diff --git a/.cargo/config.toml b/.cargo/config.toml index a590c044d8..635229119f 100644 --- a/.cargo/config.toml +++ b/.cargo/config.toml @@ -3,3 +3,6 @@ rustflags = "-C link-arg=/STACK:8000000" [target.'cfg(all(target_os = "windows", not(target_env = "msvc")))'] rustflags = "-C link-args=-Wl,--stack,8000000" + +[target.wasm32-unknown-unknown] +rustflags = ["--cfg=getrandom_backend=\"wasm_js\""] diff --git a/.cspell.dict/cpython.txt b/.cspell.dict/cpython.txt index 48059cf4e4..cb6f774ed1 100644 --- a/.cspell.dict/cpython.txt +++ b/.cspell.dict/cpython.txt @@ -1,18 +1,24 @@ argtypes asdl asname +attro augassign +badcert badsyntax basetype boolop bxor cached_tsver +cadata +cafile cellarg cellvar cellvars cmpop denom +DICTFLAG dictoffset +distpoint elts excepthandler fileutils @@ -24,6 +30,8 @@ fromlist heaptype HIGHRES IMMUTABLETYPE +Itertool +keeped kwonlyarg kwonlyargs lasti @@ -36,16 +44,23 @@ numer orelse pathconfig patma +platstdlib posonlyarg posonlyargs prec preinitialized +pythonw PYTHREAD_NAME +releasebuffer SA_ONSTACK +SOABI stackdepth +stginfo stringlib structseq subparams +swappedbytes +ticketer tok_oldval tvars unaryop @@ -53,9 +68,15 @@ unparse unparser VARKEYWORDS varkwarg +venvlauncher +venvlaunchert +venvw +venvwlauncher +venvwlaunchert wbits weakreflist +webpki withitem withs xstat -XXPRIME \ No newline at end of file +XXPRIME diff --git a/.cspell.dict/python-more.txt b/.cspell.dict/python-more.txt index 8e1c012838..d381bfe1e0 100644 --- a/.cspell.dict/python-more.txt +++ b/.cspell.dict/python-more.txt @@ -4,6 +4,7 @@ aenter aexit aiter anext +anextawaitable appendleft argcount arrayiterator @@ -147,6 +148,7 @@ nbytes ncallbacks ndigits ndim +needsfree nldecoder nlocals NOARGS @@ -167,9 +169,11 @@ pycache pycodecs pycs pyexpat +PYTHONAPI PYTHONBREAKPOINT PYTHONDEBUG PYTHONDONTWRITEBYTECODE +PYTHONFAULTHANDLER PYTHONHASHSEED PYTHONHOME PYTHONINSPECT @@ -195,6 +199,7 @@ readbuffer reconstructor refcnt releaselevel +reraised reverseitemiterator reverseiterator reversekeyiterator diff --git a/.cspell.dict/rust-more.txt b/.cspell.dict/rust-more.txt index 6f89fdfafe..ff2013e81a 100644 --- a/.cspell.dict/rust-more.txt +++ b/.cspell.dict/rust-more.txt @@ -50,6 +50,7 @@ nanos nonoverlapping objclass peekable +pemfile powc powf powi @@ -61,6 +62,7 @@ rposition rsplitn rustc rustfmt +rustls rustyline seedable seekfrom @@ -80,6 +82,7 @@ unsync wasip1 wasip2 wasmbind +wasmer wasmtime widestring winapi diff --git a/.cspell.json b/.cspell.json index 6112c35d59..89cde1ce77 100644 --- a/.cspell.json +++ b/.cspell.json @@ -60,6 +60,7 @@ "dedentations", "dedents", "deduped", + "downcastable", "downcasted", "dumpable", "emscripten", @@ -74,9 +75,9 @@ "makeunicodedata", "miri", "notrace", + "oparg", "openat", "pyarg", - "pyarg", "pyargs", "pyast", "PyAttr", @@ -106,6 +107,7 @@ "pystruct", "pystructseq", "pytrace", + "pytype", "reducelib", "richcompare", "RustPython", @@ -115,7 +117,6 @@ "sysmodule", "tracebacks", "typealiases", - "unconstructible", "unhashable", "uninit", "unraisable", @@ -123,6 +124,8 @@ "wasi", "zelf", // unix + "posixshmem", + "shm", "CLOEXEC", "codeset", "endgrent", @@ -130,6 +133,7 @@ "getrusage", "nanosleep", "sigaction", + "sighandler", "WRLCK", // win32 "birthtime", diff --git a/.gitattributes b/.gitattributes index f54bcd3b72..d1dd182a9b 100644 --- a/.gitattributes +++ b/.gitattributes @@ -5,3 +5,4 @@ vm/src/stdlib/ast/gen.rs linguist-generated -merge Lib/*.py text working-tree-encoding=UTF-8 eol=LF **/*.rs text working-tree-encoding=UTF-8 eol=LF *.pck binary +crates/rustpython_doc_db/src/*.inc.rs linguist-generated=true diff --git a/.github/copilot-instructions.md b/.github/copilot-instructions.md index e175cd5184..a2ab43a695 100644 --- a/.github/copilot-instructions.md +++ b/.github/copilot-instructions.md @@ -44,7 +44,7 @@ cargo run -- script.py cargo run # With specific features -cargo run --features ssl +cargo run --features jit # Release mode (recommended for better performance) cargo run --release -- script.py @@ -73,6 +73,14 @@ The `Lib/` directory contains Python standard library files copied from the CPyt - `unittest.skip("TODO: RustPython ")` - `unittest.expectedFailure` with `# TODO: RUSTPYTHON ` comment +### Clean Build + +When you modify bytecode instructions, a full clean is required: + +```bash +rm -r target/debug/build/rustpython-* && find . | grep -E "\.pyc$" | xargs rm -r +``` + ### Testing ```bash @@ -176,13 +184,6 @@ cargo build --target wasm32-wasip1 --no-default-features --features freeze-stdli cargo run --features jit ``` -### SSL Support - -```bash -# Enable SSL support -cargo run --features ssl -``` - ## Test Code Modification Rules **CRITICAL: Test code modification restrictions** @@ -206,7 +207,7 @@ cargo run --features ssl ## Documentation -- Check the [architecture document](architecture/architecture.md) for a high-level overview -- Read the [development guide](DEVELOPMENT.md) for detailed setup instructions +- Check the [architecture document](/architecture/architecture.md) for a high-level overview +- Read the [development guide](/DEVELOPMENT.md) for detailed setup instructions - Generate documentation with `cargo doc --no-deps --all` -- Online documentation is available at [docs.rs/rustpython](https://docs.rs/rustpython/) \ No newline at end of file +- Online documentation is available at [docs.rs/rustpython](https://docs.rs/rustpython/) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index be006de9a1..b3b7b446e4 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,13 +1,15 @@ -# Keep GitHub Actions up to date with GitHub's Dependabot... -# https://docs.github.com/en/code-security/dependabot/working-with-dependabot/keeping-your-actions-up-to-date-with-dependabot -# https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#package-ecosystem version: 2 updates: + - package-ecosystem: cargo + directory: / + schedule: + interval: weekly + ignore: + # TODO: Remove when we use ruff from crates.io + # for some reason dependabot only updates the Cargo.lock file when dealing + # with git dependencies. i.e. not updating the version in Cargo.toml + - dependency-name: "ruff_*" - package-ecosystem: github-actions directory: / - groups: - github-actions: - patterns: - - "*" # Group all Actions updates into a single larger pull request schedule: interval: weekly diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index fec9d4d838..dc5ce974e3 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -16,18 +16,20 @@ concurrency: cancel-in-progress: true env: - CARGO_ARGS: --no-default-features --features stdlib,importlib,stdio,encodings,sqlite,ssl + CARGO_ARGS: --no-default-features --features stdlib,importlib,stdio,encodings,sqlite,ssl-rustls + CARGO_ARGS_NO_SSL: --no-default-features --features stdlib,importlib,stdio,encodings,sqlite + # Crates excluded from workspace builds: + # - rustpython_wasm: requires wasm target + # - rustpython-compiler-source: deprecated + # - rustpython-venvlauncher: Windows-only + WORKSPACE_EXCLUDES: --exclude rustpython_wasm --exclude rustpython-compiler-source --exclude rustpython-venvlauncher # Skip additional tests on Windows. They are checked on Linux and MacOS. # test_glob: many failing tests - # test_io: many failing tests - # test_os: many failing tests - # test_pathlib: support.rmtree() failing + # test_pathlib: panic by surrogate chars # test_posixpath: OSError: (22, 'The filename, directory name, or volume label syntax is incorrect. (os error 123)') # test_venv: couple of failing tests WINDOWS_SKIPS: >- test_glob - test_io - test_os test_rlcompleter test_pathlib test_posixpath @@ -113,54 +115,61 @@ jobs: RUST_BACKTRACE: full name: Run rust tests runs-on: ${{ matrix.os }} + timeout-minutes: 45 strategy: matrix: os: [macos-latest, ubuntu-latest, windows-latest] fail-fast: false steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6.0.1 - uses: dtolnay/rust-toolchain@stable with: components: clippy - uses: Swatinem/rust-cache@v2 - - name: Set up the Windows environment - shell: bash - run: | - git config --system core.longpaths true - cargo install --target-dir=target -v cargo-vcpkg - cargo vcpkg -v build - if: runner.os == 'Windows' + # Only for OpenSSL builds + # - name: Set up the Windows environment + # shell: bash + # run: | + # git config --system core.longpaths true + # cargo install --target-dir=target -v cargo-vcpkg + # cargo vcpkg -v build + # if: runner.os == 'Windows' - name: Set up the Mac environment run: brew install autoconf automake libtool if: runner.os == 'macOS' - name: run clippy - run: cargo clippy ${{ env.CARGO_ARGS }} --workspace --all-targets --exclude rustpython_wasm -- -Dwarnings + run: cargo clippy ${{ env.CARGO_ARGS }} --workspace --all-targets ${{ env.WORKSPACE_EXCLUDES }} -- -Dwarnings - name: run rust tests - run: cargo test --workspace --exclude rustpython_wasm --verbose --features threading ${{ env.CARGO_ARGS }} + run: cargo test --workspace ${{ env.WORKSPACE_EXCLUDES }} --verbose --features threading ${{ env.CARGO_ARGS }} if: runner.os != 'macOS' - name: run rust tests - run: cargo test --workspace --exclude rustpython_wasm --exclude rustpython-jit --verbose --features threading ${{ env.CARGO_ARGS }} + run: cargo test --workspace ${{ env.WORKSPACE_EXCLUDES }} --exclude rustpython-jit --verbose --features threading ${{ env.CARGO_ARGS }} if: runner.os == 'macOS' - name: check compilation without threading run: cargo check ${{ env.CARGO_ARGS }} + - name: Test openssl build + run: + cargo build --no-default-features --features ssl-openssl + if: runner.os == 'Linux' + - name: Test example projects run: cargo run --manifest-path example_projects/barebone/Cargo.toml cargo run --manifest-path example_projects/frozen_stdlib/Cargo.toml if: runner.os == 'Linux' - - name: prepare AppleSilicon build + - name: prepare Intel MacOS build uses: dtolnay/rust-toolchain@stable with: - target: aarch64-apple-darwin + target: x86_64-apple-darwin if: runner.os == 'macOS' - - name: Check compilation for Apple Silicon - run: cargo check --target aarch64-apple-darwin + - name: Check compilation for Intel MacOS + run: cargo check --target x86_64-apple-darwin if: runner.os == 'macOS' - name: prepare iOS build uses: dtolnay/rust-toolchain@stable @@ -168,15 +177,16 @@ jobs: target: aarch64-apple-ios if: runner.os == 'macOS' - name: Check compilation for iOS - run: cargo check --target aarch64-apple-ios + run: cargo check --target aarch64-apple-ios ${{ env.CARGO_ARGS_NO_SSL }} if: runner.os == 'macOS' exotic_targets: if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }} name: Ensure compilation on various targets runs-on: ubuntu-latest + timeout-minutes: 30 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6.0.1 - uses: dtolnay/rust-toolchain@stable with: target: i686-unknown-linux-gnu @@ -184,14 +194,14 @@ jobs: - name: Install gcc-multilib and musl-tools run: sudo apt-get update && sudo apt-get install gcc-multilib musl-tools - name: Check compilation for x86 32bit - run: cargo check --target i686-unknown-linux-gnu + run: cargo check --target i686-unknown-linux-gnu ${{ env.CARGO_ARGS_NO_SSL }} - uses: dtolnay/rust-toolchain@stable with: target: aarch64-linux-android - name: Check compilation for android - run: cargo check --target aarch64-linux-android + run: cargo check --target aarch64-linux-android ${{ env.CARGO_ARGS_NO_SSL }} - uses: dtolnay/rust-toolchain@stable with: @@ -200,28 +210,28 @@ jobs: - name: Install gcc-aarch64-linux-gnu run: sudo apt install gcc-aarch64-linux-gnu - name: Check compilation for aarch64 linux gnu - run: cargo check --target aarch64-unknown-linux-gnu + run: cargo check --target aarch64-unknown-linux-gnu ${{ env.CARGO_ARGS_NO_SSL }} - uses: dtolnay/rust-toolchain@stable with: target: i686-unknown-linux-musl - name: Check compilation for musl - run: cargo check --target i686-unknown-linux-musl + run: cargo check --target i686-unknown-linux-musl ${{ env.CARGO_ARGS_NO_SSL }} - uses: dtolnay/rust-toolchain@stable with: target: x86_64-unknown-freebsd - name: Check compilation for freebsd - run: cargo check --target x86_64-unknown-freebsd + run: cargo check --target x86_64-unknown-freebsd ${{ env.CARGO_ARGS_NO_SSL }} - uses: dtolnay/rust-toolchain@stable with: target: x86_64-unknown-freebsd - name: Check compilation for freeBSD - run: cargo check --target x86_64-unknown-freebsd + run: cargo check --target x86_64-unknown-freebsd ${{ env.CARGO_ARGS_NO_SSL }} # - name: Prepare repository for redox compilation # run: bash scripts/redox/uncomment-cargo.sh @@ -242,19 +252,20 @@ jobs: os: [macos-latest, ubuntu-latest, windows-latest] fail-fast: false steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6.0.1 - uses: dtolnay/rust-toolchain@stable - uses: Swatinem/rust-cache@v2 - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v6.1.0 with: python-version: ${{ env.PYTHON_VERSION }} - - name: Set up the Windows environment - shell: bash - run: | - git config --system core.longpaths true - cargo install cargo-vcpkg - cargo vcpkg build - if: runner.os == 'Windows' + # Only for OpenSSL builds + # - name: Set up the Windows environment + # shell: bash + # run: | + # git config --system core.longpaths true + # cargo install cargo-vcpkg + # cargo vcpkg build + # if: runner.os == 'Windows' - name: Set up the Mac environment run: brew install autoconf automake libtool openssl@3 if: runner.os == 'macOS' @@ -264,34 +275,42 @@ jobs: - name: build rustpython run: cargo build --release --verbose --features=threading ${{ env.CARGO_ARGS }},jit if: runner.os != 'macOS' - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v6.1.0 with: python-version: ${{ env.PYTHON_VERSION }} - name: run snippets run: python -m pip install -r requirements.txt && pytest -v working-directory: ./extra_tests + - if: runner.os == 'Linux' name: run cpython platform-independent tests run: target/release/rustpython -m test -j 1 -u all --slowest --fail-env-changed -v ${{ env.PLATFORM_INDEPENDENT_TESTS }} + timeout-minutes: 35 + - if: runner.os == 'Linux' name: run cpython platform-dependent tests (Linux) run: target/release/rustpython -m test -j 1 -u all --slowest --fail-env-changed -v -x ${{ env.PLATFORM_INDEPENDENT_TESTS }} + timeout-minutes: 35 + - if: runner.os == 'macOS' name: run cpython platform-dependent tests (MacOS) run: target/release/rustpython -m test -j 1 --slowest --fail-env-changed -v -x ${{ env.PLATFORM_INDEPENDENT_TESTS }} + timeout-minutes: 35 + - if: runner.os == 'Windows' name: run cpython platform-dependent tests (windows partial - fixme) run: target/release/rustpython -m test -j 1 --slowest --fail-env-changed -v -x ${{ env.PLATFORM_INDEPENDENT_TESTS }} ${{ env.WINDOWS_SKIPS }} + timeout-minutes: 45 + - if: runner.os != 'Windows' name: check that --install-pip succeeds run: | mkdir site-packages target/release/rustpython --install-pip ensurepip --user target/release/rustpython -m pip install six - - if: runner.os != 'Windows' - name: Check that ensurepip succeeds. + - name: Check that ensurepip succeeds. run: | target/release/rustpython -m ensurepip target/release/rustpython -c "import pip" @@ -304,30 +323,32 @@ jobs: run: python -I whats_left.py lint: - name: Check Rust code with rustfmt and clippy + name: Check Rust code with clippy runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6.0.1 - uses: dtolnay/rust-toolchain@stable with: - components: rustfmt, clippy - - name: run rustfmt - run: cargo fmt --check + components: clippy - name: run clippy on wasm - run: cargo clippy --manifest-path=wasm/lib/Cargo.toml -- -Dwarnings - - uses: actions/setup-python@v5 - with: - python-version: ${{ env.PYTHON_VERSION }} - - name: install ruff - run: python -m pip install ruff==0.11.8 + run: cargo clippy --manifest-path=crates/wasm/Cargo.toml -- -Dwarnings + - name: Ensure docs generate no warnings run: cargo doc - - name: run ruff check - run: ruff check --diff - - name: run ruff format - run: ruff format --check + + - name: Install ruff + uses: astral-sh/ruff-action@57714a7c8a2e59f32539362ba31877a1957dded1 # v3.5.1 + with: + version: "0.14.9" + args: "--version" + + - run: ruff check --diff + + - run: ruff format --check + - name: install prettier run: yarn global add prettier && echo "$(yarn global bin)" >>$GITHUB_PATH + - name: check wasm code with prettier # prettier doesn't handle ignore files very well: https://github.com/prettier/prettier/issues/8506 run: cd wasm && git ls-files -z | xargs -0 prettier --check -u @@ -335,7 +356,7 @@ jobs: - name: install extra dictionaries run: npm install @cspell/dict-en_us @cspell/dict-cpp @cspell/dict-python @cspell/dict-rust @cspell/dict-win32 @cspell/dict-shell - name: spell checker - uses: streetsidesoftware/cspell-action@v7 + uses: streetsidesoftware/cspell-action@v8 with: files: '**/*.rs' incremental_files_only: true @@ -344,25 +365,33 @@ jobs: if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }} name: Run tests under miri runs-on: ubuntu-latest + timeout-minutes: 30 + env: + NIGHTLY_CHANNEL: nightly steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6.0.1 + - uses: dtolnay/rust-toolchain@master with: - toolchain: nightly + toolchain: ${{ env.NIGHTLY_CHANNEL }} components: miri - uses: Swatinem/rust-cache@v2 + - name: Run tests under miri + run: cargo +${{ env.NIGHTLY_CHANNEL }} miri test -p rustpython-vm -- miri_test + env: # miri-ignore-leaks because the type-object circular reference means that there will always be # a memory leak, at least until we have proper cyclic gc - run: MIRIFLAGS='-Zmiri-ignore-leaks' cargo +nightly miri test -p rustpython-vm -- miri_test + MIRIFLAGS: '-Zmiri-ignore-leaks' wasm: if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }} name: Check the WASM package and demo runs-on: ubuntu-latest + timeout-minutes: 30 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6.0.1 - uses: dtolnay/rust-toolchain@stable - uses: Swatinem/rust-cache@v2 @@ -373,12 +402,12 @@ jobs: wget https://github.com/mozilla/geckodriver/releases/download/v0.36.0/geckodriver-v0.36.0-linux64.tar.gz mkdir geckodriver tar -xzf geckodriver-v0.36.0-linux64.tar.gz -C geckodriver - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v6.1.0 with: python-version: ${{ env.PYTHON_VERSION }} - run: python -m pip install -r requirements.txt working-directory: ./wasm/tests - - uses: actions/setup-node@v4 + - uses: actions/setup-node@v6 with: cache: "npm" cache-dependency-path: "wasm/demo/package-lock.json" @@ -394,11 +423,13 @@ jobs: with: { wabt-version: "1.0.36" } - name: check wasm32-unknown without js run: | - cd wasm/wasm-unknown-test - cargo build --release --verbose - if wasm-objdump -xj Import target/wasm32-unknown-unknown/release/wasm_unknown_test.wasm; then - echo "ERROR: wasm32-unknown module expects imports from the host environment" >2 + cd example_projects/wasm32_without_js/rustpython-without-js + cargo build + cd .. + if wasm-objdump -xj Import rustpython-without-js/target/wasm32-unknown-unknown/debug/rustpython_without_js.wasm; then + echo "ERROR: wasm32-unknown module expects imports from the host environment" >&2 fi + cargo run --release --manifest-path wasm-runtime/Cargo.toml rustpython-without-js/target/wasm32-unknown-unknown/debug/rustpython_without_js.wasm - name: build notebook demo if: github.ref == 'refs/heads/release' run: | @@ -421,8 +452,9 @@ jobs: if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }} name: Run snippets and cpython tests on wasm-wasi runs-on: ubuntu-latest + timeout-minutes: 30 steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6.0.1 - uses: dtolnay/rust-toolchain@stable with: target: wasm32-wasip1 diff --git a/.github/workflows/comment-commands.yml b/.github/workflows/comment-commands.yml new file mode 100644 index 0000000000..d1a457c73e --- /dev/null +++ b/.github/workflows/comment-commands.yml @@ -0,0 +1,21 @@ +name: Comment Commands + +on: + issue_comment: + types: created + +jobs: + issue_assign: + if: (!github.event.issue.pull_request) && github.event.comment.body == 'take' + runs-on: ubuntu-latest + + concurrency: + group: ${{ github.actor }}-issue-assign + + permissions: + issues: write + + steps: + # Using REST API and not `gh issue edit`. https://github.com/cli/cli/issues/6235#issuecomment-1243487651 + - run: | + curl -H "Authorization: token ${{ github.token }}" -d '{"assignees": ["${{ github.event.comment.user.login }}"]}' https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.issue.number }}/assignees diff --git a/.github/workflows/cron-ci.yaml b/.github/workflows/cron-ci.yaml index 6389fee1cb..59d664e0ea 100644 --- a/.github/workflows/cron-ci.yaml +++ b/.github/workflows/cron-ci.yaml @@ -1,6 +1,6 @@ on: schedule: - - cron: '0 0 * * 6' + - cron: "0 0 * * 6" workflow_dispatch: push: paths: @@ -18,11 +18,13 @@ jobs: codecov: name: Collect code coverage data runs-on: ubuntu-latest + # Disable this scheduled job when running on a fork. + if: ${{ github.repository == 'RustPython/RustPython' || github.event_name != 'schedule' }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6.0.1 - uses: dtolnay/rust-toolchain@stable - uses: taiki-e/install-action@cargo-llvm-cov - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v6.1.0 with: python-version: ${{ env.PYTHON_VERSION }} - run: sudo apt-get update && sudo apt-get -y install lcov @@ -32,7 +34,7 @@ jobs: run: python scripts/cargo-llvm-cov.py continue-on-error: true - name: Run cargo-llvm-cov with Python test suite. - run: cargo llvm-cov --no-report run -- -m test -u all --slowest --fail-env-changed + run: cargo llvm-cov --no-report run -- -m test -u all --slowest --fail-env-changed continue-on-error: true - name: Prepare code coverage data run: cargo llvm-cov report --lcov --output-path='codecov.lcov' @@ -44,8 +46,10 @@ jobs: testdata: name: Collect regression test data runs-on: ubuntu-latest + # Disable this scheduled job when running on a fork. + if: ${{ github.repository == 'RustPython/RustPython' || github.event_name != 'schedule' }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6.0.1 - uses: dtolnay/rust-toolchain@stable - name: build rustpython run: cargo build --release --verbose @@ -73,10 +77,12 @@ jobs: whatsleft: name: Collect what is left data runs-on: ubuntu-latest + # Disable this scheduled job when running on a fork. + if: ${{ github.repository == 'RustPython/RustPython' || github.event_name != 'schedule' }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6.0.1 - uses: dtolnay/rust-toolchain@stable - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v6.1.0 with: python-version: ${{ env.PYTHON_VERSION }} - name: build rustpython @@ -103,6 +109,23 @@ jobs: rm ./_data/whats_left/modules.csv echo -e "module" > ./_data/whats_left/modules.csv cat ./_data/whats_left.temp | grep "(entire module)" | cut -d ' ' -f 1 | sort >> ./_data/whats_left/modules.csv + awk -f - ./_data/whats_left.temp > ./_data/whats_left/builtin_items.csv <<'EOF' + BEGIN { + OFS="," + print "builtin,name,is_inherited" + } + /^# builtin items/ { in_section=1; next } + /^$/ { if (in_section) exit } + in_section { + split($1, a, ".") + rest = "" + idx = index($0, " ") + if (idx > 0) { + rest = substr($0, idx+1) + } + print a[1], $1, rest + } + EOF git add -A if git -c user.name="Github Actions" -c user.email="actions@github.com" commit -m "Update what is left results" --author="$GITHUB_ACTOR"; then git push @@ -111,10 +134,12 @@ jobs: benchmark: name: Collect benchmark data runs-on: ubuntu-latest + # Disable this scheduled job when running on a fork. + if: ${{ github.repository == 'RustPython/RustPython' || github.event_name != 'schedule' }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6.0.1 - uses: dtolnay/rust-toolchain@stable - - uses: actions/setup-python@v5 + - uses: actions/setup-python@v6.1.0 with: python-version: 3.9 - run: cargo install cargo-criterion diff --git a/.github/workflows/pr-auto-commit.yaml b/.github/workflows/pr-auto-commit.yaml new file mode 100644 index 0000000000..0cbd2bfefb --- /dev/null +++ b/.github/workflows/pr-auto-commit.yaml @@ -0,0 +1,84 @@ +name: Auto-format PR + +# This workflow triggers when a PR is opened/updated +on: + pull_request_target: + types: [opened, synchronize, reopened] + branches: + - main + - release + +concurrency: + group: auto-format-${{ github.event.pull_request.number }} + cancel-in-progress: true + +jobs: + auto_format: + if: ${{ !contains(github.event.pull_request.labels.*.name, 'skip:ci') }} + permissions: + contents: write + pull-requests: write + runs-on: ubuntu-latest + timeout-minutes: 60 + steps: + - name: Checkout PR branch + uses: actions/checkout@v6.0.1 + with: + ref: ${{ github.event.pull_request.head.sha }} + repository: ${{ github.event.pull_request.head.repo.full_name }} + token: ${{ secrets.AUTO_COMMIT_PAT }} + fetch-depth: 0 + + - name: Setup Rust + uses: dtolnay/rust-toolchain@stable + with: + components: rustfmt + + - name: Run cargo fmt + run: | + echo "Running cargo fmt --all on PR #${{ github.event.pull_request.number }}" + cargo fmt --all + + - name: Install ruff + uses: astral-sh/ruff-action@57714a7c8a2e59f32539362ba31877a1957dded1 # v3.5.1 + with: + version: "0.14.9" + args: "--version" + + - run: ruff format + - run: ruff check --select I --fix + + - name: Configure git + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + + - name: Check for changes + id: check-changes + run: | + if [ -n "$(git status --porcelain)" ]; then + echo "has_changes=true" >> $GITHUB_OUTPUT + else + echo "has_changes=false" >> $GITHUB_OUTPUT + fi + + - name: Commit and push formatting changes + if: steps.check-changes.outputs.has_changes == 'true' + run: | + git add -u + git commit -m "Auto-format: cargo fmt --all" + git push origin HEAD:${{ github.event.pull_request.head.ref }} + + - name: Comment on PR + if: steps.check-changes.outputs.has_changes == 'true' + uses: marocchino/sticky-pull-request-comment@v2 + with: + number: ${{ github.event.pull_request.number }} + message: | + **Code has been automatically formatted** + + The code in this PR has been formatted using `cargo fmt --all`. + Please pull the latest changes before pushing again: + ```bash + git pull origin ${{ github.event.pull_request.head.ref }} + ``` diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index f6a1ad3209..5ad2cf3f97 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -21,6 +21,8 @@ env: jobs: build: runs-on: ${{ matrix.platform.runner }} + # Disable this scheduled job when running on a fork. + if: ${{ github.repository == 'RustPython/RustPython' || github.event_name != 'schedule' }} strategy: matrix: platform: @@ -48,7 +50,7 @@ jobs: # target: aarch64-pc-windows-msvc fail-fast: false steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6.0.1 - uses: dtolnay/rust-toolchain@stable - uses: cargo-bins/cargo-binstall@main @@ -81,15 +83,17 @@ jobs: if: runner.os == 'Windows' - name: Upload Binary Artifacts - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v6.0.0 with: name: rustpython-release-${{ runner.os }}-${{ matrix.platform.target }} path: target/rustpython-release-${{ runner.os }}-${{ matrix.platform.target }}* build-wasm: runs-on: ubuntu-latest + # Disable this scheduled job when running on a fork. + if: ${{ github.repository == 'RustPython/RustPython' || github.event_name != 'schedule' }} steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v6.0.1 - uses: dtolnay/rust-toolchain@stable with: targets: wasm32-wasip1 @@ -101,14 +105,14 @@ jobs: run: cp target/wasm32-wasip1/release/rustpython.wasm target/rustpython-release-wasm32-wasip1.wasm - name: Upload Binary Artifacts - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v6.0.0 with: name: rustpython-release-wasm32-wasip1 path: target/rustpython-release-wasm32-wasip1.wasm - name: install wasm-pack run: curl https://rustwasm.github.io/wasm-pack/installer/init.sh -sSf | sh - - uses: actions/setup-node@v4 + - uses: actions/setup-node@v6 - uses: mwilliamson/setup-wabt-action@v3 with: { wabt-version: "1.0.30" } - name: build demo @@ -136,15 +140,23 @@ jobs: release: runs-on: ubuntu-latest + # Disable this scheduled job when running on a fork. + if: ${{ github.repository == 'RustPython/RustPython' || github.event_name != 'schedule' }} needs: [build, build-wasm] steps: + - uses: actions/checkout@v6.0.1 + - name: Download Binary Artifacts - uses: actions/download-artifact@v4 + uses: actions/download-artifact@v7.0.0 with: path: bin pattern: rustpython-* merge-multiple: true + - name: Create Lib Archive + run: | + zip -r bin/rustpython-lib.zip Lib/ + - name: List Binaries run: | ls -lah bin/ @@ -168,6 +180,7 @@ jobs: --repo="$GITHUB_REPOSITORY" \ --title="RustPython $RELEASE_TYPE_NAME $today-$tag #$run" \ --target="$tag" \ + --notes "⚠️ **Important**: To run RustPython, you must download both the binary for your platform AND the \`rustpython-lib.zip\` archive. Extract the Lib directory from the archive to the same location as the binary, or set the \`RUSTPYTHONPATH\` environment variable to point to the Lib directory." \ --generate-notes \ $PRERELEASE_ARG \ bin/rustpython-release-* diff --git a/.github/workflows/update-doc-db.yml b/.github/workflows/update-doc-db.yml new file mode 100644 index 0000000000..c580e7d0ea --- /dev/null +++ b/.github/workflows/update-doc-db.yml @@ -0,0 +1,108 @@ +name: Update doc DB + +permissions: + contents: write + +on: + workflow_dispatch: + inputs: + python-version: + description: Target python version to generate doc db for + type: string + default: "3.13.9" + ref: + description: Branch to commit to (leave empty for current branch) + type: string + default: "" + +defaults: + run: + shell: bash + +jobs: + generate: + runs-on: ${{ matrix.os }} + strategy: + matrix: + os: + - ubuntu-latest + - windows-latest + - macos-latest + steps: + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + with: + persist-credentials: false + sparse-checkout: | + crates/doc + + - uses: actions/setup-python@83679a892e2d95755f2dac6acb0bfd1e9ac5d548 # v6.1.0 + with: + python-version: ${{ inputs.python-version }} + + - name: Generate docs + run: python crates/doc/generate.py + + - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: doc-db-${{ inputs.python-version }}-${{ matrix.os }} + path: "crates/doc/generated/*.json" + if-no-files-found: error + retention-days: 7 + overwrite: true + + merge: + runs-on: ubuntu-latest + needs: generate + steps: + - uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 + with: + ref: ${{ inputs.ref || github.ref }} + token: ${{ secrets.AUTO_COMMIT_PAT }} + + - name: Download generated doc DBs + uses: actions/download-artifact@37930b1c2abaa49bbe596cd826c3c89aef350131 # v7.0.0 + with: + pattern: "doc-db-${{ inputs.python-version }}-**" + path: crates/doc/generated/ + merge-multiple: true + + - name: Transform JSON + run: | + # Merge all artifacts + jq -s "add" --sort-keys crates/doc/generated/*.json > crates/doc/generated/merged.json + + # Format merged json for the phf macro + jq -r 'to_entries[] | " \(.key | @json) => \(.value | @json),"' crates/doc/generated/merged.json > crates/doc/generated/raw_entries.txt + + OUTPUT_FILE='crates/doc/src/data.inc.rs' + + echo -n '' > $OUTPUT_FILE + + echo '// This file was auto-generated by `.github/workflows/update-doc-db.yml`.' >> $OUTPUT_FILE + echo "// CPython version: ${{ inputs.python-version }}" >> $OUTPUT_FILE + echo '// spell-checker: disable' >> $OUTPUT_FILE + + echo '' >> $OUTPUT_FILE + + echo "pub static DB: phf::Map<&'static str, &'static str> = phf::phf_map! {" >> $OUTPUT_FILE + cat crates/doc/generated/raw_entries.txt >> $OUTPUT_FILE + echo '};' >> $OUTPUT_FILE + + - uses: actions/upload-artifact@b7c566a772e6b6bfb58ed0dc250532a479d7789f # v6.0.0 + with: + name: doc-db-${{ inputs.python-version }} + path: "crates/doc/src/data.inc.rs" + if-no-files-found: error + retention-days: 7 + overwrite: true + + - name: Commit and push (non-main branches only) + if: github.ref != 'refs/heads/main' && inputs.ref != 'main' + run: | + git config user.name "github-actions[bot]" + git config user.email "github-actions[bot]@users.noreply.github.com" + if [ -n "$(git status --porcelain)" ]; then + git add crates/doc/src/data.inc.rs + git commit -m "Update doc DB for CPython ${{ inputs.python-version }}" + git push + fi diff --git a/.gitignore b/.gitignore index cb7165aaca..fea93ace80 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,8 @@ flamescope.json extra_tests/snippets/resources extra_tests/not_impl.py + +Lib/site-packages/* +!Lib/site-packages/README.txt +Lib/test/data/* +!Lib/test/data/README diff --git a/Cargo.lock b/Cargo.lock index c6dee806c7..ffcc1e37c1 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -4,9 +4,9 @@ version = 4 [[package]] name = "adler2" -version = "2.0.0" +version = "2.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "512761e0bb2578dd7380c6baaa0f4ce03e84f95e960231d1dec8bf4d7d6e2627" +checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa" [[package]] name = "adler32" @@ -14,6 +14,17 @@ version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aae1277d39aeec15cb388266ecc24b11c80469deae6067e17a1a7aa9e5c1f234" +[[package]] +name = "aes" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b169f7a6d4742236a0a00c541b845991d0ac43e546831af1249753ab4c3aa3a0" +dependencies = [ + "cfg-if", + "cipher", + "cpufeatures", +] + [[package]] name = "ahash" version = "0.8.12" @@ -21,7 +32,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75" dependencies = [ "cfg-if", - "getrandom 0.3.2", + "getrandom 0.3.4", "once_cell", "version_check", "zerocopy", @@ -29,24 +40,27 @@ dependencies = [ [[package]] name = "aho-corasick" -version = "1.1.3" +version = "1.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +checksum = "ddd31a130427c27518df266943a5308ed92d4b226cc639f5a8f1002816174301" dependencies = [ "memchr", ] [[package]] -name = "allocator-api2" -version = "0.2.21" +name = "alloca" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" +checksum = "e5a7d05ea6aea7e9e64d25b9156ba2fee3fdd659e34e41063cd2fc7cd020d7f4" +dependencies = [ + "cc", +] [[package]] -name = "android-tzdata" -version = "0.1.1" +name = "allocator-api2" +version = "0.2.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e999941b234f3131b00bc13c22d06e8c5ff726d1b6318ac7eb276997bbb4fef0" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" [[package]] name = "android_system_properties" @@ -65,9 +79,9 @@ checksum = "4b46cbb362ab8752921c97e041f5e366ee6297bd428a31275b9fcf1e380f7299" [[package]] name = "anstream" -version = "0.6.18" +version = "0.6.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8acc5369981196006228e28809f761875c0327210a891e941f4c683b3a99529b" +checksum = "43d5b281e737544384e969a5ccad3f1cdd24b48086a0fc1b2a5262a26b8f4f4a" dependencies = [ "anstyle", "anstyle-parse", @@ -80,44 +94,44 @@ dependencies = [ [[package]] name = "anstyle" -version = "1.0.10" +version = "1.0.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "55cc3b69f167a1ef2e161439aa98aed94e6028e5f9a59be9a6ffb47aef1651f9" +checksum = "5192cca8006f1fd4f7237516f40fa183bb07f8fbdfedaa0036de5ea9b0b45e78" [[package]] name = "anstyle-parse" -version = "0.2.6" +version = "0.2.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b2d16507662817a6a20a9ea92df6652ee4f94f914589377d69f3b21bc5798a9" +checksum = "4e7644824f0aa2c7b9384579234ef10eb7efb6a0deb83f9630a49594dd9c15c2" dependencies = [ "utf8parse", ] [[package]] name = "anstyle-query" -version = "1.1.2" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79947af37f4177cfead1110013d678905c37501914fba0efea834c3fe9a8d60c" +checksum = "40c48f72fd53cd289104fc64099abca73db4166ad86ea0b4341abe65af83dadc" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] name = "anstyle-wincon" -version = "3.0.7" +version = "3.0.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ca3534e77181a9cc07539ad51f2141fe32f6c3ffd4df76db8ad92346b003ae4e" +checksum = "291e6a250ff86cd4a820112fb8898808a366d8f9f58ce16d1f538353ad55747d" dependencies = [ "anstyle", - "once_cell", - "windows-sys 0.59.0", + "once_cell_polyfill", + "windows-sys 0.61.2", ] [[package]] name = "anyhow" -version = "1.0.98" +version = "1.0.100" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e16d2d3311acee920a9eb8d33b8cbc1787ce4a264e85f964c2404b969bdcd487" +checksum = "a23eb6b1614318a8071c9b2521f36b424b2c83db5eb3a0fead4a6c0809af6e61" [[package]] name = "approx" @@ -130,9 +144,9 @@ dependencies = [ [[package]] name = "arbitrary" -version = "1.4.1" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dde20b3d026af13f561bdd0f15edf01fc734f0dafcedbaf42bba506a9517f223" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" [[package]] name = "ascii" @@ -141,19 +155,126 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d92bec98840b8f03a5ff5413de5293bfcd8bf96467cf5452609f939ec6f5de16" [[package]] -name = "atomic" +name = "asn1-rs" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "56624a96882bb8c26d61312ae18cb45868e5a9992ea73c58e45c3101e56a1e60" +dependencies = [ + "asn1-rs-derive", + "asn1-rs-impl", + "displaydoc", + "nom", + "num-traits", + "rusticata-macros", + "thiserror 2.0.17", + "time", +] + +[[package]] +name = "asn1-rs-derive" version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8d818003e740b63afc82337e3160717f4f63078720a810b7b903e70a5d1d2994" +checksum = "3109e49b1e4909e9db6515a30c633684d68cdeaa252f215214cb4fa1a5bfee2c" +dependencies = [ + "proc-macro2", + "quote", + "syn", + "synstructure", +] + +[[package]] +name = "asn1-rs-impl" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7b18050c2cd6fe86c3a76584ef5e0baf286d038cda203eb6223df2cc413565f7" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "atomic" +version = "0.6.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89cbf775b137e9b968e67227ef7f775587cde3fd31b0d8599dbd0f598a48340" dependencies = [ "bytemuck", ] +[[package]] +name = "attribute-derive" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "05832cdddc8f2650cc2cc187cc2e952b8c133a48eb055f35211f61ee81502d77" +dependencies = [ + "attribute-derive-macro", + "derive-where", + "manyhow", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "attribute-derive-macro" +version = "0.10.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a7cdbbd4bd005c5d3e2e9c885e6fa575db4f4a3572335b974d8db853b6beb61" +dependencies = [ + "collection_literals", + "interpolator", + "manyhow", + "proc-macro-utils", + "proc-macro2", + "quote", + "quote-use", + "syn", +] + [[package]] name = "autocfg" -version = "1.4.0" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c08606f8c3cbf4ce6ec8e28fb0014a2c086708fe954eaa885384a6165172e7e8" + +[[package]] +name = "aws-lc-fips-sys" +version = "0.13.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57900537c00a0565a35b63c4c281b372edfc9744b072fd4a3b414350a8f5ed48" +dependencies = [ + "bindgen 0.72.1", + "cc", + "cmake", + "dunce", + "fs_extra", + "regex", +] + +[[package]] +name = "aws-lc-rs" +version = "1.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" +checksum = "6a88aab2464f1f25453baa7a07c84c5b7684e274054ba06817f382357f77a288" +dependencies = [ + "aws-lc-fips-sys", + "aws-lc-sys", + "untrusted 0.7.1", + "zeroize", +] + +[[package]] +name = "aws-lc-sys" +version = "0.35.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b45afffdee1e7c9126814751f88dddc747f41d91da16c9551a0f1e8a11e788a1" +dependencies = [ + "cc", + "cmake", + "dunce", + "fs_extra", +] [[package]] name = "base64" @@ -161,13 +282,39 @@ version = "0.22.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "72b3254f16251a8381aa12e40e3c4d2f0199f8c6508fbecb9d91f575e0fbb8c6" +[[package]] +name = "base64ct" +version = "1.8.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "55248b47b0caf0546f7988906588779981c43bb1bc9d0c44087278f80cdb44ba" + [[package]] name = "bindgen" version = "0.71.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5f58bf3d7db68cfbac37cfc485a8d711e87e064c3d0fe0435b92f7a407f9d6b3" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.10.0", + "cexpr", + "clang-sys", + "itertools 0.13.0", + "log", + "prettyplease", + "proc-macro2", + "quote", + "regex", + "rustc-hash", + "shlex", + "syn", +] + +[[package]] +name = "bindgen" +version = "0.72.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "993776b509cfb49c750f11b8f07a46fa23e0a1386ffc01fb1e7d343efc387895" +dependencies = [ + "bitflags 2.10.0", "cexpr", "clang-sys", "itertools 0.13.0", @@ -178,7 +325,7 @@ dependencies = [ "regex", "rustc-hash", "shlex", - "syn 2.0.101", + "syn", ] [[package]] @@ -189,9 +336,9 @@ checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a" [[package]] name = "bitflags" -version = "2.9.0" +version = "2.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5c8214115b7bf84099f1309324e63141d4c5d7cc26862f97a0a857dbefe165bd" +checksum = "812e12b5285cc515a9c72a5c1d3b6d46a19dac5acfef5265968c166106e31dd3" [[package]] name = "blake2" @@ -211,11 +358,20 @@ dependencies = [ "generic-array", ] +[[package]] +name = "block-padding" +version = "0.3.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a8894febbff9f758034a5b8e12d87918f56dfc64a8e1fe757d65e29041538d93" +dependencies = [ + "generic-array", +] + [[package]] name = "bstr" -version = "1.12.0" +version = "1.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "234113d19d0d7d613b40e86fb654acf958910802bcceab913a4f9e7cda03b1a4" +checksum = "63044e1ae8e69f3b5a92c736ca6269b8d12fa7efe39bf34ddb06d102cf0e2cab" dependencies = [ "memchr", "regex-automata", @@ -224,37 +380,32 @@ dependencies = [ [[package]] name = "bumpalo" -version = "3.17.0" +version = "3.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1628fb46dfa0b37568d12e5edd512553eccf6a22a78e8bde00bb4aed84d5bdbf" +checksum = "46c5e41b57b8bba42a04676d81cb89e9ee8e859a1a66f80a5a72e1cb76b34d43" dependencies = [ "allocator-api2", ] [[package]] name = "bytemuck" -version = "1.23.0" +version = "1.24.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9134a6ef01ce4b366b50689c94f82c14bc72bc5d0386829828a2e2752ef7958c" +checksum = "1fbdf580320f38b612e485521afda1ee26d10cc9884efaaa750d383e13e3c5f4" [[package]] -name = "bzip2" -version = "0.5.2" +name = "bytes" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "49ecfb22d906f800d4fe833b6282cf4dc1c298f5057ca0b5445e5c209735ca47" -dependencies = [ - "bzip2-sys", - "libbz2-rs-sys", -] +checksum = "b35204fbdc0b3f4446b89fc1ac2cf84a8a68971995d0bf2e925ec7cd960f9cb3" [[package]] -name = "bzip2-sys" -version = "0.1.13+1.0.8" +name = "bzip2" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "225bff33b2141874fe80d71e07d6eec4f85c5c216453dd96388240f96e1acc14" +checksum = "f3a53fac24f34a81bc9954b5d6cfce0c21e18ec6959f44f56e8e90e4bb7c346c" dependencies = [ - "cc", - "pkg-config", + "libbz2-rs-sys", ] [[package]] @@ -274,22 +425,40 @@ checksum = "37b2a672a2cb129a2e41c10b1224bb368f9f37a2b16b612598138befd7b37eb5" [[package]] name = "castaway" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0abae9be0aaf9ea96a3b1b8b1b55c602ca751eba1b1500220cea4ecbafe7c0d5" +checksum = "dec551ab6e7578819132c713a93c022a05d60159dc86e7a7050223577484c55a" dependencies = [ "rustversion", ] +[[package]] +name = "cbc" +version = "0.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "26b52a9543ae338f279b96b0b9fed9c8093744685043739079ce85cd58f289a6" +dependencies = [ + "cipher", +] + [[package]] name = "cc" -version = "1.2.21" +version = "1.2.47" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8691782945451c1c383942c4874dbe63814f61cb57ef773cda2972682b7bb3c0" +checksum = "cd405d82c84ff7f35739f175f67d8b9fb7687a0e84ccdc78bd3568839827cf07" dependencies = [ + "find-msvc-tools", + "jobserver", + "libc", "shlex", ] +[[package]] +name = "cesu8" +version = "1.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d43a04d8753f35258c91f8ec639f792891f748a1edbd759cf1dcea3382ad83c" + [[package]] name = "cexpr" version = "0.6.0" @@ -301,9 +470,9 @@ dependencies = [ [[package]] name = "cfg-if" -version = "1.0.0" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +checksum = "9330f8b2ff13f34540b44e946ef35111825727b38d33286ef986142615121801" [[package]] name = "cfg_aliases" @@ -313,11 +482,10 @@ checksum = "613afe47fcd5fac7ccf1db93babcb082c5994d996f20b8b159f2ad1658eb5724" [[package]] name = "chrono" -version = "0.4.41" +version = "0.4.42" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c469d952047f47f91b68d1cba3f10d63c11d73e4636f24f08daf0278abf01c4d" +checksum = "145052bdd345b87320e369255277e3fb5152762ad123a901ef5c262dd38fe8d2" dependencies = [ - "android-tzdata", "iana-time-zone", "js-sys", "num-traits", @@ -352,6 +520,16 @@ dependencies = [ "half", ] +[[package]] +name = "cipher" +version = "0.4.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "773f3b9af64447d2ce9850330c473515014aa235e6a783b02db81ff39e4a3dad" +dependencies = [ + "crypto-common", + "inout", +] + [[package]] name = "clang-sys" version = "1.8.1" @@ -360,23 +538,23 @@ checksum = "0b023947811758c97c59bf9d1c188fd619ad4718dcaa767947df1cadb14f39f4" dependencies = [ "glob", "libc", - "libloading", + "libloading 0.8.9", ] [[package]] name = "clap" -version = "4.5.38" +version = "4.5.53" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed93b9805f8ba930df42c2590f05453d5ec36cbb85d018868a5b24d31f6ac000" +checksum = "c9e340e012a1bf4935f5282ed1436d1489548e8f72308207ea5df0e23d2d03f8" dependencies = [ "clap_builder", ] [[package]] name = "clap_builder" -version = "4.5.38" +version = "4.5.53" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "379026ff283facf611b0ea629334361c4211d1b12ee01024eec1591133b04120" +checksum = "d76b5d13eaa18c901fd2f7fca939fefe3a0727a953561fefdf3b2922b8569d00" dependencies = [ "anstyle", "clap_lex", @@ -384,30 +562,55 @@ dependencies = [ [[package]] name = "clap_lex" -version = "0.7.4" +version = "0.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f46ad14479a25103f283c0f10005961cf086d8dc42205bb44c46ac563475dca6" +checksum = "a1d728cc89cf3aee9ff92b05e62b19ee65a02b5702cff7d5a377e32c6ae29d8d" [[package]] name = "clipboard-win" -version = "5.4.0" +version = "5.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "15efe7a882b08f34e38556b14f2fb3daa98769d06c7f0c1b076dfd0d983bc892" +checksum = "bde03770d3df201d4fb868f2c9c59e66a3e4e2bd06692a0fe701e7103c7e84d4" dependencies = [ "error-code", ] [[package]] -name = "colorchoice" +name = "cmake" +version = "0.1.54" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7caa3f9de89ddbe2c607f4101924c5abec803763ae9534e4f4d7d8f84aa81f0" +dependencies = [ + "cc", +] + +[[package]] +name = "collection_literals" version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b63caa9aa9397e2d9480a9b13673856c78d8ac123288526c37d7839f2a86990" +checksum = "2550f75b8cfac212855f6b1885455df8eaee8fe8e246b647d69146142e016084" + +[[package]] +name = "colorchoice" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b05b61dc5112cbb17e4b6cd61790d9845d13888356391624cbe7e41efeac1e75" + +[[package]] +name = "combine" +version = "4.6.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ba5a308b75df32fe02788e748662718f03fde005016435c444eea572398219fd" +dependencies = [ + "bytes", + "memchr", +] [[package]] name = "compact_str" -version = "0.8.1" +version = "0.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3b79c4069c6cad78e2e0cdfcbd26275770669fb39fd308a752dc110e83b9af32" +checksum = "3fdb1325a1cece981e8a296ab8f0f9b63ae357bd0784a9faaf548cc7b480707a" dependencies = [ "castaway", "cfg-if", @@ -439,6 +642,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "const-oid" +version = "0.9.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c2459377285ad874054d797f3ccebf984978aa39129f6eafde5cdc8315b612f8" + [[package]] name = "constant_time_eq" version = "0.4.2" @@ -455,6 +664,16 @@ dependencies = [ "libc", ] +[[package]] +name = "core-foundation" +version = "0.10.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2a6cd9ae233e7f62ba4e9353e81a88df7fc8a5987b8d445b4d90c879bd156f6" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "core-foundation-sys" version = "0.8.7" @@ -472,9 +691,9 @@ dependencies = [ [[package]] name = "cranelift" -version = "0.119.0" +version = "0.126.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6d07c374d4da962eca0833c1d14621d5b4e32e68c8ca185b046a3b6b924ad334" +checksum = "68971376deb1edf5e9c0ac77ef00479d740ce7a60e6181adb0648afe1dc7b8f4" dependencies = [ "cranelift-codegen", "cranelift-frontend", @@ -483,42 +702,42 @@ dependencies = [ [[package]] name = "cranelift-assembler-x64" -version = "0.119.0" +version = "0.126.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "263cc79b8a23c29720eb596d251698f604546b48c34d0d84f8fd2761e5bf8888" +checksum = "30054f4aef4d614d37f27d5b77e36e165f0b27a71563be348e7c9fcfac41eed8" dependencies = [ "cranelift-assembler-x64-meta", ] [[package]] name = "cranelift-assembler-x64-meta" -version = "0.119.0" +version = "0.126.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b4a113455f8c0e13e3b3222a9c38d6940b958ff22573108be083495c72820e1" +checksum = "0beab56413879d4f515e08bcf118b1cb85f294129bb117057f573d37bfbb925a" dependencies = [ "cranelift-srcgen", ] [[package]] name = "cranelift-bforest" -version = "0.119.0" +version = "0.126.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "58f96dca41c5acf5d4312c1d04b3391e21a312f8d64ce31a2723a3bb8edd5d4d" +checksum = "6d054747549a69b264d5299c8ca1b0dd45dc6bd0ee43f1edfcc42a8b12952c7a" dependencies = [ "cranelift-entity", ] [[package]] name = "cranelift-bitset" -version = "0.119.0" +version = "0.126.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d821ed698dd83d9c012447eb63a5406c1e9c23732a2f674fb5b5015afd42202" +checksum = "98b92d481b77a7dc9d07c96e24a16f29e0c9c27d042828fdf7e49e54ee9819bf" [[package]] name = "cranelift-codegen" -version = "0.119.0" +version = "0.126.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "06c52fdec4322cb8d5545a648047819aaeaa04e630f88d3a609c0d3c1a00e9a0" +checksum = "6eeccfc043d599b0ef1806942707fc51cdd1c3965c343956dc975a55d82a920f" dependencies = [ "bumpalo", "cranelift-assembler-x64", @@ -530,55 +749,57 @@ dependencies = [ "cranelift-entity", "cranelift-isle", "gimli", - "hashbrown", + "hashbrown 0.15.5", "log", "regalloc2", "rustc-hash", "serde", "smallvec", "target-lexicon", + "wasmtime-internal-math", ] [[package]] name = "cranelift-codegen-meta" -version = "0.119.0" +version = "0.126.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af2c215e0c9afa8069aafb71d22aa0e0dde1048d9a5c3c72a83cacf9b61fcf4a" +checksum = "1174cdb9d9d43b2bdaa612a07ed82af13db9b95526bc2c286c2aec4689bcc038" dependencies = [ "cranelift-assembler-x64-meta", "cranelift-codegen-shared", "cranelift-srcgen", + "heck", ] [[package]] name = "cranelift-codegen-shared" -version = "0.119.0" +version = "0.126.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97524b2446fc26a78142132d813679dda19f620048ebc9a9fbb0ac9f2d320dcb" +checksum = "7d572be73fae802eb115f45e7e67a9ed16acb4ee683b67c4086768786545419a" [[package]] name = "cranelift-control" -version = "0.119.0" +version = "0.126.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8e32e900aee81f9e3cc493405ef667a7812cb5c79b5fc6b669e0a2795bda4b22" +checksum = "e1587465cc84c5cc793b44add928771945f3132bbf6b3621ee9473c631a87156" dependencies = [ "arbitrary", ] [[package]] name = "cranelift-entity" -version = "0.119.0" +version = "0.126.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d16a2e28e0fa6b9108d76879d60fe1cc95ba90e1bcf52bac96496371044484ee" +checksum = "063b83448b1343e79282c3c7cbda7ed5f0816f0b763a4c15f7cecb0a17d87ea6" dependencies = [ "cranelift-bitset", ] [[package]] name = "cranelift-frontend" -version = "0.119.0" +version = "0.126.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "328181a9083d99762d85954a16065d2560394a862b8dc10239f39668df528b95" +checksum = "aa4461c2d2ca48bc72883f5f5c3129d9aefac832df1db824af9db8db3efee109" dependencies = [ "cranelift-codegen", "log", @@ -588,15 +809,15 @@ dependencies = [ [[package]] name = "cranelift-isle" -version = "0.119.0" +version = "0.126.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e916f36f183e377e9a3ed71769f2721df88b72648831e95bb9fa6b0cd9b1c709" +checksum = "acd811b25e18f14810d09c504e06098acc1d9dbfa24879bf0d6b6fb44415fc66" [[package]] name = "cranelift-jit" -version = "0.119.0" +version = "0.126.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d6bb584ac927f1076d552504b0075b833b9d61e2e9178ba55df6b2d966b4375d" +checksum = "01527663ba63c10509d7c87fd1f8495d21170ba35bf714f57271495689d8fde5" dependencies = [ "anyhow", "cranelift-codegen", @@ -608,15 +829,15 @@ dependencies = [ "log", "region", "target-lexicon", - "wasmtime-jit-icache-coherence", - "windows-sys 0.59.0", + "wasmtime-internal-jit-icache-coherence", + "windows-sys 0.60.2", ] [[package]] name = "cranelift-module" -version = "0.119.0" +version = "0.126.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40c18ccb8e4861cf49cec79998af73b772a2b47212d12d3d63bf57cc4293a1e3" +checksum = "72328edb49aeafb1655818c91c476623970cb7b8a89ffbdadd82ce7d13dedc1d" dependencies = [ "anyhow", "cranelift-codegen", @@ -625,9 +846,9 @@ dependencies = [ [[package]] name = "cranelift-native" -version = "0.119.0" +version = "0.126.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc852cf04128877047dc2027aa1b85c64f681dc3a6a37ff45dcbfa26e4d52d2f" +checksum = "2417046989d8d6367a55bbab2e406a9195d176f4779be4aa484d645887217d37" dependencies = [ "cranelift-codegen", "libc", @@ -636,40 +857,39 @@ dependencies = [ [[package]] name = "cranelift-srcgen" -version = "0.119.0" +version = "0.126.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "47e1a86340a16e74b4285cc86ac69458fa1c8e7aaff313da4a89d10efd3535ee" +checksum = "8d039de901c8d928222b8128e1b9a9ab27b82a7445cb749a871c75d9cb25c57d" [[package]] name = "crc32fast" -version = "1.4.2" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a97769d94ddab943e4510d138150169a2758b5ef3eb191a9ee688de3e23ef7b3" +checksum = "9481c1c90cbf2ac953f07c8d4a58aa3945c425b7185c9154d67a65e4230da511" dependencies = [ "cfg-if", ] [[package]] name = "criterion" -version = "0.5.1" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f2b12d017a929603d80db1831cd3a24082f8137ce19c69e6447f54f5fc8d692f" +checksum = "4d883447757bb0ee46f233e9dc22eb84d93a9508c9b868687b274fc431d886bf" dependencies = [ + "alloca", "anes", "cast", "ciborium", "clap", "criterion-plot", - "is-terminal", - "itertools 0.10.5", + "itertools 0.13.0", "num-traits", - "once_cell", "oorandom", + "page_size", "plotters", "rayon", "regex", "serde", - "serde_derive", "serde_json", "tinytemplate", "walkdir", @@ -677,12 +897,12 @@ dependencies = [ [[package]] name = "criterion-plot" -version = "0.5.0" +version = "0.8.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6b50826342786a51a89e2da3a28f1c32b06e387201bc2d19791f622c673706b1" +checksum = "ed943f81ea2faa8dcecbbfa50164acf95d555afec96a27871663b300e387b2e4" dependencies = [ "cast", - "itertools 0.10.5", + "itertools 0.13.0", ] [[package]] @@ -712,15 +932,15 @@ checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28" [[package]] name = "crunchy" -version = "0.2.3" +version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "43da5946c66ffcc7745f48db692ffbb10a83bfe0afd96235c5c2a4fb23994929" +checksum = "460fbee9c2c2f33933d720630a6a0bac33ba7053db5344fac858d4b8952d77d5" [[package]] name = "crypto-common" -version = "0.1.6" +version = "0.1.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1bfb12502f3fc46cca1bb51ac28df9d618d813cdc3d2f25b9fe775a34af26bb3" +checksum = "78c8292055d1c1df0cce5d180393dc8cce0abec0a7102adb6c7b1eef6016d60a" dependencies = [ "generic-array", "typenum", @@ -728,13 +948,77 @@ dependencies = [ [[package]] name = "csv-core" -version = "0.1.12" +version = "0.1.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7d02f3b0da4c6504f86e9cd789d8dbafab48c2321be74e9987593de5a894d93d" +checksum = "704a3c26996a80471189265814dbc2c257598b96b8a7feae2d31ace646bb9782" dependencies = [ "memchr", ] +[[package]] +name = "data-encoding" +version = "2.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a2330da5de22e8a3cb63252ce2abb30116bf5265e89c0e01bc17015ce30a476" + +[[package]] +name = "der" +version = "0.7.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e7c1832837b905bbfb5101e07cc24c8deddf52f93225eee6ead5f4d63d53ddcb" +dependencies = [ + "const-oid", + "der_derive", + "flagset", + "pem-rfc7468 0.7.0", + "zeroize", +] + +[[package]] +name = "der-parser" +version = "10.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "07da5016415d5a3c4dd39b11ed26f915f52fc4e0dc197d87908bc916e51bc1a6" +dependencies = [ + "asn1-rs", + "displaydoc", + "nom", + "num-bigint", + "num-traits", + "rusticata-macros", +] + +[[package]] +name = "der_derive" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8034092389675178f570469e6c3b0465d3d30b4505c294a6550db47f3c17ad18" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "deranged" +version = "0.5.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ececcb659e7ba858fb4f10388c250a7252eb0a27373f1a72b8748afdd248e587" +dependencies = [ + "powerfmt", +] + +[[package]] +name = "derive-where" +version = "1.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ef941ded77d15ca19b40374869ac6000af1c9f2a4c0f3d4c70926287e6364a8f" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "digest" version = "0.10.7" @@ -767,23 +1051,40 @@ dependencies = [ "winapi", ] +[[package]] +name = "displaydoc" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97369cbbc041bc366949bc74d34658d6cda5621039731c6310521892a3a20ae0" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "dns-lookup" -version = "2.0.4" +version = "3.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5766087c2235fec47fafa4cfecc81e494ee679d0fd4a59887ea0919bfb0e4fc" +checksum = "6e39034cee21a2f5bbb66ba0e3689819c4bb5d00382a282006e802a7ffa6c41d" dependencies = [ "cfg-if", "libc", "socket2", - "windows-sys 0.48.0", + "windows-sys 0.60.2", ] +[[package]] +name = "dunce" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "92773504d58c093f6de2459af4af33faa518c13451eb8f2b5698ed3d36e7c813" + [[package]] name = "dyn-clone" -version = "1.0.19" +version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c7a8fb8a9fbf66c1f703fe16184d10ca0ee9d23be5b4436400408ba54a95005" +checksum = "d0881ea181b1df73ff77ffaaf9c7544ecc11e82fba9b5f27b262a3c73a332555" [[package]] name = "either" @@ -805,9 +1106,9 @@ checksum = "c34f04666d835ff5d62e058c3995147c06f42fe86ff053337632bca83e42702d" [[package]] name = "env_filter" -version = "0.1.3" +version = "0.1.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "186e05a59d4c50738528153b83b0b0194d3a29507dfec16eccd4b342903397d0" +checksum = "1bf3c259d255ca70051b30e2e95b5446cdb8949ac4cd22c0d7fd634d89f568e2" dependencies = [ "log", "regex", @@ -840,12 +1141,12 @@ checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" [[package]] name = "errno" -version = "0.3.11" +version = "0.3.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "976dd42dc7e85965fe702eb8164f21f450704bdde31faefd6471dba214cb594e" +checksum = "39cab71617ae0d63f51a36d69f866391735b51691dbda63cf6f96d042b63efeb" dependencies = [ "libc", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -866,6 +1167,12 @@ version = "0.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2acce4a10f12dc2fb14a218589d4f1f62ef011b2d0cc4b3cb1bba8e94da14649" +[[package]] +name = "fastrand" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "37909eebbb50d72f9059c3b6d82c0463f2ff062c9e95845c43a6c9c0355411be" + [[package]] name = "fd-lock" version = "4.0.4" @@ -877,13 +1184,25 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "find-msvc-tools" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3a3076410a55c90011c298b04d0cfa770b00fa04e1e3c97d3f6c9de105a03844" + +[[package]] +name = "flagset" +version = "0.4.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7ac824320a75a52197e8f2d787f6a38b6718bb6897a35142d749af3c0e8f4fe" + [[package]] name = "flame" version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1fc2706461e1ee94f55cab2ed2e3d34ae9536cfa830358ef80acff1a3dacab30" dependencies = [ - "lazy_static", + "lazy_static 0.2.11", "serde", "serde_derive", "serde_json", @@ -892,13 +1211,13 @@ dependencies = [ [[package]] name = "flamer" -version = "0.4.0" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "36b732da54fd4ea34452f2431cf464ac7be94ca4b339c9cd3d3d12eb06fe7aab" +checksum = "7693d9dd1ec1c54f52195dfe255b627f7cec7da33b679cd56de949e662b3db10" dependencies = [ "flame", "quote", - "syn 1.0.109", + "syn", ] [[package]] @@ -915,26 +1234,20 @@ dependencies = [ [[package]] name = "flate2" -version = "1.1.1" +version = "1.1.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7ced92e76e966ca2fd84c8f7aa01a4aea65b0eb6648d72f7c8f3e2764a67fece" +checksum = "bfe33edd8e85a12a67454e37f8c75e730830d83e313556ab9ebf9ee7fbeb3bfb" dependencies = [ "crc32fast", "libz-rs-sys", "miniz_oxide", ] -[[package]] -name = "fnv" -version = "1.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" - [[package]] name = "foldhash" -version = "0.1.5" +version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" +checksum = "77ce24cb58228fbb8aa041425bb1050850ac19177686ea6e0f41a70416f56fdb" [[package]] name = "foreign-types" @@ -951,6 +1264,12 @@ version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "00b0228411908ca8685dba7fc2cdd70ec9990a6e753e89b6ac91a84c40fbaf4b" +[[package]] +name = "fs_extra" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42703706b716c37f96a77aea830392ad231f44c9e9a67872fa5548707e11b11c" + [[package]] name = "generic-array" version = "0.14.7" @@ -961,23 +1280,46 @@ dependencies = [ "version_check", ] +[[package]] +name = "get-size-derive2" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ff47daa61505c85af126e9dd64af6a342a33dc0cccfe1be74ceadc7d352e6efd" +dependencies = [ + "attribute-derive", + "quote", + "syn", +] + +[[package]] +name = "get-size2" +version = "0.7.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ac7bb8710e1f09672102be7ddf39f764d8440ae74a9f4e30aaa4820dcdffa4af" +dependencies = [ + "compact_str", + "get-size-derive2", + "hashbrown 0.16.1", + "smallvec", +] + [[package]] name = "gethostname" -version = "1.0.2" +version = "1.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc257fdb4038301ce4b9cd1b3b51704509692bb3ff716a410cbd07925d9dae55" +checksum = "1bd49230192a3797a9a4d6abe9b3eed6f7fa4c8a8a4947977c6f80025f92cbd8" dependencies = [ "rustix", - "windows-targets 0.52.6", + "windows-link", ] [[package]] name = "getopts" -version = "0.2.21" +version = "0.2.24" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "14dbbfd5c71d70241ecf9e6f13737f7b5ce823821063188d7e46c41d371eebd5" +checksum = "cfe4fbac503b8d1f88e6676011885f34b7174f46e59956bba534ba83abded4df" dependencies = [ - "unicode-width 0.1.14", + "unicode-width", ] [[package]] @@ -988,28 +1330,28 @@ checksum = "335ff9f135e4384c8150d6f27c6daed433577f86b4750418338c01a1a2528592" dependencies = [ "cfg-if", "libc", - "wasi 0.11.0+wasi-snapshot-preview1", + "wasi", ] [[package]] name = "getrandom" -version = "0.3.2" +version = "0.3.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "73fea8450eea4bac3940448fb7ae50d91f034f941199fcd9d909a5a07aa455f0" +checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" dependencies = [ "cfg-if", "js-sys", "libc", "r-efi", - "wasi 0.14.2+wasi-0.2.4", + "wasip2", "wasm-bindgen", ] [[package]] name = "gimli" -version = "0.31.1" +version = "0.32.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07e28edb80900c19c28f1072f2e8aeca7fa06b23cd4169cefe1af5aa3260783f" +checksum = "e629b9b98ef3dd8afe6ca2bd0f89306cec16d43d907889945bc5d6687f2f13c7" dependencies = [ "fallible-iterator", "indexmap", @@ -1018,25 +1360,32 @@ dependencies = [ [[package]] name = "glob" -version = "0.3.2" +version = "0.3.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8d1add55171497b4705a648c6b583acafb01d58050a51727785f0b2c8e0a2b2" +checksum = "0cc23270f6e1808e30a928bdc84dea0b9b4136a8bc82338574f23baf47bbd280" [[package]] name = "half" -version = "2.6.0" +version = "2.7.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "459196ed295495a68f7d7fe1d84f6c4b7ff0e21fe3017b2f283c6fac3ad803c9" +checksum = "6ea2d84b969582b4b1864a92dc5d27cd2b77b622a8d79306834f1be5ba20d84b" dependencies = [ "cfg-if", "crunchy", + "zerocopy", ] [[package]] name = "hashbrown" -version = "0.15.3" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" + +[[package]] +name = "hashbrown" +version = "0.16.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "84b26c544d002229e640969970a2e74021aadf6e2f96372b9c58eff97de08eb3" +checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" dependencies = [ "foldhash", ] @@ -1049,15 +1398,9 @@ checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" [[package]] name = "hermit-abi" -version = "0.3.9" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" - -[[package]] -name = "hermit-abi" -version = "0.5.1" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f154ce46856750ed433c8649605bf7ed2de3bc35fd9d2a9f30cddd873c80cb08" +checksum = "fc0fef456e4baa96da950455cd02c081ca953b141298e41db3fc7e36b1da849c" [[package]] name = "hex" @@ -1071,20 +1414,29 @@ version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "dfa686283ad6dd069f105e5ab091b04c62850d3e4cf5d67debad1933f55023df" +[[package]] +name = "hmac" +version = "0.12.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6c49c37c09c17a53d937dfbb742eb3a961d65a994e6bcdcf37e7399d0cc8ab5e" +dependencies = [ + "digest", +] + [[package]] name = "home" -version = "0.5.11" +version = "0.5.12" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "589533453244b0995c858700322199b2becb13b627df2851f64a2775d024abcf" +checksum = "cc627f471c528ff0c4a49e1d5e60450c8f6461dd6d10ba9dcd3a61d3dff7728d" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] name = "iana-time-zone" -version = "0.1.63" +version = "0.1.64" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0c919e5debc312ad217002b8048a17b7d83f80703865bbfcfebb0458b0b27d8" +checksum = "33e57f83510bb73707521ebaffa789ec8caf86f9657cad665b092b581d40e9fb" dependencies = [ "android_system_properties", "core-foundation-sys", @@ -1092,7 +1444,7 @@ dependencies = [ "js-sys", "log", "wasm-bindgen", - "windows-core 0.61.0", + "windows-core", ] [[package]] @@ -1106,31 +1458,51 @@ dependencies = [ [[package]] name = "indexmap" -version = "2.9.0" +version = "2.12.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cea70ddb795996207ad57735b50c5982d8844f38ba9ee5f1aedcfb708a2aa11e" +checksum = "0ad4bb2b565bca0645f4d68c5c9af97fba094e9791da685bf83cb5f3ce74acf2" dependencies = [ "equivalent", - "hashbrown", + "hashbrown 0.16.1", ] [[package]] name = "indoc" -version = "2.0.6" +version = "2.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f4c7245a08504955605670dbf141fceab975f15ca21570696aebe9d2e71576bd" +checksum = "79cf5c93f93228cf8efb3ba362535fb11199ac548a09ce117c9b1adc3030d706" +dependencies = [ + "rustversion", +] + +[[package]] +name = "inout" +version = "0.1.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "879f10e63c20629ecabbb64a8010319738c66a5cd0c29b02d63d272b03751d01" +dependencies = [ + "block-padding", + "generic-array", +] [[package]] name = "insta" -version = "1.43.1" +version = "1.45.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "154934ea70c58054b556dd430b99a98c2a7ff5309ac9891597e339b5c28f4371" +checksum = "b76866be74d68b1595eb8060cb9191dca9c021db2316558e52ddc5d55d41b66c" dependencies = [ "console", "once_cell", "similar", + "tempfile", ] +[[package]] +name = "interpolator" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "71dd52191aae121e8611f1e8dc3e324dd0dd1dee1e6dd91d10ee07a3cfb4d9d8" + [[package]] name = "is-macro" version = "0.3.7" @@ -1140,34 +1512,14 @@ dependencies = [ "heck", "proc-macro2", "quote", - "syn 2.0.101", -] - -[[package]] -name = "is-terminal" -version = "0.4.16" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e04d7f318608d35d4b61ddd75cbdaee86b023ebe2bd5a66ee0915f0bf93095a9" -dependencies = [ - "hermit-abi 0.5.1", - "libc", - "windows-sys 0.59.0", + "syn", ] [[package]] name = "is_terminal_polyfill" -version = "1.70.1" +version = "1.70.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7943c866cc5cd64cbc25b2e01621d07fa8eb2a1a23160ee81ce38704e97b8ecf" - -[[package]] -name = "itertools" -version = "0.10.5" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b0fd2260e829bddf4cb6ea802289de2f86d6a7a690192fbe91b3f46e0f2c8473" -dependencies = [ - "either", -] +checksum = "a6cb138bb79a146c1bd460005623e142ef0181e3d0219cb493e02f7d08a35695" [[package]] name = "itertools" @@ -1195,33 +1547,65 @@ checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" [[package]] name = "jiff" -version = "0.2.13" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f02000660d30638906021176af16b17498bd0d12813dbfe7b276d8bc7f3c0806" +checksum = "49cce2b81f2098e7e3efc35bc2e0a6b7abec9d34128283d7a26fa8f32a6dbb35" dependencies = [ "jiff-static", "log", "portable-atomic", "portable-atomic-util", - "serde", + "serde_core", ] [[package]] name = "jiff-static" -version = "0.2.13" +version = "0.2.16" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f3c30758ddd7188629c6713fc45d1188af4f44c90582311d0c8d8c9907f60c48" +checksum = "980af8b43c3ad5d8d349ace167ec8170839f753a42d233ba19e08afe1850fa69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn", +] + +[[package]] +name = "jni" +version = "0.21.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1a87aa2bb7d2af34197c04845522473242e1aa17c12f4935d5856491a7fb8c97" +dependencies = [ + "cesu8", + "cfg-if", + "combine", + "jni-sys", + "log", + "thiserror 1.0.69", + "walkdir", + "windows-sys 0.45.0", +] + +[[package]] +name = "jni-sys" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8eaf4bc02d17cbdd7ff4c7438cafcdf7fb9a4613313ad11b4f8fefe7d3fa0130" + +[[package]] +name = "jobserver" +version = "0.1.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afb3de4395d6b3e67a780b6de64b51c978ecf11cb9a462c66be7d4ca9039d33" +dependencies = [ + "getrandom 0.3.4", + "libc", ] [[package]] name = "js-sys" -version = "0.3.77" +version = "0.3.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1cfaf33c695fc6e08064efbc1f72ec937429614f25eef83af942d0e227c3a28f" +checksum = "464a3709c7f55f1f721e5389aa6ea4e3bc6aba669353300af094b29ffbdde1d8" dependencies = [ "once_cell", "wasm-bindgen", @@ -1229,12 +1613,12 @@ dependencies = [ [[package]] name = "junction" -version = "1.2.0" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72bbdfd737a243da3dfc1f99ee8d6e166480f17ab4ac84d7c34aacd73fc7bd16" +checksum = "c52f6e1bf39a7894f618c9d378904a11dbd7e10fe3ec20d1173600e79b1408d8" dependencies = [ "scopeguard", - "windows-sys 0.52.0", + "windows-sys 0.60.2", ] [[package]] @@ -1252,35 +1636,36 @@ version = "0.2.11" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "76f033c7ad61445c5b347c7382dd1237847eb1bce590fe50365dcb33d546be73" +[[package]] +name = "lazy_static" +version = "1.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bbd2bcb4c963f2ddae06a2efc7e9f3591312473c50c6685e1f298068316e66fe" + [[package]] name = "lexical-parse-float" -version = "1.0.5" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "de6f9cb01fb0b08060209a057c048fcbab8717b4c1ecd2eac66ebfe39a65b0f2" +checksum = "52a9f232fbd6f550bc0137dcb5f99ab674071ac2d690ac69704593cb4abbea56" dependencies = [ "lexical-parse-integer", "lexical-util", - "static_assertions", ] [[package]] name = "lexical-parse-integer" -version = "1.0.5" +version = "1.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72207aae22fc0a121ba7b6d479e42cbfea549af1479c3f3a4f12c70dd66df12e" +checksum = "9a7a039f8fb9c19c996cd7b2fcce303c1b2874fe1aca544edc85c4a5f8489b34" dependencies = [ "lexical-util", - "static_assertions", ] [[package]] name = "lexical-util" -version = "1.0.6" +version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a82e24bf537fd24c177ffbbdc6ebcc8d54732c35b50a3f28cc3f4e4c949a0b3" -dependencies = [ - "static_assertions", -] +checksum = "2604dd126bb14f13fb5d1bd6a66155079cb9fa655b37f875b3a742c705dbed17" [[package]] name = "lexopt" @@ -1290,21 +1675,21 @@ checksum = "9fa0e2a1fcbe2f6be6c42e342259976206b383122fc152e872795338b5a3f3a7" [[package]] name = "libbz2-rs-sys" -version = "0.1.3" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0864a00c8d019e36216b69c2c4ce50b83b7bd966add3cf5ba554ec44f8bebcf5" +checksum = "2c4a545a15244c7d945065b5d392b2d2d7f21526fba56ce51467b06ed445e8f7" [[package]] name = "libc" -version = "0.2.172" +version = "0.2.178" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d750af042f7ef4f724306de029d18836c26c1765a54a6a3f094cbd23a7267ffa" +checksum = "37c93d8daa9d8a012fd8ab92f088405fb202ea0b6ab73ee2482ae66af4f42091" [[package]] name = "libffi" -version = "4.1.0" +version = "5.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebfd30a67b482a08116e753d0656cb626548cf4242543e5cc005be7639d99838" +checksum = "0444124f3ffd67e1b0b0c661a7f81a278a135eb54aaad4078e79fbc8be50c8a5" dependencies = [ "libc", "libffi-sys", @@ -1312,21 +1697,31 @@ dependencies = [ [[package]] name = "libffi-sys" -version = "3.3.1" +version = "4.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f003aa318c9f0ee69eb0ada7c78f5c9d2fedd2ceb274173b5c7ff475eee584a3" +checksum = "3d722da8817ea580d0669da6babe2262d7b86a1af1103da24102b8bb9c101ce7" dependencies = [ "cc", ] [[package]] name = "libloading" -version = "0.8.6" +version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34" +checksum = "d7c4b02199fee7c5d21a5ae7d8cfa79a6ef5bb2fc834d6e9058e89c825efdc55" dependencies = [ "cfg-if", - "windows-targets 0.52.6", + "windows-link", +] + +[[package]] +name = "libloading" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "754ca22de805bb5744484a5b151a9e1a8e837d5dc232c2d7d8c2e3492edc8b60" +dependencies = [ + "cfg-if", + "windows-link", ] [[package]] @@ -1337,19 +1732,19 @@ checksum = "f9fbbcab51052fe104eb5e5d351cf728d30a5be1fe14d9be8a3b097481fb97de" [[package]] name = "libredox" -version = "0.1.3" +version = "0.1.10" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c0ff37bd590ca25063e35af745c343cb7a0271906fb7b37e4813e8f79f00268d" +checksum = "416f7e718bdb06000964960ffa43b4335ad4012ae8b99060261aa4a8088d5ccb" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.10.0", "libc", ] [[package]] name = "libsqlite3-sys" -version = "0.28.0" +version = "0.36.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0c10584274047cb335c23d3e61bcef8e323adae7c5c8c760540f73610177fc3f" +checksum = "95b4103cffefa72eb8428cb6b47d6627161e51c2739fc5e3b734584157bc642a" dependencies = [ "cc", "pkg-config", @@ -1358,40 +1753,39 @@ dependencies = [ [[package]] name = "libz-rs-sys" -version = "0.5.0" +version = "0.5.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6489ca9bd760fe9642d7644e827b0c9add07df89857b0416ee15c1cc1a3b8c5a" +checksum = "840db8cf39d9ec4dd794376f38acc40d0fc65eec2a8f484f7fd375b84602becd" dependencies = [ "zlib-rs", ] [[package]] name = "linux-raw-sys" -version = "0.9.4" +version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cd945864f07fe9f5371a27ad7b52a172b4b499999f1d97574c9fa68373937e12" +checksum = "df1d3c3b53da64cf5760482273a98e575c651a67eec7f77df96b5b642de8f039" [[package]] name = "lock_api" -version = "0.4.12" +version = "0.4.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "07af8b9cdd281b7915f413fa73f29ebd5d55d0d3f0155584dade1ff18cea1b17" +checksum = "224399e74b87b5f3557511d98dff8b14089b3dadafcab6bb93eab67d3aace965" dependencies = [ - "autocfg", "scopeguard", ] [[package]] name = "log" -version = "0.4.27" +version = "0.4.29" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" +checksum = "5e5032e24019045c762d3c0f28f5b6b8bbf38563a65908389bf7978758920897" [[package]] name = "lz4_flex" -version = "0.11.3" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "75761162ae2b0e580d7e7c390558127e5f01b4194debd6221fd8c207fc80e3f5" +checksum = "ab6473172471198271ff72e9379150e9dfd70d8e533e0752a27e515b48dd375e" dependencies = [ "twox-hash", ] @@ -1413,26 +1807,26 @@ version = "1.1.8" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c0aeb26bf5e836cc1c341c8106051b573f1766dfa05aa87f0b98be5e51b02303" dependencies = [ - "nix", + "nix 0.29.0", "winapi", ] [[package]] name = "mach2" -version = "0.4.2" +version = "0.4.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "19b955cdeb2a02b9117f121ce63aa52d08ade45de53e48fe6a38b39c10f6f709" +checksum = "d640282b302c0bb0a2a8e0233ead9035e3bed871f0b7e81fe4a1ec829765db44" dependencies = [ "libc", ] [[package]] name = "malachite-base" -version = "0.6.0" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "554bcf7f816ff3c1eae8f2b95c4375156884c79988596a6d01b7b070710fa9e5" +checksum = "c0c91cb6071ed9ac48669d3c79bd2792db596c7e542dbadd217b385bb359f42d" dependencies = [ - "hashbrown", + "hashbrown 0.16.1", "itertools 0.14.0", "libm", "ryu", @@ -1440,9 +1834,9 @@ dependencies = [ [[package]] name = "malachite-bigint" -version = "0.6.0" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "df1acde414186498b2a6a1e271f8ce5d65eaa5c492e95271121f30718fe2f925" +checksum = "7ff3af5010102f29f2ef4ee6f7b1c5b3f08a6c261b5164e01c41cf43772b6f90" dependencies = [ "malachite-base", "malachite-nz", @@ -1453,26 +1847,50 @@ dependencies = [ [[package]] name = "malachite-nz" -version = "0.6.0" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f43d406336c42a59e07813b57efd651db00118af84c640a221d666964b2ec71f" +checksum = "1d9ecf4dd76246fd622de4811097966106aa43f9cd7cc36cb85e774fe84c8adc" dependencies = [ "itertools 0.14.0", "libm", "malachite-base", + "wide", ] [[package]] name = "malachite-q" -version = "0.6.0" +version = "0.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "25911a58ea0426e0b7bb1dffc8324e82711c82abff868b8523ae69d8a47e8062" +checksum = "b7bc9d9adf5b0a7999d84f761c809bec3dc46fe983e4de547725d2b7730462a0" dependencies = [ "itertools 0.14.0", "malachite-base", "malachite-nz", ] +[[package]] +name = "manyhow" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b33efb3ca6d3b07393750d4030418d594ab1139cee518f0dc88db70fec873587" +dependencies = [ + "manyhow-macros", + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "manyhow-macros" +version = "0.11.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46fce34d199b78b6e6073abf984c9cf5fd3e9330145a93ee0738a7443e371495" +dependencies = [ + "proc-macro-utils", + "proc-macro2", + "quote", +] + [[package]] name = "maplit" version = "1.0.2" @@ -1497,15 +1915,15 @@ dependencies = [ [[package]] name = "memchr" -version = "2.7.4" +version = "2.7.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" +checksum = "f52b00d39961fc5b2736ea853c9cc86238e165017a493d1d5c8eac6bdc4cc273" [[package]] name = "memmap2" -version = "0.5.10" +version = "0.9.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "83faa42c0a078c393f6b29d5db232d8be22776a891f8f56e5284faee4a20b327" +checksum = "744133e4a0e0a658e1374cf3bf8e415c4052a15a111acd372764c55b4177d490" dependencies = [ "libc", ] @@ -1527,11 +1945,12 @@ checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a" [[package]] name = "miniz_oxide" -version = "0.8.8" +version = "0.8.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3be647b768db090acb35d5ec5db2b0e1f1de11133ca123b9eacf5137868f892a" +checksum = "1fa76a2c86f704bdb222d66965fb3d63269ce38518b83cb0575fca855ebb6316" dependencies = [ "adler2", + "simd-adler32", ] [[package]] @@ -1558,7 +1977,20 @@ version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "71e2746dc3a24dd78b3cfcb7be93368c6de9963d30f43a6a73998a9cf4b17b46" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.10.0", + "cfg-if", + "cfg_aliases", + "libc", + "memoffset", +] + +[[package]] +name = "nix" +version = "0.30.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "74523f3a35e05aba87a1d978330aef40f67b0304ac79c1c00b294c9830543db6" +dependencies = [ + "bitflags 2.10.0", "cfg-if", "cfg_aliases", "libc", @@ -1575,6 +2007,16 @@ dependencies = [ "minimal-lexical", ] +[[package]] +name = "num-bigint" +version = "0.4.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a5e44f723f1133c9deac646763579fdb3ac745e418f2a7af9cd0c431da1f20b9" +dependencies = [ + "num-integer", + "num-traits", +] + [[package]] name = "num-complex" version = "0.4.6" @@ -1584,6 +2026,12 @@ dependencies = [ "num-traits", ] +[[package]] +name = "num-conv" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "51d515d32fb182ee37cda2ccdcb92950d6a3c2893aa280e540671c2cd0f3b1d9" + [[package]] name = "num-integer" version = "0.1.46" @@ -1604,32 +2052,42 @@ dependencies = [ [[package]] name = "num_cpus" -version = "1.16.0" +version = "1.17.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" +checksum = "91df4bbde75afed763b708b7eee1e8e7651e02d97f6d5dd763e89367e957b23b" dependencies = [ - "hermit-abi 0.3.9", + "hermit-abi", "libc", ] [[package]] name = "num_enum" -version = "0.7.3" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4e613fc340b2220f734a8595782c551f1250e969d87d3be1ae0579e8d4065179" +checksum = "b1207a7e20ad57b847bbddc6776b968420d38292bbfe2089accff5e19e82454c" dependencies = [ "num_enum_derive", + "rustversion", ] [[package]] name = "num_enum_derive" -version = "0.7.3" +version = "0.7.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "af1844ef2428cc3e1cb900be36181049ef3d3193c63e43026cfe202983b27a56" +checksum = "ff32365de1b6743cb203b710788263c44a03de03802daf96092f2da4fe6ba4d7" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn", +] + +[[package]] +name = "oid-registry" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12f40cff3dde1b6087cc5d5f5d4d65712f34016a03ed60e9c08dcc392736b5b7" +dependencies = [ + "asn1-rs", ] [[package]] @@ -1638,6 +2096,12 @@ version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +[[package]] +name = "once_cell_polyfill" +version = "1.70.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "384b8ab6d37215f3c5301a95a4accb5d64aa607f1fcb26a11b5303878451b4fe" + [[package]] name = "oorandom" version = "11.1.5" @@ -1646,11 +2110,11 @@ checksum = "d6790f58c7ff633d8771f42965289203411a5e5c68388703c06e14f24770b41e" [[package]] name = "openssl" -version = "0.10.72" +version = "0.10.75" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "fedfea7d58a1f73118430a55da6a286e7b044961736ce96a16a17068ea25e5da" +checksum = "08838db121398ad17ab8531ce9de97b244589089e290a384c900cb9ff7434328" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.10.0", "cfg-if", "foreign-types", "libc", @@ -1667,7 +2131,7 @@ checksum = "a948666b637a0f465e8564c73e89d4dde00d72d4d473cc972f390fc3dcee7d9c" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn", ] [[package]] @@ -1678,18 +2142,18 @@ checksum = "d05e27ee213611ffe7d6348b942e8f942b37114c00cc03cec254295a4a17852e" [[package]] name = "openssl-src" -version = "300.5.0+3.5.0" +version = "300.5.4+3.5.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e8ce546f549326b0e6052b649198487d91320875da901e7bd11a06d1ee3f9c2f" +checksum = "a507b3792995dae9b0df8a1c1e3771e8418b7c2d9f0baeba32e6fe8b06c7cb72" dependencies = [ "cc", ] [[package]] name = "openssl-sys" -version = "0.9.108" +version = "0.9.111" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e145e1651e858e820e4860f7b9c5e169bc1d8ce1c86043be79fa7b7634821847" +checksum = "82cab2d520aa75e3c58898289429321eb788c3106963d0dc886ec7a5f4adc321" dependencies = [ "cc", "libc", @@ -1708,48 +2172,87 @@ checksum = "978aa494585d3ca4ad74929863093e87cac9790d81fe7aba2b3dc2890643a0fc" name = "page_size" version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "30d5b2194ed13191c1999ae0704b7839fb18384fa22e49b57eeaa97d79ce40da" +checksum = "30d5b2194ed13191c1999ae0704b7839fb18384fa22e49b57eeaa97d79ce40da" +dependencies = [ + "libc", + "winapi", +] + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core", +] + +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall 0.5.18", + "smallvec", + "windows-link", +] + +[[package]] +name = "paste" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" + +[[package]] +name = "pbkdf2" +version = "0.12.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8ed6a7761f76e3b9f92dfb0a60a6a6477c61024b775147ff0973a02653abaf2" dependencies = [ - "libc", - "winapi", + "digest", + "hmac", ] [[package]] -name = "parking_lot" -version = "0.12.3" +name = "pem-rfc7468" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f1bf18183cf54e8d6059647fc3063646a1801cf30896933ec2311622cc4b9a27" +checksum = "88b39c9bfcfc231068454382784bb460aae594343fb030d46e9f50a645418412" dependencies = [ - "lock_api", - "parking_lot_core", + "base64ct", ] [[package]] -name = "parking_lot_core" -version = "0.9.10" +name = "pem-rfc7468" +version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1e401f977ab385c9e4e3ab30627d6f26d00e2c73eef317493c4ec6d468726cf8" +checksum = "a6305423e0e7738146434843d1694d621cce767262b2a86910beab705e4493d9" dependencies = [ - "cfg-if", - "libc", - "redox_syscall 0.5.12", - "smallvec", - "windows-targets 0.52.6", + "base64ct", ] [[package]] -name = "paste" -version = "1.0.15" +name = "phf" +version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "57c0d7b74b563b49d38dae00a0c37d4d6de9b432382b2892f0574ddcae73fd0a" +checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" +dependencies = [ + "phf_shared 0.11.3", +] [[package]] name = "phf" -version = "0.11.3" +version = "0.13.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fd6780a80ae0c52cc120a26a1a42c1ae51b247a253e4e06113d23d2c2edd078" +checksum = "c1562dc717473dbaa4c1f85a36410e03c047b2e7df7f45ee938fbef64ae7fadf" dependencies = [ - "phf_shared", + "phf_macros", + "phf_shared 0.13.1", + "serde", ] [[package]] @@ -1758,8 +2261,8 @@ version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "aef8048c789fa5e851558d709946d6d79a8ff88c0440c587967f8e94bfb1216a" dependencies = [ - "phf_generator", - "phf_shared", + "phf_generator 0.11.3", + "phf_shared 0.11.3", ] [[package]] @@ -1768,10 +2271,33 @@ version = "0.11.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3c80231409c20246a13fddb31776fb942c38553c51e871f8cbd687a4cfb5843d" dependencies = [ - "phf_shared", + "phf_shared 0.11.3", "rand 0.8.5", ] +[[package]] +name = "phf_generator" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "135ace3a761e564ec88c03a77317a7c6b80bb7f7135ef2544dbe054243b89737" +dependencies = [ + "fastrand", + "phf_shared 0.13.1", +] + +[[package]] +name = "phf_macros" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "812f032b54b1e759ccd5f8b6677695d5268c588701effba24601f6932f8269ef" +dependencies = [ + "phf_generator 0.13.1", + "phf_shared 0.13.1", + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "phf_shared" version = "0.11.3" @@ -1781,6 +2307,42 @@ dependencies = [ "siphasher", ] +[[package]] +name = "phf_shared" +version = "0.13.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e57fef6bc5981e38c2ce2d63bfa546861309f875b8a75f092d1d54ae2d64f266" +dependencies = [ + "siphasher", +] + +[[package]] +name = "pkcs5" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e847e2c91a18bfa887dd028ec33f2fe6f25db77db3619024764914affe8b69a6" +dependencies = [ + "aes", + "cbc", + "der", + "pbkdf2", + "scrypt", + "sha2", + "spki", +] + +[[package]] +name = "pkcs8" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f950b2377845cebe5cf8b5165cb3cc1a5e0fa5cfa3e1f7f55707d8fd82e0a7b7" +dependencies = [ + "der", + "pkcs5", + "rand_core 0.6.4", + "spki", +] + [[package]] name = "pkg-config" version = "0.3.32" @@ -1823,14 +2385,14 @@ checksum = "52a40bc70c2c58040d2d8b167ba9a5ff59fc9dab7ad44771cfde3dcfde7a09c6" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn", ] [[package]] name = "portable-atomic" -version = "1.11.0" +version = "1.11.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "350e9b48cbc6b0e028b0473b114454c6316e57336ee184ceab6e53f72c178b3e" +checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483" [[package]] name = "portable-atomic-util" @@ -1841,6 +2403,12 @@ dependencies = [ "portable-atomic", ] +[[package]] +name = "powerfmt" +version = "0.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "439ee305def115ba05938db6eb1644ff94165c5ab5e9420d1c1bcedbba909391" + [[package]] name = "ppv-lite86" version = "0.2.21" @@ -1852,19 +2420,30 @@ dependencies = [ [[package]] name = "prettyplease" -version = "0.2.32" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn", +] + +[[package]] +name = "proc-macro-utils" +version = "0.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "664ec5419c51e34154eec046ebcba56312d5a2fc3b09a06da188e1ad21afadf6" +checksum = "eeaf08a13de400bc215877b5bdc088f241b12eb42f0a548d3390dc1c56bb7071" dependencies = [ "proc-macro2", - "syn 2.0.101", + "quote", + "smallvec", ] [[package]] name = "proc-macro2" -version = "1.0.95" +version = "1.0.103" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "02b3e5e68a3a1a02aad3ec490a98007cbc13c37cbe84a3cd7b8e406d76e7f778" +checksum = "5ee95bc4ef87b8d5ba32e8b7714ccc834865276eab0aed5c9958d00ec45f49e8" dependencies = [ "unicode-ident", ] @@ -1880,11 +2459,10 @@ dependencies = [ [[package]] name = "pyo3" -version = "0.24.2" +version = "0.27.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e5203598f366b11a02b13aa20cab591229ff0a89fd121a308a5df751d5fc9219" +checksum = "ab53c047fcd1a1d2a8820fe84f05d6be69e9526be40cb03b73f86b6b03e6d87d" dependencies = [ - "cfg-if", "indoc", "libc", "memoffset", @@ -1898,19 +2476,18 @@ dependencies = [ [[package]] name = "pyo3-build-config" -version = "0.24.2" +version = "0.27.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "99636d423fa2ca130fa5acde3059308006d46f98caac629418e53f7ebb1e9999" +checksum = "b455933107de8642b4487ed26d912c2d899dec6114884214a0b3bb3be9261ea6" dependencies = [ - "once_cell", "target-lexicon", ] [[package]] name = "pyo3-ffi" -version = "0.24.2" +version = "0.27.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "78f9cf92ba9c409279bc3305b5409d90db2d2c22392d443a87df3a1adad59e33" +checksum = "1c85c9cbfaddf651b1221594209aed57e9e5cff63c4d11d1feead529b872a089" dependencies = [ "libc", "pyo3-build-config", @@ -1918,48 +2495,71 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.24.2" +version = "0.27.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b999cb1a6ce21f9a6b147dcf1be9ffedf02e0043aec74dc390f3007047cecd9" +checksum = "0a5b10c9bf9888125d917fb4d2ca2d25c8df94c7ab5a52e13313a07e050a3b02" dependencies = [ "proc-macro2", "pyo3-macros-backend", "quote", - "syn 2.0.101", + "syn", ] [[package]] name = "pyo3-macros-backend" -version = "0.24.2" +version = "0.27.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "822ece1c7e1012745607d5cf0bcb2874769f0f7cb34c4cde03b9358eb9ef911a" +checksum = "03b51720d314836e53327f5871d4c0cfb4fb37cc2c4a11cc71907a86342c40f9" dependencies = [ "heck", "proc-macro2", "pyo3-build-config", "quote", - "syn 2.0.101", + "syn", ] [[package]] name = "quote" -version = "1.0.40" +version = "1.0.42" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a338cc41d27e6cc6dce6cefc13a0729dfbb81c262b1f519331575dd80ef3067f" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "quote-use" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9619db1197b497a36178cfc736dc96b271fe918875fbf1344c436a7e93d0321e" +dependencies = [ + "quote", + "quote-use-macros", +] + +[[package]] +name = "quote-use-macros" +version = "0.8.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" +checksum = "82ebfb7faafadc06a7ab141a6f67bcfb24cb8beb158c6fe933f2f035afa99f35" dependencies = [ + "proc-macro-utils", "proc-macro2", + "quote", + "syn", ] [[package]] name = "r-efi" -version = "5.2.0" +version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "74765f6d916ee2faa39bc8e68e4f3ed8949b48cccdac59983d287a7cb71ce9c5" +checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" [[package]] name = "radium" -version = "1.1.0" -source = "git+https://github.com/youknowone/ferrilab?branch=fix-nightly#4a301c3a223e096626a2773d1a1eed1fc4e21140" +version = "1.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1775bc532a9bfde46e26eba441ca1171b91608d14a3bae71fea371f18a00cffe" dependencies = [ "cfg-if", ] @@ -1987,9 +2587,9 @@ dependencies = [ [[package]] name = "rand" -version = "0.9.1" +version = "0.9.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9fbfd9d094a40bf3ae768db9361049ace4c0e04a4fd6b359518bd7b73a73dd97" +checksum = "6db2770f06117d490610c7488547d543617b21bfa07796d7a12f6f1bd53850d1" dependencies = [ "rand_chacha 0.9.0", "rand_core 0.9.3", @@ -2030,14 +2630,14 @@ version = "0.9.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "99d9a13982dcf210057a8a78572b2217b667c3beacbf3a0d8b454f6f82837d38" dependencies = [ - "getrandom 0.3.2", + "getrandom 0.3.4", ] [[package]] name = "rayon" -version = "1.10.0" +version = "1.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b418a60154510ca1a002a752ca9714984e21e4241e804d32555251faf8b78ffa" +checksum = "368f01d005bf8fd9b1206fb6fa653e6c4a81ceb1466406b81792d87c5677a58f" dependencies = [ "either", "rayon-core", @@ -2045,9 +2645,9 @@ dependencies = [ [[package]] name = "rayon-core" -version = "1.12.1" +version = "1.13.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1465873a3dfdaa8ae7cb14b4383657caab0b3e8a0aa9ae8e04b044854c8dfce2" +checksum = "22e18b0f0062d30d4230b2e85ff77fdfe4326feb054b9783a3460d8435c8ab91" dependencies = [ "crossbeam-deque", "crossbeam-utils", @@ -2061,11 +2661,11 @@ checksum = "41cc0f7e4d5d4544e8861606a285bb08d3e70712ccc7d2b84d7c0ccfaf4b05ce" [[package]] name = "redox_syscall" -version = "0.5.12" +version = "0.5.18" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "928fca9cf2aa042393a8325b9ead81d2f0df4cb12e1e24cef072922ccd99c5af" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.10.0", ] [[package]] @@ -2081,13 +2681,13 @@ dependencies = [ [[package]] name = "regalloc2" -version = "0.11.2" +version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc06e6b318142614e4a48bc725abbf08ff166694835c43c9dae5a9009704639a" +checksum = "4e249c660440317032a71ddac302f25f1d5dff387667bcc3978d1f77aa31ac34" dependencies = [ "allocator-api2", "bumpalo", - "hashbrown", + "hashbrown 0.15.5", "log", "rustc-hash", "smallvec", @@ -2095,9 +2695,9 @@ dependencies = [ [[package]] name = "regex" -version = "1.11.1" +version = "1.12.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b544ef1b4eac5dc2db33ea63606ae9ffcfac26c1416a2806ae0bf5f56b201191" +checksum = "843bc0191f75f3e22651ae5f1e72939ab2f72a4bc30fa80a066bd66edefc24d4" dependencies = [ "aho-corasick", "memchr", @@ -2107,9 +2707,9 @@ dependencies = [ [[package]] name = "regex-automata" -version = "0.4.9" +version = "0.4.13" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "809e8dc61f6de73b46c85f4c96486310fe304c434cfa43669d7b40f711150908" +checksum = "5276caf25ac86c8d810222b3dbb938e512c55c6831a10f3e6ed1c93b84041f1c" dependencies = [ "aho-corasick", "memchr", @@ -2118,9 +2718,9 @@ dependencies = [ [[package]] name = "regex-syntax" -version = "0.8.5" +version = "0.8.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b15c43186be67a4fd63bee50d0303afffcef381492ebe2c5d87f324e1b8815c" +checksum = "7a2d987857b319362043e95f5353c0535c1f58eec5336fdfcf626430af7def58" [[package]] name = "region" @@ -2136,33 +2736,48 @@ dependencies = [ [[package]] name = "result-like" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "abf7172fef6a7d056b5c26bf6c826570267562d51697f4982ff3ba4aec68a9df" +checksum = "bffa194499266bd8a1ac7da6ac7355aa0f81ffa1a5db2baaf20dd13854fd6f4e" dependencies = [ "result-like-derive", ] [[package]] name = "result-like-derive" -version = "0.5.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8d6574c02e894d66370cfc681e5d68fedbc9a548fb55b30a96b3f0ae22d0fe5" +checksum = "01d3b03471c9700a3a6bd166550daaa6124cb4a146ea139fb028e4edaa8f4277" dependencies = [ "pmutil", "proc-macro2", "quote", - "syn 2.0.101", + "syn", +] + +[[package]] +name = "ring" +version = "0.17.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a4689e6c2294d81e88dc6261c768b63bc4fcdb852be6d1352498b114f61383b7" +dependencies = [ + "cc", + "cfg-if", + "getrandom 0.2.16", + "libc", + "untrusted 0.9.0", + "windows-sys 0.52.0", ] [[package]] name = "ruff_python_ast" version = "0.0.0" -source = "git+https://github.com/astral-sh/ruff.git?tag=0.11.0#2cd25ef6410fb5fca96af1578728a3d828d2d53a" +source = "git+https://github.com/astral-sh/ruff.git?rev=2bffef59665ce7d2630dfd72ee99846663660db8#2bffef59665ce7d2630dfd72ee99846663660db8" dependencies = [ "aho-corasick", - "bitflags 2.9.0", + "bitflags 2.10.0", "compact_str", + "get-size2", "is-macro", "itertools 0.14.0", "memchr", @@ -2170,16 +2785,18 @@ dependencies = [ "ruff_source_file", "ruff_text_size", "rustc-hash", + "thiserror 2.0.17", ] [[package]] name = "ruff_python_parser" version = "0.0.0" -source = "git+https://github.com/astral-sh/ruff.git?tag=0.11.0#2cd25ef6410fb5fca96af1578728a3d828d2d53a" +source = "git+https://github.com/astral-sh/ruff.git?rev=2bffef59665ce7d2630dfd72ee99846663660db8#2bffef59665ce7d2630dfd72ee99846663660db8" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.10.0", "bstr", "compact_str", + "get-size2", "memchr", "ruff_python_ast", "ruff_python_trivia", @@ -2188,13 +2805,13 @@ dependencies = [ "static_assertions", "unicode-ident", "unicode-normalization", - "unicode_names2", + "unicode_names2 1.3.0", ] [[package]] name = "ruff_python_trivia" version = "0.0.0" -source = "git+https://github.com/astral-sh/ruff.git?tag=0.11.0#2cd25ef6410fb5fca96af1578728a3d828d2d53a" +source = "git+https://github.com/astral-sh/ruff.git?rev=2bffef59665ce7d2630dfd72ee99846663660db8#2bffef59665ce7d2630dfd72ee99846663660db8" dependencies = [ "itertools 0.14.0", "ruff_source_file", @@ -2205,7 +2822,7 @@ dependencies = [ [[package]] name = "ruff_source_file" version = "0.0.0" -source = "git+https://github.com/astral-sh/ruff.git?tag=0.11.0#2cd25ef6410fb5fca96af1578728a3d828d2d53a" +source = "git+https://github.com/astral-sh/ruff.git?rev=2bffef59665ce7d2630dfd72ee99846663660db8#2bffef59665ce7d2630dfd72ee99846663660db8" dependencies = [ "memchr", "ruff_text_size", @@ -2214,7 +2831,10 @@ dependencies = [ [[package]] name = "ruff_text_size" version = "0.0.0" -source = "git+https://github.com/astral-sh/ruff.git?tag=0.11.0#2cd25ef6410fb5fca96af1578728a3d828d2d53a" +source = "git+https://github.com/astral-sh/ruff.git?rev=2bffef59665ce7d2630dfd72ee99846663660db8#2bffef59665ce7d2630dfd72ee99846663660db8" +dependencies = [ + "get-size2", +] [[package]] name = "rustc-hash" @@ -2222,17 +2842,109 @@ version = "2.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "357703d41365b4b27c590e3ed91eabb1b663f07c4c084095e60cbed4362dff0d" +[[package]] +name = "rusticata-macros" +version = "4.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "faf0c4a6ece9950b9abdb62b1cfcf2a68b3b67a10ba445b3bb85be2a293d0632" +dependencies = [ + "nom", +] + [[package]] name = "rustix" -version = "1.0.7" +version = "1.1.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c71e83d6afe7ff64890ec6b71d6a69bb8a610ab78ce364b3352876bb4c801266" +checksum = "cd15f8a2c5551a84d56efdc1cd049089e409ac19a3072d5037a17fd70719ff3e" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.10.0", "errno", "libc", "linux-raw-sys", - "windows-sys 0.59.0", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls" +version = "0.23.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "533f54bc6a7d4f647e46ad909549eda97bf5afc1585190ef692b4286b198bd8f" +dependencies = [ + "aws-lc-rs", + "once_cell", + "rustls-pki-types", + "rustls-webpki", + "subtle", + "zeroize", +] + +[[package]] +name = "rustls-native-certs" +version = "0.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9980d917ebb0c0536119ba501e90834767bffc3d60641457fd84a1f3fd337923" +dependencies = [ + "openssl-probe", + "rustls-pki-types", + "schannel", + "security-framework", +] + +[[package]] +name = "rustls-pemfile" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dce314e5fee3f39953d46bb63bb8a46d40c2f8fb7cc5a3b6cab2bde9721d6e50" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "rustls-pki-types" +version = "1.13.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "94182ad936a0c91c324cd46c6511b9510ed16af436d7b5bab34beab0afd55f7a" +dependencies = [ + "zeroize", +] + +[[package]] +name = "rustls-platform-verifier" +version = "0.6.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1d99feebc72bae7ab76ba994bb5e121b8d83d910ca40b36e0921f53becc41784" +dependencies = [ + "core-foundation 0.10.1", + "core-foundation-sys", + "jni", + "log", + "once_cell", + "rustls", + "rustls-native-certs", + "rustls-platform-verifier-android", + "rustls-webpki", + "security-framework", + "security-framework-sys", + "webpki-root-certs", + "windows-sys 0.61.2", +] + +[[package]] +name = "rustls-platform-verifier-android" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f87165f0995f63a9fbeea62b64d10b4d9d8e78ec6d7d51fb2125fda7bb36788f" + +[[package]] +name = "rustls-webpki" +version = "0.103.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2ffdfa2f5286e2247234e03f680868ac2815974dc39e00ea15adc445d0aafe52" +dependencies = [ + "aws-lc-rs", + "ring", + "rustls-pki-types", + "untrusted 0.9.0", ] [[package]] @@ -2263,7 +2975,7 @@ name = "rustpython-codegen" version = "0.4.0" dependencies = [ "ahash", - "bitflags 2.9.0", + "bitflags 2.10.0", "indexmap", "insta", "itertools 0.14.0", @@ -2274,14 +2986,12 @@ dependencies = [ "num-traits", "ruff_python_ast", "ruff_python_parser", - "ruff_source_file", "ruff_text_size", "rustpython-compiler-core", - "rustpython-compiler-source", "rustpython-literal", "rustpython-wtf8", - "thiserror 2.0.12", - "unicode_names2", + "thiserror 2.0.17", + "unicode_names2 2.0.0", ] [[package]] @@ -2289,17 +2999,16 @@ name = "rustpython-common" version = "0.4.0" dependencies = [ "ascii", - "bitflags 2.9.0", - "bstr", + "bitflags 2.10.0", "cfg-if", - "getrandom 0.3.2", + "getrandom 0.3.4", "itertools 0.14.0", "libc", "lock_api", "malachite-base", "malachite-bigint", "malachite-q", - "memchr", + "nix 0.30.1", "num-complex", "num-traits", "once_cell", @@ -2308,43 +3017,40 @@ dependencies = [ "rustpython-literal", "rustpython-wtf8", "siphasher", - "unicode_names2", + "unicode_names2 2.0.0", "widestring", - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] name = "rustpython-compiler" version = "0.4.0" dependencies = [ - "rand 0.9.1", "ruff_python_ast", "ruff_python_parser", "ruff_source_file", "ruff_text_size", "rustpython-codegen", "rustpython-compiler-core", - "rustpython-compiler-source", - "thiserror 2.0.12", + "thiserror 2.0.17", ] [[package]] name = "rustpython-compiler-core" version = "0.4.0" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.10.0", "itertools 0.14.0", "lz4_flex", "malachite-bigint", "num-complex", "ruff_source_file", "rustpython-wtf8", - "serde", ] [[package]] name = "rustpython-compiler-source" -version = "0.4.0" +version = "0.5.0+deprecated" dependencies = [ "ruff_source_file", "ruff_text_size", @@ -2354,10 +3060,9 @@ dependencies = [ name = "rustpython-derive" version = "0.4.0" dependencies = [ - "proc-macro2", "rustpython-compiler", "rustpython-derive-impl", - "syn 2.0.101", + "syn", ] [[package]] @@ -2370,17 +3075,16 @@ dependencies = [ "quote", "rustpython-compiler-core", "rustpython-doc", - "syn 2.0.101", + "syn", "syn-ext", "textwrap", ] [[package]] name = "rustpython-doc" -version = "0.3.0" -source = "git+https://github.com/RustPython/__doc__?tag=0.3.0#8b62ce5d796d68a091969c9fa5406276cb483f79" +version = "0.4.0" dependencies = [ - "once_cell", + "phf 0.13.1", ] [[package]] @@ -2395,7 +3099,7 @@ dependencies = [ "num-traits", "rustpython-compiler-core", "rustpython-derive", - "thiserror 2.0.12", + "thiserror 2.0.17", ] [[package]] @@ -2406,7 +3110,7 @@ dependencies = [ "is-macro", "lexical-parse-float", "num-traits", - "rand 0.9.1", + "rand 0.9.2", "rustpython-wtf8", "unic-ucd-category", ] @@ -2424,7 +3128,7 @@ dependencies = [ name = "rustpython-sre_engine" version = "0.4.0" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.10.0", "criterion", "num_enum", "optional", @@ -2438,13 +3142,16 @@ dependencies = [ "adler32", "ahash", "ascii", + "aws-lc-rs", "base64", "blake2", "bzip2", "cfg-if", + "chrono", "crc32fast", "crossbeam-utils", "csv-core", + "der", "digest", "dns-lookup", "dyn-clone", @@ -2454,7 +3161,6 @@ dependencies = [ "hex", "indexmap", "itertools 0.14.0", - "junction", "libc", "libsqlite3-sys", "libz-rs-sys", @@ -2465,20 +3171,28 @@ dependencies = [ "memchr", "memmap2", "mt19937", - "nix", + "nix 0.30.1", "num-complex", "num-integer", "num-traits", "num_enum", + "oid-registry", "openssl", "openssl-probe", "openssl-sys", "page_size", "parking_lot", "paste", + "pem-rfc7468 1.0.0", + "phf 0.13.1", + "pkcs8", "pymath", "rand_core 0.9.3", "rustix", + "rustls", + "rustls-native-certs", + "rustls-pemfile", + "rustls-platform-verifier", "rustpython-common", "rustpython-derive", "rustpython-vm", @@ -2500,21 +3214,31 @@ dependencies = [ "unic-ucd-ident", "unicode-bidi-mirroring", "unicode-casing", - "unicode_names2", + "unicode_names2 2.0.0", "uuid", + "webpki-roots", "widestring", - "windows-sys 0.59.0", - "xml-rs", + "windows-sys 0.61.2", + "x509-cert", + "x509-parser", + "xml", "xz2", ] +[[package]] +name = "rustpython-venvlauncher" +version = "0.4.0" +dependencies = [ + "windows-sys 0.61.2", +] + [[package]] name = "rustpython-vm" version = "0.4.0" dependencies = [ "ahash", "ascii", - "bitflags 2.9.0", + "bitflags 2.10.0", "bstr", "caseless", "cfg-if", @@ -2525,7 +3249,7 @@ dependencies = [ "exitcode", "flame", "flamer", - "getrandom 0.3.2", + "getrandom 0.3.4", "glob", "half", "hex", @@ -2535,12 +3259,11 @@ dependencies = [ "junction", "libc", "libffi", - "libloading", + "libloading 0.9.0", "log", "malachite-bigint", "memchr", - "memoffset", - "nix", + "nix 0.30.1", "num-complex", "num-integer", "num-traits", @@ -2553,25 +3276,24 @@ dependencies = [ "result-like", "ruff_python_ast", "ruff_python_parser", - "ruff_source_file", "ruff_text_size", "rustix", "rustpython-codegen", "rustpython-common", "rustpython-compiler", "rustpython-compiler-core", - "rustpython-compiler-source", "rustpython-derive", "rustpython-jit", "rustpython-literal", "rustpython-sre_engine", "rustyline", - "schannel", + "scoped-tls", + "scopeguard", "serde", "static_assertions", "strum", "strum_macros", - "thiserror 2.0.12", + "thiserror 2.0.17", "thread_local", "timsort", "uname", @@ -2579,13 +3301,10 @@ dependencies = [ "unic-ucd-category", "unic-ucd-ident", "unicode-casing", - "unicode_names2", "wasm-bindgen", "which", "widestring", - "windows", - "windows-sys 0.59.0", - "winreg", + "windows-sys 0.61.2", ] [[package]] @@ -2618,17 +3337,17 @@ dependencies = [ [[package]] name = "rustversion" -version = "1.0.20" +version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eded382c5f5f786b989652c49544c4877d9f015cc22e145a5ea8ea66c2921cd2" +checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" [[package]] name = "rustyline" -version = "15.0.0" +version = "17.0.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2ee1e066dc922e513bda599c6ccb5f3bb2b0ea5870a579448f2622993f0a9a2f" +checksum = "e902948a25149d50edc1a8e0141aad50f54e22ba83ff988cf8f7c9ef07f50564" dependencies = [ - "bitflags 2.9.0", + "bitflags 2.10.0", "cfg-if", "clipboard-win", "fd-lock", @@ -2636,12 +3355,12 @@ dependencies = [ "libc", "log", "memchr", - "nix", + "nix 0.30.1", "radix_trie", "unicode-segmentation", - "unicode-width 0.2.0", + "unicode-width", "utf8parse", - "windows-sys 0.59.0", + "windows-sys 0.60.2", ] [[package]] @@ -2650,6 +3369,24 @@ version = "1.0.20" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" +[[package]] +name = "safe_arch" +version = "0.9.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "629516c85c29fe757770fa03f2074cf1eac43d44c02a3de9fc2ef7b0e207dfdd" +dependencies = [ + "bytemuck", +] + +[[package]] +name = "salsa20" +version = "0.10.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "97a22f5af31f73a954c10289c93e8a50cc23d971e80ee446f1f6f7137a088213" +dependencies = [ + "cipher", +] + [[package]] name = "same-file" version = "1.0.6" @@ -2661,70 +3398,120 @@ dependencies = [ [[package]] name = "schannel" -version = "0.1.27" +version = "0.1.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1f29ebaa345f945cec9fbbc532eb307f0fdad8161f281b6369539c8d84876b3d" +checksum = "891d81b926048e76efe18581bf793546b4c0eaf8448d72be8de2bbee5fd166e1" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] +[[package]] +name = "scoped-tls" +version = "1.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e1cf6437eb19a8f4a6cc0f7dca544973b0b78843adbfeb3683d1a94a0024a294" + [[package]] name = "scopeguard" version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "scrypt" +version = "0.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0516a385866c09368f0b5bcd1caff3366aace790fcd46e2bb032697bb172fd1f" +dependencies = [ + "pbkdf2", + "salsa20", + "sha2", +] + +[[package]] +name = "security-framework" +version = "3.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b3297343eaf830f66ede390ea39da1d462b6b0c1b000f420d0a83f898bbbe6ef" +dependencies = [ + "bitflags 2.10.0", + "core-foundation 0.10.1", + "core-foundation-sys", + "libc", + "security-framework-sys", +] + +[[package]] +name = "security-framework-sys" +version = "2.15.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cc1f0cbffaac4852523ce30d8bd3c5cdc873501d96ff467ca09b6767bb8cd5c0" +dependencies = [ + "core-foundation-sys", + "libc", +] + [[package]] name = "serde" -version = "1.0.219" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6" +checksum = "9a8e94ea7f378bd32cbbd37198a4a91436180c5bb472411e48b5ec2e2124ae9e" dependencies = [ + "serde_core", "serde_derive", ] [[package]] name = "serde-wasm-bindgen" -version = "0.3.1" +version = "0.6.5" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "618365e8e586c22123d692b72a7d791d5ee697817b65a218cdf12a98870af0f7" +checksum = "8302e169f0eddcc139c70f139d19d6467353af16f9fce27e8c30158036a1e16b" dependencies = [ - "fnv", "js-sys", "serde", "wasm-bindgen", ] +[[package]] +name = "serde_core" +version = "1.0.228" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "41d385c7d4ca58e59fc732af25c3983b67ac852c1a25000afe1175de458b67ad" +dependencies = [ + "serde_derive", +] + [[package]] name = "serde_derive" -version = "1.0.219" +version = "1.0.228" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" +checksum = "d540f220d3187173da220f885ab66608367b6574e925011a9353e4badda91d79" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn", ] [[package]] name = "serde_json" -version = "1.0.140" +version = "1.0.145" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "20068b6e96dc6c9bd23e01df8827e6c7e1f2fddd43c21810382803c136b99373" +checksum = "402a6f66d8c709116cf22f558eab210f5a50187f702eb4d7e5ef38d9a7f1c79c" dependencies = [ "itoa", "memchr", "ryu", "serde", + "serde_core", ] [[package]] name = "serde_spanned" -version = "0.6.8" +version = "1.0.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "87607cb1398ed59d48732e575a4c28a7a8ebf2454b964fe3f224f2afc07909e1" +checksum = "e24345aa0fe688594e73770a5f6d1b216508b4f93484c0026d521acd30134392" dependencies = [ - "serde", + "serde_core", ] [[package]] @@ -2738,6 +3525,17 @@ dependencies = [ "digest", ] +[[package]] +name = "sha1" +version = "0.10.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e3bf829a2d51ab4a5ddf1352d8470c140cadc8301b2ae1789db023f01cedd6ba" +dependencies = [ + "cfg-if", + "cpufeatures", + "digest", +] + [[package]] name = "sha2" version = "0.10.9" @@ -2764,14 +3562,29 @@ name = "shared-build" version = "0.2.0" source = "git+https://github.com/arihant2math/tkinter.git?tag=v0.2.0#198fc35b1f18f4eda401f97a641908f321b1403a" dependencies = [ - "bindgen", + "bindgen 0.71.1", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "signature" +version = "2.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "77549399552de45a898a580c1b41d445bf730df867cc44e6c0233bbc4b8329de" +dependencies = [ + "rand_core 0.6.4", ] [[package]] -name = "shlex" -version = "1.3.0" +name = "simd-adler32" +version = "0.3.7" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe" [[package]] name = "similar" @@ -2787,25 +3600,35 @@ checksum = "56199f7ddabf13fe5074ce809e7d3f42b42ae711800501b5b16ea82ad029c39d" [[package]] name = "smallvec" -version = "1.15.0" +version = "1.15.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8917285742e9f3e1683f0a9c4e6b57960b7314d0b08d30d1ecd426713ee2eee9" +checksum = "67b1b7a3b5fe4f1376887184045fcf45c69e92af734b7aaddc05fb777b6fbd03" [[package]] name = "socket2" -version = "0.5.9" +version = "0.6.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4f5fd57c80058a56cf5c777ab8a126398ece8e442983605d280a44ce79d0edef" +checksum = "17129e116933cf371d018bb80ae557e889637989d8638274fb25622827b03881" dependencies = [ "libc", - "windows-sys 0.52.0", + "windows-sys 0.60.2", +] + +[[package]] +name = "spki" +version = "0.7.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d91ed6c858b01f942cd56b37a94b3e0a1798290327d1236e4d9cf4eaca44d29d" +dependencies = [ + "base64ct", + "der", ] [[package]] name = "stable_deref_trait" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a8f112729512f8e442d81f95a8a7ddf2b7c6b8a1a6f509a95864142b30cab2d3" +checksum = "6ce2be8dc25455e1f91df71bfa12ad37d7af1092ae736f3a6cd0e37bc7810596" [[package]] name = "static_assertions" @@ -2815,21 +3638,20 @@ checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" [[package]] name = "strum" -version = "0.27.1" +version = "0.27.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f64def088c51c9510a8579e3c5d67c65349dcf755e5479ad3d010aa6454e2c32" +checksum = "af23d6f6c1a224baef9d3f61e287d2761385a5b88fdab4eb4c6f11aeb54c4bcf" [[package]] name = "strum_macros" -version = "0.27.1" +version = "0.27.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c77a8c5abcaf0f9ce05d62342b7d298c346515365c36b673df4ebe3ced01fde8" +checksum = "7695ce3845ea4b33927c055a39dc438a45b059f7c1b3d91d38d10355fb8cbca7" dependencies = [ "heck", "proc-macro2", "quote", - "rustversion", - "syn 2.0.101", + "syn", ] [[package]] @@ -2840,9 +3662,9 @@ checksum = "13c2bddecc57b384dee18652358fb23172facb8a2c51ccc10d74c157bdea3292" [[package]] name = "syn" -version = "1.0.109" +version = "2.0.111" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "72b64191b275b66ffe2469e8af2c1cfe3bafa67b529ead792a6d0160888b4237" +checksum = "390cc9a294ab71bdb1aa2e99d13be9c753cd2d7bd6560c77118597410c4d2e87" dependencies = [ "proc-macro2", "quote", @@ -2850,35 +3672,35 @@ dependencies = [ ] [[package]] -name = "syn" -version = "2.0.101" +name = "syn-ext" +version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ce2b7fc941b3a24138a0a7cf8e858bfc6a992e7978a068a5c760deb0ed43caf" +checksum = "b126de4ef6c2a628a68609dd00733766c3b015894698a438ebdf374933fc31d1" dependencies = [ "proc-macro2", "quote", - "unicode-ident", + "syn", ] [[package]] -name = "syn-ext" -version = "0.5.0" +name = "synstructure" +version = "0.13.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b126de4ef6c2a628a68609dd00733766c3b015894698a438ebdf374933fc31d1" +checksum = "728a70f3dbaf5bab7f0c4b1ac8d7ae5ea60a4b5549c8a5914361c99147a709d2" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn", ] [[package]] name = "system-configuration" -version = "0.6.1" +version = "0.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3c879d448e9d986b661742763247d3693ed13609438cf3d006f51f5368a5ba6b" +checksum = "a13f3d0daba03132c0aa9767f98351b3488edc2c100cda2d2ec2b04f3d8d3c8b" dependencies = [ - "bitflags 2.9.0", - "core-foundation", + "bitflags 2.10.0", + "core-foundation 0.9.4", "system-configuration-sys", ] @@ -2894,9 +3716,9 @@ dependencies = [ [[package]] name = "target-lexicon" -version = "0.13.2" +version = "0.13.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e502f78cdbb8ba4718f566c418c52bc729126ffd16baee5baa718cf25dd5a69a" +checksum = "df7f62577c25e07834649fc3b39fafdc597c0a3527dc1c60129201ccfcbaa50c" [[package]] name = "tcl-sys" @@ -2907,6 +3729,19 @@ dependencies = [ "shared-build", ] +[[package]] +name = "tempfile" +version = "3.23.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2d31c77bdf42a745371d260a26ca7163f1e0924b64afa0b688e61b5a9fa02f16" +dependencies = [ + "fastrand", + "getrandom 0.3.4", + "once_cell", + "rustix", + "windows-sys 0.61.2", +] + [[package]] name = "termios" version = "0.3.3" @@ -2933,11 +3768,11 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.12" +version = "2.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "567b8a2dae586314f7be2a752ec7474332959c6460e02bde30d702a66d488708" +checksum = "f63587ca0f12b72a0600bcba1d40081f830876000bb46dd2337a3051618f4fc8" dependencies = [ - "thiserror-impl 2.0.12", + "thiserror-impl 2.0.17", ] [[package]] @@ -2948,18 +3783,18 @@ checksum = "4fee6c4efc90059e10f81e6d42c60a18f76588c3d74cb83a0b242a2b6c7504c1" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn", ] [[package]] name = "thiserror-impl" -version = "2.0.12" +version = "2.0.17" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7f7cf42b4507d8ea322120659672cf1b9dbb93f8f2d4ecfd6e51350ff5b17a1d" +checksum = "3ff15c8ecd7de3849db632e14d18d2571fa09dfc5ed93479bc4485c7a517c913" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn", ] [[package]] @@ -2975,12 +3810,42 @@ dependencies = [ [[package]] name = "thread_local" -version = "1.1.8" +version = "1.1.9" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8b9ef9bad013ada3808854ceac7b46812a6465ba368859a37e2100283d2d719c" +checksum = "f60246a4944f24f6e018aa17cdeffb7818b76356965d03b07d6a9886e8962185" dependencies = [ "cfg-if", - "once_cell", +] + +[[package]] +name = "time" +version = "0.3.44" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "91e7d9e3bb61134e77bde20dd4825b97c010155709965fedf0f49bb138e52a9d" +dependencies = [ + "deranged", + "itoa", + "num-conv", + "powerfmt", + "serde", + "time-core", + "time-macros", +] + +[[package]] +name = "time-core" +version = "0.1.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "40868e7c1d2f0b8d73e4a8c7f0ff63af4f6d19be117e90bd73eb1d62cf831c6b" + +[[package]] +name = "time-macros" +version = "0.2.24" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "30cfb0125f12d9c277f35663a0a33f8c30190f4e4574868a330595412d34ebf3" +dependencies = [ + "num-conv", + "time-core", ] [[package]] @@ -3001,9 +3866,9 @@ dependencies = [ [[package]] name = "tinyvec" -version = "1.9.0" +version = "1.10.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "09b3661f17e86524eccd4371ab0429194e0d7c008abb45f7a7495b1719463c71" +checksum = "bfa5fdc3bce6191a1dbc8c02d5c8bffcf557bafa17c124c5264a458f1b0613fa" dependencies = [ "tinyvec_macros", ] @@ -3024,61 +3889,76 @@ dependencies = [ ] [[package]] -name = "toml" -version = "0.8.22" +name = "tls_codec" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "05ae329d1f08c4d17a59bed7ff5b5a769d062e64a62d34a3261b219e62cd5aae" +checksum = "0de2e01245e2bb89d6f05801c564fa27624dbd7b1846859876c7dad82e90bf6b" dependencies = [ - "serde", - "serde_spanned", - "toml_datetime", - "toml_edit", + "tls_codec_derive", + "zeroize", ] [[package]] -name = "toml_datetime" -version = "0.6.9" +name = "tls_codec_derive" +version = "0.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "3da5db5a963e24bc68be8b17b6fa82814bb22ee8660f192bb182771d498f09a3" +checksum = "2d2e76690929402faae40aebdda620a2c0e25dd6d3b9afe48867dfd95991f4bd" dependencies = [ - "serde", + "proc-macro2", + "quote", + "syn", ] [[package]] -name = "toml_edit" -version = "0.22.26" +name = "toml" +version = "0.9.8" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "310068873db2c5b3e7659d2cc35d21855dbafa50d1ce336397c666e3cb08137e" +checksum = "f0dc8b1fb61449e27716ec0e1bdf0f6b8f3e8f6b05391e8497b8b6d7804ea6d8" dependencies = [ "indexmap", - "serde", + "serde_core", "serde_spanned", "toml_datetime", - "toml_write", + "toml_parser", + "toml_writer", "winnow", ] [[package]] -name = "toml_write" -version = "0.1.1" +name = "toml_datetime" +version = "0.7.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bfb942dfe1d8e29a7ee7fcbde5bd2b9a25fb89aa70caea2eba3bee836ff41076" +checksum = "f2cdb639ebbc97961c51720f858597f7f24c4fc295327923af55b74c3c724533" +dependencies = [ + "serde_core", +] [[package]] -name = "twox-hash" -version = "1.6.3" +name = "toml_parser" +version = "1.0.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "97fee6b57c6a41524a810daee9286c02d7752c4253064d0b05472833a438f675" +checksum = "c0cbe268d35bdb4bb5a56a2de88d0ad0eb70af5384a99d648cd4b3d04039800e" dependencies = [ - "cfg-if", - "static_assertions", + "winnow", ] +[[package]] +name = "toml_writer" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "df8b2b54733674ad286d16267dcfc7a71ed5c776e4ac7aa3c3e2561f7c637bf2" + +[[package]] +name = "twox-hash" +version = "2.1.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9ea3136b675547379c4bd395ca6b938e5ad3c3d20fad76e7fe85f9e0d011419c" + [[package]] name = "typenum" -version = "1.18.0" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1dccffe3ce07af9386bfd29e80c0ab1a8205a2fc34e4bcd40364df902cfa8f3f" +checksum = "562d481066bde0658276a35467c4af00bdc6ee726305698a55b86e61d7ad82bb" [[package]] name = "ucd" @@ -3202,27 +4082,27 @@ dependencies = [ [[package]] name = "unicode-bidi-mirroring" -version = "0.2.0" +version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "23cb788ffebc92c5948d0e997106233eeb1d8b9512f93f41651f52b6c5f5af86" +checksum = "5dfa6e8c60bb66d49db113e0125ee8711b7647b5579dc7f5f19c42357ed039fe" [[package]] name = "unicode-casing" -version = "0.1.0" +version = "0.1.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "623f59e6af2a98bdafeb93fa277ac8e1e40440973001ca15cf4ae1541cd16d56" +checksum = "061dbb8cc7f108532b6087a0065eff575e892a4bcb503dc57323a197457cc202" [[package]] name = "unicode-ident" -version = "1.0.18" +version = "1.0.22" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" +checksum = "9312f7c4f6ff9069b165498234ce8be658059c6728633667c526e27dc2cf1df5" [[package]] name = "unicode-normalization" -version = "0.1.24" +version = "0.1.25" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "5033c97c4262335cded6d6fc3e5c18ab755e1a3dc96376350f3d8e9f009ad956" +checksum = "5fd4f6878c9cb28d874b009da9e8d183b5abc80117c40bbd187a1fde336be6e8" dependencies = [ "tinyvec", ] @@ -3235,24 +4115,28 @@ checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" [[package]] name = "unicode-width" -version = "0.1.14" +version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7dd6e30e90baa6f72411720665d41d89b9a3d039dc45b8faea1ddd07f617f6af" +checksum = "b4ac048d71ede7ee76d585517add45da530660ef4390e49b098733c6e897f254" [[package]] -name = "unicode-width" -version = "0.2.0" +name = "unicode_names2" +version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1fc81956842c57dac11422a97c3b8195a1ff727f06e85c84ed2e8aa277c9a0fd" +checksum = "d1673eca9782c84de5f81b82e4109dcfb3611c8ba0d52930ec4a9478f547b2dd" +dependencies = [ + "phf 0.11.3", + "unicode_names2_generator 1.3.0", +] [[package]] name = "unicode_names2" -version = "1.3.0" +version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "d1673eca9782c84de5f81b82e4109dcfb3611c8ba0d52930ec4a9478f547b2dd" +checksum = "d189085656ca1203291e965444e7f6a2723fbdd1dd9f34f8482e79bafd8338a0" dependencies = [ - "phf", - "unicode_names2_generator", + "phf 0.11.3", + "unicode_names2_generator 2.0.0", ] [[package]] @@ -3267,12 +4151,34 @@ dependencies = [ "rand 0.8.5", ] +[[package]] +name = "unicode_names2_generator" +version = "2.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1262662dc96937c71115228ce2e1d30f41db71a7a45d3459e98783ef94052214" +dependencies = [ + "phf_codegen", + "rand 0.8.5", +] + [[package]] name = "unindent" version = "0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3" +[[package]] +name = "untrusted" +version = "0.7.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a156c684c91ea7d62626509bce3cb4e1d9ed5c4d978f7b4352658f96a4c26b4a" + +[[package]] +name = "untrusted" +version = "0.9.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8ecb6da28b8a351d773b68d5825ac39017e680750f980f3a1a85cd8dd28a47c1" + [[package]] name = "utf8parse" version = "0.2.2" @@ -3281,11 +4187,13 @@ checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" [[package]] name = "uuid" -version = "1.16.0" +version = "1.19.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "458f7a779bf54acc9f347480ac654f68407d3aab21269a6e3c9f922acd9e2da9" +checksum = "e2e054861b4bd027cd373e18e8d8d8e6548085000e41290d95ce0c373a654b4a" dependencies = [ "atomic", + "js-sys", + "wasm-bindgen", ] [[package]] @@ -3312,50 +4220,37 @@ dependencies = [ [[package]] name = "wasi" -version = "0.11.0+wasi-snapshot-preview1" +version = "0.11.1+wasi-snapshot-preview1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9c8d87e72b64a3b4db28d11ce29237c246188f4f51057d65a7eab63b7987e423" +checksum = "ccf3ec651a847eb01de73ccad15eb7d99f80485de043efb2f370cd654f4ea44b" [[package]] -name = "wasi" -version = "0.14.2+wasi-0.2.4" +name = "wasip2" +version = "1.0.1+wasi-0.2.4" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9683f9a5a998d873c0d21fcbe3c083009670149a8fab228644b8bd36b2c48cb3" +checksum = "0562428422c63773dad2c345a1882263bbf4d65cf3f42e90921f787ef5ad58e7" dependencies = [ - "wit-bindgen-rt", + "wit-bindgen", ] [[package]] name = "wasm-bindgen" -version = "0.2.100" +version = "0.2.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1edc8929d7499fc4e8f0be2262a241556cfc54a0bea223790e71446f2aab1ef5" +checksum = "0d759f433fa64a2d763d1340820e46e111a7a5ab75f993d1852d70b03dbb80fd" dependencies = [ "cfg-if", "once_cell", "rustversion", "wasm-bindgen-macro", -] - -[[package]] -name = "wasm-bindgen-backend" -version = "0.2.100" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2f0a0651a5c2bc21487bde11ee802ccaf4c51935d0d3d42a6101f98161700bc6" -dependencies = [ - "bumpalo", - "log", - "proc-macro2", - "quote", - "syn 2.0.101", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-futures" -version = "0.4.50" +version = "0.4.56" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "555d470ec0bc3bb57890405e5d4322cc9ea83cebb085523ced7be4144dac1e61" +checksum = "836d9622d604feee9e5de25ac10e3ea5f2d65b41eac0d9ce72eb5deae707ce7c" dependencies = [ "cfg-if", "js-sys", @@ -3366,9 +4261,9 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro" -version = "0.2.100" +version = "0.2.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7fe63fc6d09ed3792bd0897b314f53de8e16568c2b3f7982f468c0bf9bd0b407" +checksum = "48cb0d2638f8baedbc542ed444afc0644a29166f1595371af4fecf8ce1e7eeb3" dependencies = [ "quote", "wasm-bindgen-macro-support", @@ -3376,65 +4271,101 @@ dependencies = [ [[package]] name = "wasm-bindgen-macro-support" -version = "0.2.100" +version = "0.2.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8ae87ea40c9f689fc23f209965b6fb8a99ad69aeeb0231408be24920604395de" +checksum = "cefb59d5cd5f92d9dcf80e4683949f15ca4b511f4ac0a6e14d4e1ac60c6ecd40" dependencies = [ + "bumpalo", "proc-macro2", "quote", - "syn 2.0.101", - "wasm-bindgen-backend", + "syn", "wasm-bindgen-shared", ] [[package]] name = "wasm-bindgen-shared" -version = "0.2.100" +version = "0.2.106" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1a05d73b933a847d6cccdda8f838a22ff101ad9bf93e33684f39c1f5f0eece3d" +checksum = "cbc538057e648b67f72a982e708d485b2efa771e1ac05fec311f9f63e5800db4" dependencies = [ "unicode-ident", ] [[package]] -name = "wasmtime-jit-icache-coherence" -version = "32.0.0" +name = "wasmtime-internal-jit-icache-coherence" +version = "39.0.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "eb399eaabd7594f695e1159d236bf40ef55babcb3af97f97c027864ed2104db6" +checksum = "b97ccd36e25390258ce6720add639ffe5a7d81a5c904350aa08f5bbc60433d22" dependencies = [ "anyhow", "cfg-if", "libc", - "windows-sys 0.59.0", + "windows-sys 0.60.2", +] + +[[package]] +name = "wasmtime-internal-math" +version = "39.0.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cd1b856e1bbf0230ab560ba4204e944b141971adc4e6cdf3feb6979c1a7b7953" +dependencies = [ + "libm", ] [[package]] name = "web-sys" -version = "0.3.77" +version = "0.3.83" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33b6dd2ef9186f1f2072e409e99cd22a975331a6b3591b12c764e0e55c60d5d2" +checksum = "9b32828d774c412041098d182a8b38b16ea816958e07cf40eec2bc080ae137ac" dependencies = [ "js-sys", "wasm-bindgen", ] +[[package]] +name = "webpki-root-certs" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ee3e3b5f5e80bc89f30ce8d0343bf4e5f12341c51f3e26cbeecbc7c85443e85b" +dependencies = [ + "rustls-pki-types", +] + +[[package]] +name = "webpki-roots" +version = "1.0.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b2878ef029c47c6e8cf779119f20fcf52bde7ad42a731b2a304bc221df17571e" +dependencies = [ + "rustls-pki-types", +] + [[package]] name = "which" -version = "7.0.3" +version = "8.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "24d643ce3fd3e5b54854602a080f34fb10ab75e0b813ee32d00ca2b44fa74762" +checksum = "d3fabb953106c3c8eea8306e4393700d7657561cb43122571b172bbfb7c7ba1d" dependencies = [ - "either", "env_home", "rustix", "winsafe", ] +[[package]] +name = "wide" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "13ca908d26e4786149c48efcf6c0ea09ab0e06d1fe3c17dc1b4b0f1ca4a7e788" +dependencies = [ + "bytemuck", + "safe_arch", +] + [[package]] name = "widestring" -version = "1.2.0" +version = "1.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dd7cf3379ca1aac9eea11fba24fd7e315d621f8dfe35c8d7d2be8b793726e07d" +checksum = "72069c3113ab32ab29e5584db3c6ec55d416895e60715417b5b883a357c3e471" [[package]] name = "winapi" @@ -3454,11 +4385,11 @@ checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6" [[package]] name = "winapi-util" -version = "0.1.9" +version = "0.1.11" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" +checksum = "c2a7b1c03c876122aa43f3020e6c3c3ee5c05081c9a00739faf7503aeba10d22" dependencies = [ - "windows-sys 0.59.0", + "windows-sys 0.61.2", ] [[package]] @@ -3467,30 +4398,11 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f" -[[package]] -name = "windows" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e48a53791691ab099e5e2ad123536d0fff50652600abaf43bbf952894110d0be" -dependencies = [ - "windows-core 0.52.0", - "windows-targets 0.52.6", -] - -[[package]] -name = "windows-core" -version = "0.52.0" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "33ab640c8d7e35bf8ba19b884ba838ceb4fba93a4e8c65a9059d08afcfc683d9" -dependencies = [ - "windows-targets 0.52.6", -] - [[package]] name = "windows-core" -version = "0.61.0" +version = "0.62.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4763c1de310c86d75a878046489e2e5ba02c649d185f21c67d4cf8a56d098980" +checksum = "b8e83a14d34d0623b51dce9581199302a221863196a1dde71a7663a4c2be9deb" dependencies = [ "windows-implement", "windows-interface", @@ -3501,57 +4413,57 @@ dependencies = [ [[package]] name = "windows-implement" -version = "0.60.0" +version = "0.60.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a47fddd13af08290e67f4acabf4b459f647552718f683a7b415d290ac744a836" +checksum = "053e2e040ab57b9dc951b72c264860db7eb3b0200ba345b4e4c3b14f67855ddf" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn", ] [[package]] name = "windows-interface" -version = "0.59.1" +version = "0.59.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "bd9211b69f8dcdfa817bfd14bf1c97c9188afa36f4750130fcdf3f400eca9fa8" +checksum = "3f316c4a2570ba26bbec722032c4099d8c8bc095efccdc15688708623367e358" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn", ] [[package]] name = "windows-link" -version = "0.1.1" +version = "0.2.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "76840935b766e1b0a05c0066835fb9ec80071d4c09a16f6bd5f7e655e3c14c38" +checksum = "f0805222e57f7521d6a62e36fa9163bc891acd422f971defe97d64e70d0a4fe5" [[package]] name = "windows-result" -version = "0.3.2" +version = "0.4.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c64fd11a4fd95df68efcfee5f44a294fe71b8bc6a91993e2791938abcc712252" +checksum = "7781fa89eaf60850ac3d2da7af8e5242a5ea78d1a11c49bf2910bb5a73853eb5" dependencies = [ "windows-link", ] [[package]] name = "windows-strings" -version = "0.4.0" +version = "0.5.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7a2ba9642430ee452d5a7aa78d72907ebe8cfda358e8cb7918a2050581322f97" +checksum = "7837d08f69c77cf6b07689544538e017c1bfcf57e34b4c0ff58e6c2cd3b37091" dependencies = [ "windows-link", ] [[package]] name = "windows-sys" -version = "0.48.0" +version = "0.45.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "677d2418bec65e3338edb076e806bc1ec15693c5d0104683f2efe857f61056a9" +checksum = "75283be5efb2831d37ea142365f009c02ec203cd29a3ebecbc093d52315b66d0" dependencies = [ - "windows-targets 0.48.5", + "windows-targets 0.42.2", ] [[package]] @@ -3572,19 +4484,37 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "windows-sys" +version = "0.60.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2f500e4d28234f72040990ec9d39e3a6b950f9f22d3dba18416c35882612bcb" +dependencies = [ + "windows-targets 0.53.5", +] + +[[package]] +name = "windows-sys" +version = "0.61.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ae137229bcbd6cdf0f7b80a31df61766145077ddf49416a728b02cb3921ff3fc" +dependencies = [ + "windows-link", +] + [[package]] name = "windows-targets" -version = "0.48.5" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "9a2fa6e2155d7247be68c096456083145c183cbbbc2764150dda45a87197940c" +checksum = "8e5180c00cd44c9b1c88adb3693291f1cd93605ded80c250a75d472756b4d071" dependencies = [ - "windows_aarch64_gnullvm 0.48.5", - "windows_aarch64_msvc 0.48.5", - "windows_i686_gnu 0.48.5", - "windows_i686_msvc 0.48.5", - "windows_x86_64_gnu 0.48.5", - "windows_x86_64_gnullvm 0.48.5", - "windows_x86_64_msvc 0.48.5", + "windows_aarch64_gnullvm 0.42.2", + "windows_aarch64_msvc 0.42.2", + "windows_i686_gnu 0.42.2", + "windows_i686_msvc 0.42.2", + "windows_x86_64_gnu 0.42.2", + "windows_x86_64_gnullvm 0.42.2", + "windows_x86_64_msvc 0.42.2", ] [[package]] @@ -3596,18 +4526,35 @@ dependencies = [ "windows_aarch64_gnullvm 0.52.6", "windows_aarch64_msvc 0.52.6", "windows_i686_gnu 0.52.6", - "windows_i686_gnullvm", + "windows_i686_gnullvm 0.52.6", "windows_i686_msvc 0.52.6", "windows_x86_64_gnu 0.52.6", "windows_x86_64_gnullvm 0.52.6", "windows_x86_64_msvc 0.52.6", ] +[[package]] +name = "windows-targets" +version = "0.53.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4945f9f551b88e0d65f3db0bc25c33b8acea4d9e41163edf90dcd0b19f9069f3" +dependencies = [ + "windows-link", + "windows_aarch64_gnullvm 0.53.1", + "windows_aarch64_msvc 0.53.1", + "windows_i686_gnu 0.53.1", + "windows_i686_gnullvm 0.53.1", + "windows_i686_msvc 0.53.1", + "windows_x86_64_gnu 0.53.1", + "windows_x86_64_gnullvm 0.53.1", + "windows_x86_64_msvc 0.53.1", +] + [[package]] name = "windows_aarch64_gnullvm" -version = "0.48.5" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "2b38e32f0abccf9987a4e3079dfb67dcd799fb61361e53e2882c3cbaf0d905d8" +checksum = "597a5118570b68bc08d8d59125332c54f1ba9d9adeedeef5b99b02ba2b0698f8" [[package]] name = "windows_aarch64_gnullvm" @@ -3615,11 +4562,17 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "32a4622180e7a0ec044bb555404c800bc9fd9ec262ec147edd5989ccd0c02cd3" +[[package]] +name = "windows_aarch64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a9d8416fa8b42f5c947f8482c43e7d89e73a173cead56d044f6a56104a6d1b53" + [[package]] name = "windows_aarch64_msvc" -version = "0.48.5" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dc35310971f3b2dbbf3f0690a219f40e2d9afcf64f9ab7cc1be722937c26b4bc" +checksum = "e08e8864a60f06ef0d0ff4ba04124db8b0fb3be5776a5cd47641e942e58c4d43" [[package]] name = "windows_aarch64_msvc" @@ -3627,11 +4580,17 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "09ec2a7bb152e2252b53fa7803150007879548bc709c039df7627cabbd05d469" +[[package]] +name = "windows_aarch64_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b9d782e804c2f632e395708e99a94275910eb9100b2114651e04744e9b125006" + [[package]] name = "windows_i686_gnu" -version = "0.48.5" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a75915e7def60c94dcef72200b9a8e58e5091744960da64ec734a6c6e9b3743e" +checksum = "c61d927d8da41da96a81f029489353e68739737d3beca43145c8afec9a31a84f" [[package]] name = "windows_i686_gnu" @@ -3639,17 +4598,29 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8e9b5ad5ab802e97eb8e295ac6720e509ee4c243f69d781394014ebfe8bbfa0b" +[[package]] +name = "windows_i686_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "960e6da069d81e09becb0ca57a65220ddff016ff2d6af6a223cf372a506593a3" + [[package]] name = "windows_i686_gnullvm" version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0eee52d38c090b3caa76c563b86c3a4bd71ef1a819287c19d586d7334ae8ed66" +[[package]] +name = "windows_i686_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fa7359d10048f68ab8b09fa71c3daccfb0e9b559aed648a8f95469c27057180c" + [[package]] name = "windows_i686_msvc" -version = "0.48.5" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "8f55c233f70c4b27f66c523580f78f1004e8b5a8b659e05a4eb49d4166cca406" +checksum = "44d840b6ec649f480a41c8d80f9c65108b92d89345dd94027bfe06ac444d1060" [[package]] name = "windows_i686_msvc" @@ -3657,11 +4628,17 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "240948bc05c5e7c6dabba28bf89d89ffce3e303022809e73deaefe4f6ec56c66" +[[package]] +name = "windows_i686_msvc" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e7ac75179f18232fe9c285163565a57ef8d3c89254a30685b57d83a38d326c2" + [[package]] name = "windows_x86_64_gnu" -version = "0.48.5" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "53d40abd2583d23e4718fddf1ebec84dbff8381c07cae67ff7768bbf19c6718e" +checksum = "8de912b8b8feb55c064867cf047dda097f92d51efad5b491dfb98f6bbb70cb36" [[package]] name = "windows_x86_64_gnu" @@ -3669,11 +4646,17 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "147a5c80aabfbf0c7d901cb5895d1de30ef2907eb21fbbab29ca94c5b08b1a78" +[[package]] +name = "windows_x86_64_gnu" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9c3842cdd74a865a8066ab39c8a7a473c0778a3f29370b5fd6b4b9aa7df4a499" + [[package]] name = "windows_x86_64_gnullvm" -version = "0.48.5" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0b7b52767868a23d5bab768e390dc5f5c55825b6d30b86c844ff2dc7414044cc" +checksum = "26d41b46a36d453748aedef1486d5c7a85db22e56aff34643984ea85514e94a3" [[package]] name = "windows_x86_64_gnullvm" @@ -3681,11 +4664,17 @@ version = "0.52.6" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "24d5b23dc417412679681396f2b49f3de8c1473deb516bd34410872eff51ed0d" +[[package]] +name = "windows_x86_64_gnullvm" +version = "0.53.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0ffa179e2d07eee8ad8f57493436566c7cc30ac536a3379fdf008f47f6bb7ae1" + [[package]] name = "windows_x86_64_msvc" -version = "0.48.5" +version = "0.42.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ed94fce61571a4006852b7389a063ab983c02eb1bb37b47f8272ce92d06d9538" +checksum = "9aec5da331524158c6d1a4ac0ab1541149c0b9505fde06423b02f5ef0106b9f0" [[package]] name = "windows_x86_64_msvc" @@ -3694,29 +4683,22 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "589f6da84c646204747d1270a2a5661ea66ed1cced2631d546fdfb155959f9ec" [[package]] -name = "winnow" -version = "0.7.10" +name = "windows_x86_64_msvc" +version = "0.53.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "c06928c8748d81b05c9be96aad92e1b6ff01833332f281e8cfca3be4b35fc9ec" -dependencies = [ - "memchr", -] +checksum = "d6bbff5f0aada427a1e5a6da5f1f98158182f26556f345ac9e04d36d0ebed650" [[package]] -name = "winreg" -version = "0.55.0" +name = "winnow" +version = "0.7.14" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "cb5a765337c50e9ec252c2069be9bf91c7df47afb103b642ba3a53bf8101be97" -dependencies = [ - "cfg-if", - "windows-sys 0.59.0", -] +checksum = "5a5364e9d77fcdeeaa6062ced926ee3381faa2ee02d3eb83a5c27a8825540829" [[package]] name = "winresource" -version = "0.1.20" +version = "0.1.28" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ba4a67c78ee5782c0c1cb41bebc7e12c6e79644daa1650ebbc1de5d5b08593f7" +checksum = "6b021990998587d4438bb672b5c5f034cbc927f51b45e3807ab7323645ef4899" dependencies = [ "toml", "version_check", @@ -3729,19 +4711,47 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d135d17ab770252ad95e9a872d365cf3090e3be864a34ab46f48555993efc904" [[package]] -name = "wit-bindgen-rt" -version = "0.39.0" +name = "wit-bindgen" +version = "0.46.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f17a85883d4e6d00e8a97c586de764dabcc06133f7f1d55dce5cdc070ad7fe59" + +[[package]] +name = "x509-cert" +version = "0.2.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1301e935010a701ae5f8655edc0ad17c44bad3ac5ce8c39185f75453b720ae94" +dependencies = [ + "const-oid", + "der", + "sha1", + "signature", + "spki", + "tls_codec", +] + +[[package]] +name = "x509-parser" +version = "0.18.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "6f42320e61fe2cfd34354ecb597f86f413484a798ba44a8ca1165c58d42da6c1" +checksum = "eb3e137310115a65136898d2079f003ce33331a6c4b0d51f1531d1be082b6425" dependencies = [ - "bitflags 2.9.0", + "asn1-rs", + "data-encoding", + "der-parser", + "lazy_static 1.5.0", + "nom", + "oid-registry", + "rusticata-macros", + "thiserror 2.0.17", + "time", ] [[package]] -name = "xml-rs" -version = "0.8.26" +name = "xml" +version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a62ce76d9b56901b19a74f19431b0d8b3bc7ca4ad685a746dfd78ca8f4fc6bda" +checksum = "2df5825faced2427b2da74d9100f1e2e93c533fff063506a81ede1cf517b2e7e" [[package]] name = "xz2" @@ -3754,26 +4764,46 @@ dependencies = [ [[package]] name = "zerocopy" -version = "0.8.25" +version = "0.8.30" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a1702d9583232ddb9174e01bb7c15a2ab8fb1bc6f227aa1233858c351a3ba0cb" +checksum = "4ea879c944afe8a2b25fef16bb4ba234f47c694565e97383b36f3a878219065c" dependencies = [ "zerocopy-derive", ] [[package]] name = "zerocopy-derive" -version = "0.8.25" +version = "0.8.30" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "cf955aa904d6040f70dc8e9384444cb1030aed272ba3cb09bbc4ab9e7c1f34f5" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "zeroize" +version = "1.8.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" +dependencies = [ + "zeroize_derive", +] + +[[package]] +name = "zeroize_derive" +version = "1.4.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "28a6e20d751156648aa063f3800b706ee209a32c0b4d9f24be3d980b01be55ef" +checksum = "ce36e65b0d2999d2aafac989fb249189a141aee1f53c612c1f37d72631959f69" dependencies = [ "proc-macro2", "quote", - "syn 2.0.101", + "syn", ] [[package]] name = "zlib-rs" -version = "0.5.0" +version = "0.5.3" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "868b928d7949e09af2f6086dfc1e01936064cc7a819253bce650d4e2a2d63ba8" +checksum = "36134c44663532e6519d7a6dfdbbe06f6f8192bde8ae9ed076e9b213f0e31df7" diff --git a/Cargo.toml b/Cargo.toml index 1fdc77d261..44f9d3190f 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -10,7 +10,7 @@ repository.workspace = true license.workspace = true [features] -default = ["threading", "stdlib", "stdio", "importlib"] +default = ["threading", "stdlib", "stdio", "importlib", "ssl-rustls"] importlib = ["rustpython-vm/importlib"] encodings = ["rustpython-vm/encodings"] stdio = ["rustpython-vm/stdio"] @@ -20,8 +20,10 @@ freeze-stdlib = ["stdlib", "rustpython-vm/freeze-stdlib", "rustpython-pylib?/fre jit = ["rustpython-vm/jit"] threading = ["rustpython-vm/threading", "rustpython-stdlib/threading"] sqlite = ["rustpython-stdlib/sqlite"] -ssl = ["rustpython-stdlib/ssl"] -ssl-vendor = ["ssl", "rustpython-stdlib/ssl-vendor"] +ssl = [] +ssl-rustls = ["ssl", "rustpython-stdlib/ssl-rustls"] +ssl-openssl = ["ssl", "rustpython-stdlib/ssl-openssl"] +ssl-vendor = ["ssl-openssl", "rustpython-stdlib/ssl-vendor"] tkinter = ["rustpython-stdlib/tkinter"] [build-dependencies] @@ -51,7 +53,7 @@ rustyline = { workspace = true } [dev-dependencies] criterion = { workspace = true } -pyo3 = { version = "0.24", features = ["auto-initialize"] } +pyo3 = { version = "0.27", features = ["auto-initialize"] } [[bench]] name = "execution" @@ -73,6 +75,10 @@ opt-level = 3 # https://github.com/rust-lang/rust/issues/92869 # lto = "thin" +# Doesn't change often +[profile.release.package.rustpython-doc] +codegen-units = 1 + [profile.bench] lto = "thin" codegen-units = 1 @@ -82,7 +88,6 @@ opt-level = 3 lto = "thin" [patch.crates-io] -radium = { version = "1.1.0", git = "https://github.com/youknowone/ferrilab", branch = "fix-nightly" } # REDOX START, Uncomment when you want to compile/check with redoxer # REDOX END @@ -91,7 +96,7 @@ radium = { version = "1.1.0", git = "https://github.com/youknowone/ferrilab", br git = "https://github.com/microsoft/vcpkg" # The revision of the vcpkg repository to use # https://github.com/microsoft/vcpkg/tags -rev = "2024.02.14" +rev = "2025.09.17" [package.metadata.vcpkg.target] x86_64-pc-windows-msvc = { triplet = "x64-windows-static-md", dev-dependencies = ["openssl" ] } @@ -118,66 +123,67 @@ template = "installer-config/installer.wxs" [workspace] resolver = "2" members = [ - "compiler", "compiler/core", "compiler/codegen", "compiler/literal", "compiler/source", - ".", "common", "derive", "jit", "vm", "vm/sre_engine", "pylib", "stdlib", "derive-impl", "wtf8", - "wasm/lib", + ".", + "crates/*", ] [workspace.package] version = "0.4.0" authors = ["RustPython Team"] edition = "2024" -rust-version = "1.85.0" +rust-version = "1.89.0" repository = "https://github.com/RustPython/RustPython" license = "MIT" [workspace.dependencies] -rustpython-compiler-source = { path = "compiler/source" } -rustpython-compiler-core = { path = "compiler/core", version = "0.4.0" } -rustpython-compiler = { path = "compiler", version = "0.4.0" } -rustpython-codegen = { path = "compiler/codegen", version = "0.4.0" } -rustpython-common = { path = "common", version = "0.4.0" } -rustpython-derive = { path = "derive", version = "0.4.0" } -rustpython-derive-impl = { path = "derive-impl", version = "0.4.0" } -rustpython-jit = { path = "jit", version = "0.4.0" } -rustpython-literal = { path = "compiler/literal", version = "0.4.0" } -rustpython-vm = { path = "vm", default-features = false, version = "0.4.0" } -rustpython-pylib = { path = "pylib", version = "0.4.0" } -rustpython-stdlib = { path = "stdlib", default-features = false, version = "0.4.0" } -rustpython-sre_engine = { path = "vm/sre_engine", version = "0.4.0" } -rustpython-wtf8 = { path = "wtf8", version = "0.4.0" } -rustpython-doc = { git = "https://github.com/RustPython/__doc__", tag = "0.3.0", version = "0.3.0" } - -ruff_python_parser = { git = "https://github.com/astral-sh/ruff.git", tag = "0.11.0" } -ruff_python_ast = { git = "https://github.com/astral-sh/ruff.git", tag = "0.11.0" } -ruff_text_size = { git = "https://github.com/astral-sh/ruff.git", tag = "0.11.0" } -ruff_source_file = { git = "https://github.com/astral-sh/ruff.git", tag = "0.11.0" } - -ahash = "0.8.11" +rustpython-compiler-core = { path = "crates/compiler-core", version = "0.4.0" } +rustpython-compiler = { path = "crates/compiler", version = "0.4.0" } +rustpython-codegen = { path = "crates/codegen", version = "0.4.0" } +rustpython-common = { path = "crates/common", version = "0.4.0" } +rustpython-derive = { path = "crates/derive", version = "0.4.0" } +rustpython-derive-impl = { path = "crates/derive-impl", version = "0.4.0" } +rustpython-jit = { path = "crates/jit", version = "0.4.0" } +rustpython-literal = { path = "crates/literal", version = "0.4.0" } +rustpython-vm = { path = "crates/vm", default-features = false, version = "0.4.0" } +rustpython-pylib = { path = "crates/pylib", version = "0.4.0" } +rustpython-stdlib = { path = "crates/stdlib", default-features = false, version = "0.4.0" } +rustpython-sre_engine = { path = "crates/sre_engine", version = "0.4.0" } +rustpython-wtf8 = { path = "crates/wtf8", version = "0.4.0" } +rustpython-doc = { path = "crates/doc", version = "0.4.0" } + +# Ruff tag 0.14.1 is based on commit 2bffef59665ce7d2630dfd72ee99846663660db8 +# at the time of this capture. We use the commit hash to ensure reproducible builds. +ruff_python_parser = { git = "https://github.com/astral-sh/ruff.git", rev = "2bffef59665ce7d2630dfd72ee99846663660db8" } +ruff_python_ast = { git = "https://github.com/astral-sh/ruff.git", rev = "2bffef59665ce7d2630dfd72ee99846663660db8" } +ruff_text_size = { git = "https://github.com/astral-sh/ruff.git", rev = "2bffef59665ce7d2630dfd72ee99846663660db8" } +ruff_source_file = { git = "https://github.com/astral-sh/ruff.git", rev = "2bffef59665ce7d2630dfd72ee99846663660db8" } + +phf = { version = "0.13.1", default-features = false, features = ["macros"]} +ahash = "0.8.12" ascii = "1.1" -bitflags = "2.4.2" +bitflags = "2.9.4" bstr = "1" cfg-if = "1.0" -chrono = "0.4.39" +chrono = { version = "0.4.42", default-features = false, features = ["clock", "oldtime", "std"] } constant_time_eq = "0.4" -criterion = { version = "0.5", features = ["html_reports"] } +criterion = { version = "0.8", features = ["html_reports"] } crossbeam-utils = "0.8.21" flame = "0.2.2" getrandom = { version = "0.3", features = ["std"] } glob = "0.3" hex = "0.4.3" -indexmap = { version = "2.2.6", features = ["std"] } -insta = "1.42" +indexmap = { version = "2.11.3", features = ["std"] } +insta = "1.45" itertools = "0.14.0" is-macro = "0.3.7" -junction = "1.2.0" -libc = "0.2.169" -libffi = "4.1" -log = "0.4.27" -nix = { version = "0.29", features = ["fs", "user", "process", "term", "time", "signal", "ioctl", "socket", "sched", "zerocopy", "dir", "hostname", "net", "poll"] } -malachite-bigint = "0.6" -malachite-q = "0.6" -malachite-base = "0.6" +junction = "1.3.0" +libc = "0.2.178" +libffi = "5" +log = "0.4.29" +nix = { version = "0.30", features = ["fs", "user", "process", "term", "time", "signal", "ioctl", "socket", "sched", "zerocopy", "dir", "hostname", "net", "poll"] } +malachite-bigint = "0.8" +malachite-q = "0.8" +malachite-base = "0.8" memchr = "2.7.4" num-complex = "0.4.6" num-integer = "0.1.46" @@ -190,31 +196,33 @@ paste = "1.0.15" proc-macro2 = "1.0.93" pymath = "0.0.2" quote = "1.0.38" -radium = "1.1" +radium = "1.1.1" rand = "0.9" rand_core = { version = "0.9", features = ["os_rng"] } rustix = { version = "1.0", features = ["event"] } -rustyline = "15.0.0" -serde = { version = "1.0.133", default-features = false } -schannel = "0.1.27" +rustyline = "17.0.1" +serde = { version = "1.0.225", default-features = false } +schannel = "0.1.28" +scoped-tls = "1" +scopeguard = "1" static_assertions = "1.1" strum = "0.27" strum_macros = "0.27" syn = "2" thiserror = "2.0" -thread_local = "1.1.8" -unicode-casing = "0.1.0" +thread_local = "1.1.9" +unicode-casing = "0.1.1" unic-char-property = "0.9.0" unic-normal = "0.9.0" unic-ucd-age = "0.9.0" unic-ucd-bidi = "0.9.0" unic-ucd-category = "0.9.0" unic-ucd-ident = "0.9.0" -unicode_names2 = "1.3.0" -unicode-bidi-mirroring = "0.2" -widestring = "1.1.0" -windows-sys = "0.59.0" -wasm-bindgen = "0.2.100" +unicode_names2 = "2.0.0" +unicode-bidi-mirroring = "0.4" +widestring = "1.2.0" +windows-sys = "0.61.2" +wasm-bindgen = "0.2.106" # Lints diff --git a/DEVELOPMENT.md b/DEVELOPMENT.md index aa7d99eef3..d5c675faca 100644 --- a/DEVELOPMENT.md +++ b/DEVELOPMENT.md @@ -130,7 +130,8 @@ repository's structure: - `stdlib`: Standard library parts implemented in rust. - `src`: using the other subcrates to bring rustpython to life. - `wasm`: Binary crate and resources for WebAssembly build -- `extra_tests`: extra integration test snippets as a supplement to `Lib/test` +- `extra_tests`: extra integration test snippets as a supplement to `Lib/test`. + Add new RustPython-only regression tests here; do not place new tests under `Lib/test`. ## Understanding Internals diff --git a/Lib/_android_support.py b/Lib/_android_support.py index ae506f6a4b..a439d03a14 100644 --- a/Lib/_android_support.py +++ b/Lib/_android_support.py @@ -29,15 +29,19 @@ def init_streams(android_log_write, stdout_prio, stderr_prio): global logcat logcat = Logcat(android_log_write) - - sys.stdout = TextLogStream( - stdout_prio, "python.stdout", sys.stdout.fileno()) - sys.stderr = TextLogStream( - stderr_prio, "python.stderr", sys.stderr.fileno()) + sys.stdout = TextLogStream(stdout_prio, "python.stdout", sys.stdout) + sys.stderr = TextLogStream(stderr_prio, "python.stderr", sys.stderr) class TextLogStream(io.TextIOWrapper): - def __init__(self, prio, tag, fileno=None, **kwargs): + def __init__(self, prio, tag, original=None, **kwargs): + # Respect the -u option. + if original: + kwargs.setdefault("write_through", original.write_through) + fileno = original.fileno() + else: + fileno = None + # The default is surrogateescape for stdout and backslashreplace for # stderr, but in the context of an Android log, readability is more # important than reversibility. diff --git a/Lib/_collections_abc.py b/Lib/_collections_abc.py index de624f2e54..e02fc22738 100644 --- a/Lib/_collections_abc.py +++ b/Lib/_collections_abc.py @@ -512,7 +512,7 @@ def __getitem__(self, item): new_args = (t_args, t_result) return _CallableGenericAlias(Callable, tuple(new_args)) - # TODO: RUSTPYTHON patch for common call + # TODO: RUSTPYTHON; patch for common call def __or__(self, other): super().__or__(other) @@ -1087,7 +1087,7 @@ def __new__(cls, name, bases, namespace, **kwargs): warnings._deprecated( "collections.abc.ByteString", - remove=(3, 14), + remove=(3, 17), ) return super().__new__(cls, name, bases, namespace, **kwargs) @@ -1096,14 +1096,18 @@ def __instancecheck__(cls, instance): warnings._deprecated( "collections.abc.ByteString", - remove=(3, 14), + remove=(3, 17), ) return super().__instancecheck__(instance) class ByteString(Sequence, metaclass=_DeprecateByteStringMeta): - """This unifies bytes and bytearray. + """Deprecated ABC serving as a common supertype of ``bytes`` and ``bytearray``. - XXX Should add all their methods. + This ABC is scheduled for removal in Python 3.17. + Use ``isinstance(obj, collections.abc.Buffer)`` to test if ``obj`` + implements the buffer protocol at runtime. For use in type annotations, + either use ``Buffer`` or a union that explicitly specifies the types your + code supports (e.g., ``bytes | bytearray | memoryview``). """ __slots__ = () diff --git a/Lib/_colorize.py b/Lib/_colorize.py index 70acfd4ad0..9eb6f0933b 100644 --- a/Lib/_colorize.py +++ b/Lib/_colorize.py @@ -1,21 +1,64 @@ +from __future__ import annotations import io import os import sys COLORIZE = True +# types +if False: + from typing import IO + class ANSIColors: - BOLD_GREEN = "\x1b[1;32m" - BOLD_MAGENTA = "\x1b[1;35m" - BOLD_RED = "\x1b[1;31m" + RESET = "\x1b[0m" + + BLACK = "\x1b[30m" + BLUE = "\x1b[34m" + CYAN = "\x1b[36m" GREEN = "\x1b[32m" - GREY = "\x1b[90m" MAGENTA = "\x1b[35m" RED = "\x1b[31m" - RESET = "\x1b[0m" + WHITE = "\x1b[37m" # more like LIGHT GRAY YELLOW = "\x1b[33m" + BOLD_BLACK = "\x1b[1;30m" # DARK GRAY + BOLD_BLUE = "\x1b[1;34m" + BOLD_CYAN = "\x1b[1;36m" + BOLD_GREEN = "\x1b[1;32m" + BOLD_MAGENTA = "\x1b[1;35m" + BOLD_RED = "\x1b[1;31m" + BOLD_WHITE = "\x1b[1;37m" # actual WHITE + BOLD_YELLOW = "\x1b[1;33m" + + # intense = like bold but without being bold + INTENSE_BLACK = "\x1b[90m" + INTENSE_BLUE = "\x1b[94m" + INTENSE_CYAN = "\x1b[96m" + INTENSE_GREEN = "\x1b[92m" + INTENSE_MAGENTA = "\x1b[95m" + INTENSE_RED = "\x1b[91m" + INTENSE_WHITE = "\x1b[97m" + INTENSE_YELLOW = "\x1b[93m" + + BACKGROUND_BLACK = "\x1b[40m" + BACKGROUND_BLUE = "\x1b[44m" + BACKGROUND_CYAN = "\x1b[46m" + BACKGROUND_GREEN = "\x1b[42m" + BACKGROUND_MAGENTA = "\x1b[45m" + BACKGROUND_RED = "\x1b[41m" + BACKGROUND_WHITE = "\x1b[47m" + BACKGROUND_YELLOW = "\x1b[43m" + + INTENSE_BACKGROUND_BLACK = "\x1b[100m" + INTENSE_BACKGROUND_BLUE = "\x1b[104m" + INTENSE_BACKGROUND_CYAN = "\x1b[106m" + INTENSE_BACKGROUND_GREEN = "\x1b[102m" + INTENSE_BACKGROUND_MAGENTA = "\x1b[105m" + INTENSE_BACKGROUND_RED = "\x1b[101m" + INTENSE_BACKGROUND_WHITE = "\x1b[107m" + INTENSE_BACKGROUND_YELLOW = "\x1b[103m" + NoColors = ANSIColors() @@ -24,14 +67,16 @@ class ANSIColors: setattr(NoColors, attr, "") -def get_colors(colorize: bool = False, *, file=None) -> ANSIColors: +def get_colors( + colorize: bool = False, *, file: IO[str] | IO[bytes] | None = None +) -> ANSIColors: if colorize or can_colorize(file=file): return ANSIColors() else: return NoColors -def can_colorize(*, file=None) -> bool: +def can_colorize(*, file: IO[str] | IO[bytes] | None = None) -> bool: if file is None: file = sys.stdout @@ -64,4 +109,4 @@ def can_colorize(*, file=None) -> bool: try: return os.isatty(file.fileno()) except io.UnsupportedOperation: - return file.isatty() + return hasattr(file, "isatty") and file.isatty() diff --git a/Lib/_opcode_metadata.py b/Lib/_opcode_metadata.py new file mode 100644 index 0000000000..b3d7b8103e --- /dev/null +++ b/Lib/_opcode_metadata.py @@ -0,0 +1,343 @@ +# This file is generated by Tools/cases_generator/py_metadata_generator.py +# from: +# Python/bytecodes.c +# Do not edit! +_specializations = { + "RESUME": [ + "RESUME_CHECK", + ], + "TO_BOOL": [ + "TO_BOOL_ALWAYS_TRUE", + "TO_BOOL_BOOL", + "TO_BOOL_INT", + "TO_BOOL_LIST", + "TO_BOOL_NONE", + "TO_BOOL_STR", + ], + "BINARY_OP": [ + "BINARY_OP_MULTIPLY_INT", + "BINARY_OP_ADD_INT", + "BINARY_OP_SUBTRACT_INT", + "BINARY_OP_MULTIPLY_FLOAT", + "BINARY_OP_ADD_FLOAT", + "BINARY_OP_SUBTRACT_FLOAT", + "BINARY_OP_ADD_UNICODE", + "BINARY_OP_INPLACE_ADD_UNICODE", + ], + "BINARY_SUBSCR": [ + "BINARY_SUBSCR_DICT", + "BINARY_SUBSCR_GETITEM", + "BINARY_SUBSCR_LIST_INT", + "BINARY_SUBSCR_STR_INT", + "BINARY_SUBSCR_TUPLE_INT", + ], + "STORE_SUBSCR": [ + "STORE_SUBSCR_DICT", + "STORE_SUBSCR_LIST_INT", + ], + "SEND": [ + "SEND_GEN", + ], + "UNPACK_SEQUENCE": [ + "UNPACK_SEQUENCE_TWO_TUPLE", + "UNPACK_SEQUENCE_TUPLE", + "UNPACK_SEQUENCE_LIST", + ], + "STORE_ATTR": [ + "STORE_ATTR_INSTANCE_VALUE", + "STORE_ATTR_SLOT", + "STORE_ATTR_WITH_HINT", + ], + "LOAD_GLOBAL": [ + "LOAD_GLOBAL_MODULE", + "LOAD_GLOBAL_BUILTIN", + ], + "LOAD_SUPER_ATTR": [ + "LOAD_SUPER_ATTR_ATTR", + "LOAD_SUPER_ATTR_METHOD", + ], + "LOAD_ATTR": [ + "LOAD_ATTR_INSTANCE_VALUE", + "LOAD_ATTR_MODULE", + "LOAD_ATTR_WITH_HINT", + "LOAD_ATTR_SLOT", + "LOAD_ATTR_CLASS", + "LOAD_ATTR_PROPERTY", + "LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN", + "LOAD_ATTR_METHOD_WITH_VALUES", + "LOAD_ATTR_METHOD_NO_DICT", + "LOAD_ATTR_METHOD_LAZY_DICT", + "LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES", + "LOAD_ATTR_NONDESCRIPTOR_NO_DICT", + ], + "COMPARE_OP": [ + "COMPARE_OP_FLOAT", + "COMPARE_OP_INT", + "COMPARE_OP_STR", + ], + "CONTAINS_OP": [ + "CONTAINS_OP_SET", + "CONTAINS_OP_DICT", + ], + "FOR_ITER": [ + "FOR_ITER_LIST", + "FOR_ITER_TUPLE", + "FOR_ITER_RANGE", + "FOR_ITER_GEN", + ], + "CALL": [ + "CALL_BOUND_METHOD_EXACT_ARGS", + "CALL_PY_EXACT_ARGS", + "CALL_TYPE_1", + "CALL_STR_1", + "CALL_TUPLE_1", + "CALL_BUILTIN_CLASS", + "CALL_BUILTIN_O", + "CALL_BUILTIN_FAST", + "CALL_BUILTIN_FAST_WITH_KEYWORDS", + "CALL_LEN", + "CALL_ISINSTANCE", + "CALL_LIST_APPEND", + "CALL_METHOD_DESCRIPTOR_O", + "CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS", + "CALL_METHOD_DESCRIPTOR_NOARGS", + "CALL_METHOD_DESCRIPTOR_FAST", + "CALL_ALLOC_AND_ENTER_INIT", + "CALL_PY_GENERAL", + "CALL_BOUND_METHOD_GENERAL", + "CALL_NON_PY_GENERAL", + ], +} + +_specialized_opmap = { + 'BINARY_OP_ADD_FLOAT': 150, + 'BINARY_OP_ADD_INT': 151, + 'BINARY_OP_ADD_UNICODE': 152, + 'BINARY_OP_INPLACE_ADD_UNICODE': 3, + 'BINARY_OP_MULTIPLY_FLOAT': 153, + 'BINARY_OP_MULTIPLY_INT': 154, + 'BINARY_OP_SUBTRACT_FLOAT': 155, + 'BINARY_OP_SUBTRACT_INT': 156, + 'BINARY_SUBSCR_DICT': 157, + 'BINARY_SUBSCR_GETITEM': 158, + 'BINARY_SUBSCR_LIST_INT': 159, + 'BINARY_SUBSCR_STR_INT': 160, + 'BINARY_SUBSCR_TUPLE_INT': 161, + 'CALL_ALLOC_AND_ENTER_INIT': 162, + 'CALL_BOUND_METHOD_EXACT_ARGS': 163, + 'CALL_BOUND_METHOD_GENERAL': 164, + 'CALL_BUILTIN_CLASS': 165, + 'CALL_BUILTIN_FAST': 166, + 'CALL_BUILTIN_FAST_WITH_KEYWORDS': 167, + 'CALL_BUILTIN_O': 168, + 'CALL_ISINSTANCE': 169, + 'CALL_LEN': 170, + 'CALL_LIST_APPEND': 171, + 'CALL_METHOD_DESCRIPTOR_FAST': 172, + 'CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS': 173, + 'CALL_METHOD_DESCRIPTOR_NOARGS': 174, + 'CALL_METHOD_DESCRIPTOR_O': 175, + 'CALL_NON_PY_GENERAL': 176, + 'CALL_PY_EXACT_ARGS': 177, + 'CALL_PY_GENERAL': 178, + 'CALL_STR_1': 179, + 'CALL_TUPLE_1': 180, + 'CALL_TYPE_1': 181, + 'COMPARE_OP_FLOAT': 182, + 'COMPARE_OP_INT': 183, + 'COMPARE_OP_STR': 184, + 'CONTAINS_OP_DICT': 185, + 'CONTAINS_OP_SET': 186, + 'FOR_ITER_GEN': 187, + 'FOR_ITER_LIST': 188, + 'FOR_ITER_RANGE': 189, + 'FOR_ITER_TUPLE': 190, + 'LOAD_ATTR_CLASS': 191, + 'LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN': 192, + 'LOAD_ATTR_INSTANCE_VALUE': 193, + 'LOAD_ATTR_METHOD_LAZY_DICT': 194, + 'LOAD_ATTR_METHOD_NO_DICT': 195, + 'LOAD_ATTR_METHOD_WITH_VALUES': 196, + 'LOAD_ATTR_MODULE': 197, + 'LOAD_ATTR_NONDESCRIPTOR_NO_DICT': 198, + 'LOAD_ATTR_NONDESCRIPTOR_WITH_VALUES': 199, + 'LOAD_ATTR_PROPERTY': 200, + 'LOAD_ATTR_SLOT': 201, + 'LOAD_ATTR_WITH_HINT': 202, + 'LOAD_GLOBAL_BUILTIN': 203, + 'LOAD_GLOBAL_MODULE': 204, + 'LOAD_SUPER_ATTR_ATTR': 205, + 'LOAD_SUPER_ATTR_METHOD': 206, + 'RESUME_CHECK': 207, + 'SEND_GEN': 208, + 'STORE_ATTR_INSTANCE_VALUE': 209, + 'STORE_ATTR_SLOT': 210, + 'STORE_ATTR_WITH_HINT': 211, + 'STORE_SUBSCR_DICT': 212, + 'STORE_SUBSCR_LIST_INT': 213, + 'TO_BOOL_ALWAYS_TRUE': 214, + 'TO_BOOL_BOOL': 215, + 'TO_BOOL_INT': 216, + 'TO_BOOL_LIST': 217, + 'TO_BOOL_NONE': 218, + 'TO_BOOL_STR': 219, + 'UNPACK_SEQUENCE_LIST': 220, + 'UNPACK_SEQUENCE_TUPLE': 221, + 'UNPACK_SEQUENCE_TWO_TUPLE': 222, +} + +opmap = { + 'CACHE': 0, + 'RESERVED': 17, + 'RESUME': 149, + 'INSTRUMENTED_LINE': 254, + 'BEFORE_ASYNC_WITH': 1, + 'BEFORE_WITH': 2, + 'BINARY_SLICE': 4, + 'BINARY_SUBSCR': 5, + 'CHECK_EG_MATCH': 6, + 'CHECK_EXC_MATCH': 7, + 'CLEANUP_THROW': 8, + 'DELETE_SUBSCR': 9, + 'END_ASYNC_FOR': 10, + 'END_FOR': 11, + 'END_SEND': 12, + 'EXIT_INIT_CHECK': 13, + 'FORMAT_SIMPLE': 14, + 'FORMAT_WITH_SPEC': 15, + 'GET_AITER': 16, + 'GET_ANEXT': 18, + 'GET_ITER': 19, + 'GET_LEN': 20, + 'GET_YIELD_FROM_ITER': 21, + 'INTERPRETER_EXIT': 22, + 'LOAD_ASSERTION_ERROR': 23, + 'LOAD_BUILD_CLASS': 24, + 'LOAD_LOCALS': 25, + 'MAKE_FUNCTION': 26, + 'MATCH_KEYS': 27, + 'MATCH_MAPPING': 28, + 'MATCH_SEQUENCE': 29, + 'NOP': 30, + 'POP_EXCEPT': 31, + 'POP_TOP': 32, + 'PUSH_EXC_INFO': 33, + 'PUSH_NULL': 34, + 'RETURN_GENERATOR': 35, + 'RETURN_VALUE': 36, + 'SETUP_ANNOTATIONS': 37, + 'STORE_SLICE': 38, + 'STORE_SUBSCR': 39, + 'TO_BOOL': 40, + 'UNARY_INVERT': 41, + 'UNARY_NEGATIVE': 42, + 'UNARY_NOT': 43, + 'WITH_EXCEPT_START': 44, + 'BINARY_OP': 45, + 'BUILD_CONST_KEY_MAP': 46, + 'BUILD_LIST': 47, + 'BUILD_MAP': 48, + 'BUILD_SET': 49, + 'BUILD_SLICE': 50, + 'BUILD_STRING': 51, + 'BUILD_TUPLE': 52, + 'CALL': 53, + 'CALL_FUNCTION_EX': 54, + 'CALL_INTRINSIC_1': 55, + 'CALL_INTRINSIC_2': 56, + 'CALL_KW': 57, + 'COMPARE_OP': 58, + 'CONTAINS_OP': 59, + 'CONVERT_VALUE': 60, + 'COPY': 61, + 'COPY_FREE_VARS': 62, + 'DELETE_ATTR': 63, + 'DELETE_DEREF': 64, + 'DELETE_FAST': 65, + 'DELETE_GLOBAL': 66, + 'DELETE_NAME': 67, + 'DICT_MERGE': 68, + 'DICT_UPDATE': 69, + 'ENTER_EXECUTOR': 70, + 'EXTENDED_ARG': 71, + 'FOR_ITER': 72, + 'GET_AWAITABLE': 73, + 'IMPORT_FROM': 74, + 'IMPORT_NAME': 75, + 'IS_OP': 76, + 'JUMP_BACKWARD': 77, + 'JUMP_BACKWARD_NO_INTERRUPT': 78, + 'JUMP_FORWARD': 79, + 'LIST_APPEND': 80, + 'LIST_EXTEND': 81, + 'LOAD_ATTR': 82, + 'LOAD_CONST': 83, + 'LOAD_DEREF': 84, + 'LOAD_FAST': 85, + 'LOAD_FAST_AND_CLEAR': 86, + 'LOAD_FAST_CHECK': 87, + 'LOAD_FAST_LOAD_FAST': 88, + 'LOAD_FROM_DICT_OR_DEREF': 89, + 'LOAD_FROM_DICT_OR_GLOBALS': 90, + 'LOAD_GLOBAL': 91, + 'LOAD_NAME': 92, + 'LOAD_SUPER_ATTR': 93, + 'MAKE_CELL': 94, + 'MAP_ADD': 95, + 'MATCH_CLASS': 96, + 'POP_JUMP_IF_FALSE': 97, + 'POP_JUMP_IF_NONE': 98, + 'POP_JUMP_IF_NOT_NONE': 99, + 'POP_JUMP_IF_TRUE': 100, + 'RAISE_VARARGS': 101, + 'RERAISE': 102, + 'RETURN_CONST': 103, + 'SEND': 104, + 'SET_ADD': 105, + 'SET_FUNCTION_ATTRIBUTE': 106, + 'SET_UPDATE': 107, + 'STORE_ATTR': 108, + 'STORE_DEREF': 109, + 'STORE_FAST': 110, + 'STORE_FAST_LOAD_FAST': 111, + 'STORE_FAST_STORE_FAST': 112, + 'STORE_GLOBAL': 113, + 'STORE_NAME': 114, + 'SWAP': 115, + 'UNPACK_EX': 116, + 'UNPACK_SEQUENCE': 117, + 'YIELD_VALUE': 118, + 'INSTRUMENTED_RESUME': 236, + 'INSTRUMENTED_END_FOR': 237, + 'INSTRUMENTED_END_SEND': 238, + 'INSTRUMENTED_RETURN_VALUE': 239, + 'INSTRUMENTED_RETURN_CONST': 240, + 'INSTRUMENTED_YIELD_VALUE': 241, + 'INSTRUMENTED_LOAD_SUPER_ATTR': 242, + 'INSTRUMENTED_FOR_ITER': 243, + 'INSTRUMENTED_CALL': 244, + 'INSTRUMENTED_CALL_KW': 245, + 'INSTRUMENTED_CALL_FUNCTION_EX': 246, + 'INSTRUMENTED_INSTRUCTION': 247, + 'INSTRUMENTED_JUMP_FORWARD': 248, + 'INSTRUMENTED_JUMP_BACKWARD': 249, + 'INSTRUMENTED_POP_JUMP_IF_TRUE': 250, + 'INSTRUMENTED_POP_JUMP_IF_FALSE': 251, + 'INSTRUMENTED_POP_JUMP_IF_NONE': 252, + 'INSTRUMENTED_POP_JUMP_IF_NOT_NONE': 253, + 'JUMP': 256, + 'JUMP_NO_INTERRUPT': 257, + 'LOAD_CLOSURE': 258, + 'LOAD_METHOD': 259, + 'LOAD_SUPER_METHOD': 260, + 'LOAD_ZERO_SUPER_ATTR': 261, + 'LOAD_ZERO_SUPER_METHOD': 262, + 'POP_BLOCK': 263, + 'SETUP_CLEANUP': 264, + 'SETUP_FINALLY': 265, + 'SETUP_WITH': 266, + 'STORE_FAST_MAYBE_NULL': 267, +} + +HAVE_ARGUMENT = 44 +MIN_INSTRUMENTED_OPCODE = 236 diff --git a/Lib/_pydecimal.py b/Lib/_pydecimal.py index a0b608380a..ff80180a79 100644 --- a/Lib/_pydecimal.py +++ b/Lib/_pydecimal.py @@ -13,104 +13,7 @@ # bug) and will be backported. At this point the spec is stabilizing # and the updates are becoming fewer, smaller, and less significant. -""" -This is an implementation of decimal floating point arithmetic based on -the General Decimal Arithmetic Specification: - - http://speleotrove.com/decimal/decarith.html - -and IEEE standard 854-1987: - - http://en.wikipedia.org/wiki/IEEE_854-1987 - -Decimal floating point has finite precision with arbitrarily large bounds. - -The purpose of this module is to support arithmetic using familiar -"schoolhouse" rules and to avoid some of the tricky representation -issues associated with binary floating point. The package is especially -useful for financial applications or for contexts where users have -expectations that are at odds with binary floating point (for instance, -in binary floating point, 1.00 % 0.1 gives 0.09999999999999995 instead -of 0.0; Decimal('1.00') % Decimal('0.1') returns the expected -Decimal('0.00')). - -Here are some examples of using the decimal module: - ->>> from decimal import * ->>> setcontext(ExtendedContext) ->>> Decimal(0) -Decimal('0') ->>> Decimal('1') -Decimal('1') ->>> Decimal('-.0123') -Decimal('-0.0123') ->>> Decimal(123456) -Decimal('123456') ->>> Decimal('123.45e12345678') -Decimal('1.2345E+12345680') ->>> Decimal('1.33') + Decimal('1.27') -Decimal('2.60') ->>> Decimal('12.34') + Decimal('3.87') - Decimal('18.41') -Decimal('-2.20') ->>> dig = Decimal(1) ->>> print(dig / Decimal(3)) -0.333333333 ->>> getcontext().prec = 18 ->>> print(dig / Decimal(3)) -0.333333333333333333 ->>> print(dig.sqrt()) -1 ->>> print(Decimal(3).sqrt()) -1.73205080756887729 ->>> print(Decimal(3) ** 123) -4.85192780976896427E+58 ->>> inf = Decimal(1) / Decimal(0) ->>> print(inf) -Infinity ->>> neginf = Decimal(-1) / Decimal(0) ->>> print(neginf) --Infinity ->>> print(neginf + inf) -NaN ->>> print(neginf * inf) --Infinity ->>> print(dig / 0) -Infinity ->>> getcontext().traps[DivisionByZero] = 1 ->>> print(dig / 0) -Traceback (most recent call last): - ... - ... - ... -decimal.DivisionByZero: x / 0 ->>> c = Context() ->>> c.traps[InvalidOperation] = 0 ->>> print(c.flags[InvalidOperation]) -0 ->>> c.divide(Decimal(0), Decimal(0)) -Decimal('NaN') ->>> c.traps[InvalidOperation] = 1 ->>> print(c.flags[InvalidOperation]) -1 ->>> c.flags[InvalidOperation] = 0 ->>> print(c.flags[InvalidOperation]) -0 ->>> print(c.divide(Decimal(0), Decimal(0))) -Traceback (most recent call last): - ... - ... - ... -decimal.InvalidOperation: 0 / 0 ->>> print(c.flags[InvalidOperation]) -1 ->>> c.flags[InvalidOperation] = 0 ->>> c.traps[InvalidOperation] = 0 ->>> print(c.divide(Decimal(0), Decimal(0))) -NaN ->>> print(c.flags[InvalidOperation]) -1 ->>> -""" +"""Python decimal arithmetic module""" __all__ = [ # Two major classes @@ -140,8 +43,11 @@ # Limits for the C version for compatibility 'MAX_PREC', 'MAX_EMAX', 'MIN_EMIN', 'MIN_ETINY', - # C version: compile time choice that enables the thread local context - 'HAVE_THREADS' + # C version: compile time choice that enables the thread local context (deprecated, now always true) + 'HAVE_THREADS', + + # C version: compile time choice that enables the coroutine local context + 'HAVE_CONTEXTVAR' ] __xname__ = __name__ # sys.modules lookup (--without-threads) @@ -156,7 +62,7 @@ try: from collections import namedtuple as _namedtuple - DecimalTuple = _namedtuple('DecimalTuple', 'sign digits exponent') + DecimalTuple = _namedtuple('DecimalTuple', 'sign digits exponent', module='decimal') except ImportError: DecimalTuple = lambda *args: args @@ -172,6 +78,7 @@ # Compatibility with the C version HAVE_THREADS = True +HAVE_CONTEXTVAR = True if sys.maxsize == 2**63-1: MAX_PREC = 999999999999999999 MAX_EMAX = 999999999999999999 @@ -190,7 +97,7 @@ class DecimalException(ArithmeticError): Used exceptions derive from this. If an exception derives from another exception besides this (such as - Underflow (Inexact, Rounded, Subnormal) that indicates that it is only + Underflow (Inexact, Rounded, Subnormal)) that indicates that it is only called if the others are present. This isn't actually used for anything, though. @@ -238,7 +145,7 @@ class InvalidOperation(DecimalException): x ** (+-)INF An operand is invalid - The result of the operation after these is a quiet positive NaN, + The result of the operation after this is a quiet positive NaN, except when the cause is a signaling NaN, in which case the result is also a quiet NaN, but with the original sign, and an optional diagnostic information. @@ -431,82 +338,40 @@ class FloatOperation(DecimalException, TypeError): ##### Context Functions ################################################## # The getcontext() and setcontext() function manage access to a thread-local -# current context. Py2.4 offers direct support for thread locals. If that -# is not available, use threading.current_thread() which is slower but will -# work for older Pythons. If threads are not part of the build, create a -# mock threading object with threading.local() returning the module namespace. - -try: - import threading -except ImportError: - # Python was compiled without threads; create a mock object instead - class MockThreading(object): - def local(self, sys=sys): - return sys.modules[__xname__] - threading = MockThreading() - del MockThreading - -try: - threading.local - -except AttributeError: - - # To fix reloading, force it to create a new context - # Old contexts have different exceptions in their dicts, making problems. - if hasattr(threading.current_thread(), '__decimal_context__'): - del threading.current_thread().__decimal_context__ +# current context. - def setcontext(context): - """Set this thread's context to context.""" - if context in (DefaultContext, BasicContext, ExtendedContext): - context = context.copy() - context.clear_flags() - threading.current_thread().__decimal_context__ = context +import contextvars - def getcontext(): - """Returns this thread's context. +_current_context_var = contextvars.ContextVar('decimal_context') - If this thread does not yet have a context, returns - a new context and sets this thread's context. - New contexts are copies of DefaultContext. - """ - try: - return threading.current_thread().__decimal_context__ - except AttributeError: - context = Context() - threading.current_thread().__decimal_context__ = context - return context +_context_attributes = frozenset( + ['prec', 'Emin', 'Emax', 'capitals', 'clamp', 'rounding', 'flags', 'traps'] +) -else: +def getcontext(): + """Returns this thread's context. - local = threading.local() - if hasattr(local, '__decimal_context__'): - del local.__decimal_context__ + If this thread does not yet have a context, returns + a new context and sets this thread's context. + New contexts are copies of DefaultContext. + """ + try: + return _current_context_var.get() + except LookupError: + context = Context() + _current_context_var.set(context) + return context + +def setcontext(context): + """Set this thread's context to context.""" + if context in (DefaultContext, BasicContext, ExtendedContext): + context = context.copy() + context.clear_flags() + _current_context_var.set(context) - def getcontext(_local=local): - """Returns this thread's context. +del contextvars # Don't contaminate the namespace - If this thread does not yet have a context, returns - a new context and sets this thread's context. - New contexts are copies of DefaultContext. - """ - try: - return _local.__decimal_context__ - except AttributeError: - context = Context() - _local.__decimal_context__ = context - return context - - def setcontext(context, _local=local): - """Set this thread's context to context.""" - if context in (DefaultContext, BasicContext, ExtendedContext): - context = context.copy() - context.clear_flags() - _local.__decimal_context__ = context - - del threading, local # Don't contaminate the namespace - -def localcontext(ctx=None): +def localcontext(ctx=None, **kwargs): """Return a context manager for a copy of the supplied context Uses a copy of the current context if no context is specified @@ -542,8 +407,14 @@ def sin(x): >>> print(getcontext().prec) 28 """ - if ctx is None: ctx = getcontext() - return _ContextManager(ctx) + if ctx is None: + ctx = getcontext() + ctx_manager = _ContextManager(ctx) + for key, value in kwargs.items(): + if key not in _context_attributes: + raise TypeError(f"'{key}' is an invalid keyword argument for this function") + setattr(ctx_manager.new_context, key, value) + return ctx_manager ##### Decimal class ####################################################### @@ -553,7 +424,7 @@ def sin(x): # numbers.py for more detail. class Decimal(object): - """Floating point class for decimal arithmetic.""" + """Floating-point class for decimal arithmetic.""" __slots__ = ('_exp','_int','_sign', '_is_special') # Generally, the value of the Decimal instance is given by @@ -993,7 +864,7 @@ def __hash__(self): if self.is_snan(): raise TypeError('Cannot hash a signaling NaN value.') elif self.is_nan(): - return _PyHASH_NAN + return object.__hash__(self) else: if self._sign: return -_PyHASH_INF @@ -1674,13 +1545,13 @@ def __int__(self): __trunc__ = __int__ + @property def real(self): return self - real = property(real) + @property def imag(self): return Decimal(0) - imag = property(imag) def conjugate(self): return self @@ -2260,10 +2131,16 @@ def _power_exact(self, other, p): else: return None - if xc >= 10**p: + # An exact power of 10 is representable, but can convert to a + # string of any length. But an exact power of 10 shouldn't be + # possible at this point. + assert xc > 1, self + assert xc % 10 != 0, self + strxc = str(xc) + if len(strxc) > p: return None xe = -e-xe - return _dec_from_triple(0, str(xc), xe) + return _dec_from_triple(0, strxc, xe) # now y is positive; find m and n such that y = m/n if ye >= 0: @@ -2272,7 +2149,7 @@ def _power_exact(self, other, p): if xe != 0 and len(str(abs(yc*xe))) <= -ye: return None xc_bits = _nbits(xc) - if xc != 1 and len(str(abs(yc)*xc_bits)) <= -ye: + if len(str(abs(yc)*xc_bits)) <= -ye: return None m, n = yc, 10**(-ye) while m % 2 == n % 2 == 0: @@ -2285,7 +2162,7 @@ def _power_exact(self, other, p): # compute nth root of xc*10**xe if n > 1: # if 1 < xc < 2**n then xc isn't an nth power - if xc != 1 and xc_bits <= n: + if xc_bits <= n: return None xe, rem = divmod(xe, n) @@ -2313,13 +2190,18 @@ def _power_exact(self, other, p): return None xc = xc**m xe *= m - if xc > 10**p: + # An exact power of 10 is representable, but can convert to a string + # of any length. But an exact power of 10 shouldn't be possible at + # this point. + assert xc > 1, self + assert xc % 10 != 0, self + str_xc = str(xc) + if len(str_xc) > p: return None # by this point the result *is* exactly representable # adjust the exponent to get as close as possible to the ideal # exponent, if necessary - str_xc = str(xc) if other._isinteger() and other._sign == 0: ideal_exponent = self._exp*int(other) zeros = min(xe-ideal_exponent, p-len(str_xc)) @@ -3837,6 +3719,10 @@ def __format__(self, specifier, context=None, _localeconv=None): # represented in fixed point; rescale them to 0e0. if not self and self._exp > 0 and spec['type'] in 'fF%': self = self._rescale(0, rounding) + if not self and spec['no_neg_0'] and self._sign: + adjusted_sign = 0 + else: + adjusted_sign = self._sign # figure out placement of the decimal point leftdigits = self._exp + len(self._int) @@ -3867,7 +3753,7 @@ def __format__(self, specifier, context=None, _localeconv=None): # done with the decimal-specific stuff; hand over the rest # of the formatting to the _format_number function - return _format_number(self._sign, intpart, fracpart, exp, spec) + return _format_number(adjusted_sign, intpart, fracpart, exp, spec) def _dec_from_triple(sign, coefficient, exponent, special=False): """Create a decimal instance directly, without any validation, @@ -5677,8 +5563,6 @@ def __init__(self, value=None): def __repr__(self): return "(%r, %r, %r)" % (self.sign, self.int, self.exp) - __str__ = __repr__ - def _normalize(op1, op2, prec = 0): @@ -6187,7 +6071,7 @@ def _convert_for_comparison(self, other, equality_op=False): # # A format specifier for Decimal looks like: # -# [[fill]align][sign][#][0][minimumwidth][,][.precision][type] +# [[fill]align][sign][z][#][0][minimumwidth][,][.precision][type] _parse_format_specifier_regex = re.compile(r"""\A (?: @@ -6195,6 +6079,7 @@ def _convert_for_comparison(self, other, equality_op=False): (?P[<>=^]) )? (?P[-+ ])? +(?Pz)? (?P\#)? (?P0)? (?P(?!0)\d+)? diff --git a/Lib/_strptime.py b/Lib/_strptime.py index 798cf9f9d3..a07eb5f923 100644 --- a/Lib/_strptime.py +++ b/Lib/_strptime.py @@ -10,10 +10,13 @@ strptime -- Calculates the time struct represented by the passed-in string """ +import os import time import locale import calendar +import re from re import compile as re_compile +from re import sub as re_sub from re import IGNORECASE from re import escape as re_escape from datetime import (date as datetime_date, @@ -27,6 +30,41 @@ def _getlang(): # Figure out what the current language is set to. return locale.getlocale(locale.LC_TIME) +def _findall(haystack, needle): + # Find all positions of needle in haystack. + if not needle: + return + i = 0 + while True: + i = haystack.find(needle, i) + if i < 0: + break + yield i + i += len(needle) + +def _fixmonths(months): + yield from months + # The lower case of 'İ' ('\u0130') is 'i\u0307'. + # The re module only supports 1-to-1 character matching in + # case-insensitive mode. + for s in months: + if 'i\u0307' in s: + yield s.replace('i\u0307', '\u0130') + +lzh_TW_alt_digits = ( + # 〇:一:二:三:四:五:六:七:八:九 + '\u3007', '\u4e00', '\u4e8c', '\u4e09', '\u56db', + '\u4e94', '\u516d', '\u4e03', '\u516b', '\u4e5d', + # 十:十一:十二:十三:十四:十五:十六:十七:十八:十九 + '\u5341', '\u5341\u4e00', '\u5341\u4e8c', '\u5341\u4e09', '\u5341\u56db', + '\u5341\u4e94', '\u5341\u516d', '\u5341\u4e03', '\u5341\u516b', '\u5341\u4e5d', + # 廿:廿一:廿二:廿三:廿四:廿五:廿六:廿七:廿八:廿九 + '\u5eff', '\u5eff\u4e00', '\u5eff\u4e8c', '\u5eff\u4e09', '\u5eff\u56db', + '\u5eff\u4e94', '\u5eff\u516d', '\u5eff\u4e03', '\u5eff\u516b', '\u5eff\u4e5d', + # 卅:卅一 + '\u5345', '\u5345\u4e00') + + class LocaleTime(object): """Stores and handles locale-specific information related to time. @@ -70,6 +108,7 @@ def __init__(self): self.__calc_weekday() self.__calc_month() self.__calc_am_pm() + self.__calc_alt_digits() self.__calc_timezone() self.__calc_date_time() if _getlang() != self.lang: @@ -101,53 +140,184 @@ def __calc_am_pm(self): am_pm = [] for hour in (1, 22): time_tuple = time.struct_time((1999,3,17,hour,44,55,2,76,0)) - am_pm.append(time.strftime("%p", time_tuple).lower()) + # br_FR has AM/PM info (' ',' '). + am_pm.append(time.strftime("%p", time_tuple).lower().strip()) self.am_pm = am_pm + def __calc_alt_digits(self): + # Set self.LC_alt_digits by using time.strftime(). + + # The magic data should contain all decimal digits. + time_tuple = time.struct_time((1998, 1, 27, 10, 43, 56, 1, 27, 0)) + s = time.strftime("%x%X", time_tuple) + if s.isascii(): + # Fast path -- all digits are ASCII. + self.LC_alt_digits = () + return + + digits = ''.join(sorted(set(re.findall(r'\d', s)))) + if len(digits) == 10 and ord(digits[-1]) == ord(digits[0]) + 9: + # All 10 decimal digits from the same set. + if digits.isascii(): + # All digits are ASCII. + self.LC_alt_digits = () + return + + self.LC_alt_digits = [a + b for a in digits for b in digits] + # Test whether the numbers contain leading zero. + time_tuple2 = time.struct_time((2000, 1, 1, 1, 1, 1, 5, 1, 0)) + if self.LC_alt_digits[1] not in time.strftime("%x %X", time_tuple2): + self.LC_alt_digits[:10] = digits + return + + # Either non-Gregorian calendar or non-decimal numbers. + if {'\u4e00', '\u4e03', '\u4e5d', '\u5341', '\u5eff'}.issubset(s): + # lzh_TW + self.LC_alt_digits = lzh_TW_alt_digits + return + + self.LC_alt_digits = None + def __calc_date_time(self): - # Set self.date_time, self.date, & self.time by using - # time.strftime(). + # Set self.LC_date_time, self.LC_date, self.LC_time and + # self.LC_time_ampm by using time.strftime(). # Use (1999,3,17,22,44,55,2,76,0) for magic date because the amount of # overloaded numbers is minimized. The order in which searches for # values within the format string is very important; it eliminates # possible ambiguity for what something represents. time_tuple = time.struct_time((1999,3,17,22,44,55,2,76,0)) - date_time = [None, None, None] - date_time[0] = time.strftime("%c", time_tuple).lower() - date_time[1] = time.strftime("%x", time_tuple).lower() - date_time[2] = time.strftime("%X", time_tuple).lower() - replacement_pairs = [('%', '%%'), (self.f_weekday[2], '%A'), - (self.f_month[3], '%B'), (self.a_weekday[2], '%a'), - (self.a_month[3], '%b'), (self.am_pm[1], '%p'), - ('1999', '%Y'), ('99', '%y'), ('22', '%H'), - ('44', '%M'), ('55', '%S'), ('76', '%j'), - ('17', '%d'), ('03', '%m'), ('3', '%m'), - # '3' needed for when no leading zero. - ('2', '%w'), ('10', '%I')] - replacement_pairs.extend([(tz, "%Z") for tz_values in self.timezone - for tz in tz_values]) - for offset,directive in ((0,'%c'), (1,'%x'), (2,'%X')): - current_format = date_time[offset] - for old, new in replacement_pairs: + time_tuple2 = time.struct_time((1999,1,3,1,1,1,6,3,0)) + replacement_pairs = [] + + # Non-ASCII digits + if self.LC_alt_digits or self.LC_alt_digits is None: + for n, d in [(19, '%OC'), (99, '%Oy'), (22, '%OH'), + (44, '%OM'), (55, '%OS'), (17, '%Od'), + (3, '%Om'), (2, '%Ow'), (10, '%OI')]: + if self.LC_alt_digits is None: + s = chr(0x660 + n // 10) + chr(0x660 + n % 10) + replacement_pairs.append((s, d)) + if n < 10: + replacement_pairs.append((s[1], d)) + elif len(self.LC_alt_digits) > n: + replacement_pairs.append((self.LC_alt_digits[n], d)) + else: + replacement_pairs.append((time.strftime(d, time_tuple), d)) + replacement_pairs += [ + ('1999', '%Y'), ('99', '%y'), ('22', '%H'), + ('44', '%M'), ('55', '%S'), ('76', '%j'), + ('17', '%d'), ('03', '%m'), ('3', '%m'), + # '3' needed for when no leading zero. + ('2', '%w'), ('10', '%I'), + ] + + date_time = [] + for directive in ('%c', '%x', '%X', '%r'): + current_format = time.strftime(directive, time_tuple).lower() + current_format = current_format.replace('%', '%%') + # The month and the day of the week formats are treated specially + # because of a possible ambiguity in some locales where the full + # and abbreviated names are equal or names of different types + # are equal. See doc of __find_month_format for more details. + lst, fmt = self.__find_weekday_format(directive) + if lst: + current_format = current_format.replace(lst[2], fmt, 1) + lst, fmt = self.__find_month_format(directive) + if lst: + current_format = current_format.replace(lst[3], fmt, 1) + if self.am_pm[1]: # Must deal with possible lack of locale info # manifesting itself as the empty string (e.g., Swedish's # lack of AM/PM info) or a platform returning a tuple of empty # strings (e.g., MacOS 9 having timezone as ('','')). - if old: - current_format = current_format.replace(old, new) + current_format = current_format.replace(self.am_pm[1], '%p') + for tz_values in self.timezone: + for tz in tz_values: + if tz: + current_format = current_format.replace(tz, "%Z") + # Transform all non-ASCII digits to digits in range U+0660 to U+0669. + if not current_format.isascii() and self.LC_alt_digits is None: + current_format = re_sub(r'\d(?3[0-1]|[1-2]\d|0[1-9]|[1-9]| [1-9])", 'f': r"(?P[0-9]{1,6})", - 'H': r"(?P2[0-3]|[0-1]\d|\d)", - 'I': r"(?P1[0-2]|0[1-9]|[1-9])", + 'H': r"(?P2[0-3]|[0-1]\d|\d| \d)", + 'k': r"(?P2[0-3]|[0-1]\d|\d| \d)", + 'I': r"(?P1[0-2]|0[1-9]|[1-9]| [1-9])", + 'l': r"(?P1[0-2]|0[1-9]|[1-9]| [1-9])", 'G': r"(?P\d\d\d\d)", 'j': r"(?P36[0-6]|3[0-5]\d|[1-2]\d\d|0[1-9]\d|00[1-9]|[1-9]\d|0[1-9]|[1-9])", 'm': r"(?P1[0-2]|0[1-9]|[1-9])", @@ -198,25 +370,60 @@ def __init__(self, locale_time=None): 'V': r"(?P5[0-3]|0[1-9]|[1-4]\d|\d)", # W is set below by using 'U' 'y': r"(?P\d\d)", - #XXX: Does 'Y' need to worry about having less or more than - # 4 digits? 'Y': r"(?P\d\d\d\d)", 'z': r"(?P[+-]\d\d:?[0-5]\d(:?[0-5]\d(\.\d{1,6})?)?|(?-i:Z))", 'A': self.__seqToRE(self.locale_time.f_weekday, 'A'), 'a': self.__seqToRE(self.locale_time.a_weekday, 'a'), - 'B': self.__seqToRE(self.locale_time.f_month[1:], 'B'), - 'b': self.__seqToRE(self.locale_time.a_month[1:], 'b'), + 'B': self.__seqToRE(_fixmonths(self.locale_time.f_month[1:]), 'B'), + 'b': self.__seqToRE(_fixmonths(self.locale_time.a_month[1:]), 'b'), 'p': self.__seqToRE(self.locale_time.am_pm, 'p'), 'Z': self.__seqToRE((tz for tz_names in self.locale_time.timezone for tz in tz_names), 'Z'), - '%': '%'}) - base.__setitem__('W', base.__getitem__('U').replace('U', 'W')) - base.__setitem__('c', self.pattern(self.locale_time.LC_date_time)) - base.__setitem__('x', self.pattern(self.locale_time.LC_date)) + '%': '%'} + if self.locale_time.LC_alt_digits is None: + for d in 'dmyCHIMS': + mapping['O' + d] = r'(?P<%s>\d\d|\d| \d)' % d + mapping['Ow'] = r'(?P\d)' + else: + mapping.update({ + 'Od': self.__seqToRE(self.locale_time.LC_alt_digits[1:32], 'd', + '3[0-1]|[1-2][0-9]|0[1-9]|[1-9]'), + 'Om': self.__seqToRE(self.locale_time.LC_alt_digits[1:13], 'm', + '1[0-2]|0[1-9]|[1-9]'), + 'Ow': self.__seqToRE(self.locale_time.LC_alt_digits[:7], 'w', + '[0-6]'), + 'Oy': self.__seqToRE(self.locale_time.LC_alt_digits, 'y', + '[0-9][0-9]'), + 'OC': self.__seqToRE(self.locale_time.LC_alt_digits, 'C', + '[0-9][0-9]'), + 'OH': self.__seqToRE(self.locale_time.LC_alt_digits[:24], 'H', + '2[0-3]|[0-1][0-9]|[0-9]'), + 'OI': self.__seqToRE(self.locale_time.LC_alt_digits[1:13], 'I', + '1[0-2]|0[1-9]|[1-9]'), + 'OM': self.__seqToRE(self.locale_time.LC_alt_digits[:60], 'M', + '[0-5][0-9]|[0-9]'), + 'OS': self.__seqToRE(self.locale_time.LC_alt_digits[:62], 'S', + '6[0-1]|[0-5][0-9]|[0-9]'), + }) + mapping.update({ + 'e': mapping['d'], + 'Oe': mapping['Od'], + 'P': mapping['p'], + 'Op': mapping['p'], + 'W': mapping['U'].replace('U', 'W'), + }) + mapping['W'] = mapping['U'].replace('U', 'W') + + base.__init__(mapping) + base.__setitem__('T', self.pattern('%H:%M:%S')) + base.__setitem__('R', self.pattern('%H:%M')) + base.__setitem__('r', self.pattern(self.locale_time.LC_time_ampm)) base.__setitem__('X', self.pattern(self.locale_time.LC_time)) + base.__setitem__('x', self.pattern(self.locale_time.LC_date)) + base.__setitem__('c', self.pattern(self.locale_time.LC_date_time)) - def __seqToRE(self, to_convert, directive): + def __seqToRE(self, to_convert, directive, altregex=None): """Convert a list to a regex string for matching a directive. Want possible matching values to be from longest to shortest. This @@ -232,8 +439,9 @@ def __seqToRE(self, to_convert, directive): else: return '' regex = '|'.join(re_escape(stuff) for stuff in to_convert) - regex = '(?P<%s>%s' % (directive, regex) - return '%s)' % regex + if altregex is not None: + regex += '|' + altregex + return '(?P<%s>%s)' % (directive, regex) def pattern(self, format): """Return regex pattern for the format string. @@ -242,21 +450,36 @@ def pattern(self, format): regex syntax are escaped. """ - processed_format = '' # The sub() call escapes all characters that might be misconstrued # as regex syntax. Cannot use re.escape since we have to deal with # format directives (%m, etc.). - regex_chars = re_compile(r"([\\.^$*+?\(\){}\[\]|])") - format = regex_chars.sub(r"\\\1", format) - whitespace_replacement = re_compile(r'\s+') - format = whitespace_replacement.sub(r'\\s+', format) - while '%' in format: - directive_index = format.index('%')+1 - processed_format = "%s%s%s" % (processed_format, - format[:directive_index-1], - self[format[directive_index]]) - format = format[directive_index+1:] - return "%s%s" % (processed_format, format) + format = re_sub(r"([\\.^$*+?\(\){}\[\]|])", r"\\\1", format) + format = re_sub(r'\s+', r'\\s+', format) + format = re_sub(r"'", "['\u02bc]", format) # needed for br_FR + year_in_format = False + day_of_month_in_format = False + def repl(m): + format_char = m[1] + match format_char: + case 'Y' | 'y' | 'G': + nonlocal year_in_format + year_in_format = True + case 'd': + nonlocal day_of_month_in_format + day_of_month_in_format = True + return self[format_char] + format = re_sub(r'%[-_0^#]*[0-9]*([OE]?\\?.?)', repl, format) + if day_of_month_in_format and not year_in_format: + import warnings + warnings.warn("""\ +Parsing dates involving a day of month without a year specified is ambiguious +and fails to parse leap day. The default behavior will change in Python 3.15 +to either always raise an exception or to use a different default year (TBD). +To avoid trouble, add a specific year to the input & format. +See https://github.com/python/cpython/issues/70647.""", + DeprecationWarning, + skip_file_prefixes=(os.path.dirname(__file__),)) + return format def compile(self, format): """Return a compiled re object for the format string.""" @@ -319,14 +542,13 @@ def _strptime(data_string, format="%a %b %d %H:%M:%S %Y"): # \\, in which case it was a stray % but with a space after it except KeyError as err: bad_directive = err.args[0] - if bad_directive == "\\": - bad_directive = "%" del err + bad_directive = bad_directive.replace('\\s', '') + if not bad_directive: + raise ValueError("stray %% in format '%s'" % format) from None + bad_directive = bad_directive.replace('\\', '', 1) raise ValueError("'%s' is a bad directive in format '%s'" % (bad_directive, format)) from None - # IndexError only occurs when the format string is "%" - except IndexError: - raise ValueError("stray %% in format '%s'" % format) from None _regex_cache[format] = format_regex found = format_regex.match(data_string) if not found: @@ -348,6 +570,15 @@ def _strptime(data_string, format="%a %b %d %H:%M:%S %Y"): # values weekday = julian = None found_dict = found.groupdict() + if locale_time.LC_alt_digits: + def parse_int(s): + try: + return locale_time.LC_alt_digits.index(s) + except ValueError: + return int(s) + else: + parse_int = int + for group_key in found_dict.keys(): # Directives not explicitly handled below: # c, x, X @@ -355,30 +586,34 @@ def _strptime(data_string, format="%a %b %d %H:%M:%S %Y"): # U, W # worthless without day of the week if group_key == 'y': - year = int(found_dict['y']) - # Open Group specification for strptime() states that a %y - #value in the range of [00, 68] is in the century 2000, while - #[69,99] is in the century 1900 - if year <= 68: - year += 2000 + year = parse_int(found_dict['y']) + if 'C' in found_dict: + century = parse_int(found_dict['C']) + year += century * 100 else: - year += 1900 + # Open Group specification for strptime() states that a %y + #value in the range of [00, 68] is in the century 2000, while + #[69,99] is in the century 1900 + if year <= 68: + year += 2000 + else: + year += 1900 elif group_key == 'Y': year = int(found_dict['Y']) elif group_key == 'G': iso_year = int(found_dict['G']) elif group_key == 'm': - month = int(found_dict['m']) + month = parse_int(found_dict['m']) elif group_key == 'B': month = locale_time.f_month.index(found_dict['B'].lower()) elif group_key == 'b': month = locale_time.a_month.index(found_dict['b'].lower()) elif group_key == 'd': - day = int(found_dict['d']) + day = parse_int(found_dict['d']) elif group_key == 'H': - hour = int(found_dict['H']) + hour = parse_int(found_dict['H']) elif group_key == 'I': - hour = int(found_dict['I']) + hour = parse_int(found_dict['I']) ampm = found_dict.get('p', '').lower() # If there was no AM/PM indicator, we'll treat this like AM if ampm in ('', locale_time.am_pm[0]): @@ -394,9 +629,9 @@ def _strptime(data_string, format="%a %b %d %H:%M:%S %Y"): if hour != 12: hour += 12 elif group_key == 'M': - minute = int(found_dict['M']) + minute = parse_int(found_dict['M']) elif group_key == 'S': - second = int(found_dict['S']) + second = parse_int(found_dict['S']) elif group_key == 'f': s = found_dict['f'] # Pad to always return microseconds. diff --git a/Lib/_weakrefset.py b/Lib/_weakrefset.py index 2a27684324..489eec714e 100644 --- a/Lib/_weakrefset.py +++ b/Lib/_weakrefset.py @@ -80,8 +80,7 @@ def __contains__(self, item): return wr in self.data def __reduce__(self): - return (self.__class__, (list(self),), - getattr(self, '__dict__', None)) + return self.__class__, (list(self),), self.__getstate__() def add(self, item): if self._pending_removals: diff --git a/Lib/ast.py b/Lib/ast.py index 07044706dc..37b20206b8 100644 --- a/Lib/ast.py +++ b/Lib/ast.py @@ -1,28 +1,24 @@ """ - ast - ~~~ - - The `ast` module helps Python applications to process trees of the Python - abstract syntax grammar. The abstract syntax itself might change with - each Python release; this module helps to find out programmatically what - the current grammar looks like and allows modifications of it. - - An abstract syntax tree can be generated by passing `ast.PyCF_ONLY_AST` as - a flag to the `compile()` builtin function or by using the `parse()` - function from this module. The result will be a tree of objects whose - classes all inherit from `ast.AST`. - - A modified abstract syntax tree can be compiled into a Python code object - using the built-in `compile()` function. - - Additionally various helper functions are provided that make working with - the trees simpler. The main intention of the helper functions and this - module in general is to provide an easy to use interface for libraries - that work tightly with the python syntax (template engines for example). - - - :copyright: Copyright 2008 by Armin Ronacher. - :license: Python License. +The `ast` module helps Python applications to process trees of the Python +abstract syntax grammar. The abstract syntax itself might change with +each Python release; this module helps to find out programmatically what +the current grammar looks like and allows modifications of it. + +An abstract syntax tree can be generated by passing `ast.PyCF_ONLY_AST` as +a flag to the `compile()` builtin function or by using the `parse()` +function from this module. The result will be a tree of objects whose +classes all inherit from `ast.AST`. + +A modified abstract syntax tree can be compiled into a Python code object +using the built-in `compile()` function. + +Additionally various helper functions are provided that make working with +the trees simpler. The main intention of the helper functions and this +module in general is to provide an easy to use interface for libraries +that work tightly with the python syntax (template engines for example). + +:copyright: Copyright 2008 by Armin Ronacher. +:license: Python License. """ import sys import re @@ -32,13 +28,15 @@ def parse(source, filename='', mode='exec', *, - type_comments=False, feature_version=None): + type_comments=False, feature_version=None, optimize=-1): """ Parse the source into an AST node. Equivalent to compile(source, filename, mode, PyCF_ONLY_AST). Pass type_comments=True to get back type comments where the syntax allows. """ flags = PyCF_ONLY_AST + if optimize > 0: + flags |= PyCF_OPTIMIZED_AST if type_comments: flags |= PyCF_TYPE_COMMENTS if feature_version is None: @@ -50,7 +48,7 @@ def parse(source, filename='', mode='exec', *, feature_version = minor # Else it should be an int giving the minor version for 3.x. return compile(source, filename, mode, flags, - _feature_version=feature_version) + _feature_version=feature_version, optimize=optimize) def literal_eval(node_or_string): @@ -112,7 +110,11 @@ def _convert(node): return _convert(node_or_string) -def dump(node, annotate_fields=True, include_attributes=False, *, indent=None): +def dump( + node, annotate_fields=True, include_attributes=False, + *, + indent=None, show_empty=False, +): """ Return a formatted dump of the tree in node. This is mainly useful for debugging purposes. If annotate_fields is true (by default), @@ -123,6 +125,8 @@ def dump(node, annotate_fields=True, include_attributes=False, *, indent=None): include_attributes can be set to true. If indent is a non-negative integer or string, then the tree will be pretty-printed with that indent level. None (the default) selects the single line representation. + If show_empty is False, then empty lists and fields that are None + will be omitted from the output for better readability. """ def _format(node, level=0): if indent is not None: @@ -135,6 +139,7 @@ def _format(node, level=0): if isinstance(node, AST): cls = type(node) args = [] + args_buffer = [] allsimple = True keywords = annotate_fields for name in node._fields: @@ -146,6 +151,16 @@ def _format(node, level=0): if value is None and getattr(cls, name, ...) is None: keywords = True continue + if not show_empty: + if value == []: + field_type = cls._field_types.get(name, object) + if getattr(field_type, '__origin__', ...) is list: + if not keywords: + args_buffer.append(repr(value)) + continue + if not keywords: + args.extend(args_buffer) + args_buffer = [] value, simple = _format(value, level) allsimple = allsimple and simple if keywords: @@ -726,12 +741,11 @@ class _Unparser(NodeVisitor): output source code for the abstract syntax; original formatting is disregarded.""" - def __init__(self, *, _avoid_backslashes=False): + def __init__(self): self._source = [] self._precedences = {} self._type_ignores = {} self._indent = 0 - self._avoid_backslashes = _avoid_backslashes self._in_try_star = False def interleave(self, inter, f, seq): @@ -1104,12 +1118,21 @@ def visit_TypeVar(self, node): if node.bound: self.write(": ") self.traverse(node.bound) + if node.default_value: + self.write(" = ") + self.traverse(node.default_value) def visit_TypeVarTuple(self, node): self.write("*" + node.name) + if node.default_value: + self.write(" = ") + self.traverse(node.default_value) def visit_ParamSpec(self, node): self.write("**" + node.name) + if node.default_value: + self.write(" = ") + self.traverse(node.default_value) def visit_TypeAlias(self, node): self.fill("type ") @@ -1246,9 +1269,14 @@ def visit_JoinedStr(self, node): fallback_to_repr = True break quote_types = new_quote_types - elif "\n" in value: - quote_types = [q for q in quote_types if q in _MULTI_QUOTES] - assert quote_types + else: + if "\n" in value: + quote_types = [q for q in quote_types if q in _MULTI_QUOTES] + assert quote_types + + new_quote_types = [q for q in quote_types if q not in value] + if new_quote_types: + quote_types = new_quote_types new_fstring_parts.append(value) if fallback_to_repr: @@ -1268,13 +1296,19 @@ def visit_JoinedStr(self, node): quote_type = quote_types[0] self.write(f"{quote_type}{value}{quote_type}") - def _write_fstring_inner(self, node): + def _write_fstring_inner(self, node, is_format_spec=False): if isinstance(node, JoinedStr): # for both the f-string itself, and format_spec for value in node.values: - self._write_fstring_inner(value) + self._write_fstring_inner(value, is_format_spec=is_format_spec) elif isinstance(node, Constant) and isinstance(node.value, str): value = node.value.replace("{", "{{").replace("}", "}}") + + if is_format_spec: + value = value.replace("\\", "\\\\") + value = value.replace("'", "\\'") + value = value.replace('"', '\\"') + value = value.replace("\n", "\\n") self.write(value) elif isinstance(node, FormattedValue): self.visit_FormattedValue(node) @@ -1297,7 +1331,7 @@ def unparse_inner(inner): self.write(f"!{chr(node.conversion)}") if node.format_spec: self.write(":") - self._write_fstring_inner(node.format_spec) + self._write_fstring_inner(node.format_spec, is_format_spec=True) def visit_Name(self, node): self.write(node.id) @@ -1317,8 +1351,6 @@ def _write_constant(self, value): .replace("inf", _INFSTR) .replace("nan", f"({_INFSTR}-{_INFSTR})") ) - elif self._avoid_backslashes and isinstance(value, str): - self._write_str_avoiding_backslashes(value) else: self.write(repr(value)) @@ -1805,8 +1837,7 @@ def main(): import argparse parser = argparse.ArgumentParser(prog='python -m ast') - parser.add_argument('infile', type=argparse.FileType(mode='rb'), nargs='?', - default='-', + parser.add_argument('infile', nargs='?', default='-', help='the file to parse; defaults to stdin') parser.add_argument('-m', '--mode', default='exec', choices=('exec', 'single', 'eval', 'func_type'), @@ -1820,9 +1851,14 @@ def main(): help='indentation of nodes (number of spaces)') args = parser.parse_args() - with args.infile as infile: - source = infile.read() - tree = parse(source, args.infile.name, args.mode, type_comments=args.no_type_comments) + if args.infile == '-': + name = '' + source = sys.stdin.buffer.read() + else: + name = args.infile + with open(args.infile, 'rb') as infile: + source = infile.read() + tree = parse(source, name, args.mode, type_comments=args.no_type_comments) print(dump(tree, include_attributes=args.include_attributes, indent=args.indent)) if __name__ == '__main__': diff --git a/Lib/base64.py b/Lib/base64.py old mode 100644 new mode 100755 diff --git a/Lib/bz2.py b/Lib/bz2.py index fabe4f73c8..2420cd0190 100644 --- a/Lib/bz2.py +++ b/Lib/bz2.py @@ -17,7 +17,7 @@ from _bz2 import BZ2Compressor, BZ2Decompressor -_MODE_CLOSED = 0 +# Value 0 no longer used _MODE_READ = 1 # Value 2 no longer used _MODE_WRITE = 3 @@ -54,7 +54,7 @@ def __init__(self, filename, mode="r", *, compresslevel=9): """ self._fp = None self._closefp = False - self._mode = _MODE_CLOSED + self._mode = None if not (1 <= compresslevel <= 9): raise ValueError("compresslevel must be between 1 and 9") @@ -100,7 +100,7 @@ def close(self): May be called more than once without error. Once the file is closed, any other operation on it will raise a ValueError. """ - if self._mode == _MODE_CLOSED: + if self.closed: return try: if self._mode == _MODE_READ: @@ -115,13 +115,21 @@ def close(self): finally: self._fp = None self._closefp = False - self._mode = _MODE_CLOSED self._buffer = None @property def closed(self): """True if this file is closed.""" - return self._mode == _MODE_CLOSED + return self._fp is None + + @property + def name(self): + self._check_not_closed() + return self._fp.name + + @property + def mode(self): + return 'wb' if self._mode == _MODE_WRITE else 'rb' def fileno(self): """Return the file descriptor for the underlying file.""" diff --git a/Lib/codecs.py b/Lib/codecs.py index 82f23983e7..e4f4e1b5c0 100644 --- a/Lib/codecs.py +++ b/Lib/codecs.py @@ -111,6 +111,9 @@ def __repr__(self): (self.__class__.__module__, self.__class__.__qualname__, self.name, id(self)) + def __getnewargs__(self): + return tuple(self) + class Codec: """ Defines the interface for stateless encoders/decoders. @@ -615,7 +618,7 @@ def readlines(self, sizehint=None, keepends=True): method and are included in the list entries. sizehint, if given, is ignored since there is no efficient - way to finding the true end-of-line. + way of finding the true end-of-line. """ data = self.read() @@ -706,13 +709,13 @@ def read(self, size=-1): return self.reader.read(size) - def readline(self, size=None): + def readline(self, size=None, keepends=True): - return self.reader.readline(size) + return self.reader.readline(size, keepends) - def readlines(self, sizehint=None): + def readlines(self, sizehint=None, keepends=True): - return self.reader.readlines(sizehint) + return self.reader.readlines(sizehint, keepends) def __next__(self): diff --git a/Lib/codeop.py b/Lib/codeop.py index eea6cbc701..adf000ba29 100644 --- a/Lib/codeop.py +++ b/Lib/codeop.py @@ -44,6 +44,7 @@ # Caveat emptor: These flags are undocumented on purpose and depending # on their effect outside the standard library is **unsupported**. PyCF_DONT_IMPLY_DEDENT = 0x200 +PyCF_ONLY_AST = 0x400 PyCF_ALLOW_INCOMPLETE_INPUT = 0x4000 def _maybe_compile(compiler, source, filename, symbol): @@ -73,15 +74,6 @@ def _maybe_compile(compiler, source, filename, symbol): return compiler(source, filename, symbol, incomplete_input=False) -def _is_syntax_error(err1, err2): - rep1 = repr(err1) - rep2 = repr(err2) - if "was never closed" in rep1 and "was never closed" in rep2: - return False - if rep1 == rep2: - return True - return False - def _compile(source, filename, symbol, incomplete_input=True): flags = 0 if incomplete_input: @@ -89,7 +81,6 @@ def _compile(source, filename, symbol, incomplete_input=True): flags |= PyCF_DONT_IMPLY_DEDENT return compile(source, filename, symbol, flags) - def compile_command(source, filename="", symbol="single"): r"""Compile a command and determine whether it is incomplete. @@ -119,12 +110,14 @@ class Compile: def __init__(self): self.flags = PyCF_DONT_IMPLY_DEDENT | PyCF_ALLOW_INCOMPLETE_INPUT - def __call__(self, source, filename, symbol, **kwargs): - flags = self.flags + def __call__(self, source, filename, symbol, flags=0, **kwargs): + flags |= self.flags if kwargs.get('incomplete_input', True) is False: flags &= ~PyCF_DONT_IMPLY_DEDENT flags &= ~PyCF_ALLOW_INCOMPLETE_INPUT codeob = compile(source, filename, symbol, flags, True) + if flags & PyCF_ONLY_AST: + return codeob # this is an ast.Module in this case for feature in _features: if codeob.co_flags & feature.compiler_flag: self.flags |= feature.compiler_flag diff --git a/Lib/configparser.py b/Lib/configparser.py index e8aae21794..05b86acb91 100644 --- a/Lib/configparser.py +++ b/Lib/configparser.py @@ -18,8 +18,8 @@ delimiters=('=', ':'), comment_prefixes=('#', ';'), inline_comment_prefixes=None, strict=True, empty_lines_in_values=True, default_section='DEFAULT', - interpolation=, converters=): - + interpolation=, converters=, + allow_unnamed_section=False): Create the parser. When `defaults` is given, it is initialized into the dictionary or intrinsic defaults. The keys must be strings, the values must be appropriate for %()s string interpolation. @@ -68,6 +68,10 @@ converter gets its corresponding get*() method on the parser object and section proxies. + When `allow_unnamed_section` is True (default: False), options + without section are accepted: the section for these is + ``configparser.UNNAMED_SECTION``. + sections() Return all the configuration section names, sans DEFAULT. @@ -139,24 +143,28 @@ between keys and values are surrounded by spaces. """ -from collections.abc import MutableMapping +# Do not import dataclasses; overhead is unacceptable (gh-117703) + +from collections.abc import Iterable, MutableMapping from collections import ChainMap as _ChainMap +import contextlib import functools import io import itertools import os import re import sys -import warnings +import types __all__ = ("NoSectionError", "DuplicateOptionError", "DuplicateSectionError", "NoOptionError", "InterpolationError", "InterpolationDepthError", "InterpolationMissingOptionError", "InterpolationSyntaxError", "ParsingError", "MissingSectionHeaderError", + "MultilineContinuationError", "ConfigParser", "RawConfigParser", "Interpolation", "BasicInterpolation", "ExtendedInterpolation", - "LegacyInterpolation", "SectionProxy", "ConverterMapping", - "DEFAULTSECT", "MAX_INTERPOLATION_DEPTH") + "SectionProxy", "ConverterMapping", + "DEFAULTSECT", "MAX_INTERPOLATION_DEPTH", "UNNAMED_SECTION") _default_dict = dict DEFAULTSECT = "DEFAULT" @@ -298,15 +306,33 @@ def __init__(self, option, section, rawval): class ParsingError(Error): """Raised when a configuration file does not follow legal syntax.""" - def __init__(self, source): + def __init__(self, source, *args): super().__init__(f'Source contains parsing errors: {source!r}') self.source = source self.errors = [] self.args = (source, ) + if args: + self.append(*args) def append(self, lineno, line): self.errors.append((lineno, line)) - self.message += '\n\t[line %2d]: %s' % (lineno, line) + self.message += '\n\t[line %2d]: %s' % (lineno, repr(line)) + + def combine(self, others): + for other in others: + for error in other.errors: + self.append(*error) + return self + + @staticmethod + def _raise_all(exceptions: Iterable['ParsingError']): + """ + Combine any number of ParsingErrors into one and raise it. + """ + exceptions = iter(exceptions) + with contextlib.suppress(StopIteration): + raise next(exceptions).combine(exceptions) + class MissingSectionHeaderError(ParsingError): @@ -323,6 +349,28 @@ def __init__(self, filename, lineno, line): self.args = (filename, lineno, line) +class MultilineContinuationError(ParsingError): + """Raised when a key without value is followed by continuation line""" + def __init__(self, filename, lineno, line): + Error.__init__( + self, + "Key without value continued with an indented line.\n" + "file: %r, line: %d\n%r" + %(filename, lineno, line)) + self.source = filename + self.lineno = lineno + self.line = line + self.args = (filename, lineno, line) + +class _UnnamedSection: + + def __repr__(self): + return "" + + +UNNAMED_SECTION = _UnnamedSection() + + # Used in parser getters to indicate the default behaviour when a specific # option is not found it to raise an exception. Created to enable `None` as # a valid fallback value. @@ -478,6 +526,8 @@ def _interpolate_some(self, parser, option, accum, rest, section, map, except (KeyError, NoSectionError, NoOptionError): raise InterpolationMissingOptionError( option, section, rawval, ":".join(path)) from None + if v is None: + continue if "$" in v: self._interpolate_some(parser, opt, accum, v, sect, dict(parser.items(sect, raw=True)), @@ -491,51 +541,50 @@ def _interpolate_some(self, parser, option, accum, rest, section, map, "found: %r" % (rest,)) -class LegacyInterpolation(Interpolation): - """Deprecated interpolation used in old versions of ConfigParser. - Use BasicInterpolation or ExtendedInterpolation instead.""" +class _ReadState: + elements_added : set[str] + cursect : dict[str, str] | None = None + sectname : str | None = None + optname : str | None = None + lineno : int = 0 + indent_level : int = 0 + errors : list[ParsingError] - _KEYCRE = re.compile(r"%\(([^)]*)\)s|.") + def __init__(self): + self.elements_added = set() + self.errors = list() - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - warnings.warn( - "LegacyInterpolation has been deprecated since Python 3.2 " - "and will be removed from the configparser module in Python 3.13. " - "Use BasicInterpolation or ExtendedInterpolation instead.", - DeprecationWarning, stacklevel=2 - ) - def before_get(self, parser, section, option, value, vars): - rawval = value - depth = MAX_INTERPOLATION_DEPTH - while depth: # Loop through this until it's done - depth -= 1 - if value and "%(" in value: - replace = functools.partial(self._interpolation_replace, - parser=parser) - value = self._KEYCRE.sub(replace, value) - try: - value = value % vars - except KeyError as e: - raise InterpolationMissingOptionError( - option, section, rawval, e.args[0]) from None - else: - break - if value and "%(" in value: - raise InterpolationDepthError(option, section, rawval) - return value +class _Line(str): - def before_set(self, parser, section, option, value): - return value + def __new__(cls, val, *args, **kwargs): + return super().__new__(cls, val) - @staticmethod - def _interpolation_replace(match, parser): - s = match.group(1) - if s is None: - return match.group() - else: - return "%%(%s)s" % parser.optionxform(s) + def __init__(self, val, prefixes): + self.prefixes = prefixes + + @functools.cached_property + def clean(self): + return self._strip_full() and self._strip_inline() + + @property + def has_comments(self): + return self.strip() != self.clean + + def _strip_inline(self): + """ + Search for the earliest prefix at the beginning of the line or following a space. + """ + matcher = re.compile( + '|'.join(fr'(^|\s)({re.escape(prefix)})' for prefix in self.prefixes.inline) + # match nothing if no prefixes + or '(?!)' + ) + match = matcher.search(self) + return self[:match.start() if match else None].strip() + + def _strip_full(self): + return '' if any(map(self.strip().startswith, self.prefixes.full)) else True class RawConfigParser(MutableMapping): @@ -584,7 +633,8 @@ def __init__(self, defaults=None, dict_type=_default_dict, comment_prefixes=('#', ';'), inline_comment_prefixes=None, strict=True, empty_lines_in_values=True, default_section=DEFAULTSECT, - interpolation=_UNSET, converters=_UNSET): + interpolation=_UNSET, converters=_UNSET, + allow_unnamed_section=False,): self._dict = dict_type self._sections = self._dict() @@ -603,8 +653,10 @@ def __init__(self, defaults=None, dict_type=_default_dict, else: self._optcre = re.compile(self._OPT_TMPL.format(delim=d), re.VERBOSE) - self._comment_prefixes = tuple(comment_prefixes or ()) - self._inline_comment_prefixes = tuple(inline_comment_prefixes or ()) + self._prefixes = types.SimpleNamespace( + full=tuple(comment_prefixes or ()), + inline=tuple(inline_comment_prefixes or ()), + ) self._strict = strict self._allow_no_value = allow_no_value self._empty_lines_in_values = empty_lines_in_values @@ -623,6 +675,7 @@ def __init__(self, defaults=None, dict_type=_default_dict, self._converters.update(converters) if defaults: self._read_defaults(defaults) + self._allow_unnamed_section = allow_unnamed_section def defaults(self): return self._defaults @@ -896,13 +949,19 @@ def write(self, fp, space_around_delimiters=True): if self._defaults: self._write_section(fp, self.default_section, self._defaults.items(), d) + if UNNAMED_SECTION in self._sections: + self._write_section(fp, UNNAMED_SECTION, self._sections[UNNAMED_SECTION].items(), d, unnamed=True) + for section in self._sections: + if section is UNNAMED_SECTION: + continue self._write_section(fp, section, self._sections[section].items(), d) - def _write_section(self, fp, section_name, section_items, delimiter): - """Write a single section to the specified `fp`.""" - fp.write("[{}]\n".format(section_name)) + def _write_section(self, fp, section_name, section_items, delimiter, unnamed=False): + """Write a single section to the specified `fp'.""" + if not unnamed: + fp.write("[{}]\n".format(section_name)) for key, value in section_items: value = self._interpolation.before_write(self, section_name, key, value) @@ -988,110 +1047,113 @@ def _read(self, fp, fpname): in an otherwise empty line or may be entered in lines holding values or section names. Please note that comments get stripped off when reading configuration files. """ - elements_added = set() - cursect = None # None, or a dictionary - sectname = None - optname = None - lineno = 0 - indent_level = 0 - e = None # None, or an exception - for lineno, line in enumerate(fp, start=1): - comment_start = sys.maxsize - # strip inline comments - inline_prefixes = {p: -1 for p in self._inline_comment_prefixes} - while comment_start == sys.maxsize and inline_prefixes: - next_prefixes = {} - for prefix, index in inline_prefixes.items(): - index = line.find(prefix, index+1) - if index == -1: - continue - next_prefixes[prefix] = index - if index == 0 or (index > 0 and line[index-1].isspace()): - comment_start = min(comment_start, index) - inline_prefixes = next_prefixes - # strip full line comments - for prefix in self._comment_prefixes: - if line.strip().startswith(prefix): - comment_start = 0 - break - if comment_start == sys.maxsize: - comment_start = None - value = line[:comment_start].strip() - if not value: + + try: + ParsingError._raise_all(self._read_inner(fp, fpname)) + finally: + self._join_multiline_values() + + def _read_inner(self, fp, fpname): + st = _ReadState() + + Line = functools.partial(_Line, prefixes=self._prefixes) + for st.lineno, line in enumerate(map(Line, fp), start=1): + if not line.clean: if self._empty_lines_in_values: # add empty line to the value, but only if there was no # comment on the line - if (comment_start is None and - cursect is not None and - optname and - cursect[optname] is not None): - cursect[optname].append('') # newlines added at join + if (not line.has_comments and + st.cursect is not None and + st.optname and + st.cursect[st.optname] is not None): + st.cursect[st.optname].append('') # newlines added at join else: # empty line marks end of value - indent_level = sys.maxsize + st.indent_level = sys.maxsize continue - # continuation line? + first_nonspace = self.NONSPACECRE.search(line) - cur_indent_level = first_nonspace.start() if first_nonspace else 0 - if (cursect is not None and optname and - cur_indent_level > indent_level): - cursect[optname].append(value) - # a section header or option header? - else: - indent_level = cur_indent_level - # is it a section header? - mo = self.SECTCRE.match(value) - if mo: - sectname = mo.group('header') - if sectname in self._sections: - if self._strict and sectname in elements_added: - raise DuplicateSectionError(sectname, fpname, - lineno) - cursect = self._sections[sectname] - elements_added.add(sectname) - elif sectname == self.default_section: - cursect = self._defaults - else: - cursect = self._dict() - self._sections[sectname] = cursect - self._proxies[sectname] = SectionProxy(self, sectname) - elements_added.add(sectname) - # So sections can't start with a continuation line - optname = None - # no section header in the file? - elif cursect is None: - raise MissingSectionHeaderError(fpname, lineno, line) - # an option line? - else: - mo = self._optcre.match(value) - if mo: - optname, vi, optval = mo.group('option', 'vi', 'value') - if not optname: - e = self._handle_error(e, fpname, lineno, line) - optname = self.optionxform(optname.rstrip()) - if (self._strict and - (sectname, optname) in elements_added): - raise DuplicateOptionError(sectname, optname, - fpname, lineno) - elements_added.add((sectname, optname)) - # This check is fine because the OPTCRE cannot - # match if it would set optval to None - if optval is not None: - optval = optval.strip() - cursect[optname] = [optval] - else: - # valueless option handling - cursect[optname] = None - else: - # a non-fatal parsing error occurred. set up the - # exception but keep going. the exception will be - # raised at the end of the file and will contain a - # list of all bogus lines - e = self._handle_error(e, fpname, lineno, line) - self._join_multiline_values() - # if any parsing errors occurred, raise an exception - if e: - raise e + st.cur_indent_level = first_nonspace.start() if first_nonspace else 0 + + if self._handle_continuation_line(st, line, fpname): + continue + + self._handle_rest(st, line, fpname) + + return st.errors + + def _handle_continuation_line(self, st, line, fpname): + # continuation line? + is_continue = (st.cursect is not None and st.optname and + st.cur_indent_level > st.indent_level) + if is_continue: + if st.cursect[st.optname] is None: + raise MultilineContinuationError(fpname, st.lineno, line) + st.cursect[st.optname].append(line.clean) + return is_continue + + def _handle_rest(self, st, line, fpname): + # a section header or option header? + if self._allow_unnamed_section and st.cursect is None: + self._handle_header(st, UNNAMED_SECTION, fpname) + + st.indent_level = st.cur_indent_level + # is it a section header? + mo = self.SECTCRE.match(line.clean) + + if not mo and st.cursect is None: + raise MissingSectionHeaderError(fpname, st.lineno, line) + + self._handle_header(st, mo.group('header'), fpname) if mo else self._handle_option(st, line, fpname) + + def _handle_header(self, st, sectname, fpname): + st.sectname = sectname + if st.sectname in self._sections: + if self._strict and st.sectname in st.elements_added: + raise DuplicateSectionError(st.sectname, fpname, + st.lineno) + st.cursect = self._sections[st.sectname] + st.elements_added.add(st.sectname) + elif st.sectname == self.default_section: + st.cursect = self._defaults + else: + st.cursect = self._dict() + self._sections[st.sectname] = st.cursect + self._proxies[st.sectname] = SectionProxy(self, st.sectname) + st.elements_added.add(st.sectname) + # So sections can't start with a continuation line + st.optname = None + + def _handle_option(self, st, line, fpname): + # an option line? + st.indent_level = st.cur_indent_level + + mo = self._optcre.match(line.clean) + if not mo: + # a non-fatal parsing error occurred. set up the + # exception but keep going. the exception will be + # raised at the end of the file and will contain a + # list of all bogus lines + st.errors.append(ParsingError(fpname, st.lineno, line)) + return + + st.optname, vi, optval = mo.group('option', 'vi', 'value') + if not st.optname: + st.errors.append(ParsingError(fpname, st.lineno, line)) + st.optname = self.optionxform(st.optname.rstrip()) + if (self._strict and + (st.sectname, st.optname) in st.elements_added): + raise DuplicateOptionError(st.sectname, st.optname, + fpname, st.lineno) + st.elements_added.add((st.sectname, st.optname)) + # This check is fine because the OPTCRE cannot + # match if it would set optval to None + if optval is not None: + optval = optval.strip() + st.cursect[st.optname] = [optval] + else: + # valueless option handling + st.cursect[st.optname] = None def _join_multiline_values(self): defaults = self.default_section, self._defaults @@ -1111,12 +1173,6 @@ def _read_defaults(self, defaults): for key, value in defaults.items(): self._defaults[self.optionxform(key)] = value - def _handle_error(self, exc, fpname, lineno, line): - if not exc: - exc = ParsingError(fpname) - exc.append(lineno, repr(line)) - return exc - def _unify_values(self, section, vars): """Create a sequence of lookups with 'vars' taking priority over the 'section' which takes priority over the DEFAULTSECT. diff --git a/Lib/contextlib.py b/Lib/contextlib.py index b831d8916c..5b646fabca 100644 --- a/Lib/contextlib.py +++ b/Lib/contextlib.py @@ -20,6 +20,8 @@ class AbstractContextManager(abc.ABC): __class_getitem__ = classmethod(GenericAlias) + __slots__ = () + def __enter__(self): """Return `self` upon entering the runtime context.""" return self @@ -42,6 +44,8 @@ class AbstractAsyncContextManager(abc.ABC): __class_getitem__ = classmethod(GenericAlias) + __slots__ = () + async def __aenter__(self): """Return `self` upon entering the runtime context.""" return self @@ -565,11 +569,12 @@ def __enter__(self): return self def __exit__(self, *exc_details): - received_exc = exc_details[0] is not None + exc = exc_details[1] + received_exc = exc is not None # We manipulate the exception state so it behaves as though # we were actually nesting multiple with statements - frame_exc = sys.exc_info()[1] + frame_exc = sys.exception() def _fix_exception_context(new_exc, old_exc): # Context may not be correct, so find the end of the chain while 1: @@ -592,24 +597,28 @@ def _fix_exception_context(new_exc, old_exc): is_sync, cb = self._exit_callbacks.pop() assert is_sync try: + if exc is None: + exc_details = None, None, None + else: + exc_details = type(exc), exc, exc.__traceback__ if cb(*exc_details): suppressed_exc = True pending_raise = False - exc_details = (None, None, None) - except: - new_exc_details = sys.exc_info() + exc = None + except BaseException as new_exc: # simulate the stack of exceptions by setting the context - _fix_exception_context(new_exc_details[1], exc_details[1]) + _fix_exception_context(new_exc, exc) pending_raise = True - exc_details = new_exc_details + exc = new_exc + if pending_raise: try: - # bare "raise exc_details[1]" replaces our carefully + # bare "raise exc" replaces our carefully # set-up context - fixed_ctx = exc_details[1].__context__ - raise exc_details[1] + fixed_ctx = exc.__context__ + raise exc except BaseException: - exc_details[1].__context__ = fixed_ctx + exc.__context__ = fixed_ctx raise return received_exc and suppressed_exc @@ -705,11 +714,12 @@ async def __aenter__(self): return self async def __aexit__(self, *exc_details): - received_exc = exc_details[0] is not None + exc = exc_details[1] + received_exc = exc is not None # We manipulate the exception state so it behaves as though # we were actually nesting multiple with statements - frame_exc = sys.exc_info()[1] + frame_exc = sys.exception() def _fix_exception_context(new_exc, old_exc): # Context may not be correct, so find the end of the chain while 1: @@ -731,6 +741,10 @@ def _fix_exception_context(new_exc, old_exc): while self._exit_callbacks: is_sync, cb = self._exit_callbacks.pop() try: + if exc is None: + exc_details = None, None, None + else: + exc_details = type(exc), exc, exc.__traceback__ if is_sync: cb_suppress = cb(*exc_details) else: @@ -739,21 +753,21 @@ def _fix_exception_context(new_exc, old_exc): if cb_suppress: suppressed_exc = True pending_raise = False - exc_details = (None, None, None) - except: - new_exc_details = sys.exc_info() + exc = None + except BaseException as new_exc: # simulate the stack of exceptions by setting the context - _fix_exception_context(new_exc_details[1], exc_details[1]) + _fix_exception_context(new_exc, exc) pending_raise = True - exc_details = new_exc_details + exc = new_exc + if pending_raise: try: - # bare "raise exc_details[1]" replaces our carefully + # bare "raise exc" replaces our carefully # set-up context - fixed_ctx = exc_details[1].__context__ - raise exc_details[1] + fixed_ctx = exc.__context__ + raise exc except BaseException: - exc_details[1].__context__ = fixed_ctx + exc.__context__ = fixed_ctx raise return received_exc and suppressed_exc diff --git a/Lib/csv.py b/Lib/csv.py index 77f30c8d2b..cd20265987 100644 --- a/Lib/csv.py +++ b/Lib/csv.py @@ -1,28 +1,90 @@ -""" -csv.py - read/write/investigate CSV files +r""" +CSV parsing and writing. + +This module provides classes that assist in the reading and writing +of Comma Separated Value (CSV) files, and implements the interface +described by PEP 305. Although many CSV files are simple to parse, +the format is not formally defined by a stable specification and +is subtle enough that parsing lines of a CSV file with something +like line.split(",") is bound to fail. The module supports three +basic APIs: reading, writing, and registration of dialects. + + +DIALECT REGISTRATION: + +Readers and writers support a dialect argument, which is a convenient +handle on a group of settings. When the dialect argument is a string, +it identifies one of the dialects previously registered with the module. +If it is a class or instance, the attributes of the argument are used as +the settings for the reader or writer: + + class excel: + delimiter = ',' + quotechar = '"' + escapechar = None + doublequote = True + skipinitialspace = False + lineterminator = '\r\n' + quoting = QUOTE_MINIMAL + +SETTINGS: + + * quotechar - specifies a one-character string to use as the + quoting character. It defaults to '"'. + * delimiter - specifies a one-character string to use as the + field separator. It defaults to ','. + * skipinitialspace - specifies how to interpret spaces which + immediately follow a delimiter. It defaults to False, which + means that spaces immediately following a delimiter is part + of the following field. + * lineterminator - specifies the character sequence which should + terminate rows. + * quoting - controls when quotes should be generated by the writer. + It can take on any of the following module constants: + + csv.QUOTE_MINIMAL means only when required, for example, when a + field contains either the quotechar or the delimiter + csv.QUOTE_ALL means that quotes are always placed around fields. + csv.QUOTE_NONNUMERIC means that quotes are always placed around + fields which do not parse as integers or floating-point + numbers. + csv.QUOTE_STRINGS means that quotes are always placed around + fields which are strings. Note that the Python value None + is not a string. + csv.QUOTE_NOTNULL means that quotes are only placed around fields + that are not the Python value None. + csv.QUOTE_NONE means that quotes are never placed around fields. + * escapechar - specifies a one-character string used to escape + the delimiter when quoting is set to QUOTE_NONE. + * doublequote - controls the handling of quotes inside fields. When + True, two consecutive quotes are interpreted as one during read, + and when writing, each quote character embedded in the data is + written as two quotes """ import re import types -from _csv import Error, __version__, writer, reader, register_dialect, \ +from _csv import Error, writer, reader, register_dialect, \ unregister_dialect, get_dialect, list_dialects, \ field_size_limit, \ QUOTE_MINIMAL, QUOTE_ALL, QUOTE_NONNUMERIC, QUOTE_NONE, \ - QUOTE_STRINGS, QUOTE_NOTNULL, \ - __doc__ + QUOTE_STRINGS, QUOTE_NOTNULL from _csv import Dialect as _Dialect from io import StringIO __all__ = ["QUOTE_MINIMAL", "QUOTE_ALL", "QUOTE_NONNUMERIC", "QUOTE_NONE", "QUOTE_STRINGS", "QUOTE_NOTNULL", - "Error", "Dialect", "__doc__", "excel", "excel_tab", + "Error", "Dialect", "excel", "excel_tab", "field_size_limit", "reader", "writer", "register_dialect", "get_dialect", "list_dialects", "Sniffer", - "unregister_dialect", "__version__", "DictReader", "DictWriter", + "unregister_dialect", "DictReader", "DictWriter", "unix_dialect"] +__version__ = "1.0" + + class Dialect: """Describe a CSV dialect. @@ -51,8 +113,8 @@ def _validate(self): try: _Dialect(self) except TypeError as e: - # We do this for compatibility with py2.3 - raise Error(str(e)) + # Re-raise to get a traceback showing more user code. + raise Error(str(e)) from None class excel(Dialect): """Describe the usual properties of Excel-generated CSV files.""" diff --git a/Lib/ctypes/__init__.py b/Lib/ctypes/__init__.py index b8b005061f..80651dc64c 100644 --- a/Lib/ctypes/__init__.py +++ b/Lib/ctypes/__init__.py @@ -1,6 +1,8 @@ """create and manipulate C data types in Python""" -import os as _os, sys as _sys +import os as _os +import sys as _sys +import sysconfig as _sysconfig import types as _types __version__ = "1.1.0" @@ -36,9 +38,6 @@ FUNCFLAG_USE_ERRNO as _FUNCFLAG_USE_ERRNO, \ FUNCFLAG_USE_LASTERROR as _FUNCFLAG_USE_LASTERROR -# TODO: RUSTPYTHON remove this -from _ctypes import _non_existing_function - # WINOLEAPI -> HRESULT # WINOLEAPI_(type) # @@ -110,7 +109,7 @@ class CFunctionType(_CFuncPtr): return CFunctionType if _os.name == "nt": - from _ctypes import LoadLibrary as _dlopen + from _ctypes import LoadLibrary as _LoadLibrary from _ctypes import FUNCFLAG_STDCALL as _FUNCFLAG_STDCALL _win_functype_cache = {} @@ -299,16 +298,15 @@ def create_unicode_buffer(init, size=None): return buf elif isinstance(init, int): _sys.audit("ctypes.create_unicode_buffer", None, init) - # XXX: RUSTPYTHON - # buftype = c_wchar * init - buftype = c_wchar.__mul__(init) + buftype = c_wchar * init buf = buftype() return buf raise TypeError(init) -# XXX Deprecated def SetPointerType(pointer, cls): + import warnings + warnings._deprecated("ctypes.SetPointerType", remove=(3, 15)) if _pointer_type_cache.get(cls, None) is not None: raise RuntimeError("This type already exists in the cache") if id(pointer) not in _pointer_type_cache: @@ -317,7 +315,6 @@ def SetPointerType(pointer, cls): _pointer_type_cache[cls] = pointer del _pointer_type_cache[id(pointer)] -# XXX Deprecated def ARRAY(typ, len): return typ * len @@ -349,39 +346,59 @@ def __init__(self, name, mode=DEFAULT_MODE, handle=None, use_errno=False, use_last_error=False, winmode=None): - self._name = name - flags = self._func_flags_ - if use_errno: - flags |= _FUNCFLAG_USE_ERRNO - if use_last_error: - flags |= _FUNCFLAG_USE_LASTERROR - if _sys.platform.startswith("aix"): - """When the name contains ".a(" and ends with ")", - e.g., "libFOO.a(libFOO.so)" - this is taken to be an - archive(member) syntax for dlopen(), and the mode is adjusted. - Otherwise, name is presented to dlopen() as a file argument. - """ - if name and name.endswith(")") and ".a(" in name: - mode |= ( _os.RTLD_MEMBER | _os.RTLD_NOW ) - if _os.name == "nt": - if winmode is not None: - mode = winmode - else: - import nt - mode = nt._LOAD_LIBRARY_SEARCH_DEFAULT_DIRS - if '/' in name or '\\' in name: - self._name = nt._getfullpathname(self._name) - mode |= nt._LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR - class _FuncPtr(_CFuncPtr): - _flags_ = flags + _flags_ = self._func_flags_ _restype_ = self._func_restype_ + if use_errno: + _flags_ |= _FUNCFLAG_USE_ERRNO + if use_last_error: + _flags_ |= _FUNCFLAG_USE_LASTERROR + self._FuncPtr = _FuncPtr + if name: + name = _os.fspath(name) - if handle is None: - self._handle = _dlopen(self._name, mode) - else: - self._handle = handle + self._handle = self._load_library(name, mode, handle, winmode) + + if _os.name == "nt": + def _load_library(self, name, mode, handle, winmode): + if winmode is None: + import nt as _nt + winmode = _nt._LOAD_LIBRARY_SEARCH_DEFAULT_DIRS + # WINAPI LoadLibrary searches for a DLL if the given name + # is not fully qualified with an explicit drive. For POSIX + # compatibility, and because the DLL search path no longer + # contains the working directory, begin by fully resolving + # any name that contains a path separator. + if name is not None and ('/' in name or '\\' in name): + name = _nt._getfullpathname(name) + winmode |= _nt._LOAD_LIBRARY_SEARCH_DLL_LOAD_DIR + self._name = name + if handle is not None: + return handle + return _LoadLibrary(self._name, winmode) + + else: + def _load_library(self, name, mode, handle, winmode): + # If the filename that has been provided is an iOS/tvOS/watchOS + # .fwork file, dereference the location to the true origin of the + # binary. + if name and name.endswith(".fwork"): + with open(name) as f: + name = _os.path.join( + _os.path.dirname(_sys.executable), + f.read().strip() + ) + if _sys.platform.startswith("aix"): + """When the name contains ".a(" and ends with ")", + e.g., "libFOO.a(libFOO.so)" - this is taken to be an + archive(member) syntax for dlopen(), and the mode is adjusted. + Otherwise, name is presented to dlopen() as a file argument. + """ + if name and name.endswith(")") and ".a(" in name: + mode |= _os.RTLD_MEMBER | _os.RTLD_NOW + self._name = name + return _dlopen(name, mode) def __repr__(self): return "<%s '%s', handle %x at %#x>" % \ @@ -469,8 +486,9 @@ def LoadLibrary(self, name): if _os.name == "nt": pythonapi = PyDLL("python dll", None, _sys.dllhandle) -elif _sys.platform == "cygwin": - pythonapi = PyDLL("libpython%d.%d.dll" % _sys.version_info[:2]) +elif _sys.platform in ["android", "cygwin"]: + # These are Unix-like platforms which use a dynamically-linked libpython. + pythonapi = PyDLL(_sysconfig.get_config_var("LDLIBRARY")) else: pythonapi = PyDLL(None) @@ -500,15 +518,14 @@ def WinError(code=None, descr=None): c_ssize_t = c_longlong # functions + from _ctypes import _memmove_addr, _memset_addr, _string_at_addr, _cast_addr ## void *memmove(void *, const void *, size_t); -# XXX: RUSTPYTHON -# memmove = CFUNCTYPE(c_void_p, c_void_p, c_void_p, c_size_t)(_memmove_addr) +memmove = CFUNCTYPE(c_void_p, c_void_p, c_void_p, c_size_t)(_memmove_addr) ## void *memset(void *, int, size_t) -# XXX: RUSTPYTHON -# memset = CFUNCTYPE(c_void_p, c_void_p, c_int, c_size_t)(_memset_addr) +memset = CFUNCTYPE(c_void_p, c_void_p, c_int, c_size_t)(_memset_addr) def PYFUNCTYPE(restype, *argtypes): class CFunctionType(_CFuncPtr): @@ -517,17 +534,15 @@ class CFunctionType(_CFuncPtr): _flags_ = _FUNCFLAG_CDECL | _FUNCFLAG_PYTHONAPI return CFunctionType -# XXX: RUSTPYTHON -# _cast = PYFUNCTYPE(py_object, c_void_p, py_object, py_object)(_cast_addr) +_cast = PYFUNCTYPE(py_object, c_void_p, py_object, py_object)(_cast_addr) def cast(obj, typ): return _cast(obj, obj, typ) -# XXX: RUSTPYTHON -# _string_at = PYFUNCTYPE(py_object, c_void_p, c_int)(_string_at_addr) +_string_at = PYFUNCTYPE(py_object, c_void_p, c_int)(_string_at_addr) def string_at(ptr, size=-1): - """string_at(addr[, size]) -> string + """string_at(ptr[, size]) -> string - Return the string at addr.""" + Return the byte string at void *ptr.""" return _string_at(ptr, size) try: @@ -535,12 +550,11 @@ def string_at(ptr, size=-1): except ImportError: pass else: - # XXX: RUSTPYTHON - # _wstring_at = PYFUNCTYPE(py_object, c_void_p, c_int)(_wstring_at_addr) + _wstring_at = PYFUNCTYPE(py_object, c_void_p, c_int)(_wstring_at_addr) def wstring_at(ptr, size=-1): - """wstring_at(addr[, size]) -> string + """wstring_at(ptr[, size]) -> string - Return the string at addr.""" + Return the wide-character string at void *ptr.""" return _wstring_at(ptr, size) diff --git a/Lib/ctypes/_endian.py b/Lib/ctypes/_endian.py index 34dee64b1a..6382dd22b8 100644 --- a/Lib/ctypes/_endian.py +++ b/Lib/ctypes/_endian.py @@ -1,5 +1,5 @@ import sys -from ctypes import * +from ctypes import Array, Structure, Union _array_type = type(Array) @@ -15,8 +15,8 @@ def _other_endian(typ): # if typ is array if isinstance(typ, _array_type): return _other_endian(typ._type_) * typ._length_ - # if typ is structure - if issubclass(typ, Structure): + # if typ is structure or union + if issubclass(typ, (Structure, Union)): return typ raise TypeError("This type does not support other endian: %s" % typ) @@ -37,7 +37,7 @@ class _swapped_union_meta(_swapped_meta, type(Union)): pass ################################################################ # Note: The Structure metaclass checks for the *presence* (not the -# value!) of a _swapped_bytes_ attribute to determine the bit order in +# value!) of a _swappedbytes_ attribute to determine the bit order in # structures containing bit fields. if sys.byteorder == "little": diff --git a/Lib/ctypes/test/__init__.py b/Lib/ctypes/test/__init__.py deleted file mode 100644 index 6e496fa5a5..0000000000 --- a/Lib/ctypes/test/__init__.py +++ /dev/null @@ -1,16 +0,0 @@ -import os -import unittest -from test import support -from test.support import import_helper - - -# skip tests if _ctypes was not built -ctypes = import_helper.import_module('ctypes') -ctypes_symbols = dir(ctypes) - -def need_symbol(name): - return unittest.skipUnless(name in ctypes_symbols, - '{!r} is required'.format(name)) - -def load_tests(*args): - return support.load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/ctypes/test/__main__.py b/Lib/ctypes/test/__main__.py deleted file mode 100644 index 362a9ec8cf..0000000000 --- a/Lib/ctypes/test/__main__.py +++ /dev/null @@ -1,4 +0,0 @@ -from ctypes.test import load_tests -import unittest - -unittest.main() diff --git a/Lib/ctypes/test/test_delattr.py b/Lib/ctypes/test/test_delattr.py deleted file mode 100644 index 0f4d58691b..0000000000 --- a/Lib/ctypes/test/test_delattr.py +++ /dev/null @@ -1,21 +0,0 @@ -import unittest -from ctypes import * - -class X(Structure): - _fields_ = [("foo", c_int)] - -class TestCase(unittest.TestCase): - def test_simple(self): - self.assertRaises(TypeError, - delattr, c_int(42), "value") - - def test_chararray(self): - self.assertRaises(TypeError, - delattr, (c_char * 5)(), "value") - - def test_struct(self): - self.assertRaises(TypeError, - delattr, X(), "foo") - -if __name__ == "__main__": - unittest.main() diff --git a/Lib/ctypes/test/test_simplesubclasses.py b/Lib/ctypes/test/test_simplesubclasses.py deleted file mode 100644 index 3da2794a79..0000000000 --- a/Lib/ctypes/test/test_simplesubclasses.py +++ /dev/null @@ -1,55 +0,0 @@ -import unittest -from ctypes import * - -class MyInt(c_int): - def __eq__(self, other): - if type(other) != MyInt: - return NotImplementedError - return self.value == other.value - -class Test(unittest.TestCase): - - def test_compare(self): - self.assertEqual(MyInt(3), MyInt(3)) - self.assertNotEqual(MyInt(42), MyInt(43)) - - def test_ignore_retval(self): - # Test if the return value of a callback is ignored - # if restype is None - proto = CFUNCTYPE(None) - def func(): - return (1, "abc", None) - - cb = proto(func) - self.assertEqual(None, cb()) - - - def test_int_callback(self): - args = [] - def func(arg): - args.append(arg) - return arg - - cb = CFUNCTYPE(None, MyInt)(func) - - self.assertEqual(None, cb(42)) - self.assertEqual(type(args[-1]), MyInt) - - cb = CFUNCTYPE(c_int, c_int)(func) - - self.assertEqual(42, cb(42)) - self.assertEqual(type(args[-1]), int) - - def test_int_struct(self): - class X(Structure): - _fields_ = [("x", MyInt)] - - self.assertEqual(X().x, MyInt()) - - s = X() - s.x = MyInt(42) - - self.assertEqual(s.x, MyInt(42)) - -if __name__ == "__main__": - unittest.main() diff --git a/Lib/ctypes/util.py b/Lib/ctypes/util.py index 0c2510e161..117bf06cb0 100644 --- a/Lib/ctypes/util.py +++ b/Lib/ctypes/util.py @@ -67,7 +67,7 @@ def find_library(name): return fname return None -elif os.name == "posix" and sys.platform == "darwin": +elif os.name == "posix" and sys.platform in {"darwin", "ios", "tvos", "watchos"}: from ctypes.macholib.dyld import dyld_find as _dyld_find def find_library(name): possible = ['lib%s.dylib' % name, @@ -89,6 +89,15 @@ def find_library(name): from ctypes._aix import find_library +elif sys.platform == "android": + def find_library(name): + directory = "/system/lib" + if "64" in os.uname().machine: + directory += "64" + + fname = f"{directory}/lib{name}.so" + return fname if os.path.isfile(fname) else None + elif os.name == "posix": # Andreas Degert's find functions, using gcc, /sbin/ldconfig, objdump import re, tempfile @@ -96,8 +105,11 @@ def find_library(name): def _is_elf(filename): "Return True if the given file is an ELF file" elf_header = b'\x7fELF' - with open(filename, 'br') as thefile: - return thefile.read(4) == elf_header + try: + with open(filename, 'br') as thefile: + return thefile.read(4) == elf_header + except FileNotFoundError: + return False def _findLib_gcc(name): # Run GCC's linker with the -t (aka --trace) option and examine the diff --git a/Lib/ctypes/wintypes.py b/Lib/ctypes/wintypes.py index c619d27596..9c4e721438 100644 --- a/Lib/ctypes/wintypes.py +++ b/Lib/ctypes/wintypes.py @@ -1,7 +1,7 @@ # The most useful windows datatypes import ctypes -BYTE = ctypes.c_byte +BYTE = ctypes.c_ubyte WORD = ctypes.c_ushort DWORD = ctypes.c_ulong diff --git a/Lib/dataclasses.py b/Lib/dataclasses.py index e1687a117d..7883ce78e5 100644 --- a/Lib/dataclasses.py +++ b/Lib/dataclasses.py @@ -4,11 +4,9 @@ import types import inspect import keyword -import builtins -import functools +import itertools import abc -import _thread -from types import FunctionType, GenericAlias +from reprlib import recursive_repr __all__ = ['dataclass', @@ -222,6 +220,30 @@ def __repr__(self): # https://bugs.python.org/issue33453 for details. _MODULE_IDENTIFIER_RE = re.compile(r'^(?:\s*(\w+)\s*\.)?\s*(\w+)') +# Atomic immutable types which don't require any recursive handling and for which deepcopy +# returns the same object. We can provide a fast-path for these types in asdict and astuple. +_ATOMIC_TYPES = frozenset({ + # Common JSON Serializable types + types.NoneType, + bool, + int, + float, + str, + # Other common types + complex, + bytes, + # Other types that are also unaffected by deepcopy + types.EllipsisType, + types.NotImplementedType, + types.CodeType, + types.BuiltinFunctionType, + types.FunctionType, + type, + range, + property, +}) + + class InitVar: __slots__ = ('type', ) @@ -229,7 +251,7 @@ def __init__(self, type): self.type = type def __repr__(self): - if isinstance(self.type, type) and not isinstance(self.type, GenericAlias): + if isinstance(self.type, type): type_name = self.type.__name__ else: # typing objects, e.g. List[int] @@ -279,6 +301,7 @@ def __init__(self, default, default_factory, init, repr, hash, compare, self.kw_only = kw_only self._field_type = None + @recursive_repr() def __repr__(self): return ('Field(' f'name={self.name!r},' @@ -297,7 +320,7 @@ def __repr__(self): # This is used to support the PEP 487 __set_name__ protocol in the # case where we're using a field that contains a descriptor as a # default value. For details on __set_name__, see - # https://www.python.org/dev/peps/pep-0487/#implementation-details. + # https://peps.python.org/pep-0487/#implementation-details. # # Note that in _process_class, this Field object is overwritten # with the default value, so the end result is a descriptor that @@ -309,7 +332,7 @@ def __set_name__(self, owner, name): # it. func(self.default, owner, name) - __class_getitem__ = classmethod(GenericAlias) + __class_getitem__ = classmethod(types.GenericAlias) class _DataclassParams: @@ -319,15 +342,25 @@ class _DataclassParams: 'order', 'unsafe_hash', 'frozen', + 'match_args', + 'kw_only', + 'slots', + 'weakref_slot', ) - def __init__(self, init, repr, eq, order, unsafe_hash, frozen): + def __init__(self, + init, repr, eq, order, unsafe_hash, frozen, + match_args, kw_only, slots, weakref_slot): self.init = init self.repr = repr self.eq = eq self.order = order self.unsafe_hash = unsafe_hash self.frozen = frozen + self.match_args = match_args + self.kw_only = kw_only + self.slots = slots + self.weakref_slot = weakref_slot def __repr__(self): return ('_DataclassParams(' @@ -336,7 +369,11 @@ def __repr__(self): f'eq={self.eq!r},' f'order={self.order!r},' f'unsafe_hash={self.unsafe_hash!r},' - f'frozen={self.frozen!r}' + f'frozen={self.frozen!r},' + f'match_args={self.match_args!r},' + f'kw_only={self.kw_only!r},' + f'slots={self.slots!r},' + f'weakref_slot={self.weakref_slot!r}' ')') @@ -388,49 +425,95 @@ def _tuple_str(obj_name, fields): return f'({",".join([f"{obj_name}.{f.name}" for f in fields])},)' -# This function's logic is copied from "recursive_repr" function in -# reprlib module to avoid dependency. -def _recursive_repr(user_function): - # Decorator to make a repr function return "..." for a recursive - # call. - repr_running = set() +class _FuncBuilder: + def __init__(self, globals): + self.names = [] + self.src = [] + self.globals = globals + self.locals = {} + self.overwrite_errors = {} + self.unconditional_adds = {} + + def add_fn(self, name, args, body, *, locals=None, return_type=MISSING, + overwrite_error=False, unconditional_add=False, decorator=None): + if locals is not None: + self.locals.update(locals) + + # Keep track if this method is allowed to be overwritten if it already + # exists in the class. The error is method-specific, so keep it with + # the name. We'll use this when we generate all of the functions in + # the add_fns_to_class call. overwrite_error is either True, in which + # case we'll raise an error, or it's a string, in which case we'll + # raise an error and append this string. + if overwrite_error: + self.overwrite_errors[name] = overwrite_error + + # Should this function always overwrite anything that's already in the + # class? The default is to not overwrite a function that already + # exists. + if unconditional_add: + self.unconditional_adds[name] = True + + self.names.append(name) + + if return_type is not MISSING: + self.locals[f'__dataclass_{name}_return_type__'] = return_type + return_annotation = f'->__dataclass_{name}_return_type__' + else: + return_annotation = '' + args = ','.join(args) + body = '\n'.join(body) + + # Compute the text of the entire function, add it to the text we're generating. + self.src.append(f'{f' {decorator}\n' if decorator else ''} def {name}({args}){return_annotation}:\n{body}') - @functools.wraps(user_function) - def wrapper(self): - key = id(self), _thread.get_ident() - if key in repr_running: - return '...' - repr_running.add(key) - try: - result = user_function(self) - finally: - repr_running.discard(key) - return result - return wrapper - - -def _create_fn(name, args, body, *, globals=None, locals=None, - return_type=MISSING): - # Note that we may mutate locals. Callers beware! - # The only callers are internal to this module, so no - # worries about external callers. - if locals is None: - locals = {} - return_annotation = '' - if return_type is not MISSING: - locals['_return_type'] = return_type - return_annotation = '->_return_type' - args = ','.join(args) - body = '\n'.join(f' {b}' for b in body) - - # Compute the text of the entire function. - txt = f' def {name}({args}){return_annotation}:\n{body}' - - local_vars = ', '.join(locals.keys()) - txt = f"def __create_fn__({local_vars}):\n{txt}\n return {name}" - ns = {} - exec(txt, globals, ns) - return ns['__create_fn__'](**locals) + def add_fns_to_class(self, cls): + # The source to all of the functions we're generating. + fns_src = '\n'.join(self.src) + + # The locals they use. + local_vars = ','.join(self.locals.keys()) + + # The names of all of the functions, used for the return value of the + # outer function. Need to handle the 0-tuple specially. + if len(self.names) == 0: + return_names = '()' + else: + return_names =f'({",".join(self.names)},)' + + # txt is the entire function we're going to execute, including the + # bodies of the functions we're defining. Here's a greatly simplified + # version: + # def __create_fn__(): + # def __init__(self, x, y): + # self.x = x + # self.y = y + # @recursive_repr + # def __repr__(self): + # return f"cls(x={self.x!r},y={self.y!r})" + # return __init__,__repr__ + + txt = f"def __create_fn__({local_vars}):\n{fns_src}\n return {return_names}" + ns = {} + exec(txt, self.globals, ns) + fns = ns['__create_fn__'](**self.locals) + + # Now that we've generated the functions, assign them into cls. + for name, fn in zip(self.names, fns): + fn.__qualname__ = f"{cls.__qualname__}.{fn.__name__}" + if self.unconditional_adds.get(name, False): + setattr(cls, name, fn) + else: + already_exists = _set_new_attribute(cls, name, fn) + + # See if it's an error to overwrite this particular function. + if already_exists and (msg_extra := self.overwrite_errors.get(name)): + error_msg = (f'Cannot overwrite attribute {fn.__name__} ' + f'in class {cls.__name__}') + if not msg_extra is True: + error_msg = f'{error_msg} {msg_extra}' + + raise TypeError(error_msg) def _field_assign(frozen, name, value, self_name): @@ -441,22 +524,22 @@ def _field_assign(frozen, name, value, self_name): # self_name is what "self" is called in this function: don't # hard-code "self", since that might be a field name. if frozen: - return f'__dataclass_builtins_object__.__setattr__({self_name},{name!r},{value})' - return f'{self_name}.{name}={value}' + return f' __dataclass_builtins_object__.__setattr__({self_name},{name!r},{value})' + return f' {self_name}.{name}={value}' def _field_init(f, frozen, globals, self_name, slots): # Return the text of the line in the body of __init__ that will # initialize this field. - default_name = f'_dflt_{f.name}' + default_name = f'__dataclass_dflt_{f.name}__' if f.default_factory is not MISSING: if f.init: # This field has a default factory. If a parameter is # given, use it. If not, call the factory. globals[default_name] = f.default_factory value = (f'{default_name}() ' - f'if {f.name} is _HAS_DEFAULT_FACTORY ' + f'if {f.name} is __dataclass_HAS_DEFAULT_FACTORY__ ' f'else {f.name}') else: # This is a field that's not in the __init__ params, but @@ -517,15 +600,15 @@ def _init_param(f): elif f.default is not MISSING: # There's a default, this will be the name that's used to look # it up. - default = f'=_dflt_{f.name}' + default = f'=__dataclass_dflt_{f.name}__' elif f.default_factory is not MISSING: # There's a factory function. Set a marker. - default = '=_HAS_DEFAULT_FACTORY' - return f'{f.name}:_type_{f.name}{default}' + default = '=__dataclass_HAS_DEFAULT_FACTORY__' + return f'{f.name}:__dataclass_type_{f.name}__{default}' def _init_fn(fields, std_fields, kw_only_fields, frozen, has_post_init, - self_name, globals, slots): + self_name, func_builder, slots): # fields contains both real fields and InitVar pseudo-fields. # Make sure we don't have fields without defaults following fields @@ -534,22 +617,21 @@ def _init_fn(fields, std_fields, kw_only_fields, frozen, has_post_init, # message, and future-proofs us in case we build up the function # using ast. - seen_default = False + seen_default = None for f in std_fields: # Only consider the non-kw-only fields in the __init__ call. if f.init: if not (f.default is MISSING and f.default_factory is MISSING): - seen_default = True + seen_default = f elif seen_default: raise TypeError(f'non-default argument {f.name!r} ' - 'follows default argument') + f'follows default argument {seen_default.name!r}') - locals = {f'_type_{f.name}': f.type for f in fields} - locals.update({ - 'MISSING': MISSING, - '_HAS_DEFAULT_FACTORY': _HAS_DEFAULT_FACTORY, - '__dataclass_builtins_object__': object, - }) + locals = {**{f'__dataclass_type_{f.name}__': f.type for f in fields}, + **{'__dataclass_HAS_DEFAULT_FACTORY__': _HAS_DEFAULT_FACTORY, + '__dataclass_builtins_object__': object, + } + } body_lines = [] for f in fields: @@ -563,11 +645,11 @@ def _init_fn(fields, std_fields, kw_only_fields, frozen, has_post_init, if has_post_init: params_str = ','.join(f.name for f in fields if f._field_type is _FIELD_INITVAR) - body_lines.append(f'{self_name}.{_POST_INIT_NAME}({params_str})') + body_lines.append(f' {self_name}.{_POST_INIT_NAME}({params_str})') # If no body lines, use 'pass'. if not body_lines: - body_lines = ['pass'] + body_lines = [' pass'] _init_params = [_init_param(f) for f in std_fields] if kw_only_fields: @@ -576,70 +658,34 @@ def _init_fn(fields, std_fields, kw_only_fields, frozen, has_post_init, # (instead of just concatenting the lists together). _init_params += ['*'] _init_params += [_init_param(f) for f in kw_only_fields] - return _create_fn('__init__', - [self_name] + _init_params, - body_lines, - locals=locals, - globals=globals, - return_type=None) - - -def _repr_fn(fields, globals): - fn = _create_fn('__repr__', - ('self',), - ['return self.__class__.__qualname__ + f"(' + - ', '.join([f"{f.name}={{self.{f.name}!r}}" - for f in fields]) + - ')"'], - globals=globals) - return _recursive_repr(fn) - - -def _frozen_get_del_attr(cls, fields, globals): + func_builder.add_fn('__init__', + [self_name] + _init_params, + body_lines, + locals=locals, + return_type=None) + + +def _frozen_get_del_attr(cls, fields, func_builder): locals = {'cls': cls, 'FrozenInstanceError': FrozenInstanceError} + condition = 'type(self) is cls' if fields: - fields_str = '(' + ','.join(repr(f.name) for f in fields) + ',)' - else: - # Special case for the zero-length tuple. - fields_str = '()' - return (_create_fn('__setattr__', - ('self', 'name', 'value'), - (f'if type(self) is cls or name in {fields_str}:', - ' raise FrozenInstanceError(f"cannot assign to field {name!r}")', - f'super(cls, self).__setattr__(name, value)'), - locals=locals, - globals=globals), - _create_fn('__delattr__', - ('self', 'name'), - (f'if type(self) is cls or name in {fields_str}:', - ' raise FrozenInstanceError(f"cannot delete field {name!r}")', - f'super(cls, self).__delattr__(name)'), - locals=locals, - globals=globals), - ) - - -def _cmp_fn(name, op, self_tuple, other_tuple, globals): - # Create a comparison function. If the fields in the object are - # named 'x' and 'y', then self_tuple is the string - # '(self.x,self.y)' and other_tuple is the string - # '(other.x,other.y)'. - - return _create_fn(name, - ('self', 'other'), - [ 'if other.__class__ is self.__class__:', - f' return {self_tuple}{op}{other_tuple}', - 'return NotImplemented'], - globals=globals) - - -def _hash_fn(fields, globals): - self_tuple = _tuple_str('self', fields) - return _create_fn('__hash__', - ('self',), - [f'return hash({self_tuple})'], - globals=globals) + condition += ' or name in {' + ', '.join(repr(f.name) for f in fields) + '}' + + func_builder.add_fn('__setattr__', + ('self', 'name', 'value'), + (f' if {condition}:', + ' raise FrozenInstanceError(f"cannot assign to field {name!r}")', + f' super(cls, self).__setattr__(name, value)'), + locals=locals, + overwrite_error=True) + func_builder.add_fn('__delattr__', + ('self', 'name'), + (f' if {condition}:', + ' raise FrozenInstanceError(f"cannot delete field {name!r}")', + f' super(cls, self).__delattr__(name)'), + locals=locals, + overwrite_error=True) def _is_classvar(a_type, typing): @@ -807,26 +853,20 @@ def _get_field(cls, a_name, a_type, default_kw_only): raise TypeError(f'field {f.name} is a ClassVar but specifies ' 'kw_only') - # For real fields, disallow mutable defaults for known types. - if f._field_type is _FIELD and isinstance(f.default, (list, dict, set)): + # For real fields, disallow mutable defaults. Use unhashable as a proxy + # indicator for mutability. Read the __hash__ attribute from the class, + # not the instance. + if f._field_type is _FIELD and f.default.__class__.__hash__ is None: raise ValueError(f'mutable default {type(f.default)} for field ' f'{f.name} is not allowed: use default_factory') return f -def _set_qualname(cls, value): - # Ensure that the functions returned from _create_fn uses the proper - # __qualname__ (the class they belong to). - if isinstance(value, FunctionType): - value.__qualname__ = f"{cls.__qualname__}.{value.__name__}" - return value - def _set_new_attribute(cls, name, value): # Never overwrites an existing attribute. Returns True if the # attribute already exists. if name in cls.__dict__: return True - _set_qualname(cls, value) setattr(cls, name, value) return False @@ -836,14 +876,22 @@ def _set_new_attribute(cls, name, value): # take. The common case is to do nothing, so instead of providing a # function that is a no-op, use None to signify that. -def _hash_set_none(cls, fields, globals): - return None +def _hash_set_none(cls, fields, func_builder): + # It's sort of a hack that I'm setting this here, instead of at + # func_builder.add_fns_to_class time, but since this is an exceptional case + # (it's not setting an attribute to a function, but to a scalar value), + # just do it directly here. I might come to regret this. + cls.__hash__ = None -def _hash_add(cls, fields, globals): +def _hash_add(cls, fields, func_builder): flds = [f for f in fields if (f.compare if f.hash is None else f.hash)] - return _set_qualname(cls, _hash_fn(flds, globals)) + self_tuple = _tuple_str('self', flds) + func_builder.add_fn('__hash__', + ('self',), + [f' return hash({self_tuple})'], + unconditional_add=True) -def _hash_exception(cls, fields, globals): +def _hash_exception(cls, fields, func_builder): # Raise an exception. raise TypeError(f'Cannot overwrite attribute __hash__ ' f'in class {cls.__name__}') @@ -879,7 +927,7 @@ def _hash_exception(cls, fields, globals): def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen, - match_args, kw_only, slots): + match_args, kw_only, slots, weakref_slot): # Now that dicts retain insertion order, there's no reason to use # an ordered dict. I am leveraging that ordering here, because # derived class fields overwrite base class fields, but the order @@ -897,13 +945,18 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen, globals = {} setattr(cls, _PARAMS, _DataclassParams(init, repr, eq, order, - unsafe_hash, frozen)) + unsafe_hash, frozen, + match_args, kw_only, + slots, weakref_slot)) # Find our base classes in reverse MRO order, and exclude # ourselves. In reversed order so that more derived classes # override earlier field definitions in base classes. As long as - # we're iterating over them, see if any are frozen. + # we're iterating over them, see if all or any of them are frozen. any_frozen_base = False + # By default `all_frozen_bases` is `None` to represent a case, + # where some dataclasses does not have any bases with `_FIELDS` + all_frozen_bases = None has_dataclass_bases = False for b in cls.__mro__[-1:0:-1]: # Only process classes that have been processed by our @@ -913,13 +966,13 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen, has_dataclass_bases = True for f in base_fields.values(): fields[f.name] = f - if getattr(b, _PARAMS).frozen: - any_frozen_base = True + if all_frozen_bases is None: + all_frozen_bases = True + current_frozen = getattr(b, _PARAMS).frozen + all_frozen_bases = all_frozen_bases and current_frozen + any_frozen_base = any_frozen_base or current_frozen - # Annotations that are defined in this class (not in base - # classes). If __annotations__ isn't present, then this class - # adds no new annotations. We use this to compute fields that are - # added by this class. + # Annotations defined specifically in this class (not in base classes). # # Fields are found from cls_annotations, which is guaranteed to be # ordered. Default values are from class attributes, if a field @@ -928,7 +981,7 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen, # actual default value. Pseudo-fields ClassVars and InitVars are # included, despite the fact that they're not real fields. That's # dealt with later. - cls_annotations = cls.__dict__.get('__annotations__', {}) + cls_annotations = inspect.get_annotations(cls) # Now find fields in our class. While doing so, validate some # things, and set the default values (as class attributes) where @@ -986,7 +1039,7 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen, 'frozen one') # Raise an exception if we're frozen, but none of our bases are. - if not any_frozen_base and frozen: + if all_frozen_bases is False and frozen: raise TypeError('cannot inherit frozen dataclass from a ' 'non-frozen one') @@ -997,7 +1050,7 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen, # Was this class defined with an explicit __hash__? Note that if # __eq__ is defined in this class, then python will automatically # set __hash__ to None. This is a heuristic, as it's possible - # that such a __hash__ == None was not auto-generated, but it + # that such a __hash__ == None was not auto-generated, but it's # close enough. class_hash = cls.__dict__.get('__hash__', MISSING) has_explicit_hash = not (class_hash is MISSING or @@ -1016,24 +1069,27 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen, (std_init_fields, kw_only_init_fields) = _fields_in_init_order(all_init_fields) + func_builder = _FuncBuilder(globals) + if init: # Does this class have a post-init function? has_post_init = hasattr(cls, _POST_INIT_NAME) - _set_new_attribute(cls, '__init__', - _init_fn(all_init_fields, - std_init_fields, - kw_only_init_fields, - frozen, - has_post_init, - # The name to use for the "self" - # param in __init__. Use "self" - # if possible. - '__dataclass_self__' if 'self' in fields - else 'self', - globals, - slots, - )) + _init_fn(all_init_fields, + std_init_fields, + kw_only_init_fields, + frozen, + has_post_init, + # The name to use for the "self" + # param in __init__. Use "self" + # if possible. + '__dataclass_self__' if 'self' in fields + else 'self', + func_builder, + slots, + ) + + _set_new_attribute(cls, '__replace__', _replace) # Get the fields as a list, and include only real fields. This is # used in all of the following methods. @@ -1041,18 +1097,27 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen, if repr: flds = [f for f in field_list if f.repr] - _set_new_attribute(cls, '__repr__', _repr_fn(flds, globals)) + func_builder.add_fn('__repr__', + ('self',), + [' return f"{self.__class__.__qualname__}(' + + ', '.join([f"{f.name}={{self.{f.name}!r}}" + for f in flds]) + ')"'], + locals={'__dataclasses_recursive_repr': recursive_repr}, + decorator="@__dataclasses_recursive_repr()") if eq: # Create __eq__ method. There's no need for a __ne__ method, # since python will call __eq__ and negate it. - flds = [f for f in field_list if f.compare] - self_tuple = _tuple_str('self', flds) - other_tuple = _tuple_str('other', flds) - _set_new_attribute(cls, '__eq__', - _cmp_fn('__eq__', '==', - self_tuple, other_tuple, - globals=globals)) + cmp_fields = (field for field in field_list if field.compare) + terms = [f'self.{field.name}==other.{field.name}' for field in cmp_fields] + field_comparisons = ' and '.join(terms) or 'True' + func_builder.add_fn('__eq__', + ('self', 'other'), + [ ' if self is other:', + ' return True', + ' if other.__class__ is self.__class__:', + f' return {field_comparisons}', + ' return NotImplemented']) if order: # Create and set the ordering methods. @@ -1064,18 +1129,19 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen, ('__gt__', '>'), ('__ge__', '>='), ]: - if _set_new_attribute(cls, name, - _cmp_fn(name, op, self_tuple, other_tuple, - globals=globals)): - raise TypeError(f'Cannot overwrite attribute {name} ' - f'in class {cls.__name__}. Consider using ' - 'functools.total_ordering') + # Create a comparison function. If the fields in the object are + # named 'x' and 'y', then self_tuple is the string + # '(self.x,self.y)' and other_tuple is the string + # '(other.x,other.y)'. + func_builder.add_fn(name, + ('self', 'other'), + [ ' if other.__class__ is self.__class__:', + f' return {self_tuple}{op}{other_tuple}', + ' return NotImplemented'], + overwrite_error='Consider using functools.total_ordering') if frozen: - for fn in _frozen_get_del_attr(cls, field_list, globals): - if _set_new_attribute(cls, fn.__name__, fn): - raise TypeError(f'Cannot overwrite attribute {fn.__name__} ' - f'in class {cls.__name__}') + _frozen_get_del_attr(cls, field_list, func_builder) # Decide if/how we're going to create a hash function. hash_action = _hash_action[bool(unsafe_hash), @@ -1083,22 +1149,33 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen, bool(frozen), has_explicit_hash] if hash_action: - # No need to call _set_new_attribute here, since by the time - # we're here the overwriting is unconditional. - cls.__hash__ = hash_action(cls, field_list, globals) + cls.__hash__ = hash_action(cls, field_list, func_builder) + + # Generate the methods and add them to the class. This needs to be done + # before the __doc__ logic below, since inspect will look at the __init__ + # signature. + func_builder.add_fns_to_class(cls) if not getattr(cls, '__doc__'): # Create a class doc-string. - cls.__doc__ = (cls.__name__ + - str(inspect.signature(cls)).replace(' -> None', '')) + try: + # In some cases fetching a signature is not possible. + # But, we surely should not fail in this case. + text_sig = str(inspect.signature(cls)).replace(' -> None', '') + except (TypeError, ValueError): + text_sig = '' + cls.__doc__ = (cls.__name__ + text_sig) if match_args: - # I could probably compute this once + # I could probably compute this once. _set_new_attribute(cls, '__match_args__', tuple(f.name for f in std_init_fields)) + # It's an error to specify weakref_slot if slots is False. + if weakref_slot and not slots: + raise TypeError('weakref_slot is True but slots is False') if slots: - cls = _add_slots(cls, frozen) + cls = _add_slots(cls, frozen, weakref_slot) abc.update_abstractmethods(cls) @@ -1106,7 +1183,7 @@ def _process_class(cls, init, repr, eq, order, unsafe_hash, frozen, # _dataclass_getstate and _dataclass_setstate are needed for pickling frozen -# classes with slots. These could be slighly more performant if we generated +# classes with slots. These could be slightly more performant if we generated # the code instead of iterating over fields. But that can be a project for # another day, if performance becomes an issue. def _dataclass_getstate(self): @@ -1119,7 +1196,30 @@ def _dataclass_setstate(self, state): object.__setattr__(self, field.name, value) -def _add_slots(cls, is_frozen): +def _get_slots(cls): + match cls.__dict__.get('__slots__'): + # `__dictoffset__` and `__weakrefoffset__` can tell us whether + # the base type has dict/weakref slots, in a way that works correctly + # for both Python classes and C extension types. Extension types + # don't use `__slots__` for slot creation + case None: + slots = [] + if getattr(cls, '__weakrefoffset__', -1) != 0: + slots.append('__weakref__') + if getattr(cls, '__dictoffset__', -1) != 0: + slots.append('__dict__') + yield from slots + case str(slot): + yield slot + # Slots may be any iterable, but we cannot handle an iterator + # because it will already be (partially) consumed. + case iterable if not hasattr(iterable, '__next__'): + yield from iterable + case _: + raise TypeError(f"Slots of '{cls.__name__}' cannot be determined") + + +def _add_slots(cls, is_frozen, weakref_slot): # Need to create a new class, since we can't set __slots__ # after a class has been created. @@ -1130,7 +1230,23 @@ def _add_slots(cls, is_frozen): # Create a new dict for our new class. cls_dict = dict(cls.__dict__) field_names = tuple(f.name for f in fields(cls)) - cls_dict['__slots__'] = field_names + # Make sure slots don't overlap with those in base classes. + inherited_slots = set( + itertools.chain.from_iterable(map(_get_slots, cls.__mro__[1:-1])) + ) + # The slots for our class. Remove slots from our base classes. Add + # '__weakref__' if weakref_slot was given, unless it is already present. + cls_dict["__slots__"] = tuple( + itertools.filterfalse( + inherited_slots.__contains__, + itertools.chain( + # gh-93521: '__weakref__' also needs to be filtered out if + # already present in inherited_slots + field_names, ('__weakref__',) if weakref_slot else () + ) + ), + ) + for field_name in field_names: # Remove our attributes, if present. They'll still be # available in _MARKER. @@ -1139,6 +1255,9 @@ def _add_slots(cls, is_frozen): # Remove __dict__ itself. cls_dict.pop('__dict__', None) + # Clear existing `__weakref__` descriptor, it belongs to a previous type: + cls_dict.pop('__weakref__', None) # gh-102069 + # And finally create the class. qualname = getattr(cls, '__qualname__', None) cls = type(cls)(cls.__name__, cls.__bases__, cls_dict) @@ -1147,33 +1266,35 @@ def _add_slots(cls, is_frozen): if is_frozen: # Need this for pickling frozen classes with slots. - cls.__getstate__ = _dataclass_getstate - cls.__setstate__ = _dataclass_setstate + if '__getstate__' not in cls_dict: + cls.__getstate__ = _dataclass_getstate + if '__setstate__' not in cls_dict: + cls.__setstate__ = _dataclass_setstate return cls def dataclass(cls=None, /, *, init=True, repr=True, eq=True, order=False, unsafe_hash=False, frozen=False, match_args=True, - kw_only=False, slots=False): - """Returns the same class as was passed in, with dunder methods - added based on the fields defined in the class. + kw_only=False, slots=False, weakref_slot=False): + """Add dunder methods based on the fields defined in the class. Examines PEP 526 __annotations__ to determine fields. - If init is true, an __init__() method is added to the class. If - repr is true, a __repr__() method is added. If order is true, rich + If init is true, an __init__() method is added to the class. If repr + is true, a __repr__() method is added. If order is true, rich comparison dunder methods are added. If unsafe_hash is true, a - __hash__() method function is added. If frozen is true, fields may - not be assigned to after instance creation. If match_args is true, - the __match_args__ tuple is added. If kw_only is true, then by - default all fields are keyword-only. If slots is true, an - __slots__ attribute is added. + __hash__() method is added. If frozen is true, fields may not be + assigned to after instance creation. If match_args is true, the + __match_args__ tuple is added. If kw_only is true, then by default + all fields are keyword-only. If slots is true, a new class with a + __slots__ attribute is returned. """ def wrap(cls): return _process_class(cls, init, repr, eq, order, unsafe_hash, - frozen, match_args, kw_only, slots) + frozen, match_args, kw_only, slots, + weakref_slot) # See if we're being called as @dataclass or @dataclass(). if cls is None: @@ -1195,7 +1316,7 @@ def fields(class_or_instance): try: fields = getattr(class_or_instance, _FIELDS) except AttributeError: - raise TypeError('must be called with a dataclass type or instance') + raise TypeError('must be called with a dataclass type or instance') from None # Exclude pseudo-fields. Note that fields is sorted by insertion # order, so the order of the tuple is as the fields were defined. @@ -1210,7 +1331,7 @@ def _is_dataclass_instance(obj): def is_dataclass(obj): """Returns True if obj is a dataclass or an instance of a dataclass.""" - cls = obj if isinstance(obj, type) and not isinstance(obj, GenericAlias) else type(obj) + cls = obj if isinstance(obj, type) else type(obj) return hasattr(cls, _FIELDS) @@ -1218,7 +1339,7 @@ def asdict(obj, *, dict_factory=dict): """Return the fields of a dataclass instance as a new dictionary mapping field names to field values. - Example usage: + Example usage:: @dataclass class C: @@ -1231,7 +1352,7 @@ class C: If given, 'dict_factory' will be used instead of built-in dict. The function applies recursively to field values that are dataclass instances. This will also look into built-in containers: - tuples, lists, and dicts. + tuples, lists, and dicts. Other objects are copied with 'copy.deepcopy()'. """ if not _is_dataclass_instance(obj): raise TypeError("asdict() should be called on dataclass instances") @@ -1239,42 +1360,69 @@ class C: def _asdict_inner(obj, dict_factory): - if _is_dataclass_instance(obj): - result = [] - for f in fields(obj): - value = _asdict_inner(getattr(obj, f.name), dict_factory) - result.append((f.name, value)) - return dict_factory(result) - elif isinstance(obj, tuple) and hasattr(obj, '_fields'): - # obj is a namedtuple. Recurse into it, but the returned - # object is another namedtuple of the same type. This is - # similar to how other list- or tuple-derived classes are - # treated (see below), but we just need to create them - # differently because a namedtuple's __init__ needs to be - # called differently (see bpo-34363). - - # I'm not using namedtuple's _asdict() - # method, because: - # - it does not recurse in to the namedtuple fields and - # convert them to dicts (using dict_factory). - # - I don't actually want to return a dict here. The main - # use case here is json.dumps, and it handles converting - # namedtuples to lists. Admittedly we're losing some - # information here when we produce a json list instead of a - # dict. Note that if we returned dicts here instead of - # namedtuples, we could no longer call asdict() on a data - # structure where a namedtuple was used as a dict key. - - return type(obj)(*[_asdict_inner(v, dict_factory) for v in obj]) - elif isinstance(obj, (list, tuple)): + obj_type = type(obj) + if obj_type in _ATOMIC_TYPES: + return obj + elif hasattr(obj_type, _FIELDS): + # dataclass instance: fast path for the common case + if dict_factory is dict: + return { + f.name: _asdict_inner(getattr(obj, f.name), dict) + for f in fields(obj) + } + else: + return dict_factory([ + (f.name, _asdict_inner(getattr(obj, f.name), dict_factory)) + for f in fields(obj) + ]) + # handle the builtin types first for speed; subclasses handled below + elif obj_type is list: + return [_asdict_inner(v, dict_factory) for v in obj] + elif obj_type is dict: + return { + _asdict_inner(k, dict_factory): _asdict_inner(v, dict_factory) + for k, v in obj.items() + } + elif obj_type is tuple: + return tuple([_asdict_inner(v, dict_factory) for v in obj]) + elif issubclass(obj_type, tuple): + if hasattr(obj, '_fields'): + # obj is a namedtuple. Recurse into it, but the returned + # object is another namedtuple of the same type. This is + # similar to how other list- or tuple-derived classes are + # treated (see below), but we just need to create them + # differently because a namedtuple's __init__ needs to be + # called differently (see bpo-34363). + + # I'm not using namedtuple's _asdict() + # method, because: + # - it does not recurse in to the namedtuple fields and + # convert them to dicts (using dict_factory). + # - I don't actually want to return a dict here. The main + # use case here is json.dumps, and it handles converting + # namedtuples to lists. Admittedly we're losing some + # information here when we produce a json list instead of a + # dict. Note that if we returned dicts here instead of + # namedtuples, we could no longer call asdict() on a data + # structure where a namedtuple was used as a dict key. + return obj_type(*[_asdict_inner(v, dict_factory) for v in obj]) + else: + return obj_type(_asdict_inner(v, dict_factory) for v in obj) + elif issubclass(obj_type, dict): + if hasattr(obj_type, 'default_factory'): + # obj is a defaultdict, which has a different constructor from + # dict as it requires the default_factory as its first arg. + result = obj_type(obj.default_factory) + for k, v in obj.items(): + result[_asdict_inner(k, dict_factory)] = _asdict_inner(v, dict_factory) + return result + return obj_type((_asdict_inner(k, dict_factory), + _asdict_inner(v, dict_factory)) + for k, v in obj.items()) + elif issubclass(obj_type, list): # Assume we can create an object of this type by passing in a - # generator (which is not true for namedtuples, handled - # above). - return type(obj)(_asdict_inner(v, dict_factory) for v in obj) - elif isinstance(obj, dict): - return type(obj)((_asdict_inner(k, dict_factory), - _asdict_inner(v, dict_factory)) - for k, v in obj.items()) + # generator + return obj_type(_asdict_inner(v, dict_factory) for v in obj) else: return copy.deepcopy(obj) @@ -1289,13 +1437,13 @@ class C: x: int y: int - c = C(1, 2) - assert astuple(c) == (1, 2) + c = C(1, 2) + assert astuple(c) == (1, 2) If given, 'tuple_factory' will be used instead of built-in tuple. The function applies recursively to field values that are dataclass instances. This will also look into built-in containers: - tuples, lists, and dicts. + tuples, lists, and dicts. Other objects are copied with 'copy.deepcopy()'. """ if not _is_dataclass_instance(obj): @@ -1304,12 +1452,13 @@ class C: def _astuple_inner(obj, tuple_factory): - if _is_dataclass_instance(obj): - result = [] - for f in fields(obj): - value = _astuple_inner(getattr(obj, f.name), tuple_factory) - result.append(value) - return tuple_factory(result) + if type(obj) in _ATOMIC_TYPES: + return obj + elif _is_dataclass_instance(obj): + return tuple_factory([ + _astuple_inner(getattr(obj, f.name), tuple_factory) + for f in fields(obj) + ]) elif isinstance(obj, tuple) and hasattr(obj, '_fields'): # obj is a namedtuple. Recurse into it, but the returned # object is another namedtuple of the same type. This is @@ -1324,7 +1473,15 @@ def _astuple_inner(obj, tuple_factory): # above). return type(obj)(_astuple_inner(v, tuple_factory) for v in obj) elif isinstance(obj, dict): - return type(obj)((_astuple_inner(k, tuple_factory), _astuple_inner(v, tuple_factory)) + obj_type = type(obj) + if hasattr(obj_type, 'default_factory'): + # obj is a defaultdict, which has a different constructor from + # dict as it requires the default_factory as its first arg. + result = obj_type(getattr(obj, 'default_factory')) + for k, v in obj.items(): + result[_astuple_inner(k, tuple_factory)] = _astuple_inner(v, tuple_factory) + return result + return obj_type((_astuple_inner(k, tuple_factory), _astuple_inner(v, tuple_factory)) for k, v in obj.items()) else: return copy.deepcopy(obj) @@ -1332,17 +1489,18 @@ def _astuple_inner(obj, tuple_factory): def make_dataclass(cls_name, fields, *, bases=(), namespace=None, init=True, repr=True, eq=True, order=False, unsafe_hash=False, - frozen=False, match_args=True, kw_only=False, slots=False): + frozen=False, match_args=True, kw_only=False, slots=False, + weakref_slot=False, module=None): """Return a new dynamically created dataclass. The dataclass name will be 'cls_name'. 'fields' is an iterable of either (name), (name, type) or (name, type, Field) objects. If type is omitted, use the string 'typing.Any'. Field objects are created by - the equivalent of calling 'field(name, type [, Field-info])'. + the equivalent of calling 'field(name, type [, Field-info])'.:: C = make_dataclass('C', ['x', ('y', int), ('z', int, field(init=False))], bases=(Base,)) - is equivalent to: + is equivalent to:: @dataclass class C(Base): @@ -1352,8 +1510,11 @@ class C(Base): For the bases and namespace parameters, see the builtin type() function. - The parameters init, repr, eq, order, unsafe_hash, and frozen are passed to - dataclass(). + The parameters init, repr, eq, order, unsafe_hash, frozen, match_args, kw_only, + slots, and weakref_slot are passed to dataclass(). + + If module parameter is defined, the '__module__' attribute of the dataclass is + set to that value. """ if namespace is None: @@ -1396,16 +1557,30 @@ def exec_body_callback(ns): # of generic dataclasses. cls = types.new_class(cls_name, bases, {}, exec_body_callback) + # For pickling to work, the __module__ variable needs to be set to the frame + # where the dataclass is created. + if module is None: + try: + module = sys._getframemodulename(1) or '__main__' + except AttributeError: + try: + module = sys._getframe(1).f_globals.get('__name__', '__main__') + except (AttributeError, ValueError): + pass + if module is not None: + cls.__module__ = module + # Apply the normal decorator. return dataclass(cls, init=init, repr=repr, eq=eq, order=order, unsafe_hash=unsafe_hash, frozen=frozen, - match_args=match_args, kw_only=kw_only, slots=slots) + match_args=match_args, kw_only=kw_only, slots=slots, + weakref_slot=weakref_slot) def replace(obj, /, **changes): """Return a new object replacing specified fields with new values. - This is especially useful for frozen classes. Example usage: + This is especially useful for frozen classes. Example usage:: @dataclass(frozen=True) class C: @@ -1415,18 +1590,20 @@ class C: c = C(1, 2) c1 = replace(c, x=3) assert c1.x == 3 and c1.y == 2 - """ - - # We're going to mutate 'changes', but that's okay because it's a - # new dict, even if called with 'replace(obj, **my_changes)'. - + """ if not _is_dataclass_instance(obj): raise TypeError("replace() should be called on dataclass instances") + return _replace(obj, **changes) + + +def _replace(self, /, **changes): + # We're going to mutate 'changes', but that's okay because it's a + # new dict, even if called with 'replace(self, **my_changes)'. # It's an error to have init=False fields in 'changes'. - # If a field is not in 'changes', read its value from the provided obj. + # If a field is not in 'changes', read its value from the provided 'self'. - for f in getattr(obj, _FIELDS).values(): + for f in getattr(self, _FIELDS).values(): # Only consider normal fields or InitVars. if f._field_type is _FIELD_CLASSVAR: continue @@ -1434,20 +1611,20 @@ class C: if not f.init: # Error if this field is specified in changes. if f.name in changes: - raise ValueError(f'field {f.name} is declared with ' - 'init=False, it cannot be specified with ' - 'replace()') + raise TypeError(f'field {f.name} is declared with ' + f'init=False, it cannot be specified with ' + f'replace()') continue if f.name not in changes: if f._field_type is _FIELD_INITVAR and f.default is MISSING: - raise ValueError(f"InitVar {f.name!r} " - 'must be specified with replace()') - changes[f.name] = getattr(obj, f.name) + raise TypeError(f"InitVar {f.name!r} " + f'must be specified with replace()') + changes[f.name] = getattr(self, f.name) # Create the new object, which calls __init__() and # __post_init__() (if defined), using all of the init fields we've # added and/or left in 'changes'. If there are values supplied in # changes that aren't fields, this will correctly raise a # TypeError. - return obj.__class__(**changes) + return self.__class__(**changes) diff --git a/Lib/decimal.py b/Lib/decimal.py index 7746ea2601..ee3147f5dd 100644 --- a/Lib/decimal.py +++ b/Lib/decimal.py @@ -1,11 +1,109 @@ +"""Decimal fixed-point and floating-point arithmetic. + +This is an implementation of decimal floating-point arithmetic based on +the General Decimal Arithmetic Specification: + + http://speleotrove.com/decimal/decarith.html + +and IEEE standard 854-1987: + + http://en.wikipedia.org/wiki/IEEE_854-1987 + +Decimal floating point has finite precision with arbitrarily large bounds. + +The purpose of this module is to support arithmetic using familiar +"schoolhouse" rules and to avoid some of the tricky representation +issues associated with binary floating point. The package is especially +useful for financial applications or for contexts where users have +expectations that are at odds with binary floating point (for instance, +in binary floating point, 1.00 % 0.1 gives 0.09999999999999995 instead +of 0.0; Decimal('1.00') % Decimal('0.1') returns the expected +Decimal('0.00')). + +Here are some examples of using the decimal module: + +>>> from decimal import * +>>> setcontext(ExtendedContext) +>>> Decimal(0) +Decimal('0') +>>> Decimal('1') +Decimal('1') +>>> Decimal('-.0123') +Decimal('-0.0123') +>>> Decimal(123456) +Decimal('123456') +>>> Decimal('123.45e12345678') +Decimal('1.2345E+12345680') +>>> Decimal('1.33') + Decimal('1.27') +Decimal('2.60') +>>> Decimal('12.34') + Decimal('3.87') - Decimal('18.41') +Decimal('-2.20') +>>> dig = Decimal(1) +>>> print(dig / Decimal(3)) +0.333333333 +>>> getcontext().prec = 18 +>>> print(dig / Decimal(3)) +0.333333333333333333 +>>> print(dig.sqrt()) +1 +>>> print(Decimal(3).sqrt()) +1.73205080756887729 +>>> print(Decimal(3) ** 123) +4.85192780976896427E+58 +>>> inf = Decimal(1) / Decimal(0) +>>> print(inf) +Infinity +>>> neginf = Decimal(-1) / Decimal(0) +>>> print(neginf) +-Infinity +>>> print(neginf + inf) +NaN +>>> print(neginf * inf) +-Infinity +>>> print(dig / 0) +Infinity +>>> getcontext().traps[DivisionByZero] = 1 +>>> print(dig / 0) +Traceback (most recent call last): + ... + ... + ... +decimal.DivisionByZero: x / 0 +>>> c = Context() +>>> c.traps[InvalidOperation] = 0 +>>> print(c.flags[InvalidOperation]) +0 +>>> c.divide(Decimal(0), Decimal(0)) +Decimal('NaN') +>>> c.traps[InvalidOperation] = 1 +>>> print(c.flags[InvalidOperation]) +1 +>>> c.flags[InvalidOperation] = 0 +>>> print(c.flags[InvalidOperation]) +0 +>>> print(c.divide(Decimal(0), Decimal(0))) +Traceback (most recent call last): + ... + ... + ... +decimal.InvalidOperation: 0 / 0 +>>> print(c.flags[InvalidOperation]) +1 +>>> c.flags[InvalidOperation] = 0 +>>> c.traps[InvalidOperation] = 0 +>>> print(c.divide(Decimal(0), Decimal(0))) +NaN +>>> print(c.flags[InvalidOperation]) +1 +>>> +""" try: from _decimal import * - from _decimal import __doc__ from _decimal import __version__ from _decimal import __libmpdec_version__ except ImportError: - from _pydecimal import * - from _pydecimal import __doc__ - from _pydecimal import __version__ - from _pydecimal import __libmpdec_version__ + import _pydecimal + import sys + _pydecimal.__doc__ = __doc__ + sys.modules[__name__] = _pydecimal diff --git a/Lib/difflib.py b/Lib/difflib.py index 3425e438c9..33e7e6c165 100644 --- a/Lib/difflib.py +++ b/Lib/difflib.py @@ -1200,6 +1200,25 @@ def context_diff(a, b, fromfile='', tofile='', strings for 'fromfile', 'tofile', 'fromfiledate', and 'tofiledate'. The modification times are normally expressed in the ISO 8601 format. If not specified, the strings default to blanks. + + Example: + + >>> print(''.join(context_diff('one\ntwo\nthree\nfour\n'.splitlines(True), + ... 'zero\none\ntree\nfour\n'.splitlines(True), 'Original', 'Current')), + ... end="") + *** Original + --- Current + *************** + *** 1,4 **** + one + ! two + ! three + four + --- 1,4 ---- + + zero + one + ! tree + four """ _check_types(a, b, fromfile, tofile, fromfiledate, tofiledate, lineterm) @@ -1609,7 +1628,7 @@ def _line_pair_iterator(): """ _styles = """ - table.diff {font-family:Courier; border:medium;} + table.diff {font-family: Menlo, Consolas, Monaco, Liberation Mono, Lucida Console, monospace; border:medium} .diff_header {background-color:#e0e0e0} td.diff_header {text-align:right} .diff_next {background-color:#c0c0c0} diff --git a/Lib/dis.py b/Lib/dis.py index 53c85555bc..6583cab62b 100644 --- a/Lib/dis.py +++ b/Lib/dis.py @@ -1,18 +1,1081 @@ -from _dis import * +"""Disassembler of Python byte code into mnemonics.""" +import sys +import types +import collections +import io -# Disassembling a file by following cpython Lib/dis.py -def _test(): - """Simple test program to disassemble a file.""" +from opcode import * +from opcode import ( + __all__ as _opcodes_all, + _cache_format, + _inline_cache_entries, + _nb_ops, + _intrinsic_1_descs, + _intrinsic_2_descs, + _specializations, + _specialized_opmap, +) + +from _opcode import get_executor + + +__all__ = ["code_info", "dis", "disassemble", "distb", "disco", + "findlinestarts", "findlabels", "show_code", + "get_instructions", "Instruction", "Bytecode"] + _opcodes_all +del _opcodes_all + +_have_code = (types.MethodType, types.FunctionType, types.CodeType, + classmethod, staticmethod, type) + +CONVERT_VALUE = opmap['CONVERT_VALUE'] + +SET_FUNCTION_ATTRIBUTE = opmap['SET_FUNCTION_ATTRIBUTE'] +FUNCTION_ATTR_FLAGS = ('defaults', 'kwdefaults', 'annotations', 'closure') + +ENTER_EXECUTOR = opmap['ENTER_EXECUTOR'] +LOAD_CONST = opmap['LOAD_CONST'] +RETURN_CONST = opmap['RETURN_CONST'] +LOAD_GLOBAL = opmap['LOAD_GLOBAL'] +BINARY_OP = opmap['BINARY_OP'] +JUMP_BACKWARD = opmap['JUMP_BACKWARD'] +FOR_ITER = opmap['FOR_ITER'] +SEND = opmap['SEND'] +LOAD_ATTR = opmap['LOAD_ATTR'] +LOAD_SUPER_ATTR = opmap['LOAD_SUPER_ATTR'] +CALL_INTRINSIC_1 = opmap['CALL_INTRINSIC_1'] +CALL_INTRINSIC_2 = opmap['CALL_INTRINSIC_2'] +LOAD_FAST_LOAD_FAST = opmap['LOAD_FAST_LOAD_FAST'] +STORE_FAST_LOAD_FAST = opmap['STORE_FAST_LOAD_FAST'] +STORE_FAST_STORE_FAST = opmap['STORE_FAST_STORE_FAST'] + +CACHE = opmap["CACHE"] + +_all_opname = list(opname) +_all_opmap = dict(opmap) +for name, op in _specialized_opmap.items(): + # fill opname and opmap + assert op < len(_all_opname) + _all_opname[op] = name + _all_opmap[name] = op + +deoptmap = { + specialized: base for base, family in _specializations.items() for specialized in family +} + +def _try_compile(source, name): + """Attempts to compile the given source, first as an expression and + then as a statement if the first approach fails. + + Utility function to accept strings in functions that otherwise + expect code objects + """ + try: + return compile(source, name, 'eval') + except SyntaxError: + pass + return compile(source, name, 'exec') + +def dis(x=None, *, file=None, depth=None, show_caches=False, adaptive=False, + show_offsets=False): + """Disassemble classes, methods, functions, and other compiled objects. + + With no argument, disassemble the last traceback. + + Compiled objects currently include generator objects, async generator + objects, and coroutine objects, all of which store their code object + in a special attribute. + """ + if x is None: + distb(file=file, show_caches=show_caches, adaptive=adaptive, + show_offsets=show_offsets) + return + # Extract functions from methods. + if hasattr(x, '__func__'): + x = x.__func__ + # Extract compiled code objects from... + if hasattr(x, '__code__'): # ...a function, or + x = x.__code__ + elif hasattr(x, 'gi_code'): #...a generator object, or + x = x.gi_code + elif hasattr(x, 'ag_code'): #...an asynchronous generator object, or + x = x.ag_code + elif hasattr(x, 'cr_code'): #...a coroutine. + x = x.cr_code + # Perform the disassembly. + if hasattr(x, '__dict__'): # Class or module + items = sorted(x.__dict__.items()) + for name, x1 in items: + if isinstance(x1, _have_code): + print("Disassembly of %s:" % name, file=file) + try: + dis(x1, file=file, depth=depth, show_caches=show_caches, adaptive=adaptive, show_offsets=show_offsets) + except TypeError as msg: + print("Sorry:", msg, file=file) + print(file=file) + elif hasattr(x, 'co_code'): # Code object + _disassemble_recursive(x, file=file, depth=depth, show_caches=show_caches, adaptive=adaptive, show_offsets=show_offsets) + elif isinstance(x, (bytes, bytearray)): # Raw bytecode + labels_map = _make_labels_map(x) + label_width = 4 + len(str(len(labels_map))) + formatter = Formatter(file=file, + offset_width=len(str(max(len(x) - 2, 9999))) if show_offsets else 0, + label_width=label_width, + show_caches=show_caches) + arg_resolver = ArgResolver(labels_map=labels_map) + _disassemble_bytes(x, arg_resolver=arg_resolver, formatter=formatter) + elif isinstance(x, str): # Source code + _disassemble_str(x, file=file, depth=depth, show_caches=show_caches, adaptive=adaptive, show_offsets=show_offsets) + else: + raise TypeError("don't know how to disassemble %s objects" % + type(x).__name__) + +def distb(tb=None, *, file=None, show_caches=False, adaptive=False, show_offsets=False): + """Disassemble a traceback (default: last traceback).""" + if tb is None: + try: + if hasattr(sys, 'last_exc'): + tb = sys.last_exc.__traceback__ + else: + tb = sys.last_traceback + except AttributeError: + raise RuntimeError("no last traceback to disassemble") from None + while tb.tb_next: tb = tb.tb_next + disassemble(tb.tb_frame.f_code, tb.tb_lasti, file=file, show_caches=show_caches, adaptive=adaptive, show_offsets=show_offsets) + +# The inspect module interrogates this dictionary to build its +# list of CO_* constants. It is also used by pretty_flags to +# turn the co_flags field into a human readable list. +COMPILER_FLAG_NAMES = { + 1: "OPTIMIZED", + 2: "NEWLOCALS", + 4: "VARARGS", + 8: "VARKEYWORDS", + 16: "NESTED", + 32: "GENERATOR", + 64: "NOFREE", + 128: "COROUTINE", + 256: "ITERABLE_COROUTINE", + 512: "ASYNC_GENERATOR", +} + +def pretty_flags(flags): + """Return pretty representation of code flags.""" + names = [] + for i in range(32): + flag = 1<" + +# Sentinel to represent values that cannot be calculated +UNKNOWN = _Unknown() + +def _get_code_object(x): + """Helper to handle methods, compiled or raw code objects, and strings.""" + # Extract functions from methods. + if hasattr(x, '__func__'): + x = x.__func__ + # Extract compiled code objects from... + if hasattr(x, '__code__'): # ...a function, or + x = x.__code__ + elif hasattr(x, 'gi_code'): #...a generator object, or + x = x.gi_code + elif hasattr(x, 'ag_code'): #...an asynchronous generator object, or + x = x.ag_code + elif hasattr(x, 'cr_code'): #...a coroutine. + x = x.cr_code + # Handle source code. + if isinstance(x, str): + x = _try_compile(x, "") + # By now, if we don't have a code object, we can't disassemble x. + if hasattr(x, 'co_code'): + return x + raise TypeError("don't know how to disassemble %s objects" % + type(x).__name__) + +def _deoptop(op): + name = _all_opname[op] + return _all_opmap[deoptmap[name]] if name in deoptmap else op + +def _get_code_array(co, adaptive): + if adaptive: + code = co._co_code_adaptive + res = [] + found = False + for i in range(0, len(code), 2): + op, arg = code[i], code[i+1] + if op == ENTER_EXECUTOR: + try: + ex = get_executor(co, i) + except (ValueError, RuntimeError): + ex = None + + if ex: + op, arg = ex.get_opcode(), ex.get_oparg() + found = True + + res.append(op.to_bytes()) + res.append(arg.to_bytes()) + return code if not found else b''.join(res) + else: + return co.co_code + +def code_info(x): + """Formatted details of methods, functions, or code.""" + return _format_code_info(_get_code_object(x)) + +def _format_code_info(co): + lines = [] + lines.append("Name: %s" % co.co_name) + lines.append("Filename: %s" % co.co_filename) + lines.append("Argument count: %s" % co.co_argcount) + lines.append("Positional-only arguments: %s" % co.co_posonlyargcount) + lines.append("Kw-only arguments: %s" % co.co_kwonlyargcount) + lines.append("Number of locals: %s" % co.co_nlocals) + lines.append("Stack size: %s" % co.co_stacksize) + lines.append("Flags: %s" % pretty_flags(co.co_flags)) + if co.co_consts: + lines.append("Constants:") + for i_c in enumerate(co.co_consts): + lines.append("%4d: %r" % i_c) + if co.co_names: + lines.append("Names:") + for i_n in enumerate(co.co_names): + lines.append("%4d: %s" % i_n) + if co.co_varnames: + lines.append("Variable names:") + for i_n in enumerate(co.co_varnames): + lines.append("%4d: %s" % i_n) + if co.co_freevars: + lines.append("Free variables:") + for i_n in enumerate(co.co_freevars): + lines.append("%4d: %s" % i_n) + if co.co_cellvars: + lines.append("Cell variables:") + for i_n in enumerate(co.co_cellvars): + lines.append("%4d: %s" % i_n) + return "\n".join(lines) + +def show_code(co, *, file=None): + """Print details of methods, functions, or code to *file*. + + If *file* is not provided, the output is printed on stdout. + """ + print(code_info(co), file=file) + +Positions = collections.namedtuple( + 'Positions', + [ + 'lineno', + 'end_lineno', + 'col_offset', + 'end_col_offset', + ], + defaults=[None] * 4 +) + +_Instruction = collections.namedtuple( + "_Instruction", + [ + 'opname', + 'opcode', + 'arg', + 'argval', + 'argrepr', + 'offset', + 'start_offset', + 'starts_line', + 'line_number', + 'label', + 'positions', + 'cache_info', + ], + defaults=[None, None, None] +) + +_Instruction.opname.__doc__ = "Human readable name for operation" +_Instruction.opcode.__doc__ = "Numeric code for operation" +_Instruction.arg.__doc__ = "Numeric argument to operation (if any), otherwise None" +_Instruction.argval.__doc__ = "Resolved arg value (if known), otherwise same as arg" +_Instruction.argrepr.__doc__ = "Human readable description of operation argument" +_Instruction.offset.__doc__ = "Start index of operation within bytecode sequence" +_Instruction.start_offset.__doc__ = ( + "Start index of operation within bytecode sequence, including extended args if present; " + "otherwise equal to Instruction.offset" +) +_Instruction.starts_line.__doc__ = "True if this opcode starts a source line, otherwise False" +_Instruction.line_number.__doc__ = "source line number associated with this opcode (if any), otherwise None" +_Instruction.label.__doc__ = "A label (int > 0) if this instruction is a jump target, otherwise None" +_Instruction.positions.__doc__ = "dis.Positions object holding the span of source code covered by this instruction" +_Instruction.cache_info.__doc__ = "list of (name, size, data), one for each cache entry of the instruction" + +_ExceptionTableEntryBase = collections.namedtuple("_ExceptionTableEntryBase", + "start end target depth lasti") + +class _ExceptionTableEntry(_ExceptionTableEntryBase): + pass + +_OPNAME_WIDTH = 20 +_OPARG_WIDTH = 5 + +def _get_cache_size(opname): + return _inline_cache_entries.get(opname, 0) + +def _get_jump_target(op, arg, offset): + """Gets the bytecode offset of the jump target if this is a jump instruction. + + Otherwise return None. + """ + deop = _deoptop(op) + caches = _get_cache_size(_all_opname[deop]) + if deop in hasjrel: + if _is_backward_jump(deop): + arg = -arg + target = offset + 2 + arg*2 + target += 2 * caches + elif deop in hasjabs: + target = arg*2 + else: + target = None + return target + +class Instruction(_Instruction): + """Details for a bytecode operation. + + Defined fields: + opname - human readable name for operation + opcode - numeric code for operation + arg - numeric argument to operation (if any), otherwise None + argval - resolved arg value (if known), otherwise same as arg + argrepr - human readable description of operation argument + offset - start index of operation within bytecode sequence + start_offset - start index of operation within bytecode sequence including extended args if present; + otherwise equal to Instruction.offset + starts_line - True if this opcode starts a source line, otherwise False + line_number - source line number associated with this opcode (if any), otherwise None + label - A label if this instruction is a jump target, otherwise None + positions - Optional dis.Positions object holding the span of source code + covered by this instruction + cache_info - information about the format and content of the instruction's cache + entries (if any) + """ + + @property + def oparg(self): + """Alias for Instruction.arg.""" + return self.arg + + @property + def baseopcode(self): + """Numeric code for the base operation if operation is specialized. + + Otherwise equal to Instruction.opcode. + """ + return _deoptop(self.opcode) + + @property + def baseopname(self): + """Human readable name for the base operation if operation is specialized. + + Otherwise equal to Instruction.opname. + """ + return opname[self.baseopcode] + + @property + def cache_offset(self): + """Start index of the cache entries following the operation.""" + return self.offset + 2 + + @property + def end_offset(self): + """End index of the cache entries following the operation.""" + return self.cache_offset + _get_cache_size(_all_opname[self.opcode])*2 + + @property + def jump_target(self): + """Bytecode index of the jump target if this is a jump operation. + + Otherwise return None. + """ + return _get_jump_target(self.opcode, self.arg, self.offset) + + @property + def is_jump_target(self): + """True if other code jumps to here, otherwise False""" + return self.label is not None + + def __str__(self): + output = io.StringIO() + formatter = Formatter(file=output) + formatter.print_instruction(self, False) + return output.getvalue() + + +class Formatter: + + def __init__(self, file=None, lineno_width=0, offset_width=0, label_width=0, + line_offset=0, show_caches=False): + """Create a Formatter + + *file* where to write the output + *lineno_width* sets the width of the line number field (0 omits it) + *offset_width* sets the width of the instruction offset field + *label_width* sets the width of the label field + *show_caches* is a boolean indicating whether to display cache lines + + """ + self.file = file + self.lineno_width = lineno_width + self.offset_width = offset_width + self.label_width = label_width + self.show_caches = show_caches + + def print_instruction(self, instr, mark_as_current=False): + self.print_instruction_line(instr, mark_as_current) + if self.show_caches and instr.cache_info: + offset = instr.offset + for name, size, data in instr.cache_info: + for i in range(size): + offset += 2 + # Only show the fancy argrepr for a CACHE instruction when it's + # the first entry for a particular cache value: + if i == 0: + argrepr = f"{name}: {int.from_bytes(data, sys.byteorder)}" + else: + argrepr = "" + self.print_instruction_line( + Instruction("CACHE", CACHE, 0, None, argrepr, offset, offset, + False, None, None, instr.positions), + False) + + def print_instruction_line(self, instr, mark_as_current): + """Format instruction details for inclusion in disassembly output.""" + lineno_width = self.lineno_width + offset_width = self.offset_width + label_width = self.label_width + + new_source_line = (lineno_width > 0 and + instr.starts_line and + instr.offset > 0) + if new_source_line: + print(file=self.file) + + fields = [] + # Column: Source code line number + if lineno_width: + if instr.starts_line: + lineno_fmt = "%%%dd" if instr.line_number is not None else "%%%ds" + lineno_fmt = lineno_fmt % lineno_width + lineno = _NO_LINENO if instr.line_number is None else instr.line_number + fields.append(lineno_fmt % lineno) + else: + fields.append(' ' * lineno_width) + # Column: Label + if instr.label is not None: + lbl = f"L{instr.label}:" + fields.append(f"{lbl:>{label_width}}") + else: + fields.append(' ' * label_width) + # Column: Instruction offset from start of code sequence + if offset_width > 0: + fields.append(f"{repr(instr.offset):>{offset_width}} ") + # Column: Current instruction indicator + if mark_as_current: + fields.append('-->') + else: + fields.append(' ') + # Column: Opcode name + fields.append(instr.opname.ljust(_OPNAME_WIDTH)) + # Column: Opcode argument + if instr.arg is not None: + arg = repr(instr.arg) + # If opname is longer than _OPNAME_WIDTH, we allow it to overflow into + # the space reserved for oparg. This results in fewer misaligned opargs + # in the disassembly output. + opname_excess = max(0, len(instr.opname) - _OPNAME_WIDTH) + fields.append(repr(instr.arg).rjust(_OPARG_WIDTH - opname_excess)) + # Column: Opcode argument details + if instr.argrepr: + fields.append('(' + instr.argrepr + ')') + print(' '.join(fields).rstrip(), file=self.file) + + def print_exception_table(self, exception_entries): + file = self.file + if exception_entries: + print("ExceptionTable:", file=file) + for entry in exception_entries: + lasti = " lasti" if entry.lasti else "" + start = entry.start_label + end = entry.end_label + target = entry.target_label + print(f" L{start} to L{end} -> L{target} [{entry.depth}]{lasti}", file=file) + + +class ArgResolver: + def __init__(self, co_consts=None, names=None, varname_from_oparg=None, labels_map=None): + self.co_consts = co_consts + self.names = names + self.varname_from_oparg = varname_from_oparg + self.labels_map = labels_map or {} + + def offset_from_jump_arg(self, op, arg, offset): + deop = _deoptop(op) + if deop in hasjabs: + return arg * 2 + elif deop in hasjrel: + signed_arg = -arg if _is_backward_jump(deop) else arg + argval = offset + 2 + signed_arg*2 + caches = _get_cache_size(_all_opname[deop]) + argval += 2 * caches + return argval + return None + + def get_label_for_offset(self, offset): + return self.labels_map.get(offset, None) + + def get_argval_argrepr(self, op, arg, offset): + get_name = None if self.names is None else self.names.__getitem__ + argval = None + argrepr = '' + deop = _deoptop(op) + if arg is not None: + # Set argval to the dereferenced value of the argument when + # available, and argrepr to the string representation of argval. + # _disassemble_bytes needs the string repr of the + # raw name index for LOAD_GLOBAL, LOAD_CONST, etc. + argval = arg + if deop in hasconst: + argval, argrepr = _get_const_info(deop, arg, self.co_consts) + elif deop in hasname: + if deop == LOAD_GLOBAL: + argval, argrepr = _get_name_info(arg//2, get_name) + if (arg & 1) and argrepr: + argrepr = f"{argrepr} + NULL" + elif deop == LOAD_ATTR: + argval, argrepr = _get_name_info(arg//2, get_name) + if (arg & 1) and argrepr: + argrepr = f"{argrepr} + NULL|self" + elif deop == LOAD_SUPER_ATTR: + argval, argrepr = _get_name_info(arg//4, get_name) + if (arg & 1) and argrepr: + argrepr = f"{argrepr} + NULL|self" + else: + argval, argrepr = _get_name_info(arg, get_name) + elif deop in hasjump or deop in hasexc: + argval = self.offset_from_jump_arg(op, arg, offset) + lbl = self.get_label_for_offset(argval) + assert lbl is not None + argrepr = f"to L{lbl}" + elif deop in (LOAD_FAST_LOAD_FAST, STORE_FAST_LOAD_FAST, STORE_FAST_STORE_FAST): + arg1 = arg >> 4 + arg2 = arg & 15 + val1, argrepr1 = _get_name_info(arg1, self.varname_from_oparg) + val2, argrepr2 = _get_name_info(arg2, self.varname_from_oparg) + argrepr = argrepr1 + ", " + argrepr2 + argval = val1, val2 + elif deop in haslocal or deop in hasfree: + argval, argrepr = _get_name_info(arg, self.varname_from_oparg) + elif deop in hascompare: + argval = cmp_op[arg >> 5] + argrepr = argval + if arg & 16: + argrepr = f"bool({argrepr})" + elif deop == CONVERT_VALUE: + argval = (None, str, repr, ascii)[arg] + argrepr = ('', 'str', 'repr', 'ascii')[arg] + elif deop == SET_FUNCTION_ATTRIBUTE: + argrepr = ', '.join(s for i, s in enumerate(FUNCTION_ATTR_FLAGS) + if arg & (1<> 1 + lasti = bool(dl&1) + entries.append(_ExceptionTableEntry(start, end, target, depth, lasti)) + except StopIteration: + return entries + +def _is_backward_jump(op): + return opname[op] in ('JUMP_BACKWARD', + 'JUMP_BACKWARD_NO_INTERRUPT') + +def _get_instructions_bytes(code, linestarts=None, line_offset=0, co_positions=None, + original_code=None, arg_resolver=None): + """Iterate over the instructions in a bytecode string. + + Generates a sequence of Instruction namedtuples giving the details of each + opcode. + + """ + # Use the basic, unadaptive code for finding labels and actually walking the + # bytecode, since replacements like ENTER_EXECUTOR and INSTRUMENTED_* can + # mess that logic up pretty badly: + original_code = original_code or code + co_positions = co_positions or iter(()) + + starts_line = False + local_line_number = None + line_number = None + for offset, start_offset, op, arg in _unpack_opargs(original_code): + if linestarts is not None: + starts_line = offset in linestarts + if starts_line: + local_line_number = linestarts[offset] + if local_line_number is not None: + line_number = local_line_number + line_offset + else: + line_number = None + positions = Positions(*next(co_positions, ())) + deop = _deoptop(op) + op = code[offset] + + if arg_resolver: + argval, argrepr = arg_resolver.get_argval_argrepr(op, arg, offset) + else: + argval, argrepr = arg, repr(arg) + + caches = _get_cache_size(_all_opname[deop]) + # Advance the co_positions iterator: + for _ in range(caches): + next(co_positions, ()) + + if caches: + cache_info = [] + for name, size in _cache_format[opname[deop]].items(): + data = code[offset + 2: offset + 2 + 2 * size] + cache_info.append((name, size, data)) + else: + cache_info = None + + label = arg_resolver.get_label_for_offset(offset) if arg_resolver else None + yield Instruction(_all_opname[op], op, arg, argval, argrepr, + offset, start_offset, starts_line, line_number, + label, positions, cache_info) + + +def disassemble(co, lasti=-1, *, file=None, show_caches=False, adaptive=False, + show_offsets=False): + """Disassemble a code object.""" + linestarts = dict(findlinestarts(co)) + exception_entries = _parse_exception_table(co) + labels_map = _make_labels_map(co.co_code, exception_entries=exception_entries) + label_width = 4 + len(str(len(labels_map))) + formatter = Formatter(file=file, + lineno_width=_get_lineno_width(linestarts), + offset_width=len(str(max(len(co.co_code) - 2, 9999))) if show_offsets else 0, + label_width=label_width, + show_caches=show_caches) + arg_resolver = ArgResolver(co_consts=co.co_consts, + names=co.co_names, + varname_from_oparg=co._varname_from_oparg, + labels_map=labels_map) + _disassemble_bytes(_get_code_array(co, adaptive), lasti, linestarts, + exception_entries=exception_entries, co_positions=co.co_positions(), + original_code=co.co_code, arg_resolver=arg_resolver, formatter=formatter) + +def _disassemble_recursive(co, *, file=None, depth=None, show_caches=False, adaptive=False, show_offsets=False): + disassemble(co, file=file, show_caches=show_caches, adaptive=adaptive, show_offsets=show_offsets) + if depth is None or depth > 0: + if depth is not None: + depth = depth - 1 + for x in co.co_consts: + if hasattr(x, 'co_code'): + print(file=file) + print("Disassembly of %r:" % (x,), file=file) + _disassemble_recursive( + x, file=file, depth=depth, show_caches=show_caches, + adaptive=adaptive, show_offsets=show_offsets + ) + + +def _make_labels_map(original_code, exception_entries=()): + jump_targets = set(findlabels(original_code)) + labels = set(jump_targets) + for start, end, target, _, _ in exception_entries: + labels.add(start) + labels.add(end) + labels.add(target) + labels = sorted(labels) + labels_map = {offset: i+1 for (i, offset) in enumerate(sorted(labels))} + for e in exception_entries: + e.start_label = labels_map[e.start] + e.end_label = labels_map[e.end] + e.target_label = labels_map[e.target] + return labels_map + +_NO_LINENO = ' --' + +def _get_lineno_width(linestarts): + if linestarts is None: + return 0 + maxlineno = max(filter(None, linestarts.values()), default=-1) + if maxlineno == -1: + # Omit the line number column entirely if we have no line number info + return 0 + lineno_width = max(3, len(str(maxlineno))) + if lineno_width < len(_NO_LINENO) and None in linestarts.values(): + lineno_width = len(_NO_LINENO) + return lineno_width + + +def _disassemble_bytes(code, lasti=-1, linestarts=None, + *, line_offset=0, exception_entries=(), + co_positions=None, original_code=None, + arg_resolver=None, formatter=None): + + assert formatter is not None + assert arg_resolver is not None + + instrs = _get_instructions_bytes(code, linestarts=linestarts, + line_offset=line_offset, + co_positions=co_positions, + original_code=original_code, + arg_resolver=arg_resolver) + + print_instructions(instrs, exception_entries, formatter, lasti=lasti) + + +def print_instructions(instrs, exception_entries, formatter, lasti=-1): + for instr in instrs: + # Each CACHE takes 2 bytes + is_current_instr = instr.offset <= lasti \ + <= instr.offset + 2 * _get_cache_size(_all_opname[_deoptop(instr.opcode)]) + formatter.print_instruction(instr, is_current_instr) + + formatter.print_exception_table(exception_entries) + +def _disassemble_str(source, **kwargs): + """Compile the source string, then disassemble the code object.""" + _disassemble_recursive(_try_compile(source, ''), **kwargs) + +disco = disassemble # XXX For backwards compatibility + + +# Rely on C `int` being 32 bits for oparg +_INT_BITS = 32 +# Value for c int when it overflows +_INT_OVERFLOW = 2 ** (_INT_BITS - 1) + +def _unpack_opargs(code): + extended_arg = 0 + extended_args_offset = 0 # Number of EXTENDED_ARG instructions preceding the current instruction + caches = 0 + for i in range(0, len(code), 2): + # Skip inline CACHE entries: + if caches: + caches -= 1 + continue + op = code[i] + deop = _deoptop(op) + caches = _get_cache_size(_all_opname[deop]) + if deop in hasarg: + arg = code[i+1] | extended_arg + extended_arg = (arg << 8) if deop == EXTENDED_ARG else 0 + # The oparg is stored as a signed integer + # If the value exceeds its upper limit, it will overflow and wrap + # to a negative integer + if extended_arg >= _INT_OVERFLOW: + extended_arg -= 2 * _INT_OVERFLOW + else: + arg = None + extended_arg = 0 + if deop == EXTENDED_ARG: + extended_args_offset += 1 + yield (i, i, op, arg) + else: + start_offset = i - extended_args_offset*2 + yield (i, start_offset, op, arg) + extended_args_offset = 0 + +def findlabels(code): + """Detect all offsets in a byte code which are jump targets. + + Return the list of offsets. + + """ + labels = [] + for offset, _, op, arg in _unpack_opargs(code): + if arg is not None: + label = _get_jump_target(op, arg, offset) + if label is None: + continue + if label not in labels: + labels.append(label) + return labels + +def findlinestarts(code): + """Find the offsets in a byte code which are start of lines in the source. + + Generate pairs (offset, lineno) + lineno will be an integer or None the offset does not have a source line. + """ + + lastline = False # None is a valid line number + for start, end, line in code.co_lines(): + if line is not lastline: + lastline = line + yield start, line + return + +def _find_imports(co): + """Find import statements in the code + + Generate triplets (name, level, fromlist) where + name is the imported module and level, fromlist are + the corresponding args to __import__. + """ + IMPORT_NAME = opmap['IMPORT_NAME'] + + consts = co.co_consts + names = co.co_names + opargs = [(op, arg) for _, _, op, arg in _unpack_opargs(co.co_code) + if op != EXTENDED_ARG] + for i, (op, oparg) in enumerate(opargs): + if op == IMPORT_NAME and i >= 2: + from_op = opargs[i-1] + level_op = opargs[i-2] + if (from_op[0] in hasconst and level_op[0] in hasconst): + level = _get_const_value(level_op[0], level_op[1], consts) + fromlist = _get_const_value(from_op[0], from_op[1], consts) + yield (names[oparg], level, fromlist) + +def _find_store_names(co): + """Find names of variables which are written in the code + + Generate sequence of strings + """ + STORE_OPS = { + opmap['STORE_NAME'], + opmap['STORE_GLOBAL'] + } + + names = co.co_names + for _, _, op, arg in _unpack_opargs(co.co_code): + if op in STORE_OPS: + yield names[arg] + + +class Bytecode: + """The bytecode operations of a piece of code + + Instantiate this with a function, method, other compiled object, string of + code, or a code object (as returned by compile()). + + Iterating over this yields the bytecode operations as Instruction instances. + """ + def __init__(self, x, *, first_line=None, current_offset=None, show_caches=False, adaptive=False, show_offsets=False): + self.codeobj = co = _get_code_object(x) + if first_line is None: + self.first_line = co.co_firstlineno + self._line_offset = 0 + else: + self.first_line = first_line + self._line_offset = first_line - co.co_firstlineno + self._linestarts = dict(findlinestarts(co)) + self._original_object = x + self.current_offset = current_offset + self.exception_entries = _parse_exception_table(co) + self.show_caches = show_caches + self.adaptive = adaptive + self.show_offsets = show_offsets + + def __iter__(self): + co = self.codeobj + original_code = co.co_code + labels_map = _make_labels_map(original_code, self.exception_entries) + arg_resolver = ArgResolver(co_consts=co.co_consts, + names=co.co_names, + varname_from_oparg=co._varname_from_oparg, + labels_map=labels_map) + return _get_instructions_bytes(_get_code_array(co, self.adaptive), + linestarts=self._linestarts, + line_offset=self._line_offset, + co_positions=co.co_positions(), + original_code=original_code, + arg_resolver=arg_resolver) + + def __repr__(self): + return "{}({!r})".format(self.__class__.__name__, + self._original_object) + + @classmethod + def from_traceback(cls, tb, *, show_caches=False, adaptive=False): + """ Construct a Bytecode from the given traceback """ + while tb.tb_next: + tb = tb.tb_next + return cls( + tb.tb_frame.f_code, current_offset=tb.tb_lasti, show_caches=show_caches, adaptive=adaptive + ) + + def info(self): + """Return formatted information about the code object.""" + return _format_code_info(self.codeobj) + + def dis(self): + """Return a formatted view of the bytecode operations.""" + co = self.codeobj + if self.current_offset is not None: + offset = self.current_offset + else: + offset = -1 + with io.StringIO() as output: + code = _get_code_array(co, self.adaptive) + offset_width = len(str(max(len(code) - 2, 9999))) if self.show_offsets else 0 + + + labels_map = _make_labels_map(co.co_code, self.exception_entries) + label_width = 4 + len(str(len(labels_map))) + formatter = Formatter(file=output, + lineno_width=_get_lineno_width(self._linestarts), + offset_width=offset_width, + label_width=label_width, + line_offset=self._line_offset, + show_caches=self.show_caches) + + arg_resolver = ArgResolver(co_consts=co.co_consts, + names=co.co_names, + varname_from_oparg=co._varname_from_oparg, + labels_map=labels_map) + _disassemble_bytes(code, + linestarts=self._linestarts, + line_offset=self._line_offset, + lasti=offset, + exception_entries=self.exception_entries, + co_positions=co.co_positions(), + original_code=co.co_code, + arg_resolver=arg_resolver, + formatter=formatter) + return output.getvalue() + + +from _dis import * # TODO: RUSTPYTHON; Remove this import (and module) + + +def main(args=None): import argparse parser = argparse.ArgumentParser() - parser.add_argument('infile', type=argparse.FileType('rb'), nargs='?', default='-') - args = parser.parse_args() - with args.infile as infile: - source = infile.read() - code = compile(source, args.infile.name, "exec") + parser.add_argument('-C', '--show-caches', action='store_true', + help='show inline caches') + parser.add_argument('-O', '--show-offsets', action='store_true', + help='show instruction offsets') + parser.add_argument('infile', nargs='?', default='-') + args = parser.parse_args(args=args) + if args.infile == '-': + name = '' + source = sys.stdin.buffer.read() + else: + name = args.infile + with open(args.infile, 'rb') as infile: + source = infile.read() + code = compile(source, name, "exec") + # TODO: RUSTPYTHON; Add support for `show_caches` & `show_offsets` arguments + # dis(code, show_caches=args.show_caches, show_offsets=args.show_offsets) dis(code) if __name__ == "__main__": - _test() + main() diff --git a/Lib/encodings/aliases.py b/Lib/encodings/aliases.py index d85afd6d5c..6a5ca046b5 100644 --- a/Lib/encodings/aliases.py +++ b/Lib/encodings/aliases.py @@ -209,6 +209,7 @@ 'ms932' : 'cp932', 'mskanji' : 'cp932', 'ms_kanji' : 'cp932', + 'windows_31j' : 'cp932', # cp949 codec '949' : 'cp949', diff --git a/Lib/encodings/idna.py b/Lib/encodings/idna.py index 5396047a7f..0c90b4c9fe 100644 --- a/Lib/encodings/idna.py +++ b/Lib/encodings/idna.py @@ -11,7 +11,7 @@ sace_prefix = "xn--" # This assumes query strings, so AllowUnassigned is true -def nameprep(label): +def nameprep(label): # type: (str) -> str # Map newlabel = [] for c in label: @@ -25,7 +25,7 @@ def nameprep(label): label = unicodedata.normalize("NFKC", label) # Prohibit - for c in label: + for i, c in enumerate(label): if stringprep.in_table_c12(c) or \ stringprep.in_table_c22(c) or \ stringprep.in_table_c3(c) or \ @@ -35,7 +35,7 @@ def nameprep(label): stringprep.in_table_c7(c) or \ stringprep.in_table_c8(c) or \ stringprep.in_table_c9(c): - raise UnicodeError("Invalid character %r" % c) + raise UnicodeEncodeError("idna", label, i, i+1, f"Invalid character {c!r}") # Check bidi RandAL = [stringprep.in_table_d1(x) for x in label] @@ -46,29 +46,38 @@ def nameprep(label): # This is table C.8, which was already checked # 2) If a string contains any RandALCat character, the string # MUST NOT contain any LCat character. - if any(stringprep.in_table_d2(x) for x in label): - raise UnicodeError("Violation of BIDI requirement 2") + for i, x in enumerate(label): + if stringprep.in_table_d2(x): + raise UnicodeEncodeError("idna", label, i, i+1, + "Violation of BIDI requirement 2") # 3) If a string contains any RandALCat character, a # RandALCat character MUST be the first character of the # string, and a RandALCat character MUST be the last # character of the string. - if not RandAL[0] or not RandAL[-1]: - raise UnicodeError("Violation of BIDI requirement 3") + if not RandAL[0]: + raise UnicodeEncodeError("idna", label, 0, 1, + "Violation of BIDI requirement 3") + if not RandAL[-1]: + raise UnicodeEncodeError("idna", label, len(label)-1, len(label), + "Violation of BIDI requirement 3") return label -def ToASCII(label): +def ToASCII(label): # type: (str) -> bytes try: # Step 1: try ASCII - label = label.encode("ascii") - except UnicodeError: + label_ascii = label.encode("ascii") + except UnicodeEncodeError: pass else: # Skip to step 3: UseSTD3ASCIIRules is false, so # Skip to step 8. - if 0 < len(label) < 64: - return label - raise UnicodeError("label empty or too long") + if 0 < len(label_ascii) < 64: + return label_ascii + if len(label) == 0: + raise UnicodeEncodeError("idna", label, 0, 1, "label empty") + else: + raise UnicodeEncodeError("idna", label, 0, len(label), "label too long") # Step 2: nameprep label = nameprep(label) @@ -76,29 +85,34 @@ def ToASCII(label): # Step 3: UseSTD3ASCIIRules is false # Step 4: try ASCII try: - label = label.encode("ascii") - except UnicodeError: + label_ascii = label.encode("ascii") + except UnicodeEncodeError: pass else: # Skip to step 8. if 0 < len(label) < 64: - return label - raise UnicodeError("label empty or too long") + return label_ascii + if len(label) == 0: + raise UnicodeEncodeError("idna", label, 0, 1, "label empty") + else: + raise UnicodeEncodeError("idna", label, 0, len(label), "label too long") # Step 5: Check ACE prefix - if label.startswith(sace_prefix): - raise UnicodeError("Label starts with ACE prefix") + if label.lower().startswith(sace_prefix): + raise UnicodeEncodeError( + "idna", label, 0, len(sace_prefix), "Label starts with ACE prefix") # Step 6: Encode with PUNYCODE - label = label.encode("punycode") + label_ascii = label.encode("punycode") # Step 7: Prepend ACE prefix - label = ace_prefix + label + label_ascii = ace_prefix + label_ascii # Step 8: Check size - if 0 < len(label) < 64: - return label - raise UnicodeError("label empty or too long") + # do not check for empty as we prepend ace_prefix. + if len(label_ascii) < 64: + return label_ascii + raise UnicodeEncodeError("idna", label, 0, len(label), "label too long") def ToUnicode(label): if len(label) > 1024: @@ -110,7 +124,9 @@ def ToUnicode(label): # per https://www.rfc-editor.org/rfc/rfc3454#section-3.1 while still # preventing us from wasting time decoding a big thing that'll just # hit the actual <= 63 length limit in Step 6. - raise UnicodeError("label way too long") + if isinstance(label, str): + label = label.encode("utf-8", errors="backslashreplace") + raise UnicodeDecodeError("idna", label, 0, len(label), "label way too long") # Step 1: Check for ASCII if isinstance(label, bytes): pure_ascii = True @@ -118,25 +134,32 @@ def ToUnicode(label): try: label = label.encode("ascii") pure_ascii = True - except UnicodeError: + except UnicodeEncodeError: pure_ascii = False if not pure_ascii: + assert isinstance(label, str) # Step 2: Perform nameprep label = nameprep(label) # It doesn't say this, but apparently, it should be ASCII now try: label = label.encode("ascii") - except UnicodeError: - raise UnicodeError("Invalid character in IDN label") + except UnicodeEncodeError as exc: + raise UnicodeEncodeError("idna", label, exc.start, exc.end, + "Invalid character in IDN label") # Step 3: Check for ACE prefix - if not label.startswith(ace_prefix): + assert isinstance(label, bytes) + if not label.lower().startswith(ace_prefix): return str(label, "ascii") # Step 4: Remove ACE prefix label1 = label[len(ace_prefix):] # Step 5: Decode using PUNYCODE - result = label1.decode("punycode") + try: + result = label1.decode("punycode") + except UnicodeDecodeError as exc: + offset = len(ace_prefix) + raise UnicodeDecodeError("idna", label, offset+exc.start, offset+exc.end, exc.reason) # Step 6: Apply ToASCII label2 = ToASCII(result) @@ -144,7 +167,8 @@ def ToUnicode(label): # Step 7: Compare the result of step 6 with the one of step 3 # label2 will already be in lower case. if str(label, "ascii").lower() != str(label2, "ascii"): - raise UnicodeError("IDNA does not round-trip", label, label2) + raise UnicodeDecodeError("idna", label, 0, len(label), + f"IDNA does not round-trip, '{label!r}' != '{label2!r}'") # Step 8: return the result of step 5 return result @@ -156,7 +180,7 @@ def encode(self, input, errors='strict'): if errors != 'strict': # IDNA is quite clear that implementations must be strict - raise UnicodeError("unsupported error handling "+errors) + raise UnicodeError(f"Unsupported error handling: {errors}") if not input: return b'', 0 @@ -168,11 +192,16 @@ def encode(self, input, errors='strict'): else: # ASCII name: fast path labels = result.split(b'.') - for label in labels[:-1]: - if not (0 < len(label) < 64): - raise UnicodeError("label empty or too long") - if len(labels[-1]) >= 64: - raise UnicodeError("label too long") + for i, label in enumerate(labels[:-1]): + if len(label) == 0: + offset = sum(len(l) for l in labels[:i]) + i + raise UnicodeEncodeError("idna", input, offset, offset+1, + "label empty") + for i, label in enumerate(labels): + if len(label) >= 64: + offset = sum(len(l) for l in labels[:i]) + i + raise UnicodeEncodeError("idna", input, offset, offset+len(label), + "label too long") return result, len(input) result = bytearray() @@ -182,17 +211,27 @@ def encode(self, input, errors='strict'): del labels[-1] else: trailing_dot = b'' - for label in labels: + for i, label in enumerate(labels): if result: # Join with U+002E result.extend(b'.') - result.extend(ToASCII(label)) + try: + result.extend(ToASCII(label)) + except (UnicodeEncodeError, UnicodeDecodeError) as exc: + offset = sum(len(l) for l in labels[:i]) + i + raise UnicodeEncodeError( + "idna", + input, + offset + exc.start, + offset + exc.end, + exc.reason, + ) return bytes(result+trailing_dot), len(input) def decode(self, input, errors='strict'): if errors != 'strict': - raise UnicodeError("Unsupported error handling "+errors) + raise UnicodeError(f"Unsupported error handling: {errors}") if not input: return "", 0 @@ -202,7 +241,7 @@ def decode(self, input, errors='strict'): # XXX obviously wrong, see #3232 input = bytes(input) - if ace_prefix not in input: + if ace_prefix not in input.lower(): # Fast path try: return input.decode('ascii'), len(input) @@ -218,8 +257,15 @@ def decode(self, input, errors='strict'): trailing_dot = '' result = [] - for label in labels: - result.append(ToUnicode(label)) + for i, label in enumerate(labels): + try: + u_label = ToUnicode(label) + except (UnicodeEncodeError, UnicodeDecodeError) as exc: + offset = sum(len(x) for x in labels[:i]) + len(labels[:i]) + raise UnicodeDecodeError( + "idna", input, offset+exc.start, offset+exc.end, exc.reason) + else: + result.append(u_label) return ".".join(result)+trailing_dot, len(input) @@ -227,7 +273,7 @@ class IncrementalEncoder(codecs.BufferedIncrementalEncoder): def _buffer_encode(self, input, errors, final): if errors != 'strict': # IDNA is quite clear that implementations must be strict - raise UnicodeError("unsupported error handling "+errors) + raise UnicodeError(f"Unsupported error handling: {errors}") if not input: return (b'', 0) @@ -251,7 +297,16 @@ def _buffer_encode(self, input, errors, final): # Join with U+002E result.extend(b'.') size += 1 - result.extend(ToASCII(label)) + try: + result.extend(ToASCII(label)) + except (UnicodeEncodeError, UnicodeDecodeError) as exc: + raise UnicodeEncodeError( + "idna", + input, + size + exc.start, + size + exc.end, + exc.reason, + ) size += len(label) result += trailing_dot @@ -261,7 +316,7 @@ def _buffer_encode(self, input, errors, final): class IncrementalDecoder(codecs.BufferedIncrementalDecoder): def _buffer_decode(self, input, errors, final): if errors != 'strict': - raise UnicodeError("Unsupported error handling "+errors) + raise UnicodeError(f"Unsupported error handling: {errors}") if not input: return ("", 0) @@ -271,7 +326,11 @@ def _buffer_decode(self, input, errors, final): labels = dots.split(input) else: # Must be ASCII string - input = str(input, "ascii") + try: + input = str(input, "ascii") + except (UnicodeEncodeError, UnicodeDecodeError) as exc: + raise UnicodeDecodeError("idna", input, + exc.start, exc.end, exc.reason) labels = input.split(".") trailing_dot = '' @@ -288,7 +347,18 @@ def _buffer_decode(self, input, errors, final): result = [] size = 0 for label in labels: - result.append(ToUnicode(label)) + try: + u_label = ToUnicode(label) + except (UnicodeEncodeError, UnicodeDecodeError) as exc: + raise UnicodeDecodeError( + "idna", + input.encode("ascii", errors="backslashreplace"), + size + exc.start, + size + exc.end, + exc.reason, + ) + else: + result.append(u_label) if size: size += 1 size += len(label) diff --git a/Lib/encodings/palmos.py b/Lib/encodings/palmos.py index c506d65452..df164ca5b9 100644 --- a/Lib/encodings/palmos.py +++ b/Lib/encodings/palmos.py @@ -201,7 +201,7 @@ def getregentry(): '\u02dc' # 0x98 -> SMALL TILDE '\u2122' # 0x99 -> TRADE MARK SIGN '\u0161' # 0x9A -> LATIN SMALL LETTER S WITH CARON - '\x9b' # 0x9B -> + '\u203a' # 0x9B -> SINGLE RIGHT-POINTING ANGLE QUOTATION MARK '\u0153' # 0x9C -> LATIN SMALL LIGATURE OE '\x9d' # 0x9D -> '\x9e' # 0x9E -> diff --git a/Lib/encodings/punycode.py b/Lib/encodings/punycode.py index 1c57264470..4622fc8c92 100644 --- a/Lib/encodings/punycode.py +++ b/Lib/encodings/punycode.py @@ -1,4 +1,4 @@ -""" Codec for the Punicode encoding, as specified in RFC 3492 +""" Codec for the Punycode encoding, as specified in RFC 3492 Written by Martin v. Löwis. """ @@ -131,10 +131,11 @@ def decode_generalized_number(extended, extpos, bias, errors): j = 0 while 1: try: - char = ord(extended[extpos]) + char = extended[extpos] except IndexError: if errors == "strict": - raise UnicodeError("incomplete punicode string") + raise UnicodeDecodeError("punycode", extended, extpos, extpos+1, + "incomplete punycode string") return extpos + 1, None extpos += 1 if 0x41 <= char <= 0x5A: # A-Z @@ -142,8 +143,8 @@ def decode_generalized_number(extended, extpos, bias, errors): elif 0x30 <= char <= 0x39: digit = char - 22 # 0x30-26 elif errors == "strict": - raise UnicodeError("Invalid extended code point '%s'" - % extended[extpos-1]) + raise UnicodeDecodeError("punycode", extended, extpos-1, extpos, + f"Invalid extended code point '{extended[extpos-1]}'") else: return extpos, None t = T(j, bias) @@ -155,11 +156,14 @@ def decode_generalized_number(extended, extpos, bias, errors): def insertion_sort(base, extended, errors): - """3.2 Insertion unsort coding""" + """3.2 Insertion sort coding""" + # This function raises UnicodeDecodeError with position in the extended. + # Caller should add the offset. char = 0x80 pos = -1 bias = 72 extpos = 0 + while extpos < len(extended): newpos, delta = decode_generalized_number(extended, extpos, bias, errors) @@ -171,7 +175,9 @@ def insertion_sort(base, extended, errors): char += pos // (len(base) + 1) if char > 0x10FFFF: if errors == "strict": - raise UnicodeError("Invalid character U+%x" % char) + raise UnicodeDecodeError( + "punycode", extended, pos-1, pos, + f"Invalid character U+{char:x}") char = ord('?') pos = pos % (len(base) + 1) base = base[:pos] + chr(char) + base[pos:] @@ -187,11 +193,21 @@ def punycode_decode(text, errors): pos = text.rfind(b"-") if pos == -1: base = "" - extended = str(text, "ascii").upper() + extended = text.upper() else: - base = str(text[:pos], "ascii", errors) - extended = str(text[pos+1:], "ascii").upper() - return insertion_sort(base, extended, errors) + try: + base = str(text[:pos], "ascii", errors) + except UnicodeDecodeError as exc: + raise UnicodeDecodeError("ascii", text, exc.start, exc.end, + exc.reason) from None + extended = text[pos+1:].upper() + try: + return insertion_sort(base, extended, errors) + except UnicodeDecodeError as exc: + offset = pos + 1 + raise UnicodeDecodeError("punycode", text, + offset+exc.start, offset+exc.end, + exc.reason) from None ### Codec APIs @@ -203,7 +219,7 @@ def encode(self, input, errors='strict'): def decode(self, input, errors='strict'): if errors not in ('strict', 'replace', 'ignore'): - raise UnicodeError("Unsupported error handling "+errors) + raise UnicodeError(f"Unsupported error handling: {errors}") res = punycode_decode(input, errors) return res, len(input) @@ -214,7 +230,7 @@ def encode(self, input, final=False): class IncrementalDecoder(codecs.IncrementalDecoder): def decode(self, input, final=False): if self.errors not in ('strict', 'replace', 'ignore'): - raise UnicodeError("Unsupported error handling "+self.errors) + raise UnicodeError(f"Unsupported error handling: {self.errors}") return punycode_decode(input, self.errors) class StreamWriter(Codec,codecs.StreamWriter): diff --git a/Lib/encodings/undefined.py b/Lib/encodings/undefined.py index 4690288355..082771e1c8 100644 --- a/Lib/encodings/undefined.py +++ b/Lib/encodings/undefined.py @@ -1,6 +1,6 @@ """ Python 'undefined' Codec - This codec will always raise a ValueError exception when being + This codec will always raise a UnicodeError exception when being used. It is intended for use by the site.py file to switch off automatic string to Unicode coercion. diff --git a/Lib/encodings/utf_16.py b/Lib/encodings/utf_16.py index c61248242b..d3b9980026 100644 --- a/Lib/encodings/utf_16.py +++ b/Lib/encodings/utf_16.py @@ -64,7 +64,7 @@ def _buffer_decode(self, input, errors, final): elif byteorder == 1: self.decoder = codecs.utf_16_be_decode elif consumed >= 2: - raise UnicodeError("UTF-16 stream does not start with BOM") + raise UnicodeDecodeError("utf-16", input, 0, 2, "Stream does not start with BOM") return (output, consumed) return self.decoder(input, self.errors, final) @@ -138,7 +138,7 @@ def decode(self, input, errors='strict'): elif byteorder == 1: self.decode = codecs.utf_16_be_decode elif consumed>=2: - raise UnicodeError("UTF-16 stream does not start with BOM") + raise UnicodeDecodeError("utf-16", input, 0, 2, "Stream does not start with BOM") return (object, consumed) ### encodings module API diff --git a/Lib/encodings/utf_32.py b/Lib/encodings/utf_32.py index cdf84d1412..1924bedbb7 100644 --- a/Lib/encodings/utf_32.py +++ b/Lib/encodings/utf_32.py @@ -59,7 +59,7 @@ def _buffer_decode(self, input, errors, final): elif byteorder == 1: self.decoder = codecs.utf_32_be_decode elif consumed >= 4: - raise UnicodeError("UTF-32 stream does not start with BOM") + raise UnicodeDecodeError("utf-32", input, 0, 4, "Stream does not start with BOM") return (output, consumed) return self.decoder(input, self.errors, final) @@ -132,8 +132,8 @@ def decode(self, input, errors='strict'): self.decode = codecs.utf_32_le_decode elif byteorder == 1: self.decode = codecs.utf_32_be_decode - elif consumed>=4: - raise UnicodeError("UTF-32 stream does not start with BOM") + elif consumed >= 4: + raise UnicodeDecodeError("utf-32", input, 0, 4, "Stream does not start with BOM") return (object, consumed) ### encodings module API diff --git a/Lib/ensurepip/__init__.py b/Lib/ensurepip/__init__.py index 1fb1d505cf..ab6d32478e 100644 --- a/Lib/ensurepip/__init__.py +++ b/Lib/ensurepip/__init__.py @@ -1,78 +1,64 @@ -import collections import os -import os.path import subprocess import sys import sysconfig import tempfile +from contextlib import nullcontext from importlib import resources +from pathlib import Path +from shutil import copy2 __all__ = ["version", "bootstrap"] -_PACKAGE_NAMES = ('pip',) -_PIP_VERSION = "23.2.1" -_PROJECTS = [ - ("pip", _PIP_VERSION, "py3"), -] - -# Packages bundled in ensurepip._bundled have wheel_name set. -# Packages from WHEEL_PKG_DIR have wheel_path set. -_Package = collections.namedtuple('Package', - ('version', 'wheel_name', 'wheel_path')) +_PIP_VERSION = "25.2" # Directory of system wheel packages. Some Linux distribution packaging # policies recommend against bundling dependencies. For example, Fedora # installs wheel packages in the /usr/share/python-wheels/ directory and don't # install the ensurepip._bundled package. -_WHEEL_PKG_DIR = sysconfig.get_config_var('WHEEL_PKG_DIR') +if (_pkg_dir := sysconfig.get_config_var('WHEEL_PKG_DIR')) is not None: + _WHEEL_PKG_DIR = Path(_pkg_dir).resolve() +else: + _WHEEL_PKG_DIR = None + +def _find_wheel_pkg_dir_pip(): + if _WHEEL_PKG_DIR is None: + # NOTE: The compile-time `WHEEL_PKG_DIR` is unset so there is no place + # NOTE: for looking up the wheels. + return None -def _find_packages(path): - packages = {} + dist_matching_wheels = _WHEEL_PKG_DIR.glob('pip-*.whl') try: - filenames = os.listdir(path) - except OSError: - # Ignore: path doesn't exist or permission error - filenames = () - # Make the code deterministic if a directory contains multiple wheel files - # of the same package, but don't attempt to implement correct version - # comparison since this case should not happen. - filenames = sorted(filenames) - for filename in filenames: - # filename is like 'pip-21.2.4-py3-none-any.whl' - if not filename.endswith(".whl"): - continue - for name in _PACKAGE_NAMES: - prefix = name + '-' - if filename.startswith(prefix): - break - else: - continue - - # Extract '21.2.4' from 'pip-21.2.4-py3-none-any.whl' - version = filename.removeprefix(prefix).partition('-')[0] - wheel_path = os.path.join(path, filename) - packages[name] = _Package(version, None, wheel_path) - return packages - - -def _get_packages(): - global _PACKAGES, _WHEEL_PKG_DIR - if _PACKAGES is not None: - return _PACKAGES - - packages = {} - for name, version, py_tag in _PROJECTS: - wheel_name = f"{name}-{version}-{py_tag}-none-any.whl" - packages[name] = _Package(version, wheel_name, None) - if _WHEEL_PKG_DIR: - dir_packages = _find_packages(_WHEEL_PKG_DIR) - # only used the wheel package directory if all packages are found there - if all(name in dir_packages for name in _PACKAGE_NAMES): - packages = dir_packages - _PACKAGES = packages - return packages -_PACKAGES = None + last_matching_dist_wheel = sorted(dist_matching_wheels)[-1] + except IndexError: + # NOTE: `WHEEL_PKG_DIR` does not contain any wheel files for `pip`. + return None + + return nullcontext(last_matching_dist_wheel) + + +def _get_pip_whl_path_ctx(): + # Prefer pip from the wheel package directory, if present. + if (alternative_pip_wheel_path := _find_wheel_pkg_dir_pip()) is not None: + return alternative_pip_wheel_path + + return resources.as_file( + resources.files('ensurepip') + / '_bundled' + / f'pip-{_PIP_VERSION}-py3-none-any.whl' + ) + + +def _get_pip_version(): + with _get_pip_whl_path_ctx() as bundled_wheel_path: + wheel_name = bundled_wheel_path.name + return ( + # Extract '21.2.4' from 'pip-21.2.4-py3-none-any.whl' + wheel_name. + removeprefix('pip-'). + partition('-')[0] + ) def _run_pip(args, additional_paths=None): @@ -105,7 +91,7 @@ def version(): """ Returns a string specifying the bundled version of pip. """ - return _get_packages()['pip'].version + return _get_pip_version() def _disable_pip_configuration_settings(): @@ -167,24 +153,10 @@ def _bootstrap(*, root=None, upgrade=False, user=False, with tempfile.TemporaryDirectory() as tmpdir: # Put our bundled wheels into a temporary directory and construct the # additional paths that need added to sys.path - additional_paths = [] - for name, package in _get_packages().items(): - if package.wheel_name: - # Use bundled wheel package - wheel_name = package.wheel_name - wheel_path = resources.files("ensurepip") / "_bundled" / wheel_name - whl = wheel_path.read_bytes() - else: - # Use the wheel package directory - with open(package.wheel_path, "rb") as fp: - whl = fp.read() - wheel_name = os.path.basename(package.wheel_path) - - filename = os.path.join(tmpdir, wheel_name) - with open(filename, "wb") as fp: - fp.write(whl) - - additional_paths.append(filename) + tmpdir_path = Path(tmpdir) + with _get_pip_whl_path_ctx() as bundled_wheel_path: + tmp_wheel_path = tmpdir_path / bundled_wheel_path.name + copy2(bundled_wheel_path, tmp_wheel_path) # Construct the arguments to be passed to the pip command args = ["install", "--no-cache-dir", "--no-index", "--find-links", tmpdir] @@ -197,7 +169,8 @@ def _bootstrap(*, root=None, upgrade=False, user=False, if verbosity: args += ["-" + "v" * verbosity] - return _run_pip([*args, *_PACKAGE_NAMES], additional_paths) + return _run_pip([*args, "pip"], [os.fsdecode(tmp_wheel_path)]) + def _uninstall_helper(*, verbosity=0): """Helper to support a clean default uninstall process on Windows @@ -227,7 +200,7 @@ def _uninstall_helper(*, verbosity=0): if verbosity: args += ["-" + "v" * verbosity] - return _run_pip([*args, *reversed(_PACKAGE_NAMES)]) + return _run_pip([*args, "pip"]) def _main(argv=None): diff --git a/Lib/ensurepip/_bundled/pip-23.2.1-py3-none-any.whl b/Lib/ensurepip/_bundled/pip-23.2.1-py3-none-any.whl deleted file mode 100644 index ba28ef02e2..0000000000 Binary files a/Lib/ensurepip/_bundled/pip-23.2.1-py3-none-any.whl and /dev/null differ diff --git a/Lib/ensurepip/_bundled/pip-25.2-py3-none-any.whl b/Lib/ensurepip/_bundled/pip-25.2-py3-none-any.whl new file mode 100644 index 0000000000..4db7e72071 Binary files /dev/null and b/Lib/ensurepip/_bundled/pip-25.2-py3-none-any.whl differ diff --git a/Lib/fnmatch.py b/Lib/fnmatch.py index fee59bf73f..73acb1fe8d 100644 --- a/Lib/fnmatch.py +++ b/Lib/fnmatch.py @@ -16,12 +16,6 @@ __all__ = ["filter", "fnmatch", "fnmatchcase", "translate"] -# Build a thread-safe incrementing counter to help create unique regexp group -# names across calls. -from itertools import count -_nextgroupnum = count().__next__ -del count - def fnmatch(name, pat): """Test whether FILENAME matches PATTERN. @@ -41,7 +35,7 @@ def fnmatch(name, pat): pat = os.path.normcase(pat) return fnmatchcase(name, pat) -@functools.lru_cache(maxsize=256, typed=True) +@functools.lru_cache(maxsize=32768, typed=True) def _compile_pattern(pat): if isinstance(pat, bytes): pat_str = str(pat, 'ISO-8859-1') @@ -84,6 +78,11 @@ def translate(pat): """ STAR = object() + parts = _translate(pat, STAR, '.') + return _join_translated_parts(parts, STAR) + + +def _translate(pat, STAR, QUESTION_MARK): res = [] add = res.append i, n = 0, len(pat) @@ -95,7 +94,7 @@ def translate(pat): if (not res) or res[-1] is not STAR: add(STAR) elif c == '?': - add('.') + add(QUESTION_MARK) elif c == '[': j = i if j < n and pat[j] == '!': @@ -152,9 +151,11 @@ def translate(pat): else: add(re.escape(c)) assert i == n + return res + +def _join_translated_parts(inp, STAR): # Deal with STARs. - inp = res res = [] add = res.append i, n = 0, len(inp) @@ -165,17 +166,10 @@ def translate(pat): # Now deal with STAR fixed STAR fixed ... # For an interior `STAR fixed` pairing, we want to do a minimal # .*? match followed by `fixed`, with no possibility of backtracking. - # We can't spell that directly, but can trick it into working by matching - # .*?fixed - # in a lookahead assertion, save the matched part in a group, then - # consume that group via a backreference. If the overall match fails, - # the lookahead assertion won't try alternatives. So the translation is: - # (?=(?P.*?fixed))(?P=name) - # Group names are created as needed: g0, g1, g2, ... - # The numbers are obtained from _nextgroupnum() to ensure they're unique - # across calls and across threads. This is because people rely on the - # undocumented ability to join multiple translate() results together via - # "|" to build large regexps matching "one of many" shell patterns. + # Atomic groups ("(?>...)") allow us to spell that directly. + # Note: people rely on the undocumented ability to join multiple + # translate() results together via "|" to build large regexps matching + # "one of many" shell patterns. while i < n: assert inp[i] is STAR i += 1 @@ -192,8 +186,7 @@ def translate(pat): add(".*") add(fixed) else: - groupnum = _nextgroupnum() - add(f"(?=(?P.*?{fixed}))(?P=g{groupnum})") + add(f"(?>.*?{fixed})") assert i == n res = "".join(res) return fr'(?s:{res})\Z' diff --git a/Lib/functools.py b/Lib/functools.py index 2ae4290f98..4c1175b815 100644 --- a/Lib/functools.py +++ b/Lib/functools.py @@ -19,8 +19,9 @@ # import types, weakref # Deferred to single_dispatch() from reprlib import recursive_repr from _thread import RLock -from types import GenericAlias +# Avoid importing types, so we can speedup import time +GenericAlias = type(list[int]) ################################################################################ ### update_wrapper() and wraps() decorator @@ -236,14 +237,16 @@ def __ge__(self, other): def reduce(function, sequence, initial=_initial_missing): """ - reduce(function, iterable[, initial]) -> value - - Apply a function of two arguments cumulatively to the items of a sequence - or iterable, from left to right, so as to reduce the iterable to a single - value. For example, reduce(lambda x, y: x+y, [1, 2, 3, 4, 5]) calculates - ((((1+2)+3)+4)+5). If initial is present, it is placed before the items - of the iterable in the calculation, and serves as a default when the - iterable is empty. + reduce(function, iterable[, initial], /) -> value + + Apply a function of two arguments cumulatively to the items of an iterable, from left to right. + + This effectively reduces the iterable to a single value. If initial is present, + it is placed before the items of the iterable in the calculation, and serves as + a default when the iterable is empty. + + For example, reduce(lambda x, y: x+y, [1, 2, 3, 4, 5]) + calculates ((((1 + 2) + 3) + 4) + 5). """ it = iter(sequence) @@ -284,7 +287,7 @@ def __new__(cls, func, /, *args, **keywords): if not callable(func): raise TypeError("the first argument must be callable") - if hasattr(func, "func"): + if isinstance(func, partial): args = func.args + args keywords = {**func.keywords, **keywords} func = func.func @@ -302,13 +305,23 @@ def __call__(self, /, *args, **keywords): @recursive_repr() def __repr__(self): - qualname = type(self).__qualname__ + cls = type(self) + qualname = cls.__qualname__ + module = cls.__module__ args = [repr(self.func)] args.extend(repr(x) for x in self.args) args.extend(f"{k}={v!r}" for (k, v) in self.keywords.items()) - if type(self).__module__ == "functools": - return f"functools.{qualname}({', '.join(args)})" - return f"{qualname}({', '.join(args)})" + return f"{module}.{qualname}({', '.join(args)})" + + def __get__(self, obj, objtype=None): + if obj is None: + return self + import warnings + warnings.warn('functools.partial will be a method descriptor in ' + 'future Python versions; wrap it in staticmethod() ' + 'if you want to preserve the old behavior', + FutureWarning, 2) + return self def __reduce__(self): return type(self), (self.func,), (self.func, self.args, @@ -338,6 +351,9 @@ def __setstate__(self, state): self.args = args self.keywords = kwds + __class_getitem__ = classmethod(GenericAlias) + + try: from _functools import partial except ImportError: @@ -372,28 +388,26 @@ def __init__(self, func, /, *args, **keywords): self.keywords = keywords def __repr__(self): - args = ", ".join(map(repr, self.args)) - keywords = ", ".join("{}={!r}".format(k, v) - for k, v in self.keywords.items()) - format_string = "{module}.{cls}({func}, {args}, {keywords})" - return format_string.format(module=self.__class__.__module__, - cls=self.__class__.__qualname__, - func=self.func, - args=args, - keywords=keywords) + cls = type(self) + module = cls.__module__ + qualname = cls.__qualname__ + args = [repr(self.func)] + args.extend(map(repr, self.args)) + args.extend(f"{k}={v!r}" for k, v in self.keywords.items()) + return f"{module}.{qualname}({', '.join(args)})" def _make_unbound_method(self): def _method(cls_or_self, /, *args, **keywords): keywords = {**self.keywords, **keywords} return self.func(cls_or_self, *self.args, *args, **keywords) _method.__isabstractmethod__ = self.__isabstractmethod__ - _method._partialmethod = self + _method.__partialmethod__ = self return _method def __get__(self, obj, cls=None): get = getattr(self.func, "__get__", None) result = None - if get is not None: + if get is not None and not isinstance(self.func, partial): new_func = get(obj, cls) if new_func is not self.func: # Assume __get__ returning something new indicates the @@ -423,6 +437,17 @@ def _unwrap_partial(func): func = func.func return func +def _unwrap_partialmethod(func): + prev = None + while func is not prev: + prev = func + while isinstance(getattr(func, "__partialmethod__", None), partialmethod): + func = func.__partialmethod__ + while isinstance(func, partialmethod): + func = getattr(func, 'func') + func = _unwrap_partial(func) + return func + ################################################################################ ### LRU Cache function decorator ################################################################################ @@ -483,8 +508,9 @@ def lru_cache(maxsize=128, typed=False): can grow without bound. If *typed* is True, arguments of different types will be cached separately. - For example, f(3.0) and f(3) will be treated as distinct calls with - distinct results. + For example, f(decimal.Decimal("3.0")) and f(3.0) will be treated as + distinct calls with distinct results. Some types such as str and int may + be cached separately even when typed is false. Arguments to the cached function must be hashable. @@ -660,7 +686,7 @@ def cache(user_function, /): def _c3_merge(sequences): """Merges MROs in *sequences* to a single MRO using the C3 algorithm. - Adapted from https://www.python.org/download/releases/2.3/mro/. + Adapted from https://docs.python.org/3/howto/mro.html. """ result = [] @@ -905,7 +931,6 @@ def wrapper(*args, **kw): if not args: raise TypeError(f'{funcname} requires at least ' '1 positional argument') - return dispatch(args[0].__class__)(*args, **kw) funcname = getattr(func, '__name__', 'singledispatch function') @@ -941,13 +966,18 @@ def register(self, cls, method=None): return self.dispatcher.register(cls, func=method) def __get__(self, obj, cls=None): + dispatch = self.dispatcher.dispatch + funcname = getattr(self.func, '__name__', 'singledispatchmethod method') def _method(*args, **kwargs): - method = self.dispatcher.dispatch(args[0].__class__) - return method.__get__(obj, cls)(*args, **kwargs) + if not args: + raise TypeError(f'{funcname} requires at least ' + '1 positional argument') + return dispatch(args[0].__class__).__get__(obj, cls)(*args, **kwargs) _method.__isabstractmethod__ = self.__isabstractmethod__ _method.register = self.register update_wrapper(_method, self.func) + return _method @property @@ -966,6 +996,7 @@ def __init__(self, func): self.func = func self.attrname = None self.__doc__ = func.__doc__ + self.__module__ = func.__module__ def __set_name__(self, owner, name): if self.attrname is None: diff --git a/Lib/genericpath.py b/Lib/genericpath.py index 1bd5b3897c..9363f564aa 100644 --- a/Lib/genericpath.py +++ b/Lib/genericpath.py @@ -7,8 +7,8 @@ import stat __all__ = ['commonprefix', 'exists', 'getatime', 'getctime', 'getmtime', - 'getsize', 'isdir', 'isfile', 'islink', 'samefile', 'sameopenfile', - 'samestat'] + 'getsize', 'isdevdrive', 'isdir', 'isfile', 'isjunction', 'islink', + 'lexists', 'samefile', 'sameopenfile', 'samestat', 'ALLOW_MISSING'] # Does a path exist? @@ -22,6 +22,15 @@ def exists(path): return True +# Being true for dangling symbolic links is also useful. +def lexists(path): + """Test whether a path exists. Returns True for broken symbolic links""" + try: + os.lstat(path) + except (OSError, ValueError): + return False + return True + # This follows symbolic links, so both islink() and isdir() can be true # for the same path on systems that support symlinks def isfile(path): @@ -57,6 +66,21 @@ def islink(path): return stat.S_ISLNK(st.st_mode) +# Is a path a junction? +def isjunction(path): + """Test whether a path is a junction + Junctions are not supported on the current platform""" + os.fspath(path) + return False + + +def isdevdrive(path): + """Determines whether the specified path is on a Windows Dev Drive. + Dev Drives are not supported on the current platform""" + os.fspath(path) + return False + + def getsize(filename): """Return the size of a file, reported by os.stat().""" return os.stat(filename).st_size @@ -165,3 +189,12 @@ def _check_arg_types(funcname, *args): f'os.PathLike object, not {s.__class__.__name__!r}') from None if hasstr and hasbytes: raise TypeError("Can't mix strings and bytes in path components") from None + +# A singleton with a true boolean value. +@object.__new__ +class ALLOW_MISSING: + """Special value for use in realpath().""" + def __repr__(self): + return 'os.path.ALLOW_MISSING' + def __reduce__(self): + return self.__class__.__name__ diff --git a/Lib/gettext.py b/Lib/gettext.py index b72b15f82d..62cff81b7b 100644 --- a/Lib/gettext.py +++ b/Lib/gettext.py @@ -46,6 +46,7 @@ # find this format documented anywhere. +import operator import os import re import sys @@ -166,14 +167,28 @@ def _parse(tokens, priority=-1): def _as_int(n): try: - i = round(n) + round(n) except TypeError: raise TypeError('Plural value must be an integer, got %s' % (n.__class__.__name__,)) from None + return _as_int2(n) + +def _as_int2(n): + try: + return operator.index(n) + except TypeError: + pass + import warnings + frame = sys._getframe(1) + stacklevel = 2 + while frame.f_back is not None and frame.f_globals.get('__name__') == __name__: + stacklevel += 1 + frame = frame.f_back warnings.warn('Plural value must be an integer, got %s' % (n.__class__.__name__,), - DeprecationWarning, 4) + DeprecationWarning, + stacklevel) return n @@ -200,7 +215,7 @@ def c2py(plural): elif c == ')': depth -= 1 - ns = {'_as_int': _as_int} + ns = {'_as_int': _as_int, '__name__': __name__} exec('''if True: def func(n): if not isinstance(n, int): @@ -280,6 +295,7 @@ def gettext(self, message): def ngettext(self, msgid1, msgid2, n): if self._fallback: return self._fallback.ngettext(msgid1, msgid2, n) + n = _as_int2(n) if n == 1: return msgid1 else: @@ -293,6 +309,7 @@ def pgettext(self, context, message): def npgettext(self, context, msgid1, msgid2, n): if self._fallback: return self._fallback.npgettext(context, msgid1, msgid2, n) + n = _as_int2(n) if n == 1: return msgid1 else: @@ -579,6 +596,7 @@ def dngettext(domain, msgid1, msgid2, n): try: t = translation(domain, _localedirs.get(domain, None)) except OSError: + n = _as_int2(n) if n == 1: return msgid1 else: @@ -598,6 +616,7 @@ def dnpgettext(domain, context, msgid1, msgid2, n): try: t = translation(domain, _localedirs.get(domain, None)) except OSError: + n = _as_int2(n) if n == 1: return msgid1 else: diff --git a/Lib/glob.py b/Lib/glob.py index 50beef37f4..c506e0e215 100644 --- a/Lib/glob.py +++ b/Lib/glob.py @@ -4,11 +4,14 @@ import os import re import fnmatch +import functools import itertools +import operator import stat import sys -__all__ = ["glob", "iglob", "escape"] + +__all__ = ["glob", "iglob", "escape", "translate"] def glob(pathname, *, root_dir=None, dir_fd=None, recursive=False, include_hidden=False): @@ -104,8 +107,8 @@ def _iglob(pathname, root_dir, dir_fd, recursive, dironly, def _glob1(dirname, pattern, dir_fd, dironly, include_hidden=False): names = _listdir(dirname, dir_fd, dironly) - if include_hidden or not _ishidden(pattern): - names = (x for x in names if include_hidden or not _ishidden(x)) + if not (include_hidden or _ishidden(pattern)): + names = (x for x in names if not _ishidden(x)) return fnmatch.filter(names, pattern) def _glob0(dirname, basename, dir_fd, dironly, include_hidden=False): @@ -119,12 +122,19 @@ def _glob0(dirname, basename, dir_fd, dironly, include_hidden=False): return [basename] return [] -# Following functions are not public but can be used by third-party code. +_deprecated_function_message = ( + "{name} is deprecated and will be removed in Python {remove}. Use " + "glob.glob and pass a directory to its root_dir argument instead." +) def glob0(dirname, pattern): + import warnings + warnings._deprecated("glob.glob0", _deprecated_function_message, remove=(3, 15)) return _glob0(dirname, pattern, None, False) def glob1(dirname, pattern): + import warnings + warnings._deprecated("glob.glob1", _deprecated_function_message, remove=(3, 15)) return _glob1(dirname, pattern, None, False) # This helper function recursively yields relative pathnames inside a literal @@ -249,4 +259,287 @@ def escape(pathname): return drive + pathname +_special_parts = ('', '.', '..') _dir_open_flags = os.O_RDONLY | getattr(os, 'O_DIRECTORY', 0) +_no_recurse_symlinks = object() + + +def translate(pat, *, recursive=False, include_hidden=False, seps=None): + """Translate a pathname with shell wildcards to a regular expression. + + If `recursive` is true, the pattern segment '**' will match any number of + path segments. + + If `include_hidden` is true, wildcards can match path segments beginning + with a dot ('.'). + + If a sequence of separator characters is given to `seps`, they will be + used to split the pattern into segments and match path separators. If not + given, os.path.sep and os.path.altsep (where available) are used. + """ + if not seps: + if os.path.altsep: + seps = (os.path.sep, os.path.altsep) + else: + seps = os.path.sep + escaped_seps = ''.join(map(re.escape, seps)) + any_sep = f'[{escaped_seps}]' if len(seps) > 1 else escaped_seps + not_sep = f'[^{escaped_seps}]' + if include_hidden: + one_last_segment = f'{not_sep}+' + one_segment = f'{one_last_segment}{any_sep}' + any_segments = f'(?:.+{any_sep})?' + any_last_segments = '.*' + else: + one_last_segment = f'[^{escaped_seps}.]{not_sep}*' + one_segment = f'{one_last_segment}{any_sep}' + any_segments = f'(?:{one_segment})*' + any_last_segments = f'{any_segments}(?:{one_last_segment})?' + + results = [] + parts = re.split(any_sep, pat) + last_part_idx = len(parts) - 1 + for idx, part in enumerate(parts): + if part == '*': + results.append(one_segment if idx < last_part_idx else one_last_segment) + elif recursive and part == '**': + if idx < last_part_idx: + if parts[idx + 1] != '**': + results.append(any_segments) + else: + results.append(any_last_segments) + else: + if part: + if not include_hidden and part[0] in '*?': + results.append(r'(?!\.)') + results.extend(fnmatch._translate(part, f'{not_sep}*', not_sep)) + if idx < last_part_idx: + results.append(any_sep) + res = ''.join(results) + return fr'(?s:{res})\Z' + + +@functools.lru_cache(maxsize=512) +def _compile_pattern(pat, sep, case_sensitive, recursive=True): + """Compile given glob pattern to a re.Pattern object (observing case + sensitivity).""" + flags = re.NOFLAG if case_sensitive else re.IGNORECASE + regex = translate(pat, recursive=recursive, include_hidden=True, seps=sep) + return re.compile(regex, flags=flags).match + + +class _Globber: + """Class providing shell-style pattern matching and globbing. + """ + + def __init__(self, sep, case_sensitive, case_pedantic=False, recursive=False): + self.sep = sep + self.case_sensitive = case_sensitive + self.case_pedantic = case_pedantic + self.recursive = recursive + + # Low-level methods + + lstat = operator.methodcaller('lstat') + add_slash = operator.methodcaller('joinpath', '') + + @staticmethod + def scandir(path): + """Emulates os.scandir(), which returns an object that can be used as + a context manager. This method is called by walk() and glob(). + """ + return contextlib.nullcontext(path.iterdir()) + + @staticmethod + def concat_path(path, text): + """Appends text to the given path. + """ + return path.with_segments(path._raw_path + text) + + @staticmethod + def parse_entry(entry): + """Returns the path of an entry yielded from scandir(). + """ + return entry + + # High-level methods + + def compile(self, pat): + return _compile_pattern(pat, self.sep, self.case_sensitive, self.recursive) + + def selector(self, parts): + """Returns a function that selects from a given path, walking and + filtering according to the glob-style pattern parts in *parts*. + """ + if not parts: + return self.select_exists + part = parts.pop() + if self.recursive and part == '**': + selector = self.recursive_selector + elif part in _special_parts: + selector = self.special_selector + elif not self.case_pedantic and magic_check.search(part) is None: + selector = self.literal_selector + else: + selector = self.wildcard_selector + return selector(part, parts) + + def special_selector(self, part, parts): + """Returns a function that selects special children of the given path. + """ + select_next = self.selector(parts) + + def select_special(path, exists=False): + path = self.concat_path(self.add_slash(path), part) + return select_next(path, exists) + return select_special + + def literal_selector(self, part, parts): + """Returns a function that selects a literal descendant of a path. + """ + + # Optimization: consume and join any subsequent literal parts here, + # rather than leaving them for the next selector. This reduces the + # number of string concatenation operations and calls to add_slash(). + while parts and magic_check.search(parts[-1]) is None: + part += self.sep + parts.pop() + + select_next = self.selector(parts) + + def select_literal(path, exists=False): + path = self.concat_path(self.add_slash(path), part) + return select_next(path, exists=False) + return select_literal + + def wildcard_selector(self, part, parts): + """Returns a function that selects direct children of a given path, + filtering by pattern. + """ + + match = None if part == '*' else self.compile(part) + dir_only = bool(parts) + if dir_only: + select_next = self.selector(parts) + + def select_wildcard(path, exists=False): + try: + # We must close the scandir() object before proceeding to + # avoid exhausting file descriptors when globbing deep trees. + with self.scandir(path) as scandir_it: + entries = list(scandir_it) + except OSError: + pass + else: + for entry in entries: + if match is None or match(entry.name): + if dir_only: + try: + if not entry.is_dir(): + continue + except OSError: + continue + entry_path = self.parse_entry(entry) + if dir_only: + yield from select_next(entry_path, exists=True) + else: + yield entry_path + return select_wildcard + + def recursive_selector(self, part, parts): + """Returns a function that selects a given path and all its children, + recursively, filtering by pattern. + """ + # Optimization: consume following '**' parts, which have no effect. + while parts and parts[-1] == '**': + parts.pop() + + # Optimization: consume and join any following non-special parts here, + # rather than leaving them for the next selector. They're used to + # build a regular expression, which we use to filter the results of + # the recursive walk. As a result, non-special pattern segments + # following a '**' wildcard don't require additional filesystem access + # to expand. + follow_symlinks = self.recursive is not _no_recurse_symlinks + if follow_symlinks: + while parts and parts[-1] not in _special_parts: + part += self.sep + parts.pop() + + match = None if part == '**' else self.compile(part) + dir_only = bool(parts) + select_next = self.selector(parts) + + def select_recursive(path, exists=False): + path = self.add_slash(path) + match_pos = len(str(path)) + if match is None or match(str(path), match_pos): + yield from select_next(path, exists) + stack = [path] + while stack: + yield from select_recursive_step(stack, match_pos) + + def select_recursive_step(stack, match_pos): + path = stack.pop() + try: + # We must close the scandir() object before proceeding to + # avoid exhausting file descriptors when globbing deep trees. + with self.scandir(path) as scandir_it: + entries = list(scandir_it) + except OSError: + pass + else: + for entry in entries: + is_dir = False + try: + if entry.is_dir(follow_symlinks=follow_symlinks): + is_dir = True + except OSError: + pass + + if is_dir or not dir_only: + entry_path = self.parse_entry(entry) + if match is None or match(str(entry_path), match_pos): + if dir_only: + yield from select_next(entry_path, exists=True) + else: + # Optimization: directly yield the path if this is + # last pattern part. + yield entry_path + if is_dir: + stack.append(entry_path) + + return select_recursive + + def select_exists(self, path, exists=False): + """Yields the given path, if it exists. + """ + if exists: + # Optimization: this path is already known to exist, e.g. because + # it was returned from os.scandir(), so we skip calling lstat(). + yield path + else: + try: + self.lstat(path) + yield path + except OSError: + pass + + +class _StringGlobber(_Globber): + lstat = staticmethod(os.lstat) + scandir = staticmethod(os.scandir) + parse_entry = operator.attrgetter('path') + concat_path = operator.add + + if os.name == 'nt': + @staticmethod + def add_slash(pathname): + tail = os.path.splitroot(pathname)[2] + if not tail or tail[-1] in '\\/': + return pathname + return f'{pathname}\\' + else: + @staticmethod + def add_slash(pathname): + if not pathname or pathname[-1] == '/': + return pathname + return f'{pathname}/' diff --git a/Lib/html/__init__.py b/Lib/html/__init__.py index da0a0a3ce7..1543460ca3 100644 --- a/Lib/html/__init__.py +++ b/Lib/html/__init__.py @@ -25,7 +25,7 @@ def escape(s, quote=True): return s -# see http://www.w3.org/TR/html5/syntax.html#tokenizing-character-references +# see https://html.spec.whatwg.org/multipage/parsing.html#numeric-character-reference-end-state _invalid_charrefs = { 0x00: '\ufffd', # REPLACEMENT CHARACTER diff --git a/Lib/html/entities.py b/Lib/html/entities.py index dc508631ac..eb6dc12190 100644 --- a/Lib/html/entities.py +++ b/Lib/html/entities.py @@ -3,8 +3,7 @@ __all__ = ['html5', 'name2codepoint', 'codepoint2name', 'entitydefs'] -# maps the HTML entity name to the Unicode code point -# from https://html.spec.whatwg.org/multipage/named-characters.html +# maps HTML4 entity name to the Unicode code point name2codepoint = { 'AElig': 0x00c6, # latin capital letter AE = latin capital ligature AE, U+00C6 ISOlat1 'Aacute': 0x00c1, # latin capital letter A with acute, U+00C1 ISOlat1 @@ -261,7 +260,11 @@ } -# maps the HTML5 named character references to the equivalent Unicode character(s) +# HTML5 named character references +# Generated by Tools/build/parse_html5_entities.py +# from https://html.spec.whatwg.org/entities.json and +# https://html.spec.whatwg.org/multipage/named-characters.html. +# Map HTML5 named character references to the equivalent Unicode character(s). html5 = { 'Aacute': '\xc1', 'aacute': '\xe1', diff --git a/Lib/html/parser.py b/Lib/html/parser.py index bef0f4fe4b..5d7050dad2 100644 --- a/Lib/html/parser.py +++ b/Lib/html/parser.py @@ -12,6 +12,7 @@ import _markupbase from html import unescape +from html.entities import html5 as html5_entities __all__ = ['HTMLParser'] @@ -23,20 +24,51 @@ entityref = re.compile('&([a-zA-Z][-.a-zA-Z0-9]*)[^a-zA-Z0-9]') charref = re.compile('&#(?:[0-9]+|[xX][0-9a-fA-F]+)[^0-9a-fA-F]') +attr_charref = re.compile(r'&(#[0-9]+|#[xX][0-9a-fA-F]+|[a-zA-Z][a-zA-Z0-9]*)[;=]?') starttagopen = re.compile('<[a-zA-Z]') +endtagopen = re.compile('') -commentclose = re.compile(r'--\s*>') +commentclose = re.compile(r'--!?>') +commentabruptclose = re.compile(r'-?>') # Note: -# 1) if you change tagfind/attrfind remember to update locatestarttagend too; -# 2) if you change tagfind/attrfind and/or locatestarttagend the parser will +# 1) if you change tagfind/attrfind remember to update locatetagend too; +# 2) if you change tagfind/attrfind and/or locatetagend the parser will # explode, so don't do it. -# see http://www.w3.org/TR/html5/tokenization.html#tag-open-state -# and http://www.w3.org/TR/html5/tokenization.html#tag-name-state -tagfind_tolerant = re.compile(r'([a-zA-Z][^\t\n\r\f />\x00]*)(?:\s|/(?!>))*') -attrfind_tolerant = re.compile( - r'((?<=[\'"\s/])[^\s/>][^\s/=>]*)(\s*=+\s*' - r'(\'[^\']*\'|"[^"]*"|(?![\'"])[^>\s]*))?(?:\s|/(?!>))*') +# see the HTML5 specs section "13.2.5.6 Tag open state", +# "13.2.5.8 Tag name state" and "13.2.5.33 Attribute name state". +# https://html.spec.whatwg.org/multipage/parsing.html#tag-open-state +# https://html.spec.whatwg.org/multipage/parsing.html#tag-name-state +# https://html.spec.whatwg.org/multipage/parsing.html#attribute-name-state +tagfind_tolerant = re.compile(r'([a-zA-Z][^\t\n\r\f />]*)(?:[\t\n\r\f ]|/(?!>))*') +attrfind_tolerant = re.compile(r""" + ( + (?<=['"\t\n\r\f /])[^\t\n\r\f />][^\t\n\r\f /=>]* # attribute name + ) + ([\t\n\r\f ]*=[\t\n\r\f ]* # value indicator + ('[^']*' # LITA-enclosed value + |"[^"]*" # LIT-enclosed value + |(?!['"])[^>\t\n\r\f ]* # bare value + ) + )? + (?:[\t\n\r\f ]|/(?!>))* # possibly followed by a space +""", re.VERBOSE) +locatetagend = re.compile(r""" + [a-zA-Z][^\t\n\r\f />]* # tag name + [\t\n\r\f /]* # optional whitespace before attribute name + (?:(?<=['"\t\n\r\f /])[^\t\n\r\f />][^\t\n\r\f /=>]* # attribute name + (?:[\t\n\r\f ]*=[\t\n\r\f ]* # value indicator + (?:'[^']*' # LITA-enclosed value + |"[^"]*" # LIT-enclosed value + |(?!['"])[^>\t\n\r\f ]* # bare value + ) + )? + [\t\n\r\f /]* # possibly followed by a space + )* + >? +""", re.VERBOSE) +# The following variables are not used, but are temporarily left for +# backward compatibility. locatestarttagend_tolerant = re.compile(r""" <[a-zA-Z][^\t\n\r\f />\x00]* # tag name (?:[\s/]* # optional whitespace before attribute name @@ -53,10 +85,24 @@ \s* # trailing whitespace """, re.VERBOSE) endendtag = re.compile('>') -# the HTML 5 spec, section 8.1.2.2, doesn't allow spaces between -# ') +# Character reference processing logic specific to attribute values +# See: https://html.spec.whatwg.org/multipage/parsing.html#named-character-reference-state +def _replace_attr_charref(match): + ref = match.group(0) + # Numeric / hex char refs must always be unescaped + if ref.startswith('&#'): + return unescape(ref) + # Named character / entity references must only be unescaped + # if they are an exact match, and they are not followed by an equals sign + if not ref.endswith('=') and ref[1:] in html5_entities: + return unescape(ref) + # Otherwise do not unescape + return ref + +def _unescape_attrvalue(s): + return attr_charref.sub(_replace_attr_charref, s) class HTMLParser(_markupbase.ParserBase): @@ -82,6 +128,7 @@ class HTMLParser(_markupbase.ParserBase): """ CDATA_CONTENT_ELEMENTS = ("script", "style") + RCDATA_CONTENT_ELEMENTS = ("textarea", "title") def __init__(self, *, convert_charrefs=True): """Initialize and reset this instance. @@ -89,6 +136,7 @@ def __init__(self, *, convert_charrefs=True): If convert_charrefs is True (the default), all character references are automatically converted to the corresponding Unicode characters. """ + super().__init__() self.convert_charrefs = convert_charrefs self.reset() @@ -98,7 +146,9 @@ def reset(self): self.lasttag = '???' self.interesting = interesting_normal self.cdata_elem = None - _markupbase.ParserBase.reset(self) + self._support_cdata = True + self._escapable = True + super().reset() def feed(self, data): r"""Feed data to the parser. @@ -119,13 +169,33 @@ def get_starttag_text(self): """Return full source of start tag: '<...>'.""" return self.__starttag_text - def set_cdata_mode(self, elem): + def set_cdata_mode(self, elem, *, escapable=False): self.cdata_elem = elem.lower() - self.interesting = re.compile(r'' % self.cdata_elem, re.I) + self._escapable = escapable + if escapable and not self.convert_charrefs: + self.interesting = re.compile(r'&|])' % self.cdata_elem, + re.IGNORECASE|re.ASCII) + else: + self.interesting = re.compile(r'])' % self.cdata_elem, + re.IGNORECASE|re.ASCII) def clear_cdata_mode(self): self.interesting = interesting_normal self.cdata_elem = None + self._escapable = True + + def _set_support_cdata(self, flag=True): + """Enable or disable support of the CDATA sections. + If enabled, "<[CDATA[" starts a CDATA section which ends with "]]>". + If disabled, "<[CDATA[" starts a bogus comments which ends with ">". + + This method is not called by default. Its purpose is to be called + in custom handle_starttag() and handle_endtag() methods, with + value that depends on the adjusted current node. + See https://html.spec.whatwg.org/multipage/parsing.html#markup-declaration-open-state + for details. + """ + self._support_cdata = flag # Internal -- handle data as far as reasonable. May leave state # and data to be processed by a subsequent call. If 'end' is @@ -146,7 +216,7 @@ def goahead(self, end): # & near the end and see if it's followed by a space or ;. amppos = rawdata.rfind('&', max(i, n-34)) if (amppos >= 0 and - not re.compile(r'[\s;]').search(rawdata, amppos)): + not re.compile(r'[\t\n\r\f ;]').search(rawdata, amppos)): break # wait till we get all the text j = n else: @@ -158,7 +228,7 @@ def goahead(self, end): break j = n if i < j: - if self.convert_charrefs and not self.cdata_elem: + if self.convert_charrefs and self._escapable: self.handle_data(unescape(rawdata[i:j])) else: self.handle_data(rawdata[i:j]) @@ -176,7 +246,7 @@ def goahead(self, end): k = self.parse_pi(i) elif startswith("', i + 1) - if k < 0: - k = rawdata.find('<', i + 1) - if k < 0: - k = i + 1 - else: - k += 1 - if self.convert_charrefs and not self.cdata_elem: - self.handle_data(unescape(rawdata[i:k])) + if starttagopen.match(rawdata, i): # < + letter + pass + elif startswith("', i+9) + if j < 0: + return -1 + self.unknown_decl(rawdata[i+3: j]) + return j + 3 elif rawdata[i:i+9].lower() == ' gtpos = rawdata.find('>', i+9) @@ -271,12 +363,27 @@ def parse_html_declaration(self, i): else: return self.parse_bogus_comment(i) + # Internal -- parse comment, return length or -1 if not terminated + # see https://html.spec.whatwg.org/multipage/parsing.html#comment-start-state + def parse_comment(self, i, report=True): + rawdata = self.rawdata + assert rawdata.startswith(' \n # \" --> " # - i = 0 - n = len(str) - res = [] - while 0 <= i < n: - o_match = _OctalPatt.search(str, i) - q_match = _QuotePatt.search(str, i) - if not o_match and not q_match: # Neither matched - res.append(str[i:]) - break - # else: - j = k = -1 - if o_match: - j = o_match.start(0) - if q_match: - k = q_match.start(0) - if q_match and (not o_match or k < j): # QuotePatt matched - res.append(str[i:k]) - res.append(str[k+1]) - i = k + 2 - else: # OctalPatt matched - res.append(str[i:j]) - res.append(chr(int(str[j+1:j+4], 8))) - i = j + 4 - return _nulljoin(res) + return _unquote_sub(_unquote_replace, str) # The _getdate() routine is used to set the expiration time in the cookie's HTTP # header. By default, _getdate() returns the current time in the appropriate # "expires" format for a Set-Cookie header. The one optional argument is an # offset from now, in seconds. For example, an offset of -3600 means "one hour -# ago". The offset may be a floating point number. +# ago". The offset may be a floating-point number. # _weekdayname = ['Mon', 'Tue', 'Wed', 'Thu', 'Fri', 'Sat', 'Sun'] @@ -442,9 +424,11 @@ def OutputString(self, attrs=None): ( # Optional group: there may not be a value. \s*=\s* # Equal Sign (?P # Start of group 'val' - "(?:[^\\"]|\\.)*" # Any doublequoted string + "(?:[^\\"]|\\.)*" # Any double-quoted string | # or - \w{3},\s[\w\d\s-]{9,11}\s[\d:]{8}\sGMT # Special case for "expires" attr + # Special case for "expires" attr + (\w{3,6}day|\w{3}),\s # Day of the week or abbreviated day + [\w\d\s-]{9,11}\s[\d:]{8}\sGMT # Date and time in specific format | # or [""" + _LegalValueChars + r"""]* # Any word or empty string ) # End of group 'val' diff --git a/Lib/importlib/metadata/__init__.py b/Lib/importlib/metadata/__init__.py index 56ee403832..82e0ce1b28 100644 --- a/Lib/importlib/metadata/__init__.py +++ b/Lib/importlib/metadata/__init__.py @@ -56,12 +56,6 @@ def name(self): (name,) = self.args return name - # TODO: RUSTPYTHON; the entire setter is added to avoid errors - @name.setter - def name(self, value): - import sys - sys.stderr.write("set value to PackageNotFoundError ignored\n") - class Sectioned: """ diff --git a/Lib/inspect.py b/Lib/inspect.py index a84f3346b3..5a814f97b5 100644 --- a/Lib/inspect.py +++ b/Lib/inspect.py @@ -24,6 +24,8 @@ stack(), trace() - get info about frames on the stack or in a traceback signature() - get a Signature object for the callable + + get_annotations() - safely compute an object's annotations """ # This module is in the public domain. No warranties. @@ -31,7 +33,116 @@ __author__ = ('Ka-Ping Yee ', 'Yury Selivanov ') +__all__ = [ + "AGEN_CLOSED", + "AGEN_CREATED", + "AGEN_RUNNING", + "AGEN_SUSPENDED", + "ArgInfo", + "Arguments", + "Attribute", + "BlockFinder", + "BoundArguments", + "BufferFlags", + "CORO_CLOSED", + "CORO_CREATED", + "CORO_RUNNING", + "CORO_SUSPENDED", + "CO_ASYNC_GENERATOR", + "CO_COROUTINE", + "CO_GENERATOR", + "CO_ITERABLE_COROUTINE", + "CO_NESTED", + "CO_NEWLOCALS", + "CO_NOFREE", + "CO_OPTIMIZED", + "CO_VARARGS", + "CO_VARKEYWORDS", + "ClassFoundException", + "ClosureVars", + "EndOfBlock", + "FrameInfo", + "FullArgSpec", + "GEN_CLOSED", + "GEN_CREATED", + "GEN_RUNNING", + "GEN_SUSPENDED", + "Parameter", + "Signature", + "TPFLAGS_IS_ABSTRACT", + "Traceback", + "classify_class_attrs", + "cleandoc", + "currentframe", + "findsource", + "formatannotation", + "formatannotationrelativeto", + "formatargvalues", + "get_annotations", + "getabsfile", + "getargs", + "getargvalues", + "getasyncgenlocals", + "getasyncgenstate", + "getattr_static", + "getblock", + "getcallargs", + "getclasstree", + "getclosurevars", + "getcomments", + "getcoroutinelocals", + "getcoroutinestate", + "getdoc", + "getfile", + "getframeinfo", + "getfullargspec", + "getgeneratorlocals", + "getgeneratorstate", + "getinnerframes", + "getlineno", + "getmembers", + "getmembers_static", + "getmodule", + "getmodulename", + "getmro", + "getouterframes", + "getsource", + "getsourcefile", + "getsourcelines", + "indentsize", + "isabstract", + "isasyncgen", + "isasyncgenfunction", + "isawaitable", + "isbuiltin", + "isclass", + "iscode", + "iscoroutine", + "iscoroutinefunction", + "isdatadescriptor", + "isframe", + "isfunction", + "isgenerator", + "isgeneratorfunction", + "isgetsetdescriptor", + "ismemberdescriptor", + "ismethod", + "ismethoddescriptor", + "ismethodwrapper", + "ismodule", + "isroutine", + "istraceback", + "markcoroutinefunction", + "signature", + "stack", + "trace", + "unwrap", + "walktree", +] + + import abc +import ast import dis import collections.abc import enum @@ -44,47 +155,156 @@ import tokenize import token import types -import warnings import functools import builtins +from keyword import iskeyword from operator import attrgetter from collections import namedtuple, OrderedDict +from weakref import ref as make_weakref # Create constants for the compiler flags in Include/code.h # We try to get them from dis to avoid duplication mod_dict = globals() for k, v in dis.COMPILER_FLAG_NAMES.items(): mod_dict["CO_" + v] = k +del k, v, mod_dict # See Include/object.h TPFLAGS_IS_ABSTRACT = 1 << 20 + +def get_annotations(obj, *, globals=None, locals=None, eval_str=False): + """Compute the annotations dict for an object. + + obj may be a callable, class, or module. + Passing in an object of any other type raises TypeError. + + Returns a dict. get_annotations() returns a new dict every time + it's called; calling it twice on the same object will return two + different but equivalent dicts. + + This function handles several details for you: + + * If eval_str is true, values of type str will + be un-stringized using eval(). This is intended + for use with stringized annotations + ("from __future__ import annotations"). + * If obj doesn't have an annotations dict, returns an + empty dict. (Functions and methods always have an + annotations dict; classes, modules, and other types of + callables may not.) + * Ignores inherited annotations on classes. If a class + doesn't have its own annotations dict, returns an empty dict. + * All accesses to object members and dict values are done + using getattr() and dict.get() for safety. + * Always, always, always returns a freshly-created dict. + + eval_str controls whether or not values of type str are replaced + with the result of calling eval() on those values: + + * If eval_str is true, eval() is called on values of type str. + * If eval_str is false (the default), values of type str are unchanged. + + globals and locals are passed in to eval(); see the documentation + for eval() for more information. If either globals or locals is + None, this function may replace that value with a context-specific + default, contingent on type(obj): + + * If obj is a module, globals defaults to obj.__dict__. + * If obj is a class, globals defaults to + sys.modules[obj.__module__].__dict__ and locals + defaults to the obj class namespace. + * If obj is a callable, globals defaults to obj.__globals__, + although if obj is a wrapped function (using + functools.update_wrapper()) it is first unwrapped. + """ + if isinstance(obj, type): + # class + obj_dict = getattr(obj, '__dict__', None) + if obj_dict and hasattr(obj_dict, 'get'): + ann = obj_dict.get('__annotations__', None) + if isinstance(ann, types.GetSetDescriptorType): + ann = None + else: + ann = None + + obj_globals = None + module_name = getattr(obj, '__module__', None) + if module_name: + module = sys.modules.get(module_name, None) + if module: + obj_globals = getattr(module, '__dict__', None) + obj_locals = dict(vars(obj)) + unwrap = obj + elif isinstance(obj, types.ModuleType): + # module + ann = getattr(obj, '__annotations__', None) + obj_globals = getattr(obj, '__dict__') + obj_locals = None + unwrap = None + elif callable(obj): + # this includes types.Function, types.BuiltinFunctionType, + # types.BuiltinMethodType, functools.partial, functools.singledispatch, + # "class funclike" from Lib/test/test_inspect... on and on it goes. + ann = getattr(obj, '__annotations__', None) + obj_globals = getattr(obj, '__globals__', None) + obj_locals = None + unwrap = obj + else: + raise TypeError(f"{obj!r} is not a module, class, or callable.") + + if ann is None: + return {} + + if not isinstance(ann, dict): + raise ValueError(f"{obj!r}.__annotations__ is neither a dict nor None") + + if not ann: + return {} + + if not eval_str: + return dict(ann) + + if unwrap is not None: + while True: + if hasattr(unwrap, '__wrapped__'): + unwrap = unwrap.__wrapped__ + continue + if isinstance(unwrap, functools.partial): + unwrap = unwrap.func + continue + break + if hasattr(unwrap, "__globals__"): + obj_globals = unwrap.__globals__ + + if globals is None: + globals = obj_globals + if locals is None: + locals = obj_locals or {} + + # "Inject" type parameters into the local namespace + # (unless they are shadowed by assignments *in* the local namespace), + # as a way of emulating annotation scopes when calling `eval()` + if type_params := getattr(obj, "__type_params__", ()): + locals = {param.__name__: param for param in type_params} | locals + + return_value = {key: + value if not isinstance(value, str) else eval(value, globals, locals) + for key, value in ann.items() } + return return_value + + # ----------------------------------------------------------- type-checking def ismodule(object): - """Return true if the object is a module. - - Module objects provide these attributes: - __cached__ pathname to byte compiled file - __doc__ documentation string - __file__ filename (missing for built-in modules)""" + """Return true if the object is a module.""" return isinstance(object, types.ModuleType) def isclass(object): - """Return true if the object is a class. - - Class objects provide these attributes: - __doc__ documentation string - __module__ name of module in which this class was defined""" + """Return true if the object is a class.""" return isinstance(object, type) def ismethod(object): - """Return true if the object is an instance method. - - Instance method objects provide these attributes: - __doc__ documentation string - __name__ name with which this method was defined - __func__ function object containing implementation of method - __self__ instance to which this method is bound""" + """Return true if the object is an instance method.""" return isinstance(object, types.MethodType) def ismethoddescriptor(object): @@ -93,9 +313,10 @@ def ismethoddescriptor(object): But not if ismethod() or isclass() or isfunction() are true. This is new in Python 2.2, and, for example, is true of int.__add__. - An object passing this test has a __get__ attribute but not a __set__ - attribute, but beyond that the set of attributes varies. __name__ is - usually sensible, and __doc__ often is. + An object passing this test has a __get__ attribute, but not a + __set__ attribute or a __delete__ attribute. Beyond that, the set + of attributes varies; __name__ is usually sensible, and __doc__ + often is. Methods implemented via descriptors that also pass one of the other tests return false from the ismethoddescriptor() test, simply because @@ -104,8 +325,15 @@ def ismethoddescriptor(object): if isclass(object) or ismethod(object) or isfunction(object): # mutual exclusion return False + if isinstance(object, functools.partial): + # Lie for children. The addition of partial.__get__ + # doesn't currently change the partial objects behaviour, + # not counting a warning about future changes. + return False tp = type(object) - return hasattr(tp, "__get__") and not hasattr(tp, "__set__") + return (hasattr(tp, "__get__") + and not hasattr(tp, "__set__") + and not hasattr(tp, "__delete__")) def isdatadescriptor(object): """Return true if the object is a data descriptor. @@ -170,12 +398,14 @@ def isfunction(object): def _has_code_flag(f, flag): """Return true if ``f`` is a function (or a method or functools.partial - wrapper wrapping a function) whose code object has the given ``flag`` + wrapper wrapping a function or a functools.partialmethod wrapping a + function) whose code object has the given ``flag`` set in its flags.""" + f = functools._unwrap_partialmethod(f) while ismethod(f): f = f.__func__ f = functools._unwrap_partial(f) - if not isfunction(f): + if not (isfunction(f) or _signature_is_functionlike(f)): return False return bool(f.__code__.co_flags & flag) @@ -186,12 +416,31 @@ def isgeneratorfunction(obj): See help(isfunction) for a list of attributes.""" return _has_code_flag(obj, CO_GENERATOR) +# A marker for markcoroutinefunction and iscoroutinefunction. +_is_coroutine_mark = object() + +def _has_coroutine_mark(f): + while ismethod(f): + f = f.__func__ + f = functools._unwrap_partial(f) + return getattr(f, "_is_coroutine_marker", None) is _is_coroutine_mark + +def markcoroutinefunction(func): + """ + Decorator to ensure callable is recognised as a coroutine function. + """ + if hasattr(func, '__func__'): + func = func.__func__ + func._is_coroutine_marker = _is_coroutine_mark + return func + def iscoroutinefunction(obj): """Return true if the object is a coroutine function. - Coroutine functions are defined with "async def" syntax. + Coroutine functions are normally defined with "async def" syntax, but may + be marked via markcoroutinefunction. """ - return _has_code_flag(obj, CO_COROUTINE) + return _has_code_flag(obj, CO_COROUTINE) or _has_coroutine_mark(obj) def isasyncgenfunction(obj): """Return true if the object is an asynchronous generator function. @@ -276,7 +525,7 @@ def iscode(object): co_kwonlyargcount number of keyword only arguments (not including ** arg) co_lnotab encoded mapping of line numbers to bytecode indices co_name name with which this code object was defined - co_names tuple of names of local variables + co_names tuple of names other than arguments and function locals co_nlocals number of local variables co_stacksize virtual machine stack space required co_varnames tuple of names of arguments and local variables""" @@ -291,28 +540,30 @@ def isbuiltin(object): __self__ instance to which a method is bound, or None""" return isinstance(object, types.BuiltinFunctionType) +def ismethodwrapper(object): + """Return true if the object is a method wrapper.""" + return isinstance(object, types.MethodWrapperType) + def isroutine(object): """Return true if the object is any kind of function or method.""" return (isbuiltin(object) or isfunction(object) or ismethod(object) - or ismethoddescriptor(object)) + or ismethoddescriptor(object) + or ismethodwrapper(object)) def isabstract(object): """Return true if the object is an abstract base class (ABC).""" if not isinstance(object, type): return False - # TODO: RUSTPYTHON - # TPFLAGS_IS_ABSTRACT is not being set for abstract classes, so this implementation differs from CPython. - # if object.__flags__ & TPFLAGS_IS_ABSTRACT: - # return True + if object.__flags__ & TPFLAGS_IS_ABSTRACT: + return True if not issubclass(type(object), abc.ABCMeta): return False if hasattr(object, '__abstractmethods__'): # It looks like ABCMeta.__new__ has finished running; # TPFLAGS_IS_ABSTRACT should have been accurate. - # return False - return bool(getattr(object, '__abstractmethods__')) + return False # It looks like ABCMeta.__new__ has not finished running yet; we're # probably in __init_subclass__. We'll look for abstractmethods manually. for name, value in object.__dict__.items(): @@ -325,32 +576,30 @@ def isabstract(object): return True return False -def getmembers(object, predicate=None): - """Return all members of an object as (name, value) pairs sorted by name. - Optionally, only return members that satisfy a given predicate.""" - if isclass(object): - mro = (object,) + getmro(object) - else: - mro = () +def _getmembers(object, predicate, getter): results = [] processed = set() names = dir(object) - # :dd any DynamicClassAttributes to the list of names if object is a class; - # this may result in duplicate entries if, for example, a virtual - # attribute with the same name as a DynamicClassAttribute exists - try: - for base in object.__bases__: - for k, v in base.__dict__.items(): - if isinstance(v, types.DynamicClassAttribute): - names.append(k) - except AttributeError: - pass + if isclass(object): + mro = getmro(object) + # add any DynamicClassAttributes to the list of names if object is a class; + # this may result in duplicate entries if, for example, a virtual + # attribute with the same name as a DynamicClassAttribute exists + try: + for base in object.__bases__: + for k, v in base.__dict__.items(): + if isinstance(v, types.DynamicClassAttribute): + names.append(k) + except AttributeError: + pass + else: + mro = () for key in names: # First try to get the value via getattr. Some descriptors don't # like calling their __get__ (see bug #1785), so fall back to # looking in the __dict__. try: - value = getattr(object, key) + value = getter(object, key) # handle the duplicate key if key in processed: raise AttributeError @@ -369,6 +618,25 @@ def getmembers(object, predicate=None): results.sort(key=lambda pair: pair[0]) return results +def getmembers(object, predicate=None): + """Return all members of an object as (name, value) pairs sorted by name. + Optionally, only return members that satisfy a given predicate.""" + return _getmembers(object, predicate, getattr) + +def getmembers_static(object, predicate=None): + """Return all members of an object as (name, value) pairs sorted by name + without triggering dynamic lookup via the descriptor protocol, + __getattr__ or __getattribute__. Optionally, only return members that + satisfy a given predicate. + + Note: this function may not be able to retrieve all members + that getmembers can fetch (like dynamically created attributes) + and may find members that getmembers can't (like descriptors + that raise AttributeError). It can also return descriptor objects + instead of instance members in some cases. + """ + return _getmembers(object, predicate, getattr_static) + Attribute = namedtuple('Attribute', 'name kind defining_class object') def classify_class_attrs(cls): @@ -409,7 +677,7 @@ def classify_class_attrs(cls): # attribute with the same name as a DynamicClassAttribute exists. for base in mro: for k, v in base.__dict__.items(): - if isinstance(v, types.DynamicClassAttribute): + if isinstance(v, types.DynamicClassAttribute) and v.fget is not None: names.append(k) result = [] processed = set() @@ -432,7 +700,7 @@ def classify_class_attrs(cls): if name == '__dict__': raise Exception("__dict__ is special, don't want the proxy") get_obj = getattr(cls, name) - except Exception as exc: + except Exception: pass else: homecls = getattr(get_obj, "__objclass__", homecls) @@ -509,18 +777,14 @@ def unwrap(func, *, stop=None): :exc:`ValueError` is raised if a cycle is encountered. """ - if stop is None: - def _is_wrapper(f): - return hasattr(f, '__wrapped__') - else: - def _is_wrapper(f): - return hasattr(f, '__wrapped__') and not stop(f) f = func # remember the original func for error reporting # Memoise by id to tolerate non-hashable objects, but store objects to # ensure they aren't destroyed, which would allow their IDs to be reused. memo = {id(f): f} recursion_limit = sys.getrecursionlimit() - while _is_wrapper(func): + while not isinstance(func, type) and hasattr(func, '__wrapped__'): + if stop is not None and stop(func): + break func = func.__wrapped__ id_func = id(func) if (id_func in memo) or (len(memo) >= recursion_limit): @@ -581,9 +845,8 @@ def _finddoc(obj): cls = self.__class__ # Should be tested before isdatadescriptor(). elif isinstance(obj, property): - func = obj.fget - name = func.__name__ - cls = _findclass(func) + name = obj.__name__ + cls = _findclass(obj.fget) if cls is None or getattr(cls, name) is not obj: return None elif ismethoddescriptor(obj) or isdatadescriptor(obj): @@ -630,29 +893,28 @@ def cleandoc(doc): Any whitespace that can be uniformly removed from the second line onwards is removed.""" - try: - lines = doc.expandtabs().split('\n') - except UnicodeError: - return None - else: - # Find minimum indentation of any non-blank lines after first line. - margin = sys.maxsize - for line in lines[1:]: - content = len(line.lstrip()) - if content: - indent = len(line) - content - margin = min(margin, indent) - # Remove indentation. - if lines: - lines[0] = lines[0].lstrip() - if margin < sys.maxsize: - for i in range(1, len(lines)): lines[i] = lines[i][margin:] - # Remove any trailing or leading blank lines. - while lines and not lines[-1]: - lines.pop() - while lines and not lines[0]: - lines.pop(0) - return '\n'.join(lines) + lines = doc.expandtabs().split('\n') + + # Find minimum indentation of any non-blank lines after first line. + margin = sys.maxsize + for line in lines[1:]: + content = len(line.lstrip(' ')) + if content: + indent = len(line) - content + margin = min(margin, indent) + # Remove indentation. + if lines: + lines[0] = lines[0].lstrip(' ') + if margin < sys.maxsize: + for i in range(1, len(lines)): + lines[i] = lines[i][margin:] + # Remove any trailing or leading blank lines. + while lines and not lines[-1]: + lines.pop() + while lines and not lines[0]: + lines.pop(0) + return '\n'.join(lines) + def getfile(object): """Work out which source or compiled file an object was defined in.""" @@ -665,6 +927,8 @@ def getfile(object): module = sys.modules.get(object.__module__) if getattr(module, '__file__', None): return module.__file__ + if object.__module__ == '__main__': + raise OSError('source code not available') raise TypeError('{!r} is a built-in class'.format(object)) if ismethod(object): object = object.__func__ @@ -705,13 +969,20 @@ def getsourcefile(object): elif any(filename.endswith(s) for s in importlib.machinery.EXTENSION_SUFFIXES): return None + elif filename.endswith(".fwork"): + # Apple mobile framework markers are another type of non-source file + return None + + # return a filename found in the linecache even if it doesn't exist on disk + if filename in linecache.cache: + return filename if os.path.exists(filename): return filename # only return a non-existent filename if the module has a PEP 302 loader - if getattr(getmodule(object, filename), '__loader__', None) is not None: + module = getmodule(object, filename) + if getattr(module, '__loader__', None) is not None: return filename - # or it is in the linecache - if filename in linecache.cache: + elif getattr(getattr(module, "__spec__", None), "loader", None) is not None: return filename def getabsfile(object, _filename=None): @@ -732,19 +1003,20 @@ def getmodule(object, _filename=None): return object if hasattr(object, '__module__'): return sys.modules.get(object.__module__) + # Try the filename to modulename cache if _filename is not None and _filename in modulesbyfile: return sys.modules.get(modulesbyfile[_filename]) # Try the cache again with the absolute file name try: file = getabsfile(object, _filename) - except TypeError: + except (TypeError, FileNotFoundError): return None if file in modulesbyfile: return sys.modules.get(modulesbyfile[file]) # Update the filename to module name cache and check yet again # Copy sys.modules in order to cope with changes while iterating - for modname, module in list(sys.modules.items()): + for modname, module in sys.modules.copy().items(): if ismodule(module) and hasattr(module, '__file__'): f = module.__file__ if f == _filesbymodname.get(modname, None): @@ -772,6 +1044,11 @@ def getmodule(object, _filename=None): if builtinobject is object: return builtin + +class ClassFoundException(Exception): + pass + + def findsource(object): """Return the entire source file and starting line number for an object. @@ -789,12 +1066,14 @@ def findsource(object): # Allow filenames in form of "" to pass through. # `doctest` monkeypatches `linecache` module to enable # inspection, so let `linecache.getlines` to be called. - if not (file.startswith('<') and file.endswith('>')): + if (not (file.startswith('<') and file.endswith('>'))) or file.endswith('.fwork'): raise OSError('source code not available') module = getmodule(object, file) if module: lines = linecache.getlines(file, module.__dict__) + if not lines and file.startswith('<') and hasattr(object, "__code__"): + lines = linecache._getlines_from_code(object.__code__) else: lines = linecache.getlines(file) if not lines: @@ -804,27 +1083,13 @@ def findsource(object): return lines, 0 if isclass(object): - name = object.__name__ - pat = re.compile(r'^(\s*)class\s*' + name + r'\b') - # make some effort to find the best matching class definition: - # use the one with the least indentation, which is the one - # that's most probably not inside a function definition. - candidates = [] - for i in range(len(lines)): - match = pat.match(lines[i]) - if match: - # if it's at toplevel, it's already the best one - if lines[i][0] == 'c': - return lines, i - # else add whitespace to candidate list - candidates.append((match.group(1), i)) - if candidates: - # this will sort by whitespace, and by line number, - # less whitespace first - candidates.sort() - return lines, candidates[0][1] - else: - raise OSError('could not find class definition') + try: + lnum = vars(object)['__firstlineno__'] - 1 + except (TypeError, KeyError): + raise OSError('source code not available') + if lnum >= len(lines): + raise OSError('lineno is out of bounds') + return lines, lnum if ismethod(object): object = object.__func__ @@ -838,10 +1103,8 @@ def findsource(object): if not hasattr(object, 'co_firstlineno'): raise OSError('could not find function definition') lnum = object.co_firstlineno - 1 - pat = re.compile(r'^(\s*def\s)|(\s*async\s+def\s)|(.*(? 0: - if pat.match(lines[lnum]): break - lnum = lnum - 1 + if lnum >= len(lines): + raise OSError('lineno is out of bounds') return lines, lnum raise OSError('could not find code object') @@ -896,43 +1159,43 @@ class BlockFinder: """Provide a tokeneater() method to detect the end of a code block.""" def __init__(self): self.indent = 0 - self.islambda = False + self.singleline = False self.started = False self.passline = False self.indecorator = False - self.decoratorhasargs = False self.last = 1 + self.body_col0 = None def tokeneater(self, type, token, srowcol, erowcol, line): if not self.started and not self.indecorator: + if type in (tokenize.INDENT, tokenize.COMMENT, tokenize.NL): + pass + elif token == "async": + pass # skip any decorators - if token == "@": + elif token == "@": self.indecorator = True - # look for the first "def", "class" or "lambda" - elif token in ("def", "class", "lambda"): - if token == "lambda": - self.islambda = True + else: + # For "def" and "class" scan to the end of the block. + # For "lambda" and generator expression scan to + # the end of the logical line. + self.singleline = token not in ("def", "class") self.started = True self.passline = True # skip to the end of the line - elif token == "(": - if self.indecorator: - self.decoratorhasargs = True - elif token == ")": - if self.indecorator: - self.indecorator = False - self.decoratorhasargs = False elif type == tokenize.NEWLINE: self.passline = False # stop skipping when a NEWLINE is seen self.last = srowcol[0] - if self.islambda: # lambdas always end at the first NEWLINE + if self.singleline: raise EndOfBlock # hitting a NEWLINE when in a decorator without args # ends the decorator - if self.indecorator and not self.decoratorhasargs: + if self.indecorator: self.indecorator = False elif self.passline: pass elif type == tokenize.INDENT: + if self.body_col0 is None and self.started: + self.body_col0 = erowcol[1] self.indent = self.indent + 1 self.passline = True elif type == tokenize.DEDENT: @@ -942,6 +1205,10 @@ def tokeneater(self, type, token, srowcol, erowcol, line): # not e.g. for "if: else:" or "try: finally:" blocks) if self.indent <= 0: raise EndOfBlock + elif type == tokenize.COMMENT: + if self.body_col0 is not None and srowcol[1] >= self.body_col0: + # Include comments if indented at least as much as the block + self.last = srowcol[0] elif self.indent == 0 and type not in (tokenize.COMMENT, tokenize.NL): # any other token on the same indentation level end the previous # block as well, except the pseudo-tokens COMMENT and NL. @@ -956,6 +1223,14 @@ def getblock(lines): blockfinder.tokeneater(*_token) except (EndOfBlock, IndentationError): pass + except SyntaxError as e: + if "unmatched" not in e.msg: + raise e from None + _, *_token_info = _token + try: + blockfinder.tokeneater(tokenize.NEWLINE, *_token_info) + except (EndOfBlock, IndentationError): + pass return lines[:blockfinder.last] def getsourcelines(object): @@ -1043,7 +1318,6 @@ def getargs(co): nkwargs = co.co_kwonlyargcount args = list(names[:nargs]) kwonlyargs = list(names[nargs:nargs+nkwargs]) - step = 0 nargs += nkwargs varargs = None @@ -1055,37 +1329,6 @@ def getargs(co): varkw = co.co_varnames[nargs] return Arguments(args + kwonlyargs, varargs, varkw) -ArgSpec = namedtuple('ArgSpec', 'args varargs keywords defaults') - -def getargspec(func): - """Get the names and default values of a function's parameters. - - A tuple of four things is returned: (args, varargs, keywords, defaults). - 'args' is a list of the argument names, including keyword-only argument names. - 'varargs' and 'keywords' are the names of the * and ** parameters or None. - 'defaults' is an n-tuple of the default values of the last n parameters. - - This function is deprecated, as it does not support annotations or - keyword-only parameters and will raise ValueError if either is present - on the supplied callable. - - For a more structured introspection API, use inspect.signature() instead. - - Alternatively, use getfullargspec() for an API with a similar namedtuple - based interface, but full support for annotations and keyword-only - parameters. - - Deprecated since Python 3.5, use `inspect.getfullargspec()`. - """ - warnings.warn("inspect.getargspec() is deprecated since Python 3.0, " - "use inspect.signature() or inspect.getfullargspec()", - DeprecationWarning, stacklevel=2) - args, varargs, varkw, defaults, kwonlyargs, kwonlydefaults, ann = \ - getfullargspec(func) - if kwonlyargs or ann: - raise ValueError("Function has keyword-only parameters or annotations" - ", use inspect.signature() API which can support them") - return ArgSpec(args, varargs, varkw, defaults) FullArgSpec = namedtuple('FullArgSpec', 'args, varargs, varkw, defaults, kwonlyargs, kwonlydefaults, annotations') @@ -1126,7 +1369,8 @@ def getfullargspec(func): sig = _signature_from_callable(func, follow_wrapper_chains=False, skip_bound_arg=False, - sigcls=Signature) + sigcls=Signature, + eval_str=False) except Exception as ex: # Most of the times 'signature' will raise ValueError. # But, it can also raise AttributeError, and, maybe something @@ -1139,7 +1383,6 @@ def getfullargspec(func): varkw = None posonlyargs = [] kwonlyargs = [] - defaults = () annotations = {} defaults = () kwdefaults = {} @@ -1197,7 +1440,12 @@ def getargvalues(frame): def formatannotation(annotation, base_module=None): if getattr(annotation, '__module__', None) == 'typing': - return repr(annotation).replace('typing.', '') + def repl(match): + text = match.group() + return text.removeprefix('typing.') + return re.sub(r'[\w\.]+', repl, repr(annotation)) + if isinstance(annotation, types.GenericAlias): + return str(annotation) if isinstance(annotation, type): if annotation.__module__ in ('builtins', base_module): return annotation.__qualname__ @@ -1210,63 +1458,6 @@ def _formatannotation(annotation): return formatannotation(annotation, module) return _formatannotation -def formatargspec(args, varargs=None, varkw=None, defaults=None, - kwonlyargs=(), kwonlydefaults={}, annotations={}, - formatarg=str, - formatvarargs=lambda name: '*' + name, - formatvarkw=lambda name: '**' + name, - formatvalue=lambda value: '=' + repr(value), - formatreturns=lambda text: ' -> ' + text, - formatannotation=formatannotation): - """Format an argument spec from the values returned by getfullargspec. - - The first seven arguments are (args, varargs, varkw, defaults, - kwonlyargs, kwonlydefaults, annotations). The other five arguments - are the corresponding optional formatting functions that are called to - turn names and values into strings. The last argument is an optional - function to format the sequence of arguments. - - Deprecated since Python 3.5: use the `signature` function and `Signature` - objects. - """ - - from warnings import warn - - warn("`formatargspec` is deprecated since Python 3.5. Use `signature` and " - "the `Signature` object directly", - DeprecationWarning, - stacklevel=2) - - def formatargandannotation(arg): - result = formatarg(arg) - if arg in annotations: - result += ': ' + formatannotation(annotations[arg]) - return result - specs = [] - if defaults: - firstdefault = len(args) - len(defaults) - for i, arg in enumerate(args): - spec = formatargandannotation(arg) - if defaults and i >= firstdefault: - spec = spec + formatvalue(defaults[i - firstdefault]) - specs.append(spec) - if varargs is not None: - specs.append(formatvarargs(formatargandannotation(varargs))) - else: - if kwonlyargs: - specs.append('*') - if kwonlyargs: - for kwonlyarg in kwonlyargs: - spec = formatargandannotation(kwonlyarg) - if kwonlydefaults and kwonlyarg in kwonlydefaults: - spec += formatvalue(kwonlydefaults[kwonlyarg]) - specs.append(spec) - if varkw is not None: - specs.append(formatvarkw(formatargandannotation(varkw))) - result = '(' + ', '.join(specs) + ')' - if 'return' in annotations: - result += formatreturns(formatannotation(annotations['return'])) - return result def formatargvalues(args, varargs, varkw, locals, formatarg=str, @@ -1425,11 +1616,15 @@ def getclosurevars(func): global_vars = {} builtin_vars = {} unbound_names = set() - for name in code.co_names: - if name in ("None", "True", "False"): - # Because these used to be builtins instead of keywords, they - # may still show up as name references. We ignore them. - continue + global_names = set() + for instruction in dis.get_instructions(code): + opname = instruction.opname + name = instruction.argval + if opname == "LOAD_ATTR": + unbound_names.add(name) + elif opname == "LOAD_GLOBAL": + global_names.add(name) + for name in global_names: try: global_vars[name] = global_ns[name] except KeyError: @@ -1443,7 +1638,30 @@ def getclosurevars(func): # -------------------------------------------------- stack frame extraction -Traceback = namedtuple('Traceback', 'filename lineno function code_context index') +_Traceback = namedtuple('_Traceback', 'filename lineno function code_context index') + +class Traceback(_Traceback): + def __new__(cls, filename, lineno, function, code_context, index, *, positions=None): + instance = super().__new__(cls, filename, lineno, function, code_context, index) + instance.positions = positions + return instance + + def __repr__(self): + return ('Traceback(filename={!r}, lineno={!r}, function={!r}, ' + 'code_context={!r}, index={!r}, positions={!r})'.format( + self.filename, self.lineno, self.function, self.code_context, + self.index, self.positions)) + +def _get_code_position_from_tb(tb): + code, instruction_index = tb.tb_frame.f_code, tb.tb_lasti + return _get_code_position(code, instruction_index) + +def _get_code_position(code, instruction_index): + if instruction_index < 0: + return (None, None, None, None) + positions_gen = code.co_positions() + # The nth entry in code.co_positions() corresponds to instruction (2*n)th since Python 3.10+ + return next(itertools.islice(positions_gen, instruction_index // 2, None)) def getframeinfo(frame, context=1): """Get information about a frame or traceback object. @@ -1454,10 +1672,20 @@ def getframeinfo(frame, context=1): The optional second argument specifies the number of lines of context to return, which are centered around the current line.""" if istraceback(frame): + positions = _get_code_position_from_tb(frame) lineno = frame.tb_lineno frame = frame.tb_frame else: lineno = frame.f_lineno + positions = _get_code_position(frame.f_code, frame.f_lasti) + + if positions[0] is None: + frame, *positions = (frame, lineno, *positions[1:]) + else: + frame, *positions = (frame, *positions) + + lineno = positions[0] + if not isframe(frame): raise TypeError('{!r} is not a frame or traceback object'.format(frame)) @@ -1475,14 +1703,26 @@ def getframeinfo(frame, context=1): else: lines = index = None - return Traceback(filename, lineno, frame.f_code.co_name, lines, index) + return Traceback(filename, lineno, frame.f_code.co_name, lines, + index, positions=dis.Positions(*positions)) def getlineno(frame): """Get the line number from a frame object, allowing for optimization.""" # FrameType.f_lineno is now a descriptor that grovels co_lnotab return frame.f_lineno -FrameInfo = namedtuple('FrameInfo', ('frame',) + Traceback._fields) +_FrameInfo = namedtuple('_FrameInfo', ('frame',) + Traceback._fields) +class FrameInfo(_FrameInfo): + def __new__(cls, frame, filename, lineno, function, code_context, index, *, positions=None): + instance = super().__new__(cls, frame, filename, lineno, function, code_context, index) + instance.positions = positions + return instance + + def __repr__(self): + return ('FrameInfo(frame={!r}, filename={!r}, lineno={!r}, function={!r}, ' + 'code_context={!r}, index={!r}, positions={!r})'.format( + self.frame, self.filename, self.lineno, self.function, + self.code_context, self.index, self.positions)) def getouterframes(frame, context=1): """Get a list of records for a frame and all higher (calling) frames. @@ -1491,8 +1731,9 @@ def getouterframes(frame, context=1): name, a list of lines of context, and index within the context.""" framelist = [] while frame: - frameinfo = (frame,) + getframeinfo(frame, context) - framelist.append(FrameInfo(*frameinfo)) + traceback_info = getframeinfo(frame, context) + frameinfo = (frame,) + traceback_info + framelist.append(FrameInfo(*frameinfo, positions=traceback_info.positions)) frame = frame.f_back return framelist @@ -1503,8 +1744,9 @@ def getinnerframes(tb, context=1): name, a list of lines of context, and index within the context.""" framelist = [] while tb: - frameinfo = (tb.tb_frame,) + getframeinfo(tb, context) - framelist.append(FrameInfo(*frameinfo)) + traceback_info = getframeinfo(tb, context) + frameinfo = (tb.tb_frame,) + traceback_info + framelist.append(FrameInfo(*frameinfo, positions=traceback_info.positions)) tb = tb.tb_next return framelist @@ -1518,15 +1760,17 @@ def stack(context=1): def trace(context=1): """Return a list of records for the stack below the current exception.""" - return getinnerframes(sys.exc_info()[2], context) + exc = sys.exception() + tb = None if exc is None else exc.__traceback__ + return getinnerframes(tb, context) # ------------------------------------------------ static version of getattr _sentinel = object() +_static_getmro = type.__dict__['__mro__'].__get__ +_get_dunder_dict_of_class = type.__dict__["__dict__"].__get__ -def _static_getmro(klass): - return type.__dict__['__mro__'].__get__(klass) def _check_instance(obj, attr): instance_dict = {} @@ -1539,34 +1783,43 @@ def _check_instance(obj, attr): def _check_class(klass, attr): for entry in _static_getmro(klass): - if _shadowed_dict(type(entry)) is _sentinel: - try: - return entry.__dict__[attr] - except KeyError: - pass + if _shadowed_dict(type(entry)) is _sentinel and attr in entry.__dict__: + return entry.__dict__[attr] return _sentinel -def _is_type(obj): - try: - _static_getmro(obj) - except TypeError: - return False - return True -def _shadowed_dict(klass): - dict_attr = type.__dict__["__dict__"] - for entry in _static_getmro(klass): - try: - class_dict = dict_attr.__get__(entry)["__dict__"] - except KeyError: - pass - else: +@functools.lru_cache() +def _shadowed_dict_from_weakref_mro_tuple(*weakref_mro): + for weakref_entry in weakref_mro: + # Normally we'd have to check whether the result of weakref_entry() + # is None here, in case the object the weakref is pointing to has died. + # In this specific case, however, we know that the only caller of this + # function is `_shadowed_dict()`, and that therefore this weakref is + # guaranteed to point to an object that is still alive. + entry = weakref_entry() + dunder_dict = _get_dunder_dict_of_class(entry) + if '__dict__' in dunder_dict: + class_dict = dunder_dict['__dict__'] if not (type(class_dict) is types.GetSetDescriptorType and class_dict.__name__ == "__dict__" and class_dict.__objclass__ is entry): return class_dict return _sentinel + +def _shadowed_dict(klass): + # gh-118013: the inner function here is decorated with lru_cache for + # performance reasons, *but* make sure not to pass strong references + # to the items in the mro. Doing so can lead to unexpected memory + # consumption in cases where classes are dynamically created and + # destroyed, and the dynamically created classes happen to be the only + # objects that hold strong references to other objects that take up a + # significant amount of memory. + return _shadowed_dict_from_weakref_mro_tuple( + *[make_weakref(entry) for entry in _static_getmro(klass)] + ) + + def getattr_static(obj, attr, default=_sentinel): """Retrieve attributes without triggering dynamic lookup via the descriptor protocol, __getattr__ or __getattribute__. @@ -1579,8 +1832,10 @@ def getattr_static(obj, attr, default=_sentinel): documentation for details. """ instance_result = _sentinel - if not _is_type(obj): - klass = type(obj) + + objtype = type(obj) + if type not in _static_getmro(objtype): + klass = objtype dict_attr = _shadowed_dict(klass) if (dict_attr is _sentinel or type(dict_attr) is types.MemberDescriptorType): @@ -1591,8 +1846,10 @@ def getattr_static(obj, attr, default=_sentinel): klass_result = _check_class(klass, attr) if instance_result is not _sentinel and klass_result is not _sentinel: - if (_check_class(type(klass_result), '__get__') is not _sentinel and - _check_class(type(klass_result), '__set__') is not _sentinel): + if _check_class(type(klass_result), "__get__") is not _sentinel and ( + _check_class(type(klass_result), "__set__") is not _sentinel + or _check_class(type(klass_result), "__delete__") is not _sentinel + ): return klass_result if instance_result is not _sentinel: @@ -1603,11 +1860,11 @@ def getattr_static(obj, attr, default=_sentinel): if obj is klass: # for types we check the metaclass too for entry in _static_getmro(type(klass)): - if _shadowed_dict(type(entry)) is _sentinel: - try: - return entry.__dict__[attr] - except KeyError: - pass + if ( + _shadowed_dict(type(entry)) is _sentinel + and attr in entry.__dict__ + ): + return entry.__dict__[attr] if default is not _sentinel: return default raise AttributeError(attr) @@ -1631,11 +1888,11 @@ def getgeneratorstate(generator): """ if generator.gi_running: return GEN_RUNNING + if generator.gi_suspended: + return GEN_SUSPENDED if generator.gi_frame is None: return GEN_CLOSED - if generator.gi_frame.f_lasti == -1: - return GEN_CREATED - return GEN_SUSPENDED + return GEN_CREATED def getgeneratorlocals(generator): @@ -1673,11 +1930,11 @@ def getcoroutinestate(coroutine): """ if coroutine.cr_running: return CORO_RUNNING + if coroutine.cr_suspended: + return CORO_SUSPENDED if coroutine.cr_frame is None: return CORO_CLOSED - if coroutine.cr_frame.f_lasti == -1: - return CORO_CREATED - return CORO_SUSPENDED + return CORO_CREATED def getcoroutinelocals(coroutine): @@ -1693,35 +1950,89 @@ def getcoroutinelocals(coroutine): return {} +# ----------------------------------- asynchronous generator introspection + +AGEN_CREATED = 'AGEN_CREATED' +AGEN_RUNNING = 'AGEN_RUNNING' +AGEN_SUSPENDED = 'AGEN_SUSPENDED' +AGEN_CLOSED = 'AGEN_CLOSED' + + +def getasyncgenstate(agen): + """Get current state of an asynchronous generator object. + + Possible states are: + AGEN_CREATED: Waiting to start execution. + AGEN_RUNNING: Currently being executed by the interpreter. + AGEN_SUSPENDED: Currently suspended at a yield expression. + AGEN_CLOSED: Execution has completed. + """ + if agen.ag_running: + return AGEN_RUNNING + if agen.ag_suspended: + return AGEN_SUSPENDED + if agen.ag_frame is None: + return AGEN_CLOSED + return AGEN_CREATED + + +def getasyncgenlocals(agen): + """ + Get the mapping of asynchronous generator local variables to their current + values. + + A dict is returned, with the keys the local variable names and values the + bound values.""" + + if not isasyncgen(agen): + raise TypeError(f"{agen!r} is not a Python async generator") + + frame = getattr(agen, "ag_frame", None) + if frame is not None: + return agen.ag_frame.f_locals + else: + return {} + + ############################################################################### ### Function Signature Object (PEP 362) ############################################################################### -_WrapperDescriptor = type(type.__call__) -_MethodWrapper = type(all.__call__) -_ClassMethodWrapper = type(int.__dict__['from_bytes']) - -_NonUserDefinedCallables = (_WrapperDescriptor, - _MethodWrapper, - _ClassMethodWrapper, +_NonUserDefinedCallables = (types.WrapperDescriptorType, + types.MethodWrapperType, + types.ClassMethodDescriptorType, types.BuiltinFunctionType) -def _signature_get_user_defined_method(cls, method_name): +def _signature_get_user_defined_method(cls, method_name, *, follow_wrapper_chains=True): """Private helper. Checks if ``cls`` has an attribute named ``method_name`` and returns it only if it is a pure python function. """ - try: - meth = getattr(cls, method_name) - except AttributeError: - return + if method_name == '__new__': + meth = getattr(cls, method_name, None) else: - if not isinstance(meth, _NonUserDefinedCallables): - # Once '__signature__' will be added to 'C'-level - # callables, this check won't be necessary - return meth + meth = getattr_static(cls, method_name, None) + if meth is None: + return None + + # NOTE: The meth may wraps a non-user-defined callable. + # In this case, we treat the meth as non-user-defined callable too. + # (e.g. cls.__new__ generated by @warnings.deprecated) + unwrapped_meth = None + if follow_wrapper_chains: + unwrapped_meth = unwrap(meth, stop=(lambda m: hasattr(m, "__signature__") + or _signature_is_builtin(m))) + + if (isinstance(meth, _NonUserDefinedCallables) + or isinstance(unwrapped_meth, _NonUserDefinedCallables)): + # Once '__signature__' will be added to 'C'-level + # callables, this check won't be necessary + return None + if method_name != '__new__': + meth = _descriptor_get(meth, cls) + return meth def _signature_get_partial(wrapped_sig, partial, extra_args=()): @@ -1834,8 +2145,10 @@ def _signature_is_builtin(obj): ismethoddescriptor(obj) or isinstance(obj, _NonUserDefinedCallables) or # Can't test 'isinstance(type)' here, as it would - # also be True for regular python classes - obj in (type, object)) + # also be True for regular python classes. + # Can't use the `in` operator here, as it would + # invoke the custom __eq__ method. + obj is type or obj is object) def _signature_is_functionlike(obj): @@ -1860,30 +2173,7 @@ def _signature_is_functionlike(obj): isinstance(name, str) and (defaults is None or isinstance(defaults, tuple)) and (kwdefaults is None or isinstance(kwdefaults, dict)) and - isinstance(annotations, dict)) - - -def _signature_get_bound_param(spec): - """ Private helper to get first parameter name from a - __text_signature__ of a builtin method, which should - be in the following format: '($param1, ...)'. - Assumptions are that the first argument won't have - a default value or an annotation. - """ - - assert spec.startswith('($') - - pos = spec.find(',') - if pos == -1: - pos = spec.find(')') - - cpos = spec.find(':') - assert cpos == -1 or cpos > pos - - cpos = spec.find('=') - assert cpos == -1 or cpos > pos - - return spec[2:pos] + (isinstance(annotations, (dict)) or annotations is None) ) def _signature_strip_non_python_syntax(signature): @@ -1891,26 +2181,21 @@ def _signature_strip_non_python_syntax(signature): Private helper function. Takes a signature in Argument Clinic's extended signature format. - Returns a tuple of three things: - * that signature re-rendered in standard Python syntax, + Returns a tuple of two things: + * that signature re-rendered in standard Python syntax, and * the index of the "self" parameter (generally 0), or None if - the function does not have a "self" parameter, and - * the index of the last "positional only" parameter, - or None if the signature has no positional-only parameters. + the function does not have a "self" parameter. """ if not signature: - return signature, None, None + return signature, None self_parameter = None - last_positional_only = None - lines = [l.encode('ascii') for l in signature.split('\n')] + lines = [l.encode('ascii') for l in signature.split('\n') if l] generator = iter(lines).__next__ token_stream = tokenize.tokenize(generator) - delayed_comma = False - skip_next_comma = False text = [] add = text.append @@ -1927,49 +2212,27 @@ def _signature_strip_non_python_syntax(signature): if type == OP: if string == ',': - if skip_next_comma: - skip_next_comma = False - else: - assert not delayed_comma - delayed_comma = True - current_parameter += 1 - continue + current_parameter += 1 - if string == '/': - assert not skip_next_comma - assert last_positional_only is None - skip_next_comma = True - last_positional_only = current_parameter - 1 - continue - - if (type == ERRORTOKEN) and (string == '$'): + if (type == OP) and (string == '$'): assert self_parameter is None self_parameter = current_parameter continue - if delayed_comma: - delayed_comma = False - if not ((type == OP) and (string == ')')): - add(', ') add(string) if (string == ','): add(' ') - clean_signature = ''.join(text) - return clean_signature, self_parameter, last_positional_only + clean_signature = ''.join(text).strip().replace("\n", "") + return clean_signature, self_parameter def _signature_fromstr(cls, obj, s, skip_bound_arg=True): """Private helper to parse content of '__text_signature__' and return a Signature based on it. """ - # Lazy import ast because it's relatively heavy and - # it's not used for other than this function. - import ast - Parameter = cls._parameter_cls - clean_signature, self_parameter, last_positional_only = \ - _signature_strip_non_python_syntax(s) + clean_signature, self_parameter = _signature_strip_non_python_syntax(s) program = "def foo" + clean_signature + ": pass" @@ -1985,11 +2248,15 @@ def _signature_fromstr(cls, obj, s, skip_bound_arg=True): parameters = [] empty = Parameter.empty - invalid = object() module = None module_dict = {} + module_name = getattr(obj, '__module__', None) + if not module_name: + objclass = getattr(obj, '__objclass__', None) + module_name = getattr(objclass, '__module__', None) + if module_name: module = sys.modules.get(module_name, None) if module: @@ -2009,11 +2276,11 @@ def wrap_value(s): try: value = eval(s, sys_module_dict) except NameError: - raise RuntimeError() + raise ValueError if isinstance(value, (str, int, float, bytes, bool, type(None))): return ast.Constant(value) - raise RuntimeError() + raise ValueError class RewriteSymbolics(ast.NodeTransformer): def visit_Attribute(self, node): @@ -2023,7 +2290,7 @@ def visit_Attribute(self, node): a.append(n.attr) n = n.value if not isinstance(n, ast.Name): - raise RuntimeError() + raise ValueError a.append(n.id) value = ".".join(reversed(a)) return wrap_value(value) @@ -2033,33 +2300,43 @@ def visit_Name(self, node): raise ValueError() return wrap_value(node.id) + def visit_BinOp(self, node): + # Support constant folding of a couple simple binary operations + # commonly used to define default values in text signatures + left = self.visit(node.left) + right = self.visit(node.right) + if not isinstance(left, ast.Constant) or not isinstance(right, ast.Constant): + raise ValueError + if isinstance(node.op, ast.Add): + return ast.Constant(left.value + right.value) + elif isinstance(node.op, ast.Sub): + return ast.Constant(left.value - right.value) + elif isinstance(node.op, ast.BitOr): + return ast.Constant(left.value | right.value) + raise ValueError + def p(name_node, default_node, default=empty): name = parse_name(name_node) - if name is invalid: - return None if default_node and default_node is not _empty: try: default_node = RewriteSymbolics().visit(default_node) - o = ast.literal_eval(default_node) + default = ast.literal_eval(default_node) except ValueError: - o = invalid - if o is invalid: - return None - default = o if o is not invalid else default + raise ValueError("{!r} builtin has invalid signature".format(obj)) from None parameters.append(Parameter(name, kind, default=default, annotation=empty)) # non-keyword-only parameters - args = reversed(f.args.args) - defaults = reversed(f.args.defaults) - iter = itertools.zip_longest(args, defaults, fillvalue=None) - if last_positional_only is not None: - kind = Parameter.POSITIONAL_ONLY - else: - kind = Parameter.POSITIONAL_OR_KEYWORD - for i, (name, default) in enumerate(reversed(list(iter))): + total_non_kw_args = len(f.args.posonlyargs) + len(f.args.args) + required_non_kw_args = total_non_kw_args - len(f.args.defaults) + defaults = itertools.chain(itertools.repeat(None, required_non_kw_args), f.args.defaults) + + kind = Parameter.POSITIONAL_ONLY + for (name, default) in zip(f.args.posonlyargs, defaults): + p(name, default) + + kind = Parameter.POSITIONAL_OR_KEYWORD + for (name, default) in zip(f.args.args, defaults): p(name, default) - if i == last_positional_only: - kind = Parameter.POSITIONAL_OR_KEYWORD # *args if f.args.vararg: @@ -2112,7 +2389,8 @@ def _signature_from_builtin(cls, func, skip_bound_arg=True): return _signature_fromstr(cls, func, s, skip_bound_arg) -def _signature_from_function(cls, func, skip_bound_arg=True): +def _signature_from_function(cls, func, skip_bound_arg=True, + globals=None, locals=None, eval_str=False): """Private helper: constructs Signature for the given python function.""" is_duck_function = False @@ -2138,7 +2416,7 @@ def _signature_from_function(cls, func, skip_bound_arg=True): positional = arg_names[:pos_count] keyword_only_count = func_code.co_kwonlyargcount keyword_only = arg_names[pos_count:pos_count + keyword_only_count] - annotations = func.__annotations__ + annotations = get_annotations(func, globals=globals, locals=locals, eval_str=eval_str) defaults = func.__defaults__ kwdefaults = func.__kwdefaults__ @@ -2206,26 +2484,42 @@ def _signature_from_function(cls, func, skip_bound_arg=True): __validate_parameters__=is_duck_function) +def _descriptor_get(descriptor, obj): + if isclass(descriptor): + return descriptor + get = getattr(type(descriptor), '__get__', _sentinel) + if get is _sentinel: + return descriptor + return get(descriptor, obj, type(obj)) + + def _signature_from_callable(obj, *, follow_wrapper_chains=True, skip_bound_arg=True, + globals=None, + locals=None, + eval_str=False, sigcls): """Private helper function to get signature for arbitrary callable objects. """ + _get_signature_of = functools.partial(_signature_from_callable, + follow_wrapper_chains=follow_wrapper_chains, + skip_bound_arg=skip_bound_arg, + globals=globals, + locals=locals, + sigcls=sigcls, + eval_str=eval_str) + if not callable(obj): raise TypeError('{!r} is not a callable object'.format(obj)) if isinstance(obj, types.MethodType): # In this case we skip the first parameter of the underlying # function (usually `self` or `cls`). - sig = _signature_from_callable( - obj.__func__, - follow_wrapper_chains=follow_wrapper_chains, - skip_bound_arg=skip_bound_arg, - sigcls=sigcls) + sig = _get_signature_of(obj.__func__) if skip_bound_arg: return _signature_bound_method(sig) @@ -2234,16 +2528,15 @@ def _signature_from_callable(obj, *, # Was this function wrapped by a decorator? if follow_wrapper_chains: - obj = unwrap(obj, stop=(lambda f: hasattr(f, "__signature__"))) + # Unwrap until we find an explicit signature or a MethodType (which will be + # handled explicitly below). + obj = unwrap(obj, stop=(lambda f: hasattr(f, "__signature__") + or isinstance(f, types.MethodType))) if isinstance(obj, types.MethodType): # If the unwrapped object is a *method*, we might want to # skip its first parameter (self). # See test_signature_wrapped_bound_method for details. - return _signature_from_callable( - obj, - follow_wrapper_chains=follow_wrapper_chains, - skip_bound_arg=skip_bound_arg, - sigcls=sigcls) + return _get_signature_of(obj) try: sig = obj.__signature__ @@ -2251,14 +2544,22 @@ def _signature_from_callable(obj, *, pass else: if sig is not None: + # since __text_signature__ is not writable on classes, __signature__ + # may contain text (or be a callable that returns text); + # if so, convert it + o_sig = sig + if not isinstance(sig, (Signature, str)) and callable(sig): + sig = sig() + if isinstance(sig, str): + sig = _signature_fromstr(sigcls, obj, sig) if not isinstance(sig, Signature): raise TypeError( 'unexpected object {!r} in __signature__ ' - 'attribute'.format(sig)) + 'attribute'.format(o_sig)) return sig try: - partialmethod = obj._partialmethod + partialmethod = obj.__partialmethod__ except AttributeError: pass else: @@ -2270,11 +2571,7 @@ def _signature_from_callable(obj, *, # (usually `self`, or `cls`) will not be passed # automatically (as for boundmethods) - wrapped_sig = _signature_from_callable( - partialmethod.func, - follow_wrapper_chains=follow_wrapper_chains, - skip_bound_arg=skip_bound_arg, - sigcls=sigcls) + wrapped_sig = _get_signature_of(partialmethod.func) sig = _signature_get_partial(wrapped_sig, partialmethod, (None,)) first_wrapped_param = tuple(wrapped_sig.parameters.values())[0] @@ -2289,121 +2586,112 @@ def _signature_from_callable(obj, *, new_params = (first_wrapped_param,) + sig_params return sig.replace(parameters=new_params) + if isinstance(obj, functools.partial): + wrapped_sig = _get_signature_of(obj.func) + return _signature_get_partial(wrapped_sig, obj) + if isfunction(obj) or _signature_is_functionlike(obj): # If it's a pure Python function, or an object that is duck type # of a Python function (Cython functions, for instance), then: return _signature_from_function(sigcls, obj, - skip_bound_arg=skip_bound_arg) + skip_bound_arg=skip_bound_arg, + globals=globals, locals=locals, eval_str=eval_str) if _signature_is_builtin(obj): return _signature_from_builtin(sigcls, obj, skip_bound_arg=skip_bound_arg) - if isinstance(obj, functools.partial): - wrapped_sig = _signature_from_callable( - obj.func, - follow_wrapper_chains=follow_wrapper_chains, - skip_bound_arg=skip_bound_arg, - sigcls=sigcls) - return _signature_get_partial(wrapped_sig, obj) - - sig = None if isinstance(obj, type): # obj is a class or a metaclass # First, let's see if it has an overloaded __call__ defined # in its metaclass - call = _signature_get_user_defined_method(type(obj), '__call__') + call = _signature_get_user_defined_method( + type(obj), + '__call__', + follow_wrapper_chains=follow_wrapper_chains, + ) if call is not None: - sig = _signature_from_callable( - call, - follow_wrapper_chains=follow_wrapper_chains, - skip_bound_arg=skip_bound_arg, - sigcls=sigcls) - else: - # Now we check if the 'obj' class has a '__new__' method - new = _signature_get_user_defined_method(obj, '__new__') - if new is not None: - sig = _signature_from_callable( - new, - follow_wrapper_chains=follow_wrapper_chains, - skip_bound_arg=skip_bound_arg, - sigcls=sigcls) + return _get_signature_of(call) + + # NOTE: The user-defined method can be a function with a thin wrapper + # around object.__new__ (e.g., generated by `@warnings.deprecated`) + new = _signature_get_user_defined_method( + obj, + '__new__', + follow_wrapper_chains=follow_wrapper_chains, + ) + init = _signature_get_user_defined_method( + obj, + '__init__', + follow_wrapper_chains=follow_wrapper_chains, + ) + + # Go through the MRO and see if any class has user-defined + # pure Python __new__ or __init__ method + for base in obj.__mro__: + # Now we check if the 'obj' class has an own '__new__' method + if new is not None and '__new__' in base.__dict__: + sig = _get_signature_of(new) + if skip_bound_arg: + sig = _signature_bound_method(sig) + return sig + # or an own '__init__' method + elif init is not None and '__init__' in base.__dict__: + return _get_signature_of(init) + + # At this point we know, that `obj` is a class, with no user- + # defined '__init__', '__new__', or class-level '__call__' + + for base in obj.__mro__[:-1]: + # Since '__text_signature__' is implemented as a + # descriptor that extracts text signature from the + # class docstring, if 'obj' is derived from a builtin + # class, its own '__text_signature__' may be 'None'. + # Therefore, we go through the MRO (except the last + # class in there, which is 'object') to find the first + # class with non-empty text signature. + try: + text_sig = base.__text_signature__ + except AttributeError: + pass else: - # Finally, we should have at least __init__ implemented - init = _signature_get_user_defined_method(obj, '__init__') - if init is not None: - sig = _signature_from_callable( - init, - follow_wrapper_chains=follow_wrapper_chains, - skip_bound_arg=skip_bound_arg, - sigcls=sigcls) - - if sig is None: - # At this point we know, that `obj` is a class, with no user- - # defined '__init__', '__new__', or class-level '__call__' - - for base in obj.__mro__[:-1]: - # Since '__text_signature__' is implemented as a - # descriptor that extracts text signature from the - # class docstring, if 'obj' is derived from a builtin - # class, its own '__text_signature__' may be 'None'. - # Therefore, we go through the MRO (except the last - # class in there, which is 'object') to find the first - # class with non-empty text signature. - try: - text_sig = base.__text_signature__ - except AttributeError: - pass - else: - if text_sig: - # If 'obj' class has a __text_signature__ attribute: - # return a signature based on it - return _signature_fromstr(sigcls, obj, text_sig) - - # No '__text_signature__' was found for the 'obj' class. - # Last option is to check if its '__init__' is - # object.__init__ or type.__init__. - if type not in obj.__mro__: - # We have a class (not metaclass), but no user-defined - # __init__ or __new__ for it - if (obj.__init__ is object.__init__ and - obj.__new__ is object.__new__): - # Return a signature of 'object' builtin. - return sigcls.from_callable(object) - else: - raise ValueError( - 'no signature found for builtin type {!r}'.format(obj)) + if text_sig: + # If 'base' class has a __text_signature__ attribute: + # return a signature based on it + return _signature_fromstr(sigcls, base, text_sig) + + # No '__text_signature__' was found for the 'obj' class. + # Last option is to check if its '__init__' is + # object.__init__ or type.__init__. + if type not in obj.__mro__: + obj_init = obj.__init__ + obj_new = obj.__new__ + if follow_wrapper_chains: + obj_init = unwrap(obj_init) + obj_new = unwrap(obj_new) + # We have a class (not metaclass), but no user-defined + # __init__ or __new__ for it + if obj_init is object.__init__ and obj_new is object.__new__: + # Return a signature of 'object' builtin. + return sigcls.from_callable(object) + else: + raise ValueError( + 'no signature found for builtin type {!r}'.format(obj)) - elif not isinstance(obj, _NonUserDefinedCallables): + else: # An object with __call__ - # We also check that the 'obj' is not an instance of - # _WrapperDescriptor or _MethodWrapper to avoid - # infinite recursion (and even potential segfault) - call = _signature_get_user_defined_method(type(obj), '__call__') + call = getattr_static(type(obj), '__call__', None) if call is not None: try: - sig = _signature_from_callable( - call, - follow_wrapper_chains=follow_wrapper_chains, - skip_bound_arg=skip_bound_arg, - sigcls=sigcls) - except ValueError as ex: - msg = 'no signature found for {!r}'.format(obj) - raise ValueError(msg) from ex - - if sig is not None: - # For classes and objects we skip the first parameter of their - # __call__, __new__, or __init__ methods - if skip_bound_arg: - return _signature_bound_method(sig) - else: - return sig - - if isinstance(obj, types.BuiltinFunctionType): - # Raise a nicer error message for builtins - msg = 'no signature found for builtin function {!r}'.format(obj) - raise ValueError(msg) + text_sig = obj.__text_signature__ + except AttributeError: + pass + else: + if text_sig: + return _signature_fromstr(sigcls, obj, text_sig) + call = _descriptor_get(call, obj) + return _get_signature_of(call) raise ValueError('callable {!r} is not supported by signature'.format(obj)) @@ -2417,18 +2705,21 @@ class _empty: class _ParameterKind(enum.IntEnum): - POSITIONAL_ONLY = 0 - POSITIONAL_OR_KEYWORD = 1 - VAR_POSITIONAL = 2 - KEYWORD_ONLY = 3 - VAR_KEYWORD = 4 + POSITIONAL_ONLY = 'positional-only' + POSITIONAL_OR_KEYWORD = 'positional or keyword' + VAR_POSITIONAL = 'variadic positional' + KEYWORD_ONLY = 'keyword-only' + VAR_KEYWORD = 'variadic keyword' + + def __new__(cls, description): + value = len(cls.__members__) + member = int.__new__(cls, value) + member._value_ = value + member.description = description + return member def __str__(self): - return self._name_ - - @property - def description(self): - return _PARAM_NAME_MAPPING[self] + return self.name _POSITIONAL_ONLY = _ParameterKind.POSITIONAL_ONLY _POSITIONAL_OR_KEYWORD = _ParameterKind.POSITIONAL_OR_KEYWORD @@ -2436,14 +2727,6 @@ def description(self): _KEYWORD_ONLY = _ParameterKind.KEYWORD_ONLY _VAR_KEYWORD = _ParameterKind.VAR_KEYWORD -_PARAM_NAME_MAPPING = { - _POSITIONAL_ONLY: 'positional-only', - _POSITIONAL_OR_KEYWORD: 'positional or keyword', - _VAR_POSITIONAL: 'variadic positional', - _KEYWORD_ONLY: 'keyword-only', - _VAR_KEYWORD: 'variadic keyword' -} - class Parameter: """Represents a parameter in a function signature. @@ -2512,7 +2795,10 @@ def __init__(self, name, kind, *, default=_empty, annotation=_empty): self._kind = _POSITIONAL_ONLY name = 'implicit{}'.format(name[1:]) - if not name.isidentifier(): + # It's possible for C functions to have a positional-only parameter + # where the name is a keyword, so for compatibility we'll allow it. + is_keyword = iskeyword(name) and self._kind is not _POSITIONAL_ONLY + if is_keyword or not name.isidentifier(): raise ValueError('{!r} is not a valid parameter name'.format(name)) self._name = name @@ -2583,11 +2869,13 @@ def __str__(self): return formatted + __replace__ = replace + def __repr__(self): return '<{} "{}">'.format(self.__class__.__name__, self) def __hash__(self): - return hash((self.name, self.kind, self.annotation, self.default)) + return hash((self._name, self._kind, self._annotation, self._default)) def __eq__(self, other): if self is other: @@ -2606,7 +2894,7 @@ class BoundArguments: Has the following public attributes: - * arguments : OrderedDict + * arguments : dict An ordered mutable mapping of parameters' names to arguments' values. Does not contain arguments' default values. * signature : Signature @@ -2706,7 +2994,7 @@ def apply_defaults(self): # Signature.bind_partial(). continue new_arguments.append((name, val)) - self.arguments = OrderedDict(new_arguments) + self.arguments = dict(new_arguments) def __eq__(self, other): if self is other: @@ -2772,9 +3060,9 @@ def __init__(self, parameters=None, *, return_annotation=_empty, if __validate_parameters__: params = OrderedDict() top_kind = _POSITIONAL_ONLY - kind_defaults = False + seen_default = False - for idx, param in enumerate(parameters): + for param in parameters: kind = param.kind name = param.name @@ -2787,21 +3075,19 @@ def __init__(self, parameters=None, *, return_annotation=_empty, kind.description) raise ValueError(msg) elif kind > top_kind: - kind_defaults = False top_kind = kind if kind in (_POSITIONAL_ONLY, _POSITIONAL_OR_KEYWORD): if param.default is _empty: - if kind_defaults: + if seen_default: # No default for this parameter, but the - # previous parameter of the same kind had - # a default + # previous parameter of had a default msg = 'non-default argument follows default ' \ 'argument' raise ValueError(msg) else: # There is a default for this parameter. - kind_defaults = True + seen_default = True if name in params: msg = 'duplicate parameter name: {!r}'.format(name) @@ -2809,41 +3095,18 @@ def __init__(self, parameters=None, *, return_annotation=_empty, params[name] = param else: - params = OrderedDict(((param.name, param) - for param in parameters)) + params = OrderedDict((param.name, param) for param in parameters) self._parameters = types.MappingProxyType(params) self._return_annotation = return_annotation @classmethod - def from_function(cls, func): - """Constructs Signature for the given python function. - - Deprecated since Python 3.5, use `Signature.from_callable()`. - """ - - warnings.warn("inspect.Signature.from_function() is deprecated since " - "Python 3.5, use Signature.from_callable()", - DeprecationWarning, stacklevel=2) - return _signature_from_function(cls, func) - - @classmethod - def from_builtin(cls, func): - """Constructs Signature for the given builtin function. - - Deprecated since Python 3.5, use `Signature.from_callable()`. - """ - - warnings.warn("inspect.Signature.from_builtin() is deprecated since " - "Python 3.5, use Signature.from_callable()", - DeprecationWarning, stacklevel=2) - return _signature_from_builtin(cls, func) - - @classmethod - def from_callable(cls, obj, *, follow_wrapped=True): + def from_callable(cls, obj, *, + follow_wrapped=True, globals=None, locals=None, eval_str=False): """Constructs Signature for the given callable object.""" return _signature_from_callable(obj, sigcls=cls, - follow_wrapper_chains=follow_wrapped) + follow_wrapper_chains=follow_wrapped, + globals=globals, locals=locals, eval_str=eval_str) @property def parameters(self): @@ -2868,6 +3131,8 @@ def replace(self, *, parameters=_void, return_annotation=_void): return type(self)(parameters, return_annotation=return_annotation) + __replace__ = replace + def _hash_basis(self): params = tuple(param for param in self.parameters.values() if param.kind != _KEYWORD_ONLY) @@ -2892,12 +3157,14 @@ def __eq__(self, other): def _bind(self, args, kwargs, *, partial=False): """Private method. Don't use directly.""" - arguments = OrderedDict() + arguments = {} parameters = iter(self.parameters.values()) parameters_ex = () arg_vals = iter(args) + pos_only_param_in_kwargs = [] + while True: # Let's iterate through the positional arguments and corresponding # parameters @@ -2918,10 +3185,13 @@ def _bind(self, args, kwargs, *, partial=False): break elif param.name in kwargs: if param.kind == _POSITIONAL_ONLY: - msg = '{arg!r} parameter is positional only, ' \ - 'but was passed as a keyword' - msg = msg.format(arg=param.name) - raise TypeError(msg) from None + if param.default is _empty: + msg = f'missing a required positional-only argument: {param.name!r}' + raise TypeError(msg) + # Raise a TypeError once we are sure there is no + # **kwargs param later. + pos_only_param_in_kwargs.append(param) + continue parameters_ex = (param,) break elif (param.kind == _VAR_KEYWORD or @@ -2938,8 +3208,12 @@ def _bind(self, args, kwargs, *, partial=False): parameters_ex = (param,) break else: - msg = 'missing a required argument: {arg!r}' - msg = msg.format(arg=param.name) + if param.kind == _KEYWORD_ONLY: + argtype = ' keyword-only' + else: + argtype = '' + msg = 'missing a required{argtype} argument: {arg!r}' + msg = msg.format(arg=param.name, argtype=argtype) raise TypeError(msg) from None else: # We have a positional argument to process @@ -2999,20 +3273,22 @@ def _bind(self, args, kwargs, *, partial=False): format(arg=param_name)) from None else: - if param.kind == _POSITIONAL_ONLY: - # This should never happen in case of a properly built - # Signature object (but let's have this check here - # to ensure correct behaviour just in case) - raise TypeError('{arg!r} parameter is positional only, ' - 'but was passed as a keyword'. \ - format(arg=param.name)) - arguments[param_name] = arg_val if kwargs: if kwargs_param is not None: # Process our '**kwargs'-like parameter arguments[kwargs_param.name] = kwargs + elif pos_only_param_in_kwargs: + raise TypeError( + 'got some positional-only arguments passed as ' + 'keyword arguments: {arg!r}'.format( + arg=', '.join( + param.name + for param in pos_only_param_in_kwargs + ), + ), + ) else: raise TypeError( 'got an unexpected keyword argument {arg!r}'.format( @@ -3046,6 +3322,16 @@ def __repr__(self): return '<{} {}>'.format(self.__class__.__name__, self) def __str__(self): + return self.format() + + def format(self, *, max_width=None): + """Create a string representation of the Signature object. + + If *max_width* integer is passed, + signature will try to fit into the *max_width*. + If signature is longer than *max_width*, + all parameters will be on separate lines. + """ result = [] render_pos_only_separator = False render_kw_only_separator = True @@ -3083,6 +3369,8 @@ def __str__(self): result.append('/') rendered = '({})'.format(', '.join(result)) + if max_width is not None and len(rendered) > max_width: + rendered = '(\n {}\n)'.format(',\n '.join(result)) if self.return_annotation is not _empty: anno = formatannotation(self.return_annotation) @@ -3091,9 +3379,32 @@ def __str__(self): return rendered -def signature(obj, *, follow_wrapped=True): +def signature(obj, *, follow_wrapped=True, globals=None, locals=None, eval_str=False): """Get a signature object for the passed callable.""" - return Signature.from_callable(obj, follow_wrapped=follow_wrapped) + return Signature.from_callable(obj, follow_wrapped=follow_wrapped, + globals=globals, locals=locals, eval_str=eval_str) + + +class BufferFlags(enum.IntFlag): + SIMPLE = 0x0 + WRITABLE = 0x1 + FORMAT = 0x4 + ND = 0x8 + STRIDES = 0x10 | ND + C_CONTIGUOUS = 0x20 | STRIDES + F_CONTIGUOUS = 0x40 | STRIDES + ANY_CONTIGUOUS = 0x80 | STRIDES + INDIRECT = 0x100 | STRIDES + CONTIG = ND | WRITABLE + CONTIG_RO = ND + STRIDED = STRIDES | WRITABLE + STRIDED_RO = STRIDES + RECORDS = STRIDES | WRITABLE | FORMAT + RECORDS_RO = STRIDES | FORMAT + FULL = INDIRECT | WRITABLE | FORMAT + FULL_RO = INDIRECT | FORMAT + READ = 0x100 + WRITE = 0x200 def _main(): diff --git a/Lib/io.py b/Lib/io.py index c2812876d3..f0e2fa15d5 100644 --- a/Lib/io.py +++ b/Lib/io.py @@ -46,23 +46,17 @@ "BufferedReader", "BufferedWriter", "BufferedRWPair", "BufferedRandom", "TextIOBase", "TextIOWrapper", "UnsupportedOperation", "SEEK_SET", "SEEK_CUR", "SEEK_END", - "DEFAULT_BUFFER_SIZE", "text_encoding", - "IncrementalNewlineDecoder" - ] + "DEFAULT_BUFFER_SIZE", "text_encoding", "IncrementalNewlineDecoder"] import _io import abc from _io import (DEFAULT_BUFFER_SIZE, BlockingIOError, UnsupportedOperation, - open, open_code, BytesIO, StringIO, BufferedReader, + open, open_code, FileIO, BytesIO, StringIO, BufferedReader, BufferedWriter, BufferedRWPair, BufferedRandom, IncrementalNewlineDecoder, text_encoding, TextIOWrapper) -try: - from _io import FileIO -except ImportError: - pass # Pretend this exception was created here. UnsupportedOperation.__module__ = "io" @@ -87,10 +81,7 @@ class BufferedIOBase(_io._BufferedIOBase, IOBase): class TextIOBase(_io._TextIOBase, IOBase): __doc__ = _io._TextIOBase.__doc__ -try: - RawIOBase.register(FileIO) -except NameError: - pass +RawIOBase.register(FileIO) for klass in (BytesIO, BufferedReader, BufferedWriter, BufferedRandom, BufferedRWPair): diff --git a/Lib/json/__init__.py b/Lib/json/__init__.py index e4c21daaf3..ed2c74771e 100644 --- a/Lib/json/__init__.py +++ b/Lib/json/__init__.py @@ -1,4 +1,4 @@ -r"""JSON (JavaScript Object Notation) is a subset of +r"""JSON (JavaScript Object Notation) is a subset of JavaScript syntax (ECMA-262 3rd edition) used as a lightweight data interchange format. diff --git a/Lib/json/decoder.py b/Lib/json/decoder.py index 239bacdb0f..9e6ca981d7 100644 --- a/Lib/json/decoder.py +++ b/Lib/json/decoder.py @@ -41,7 +41,6 @@ def _from_serde(cls, msg, doc, line, col): pos += col return cls(msg, doc, pos) - # Note that this exception is used from _json def __init__(self, msg, doc, pos): lineno = doc.count('\n', 0, pos) + 1 @@ -65,17 +64,18 @@ def __reduce__(self): } +HEXDIGITS = re.compile(r'[0-9A-Fa-f]{4}', FLAGS) STRINGCHUNK = re.compile(r'(.*?)(["\\\x00-\x1f])', FLAGS) BACKSLASH = { '"': '"', '\\': '\\', '/': '/', 'b': '\b', 'f': '\f', 'n': '\n', 'r': '\r', 't': '\t', } -def _decode_uXXXX(s, pos): - esc = s[pos + 1:pos + 5] - if len(esc) == 4 and esc[1] not in 'xX': +def _decode_uXXXX(s, pos, _m=HEXDIGITS.match): + esc = _m(s, pos + 1) + if esc is not None: try: - return int(esc, 16) + return int(esc.group(), 16) except ValueError: pass msg = "Invalid \\uXXXX escape" @@ -215,10 +215,13 @@ def JSONObject(s_and_end, strict, scan_once, object_hook, object_pairs_hook, break elif nextchar != ',': raise JSONDecodeError("Expecting ',' delimiter", s, end - 1) + comma_idx = end - 1 end = _w(s, end).end() nextchar = s[end:end + 1] end += 1 if nextchar != '"': + if nextchar == '}': + raise JSONDecodeError("Illegal trailing comma before end of object", s, comma_idx) raise JSONDecodeError( "Expecting property name enclosed in double quotes", s, end - 1) if object_pairs_hook is not None: @@ -255,19 +258,23 @@ def JSONArray(s_and_end, scan_once, _w=WHITESPACE.match, _ws=WHITESPACE_STR): break elif nextchar != ',': raise JSONDecodeError("Expecting ',' delimiter", s, end - 1) + comma_idx = end - 1 try: if s[end] in _ws: end += 1 if s[end] in _ws: end = _w(s, end + 1).end() + nextchar = s[end:end + 1] except IndexError: pass + if nextchar == ']': + raise JSONDecodeError("Illegal trailing comma before end of array", s, comma_idx) return values, end class JSONDecoder(object): - """Simple JSON decoder + """Simple JSON decoder Performs the following translations in decoding by default: diff --git a/Lib/json/encoder.py b/Lib/json/encoder.py index 21bff2c1a1..08ef39d159 100644 --- a/Lib/json/encoder.py +++ b/Lib/json/encoder.py @@ -30,6 +30,7 @@ for i in range(0x20): ESCAPE_DCT.setdefault(chr(i), '\\u{0:04x}'.format(i)) #ESCAPE_DCT.setdefault(chr(i), '\\u%04x' % (i,)) +del i INFINITY = float('inf') @@ -71,7 +72,7 @@ def replace(match): c_encode_basestring_ascii or py_encode_basestring_ascii) class JSONEncoder(object): - """Extensible JSON encoder for Python data structures. + """Extensible JSON encoder for Python data structures. Supports the following objects and types by default: @@ -107,8 +108,8 @@ def __init__(self, *, skipkeys=False, ensure_ascii=True, """Constructor for JSONEncoder, with sensible defaults. If skipkeys is false, then it is a TypeError to attempt - encoding of keys that are not str, int, float or None. If - skipkeys is True, such items are simply skipped. + encoding of keys that are not str, int, float, bool or None. + If skipkeys is True, such items are simply skipped. If ensure_ascii is true, the output is guaranteed to be str objects with all incoming non-ASCII characters escaped. If @@ -173,7 +174,7 @@ def default(self, o): else: return list(iterable) # Let the base class default method raise the TypeError - return JSONEncoder.default(self, o) + return super().default(o) """ raise TypeError(f'Object of type {o.__class__.__name__} ' @@ -243,15 +244,18 @@ def floatstr(o, allow_nan=self.allow_nan, return text - if (_one_shot and c_make_encoder is not None - and self.indent is None): + if self.indent is None or isinstance(self.indent, str): + indent = self.indent + else: + indent = ' ' * self.indent + if _one_shot and c_make_encoder is not None: _iterencode = c_make_encoder( - markers, self.default, _encoder, self.indent, + markers, self.default, _encoder, indent, self.key_separator, self.item_separator, self.sort_keys, self.skipkeys, self.allow_nan) else: _iterencode = _make_iterencode( - markers, self.default, _encoder, self.indent, floatstr, + markers, self.default, _encoder, indent, floatstr, self.key_separator, self.item_separator, self.sort_keys, self.skipkeys, _one_shot) return _iterencode(o, 0) @@ -271,9 +275,6 @@ def _make_iterencode(markers, _default, _encoder, _indent, _floatstr, _intstr=int.__repr__, ): - if _indent is not None and not isinstance(_indent, str): - _indent = ' ' * _indent - def _iterencode_list(lst, _current_indent_level): if not lst: yield '[]' @@ -344,7 +345,6 @@ def _iterencode_dict(dct, _current_indent_level): _current_indent_level += 1 newline_indent = '\n' + _indent * _current_indent_level item_separator = _item_separator + newline_indent - yield newline_indent else: newline_indent = None item_separator = _item_separator @@ -377,6 +377,8 @@ def _iterencode_dict(dct, _current_indent_level): f'not {key.__class__.__name__}') if first: first = False + if newline_indent is not None: + yield newline_indent else: yield item_separator yield _encoder(key) @@ -403,7 +405,7 @@ def _iterencode_dict(dct, _current_indent_level): else: chunks = _iterencode(value, _current_indent_level) yield from chunks - if newline_indent is not None: + if not first and newline_indent is not None: _current_indent_level -= 1 yield '\n' + _indent * _current_indent_level yield '}' diff --git a/Lib/json/scanner.py b/Lib/json/scanner.py index 7a61cfc2d2..090897515f 100644 --- a/Lib/json/scanner.py +++ b/Lib/json/scanner.py @@ -9,7 +9,7 @@ __all__ = ['make_scanner'] NUMBER_RE = re.compile( - r'(-?(?:0|[1-9]\d*))(\.\d+)?([eE][-+]?\d+)?', + r'(-?(?:0|[1-9][0-9]*))(\.[0-9]+)?([eE][-+]?[0-9]+)?', (re.VERBOSE | re.MULTILINE | re.DOTALL)) def py_make_scanner(context): diff --git a/Lib/json/tool.py b/Lib/json/tool.py index 0490b8c0be..fdfc3372bc 100644 --- a/Lib/json/tool.py +++ b/Lib/json/tool.py @@ -13,7 +13,6 @@ import argparse import json import sys -from pathlib import Path def main(): @@ -22,11 +21,9 @@ def main(): 'to validate and pretty-print JSON objects.') parser = argparse.ArgumentParser(prog=prog, description=description) parser.add_argument('infile', nargs='?', - type=argparse.FileType(encoding="utf-8"), help='a JSON file to be validated or pretty-printed', - default=sys.stdin) + default='-') parser.add_argument('outfile', nargs='?', - type=Path, help='write the output of infile to outfile', default=None) parser.add_argument('--sort-keys', action='store_true', default=False, @@ -59,23 +56,30 @@ def main(): dump_args['indent'] = None dump_args['separators'] = ',', ':' - with options.infile as infile: + try: + if options.infile == '-': + infile = sys.stdin + else: + infile = open(options.infile, encoding='utf-8') try: if options.json_lines: objs = (json.loads(line) for line in infile) else: objs = (json.load(infile),) + finally: + if infile is not sys.stdin: + infile.close() - if options.outfile is None: - out = sys.stdout - else: - out = options.outfile.open('w', encoding='utf-8') - with out as outfile: - for obj in objs: - json.dump(obj, outfile, **dump_args) - outfile.write('\n') - except ValueError as e: - raise SystemExit(e) + if options.outfile is None: + outfile = sys.stdout + else: + outfile = open(options.outfile, 'w', encoding='utf-8') + with outfile: + for obj in objs: + json.dump(obj, outfile, **dump_args) + outfile.write('\n') + except ValueError as e: + raise SystemExit(e) if __name__ == '__main__': diff --git a/Lib/locale.py b/Lib/locale.py index 7a7694e1bf..db6d0abb26 100644 --- a/Lib/locale.py +++ b/Lib/locale.py @@ -25,8 +25,8 @@ # Yuck: LC_MESSAGES is non-standard: can't tell whether it exists before # trying the import. So __all__ is also fiddled at the end of the file. __all__ = ["getlocale", "getdefaultlocale", "getpreferredencoding", "Error", - "setlocale", "resetlocale", "localeconv", "strcoll", "strxfrm", - "str", "atof", "atoi", "format", "format_string", "currency", + "setlocale", "localeconv", "strcoll", "strxfrm", + "str", "atof", "atoi", "format_string", "currency", "normalize", "LC_CTYPE", "LC_COLLATE", "LC_TIME", "LC_MONETARY", "LC_NUMERIC", "LC_ALL", "CHAR_MAX", "getencoding"] @@ -247,21 +247,6 @@ def format_string(f, val, grouping=False, monetary=False): return new_f % val -def format(percent, value, grouping=False, monetary=False, *additional): - """Deprecated, use format_string instead.""" - import warnings - warnings.warn( - "This method will be removed in a future version of Python. " - "Use 'locale.format_string()' instead.", - DeprecationWarning, stacklevel=2 - ) - - match = _percent_re.match(percent) - if not match or len(match.group())!= len(percent): - raise ValueError(("format() must be given exactly one %%char " - "format specifier, %s not valid") % repr(percent)) - return _format(percent, value, grouping, monetary, *additional) - def currency(val, symbol=True, grouping=False, international=False): """Formats val according to the currency settings in the current locale.""" @@ -556,11 +541,15 @@ def getdefaultlocale(envvars=('LC_ALL', 'LC_CTYPE', 'LANG', 'LANGUAGE')): """ import warnings - warnings.warn( - "Use setlocale(), getencoding() and getlocale() instead", - DeprecationWarning, stacklevel=2 - ) + warnings._deprecated( + "locale.getdefaultlocale", + "{name!r} is deprecated and slated for removal in Python {remove}. " + "Use setlocale(), getencoding() and getlocale() instead.", + remove=(3, 15)) + return _getdefaultlocale(envvars) + +def _getdefaultlocale(envvars=('LC_ALL', 'LC_CTYPE', 'LANG', 'LANGUAGE')): try: # check if it's supported by the _locale module import _locale @@ -625,40 +614,15 @@ def setlocale(category, locale=None): locale = normalize(_build_localename(locale)) return _setlocale(category, locale) -def resetlocale(category=LC_ALL): - - """ Sets the locale for category to the default setting. - - The default setting is determined by calling - getdefaultlocale(). category defaults to LC_ALL. - - """ - import warnings - warnings.warn( - 'Use locale.setlocale(locale.LC_ALL, "") instead', - DeprecationWarning, stacklevel=2 - ) - - with warnings.catch_warnings(): - warnings.simplefilter('ignore', category=DeprecationWarning) - loc = getdefaultlocale() - - _setlocale(category, _build_localename(loc)) - try: from _locale import getencoding except ImportError: + # When _locale.getencoding() is missing, locale.getencoding() uses the + # Python filesystem encoding. def getencoding(): - if hasattr(sys, 'getandroidapilevel'): - # On Android langinfo.h and CODESET are missing, and UTF-8 is - # always used in mbstowcs() and wcstombs(). - return 'utf-8' - encoding = getdefaultlocale()[1] - if encoding is None: - # LANG not set, default to UTF-8 - encoding = 'utf-8' - return encoding + return sys.getfilesystemencoding() + try: CODESET @@ -896,6 +860,28 @@ def getpreferredencoding(do_setlocale=True): # updated 'ca_es@valencia' -> 'ca_ES.ISO8859-15@valencia' to 'ca_ES.UTF-8@valencia' # updated 'kk_kz' -> 'kk_KZ.RK1048' to 'kk_KZ.ptcp154' # updated 'russian' -> 'ru_RU.ISO8859-5' to 'ru_RU.KOI8-R' +# +# SS 2025-02-04: +# Updated alias mapping with glibc 2.41 supported locales and the latest +# X lib alias mapping. +# +# These are the differences compared to the old mapping (Python 3.13.1 +# and older): +# +# updated 'c.utf8' -> 'C.UTF-8' to 'en_US.UTF-8' +# updated 'de_it' -> 'de_IT.ISO8859-1' to 'de_IT.UTF-8' +# removed 'de_li.utf8' +# updated 'en_il' -> 'en_IL.UTF-8' to 'en_IL.ISO8859-1' +# removed 'english.iso88591' +# updated 'es_cu' -> 'es_CU.UTF-8' to 'es_CU.ISO8859-1' +# updated 'russian' -> 'ru_RU.KOI8-R' to 'ru_RU.ISO8859-5' +# updated 'sr@latn' -> 'sr_CS.UTF-8@latin' to 'sr_RS.UTF-8@latin' +# removed 'univ' +# removed 'universal' +# +# SS 2025-06-10: +# Remove 'c.utf8' -> 'en_US.UTF-8' because 'en_US.UTF-8' does not exist +# on all platforms. locale_alias = { 'a3': 'az_AZ.KOI8-C', @@ -975,7 +961,6 @@ def getpreferredencoding(do_setlocale=True): 'c.ascii': 'C', 'c.en': 'C', 'c.iso88591': 'en_US.ISO8859-1', - 'c.utf8': 'en_US.UTF-8', 'c_c': 'C', 'c_c.c': 'C', 'ca': 'ca_ES.ISO8859-1', @@ -992,6 +977,7 @@ def getpreferredencoding(do_setlocale=True): 'chr_us': 'chr_US.UTF-8', 'ckb_iq': 'ckb_IQ.UTF-8', 'cmn_tw': 'cmn_TW.UTF-8', + 'crh_ru': 'crh_RU.UTF-8', 'crh_ua': 'crh_UA.UTF-8', 'croatian': 'hr_HR.ISO8859-2', 'cs': 'cs_CZ.ISO8859-2', @@ -1013,11 +999,12 @@ def getpreferredencoding(do_setlocale=True): 'de_be': 'de_BE.ISO8859-1', 'de_ch': 'de_CH.ISO8859-1', 'de_de': 'de_DE.ISO8859-1', - 'de_it': 'de_IT.ISO8859-1', - 'de_li.utf8': 'de_LI.UTF-8', + 'de_it': 'de_IT.UTF-8', + 'de_li': 'de_LI.ISO8859-1', 'de_lu': 'de_LU.ISO8859-1', 'deutsch': 'de_DE.ISO8859-1', 'doi_in': 'doi_IN.UTF-8', + 'dsb_de': 'dsb_DE.UTF-8', 'dutch': 'nl_NL.ISO8859-1', 'dutch.iso88591': 'nl_BE.ISO8859-1', 'dv_mv': 'dv_MV.UTF-8', @@ -1040,7 +1027,7 @@ def getpreferredencoding(do_setlocale=True): 'en_gb': 'en_GB.ISO8859-1', 'en_hk': 'en_HK.ISO8859-1', 'en_ie': 'en_IE.ISO8859-1', - 'en_il': 'en_IL.UTF-8', + 'en_il': 'en_IL.ISO8859-1', 'en_in': 'en_IN.ISO8859-1', 'en_ng': 'en_NG.UTF-8', 'en_nz': 'en_NZ.ISO8859-1', @@ -1056,7 +1043,6 @@ def getpreferredencoding(do_setlocale=True): 'en_zw.utf8': 'en_ZS.UTF-8', 'eng_gb': 'en_GB.ISO8859-1', 'english': 'en_EN.ISO8859-1', - 'english.iso88591': 'en_US.ISO8859-1', 'english_uk': 'en_GB.ISO8859-1', 'english_united-states': 'en_US.ISO8859-1', 'english_united-states.437': 'C', @@ -1072,7 +1058,7 @@ def getpreferredencoding(do_setlocale=True): 'es_cl': 'es_CL.ISO8859-1', 'es_co': 'es_CO.ISO8859-1', 'es_cr': 'es_CR.ISO8859-1', - 'es_cu': 'es_CU.UTF-8', + 'es_cu': 'es_CU.ISO8859-1', 'es_do': 'es_DO.ISO8859-1', 'es_ec': 'es_EC.ISO8859-1', 'es_es': 'es_ES.ISO8859-1', @@ -1122,6 +1108,7 @@ def getpreferredencoding(do_setlocale=True): 'ga_ie': 'ga_IE.ISO8859-1', 'galego': 'gl_ES.ISO8859-1', 'galician': 'gl_ES.ISO8859-1', + 'gbm_in': 'gbm_IN.UTF-8', 'gd': 'gd_GB.ISO8859-1', 'gd_gb': 'gd_GB.ISO8859-1', 'ger_de': 'de_DE.ISO8859-1', @@ -1162,6 +1149,7 @@ def getpreferredencoding(do_setlocale=True): 'icelandic': 'is_IS.ISO8859-1', 'id': 'id_ID.ISO8859-1', 'id_id': 'id_ID.ISO8859-1', + 'ie': 'ie.UTF-8', 'ig_ng': 'ig_NG.UTF-8', 'ik_ca': 'ik_CA.UTF-8', 'in': 'id_ID.ISO8859-1', @@ -1216,6 +1204,7 @@ def getpreferredencoding(do_setlocale=True): 'ks_in': 'ks_IN.UTF-8', 'ks_in@devanagari.utf8': 'ks_IN.UTF-8@devanagari', 'ku_tr': 'ku_TR.ISO8859-9', + 'kv_ru': 'kv_RU.UTF-8', 'kw': 'kw_GB.ISO8859-1', 'kw_gb': 'kw_GB.ISO8859-1', 'ky': 'ky_KG.UTF-8', @@ -1234,6 +1223,7 @@ def getpreferredencoding(do_setlocale=True): 'lo_la.mulelao1': 'lo_LA.MULELAO-1', 'lt': 'lt_LT.ISO8859-13', 'lt_lt': 'lt_LT.ISO8859-13', + 'ltg_lv.utf8': 'ltg_LV.UTF-8', 'lv': 'lv_LV.ISO8859-13', 'lv_lv': 'lv_LV.ISO8859-13', 'lzh_tw': 'lzh_TW.UTF-8', @@ -1241,6 +1231,7 @@ def getpreferredencoding(do_setlocale=True): 'mai': 'mai_IN.UTF-8', 'mai_in': 'mai_IN.UTF-8', 'mai_np': 'mai_NP.UTF-8', + 'mdf_ru': 'mdf_RU.UTF-8', 'mfe_mu': 'mfe_MU.UTF-8', 'mg_mg': 'mg_MG.ISO8859-15', 'mhr_ru': 'mhr_RU.UTF-8', @@ -1254,6 +1245,7 @@ def getpreferredencoding(do_setlocale=True): 'ml_in': 'ml_IN.UTF-8', 'mn_mn': 'mn_MN.UTF-8', 'mni_in': 'mni_IN.UTF-8', + 'mnw_mm': 'mnw_MM.UTF-8', 'mr': 'mr_IN.UTF-8', 'mr_in': 'mr_IN.UTF-8', 'ms': 'ms_MY.ISO8859-1', @@ -1322,6 +1314,7 @@ def getpreferredencoding(do_setlocale=True): 'pt_pt': 'pt_PT.ISO8859-1', 'quz_pe': 'quz_PE.UTF-8', 'raj_in': 'raj_IN.UTF-8', + 'rif_ma': 'rif_MA.UTF-8', 'ro': 'ro_RO.ISO8859-2', 'ro_ro': 'ro_RO.ISO8859-2', 'romanian': 'ro_RO.ISO8859-2', @@ -1329,12 +1322,14 @@ def getpreferredencoding(do_setlocale=True): 'ru_ru': 'ru_RU.UTF-8', 'ru_ua': 'ru_UA.KOI8-U', 'rumanian': 'ro_RO.ISO8859-2', - 'russian': 'ru_RU.KOI8-R', + 'russian': 'ru_RU.ISO8859-5', 'rw': 'rw_RW.ISO8859-1', 'rw_rw': 'rw_RW.ISO8859-1', 'sa_in': 'sa_IN.UTF-8', + 'sah_ru': 'sah_RU.UTF-8', 'sat_in': 'sat_IN.UTF-8', 'sc_it': 'sc_IT.UTF-8', + 'scn_it': 'scn_IT.UTF-8', 'sd': 'sd_IN.UTF-8', 'sd_in': 'sd_IN.UTF-8', 'sd_in@devanagari.utf8': 'sd_IN.UTF-8@devanagari', @@ -1376,7 +1371,7 @@ def getpreferredencoding(do_setlocale=True): 'sq_mk': 'sq_MK.UTF-8', 'sr': 'sr_RS.UTF-8', 'sr@cyrillic': 'sr_RS.UTF-8', - 'sr@latn': 'sr_CS.UTF-8@latin', + 'sr@latn': 'sr_RS.UTF-8@latin', 'sr_cs': 'sr_CS.UTF-8', 'sr_cs.iso88592@latn': 'sr_CS.ISO8859-2', 'sr_cs@latn': 'sr_CS.UTF-8@latin', @@ -1395,14 +1390,17 @@ def getpreferredencoding(do_setlocale=True): 'sr_yu@cyrillic': 'sr_RS.UTF-8', 'ss': 'ss_ZA.ISO8859-1', 'ss_za': 'ss_ZA.ISO8859-1', + 'ssy_er': 'ssy_ER.UTF-8', 'st': 'st_ZA.ISO8859-1', 'st_za': 'st_ZA.ISO8859-1', + 'su_id': 'su_ID.UTF-8', 'sv': 'sv_SE.ISO8859-1', 'sv_fi': 'sv_FI.ISO8859-1', 'sv_se': 'sv_SE.ISO8859-1', 'sw_ke': 'sw_KE.UTF-8', 'sw_tz': 'sw_TZ.UTF-8', 'swedish': 'sv_SE.ISO8859-1', + 'syr': 'syr.UTF-8', 'szl_pl': 'szl_PL.UTF-8', 'ta': 'ta_IN.TSCII-0', 'ta_in': 'ta_IN.TSCII-0', @@ -1429,6 +1427,7 @@ def getpreferredencoding(do_setlocale=True): 'tn': 'tn_ZA.ISO8859-15', 'tn_za': 'tn_ZA.ISO8859-15', 'to_to': 'to_TO.UTF-8', + 'tok': 'tok.UTF-8', 'tpi_pg': 'tpi_PG.UTF-8', 'tr': 'tr_TR.ISO8859-9', 'tr_cy': 'tr_CY.ISO8859-9', @@ -1443,8 +1442,7 @@ def getpreferredencoding(do_setlocale=True): 'ug_cn': 'ug_CN.UTF-8', 'uk': 'uk_UA.KOI8-U', 'uk_ua': 'uk_UA.KOI8-U', - 'univ': 'en_US.utf', - 'universal': 'en_US.utf', + 'univ.utf8': 'en_US.UTF-8', 'universal.utf8@ucs4': 'en_US.UTF-8', 'unm_us': 'unm_US.UTF-8', 'ur': 'ur_PK.CP1256', @@ -1473,6 +1471,7 @@ def getpreferredencoding(do_setlocale=True): 'yo_ng': 'yo_NG.UTF-8', 'yue_hk': 'yue_HK.UTF-8', 'yuw_pg': 'yuw_PG.UTF-8', + 'zgh_ma': 'zgh_MA.UTF-8', 'zh': 'zh_CN.eucCN', 'zh_cn': 'zh_CN.gb2312', 'zh_cn.big5': 'zh_TW.big5', @@ -1496,7 +1495,8 @@ def getpreferredencoding(do_setlocale=True): # to include every locale up to Windows Vista. # # NOTE: this mapping is incomplete. If your language is missing, please -# submit a bug report to the Python bug tracker at http://bugs.python.org/ +# submit a bug report as detailed in the Python devguide at: +# https://devguide.python.org/triage/issue-tracker/ # Make sure you include the missing language identifier and the suggested # locale code. # @@ -1742,17 +1742,6 @@ def _init_categories(categories=categories): print(' Encoding: ', enc or '(undefined)') print() - print() - print('Locale settings after calling resetlocale():') - print('-'*72) - resetlocale() - for name,category in categories.items(): - print(name, '...') - lang, enc = getlocale(category) - print(' Language: ', lang or '(undefined)') - print(' Encoding: ', enc or '(undefined)') - print() - try: setlocale(LC_ALL, "") except: diff --git a/Lib/logging/__init__.py b/Lib/logging/__init__.py index 28df075dcd..357d127c09 100644 --- a/Lib/logging/__init__.py +++ b/Lib/logging/__init__.py @@ -56,7 +56,7 @@ # #_startTime is used as the base when calculating the relative time of events # -_startTime = time.time() +_startTime = time.time_ns() # #raiseExceptions is used to see if exceptions during handling should be @@ -159,12 +159,9 @@ def addLevelName(level, levelName): This is used when converting levels to text during message formatting. """ - _acquireLock() - try: #unlikely to cause an exception, but you never know... + with _lock: _levelToName[level] = levelName _nameToLevel[levelName] = level - finally: - _releaseLock() if hasattr(sys, "_getframe"): currentframe = lambda: sys._getframe(1) @@ -201,7 +198,7 @@ def _is_internal_frame(frame): """Signal whether the frame is a CPython or logging module internal.""" filename = os.path.normcase(frame.f_code.co_filename) return filename == _srcfile or ( - "importlib" in filename and "_bootstrap" in filename + "importlib" in filename and "_bootstrap" in filename ) @@ -231,21 +228,27 @@ def _checkLevel(level): # _lock = threading.RLock() -def _acquireLock(): +def _prepareFork(): """ - Acquire the module-level lock for serializing access to shared data. + Prepare to fork a new child process by acquiring the module-level lock. - This should be released with _releaseLock(). + This should be used in conjunction with _afterFork(). """ - if _lock: + # Wrap the lock acquisition in a try-except to prevent the lock from being + # abandoned in the event of an asynchronous exception. See gh-106238. + try: _lock.acquire() + except BaseException: + _lock.release() + raise -def _releaseLock(): +def _afterFork(): """ - Release the module-level lock acquired by calling _acquireLock(). + After a new child process has been forked, release the module-level lock. + + This should be used in conjunction with _prepareFork(). """ - if _lock: - _lock.release() + _lock.release() # Prevent a held logging lock from blocking a child from logging. @@ -260,23 +263,20 @@ def _register_at_fork_reinit_lock(instance): _at_fork_reinit_lock_weakset = weakref.WeakSet() def _register_at_fork_reinit_lock(instance): - _acquireLock() - try: + with _lock: _at_fork_reinit_lock_weakset.add(instance) - finally: - _releaseLock() def _after_at_fork_child_reinit_locks(): for handler in _at_fork_reinit_lock_weakset: handler._at_fork_reinit() - # _acquireLock() was called in the parent before forking. + # _prepareFork() was called in the parent before forking. # The lock is reinitialized to unlocked state. _lock._at_fork_reinit() - os.register_at_fork(before=_acquireLock, + os.register_at_fork(before=_prepareFork, after_in_child=_after_at_fork_child_reinit_locks, - after_in_parent=_releaseLock) + after_in_parent=_afterFork) #--------------------------------------------------------------------------- @@ -300,7 +300,7 @@ def __init__(self, name, level, pathname, lineno, """ Initialize a logging record with interesting information. """ - ct = time.time() + ct = time.time_ns() self.name = name self.msg = msg # @@ -322,7 +322,7 @@ def __init__(self, name, level, pathname, lineno, # Thus, while not removing the isinstance check, it does now look # for collections.abc.Mapping rather than, as before, dict. if (args and len(args) == 1 and isinstance(args[0], collections.abc.Mapping) - and args[0]): + and args[0]): args = args[0] self.args = args self.levelname = getLevelName(level) @@ -339,9 +339,17 @@ def __init__(self, name, level, pathname, lineno, self.stack_info = sinfo self.lineno = lineno self.funcName = func - self.created = ct - self.msecs = int((ct - int(ct)) * 1000) + 0.0 # see gh-89047 - self.relativeCreated = (self.created - _startTime) * 1000 + self.created = ct / 1e9 # ns to float seconds + # Get the number of whole milliseconds (0-999) in the fractional part of seconds. + # Eg: 1_677_903_920_999_998_503 ns --> 999_998_503 ns--> 999 ms + # Convert to float by adding 0.0 for historical reasons. See gh-89047 + self.msecs = (ct % 1_000_000_000) // 1_000_000 + 0.0 + if self.msecs == 999.0 and int(self.created) != ct // 1_000_000_000: + # ns -> sec conversion can round up, e.g: + # 1_677_903_920_999_999_900 ns --> 1_677_903_921.0 sec + self.msecs = 0.0 + + self.relativeCreated = (ct - _startTime) / 1e6 if logThreads: self.thread = threading.get_ident() self.threadName = threading.current_thread().name @@ -378,7 +386,7 @@ def __init__(self, name, level, pathname, lineno, def __repr__(self): return ''%(self.name, self.levelno, - self.pathname, self.lineno, self.msg) + self.pathname, self.lineno, self.msg) def getMessage(self): """ @@ -572,7 +580,7 @@ class Formatter(object): %(lineno)d Source line number where the logging call was issued (if available) %(funcName)s Function name - %(created)f Time when the LogRecord was created (time.time() + %(created)f Time when the LogRecord was created (time.time_ns() / 1e9 return value) %(asctime)s Textual time when the LogRecord was created %(msecs)d Millisecond portion of the creation time @@ -583,6 +591,7 @@ class Formatter(object): %(threadName)s Thread name (if available) %(taskName)s Task name (if available) %(process)d Process ID (if available) + %(processName)s Process name (if available) %(message)s The result of record.getMessage(), computed just as the record is emitted """ @@ -608,7 +617,7 @@ def __init__(self, fmt=None, datefmt=None, style='%', validate=True, *, """ if style not in _STYLES: raise ValueError('Style must be one of: %s' % ','.join( - _STYLES.keys())) + _STYLES.keys())) self._style = _STYLES[style][0](fmt, defaults=defaults) if validate: self._style.validate() @@ -658,7 +667,7 @@ def formatException(self, ei): # See issues #9427, #1553375. Commented out for now. #if getattr(self, 'fullstack', False): # traceback.print_stack(tb.tb_frame.f_back, file=sio) - traceback.print_exception(ei[0], ei[1], tb, None, sio) + traceback.print_exception(ei[0], ei[1], tb, limit=None, file=sio) s = sio.getvalue() sio.close() if s[-1:] == "\n": @@ -879,25 +888,20 @@ def _removeHandlerRef(wr): # set to None. It can also be called from another thread. So we need to # pre-emptively grab the necessary globals and check if they're None, # to prevent race conditions and failures during interpreter shutdown. - acquire, release, handlers = _acquireLock, _releaseLock, _handlerList - if acquire and release and handlers: - acquire() - try: - handlers.remove(wr) - except ValueError: - pass - finally: - release() + handlers, lock = _handlerList, _lock + if lock and handlers: + with lock: + try: + handlers.remove(wr) + except ValueError: + pass def _addHandlerRef(handler): """ Add a handler to the internal cleanup list using a weak reference. """ - _acquireLock() - try: + with _lock: _handlerList.append(weakref.ref(handler, _removeHandlerRef)) - finally: - _releaseLock() def getHandlerByName(name): @@ -912,8 +916,7 @@ def getHandlerNames(): """ Return all known handler names as an immutable set. """ - result = set(_handlers.keys()) - return frozenset(result) + return frozenset(_handlers) class Handler(Filterer): @@ -943,15 +946,12 @@ def get_name(self): return self._name def set_name(self, name): - _acquireLock() - try: + with _lock: if self._name in _handlers: del _handlers[self._name] self._name = name if name: _handlers[name] = self - finally: - _releaseLock() name = property(get_name, set_name) @@ -1023,11 +1023,8 @@ def handle(self, record): if isinstance(rv, LogRecord): record = rv if rv: - self.acquire() - try: + with self.lock: self.emit(record) - finally: - self.release() return rv def setFormatter(self, fmt): @@ -1055,13 +1052,10 @@ def close(self): methods. """ #get the module data lock, as we're updating a shared structure. - _acquireLock() - try: #unlikely to raise an exception, but you never know... + with _lock: self._closed = True if self._name and self._name in _handlers: del _handlers[self._name] - finally: - _releaseLock() def handleError(self, record): """ @@ -1076,14 +1070,14 @@ def handleError(self, record): The record which was being processed is passed in to this method. """ if raiseExceptions and sys.stderr: # see issue 13807 - t, v, tb = sys.exc_info() + exc = sys.exception() try: sys.stderr.write('--- Logging error ---\n') - traceback.print_exception(t, v, tb, None, sys.stderr) + traceback.print_exception(exc, limit=None, file=sys.stderr) sys.stderr.write('Call stack:\n') # Walk the stack frame up until we're out of logging, # so as to print the calling context. - frame = tb.tb_frame + frame = exc.__traceback__.tb_frame while (frame and os.path.dirname(frame.f_code.co_filename) == __path__[0]): frame = frame.f_back @@ -1092,7 +1086,7 @@ def handleError(self, record): else: # couldn't find the right stack frame, for some reason sys.stderr.write('Logged from file %s, line %s\n' % ( - record.filename, record.lineno)) + record.filename, record.lineno)) # Issue 18671: output logging message and arguments try: sys.stderr.write('Message: %r\n' @@ -1104,11 +1098,11 @@ def handleError(self, record): sys.stderr.write('Unable to print the message and arguments' ' - possible formatting error.\nUse the' ' traceback above to help find the error.\n' - ) + ) except OSError: #pragma: no cover pass # see issue 5971 finally: - del t, v, tb + del exc def __repr__(self): level = getLevelName(self.level) @@ -1138,12 +1132,9 @@ def flush(self): """ Flushes the stream. """ - self.acquire() - try: + with self.lock: if self.stream and hasattr(self.stream, "flush"): self.stream.flush() - finally: - self.release() def emit(self, record): """ @@ -1179,12 +1170,9 @@ def setStream(self, stream): result = None else: result = self.stream - self.acquire() - try: + with self.lock: self.flush() self.stream = stream - finally: - self.release() return result def __repr__(self): @@ -1234,8 +1222,7 @@ def close(self): """ Closes the stream. """ - self.acquire() - try: + with self.lock: try: if self.stream: try: @@ -1251,8 +1238,6 @@ def close(self): # Also see Issue #42378: we also rely on # self._closed being set to True there StreamHandler.close(self) - finally: - self.release() def _open(self): """ @@ -1388,8 +1373,7 @@ def getLogger(self, name): rv = None if not isinstance(name, str): raise TypeError('A logger name must be a string') - _acquireLock() - try: + with _lock: if name in self.loggerDict: rv = self.loggerDict[name] if isinstance(rv, PlaceHolder): @@ -1404,8 +1388,6 @@ def getLogger(self, name): rv.manager = self self.loggerDict[name] = rv self._fixupParents(rv) - finally: - _releaseLock() return rv def setLoggerClass(self, klass): @@ -1468,12 +1450,11 @@ def _clear_cache(self): Called when level changes are made """ - _acquireLock() - for logger in self.loggerDict.values(): - if isinstance(logger, Logger): - logger._cache.clear() - self.root._cache.clear() - _releaseLock() + with _lock: + for logger in self.loggerDict.values(): + if isinstance(logger, Logger): + logger._cache.clear() + self.root._cache.clear() #--------------------------------------------------------------------------- # Logger classes and functions @@ -1494,6 +1475,8 @@ class Logger(Filterer): level, and "input.csv", "input.xls" and "input.gnu" for the sub-levels. There is no arbitrary limit to the depth of nesting. """ + _tls = threading.local() + def __init__(self, name, level=NOTSET): """ Initialize the logger with a name and an optional level. @@ -1552,7 +1535,7 @@ def warning(self, msg, *args, **kwargs): def warn(self, msg, *args, **kwargs): warnings.warn("The 'warn' method is deprecated, " - "use 'warning' instead", DeprecationWarning, 2) + "use 'warning' instead", DeprecationWarning, 2) self.warning(msg, *args, **kwargs) def error(self, msg, *args, **kwargs): @@ -1649,7 +1632,7 @@ def makeRecord(self, name, level, fn, lno, msg, args, exc_info, specialized LogRecords. """ rv = _logRecordFactory(name, level, fn, lno, msg, args, exc_info, func, - sinfo) + sinfo) if extra is not None: for key in extra: if (key in ["message", "asctime"]) or (key in rv.__dict__): @@ -1690,36 +1673,35 @@ def handle(self, record): This method is used for unpickled records received from a socket, as well as those created locally. Logger-level filtering is applied. """ - if self.disabled: - return - maybe_record = self.filter(record) - if not maybe_record: + if self._is_disabled(): return - if isinstance(maybe_record, LogRecord): - record = maybe_record - self.callHandlers(record) + + self._tls.in_progress = True + try: + maybe_record = self.filter(record) + if not maybe_record: + return + if isinstance(maybe_record, LogRecord): + record = maybe_record + self.callHandlers(record) + finally: + self._tls.in_progress = False def addHandler(self, hdlr): """ Add the specified handler to this logger. """ - _acquireLock() - try: + with _lock: if not (hdlr in self.handlers): self.handlers.append(hdlr) - finally: - _releaseLock() def removeHandler(self, hdlr): """ Remove the specified handler from this logger. """ - _acquireLock() - try: + with _lock: if hdlr in self.handlers: self.handlers.remove(hdlr) - finally: - _releaseLock() def hasHandlers(self): """ @@ -1791,22 +1773,19 @@ def isEnabledFor(self, level): """ Is this logger enabled for level 'level'? """ - if self.disabled: + if self._is_disabled(): return False try: return self._cache[level] except KeyError: - _acquireLock() - try: + with _lock: if self.manager.disable >= level: is_enabled = self._cache[level] = False else: is_enabled = self._cache[level] = ( - level >= self.getEffectiveLevel() + level >= self.getEffectiveLevel() ) - finally: - _releaseLock() return is_enabled def getChild(self, suffix): @@ -1836,16 +1815,18 @@ def _hierlevel(logger): return 1 + logger.name.count('.') d = self.manager.loggerDict - _acquireLock() - try: + with _lock: # exclude PlaceHolders - the last check is to ensure that lower-level # descendants aren't returned - if there are placeholders, a logger's # parent field might point to a grandparent or ancestor thereof. return set(item for item in d.values() if isinstance(item, Logger) and item.parent is self and _hierlevel(item) == 1 + _hierlevel(item.parent)) - finally: - _releaseLock() + + def _is_disabled(self): + # We need to use getattr as it will only be set the first time a log + # message is recorded on any given thread + return self.disabled or getattr(self._tls, 'in_progress', False) def __repr__(self): level = getLevelName(self.getEffectiveLevel()) @@ -1881,7 +1862,7 @@ class LoggerAdapter(object): information in logging output. """ - def __init__(self, logger, extra=None): + def __init__(self, logger, extra=None, merge_extra=False): """ Initialize the adapter with a logger and a dict-like object which provides contextual information. This constructor signature allows @@ -1891,9 +1872,20 @@ def __init__(self, logger, extra=None): following example: adapter = LoggerAdapter(someLogger, dict(p1=v1, p2="v2")) + + By default, LoggerAdapter objects will drop the "extra" argument + passed on the individual log calls to use its own instead. + + Initializing it with merge_extra=True will instead merge both + maps when logging, the individual call extra taking precedence + over the LoggerAdapter instance extra + + .. versionchanged:: 3.13 + The *merge_extra* argument was added. """ self.logger = logger self.extra = extra + self.merge_extra = merge_extra def process(self, msg, kwargs): """ @@ -1905,7 +1897,10 @@ def process(self, msg, kwargs): Normally, you'll only need to override this one method in a LoggerAdapter subclass for your specific needs. """ - kwargs["extra"] = self.extra + if self.merge_extra and "extra" in kwargs: + kwargs["extra"] = {**self.extra, **kwargs["extra"]} + else: + kwargs["extra"] = self.extra return msg, kwargs # @@ -1931,7 +1926,7 @@ def warning(self, msg, *args, **kwargs): def warn(self, msg, *args, **kwargs): warnings.warn("The 'warn' method is deprecated, " - "use 'warning' instead", DeprecationWarning, 2) + "use 'warning' instead", DeprecationWarning, 2) self.warning(msg, *args, **kwargs) def error(self, msg, *args, **kwargs): @@ -2088,8 +2083,7 @@ def basicConfig(**kwargs): """ # Add thread safety in case someone mistakenly calls # basicConfig() from multiple threads - _acquireLock() - try: + with _lock: force = kwargs.pop('force', False) encoding = kwargs.pop('encoding', None) errors = kwargs.pop('errors', 'backslashreplace') @@ -2125,7 +2119,7 @@ def basicConfig(**kwargs): style = kwargs.pop("style", '%') if style not in _STYLES: raise ValueError('Style must be one of: %s' % ','.join( - _STYLES.keys())) + _STYLES.keys())) fs = kwargs.pop("format", _STYLES[style][1]) fmt = Formatter(fs, dfs, style) for h in handlers: @@ -2138,8 +2132,6 @@ def basicConfig(**kwargs): if kwargs: keys = ', '.join(kwargs.keys()) raise ValueError('Unrecognised argument(s): %s' % keys) - finally: - _releaseLock() #--------------------------------------------------------------------------- # Utility functions at module level. @@ -2202,7 +2194,7 @@ def warning(msg, *args, **kwargs): def warn(msg, *args, **kwargs): warnings.warn("The 'warn' function is deprecated, " - "use 'warning' instead", DeprecationWarning, 2) + "use 'warning' instead", DeprecationWarning, 2) warning(msg, *args, **kwargs) def info(msg, *args, **kwargs): diff --git a/Lib/logging/config.py b/Lib/logging/config.py index ef04a35168..190b4f9225 100644 --- a/Lib/logging/config.py +++ b/Lib/logging/config.py @@ -83,15 +83,12 @@ def fileConfig(fname, defaults=None, disable_existing_loggers=True, encoding=Non formatters = _create_formatters(cp) # critical section - logging._acquireLock() - try: + with logging._lock: _clearExistingHandlers() # Handlers add themselves to logging._handlers handlers = _install_handlers(cp, formatters) _install_loggers(cp, handlers, disable_existing_loggers) - finally: - logging._releaseLock() def _resolve(name): @@ -314,7 +311,7 @@ def convert_with_key(self, key, value, replace=True): if replace: self[key] = result if type(result) in (ConvertingDict, ConvertingList, - ConvertingTuple): + ConvertingTuple): result.parent = self result.key = key return result @@ -323,7 +320,7 @@ def convert(self, value): result = self.configurator.convert(value) if value is not result: if type(result) in (ConvertingDict, ConvertingList, - ConvertingTuple): + ConvertingTuple): result.parent = self return result @@ -378,7 +375,7 @@ class BaseConfigurator(object): WORD_PATTERN = re.compile(r'^\s*(\w+)\s*') DOT_PATTERN = re.compile(r'^\.\s*(\w+)\s*') - INDEX_PATTERN = re.compile(r'^\[\s*(\w+)\s*\]\s*') + INDEX_PATTERN = re.compile(r'^\[([^\[\]]*)\]\s*') DIGIT_PATTERN = re.compile(r'^\d+$') value_converters = { @@ -464,8 +461,8 @@ def convert(self, value): elif not isinstance(value, ConvertingList) and isinstance(value, list): value = ConvertingList(value) value.configurator = self - elif not isinstance(value, ConvertingTuple) and \ - isinstance(value, tuple) and not hasattr(value, '_fields'): + elif not isinstance(value, ConvertingTuple) and\ + isinstance(value, tuple) and not hasattr(value, '_fields'): value = ConvertingTuple(value) value.configurator = self elif isinstance(value, str): # str for py3k @@ -543,8 +540,7 @@ def configure(self): raise ValueError("Unsupported version: %s" % config['version']) incremental = config.pop('incremental', False) EMPTY_DICT = {} - logging._acquireLock() - try: + with logging._lock: if incremental: handlers = config.get('handlers', EMPTY_DICT) for name in handlers: @@ -585,7 +581,7 @@ def configure(self): for name in formatters: try: formatters[name] = self.configure_formatter( - formatters[name]) + formatters[name]) except Exception as e: raise ValueError('Unable to configure ' 'formatter %r' % name) from e @@ -688,8 +684,6 @@ def configure(self): except Exception as e: raise ValueError('Unable to configure root ' 'logger') from e - finally: - logging._releaseLock() def configure_formatter(self, config): """Configure a formatter from a dictionary.""" @@ -700,10 +694,9 @@ def configure_formatter(self, config): except TypeError as te: if "'format'" not in str(te): raise - #Name of parameter changed from fmt to format. - #Retry with old name. - #This is so that code can be used with older Python versions - #(e.g. by Django) + # logging.Formatter and its subclasses expect the `fmt` + # parameter instead of `format`. Retry passing configuration + # with `fmt`. config['fmt'] = config.pop('format') config['()'] = factory result = self.configure_custom(config) @@ -812,7 +805,7 @@ def configure_handler(self, config): elif issubclass(klass, logging.handlers.QueueHandler): # Another special case for handler which refers to other handlers # if 'handlers' not in config: - # raise ValueError('No handlers specified for a QueueHandler') + # raise ValueError('No handlers specified for a QueueHandler') if 'queue' in config: qspec = config['queue'] @@ -836,8 +829,8 @@ def configure_handler(self, config): else: if isinstance(lspec, str): listener = self.resolve(lspec) - if isinstance(listener, type) and \ - not issubclass(listener, logging.handlers.QueueListener): + if isinstance(listener, type) and\ + not issubclass(listener, logging.handlers.QueueListener): raise TypeError('Invalid listener specifier %r' % lspec) elif isinstance(lspec, dict): if '()' not in lspec: @@ -861,11 +854,11 @@ def configure_handler(self, config): except Exception as e: raise ValueError('Unable to set required handler %r' % hn) from e config['handlers'] = hlist - elif issubclass(klass, logging.handlers.SMTPHandler) and \ - 'mailhost' in config: + elif issubclass(klass, logging.handlers.SMTPHandler) and\ + 'mailhost' in config: config['mailhost'] = self.as_tuple(config['mailhost']) - elif issubclass(klass, logging.handlers.SysLogHandler) and \ - 'address' in config: + elif issubclass(klass, logging.handlers.SysLogHandler) and\ + 'address' in config: config['address'] = self.as_tuple(config['address']) if issubclass(klass, logging.handlers.QueueHandler): factory = functools.partial(self._configure_queue_handler, klass) @@ -1018,9 +1011,8 @@ class ConfigSocketReceiver(ThreadingTCPServer): def __init__(self, host='localhost', port=DEFAULT_LOGGING_CONFIG_PORT, handler=None, ready=None, verify=None): ThreadingTCPServer.__init__(self, (host, port), handler) - logging._acquireLock() - self.abort = 0 - logging._releaseLock() + with logging._lock: + self.abort = 0 self.timeout = 1 self.ready = ready self.verify = verify @@ -1034,9 +1026,8 @@ def serve_until_stopped(self): self.timeout) if rd: self.handle_request() - logging._acquireLock() - abort = self.abort - logging._releaseLock() + with logging._lock: + abort = self.abort self.server_close() class Server(threading.Thread): @@ -1057,9 +1048,8 @@ def run(self): self.port = server.server_address[1] self.ready.set() global _listener - logging._acquireLock() - _listener = server - logging._releaseLock() + with logging._lock: + _listener = server server.serve_until_stopped() return Server(ConfigSocketReceiver, ConfigStreamHandler, port, verify) @@ -1069,10 +1059,7 @@ def stopListening(): Stop the listening server which was created with a call to listen(). """ global _listener - logging._acquireLock() - try: + with logging._lock: if _listener: _listener.abort = 1 _listener = None - finally: - logging._releaseLock() diff --git a/Lib/logging/handlers.py b/Lib/logging/handlers.py index bf42ea1103..d3ea06c731 100644 --- a/Lib/logging/handlers.py +++ b/Lib/logging/handlers.py @@ -23,11 +23,17 @@ To use, simply 'import logging.handlers' and log away! """ -import io, logging, socket, os, pickle, struct, time, re -from stat import ST_DEV, ST_INO, ST_MTIME +import copy +import io +import logging +import os +import pickle import queue +import re +import socket +import struct import threading -import copy +import time # # Some constants... @@ -272,7 +278,7 @@ def __init__(self, filename, when='h', interval=1, backupCount=0, # path object (see Issue #27493), but self.baseFilename will be a string filename = self.baseFilename if os.path.exists(filename): - t = os.stat(filename)[ST_MTIME] + t = int(os.stat(filename).st_mtime) else: t = int(time.time()) self.rolloverAt = self.computeRollover(t) @@ -304,10 +310,10 @@ def computeRollover(self, currentTime): rotate_ts = _MIDNIGHT else: rotate_ts = ((self.atTime.hour * 60 + self.atTime.minute)*60 + - self.atTime.second) + self.atTime.second) r = rotate_ts - ((currentHour * 60 + currentMinute) * 60 + - currentSecond) + currentSecond) if r <= 0: # Rotate time is before the current time (for example when # self.rotateAt is 13:45 and it now 14:15), rotation is @@ -465,8 +471,7 @@ class WatchedFileHandler(logging.FileHandler): This handler is not appropriate for use under Windows, because under Windows open files cannot be moved or renamed - logging opens the files with exclusive locks - and so there is no need - for such a handler. Furthermore, ST_INO is not supported under - Windows; stat always returns zero for this value. + for such a handler. This handler is based on a suggestion and patch by Chad J. Schroeder. @@ -482,9 +487,11 @@ def __init__(self, filename, mode='a', encoding=None, delay=False, self._statstream() def _statstream(self): - if self.stream: - sres = os.fstat(self.stream.fileno()) - self.dev, self.ino = sres[ST_DEV], sres[ST_INO] + if self.stream is None: + return + sres = os.fstat(self.stream.fileno()) + self.dev = sres.st_dev + self.ino = sres.st_ino def reopenIfNeeded(self): """ @@ -494,6 +501,9 @@ def reopenIfNeeded(self): has, close the old stream and reopen the file to get the current stream. """ + if self.stream is None: + return + # Reduce the chance of race conditions by stat'ing by path only # once and then fstat'ing our new fd if we opened a new log stream. # See issue #14632: Thanks to John Mulligan for the problem report @@ -501,18 +511,23 @@ def reopenIfNeeded(self): try: # stat the file by path, checking for existence sres = os.stat(self.baseFilename) + + # compare file system stat with that of our stream file handle + reopen = (sres.st_dev != self.dev or sres.st_ino != self.ino) except FileNotFoundError: - sres = None - # compare file system stat with that of our stream file handle - if not sres or sres[ST_DEV] != self.dev or sres[ST_INO] != self.ino: - if self.stream is not None: - # we have an open file handle, clean it up - self.stream.flush() - self.stream.close() - self.stream = None # See Issue #21742: _open () might fail. - # open a new file handle and get new stat info from that fd - self.stream = self._open() - self._statstream() + reopen = True + + if not reopen: + return + + # we have an open file handle, clean it up + self.stream.flush() + self.stream.close() + self.stream = None # See Issue #21742: _open () might fail. + + # open a new file handle and get new stat info from that fd + self.stream = self._open() + self._statstream() def emit(self, record): """ @@ -682,15 +697,12 @@ def close(self): """ Closes the socket. """ - self.acquire() - try: + with self.lock: sock = self.sock if sock: self.sock = None sock.close() logging.Handler.close(self) - finally: - self.release() class DatagramHandler(SocketHandler): """ @@ -803,7 +815,7 @@ class SysLogHandler(logging.Handler): "panic": LOG_EMERG, # DEPRECATED "warn": LOG_WARNING, # DEPRECATED "warning": LOG_WARNING, - } + } facility_names = { "auth": LOG_AUTH, @@ -830,7 +842,7 @@ class SysLogHandler(logging.Handler): "local5": LOG_LOCAL5, "local6": LOG_LOCAL6, "local7": LOG_LOCAL7, - } + } # Originally added to work around GH-43683. Unnecessary since GH-50043 but kept # for backwards compatibility. @@ -950,15 +962,12 @@ def close(self): """ Closes the socket. """ - self.acquire() - try: + with self.lock: sock = self.socket if sock: self.socket = None sock.close() logging.Handler.close(self) - finally: - self.release() def mapPriority(self, levelName): """ @@ -1031,7 +1040,8 @@ def __init__(self, mailhost, fromaddr, toaddrs, subject, only be used when authentication credentials are supplied. The tuple will be either an empty tuple, or a single-value tuple with the name of a keyfile, or a 2-value tuple with the names of the keyfile and - certificate file. (This tuple is passed to the `starttls` method). + certificate file. (This tuple is passed to the + `ssl.SSLContext.load_cert_chain` method). A timeout in seconds can be specified for the SMTP connection (the default is one second). """ @@ -1084,8 +1094,23 @@ def emit(self, record): msg.set_content(self.format(record)) if self.username: if self.secure is not None: + import ssl + + try: + keyfile = self.secure[0] + except IndexError: + keyfile = None + + try: + certfile = self.secure[1] + except IndexError: + certfile = None + + context = ssl._create_stdlib_context( + certfile=certfile, keyfile=keyfile + ) smtp.ehlo() - smtp.starttls(*self.secure) + smtp.starttls(context=context) smtp.ehlo() smtp.login(self.username, self.password) smtp.send_message(msg) @@ -1132,10 +1157,10 @@ def __init__(self, appname, dllname=None, logtype="Application"): logging.WARNING : win32evtlog.EVENTLOG_WARNING_TYPE, logging.ERROR : win32evtlog.EVENTLOG_ERROR_TYPE, logging.CRITICAL: win32evtlog.EVENTLOG_ERROR_TYPE, - } + } except ImportError: - print("The Python Win32 extensions for NT (service, event " \ - "logging) appear not to be available.") + print("The Python Win32 extensions for NT (service, event "\ + "logging) appear not to be available.") self._welu = None def getMessageID(self, record): @@ -1330,11 +1355,8 @@ def flush(self): This version just zaps the buffer to empty. """ - self.acquire() - try: + with self.lock: self.buffer.clear() - finally: - self.release() def close(self): """ @@ -1378,17 +1400,14 @@ def shouldFlush(self, record): Check for buffer full or a record at the flushLevel or higher. """ return (len(self.buffer) >= self.capacity) or \ - (record.levelno >= self.flushLevel) + (record.levelno >= self.flushLevel) def setTarget(self, target): """ Set the target handler for this handler. """ - self.acquire() - try: + with self.lock: self.target = target - finally: - self.release() def flush(self): """ @@ -1398,14 +1417,11 @@ def flush(self): The record buffer is only cleared if a target has been set. """ - self.acquire() - try: + with self.lock: if self.target: for record in self.buffer: self.target.handle(record) self.buffer.clear() - finally: - self.release() def close(self): """ @@ -1416,12 +1432,9 @@ def close(self): if self.flushOnClose: self.flush() finally: - self.acquire() - try: + with self.lock: self.target = None BufferingHandler.close(self) - finally: - self.release() class QueueHandler(logging.Handler): @@ -1532,6 +1545,9 @@ def start(self): This starts up a background thread to monitor the queue for LogRecords to process. """ + if self._thread is not None: + raise RuntimeError("Listener already started") + self._thread = t = threading.Thread(target=self._monitor) t.daemon = True t.start() @@ -1603,6 +1619,7 @@ def stop(self): Note that if you don't call this before your application exits, there may be some records still left on the queue, which won't be processed. """ - self.enqueue_sentinel() - self._thread.join() - self._thread = None + if self._thread: # see gh-114706 - allow calling this more than once + self.enqueue_sentinel() + self._thread.join() + self._thread = None diff --git a/Lib/lzma.py b/Lib/lzma.py index 6668921f00..c1e3d33deb 100644 --- a/Lib/lzma.py +++ b/Lib/lzma.py @@ -128,7 +128,7 @@ def __init__(self, filename=None, mode="r", *, if self._mode == _MODE_READ: raw = _compression.DecompressReader(self._fp, LZMADecompressor, - trailing_error=LZMAError, format=format, filters=filters) + trailing_error=LZMAError, format=format, filters=filters) self._buffer = io.BufferedReader(raw) def close(self): diff --git a/Lib/multiprocessing/connection.py b/Lib/multiprocessing/connection.py index d0582e3cd5..8caddd204d 100644 --- a/Lib/multiprocessing/connection.py +++ b/Lib/multiprocessing/connection.py @@ -19,7 +19,6 @@ import tempfile import itertools -import _multiprocessing from . import util @@ -28,6 +27,7 @@ _ForkingPickler = reduction.ForkingPickler try: + import _multiprocessing import _winapi from _winapi import WAIT_OBJECT_0, WAIT_ABANDONED_0, WAIT_TIMEOUT, INFINITE except ImportError: @@ -846,7 +846,7 @@ def PipeClient(address): _LEGACY_LENGTHS = (_MD5ONLY_MESSAGE_LENGTH, _MD5_DIGEST_LEN) -def _get_digest_name_and_payload(message: bytes) -> (str, bytes): +def _get_digest_name_and_payload(message): # type: (bytes) -> tuple[str, bytes] """Returns a digest name and the payload for a response hash. If a legacy protocol is detected based on the message length @@ -956,7 +956,7 @@ def answer_challenge(connection, authkey: bytes): f'Protocol error, expected challenge: {message=}') message = message[len(_CHALLENGE):] if len(message) < _MD5ONLY_MESSAGE_LENGTH: - raise AuthenticationError('challenge too short: {len(message)} bytes') + raise AuthenticationError(f'challenge too short: {len(message)} bytes') digest = _create_response(authkey, message) connection.send_bytes(digest) response = connection.recv_bytes(256) # reject large message @@ -1012,8 +1012,20 @@ def _exhaustive_wait(handles, timeout): # returning the first signalled might create starvation issues.) L = list(handles) ready = [] + # Windows limits WaitForMultipleObjects at 64 handles, and we use a + # few for synchronisation, so we switch to batched waits at 60. + if len(L) > 60: + try: + res = _winapi.BatchedWaitForMultipleObjects(L, False, timeout) + except TimeoutError: + return [] + ready.extend(L[i] for i in res) + if res: + L = [h for i, h in enumerate(L) if i > res[0] & i not in res] + timeout = 0 while L: - res = _winapi.WaitForMultipleObjects(L, False, timeout) + short_L = L[:60] if len(L) > 60 else L + res = _winapi.WaitForMultipleObjects(short_L, False, timeout) if res == WAIT_TIMEOUT: break elif WAIT_OBJECT_0 <= res < WAIT_OBJECT_0 + len(L): diff --git a/Lib/multiprocessing/context.py b/Lib/multiprocessing/context.py index de8a264829..f395e8b04d 100644 --- a/Lib/multiprocessing/context.py +++ b/Lib/multiprocessing/context.py @@ -145,7 +145,7 @@ def freeze_support(self): '''Check whether this is a fake forked process in a frozen executable. If so then run code specified by commandline and exit. ''' - if sys.platform == 'win32' and getattr(sys, 'frozen', False): + if self.get_start_method() == 'spawn' and getattr(sys, 'frozen', False): from .spawn import freeze_support freeze_support() diff --git a/Lib/multiprocessing/forkserver.py b/Lib/multiprocessing/forkserver.py index 4642707dae..e243442e7a 100644 --- a/Lib/multiprocessing/forkserver.py +++ b/Lib/multiprocessing/forkserver.py @@ -1,3 +1,4 @@ +import atexit import errno import os import selectors @@ -126,12 +127,13 @@ def ensure_running(self): cmd = ('from multiprocessing.forkserver import main; ' + 'main(%d, %d, %r, **%r)') + main_kws = {} if self._preload_modules: - desired_keys = {'main_path', 'sys_path'} data = spawn.get_preparation_data('ignore') - data = {x: y for x, y in data.items() if x in desired_keys} - else: - data = {} + if 'sys_path' in data: + main_kws['sys_path'] = data['sys_path'] + if 'init_main_from_path' in data: + main_kws['main_path'] = data['init_main_from_path'] with socket.socket(socket.AF_UNIX) as listener: address = connection.arbitrary_address('AF_UNIX') @@ -146,7 +148,7 @@ def ensure_running(self): try: fds_to_pass = [listener.fileno(), alive_r] cmd %= (listener.fileno(), alive_r, self._preload_modules, - data) + main_kws) exe = spawn.get_executable() args = [exe] + util._args_from_interpreter_flags() args += ['-c', cmd] @@ -167,6 +169,8 @@ def ensure_running(self): def main(listener_fd, alive_r, preload, main_path=None, sys_path=None): '''Run forkserver.''' if preload: + if sys_path is not None: + sys.path[:] = sys_path if '__main__' in preload and main_path is not None: process.current_process()._inheriting = True try: @@ -179,6 +183,10 @@ def main(listener_fd, alive_r, preload, main_path=None, sys_path=None): except ImportError: pass + # gh-135335: flush stdout/stderr in case any of the preloaded modules + # wrote to them, otherwise children might inherit buffered data + util._flush_std_streams() + util._close_stdin() sig_r, sig_w = os.pipe() @@ -271,6 +279,8 @@ def sigchld_handler(*_unused): selector.close() unused_fds = [alive_r, child_w, sig_r, sig_w] unused_fds.extend(pid_to_fd.values()) + atexit._clear() + atexit.register(util._exit_function) code = _serve_one(child_r, fds, unused_fds, old_handlers) @@ -278,6 +288,7 @@ def sigchld_handler(*_unused): sys.excepthook(*sys.exc_info()) sys.stderr.flush() finally: + atexit._run_exitfuncs() os._exit(code) else: # Send pid to client process diff --git a/Lib/multiprocessing/managers.py b/Lib/multiprocessing/managers.py index 75d9c18c20..ef791c2751 100644 --- a/Lib/multiprocessing/managers.py +++ b/Lib/multiprocessing/managers.py @@ -90,7 +90,10 @@ def dispatch(c, id, methodname, args=(), kwds={}): kind, result = c.recv() if kind == '#RETURN': return result - raise convert_to_error(kind, result) + try: + raise convert_to_error(kind, result) + finally: + del result # break reference cycle def convert_to_error(kind, result): if kind == '#ERROR': @@ -755,22 +758,29 @@ class BaseProxy(object): _address_to_local = {} _mutex = util.ForkAwareThreadLock() + # Each instance gets a `_serial` number. Unlike `id(...)`, this number + # is never reused. + _next_serial = 1 + def __init__(self, token, serializer, manager=None, authkey=None, exposed=None, incref=True, manager_owned=False): with BaseProxy._mutex: - tls_idset = BaseProxy._address_to_local.get(token.address, None) - if tls_idset is None: - tls_idset = util.ForkAwareLocal(), ProcessLocalSet() - BaseProxy._address_to_local[token.address] = tls_idset + tls_serials = BaseProxy._address_to_local.get(token.address, None) + if tls_serials is None: + tls_serials = util.ForkAwareLocal(), ProcessLocalSet() + BaseProxy._address_to_local[token.address] = tls_serials + + self._serial = BaseProxy._next_serial + BaseProxy._next_serial += 1 # self._tls is used to record the connection used by this # thread to communicate with the manager at token.address - self._tls = tls_idset[0] + self._tls = tls_serials[0] - # self._idset is used to record the identities of all shared - # objects for which the current process owns references and + # self._all_serials is a set used to record the identities of all + # shared objects for which the current process owns references and # which are in the manager at token.address - self._idset = tls_idset[1] + self._all_serials = tls_serials[1] self._token = token self._id = self._token.id @@ -833,7 +843,10 @@ def _callmethod(self, methodname, args=(), kwds={}): conn = self._Client(token.address, authkey=self._authkey) dispatch(conn, None, 'decref', (token.id,)) return proxy - raise convert_to_error(kind, result) + try: + raise convert_to_error(kind, result) + finally: + del result # break reference cycle def _getvalue(self): ''' @@ -850,20 +863,20 @@ def _incref(self): dispatch(conn, None, 'incref', (self._id,)) util.debug('INCREF %r', self._token.id) - self._idset.add(self._id) + self._all_serials.add(self._serial) state = self._manager and self._manager._state self._close = util.Finalize( self, BaseProxy._decref, - args=(self._token, self._authkey, state, - self._tls, self._idset, self._Client), + args=(self._token, self._serial, self._authkey, state, + self._tls, self._all_serials, self._Client), exitpriority=10 ) @staticmethod - def _decref(token, authkey, state, tls, idset, _Client): - idset.discard(token.id) + def _decref(token, serial, authkey, state, tls, idset, _Client): + idset.discard(serial) # check whether manager is still alive if state is None or state.value == State.STARTED: @@ -1159,15 +1172,19 @@ def __imul__(self, value): self._callmethod('__imul__', (value,)) return self + __class_getitem__ = classmethod(types.GenericAlias) -DictProxy = MakeProxyType('DictProxy', ( + +_BaseDictProxy = MakeProxyType('DictProxy', ( '__contains__', '__delitem__', '__getitem__', '__iter__', '__len__', '__setitem__', 'clear', 'copy', 'get', 'items', 'keys', 'pop', 'popitem', 'setdefault', 'update', 'values' )) -DictProxy._method_to_typeid_ = { +_BaseDictProxy._method_to_typeid_ = { '__iter__': 'Iterator', } +class DictProxy(_BaseDictProxy): + __class_getitem__ = classmethod(types.GenericAlias) ArrayProxy = MakeProxyType('ArrayProxy', ( diff --git a/Lib/multiprocessing/pool.py b/Lib/multiprocessing/pool.py index 4f5d88cb97..f979890170 100644 --- a/Lib/multiprocessing/pool.py +++ b/Lib/multiprocessing/pool.py @@ -200,7 +200,7 @@ def __init__(self, processes=None, initializer=None, initargs=(), self._initargs = initargs if processes is None: - processes = os.cpu_count() or 1 + processes = os.process_cpu_count() or 1 if processes < 1: raise ValueError("Number of processes must be at least 1") if maxtasksperchild is not None: diff --git a/Lib/multiprocessing/popen_fork.py b/Lib/multiprocessing/popen_fork.py index 625981cf47..a57ef6bdad 100644 --- a/Lib/multiprocessing/popen_fork.py +++ b/Lib/multiprocessing/popen_fork.py @@ -1,3 +1,4 @@ +import atexit import os import signal @@ -66,10 +67,13 @@ def _launch(self, process_obj): self.pid = os.fork() if self.pid == 0: try: + atexit._clear() + atexit.register(util._exit_function) os.close(parent_r) os.close(parent_w) code = process_obj._bootstrap(parent_sentinel=child_r) finally: + atexit._run_exitfuncs() os._exit(code) else: os.close(child_w) diff --git a/Lib/multiprocessing/popen_spawn_posix.py b/Lib/multiprocessing/popen_spawn_posix.py index 24b8634523..cccd659ae7 100644 --- a/Lib/multiprocessing/popen_spawn_posix.py +++ b/Lib/multiprocessing/popen_spawn_posix.py @@ -57,6 +57,10 @@ def _launch(self, process_obj): self._fds.extend([child_r, child_w]) self.pid = util.spawnv_passfds(spawn.get_executable(), cmd, self._fds) + os.close(child_r) + child_r = None + os.close(child_w) + child_w = None self.sentinel = parent_r with open(parent_w, 'wb', closefd=False) as f: f.write(fp.getbuffer()) diff --git a/Lib/multiprocessing/popen_spawn_win32.py b/Lib/multiprocessing/popen_spawn_win32.py index 49d4c7eea2..62fb0ddbf9 100644 --- a/Lib/multiprocessing/popen_spawn_win32.py +++ b/Lib/multiprocessing/popen_spawn_win32.py @@ -3,6 +3,7 @@ import signal import sys import _winapi +from subprocess import STARTUPINFO, STARTF_FORCEOFFFEEDBACK from .context import reduction, get_spawning_popen, set_spawning_popen from . import spawn @@ -74,7 +75,8 @@ def __init__(self, process_obj): try: hp, ht, pid, tid = _winapi.CreateProcess( python_exe, cmd, - None, None, False, 0, env, None, None) + None, None, False, 0, env, None, + STARTUPINFO(dwFlags=STARTF_FORCEOFFFEEDBACK)) _winapi.CloseHandle(ht) except: _winapi.CloseHandle(rhandle) diff --git a/Lib/multiprocessing/process.py b/Lib/multiprocessing/process.py index 271ba3fd32..b45f7df476 100644 --- a/Lib/multiprocessing/process.py +++ b/Lib/multiprocessing/process.py @@ -310,11 +310,8 @@ def _bootstrap(self, parent_sentinel=None): # _run_after_forkers() is executed del old_process util.info('child process calling self.run()') - try: - self.run() - exitcode = 0 - finally: - util._exit_function() + self.run() + exitcode = 0 except SystemExit as e: if e.code is None: exitcode = 0 diff --git a/Lib/multiprocessing/queues.py b/Lib/multiprocessing/queues.py index 852ae87b27..925f043900 100644 --- a/Lib/multiprocessing/queues.py +++ b/Lib/multiprocessing/queues.py @@ -20,8 +20,6 @@ from queue import Empty, Full -import _multiprocessing - from . import connection from . import context _ForkingPickler = context.reduction.ForkingPickler diff --git a/Lib/multiprocessing/resource_tracker.py b/Lib/multiprocessing/resource_tracker.py index 79e96ecf32..05633ac21a 100644 --- a/Lib/multiprocessing/resource_tracker.py +++ b/Lib/multiprocessing/resource_tracker.py @@ -29,8 +29,12 @@ _HAVE_SIGMASK = hasattr(signal, 'pthread_sigmask') _IGNORED_SIGNALS = (signal.SIGINT, signal.SIGTERM) +def cleanup_noop(name): + raise RuntimeError('noop should never be registered or cleaned up') + _CLEANUP_FUNCS = { - 'noop': lambda: None, + 'noop': cleanup_noop, + 'dummy': lambda name: None, # Dummy resource used in tests } if os.name == 'posix': @@ -61,6 +65,7 @@ def __init__(self): self._lock = threading.RLock() self._fd = None self._pid = None + self._exitcode = None def _reentrant_call_error(self): # gh-109629: this happens if an explicit call to the ResourceTracker @@ -70,22 +75,53 @@ def _reentrant_call_error(self): raise ReentrantCallError( "Reentrant call into the multiprocessing resource tracker") - def _stop(self): - with self._lock: - # This should not happen (_stop() isn't called by a finalizer) - # but we check for it anyway. - if self._lock._recursion_count() > 1: - return self._reentrant_call_error() - if self._fd is None: - # not running - return + def __del__(self): + # making sure child processess are cleaned before ResourceTracker + # gets destructed. + # see https://github.com/python/cpython/issues/88887 + self._stop(use_blocking_lock=False) - # closing the "alive" file descriptor stops main() - os.close(self._fd) - self._fd = None + def _stop(self, use_blocking_lock=True): + if use_blocking_lock: + with self._lock: + self._stop_locked() + else: + acquired = self._lock.acquire(blocking=False) + try: + self._stop_locked() + finally: + if acquired: + self._lock.release() + + def _stop_locked( + self, + close=os.close, + waitpid=os.waitpid, + waitstatus_to_exitcode=os.waitstatus_to_exitcode, + ): + # This shouldn't happen (it might when called by a finalizer) + # so we check for it anyway. + if self._lock._recursion_count() > 1: + return self._reentrant_call_error() + if self._fd is None: + # not running + return + if self._pid is None: + return + + # closing the "alive" file descriptor stops main() + close(self._fd) + self._fd = None - os.waitpid(self._pid, 0) - self._pid = None + _, status = waitpid(self._pid, 0) + + self._pid = None + + try: + self._exitcode = waitstatus_to_exitcode(status) + except ValueError: + # os.waitstatus_to_exitcode may raise an exception for invalid values + self._exitcode = None def getfd(self): self.ensure_running() @@ -119,6 +155,7 @@ def ensure_running(self): pass self._fd = None self._pid = None + self._exitcode = None warnings.warn('resource_tracker: process died unexpectedly, ' 'relaunching. Some resources might leak.') @@ -142,13 +179,14 @@ def ensure_running(self): # that can make the child die before it registers signal handlers # for SIGINT and SIGTERM. The mask is unregistered after spawning # the child. + prev_sigmask = None try: if _HAVE_SIGMASK: - signal.pthread_sigmask(signal.SIG_BLOCK, _IGNORED_SIGNALS) + prev_sigmask = signal.pthread_sigmask(signal.SIG_BLOCK, _IGNORED_SIGNALS) pid = util.spawnv_passfds(exe, args, fds_to_pass) finally: - if _HAVE_SIGMASK: - signal.pthread_sigmask(signal.SIG_UNBLOCK, _IGNORED_SIGNALS) + if prev_sigmask is not None: + signal.pthread_sigmask(signal.SIG_SETMASK, prev_sigmask) except: os.close(w) raise @@ -221,6 +259,8 @@ def main(fd): pass cache = {rtype: set() for rtype in _CLEANUP_FUNCS.keys()} + exit_code = 0 + try: # keep track of registered/unregistered resources with open(fd, 'rb') as f: @@ -242,6 +282,7 @@ def main(fd): else: raise RuntimeError('unrecognized command %r' % cmd) except Exception: + exit_code = 3 try: sys.excepthook(*sys.exc_info()) except: @@ -251,9 +292,17 @@ def main(fd): for rtype, rtype_cache in cache.items(): if rtype_cache: try: - warnings.warn('resource_tracker: There appear to be %d ' - 'leaked %s objects to clean up at shutdown' % - (len(rtype_cache), rtype)) + exit_code = 1 + if rtype == 'dummy': + # The test 'dummy' resource is expected to leak. + # We skip the warning (and *only* the warning) for it. + pass + else: + warnings.warn( + f'resource_tracker: There appear to be ' + f'{len(rtype_cache)} leaked {rtype} objects to ' + f'clean up at shutdown: {rtype_cache}' + ) except Exception: pass for name in rtype_cache: @@ -264,6 +313,9 @@ def main(fd): try: _CLEANUP_FUNCS[rtype](name) except Exception as e: + exit_code = 2 warnings.warn('resource_tracker: %r: %s' % (name, e)) finally: pass + + sys.exit(exit_code) diff --git a/Lib/multiprocessing/shared_memory.py b/Lib/multiprocessing/shared_memory.py index 9a1e5aa17b..67e70fdc27 100644 --- a/Lib/multiprocessing/shared_memory.py +++ b/Lib/multiprocessing/shared_memory.py @@ -71,8 +71,9 @@ class SharedMemory: _flags = os.O_RDWR _mode = 0o600 _prepend_leading_slash = True if _USE_POSIX else False + _track = True - def __init__(self, name=None, create=False, size=0): + def __init__(self, name=None, create=False, size=0, *, track=True): if not size >= 0: raise ValueError("'size' must be a positive integer") if create: @@ -82,6 +83,7 @@ def __init__(self, name=None, create=False, size=0): if name is None and not self._flags & os.O_EXCL: raise ValueError("'name' can only be None if create=True") + self._track = track if _USE_POSIX: # POSIX Shared Memory @@ -116,8 +118,8 @@ def __init__(self, name=None, create=False, size=0): except OSError: self.unlink() raise - - resource_tracker.register(self._name, "shared_memory") + if self._track: + resource_tracker.register(self._name, "shared_memory") else: @@ -236,12 +238,20 @@ def close(self): def unlink(self): """Requests that the underlying shared memory block be destroyed. - In order to ensure proper cleanup of resources, unlink should be - called once (and only once) across all processes which have access - to the shared memory block.""" + Unlink should be called once (and only once) across all handles + which have access to the shared memory block, even if these + handles belong to different processes. Closing and unlinking may + happen in any order, but trying to access data inside a shared + memory block after unlinking may result in memory errors, + depending on platform. + + This method has no effect on Windows, where the only way to + delete a shared memory block is to close all handles.""" + if _USE_POSIX and self._name: _posixshmem.shm_unlink(self._name) - resource_tracker.unregister(self._name, "shared_memory") + if self._track: + resource_tracker.unregister(self._name, "shared_memory") _encoding = "utf8" diff --git a/Lib/multiprocessing/synchronize.py b/Lib/multiprocessing/synchronize.py index 3ccbfe311c..870c91349b 100644 --- a/Lib/multiprocessing/synchronize.py +++ b/Lib/multiprocessing/synchronize.py @@ -174,7 +174,7 @@ def __repr__(self): name = process.current_process().name if threading.current_thread().name != 'MainThread': name += '|' + threading.current_thread().name - elif self._semlock._get_value() == 1: + elif not self._semlock._is_zero(): name = 'None' elif self._semlock._count() > 0: name = 'SomeOtherThread' @@ -200,7 +200,7 @@ def __repr__(self): if threading.current_thread().name != 'MainThread': name += '|' + threading.current_thread().name count = self._semlock._count() - elif self._semlock._get_value() == 1: + elif not self._semlock._is_zero(): name, count = 'None', 0 elif self._semlock._count() > 0: name, count = 'SomeOtherThread', 'nonzero' @@ -360,7 +360,7 @@ def wait(self, timeout=None): return True return False - def __repr__(self) -> str: + def __repr__(self): set_status = 'set' if self.is_set() else 'unset' return f"<{type(self).__qualname__} at {id(self):#x} {set_status}>" # diff --git a/Lib/multiprocessing/util.py b/Lib/multiprocessing/util.py index 79559823fb..75dde02d88 100644 --- a/Lib/multiprocessing/util.py +++ b/Lib/multiprocessing/util.py @@ -64,8 +64,7 @@ def get_logger(): global _logger import logging - logging._acquireLock() - try: + with logging._lock: if not _logger: _logger = logging.getLogger(LOGGER_NAME) @@ -79,9 +78,6 @@ def get_logger(): atexit._exithandlers.remove((_exit_function, (), {})) atexit._exithandlers.append((_exit_function, (), {})) - finally: - logging._releaseLock() - return _logger def log_to_stderr(level=None): @@ -106,11 +102,7 @@ def log_to_stderr(level=None): # Abstract socket support def _platform_supports_abstract_sockets(): - if sys.platform == "linux": - return True - if hasattr(sys, 'getandroidapilevel'): - return True - return False + return sys.platform in ("linux", "android") def is_abstract_socket_namespace(address): @@ -130,10 +122,7 @@ def is_abstract_socket_namespace(address): # def _remove_temp_dir(rmtree, tempdir): - def onerror(func, path, err_info): - if not issubclass(err_info[0], FileNotFoundError): - raise - rmtree(tempdir, onerror=onerror) + rmtree(tempdir) current_process = process.current_process() # current_process() can be None if the finalizer is called diff --git a/Lib/netrc.py b/Lib/netrc.py index c1358aac6a..bd003e80a4 100644 --- a/Lib/netrc.py +++ b/Lib/netrc.py @@ -2,11 +2,24 @@ # Module and documentation by Eric S. Raymond, 21 Dec 1998 -import os, shlex, stat +import os, stat __all__ = ["netrc", "NetrcParseError"] +def _can_security_check(): + # On WASI, getuid() is indicated as a stub but it may also be missing. + return os.name == 'posix' and hasattr(os, 'getuid') + + +def _getpwuid(uid): + try: + import pwd + return pwd.getpwuid(uid)[0] + except (ImportError, LookupError): + return f'uid {uid}' + + class NetrcParseError(Exception): """Exception raised on syntax errors in the .netrc file.""" def __init__(self, msg, filename=None, lineno=None): @@ -142,18 +155,12 @@ def _parse(self, file, fp, default_netrc): self._security_check(fp, default_netrc, self.hosts[entryname][0]) def _security_check(self, fp, default_netrc, login): - if os.name == 'posix' and default_netrc and login != "anonymous": + if _can_security_check() and default_netrc and login != "anonymous": prop = os.fstat(fp.fileno()) - if prop.st_uid != os.getuid(): - import pwd - try: - fowner = pwd.getpwuid(prop.st_uid)[0] - except KeyError: - fowner = 'uid %s' % prop.st_uid - try: - user = pwd.getpwuid(os.getuid())[0] - except KeyError: - user = 'uid %s' % os.getuid() + current_user_id = os.getuid() + if prop.st_uid != current_user_id: + fowner = _getpwuid(prop.st_uid) + user = _getpwuid(current_user_id) raise NetrcParseError( (f"~/.netrc file owner ({fowner}, {user}) does not match" " current user")) diff --git a/Lib/ntpath.py b/Lib/ntpath.py index df3402d46c..9cdc16480f 100644 --- a/Lib/ntpath.py +++ b/Lib/ntpath.py @@ -19,18 +19,17 @@ import os import sys -import stat import genericpath from genericpath import * - __all__ = ["normcase","isabs","join","splitdrive","splitroot","split","splitext", "basename","dirname","commonprefix","getsize","getmtime", "getatime","getctime", "islink","exists","lexists","isdir","isfile", - "ismount", "expanduser","expandvars","normpath","abspath", - "curdir","pardir","sep","pathsep","defpath","altsep", + "ismount","isreserved","expanduser","expandvars","normpath", + "abspath","curdir","pardir","sep","pathsep","defpath","altsep", "extsep","devnull","realpath","supports_unicode_filenames","relpath", - "samefile", "sameopenfile", "samestat", "commonpath", "isjunction"] + "samefile", "sameopenfile", "samestat", "commonpath", "isjunction", + "isdevdrive", "ALLOW_MISSING"] def _get_bothseps(path): if isinstance(path, bytes): @@ -78,12 +77,6 @@ def normcase(s): return s.replace('/', '\\').lower() -# Return whether a path is absolute. -# Trivial in Posix, harder on Windows. -# For Windows it is absolute if it starts with a slash or backslash (current -# volume), or if a pathname after the volume-letter-and-colon or UNC-resource -# starts with a slash or backslash. - def isabs(s): """Test whether a path is absolute""" s = os.fspath(s) @@ -91,16 +84,15 @@ def isabs(s): sep = b'\\' altsep = b'/' colon_sep = b':\\' + double_sep = b'\\\\' else: sep = '\\' altsep = '/' colon_sep = ':\\' + double_sep = '\\\\' s = s[:3].replace(altsep, sep) # Absolute: UNC, device, and paths with a drive and root. - # LEGACY BUG: isabs("/x") should be false since the path has no drive. - if s.startswith(sep) or s.startswith(colon_sep, 1): - return True - return False + return s.startswith(colon_sep, 1) or s.startswith(double_sep) # Join two (or more) paths. @@ -109,16 +101,14 @@ def join(path, *paths): if isinstance(path, bytes): sep = b'\\' seps = b'\\/' - colon = b':' + colon_seps = b':\\/' else: sep = '\\' seps = '\\/' - colon = ':' + colon_seps = ':\\/' try: - if not paths: - path[:0] + sep #23780: Ensure compatible data type even if p is null. result_drive, result_root, result_path = splitroot(path) - for p in map(os.fspath, paths): + for p in paths: p_drive, p_root, p_path = splitroot(p) if p_root: # Second path is absolute @@ -142,7 +132,7 @@ def join(path, *paths): result_path = result_path + p_path ## add separator between UNC and non-absolute path if (result_path and not result_root and - result_drive and result_drive[-1:] not in colon + seps): + result_drive and result_drive[-1] not in colon_seps): return result_drive + sep + result_path return result_drive + result_root + result_path except (TypeError, AttributeError, BytesWarning): @@ -176,56 +166,52 @@ def splitdrive(p): return drive, root + tail -def splitroot(p): - """Split a pathname into drive, root and tail. The drive is defined - exactly as in splitdrive(). On Windows, the root may be a single path - separator or an empty string. The tail contains anything after the root. - For example: +try: + from nt import _path_splitroot_ex as splitroot +except ImportError: + def splitroot(p): + """Split a pathname into drive, root and tail. - splitroot('//server/share/') == ('//server/share', '/', '') - splitroot('C:/Users/Barney') == ('C:', '/', 'Users/Barney') - splitroot('C:///spam///ham') == ('C:', '/', '//spam///ham') - splitroot('Windows/notepad') == ('', '', 'Windows/notepad') - """ - p = os.fspath(p) - if isinstance(p, bytes): - sep = b'\\' - altsep = b'/' - colon = b':' - unc_prefix = b'\\\\?\\UNC\\' - empty = b'' - else: - sep = '\\' - altsep = '/' - colon = ':' - unc_prefix = '\\\\?\\UNC\\' - empty = '' - normp = p.replace(altsep, sep) - if normp[:1] == sep: - if normp[1:2] == sep: - # UNC drives, e.g. \\server\share or \\?\UNC\server\share - # Device drives, e.g. \\.\device or \\?\device - start = 8 if normp[:8].upper() == unc_prefix else 2 - index = normp.find(sep, start) - if index == -1: - return p, empty, empty - index2 = normp.find(sep, index + 1) - if index2 == -1: - return p, empty, empty - return p[:index2], p[index2:index2 + 1], p[index2 + 1:] + The tail contains anything after the root.""" + p = os.fspath(p) + if isinstance(p, bytes): + sep = b'\\' + altsep = b'/' + colon = b':' + unc_prefix = b'\\\\?\\UNC\\' + empty = b'' else: - # Relative path with root, e.g. \Windows - return empty, p[:1], p[1:] - elif normp[1:2] == colon: - if normp[2:3] == sep: - # Absolute drive-letter path, e.g. X:\Windows - return p[:2], p[2:3], p[3:] + sep = '\\' + altsep = '/' + colon = ':' + unc_prefix = '\\\\?\\UNC\\' + empty = '' + normp = p.replace(altsep, sep) + if normp[:1] == sep: + if normp[1:2] == sep: + # UNC drives, e.g. \\server\share or \\?\UNC\server\share + # Device drives, e.g. \\.\device or \\?\device + start = 8 if normp[:8].upper() == unc_prefix else 2 + index = normp.find(sep, start) + if index == -1: + return p, empty, empty + index2 = normp.find(sep, index + 1) + if index2 == -1: + return p, empty, empty + return p[:index2], p[index2:index2 + 1], p[index2 + 1:] + else: + # Relative path with root, e.g. \Windows + return empty, p[:1], p[1:] + elif normp[1:2] == colon: + if normp[2:3] == sep: + # Absolute drive-letter path, e.g. X:\Windows + return p[:2], p[2:3], p[3:] + else: + # Relative path with drive, e.g. X:Windows + return p[:2], empty, p[2:] else: - # Relative path with drive, e.g. X:Windows - return p[:2], empty, p[2:] - else: - # Relative path, e.g. Windows - return empty, empty, p + # Relative path, e.g. Windows + return empty, empty, p # Split a path in head (everything up to the last '/') and tail (the @@ -277,33 +263,6 @@ def dirname(p): return split(p)[0] -# Is a path a junction? - -if hasattr(os.stat_result, 'st_reparse_tag'): - def isjunction(path): - """Test whether a path is a junction""" - try: - st = os.lstat(path) - except (OSError, ValueError, AttributeError): - return False - return bool(st.st_reparse_tag == stat.IO_REPARSE_TAG_MOUNT_POINT) -else: - def isjunction(path): - """Test whether a path is a junction""" - os.fspath(path) - return False - - -# Being true for dangling symbolic links is also useful. - -def lexists(path): - """Test whether a path exists. Returns True for broken symbolic links""" - try: - st = os.lstat(path) - except (OSError, ValueError): - return False - return True - # Is a path a mount point? # Any drive letter root (eg c:\) # Any share UNC (eg \\server\share) @@ -338,6 +297,40 @@ def ismount(path): return False +_reserved_chars = frozenset( + {chr(i) for i in range(32)} | + {'"', '*', ':', '<', '>', '?', '|', '/', '\\'} +) + +_reserved_names = frozenset( + {'CON', 'PRN', 'AUX', 'NUL', 'CONIN$', 'CONOUT$'} | + {f'COM{c}' for c in '123456789\xb9\xb2\xb3'} | + {f'LPT{c}' for c in '123456789\xb9\xb2\xb3'} +) + +def isreserved(path): + """Return true if the pathname is reserved by the system.""" + # Refer to "Naming Files, Paths, and Namespaces": + # https://docs.microsoft.com/en-us/windows/win32/fileio/naming-a-file + path = os.fsdecode(splitroot(path)[2]).replace(altsep, sep) + return any(_isreservedname(name) for name in reversed(path.split(sep))) + +def _isreservedname(name): + """Return true if the filename is reserved by the system.""" + # Trailing dots and spaces are reserved. + if name[-1:] in ('.', ' '): + return name not in ('.', '..') + # Wildcards, separators, colon, and pipe (*?"<>/\:|) are reserved. + # ASCII control characters (0-31) are reserved. + # Colon is reserved for file streams (e.g. "name:stream[:type]"). + if _reserved_chars.intersection(name): + return True + # DOS device names are reserved (e.g. "nul" or "nul .txt"). The rules + # are complex and vary across Windows versions. On the side of + # caution, return True for names that may not be reserved. + return name.partition('.')[0].rstrip(' ').upper() in _reserved_names + + # Expand paths beginning with '~' or '~user'. # '~' means $HOME; '~user' means that user's home directory. # If the path doesn't begin with '~', or if the user or $HOME is unknown, @@ -353,24 +346,23 @@ def expanduser(path): If user or $HOME is unknown, do nothing.""" path = os.fspath(path) if isinstance(path, bytes): + seps = b'\\/' tilde = b'~' else: + seps = '\\/' tilde = '~' if not path.startswith(tilde): return path i, n = 1, len(path) - while i < n and path[i] not in _get_bothseps(path): + while i < n and path[i] not in seps: i += 1 if 'USERPROFILE' in os.environ: userhome = os.environ['USERPROFILE'] - elif not 'HOMEPATH' in os.environ: + elif 'HOMEPATH' not in os.environ: return path else: - try: - drive = os.environ['HOMEDRIVE'] - except KeyError: - drive = '' + drive = os.environ.get('HOMEDRIVE', '') userhome = join(drive, os.environ['HOMEPATH']) if i != 1: #~user @@ -521,7 +513,7 @@ def expandvars(path): # Previously, this function also truncated pathnames to 8+3 format, # but as this module is called "ntpath", that's obviously wrong! try: - from nt import _path_normpath + from nt import _path_normpath as normpath except ImportError: def normpath(path): @@ -560,37 +552,22 @@ def normpath(path): comps.append(curdir) return prefix + sep.join(comps) -else: - def normpath(path): - """Normalize path, eliminating double slashes, etc.""" - path = os.fspath(path) - if isinstance(path, bytes): - return os.fsencode(_path_normpath(os.fsdecode(path))) or b"." - return _path_normpath(path) or "." - - -def _abspath_fallback(path): - """Return the absolute version of a path as a fallback function in case - `nt._getfullpathname` is not available or raises OSError. See bpo-31047 for - more. - - """ - - path = os.fspath(path) - if not isabs(path): - if isinstance(path, bytes): - cwd = os.getcwdb() - else: - cwd = os.getcwd() - path = join(cwd, path) - return normpath(path) # Return an absolute path. try: from nt import _getfullpathname except ImportError: # not running on Windows - mock up something sensible - abspath = _abspath_fallback + def abspath(path): + """Return the absolute version of a path.""" + path = os.fspath(path) + if not isabs(path): + if isinstance(path, bytes): + cwd = os.getcwdb() + else: + cwd = os.getcwd() + path = join(cwd, path) + return normpath(path) else: # use native Windows method on Windows def abspath(path): @@ -598,15 +575,36 @@ def abspath(path): try: return _getfullpathname(normpath(path)) except (OSError, ValueError): - return _abspath_fallback(path) + # See gh-75230, handle outside for cleaner traceback + pass + path = os.fspath(path) + if not isabs(path): + if isinstance(path, bytes): + sep = b'\\' + getcwd = os.getcwdb + else: + sep = '\\' + getcwd = os.getcwd + drive, root, path = splitroot(path) + # Either drive or root can be nonempty, but not both. + if drive or root: + try: + path = join(_getfullpathname(drive + root), path) + except (OSError, ValueError): + # Drive "\0:" cannot exist; use the root directory. + path = drive + sep + path + else: + path = join(getcwd(), path) + return normpath(path) try: - from nt import _getfinalpathname, readlink as _nt_readlink + from nt import _findfirstfile, _getfinalpathname, readlink as _nt_readlink except ImportError: # realpath is a no-op on systems without _getfinalpathname support. - realpath = abspath + def realpath(path, *, strict=False): + return abspath(path) else: - def _readlink_deep(path): + def _readlink_deep(path, ignored_error=OSError): # These error codes indicate that we should stop reading links and # return the path we currently have. # 1: ERROR_INVALID_FUNCTION @@ -639,7 +637,7 @@ def _readlink_deep(path): path = old_path break path = normpath(join(dirname(old_path), path)) - except OSError as ex: + except ignored_error as ex: if ex.winerror in allowed_winerror: break raise @@ -648,7 +646,7 @@ def _readlink_deep(path): break return path - def _getfinalpathname_nonstrict(path): + def _getfinalpathname_nonstrict(path, ignored_error=OSError): # These error codes indicate that we should stop resolving the path # and return the value we currently have. # 1: ERROR_INVALID_FUNCTION @@ -664,9 +662,10 @@ def _getfinalpathname_nonstrict(path): # 87: ERROR_INVALID_PARAMETER # 123: ERROR_INVALID_NAME # 161: ERROR_BAD_PATHNAME + # 1005: ERROR_UNRECOGNIZED_VOLUME # 1920: ERROR_CANT_ACCESS_FILE # 1921: ERROR_CANT_RESOLVE_FILENAME (implies unfollowable symlink) - allowed_winerror = 1, 2, 3, 5, 21, 32, 50, 53, 65, 67, 87, 123, 161, 1920, 1921 + allowed_winerror = 1, 2, 3, 5, 21, 32, 50, 53, 65, 67, 87, 123, 161, 1005, 1920, 1921 # Non-strict algorithm is to find as much of the target directory # as we can and join the rest. @@ -675,23 +674,29 @@ def _getfinalpathname_nonstrict(path): try: path = _getfinalpathname(path) return join(path, tail) if tail else path - except OSError as ex: + except ignored_error as ex: if ex.winerror not in allowed_winerror: raise try: # The OS could not resolve this path fully, so we attempt # to follow the link ourselves. If we succeed, join the tail # and return. - new_path = _readlink_deep(path) + new_path = _readlink_deep(path, + ignored_error=ignored_error) if new_path != path: return join(new_path, tail) if tail else new_path - except OSError: + except ignored_error: # If we fail to readlink(), let's keep traversing pass - path, name = split(path) - # TODO (bpo-38186): Request the real file name from the directory - # entry using FindFirstFileW. For now, we will return the path - # as best we have it + # If we get these errors, try to get the real name of the file without accessing it. + if ex.winerror in (1, 5, 32, 50, 87, 1920, 1921): + try: + name = _findfirstfile(path) + path, _ = split(path) + except ignored_error: + path, name = split(path) + else: + path, name = split(path) if path and not name: return path + tail tail = join(name, tail) if tail else name @@ -705,7 +710,8 @@ def realpath(path, *, strict=False): new_unc_prefix = b'\\\\' cwd = os.getcwdb() # bpo-38081: Special case for realpath(b'nul') - if normcase(path) == normcase(os.fsencode(devnull)): + devnull = b'nul' + if normcase(path) == devnull: return b'\\\\.\\NUL' else: prefix = '\\\\?\\' @@ -713,9 +719,19 @@ def realpath(path, *, strict=False): new_unc_prefix = '\\\\' cwd = os.getcwd() # bpo-38081: Special case for realpath('nul') - if normcase(path) == normcase(devnull): + devnull = 'nul' + if normcase(path) == devnull: return '\\\\.\\NUL' had_prefix = path.startswith(prefix) + + if strict is ALLOW_MISSING: + ignored_error = FileNotFoundError + strict = True + elif strict: + ignored_error = () + else: + ignored_error = OSError + if not had_prefix and not isabs(path): path = join(cwd, path) try: @@ -723,17 +739,16 @@ def realpath(path, *, strict=False): initial_winerror = 0 except ValueError as ex: # gh-106242: Raised for embedded null characters - # In strict mode, we convert into an OSError. + # In strict modes, we convert into an OSError. # Non-strict mode returns the path as-is, since we've already # made it absolute. if strict: raise OSError(str(ex)) from None path = normpath(path) - except OSError as ex: - if strict: - raise + except ignored_error as ex: initial_winerror = ex.winerror - path = _getfinalpathname_nonstrict(path) + path = _getfinalpathname_nonstrict(path, + ignored_error=ignored_error) # The path returned by _getfinalpathname will always start with \\?\ - # strip off that prefix unless it was already provided on the original # path. @@ -766,6 +781,9 @@ def realpath(path, *, strict=False): def relpath(path, start=None): """Return a relative version of a path""" path = os.fspath(path) + if not path: + raise ValueError("no path specified") + if isinstance(path, bytes): sep = b'\\' curdir = b'.' @@ -777,22 +795,20 @@ def relpath(path, start=None): if start is None: start = curdir + else: + start = os.fspath(start) - if not path: - raise ValueError("no path specified") - - start = os.fspath(start) try: - start_abs = abspath(normpath(start)) - path_abs = abspath(normpath(path)) + start_abs = abspath(start) + path_abs = abspath(path) start_drive, _, start_rest = splitroot(start_abs) path_drive, _, path_rest = splitroot(path_abs) if normcase(start_drive) != normcase(path_drive): raise ValueError("path is on mount %r, start on mount %r" % ( path_drive, start_drive)) - start_list = [x for x in start_rest.split(sep) if x] - path_list = [x for x in path_rest.split(sep) if x] + start_list = start_rest.split(sep) if start_rest else [] + path_list = path_rest.split(sep) if path_rest else [] # Work out how much of the filepath is shared by start and path. i = 0 for e1, e2 in zip(start_list, path_list): @@ -803,29 +819,28 @@ def relpath(path, start=None): rel_list = [pardir] * (len(start_list)-i) + path_list[i:] if not rel_list: return curdir - return join(*rel_list) + return sep.join(rel_list) except (TypeError, ValueError, AttributeError, BytesWarning, DeprecationWarning): genericpath._check_arg_types('relpath', path, start) raise -# Return the longest common sub-path of the sequence of paths given as input. +# Return the longest common sub-path of the iterable of paths given as input. # The function is case-insensitive and 'separator-insensitive', i.e. if the # only difference between two paths is the use of '\' versus '/' as separator, # they are deemed to be equal. # # However, the returned path will have the standard '\' separator (even if the # given paths had the alternative '/' separator) and will have the case of the -# first path given in the sequence. Additionally, any trailing separator is +# first path given in the iterable. Additionally, any trailing separator is # stripped from the returned path. def commonpath(paths): - """Given a sequence of path names, returns the longest common sub-path.""" - + """Given an iterable of path names, returns the longest common sub-path.""" + paths = tuple(map(os.fspath, paths)) if not paths: - raise ValueError('commonpath() arg is an empty sequence') + raise ValueError('commonpath() arg is an empty iterable') - paths = tuple(map(os.fspath, paths)) if isinstance(paths[0], bytes): sep = b'\\' altsep = b'/' @@ -839,9 +854,6 @@ def commonpath(paths): drivesplits = [splitroot(p.replace(altsep, sep).lower()) for p in paths] split_paths = [p.split(sep) for d, r, p in drivesplits] - if len({r for d, r, p in drivesplits}) != 1: - raise ValueError("Can't mix absolute and relative paths") - # Check that all drive letters or UNC paths match. The check is made only # now otherwise type errors for mixing strings and bytes would not be # caught. @@ -849,6 +861,12 @@ def commonpath(paths): raise ValueError("Paths don't have the same drive") drive, root, path = splitroot(paths[0].replace(altsep, sep)) + if len({r for d, r, p in drivesplits}) != 1: + if drive: + raise ValueError("Can't mix absolute and relative paths") + else: + raise ValueError("Can't mix rooted and not-rooted paths") + common = path.split(sep) common = [c for c in common if c and c != curdir] @@ -869,13 +887,15 @@ def commonpath(paths): try: - # The isdir(), isfile(), islink() and exists() implementations in - # genericpath use os.stat(). This is overkill on Windows. Use simpler + # The isdir(), isfile(), islink(), exists() and lexists() implementations + # in genericpath use os.stat(). This is overkill on Windows. Use simpler # builtin functions if they are available. from nt import _path_isdir as isdir from nt import _path_isfile as isfile from nt import _path_islink as islink + from nt import _path_isjunction as isjunction from nt import _path_exists as exists + from nt import _path_lexists as lexists except ImportError: # Use genericpath.* as imported above pass @@ -883,15 +903,12 @@ def commonpath(paths): try: from nt import _path_isdevdrive -except ImportError: - def isdevdrive(path): - """Determines whether the specified path is on a Windows Dev Drive.""" - # Never a Dev Drive - return False -else: def isdevdrive(path): """Determines whether the specified path is on a Windows Dev Drive.""" try: return _path_isdevdrive(abspath(path)) except OSError: return False +except ImportError: + # Use genericpath.isdevdrive as imported above + pass diff --git a/Lib/numbers.py b/Lib/numbers.py index a2913e32cf..37fddb8917 100644 --- a/Lib/numbers.py +++ b/Lib/numbers.py @@ -290,18 +290,27 @@ def conjugate(self): class Rational(Real): - """.numerator and .denominator should be in lowest terms.""" + """To Real, Rational adds numerator and denominator properties. + + The numerator and denominator values should be in lowest terms, + with a positive denominator. + """ __slots__ = () @property @abstractmethod def numerator(self): + """The numerator of a rational number in lowest terms.""" raise NotImplementedError @property @abstractmethod def denominator(self): + """The denominator of a rational number in lowest terms. + + This denominator should be positive. + """ raise NotImplementedError # Concrete implementation of Real's conversion to float. diff --git a/Lib/opcode.py b/Lib/opcode.py index ab6b765b4b..5735686fa7 100644 --- a/Lib/opcode.py +++ b/Lib/opcode.py @@ -4,404 +4,47 @@ operate on bytecodes (e.g. peephole optimizers). """ -__all__ = ["cmp_op", "hasarg", "hasconst", "hasname", "hasjrel", "hasjabs", - "haslocal", "hascompare", "hasfree", "hasexc", "opname", "opmap", - "HAVE_ARGUMENT", "EXTENDED_ARG"] -# It's a chicken-and-egg I'm afraid: -# We're imported before _opcode's made. -# With exception unheeded -# (stack_effect is not needed) -# Both our chickens and eggs are allayed. -# --Larry Hastings, 2013/11/23 +__all__ = ["cmp_op", "stack_effect", "hascompare", "opname", "opmap", + "HAVE_ARGUMENT", "EXTENDED_ARG", "hasarg", "hasconst", "hasname", + "hasjump", "hasjrel", "hasjabs", "hasfree", "haslocal", "hasexc"] -try: - from _opcode import stack_effect - __all__.append('stack_effect') -except ImportError: - pass +import _opcode +from _opcode import stack_effect -cmp_op = ('<', '<=', '==', '!=', '>', '>=') - -hasarg = [] -hasconst = [] -hasname = [] -hasjrel = [] -hasjabs = [] -haslocal = [] -hascompare = [] -hasfree = [] -hasexc = [] - -def is_pseudo(op): - return op >= MIN_PSEUDO_OPCODE and op <= MAX_PSEUDO_OPCODE - -oplists = [hasarg, hasconst, hasname, hasjrel, hasjabs, - haslocal, hascompare, hasfree, hasexc] - -opmap = {} - -## pseudo opcodes (used in the compiler) mapped to the values -## they can become in the actual code. -_pseudo_ops = {} - -def def_op(name, op): - opmap[name] = op - -def name_op(name, op): - def_op(name, op) - hasname.append(op) - -def jrel_op(name, op): - def_op(name, op) - hasjrel.append(op) - -def jabs_op(name, op): - def_op(name, op) - hasjabs.append(op) - -def pseudo_op(name, op, real_ops): - def_op(name, op) - _pseudo_ops[name] = real_ops - # add the pseudo opcode to the lists its targets are in - for oplist in oplists: - res = [opmap[rop] in oplist for rop in real_ops] - if any(res): - assert all(res) - oplist.append(op) - - -# Instruction opcodes for compiled code -# Blank lines correspond to available opcodes - -def_op('CACHE', 0) -def_op('POP_TOP', 1) -def_op('PUSH_NULL', 2) - -def_op('NOP', 9) -def_op('UNARY_POSITIVE', 10) -def_op('UNARY_NEGATIVE', 11) -def_op('UNARY_NOT', 12) - -def_op('UNARY_INVERT', 15) - -def_op('BINARY_SUBSCR', 25) -def_op('BINARY_SLICE', 26) -def_op('STORE_SLICE', 27) - -def_op('GET_LEN', 30) -def_op('MATCH_MAPPING', 31) -def_op('MATCH_SEQUENCE', 32) -def_op('MATCH_KEYS', 33) - -def_op('PUSH_EXC_INFO', 35) -def_op('CHECK_EXC_MATCH', 36) -def_op('CHECK_EG_MATCH', 37) - -def_op('WITH_EXCEPT_START', 49) -def_op('GET_AITER', 50) -def_op('GET_ANEXT', 51) -def_op('BEFORE_ASYNC_WITH', 52) -def_op('BEFORE_WITH', 53) -def_op('END_ASYNC_FOR', 54) -def_op('CLEANUP_THROW', 55) - -def_op('STORE_SUBSCR', 60) -def_op('DELETE_SUBSCR', 61) - -# TODO: RUSTPYTHON -# Delete below def_op after updating coroutines.py -def_op('YIELD_FROM', 72) - -def_op('GET_ITER', 68) -def_op('GET_YIELD_FROM_ITER', 69) -def_op('PRINT_EXPR', 70) -def_op('LOAD_BUILD_CLASS', 71) - -def_op('LOAD_ASSERTION_ERROR', 74) -def_op('RETURN_GENERATOR', 75) - -def_op('LIST_TO_TUPLE', 82) -def_op('RETURN_VALUE', 83) -def_op('IMPORT_STAR', 84) -def_op('SETUP_ANNOTATIONS', 85) - -def_op('ASYNC_GEN_WRAP', 87) -def_op('PREP_RERAISE_STAR', 88) -def_op('POP_EXCEPT', 89) - -HAVE_ARGUMENT = 90 # real opcodes from here have an argument: - -name_op('STORE_NAME', 90) # Index in name list -name_op('DELETE_NAME', 91) # "" -def_op('UNPACK_SEQUENCE', 92) # Number of tuple items -jrel_op('FOR_ITER', 93) -def_op('UNPACK_EX', 94) -name_op('STORE_ATTR', 95) # Index in name list -name_op('DELETE_ATTR', 96) # "" -name_op('STORE_GLOBAL', 97) # "" -name_op('DELETE_GLOBAL', 98) # "" -def_op('SWAP', 99) -def_op('LOAD_CONST', 100) # Index in const list -hasconst.append(100) -name_op('LOAD_NAME', 101) # Index in name list -def_op('BUILD_TUPLE', 102) # Number of tuple items -def_op('BUILD_LIST', 103) # Number of list items -def_op('BUILD_SET', 104) # Number of set items -def_op('BUILD_MAP', 105) # Number of dict entries -name_op('LOAD_ATTR', 106) # Index in name list -def_op('COMPARE_OP', 107) # Comparison operator -hascompare.append(107) -name_op('IMPORT_NAME', 108) # Index in name list -name_op('IMPORT_FROM', 109) # Index in name list -jrel_op('JUMP_FORWARD', 110) # Number of words to skip -jrel_op('JUMP_IF_FALSE_OR_POP', 111) # Number of words to skip -jrel_op('JUMP_IF_TRUE_OR_POP', 112) # "" -jrel_op('POP_JUMP_IF_FALSE', 114) -jrel_op('POP_JUMP_IF_TRUE', 115) -name_op('LOAD_GLOBAL', 116) # Index in name list -def_op('IS_OP', 117) -def_op('CONTAINS_OP', 118) -def_op('RERAISE', 119) -def_op('COPY', 120) -def_op('BINARY_OP', 122) -jrel_op('SEND', 123) # Number of bytes to skip -def_op('LOAD_FAST', 124) # Local variable number, no null check -haslocal.append(124) -def_op('STORE_FAST', 125) # Local variable number -haslocal.append(125) -def_op('DELETE_FAST', 126) # Local variable number -haslocal.append(126) -def_op('LOAD_FAST_CHECK', 127) # Local variable number -haslocal.append(127) -jrel_op('POP_JUMP_IF_NOT_NONE', 128) -jrel_op('POP_JUMP_IF_NONE', 129) -def_op('RAISE_VARARGS', 130) # Number of raise arguments (1, 2, or 3) -def_op('GET_AWAITABLE', 131) -def_op('MAKE_FUNCTION', 132) # Flags -def_op('BUILD_SLICE', 133) # Number of items -jrel_op('JUMP_BACKWARD_NO_INTERRUPT', 134) # Number of words to skip (backwards) -def_op('MAKE_CELL', 135) -hasfree.append(135) -def_op('LOAD_CLOSURE', 136) -hasfree.append(136) -def_op('LOAD_DEREF', 137) -hasfree.append(137) -def_op('STORE_DEREF', 138) -hasfree.append(138) -def_op('DELETE_DEREF', 139) -hasfree.append(139) -jrel_op('JUMP_BACKWARD', 140) # Number of words to skip (backwards) - -def_op('CALL_FUNCTION_EX', 142) # Flags - -def_op('EXTENDED_ARG', 144) -EXTENDED_ARG = 144 -def_op('LIST_APPEND', 145) -def_op('SET_ADD', 146) -def_op('MAP_ADD', 147) -def_op('LOAD_CLASSDEREF', 148) -hasfree.append(148) -def_op('COPY_FREE_VARS', 149) -def_op('YIELD_VALUE', 150) -def_op('RESUME', 151) # This must be kept in sync with deepfreeze.py -def_op('MATCH_CLASS', 152) - -def_op('FORMAT_VALUE', 155) -def_op('BUILD_CONST_KEY_MAP', 156) -def_op('BUILD_STRING', 157) - -def_op('LIST_EXTEND', 162) -def_op('SET_UPDATE', 163) -def_op('DICT_MERGE', 164) -def_op('DICT_UPDATE', 165) - -def_op('CALL', 171) -def_op('KW_NAMES', 172) -hasconst.append(172) +from _opcode_metadata import (_specializations, _specialized_opmap, opmap, + HAVE_ARGUMENT, MIN_INSTRUMENTED_OPCODE) +EXTENDED_ARG = opmap['EXTENDED_ARG'] - -hasarg.extend([op for op in opmap.values() if op >= HAVE_ARGUMENT]) - -MIN_PSEUDO_OPCODE = 256 - -pseudo_op('SETUP_FINALLY', 256, ['NOP']) -hasexc.append(256) -pseudo_op('SETUP_CLEANUP', 257, ['NOP']) -hasexc.append(257) -pseudo_op('SETUP_WITH', 258, ['NOP']) -hasexc.append(258) -pseudo_op('POP_BLOCK', 259, ['NOP']) - -pseudo_op('JUMP', 260, ['JUMP_FORWARD', 'JUMP_BACKWARD']) -pseudo_op('JUMP_NO_INTERRUPT', 261, ['JUMP_FORWARD', 'JUMP_BACKWARD_NO_INTERRUPT']) - -pseudo_op('LOAD_METHOD', 262, ['LOAD_ATTR']) - -MAX_PSEUDO_OPCODE = MIN_PSEUDO_OPCODE + len(_pseudo_ops) - 1 - -del def_op, name_op, jrel_op, jabs_op, pseudo_op - -opname = ['<%r>' % (op,) for op in range(MAX_PSEUDO_OPCODE + 1)] +opname = ['<%r>' % (op,) for op in range(max(opmap.values()) + 1)] for op, i in opmap.items(): opname[i] = op +cmp_op = ('<', '<=', '==', '!=', '>', '>=') -_nb_ops = [ - ("NB_ADD", "+"), - ("NB_AND", "&"), - ("NB_FLOOR_DIVIDE", "//"), - ("NB_LSHIFT", "<<"), - ("NB_MATRIX_MULTIPLY", "@"), - ("NB_MULTIPLY", "*"), - ("NB_REMAINDER", "%"), - ("NB_OR", "|"), - ("NB_POWER", "**"), - ("NB_RSHIFT", ">>"), - ("NB_SUBTRACT", "-"), - ("NB_TRUE_DIVIDE", "/"), - ("NB_XOR", "^"), - ("NB_INPLACE_ADD", "+="), - ("NB_INPLACE_AND", "&="), - ("NB_INPLACE_FLOOR_DIVIDE", "//="), - ("NB_INPLACE_LSHIFT", "<<="), - ("NB_INPLACE_MATRIX_MULTIPLY", "@="), - ("NB_INPLACE_MULTIPLY", "*="), - ("NB_INPLACE_REMAINDER", "%="), - ("NB_INPLACE_OR", "|="), - ("NB_INPLACE_POWER", "**="), - ("NB_INPLACE_RSHIFT", ">>="), - ("NB_INPLACE_SUBTRACT", "-="), - ("NB_INPLACE_TRUE_DIVIDE", "/="), - ("NB_INPLACE_XOR", "^="), -] +# These lists are documented as part of the dis module's API +hasarg = [op for op in opmap.values() if _opcode.has_arg(op)] +hasconst = [op for op in opmap.values() if _opcode.has_const(op)] +hasname = [op for op in opmap.values() if _opcode.has_name(op)] +hasjump = [op for op in opmap.values() if _opcode.has_jump(op)] +hasjrel = hasjump # for backward compatibility +hasjabs = [] +hasfree = [op for op in opmap.values() if _opcode.has_free(op)] +haslocal = [op for op in opmap.values() if _opcode.has_local(op)] +hasexc = [op for op in opmap.values() if _opcode.has_exc(op)] -_specializations = { - "BINARY_OP": [ - "BINARY_OP_ADAPTIVE", - "BINARY_OP_ADD_FLOAT", - "BINARY_OP_ADD_INT", - "BINARY_OP_ADD_UNICODE", - "BINARY_OP_INPLACE_ADD_UNICODE", - "BINARY_OP_MULTIPLY_FLOAT", - "BINARY_OP_MULTIPLY_INT", - "BINARY_OP_SUBTRACT_FLOAT", - "BINARY_OP_SUBTRACT_INT", - ], - "BINARY_SUBSCR": [ - "BINARY_SUBSCR_ADAPTIVE", - "BINARY_SUBSCR_DICT", - "BINARY_SUBSCR_GETITEM", - "BINARY_SUBSCR_LIST_INT", - "BINARY_SUBSCR_TUPLE_INT", - ], - "CALL": [ - "CALL_ADAPTIVE", - "CALL_PY_EXACT_ARGS", - "CALL_PY_WITH_DEFAULTS", - "CALL_BOUND_METHOD_EXACT_ARGS", - "CALL_BUILTIN_CLASS", - "CALL_BUILTIN_FAST_WITH_KEYWORDS", - "CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS", - "CALL_NO_KW_BUILTIN_FAST", - "CALL_NO_KW_BUILTIN_O", - "CALL_NO_KW_ISINSTANCE", - "CALL_NO_KW_LEN", - "CALL_NO_KW_LIST_APPEND", - "CALL_NO_KW_METHOD_DESCRIPTOR_FAST", - "CALL_NO_KW_METHOD_DESCRIPTOR_NOARGS", - "CALL_NO_KW_METHOD_DESCRIPTOR_O", - "CALL_NO_KW_STR_1", - "CALL_NO_KW_TUPLE_1", - "CALL_NO_KW_TYPE_1", - ], - "COMPARE_OP": [ - "COMPARE_OP_ADAPTIVE", - "COMPARE_OP_FLOAT_JUMP", - "COMPARE_OP_INT_JUMP", - "COMPARE_OP_STR_JUMP", - ], - "EXTENDED_ARG": [ - "EXTENDED_ARG_QUICK", - ], - "FOR_ITER": [ - "FOR_ITER_ADAPTIVE", - "FOR_ITER_LIST", - "FOR_ITER_RANGE", - ], - "JUMP_BACKWARD": [ - "JUMP_BACKWARD_QUICK", - ], - "LOAD_ATTR": [ - "LOAD_ATTR_ADAPTIVE", - # These potentially push [NULL, bound method] onto the stack. - "LOAD_ATTR_CLASS", - "LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN", - "LOAD_ATTR_INSTANCE_VALUE", - "LOAD_ATTR_MODULE", - "LOAD_ATTR_PROPERTY", - "LOAD_ATTR_SLOT", - "LOAD_ATTR_WITH_HINT", - # These will always push [unbound method, self] onto the stack. - "LOAD_ATTR_METHOD_LAZY_DICT", - "LOAD_ATTR_METHOD_NO_DICT", - "LOAD_ATTR_METHOD_WITH_DICT", - "LOAD_ATTR_METHOD_WITH_VALUES", - ], - "LOAD_CONST": [ - "LOAD_CONST__LOAD_FAST", - ], - "LOAD_FAST": [ - "LOAD_FAST__LOAD_CONST", - "LOAD_FAST__LOAD_FAST", - ], - "LOAD_GLOBAL": [ - "LOAD_GLOBAL_ADAPTIVE", - "LOAD_GLOBAL_BUILTIN", - "LOAD_GLOBAL_MODULE", - ], - "RESUME": [ - "RESUME_QUICK", - ], - "STORE_ATTR": [ - "STORE_ATTR_ADAPTIVE", - "STORE_ATTR_INSTANCE_VALUE", - "STORE_ATTR_SLOT", - "STORE_ATTR_WITH_HINT", - ], - "STORE_FAST": [ - "STORE_FAST__LOAD_FAST", - "STORE_FAST__STORE_FAST", - ], - "STORE_SUBSCR": [ - "STORE_SUBSCR_ADAPTIVE", - "STORE_SUBSCR_DICT", - "STORE_SUBSCR_LIST_INT", - ], - "UNPACK_SEQUENCE": [ - "UNPACK_SEQUENCE_ADAPTIVE", - "UNPACK_SEQUENCE_LIST", - "UNPACK_SEQUENCE_TUPLE", - "UNPACK_SEQUENCE_TWO_TUPLE", - ], -} -_specialized_instructions = [ - opcode for family in _specializations.values() for opcode in family -] -_specialization_stats = [ - "success", - "failure", - "hit", - "deferred", - "miss", - "deopt", -] + +_intrinsic_1_descs = _opcode.get_intrinsic1_descs() +_intrinsic_2_descs = _opcode.get_intrinsic2_descs() +_nb_ops = _opcode.get_nb_ops() + +hascompare = [opmap["COMPARE_OP"]] _cache_format = { "LOAD_GLOBAL": { "counter": 1, "index": 1, - "module_keys_version": 2, + "module_keys_version": 1, "builtin_keys_version": 1, }, "BINARY_OP": { @@ -412,16 +55,19 @@ def pseudo_op(name, op, real_ops): }, "COMPARE_OP": { "counter": 1, - "mask": 1, + }, + "CONTAINS_OP": { + "counter": 1, }, "BINARY_SUBSCR": { "counter": 1, - "type_version": 2, - "func_version": 1, }, "FOR_ITER": { "counter": 1, }, + "LOAD_SUPER_ATTR": { + "counter": 1, + }, "LOAD_ATTR": { "counter": 1, "version": 2, @@ -436,13 +82,34 @@ def pseudo_op(name, op, real_ops): "CALL": { "counter": 1, "func_version": 2, - "min_args": 1, }, "STORE_SUBSCR": { "counter": 1, }, + "SEND": { + "counter": 1, + }, + "JUMP_BACKWARD": { + "counter": 1, + }, + "TO_BOOL": { + "counter": 1, + "version": 2, + }, + "POP_JUMP_IF_TRUE": { + "counter": 1, + }, + "POP_JUMP_IF_FALSE": { + "counter": 1, + }, + "POP_JUMP_IF_NONE": { + "counter": 1, + }, + "POP_JUMP_IF_NOT_NONE": { + "counter": 1, + }, } -_inline_cache_entries = [ - sum(_cache_format.get(opname[opcode], {}).values()) for opcode in range(256) -] +_inline_cache_entries = { + name : sum(value.values()) for (name, value) in _cache_format.items() +} diff --git a/Lib/operator.py b/Lib/operator.py index 30116c1189..02ccdaa13d 100644 --- a/Lib/operator.py +++ b/Lib/operator.py @@ -239,7 +239,7 @@ class attrgetter: """ __slots__ = ('_attrs', '_call') - def __init__(self, attr, *attrs): + def __init__(self, attr, /, *attrs): if not attrs: if not isinstance(attr, str): raise TypeError('attribute name must be a string') @@ -257,7 +257,7 @@ def func(obj): return tuple(getter(obj) for getter in getters) self._call = func - def __call__(self, obj): + def __call__(self, obj, /): return self._call(obj) def __repr__(self): @@ -276,7 +276,7 @@ class itemgetter: """ __slots__ = ('_items', '_call') - def __init__(self, item, *items): + def __init__(self, item, /, *items): if not items: self._items = (item,) def func(obj): @@ -288,7 +288,7 @@ def func(obj): return tuple(obj[i] for i in items) self._call = func - def __call__(self, obj): + def __call__(self, obj, /): return self._call(obj) def __repr__(self): @@ -315,7 +315,7 @@ def __init__(self, name, /, *args, **kwargs): self._args = args self._kwargs = kwargs - def __call__(self, obj): + def __call__(self, obj, /): return getattr(obj, self._name)(*self._args, **self._kwargs) def __repr__(self): diff --git a/Lib/os.py b/Lib/os.py index 7ee7d695d9..b4c9f84c36 100644 --- a/Lib/os.py +++ b/Lib/os.py @@ -110,6 +110,7 @@ def _add(str, fn): _add("HAVE_FCHMODAT", "chmod") _add("HAVE_FCHOWNAT", "chown") _add("HAVE_FSTATAT", "stat") + _add("HAVE_LSTAT", "lstat") _add("HAVE_FUTIMESAT", "utime") _add("HAVE_LINKAT", "link") _add("HAVE_MKDIRAT", "mkdir") @@ -131,6 +132,7 @@ def _add(str, fn): _set = set() _add("HAVE_FCHDIR", "chdir") _add("HAVE_FCHMOD", "chmod") + _add("MS_WINDOWS", "chmod") _add("HAVE_FCHOWN", "chown") _add("HAVE_FDOPENDIR", "listdir") _add("HAVE_FDOPENDIR", "scandir") @@ -171,6 +173,7 @@ def _add(str, fn): _add("HAVE_FSTATAT", "stat") _add("HAVE_LCHFLAGS", "chflags") _add("HAVE_LCHMOD", "chmod") + _add("MS_WINDOWS", "chmod") if _exists("lchown"): # mac os x10.3 _add("HAVE_LCHOWN", "chown") _add("HAVE_LINKAT", "link") @@ -279,6 +282,10 @@ def renames(old, new): __all__.extend(["makedirs", "removedirs", "renames"]) +# Private sentinel that makes walk() classify all symlinks and junctions as +# regular files. +_walk_symlinks_as_files = object() + def walk(top, topdown=True, onerror=None, followlinks=False): """Directory tree generator. @@ -331,12 +338,12 @@ def walk(top, topdown=True, onerror=None, followlinks=False): import os from os.path import join, getsize - for root, dirs, files in os.walk('python/Lib/email'): + for root, dirs, files in os.walk('python/Lib/xml'): print(root, "consumes ") print(sum(getsize(join(root, name)) for name in files), end=" ") print("bytes in", len(files), "non-directory files") - if 'CVS' in dirs: - dirs.remove('CVS') # don't visit CVS directories + if '__pycache__' in dirs: + dirs.remove('__pycache__') # don't visit __pycache__ directories """ sys.audit("os.walk", top, topdown, onerror, followlinks) @@ -380,7 +387,10 @@ def walk(top, topdown=True, onerror=None, followlinks=False): break try: - is_dir = entry.is_dir() + if followlinks is _walk_symlinks_as_files: + is_dir = entry.is_dir(follow_symlinks=False) and not entry.is_junction() + else: + is_dir = entry.is_dir() except OSError: # If is_dir() raises an OSError, consider the entry not to # be a directory, same behaviour as os.path.isdir(). @@ -459,34 +469,69 @@ def fwalk(top=".", topdown=True, onerror=None, *, follow_symlinks=False, dir_fd= Example: import os - for root, dirs, files, rootfd in os.fwalk('python/Lib/email'): + for root, dirs, files, rootfd in os.fwalk('python/Lib/xml'): print(root, "consumes", end="") print(sum(os.stat(name, dir_fd=rootfd).st_size for name in files), end="") print("bytes in", len(files), "non-directory files") - if 'CVS' in dirs: - dirs.remove('CVS') # don't visit CVS directories + if '__pycache__' in dirs: + dirs.remove('__pycache__') # don't visit __pycache__ directories """ sys.audit("os.fwalk", top, topdown, onerror, follow_symlinks, dir_fd) top = fspath(top) - # Note: To guard against symlink races, we use the standard - # lstat()/open()/fstat() trick. - if not follow_symlinks: - orig_st = stat(top, follow_symlinks=False, dir_fd=dir_fd) - topfd = open(top, O_RDONLY | O_NONBLOCK, dir_fd=dir_fd) + stack = [(_fwalk_walk, (True, dir_fd, top, top, None))] + isbytes = isinstance(top, bytes) try: - if (follow_symlinks or (st.S_ISDIR(orig_st.st_mode) and - path.samestat(orig_st, stat(topfd)))): - yield from _fwalk(topfd, top, isinstance(top, bytes), - topdown, onerror, follow_symlinks) + while stack: + yield from _fwalk(stack, isbytes, topdown, onerror, follow_symlinks) finally: - close(topfd) - - def _fwalk(topfd, toppath, isbytes, topdown, onerror, follow_symlinks): + # Close any file descriptors still on the stack. + while stack: + action, value = stack.pop() + if action == _fwalk_close: + close(value) + + # Each item in the _fwalk() stack is a pair (action, args). + _fwalk_walk = 0 # args: (isroot, dirfd, toppath, topname, entry) + _fwalk_yield = 1 # args: (toppath, dirnames, filenames, topfd) + _fwalk_close = 2 # args: dirfd + + def _fwalk(stack, isbytes, topdown, onerror, follow_symlinks): # Note: This uses O(depth of the directory tree) file descriptors: if # necessary, it can be adapted to only require O(1) FDs, see issue # #13734. + action, value = stack.pop() + if action == _fwalk_close: + close(value) + return + elif action == _fwalk_yield: + yield value + return + assert action == _fwalk_walk + isroot, dirfd, toppath, topname, entry = value + try: + if not follow_symlinks: + # Note: To guard against symlink races, we use the standard + # lstat()/open()/fstat() trick. + if entry is None: + orig_st = stat(topname, follow_symlinks=False, dir_fd=dirfd) + else: + orig_st = entry.stat(follow_symlinks=False) + topfd = open(topname, O_RDONLY | O_NONBLOCK, dir_fd=dirfd) + except OSError as err: + if isroot: + raise + if onerror is not None: + onerror(err) + return + stack.append((_fwalk_close, topfd)) + if not follow_symlinks: + if isroot and not st.S_ISDIR(orig_st.st_mode): + return + if not path.samestat(orig_st, stat(topfd)): + return + scandir_it = scandir(topfd) dirs = [] nondirs = [] @@ -512,31 +557,18 @@ def _fwalk(topfd, toppath, isbytes, topdown, onerror, follow_symlinks): if topdown: yield toppath, dirs, nondirs, topfd + else: + stack.append((_fwalk_yield, (toppath, dirs, nondirs, topfd))) - for name in dirs if entries is None else zip(dirs, entries): - try: - if not follow_symlinks: - if topdown: - orig_st = stat(name, dir_fd=topfd, follow_symlinks=False) - else: - assert entries is not None - name, entry = name - orig_st = entry.stat(follow_symlinks=False) - dirfd = open(name, O_RDONLY | O_NONBLOCK, dir_fd=topfd) - except OSError as err: - if onerror is not None: - onerror(err) - continue - try: - if follow_symlinks or path.samestat(orig_st, stat(dirfd)): - dirpath = path.join(toppath, name) - yield from _fwalk(dirfd, dirpath, isbytes, - topdown, onerror, follow_symlinks) - finally: - close(dirfd) - - if not topdown: - yield toppath, dirs, nondirs, topfd + toppath = path.join(toppath, toppath[:0]) # Add trailing slash. + if entries is None: + stack.extend( + (_fwalk_walk, (False, topfd, toppath + name, name, None)) + for name in dirs[::-1]) + else: + stack.extend( + (_fwalk_walk, (False, topfd, toppath + name, name, entry)) + for name, entry in zip(dirs[::-1], entries[::-1])) __all__.append("fwalk") @@ -1061,6 +1093,12 @@ def _fspath(path): else: raise TypeError("expected str, bytes or os.PathLike object, " "not " + path_type.__name__) + except TypeError: + if path_type.__fspath__ is None: + raise TypeError("expected str, bytes or os.PathLike object, " + "not " + path_type.__name__) from None + else: + raise if isinstance(path_repr, (str, bytes)): return path_repr else: @@ -1079,6 +1117,8 @@ class PathLike(abc.ABC): """Abstract base class for implementing the file system path protocol.""" + __slots__ = () + @abc.abstractmethod def __fspath__(self): """Return the file system path representation of the object.""" @@ -1128,3 +1168,17 @@ def add_dll_directory(path): cookie, nt._remove_dll_directory ) + + +if _exists('sched_getaffinity') and sys._get_cpu_count_config() < 0: + def process_cpu_count(): + """ + Get the number of CPUs of the current process. + + Return the number of logical CPUs usable by the calling thread of the + current process. Return None if indeterminable. + """ + return len(sched_getaffinity(0)) +else: + # Just an alias to cpu_count() (same docstring) + process_cpu_count = cpu_count diff --git a/Lib/pickle.py b/Lib/pickle.py index 6e3c61fd0b..550f8675f2 100644 --- a/Lib/pickle.py +++ b/Lib/pickle.py @@ -314,16 +314,17 @@ def load_frame(self, frame_size): # Tools used for pickling. def _getattribute(obj, name): + top = obj for subpath in name.split('.'): if subpath == '': raise AttributeError("Can't get local attribute {!r} on {!r}" - .format(name, obj)) + .format(name, top)) try: parent = obj obj = getattr(obj, subpath) except AttributeError: raise AttributeError("Can't get attribute {!r} on {!r}" - .format(name, obj)) from None + .format(name, top)) from None return obj, parent def whichmodule(obj, name): @@ -396,6 +397,8 @@ def decode_long(data): return int.from_bytes(data, byteorder='little', signed=True) +_NoValue = object() + # Pickling machinery class _Pickler: @@ -530,10 +533,11 @@ def save(self, obj, save_persistent_id=True): self.framer.commit_frame() # Check for persistent id (defined by a subclass) - pid = self.persistent_id(obj) - if pid is not None and save_persistent_id: - self.save_pers(pid) - return + if save_persistent_id: + pid = self.persistent_id(obj) + if pid is not None: + self.save_pers(pid) + return # Check the memo x = self.memo.get(id(obj)) @@ -542,8 +546,8 @@ def save(self, obj, save_persistent_id=True): return rv = NotImplemented - reduce = getattr(self, "reducer_override", None) - if reduce is not None: + reduce = getattr(self, "reducer_override", _NoValue) + if reduce is not _NoValue: rv = reduce(obj) if rv is NotImplemented: @@ -556,8 +560,8 @@ def save(self, obj, save_persistent_id=True): # Check private dispatch table if any, or else # copyreg.dispatch_table - reduce = getattr(self, 'dispatch_table', dispatch_table).get(t) - if reduce is not None: + reduce = getattr(self, 'dispatch_table', dispatch_table).get(t, _NoValue) + if reduce is not _NoValue: rv = reduce(obj) else: # Check for a class with a custom metaclass; treat as regular @@ -567,12 +571,12 @@ def save(self, obj, save_persistent_id=True): return # Check for a __reduce_ex__ method, fall back to __reduce__ - reduce = getattr(obj, "__reduce_ex__", None) - if reduce is not None: + reduce = getattr(obj, "__reduce_ex__", _NoValue) + if reduce is not _NoValue: rv = reduce(self.proto) else: - reduce = getattr(obj, "__reduce__", None) - if reduce is not None: + reduce = getattr(obj, "__reduce__", _NoValue) + if reduce is not _NoValue: rv = reduce() else: raise PicklingError("Can't pickle %r object: %r" % @@ -780,14 +784,10 @@ def save_float(self, obj): self.write(FLOAT + repr(obj).encode("ascii") + b'\n') dispatch[float] = save_float - def save_bytes(self, obj): - if self.proto < 3: - if not obj: # bytes object is empty - self.save_reduce(bytes, (), obj=obj) - else: - self.save_reduce(codecs.encode, - (str(obj, 'latin1'), 'latin1'), obj=obj) - return + def _save_bytes_no_memo(self, obj): + # helper for writing bytes objects for protocol >= 3 + # without memoizing them + assert self.proto >= 3 n = len(obj) if n <= 0xff: self.write(SHORT_BINBYTES + pack("= 5 + # without memoizing them + assert self.proto >= 5 + n = len(obj) + if n >= self.framer._FRAME_SIZE_TARGET: + self._write_large_bytes(BYTEARRAY8 + pack("= self.framer._FRAME_SIZE_TARGET: - self._write_large_bytes(BYTEARRAY8 + pack("= 5") with obj.raw() as m: if not m.contiguous: @@ -830,10 +846,18 @@ def save_picklebuffer(self, obj): if in_band: # Write data in-band # XXX The C implementation avoids a copy here + buf = m.tobytes() + in_memo = id(buf) in self.memo if m.readonly: - self.save_bytes(m.tobytes()) + if in_memo: + self._save_bytes_no_memo(buf) + else: + self.save_bytes(buf) else: - self.save_bytearray(m.tobytes()) + if in_memo: + self._save_bytearray_no_memo(buf) + else: + self.save_bytearray(buf) else: # Write data out-of-band self.write(NEXT_BUFFER) @@ -1070,11 +1094,16 @@ def save_global(self, obj, name=None): (obj, module_name, name)) if self.proto >= 2: - code = _extension_registry.get((module_name, name)) - if code: - assert code > 0 + code = _extension_registry.get((module_name, name), _NoValue) + if code is not _NoValue: if code <= 0xff: - write(EXT1 + pack("= 3: - write(GLOBAL + bytes(module_name, "utf-8") + b'\n' + - bytes(name, "utf-8") + b'\n') + elif '.' in name: + # In protocol < 4, objects with multi-part __qualname__ + # are represented as + # getattr(getattr(..., attrname1), attrname2). + dotted_path = name.split('.') + name = dotted_path.pop(0) + save = self.save + for attrname in dotted_path: + save(getattr) + if self.proto < 2: + write(MARK) + self._save_toplevel_by_name(module_name, name) + for attrname in dotted_path: + save(attrname) + if self.proto < 2: + write(TUPLE) + else: + write(TUPLE2) + write(REDUCE) + else: + self._save_toplevel_by_name(module_name, name) + + self.memoize(obj) + + def _save_toplevel_by_name(self, module_name, name): + if self.proto >= 3: + # Non-ASCII identifiers are supported only with protocols >= 3. + self.write(GLOBAL + bytes(module_name, "utf-8") + b'\n' + + bytes(name, "utf-8") + b'\n') else: if self.fix_imports: r_name_mapping = _compat_pickle.REVERSE_NAME_MAPPING @@ -1102,14 +1155,12 @@ def save_global(self, obj, name=None): elif module_name in r_import_mapping: module_name = r_import_mapping[module_name] try: - write(GLOBAL + bytes(module_name, "ascii") + b'\n' + - bytes(name, "ascii") + b'\n') + self.write(GLOBAL + bytes(module_name, "ascii") + b'\n' + + bytes(name, "ascii") + b'\n') except UnicodeEncodeError: raise PicklingError( "can't pickle global identifier '%s.%s' using " - "pickle protocol %i" % (module, name, self.proto)) from None - - self.memoize(obj) + "pickle protocol %i" % (module_name, name, self.proto)) from None def save_type(self, obj): if obj is type(None): @@ -1546,9 +1597,8 @@ def load_ext4(self): dispatch[EXT4[0]] = load_ext4 def get_extension(self, code): - nil = [] - obj = _extension_cache.get(code, nil) - if obj is not nil: + obj = _extension_cache.get(code, _NoValue) + if obj is not _NoValue: self.append(obj) return key = _inverted_registry.get(code) @@ -1705,8 +1755,8 @@ def load_build(self): stack = self.stack state = stack.pop() inst = stack[-1] - setstate = getattr(inst, "__setstate__", None) - if setstate is not None: + setstate = getattr(inst, "__setstate__", _NoValue) + if setstate is not _NoValue: setstate(state) return slotstate = None diff --git a/Lib/pickletools.py b/Lib/pickletools.py index 51ee4a7a26..33a51492ea 100644 --- a/Lib/pickletools.py +++ b/Lib/pickletools.py @@ -312,7 +312,7 @@ def read_uint8(f): doc="Eight-byte unsigned integer, little-endian.") -def read_stringnl(f, decode=True, stripquotes=True): +def read_stringnl(f, decode=True, stripquotes=True, *, encoding='latin-1'): r""" >>> import io >>> read_stringnl(io.BytesIO(b"'abcd'\nefg\n")) @@ -356,7 +356,7 @@ def read_stringnl(f, decode=True, stripquotes=True): raise ValueError("no string quotes around %r" % data) if decode: - data = codecs.escape_decode(data)[0].decode("ascii") + data = codecs.escape_decode(data)[0].decode(encoding) return data stringnl = ArgumentDescriptor( @@ -370,7 +370,7 @@ def read_stringnl(f, decode=True, stripquotes=True): """) def read_stringnl_noescape(f): - return read_stringnl(f, stripquotes=False) + return read_stringnl(f, stripquotes=False, encoding='utf-8') stringnl_noescape = ArgumentDescriptor( name='stringnl_noescape', @@ -2513,7 +2513,10 @@ def dis(pickle, out=None, memo=None, indentlevel=4, annotate=0): # make a mild effort to align arguments line += ' ' * (10 - len(opcode.name)) if arg is not None: - line += ' ' + repr(arg) + if opcode.name in ("STRING", "BINSTRING", "SHORT_BINSTRING"): + line += ' ' + ascii(arg) + else: + line += ' ' + repr(arg) if markmsg: line += ' ' + markmsg if annotate: diff --git a/Lib/platform.py b/Lib/platform.py index 58b66078e1..c64c6d2c6f 100755 --- a/Lib/platform.py +++ b/Lib/platform.py @@ -1080,7 +1080,7 @@ def _sys_version(sys_version=None): match.groups() # XXX: RUSTPYTHON support - if "rustc" in sys_version: + if "RustPython" in sys_version: name = "RustPython" else: name = 'CPython' diff --git a/Lib/posixpath.py b/Lib/posixpath.py index e4f155e41a..80561ae7e5 100644 --- a/Lib/posixpath.py +++ b/Lib/posixpath.py @@ -22,6 +22,7 @@ altsep = None devnull = '/dev/null' +import errno import os import sys import stat @@ -35,7 +36,7 @@ "samefile","sameopenfile","samestat", "curdir","pardir","sep","pathsep","defpath","altsep","extsep", "devnull","realpath","supports_unicode_filenames","relpath", - "commonpath", "isjunction"] + "commonpath", "isjunction","isdevdrive","ALLOW_MISSING"] def _get_sep(path): @@ -77,12 +78,11 @@ def join(a, *p): sep = _get_sep(a) path = a try: - if not p: - path[:0] + sep #23780: Ensure compatible data type even if p is null. - for b in map(os.fspath, p): - if b.startswith(sep): + for b in p: + b = os.fspath(b) + if b.startswith(sep) or not path: path = b - elif not path or path.endswith(sep): + elif path.endswith(sep): path += b else: path += sep + b @@ -135,33 +135,30 @@ def splitdrive(p): return p[:0], p -def splitroot(p): - """Split a pathname into drive, root and tail. On Posix, drive is always - empty; the root may be empty, a single slash, or two slashes. The tail - contains anything after the root. For example: +try: + from posix import _path_splitroot_ex as splitroot +except ImportError: + def splitroot(p): + """Split a pathname into drive, root and tail. - splitroot('foo/bar') == ('', '', 'foo/bar') - splitroot('/foo/bar') == ('', '/', 'foo/bar') - splitroot('//foo/bar') == ('', '//', 'foo/bar') - splitroot('///foo/bar') == ('', '/', '//foo/bar') - """ - p = os.fspath(p) - if isinstance(p, bytes): - sep = b'/' - empty = b'' - else: - sep = '/' - empty = '' - if p[:1] != sep: - # Relative path, e.g.: 'foo' - return empty, empty, p - elif p[1:2] != sep or p[2:3] == sep: - # Absolute path, e.g.: '/foo', '///foo', '////foo', etc. - return empty, sep, p[1:] - else: - # Precisely two leading slashes, e.g.: '//foo'. Implementation defined per POSIX, see - # https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap04.html#tag_04_13 - return empty, p[:2], p[2:] + The tail contains anything after the root.""" + p = os.fspath(p) + if isinstance(p, bytes): + sep = b'/' + empty = b'' + else: + sep = '/' + empty = '' + if p[:1] != sep: + # Relative path, e.g.: 'foo' + return empty, empty, p + elif p[1:2] != sep or p[2:3] == sep: + # Absolute path, e.g.: '/foo', '///foo', '////foo', etc. + return empty, sep, p[1:] + else: + # Precisely two leading slashes, e.g.: '//foo'. Implementation defined per POSIX, see + # https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap04.html#tag_04_13 + return empty, p[:2], p[2:] # Return the tail (basename) part of a path, same as split(path)[1]. @@ -187,26 +184,6 @@ def dirname(p): return head -# Is a path a junction? - -def isjunction(path): - """Test whether a path is a junction - Junctions are not a part of posix semantics""" - os.fspath(path) - return False - - -# Being true for dangling symbolic links is also useful. - -def lexists(path): - """Test whether a path exists. Returns True for broken symbolic links""" - try: - os.lstat(path) - except (OSError, ValueError): - return False - return True - - # Is a path a mount point? # (Does this work for all UNIXes? Is it even guaranteed to work by Posix?) @@ -227,21 +204,17 @@ def ismount(path): parent = join(path, b'..') else: parent = join(path, '..') - parent = realpath(parent) try: s2 = os.lstat(parent) - except (OSError, ValueError): - return False + except OSError: + parent = realpath(parent) + try: + s2 = os.lstat(parent) + except OSError: + return False - dev1 = s1.st_dev - dev2 = s2.st_dev - if dev1 != dev2: - return True # path/.. on a different device as path - ino1 = s1.st_ino - ino2 = s2.st_ino - if ino1 == ino2: - return True # path/.. is the same i-node as path - return False + # path/.. on a different device as path or the same i-node as path + return s1.st_dev != s2.st_dev or s1.st_ino == s2.st_ino # Expand paths beginning with '~' or '~user'. @@ -290,7 +263,7 @@ def expanduser(path): return path name = path[1:i] if isinstance(name, bytes): - name = str(name, 'ASCII') + name = os.fsdecode(name) try: pwent = pwd.getpwnam(name) except KeyError: @@ -303,11 +276,8 @@ def expanduser(path): return path if isinstance(path, bytes): userhome = os.fsencode(userhome) - root = b'/' - else: - root = '/' - userhome = userhome.rstrip(root) - return (userhome + path[i:]) or root + userhome = userhome.rstrip(sep) + return (userhome + path[i:]) or sep # Expand paths containing shell variable substitutions. @@ -371,7 +341,7 @@ def expandvars(path): # if it contains symbolic links! try: - from posix import _path_normpath + from posix import _path_normpath as normpath except ImportError: def normpath(path): @@ -379,21 +349,19 @@ def normpath(path): path = os.fspath(path) if isinstance(path, bytes): sep = b'/' - empty = b'' dot = b'.' dotdot = b'..' else: sep = '/' - empty = '' dot = '.' dotdot = '..' - if path == empty: + if not path: return dot _, initial_slashes, path = splitroot(path) comps = path.split(sep) new_comps = [] for comp in comps: - if comp in (empty, dot): + if not comp or comp == dot: continue if (comp != dotdot or (not initial_slashes and not new_comps) or (new_comps and new_comps[-1] == dotdot)): @@ -404,24 +372,16 @@ def normpath(path): path = initial_slashes + sep.join(comps) return path or dot -else: - def normpath(path): - """Normalize path, eliminating double slashes, etc.""" - path = os.fspath(path) - if isinstance(path, bytes): - return os.fsencode(_path_normpath(os.fsdecode(path))) or b"." - return _path_normpath(path) or "." - def abspath(path): """Return an absolute path.""" path = os.fspath(path) - if not isabs(path): - if isinstance(path, bytes): - cwd = os.getcwdb() - else: - cwd = os.getcwd() - path = join(cwd, path) + if isinstance(path, bytes): + if not path.startswith(b'/'): + path = join(os.getcwdb(), path) + else: + if not path.startswith('/'): + path = join(os.getcwd(), path) return normpath(path) @@ -432,72 +392,109 @@ def realpath(filename, *, strict=False): """Return the canonical path of the specified filename, eliminating any symbolic links encountered in the path.""" filename = os.fspath(filename) - path, ok = _joinrealpath(filename[:0], filename, strict, {}) - return abspath(path) - -# Join two paths, normalizing and eliminating any symbolic links -# encountered in the second path. -def _joinrealpath(path, rest, strict, seen): - if isinstance(path, bytes): + if isinstance(filename, bytes): sep = b'/' curdir = b'.' pardir = b'..' + getcwd = os.getcwdb else: sep = '/' curdir = '.' pardir = '..' - - if isabs(rest): - rest = rest[1:] - path = sep - - while rest: - name, _, rest = rest.partition(sep) + getcwd = os.getcwd + if strict is ALLOW_MISSING: + ignored_error = FileNotFoundError + strict = True + elif strict: + ignored_error = () + else: + ignored_error = OSError + + maxlinks = None + + # The stack of unresolved path parts. When popped, a special value of None + # indicates that a symlink target has been resolved, and that the original + # symlink path can be retrieved by popping again. The [::-1] slice is a + # very fast way of spelling list(reversed(...)). + rest = filename.split(sep)[::-1] + + # Number of unprocessed parts in 'rest'. This can differ from len(rest) + # later, because 'rest' might contain markers for unresolved symlinks. + part_count = len(rest) + + # The resolved path, which is absolute throughout this function. + # Note: getcwd() returns a normalized and symlink-free path. + path = sep if filename.startswith(sep) else getcwd() + + # Mapping from symlink paths to *fully resolved* symlink targets. If a + # symlink is encountered but not yet resolved, the value is None. This is + # used both to detect symlink loops and to speed up repeated traversals of + # the same links. + seen = {} + + while part_count: + name = rest.pop() + if name is None: + # resolved symlink target + seen[rest.pop()] = path + continue + part_count -= 1 if not name or name == curdir: # current dir continue if name == pardir: # parent dir - if path: - path, name = split(path) - if name == pardir: - path = join(path, pardir, pardir) - else: - path = pardir + path = path[:path.rindex(sep)] or sep continue - newpath = join(path, name) + if path == sep: + newpath = path + name + else: + newpath = path + sep + name try: - st = os.lstat(newpath) - except OSError: - if strict: - raise - is_link = False + st_mode = os.lstat(newpath).st_mode + if not stat.S_ISLNK(st_mode): + if strict and part_count and not stat.S_ISDIR(st_mode): + raise OSError(errno.ENOTDIR, os.strerror(errno.ENOTDIR), + newpath) + path = newpath + continue + if newpath in seen: + # Already seen this path + path = seen[newpath] + if path is not None: + # use cached value + continue + # The symlink is not resolved, so we must have a symlink loop. + if strict: + # Raise OSError(errno.ELOOP) + os.stat(newpath) + path = newpath + continue + target = os.readlink(newpath) + except ignored_error: + pass else: - is_link = stat.S_ISLNK(st.st_mode) - if not is_link: - path = newpath + # Resolve the symbolic link + if target.startswith(sep): + # Symlink target is absolute; reset resolved path. + path = sep + if maxlinks is None: + # Mark this symlink as seen but not fully resolved. + seen[newpath] = None + # Push the symlink path onto the stack, and signal its specialness + # by also pushing None. When these entries are popped, we'll + # record the fully-resolved symlink target in the 'seen' mapping. + rest.append(newpath) + rest.append(None) + # Push the unresolved symlink target parts onto the stack. + target_parts = target.split(sep)[::-1] + rest.extend(target_parts) + part_count += len(target_parts) continue - # Resolve the symbolic link - if newpath in seen: - # Already seen this path - path = seen[newpath] - if path is not None: - # use cached value - continue - # The symlink is not resolved, so we must have a symlink loop. - if strict: - # Raise OSError(errno.ELOOP) - os.stat(newpath) - else: - # Return already resolved part + rest of the path unchanged. - return join(newpath, rest), False - seen[newpath] = None # not resolved symlink - path, ok = _joinrealpath(path, os.readlink(newpath), strict, seen) - if not ok: - return join(path, rest), False - seen[newpath] = path # resolved symlink + # An error occurred and was ignored. + path = newpath - return path, True + return path supports_unicode_filenames = (sys.platform == 'darwin') @@ -505,10 +502,10 @@ def _joinrealpath(path, rest, strict, seen): def relpath(path, start=None): """Return a relative version of a path""" + path = os.fspath(path) if not path: raise ValueError("no path specified") - path = os.fspath(path) if isinstance(path, bytes): curdir = b'.' sep = b'/' @@ -524,15 +521,17 @@ def relpath(path, start=None): start = os.fspath(start) try: - start_list = [x for x in abspath(start).split(sep) if x] - path_list = [x for x in abspath(path).split(sep) if x] + start_tail = abspath(start).lstrip(sep) + path_tail = abspath(path).lstrip(sep) + start_list = start_tail.split(sep) if start_tail else [] + path_list = path_tail.split(sep) if path_tail else [] # Work out how much of the filepath is shared by start and path. i = len(commonprefix([start_list, path_list])) rel_list = [pardir] * (len(start_list)-i) + path_list[i:] if not rel_list: return curdir - return join(*rel_list) + return sep.join(rel_list) except (TypeError, AttributeError, BytesWarning, DeprecationWarning): genericpath._check_arg_types('relpath', path, start) raise @@ -546,10 +545,11 @@ def relpath(path, start=None): def commonpath(paths): """Given a sequence of path names, returns the longest common sub-path.""" + paths = tuple(map(os.fspath, paths)) + if not paths: raise ValueError('commonpath() arg is an empty sequence') - paths = tuple(map(os.fspath, paths)) if isinstance(paths[0], bytes): sep = b'/' curdir = b'.' @@ -561,7 +561,7 @@ def commonpath(paths): split_paths = [path.split(sep) for path in paths] try: - isabs, = set(p[:1] == sep for p in paths) + isabs, = {p.startswith(sep) for p in paths} except ValueError: raise ValueError("Can't mix absolute and relative paths") from None diff --git a/Lib/pyclbr.py b/Lib/pyclbr.py new file mode 100644 index 0000000000..37f86995d6 --- /dev/null +++ b/Lib/pyclbr.py @@ -0,0 +1,314 @@ +"""Parse a Python module and describe its classes and functions. + +Parse enough of a Python file to recognize imports and class and +function definitions, and to find out the superclasses of a class. + +The interface consists of a single function: + readmodule_ex(module, path=None) +where module is the name of a Python module, and path is an optional +list of directories where the module is to be searched. If present, +path is prepended to the system search path sys.path. The return value +is a dictionary. The keys of the dictionary are the names of the +classes and functions defined in the module (including classes that are +defined via the from XXX import YYY construct). The values are +instances of classes Class and Function. One special key/value pair is +present for packages: the key '__path__' has a list as its value which +contains the package search path. + +Classes and Functions have a common superclass: _Object. Every instance +has the following attributes: + module -- name of the module; + name -- name of the object; + file -- file in which the object is defined; + lineno -- line in the file where the object's definition starts; + end_lineno -- line in the file where the object's definition ends; + parent -- parent of this object, if any; + children -- nested objects contained in this object. +The 'children' attribute is a dictionary mapping names to objects. + +Instances of Function describe functions with the attributes from _Object, +plus the following: + is_async -- if a function is defined with an 'async' prefix + +Instances of Class describe classes with the attributes from _Object, +plus the following: + super -- list of super classes (Class instances if possible); + methods -- mapping of method names to beginning line numbers. +If the name of a super class is not recognized, the corresponding +entry in the list of super classes is not a class instance but a +string giving the name of the super class. Since import statements +are recognized and imported modules are scanned as well, this +shouldn't happen often. +""" + +import ast +import sys +import importlib.util + +__all__ = ["readmodule", "readmodule_ex", "Class", "Function"] + +_modules = {} # Initialize cache of modules we've seen. + + +class _Object: + "Information about Python class or function." + def __init__(self, module, name, file, lineno, end_lineno, parent): + self.module = module + self.name = name + self.file = file + self.lineno = lineno + self.end_lineno = end_lineno + self.parent = parent + self.children = {} + if parent is not None: + parent.children[name] = self + + +# Odd Function and Class signatures are for back-compatibility. +class Function(_Object): + "Information about a Python function, including methods." + def __init__(self, module, name, file, lineno, + parent=None, is_async=False, *, end_lineno=None): + super().__init__(module, name, file, lineno, end_lineno, parent) + self.is_async = is_async + if isinstance(parent, Class): + parent.methods[name] = lineno + + +class Class(_Object): + "Information about a Python class." + def __init__(self, module, name, super_, file, lineno, + parent=None, *, end_lineno=None): + super().__init__(module, name, file, lineno, end_lineno, parent) + self.super = super_ or [] + self.methods = {} + + +# These 2 functions are used in these tests +# Lib/test/test_pyclbr, Lib/idlelib/idle_test/test_browser.py +def _nest_function(ob, func_name, lineno, end_lineno, is_async=False): + "Return a Function after nesting within ob." + return Function(ob.module, func_name, ob.file, lineno, + parent=ob, is_async=is_async, end_lineno=end_lineno) + +def _nest_class(ob, class_name, lineno, end_lineno, super=None): + "Return a Class after nesting within ob." + return Class(ob.module, class_name, super, ob.file, lineno, + parent=ob, end_lineno=end_lineno) + + +def readmodule(module, path=None): + """Return Class objects for the top-level classes in module. + + This is the original interface, before Functions were added. + """ + + res = {} + for key, value in _readmodule(module, path or []).items(): + if isinstance(value, Class): + res[key] = value + return res + +def readmodule_ex(module, path=None): + """Return a dictionary with all functions and classes in module. + + Search for module in PATH + sys.path. + If possible, include imported superclasses. + Do this by reading source, without importing (and executing) it. + """ + return _readmodule(module, path or []) + + +def _readmodule(module, path, inpackage=None): + """Do the hard work for readmodule[_ex]. + + If inpackage is given, it must be the dotted name of the package in + which we are searching for a submodule, and then PATH must be the + package search path; otherwise, we are searching for a top-level + module, and path is combined with sys.path. + """ + # Compute the full module name (prepending inpackage if set). + if inpackage is not None: + fullmodule = "%s.%s" % (inpackage, module) + else: + fullmodule = module + + # Check in the cache. + if fullmodule in _modules: + return _modules[fullmodule] + + # Initialize the dict for this module's contents. + tree = {} + + # Check if it is a built-in module; we don't do much for these. + if module in sys.builtin_module_names and inpackage is None: + _modules[module] = tree + return tree + + # Check for a dotted module name. + i = module.rfind('.') + if i >= 0: + package = module[:i] + submodule = module[i+1:] + parent = _readmodule(package, path, inpackage) + if inpackage is not None: + package = "%s.%s" % (inpackage, package) + if not '__path__' in parent: + raise ImportError('No package named {}'.format(package)) + return _readmodule(submodule, parent['__path__'], package) + + # Search the path for the module. + f = None + if inpackage is not None: + search_path = path + else: + search_path = path + sys.path + spec = importlib.util._find_spec_from_path(fullmodule, search_path) + if spec is None: + raise ModuleNotFoundError(f"no module named {fullmodule!r}", name=fullmodule) + _modules[fullmodule] = tree + # Is module a package? + if spec.submodule_search_locations is not None: + tree['__path__'] = spec.submodule_search_locations + try: + source = spec.loader.get_source(fullmodule) + except (AttributeError, ImportError): + # If module is not Python source, we cannot do anything. + return tree + else: + if source is None: + return tree + + fname = spec.loader.get_filename(fullmodule) + return _create_tree(fullmodule, path, fname, source, tree, inpackage) + + +class _ModuleBrowser(ast.NodeVisitor): + def __init__(self, module, path, file, tree, inpackage): + self.path = path + self.tree = tree + self.file = file + self.module = module + self.inpackage = inpackage + self.stack = [] + + def visit_ClassDef(self, node): + bases = [] + for base in node.bases: + name = ast.unparse(base) + if name in self.tree: + # We know this super class. + bases.append(self.tree[name]) + elif len(names := name.split(".")) > 1: + # Super class form is module.class: + # look in module for class. + *_, module, class_ = names + if module in _modules: + bases.append(_modules[module].get(class_, name)) + else: + bases.append(name) + + parent = self.stack[-1] if self.stack else None + class_ = Class(self.module, node.name, bases, self.file, node.lineno, + parent=parent, end_lineno=node.end_lineno) + if parent is None: + self.tree[node.name] = class_ + self.stack.append(class_) + self.generic_visit(node) + self.stack.pop() + + def visit_FunctionDef(self, node, *, is_async=False): + parent = self.stack[-1] if self.stack else None + function = Function(self.module, node.name, self.file, node.lineno, + parent, is_async, end_lineno=node.end_lineno) + if parent is None: + self.tree[node.name] = function + self.stack.append(function) + self.generic_visit(node) + self.stack.pop() + + def visit_AsyncFunctionDef(self, node): + self.visit_FunctionDef(node, is_async=True) + + def visit_Import(self, node): + if node.col_offset != 0: + return + + for module in node.names: + try: + try: + _readmodule(module.name, self.path, self.inpackage) + except ImportError: + _readmodule(module.name, []) + except (ImportError, SyntaxError): + # If we can't find or parse the imported module, + # too bad -- don't die here. + continue + + def visit_ImportFrom(self, node): + if node.col_offset != 0: + return + try: + module = "." * node.level + if node.module: + module += node.module + module = _readmodule(module, self.path, self.inpackage) + except (ImportError, SyntaxError): + return + + for name in node.names: + if name.name in module: + self.tree[name.asname or name.name] = module[name.name] + elif name.name == "*": + for import_name, import_value in module.items(): + if import_name.startswith("_"): + continue + self.tree[import_name] = import_value + + +def _create_tree(fullmodule, path, fname, source, tree, inpackage): + mbrowser = _ModuleBrowser(fullmodule, path, fname, tree, inpackage) + mbrowser.visit(ast.parse(source)) + return mbrowser.tree + + +def _main(): + "Print module output (default this file) for quick visual check." + import os + try: + mod = sys.argv[1] + except: + mod = __file__ + if os.path.exists(mod): + path = [os.path.dirname(mod)] + mod = os.path.basename(mod) + if mod.lower().endswith(".py"): + mod = mod[:-3] + else: + path = [] + tree = readmodule_ex(mod, path) + lineno_key = lambda a: getattr(a, 'lineno', 0) + objs = sorted(tree.values(), key=lineno_key, reverse=True) + indent_level = 2 + while objs: + obj = objs.pop() + if isinstance(obj, list): + # Value is a __path__ key. + continue + if not hasattr(obj, 'indent'): + obj.indent = 0 + + if isinstance(obj, _Object): + new_objs = sorted(obj.children.values(), + key=lineno_key, reverse=True) + for ob in new_objs: + ob.indent = obj.indent + indent_level + objs.extend(new_objs) + if isinstance(obj, Class): + print("{}class {} {} {}" + .format(' ' * obj.indent, obj.name, obj.super, obj.lineno)) + elif isinstance(obj, Function): + print("{}def {} {}".format(' ' * obj.indent, obj.name, obj.lineno)) + +if __name__ == "__main__": + _main() diff --git a/Lib/quopri.py b/Lib/quopri.py index 08899c5cb7..f36cf7b395 100755 --- a/Lib/quopri.py +++ b/Lib/quopri.py @@ -67,10 +67,7 @@ def write(s, output=output, lineEnd=b'\n'): output.write(s + lineEnd) prevline = None - while 1: - line = input.readline() - if not line: - break + while line := input.readline(): outline = [] # Strip off any readline induced trailing newline stripped = b'' @@ -126,9 +123,7 @@ def decode(input, output, header=False): return new = b'' - while 1: - line = input.readline() - if not line: break + while line := input.readline(): i, n = 0, len(line) if n > 0 and line[n-1:n] == b'\n': partial = 0; n = n-1 diff --git a/Lib/rlcompleter.py b/Lib/rlcompleter.py index bca4a7bc52..23eb0020f4 100644 --- a/Lib/rlcompleter.py +++ b/Lib/rlcompleter.py @@ -31,7 +31,11 @@ import atexit import builtins +import inspect +import keyword +import re import __main__ +import warnings __all__ = ["Completer"] @@ -85,10 +89,11 @@ def complete(self, text, state): return None if state == 0: - if "." in text: - self.matches = self.attr_matches(text) - else: - self.matches = self.global_matches(text) + with warnings.catch_warnings(action="ignore"): + if "." in text: + self.matches = self.attr_matches(text) + else: + self.matches = self.global_matches(text) try: return self.matches[state] except IndexError: @@ -96,7 +101,13 @@ def complete(self, text, state): def _callable_postfix(self, val, word): if callable(val): - word = word + "(" + word += "(" + try: + if not inspect.signature(val).parameters: + word += ")" + except ValueError: + pass + return word def global_matches(self, text): @@ -106,18 +117,17 @@ def global_matches(self, text): defined in self.namespace that match. """ - import keyword matches = [] seen = {"__builtins__"} n = len(text) - for word in keyword.kwlist: + for word in keyword.kwlist + keyword.softkwlist: if word[:n] == text: seen.add(word) if word in {'finally', 'try'}: word = word + ':' elif word not in {'False', 'None', 'True', 'break', 'continue', 'pass', - 'else'}: + 'else', '_'}: word = word + ' ' matches.append(word) for nspace in [self.namespace, builtins.__dict__]: @@ -139,7 +149,6 @@ def attr_matches(self, text): with a __getattr__ hook is evaluated. """ - import re m = re.match(r"(\w+(\.\w+)*)\.(\w*)", text) if not m: return [] @@ -169,13 +178,20 @@ def attr_matches(self, text): if (word[:n] == attr and not (noprefix and word[:n+1] == noprefix)): match = "%s.%s" % (expr, word) - try: - val = getattr(thisobject, word) - except Exception: - pass # Include even if attribute not set + if isinstance(getattr(type(thisobject), word, None), + property): + # bpo-44752: thisobject.word is a method decorated by + # `@property`. What follows applies a postfix if + # thisobject.word is callable, but know we know that + # this is not callable (because it is a property). + # Also, getattr(thisobject, word) will evaluate the + # property method, which is not desirable. + matches.append(match) + continue + if (value := getattr(thisobject, word, None)) is not None: + matches.append(self._callable_postfix(value, match)) else: - match = self._callable_postfix(val, match) - matches.append(match) + matches.append(match) if matches or not noprefix: break if noprefix == '_': diff --git a/Lib/runpy.py b/Lib/runpy.py index c7d3d8caad..ef54d3282e 100644 --- a/Lib/runpy.py +++ b/Lib/runpy.py @@ -14,18 +14,20 @@ import importlib.machinery # importlib first so we can test #15386 via -m import importlib.util import io -import types import os __all__ = [ "run_module", "run_path", ] +# avoid 'import types' just for ModuleType +ModuleType = type(sys) + class _TempModule(object): """Temporarily replace a module in sys.modules with an empty namespace""" def __init__(self, mod_name): self.mod_name = mod_name - self.module = types.ModuleType(mod_name) + self.module = ModuleType(mod_name) self._saved_module = [] def __enter__(self): @@ -245,17 +247,17 @@ def _get_main_module_details(error=ImportError): sys.modules[main_name] = saved_main -def _get_code_from_file(run_name, fname): +def _get_code_from_file(fname): # Check for a compiled file first from pkgutil import read_code - decoded_path = os.path.abspath(os.fsdecode(fname)) - with io.open_code(decoded_path) as f: + code_path = os.path.abspath(fname) + with io.open_code(code_path) as f: code = read_code(f) if code is None: # That didn't work, so try it as normal source code - with io.open_code(decoded_path) as f: + with io.open_code(code_path) as f: code = compile(f.read(), fname, 'exec') - return code, fname + return code def run_path(path_name, init_globals=None, run_name=None): """Execute code located at the specified filesystem location. @@ -277,17 +279,13 @@ def run_path(path_name, init_globals=None, run_name=None): pkg_name = run_name.rpartition(".")[0] from pkgutil import get_importer importer = get_importer(path_name) - # Trying to avoid importing imp so as to not consume the deprecation warning. - is_NullImporter = False - if type(importer).__module__ == 'imp': - if type(importer).__name__ == 'NullImporter': - is_NullImporter = True - if isinstance(importer, type(None)) or is_NullImporter: + path_name = os.fsdecode(path_name) + if isinstance(importer, type(None)): # Not a valid sys.path entry, so run the code directly # execfile() doesn't help as we want to allow compiled files - code, fname = _get_code_from_file(run_name, path_name) + code = _get_code_from_file(path_name) return _run_module_code(code, init_globals, run_name, - pkg_name=pkg_name, script_name=fname) + pkg_name=pkg_name, script_name=path_name) else: # Finder is defined for path, so add it to # the start of sys.path diff --git a/Lib/secrets.py b/Lib/secrets.py index a546efbdd4..566a09b731 100644 --- a/Lib/secrets.py +++ b/Lib/secrets.py @@ -2,7 +2,7 @@ managing secrets such as account authentication, tokens, and similar. See PEP 506 for more information. -https://www.python.org/dev/peps/pep-0506/ +https://peps.python.org/pep-0506/ """ @@ -13,7 +13,6 @@ import base64 -import binascii from hmac import compare_digest from random import SystemRandom @@ -56,7 +55,7 @@ def token_hex(nbytes=None): 'f9bf78b9a18ce6d46a0cd2b0b86df9da' """ - return binascii.hexlify(token_bytes(nbytes)).decode('ascii') + return token_bytes(nbytes).hex() def token_urlsafe(nbytes=None): """Return a random URL-safe text string, in Base64 encoding. diff --git a/Lib/selectors.py b/Lib/selectors.py index c3b065b522..b8e5f6a4f7 100644 --- a/Lib/selectors.py +++ b/Lib/selectors.py @@ -66,12 +66,16 @@ def __init__(self, selector): def __len__(self): return len(self._selector._fd_to_key) + def get(self, fileobj, default=None): + fd = self._selector._fileobj_lookup(fileobj) + return self._selector._fd_to_key.get(fd, default) + def __getitem__(self, fileobj): - try: - fd = self._selector._fileobj_lookup(fileobj) - return self._selector._fd_to_key[fd] - except KeyError: - raise KeyError("{!r} is not registered".format(fileobj)) from None + fd = self._selector._fileobj_lookup(fileobj) + key = self._selector._fd_to_key.get(fd) + if key is None: + raise KeyError("{!r} is not registered".format(fileobj)) + return key def __iter__(self): return iter(self._selector._fd_to_key) @@ -272,19 +276,6 @@ def close(self): def get_map(self): return self._map - def _key_from_fd(self, fd): - """Return the key associated to a given file descriptor. - - Parameters: - fd -- file descriptor - - Returns: - corresponding key, or None if not found - """ - try: - return self._fd_to_key[fd] - except KeyError: - return None class SelectSelector(_BaseSelectorImpl): @@ -323,17 +314,15 @@ def select(self, timeout=None): r, w, _ = self._select(self._readers, self._writers, [], timeout) except InterruptedError: return ready - r = set(r) - w = set(w) - for fd in r | w: - events = 0 - if fd in r: - events |= EVENT_READ - if fd in w: - events |= EVENT_WRITE - - key = self._key_from_fd(fd) + r = frozenset(r) + w = frozenset(w) + rw = r | w + fd_to_key_get = self._fd_to_key.get + for fd in rw: + key = fd_to_key_get(fd) if key: + events = ((fd in r and EVENT_READ) + | (fd in w and EVENT_WRITE)) ready.append((key, events & key.events)) return ready @@ -350,11 +339,8 @@ def __init__(self): def register(self, fileobj, events, data=None): key = super().register(fileobj, events, data) - poller_events = 0 - if events & EVENT_READ: - poller_events |= self._EVENT_READ - if events & EVENT_WRITE: - poller_events |= self._EVENT_WRITE + poller_events = ((events & EVENT_READ and self._EVENT_READ) + | (events & EVENT_WRITE and self._EVENT_WRITE) ) try: self._selector.register(key.fd, poller_events) except: @@ -380,11 +366,8 @@ def modify(self, fileobj, events, data=None): changed = False if events != key.events: - selector_events = 0 - if events & EVENT_READ: - selector_events |= self._EVENT_READ - if events & EVENT_WRITE: - selector_events |= self._EVENT_WRITE + selector_events = ((events & EVENT_READ and self._EVENT_READ) + | (events & EVENT_WRITE and self._EVENT_WRITE)) try: self._selector.modify(key.fd, selector_events) except: @@ -415,15 +398,13 @@ def select(self, timeout=None): fd_event_list = self._selector.poll(timeout) except InterruptedError: return ready - for fd, event in fd_event_list: - events = 0 - if event & ~self._EVENT_READ: - events |= EVENT_WRITE - if event & ~self._EVENT_WRITE: - events |= EVENT_READ - key = self._key_from_fd(fd) + fd_to_key_get = self._fd_to_key.get + for fd, event in fd_event_list: + key = fd_to_key_get(fd) if key: + events = ((event & ~self._EVENT_READ and EVENT_WRITE) + | (event & ~self._EVENT_WRITE and EVENT_READ)) ready.append((key, events & key.events)) return ready @@ -439,6 +420,9 @@ class PollSelector(_PollLikeSelector): if hasattr(select, 'epoll'): + _NOT_EPOLLIN = ~select.EPOLLIN + _NOT_EPOLLOUT = ~select.EPOLLOUT + class EpollSelector(_PollLikeSelector): """Epoll-based selector.""" _selector_cls = select.epoll @@ -461,22 +445,20 @@ def select(self, timeout=None): # epoll_wait() expects `maxevents` to be greater than zero; # we want to make sure that `select()` can be called when no # FD is registered. - max_ev = max(len(self._fd_to_key), 1) + max_ev = len(self._fd_to_key) or 1 ready = [] try: fd_event_list = self._selector.poll(timeout, max_ev) except InterruptedError: return ready - for fd, event in fd_event_list: - events = 0 - if event & ~select.EPOLLIN: - events |= EVENT_WRITE - if event & ~select.EPOLLOUT: - events |= EVENT_READ - key = self._key_from_fd(fd) + fd_to_key = self._fd_to_key + for fd, event in fd_event_list: + key = fd_to_key.get(fd) if key: + events = ((event & _NOT_EPOLLIN and EVENT_WRITE) + | (event & _NOT_EPOLLOUT and EVENT_READ)) ready.append((key, events & key.events)) return ready @@ -566,17 +548,15 @@ def select(self, timeout=None): kev_list = self._selector.control(None, max_ev, timeout) except InterruptedError: return ready + + fd_to_key_get = self._fd_to_key.get for kev in kev_list: fd = kev.ident flag = kev.filter - events = 0 - if flag == select.KQ_FILTER_READ: - events |= EVENT_READ - if flag == select.KQ_FILTER_WRITE: - events |= EVENT_WRITE - - key = self._key_from_fd(fd) + key = fd_to_key_get(fd) if key: + events = ((flag == select.KQ_FILTER_READ and EVENT_READ) + | (flag == select.KQ_FILTER_WRITE and EVENT_WRITE)) ready.append((key, events & key.events)) return ready diff --git a/Lib/shlex.py b/Lib/shlex.py index 4801a6c1d4..f4821616b6 100644 --- a/Lib/shlex.py +++ b/Lib/shlex.py @@ -305,9 +305,7 @@ def __next__(self): def split(s, comments=False, posix=True): """Split the string *s* using shell-like syntax.""" if s is None: - import warnings - warnings.warn("Passing None for 's' to shlex.split() is deprecated.", - DeprecationWarning, stacklevel=2) + raise ValueError("s argument must not be None") lex = shlex(s, posix=posix) lex.whitespace_split = True if not comments: @@ -335,10 +333,7 @@ def quote(s): def _print_tokens(lexer): - while 1: - tt = lexer.get_token() - if not tt: - break + while tt := lexer.get_token(): print("Token: " + repr(tt)) if __name__ == '__main__': diff --git a/Lib/shutil.py b/Lib/shutil.py index 6803ee3ce6..7df972012c 100644 --- a/Lib/shutil.py +++ b/Lib/shutil.py @@ -10,7 +10,6 @@ import fnmatch import collections import errno -import warnings try: import zlib @@ -48,7 +47,8 @@ COPY_BUFSIZE = 1024 * 1024 if _WINDOWS else 64 * 1024 # This should never be removed, see rationale in: # https://bugs.python.org/issue43743#msg393429 -_USE_CP_SENDFILE = hasattr(os, "sendfile") and sys.platform.startswith("linux") +_USE_CP_SENDFILE = (hasattr(os, "sendfile") + and sys.platform.startswith(("linux", "android"))) _HAS_FCOPYFILE = posix and hasattr(posix, "_fcopyfile") # macOS # CMD defaults in Windows 10 @@ -153,16 +153,14 @@ def _fastcopy_sendfile(fsrc, fdst): err.filename = fsrc.name err.filename2 = fdst.name - # XXX RUSTPYTHON TODO: consistent OSError.errno - if hasattr(err, "errno") and err.errno == errno.ENOTSOCK: + if err.errno == errno.ENOTSOCK: # sendfile() on this platform (probably Linux < 2.6.33) # does not support copies between regular files (only # sockets). _USE_CP_SENDFILE = False raise _GiveupOnFastCopy(err) - # XXX RUSTPYTHON TODO: consistent OSError.errno - if hasattr(err, "errno") and err.errno == errno.ENOSPC: # filesystem is full + if err.errno == errno.ENOSPC: # filesystem is full raise err from None # Give up on first call and if no data was copied. @@ -304,16 +302,17 @@ def copymode(src, dst, *, follow_symlinks=True): sys.audit("shutil.copymode", src, dst) if not follow_symlinks and _islink(src) and os.path.islink(dst): - if os.name == 'nt': - stat_func, chmod_func = os.lstat, os.chmod - elif hasattr(os, 'lchmod'): + if hasattr(os, 'lchmod'): stat_func, chmod_func = os.lstat, os.lchmod else: return else: + stat_func = _stat if os.name == 'nt' and os.path.islink(dst): - dst = os.path.realpath(dst, strict=True) - stat_func, chmod_func = _stat, os.chmod + def chmod_func(*args): + os.chmod(*args, follow_symlinks=True) + else: + chmod_func = os.chmod st = stat_func(src) chmod_func(dst, stat.S_IMODE(st.st_mode)) @@ -388,16 +387,8 @@ def lookup(name): # We must copy extended attributes before the file is (potentially) # chmod()'ed read-only, otherwise setxattr() will error with -EACCES. _copyxattr(src, dst, follow_symlinks=follow) - _chmod = lookup("chmod") - if os.name == 'nt': - if follow: - if os.path.islink(dst): - dst = os.path.realpath(dst, strict=True) - else: - def _chmod(*args, **kwargs): - os.chmod(*args) try: - _chmod(dst, mode, follow_symlinks=follow) + lookup("chmod")(dst, mode, follow_symlinks=follow) except NotImplementedError: # if we got a NotImplementedError, it's because # * follow_symlinks=False, @@ -565,7 +556,7 @@ def copytree(src, dst, symlinks=False, ignore=None, copy_function=copy2, If the optional symlinks flag is true, symbolic links in the source tree result in symbolic links in the destination tree; if it is false, the contents of the files pointed to by symbolic - links are copied. If the file pointed by the symlink doesn't + links are copied. If the file pointed to by the symlink doesn't exist, an exception will be added in the list of errors raised in an Error exception at the end of the copy process. @@ -605,118 +596,115 @@ def copytree(src, dst, symlinks=False, ignore=None, copy_function=copy2, dirs_exist_ok=dirs_exist_ok) if hasattr(os.stat_result, 'st_file_attributes'): - def _rmtree_islink(path): - try: - st = os.lstat(path) - return (stat.S_ISLNK(st.st_mode) or - (st.st_file_attributes & stat.FILE_ATTRIBUTE_REPARSE_POINT - and st.st_reparse_tag == stat.IO_REPARSE_TAG_MOUNT_POINT)) - except OSError: - return False + def _rmtree_islink(st): + return (stat.S_ISLNK(st.st_mode) or + (st.st_file_attributes & stat.FILE_ATTRIBUTE_REPARSE_POINT + and st.st_reparse_tag == stat.IO_REPARSE_TAG_MOUNT_POINT)) else: - def _rmtree_islink(path): - return os.path.islink(path) + def _rmtree_islink(st): + return stat.S_ISLNK(st.st_mode) # version vulnerable to race conditions def _rmtree_unsafe(path, onexc): - try: - with os.scandir(path) as scandir_it: - entries = list(scandir_it) - except OSError as err: - onexc(os.scandir, path, err) - entries = [] - for entry in entries: - fullname = entry.path - try: - is_dir = entry.is_dir(follow_symlinks=False) - except OSError: - is_dir = False - - if is_dir and not entry.is_junction(): + def onerror(err): + if not isinstance(err, FileNotFoundError): + onexc(os.scandir, err.filename, err) + results = os.walk(path, topdown=False, onerror=onerror, followlinks=os._walk_symlinks_as_files) + for dirpath, dirnames, filenames in results: + for name in dirnames: + fullname = os.path.join(dirpath, name) try: - if entry.is_symlink(): - # This can only happen if someone replaces - # a directory with a symlink after the call to - # os.scandir or entry.is_dir above. - raise OSError("Cannot call rmtree on a symbolic link") - except OSError as err: - onexc(os.path.islink, fullname, err) + os.rmdir(fullname) + except FileNotFoundError: continue - _rmtree_unsafe(fullname, onexc) - else: + except OSError as err: + onexc(os.rmdir, fullname, err) + for name in filenames: + fullname = os.path.join(dirpath, name) try: os.unlink(fullname) + except FileNotFoundError: + continue except OSError as err: onexc(os.unlink, fullname, err) try: os.rmdir(path) + except FileNotFoundError: + pass except OSError as err: onexc(os.rmdir, path, err) # Version using fd-based APIs to protect against races -def _rmtree_safe_fd(topfd, path, onexc): +def _rmtree_safe_fd(stack, onexc): + # Each stack item has four elements: + # * func: The first operation to perform: os.lstat, os.close or os.rmdir. + # Walking a directory starts with an os.lstat() to detect symlinks; in + # this case, func is updated before subsequent operations and passed to + # onexc() if an error occurs. + # * dirfd: Open file descriptor, or None if we're processing the top-level + # directory given to rmtree() and the user didn't supply dir_fd. + # * path: Path of file to operate upon. This is passed to onexc() if an + # error occurs. + # * orig_entry: os.DirEntry, or None if we're processing the top-level + # directory given to rmtree(). We used the cached stat() of the entry to + # save a call to os.lstat() when walking subdirectories. + func, dirfd, path, orig_entry = stack.pop() + name = path if orig_entry is None else orig_entry.name try: + if func is os.close: + os.close(dirfd) + return + if func is os.rmdir: + os.rmdir(name, dir_fd=dirfd) + return + + # Note: To guard against symlink races, we use the standard + # lstat()/open()/fstat() trick. + assert func is os.lstat + if orig_entry is None: + orig_st = os.lstat(name, dir_fd=dirfd) + else: + orig_st = orig_entry.stat(follow_symlinks=False) + + func = os.open # For error reporting. + topfd = os.open(name, os.O_RDONLY | os.O_NONBLOCK, dir_fd=dirfd) + + func = os.path.islink # For error reporting. + try: + if not os.path.samestat(orig_st, os.fstat(topfd)): + # Symlinks to directories are forbidden, see GH-46010. + raise OSError("Cannot call rmtree on a symbolic link") + stack.append((os.rmdir, dirfd, path, orig_entry)) + finally: + stack.append((os.close, topfd, path, orig_entry)) + + func = os.scandir # For error reporting. with os.scandir(topfd) as scandir_it: entries = list(scandir_it) - except OSError as err: - err.filename = path - onexc(os.scandir, path, err) - return - for entry in entries: - fullname = os.path.join(path, entry.name) - try: - is_dir = entry.is_dir(follow_symlinks=False) - except OSError: - is_dir = False - else: - if is_dir: - try: - orig_st = entry.stat(follow_symlinks=False) - is_dir = stat.S_ISDIR(orig_st.st_mode) - except OSError as err: - onexc(os.lstat, fullname, err) - continue - if is_dir: + for entry in entries: + fullname = os.path.join(path, entry.name) try: - dirfd = os.open(entry.name, os.O_RDONLY | os.O_NONBLOCK, dir_fd=topfd) - dirfd_closed = False - except OSError as err: - onexc(os.open, fullname, err) - else: - try: - if os.path.samestat(orig_st, os.fstat(dirfd)): - _rmtree_safe_fd(dirfd, fullname, onexc) - try: - os.close(dirfd) - except OSError as err: - # close() should not be retried after an error. - dirfd_closed = True - onexc(os.close, fullname, err) - dirfd_closed = True - try: - os.rmdir(entry.name, dir_fd=topfd) - except OSError as err: - onexc(os.rmdir, fullname, err) - else: - try: - # This can only happen if someone replaces - # a directory with a symlink after the call to - # os.scandir or stat.S_ISDIR above. - raise OSError("Cannot call rmtree on a symbolic " - "link") - except OSError as err: - onexc(os.path.islink, fullname, err) - finally: - if not dirfd_closed: - try: - os.close(dirfd) - except OSError as err: - onexc(os.close, fullname, err) - else: + if entry.is_dir(follow_symlinks=False): + # Traverse into sub-directory. + stack.append((os.lstat, topfd, fullname, entry)) + continue + except FileNotFoundError: + continue + except OSError: + pass try: os.unlink(entry.name, dir_fd=topfd) + except FileNotFoundError: + continue except OSError as err: onexc(os.unlink, fullname, err) + except FileNotFoundError as err: + if orig_entry is None or func is os.close: + err.filename = path + onexc(func, path, err) + except OSError as err: + err.filename = path + onexc(func, path, err) _use_fd_functions = ({os.open, os.stat, os.unlink, os.rmdir} <= os.supports_dir_fd and @@ -769,41 +757,16 @@ def onexc(*args): # While the unsafe rmtree works fine on bytes, the fd based does not. if isinstance(path, bytes): path = os.fsdecode(path) - # Note: To guard against symlink races, we use the standard - # lstat()/open()/fstat() trick. - try: - orig_st = os.lstat(path, dir_fd=dir_fd) - except Exception as err: - onexc(os.lstat, path, err) - return + stack = [(os.lstat, dir_fd, path, None)] try: - fd = os.open(path, os.O_RDONLY | os.O_NONBLOCK, dir_fd=dir_fd) - fd_closed = False - except Exception as err: - onexc(os.open, path, err) - return - try: - if os.path.samestat(orig_st, os.fstat(fd)): - _rmtree_safe_fd(fd, path, onexc) - try: - os.close(fd) - except OSError as err: - # close() should not be retried after an error. - fd_closed = True - onexc(os.close, path, err) - fd_closed = True - try: - os.rmdir(path, dir_fd=dir_fd) - except OSError as err: - onexc(os.rmdir, path, err) - else: - try: - # symlinks to directories are forbidden, see bug #1669 - raise OSError("Cannot call rmtree on a symbolic link") - except OSError as err: - onexc(os.path.islink, path, err) + while stack: + _rmtree_safe_fd(stack, onexc) finally: - if not fd_closed: + # Close any file descriptors still on the stack. + while stack: + func, fd, path, entry = stack.pop() + if func is not os.close: + continue try: os.close(fd) except OSError as err: @@ -812,7 +775,12 @@ def onexc(*args): if dir_fd is not None: raise NotImplementedError("dir_fd unavailable on this platform") try: - if _rmtree_islink(path): + st = os.lstat(path) + except OSError as err: + onexc(os.lstat, path, err) + return + try: + if _rmtree_islink(st): # symlinks to directories are forbidden, see bug #1669 raise OSError("Cannot call rmtree on a symbolic link") except OSError as err: @@ -1428,11 +1396,18 @@ def disk_usage(path): return _ntuple_diskusage(total, used, free) -def chown(path, user=None, group=None): +def chown(path, user=None, group=None, *, dir_fd=None, follow_symlinks=True): """Change owner user and group of the given path. user and group can be the uid/gid or the user/group names, and in that case, they are converted to their respective uid/gid. + + If dir_fd is set, it should be an open file descriptor to the directory to + be used as the root of *path* if it is relative. + + If follow_symlinks is set to False and the last element of the path is a + symbolic link, chown will modify the link itself and not the file being + referenced by the link. """ sys.audit('shutil.chown', path, user, group) @@ -1458,7 +1433,8 @@ def chown(path, user=None, group=None): if _group is None: raise LookupError("no such group: {!r}".format(group)) - os.chown(path, _user, _group) + os.chown(path, _user, _group, dir_fd=dir_fd, + follow_symlinks=follow_symlinks) def get_terminal_size(fallback=(80, 24)): """Get the size of the terminal window. @@ -1575,21 +1551,21 @@ def which(cmd, mode=os.F_OK | os.X_OK, path=None): if sys.platform == "win32": # PATHEXT is necessary to check on Windows. pathext_source = os.getenv("PATHEXT") or _WIN_DEFAULT_PATHEXT - pathext = [ext for ext in pathext_source.split(os.pathsep) if ext] + pathext = pathext_source.split(os.pathsep) + pathext = [ext.rstrip('.') for ext in pathext if ext] if use_bytes: pathext = [os.fsencode(ext) for ext in pathext] - files = ([cmd] + [cmd + ext for ext in pathext]) + files = [cmd + ext for ext in pathext] - # gh-109590. If we are looking for an executable, we need to look - # for a PATHEXT match. The first cmd is the direct match - # (e.g. python.exe instead of python) - # Check that direct match first if and only if the extension is in PATHEXT - # Otherwise check it last - suffix = os.path.splitext(files[0])[1].upper() - if mode & os.X_OK and not any(suffix == ext.upper() for ext in pathext): - files.append(files.pop(0)) + # If X_OK in mode, simulate the cmd.exe behavior: look at direct + # match if and only if the extension is in PATHEXT. + # If X_OK not in mode, simulate the first result of where.exe: + # always look at direct match before a PATHEXT match. + normcmd = cmd.upper() + if not (mode & os.X_OK) or any(normcmd.endswith(ext.upper()) for ext in pathext): + files.insert(0, cmd) else: # On other platforms you don't have things like PATHEXT to tell you # what file suffixes are executable, so just pass on cmd as-is. @@ -1598,7 +1574,7 @@ def which(cmd, mode=os.F_OK | os.X_OK, path=None): seen = set() for dir in path: normdir = os.path.normcase(dir) - if not normdir in seen: + if normdir not in seen: seen.add(normdir) for thefile in files: name = os.path.join(dir, thefile) diff --git a/Lib/site.py b/Lib/site.py index acc8481b13..2983ca7154 100644 --- a/Lib/site.py +++ b/Lib/site.py @@ -75,6 +75,7 @@ import _sitebuiltins import io import stat +import errno # Prefixes for site-packages; add additional prefixes like /usr/local here PREFIXES = [sys.prefix, sys.exec_prefix] @@ -179,35 +180,46 @@ def addpackage(sitedir, name, known_paths): return _trace(f"Processing .pth file: {fullname!r}") try: - # locale encoding is not ideal especially on Windows. But we have used - # it for a long time. setuptools uses the locale encoding too. - f = io.TextIOWrapper(io.open_code(fullname), encoding="locale") + with io.open_code(fullname) as f: + pth_content = f.read() except OSError: return - with f: - for n, line in enumerate(f): - if line.startswith("#"): - continue - if line.strip() == "": + + try: + # Accept BOM markers in .pth files as we do in source files + # (Windows PowerShell 5.1 makes it hard to emit UTF-8 files without a BOM) + pth_content = pth_content.decode("utf-8-sig") + except UnicodeDecodeError: + # Fallback to locale encoding for backward compatibility. + # We will deprecate this fallback in the future. + import locale + pth_content = pth_content.decode(locale.getencoding()) + _trace(f"Cannot read {fullname!r} as UTF-8. " + f"Using fallback encoding {locale.getencoding()!r}") + + for n, line in enumerate(pth_content.splitlines(), 1): + if line.startswith("#"): + continue + if line.strip() == "": + continue + try: + if line.startswith(("import ", "import\t")): + exec(line) continue - try: - if line.startswith(("import ", "import\t")): - exec(line) - continue - line = line.rstrip() - dir, dircase = makepath(sitedir, line) - if not dircase in known_paths and os.path.exists(dir): - sys.path.append(dir) - known_paths.add(dircase) - except Exception as exc: - print("Error processing line {:d} of {}:\n".format(n+1, fullname), - file=sys.stderr) - import traceback - for record in traceback.format_exception(exc): - for line in record.splitlines(): - print(' '+line, file=sys.stderr) - print("\nRemainder of file ignored", file=sys.stderr) - break + line = line.rstrip() + dir, dircase = makepath(sitedir, line) + if dircase not in known_paths and os.path.exists(dir): + sys.path.append(dir) + known_paths.add(dircase) + except Exception as exc: + print(f"Error processing line {n:d} of {fullname}:\n", + file=sys.stderr) + import traceback + for record in traceback.format_exception(exc): + for line in record.splitlines(): + print(' '+line, file=sys.stderr) + print("\nRemainder of file ignored", file=sys.stderr) + break if reset: known_paths = None return known_paths @@ -270,14 +282,18 @@ def check_enableusersite(): # # See https://bugs.python.org/issue29585 +# Copy of sysconfig._get_implementation() +def _get_implementation(): + return 'RustPython' # XXX: RustPython; for site-packages + # Copy of sysconfig._getuserbase() def _getuserbase(): env_base = os.environ.get("PYTHONUSERBASE", None) if env_base: return env_base - # Emscripten, VxWorks, and WASI have no home directories - if sys.platform in {"emscripten", "vxworks", "wasi"}: + # Emscripten, iOS, tvOS, VxWorks, WASI, and watchOS have no home directories + if sys.platform in {"emscripten", "ios", "tvos", "vxworks", "wasi", "watchos"}: return None def joinuser(*args): @@ -285,8 +301,7 @@ def joinuser(*args): if os.name == "nt": base = os.environ.get("APPDATA") or "~" - # XXX: RUSTPYTHON; please keep this change for site-packages - return joinuser(base, "RustPython") + return joinuser(base, _get_implementation()) if sys.platform == "darwin" and sys._framework: return joinuser("~", "Library", sys._framework, @@ -298,15 +313,22 @@ def joinuser(*args): # Same to sysconfig.get_path('purelib', os.name+'_user') def _get_path(userbase): version = sys.version_info + if hasattr(sys, 'abiflags') and 't' in sys.abiflags: + abi_thread = 't' + else: + abi_thread = '' + implementation = _get_implementation() + implementation_lower = implementation.lower() if os.name == 'nt': ver_nodot = sys.winver.replace('.', '') - return f'{userbase}\\RustPython{ver_nodot}\\site-packages' + return f'{userbase}\\{implementation}{ver_nodot}\\site-packages' if sys.platform == 'darwin' and sys._framework: - return f'{userbase}/lib/rustpython/site-packages' + return f'{userbase}/lib/{implementation_lower}/site-packages' - return f'{userbase}/lib/rustpython{version[0]}.{version[1]}/site-packages' + # XXX: RUSTPYTHON + return f'{userbase}/lib/rustpython{version[0]}.{version[1]}{abi_thread}/site-packages' def getuserbase(): @@ -372,6 +394,12 @@ def getsitepackages(prefixes=None): continue seen.add(prefix) + implementation = _get_implementation().lower() + ver = sys.version_info + if hasattr(sys, 'abiflags') and 't' in sys.abiflags: + abi_thread = 't' + else: + abi_thread = '' if os.sep == '/': libdirs = [sys.platlibdir] if sys.platlibdir != "lib": @@ -379,8 +407,7 @@ def getsitepackages(prefixes=None): for libdir in libdirs: path = os.path.join(prefix, libdir, - # XXX: RUSTPYTHON; please keep this change for site-packages - "rustpython%d.%d" % sys.version_info[:2], + f"{implementation}{ver[0]}.{ver[1]}{abi_thread}", "site-packages") sitepackages.append(path) else: @@ -417,8 +444,9 @@ def setcopyright(): """Set 'copyright' and 'credits' in builtins""" builtins.copyright = _sitebuiltins._Printer("copyright", sys.copyright) builtins.credits = _sitebuiltins._Printer("credits", """\ - Thanks to CWI, CNRI, BeOpen.com, Zope Corporation and a cast of thousands - for supporting Python development. See www.python.org for more information.""") + Thanks to CWI, CNRI, BeOpen, Zope Corporation, the Python Software + Foundation, and a cast of thousands for supporting Python + development. See www.python.org for more information.""") files, dirs = [], [] # Not all modules are required to have a __file__ attribute. See # PEP 420 for more details. @@ -437,27 +465,76 @@ def setcopyright(): def sethelper(): builtins.help = _sitebuiltins._Helper() + +def gethistoryfile(): + """Check if the PYTHON_HISTORY environment variable is set and define + it as the .python_history file. If PYTHON_HISTORY is not set, use the + default .python_history file. + """ + if not sys.flags.ignore_environment: + history = os.environ.get("PYTHON_HISTORY") + if history: + return history + return os.path.join(os.path.expanduser('~'), + '.python_history') + + def enablerlcompleter(): """Enable default readline configuration on interactive prompts, by registering a sys.__interactivehook__. + """ + sys.__interactivehook__ = register_readline + + +def register_readline(): + """Configure readline completion on interactive prompts. If the readline module can be imported, the hook will set the Tab key as completion key and register ~/.python_history as history file. This can be overridden in the sitecustomize or usercustomize module, or in a PYTHONSTARTUP file. """ - def register_readline(): - import atexit + if not sys.flags.ignore_environment: + PYTHON_BASIC_REPL = os.getenv("PYTHON_BASIC_REPL") + else: + PYTHON_BASIC_REPL = False + + import atexit + + try: try: import readline - import rlcompleter except ImportError: - return + readline = None + else: + import rlcompleter # noqa: F401 + except ImportError: + return + try: + if PYTHON_BASIC_REPL: + CAN_USE_PYREPL = False + else: + original_path = sys.path + sys.path = [p for p in original_path if p != ''] + try: + import _pyrepl.readline + if os.name == "nt": + import _pyrepl.windows_console + console_errors = (_pyrepl.windows_console._error,) + else: + import _pyrepl.unix_console + console_errors = _pyrepl.unix_console._error + from _pyrepl.main import CAN_USE_PYREPL + finally: + sys.path = original_path + except ImportError: + return + + if readline is not None: # Reading the initialization (config) file may not be enough to set a # completion key, so we set one first and then read the file. - readline_doc = getattr(readline, '__doc__', '') - if readline_doc is not None and 'libedit' in readline_doc: + if readline.backend == 'editline': readline.parse_and_bind('bind ^I rl_complete') else: readline.parse_and_bind('tab: complete') @@ -471,30 +548,44 @@ def register_readline(): # want to ignore the exception. pass - if readline.get_current_history_length() == 0: - # If no history was loaded, default to .python_history. - # The guard is necessary to avoid doubling history size at - # each interpreter exit when readline was already configured - # through a PYTHONSTARTUP hook, see: - # http://bugs.python.org/issue5845#msg198636 - history = os.path.join(os.path.expanduser('~'), - '.python_history') + if readline is None or readline.get_current_history_length() == 0: + # If no history was loaded, default to .python_history, + # or PYTHON_HISTORY. + # The guard is necessary to avoid doubling history size at + # each interpreter exit when readline was already configured + # through a PYTHONSTARTUP hook, see: + # http://bugs.python.org/issue5845#msg198636 + history = gethistoryfile() + + if CAN_USE_PYREPL: + readline_module = _pyrepl.readline + exceptions = (OSError, *console_errors) + else: + if readline is None: + return + readline_module = readline + exceptions = OSError + + try: + readline_module.read_history_file(history) + except exceptions: + pass + + def write_history(): try: - readline.read_history_file(history) - except OSError: + readline_module.write_history_file(history) + except (FileNotFoundError, PermissionError): + # home directory does not exist or is not writable + # https://bugs.python.org/issue19891 pass + except OSError: + if errno.EROFS: + pass # gh-128066: read-only file system + else: + raise - def write_history(): - try: - readline.write_history_file(history) - except OSError: - # bpo-19891, bpo-41193: Home directory does not exist - # or is not writable, or the filesystem is read-only. - pass - - atexit.register(write_history) + atexit.register(write_history) - sys.__interactivehook__ = register_readline def venv(known_paths): global PREFIXES, ENABLE_USER_SITE @@ -679,17 +770,5 @@ def exists(path): print(textwrap.dedent(help % (sys.argv[0], os.pathsep))) sys.exit(10) -def gethistoryfile(): - """Check if the PYTHON_HISTORY environment variable is set and define - it as the .python_history file. If PYTHON_HISTORY is not set, use the - default .python_history file. - """ - if not sys.flags.ignore_environment: - history = os.environ.get("PYTHON_HISTORY") - if history: - return history - return os.path.join(os.path.expanduser('~'), - '.python_history') - if __name__ == '__main__': _script() diff --git a/Lib/smtplib.py b/Lib/smtplib.py old mode 100644 new mode 100755 index 912233d817..9b81bcfbc4 --- a/Lib/smtplib.py +++ b/Lib/smtplib.py @@ -171,7 +171,7 @@ def quotedata(data): internet CRLF end-of-line. """ return re.sub(r'(?m)^\.', '..', - re.sub(r'(?:\r\n|\n|\r(?!\n))', CRLF, data)) + re.sub(r'(?:\r\n|\n|\r(?!\n))', CRLF, data)) def _quote_periods(bindata): return re.sub(br'(?m)^\.', b'..', bindata) @@ -179,6 +179,16 @@ def _quote_periods(bindata): def _fix_eols(data): return re.sub(r'(?:\r\n|\n|\r(?!\n))', CRLF, data) + +try: + hmac.digest(b'', b'', 'md5') +# except ValueError: +except (ValueError, AttributeError): # TODO: RUSTPYTHON + _have_cram_md5_support = False +else: + _have_cram_md5_support = True + + try: import ssl except ImportError: @@ -475,7 +485,7 @@ def ehlo(self, name=''): if auth_match: # This doesn't remove duplicates, but that's no problem self.esmtp_features["auth"] = self.esmtp_features.get("auth", "") \ - + " " + auth_match.groups(0)[0] + + " " + auth_match.groups(0)[0] continue # RFC 1869 requires a space between ehlo keyword and parameters. @@ -488,7 +498,7 @@ def ehlo(self, name=''): params = m.string[m.end("feature"):].strip() if feature == "auth": self.esmtp_features[feature] = self.esmtp_features.get(feature, "") \ - + " " + params + + " " + params else: self.esmtp_features[feature] = params return (code, msg) @@ -542,7 +552,7 @@ def mail(self, sender, options=()): raise SMTPNotSupportedError( 'SMTPUTF8 not supported by server') optionlist = ' ' + ' '.join(options) - self.putcmd("mail", "FROM:%s%s" % (quoteaddr(sender), optionlist)) + self.putcmd("mail", "from:%s%s" % (quoteaddr(sender), optionlist)) return self.getreply() def rcpt(self, recip, options=()): @@ -550,7 +560,7 @@ def rcpt(self, recip, options=()): optionlist = '' if options and self.does_esmtp: optionlist = ' ' + ' '.join(options) - self.putcmd("rcpt", "TO:%s%s" % (quoteaddr(recip), optionlist)) + self.putcmd("rcpt", "to:%s%s" % (quoteaddr(recip), optionlist)) return self.getreply() def data(self, msg): @@ -667,8 +677,11 @@ def auth_cram_md5(self, challenge=None): # CRAM-MD5 does not support initial-response. if challenge is None: return None - return self.user + " " + hmac.HMAC( - self.password.encode('ascii'), challenge, 'md5').hexdigest() + if not _have_cram_md5_support: + raise SMTPException("CRAM-MD5 is not supported") + password = self.password.encode('ascii') + authcode = hmac.HMAC(password, challenge, 'md5') + return f"{self.user} {authcode.hexdigest()}" def auth_plain(self, challenge=None): """ Authobject to use with PLAIN authentication. Requires self.user and @@ -720,8 +733,10 @@ def login(self, user, password, *, initial_response_ok=True): advertised_authlist = self.esmtp_features["auth"].split() # Authentication methods we can handle in our preferred order: - preferred_auths = ['CRAM-MD5', 'PLAIN', 'LOGIN'] - + if _have_cram_md5_support: + preferred_auths = ['CRAM-MD5', 'PLAIN', 'LOGIN'] + else: + preferred_auths = ['PLAIN', 'LOGIN'] # We try the supported authentications in our preferred order, if # the server supports them. authlist = [auth for auth in preferred_auths @@ -905,7 +920,7 @@ def send_message(self, msg, from_addr=None, to_addrs=None, The arguments are as for sendmail, except that msg is an email.message.Message object. If from_addr is None or to_addrs is None, these arguments are taken from the headers of the Message as - described in RFC 2822 (a ValueError is raised if there is more than + described in RFC 5322 (a ValueError is raised if there is more than one set of 'Resent-' headers). Regardless of the values of from_addr and to_addr, any Bcc field (or Resent-Bcc field, when the Message is a resent) of the Message object won't be transmitted. The Message @@ -919,7 +934,7 @@ def send_message(self, msg, from_addr=None, to_addrs=None, policy. """ - # 'Resent-Date' is a mandatory field if the Message is resent (RFC 2822 + # 'Resent-Date' is a mandatory field if the Message is resent (RFC 5322 # Section 3.6.6). In such a case, we use the 'Resent-*' fields. However, # if there is more than one 'Resent-' block there's no way to # unambiguously determine which one is the most recent in all cases, @@ -938,10 +953,10 @@ def send_message(self, msg, from_addr=None, to_addrs=None, else: raise ValueError("message has more than one 'Resent-' header block") if from_addr is None: - # Prefer the sender field per RFC 2822:3.6.2. + # Prefer the sender field per RFC 5322 section 3.6.2. from_addr = (msg[header_prefix + 'Sender'] - if (header_prefix + 'Sender') in msg - else msg[header_prefix + 'From']) + if (header_prefix + 'Sender') in msg + else msg[header_prefix + 'From']) from_addr = email.utils.getaddresses([from_addr])[0][1] if to_addrs is None: addr_fields = [f for f in (msg[header_prefix + 'To'], diff --git a/Lib/ssl.py b/Lib/ssl.py index 1200d7d993..c8703b046c 100644 --- a/Lib/ssl.py +++ b/Lib/ssl.py @@ -18,9 +18,10 @@ seconds past the Epoch (the time values returned from time.time()) - fetch_server_certificate (HOST, PORT) -- fetch the certificate provided - by the server running on HOST at port PORT. No - validation of the certificate is performed. + get_server_certificate (addr, ssl_version, ca_certs, timeout) -- Retrieve the + certificate from the server at the specified + address and return it as a PEM-encoded string + Integer constants: @@ -94,31 +95,31 @@ import os from collections import namedtuple from enum import Enum as _Enum, IntEnum as _IntEnum, IntFlag as _IntFlag +from enum import _simple_enum import _ssl # if we can't import it, let the error propagate from _ssl import OPENSSL_VERSION_NUMBER, OPENSSL_VERSION_INFO, OPENSSL_VERSION -from _ssl import _SSLContext#, MemoryBIO, SSLSession +from _ssl import _SSLContext, MemoryBIO, SSLSession from _ssl import ( SSLError, SSLZeroReturnError, SSLWantReadError, SSLWantWriteError, SSLSyscallError, SSLEOFError, SSLCertVerificationError ) from _ssl import txt2obj as _txt2obj, nid2obj as _nid2obj -from _ssl import RAND_status, RAND_add, RAND_bytes, RAND_pseudo_bytes +from _ssl import RAND_status, RAND_add, RAND_bytes try: from _ssl import RAND_egd except ImportError: - # LibreSSL does not provide RAND_egd + # RAND_egd is not supported on some platforms pass from _ssl import ( HAS_SNI, HAS_ECDH, HAS_NPN, HAS_ALPN, HAS_SSLv2, HAS_SSLv3, HAS_TLSv1, - HAS_TLSv1_1, HAS_TLSv1_2, HAS_TLSv1_3 + HAS_TLSv1_1, HAS_TLSv1_2, HAS_TLSv1_3, HAS_PSK ) from _ssl import _DEFAULT_CIPHERS, _OPENSSL_API_VERSION - _IntEnum._convert_( '_SSLMethod', __name__, lambda name: name.startswith('PROTOCOL_') and name != 'PROTOCOL_SSLv23', @@ -155,7 +156,8 @@ _SSLv2_IF_EXISTS = getattr(_SSLMethod, 'PROTOCOL_SSLv2', None) -class TLSVersion(_IntEnum): +@_simple_enum(_IntEnum) +class TLSVersion: MINIMUM_SUPPORTED = _ssl.PROTO_MINIMUM_SUPPORTED SSLv3 = _ssl.PROTO_SSLv3 TLSv1 = _ssl.PROTO_TLSv1 @@ -165,7 +167,8 @@ class TLSVersion(_IntEnum): MAXIMUM_SUPPORTED = _ssl.PROTO_MAXIMUM_SUPPORTED -class _TLSContentType(_IntEnum): +@_simple_enum(_IntEnum) +class _TLSContentType: """Content types (record layer) See RFC 8446, section B.1 @@ -179,7 +182,8 @@ class _TLSContentType(_IntEnum): INNER_CONTENT_TYPE = 0x101 -class _TLSAlertType(_IntEnum): +@_simple_enum(_IntEnum) +class _TLSAlertType: """Alert types for TLSContentType.ALERT messages See RFC 8466, section B.2 @@ -220,7 +224,8 @@ class _TLSAlertType(_IntEnum): NO_APPLICATION_PROTOCOL = 120 -class _TLSMessageType(_IntEnum): +@_simple_enum(_IntEnum) +class _TLSMessageType: """Message types (handshake protocol) See RFC 8446, section B.3 @@ -250,10 +255,10 @@ class _TLSMessageType(_IntEnum): if sys.platform == "win32": - from _ssl import enum_certificates #, enum_crls + from _ssl import enum_certificates, enum_crls -from socket import socket, AF_INET, SOCK_STREAM, create_connection -from socket import SOL_SOCKET, SO_TYPE +from socket import socket, SOCK_STREAM, create_connection +from socket import SOL_SOCKET, SO_TYPE, _GLOBAL_DEFAULT_TIMEOUT import socket as _socket import base64 # for DER-to-PEM translation import errno @@ -275,7 +280,7 @@ class _TLSMessageType(_IntEnum): def _dnsname_match(dn, hostname): """Matching according to RFC 6125, section 6.4.3 - - Hostnames are compared lower case. + - Hostnames are compared lower-case. - For IDNA, both dn and hostname must be encoded as IDN A-label (ACE). - Partial wildcards like 'www*.example.org', multiple wildcards, sole wildcard or wildcards in labels other then the left-most label are not @@ -363,68 +368,11 @@ def _ipaddress_match(cert_ipaddress, host_ip): (section 1.7.2 - "Out of Scope"). """ # OpenSSL may add a trailing newline to a subjectAltName's IP address, - # commonly woth IPv6 addresses. Strip off trailing \n. + # commonly with IPv6 addresses. Strip off trailing \n. ip = _inet_paton(cert_ipaddress.rstrip()) return ip == host_ip -def match_hostname(cert, hostname): - """Verify that *cert* (in decoded format as returned by - SSLSocket.getpeercert()) matches the *hostname*. RFC 2818 and RFC 6125 - rules are followed. - - The function matches IP addresses rather than dNSNames if hostname is a - valid ipaddress string. IPv4 addresses are supported on all platforms. - IPv6 addresses are supported on platforms with IPv6 support (AF_INET6 - and inet_pton). - - CertificateError is raised on failure. On success, the function - returns nothing. - """ - if not cert: - raise ValueError("empty or no certificate, match_hostname needs a " - "SSL socket or SSL context with either " - "CERT_OPTIONAL or CERT_REQUIRED") - try: - host_ip = _inet_paton(hostname) - except ValueError: - # Not an IP address (common case) - host_ip = None - dnsnames = [] - san = cert.get('subjectAltName', ()) - for key, value in san: - if key == 'DNS': - if host_ip is None and _dnsname_match(value, hostname): - return - dnsnames.append(value) - elif key == 'IP Address': - if host_ip is not None and _ipaddress_match(value, host_ip): - return - dnsnames.append(value) - if not dnsnames: - # The subject is only checked when there is no dNSName entry - # in subjectAltName - for sub in cert.get('subject', ()): - for key, value in sub: - # XXX according to RFC 2818, the most specific Common Name - # must be used. - if key == 'commonName': - if _dnsname_match(value, hostname): - return - dnsnames.append(value) - if len(dnsnames) > 1: - raise CertificateError("hostname %r " - "doesn't match either of %s" - % (hostname, ', '.join(map(repr, dnsnames)))) - elif len(dnsnames) == 1: - raise CertificateError("hostname %r " - "doesn't match %r" - % (hostname, dnsnames[0])) - else: - raise CertificateError("no appropriate commonName or " - "subjectAltName fields were found") - - DefaultVerifyPaths = namedtuple("DefaultVerifyPaths", "cafile capath openssl_cafile_env openssl_cafile openssl_capath_env " "openssl_capath") @@ -479,7 +427,14 @@ class SSLContext(_SSLContext): sslsocket_class = None # SSLSocket is assigned later. sslobject_class = None # SSLObject is assigned later. - def __new__(cls, protocol=PROTOCOL_TLS, *args, **kwargs): + def __new__(cls, protocol=None, *args, **kwargs): + if protocol is None: + warnings.warn( + "ssl.SSLContext() without protocol argument is deprecated.", + category=DeprecationWarning, + stacklevel=2 + ) + protocol = PROTOCOL_TLS self = _SSLContext.__new__(cls, protocol) return self @@ -518,6 +473,11 @@ def wrap_bio(self, incoming, outgoing, server_side=False, ) def set_npn_protocols(self, npn_protocols): + warnings.warn( + "ssl NPN is deprecated, use ALPN instead", + DeprecationWarning, + stacklevel=2 + ) protos = bytearray() for protocol in npn_protocols: b = bytes(protocol, 'ascii') @@ -553,18 +513,17 @@ def set_alpn_protocols(self, alpn_protocols): self._set_alpn_protocols(protos) def _load_windows_store_certs(self, storename, purpose): - certs = bytearray() try: for cert, encoding, trust in enum_certificates(storename): # CA certs are never PKCS#7 encoded if encoding == "x509_asn": if trust is True or purpose.oid in trust: - certs.extend(cert) + try: + self.load_verify_locations(cadata=cert) + except SSLError as exc: + warnings.warn(f"Bad certificate in Windows certificate store: {exc!s}") except PermissionError: warnings.warn("unable to enumerate Windows certificate store") - if certs: - self.load_verify_locations(cadata=certs) - return certs def load_default_certs(self, purpose=Purpose.SERVER_AUTH): if not isinstance(purpose, _ASN1Object): @@ -734,12 +693,25 @@ def create_default_context(purpose=Purpose.SERVER_AUTH, *, cafile=None, # SSLContext sets OP_NO_SSLv2, OP_NO_SSLv3, OP_NO_COMPRESSION, # OP_CIPHER_SERVER_PREFERENCE, OP_SINGLE_DH_USE and OP_SINGLE_ECDH_USE # by default. - context = SSLContext(PROTOCOL_TLS) - if purpose == Purpose.SERVER_AUTH: # verify certs and host name in client mode + context = SSLContext(PROTOCOL_TLS_CLIENT) context.verify_mode = CERT_REQUIRED context.check_hostname = True + elif purpose == Purpose.CLIENT_AUTH: + context = SSLContext(PROTOCOL_TLS_SERVER) + else: + raise ValueError(purpose) + + # `VERIFY_X509_PARTIAL_CHAIN` makes OpenSSL's chain building behave more + # like RFC 3280 and 5280, which specify that chain building stops with the + # first trust anchor, even if that anchor is not self-signed. + # + # `VERIFY_X509_STRICT` makes OpenSSL more conservative about the + # certificates it accepts, including "disabling workarounds for + # some broken certificates." + context.verify_flags |= (_ssl.VERIFY_X509_PARTIAL_CHAIN | + _ssl.VERIFY_X509_STRICT) if cafile or capath or cadata: context.load_verify_locations(cafile, capath, cadata) @@ -755,7 +727,7 @@ def create_default_context(purpose=Purpose.SERVER_AUTH, *, cafile=None, context.keylog_filename = keylogfile return context -def _create_unverified_context(protocol=PROTOCOL_TLS, *, cert_reqs=CERT_NONE, +def _create_unverified_context(protocol=None, *, cert_reqs=CERT_NONE, check_hostname=False, purpose=Purpose.SERVER_AUTH, certfile=None, keyfile=None, cafile=None, capath=None, cadata=None): @@ -772,10 +744,18 @@ def _create_unverified_context(protocol=PROTOCOL_TLS, *, cert_reqs=CERT_NONE, # SSLContext sets OP_NO_SSLv2, OP_NO_SSLv3, OP_NO_COMPRESSION, # OP_CIPHER_SERVER_PREFERENCE, OP_SINGLE_DH_USE and OP_SINGLE_ECDH_USE # by default. - context = SSLContext(protocol) + if purpose == Purpose.SERVER_AUTH: + # verify certs and host name in client mode + if protocol is None: + protocol = PROTOCOL_TLS_CLIENT + elif purpose == Purpose.CLIENT_AUTH: + if protocol is None: + protocol = PROTOCOL_TLS_SERVER + else: + raise ValueError(purpose) - if not check_hostname: - context.check_hostname = False + context = SSLContext(protocol) + context.check_hostname = check_hostname if cert_reqs is not None: context.verify_mode = cert_reqs if check_hostname: @@ -905,19 +885,46 @@ def getpeercert(self, binary_form=False): """ return self._sslobj.getpeercert(binary_form) + def get_verified_chain(self): + """Returns verified certificate chain provided by the other + end of the SSL channel as a list of DER-encoded bytes. + + If certificate verification was disabled method acts the same as + ``SSLSocket.get_unverified_chain``. + """ + chain = self._sslobj.get_verified_chain() + + if chain is None: + return [] + + return [cert.public_bytes(_ssl.ENCODING_DER) for cert in chain] + + def get_unverified_chain(self): + """Returns raw certificate chain provided by the other + end of the SSL channel as a list of DER-encoded bytes. + """ + chain = self._sslobj.get_unverified_chain() + + if chain is None: + return [] + + return [cert.public_bytes(_ssl.ENCODING_DER) for cert in chain] + def selected_npn_protocol(self): """Return the currently selected NPN protocol as a string, or ``None`` if a next protocol was not negotiated or if NPN is not supported by one of the peers.""" - if _ssl.HAS_NPN: - return self._sslobj.selected_npn_protocol() + warnings.warn( + "ssl NPN is deprecated, use ALPN instead", + DeprecationWarning, + stacklevel=2 + ) def selected_alpn_protocol(self): """Return the currently selected ALPN protocol as a string, or ``None`` if a next protocol was not negotiated or if ALPN is not supported by one of the peers.""" - if _ssl.HAS_ALPN: - return self._sslobj.selected_alpn_protocol() + return self._sslobj.selected_alpn_protocol() def cipher(self): """Return the currently selected cipher as a 3-tuple ``(name, @@ -996,38 +1003,67 @@ def _create(cls, sock, server_side=False, do_handshake_on_connect=True, if context.check_hostname and not server_hostname: raise ValueError("check_hostname requires server_hostname") + sock_timeout = sock.gettimeout() kwargs = dict( family=sock.family, type=sock.type, proto=sock.proto, fileno=sock.fileno() ) self = cls.__new__(cls, **kwargs) super(SSLSocket, self).__init__(**kwargs) - self.settimeout(sock.gettimeout()) sock.detach() - - self._context = context - self._session = session - self._closed = False - self._sslobj = None - self.server_side = server_side - self.server_hostname = context._encode_hostname(server_hostname) - self.do_handshake_on_connect = do_handshake_on_connect - self.suppress_ragged_eofs = suppress_ragged_eofs - - # See if we are connected + # Now SSLSocket is responsible for closing the file descriptor. try: - self.getpeername() - except OSError as e: - if e.errno != errno.ENOTCONN: - raise - connected = False - else: - connected = True + self._context = context + self._session = session + self._closed = False + self._sslobj = None + self.server_side = server_side + self.server_hostname = context._encode_hostname(server_hostname) + self.do_handshake_on_connect = do_handshake_on_connect + self.suppress_ragged_eofs = suppress_ragged_eofs - self._connected = connected - if connected: - # create the SSL object + # See if we are connected try: + self.getpeername() + except OSError as e: + if e.errno != errno.ENOTCONN: + raise + connected = False + blocking = self.getblocking() + self.setblocking(False) + try: + # We are not connected so this is not supposed to block, but + # testing revealed otherwise on macOS and Windows so we do + # the non-blocking dance regardless. Our raise when any data + # is found means consuming the data is harmless. + notconn_pre_handshake_data = self.recv(1) + except OSError as e: + # EINVAL occurs for recv(1) on non-connected on unix sockets. + if e.errno not in (errno.ENOTCONN, errno.EINVAL): + raise + notconn_pre_handshake_data = b'' + self.setblocking(blocking) + if notconn_pre_handshake_data: + # This prevents pending data sent to the socket before it was + # closed from escaping to the caller who could otherwise + # presume it came through a successful TLS connection. + reason = "Closed before TLS handshake with data in recv buffer." + notconn_pre_handshake_data_error = SSLError(e.errno, reason) + # Add the SSLError attributes that _ssl.c always adds. + notconn_pre_handshake_data_error.reason = reason + notconn_pre_handshake_data_error.library = None + try: + raise notconn_pre_handshake_data_error + finally: + # Explicitly break the reference cycle. + notconn_pre_handshake_data_error = None + else: + connected = True + + self.settimeout(sock_timeout) # Must come after setblocking() calls. + self._connected = connected + if connected: + # create the SSL object self._sslobj = self._context._wrap_socket( self, server_side, self.server_hostname, owner=self, session=self._session, @@ -1038,9 +1074,12 @@ def _create(cls, sock, server_side=False, do_handshake_on_connect=True, # non-blocking raise ValueError("do_handshake_on_connect should not be specified for non-blocking sockets") self.do_handshake() - except (OSError, ValueError): + except: + try: self.close() - raise + except OSError: + pass + raise return self @property @@ -1123,13 +1162,33 @@ def getpeercert(self, binary_form=False): self._check_connected() return self._sslobj.getpeercert(binary_form) + @_sslcopydoc + def get_verified_chain(self): + chain = self._sslobj.get_verified_chain() + + if chain is None: + return [] + + return [cert.public_bytes(_ssl.ENCODING_DER) for cert in chain] + + @_sslcopydoc + def get_unverified_chain(self): + chain = self._sslobj.get_unverified_chain() + + if chain is None: + return [] + + return [cert.public_bytes(_ssl.ENCODING_DER) for cert in chain] + @_sslcopydoc def selected_npn_protocol(self): self._checkClosed() - if self._sslobj is None or not _ssl.HAS_NPN: - return None - else: - return self._sslobj.selected_npn_protocol() + warnings.warn( + "ssl NPN is deprecated, use ALPN instead", + DeprecationWarning, + stacklevel=2 + ) + return None @_sslcopydoc def selected_alpn_protocol(self): @@ -1229,10 +1288,14 @@ def recv(self, buflen=1024, flags=0): def recv_into(self, buffer, nbytes=None, flags=0): self._checkClosed() - if buffer and (nbytes is None): - nbytes = len(buffer) - elif nbytes is None: - nbytes = 1024 + if nbytes is None: + if buffer is not None: + with memoryview(buffer) as view: + nbytes = view.nbytes + if not nbytes: + nbytes = 1024 + else: + nbytes = 1024 if self._sslobj is not None: if flags != 0: raise ValueError( @@ -1382,32 +1445,6 @@ def version(self): SSLContext.sslobject_class = SSLObject -def wrap_socket(sock, keyfile=None, certfile=None, - server_side=False, cert_reqs=CERT_NONE, - ssl_version=PROTOCOL_TLS, ca_certs=None, - do_handshake_on_connect=True, - suppress_ragged_eofs=True, - ciphers=None): - - if server_side and not certfile: - raise ValueError("certfile must be specified for server-side " - "operations") - if keyfile and not certfile: - raise ValueError("certfile must be specified") - context = SSLContext(ssl_version) - context.verify_mode = cert_reqs - if ca_certs: - context.load_verify_locations(ca_certs) - if certfile: - context.load_cert_chain(certfile, keyfile) - if ciphers: - context.set_ciphers(ciphers) - return context.wrap_socket( - sock=sock, server_side=server_side, - do_handshake_on_connect=do_handshake_on_connect, - suppress_ragged_eofs=suppress_ragged_eofs - ) - # some utility functions def cert_time_to_seconds(cert_time): @@ -1466,11 +1503,14 @@ def PEM_cert_to_DER_cert(pem_cert_string): d = pem_cert_string.strip()[len(PEM_HEADER):-len(PEM_FOOTER)] return base64.decodebytes(d.encode('ASCII', 'strict')) -def get_server_certificate(addr, ssl_version=PROTOCOL_TLS, ca_certs=None): +def get_server_certificate(addr, ssl_version=PROTOCOL_TLS_CLIENT, + ca_certs=None, timeout=_GLOBAL_DEFAULT_TIMEOUT): """Retrieve the certificate from the server at the specified address, and return it as a PEM-encoded string. If 'ca_certs' is specified, validate the server cert against it. - If 'ssl_version' is specified, use it in the connection attempt.""" + If 'ssl_version' is specified, use it in the connection attempt. + If 'timeout' is specified, use it in the connection attempt. + """ host, port = addr if ca_certs is not None: @@ -1480,8 +1520,8 @@ def get_server_certificate(addr, ssl_version=PROTOCOL_TLS, ca_certs=None): context = _create_stdlib_context(ssl_version, cert_reqs=cert_reqs, cafile=ca_certs) - with create_connection(addr) as sock: - with context.wrap_socket(sock) as sslsock: + with create_connection(addr, timeout=timeout) as sock: + with context.wrap_socket(sock, server_hostname=host) as sslsock: dercert = sslsock.getpeercert(True) return DER_cert_to_PEM_cert(dercert) diff --git a/Lib/stat.py b/Lib/stat.py index fc024db3f4..1b4ed1ebc9 100644 --- a/Lib/stat.py +++ b/Lib/stat.py @@ -110,22 +110,30 @@ def S_ISWHT(mode): S_IXOTH = 0o0001 # execute by others # Names for file flags - +UF_SETTABLE = 0x0000ffff # owner settable flags UF_NODUMP = 0x00000001 # do not dump file UF_IMMUTABLE = 0x00000002 # file may not be changed UF_APPEND = 0x00000004 # file may only be appended to UF_OPAQUE = 0x00000008 # directory is opaque when viewed through a union stack UF_NOUNLINK = 0x00000010 # file may not be renamed or deleted -UF_COMPRESSED = 0x00000020 # OS X: file is hfs-compressed -UF_HIDDEN = 0x00008000 # OS X: file should not be displayed +UF_COMPRESSED = 0x00000020 # macOS: file is compressed +UF_TRACKED = 0x00000040 # macOS: used for handling document IDs +UF_DATAVAULT = 0x00000080 # macOS: entitlement needed for I/O +UF_HIDDEN = 0x00008000 # macOS: file should not be displayed +SF_SETTABLE = 0xffff0000 # superuser settable flags SF_ARCHIVED = 0x00010000 # file may be archived SF_IMMUTABLE = 0x00020000 # file may not be changed SF_APPEND = 0x00040000 # file may only be appended to +SF_RESTRICTED = 0x00080000 # macOS: entitlement needed for writing SF_NOUNLINK = 0x00100000 # file may not be renamed or deleted SF_SNAPSHOT = 0x00200000 # file is a snapshot file +SF_FIRMLINK = 0x00800000 # macOS: file is a firmlink +SF_DATALESS = 0x40000000 # macOS: file is a dataless object _filemode_table = ( + # File type chars according to: + # http://en.wikibooks.org/wiki/C_Programming/POSIX_Reference/sys/stat.h ((S_IFLNK, "l"), (S_IFSOCK, "s"), # Must appear before IFREG and IFDIR as IFSOCK == IFREG | IFDIR (S_IFREG, "-"), @@ -156,13 +164,17 @@ def S_ISWHT(mode): def filemode(mode): """Convert a file's mode to a string of the form '-rwxrwxrwx'.""" perm = [] - for table in _filemode_table: + for index, table in enumerate(_filemode_table): for bit, char in table: if mode & bit == bit: perm.append(char) break else: - perm.append("-") + if index == 0: + # Unknown filetype + perm.append("?") + else: + perm.append("-") return "".join(perm) diff --git a/Lib/statistics.py b/Lib/statistics.py index f66245380a..ad4a94219c 100644 --- a/Lib/statistics.py +++ b/Lib/statistics.py @@ -11,7 +11,7 @@ Function Description ================== ================================================== mean Arithmetic mean (average) of data. -fmean Fast, floating point arithmetic mean. +fmean Fast, floating-point arithmetic mean. geometric_mean Geometric mean of data. harmonic_mean Harmonic mean of data. median Median (middle value) of data. @@ -112,6 +112,8 @@ 'fmean', 'geometric_mean', 'harmonic_mean', + 'kde', + 'kde_random', 'linear_regression', 'mean', 'median', @@ -130,14 +132,20 @@ import math import numbers import random +import sys from fractions import Fraction from decimal import Decimal -from itertools import groupby, repeat +from itertools import count, groupby, repeat from bisect import bisect_left, bisect_right -from math import hypot, sqrt, fabs, exp, erf, tau, log, fsum +from math import hypot, sqrt, fabs, exp, erf, tau, log, fsum, sumprod +from math import isfinite, isinf, pi, cos, sin, tan, cosh, asin, atan, acos +from functools import reduce from operator import itemgetter -from collections import Counter, namedtuple +from collections import Counter, namedtuple, defaultdict + +_SQRT2 = sqrt(2.0) +_random = random # === Exceptions === @@ -180,11 +188,12 @@ def _sum(data): allowed. """ count = 0 + types = set() + types_add = types.add partials = {} partials_get = partials.get - T = int for typ, values in groupby(data, type): - T = _coerce(T, typ) # or raise TypeError + types_add(typ) for n, d in map(_exact_ratio, values): count += 1 partials[d] = partials_get(d, 0) + n @@ -196,9 +205,51 @@ def _sum(data): else: # Sum all the partial sums using builtin sum. total = sum(Fraction(n, d) for d, n in partials.items()) + T = reduce(_coerce, types, int) # or raise TypeError return (T, total, count) +def _ss(data, c=None): + """Return the exact mean and sum of square deviations of sequence data. + + Calculations are done in a single pass, allowing the input to be an iterator. + + If given *c* is used the mean; otherwise, it is calculated from the data. + Use the *c* argument with care, as it can lead to garbage results. + + """ + if c is not None: + T, ssd, count = _sum((d := x - c) * d for x in data) + return (T, ssd, c, count) + count = 0 + types = set() + types_add = types.add + sx_partials = defaultdict(int) + sxx_partials = defaultdict(int) + for typ, values in groupby(data, type): + types_add(typ) + for n, d in map(_exact_ratio, values): + count += 1 + sx_partials[d] += n + sxx_partials[d] += n * n + if not count: + ssd = c = Fraction(0) + elif None in sx_partials: + # The sum will be a NAN or INF. We can ignore all the finite + # partials, and just look at this special one. + ssd = c = sx_partials[None] + assert not _isfinite(ssd) + else: + sx = sum(Fraction(n, d) for d, n in sx_partials.items()) + sxx = sum(Fraction(n, d*d) for d, n in sxx_partials.items()) + # This formula has poor numeric properties for floats, + # but with fractions it is exact. + ssd = (count * sxx - sx * sx) / count + c = sx / count + T = reduce(_coerce, types, int) # or raise TypeError + return (T, ssd, c, count) + + def _isfinite(x): try: return x.is_finite() # Likely a Decimal. @@ -245,6 +296,28 @@ def _exact_ratio(x): x is expected to be an int, Fraction, Decimal or float. """ + + # XXX We should revisit whether using fractions to accumulate exact + # ratios is the right way to go. + + # The integer ratios for binary floats can have numerators or + # denominators with over 300 decimal digits. The problem is more + # acute with decimal floats where the default decimal context + # supports a huge range of exponents from Emin=-999999 to + # Emax=999999. When expanded with as_integer_ratio(), numbers like + # Decimal('3.14E+5000') and Decimal('3.14E-5000') have large + # numerators or denominators that will slow computation. + + # When the integer ratios are accumulated as fractions, the size + # grows to cover the full range from the smallest magnitude to the + # largest. For example, Fraction(3.14E+300) + Fraction(3.14E-300), + # has a 616 digit numerator. Likewise, + # Fraction(Decimal('3.14E+5000')) + Fraction(Decimal('3.14E-5000')) + # has 10,003 digit numerator. + + # This doesn't seem to have been problem in practice, but it is a + # potential pitfall. + try: return x.as_integer_ratio() except AttributeError: @@ -279,22 +352,6 @@ def _convert(value, T): raise -def _find_lteq(a, x): - 'Locate the leftmost value exactly equal to x' - i = bisect_left(a, x) - if i != len(a) and a[i] == x: - return i - raise ValueError - - -def _find_rteq(a, l, x): - 'Locate the rightmost value exactly equal to x' - i = bisect_right(a, x, lo=l) - if i != (len(a) + 1) and a[i - 1] == x: - return i - 1 - raise ValueError - - def _fail_neg(values, errmsg='negative value'): """Iterate over values, failing if any are less than zero.""" for x in values: @@ -303,6 +360,113 @@ def _fail_neg(values, errmsg='negative value'): yield x +def _rank(data, /, *, key=None, reverse=False, ties='average', start=1) -> list[float]: + """Rank order a dataset. The lowest value has rank 1. + + Ties are averaged so that equal values receive the same rank: + + >>> data = [31, 56, 31, 25, 75, 18] + >>> _rank(data) + [3.5, 5.0, 3.5, 2.0, 6.0, 1.0] + + The operation is idempotent: + + >>> _rank([3.5, 5.0, 3.5, 2.0, 6.0, 1.0]) + [3.5, 5.0, 3.5, 2.0, 6.0, 1.0] + + It is possible to rank the data in reverse order so that the + highest value has rank 1. Also, a key-function can extract + the field to be ranked: + + >>> goals = [('eagles', 45), ('bears', 48), ('lions', 44)] + >>> _rank(goals, key=itemgetter(1), reverse=True) + [2.0, 1.0, 3.0] + + Ranks are conventionally numbered starting from one; however, + setting *start* to zero allows the ranks to be used as array indices: + + >>> prize = ['Gold', 'Silver', 'Bronze', 'Certificate'] + >>> scores = [8.1, 7.3, 9.4, 8.3] + >>> [prize[int(i)] for i in _rank(scores, start=0, reverse=True)] + ['Bronze', 'Certificate', 'Gold', 'Silver'] + + """ + # If this function becomes public at some point, more thought + # needs to be given to the signature. A list of ints is + # plausible when ties is "min" or "max". When ties is "average", + # either list[float] or list[Fraction] is plausible. + + # Default handling of ties matches scipy.stats.mstats.spearmanr. + if ties != 'average': + raise ValueError(f'Unknown tie resolution method: {ties!r}') + if key is not None: + data = map(key, data) + val_pos = sorted(zip(data, count()), reverse=reverse) + i = start - 1 + result = [0] * len(val_pos) + for _, g in groupby(val_pos, key=itemgetter(0)): + group = list(g) + size = len(group) + rank = i + (size + 1) / 2 + for value, orig_pos in group: + result[orig_pos] = rank + i += size + return result + + +def _integer_sqrt_of_frac_rto(n: int, m: int) -> int: + """Square root of n/m, rounded to the nearest integer using round-to-odd.""" + # Reference: https://www.lri.fr/~melquion/doc/05-imacs17_1-expose.pdf + a = math.isqrt(n // m) + return a | (a*a*m != n) + + +# For 53 bit precision floats, the bit width used in +# _float_sqrt_of_frac() is 109. +_sqrt_bit_width: int = 2 * sys.float_info.mant_dig + 3 + + +def _float_sqrt_of_frac(n: int, m: int) -> float: + """Square root of n/m as a float, correctly rounded.""" + # See principle and proof sketch at: https://bugs.python.org/msg407078 + q = (n.bit_length() - m.bit_length() - _sqrt_bit_width) // 2 + if q >= 0: + numerator = _integer_sqrt_of_frac_rto(n, m << 2 * q) << q + denominator = 1 + else: + numerator = _integer_sqrt_of_frac_rto(n << -2 * q, m) + denominator = 1 << -q + return numerator / denominator # Convert to float + + +def _decimal_sqrt_of_frac(n: int, m: int) -> Decimal: + """Square root of n/m as a Decimal, correctly rounded.""" + # Premise: For decimal, computing (n/m).sqrt() can be off + # by 1 ulp from the correctly rounded result. + # Method: Check the result, moving up or down a step if needed. + if n <= 0: + if not n: + return Decimal('0.0') + n, m = -n, -m + + root = (Decimal(n) / Decimal(m)).sqrt() + nr, dr = root.as_integer_ratio() + + plus = root.next_plus() + np, dp = plus.as_integer_ratio() + # test: n / m > ((root + plus) / 2) ** 2 + if 4 * n * (dr*dp)**2 > m * (dr*np + dp*nr)**2: + return plus + + minus = root.next_minus() + nm, dm = minus.as_integer_ratio() + # test: n / m < ((root + minus) / 2) ** 2 + if 4 * n * (dr*dm)**2 < m * (dr*nm + dm*nr)**2: + return minus + + return root + + # === Measures of central tendency (averages) === def mean(data): @@ -321,17 +485,13 @@ def mean(data): If ``data`` is empty, StatisticsError will be raised. """ - if iter(data) is data: - data = list(data) - n = len(data) + T, total, n = _sum(data) if n < 1: raise StatisticsError('mean requires at least one data point') - T, total, count = _sum(data) - assert count == n return _convert(total / n, T) -def fmean(data): +def fmean(data, weights=None): """Convert data to floats and compute the arithmetic mean. This runs faster than the mean() function and it always returns a float. @@ -340,29 +500,40 @@ def fmean(data): >>> fmean([3.5, 4.0, 5.25]) 4.25 """ - try: - n = len(data) - except TypeError: - # Handle iterators that do not define __len__(). - n = 0 - def count(iterable): - nonlocal n - for n, x in enumerate(iterable, start=1): - yield x - total = fsum(count(data)) - else: + if weights is None: + try: + n = len(data) + except TypeError: + # Handle iterators that do not define __len__(). + n = 0 + def count(iterable): + nonlocal n + for n, x in enumerate(iterable, start=1): + yield x + data = count(data) total = fsum(data) - try: + if not n: + raise StatisticsError('fmean requires at least one data point') return total / n - except ZeroDivisionError: - raise StatisticsError('fmean requires at least one data point') from None + if not isinstance(weights, (list, tuple)): + weights = list(weights) + try: + num = sumprod(data, weights) + except ValueError: + raise StatisticsError('data and weights must be the same length') + den = fsum(weights) + if not den: + raise StatisticsError('sum of weights must be non-zero') + return num / den def geometric_mean(data): """Convert data to floats and compute the geometric mean. - Raises a StatisticsError if the input dataset is empty, - if it contains a zero, or if it contains a negative value. + Raises a StatisticsError if the input dataset is empty + or if it contains a negative value. + + Returns zero if the product of inputs is zero. No special efforts are made to achieve exact results. (However, this may change in the future.) @@ -370,11 +541,25 @@ def geometric_mean(data): >>> round(geometric_mean([54, 24, 36]), 9) 36.0 """ - try: - return exp(fmean(map(log, data))) - except ValueError: - raise StatisticsError('geometric mean requires a non-empty dataset ' - 'containing positive numbers') from None + n = 0 + found_zero = False + def count_positive(iterable): + nonlocal n, found_zero + for n, x in enumerate(iterable, start=1): + if x > 0.0 or math.isnan(x): + yield x + elif x == 0.0: + found_zero = True + else: + raise StatisticsError('No negative inputs allowed', x) + total = fsum(map(log, count_positive(data))) + if not n: + raise StatisticsError('Must have a non-empty dataset') + if math.isnan(total): + return math.nan + if found_zero: + return math.nan if total == math.inf else 0.0 + return exp(total / n) def harmonic_mean(data, weights=None): @@ -498,58 +683,75 @@ def median_high(data): return data[n // 2] -def median_grouped(data, interval=1): - """Return the 50th percentile (median) of grouped continuous data. +def median_grouped(data, interval=1.0): + """Estimates the median for numeric data binned around the midpoints + of consecutive, fixed-width intervals. - >>> median_grouped([1, 2, 2, 3, 4, 4, 4, 4, 4, 5]) - 3.7 - >>> median_grouped([52, 52, 53, 54]) - 52.5 + The *data* can be any iterable of numeric data with each value being + exactly the midpoint of a bin. At least one value must be present. - This calculates the median as the 50th percentile, and should be - used when your data is continuous and grouped. In the above example, - the values 1, 2, 3, etc. actually represent the midpoint of classes - 0.5-1.5, 1.5-2.5, 2.5-3.5, etc. The middle value falls somewhere in - class 3.5-4.5, and interpolation is used to estimate it. + The *interval* is width of each bin. - Optional argument ``interval`` represents the class interval, and - defaults to 1. Changing the class interval naturally will change the - interpolated 50th percentile value: + For example, demographic information may have been summarized into + consecutive ten-year age groups with each group being represented + by the 5-year midpoints of the intervals: - >>> median_grouped([1, 3, 3, 5, 7], interval=1) - 3.25 - >>> median_grouped([1, 3, 3, 5, 7], interval=2) - 3.5 + >>> demographics = Counter({ + ... 25: 172, # 20 to 30 years old + ... 35: 484, # 30 to 40 years old + ... 45: 387, # 40 to 50 years old + ... 55: 22, # 50 to 60 years old + ... 65: 6, # 60 to 70 years old + ... }) + + The 50th percentile (median) is the 536th person out of the 1071 + member cohort. That person is in the 30 to 40 year old age group. + + The regular median() function would assume that everyone in the + tricenarian age group was exactly 35 years old. A more tenable + assumption is that the 484 members of that age group are evenly + distributed between 30 and 40. For that, we use median_grouped(). + + >>> data = list(demographics.elements()) + >>> median(data) + 35 + >>> round(median_grouped(data, interval=10), 1) + 37.5 + + The caller is responsible for making sure the data points are separated + by exact multiples of *interval*. This is essential for getting a + correct result. The function does not check this precondition. + + Inputs may be any numeric type that can be coerced to a float during + the interpolation step. - This function does not check whether the data points are at least - ``interval`` apart. """ data = sorted(data) n = len(data) - if n == 0: + if not n: raise StatisticsError("no median for empty data") - elif n == 1: - return data[0] + # Find the value at the midpoint. Remember this corresponds to the - # centre of the class interval. + # midpoint of the class interval. x = data[n // 2] - for obj in (x, interval): - if isinstance(obj, (str, bytes)): - raise TypeError('expected number but got %r' % obj) + + # Using O(log n) bisection, find where all the x values occur in the data. + # All x will lie within data[i:j]. + i = bisect_left(data, x) + j = bisect_right(data, x, lo=i) + + # Coerce to floats, raising a TypeError if not possible try: - L = x - interval / 2 # The lower limit of the median interval. - except TypeError: - # Mixed type. For now we just coerce to float. - L = float(x) - float(interval) / 2 - - # Uses bisection search to search for x in data with log(n) time complexity - # Find the position of leftmost occurrence of x in data - l1 = _find_lteq(data, x) - # Find the position of rightmost occurrence of x in data[l1...len(data)] - # Assuming always l1 <= l2 - l2 = _find_rteq(data, l1, x) - cf = l1 - f = l2 - l1 + 1 + interval = float(interval) + x = float(x) + except ValueError: + raise TypeError(f'Value cannot be converted to a float') + + # Interpolate the median using the formula found at: + # https://www.cuemath.com/data/median-of-grouped-data/ + L = x - interval / 2.0 # Lower limit of the median interval + cf = i # Cumulative frequency of the preceding interval + f = j - i # Number of elements in the median internal return L + interval * (n / 2 - cf) / f @@ -596,9 +798,223 @@ def multimode(data): >>> multimode('') [] """ - counts = Counter(iter(data)).most_common() - maxcount, mode_items = next(groupby(counts, key=itemgetter(1)), (0, [])) - return list(map(itemgetter(0), mode_items)) + counts = Counter(iter(data)) + if not counts: + return [] + maxcount = max(counts.values()) + return [value for value, count in counts.items() if count == maxcount] + + +def kde(data, h, kernel='normal', *, cumulative=False): + """Kernel Density Estimation: Create a continuous probability density + function or cumulative distribution function from discrete samples. + + The basic idea is to smooth the data using a kernel function + to help draw inferences about a population from a sample. + + The degree of smoothing is controlled by the scaling parameter h + which is called the bandwidth. Smaller values emphasize local + features while larger values give smoother results. + + The kernel determines the relative weights of the sample data + points. Generally, the choice of kernel shape does not matter + as much as the more influential bandwidth smoothing parameter. + + Kernels that give some weight to every sample point: + + normal (gauss) + logistic + sigmoid + + Kernels that only give weight to sample points within + the bandwidth: + + rectangular (uniform) + triangular + parabolic (epanechnikov) + quartic (biweight) + triweight + cosine + + If *cumulative* is true, will return a cumulative distribution function. + + A StatisticsError will be raised if the data sequence is empty. + + Example + ------- + + Given a sample of six data points, construct a continuous + function that estimates the underlying probability density: + + >>> sample = [-2.1, -1.3, -0.4, 1.9, 5.1, 6.2] + >>> f_hat = kde(sample, h=1.5) + + Compute the area under the curve: + + >>> area = sum(f_hat(x) for x in range(-20, 20)) + >>> round(area, 4) + 1.0 + + Plot the estimated probability density function at + evenly spaced points from -6 to 10: + + >>> for x in range(-6, 11): + ... density = f_hat(x) + ... plot = ' ' * int(density * 400) + 'x' + ... print(f'{x:2}: {density:.3f} {plot}') + ... + -6: 0.002 x + -5: 0.009 x + -4: 0.031 x + -3: 0.070 x + -2: 0.111 x + -1: 0.125 x + 0: 0.110 x + 1: 0.086 x + 2: 0.068 x + 3: 0.059 x + 4: 0.066 x + 5: 0.082 x + 6: 0.082 x + 7: 0.058 x + 8: 0.028 x + 9: 0.009 x + 10: 0.002 x + + Estimate P(4.5 < X <= 7.5), the probability that a new sample value + will be between 4.5 and 7.5: + + >>> cdf = kde(sample, h=1.5, cumulative=True) + >>> round(cdf(7.5) - cdf(4.5), 2) + 0.22 + + References + ---------- + + Kernel density estimation and its application: + https://www.itm-conferences.org/articles/itmconf/pdf/2018/08/itmconf_sam2018_00037.pdf + + Kernel functions in common use: + https://en.wikipedia.org/wiki/Kernel_(statistics)#kernel_functions_in_common_use + + Interactive graphical demonstration and exploration: + https://demonstrations.wolfram.com/KernelDensityEstimation/ + + Kernel estimation of cumulative distribution function of a random variable with bounded support + https://www.econstor.eu/bitstream/10419/207829/1/10.21307_stattrans-2016-037.pdf + + """ + + n = len(data) + if not n: + raise StatisticsError('Empty data sequence') + + if not isinstance(data[0], (int, float)): + raise TypeError('Data sequence must contain ints or floats') + + if h <= 0.0: + raise StatisticsError(f'Bandwidth h must be positive, not {h=!r}') + + match kernel: + + case 'normal' | 'gauss': + sqrt2pi = sqrt(2 * pi) + sqrt2 = sqrt(2) + K = lambda t: exp(-1/2 * t * t) / sqrt2pi + W = lambda t: 1/2 * (1.0 + erf(t / sqrt2)) + support = None + + case 'logistic': + # 1.0 / (exp(t) + 2.0 + exp(-t)) + K = lambda t: 1/2 / (1.0 + cosh(t)) + W = lambda t: 1.0 - 1.0 / (exp(t) + 1.0) + support = None + + case 'sigmoid': + # (2/pi) / (exp(t) + exp(-t)) + c1 = 1 / pi + c2 = 2 / pi + K = lambda t: c1 / cosh(t) + W = lambda t: c2 * atan(exp(t)) + support = None + + case 'rectangular' | 'uniform': + K = lambda t: 1/2 + W = lambda t: 1/2 * t + 1/2 + support = 1.0 + + case 'triangular': + K = lambda t: 1.0 - abs(t) + W = lambda t: t*t * (1/2 if t < 0.0 else -1/2) + t + 1/2 + support = 1.0 + + case 'parabolic' | 'epanechnikov': + K = lambda t: 3/4 * (1.0 - t * t) + W = lambda t: -1/4 * t**3 + 3/4 * t + 1/2 + support = 1.0 + + case 'quartic' | 'biweight': + K = lambda t: 15/16 * (1.0 - t * t) ** 2 + W = lambda t: 3/16 * t**5 - 5/8 * t**3 + 15/16 * t + 1/2 + support = 1.0 + + case 'triweight': + K = lambda t: 35/32 * (1.0 - t * t) ** 3 + W = lambda t: 35/32 * (-1/7*t**7 + 3/5*t**5 - t**3 + t) + 1/2 + support = 1.0 + + case 'cosine': + c1 = pi / 4 + c2 = pi / 2 + K = lambda t: c1 * cos(c2 * t) + W = lambda t: 1/2 * sin(c2 * t) + 1/2 + support = 1.0 + + case _: + raise StatisticsError(f'Unknown kernel name: {kernel!r}') + + if support is None: + + def pdf(x): + n = len(data) + return sum(K((x - x_i) / h) for x_i in data) / (n * h) + + def cdf(x): + n = len(data) + return sum(W((x - x_i) / h) for x_i in data) / n + + else: + + sample = sorted(data) + bandwidth = h * support + + def pdf(x): + nonlocal n, sample + if len(data) != n: + sample = sorted(data) + n = len(data) + i = bisect_left(sample, x - bandwidth) + j = bisect_right(sample, x + bandwidth) + supported = sample[i : j] + return sum(K((x - x_i) / h) for x_i in supported) / (n * h) + + def cdf(x): + nonlocal n, sample + if len(data) != n: + sample = sorted(data) + n = len(data) + i = bisect_left(sample, x - bandwidth) + j = bisect_right(sample, x + bandwidth) + supported = sample[i : j] + return sum((W((x - x_i) / h) for x_i in supported), i) / n + + if cumulative: + cdf.__doc__ = f'CDF estimate with {h=!r} and {kernel=!r}' + return cdf + + else: + pdf.__doc__ = f'PDF estimate with {h=!r} and {kernel=!r}' + return pdf # Notes on methods for computing quantiles @@ -659,7 +1075,10 @@ def quantiles(data, *, n=4, method='exclusive'): data = sorted(data) ld = len(data) if ld < 2: - raise StatisticsError('must have at least two data points') + if ld == 1: + return data * (n - 1) + raise StatisticsError('must have at least one data point') + if method == 'inclusive': m = ld - 1 result = [] @@ -668,6 +1087,7 @@ def quantiles(data, *, n=4, method='exclusive'): interpolated = (data[j] * (n - delta) + data[j + 1] * delta) / n result.append(interpolated) return result + if method == 'exclusive': m = ld + 1 result = [] @@ -678,6 +1098,7 @@ def quantiles(data, *, n=4, method='exclusive'): interpolated = (data[j - 1] * (n - delta) + data[j] * delta) / n result.append(interpolated) return result + raise ValueError(f'Unknown method: {method!r}') @@ -685,41 +1106,6 @@ def quantiles(data, *, n=4, method='exclusive'): # See http://mathworld.wolfram.com/Variance.html # http://mathworld.wolfram.com/SampleVariance.html -# http://en.wikipedia.org/wiki/Algorithms_for_calculating_variance -# -# Under no circumstances use the so-called "computational formula for -# variance", as that is only suitable for hand calculations with a small -# amount of low-precision data. It has terrible numeric properties. -# -# See a comparison of three computational methods here: -# http://www.johndcook.com/blog/2008/09/26/comparing-three-methods-of-computing-standard-deviation/ - -def _ss(data, c=None): - """Return sum of square deviations of sequence data. - - If ``c`` is None, the mean is calculated in one pass, and the deviations - from the mean are calculated in a second pass. Otherwise, deviations are - calculated from ``c`` as given. Use the second case with care, as it can - lead to garbage results. - """ - if c is not None: - T, total, count = _sum((x-c)**2 for x in data) - return (T, total) - T, total, count = _sum(data) - mean_n, mean_d = (total / count).as_integer_ratio() - partials = Counter() - for n, d in map(_exact_ratio, data): - diff_n = n * mean_d - d * mean_n - diff_d = d * mean_d - partials[diff_d * diff_d] += diff_n * diff_n - if None in partials: - # The sum will be a NAN or INF. We can ignore all the finite - # partials, and just look at this special one. - total = partials[None] - assert not _isfinite(total) - else: - total = sum(Fraction(n, d) for d, n in partials.items()) - return (T, total) def variance(data, xbar=None): @@ -760,12 +1146,9 @@ def variance(data, xbar=None): Fraction(67, 108) """ - if iter(data) is data: - data = list(data) - n = len(data) + T, ss, c, n = _ss(data, xbar) if n < 2: raise StatisticsError('variance requires at least two data points') - T, ss = _ss(data, xbar) return _convert(ss / (n - 1), T) @@ -804,12 +1187,9 @@ def pvariance(data, mu=None): Fraction(13, 72) """ - if iter(data) is data: - data = list(data) - n = len(data) + T, ss, c, n = _ss(data, mu) if n < 1: raise StatisticsError('pvariance requires at least one data point') - T, ss = _ss(data, mu) return _convert(ss / n, T) @@ -822,14 +1202,13 @@ def stdev(data, xbar=None): 1.0810874155219827 """ - # Fixme: Despite the exact sum of squared deviations, some inaccuracy - # remain because there are two rounding steps. The first occurs in - # the _convert() step for variance(), the second occurs in math.sqrt(). - var = variance(data, xbar) - try: - return var.sqrt() - except AttributeError: - return math.sqrt(var) + T, ss, c, n = _ss(data, xbar) + if n < 2: + raise StatisticsError('stdev requires at least two data points') + mss = ss / (n - 1) + if issubclass(T, Decimal): + return _decimal_sqrt_of_frac(mss.numerator, mss.denominator) + return _float_sqrt_of_frac(mss.numerator, mss.denominator) def pstdev(data, mu=None): @@ -841,14 +1220,47 @@ def pstdev(data, mu=None): 0.986893273527251 """ - # Fixme: Despite the exact sum of squared deviations, some inaccuracy - # remain because there are two rounding steps. The first occurs in - # the _convert() step for pvariance(), the second occurs in math.sqrt(). - var = pvariance(data, mu) + T, ss, c, n = _ss(data, mu) + if n < 1: + raise StatisticsError('pstdev requires at least one data point') + mss = ss / n + if issubclass(T, Decimal): + return _decimal_sqrt_of_frac(mss.numerator, mss.denominator) + return _float_sqrt_of_frac(mss.numerator, mss.denominator) + + +def _mean_stdev(data): + """In one pass, compute the mean and sample standard deviation as floats.""" + T, ss, xbar, n = _ss(data) + if n < 2: + raise StatisticsError('stdev requires at least two data points') + mss = ss / (n - 1) try: - return var.sqrt() + return float(xbar), _float_sqrt_of_frac(mss.numerator, mss.denominator) except AttributeError: - return math.sqrt(var) + # Handle Nans and Infs gracefully + return float(xbar), float(xbar) / float(ss) + +def _sqrtprod(x: float, y: float) -> float: + "Return sqrt(x * y) computed with improved accuracy and without overflow/underflow." + h = sqrt(x * y) + if not isfinite(h): + if isinf(h) and not isinf(x) and not isinf(y): + # Finite inputs overflowed, so scale down, and recompute. + scale = 2.0 ** -512 # sqrt(1 / sys.float_info.max) + return _sqrtprod(scale * x, scale * y) / scale + return h + if not h: + if x and y: + # Non-zero inputs underflowed, so scale up, and recompute. + # Scale: 1 / sqrt(sys.float_info.min * sys.float_info.epsilon) + scale = 2.0 ** 537 + return _sqrtprod(scale * x, scale * y) / scale + return h + # Improve accuracy with a differential correction. + # https://www.wolframalpha.com/input/?i=Maclaurin+series+sqrt%28h**2+%2B+x%29+at+x%3D0 + d = sumprod((x, h), (y, -h)) + return h + d / (2.0 * h) # === Statistics for relations between two inputs === @@ -882,18 +1294,16 @@ def covariance(x, y, /): raise StatisticsError('covariance requires at least two data points') xbar = fsum(x) / n ybar = fsum(y) / n - sxy = fsum((xi - xbar) * (yi - ybar) for xi, yi in zip(x, y)) + sxy = sumprod((xi - xbar for xi in x), (yi - ybar for yi in y)) return sxy / (n - 1) -def correlation(x, y, /): +def correlation(x, y, /, *, method='linear'): """Pearson's correlation coefficient Return the Pearson's correlation coefficient for two inputs. Pearson's - correlation coefficient *r* takes values between -1 and +1. It measures the - strength and direction of the linear relationship, where +1 means very - strong, positive linear relationship, -1 very strong, negative linear - relationship, and 0 no linear relationship. + correlation coefficient *r* takes values between -1 and +1. It measures + the strength and direction of a linear relationship. >>> x = [1, 2, 3, 4, 5, 6, 7, 8, 9] >>> y = [9, 8, 7, 6, 5, 4, 3, 2, 1] @@ -902,19 +1312,36 @@ def correlation(x, y, /): >>> correlation(x, y) -1.0 + If *method* is "ranked", computes Spearman's rank correlation coefficient + for two inputs. The data is replaced by ranks. Ties are averaged + so that equal values receive the same rank. The resulting coefficient + measures the strength of a monotonic relationship. + + Spearman's rank correlation coefficient is appropriate for ordinal + data or for continuous data that doesn't meet the linear proportion + requirement for Pearson's correlation coefficient. """ n = len(x) if len(y) != n: raise StatisticsError('correlation requires that both inputs have same number of data points') if n < 2: raise StatisticsError('correlation requires at least two data points') - xbar = fsum(x) / n - ybar = fsum(y) / n - sxy = fsum((xi - xbar) * (yi - ybar) for xi, yi in zip(x, y)) - sxx = fsum((xi - xbar) ** 2.0 for xi in x) - syy = fsum((yi - ybar) ** 2.0 for yi in y) + if method not in {'linear', 'ranked'}: + raise ValueError(f'Unknown method: {method!r}') + if method == 'ranked': + start = (n - 1) / -2 # Center rankings around zero + x = _rank(x, start=start) + y = _rank(y, start=start) + else: + xbar = fsum(x) / n + ybar = fsum(y) / n + x = [xi - xbar for xi in x] + y = [yi - ybar for yi in y] + sxy = sumprod(x, y) + sxx = sumprod(x, x) + syy = sumprod(y, y) try: - return sxy / sqrt(sxx * syy) + return sxy / _sqrtprod(sxx, syy) except ZeroDivisionError: raise StatisticsError('at least one of the inputs is constant') @@ -922,13 +1349,13 @@ def correlation(x, y, /): LinearRegression = namedtuple('LinearRegression', ('slope', 'intercept')) -def linear_regression(x, y, /): +def linear_regression(x, y, /, *, proportional=False): """Slope and intercept for simple linear regression. Return the slope and intercept of simple linear regression parameters estimated using ordinary least squares. Simple linear regression describes relationship between an independent variable - *x* and a dependent variable *y* in terms of linear function: + *x* and a dependent variable *y* in terms of a linear function: y = slope * x + intercept + noise @@ -944,7 +1371,20 @@ def linear_regression(x, y, /): >>> noise = NormalDist().samples(5, seed=42) >>> y = [3 * x[i] + 2 + noise[i] for i in range(5)] >>> linear_regression(x, y) #doctest: +ELLIPSIS - LinearRegression(slope=3.09078914170..., intercept=1.75684970486...) + LinearRegression(slope=3.17495..., intercept=1.00925...) + + If *proportional* is true, the independent variable *x* and the + dependent variable *y* are assumed to be directly proportional. + The data is fit to a line passing through the origin. + + Since the *intercept* will always be 0.0, the underlying linear + function simplifies to: + + y = slope * x + noise + + >>> y = [3 * x[i] + noise[i] for i in range(5)] + >>> linear_regression(x, y, proportional=True) #doctest: +ELLIPSIS + LinearRegression(slope=2.90475..., intercept=0.0) """ n = len(x) @@ -952,15 +1392,18 @@ def linear_regression(x, y, /): raise StatisticsError('linear regression requires that both inputs have same number of data points') if n < 2: raise StatisticsError('linear regression requires at least two data points') - xbar = fsum(x) / n - ybar = fsum(y) / n - sxy = fsum((xi - xbar) * (yi - ybar) for xi, yi in zip(x, y)) - sxx = fsum((xi - xbar) ** 2.0 for xi in x) + if not proportional: + xbar = fsum(x) / n + ybar = fsum(y) / n + x = [xi - xbar for xi in x] # List because used three times below + y = (yi - ybar for yi in y) # Generator because only used once below + sxy = sumprod(x, y) + 0.0 # Add zero to coerce result to a float + sxx = sumprod(x, x) try: slope = sxy / sxx # equivalent to: covariance(x, y) / variance(x) except ZeroDivisionError: raise StatisticsError('x is constant') - intercept = ybar - slope * xbar + intercept = 0.0 if proportional else ybar - slope * xbar return LinearRegression(slope=slope, intercept=intercept) @@ -1068,29 +1511,29 @@ def __init__(self, mu=0.0, sigma=1.0): @classmethod def from_samples(cls, data): "Make a normal distribution instance from sample data." - if not isinstance(data, (list, tuple)): - data = list(data) - xbar = fmean(data) - return cls(xbar, stdev(data, xbar)) + return cls(*_mean_stdev(data)) def samples(self, n, *, seed=None): "Generate *n* samples for a given mean and standard deviation." - gauss = random.gauss if seed is None else random.Random(seed).gauss - mu, sigma = self._mu, self._sigma - return [gauss(mu, sigma) for i in range(n)] + rnd = random.random if seed is None else random.Random(seed).random + inv_cdf = _normal_dist_inv_cdf + mu = self._mu + sigma = self._sigma + return [inv_cdf(rnd(), mu, sigma) for _ in repeat(None, n)] def pdf(self, x): "Probability density function. P(x <= X < x+dx) / dx" - variance = self._sigma ** 2.0 + variance = self._sigma * self._sigma if not variance: raise StatisticsError('pdf() not defined when sigma is zero') - return exp((x - self._mu)**2.0 / (-2.0*variance)) / sqrt(tau*variance) + diff = x - self._mu + return exp(diff * diff / (-2.0 * variance)) / sqrt(tau * variance) def cdf(self, x): "Cumulative distribution function. P(X <= x)" if not self._sigma: raise StatisticsError('cdf() not defined when sigma is zero') - return 0.5 * (1.0 + erf((x - self._mu) / (self._sigma * sqrt(2.0)))) + return 0.5 * (1.0 + erf((x - self._mu) / (self._sigma * _SQRT2))) def inv_cdf(self, p): """Inverse cumulative distribution function. x : P(X <= x) = p @@ -1104,8 +1547,6 @@ def inv_cdf(self, p): """ if p <= 0.0 or p >= 1.0: raise StatisticsError('p must be in the range 0.0 < p < 1.0') - if self._sigma <= 0.0: - raise StatisticsError('cdf() not defined when sigma at or below zero') return _normal_dist_inv_cdf(p, self._mu, self._sigma) def quantiles(self, n=4): @@ -1146,9 +1587,9 @@ def overlap(self, other): dv = Y_var - X_var dm = fabs(Y._mu - X._mu) if not dv: - return 1.0 - erf(dm / (2.0 * X._sigma * sqrt(2.0))) + return 1.0 - erf(dm / (2.0 * X._sigma * _SQRT2)) a = X._mu * Y_var - Y._mu * X_var - b = X._sigma * Y._sigma * sqrt(dm**2.0 + dv * log(Y_var / X_var)) + b = X._sigma * Y._sigma * sqrt(dm * dm + dv * log(Y_var / X_var)) x1 = (a + b) / dv x2 = (a - b) / dv return 1.0 - (fabs(Y.cdf(x1) - X.cdf(x1)) + fabs(Y.cdf(x2) - X.cdf(x2))) @@ -1191,7 +1632,7 @@ def stdev(self): @property def variance(self): "Square of the standard deviation." - return self._sigma ** 2.0 + return self._sigma * self._sigma def __add__(x1, x2): """Add a constant or another NormalDist instance. @@ -1265,3 +1706,102 @@ def __hash__(self): def __repr__(self): return f'{type(self).__name__}(mu={self._mu!r}, sigma={self._sigma!r})' + + def __getstate__(self): + return self._mu, self._sigma + + def __setstate__(self, state): + self._mu, self._sigma = state + + +## kde_random() ############################################################## + +def _newton_raphson(f_inv_estimate, f, f_prime, tolerance=1e-12): + def f_inv(y): + "Return x such that f(x) ≈ y within the specified tolerance." + x = f_inv_estimate(y) + while abs(diff := f(x) - y) > tolerance: + x -= diff / f_prime(x) + return x + return f_inv + +def _quartic_invcdf_estimate(p): + sign, p = (1.0, p) if p <= 1/2 else (-1.0, 1.0 - p) + x = (2.0 * p) ** 0.4258865685331 - 1.0 + if p >= 0.004 < 0.499: + x += 0.026818732 * sin(7.101753784 * p + 2.73230839482953) + return x * sign + +_quartic_invcdf = _newton_raphson( + f_inv_estimate = _quartic_invcdf_estimate, + f = lambda t: 3/16 * t**5 - 5/8 * t**3 + 15/16 * t + 1/2, + f_prime = lambda t: 15/16 * (1.0 - t * t) ** 2) + +def _triweight_invcdf_estimate(p): + sign, p = (1.0, p) if p <= 1/2 else (-1.0, 1.0 - p) + x = (2.0 * p) ** 0.3400218741872791 - 1.0 + return x * sign + +_triweight_invcdf = _newton_raphson( + f_inv_estimate = _triweight_invcdf_estimate, + f = lambda t: 35/32 * (-1/7*t**7 + 3/5*t**5 - t**3 + t) + 1/2, + f_prime = lambda t: 35/32 * (1.0 - t * t) ** 3) + +_kernel_invcdfs = { + 'normal': NormalDist().inv_cdf, + 'logistic': lambda p: log(p / (1 - p)), + 'sigmoid': lambda p: log(tan(p * pi/2)), + 'rectangular': lambda p: 2*p - 1, + 'parabolic': lambda p: 2 * cos((acos(2*p-1) + pi) / 3), + 'quartic': _quartic_invcdf, + 'triweight': _triweight_invcdf, + 'triangular': lambda p: sqrt(2*p) - 1 if p < 1/2 else 1 - sqrt(2 - 2*p), + 'cosine': lambda p: 2 * asin(2*p - 1) / pi, +} +_kernel_invcdfs['gauss'] = _kernel_invcdfs['normal'] +_kernel_invcdfs['uniform'] = _kernel_invcdfs['rectangular'] +_kernel_invcdfs['epanechnikov'] = _kernel_invcdfs['parabolic'] +_kernel_invcdfs['biweight'] = _kernel_invcdfs['quartic'] + +def kde_random(data, h, kernel='normal', *, seed=None): + """Return a function that makes a random selection from the estimated + probability density function created by kde(data, h, kernel). + + Providing a *seed* allows reproducible selections within a single + thread. The seed may be an integer, float, str, or bytes. + + A StatisticsError will be raised if the *data* sequence is empty. + + Example: + + >>> data = [-2.1, -1.3, -0.4, 1.9, 5.1, 6.2] + >>> rand = kde_random(data, h=1.5, seed=8675309) + >>> new_selections = [rand() for i in range(10)] + >>> [round(x, 1) for x in new_selections] + [0.7, 6.2, 1.2, 6.9, 7.0, 1.8, 2.5, -0.5, -1.8, 5.6] + + """ + n = len(data) + if not n: + raise StatisticsError('Empty data sequence') + + if not isinstance(data[0], (int, float)): + raise TypeError('Data sequence must contain ints or floats') + + if h <= 0.0: + raise StatisticsError(f'Bandwidth h must be positive, not {h=!r}') + + kernel_invcdf = _kernel_invcdfs.get(kernel) + if kernel_invcdf is None: + raise StatisticsError(f'Unknown kernel name: {kernel!r}') + + prng = _random.Random(seed) + random = prng.random + choice = prng.choice + + def rand(): + return choice(data) + h * kernel_invcdf(random()) + + rand.__doc__ = f'Random KDE selection with {h=!r} and {kernel=!r}' + + return rand diff --git a/Lib/symtable.py b/Lib/symtable.py new file mode 100644 index 0000000000..672ec0ce1f --- /dev/null +++ b/Lib/symtable.py @@ -0,0 +1,414 @@ +"""Interface to the compiler's internal symbol tables""" + +import _symtable +from _symtable import (USE, DEF_GLOBAL, DEF_NONLOCAL, DEF_LOCAL, DEF_PARAM, + DEF_IMPORT, DEF_BOUND, DEF_ANNOT, SCOPE_OFF, SCOPE_MASK, FREE, + LOCAL, GLOBAL_IMPLICIT, GLOBAL_EXPLICIT, CELL) + +import weakref +from enum import StrEnum + +__all__ = ["symtable", "SymbolTableType", "SymbolTable", "Class", "Function", "Symbol"] + +def symtable(code, filename, compile_type): + """ Return the toplevel *SymbolTable* for the source code. + + *filename* is the name of the file with the code + and *compile_type* is the *compile()* mode argument. + """ + top = _symtable.symtable(code, filename, compile_type) + return _newSymbolTable(top, filename) + +class SymbolTableFactory: + def __init__(self): + self.__memo = weakref.WeakValueDictionary() + + def new(self, table, filename): + if table.type == _symtable.TYPE_FUNCTION: + return Function(table, filename) + if table.type == _symtable.TYPE_CLASS: + return Class(table, filename) + return SymbolTable(table, filename) + + def __call__(self, table, filename): + key = table, filename + obj = self.__memo.get(key, None) + if obj is None: + obj = self.__memo[key] = self.new(table, filename) + return obj + +_newSymbolTable = SymbolTableFactory() + + +class SymbolTableType(StrEnum): + MODULE = "module" + FUNCTION = "function" + CLASS = "class" + ANNOTATION = "annotation" + TYPE_ALIAS = "type alias" + TYPE_PARAMETERS = "type parameters" + TYPE_VARIABLE = "type variable" + + +class SymbolTable: + + def __init__(self, raw_table, filename): + self._table = raw_table + self._filename = filename + self._symbols = {} + + def __repr__(self): + if self.__class__ == SymbolTable: + kind = "" + else: + kind = "%s " % self.__class__.__name__ + + if self._table.name == "top": + return "<{0}SymbolTable for module {1}>".format(kind, self._filename) + else: + return "<{0}SymbolTable for {1} in {2}>".format(kind, + self._table.name, + self._filename) + + def get_type(self): + """Return the type of the symbol table. + + The value returned is one of the values in + the ``SymbolTableType`` enumeration. + """ + if self._table.type == _symtable.TYPE_MODULE: + return SymbolTableType.MODULE + if self._table.type == _symtable.TYPE_FUNCTION: + return SymbolTableType.FUNCTION + if self._table.type == _symtable.TYPE_CLASS: + return SymbolTableType.CLASS + if self._table.type == _symtable.TYPE_ANNOTATION: + return SymbolTableType.ANNOTATION + if self._table.type == _symtable.TYPE_TYPE_ALIAS: + return SymbolTableType.TYPE_ALIAS + if self._table.type == _symtable.TYPE_TYPE_PARAMETERS: + return SymbolTableType.TYPE_PARAMETERS + if self._table.type == _symtable.TYPE_TYPE_VARIABLE: + return SymbolTableType.TYPE_VARIABLE + assert False, f"unexpected type: {self._table.type}" + + def get_id(self): + """Return an identifier for the table. + """ + return self._table.id + + def get_name(self): + """Return the table's name. + + This corresponds to the name of the class, function + or 'top' if the table is for a class, function or + global respectively. + """ + return self._table.name + + def get_lineno(self): + """Return the number of the first line in the + block for the table. + """ + return self._table.lineno + + def is_optimized(self): + """Return *True* if the locals in the table + are optimizable. + """ + return bool(self._table.type == _symtable.TYPE_FUNCTION) + + def is_nested(self): + """Return *True* if the block is a nested class + or function.""" + return bool(self._table.nested) + + def has_children(self): + """Return *True* if the block has nested namespaces. + """ + return bool(self._table.children) + + def get_identifiers(self): + """Return a view object containing the names of symbols in the table. + """ + return self._table.symbols.keys() + + def lookup(self, name): + """Lookup a *name* in the table. + + Returns a *Symbol* instance. + """ + sym = self._symbols.get(name) + if sym is None: + flags = self._table.symbols[name] + namespaces = self.__check_children(name) + module_scope = (self._table.name == "top") + sym = self._symbols[name] = Symbol(name, flags, namespaces, + module_scope=module_scope) + return sym + + def get_symbols(self): + """Return a list of *Symbol* instances for + names in the table. + """ + return [self.lookup(ident) for ident in self.get_identifiers()] + + def __check_children(self, name): + return [_newSymbolTable(st, self._filename) + for st in self._table.children + if st.name == name] + + def get_children(self): + """Return a list of the nested symbol tables. + """ + return [_newSymbolTable(st, self._filename) + for st in self._table.children] + + +class Function(SymbolTable): + + # Default values for instance variables + __params = None + __locals = None + __frees = None + __globals = None + __nonlocals = None + + def __idents_matching(self, test_func): + return tuple(ident for ident in self.get_identifiers() + if test_func(self._table.symbols[ident])) + + def get_parameters(self): + """Return a tuple of parameters to the function. + """ + if self.__params is None: + self.__params = self.__idents_matching(lambda x:x & DEF_PARAM) + return self.__params + + def get_locals(self): + """Return a tuple of locals in the function. + """ + if self.__locals is None: + locs = (LOCAL, CELL) + test = lambda x: ((x >> SCOPE_OFF) & SCOPE_MASK) in locs + self.__locals = self.__idents_matching(test) + return self.__locals + + def get_globals(self): + """Return a tuple of globals in the function. + """ + if self.__globals is None: + glob = (GLOBAL_IMPLICIT, GLOBAL_EXPLICIT) + test = lambda x:((x >> SCOPE_OFF) & SCOPE_MASK) in glob + self.__globals = self.__idents_matching(test) + return self.__globals + + def get_nonlocals(self): + """Return a tuple of nonlocals in the function. + """ + if self.__nonlocals is None: + self.__nonlocals = self.__idents_matching(lambda x:x & DEF_NONLOCAL) + return self.__nonlocals + + def get_frees(self): + """Return a tuple of free variables in the function. + """ + if self.__frees is None: + is_free = lambda x:((x >> SCOPE_OFF) & SCOPE_MASK) == FREE + self.__frees = self.__idents_matching(is_free) + return self.__frees + + +class Class(SymbolTable): + + __methods = None + + def get_methods(self): + """Return a tuple of methods declared in the class. + """ + if self.__methods is None: + d = {} + + def is_local_symbol(ident): + flags = self._table.symbols.get(ident, 0) + return ((flags >> SCOPE_OFF) & SCOPE_MASK) == LOCAL + + for st in self._table.children: + # pick the function-like symbols that are local identifiers + if is_local_symbol(st.name): + match st.type: + case _symtable.TYPE_FUNCTION: + # generators are of type TYPE_FUNCTION with a ".0" + # parameter as a first parameter (which makes them + # distinguishable from a function named 'genexpr') + if st.name == 'genexpr' and '.0' in st.varnames: + continue + d[st.name] = 1 + case _symtable.TYPE_TYPE_PARAMETERS: + # Get the function-def block in the annotation + # scope 'st' with the same identifier, if any. + scope_name = st.name + for c in st.children: + if c.name == scope_name and c.type == _symtable.TYPE_FUNCTION: + # A generic generator of type TYPE_FUNCTION + # cannot be a direct child of 'st' (but it + # can be a descendant), e.g.: + # + # class A: + # type genexpr[genexpr] = (x for x in []) + assert scope_name != 'genexpr' or '.0' not in c.varnames + d[scope_name] = 1 + break + self.__methods = tuple(d) + return self.__methods + + +class Symbol: + + def __init__(self, name, flags, namespaces=None, *, module_scope=False): + self.__name = name + self.__flags = flags + self.__scope = (flags >> SCOPE_OFF) & SCOPE_MASK # like PyST_GetScope() + self.__namespaces = namespaces or () + self.__module_scope = module_scope + + def __repr__(self): + flags_str = '|'.join(self._flags_str()) + return f'' + + def _scope_str(self): + return _scopes_value_to_name.get(self.__scope) or str(self.__scope) + + def _flags_str(self): + for flagname, flagvalue in _flags: + if self.__flags & flagvalue == flagvalue: + yield flagname + + def get_name(self): + """Return a name of a symbol. + """ + return self.__name + + def is_referenced(self): + """Return *True* if the symbol is used in + its block. + """ + return bool(self.__flags & _symtable.USE) + + def is_parameter(self): + """Return *True* if the symbol is a parameter. + """ + return bool(self.__flags & DEF_PARAM) + + def is_global(self): + """Return *True* if the symbol is global. + """ + return bool(self.__scope in (GLOBAL_IMPLICIT, GLOBAL_EXPLICIT) + or (self.__module_scope and self.__flags & DEF_BOUND)) + + def is_nonlocal(self): + """Return *True* if the symbol is nonlocal.""" + return bool(self.__flags & DEF_NONLOCAL) + + def is_declared_global(self): + """Return *True* if the symbol is declared global + with a global statement.""" + return bool(self.__scope == GLOBAL_EXPLICIT) + + def is_local(self): + """Return *True* if the symbol is local. + """ + return bool(self.__scope in (LOCAL, CELL) + or (self.__module_scope and self.__flags & DEF_BOUND)) + + def is_annotated(self): + """Return *True* if the symbol is annotated. + """ + return bool(self.__flags & DEF_ANNOT) + + def is_free(self): + """Return *True* if a referenced symbol is + not assigned to. + """ + return bool(self.__scope == FREE) + + def is_imported(self): + """Return *True* if the symbol is created from + an import statement. + """ + return bool(self.__flags & DEF_IMPORT) + + def is_assigned(self): + """Return *True* if a symbol is assigned to.""" + return bool(self.__flags & DEF_LOCAL) + + def is_namespace(self): + """Returns *True* if name binding introduces new namespace. + + If the name is used as the target of a function or class + statement, this will be true. + + Note that a single name can be bound to multiple objects. If + is_namespace() is true, the name may also be bound to other + objects, like an int or list, that does not introduce a new + namespace. + """ + return bool(self.__namespaces) + + def get_namespaces(self): + """Return a list of namespaces bound to this name""" + return self.__namespaces + + def get_namespace(self): + """Return the single namespace bound to this name. + + Raises ValueError if the name is bound to multiple namespaces + or no namespace. + """ + if len(self.__namespaces) == 0: + raise ValueError("name is not bound to any namespaces") + elif len(self.__namespaces) > 1: + raise ValueError("name is bound to multiple namespaces") + else: + return self.__namespaces[0] + + +_flags = [('USE', USE)] +_flags.extend(kv for kv in globals().items() if kv[0].startswith('DEF_')) +_scopes_names = ('FREE', 'LOCAL', 'GLOBAL_IMPLICIT', 'GLOBAL_EXPLICIT', 'CELL') +_scopes_value_to_name = {globals()[n]: n for n in _scopes_names} + + +def main(args): + import sys + def print_symbols(table, level=0): + indent = ' ' * level + nested = "nested " if table.is_nested() else "" + if table.get_type() == 'module': + what = f'from file {table._filename!r}' + else: + what = f'{table.get_name()!r}' + print(f'{indent}symbol table for {nested}{table.get_type()} {what}:') + for ident in table.get_identifiers(): + symbol = table.lookup(ident) + flags = ', '.join(symbol._flags_str()).lower() + print(f' {indent}{symbol._scope_str().lower()} symbol {symbol.get_name()!r}: {flags}') + print() + + for table2 in table.get_children(): + print_symbols(table2, level + 1) + + for filename in args or ['-']: + if filename == '-': + src = sys.stdin.read() + filename = '' + else: + with open(filename, 'rb') as f: + src = f.read() + mod = symtable(src, filename, 'exec') + print_symbols(mod) + + +if __name__ == "__main__": + import sys + main(sys.argv[1:]) diff --git a/Lib/sysconfig.py b/Lib/sysconfig/__init__.py similarity index 52% rename from Lib/sysconfig.py rename to Lib/sysconfig/__init__.py index 9999d6bbd5..8365236f61 100644 --- a/Lib/sysconfig.py +++ b/Lib/sysconfig/__init__.py @@ -1,10 +1,9 @@ -# XXX: RUSTPYTHON; Trick to make sysconfig work as RustPython -exec(r''' """Access to Python's configuration information.""" import os import sys -from os.path import pardir, realpath +import threading +from os.path import realpath __all__ = [ 'get_config_h_filename', @@ -22,29 +21,30 @@ # Keys for get_config_var() that are never converted to Python integers. _ALWAYS_STR = { + 'IPHONEOS_DEPLOYMENT_TARGET', 'MACOSX_DEPLOYMENT_TARGET', } _INSTALL_SCHEMES = { 'posix_prefix': { - 'stdlib': '{installed_base}/{platlibdir}/python{py_version_short}', - 'platstdlib': '{platbase}/{platlibdir}/python{py_version_short}', - 'purelib': '{base}/lib/python{py_version_short}/site-packages', - 'platlib': '{platbase}/{platlibdir}/python{py_version_short}/site-packages', + 'stdlib': '{installed_base}/{platlibdir}/{implementation_lower}{py_version_short}{abi_thread}', + 'platstdlib': '{platbase}/{platlibdir}/{implementation_lower}{py_version_short}{abi_thread}', + 'purelib': '{base}/lib/{implementation_lower}{py_version_short}{abi_thread}/site-packages', + 'platlib': '{platbase}/{platlibdir}/{implementation_lower}{py_version_short}{abi_thread}/site-packages', 'include': - '{installed_base}/include/python{py_version_short}{abiflags}', + '{installed_base}/include/{implementation_lower}{py_version_short}{abiflags}', 'platinclude': - '{installed_platbase}/include/python{py_version_short}{abiflags}', + '{installed_platbase}/include/{implementation_lower}{py_version_short}{abiflags}', 'scripts': '{base}/bin', 'data': '{base}', }, 'posix_home': { - 'stdlib': '{installed_base}/lib/python', - 'platstdlib': '{base}/lib/python', - 'purelib': '{base}/lib/python', - 'platlib': '{base}/lib/python', - 'include': '{installed_base}/include/python', - 'platinclude': '{installed_base}/include/python', + 'stdlib': '{installed_base}/lib/{implementation_lower}', + 'platstdlib': '{base}/lib/{implementation_lower}', + 'purelib': '{base}/lib/{implementation_lower}', + 'platlib': '{base}/lib/{implementation_lower}', + 'include': '{installed_base}/include/{implementation_lower}', + 'platinclude': '{installed_base}/include/{implementation_lower}', 'scripts': '{base}/bin', 'data': '{base}', }, @@ -58,6 +58,7 @@ 'scripts': '{base}/Scripts', 'data': '{base}', }, + # Downstream distributors can overwrite the default install scheme. # This is done to support downstream modifications where distributors change # the installation layout (eg. different site-packages directory). @@ -76,14 +77,14 @@ # Downstream distributors who patch posix_prefix/nt scheme are encouraged to # leave the following schemes unchanged 'posix_venv': { - 'stdlib': '{installed_base}/{platlibdir}/python{py_version_short}', - 'platstdlib': '{platbase}/{platlibdir}/python{py_version_short}', - 'purelib': '{base}/lib/python{py_version_short}/site-packages', - 'platlib': '{platbase}/{platlibdir}/python{py_version_short}/site-packages', + 'stdlib': '{installed_base}/{platlibdir}/{implementation_lower}{py_version_short}{abi_thread}', + 'platstdlib': '{platbase}/{platlibdir}/{implementation_lower}{py_version_short}{abi_thread}', + 'purelib': '{base}/lib/{implementation_lower}{py_version_short}{abi_thread}/site-packages', + 'platlib': '{platbase}/{platlibdir}/{implementation_lower}{py_version_short}{abi_thread}/site-packages', 'include': - '{installed_base}/include/python{py_version_short}{abiflags}', + '{installed_base}/include/{implementation_lower}{py_version_short}{abiflags}', 'platinclude': - '{installed_platbase}/include/python{py_version_short}{abiflags}', + '{installed_platbase}/include/{implementation_lower}{py_version_short}{abiflags}', 'scripts': '{base}/bin', 'data': '{base}', }, @@ -105,6 +106,8 @@ else: _INSTALL_SCHEMES['venv'] = _INSTALL_SCHEMES['posix_venv'] +def _get_implementation(): + return 'RustPython' # XXX: For site-packages # NOTE: site.py has copy of this function. # Sync it when modify this function. @@ -113,8 +116,8 @@ def _getuserbase(): if env_base: return env_base - # Emscripten, VxWorks, and WASI have no home directories - if sys.platform in {"emscripten", "vxworks", "wasi"}: + # Emscripten, iOS, tvOS, VxWorks, WASI, and watchOS have no home directories + if sys.platform in {"emscripten", "ios", "tvos", "vxworks", "wasi", "watchos"}: return None def joinuser(*args): @@ -122,7 +125,7 @@ def joinuser(*args): if os.name == "nt": base = os.environ.get("APPDATA") or "~" - return joinuser(base, "Python") + return joinuser(base, _get_implementation()) if sys.platform == "darwin" and sys._framework: return joinuser("~", "Library", sys._framework, @@ -136,29 +139,29 @@ def joinuser(*args): _INSTALL_SCHEMES |= { # NOTE: When modifying "purelib" scheme, update site._get_path() too. 'nt_user': { - 'stdlib': '{userbase}/Python{py_version_nodot_plat}', - 'platstdlib': '{userbase}/Python{py_version_nodot_plat}', - 'purelib': '{userbase}/Python{py_version_nodot_plat}/site-packages', - 'platlib': '{userbase}/Python{py_version_nodot_plat}/site-packages', - 'include': '{userbase}/Python{py_version_nodot_plat}/Include', - 'scripts': '{userbase}/Python{py_version_nodot_plat}/Scripts', + 'stdlib': '{userbase}/{implementation}{py_version_nodot_plat}', + 'platstdlib': '{userbase}/{implementation}{py_version_nodot_plat}', + 'purelib': '{userbase}/{implementation}{py_version_nodot_plat}/site-packages', + 'platlib': '{userbase}/{implementation}{py_version_nodot_plat}/site-packages', + 'include': '{userbase}/{implementation}{py_version_nodot_plat}/Include', + 'scripts': '{userbase}/{implementation}{py_version_nodot_plat}/Scripts', 'data': '{userbase}', }, 'posix_user': { - 'stdlib': '{userbase}/{platlibdir}/python{py_version_short}', - 'platstdlib': '{userbase}/{platlibdir}/python{py_version_short}', - 'purelib': '{userbase}/lib/python{py_version_short}/site-packages', - 'platlib': '{userbase}/lib/python{py_version_short}/site-packages', - 'include': '{userbase}/include/python{py_version_short}', + 'stdlib': '{userbase}/{platlibdir}/{implementation_lower}{py_version_short}{abi_thread}', + 'platstdlib': '{userbase}/{platlibdir}/{implementation_lower}{py_version_short}{abi_thread}', + 'purelib': '{userbase}/lib/{implementation_lower}{py_version_short}{abi_thread}/site-packages', + 'platlib': '{userbase}/lib/{implementation_lower}{py_version_short}{abi_thread}/site-packages', + 'include': '{userbase}/include/{implementation_lower}{py_version_short}{abi_thread}', 'scripts': '{userbase}/bin', 'data': '{userbase}', }, 'osx_framework_user': { - 'stdlib': '{userbase}/lib/python', - 'platstdlib': '{userbase}/lib/python', - 'purelib': '{userbase}/lib/python/site-packages', - 'platlib': '{userbase}/lib/python/site-packages', - 'include': '{userbase}/include/python{py_version_short}', + 'stdlib': '{userbase}/lib/{implementation_lower}', + 'platstdlib': '{userbase}/lib/{implementation_lower}', + 'purelib': '{userbase}/lib/{implementation_lower}/site-packages', + 'platlib': '{userbase}/lib/{implementation_lower}/site-packages', + 'include': '{userbase}/include/{implementation_lower}{py_version_short}', 'scripts': '{userbase}/bin', 'data': '{userbase}', }, @@ -170,19 +173,15 @@ def joinuser(*args): _PY_VERSION = sys.version.split()[0] _PY_VERSION_SHORT = f'{sys.version_info[0]}.{sys.version_info[1]}' _PY_VERSION_SHORT_NO_DOT = f'{sys.version_info[0]}{sys.version_info[1]}' -_PREFIX = os.path.normpath(sys.prefix) _BASE_PREFIX = os.path.normpath(sys.base_prefix) -_EXEC_PREFIX = os.path.normpath(sys.exec_prefix) _BASE_EXEC_PREFIX = os.path.normpath(sys.base_exec_prefix) +# Mutex guarding initialization of _CONFIG_VARS. +_CONFIG_VARS_LOCK = threading.RLock() _CONFIG_VARS = None +# True iff _CONFIG_VARS has been fully initialized. +_CONFIG_VARS_INITIALIZED = False _USER_BASE = None -# Regexes needed for parsing Makefile (and similar syntaxes, -# like old-style Setup files). -_variable_rx = r"([a-zA-Z][a-zA-Z0-9_]+)\s*=\s*(.*)" -_findvar1_rx = r"\$\(([A-Za-z][A-Za-z0-9_]*)\)" -_findvar2_rx = r"\${([A-Za-z][A-Za-z0-9_]*)}" - def _safe_realpath(path): try: @@ -221,8 +220,15 @@ def _safe_realpath(path): def is_python_build(check_home=None): if check_home is not None: import warnings - warnings.warn("check_home argument is deprecated and ignored.", - DeprecationWarning, stacklevel=2) + warnings.warn( + ( + 'The check_home argument of sysconfig.is_python_build is ' + 'deprecated and its value is ignored. ' + 'It will be removed in Python 3.15.' + ), + DeprecationWarning, + stacklevel=2, + ) for fn in ("Setup", "Setup.local"): if os.path.isfile(os.path.join(_PROJECT_BASE, "Modules", fn)): return True @@ -291,6 +297,7 @@ def _get_preferred_schemes(): 'home': 'posix_home', 'user': 'osx_framework_user', } + return { 'prefix': 'posix_prefix', 'home': 'posix_home', @@ -314,134 +321,6 @@ def get_default_scheme(): return get_preferred_scheme('prefix') -def _parse_makefile(filename, vars=None, keep_unresolved=True): - """Parse a Makefile-style file. - - A dictionary containing name/value pairs is returned. If an - optional dictionary is passed in as the second argument, it is - used instead of a new dictionary. - """ - import re - - if vars is None: - vars = {} - done = {} - notdone = {} - - with open(filename, encoding=sys.getfilesystemencoding(), - errors="surrogateescape") as f: - lines = f.readlines() - - for line in lines: - if line.startswith('#') or line.strip() == '': - continue - m = re.match(_variable_rx, line) - if m: - n, v = m.group(1, 2) - v = v.strip() - # `$$' is a literal `$' in make - tmpv = v.replace('$$', '') - - if "$" in tmpv: - notdone[n] = v - else: - try: - if n in _ALWAYS_STR: - raise ValueError - - v = int(v) - except ValueError: - # insert literal `$' - done[n] = v.replace('$$', '$') - else: - done[n] = v - - # do variable interpolation here - variables = list(notdone.keys()) - - # Variables with a 'PY_' prefix in the makefile. These need to - # be made available without that prefix through sysconfig. - # Special care is needed to ensure that variable expansion works, even - # if the expansion uses the name without a prefix. - renamed_variables = ('CFLAGS', 'LDFLAGS', 'CPPFLAGS') - - while len(variables) > 0: - for name in tuple(variables): - value = notdone[name] - m1 = re.search(_findvar1_rx, value) - m2 = re.search(_findvar2_rx, value) - if m1 and m2: - m = m1 if m1.start() < m2.start() else m2 - else: - m = m1 if m1 else m2 - if m is not None: - n = m.group(1) - found = True - if n in done: - item = str(done[n]) - elif n in notdone: - # get it on a subsequent round - found = False - elif n in os.environ: - # do it like make: fall back to environment - item = os.environ[n] - - elif n in renamed_variables: - if (name.startswith('PY_') and - name[3:] in renamed_variables): - item = "" - - elif 'PY_' + n in notdone: - found = False - - else: - item = str(done['PY_' + n]) - - else: - done[n] = item = "" - - if found: - after = value[m.end():] - value = value[:m.start()] + item + after - if "$" in after: - notdone[name] = value - else: - try: - if name in _ALWAYS_STR: - raise ValueError - value = int(value) - except ValueError: - done[name] = value.strip() - else: - done[name] = value - variables.remove(name) - - if name.startswith('PY_') \ - and name[3:] in renamed_variables: - - name = name[3:] - if name not in done: - done[name] = value - - else: - # Adds unresolved variables to the done dict. - # This is disabled when called from distutils.sysconfig - if keep_unresolved: - done[name] = value - # bogus variable reference (e.g. "prefix=$/opt/python"); - # just drop it since we can't deal - variables.remove(name) - - # strip spurious spaces - for k, v in done.items(): - if isinstance(v, str): - done[k] = v.strip() - - # save the results in the global dictionary - vars.update(done) - return vars - - def get_makefile_filename(): """Return the path of the Makefile.""" if _PYTHON_BUILD: @@ -462,91 +341,44 @@ def _get_sysconfigdata_name(): f'_sysconfigdata_{sys.abiflags}_{sys.platform}_{multiarch}', ) - -def _generate_posix_vars(): - """Generate the Python module containing build-time variables.""" - import pprint - vars = {} - # load the installed Makefile: - makefile = get_makefile_filename() - try: - _parse_makefile(makefile, vars) - except OSError as e: - msg = f"invalid Python installation: unable to open {makefile}" - if hasattr(e, "strerror"): - msg = f"{msg} ({e.strerror})" - raise OSError(msg) - # load the installed pyconfig.h: - config_h = get_config_h_filename() - try: - with open(config_h, encoding="utf-8") as f: - parse_config_h(f, vars) - except OSError as e: - msg = f"invalid Python installation: unable to open {config_h}" - if hasattr(e, "strerror"): - msg = f"{msg} ({e.strerror})" - raise OSError(msg) - # On AIX, there are wrong paths to the linker scripts in the Makefile - # -- these paths are relative to the Python source, but when installed - # the scripts are in another directory. - if _PYTHON_BUILD: - vars['BLDSHARED'] = vars['LDSHARED'] - - # There's a chicken-and-egg situation on OS X with regards to the - # _sysconfigdata module after the changes introduced by #15298: - # get_config_vars() is called by get_platform() as part of the - # `make pybuilddir.txt` target -- which is a precursor to the - # _sysconfigdata.py module being constructed. Unfortunately, - # get_config_vars() eventually calls _init_posix(), which attempts - # to import _sysconfigdata, which we won't have built yet. In order - # for _init_posix() to work, if we're on Darwin, just mock up the - # _sysconfigdata module manually and populate it with the build vars. - # This is more than sufficient for ensuring the subsequent call to - # get_platform() succeeds. - name = _get_sysconfigdata_name() - if 'darwin' in sys.platform: - import types - module = types.ModuleType(name) - module.build_time_vars = vars - sys.modules[name] = module - - pybuilddir = f'build/lib.{get_platform()}-{_PY_VERSION_SHORT}' - if hasattr(sys, "gettotalrefcount"): - pybuilddir += '-pydebug' - os.makedirs(pybuilddir, exist_ok=True) - destfile = os.path.join(pybuilddir, name + '.py') - - with open(destfile, 'w', encoding='utf8') as f: - f.write('# system configuration generated and used by' - ' the sysconfig module\n') - f.write('build_time_vars = ') - pprint.pprint(vars, stream=f) - - # Create file used for sys.path fixup -- see Modules/getpath.c - with open('pybuilddir.txt', 'w', encoding='utf8') as f: - f.write(pybuilddir) - def _init_posix(vars): """Initialize the module as appropriate for POSIX systems.""" # _sysconfigdata is generated at build time, see _generate_posix_vars() name = _get_sysconfigdata_name() - _temp = __import__(name, globals(), locals(), ['build_time_vars'], 0) + + # For cross builds, the path to the target's sysconfigdata must be specified + # so it can be imported. It cannot be in PYTHONPATH, as foreign modules in + # sys.path can cause crashes when loaded by the host interpreter. + # Rely on truthiness as a valueless env variable is still an empty string. + # See OS X note in _generate_posix_vars re _sysconfigdata. + if (path := os.environ.get('_PYTHON_SYSCONFIGDATA_PATH')): + from importlib.machinery import FileFinder, SourceFileLoader, SOURCE_SUFFIXES + from importlib.util import module_from_spec + spec = FileFinder(path, (SourceFileLoader, SOURCE_SUFFIXES)).find_spec(name) + _temp = module_from_spec(spec) + spec.loader.exec_module(_temp) + else: + _temp = __import__(name, globals(), locals(), ['build_time_vars'], 0) build_time_vars = _temp.build_time_vars vars.update(build_time_vars) def _init_non_posix(vars): """Initialize the module as appropriate for NT""" # set basic install directories - import _imp + import _winapi + import _sysconfig vars['LIBDEST'] = get_path('stdlib') vars['BINLIBDEST'] = get_path('platstdlib') vars['INCLUDEPY'] = get_path('include') - try: - # GH-99201: _imp.extension_suffixes may be empty when - # HAVE_DYNAMIC_LOADING is not set. In this case, don't set EXT_SUFFIX. - vars['EXT_SUFFIX'] = _imp.extension_suffixes()[0] - except IndexError: - pass + + # Add EXT_SUFFIX, SOABI, and Py_GIL_DISABLED + vars.update(_sysconfig.config_vars()) + + vars['LIBDIR'] = _safe_realpath(os.path.join(get_config_var('installed_base'), 'libs')) + if hasattr(sys, 'dllhandle'): + dllhandle = _winapi.GetModuleFileName(sys.dllhandle) + vars['LIBRARY'] = os.path.basename(_safe_realpath(dllhandle)) + vars['LDLIBRARY'] = vars['LIBRARY'] vars['EXE'] = '.exe' vars['VERSION'] = _PY_VERSION_SHORT_NO_DOT vars['BINDIR'] = os.path.dirname(_safe_realpath(sys.executable)) @@ -595,7 +427,7 @@ def get_config_h_filename(): """Return the path of pyconfig.h.""" if _PYTHON_BUILD: if os.name == "nt": - inc_dir = os.path.join(_PROJECT_BASE, "PC") + inc_dir = os.path.dirname(sys._base_executable) else: inc_dir = _PROJECT_BASE else: @@ -633,6 +465,78 @@ def get_path(name, scheme=get_default_scheme(), vars=None, expand=True): return get_paths(scheme, vars, expand)[name] +def _init_config_vars(): + global _CONFIG_VARS + _CONFIG_VARS = {} + # Normalized versions of prefix and exec_prefix are handy to have; + # in fact, these are the standard versions used most places in the + # Distutils. + _PREFIX = os.path.normpath(sys.prefix) + _EXEC_PREFIX = os.path.normpath(sys.exec_prefix) + _CONFIG_VARS['prefix'] = _PREFIX # FIXME: This gets overwriten by _init_posix. + _CONFIG_VARS['exec_prefix'] = _EXEC_PREFIX # FIXME: This gets overwriten by _init_posix. + _CONFIG_VARS['py_version'] = _PY_VERSION + _CONFIG_VARS['py_version_short'] = _PY_VERSION_SHORT + _CONFIG_VARS['py_version_nodot'] = _PY_VERSION_SHORT_NO_DOT + _CONFIG_VARS['installed_base'] = _BASE_PREFIX + _CONFIG_VARS['base'] = _PREFIX + _CONFIG_VARS['installed_platbase'] = _BASE_EXEC_PREFIX + _CONFIG_VARS['platbase'] = _EXEC_PREFIX + _CONFIG_VARS['projectbase'] = _PROJECT_BASE + _CONFIG_VARS['platlibdir'] = sys.platlibdir + _CONFIG_VARS['implementation'] = _get_implementation() + _CONFIG_VARS['implementation_lower'] = _get_implementation().lower() + try: + _CONFIG_VARS['abiflags'] = sys.abiflags + except AttributeError: + # sys.abiflags may not be defined on all platforms. + _CONFIG_VARS['abiflags'] = '' + try: + _CONFIG_VARS['py_version_nodot_plat'] = sys.winver.replace('.', '') + except AttributeError: + _CONFIG_VARS['py_version_nodot_plat'] = '' + + if os.name == 'nt': + _init_non_posix(_CONFIG_VARS) + _CONFIG_VARS['VPATH'] = sys._vpath + if os.name == 'posix': + _init_posix(_CONFIG_VARS) + if _HAS_USER_BASE: + # Setting 'userbase' is done below the call to the + # init function to enable using 'get_config_var' in + # the init-function. + _CONFIG_VARS['userbase'] = _getuserbase() + + # e.g., 't' for free-threaded or '' for default build + _CONFIG_VARS['abi_thread'] = 't' if _CONFIG_VARS.get('Py_GIL_DISABLED') else '' + + # Always convert srcdir to an absolute path + srcdir = _CONFIG_VARS.get('srcdir', _PROJECT_BASE) + if os.name == 'posix': + if _PYTHON_BUILD: + # If srcdir is a relative path (typically '.' or '..') + # then it should be interpreted relative to the directory + # containing Makefile. + base = os.path.dirname(get_makefile_filename()) + srcdir = os.path.join(base, srcdir) + else: + # srcdir is not meaningful since the installation is + # spread about the filesystem. We choose the + # directory containing the Makefile since we know it + # exists. + srcdir = os.path.dirname(get_makefile_filename()) + _CONFIG_VARS['srcdir'] = _safe_realpath(srcdir) + + # OS X platforms require special customization to handle + # multi-architecture, multi-os-version installers + if sys.platform == 'darwin': + import _osx_support + _osx_support.customize_config_vars(_CONFIG_VARS) + + global _CONFIG_VARS_INITIALIZED + _CONFIG_VARS_INITIALIZED = True + + def get_config_vars(*args): """With no arguments, return a dictionary of all configuration variables relevant for the current platform. @@ -643,66 +547,26 @@ def get_config_vars(*args): With arguments, return a list of values that result from looking up each argument in the configuration variable dictionary. """ - global _CONFIG_VARS - if _CONFIG_VARS is None: - _CONFIG_VARS = {} - # Normalized versions of prefix and exec_prefix are handy to have; - # in fact, these are the standard versions used most places in the - # Distutils. - _CONFIG_VARS['prefix'] = _PREFIX - _CONFIG_VARS['exec_prefix'] = _EXEC_PREFIX - _CONFIG_VARS['py_version'] = _PY_VERSION - _CONFIG_VARS['py_version_short'] = _PY_VERSION_SHORT - _CONFIG_VARS['py_version_nodot'] = _PY_VERSION_SHORT_NO_DOT - _CONFIG_VARS['installed_base'] = _BASE_PREFIX - _CONFIG_VARS['base'] = _PREFIX - _CONFIG_VARS['installed_platbase'] = _BASE_EXEC_PREFIX - _CONFIG_VARS['platbase'] = _EXEC_PREFIX - _CONFIG_VARS['projectbase'] = _PROJECT_BASE - _CONFIG_VARS['platlibdir'] = sys.platlibdir - try: - _CONFIG_VARS['abiflags'] = sys.abiflags - except AttributeError: - # sys.abiflags may not be defined on all platforms. - _CONFIG_VARS['abiflags'] = '' - try: - _CONFIG_VARS['py_version_nodot_plat'] = sys.winver.replace('.', '') - except AttributeError: - _CONFIG_VARS['py_version_nodot_plat'] = '' - - if os.name == 'nt': - _init_non_posix(_CONFIG_VARS) - _CONFIG_VARS['VPATH'] = sys._vpath - if os.name == 'posix': - _init_posix(_CONFIG_VARS) - if _HAS_USER_BASE: - # Setting 'userbase' is done below the call to the - # init function to enable using 'get_config_var' in - # the init-function. - _CONFIG_VARS['userbase'] = _getuserbase() - - # Always convert srcdir to an absolute path - srcdir = _CONFIG_VARS.get('srcdir', _PROJECT_BASE) - if os.name == 'posix': - if _PYTHON_BUILD: - # If srcdir is a relative path (typically '.' or '..') - # then it should be interpreted relative to the directory - # containing Makefile. - base = os.path.dirname(get_makefile_filename()) - srcdir = os.path.join(base, srcdir) - else: - # srcdir is not meaningful since the installation is - # spread about the filesystem. We choose the - # directory containing the Makefile since we know it - # exists. - srcdir = os.path.dirname(get_makefile_filename()) - _CONFIG_VARS['srcdir'] = _safe_realpath(srcdir) - - # OS X platforms require special customization to handle - # multi-architecture, multi-os-version installers - if sys.platform == 'darwin': - import _osx_support - _osx_support.customize_config_vars(_CONFIG_VARS) + global _CONFIG_VARS_INITIALIZED + + # Avoid claiming the lock once initialization is complete. + if not _CONFIG_VARS_INITIALIZED: + with _CONFIG_VARS_LOCK: + # Test again with the lock held to avoid races. Note that + # we test _CONFIG_VARS here, not _CONFIG_VARS_INITIALIZED, + # to ensure that recursive calls to get_config_vars() + # don't re-enter init_config_vars(). + if _CONFIG_VARS is None: + _init_config_vars() + else: + # If the site module initialization happened after _CONFIG_VARS was + # initialized, a virtual environment might have been activated, resulting in + # variables like sys.prefix changing their value, so we need to re-init the + # config vars (see GH-126789). + if _CONFIG_VARS['base'] != os.path.normpath(sys.prefix): + with _CONFIG_VARS_LOCK: + _CONFIG_VARS_INITIALIZED = False + _init_config_vars() if args: vals = [] @@ -732,17 +596,22 @@ def get_platform(): isn't particularly important. Examples of returned values: - linux-i586 - linux-alpha (?) - solaris-2.6-sun4u - Windows will return one of: - win-amd64 (64bit Windows on AMD64 (aka x86_64, Intel64, EM64T, etc) - win32 (all others - specifically, sys.platform is returned) - For other non-POSIX platforms, currently just returns 'sys.platform'. + Windows: - """ + - win-amd64 (64-bit Windows on AMD64, aka x86_64, Intel64, and EM64T) + - win-arm64 (64-bit Windows on ARM64, aka AArch64) + - win32 (all others - specifically, sys.platform is returned) + + POSIX based OS: + + - linux-x86_64 + - macosx-15.5-arm64 + - macosx-26.0-universal2 (macOS on Apple Silicon or Intel) + - android-24-arm64_v8a + + For other non-POSIX platforms, currently just returns :data:`sys.platform`.""" if os.name == 'nt': if 'amd64' in sys.version.lower(): return 'win-amd64' @@ -770,10 +639,22 @@ def get_platform(): machine = machine.replace('/', '-') if osname[:5] == "linux": - # At least on Linux/Intel, 'machine' is the processor -- - # i386, etc. - # XXX what about Alpha, SPARC, etc? - return f"{osname}-{machine}" + if sys.platform == "android": + osname = "android" + release = get_config_var("ANDROID_API_LEVEL") + + # Wheel tags use the ABI names from Android's own tools. + machine = { + "x86_64": "x86_64", + "i686": "x86", + "aarch64": "arm64_v8a", + "armv7l": "armeabi_v7a", + }[machine] + else: + # At least on Linux/Intel, 'machine' is the processor -- + # i386, etc. + # XXX what about Alpha, SPARC, etc? + return f"{osname}-{machine}" elif osname[:5] == "sunos": if release[0] >= "5": # SunOS 5 == Solaris 2 osname = "solaris" @@ -795,10 +676,15 @@ def get_platform(): if m: release = m.group() elif osname[:6] == "darwin": - import _osx_support - osname, release, machine = _osx_support.get_platform_osx( - get_config_vars(), - osname, release, machine) + if sys.platform == "ios": + release = get_config_vars().get("IPHONEOS_DEPLOYMENT_TARGET", "13.0") + osname = sys.platform + machine = sys.implementation._multiarch + else: + import _osx_support + osname, release, machine = _osx_support.get_platform_osx( + get_config_vars(), + osname, release, machine) return f"{osname}-{release}-{machine}" @@ -807,6 +693,10 @@ def get_python_version(): return _PY_VERSION_SHORT +def _get_python_version_abi(): + return _PY_VERSION_SHORT + get_config_var("abi_thread") + + def expand_makefile_vars(s, vars): """Expand Makefile-style variables -- "${foo}" or "$(foo)" -- in 'string' according to 'vars' (a dictionary mapping variable names to @@ -817,6 +707,9 @@ def expand_makefile_vars(s, vars): """ import re + _findvar1_rx = r"\$\(([A-Za-z][A-Za-z0-9_]*)\)" + _findvar2_rx = r"\${([A-Za-z][A-Za-z0-9_]*)}" + # This algorithm does multiple expansion, so if vars['foo'] contains # "${bar}", it will expand ${foo} to ${bar}, and then expand # ${bar}... and so forth. This is fine as long as 'vars' comes from @@ -831,28 +724,3 @@ def expand_makefile_vars(s, vars): else: break return s - - -def _print_dict(title, data): - for index, (key, value) in enumerate(sorted(data.items())): - if index == 0: - print(f'{title}: ') - print(f'\t{key} = "{value}"') - - -def _main(): - """Display all information sysconfig detains.""" - if '--generate-posix-vars' in sys.argv: - _generate_posix_vars() - return - print(f'Platform: "{get_platform()}"') - print(f'Python version: "{get_python_version()}"') - print(f'Current installation scheme: "{get_default_scheme()}"') - print() - _print_dict('Paths', get_paths()) - print() - _print_dict('Variables', get_config_vars()) - -if __name__ == '__main__': - _main() -'''.replace("Python", "RustPython").replace("/python", "/rustpython")) diff --git a/Lib/sysconfig/__main__.py b/Lib/sysconfig/__main__.py new file mode 100644 index 0000000000..d7257b9d2d --- /dev/null +++ b/Lib/sysconfig/__main__.py @@ -0,0 +1,248 @@ +import os +import sys +from sysconfig import ( + _ALWAYS_STR, + _PYTHON_BUILD, + _get_sysconfigdata_name, + get_config_h_filename, + get_config_vars, + get_default_scheme, + get_makefile_filename, + get_paths, + get_platform, + get_python_version, + parse_config_h, +) + + +# Regexes needed for parsing Makefile (and similar syntaxes, +# like old-style Setup files). +_variable_rx = r"([a-zA-Z][a-zA-Z0-9_]+)\s*=\s*(.*)" +_findvar1_rx = r"\$\(([A-Za-z][A-Za-z0-9_]*)\)" +_findvar2_rx = r"\${([A-Za-z][A-Za-z0-9_]*)}" + + +def _parse_makefile(filename, vars=None, keep_unresolved=True): + """Parse a Makefile-style file. + + A dictionary containing name/value pairs is returned. If an + optional dictionary is passed in as the second argument, it is + used instead of a new dictionary. + """ + import re + + if vars is None: + vars = {} + done = {} + notdone = {} + + with open(filename, encoding=sys.getfilesystemencoding(), + errors="surrogateescape") as f: + lines = f.readlines() + + for line in lines: + if line.startswith('#') or line.strip() == '': + continue + m = re.match(_variable_rx, line) + if m: + n, v = m.group(1, 2) + v = v.strip() + # `$$' is a literal `$' in make + tmpv = v.replace('$$', '') + + if "$" in tmpv: + notdone[n] = v + else: + try: + if n in _ALWAYS_STR: + raise ValueError + + v = int(v) + except ValueError: + # insert literal `$' + done[n] = v.replace('$$', '$') + else: + done[n] = v + + # do variable interpolation here + variables = list(notdone.keys()) + + # Variables with a 'PY_' prefix in the makefile. These need to + # be made available without that prefix through sysconfig. + # Special care is needed to ensure that variable expansion works, even + # if the expansion uses the name without a prefix. + renamed_variables = ('CFLAGS', 'LDFLAGS', 'CPPFLAGS') + + while len(variables) > 0: + for name in tuple(variables): + value = notdone[name] + m1 = re.search(_findvar1_rx, value) + m2 = re.search(_findvar2_rx, value) + if m1 and m2: + m = m1 if m1.start() < m2.start() else m2 + else: + m = m1 if m1 else m2 + if m is not None: + n = m.group(1) + found = True + if n in done: + item = str(done[n]) + elif n in notdone: + # get it on a subsequent round + found = False + elif n in os.environ: + # do it like make: fall back to environment + item = os.environ[n] + + elif n in renamed_variables: + if (name.startswith('PY_') and + name[3:] in renamed_variables): + item = "" + + elif 'PY_' + n in notdone: + found = False + + else: + item = str(done['PY_' + n]) + + else: + done[n] = item = "" + + if found: + after = value[m.end():] + value = value[:m.start()] + item + after + if "$" in after: + notdone[name] = value + else: + try: + if name in _ALWAYS_STR: + raise ValueError + value = int(value) + except ValueError: + done[name] = value.strip() + else: + done[name] = value + variables.remove(name) + + if name.startswith('PY_') \ + and name[3:] in renamed_variables: + + name = name[3:] + if name not in done: + done[name] = value + + else: + # Adds unresolved variables to the done dict. + # This is disabled when called from distutils.sysconfig + if keep_unresolved: + done[name] = value + # bogus variable reference (e.g. "prefix=$/opt/python"); + # just drop it since we can't deal + variables.remove(name) + + # strip spurious spaces + for k, v in done.items(): + if isinstance(v, str): + done[k] = v.strip() + + # save the results in the global dictionary + vars.update(done) + return vars + + +def _print_config_dict(d, stream): + print ("{", file=stream) + for k, v in sorted(d.items()): + print(f" {k!r}: {v!r},", file=stream) + print ("}", file=stream) + + +def _generate_posix_vars(): + """Generate the Python module containing build-time variables.""" + vars = {} + # load the installed Makefile: + makefile = get_makefile_filename() + try: + _parse_makefile(makefile, vars) + except OSError as e: + msg = f"invalid Python installation: unable to open {makefile}" + if hasattr(e, "strerror"): + msg = f"{msg} ({e.strerror})" + raise OSError(msg) + # load the installed pyconfig.h: + config_h = get_config_h_filename() + try: + with open(config_h, encoding="utf-8") as f: + parse_config_h(f, vars) + except OSError as e: + msg = f"invalid Python installation: unable to open {config_h}" + if hasattr(e, "strerror"): + msg = f"{msg} ({e.strerror})" + raise OSError(msg) + # On AIX, there are wrong paths to the linker scripts in the Makefile + # -- these paths are relative to the Python source, but when installed + # the scripts are in another directory. + if _PYTHON_BUILD: + vars['BLDSHARED'] = vars['LDSHARED'] + + # There's a chicken-and-egg situation on OS X with regards to the + # _sysconfigdata module after the changes introduced by #15298: + # get_config_vars() is called by get_platform() as part of the + # `make pybuilddir.txt` target -- which is a precursor to the + # _sysconfigdata.py module being constructed. Unfortunately, + # get_config_vars() eventually calls _init_posix(), which attempts + # to import _sysconfigdata, which we won't have built yet. In order + # for _init_posix() to work, if we're on Darwin, just mock up the + # _sysconfigdata module manually and populate it with the build vars. + # This is more than sufficient for ensuring the subsequent call to + # get_platform() succeeds. + name = _get_sysconfigdata_name() + if 'darwin' in sys.platform: + import types + module = types.ModuleType(name) + module.build_time_vars = vars + sys.modules[name] = module + + pybuilddir = f'build/lib.{get_platform()}-{get_python_version()}' + if hasattr(sys, "gettotalrefcount"): + pybuilddir += '-pydebug' + os.makedirs(pybuilddir, exist_ok=True) + destfile = os.path.join(pybuilddir, name + '.py') + + with open(destfile, 'w', encoding='utf8') as f: + f.write('# system configuration generated and used by' + ' the sysconfig module\n') + f.write('build_time_vars = ') + _print_config_dict(vars, stream=f) + + # Create file used for sys.path fixup -- see Modules/getpath.c + with open('pybuilddir.txt', 'w', encoding='utf8') as f: + f.write(pybuilddir) + + +def _print_dict(title, data): + for index, (key, value) in enumerate(sorted(data.items())): + if index == 0: + print(f'{title}: ') + print(f'\t{key} = "{value}"') + + +def _main(): + """Display all information sysconfig detains.""" + if '--generate-posix-vars' in sys.argv: + _generate_posix_vars() + return + print(f'Platform: "{get_platform()}"') + print(f'Python version: "{get_python_version()}"') + print(f'Current installation scheme: "{get_default_scheme()}"') + print() + _print_dict('Paths', get_paths()) + print() + _print_dict('Variables', get_config_vars()) + + +if __name__ == '__main__': + try: + _main() + except BrokenPipeError: + pass diff --git a/Lib/tabnanny.py b/Lib/tabnanny.py index 7973f26f98..d06c4c221e 100755 --- a/Lib/tabnanny.py +++ b/Lib/tabnanny.py @@ -23,8 +23,6 @@ import os import sys import tokenize -if not hasattr(tokenize, 'NL'): - raise ValueError("tokenize.NL doesn't exist -- tokenize module too old") __all__ = ["check", "NannyNag", "process_tokens"] @@ -37,6 +35,7 @@ def errprint(*args): sys.stderr.write(sep + str(arg)) sep = " " sys.stderr.write("\n") + sys.exit(1) def main(): import getopt @@ -46,7 +45,6 @@ def main(): opts, args = getopt.getopt(sys.argv[1:], "qv") except getopt.error as msg: errprint(msg) - return for o, a in opts: if o == '-q': filename_only = filename_only + 1 @@ -54,7 +52,6 @@ def main(): verbose = verbose + 1 if not args: errprint("Usage:", sys.argv[0], "[-v] file_or_directory ...") - return for arg in args: check(arg) @@ -114,6 +111,10 @@ def check(file): errprint("%r: Indentation Error: %s" % (file, msg)) return + except SyntaxError as msg: + errprint("%r: Syntax Error: %s" % (file, msg)) + return + except NannyNag as nag: badline = nag.get_lineno() line = nag.get_line() @@ -275,6 +276,12 @@ def format_witnesses(w): return prefix + " " + ', '.join(firsts) def process_tokens(tokens): + try: + _process_tokens(tokens) + except TabError as e: + raise NannyNag(e.lineno, e.msg, e.text) + +def _process_tokens(tokens): INDENT = tokenize.INDENT DEDENT = tokenize.DEDENT NEWLINE = tokenize.NEWLINE diff --git a/Lib/test/_test_multiprocessing.py b/Lib/test/_test_multiprocessing.py index 9e688efb1e..c22ce769c4 100644 --- a/Lib/test/_test_multiprocessing.py +++ b/Lib/test/_test_multiprocessing.py @@ -12,6 +12,7 @@ import sys import os import gc +import importlib import errno import functools import signal @@ -19,10 +20,11 @@ import socket import random import logging +import shutil import subprocess import struct +import tempfile import operator -import pathlib import pickle import weakref import warnings @@ -50,7 +52,7 @@ import multiprocessing.managers import multiprocessing.pool import multiprocessing.queues -from multiprocessing.connection import wait, AuthenticationError +from multiprocessing.connection import wait from multiprocessing import util @@ -255,6 +257,9 @@ def __call__(self, *args, **kwds): class BaseTestCase(object): ALLOWED_TYPES = ('processes', 'manager', 'threads') + # If not empty, limit which start method suites run this class. + START_METHODS: set[str] = set() + start_method = None # set by install_tests_in_module_dict() def assertTimingAlmostEqual(self, a, b): if CHECK_TIMINGS: @@ -324,8 +329,9 @@ def test_set_executable(self): self.skipTest(f'test not appropriate for {self.TYPE}') paths = [ sys.executable, # str - sys.executable.encode(), # bytes - pathlib.Path(sys.executable) # os.PathLike + os.fsencode(sys.executable), # bytes + os_helper.FakePath(sys.executable), # os.PathLike + os_helper.FakePath(os.fsencode(sys.executable)), # os.PathLike bytes ] for path in paths: self.set_executable(path) @@ -505,6 +511,11 @@ def _test_process_mainthread_native_id(cls, q): def _sleep_some(cls): time.sleep(100) + @classmethod + def _sleep_some_event(cls, event): + event.set() + time.sleep(100) + @classmethod def _test_sleep(cls, delay): time.sleep(delay) @@ -513,7 +524,8 @@ def _kill_process(self, meth): if self.TYPE == 'threads': self.skipTest('test not appropriate for {}'.format(self.TYPE)) - p = self.Process(target=self._sleep_some) + event = self.Event() + p = self.Process(target=self._sleep_some_event, args=(event,)) p.daemon = True p.start() @@ -531,8 +543,11 @@ def _kill_process(self, meth): self.assertTimingAlmostEqual(join.elapsed, 0.0) self.assertEqual(p.is_alive(), True) - # XXX maybe terminating too soon causes the problems on Gentoo... - time.sleep(1) + timeout = support.SHORT_TIMEOUT + if not event.wait(timeout): + p.terminate() + p.join() + self.fail(f"event not signaled in {timeout} seconds") meth(p) @@ -582,12 +597,16 @@ def test_cpu_count(self): def test_active_children(self): self.assertEqual(type(self.active_children()), list) - p = self.Process(target=time.sleep, args=(DELTA,)) + event = self.Event() + p = self.Process(target=event.wait, args=()) self.assertNotIn(p, self.active_children()) - p.daemon = True - p.start() - self.assertIn(p, self.active_children()) + try: + p.daemon = True + p.start() + self.assertIn(p, self.active_children()) + finally: + event.set() p.join() self.assertNotIn(p, self.active_children()) @@ -1332,6 +1351,23 @@ def _on_queue_feeder_error(e, obj): self.assertTrue(not_serializable_obj.reduce_was_called) self.assertTrue(not_serializable_obj.on_queue_feeder_error_was_called) + def test_closed_queue_empty_exceptions(self): + # Assert that checking the emptiness of an unused closed queue + # does not raise an OSError. The rationale is that q.close() is + # a no-op upon construction and becomes effective once the queue + # has been used (e.g., by calling q.put()). + for q in multiprocessing.Queue(), multiprocessing.JoinableQueue(): + q.close() # this is a no-op since the feeder thread is None + q.join_thread() # this is also a no-op + self.assertTrue(q.empty()) + + for q in multiprocessing.Queue(), multiprocessing.JoinableQueue(): + q.put('foo') # make sure that the queue is 'used' + q.close() # close the feeder thread + q.join_thread() # make sure to join the feeder thread + with self.assertRaisesRegex(OSError, 'is closed'): + q.empty() + def test_closed_queue_put_get_exceptions(self): for q in multiprocessing.Queue(), multiprocessing.JoinableQueue(): q.close() @@ -1345,6 +1381,66 @@ def test_closed_queue_put_get_exceptions(self): class _TestLock(BaseTestCase): + @staticmethod + def _acquire(lock, l=None): + lock.acquire() + if l is not None: + l.append(repr(lock)) + + @staticmethod + def _acquire_event(lock, event): + lock.acquire() + event.set() + time.sleep(1.0) + + def test_repr_lock(self): + if self.TYPE != 'processes': + self.skipTest('test not appropriate for {}'.format(self.TYPE)) + + lock = self.Lock() + self.assertEqual(f'', repr(lock)) + + lock.acquire() + self.assertEqual(f'', repr(lock)) + lock.release() + + tname = 'T1' + l = [] + t = threading.Thread(target=self._acquire, + args=(lock, l), + name=tname) + t.start() + time.sleep(0.1) + self.assertEqual(f'', l[0]) + lock.release() + + t = threading.Thread(target=self._acquire, + args=(lock,), + name=tname) + t.start() + time.sleep(0.1) + self.assertEqual('', repr(lock)) + lock.release() + + pname = 'P1' + l = multiprocessing.Manager().list() + p = self.Process(target=self._acquire, + args=(lock, l), + name=pname) + p.start() + p.join() + self.assertEqual(f'', l[0]) + + lock = self.Lock() + event = self.Event() + p = self.Process(target=self._acquire_event, + args=(lock, event), + name='P2') + p.start() + event.wait() + self.assertEqual(f'', repr(lock)) + p.terminate() + def test_lock(self): lock = self.Lock() self.assertEqual(lock.acquire(), True) @@ -1352,6 +1448,68 @@ def test_lock(self): self.assertEqual(lock.release(), None) self.assertRaises((ValueError, threading.ThreadError), lock.release) + @staticmethod + def _acquire_release(lock, timeout, l=None, n=1): + for _ in range(n): + lock.acquire() + if l is not None: + l.append(repr(lock)) + time.sleep(timeout) + for _ in range(n): + lock.release() + + def test_repr_rlock(self): + if self.TYPE != 'processes': + self.skipTest('test not appropriate for {}'.format(self.TYPE)) + + lock = self.RLock() + self.assertEqual('', repr(lock)) + + n = 3 + for _ in range(n): + lock.acquire() + self.assertEqual(f'', repr(lock)) + for _ in range(n): + lock.release() + + t, l = [], [] + for i in range(n): + t.append(threading.Thread(target=self._acquire_release, + args=(lock, 0.1, l, i+1), + name=f'T{i+1}')) + t[-1].start() + for t_ in t: + t_.join() + for i in range(n): + self.assertIn(f'', l) + + + t = threading.Thread(target=self._acquire_release, + args=(lock, 0.2), + name=f'T1') + t.start() + time.sleep(0.1) + self.assertEqual('', repr(lock)) + time.sleep(0.2) + + pname = 'P1' + l = multiprocessing.Manager().list() + p = self.Process(target=self._acquire_release, + args=(lock, 0.1, l), + name=pname) + p.start() + p.join() + self.assertEqual(f'', l[0]) + + event = self.Event() + lock = self.RLock() + p = self.Process(target=self._acquire_event, + args=(lock, event)) + p.start() + event.wait() + self.assertEqual('', repr(lock)) + p.join() + def test_rlock(self): lock = self.RLock() self.assertEqual(lock.acquire(), True) @@ -1432,14 +1590,13 @@ def f(cls, cond, sleeping, woken, timeout=None): cond.release() def assertReachesEventually(self, func, value): - for i in range(10): + for _ in support.sleeping_retry(support.SHORT_TIMEOUT): try: if func() == value: break except NotImplementedError: break - time.sleep(DELTA) - time.sleep(DELTA) + self.assertReturnsIfImplemented(value, func) def check_invariant(self, cond): @@ -1461,20 +1618,17 @@ def test_notify(self): p = self.Process(target=self.f, args=(cond, sleeping, woken)) p.daemon = True p.start() - self.addCleanup(p.join) - p = threading.Thread(target=self.f, args=(cond, sleeping, woken)) - p.daemon = True - p.start() - self.addCleanup(p.join) + t = threading.Thread(target=self.f, args=(cond, sleeping, woken)) + t.daemon = True + t.start() # wait for both children to start sleeping sleeping.acquire() sleeping.acquire() # check no process/thread has woken up - time.sleep(DELTA) - self.assertReturnsIfImplemented(0, get_value, woken) + self.assertReachesEventually(lambda: get_value(woken), 0) # wake up one process/thread cond.acquire() @@ -1482,8 +1636,7 @@ def test_notify(self): cond.release() # check one process/thread has woken up - time.sleep(DELTA) - self.assertReturnsIfImplemented(1, get_value, woken) + self.assertReachesEventually(lambda: get_value(woken), 1) # wake up another cond.acquire() @@ -1491,12 +1644,13 @@ def test_notify(self): cond.release() # check other has woken up - time.sleep(DELTA) - self.assertReturnsIfImplemented(2, get_value, woken) + self.assertReachesEventually(lambda: get_value(woken), 2) # check state is not mucked up self.check_invariant(cond) - p.join() + + threading_helper.join_thread(t) + join_process(p) def test_notify_all(self): cond = self.Condition() @@ -1504,18 +1658,19 @@ def test_notify_all(self): woken = self.Semaphore(0) # start some threads/processes which will timeout + workers = [] for i in range(3): p = self.Process(target=self.f, args=(cond, sleeping, woken, TIMEOUT1)) p.daemon = True p.start() - self.addCleanup(p.join) + workers.append(p) t = threading.Thread(target=self.f, args=(cond, sleeping, woken, TIMEOUT1)) t.daemon = True t.start() - self.addCleanup(t.join) + workers.append(t) # wait for them all to sleep for i in range(6): @@ -1534,12 +1689,12 @@ def test_notify_all(self): p = self.Process(target=self.f, args=(cond, sleeping, woken)) p.daemon = True p.start() - self.addCleanup(p.join) + workers.append(p) t = threading.Thread(target=self.f, args=(cond, sleeping, woken)) t.daemon = True t.start() - self.addCleanup(t.join) + workers.append(t) # wait for them to all sleep for i in range(6): @@ -1555,27 +1710,34 @@ def test_notify_all(self): cond.release() # check they have all woken - self.assertReachesEventually(lambda: get_value(woken), 6) + for i in range(6): + woken.acquire() + self.assertReturnsIfImplemented(0, get_value, woken) # check state is not mucked up self.check_invariant(cond) + for w in workers: + # NOTE: join_process and join_thread are the same + threading_helper.join_thread(w) + def test_notify_n(self): cond = self.Condition() sleeping = self.Semaphore(0) woken = self.Semaphore(0) # start some threads/processes + workers = [] for i in range(3): p = self.Process(target=self.f, args=(cond, sleeping, woken)) p.daemon = True p.start() - self.addCleanup(p.join) + workers.append(p) t = threading.Thread(target=self.f, args=(cond, sleeping, woken)) t.daemon = True t.start() - self.addCleanup(t.join) + workers.append(t) # wait for them to all sleep for i in range(6): @@ -1610,6 +1772,10 @@ def test_notify_n(self): # check state is not mucked up self.check_invariant(cond) + for w in workers: + # NOTE: join_process and join_thread are the same + threading_helper.join_thread(w) + def test_timeout(self): cond = self.Condition() wait = TimingWrapper(cond.wait) @@ -2812,8 +2978,8 @@ def test_release_task_refs(self): self.pool.map(identity, objs) del objs - gc.collect() # For PyPy or other GCs. time.sleep(DELTA) # let threaded cleanup code run + support.gc_collect() # For PyPy or other GCs. self.assertEqual(set(wr() for wr in refs), {None}) # With a process pool, copies of the objects are returned, check # they were released too. @@ -3174,6 +3340,44 @@ def test_rapid_restart(self): if hasattr(manager, "shutdown"): self.addCleanup(manager.shutdown) + +class FakeConnection: + def send(self, payload): + pass + + def recv(self): + return '#ERROR', pyqueue.Empty() + +class TestManagerExceptions(unittest.TestCase): + # Issue 106558: Manager exceptions avoids creating cyclic references. + def setUp(self): + self.mgr = multiprocessing.Manager() + + def tearDown(self): + self.mgr.shutdown() + self.mgr.join() + + def test_queue_get(self): + queue = self.mgr.Queue() + if gc.isenabled(): + gc.disable() + self.addCleanup(gc.enable) + try: + queue.get_nowait() + except pyqueue.Empty as e: + wr = weakref.ref(e) + self.assertEqual(wr(), None) + + def test_dispatch(self): + if gc.isenabled(): + gc.disable() + self.addCleanup(gc.enable) + try: + multiprocessing.managers.dispatch(FakeConnection(), None, None) + except pyqueue.Empty as e: + wr = weakref.ref(e) + self.assertEqual(wr(), None) + # # # @@ -4462,6 +4666,59 @@ def test_shared_memory_cleaned_after_process_termination(self): "resource_tracker: There appear to be 1 leaked " "shared_memory objects to clean up at shutdown", err) + @unittest.skipIf(os.name != "posix", "resource_tracker is posix only") + def test_shared_memory_untracking(self): + # gh-82300: When a separate Python process accesses shared memory + # with track=False, it must not cause the memory to be deleted + # when terminating. + cmd = '''if 1: + import sys + from multiprocessing.shared_memory import SharedMemory + mem = SharedMemory(create=False, name=sys.argv[1], track=False) + mem.close() + ''' + mem = shared_memory.SharedMemory(create=True, size=10) + # The resource tracker shares pipes with the subprocess, and so + # err existing means that the tracker process has terminated now. + try: + rc, out, err = script_helper.assert_python_ok("-c", cmd, mem.name) + self.assertNotIn(b"resource_tracker", err) + self.assertEqual(rc, 0) + mem2 = shared_memory.SharedMemory(create=False, name=mem.name) + mem2.close() + finally: + try: + mem.unlink() + except OSError: + pass + mem.close() + + @unittest.skipIf(os.name != "posix", "resource_tracker is posix only") + def test_shared_memory_tracking(self): + # gh-82300: When a separate Python process accesses shared memory + # with track=True, it must cause the memory to be deleted when + # terminating. + cmd = '''if 1: + import sys + from multiprocessing.shared_memory import SharedMemory + mem = SharedMemory(create=False, name=sys.argv[1], track=True) + mem.close() + ''' + mem = shared_memory.SharedMemory(create=True, size=10) + try: + rc, out, err = script_helper.assert_python_ok("-c", cmd, mem.name) + self.assertEqual(rc, 0) + self.assertIn( + b"resource_tracker: There appear to be 1 leaked " + b"shared_memory objects to clean up at shutdown", err) + finally: + try: + mem.unlink() + except OSError: + pass + resource_tracker.unregister(mem._name, "shared_memory") + mem.close() + # # Test to verify that `Finalize` works. # @@ -4571,7 +4828,7 @@ def make_finalizers(): old_interval = sys.getswitchinterval() old_threshold = gc.get_threshold() try: - sys.setswitchinterval(1e-6) + support.setswitchinterval(1e-6) gc.set_threshold(5, 5, 5) threads = [threading.Thread(target=run_finalizers), threading.Thread(target=make_finalizers)] @@ -5557,8 +5814,9 @@ def create_and_register_resource(rtype): ''' for rtype in resource_tracker._CLEANUP_FUNCS: with self.subTest(rtype=rtype): - if rtype == "noop": + if rtype in ("noop", "dummy"): # Artefact resource type used by the resource_tracker + # or tests continue r, w = os.pipe() p = subprocess.Popen([sys.executable, @@ -5638,6 +5896,8 @@ def test_resource_tracker_sigterm(self): # Catchable signal (ignored by semaphore tracker) self.check_resource_tracker_death(signal.SIGTERM, False) + @unittest.skipIf(sys.platform.startswith("netbsd"), + "gh-125620: Skip on NetBSD due to long wait for SIGKILL process termination.") def test_resource_tracker_sigkill(self): # Uncatchable signal. self.check_resource_tracker_death(signal.SIGKILL, True) @@ -5678,6 +5938,59 @@ def test_too_long_name_resource(self): with self.assertRaises(ValueError): resource_tracker.register(too_long_name_resource, rtype) + def _test_resource_tracker_leak_resources(self, cleanup): + # We use a separate instance for testing, since the main global + # _resource_tracker may be used to watch test infrastructure. + from multiprocessing.resource_tracker import ResourceTracker + tracker = ResourceTracker() + tracker.ensure_running() + self.assertTrue(tracker._check_alive()) + + self.assertIsNone(tracker._exitcode) + tracker.register('somename', 'dummy') + if cleanup: + tracker.unregister('somename', 'dummy') + expected_exit_code = 0 + else: + expected_exit_code = 1 + + self.assertTrue(tracker._check_alive()) + self.assertIsNone(tracker._exitcode) + tracker._stop() + self.assertEqual(tracker._exitcode, expected_exit_code) + + def test_resource_tracker_exit_code(self): + """ + Test the exit code of the resource tracker. + + If no leaked resources were found, exit code should be 0, otherwise 1 + """ + for cleanup in [True, False]: + with self.subTest(cleanup=cleanup): + self._test_resource_tracker_leak_resources( + cleanup=cleanup, + ) + + @unittest.skipUnless(hasattr(signal, "pthread_sigmask"), "pthread_sigmask is not available") + def test_resource_tracker_blocked_signals(self): + # + # gh-127586: Check that resource_tracker does not override blocked signals of caller. + # + from multiprocessing.resource_tracker import ResourceTracker + orig_sigmask = signal.pthread_sigmask(signal.SIG_BLOCK, set()) + signals = {signal.SIGTERM, signal.SIGINT, signal.SIGUSR1} + + try: + for sig in signals: + signal.pthread_sigmask(signal.SIG_SETMASK, {sig}) + self.assertEqual(signal.pthread_sigmask(signal.SIG_BLOCK, set()), {sig}) + tracker = ResourceTracker() + tracker.ensure_running() + self.assertEqual(signal.pthread_sigmask(signal.SIG_BLOCK, set()), {sig}) + tracker._stop() + finally: + # restore sigmask to what it was before executing test + signal.pthread_sigmask(signal.SIG_SETMASK, orig_sigmask) class TestSimpleQueue(unittest.TestCase): @@ -5691,6 +6004,15 @@ def _test_empty(cls, queue, child_can_start, parent_can_continue): finally: parent_can_continue.set() + def test_empty_exceptions(self): + # Assert that checking emptiness of a closed queue raises + # an OSError, independently of whether the queue was used + # or not. This differs from Queue and JoinableQueue. + q = multiprocessing.SimpleQueue() + q.close() # close the pipe + with self.assertRaisesRegex(OSError, 'is closed'): + q.empty() + def test_empty(self): queue = multiprocessing.SimpleQueue() child_can_start = multiprocessing.Event() @@ -6037,6 +6359,128 @@ def submain(): pass self.assertFalse(err, msg=err.decode('utf-8')) +class _TestAtExit(BaseTestCase): + + ALLOWED_TYPES = ('processes',) + + @classmethod + def _write_file_at_exit(self, output_path): + import atexit + def exit_handler(): + with open(output_path, 'w') as f: + f.write("deadbeef") + atexit.register(exit_handler) + + def test_atexit(self): + # gh-83856 + with os_helper.temp_dir() as temp_dir: + output_path = os.path.join(temp_dir, 'output.txt') + p = self.Process(target=self._write_file_at_exit, args=(output_path,)) + p.start() + p.join() + with open(output_path) as f: + self.assertEqual(f.read(), 'deadbeef') + + +class _TestSpawnedSysPath(BaseTestCase): + """Test that sys.path is setup in forkserver and spawn processes.""" + + ALLOWED_TYPES = {'processes'} + # Not applicable to fork which inherits everything from the process as is. + START_METHODS = {"forkserver", "spawn"} + + def setUp(self): + self._orig_sys_path = list(sys.path) + self._temp_dir = tempfile.mkdtemp(prefix="test_sys_path-") + self._mod_name = "unique_test_mod" + module_path = os.path.join(self._temp_dir, f"{self._mod_name}.py") + with open(module_path, "w", encoding="utf-8") as mod: + mod.write("# A simple test module\n") + sys.path[:] = [p for p in sys.path if p] # remove any existing ""s + sys.path.insert(0, self._temp_dir) + sys.path.insert(0, "") # Replaced with an abspath in child. + self.assertIn(self.start_method, self.START_METHODS) + self._ctx = multiprocessing.get_context(self.start_method) + + def tearDown(self): + sys.path[:] = self._orig_sys_path + shutil.rmtree(self._temp_dir, ignore_errors=True) + + @staticmethod + def enq_imported_module_names(queue): + queue.put(tuple(sys.modules)) + + def test_forkserver_preload_imports_sys_path(self): + if self._ctx.get_start_method() != "forkserver": + self.skipTest("forkserver specific test.") + self.assertNotIn(self._mod_name, sys.modules) + multiprocessing.forkserver._forkserver._stop() # Must be fresh. + self._ctx.set_forkserver_preload( + ["test.test_multiprocessing_forkserver", self._mod_name]) + q = self._ctx.Queue() + proc = self._ctx.Process( + target=self.enq_imported_module_names, args=(q,)) + proc.start() + proc.join() + child_imported_modules = q.get() + q.close() + self.assertIn(self._mod_name, child_imported_modules) + + @staticmethod + def enq_sys_path_and_import(queue, mod_name): + queue.put(sys.path) + try: + importlib.import_module(mod_name) + except ImportError as exc: + queue.put(exc) + else: + queue.put(None) + + def test_child_sys_path(self): + q = self._ctx.Queue() + proc = self._ctx.Process( + target=self.enq_sys_path_and_import, args=(q, self._mod_name)) + proc.start() + proc.join() + child_sys_path = q.get() + import_error = q.get() + q.close() + self.assertNotIn("", child_sys_path) # replaced by an abspath + self.assertIn(self._temp_dir, child_sys_path) # our addition + # ignore the first element, it is the absolute "" replacement + self.assertEqual(child_sys_path[1:], sys.path[1:]) + self.assertIsNone(import_error, msg=f"child could not import {self._mod_name}") + + def test_std_streams_flushed_after_preload(self): + # gh-135335: Check fork server flushes standard streams after + # preloading modules + if multiprocessing.get_start_method() != "forkserver": + self.skipTest("forkserver specific test") + + # Create a test module in the temporary directory on the child's path + # TODO: This can all be simplified once gh-126631 is fixed and we can + # use __main__ instead of a module. + dirname = os.path.join(self._temp_dir, 'preloaded_module') + init_name = os.path.join(dirname, '__init__.py') + os.mkdir(dirname) + with open(init_name, "w") as f: + cmd = '''if 1: + import sys + print('stderr', end='', file=sys.stderr) + print('stdout', end='', file=sys.stdout) + ''' + f.write(cmd) + + name = os.path.join(os.path.dirname(__file__), 'mp_preload_flush.py') + env = {'PYTHONPATH': self._temp_dir} + _, out, err = test.support.script_helper.assert_python_ok(name, **env) + + # Check stderr first, as it is more likely to be useful to see in the + # event of a failure. + self.assertEqual(err.decode().rstrip(), 'stderr') + self.assertEqual(out.decode().rstrip(), 'stdout') + + class MiscTestCase(unittest.TestCase): def test__all__(self): # Just make sure names in not_exported are excluded @@ -6061,6 +6505,58 @@ def test_spawn_sys_executable_none_allows_import(self): self.assertEqual(rc, 0) self.assertFalse(err, msg=err.decode('utf-8')) + def test_large_pool(self): + # + # gh-89240: Check that large pools are always okay + # + testfn = os_helper.TESTFN + self.addCleanup(os_helper.unlink, testfn) + with open(testfn, 'w', encoding='utf-8') as f: + f.write(textwrap.dedent('''\ + import multiprocessing + def f(x): return x*x + if __name__ == '__main__': + with multiprocessing.Pool(200) as p: + print(sum(p.map(f, range(1000)))) + ''')) + rc, out, err = script_helper.assert_python_ok(testfn) + self.assertEqual("332833500", out.decode('utf-8').strip()) + self.assertFalse(err, msg=err.decode('utf-8')) + + def test_forked_thread_not_started(self): + # gh-134381: Ensure that a thread that has not been started yet in + # the parent process can be started within a forked child process. + + if multiprocessing.get_start_method() != "fork": + self.skipTest("fork specific test") + + q = multiprocessing.Queue() + t = threading.Thread(target=lambda: q.put("done"), daemon=True) + + def child(): + t.start() + t.join() + + p = multiprocessing.Process(target=child) + p.start() + p.join(support.SHORT_TIMEOUT) + + self.assertEqual(p.exitcode, 0) + self.assertEqual(q.get_nowait(), "done") + close_queue(q) + + def test_preload_main(self): + # gh-126631: Check that __main__ can be pre-loaded + if multiprocessing.get_start_method() != "forkserver": + self.skipTest("forkserver specific test") + + name = os.path.join(os.path.dirname(__file__), 'mp_preload_main.py') + _, out, err = test.support.script_helper.assert_python_ok(name) + self.assertEqual(err, b'') + + # The trailing empty string comes from split() on output ending with \n + out = out.decode().split("\n") + self.assertEqual(out, ['__main__', '__mp_main__', 'f', 'f', '']) # # Mixins @@ -6213,6 +6709,8 @@ def install_tests_in_module_dict(remote_globs, start_method, if base is BaseTestCase: continue assert set(base.ALLOWED_TYPES) <= ALL_TYPES, base.ALLOWED_TYPES + if base.START_METHODS and start_method not in base.START_METHODS: + continue # class not intended for this start method. for type_ in base.ALLOWED_TYPES: if only_type and type_ != only_type: continue @@ -6226,6 +6724,7 @@ class Temp(base, Mixin, unittest.TestCase): Temp = hashlib_helper.requires_hashdigest('sha256')(Temp) Temp.__name__ = Temp.__qualname__ = newname Temp.__module__ = __module__ + Temp.start_method = start_method remote_globs[newname] = Temp elif issubclass(base, unittest.TestCase): if only_type: diff --git a/Lib/test/_test_venv_multiprocessing.py b/Lib/test/_test_venv_multiprocessing.py new file mode 100644 index 0000000000..ad985dd8d5 --- /dev/null +++ b/Lib/test/_test_venv_multiprocessing.py @@ -0,0 +1,40 @@ +import multiprocessing +import random +import sys + +def fill_queue(queue, code): + queue.put(code) + + +def drain_queue(queue, code): + if code != queue.get(): + sys.exit(1) + + +def test_func(): + code = random.randrange(0, 1000) + queue = multiprocessing.Queue() + fill_pool = multiprocessing.Process( + target=fill_queue, + args=(queue, code) + ) + drain_pool = multiprocessing.Process( + target=drain_queue, + args=(queue, code) + ) + drain_pool.start() + fill_pool.start() + fill_pool.join() + drain_pool.join() + + +def main(): + multiprocessing.set_start_method('spawn') + test_pool = multiprocessing.Process(target=test_func) + test_pool.start() + test_pool.join() + sys.exit(test_pool.exitcode) + + +if __name__ == "__main__": + main() diff --git a/Lib/test/archivetestdata/README.md b/Lib/test/archivetestdata/README.md new file mode 100644 index 0000000000..7b555fa327 --- /dev/null +++ b/Lib/test/archivetestdata/README.md @@ -0,0 +1,36 @@ +# Test data for `test_zipfile`, `test_tarfile` (and even some others) + +## `test_zipfile` + +The test executables in this directory are created manually from `header.sh` and +the `testdata_module_inside_zip.py` file. You must have Info-ZIP's zip utility +installed (`apt install zip` on Debian). + +### Purpose of `exe_with_zip` and `exe_with_z64` + +These are used to test executable files with an appended zipfile, in a scenario +where the executable is _not_ a Python interpreter itself so our automatic +zipimport machinery (that'd look for `__main__.py`) is not being used. + +### Updating the test executables + +If you update header.sh or the testdata_module_inside_zip.py file, rerun the +commands below. These are expected to be rarely changed, if ever. + +#### Standard old format (2.0) zip file + +``` +zip -0 zip2.zip testdata_module_inside_zip.py +cat header.sh zip2.zip >exe_with_zip +rm zip2.zip +``` + +#### Modern format (4.5) zip64 file + +Redirecting from stdin forces Info-ZIP's zip tool to create a zip64. + +``` +zip -0 zip64.zip +cat header.sh zip64.zip >exe_with_z64 +rm zip64.zip +``` diff --git a/Lib/test/archivetestdata/exe_with_z64 b/Lib/test/archivetestdata/exe_with_z64 new file mode 100755 index 0000000000..82b03cf39d Binary files /dev/null and b/Lib/test/archivetestdata/exe_with_z64 differ diff --git a/Lib/test/archivetestdata/exe_with_zip b/Lib/test/archivetestdata/exe_with_zip new file mode 100755 index 0000000000..c833cdf9f9 Binary files /dev/null and b/Lib/test/archivetestdata/exe_with_zip differ diff --git a/Lib/test/archivetestdata/header.sh b/Lib/test/archivetestdata/header.sh new file mode 100755 index 0000000000..52dc91acf7 --- /dev/null +++ b/Lib/test/archivetestdata/header.sh @@ -0,0 +1,24 @@ +#!/bin/bash +INTERPRETER_UNDER_TEST="$1" +if [[ ! -x "${INTERPRETER_UNDER_TEST}" ]]; then + echo "Interpreter must be the command line argument." + exit 4 +fi +EXECUTABLE="$0" exec "${INTERPRETER_UNDER_TEST}" -E - <, email:user@example.org, DNS:www.example.org, DirName:/C=XY/L=Castle Anthrax/O=Python Software Foundation/CN=dirname example, URI:https://www.python.org/, IP Address:127.0.0.1, IP Address:0:0:0:0:0:0:0:1, Registered ID:1.2.3.4.5 + X509v3 Key Usage: critical + Digital Signature, Key Encipherment + X509v3 Extended Key Usage: + TLS Web Server Authentication, TLS Web Client Authentication + X509v3 Basic Constraints: critical + CA:FALSE + X509v3 Subject Key Identifier: + 31:5E:C0:5E:2F:47:FF:8B:92:F9:EE:3D:B1:87:D0:53:75:3B:B1:48 + X509v3 Authority Key Identifier: + keyid:F3:EC:94:8E:F2:8E:30:C4:8E:68:C2:BF:8E:6A:19:C0:C1:9F:76:65 + DirName:/C=XY/O=Python Software Foundation CA/CN=our-ca-server + serial:CB:2D:80:99:5A:69:52:5B + Authority Information Access: + CA Issuers - URI:http://testca.pythontest.net/testca/pycacert.cer + OCSP - URI:http://testca.pythontest.net/testca/ocsp/ + X509v3 CRL Distribution Points: + Full Name: + URI:http://testca.pythontest.net/testca/revocation.crl + Signature Algorithm: sha256WithRSAEncryption + Signature Value: + 72:42:a6:fc:ee:3c:21:47:05:33:e8:8c:6b:27:07:4a:ed:e2: + 81:47:96:79:43:ff:0f:ef:5a:06:aa:4c:01:70:5b:21:c4:b7: + 5d:17:29:c8:10:02:c3:08:7b:8c:86:56:9e:e9:7c:6e:a8:b6: + 26:13:9e:1e:1f:93:66:85:67:63:9e:08:fb:55:39:56:82:f5: + be:0c:38:1e:eb:c4:54:b2:a7:7b:18:55:bb:00:87:43:50:50: + bb:e1:29:10:cf:3d:c9:07:c7:d2:5d:b6:45:68:1f:d6:de:00: + 96:3e:29:73:f6:22:70:21:a2:ba:68:28:94:ec:37:bc:a7:00: + 70:58:4e:d1:48:ae:ef:8d:11:a4:6e:10:2f:92:83:07:e2:76: + ac:bf:4f:bb:d6:9f:47:9e:a4:02:03:16:f8:a8:0a:3d:67:17: + 31:44:0e:68:d0:d3:24:d5:e7:bf:67:30:8f:88:97:92:0a:1e: + d7:74:df:7e:7b:4c:c6:d9:c3:84:92:2b:a0:89:11:08:4c:dd: + 32:49:df:36:23:d4:63:56:e4:f1:68:5a:6f:a0:c3:3c:e2:36: + ee:f3:46:60:78:4d:76:a5:5a:4a:61:c6:f8:ae:18:68:c2:8d: + 0e:2f:76:50:bb:be:b9:56:f1:04:5c:ac:ad:d7:d6:a4:1e:45: + 45:52:f4:10:a2:0f:9b:e3:d9:73:17:b6:52:42:a6:5b:c9:e9: + 8d:60:74:68:d0:1f:7a:ce:01:8e:9e:55:cb:cf:64:c1:cc:9a: + 72:aa:b4:5f:b5:55:13:41:10:51:a0:2c:a5:5b:43:12:ca:cc: + b7:c4:ac:f2:6f:72:fd:0d:50:6a:d6:81:c1:91:93:21:fe:de: + 9a:be:e5:3c:2a:98:95:a1:42:f8:f2:5c:75:c6:f1:fd:11:b1: + 22:26:33:5b:43:63:21:06:61:d2:cd:04:f3:30:c6:a8:3f:17: + d3:05:a3:87:45:2e:52:1e:51:88:e3:59:4c:78:51:b0:7b:b4: + 58:d9:27:22:6e:8c +-----BEGIN CERTIFICATE----- +MIIHDTCCBXWgAwIBAgIJAMstgJlaaVJfMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV +BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW +MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xODA4MjkxNDIzMTZaFw0zNzEwMjgx +NDIzMTZaMF0xCzAJBgNVBAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEj +MCEGA1UECgwaUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24xEDAOBgNVBAMMB2Fs +bHNhbnMwggGiMA0GCSqGSIb3DQEBAQUAA4IBjwAwggGKAoIBgQCczEVv5D2UDtn6 +DMmZ/uCWCLyL+K5xTZp/5j3cyISoaTuU1Ku3kD97eLgpHj4Fgk5ZJi21zsQqepCj +jAhBk6tj6RYUcnMbb8MuxUkQMEDW+5LfSyp+HCaetlHosWdhEDqX4kpJ5ajBwNRt +07mxQExtC4kcno0ut9rG5XzLN29XpCpRHlFFrntOgQAEoiz9/fc8qaTgb37RgGYP +Qsxh7PcRDRe4ZGx1l06Irr8Y+2W50zWCfkwCS3DaLDOKIjSOfPHNqmfcfsTpzrj8 +330cdPklrMIuiBv+iGklCjkPZJiEhxvY2k6ERM4HAxxuPCivrH5MCeMNYvBVUcLr +GROm7JRXRllI/XubwwoAaAb+y+dZtCZ9AnzHIb+nyKiJxWAjzjR+QPL6jHrVWBVA +WTc83YP5FvxUXMfY3sVv9tNSCV3cpYOW5+iXcQzLuczXnOLRYk7p9wkb0/hk9KuK +4BMA90eBhvFMCFgHJ1/xJg2nFmBHPo/xbcwPG/ma5T/McA8mAlECAwEAAaOCAt4w +ggLaMIIBMAYDVR0RBIIBJzCCASOCB2FsbHNhbnOgHgYDKgMEoBcMFXNvbWUgb3Ro +ZXIgaWRlbnRpZmllcqA1BgYrBgEFAgKgKzApoBAbDktFUkJFUk9TLlJFQUxNoRUw +E6ADAgEBoQwwChsIdXNlcm5hbWWBEHVzZXJAZXhhbXBsZS5vcmeCD3d3dy5leGFt +cGxlLm9yZ6RnMGUxCzAJBgNVBAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJh +eDEjMCEGA1UECgwaUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24xGDAWBgNVBAMM +D2Rpcm5hbWUgZXhhbXBsZYYXaHR0cHM6Ly93d3cucHl0aG9uLm9yZy+HBH8AAAGH +EAAAAAAAAAAAAAAAAAAAAAGIBCoDBAUwDgYDVR0PAQH/BAQDAgWgMB0GA1UdJQQW +MBQGCCsGAQUFBwMBBggrBgEFBQcDAjAMBgNVHRMBAf8EAjAAMB0GA1UdDgQWBBQx +XsBeL0f/i5L57j2xh9BTdTuxSDB9BgNVHSMEdjB0gBTz7JSO8o4wxI5owr+OahnA +wZ92ZaFRpE8wTTELMAkGA1UEBhMCWFkxJjAkBgNVBAoMHVB5dGhvbiBTb2Z0d2Fy +ZSBGb3VuZGF0aW9uIENBMRYwFAYDVQQDDA1vdXItY2Etc2VydmVyggkAyy2AmVpp +UlswgYMGCCsGAQUFBwEBBHcwdTA8BggrBgEFBQcwAoYwaHR0cDovL3Rlc3RjYS5w +eXRob250ZXN0Lm5ldC90ZXN0Y2EvcHljYWNlcnQuY2VyMDUGCCsGAQUFBzABhilo +dHRwOi8vdGVzdGNhLnB5dGhvbnRlc3QubmV0L3Rlc3RjYS9vY3NwLzBDBgNVHR8E +PDA6MDigNqA0hjJodHRwOi8vdGVzdGNhLnB5dGhvbnRlc3QubmV0L3Rlc3RjYS9y +ZXZvY2F0aW9uLmNybDANBgkqhkiG9w0BAQsFAAOCAYEAckKm/O48IUcFM+iMaycH +Su3igUeWeUP/D+9aBqpMAXBbIcS3XRcpyBACwwh7jIZWnul8bqi2JhOeHh+TZoVn +Y54I+1U5VoL1vgw4HuvEVLKnexhVuwCHQ1BQu+EpEM89yQfH0l22RWgf1t4Alj4p +c/YicCGiumgolOw3vKcAcFhO0Uiu740RpG4QL5KDB+J2rL9Pu9afR56kAgMW+KgK +PWcXMUQOaNDTJNXnv2cwj4iXkgoe13TffntMxtnDhJIroIkRCEzdMknfNiPUY1bk +8Whab6DDPOI27vNGYHhNdqVaSmHG+K4YaMKNDi92ULu+uVbxBFysrdfWpB5FRVL0 +EKIPm+PZcxe2UkKmW8npjWB0aNAfes4Bjp5Vy89kwcyacqq0X7VVE0EQUaAspVtD +EsrMt8Ss8m9y/Q1QataBwZGTIf7emr7lPCqYlaFC+PJcdcbx/RGxIiYzW0NjIQZh +0s0E8zDGqD8X0wWjh0UuUh5RiONZTHhRsHu0WNknIm6M +-----END CERTIFICATE----- diff --git a/Lib/test/certdata/badcert.pem b/Lib/test/certdata/badcert.pem new file mode 100644 index 0000000000..c4191460f9 --- /dev/null +++ b/Lib/test/certdata/badcert.pem @@ -0,0 +1,36 @@ +-----BEGIN RSA PRIVATE KEY----- +MIICXwIBAAKBgQC8ddrhm+LutBvjYcQlnH21PPIseJ1JVG2HMmN2CmZk2YukO+9L +opdJhTvbGfEj0DQs1IE8M+kTUyOmuKfVrFMKwtVeCJphrAnhoz7TYOuLBSqt7lVH +fhi/VwovESJlaBOp+WMnfhcduPEYHYx/6cnVapIkZnLt30zu2um+DzA9jQIDAQAB +AoGBAK0FZpaKj6WnJZN0RqhhK+ggtBWwBnc0U/ozgKz2j1s3fsShYeiGtW6CK5nU +D1dZ5wzhbGThI7LiOXDvRucc9n7vUgi0alqPQ/PFodPxAN/eEYkmXQ7W2k7zwsDA +IUK0KUhktQbLu8qF/m8qM86ba9y9/9YkXuQbZ3COl5ahTZrhAkEA301P08RKv3KM +oXnGU2UHTuJ1MAD2hOrPxjD4/wxA/39EWG9bZczbJyggB4RHu0I3NOSFjAm3HQm0 +ANOu5QK9owJBANgOeLfNNcF4pp+UikRFqxk5hULqRAWzVxVrWe85FlPm0VVmHbb/ +loif7mqjU8o1jTd/LM7RD9f2usZyE2psaw8CQQCNLhkpX3KO5kKJmS9N7JMZSc4j +oog58yeYO8BBqKKzpug0LXuQultYv2K4veaIO04iL9VLe5z9S/Q1jaCHBBuXAkEA +z8gjGoi1AOp6PBBLZNsncCvcV/0aC+1se4HxTNo2+duKSDnbq+ljqOM+E7odU+Nq +ewvIWOG//e8fssd0mq3HywJBAJ8l/c8GVmrpFTx8r/nZ2Pyyjt3dH1widooDXYSV +q6Gbf41Llo5sYAtmxdndTLASuHKecacTgZVhy0FryZpLKrU= +-----END RSA PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +Just bad cert data +-----END CERTIFICATE----- +-----BEGIN RSA PRIVATE KEY----- +MIICXwIBAAKBgQC8ddrhm+LutBvjYcQlnH21PPIseJ1JVG2HMmN2CmZk2YukO+9L +opdJhTvbGfEj0DQs1IE8M+kTUyOmuKfVrFMKwtVeCJphrAnhoz7TYOuLBSqt7lVH +fhi/VwovESJlaBOp+WMnfhcduPEYHYx/6cnVapIkZnLt30zu2um+DzA9jQIDAQAB +AoGBAK0FZpaKj6WnJZN0RqhhK+ggtBWwBnc0U/ozgKz2j1s3fsShYeiGtW6CK5nU +D1dZ5wzhbGThI7LiOXDvRucc9n7vUgi0alqPQ/PFodPxAN/eEYkmXQ7W2k7zwsDA +IUK0KUhktQbLu8qF/m8qM86ba9y9/9YkXuQbZ3COl5ahTZrhAkEA301P08RKv3KM +oXnGU2UHTuJ1MAD2hOrPxjD4/wxA/39EWG9bZczbJyggB4RHu0I3NOSFjAm3HQm0 +ANOu5QK9owJBANgOeLfNNcF4pp+UikRFqxk5hULqRAWzVxVrWe85FlPm0VVmHbb/ +loif7mqjU8o1jTd/LM7RD9f2usZyE2psaw8CQQCNLhkpX3KO5kKJmS9N7JMZSc4j +oog58yeYO8BBqKKzpug0LXuQultYv2K4veaIO04iL9VLe5z9S/Q1jaCHBBuXAkEA +z8gjGoi1AOp6PBBLZNsncCvcV/0aC+1se4HxTNo2+duKSDnbq+ljqOM+E7odU+Nq +ewvIWOG//e8fssd0mq3HywJBAJ8l/c8GVmrpFTx8r/nZ2Pyyjt3dH1widooDXYSV +q6Gbf41Llo5sYAtmxdndTLASuHKecacTgZVhy0FryZpLKrU= +-----END RSA PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +Just bad cert data +-----END CERTIFICATE----- diff --git a/Lib/test/certdata/badkey.pem b/Lib/test/certdata/badkey.pem new file mode 100644 index 0000000000..1c8a955719 --- /dev/null +++ b/Lib/test/certdata/badkey.pem @@ -0,0 +1,40 @@ +-----BEGIN RSA PRIVATE KEY----- +Bad Key, though the cert should be OK +-----END RSA PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIICpzCCAhCgAwIBAgIJAP+qStv1cIGNMA0GCSqGSIb3DQEBBQUAMIGJMQswCQYD +VQQGEwJVUzERMA8GA1UECBMIRGVsYXdhcmUxEzARBgNVBAcTCldpbG1pbmd0b24x +IzAhBgNVBAoTGlB5dGhvbiBTb2Z0d2FyZSBGb3VuZGF0aW9uMQwwCgYDVQQLEwNT +U0wxHzAdBgNVBAMTFnNvbWVtYWNoaW5lLnB5dGhvbi5vcmcwHhcNMDcwODI3MTY1 +NDUwWhcNMTMwMjE2MTY1NDUwWjCBiTELMAkGA1UEBhMCVVMxETAPBgNVBAgTCERl +bGF3YXJlMRMwEQYDVQQHEwpXaWxtaW5ndG9uMSMwIQYDVQQKExpQeXRob24gU29m +dHdhcmUgRm91bmRhdGlvbjEMMAoGA1UECxMDU1NMMR8wHQYDVQQDExZzb21lbWFj +aGluZS5weXRob24ub3JnMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQC8ddrh +m+LutBvjYcQlnH21PPIseJ1JVG2HMmN2CmZk2YukO+9LopdJhTvbGfEj0DQs1IE8 +M+kTUyOmuKfVrFMKwtVeCJphrAnhoz7TYOuLBSqt7lVHfhi/VwovESJlaBOp+WMn +fhcduPEYHYx/6cnVapIkZnLt30zu2um+DzA9jQIDAQABoxUwEzARBglghkgBhvhC +AQEEBAMCBkAwDQYJKoZIhvcNAQEFBQADgYEAF4Q5BVqmCOLv1n8je/Jw9K669VXb +08hyGzQhkemEBYQd6fzQ9A/1ZzHkJKb1P6yreOLSEh4KcxYPyrLRC1ll8nr5OlCx +CMhKkTnR6qBsdNV0XtdU2+N25hqW+Ma4ZeqsN/iiJVCGNOZGnvQuvCAGWF8+J/f/ +iHkC6gGdBJhogs4= +-----END CERTIFICATE----- +-----BEGIN RSA PRIVATE KEY----- +Bad Key, though the cert should be OK +-----END RSA PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIICpzCCAhCgAwIBAgIJAP+qStv1cIGNMA0GCSqGSIb3DQEBBQUAMIGJMQswCQYD +VQQGEwJVUzERMA8GA1UECBMIRGVsYXdhcmUxEzARBgNVBAcTCldpbG1pbmd0b24x +IzAhBgNVBAoTGlB5dGhvbiBTb2Z0d2FyZSBGb3VuZGF0aW9uMQwwCgYDVQQLEwNT +U0wxHzAdBgNVBAMTFnNvbWVtYWNoaW5lLnB5dGhvbi5vcmcwHhcNMDcwODI3MTY1 +NDUwWhcNMTMwMjE2MTY1NDUwWjCBiTELMAkGA1UEBhMCVVMxETAPBgNVBAgTCERl +bGF3YXJlMRMwEQYDVQQHEwpXaWxtaW5ndG9uMSMwIQYDVQQKExpQeXRob24gU29m +dHdhcmUgRm91bmRhdGlvbjEMMAoGA1UECxMDU1NMMR8wHQYDVQQDExZzb21lbWFj +aGluZS5weXRob24ub3JnMIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQC8ddrh +m+LutBvjYcQlnH21PPIseJ1JVG2HMmN2CmZk2YukO+9LopdJhTvbGfEj0DQs1IE8 +M+kTUyOmuKfVrFMKwtVeCJphrAnhoz7TYOuLBSqt7lVHfhi/VwovESJlaBOp+WMn +fhcduPEYHYx/6cnVapIkZnLt30zu2um+DzA9jQIDAQABoxUwEzARBglghkgBhvhC +AQEEBAMCBkAwDQYJKoZIhvcNAQEFBQADgYEAF4Q5BVqmCOLv1n8je/Jw9K669VXb +08hyGzQhkemEBYQd6fzQ9A/1ZzHkJKb1P6yreOLSEh4KcxYPyrLRC1ll8nr5OlCx +CMhKkTnR6qBsdNV0XtdU2+N25hqW+Ma4ZeqsN/iiJVCGNOZGnvQuvCAGWF8+J/f/ +iHkC6gGdBJhogs4= +-----END CERTIFICATE----- diff --git a/Lib/test/certdata/capath/4e1295a3.0 b/Lib/test/certdata/capath/4e1295a3.0 new file mode 100644 index 0000000000..9d7ac238d8 --- /dev/null +++ b/Lib/test/certdata/capath/4e1295a3.0 @@ -0,0 +1,14 @@ +-----BEGIN CERTIFICATE----- +MIICLDCCAdYCAQAwDQYJKoZIhvcNAQEEBQAwgaAxCzAJBgNVBAYTAlBUMRMwEQYD +VQQIEwpRdWVlbnNsYW5kMQ8wDQYDVQQHEwZMaXNib2ExFzAVBgNVBAoTDk5ldXJv +bmlvLCBMZGEuMRgwFgYDVQQLEw9EZXNlbnZvbHZpbWVudG8xGzAZBgNVBAMTEmJy +dXR1cy5uZXVyb25pby5wdDEbMBkGCSqGSIb3DQEJARYMc2FtcG9AaWtpLmZpMB4X +DTk2MDkwNTAzNDI0M1oXDTk2MTAwNTAzNDI0M1owgaAxCzAJBgNVBAYTAlBUMRMw +EQYDVQQIEwpRdWVlbnNsYW5kMQ8wDQYDVQQHEwZMaXNib2ExFzAVBgNVBAoTDk5l +dXJvbmlvLCBMZGEuMRgwFgYDVQQLEw9EZXNlbnZvbHZpbWVudG8xGzAZBgNVBAMT +EmJydXR1cy5uZXVyb25pby5wdDEbMBkGCSqGSIb3DQEJARYMc2FtcG9AaWtpLmZp +MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBAL7+aty3S1iBA/+yxjxv4q1MUTd1kjNw +L4lYKbpzzlmC5beaQXeQ2RmGMTXU+mDvuqItjVHOK3DvPK7lTcSGftUCAwEAATAN +BgkqhkiG9w0BAQQFAANBAFqPEKFjk6T6CKTHvaQeEAsX0/8YHPHqH/9AnhSjrwuX +9EBc0n6bVGhN7XaXd6sJ7dym9sbsWxb+pJdurnkxjx4= +-----END CERTIFICATE----- diff --git a/Lib/test/certdata/capath/5ed36f99.0 b/Lib/test/certdata/capath/5ed36f99.0 new file mode 100644 index 0000000000..e7dfc82947 --- /dev/null +++ b/Lib/test/certdata/capath/5ed36f99.0 @@ -0,0 +1,41 @@ +-----BEGIN CERTIFICATE----- +MIIHPTCCBSWgAwIBAgIBADANBgkqhkiG9w0BAQQFADB5MRAwDgYDVQQKEwdSb290 +IENBMR4wHAYDVQQLExVodHRwOi8vd3d3LmNhY2VydC5vcmcxIjAgBgNVBAMTGUNB +IENlcnQgU2lnbmluZyBBdXRob3JpdHkxITAfBgkqhkiG9w0BCQEWEnN1cHBvcnRA +Y2FjZXJ0Lm9yZzAeFw0wMzAzMzAxMjI5NDlaFw0zMzAzMjkxMjI5NDlaMHkxEDAO +BgNVBAoTB1Jvb3QgQ0ExHjAcBgNVBAsTFWh0dHA6Ly93d3cuY2FjZXJ0Lm9yZzEi +MCAGA1UEAxMZQ0EgQ2VydCBTaWduaW5nIEF1dGhvcml0eTEhMB8GCSqGSIb3DQEJ +ARYSc3VwcG9ydEBjYWNlcnQub3JnMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIIC +CgKCAgEAziLA4kZ97DYoB1CW8qAzQIxL8TtmPzHlawI229Z89vGIj053NgVBlfkJ +8BLPRoZzYLdufujAWGSuzbCtRRcMY/pnCujW0r8+55jE8Ez64AO7NV1sId6eINm6 +zWYyN3L69wj1x81YyY7nDl7qPv4coRQKFWyGhFtkZip6qUtTefWIonvuLwphK42y +fk1WpRPs6tqSnqxEQR5YYGUFZvjARL3LlPdCfgv3ZWiYUQXw8wWRBB0bF4LsyFe7 +w2t6iPGwcswlWyCR7BYCEo8y6RcYSNDHBS4CMEK4JZwFaz+qOqfrU0j36NK2B5jc +G8Y0f3/JHIJ6BVgrCFvzOKKrF11myZjXnhCLotLddJr3cQxyYN/Nb5gznZY0dj4k +epKwDpUeb+agRThHqtdB7Uq3EvbXG4OKDy7YCbZZ16oE/9KTfWgu3YtLq1i6L43q +laegw1SJpfvbi1EinbLDvhG+LJGGi5Z4rSDTii8aP8bQUWWHIbEZAWV/RRyH9XzQ +QUxPKZgh/TMfdQwEUfoZd9vUFBzugcMd9Zi3aQaRIt0AUMyBMawSB3s42mhb5ivU +fslfrejrckzzAeVLIL+aplfKkQABi6F1ITe1Yw1nPkZPcCBnzsXWWdsC4PDSy826 +YreQQejdIOQpvGQpQsgi3Hia/0PsmBsJUUtaWsJx8cTLc6nloQsCAwEAAaOCAc4w +ggHKMB0GA1UdDgQWBBQWtTIb1Mfz4OaO873SsDrusjkY0TCBowYDVR0jBIGbMIGY +gBQWtTIb1Mfz4OaO873SsDrusjkY0aF9pHsweTEQMA4GA1UEChMHUm9vdCBDQTEe +MBwGA1UECxMVaHR0cDovL3d3dy5jYWNlcnQub3JnMSIwIAYDVQQDExlDQSBDZXJ0 +IFNpZ25pbmcgQXV0aG9yaXR5MSEwHwYJKoZIhvcNAQkBFhJzdXBwb3J0QGNhY2Vy +dC5vcmeCAQAwDwYDVR0TAQH/BAUwAwEB/zAyBgNVHR8EKzApMCegJaAjhiFodHRw +czovL3d3dy5jYWNlcnQub3JnL3Jldm9rZS5jcmwwMAYJYIZIAYb4QgEEBCMWIWh0 +dHBzOi8vd3d3LmNhY2VydC5vcmcvcmV2b2tlLmNybDA0BglghkgBhvhCAQgEJxYl +aHR0cDovL3d3dy5jYWNlcnQub3JnL2luZGV4LnBocD9pZD0xMDBWBglghkgBhvhC +AQ0ESRZHVG8gZ2V0IHlvdXIgb3duIGNlcnRpZmljYXRlIGZvciBGUkVFIGhlYWQg +b3ZlciB0byBodHRwOi8vd3d3LmNhY2VydC5vcmcwDQYJKoZIhvcNAQEEBQADggIB +ACjH7pyCArpcgBLKNQodgW+JapnM8mgPf6fhjViVPr3yBsOQWqy1YPaZQwGjiHCc +nWKdpIevZ1gNMDY75q1I08t0AoZxPuIrA2jxNGJARjtT6ij0rPtmlVOKTV39O9lg +18p5aTuxZZKmxoGCXJzN600BiqXfEVWqFcofN8CCmHBh22p8lqOOLlQ+TyGpkO/c +gr/c6EWtTZBzCDyUZbAEmXZ/4rzCahWqlwQ3JNgelE5tDlG+1sSPypZt90Pf6DBl +Jzt7u0NDY8RD97LsaMzhGY4i+5jhe1o+ATc7iwiwovOVThrLm82asduycPAtStvY +sONvRUgzEv/+PDIqVPfE94rwiCPCR/5kenHA0R6mY7AHfqQv0wGP3J8rtsYIqQ+T +SCX8Ev2fQtzzxD72V7DX3WnRBnc0CkvSyqD/HMaMyRa+xMwyN2hzXwj7UfdJUzYF +CpUCTPJ5GhD22Dp1nPMd8aINcGeGG7MW9S/lpOt5hvk9C8JzC6WZrG/8Z7jlLwum +GCSNe9FINSkYQKyTYOGWhlC0elnYjyELn8+CkcY7v2vcB5G5l1YjqrZslMZIBjzk +zk6q5PYvCdxTby78dOs6Y5nCpqyJvKeyRKANihDjbPIky/qbn3BHLt4Ui9SyIAmW +omTxJBzcoTWcFbLUvFUufQb1nA5V9FrWk9p2rSVzTMVD +-----END CERTIFICATE----- diff --git a/Lib/test/certdata/capath/6e88d7b8.0 b/Lib/test/certdata/capath/6e88d7b8.0 new file mode 100644 index 0000000000..9d7ac238d8 --- /dev/null +++ b/Lib/test/certdata/capath/6e88d7b8.0 @@ -0,0 +1,14 @@ +-----BEGIN CERTIFICATE----- +MIICLDCCAdYCAQAwDQYJKoZIhvcNAQEEBQAwgaAxCzAJBgNVBAYTAlBUMRMwEQYD +VQQIEwpRdWVlbnNsYW5kMQ8wDQYDVQQHEwZMaXNib2ExFzAVBgNVBAoTDk5ldXJv +bmlvLCBMZGEuMRgwFgYDVQQLEw9EZXNlbnZvbHZpbWVudG8xGzAZBgNVBAMTEmJy +dXR1cy5uZXVyb25pby5wdDEbMBkGCSqGSIb3DQEJARYMc2FtcG9AaWtpLmZpMB4X +DTk2MDkwNTAzNDI0M1oXDTk2MTAwNTAzNDI0M1owgaAxCzAJBgNVBAYTAlBUMRMw +EQYDVQQIEwpRdWVlbnNsYW5kMQ8wDQYDVQQHEwZMaXNib2ExFzAVBgNVBAoTDk5l +dXJvbmlvLCBMZGEuMRgwFgYDVQQLEw9EZXNlbnZvbHZpbWVudG8xGzAZBgNVBAMT +EmJydXR1cy5uZXVyb25pby5wdDEbMBkGCSqGSIb3DQEJARYMc2FtcG9AaWtpLmZp +MFwwDQYJKoZIhvcNAQEBBQADSwAwSAJBAL7+aty3S1iBA/+yxjxv4q1MUTd1kjNw +L4lYKbpzzlmC5beaQXeQ2RmGMTXU+mDvuqItjVHOK3DvPK7lTcSGftUCAwEAATAN +BgkqhkiG9w0BAQQFAANBAFqPEKFjk6T6CKTHvaQeEAsX0/8YHPHqH/9AnhSjrwuX +9EBc0n6bVGhN7XaXd6sJ7dym9sbsWxb+pJdurnkxjx4= +-----END CERTIFICATE----- diff --git a/Lib/test/certdata/capath/99d0fa06.0 b/Lib/test/certdata/capath/99d0fa06.0 new file mode 100644 index 0000000000..e7dfc82947 --- /dev/null +++ b/Lib/test/certdata/capath/99d0fa06.0 @@ -0,0 +1,41 @@ +-----BEGIN CERTIFICATE----- +MIIHPTCCBSWgAwIBAgIBADANBgkqhkiG9w0BAQQFADB5MRAwDgYDVQQKEwdSb290 +IENBMR4wHAYDVQQLExVodHRwOi8vd3d3LmNhY2VydC5vcmcxIjAgBgNVBAMTGUNB +IENlcnQgU2lnbmluZyBBdXRob3JpdHkxITAfBgkqhkiG9w0BCQEWEnN1cHBvcnRA +Y2FjZXJ0Lm9yZzAeFw0wMzAzMzAxMjI5NDlaFw0zMzAzMjkxMjI5NDlaMHkxEDAO +BgNVBAoTB1Jvb3QgQ0ExHjAcBgNVBAsTFWh0dHA6Ly93d3cuY2FjZXJ0Lm9yZzEi +MCAGA1UEAxMZQ0EgQ2VydCBTaWduaW5nIEF1dGhvcml0eTEhMB8GCSqGSIb3DQEJ +ARYSc3VwcG9ydEBjYWNlcnQub3JnMIICIjANBgkqhkiG9w0BAQEFAAOCAg8AMIIC +CgKCAgEAziLA4kZ97DYoB1CW8qAzQIxL8TtmPzHlawI229Z89vGIj053NgVBlfkJ +8BLPRoZzYLdufujAWGSuzbCtRRcMY/pnCujW0r8+55jE8Ez64AO7NV1sId6eINm6 +zWYyN3L69wj1x81YyY7nDl7qPv4coRQKFWyGhFtkZip6qUtTefWIonvuLwphK42y +fk1WpRPs6tqSnqxEQR5YYGUFZvjARL3LlPdCfgv3ZWiYUQXw8wWRBB0bF4LsyFe7 +w2t6iPGwcswlWyCR7BYCEo8y6RcYSNDHBS4CMEK4JZwFaz+qOqfrU0j36NK2B5jc +G8Y0f3/JHIJ6BVgrCFvzOKKrF11myZjXnhCLotLddJr3cQxyYN/Nb5gznZY0dj4k +epKwDpUeb+agRThHqtdB7Uq3EvbXG4OKDy7YCbZZ16oE/9KTfWgu3YtLq1i6L43q +laegw1SJpfvbi1EinbLDvhG+LJGGi5Z4rSDTii8aP8bQUWWHIbEZAWV/RRyH9XzQ +QUxPKZgh/TMfdQwEUfoZd9vUFBzugcMd9Zi3aQaRIt0AUMyBMawSB3s42mhb5ivU +fslfrejrckzzAeVLIL+aplfKkQABi6F1ITe1Yw1nPkZPcCBnzsXWWdsC4PDSy826 +YreQQejdIOQpvGQpQsgi3Hia/0PsmBsJUUtaWsJx8cTLc6nloQsCAwEAAaOCAc4w +ggHKMB0GA1UdDgQWBBQWtTIb1Mfz4OaO873SsDrusjkY0TCBowYDVR0jBIGbMIGY +gBQWtTIb1Mfz4OaO873SsDrusjkY0aF9pHsweTEQMA4GA1UEChMHUm9vdCBDQTEe +MBwGA1UECxMVaHR0cDovL3d3dy5jYWNlcnQub3JnMSIwIAYDVQQDExlDQSBDZXJ0 +IFNpZ25pbmcgQXV0aG9yaXR5MSEwHwYJKoZIhvcNAQkBFhJzdXBwb3J0QGNhY2Vy +dC5vcmeCAQAwDwYDVR0TAQH/BAUwAwEB/zAyBgNVHR8EKzApMCegJaAjhiFodHRw +czovL3d3dy5jYWNlcnQub3JnL3Jldm9rZS5jcmwwMAYJYIZIAYb4QgEEBCMWIWh0 +dHBzOi8vd3d3LmNhY2VydC5vcmcvcmV2b2tlLmNybDA0BglghkgBhvhCAQgEJxYl +aHR0cDovL3d3dy5jYWNlcnQub3JnL2luZGV4LnBocD9pZD0xMDBWBglghkgBhvhC +AQ0ESRZHVG8gZ2V0IHlvdXIgb3duIGNlcnRpZmljYXRlIGZvciBGUkVFIGhlYWQg +b3ZlciB0byBodHRwOi8vd3d3LmNhY2VydC5vcmcwDQYJKoZIhvcNAQEEBQADggIB +ACjH7pyCArpcgBLKNQodgW+JapnM8mgPf6fhjViVPr3yBsOQWqy1YPaZQwGjiHCc +nWKdpIevZ1gNMDY75q1I08t0AoZxPuIrA2jxNGJARjtT6ij0rPtmlVOKTV39O9lg +18p5aTuxZZKmxoGCXJzN600BiqXfEVWqFcofN8CCmHBh22p8lqOOLlQ+TyGpkO/c +gr/c6EWtTZBzCDyUZbAEmXZ/4rzCahWqlwQ3JNgelE5tDlG+1sSPypZt90Pf6DBl +Jzt7u0NDY8RD97LsaMzhGY4i+5jhe1o+ATc7iwiwovOVThrLm82asduycPAtStvY +sONvRUgzEv/+PDIqVPfE94rwiCPCR/5kenHA0R6mY7AHfqQv0wGP3J8rtsYIqQ+T +SCX8Ev2fQtzzxD72V7DX3WnRBnc0CkvSyqD/HMaMyRa+xMwyN2hzXwj7UfdJUzYF +CpUCTPJ5GhD22Dp1nPMd8aINcGeGG7MW9S/lpOt5hvk9C8JzC6WZrG/8Z7jlLwum +GCSNe9FINSkYQKyTYOGWhlC0elnYjyELn8+CkcY7v2vcB5G5l1YjqrZslMZIBjzk +zk6q5PYvCdxTby78dOs6Y5nCpqyJvKeyRKANihDjbPIky/qbn3BHLt4Ui9SyIAmW +omTxJBzcoTWcFbLUvFUufQb1nA5V9FrWk9p2rSVzTMVD +-----END CERTIFICATE----- diff --git a/Lib/test/certdata/capath/b1930218.0 b/Lib/test/certdata/capath/b1930218.0 new file mode 100644 index 0000000000..aa9dbbe294 --- /dev/null +++ b/Lib/test/certdata/capath/b1930218.0 @@ -0,0 +1,27 @@ +-----BEGIN CERTIFICATE----- +MIIEgDCCAuigAwIBAgIJAMstgJlaaVJbMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV +BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW +MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xODA4MjkxNDIzMTZaFw0zNzEwMjgx +NDIzMTZaME0xCzAJBgNVBAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUg +Rm91bmRhdGlvbiBDQTEWMBQGA1UEAwwNb3VyLWNhLXNlcnZlcjCCAaIwDQYJKoZI +hvcNAQEBBQADggGPADCCAYoCggGBANCgm7G5O3nuMS+4URwBde0JWUysyL9qCvh6 +CPAl4yV7avjE2KqgYAclsM9zcQVSaL8Gk64QYZa8s2mBGn0Z/CCGj5poG+3N4mxh +Z8dOVepDBiEb6bm+hF/C2uuJiOBCpkVJKtC5a4yTyUQ7yvw8lH/dcMWt2Es73B74 +VUu1J4b437CDz/cWN78TFzTUyVXtaxbJf60gTvAe2Ru/jbrNypbvHmnLUWZhSA3o +eaNZYdQQjeANOwuFttWFEt2lB8VL+iP6VDn3lwvJREceVnc8PBMBC2131hS6RPRT +NVbZPbk+NV/bM5pPWrk4RMkySf5m9h8al6rKTEr2uF5Af/sLHfhbodz4wC7QbUn1 +0kbUkFf+koE0ri04u6gXDOHlP+L3JgVUUPVksxxuRP9vqbQDlukOwojYclKQmcZB +D0aQWbg+b9Linh02gpXTWIoS8+LYDSBRI/CQLZo+fSaGsqfX+ShgA+N3x4gEyf6J +d3AQT8Ogijv0q0J74xSS2K4W1qHefQIDAQABo2MwYTAdBgNVHQ4EFgQU8+yUjvKO +MMSOaMK/jmoZwMGfdmUwHwYDVR0jBBgwFoAU8+yUjvKOMMSOaMK/jmoZwMGfdmUw +DwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAYYwDQYJKoZIhvcNAQELBQAD +ggGBAIsAVHKzjevzrzSf1mDq3oQ/jASPGaa+AmfEY8V040c3WYOUBvFFGegHL9ZO +S0+oPccHByeS9H5zT4syGZRGeiXE2cQnsBFjOmCLheFzTzQ7a6Q0jEmOzc9PsmUn +QRmw/IAxePJzapt9cTRQ/Hio2gW0nFs6mXprXe870+k7MwESZc9eB9gZr9VT6vAQ +rMS2Jjw0LnTuZN0dNnWJRACwDf0vswHMGosCzWzogILKv4LXAJ3YNhXSBzf8bHMd +2qgc6CCOMnr+bScW5Fhs6z7w/iRSKXG4lntTS0UgVUBehhvsyUaRku6sk2WRLpS2 +tqzoozSJpBoSDU1EpVLti5HuL6avpJUl+c7HW6cA05PKtDxdTfexPMxttEW+gu0Y +kMiG0XVRUARM6E/S1lCqdede/6F7Jxkca0ksbE1rY8w7cwDzmSbQgofTqTactD25 +SGiokvAnjgzNFXZChIDJP6N+tN3X+Kx2umCXPFofTt5x7gk5EN0x1WhXXRrlQroO +aOZF0w== +-----END CERTIFICATE----- diff --git a/Lib/test/certdata/capath/ceff1710.0 b/Lib/test/certdata/capath/ceff1710.0 new file mode 100644 index 0000000000..aa9dbbe294 --- /dev/null +++ b/Lib/test/certdata/capath/ceff1710.0 @@ -0,0 +1,27 @@ +-----BEGIN CERTIFICATE----- +MIIEgDCCAuigAwIBAgIJAMstgJlaaVJbMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV +BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW +MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xODA4MjkxNDIzMTZaFw0zNzEwMjgx +NDIzMTZaME0xCzAJBgNVBAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUg +Rm91bmRhdGlvbiBDQTEWMBQGA1UEAwwNb3VyLWNhLXNlcnZlcjCCAaIwDQYJKoZI +hvcNAQEBBQADggGPADCCAYoCggGBANCgm7G5O3nuMS+4URwBde0JWUysyL9qCvh6 +CPAl4yV7avjE2KqgYAclsM9zcQVSaL8Gk64QYZa8s2mBGn0Z/CCGj5poG+3N4mxh +Z8dOVepDBiEb6bm+hF/C2uuJiOBCpkVJKtC5a4yTyUQ7yvw8lH/dcMWt2Es73B74 +VUu1J4b437CDz/cWN78TFzTUyVXtaxbJf60gTvAe2Ru/jbrNypbvHmnLUWZhSA3o +eaNZYdQQjeANOwuFttWFEt2lB8VL+iP6VDn3lwvJREceVnc8PBMBC2131hS6RPRT +NVbZPbk+NV/bM5pPWrk4RMkySf5m9h8al6rKTEr2uF5Af/sLHfhbodz4wC7QbUn1 +0kbUkFf+koE0ri04u6gXDOHlP+L3JgVUUPVksxxuRP9vqbQDlukOwojYclKQmcZB +D0aQWbg+b9Linh02gpXTWIoS8+LYDSBRI/CQLZo+fSaGsqfX+ShgA+N3x4gEyf6J +d3AQT8Ogijv0q0J74xSS2K4W1qHefQIDAQABo2MwYTAdBgNVHQ4EFgQU8+yUjvKO +MMSOaMK/jmoZwMGfdmUwHwYDVR0jBBgwFoAU8+yUjvKOMMSOaMK/jmoZwMGfdmUw +DwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAYYwDQYJKoZIhvcNAQELBQAD +ggGBAIsAVHKzjevzrzSf1mDq3oQ/jASPGaa+AmfEY8V040c3WYOUBvFFGegHL9ZO +S0+oPccHByeS9H5zT4syGZRGeiXE2cQnsBFjOmCLheFzTzQ7a6Q0jEmOzc9PsmUn +QRmw/IAxePJzapt9cTRQ/Hio2gW0nFs6mXprXe870+k7MwESZc9eB9gZr9VT6vAQ +rMS2Jjw0LnTuZN0dNnWJRACwDf0vswHMGosCzWzogILKv4LXAJ3YNhXSBzf8bHMd +2qgc6CCOMnr+bScW5Fhs6z7w/iRSKXG4lntTS0UgVUBehhvsyUaRku6sk2WRLpS2 +tqzoozSJpBoSDU1EpVLti5HuL6avpJUl+c7HW6cA05PKtDxdTfexPMxttEW+gu0Y +kMiG0XVRUARM6E/S1lCqdede/6F7Jxkca0ksbE1rY8w7cwDzmSbQgofTqTactD25 +SGiokvAnjgzNFXZChIDJP6N+tN3X+Kx2umCXPFofTt5x7gk5EN0x1WhXXRrlQroO +aOZF0w== +-----END CERTIFICATE----- diff --git a/Lib/test/certdata/cert3.pem b/Lib/test/certdata/cert3.pem new file mode 100644 index 0000000000..4ab0f5ff13 --- /dev/null +++ b/Lib/test/certdata/cert3.pem @@ -0,0 +1,34 @@ +-----BEGIN CERTIFICATE----- +MIIF8TCCBFmgAwIBAgIJAMstgJlaaVJcMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV +BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW +MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xODA4MjkxNDIzMTZaFw0zNzEwMjgx +NDIzMTZaMF8xCzAJBgNVBAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEj +MCEGA1UECgwaUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24xEjAQBgNVBAMMCWxv +Y2FsaG9zdDCCAaIwDQYJKoZIhvcNAQEBBQADggGPADCCAYoCggGBAKAqKHEL7aDt +3swl8hQF8VaK4zDGDRaF3E/IZTMwCN7FsQ4ejSiOe3E90f0phHCIpEpv2OebNenY +IpOGoFgkh62r/cthmnhu8Mn+FUIv17iOq7WX7B30OSqEpnr1voLX93XYkAq8LlMh +P79vsSCVhTwow3HZY7krEgl5WlfryOfj1i1TODSFPRCJePh66BsOTUvV/33GC+Qd +pVZVDGLowU1Ycmr/FdRvwT+F39Dehp03UFcxaX0/joPhH5gYpBB1kWTAQmxuqKMW +9ZZs6hrPtMXF/yfSrrXrzTdpct9paKR8RcufOcS8qju/ISK+1P/LXg2b5KJHedLo +TTIO3yCZ4d1odyuZBP7JDrI05gMJx95gz6sG685Qc+52MzLSTwr/Qg+MOjQoBy0o +8fRRVvIMEwoN0ZDb4uFEUuwZceUP1vTk/GGpNQt7ct4ropn6K4Zta3BUtovlLjZa +IIBhc1KETUqjRDvC6ACKmlcJ/5pY/dbH1lOux+IMFsh+djmaV90b3QIDAQABo4IB +wDCCAbwwFAYDVR0RBA0wC4IJbG9jYWxob3N0MA4GA1UdDwEB/wQEAwIFoDAdBgNV +HSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwDAYDVR0TAQH/BAIwADAdBgNVHQ4E +FgQUP7HpT6C+MGY+ChjID0caTzRqD0IwfQYDVR0jBHYwdIAU8+yUjvKOMMSOaMK/ +jmoZwMGfdmWhUaRPME0xCzAJBgNVBAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29m +dHdhcmUgRm91bmRhdGlvbiBDQTEWMBQGA1UEAwwNb3VyLWNhLXNlcnZlcoIJAMst +gJlaaVJbMIGDBggrBgEFBQcBAQR3MHUwPAYIKwYBBQUHMAKGMGh0dHA6Ly90ZXN0 +Y2EucHl0aG9udGVzdC5uZXQvdGVzdGNhL3B5Y2FjZXJ0LmNlcjA1BggrBgEFBQcw +AYYpaHR0cDovL3Rlc3RjYS5weXRob250ZXN0Lm5ldC90ZXN0Y2Evb2NzcC8wQwYD +VR0fBDwwOjA4oDagNIYyaHR0cDovL3Rlc3RjYS5weXRob250ZXN0Lm5ldC90ZXN0 +Y2EvcmV2b2NhdGlvbi5jcmwwDQYJKoZIhvcNAQELBQADggGBAMo0usXQzycxMtYN +JzC42xfftzmnu7E7hsQx/fur22MazJCruU6rNEkMXow+cKOnay+nmiV7AVoYlkh2 ++DZ4dPq8fWh/5cqmnXvccr2jJVEXaOjp1wKGLH0WfLXcRLIK4/fJM6NRNoO81HDN +hJGfBrot0gUKZcPZVQmouAlpu5OGwrfCkHR8v/BdvA5jE4zr+g/x+uUScE0M64wu +okJCAAQP/PkfQZxjePBmk7KPLuiTHFDLLX+2uldvUmLXOQsJgqumU03MBT4Z8NTA +zqmtEM65ceSP8lo8Zbrcy+AEkCulFaZ92tyjtbe8oN4wTmTLFw06oFLSZzuiOgDV +OaphdVKf/pvA6KBpr6izox0KQFIE5z3AAJZfKzMGDDD20xhy7jjQZNMAhjfsT+k4 +SeYB/6KafNxq08uoulj7w4Z4R/EGpkXnU96ZHYHmvGN0RnxwI1cpYHCazG8AjsK/ +anN9brBi5twTGrn+D8LRBqF5Yn+2MKkD0EdXJdtIENHP+32sPQ== +-----END CERTIFICATE----- diff --git a/Lib/test/certdata/ffdh3072.pem b/Lib/test/certdata/ffdh3072.pem new file mode 100644 index 0000000000..ad69bac8d0 --- /dev/null +++ b/Lib/test/certdata/ffdh3072.pem @@ -0,0 +1,41 @@ + DH Parameters: (3072 bit) + prime: + 00:ff:ff:ff:ff:ff:ff:ff:ff:ad:f8:54:58:a2:bb: + 4a:9a:af:dc:56:20:27:3d:3c:f1:d8:b9:c5:83:ce: + 2d:36:95:a9:e1:36:41:14:64:33:fb:cc:93:9d:ce: + 24:9b:3e:f9:7d:2f:e3:63:63:0c:75:d8:f6:81:b2: + 02:ae:c4:61:7a:d3:df:1e:d5:d5:fd:65:61:24:33: + f5:1f:5f:06:6e:d0:85:63:65:55:3d:ed:1a:f3:b5: + 57:13:5e:7f:57:c9:35:98:4f:0c:70:e0:e6:8b:77: + e2:a6:89:da:f3:ef:e8:72:1d:f1:58:a1:36:ad:e7: + 35:30:ac:ca:4f:48:3a:79:7a:bc:0a:b1:82:b3:24: + fb:61:d1:08:a9:4b:b2:c8:e3:fb:b9:6a:da:b7:60: + d7:f4:68:1d:4f:42:a3:de:39:4d:f4:ae:56:ed:e7: + 63:72:bb:19:0b:07:a7:c8:ee:0a:6d:70:9e:02:fc: + e1:cd:f7:e2:ec:c0:34:04:cd:28:34:2f:61:91:72: + fe:9c:e9:85:83:ff:8e:4f:12:32:ee:f2:81:83:c3: + fe:3b:1b:4c:6f:ad:73:3b:b5:fc:bc:2e:c2:20:05: + c5:8e:f1:83:7d:16:83:b2:c6:f3:4a:26:c1:b2:ef: + fa:88:6b:42:38:61:1f:cf:dc:de:35:5b:3b:65:19: + 03:5b:bc:34:f4:de:f9:9c:02:38:61:b4:6f:c9:d6: + e6:c9:07:7a:d9:1d:26:91:f7:f7:ee:59:8c:b0:fa: + c1:86:d9:1c:ae:fe:13:09:85:13:92:70:b4:13:0c: + 93:bc:43:79:44:f4:fd:44:52:e2:d7:4d:d3:64:f2: + e2:1e:71:f5:4b:ff:5c:ae:82:ab:9c:9d:f6:9e:e8: + 6d:2b:c5:22:36:3a:0d:ab:c5:21:97:9b:0d:ea:da: + 1d:bf:9a:42:d5:c4:48:4e:0a:bc:d0:6b:fa:53:dd: + ef:3c:1b:20:ee:3f:d5:9d:7c:25:e4:1d:2b:66:c6: + 2e:37:ff:ff:ff:ff:ff:ff:ff:ff + generator: 2 (0x2) + recommended-private-length: 276 bits +-----BEGIN DH PARAMETERS----- +MIIBjAKCAYEA//////////+t+FRYortKmq/cViAnPTzx2LnFg84tNpWp4TZBFGQz ++8yTnc4kmz75fS/jY2MMddj2gbICrsRhetPfHtXV/WVhJDP1H18GbtCFY2VVPe0a +87VXE15/V8k1mE8McODmi3fipona8+/och3xWKE2rec1MKzKT0g6eXq8CrGCsyT7 +YdEIqUuyyOP7uWrat2DX9GgdT0Kj3jlN9K5W7edjcrsZCwenyO4KbXCeAvzhzffi +7MA0BM0oNC9hkXL+nOmFg/+OTxIy7vKBg8P+OxtMb61zO7X8vC7CIAXFjvGDfRaD +ssbzSibBsu/6iGtCOGEfz9zeNVs7ZRkDW7w09N75nAI4YbRvydbmyQd62R0mkff3 +7lmMsPrBhtkcrv4TCYUTknC0EwyTvEN5RPT9RFLi103TZPLiHnH1S/9croKrnJ32 +nuhtK8UiNjoNq8Uhl5sN6todv5pC1cRITgq80Gv6U93vPBsg7j/VnXwl5B0rZsYu +N///////////AgECAgIBFA== +-----END DH PARAMETERS----- diff --git a/Lib/test/certdata/idnsans.pem b/Lib/test/certdata/idnsans.pem new file mode 100644 index 0000000000..07a42422af --- /dev/null +++ b/Lib/test/certdata/idnsans.pem @@ -0,0 +1,166 @@ +-----BEGIN PRIVATE KEY----- +MIIG/AIBADANBgkqhkiG9w0BAQEFAASCBuYwggbiAgEAAoIBgQCp6zt40WB3K7yj +BGugnRuqI3ApftThZWDIpvW0cVmN0nqQxsO6CCnS4dS7SYhGFiIqWjNVc2WG0gv7 +nC5DFguqbndNZk9/SjX8EOxKz4ANjd61WnTkDO5Tbiiyd+TuEBxhmbEF69bF9dtd +1Sgo8jmM7j+aa6ClYh/49bx+blJDF76EGSrmB1q+obMeZURhPXNBeoiqKR83x5Hc +LTJYMocvb6m8uABwuSka13Gb3QGu06p5ldK6TDK38HsoOy6MFO5F1PrkakG/eBHO +jcBOGPfNmTwWOqvwlcQWykr4QspWS+yTzdkgZ+mxar/yQuq7wuYSNaEfGH5yoYtV +WIgKwwZRDPqpSQuVe+J+MWLPQ6RTM+rXIHVzHtPk1f8DrgN+hSepJy/sVBBEQCzj +nyB+scn76ETWch3iyVoMj3oVOGs0b4XTDMmUw/DmEt5TDah7TqE3G+fpBIbgMSjx +MzUQZl27izmM9nQCJRAosNoNwXqlM754K9WcY6gT8kkcj1CfTmMCAwEAAQKCAYAz +9ZdHkDsf5fN2pAznXfOOOOz8+2hMjmwkn42GAp1gdWr+Z5GFiyaC8oTTSp6N1AnZ +iqCk8jcrHYMFi1JIOG8TzFjWBcGsinxsmp4vGDmvq2Ddcw5IiD2+rHJsdKZAOBP9 +snpD9cTE3zQYAu0XbE617krrxRqoSBO/1SExRjoIgzPCgFGyarBQl/DGjC/3Tku2 +y6oL4qxFqdTMD9QTzUuycUJlz5xu2+gaaaQ3hcMUe2xnZq28Qz3FKpf2ivZmZqWf +4+AIe0lRosmFoLAFjIyyuGCkWZ2t9KDIZV0OOS4+DvVOC/Um9r4VojeikripCGKY +2FzkkuQP3jz6pJ1UxCDg7YXZdR2IbcS18F1OYmLViU8oLDR6T01s0Npmp39dDazf +A4U+WyV3o1ydiSpwAiN8pJadmr5BHrCSmawV8ztW5yholufrO+FR5ze2+QG99txm +6l7lUI8Nz01lYG6D10MjaQ9INk2WSjBPVNbfsTl73/hR76/319ctfOINRMHilJ0C +gcEAvFgTdc5Qf9E7xEZE5UtpSTPvZ24DxQ7hmGW3pTs6+Nw4ccknnW0lLkWvY4Sf +gXl4TyTTUhe6PAN3UHzbjN2vdsTmAVUlnkNH40ymF8FgRGFNDuvuCZvf5ZWNddSl +6Vu/e5TFPDAqS8rGnl23DgXhlT+3Y0/mrftWoigdOxgximLAsmmqp/tANGi9jqE1 +3L0BN+xKqMMKSXkMtuzJHuini8MaWQgQcW/4czh4ArdesMzuVrstOD8947XHreY9 +pesVAoHBAOb0y/AFEdR+mhk/cfqjTzsNU2sS9yHlzMVgecF8BA26xcfAwg4d47VS ++LK8fV6KC4hHk4uQWjQzCG2PYXBbFT52xeJ3EC8DwWxJP09b4HV/1mWxXl5htjnr +dfyTmXKvEe5ZBpKGWc8i7s7jBi7R5EpgIfc586iNRyjYAk60dyG0iP13SurRvXBg +ID25VR4wABl3HQ3Hhv61dqC9FPrdHZQJdysfUqNrAFniWsSR2eyG5i4S1uHa3G+i +MzBTOuBRlwKBwBNXUBhG6YlWqTaMqMKLLfKwfKM4bvargost1uAG5xVrN/inWYQX +EzxfN5WWpvKa0Ln/5BuICD3ldTk0uS8MDNq7eYslfUl1S0qSMnQ6DXK4MzuXCsi9 +0w42f2JcRfVi0JUWP/LgV1eVKTRWF1g/Tl0PP/vY1q2DI/BfAjFxWJUHcxZfN4Es +kflP0Dd3YpqaZieiAkC2VrYY0i9uvXCJH7uAe5Is+9NKVk8uu1Q8FGM/iDIr4obm +J6rcnfbDsAz7yQKBwGtIbW9qO3UU9ioiQaTmtYg90XEclzXk1HEfNo+9NvjVuMfo +b3w1QDBbgXEtg6MlxuOgNBaRkIVM625ROzcA6GZir9tZ6Wede/z8LW+Ew0hxgLsu +YCLBiu9uxBj2y0HttwubySTJSfChToNGC/o1v7EY5M492kSCk/qSFMhQpkI+5Z+w +CVn44eHQlUl2zOY/79vka9eZxsiMrLVP/+3kRrgciYG7hByrOLeIIRfMlIl9xHDE +iZLSorEsjFC3aNMIswKBwFELC2fvlziW9rECQcGXnvc1DPmZcxm1ATFZ93FpKleF +TjLIWSdst0PmO8CSIuEZ2ZXPoK9CMJyQG+kt3k7IgZ1xKXg9y6ThwbznurXp1jaW +NjEnYtFMBK9Ur3oaAsrG2XwZ2PMvnI/Yp8tciGvjJlzSM8gHJ9BL8Yf+3gIJi/0D +KtaF9ha9J/SDDZdEiLIQ4LvSqYmlUgsCgiUvY3SVwCh8xDfBWD1hKw9vUiZu5cnJ +81hAHFgeD4f+C8fLols/sA== +-----END PRIVATE KEY----- +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + cb:2d:80:99:5a:69:52:60 + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=XY, O=Python Software Foundation CA, CN=our-ca-server + Validity + Not Before: Aug 29 14:23:16 2018 GMT + Not After : Oct 28 14:23:16 2037 GMT + Subject: C=XY, L=Castle Anthrax, O=Python Software Foundation, CN=idnsans + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (3072 bit) + Modulus: + 00:a9:eb:3b:78:d1:60:77:2b:bc:a3:04:6b:a0:9d: + 1b:aa:23:70:29:7e:d4:e1:65:60:c8:a6:f5:b4:71: + 59:8d:d2:7a:90:c6:c3:ba:08:29:d2:e1:d4:bb:49: + 88:46:16:22:2a:5a:33:55:73:65:86:d2:0b:fb:9c: + 2e:43:16:0b:aa:6e:77:4d:66:4f:7f:4a:35:fc:10: + ec:4a:cf:80:0d:8d:de:b5:5a:74:e4:0c:ee:53:6e: + 28:b2:77:e4:ee:10:1c:61:99:b1:05:eb:d6:c5:f5: + db:5d:d5:28:28:f2:39:8c:ee:3f:9a:6b:a0:a5:62: + 1f:f8:f5:bc:7e:6e:52:43:17:be:84:19:2a:e6:07: + 5a:be:a1:b3:1e:65:44:61:3d:73:41:7a:88:aa:29: + 1f:37:c7:91:dc:2d:32:58:32:87:2f:6f:a9:bc:b8: + 00:70:b9:29:1a:d7:71:9b:dd:01:ae:d3:aa:79:95: + d2:ba:4c:32:b7:f0:7b:28:3b:2e:8c:14:ee:45:d4: + fa:e4:6a:41:bf:78:11:ce:8d:c0:4e:18:f7:cd:99: + 3c:16:3a:ab:f0:95:c4:16:ca:4a:f8:42:ca:56:4b: + ec:93:cd:d9:20:67:e9:b1:6a:bf:f2:42:ea:bb:c2: + e6:12:35:a1:1f:18:7e:72:a1:8b:55:58:88:0a:c3: + 06:51:0c:fa:a9:49:0b:95:7b:e2:7e:31:62:cf:43: + a4:53:33:ea:d7:20:75:73:1e:d3:e4:d5:ff:03:ae: + 03:7e:85:27:a9:27:2f:ec:54:10:44:40:2c:e3:9f: + 20:7e:b1:c9:fb:e8:44:d6:72:1d:e2:c9:5a:0c:8f: + 7a:15:38:6b:34:6f:85:d3:0c:c9:94:c3:f0:e6:12: + de:53:0d:a8:7b:4e:a1:37:1b:e7:e9:04:86:e0:31: + 28:f1:33:35:10:66:5d:bb:8b:39:8c:f6:74:02:25: + 10:28:b0:da:0d:c1:7a:a5:33:be:78:2b:d5:9c:63: + a8:13:f2:49:1c:8f:50:9f:4e:63 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Alternative Name: + DNS:idnsans, DNS:xn--knig-5qa.idn.pythontest.net, DNS:xn--knigsgsschen-lcb0w.idna2003.pythontest.net, DNS:xn--knigsgchen-b4a3dun.idna2008.pythontest.net, DNS:xn--nxasmq6b.idna2003.pythontest.net, DNS:xn--nxasmm1c.idna2008.pythontest.net + X509v3 Key Usage: critical + Digital Signature, Key Encipherment + X509v3 Extended Key Usage: + TLS Web Server Authentication, TLS Web Client Authentication + X509v3 Basic Constraints: critical + CA:FALSE + X509v3 Subject Key Identifier: + 5B:93:42:58:B0:B4:18:CC:41:4C:15:EB:42:33:66:77:4C:71:2F:42 + X509v3 Authority Key Identifier: + keyid:F3:EC:94:8E:F2:8E:30:C4:8E:68:C2:BF:8E:6A:19:C0:C1:9F:76:65 + DirName:/C=XY/O=Python Software Foundation CA/CN=our-ca-server + serial:CB:2D:80:99:5A:69:52:5B + Authority Information Access: + CA Issuers - URI:http://testca.pythontest.net/testca/pycacert.cer + OCSP - URI:http://testca.pythontest.net/testca/ocsp/ + X509v3 CRL Distribution Points: + Full Name: + URI:http://testca.pythontest.net/testca/revocation.crl + Signature Algorithm: sha256WithRSAEncryption + Signature Value: + 5f:d8:9b:dc:22:55:80:47:e1:9b:04:3e:46:53:9b:e5:a7:4a: + 8f:eb:53:01:39:d5:04:f6:cf:dc:48:84:8a:a9:c3:a5:35:22: + 2f:ab:74:77:ec:a6:fd:b1:e6:e6:74:82:38:54:0b:27:36:e6: + ec:3d:fe:92:1a:b2:7a:35:0d:a3:e5:7c:ff:e5:5b:1a:28:4b: + 29:1f:99:1b:3e:11:e9:e2:e0:d7:da:06:4f:e3:7b:8c:ad:30: + f4:39:24:e8:ad:2a:0e:71:74:ab:ed:62:e9:9f:85:7e:6a:b0: + bb:53:b4:d7:6b:b8:da:54:15:5c:9a:41:cf:61:f1:ab:67:d6: + 27:5c:0c:a3:d7:41:e7:27:3e:58:89:d6:1f:3f:2a:52:cc:13: + 0b:4b:e6:d6:ba:a0:c7:fd:e3:17:a4:b8:da:cc:cb:88:70:21: + 3b:70:df:09:40:6c:e7:02:81:08:80:b0:36:77:fb:44:c5:cf: + bf:19:54:7c:d1:4e:1f:a2:44:9e:d8:56:0e:bf:4b:0b:e0:84: + 6f:bc:f6:c6:7f:35:7a:17:ca:83:b3:82:c6:4e:d3:f3:d8:30: + 05:fd:6d:3c:8a:ab:63:55:6f:c5:18:ba:66:fe:e2:35:04:2b: + ae:76:34:f0:56:18:e8:54:db:83:b2:1b:93:0a:25:81:81:f0: + 25:ca:0a:95:be:8e:2f:05:3f:6c:e7:de:d1:7c:b8:a3:71:7c: + 6f:8a:05:c3:69:eb:6f:e6:76:8c:11:e1:59:0b:12:53:07:42: + 84:e8:89:ee:ab:7d:28:81:48:e8:79:d5:cf:a2:05:a4:fd:72: + 2c:7d:b4:1c:08:90:4e:0d:10:05:d1:9a:c0:69:4c:0a:14:39: + 17:fb:4d:5b:f6:42:bb:46:27:23:0f:5e:57:5b:b8:ae:9b:a3: + 0e:23:59:41:63:41:a4:f1:69:df:b3:a3:5c:10:d5:63:30:74: + a8:3c:0c:8e:1c:6b:10:e1:13:27:02:26:9b:fd:88:93:7e:91: + 9c:f9:c2:07:27:a4 +-----BEGIN CERTIFICATE----- +MIIGvTCCBSWgAwIBAgIJAMstgJlaaVJgMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV +BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW +MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xODA4MjkxNDIzMTZaFw0zNzEwMjgx +NDIzMTZaMF0xCzAJBgNVBAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEj +MCEGA1UECgwaUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24xEDAOBgNVBAMMB2lk +bnNhbnMwggGiMA0GCSqGSIb3DQEBAQUAA4IBjwAwggGKAoIBgQCp6zt40WB3K7yj +BGugnRuqI3ApftThZWDIpvW0cVmN0nqQxsO6CCnS4dS7SYhGFiIqWjNVc2WG0gv7 +nC5DFguqbndNZk9/SjX8EOxKz4ANjd61WnTkDO5Tbiiyd+TuEBxhmbEF69bF9dtd +1Sgo8jmM7j+aa6ClYh/49bx+blJDF76EGSrmB1q+obMeZURhPXNBeoiqKR83x5Hc +LTJYMocvb6m8uABwuSka13Gb3QGu06p5ldK6TDK38HsoOy6MFO5F1PrkakG/eBHO +jcBOGPfNmTwWOqvwlcQWykr4QspWS+yTzdkgZ+mxar/yQuq7wuYSNaEfGH5yoYtV +WIgKwwZRDPqpSQuVe+J+MWLPQ6RTM+rXIHVzHtPk1f8DrgN+hSepJy/sVBBEQCzj +nyB+scn76ETWch3iyVoMj3oVOGs0b4XTDMmUw/DmEt5TDah7TqE3G+fpBIbgMSjx +MzUQZl27izmM9nQCJRAosNoNwXqlM754K9WcY6gT8kkcj1CfTmMCAwEAAaOCAo4w +ggKKMIHhBgNVHREEgdkwgdaCB2lkbnNhbnOCH3huLS1rbmlnLTVxYS5pZG4ucHl0 +aG9udGVzdC5uZXSCLnhuLS1rbmlnc2dzc2NoZW4tbGNiMHcuaWRuYTIwMDMucHl0 +aG9udGVzdC5uZXSCLnhuLS1rbmlnc2djaGVuLWI0YTNkdW4uaWRuYTIwMDgucHl0 +aG9udGVzdC5uZXSCJHhuLS1ueGFzbXE2Yi5pZG5hMjAwMy5weXRob250ZXN0Lm5l +dIIkeG4tLW54YXNtbTFjLmlkbmEyMDA4LnB5dGhvbnRlc3QubmV0MA4GA1UdDwEB +/wQEAwIFoDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwDAYDVR0TAQH/ +BAIwADAdBgNVHQ4EFgQUW5NCWLC0GMxBTBXrQjNmd0xxL0IwfQYDVR0jBHYwdIAU +8+yUjvKOMMSOaMK/jmoZwMGfdmWhUaRPME0xCzAJBgNVBAYTAlhZMSYwJAYDVQQK +DB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEWMBQGA1UEAwwNb3VyLWNh +LXNlcnZlcoIJAMstgJlaaVJbMIGDBggrBgEFBQcBAQR3MHUwPAYIKwYBBQUHMAKG +MGh0dHA6Ly90ZXN0Y2EucHl0aG9udGVzdC5uZXQvdGVzdGNhL3B5Y2FjZXJ0LmNl +cjA1BggrBgEFBQcwAYYpaHR0cDovL3Rlc3RjYS5weXRob250ZXN0Lm5ldC90ZXN0 +Y2Evb2NzcC8wQwYDVR0fBDwwOjA4oDagNIYyaHR0cDovL3Rlc3RjYS5weXRob250 +ZXN0Lm5ldC90ZXN0Y2EvcmV2b2NhdGlvbi5jcmwwDQYJKoZIhvcNAQELBQADggGB +AF/Ym9wiVYBH4ZsEPkZTm+WnSo/rUwE51QT2z9xIhIqpw6U1Ii+rdHfspv2x5uZ0 +gjhUCyc25uw9/pIasno1DaPlfP/lWxooSykfmRs+Eeni4NfaBk/je4ytMPQ5JOit +Kg5xdKvtYumfhX5qsLtTtNdruNpUFVyaQc9h8atn1idcDKPXQecnPliJ1h8/KlLM +EwtL5ta6oMf94xekuNrMy4hwITtw3wlAbOcCgQiAsDZ3+0TFz78ZVHzRTh+iRJ7Y +Vg6/SwvghG+89sZ/NXoXyoOzgsZO0/PYMAX9bTyKq2NVb8UYumb+4jUEK652NPBW +GOhU24OyG5MKJYGB8CXKCpW+ji8FP2zn3tF8uKNxfG+KBcNp62/mdowR4VkLElMH +QoToie6rfSiBSOh51c+iBaT9cix9tBwIkE4NEAXRmsBpTAoUORf7TVv2QrtGJyMP +XldbuK6bow4jWUFjQaTxad+zo1wQ1WMwdKg8DI4caxDhEycCJpv9iJN+kZz5wgcn +pA== +-----END CERTIFICATE----- diff --git a/Lib/test/certdata/keycert.passwd.pem b/Lib/test/certdata/keycert.passwd.pem new file mode 100644 index 0000000000..187021b8ee --- /dev/null +++ b/Lib/test/certdata/keycert.passwd.pem @@ -0,0 +1,69 @@ +-----BEGIN ENCRYPTED PRIVATE KEY----- +MIIHbTBXBgkqhkiG9w0BBQ0wSjApBgkqhkiG9w0BBQwwHAQIc17oH9riZswCAggA +MAwGCCqGSIb3DQIJBQAwHQYJYIZIAWUDBAEqBBDwi0Mkj59S0hplpnDSNHwPBIIH +EFGdZuO4Cwzg0bspLhE1UpBN5cBq1rKbf4PyVtCczIqJt3KjO3H5I4KdQd9zihkN +A1qzMiqVZOnQZw1eWFXMdyWuCgvNe1S/PRLWY3iZfnuZ9gZXQvyMEHy4JU7pe2Ib +GNm9mzadzJtGv0YZ05Kkza20zRlOxC/cgaNUV6TPeTSwW9CR2bylxw0lTFKBph+o +uFGcAzhqQuw9vsURYJf1f1iE7bQsnWU2kKmb9cx6kaUXiGJpkUMUraBL/rShoHa0 +eet6saiFnK3XGMCIK0mhS9s92CIQV5H9oQQPo/7s6MOoUHjC/gFoWBXoIDOcN9aR +ngybosCLtofY2m14WcHXvu4NJnfnKStx73K3dy3ZLr2iyjnsqGD1OhqGEWOVG/ho +QiZEhZ+9sOnqWI2OuMhMoQJNvrLj7AY4QbdkahdjNvLjDAQSuMI2uSUDFDNfkQdy +hqF/iiEM28PmSHCapgCpzR4+VfEfXBoyBCqs973asa9qhrorfnBVxXnvsqmKNLGH +dymtEPei9scpoftE5T9TPqQj46446bXk23Xpg8QIFa8InQC2Y+yZqqlqvzCAbN6S +Qcq1DcTSAMnbmBXVu9hPmJYIYOlBMHL8JGbsGrkVOhLiiIou4w3G+DyAvIwPj6j9 +BHLqa7HgUnUEC+zL4azVHOSMqmDsOiF3w9fkBWNSkOyNoZpe+gBjbxq7sp+GjAJv +1CemRC3LSoNzLcjRG2IEGs1jlEHSSfijvwlE4lEy3JVc+QK8BOkKXXDVhY1SQHcS +pniEnj95RFVmAujdFDBoUgySyxK/y6Ju/tHPpSTG9VMNbJTKTdBWAVWWHVJzBFhR +0Ug62VrBK7fmfUdH1b37aIxqsPND2De6WLm0RX+7r3XPDJ7hm+baKCchI5CvnG19 +ky8InhMbU4qV+9LceMETmNKKDkhKl4Zx/Y3nab7DG9s/RZfrTdCHojc9Va/t0Ykp +qlVrvdj/893CdI78SW3VjWBJGWfKMyT16hBMY3TPz6ulbFXk6Pul/KcLLWslghS+ +GKZjyBe96UwfH4C7WjuIB+zo+De3Wr8xOCdJR5zwEutBMM+L/Wul8B6wIEGS71kB +TN/CAoeIgHLQFbcw4YE80dllTnSEsqF+ahVTTcCt3iLUaOgeTUxteMbXY9+nekSX +x8aUcvkMhbU9omdEowFr5/HIMKXo4UXat4fIGgh2pG8v8fA46hZXkhWUh/PhbnQw +StXzn4fA13erqVI679kHMmOIQebv4oqdcwkImrH5fEsACNjQbkYZF5fD4z+1GHkA +e2eGqejVT+OV14I8qfx9oqs2f8aqijH8fYLU0TymE7p53DYZy4WvDwk22I4rMzoQ +sGkOZwfKUYpdBI2t6tEf1ROBjoNG0E2Onq+5iooibN08rKXKAQMWsK+2vNHNHwBW +49vRheQNnRqSuLY+b7QAjA0KuRWo9YptCbnXyF/Aw64jMfAGjggDLoaZfALGZk3n +P+ZoL9xc7rYRpIca44BeYI6AhHFcWWIOX7Sm69FvmyHlfsgTAXVgY1lQPuGy68Au +PHSkgUyydDtkrfb2W2gJuqD/+h+9X2z+o/+nETYPCZm3sH5xvTY/DTcTx9kTpXxx +YQBaFTt12eVX7wZVr5K3u9M371rg+SeXC2SzL4T6APHD52cxbA1jgM0JFh3KJTuk +fADxIzM1NdzYQ45J6i2w+/Fh4VPnXZ0oiUSwE094XTBlvhI6zHgar2Q0Qx1P51vB +odd9XzyDLULuIzei0DYjTIg0KhE+wAGq1I5qtiMhmy5TdCKKNA9WGb1Pq38zpyjU +wGmztzSzCEjfLyhChaUObVRRxEfD5ioxKer/fczOhKQe8FXmGy5u/04tVmmEyNOO +JkkDtZy+UbKuJ257QnY72wPjgtHNy+S4Iv7zHUbNJNhxk+xBlRcmRNWCEM20LBSO +Tj4S9gyan+gH2+WFxy8FaENUhM+vHFEeJcjQIBFBeWICmSmdkh/r0YK1UVJ9NLfR +l0HiKm3lKg+kNCexTAPLMt2rGZ4PAKVnhVaxtuHMYYDpl2GYmyH73B9BfcPdA/Zx +GUBmd9hwcLz9SuUg+fjHcogZRRRlcZlKhw3zUCsqHSCQXZCQm7mBlG/5C/7cM7wQ +IRtsNospLStOg51gv21ClQ+uWx30XEcwmnIfVoLl1vMaguuf1u5u3dWBD/UgmqiP +1Ym8jv0BF/AS+u/CtUpwe7ZWxFT0vbyi10xxIF7O07fwFa+5dME3ycZwcyiE95K1 +ftcHlGOIhuVBMSNZXC4I9LM+7IWy+hanUcK+v5RvwBDSJV3fnAOdfrka1L/HyEEb +x/FYKEiU/TAjXDw2NtZ2itpADTSG5KbdJSwPr01Ak7aE+QYe7TIKJhBDZXGQlqq8 +1wv77zyv7V5Xq2cxSEKgSqzB9fhYZCASe8+HWlV2T+Sd +-----END ENCRYPTED PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIIEgzCCAuugAwIBAgIUU+FIM/dUbCklbdDwNPd2xemDAEwwDQYJKoZIhvcNAQEL +BQAwXzELMAkGA1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMwIQYD +VQQKDBpQeXRob24gU29mdHdhcmUgRm91bmRhdGlvbjESMBAGA1UEAwwJbG9jYWxo +b3N0MB4XDTIzMTEyNTA0MjEzNloXDTQzMDEyNDA0MjEzNlowXzELMAkGA1UEBhMC +WFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMwIQYDVQQKDBpQeXRob24gU29m +dHdhcmUgRm91bmRhdGlvbjESMBAGA1UEAwwJbG9jYWxob3N0MIIBojANBgkqhkiG +9w0BAQEFAAOCAY8AMIIBigKCAYEAzXTIl1su11AGu6sDPsoxqcRGyAX0yjxIcswF +vj+eW/fBs2GcBby95VEOKpJPKRYYB7fAEAjAKK59zFdsDX/ynxPZLqyLQocBkFVq +tclhCRZu//KZND+uQuHSx3PjGkSvK/nrGjg5T0bkM4SFeb0YdLb+0aDTKGozUC82 +oBAilNcrFz1VXpEF0qUe9QeKQhyd0MaW5T1oSn+U3RAj2MXm3TGExyZeaicpIM5O +HFlnwUxsYSDZo0jUj342MbPOZh8szZDWi042jdtSA3i8uMSplEf4O8ZPmX0JCtrz +fVjRVdaKXIjrhMNWB8K44q6AeyhqJcVHtOmPYoHDm0qIjcrurt0LZaGhmCuKimNd +njcPxW0VQmDIS/mO5+s24SK+Mpznm5q/clXEwyD8FbrtrzV5cHCE8eNkxjuQjkmi +wW9uadK1s54tDwRWMl6DRWRyxoF0an885UQWmbsgEB5aRmEx2L0JeD0/q6Iw1Nta +As8DG4AaWuYMrgZXz7XvyiMq3IxVAgMBAAGjNzA1MBQGA1UdEQQNMAuCCWxvY2Fs +aG9zdDAdBgNVHQ4EFgQUl2wd7iWE1JTZUVq2yFBKGm9N36owDQYJKoZIhvcNAQEL +BQADggGBAF0f5x6QXFbgdyLOyeAPD/1DDxNjM68fJSmNM/6vxHJeDFzK0Pja+iJo +xv54YiS9F2tiKPpejk4ujvLQgvrYrTQvliIE+7fUT0dV74wZKPdLphftT9uEo1dH +TeIld+549fqcfZCJfVPE2Ka4vfyMGij9hVfY5FoZL1Xpnq/ZGYyWZNAPbkG292p8 +KrfLZm/0fFYAhq8tG/6DX7+2btxeX4MP/49tzskcYWgOjlkknyhJ76aMG9BJ1D7F +/TIEh5ihNwRTmyt023RBz/xWiN4xBLyIlpQ6d5ECKmFNFr0qnEui6UovfCHUF6lZ +qcAQ5VFQQ2CayNlVmQ+UGmWIqANlacYWBt7Q6VqpGg24zTMec1/Pqd6X07ScSfrm +MAtywrWrU7p1aEkN5lBa4n/XKZHGYMjor/YcMdF5yjdSrZr274YYO1pafmTFwRwH +5o16c8WPc0aPvTFbkGIFT5ddxYstw+QwsBtLKE2lJ4Qfmxt0Ew/0L7xkbK1BaCOo +EGD2IF7VDQ== +-----END CERTIFICATE----- diff --git a/Lib/test/certdata/keycert.pem b/Lib/test/certdata/keycert.pem new file mode 100644 index 0000000000..a30d15ca4d --- /dev/null +++ b/Lib/test/certdata/keycert.pem @@ -0,0 +1,67 @@ +-----BEGIN PRIVATE KEY----- +MIIG/QIBADANBgkqhkiG9w0BAQEFAASCBucwggbjAgEAAoIBgQDNdMiXWy7XUAa7 +qwM+yjGpxEbIBfTKPEhyzAW+P55b98GzYZwFvL3lUQ4qkk8pFhgHt8AQCMAorn3M +V2wNf/KfE9kurItChwGQVWq1yWEJFm7/8pk0P65C4dLHc+MaRK8r+esaODlPRuQz +hIV5vRh0tv7RoNMoajNQLzagECKU1ysXPVVekQXSpR71B4pCHJ3QxpblPWhKf5Td +ECPYxebdMYTHJl5qJykgzk4cWWfBTGxhINmjSNSPfjYxs85mHyzNkNaLTjaN21ID +eLy4xKmUR/g7xk+ZfQkK2vN9WNFV1opciOuEw1YHwrjiroB7KGolxUe06Y9igcOb +SoiNyu6u3QtloaGYK4qKY12eNw/FbRVCYMhL+Y7n6zbhIr4ynOebmr9yVcTDIPwV +uu2vNXlwcITx42TGO5COSaLBb25p0rWzni0PBFYyXoNFZHLGgXRqfzzlRBaZuyAQ +HlpGYTHYvQl4PT+rojDU21oCzwMbgBpa5gyuBlfPte/KIyrcjFUCAwEAAQKCAYAO +M1r0+TCy4Z1hhceu5JdLql0RELZTbxi71IW2GVwW87gv75hy3hGLAs/1mdC+YIBP +MkBka1JqzWq0/7rgcP5CSAMsInFqqv2s7fZ286ERGXuZFbnInnkrNsQUlJo3E9W+ +tqKtGIM/i0EVHX0DRdJlqMtSjmjh43tB+M1wAUV+n6OjEtJue5wZK+AIpBmGicdP +qZY+6IBnm8tcfzPXFRCoq7ZHdIu0jxnc4l2MQJK3DdL04KoiStOkSl8xDsI+lTtq +D3qa41LE0TY8X2jJ/w6KK3cUeK7F4DQYs+kfCKWMVPpn0/5u6TbC1F7gLvkrseph +7cIgrruNNs9iKacnR1w3U72R+hNxHsNfo4RGHFa192p/Mfc+kiBd5RNR/M9oHdeq +U6T/+KM+QyF5dDOyonY0QjwfAcEx+ZsV72nj8AerjM907I6dgHo/9YZ2S1Dt/xuG +ntD+76GDzmrOvXmmpF0DsTn+Wql7AC4uzaOjv6PVziqz03pR61RpjPDemyJEWMkC +gcEA7BkGGX3enBENs3X6BYFoeXfGO/hV7/aNpA6ykLzw657dqwy2b6bWLiIaqZdZ +u0oiY6+SpOtavkZBFTq4bTVD58FHL0n73Yvvaft507kijpYBrxyDOfTJOETv+dVG +XiY8AUSAE6GjPi0ebuYIVUxoDnMeWDuRJNvTck4byn1hJ1aVlEhwXNxt/nAjq48s +5QDuR6Z9F8lqEACRYCHSMQYFm35c7c1pPsHJnElX8a7eZ9lT7HGPXHaf/ypMkOzo +dvJNAoHBAN7GhDomff/kSgQLyzmqKqQowTZlyihnReapygwr8YpNcqKDqq6VlnfH +Jl1+qtSMSVI0csmccwJWkz1WtSjDsvY+oMdv4gUK3028vQAMQZo+Sh7OElFPFET3 +UmL+Nh73ACPgpiommsdLZQPcIqpWNT5NzO+Jm5xa+U9ToVZgQ7xjrqee5NUiMutr +r7UWAz7vDWu3x7bzYRRdUJxU18NogGbFGWJ1KM0c67GUXu2E7wBQdjVdS78UWs+4 +XBxKQkG2KQKBwQCtO+M82x122BB8iGkulvhogBjlMd8klnzxTpN5HhmMWWH+uvI1 +1G29Jer4WwRNJyU6jb4E4mgPyw7AG/jssLOlniy0Jw32TlIaKpoGXwZbJvgPW9Vx +tgnbDsIiR3o9ZMKMj42GWgike4ikCIc+xzRmvdMbHIHwUJfCfEtp9TtPGPnh9pDz +og3XLsMNg52GXnt3+VI6HOCE41XH+qj2rZt5r2tSVXEOyjQ7R5mOzSeFfXJVwDFX +v/a/zHKnuB0OAdUCgcBLrxPTEaqy2eMPdtZHM/mipbnmejRw/4zu7XYYJoG7483z +SlodT/K7pKvzDYqKBVMPm4P33K/x9mm1aBTJ0ZqmL+a9etRFtEjjByEKuB89gLX7 +uzTb7MrNF10lBopqgK3KgpLRNSZWWNXrtskMJ5eVICdkpdJ5Dyst+RKR3siEYzU9 ++yxxAFpeQsqB8gWORva/RsOR8yNjIMS3J9fZqlIdGA8ktPr0nEOyo96QQR5VdACE +5rpKI2cqtM6OSegynOkCgcAnr2Xzjef6tdcrxrQrq0DjEFTMoCAxQRa6tuF/NYHV +AK70Y4hBNX84Bvym4hmfbMUEuOCJU+QHQf/iDQrHXPhtX3X2/t8M+AlIzmwLKf2o +VwCYnZ8SqiwSaWVg+GANWLh0JuKn/ZYyR8urR79dAXFfp0UK+N39vIxNoBisBf+F +G8mca7zx3UtK2eOW8WgGHz+Y20VZy0m/nkNekd1ZTXoSGhL+iN4XsTRn1YQIn69R +kNdcwhtZZ3dpChUdf+w/LIc= +-----END PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIIEgzCCAuugAwIBAgIUU+FIM/dUbCklbdDwNPd2xemDAEwwDQYJKoZIhvcNAQEL +BQAwXzELMAkGA1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMwIQYD +VQQKDBpQeXRob24gU29mdHdhcmUgRm91bmRhdGlvbjESMBAGA1UEAwwJbG9jYWxo +b3N0MB4XDTIzMTEyNTA0MjEzNloXDTQzMDEyNDA0MjEzNlowXzELMAkGA1UEBhMC +WFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMwIQYDVQQKDBpQeXRob24gU29m +dHdhcmUgRm91bmRhdGlvbjESMBAGA1UEAwwJbG9jYWxob3N0MIIBojANBgkqhkiG +9w0BAQEFAAOCAY8AMIIBigKCAYEAzXTIl1su11AGu6sDPsoxqcRGyAX0yjxIcswF +vj+eW/fBs2GcBby95VEOKpJPKRYYB7fAEAjAKK59zFdsDX/ynxPZLqyLQocBkFVq +tclhCRZu//KZND+uQuHSx3PjGkSvK/nrGjg5T0bkM4SFeb0YdLb+0aDTKGozUC82 +oBAilNcrFz1VXpEF0qUe9QeKQhyd0MaW5T1oSn+U3RAj2MXm3TGExyZeaicpIM5O +HFlnwUxsYSDZo0jUj342MbPOZh8szZDWi042jdtSA3i8uMSplEf4O8ZPmX0JCtrz +fVjRVdaKXIjrhMNWB8K44q6AeyhqJcVHtOmPYoHDm0qIjcrurt0LZaGhmCuKimNd +njcPxW0VQmDIS/mO5+s24SK+Mpznm5q/clXEwyD8FbrtrzV5cHCE8eNkxjuQjkmi +wW9uadK1s54tDwRWMl6DRWRyxoF0an885UQWmbsgEB5aRmEx2L0JeD0/q6Iw1Nta +As8DG4AaWuYMrgZXz7XvyiMq3IxVAgMBAAGjNzA1MBQGA1UdEQQNMAuCCWxvY2Fs +aG9zdDAdBgNVHQ4EFgQUl2wd7iWE1JTZUVq2yFBKGm9N36owDQYJKoZIhvcNAQEL +BQADggGBAF0f5x6QXFbgdyLOyeAPD/1DDxNjM68fJSmNM/6vxHJeDFzK0Pja+iJo +xv54YiS9F2tiKPpejk4ujvLQgvrYrTQvliIE+7fUT0dV74wZKPdLphftT9uEo1dH +TeIld+549fqcfZCJfVPE2Ka4vfyMGij9hVfY5FoZL1Xpnq/ZGYyWZNAPbkG292p8 +KrfLZm/0fFYAhq8tG/6DX7+2btxeX4MP/49tzskcYWgOjlkknyhJ76aMG9BJ1D7F +/TIEh5ihNwRTmyt023RBz/xWiN4xBLyIlpQ6d5ECKmFNFr0qnEui6UovfCHUF6lZ +qcAQ5VFQQ2CayNlVmQ+UGmWIqANlacYWBt7Q6VqpGg24zTMec1/Pqd6X07ScSfrm +MAtywrWrU7p1aEkN5lBa4n/XKZHGYMjor/YcMdF5yjdSrZr274YYO1pafmTFwRwH +5o16c8WPc0aPvTFbkGIFT5ddxYstw+QwsBtLKE2lJ4Qfmxt0Ew/0L7xkbK1BaCOo +EGD2IF7VDQ== +-----END CERTIFICATE----- diff --git a/Lib/test/certdata/keycert2.pem b/Lib/test/certdata/keycert2.pem new file mode 100644 index 0000000000..c7c4aa7458 --- /dev/null +++ b/Lib/test/certdata/keycert2.pem @@ -0,0 +1,67 @@ +-----BEGIN PRIVATE KEY----- +MIIG/gIBADANBgkqhkiG9w0BAQEFAASCBugwggbkAgEAAoIBgQCyAUXjczgUEn7m +mOwDMi/++wDRxqJAJ2f7F9ADxTuOm+EtdpfYr4mBn8Uz9e3I+ZheG5y3QZ1ddBYA +9YTfcUL0on8UXLOOBVZCetxsQXoSAuDMPV0IXeEgtZZDXe7STqKSQeYk7Cz+VtHe +lZ8j7oOOcx5sJgpbaD+OGJnPoAdB8l8nQfxqAG45sW4P6gfLKoJLviKctDe5pvgi +JC8tvytg/IhESKeefLZ4ix2dNjj2GNUaL+khU6UEuM1kJHcPVjPoYc+y8fop/qhQ +0ithBhO2OvJ+YmOFdCE67SyCwU3p8zJpN+XkwbHttgmNg4OSs7H6V7E52/CsTNTA +SthBHXtxqaM+vjbGARrz2Fpc/n+LwRt7MGIR0gVtntTgnP0HoeHskhAIeDtaPrZ6 +zHdl3aDwgAecVebTEBT5YPboz+X1lWdOrRD2JW3bqXSRIN3E4qz5IMuNx3VvhpSR +eFZzR6QIbxQqzO/Vp93Ivy8hPZ6WMgfSYWs7CGtu4NP79PJfdMsCAwEAAQKCAYAc +e3yp2NlbyNvaXRTCrCim5ZXrexuiJUwLjvNfbxNJDeM5iZThfLEFd0GwP0U1l86M +HGH2pr6d4gHVVHPW5wIeL9Qit3SZoHv9djhH8DAuqpw6wgTdXlw0BipNjD23FBMK +URYYyVuntM+vDITi1Hrjc8Ml7e5RUvx8aa5O3R3cLQKRvwq7EWeRvrTMQhfOJ/ai +VQGnzmRuRevFVsHf0YuI4M+TEYcUooL2BdiOu8rggfezUYA9r2sjtshSok0UvKeb +79pNzWmg9EWVeFk+A0HQpyLq+3EVyB5UZ3CZRkT0XhEm1B7mpKrtcGMjaumNAam7 +jkhidGdhT/PV9BB1TttcqwTf+JH9P9sSpY9ZTA1LkkeWe9Rwqpxjjssqxxl6Xnds ++wUfuovVvHuBinsO+5PLE5UQByn21WcIBNnPCAMvALy/68T7z8+ATJ+I2CqBXuM2 +z5663hNrvdCu93PpK4j/k/1l3NTrthaorbnPhkmNYHQkBicpAfFQywrv6enD+30C +gcEA7Vlw76og4oxI7SSD6gTfo85OqTLp2CUZhNNxzYiUOOssRnGHBKsGZ8p0OhLN +vk9/SgbeHL5jsmnyd8ZuYWmsOQHRWgg1zO3S25iuq+VAo4cL/7IynoJ0RP5xP0Pw +QD+xJLZQp0XuLUtXnlc6dM5Hg7tOTthOP9UxA1i57lzpYfkRnKmSeWi+4IDJymOt +WoWnEK7Yr7qSg6aScLWRyIvAPVmKF9LToSFaTq0eOD0GIwAQxqNbIwN3U0UJ5Ruc +KRBVAoHBAL/+DNGqnEzhhWS6zqZp2eH90YR+b3R4yOK4PROm2AVA3h1GhIAiWX68 +PvKYZK9dZ9EdAswlFf9PVQYIXUraR3az0UiIywnNMri+kO1ZxwofGvljrOfRRLg0 +B46wuHi6dVgTWzjTl503G9+FpAYNHv184xsr1tne0pf2TKEnN7oyQciCV8qtr8vV +HrL46uaD0w1fcXIXbO3F/7ErLsvsgLzKfxR5BeQo6Fq0GmzD+lCmzVNirtfl2CZj +2ukROXUQnwKBwQDR1CqFlm/wGHk4PPnp31ke5XqhFoOpNFM1HAEV5VK0ZyQDOsZU +mCXXiCHsXUdKodk0RpIB80cMKaHTxbc7o0JAO50q7OszOmUZAggZq1jTuMYgzRb3 +DvlfLVpMxfEVu7kNbagr2STRIjRZpV/md569lM+L4Kp8wCrOfJgTZExm8txhFYCK +mNF2hCThKfHNfy7NDuY9pMF2ZcI8pig1lWbkVc5BdX7miifeOinnKfvM4XfzQ+OE +NsI8+WHgC+KoYukCgcBwrOpdCmHchOZCbZfl9m1Wwh16QrGqi1BqLnI53EsfGijA +yaftgzs+s7/FpEZC3PCWuw3vPTyhr69YcQQ/b8dNFM8YYJ+4SuMfpUds5Kl5eTPd +dO/+xMQtzus4hOJeiB9h50o8GYH7VGJZVhcjLgQoBGlMgvf+uVSitnvWgCumbORK +hqR7YF+xoov3wToquubcDE2KBdF54h/jnFJEf7I2GilmnHgmpRNoWBbCCmoXdy09 +aMbwEgY+0Y+iBOfRmkUCgcEAoHJLw7VnZQGQQL4l3lnoGU9o06RPkNbpda9G/Ptz +v+K7DXmHiLFVDszvZaPVreohuc0tKdrT0cZpZ21h0GQD4B6JX66R/y6CCAr0QpdA +pFZO9sc5ky6RJ4xZCoCsNJzORNUb36cagEzBWExb7Jz2v6gNa044K5bs3CVv5h15 +rJtTxZNn/gcnIk+gt//67WUnKLS4PR5PVCCqYhSbhFwx/OvVTJmflIBUinAclf2Q +M4mhHOfwBicqYzzEYbOE9Vk9 +-----END PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIIEjDCCAvSgAwIBAgIUQ2S3jJ5nve5k5956sgsrWY3vw9MwDQYJKoZIhvcNAQEL +BQAwYjELMAkGA1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMwIQYD +VQQKDBpQeXRob24gU29mdHdhcmUgRm91bmRhdGlvbjEVMBMGA1UEAwwMZmFrZWhv +c3RuYW1lMB4XDTIzMTEyNTA0MjEzN1oXDTQzMDEyNDA0MjEzN1owYjELMAkGA1UE +BhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMwIQYDVQQKDBpQeXRob24g +U29mdHdhcmUgRm91bmRhdGlvbjEVMBMGA1UEAwwMZmFrZWhvc3RuYW1lMIIBojAN +BgkqhkiG9w0BAQEFAAOCAY8AMIIBigKCAYEAsgFF43M4FBJ+5pjsAzIv/vsA0cai +QCdn+xfQA8U7jpvhLXaX2K+JgZ/FM/XtyPmYXhuct0GdXXQWAPWE33FC9KJ/FFyz +jgVWQnrcbEF6EgLgzD1dCF3hILWWQ13u0k6ikkHmJOws/lbR3pWfI+6DjnMebCYK +W2g/jhiZz6AHQfJfJ0H8agBuObFuD+oHyyqCS74inLQ3uab4IiQvLb8rYPyIREin +nny2eIsdnTY49hjVGi/pIVOlBLjNZCR3D1Yz6GHPsvH6Kf6oUNIrYQYTtjryfmJj +hXQhOu0sgsFN6fMyaTfl5MGx7bYJjYODkrOx+lexOdvwrEzUwErYQR17camjPr42 +xgEa89haXP5/i8EbezBiEdIFbZ7U4Jz9B6Hh7JIQCHg7Wj62esx3Zd2g8IAHnFXm +0xAU+WD26M/l9ZVnTq0Q9iVt26l0kSDdxOKs+SDLjcd1b4aUkXhWc0ekCG8UKszv +1afdyL8vIT2eljIH0mFrOwhrbuDT+/TyX3TLAgMBAAGjOjA4MBcGA1UdEQQQMA6C +DGZha2Vob3N0bmFtZTAdBgNVHQ4EFgQU5wVOIuQD/Jxmam/97g91+igosWQwDQYJ +KoZIhvcNAQELBQADggGBAFv5gW5x4ET5NXEw6vILlOtwxwplEbU/x6eUVR/AXtEz +jtq9zIk2svX/JIzSLRQnjJmb/nCDCeNcFMkkgIiB64I3yMJT9n50fO4EhSGEaITZ +vYAw0/U6QXw+B1VS1ijNA44X2zvC+aw1q9W+0SKtvnu7l16TQ654ey0Qh9hOF1HS +AZQ46593T9gaZMeexz4CShoBZ80oFOJezfNhyT3FK6tzXNbkVoJDhlLvr/ep81GG +mABUGtKQYYMhuSSp0TDvf7jnXxtQcZI5lQOxZp0fnWUcK4gMVJqFVicwY8NiOhAG +6TlvXYP4COLAvGmqBB+xUhekIS0jVzaMyek+hKK0sT/OE+W/fR5V9YT5QlHFJCf5 +IUIfDCpBZrBpsOTwsUm8eL0krLiBjYf0HgH5oFBc7aF4w1kuUJjlsJ68bzO9mLEF +HXDaOWbe00+7BMMDnyuEyLN8KaAGiN8x0NQRX+nTAjCdPs6E0NftcXtznWBID6tA +j5m7qjsoGurj6TlDsBJb1A== +-----END CERTIFICATE----- diff --git a/Lib/test/certdata/keycert3.pem b/Lib/test/certdata/keycert3.pem new file mode 100644 index 0000000000..20d9bd14e9 --- /dev/null +++ b/Lib/test/certdata/keycert3.pem @@ -0,0 +1,161 @@ +-----BEGIN PRIVATE KEY----- +MIIG/gIBADANBgkqhkiG9w0BAQEFAASCBugwggbkAgEAAoIBgQCgKihxC+2g7d7M +JfIUBfFWiuMwxg0WhdxPyGUzMAjexbEOHo0ojntxPdH9KYRwiKRKb9jnmzXp2CKT +hqBYJIetq/3LYZp4bvDJ/hVCL9e4jqu1l+wd9DkqhKZ69b6C1/d12JAKvC5TIT+/ +b7EglYU8KMNx2WO5KxIJeVpX68jn49YtUzg0hT0QiXj4eugbDk1L1f99xgvkHaVW +VQxi6MFNWHJq/xXUb8E/hd/Q3oadN1BXMWl9P46D4R+YGKQQdZFkwEJsbqijFvWW +bOoaz7TFxf8n0q616803aXLfaWikfEXLnznEvKo7vyEivtT/y14Nm+SiR3nS6E0y +Dt8gmeHdaHcrmQT+yQ6yNOYDCcfeYM+rBuvOUHPudjMy0k8K/0IPjDo0KActKPH0 +UVbyDBMKDdGQ2+LhRFLsGXHlD9b05PxhqTULe3LeK6KZ+iuGbWtwVLaL5S42WiCA +YXNShE1Ko0Q7wugAippXCf+aWP3Wx9ZTrsfiDBbIfnY5mlfdG90CAwEAAQKCAYAA +ogoE4FoxD5+YyPGa+KcKg4QAVlgI5cCIJC+aMy9lyfw4JRDDv0RnnynsSTS3ySJ1 +FNoTmD5vTSZd1ONfVc2fdxWKrzkQDsgu1C07VLsShKXTEuWg/K0ZKOsLg1scY0Qc +GB4BnNrGA1SgKg3WJiEfqr2S/pvxSGVK2krsHAdwOytGhJStSHWEUjbDLKEsMjNG +AHOBCL5VSXS00aM55NeWuanCGH36l/J4kMvgpHB9wJE1twFGuHCUvtgEHtzPH9fQ +plmI0QDREm6UE6Qh01lxmwx3Xc5ASBURmxs+bxpk94BPRpj8/eF2HPiJalrkJksj +Xk3QQ7k23v6XnmHKV3QqpjUgJTdbuMoTrVMu14cIH6FtXfwVhtthPnCI8rk5Lh8N +cqLC7HT+NE1JyygzuMToOHMmSJTQ8L6BTIaRCZjvGTPYaZfFgeMHvvhAJtP5zAcc +xQzyCyNBU8RdPGT8tJTyDUIRs20poqe7dKrPEIocKJX7tvNSI2QxkQ96Adxo1gEC +gcEAvI8m6QCDGgDWI8yTH9EvZQwq+tF8I+WMC+jbPuDwKg5ZKC7VjRO//9JzPY+c +TxmLnQu64OkECHbu7pswDBbtnPMbodF9inYEY5RkfufEjEMJGEdkoBJWnNx78EkV +bcffWik0wXwdt6jd1CAnjmS9qaPz0T1NV8m5rQQn5JUYXlC9eB2kOojZYLbZBl3g +xUSRbIqHC7h8HuyAU26EPiprHsIxrOpbxABFOdvo2optr50U7X10Eqb4mRQ4z22W +ojJdAoHBANlzJjjEgGVB9W50pJqkTw8wXiTUG8AmhqrVvqEttLPcWpK6QwRkRC+i +5N1iUObf/kOlun2gNfHF6gM68Ja9wb2eGvE5sApq9hPpyYF0LS3g8BbJ9GOs6NU9 +BfM1CkPrDCdc4kzlUpDibvc6Fc9raCqvrZRlKEukqQS8dumVdb74IaPsP6q8sZMz +jibOk0eUrbx2c5vEnd0W8zMeNCuCwO1oXbfenPp/GLX9ZRlolWS/3cQoZYOSQc9J +lFQYkxL3gQKBwQCy3Pwk9AZoqTh4dvtsqArUSImQqRygFIQXXAh1ifxneHrcYijS +jVSIwEHuuIamhe3oyBK6fG8F9IPLtUwLe8hkJDwm8Misiiy5pS77LrFD9+btr/Nk +4GBmpcOveDQqkflt1j6j9y9dY4MhUGsVaLx86fhDmGoAh2tpEtMgwsl91gsUoNGD +cQL6+he+MVkg510nX/Sgipy63M8R1Xj+W1CHueBTTXBE6ZjBPLiSbdOETXZnnaR4 +eQjCdOs64JKOQ0UCgcBZ4kFAYel48aTT/Z801QphCus/afX2nXY5E5Vy5oO1fTZr +RFcDb7bHwhu8bzFl3d0qdUz7NMhXoimzIB/nD5UQHlSgtenQxJnnbVIAEtfCCSL1 +KJG+yfCMhGb7O0d8/6HMe5aHlptkjFS2GOp/DLTIQEoN9yqK6gt7i7PTphY/1C2D +ptpCZzE32a2+2NEEW67dIlFzZ/ihNSVeUfPasHezKtricECPQw4h3BZ4RETMmoq+ +1LvxgPl3B8EqaeYRhwECgcEAjjp/0hu/ukQhiNeR5a9p1ECBFP8qFh6Cpo0Az/DT +1kX0qU8tnT3cYYhwbVGwLxn2HVRdLrbjMj/t88W/LM2IaQ162m7TvvBMxNmr058y +sW/LADp5YWWsY70EJ8AfaTmdQriqKsNiLLpNdgcm1bkwHJ1CNlvEpDs1OOI3cCGi +BEuUmeKxpRhwCaZeaR5tREmbD70My+BMDTDLfrXoKqzl4JrRua4jFTpHeZaFdkkh +gDq3K6+KpVREQFEhyOtIB2kk +-----END PRIVATE KEY----- +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + cb:2d:80:99:5a:69:52:5c + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=XY, O=Python Software Foundation CA, CN=our-ca-server + Validity + Not Before: Aug 29 14:23:16 2018 GMT + Not After : Oct 28 14:23:16 2037 GMT + Subject: C=XY, L=Castle Anthrax, O=Python Software Foundation, CN=localhost + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (3072 bit) + Modulus: + 00:a0:2a:28:71:0b:ed:a0:ed:de:cc:25:f2:14:05: + f1:56:8a:e3:30:c6:0d:16:85:dc:4f:c8:65:33:30: + 08:de:c5:b1:0e:1e:8d:28:8e:7b:71:3d:d1:fd:29: + 84:70:88:a4:4a:6f:d8:e7:9b:35:e9:d8:22:93:86: + a0:58:24:87:ad:ab:fd:cb:61:9a:78:6e:f0:c9:fe: + 15:42:2f:d7:b8:8e:ab:b5:97:ec:1d:f4:39:2a:84: + a6:7a:f5:be:82:d7:f7:75:d8:90:0a:bc:2e:53:21: + 3f:bf:6f:b1:20:95:85:3c:28:c3:71:d9:63:b9:2b: + 12:09:79:5a:57:eb:c8:e7:e3:d6:2d:53:38:34:85: + 3d:10:89:78:f8:7a:e8:1b:0e:4d:4b:d5:ff:7d:c6: + 0b:e4:1d:a5:56:55:0c:62:e8:c1:4d:58:72:6a:ff: + 15:d4:6f:c1:3f:85:df:d0:de:86:9d:37:50:57:31: + 69:7d:3f:8e:83:e1:1f:98:18:a4:10:75:91:64:c0: + 42:6c:6e:a8:a3:16:f5:96:6c:ea:1a:cf:b4:c5:c5: + ff:27:d2:ae:b5:eb:cd:37:69:72:df:69:68:a4:7c: + 45:cb:9f:39:c4:bc:aa:3b:bf:21:22:be:d4:ff:cb: + 5e:0d:9b:e4:a2:47:79:d2:e8:4d:32:0e:df:20:99: + e1:dd:68:77:2b:99:04:fe:c9:0e:b2:34:e6:03:09: + c7:de:60:cf:ab:06:eb:ce:50:73:ee:76:33:32:d2: + 4f:0a:ff:42:0f:8c:3a:34:28:07:2d:28:f1:f4:51: + 56:f2:0c:13:0a:0d:d1:90:db:e2:e1:44:52:ec:19: + 71:e5:0f:d6:f4:e4:fc:61:a9:35:0b:7b:72:de:2b: + a2:99:fa:2b:86:6d:6b:70:54:b6:8b:e5:2e:36:5a: + 20:80:61:73:52:84:4d:4a:a3:44:3b:c2:e8:00:8a: + 9a:57:09:ff:9a:58:fd:d6:c7:d6:53:ae:c7:e2:0c: + 16:c8:7e:76:39:9a:57:dd:1b:dd + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Alternative Name: + DNS:localhost + X509v3 Key Usage: critical + Digital Signature, Key Encipherment + X509v3 Extended Key Usage: + TLS Web Server Authentication, TLS Web Client Authentication + X509v3 Basic Constraints: critical + CA:FALSE + X509v3 Subject Key Identifier: + 3F:B1:E9:4F:A0:BE:30:66:3E:0A:18:C8:0F:47:1A:4F:34:6A:0F:42 + X509v3 Authority Key Identifier: + keyid:F3:EC:94:8E:F2:8E:30:C4:8E:68:C2:BF:8E:6A:19:C0:C1:9F:76:65 + DirName:/C=XY/O=Python Software Foundation CA/CN=our-ca-server + serial:CB:2D:80:99:5A:69:52:5B + Authority Information Access: + CA Issuers - URI:http://testca.pythontest.net/testca/pycacert.cer + OCSP - URI:http://testca.pythontest.net/testca/ocsp/ + X509v3 CRL Distribution Points: + Full Name: + URI:http://testca.pythontest.net/testca/revocation.crl + Signature Algorithm: sha256WithRSAEncryption + Signature Value: + ca:34:ba:c5:d0:cf:27:31:32:d6:0d:27:30:b8:db:17:df:b7: + 39:a7:bb:b1:3b:86:c4:31:fd:fb:ab:db:63:1a:cc:90:ab:b9: + 4e:ab:34:49:0c:5e:8c:3e:70:a3:a7:6b:2f:a7:9a:25:7b:01: + 5a:18:96:48:76:f8:36:78:74:fa:bc:7d:68:7f:e5:ca:a6:9d: + 7b:dc:72:bd:a3:25:51:17:68:e8:e9:d7:02:86:2c:7d:16:7c: + b5:dc:44:b2:0a:e3:f7:c9:33:a3:51:36:83:bc:d4:70:cd:84: + 91:9f:06:ba:2d:d2:05:0a:65:c3:d9:55:09:a8:b8:09:69:bb: + 93:86:c2:b7:c2:90:74:7c:bf:f0:5d:bc:0e:63:13:8c:eb:fa: + 0f:f1:fa:e5:12:70:4d:0c:eb:8c:2e:a2:42:42:00:04:0f:fc: + f9:1f:41:9c:63:78:f0:66:93:b2:8f:2e:e8:93:1c:50:cb:2d: + 7f:b6:ba:57:6f:52:62:d7:39:0b:09:82:ab:a6:53:4d:cc:05: + 3e:19:f0:d4:c0:ce:a9:ad:10:ce:b9:71:e4:8f:f2:5a:3c:65: + ba:dc:cb:e0:04:90:2b:a5:15:a6:7d:da:dc:a3:b5:b7:bc:a0: + de:30:4e:64:cb:17:0d:3a:a0:52:d2:67:3b:a2:3a:00:d5:39: + aa:61:75:52:9f:fe:9b:c0:e8:a0:69:af:a8:b3:a3:1d:0a:40: + 52:04:e7:3d:c0:00:96:5f:2b:33:06:0c:30:f6:d3:18:72:ee: + 38:d0:64:d3:00:86:37:ec:4f:e9:38:49:e6:01:ff:a2:9a:7c: + dc:6a:d3:cb:a8:ba:58:fb:c3:86:78:47:f1:06:a6:45:e7:53: + de:99:1d:81:e6:bc:63:74:46:7c:70:23:57:29:60:70:9a:cc: + 6f:00:8e:c2:bf:6a:73:7d:6e:b0:62:e6:dc:13:1a:b9:fe:0f: + c2:d1:06:a1:79:62:7f:b6:30:a9:03:d0:47:57:25:db:48:10: + d1:cf:fb:7d:ac:3d +-----BEGIN CERTIFICATE----- +MIIF8TCCBFmgAwIBAgIJAMstgJlaaVJcMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV +BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW +MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xODA4MjkxNDIzMTZaFw0zNzEwMjgx +NDIzMTZaMF8xCzAJBgNVBAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEj +MCEGA1UECgwaUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24xEjAQBgNVBAMMCWxv +Y2FsaG9zdDCCAaIwDQYJKoZIhvcNAQEBBQADggGPADCCAYoCggGBAKAqKHEL7aDt +3swl8hQF8VaK4zDGDRaF3E/IZTMwCN7FsQ4ejSiOe3E90f0phHCIpEpv2OebNenY +IpOGoFgkh62r/cthmnhu8Mn+FUIv17iOq7WX7B30OSqEpnr1voLX93XYkAq8LlMh +P79vsSCVhTwow3HZY7krEgl5WlfryOfj1i1TODSFPRCJePh66BsOTUvV/33GC+Qd +pVZVDGLowU1Ycmr/FdRvwT+F39Dehp03UFcxaX0/joPhH5gYpBB1kWTAQmxuqKMW +9ZZs6hrPtMXF/yfSrrXrzTdpct9paKR8RcufOcS8qju/ISK+1P/LXg2b5KJHedLo +TTIO3yCZ4d1odyuZBP7JDrI05gMJx95gz6sG685Qc+52MzLSTwr/Qg+MOjQoBy0o +8fRRVvIMEwoN0ZDb4uFEUuwZceUP1vTk/GGpNQt7ct4ropn6K4Zta3BUtovlLjZa +IIBhc1KETUqjRDvC6ACKmlcJ/5pY/dbH1lOux+IMFsh+djmaV90b3QIDAQABo4IB +wDCCAbwwFAYDVR0RBA0wC4IJbG9jYWxob3N0MA4GA1UdDwEB/wQEAwIFoDAdBgNV +HSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwDAYDVR0TAQH/BAIwADAdBgNVHQ4E +FgQUP7HpT6C+MGY+ChjID0caTzRqD0IwfQYDVR0jBHYwdIAU8+yUjvKOMMSOaMK/ +jmoZwMGfdmWhUaRPME0xCzAJBgNVBAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29m +dHdhcmUgRm91bmRhdGlvbiBDQTEWMBQGA1UEAwwNb3VyLWNhLXNlcnZlcoIJAMst +gJlaaVJbMIGDBggrBgEFBQcBAQR3MHUwPAYIKwYBBQUHMAKGMGh0dHA6Ly90ZXN0 +Y2EucHl0aG9udGVzdC5uZXQvdGVzdGNhL3B5Y2FjZXJ0LmNlcjA1BggrBgEFBQcw +AYYpaHR0cDovL3Rlc3RjYS5weXRob250ZXN0Lm5ldC90ZXN0Y2Evb2NzcC8wQwYD +VR0fBDwwOjA4oDagNIYyaHR0cDovL3Rlc3RjYS5weXRob250ZXN0Lm5ldC90ZXN0 +Y2EvcmV2b2NhdGlvbi5jcmwwDQYJKoZIhvcNAQELBQADggGBAMo0usXQzycxMtYN +JzC42xfftzmnu7E7hsQx/fur22MazJCruU6rNEkMXow+cKOnay+nmiV7AVoYlkh2 ++DZ4dPq8fWh/5cqmnXvccr2jJVEXaOjp1wKGLH0WfLXcRLIK4/fJM6NRNoO81HDN +hJGfBrot0gUKZcPZVQmouAlpu5OGwrfCkHR8v/BdvA5jE4zr+g/x+uUScE0M64wu +okJCAAQP/PkfQZxjePBmk7KPLuiTHFDLLX+2uldvUmLXOQsJgqumU03MBT4Z8NTA +zqmtEM65ceSP8lo8Zbrcy+AEkCulFaZ92tyjtbe8oN4wTmTLFw06oFLSZzuiOgDV +OaphdVKf/pvA6KBpr6izox0KQFIE5z3AAJZfKzMGDDD20xhy7jjQZNMAhjfsT+k4 +SeYB/6KafNxq08uoulj7w4Z4R/EGpkXnU96ZHYHmvGN0RnxwI1cpYHCazG8AjsK/ +anN9brBi5twTGrn+D8LRBqF5Yn+2MKkD0EdXJdtIENHP+32sPQ== +-----END CERTIFICATE----- diff --git a/Lib/test/certdata/keycert4.pem b/Lib/test/certdata/keycert4.pem new file mode 100644 index 0000000000..ff4dceac79 --- /dev/null +++ b/Lib/test/certdata/keycert4.pem @@ -0,0 +1,161 @@ +-----BEGIN PRIVATE KEY----- +MIIG/QIBADANBgkqhkiG9w0BAQEFAASCBucwggbjAgEAAoIBgQDGKA1zZDjeNPh2 +J9WHVXXMUf8h5N4/bHCM3CbIaZ1dShkCgfmFWmOtruEihgbfRYaSWZAwCmVAQGjm +gvUfgOIgsFfM8yO+zDByPhza7XvWPZfEe7mNRFe5ZlYntbeM/vuWCM4VzwDq/mqF +TFxNRmwInqE7hx0WnfCoQWe9N41hJyl1K0OjADb+SjlpJ0/UJ63hsB+dowGjaaBv +J8HduQcRqNg8s6FcyJJ8Mjss1uRMFK2j9QrmgbA61XuIPCxzc3J57mW8FN2KsR8D +2HOhe9nsTGlxp+O5Cudf/RBWB443xcoyduwRXOFTdEAU45MS4tKGP2hzezuxMFQn +LKARXVW4/gFxZk7kU8TweZUS6LAYPfYJnlfteb6z37LAbtoDvzKUKBEDf/nmoa7C +uKxSPC5HIKhLbjU/6kuPglSVEfJPJWu2bZJDAkFL85Ot3gPs10EX2lMUy0Jt3tf+ +TaQjEvFZhpKN8KAdYj3eVgOfzIBbQyjotHJjFe9Jkq4q7RoI+ncCAwEAAQKCAYAH +tRsdRh1Z7JmHOasy+tPDsvhVuWLHMaYlScvAYhJh/W65YSKd56+zFKINlX3fYcp5 +Fz67Yy+uWahXVE2QgFou3KX0u+9ucRiLFXfYheWL3xSMXJgRee0LI/T7tRe7uAHu +CnoURqKCulIqzLOO1efx1eKasXmVuhEtmjhVpcmDGv8SChSKTIjzgOjqT7QGE9Xq +eSRhq7mulpq9zWq+/369yG+0SvPs60vTxNovDIaBn/RHSW5FjeDss5QnmYMh/ukN +dggoKllQlkTzHSxHmKrIJuryZC+bsqvEPUFXN0NMUYcZRvt1lwdjzq/A+w4gDDZG +7QqAzYMYQZMw9PJeHqu4mxfUX5hJWuAwG5I2eV3kBRheoFw7MxP0tw40fPlFU+Zh +pRXbKwhMAlIHi0D8NyMn3hkVPyToWVVY3vHRknBB/52RqRq3MjqEFaAZfp0nFkiF +ytv3Dd5aeBb1vraOIREyhxIxE/qY8CtZC+6JI8CpufLmFXB412WPwl0OrVpWYfEC +gcEA486zOI46xRDgDw0jqTpOFHzh+3VZ8UoPoiqCjKzJGnrh2EeLvTsXX/GZOj0m +5zl6RHEGFjm5vKCh2C72Vj/m+AFVy7V9iJRzTYzP8So/3paaqo7ZaROTa6uStxdD +VPnY1uIgVQz9w5coN4dmr+RLBpFvvWaHp1wuC08YIWxcC9HSTQpbi1EP5eo08fOk +8reNkDEHxihDGHr1xW0z0qJqK1IVyLP7wDkmapudMZjkjqJSGJwwefV4qyGMTV2b +suW1AoHBAN6t9n6LBH553MF5iUrNJYxXh/SCom4Zft9aD6W4bZV/xL4XPpKBB4HX +aWdeI0iYZU9U+CZ88tBoQCt+JMrJ9cz03ENOvA/MBMREwbZ2hKmQgnoDZsV0vNry +6UsxeQmeNpGQFUz9foVJQVRdQCceN2YEABdehV1HZoSBbuGZkzqGJXrWwaf/ZhpB +dPYGUGOsczoD2/QLuWy2M7f7v0Ews6Heww3zipWzvdxKE0IpyVs30ZwVi8CRQiWU +bEcleXP6+wKBwAi3xEwJxV39Q1XQHuk+/fXywYMp/oMpXmfKUKypgBivUy0/r61S +MZbOXBrKdE6s+GzeFmmLU/xP+WGYinzKfUBIbMwa6e7sH218UgjcoQ0Xnlugk9ld +kmqwajDvhvgdh5rRlIMsuBlgE33shJV+mxBpSGlrHw3cjTaJlFbTGsKpCO9B0jcG +pyEZUWVg+ZMASz6VYcLHj6nEKtufTjhlVsLJpWPE34F/rmSuB9n6C+UZeSLP91rz +dea2pfPf/TFfcQKBwF4DSj9Qx/vxzS7t9fXbuM+QoPitMpCTOQppRpPr0nA8uj6b +J7LIwPejj3+xsenTVWpx8DanqAgvC3CRWE05iQoYEupj0mhE9Xo7oSE81nOUbFHB +H+GbkKRLzA0P/Q7/egBouWWA3Kq/K9LHb+9UBYWPiM5U/K9OFs04rCyZHxylSCud +gbNA08Wf/xZjwgri4t8KhBF75bQtFJbHtY57Vkuv9d/tA4SCl1Tq/UiAxd86KMfi +HNeXPDsLd89t1eIOgwKBwQDJkqwZXkhwkhoNuHRdzPO/1f5FyKpQxFs+x+OBulzG +zuwVKIawsLlUR4TBtF7PChOSZSH50VZaBI5kVti79kEtfNjfAzg4kleHrY8jQ/eq +HludZ3nmiPqqlbH4MH8NWczPEjee6z4ODROsAe31pz3S8YQK7KVoEuSf0+usJ894 +FtzS5wl6POAXTo2QeSNg9zTbb6JjVYcq6KCTnflDm4YEvFKI+ARqAXQHxm05wEOe +DbKC6hxxQbDaNOvXEAda8wU= +-----END PRIVATE KEY----- +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + cb:2d:80:99:5a:69:52:5d + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=XY, O=Python Software Foundation CA, CN=our-ca-server + Validity + Not Before: Aug 29 14:23:16 2018 GMT + Not After : Oct 28 14:23:16 2037 GMT + Subject: C=XY, L=Castle Anthrax, O=Python Software Foundation, CN=fakehostname + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (3072 bit) + Modulus: + 00:c6:28:0d:73:64:38:de:34:f8:76:27:d5:87:55: + 75:cc:51:ff:21:e4:de:3f:6c:70:8c:dc:26:c8:69: + 9d:5d:4a:19:02:81:f9:85:5a:63:ad:ae:e1:22:86: + 06:df:45:86:92:59:90:30:0a:65:40:40:68:e6:82: + f5:1f:80:e2:20:b0:57:cc:f3:23:be:cc:30:72:3e: + 1c:da:ed:7b:d6:3d:97:c4:7b:b9:8d:44:57:b9:66: + 56:27:b5:b7:8c:fe:fb:96:08:ce:15:cf:00:ea:fe: + 6a:85:4c:5c:4d:46:6c:08:9e:a1:3b:87:1d:16:9d: + f0:a8:41:67:bd:37:8d:61:27:29:75:2b:43:a3:00: + 36:fe:4a:39:69:27:4f:d4:27:ad:e1:b0:1f:9d:a3: + 01:a3:69:a0:6f:27:c1:dd:b9:07:11:a8:d8:3c:b3: + a1:5c:c8:92:7c:32:3b:2c:d6:e4:4c:14:ad:a3:f5: + 0a:e6:81:b0:3a:d5:7b:88:3c:2c:73:73:72:79:ee: + 65:bc:14:dd:8a:b1:1f:03:d8:73:a1:7b:d9:ec:4c: + 69:71:a7:e3:b9:0a:e7:5f:fd:10:56:07:8e:37:c5: + ca:32:76:ec:11:5c:e1:53:74:40:14:e3:93:12:e2: + d2:86:3f:68:73:7b:3b:b1:30:54:27:2c:a0:11:5d: + 55:b8:fe:01:71:66:4e:e4:53:c4:f0:79:95:12:e8: + b0:18:3d:f6:09:9e:57:ed:79:be:b3:df:b2:c0:6e: + da:03:bf:32:94:28:11:03:7f:f9:e6:a1:ae:c2:b8: + ac:52:3c:2e:47:20:a8:4b:6e:35:3f:ea:4b:8f:82: + 54:95:11:f2:4f:25:6b:b6:6d:92:43:02:41:4b:f3: + 93:ad:de:03:ec:d7:41:17:da:53:14:cb:42:6d:de: + d7:fe:4d:a4:23:12:f1:59:86:92:8d:f0:a0:1d:62: + 3d:de:56:03:9f:cc:80:5b:43:28:e8:b4:72:63:15: + ef:49:92:ae:2a:ed:1a:08:fa:77 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Alternative Name: + DNS:fakehostname + X509v3 Key Usage: critical + Digital Signature, Key Encipherment + X509v3 Extended Key Usage: + TLS Web Server Authentication, TLS Web Client Authentication + X509v3 Basic Constraints: critical + CA:FALSE + X509v3 Subject Key Identifier: + 1C:70:14:B0:20:DD:08:76:A4:3B:56:59:FA:5F:34:F8:36:66:E8:56 + X509v3 Authority Key Identifier: + keyid:F3:EC:94:8E:F2:8E:30:C4:8E:68:C2:BF:8E:6A:19:C0:C1:9F:76:65 + DirName:/C=XY/O=Python Software Foundation CA/CN=our-ca-server + serial:CB:2D:80:99:5A:69:52:5B + Authority Information Access: + CA Issuers - URI:http://testca.pythontest.net/testca/pycacert.cer + OCSP - URI:http://testca.pythontest.net/testca/ocsp/ + X509v3 CRL Distribution Points: + Full Name: + URI:http://testca.pythontest.net/testca/revocation.crl + Signature Algorithm: sha256WithRSAEncryption + Signature Value: + 75:14:e5:68:45:8d:ed:6c:f1:27:1e:0e:f3:35:ae:0e:60:c1: + 65:36:62:b8:07:78:e1:b9:8d:7a:50:70:af:06:c9:d4:ee:50: + ef:d2:76:b2:a2:b6:cb:dc:a6:18:b5:3d:d2:f7:eb:0e:ec:b7: + 95:cd:2e:b1:36:6f:a8:9f:b8:4d:ff:ce:8a:c4:8e:62:37:32: + 80:3e:05:4a:4d:39:87:69:09:00:e8:40:64:d2:9d:f9:1f:9f: + ab:67:1f:f9:c6:84:ba:7e:17:6c:8b:8d:08:ee:fb:8a:d7:cd: + 06:25:72:9f:4e:1a:c2:71:e1:1b:cf:a2:d7:1c:05:12:95:d6: + 49:4b:e9:95:95:89:cf:68:18:46:a3:ea:0d:9d:8e:ca:1c:28: + 55:49:6b:c0:4b:58:f5:42:b9:0a:ec:0e:6e:21:a4:ff:60:c0: + 1b:6e:40:72:d0:a5:c5:b5:db:4e:87:67:3a:31:70:cb:32:84: + 70:a9:e2:ff:e0:f2:db:cd:03:b4:85:45:d3:07:cc:0f:c7:49: + d8:c2:17:eb:73:f7:4a:c0:d9:8c:59:ef:c0:0a:ce:13:0b:84: + c9:aa:0d:11:14:b4:e5:74:aa:ec:18:de:5f:26:18:98:4a:76: + f0:7f:cd:e6:c4:b5:58:03:03:f5:10:01:5d:8f:63:88:ba:65: + d7:b4:7f:5a:1a:51:0e:ed:e5:68:fa:18:03:72:15:a1:ec:27: + 1f:ea:ac:24:46:18:6e:f1:97:db:4a:f4:d6:a1:91:a0:8c:b0: + 2f:be:87:3b:44:b0:8d:2a:89:85:5f:f2:d9:e3:2e:66:b2:88: + 98:04:2c:96:32:38:99:19:a9:83:fd:94:0c:dd:63:d4:1b:60: + 9d:43:98:35:ac:b4:23:38:de:7f:85:52:57:a0:37:df:a5:cf: + be:54:2c:3c:50:27:2b:d4:54:a9:9d:a3:d4:a5:b3:c0:ea:3d: + 0e:e2:70:6b:fb:cb:a5:56:05:ec:64:72:f0:1a:db:64:01:cb: + 5d:27:c4:a1:c4:63 +-----BEGIN CERTIFICATE----- +MIIF9zCCBF+gAwIBAgIJAMstgJlaaVJdMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV +BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW +MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xODA4MjkxNDIzMTZaFw0zNzEwMjgx +NDIzMTZaMGIxCzAJBgNVBAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEj +MCEGA1UECgwaUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24xFTATBgNVBAMMDGZh +a2Vob3N0bmFtZTCCAaIwDQYJKoZIhvcNAQEBBQADggGPADCCAYoCggGBAMYoDXNk +ON40+HYn1YdVdcxR/yHk3j9scIzcJshpnV1KGQKB+YVaY62u4SKGBt9FhpJZkDAK +ZUBAaOaC9R+A4iCwV8zzI77MMHI+HNrte9Y9l8R7uY1EV7lmVie1t4z++5YIzhXP +AOr+aoVMXE1GbAieoTuHHRad8KhBZ703jWEnKXUrQ6MANv5KOWknT9QnreGwH52j +AaNpoG8nwd25BxGo2DyzoVzIknwyOyzW5EwUraP1CuaBsDrVe4g8LHNzcnnuZbwU +3YqxHwPYc6F72exMaXGn47kK51/9EFYHjjfFyjJ27BFc4VN0QBTjkxLi0oY/aHN7 +O7EwVCcsoBFdVbj+AXFmTuRTxPB5lRLosBg99gmeV+15vrPfssBu2gO/MpQoEQN/ ++eahrsK4rFI8LkcgqEtuNT/qS4+CVJUR8k8la7ZtkkMCQUvzk63eA+zXQRfaUxTL +Qm3e1/5NpCMS8VmGko3woB1iPd5WA5/MgFtDKOi0cmMV70mSrirtGgj6dwIDAQAB +o4IBwzCCAb8wFwYDVR0RBBAwDoIMZmFrZWhvc3RuYW1lMA4GA1UdDwEB/wQEAwIF +oDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwDAYDVR0TAQH/BAIwADAd +BgNVHQ4EFgQUHHAUsCDdCHakO1ZZ+l80+DZm6FYwfQYDVR0jBHYwdIAU8+yUjvKO +MMSOaMK/jmoZwMGfdmWhUaRPME0xCzAJBgNVBAYTAlhZMSYwJAYDVQQKDB1QeXRo +b24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEWMBQGA1UEAwwNb3VyLWNhLXNlcnZl +coIJAMstgJlaaVJbMIGDBggrBgEFBQcBAQR3MHUwPAYIKwYBBQUHMAKGMGh0dHA6 +Ly90ZXN0Y2EucHl0aG9udGVzdC5uZXQvdGVzdGNhL3B5Y2FjZXJ0LmNlcjA1Bggr +BgEFBQcwAYYpaHR0cDovL3Rlc3RjYS5weXRob250ZXN0Lm5ldC90ZXN0Y2Evb2Nz +cC8wQwYDVR0fBDwwOjA4oDagNIYyaHR0cDovL3Rlc3RjYS5weXRob250ZXN0Lm5l +dC90ZXN0Y2EvcmV2b2NhdGlvbi5jcmwwDQYJKoZIhvcNAQELBQADggGBAHUU5WhF +je1s8SceDvM1rg5gwWU2YrgHeOG5jXpQcK8GydTuUO/SdrKitsvcphi1PdL36w7s +t5XNLrE2b6ifuE3/zorEjmI3MoA+BUpNOYdpCQDoQGTSnfkfn6tnH/nGhLp+F2yL +jQju+4rXzQYlcp9OGsJx4RvPotccBRKV1klL6ZWVic9oGEaj6g2djsocKFVJa8BL +WPVCuQrsDm4hpP9gwBtuQHLQpcW1206HZzoxcMsyhHCp4v/g8tvNA7SFRdMHzA/H +SdjCF+tz90rA2YxZ78AKzhMLhMmqDREUtOV0quwY3l8mGJhKdvB/zebEtVgDA/UQ +AV2PY4i6Zde0f1oaUQ7t5Wj6GANyFaHsJx/qrCRGGG7xl9tK9NahkaCMsC++hztE +sI0qiYVf8tnjLmayiJgELJYyOJkZqYP9lAzdY9QbYJ1DmDWstCM43n+FUlegN9+l +z75ULDxQJyvUVKmdo9Sls8DqPQ7icGv7y6VWBexkcvAa22QBy10nxKHEYw== +-----END CERTIFICATE----- diff --git a/Lib/test/certdata/keycertecc.pem b/Lib/test/certdata/keycertecc.pem new file mode 100644 index 0000000000..bd10992189 --- /dev/null +++ b/Lib/test/certdata/keycertecc.pem @@ -0,0 +1,103 @@ +-----BEGIN PRIVATE KEY----- +MIG2AgEAMBAGByqGSM49AgEGBSuBBAAiBIGeMIGbAgEBBDDRUbCeT3hMph4Y/ahL +1sy9Qfy4DYotuAP06UetzG6syv+EoQ02kX3xvazqwiJDrEyhZANiAAQef97STEPn +4Nk6C153VEx24MNkJUcmLe771u6lr3Q8Em3J/YPaA1i9Ys7KZA3WvoKBPoWaaikn +4yLQbd/6YE6AAjMuaThlR1/cqH5QnmS3DXHUjmxnLjWy/dZl0CJG1qo= +-----END PRIVATE KEY----- +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + cb:2d:80:99:5a:69:52:5e + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=XY, O=Python Software Foundation CA, CN=our-ca-server + Validity + Not Before: Aug 29 14:23:16 2018 GMT + Not After : Oct 28 14:23:16 2037 GMT + Subject: C=XY, L=Castle Anthrax, O=Python Software Foundation, CN=localhost-ecc + Subject Public Key Info: + Public Key Algorithm: id-ecPublicKey + Public-Key: (384 bit) + pub: + 04:1e:7f:de:d2:4c:43:e7:e0:d9:3a:0b:5e:77:54: + 4c:76:e0:c3:64:25:47:26:2d:ee:fb:d6:ee:a5:af: + 74:3c:12:6d:c9:fd:83:da:03:58:bd:62:ce:ca:64: + 0d:d6:be:82:81:3e:85:9a:6a:29:27:e3:22:d0:6d: + df:fa:60:4e:80:02:33:2e:69:38:65:47:5f:dc:a8: + 7e:50:9e:64:b7:0d:71:d4:8e:6c:67:2e:35:b2:fd: + d6:65:d0:22:46:d6:aa + ASN1 OID: secp384r1 + NIST CURVE: P-384 + X509v3 extensions: + X509v3 Subject Alternative Name: + DNS:localhost-ecc + X509v3 Key Usage: critical + Digital Signature, Key Encipherment + X509v3 Extended Key Usage: + TLS Web Server Authentication, TLS Web Client Authentication + X509v3 Basic Constraints: critical + CA:FALSE + X509v3 Subject Key Identifier: + 45:ED:32:14:6D:51:A2:3B:B0:80:55:E0:A6:9B:74:4C:A5:56:88:B1 + X509v3 Authority Key Identifier: + keyid:F3:EC:94:8E:F2:8E:30:C4:8E:68:C2:BF:8E:6A:19:C0:C1:9F:76:65 + DirName:/C=XY/O=Python Software Foundation CA/CN=our-ca-server + serial:CB:2D:80:99:5A:69:52:5B + Authority Information Access: + CA Issuers - URI:http://testca.pythontest.net/testca/pycacert.cer + OCSP - URI:http://testca.pythontest.net/testca/ocsp/ + X509v3 CRL Distribution Points: + Full Name: + URI:http://testca.pythontest.net/testca/revocation.crl + Signature Algorithm: sha256WithRSAEncryption + Signature Value: + 07:e4:91:0b:d3:ed:4b:52:7f:50:68:c7:8d:80:48:9f:b7:4a: + 13:66:bf:9d:4c:2d:18:19:68:a0:da:3b:12:85:05:16:fa:8d: + 9c:58:c6:81:b3:96:ba:11:62:65:d3:76:f1:1c:ab:95:e4:d8: + 2a:e0:1f:7b:c5:20:2e:7c:8f:de:87:7a:2b:52:54:ca:d1:41: + b0:5e:20:72:df:44:00:4a:69:1a:ef:10:63:52:13:ed:49:02: + ee:dc:9d:f3:c8:ba:c4:01:81:5a:a9:1c:15:12:b6:21:de:44: + a5:fd:7e:f9:22:d1:3e:ee:22:dd:31:55:32:4e:41:68:27:c5: + 95:1b:7e:6b:18:74:f9:22:d6:b7:b9:31:72:51:a0:5a:2c:ff: + 62:76:e9:a0:55:8d:78:33:52:4a:58:b2:f4:4b:0c:43:82:2f: + a9:84:68:05:dd:11:47:70:24:fe:5c:92:fd:17:21:63:bb:fa: + 93:fa:54:54:05:72:48:ed:81:48:ab:95:fc:6d:a8:62:96:f9: + 3b:e2:71:18:05:3e:76:bb:df:95:17:7b:81:4b:1f:7f:e1:67: + 76:c4:07:cb:65:a7:f2:cf:e6:b4:fb:75:7c:ee:df:a1:f5:34: + 20:2b:48:fd:2e:49:ff:f3:a6:3b:00:49:6c:88:79:ed:9c:16: + 2a:04:72:e2:93:e4:7e:3f:2a:dd:30:47:9a:99:84:2a:b9:c4: + 40:31:a6:68:f3:20:d1:75:f1:1e:c8:18:64:5b:f8:4c:ce:9a: + 3c:57:2c:e3:63:64:29:0a:c2:b6:8e:20:01:55:9f:fe:10:ba: + 12:42:38:0a:9b:53:01:a5:b4:08:76:ec:e8:a6:fc:69:2c:f7: + 7f:5e:0f:44:07:55:e1:7c:2e:58:e5:d6:fc:6f:c2:4d:83:65: + bd:f3:32:e3:14:48:22:8d:80:18:ea:44:f8:24:79:ff:ff:c6: + 04:c2:e9:90:34:40:d6:59:3f:59:1e:4a:9a:58:60:ce:ab:f9: + 76:0e:ef:f7:05:17 +-----BEGIN CERTIFICATE----- +MIIEyzCCAzOgAwIBAgIJAMstgJlaaVJeMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV +BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW +MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xODA4MjkxNDIzMTZaFw0zNzEwMjgx +NDIzMTZaMGMxCzAJBgNVBAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEj +MCEGA1UECgwaUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24xFjAUBgNVBAMMDWxv +Y2FsaG9zdC1lY2MwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAAQef97STEPn4Nk6C153 +VEx24MNkJUcmLe771u6lr3Q8Em3J/YPaA1i9Ys7KZA3WvoKBPoWaaikn4yLQbd/6 +YE6AAjMuaThlR1/cqH5QnmS3DXHUjmxnLjWy/dZl0CJG1qqjggHEMIIBwDAYBgNV +HREEETAPgg1sb2NhbGhvc3QtZWNjMA4GA1UdDwEB/wQEAwIFoDAdBgNVHSUEFjAU +BggrBgEFBQcDAQYIKwYBBQUHAwIwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQURe0y +FG1RojuwgFXgppt0TKVWiLEwfQYDVR0jBHYwdIAU8+yUjvKOMMSOaMK/jmoZwMGf +dmWhUaRPME0xCzAJBgNVBAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUg +Rm91bmRhdGlvbiBDQTEWMBQGA1UEAwwNb3VyLWNhLXNlcnZlcoIJAMstgJlaaVJb +MIGDBggrBgEFBQcBAQR3MHUwPAYIKwYBBQUHMAKGMGh0dHA6Ly90ZXN0Y2EucHl0 +aG9udGVzdC5uZXQvdGVzdGNhL3B5Y2FjZXJ0LmNlcjA1BggrBgEFBQcwAYYpaHR0 +cDovL3Rlc3RjYS5weXRob250ZXN0Lm5ldC90ZXN0Y2Evb2NzcC8wQwYDVR0fBDww +OjA4oDagNIYyaHR0cDovL3Rlc3RjYS5weXRob250ZXN0Lm5ldC90ZXN0Y2EvcmV2 +b2NhdGlvbi5jcmwwDQYJKoZIhvcNAQELBQADggGBAAfkkQvT7UtSf1Box42ASJ+3 +ShNmv51MLRgZaKDaOxKFBRb6jZxYxoGzlroRYmXTdvEcq5Xk2CrgH3vFIC58j96H +eitSVMrRQbBeIHLfRABKaRrvEGNSE+1JAu7cnfPIusQBgVqpHBUStiHeRKX9fvki +0T7uIt0xVTJOQWgnxZUbfmsYdPki1re5MXJRoFos/2J26aBVjXgzUkpYsvRLDEOC +L6mEaAXdEUdwJP5ckv0XIWO7+pP6VFQFckjtgUirlfxtqGKW+TvicRgFPna735UX +e4FLH3/hZ3bEB8tlp/LP5rT7dXzu36H1NCArSP0uSf/zpjsASWyIee2cFioEcuKT +5H4/Kt0wR5qZhCq5xEAxpmjzINF18R7IGGRb+EzOmjxXLONjZCkKwraOIAFVn/4Q +uhJCOAqbUwGltAh27Oim/Gks939eD0QHVeF8Lljl1vxvwk2DZb3zMuMUSCKNgBjq +RPgkef//xgTC6ZA0QNZZP1keSppYYM6r+XYO7/cFFw== +-----END CERTIFICATE----- diff --git a/Lib/test/certdata/leaf-missing-aki.ca.pem b/Lib/test/certdata/leaf-missing-aki.ca.pem new file mode 100644 index 0000000000..36b202ae02 --- /dev/null +++ b/Lib/test/certdata/leaf-missing-aki.ca.pem @@ -0,0 +1,13 @@ +# Taken from x509-limbo's `rfc5280::aki::leaf-missing-aki` testcase. +# See: https://x509-limbo.com/testcases/rfc5280/#rfc5280akileaf-missing-aki +-----BEGIN CERTIFICATE----- +MIIBkDCCATWgAwIBAgIUGjIb/aYm9u9fBh2o4GAYRJwk5XIwCgYIKoZIzj0EAwIw +GjEYMBYGA1UEAwwPeDUwOS1saW1iby1yb290MCAXDTcwMDEwMTAwMDAwMVoYDzI5 +NjkwNTAzMDAwMDAxWjAaMRgwFgYDVQQDDA94NTA5LWxpbWJvLXJvb3QwWTATBgcq +hkjOPQIBBggqhkjOPQMBBwNCAARUzBhjMOkO911U65Fvs4YmL1YPNj63P9Fa+g9U +KrUqiIy8WjaDXdIe8g8Zj0TalpbU1gYCs3atteMxgIp6qxwHo1cwVTAPBgNVHRMB +Af8EBTADAQH/MAsGA1UdDwQEAwICBDAWBgNVHREEDzANggtleGFtcGxlLmNvbTAd +BgNVHQ4EFgQUcv1fyqgezMGzmo+lhmUkdUuAbIowCgYIKoZIzj0EAwIDSQAwRgIh +AIOErPSRlWpnyMub9UgtPF/lSzdvnD4Q8KjLQppHx6oPAiEA373p4L/HvUbs0xg8 +6/pLyn0RT02toKKJcMV3ChohLtM= +-----END CERTIFICATE----- diff --git a/Lib/test/certdata/leaf-missing-aki.keycert.pem b/Lib/test/certdata/leaf-missing-aki.keycert.pem new file mode 100644 index 0000000000..0fd2ab39bf --- /dev/null +++ b/Lib/test/certdata/leaf-missing-aki.keycert.pem @@ -0,0 +1,18 @@ +# Taken from x509-limbo's `rfc5280::aki::leaf-missing-aki` testcase. +# See: https://x509-limbo.com/testcases/rfc5280/#rfc5280akileaf-missing-aki +-----BEGIN EC PRIVATE KEY----- +MHcCAQEEIF5Re+/FP3rg+7c1odKEQPXhb9V65kXnlZIWHDG9gKrLoAoGCCqGSM49 +AwEHoUQDQgAE1WAQMdC7ims7T9lpK9uzaCuKqHb/oNMbGjh1f10pOHv3Z+oAvsqF +Sv3hGzreu69YLy01afA6sUCf1AA/95dKkg== +-----END EC PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +MIIBjjCCATWgAwIBAgIUVlBgclml+OXlrWzZfcgYCiNm96UwCgYIKoZIzj0EAwIw +GjEYMBYGA1UEAwwPeDUwOS1saW1iby1yb290MCAXDTcwMDEwMTAwMDAwMVoYDzI5 +NjkwNTAzMDAwMDAxWjAWMRQwEgYDVQQDDAtleGFtcGxlLmNvbTBZMBMGByqGSM49 +AgEGCCqGSM49AwEHA0IABNVgEDHQu4prO0/ZaSvbs2griqh2/6DTGxo4dX9dKTh7 +92fqAL7KhUr94Rs63ruvWC8tNWnwOrFAn9QAP/eXSpKjWzBZMB0GA1UdDgQWBBS3 +yYRQQwo3syjGVQ8Yw7/XRZHbpzALBgNVHQ8EBAMCB4AwEwYDVR0lBAwwCgYIKwYB +BQUHAwEwFgYDVR0RBA8wDYILZXhhbXBsZS5jb20wCgYIKoZIzj0EAwIDRwAwRAIg +BVq7lw4Y5MPEyisPhowMWd4KnERupdM5qeImDO+dD7ICIE/ksd6Wz1b8rMAfllNV +yiYst9lfwTd2SkFgdDNUDFud +-----END CERTIFICATE----- diff --git a/Lib/test/certdata/make_ssl_certs.py b/Lib/test/certdata/make_ssl_certs.py new file mode 100644 index 0000000000..ed2037c1fd --- /dev/null +++ b/Lib/test/certdata/make_ssl_certs.py @@ -0,0 +1,315 @@ +"""Make the custom certificate and private key files used by test_ssl +and friends.""" + +import os +import pprint +import shutil +import tempfile +from subprocess import * + +startdate = "20180829142316Z" +enddate = "20371028142316Z" + +req_template = """ + [ default ] + base_url = http://testca.pythontest.net/testca + + [req] + distinguished_name = req_distinguished_name + prompt = no + + [req_distinguished_name] + C = XY + L = Castle Anthrax + O = Python Software Foundation + CN = {hostname} + + [req_x509_extensions_nosan] + + [req_x509_extensions_simple] + subjectAltName = @san + + [req_x509_extensions_full] + subjectAltName = @san + keyUsage = critical,keyEncipherment,digitalSignature + extendedKeyUsage = serverAuth,clientAuth + basicConstraints = critical,CA:false + subjectKeyIdentifier = hash + authorityKeyIdentifier = keyid:always,issuer:always + authorityInfoAccess = @issuer_ocsp_info + crlDistributionPoints = @crl_info + + [ issuer_ocsp_info ] + caIssuers;URI.0 = $base_url/pycacert.cer + OCSP;URI.0 = $base_url/ocsp/ + + [ crl_info ] + URI.0 = $base_url/revocation.crl + + [san] + DNS.1 = {hostname} + {extra_san} + + [dir_sect] + C = XY + L = Castle Anthrax + O = Python Software Foundation + CN = dirname example + + [princ_name] + realm = EXP:0, GeneralString:KERBEROS.REALM + principal_name = EXP:1, SEQUENCE:principal_seq + + [principal_seq] + name_type = EXP:0, INTEGER:1 + name_string = EXP:1, SEQUENCE:principals + + [principals] + princ1 = GeneralString:username + + [ ca ] + default_ca = CA_default + + [ CA_default ] + dir = cadir + database = $dir/index.txt + crlnumber = $dir/crl.txt + default_md = sha256 + startdate = {startdate} + default_startdate = {startdate} + enddate = {enddate} + default_enddate = {enddate} + default_days = 7000 + default_crl_days = 7000 + certificate = pycacert.pem + private_key = pycakey.pem + serial = $dir/serial + RANDFILE = $dir/.rand + policy = policy_match + + [ policy_match ] + countryName = match + stateOrProvinceName = optional + organizationName = match + organizationalUnitName = optional + commonName = supplied + emailAddress = optional + + [ policy_anything ] + countryName = optional + stateOrProvinceName = optional + localityName = optional + organizationName = optional + organizationalUnitName = optional + commonName = supplied + emailAddress = optional + + + [ v3_ca ] + + subjectKeyIdentifier=hash + authorityKeyIdentifier=keyid:always,issuer + basicConstraints = critical, CA:true + keyUsage = critical, digitalSignature, keyCertSign, cRLSign + + """ + +here = os.path.abspath(os.path.dirname(__file__)) + + +def make_cert_key(hostname, sign=False, extra_san='', + ext='req_x509_extensions_full', key='rsa:3072'): + print("creating cert for " + hostname) + tempnames = [] + for i in range(3): + with tempfile.NamedTemporaryFile(delete=False) as f: + tempnames.append(f.name) + req_file, cert_file, key_file = tempnames + try: + req = req_template.format( + hostname=hostname, + extra_san=extra_san, + startdate=startdate, + enddate=enddate + ) + with open(req_file, 'w') as f: + f.write(req) + args = ['req', '-new', '-nodes', '-days', '7000', + '-newkey', key, '-keyout', key_file, + '-extensions', ext, + '-config', req_file] + if sign: + with tempfile.NamedTemporaryFile(delete=False) as f: + tempnames.append(f.name) + reqfile = f.name + args += ['-out', reqfile ] + + else: + args += ['-x509', '-out', cert_file ] + check_call(['openssl'] + args) + + if sign: + args = [ + 'ca', + '-config', req_file, + '-extensions', ext, + '-out', cert_file, + '-outdir', 'cadir', + '-policy', 'policy_anything', + '-batch', '-infiles', reqfile + ] + check_call(['openssl'] + args) + + + with open(cert_file, 'r') as f: + cert = f.read() + with open(key_file, 'r') as f: + key = f.read() + return cert, key + finally: + for name in tempnames: + os.remove(name) + +TMP_CADIR = 'cadir' + +def unmake_ca(): + shutil.rmtree(TMP_CADIR) + +def make_ca(): + os.mkdir(TMP_CADIR) + with open(os.path.join('cadir','index.txt'),'a+') as f: + pass # empty file + with open(os.path.join('cadir','crl.txt'),'a+') as f: + f.write("00") + with open(os.path.join('cadir','index.txt.attr'),'w+') as f: + f.write('unique_subject = no') + # random start value for serial numbers + with open(os.path.join('cadir','serial'), 'w') as f: + f.write('CB2D80995A69525B\n') + + with tempfile.NamedTemporaryFile("w") as t: + req = req_template.format( + hostname='our-ca-server', + extra_san='', + startdate=startdate, + enddate=enddate + ) + t.write(req) + t.flush() + with tempfile.NamedTemporaryFile() as f: + args = ['req', '-config', t.name, '-new', + '-nodes', + '-newkey', 'rsa:3072', + '-keyout', 'pycakey.pem', + '-out', f.name, + '-subj', '/C=XY/L=Castle Anthrax/O=Python Software Foundation CA/CN=our-ca-server'] + check_call(['openssl'] + args) + args = ['ca', '-config', t.name, + '-out', 'pycacert.pem', '-batch', '-outdir', TMP_CADIR, + '-keyfile', 'pycakey.pem', + '-selfsign', '-extensions', 'v3_ca', '-infiles', f.name ] + check_call(['openssl'] + args) + args = ['ca', '-config', t.name, '-gencrl', '-out', 'revocation.crl'] + check_call(['openssl'] + args) + + # capath hashes depend on subject! + check_call([ + 'openssl', 'x509', '-in', 'pycacert.pem', '-out', 'capath/ceff1710.0' + ]) + shutil.copy('capath/ceff1710.0', 'capath/b1930218.0') + + +def print_cert(path): + import _ssl + pprint.pprint(_ssl._test_decode_cert(path)) + + +if __name__ == '__main__': + os.chdir(here) + cert, key = make_cert_key('localhost', ext='req_x509_extensions_simple') + with open('ssl_cert.pem', 'w') as f: + f.write(cert) + with open('ssl_key.pem', 'w') as f: + f.write(key) + print("password protecting ssl_key.pem in ssl_key.passwd.pem") + check_call(['openssl','pkey','-in','ssl_key.pem','-out','ssl_key.passwd.pem','-aes256','-passout','pass:somepass']) + check_call(['openssl','pkey','-in','ssl_key.pem','-out','keycert.passwd.pem','-aes256','-passout','pass:somepass']) + + with open('keycert.pem', 'w') as f: + f.write(key) + f.write(cert) + + with open('keycert.passwd.pem', 'a+') as f: + f.write(cert) + + # For certificate matching tests + make_ca() + cert, key = make_cert_key('fakehostname', ext='req_x509_extensions_simple') + with open('keycert2.pem', 'w') as f: + f.write(key) + f.write(cert) + + cert, key = make_cert_key('localhost', sign=True) + with open('keycert3.pem', 'w') as f: + f.write(key) + f.write(cert) + + check_call(['openssl', 'x509', '-outform', 'pem', '-in', 'keycert3.pem', '-out', 'cert3.pem']) + + cert, key = make_cert_key('fakehostname', sign=True) + with open('keycert4.pem', 'w') as f: + f.write(key) + f.write(cert) + + cert, key = make_cert_key( + 'localhost-ecc', sign=True, key='param:secp384r1.pem' + ) + with open('keycertecc.pem', 'w') as f: + f.write(key) + f.write(cert) + + extra_san = [ + 'otherName.1 = 1.2.3.4;UTF8:some other identifier', + 'otherName.2 = 1.3.6.1.5.2.2;SEQUENCE:princ_name', + 'email.1 = user@example.org', + 'DNS.2 = www.example.org', + # GEN_X400 + 'dirName.1 = dir_sect', + # GEN_EDIPARTY + 'URI.1 = https://www.python.org/', + 'IP.1 = 127.0.0.1', + 'IP.2 = ::1', + 'RID.1 = 1.2.3.4.5', + ] + + cert, key = make_cert_key('allsans', sign=True, extra_san='\n'.join(extra_san)) + with open('allsans.pem', 'w') as f: + f.write(key) + f.write(cert) + + extra_san = [ + # könig (king) + 'DNS.2 = xn--knig-5qa.idn.pythontest.net', + # königsgäßchen (king's alleyway) + 'DNS.3 = xn--knigsgsschen-lcb0w.idna2003.pythontest.net', + 'DNS.4 = xn--knigsgchen-b4a3dun.idna2008.pythontest.net', + # βόλοσ (marble) + 'DNS.5 = xn--nxasmq6b.idna2003.pythontest.net', + 'DNS.6 = xn--nxasmm1c.idna2008.pythontest.net', + ] + + # IDN SANS, signed + cert, key = make_cert_key('idnsans', sign=True, extra_san='\n'.join(extra_san)) + with open('idnsans.pem', 'w') as f: + f.write(key) + f.write(cert) + + cert, key = make_cert_key('nosan', sign=True, ext='req_x509_extensions_nosan') + with open('nosan.pem', 'w') as f: + f.write(key) + f.write(cert) + + unmake_ca() + print("update Lib/test/test_ssl.py and Lib/test/test_asyncio/utils.py") + print_cert('keycert.pem') + print_cert('keycert3.pem') diff --git a/Lib/test/certdata/nokia.pem b/Lib/test/certdata/nokia.pem new file mode 100644 index 0000000000..0d044df436 --- /dev/null +++ b/Lib/test/certdata/nokia.pem @@ -0,0 +1,31 @@ +# Certificate for projects.developer.nokia.com:443 (see issue 13034) +-----BEGIN CERTIFICATE----- +MIIFLDCCBBSgAwIBAgIQLubqdkCgdc7lAF9NfHlUmjANBgkqhkiG9w0BAQUFADCB +vDELMAkGA1UEBhMCVVMxFzAVBgNVBAoTDlZlcmlTaWduLCBJbmMuMR8wHQYDVQQL +ExZWZXJpU2lnbiBUcnVzdCBOZXR3b3JrMTswOQYDVQQLEzJUZXJtcyBvZiB1c2Ug +YXQgaHR0cHM6Ly93d3cudmVyaXNpZ24uY29tL3JwYSAoYykxMDE2MDQGA1UEAxMt +VmVyaVNpZ24gQ2xhc3MgMyBJbnRlcm5hdGlvbmFsIFNlcnZlciBDQSAtIEczMB4X +DTExMDkyMTAwMDAwMFoXDTEyMDkyMDIzNTk1OVowcTELMAkGA1UEBhMCRkkxDjAM +BgNVBAgTBUVzcG9vMQ4wDAYDVQQHFAVFc3BvbzEOMAwGA1UEChQFTm9raWExCzAJ +BgNVBAsUAkJJMSUwIwYDVQQDFBxwcm9qZWN0cy5kZXZlbG9wZXIubm9raWEuY29t +MIGfMA0GCSqGSIb3DQEBAQUAA4GNADCBiQKBgQCr92w1bpHYSYxUEx8N/8Iddda2 +lYi+aXNtQfV/l2Fw9Ykv3Ipw4nLeGTj18FFlAZgMdPRlgrzF/NNXGw/9l3/qKdow +CypkQf8lLaxb9Ze1E/KKmkRJa48QTOqvo6GqKuTI6HCeGlG1RxDb8YSKcQWLiytn +yj3Wp4MgRQO266xmMQIDAQABo4IB9jCCAfIwQQYDVR0RBDowOIIccHJvamVjdHMu +ZGV2ZWxvcGVyLm5va2lhLmNvbYIYcHJvamVjdHMuZm9ydW0ubm9raWEuY29tMAkG +A1UdEwQCMAAwCwYDVR0PBAQDAgWgMEEGA1UdHwQ6MDgwNqA0oDKGMGh0dHA6Ly9T +VlJJbnRsLUczLWNybC52ZXJpc2lnbi5jb20vU1ZSSW50bEczLmNybDBEBgNVHSAE +PTA7MDkGC2CGSAGG+EUBBxcDMCowKAYIKwYBBQUHAgEWHGh0dHBzOi8vd3d3LnZl +cmlzaWduLmNvbS9ycGEwKAYDVR0lBCEwHwYJYIZIAYb4QgQBBggrBgEFBQcDAQYI +KwYBBQUHAwIwcgYIKwYBBQUHAQEEZjBkMCQGCCsGAQUFBzABhhhodHRwOi8vb2Nz +cC52ZXJpc2lnbi5jb20wPAYIKwYBBQUHMAKGMGh0dHA6Ly9TVlJJbnRsLUczLWFp +YS52ZXJpc2lnbi5jb20vU1ZSSW50bEczLmNlcjBuBggrBgEFBQcBDARiMGChXqBc +MFowWDBWFglpbWFnZS9naWYwITAfMAcGBSsOAwIaBBRLa7kolgYMu9BSOJsprEsH +iyEFGDAmFiRodHRwOi8vbG9nby52ZXJpc2lnbi5jb20vdnNsb2dvMS5naWYwDQYJ +KoZIhvcNAQEFBQADggEBACQuPyIJqXwUyFRWw9x5yDXgMW4zYFopQYOw/ItRY522 +O5BsySTh56BWS6mQB07XVfxmYUGAvRQDA5QHpmY8jIlNwSmN3s8RKo+fAtiNRlcL +x/mWSfuMs3D/S6ev3D6+dpEMZtjrhOdctsarMKp8n/hPbwhAbg5hVjpkW5n8vz2y +0KxvvkA1AxpLwpVv7OlK17ttzIHw8bp9HTlHBU5s8bKz4a565V/a5HI0CSEv/+0y +ko4/ghTnZc1CkmUngKKeFMSah/mT/xAh8XnE2l1AazFa8UKuYki1e+ArHaGZc4ix +UYOtiRphwfuYQhRZ7qX9q2MMkCMI65XNK/SaFrAbbG0= +-----END CERTIFICATE----- diff --git a/Lib/test/certdata/nosan.pem b/Lib/test/certdata/nosan.pem new file mode 100644 index 0000000000..c6ff8ea31b --- /dev/null +++ b/Lib/test/certdata/nosan.pem @@ -0,0 +1,131 @@ +-----BEGIN PRIVATE KEY----- +MIIG/gIBADANBgkqhkiG9w0BAQEFAASCBugwggbkAgEAAoIBgQC99xEYPTwFN/ji +i0lm11ckEGhcxciSsIgTgior54CLgQy7JXllTYmAWFTTg2zNBvDMexGI0h+xtZ4q +1Renghgt33N3Y6CT3v/L7JkE1abQbFveKW/ydlxH0+jLlsENSWjySwC80+f9L3bX +TcD8T4Fu9Uty2Rg1a/Eyekng5RmfkmLNgxfnX5R5nWhh0Aia7h3Ax2zCALfxqZIm +fxwavEgHsW/yZi+T+eoJwe0i7a6LaUoLqsPV9ZhagziNDaappPHH42NW39WlRhx1 +UjtiRm2Jihnzxcfs+90zLXSp5pxo/cE9Ia4d8ieq3Rxd/XgjlF6FXXFJjwfL36Dw +ehy8m3PKKAuO+fyMgPPPMQb7oaRy/MBG0NayRreTwyKILS2zafIW/iKpgICbxrWJ +r/H1b3S6PBKYUE2uQs0/ZPnRjjh0VeNnue7JcRoNbe27I2d56KUBsVEPdokjU59v +NYi6Se+ViZXtUbM1u/I0kvDMprAiobwtJFYgcE86N1lFJjHSwDMCAwEAAQKCAYBb +lvnJBA0iPwBiyeFUElNTcg2/XST9hNu2/DU1AeM6X7gxqznCnAXFudD8Qgt9NvF2 +xYeIvjbFydk+sYs8Gj9qLqhPUdukMAqI2cRVTmWla/lHPhdZgbOwdf1x23es3k4Z +NAxg/pKFwhK8cCKyA+tWAjKkZwODDk42ljt0kUEvbLbye1hVGAJQOJKRRmo/uLrj +rcNELnCBtc5ffT2hrlHUU7qz1ozt/brXhYa+JnbXhKZMxcKyMD2KtmXXrFNEy99o +jXbrpDCos82bzQfPDo8IpCbVbEd2J00aFmrNjQWhZuXX5dXflrujW4J0nzeHrZ78 +rNAz2/YuZ543BTB3XbogeFuLC5RqBgAMmw2WJ96Oa/UG8nZNvEw54N5r6dhfXj6A +VlJFLVwlfBQdAdaM3P4uZ6WECrH3EerQa27qyUdRrcDaGPLt7wG9FmMivnW1KQsy +5ow/gM0CsxFj2xNoGw1S5jtclbgSy8HNJaBsNk4XMQ+ORABZdG1MTTE+GMSjD/EC +gcEA+6JYiZEo+QrvItIZYB6Go4suu/F8df1pEjJlxwp2GmObitRhtV6r9g9IySFv +5SL7ZxARr4aQxvM7fNp57p9ssmkBtY0ofMjJAxhvs4T37bAcGK/2xCegNSmbqh24 +FAWpRDMgE5PjtuWC5jTvSOYFeUxwI/cu0HxWdxJl2dPUSL1nI2jP+ok3pZobEQk9 +E2+MlHpKmU+s/lAkuQiP+AW9a4M+ZJNWxocJjmtwj4OjJXPm7GddA/5x622DxFe6 +4K2vAoHBAMFC0An25bwGoFrCV/96s45K4qZcZcJ660+aK3xXaq6/8KfiusJnWds2 +nc0B6jYjKs8A7yTAGXke6fmyVsoLosZiXsbpW2m16g8jL79Tc85O9oDNmDIGk1uT +tRLZc2BvmHmy/dNrdbT/EHC3FKNWQVqWc2sHhPeB6F3hIEXDSUO/GB0njMZNXrPJ +547RlhN0xCLb3vTzzGHwNmwfI81YjV/XI4vpJjq1YceN8Xyd1r5ZOFfU8woIACO3 +I4dvBQ1avQKBwQCLDs9wzohfAFzg2Exvos7y6AKemDgYmD8NcE5wbWaQ9MTLNsz8 +RuIu64lkpRbKAMf/z5CGeI3fdCFGwRGq/e06tu7b3rMmKmtzS3jHM08zyiPsvKlZ +AzD00BaXLy8/2VUOPFaYmxy3QSRShaRKm9sgik5agcocKuo5iTBB7V8eB5VMqyps +IJJg8MXOZ1WaPQXqM56wFKjcLXvtyT6OaNWh6Xh8ajQFKDDuxI8CsFNjaiaONBzi +DSX1XaL4ySab7T8CgcEAsI+7xP6+EDP1mDVpc8zD8kHUI6zSgwUNqiHtjKHIo3JU +CO2JNkZ5v158eGlBcshaOdheozKlkxR9KlSWGezbf2crs4pKq585AS9iVeeGK3vU +lQRAAaQkSEv/6AKl9/q8UKMIZnkMhploibGZt0f8WSiOtb+e6QjUI8CjXVj2vF// +RdN2N01EMflKBh7Qf2H0NuytGxkJJojxD4K7kMVQE7lXjmEpPgWsGUZC01jYcfrN +EOFKUWXRys9sNDVnZjX5AoHAFRyOC1BlmVEtcOsgzum4+JEDWvRnO1hP1tm68eTZ +ijB/XppDtVESpq3+1+gx2YOmzlUNEhKlcn6eHPWEJwdVxJ87Gdh03rIV/ZQUKe46 +3+j6l/5coN4WfCBloy4b+Tcj+ZTL4sKaLm33RoD2UEWS5mmItfZuoEFQB958h3JD +1Ka1tgsLnuYGjcrg+ALvbM5nQlefzPqPJh0C8UV3Ny/4Gd02YgHw7Yoe4m6OUqQv +hctFUL/gjIGg9PVqTWzVVKaI +-----END PRIVATE KEY----- +Certificate: + Data: + Version: 1 (0x0) + Serial Number: + cb:2d:80:99:5a:69:52:61 + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=XY, O=Python Software Foundation CA, CN=our-ca-server + Validity + Not Before: Aug 29 14:23:16 2018 GMT + Not After : Oct 28 14:23:16 2037 GMT + Subject: C=XY, L=Castle Anthrax, O=Python Software Foundation, CN=nosan + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (3072 bit) + Modulus: + 00:bd:f7:11:18:3d:3c:05:37:f8:e2:8b:49:66:d7: + 57:24:10:68:5c:c5:c8:92:b0:88:13:82:2a:2b:e7: + 80:8b:81:0c:bb:25:79:65:4d:89:80:58:54:d3:83: + 6c:cd:06:f0:cc:7b:11:88:d2:1f:b1:b5:9e:2a:d5: + 17:a7:82:18:2d:df:73:77:63:a0:93:de:ff:cb:ec: + 99:04:d5:a6:d0:6c:5b:de:29:6f:f2:76:5c:47:d3: + e8:cb:96:c1:0d:49:68:f2:4b:00:bc:d3:e7:fd:2f: + 76:d7:4d:c0:fc:4f:81:6e:f5:4b:72:d9:18:35:6b: + f1:32:7a:49:e0:e5:19:9f:92:62:cd:83:17:e7:5f: + 94:79:9d:68:61:d0:08:9a:ee:1d:c0:c7:6c:c2:00: + b7:f1:a9:92:26:7f:1c:1a:bc:48:07:b1:6f:f2:66: + 2f:93:f9:ea:09:c1:ed:22:ed:ae:8b:69:4a:0b:aa: + c3:d5:f5:98:5a:83:38:8d:0d:a6:a9:a4:f1:c7:e3: + 63:56:df:d5:a5:46:1c:75:52:3b:62:46:6d:89:8a: + 19:f3:c5:c7:ec:fb:dd:33:2d:74:a9:e6:9c:68:fd: + c1:3d:21:ae:1d:f2:27:aa:dd:1c:5d:fd:78:23:94: + 5e:85:5d:71:49:8f:07:cb:df:a0:f0:7a:1c:bc:9b: + 73:ca:28:0b:8e:f9:fc:8c:80:f3:cf:31:06:fb:a1: + a4:72:fc:c0:46:d0:d6:b2:46:b7:93:c3:22:88:2d: + 2d:b3:69:f2:16:fe:22:a9:80:80:9b:c6:b5:89:af: + f1:f5:6f:74:ba:3c:12:98:50:4d:ae:42:cd:3f:64: + f9:d1:8e:38:74:55:e3:67:b9:ee:c9:71:1a:0d:6d: + ed:bb:23:67:79:e8:a5:01:b1:51:0f:76:89:23:53: + 9f:6f:35:88:ba:49:ef:95:89:95:ed:51:b3:35:bb: + f2:34:92:f0:cc:a6:b0:22:a1:bc:2d:24:56:20:70: + 4f:3a:37:59:45:26:31:d2:c0:33 + Exponent: 65537 (0x10001) + Signature Algorithm: sha256WithRSAEncryption + Signature Value: + 7e:dd:64:64:92:6c:b9:41:ce:f3:e3:f8:e6:9f:c8:5b:32:39: + 8c:03:5b:5e:7e:b3:23:ca:6c:d1:99:2f:53:af:9d:3c:84:cd: + c6:ce:0a:ee:94:de:ff:a7:06:81:7e:e2:38:a5:05:39:58:22: + dc:13:83:53:e7:f8:16:cb:93:dc:cf:4b:e6:1b:9f:9e:71:ef: + ee:ba:ea:b6:68:5c:32:22:7e:54:4f:46:a6:0b:11:8f:ef:05: + 6e:d3:0b:d0:a8:be:95:23:a2:e4:e7:a8:a2:a4:7d:98:52:86: + a4:15:fb:74:7a:9a:89:23:43:20:26:3a:56:9e:a3:6e:54:02: + 76:4e:25:9c:a1:8c:03:99:e5:eb:a6:61:b4:9c:2a:b1:ed:eb: + 94:f9:14:aa:a4:c3:f0:f7:7a:03:a3:b1:f8:c0:83:79:ab:8a: + 93:7f:0a:95:08:50:ff:55:19:ac:28:a2:c8:9f:a6:77:72:a3: + da:37:a9:ff:f3:57:70:c8:65:d9:55:14:84:b4:b3:78:86:82: + da:84:2c:48:19:51:ec:9d:20:b1:4d:18:fb:82:9f:7b:a7:80: + 22:69:25:83:4d:bf:ac:31:64:f5:39:11:f1:ed:53:fb:67:ab: + 91:86:c5:4d:87:e8:6b:fe:9a:84:fe:6a:92:6b:62:c1:ae:d2: + f0:cb:06:6e:f3:50:f4:8d:6d:fa:7d:6a:1c:64:c3:98:91:da: + c9:8c:a9:79:e5:48:4c:a2:de:42:28:e8:0e:9f:52:6a:a4:e0: + c7:ac:11:9c:ba:5d:d6:84:93:56:28:f1:6d:83:aa:62:b2:b7: + 56:c6:64:d9:96:4e:97:ab:4e:8f:ba:f6:ab:b9:17:52:98:32: + 7f:b5:12:fa:39:d7:34:2a:f3:ed:40:90:6f:66:7b:b6:c1:9d: + b9:53:d0:e3:e9:69:8c:cf:7a:fd:08:0a:62:47:d4:ce:72:f7: + 6f:80:b4:1d:18:7a:ba:a2:a9:45:49:ef:9c:0b:99:89:03:ab: + 5f:7a:9d:c5:77:b7 +-----BEGIN CERTIFICATE----- +MIIEJDCCAowCCQDLLYCZWmlSYTANBgkqhkiG9w0BAQsFADBNMQswCQYDVQQGEwJY +WTEmMCQGA1UECgwdUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24gQ0ExFjAUBgNV +BAMMDW91ci1jYS1zZXJ2ZXIwHhcNMTgwODI5MTQyMzE2WhcNMzcxMDI4MTQyMzE2 +WjBbMQswCQYDVQQGEwJYWTEXMBUGA1UEBwwOQ2FzdGxlIEFudGhyYXgxIzAhBgNV +BAoMGlB5dGhvbiBTb2Z0d2FyZSBGb3VuZGF0aW9uMQ4wDAYDVQQDDAVub3NhbjCC +AaIwDQYJKoZIhvcNAQEBBQADggGPADCCAYoCggGBAL33ERg9PAU3+OKLSWbXVyQQ +aFzFyJKwiBOCKivngIuBDLsleWVNiYBYVNODbM0G8Mx7EYjSH7G1nirVF6eCGC3f +c3djoJPe/8vsmQTVptBsW94pb/J2XEfT6MuWwQ1JaPJLALzT5/0vdtdNwPxPgW71 +S3LZGDVr8TJ6SeDlGZ+SYs2DF+dflHmdaGHQCJruHcDHbMIAt/GpkiZ/HBq8SAex +b/JmL5P56gnB7SLtrotpSguqw9X1mFqDOI0Npqmk8cfjY1bf1aVGHHVSO2JGbYmK +GfPFx+z73TMtdKnmnGj9wT0hrh3yJ6rdHF39eCOUXoVdcUmPB8vfoPB6HLybc8oo +C475/IyA888xBvuhpHL8wEbQ1rJGt5PDIogtLbNp8hb+IqmAgJvGtYmv8fVvdLo8 +EphQTa5CzT9k+dGOOHRV42e57slxGg1t7bsjZ3nopQGxUQ92iSNTn281iLpJ75WJ +le1RszW78jSS8MymsCKhvC0kViBwTzo3WUUmMdLAMwIDAQABMA0GCSqGSIb3DQEB +CwUAA4IBgQB+3WRkkmy5Qc7z4/jmn8hbMjmMA1tefrMjymzRmS9Tr508hM3Gzgru +lN7/pwaBfuI4pQU5WCLcE4NT5/gWy5Pcz0vmG5+ece/uuuq2aFwyIn5UT0amCxGP +7wVu0wvQqL6VI6Lk56iipH2YUoakFft0epqJI0MgJjpWnqNuVAJ2TiWcoYwDmeXr +pmG0nCqx7euU+RSqpMPw93oDo7H4wIN5q4qTfwqVCFD/VRmsKKLIn6Z3cqPaN6n/ +81dwyGXZVRSEtLN4hoLahCxIGVHsnSCxTRj7gp97p4AiaSWDTb+sMWT1ORHx7VP7 +Z6uRhsVNh+hr/pqE/mqSa2LBrtLwywZu81D0jW36fWocZMOYkdrJjKl55UhMot5C +KOgOn1JqpODHrBGcul3WhJNWKPFtg6pisrdWxmTZlk6Xq06PuvaruRdSmDJ/tRL6 +Odc0KvPtQJBvZnu2wZ25U9Dj6WmMz3r9CApiR9TOcvdvgLQdGHq6oqlFSe+cC5mJ +A6tfep3Fd7c= +-----END CERTIFICATE----- diff --git a/Lib/test/certdata/nullbytecert.pem b/Lib/test/certdata/nullbytecert.pem new file mode 100644 index 0000000000..447186c950 --- /dev/null +++ b/Lib/test/certdata/nullbytecert.pem @@ -0,0 +1,90 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: 0 (0x0) + Signature Algorithm: sha1WithRSAEncryption + Issuer: C=US, ST=Oregon, L=Beaverton, O=Python Software Foundation, OU=Python Core Development, CN=null.python.org\x00example.org/emailAddress=python-dev@python.org + Validity + Not Before: Aug 7 13:11:52 2013 GMT + Not After : Aug 7 13:12:52 2013 GMT + Subject: C=US, ST=Oregon, L=Beaverton, O=Python Software Foundation, OU=Python Core Development, CN=null.python.org\x00example.org/emailAddress=python-dev@python.org + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (2048 bit) + Modulus: + 00:b5:ea:ed:c9:fb:46:7d:6f:3b:76:80:dd:3a:f3: + 03:94:0b:a7:a6:db:ec:1d:df:ff:23:74:08:9d:97: + 16:3f:a3:a4:7b:3e:1b:0e:96:59:25:03:a7:26:e2: + 88:a9:cf:79:cd:f7:04:56:b0:ab:79:32:6e:59:c1: + 32:30:54:eb:58:a8:cb:91:f0:42:a5:64:27:cb:d4: + 56:31:88:52:ad:cf:bd:7f:f0:06:64:1f:cc:27:b8: + a3:8b:8c:f3:d8:29:1f:25:0b:f5:46:06:1b:ca:02: + 45:ad:7b:76:0a:9c:bf:bb:b9:ae:0d:16:ab:60:75: + ae:06:3e:9c:7c:31:dc:92:2f:29:1a:e0:4b:0c:91: + 90:6c:e9:37:c5:90:d7:2a:d7:97:15:a3:80:8f:5d: + 7b:49:8f:54:30:d4:97:2c:1c:5b:37:b5:ab:69:30: + 68:43:d3:33:78:4b:02:60:f5:3c:44:80:a1:8f:e7: + f0:0f:d1:5e:87:9e:46:cf:62:fc:f9:bf:0c:65:12: + f1:93:c8:35:79:3f:c8:ec:ec:47:f5:ef:be:44:d5: + ae:82:1e:2d:9a:9f:98:5a:67:65:e1:74:70:7c:cb: + d3:c2:ce:0e:45:49:27:dc:e3:2d:d4:fb:48:0e:2f: + 9e:77:b8:14:46:c0:c4:36:ca:02:ae:6a:91:8c:da: + 2f:85 + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Basic Constraints: critical + CA:FALSE + X509v3 Subject Key Identifier: + 88:5A:55:C0:52:FF:61:CD:52:A3:35:0F:EA:5A:9C:24:38:22:F7:5C + X509v3 Key Usage: + Digital Signature, Non Repudiation, Key Encipherment + X509v3 Subject Alternative Name: + ************************************************************* + WARNING: The values for DNS, email and URI are WRONG. OpenSSL + doesn't print the text after a NULL byte. + ************************************************************* + DNS:altnull.python.org, email:null@python.org, URI:http://null.python.org, IP Address:192.0.2.1, IP Address:2001:DB8:0:0:0:0:0:1 + Signature Algorithm: sha1WithRSAEncryption + ac:4f:45:ef:7d:49:a8:21:70:8e:88:59:3e:d4:36:42:70:f5: + a3:bd:8b:d7:a8:d0:58:f6:31:4a:b1:a4:a6:dd:6f:d9:e8:44: + 3c:b6:0a:71:d6:7f:b1:08:61:9d:60:ce:75:cf:77:0c:d2:37: + 86:02:8d:5e:5d:f9:0f:71:b4:16:a8:c1:3d:23:1c:f1:11:b3: + 56:6e:ca:d0:8d:34:94:e6:87:2a:99:f2:ae:ae:cc:c2:e8:86: + de:08:a8:7f:c5:05:fa:6f:81:a7:82:e6:d0:53:9d:34:f4:ac: + 3e:40:fe:89:57:7a:29:a4:91:7e:0b:c6:51:31:e5:10:2f:a4: + 60:76:cd:95:51:1a:be:8b:a1:b0:fd:ad:52:bd:d7:1b:87:60: + d2:31:c7:17:c4:18:4f:2d:08:25:a3:a7:4f:b7:92:ca:e2:f5: + 25:f1:54:75:81:9d:b3:3d:61:a2:f7:da:ed:e1:c6:6f:2c:60: + 1f:d8:6f:c5:92:05:ab:c9:09:62:49:a9:14:ad:55:11:cc:d6: + 4a:19:94:99:97:37:1d:81:5f:8b:cf:a3:a8:96:44:51:08:3d: + 0b:05:65:12:eb:b6:70:80:88:48:72:4f:c6:c2:da:cf:cd:8e: + 5b:ba:97:2f:60:b4:96:56:49:5e:3a:43:76:63:04:be:2a:f6: + c1:ca:a9:94 +-----BEGIN CERTIFICATE----- +MIIE2DCCA8CgAwIBAgIBADANBgkqhkiG9w0BAQUFADCBxTELMAkGA1UEBhMCVVMx +DzANBgNVBAgMBk9yZWdvbjESMBAGA1UEBwwJQmVhdmVydG9uMSMwIQYDVQQKDBpQ +eXRob24gU29mdHdhcmUgRm91bmRhdGlvbjEgMB4GA1UECwwXUHl0aG9uIENvcmUg +RGV2ZWxvcG1lbnQxJDAiBgNVBAMMG251bGwucHl0aG9uLm9yZwBleGFtcGxlLm9y +ZzEkMCIGCSqGSIb3DQEJARYVcHl0aG9uLWRldkBweXRob24ub3JnMB4XDTEzMDgw +NzEzMTE1MloXDTEzMDgwNzEzMTI1MlowgcUxCzAJBgNVBAYTAlVTMQ8wDQYDVQQI +DAZPcmVnb24xEjAQBgNVBAcMCUJlYXZlcnRvbjEjMCEGA1UECgwaUHl0aG9uIFNv +ZnR3YXJlIEZvdW5kYXRpb24xIDAeBgNVBAsMF1B5dGhvbiBDb3JlIERldmVsb3Bt +ZW50MSQwIgYDVQQDDBtudWxsLnB5dGhvbi5vcmcAZXhhbXBsZS5vcmcxJDAiBgkq +hkiG9w0BCQEWFXB5dGhvbi1kZXZAcHl0aG9uLm9yZzCCASIwDQYJKoZIhvcNAQEB +BQADggEPADCCAQoCggEBALXq7cn7Rn1vO3aA3TrzA5QLp6bb7B3f/yN0CJ2XFj+j +pHs+Gw6WWSUDpybiiKnPec33BFawq3kyblnBMjBU61ioy5HwQqVkJ8vUVjGIUq3P +vX/wBmQfzCe4o4uM89gpHyUL9UYGG8oCRa17dgqcv7u5rg0Wq2B1rgY+nHwx3JIv +KRrgSwyRkGzpN8WQ1yrXlxWjgI9de0mPVDDUlywcWze1q2kwaEPTM3hLAmD1PESA +oY/n8A/RXoeeRs9i/Pm/DGUS8ZPINXk/yOzsR/XvvkTVroIeLZqfmFpnZeF0cHzL +08LODkVJJ9zjLdT7SA4vnne4FEbAxDbKAq5qkYzaL4UCAwEAAaOB0DCBzTAMBgNV +HRMBAf8EAjAAMB0GA1UdDgQWBBSIWlXAUv9hzVKjNQ/qWpwkOCL3XDALBgNVHQ8E +BAMCBeAwgZAGA1UdEQSBiDCBhYIeYWx0bnVsbC5weXRob24ub3JnAGV4YW1wbGUu +Y29tgSBudWxsQHB5dGhvbi5vcmcAdXNlckBleGFtcGxlLm9yZ4YpaHR0cDovL251 +bGwucHl0aG9uLm9yZwBodHRwOi8vZXhhbXBsZS5vcmeHBMAAAgGHECABDbgAAAAA +AAAAAAAAAAEwDQYJKoZIhvcNAQEFBQADggEBAKxPRe99SaghcI6IWT7UNkJw9aO9 +i9eo0Fj2MUqxpKbdb9noRDy2CnHWf7EIYZ1gznXPdwzSN4YCjV5d+Q9xtBaowT0j +HPERs1ZuytCNNJTmhyqZ8q6uzMLoht4IqH/FBfpvgaeC5tBTnTT0rD5A/olXeimk +kX4LxlEx5RAvpGB2zZVRGr6LobD9rVK91xuHYNIxxxfEGE8tCCWjp0+3ksri9SXx +VHWBnbM9YaL32u3hxm8sYB/Yb8WSBavJCWJJqRStVRHM1koZlJmXNx2BX4vPo6iW +RFEIPQsFZRLrtnCAiEhyT8bC2s/Njlu6ly9gtJZWSV46Q3ZjBL4q9sHKqZQ= +-----END CERTIFICATE----- diff --git a/Lib/test/certdata/nullcert.pem b/Lib/test/certdata/nullcert.pem new file mode 100644 index 0000000000..e69de29bb2 diff --git a/Lib/test/certdata/pycacert.pem b/Lib/test/certdata/pycacert.pem new file mode 100644 index 0000000000..0a48bf7d23 --- /dev/null +++ b/Lib/test/certdata/pycacert.pem @@ -0,0 +1,102 @@ +Certificate: + Data: + Version: 3 (0x2) + Serial Number: + cb:2d:80:99:5a:69:52:5b + Signature Algorithm: sha256WithRSAEncryption + Issuer: C=XY, O=Python Software Foundation CA, CN=our-ca-server + Validity + Not Before: Aug 29 14:23:16 2018 GMT + Not After : Oct 28 14:23:16 2037 GMT + Subject: C=XY, O=Python Software Foundation CA, CN=our-ca-server + Subject Public Key Info: + Public Key Algorithm: rsaEncryption + Public-Key: (3072 bit) + Modulus: + 00:d0:a0:9b:b1:b9:3b:79:ee:31:2f:b8:51:1c:01: + 75:ed:09:59:4c:ac:c8:bf:6a:0a:f8:7a:08:f0:25: + e3:25:7b:6a:f8:c4:d8:aa:a0:60:07:25:b0:cf:73: + 71:05:52:68:bf:06:93:ae:10:61:96:bc:b3:69:81: + 1a:7d:19:fc:20:86:8f:9a:68:1b:ed:cd:e2:6c:61: + 67:c7:4e:55:ea:43:06:21:1b:e9:b9:be:84:5f:c2: + da:eb:89:88:e0:42:a6:45:49:2a:d0:b9:6b:8c:93: + c9:44:3b:ca:fc:3c:94:7f:dd:70:c5:ad:d8:4b:3b: + dc:1e:f8:55:4b:b5:27:86:f8:df:b0:83:cf:f7:16: + 37:bf:13:17:34:d4:c9:55:ed:6b:16:c9:7f:ad:20: + 4e:f0:1e:d9:1b:bf:8d:ba:cd:ca:96:ef:1e:69:cb: + 51:66:61:48:0d:e8:79:a3:59:61:d4:10:8d:e0:0d: + 3b:0b:85:b6:d5:85:12:dd:a5:07:c5:4b:fa:23:fa: + 54:39:f7:97:0b:c9:44:47:1e:56:77:3c:3c:13:01: + 0b:6d:77:d6:14:ba:44:f4:53:35:56:d9:3d:b9:3e: + 35:5f:db:33:9a:4f:5a:b9:38:44:c9:32:49:fe:66: + f6:1f:1a:97:aa:ca:4c:4a:f6:b8:5e:40:7f:fb:0b: + 1d:f8:5b:a1:dc:f8:c0:2e:d0:6d:49:f5:d2:46:d4: + 90:57:fe:92:81:34:ae:2d:38:bb:a8:17:0c:e1:e5: + 3f:e2:f7:26:05:54:50:f5:64:b3:1c:6e:44:ff:6f: + a9:b4:03:96:e9:0e:c2:88:d8:72:52:90:99:c6:41: + 0f:46:90:59:b8:3e:6f:d2:e2:9e:1d:36:82:95:d3: + 58:8a:12:f3:e2:d8:0d:20:51:23:f0:90:2d:9a:3e: + 7d:26:86:b2:a7:d7:f9:28:60:03:e3:77:c7:88:04: + c9:fe:89:77:70:10:4f:c3:a0:8a:3b:f4:ab:42:7b: + e3:14:92:d8:ae:16:d6:a1:de:7d + Exponent: 65537 (0x10001) + X509v3 extensions: + X509v3 Subject Key Identifier: + F3:EC:94:8E:F2:8E:30:C4:8E:68:C2:BF:8E:6A:19:C0:C1:9F:76:65 + X509v3 Authority Key Identifier: + F3:EC:94:8E:F2:8E:30:C4:8E:68:C2:BF:8E:6A:19:C0:C1:9F:76:65 + X509v3 Basic Constraints: critical + CA:TRUE + X509v3 Key Usage: critical + Digital Signature, Certificate Sign, CRL Sign + Signature Algorithm: sha256WithRSAEncryption + Signature Value: + 8b:00:54:72:b3:8d:eb:f3:af:34:9f:d6:60:ea:de:84:3f:8c: + 04:8f:19:a6:be:02:67:c4:63:c5:74:e3:47:37:59:83:94:06: + f1:45:19:e8:07:2f:d6:4e:4b:4f:a8:3d:c7:07:07:27:92:f4: + 7e:73:4f:8b:32:19:94:46:7a:25:c4:d9:c4:27:b0:11:63:3a: + 60:8b:85:e1:73:4f:34:3b:6b:a4:34:8c:49:8e:cd:cf:4f:b2: + 65:27:41:19:b0:fc:80:31:78:f2:73:6a:9b:7d:71:34:50:fc: + 78:a8:da:05:b4:9c:5b:3a:99:7a:6b:5d:ef:3b:d3:e9:3b:33: + 01:12:65:cf:5e:07:d8:19:af:d5:53:ea:f0:10:ac:c4:b6:26: + 3c:34:2e:74:ee:64:dd:1d:36:75:89:44:00:b0:0d:fd:2f:b3: + 01:cc:1a:8b:02:cd:6c:e8:80:82:ca:bf:82:d7:00:9d:d8:36: + 15:d2:07:37:fc:6c:73:1d:da:a8:1c:e8:20:8e:32:7a:fe:6d: + 27:16:e4:58:6c:eb:3e:f0:fe:24:52:29:71:b8:96:7b:53:4b: + 45:20:55:40:5e:86:1b:ec:c9:46:91:92:ee:ac:93:65:91:2e: + 94:b6:b6:ac:e8:a3:34:89:a4:1a:12:0d:4d:44:a5:52:ed:8b: + 91:ee:2f:a6:af:a4:95:25:f9:ce:c7:5b:a7:00:d3:93:ca:b4: + 3c:5d:4d:f7:b1:3c:cc:6d:b4:45:be:82:ed:18:90:c8:86:d1: + 75:51:50:04:4c:e8:4f:d2:d6:50:aa:75:e7:5e:ff:a1:7b:27: + 19:1c:6b:49:2c:6c:4d:6b:63:cc:3b:73:00:f3:99:26:d0:82: + 87:d3:a9:36:9c:b4:3d:b9:48:68:a8:92:f0:27:8e:0c:cd:15: + 76:42:84:80:c9:3f:a3:7e:b4:dd:d7:f8:ac:76:ba:60:97:3c: + 5a:1f:4e:de:71:ee:09:39:10:dd:31:d5:68:57:5d:1a:e5:42: + ba:0e:68:e6:45:d3 +-----BEGIN CERTIFICATE----- +MIIEgDCCAuigAwIBAgIJAMstgJlaaVJbMA0GCSqGSIb3DQEBCwUAME0xCzAJBgNV +BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW +MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xODA4MjkxNDIzMTZaFw0zNzEwMjgx +NDIzMTZaME0xCzAJBgNVBAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUg +Rm91bmRhdGlvbiBDQTEWMBQGA1UEAwwNb3VyLWNhLXNlcnZlcjCCAaIwDQYJKoZI +hvcNAQEBBQADggGPADCCAYoCggGBANCgm7G5O3nuMS+4URwBde0JWUysyL9qCvh6 +CPAl4yV7avjE2KqgYAclsM9zcQVSaL8Gk64QYZa8s2mBGn0Z/CCGj5poG+3N4mxh +Z8dOVepDBiEb6bm+hF/C2uuJiOBCpkVJKtC5a4yTyUQ7yvw8lH/dcMWt2Es73B74 +VUu1J4b437CDz/cWN78TFzTUyVXtaxbJf60gTvAe2Ru/jbrNypbvHmnLUWZhSA3o +eaNZYdQQjeANOwuFttWFEt2lB8VL+iP6VDn3lwvJREceVnc8PBMBC2131hS6RPRT +NVbZPbk+NV/bM5pPWrk4RMkySf5m9h8al6rKTEr2uF5Af/sLHfhbodz4wC7QbUn1 +0kbUkFf+koE0ri04u6gXDOHlP+L3JgVUUPVksxxuRP9vqbQDlukOwojYclKQmcZB +D0aQWbg+b9Linh02gpXTWIoS8+LYDSBRI/CQLZo+fSaGsqfX+ShgA+N3x4gEyf6J +d3AQT8Ogijv0q0J74xSS2K4W1qHefQIDAQABo2MwYTAdBgNVHQ4EFgQU8+yUjvKO +MMSOaMK/jmoZwMGfdmUwHwYDVR0jBBgwFoAU8+yUjvKOMMSOaMK/jmoZwMGfdmUw +DwYDVR0TAQH/BAUwAwEB/zAOBgNVHQ8BAf8EBAMCAYYwDQYJKoZIhvcNAQELBQAD +ggGBAIsAVHKzjevzrzSf1mDq3oQ/jASPGaa+AmfEY8V040c3WYOUBvFFGegHL9ZO +S0+oPccHByeS9H5zT4syGZRGeiXE2cQnsBFjOmCLheFzTzQ7a6Q0jEmOzc9PsmUn +QRmw/IAxePJzapt9cTRQ/Hio2gW0nFs6mXprXe870+k7MwESZc9eB9gZr9VT6vAQ +rMS2Jjw0LnTuZN0dNnWJRACwDf0vswHMGosCzWzogILKv4LXAJ3YNhXSBzf8bHMd +2qgc6CCOMnr+bScW5Fhs6z7w/iRSKXG4lntTS0UgVUBehhvsyUaRku6sk2WRLpS2 +tqzoozSJpBoSDU1EpVLti5HuL6avpJUl+c7HW6cA05PKtDxdTfexPMxttEW+gu0Y +kMiG0XVRUARM6E/S1lCqdede/6F7Jxkca0ksbE1rY8w7cwDzmSbQgofTqTactD25 +SGiokvAnjgzNFXZChIDJP6N+tN3X+Kx2umCXPFofTt5x7gk5EN0x1WhXXRrlQroO +aOZF0w== +-----END CERTIFICATE----- diff --git a/Lib/test/certdata/pycakey.pem b/Lib/test/certdata/pycakey.pem new file mode 100644 index 0000000000..a6bf7356f4 --- /dev/null +++ b/Lib/test/certdata/pycakey.pem @@ -0,0 +1,40 @@ +-----BEGIN PRIVATE KEY----- +MIIG/gIBADANBgkqhkiG9w0BAQEFAASCBugwggbkAgEAAoIBgQDQoJuxuTt57jEv +uFEcAXXtCVlMrMi/agr4egjwJeMle2r4xNiqoGAHJbDPc3EFUmi/BpOuEGGWvLNp +gRp9Gfwgho+aaBvtzeJsYWfHTlXqQwYhG+m5voRfwtrriYjgQqZFSSrQuWuMk8lE +O8r8PJR/3XDFrdhLO9we+FVLtSeG+N+wg8/3Fje/Exc01MlV7WsWyX+tIE7wHtkb +v426zcqW7x5py1FmYUgN6HmjWWHUEI3gDTsLhbbVhRLdpQfFS/oj+lQ595cLyURH +HlZ3PDwTAQttd9YUukT0UzVW2T25PjVf2zOaT1q5OETJMkn+ZvYfGpeqykxK9rhe +QH/7Cx34W6Hc+MAu0G1J9dJG1JBX/pKBNK4tOLuoFwzh5T/i9yYFVFD1ZLMcbkT/ +b6m0A5bpDsKI2HJSkJnGQQ9GkFm4Pm/S4p4dNoKV01iKEvPi2A0gUSPwkC2aPn0m +hrKn1/koYAPjd8eIBMn+iXdwEE/DoIo79KtCe+MUktiuFtah3n0CAwEAAQKCAYAD +iUK0/k2ZRqXJHXKBKy8rWjYMHCj3lvMM/M3g+tYWS7i88w00cIJ1geM006FDSf8i +LxjatvFd2OCg9ay+w8LSbvrJJGGbeXAQjo1v7ePRPttAPWphQ8RCS+8NAKhJcNJu +UzapZ13WJKfL2HLw1+VbziORXjMlLKRnAVDkzHMZO70C5MEQ0EIX+C6zrmBOl2HH +du6LPy8crSaDQg8YxFCI7WWnvRKp+Gp8aIfYnR+7ifT1qr5o9sEUw8GAReyooJ3a +yJ9uBUbcelO8fNjEABf9xjx+jOmOVsQfig2KuBEi0qXlQSpilZfUdYJhtNke9ADu +Hui6MBn04D4RIzeKXV+OLjiLwqkJyNlPuxJ2EGpIHNMcx3gpjXIApAwc47BQwLKJ +VhMWMXS0EWhCLtEzf5UrbMNX+Io3J7noEUu6jxmJV1BKhrnlYeoo4JryN0DUpkSb +rOAOJLOkpfj7+gvqmWI4MT6SQXSr6BK+3m4J5bVSq4pej9uG5NR3Utghi5hF7DEC +gcEA3cYNPYPFSTj9YAR3GUZvwUPVL3ZEFcwjrIeg87JhuZOH/hSQ33SgeEoAtaqL +cLbimj7YzUYx3FOUCp/7yK/bAF1dhAbFab1yZE46Qv2Vi4e+/KEBBftqxyJl5KyV +vc/HE1dXZGZIO1X5Z5MX8nO3rz/YayiozYVmMibrbHxgTEDC4BrbWtPJQNkflWEb +FXNjkm0s2+J3kFANpL94NUKMGtArxQV3hWydGN8wS3Fn7LDnHDoM5mOt/naeKRES +fwwpAoHBAPDTKsKs2LEe4YFzO1EClycDelppjxh5pHSnzTWSq40aKx533SG4aLyI +DmghzoA3OmY0xpAy1WpT9FeiDNbYpiFCH3qBkArQR2QCu+WGUQ9tDoeN0C2Dje4e +Yix49BjcGSWzSNvh+tU9PzRc/9eVBMAQuaCm3yNEL+Z7hFTzkrCWK23+jP/OzIIC +XhnKdOveIYVAjlVgv8CoWIy3xhwXyqPAcstcPmlv9sDAYn37Ot7rGIS7e0WyQxvg +gxnOxFzKNQKBwQDOPOn/NNV5HKh0bHKdbKVs4zoT4zW515etUIvbVR4QSCSFonZ/ +d6PreVZjmvAFp+3fZ2aSrx6bOJZJszGhFfjhw/G9X9aiWO1SXnVL6yrxERIJOWkM +ORy5h0GegOjYFauaTvUUhxHRLEi9i0sPy5EcRpFqReuFBPNe3Fa/EoMzJl6TriYj +tyRHTCNU9XMMZbxJZYH8EgUCjY/Cj9SoIvTL0p+Bn23hBHqrsJLm9dWhhXnHBC0O +68/Y/lJi+l9rCtECgcEAt6PfTJovl0j8HxF23vyBtK9TQtSR2NERlh9LPZn9lViq +Hs66YndT7sg1bDSzWlRDBSMjc1xAH5erkJOzBLYqYNwiUvGvnH9coSfwjkMRVxkL +ZlS+taZGuZiTtmP5h2d3CaegXIQDGU5d/xkXwxYQjEF0u8vkBel+OVxg+cLPTjcF +IRhl/r98dXtGtJYM+LvnhcxHfVWMg2YcOBn/SPbfgGVFZEuQECjf2fYaZQUJzGkr +xjOM+gXIZN6cOjbQyA0tAoHADgR5/bMbcf6Jk0w56c/khFZz/pusne5cjXw5a6qq +fClAqnqjGBpkRxs7HoCR3aje0Pd0pCS93a6Wiqneo4x4HDrpo+pWR2KGAAF4MeO3 +3K94hncmiLAiZo8iqULLKCqJW2EGB2b7QzGpY7jCPiI1g80KuYPesf4ZohSfrr1w +DoqGoNrcIVdVmUgX47lLqIiWarbbDRY0Am9j58dovmNINYr5wCYGbeh2RuUmHr4u +E2bb0CdekSHf05HPiF9QpK1z +-----END PRIVATE KEY----- diff --git a/Lib/test/certdata/revocation.crl b/Lib/test/certdata/revocation.crl new file mode 100644 index 0000000000..431a831c4f --- /dev/null +++ b/Lib/test/certdata/revocation.crl @@ -0,0 +1,14 @@ +-----BEGIN X509 CRL----- +MIICJjCBjwIBATANBgkqhkiG9w0BAQsFADBNMQswCQYDVQQGEwJYWTEmMCQGA1UE +CgwdUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24gQ0ExFjAUBgNVBAMMDW91ci1j +YS1zZXJ2ZXIXDTIzMTEyNTA0MjEzNloXDTQzMDEyNDA0MjEzNlqgDjAMMAoGA1Ud +FAQDAgEAMA0GCSqGSIb3DQEBCwUAA4IBgQDMZ4XLQlzUrqBbszEq9I/nXK3jN8/p +VZ2aScU2le0ySJqIthe0yXEYuoFu+I4ZULyNkCA79baStIl8/Lt48DOHfBVv8SVx +ZqF7/fdUZBCLJV1kuhuSSknbtNmja5NI4/lcRRXrodRWDMcOmqlKbAC6RMQz/gMG +vpewGPX1oj5AQnqqd9spKtHbeqeDiyyWYr9ZZFO/433lP7GdsoriTPggYJJMWJvs +819buE0iGwWf+rTLB51VyGluhcz2pqimej6Ra2cdnYh5IztZlDFR99HywzWhVz/A +2fwUA91GR7zATerweXVKNd59mcgF4PZWiXmQMwcE0qQOMqMmAqYPLim1mretZsAs +t1X+nDM0Ak3sKumIjteQF7I6VpSsG4NCtq23G8KpNHnBZVOt0U065lQEvx0ZmB94 +1z7SzjfSZMVXYxBjSXljwuoc1keGpNT5xCmHyrOIxaHsmizzwNESW4dGVLu7/JfK +w40uGbwH09w4Cfbwuo7w6sRWDWPnlW2mkoc= +-----END X509 CRL----- diff --git a/Lib/test/certdata/secp384r1.pem b/Lib/test/certdata/secp384r1.pem new file mode 100644 index 0000000000..eef7117af7 --- /dev/null +++ b/Lib/test/certdata/secp384r1.pem @@ -0,0 +1,7 @@ +$ openssl genpkey -genparam -algorithm EC -pkeyopt ec_paramgen_curve:secp384r1 -pkeyopt ec_param_enc:named_curve -text +-----BEGIN EC PARAMETERS----- +BgUrgQQAIg== +-----END EC PARAMETERS----- +ECDSA-Parameters: (384 bit) +ASN1 OID: secp384r1 +NIST CURVE: P-384 diff --git a/Lib/test/certdata/selfsigned_pythontestdotnet.pem b/Lib/test/certdata/selfsigned_pythontestdotnet.pem new file mode 100644 index 0000000000..2b1760747b --- /dev/null +++ b/Lib/test/certdata/selfsigned_pythontestdotnet.pem @@ -0,0 +1,34 @@ +-----BEGIN CERTIFICATE----- +MIIF9zCCA9+gAwIBAgIUH98b4Fw/DyugC9cV7VK7ZODzHsIwDQYJKoZIhvcNAQEL +BQAwgYoxCzAJBgNVBAYTAlhZMRcwFQYDVQQIDA5DYXN0bGUgQW50aHJheDEYMBYG +A1UEBwwPQXJndW1lbnQgQ2xpbmljMSMwIQYDVQQKDBpQeXRob24gU29mdHdhcmUg +Rm91bmRhdGlvbjEjMCEGA1UEAwwac2VsZi1zaWduZWQucHl0aG9udGVzdC5uZXQw +HhcNMTkwNTA4MDEwMjQzWhcNMjcwNzI0MDEwMjQzWjCBijELMAkGA1UEBhMCWFkx +FzAVBgNVBAgMDkNhc3RsZSBBbnRocmF4MRgwFgYDVQQHDA9Bcmd1bWVudCBDbGlu +aWMxIzAhBgNVBAoMGlB5dGhvbiBTb2Z0d2FyZSBGb3VuZGF0aW9uMSMwIQYDVQQD +DBpzZWxmLXNpZ25lZC5weXRob250ZXN0Lm5ldDCCAiIwDQYJKoZIhvcNAQEBBQAD +ggIPADCCAgoCggIBAMKdJlyCThkahwoBb7pl5q64Pe9Fn5jrIvzsveHTc97TpjV2 +RLfICnXKrltPk/ohkVl6K5SUZQZwMVzFubkyxE0nZPHYHlpiKWQxbsYVkYv01rix +IFdLvaxxbGYke2jwQao31s4o61AdlsfK1SdpHQUynBBMssqI3SB4XPmcA7e+wEEx +jxjVish4ixA1vuIZOx8yibu+CFCf/geEjoBMF3QPdzULzlrCSw8k/45iZCSoNbvK +DoL4TVV07PHOxpheDh8ZQmepGvU6pVqhb9m4lgmV0OGWHgozd5Ur9CbTVDmxIEz3 +TSoRtNJK7qtyZdGNqwjksQxgZTjM/d/Lm/BJG99AiOmYOjsl9gbQMZgvQmMAtUsI +aMJnQuZ6R+KEpW/TR5qSKLWZSG45z/op+tzI2m+cE6HwTRVAWbcuJxcAA55MZjqU +OOOu3BBYMjS5nf2sQ9uoXsVBFH7i0mQqoW1SLzr9opI8KsWwFxQmO2vBxWYaN+lH +OmwBZBwyODIsmI1YGXmTp09NxRYz3Qe5GCgFzYowpMrcxUC24iduIdMwwhRM7rKg +7GtIWMSrFfuI1XCLRmSlhDbhNN6fVg2f8Bo9PdH9ihiIyxSrc+FOUasUYCCJvlSZ +8hFUlLvcmrZlWuazohm0lsXuMK1JflmQr/DA/uXxP9xzFfRy+RU3jDyxJbRHAgMB +AAGjUzBRMB0GA1UdDgQWBBSQJyxiPMRK01i+0BsV9zUwDiBaHzAfBgNVHSMEGDAW +gBSQJyxiPMRK01i+0BsV9zUwDiBaHzAPBgNVHRMBAf8EBTADAQH/MA0GCSqGSIb3 +DQEBCwUAA4ICAQCR+7a7N/m+WLkxPPIA/CB4MOr2Uf8ixTv435Nyv6rXOun0+lTP +ExSZ0uYQ+L0WylItI3cQHULldDueD+s8TGzxf5woaLKf6tqyr0NYhKs+UeNEzDnN +9PHQIhX0SZw3XyXGUgPNBfRCg2ZDdtMMdOU4XlQN/IN/9hbYTrueyY7eXq9hmtI9 +1srftAMqr9SR1JP7aHI6DVgrEsZVMTDnfT8WmLSGLlY1HmGfdEn1Ip5sbo9uSkiH +AEPgPfjYIvR5LqTOMn4KsrlZyBbFIDh9Sl99M1kZzgH6zUGVLCDg1y6Cms69fx/e +W1HoIeVkY4b4TY7Bk7JsqyNhIuqu7ARaxkdaZWhYaA2YyknwANdFfNpfH+elCLIk +BUt5S3f4i7DaUePTvKukCZiCq4Oyln7RcOn5If73wCeLB/ZM9Ei1HforyLWP1CN8 +XLfpHaoeoPSWIveI0XHUl65LsPN2UbMbul/F23hwl+h8+BLmyAS680Yhn4zEN6Ku +B7Po90HoFa1Du3bmx4jsN73UkT/dwMTi6K072FbipnC1904oGlWmLwvAHvrtxxmL +Pl3pvEaZIu8wa/PNF6Y7J7VIewikIJq6Ta6FrWeFfzMWOj2qA1ZZi6fUaDSNYvuV +J5quYKCc/O+I/yDDf8wyBbZ/gvUXzUHTMYGG+bFrn1p7XDbYYeEJ6R/xEg== +-----END CERTIFICATE----- diff --git a/Lib/test/certdata/ssl_cert.pem b/Lib/test/certdata/ssl_cert.pem new file mode 100644 index 0000000000..427948252b --- /dev/null +++ b/Lib/test/certdata/ssl_cert.pem @@ -0,0 +1,27 @@ +-----BEGIN CERTIFICATE----- +MIIEgzCCAuugAwIBAgIUU+FIM/dUbCklbdDwNPd2xemDAEwwDQYJKoZIhvcNAQEL +BQAwXzELMAkGA1UEBhMCWFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMwIQYD +VQQKDBpQeXRob24gU29mdHdhcmUgRm91bmRhdGlvbjESMBAGA1UEAwwJbG9jYWxo +b3N0MB4XDTIzMTEyNTA0MjEzNloXDTQzMDEyNDA0MjEzNlowXzELMAkGA1UEBhMC +WFkxFzAVBgNVBAcMDkNhc3RsZSBBbnRocmF4MSMwIQYDVQQKDBpQeXRob24gU29m +dHdhcmUgRm91bmRhdGlvbjESMBAGA1UEAwwJbG9jYWxob3N0MIIBojANBgkqhkiG +9w0BAQEFAAOCAY8AMIIBigKCAYEAzXTIl1su11AGu6sDPsoxqcRGyAX0yjxIcswF +vj+eW/fBs2GcBby95VEOKpJPKRYYB7fAEAjAKK59zFdsDX/ynxPZLqyLQocBkFVq +tclhCRZu//KZND+uQuHSx3PjGkSvK/nrGjg5T0bkM4SFeb0YdLb+0aDTKGozUC82 +oBAilNcrFz1VXpEF0qUe9QeKQhyd0MaW5T1oSn+U3RAj2MXm3TGExyZeaicpIM5O +HFlnwUxsYSDZo0jUj342MbPOZh8szZDWi042jdtSA3i8uMSplEf4O8ZPmX0JCtrz +fVjRVdaKXIjrhMNWB8K44q6AeyhqJcVHtOmPYoHDm0qIjcrurt0LZaGhmCuKimNd +njcPxW0VQmDIS/mO5+s24SK+Mpznm5q/clXEwyD8FbrtrzV5cHCE8eNkxjuQjkmi +wW9uadK1s54tDwRWMl6DRWRyxoF0an885UQWmbsgEB5aRmEx2L0JeD0/q6Iw1Nta +As8DG4AaWuYMrgZXz7XvyiMq3IxVAgMBAAGjNzA1MBQGA1UdEQQNMAuCCWxvY2Fs +aG9zdDAdBgNVHQ4EFgQUl2wd7iWE1JTZUVq2yFBKGm9N36owDQYJKoZIhvcNAQEL +BQADggGBAF0f5x6QXFbgdyLOyeAPD/1DDxNjM68fJSmNM/6vxHJeDFzK0Pja+iJo +xv54YiS9F2tiKPpejk4ujvLQgvrYrTQvliIE+7fUT0dV74wZKPdLphftT9uEo1dH +TeIld+549fqcfZCJfVPE2Ka4vfyMGij9hVfY5FoZL1Xpnq/ZGYyWZNAPbkG292p8 +KrfLZm/0fFYAhq8tG/6DX7+2btxeX4MP/49tzskcYWgOjlkknyhJ76aMG9BJ1D7F +/TIEh5ihNwRTmyt023RBz/xWiN4xBLyIlpQ6d5ECKmFNFr0qnEui6UovfCHUF6lZ +qcAQ5VFQQ2CayNlVmQ+UGmWIqANlacYWBt7Q6VqpGg24zTMec1/Pqd6X07ScSfrm +MAtywrWrU7p1aEkN5lBa4n/XKZHGYMjor/YcMdF5yjdSrZr274YYO1pafmTFwRwH +5o16c8WPc0aPvTFbkGIFT5ddxYstw+QwsBtLKE2lJ4Qfmxt0Ew/0L7xkbK1BaCOo +EGD2IF7VDQ== +-----END CERTIFICATE----- diff --git a/Lib/test/certdata/ssl_key.passwd.pem b/Lib/test/certdata/ssl_key.passwd.pem new file mode 100644 index 0000000000..6ab7d57d00 --- /dev/null +++ b/Lib/test/certdata/ssl_key.passwd.pem @@ -0,0 +1,42 @@ +-----BEGIN ENCRYPTED PRIVATE KEY----- +MIIHbTBXBgkqhkiG9w0BBQ0wSjApBgkqhkiG9w0BBQwwHAQIsc9l0YPybNICAggA +MAwGCCqGSIb3DQIJBQAwHQYJYIZIAWUDBAEqBBDxb9ekR9MERvIff73hFLc6BIIH +ENhkFePApZj7ZqpjBltINRnaZhu8sEfG1/y3ejDBOa5Sq3C/UPykPfJh0IXsraAB +STZO22UQEDpJzDnf1aLCo2cJpdz4Mr+Uj8OUdPiX83OlhC36gMrkgSYUdhSFQEas +MLiBnXU6Z5Mv1Lxe7TJrnMyA4A8JYXXu5XVTErJrC0YT6iCPQh7eAoEtml9a/tJM +OPg6kn58zmzVDp8LAau4Th1yhdD/cUQM09wg2i5JHLeC9akD+CkNlujVoAirLMTh +xoMXTy2dkv/lIwI9QVx6WE/VKIngBAPIi3Q+YCIm0PaTgWj5U10C8j4t7kW2AEZK +z82+vDOpLRGLo/ItNCO9F/a9e4PK4xxwFCOfR80tQNhs5gjKnbDz5IQv2p+pUfUX +u+AIO0rBb3M9Yya1MC2pc5VLAeQ3UF6YPrNyNjoDsQOytY3YtRVyxiKW72QzeUcX +Vpc3U6u8ZyHhkxK6bMv3dkPHGW1MOBd9/U5z+9lhHOfCGFStIQ9M8N48ZCWEGyty +oZT3UApxgqiBAi1h14ZyagA2mjsMNtTmmkSa3v26WUfrwnjm7LD1/0Vm+ptBOFH2 +CkP/aAvr8Ie+ehWobXGpqwB6rlOAwdpPrePtEZiZtdt58anmCquRgE5GIYtVz30f +flRABM8waJ196RDGkNAmDA3p/sqHy4vbsIOMl8faZ3QxvGVZlPbUEwPhiTIetA5Q +95fT/uIcuBLfpbaN23j/Av3LiJAeABSmGZ+dA+NXC5UMvuX8COyBU0YF2V6ofpIu +gP3UC7Tn4yV3Pbes81LEDCskaN6qVRil47l0G+dNcEHVkrGKcSaRCN+joBSCbuin +Rol34ir9azh8DqHRKdVlLlzTmDQcOwmi0Vx0ASgBXx4UI3IfK45gLJVoz6dkUz+3 +GIPrnh5cw2DvIgIApwmuCQUXPbWZwUW0zuyzhtny9W6S72GUE/P5oUCV+kGYBsup +FNiAyR9+n/xUuzB5HqIosj4rX+M4il4Ovt+KaCO6/COi+YjAO/9EnSttu8OTxsXl +wvgblsT7Y1d+iUfmIVNGtbc5NX46ktrbGiqgPX7oR7YDy5/+FQlnPS1YL0ThUiAC +2RbItu6b0uUyfu2jfWaGqy+SiRZ81rLwKPU3vJSEPfooVcJTG49EE006ZC4TvRzu +fNkId+P+BxvhEpUM4+VKzfzViEPuzR1u/DuwLAavS7nr5qb+zaUq+Fte5vDQmjjC +fflT8hS0BGpYEGndeZT4k+mZunHgs3NVUQ4/HW0nflf1j6qAn4+yIB79dH9d/ubt +RyBG29K+rN0TI/kH9BQZfsAcbnmhpT/ud0mJfeHZ0Lknn6mdJ/k4LXN0T1IlLKz3 +cSleOWY3zjKaOsbuju1o5IiVIr+AF/w+M4nzzDX6DDVpBPAt9iUnDGqjh6mJ3QWQ +CyCJDLNP0X8rZ8va2KOPorIBhmfDwJKEtIoXkb2hqWURTE0chC444QqiMsMXsX6+ +mOmiWGkdBFnEpGITISFTGERCjEfqOgTMweCANpquiLymJXgDURL603N2WexSgwnu +Gy1Ws1cA+1cT65ZLqjSqayZ6WdQvsKBBAnGW5LbwBhoCkX0vahs5nZiw0KnskP60 +wNMnyxaS1SuDJ65n+vuLUl7WeysRyz10RWliYZFiUE7jIXfWeYGonAo4eyCEeV/f +HInxxpswsg/na8BGBPMsx2SfBIiIvSIT4VNxHrL3sIfDrnb2HH/ut/oSLBgSKzY5 +DdkPz309kMM5dqnHANAgRrtVhqzLQE3kNGZ9mO/X1FAyXx8eB7NSeB6ysD8CAHvm +lkyfsGTzVsnuWWpeHqplds0wx5+XouVtFRI5J3RGa39mbpM1hMyIbS0O24CBKW6K +7n2UunbABwepL1hSa4e01OPdz4Zx/oayOevTtlfVqh68cEEc6ePdzf7z69pjot7B +eqlNaqa1POOmkuygL+fiP1BAR3rGEoQKXqb+6JjzLM9CnhCQHHPR2UdqukkEYwsa +bh9CU8AlfAJ19KFDria4JZXtl8LLMLLqWIO8fmQx7VqkEkEkl8jecO8YMaZTzFEb +bW7QtIZ1qHWH0UIHH3Qlav72NJTKvGIbtp1JNrLdsHcYNcojLZkEeA83UPaiTB2R +udltVUd016cktRVzLOKrust8kzPq3iSjpoIXFyFqIYHvWxGHgc7qD5gVBlazqSsV +qudDv+0PCBjLWLjS6HkFI8BfyXd3ME2wvSmTzSSgSh4nVJNNrZ/RVTtQ5MLVcdh0 +sJ3qsq2Pokf61XXjsTiQorX+cgI9zF6zETXHvnLf9FL+G/VSlcLUsQ0wC584qwQt +OSASYTbM79xgmjRmolZOptcYXGktfi2C4iq6V6zpFJuNMVgzZ+SbaQw9bvzUo2jG +VMwrTuQQ+fsAyn66WZvtkSGAdp58+3PNq31ZjafJXBzN +-----END ENCRYPTED PRIVATE KEY----- diff --git a/Lib/test/certdata/ssl_key.pem b/Lib/test/certdata/ssl_key.pem new file mode 100644 index 0000000000..ee92721051 --- /dev/null +++ b/Lib/test/certdata/ssl_key.pem @@ -0,0 +1,40 @@ +-----BEGIN PRIVATE KEY----- +MIIG/QIBADANBgkqhkiG9w0BAQEFAASCBucwggbjAgEAAoIBgQDNdMiXWy7XUAa7 +qwM+yjGpxEbIBfTKPEhyzAW+P55b98GzYZwFvL3lUQ4qkk8pFhgHt8AQCMAorn3M +V2wNf/KfE9kurItChwGQVWq1yWEJFm7/8pk0P65C4dLHc+MaRK8r+esaODlPRuQz +hIV5vRh0tv7RoNMoajNQLzagECKU1ysXPVVekQXSpR71B4pCHJ3QxpblPWhKf5Td +ECPYxebdMYTHJl5qJykgzk4cWWfBTGxhINmjSNSPfjYxs85mHyzNkNaLTjaN21ID +eLy4xKmUR/g7xk+ZfQkK2vN9WNFV1opciOuEw1YHwrjiroB7KGolxUe06Y9igcOb +SoiNyu6u3QtloaGYK4qKY12eNw/FbRVCYMhL+Y7n6zbhIr4ynOebmr9yVcTDIPwV +uu2vNXlwcITx42TGO5COSaLBb25p0rWzni0PBFYyXoNFZHLGgXRqfzzlRBaZuyAQ +HlpGYTHYvQl4PT+rojDU21oCzwMbgBpa5gyuBlfPte/KIyrcjFUCAwEAAQKCAYAO +M1r0+TCy4Z1hhceu5JdLql0RELZTbxi71IW2GVwW87gv75hy3hGLAs/1mdC+YIBP +MkBka1JqzWq0/7rgcP5CSAMsInFqqv2s7fZ286ERGXuZFbnInnkrNsQUlJo3E9W+ +tqKtGIM/i0EVHX0DRdJlqMtSjmjh43tB+M1wAUV+n6OjEtJue5wZK+AIpBmGicdP +qZY+6IBnm8tcfzPXFRCoq7ZHdIu0jxnc4l2MQJK3DdL04KoiStOkSl8xDsI+lTtq +D3qa41LE0TY8X2jJ/w6KK3cUeK7F4DQYs+kfCKWMVPpn0/5u6TbC1F7gLvkrseph +7cIgrruNNs9iKacnR1w3U72R+hNxHsNfo4RGHFa192p/Mfc+kiBd5RNR/M9oHdeq +U6T/+KM+QyF5dDOyonY0QjwfAcEx+ZsV72nj8AerjM907I6dgHo/9YZ2S1Dt/xuG +ntD+76GDzmrOvXmmpF0DsTn+Wql7AC4uzaOjv6PVziqz03pR61RpjPDemyJEWMkC +gcEA7BkGGX3enBENs3X6BYFoeXfGO/hV7/aNpA6ykLzw657dqwy2b6bWLiIaqZdZ +u0oiY6+SpOtavkZBFTq4bTVD58FHL0n73Yvvaft507kijpYBrxyDOfTJOETv+dVG +XiY8AUSAE6GjPi0ebuYIVUxoDnMeWDuRJNvTck4byn1hJ1aVlEhwXNxt/nAjq48s +5QDuR6Z9F8lqEACRYCHSMQYFm35c7c1pPsHJnElX8a7eZ9lT7HGPXHaf/ypMkOzo +dvJNAoHBAN7GhDomff/kSgQLyzmqKqQowTZlyihnReapygwr8YpNcqKDqq6VlnfH +Jl1+qtSMSVI0csmccwJWkz1WtSjDsvY+oMdv4gUK3028vQAMQZo+Sh7OElFPFET3 +UmL+Nh73ACPgpiommsdLZQPcIqpWNT5NzO+Jm5xa+U9ToVZgQ7xjrqee5NUiMutr +r7UWAz7vDWu3x7bzYRRdUJxU18NogGbFGWJ1KM0c67GUXu2E7wBQdjVdS78UWs+4 +XBxKQkG2KQKBwQCtO+M82x122BB8iGkulvhogBjlMd8klnzxTpN5HhmMWWH+uvI1 +1G29Jer4WwRNJyU6jb4E4mgPyw7AG/jssLOlniy0Jw32TlIaKpoGXwZbJvgPW9Vx +tgnbDsIiR3o9ZMKMj42GWgike4ikCIc+xzRmvdMbHIHwUJfCfEtp9TtPGPnh9pDz +og3XLsMNg52GXnt3+VI6HOCE41XH+qj2rZt5r2tSVXEOyjQ7R5mOzSeFfXJVwDFX +v/a/zHKnuB0OAdUCgcBLrxPTEaqy2eMPdtZHM/mipbnmejRw/4zu7XYYJoG7483z +SlodT/K7pKvzDYqKBVMPm4P33K/x9mm1aBTJ0ZqmL+a9etRFtEjjByEKuB89gLX7 +uzTb7MrNF10lBopqgK3KgpLRNSZWWNXrtskMJ5eVICdkpdJ5Dyst+RKR3siEYzU9 ++yxxAFpeQsqB8gWORva/RsOR8yNjIMS3J9fZqlIdGA8ktPr0nEOyo96QQR5VdACE +5rpKI2cqtM6OSegynOkCgcAnr2Xzjef6tdcrxrQrq0DjEFTMoCAxQRa6tuF/NYHV +AK70Y4hBNX84Bvym4hmfbMUEuOCJU+QHQf/iDQrHXPhtX3X2/t8M+AlIzmwLKf2o +VwCYnZ8SqiwSaWVg+GANWLh0JuKn/ZYyR8urR79dAXFfp0UK+N39vIxNoBisBf+F +G8mca7zx3UtK2eOW8WgGHz+Y20VZy0m/nkNekd1ZTXoSGhL+iN4XsTRn1YQIn69R +kNdcwhtZZ3dpChUdf+w/LIc= +-----END PRIVATE KEY----- diff --git a/Lib/test/certdata/talos-2019-0758.pem b/Lib/test/certdata/talos-2019-0758.pem new file mode 100644 index 0000000000..13b95a77fd --- /dev/null +++ b/Lib/test/certdata/talos-2019-0758.pem @@ -0,0 +1,22 @@ +-----BEGIN CERTIFICATE----- +MIIDqDCCApKgAwIBAgIBAjALBgkqhkiG9w0BAQswHzELMAkGA1UEBhMCVUsxEDAO +BgNVBAMTB2NvZHktY2EwHhcNMTgwNjE4MTgwMDU4WhcNMjgwNjE0MTgwMDU4WjA7 +MQswCQYDVQQGEwJVSzEsMCoGA1UEAxMjY29kZW5vbWljb24tdm0tMi50ZXN0Lmxh +bC5jaXNjby5jb20wggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQC63fGB +J80A9Av1GB0bptslKRIUtJm8EeEu34HkDWbL6AJY0P8WfDtlXjlPaLqFa6sqH6ES +V48prSm1ZUbDSVL8R6BYVYpOlK8/48xk4pGTgRzv69gf5SGtQLwHy8UPBKgjSZoD +5a5k5wJXGswhKFFNqyyxqCvWmMnJWxXTt2XDCiWc4g4YAWi4O4+6SeeHVAV9rV7C +1wxqjzKovVe2uZOHjKEzJbbIU6JBPb6TRfMdRdYOw98n1VXDcKVgdX2DuuqjCzHP +WhU4Tw050M9NaK3eXp4Mh69VuiKoBGOLSOcS8reqHIU46Reg0hqeL8LIL6OhFHIF +j7HR6V1X6F+BfRS/AgMBAAGjgdYwgdMwCQYDVR0TBAIwADAdBgNVHQ4EFgQUOktp +HQjxDXXUg8prleY9jeLKeQ4wTwYDVR0jBEgwRoAUx6zgPygZ0ZErF9sPC4+5e2Io +UU+hI6QhMB8xCzAJBgNVBAYTAlVLMRAwDgYDVQQDEwdjb2R5LWNhggkA1QEAuwb7 +2s0wCQYDVR0SBAIwADAuBgNVHREEJzAlgiNjb2Rlbm9taWNvbi12bS0yLnRlc3Qu +bGFsLmNpc2NvLmNvbTAOBgNVHQ8BAf8EBAMCBaAwCwYDVR0fBAQwAjAAMAsGCSqG +SIb3DQEBCwOCAQEAvqantx2yBlM11RoFiCfi+AfSblXPdrIrHvccepV4pYc/yO6p +t1f2dxHQb8rWH3i6cWag/EgIZx+HJQvo0rgPY1BFJsX1WnYf1/znZpkUBGbVmlJr +t/dW1gSkNS6sPsM0Q+7HPgEv8CPDNK5eo7vU2seE0iWOkxSyVUuiCEY9ZVGaLVit +p0C78nZ35Pdv4I+1cosmHl28+es1WI22rrnmdBpH8J1eY6WvUw2xuZHLeNVN0TzV +Q3qq53AaCWuLOD1AjESWuUCxMZTK9DPS4JKXTK8RLyDeqOvJGjsSWp3kL0y3GaQ+ +10T1rfkKJub2+m9A9duin1fn6tHc2wSvB7m3DA== +-----END CERTIFICATE----- diff --git a/Lib/test/cfgparser.1 b/Lib/test/configdata/cfgparser.1 similarity index 100% rename from Lib/test/cfgparser.1 rename to Lib/test/configdata/cfgparser.1 diff --git a/Lib/test/cfgparser.2 b/Lib/test/configdata/cfgparser.2 similarity index 100% rename from Lib/test/cfgparser.2 rename to Lib/test/configdata/cfgparser.2 diff --git a/Lib/test/cfgparser.3 b/Lib/test/configdata/cfgparser.3 similarity index 100% rename from Lib/test/cfgparser.3 rename to Lib/test/configdata/cfgparser.3 diff --git a/Lib/test/data/README b/Lib/test/data/README new file mode 100644 index 0000000000..bd05984e43 --- /dev/null +++ b/Lib/test/data/README @@ -0,0 +1,2 @@ +This empty directory serves as destination for temporary files +created by some tests, in particular, the test_codecmaps_* tests. diff --git a/Lib/test/datetimetester.py b/Lib/test/datetimetester.py index 7c6c8d7dd2..2636c1fbb9 100644 --- a/Lib/test/datetimetester.py +++ b/Lib/test/datetimetester.py @@ -2943,8 +2943,6 @@ def newmeth(self, start): self.assertEqual(dt2.newmeth(-7), dt1.year + dt1.month + dt1.second - 7) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_subclass_alternate_constructors_datetime(self): # Test that alternate constructors call the constructor class DateTimeSubclass(self.theclass): @@ -5457,8 +5455,6 @@ def test_bug_1028306(self): self.assertEqual(as_datetime, datetime_sc) self.assertEqual(datetime_sc, as_datetime) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_extra_attributes(self): with self.assertWarns(DeprecationWarning): utcnow = datetime.utcnow() diff --git a/Lib/test/decimaltestdata/abs.decTest b/Lib/test/decimaltestdata/abs.decTest index 01f73d7766..569b8fcd84 100644 --- a/Lib/test/decimaltestdata/abs.decTest +++ b/Lib/test/decimaltestdata/abs.decTest @@ -20,7 +20,7 @@ version: 2.59 -- This set of tests primarily tests the existence of the operator. --- Additon, subtraction, rounding, and more overflows are tested +-- Addition, subtraction, rounding, and more overflows are tested -- elsewhere. precision: 9 diff --git a/Lib/test/decimaltestdata/ddFMA.decTest b/Lib/test/decimaltestdata/ddFMA.decTest index 9094fc015b..7f2e523037 100644 --- a/Lib/test/decimaltestdata/ddFMA.decTest +++ b/Lib/test/decimaltestdata/ddFMA.decTest @@ -1663,7 +1663,7 @@ ddfma375087 fma 1 12345678 1E-33 -> 12345678.00000001 Inexac ddfma375088 fma 1 12345678 1E-34 -> 12345678.00000001 Inexact Rounded ddfma375089 fma 1 12345678 1E-35 -> 12345678.00000001 Inexact Rounded --- desctructive subtraction (from remainder tests) +-- destructive subtraction (from remainder tests) -- +++ some of these will be off-by-one remainder vs remainderNear diff --git a/Lib/test/decimaltestdata/ddQuantize.decTest b/Lib/test/decimaltestdata/ddQuantize.decTest index 9177620169..e1c5674d9a 100644 --- a/Lib/test/decimaltestdata/ddQuantize.decTest +++ b/Lib/test/decimaltestdata/ddQuantize.decTest @@ -462,7 +462,7 @@ ddqua520 quantize 1.234 1e359 -> 0E+359 Inexact Rounded ddqua521 quantize 123.456 1e359 -> 0E+359 Inexact Rounded ddqua522 quantize 1.234 1e359 -> 0E+359 Inexact Rounded ddqua523 quantize 123.456 1e359 -> 0E+359 Inexact Rounded --- next four are "won't fit" overfl +-- next four are "won't fit" overflow ddqua526 quantize 1.234 1e-299 -> NaN Invalid_operation ddqua527 quantize 123.456 1e-299 -> NaN Invalid_operation ddqua528 quantize 1.234 1e-299 -> NaN Invalid_operation diff --git a/Lib/test/decimaltestdata/ddRemainder.decTest b/Lib/test/decimaltestdata/ddRemainder.decTest index 5bd1e32d01..b1866d39a2 100644 --- a/Lib/test/decimaltestdata/ddRemainder.decTest +++ b/Lib/test/decimaltestdata/ddRemainder.decTest @@ -422,7 +422,7 @@ ddrem757 remainder 1 sNaN -> NaN Invalid_operation ddrem758 remainder 1000 sNaN -> NaN Invalid_operation ddrem759 remainder Inf -sNaN -> -NaN Invalid_operation --- propaging NaNs +-- propagating NaNs ddrem760 remainder NaN1 NaN7 -> NaN1 ddrem761 remainder sNaN2 NaN8 -> NaN2 Invalid_operation ddrem762 remainder NaN3 sNaN9 -> NaN9 Invalid_operation diff --git a/Lib/test/decimaltestdata/ddRemainderNear.decTest b/Lib/test/decimaltestdata/ddRemainderNear.decTest index 6ba64ebafe..bbe82ea374 100644 --- a/Lib/test/decimaltestdata/ddRemainderNear.decTest +++ b/Lib/test/decimaltestdata/ddRemainderNear.decTest @@ -450,7 +450,7 @@ ddrmn757 remaindernear 1 sNaN -> NaN Invalid_operation ddrmn758 remaindernear 1000 sNaN -> NaN Invalid_operation ddrmn759 remaindernear Inf -sNaN -> -NaN Invalid_operation --- propaging NaNs +-- propagating NaNs ddrmn760 remaindernear NaN1 NaN7 -> NaN1 ddrmn761 remaindernear sNaN2 NaN8 -> NaN2 Invalid_operation ddrmn762 remaindernear NaN3 sNaN9 -> NaN9 Invalid_operation diff --git a/Lib/test/decimaltestdata/dqRemainder.decTest b/Lib/test/decimaltestdata/dqRemainder.decTest index bae8eae526..e0aaca3747 100644 --- a/Lib/test/decimaltestdata/dqRemainder.decTest +++ b/Lib/test/decimaltestdata/dqRemainder.decTest @@ -418,7 +418,7 @@ dqrem757 remainder 1 sNaN -> NaN Invalid_operation dqrem758 remainder 1000 sNaN -> NaN Invalid_operation dqrem759 remainder Inf -sNaN -> -NaN Invalid_operation --- propaging NaNs +-- propagating NaNs dqrem760 remainder NaN1 NaN7 -> NaN1 dqrem761 remainder sNaN2 NaN8 -> NaN2 Invalid_operation dqrem762 remainder NaN3 sNaN9 -> NaN9 Invalid_operation diff --git a/Lib/test/decimaltestdata/dqRemainderNear.decTest b/Lib/test/decimaltestdata/dqRemainderNear.decTest index b850626fe4..2c5c3f5074 100644 --- a/Lib/test/decimaltestdata/dqRemainderNear.decTest +++ b/Lib/test/decimaltestdata/dqRemainderNear.decTest @@ -450,7 +450,7 @@ dqrmn757 remaindernear 1 sNaN -> NaN Invalid_operation dqrmn758 remaindernear 1000 sNaN -> NaN Invalid_operation dqrmn759 remaindernear Inf -sNaN -> -NaN Invalid_operation --- propaging NaNs +-- propagating NaNs dqrmn760 remaindernear NaN1 NaN7 -> NaN1 dqrmn761 remaindernear sNaN2 NaN8 -> NaN2 Invalid_operation dqrmn762 remaindernear NaN3 sNaN9 -> NaN9 Invalid_operation diff --git a/Lib/test/decimaltestdata/exp.decTest b/Lib/test/decimaltestdata/exp.decTest index 6a7af23b62..e01d7a8f92 100644 --- a/Lib/test/decimaltestdata/exp.decTest +++ b/Lib/test/decimaltestdata/exp.decTest @@ -28,7 +28,7 @@ rounding: half_even maxExponent: 384 minexponent: -383 --- basics (examples in specificiation, etc.) +-- basics (examples in specification, etc.) expx001 exp -Infinity -> 0 expx002 exp -10 -> 0.0000453999298 Inexact Rounded expx003 exp -1 -> 0.367879441 Inexact Rounded diff --git a/Lib/test/decimaltestdata/extra.decTest b/Lib/test/decimaltestdata/extra.decTest index b630d8e3f9..31291202a3 100644 --- a/Lib/test/decimaltestdata/extra.decTest +++ b/Lib/test/decimaltestdata/extra.decTest @@ -156,7 +156,7 @@ extr1302 fma -Inf 0E-456 sNaN148 -> NaN Invalid_operation -- max/min/max_mag/min_mag bug in 2.5.2/2.6/3.0: max(NaN, finite) gave -- incorrect answers when the finite number required rounding; similarly --- for the other thre functions +-- for the other three functions maxexponent: 999 minexponent: -999 precision: 6 diff --git a/Lib/test/decimaltestdata/remainder.decTest b/Lib/test/decimaltestdata/remainder.decTest index 7a1061b1e6..4f59b33287 100644 --- a/Lib/test/decimaltestdata/remainder.decTest +++ b/Lib/test/decimaltestdata/remainder.decTest @@ -435,7 +435,7 @@ remx757 remainder 1 sNaN -> NaN Invalid_operation remx758 remainder 1000 sNaN -> NaN Invalid_operation remx759 remainder Inf -sNaN -> -NaN Invalid_operation --- propaging NaNs +-- propagating NaNs remx760 remainder NaN1 NaN7 -> NaN1 remx761 remainder sNaN2 NaN8 -> NaN2 Invalid_operation remx762 remainder NaN3 sNaN9 -> NaN9 Invalid_operation diff --git a/Lib/test/decimaltestdata/remainderNear.decTest b/Lib/test/decimaltestdata/remainderNear.decTest index b768b9e0cf..000b1424d8 100644 --- a/Lib/test/decimaltestdata/remainderNear.decTest +++ b/Lib/test/decimaltestdata/remainderNear.decTest @@ -498,7 +498,7 @@ rmnx758 remaindernear 1000 sNaN -> NaN Invalid_operation rmnx759 remaindernear Inf sNaN -> NaN Invalid_operation rmnx760 remaindernear NaN sNaN -> NaN Invalid_operation --- propaging NaNs +-- propagating NaNs rmnx761 remaindernear NaN1 NaN7 -> NaN1 rmnx762 remaindernear sNaN2 NaN8 -> NaN2 Invalid_operation rmnx763 remaindernear NaN3 -sNaN9 -> -NaN9 Invalid_operation diff --git a/Lib/test/dtracedata/assert_usable.d b/Lib/test/dtracedata/assert_usable.d new file mode 100644 index 0000000000..0b2d4da66e --- /dev/null +++ b/Lib/test/dtracedata/assert_usable.d @@ -0,0 +1,5 @@ +BEGIN +{ + printf("probe: success\n"); + exit(0); +} diff --git a/Lib/test/dtracedata/assert_usable.stp b/Lib/test/dtracedata/assert_usable.stp new file mode 100644 index 0000000000..88e7e68e2c --- /dev/null +++ b/Lib/test/dtracedata/assert_usable.stp @@ -0,0 +1,5 @@ +probe begin +{ + println("probe: success") + exit () +} diff --git a/Lib/test/dtracedata/call_stack.d b/Lib/test/dtracedata/call_stack.d new file mode 100644 index 0000000000..761d30fd55 --- /dev/null +++ b/Lib/test/dtracedata/call_stack.d @@ -0,0 +1,31 @@ +self int indent; + +python$target:::function-entry +/copyinstr(arg1) == "start"/ +{ + self->trace = 1; +} + +python$target:::function-entry +/self->trace/ +{ + printf("%d\t%*s:", timestamp, 15, probename); + printf("%*s", self->indent, ""); + printf("%s:%s:%d\n", basename(copyinstr(arg0)), copyinstr(arg1), arg2); + self->indent++; +} + +python$target:::function-return +/self->trace/ +{ + self->indent--; + printf("%d\t%*s:", timestamp, 15, probename); + printf("%*s", self->indent, ""); + printf("%s:%s:%d\n", basename(copyinstr(arg0)), copyinstr(arg1), arg2); +} + +python$target:::function-return +/copyinstr(arg1) == "start"/ +{ + self->trace = 0; +} diff --git a/Lib/test/dtracedata/call_stack.d.expected b/Lib/test/dtracedata/call_stack.d.expected new file mode 100644 index 0000000000..27849d1549 --- /dev/null +++ b/Lib/test/dtracedata/call_stack.d.expected @@ -0,0 +1,18 @@ + function-entry:call_stack.py:start:23 + function-entry: call_stack.py:function_1:1 + function-entry: call_stack.py:function_3:9 +function-return: call_stack.py:function_3:10 +function-return: call_stack.py:function_1:2 + function-entry: call_stack.py:function_2:5 + function-entry: call_stack.py:function_1:1 + function-entry: call_stack.py:function_3:9 +function-return: call_stack.py:function_3:10 +function-return: call_stack.py:function_1:2 +function-return: call_stack.py:function_2:6 + function-entry: call_stack.py:function_3:9 +function-return: call_stack.py:function_3:10 + function-entry: call_stack.py:function_4:13 +function-return: call_stack.py:function_4:14 + function-entry: call_stack.py:function_5:18 +function-return: call_stack.py:function_5:21 +function-return:call_stack.py:start:28 diff --git a/Lib/test/dtracedata/call_stack.py b/Lib/test/dtracedata/call_stack.py new file mode 100644 index 0000000000..ee9f3ae8d6 --- /dev/null +++ b/Lib/test/dtracedata/call_stack.py @@ -0,0 +1,30 @@ +def function_1(): + function_3(1, 2) + +# Check stacktrace +def function_2(): + function_1() + +# CALL_FUNCTION_VAR +def function_3(dummy, dummy2): + pass + +# CALL_FUNCTION_KW +def function_4(**dummy): + return 1 + return 2 # unreachable + +# CALL_FUNCTION_VAR_KW +def function_5(dummy, dummy2, **dummy3): + if False: + return 7 + return 8 + +def start(): + function_1() + function_2() + function_3(1, 2) + function_4(test=42) + function_5(*(1, 2), **{"test": 42}) + +start() diff --git a/Lib/test/dtracedata/call_stack.stp b/Lib/test/dtracedata/call_stack.stp new file mode 100644 index 0000000000..54082c202f --- /dev/null +++ b/Lib/test/dtracedata/call_stack.stp @@ -0,0 +1,41 @@ +global tracing + +function basename:string(path:string) +{ + last_token = token = tokenize(path, "/"); + while (token != "") { + last_token = token; + token = tokenize("", "/"); + } + return last_token; +} + +probe process.mark("function__entry") +{ + funcname = user_string($arg2); + + if (funcname == "start") { + tracing = 1; + } +} + +probe process.mark("function__entry"), process.mark("function__return") +{ + filename = user_string($arg1); + funcname = user_string($arg2); + lineno = $arg3; + + if (tracing) { + printf("%d\t%s:%s:%s:%d\n", gettimeofday_us(), $$name, + basename(filename), funcname, lineno); + } +} + +probe process.mark("function__return") +{ + funcname = user_string($arg2); + + if (funcname == "start") { + tracing = 0; + } +} diff --git a/Lib/test/dtracedata/call_stack.stp.expected b/Lib/test/dtracedata/call_stack.stp.expected new file mode 100644 index 0000000000..32cf396f82 --- /dev/null +++ b/Lib/test/dtracedata/call_stack.stp.expected @@ -0,0 +1,14 @@ +function__entry:call_stack.py:start:23 +function__entry:call_stack.py:function_1:1 +function__return:call_stack.py:function_1:2 +function__entry:call_stack.py:function_2:5 +function__entry:call_stack.py:function_1:1 +function__return:call_stack.py:function_1:2 +function__return:call_stack.py:function_2:6 +function__entry:call_stack.py:function_3:9 +function__return:call_stack.py:function_3:10 +function__entry:call_stack.py:function_4:13 +function__return:call_stack.py:function_4:14 +function__entry:call_stack.py:function_5:18 +function__return:call_stack.py:function_5:21 +function__return:call_stack.py:start:28 diff --git a/Lib/test/dtracedata/gc.d b/Lib/test/dtracedata/gc.d new file mode 100644 index 0000000000..4d91487b7e --- /dev/null +++ b/Lib/test/dtracedata/gc.d @@ -0,0 +1,18 @@ +python$target:::function-entry +/copyinstr(arg1) == "start"/ +{ + self->trace = 1; +} + +python$target:::gc-start, +python$target:::gc-done +/self->trace/ +{ + printf("%d\t%s:%ld\n", timestamp, probename, arg0); +} + +python$target:::function-return +/copyinstr(arg1) == "start"/ +{ + self->trace = 0; +} diff --git a/Lib/test/dtracedata/gc.d.expected b/Lib/test/dtracedata/gc.d.expected new file mode 100644 index 0000000000..8e5ac2a6d5 --- /dev/null +++ b/Lib/test/dtracedata/gc.d.expected @@ -0,0 +1,8 @@ +gc-start:0 +gc-done:0 +gc-start:1 +gc-done:0 +gc-start:2 +gc-done:0 +gc-start:2 +gc-done:1 diff --git a/Lib/test/dtracedata/gc.py b/Lib/test/dtracedata/gc.py new file mode 100644 index 0000000000..144a783b7b --- /dev/null +++ b/Lib/test/dtracedata/gc.py @@ -0,0 +1,13 @@ +import gc + +def start(): + gc.collect(0) + gc.collect(1) + gc.collect(2) + l = [] + l.append(l) + del l + gc.collect(2) + +gc.collect() +start() diff --git a/Lib/test/dtracedata/gc.stp b/Lib/test/dtracedata/gc.stp new file mode 100644 index 0000000000..162c6d3a22 --- /dev/null +++ b/Lib/test/dtracedata/gc.stp @@ -0,0 +1,26 @@ +global tracing + +probe process.mark("function__entry") +{ + funcname = user_string($arg2); + + if (funcname == "start") { + tracing = 1; + } +} + +probe process.mark("gc__start"), process.mark("gc__done") +{ + if (tracing) { + printf("%d\t%s:%ld\n", gettimeofday_us(), $$name, $arg1); + } +} + +probe process.mark("function__return") +{ + funcname = user_string($arg2); + + if (funcname == "start") { + tracing = 0; + } +} diff --git a/Lib/test/dtracedata/gc.stp.expected b/Lib/test/dtracedata/gc.stp.expected new file mode 100644 index 0000000000..7e6e6227fb --- /dev/null +++ b/Lib/test/dtracedata/gc.stp.expected @@ -0,0 +1,8 @@ +gc__start:0 +gc__done:0 +gc__start:1 +gc__done:0 +gc__start:2 +gc__done:0 +gc__start:2 +gc__done:1 diff --git a/Lib/test/dtracedata/instance.py b/Lib/test/dtracedata/instance.py new file mode 100644 index 0000000000..f1421378b0 --- /dev/null +++ b/Lib/test/dtracedata/instance.py @@ -0,0 +1,24 @@ +import gc + +class old_style_class(): + pass +class new_style_class(object): + pass + +a = old_style_class() +del a +gc.collect() +b = new_style_class() +del b +gc.collect() + +a = old_style_class() +del old_style_class +gc.collect() +b = new_style_class() +del new_style_class +gc.collect() +del a +gc.collect() +del b +gc.collect() diff --git a/Lib/test/dtracedata/line.d b/Lib/test/dtracedata/line.d new file mode 100644 index 0000000000..03f22db6fc --- /dev/null +++ b/Lib/test/dtracedata/line.d @@ -0,0 +1,7 @@ +python$target:::line +/(copyinstr(arg1)=="test_line")/ +{ + printf("%d\t%s:%s:%s:%d\n", timestamp, + probename, basename(copyinstr(arg0)), + copyinstr(arg1), arg2); +} diff --git a/Lib/test/dtracedata/line.d.expected b/Lib/test/dtracedata/line.d.expected new file mode 100644 index 0000000000..9b16ce76ee --- /dev/null +++ b/Lib/test/dtracedata/line.d.expected @@ -0,0 +1,20 @@ +line:line.py:test_line:2 +line:line.py:test_line:3 +line:line.py:test_line:4 +line:line.py:test_line:5 +line:line.py:test_line:6 +line:line.py:test_line:7 +line:line.py:test_line:8 +line:line.py:test_line:9 +line:line.py:test_line:10 +line:line.py:test_line:11 +line:line.py:test_line:4 +line:line.py:test_line:5 +line:line.py:test_line:6 +line:line.py:test_line:7 +line:line.py:test_line:8 +line:line.py:test_line:10 +line:line.py:test_line:11 +line:line.py:test_line:4 +line:line.py:test_line:12 +line:line.py:test_line:13 diff --git a/Lib/test/dtracedata/line.py b/Lib/test/dtracedata/line.py new file mode 100644 index 0000000000..0930ff391f --- /dev/null +++ b/Lib/test/dtracedata/line.py @@ -0,0 +1,17 @@ +def test_line(): + a = 1 + print('# Preamble', a) + for i in range(2): + a = i + b = i+2 + c = i+3 + if c < 4: + a = c + d = a + b +c + print('#', a, b, c, d) + a = 1 + print('# Epilogue', a) + + +if __name__ == '__main__': + test_line() diff --git a/Lib/test/fork_wait.py b/Lib/test/fork_wait.py new file mode 100644 index 0000000000..8c32895f5e --- /dev/null +++ b/Lib/test/fork_wait.py @@ -0,0 +1,80 @@ +"""This test case provides support for checking forking and wait behavior. + +To test different wait behavior, override the wait_impl method. + +We want fork1() semantics -- only the forking thread survives in the +child after a fork(). + +On some systems (e.g. Solaris without posix threads) we find that all +active threads survive in the child after a fork(); this is an error. +""" + +import os, time, unittest +import threading +from test import support +from test.support import threading_helper +import warnings + + +LONGSLEEP = 2 +SHORTSLEEP = 0.5 +NUM_THREADS = 4 + +class ForkWait(unittest.TestCase): + + def setUp(self): + self._threading_key = threading_helper.threading_setup() + self.alive = {} + self.stop = 0 + self.threads = [] + + def tearDown(self): + # Stop threads + self.stop = 1 + for thread in self.threads: + thread.join() + thread = None + self.threads.clear() + threading_helper.threading_cleanup(*self._threading_key) + + def f(self, id): + while not self.stop: + self.alive[id] = os.getpid() + try: + time.sleep(SHORTSLEEP) + except OSError: + pass + + def wait_impl(self, cpid, *, exitcode): + support.wait_process(cpid, exitcode=exitcode) + + def test_wait(self): + for i in range(NUM_THREADS): + thread = threading.Thread(target=self.f, args=(i,)) + thread.start() + self.threads.append(thread) + + # busy-loop to wait for threads + for _ in support.sleeping_retry(support.SHORT_TIMEOUT): + if len(self.alive) >= NUM_THREADS: + break + + a = sorted(self.alive.keys()) + self.assertEqual(a, list(range(NUM_THREADS))) + + prefork_lives = self.alive.copy() + + # Ignore the warning about fork with threads. + with warnings.catch_warnings(category=DeprecationWarning, + action="ignore"): + if (cpid := os.fork()) == 0: + # Child + time.sleep(LONGSLEEP) + n = 0 + for key in self.alive: + if self.alive[key] != prefork_lives[key]: + n += 1 + os._exit(n) + else: + # Parent + self.wait_impl(cpid, exitcode=0) diff --git a/Lib/test/keycert.passwd.pem b/Lib/test/keycert.passwd.pem deleted file mode 100644 index 0ad6960551..0000000000 --- a/Lib/test/keycert.passwd.pem +++ /dev/null @@ -1,50 +0,0 @@ ------BEGIN RSA PRIVATE KEY----- -Proc-Type: 4,ENCRYPTED -DEK-Info: DES-EDE3-CBC,E74528136B90D2DD - -WRHVD2PJXPqjFSHg92HURIsUzvsTE4a9oi0SC5yMBFKNWA5Z933gK3XTifp6jul5 -zpNYi8jBXZ2EqJJBxCuVcefmXSxL0q7CMej25TdIC4BVAFJVveeprHPUFkNB0IM1 -go5Lg4YofYqTCg3OE3k7WvfR3Zg1cRYxksDKO+WNZgWyKBex5X4vjOiyUqDl3GKt -kQXnkg1VgPV2Vrx93S9XNdELNRTguwf+XG0fkhtYhp/zCto8uKTgy5elK2P/ulGp -7fe6uj7h/uN9L7EOC6CjRkitywfeBUER739mOcGT4imSFJ9G27TCqPzj2ea3uuaf -/v1xhkQ4M6lNY/gcRfgVpCXhW43aAQV8XXQRMJTqLmz5Y5hYTKn+Ugq5vJ/ngyRM -lu1gUJnYYaemBTb4hbm6aBvnYK9mORa891Pmf+vxU9rYuQIdVAhvvXh4KBreSEBI -1AFy6dFKXl8ZKs6Wrq5wPefmFFkRmZ8OBiiq0fp2ApCRGZw6LsjCgwrRM38JiY7d -3OdsJpKvRYufgUyuuzUE0xA+E4yMvD48M9pPq2fC8O5giuGL1uEekQWXJuq+6ZRI -XYKIeSkuQALbX3RAzCPXTUEMtCYXKm/gxrrwJ+Bet4ob2amf3MX0uvWwOuAq++Fk -J0HFSBxrwvIWOhyQXOPuJdAN8PXA7dWOXfOgOMF0hQYqZCl3T4TiVZJbwVQtg1sN -dO7oAD5ZPHiKzvveZuB6k1FlBG8j0TyAC+44ChxkPDD3jF4dd6zGe62sDf85p4/d -W80gxJeD3xnDxG0ePPns+GuKUpUaWS7WvHtDpeFW1JEhvOqf8p1Li9a7RzWVo8ML -mGTdQgBIYIf6/fk69pFKl0nKtBU75KaunZz4nAmd9bNED4naDurMBg44u5TvODbJ -vgYIYXIYjNvONbskJatVrrTS8zch2NwVIjCi8L/hecwBXbIXzo1pECpc6BU7sQT8 -+i9sDKBeJcRipzfKZNHvnO19mUZaPCY8+a/f9c21DgKXz+bgLcJbohpSaeGM8Gfc -aZd3Vp9n3OJ3g2zQR1++HO9v1vR/wLELu6MeydkvMduHLmOPCn54gZ9z51ZNPAwa -qfFIsH+mLh9ks0H74ssF59uIlstkgB9zmZHv/Q0dK9ZfG/VEH6rSgdETWhZxhoMQ -Z92jXBEFT0zhI3rrIPNY+XS7eJCQIc1wc84Ea3cRk7SP+S1og3JtAxX56ykUwtkM -LQ/Dwwa6h1aqD0l2d5x1/BSdavtTuSegISRWQ4iOmSvEdlFP7H4g6RZk/okbLzMD -Evq5gNc7vlXhVawoQU8JCanJ5ZbbWnIRZfiXxBQS4lpYPKvJt4ML9z/x+82XxcXv -Z93N2Wep7wWW5OwS2LcQcOgZRDSIPompwo/0pMFGOS+5oort0ZDRHdmmGLjvBcCb -1KQmKQ4+8brI/3rjRzts6uDLjTGNxSCieNsnqhwHUv9Mg9WDSWupcGa+x27L89x3 -rObf6+3umcFLSjIzU8wuv1hx/e/y98Kv7BDBNYpAr6kVMrLnzYjAfJbBmqlxkzkQ -IgQzgrk2QZoTdgwR+S374NAMO0AE5IlO+/qa6qp2SORGTDX64I3UNw== ------END RSA PRIVATE KEY----- ------BEGIN CERTIFICATE----- -MIIDWTCCAkGgAwIBAgIJAPm6B21bar2bMA0GCSqGSIb3DQEBCwUAMF8xCzAJBgNV -BAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9u -IFNvZnR3YXJlIEZvdW5kYXRpb24xEjAQBgNVBAMMCWxvY2FsaG9zdDAeFw0xODAx -MTkxOTA5MDZaFw0yODAxMTcxOTA5MDZaMF8xCzAJBgNVBAYTAlhZMRcwFQYDVQQH -DA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9uIFNvZnR3YXJlIEZvdW5k -YXRpb24xEjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEP -ADCCAQoCggEBAKvvsX2gEti4shve3iYMc+jE4Se7WHs1Bol2f21H8qNboDOFdeb1 -RKHjmq3exHpajywOUEgne9nKHJY/3f2phR4Y5klqG6liLgiSpVyRlcBGbeT2qEAj -9oLiLFUXLGfGDds2mTwivQDLJBWi51j7ff5k2Pr58fN5ugYMn24T9FNyn0moT+qj -SFoBNm58l9jrdkJSlgWfqPlbiMa+mqDn/SFtrwLF2Trbfzu42Sd9UdIzMaSSrzbN -sGm53pNhCh8KndWUQ8GPP2IsLPoUU4qAtmZuTxCx2S1cXrN9EkmT69tlOH84YfSn -96Ih9bWRc7M5y5bfVdEVM+fKQl3hBRf05qMCAwEAAaMYMBYwFAYDVR0RBA0wC4IJ -bG9jYWxob3N0MA0GCSqGSIb3DQEBCwUAA4IBAQAtQ8f37cCEk7/rAcbYR53ce3iK -Vpihb0U2ni1QjG9Tg9UIExkIGkwTiCm7kwQL+GEStBu9AG/QVrOjeTriRiddhWkk -ze8kRaI3AC/63t6Vh9Q1x6PESgeE4OtAO9JpJCf4GILglA789Y/b/GF8zJZQxR13 -qpB4ZwWw7gCBhdEW59u6CFeBmfDa58hM8lWvuVoRrTi7bjUeC6PAn5HVMzZSykhu -4HaUfBp6bKFjuym2+h/VvM1n8C3chjVSmutsLb6ELdD8IK0vPV/yf5+LN256zSsS -dyUZYd8XwQaioEMKdbhLvnehyzHiWfQIUR3BdhONxoIJhHv/EAo8eCkHHYIF ------END CERTIFICATE----- diff --git a/Lib/test/keycert.pem b/Lib/test/keycert.pem deleted file mode 100644 index 9545dcf4b9..0000000000 --- a/Lib/test/keycert.pem +++ /dev/null @@ -1,48 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQCr77F9oBLYuLIb -3t4mDHPoxOEnu1h7NQaJdn9tR/KjW6AzhXXm9USh45qt3sR6Wo8sDlBIJ3vZyhyW -P939qYUeGOZJahupYi4IkqVckZXARm3k9qhAI/aC4ixVFyxnxg3bNpk8Ir0AyyQV -oudY+33+ZNj6+fHzeboGDJ9uE/RTcp9JqE/qo0haATZufJfY63ZCUpYFn6j5W4jG -vpqg5/0hba8Cxdk62387uNknfVHSMzGkkq82zbBpud6TYQofCp3VlEPBjz9iLCz6 -FFOKgLZmbk8QsdktXF6zfRJJk+vbZTh/OGH0p/eiIfW1kXOzOcuW31XRFTPnykJd -4QUX9OajAgMBAAECggEAHppmXDbuw9Z0FVPg9KLIysioTtsgz6VLiZIm8juZK4x2 -glUh/D7xvWL2uDXrgN+3lh7iGUW13LkFx5SMncbbo9TIwI57Z/XKvcnkVwquve+L -RfLFVc1Q5lD9lROv2rS86KTaN4LzYz3FKXi6dvMkpPAsUtfEQhMLkmISypQQq/1z -EJaqo7r85OjN7e0wKazlKZpOzJEa5FQLMVRjTRFhLFNbHXX/tAet2jw+umATKbw8 -hYgiuZ44TwSEd9JeIV/oSYWfI/3HetuYW0ru3caiztRF2NySNu8lcsWgNC7fIku9 -mcHjtSNzs91QN1Qlu7GQvvhpt6OWDirNDCW+49WGaQKBgQDg9SDhfF0jRYslgYbH -cqO4ggaFdHjrAAYpwnAgvanhFZL/zEqm5G1E7l/e2fCkJ9VOSFO0A208chvwMcr+ -dCjHE2tVdE81aQ2v/Eo83VdS1RcOV4Y75yPH48rMhxPaHvxWD/FFDbf0/P2mtPB7 -SZ3kIeZMkE1wxdaO3AKUbQoozwKBgQDDqYgg7kVtygyICE1mB8Hwp6nUxFTczG7y -4XcsDqMIrKmw+PbQluvkoHoStxeVrsTloDhkTjIrpmYLyAiazg+PUJdkd6xrfLSj -VV6X93W0S/1egEb1F1CGFxtk8v/PWH4K76EPL2vxXdxjywz3GWlrL9yDYaB2szzS -DqgwVMqx7QKBgDCD7UF0Bsoyl13RX3XoPXLvZ+SkR+e2q52Z94C4JskKVBeiwX7Y -yNAS8M4pBoMArDoj0xmBm69rlKbqtjLGbnzwrTdSzDpim7cWnBQgUFLm7gAD1Elb -AhZ8BCK0Bw4FnLoa2hfga4oEfdfUMgEE0W5/+SEOBgWKRUmuHUhRc911AoGAY2EN -YmSDYSM5wDIvVb5k9B3EtevOiqNPSw/XnsoEZtiEC/44JnQxdltIBY93bDBrk5IQ -cmoBM4h91kgQjshQwOMXMhFSwvmBKmCm/hrTbvMVytTutXfVD3ZXFKwT4DW7N0TF -ElhsxBh/YzRz7mG62JVjtFt2zDN3ld2Z8YpvtXUCgYEA4EJ4ObS5YyvcXAKHJFo6 -Fxmavyrf8LSm3MFA65uSnFvWukMVqqRMReQc5jvpxHKCis+XvnHzyOfL0gW9ZTi7 -tWGGbBi0TRJCa8BkvgngUZxOxUlMfg/7cVxOIB0TPoUSgxFd/+qVz4GZMvr0dPu7 -eAF7J/8ECVvb0wSPTUI1N3c= ------END PRIVATE KEY----- ------BEGIN CERTIFICATE----- -MIIDWTCCAkGgAwIBAgIJAPm6B21bar2bMA0GCSqGSIb3DQEBCwUAMF8xCzAJBgNV -BAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9u -IFNvZnR3YXJlIEZvdW5kYXRpb24xEjAQBgNVBAMMCWxvY2FsaG9zdDAeFw0xODAx -MTkxOTA5MDZaFw0yODAxMTcxOTA5MDZaMF8xCzAJBgNVBAYTAlhZMRcwFQYDVQQH -DA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9uIFNvZnR3YXJlIEZvdW5k -YXRpb24xEjAQBgNVBAMMCWxvY2FsaG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEP -ADCCAQoCggEBAKvvsX2gEti4shve3iYMc+jE4Se7WHs1Bol2f21H8qNboDOFdeb1 -RKHjmq3exHpajywOUEgne9nKHJY/3f2phR4Y5klqG6liLgiSpVyRlcBGbeT2qEAj -9oLiLFUXLGfGDds2mTwivQDLJBWi51j7ff5k2Pr58fN5ugYMn24T9FNyn0moT+qj -SFoBNm58l9jrdkJSlgWfqPlbiMa+mqDn/SFtrwLF2Trbfzu42Sd9UdIzMaSSrzbN -sGm53pNhCh8KndWUQ8GPP2IsLPoUU4qAtmZuTxCx2S1cXrN9EkmT69tlOH84YfSn -96Ih9bWRc7M5y5bfVdEVM+fKQl3hBRf05qMCAwEAAaMYMBYwFAYDVR0RBA0wC4IJ -bG9jYWxob3N0MA0GCSqGSIb3DQEBCwUAA4IBAQAtQ8f37cCEk7/rAcbYR53ce3iK -Vpihb0U2ni1QjG9Tg9UIExkIGkwTiCm7kwQL+GEStBu9AG/QVrOjeTriRiddhWkk -ze8kRaI3AC/63t6Vh9Q1x6PESgeE4OtAO9JpJCf4GILglA789Y/b/GF8zJZQxR13 -qpB4ZwWw7gCBhdEW59u6CFeBmfDa58hM8lWvuVoRrTi7bjUeC6PAn5HVMzZSykhu -4HaUfBp6bKFjuym2+h/VvM1n8C3chjVSmutsLb6ELdD8IK0vPV/yf5+LN256zSsS -dyUZYd8XwQaioEMKdbhLvnehyzHiWfQIUR3BdhONxoIJhHv/EAo8eCkHHYIF ------END CERTIFICATE----- diff --git a/Lib/test/keycert2.pem b/Lib/test/keycert2.pem deleted file mode 100644 index bb5fa65a8a..0000000000 --- a/Lib/test/keycert2.pem +++ /dev/null @@ -1,49 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIEvwIBADANBgkqhkiG9w0BAQEFAASCBKkwggSlAgEAAoIBAQC3ulRNfhbOAey/ -B+wIVYx+d5az7EV4riR6yi/qE6G+bxbTvay2pqySHtDweuaYSh2cVmcasBKKIFJm -rCD1zR8UmLb5i2XFIina1t3eePCuBZMrvZZwkzlQUSM1AZtjGOO/W0I3FwO6y645 -9xA5PduKI7SMYkH/VL3zE5W1JwMovv6bvNiT+GU5l6mB9ylCTgLpmUqoQhRqz/35 -zCzVyoh+ppDvVcpWYfvXywsXsgQwbAF0QJm8SSFi0TZm5ykv4WE16afQp08yuZS0 -3U4K3MJCa4rxO58edcxBopWYfQ29K3iINM8enRfr5q+u5mAAbALAEEvyFjgLWl/u -7arxn7bJAgMBAAECggEBAJfMt8KfHzBunrDnVrk8FayYGkfmOzAOkc1yKEx6k/TH -zFB+Mqlm5MaF95P5t3S0J+r36JBAUdEWC38RUNpF9BwMYYGlDxzlsTdCuGYL/q+J -o6NMLXQt7/jQUQqGnWAvPFzqhbcGqOo5R2ZVH25sEWv9PDuRI35XAepIkDTwWsfa -P6UcJJoP+4v9B++fb3sSL4zNwp1BqS4wxR8YTR0t1zQqOxJ5BGPw1J8aBMs1sq5t -qyosAQAT63kLrdqWotHaM26QxjqEQUMlh12XMWb5GdBXUxbvyGtEabsqskGa/f8B -RdHE437J8D8l+jxb2mZLzrlaH3dq2tbFGCe1rT8qLRECgYEA5CWIvoD/YnQydLGA -OlEhCSocqURuqcotg9Ev0nt/C60jkr/NHFLGppz9lhqjIDjixt3sIMGZMFzxRtwM -pSYal3XiR7rZuHau9iM35yDhpuytEiGbYy1ADakJRzY5jq/Qa8RfPP9Atua5xAeP -q6DiSnq9vhHv9G+O4MxzHBmrw9sCgYEAziiJWFthcwvuXn3Jv9xFYKEb/06puZAx -EgQCz/3rPzv5fmGD/sKVo1U/K4z/eA82DNeKG8QRTFJCxT8TCNRxOmGV7HdCYo/B -4BTNNvbKcdi3l0j75kKoADg+nt5CD5lz6gLG0GrUEnVO1y5HVfCTb3BEAfa36C85 -9i0sfQGiwysCgYEAuus9k8cgdct5oz3iLuVVSark/JGCkT2B+OOkaLChsDFUWeEm -7TOsaclpwldkmvvAYOplkZjMJ2GelE2pVo1XcAw3LkmaI5WpVyQXoxe/iQGT8qzy -IFlsh0Scw2lb0tmcyw6CcPk4TiHOxRrkzNrtS9QwLM+JZx0XVHptPPKTVc0CgYAu -j/VFYY5G/8Dc0qhIjyWUR48dQNUQtkJ/ASzpcT46z/7vznKTjbtiYpSb74KbyUO5 -7sygrM4DYOj3x+Eys1jHiNbly6HQxQtS4x/edCsRP5NntfI+9XsgYZOzKhvdjhki -F3J0DEzNxnUCIM+311hVaRPTJbgv1srOkTFlIoNydQKBgQC6/OHGaC/OewQqRlRK -Mg5KZm01/pk4iKrpA5nG7OTAeoa70NzXNtG8J3WnaJ4mWanNwNUOyRMAMrsUAy9q -EeGqHM5mMFpY4TeVuNLL21lu/x3KYw6mKL3Ctinn+JLAoYoqEy8deZnEA5/tjYlz -YhFBchnUicjoUN1chdpM6SpV2Q== ------END PRIVATE KEY----- ------BEGIN CERTIFICATE----- -MIIDYjCCAkqgAwIBAgIJALJXRr8qF6oIMA0GCSqGSIb3DQEBCwUAMGIxCzAJBgNV -BAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9u -IFNvZnR3YXJlIEZvdW5kYXRpb24xFTATBgNVBAMMDGZha2Vob3N0bmFtZTAeFw0x -ODAxMTkxOTA5MDZaFw0yODAxMTcxOTA5MDZaMGIxCzAJBgNVBAYTAlhZMRcwFQYD -VQQHDA5DYXN0bGUgQW50aHJheDEjMCEGA1UECgwaUHl0aG9uIFNvZnR3YXJlIEZv -dW5kYXRpb24xFTATBgNVBAMMDGZha2Vob3N0bmFtZTCCASIwDQYJKoZIhvcNAQEB -BQADggEPADCCAQoCggEBALe6VE1+Fs4B7L8H7AhVjH53lrPsRXiuJHrKL+oTob5v -FtO9rLamrJIe0PB65phKHZxWZxqwEoogUmasIPXNHxSYtvmLZcUiKdrW3d548K4F -kyu9lnCTOVBRIzUBm2MY479bQjcXA7rLrjn3EDk924ojtIxiQf9UvfMTlbUnAyi+ -/pu82JP4ZTmXqYH3KUJOAumZSqhCFGrP/fnMLNXKiH6mkO9VylZh+9fLCxeyBDBs -AXRAmbxJIWLRNmbnKS/hYTXpp9CnTzK5lLTdTgrcwkJrivE7nx51zEGilZh9Db0r -eIg0zx6dF+vmr67mYABsAsAQS/IWOAtaX+7tqvGftskCAwEAAaMbMBkwFwYDVR0R -BBAwDoIMZmFrZWhvc3RuYW1lMA0GCSqGSIb3DQEBCwUAA4IBAQCZhHhGItpkqhEq -ntMRd6Hv0GoOJixNvgeMwK4NJSRT/no3OirtUTzccn46h+SWibSa2eVssAV+pAVJ -HbzkN/DH27A1mMx1zJL1ekcOKA1AF6MXhUnrUGXMqW36YNtzHfXJLrwvpLJ13OQg -/Kxo4Nw68bGzM+PyRtKU/mpgYyfcvwR+ZSeIDh1fvUZK/IEVCf8ub42GPVs5wPfv -M+k5aHxWTxeif3K1byTRzxHupYNG2yWO4XEdnBGOuOwzzN4/iQyNcsuQKeuKHGrt -YvIlG/ri04CQ7xISZCj74yjTZ+/A2bXre2mQXAHqKPumHL7cl34+erzbUaxYxbTE -u5FcOmLQ ------END CERTIFICATE----- diff --git a/Lib/test/keycert3.pem b/Lib/test/keycert3.pem deleted file mode 100644 index 621eb08bb0..0000000000 --- a/Lib/test/keycert3.pem +++ /dev/null @@ -1,132 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDgV4G+Zzf2DT5n -oAisIGFhn/bz7Vn5WiXUqbDsxROJOh/7BtOlduZka0pPhFylGbnxS8l1kEWHRI2T -6hOoWzumB6ItKiN+T5J30lAvSyo7iwdFoAQ/S5nPXQfhNARQe/NEOhRtpcuNdyx4 -BWdPdPuJQA1ASNJCLwcLOoRxaLbKLvb2V5T5FCAkeNPtRvPuT4gKQItMmiHfAhoV -C8MZWF/GC0RukHINys5MwqeFexam8CznmQPMYrLdhmKTj3DTivCPoh97EDIFGlgZ -SCaaYDVQA+aqlo/q2pi52PFwC1KzhNEA7EeOqSwC1NQjwjHuhcnf9WbxrgTq2zh3 -rv5YEW2ZAgMBAAECggEAPfSMtTumPcJskIumuXp7yk02EyliZrWZqwBuBwVqHsS5 -nkbFXnXWrLbgn9MrDsFrE5NdgKUmPnQVMVs8sIr5jyGejSCNCs4I4iRn1pfIgwcj -K/xEEALd6GGF0pDd/CgvB5GOoLVf4KKf2kmLvWrOKJpSzoUN5A8+v8AaYYOMr4sC -czbvfGomzEIewEG+Rw9zOVUDlmwyEKPQZ47E7PQ+EEA7oeFdR+1Zj6eT9ndegf8B -54frySYCLRUCk/sHCpWhaJBtBrcpht7Y8CfY7hiH/7x866fvuLnYPz4YALtUb0wN -7zUCNS9ol3n4LbjFFKfZtiRjKaCBRzMjK0rz6ydFcQKBgQDyLI3oGbnW73vqcDe/ -6eR0w++fiCAVhfMs3AO/gOaJi2la2JHlJ5u+cIHQIOFwEhn6Zq0AtdmnFx1TS5IQ -C0MdXI0XoQQw7rEF8EJcvfe85Z0QxENVhzydtdb8QpJfnQGfBfLyQlaaRYzRRHB6 -VdYUHF3EIPVIhbjbghal+Qep/QKBgQDtJlRPHkWwTMevu0J0fYbWN1ywtVTFUR// -k7VyORSf8yuuSnaQRop4cbcqONxmDKH6Or1fl3NYBsAxtXkkOK1E2OZNo2sfQdRa -wpA7o7mPHRhztQFpT5vflp+8P6+PEFat8D04eBOhNwrwwfhiPjD4gv5KvN4XutRW -VWv/2pnmzQKBgHPvHGg2mJ7quvm6ixXW1MWJX1eSBToIjCe3lBvDi5nhIaiZ8Q4w -7gA3QA3xD7tlDwauzLeAVxgEmsdbcCs6GQEfY3QiYy1Bt4FOSZa4YrcNfSmfq1Rw -j3Y4rRjKjeQz96i3YlzToT3tecJc7zPBj+DEy6au2H3Fdn+vQURneWHJAoGBANG7 -XES8mRVaUh/wlM1BVsaNH8SIGfiHzqzRjV7/bGYpQTBbWpAuUrhCmaMVtpXqBjav -TFwGLVRkZAWSYRjPpy2ERenT5SE3rv61o6mbGrifGsj6A82HQmtzYsGx8SmtYXtj -REF0sKebbmmOooUAS379GrguYJzL9o6D7YfRZNrhAoGAVfb/tiFU4S67DSpYpQey -ULhgfsFpDByICY6Potsg67gVFf9jIaB83NPTx3u/r6sHFgxFw7lQsuZcgSuWMu7t -glzOXVIP11Y5sl5CJ5OsfeK1/0umMZF5MWPyAQCx/qrPlZL86vXjt24Y/VaOxsAi -CZYdyJsjgOrJrWoMbo5ta54= ------END PRIVATE KEY----- -Certificate: - Data: - Version: 3 (0x2) - Serial Number: - 82:ed:bf:41:c8:80:91:9c - Signature Algorithm: sha1WithRSAEncryption - Issuer: C=XY, O=Python Software Foundation CA, CN=our-ca-server - Validity - Not Before: Jan 19 19:09:06 2018 GMT - Not After : Nov 28 19:09:06 2027 GMT - Subject: C=XY, L=Castle Anthrax, O=Python Software Foundation, CN=localhost - Subject Public Key Info: - Public Key Algorithm: rsaEncryption - Public-Key: (2048 bit) - Modulus: - 00:e0:57:81:be:67:37:f6:0d:3e:67:a0:08:ac:20: - 61:61:9f:f6:f3:ed:59:f9:5a:25:d4:a9:b0:ec:c5: - 13:89:3a:1f:fb:06:d3:a5:76:e6:64:6b:4a:4f:84: - 5c:a5:19:b9:f1:4b:c9:75:90:45:87:44:8d:93:ea: - 13:a8:5b:3b:a6:07:a2:2d:2a:23:7e:4f:92:77:d2: - 50:2f:4b:2a:3b:8b:07:45:a0:04:3f:4b:99:cf:5d: - 07:e1:34:04:50:7b:f3:44:3a:14:6d:a5:cb:8d:77: - 2c:78:05:67:4f:74:fb:89:40:0d:40:48:d2:42:2f: - 07:0b:3a:84:71:68:b6:ca:2e:f6:f6:57:94:f9:14: - 20:24:78:d3:ed:46:f3:ee:4f:88:0a:40:8b:4c:9a: - 21:df:02:1a:15:0b:c3:19:58:5f:c6:0b:44:6e:90: - 72:0d:ca:ce:4c:c2:a7:85:7b:16:a6:f0:2c:e7:99: - 03:cc:62:b2:dd:86:62:93:8f:70:d3:8a:f0:8f:a2: - 1f:7b:10:32:05:1a:58:19:48:26:9a:60:35:50:03: - e6:aa:96:8f:ea:da:98:b9:d8:f1:70:0b:52:b3:84: - d1:00:ec:47:8e:a9:2c:02:d4:d4:23:c2:31:ee:85: - c9:df:f5:66:f1:ae:04:ea:db:38:77:ae:fe:58:11: - 6d:99 - Exponent: 65537 (0x10001) - X509v3 extensions: - X509v3 Subject Alternative Name: - DNS:localhost - X509v3 Key Usage: critical - Digital Signature, Key Encipherment - X509v3 Extended Key Usage: - TLS Web Server Authentication, TLS Web Client Authentication - X509v3 Basic Constraints: critical - CA:FALSE - X509v3 Subject Key Identifier: - 85:11:BE:16:47:04:D1:30:EE:86:8A:18:70:BE:A8:28:6F:82:3D:CE - X509v3 Authority Key Identifier: - keyid:9A:CF:CF:6E:EB:71:3D:DB:3C:F1:AE:88:6B:56:72:03:CB:08:A7:48 - DirName:/C=XY/O=Python Software Foundation CA/CN=our-ca-server - serial:82:ED:BF:41:C8:80:91:9B - - Authority Information Access: - CA Issuers - URI:http://testca.pythontest.net/testca/pycacert.cer - OCSP - URI:http://testca.pythontest.net/testca/ocsp/ - - X509v3 CRL Distribution Points: - - Full Name: - URI:http://testca.pythontest.net/testca/revocation.crl - - Signature Algorithm: sha1WithRSAEncryption - 7f:a1:7e:3e:68:01:b0:32:b8:57:b8:03:68:13:13:b3:e3:f4: - 70:2f:15:e5:0f:87:b9:fd:e0:12:e3:16:f2:91:53:c7:4e:25: - af:ca:cb:a7:d9:9d:57:4d:bf:a2:80:d4:78:aa:04:31:fd:6d: - cc:6d:82:43:e9:62:16:0d:0e:26:8b:e7:f1:3d:57:5c:68:02: - 9c:2b:b6:c9:fd:62:2f:10:85:88:cc:44:a5:e7:a2:3e:89:f2: - 1f:02:6a:3f:d0:3c:6c:24:2d:bc:51:62:7a:ec:25:c5:86:87: - 77:35:8f:f9:7e:d0:17:3d:77:56:bf:1a:0c:be:09:78:ee:ea: - 73:97:65:60:94:91:35:b3:5c:46:8a:5e:6d:94:52:de:48:b7: - 1f:6c:28:79:7f:ff:08:8d:e4:7d:d0:b9:0b:7c:ae:c4:1d:2a: - a1:b3:50:11:82:03:5e:6c:e7:26:fa:05:32:39:07:83:49:b9: - a2:fa:04:da:0d:e5:ff:4c:db:97:d0:c3:a7:43:37:4c:16:de: - 3c:b5:e9:7e:82:d4:b3:10:df:d1:c1:66:72:9c:15:67:19:3b: - 7b:91:0a:82:07:67:c5:06:03:5f:80:54:08:81:8a:b1:5c:7c: - 4c:d2:07:38:92:eb:12:f5:71:ae:de:05:15:c8:e1:33:f0:e4: - 96:0f:0f:1e ------BEGIN CERTIFICATE----- -MIIE8TCCA9mgAwIBAgIJAILtv0HIgJGcMA0GCSqGSIb3DQEBBQUAME0xCzAJBgNV -BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW -MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xODAxMTkxOTA5MDZaFw0yNzExMjgx -OTA5MDZaMF8xCzAJBgNVBAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEj -MCEGA1UECgwaUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24xEjAQBgNVBAMMCWxv -Y2FsaG9zdDCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAOBXgb5nN/YN -PmegCKwgYWGf9vPtWflaJdSpsOzFE4k6H/sG06V25mRrSk+EXKUZufFLyXWQRYdE -jZPqE6hbO6YHoi0qI35PknfSUC9LKjuLB0WgBD9Lmc9dB+E0BFB780Q6FG2ly413 -LHgFZ090+4lADUBI0kIvBws6hHFotsou9vZXlPkUICR40+1G8+5PiApAi0yaId8C -GhULwxlYX8YLRG6Qcg3KzkzCp4V7FqbwLOeZA8xist2GYpOPcNOK8I+iH3sQMgUa -WBlIJppgNVAD5qqWj+ramLnY8XALUrOE0QDsR46pLALU1CPCMe6Fyd/1ZvGuBOrb -OHeu/lgRbZkCAwEAAaOCAcAwggG8MBQGA1UdEQQNMAuCCWxvY2FsaG9zdDAOBgNV -HQ8BAf8EBAMCBaAwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMCMAwGA1Ud -EwEB/wQCMAAwHQYDVR0OBBYEFIURvhZHBNEw7oaKGHC+qChvgj3OMH0GA1UdIwR2 -MHSAFJrPz27rcT3bPPGuiGtWcgPLCKdIoVGkTzBNMQswCQYDVQQGEwJYWTEmMCQG -A1UECgwdUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24gQ0ExFjAUBgNVBAMMDW91 -ci1jYS1zZXJ2ZXKCCQCC7b9ByICRmzCBgwYIKwYBBQUHAQEEdzB1MDwGCCsGAQUF -BzAChjBodHRwOi8vdGVzdGNhLnB5dGhvbnRlc3QubmV0L3Rlc3RjYS9weWNhY2Vy -dC5jZXIwNQYIKwYBBQUHMAGGKWh0dHA6Ly90ZXN0Y2EucHl0aG9udGVzdC5uZXQv -dGVzdGNhL29jc3AvMEMGA1UdHwQ8MDowOKA2oDSGMmh0dHA6Ly90ZXN0Y2EucHl0 -aG9udGVzdC5uZXQvdGVzdGNhL3Jldm9jYXRpb24uY3JsMA0GCSqGSIb3DQEBBQUA -A4IBAQB/oX4+aAGwMrhXuANoExOz4/RwLxXlD4e5/eAS4xbykVPHTiWvysun2Z1X -Tb+igNR4qgQx/W3MbYJD6WIWDQ4mi+fxPVdcaAKcK7bJ/WIvEIWIzESl56I+ifIf -Amo/0DxsJC28UWJ67CXFhod3NY/5ftAXPXdWvxoMvgl47upzl2VglJE1s1xGil5t -lFLeSLcfbCh5f/8IjeR90LkLfK7EHSqhs1ARggNebOcm+gUyOQeDSbmi+gTaDeX/ -TNuX0MOnQzdMFt48tel+gtSzEN/RwWZynBVnGTt7kQqCB2fFBgNfgFQIgYqxXHxM -0gc4kusS9XGu3gUVyOEz8OSWDw8e ------END CERTIFICATE----- diff --git a/Lib/test/keycert4.pem b/Lib/test/keycert4.pem deleted file mode 100644 index b7df7f3f2c..0000000000 --- a/Lib/test/keycert4.pem +++ /dev/null @@ -1,132 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIIEvQIBADANBgkqhkiG9w0BAQEFAASCBKcwggSjAgEAAoIBAQDH/76hZAZH4cSV -CmVZa5HEqKCjCKrcPwBECs9BS+3ibwN4x9NnFNP+tCeFGgJXl7WGFoeXgg3oK+1p -FsOWpsRHuF3BdqkCnShSydmT8bLaGHwKeL0cPxJP5T/uW7ezPKW2VWXGMwmwRaRJ -9dj2VCUu20vDZWSGFr9zjnjoJczBtH3RsVUgpK7euEHuQ5pIM9QSOaCo+5FPR7s7 -1nU7YqbFWtd+NhC8Og1G497B31DQlHciF6BRm6/cNGAmHaAErKUGBFdkGtFPHBn4 -vktoEg9fwxJAZLvGpoTZWrB4HRsRwVTmFdGvK+JXK225xF23AXRXp/snhSuSFeLj -E5cpyJJ7AgMBAAECggEAQOv527X2e/sDr0XSpHZQuT/r9UBpBlnFIlFH+fBF5k0X -GWv0ae/O6U1dzs0kmX57xG0n0ry6+vTXeleTYiH8cTOd66EzN9AAOO+hG29IGZf9 -HAEZkkO/FARc/mjzdtFnEYsjIHWM3ZWdwQx3Q28JKu6w51rQiN51g3NqOCGdF/uF -rE5XPKsKndn+nLHvsNuApFgUYZEwdrozgUueEgRaPTUCNhzotcA9eWoBdA24XNhk -x8Cm/bZWabXm7gBO75zl3Cu2F21ay+EuwyOZTsx6lZi6YX9/zo1mkO81Zi3tQk50 -NMEI0feLNwsdxTbmOcVJadjOgd+QVghlFyr5HGBWMQKBgQD3AH3rhnAo6tOyNkGN -+IzIU1MhUS452O7IavykUYO9sM24BVChpRtlI9Dpev4yE/q3BAO3+oWT3cJrN7/3 -iyo1dzAkpGvI65XWfElXFM4nLjEiZzx4W9fiPN91Oucpr0ED6+BZXTtz4gVm0TP/ -TUc2xvTB6EKvIyWmKOYEi0snxQKBgQDPSOjbz9jWOrC9XY7PmtLB6QJDDz7XSGVK -wzD+gDAPpAwhk58BEokdOhBx2Lwl8zMJi0CRHgH2vNvkRyhvUQ4UFzisrqann/Tw -klp5sw3iWC6ERC8z9zL7GfHs7sK3mOVeAdK6ffowPM3JrZ2vPusVBdr0MN3oZwki -CtNXqbY1PwKBgGheQNbAW6wubX0kB9chavtKmhm937Z5v4vYCSC1gOEqUAKt3EAx -L74wwBmn6rjmUE382EVpCgBM99WuHONQXmlxD1qsTw763LlgkuzE0cckcYaD8L06 -saHa7uDuHrcyYlpx1L5t8q0ol/e19i6uTKUMtGcq6OJwC3yGU4sgAIWxAoGBAMVq -qiQXm2vFL+jafxYoXUvDMJ1PmskMsTP4HOR2j8+FrOwZnVk3HxGP6HOVOPRn4JbZ -YiAT1Uj6a+7I+rCyINdvmlGUcTK6fFzW9oZryvBkjcD483/pkktmVWwTpa2YV/Ml -h16IdsyUTGYlDUYHhXtbPUJOfDpIT4F1j/0wrFGfAoGAO82BcUsehEUQE0xvQLIn -7QaFtUI5z19WW730jVuEobiYlh9Ka4DPbKMvka8MwyOxEwhk39gZQavmfG6+wZm+ -kjERU23LhHziJGWS2Um4yIhC7myKbWaLzjHEq72dszLpQku4BzE5fT60fxI7cURD -WGm/Z3Q2weS3ZGIoMj1RNPI= ------END PRIVATE KEY----- -Certificate: - Data: - Version: 3 (0x2) - Serial Number: - 82:ed:bf:41:c8:80:91:9d - Signature Algorithm: sha1WithRSAEncryption - Issuer: C=XY, O=Python Software Foundation CA, CN=our-ca-server - Validity - Not Before: Jan 19 19:09:06 2018 GMT - Not After : Nov 28 19:09:06 2027 GMT - Subject: C=XY, L=Castle Anthrax, O=Python Software Foundation, CN=fakehostname - Subject Public Key Info: - Public Key Algorithm: rsaEncryption - Public-Key: (2048 bit) - Modulus: - 00:c7:ff:be:a1:64:06:47:e1:c4:95:0a:65:59:6b: - 91:c4:a8:a0:a3:08:aa:dc:3f:00:44:0a:cf:41:4b: - ed:e2:6f:03:78:c7:d3:67:14:d3:fe:b4:27:85:1a: - 02:57:97:b5:86:16:87:97:82:0d:e8:2b:ed:69:16: - c3:96:a6:c4:47:b8:5d:c1:76:a9:02:9d:28:52:c9: - d9:93:f1:b2:da:18:7c:0a:78:bd:1c:3f:12:4f:e5: - 3f:ee:5b:b7:b3:3c:a5:b6:55:65:c6:33:09:b0:45: - a4:49:f5:d8:f6:54:25:2e:db:4b:c3:65:64:86:16: - bf:73:8e:78:e8:25:cc:c1:b4:7d:d1:b1:55:20:a4: - ae:de:b8:41:ee:43:9a:48:33:d4:12:39:a0:a8:fb: - 91:4f:47:bb:3b:d6:75:3b:62:a6:c5:5a:d7:7e:36: - 10:bc:3a:0d:46:e3:de:c1:df:50:d0:94:77:22:17: - a0:51:9b:af:dc:34:60:26:1d:a0:04:ac:a5:06:04: - 57:64:1a:d1:4f:1c:19:f8:be:4b:68:12:0f:5f:c3: - 12:40:64:bb:c6:a6:84:d9:5a:b0:78:1d:1b:11:c1: - 54:e6:15:d1:af:2b:e2:57:2b:6d:b9:c4:5d:b7:01: - 74:57:a7:fb:27:85:2b:92:15:e2:e3:13:97:29:c8: - 92:7b - Exponent: 65537 (0x10001) - X509v3 extensions: - X509v3 Subject Alternative Name: - DNS:fakehostname - X509v3 Key Usage: critical - Digital Signature, Key Encipherment - X509v3 Extended Key Usage: - TLS Web Server Authentication, TLS Web Client Authentication - X509v3 Basic Constraints: critical - CA:FALSE - X509v3 Subject Key Identifier: - F8:76:79:CB:11:85:F0:46:E5:95:E6:7E:69:CB:12:5E:4E:AA:EC:4D - X509v3 Authority Key Identifier: - keyid:9A:CF:CF:6E:EB:71:3D:DB:3C:F1:AE:88:6B:56:72:03:CB:08:A7:48 - DirName:/C=XY/O=Python Software Foundation CA/CN=our-ca-server - serial:82:ED:BF:41:C8:80:91:9B - - Authority Information Access: - CA Issuers - URI:http://testca.pythontest.net/testca/pycacert.cer - OCSP - URI:http://testca.pythontest.net/testca/ocsp/ - - X509v3 CRL Distribution Points: - - Full Name: - URI:http://testca.pythontest.net/testca/revocation.crl - - Signature Algorithm: sha1WithRSAEncryption - 6d:50:8d:fb:ee:4e:93:8b:eb:47:56:ba:38:cc:80:e1:9d:c7: - e1:9e:1f:9c:22:0c:d2:08:9b:ed:bf:31:d9:00:ee:af:8c:56: - 78:92:d1:7c:ba:4e:81:7f:82:1f:f4:68:99:86:91:c6:cb:57: - d3:b9:41:12:fa:75:53:fd:22:32:21:50:af:6b:4c:b1:34:36: - d1:a8:25:0a:d0:f0:f8:81:7d:69:58:6e:af:e3:d2:c4:32:87: - 79:d7:cd:ad:0c:56:f3:15:27:10:0c:f9:57:59:53:00:ed:af: - 5d:4d:07:86:7a:e5:f3:97:88:bc:86:b4:f1:17:46:33:55:28: - 66:7b:70:d3:a5:12:b9:4f:c7:ed:e6:13:20:2d:f0:9e:ec:17: - 64:cf:fd:13:14:1b:76:ba:64:ac:c5:51:b6:cd:13:0a:93:b1: - fd:43:09:a0:0b:44:6c:77:45:43:0b:e5:ed:70:b2:76:dc:08: - 4a:5b:73:5f:c1:fc:7f:63:70:f8:b9:ca:3c:98:06:5f:fd:98: - d1:e4:e6:61:5f:09:8f:6c:18:86:98:9c:cb:3f:73:7b:3f:38: - f5:a7:09:20:ee:a5:63:1c:ff:8b:a6:d1:8c:e8:f4:84:3d:99: - 38:0f:cc:e0:52:03:f9:18:05:23:76:39:de:52:ce:8e:fb:a6: - 6e:f5:4f:c3 ------BEGIN CERTIFICATE----- -MIIE9zCCA9+gAwIBAgIJAILtv0HIgJGdMA0GCSqGSIb3DQEBBQUAME0xCzAJBgNV -BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW -MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xODAxMTkxOTA5MDZaFw0yNzExMjgx -OTA5MDZaMGIxCzAJBgNVBAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEj -MCEGA1UECgwaUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24xFTATBgNVBAMMDGZh -a2Vob3N0bmFtZTCCASIwDQYJKoZIhvcNAQEBBQADggEPADCCAQoCggEBAMf/vqFk -BkfhxJUKZVlrkcSooKMIqtw/AEQKz0FL7eJvA3jH02cU0/60J4UaAleXtYYWh5eC -Degr7WkWw5amxEe4XcF2qQKdKFLJ2ZPxstoYfAp4vRw/Ek/lP+5bt7M8pbZVZcYz -CbBFpEn12PZUJS7bS8NlZIYWv3OOeOglzMG0fdGxVSCkrt64Qe5Dmkgz1BI5oKj7 -kU9HuzvWdTtipsVa1342ELw6DUbj3sHfUNCUdyIXoFGbr9w0YCYdoASspQYEV2Qa -0U8cGfi+S2gSD1/DEkBku8amhNlasHgdGxHBVOYV0a8r4lcrbbnEXbcBdFen+yeF -K5IV4uMTlynIknsCAwEAAaOCAcMwggG/MBcGA1UdEQQQMA6CDGZha2Vob3N0bmFt -ZTAOBgNVHQ8BAf8EBAMCBaAwHQYDVR0lBBYwFAYIKwYBBQUHAwEGCCsGAQUFBwMC -MAwGA1UdEwEB/wQCMAAwHQYDVR0OBBYEFPh2ecsRhfBG5ZXmfmnLEl5OquxNMH0G -A1UdIwR2MHSAFJrPz27rcT3bPPGuiGtWcgPLCKdIoVGkTzBNMQswCQYDVQQGEwJY -WTEmMCQGA1UECgwdUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24gQ0ExFjAUBgNV -BAMMDW91ci1jYS1zZXJ2ZXKCCQCC7b9ByICRmzCBgwYIKwYBBQUHAQEEdzB1MDwG -CCsGAQUFBzAChjBodHRwOi8vdGVzdGNhLnB5dGhvbnRlc3QubmV0L3Rlc3RjYS9w -eWNhY2VydC5jZXIwNQYIKwYBBQUHMAGGKWh0dHA6Ly90ZXN0Y2EucHl0aG9udGVz -dC5uZXQvdGVzdGNhL29jc3AvMEMGA1UdHwQ8MDowOKA2oDSGMmh0dHA6Ly90ZXN0 -Y2EucHl0aG9udGVzdC5uZXQvdGVzdGNhL3Jldm9jYXRpb24uY3JsMA0GCSqGSIb3 -DQEBBQUAA4IBAQBtUI377k6Ti+tHVro4zIDhncfhnh+cIgzSCJvtvzHZAO6vjFZ4 -ktF8uk6Bf4If9GiZhpHGy1fTuUES+nVT/SIyIVCva0yxNDbRqCUK0PD4gX1pWG6v -49LEMod5182tDFbzFScQDPlXWVMA7a9dTQeGeuXzl4i8hrTxF0YzVShme3DTpRK5 -T8ft5hMgLfCe7Bdkz/0TFBt2umSsxVG2zRMKk7H9QwmgC0Rsd0VDC+XtcLJ23AhK -W3Nfwfx/Y3D4uco8mAZf/ZjR5OZhXwmPbBiGmJzLP3N7Pzj1pwkg7qVjHP+LptGM -6PSEPZk4D8zgUgP5GAUjdjneUs6O+6Zu9U/D ------END CERTIFICATE----- diff --git a/Lib/test/keycertecc.pem b/Lib/test/keycertecc.pem deleted file mode 100644 index deb484f992..0000000000 --- a/Lib/test/keycertecc.pem +++ /dev/null @@ -1,96 +0,0 @@ ------BEGIN PRIVATE KEY----- -MIG2AgEAMBAGByqGSM49AgEGBSuBBAAiBIGeMIGbAgEBBDDe3QWmhZX07HZbntz4 -CFqAOaoYMdYwD7Z3WPNIc2zR7p4D6BMOa7NAWjLV5A7CUw6hZANiAAQ5IVKzLLz4 -LCfcpy6fMOp+jk5KwywsU3upPtjA6E3UetxPcfnnv+gghRyDAYLN2OVqZgLMEmUo -F1j1SM1QrbhHIuNcVxI9gPPMdumcNFSz/hqxrBRtA/8Z2gywczdNLjc= ------END PRIVATE KEY----- -Certificate: - Data: - Version: 3 (0x2) - Serial Number: - 82:ed:bf:41:c8:80:91:9e - Signature Algorithm: sha1WithRSAEncryption - Issuer: C=XY, O=Python Software Foundation CA, CN=our-ca-server - Validity - Not Before: Jan 19 19:09:06 2018 GMT - Not After : Nov 28 19:09:06 2027 GMT - Subject: C=XY, L=Castle Anthrax, O=Python Software Foundation, CN=localhost-ecc - Subject Public Key Info: - Public Key Algorithm: id-ecPublicKey - Public-Key: (384 bit) - pub: - 04:39:21:52:b3:2c:bc:f8:2c:27:dc:a7:2e:9f:30: - ea:7e:8e:4e:4a:c3:2c:2c:53:7b:a9:3e:d8:c0:e8: - 4d:d4:7a:dc:4f:71:f9:e7:bf:e8:20:85:1c:83:01: - 82:cd:d8:e5:6a:66:02:cc:12:65:28:17:58:f5:48: - cd:50:ad:b8:47:22:e3:5c:57:12:3d:80:f3:cc:76: - e9:9c:34:54:b3:fe:1a:b1:ac:14:6d:03:ff:19:da: - 0c:b0:73:37:4d:2e:37 - ASN1 OID: secp384r1 - NIST CURVE: P-384 - X509v3 extensions: - X509v3 Subject Alternative Name: - DNS:localhost-ecc - X509v3 Key Usage: critical - Digital Signature, Key Encipherment - X509v3 Extended Key Usage: - TLS Web Server Authentication, TLS Web Client Authentication - X509v3 Basic Constraints: critical - CA:FALSE - X509v3 Subject Key Identifier: - 33:23:0E:15:04:83:2E:3D:BF:DA:81:6D:10:38:80:C3:C2:B0:A4:74 - X509v3 Authority Key Identifier: - keyid:9A:CF:CF:6E:EB:71:3D:DB:3C:F1:AE:88:6B:56:72:03:CB:08:A7:48 - DirName:/C=XY/O=Python Software Foundation CA/CN=our-ca-server - serial:82:ED:BF:41:C8:80:91:9B - - Authority Information Access: - CA Issuers - URI:http://testca.pythontest.net/testca/pycacert.cer - OCSP - URI:http://testca.pythontest.net/testca/ocsp/ - - X509v3 CRL Distribution Points: - - Full Name: - URI:http://testca.pythontest.net/testca/revocation.crl - - Signature Algorithm: sha1WithRSAEncryption - 3b:6f:97:af:7e:5f:e0:14:34:ed:57:7e:de:ce:c4:85:1e:aa: - 84:52:94:7c:e5:ce:e9:9c:88:8b:ad:b5:4d:16:ac:af:81:ea: - b8:a2:e2:50:2e:cb:e9:11:bd:1b:a6:3f:0c:a2:d7:7b:67:72: - b3:43:16:ad:c6:87:ac:6e:ac:47:78:ef:2f:8c:86:e8:9b:d1: - 43:8c:c1:7a:91:30:e9:14:d6:9f:41:8b:9b:0b:24:9b:78:86: - 11:8a:fc:2b:cd:c9:13:ee:90:4f:14:33:51:a3:c4:9e:d6:06: - 48:f5:41:12:af:f0:f2:71:40:78:f5:96:c2:5d:cf:e1:38:ff: - bf:10:eb:74:2f:c2:23:21:3e:27:f5:f1:f2:af:2c:62:82:31: - 00:c8:96:1b:c3:7e:8d:71:89:e7:40:b5:67:1a:33:fb:c0:8b: - 96:0c:36:78:25:27:82:d8:27:27:52:0f:f7:69:cd:ff:2b:92: - 10:d3:d2:0a:db:65:ed:af:90:eb:db:76:f3:8a:7a:13:9e:c6: - 33:57:15:42:06:13:d6:54:49:fa:84:a7:0e:1d:14:72:ca:19: - 8e:2b:aa:a4:02:54:3c:f6:1c:23:81:7a:59:54:b0:92:65:72: - c8:e5:ba:9f:03:4e:30:f2:4d:45:85:e3:35:a8:b1:68:58:b9: - 3b:20:a3:eb ------BEGIN CERTIFICATE----- -MIIESzCCAzOgAwIBAgIJAILtv0HIgJGeMA0GCSqGSIb3DQEBBQUAME0xCzAJBgNV -BAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUgRm91bmRhdGlvbiBDQTEW -MBQGA1UEAwwNb3VyLWNhLXNlcnZlcjAeFw0xODAxMTkxOTA5MDZaFw0yNzExMjgx -OTA5MDZaMGMxCzAJBgNVBAYTAlhZMRcwFQYDVQQHDA5DYXN0bGUgQW50aHJheDEj -MCEGA1UECgwaUHl0aG9uIFNvZnR3YXJlIEZvdW5kYXRpb24xFjAUBgNVBAMMDWxv -Y2FsaG9zdC1lY2MwdjAQBgcqhkjOPQIBBgUrgQQAIgNiAAQ5IVKzLLz4LCfcpy6f -MOp+jk5KwywsU3upPtjA6E3UetxPcfnnv+gghRyDAYLN2OVqZgLMEmUoF1j1SM1Q -rbhHIuNcVxI9gPPMdumcNFSz/hqxrBRtA/8Z2gywczdNLjejggHEMIIBwDAYBgNV -HREEETAPgg1sb2NhbGhvc3QtZWNjMA4GA1UdDwEB/wQEAwIFoDAdBgNVHSUEFjAU -BggrBgEFBQcDAQYIKwYBBQUHAwIwDAYDVR0TAQH/BAIwADAdBgNVHQ4EFgQUMyMO -FQSDLj2/2oFtEDiAw8KwpHQwfQYDVR0jBHYwdIAUms/PbutxPds88a6Ia1ZyA8sI -p0ihUaRPME0xCzAJBgNVBAYTAlhZMSYwJAYDVQQKDB1QeXRob24gU29mdHdhcmUg -Rm91bmRhdGlvbiBDQTEWMBQGA1UEAwwNb3VyLWNhLXNlcnZlcoIJAILtv0HIgJGb -MIGDBggrBgEFBQcBAQR3MHUwPAYIKwYBBQUHMAKGMGh0dHA6Ly90ZXN0Y2EucHl0 -aG9udGVzdC5uZXQvdGVzdGNhL3B5Y2FjZXJ0LmNlcjA1BggrBgEFBQcwAYYpaHR0 -cDovL3Rlc3RjYS5weXRob250ZXN0Lm5ldC90ZXN0Y2Evb2NzcC8wQwYDVR0fBDww -OjA4oDagNIYyaHR0cDovL3Rlc3RjYS5weXRob250ZXN0Lm5ldC90ZXN0Y2EvcmV2 -b2NhdGlvbi5jcmwwDQYJKoZIhvcNAQEFBQADggEBADtvl69+X+AUNO1Xft7OxIUe -qoRSlHzlzumciIuttU0WrK+B6rii4lAuy+kRvRumPwyi13tncrNDFq3Gh6xurEd4 -7y+Mhuib0UOMwXqRMOkU1p9Bi5sLJJt4hhGK/CvNyRPukE8UM1GjxJ7WBkj1QRKv -8PJxQHj1lsJdz+E4/78Q63QvwiMhPif18fKvLGKCMQDIlhvDfo1xiedAtWcaM/vA -i5YMNnglJ4LYJydSD/dpzf8rkhDT0grbZe2vkOvbdvOKehOexjNXFUIGE9ZUSfqE -pw4dFHLKGY4rqqQCVDz2HCOBellUsJJlcsjlup8DTjDyTUWF4zWosWhYuTsgo+s= ------END CERTIFICATE----- diff --git a/Lib/test/list_tests.py b/Lib/test/list_tests.py index 0e11af6f36..65dfa41b26 100644 --- a/Lib/test/list_tests.py +++ b/Lib/test/list_tests.py @@ -2,13 +2,11 @@ Tests common to list and UserList.UserList """ -import unittest import sys -import os from functools import cmp_to_key -from test import support, seq_tests -from test.support import ALWAYS_EQ, NEVER_EQ +from test import seq_tests +from test.support import ALWAYS_EQ, NEVER_EQ, get_c_recursion_limit class CommonTest(seq_tests.CommonTest): @@ -33,13 +31,13 @@ def test_init(self): self.assertEqual(a, b) def test_getitem_error(self): - a = [] + a = self.type2test([]) msg = "list indices must be integers or slices" with self.assertRaisesRegex(TypeError, msg): a['a'] def test_setitem_error(self): - a = [] + a = self.type2test([]) msg = "list indices must be integers or slices" with self.assertRaisesRegex(TypeError, msg): a['a'] = "python" @@ -63,7 +61,7 @@ def test_repr(self): def test_repr_deep(self): a = self.type2test([]) - for i in range(sys.getrecursionlimit() + 100): + for i in range(get_c_recursion_limit() + 1): a = self.type2test([a]) self.assertRaises(RecursionError, repr, a) @@ -193,6 +191,14 @@ def test_setslice(self): self.assertRaises(TypeError, a.__setitem__) + def test_slice_assign_iterator(self): + x = self.type2test(range(5)) + x[0:3] = reversed(range(3)) + self.assertEqual(x, self.type2test([2, 1, 0, 3, 4])) + + x[:] = reversed(range(3)) + self.assertEqual(x, self.type2test([2, 1, 0])) + def test_delslice(self): a = self.type2test([0, 1]) del a[1:2] @@ -552,7 +558,7 @@ def test_constructor_exception_handling(self): class F(object): def __iter__(self): raise KeyboardInterrupt - self.assertRaises(KeyboardInterrupt, list, F()) + self.assertRaises(KeyboardInterrupt, self.type2test, F()) def test_exhausted_iterator(self): a = self.type2test([1, 2, 3]) @@ -564,3 +570,8 @@ def test_exhausted_iterator(self): self.assertEqual(list(exhit), []) self.assertEqual(list(empit), [9]) self.assertEqual(a, self.type2test([1, 2, 3, 9])) + + # gh-115733: Crash when iterating over exhausted iterator + exhit = iter(self.type2test([1, 2, 3])) + for _ in exhit: + next(exhit, 1) diff --git a/Lib/test/mathdata/floating_points.txt b/Lib/test/mathdata/floating_points.txt new file mode 100644 index 0000000000..539073d19d --- /dev/null +++ b/Lib/test/mathdata/floating_points.txt @@ -0,0 +1,1028 @@ +# These numbers are used to test floating point binary-to-decimal conversion. +# They are based on the TCL test suite (tests/expr.test), which is based on +# test data from: +# Brigitte Verdonk, Annie Cuyt, Dennis Verschaeren, A precision and range +# independent tool for testing floating-point arithmetic II: Conversions, +# ACM Transactions on Mathematical Software 27:2 (March 2001), pp. 119-140. + +0E0 +-0E0 +1E0 +15E-1 +125E-2 +1125E-3 +10625E-4 +103125E-5 +1015625E-6 +10078125E-7 +100390625E-8 +1001953125E-9 +10009765625E-10 +100048828125E-11 +1000244140625E-12 +10001220703125E-13 +100006103515625E-14 +1000030517578125E-15 +10000152587890625E-16 ++8E153 +-1E153 ++9E306 +-2E153 ++7E-304 +-3E-49 ++7E-303 +-6E-49 ++9E43 +-9E44 ++8E303 +-1E303 ++7E-287 +-2E-204 ++2E-205 +-9E-47 ++34E195 +-68E195 ++85E194 +-67E97 ++93E-234 +-19E-87 ++38E-87 +-38E-88 +-69E220 ++18E43 +-36E43 ++61E-99 +-43E-92 ++86E-92 +-51E-74 ++283E85 +-566E85 ++589E187 +-839E143 +-744E-234 ++930E-235 +-186E-234 ++604E175 +-302E175 ++755E174 +-151E175 ++662E-213 +-408E-74 ++510E-75 ++6782E55 +-2309E92 ++7963E34 +-3391E55 ++7903E-96 +-7611E-226 ++4907E-196 +-5547E-311 ++5311E241 +-5311E243 ++5311E242 ++9269E-45 +-8559E-289 ++8699E-276 +-8085E-64 ++74819E201 +-82081E41 ++51881E37 +-55061E157 ++77402E-215 +-33891E-92 ++38701E-215 +-82139E-76 ++75859E25 ++89509E140 +-57533E287 ++46073E-32 +-92146E-32 ++83771E-74 +-34796E-276 ++584169E229 ++164162E41 +-328324E41 ++209901E-11 +-419802E-11 ++940189E-112 +-892771E-213 ++757803E120 +-252601E120 ++252601E121 +-505202E120 ++970811E-264 +-654839E-60 ++289767E-178 +-579534E-178 +-8823691E130 ++9346704E229 +-1168338E229 +-6063369E-136 ++3865421E-225 +-5783893E-127 ++2572231E223 +-5144462E223 ++1817623E109 ++6431543E-97 +-5444097E-21 ++8076999E-121 +-9997649E-270 ++50609263E157 ++70589528E130 +-88236910E129 ++87575437E-310 +-23135572E-127 ++85900881E177 +-84863171E113 ++68761586E232 +-50464069E286 ++27869147E-248 +-55738294E-248 ++70176353E-53 +-80555086E-32 +-491080654E121 ++526250918E287 +-245540327E121 +-175150874E-310 ++350301748E-310 +-437877185E-311 ++458117166E52 +-916234332E52 ++229058583E52 +-525789935E98 ++282926897E-227 +-565853794E-227 ++667284113E-240 +-971212611E-126 ++9981396317E-182 +-5035231965E-156 ++8336960483E-153 +-8056371144E-155 ++6418488827E79 +-3981006983E252 ++7962013966E252 +-4713898551E261 ++8715380633E-58 +-9078555839E-109 ++9712126110E-127 ++42333842451E201 +-84667684902E201 ++23792120709E-315 +-78564021519E-227 ++71812054883E-188 +-30311163631E-116 ++71803914657E292 ++36314223356E-109 ++18157111678E-109 +-45392779195E-110 ++778380362293E218 +-685763015669E280 ++952918668151E70 +-548357443505E32 ++384865004907E-285 +-769730009814E-285 ++697015418417E-93 +-915654049301E-28 ++178548656339E169 +-742522891517E259 ++742522891517E258 +-357097312678E169 +-3113521449172E218 ++3891901811465E217 +-1556760724586E218 ++9997878507563E-195 +-7247563029154E-319 ++3623781514577E-319 +-3092446298323E-200 ++6363857920591E145 +-8233559360849E94 ++2689845954547E49 +-5379691909094E49 ++5560322501926E-301 +-7812878489261E-179 ++8439398533053E-256 +-2780161250963E-301 +-87605699161665E155 +-17521139832333E156 +-88218101363513E-170 ++38639244311627E-115 ++35593959807306E261 +-53390939710959E260 ++71187919614612E261 +-88984899518265E260 ++77003665618895E-73 +-15400733123779E-72 ++61602932495116E-72 +-30801466247558E-72 ++834735494917063E-300 +-589795149206434E-151 ++475603213226859E-42 +-294897574603217E-151 ++850813008001913E93 +-203449172043339E185 ++406898344086678E185 +-813796688173356E185 ++6045338514609393E244 +-5145963778954906E142 ++2572981889477453E142 +-6965949469487146E74 ++6182410494241627E-119 +-8510309498186985E-277 ++6647704637273331E-212 +-2215901545757777E-212 ++3771476185376383E276 +-3729901848043846E212 ++3771476185376383E277 +-9977830465649166E119 ++8439928496349319E-142 +-8204230082070882E-59 ++8853686434843997E-244 +-5553274272288559E-104 ++36149023611096162E144 +-36149023611096162E147 ++18074511805548081E146 +-18074511805548081E147 ++97338774138954421E-290 +-88133809804950961E-308 ++94080055902682397E-243 +-24691002732654881E-115 ++52306490527514614E49 +-26153245263757307E49 ++55188692254193604E165 +-68985865317742005E164 ++27176258005319167E-261 +-73169230107256116E-248 ++91461537634070145E-249 +-54352516010638334E-261 ++586144289638535878E280 +-601117006785295431E245 ++293072144819267939E280 +-953184713238516652E272 ++902042358290366539E-281 +-557035730189854663E-294 ++902042358290366539E-280 +-354944100507554393E-238 ++272104041512242479E199 +-816312124536727437E199 ++544208083024484958E199 +-792644927852378159E78 +-679406450132979175E-263 ++543525160106383340E-262 ++7400253695682920196E215 +-1850063423920730049E215 ++3700126847841460098E215 +-9250317119603650245E214 ++8396094300569779681E-252 +-3507665085003296281E-75 ++7015330170006592562E-75 +-7015330170006592562E-74 ++7185620434951919351E205 +-1360520207561212395E198 ++2178999185345151731E-184 +-8691089486201567102E-218 ++4345544743100783551E-218 +-4357998370690303462E-184 ++59825267349106892461E177 +-62259110684423957791E47 ++58380168477038565599E265 +-62259110684423957791E48 +-33584377202279118724E-252 +-57484963479615354808E205 ++71856204349519193510E204 +-14371240869903838702E205 ++36992084760177624177E-318 +-73984169520355248354E-318 ++99257763227713890244E-115 +-87336362425182547697E-280 ++7E289 +-3E153 ++6E153 +-5E243 ++7E-161 +-7E-172 ++8E-63 +-7E-113 ++8E126 +-4E126 ++5E125 +-1E126 ++8E-163 +-1E-163 ++2E-163 +-4E-163 ++51E195 +-37E46 ++74E46 +-56E289 ++69E-145 +-70E-162 ++56E-161 +-21E-303 ++34E-276 +-68E-276 ++85E-277 +-87E-274 ++829E102 +-623E100 ++723E-162 +-457E-102 ++914E-102 +-323E-135 ++151E176 +-302E176 ++921E90 +-604E176 ++823E-206 +-463E-114 ++348E-274 ++9968E100 +-6230E99 ++1246E100 ++6676E-296 +-8345E-297 ++1669E-296 +-3338E-296 ++3257E58 +-6514E58 ++2416E176 ++8085E-63 +-3234E-62 ++1617E-62 +-6468E-62 ++53418E111 +-60513E160 ++26709E111 +-99447E166 ++12549E48 +-25098E48 ++50196E48 +-62745E47 ++83771E-73 +-97451E-167 ++86637E-203 +-75569E-254 ++473806E83 +-947612E83 ++292369E76 +-584738E76 ++933587E-140 +-720919E-14 ++535001E-149 +-890521E-235 ++548057E81 +-706181E88 ++820997E106 +-320681E63 ++928609E-261 +-302276E-254 ++151138E-254 ++4691773E45 +-9383546E45 ++3059949E-243 +-6119898E-243 ++5356626E-213 +-4877378E-199 ++7716693E223 +-5452869E109 ++4590831E156 +-9181662E156 +-3714436E-261 ++4643045E-262 +-7428872E-261 ++52942146E130 +-27966061E145 ++26471073E130 +-55932122E145 ++95412548E-99 +-47706274E-99 ++23853137E-99 +-78493654E-301 ++65346417E29 +-51083099E167 ++89396333E264 +-84863171E114 ++59540836E-251 +-74426045E-252 ++14885209E-251 +-29770418E-251 ++982161308E122 +-245540327E122 ++491080654E122 ++525452622E-310 +-771837113E-134 ++820858081E-150 +-262726311E-310 ++923091487E209 +-653777767E273 ++842116236E-53 +-741111169E-202 ++839507247E-284 +-951487269E-264 +-9821613080E121 ++6677856011E-31 +-3573796826E-266 ++7147593652E-266 +-9981396317E-181 ++3268888835E272 +-2615111068E273 ++1307555534E273 ++2990671154E-190 +-1495335577E-190 ++5981342308E-190 +-7476677885E-191 ++82259684194E-202 +-93227267727E-49 ++41129842097E-202 +-47584241418E-314 +-79360293406E92 ++57332259349E225 +-57202326162E111 ++86860597053E-206 +-53827010643E-200 ++53587107423E-61 ++635007636765E200 ++508006109412E201 +-254003054706E201 ++561029718715E-72 +-897647549944E-71 ++112205943743E-71 +-873947086081E-236 ++809184709177E116 +-573112917422E81 ++286556458711E81 ++952805821491E-259 +-132189992873E-44 +-173696038493E-144 ++1831132757599E-107 +-9155663787995E-108 ++7324531030396E-107 +-9277338894969E-200 ++8188292423973E287 +-5672557437938E59 ++2836278718969E59 +-9995153153494E54 ++9224786422069E-291 +-3142213164987E-294 ++6284426329974E-294 +-8340483752889E-301 ++67039371486466E89 +-62150786615239E197 ++33519685743233E89 +-52563419496999E156 ++32599460466991E-65 +-41010988798007E-133 ++65198920933982E-65 +-82021977596014E-133 ++80527976643809E61 +-74712611505209E158 ++53390939710959E261 +-69277302659155E225 ++46202199371337E-72 +-23438635467783E-179 ++41921560615349E-67 +-92404398742674E-72 ++738545606647197E124 +-972708181182949E117 +-837992143580825E87 ++609610927149051E-255 +-475603213226859E-41 ++563002800671023E-177 +-951206426453718E-41 ++805416432656519E202 +-530658674694337E159 ++946574173863918E208 +-318329953318553E113 +-462021993713370E-73 ++369617594970696E-72 ++3666156212014994E233 +-1833078106007497E233 ++8301790508624232E174 +-1037723813578029E174 ++7297662880581139E-286 +-5106185698912191E-276 ++7487252720986826E-165 +-3743626360493413E-165 ++3773057430100257E230 +-7546114860200514E230 ++4321222892463822E58 +-7793560217139653E51 ++26525993941010681E112 +-53051987882021362E112 ++72844871414247907E77 +-88839359596763261E105 ++18718131802467065E-166 +-14974505441973652E-165 ++73429396004640239E106 +-58483921078398283E57 ++41391519190645203E165 +-82783038381290406E165 ++58767043776702677E-163 +-90506231831231999E-129 ++64409240769861689E-159 +-77305427432277771E-190 ++476592356619258326E273 +-953184713238516652E273 ++899810892172646163E283 +-929167076892018333E187 ++647761278967534239E-312 +-644290479820542942E-180 ++926145344610700019E-225 +-958507931896511964E-246 ++272104041512242479E200 +-792644927852378159E79 ++544208083024484958E200 +-929963218616126365E290 ++305574339166810102E-219 +-152787169583405051E-219 ++611148678333620204E-219 +-763935847917025255E-220 ++7439550220920798612E158 +-3719775110460399306E158 ++9299437776150998265E157 +-7120190517612959703E120 ++3507665085003296281E-73 +-7015330170006592562E-73 +-6684428762278255956E-294 +-1088416166048969916E200 +-8707329328391759328E200 ++4439021781608558002E-65 +-8878043563217116004E-65 ++2219510890804279001E-65 ++33051223951904955802E55 +-56961524140903677624E120 ++71201905176129597030E119 ++14030660340013185124E-73 +-17538325425016481405E-74 ++67536228609141569109E-133 +-35620497849450218807E-306 ++66550376797582521751E-126 +-71240995698900437614E-306 ++3E24 +-6E24 ++6E26 +-7E25 ++1E-14 +-2E-14 ++4E-14 +-8E-14 ++5E26 +-8E27 ++1E27 +-4E27 ++9E-13 +-7E-20 ++56E25 +-70E24 ++51E26 ++71E-17 +-31E-5 ++62E-5 +-94E-8 ++67E27 +-81E24 ++54E23 +-54E25 ++63E-22 +-63E-23 ++43E-4 +-86E-4 ++942E26 +-471E25 ++803E24 +-471E26 +-409E-21 ++818E-21 +-867E-8 ++538E27 +-857E24 ++269E27 +-403E26 ++959E-7 +-959E-6 ++373E-27 +-746E-27 ++4069E24 +-4069E23 +-8138E24 ++8294E-15 +-4147E-14 ++4147E-15 +-8294E-14 ++538E27 +-2690E26 ++269E27 +-2152E27 ++1721E-17 +-7979E-27 ++6884E-17 +-8605E-18 ++82854E27 +-55684E24 ++27842E24 +-48959E25 ++81921E-17 +-76207E-8 ++4147E-15 +-41470E-16 ++89309E24 ++75859E26 +-75859E25 ++14257E-23 +-28514E-23 ++57028E-23 +-71285E-24 ++344863E27 +-951735E27 ++200677E23 +-401354E24 ++839604E-11 +-209901E-11 ++419802E-11 +-537734E-24 ++910308E26 +-227577E26 ++455154E26 +-531013E25 ++963019E-21 +-519827E-13 ++623402E-27 +-311701E-27 ++9613651E26 +-9191316E23 ++4595658E23 +-2297829E23 +-1679208E-11 ++3379223E27 +-6758446E27 ++5444097E-21 +-8399969E-27 ++8366487E-16 +-8366487E-15 ++65060671E25 ++65212389E23 ++55544957E-13 +-51040905E-20 ++99585767E-22 +-99585767E-23 ++40978393E26 +-67488159E24 ++69005339E23 +-81956786E26 +-87105552E-21 ++10888194E-21 +-21776388E-21 ++635806667E27 +-670026614E25 ++335013307E26 +-335013307E25 ++371790617E-24 +-371790617E-25 ++743581234E-24 +-743581234E-25 ++202464477E24 +-404928954E24 ++997853758E27 +-997853758E26 ++405498418E-17 +-582579084E-14 ++608247627E-18 +-291289542E-14 +-9537100005E26 ++6358066670E27 +-1271613334E27 ++5229646999E-16 ++5229646999E-17 ++4429943614E24 +-8859887228E24 ++2214971807E24 +-4176887093E26 ++4003495257E-20 +-4361901637E-23 ++8723803274E-23 +-8006990514E-20 ++72835110098E27 +-36417555049E27 ++84279630104E25 +-84279630104E24 ++21206176437E-27 +-66461566917E-22 ++64808355539E-16 +-84932679673E-19 ++65205430094E26 +-68384463429E25 ++32602715047E26 +-62662203426E27 ++58784444678E-18 +-50980203373E-21 ++29392222339E-18 +-75529940323E-27 +-937495906299E26 ++842642485799E-20 +-387824150699E-23 ++924948814726E-27 +-775648301398E-23 ++547075707432E25 ++683844634290E24 +-136768926858E25 ++509802033730E-22 ++101960406746E-21 +-815683253968E-21 ++7344124123524E24 +-9180155154405E23 ++6479463327323E27 +-1836031030881E24 ++4337269293039E-19 +-4599163554373E-23 ++9198327108746E-23 ++4812803938347E27 +-8412030890011E23 ++9625607876694E27 +-4739968828249E24 ++9697183891673E-23 +-7368108517543E-20 ++51461358161422E25 +-77192037242133E26 ++77192037242133E25 +-51461358161422E27 ++43999661561541E-21 +-87999323123082E-21 ++48374886826137E-26 +-57684246567111E-23 ++87192805957686E23 +-75108713005913E24 ++64233110587487E27 +-77577471133384E-23 ++48485919458365E-24 +-56908598265713E-26 ++589722294620133E23 ++652835804449289E-22 +-656415363936202E-23 ++579336749585745E-25 +-381292764980839E-26 ++965265859649698E23 +-848925235434882E27 ++536177612222491E23 +-424462617717441E27 ++276009279888989E-27 +-608927158043691E-26 ++552018559777978E-27 +-425678377667758E-22 ++8013702726927119E26 ++8862627962362001E27 +-5068007907757162E26 +-7379714799828406E-23 ++4114538064016107E-27 +-3689857399914203E-23 ++5575954851815478E23 ++3395700941739528E27 ++4115535777581961E-23 +-8231071555163922E-23 ++6550246696190871E-26 +-68083046403986701E27 ++43566388595783643E27 +-87132777191567286E27 ++59644881059342141E25 +-83852770718576667E23 ++99482967418206961E-25 +-99482967418206961E-26 ++87446669969994614E-27 +-43723334984997307E-27 ++5E24 +-8E25 ++1E25 +-4E25 ++2E-5 +-5E-6 ++4E-5 +-3E-20 ++3E27 +-9E26 ++7E25 +-6E27 ++2E-21 +-5E-22 +-4E-21 ++87E25 +-97E24 ++82E-24 +-41E-24 ++76E-23 ++83E25 +-50E27 ++25E27 +-99E27 ++97E-10 +-57E-20 ++997E23 ++776E24 +-388E24 ++521E-10 +-506E-26 ++739E-10 +-867E-7 +-415E24 ++332E25 +-664E25 ++291E-13 +-982E-8 ++582E-13 +-491E-8 ++4574E26 +-8609E26 ++2287E26 +-4818E24 ++6529E-8 +-8151E-21 ++1557E-12 +-2573E-18 ++4929E-16 +-3053E-22 ++9858E-16 +-7767E-11 ++54339E26 +-62409E25 ++32819E27 +-89849E27 ++63876E-20 +-15969E-20 ++31938E-20 +-79845E-21 ++89306E27 +-25487E24 ++79889E24 +-97379E26 ++81002E-8 +-43149E-25 ++40501E-8 +-60318E-10 +-648299E27 ++780649E24 ++720919E-14 +-629703E-11 ++557913E24 +-847899E23 ++565445E27 +-736531E24 ++680013E-19 +-529981E-10 ++382923E-23 +-633614E-18 ++2165479E27 +-8661916E27 ++4330958E27 +-9391993E22 +-5767352E-14 ++7209190E-15 +-1441838E-14 ++8478990E22 ++1473062E24 ++8366487E-14 +-8399969E-25 ++9366737E-12 +-9406141E-13 ++65970979E24 +-65060671E26 ++54923002E27 +-63846927E25 ++99585767E-21 ++67488159E25 +-69005339E24 ++81956786E27 +-40978393E27 ++77505754E-12 +-38752877E-12 ++82772981E-15 +-95593517E-25 ++200036989E25 +-772686455E27 ++859139907E23 +-400073978E25 ++569014327E-14 +-794263862E-15 ++397131931E-15 +-380398957E-16 ++567366773E27 +-337440795E24 ++134976318E25 +-269952636E25 ++932080597E-20 +-331091924E-15 +-413864905E-16 ++8539246247E26 +-5859139791E26 ++6105010149E24 +-3090745820E27 ++3470877773E-20 +-6136309089E-27 ++8917758713E-19 +-6941755546E-20 ++9194900535E25 +-1838980107E26 ++7355920428E26 +-3677960214E26 ++8473634343E-17 +-8870766274E-16 ++4435383137E-16 +-9598990129E-15 ++71563496764E26 +-89454370955E25 ++17890874191E26 +-35781748382E26 ++57973447842E-19 +-28986723921E-19 ++76822711313E-19 +-97699466874E-20 ++67748656762E27 +-19394840991E24 ++38789681982E24 +-33874328381E27 ++54323763886E-27 +-58987193887E-20 ++27161881943E-27 +-93042648033E-19 ++520831059055E27 +-768124264394E25 ++384062132197E25 ++765337749889E-25 ++794368912771E25 +-994162090146E23 ++781652779431E26 ++910077190046E-26 +-455038595023E-26 ++471897551096E-20 +-906698409911E-21 ++8854128003935E25 +-8146122716299E27 ++7083302403148E26 +-3541651201574E26 ++8394920649291E-25 +-7657975756753E-22 ++5473834002228E-20 +-6842292502785E-21 +-2109568884597E25 ++8438275538388E25 +-4219137769194E25 ++3200141789841E-25 +-8655689322607E-22 ++6400283579682E-25 +-8837719634493E-21 ++19428217075297E24 +-38856434150594E24 ++77712868301188E24 +-77192037242133E27 ++76579757567530E-23 ++15315951513506E-22 +-38289878783765E-23 ++49378033925202E25 +-50940527102367E24 ++98756067850404E25 +-99589397544892E26 +-56908598265713E-25 ++97470695699657E-22 +-35851901247343E-25 ++154384074484266E27 +-308768148968532E27 ++910990389005985E23 ++271742424169201E-27 +-543484848338402E-27 ++162192083357563E-26 +-869254552770081E-23 ++664831007626046E24 +-332415503813023E24 ++943701829041427E24 +-101881054204734E24 ++828027839666967E-27 +-280276135608777E-27 ++212839188833879E-21 +-113817196531426E-25 ++9711553197796883E27 +-2739849386524269E26 ++5479698773048538E26 ++6124568318523113E-25 +-1139777988171071E-24 ++6322612303128019E-27 +-2955864564844617E-25 +-9994029144998961E25 +-2971238324022087E27 +-1656055679333934E-27 +-1445488709150234E-26 ++55824717499885172E27 +-69780896874856465E26 ++84161538867545199E25 +-27912358749942586E27 ++24711112462926331E-25 +-12645224606256038E-27 +-12249136637046226E-25 ++74874448287465757E27 +-35642836832753303E24 +-71285673665506606E24 ++43723334984997307E-26 ++10182419849537963E-24 +-93501703572661982E-26 + +# A value that caused a crash in debug builds for Python >= 2.7, 3.1 +# See http://bugs.python.org/issue7632 +2183167012312112312312.23538020374420446192e-370 + +# Another value designed to test a corner case of Python's strtod code. +0.99999999999999999999999999999999999999999e+23 diff --git a/Lib/test/mathdata/formatfloat_testcases.txt b/Lib/test/mathdata/formatfloat_testcases.txt new file mode 100644 index 0000000000..25c07ba293 --- /dev/null +++ b/Lib/test/mathdata/formatfloat_testcases.txt @@ -0,0 +1,355 @@ +-- 'f' code formatting, with explicit precision (>= 0). Output always +-- has the given number of places after the point; zeros are added if +-- necessary to make this true. + +-- zeros +%.0f 0 -> 0 +%.1f 0 -> 0.0 +%.2f 0 -> 0.00 +%.3f 0 -> 0.000 +%.50f 0 -> 0.00000000000000000000000000000000000000000000000000 + +-- precision 0; result should never include a . +%.0f 1.5 -> 2 +%.0f 2.5 -> 2 +%.0f 3.5 -> 4 +%.0f 0.0 -> 0 +%.0f 0.1 -> 0 +%.0f 0.001 -> 0 +%.0f 10.0 -> 10 +%.0f 10.1 -> 10 +%.0f 10.01 -> 10 +%.0f 123.456 -> 123 +%.0f 1234.56 -> 1235 +%.0f 1e49 -> 9999999999999999464902769475481793196872414789632 +%.0f 9.9999999999999987e+49 -> 99999999999999986860582406952576489172979654066176 +%.0f 1e50 -> 100000000000000007629769841091887003294964970946560 + +-- precision 1 +%.1f 0.0001 -> 0.0 +%.1f 0.001 -> 0.0 +%.1f 0.01 -> 0.0 +%.1f 0.04 -> 0.0 +%.1f 0.06 -> 0.1 +%.1f 0.25 -> 0.2 +%.1f 0.75 -> 0.8 +%.1f 1.4 -> 1.4 +%.1f 1.5 -> 1.5 +%.1f 10.0 -> 10.0 +%.1f 1000.03 -> 1000.0 +%.1f 1234.5678 -> 1234.6 +%.1f 1234.7499 -> 1234.7 +%.1f 1234.75 -> 1234.8 + +-- precision 2 +%.2f 0.0001 -> 0.00 +%.2f 0.001 -> 0.00 +%.2f 0.004999 -> 0.00 +%.2f 0.005001 -> 0.01 +%.2f 0.01 -> 0.01 +%.2f 0.125 -> 0.12 +%.2f 0.375 -> 0.38 +%.2f 1234500 -> 1234500.00 +%.2f 1234560 -> 1234560.00 +%.2f 1234567 -> 1234567.00 +%.2f 1234567.8 -> 1234567.80 +%.2f 1234567.89 -> 1234567.89 +%.2f 1234567.891 -> 1234567.89 +%.2f 1234567.8912 -> 1234567.89 + +-- alternate form always includes a decimal point. This only +-- makes a difference when the precision is 0. +%#.0f 0 -> 0. +%#.1f 0 -> 0.0 +%#.0f 1.5 -> 2. +%#.0f 2.5 -> 2. +%#.0f 10.1 -> 10. +%#.0f 1234.56 -> 1235. +%#.1f 1.4 -> 1.4 +%#.2f 0.375 -> 0.38 + +-- if precision is omitted it defaults to 6 +%f 0 -> 0.000000 +%f 1230000 -> 1230000.000000 +%f 1234567 -> 1234567.000000 +%f 123.4567 -> 123.456700 +%f 1.23456789 -> 1.234568 +%f 0.00012 -> 0.000120 +%f 0.000123 -> 0.000123 +%f 0.00012345 -> 0.000123 +%f 0.000001 -> 0.000001 +%f 0.0000005001 -> 0.000001 +%f 0.0000004999 -> 0.000000 + +-- 'e' code formatting with explicit precision (>= 0). Output should +-- always have exactly the number of places after the point that were +-- requested. + +-- zeros +%.0e 0 -> 0e+00 +%.1e 0 -> 0.0e+00 +%.2e 0 -> 0.00e+00 +%.10e 0 -> 0.0000000000e+00 +%.50e 0 -> 0.00000000000000000000000000000000000000000000000000e+00 + +-- precision 0. no decimal point in the output +%.0e 0.01 -> 1e-02 +%.0e 0.1 -> 1e-01 +%.0e 1 -> 1e+00 +%.0e 10 -> 1e+01 +%.0e 100 -> 1e+02 +%.0e 0.012 -> 1e-02 +%.0e 0.12 -> 1e-01 +%.0e 1.2 -> 1e+00 +%.0e 12 -> 1e+01 +%.0e 120 -> 1e+02 +%.0e 123.456 -> 1e+02 +%.0e 0.000123456 -> 1e-04 +%.0e 123456000 -> 1e+08 +%.0e 0.5 -> 5e-01 +%.0e 1.4 -> 1e+00 +%.0e 1.5 -> 2e+00 +%.0e 1.6 -> 2e+00 +%.0e 2.4999999 -> 2e+00 +%.0e 2.5 -> 2e+00 +%.0e 2.5000001 -> 3e+00 +%.0e 3.499999999999 -> 3e+00 +%.0e 3.5 -> 4e+00 +%.0e 4.5 -> 4e+00 +%.0e 5.5 -> 6e+00 +%.0e 6.5 -> 6e+00 +%.0e 7.5 -> 8e+00 +%.0e 8.5 -> 8e+00 +%.0e 9.4999 -> 9e+00 +%.0e 9.5 -> 1e+01 +%.0e 10.5 -> 1e+01 +%.0e 14.999 -> 1e+01 +%.0e 15 -> 2e+01 + +-- precision 1 +%.1e 0.0001 -> 1.0e-04 +%.1e 0.001 -> 1.0e-03 +%.1e 0.01 -> 1.0e-02 +%.1e 0.1 -> 1.0e-01 +%.1e 1 -> 1.0e+00 +%.1e 10 -> 1.0e+01 +%.1e 100 -> 1.0e+02 +%.1e 120 -> 1.2e+02 +%.1e 123 -> 1.2e+02 +%.1e 123.4 -> 1.2e+02 + +-- precision 2 +%.2e 0.00013 -> 1.30e-04 +%.2e 0.000135 -> 1.35e-04 +%.2e 0.0001357 -> 1.36e-04 +%.2e 0.0001 -> 1.00e-04 +%.2e 0.001 -> 1.00e-03 +%.2e 0.01 -> 1.00e-02 +%.2e 0.1 -> 1.00e-01 +%.2e 1 -> 1.00e+00 +%.2e 10 -> 1.00e+01 +%.2e 100 -> 1.00e+02 +%.2e 1000 -> 1.00e+03 +%.2e 1500 -> 1.50e+03 +%.2e 1590 -> 1.59e+03 +%.2e 1598 -> 1.60e+03 +%.2e 1598.7 -> 1.60e+03 +%.2e 1598.76 -> 1.60e+03 +%.2e 9999 -> 1.00e+04 + +-- omitted precision defaults to 6 +%e 0 -> 0.000000e+00 +%e 165 -> 1.650000e+02 +%e 1234567 -> 1.234567e+06 +%e 12345678 -> 1.234568e+07 +%e 1.1 -> 1.100000e+00 + +-- alternate form always contains a decimal point. This only makes +-- a difference when precision is 0. + +%#.0e 0.01 -> 1.e-02 +%#.0e 0.1 -> 1.e-01 +%#.0e 1 -> 1.e+00 +%#.0e 10 -> 1.e+01 +%#.0e 100 -> 1.e+02 +%#.0e 0.012 -> 1.e-02 +%#.0e 0.12 -> 1.e-01 +%#.0e 1.2 -> 1.e+00 +%#.0e 12 -> 1.e+01 +%#.0e 120 -> 1.e+02 +%#.0e 123.456 -> 1.e+02 +%#.0e 0.000123456 -> 1.e-04 +%#.0e 123456000 -> 1.e+08 +%#.0e 0.5 -> 5.e-01 +%#.0e 1.4 -> 1.e+00 +%#.0e 1.5 -> 2.e+00 +%#.0e 1.6 -> 2.e+00 +%#.0e 2.4999999 -> 2.e+00 +%#.0e 2.5 -> 2.e+00 +%#.0e 2.5000001 -> 3.e+00 +%#.0e 3.499999999999 -> 3.e+00 +%#.0e 3.5 -> 4.e+00 +%#.0e 4.5 -> 4.e+00 +%#.0e 5.5 -> 6.e+00 +%#.0e 6.5 -> 6.e+00 +%#.0e 7.5 -> 8.e+00 +%#.0e 8.5 -> 8.e+00 +%#.0e 9.4999 -> 9.e+00 +%#.0e 9.5 -> 1.e+01 +%#.0e 10.5 -> 1.e+01 +%#.0e 14.999 -> 1.e+01 +%#.0e 15 -> 2.e+01 +%#.1e 123.4 -> 1.2e+02 +%#.2e 0.0001357 -> 1.36e-04 + +-- 'g' code formatting. + +-- zeros +%.0g 0 -> 0 +%.1g 0 -> 0 +%.2g 0 -> 0 +%.3g 0 -> 0 +%.4g 0 -> 0 +%.10g 0 -> 0 +%.50g 0 -> 0 +%.100g 0 -> 0 + +-- precision 0 doesn't make a lot of sense for the 'g' code (what does +-- it mean to have no significant digits?); in practice, it's interpreted +-- as identical to precision 1 +%.0g 1000 -> 1e+03 +%.0g 100 -> 1e+02 +%.0g 10 -> 1e+01 +%.0g 1 -> 1 +%.0g 0.1 -> 0.1 +%.0g 0.01 -> 0.01 +%.0g 1e-3 -> 0.001 +%.0g 1e-4 -> 0.0001 +%.0g 1e-5 -> 1e-05 +%.0g 1e-6 -> 1e-06 +%.0g 12 -> 1e+01 +%.0g 120 -> 1e+02 +%.0g 1.2 -> 1 +%.0g 0.12 -> 0.1 +%.0g 0.012 -> 0.01 +%.0g 0.0012 -> 0.001 +%.0g 0.00012 -> 0.0001 +%.0g 0.000012 -> 1e-05 +%.0g 0.0000012 -> 1e-06 + +-- precision 1 identical to precision 0 +%.1g 1000 -> 1e+03 +%.1g 100 -> 1e+02 +%.1g 10 -> 1e+01 +%.1g 1 -> 1 +%.1g 0.1 -> 0.1 +%.1g 0.01 -> 0.01 +%.1g 1e-3 -> 0.001 +%.1g 1e-4 -> 0.0001 +%.1g 1e-5 -> 1e-05 +%.1g 1e-6 -> 1e-06 +%.1g 12 -> 1e+01 +%.1g 120 -> 1e+02 +%.1g 1.2 -> 1 +%.1g 0.12 -> 0.1 +%.1g 0.012 -> 0.01 +%.1g 0.0012 -> 0.001 +%.1g 0.00012 -> 0.0001 +%.1g 0.000012 -> 1e-05 +%.1g 0.0000012 -> 1e-06 + +-- precision 2 +%.2g 1000 -> 1e+03 +%.2g 100 -> 1e+02 +%.2g 10 -> 10 +%.2g 1 -> 1 +%.2g 0.1 -> 0.1 +%.2g 0.01 -> 0.01 +%.2g 0.001 -> 0.001 +%.2g 1e-4 -> 0.0001 +%.2g 1e-5 -> 1e-05 +%.2g 1e-6 -> 1e-06 +%.2g 1234 -> 1.2e+03 +%.2g 123 -> 1.2e+02 +%.2g 12.3 -> 12 +%.2g 1.23 -> 1.2 +%.2g 0.123 -> 0.12 +%.2g 0.0123 -> 0.012 +%.2g 0.00123 -> 0.0012 +%.2g 0.000123 -> 0.00012 +%.2g 0.0000123 -> 1.2e-05 + +-- bad cases from http://bugs.python.org/issue9980 +%.12g 38210.0 -> 38210 +%.12g 37210.0 -> 37210 +%.12g 36210.0 -> 36210 + +-- alternate g formatting: always include decimal point and +-- exactly significant digits. +%#.0g 0 -> 0. +%#.1g 0 -> 0. +%#.2g 0 -> 0.0 +%#.3g 0 -> 0.00 +%#.4g 0 -> 0.000 + +%#.0g 0.2 -> 0.2 +%#.1g 0.2 -> 0.2 +%#.2g 0.2 -> 0.20 +%#.3g 0.2 -> 0.200 +%#.4g 0.2 -> 0.2000 +%#.10g 0.2 -> 0.2000000000 + +%#.0g 2 -> 2. +%#.1g 2 -> 2. +%#.2g 2 -> 2.0 +%#.3g 2 -> 2.00 +%#.4g 2 -> 2.000 + +%#.0g 20 -> 2.e+01 +%#.1g 20 -> 2.e+01 +%#.2g 20 -> 20. +%#.3g 20 -> 20.0 +%#.4g 20 -> 20.00 + +%#.0g 234.56 -> 2.e+02 +%#.1g 234.56 -> 2.e+02 +%#.2g 234.56 -> 2.3e+02 +%#.3g 234.56 -> 235. +%#.4g 234.56 -> 234.6 +%#.5g 234.56 -> 234.56 +%#.6g 234.56 -> 234.560 + +-- repr formatting. Result always includes decimal point and at +-- least one digit after the point, or an exponent. +%r 0 -> 0.0 +%r 1 -> 1.0 + +%r 0.01 -> 0.01 +%r 0.02 -> 0.02 +%r 0.03 -> 0.03 +%r 0.04 -> 0.04 +%r 0.05 -> 0.05 + +-- values >= 1e16 get an exponent +%r 10 -> 10.0 +%r 100 -> 100.0 +%r 1e15 -> 1000000000000000.0 +%r 9.999e15 -> 9999000000000000.0 +%r 9999999999999998 -> 9999999999999998.0 +%r 9999999999999999 -> 1e+16 +%r 1e16 -> 1e+16 +%r 1e17 -> 1e+17 + +-- as do values < 1e-4 +%r 1e-3 -> 0.001 +%r 1.001e-4 -> 0.0001001 +%r 1.0000000000000001e-4 -> 0.0001 +%r 1.000000000000001e-4 -> 0.0001000000000000001 +%r 1.00000000001e-4 -> 0.000100000000001 +%r 1.0000000001e-4 -> 0.00010000000001 +%r 1e-4 -> 0.0001 +%r 0.99999999999999999e-4 -> 0.0001 +%r 0.9999999999999999e-4 -> 9.999999999999999e-05 +%r 0.999999999999e-4 -> 9.99999999999e-05 +%r 0.999e-4 -> 9.99e-05 +%r 1e-5 -> 1e-05 diff --git a/Lib/test/mp_preload_main.py b/Lib/test/mp_preload_main.py new file mode 100644 index 0000000000..acb342822e --- /dev/null +++ b/Lib/test/mp_preload_main.py @@ -0,0 +1,14 @@ +import multiprocessing + +print(f"{__name__}") + +def f(): + print("f") + +if __name__ == "__main__": + ctx = multiprocessing.get_context("forkserver") + ctx.set_forkserver_preload(['__main__']) + for _ in range(2): + p = ctx.Process(target=f) + p.start() + p.join() diff --git a/Lib/test/pickletester.py b/Lib/test/pickletester.py index 177e2ed2ca..c0d4c8f43b 100644 --- a/Lib/test/pickletester.py +++ b/Lib/test/pickletester.py @@ -26,7 +26,7 @@ from test import support from test.support import os_helper from test.support import ( - TestFailed, run_with_locale, no_tracing, + TestFailed, run_with_locales, no_tracing, _2G, _4G, bigmemtest ) from test.support.import_helper import forget @@ -144,6 +144,14 @@ class E(C): def __getinitargs__(self): return () +import __main__ +__main__.C = C +C.__module__ = "__main__" +__main__.D = D +D.__module__ = "__main__" +__main__.E = E +E.__module__ = "__main__" + # Simple mutable object. class Object: pass @@ -157,14 +165,6 @@ def __reduce__(self): # Shouldn't support the recursion itself return K, (self.value,) -import __main__ -__main__.C = C -C.__module__ = "__main__" -__main__.D = D -D.__module__ = "__main__" -__main__.E = E -E.__module__ = "__main__" - class myint(int): def __init__(self, x): self.str = str(x) @@ -1080,6 +1080,11 @@ def test_large_32b_binunicode8(self): self.check_unpickling_error((pickle.UnpicklingError, OverflowError), dumped) + def test_large_binstring(self): + errmsg = 'BINSTRING pickle has negative byte count' + with self.assertRaisesRegex(pickle.UnpicklingError, errmsg): + self.loads(b'T\0\0\0\x80') + def test_get(self): pickled = b'((lp100000\ng100000\nt.' unpickled = self.loads(pickled) @@ -1179,6 +1184,153 @@ def test_compat_unpickle(self): self.assertIs(type(unpickled), collections.UserDict) self.assertEqual(unpickled, collections.UserDict({1: 2})) + def test_load_global(self): + self.assertIs(self.loads(b'cbuiltins\nstr\n.'), str) + self.assertIs(self.loads(b'cmath\nlog\n.'), math.log) + self.assertIs(self.loads(b'cos.path\njoin\n.'), os.path.join) + self.assertIs(self.loads(b'\x80\x04cbuiltins\nstr.upper\n.'), str.upper) + with support.swap_item(sys.modules, 'mödule', types.SimpleNamespace(glöbal=42)): + self.assertEqual(self.loads(b'\x80\x04cm\xc3\xb6dule\ngl\xc3\xb6bal\n.'), 42) + + self.assertRaises(UnicodeDecodeError, self.loads, b'c\xff\nlog\n.') + self.assertRaises(UnicodeDecodeError, self.loads, b'cmath\n\xff\n.') + self.assertRaises(self.truncated_errors, self.loads, b'c\nlog\n.') + self.assertRaises(self.truncated_errors, self.loads, b'cmath\n\n.') + self.assertRaises(self.truncated_errors, self.loads, b'\x80\x04cmath\n\n.') + + def test_load_stack_global(self): + self.assertIs(self.loads(b'\x8c\x08builtins\x8c\x03str\x93.'), str) + self.assertIs(self.loads(b'\x8c\x04math\x8c\x03log\x93.'), math.log) + self.assertIs(self.loads(b'\x8c\x07os.path\x8c\x04join\x93.'), + os.path.join) + self.assertIs(self.loads(b'\x80\x04\x8c\x08builtins\x8c\x09str.upper\x93.'), + str.upper) + with support.swap_item(sys.modules, 'mödule', types.SimpleNamespace(glöbal=42)): + self.assertEqual(self.loads(b'\x80\x04\x8c\x07m\xc3\xb6dule\x8c\x07gl\xc3\xb6bal\x93.'), 42) + + self.assertRaises(UnicodeDecodeError, self.loads, b'\x8c\x01\xff\x8c\x03log\x93.') + self.assertRaises(UnicodeDecodeError, self.loads, b'\x8c\x04math\x8c\x01\xff\x93.') + self.assertRaises(ValueError, self.loads, b'\x8c\x00\x8c\x03log\x93.') + self.assertRaises(AttributeError, self.loads, b'\x8c\x04math\x8c\x00\x93.') + self.assertRaises(AttributeError, self.loads, b'\x80\x04\x8c\x04math\x8c\x00\x93.') + + self.assertRaises(pickle.UnpicklingError, self.loads, b'N\x8c\x03log\x93.') + self.assertRaises(pickle.UnpicklingError, self.loads, b'\x8c\x04mathN\x93.') + self.assertRaises(pickle.UnpicklingError, self.loads, b'\x80\x04\x8c\x04mathN\x93.') + + def test_find_class(self): + unpickler = self.unpickler(io.BytesIO()) + unpickler_nofix = self.unpickler(io.BytesIO(), fix_imports=False) + unpickler4 = self.unpickler(io.BytesIO(b'\x80\x04N.')) + unpickler4.load() + + self.assertIs(unpickler.find_class('__builtin__', 'str'), str) + self.assertRaises(ModuleNotFoundError, + unpickler_nofix.find_class, '__builtin__', 'str') + self.assertIs(unpickler.find_class('builtins', 'str'), str) + self.assertIs(unpickler_nofix.find_class('builtins', 'str'), str) + self.assertIs(unpickler.find_class('math', 'log'), math.log) + self.assertIs(unpickler.find_class('os.path', 'join'), os.path.join) + self.assertIs(unpickler.find_class('os.path', 'join'), os.path.join) + + self.assertIs(unpickler4.find_class('builtins', 'str.upper'), str.upper) + with self.assertRaises(AttributeError): + unpickler.find_class('builtins', 'str.upper') + + with self.assertRaises(AttributeError): + unpickler.find_class('math', 'spam') + with self.assertRaises(AttributeError): + unpickler4.find_class('math', 'spam') + with self.assertRaises(AttributeError): + unpickler.find_class('math', 'log.spam') + with self.assertRaises(AttributeError): + unpickler4.find_class('math', 'log.spam') + with self.assertRaises(AttributeError): + unpickler.find_class('math', 'log..spam') + with self.assertRaises(AttributeError): + unpickler4.find_class('math', 'log..spam') + with self.assertRaises(AttributeError): + unpickler.find_class('math', '') + with self.assertRaises(AttributeError): + unpickler4.find_class('math', '') + self.assertRaises(ModuleNotFoundError, unpickler.find_class, 'spam', 'log') + self.assertRaises(ValueError, unpickler.find_class, '', 'log') + + self.assertRaises(TypeError, unpickler.find_class, None, 'log') + self.assertRaises(TypeError, unpickler.find_class, 'math', None) + self.assertRaises((TypeError, AttributeError), unpickler4.find_class, 'math', None) + + def test_custom_find_class(self): + def loads(data): + class Unpickler(self.unpickler): + def find_class(self, module_name, global_name): + return (module_name, global_name) + return Unpickler(io.BytesIO(data)).load() + + self.assertEqual(loads(b'cmath\nlog\n.'), ('math', 'log')) + self.assertEqual(loads(b'\x8c\x04math\x8c\x03log\x93.'), ('math', 'log')) + + def loads(data): + class Unpickler(self.unpickler): + @staticmethod + def find_class(module_name, global_name): + return (module_name, global_name) + return Unpickler(io.BytesIO(data)).load() + + self.assertEqual(loads(b'cmath\nlog\n.'), ('math', 'log')) + self.assertEqual(loads(b'\x8c\x04math\x8c\x03log\x93.'), ('math', 'log')) + + def loads(data): + class Unpickler(self.unpickler): + @classmethod + def find_class(cls, module_name, global_name): + return (module_name, global_name) + return Unpickler(io.BytesIO(data)).load() + + self.assertEqual(loads(b'cmath\nlog\n.'), ('math', 'log')) + self.assertEqual(loads(b'\x8c\x04math\x8c\x03log\x93.'), ('math', 'log')) + + def loads(data): + class Unpickler(self.unpickler): + pass + def find_class(module_name, global_name): + return (module_name, global_name) + unpickler = Unpickler(io.BytesIO(data)) + unpickler.find_class = find_class + return unpickler.load() + + self.assertEqual(loads(b'cmath\nlog\n.'), ('math', 'log')) + self.assertEqual(loads(b'\x8c\x04math\x8c\x03log\x93.'), ('math', 'log')) + + def test_bad_ext_code(self): + # unregistered extension code + self.check_unpickling_error(ValueError, b'\x82\x01.') + self.check_unpickling_error(ValueError, b'\x82\xff.') + self.check_unpickling_error(ValueError, b'\x83\x01\x00.') + self.check_unpickling_error(ValueError, b'\x83\xff\xff.') + self.check_unpickling_error(ValueError, b'\x84\x01\x00\x00\x00.') + self.check_unpickling_error(ValueError, b'\x84\xff\xff\xff\x7f.') + # EXT specifies code <= 0 + self.check_unpickling_error(pickle.UnpicklingError, b'\x82\x00.') + self.check_unpickling_error(pickle.UnpicklingError, b'\x83\x00\x00.') + self.check_unpickling_error(pickle.UnpicklingError, b'\x84\x00\x00\x00\x00.') + self.check_unpickling_error(pickle.UnpicklingError, b'\x84\x00\x00\x00\x80.') + self.check_unpickling_error(pickle.UnpicklingError, b'\x84\xff\xff\xff\xff.') + + @support.cpython_only + def test_bad_ext_inverted_registry(self): + code = 1 + def check(key, exc): + with support.swap_item(copyreg._inverted_registry, code, key): + with self.assertRaises(exc): + self.loads(b'\x82\x01.') + check(None, ValueError) + check((), ValueError) + check((__name__,), (TypeError, ValueError)) + check((__name__, "MyList", "x"), (TypeError, ValueError)) + check((__name__, None), (TypeError, ValueError)) + check((None, "MyList"), (TypeError, ValueError)) + def test_bad_reduce(self): self.assertEqual(self.loads(b'cbuiltins\nint\n)R.'), 0) self.check_unpickling_error(TypeError, b'N)R.') @@ -1197,6 +1349,41 @@ def test_bad_newobj_ex(self): self.check_unpickling_error(error, b'cbuiltins\nint\nN}\x92.') self.check_unpickling_error(error, b'cbuiltins\nint\n)N\x92.') + def test_bad_state(self): + c = C() + c.x = None + base = b'c__main__\nC\n)\x81' + self.assertEqual(self.loads(base + b'}X\x01\x00\x00\x00xNsb.'), c) + self.assertEqual(self.loads(base + b'N}X\x01\x00\x00\x00xNs\x86b.'), c) + # non-hashable dict key + self.check_unpickling_error(TypeError, base + b'}]Nsb.') + # state = list + error = (pickle.UnpicklingError, AttributeError) + self.check_unpickling_error(error, base + b'](}}eb.') + # state = 1-tuple + self.check_unpickling_error(error, base + b'}\x85b.') + # state = 3-tuple + self.check_unpickling_error(error, base + b'}}}\x87b.') + # non-hashable slot name + self.check_unpickling_error(TypeError, base + b'}}]Ns\x86b.') + # non-string slot name + self.check_unpickling_error(TypeError, base + b'}}NNs\x86b.') + # dict = True + self.check_unpickling_error(error, base + b'\x88}\x86b.') + # slots dict = True + self.check_unpickling_error(error, base + b'}\x88\x86b.') + + class BadKey1: + count = 1 + def __hash__(self): + if not self.count: + raise CustomError + self.count -= 1 + return 42 + __main__.BadKey1 = BadKey1 + # bad hashable dict key + self.check_unpickling_error(CustomError, base + b'}c__main__\nBadKey1\n)\x81Nsb.') + def test_bad_stack(self): badpickles = [ b'.', # STOP @@ -1443,6 +1630,502 @@ def t(): [ToBeUnpickled] * 2) +class AbstractPicklingErrorTests: + # Subclass must define self.dumps, self.pickler. + + def test_bad_reduce_result(self): + obj = REX([print, ()]) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError): + self.dumps(obj, proto) + + obj = REX((print,)) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError): + self.dumps(obj, proto) + + obj = REX((print, (), None, None, None, None, None)) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError): + self.dumps(obj, proto) + + def test_bad_reconstructor(self): + obj = REX((42, ())) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError): + self.dumps(obj, proto) + + def test_unpickleable_reconstructor(self): + obj = REX((UnpickleableCallable(), ())) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(CustomError): + self.dumps(obj, proto) + + def test_bad_reconstructor_args(self): + obj = REX((print, [])) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError): + self.dumps(obj, proto) + + def test_unpickleable_reconstructor_args(self): + obj = REX((print, (1, 2, UNPICKLEABLE))) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(CustomError): + self.dumps(obj, proto) + + def test_bad_newobj_args(self): + obj = REX((copyreg.__newobj__, ())) + for proto in protocols[2:]: + with self.subTest(proto=proto): + with self.assertRaises((IndexError, pickle.PicklingError)) as cm: + self.dumps(obj, proto) + + obj = REX((copyreg.__newobj__, [REX])) + for proto in protocols[2:]: + with self.subTest(proto=proto): + with self.assertRaises((IndexError, pickle.PicklingError)): + self.dumps(obj, proto) + + def test_bad_newobj_class(self): + obj = REX((copyreg.__newobj__, (NoNew(),))) + for proto in protocols[2:]: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError): + self.dumps(obj, proto) + + def test_wrong_newobj_class(self): + obj = REX((copyreg.__newobj__, (str,))) + for proto in protocols[2:]: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError): + self.dumps(obj, proto) + + def test_unpickleable_newobj_class(self): + class LocalREX(REX): pass + obj = LocalREX((copyreg.__newobj__, (LocalREX,))) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises((pickle.PicklingError, AttributeError)): + self.dumps(obj, proto) + + def test_unpickleable_newobj_args(self): + obj = REX((copyreg.__newobj__, (REX, 1, 2, UNPICKLEABLE))) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(CustomError): + self.dumps(obj, proto) + + def test_bad_newobj_ex_args(self): + obj = REX((copyreg.__newobj_ex__, ())) + for proto in protocols[2:]: + with self.subTest(proto=proto): + with self.assertRaises((ValueError, pickle.PicklingError)): + self.dumps(obj, proto) + + obj = REX((copyreg.__newobj_ex__, 42)) + for proto in protocols[2:]: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError): + self.dumps(obj, proto) + + obj = REX((copyreg.__newobj_ex__, (REX, 42, {}))) + is_py = self.pickler is pickle._Pickler + for proto in protocols[2:4] if is_py else protocols[2:]: + with self.subTest(proto=proto): + with self.assertRaises((TypeError, pickle.PicklingError)): + self.dumps(obj, proto) + + obj = REX((copyreg.__newobj_ex__, (REX, (), []))) + for proto in protocols[2:4] if is_py else protocols[2:]: + with self.subTest(proto=proto): + with self.assertRaises((TypeError, pickle.PicklingError)): + self.dumps(obj, proto) + + def test_bad_newobj_ex__class(self): + obj = REX((copyreg.__newobj_ex__, (NoNew(), (), {}))) + for proto in protocols[2:]: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError): + self.dumps(obj, proto) + + def test_wrong_newobj_ex_class(self): + if self.pickler is not pickle._Pickler: + self.skipTest('only verified in the Python implementation') + obj = REX((copyreg.__newobj_ex__, (str, (), {}))) + for proto in protocols[2:]: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError): + self.dumps(obj, proto) + + def test_unpickleable_newobj_ex_class(self): + class LocalREX(REX): pass + obj = LocalREX((copyreg.__newobj_ex__, (LocalREX, (), {}))) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises((pickle.PicklingError, AttributeError)): + self.dumps(obj, proto) + + def test_unpickleable_newobj_ex_args(self): + obj = REX((copyreg.__newobj_ex__, (REX, (1, 2, UNPICKLEABLE), {}))) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(CustomError): + self.dumps(obj, proto) + + def test_unpickleable_newobj_ex_kwargs(self): + obj = REX((copyreg.__newobj_ex__, (REX, (), {'a': UNPICKLEABLE}))) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(CustomError): + self.dumps(obj, proto) + + def test_unpickleable_state(self): + obj = REX_state(UNPICKLEABLE) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(CustomError): + self.dumps(obj, proto) + + def test_bad_state_setter(self): + if self.pickler is pickle._Pickler: + self.skipTest('only verified in the C implementation') + obj = REX((print, (), 'state', None, None, 42)) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError): + self.dumps(obj, proto) + + def test_unpickleable_state_setter(self): + obj = REX((print, (), 'state', None, None, UnpickleableCallable())) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(CustomError): + self.dumps(obj, proto) + + def test_unpickleable_state_with_state_setter(self): + obj = REX((print, (), UNPICKLEABLE, None, None, print)) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(CustomError): + self.dumps(obj, proto) + + def test_bad_object_list_items(self): + # Issue4176: crash when 4th and 5th items of __reduce__() + # are not iterators + obj = REX((list, (), None, 42)) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises((TypeError, pickle.PicklingError)): + self.dumps(obj, proto) + + if self.pickler is not pickle._Pickler: + # Python implementation is less strict and also accepts iterables. + obj = REX((list, (), None, [])) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises((TypeError, pickle.PicklingError)): + self.dumps(obj, proto) + + def test_unpickleable_object_list_items(self): + obj = REX_six([1, 2, UNPICKLEABLE]) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(CustomError): + self.dumps(obj, proto) + + def test_bad_object_dict_items(self): + # Issue4176: crash when 4th and 5th items of __reduce__() + # are not iterators + obj = REX((dict, (), None, None, 42)) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises((TypeError, pickle.PicklingError)): + self.dumps(obj, proto) + + for proto in protocols: + obj = REX((dict, (), None, None, iter([('a',)]))) + with self.subTest(proto=proto): + with self.assertRaises((ValueError, TypeError)): + self.dumps(obj, proto) + + if self.pickler is not pickle._Pickler: + # Python implementation is less strict and also accepts iterables. + obj = REX((dict, (), None, None, [])) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises((TypeError, pickle.PicklingError)): + self.dumps(obj, proto) + + def test_unpickleable_object_dict_items(self): + obj = REX_seven({'a': UNPICKLEABLE}) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(CustomError): + self.dumps(obj, proto) + + def test_unpickleable_list_items(self): + obj = [1, [2, 3, UNPICKLEABLE]] + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(CustomError): + self.dumps(obj, proto) + for n in [0, 1, 1000, 1005]: + obj = [*range(n), UNPICKLEABLE] + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(CustomError): + self.dumps(obj, proto) + + def test_unpickleable_tuple_items(self): + obj = (1, (2, 3, UNPICKLEABLE)) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(CustomError): + self.dumps(obj, proto) + obj = (*range(10), UNPICKLEABLE) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(CustomError): + self.dumps(obj, proto) + + def test_unpickleable_dict_items(self): + obj = {'a': {'b': UNPICKLEABLE}} + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(CustomError): + self.dumps(obj, proto) + for n in [0, 1, 1000, 1005]: + obj = dict.fromkeys(range(n)) + obj['a'] = UNPICKLEABLE + for proto in protocols: + with self.subTest(proto=proto, n=n): + with self.assertRaises(CustomError): + self.dumps(obj, proto) + + def test_unpickleable_set_items(self): + obj = {UNPICKLEABLE} + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(CustomError): + self.dumps(obj, proto) + + def test_unpickleable_frozenset_items(self): + obj = frozenset({frozenset({UNPICKLEABLE})}) + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(CustomError): + self.dumps(obj, proto) + + def test_global_lookup_error(self): + # Global name does not exist + obj = REX('spam') + obj.__module__ = __name__ + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError): + self.dumps(obj, proto) + + obj.__module__ = 'nonexisting' + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError): + self.dumps(obj, proto) + + obj.__module__ = '' + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises((ValueError, pickle.PicklingError)): + self.dumps(obj, proto) + + obj.__module__ = None + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError): + self.dumps(obj, proto) + + def test_nonencodable_global_name_error(self): + for proto in protocols[:4]: + with self.subTest(proto=proto): + name = 'nonascii\xff' if proto < 3 else 'nonencodable\udbff' + obj = REX(name) + obj.__module__ = __name__ + with support.swap_item(globals(), name, obj): + with self.assertRaises((UnicodeEncodeError, pickle.PicklingError)): + self.dumps(obj, proto) + + def test_nonencodable_module_name_error(self): + for proto in protocols[:4]: + with self.subTest(proto=proto): + name = 'nonascii\xff' if proto < 3 else 'nonencodable\udbff' + obj = REX('test') + obj.__module__ = name + mod = types.SimpleNamespace(test=obj) + with support.swap_item(sys.modules, name, mod): + with self.assertRaises((UnicodeEncodeError, pickle.PicklingError)): + self.dumps(obj, proto) + + def test_nested_lookup_error(self): + # Nested name does not exist + global TestGlobal + class TestGlobal: + class A: + pass + obj = REX('TestGlobal.A.B.C') + obj.__module__ = __name__ + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError): + self.dumps(obj, proto) + + obj.__module__ = None + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError): + self.dumps(obj, proto) + + def test_wrong_object_lookup_error(self): + # Name is bound to different object + global TestGlobal + class TestGlobal: + pass + obj = REX('TestGlobal') + obj.__module__ = __name__ + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError): + self.dumps(obj, proto) + + obj.__module__ = None + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises(pickle.PicklingError): + self.dumps(obj, proto) + + def test_local_lookup_error(self): + # Test that whichmodule() errors out cleanly when looking up + # an assumed globally-reachable object fails. + def f(): + pass + # Since the function is local, lookup will fail + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises((AttributeError, pickle.PicklingError)): + self.dumps(f, proto) + # Same without a __module__ attribute (exercises a different path + # in _pickle.c). + del f.__module__ + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises((AttributeError, pickle.PicklingError)): + self.dumps(f, proto) + # Yet a different path. + f.__name__ = f.__qualname__ + for proto in protocols: + with self.subTest(proto=proto): + with self.assertRaises((AttributeError, pickle.PicklingError)): + self.dumps(f, proto) + + def test_reduce_ex_None(self): + c = REX_None() + with self.assertRaises(TypeError): + self.dumps(c) + + def test_reduce_None(self): + c = R_None() + with self.assertRaises(TypeError): + self.dumps(c) + + @no_tracing + def test_bad_getattr(self): + # Issue #3514: crash when there is an infinite loop in __getattr__ + x = BadGetattr() + for proto in range(2): + with support.infinite_recursion(25): + self.assertRaises(RuntimeError, self.dumps, x, proto) + for proto in range(2, pickle.HIGHEST_PROTOCOL + 1): + s = self.dumps(x, proto) + + def test_picklebuffer_error(self): + # PickleBuffer forbidden with protocol < 5 + pb = pickle.PickleBuffer(b"foobar") + for proto in range(0, 5): + with self.subTest(proto=proto): + with self.assertRaises(pickle.PickleError) as cm: + self.dumps(pb, proto) + self.assertEqual(str(cm.exception), + 'PickleBuffer can only be pickled with protocol >= 5') + + def test_non_continuous_buffer(self): + for proto in protocols[5:]: + with self.subTest(proto=proto): + pb = pickle.PickleBuffer(memoryview(b"foobar")[::2]) + with self.assertRaises((pickle.PicklingError, BufferError)): + self.dumps(pb, proto) + + def test_buffer_callback_error(self): + def buffer_callback(buffers): + raise CustomError + pb = pickle.PickleBuffer(b"foobar") + with self.assertRaises(CustomError): + self.dumps(pb, 5, buffer_callback=buffer_callback) + + def test_evil_pickler_mutating_collection(self): + # https://github.com/python/cpython/issues/92930 + global Clearer + class Clearer: + pass + + def check(collection): + class EvilPickler(self.pickler): + def persistent_id(self, obj): + if isinstance(obj, Clearer): + collection.clear() + return None + pickler = EvilPickler(io.BytesIO(), proto) + try: + pickler.dump(collection) + except RuntimeError as e: + expected = "changed size during iteration" + self.assertIn(expected, str(e)) + + for proto in protocols: + check([Clearer()]) + check([Clearer(), Clearer()]) + check({Clearer()}) + check({Clearer(), Clearer()}) + check({Clearer(): 1}) + check({Clearer(): 1, Clearer(): 2}) + check({1: Clearer(), 2: Clearer()}) + + @support.cpython_only + def test_bad_ext_code(self): + # This should never happen in normal circumstances, because the type + # and the value of the extension code is checked in copyreg.add_extension(). + key = (__name__, 'MyList') + def check(code, exc): + assert key not in copyreg._extension_registry + assert code not in copyreg._inverted_registry + with (support.swap_item(copyreg._extension_registry, key, code), + support.swap_item(copyreg._inverted_registry, code, key)): + for proto in protocols[2:]: + with self.assertRaises(exc): + self.dumps(MyList, proto) + + check(object(), TypeError) + check(None, TypeError) + check(-1, (RuntimeError, struct.error)) + check(0, RuntimeError) + check(2**31, (RuntimeError, OverflowError, struct.error)) + check(2**1000, (OverflowError, struct.error)) + check(-2**1000, (OverflowError, struct.error)) + class AbstractPickleTests: # Subclass must define self.dumps, self.loads. @@ -1845,6 +2528,25 @@ def test_bytes(self): p = self.dumps(s, proto) self.assert_is_copy(s, self.loads(p)) + def test_bytes_memoization(self): + for proto in protocols: + for array_type in [bytes, ZeroCopyBytes]: + for s in b'', b'xyz', b'xyz'*100: + with self.subTest(proto=proto, array_type=array_type, s=s, independent=False): + b = array_type(s) + p = self.dumps((b, b), proto) + x, y = self.loads(p) + self.assertIs(x, y) + self.assert_is_copy((b, b), (x, y)) + + with self.subTest(proto=proto, array_type=array_type, s=s, independent=True): + b1, b2 = array_type(s), array_type(s) + p = self.dumps((b1, b2), proto) + # Note that (b1, b2) = self.loads(p) might have identical + # components, i.e., b1 is b2, but this is not always the + # case if the content is large (equality still holds). + self.assert_is_copy((b1, b2), self.loads(p)) + def test_bytearray(self): for proto in protocols: for s in b'', b'xyz', b'xyz'*100: @@ -1864,13 +2566,31 @@ def test_bytearray(self): self.assertNotIn(b'bytearray', p) self.assertTrue(opcode_in_pickle(pickle.BYTEARRAY8, p)) - def test_bytearray_memoization_bug(self): + def test_bytearray_memoization(self): for proto in protocols: - for s in b'', b'xyz', b'xyz'*100: - b = bytearray(s) - p = self.dumps((b, b), proto) - b1, b2 = self.loads(p) - self.assertIs(b1, b2) + for array_type in [bytearray, ZeroCopyBytearray]: + for s in b'', b'xyz', b'xyz'*100: + with self.subTest(proto=proto, array_type=array_type, s=s, independent=False): + b = array_type(s) + p = self.dumps((b, b), proto) + b1, b2 = self.loads(p) + self.assertIs(b1, b2) + + with self.subTest(proto=proto, array_type=array_type, s=s, independent=True): + b1a, b2a = array_type(s), array_type(s) + # Unlike bytes, equal but independent bytearray objects are + # never identical. + self.assertIsNot(b1a, b2a) + + p = self.dumps((b1a, b2a), proto) + b1b, b2b = self.loads(p) + self.assertIsNot(b1b, b2b) + + self.assertIsNot(b1a, b1b) + self.assert_is_copy(b1a, b1b) + + self.assertIsNot(b2a, b2b) + self.assert_is_copy(b2a, b2b) def test_ints(self): for proto in protocols: @@ -1915,7 +2635,7 @@ def test_float(self): got = self.loads(pickle) self.assert_is_copy(value, got) - @run_with_locale('LC_ALL', 'de_DE', 'fr_FR') + @run_with_locales('LC_ALL', 'de_DE', 'fr_FR', '') def test_float_format(self): # make sure that floats are formatted locale independent with proto 0 self.assertEqual(self.dumps(1.2, 0)[0:3], b'F1.') @@ -2010,8 +2730,6 @@ def test_builtin_exceptions(self): else: self.assertIs(u, t) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_builtin_functions(self): for t in builtins.__dict__.values(): if isinstance(t, types.BuiltinFunctionType): @@ -2418,40 +3136,11 @@ def test_reduce_calls_base(self): y = self.loads(s) self.assertEqual(y._reduce_called, 1) - # TODO: RUSTPYTHON - @unittest.expectedFailure - @no_tracing - def test_bad_getattr(self): - # Issue #3514: crash when there is an infinite loop in __getattr__ - x = BadGetattr() - for proto in range(2): - with support.infinite_recursion(): - self.assertRaises(RuntimeError, self.dumps, x, proto) - for proto in range(2, pickle.HIGHEST_PROTOCOL + 1): - s = self.dumps(x, proto) - - def test_reduce_bad_iterator(self): - # Issue4176: crash when 4th and 5th items of __reduce__() - # are not iterators - class C(object): - def __reduce__(self): - # 4th item is not an iterator - return list, (), None, [], None - class D(object): - def __reduce__(self): - # 5th item is not an iterator - return dict, (), None, None, [] - - # Python implementation is less strict and also accepts iterables. - for proto in protocols: - try: - self.dumps(C(), proto) - except pickle.PicklingError: - pass - try: - self.dumps(D(), proto) - except pickle.PicklingError: - pass + def test_pickle_setstate_None(self): + c = C_None_setstate() + p = self.dumps(c) + with self.assertRaises(TypeError): + self.loads(p) def test_many_puts_and_gets(self): # Test that internal data structures correctly deal with lots of @@ -2769,6 +3458,18 @@ class Recursive: self.assertIs(unpickled, Recursive) del Recursive.mod # break reference loop + def test_recursive_nested_names2(self): + global Recursive + class Recursive: + pass + Recursive.ref = Recursive + Recursive.__qualname__ = 'Recursive.ref' + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(proto=proto): + unpickled = self.loads(self.dumps(Recursive, proto)) + self.assertIs(unpickled, Recursive) + del Recursive.ref # break reference loop + def test_py_methods(self): global PyMethodsTest class PyMethodsTest: @@ -2818,6 +3519,15 @@ def pie(self): unpickled = self.loads(self.dumps(method, proto)) self.assertEqual(method(obj), unpickled(obj)) + descriptors = ( + PyMethodsTest.__dict__['cheese'], # static method descriptor + PyMethodsTest.__dict__['wine'], # class method descriptor + ) + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + for descr in descriptors: + with self.subTest(proto=proto, descr=descr): + self.assertRaises(TypeError, self.dumps, descr, proto) + def test_c_methods(self): global Subclass class Subclass(tuple): @@ -2853,6 +3563,15 @@ class Nested(str): unpickled = self.loads(self.dumps(method, proto)) self.assertEqual(method(*args), unpickled(*args)) + descriptors = ( + bytearray.__dict__['maketrans'], # built-in static method descriptor + dict.__dict__['fromkeys'], # built-in class method descriptor + ) + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + for descr in descriptors: + with self.subTest(proto=proto, descr=descr): + self.assertRaises(TypeError, self.dumps, descr, proto) + def test_compat_pickle(self): tests = [ (range(1, 7), '__builtin__', 'xrange'), @@ -2871,27 +3590,6 @@ def test_compat_pickle(self): self.assertIn(('c%s\n%s' % (mod, name)).encode(), pickled) self.assertIs(type(self.loads(pickled)), type(val)) - def test_local_lookup_error(self): - # Test that whichmodule() errors out cleanly when looking up - # an assumed globally-reachable object fails. - def f(): - pass - # Since the function is local, lookup will fail - for proto in range(0, pickle.HIGHEST_PROTOCOL + 1): - with self.assertRaises((AttributeError, pickle.PicklingError)): - pickletools.dis(self.dumps(f, proto)) - # Same without a __module__ attribute (exercises a different path - # in _pickle.c). - del f.__module__ - for proto in range(0, pickle.HIGHEST_PROTOCOL + 1): - with self.assertRaises((AttributeError, pickle.PicklingError)): - pickletools.dis(self.dumps(f, proto)) - # Yet a different path. - f.__name__ = f.__qualname__ - for proto in range(0, pickle.HIGHEST_PROTOCOL + 1): - with self.assertRaises((AttributeError, pickle.PicklingError)): - pickletools.dis(self.dumps(f, proto)) - # # PEP 574 tests below # @@ -3002,20 +3700,6 @@ def test_oob_buffers_writable_to_readonly(self): self.assertIs(type(new), type(obj)) self.assertEqual(new, obj) - def test_picklebuffer_error(self): - # PickleBuffer forbidden with protocol < 5 - pb = pickle.PickleBuffer(b"foobar") - for proto in range(0, 5): - with self.assertRaises(pickle.PickleError): - self.dumps(pb, proto) - - def test_buffer_callback_error(self): - def buffer_callback(buffers): - 1/0 - pb = pickle.PickleBuffer(b"foobar") - with self.assertRaises(ZeroDivisionError): - self.dumps(pb, 5, buffer_callback=buffer_callback) - def test_buffers_error(self): pb = pickle.PickleBuffer(b"foobar") for proto in range(5, pickle.HIGHEST_PROTOCOL + 1): @@ -3107,37 +3791,6 @@ def __reduce__(self): expected = "changed size during iteration" self.assertIn(expected, str(e)) - def test_evil_pickler_mutating_collection(self): - # https://github.com/python/cpython/issues/92930 - if not hasattr(self, "pickler"): - raise self.skipTest(f"{type(self)} has no associated pickler type") - - global Clearer - class Clearer: - pass - - def check(collection): - class EvilPickler(self.pickler): - def persistent_id(self, obj): - if isinstance(obj, Clearer): - collection.clear() - return None - pickler = EvilPickler(io.BytesIO(), proto) - try: - pickler.dump(collection) - except RuntimeError as e: - expected = "changed size during iteration" - self.assertIn(expected, str(e)) - - for proto in protocols: - check([Clearer()]) - check([Clearer(), Clearer()]) - check({Clearer()}) - check({Clearer(), Clearer()}) - check({Clearer(): 1}) - check({Clearer(): 1, Clearer(): 2}) - check({1: Clearer(), 2: Clearer()}) - class BigmemPickleTests: @@ -3268,6 +3921,18 @@ def test_huge_str_64b(self, size): # Test classes for reduce_ex +class R: + def __init__(self, reduce=None): + self.reduce = reduce + def __reduce__(self, proto): + return self.reduce + +class REX: + def __init__(self, reduce_ex=None): + self.reduce_ex = reduce_ex + def __reduce_ex__(self, proto): + return self.reduce_ex + class REX_one(object): """No __reduce_ex__ here, but inheriting it from object""" _reduce_called = 0 @@ -3343,6 +4008,34 @@ def __setstate__(self, state): def __reduce__(self): return type(self), (), self.state +class REX_None: + """ Setting __reduce_ex__ to None should fail """ + __reduce_ex__ = None + +class R_None: + """ Setting __reduce__ to None should fail """ + __reduce__ = None + +class C_None_setstate: + """ Setting __setstate__ to None should fail """ + def __getstate__(self): + return 1 + + __setstate__ = None + +class CustomError(Exception): + pass + +class Unpickleable: + def __reduce__(self): + raise CustomError + +UNPICKLEABLE = Unpickleable() + +class UnpickleableCallable(Unpickleable): + def __call__(self, *args, **kwargs): + pass + # Test classes for newobj @@ -3392,7 +4085,9 @@ class MyIntWithNew2(MyIntWithNew): class SlotList(MyList): __slots__ = ["foo"] -class SimpleNewObj(int): +# Ruff "redefined while unused" false positive here due to `global` variables +# being assigned (and then restored) from within test methods earlier in the file +class SimpleNewObj(int): # noqa: F811 def __init__(self, *args, **kwargs): # raise an error, to make sure this isn't called raise TypeError("SimpleNewObj.__init__() didn't expect to get called") @@ -3411,6 +4106,12 @@ class BadGetattr: def __getattr__(self, key): self.foo +class NoNew: + def __getattribute__(self, name): + if name == '__new__': + raise AttributeError + return super().__getattribute__(name) + class AbstractPickleModuleTests: @@ -3483,7 +4184,7 @@ def raises_oserror(self, *args, **kwargs): raise OSError @property def bad_property(self): - 1/0 + raise CustomError # File without read and readline class F: @@ -3504,23 +4205,23 @@ class F: class F: read = bad_property readline = raises_oserror - self.assertRaises(ZeroDivisionError, self.Unpickler, F()) + self.assertRaises(CustomError, self.Unpickler, F()) # File with bad readline class F: readline = bad_property read = raises_oserror - self.assertRaises(ZeroDivisionError, self.Unpickler, F()) + self.assertRaises(CustomError, self.Unpickler, F()) # File with bad readline, no read class F: readline = bad_property - self.assertRaises(ZeroDivisionError, self.Unpickler, F()) + self.assertRaises(CustomError, self.Unpickler, F()) # File with bad read, no readline class F: read = bad_property - self.assertRaises((AttributeError, ZeroDivisionError), self.Unpickler, F()) + self.assertRaises((AttributeError, CustomError), self.Unpickler, F()) # File with bad peek class F: @@ -3529,7 +4230,7 @@ class F: readline = raises_oserror try: self.Unpickler(F()) - except ZeroDivisionError: + except CustomError: pass # File with bad readinto @@ -3539,7 +4240,7 @@ class F: readline = raises_oserror try: self.Unpickler(F()) - except ZeroDivisionError: + except CustomError: pass def test_pickler_bad_file(self): @@ -3552,8 +4253,8 @@ class F: class F: @property def write(self): - 1/0 - self.assertRaises(ZeroDivisionError, self.Pickler, F()) + raise CustomError + self.assertRaises(CustomError, self.Pickler, F()) def check_dumps_loads_oob_buffers(self, dumps, loads): # No need to do the full gamut of tests here, just enough to @@ -3661,9 +4362,15 @@ def test_return_correct_type(self): def test_protocol0_is_ascii_only(self): non_ascii_str = "\N{EMPTY SET}" - self.assertRaises(pickle.PicklingError, self.dumps, non_ascii_str, 0) + with self.assertRaises(pickle.PicklingError) as cm: + self.dumps(non_ascii_str, 0) + self.assertEqual(str(cm.exception), + 'persistent IDs in protocol 0 must be ASCII strings') pickled = pickle.PERSID + non_ascii_str.encode('utf-8') + b'\n.' - self.assertRaises(pickle.UnpicklingError, self.loads, pickled) + with self.assertRaises(pickle.UnpicklingError) as cm: + self.loads(pickled) + self.assertEqual(str(cm.exception), + 'persistent IDs in protocol 0 must be ASCII strings') class AbstractPicklerUnpicklerObjectTests: @@ -3824,6 +4531,25 @@ def test_unpickling_buffering_readline(self): unpickler = self.unpickler_class(f) self.assertEqual(unpickler.load(), data) + def test_pickle_invalid_reducer_override(self): + # gh-103035 + obj = object() + + f = io.BytesIO() + class MyPickler(self.pickler_class): + pass + pickler = MyPickler(f) + pickler.dump(obj) + + pickler.clear_memo() + pickler.reducer_override = None + with self.assertRaises(TypeError): + pickler.dump(obj) + + pickler.clear_memo() + pickler.reducer_override = 10 + with self.assertRaises(TypeError): + pickler.dump(obj) # Tests for dispatch_table attribute @@ -3986,6 +4712,15 @@ def dumps(obj, protocol=None): self._test_dispatch_table(dumps, dt) + def test_dispatch_table_None_item(self): + # gh-93627 + obj = object() + f = io.BytesIO() + pickler = self.pickler_class(f) + pickler.dispatch_table = {type(obj): None} + with self.assertRaises(TypeError): + pickler.dump(obj) + def _test_dispatch_table(self, dumps, dispatch_table): def custom_load_dump(obj): return pickle.loads(dumps(obj, 0)) diff --git a/Lib/test/pyclbr_input.py b/Lib/test/pyclbr_input.py new file mode 100644 index 0000000000..5535edbfa7 --- /dev/null +++ b/Lib/test/pyclbr_input.py @@ -0,0 +1,85 @@ +"""Test cases for test_pyclbr.py""" + +def f(): pass + +class Other(object): + @classmethod + def foo(c): pass + + def om(self): pass + +class B (object): + def bm(self): pass + +class C (B): + d = 10 + + # This one is correctly considered by both test_pyclbr.py and pyclbr.py + # as a non-method of C. + foo = Other().foo + + # This causes test_pyclbr.py to fail, but only because the + # introspection-based is_method() code in the test can't + # distinguish between this and a genuine method function like m(). + # + # The pyclbr.py module gets this right as it parses the text. + om = Other.om + f = f + + def m(self): pass + + @staticmethod + def sm(self): pass + + @classmethod + def cm(self): pass + +# Check that mangling is correctly handled + +class a: + def a(self): pass + def _(self): pass + def _a(self): pass + def __(self): pass + def ___(self): pass + def __a(self): pass + +class _: + def a(self): pass + def _(self): pass + def _a(self): pass + def __(self): pass + def ___(self): pass + def __a(self): pass + +class __: + def a(self): pass + def _(self): pass + def _a(self): pass + def __(self): pass + def ___(self): pass + def __a(self): pass + +class ___: + def a(self): pass + def _(self): pass + def _a(self): pass + def __(self): pass + def ___(self): pass + def __a(self): pass + +class _a: + def a(self): pass + def _(self): pass + def _a(self): pass + def __(self): pass + def ___(self): pass + def __a(self): pass + +class __a: + def a(self): pass + def _(self): pass + def _a(self): pass + def __(self): pass + def ___(self): pass + def __a(self): pass diff --git a/Lib/test/pythoninfo.py b/Lib/test/pythoninfo.py new file mode 100644 index 0000000000..19b336ba96 --- /dev/null +++ b/Lib/test/pythoninfo.py @@ -0,0 +1,1100 @@ +""" +Collect various information about Python to help debugging test failures. +""" +import errno +import re +import sys +import traceback +import warnings + + +def normalize_text(text): + if text is None: + return None + text = str(text) + text = re.sub(r'\s+', ' ', text) + return text.strip() + + +class PythonInfo: + def __init__(self): + self.info = {} + + def add(self, key, value): + if key in self.info: + raise ValueError("duplicate key: %r" % key) + + if value is None: + return + + if not isinstance(value, int): + if not isinstance(value, str): + # convert other objects like sys.flags to string + value = str(value) + + value = value.strip() + if not value: + return + + self.info[key] = value + + def get_infos(self): + """ + Get information as a key:value dictionary where values are strings. + """ + return {key: str(value) for key, value in self.info.items()} + + +def copy_attributes(info_add, obj, name_fmt, attributes, *, formatter=None): + for attr in attributes: + value = getattr(obj, attr, None) + if value is None: + continue + name = name_fmt % attr + if formatter is not None: + value = formatter(attr, value) + info_add(name, value) + + +def copy_attr(info_add, name, mod, attr_name): + try: + value = getattr(mod, attr_name) + except AttributeError: + return + info_add(name, value) + + +def call_func(info_add, name, mod, func_name, *, formatter=None): + try: + func = getattr(mod, func_name) + except AttributeError: + return + value = func() + if formatter is not None: + value = formatter(value) + info_add(name, value) + + +def collect_sys(info_add): + attributes = ( + '_emscripten_info', + '_framework', + 'abiflags', + 'api_version', + 'builtin_module_names', + 'byteorder', + 'dont_write_bytecode', + 'executable', + 'flags', + 'float_info', + 'float_repr_style', + 'hash_info', + 'hexversion', + 'implementation', + 'int_info', + 'maxsize', + 'maxunicode', + 'path', + 'platform', + 'platlibdir', + 'prefix', + 'thread_info', + 'version', + 'version_info', + 'winver', + ) + copy_attributes(info_add, sys, 'sys.%s', attributes) + + for func in ( + '_is_gil_enabled', + 'getandroidapilevel', + 'getrecursionlimit', + 'getwindowsversion', + ): + call_func(info_add, f'sys.{func}', sys, func) + + encoding = sys.getfilesystemencoding() + if hasattr(sys, 'getfilesystemencodeerrors'): + encoding = '%s/%s' % (encoding, sys.getfilesystemencodeerrors()) + info_add('sys.filesystem_encoding', encoding) + + for name in ('stdin', 'stdout', 'stderr'): + stream = getattr(sys, name) + if stream is None: + continue + encoding = getattr(stream, 'encoding', None) + if not encoding: + continue + errors = getattr(stream, 'errors', None) + if errors: + encoding = '%s/%s' % (encoding, errors) + info_add('sys.%s.encoding' % name, encoding) + + # Were we compiled --with-pydebug? + Py_DEBUG = hasattr(sys, 'gettotalrefcount') + if Py_DEBUG: + text = 'Yes (sys.gettotalrefcount() present)' + else: + text = 'No (sys.gettotalrefcount() missing)' + info_add('build.Py_DEBUG', text) + + # Were we compiled --with-trace-refs? + Py_TRACE_REFS = hasattr(sys, 'getobjects') + if Py_TRACE_REFS: + text = 'Yes (sys.getobjects() present)' + else: + text = 'No (sys.getobjects() missing)' + info_add('build.Py_TRACE_REFS', text) + + +def collect_platform(info_add): + import platform + + arch = platform.architecture() + arch = ' '.join(filter(bool, arch)) + info_add('platform.architecture', arch) + + info_add('platform.python_implementation', + platform.python_implementation()) + info_add('platform.platform', + platform.platform(aliased=True)) + + libc_ver = ('%s %s' % platform.libc_ver()).strip() + if libc_ver: + info_add('platform.libc_ver', libc_ver) + + try: + os_release = platform.freedesktop_os_release() + except OSError: + pass + else: + for key in ( + 'ID', + 'NAME', + 'PRETTY_NAME' + 'VARIANT', + 'VARIANT_ID', + 'VERSION', + 'VERSION_CODENAME', + 'VERSION_ID', + ): + if key not in os_release: + continue + info_add(f'platform.freedesktop_os_release[{key}]', + os_release[key]) + + if sys.platform == 'android': + call_func(info_add, 'platform.android_ver', platform, 'android_ver') + + +def collect_locale(info_add): + import locale + + info_add('locale.getencoding', locale.getencoding()) + + +def collect_builtins(info_add): + info_add('builtins.float.float_format', float.__getformat__("float")) + info_add('builtins.float.double_format', float.__getformat__("double")) + + +def collect_urandom(info_add): + import os + + if hasattr(os, 'getrandom'): + # PEP 524: Check if system urandom is initialized + try: + try: + os.getrandom(1, os.GRND_NONBLOCK) + state = 'ready (initialized)' + except BlockingIOError as exc: + state = 'not seeded yet (%s)' % exc + info_add('os.getrandom', state) + except OSError as exc: + # Python was compiled on a more recent Linux version + # than the current Linux kernel: ignore OSError(ENOSYS) + if exc.errno != errno.ENOSYS: + raise + + +def collect_os(info_add): + import os + + def format_attr(attr, value): + if attr in ('supports_follow_symlinks', 'supports_fd', + 'supports_effective_ids'): + return str(sorted(func.__name__ for func in value)) + else: + return value + + attributes = ( + 'name', + 'supports_bytes_environ', + 'supports_effective_ids', + 'supports_fd', + 'supports_follow_symlinks', + ) + copy_attributes(info_add, os, 'os.%s', attributes, formatter=format_attr) + + for func in ( + 'cpu_count', + 'getcwd', + 'getegid', + 'geteuid', + 'getgid', + 'getloadavg', + 'getresgid', + 'getresuid', + 'getuid', + 'process_cpu_count', + 'uname', + ): + call_func(info_add, 'os.%s' % func, os, func) + + def format_groups(groups): + return ', '.join(map(str, groups)) + + call_func(info_add, 'os.getgroups', os, 'getgroups', formatter=format_groups) + + if hasattr(os, 'getlogin'): + try: + login = os.getlogin() + except OSError: + # getlogin() fails with "OSError: [Errno 25] Inappropriate ioctl + # for device" on Travis CI + pass + else: + info_add("os.login", login) + + # Environment variables used by the stdlib and tests. Don't log the full + # environment: filter to list to not leak sensitive information. + # + # HTTP_PROXY is not logged because it can contain a password. + ENV_VARS = frozenset(( + "APPDATA", + "AR", + "ARCHFLAGS", + "ARFLAGS", + "AUDIODEV", + "BUILDPYTHON", + "CC", + "CFLAGS", + "COLUMNS", + "COMPUTERNAME", + "COMSPEC", + "CPP", + "CPPFLAGS", + "DISPLAY", + "DISTUTILS_DEBUG", + "DISTUTILS_USE_SDK", + "DYLD_LIBRARY_PATH", + "ENSUREPIP_OPTIONS", + "HISTORY_FILE", + "HOME", + "HOMEDRIVE", + "HOMEPATH", + "IDLESTARTUP", + "IPHONEOS_DEPLOYMENT_TARGET", + "LANG", + "LDFLAGS", + "LDSHARED", + "LD_LIBRARY_PATH", + "LINES", + "MACOSX_DEPLOYMENT_TARGET", + "MAILCAPS", + "MAKEFLAGS", + "MIXERDEV", + "MSSDK", + "PATH", + "PATHEXT", + "PIP_CONFIG_FILE", + "PLAT", + "POSIXLY_CORRECT", + "PY_SAX_PARSER", + "ProgramFiles", + "ProgramFiles(x86)", + "RUNNING_ON_VALGRIND", + "SDK_TOOLS_BIN", + "SERVER_SOFTWARE", + "SHELL", + "SOURCE_DATE_EPOCH", + "SYSTEMROOT", + "TEMP", + "TERM", + "TILE_LIBRARY", + "TMP", + "TMPDIR", + "TRAVIS", + "TZ", + "USERPROFILE", + "VIRTUAL_ENV", + "WAYLAND_DISPLAY", + "WINDIR", + "_PYTHON_HOSTRUNNER", + "_PYTHON_HOST_PLATFORM", + "_PYTHON_PROJECT_BASE", + "_PYTHON_SYSCONFIGDATA_NAME", + "_PYTHON_SYSCONFIGDATA_PATH", + "__PYVENV_LAUNCHER__", + + # Sanitizer options + "ASAN_OPTIONS", + "LSAN_OPTIONS", + "MSAN_OPTIONS", + "TSAN_OPTIONS", + "UBSAN_OPTIONS", + )) + for name, value in os.environ.items(): + uname = name.upper() + if (uname in ENV_VARS + # Copy PYTHON* variables like PYTHONPATH + # Copy LC_* variables like LC_ALL + or uname.startswith(("PYTHON", "LC_")) + # Visual Studio: VS140COMNTOOLS + or (uname.startswith("VS") and uname.endswith("COMNTOOLS"))): + info_add('os.environ[%s]' % name, value) + + if hasattr(os, 'umask'): + mask = os.umask(0) + os.umask(mask) + info_add("os.umask", '0o%03o' % mask) + + +def collect_pwd(info_add): + try: + import pwd + except ImportError: + return + import os + + uid = os.getuid() + try: + entry = pwd.getpwuid(uid) + except KeyError: + entry = None + + info_add('pwd.getpwuid(%s)'% uid, + entry if entry is not None else '') + + if entry is None: + # there is nothing interesting to read if the current user identifier + # is not the password database + return + + if hasattr(os, 'getgrouplist'): + groups = os.getgrouplist(entry.pw_name, entry.pw_gid) + groups = ', '.join(map(str, groups)) + info_add('os.getgrouplist', groups) + + +def collect_readline(info_add): + try: + import readline + except ImportError: + return + + def format_attr(attr, value): + if isinstance(value, int): + return "%#x" % value + else: + return value + + attributes = ( + "_READLINE_VERSION", + "_READLINE_RUNTIME_VERSION", + "_READLINE_LIBRARY_VERSION", + ) + copy_attributes(info_add, readline, 'readline.%s', attributes, + formatter=format_attr) + + if not hasattr(readline, "_READLINE_LIBRARY_VERSION"): + # _READLINE_LIBRARY_VERSION has been added to CPython 3.7 + doc = getattr(readline, '__doc__', '') + if 'libedit readline' in doc: + info_add('readline.library', 'libedit readline') + elif 'GNU readline' in doc: + info_add('readline.library', 'GNU readline') + + +def collect_gdb(info_add): + import subprocess + + try: + proc = subprocess.Popen(["gdb", "-nx", "--version"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + universal_newlines=True) + version = proc.communicate()[0] + if proc.returncode: + # ignore gdb failure: test_gdb will log the error + return + except OSError: + return + + # Only keep the first line + version = version.splitlines()[0] + info_add('gdb_version', version) + + +def collect_tkinter(info_add): + try: + import _tkinter + except ImportError: + pass + else: + attributes = ('TK_VERSION', 'TCL_VERSION') + copy_attributes(info_add, _tkinter, 'tkinter.%s', attributes) + + try: + import tkinter + except ImportError: + pass + else: + tcl = tkinter.Tcl() + patchlevel = tcl.call('info', 'patchlevel') + info_add('tkinter.info_patchlevel', patchlevel) + + +def collect_time(info_add): + import time + + info_add('time.time', time.time()) + + attributes = ( + 'altzone', + 'daylight', + 'timezone', + 'tzname', + ) + copy_attributes(info_add, time, 'time.%s', attributes) + + if hasattr(time, 'get_clock_info'): + for clock in ('clock', 'monotonic', 'perf_counter', + 'process_time', 'thread_time', 'time'): + try: + # prevent DeprecatingWarning on get_clock_info('clock') + with warnings.catch_warnings(record=True): + clock_info = time.get_clock_info(clock) + except ValueError: + # missing clock like time.thread_time() + pass + else: + info_add('time.get_clock_info(%s)' % clock, clock_info) + + +def collect_curses(info_add): + try: + import curses + except ImportError: + return + + copy_attr(info_add, 'curses.ncurses_version', curses, 'ncurses_version') + + +def collect_datetime(info_add): + try: + import datetime + except ImportError: + return + + info_add('datetime.datetime.now', datetime.datetime.now()) + + +def collect_sysconfig(info_add): + import sysconfig + + info_add('sysconfig.is_python_build', sysconfig.is_python_build()) + + for name in ( + 'ABIFLAGS', + 'ANDROID_API_LEVEL', + 'CC', + 'CCSHARED', + 'CFLAGS', + 'CFLAGSFORSHARED', + 'CONFIG_ARGS', + 'HOSTRUNNER', + 'HOST_GNU_TYPE', + 'MACHDEP', + 'MULTIARCH', + 'OPT', + 'PGO_PROF_USE_FLAG', + 'PY_CFLAGS', + 'PY_CFLAGS_NODIST', + 'PY_CORE_LDFLAGS', + 'PY_LDFLAGS', + 'PY_LDFLAGS_NODIST', + 'PY_STDMODULE_CFLAGS', + 'Py_DEBUG', + 'Py_ENABLE_SHARED', + 'Py_GIL_DISABLED', + 'SHELL', + 'SOABI', + 'TEST_MODULES', + 'abs_builddir', + 'abs_srcdir', + 'prefix', + 'srcdir', + ): + value = sysconfig.get_config_var(name) + if name == 'ANDROID_API_LEVEL' and not value: + # skip ANDROID_API_LEVEL=0 + continue + value = normalize_text(value) + info_add('sysconfig[%s]' % name, value) + + PY_CFLAGS = sysconfig.get_config_var('PY_CFLAGS') + NDEBUG = (PY_CFLAGS and '-DNDEBUG' in PY_CFLAGS) + if NDEBUG: + text = 'ignore assertions (macro defined)' + else: + text= 'build assertions (macro not defined)' + info_add('build.NDEBUG',text) + + for name in ( + 'WITH_DOC_STRINGS', + 'WITH_DTRACE', + 'WITH_FREELISTS', + 'WITH_MIMALLOC', + 'WITH_PYMALLOC', + 'WITH_VALGRIND', + ): + value = sysconfig.get_config_var(name) + if value: + text = 'Yes' + else: + text = 'No' + info_add(f'build.{name}', text) + + +def collect_ssl(info_add): + import os + try: + import ssl + except ImportError: + return + try: + import _ssl + except ImportError: + _ssl = None + + def format_attr(attr, value): + if attr.startswith('OP_'): + return '%#8x' % value + else: + return value + + attributes = ( + 'OPENSSL_VERSION', + 'OPENSSL_VERSION_INFO', + 'HAS_SNI', + 'OP_ALL', + 'OP_NO_TLSv1_1', + ) + copy_attributes(info_add, ssl, 'ssl.%s', attributes, formatter=format_attr) + + for name, ctx in ( + ('SSLContext', ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT)), + ('default_https_context', ssl._create_default_https_context()), + ('stdlib_context', ssl._create_stdlib_context()), + ): + attributes = ( + 'minimum_version', + 'maximum_version', + 'protocol', + 'options', + 'verify_mode', + ) + copy_attributes(info_add, ctx, f'ssl.{name}.%s', attributes) + + env_names = ["OPENSSL_CONF", "SSLKEYLOGFILE"] + if _ssl is not None and hasattr(_ssl, 'get_default_verify_paths'): + parts = _ssl.get_default_verify_paths() + env_names.extend((parts[0], parts[2])) + + for name in env_names: + try: + value = os.environ[name] + except KeyError: + continue + info_add('ssl.environ[%s]' % name, value) + + +def collect_socket(info_add): + try: + import socket + except ImportError: + return + + try: + hostname = socket.gethostname() + except (OSError, AttributeError): + # WASI SDK 16.0 does not have gethostname(2). + if sys.platform != "wasi": + raise + else: + info_add('socket.hostname', hostname) + + +def collect_sqlite(info_add): + try: + import sqlite3 + except ImportError: + return + + attributes = ('sqlite_version',) + copy_attributes(info_add, sqlite3, 'sqlite3.%s', attributes) + + +def collect_zlib(info_add): + try: + import zlib + except ImportError: + return + + attributes = ('ZLIB_VERSION', 'ZLIB_RUNTIME_VERSION') + copy_attributes(info_add, zlib, 'zlib.%s', attributes) + + +def collect_expat(info_add): + try: + from xml.parsers import expat + except ImportError: + return + + attributes = ('EXPAT_VERSION',) + copy_attributes(info_add, expat, 'expat.%s', attributes) + + +def collect_decimal(info_add): + try: + import _decimal + except ImportError: + return + + attributes = ('__libmpdec_version__',) + copy_attributes(info_add, _decimal, '_decimal.%s', attributes) + + +def collect_testcapi(info_add): + try: + import _testcapi + except ImportError: + return + + for name in ( + 'LONG_MAX', # always 32-bit on Windows, 64-bit on 64-bit Unix + 'PY_SSIZE_T_MAX', + 'Py_C_RECURSION_LIMIT', + 'SIZEOF_TIME_T', # 32-bit or 64-bit depending on the platform + 'SIZEOF_WCHAR_T', # 16-bit or 32-bit depending on the platform + ): + copy_attr(info_add, f'_testcapi.{name}', _testcapi, name) + + +def collect_testinternalcapi(info_add): + try: + import _testinternalcapi + except ImportError: + return + + call_func(info_add, 'pymem.allocator', _testinternalcapi, 'pymem_getallocatorsname') + + for name in ( + 'SIZEOF_PYGC_HEAD', + 'SIZEOF_PYOBJECT', + ): + copy_attr(info_add, f'_testinternalcapi.{name}', _testinternalcapi, name) + + +def collect_resource(info_add): + try: + import resource + except ImportError: + return + + limits = [attr for attr in dir(resource) if attr.startswith('RLIMIT_')] + for name in limits: + key = getattr(resource, name) + value = resource.getrlimit(key) + info_add('resource.%s' % name, value) + + call_func(info_add, 'resource.pagesize', resource, 'getpagesize') + + +def collect_test_socket(info_add): + import unittest + try: + from test import test_socket + except (ImportError, unittest.SkipTest): + return + + # all check attributes like HAVE_SOCKET_CAN + attributes = [name for name in dir(test_socket) + if name.startswith('HAVE_')] + copy_attributes(info_add, test_socket, 'test_socket.%s', attributes) + + +def collect_support(info_add): + try: + from test import support + except ImportError: + return + + attributes = ( + 'MS_WINDOWS', + 'has_fork_support', + 'has_socket_support', + 'has_strftime_extensions', + 'has_subprocess_support', + 'is_android', + 'is_emscripten', + 'is_jython', + 'is_wasi', + ) + copy_attributes(info_add, support, 'support.%s', attributes) + + call_func(info_add, 'support._is_gui_available', support, '_is_gui_available') + call_func(info_add, 'support.python_is_optimized', support, 'python_is_optimized') + + info_add('support.check_sanitizer(address=True)', + support.check_sanitizer(address=True)) + info_add('support.check_sanitizer(memory=True)', + support.check_sanitizer(memory=True)) + info_add('support.check_sanitizer(ub=True)', + support.check_sanitizer(ub=True)) + + +def collect_support_os_helper(info_add): + try: + from test.support import os_helper + except ImportError: + return + + for name in ( + 'can_symlink', + 'can_xattr', + 'can_chmod', + 'can_dac_override', + ): + func = getattr(os_helper, name) + info_add(f'support_os_helper.{name}', func()) + + +def collect_support_socket_helper(info_add): + try: + from test.support import socket_helper + except ImportError: + return + + attributes = ( + 'IPV6_ENABLED', + 'has_gethostname', + ) + copy_attributes(info_add, socket_helper, 'support_socket_helper.%s', attributes) + + for name in ( + 'tcp_blackhole', + ): + func = getattr(socket_helper, name) + info_add(f'support_socket_helper.{name}', func()) + + +def collect_support_threading_helper(info_add): + try: + from test.support import threading_helper + except ImportError: + return + + attributes = ( + 'can_start_thread', + ) + copy_attributes(info_add, threading_helper, 'support_threading_helper.%s', attributes) + + +def collect_cc(info_add): + import subprocess + import sysconfig + + CC = sysconfig.get_config_var('CC') + if not CC: + return + + try: + import shlex + args = shlex.split(CC) + except ImportError: + args = CC.split() + args.append('--version') + try: + proc = subprocess.Popen(args, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + universal_newlines=True) + except OSError: + # Cannot run the compiler, for example when Python has been + # cross-compiled and installed on the target platform where the + # compiler is missing. + return + + stdout = proc.communicate()[0] + if proc.returncode: + # CC --version failed: ignore error + return + + text = stdout.splitlines()[0] + text = normalize_text(text) + info_add('CC.version', text) + + +def collect_gdbm(info_add): + try: + from _gdbm import _GDBM_VERSION + except ImportError: + return + + info_add('gdbm.GDBM_VERSION', '.'.join(map(str, _GDBM_VERSION))) + + +def collect_get_config(info_add): + # Get global configuration variables, _PyPreConfig and _PyCoreConfig + try: + from _testinternalcapi import get_configs + except ImportError: + return + + all_configs = get_configs() + for config_type in sorted(all_configs): + config = all_configs[config_type] + for key in sorted(config): + info_add('%s[%s]' % (config_type, key), repr(config[key])) + + +def collect_subprocess(info_add): + import subprocess + copy_attributes(info_add, subprocess, 'subprocess.%s', ('_USE_POSIX_SPAWN',)) + + +def collect_windows(info_add): + if sys.platform != "win32": + # Code specific to Windows + return + + # windows.RtlAreLongPathsEnabled: RtlAreLongPathsEnabled() + # windows.is_admin: IsUserAnAdmin() + try: + import ctypes + if not hasattr(ctypes, 'WinDLL'): + raise ImportError + except ImportError: + pass + else: + ntdll = ctypes.WinDLL('ntdll') + BOOLEAN = ctypes.c_ubyte + try: + RtlAreLongPathsEnabled = ntdll.RtlAreLongPathsEnabled + except AttributeError: + res = '' + else: + RtlAreLongPathsEnabled.restype = BOOLEAN + RtlAreLongPathsEnabled.argtypes = () + res = bool(RtlAreLongPathsEnabled()) + info_add('windows.RtlAreLongPathsEnabled', res) + + shell32 = ctypes.windll.shell32 + IsUserAnAdmin = shell32.IsUserAnAdmin + IsUserAnAdmin.restype = BOOLEAN + IsUserAnAdmin.argtypes = () + info_add('windows.is_admin', IsUserAnAdmin()) + + try: + import _winapi + dll_path = _winapi.GetModuleFileName(sys.dllhandle) + info_add('windows.dll_path', dll_path) + except (ImportError, AttributeError): + pass + + # windows.version_caption: "wmic os get Caption,Version /value" command + import subprocess + try: + # When wmic.exe output is redirected to a pipe, + # it uses the OEM code page + proc = subprocess.Popen(["wmic", "os", "get", "Caption,Version", "/value"], + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + encoding="oem", + text=True) + output, stderr = proc.communicate() + if proc.returncode: + output = "" + except OSError: + pass + else: + for line in output.splitlines(): + line = line.strip() + if line.startswith('Caption='): + line = line.removeprefix('Caption=').strip() + if line: + info_add('windows.version_caption', line) + elif line.startswith('Version='): + line = line.removeprefix('Version=').strip() + if line: + info_add('windows.version', line) + + # windows.ver: "ver" command + try: + proc = subprocess.Popen(["ver"], shell=True, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, + text=True) + output = proc.communicate()[0] + if proc.returncode == 0xc0000142: + return + if proc.returncode: + output = "" + except OSError: + return + else: + output = output.strip() + line = output.splitlines()[0] + if line: + info_add('windows.ver', line) + + # windows.developer_mode: get AllowDevelopmentWithoutDevLicense registry + import winreg + try: + key = winreg.OpenKey( + winreg.HKEY_LOCAL_MACHINE, + r"SOFTWARE\Microsoft\Windows\CurrentVersion\AppModelUnlock") + subkey = "AllowDevelopmentWithoutDevLicense" + try: + value, value_type = winreg.QueryValueEx(key, subkey) + finally: + winreg.CloseKey(key) + except OSError: + pass + else: + info_add('windows.developer_mode', "enabled" if value else "disabled") + + +def collect_fips(info_add): + try: + import _hashlib + except ImportError: + _hashlib = None + + if _hashlib is not None: + call_func(info_add, 'fips.openssl_fips_mode', _hashlib, 'get_fips_mode') + + try: + with open("/proc/sys/crypto/fips_enabled", encoding="utf-8") as fp: + line = fp.readline().rstrip() + + if line: + info_add('fips.linux_crypto_fips_enabled', line) + except OSError: + pass + + +def collect_tempfile(info_add): + import tempfile + + info_add('tempfile.gettempdir', tempfile.gettempdir()) + + +def collect_libregrtest_utils(info_add): + try: + from test.libregrtest import utils + except ImportError: + return + + info_add('libregrtests.build_info', ' '.join(utils.get_build_info())) + + +def collect_info(info): + error = False + info_add = info.add + + for collect_func in ( + # collect_urandom() must be the first, to check the getrandom() status. + # Other functions may block on os.urandom() indirectly and so change + # its state. + collect_urandom, + + collect_builtins, + collect_cc, + collect_curses, + collect_datetime, + collect_decimal, + collect_expat, + collect_fips, + collect_gdb, + collect_gdbm, + collect_get_config, + collect_locale, + collect_os, + collect_platform, + collect_pwd, + collect_readline, + collect_resource, + collect_socket, + collect_sqlite, + collect_ssl, + collect_subprocess, + collect_sys, + collect_sysconfig, + collect_testcapi, + collect_testinternalcapi, + collect_tempfile, + collect_time, + collect_tkinter, + collect_windows, + collect_zlib, + collect_libregrtest_utils, + + # Collecting from tests should be last as they have side effects. + collect_test_socket, + collect_support, + collect_support_os_helper, + collect_support_socket_helper, + collect_support_threading_helper, + ): + try: + collect_func(info_add) + except Exception: + error = True + print("ERROR: %s() failed" % (collect_func.__name__), + file=sys.stderr) + traceback.print_exc(file=sys.stderr) + print(file=sys.stderr) + sys.stderr.flush() + + return error + + +def dump_info(info, file=None): + title = "Python debug information" + print(title) + print("=" * len(title)) + print() + + infos = info.get_infos() + infos = sorted(infos.items()) + for key, value in infos: + value = value.replace("\n", " ") + print("%s: %s" % (key, value)) + + +def main(): + info = PythonInfo() + error = collect_info(info) + dump_info(info) + + if error: + print() + print("Collection failed: exit with error", file=sys.stderr) + sys.exit(1) + + +if __name__ == "__main__": + main() diff --git a/Lib/test/seq_tests.py b/Lib/test/seq_tests.py index 59db2664a0..54eb5e65ac 100644 --- a/Lib/test/seq_tests.py +++ b/Lib/test/seq_tests.py @@ -145,6 +145,9 @@ def __getitem__(self, i): self.assertEqual(self.type2test(LyingTuple((2,))), self.type2test((1,))) self.assertEqual(self.type2test(LyingList([2])), self.type2test([1])) + with self.assertRaises(TypeError): + self.type2test(unsupported_arg=[]) + def test_truth(self): self.assertFalse(self.type2test()) self.assertTrue(self.type2test([42])) @@ -423,8 +426,8 @@ def test_pickle(self): self.assertEqual(lst2, lst) self.assertNotEqual(id(lst2), id(lst)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON + @support.suppress_immortalization() def test_free_after_iterating(self): support.check_free_after_iterating(self, iter, self.type2test) support.check_free_after_iterating(self, reversed, self.type2test) diff --git a/Lib/test/site-packages/README.txt b/Lib/test/site-packages/README.txt new file mode 100644 index 0000000000..273f6251a7 --- /dev/null +++ b/Lib/test/site-packages/README.txt @@ -0,0 +1,2 @@ +This directory exists so that 3rd party packages can be installed +here. Read the source for site.py for more details. diff --git a/Lib/test/ssl_servers.py b/Lib/test/ssl_servers.py index a4bd7455d4..c45411f49d 100644 --- a/Lib/test/ssl_servers.py +++ b/Lib/test/ssl_servers.py @@ -14,7 +14,7 @@ here = os.path.dirname(__file__) HOST = socket_helper.HOST -CERTFILE = os.path.join(here, 'keycert.pem') +CERTFILE = os.path.join(here, 'certdata/keycert.pem') # This one's based on HTTPServer, which is based on socketserver diff --git a/Lib/test/string_tests.py b/Lib/test/string_tests.py index 6f402513fd..c5831c47fc 100644 --- a/Lib/test/string_tests.py +++ b/Lib/test/string_tests.py @@ -8,18 +8,12 @@ from collections import UserList import random + class Sequence: def __init__(self, seq='wxyz'): self.seq = seq def __len__(self): return len(self.seq) def __getitem__(self, i): return self.seq[i] -class BadSeq1(Sequence): - def __init__(self): self.seq = [7, 'hello', 123] - def __str__(self): return '{0} {1} {2}'.format(*self.seq) - -class BadSeq2(Sequence): - def __init__(self): self.seq = ['a', 'b', 'c'] - def __len__(self): return 8 class BaseTest: # These tests are for buffers of values (bytes) and not @@ -27,7 +21,7 @@ class BaseTest: # and various string implementations # The type to be tested - # Change in subclasses to change the behaviour of fixtesttype() + # Change in subclasses to change the behaviour of fixtype() type2test = None # Whether the "contained items" of the container are integers in @@ -36,7 +30,7 @@ class BaseTest: contains_bytes = False # All tests pass their arguments to the testing methods - # as str objects. fixtesttype() can be used to propagate + # as str objects. fixtype() can be used to propagate # these arguments to the appropriate type def fixtype(self, obj): if isinstance(obj, str): @@ -160,6 +154,12 @@ def test_count(self): self.assertEqual(rem, 0, '%s != 0 for %s' % (rem, i)) self.assertEqual(r1, r2, '%s != %s for %s' % (r1, r2, i)) + def test_count_keyword(self): + self.assertEqual('aa'.replace('a', 'b', 0), 'aa'.replace('a', 'b', count=0)) + self.assertEqual('aa'.replace('a', 'b', 1), 'aa'.replace('a', 'b', count=1)) + self.assertEqual('aa'.replace('a', 'b', 2), 'aa'.replace('a', 'b', count=2)) + self.assertEqual('aa'.replace('a', 'b', 3), 'aa'.replace('a', 'b', count=3)) + def test_find(self): self.checkequal(0, 'abcdefghiabc', 'find', 'abc') self.checkequal(9, 'abcdefghiabc', 'find', 'abc', 1) @@ -327,11 +327,12 @@ def reference_find(p, s): for i in range(len(s)): if s.startswith(p, i): return i + if p == '' and s == '': + return 0 return -1 - rr = random.randrange - choices = random.choices - for _ in range(1000): + def check_pattern(rr): + choices = random.choices p0 = ''.join(choices('abcde', k=rr(10))) * rr(10, 20) p = p0[:len(p0) - rr(10)] # pop off some characters left = ''.join(choices('abcdef', k=rr(2000))) @@ -341,6 +342,49 @@ def reference_find(p, s): self.checkequal(reference_find(p, text), text, 'find', p) + rr = random.randrange + for _ in range(1000): + check_pattern(rr) + + # Test that empty string always work: + check_pattern(lambda *args: 0) + + def test_find_many_lengths(self): + haystack_repeats = [a * 10**e for e in range(6) for a in (1,2,5)] + haystacks = [(n, self.fixtype("abcab"*n + "da")) for n in haystack_repeats] + + needle_repeats = [a * 10**e for e in range(6) for a in (1, 3)] + needles = [(m, self.fixtype("abcab"*m + "da")) for m in needle_repeats] + + for n, haystack1 in haystacks: + haystack2 = haystack1[:-1] + for m, needle in needles: + answer1 = 5 * (n - m) if m <= n else -1 + self.assertEqual(haystack1.find(needle), answer1, msg=(n,m)) + self.assertEqual(haystack2.find(needle), -1, msg=(n,m)) + + def test_adaptive_find(self): + # This would be very slow for the naive algorithm, + # but str.find() should be O(n + m). + for N in 1000, 10_000, 100_000, 1_000_000: + A, B = 'a' * N, 'b' * N + haystack = A + A + B + A + A + needle = A + B + B + A + self.checkequal(-1, haystack, 'find', needle) + self.checkequal(0, haystack, 'count', needle) + self.checkequal(len(haystack), haystack + needle, 'find', needle) + self.checkequal(1, haystack + needle, 'count', needle) + + def test_find_with_memory(self): + # Test the "Skip with memory" path in the two-way algorithm. + for N in 1000, 3000, 10_000, 30_000: + needle = 'ab' * N + haystack = ('ab'*(N-1) + 'b') * 2 + self.checkequal(-1, haystack, 'find', needle) + self.checkequal(0, haystack, 'count', needle) + self.checkequal(len(haystack), haystack + needle, 'find', needle) + self.checkequal(1, haystack + needle, 'count', needle) + def test_find_shift_table_overflow(self): """When the table of 8-bit shifts overflows.""" N = 2**8 + 100 @@ -724,6 +768,18 @@ def test_replace(self): self.checkraises(TypeError, 'hello', 'replace', 42, 'h') self.checkraises(TypeError, 'hello', 'replace', 'h', 42) + def test_replace_uses_two_way_maxcount(self): + # Test that maxcount works in _two_way_count in fastsearch.h + A, B = "A"*1000, "B"*1000 + AABAA = A + A + B + A + A + ABBA = A + B + B + A + self.checkequal(AABAA + ABBA, + AABAA + ABBA, 'replace', ABBA, "ccc", 0) + self.checkequal(AABAA + "ccc", + AABAA + ABBA, 'replace', ABBA, "ccc", 1) + self.checkequal(AABAA + "ccc", + AABAA + ABBA, 'replace', ABBA, "ccc", 2) + @unittest.skip("TODO: RUSTPYTHON, may only apply to 32-bit platforms") @unittest.skipIf(sys.maxsize > (1 << 32) or struct.calcsize('P') != 4, 'only applies to 32-bit platforms') @@ -734,8 +790,6 @@ def test_replace_overflow(self): self.checkraises(OverflowError, A2_16, "replace", "A", A2_16) self.checkraises(OverflowError, A2_16, "replace", "AA", A2_16+A2_16) - - # Python 3.9 def test_removeprefix(self): self.checkequal('am', 'spam', 'removeprefix', 'sp') self.checkequal('spamspam', 'spamspamspam', 'removeprefix', 'spam') @@ -754,7 +808,6 @@ def test_removeprefix(self): self.checkraises(TypeError, 'hello', 'removeprefix', 'h', 42) self.checkraises(TypeError, 'hello', 'removeprefix', ("he", "l")) - # Python 3.9 def test_removesuffix(self): self.checkequal('sp', 'spam', 'removesuffix', 'am') self.checkequal('spamspam', 'spamspamspam', 'removesuffix', 'spam') @@ -1053,7 +1106,7 @@ def test_splitlines(self): self.checkraises(TypeError, 'abc', 'splitlines', 42, 42) -class CommonTest(BaseTest): +class StringLikeTest(BaseTest): # This testcase contains tests that can be used in all # stringlike classes. Currently this is str and UserString. @@ -1084,11 +1137,6 @@ def test_capitalize_nonascii(self): self.checkequal('\u019b\u1d00\u1d86\u0221\u1fb7', '\u019b\u1d00\u1d86\u0221\u1fb7', 'capitalize') - -class MixinStrUnicodeUserStringTest: - # additional tests that only work for - # stringlike objects, i.e. str, UserString - def test_startswith(self): self.checkequal(True, 'hello', 'startswith', 'he') self.checkequal(True, 'hello', 'startswith', 'hello') @@ -1273,8 +1321,11 @@ def test_join(self): self.checkequal(((('a' * i) + '-') * i)[:-1], '-', 'join', ('a' * i,) * i) - #self.checkequal(str(BadSeq1()), ' ', 'join', BadSeq1()) - self.checkequal('a b c', ' ', 'join', BadSeq2()) + class LiesAboutLengthSeq(Sequence): + def __init__(self): self.seq = ['a', 'b', 'c'] + def __len__(self): return 8 + + self.checkequal('a b c', ' ', 'join', LiesAboutLengthSeq()) self.checkraises(TypeError, ' ', 'join') self.checkraises(TypeError, ' ', 'join', None) @@ -1459,19 +1510,19 @@ def test_find_etc_raise_correct_error_messages(self): # issue 11828 s = 'hello' x = 'x' - self.assertRaisesRegex(TypeError, r'^find\(', s.find, + self.assertRaisesRegex(TypeError, r'^find\b', s.find, x, None, None, None) - self.assertRaisesRegex(TypeError, r'^rfind\(', s.rfind, + self.assertRaisesRegex(TypeError, r'^rfind\b', s.rfind, x, None, None, None) - self.assertRaisesRegex(TypeError, r'^index\(', s.index, + self.assertRaisesRegex(TypeError, r'^index\b', s.index, x, None, None, None) - self.assertRaisesRegex(TypeError, r'^rindex\(', s.rindex, + self.assertRaisesRegex(TypeError, r'^rindex\b', s.rindex, x, None, None, None) - self.assertRaisesRegex(TypeError, r'^count\(', s.count, + self.assertRaisesRegex(TypeError, r'^count\b', s.count, x, None, None, None) - self.assertRaisesRegex(TypeError, r'^startswith\(', s.startswith, + self.assertRaisesRegex(TypeError, r'^startswith\b', s.startswith, x, None, None, None) - self.assertRaisesRegex(TypeError, r'^endswith\(', s.endswith, + self.assertRaisesRegex(TypeError, r'^endswith\b', s.endswith, x, None, None, None) # issue #15534 diff --git a/Lib/test/support/__init__.py b/Lib/test/support/__init__.py index 948bad1ca8..444ca2219c 100644 --- a/Lib/test/support/__init__.py +++ b/Lib/test/support/__init__.py @@ -6,8 +6,8 @@ import contextlib import dataclasses import functools -import getpass -import opcode +import logging +import _opcode import os import re import stat @@ -19,8 +19,6 @@ import unittest import warnings -from .testresult import get_test_runner - __all__ = [ # globals @@ -29,12 +27,11 @@ "Error", "TestFailed", "TestDidNotRun", "ResourceDenied", # io "record_original_stdout", "get_original_stdout", "captured_stdout", - "captured_stdin", "captured_stderr", + "captured_stdin", "captured_stderr", "captured_output", # unittest "is_resource_enabled", "requires", "requires_freebsd_version", - "requires_linux_version", "requires_mac_ver", + "requires_gil_enabled", "requires_linux_version", "requires_mac_ver", "check_syntax_error", - "run_unittest", "run_doctest", "requires_gzip", "requires_bz2", "requires_lzma", "bigmemtest", "bigaddrspacetest", "cpython_only", "get_attribute", "requires_IEEE_754", "requires_zlib", @@ -46,8 +43,8 @@ "check_disallow_instantiation", "check_sanitizer", "skip_if_sanitizer", "requires_limited_api", "requires_specialization", # sys - "is_jython", "is_android", "is_emscripten", "is_wasi", - "check_impl_detail", "unix_shell", "setswitchinterval", + "MS_WINDOWS", "is_jython", "is_android", "is_emscripten", "is_wasi", + "is_apple_mobile", "check_impl_detail", "unix_shell", "setswitchinterval", # os "get_pagesize", # network @@ -60,8 +57,13 @@ "run_with_tz", "PGO", "missing_compiler_executable", "ALWAYS_EQ", "NEVER_EQ", "LARGEST", "SMALLEST", "LOOPBACK_TIMEOUT", "INTERNET_TIMEOUT", "SHORT_TIMEOUT", "LONG_TIMEOUT", - "Py_DEBUG", "EXCEEDS_RECURSION_LIMIT", "C_RECURSION_LIMIT", + "Py_DEBUG", "exceeds_recursion_limit", "get_c_recursion_limit", "skip_on_s390x", + "without_optimizer", + "force_not_colorized", + "force_not_colorized_test_class", + "make_clean_env", + "BrokenIter", ] @@ -107,6 +109,7 @@ STDLIB_DIR = os.path.dirname(TEST_HOME_DIR) REPO_ROOT = os.path.dirname(STDLIB_DIR) + class Error(Exception): """Base class for regression test exceptions.""" @@ -302,6 +305,16 @@ def requires(resource, msg=None): if resource == 'gui' and not _is_gui_available(): raise ResourceDenied(_is_gui_available.reason) +def _get_kernel_version(sysname="Linux"): + import platform + if platform.system() != sysname: + return None + version_txt = platform.release().split('-', 1)[0] + try: + return tuple(map(int, version_txt.split('.'))) + except ValueError: + return None + def _requires_unix_version(sysname, min_version): """Decorator raising SkipTest if the OS is `sysname` and the version is less than `min_version`. @@ -384,7 +397,7 @@ def skip_if_buildbot(reason=None): try: isbuildbot = getpass.getuser().lower() == 'buildbot' except (KeyError, OSError) as err: - warnings.warn(f'getpass.getuser() failed {err}.', RuntimeWarning) + logging.getLogger(__name__).warning('getpass.getuser() failed %s.', err, exc_info=err) isbuildbot = False return unittest.skipIf(isbuildbot, reason) @@ -498,8 +511,7 @@ def requires_lzma(reason='requires lzma'): import lzma except ImportError: lzma = None - # XXX: RUSTPYTHON; xz is not supported yet - lzma = None + lzma = None # XXX: RUSTPYTHON; xz is not supported yet return unittest.skipUnless(lzma, reason) def has_no_debug_ranges(): @@ -511,27 +523,50 @@ def has_no_debug_ranges(): return not bool(config['code_debug_ranges']) def requires_debug_ranges(reason='requires co_positions / debug_ranges'): - return unittest.skipIf(has_no_debug_ranges(), reason) + try: + skip = has_no_debug_ranges() + except unittest.SkipTest as e: + skip = True + reason = e.args[0] if e.args else reason + return unittest.skipIf(skip, reason) -@contextlib.contextmanager -def suppress_immortalization(suppress=True): - """Suppress immortalization of deferred objects.""" + +def can_use_suppress_immortalization(suppress=True): + """Check if suppress_immortalization(suppress) can be used. + + Use this helper in code where SkipTest must be eagerly handled. + """ + if not suppress: + return True try: import _testinternalcapi except ImportError: - yield - return + return False + return True + + +@contextlib.contextmanager +def suppress_immortalization(suppress=True): + """Suppress immortalization of deferred objects. + If _testinternalcapi is not available, the decorated test or class + is skipped. Use can_use_suppress_immortalization() outside test cases + to check if this decorator can be used. + """ if not suppress: - yield + yield # no-op return + from .import_helper import import_module + + _testinternalcapi = import_module("_testinternalcapi") _testinternalcapi.suppress_immortalization(True) try: yield finally: _testinternalcapi.suppress_immortalization(False) + def skip_if_suppress_immortalization(): try: import _testinternalcapi @@ -807,28 +842,47 @@ def gc_collect(): longer than expected. This function tries its best to force all garbage objects to disappear. """ - # TODO: RUSTPYTHON (comment out before) - # import gc - # gc.collect() - # if is_jython: - # time.sleep(0.1) - # gc.collect() - # gc.collect() - pass + return # TODO: RUSTPYTHON + + import gc + gc.collect() + gc.collect() + gc.collect() @contextlib.contextmanager def disable_gc(): - # TODO: RUSTPYTHON (comment out before) - # import gc - # have_gc = gc.isenabled() - # gc.disable() - # try: - # yield - # finally: - # if have_gc: - # gc.enable() - yield + # TODO: RUSTPYTHON; GC is not supported yet + try: + yield + finally: + pass + return + + import gc + have_gc = gc.isenabled() + gc.disable() + try: + yield + finally: + if have_gc: + gc.enable() +@contextlib.contextmanager +def gc_threshold(*args): + # TODO: RUSTPYTHON; GC is not supported yet + try: + yield + finally: + pass + return + + import gc + old_threshold = gc.get_threshold() + gc.set_threshold(*args) + try: + yield + finally: + gc.set_threshold(*old_threshold) def python_is_optimized(): """Find if Python was built with optimizations.""" @@ -837,14 +891,57 @@ def python_is_optimized(): for opt in cflags.split(): if opt.startswith('-O'): final_opt = opt - return final_opt not in ('', '-O0', '-Og') + if sysconfig.get_config_var("CC") == "gcc": + non_opts = ('', '-O0', '-Og') + else: + non_opts = ('', '-O0') + return final_opt not in non_opts + + +def check_cflags_pgo(): + # Check if Python was built with ./configure --enable-optimizations: + # with Profile Guided Optimization (PGO). + cflags_nodist = sysconfig.get_config_var('PY_CFLAGS_NODIST') or '' + pgo_options = [ + # GCC + '-fprofile-use', + # clang: -fprofile-instr-use=code.profclangd + '-fprofile-instr-use', + # ICC + "-prof-use", + ] + PGO_PROF_USE_FLAG = sysconfig.get_config_var('PGO_PROF_USE_FLAG') + if PGO_PROF_USE_FLAG: + pgo_options.append(PGO_PROF_USE_FLAG) + return any(option in cflags_nodist for option in pgo_options) + + +def check_bolt_optimized(): + # Always return false, if the platform is WASI, + # because BOLT optimization does not support WASM binary. + if is_wasi: + return False + config_args = sysconfig.get_config_var('CONFIG_ARGS') or '' + return '--enable-bolt' in config_args + +Py_GIL_DISABLED = bool(sysconfig.get_config_var('Py_GIL_DISABLED')) -_header = 'nP' +def requires_gil_enabled(msg="needs the GIL enabled"): + """Decorator for skipping tests on the free-threaded build.""" + return unittest.skipIf(Py_GIL_DISABLED, msg) + +def expected_failure_if_gil_disabled(): + """Expect test failure if the GIL is disabled.""" + if Py_GIL_DISABLED: + return unittest.expectedFailure + return lambda test_case: test_case + +if Py_GIL_DISABLED: + _header = 'PHBBInP' +else: + _header = 'nP' _align = '0n' -if hasattr(sys, "getobjects"): - _header = '2P' + _header - _align = '0P' _vheader = _header + 'n' def calcobjsize(fmt): @@ -873,9 +970,34 @@ def check_sizeof(test, o, size): % (type(o), result, size) test.assertEqual(result, size, msg) +def subTests(arg_names, arg_values, /, *, _do_cleanups=False): + """Run multiple subtests with different parameters. + """ + single_param = False + if isinstance(arg_names, str): + arg_names = arg_names.replace(',',' ').split() + if len(arg_names) == 1: + single_param = True + arg_values = tuple(arg_values) + def decorator(func): + if isinstance(func, type): + raise TypeError('subTests() can only decorate methods, not classes') + @functools.wraps(func) + def wrapper(self, /, *args, **kwargs): + for values in arg_values: + if single_param: + values = (values,) + subtest_kwargs = dict(zip(arg_names, values)) + with self.subTest(**subtest_kwargs): + func(self, *args, **kwargs, **subtest_kwargs) + if _do_cleanups: + self.doCleanups() + return wrapper + return decorator + #======================================================================= -# Decorator for running a function in a different locale, correctly resetting -# it afterwards. +# Decorator/context manager for running a code in a different locale, +# correctly resetting it afterwards. @contextlib.contextmanager def run_with_locale(catstr, *locales): @@ -886,16 +1008,21 @@ def run_with_locale(catstr, *locales): except AttributeError: # if the test author gives us an invalid category string raise - except: + except Exception: # cannot retrieve original locale, so do nothing locale = orig_locale = None + if '' not in locales: + raise unittest.SkipTest('no locales') else: for loc in locales: try: locale.setlocale(category, loc) break - except: + except locale.Error: pass + else: + if '' not in locales: + raise unittest.SkipTest(f'no locales {locales}') try: yield @@ -903,6 +1030,46 @@ def run_with_locale(catstr, *locales): if locale and orig_locale: locale.setlocale(category, orig_locale) +#======================================================================= +# Decorator for running a function in multiple locales (if they are +# availasble) and resetting the original locale afterwards. + +def run_with_locales(catstr, *locales): + def deco(func): + @functools.wraps(func) + def wrapper(self, /, *args, **kwargs): + dry_run = '' in locales + try: + import locale + category = getattr(locale, catstr) + orig_locale = locale.setlocale(category) + except AttributeError: + # if the test author gives us an invalid category string + raise + except Exception: + # cannot retrieve original locale, so do nothing + pass + else: + try: + for loc in locales: + with self.subTest(locale=loc): + try: + locale.setlocale(category, loc) + except locale.Error: + self.skipTest(f'no locale {loc!r}') + else: + dry_run = False + func(self, *args, **kwargs) + finally: + locale.setlocale(category, orig_locale) + if dry_run: + # no locales available, so just run the test + # with the current locale + with self.subTest(locale=None): + func(self, *args, **kwargs) + return wrapper + return deco + #======================================================================= # Decorator for running a function in a specific timezone, correctly # resetting it afterwards. @@ -949,27 +1116,31 @@ def inner(*args, **kwds): MAX_Py_ssize_t = sys.maxsize -def set_memlimit(limit): - global max_memuse - global real_max_memuse +def _parse_memlimit(limit: str) -> int: sizes = { 'k': 1024, 'm': _1M, 'g': _1G, 't': 1024*_1G, } - m = re.match(r'(\d+(\.\d+)?) (K|M|G|T)b?$', limit, + m = re.match(r'(\d+(?:\.\d+)?) (K|M|G|T)b?$', limit, re.IGNORECASE | re.VERBOSE) if m is None: - raise ValueError('Invalid memory limit %r' % (limit,)) - memlimit = int(float(m.group(1)) * sizes[m.group(3).lower()]) - real_max_memuse = memlimit - if memlimit > MAX_Py_ssize_t: - memlimit = MAX_Py_ssize_t + raise ValueError(f'Invalid memory limit: {limit!r}') + return int(float(m.group(1)) * sizes[m.group(2).lower()]) + +def set_memlimit(limit: str) -> None: + global max_memuse + global real_max_memuse + memlimit = _parse_memlimit(limit) if memlimit < _2G - 1: - raise ValueError('Memory limit %r too low to be useful' % (limit,)) + raise ValueError(f'Memory limit {limit!r} too low to be useful') + + real_max_memuse = memlimit + memlimit = min(memlimit, MAX_Py_ssize_t) max_memuse = memlimit + class _MemoryWatchdog: """An object which periodically watches the process' memory consumption and prints it out. @@ -984,8 +1155,7 @@ def start(self): try: f = open(self.procfile, 'r') except OSError as e: - warnings.warn('/proc not available for stats: {}'.format(e), - RuntimeWarning) + logging.getLogger(__name__).warning('/proc not available for stats: %s', e, exc_info=e) sys.stderr.flush() return @@ -1122,18 +1292,30 @@ def check_impl_detail(**guards): def no_tracing(func): """Decorator to temporarily turn off tracing for the duration of a test.""" - if not hasattr(sys, 'gettrace'): - return func - else: + trace_wrapper = func + if hasattr(sys, 'gettrace'): @functools.wraps(func) - def wrapper(*args, **kwargs): + def trace_wrapper(*args, **kwargs): original_trace = sys.gettrace() try: sys.settrace(None) return func(*args, **kwargs) finally: sys.settrace(original_trace) - return wrapper + + coverage_wrapper = trace_wrapper + if 'test.cov' in sys.modules: # -Xpresite=test.cov used + cov = sys.monitoring.COVERAGE_ID + @functools.wraps(func) + def coverage_wrapper(*args, **kwargs): + original_events = sys.monitoring.get_events(cov) + try: + sys.monitoring.set_events(cov, 0) + return trace_wrapper(*args, **kwargs) + finally: + sys.monitoring.set_events(cov, original_events) + + return coverage_wrapper def refcount_test(test): @@ -1150,305 +1332,109 @@ def refcount_test(test): def requires_limited_api(test): try: import _testcapi + import _testlimitedcapi except ImportError: - return unittest.skip('needs _testcapi module')(test) - return unittest.skipUnless( - _testcapi.LIMITED_API_AVAILABLE, 'needs Limited API support')(test) + return unittest.skip('needs _testcapi and _testlimitedcapi modules')(test) + return test + + +# Windows build doesn't support --disable-test-modules feature, so there's no +# 'TEST_MODULES' var in config +TEST_MODULES_ENABLED = (sysconfig.get_config_var('TEST_MODULES') or 'yes') == 'yes' def requires_specialization(test): return unittest.skipUnless( - opcode.ENABLE_SPECIALIZATION, "requires specialization")(test) - -def _filter_suite(suite, pred): - """Recursively filter test cases in a suite based on a predicate.""" - newtests = [] - for test in suite._tests: - if isinstance(test, unittest.TestSuite): - _filter_suite(test, pred) - newtests.append(test) - else: - if pred(test): - newtests.append(test) - suite._tests = newtests + _opcode.ENABLE_SPECIALIZATION, "requires specialization")(test) -@dataclasses.dataclass(slots=True) -class TestStats: - tests_run: int = 0 - failures: int = 0 - skipped: int = 0 - @staticmethod - def from_unittest(result): - return TestStats(result.testsRun, - len(result.failures), - len(result.skipped)) +#======================================================================= +# Check for the presence of docstrings. - @staticmethod - def from_doctest(results): - return TestStats(results.attempted, - results.failed) +# Rather than trying to enumerate all the cases where docstrings may be +# disabled, we just check for that directly - def accumulate(self, stats): - self.tests_run += stats.tests_run - self.failures += stats.failures - self.skipped += stats.skipped +def _check_docstrings(): + """Just used to check if docstrings are enabled""" +MISSING_C_DOCSTRINGS = (check_impl_detail() and + sys.platform != 'win32' and + not sysconfig.get_config_var('WITH_DOC_STRINGS')) -def _run_suite(suite): - """Run tests from a unittest.TestSuite-derived class.""" - runner = get_test_runner(sys.stdout, - verbosity=verbose, - capture_output=(junit_xml_list is not None)) +HAVE_PY_DOCSTRINGS = _check_docstrings.__doc__ is not None +HAVE_DOCSTRINGS = (HAVE_PY_DOCSTRINGS and not MISSING_C_DOCSTRINGS) - result = runner.run(suite) +requires_docstrings = unittest.skipUnless(HAVE_DOCSTRINGS, + "test requires docstrings") - if junit_xml_list is not None: - junit_xml_list.append(result.get_xml_element()) - if not result.testsRun and not result.skipped and not result.errors: - raise TestDidNotRun - if not result.wasSuccessful(): - stats = TestStats.from_unittest(result) - if len(result.errors) == 1 and not result.failures: - err = result.errors[0][1] - elif len(result.failures) == 1 and not result.errors: - err = result.failures[0][1] - else: - err = "multiple errors occurred" - if not verbose: err += "; run in verbose mode for details" - errors = [(str(tc), exc_str) for tc, exc_str in result.errors] - failures = [(str(tc), exc_str) for tc, exc_str in result.failures] - raise TestFailedWithDetails(err, errors, failures, stats=stats) - return result +#======================================================================= +# Support for saving and restoring the imported modules. +def flush_std_streams(): + if sys.stdout is not None: + sys.stdout.flush() + if sys.stderr is not None: + sys.stderr.flush() -# By default, don't filter tests -_match_test_func = None -_accept_test_patterns = None -_ignore_test_patterns = None +def print_warning(msg): + # bpo-45410: Explicitly flush stdout to keep logs in order + flush_std_streams() + stream = print_warning.orig_stderr + for line in msg.splitlines(): + print(f"Warning -- {line}", file=stream) + stream.flush() +# bpo-39983: Store the original sys.stderr at Python startup to be able to +# log warnings even if sys.stderr is captured temporarily by a test. +print_warning.orig_stderr = sys.stderr -def match_test(test): - # Function used by support.run_unittest() and regrtest --list-cases - if _match_test_func is None: - return True - else: - return _match_test_func(test.id()) +# Flag used by saved_test_environment of test.libregrtest.save_env, +# to check if a test modified the environment. The flag should be set to False +# before running a new test. +# +# For example, threading_helper.threading_cleanup() sets the flag is the function fails +# to cleanup threads. +environment_altered = False -def _is_full_match_test(pattern): - # If a pattern contains at least one dot, it's considered - # as a full test identifier. - # Example: 'test.test_os.FileTests.test_access'. - # - # ignore patterns which contain fnmatch patterns: '*', '?', '[...]' - # or '[!...]'. For example, ignore 'test_access*'. - return ('.' in pattern) and (not re.search(r'[?*\[\]]', pattern)) +def reap_children(): + """Use this function at the end of test_main() whenever sub-processes + are started. This will help ensure that no extra children (zombies) + stick around to hog resources and create problems when looking + for refleaks. + """ + global environment_altered + # Need os.waitpid(-1, os.WNOHANG): Windows is not supported + if not (hasattr(os, 'waitpid') and hasattr(os, 'WNOHANG')): + return + elif not has_subprocess_support: + return -def set_match_tests(accept_patterns=None, ignore_patterns=None): - global _match_test_func, _accept_test_patterns, _ignore_test_patterns + # Reap all our dead child processes so we don't leave zombies around. + # These hog resources and might be causing some of the buildbots to die. + while True: + try: + # Read the exit status of any child process which already completed + pid, status = os.waitpid(-1, os.WNOHANG) + except OSError: + break - if accept_patterns is None: - accept_patterns = () - if ignore_patterns is None: - ignore_patterns = () + if pid == 0: + break - accept_func = ignore_func = None + print_warning(f"reap_children() reaped child process {pid}") + environment_altered = True - if accept_patterns != _accept_test_patterns: - accept_patterns, accept_func = _compile_match_function(accept_patterns) - if ignore_patterns != _ignore_test_patterns: - ignore_patterns, ignore_func = _compile_match_function(ignore_patterns) - # Create a copy since patterns can be mutable and so modified later - _accept_test_patterns = tuple(accept_patterns) - _ignore_test_patterns = tuple(ignore_patterns) +@contextlib.contextmanager +def swap_attr(obj, attr, new_val): + """Temporary swap out an attribute with a new object. - if accept_func is not None or ignore_func is not None: - def match_function(test_id): - accept = True - ignore = False - if accept_func: - accept = accept_func(test_id) - if ignore_func: - ignore = ignore_func(test_id) - return accept and not ignore - - _match_test_func = match_function - - -def _compile_match_function(patterns): - if not patterns: - func = None - # set_match_tests(None) behaves as set_match_tests(()) - patterns = () - elif all(map(_is_full_match_test, patterns)): - # Simple case: all patterns are full test identifier. - # The test.bisect_cmd utility only uses such full test identifiers. - func = set(patterns).__contains__ - else: - import fnmatch - regex = '|'.join(map(fnmatch.translate, patterns)) - # The search *is* case sensitive on purpose: - # don't use flags=re.IGNORECASE - regex_match = re.compile(regex).match - - def match_test_regex(test_id): - if regex_match(test_id): - # The regex matches the whole identifier, for example - # 'test.test_os.FileTests.test_access'. - return True - else: - # Try to match parts of the test identifier. - # For example, split 'test.test_os.FileTests.test_access' - # into: 'test', 'test_os', 'FileTests' and 'test_access'. - return any(map(regex_match, test_id.split("."))) - - func = match_test_regex - - return patterns, func - - -def run_unittest(*classes): - """Run tests from unittest.TestCase-derived classes.""" - valid_types = (unittest.TestSuite, unittest.TestCase) - loader = unittest.TestLoader() - suite = unittest.TestSuite() - for cls in classes: - if isinstance(cls, str): - if cls in sys.modules: - suite.addTest(loader.loadTestsFromModule(sys.modules[cls])) - else: - raise ValueError("str arguments must be keys in sys.modules") - elif isinstance(cls, valid_types): - suite.addTest(cls) - else: - suite.addTest(loader.loadTestsFromTestCase(cls)) - _filter_suite(suite, match_test) - return _run_suite(suite) - -#======================================================================= -# Check for the presence of docstrings. - -# Rather than trying to enumerate all the cases where docstrings may be -# disabled, we just check for that directly - -def _check_docstrings(): - """Just used to check if docstrings are enabled""" - -MISSING_C_DOCSTRINGS = (check_impl_detail() and - sys.platform != 'win32' and - not sysconfig.get_config_var('WITH_DOC_STRINGS')) - -HAVE_DOCSTRINGS = (_check_docstrings.__doc__ is not None and - not MISSING_C_DOCSTRINGS) - -requires_docstrings = unittest.skipUnless(HAVE_DOCSTRINGS, - "test requires docstrings") - - -#======================================================================= -# doctest driver. - -def run_doctest(module, verbosity=None, optionflags=0): - """Run doctest on the given module. Return (#failures, #tests). - - If optional argument verbosity is not specified (or is None), pass - support's belief about verbosity on to doctest. Else doctest's - usual behavior is used (it searches sys.argv for -v). - """ - - import doctest - - if verbosity is None: - verbosity = verbose - else: - verbosity = None - - results = doctest.testmod(module, - verbose=verbosity, - optionflags=optionflags) - if results.failed: - stats = TestStats.from_doctest(results) - raise TestFailed(f"{results.failed} of {results.attempted} " - f"doctests failed", - stats=stats) - if verbose: - print('doctest (%s) ... %d tests with zero failures' % - (module.__name__, results.attempted)) - return results - - -#======================================================================= -# Support for saving and restoring the imported modules. - -def flush_std_streams(): - if sys.stdout is not None: - sys.stdout.flush() - if sys.stderr is not None: - sys.stderr.flush() - - -def print_warning(msg): - # bpo-45410: Explicitly flush stdout to keep logs in order - flush_std_streams() - stream = print_warning.orig_stderr - for line in msg.splitlines(): - print(f"Warning -- {line}", file=stream) - stream.flush() - -# bpo-39983: Store the original sys.stderr at Python startup to be able to -# log warnings even if sys.stderr is captured temporarily by a test. -print_warning.orig_stderr = sys.stderr - - -# Flag used by saved_test_environment of test.libregrtest.save_env, -# to check if a test modified the environment. The flag should be set to False -# before running a new test. -# -# For example, threading_helper.threading_cleanup() sets the flag is the function fails -# to cleanup threads. -environment_altered = False - -def reap_children(): - """Use this function at the end of test_main() whenever sub-processes - are started. This will help ensure that no extra children (zombies) - stick around to hog resources and create problems when looking - for refleaks. - """ - global environment_altered - - # Need os.waitpid(-1, os.WNOHANG): Windows is not supported - if not (hasattr(os, 'waitpid') and hasattr(os, 'WNOHANG')): - return - elif not has_subprocess_support: - return - - # Reap all our dead child processes so we don't leave zombies around. - # These hog resources and might be causing some of the buildbots to die. - while True: - try: - # Read the exit status of any child process which already completed - pid, status = os.waitpid(-1, os.WNOHANG) - except OSError: - break - - if pid == 0: - break - - print_warning(f"reap_children() reaped child process {pid}") - environment_altered = True - - -@contextlib.contextmanager -def swap_attr(obj, attr, new_val): - """Temporary swap out an attribute with a new object. - - Usage: - with swap_attr(obj, "attr", 5): - ... + Usage: + with swap_attr(obj, "attr", 5): + ... This will set obj.attr to 5 for the duration of the with: block, restoring the old value at the end of the block. If `attr` doesn't @@ -1686,7 +1672,7 @@ def check__all__(test_case, module, name_of_module=None, extra=(), 'module'. The 'name_of_module' argument can specify (as a string or tuple thereof) - what module(s) an API could be defined in in order to be detected as a + what module(s) an API could be defined in order to be detected as a public API. One case for this is when 'module' imports part of its public API from other modules, possibly a C backend (like 'csv' and its '_csv'). @@ -1903,7 +1889,10 @@ def run_in_subinterp(code): module is enabled. """ _check_tracemalloc() - import _testcapi + try: + import _testcapi + except ImportError: + raise unittest.SkipTest("requires _testcapi") return _testcapi.run_in_subinterp(code) @@ -1913,11 +1902,25 @@ def run_in_subinterp_with_config(code, *, own_gil=None, **config): module is enabled. """ _check_tracemalloc() - import _testcapi + try: + import _testinternalcapi + except ImportError: + raise unittest.SkipTest("requires _testinternalcapi") if own_gil is not None: assert 'gil' not in config, (own_gil, config) - config['gil'] = 2 if own_gil else 1 - return _testcapi.run_in_subinterp_with_config(code, **config) + config['gil'] = 'own' if own_gil else 'shared' + else: + gil = config['gil'] + if gil == 0: + config['gil'] = 'default' + elif gil == 1: + config['gil'] = 'shared' + elif gil == 2: + config['gil'] = 'own' + elif not isinstance(gil, str): + raise NotImplementedError(gil) + config = types.SimpleNamespace(**config) + return _testinternalcapi.run_in_subinterp_with_config(code, config) def _check_tracemalloc(): @@ -1933,24 +1936,31 @@ def _check_tracemalloc(): "if tracemalloc module is tracing " "memory allocations") -# TODO: RUSTPYTHON; GC is not supported yet -# def check_free_after_iterating(test, iter, cls, args=()): -# class A(cls): -# def __del__(self): -# nonlocal done -# done = True -# try: -# next(it) -# except StopIteration: -# pass - -# done = False -# it = iter(A(*args)) -# # Issue 26494: Shouldn't crash -# test.assertRaises(StopIteration, next, it) -# # The sequence should be deallocated just after the end of iterating -# gc_collect() -# test.assertTrue(done) + +def check_free_after_iterating(test, iter, cls, args=()): + # TODO: RUSTPYTHON; GC is not supported yet + test.assertTrue(False) + return + + done = False + def wrapper(): + class A(cls): + def __del__(self): + nonlocal done + done = True + try: + next(it) + except StopIteration: + pass + + it = iter(A(*args)) + # Issue 26494: Shouldn't crash + test.assertRaises(StopIteration, next, it) + + wrapper() + # The sequence should be deallocated just after the end of iterating + gc_collect() + test.assertTrue(done) def missing_compiler_executable(cmd_names=[]): @@ -1962,8 +1972,9 @@ def missing_compiler_executable(cmd_names=[]): missing. """ - from setuptools._distutils import ccompiler, sysconfig, spawn + from setuptools._distutils import ccompiler, sysconfig from setuptools import errors + import shutil compiler = ccompiler.new_compiler() sysconfig.customize_compiler(compiler) @@ -1982,22 +1993,22 @@ def missing_compiler_executable(cmd_names=[]): "the '%s' executable is not configured" % name elif not cmd: continue - if spawn.find_executable(cmd[0]) is None: + if shutil.which(cmd[0]) is None: return cmd[0] -_is_android_emulator = None +_old_android_emulator = None def setswitchinterval(interval): # Setting a very low gil interval on the Android emulator causes python # to hang (issue #26939). - minimum_interval = 1e-5 + minimum_interval = 1e-4 # 100 us if is_android and interval < minimum_interval: - global _is_android_emulator - if _is_android_emulator is None: - import subprocess - _is_android_emulator = (subprocess.check_output( - ['getprop', 'ro.kernel.qemu']).strip() == b'1') - if _is_android_emulator: + global _old_android_emulator + if _old_android_emulator is None: + import platform + av = platform.android_ver() + _old_android_emulator = av.is_emulator and av.api_level < 24 + if _old_android_emulator: interval = minimum_interval return sys.setswitchinterval(interval) @@ -2072,8 +2083,19 @@ def restore(self): def with_pymalloc(): - import _testcapi - return _testcapi.WITH_PYMALLOC + try: + import _testcapi + except ImportError: + raise unittest.SkipTest("requires _testcapi") + return _testcapi.WITH_PYMALLOC and not Py_GIL_DISABLED + + +def with_mimalloc(): + try: + import _testcapi + except ImportError: + raise unittest.SkipTest("requires _testcapi") + return _testcapi.WITH_MIMALLOC class _ALWAYS_EQ: @@ -2301,6 +2323,7 @@ def check_disallow_instantiation(testcase, tp, *args, **kwds): qualname = f"{name}" msg = f"cannot create '{re.escape(qualname)}' instances" testcase.assertRaisesRegex(TypeError, msg, tp, *args, **kwds) + testcase.assertRaisesRegex(TypeError, msg, tp.__new__, tp, *args, **kwds) def get_recursion_depth(): """Get the recursion depth of the caller function. @@ -2345,14 +2368,14 @@ def set_recursion_limit(limit): finally: sys.setrecursionlimit(original_limit) -def infinite_recursion(max_depth=100): - """Set a lower limit for tests that interact with infinite recursions - (e.g test_ast.ASTHelpers_Test.test_recursion_direct) since on some - debug windows builds, due to not enough functions being inlined the - stack size might not handle the default recursion limit (1000). See - bpo-11105 for details.""" - if max_depth < 3: - raise ValueError("max_depth must be at least 3, got {max_depth}") +def infinite_recursion(max_depth=None): + if max_depth is None: + # Pick a number large enough to cause problems + # but not take too long for code that can handle + # very deep recursion. + max_depth = 20_000 + elif max_depth < 3: + raise ValueError(f"max_depth must be at least 3, got {max_depth}") depth = get_recursion_depth() depth = max(depth - 1, 1) # Ignore infinite_recursion() frame. limit = depth + max_depth @@ -2412,11 +2435,13 @@ def _findwheel(pkgname): If set, the wheels are searched for in WHEEL_PKG_DIR (see ensurepip). Otherwise, they are searched for in the test directory. """ - wheel_dir = sysconfig.get_config_var('WHEEL_PKG_DIR') or TEST_HOME_DIR + wheel_dir = sysconfig.get_config_var('WHEEL_PKG_DIR') or os.path.join( + TEST_HOME_DIR, 'wheeldata', + ) filenames = os.listdir(wheel_dir) filenames = sorted(filenames, reverse=True) # approximate "newest" first for filename in filenames: - # filename is like 'setuptools-67.6.1-py3-none-any.whl' + # filename is like 'setuptools-{version}-py3-none-any.whl' if not filename.endswith(".whl"): continue prefix = pkgname + '-' @@ -2425,20 +2450,29 @@ def _findwheel(pkgname): raise FileNotFoundError(f"No wheel for {pkgname} found in {wheel_dir}") -# Context manager that creates a virtual environment, install setuptools and wheel in it -# and returns the path to the venv directory and the path to the python executable +# Context manager that creates a virtual environment, install setuptools in it, +# and returns the paths to the venv directory and the python executable @contextlib.contextmanager -def setup_venv_with_pip_setuptools_wheel(venv_dir): +def setup_venv_with_pip_setuptools(venv_dir): import subprocess from .os_helper import temp_cwd + def run_command(cmd): + if verbose: + import shlex + print() + print('Run:', ' '.join(map(shlex.quote, cmd))) + subprocess.run(cmd, check=True) + else: + subprocess.run(cmd, + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + check=True) + with temp_cwd() as temp_dir: # Create virtual environment to get setuptools cmd = [sys.executable, '-X', 'dev', '-m', 'venv', venv_dir] - if verbose: - print() - print('Run:', ' '.join(cmd)) - subprocess.run(cmd, check=True) + run_command(cmd) venv = os.path.join(temp_dir, venv_dir) @@ -2449,14 +2483,11 @@ def setup_venv_with_pip_setuptools_wheel(venv_dir): else: python = os.path.join(venv, 'bin', python_exe) - cmd = [python, '-X', 'dev', + cmd = (python, '-X', 'dev', '-m', 'pip', 'install', _findwheel('setuptools'), - _findwheel('wheel')] - if verbose: - print() - print('Run:', ' '.join(cmd)) - subprocess.run(cmd, check=True) + ) + run_command(cmd) yield python @@ -2579,6 +2610,46 @@ def sleeping_retry(timeout, err_msg=None, /, delay = min(delay * 2, max_delay) +class CPUStopwatch: + """Context manager to roughly time a CPU-bound operation. + + Disables GC. Uses CPU time if it can (i.e. excludes sleeps & time of + other processes). + + N.B.: + - This *includes* time spent in other threads. + - Some systems only have a coarse resolution; check + stopwatch.clock_info.rseolution if. + + Usage: + + with ProcessStopwatch() as stopwatch: + ... + elapsed = stopwatch.seconds + resolution = stopwatch.clock_info.resolution + """ + def __enter__(self): + get_time = time.process_time + clock_info = time.get_clock_info('process_time') + if get_time() <= 0: # some platforms like WASM lack process_time() + get_time = time.monotonic + clock_info = time.get_clock_info('monotonic') + self.context = disable_gc() + self.context.__enter__() + self.get_time = get_time + self.clock_info = clock_info + self.start_time = get_time() + return self + + def __exit__(self, *exc): + try: + end_time = self.get_time() + finally: + result = self.context.__exit__(*exc) + self.seconds = end_time - self.start_time + return result + + @contextlib.contextmanager def adjust_int_max_str_digits(max_digits): """Temporarily change the integer string conversion length limit.""" @@ -2589,33 +2660,139 @@ def adjust_int_max_str_digits(max_digits): finally: sys.set_int_max_str_digits(current) -#For recursion tests, easily exceeds default recursion limit -EXCEEDS_RECURSION_LIMIT = 5000 -# The default C recursion limit (from Include/cpython/pystate.h). -C_RECURSION_LIMIT = 1500 +def get_c_recursion_limit(): + try: + import _testcapi + return _testcapi.Py_C_RECURSION_LIMIT + except ImportError: + raise unittest.SkipTest('requires _testcapi') + + +def exceeds_recursion_limit(): + """For recursion tests, easily exceeds default recursion limit.""" + return get_c_recursion_limit() * 3 + # Windows doesn't have os.uname() but it doesn't support s390x. is_s390x = hasattr(os, 'uname') and os.uname().machine == 's390x' -skip_on_s390x = unittest.skipIf(hasattr(os, 'uname') and os.uname().machine == 's390x', - 'skipped on s390x') -HAVE_ASAN_FORK_BUG = check_sanitizer(address=True) +skip_on_s390x = unittest.skipIf(is_s390x, 'skipped on s390x') + +Py_TRACE_REFS = hasattr(sys, 'getobjects') + +# Decorator to disable optimizer while a function run +def without_optimizer(func): + try: + from _testinternalcapi import get_optimizer, set_optimizer + except ImportError: + return func + @functools.wraps(func) + def wrapper(*args, **kwargs): + save_opt = get_optimizer() + try: + set_optimizer(None) + return func(*args, **kwargs) + finally: + set_optimizer(save_opt) + return wrapper + + +_BASE_COPY_SRC_DIR_IGNORED_NAMES = frozenset({ + # SRC_DIR/.git + '.git', + # ignore all __pycache__/ sub-directories + '__pycache__', +}) + +# Ignore function for shutil.copytree() to copy the Python source code. +def copy_python_src_ignore(path, names): + ignored = _BASE_COPY_SRC_DIR_IGNORED_NAMES + if os.path.basename(path) == 'Doc': + ignored |= { + # SRC_DIR/Doc/build/ + 'build', + # SRC_DIR/Doc/venv/ + 'venv', + } + + # check if we are at the root of the source code + elif 'Modules' in names: + ignored |= { + # SRC_DIR/build/ + 'build', + } + return ignored + + +def iter_builtin_types(): + for obj in __builtins__.values(): + if not isinstance(obj, type): + continue + cls = obj + if cls.__module__ != 'builtins': + continue + yield cls + + +def iter_slot_wrappers(cls): + assert cls.__module__ == 'builtins', cls + + def is_slot_wrapper(name, value): + if not isinstance(value, types.WrapperDescriptorType): + assert not repr(value).startswith(' dict[str, str]: + clean_env = os.environ.copy() + for k in clean_env.copy(): + if k.startswith("PYTHON"): + clean_env.pop(k) + clean_env.pop("FORCE_COLOR", None) + clean_env.pop("NO_COLOR", None) + return clean_env + + +def initialized_with_pyrepl(): + """Detect whether PyREPL was used during Python initialization.""" + # If the main module has a __file__ attribute it's a Python module, which means PyREPL. + return hasattr(sys.modules["__main__"], "__file__") + + class BrokenIter: def __init__(self, init_raises=False, next_raises=False, iter_raises=False): if init_raises: @@ -2641,3 +2847,224 @@ def __iter__(self): if self.iter_raises: 1/0 return self + + +def linked_to_musl(): + """ + Test if the Python executable is linked to the musl C library. + """ + if sys.platform != 'linux': + return False + + import subprocess + exe = getattr(sys, '_base_executable', sys.executable) + cmd = ['ldd', exe] + try: + stdout = subprocess.check_output(cmd, + text=True, + stderr=subprocess.STDOUT) + except (OSError, subprocess.CalledProcessError): + return False + return ('musl' in stdout) + + +# TODO: RUSTPYTHON +# Every line of code below allowed us to update `Lib/test/support/__init__.py` without +# needing to update `libregtest` and its dependencies. +# Ideally we want to remove all code below and update `libregtest`. +# +# Code below was copied from: https://github.com/RustPython/RustPython/blob/9499d39f55b73535e2405bf208d5380241f79ada/Lib/test/support/__init__.py + +from .testresult import get_test_runner + +def _filter_suite(suite, pred): + """Recursively filter test cases in a suite based on a predicate.""" + newtests = [] + for test in suite._tests: + if isinstance(test, unittest.TestSuite): + _filter_suite(test, pred) + newtests.append(test) + else: + if pred(test): + newtests.append(test) + suite._tests = newtests + +# By default, don't filter tests +_match_test_func = None + +_accept_test_patterns = None +_ignore_test_patterns = None + +def match_test(test): + # Function used by support.run_unittest() and regrtest --list-cases + if _match_test_func is None: + return True + else: + return _match_test_func(test.id()) + +def _is_full_match_test(pattern): + # If a pattern contains at least one dot, it's considered + # as a full test identifier. + # Example: 'test.test_os.FileTests.test_access'. + # + # ignore patterns which contain fnmatch patterns: '*', '?', '[...]' + # or '[!...]'. For example, ignore 'test_access*'. + return ('.' in pattern) and (not re.search(r'[?*\[\]]', pattern)) + +def set_match_tests(accept_patterns=None, ignore_patterns=None): + global _match_test_func, _accept_test_patterns, _ignore_test_patterns + + if accept_patterns is None: + accept_patterns = () + if ignore_patterns is None: + ignore_patterns = () + + accept_func = ignore_func = None + + if accept_patterns != _accept_test_patterns: + accept_patterns, accept_func = _compile_match_function(accept_patterns) + if ignore_patterns != _ignore_test_patterns: + ignore_patterns, ignore_func = _compile_match_function(ignore_patterns) + + # Create a copy since patterns can be mutable and so modified later + _accept_test_patterns = tuple(accept_patterns) + _ignore_test_patterns = tuple(ignore_patterns) + + if accept_func is not None or ignore_func is not None: + def match_function(test_id): + accept = True + ignore = False + if accept_func: + accept = accept_func(test_id) + if ignore_func: + ignore = ignore_func(test_id) + return accept and not ignore + + _match_test_func = match_function + +def _compile_match_function(patterns): + if not patterns: + func = None + # set_match_tests(None) behaves as set_match_tests(()) + patterns = () + elif all(map(_is_full_match_test, patterns)): + # Simple case: all patterns are full test identifier. + # The test.bisect_cmd utility only uses such full test identifiers. + func = set(patterns).__contains__ + else: + import fnmatch + regex = '|'.join(map(fnmatch.translate, patterns)) + # The search *is* case sensitive on purpose: + # don't use flags=re.IGNORECASE + regex_match = re.compile(regex).match + + def match_test_regex(test_id): + if regex_match(test_id): + # The regex matches the whole identifier, for example + # 'test.test_os.FileTests.test_access'. + return True + else: + # Try to match parts of the test identifier. + # For example, split 'test.test_os.FileTests.test_access' + # into: 'test', 'test_os', 'FileTests' and 'test_access'. + return any(map(regex_match, test_id.split("."))) + + func = match_test_regex + + return patterns, func + +def run_unittest(*classes): + """Run tests from unittest.TestCase-derived classes.""" + valid_types = (unittest.TestSuite, unittest.TestCase) + loader = unittest.TestLoader() + suite = unittest.TestSuite() + for cls in classes: + if isinstance(cls, str): + if cls in sys.modules: + suite.addTest(loader.loadTestsFromModule(sys.modules[cls])) + else: + raise ValueError("str arguments must be keys in sys.modules") + elif isinstance(cls, valid_types): + suite.addTest(cls) + else: + suite.addTest(loader.loadTestsFromTestCase(cls)) + _filter_suite(suite, match_test) + return _run_suite(suite) + +def _run_suite(suite): + """Run tests from a unittest.TestSuite-derived class.""" + runner = get_test_runner(sys.stdout, + verbosity=verbose, + capture_output=(junit_xml_list is not None)) + + result = runner.run(suite) + + if junit_xml_list is not None: + junit_xml_list.append(result.get_xml_element()) + + if not result.testsRun and not result.skipped and not result.errors: + raise TestDidNotRun + if not result.wasSuccessful(): + stats = TestStats.from_unittest(result) + if len(result.errors) == 1 and not result.failures: + err = result.errors[0][1] + elif len(result.failures) == 1 and not result.errors: + err = result.failures[0][1] + else: + err = "multiple errors occurred" + if not verbose: err += "; run in verbose mode for details" + errors = [(str(tc), exc_str) for tc, exc_str in result.errors] + failures = [(str(tc), exc_str) for tc, exc_str in result.failures] + raise TestFailedWithDetails(err, errors, failures, stats=stats) + return result + +@dataclasses.dataclass(slots=True) +class TestStats: + tests_run: int = 0 + failures: int = 0 + skipped: int = 0 + + @staticmethod + def from_unittest(result): + return TestStats(result.testsRun, + len(result.failures), + len(result.skipped)) + + @staticmethod + def from_doctest(results): + return TestStats(results.attempted, + results.failed) + + def accumulate(self, stats): + self.tests_run += stats.tests_run + self.failures += stats.failures + self.skipped += stats.skipped + + +def run_doctest(module, verbosity=None, optionflags=0): + """Run doctest on the given module. Return (#failures, #tests). + + If optional argument verbosity is not specified (or is None), pass + support's belief about verbosity on to doctest. Else doctest's + usual behavior is used (it searches sys.argv for -v). + """ + + import doctest + + if verbosity is None: + verbosity = verbose + else: + verbosity = None + + results = doctest.testmod(module, + verbose=verbosity, + optionflags=optionflags) + if results.failed: + stats = TestStats.from_doctest(results) + raise TestFailed(f"{results.failed} of {results.attempted} " + f"doctests failed", + stats=stats) + if verbose: + print('doctest (%s) ... %d tests with zero failures' % + (module.__name__, results.attempted)) + return results diff --git a/Lib/test/support/bytecode_helper.py b/Lib/test/support/bytecode_helper.py index 388d126677..85bcd1f0f1 100644 --- a/Lib/test/support/bytecode_helper.py +++ b/Lib/test/support/bytecode_helper.py @@ -3,10 +3,26 @@ import unittest import dis import io -from _testinternalcapi import compiler_codegen, optimize_cfg, assemble_code_object +import opcode +try: + import _testinternalcapi +except ImportError: + _testinternalcapi = None _UNSPECIFIED = object() +def instructions_with_positions(instrs, co_positions): + # Return (instr, positions) pairs from the instrs list and co_positions + # iterator. The latter contains items for cache lines and the former + # doesn't, so those need to be skipped. + + co_positions = co_positions or iter(()) + for instr in instrs: + yield instr, next(co_positions, ()) + for _, size, _ in (instr.cache_info or ()): + for i in range(size): + next(co_positions, ()) + class BytecodeTestCase(unittest.TestCase): """Custom assertion methods for inspecting bytecode.""" @@ -53,16 +69,14 @@ class CompilationStepTestCase(unittest.TestCase): class Label: pass - def assertInstructionsMatch(self, actual_, expected_): - # get two lists where each entry is a label or - # an instruction tuple. Normalize the labels to the - # instruction count of the target, and compare the lists. - - self.assertIsInstance(actual_, list) - self.assertIsInstance(expected_, list) + def assertInstructionsMatch(self, actual_seq, expected): + # get an InstructionSequence and an expected list, where each + # entry is a label or an instruction tuple. Construct an expcted + # instruction sequence and compare with the one given. - actual = self.normalize_insts(actual_) - expected = self.normalize_insts(expected_) + self.assertIsInstance(expected, list) + actual = actual_seq.get_instructions() + expected = self.seq_from_insts(expected).get_instructions() self.assertEqual(len(actual), len(expected)) # compare instructions @@ -72,10 +86,8 @@ def assertInstructionsMatch(self, actual_, expected_): continue self.assertIsInstance(exp, tuple) self.assertIsInstance(act, tuple) - # crop comparison to the provided expected values - if len(act) > len(exp): - act = act[:len(exp)] - self.assertEqual(exp, act) + idx = max([p[0] for p in enumerate(exp) if p[1] != -1]) + self.assertEqual(exp[:idx], act[:idx]) def resolveAndRemoveLabels(self, insts): idx = 0 @@ -90,54 +102,57 @@ def resolveAndRemoveLabels(self, insts): return res - def normalize_insts(self, insts): - """ Map labels to instruction index. - Map opcodes to opnames. - """ - insts = self.resolveAndRemoveLabels(insts) - res = [] - for item in insts: - assert isinstance(item, tuple) - opcode, oparg, *loc = item - opcode = dis.opmap.get(opcode, opcode) - if isinstance(oparg, self.Label): - arg = oparg.value - else: - arg = oparg if opcode in self.HAS_ARG else None - opcode = dis.opname[opcode] - res.append((opcode, arg, *loc)) - return res + def seq_from_insts(self, insts): + labels = {item for item in insts if isinstance(item, self.Label)} + for i, lbl in enumerate(labels): + lbl.value = i - def complete_insts_info(self, insts): - # fill in omitted fields in location, and oparg 0 for ops with no arg. - res = [] + seq = _testinternalcapi.new_instruction_sequence() for item in insts: - assert isinstance(item, tuple) - inst = list(item) - opcode = dis.opmap[inst[0]] - oparg = inst[1] - loc = inst[2:] + [-1] * (6 - len(inst)) - res.append((opcode, oparg, *loc)) - return res + if isinstance(item, self.Label): + seq.use_label(item.value) + else: + op = item[0] + if isinstance(op, str): + op = opcode.opmap[op] + arg, *loc = item[1:] + if isinstance(arg, self.Label): + arg = arg.value + loc = loc + [-1] * (4 - len(loc)) + seq.addop(op, arg or 0, *loc) + return seq + + def check_instructions(self, insts): + for inst in insts: + if isinstance(inst, self.Label): + continue + op, arg, *loc = inst + if isinstance(op, str): + op = opcode.opmap[op] + self.assertEqual(op in opcode.hasarg, + arg is not None, + f"{opcode.opname[op]=} {arg=}") + self.assertTrue(all(isinstance(l, int) for l in loc)) +@unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi") class CodegenTestCase(CompilationStepTestCase): def generate_code(self, ast): - insts, _ = compiler_codegen(ast, "my_file.py", 0) + insts, _ = _testinternalcapi.compiler_codegen(ast, "my_file.py", 0) return insts +@unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi") class CfgOptimizationTestCase(CompilationStepTestCase): - def get_optimized(self, insts, consts, nlocals=0): - insts = self.normalize_insts(insts) - insts = self.complete_insts_info(insts) - insts = optimize_cfg(insts, consts, nlocals) + def get_optimized(self, seq, consts, nlocals=0): + insts = _testinternalcapi.optimize_cfg(seq, consts, nlocals) return insts, consts +@unittest.skipIf(_testinternalcapi is None, "requires _testinternalcapi") class AssemblerTestCase(CompilationStepTestCase): def get_code_object(self, filename, insts, metadata): - co = assemble_code_object(filename, insts, metadata) + co = _testinternalcapi.assemble_code_object(filename, insts, metadata) return co diff --git a/Lib/test/support/hypothesis_helper.py b/Lib/test/support/hypothesis_helper.py index 40f58a2f59..a99a4963ff 100644 --- a/Lib/test/support/hypothesis_helper.py +++ b/Lib/test/support/hypothesis_helper.py @@ -7,7 +7,8 @@ else: # Regrtest changes to use a tempdir as the working directory, so we have # to tell Hypothesis to use the original in order to persist the database. - from .os_helper import SAVEDCWD + from test.support import has_socket_support + from test.support.os_helper import SAVEDCWD from hypothesis.configuration import set_hypothesis_home_dir set_hypothesis_home_dir(os.path.join(SAVEDCWD, ".hypothesis")) @@ -28,7 +29,14 @@ # of failing examples, and also use a pull-through cache to automatically # replay any failing examples discovered in CI. For details on how this # works, see https://hypothesis.readthedocs.io/en/latest/database.html - if "CI" not in os.environ: + # We only do that if a GITHUB_TOKEN env var is provided, see: + # https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/managing-your-personal-access-tokens + # And Python is built with socket support: + if ( + has_socket_support + and "CI" not in os.environ + and "GITHUB_TOKEN" in os.environ + ): from hypothesis.database import ( GitHubArtifactDatabase, MultiplexedDatabase, diff --git a/Lib/test/support/import_helper.py b/Lib/test/support/import_helper.py index 67f18e530e..2b91bdcf9c 100644 --- a/Lib/test/support/import_helper.py +++ b/Lib/test/support/import_helper.py @@ -8,7 +8,7 @@ import unittest import warnings -from .os_helper import unlink +from .os_helper import unlink, temp_dir @contextlib.contextmanager @@ -58,8 +58,8 @@ def make_legacy_pyc(source): :return: The file system path to the legacy pyc file. """ pyc_file = importlib.util.cache_from_source(source) - up_one = os.path.dirname(os.path.abspath(source)) - legacy_pyc = os.path.join(up_one, source + 'c') + assert source.endswith('.py') + legacy_pyc = source + 'c' shutil.move(pyc_file, legacy_pyc) return legacy_pyc @@ -114,7 +114,7 @@ def multi_interp_extensions_check(enabled=True): This only applies to modules that haven't been imported yet. It overrides the PyInterpreterConfig.check_multi_interp_extensions setting (see support.run_in_subinterp_with_config() and - _xxsubinterpreters.create()). + _interpreters.create()). Also see importlib.utils.allowing_all_extensions(). """ @@ -268,9 +268,44 @@ def modules_cleanup(oldmodules): sys.modules.update(oldmodules) +@contextlib.contextmanager +def isolated_modules(): + """ + Save modules on entry and cleanup on exit. + """ + (saved,) = modules_setup() + try: + yield + finally: + modules_cleanup(saved) + + def mock_register_at_fork(func): # bpo-30599: Mock os.register_at_fork() when importing the random module, # since this function doesn't allow to unregister callbacks and would leak # memory. from unittest import mock return mock.patch('os.register_at_fork', create=True)(func) + + +@contextlib.contextmanager +def ready_to_import(name=None, source=""): + from test.support import script_helper + + # 1. Sets up a temporary directory and removes it afterwards + # 2. Creates the module file + # 3. Temporarily clears the module from sys.modules (if any) + # 4. Reverts or removes the module when cleaning up + name = name or "spam" + with temp_dir() as tempdir: + path = script_helper.make_script(tempdir, name, source) + old_module = sys.modules.pop(name, None) + try: + sys.path.insert(0, tempdir) + yield name, path + sys.path.remove(tempdir) + finally: + if old_module is not None: + sys.modules[name] = old_module + else: + sys.modules.pop(name, None) diff --git a/Lib/test/support/interpreters/__init__.py b/Lib/test/support/interpreters/__init__.py new file mode 100644 index 0000000000..e067f25936 --- /dev/null +++ b/Lib/test/support/interpreters/__init__.py @@ -0,0 +1,258 @@ +"""Subinterpreters High Level Module.""" + +import threading +import weakref +import _interpreters + +# aliases: +from _interpreters import ( + InterpreterError, InterpreterNotFoundError, NotShareableError, + is_shareable, +) + + +__all__ = [ + 'get_current', 'get_main', 'create', 'list_all', 'is_shareable', + 'Interpreter', + 'InterpreterError', 'InterpreterNotFoundError', 'ExecutionFailed', + 'NotShareableError', + 'create_queue', 'Queue', 'QueueEmpty', 'QueueFull', +] + + +_queuemod = None + +def __getattr__(name): + if name in ('Queue', 'QueueEmpty', 'QueueFull', 'create_queue'): + global create_queue, Queue, QueueEmpty, QueueFull + ns = globals() + from .queues import ( + create as create_queue, + Queue, QueueEmpty, QueueFull, + ) + return ns[name] + else: + raise AttributeError(name) + + +_EXEC_FAILURE_STR = """ +{superstr} + +Uncaught in the interpreter: + +{formatted} +""".strip() + +class ExecutionFailed(InterpreterError): + """An unhandled exception happened during execution. + + This is raised from Interpreter.exec() and Interpreter.call(). + """ + + def __init__(self, excinfo): + msg = excinfo.formatted + if not msg: + if excinfo.type and excinfo.msg: + msg = f'{excinfo.type.__name__}: {excinfo.msg}' + else: + msg = excinfo.type.__name__ or excinfo.msg + super().__init__(msg) + self.excinfo = excinfo + + def __str__(self): + try: + formatted = self.excinfo.errdisplay + except Exception: + return super().__str__() + else: + return _EXEC_FAILURE_STR.format( + superstr=super().__str__(), + formatted=formatted, + ) + + +def create(): + """Return a new (idle) Python interpreter.""" + id = _interpreters.create(reqrefs=True) + return Interpreter(id, _ownsref=True) + + +def list_all(): + """Return all existing interpreters.""" + return [Interpreter(id, _whence=whence) + for id, whence in _interpreters.list_all(require_ready=True)] + + +def get_current(): + """Return the currently running interpreter.""" + id, whence = _interpreters.get_current() + return Interpreter(id, _whence=whence) + + +def get_main(): + """Return the main interpreter.""" + id, whence = _interpreters.get_main() + assert whence == _interpreters.WHENCE_RUNTIME, repr(whence) + return Interpreter(id, _whence=whence) + + +_known = weakref.WeakValueDictionary() + +class Interpreter: + """A single Python interpreter. + + Attributes: + + "id" - the unique process-global ID number for the interpreter + "whence" - indicates where the interpreter was created + + If the interpreter wasn't created by this module + then any method that modifies the interpreter will fail, + i.e. .close(), .prepare_main(), .exec(), and .call() + """ + + _WHENCE_TO_STR = { + _interpreters.WHENCE_UNKNOWN: 'unknown', + _interpreters.WHENCE_RUNTIME: 'runtime init', + _interpreters.WHENCE_LEGACY_CAPI: 'legacy C-API', + _interpreters.WHENCE_CAPI: 'C-API', + _interpreters.WHENCE_XI: 'cross-interpreter C-API', + _interpreters.WHENCE_STDLIB: '_interpreters module', + } + + def __new__(cls, id, /, _whence=None, _ownsref=None): + # There is only one instance for any given ID. + if not isinstance(id, int): + raise TypeError(f'id must be an int, got {id!r}') + id = int(id) + if _whence is None: + if _ownsref: + _whence = _interpreters.WHENCE_STDLIB + else: + _whence = _interpreters.whence(id) + assert _whence in cls._WHENCE_TO_STR, repr(_whence) + if _ownsref is None: + _ownsref = (_whence == _interpreters.WHENCE_STDLIB) + try: + self = _known[id] + assert hasattr(self, '_ownsref') + except KeyError: + self = super().__new__(cls) + _known[id] = self + self._id = id + self._whence = _whence + self._ownsref = _ownsref + if _ownsref: + # This may raise InterpreterNotFoundError: + _interpreters.incref(id) + return self + + def __repr__(self): + return f'{type(self).__name__}({self.id})' + + def __hash__(self): + return hash(self._id) + + def __del__(self): + self._decref() + + # for pickling: + def __getnewargs__(self): + return (self._id,) + + # for pickling: + def __getstate__(self): + return None + + def _decref(self): + if not self._ownsref: + return + self._ownsref = False + try: + _interpreters.decref(self._id) + except InterpreterNotFoundError: + pass + + @property + def id(self): + return self._id + + @property + def whence(self): + return self._WHENCE_TO_STR[self._whence] + + def is_running(self): + """Return whether or not the identified interpreter is running.""" + return _interpreters.is_running(self._id) + + # Everything past here is available only to interpreters created by + # interpreters.create(). + + def close(self): + """Finalize and destroy the interpreter. + + Attempting to destroy the current interpreter results + in an InterpreterError. + """ + return _interpreters.destroy(self._id, restrict=True) + + def prepare_main(self, ns=None, /, **kwargs): + """Bind the given values into the interpreter's __main__. + + The values must be shareable. + """ + ns = dict(ns, **kwargs) if ns is not None else kwargs + _interpreters.set___main___attrs(self._id, ns, restrict=True) + + def exec(self, code, /): + """Run the given source code in the interpreter. + + This is essentially the same as calling the builtin "exec" + with this interpreter, using the __dict__ of its __main__ + module as both globals and locals. + + There is no return value. + + If the code raises an unhandled exception then an ExecutionFailed + exception is raised, which summarizes the unhandled exception. + The actual exception is discarded because objects cannot be + shared between interpreters. + + This blocks the current Python thread until done. During + that time, the previous interpreter is allowed to run + in other threads. + """ + excinfo = _interpreters.exec(self._id, code, restrict=True) + if excinfo is not None: + raise ExecutionFailed(excinfo) + + def call(self, callable, /): + """Call the object in the interpreter with given args/kwargs. + + Only functions that take no arguments and have no closure + are supported. + + The return value is discarded. + + If the callable raises an exception then the error display + (including full traceback) is send back between the interpreters + and an ExecutionFailed exception is raised, much like what + happens with Interpreter.exec(). + """ + # XXX Support args and kwargs. + # XXX Support arbitrary callables. + # XXX Support returning the return value (e.g. via pickle). + excinfo = _interpreters.call(self._id, callable, restrict=True) + if excinfo is not None: + raise ExecutionFailed(excinfo) + + def call_in_thread(self, callable, /): + """Return a new thread that calls the object in the interpreter. + + The return value and any raised exception are discarded. + """ + def task(): + self.call(callable) + t = threading.Thread(target=task) + t.start() + return t diff --git a/Lib/test/support/interpreters/_crossinterp.py b/Lib/test/support/interpreters/_crossinterp.py new file mode 100644 index 0000000000..544e197ba4 --- /dev/null +++ b/Lib/test/support/interpreters/_crossinterp.py @@ -0,0 +1,102 @@ +"""Common code between queues and channels.""" + + +class ItemInterpreterDestroyed(Exception): + """Raised when trying to get an item whose interpreter was destroyed.""" + + +class classonly: + """A non-data descriptor that makes a value only visible on the class. + + This is like the "classmethod" builtin, but does not show up on + instances of the class. It may be used as a decorator. + """ + + def __init__(self, value): + self.value = value + self.getter = classmethod(value).__get__ + self.name = None + + def __set_name__(self, cls, name): + if self.name is not None: + raise TypeError('already used') + self.name = name + + def __get__(self, obj, cls): + if obj is not None: + raise AttributeError(self.name) + # called on the class + return self.getter(None, cls) + + +class UnboundItem: + """Represents a cross-interpreter item no longer bound to an interpreter. + + An item is unbound when the interpreter that added it to the + cross-interpreter container is destroyed. + """ + + __slots__ = () + + @classonly + def singleton(cls, kind, module, name='UNBOUND'): + doc = cls.__doc__.replace('cross-interpreter container', kind) + doc = doc.replace('cross-interpreter', kind) + subclass = type( + f'Unbound{kind.capitalize()}Item', + (cls,), + dict( + _MODULE=module, + _NAME=name, + __doc__=doc, + ), + ) + return object.__new__(subclass) + + _MODULE = __name__ + _NAME = 'UNBOUND' + + def __new__(cls): + raise Exception(f'use {cls._MODULE}.{cls._NAME}') + + def __repr__(self): + return f'{self._MODULE}.{self._NAME}' +# return f'interpreters.queues.UNBOUND' + + +UNBOUND = object.__new__(UnboundItem) +UNBOUND_ERROR = object() +UNBOUND_REMOVE = object() + +_UNBOUND_CONSTANT_TO_FLAG = { + UNBOUND_REMOVE: 1, + UNBOUND_ERROR: 2, + UNBOUND: 3, +} +_UNBOUND_FLAG_TO_CONSTANT = {v: k + for k, v in _UNBOUND_CONSTANT_TO_FLAG.items()} + + +def serialize_unbound(unbound): + op = unbound + try: + flag = _UNBOUND_CONSTANT_TO_FLAG[op] + except KeyError: + raise NotImplementedError(f'unsupported unbound replacement op {op!r}') + return flag, + + +def resolve_unbound(flag, exctype_destroyed): + try: + op = _UNBOUND_FLAG_TO_CONSTANT[flag] + except KeyError: + raise NotImplementedError(f'unsupported unbound replacement op {flag!r}') + if op is UNBOUND_REMOVE: + # "remove" not possible here + raise NotImplementedError + elif op is UNBOUND_ERROR: + raise exctype_destroyed("item's original interpreter destroyed") + elif op is UNBOUND: + return UNBOUND + else: + raise NotImplementedError(repr(op)) diff --git a/Lib/test/support/interpreters/channels.py b/Lib/test/support/interpreters/channels.py new file mode 100644 index 0000000000..d2bd93d77f --- /dev/null +++ b/Lib/test/support/interpreters/channels.py @@ -0,0 +1,257 @@ +"""Cross-interpreter Channels High Level Module.""" + +import time +import _interpchannels as _channels +from . import _crossinterp + +# aliases: +from _interpchannels import ( + ChannelError, ChannelNotFoundError, ChannelClosedError, + ChannelEmptyError, ChannelNotEmptyError, +) +from ._crossinterp import ( + UNBOUND_ERROR, UNBOUND_REMOVE, +) + + +__all__ = [ + 'UNBOUND', 'UNBOUND_ERROR', 'UNBOUND_REMOVE', + 'create', 'list_all', + 'SendChannel', 'RecvChannel', + 'ChannelError', 'ChannelNotFoundError', 'ChannelEmptyError', + 'ItemInterpreterDestroyed', +] + + +class ItemInterpreterDestroyed(ChannelError, + _crossinterp.ItemInterpreterDestroyed): + """Raised from get() and get_nowait().""" + + +UNBOUND = _crossinterp.UnboundItem.singleton('queue', __name__) + + +def _serialize_unbound(unbound): + if unbound is UNBOUND: + unbound = _crossinterp.UNBOUND + return _crossinterp.serialize_unbound(unbound) + + +def _resolve_unbound(flag): + resolved = _crossinterp.resolve_unbound(flag, ItemInterpreterDestroyed) + if resolved is _crossinterp.UNBOUND: + resolved = UNBOUND + return resolved + + +def create(*, unbounditems=UNBOUND): + """Return (recv, send) for a new cross-interpreter channel. + + The channel may be used to pass data safely between interpreters. + + "unbounditems" sets the default for the send end of the channel. + See SendChannel.send() for supported values. The default value + is UNBOUND, which replaces the unbound item when received. + """ + unbound = _serialize_unbound(unbounditems) + unboundop, = unbound + cid = _channels.create(unboundop) + recv, send = RecvChannel(cid), SendChannel(cid, _unbound=unbound) + return recv, send + + +def list_all(): + """Return a list of (recv, send) for all open channels.""" + return [(RecvChannel(cid), SendChannel(cid, _unbound=unbound)) + for cid, unbound in _channels.list_all()] + + +class _ChannelEnd: + """The base class for RecvChannel and SendChannel.""" + + _end = None + + def __new__(cls, cid): + self = super().__new__(cls) + if self._end == 'send': + cid = _channels._channel_id(cid, send=True, force=True) + elif self._end == 'recv': + cid = _channels._channel_id(cid, recv=True, force=True) + else: + raise NotImplementedError(self._end) + self._id = cid + return self + + def __repr__(self): + return f'{type(self).__name__}(id={int(self._id)})' + + def __hash__(self): + return hash(self._id) + + def __eq__(self, other): + if isinstance(self, RecvChannel): + if not isinstance(other, RecvChannel): + return NotImplemented + elif not isinstance(other, SendChannel): + return NotImplemented + return other._id == self._id + + # for pickling: + def __getnewargs__(self): + return (int(self._id),) + + # for pickling: + def __getstate__(self): + return None + + @property + def id(self): + return self._id + + @property + def _info(self): + return _channels.get_info(self._id) + + @property + def is_closed(self): + return self._info.closed + + +_NOT_SET = object() + + +class RecvChannel(_ChannelEnd): + """The receiving end of a cross-interpreter channel.""" + + _end = 'recv' + + def recv(self, timeout=None, *, + _sentinel=object(), + _delay=10 / 1000, # 10 milliseconds + ): + """Return the next object from the channel. + + This blocks until an object has been sent, if none have been + sent already. + """ + if timeout is not None: + timeout = int(timeout) + if timeout < 0: + raise ValueError(f'timeout value must be non-negative') + end = time.time() + timeout + obj, unboundop = _channels.recv(self._id, _sentinel) + while obj is _sentinel: + time.sleep(_delay) + if timeout is not None and time.time() >= end: + raise TimeoutError + obj, unboundop = _channels.recv(self._id, _sentinel) + if unboundop is not None: + assert obj is None, repr(obj) + return _resolve_unbound(unboundop) + return obj + + def recv_nowait(self, default=_NOT_SET): + """Return the next object from the channel. + + If none have been sent then return the default if one + is provided or fail with ChannelEmptyError. Otherwise this + is the same as recv(). + """ + if default is _NOT_SET: + obj, unboundop = _channels.recv(self._id) + else: + obj, unboundop = _channels.recv(self._id, default) + if unboundop is not None: + assert obj is None, repr(obj) + return _resolve_unbound(unboundop) + return obj + + def close(self): + _channels.close(self._id, recv=True) + + +class SendChannel(_ChannelEnd): + """The sending end of a cross-interpreter channel.""" + + _end = 'send' + + def __new__(cls, cid, *, _unbound=None): + if _unbound is None: + try: + op = _channels.get_channel_defaults(cid) + _unbound = (op,) + except ChannelNotFoundError: + _unbound = _serialize_unbound(UNBOUND) + self = super().__new__(cls, cid) + self._unbound = _unbound + return self + + @property + def is_closed(self): + info = self._info + return info.closed or info.closing + + def send(self, obj, timeout=None, *, + unbound=None, + ): + """Send the object (i.e. its data) to the channel's receiving end. + + This blocks until the object is received. + """ + if unbound is None: + unboundop, = self._unbound + else: + unboundop, = _serialize_unbound(unbound) + _channels.send(self._id, obj, unboundop, timeout=timeout, blocking=True) + + def send_nowait(self, obj, *, + unbound=None, + ): + """Send the object to the channel's receiving end. + + If the object is immediately received then return True + (else False). Otherwise this is the same as send(). + """ + if unbound is None: + unboundop, = self._unbound + else: + unboundop, = _serialize_unbound(unbound) + # XXX Note that at the moment channel_send() only ever returns + # None. This should be fixed when channel_send_wait() is added. + # See bpo-32604 and gh-19829. + return _channels.send(self._id, obj, unboundop, blocking=False) + + def send_buffer(self, obj, timeout=None, *, + unbound=None, + ): + """Send the object's buffer to the channel's receiving end. + + This blocks until the object is received. + """ + if unbound is None: + unboundop, = self._unbound + else: + unboundop, = _serialize_unbound(unbound) + _channels.send_buffer(self._id, obj, unboundop, + timeout=timeout, blocking=True) + + def send_buffer_nowait(self, obj, *, + unbound=None, + ): + """Send the object's buffer to the channel's receiving end. + + If the object is immediately received then return True + (else False). Otherwise this is the same as send(). + """ + if unbound is None: + unboundop, = self._unbound + else: + unboundop, = _serialize_unbound(unbound) + return _channels.send_buffer(self._id, obj, unboundop, blocking=False) + + def close(self): + _channels.close(self._id, send=True) + + +# XXX This is causing leaks (gh-110318): +_channels._register_end_types(SendChannel, RecvChannel) diff --git a/Lib/test/support/interpreters/queues.py b/Lib/test/support/interpreters/queues.py new file mode 100644 index 0000000000..deb8e8613a --- /dev/null +++ b/Lib/test/support/interpreters/queues.py @@ -0,0 +1,313 @@ +"""Cross-interpreter Queues High Level Module.""" + +import pickle +import queue +import time +import weakref +import _interpqueues as _queues +from . import _crossinterp + +# aliases: +from _interpqueues import ( + QueueError, QueueNotFoundError, +) +from ._crossinterp import ( + UNBOUND_ERROR, UNBOUND_REMOVE, +) + +__all__ = [ + 'UNBOUND', 'UNBOUND_ERROR', 'UNBOUND_REMOVE', + 'create', 'list_all', + 'Queue', + 'QueueError', 'QueueNotFoundError', 'QueueEmpty', 'QueueFull', + 'ItemInterpreterDestroyed', +] + + +class QueueEmpty(QueueError, queue.Empty): + """Raised from get_nowait() when the queue is empty. + + It is also raised from get() if it times out. + """ + + +class QueueFull(QueueError, queue.Full): + """Raised from put_nowait() when the queue is full. + + It is also raised from put() if it times out. + """ + + +class ItemInterpreterDestroyed(QueueError, + _crossinterp.ItemInterpreterDestroyed): + """Raised from get() and get_nowait().""" + + +_SHARED_ONLY = 0 +_PICKLED = 1 + + +UNBOUND = _crossinterp.UnboundItem.singleton('queue', __name__) + + +def _serialize_unbound(unbound): + if unbound is UNBOUND: + unbound = _crossinterp.UNBOUND + return _crossinterp.serialize_unbound(unbound) + + +def _resolve_unbound(flag): + resolved = _crossinterp.resolve_unbound(flag, ItemInterpreterDestroyed) + if resolved is _crossinterp.UNBOUND: + resolved = UNBOUND + return resolved + + +def create(maxsize=0, *, syncobj=False, unbounditems=UNBOUND): + """Return a new cross-interpreter queue. + + The queue may be used to pass data safely between interpreters. + + "syncobj" sets the default for Queue.put() + and Queue.put_nowait(). + + "unbounditems" likewise sets the default. See Queue.put() for + supported values. The default value is UNBOUND, which replaces + the unbound item. + """ + fmt = _SHARED_ONLY if syncobj else _PICKLED + unbound = _serialize_unbound(unbounditems) + unboundop, = unbound + qid = _queues.create(maxsize, fmt, unboundop) + return Queue(qid, _fmt=fmt, _unbound=unbound) + + +def list_all(): + """Return a list of all open queues.""" + return [Queue(qid, _fmt=fmt, _unbound=(unboundop,)) + for qid, fmt, unboundop in _queues.list_all()] + + +_known_queues = weakref.WeakValueDictionary() + +class Queue: + """A cross-interpreter queue.""" + + def __new__(cls, id, /, *, _fmt=None, _unbound=None): + # There is only one instance for any given ID. + if isinstance(id, int): + id = int(id) + else: + raise TypeError(f'id must be an int, got {id!r}') + if _fmt is None: + if _unbound is None: + _fmt, op = _queues.get_queue_defaults(id) + _unbound = (op,) + else: + _fmt, _ = _queues.get_queue_defaults(id) + elif _unbound is None: + _, op = _queues.get_queue_defaults(id) + _unbound = (op,) + try: + self = _known_queues[id] + except KeyError: + self = super().__new__(cls) + self._id = id + self._fmt = _fmt + self._unbound = _unbound + _known_queues[id] = self + _queues.bind(id) + return self + + def __del__(self): + try: + _queues.release(self._id) + except QueueNotFoundError: + pass + try: + del _known_queues[self._id] + except KeyError: + pass + + def __repr__(self): + return f'{type(self).__name__}({self.id})' + + def __hash__(self): + return hash(self._id) + + # for pickling: + def __getnewargs__(self): + return (self._id,) + + # for pickling: + def __getstate__(self): + return None + + @property + def id(self): + return self._id + + @property + def maxsize(self): + try: + return self._maxsize + except AttributeError: + self._maxsize = _queues.get_maxsize(self._id) + return self._maxsize + + def empty(self): + return self.qsize() == 0 + + def full(self): + return _queues.is_full(self._id) + + def qsize(self): + return _queues.get_count(self._id) + + def put(self, obj, timeout=None, *, + syncobj=None, + unbound=None, + _delay=10 / 1000, # 10 milliseconds + ): + """Add the object to the queue. + + This blocks while the queue is full. + + If "syncobj" is None (the default) then it uses the + queue's default, set with create_queue(). + + If "syncobj" is false then all objects are supported, + at the expense of worse performance. + + If "syncobj" is true then the object must be "shareable". + Examples of "shareable" objects include the builtin singletons, + str, and memoryview. One benefit is that such objects are + passed through the queue efficiently. + + The key difference, though, is conceptual: the corresponding + object returned from Queue.get() will be strictly equivalent + to the given obj. In other words, the two objects will be + effectively indistinguishable from each other, even if the + object is mutable. The received object may actually be the + same object, or a copy (immutable values only), or a proxy. + Regardless, the received object should be treated as though + the original has been shared directly, whether or not it + actually is. That's a slightly different and stronger promise + than just (initial) equality, which is all "syncobj=False" + can promise. + + "unbound" controls the behavior of Queue.get() for the given + object if the current interpreter (calling put()) is later + destroyed. + + If "unbound" is None (the default) then it uses the + queue's default, set with create_queue(), + which is usually UNBOUND. + + If "unbound" is UNBOUND_ERROR then get() will raise an + ItemInterpreterDestroyed exception if the original interpreter + has been destroyed. This does not otherwise affect the queue; + the next call to put() will work like normal, returning the next + item in the queue. + + If "unbound" is UNBOUND_REMOVE then the item will be removed + from the queue as soon as the original interpreter is destroyed. + Be aware that this will introduce an imbalance between put() + and get() calls. + + If "unbound" is UNBOUND then it is returned by get() in place + of the unbound item. + """ + if syncobj is None: + fmt = self._fmt + else: + fmt = _SHARED_ONLY if syncobj else _PICKLED + if unbound is None: + unboundop, = self._unbound + else: + unboundop, = _serialize_unbound(unbound) + if timeout is not None: + timeout = int(timeout) + if timeout < 0: + raise ValueError(f'timeout value must be non-negative') + end = time.time() + timeout + if fmt is _PICKLED: + obj = pickle.dumps(obj) + while True: + try: + _queues.put(self._id, obj, fmt, unboundop) + except QueueFull as exc: + if timeout is not None and time.time() >= end: + raise # re-raise + time.sleep(_delay) + else: + break + + def put_nowait(self, obj, *, syncobj=None, unbound=None): + if syncobj is None: + fmt = self._fmt + else: + fmt = _SHARED_ONLY if syncobj else _PICKLED + if unbound is None: + unboundop, = self._unbound + else: + unboundop, = _serialize_unbound(unbound) + if fmt is _PICKLED: + obj = pickle.dumps(obj) + _queues.put(self._id, obj, fmt, unboundop) + + def get(self, timeout=None, *, + _delay=10 / 1000, # 10 milliseconds + ): + """Return the next object from the queue. + + This blocks while the queue is empty. + + If the next item's original interpreter has been destroyed + then the "next object" is determined by the value of the + "unbound" argument to put(). + """ + if timeout is not None: + timeout = int(timeout) + if timeout < 0: + raise ValueError(f'timeout value must be non-negative') + end = time.time() + timeout + while True: + try: + obj, fmt, unboundop = _queues.get(self._id) + except QueueEmpty as exc: + if timeout is not None and time.time() >= end: + raise # re-raise + time.sleep(_delay) + else: + break + if unboundop is not None: + assert obj is None, repr(obj) + return _resolve_unbound(unboundop) + if fmt == _PICKLED: + obj = pickle.loads(obj) + else: + assert fmt == _SHARED_ONLY + return obj + + def get_nowait(self): + """Return the next object from the channel. + + If the queue is empty then raise QueueEmpty. Otherwise this + is the same as get(). + """ + try: + obj, fmt, unboundop = _queues.get(self._id) + except QueueEmpty as exc: + raise # re-raise + if unboundop is not None: + assert obj is None, repr(obj) + return _resolve_unbound(unboundop) + if fmt == _PICKLED: + obj = pickle.loads(obj) + else: + assert fmt == _SHARED_ONLY + return obj + + +_queues._register_heap_types(Queue, QueueEmpty, QueueFull) diff --git a/Lib/test/support/numbers.py b/Lib/test/support/numbers.py new file mode 100644 index 0000000000..d5dbb41ace --- /dev/null +++ b/Lib/test/support/numbers.py @@ -0,0 +1,80 @@ +# These are shared with test_tokenize and other test modules. +# +# Note: since several test cases filter out floats by looking for "e" and ".", +# don't add hexadecimal literals that contain "e" or "E". +VALID_UNDERSCORE_LITERALS = [ + '0_0_0', + '4_2', + '1_0000_0000', + '0b1001_0100', + '0xffff_ffff', + '0o5_7_7', + '1_00_00.5', + '1_00_00.5e5', + '1_00_00e5_1', + '1e1_0', + '.1_4', + '.1_4e1', + '0b_0', + '0x_f', + '0o_5', + '1_00_00j', + '1_00_00.5j', + '1_00_00e5_1j', + '.1_4j', + '(1_2.5+3_3j)', + '(.5_6j)', +] +INVALID_UNDERSCORE_LITERALS = [ + # Trailing underscores: + '0_', + '42_', + '1.4j_', + '0x_', + '0b1_', + '0xf_', + '0o5_', + '0 if 1_Else 1', + # Underscores in the base selector: + '0_b0', + '0_xf', + '0_o5', + # Old-style octal, still disallowed: + '0_7', + '09_99', + # Multiple consecutive underscores: + '4_______2', + '0.1__4', + '0.1__4j', + '0b1001__0100', + '0xffff__ffff', + '0x___', + '0o5__77', + '1e1__0', + '1e1__0j', + # Underscore right before a dot: + '1_.4', + '1_.4j', + # Underscore right after a dot: + '1._4', + '1._4j', + '._5', + '._5j', + # Underscore right after a sign: + '1.0e+_1', + '1.0e+_1j', + # Underscore right before j: + '1.4_j', + '1.4e5_j', + # Underscore right before e: + '1_e1', + '1.4_e1', + '1.4_e1j', + # Underscore right after e: + '1e_1', + '1.4e_1', + '1.4e_1j', + # Complex cases with parens: + '(1+1.5_j_)', + '(1+1.5_j)', +] diff --git a/Lib/test/support/smtpd.py b/Lib/test/support/smtpd.py index ec4e7d2f4c..6052232ec2 100644 --- a/Lib/test/support/smtpd.py +++ b/Lib/test/support/smtpd.py @@ -180,122 +180,122 @@ def _set_rset_state(self): @property def __server(self): warn("Access to __server attribute on SMTPChannel is deprecated, " - "use 'smtp_server' instead", DeprecationWarning, 2) + "use 'smtp_server' instead", DeprecationWarning, 2) return self.smtp_server @__server.setter def __server(self, value): warn("Setting __server attribute on SMTPChannel is deprecated, " - "set 'smtp_server' instead", DeprecationWarning, 2) + "set 'smtp_server' instead", DeprecationWarning, 2) self.smtp_server = value @property def __line(self): warn("Access to __line attribute on SMTPChannel is deprecated, " - "use 'received_lines' instead", DeprecationWarning, 2) + "use 'received_lines' instead", DeprecationWarning, 2) return self.received_lines @__line.setter def __line(self, value): warn("Setting __line attribute on SMTPChannel is deprecated, " - "set 'received_lines' instead", DeprecationWarning, 2) + "set 'received_lines' instead", DeprecationWarning, 2) self.received_lines = value @property def __state(self): warn("Access to __state attribute on SMTPChannel is deprecated, " - "use 'smtp_state' instead", DeprecationWarning, 2) + "use 'smtp_state' instead", DeprecationWarning, 2) return self.smtp_state @__state.setter def __state(self, value): warn("Setting __state attribute on SMTPChannel is deprecated, " - "set 'smtp_state' instead", DeprecationWarning, 2) + "set 'smtp_state' instead", DeprecationWarning, 2) self.smtp_state = value @property def __greeting(self): warn("Access to __greeting attribute on SMTPChannel is deprecated, " - "use 'seen_greeting' instead", DeprecationWarning, 2) + "use 'seen_greeting' instead", DeprecationWarning, 2) return self.seen_greeting @__greeting.setter def __greeting(self, value): warn("Setting __greeting attribute on SMTPChannel is deprecated, " - "set 'seen_greeting' instead", DeprecationWarning, 2) + "set 'seen_greeting' instead", DeprecationWarning, 2) self.seen_greeting = value @property def __mailfrom(self): warn("Access to __mailfrom attribute on SMTPChannel is deprecated, " - "use 'mailfrom' instead", DeprecationWarning, 2) + "use 'mailfrom' instead", DeprecationWarning, 2) return self.mailfrom @__mailfrom.setter def __mailfrom(self, value): warn("Setting __mailfrom attribute on SMTPChannel is deprecated, " - "set 'mailfrom' instead", DeprecationWarning, 2) + "set 'mailfrom' instead", DeprecationWarning, 2) self.mailfrom = value @property def __rcpttos(self): warn("Access to __rcpttos attribute on SMTPChannel is deprecated, " - "use 'rcpttos' instead", DeprecationWarning, 2) + "use 'rcpttos' instead", DeprecationWarning, 2) return self.rcpttos @__rcpttos.setter def __rcpttos(self, value): warn("Setting __rcpttos attribute on SMTPChannel is deprecated, " - "set 'rcpttos' instead", DeprecationWarning, 2) + "set 'rcpttos' instead", DeprecationWarning, 2) self.rcpttos = value @property def __data(self): warn("Access to __data attribute on SMTPChannel is deprecated, " - "use 'received_data' instead", DeprecationWarning, 2) + "use 'received_data' instead", DeprecationWarning, 2) return self.received_data @__data.setter def __data(self, value): warn("Setting __data attribute on SMTPChannel is deprecated, " - "set 'received_data' instead", DeprecationWarning, 2) + "set 'received_data' instead", DeprecationWarning, 2) self.received_data = value @property def __fqdn(self): warn("Access to __fqdn attribute on SMTPChannel is deprecated, " - "use 'fqdn' instead", DeprecationWarning, 2) + "use 'fqdn' instead", DeprecationWarning, 2) return self.fqdn @__fqdn.setter def __fqdn(self, value): warn("Setting __fqdn attribute on SMTPChannel is deprecated, " - "set 'fqdn' instead", DeprecationWarning, 2) + "set 'fqdn' instead", DeprecationWarning, 2) self.fqdn = value @property def __peer(self): warn("Access to __peer attribute on SMTPChannel is deprecated, " - "use 'peer' instead", DeprecationWarning, 2) + "use 'peer' instead", DeprecationWarning, 2) return self.peer @__peer.setter def __peer(self, value): warn("Setting __peer attribute on SMTPChannel is deprecated, " - "set 'peer' instead", DeprecationWarning, 2) + "set 'peer' instead", DeprecationWarning, 2) self.peer = value @property def __conn(self): warn("Access to __conn attribute on SMTPChannel is deprecated, " - "use 'conn' instead", DeprecationWarning, 2) + "use 'conn' instead", DeprecationWarning, 2) return self.conn @__conn.setter def __conn(self, value): warn("Setting __conn attribute on SMTPChannel is deprecated, " - "set 'conn' instead", DeprecationWarning, 2) + "set 'conn' instead", DeprecationWarning, 2) self.conn = value @property def __addr(self): warn("Access to __addr attribute on SMTPChannel is deprecated, " - "use 'addr' instead", DeprecationWarning, 2) + "use 'addr' instead", DeprecationWarning, 2) return self.addr @__addr.setter def __addr(self, value): warn("Setting __addr attribute on SMTPChannel is deprecated, " - "set 'addr' instead", DeprecationWarning, 2) + "set 'addr' instead", DeprecationWarning, 2) self.addr = value # Overrides base class for convenience. @@ -339,7 +339,7 @@ def found_terminator(self): command = line[:i].upper() arg = line[i+1:].strip() max_sz = (self.command_size_limits[command] - if self.extended_smtp else self.command_size_limit) + if self.extended_smtp else self.command_size_limit) if sz > max_sz: self.push('500 Error: line too long') return diff --git a/Lib/test/support/venv.py b/Lib/test/support/venv.py new file mode 100644 index 0000000000..78e6a51ec1 --- /dev/null +++ b/Lib/test/support/venv.py @@ -0,0 +1,70 @@ +import contextlib +import logging +import os +import subprocess +import shlex +import sys +import sysconfig +import tempfile +import venv + + +class VirtualEnvironment: + def __init__(self, prefix, **venv_create_args): + self._logger = logging.getLogger(self.__class__.__name__) + venv.create(prefix, **venv_create_args) + self._prefix = prefix + self._paths = sysconfig.get_paths( + scheme='venv', + vars={'base': self.prefix}, + expand=True, + ) + + @classmethod + @contextlib.contextmanager + def from_tmpdir(cls, *, prefix=None, dir=None, **venv_create_args): + delete = not bool(os.environ.get('PYTHON_TESTS_KEEP_VENV')) + with tempfile.TemporaryDirectory(prefix=prefix, dir=dir, delete=delete) as tmpdir: + yield cls(tmpdir, **venv_create_args) + + @property + def prefix(self): + return self._prefix + + @property + def paths(self): + return self._paths + + @property + def interpreter(self): + return os.path.join(self.paths['scripts'], os.path.basename(sys.executable)) + + def _format_output(self, name, data, indent='\t'): + if not data: + return indent + f'{name}: (none)' + if len(data.splitlines()) == 1: + return indent + f'{name}: {data}' + else: + prefixed_lines = '\n'.join(indent + '> ' + line for line in data.splitlines()) + return indent + f'{name}:\n' + prefixed_lines + + def run(self, *args, **subprocess_args): + if subprocess_args.get('shell'): + raise ValueError('Running the subprocess in shell mode is not supported.') + default_args = { + 'capture_output': True, + 'check': True, + } + try: + result = subprocess.run([self.interpreter, *args], **default_args | subprocess_args) + except subprocess.CalledProcessError as e: + if e.returncode != 0: + self._logger.error( + f'Interpreter returned non-zero exit status {e.returncode}.\n' + + self._format_output('COMMAND', shlex.join(e.cmd)) + '\n' + + self._format_output('STDOUT', e.stdout.decode()) + '\n' + + self._format_output('STDERR', e.stderr.decode()) + '\n' + ) + raise + else: + return result diff --git a/Lib/test/test__colorize.py b/Lib/test/test__colorize.py index 056a5306ce..b2f0bb1386 100644 --- a/Lib/test/test__colorize.py +++ b/Lib/test/test__colorize.py @@ -10,8 +10,7 @@ @contextlib.contextmanager def clear_env(): with EnvironmentVarGuard() as mock_env: - for var in "FORCE_COLOR", "NO_COLOR", "PYTHON_COLORS": - mock_env.unset(var) + mock_env.unset("FORCE_COLOR", "NO_COLOR", "PYTHON_COLORS", "TERM") yield mock_env diff --git a/Lib/test/test__opcode.py b/Lib/test/test__opcode.py new file mode 100644 index 0000000000..dd4f30ab17 --- /dev/null +++ b/Lib/test/test__opcode.py @@ -0,0 +1,141 @@ +import dis +from test.support.import_helper import import_module +import unittest +import opcode + +_opcode = import_module("_opcode") +from _opcode import stack_effect + + +class OpListTests(unittest.TestCase): + def check_bool_function_result(self, func, ops, expected): + for op in ops: + if isinstance(op, str): + op = dis.opmap[op] + with self.subTest(opcode=op, func=func): + self.assertIsInstance(func(op), bool) + self.assertEqual(func(op), expected) + + def test_invalid_opcodes(self): + invalid = [-100, -1, 255, 512, 513, 1000] + self.check_bool_function_result(_opcode.is_valid, invalid, False) + self.check_bool_function_result(_opcode.has_arg, invalid, False) + self.check_bool_function_result(_opcode.has_const, invalid, False) + self.check_bool_function_result(_opcode.has_name, invalid, False) + self.check_bool_function_result(_opcode.has_jump, invalid, False) + self.check_bool_function_result(_opcode.has_free, invalid, False) + self.check_bool_function_result(_opcode.has_local, invalid, False) + self.check_bool_function_result(_opcode.has_exc, invalid, False) + + def test_is_valid(self): + names = [ + 'CACHE', + 'POP_TOP', + 'IMPORT_NAME', + 'JUMP', + 'INSTRUMENTED_RETURN_VALUE', + ] + opcodes = [dis.opmap[opname] for opname in names] + self.check_bool_function_result(_opcode.is_valid, opcodes, True) + + def test_oplists(self): + def check_function(self, func, expected): + for op in [-10, 520]: + with self.subTest(opcode=op, func=func): + res = func(op) + self.assertIsInstance(res, bool) + self.assertEqual(res, op in expected) + + check_function(self, _opcode.has_arg, dis.hasarg) + check_function(self, _opcode.has_const, dis.hasconst) + check_function(self, _opcode.has_name, dis.hasname) + check_function(self, _opcode.has_jump, dis.hasjump) + check_function(self, _opcode.has_free, dis.hasfree) + check_function(self, _opcode.has_local, dis.haslocal) + check_function(self, _opcode.has_exc, dis.hasexc) + + +class StackEffectTests(unittest.TestCase): + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_stack_effect(self): + self.assertEqual(stack_effect(dis.opmap['POP_TOP']), -1) + self.assertEqual(stack_effect(dis.opmap['BUILD_SLICE'], 0), -1) + self.assertEqual(stack_effect(dis.opmap['BUILD_SLICE'], 1), -1) + self.assertEqual(stack_effect(dis.opmap['BUILD_SLICE'], 3), -2) + self.assertRaises(ValueError, stack_effect, 30000) + # All defined opcodes + has_arg = dis.hasarg + for name, code in filter(lambda item: item[0] not in dis.deoptmap, dis.opmap.items()): + if code >= opcode.MIN_INSTRUMENTED_OPCODE: + continue + with self.subTest(opname=name): + stack_effect(code) + stack_effect(code, 0) + # All not defined opcodes + for code in set(range(256)) - set(dis.opmap.values()): + with self.subTest(opcode=code): + self.assertRaises(ValueError, stack_effect, code) + self.assertRaises(ValueError, stack_effect, code, 0) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_stack_effect_jump(self): + FOR_ITER = dis.opmap['FOR_ITER'] + self.assertEqual(stack_effect(FOR_ITER, 0), 1) + self.assertEqual(stack_effect(FOR_ITER, 0, jump=True), 1) + self.assertEqual(stack_effect(FOR_ITER, 0, jump=False), 1) + JUMP_FORWARD = dis.opmap['JUMP_FORWARD'] + self.assertEqual(stack_effect(JUMP_FORWARD, 0), 0) + self.assertEqual(stack_effect(JUMP_FORWARD, 0, jump=True), 0) + self.assertEqual(stack_effect(JUMP_FORWARD, 0, jump=False), 0) + # All defined opcodes + has_arg = dis.hasarg + has_exc = dis.hasexc + has_jump = dis.hasjabs + dis.hasjrel + for name, code in filter(lambda item: item[0] not in dis.deoptmap, dis.opmap.items()): + if code >= opcode.MIN_INSTRUMENTED_OPCODE: + continue + with self.subTest(opname=name): + if code not in has_arg: + common = stack_effect(code) + jump = stack_effect(code, jump=True) + nojump = stack_effect(code, jump=False) + else: + common = stack_effect(code, 0) + jump = stack_effect(code, 0, jump=True) + nojump = stack_effect(code, 0, jump=False) + if code in has_jump or code in has_exc: + self.assertEqual(common, max(jump, nojump)) + else: + self.assertEqual(jump, common) + self.assertEqual(nojump, common) + + +class SpecializationStatsTests(unittest.TestCase): + def test_specialization_stats(self): + stat_names = ["success", "failure", "hit", "deferred", "miss", "deopt"] + specialized_opcodes = [ + op.lower() + for op in opcode._specializations + if opcode._inline_cache_entries.get(op, 0) + ] + self.assertIn('load_attr', specialized_opcodes) + self.assertIn('binary_subscr', specialized_opcodes) + + stats = _opcode.get_specialization_stats() + if stats is not None: + self.assertIsInstance(stats, dict) + self.assertCountEqual(stats.keys(), specialized_opcodes) + self.assertCountEqual( + stats['load_attr'].keys(), + stat_names + ['failure_kinds']) + for sn in stat_names: + self.assertIsInstance(stats['load_attr'][sn], int) + self.assertIsInstance( + stats['load_attr']['failure_kinds'], + tuple) + for v in stats['load_attr']['failure_kinds']: + self.assertIsInstance(v, int) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_abc.py b/Lib/test/test_abc.py index ac46ea67bb..92bd955855 100644 --- a/Lib/test/test_abc.py +++ b/Lib/test/test_abc.py @@ -168,6 +168,8 @@ def method_two(self): msg = r"class C without an implementation for abstract methods 'method_one', 'method_two'" self.assertRaisesRegex(TypeError, msg, C) + # TODO: RUSTPYTHON; AssertionError: False is not true + @unittest.expectedFailure def test_abstractmethod_integration(self): for abstractthing in [abc.abstractmethod, abc.abstractproperty, abc.abstractclassmethod, diff --git a/Lib/test/test_argparse.py b/Lib/test/test_argparse.py index dc2df795a7..b7e995334f 100644 --- a/Lib/test/test_argparse.py +++ b/Lib/test/test_argparse.py @@ -6747,8 +6747,6 @@ def test_ambiguous_option(self): "ambiguous option: --foob=1 could match --foobaz, --fooble$", self.parser.parse_args, ['--foob=1', '--foogle', '2']) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_os_error(self): self.parser.add_argument('file') self.assertRaisesRegex(argparse.ArgumentError, diff --git a/Lib/test/test_array.py b/Lib/test/test_array.py index be89bec522..0c20e27cfd 100644 --- a/Lib/test/test_array.py +++ b/Lib/test/test_array.py @@ -1285,8 +1285,6 @@ def check_overflow(self, lower, upper): self.assertRaises(OverflowError, array.array, self.typecode, [upper+1]) self.assertRaises(OverflowError, a.__setitem__, 0, upper+1) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_subclassing(self): typecode = self.typecode class ExaggeratingArray(array.array): diff --git a/Lib/test/test_ast.py b/Lib/test/test_ast.py deleted file mode 100644 index 1cac438250..0000000000 --- a/Lib/test/test_ast.py +++ /dev/null @@ -1,3236 +0,0 @@ -import ast -import builtins -import dis -import enum -import os -import re -import sys -import textwrap -import types -import unittest -import warnings -import weakref -from functools import partial -from textwrap import dedent - -from test import support -from test.support.import_helper import import_fresh_module -from test.support import os_helper, script_helper -from test.support.ast_helper import ASTTestMixin - -def to_tuple(t): - if t is None or isinstance(t, (str, int, complex)) or t is Ellipsis: - return t - elif isinstance(t, list): - return [to_tuple(e) for e in t] - result = [t.__class__.__name__] - if hasattr(t, 'lineno') and hasattr(t, 'col_offset'): - result.append((t.lineno, t.col_offset)) - if hasattr(t, 'end_lineno') and hasattr(t, 'end_col_offset'): - result[-1] += (t.end_lineno, t.end_col_offset) - if t._fields is None: - return tuple(result) - for f in t._fields: - result.append(to_tuple(getattr(t, f))) - return tuple(result) - - -# These tests are compiled through "exec" -# There should be at least one test per statement -exec_tests = [ - # None - "None", - # Module docstring - "'module docstring'", - # FunctionDef - "def f(): pass", - # FunctionDef with docstring - "def f(): 'function docstring'", - # FunctionDef with arg - "def f(a): pass", - # FunctionDef with arg and default value - "def f(a=0): pass", - # FunctionDef with varargs - "def f(*args): pass", - # FunctionDef with varargs as TypeVarTuple - "def f(*args: *Ts): pass", - # FunctionDef with varargs as unpacked Tuple - "def f(*args: *tuple[int, ...]): pass", - # FunctionDef with varargs as unpacked Tuple *and* TypeVarTuple - "def f(*args: *tuple[int, *Ts]): pass", - # FunctionDef with kwargs - "def f(**kwargs): pass", - # FunctionDef with all kind of args and docstring - "def f(a, b=1, c=None, d=[], e={}, *args, f=42, **kwargs): 'doc for f()'", - # FunctionDef with type annotation on return involving unpacking - "def f() -> tuple[*Ts]: pass", - "def f() -> tuple[int, *Ts]: pass", - "def f() -> tuple[int, *tuple[int, ...]]: pass", - # ClassDef - "class C:pass", - # ClassDef with docstring - "class C: 'docstring for class C'", - # ClassDef, new style class - "class C(object): pass", - # Return - "def f():return 1", - # Delete - "del v", - # Assign - "v = 1", - "a,b = c", - "(a,b) = c", - "[a,b] = c", - # AnnAssign with unpacked types - "x: tuple[*Ts]", - "x: tuple[int, *Ts]", - "x: tuple[int, *tuple[str, ...]]", - # AugAssign - "v += 1", - # For - "for v in v:pass", - # While - "while v:pass", - # If - "if v:pass", - # If-Elif - "if a:\n pass\nelif b:\n pass", - # If-Elif-Else - "if a:\n pass\nelif b:\n pass\nelse:\n pass", - # With - "with x as y: pass", - "with x as y, z as q: pass", - # Raise - "raise Exception('string')", - # TryExcept - "try:\n pass\nexcept Exception:\n pass", - # TryFinally - "try:\n pass\nfinally:\n pass", - # TryStarExcept - "try:\n pass\nexcept* Exception:\n pass", - # Assert - "assert v", - # Import - "import sys", - # ImportFrom - "from sys import v", - # Global - "global v", - # Expr - "1", - # Pass, - "pass", - # Break - "for v in v:break", - # Continue - "for v in v:continue", - # for statements with naked tuples (see http://bugs.python.org/issue6704) - "for a,b in c: pass", - "for (a,b) in c: pass", - "for [a,b] in c: pass", - # Multiline generator expression (test for .lineno & .col_offset) - """( - ( - Aa - , - Bb - ) - for - Aa - , - Bb in Cc - )""", - # dictcomp - "{a : b for w in x for m in p if g}", - # dictcomp with naked tuple - "{a : b for v,w in x}", - # setcomp - "{r for l in x if g}", - # setcomp with naked tuple - "{r for l,m in x}", - # AsyncFunctionDef - "async def f():\n 'async function'\n await something()", - # AsyncFor - "async def f():\n async for e in i: 1\n else: 2", - # AsyncWith - "async def f():\n async with a as b: 1", - # PEP 448: Additional Unpacking Generalizations - "{**{1:2}, 2:3}", - "{*{1, 2}, 3}", - # Asynchronous comprehensions - "async def f():\n [i async for b in c]", - # Decorated FunctionDef - "@deco1\n@deco2()\n@deco3(1)\ndef f(): pass", - # Decorated AsyncFunctionDef - "@deco1\n@deco2()\n@deco3(1)\nasync def f(): pass", - # Decorated ClassDef - "@deco1\n@deco2()\n@deco3(1)\nclass C: pass", - # Decorator with generator argument - "@deco(a for a in b)\ndef f(): pass", - # Decorator with attribute - "@a.b.c\ndef f(): pass", - # Simple assignment expression - "(a := 1)", - # Positional-only arguments - "def f(a, /,): pass", - "def f(a, /, c, d, e): pass", - "def f(a, /, c, *, d, e): pass", - "def f(a, /, c, *, d, e, **kwargs): pass", - # Positional-only arguments with defaults - "def f(a=1, /,): pass", - "def f(a=1, /, b=2, c=4): pass", - "def f(a=1, /, b=2, *, c=4): pass", - "def f(a=1, /, b=2, *, c): pass", - "def f(a=1, /, b=2, *, c=4, **kwargs): pass", - "def f(a=1, /, b=2, *, c, **kwargs): pass", - # Type aliases - "type X = int", - "type X[T] = int", - "type X[T, *Ts, **P] = (T, Ts, P)", - "type X[T: int, *Ts, **P] = (T, Ts, P)", - "type X[T: (int, str), *Ts, **P] = (T, Ts, P)", - # Generic classes - "class X[T]: pass", - "class X[T, *Ts, **P]: pass", - "class X[T: int, *Ts, **P]: pass", - "class X[T: (int, str), *Ts, **P]: pass", - # Generic functions - "def f[T](): pass", - "def f[T, *Ts, **P](): pass", - "def f[T: int, *Ts, **P](): pass", - "def f[T: (int, str), *Ts, **P](): pass", -] - -# These are compiled through "single" -# because of overlap with "eval", it just tests what -# can't be tested with "eval" -single_tests = [ - "1+2" -] - -# These are compiled through "eval" -# It should test all expressions -eval_tests = [ - # None - "None", - # BoolOp - "a and b", - # BinOp - "a + b", - # UnaryOp - "not v", - # Lambda - "lambda:None", - # Dict - "{ 1:2 }", - # Empty dict - "{}", - # Set - "{None,}", - # Multiline dict (test for .lineno & .col_offset) - """{ - 1 - : - 2 - }""", - # ListComp - "[a for b in c if d]", - # GeneratorExp - "(a for b in c if d)", - # Comprehensions with multiple for targets - "[(a,b) for a,b in c]", - "[(a,b) for (a,b) in c]", - "[(a,b) for [a,b] in c]", - "{(a,b) for a,b in c}", - "{(a,b) for (a,b) in c}", - "{(a,b) for [a,b] in c}", - "((a,b) for a,b in c)", - "((a,b) for (a,b) in c)", - "((a,b) for [a,b] in c)", - # Yield - yield expressions can't work outside a function - # - # Compare - "1 < 2 < 3", - # Call - "f(1,2,c=3,*d,**e)", - # Call with multi-character starred - "f(*[0, 1])", - # Call with a generator argument - "f(a for a in b)", - # Num - "10", - # Str - "'string'", - # Attribute - "a.b", - # Subscript - "a[b:c]", - # Name - "v", - # List - "[1,2,3]", - # Empty list - "[]", - # Tuple - "1,2,3", - # Tuple - "(1,2,3)", - # Empty tuple - "()", - # Combination - "a.b.c.d(a.b[1:2])", -] - -# TODO: expr_context, slice, boolop, operator, unaryop, cmpop, comprehension -# excepthandler, arguments, keywords, alias - -class AST_Tests(unittest.TestCase): - maxDiff = None - - def _is_ast_node(self, name, node): - if not isinstance(node, type): - return False - if "ast" not in node.__module__: - return False - return name != 'AST' and name[0].isupper() - - def _assertTrueorder(self, ast_node, parent_pos): - if not isinstance(ast_node, ast.AST) or ast_node._fields is None: - return - if isinstance(ast_node, (ast.expr, ast.stmt, ast.excepthandler)): - node_pos = (ast_node.lineno, ast_node.col_offset) - self.assertGreaterEqual(node_pos, parent_pos) - parent_pos = (ast_node.lineno, ast_node.col_offset) - for name in ast_node._fields: - value = getattr(ast_node, name) - if isinstance(value, list): - first_pos = parent_pos - if value and name == 'decorator_list': - first_pos = (value[0].lineno, value[0].col_offset) - for child in value: - self._assertTrueorder(child, first_pos) - elif value is not None: - self._assertTrueorder(value, parent_pos) - self.assertEqual(ast_node._fields, ast_node.__match_args__) - - def test_AST_objects(self): - x = ast.AST() - self.assertEqual(x._fields, ()) - x.foobar = 42 - self.assertEqual(x.foobar, 42) - self.assertEqual(x.__dict__["foobar"], 42) - - with self.assertRaises(AttributeError): - x.vararg - - with self.assertRaises(TypeError): - # "ast.AST constructor takes 0 positional arguments" - ast.AST(2) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_AST_garbage_collection(self): - class X: - pass - a = ast.AST() - a.x = X() - a.x.a = a - ref = weakref.ref(a.x) - del a - support.gc_collect() - self.assertIsNone(ref()) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_snippets(self): - for input, output, kind in ((exec_tests, exec_results, "exec"), - (single_tests, single_results, "single"), - (eval_tests, eval_results, "eval")): - for i, o in zip(input, output): - with self.subTest(action="parsing", input=i): - ast_tree = compile(i, "?", kind, ast.PyCF_ONLY_AST) - self.assertEqual(to_tuple(ast_tree), o) - self._assertTrueorder(ast_tree, (0, 0)) - with self.subTest(action="compiling", input=i, kind=kind): - compile(ast_tree, "?", kind) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_ast_validation(self): - # compile() is the only function that calls PyAST_Validate - snippets_to_validate = exec_tests + single_tests + eval_tests - for snippet in snippets_to_validate: - tree = ast.parse(snippet) - compile(tree, '', 'exec') - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_invalid_position_information(self): - invalid_linenos = [ - (10, 1), (-10, -11), (10, -11), (-5, -2), (-5, 1) - ] - - for lineno, end_lineno in invalid_linenos: - with self.subTest(f"Check invalid linenos {lineno}:{end_lineno}"): - snippet = "a = 1" - tree = ast.parse(snippet) - tree.body[0].lineno = lineno - tree.body[0].end_lineno = end_lineno - with self.assertRaises(ValueError): - compile(tree, '', 'exec') - - invalid_col_offsets = [ - (10, 1), (-10, -11), (10, -11), (-5, -2), (-5, 1) - ] - for col_offset, end_col_offset in invalid_col_offsets: - with self.subTest(f"Check invalid col_offset {col_offset}:{end_col_offset}"): - snippet = "a = 1" - tree = ast.parse(snippet) - tree.body[0].col_offset = col_offset - tree.body[0].end_col_offset = end_col_offset - with self.assertRaises(ValueError): - compile(tree, '', 'exec') - - # XXX RUSTPYTHON: we always require that end ranges be present - @unittest.expectedFailure - def test_compilation_of_ast_nodes_with_default_end_position_values(self): - tree = ast.Module(body=[ - ast.Import(names=[ast.alias(name='builtins', lineno=1, col_offset=0)], lineno=1, col_offset=0), - ast.Import(names=[ast.alias(name='traceback', lineno=0, col_offset=0)], lineno=0, col_offset=1) - ], type_ignores=[]) - - # Check that compilation doesn't crash. Note: this may crash explicitly only on debug mode. - compile(tree, "", "exec") - - def test_slice(self): - slc = ast.parse("x[::]").body[0].value.slice - self.assertIsNone(slc.upper) - self.assertIsNone(slc.lower) - self.assertIsNone(slc.step) - - def test_from_import(self): - im = ast.parse("from . import y").body[0] - self.assertIsNone(im.module) - - def test_non_interned_future_from_ast(self): - mod = ast.parse("from __future__ import division") - self.assertIsInstance(mod.body[0], ast.ImportFrom) - mod.body[0].module = " __future__ ".strip() - compile(mod, "", "exec") - - def test_alias(self): - im = ast.parse("from bar import y").body[0] - self.assertEqual(len(im.names), 1) - alias = im.names[0] - self.assertEqual(alias.name, 'y') - self.assertIsNone(alias.asname) - self.assertEqual(alias.lineno, 1) - self.assertEqual(alias.end_lineno, 1) - self.assertEqual(alias.col_offset, 16) - self.assertEqual(alias.end_col_offset, 17) - - im = ast.parse("from bar import *").body[0] - alias = im.names[0] - self.assertEqual(alias.name, '*') - self.assertIsNone(alias.asname) - self.assertEqual(alias.lineno, 1) - self.assertEqual(alias.end_lineno, 1) - self.assertEqual(alias.col_offset, 16) - self.assertEqual(alias.end_col_offset, 17) - - im = ast.parse("from bar import y as z").body[0] - alias = im.names[0] - self.assertEqual(alias.name, "y") - self.assertEqual(alias.asname, "z") - self.assertEqual(alias.lineno, 1) - self.assertEqual(alias.end_lineno, 1) - self.assertEqual(alias.col_offset, 16) - self.assertEqual(alias.end_col_offset, 22) - - im = ast.parse("import bar as foo").body[0] - alias = im.names[0] - self.assertEqual(alias.name, "bar") - self.assertEqual(alias.asname, "foo") - self.assertEqual(alias.lineno, 1) - self.assertEqual(alias.end_lineno, 1) - self.assertEqual(alias.col_offset, 7) - self.assertEqual(alias.end_col_offset, 17) - - def test_base_classes(self): - self.assertTrue(issubclass(ast.For, ast.stmt)) - self.assertTrue(issubclass(ast.Name, ast.expr)) - self.assertTrue(issubclass(ast.stmt, ast.AST)) - self.assertTrue(issubclass(ast.expr, ast.AST)) - self.assertTrue(issubclass(ast.comprehension, ast.AST)) - self.assertTrue(issubclass(ast.Gt, ast.AST)) - - def test_import_deprecated(self): - ast = import_fresh_module('ast') - depr_regex = ( - r'ast\.{} is deprecated and will be removed in Python 3.14; ' - r'use ast\.Constant instead' - ) - for name in 'Num', 'Str', 'Bytes', 'NameConstant', 'Ellipsis': - with self.assertWarnsRegex(DeprecationWarning, depr_regex.format(name)): - getattr(ast, name) - - def test_field_attr_existence_deprecated(self): - with warnings.catch_warnings(): - warnings.filterwarnings('ignore', '', DeprecationWarning) - from ast import Num, Str, Bytes, NameConstant, Ellipsis - - for name in ('Num', 'Str', 'Bytes', 'NameConstant', 'Ellipsis'): - item = getattr(ast, name) - if self._is_ast_node(name, item): - with self.subTest(item): - with self.assertWarns(DeprecationWarning): - x = item() - if isinstance(x, ast.AST): - self.assertIs(type(x._fields), tuple) - - def test_field_attr_existence(self): - for name, item in ast.__dict__.items(): - # These emit DeprecationWarnings - if name in {'Num', 'Str', 'Bytes', 'NameConstant', 'Ellipsis'}: - continue - # constructor has a different signature - if name == 'Index': - continue - if self._is_ast_node(name, item): - x = item() - if isinstance(x, ast.AST): - self.assertIs(type(x._fields), tuple) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_arguments(self): - x = ast.arguments() - self.assertEqual(x._fields, ('posonlyargs', 'args', 'vararg', 'kwonlyargs', - 'kw_defaults', 'kwarg', 'defaults')) - - with self.assertRaises(AttributeError): - x.args - self.assertIsNone(x.vararg) - - x = ast.arguments(*range(1, 8)) - self.assertEqual(x.args, 2) - self.assertEqual(x.vararg, 3) - - def test_field_attr_writable_deprecated(self): - with warnings.catch_warnings(): - warnings.filterwarnings('ignore', '', DeprecationWarning) - x = ast.Num() - # We can assign to _fields - x._fields = 666 - self.assertEqual(x._fields, 666) - - def test_field_attr_writable(self): - x = ast.Constant() - # We can assign to _fields - x._fields = 666 - self.assertEqual(x._fields, 666) - - def test_classattrs_deprecated(self): - with warnings.catch_warnings(): - warnings.filterwarnings('ignore', '', DeprecationWarning) - from ast import Num, Str, Bytes, NameConstant, Ellipsis - - with warnings.catch_warnings(record=True) as wlog: - warnings.filterwarnings('always', '', DeprecationWarning) - x = ast.Num() - self.assertEqual(x._fields, ('value', 'kind')) - - with self.assertRaises(AttributeError): - x.value - - with self.assertRaises(AttributeError): - x.n - - x = ast.Num(42) - self.assertEqual(x.value, 42) - self.assertEqual(x.n, 42) - - with self.assertRaises(AttributeError): - x.lineno - - with self.assertRaises(AttributeError): - x.foobar - - x = ast.Num(lineno=2) - self.assertEqual(x.lineno, 2) - - x = ast.Num(42, lineno=0) - self.assertEqual(x.lineno, 0) - self.assertEqual(x._fields, ('value', 'kind')) - self.assertEqual(x.value, 42) - self.assertEqual(x.n, 42) - - self.assertRaises(TypeError, ast.Num, 1, None, 2) - self.assertRaises(TypeError, ast.Num, 1, None, 2, lineno=0) - - # Arbitrary keyword arguments are supported - self.assertEqual(ast.Num(1, foo='bar').foo, 'bar') - - with self.assertRaisesRegex(TypeError, "Num got multiple values for argument 'n'"): - ast.Num(1, n=2) - - self.assertEqual(ast.Num(42).n, 42) - self.assertEqual(ast.Num(4.25).n, 4.25) - self.assertEqual(ast.Num(4.25j).n, 4.25j) - self.assertEqual(ast.Str('42').s, '42') - self.assertEqual(ast.Bytes(b'42').s, b'42') - self.assertIs(ast.NameConstant(True).value, True) - self.assertIs(ast.NameConstant(False).value, False) - self.assertIs(ast.NameConstant(None).value, None) - - self.assertEqual([str(w.message) for w in wlog], [ - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', - 'ast.Str is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'Attribute s is deprecated and will be removed in Python 3.14; use value instead', - 'ast.Bytes is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'Attribute s is deprecated and will be removed in Python 3.14; use value instead', - 'ast.NameConstant is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.NameConstant is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.NameConstant is deprecated and will be removed in Python 3.14; use ast.Constant instead', - ]) - - def test_classattrs(self): - x = ast.Constant() - self.assertEqual(x._fields, ('value', 'kind')) - - with self.assertRaises(AttributeError): - x.value - - x = ast.Constant(42) - self.assertEqual(x.value, 42) - - with self.assertRaises(AttributeError): - x.lineno - - with self.assertRaises(AttributeError): - x.foobar - - x = ast.Constant(lineno=2) - self.assertEqual(x.lineno, 2) - - x = ast.Constant(42, lineno=0) - self.assertEqual(x.lineno, 0) - self.assertEqual(x._fields, ('value', 'kind')) - self.assertEqual(x.value, 42) - - self.assertRaises(TypeError, ast.Constant, 1, None, 2) - self.assertRaises(TypeError, ast.Constant, 1, None, 2, lineno=0) - - # Arbitrary keyword arguments are supported - self.assertEqual(ast.Constant(1, foo='bar').foo, 'bar') - - with self.assertRaisesRegex(TypeError, "Constant got multiple values for argument 'value'"): - ast.Constant(1, value=2) - - self.assertEqual(ast.Constant(42).value, 42) - self.assertEqual(ast.Constant(4.25).value, 4.25) - self.assertEqual(ast.Constant(4.25j).value, 4.25j) - self.assertEqual(ast.Constant('42').value, '42') - self.assertEqual(ast.Constant(b'42').value, b'42') - self.assertIs(ast.Constant(True).value, True) - self.assertIs(ast.Constant(False).value, False) - self.assertIs(ast.Constant(None).value, None) - self.assertIs(ast.Constant(...).value, ...) - - def test_realtype(self): - with warnings.catch_warnings(): - warnings.filterwarnings('ignore', '', DeprecationWarning) - from ast import Num, Str, Bytes, NameConstant, Ellipsis - - with warnings.catch_warnings(record=True) as wlog: - warnings.filterwarnings('always', '', DeprecationWarning) - self.assertIs(type(ast.Num(42)), ast.Constant) - self.assertIs(type(ast.Num(4.25)), ast.Constant) - self.assertIs(type(ast.Num(4.25j)), ast.Constant) - self.assertIs(type(ast.Str('42')), ast.Constant) - self.assertIs(type(ast.Bytes(b'42')), ast.Constant) - self.assertIs(type(ast.NameConstant(True)), ast.Constant) - self.assertIs(type(ast.NameConstant(False)), ast.Constant) - self.assertIs(type(ast.NameConstant(None)), ast.Constant) - self.assertIs(type(ast.Ellipsis()), ast.Constant) - - self.assertEqual([str(w.message) for w in wlog], [ - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.Str is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.Bytes is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.NameConstant is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.NameConstant is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.NameConstant is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.Ellipsis is deprecated and will be removed in Python 3.14; use ast.Constant instead', - ]) - - def test_isinstance(self): - from ast import Constant - - with warnings.catch_warnings(): - warnings.filterwarnings('ignore', '', DeprecationWarning) - from ast import Num, Str, Bytes, NameConstant, Ellipsis - - cls_depr_msg = ( - 'ast.{} is deprecated and will be removed in Python 3.14; ' - 'use ast.Constant instead' - ) - - assertNumDeprecated = partial( - self.assertWarnsRegex, DeprecationWarning, cls_depr_msg.format("Num") - ) - assertStrDeprecated = partial( - self.assertWarnsRegex, DeprecationWarning, cls_depr_msg.format("Str") - ) - assertBytesDeprecated = partial( - self.assertWarnsRegex, DeprecationWarning, cls_depr_msg.format("Bytes") - ) - assertNameConstantDeprecated = partial( - self.assertWarnsRegex, - DeprecationWarning, - cls_depr_msg.format("NameConstant") - ) - assertEllipsisDeprecated = partial( - self.assertWarnsRegex, DeprecationWarning, cls_depr_msg.format("Ellipsis") - ) - - for arg in 42, 4.2, 4.2j: - with self.subTest(arg=arg): - with assertNumDeprecated(): - n = Num(arg) - with assertNumDeprecated(): - self.assertIsInstance(n, Num) - - with assertStrDeprecated(): - s = Str('42') - with assertStrDeprecated(): - self.assertIsInstance(s, Str) - - with assertBytesDeprecated(): - b = Bytes(b'42') - with assertBytesDeprecated(): - self.assertIsInstance(b, Bytes) - - for arg in True, False, None: - with self.subTest(arg=arg): - with assertNameConstantDeprecated(): - n = NameConstant(arg) - with assertNameConstantDeprecated(): - self.assertIsInstance(n, NameConstant) - - with assertEllipsisDeprecated(): - e = Ellipsis() - with assertEllipsisDeprecated(): - self.assertIsInstance(e, Ellipsis) - - for arg in 42, 4.2, 4.2j: - with self.subTest(arg=arg): - with assertNumDeprecated(): - self.assertIsInstance(Constant(arg), Num) - - with assertStrDeprecated(): - self.assertIsInstance(Constant('42'), Str) - - with assertBytesDeprecated(): - self.assertIsInstance(Constant(b'42'), Bytes) - - for arg in True, False, None: - with self.subTest(arg=arg): - with assertNameConstantDeprecated(): - self.assertIsInstance(Constant(arg), NameConstant) - - with assertEllipsisDeprecated(): - self.assertIsInstance(Constant(...), Ellipsis) - - with assertStrDeprecated(): - s = Str('42') - assertNumDeprecated(self.assertNotIsInstance, s, Num) - assertBytesDeprecated(self.assertNotIsInstance, s, Bytes) - - with assertNumDeprecated(): - n = Num(42) - assertStrDeprecated(self.assertNotIsInstance, n, Str) - assertNameConstantDeprecated(self.assertNotIsInstance, n, NameConstant) - assertEllipsisDeprecated(self.assertNotIsInstance, n, Ellipsis) - - with assertNameConstantDeprecated(): - n = NameConstant(True) - with assertNumDeprecated(): - self.assertNotIsInstance(n, Num) - - with assertNameConstantDeprecated(): - n = NameConstant(False) - with assertNumDeprecated(): - self.assertNotIsInstance(n, Num) - - for arg in '42', True, False: - with self.subTest(arg=arg): - with assertNumDeprecated(): - self.assertNotIsInstance(Constant(arg), Num) - - assertStrDeprecated(self.assertNotIsInstance, Constant(42), Str) - assertBytesDeprecated(self.assertNotIsInstance, Constant('42'), Bytes) - assertNameConstantDeprecated(self.assertNotIsInstance, Constant(42), NameConstant) - assertEllipsisDeprecated(self.assertNotIsInstance, Constant(42), Ellipsis) - assertNumDeprecated(self.assertNotIsInstance, Constant(), Num) - assertStrDeprecated(self.assertNotIsInstance, Constant(), Str) - assertBytesDeprecated(self.assertNotIsInstance, Constant(), Bytes) - assertNameConstantDeprecated(self.assertNotIsInstance, Constant(), NameConstant) - assertEllipsisDeprecated(self.assertNotIsInstance, Constant(), Ellipsis) - - class S(str): pass - with assertStrDeprecated(): - self.assertIsInstance(Constant(S('42')), Str) - with assertNumDeprecated(): - self.assertNotIsInstance(Constant(S('42')), Num) - - def test_constant_subclasses_deprecated(self): - with warnings.catch_warnings(): - warnings.filterwarnings('ignore', '', DeprecationWarning) - from ast import Num - - with warnings.catch_warnings(record=True) as wlog: - warnings.filterwarnings('always', '', DeprecationWarning) - class N(ast.Num): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.z = 'spam' - class N2(ast.Num): - pass - - n = N(42) - self.assertEqual(n.n, 42) - self.assertEqual(n.z, 'spam') - self.assertIs(type(n), N) - self.assertIsInstance(n, N) - self.assertIsInstance(n, ast.Num) - self.assertNotIsInstance(n, N2) - self.assertNotIsInstance(ast.Num(42), N) - n = N(n=42) - self.assertEqual(n.n, 42) - self.assertIs(type(n), N) - - self.assertEqual([str(w.message) for w in wlog], [ - 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', - 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', - 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', - ]) - - def test_constant_subclasses(self): - class N(ast.Constant): - def __init__(self, *args, **kwargs): - super().__init__(*args, **kwargs) - self.z = 'spam' - class N2(ast.Constant): - pass - - n = N(42) - self.assertEqual(n.value, 42) - self.assertEqual(n.z, 'spam') - self.assertEqual(type(n), N) - self.assertTrue(isinstance(n, N)) - self.assertTrue(isinstance(n, ast.Constant)) - self.assertFalse(isinstance(n, N2)) - self.assertFalse(isinstance(ast.Constant(42), N)) - n = N(value=42) - self.assertEqual(n.value, 42) - self.assertEqual(type(n), N) - - def test_module(self): - body = [ast.Constant(42)] - x = ast.Module(body, []) - self.assertEqual(x.body, body) - - def test_nodeclasses(self): - # Zero arguments constructor explicitly allowed - x = ast.BinOp() - self.assertEqual(x._fields, ('left', 'op', 'right')) - - # Random attribute allowed too - x.foobarbaz = 5 - self.assertEqual(x.foobarbaz, 5) - - n1 = ast.Constant(1) - n3 = ast.Constant(3) - addop = ast.Add() - x = ast.BinOp(n1, addop, n3) - self.assertEqual(x.left, n1) - self.assertEqual(x.op, addop) - self.assertEqual(x.right, n3) - - x = ast.BinOp(1, 2, 3) - self.assertEqual(x.left, 1) - self.assertEqual(x.op, 2) - self.assertEqual(x.right, 3) - - x = ast.BinOp(1, 2, 3, lineno=0) - self.assertEqual(x.left, 1) - self.assertEqual(x.op, 2) - self.assertEqual(x.right, 3) - self.assertEqual(x.lineno, 0) - - # node raises exception when given too many arguments - self.assertRaises(TypeError, ast.BinOp, 1, 2, 3, 4) - # node raises exception when given too many arguments - self.assertRaises(TypeError, ast.BinOp, 1, 2, 3, 4, lineno=0) - - # can set attributes through kwargs too - x = ast.BinOp(left=1, op=2, right=3, lineno=0) - self.assertEqual(x.left, 1) - self.assertEqual(x.op, 2) - self.assertEqual(x.right, 3) - self.assertEqual(x.lineno, 0) - - # Random kwargs also allowed - x = ast.BinOp(1, 2, 3, foobarbaz=42) - self.assertEqual(x.foobarbaz, 42) - - def test_no_fields(self): - # this used to fail because Sub._fields was None - x = ast.Sub() - self.assertEqual(x._fields, ()) - - # TODO: RUSTPYTHON _ast classes should be HEAPTYPES (except for _ast.AST) - @unittest.expectedFailure - def test_pickling(self): - import pickle - - for protocol in range(pickle.HIGHEST_PROTOCOL + 1): - for ast in (compile(i, "?", "exec", 0x400) for i in exec_tests): - ast2 = pickle.loads(pickle.dumps(ast, protocol)) - self.assertEqual(to_tuple(ast2), to_tuple(ast)) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_invalid_sum(self): - pos = dict(lineno=2, col_offset=3) - m = ast.Module([ast.Expr(ast.expr(**pos), **pos)], []) - with self.assertRaises(TypeError) as cm: - compile(m, "", "exec") - self.assertIn("but got ", "exec") - self.assertIn("identifier must be of type str", str(cm.exception)) - - def test_invalid_constant(self): - for invalid_constant in int, (1, 2, int), frozenset((1, 2, int)): - e = ast.Expression(body=ast.Constant(invalid_constant)) - ast.fix_missing_locations(e) - with self.assertRaisesRegex( - TypeError, "invalid type in Constant: type" - ): - compile(e, "", "eval") - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_empty_yield_from(self): - # Issue 16546: yield from value is not optional. - empty_yield_from = ast.parse("def f():\n yield from g()") - empty_yield_from.body[0].body[0].value.value = None - with self.assertRaises(ValueError) as cm: - compile(empty_yield_from, "", "exec") - self.assertIn("field 'value' is required", str(cm.exception)) - - @support.cpython_only - def test_issue31592(self): - # There shouldn't be an assertion failure in case of a bad - # unicodedata.normalize(). - import unicodedata - def bad_normalize(*args): - return None - with support.swap_attr(unicodedata, 'normalize', bad_normalize): - self.assertRaises(TypeError, ast.parse, '\u03D5') - - def test_issue18374_binop_col_offset(self): - tree = ast.parse('4+5+6+7') - parent_binop = tree.body[0].value - child_binop = parent_binop.left - grandchild_binop = child_binop.left - self.assertEqual(parent_binop.col_offset, 0) - self.assertEqual(parent_binop.end_col_offset, 7) - self.assertEqual(child_binop.col_offset, 0) - self.assertEqual(child_binop.end_col_offset, 5) - self.assertEqual(grandchild_binop.col_offset, 0) - self.assertEqual(grandchild_binop.end_col_offset, 3) - - tree = ast.parse('4+5-\\\n 6-7') - parent_binop = tree.body[0].value - child_binop = parent_binop.left - grandchild_binop = child_binop.left - self.assertEqual(parent_binop.col_offset, 0) - self.assertEqual(parent_binop.lineno, 1) - self.assertEqual(parent_binop.end_col_offset, 4) - self.assertEqual(parent_binop.end_lineno, 2) - - self.assertEqual(child_binop.col_offset, 0) - self.assertEqual(child_binop.lineno, 1) - self.assertEqual(child_binop.end_col_offset, 2) - self.assertEqual(child_binop.end_lineno, 2) - - self.assertEqual(grandchild_binop.col_offset, 0) - self.assertEqual(grandchild_binop.lineno, 1) - self.assertEqual(grandchild_binop.end_col_offset, 3) - self.assertEqual(grandchild_binop.end_lineno, 1) - - def test_issue39579_dotted_name_end_col_offset(self): - tree = ast.parse('@a.b.c\ndef f(): pass') - attr_b = tree.body[0].decorator_list[0].value - self.assertEqual(attr_b.end_col_offset, 4) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_ast_asdl_signature(self): - self.assertEqual(ast.withitem.__doc__, "withitem(expr context_expr, expr? optional_vars)") - self.assertEqual(ast.GtE.__doc__, "GtE") - self.assertEqual(ast.Name.__doc__, "Name(identifier id, expr_context ctx)") - self.assertEqual(ast.cmpop.__doc__, "cmpop = Eq | NotEq | Lt | LtE | Gt | GtE | Is | IsNot | In | NotIn") - expressions = [f" | {node.__doc__}" for node in ast.expr.__subclasses__()] - expressions[0] = f"expr = {ast.expr.__subclasses__()[0].__doc__}" - self.assertCountEqual(ast.expr.__doc__.split("\n"), expressions) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_positional_only_feature_version(self): - ast.parse('def foo(x, /): ...', feature_version=(3, 8)) - ast.parse('def bar(x=1, /): ...', feature_version=(3, 8)) - with self.assertRaises(SyntaxError): - ast.parse('def foo(x, /): ...', feature_version=(3, 7)) - with self.assertRaises(SyntaxError): - ast.parse('def bar(x=1, /): ...', feature_version=(3, 7)) - - ast.parse('lambda x, /: ...', feature_version=(3, 8)) - ast.parse('lambda x=1, /: ...', feature_version=(3, 8)) - with self.assertRaises(SyntaxError): - ast.parse('lambda x, /: ...', feature_version=(3, 7)) - with self.assertRaises(SyntaxError): - ast.parse('lambda x=1, /: ...', feature_version=(3, 7)) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_parenthesized_with_feature_version(self): - ast.parse('with (CtxManager() as example): ...', feature_version=(3, 10)) - # While advertised as a feature in Python 3.10, this was allowed starting 3.9 - ast.parse('with (CtxManager() as example): ...', feature_version=(3, 9)) - with self.assertRaises(SyntaxError): - ast.parse('with (CtxManager() as example): ...', feature_version=(3, 8)) - ast.parse('with CtxManager() as example: ...', feature_version=(3, 8)) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_assignment_expression_feature_version(self): - ast.parse('(x := 0)', feature_version=(3, 8)) - with self.assertRaises(SyntaxError): - ast.parse('(x := 0)', feature_version=(3, 7)) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_exception_groups_feature_version(self): - code = dedent(''' - try: ... - except* Exception: ... - ''') - ast.parse(code) - with self.assertRaises(SyntaxError): - ast.parse(code, feature_version=(3, 10)) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_type_params_feature_version(self): - samples = [ - "type X = int", - "class X[T]: pass", - "def f[T](): pass", - ] - for sample in samples: - with self.subTest(sample): - ast.parse(sample) - with self.assertRaises(SyntaxError): - ast.parse(sample, feature_version=(3, 11)) - - def test_invalid_major_feature_version(self): - with self.assertRaises(ValueError): - ast.parse('pass', feature_version=(2, 7)) - with self.assertRaises(ValueError): - ast.parse('pass', feature_version=(4, 0)) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_constant_as_name(self): - for constant in "True", "False", "None": - expr = ast.Expression(ast.Name(constant, ast.Load())) - ast.fix_missing_locations(expr) - with self.assertRaisesRegex(ValueError, f"identifier field can't represent '{constant}' constant"): - compile(expr, "", "eval") - - @unittest.skip("TODO: RUSTPYTHON, TypeError: enum mismatch") - def test_precedence_enum(self): - class _Precedence(enum.IntEnum): - """Precedence table that originated from python grammar.""" - NAMED_EXPR = enum.auto() # := - TUPLE = enum.auto() # , - YIELD = enum.auto() # 'yield', 'yield from' - TEST = enum.auto() # 'if'-'else', 'lambda' - OR = enum.auto() # 'or' - AND = enum.auto() # 'and' - NOT = enum.auto() # 'not' - CMP = enum.auto() # '<', '>', '==', '>=', '<=', '!=', - # 'in', 'not in', 'is', 'is not' - EXPR = enum.auto() - BOR = EXPR # '|' - BXOR = enum.auto() # '^' - BAND = enum.auto() # '&' - SHIFT = enum.auto() # '<<', '>>' - ARITH = enum.auto() # '+', '-' - TERM = enum.auto() # '*', '@', '/', '%', '//' - FACTOR = enum.auto() # unary '+', '-', '~' - POWER = enum.auto() # '**' - AWAIT = enum.auto() # 'await' - ATOM = enum.auto() - def next(self): - try: - return self.__class__(self + 1) - except ValueError: - return self - enum._test_simple_enum(_Precedence, ast._Precedence) - - @unittest.skipIf(support.is_wasi, "exhausts limited stack on WASI") - @support.cpython_only - def test_ast_recursion_limit(self): - fail_depth = support.EXCEEDS_RECURSION_LIMIT - crash_depth = 100_000 - success_depth = 1200 - - def check_limit(prefix, repeated): - expect_ok = prefix + repeated * success_depth - ast.parse(expect_ok) - for depth in (fail_depth, crash_depth): - broken = prefix + repeated * depth - details = "Compiling ({!r} + {!r} * {})".format( - prefix, repeated, depth) - with self.assertRaises(RecursionError, msg=details): - with support.infinite_recursion(): - ast.parse(broken) - - check_limit("a", "()") - check_limit("a", ".b") - check_limit("a", "[0]") - check_limit("a", "*a") - - def test_null_bytes(self): - with self.assertRaises(SyntaxError, - msg="source code string cannot contain null bytes"): - ast.parse("a\0b") - - def assert_none_check(self, node: type[ast.AST], attr: str, source: str) -> None: - with self.subTest(f"{node.__name__}.{attr}"): - tree = ast.parse(source) - found = 0 - for child in ast.walk(tree): - if isinstance(child, node): - setattr(child, attr, None) - found += 1 - self.assertEqual(found, 1) - e = re.escape(f"field '{attr}' is required for {node.__name__}") - with self.assertRaisesRegex(ValueError, f"^{e}$"): - compile(tree, "", "exec") - - @unittest.skip("TODO: RUSTPYTHON, TypeError: Expected type 'str' but 'NoneType' found") - def test_none_checks(self) -> None: - tests = [ - (ast.alias, "name", "import spam as SPAM"), - (ast.arg, "arg", "def spam(SPAM): spam"), - (ast.comprehension, "target", "[spam for SPAM in spam]"), - (ast.comprehension, "iter", "[spam for spam in SPAM]"), - (ast.keyword, "value", "spam(**SPAM)"), - (ast.match_case, "pattern", "match spam:\n case SPAM: spam"), - (ast.withitem, "context_expr", "with SPAM: spam"), - ] - for node, attr, source in tests: - self.assert_none_check(node, attr, source) - -class ASTHelpers_Test(unittest.TestCase): - maxDiff = None - - def test_parse(self): - a = ast.parse('foo(1 + 1)') - b = compile('foo(1 + 1)', '', 'exec', ast.PyCF_ONLY_AST) - self.assertEqual(ast.dump(a), ast.dump(b)) - - def test_parse_in_error(self): - try: - 1/0 - except Exception: - with self.assertRaises(SyntaxError) as e: - ast.literal_eval(r"'\U'") - self.assertIsNotNone(e.exception.__context__) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_dump(self): - node = ast.parse('spam(eggs, "and cheese")') - self.assertEqual(ast.dump(node), - "Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load()), " - "args=[Name(id='eggs', ctx=Load()), Constant(value='and cheese')], " - "keywords=[]))], type_ignores=[])" - ) - self.assertEqual(ast.dump(node, annotate_fields=False), - "Module([Expr(Call(Name('spam', Load()), [Name('eggs', Load()), " - "Constant('and cheese')], []))], [])" - ) - self.assertEqual(ast.dump(node, include_attributes=True), - "Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load(), " - "lineno=1, col_offset=0, end_lineno=1, end_col_offset=4), " - "args=[Name(id='eggs', ctx=Load(), lineno=1, col_offset=5, " - "end_lineno=1, end_col_offset=9), Constant(value='and cheese', " - "lineno=1, col_offset=11, end_lineno=1, end_col_offset=23)], keywords=[], " - "lineno=1, col_offset=0, end_lineno=1, end_col_offset=24), " - "lineno=1, col_offset=0, end_lineno=1, end_col_offset=24)], type_ignores=[])" - ) - - # TODO: RUSTPYTHON; redundant kind for Contant node - @unittest.expectedFailure - def test_dump_indent(self): - node = ast.parse('spam(eggs, "and cheese")') - self.assertEqual(ast.dump(node, indent=3), """\ -Module( - body=[ - Expr( - value=Call( - func=Name(id='spam', ctx=Load()), - args=[ - Name(id='eggs', ctx=Load()), - Constant(value='and cheese')], - keywords=[]))], - type_ignores=[])""") - self.assertEqual(ast.dump(node, annotate_fields=False, indent='\t'), """\ -Module( -\t[ -\t\tExpr( -\t\t\tCall( -\t\t\t\tName('spam', Load()), -\t\t\t\t[ -\t\t\t\t\tName('eggs', Load()), -\t\t\t\t\tConstant('and cheese')], -\t\t\t\t[]))], -\t[])""") - self.assertEqual(ast.dump(node, include_attributes=True, indent=3), """\ -Module( - body=[ - Expr( - value=Call( - func=Name( - id='spam', - ctx=Load(), - lineno=1, - col_offset=0, - end_lineno=1, - end_col_offset=4), - args=[ - Name( - id='eggs', - ctx=Load(), - lineno=1, - col_offset=5, - end_lineno=1, - end_col_offset=9), - Constant( - value='and cheese', - lineno=1, - col_offset=11, - end_lineno=1, - end_col_offset=23)], - keywords=[], - lineno=1, - col_offset=0, - end_lineno=1, - end_col_offset=24), - lineno=1, - col_offset=0, - end_lineno=1, - end_col_offset=24)], - type_ignores=[])""") - - def test_dump_incomplete(self): - node = ast.Raise(lineno=3, col_offset=4) - self.assertEqual(ast.dump(node), - "Raise()" - ) - self.assertEqual(ast.dump(node, include_attributes=True), - "Raise(lineno=3, col_offset=4)" - ) - node = ast.Raise(exc=ast.Name(id='e', ctx=ast.Load()), lineno=3, col_offset=4) - self.assertEqual(ast.dump(node), - "Raise(exc=Name(id='e', ctx=Load()))" - ) - self.assertEqual(ast.dump(node, annotate_fields=False), - "Raise(Name('e', Load()))" - ) - self.assertEqual(ast.dump(node, include_attributes=True), - "Raise(exc=Name(id='e', ctx=Load()), lineno=3, col_offset=4)" - ) - self.assertEqual(ast.dump(node, annotate_fields=False, include_attributes=True), - "Raise(Name('e', Load()), lineno=3, col_offset=4)" - ) - node = ast.Raise(cause=ast.Name(id='e', ctx=ast.Load())) - self.assertEqual(ast.dump(node), - "Raise(cause=Name(id='e', ctx=Load()))" - ) - self.assertEqual(ast.dump(node, annotate_fields=False), - "Raise(cause=Name('e', Load()))" - ) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_copy_location(self): - src = ast.parse('1 + 1', mode='eval') - src.body.right = ast.copy_location(ast.Constant(2), src.body.right) - self.assertEqual(ast.dump(src, include_attributes=True), - 'Expression(body=BinOp(left=Constant(value=1, lineno=1, col_offset=0, ' - 'end_lineno=1, end_col_offset=1), op=Add(), right=Constant(value=2, ' - 'lineno=1, col_offset=4, end_lineno=1, end_col_offset=5), lineno=1, ' - 'col_offset=0, end_lineno=1, end_col_offset=5))' - ) - src = ast.Call(col_offset=1, lineno=1, end_lineno=1, end_col_offset=1) - new = ast.copy_location(src, ast.Call(col_offset=None, lineno=None)) - self.assertIsNone(new.end_lineno) - self.assertIsNone(new.end_col_offset) - self.assertEqual(new.lineno, 1) - self.assertEqual(new.col_offset, 1) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_fix_missing_locations(self): - src = ast.parse('write("spam")') - src.body.append(ast.Expr(ast.Call(ast.Name('spam', ast.Load()), - [ast.Constant('eggs')], []))) - self.assertEqual(src, ast.fix_missing_locations(src)) - self.maxDiff = None - self.assertEqual(ast.dump(src, include_attributes=True), - "Module(body=[Expr(value=Call(func=Name(id='write', ctx=Load(), " - "lineno=1, col_offset=0, end_lineno=1, end_col_offset=5), " - "args=[Constant(value='spam', lineno=1, col_offset=6, end_lineno=1, " - "end_col_offset=12)], keywords=[], lineno=1, col_offset=0, end_lineno=1, " - "end_col_offset=13), lineno=1, col_offset=0, end_lineno=1, " - "end_col_offset=13), Expr(value=Call(func=Name(id='spam', ctx=Load(), " - "lineno=1, col_offset=0, end_lineno=1, end_col_offset=0), " - "args=[Constant(value='eggs', lineno=1, col_offset=0, end_lineno=1, " - "end_col_offset=0)], keywords=[], lineno=1, col_offset=0, end_lineno=1, " - "end_col_offset=0), lineno=1, col_offset=0, end_lineno=1, end_col_offset=0)], " - "type_ignores=[])" - ) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_increment_lineno(self): - src = ast.parse('1 + 1', mode='eval') - self.assertEqual(ast.increment_lineno(src, n=3), src) - self.assertEqual(ast.dump(src, include_attributes=True), - 'Expression(body=BinOp(left=Constant(value=1, lineno=4, col_offset=0, ' - 'end_lineno=4, end_col_offset=1), op=Add(), right=Constant(value=1, ' - 'lineno=4, col_offset=4, end_lineno=4, end_col_offset=5), lineno=4, ' - 'col_offset=0, end_lineno=4, end_col_offset=5))' - ) - # issue10869: do not increment lineno of root twice - src = ast.parse('1 + 1', mode='eval') - self.assertEqual(ast.increment_lineno(src.body, n=3), src.body) - self.assertEqual(ast.dump(src, include_attributes=True), - 'Expression(body=BinOp(left=Constant(value=1, lineno=4, col_offset=0, ' - 'end_lineno=4, end_col_offset=1), op=Add(), right=Constant(value=1, ' - 'lineno=4, col_offset=4, end_lineno=4, end_col_offset=5), lineno=4, ' - 'col_offset=0, end_lineno=4, end_col_offset=5))' - ) - src = ast.Call( - func=ast.Name("test", ast.Load()), args=[], keywords=[], lineno=1 - ) - self.assertEqual(ast.increment_lineno(src).lineno, 2) - self.assertIsNone(ast.increment_lineno(src).end_lineno) - - @unittest.skip("TODO: RUSTPYTHON, NameError: name 'PyCF_TYPE_COMMENTS' is not defined") - def test_increment_lineno_on_module(self): - src = ast.parse(dedent("""\ - a = 1 - b = 2 # type: ignore - c = 3 - d = 4 # type: ignore@tag - """), type_comments=True) - ast.increment_lineno(src, n=5) - self.assertEqual(src.type_ignores[0].lineno, 7) - self.assertEqual(src.type_ignores[1].lineno, 9) - self.assertEqual(src.type_ignores[1].tag, '@tag') - - def test_iter_fields(self): - node = ast.parse('foo()', mode='eval') - d = dict(ast.iter_fields(node.body)) - self.assertEqual(d.pop('func').id, 'foo') - self.assertEqual(d, {'keywords': [], 'args': []}) - - # TODO: RUSTPYTHON; redundant kind for Constant node - @unittest.expectedFailure - def test_iter_child_nodes(self): - node = ast.parse("spam(23, 42, eggs='leek')", mode='eval') - self.assertEqual(len(list(ast.iter_child_nodes(node.body))), 4) - iterator = ast.iter_child_nodes(node.body) - self.assertEqual(next(iterator).id, 'spam') - self.assertEqual(next(iterator).value, 23) - self.assertEqual(next(iterator).value, 42) - self.assertEqual(ast.dump(next(iterator)), - "keyword(arg='eggs', value=Constant(value='leek'))" - ) - - def test_get_docstring(self): - node = ast.parse('"""line one\n line two"""') - self.assertEqual(ast.get_docstring(node), - 'line one\nline two') - - node = ast.parse('class foo:\n """line one\n line two"""') - self.assertEqual(ast.get_docstring(node.body[0]), - 'line one\nline two') - - node = ast.parse('def foo():\n """line one\n line two"""') - self.assertEqual(ast.get_docstring(node.body[0]), - 'line one\nline two') - - node = ast.parse('async def foo():\n """spam\n ham"""') - self.assertEqual(ast.get_docstring(node.body[0]), 'spam\nham') - - def test_get_docstring_none(self): - self.assertIsNone(ast.get_docstring(ast.parse(''))) - node = ast.parse('x = "not docstring"') - self.assertIsNone(ast.get_docstring(node)) - node = ast.parse('def foo():\n pass') - self.assertIsNone(ast.get_docstring(node)) - - node = ast.parse('class foo:\n pass') - self.assertIsNone(ast.get_docstring(node.body[0])) - node = ast.parse('class foo:\n x = "not docstring"') - self.assertIsNone(ast.get_docstring(node.body[0])) - node = ast.parse('class foo:\n def bar(self): pass') - self.assertIsNone(ast.get_docstring(node.body[0])) - - node = ast.parse('def foo():\n pass') - self.assertIsNone(ast.get_docstring(node.body[0])) - node = ast.parse('def foo():\n x = "not docstring"') - self.assertIsNone(ast.get_docstring(node.body[0])) - - node = ast.parse('async def foo():\n pass') - self.assertIsNone(ast.get_docstring(node.body[0])) - node = ast.parse('async def foo():\n x = "not docstring"') - self.assertIsNone(ast.get_docstring(node.body[0])) - - def test_multi_line_docstring_col_offset_and_lineno_issue16806(self): - node = ast.parse( - '"""line one\nline two"""\n\n' - 'def foo():\n """line one\n line two"""\n\n' - ' def bar():\n """line one\n line two"""\n' - ' """line one\n line two"""\n' - '"""line one\nline two"""\n\n' - ) - self.assertEqual(node.body[0].col_offset, 0) - self.assertEqual(node.body[0].lineno, 1) - self.assertEqual(node.body[1].body[0].col_offset, 2) - self.assertEqual(node.body[1].body[0].lineno, 5) - self.assertEqual(node.body[1].body[1].body[0].col_offset, 4) - self.assertEqual(node.body[1].body[1].body[0].lineno, 9) - self.assertEqual(node.body[1].body[2].col_offset, 2) - self.assertEqual(node.body[1].body[2].lineno, 11) - self.assertEqual(node.body[2].col_offset, 0) - self.assertEqual(node.body[2].lineno, 13) - - def test_elif_stmt_start_position(self): - node = ast.parse('if a:\n pass\nelif b:\n pass\n') - elif_stmt = node.body[0].orelse[0] - self.assertEqual(elif_stmt.lineno, 3) - self.assertEqual(elif_stmt.col_offset, 0) - - def test_elif_stmt_start_position_with_else(self): - node = ast.parse('if a:\n pass\nelif b:\n pass\nelse:\n pass\n') - elif_stmt = node.body[0].orelse[0] - self.assertEqual(elif_stmt.lineno, 3) - self.assertEqual(elif_stmt.col_offset, 0) - - def test_starred_expr_end_position_within_call(self): - node = ast.parse('f(*[0, 1])') - starred_expr = node.body[0].value.args[0] - self.assertEqual(starred_expr.end_lineno, 1) - self.assertEqual(starred_expr.end_col_offset, 9) - - def test_literal_eval(self): - self.assertEqual(ast.literal_eval('[1, 2, 3]'), [1, 2, 3]) - self.assertEqual(ast.literal_eval('{"foo": 42}'), {"foo": 42}) - self.assertEqual(ast.literal_eval('(True, False, None)'), (True, False, None)) - self.assertEqual(ast.literal_eval('{1, 2, 3}'), {1, 2, 3}) - self.assertEqual(ast.literal_eval('b"hi"'), b"hi") - self.assertEqual(ast.literal_eval('set()'), set()) - self.assertRaises(ValueError, ast.literal_eval, 'foo()') - self.assertEqual(ast.literal_eval('6'), 6) - self.assertEqual(ast.literal_eval('+6'), 6) - self.assertEqual(ast.literal_eval('-6'), -6) - self.assertEqual(ast.literal_eval('3.25'), 3.25) - self.assertEqual(ast.literal_eval('+3.25'), 3.25) - self.assertEqual(ast.literal_eval('-3.25'), -3.25) - self.assertEqual(repr(ast.literal_eval('-0.0')), '-0.0') - self.assertRaises(ValueError, ast.literal_eval, '++6') - self.assertRaises(ValueError, ast.literal_eval, '+True') - self.assertRaises(ValueError, ast.literal_eval, '2+3') - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_literal_eval_str_int_limit(self): - with support.adjust_int_max_str_digits(4000): - ast.literal_eval('3'*4000) # no error - with self.assertRaises(SyntaxError) as err_ctx: - ast.literal_eval('3'*4001) - self.assertIn('Exceeds the limit ', str(err_ctx.exception)) - self.assertIn(' Consider hexadecimal ', str(err_ctx.exception)) - - def test_literal_eval_complex(self): - # Issue #4907 - self.assertEqual(ast.literal_eval('6j'), 6j) - self.assertEqual(ast.literal_eval('-6j'), -6j) - self.assertEqual(ast.literal_eval('6.75j'), 6.75j) - self.assertEqual(ast.literal_eval('-6.75j'), -6.75j) - self.assertEqual(ast.literal_eval('3+6j'), 3+6j) - self.assertEqual(ast.literal_eval('-3+6j'), -3+6j) - self.assertEqual(ast.literal_eval('3-6j'), 3-6j) - self.assertEqual(ast.literal_eval('-3-6j'), -3-6j) - self.assertEqual(ast.literal_eval('3.25+6.75j'), 3.25+6.75j) - self.assertEqual(ast.literal_eval('-3.25+6.75j'), -3.25+6.75j) - self.assertEqual(ast.literal_eval('3.25-6.75j'), 3.25-6.75j) - self.assertEqual(ast.literal_eval('-3.25-6.75j'), -3.25-6.75j) - self.assertEqual(ast.literal_eval('(3+6j)'), 3+6j) - self.assertRaises(ValueError, ast.literal_eval, '-6j+3') - self.assertRaises(ValueError, ast.literal_eval, '-6j+3j') - self.assertRaises(ValueError, ast.literal_eval, '3+-6j') - self.assertRaises(ValueError, ast.literal_eval, '3+(0+6j)') - self.assertRaises(ValueError, ast.literal_eval, '-(3+6j)') - - def test_literal_eval_malformed_dict_nodes(self): - malformed = ast.Dict(keys=[ast.Constant(1), ast.Constant(2)], values=[ast.Constant(3)]) - self.assertRaises(ValueError, ast.literal_eval, malformed) - malformed = ast.Dict(keys=[ast.Constant(1)], values=[ast.Constant(2), ast.Constant(3)]) - self.assertRaises(ValueError, ast.literal_eval, malformed) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_literal_eval_trailing_ws(self): - self.assertEqual(ast.literal_eval(" -1"), -1) - self.assertEqual(ast.literal_eval("\t\t-1"), -1) - self.assertEqual(ast.literal_eval(" \t -1"), -1) - self.assertRaises(IndentationError, ast.literal_eval, "\n -1") - - def test_literal_eval_malformed_lineno(self): - msg = r'malformed node or string on line 3:' - with self.assertRaisesRegex(ValueError, msg): - ast.literal_eval("{'a': 1,\n'b':2,\n'c':++3,\n'd':4}") - - node = ast.UnaryOp( - ast.UAdd(), ast.UnaryOp(ast.UAdd(), ast.Constant(6))) - self.assertIsNone(getattr(node, 'lineno', None)) - msg = r'malformed node or string:' - with self.assertRaisesRegex(ValueError, msg): - ast.literal_eval(node) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_literal_eval_syntax_errors(self): - with self.assertRaisesRegex(SyntaxError, "unexpected indent"): - ast.literal_eval(r''' - \ - (\ - \ ''') - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_bad_integer(self): - # issue13436: Bad error message with invalid numeric values - body = [ast.ImportFrom(module='time', - names=[ast.alias(name='sleep')], - level=None, - lineno=None, col_offset=None)] - mod = ast.Module(body, []) - with self.assertRaises(ValueError) as cm: - compile(mod, 'test', 'exec') - self.assertIn("invalid integer value: None", str(cm.exception)) - - # XXX RUSTPYTHON: we always require that end ranges be present - @unittest.expectedFailure - def test_level_as_none(self): - body = [ast.ImportFrom(module='time', - names=[ast.alias(name='sleep', - lineno=0, col_offset=0)], - level=None, - lineno=0, col_offset=0)] - mod = ast.Module(body, []) - code = compile(mod, 'test', 'exec') - ns = {} - exec(code, ns) - self.assertIn('sleep', ns) - - @unittest.skip("TODO: RUSTPYTHON; crash") - def test_recursion_direct(self): - e = ast.UnaryOp(op=ast.Not(), lineno=0, col_offset=0) - e.operand = e - with self.assertRaises(RecursionError): - with support.infinite_recursion(): - compile(ast.Expression(e), "", "eval") - - @unittest.skip("TODO: RUSTPYTHON; crash") - def test_recursion_indirect(self): - e = ast.UnaryOp(op=ast.Not(), lineno=0, col_offset=0) - f = ast.UnaryOp(op=ast.Not(), lineno=0, col_offset=0) - e.operand = f - f.operand = e - with self.assertRaises(RecursionError): - with support.infinite_recursion(): - compile(ast.Expression(e), "", "eval") - - -class ASTValidatorTests(unittest.TestCase): - - def mod(self, mod, msg=None, mode="exec", *, exc=ValueError): - mod.lineno = mod.col_offset = 0 - ast.fix_missing_locations(mod) - if msg is None: - compile(mod, "", mode) - else: - with self.assertRaises(exc) as cm: - compile(mod, "", mode) - self.assertIn(msg, str(cm.exception)) - - def expr(self, node, msg=None, *, exc=ValueError): - mod = ast.Module([ast.Expr(node)], []) - self.mod(mod, msg, exc=exc) - - def stmt(self, stmt, msg=None): - mod = ast.Module([stmt], []) - self.mod(mod, msg) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_module(self): - m = ast.Interactive([ast.Expr(ast.Name("x", ast.Store()))]) - self.mod(m, "must have Load context", "single") - m = ast.Expression(ast.Name("x", ast.Store())) - self.mod(m, "must have Load context", "eval") - - def _check_arguments(self, fac, check): - def arguments(args=None, posonlyargs=None, vararg=None, - kwonlyargs=None, kwarg=None, - defaults=None, kw_defaults=None): - if args is None: - args = [] - if posonlyargs is None: - posonlyargs = [] - if kwonlyargs is None: - kwonlyargs = [] - if defaults is None: - defaults = [] - if kw_defaults is None: - kw_defaults = [] - args = ast.arguments(args, posonlyargs, vararg, kwonlyargs, - kw_defaults, kwarg, defaults) - return fac(args) - args = [ast.arg("x", ast.Name("x", ast.Store()))] - check(arguments(args=args), "must have Load context") - check(arguments(posonlyargs=args), "must have Load context") - check(arguments(kwonlyargs=args), "must have Load context") - check(arguments(defaults=[ast.Constant(3)]), - "more positional defaults than args") - check(arguments(kw_defaults=[ast.Constant(4)]), - "length of kwonlyargs is not the same as kw_defaults") - args = [ast.arg("x", ast.Name("x", ast.Load()))] - check(arguments(args=args, defaults=[ast.Name("x", ast.Store())]), - "must have Load context") - args = [ast.arg("a", ast.Name("x", ast.Load())), - ast.arg("b", ast.Name("y", ast.Load()))] - check(arguments(kwonlyargs=args, - kw_defaults=[None, ast.Name("x", ast.Store())]), - "must have Load context") - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_funcdef(self): - a = ast.arguments([], [], None, [], [], None, []) - f = ast.FunctionDef("x", a, [], [], None, None, []) - self.stmt(f, "empty body on FunctionDef") - f = ast.FunctionDef("x", a, [ast.Pass()], [ast.Name("x", ast.Store())], None, None, []) - self.stmt(f, "must have Load context") - f = ast.FunctionDef("x", a, [ast.Pass()], [], - ast.Name("x", ast.Store()), None, []) - self.stmt(f, "must have Load context") - f = ast.FunctionDef("x", ast.arguments(), [ast.Pass()]) - self.stmt(f) - def fac(args): - return ast.FunctionDef("x", args, [ast.Pass()], [], None, None, []) - self._check_arguments(fac, self.stmt) - - # TODO: RUSTPYTHON, match expression is not implemented yet - # def test_funcdef_pattern_matching(self): - # # gh-104799: New fields on FunctionDef should be added at the end - # def matcher(node): - # match node: - # case ast.FunctionDef("foo", ast.arguments(args=[ast.arg("bar")]), - # [ast.Pass()], - # [ast.Name("capybara", ast.Load())], - # ast.Name("pacarana", ast.Load())): - # return True - # case _: - # return False - - # code = """ - # @capybara - # def foo(bar) -> pacarana: - # pass - # """ - # source = ast.parse(textwrap.dedent(code)) - # funcdef = source.body[0] - # self.assertIsInstance(funcdef, ast.FunctionDef) - # self.assertTrue(matcher(funcdef)) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_classdef(self): - def cls(bases=None, keywords=None, body=None, decorator_list=None, type_params=None): - if bases is None: - bases = [] - if keywords is None: - keywords = [] - if body is None: - body = [ast.Pass()] - if decorator_list is None: - decorator_list = [] - if type_params is None: - type_params = [] - return ast.ClassDef("myclass", bases, keywords, - body, decorator_list, type_params) - self.stmt(cls(bases=[ast.Name("x", ast.Store())]), - "must have Load context") - self.stmt(cls(keywords=[ast.keyword("x", ast.Name("x", ast.Store()))]), - "must have Load context") - self.stmt(cls(body=[]), "empty body on ClassDef") - self.stmt(cls(body=[None]), "None disallowed") - self.stmt(cls(decorator_list=[ast.Name("x", ast.Store())]), - "must have Load context") - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_delete(self): - self.stmt(ast.Delete([]), "empty targets on Delete") - self.stmt(ast.Delete([None]), "None disallowed") - self.stmt(ast.Delete([ast.Name("x", ast.Load())]), - "must have Del context") - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_assign(self): - self.stmt(ast.Assign([], ast.Constant(3)), "empty targets on Assign") - self.stmt(ast.Assign([None], ast.Constant(3)), "None disallowed") - self.stmt(ast.Assign([ast.Name("x", ast.Load())], ast.Constant(3)), - "must have Store context") - self.stmt(ast.Assign([ast.Name("x", ast.Store())], - ast.Name("y", ast.Store())), - "must have Load context") - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_augassign(self): - aug = ast.AugAssign(ast.Name("x", ast.Load()), ast.Add(), - ast.Name("y", ast.Load())) - self.stmt(aug, "must have Store context") - aug = ast.AugAssign(ast.Name("x", ast.Store()), ast.Add(), - ast.Name("y", ast.Store())) - self.stmt(aug, "must have Load context") - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_for(self): - x = ast.Name("x", ast.Store()) - y = ast.Name("y", ast.Load()) - p = ast.Pass() - self.stmt(ast.For(x, y, [], []), "empty body on For") - self.stmt(ast.For(ast.Name("x", ast.Load()), y, [p], []), - "must have Store context") - self.stmt(ast.For(x, ast.Name("y", ast.Store()), [p], []), - "must have Load context") - e = ast.Expr(ast.Name("x", ast.Store())) - self.stmt(ast.For(x, y, [e], []), "must have Load context") - self.stmt(ast.For(x, y, [p], [e]), "must have Load context") - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_while(self): - self.stmt(ast.While(ast.Constant(3), [], []), "empty body on While") - self.stmt(ast.While(ast.Name("x", ast.Store()), [ast.Pass()], []), - "must have Load context") - self.stmt(ast.While(ast.Constant(3), [ast.Pass()], - [ast.Expr(ast.Name("x", ast.Store()))]), - "must have Load context") - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_if(self): - self.stmt(ast.If(ast.Constant(3), [], []), "empty body on If") - i = ast.If(ast.Name("x", ast.Store()), [ast.Pass()], []) - self.stmt(i, "must have Load context") - i = ast.If(ast.Constant(3), [ast.Expr(ast.Name("x", ast.Store()))], []) - self.stmt(i, "must have Load context") - i = ast.If(ast.Constant(3), [ast.Pass()], - [ast.Expr(ast.Name("x", ast.Store()))]) - self.stmt(i, "must have Load context") - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_with(self): - p = ast.Pass() - self.stmt(ast.With([], [p]), "empty items on With") - i = ast.withitem(ast.Constant(3), None) - self.stmt(ast.With([i], []), "empty body on With") - i = ast.withitem(ast.Name("x", ast.Store()), None) - self.stmt(ast.With([i], [p]), "must have Load context") - i = ast.withitem(ast.Constant(3), ast.Name("x", ast.Load())) - self.stmt(ast.With([i], [p]), "must have Store context") - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_raise(self): - r = ast.Raise(None, ast.Constant(3)) - self.stmt(r, "Raise with cause but no exception") - r = ast.Raise(ast.Name("x", ast.Store()), None) - self.stmt(r, "must have Load context") - r = ast.Raise(ast.Constant(4), ast.Name("x", ast.Store())) - self.stmt(r, "must have Load context") - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_try(self): - p = ast.Pass() - t = ast.Try([], [], [], [p]) - self.stmt(t, "empty body on Try") - t = ast.Try([ast.Expr(ast.Name("x", ast.Store()))], [], [], [p]) - self.stmt(t, "must have Load context") - t = ast.Try([p], [], [], []) - self.stmt(t, "Try has neither except handlers nor finalbody") - t = ast.Try([p], [], [p], [p]) - self.stmt(t, "Try has orelse but no except handlers") - t = ast.Try([p], [ast.ExceptHandler(None, "x", [])], [], []) - self.stmt(t, "empty body on ExceptHandler") - e = [ast.ExceptHandler(ast.Name("x", ast.Store()), "y", [p])] - self.stmt(ast.Try([p], e, [], []), "must have Load context") - e = [ast.ExceptHandler(None, "x", [p])] - t = ast.Try([p], e, [ast.Expr(ast.Name("x", ast.Store()))], [p]) - self.stmt(t, "must have Load context") - t = ast.Try([p], e, [p], [ast.Expr(ast.Name("x", ast.Store()))]) - self.stmt(t, "must have Load context") - - # TODO: RUSTPYTHON - @unittest.skip("TODO: RUSTPYTHON, SyntaxError: RustPython does not implement this feature yet") - def test_try_star(self): - p = ast.Pass() - t = ast.TryStar([], [], [], [p]) - self.stmt(t, "empty body on TryStar") - t = ast.TryStar([ast.Expr(ast.Name("x", ast.Store()))], [], [], [p]) - self.stmt(t, "must have Load context") - t = ast.TryStar([p], [], [], []) - self.stmt(t, "TryStar has neither except handlers nor finalbody") - t = ast.TryStar([p], [], [p], [p]) - self.stmt(t, "TryStar has orelse but no except handlers") - t = ast.TryStar([p], [ast.ExceptHandler(None, "x", [])], [], []) - self.stmt(t, "empty body on ExceptHandler") - e = [ast.ExceptHandler(ast.Name("x", ast.Store()), "y", [p])] - self.stmt(ast.TryStar([p], e, [], []), "must have Load context") - e = [ast.ExceptHandler(None, "x", [p])] - t = ast.TryStar([p], e, [ast.Expr(ast.Name("x", ast.Store()))], [p]) - self.stmt(t, "must have Load context") - t = ast.TryStar([p], e, [p], [ast.Expr(ast.Name("x", ast.Store()))]) - self.stmt(t, "must have Load context") - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_assert(self): - self.stmt(ast.Assert(ast.Name("x", ast.Store()), None), - "must have Load context") - assrt = ast.Assert(ast.Name("x", ast.Load()), - ast.Name("y", ast.Store())) - self.stmt(assrt, "must have Load context") - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_import(self): - self.stmt(ast.Import([]), "empty names on Import") - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_importfrom(self): - imp = ast.ImportFrom(None, [ast.alias("x", None)], -42) - self.stmt(imp, "Negative ImportFrom level") - self.stmt(ast.ImportFrom(None, [], 0), "empty names on ImportFrom") - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_global(self): - self.stmt(ast.Global([]), "empty names on Global") - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_nonlocal(self): - self.stmt(ast.Nonlocal([]), "empty names on Nonlocal") - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_expr(self): - e = ast.Expr(ast.Name("x", ast.Store())) - self.stmt(e, "must have Load context") - - @unittest.skip("TODO: RUSTPYTHON, thread 'main' panicked at 'called `Option::unwrap()` on a `None` value'") - def test_boolop(self): - b = ast.BoolOp(ast.And(), []) - self.expr(b, "less than 2 values") - b = ast.BoolOp(ast.And(), [ast.Constant(3)]) - self.expr(b, "less than 2 values") - b = ast.BoolOp(ast.And(), [ast.Constant(4), None]) - self.expr(b, "None disallowed") - b = ast.BoolOp(ast.And(), [ast.Constant(4), ast.Name("x", ast.Store())]) - self.expr(b, "must have Load context") - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_unaryop(self): - u = ast.UnaryOp(ast.Not(), ast.Name("x", ast.Store())) - self.expr(u, "must have Load context") - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_lambda(self): - a = ast.arguments([], [], None, [], [], None, []) - self.expr(ast.Lambda(a, ast.Name("x", ast.Store())), - "must have Load context") - def fac(args): - return ast.Lambda(args, ast.Name("x", ast.Load())) - self._check_arguments(fac, self.expr) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_ifexp(self): - l = ast.Name("x", ast.Load()) - s = ast.Name("y", ast.Store()) - for args in (s, l, l), (l, s, l), (l, l, s): - self.expr(ast.IfExp(*args), "must have Load context") - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_dict(self): - d = ast.Dict([], [ast.Name("x", ast.Load())]) - self.expr(d, "same number of keys as values") - d = ast.Dict([ast.Name("x", ast.Load())], [None]) - self.expr(d, "None disallowed") - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_set(self): - self.expr(ast.Set([None]), "None disallowed") - s = ast.Set([ast.Name("x", ast.Store())]) - self.expr(s, "must have Load context") - - def _check_comprehension(self, fac): - self.expr(fac([]), "comprehension with no generators") - g = ast.comprehension(ast.Name("x", ast.Load()), - ast.Name("x", ast.Load()), [], 0) - self.expr(fac([g]), "must have Store context") - g = ast.comprehension(ast.Name("x", ast.Store()), - ast.Name("x", ast.Store()), [], 0) - self.expr(fac([g]), "must have Load context") - x = ast.Name("x", ast.Store()) - y = ast.Name("y", ast.Load()) - g = ast.comprehension(x, y, [None], 0) - self.expr(fac([g]), "None disallowed") - g = ast.comprehension(x, y, [ast.Name("x", ast.Store())], 0) - self.expr(fac([g]), "must have Load context") - - def _simple_comp(self, fac): - g = ast.comprehension(ast.Name("x", ast.Store()), - ast.Name("x", ast.Load()), [], 0) - self.expr(fac(ast.Name("x", ast.Store()), [g]), - "must have Load context") - def wrap(gens): - return fac(ast.Name("x", ast.Store()), gens) - self._check_comprehension(wrap) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_listcomp(self): - self._simple_comp(ast.ListComp) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_setcomp(self): - self._simple_comp(ast.SetComp) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_generatorexp(self): - self._simple_comp(ast.GeneratorExp) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_dictcomp(self): - g = ast.comprehension(ast.Name("y", ast.Store()), - ast.Name("p", ast.Load()), [], 0) - c = ast.DictComp(ast.Name("x", ast.Store()), - ast.Name("y", ast.Load()), [g]) - self.expr(c, "must have Load context") - c = ast.DictComp(ast.Name("x", ast.Load()), - ast.Name("y", ast.Store()), [g]) - self.expr(c, "must have Load context") - def factory(comps): - k = ast.Name("x", ast.Load()) - v = ast.Name("y", ast.Load()) - return ast.DictComp(k, v, comps) - self._check_comprehension(factory) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_yield(self): - self.expr(ast.Yield(ast.Name("x", ast.Store())), "must have Load") - self.expr(ast.YieldFrom(ast.Name("x", ast.Store())), "must have Load") - - @unittest.skip("TODO: RUSTPYTHON, thread 'main' panicked at 'assertion failed: `(left == right)` left: `0`, right: `1`'") - def test_compare(self): - left = ast.Name("x", ast.Load()) - comp = ast.Compare(left, [ast.In()], []) - self.expr(comp, "no comparators") - comp = ast.Compare(left, [ast.In()], [ast.Constant(4), ast.Constant(5)]) - self.expr(comp, "different number of comparators and operands") - comp = ast.Compare(ast.Constant("blah"), [ast.In()], [left]) - self.expr(comp) - comp = ast.Compare(left, [ast.In()], [ast.Constant("blah")]) - self.expr(comp) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_call(self): - func = ast.Name("x", ast.Load()) - args = [ast.Name("y", ast.Load())] - keywords = [ast.keyword("w", ast.Name("z", ast.Load()))] - call = ast.Call(ast.Name("x", ast.Store()), args, keywords) - self.expr(call, "must have Load context") - call = ast.Call(func, [None], keywords) - self.expr(call, "None disallowed") - bad_keywords = [ast.keyword("w", ast.Name("z", ast.Store()))] - call = ast.Call(func, args, bad_keywords) - self.expr(call, "must have Load context") - - def test_num(self): - with warnings.catch_warnings(record=True) as wlog: - warnings.filterwarnings('ignore', '', DeprecationWarning) - from ast import Num - - with warnings.catch_warnings(record=True) as wlog: - warnings.filterwarnings('always', '', DeprecationWarning) - class subint(int): - pass - class subfloat(float): - pass - class subcomplex(complex): - pass - for obj in "0", "hello": - self.expr(ast.Num(obj)) - for obj in subint(), subfloat(), subcomplex(): - self.expr(ast.Num(obj), "invalid type", exc=TypeError) - - self.assertEqual([str(w.message) for w in wlog], [ - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - 'ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead', - ]) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_attribute(self): - attr = ast.Attribute(ast.Name("x", ast.Store()), "y", ast.Load()) - self.expr(attr, "must have Load context") - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_subscript(self): - sub = ast.Subscript(ast.Name("x", ast.Store()), ast.Constant(3), - ast.Load()) - self.expr(sub, "must have Load context") - x = ast.Name("x", ast.Load()) - sub = ast.Subscript(x, ast.Name("y", ast.Store()), - ast.Load()) - self.expr(sub, "must have Load context") - s = ast.Name("x", ast.Store()) - for args in (s, None, None), (None, s, None), (None, None, s): - sl = ast.Slice(*args) - self.expr(ast.Subscript(x, sl, ast.Load()), - "must have Load context") - sl = ast.Tuple([], ast.Load()) - self.expr(ast.Subscript(x, sl, ast.Load())) - sl = ast.Tuple([s], ast.Load()) - self.expr(ast.Subscript(x, sl, ast.Load()), "must have Load context") - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_starred(self): - left = ast.List([ast.Starred(ast.Name("x", ast.Load()), ast.Store())], - ast.Store()) - assign = ast.Assign([left], ast.Constant(4)) - self.stmt(assign, "must have Store context") - - def _sequence(self, fac): - self.expr(fac([None], ast.Load()), "None disallowed") - self.expr(fac([ast.Name("x", ast.Store())], ast.Load()), - "must have Load context") - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_list(self): - self._sequence(ast.List) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_tuple(self): - self._sequence(ast.Tuple) - - def test_nameconstant(self): - with warnings.catch_warnings(record=True) as wlog: - warnings.filterwarnings('ignore', '', DeprecationWarning) - from ast import NameConstant - - with warnings.catch_warnings(record=True) as wlog: - warnings.filterwarnings('always', '', DeprecationWarning) - self.expr(ast.NameConstant(4)) - - self.assertEqual([str(w.message) for w in wlog], [ - 'ast.NameConstant is deprecated and will be removed in Python 3.14; use ast.Constant instead', - ]) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - @support.requires_resource('cpu') - def test_stdlib_validates(self): - stdlib = os.path.dirname(ast.__file__) - tests = [fn for fn in os.listdir(stdlib) if fn.endswith(".py")] - tests.extend(["test/test_grammar.py", "test/test_unpack_ex.py"]) - for module in tests: - with self.subTest(module): - fn = os.path.join(stdlib, module) - with open(fn, "r", encoding="utf-8") as fp: - source = fp.read() - mod = ast.parse(source, fn) - compile(mod, fn, "exec") - - constant_1 = ast.Constant(1) - pattern_1 = ast.MatchValue(constant_1) - - constant_x = ast.Constant('x') - pattern_x = ast.MatchValue(constant_x) - - constant_true = ast.Constant(True) - pattern_true = ast.MatchSingleton(True) - - name_carter = ast.Name('carter', ast.Load()) - - _MATCH_PATTERNS = [ - ast.MatchValue( - ast.Attribute( - ast.Attribute( - ast.Name('x', ast.Store()), - 'y', ast.Load() - ), - 'z', ast.Load() - ) - ), - ast.MatchValue( - ast.Attribute( - ast.Attribute( - ast.Name('x', ast.Load()), - 'y', ast.Store() - ), - 'z', ast.Load() - ) - ), - ast.MatchValue( - ast.Constant(...) - ), - ast.MatchValue( - ast.Constant(True) - ), - ast.MatchValue( - ast.Constant((1,2,3)) - ), - ast.MatchSingleton('string'), - ast.MatchSequence([ - ast.MatchSingleton('string') - ]), - ast.MatchSequence( - [ - ast.MatchSequence( - [ - ast.MatchSingleton('string') - ] - ) - ] - ), - ast.MatchMapping( - [constant_1, constant_true], - [pattern_x] - ), - ast.MatchMapping( - [constant_true, constant_1], - [pattern_x, pattern_1], - rest='True' - ), - ast.MatchMapping( - [constant_true, ast.Starred(ast.Name('lol', ast.Load()), ast.Load())], - [pattern_x, pattern_1], - rest='legit' - ), - ast.MatchClass( - ast.Attribute( - ast.Attribute( - constant_x, - 'y', ast.Load()), - 'z', ast.Load()), - patterns=[], kwd_attrs=[], kwd_patterns=[] - ), - ast.MatchClass( - name_carter, - patterns=[], - kwd_attrs=['True'], - kwd_patterns=[pattern_1] - ), - ast.MatchClass( - name_carter, - patterns=[], - kwd_attrs=[], - kwd_patterns=[pattern_1] - ), - ast.MatchClass( - name_carter, - patterns=[ast.MatchSingleton('string')], - kwd_attrs=[], - kwd_patterns=[] - ), - ast.MatchClass( - name_carter, - patterns=[ast.MatchStar()], - kwd_attrs=[], - kwd_patterns=[] - ), - ast.MatchClass( - name_carter, - patterns=[], - kwd_attrs=[], - kwd_patterns=[ast.MatchStar()] - ), - ast.MatchClass( - constant_true, # invalid name - patterns=[], - kwd_attrs=['True'], - kwd_patterns=[pattern_1] - ), - ast.MatchSequence( - [ - ast.MatchStar("True") - ] - ), - ast.MatchAs( - name='False' - ), - ast.MatchOr( - [] - ), - ast.MatchOr( - [pattern_1] - ), - ast.MatchOr( - [pattern_1, pattern_x, ast.MatchSingleton('xxx')] - ), - ast.MatchAs(name="_"), - ast.MatchStar(name="x"), - ast.MatchSequence([ast.MatchStar("_")]), - ast.MatchMapping([], [], rest="_"), - ] - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_match_validation_pattern(self): - name_x = ast.Name('x', ast.Load()) - for pattern in self._MATCH_PATTERNS: - with self.subTest(ast.dump(pattern, indent=4)): - node = ast.Match( - subject=name_x, - cases = [ - ast.match_case( - pattern=pattern, - body = [ast.Pass()] - ) - ] - ) - node = ast.fix_missing_locations(node) - module = ast.Module([node], []) - with self.assertRaises(ValueError): - compile(module, "", "exec") - - -class ConstantTests(unittest.TestCase): - """Tests on the ast.Constant node type.""" - - def compile_constant(self, value): - tree = ast.parse("x = 123") - - node = tree.body[0].value - new_node = ast.Constant(value=value) - ast.copy_location(new_node, node) - tree.body[0].value = new_node - - code = compile(tree, "", "exec") - - ns = {} - exec(code, ns) - return ns['x'] - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_validation(self): - with self.assertRaises(TypeError) as cm: - self.compile_constant([1, 2, 3]) - self.assertEqual(str(cm.exception), - "got an invalid type in Constant: list") - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_singletons(self): - for const in (None, False, True, Ellipsis, b'', frozenset()): - with self.subTest(const=const): - value = self.compile_constant(const) - self.assertIs(value, const) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_values(self): - nested_tuple = (1,) - nested_frozenset = frozenset({1}) - for level in range(3): - nested_tuple = (nested_tuple, 2) - nested_frozenset = frozenset({nested_frozenset, 2}) - values = (123, 123.0, 123j, - "unicode", b'bytes', - tuple("tuple"), frozenset("frozenset"), - nested_tuple, nested_frozenset) - for value in values: - with self.subTest(value=value): - result = self.compile_constant(value) - self.assertEqual(result, value) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_assign_to_constant(self): - tree = ast.parse("x = 1") - - target = tree.body[0].targets[0] - new_target = ast.Constant(value=1) - ast.copy_location(new_target, target) - tree.body[0].targets[0] = new_target - - with self.assertRaises(ValueError) as cm: - compile(tree, "string", "exec") - self.assertEqual(str(cm.exception), - "expression which can't be assigned " - "to in Store context") - - def test_get_docstring(self): - tree = ast.parse("'docstring'\nx = 1") - self.assertEqual(ast.get_docstring(tree), 'docstring') - - def get_load_const(self, tree): - # Compile to bytecode, disassemble and get parameter of LOAD_CONST - # instructions - co = compile(tree, '', 'exec') - consts = [] - for instr in dis.get_instructions(co): - if instr.opname == 'LOAD_CONST' or instr.opname == 'RETURN_CONST': - consts.append(instr.argval) - return consts - - @support.cpython_only - def test_load_const(self): - consts = [None, - True, False, - 124, - 2.0, - 3j, - "unicode", - b'bytes', - (1, 2, 3)] - - code = '\n'.join(['x={!r}'.format(const) for const in consts]) - code += '\nx = ...' - consts.extend((Ellipsis, None)) - - tree = ast.parse(code) - self.assertEqual(self.get_load_const(tree), - consts) - - # Replace expression nodes with constants - for assign, const in zip(tree.body, consts): - assert isinstance(assign, ast.Assign), ast.dump(assign) - new_node = ast.Constant(value=const) - ast.copy_location(new_node, assign.value) - assign.value = new_node - - self.assertEqual(self.get_load_const(tree), - consts) - - def test_literal_eval(self): - tree = ast.parse("1 + 2") - binop = tree.body[0].value - - new_left = ast.Constant(value=10) - ast.copy_location(new_left, binop.left) - binop.left = new_left - - new_right = ast.Constant(value=20j) - ast.copy_location(new_right, binop.right) - binop.right = new_right - - self.assertEqual(ast.literal_eval(binop), 10+20j) - - def test_string_kind(self): - c = ast.parse('"x"', mode='eval').body - self.assertEqual(c.value, "x") - self.assertEqual(c.kind, None) - - c = ast.parse('u"x"', mode='eval').body - self.assertEqual(c.value, "x") - self.assertEqual(c.kind, "u") - - c = ast.parse('r"x"', mode='eval').body - self.assertEqual(c.value, "x") - self.assertEqual(c.kind, None) - - c = ast.parse('b"x"', mode='eval').body - self.assertEqual(c.value, b"x") - self.assertEqual(c.kind, None) - - -class EndPositionTests(unittest.TestCase): - """Tests for end position of AST nodes. - - Testing end positions of nodes requires a bit of extra care - because of how LL parsers work. - """ - def _check_end_pos(self, ast_node, end_lineno, end_col_offset): - self.assertEqual(ast_node.end_lineno, end_lineno) - self.assertEqual(ast_node.end_col_offset, end_col_offset) - - def _check_content(self, source, ast_node, content): - self.assertEqual(ast.get_source_segment(source, ast_node), content) - - def _parse_value(self, s): - # Use duck-typing to support both single expression - # and a right hand side of an assignment statement. - return ast.parse(s).body[0].value - - def test_lambda(self): - s = 'lambda x, *y: None' - lam = self._parse_value(s) - self._check_content(s, lam.body, 'None') - self._check_content(s, lam.args.args[0], 'x') - self._check_content(s, lam.args.vararg, 'y') - - def test_func_def(self): - s = dedent(''' - def func(x: int, - *args: str, - z: float = 0, - **kwargs: Any) -> bool: - return True - ''').strip() - fdef = ast.parse(s).body[0] - self._check_end_pos(fdef, 5, 15) - self._check_content(s, fdef.body[0], 'return True') - self._check_content(s, fdef.args.args[0], 'x: int') - self._check_content(s, fdef.args.args[0].annotation, 'int') - self._check_content(s, fdef.args.kwarg, 'kwargs: Any') - self._check_content(s, fdef.args.kwarg.annotation, 'Any') - - def test_call(self): - s = 'func(x, y=2, **kw)' - call = self._parse_value(s) - self._check_content(s, call.func, 'func') - self._check_content(s, call.keywords[0].value, '2') - self._check_content(s, call.keywords[1].value, 'kw') - - def test_call_noargs(self): - s = 'x[0]()' - call = self._parse_value(s) - self._check_content(s, call.func, 'x[0]') - self._check_end_pos(call, 1, 6) - - def test_class_def(self): - s = dedent(''' - class C(A, B): - x: int = 0 - ''').strip() - cdef = ast.parse(s).body[0] - self._check_end_pos(cdef, 2, 14) - self._check_content(s, cdef.bases[1], 'B') - self._check_content(s, cdef.body[0], 'x: int = 0') - - def test_class_kw(self): - s = 'class S(metaclass=abc.ABCMeta): pass' - cdef = ast.parse(s).body[0] - self._check_content(s, cdef.keywords[0].value, 'abc.ABCMeta') - - def test_multi_line_str(self): - s = dedent(''' - x = """Some multi-line text. - - It goes on starting from same indent.""" - ''').strip() - assign = ast.parse(s).body[0] - self._check_end_pos(assign, 3, 40) - self._check_end_pos(assign.value, 3, 40) - - def test_continued_str(self): - s = dedent(''' - x = "first part" \\ - "second part" - ''').strip() - assign = ast.parse(s).body[0] - self._check_end_pos(assign, 2, 13) - self._check_end_pos(assign.value, 2, 13) - - def test_suites(self): - # We intentionally put these into the same string to check - # that empty lines are not part of the suite. - s = dedent(''' - while True: - pass - - if one(): - x = None - elif other(): - y = None - else: - z = None - - for x, y in stuff: - assert True - - try: - raise RuntimeError - except TypeError as e: - pass - - pass - ''').strip() - mod = ast.parse(s) - while_loop = mod.body[0] - if_stmt = mod.body[1] - for_loop = mod.body[2] - try_stmt = mod.body[3] - pass_stmt = mod.body[4] - - self._check_end_pos(while_loop, 2, 8) - self._check_end_pos(if_stmt, 9, 12) - self._check_end_pos(for_loop, 12, 15) - self._check_end_pos(try_stmt, 17, 8) - self._check_end_pos(pass_stmt, 19, 4) - - self._check_content(s, while_loop.test, 'True') - self._check_content(s, if_stmt.body[0], 'x = None') - self._check_content(s, if_stmt.orelse[0].test, 'other()') - self._check_content(s, for_loop.target, 'x, y') - self._check_content(s, try_stmt.body[0], 'raise RuntimeError') - self._check_content(s, try_stmt.handlers[0].type, 'TypeError') - - def test_fstring(self): - s = 'x = f"abc {x + y} abc"' - fstr = self._parse_value(s) - binop = fstr.values[1].value - self._check_content(s, binop, 'x + y') - - def test_fstring_multi_line(self): - s = dedent(''' - f"""Some multi-line text. - { - arg_one - + - arg_two - } - It goes on...""" - ''').strip() - fstr = self._parse_value(s) - binop = fstr.values[1].value - self._check_end_pos(binop, 5, 7) - self._check_content(s, binop.left, 'arg_one') - self._check_content(s, binop.right, 'arg_two') - - def test_import_from_multi_line(self): - s = dedent(''' - from x.y.z import ( - a, b, c as c - ) - ''').strip() - imp = ast.parse(s).body[0] - self._check_end_pos(imp, 3, 1) - self._check_end_pos(imp.names[2], 2, 16) - - def test_slices(self): - s1 = 'f()[1, 2] [0]' - s2 = 'x[ a.b: c.d]' - sm = dedent(''' - x[ a.b: f () , - g () : c.d - ] - ''').strip() - i1, i2, im = map(self._parse_value, (s1, s2, sm)) - self._check_content(s1, i1.value, 'f()[1, 2]') - self._check_content(s1, i1.value.slice, '1, 2') - self._check_content(s2, i2.slice.lower, 'a.b') - self._check_content(s2, i2.slice.upper, 'c.d') - self._check_content(sm, im.slice.elts[0].upper, 'f ()') - self._check_content(sm, im.slice.elts[1].lower, 'g ()') - self._check_end_pos(im, 3, 3) - - def test_binop(self): - s = dedent(''' - (1 * 2 + (3 ) + - 4 - ) - ''').strip() - binop = self._parse_value(s) - self._check_end_pos(binop, 2, 6) - self._check_content(s, binop.right, '4') - self._check_content(s, binop.left, '1 * 2 + (3 )') - self._check_content(s, binop.left.right, '3') - - def test_boolop(self): - s = dedent(''' - if (one_condition and - (other_condition or yet_another_one)): - pass - ''').strip() - bop = ast.parse(s).body[0].test - self._check_end_pos(bop, 2, 44) - self._check_content(s, bop.values[1], - 'other_condition or yet_another_one') - - def test_tuples(self): - s1 = 'x = () ;' - s2 = 'x = 1 , ;' - s3 = 'x = (1 , 2 ) ;' - sm = dedent(''' - x = ( - a, b, - ) - ''').strip() - t1, t2, t3, tm = map(self._parse_value, (s1, s2, s3, sm)) - self._check_content(s1, t1, '()') - self._check_content(s2, t2, '1 ,') - self._check_content(s3, t3, '(1 , 2 )') - self._check_end_pos(tm, 3, 1) - - def test_attribute_spaces(self): - s = 'func(x. y .z)' - call = self._parse_value(s) - self._check_content(s, call, s) - self._check_content(s, call.args[0], 'x. y .z') - - def test_redundant_parenthesis(self): - s = '( ( ( a + b ) ) )' - v = ast.parse(s).body[0].value - self.assertEqual(type(v).__name__, 'BinOp') - self._check_content(s, v, 'a + b') - s2 = 'await ' + s - v = ast.parse(s2).body[0].value.value - self.assertEqual(type(v).__name__, 'BinOp') - self._check_content(s2, v, 'a + b') - - def test_trailers_with_redundant_parenthesis(self): - tests = ( - ('( ( ( a ) ) ) ( )', 'Call'), - ('( ( ( a ) ) ) ( b )', 'Call'), - ('( ( ( a ) ) ) [ b ]', 'Subscript'), - ('( ( ( a ) ) ) . b', 'Attribute'), - ) - for s, t in tests: - with self.subTest(s): - v = ast.parse(s).body[0].value - self.assertEqual(type(v).__name__, t) - self._check_content(s, v, s) - s2 = 'await ' + s - v = ast.parse(s2).body[0].value.value - self.assertEqual(type(v).__name__, t) - self._check_content(s2, v, s) - - def test_displays(self): - s1 = '[{}, {1, }, {1, 2,} ]' - s2 = '{a: b, f (): g () ,}' - c1 = self._parse_value(s1) - c2 = self._parse_value(s2) - self._check_content(s1, c1.elts[0], '{}') - self._check_content(s1, c1.elts[1], '{1, }') - self._check_content(s1, c1.elts[2], '{1, 2,}') - self._check_content(s2, c2.keys[1], 'f ()') - self._check_content(s2, c2.values[1], 'g ()') - - def test_comprehensions(self): - s = dedent(''' - x = [{x for x, y in stuff - if cond.x} for stuff in things] - ''').strip() - cmp = self._parse_value(s) - self._check_end_pos(cmp, 2, 37) - self._check_content(s, cmp.generators[0].iter, 'things') - self._check_content(s, cmp.elt.generators[0].iter, 'stuff') - self._check_content(s, cmp.elt.generators[0].ifs[0], 'cond.x') - self._check_content(s, cmp.elt.generators[0].target, 'x, y') - - def test_yield_await(self): - s = dedent(''' - async def f(): - yield x - await y - ''').strip() - fdef = ast.parse(s).body[0] - self._check_content(s, fdef.body[0].value, 'yield x') - self._check_content(s, fdef.body[1].value, 'await y') - - def test_source_segment_multi(self): - s_orig = dedent(''' - x = ( - a, b, - ) + () - ''').strip() - s_tuple = dedent(''' - ( - a, b, - ) - ''').strip() - binop = self._parse_value(s_orig) - self.assertEqual(ast.get_source_segment(s_orig, binop.left), s_tuple) - - def test_source_segment_padded(self): - s_orig = dedent(''' - class C: - def fun(self) -> None: - "ЖЖЖЖЖ" - ''').strip() - s_method = ' def fun(self) -> None:\n' \ - ' "ЖЖЖЖЖ"' - cdef = ast.parse(s_orig).body[0] - self.assertEqual(ast.get_source_segment(s_orig, cdef.body[0], padded=True), - s_method) - - def test_source_segment_endings(self): - s = 'v = 1\r\nw = 1\nx = 1\n\ry = 1\rz = 1\r\n' - v, w, x, y, z = ast.parse(s).body - self._check_content(s, v, 'v = 1') - self._check_content(s, w, 'w = 1') - self._check_content(s, x, 'x = 1') - self._check_content(s, y, 'y = 1') - self._check_content(s, z, 'z = 1') - - def test_source_segment_tabs(self): - s = dedent(''' - class C: - \t\f def fun(self) -> None: - \t\f pass - ''').strip() - s_method = ' \t\f def fun(self) -> None:\n' \ - ' \t\f pass' - - cdef = ast.parse(s).body[0] - self.assertEqual(ast.get_source_segment(s, cdef.body[0], padded=True), s_method) - - def test_source_segment_newlines(self): - s = 'def f():\n pass\ndef g():\r pass\r\ndef h():\r\n pass\r\n' - f, g, h = ast.parse(s).body - self._check_content(s, f, 'def f():\n pass') - self._check_content(s, g, 'def g():\r pass') - self._check_content(s, h, 'def h():\r\n pass') - - s = 'def f():\n a = 1\r b = 2\r\n c = 3\n' - f = ast.parse(s).body[0] - self._check_content(s, f, s.rstrip()) - - def test_source_segment_missing_info(self): - s = 'v = 1\r\nw = 1\nx = 1\n\ry = 1\r\n' - v, w, x, y = ast.parse(s).body - del v.lineno - del w.end_lineno - del x.col_offset - del y.end_col_offset - self.assertIsNone(ast.get_source_segment(s, v)) - self.assertIsNone(ast.get_source_segment(s, w)) - self.assertIsNone(ast.get_source_segment(s, x)) - self.assertIsNone(ast.get_source_segment(s, y)) - -class BaseNodeVisitorCases: - # Both `NodeVisitor` and `NodeTranformer` must raise these warnings: - def test_old_constant_nodes(self): - class Visitor(self.visitor_class): - def visit_Num(self, node): - log.append((node.lineno, 'Num', node.n)) - def visit_Str(self, node): - log.append((node.lineno, 'Str', node.s)) - def visit_Bytes(self, node): - log.append((node.lineno, 'Bytes', node.s)) - def visit_NameConstant(self, node): - log.append((node.lineno, 'NameConstant', node.value)) - def visit_Ellipsis(self, node): - log.append((node.lineno, 'Ellipsis', ...)) - mod = ast.parse(dedent('''\ - i = 42 - f = 4.25 - c = 4.25j - s = 'string' - b = b'bytes' - t = True - n = None - e = ... - ''')) - visitor = Visitor() - log = [] - with warnings.catch_warnings(record=True) as wlog: - warnings.filterwarnings('always', '', DeprecationWarning) - visitor.visit(mod) - self.assertEqual(log, [ - (1, 'Num', 42), - (2, 'Num', 4.25), - (3, 'Num', 4.25j), - (4, 'Str', 'string'), - (5, 'Bytes', b'bytes'), - (6, 'NameConstant', True), - (7, 'NameConstant', None), - (8, 'Ellipsis', ...), - ]) - self.assertEqual([str(w.message) for w in wlog], [ - 'visit_Num is deprecated; add visit_Constant', - 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', - 'visit_Num is deprecated; add visit_Constant', - 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', - 'visit_Num is deprecated; add visit_Constant', - 'Attribute n is deprecated and will be removed in Python 3.14; use value instead', - 'visit_Str is deprecated; add visit_Constant', - 'Attribute s is deprecated and will be removed in Python 3.14; use value instead', - 'visit_Bytes is deprecated; add visit_Constant', - 'Attribute s is deprecated and will be removed in Python 3.14; use value instead', - 'visit_NameConstant is deprecated; add visit_Constant', - 'visit_NameConstant is deprecated; add visit_Constant', - 'visit_Ellipsis is deprecated; add visit_Constant', - ]) - - -class NodeVisitorTests(BaseNodeVisitorCases, unittest.TestCase): - visitor_class = ast.NodeVisitor - - -class NodeTransformerTests(ASTTestMixin, BaseNodeVisitorCases, unittest.TestCase): - visitor_class = ast.NodeTransformer - - def assertASTTransformation(self, tranformer_class, - initial_code, expected_code): - initial_ast = ast.parse(dedent(initial_code)) - expected_ast = ast.parse(dedent(expected_code)) - - tranformer = tranformer_class() - result_ast = ast.fix_missing_locations(tranformer.visit(initial_ast)) - - self.assertASTEqual(result_ast, expected_ast) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_node_remove_single(self): - code = 'def func(arg) -> SomeType: ...' - expected = 'def func(arg): ...' - - # Since `FunctionDef.returns` is defined as a single value, we test - # the `if isinstance(old_value, AST):` branch here. - class SomeTypeRemover(ast.NodeTransformer): - def visit_Name(self, node: ast.Name): - self.generic_visit(node) - if node.id == 'SomeType': - return None - return node - - self.assertASTTransformation(SomeTypeRemover, code, expected) - - def test_node_remove_from_list(self): - code = """ - def func(arg): - print(arg) - yield arg - """ - expected = """ - def func(arg): - print(arg) - """ - - # Since `FunctionDef.body` is defined as a list, we test - # the `if isinstance(old_value, list):` branch here. - class YieldRemover(ast.NodeTransformer): - def visit_Expr(self, node: ast.Expr): - self.generic_visit(node) - if isinstance(node.value, ast.Yield): - return None # Remove `yield` from a function - return node - - self.assertASTTransformation(YieldRemover, code, expected) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_node_return_list(self): - code = """ - class DSL(Base, kw1=True): ... - """ - expected = """ - class DSL(Base, kw1=True, kw2=True, kw3=False): ... - """ - - class ExtendKeywords(ast.NodeTransformer): - def visit_keyword(self, node: ast.keyword): - self.generic_visit(node) - if node.arg == 'kw1': - return [ - node, - ast.keyword('kw2', ast.Constant(True)), - ast.keyword('kw3', ast.Constant(False)), - ] - return node - - self.assertASTTransformation(ExtendKeywords, code, expected) - - def test_node_mutate(self): - code = """ - def func(arg): - print(arg) - """ - expected = """ - def func(arg): - log(arg) - """ - - class PrintToLog(ast.NodeTransformer): - def visit_Call(self, node: ast.Call): - self.generic_visit(node) - if isinstance(node.func, ast.Name) and node.func.id == 'print': - node.func.id = 'log' - return node - - self.assertASTTransformation(PrintToLog, code, expected) - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_node_replace(self): - code = """ - def func(arg): - print(arg) - """ - expected = """ - def func(arg): - logger.log(arg, debug=True) - """ - - class PrintToLog(ast.NodeTransformer): - def visit_Call(self, node: ast.Call): - self.generic_visit(node) - if isinstance(node.func, ast.Name) and node.func.id == 'print': - return ast.Call( - func=ast.Attribute( - ast.Name('logger', ctx=ast.Load()), - attr='log', - ctx=ast.Load(), - ), - args=node.args, - keywords=[ast.keyword('debug', ast.Constant(True))], - ) - return node - - self.assertASTTransformation(PrintToLog, code, expected) - - -@support.cpython_only -class ModuleStateTests(unittest.TestCase): - # bpo-41194, bpo-41261, bpo-41631: The _ast module uses a global state. - - def check_ast_module(self): - # Check that the _ast module still works as expected - code = 'x + 1' - filename = '' - mode = 'eval' - - # Create _ast.AST subclasses instances - ast_tree = compile(code, filename, mode, flags=ast.PyCF_ONLY_AST) - - # Call PyAST_Check() - code = compile(ast_tree, filename, mode) - self.assertIsInstance(code, types.CodeType) - - def test_reload_module(self): - # bpo-41194: Importing the _ast module twice must not crash. - with support.swap_item(sys.modules, '_ast', None): - del sys.modules['_ast'] - import _ast as ast1 - - del sys.modules['_ast'] - import _ast as ast2 - - self.check_ast_module() - - # Unloading the two _ast module instances must not crash. - del ast1 - del ast2 - support.gc_collect() - - self.check_ast_module() - - def test_sys_modules(self): - # bpo-41631: Test reproducing a Mercurial crash when PyAST_Check() - # imported the _ast module internally. - lazy_mod = object() - - def my_import(name, *args, **kw): - sys.modules[name] = lazy_mod - return lazy_mod - - with support.swap_item(sys.modules, '_ast', None): - del sys.modules['_ast'] - - with support.swap_attr(builtins, '__import__', my_import): - # Test that compile() does not import the _ast module - self.check_ast_module() - self.assertNotIn('_ast', sys.modules) - - # Sanity check of the test itself - import _ast - self.assertIs(_ast, lazy_mod) - - def test_subinterpreter(self): - # bpo-41631: Importing and using the _ast module in a subinterpreter - # must not crash. - code = dedent(''' - import _ast - import ast - import gc - import sys - import types - - # Create _ast.AST subclasses instances and call PyAST_Check() - ast_tree = compile('x+1', '', 'eval', - flags=ast.PyCF_ONLY_AST) - code = compile(ast_tree, 'string', 'eval') - if not isinstance(code, types.CodeType): - raise AssertionError - - # Unloading the _ast module must not crash. - del ast, _ast - del sys.modules['ast'], sys.modules['_ast'] - gc.collect() - ''') - res = support.run_in_subinterp(code) - self.assertEqual(res, 0) - - -class ASTMainTests(unittest.TestCase): - # Tests `ast.main()` function. - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_cli_file_input(self): - code = "print(1, 2, 3)" - expected = ast.dump(ast.parse(code), indent=3) - - with os_helper.temp_dir() as tmp_dir: - filename = os.path.join(tmp_dir, "test_module.py") - with open(filename, 'w', encoding='utf-8') as f: - f.write(code) - res, _ = script_helper.run_python_until_end("-m", "ast", filename) - - self.assertEqual(res.err, b"") - self.assertEqual(expected.splitlines(), - res.out.decode("utf8").splitlines()) - self.assertEqual(res.rc, 0) - - -def main(): - if __name__ != '__main__': - return - if sys.argv[1:] == ['-g']: - for statements, kind in ((exec_tests, "exec"), (single_tests, "single"), - (eval_tests, "eval")): - print(kind+"_results = [") - for statement in statements: - tree = ast.parse(statement, "?", kind) - print("%r," % (to_tuple(tree),)) - print("]") - print("main()") - raise SystemExit - unittest.main() - -#### EVERYTHING BELOW IS GENERATED BY python Lib/test/test_ast.py -g ##### -exec_results = [ -('Module', [('Expr', (1, 0, 1, 4), ('Constant', (1, 0, 1, 4), None, None))], []), -('Module', [('Expr', (1, 0, 1, 18), ('Constant', (1, 0, 1, 18), 'module docstring', None))], []), -('Module', [('FunctionDef', (1, 0, 1, 13), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 9, 1, 13))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 29), 'f', ('arguments', [], [], None, [], [], None, []), [('Expr', (1, 9, 1, 29), ('Constant', (1, 9, 1, 29), 'function docstring', None))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 14), 'f', ('arguments', [], [('arg', (1, 6, 1, 7), 'a', None, None)], None, [], [], None, []), [('Pass', (1, 10, 1, 14))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 16), 'f', ('arguments', [], [('arg', (1, 6, 1, 7), 'a', None, None)], None, [], [], None, [('Constant', (1, 8, 1, 9), 0, None)]), [('Pass', (1, 12, 1, 16))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 18), 'f', ('arguments', [], [], ('arg', (1, 7, 1, 11), 'args', None, None), [], [], None, []), [('Pass', (1, 14, 1, 18))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 23), 'f', ('arguments', [], [], ('arg', (1, 7, 1, 16), 'args', ('Starred', (1, 13, 1, 16), ('Name', (1, 14, 1, 16), 'Ts', ('Load',)), ('Load',)), None), [], [], None, []), [('Pass', (1, 19, 1, 23))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 36), 'f', ('arguments', [], [], ('arg', (1, 7, 1, 29), 'args', ('Starred', (1, 13, 1, 29), ('Subscript', (1, 14, 1, 29), ('Name', (1, 14, 1, 19), 'tuple', ('Load',)), ('Tuple', (1, 20, 1, 28), [('Name', (1, 20, 1, 23), 'int', ('Load',)), ('Constant', (1, 25, 1, 28), Ellipsis, None)], ('Load',)), ('Load',)), ('Load',)), None), [], [], None, []), [('Pass', (1, 32, 1, 36))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 36), 'f', ('arguments', [], [], ('arg', (1, 7, 1, 29), 'args', ('Starred', (1, 13, 1, 29), ('Subscript', (1, 14, 1, 29), ('Name', (1, 14, 1, 19), 'tuple', ('Load',)), ('Tuple', (1, 20, 1, 28), [('Name', (1, 20, 1, 23), 'int', ('Load',)), ('Starred', (1, 25, 1, 28), ('Name', (1, 26, 1, 28), 'Ts', ('Load',)), ('Load',))], ('Load',)), ('Load',)), ('Load',)), None), [], [], None, []), [('Pass', (1, 32, 1, 36))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 21), 'f', ('arguments', [], [], None, [], [], ('arg', (1, 8, 1, 14), 'kwargs', None, None), []), [('Pass', (1, 17, 1, 21))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 71), 'f', ('arguments', [], [('arg', (1, 6, 1, 7), 'a', None, None), ('arg', (1, 9, 1, 10), 'b', None, None), ('arg', (1, 14, 1, 15), 'c', None, None), ('arg', (1, 22, 1, 23), 'd', None, None), ('arg', (1, 28, 1, 29), 'e', None, None)], ('arg', (1, 35, 1, 39), 'args', None, None), [('arg', (1, 41, 1, 42), 'f', None, None)], [('Constant', (1, 43, 1, 45), 42, None)], ('arg', (1, 49, 1, 55), 'kwargs', None, None), [('Constant', (1, 11, 1, 12), 1, None), ('Constant', (1, 16, 1, 20), None, None), ('List', (1, 24, 1, 26), [], ('Load',)), ('Dict', (1, 30, 1, 32), [], [])]), [('Expr', (1, 58, 1, 71), ('Constant', (1, 58, 1, 71), 'doc for f()', None))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 27), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 23, 1, 27))], [], ('Subscript', (1, 11, 1, 21), ('Name', (1, 11, 1, 16), 'tuple', ('Load',)), ('Tuple', (1, 17, 1, 20), [('Starred', (1, 17, 1, 20), ('Name', (1, 18, 1, 20), 'Ts', ('Load',)), ('Load',))], ('Load',)), ('Load',)), None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 32), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 28, 1, 32))], [], ('Subscript', (1, 11, 1, 26), ('Name', (1, 11, 1, 16), 'tuple', ('Load',)), ('Tuple', (1, 17, 1, 25), [('Name', (1, 17, 1, 20), 'int', ('Load',)), ('Starred', (1, 22, 1, 25), ('Name', (1, 23, 1, 25), 'Ts', ('Load',)), ('Load',))], ('Load',)), ('Load',)), None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 45), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 41, 1, 45))], [], ('Subscript', (1, 11, 1, 39), ('Name', (1, 11, 1, 16), 'tuple', ('Load',)), ('Tuple', (1, 17, 1, 38), [('Name', (1, 17, 1, 20), 'int', ('Load',)), ('Starred', (1, 22, 1, 38), ('Subscript', (1, 23, 1, 38), ('Name', (1, 23, 1, 28), 'tuple', ('Load',)), ('Tuple', (1, 29, 1, 37), [('Name', (1, 29, 1, 32), 'int', ('Load',)), ('Constant', (1, 34, 1, 37), Ellipsis, None)], ('Load',)), ('Load',)), ('Load',))], ('Load',)), ('Load',)), None, [])], []), -('Module', [('ClassDef', (1, 0, 1, 12), 'C', [], [], [('Pass', (1, 8, 1, 12))], [], [])], []), -('Module', [('ClassDef', (1, 0, 1, 32), 'C', [], [], [('Expr', (1, 9, 1, 32), ('Constant', (1, 9, 1, 32), 'docstring for class C', None))], [], [])], []), -('Module', [('ClassDef', (1, 0, 1, 21), 'C', [('Name', (1, 8, 1, 14), 'object', ('Load',))], [], [('Pass', (1, 17, 1, 21))], [], [])], []), -('Module', [('FunctionDef', (1, 0, 1, 16), 'f', ('arguments', [], [], None, [], [], None, []), [('Return', (1, 8, 1, 16), ('Constant', (1, 15, 1, 16), 1, None))], [], None, None, [])], []), -('Module', [('Delete', (1, 0, 1, 5), [('Name', (1, 4, 1, 5), 'v', ('Del',))])], []), -('Module', [('Assign', (1, 0, 1, 5), [('Name', (1, 0, 1, 1), 'v', ('Store',))], ('Constant', (1, 4, 1, 5), 1, None), None)], []), -('Module', [('Assign', (1, 0, 1, 7), [('Tuple', (1, 0, 1, 3), [('Name', (1, 0, 1, 1), 'a', ('Store',)), ('Name', (1, 2, 1, 3), 'b', ('Store',))], ('Store',))], ('Name', (1, 6, 1, 7), 'c', ('Load',)), None)], []), -('Module', [('Assign', (1, 0, 1, 9), [('Tuple', (1, 0, 1, 5), [('Name', (1, 1, 1, 2), 'a', ('Store',)), ('Name', (1, 3, 1, 4), 'b', ('Store',))], ('Store',))], ('Name', (1, 8, 1, 9), 'c', ('Load',)), None)], []), -('Module', [('Assign', (1, 0, 1, 9), [('List', (1, 0, 1, 5), [('Name', (1, 1, 1, 2), 'a', ('Store',)), ('Name', (1, 3, 1, 4), 'b', ('Store',))], ('Store',))], ('Name', (1, 8, 1, 9), 'c', ('Load',)), None)], []), -('Module', [('AnnAssign', (1, 0, 1, 13), ('Name', (1, 0, 1, 1), 'x', ('Store',)), ('Subscript', (1, 3, 1, 13), ('Name', (1, 3, 1, 8), 'tuple', ('Load',)), ('Tuple', (1, 9, 1, 12), [('Starred', (1, 9, 1, 12), ('Name', (1, 10, 1, 12), 'Ts', ('Load',)), ('Load',))], ('Load',)), ('Load',)), None, 1)], []), -('Module', [('AnnAssign', (1, 0, 1, 18), ('Name', (1, 0, 1, 1), 'x', ('Store',)), ('Subscript', (1, 3, 1, 18), ('Name', (1, 3, 1, 8), 'tuple', ('Load',)), ('Tuple', (1, 9, 1, 17), [('Name', (1, 9, 1, 12), 'int', ('Load',)), ('Starred', (1, 14, 1, 17), ('Name', (1, 15, 1, 17), 'Ts', ('Load',)), ('Load',))], ('Load',)), ('Load',)), None, 1)], []), -('Module', [('AnnAssign', (1, 0, 1, 31), ('Name', (1, 0, 1, 1), 'x', ('Store',)), ('Subscript', (1, 3, 1, 31), ('Name', (1, 3, 1, 8), 'tuple', ('Load',)), ('Tuple', (1, 9, 1, 30), [('Name', (1, 9, 1, 12), 'int', ('Load',)), ('Starred', (1, 14, 1, 30), ('Subscript', (1, 15, 1, 30), ('Name', (1, 15, 1, 20), 'tuple', ('Load',)), ('Tuple', (1, 21, 1, 29), [('Name', (1, 21, 1, 24), 'str', ('Load',)), ('Constant', (1, 26, 1, 29), Ellipsis, None)], ('Load',)), ('Load',)), ('Load',))], ('Load',)), ('Load',)), None, 1)], []), -('Module', [('AugAssign', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('Add',), ('Constant', (1, 5, 1, 6), 1, None))], []), -('Module', [('For', (1, 0, 1, 15), ('Name', (1, 4, 1, 5), 'v', ('Store',)), ('Name', (1, 9, 1, 10), 'v', ('Load',)), [('Pass', (1, 11, 1, 15))], [], None)], []), -('Module', [('While', (1, 0, 1, 12), ('Name', (1, 6, 1, 7), 'v', ('Load',)), [('Pass', (1, 8, 1, 12))], [])], []), -('Module', [('If', (1, 0, 1, 9), ('Name', (1, 3, 1, 4), 'v', ('Load',)), [('Pass', (1, 5, 1, 9))], [])], []), -('Module', [('If', (1, 0, 4, 6), ('Name', (1, 3, 1, 4), 'a', ('Load',)), [('Pass', (2, 2, 2, 6))], [('If', (3, 0, 4, 6), ('Name', (3, 5, 3, 6), 'b', ('Load',)), [('Pass', (4, 2, 4, 6))], [])])], []), -('Module', [('If', (1, 0, 6, 6), ('Name', (1, 3, 1, 4), 'a', ('Load',)), [('Pass', (2, 2, 2, 6))], [('If', (3, 0, 6, 6), ('Name', (3, 5, 3, 6), 'b', ('Load',)), [('Pass', (4, 2, 4, 6))], [('Pass', (6, 2, 6, 6))])])], []), -('Module', [('With', (1, 0, 1, 17), [('withitem', ('Name', (1, 5, 1, 6), 'x', ('Load',)), ('Name', (1, 10, 1, 11), 'y', ('Store',)))], [('Pass', (1, 13, 1, 17))], None)], []), -('Module', [('With', (1, 0, 1, 25), [('withitem', ('Name', (1, 5, 1, 6), 'x', ('Load',)), ('Name', (1, 10, 1, 11), 'y', ('Store',))), ('withitem', ('Name', (1, 13, 1, 14), 'z', ('Load',)), ('Name', (1, 18, 1, 19), 'q', ('Store',)))], [('Pass', (1, 21, 1, 25))], None)], []), -('Module', [('Raise', (1, 0, 1, 25), ('Call', (1, 6, 1, 25), ('Name', (1, 6, 1, 15), 'Exception', ('Load',)), [('Constant', (1, 16, 1, 24), 'string', None)], []), None)], []), -('Module', [('Try', (1, 0, 4, 6), [('Pass', (2, 2, 2, 6))], [('ExceptHandler', (3, 0, 4, 6), ('Name', (3, 7, 3, 16), 'Exception', ('Load',)), None, [('Pass', (4, 2, 4, 6))])], [], [])], []), -('Module', [('Try', (1, 0, 4, 6), [('Pass', (2, 2, 2, 6))], [], [], [('Pass', (4, 2, 4, 6))])], []), -('Module', [('TryStar', (1, 0, 4, 6), [('Pass', (2, 2, 2, 6))], [('ExceptHandler', (3, 0, 4, 6), ('Name', (3, 8, 3, 17), 'Exception', ('Load',)), None, [('Pass', (4, 2, 4, 6))])], [], [])], []), -('Module', [('Assert', (1, 0, 1, 8), ('Name', (1, 7, 1, 8), 'v', ('Load',)), None)], []), -('Module', [('Import', (1, 0, 1, 10), [('alias', (1, 7, 1, 10), 'sys', None)])], []), -('Module', [('ImportFrom', (1, 0, 1, 17), 'sys', [('alias', (1, 16, 1, 17), 'v', None)], 0)], []), -('Module', [('Global', (1, 0, 1, 8), ['v'])], []), -('Module', [('Expr', (1, 0, 1, 1), ('Constant', (1, 0, 1, 1), 1, None))], []), -('Module', [('Pass', (1, 0, 1, 4))], []), -('Module', [('For', (1, 0, 1, 16), ('Name', (1, 4, 1, 5), 'v', ('Store',)), ('Name', (1, 9, 1, 10), 'v', ('Load',)), [('Break', (1, 11, 1, 16))], [], None)], []), -('Module', [('For', (1, 0, 1, 19), ('Name', (1, 4, 1, 5), 'v', ('Store',)), ('Name', (1, 9, 1, 10), 'v', ('Load',)), [('Continue', (1, 11, 1, 19))], [], None)], []), -('Module', [('For', (1, 0, 1, 18), ('Tuple', (1, 4, 1, 7), [('Name', (1, 4, 1, 5), 'a', ('Store',)), ('Name', (1, 6, 1, 7), 'b', ('Store',))], ('Store',)), ('Name', (1, 11, 1, 12), 'c', ('Load',)), [('Pass', (1, 14, 1, 18))], [], None)], []), -('Module', [('For', (1, 0, 1, 20), ('Tuple', (1, 4, 1, 9), [('Name', (1, 5, 1, 6), 'a', ('Store',)), ('Name', (1, 7, 1, 8), 'b', ('Store',))], ('Store',)), ('Name', (1, 13, 1, 14), 'c', ('Load',)), [('Pass', (1, 16, 1, 20))], [], None)], []), -('Module', [('For', (1, 0, 1, 20), ('List', (1, 4, 1, 9), [('Name', (1, 5, 1, 6), 'a', ('Store',)), ('Name', (1, 7, 1, 8), 'b', ('Store',))], ('Store',)), ('Name', (1, 13, 1, 14), 'c', ('Load',)), [('Pass', (1, 16, 1, 20))], [], None)], []), -('Module', [('Expr', (1, 0, 11, 5), ('GeneratorExp', (1, 0, 11, 5), ('Tuple', (2, 4, 6, 5), [('Name', (3, 4, 3, 6), 'Aa', ('Load',)), ('Name', (5, 7, 5, 9), 'Bb', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (8, 4, 10, 6), [('Name', (8, 4, 8, 6), 'Aa', ('Store',)), ('Name', (10, 4, 10, 6), 'Bb', ('Store',))], ('Store',)), ('Name', (10, 10, 10, 12), 'Cc', ('Load',)), [], 0)]))], []), -('Module', [('Expr', (1, 0, 1, 34), ('DictComp', (1, 0, 1, 34), ('Name', (1, 1, 1, 2), 'a', ('Load',)), ('Name', (1, 5, 1, 6), 'b', ('Load',)), [('comprehension', ('Name', (1, 11, 1, 12), 'w', ('Store',)), ('Name', (1, 16, 1, 17), 'x', ('Load',)), [], 0), ('comprehension', ('Name', (1, 22, 1, 23), 'm', ('Store',)), ('Name', (1, 27, 1, 28), 'p', ('Load',)), [('Name', (1, 32, 1, 33), 'g', ('Load',))], 0)]))], []), -('Module', [('Expr', (1, 0, 1, 20), ('DictComp', (1, 0, 1, 20), ('Name', (1, 1, 1, 2), 'a', ('Load',)), ('Name', (1, 5, 1, 6), 'b', ('Load',)), [('comprehension', ('Tuple', (1, 11, 1, 14), [('Name', (1, 11, 1, 12), 'v', ('Store',)), ('Name', (1, 13, 1, 14), 'w', ('Store',))], ('Store',)), ('Name', (1, 18, 1, 19), 'x', ('Load',)), [], 0)]))], []), -('Module', [('Expr', (1, 0, 1, 19), ('SetComp', (1, 0, 1, 19), ('Name', (1, 1, 1, 2), 'r', ('Load',)), [('comprehension', ('Name', (1, 7, 1, 8), 'l', ('Store',)), ('Name', (1, 12, 1, 13), 'x', ('Load',)), [('Name', (1, 17, 1, 18), 'g', ('Load',))], 0)]))], []), -('Module', [('Expr', (1, 0, 1, 16), ('SetComp', (1, 0, 1, 16), ('Name', (1, 1, 1, 2), 'r', ('Load',)), [('comprehension', ('Tuple', (1, 7, 1, 10), [('Name', (1, 7, 1, 8), 'l', ('Store',)), ('Name', (1, 9, 1, 10), 'm', ('Store',))], ('Store',)), ('Name', (1, 14, 1, 15), 'x', ('Load',)), [], 0)]))], []), -('Module', [('AsyncFunctionDef', (1, 0, 3, 18), 'f', ('arguments', [], [], None, [], [], None, []), [('Expr', (2, 1, 2, 17), ('Constant', (2, 1, 2, 17), 'async function', None)), ('Expr', (3, 1, 3, 18), ('Await', (3, 1, 3, 18), ('Call', (3, 7, 3, 18), ('Name', (3, 7, 3, 16), 'something', ('Load',)), [], [])))], [], None, None, [])], []), -('Module', [('AsyncFunctionDef', (1, 0, 3, 8), 'f', ('arguments', [], [], None, [], [], None, []), [('AsyncFor', (2, 1, 3, 8), ('Name', (2, 11, 2, 12), 'e', ('Store',)), ('Name', (2, 16, 2, 17), 'i', ('Load',)), [('Expr', (2, 19, 2, 20), ('Constant', (2, 19, 2, 20), 1, None))], [('Expr', (3, 7, 3, 8), ('Constant', (3, 7, 3, 8), 2, None))], None)], [], None, None, [])], []), -('Module', [('AsyncFunctionDef', (1, 0, 2, 21), 'f', ('arguments', [], [], None, [], [], None, []), [('AsyncWith', (2, 1, 2, 21), [('withitem', ('Name', (2, 12, 2, 13), 'a', ('Load',)), ('Name', (2, 17, 2, 18), 'b', ('Store',)))], [('Expr', (2, 20, 2, 21), ('Constant', (2, 20, 2, 21), 1, None))], None)], [], None, None, [])], []), -('Module', [('Expr', (1, 0, 1, 14), ('Dict', (1, 0, 1, 14), [None, ('Constant', (1, 10, 1, 11), 2, None)], [('Dict', (1, 3, 1, 8), [('Constant', (1, 4, 1, 5), 1, None)], [('Constant', (1, 6, 1, 7), 2, None)]), ('Constant', (1, 12, 1, 13), 3, None)]))], []), -('Module', [('Expr', (1, 0, 1, 12), ('Set', (1, 0, 1, 12), [('Starred', (1, 1, 1, 8), ('Set', (1, 2, 1, 8), [('Constant', (1, 3, 1, 4), 1, None), ('Constant', (1, 6, 1, 7), 2, None)]), ('Load',)), ('Constant', (1, 10, 1, 11), 3, None)]))], []), -('Module', [('AsyncFunctionDef', (1, 0, 2, 21), 'f', ('arguments', [], [], None, [], [], None, []), [('Expr', (2, 1, 2, 21), ('ListComp', (2, 1, 2, 21), ('Name', (2, 2, 2, 3), 'i', ('Load',)), [('comprehension', ('Name', (2, 14, 2, 15), 'b', ('Store',)), ('Name', (2, 19, 2, 20), 'c', ('Load',)), [], 1)]))], [], None, None, [])], []), -('Module', [('FunctionDef', (4, 0, 4, 13), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (4, 9, 4, 13))], [('Name', (1, 1, 1, 6), 'deco1', ('Load',)), ('Call', (2, 1, 2, 8), ('Name', (2, 1, 2, 6), 'deco2', ('Load',)), [], []), ('Call', (3, 1, 3, 9), ('Name', (3, 1, 3, 6), 'deco3', ('Load',)), [('Constant', (3, 7, 3, 8), 1, None)], [])], None, None, [])], []), -('Module', [('AsyncFunctionDef', (4, 0, 4, 19), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (4, 15, 4, 19))], [('Name', (1, 1, 1, 6), 'deco1', ('Load',)), ('Call', (2, 1, 2, 8), ('Name', (2, 1, 2, 6), 'deco2', ('Load',)), [], []), ('Call', (3, 1, 3, 9), ('Name', (3, 1, 3, 6), 'deco3', ('Load',)), [('Constant', (3, 7, 3, 8), 1, None)], [])], None, None, [])], []), -('Module', [('ClassDef', (4, 0, 4, 13), 'C', [], [], [('Pass', (4, 9, 4, 13))], [('Name', (1, 1, 1, 6), 'deco1', ('Load',)), ('Call', (2, 1, 2, 8), ('Name', (2, 1, 2, 6), 'deco2', ('Load',)), [], []), ('Call', (3, 1, 3, 9), ('Name', (3, 1, 3, 6), 'deco3', ('Load',)), [('Constant', (3, 7, 3, 8), 1, None)], [])], [])], []), -('Module', [('FunctionDef', (2, 0, 2, 13), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (2, 9, 2, 13))], [('Call', (1, 1, 1, 19), ('Name', (1, 1, 1, 5), 'deco', ('Load',)), [('GeneratorExp', (1, 5, 1, 19), ('Name', (1, 6, 1, 7), 'a', ('Load',)), [('comprehension', ('Name', (1, 12, 1, 13), 'a', ('Store',)), ('Name', (1, 17, 1, 18), 'b', ('Load',)), [], 0)])], [])], None, None, [])], []), -('Module', [('FunctionDef', (2, 0, 2, 13), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (2, 9, 2, 13))], [('Attribute', (1, 1, 1, 6), ('Attribute', (1, 1, 1, 4), ('Name', (1, 1, 1, 2), 'a', ('Load',)), 'b', ('Load',)), 'c', ('Load',))], None, None, [])], []), -('Module', [('Expr', (1, 0, 1, 8), ('NamedExpr', (1, 1, 1, 7), ('Name', (1, 1, 1, 2), 'a', ('Store',)), ('Constant', (1, 6, 1, 7), 1, None)))], []), -('Module', [('FunctionDef', (1, 0, 1, 18), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [], None, [], [], None, []), [('Pass', (1, 14, 1, 18))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 26), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 12, 1, 13), 'c', None, None), ('arg', (1, 15, 1, 16), 'd', None, None), ('arg', (1, 18, 1, 19), 'e', None, None)], None, [], [], None, []), [('Pass', (1, 22, 1, 26))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 29), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 12, 1, 13), 'c', None, None)], None, [('arg', (1, 18, 1, 19), 'd', None, None), ('arg', (1, 21, 1, 22), 'e', None, None)], [None, None], None, []), [('Pass', (1, 25, 1, 29))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 39), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 12, 1, 13), 'c', None, None)], None, [('arg', (1, 18, 1, 19), 'd', None, None), ('arg', (1, 21, 1, 22), 'e', None, None)], [None, None], ('arg', (1, 26, 1, 32), 'kwargs', None, None), []), [('Pass', (1, 35, 1, 39))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 20), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [], None, [], [], None, [('Constant', (1, 8, 1, 9), 1, None)]), [('Pass', (1, 16, 1, 20))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 29), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 14, 1, 15), 'b', None, None), ('arg', (1, 19, 1, 20), 'c', None, None)], None, [], [], None, [('Constant', (1, 8, 1, 9), 1, None), ('Constant', (1, 16, 1, 17), 2, None), ('Constant', (1, 21, 1, 22), 4, None)]), [('Pass', (1, 25, 1, 29))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 32), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 14, 1, 15), 'b', None, None)], None, [('arg', (1, 22, 1, 23), 'c', None, None)], [('Constant', (1, 24, 1, 25), 4, None)], None, [('Constant', (1, 8, 1, 9), 1, None), ('Constant', (1, 16, 1, 17), 2, None)]), [('Pass', (1, 28, 1, 32))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 30), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 14, 1, 15), 'b', None, None)], None, [('arg', (1, 22, 1, 23), 'c', None, None)], [None], None, [('Constant', (1, 8, 1, 9), 1, None), ('Constant', (1, 16, 1, 17), 2, None)]), [('Pass', (1, 26, 1, 30))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 42), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 14, 1, 15), 'b', None, None)], None, [('arg', (1, 22, 1, 23), 'c', None, None)], [('Constant', (1, 24, 1, 25), 4, None)], ('arg', (1, 29, 1, 35), 'kwargs', None, None), [('Constant', (1, 8, 1, 9), 1, None), ('Constant', (1, 16, 1, 17), 2, None)]), [('Pass', (1, 38, 1, 42))], [], None, None, [])], []), -('Module', [('FunctionDef', (1, 0, 1, 40), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 14, 1, 15), 'b', None, None)], None, [('arg', (1, 22, 1, 23), 'c', None, None)], [None], ('arg', (1, 27, 1, 33), 'kwargs', None, None), [('Constant', (1, 8, 1, 9), 1, None), ('Constant', (1, 16, 1, 17), 2, None)]), [('Pass', (1, 36, 1, 40))], [], None, None, [])], []), -('Module', [('TypeAlias', (1, 0, 1, 12), ('Name', (1, 5, 1, 6), 'X', ('Store',)), [], ('Name', (1, 9, 1, 12), 'int', ('Load',)))], []), -('Module', [('TypeAlias', (1, 0, 1, 15), ('Name', (1, 5, 1, 6), 'X', ('Store',)), [('TypeVar', (1, 7, 1, 8), 'T', None)], ('Name', (1, 12, 1, 15), 'int', ('Load',)))], []), -('Module', [('TypeAlias', (1, 0, 1, 32), ('Name', (1, 5, 1, 6), 'X', ('Store',)), [('TypeVar', (1, 7, 1, 8), 'T', None), ('TypeVarTuple', (1, 10, 1, 13), 'Ts'), ('ParamSpec', (1, 15, 1, 18), 'P')], ('Tuple', (1, 22, 1, 32), [('Name', (1, 23, 1, 24), 'T', ('Load',)), ('Name', (1, 26, 1, 28), 'Ts', ('Load',)), ('Name', (1, 30, 1, 31), 'P', ('Load',))], ('Load',)))], []), -('Module', [('TypeAlias', (1, 0, 1, 37), ('Name', (1, 5, 1, 6), 'X', ('Store',)), [('TypeVar', (1, 7, 1, 13), 'T', ('Name', (1, 10, 1, 13), 'int', ('Load',))), ('TypeVarTuple', (1, 15, 1, 18), 'Ts'), ('ParamSpec', (1, 20, 1, 23), 'P')], ('Tuple', (1, 27, 1, 37), [('Name', (1, 28, 1, 29), 'T', ('Load',)), ('Name', (1, 31, 1, 33), 'Ts', ('Load',)), ('Name', (1, 35, 1, 36), 'P', ('Load',))], ('Load',)))], []), -('Module', [('TypeAlias', (1, 0, 1, 44), ('Name', (1, 5, 1, 6), 'X', ('Store',)), [('TypeVar', (1, 7, 1, 20), 'T', ('Tuple', (1, 10, 1, 20), [('Name', (1, 11, 1, 14), 'int', ('Load',)), ('Name', (1, 16, 1, 19), 'str', ('Load',))], ('Load',))), ('TypeVarTuple', (1, 22, 1, 25), 'Ts'), ('ParamSpec', (1, 27, 1, 30), 'P')], ('Tuple', (1, 34, 1, 44), [('Name', (1, 35, 1, 36), 'T', ('Load',)), ('Name', (1, 38, 1, 40), 'Ts', ('Load',)), ('Name', (1, 42, 1, 43), 'P', ('Load',))], ('Load',)))], []), -('Module', [('ClassDef', (1, 0, 1, 16), 'X', [], [], [('Pass', (1, 12, 1, 16))], [], [('TypeVar', (1, 8, 1, 9), 'T', None)])], []), -('Module', [('ClassDef', (1, 0, 1, 26), 'X', [], [], [('Pass', (1, 22, 1, 26))], [], [('TypeVar', (1, 8, 1, 9), 'T', None), ('TypeVarTuple', (1, 11, 1, 14), 'Ts'), ('ParamSpec', (1, 16, 1, 19), 'P')])], []), -('Module', [('ClassDef', (1, 0, 1, 31), 'X', [], [], [('Pass', (1, 27, 1, 31))], [], [('TypeVar', (1, 8, 1, 14), 'T', ('Name', (1, 11, 1, 14), 'int', ('Load',))), ('TypeVarTuple', (1, 16, 1, 19), 'Ts'), ('ParamSpec', (1, 21, 1, 24), 'P')])], []), -('Module', [('ClassDef', (1, 0, 1, 38), 'X', [], [], [('Pass', (1, 34, 1, 38))], [], [('TypeVar', (1, 8, 1, 21), 'T', ('Tuple', (1, 11, 1, 21), [('Name', (1, 12, 1, 15), 'int', ('Load',)), ('Name', (1, 17, 1, 20), 'str', ('Load',))], ('Load',))), ('TypeVarTuple', (1, 23, 1, 26), 'Ts'), ('ParamSpec', (1, 28, 1, 31), 'P')])], []), -('Module', [('FunctionDef', (1, 0, 1, 16), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 12, 1, 16))], [], None, None, [('TypeVar', (1, 6, 1, 7), 'T', None)])], []), -('Module', [('FunctionDef', (1, 0, 1, 26), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 22, 1, 26))], [], None, None, [('TypeVar', (1, 6, 1, 7), 'T', None), ('TypeVarTuple', (1, 9, 1, 12), 'Ts'), ('ParamSpec', (1, 14, 1, 17), 'P')])], []), -('Module', [('FunctionDef', (1, 0, 1, 31), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 27, 1, 31))], [], None, None, [('TypeVar', (1, 6, 1, 12), 'T', ('Name', (1, 9, 1, 12), 'int', ('Load',))), ('TypeVarTuple', (1, 14, 1, 17), 'Ts'), ('ParamSpec', (1, 19, 1, 22), 'P')])], []), -('Module', [('FunctionDef', (1, 0, 1, 38), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 34, 1, 38))], [], None, None, [('TypeVar', (1, 6, 1, 19), 'T', ('Tuple', (1, 9, 1, 19), [('Name', (1, 10, 1, 13), 'int', ('Load',)), ('Name', (1, 15, 1, 18), 'str', ('Load',))], ('Load',))), ('TypeVarTuple', (1, 21, 1, 24), 'Ts'), ('ParamSpec', (1, 26, 1, 29), 'P')])], []), -] -single_results = [ -('Interactive', [('Expr', (1, 0, 1, 3), ('BinOp', (1, 0, 1, 3), ('Constant', (1, 0, 1, 1), 1, None), ('Add',), ('Constant', (1, 2, 1, 3), 2, None)))]), -] -eval_results = [ -('Expression', ('Constant', (1, 0, 1, 4), None, None)), -('Expression', ('BoolOp', (1, 0, 1, 7), ('And',), [('Name', (1, 0, 1, 1), 'a', ('Load',)), ('Name', (1, 6, 1, 7), 'b', ('Load',))])), -('Expression', ('BinOp', (1, 0, 1, 5), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('Add',), ('Name', (1, 4, 1, 5), 'b', ('Load',)))), -('Expression', ('UnaryOp', (1, 0, 1, 5), ('Not',), ('Name', (1, 4, 1, 5), 'v', ('Load',)))), -('Expression', ('Lambda', (1, 0, 1, 11), ('arguments', [], [], None, [], [], None, []), ('Constant', (1, 7, 1, 11), None, None))), -('Expression', ('Dict', (1, 0, 1, 7), [('Constant', (1, 2, 1, 3), 1, None)], [('Constant', (1, 4, 1, 5), 2, None)])), -('Expression', ('Dict', (1, 0, 1, 2), [], [])), -('Expression', ('Set', (1, 0, 1, 7), [('Constant', (1, 1, 1, 5), None, None)])), -('Expression', ('Dict', (1, 0, 5, 6), [('Constant', (2, 6, 2, 7), 1, None)], [('Constant', (4, 10, 4, 11), 2, None)])), -('Expression', ('ListComp', (1, 0, 1, 19), ('Name', (1, 1, 1, 2), 'a', ('Load',)), [('comprehension', ('Name', (1, 7, 1, 8), 'b', ('Store',)), ('Name', (1, 12, 1, 13), 'c', ('Load',)), [('Name', (1, 17, 1, 18), 'd', ('Load',))], 0)])), -('Expression', ('GeneratorExp', (1, 0, 1, 19), ('Name', (1, 1, 1, 2), 'a', ('Load',)), [('comprehension', ('Name', (1, 7, 1, 8), 'b', ('Store',)), ('Name', (1, 12, 1, 13), 'c', ('Load',)), [('Name', (1, 17, 1, 18), 'd', ('Load',))], 0)])), -('Expression', ('ListComp', (1, 0, 1, 20), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11, 1, 14), [('Name', (1, 11, 1, 12), 'a', ('Store',)), ('Name', (1, 13, 1, 14), 'b', ('Store',))], ('Store',)), ('Name', (1, 18, 1, 19), 'c', ('Load',)), [], 0)])), -('Expression', ('ListComp', (1, 0, 1, 22), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11, 1, 16), [('Name', (1, 12, 1, 13), 'a', ('Store',)), ('Name', (1, 14, 1, 15), 'b', ('Store',))], ('Store',)), ('Name', (1, 20, 1, 21), 'c', ('Load',)), [], 0)])), -('Expression', ('ListComp', (1, 0, 1, 22), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('List', (1, 11, 1, 16), [('Name', (1, 12, 1, 13), 'a', ('Store',)), ('Name', (1, 14, 1, 15), 'b', ('Store',))], ('Store',)), ('Name', (1, 20, 1, 21), 'c', ('Load',)), [], 0)])), -('Expression', ('SetComp', (1, 0, 1, 20), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11, 1, 14), [('Name', (1, 11, 1, 12), 'a', ('Store',)), ('Name', (1, 13, 1, 14), 'b', ('Store',))], ('Store',)), ('Name', (1, 18, 1, 19), 'c', ('Load',)), [], 0)])), -('Expression', ('SetComp', (1, 0, 1, 22), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11, 1, 16), [('Name', (1, 12, 1, 13), 'a', ('Store',)), ('Name', (1, 14, 1, 15), 'b', ('Store',))], ('Store',)), ('Name', (1, 20, 1, 21), 'c', ('Load',)), [], 0)])), -('Expression', ('SetComp', (1, 0, 1, 22), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('List', (1, 11, 1, 16), [('Name', (1, 12, 1, 13), 'a', ('Store',)), ('Name', (1, 14, 1, 15), 'b', ('Store',))], ('Store',)), ('Name', (1, 20, 1, 21), 'c', ('Load',)), [], 0)])), -('Expression', ('GeneratorExp', (1, 0, 1, 20), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11, 1, 14), [('Name', (1, 11, 1, 12), 'a', ('Store',)), ('Name', (1, 13, 1, 14), 'b', ('Store',))], ('Store',)), ('Name', (1, 18, 1, 19), 'c', ('Load',)), [], 0)])), -('Expression', ('GeneratorExp', (1, 0, 1, 22), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11, 1, 16), [('Name', (1, 12, 1, 13), 'a', ('Store',)), ('Name', (1, 14, 1, 15), 'b', ('Store',))], ('Store',)), ('Name', (1, 20, 1, 21), 'c', ('Load',)), [], 0)])), -('Expression', ('GeneratorExp', (1, 0, 1, 22), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('List', (1, 11, 1, 16), [('Name', (1, 12, 1, 13), 'a', ('Store',)), ('Name', (1, 14, 1, 15), 'b', ('Store',))], ('Store',)), ('Name', (1, 20, 1, 21), 'c', ('Load',)), [], 0)])), -('Expression', ('Compare', (1, 0, 1, 9), ('Constant', (1, 0, 1, 1), 1, None), [('Lt',), ('Lt',)], [('Constant', (1, 4, 1, 5), 2, None), ('Constant', (1, 8, 1, 9), 3, None)])), -('Expression', ('Call', (1, 0, 1, 17), ('Name', (1, 0, 1, 1), 'f', ('Load',)), [('Constant', (1, 2, 1, 3), 1, None), ('Constant', (1, 4, 1, 5), 2, None), ('Starred', (1, 10, 1, 12), ('Name', (1, 11, 1, 12), 'd', ('Load',)), ('Load',))], [('keyword', (1, 6, 1, 9), 'c', ('Constant', (1, 8, 1, 9), 3, None)), ('keyword', (1, 13, 1, 16), None, ('Name', (1, 15, 1, 16), 'e', ('Load',)))])), -('Expression', ('Call', (1, 0, 1, 10), ('Name', (1, 0, 1, 1), 'f', ('Load',)), [('Starred', (1, 2, 1, 9), ('List', (1, 3, 1, 9), [('Constant', (1, 4, 1, 5), 0, None), ('Constant', (1, 7, 1, 8), 1, None)], ('Load',)), ('Load',))], [])), -('Expression', ('Call', (1, 0, 1, 15), ('Name', (1, 0, 1, 1), 'f', ('Load',)), [('GeneratorExp', (1, 1, 1, 15), ('Name', (1, 2, 1, 3), 'a', ('Load',)), [('comprehension', ('Name', (1, 8, 1, 9), 'a', ('Store',)), ('Name', (1, 13, 1, 14), 'b', ('Load',)), [], 0)])], [])), -('Expression', ('Constant', (1, 0, 1, 2), 10, None)), -('Expression', ('Constant', (1, 0, 1, 8), 'string', None)), -('Expression', ('Attribute', (1, 0, 1, 3), ('Name', (1, 0, 1, 1), 'a', ('Load',)), 'b', ('Load',))), -('Expression', ('Subscript', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('Slice', (1, 2, 1, 5), ('Name', (1, 2, 1, 3), 'b', ('Load',)), ('Name', (1, 4, 1, 5), 'c', ('Load',)), None), ('Load',))), -('Expression', ('Name', (1, 0, 1, 1), 'v', ('Load',))), -('Expression', ('List', (1, 0, 1, 7), [('Constant', (1, 1, 1, 2), 1, None), ('Constant', (1, 3, 1, 4), 2, None), ('Constant', (1, 5, 1, 6), 3, None)], ('Load',))), -('Expression', ('List', (1, 0, 1, 2), [], ('Load',))), -('Expression', ('Tuple', (1, 0, 1, 5), [('Constant', (1, 0, 1, 1), 1, None), ('Constant', (1, 2, 1, 3), 2, None), ('Constant', (1, 4, 1, 5), 3, None)], ('Load',))), -('Expression', ('Tuple', (1, 0, 1, 7), [('Constant', (1, 1, 1, 2), 1, None), ('Constant', (1, 3, 1, 4), 2, None), ('Constant', (1, 5, 1, 6), 3, None)], ('Load',))), -('Expression', ('Tuple', (1, 0, 1, 2), [], ('Load',))), -('Expression', ('Call', (1, 0, 1, 17), ('Attribute', (1, 0, 1, 7), ('Attribute', (1, 0, 1, 5), ('Attribute', (1, 0, 1, 3), ('Name', (1, 0, 1, 1), 'a', ('Load',)), 'b', ('Load',)), 'c', ('Load',)), 'd', ('Load',)), [('Subscript', (1, 8, 1, 16), ('Attribute', (1, 8, 1, 11), ('Name', (1, 8, 1, 9), 'a', ('Load',)), 'b', ('Load',)), ('Slice', (1, 12, 1, 15), ('Constant', (1, 12, 1, 13), 1, None), ('Constant', (1, 14, 1, 15), 2, None), None), ('Load',))], [])), -] -main() diff --git a/Lib/test/test_ast/__init__.py b/Lib/test/test_ast/__init__.py new file mode 100644 index 0000000000..9a89d27ba9 --- /dev/null +++ b/Lib/test/test_ast/__init__.py @@ -0,0 +1,7 @@ +import os + +from test import support + + +def load_tests(*args): + return support.load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_ast/snippets.py b/Lib/test/test_ast/snippets.py new file mode 100644 index 0000000000..28d32b2941 --- /dev/null +++ b/Lib/test/test_ast/snippets.py @@ -0,0 +1,601 @@ +import ast +import sys + +from test.test_ast.utils import to_tuple + + +# These tests are compiled through "exec" +# There should be at least one test per statement +exec_tests = [ + # Module docstring + "'module docstring'", + # FunctionDef + "def f(): pass", + # FunctionDef with docstring + "def f(): 'function docstring'", + # FunctionDef with arg + "def f(a): pass", + # FunctionDef with arg and default value + "def f(a=0): pass", + # FunctionDef with varargs + "def f(*args): pass", + # FunctionDef with varargs as TypeVarTuple + "def f(*args: *Ts): pass", + # FunctionDef with varargs as unpacked Tuple + "def f(*args: *tuple[int, ...]): pass", + # FunctionDef with varargs as unpacked Tuple *and* TypeVarTuple + "def f(*args: *tuple[int, *Ts]): pass", + # FunctionDef with kwargs + "def f(**kwargs): pass", + # FunctionDef with all kind of args and docstring + "def f(a, b=1, c=None, d=[], e={}, *args, f=42, **kwargs): 'doc for f()'", + # FunctionDef with type annotation on return involving unpacking + "def f() -> tuple[*Ts]: pass", + "def f() -> tuple[int, *Ts]: pass", + "def f() -> tuple[int, *tuple[int, ...]]: pass", + # ClassDef + "class C:pass", + # ClassDef with docstring + "class C: 'docstring for class C'", + # ClassDef, new style class + "class C(object): pass", + # Classdef with multiple bases + "class C(A, B): pass", + # Return + "def f():return 1", + "def f():return", + # Delete + "del v", + # Assign + "v = 1", + "a,b = c", + "(a,b) = c", + "[a,b] = c", + "a[b] = c", + # AnnAssign with unpacked types + "x: tuple[*Ts]", + "x: tuple[int, *Ts]", + "x: tuple[int, *tuple[str, ...]]", + # AugAssign + "v += 1", + "v -= 1", + "v *= 1", + "v @= 1", + "v /= 1", + "v %= 1", + "v **= 1", + "v <<= 1", + "v >>= 1", + "v |= 1", + "v ^= 1", + "v &= 1", + "v //= 1", + # For + "for v in v:pass", + # For-Else + "for v in v:\n pass\nelse:\n pass", + # While + "while v:pass", + # While-Else + "while v:\n pass\nelse:\n pass", + # If-Elif-Else + "if v:pass", + "if a:\n pass\nelif b:\n pass", + "if a:\n pass\nelse:\n pass", + "if a:\n pass\nelif b:\n pass\nelse:\n pass", + "if a:\n pass\nelif b:\n pass\nelif b:\n pass\nelif b:\n pass\nelse:\n pass", + # With + "with x: pass", + "with x, y: pass", + "with x as y: pass", + "with x as y, z as q: pass", + "with (x as y): pass", + "with (x, y): pass", + # Raise + "raise", + "raise Exception('string')", + "raise Exception", + "raise Exception('string') from None", + # TryExcept + "try:\n pass\nexcept Exception:\n pass", + "try:\n pass\nexcept Exception as exc:\n pass", + # TryFinally + "try:\n pass\nfinally:\n pass", + # TryStarExcept + "try:\n pass\nexcept* Exception:\n pass", + "try:\n pass\nexcept* Exception as exc:\n pass", + # TryExceptFinallyElse + "try:\n pass\nexcept Exception:\n pass\nelse: pass\nfinally:\n pass", + "try:\n pass\nexcept Exception as exc:\n pass\nelse: pass\nfinally:\n pass", + "try:\n pass\nexcept* Exception as exc:\n pass\nelse: pass\nfinally:\n pass", + # Assert + "assert v", + # Assert with message + "assert v, 'message'", + # Import + "import sys", + "import foo as bar", + # ImportFrom + "from sys import x as y", + "from sys import v", + # Global + "global v", + # Expr + "1", + # Pass, + "pass", + # Break + "for v in v:break", + # Continue + "for v in v:continue", + # for statements with naked tuples (see http://bugs.python.org/issue6704) + "for a,b in c: pass", + "for (a,b) in c: pass", + "for [a,b] in c: pass", + # Multiline generator expression (test for .lineno & .col_offset) + """( + ( + Aa + , + Bb + ) + for + Aa + , + Bb in Cc + )""", + # dictcomp + "{a : b for w in x for m in p if g}", + # dictcomp with naked tuple + "{a : b for v,w in x}", + # setcomp + "{r for l in x if g}", + # setcomp with naked tuple + "{r for l,m in x}", + # AsyncFunctionDef + "async def f():\n 'async function'\n await something()", + # AsyncFor + "async def f():\n async for e in i: 1\n else: 2", + # AsyncWith + "async def f():\n async with a as b: 1", + # PEP 448: Additional Unpacking Generalizations + "{**{1:2}, 2:3}", + "{*{1, 2}, 3}", + # Function with yield (from) + "def f(): yield 1", + "def f(): yield from []", + # Asynchronous comprehensions + "async def f():\n [i async for b in c]", + # Decorated FunctionDef + "@deco1\n@deco2()\n@deco3(1)\ndef f(): pass", + # Decorated AsyncFunctionDef + "@deco1\n@deco2()\n@deco3(1)\nasync def f(): pass", + # Decorated ClassDef + "@deco1\n@deco2()\n@deco3(1)\nclass C: pass", + # Decorator with generator argument + "@deco(a for a in b)\ndef f(): pass", + # Decorator with attribute + "@a.b.c\ndef f(): pass", + # Simple assignment expression + "(a := 1)", + # Assignment expression in if statement + "if a := foo(): pass", + # Assignment expression in while + "while a := foo(): pass", + # Positional-only arguments + "def f(a, /,): pass", + "def f(a, /, c, d, e): pass", + "def f(a, /, c, *, d, e): pass", + "def f(a, /, c, *, d, e, **kwargs): pass", + # Positional-only arguments with defaults + "def f(a=1, /,): pass", + "def f(a=1, /, b=2, c=4): pass", + "def f(a=1, /, b=2, *, c=4): pass", + "def f(a=1, /, b=2, *, c): pass", + "def f(a=1, /, b=2, *, c=4, **kwargs): pass", + "def f(a=1, /, b=2, *, c, **kwargs): pass", + # Type aliases + "type X = int", + "type X[T] = int", + "type X[T, *Ts, **P] = (T, Ts, P)", + "type X[T: int, *Ts, **P] = (T, Ts, P)", + "type X[T: (int, str), *Ts, **P] = (T, Ts, P)", + "type X[T: int = 1, *Ts = 2, **P =3] = (T, Ts, P)", + # Generic classes + "class X[T]: pass", + "class X[T, *Ts, **P]: pass", + "class X[T: int, *Ts, **P]: pass", + "class X[T: (int, str), *Ts, **P]: pass", + "class X[T: int = 1, *Ts = 2, **P = 3]: pass", + # Generic functions + "def f[T](): pass", + "def f[T, *Ts, **P](): pass", + "def f[T: int, *Ts, **P](): pass", + "def f[T: (int, str), *Ts, **P](): pass", + "def f[T: int = 1, *Ts = 2, **P = 3](): pass", + # Match + "match x:\n\tcase 1:\n\t\tpass", + # Match with _ + "match x:\n\tcase 1:\n\t\tpass\n\tcase _:\n\t\tpass", +] + +# These are compiled through "single" +# because of overlap with "eval", it just tests what +# can't be tested with "eval" +single_tests = [ + "1+2" +] + +# These are compiled through "eval" +# It should test all expressions +eval_tests = [ + # Constant(value=None) + "None", + # True + "True", + # False + "False", + # BoolOp + "a and b", + "a or b", + # BinOp + "a + b", + "a - b", + "a * b", + "a / b", + "a @ b", + "a // b", + "a ** b", + "a % b", + "a >> b", + "a << b", + "a ^ b", + "a | b", + "a & b", + # UnaryOp + "not v", + "+v", + "-v", + "~v", + # Lambda + "lambda:None", + # Dict + "{ 1:2 }", + # Empty dict + "{}", + # Set + "{None,}", + # Multiline dict (test for .lineno & .col_offset) + """{ + 1 + : + 2 + }""", + # Multiline list + """[ + 1 + , + 1 + ]""", + # Multiline tuple + """( + 1 + , + )""", + # Multiline set + """{ + 1 + , + 1 + }""", + # ListComp + "[a for b in c if d]", + # GeneratorExp + "(a for b in c if d)", + # SetComp + "{a for b in c if d}", + # DictComp + "{k: v for k, v in c if d}", + # Comprehensions with multiple for targets + "[(a,b) for a,b in c]", + "[(a,b) for (a,b) in c]", + "[(a,b) for [a,b] in c]", + "{(a,b) for a,b in c}", + "{(a,b) for (a,b) in c}", + "{(a,b) for [a,b] in c}", + "((a,b) for a,b in c)", + "((a,b) for (a,b) in c)", + "((a,b) for [a,b] in c)", + # Async comprehensions - async comprehensions can't work outside an asynchronous function + # + # Yield - yield expressions can't work outside a function + # + # Compare + "1 < 2 < 3", + "a == b", + "a <= b", + "a >= b", + "a != b", + "a is b", + "a is not b", + "a in b", + "a not in b", + # Call without argument + "f()", + # Call + "f(1,2,c=3,*d,**e)", + # Call with multi-character starred + "f(*[0, 1])", + # Call with a generator argument + "f(a for a in b)", + # Constant(value=int()) + "10", + # Complex num + "1j", + # Constant(value=str()) + "'string'", + # Attribute + "a.b", + # Subscript + "a[b:c]", + # Name + "v", + # List + "[1,2,3]", + # Empty list + "[]", + # Tuple + "1,2,3", + # Tuple + "(1,2,3)", + # Empty tuple + "()", + # Combination + "a.b.c.d(a.b[1:2])", + # Slice + "[5][1:]", + "[5][:1]", + "[5][::1]", + "[5][1:1:1]", + # IfExp + "foo() if x else bar()", + # JoinedStr and FormattedValue + "f'{a}'", + "f'{a:.2f}'", + "f'{a!r}'", + "f'foo({a})'", +] + + +def main(): + if __name__ != '__main__': + return + if sys.argv[1:] == ['-g']: + for statements, kind in ((exec_tests, "exec"), (single_tests, "single"), + (eval_tests, "eval")): + print(kind+"_results = [") + for statement in statements: + tree = ast.parse(statement, "?", kind) + print("%r," % (to_tuple(tree),)) + print("]") + print("main()") + raise SystemExit + +#### EVERYTHING BELOW IS GENERATED BY python Lib/test/test_ast/snippets.py -g ##### +exec_results = [ +('Module', [('Expr', (1, 0, 1, 18), ('Constant', (1, 0, 1, 18), 'module docstring', None))], []), +('Module', [('FunctionDef', (1, 0, 1, 13), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 9, 1, 13))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 29), 'f', ('arguments', [], [], None, [], [], None, []), [('Expr', (1, 9, 1, 29), ('Constant', (1, 9, 1, 29), 'function docstring', None))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 14), 'f', ('arguments', [], [('arg', (1, 6, 1, 7), 'a', None, None)], None, [], [], None, []), [('Pass', (1, 10, 1, 14))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 16), 'f', ('arguments', [], [('arg', (1, 6, 1, 7), 'a', None, None)], None, [], [], None, [('Constant', (1, 8, 1, 9), 0, None)]), [('Pass', (1, 12, 1, 16))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 18), 'f', ('arguments', [], [], ('arg', (1, 7, 1, 11), 'args', None, None), [], [], None, []), [('Pass', (1, 14, 1, 18))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 23), 'f', ('arguments', [], [], ('arg', (1, 7, 1, 16), 'args', ('Starred', (1, 13, 1, 16), ('Name', (1, 14, 1, 16), 'Ts', ('Load',)), ('Load',)), None), [], [], None, []), [('Pass', (1, 19, 1, 23))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 36), 'f', ('arguments', [], [], ('arg', (1, 7, 1, 29), 'args', ('Starred', (1, 13, 1, 29), ('Subscript', (1, 14, 1, 29), ('Name', (1, 14, 1, 19), 'tuple', ('Load',)), ('Tuple', (1, 20, 1, 28), [('Name', (1, 20, 1, 23), 'int', ('Load',)), ('Constant', (1, 25, 1, 28), Ellipsis, None)], ('Load',)), ('Load',)), ('Load',)), None), [], [], None, []), [('Pass', (1, 32, 1, 36))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 36), 'f', ('arguments', [], [], ('arg', (1, 7, 1, 29), 'args', ('Starred', (1, 13, 1, 29), ('Subscript', (1, 14, 1, 29), ('Name', (1, 14, 1, 19), 'tuple', ('Load',)), ('Tuple', (1, 20, 1, 28), [('Name', (1, 20, 1, 23), 'int', ('Load',)), ('Starred', (1, 25, 1, 28), ('Name', (1, 26, 1, 28), 'Ts', ('Load',)), ('Load',))], ('Load',)), ('Load',)), ('Load',)), None), [], [], None, []), [('Pass', (1, 32, 1, 36))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 21), 'f', ('arguments', [], [], None, [], [], ('arg', (1, 8, 1, 14), 'kwargs', None, None), []), [('Pass', (1, 17, 1, 21))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 71), 'f', ('arguments', [], [('arg', (1, 6, 1, 7), 'a', None, None), ('arg', (1, 9, 1, 10), 'b', None, None), ('arg', (1, 14, 1, 15), 'c', None, None), ('arg', (1, 22, 1, 23), 'd', None, None), ('arg', (1, 28, 1, 29), 'e', None, None)], ('arg', (1, 35, 1, 39), 'args', None, None), [('arg', (1, 41, 1, 42), 'f', None, None)], [('Constant', (1, 43, 1, 45), 42, None)], ('arg', (1, 49, 1, 55), 'kwargs', None, None), [('Constant', (1, 11, 1, 12), 1, None), ('Constant', (1, 16, 1, 20), None, None), ('List', (1, 24, 1, 26), [], ('Load',)), ('Dict', (1, 30, 1, 32), [], [])]), [('Expr', (1, 58, 1, 71), ('Constant', (1, 58, 1, 71), 'doc for f()', None))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 27), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 23, 1, 27))], [], ('Subscript', (1, 11, 1, 21), ('Name', (1, 11, 1, 16), 'tuple', ('Load',)), ('Tuple', (1, 17, 1, 20), [('Starred', (1, 17, 1, 20), ('Name', (1, 18, 1, 20), 'Ts', ('Load',)), ('Load',))], ('Load',)), ('Load',)), None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 32), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 28, 1, 32))], [], ('Subscript', (1, 11, 1, 26), ('Name', (1, 11, 1, 16), 'tuple', ('Load',)), ('Tuple', (1, 17, 1, 25), [('Name', (1, 17, 1, 20), 'int', ('Load',)), ('Starred', (1, 22, 1, 25), ('Name', (1, 23, 1, 25), 'Ts', ('Load',)), ('Load',))], ('Load',)), ('Load',)), None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 45), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 41, 1, 45))], [], ('Subscript', (1, 11, 1, 39), ('Name', (1, 11, 1, 16), 'tuple', ('Load',)), ('Tuple', (1, 17, 1, 38), [('Name', (1, 17, 1, 20), 'int', ('Load',)), ('Starred', (1, 22, 1, 38), ('Subscript', (1, 23, 1, 38), ('Name', (1, 23, 1, 28), 'tuple', ('Load',)), ('Tuple', (1, 29, 1, 37), [('Name', (1, 29, 1, 32), 'int', ('Load',)), ('Constant', (1, 34, 1, 37), Ellipsis, None)], ('Load',)), ('Load',)), ('Load',))], ('Load',)), ('Load',)), None, [])], []), +('Module', [('ClassDef', (1, 0, 1, 12), 'C', [], [], [('Pass', (1, 8, 1, 12))], [], [])], []), +('Module', [('ClassDef', (1, 0, 1, 32), 'C', [], [], [('Expr', (1, 9, 1, 32), ('Constant', (1, 9, 1, 32), 'docstring for class C', None))], [], [])], []), +('Module', [('ClassDef', (1, 0, 1, 21), 'C', [('Name', (1, 8, 1, 14), 'object', ('Load',))], [], [('Pass', (1, 17, 1, 21))], [], [])], []), +('Module', [('ClassDef', (1, 0, 1, 19), 'C', [('Name', (1, 8, 1, 9), 'A', ('Load',)), ('Name', (1, 11, 1, 12), 'B', ('Load',))], [], [('Pass', (1, 15, 1, 19))], [], [])], []), +('Module', [('FunctionDef', (1, 0, 1, 16), 'f', ('arguments', [], [], None, [], [], None, []), [('Return', (1, 8, 1, 16), ('Constant', (1, 15, 1, 16), 1, None))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 14), 'f', ('arguments', [], [], None, [], [], None, []), [('Return', (1, 8, 1, 14), None)], [], None, None, [])], []), +('Module', [('Delete', (1, 0, 1, 5), [('Name', (1, 4, 1, 5), 'v', ('Del',))])], []), +('Module', [('Assign', (1, 0, 1, 5), [('Name', (1, 0, 1, 1), 'v', ('Store',))], ('Constant', (1, 4, 1, 5), 1, None), None)], []), +('Module', [('Assign', (1, 0, 1, 7), [('Tuple', (1, 0, 1, 3), [('Name', (1, 0, 1, 1), 'a', ('Store',)), ('Name', (1, 2, 1, 3), 'b', ('Store',))], ('Store',))], ('Name', (1, 6, 1, 7), 'c', ('Load',)), None)], []), +('Module', [('Assign', (1, 0, 1, 9), [('Tuple', (1, 0, 1, 5), [('Name', (1, 1, 1, 2), 'a', ('Store',)), ('Name', (1, 3, 1, 4), 'b', ('Store',))], ('Store',))], ('Name', (1, 8, 1, 9), 'c', ('Load',)), None)], []), +('Module', [('Assign', (1, 0, 1, 9), [('List', (1, 0, 1, 5), [('Name', (1, 1, 1, 2), 'a', ('Store',)), ('Name', (1, 3, 1, 4), 'b', ('Store',))], ('Store',))], ('Name', (1, 8, 1, 9), 'c', ('Load',)), None)], []), +('Module', [('Assign', (1, 0, 1, 8), [('Subscript', (1, 0, 1, 4), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('Name', (1, 2, 1, 3), 'b', ('Load',)), ('Store',))], ('Name', (1, 7, 1, 8), 'c', ('Load',)), None)], []), +('Module', [('AnnAssign', (1, 0, 1, 13), ('Name', (1, 0, 1, 1), 'x', ('Store',)), ('Subscript', (1, 3, 1, 13), ('Name', (1, 3, 1, 8), 'tuple', ('Load',)), ('Tuple', (1, 9, 1, 12), [('Starred', (1, 9, 1, 12), ('Name', (1, 10, 1, 12), 'Ts', ('Load',)), ('Load',))], ('Load',)), ('Load',)), None, 1)], []), +('Module', [('AnnAssign', (1, 0, 1, 18), ('Name', (1, 0, 1, 1), 'x', ('Store',)), ('Subscript', (1, 3, 1, 18), ('Name', (1, 3, 1, 8), 'tuple', ('Load',)), ('Tuple', (1, 9, 1, 17), [('Name', (1, 9, 1, 12), 'int', ('Load',)), ('Starred', (1, 14, 1, 17), ('Name', (1, 15, 1, 17), 'Ts', ('Load',)), ('Load',))], ('Load',)), ('Load',)), None, 1)], []), +('Module', [('AnnAssign', (1, 0, 1, 31), ('Name', (1, 0, 1, 1), 'x', ('Store',)), ('Subscript', (1, 3, 1, 31), ('Name', (1, 3, 1, 8), 'tuple', ('Load',)), ('Tuple', (1, 9, 1, 30), [('Name', (1, 9, 1, 12), 'int', ('Load',)), ('Starred', (1, 14, 1, 30), ('Subscript', (1, 15, 1, 30), ('Name', (1, 15, 1, 20), 'tuple', ('Load',)), ('Tuple', (1, 21, 1, 29), [('Name', (1, 21, 1, 24), 'str', ('Load',)), ('Constant', (1, 26, 1, 29), Ellipsis, None)], ('Load',)), ('Load',)), ('Load',))], ('Load',)), ('Load',)), None, 1)], []), +('Module', [('AugAssign', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('Add',), ('Constant', (1, 5, 1, 6), 1, None))], []), +('Module', [('AugAssign', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('Sub',), ('Constant', (1, 5, 1, 6), 1, None))], []), +('Module', [('AugAssign', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('Mult',), ('Constant', (1, 5, 1, 6), 1, None))], []), +('Module', [('AugAssign', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('MatMult',), ('Constant', (1, 5, 1, 6), 1, None))], []), +('Module', [('AugAssign', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('Div',), ('Constant', (1, 5, 1, 6), 1, None))], []), +('Module', [('AugAssign', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('Mod',), ('Constant', (1, 5, 1, 6), 1, None))], []), +('Module', [('AugAssign', (1, 0, 1, 7), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('Pow',), ('Constant', (1, 6, 1, 7), 1, None))], []), +('Module', [('AugAssign', (1, 0, 1, 7), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('LShift',), ('Constant', (1, 6, 1, 7), 1, None))], []), +('Module', [('AugAssign', (1, 0, 1, 7), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('RShift',), ('Constant', (1, 6, 1, 7), 1, None))], []), +('Module', [('AugAssign', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('BitOr',), ('Constant', (1, 5, 1, 6), 1, None))], []), +('Module', [('AugAssign', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('BitXor',), ('Constant', (1, 5, 1, 6), 1, None))], []), +('Module', [('AugAssign', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('BitAnd',), ('Constant', (1, 5, 1, 6), 1, None))], []), +('Module', [('AugAssign', (1, 0, 1, 7), ('Name', (1, 0, 1, 1), 'v', ('Store',)), ('FloorDiv',), ('Constant', (1, 6, 1, 7), 1, None))], []), +('Module', [('For', (1, 0, 1, 15), ('Name', (1, 4, 1, 5), 'v', ('Store',)), ('Name', (1, 9, 1, 10), 'v', ('Load',)), [('Pass', (1, 11, 1, 15))], [], None)], []), +('Module', [('For', (1, 0, 4, 6), ('Name', (1, 4, 1, 5), 'v', ('Store',)), ('Name', (1, 9, 1, 10), 'v', ('Load',)), [('Pass', (2, 2, 2, 6))], [('Pass', (4, 2, 4, 6))], None)], []), +('Module', [('While', (1, 0, 1, 12), ('Name', (1, 6, 1, 7), 'v', ('Load',)), [('Pass', (1, 8, 1, 12))], [])], []), +('Module', [('While', (1, 0, 4, 6), ('Name', (1, 6, 1, 7), 'v', ('Load',)), [('Pass', (2, 2, 2, 6))], [('Pass', (4, 2, 4, 6))])], []), +('Module', [('If', (1, 0, 1, 9), ('Name', (1, 3, 1, 4), 'v', ('Load',)), [('Pass', (1, 5, 1, 9))], [])], []), +('Module', [('If', (1, 0, 4, 6), ('Name', (1, 3, 1, 4), 'a', ('Load',)), [('Pass', (2, 2, 2, 6))], [('If', (3, 0, 4, 6), ('Name', (3, 5, 3, 6), 'b', ('Load',)), [('Pass', (4, 2, 4, 6))], [])])], []), +('Module', [('If', (1, 0, 4, 6), ('Name', (1, 3, 1, 4), 'a', ('Load',)), [('Pass', (2, 2, 2, 6))], [('Pass', (4, 2, 4, 6))])], []), +('Module', [('If', (1, 0, 6, 6), ('Name', (1, 3, 1, 4), 'a', ('Load',)), [('Pass', (2, 2, 2, 6))], [('If', (3, 0, 6, 6), ('Name', (3, 5, 3, 6), 'b', ('Load',)), [('Pass', (4, 2, 4, 6))], [('Pass', (6, 2, 6, 6))])])], []), +('Module', [('If', (1, 0, 10, 6), ('Name', (1, 3, 1, 4), 'a', ('Load',)), [('Pass', (2, 2, 2, 6))], [('If', (3, 0, 10, 6), ('Name', (3, 5, 3, 6), 'b', ('Load',)), [('Pass', (4, 2, 4, 6))], [('If', (5, 0, 10, 6), ('Name', (5, 5, 5, 6), 'b', ('Load',)), [('Pass', (6, 2, 6, 6))], [('If', (7, 0, 10, 6), ('Name', (7, 5, 7, 6), 'b', ('Load',)), [('Pass', (8, 2, 8, 6))], [('Pass', (10, 2, 10, 6))])])])])], []), +('Module', [('With', (1, 0, 1, 12), [('withitem', ('Name', (1, 5, 1, 6), 'x', ('Load',)), None)], [('Pass', (1, 8, 1, 12))], None)], []), +('Module', [('With', (1, 0, 1, 15), [('withitem', ('Name', (1, 5, 1, 6), 'x', ('Load',)), None), ('withitem', ('Name', (1, 8, 1, 9), 'y', ('Load',)), None)], [('Pass', (1, 11, 1, 15))], None)], []), +('Module', [('With', (1, 0, 1, 17), [('withitem', ('Name', (1, 5, 1, 6), 'x', ('Load',)), ('Name', (1, 10, 1, 11), 'y', ('Store',)))], [('Pass', (1, 13, 1, 17))], None)], []), +('Module', [('With', (1, 0, 1, 25), [('withitem', ('Name', (1, 5, 1, 6), 'x', ('Load',)), ('Name', (1, 10, 1, 11), 'y', ('Store',))), ('withitem', ('Name', (1, 13, 1, 14), 'z', ('Load',)), ('Name', (1, 18, 1, 19), 'q', ('Store',)))], [('Pass', (1, 21, 1, 25))], None)], []), +('Module', [('With', (1, 0, 1, 19), [('withitem', ('Name', (1, 6, 1, 7), 'x', ('Load',)), ('Name', (1, 11, 1, 12), 'y', ('Store',)))], [('Pass', (1, 15, 1, 19))], None)], []), +('Module', [('With', (1, 0, 1, 17), [('withitem', ('Name', (1, 6, 1, 7), 'x', ('Load',)), None), ('withitem', ('Name', (1, 9, 1, 10), 'y', ('Load',)), None)], [('Pass', (1, 13, 1, 17))], None)], []), +('Module', [('Raise', (1, 0, 1, 5), None, None)], []), +('Module', [('Raise', (1, 0, 1, 25), ('Call', (1, 6, 1, 25), ('Name', (1, 6, 1, 15), 'Exception', ('Load',)), [('Constant', (1, 16, 1, 24), 'string', None)], []), None)], []), +('Module', [('Raise', (1, 0, 1, 15), ('Name', (1, 6, 1, 15), 'Exception', ('Load',)), None)], []), +('Module', [('Raise', (1, 0, 1, 35), ('Call', (1, 6, 1, 25), ('Name', (1, 6, 1, 15), 'Exception', ('Load',)), [('Constant', (1, 16, 1, 24), 'string', None)], []), ('Constant', (1, 31, 1, 35), None, None))], []), +('Module', [('Try', (1, 0, 4, 6), [('Pass', (2, 2, 2, 6))], [('ExceptHandler', (3, 0, 4, 6), ('Name', (3, 7, 3, 16), 'Exception', ('Load',)), None, [('Pass', (4, 2, 4, 6))])], [], [])], []), +('Module', [('Try', (1, 0, 4, 6), [('Pass', (2, 2, 2, 6))], [('ExceptHandler', (3, 0, 4, 6), ('Name', (3, 7, 3, 16), 'Exception', ('Load',)), 'exc', [('Pass', (4, 2, 4, 6))])], [], [])], []), +('Module', [('Try', (1, 0, 4, 6), [('Pass', (2, 2, 2, 6))], [], [], [('Pass', (4, 2, 4, 6))])], []), +('Module', [('TryStar', (1, 0, 4, 6), [('Pass', (2, 2, 2, 6))], [('ExceptHandler', (3, 0, 4, 6), ('Name', (3, 8, 3, 17), 'Exception', ('Load',)), None, [('Pass', (4, 2, 4, 6))])], [], [])], []), +('Module', [('TryStar', (1, 0, 4, 6), [('Pass', (2, 2, 2, 6))], [('ExceptHandler', (3, 0, 4, 6), ('Name', (3, 8, 3, 17), 'Exception', ('Load',)), 'exc', [('Pass', (4, 2, 4, 6))])], [], [])], []), +('Module', [('Try', (1, 0, 7, 6), [('Pass', (2, 2, 2, 6))], [('ExceptHandler', (3, 0, 4, 6), ('Name', (3, 7, 3, 16), 'Exception', ('Load',)), None, [('Pass', (4, 2, 4, 6))])], [('Pass', (5, 7, 5, 11))], [('Pass', (7, 2, 7, 6))])], []), +('Module', [('Try', (1, 0, 7, 6), [('Pass', (2, 2, 2, 6))], [('ExceptHandler', (3, 0, 4, 6), ('Name', (3, 7, 3, 16), 'Exception', ('Load',)), 'exc', [('Pass', (4, 2, 4, 6))])], [('Pass', (5, 7, 5, 11))], [('Pass', (7, 2, 7, 6))])], []), +('Module', [('TryStar', (1, 0, 7, 6), [('Pass', (2, 2, 2, 6))], [('ExceptHandler', (3, 0, 4, 6), ('Name', (3, 8, 3, 17), 'Exception', ('Load',)), 'exc', [('Pass', (4, 2, 4, 6))])], [('Pass', (5, 7, 5, 11))], [('Pass', (7, 2, 7, 6))])], []), +('Module', [('Assert', (1, 0, 1, 8), ('Name', (1, 7, 1, 8), 'v', ('Load',)), None)], []), +('Module', [('Assert', (1, 0, 1, 19), ('Name', (1, 7, 1, 8), 'v', ('Load',)), ('Constant', (1, 10, 1, 19), 'message', None))], []), +('Module', [('Import', (1, 0, 1, 10), [('alias', (1, 7, 1, 10), 'sys', None)])], []), +('Module', [('Import', (1, 0, 1, 17), [('alias', (1, 7, 1, 17), 'foo', 'bar')])], []), +('Module', [('ImportFrom', (1, 0, 1, 22), 'sys', [('alias', (1, 16, 1, 22), 'x', 'y')], 0)], []), +('Module', [('ImportFrom', (1, 0, 1, 17), 'sys', [('alias', (1, 16, 1, 17), 'v', None)], 0)], []), +('Module', [('Global', (1, 0, 1, 8), ['v'])], []), +('Module', [('Expr', (1, 0, 1, 1), ('Constant', (1, 0, 1, 1), 1, None))], []), +('Module', [('Pass', (1, 0, 1, 4))], []), +('Module', [('For', (1, 0, 1, 16), ('Name', (1, 4, 1, 5), 'v', ('Store',)), ('Name', (1, 9, 1, 10), 'v', ('Load',)), [('Break', (1, 11, 1, 16))], [], None)], []), +('Module', [('For', (1, 0, 1, 19), ('Name', (1, 4, 1, 5), 'v', ('Store',)), ('Name', (1, 9, 1, 10), 'v', ('Load',)), [('Continue', (1, 11, 1, 19))], [], None)], []), +('Module', [('For', (1, 0, 1, 18), ('Tuple', (1, 4, 1, 7), [('Name', (1, 4, 1, 5), 'a', ('Store',)), ('Name', (1, 6, 1, 7), 'b', ('Store',))], ('Store',)), ('Name', (1, 11, 1, 12), 'c', ('Load',)), [('Pass', (1, 14, 1, 18))], [], None)], []), +('Module', [('For', (1, 0, 1, 20), ('Tuple', (1, 4, 1, 9), [('Name', (1, 5, 1, 6), 'a', ('Store',)), ('Name', (1, 7, 1, 8), 'b', ('Store',))], ('Store',)), ('Name', (1, 13, 1, 14), 'c', ('Load',)), [('Pass', (1, 16, 1, 20))], [], None)], []), +('Module', [('For', (1, 0, 1, 20), ('List', (1, 4, 1, 9), [('Name', (1, 5, 1, 6), 'a', ('Store',)), ('Name', (1, 7, 1, 8), 'b', ('Store',))], ('Store',)), ('Name', (1, 13, 1, 14), 'c', ('Load',)), [('Pass', (1, 16, 1, 20))], [], None)], []), +('Module', [('Expr', (1, 0, 11, 5), ('GeneratorExp', (1, 0, 11, 5), ('Tuple', (2, 4, 6, 5), [('Name', (3, 4, 3, 6), 'Aa', ('Load',)), ('Name', (5, 7, 5, 9), 'Bb', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (8, 4, 10, 6), [('Name', (8, 4, 8, 6), 'Aa', ('Store',)), ('Name', (10, 4, 10, 6), 'Bb', ('Store',))], ('Store',)), ('Name', (10, 10, 10, 12), 'Cc', ('Load',)), [], 0)]))], []), +('Module', [('Expr', (1, 0, 1, 34), ('DictComp', (1, 0, 1, 34), ('Name', (1, 1, 1, 2), 'a', ('Load',)), ('Name', (1, 5, 1, 6), 'b', ('Load',)), [('comprehension', ('Name', (1, 11, 1, 12), 'w', ('Store',)), ('Name', (1, 16, 1, 17), 'x', ('Load',)), [], 0), ('comprehension', ('Name', (1, 22, 1, 23), 'm', ('Store',)), ('Name', (1, 27, 1, 28), 'p', ('Load',)), [('Name', (1, 32, 1, 33), 'g', ('Load',))], 0)]))], []), +('Module', [('Expr', (1, 0, 1, 20), ('DictComp', (1, 0, 1, 20), ('Name', (1, 1, 1, 2), 'a', ('Load',)), ('Name', (1, 5, 1, 6), 'b', ('Load',)), [('comprehension', ('Tuple', (1, 11, 1, 14), [('Name', (1, 11, 1, 12), 'v', ('Store',)), ('Name', (1, 13, 1, 14), 'w', ('Store',))], ('Store',)), ('Name', (1, 18, 1, 19), 'x', ('Load',)), [], 0)]))], []), +('Module', [('Expr', (1, 0, 1, 19), ('SetComp', (1, 0, 1, 19), ('Name', (1, 1, 1, 2), 'r', ('Load',)), [('comprehension', ('Name', (1, 7, 1, 8), 'l', ('Store',)), ('Name', (1, 12, 1, 13), 'x', ('Load',)), [('Name', (1, 17, 1, 18), 'g', ('Load',))], 0)]))], []), +('Module', [('Expr', (1, 0, 1, 16), ('SetComp', (1, 0, 1, 16), ('Name', (1, 1, 1, 2), 'r', ('Load',)), [('comprehension', ('Tuple', (1, 7, 1, 10), [('Name', (1, 7, 1, 8), 'l', ('Store',)), ('Name', (1, 9, 1, 10), 'm', ('Store',))], ('Store',)), ('Name', (1, 14, 1, 15), 'x', ('Load',)), [], 0)]))], []), +('Module', [('AsyncFunctionDef', (1, 0, 3, 18), 'f', ('arguments', [], [], None, [], [], None, []), [('Expr', (2, 1, 2, 17), ('Constant', (2, 1, 2, 17), 'async function', None)), ('Expr', (3, 1, 3, 18), ('Await', (3, 1, 3, 18), ('Call', (3, 7, 3, 18), ('Name', (3, 7, 3, 16), 'something', ('Load',)), [], [])))], [], None, None, [])], []), +('Module', [('AsyncFunctionDef', (1, 0, 3, 8), 'f', ('arguments', [], [], None, [], [], None, []), [('AsyncFor', (2, 1, 3, 8), ('Name', (2, 11, 2, 12), 'e', ('Store',)), ('Name', (2, 16, 2, 17), 'i', ('Load',)), [('Expr', (2, 19, 2, 20), ('Constant', (2, 19, 2, 20), 1, None))], [('Expr', (3, 7, 3, 8), ('Constant', (3, 7, 3, 8), 2, None))], None)], [], None, None, [])], []), +('Module', [('AsyncFunctionDef', (1, 0, 2, 21), 'f', ('arguments', [], [], None, [], [], None, []), [('AsyncWith', (2, 1, 2, 21), [('withitem', ('Name', (2, 12, 2, 13), 'a', ('Load',)), ('Name', (2, 17, 2, 18), 'b', ('Store',)))], [('Expr', (2, 20, 2, 21), ('Constant', (2, 20, 2, 21), 1, None))], None)], [], None, None, [])], []), +('Module', [('Expr', (1, 0, 1, 14), ('Dict', (1, 0, 1, 14), [None, ('Constant', (1, 10, 1, 11), 2, None)], [('Dict', (1, 3, 1, 8), [('Constant', (1, 4, 1, 5), 1, None)], [('Constant', (1, 6, 1, 7), 2, None)]), ('Constant', (1, 12, 1, 13), 3, None)]))], []), +('Module', [('Expr', (1, 0, 1, 12), ('Set', (1, 0, 1, 12), [('Starred', (1, 1, 1, 8), ('Set', (1, 2, 1, 8), [('Constant', (1, 3, 1, 4), 1, None), ('Constant', (1, 6, 1, 7), 2, None)]), ('Load',)), ('Constant', (1, 10, 1, 11), 3, None)]))], []), +('Module', [('FunctionDef', (1, 0, 1, 16), 'f', ('arguments', [], [], None, [], [], None, []), [('Expr', (1, 9, 1, 16), ('Yield', (1, 9, 1, 16), ('Constant', (1, 15, 1, 16), 1, None)))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 22), 'f', ('arguments', [], [], None, [], [], None, []), [('Expr', (1, 9, 1, 22), ('YieldFrom', (1, 9, 1, 22), ('List', (1, 20, 1, 22), [], ('Load',))))], [], None, None, [])], []), +('Module', [('AsyncFunctionDef', (1, 0, 2, 21), 'f', ('arguments', [], [], None, [], [], None, []), [('Expr', (2, 1, 2, 21), ('ListComp', (2, 1, 2, 21), ('Name', (2, 2, 2, 3), 'i', ('Load',)), [('comprehension', ('Name', (2, 14, 2, 15), 'b', ('Store',)), ('Name', (2, 19, 2, 20), 'c', ('Load',)), [], 1)]))], [], None, None, [])], []), +('Module', [('FunctionDef', (4, 0, 4, 13), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (4, 9, 4, 13))], [('Name', (1, 1, 1, 6), 'deco1', ('Load',)), ('Call', (2, 1, 2, 8), ('Name', (2, 1, 2, 6), 'deco2', ('Load',)), [], []), ('Call', (3, 1, 3, 9), ('Name', (3, 1, 3, 6), 'deco3', ('Load',)), [('Constant', (3, 7, 3, 8), 1, None)], [])], None, None, [])], []), +('Module', [('AsyncFunctionDef', (4, 0, 4, 19), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (4, 15, 4, 19))], [('Name', (1, 1, 1, 6), 'deco1', ('Load',)), ('Call', (2, 1, 2, 8), ('Name', (2, 1, 2, 6), 'deco2', ('Load',)), [], []), ('Call', (3, 1, 3, 9), ('Name', (3, 1, 3, 6), 'deco3', ('Load',)), [('Constant', (3, 7, 3, 8), 1, None)], [])], None, None, [])], []), +('Module', [('ClassDef', (4, 0, 4, 13), 'C', [], [], [('Pass', (4, 9, 4, 13))], [('Name', (1, 1, 1, 6), 'deco1', ('Load',)), ('Call', (2, 1, 2, 8), ('Name', (2, 1, 2, 6), 'deco2', ('Load',)), [], []), ('Call', (3, 1, 3, 9), ('Name', (3, 1, 3, 6), 'deco3', ('Load',)), [('Constant', (3, 7, 3, 8), 1, None)], [])], [])], []), +('Module', [('FunctionDef', (2, 0, 2, 13), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (2, 9, 2, 13))], [('Call', (1, 1, 1, 19), ('Name', (1, 1, 1, 5), 'deco', ('Load',)), [('GeneratorExp', (1, 5, 1, 19), ('Name', (1, 6, 1, 7), 'a', ('Load',)), [('comprehension', ('Name', (1, 12, 1, 13), 'a', ('Store',)), ('Name', (1, 17, 1, 18), 'b', ('Load',)), [], 0)])], [])], None, None, [])], []), +('Module', [('FunctionDef', (2, 0, 2, 13), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (2, 9, 2, 13))], [('Attribute', (1, 1, 1, 6), ('Attribute', (1, 1, 1, 4), ('Name', (1, 1, 1, 2), 'a', ('Load',)), 'b', ('Load',)), 'c', ('Load',))], None, None, [])], []), +('Module', [('Expr', (1, 0, 1, 8), ('NamedExpr', (1, 1, 1, 7), ('Name', (1, 1, 1, 2), 'a', ('Store',)), ('Constant', (1, 6, 1, 7), 1, None)))], []), +('Module', [('If', (1, 0, 1, 19), ('NamedExpr', (1, 3, 1, 13), ('Name', (1, 3, 1, 4), 'a', ('Store',)), ('Call', (1, 8, 1, 13), ('Name', (1, 8, 1, 11), 'foo', ('Load',)), [], [])), [('Pass', (1, 15, 1, 19))], [])], []), +('Module', [('While', (1, 0, 1, 22), ('NamedExpr', (1, 6, 1, 16), ('Name', (1, 6, 1, 7), 'a', ('Store',)), ('Call', (1, 11, 1, 16), ('Name', (1, 11, 1, 14), 'foo', ('Load',)), [], [])), [('Pass', (1, 18, 1, 22))], [])], []), +('Module', [('FunctionDef', (1, 0, 1, 18), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [], None, [], [], None, []), [('Pass', (1, 14, 1, 18))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 26), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 12, 1, 13), 'c', None, None), ('arg', (1, 15, 1, 16), 'd', None, None), ('arg', (1, 18, 1, 19), 'e', None, None)], None, [], [], None, []), [('Pass', (1, 22, 1, 26))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 29), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 12, 1, 13), 'c', None, None)], None, [('arg', (1, 18, 1, 19), 'd', None, None), ('arg', (1, 21, 1, 22), 'e', None, None)], [None, None], None, []), [('Pass', (1, 25, 1, 29))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 39), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 12, 1, 13), 'c', None, None)], None, [('arg', (1, 18, 1, 19), 'd', None, None), ('arg', (1, 21, 1, 22), 'e', None, None)], [None, None], ('arg', (1, 26, 1, 32), 'kwargs', None, None), []), [('Pass', (1, 35, 1, 39))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 20), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [], None, [], [], None, [('Constant', (1, 8, 1, 9), 1, None)]), [('Pass', (1, 16, 1, 20))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 29), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 14, 1, 15), 'b', None, None), ('arg', (1, 19, 1, 20), 'c', None, None)], None, [], [], None, [('Constant', (1, 8, 1, 9), 1, None), ('Constant', (1, 16, 1, 17), 2, None), ('Constant', (1, 21, 1, 22), 4, None)]), [('Pass', (1, 25, 1, 29))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 32), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 14, 1, 15), 'b', None, None)], None, [('arg', (1, 22, 1, 23), 'c', None, None)], [('Constant', (1, 24, 1, 25), 4, None)], None, [('Constant', (1, 8, 1, 9), 1, None), ('Constant', (1, 16, 1, 17), 2, None)]), [('Pass', (1, 28, 1, 32))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 30), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 14, 1, 15), 'b', None, None)], None, [('arg', (1, 22, 1, 23), 'c', None, None)], [None], None, [('Constant', (1, 8, 1, 9), 1, None), ('Constant', (1, 16, 1, 17), 2, None)]), [('Pass', (1, 26, 1, 30))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 42), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 14, 1, 15), 'b', None, None)], None, [('arg', (1, 22, 1, 23), 'c', None, None)], [('Constant', (1, 24, 1, 25), 4, None)], ('arg', (1, 29, 1, 35), 'kwargs', None, None), [('Constant', (1, 8, 1, 9), 1, None), ('Constant', (1, 16, 1, 17), 2, None)]), [('Pass', (1, 38, 1, 42))], [], None, None, [])], []), +('Module', [('FunctionDef', (1, 0, 1, 40), 'f', ('arguments', [('arg', (1, 6, 1, 7), 'a', None, None)], [('arg', (1, 14, 1, 15), 'b', None, None)], None, [('arg', (1, 22, 1, 23), 'c', None, None)], [None], ('arg', (1, 27, 1, 33), 'kwargs', None, None), [('Constant', (1, 8, 1, 9), 1, None), ('Constant', (1, 16, 1, 17), 2, None)]), [('Pass', (1, 36, 1, 40))], [], None, None, [])], []), +('Module', [('TypeAlias', (1, 0, 1, 12), ('Name', (1, 5, 1, 6), 'X', ('Store',)), [], ('Name', (1, 9, 1, 12), 'int', ('Load',)))], []), +('Module', [('TypeAlias', (1, 0, 1, 15), ('Name', (1, 5, 1, 6), 'X', ('Store',)), [('TypeVar', (1, 7, 1, 8), 'T', None, None)], ('Name', (1, 12, 1, 15), 'int', ('Load',)))], []), +('Module', [('TypeAlias', (1, 0, 1, 32), ('Name', (1, 5, 1, 6), 'X', ('Store',)), [('TypeVar', (1, 7, 1, 8), 'T', None, None), ('TypeVarTuple', (1, 10, 1, 13), 'Ts', None), ('ParamSpec', (1, 15, 1, 18), 'P', None)], ('Tuple', (1, 22, 1, 32), [('Name', (1, 23, 1, 24), 'T', ('Load',)), ('Name', (1, 26, 1, 28), 'Ts', ('Load',)), ('Name', (1, 30, 1, 31), 'P', ('Load',))], ('Load',)))], []), +('Module', [('TypeAlias', (1, 0, 1, 37), ('Name', (1, 5, 1, 6), 'X', ('Store',)), [('TypeVar', (1, 7, 1, 13), 'T', ('Name', (1, 10, 1, 13), 'int', ('Load',)), None), ('TypeVarTuple', (1, 15, 1, 18), 'Ts', None), ('ParamSpec', (1, 20, 1, 23), 'P', None)], ('Tuple', (1, 27, 1, 37), [('Name', (1, 28, 1, 29), 'T', ('Load',)), ('Name', (1, 31, 1, 33), 'Ts', ('Load',)), ('Name', (1, 35, 1, 36), 'P', ('Load',))], ('Load',)))], []), +('Module', [('TypeAlias', (1, 0, 1, 44), ('Name', (1, 5, 1, 6), 'X', ('Store',)), [('TypeVar', (1, 7, 1, 20), 'T', ('Tuple', (1, 10, 1, 20), [('Name', (1, 11, 1, 14), 'int', ('Load',)), ('Name', (1, 16, 1, 19), 'str', ('Load',))], ('Load',)), None), ('TypeVarTuple', (1, 22, 1, 25), 'Ts', None), ('ParamSpec', (1, 27, 1, 30), 'P', None)], ('Tuple', (1, 34, 1, 44), [('Name', (1, 35, 1, 36), 'T', ('Load',)), ('Name', (1, 38, 1, 40), 'Ts', ('Load',)), ('Name', (1, 42, 1, 43), 'P', ('Load',))], ('Load',)))], []), +('Module', [('TypeAlias', (1, 0, 1, 48), ('Name', (1, 5, 1, 6), 'X', ('Store',)), [('TypeVar', (1, 7, 1, 17), 'T', ('Name', (1, 10, 1, 13), 'int', ('Load',)), ('Constant', (1, 16, 1, 17), 1, None)), ('TypeVarTuple', (1, 19, 1, 26), 'Ts', ('Constant', (1, 25, 1, 26), 2, None)), ('ParamSpec', (1, 28, 1, 34), 'P', ('Constant', (1, 33, 1, 34), 3, None))], ('Tuple', (1, 38, 1, 48), [('Name', (1, 39, 1, 40), 'T', ('Load',)), ('Name', (1, 42, 1, 44), 'Ts', ('Load',)), ('Name', (1, 46, 1, 47), 'P', ('Load',))], ('Load',)))], []), +('Module', [('ClassDef', (1, 0, 1, 16), 'X', [], [], [('Pass', (1, 12, 1, 16))], [], [('TypeVar', (1, 8, 1, 9), 'T', None, None)])], []), +('Module', [('ClassDef', (1, 0, 1, 26), 'X', [], [], [('Pass', (1, 22, 1, 26))], [], [('TypeVar', (1, 8, 1, 9), 'T', None, None), ('TypeVarTuple', (1, 11, 1, 14), 'Ts', None), ('ParamSpec', (1, 16, 1, 19), 'P', None)])], []), +('Module', [('ClassDef', (1, 0, 1, 31), 'X', [], [], [('Pass', (1, 27, 1, 31))], [], [('TypeVar', (1, 8, 1, 14), 'T', ('Name', (1, 11, 1, 14), 'int', ('Load',)), None), ('TypeVarTuple', (1, 16, 1, 19), 'Ts', None), ('ParamSpec', (1, 21, 1, 24), 'P', None)])], []), +('Module', [('ClassDef', (1, 0, 1, 38), 'X', [], [], [('Pass', (1, 34, 1, 38))], [], [('TypeVar', (1, 8, 1, 21), 'T', ('Tuple', (1, 11, 1, 21), [('Name', (1, 12, 1, 15), 'int', ('Load',)), ('Name', (1, 17, 1, 20), 'str', ('Load',))], ('Load',)), None), ('TypeVarTuple', (1, 23, 1, 26), 'Ts', None), ('ParamSpec', (1, 28, 1, 31), 'P', None)])], []), +('Module', [('ClassDef', (1, 0, 1, 43), 'X', [], [], [('Pass', (1, 39, 1, 43))], [], [('TypeVar', (1, 8, 1, 18), 'T', ('Name', (1, 11, 1, 14), 'int', ('Load',)), ('Constant', (1, 17, 1, 18), 1, None)), ('TypeVarTuple', (1, 20, 1, 27), 'Ts', ('Constant', (1, 26, 1, 27), 2, None)), ('ParamSpec', (1, 29, 1, 36), 'P', ('Constant', (1, 35, 1, 36), 3, None))])], []), +('Module', [('FunctionDef', (1, 0, 1, 16), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 12, 1, 16))], [], None, None, [('TypeVar', (1, 6, 1, 7), 'T', None, None)])], []), +('Module', [('FunctionDef', (1, 0, 1, 26), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 22, 1, 26))], [], None, None, [('TypeVar', (1, 6, 1, 7), 'T', None, None), ('TypeVarTuple', (1, 9, 1, 12), 'Ts', None), ('ParamSpec', (1, 14, 1, 17), 'P', None)])], []), +('Module', [('FunctionDef', (1, 0, 1, 31), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 27, 1, 31))], [], None, None, [('TypeVar', (1, 6, 1, 12), 'T', ('Name', (1, 9, 1, 12), 'int', ('Load',)), None), ('TypeVarTuple', (1, 14, 1, 17), 'Ts', None), ('ParamSpec', (1, 19, 1, 22), 'P', None)])], []), +('Module', [('FunctionDef', (1, 0, 1, 38), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 34, 1, 38))], [], None, None, [('TypeVar', (1, 6, 1, 19), 'T', ('Tuple', (1, 9, 1, 19), [('Name', (1, 10, 1, 13), 'int', ('Load',)), ('Name', (1, 15, 1, 18), 'str', ('Load',))], ('Load',)), None), ('TypeVarTuple', (1, 21, 1, 24), 'Ts', None), ('ParamSpec', (1, 26, 1, 29), 'P', None)])], []), +('Module', [('FunctionDef', (1, 0, 1, 43), 'f', ('arguments', [], [], None, [], [], None, []), [('Pass', (1, 39, 1, 43))], [], None, None, [('TypeVar', (1, 6, 1, 16), 'T', ('Name', (1, 9, 1, 12), 'int', ('Load',)), ('Constant', (1, 15, 1, 16), 1, None)), ('TypeVarTuple', (1, 18, 1, 25), 'Ts', ('Constant', (1, 24, 1, 25), 2, None)), ('ParamSpec', (1, 27, 1, 34), 'P', ('Constant', (1, 33, 1, 34), 3, None))])], []), +('Module', [('Match', (1, 0, 3, 6), ('Name', (1, 6, 1, 7), 'x', ('Load',)), [('match_case', ('MatchValue', (2, 6, 2, 7), ('Constant', (2, 6, 2, 7), 1, None)), None, [('Pass', (3, 2, 3, 6))])])], []), +('Module', [('Match', (1, 0, 5, 6), ('Name', (1, 6, 1, 7), 'x', ('Load',)), [('match_case', ('MatchValue', (2, 6, 2, 7), ('Constant', (2, 6, 2, 7), 1, None)), None, [('Pass', (3, 2, 3, 6))]), ('match_case', ('MatchAs', (4, 6, 4, 7), None, None), None, [('Pass', (5, 2, 5, 6))])])], []), +] +single_results = [ +('Interactive', [('Expr', (1, 0, 1, 3), ('BinOp', (1, 0, 1, 3), ('Constant', (1, 0, 1, 1), 1, None), ('Add',), ('Constant', (1, 2, 1, 3), 2, None)))]), +] +eval_results = [ +('Expression', ('Constant', (1, 0, 1, 4), None, None)), +('Expression', ('Constant', (1, 0, 1, 4), True, None)), +('Expression', ('Constant', (1, 0, 1, 5), False, None)), +('Expression', ('BoolOp', (1, 0, 1, 7), ('And',), [('Name', (1, 0, 1, 1), 'a', ('Load',)), ('Name', (1, 6, 1, 7), 'b', ('Load',))])), +('Expression', ('BoolOp', (1, 0, 1, 6), ('Or',), [('Name', (1, 0, 1, 1), 'a', ('Load',)), ('Name', (1, 5, 1, 6), 'b', ('Load',))])), +('Expression', ('BinOp', (1, 0, 1, 5), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('Add',), ('Name', (1, 4, 1, 5), 'b', ('Load',)))), +('Expression', ('BinOp', (1, 0, 1, 5), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('Sub',), ('Name', (1, 4, 1, 5), 'b', ('Load',)))), +('Expression', ('BinOp', (1, 0, 1, 5), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('Mult',), ('Name', (1, 4, 1, 5), 'b', ('Load',)))), +('Expression', ('BinOp', (1, 0, 1, 5), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('Div',), ('Name', (1, 4, 1, 5), 'b', ('Load',)))), +('Expression', ('BinOp', (1, 0, 1, 5), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('MatMult',), ('Name', (1, 4, 1, 5), 'b', ('Load',)))), +('Expression', ('BinOp', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('FloorDiv',), ('Name', (1, 5, 1, 6), 'b', ('Load',)))), +('Expression', ('BinOp', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('Pow',), ('Name', (1, 5, 1, 6), 'b', ('Load',)))), +('Expression', ('BinOp', (1, 0, 1, 5), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('Mod',), ('Name', (1, 4, 1, 5), 'b', ('Load',)))), +('Expression', ('BinOp', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('RShift',), ('Name', (1, 5, 1, 6), 'b', ('Load',)))), +('Expression', ('BinOp', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('LShift',), ('Name', (1, 5, 1, 6), 'b', ('Load',)))), +('Expression', ('BinOp', (1, 0, 1, 5), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('BitXor',), ('Name', (1, 4, 1, 5), 'b', ('Load',)))), +('Expression', ('BinOp', (1, 0, 1, 5), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('BitOr',), ('Name', (1, 4, 1, 5), 'b', ('Load',)))), +('Expression', ('BinOp', (1, 0, 1, 5), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('BitAnd',), ('Name', (1, 4, 1, 5), 'b', ('Load',)))), +('Expression', ('UnaryOp', (1, 0, 1, 5), ('Not',), ('Name', (1, 4, 1, 5), 'v', ('Load',)))), +('Expression', ('UnaryOp', (1, 0, 1, 2), ('UAdd',), ('Name', (1, 1, 1, 2), 'v', ('Load',)))), +('Expression', ('UnaryOp', (1, 0, 1, 2), ('USub',), ('Name', (1, 1, 1, 2), 'v', ('Load',)))), +('Expression', ('UnaryOp', (1, 0, 1, 2), ('Invert',), ('Name', (1, 1, 1, 2), 'v', ('Load',)))), +('Expression', ('Lambda', (1, 0, 1, 11), ('arguments', [], [], None, [], [], None, []), ('Constant', (1, 7, 1, 11), None, None))), +('Expression', ('Dict', (1, 0, 1, 7), [('Constant', (1, 2, 1, 3), 1, None)], [('Constant', (1, 4, 1, 5), 2, None)])), +('Expression', ('Dict', (1, 0, 1, 2), [], [])), +('Expression', ('Set', (1, 0, 1, 7), [('Constant', (1, 1, 1, 5), None, None)])), +('Expression', ('Dict', (1, 0, 5, 6), [('Constant', (2, 6, 2, 7), 1, None)], [('Constant', (4, 10, 4, 11), 2, None)])), +('Expression', ('List', (1, 0, 5, 6), [('Constant', (2, 6, 2, 7), 1, None), ('Constant', (4, 8, 4, 9), 1, None)], ('Load',))), +('Expression', ('Tuple', (1, 0, 4, 6), [('Constant', (2, 6, 2, 7), 1, None)], ('Load',))), +('Expression', ('Set', (1, 0, 5, 6), [('Constant', (2, 6, 2, 7), 1, None), ('Constant', (4, 8, 4, 9), 1, None)])), +('Expression', ('ListComp', (1, 0, 1, 19), ('Name', (1, 1, 1, 2), 'a', ('Load',)), [('comprehension', ('Name', (1, 7, 1, 8), 'b', ('Store',)), ('Name', (1, 12, 1, 13), 'c', ('Load',)), [('Name', (1, 17, 1, 18), 'd', ('Load',))], 0)])), +('Expression', ('GeneratorExp', (1, 0, 1, 19), ('Name', (1, 1, 1, 2), 'a', ('Load',)), [('comprehension', ('Name', (1, 7, 1, 8), 'b', ('Store',)), ('Name', (1, 12, 1, 13), 'c', ('Load',)), [('Name', (1, 17, 1, 18), 'd', ('Load',))], 0)])), +('Expression', ('SetComp', (1, 0, 1, 19), ('Name', (1, 1, 1, 2), 'a', ('Load',)), [('comprehension', ('Name', (1, 7, 1, 8), 'b', ('Store',)), ('Name', (1, 12, 1, 13), 'c', ('Load',)), [('Name', (1, 17, 1, 18), 'd', ('Load',))], 0)])), +('Expression', ('DictComp', (1, 0, 1, 25), ('Name', (1, 1, 1, 2), 'k', ('Load',)), ('Name', (1, 4, 1, 5), 'v', ('Load',)), [('comprehension', ('Tuple', (1, 10, 1, 14), [('Name', (1, 10, 1, 11), 'k', ('Store',)), ('Name', (1, 13, 1, 14), 'v', ('Store',))], ('Store',)), ('Name', (1, 18, 1, 19), 'c', ('Load',)), [('Name', (1, 23, 1, 24), 'd', ('Load',))], 0)])), +('Expression', ('ListComp', (1, 0, 1, 20), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11, 1, 14), [('Name', (1, 11, 1, 12), 'a', ('Store',)), ('Name', (1, 13, 1, 14), 'b', ('Store',))], ('Store',)), ('Name', (1, 18, 1, 19), 'c', ('Load',)), [], 0)])), +('Expression', ('ListComp', (1, 0, 1, 22), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11, 1, 16), [('Name', (1, 12, 1, 13), 'a', ('Store',)), ('Name', (1, 14, 1, 15), 'b', ('Store',))], ('Store',)), ('Name', (1, 20, 1, 21), 'c', ('Load',)), [], 0)])), +('Expression', ('ListComp', (1, 0, 1, 22), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('List', (1, 11, 1, 16), [('Name', (1, 12, 1, 13), 'a', ('Store',)), ('Name', (1, 14, 1, 15), 'b', ('Store',))], ('Store',)), ('Name', (1, 20, 1, 21), 'c', ('Load',)), [], 0)])), +('Expression', ('SetComp', (1, 0, 1, 20), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11, 1, 14), [('Name', (1, 11, 1, 12), 'a', ('Store',)), ('Name', (1, 13, 1, 14), 'b', ('Store',))], ('Store',)), ('Name', (1, 18, 1, 19), 'c', ('Load',)), [], 0)])), +('Expression', ('SetComp', (1, 0, 1, 22), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11, 1, 16), [('Name', (1, 12, 1, 13), 'a', ('Store',)), ('Name', (1, 14, 1, 15), 'b', ('Store',))], ('Store',)), ('Name', (1, 20, 1, 21), 'c', ('Load',)), [], 0)])), +('Expression', ('SetComp', (1, 0, 1, 22), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('List', (1, 11, 1, 16), [('Name', (1, 12, 1, 13), 'a', ('Store',)), ('Name', (1, 14, 1, 15), 'b', ('Store',))], ('Store',)), ('Name', (1, 20, 1, 21), 'c', ('Load',)), [], 0)])), +('Expression', ('GeneratorExp', (1, 0, 1, 20), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11, 1, 14), [('Name', (1, 11, 1, 12), 'a', ('Store',)), ('Name', (1, 13, 1, 14), 'b', ('Store',))], ('Store',)), ('Name', (1, 18, 1, 19), 'c', ('Load',)), [], 0)])), +('Expression', ('GeneratorExp', (1, 0, 1, 22), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('Tuple', (1, 11, 1, 16), [('Name', (1, 12, 1, 13), 'a', ('Store',)), ('Name', (1, 14, 1, 15), 'b', ('Store',))], ('Store',)), ('Name', (1, 20, 1, 21), 'c', ('Load',)), [], 0)])), +('Expression', ('GeneratorExp', (1, 0, 1, 22), ('Tuple', (1, 1, 1, 6), [('Name', (1, 2, 1, 3), 'a', ('Load',)), ('Name', (1, 4, 1, 5), 'b', ('Load',))], ('Load',)), [('comprehension', ('List', (1, 11, 1, 16), [('Name', (1, 12, 1, 13), 'a', ('Store',)), ('Name', (1, 14, 1, 15), 'b', ('Store',))], ('Store',)), ('Name', (1, 20, 1, 21), 'c', ('Load',)), [], 0)])), +('Expression', ('Compare', (1, 0, 1, 9), ('Constant', (1, 0, 1, 1), 1, None), [('Lt',), ('Lt',)], [('Constant', (1, 4, 1, 5), 2, None), ('Constant', (1, 8, 1, 9), 3, None)])), +('Expression', ('Compare', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'a', ('Load',)), [('Eq',)], [('Name', (1, 5, 1, 6), 'b', ('Load',))])), +('Expression', ('Compare', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'a', ('Load',)), [('LtE',)], [('Name', (1, 5, 1, 6), 'b', ('Load',))])), +('Expression', ('Compare', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'a', ('Load',)), [('GtE',)], [('Name', (1, 5, 1, 6), 'b', ('Load',))])), +('Expression', ('Compare', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'a', ('Load',)), [('NotEq',)], [('Name', (1, 5, 1, 6), 'b', ('Load',))])), +('Expression', ('Compare', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'a', ('Load',)), [('Is',)], [('Name', (1, 5, 1, 6), 'b', ('Load',))])), +('Expression', ('Compare', (1, 0, 1, 10), ('Name', (1, 0, 1, 1), 'a', ('Load',)), [('IsNot',)], [('Name', (1, 9, 1, 10), 'b', ('Load',))])), +('Expression', ('Compare', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'a', ('Load',)), [('In',)], [('Name', (1, 5, 1, 6), 'b', ('Load',))])), +('Expression', ('Compare', (1, 0, 1, 10), ('Name', (1, 0, 1, 1), 'a', ('Load',)), [('NotIn',)], [('Name', (1, 9, 1, 10), 'b', ('Load',))])), +('Expression', ('Call', (1, 0, 1, 3), ('Name', (1, 0, 1, 1), 'f', ('Load',)), [], [])), +('Expression', ('Call', (1, 0, 1, 17), ('Name', (1, 0, 1, 1), 'f', ('Load',)), [('Constant', (1, 2, 1, 3), 1, None), ('Constant', (1, 4, 1, 5), 2, None), ('Starred', (1, 10, 1, 12), ('Name', (1, 11, 1, 12), 'd', ('Load',)), ('Load',))], [('keyword', (1, 6, 1, 9), 'c', ('Constant', (1, 8, 1, 9), 3, None)), ('keyword', (1, 13, 1, 16), None, ('Name', (1, 15, 1, 16), 'e', ('Load',)))])), +('Expression', ('Call', (1, 0, 1, 10), ('Name', (1, 0, 1, 1), 'f', ('Load',)), [('Starred', (1, 2, 1, 9), ('List', (1, 3, 1, 9), [('Constant', (1, 4, 1, 5), 0, None), ('Constant', (1, 7, 1, 8), 1, None)], ('Load',)), ('Load',))], [])), +('Expression', ('Call', (1, 0, 1, 15), ('Name', (1, 0, 1, 1), 'f', ('Load',)), [('GeneratorExp', (1, 1, 1, 15), ('Name', (1, 2, 1, 3), 'a', ('Load',)), [('comprehension', ('Name', (1, 8, 1, 9), 'a', ('Store',)), ('Name', (1, 13, 1, 14), 'b', ('Load',)), [], 0)])], [])), +('Expression', ('Constant', (1, 0, 1, 2), 10, None)), +('Expression', ('Constant', (1, 0, 1, 2), 1j, None)), +('Expression', ('Constant', (1, 0, 1, 8), 'string', None)), +('Expression', ('Attribute', (1, 0, 1, 3), ('Name', (1, 0, 1, 1), 'a', ('Load',)), 'b', ('Load',))), +('Expression', ('Subscript', (1, 0, 1, 6), ('Name', (1, 0, 1, 1), 'a', ('Load',)), ('Slice', (1, 2, 1, 5), ('Name', (1, 2, 1, 3), 'b', ('Load',)), ('Name', (1, 4, 1, 5), 'c', ('Load',)), None), ('Load',))), +('Expression', ('Name', (1, 0, 1, 1), 'v', ('Load',))), +('Expression', ('List', (1, 0, 1, 7), [('Constant', (1, 1, 1, 2), 1, None), ('Constant', (1, 3, 1, 4), 2, None), ('Constant', (1, 5, 1, 6), 3, None)], ('Load',))), +('Expression', ('List', (1, 0, 1, 2), [], ('Load',))), +('Expression', ('Tuple', (1, 0, 1, 5), [('Constant', (1, 0, 1, 1), 1, None), ('Constant', (1, 2, 1, 3), 2, None), ('Constant', (1, 4, 1, 5), 3, None)], ('Load',))), +('Expression', ('Tuple', (1, 0, 1, 7), [('Constant', (1, 1, 1, 2), 1, None), ('Constant', (1, 3, 1, 4), 2, None), ('Constant', (1, 5, 1, 6), 3, None)], ('Load',))), +('Expression', ('Tuple', (1, 0, 1, 2), [], ('Load',))), +('Expression', ('Call', (1, 0, 1, 17), ('Attribute', (1, 0, 1, 7), ('Attribute', (1, 0, 1, 5), ('Attribute', (1, 0, 1, 3), ('Name', (1, 0, 1, 1), 'a', ('Load',)), 'b', ('Load',)), 'c', ('Load',)), 'd', ('Load',)), [('Subscript', (1, 8, 1, 16), ('Attribute', (1, 8, 1, 11), ('Name', (1, 8, 1, 9), 'a', ('Load',)), 'b', ('Load',)), ('Slice', (1, 12, 1, 15), ('Constant', (1, 12, 1, 13), 1, None), ('Constant', (1, 14, 1, 15), 2, None), None), ('Load',))], [])), +('Expression', ('Subscript', (1, 0, 1, 7), ('List', (1, 0, 1, 3), [('Constant', (1, 1, 1, 2), 5, None)], ('Load',)), ('Slice', (1, 4, 1, 6), ('Constant', (1, 4, 1, 5), 1, None), None, None), ('Load',))), +('Expression', ('Subscript', (1, 0, 1, 7), ('List', (1, 0, 1, 3), [('Constant', (1, 1, 1, 2), 5, None)], ('Load',)), ('Slice', (1, 4, 1, 6), None, ('Constant', (1, 5, 1, 6), 1, None), None), ('Load',))), +('Expression', ('Subscript', (1, 0, 1, 8), ('List', (1, 0, 1, 3), [('Constant', (1, 1, 1, 2), 5, None)], ('Load',)), ('Slice', (1, 4, 1, 7), None, None, ('Constant', (1, 6, 1, 7), 1, None)), ('Load',))), +('Expression', ('Subscript', (1, 0, 1, 10), ('List', (1, 0, 1, 3), [('Constant', (1, 1, 1, 2), 5, None)], ('Load',)), ('Slice', (1, 4, 1, 9), ('Constant', (1, 4, 1, 5), 1, None), ('Constant', (1, 6, 1, 7), 1, None), ('Constant', (1, 8, 1, 9), 1, None)), ('Load',))), +('Expression', ('IfExp', (1, 0, 1, 21), ('Name', (1, 9, 1, 10), 'x', ('Load',)), ('Call', (1, 0, 1, 5), ('Name', (1, 0, 1, 3), 'foo', ('Load',)), [], []), ('Call', (1, 16, 1, 21), ('Name', (1, 16, 1, 19), 'bar', ('Load',)), [], []))), +('Expression', ('JoinedStr', (1, 0, 1, 6), [('FormattedValue', (1, 2, 1, 5), ('Name', (1, 3, 1, 4), 'a', ('Load',)), -1, None)])), +('Expression', ('JoinedStr', (1, 0, 1, 10), [('FormattedValue', (1, 2, 1, 9), ('Name', (1, 3, 1, 4), 'a', ('Load',)), -1, ('JoinedStr', (1, 4, 1, 8), [('Constant', (1, 5, 1, 8), '.2f', None)]))])), +('Expression', ('JoinedStr', (1, 0, 1, 8), [('FormattedValue', (1, 2, 1, 7), ('Name', (1, 3, 1, 4), 'a', ('Load',)), 114, None)])), +('Expression', ('JoinedStr', (1, 0, 1, 11), [('Constant', (1, 2, 1, 6), 'foo(', None), ('FormattedValue', (1, 6, 1, 9), ('Name', (1, 7, 1, 8), 'a', ('Load',)), -1, None), ('Constant', (1, 9, 1, 10), ')', None)])), +] +main() diff --git a/Lib/test/test_ast/test_ast.py b/Lib/test/test_ast/test_ast.py new file mode 100644 index 0000000000..628c243f2f --- /dev/null +++ b/Lib/test/test_ast/test_ast.py @@ -0,0 +1,3643 @@ +import ast +import builtins +import copy +import dis +import enum +import os +import re +import sys +import textwrap +import types +import unittest +import warnings +import weakref +from functools import partial +from textwrap import dedent + +try: + import _testinternalcapi +except ImportError: + _testinternalcapi = None + +from test import support +from test.support.import_helper import import_fresh_module +from test.support import os_helper, script_helper +from test.support.ast_helper import ASTTestMixin +from test.test_ast.utils import to_tuple +from test.test_ast.snippets import ( + eval_tests, eval_results, exec_tests, exec_results, single_tests, single_results +) + + +class AST_Tests(unittest.TestCase): + maxDiff = None + + def _is_ast_node(self, name, node): + if not isinstance(node, type): + return False + if "ast" not in node.__module__: + return False + return name != "AST" and name[0].isupper() + + def _assertTrueorder(self, ast_node, parent_pos): + if not isinstance(ast_node, ast.AST) or ast_node._fields is None: + return + if isinstance(ast_node, (ast.expr, ast.stmt, ast.excepthandler)): + node_pos = (ast_node.lineno, ast_node.col_offset) + self.assertGreaterEqual(node_pos, parent_pos) + parent_pos = (ast_node.lineno, ast_node.col_offset) + for name in ast_node._fields: + value = getattr(ast_node, name) + if isinstance(value, list): + first_pos = parent_pos + if value and name == "decorator_list": + first_pos = (value[0].lineno, value[0].col_offset) + for child in value: + self._assertTrueorder(child, first_pos) + elif value is not None: + self._assertTrueorder(value, parent_pos) + self.assertEqual(ast_node._fields, ast_node.__match_args__) + + def test_AST_objects(self): + x = ast.AST() + self.assertEqual(x._fields, ()) + x.foobar = 42 + self.assertEqual(x.foobar, 42) + self.assertEqual(x.__dict__["foobar"], 42) + + with self.assertRaises(AttributeError): + x.vararg + + with self.assertRaises(TypeError): + # "ast.AST constructor takes 0 positional arguments" + ast.AST(2) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_AST_fields_NULL_check(self): + # See: https://github.com/python/cpython/issues/126105 + old_value = ast.AST._fields + + def cleanup(): + ast.AST._fields = old_value + self.addCleanup(cleanup) + + del ast.AST._fields + + msg = "type object 'ast.AST' has no attribute '_fields'" + # Both examples used to crash: + with self.assertRaisesRegex(AttributeError, msg): + ast.AST(arg1=123) + with self.assertRaisesRegex(AttributeError, msg): + ast.AST() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_AST_garbage_collection(self): + class X: + pass + + a = ast.AST() + a.x = X() + a.x.a = a + ref = weakref.ref(a.x) + del a + support.gc_collect() + self.assertIsNone(ref()) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_snippets(self): + for input, output, kind in ( + (exec_tests, exec_results, "exec"), + (single_tests, single_results, "single"), + (eval_tests, eval_results, "eval"), + ): + for i, o in zip(input, output): + with self.subTest(action="parsing", input=i): + ast_tree = compile(i, "?", kind, ast.PyCF_ONLY_AST) + self.assertEqual(to_tuple(ast_tree), o) + self._assertTrueorder(ast_tree, (0, 0)) + with self.subTest(action="compiling", input=i, kind=kind): + compile(ast_tree, "?", kind) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_ast_validation(self): + # compile() is the only function that calls PyAST_Validate + snippets_to_validate = exec_tests + single_tests + eval_tests + for snippet in snippets_to_validate: + tree = ast.parse(snippet) + compile(tree, "", "exec") + + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags + def test_optimization_levels__debug__(self): + cases = [(-1, "__debug__"), (0, "__debug__"), (1, False), (2, False)] + for optval, expected in cases: + with self.subTest(optval=optval, expected=expected): + res1 = ast.parse("__debug__", optimize=optval) + res2 = ast.parse(ast.parse("__debug__"), optimize=optval) + for res in [res1, res2]: + self.assertIsInstance(res.body[0], ast.Expr) + if isinstance(expected, bool): + self.assertIsInstance(res.body[0].value, ast.Constant) + self.assertEqual(res.body[0].value.value, expected) + else: + self.assertIsInstance(res.body[0].value, ast.Name) + self.assertEqual(res.body[0].value.id, expected) + + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags + def test_optimization_levels_const_folding(self): + folded = ("Expr", (1, 0, 1, 5), ("Constant", (1, 0, 1, 5), 3, None)) + not_folded = ( + "Expr", + (1, 0, 1, 5), + ( + "BinOp", + (1, 0, 1, 5), + ("Constant", (1, 0, 1, 1), 1, None), + ("Add",), + ("Constant", (1, 4, 1, 5), 2, None), + ), + ) + + cases = [(-1, not_folded), (0, not_folded), (1, folded), (2, folded)] + for optval, expected in cases: + with self.subTest(optval=optval): + tree1 = ast.parse("1 + 2", optimize=optval) + tree2 = ast.parse(ast.parse("1 + 2"), optimize=optval) + for tree in [tree1, tree2]: + res = to_tuple(tree.body[0]) + self.assertEqual(res, expected) + + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised + def test_invalid_position_information(self): + invalid_linenos = [(10, 1), (-10, -11), (10, -11), (-5, -2), (-5, 1)] + + for lineno, end_lineno in invalid_linenos: + with self.subTest(f"Check invalid linenos {lineno}:{end_lineno}"): + snippet = "a = 1" + tree = ast.parse(snippet) + tree.body[0].lineno = lineno + tree.body[0].end_lineno = end_lineno + with self.assertRaises(ValueError): + compile(tree, "", "exec") + + invalid_col_offsets = [(10, 1), (-10, -11), (10, -11), (-5, -2), (-5, 1)] + for col_offset, end_col_offset in invalid_col_offsets: + with self.subTest( + f"Check invalid col_offset {col_offset}:{end_col_offset}" + ): + snippet = "a = 1" + tree = ast.parse(snippet) + tree.body[0].col_offset = col_offset + tree.body[0].end_col_offset = end_col_offset + with self.assertRaises(ValueError): + compile(tree, "", "exec") + + def test_compilation_of_ast_nodes_with_default_end_position_values(self): + tree = ast.Module( + body=[ + ast.Import( + names=[ast.alias(name="builtins", lineno=1, col_offset=0)], + lineno=1, + col_offset=0, + ), + ast.Import( + names=[ast.alias(name="traceback", lineno=0, col_offset=0)], + lineno=0, + col_offset=1, + ), + ], + type_ignores=[], + ) + + # Check that compilation doesn't crash. Note: this may crash explicitly only on debug mode. + compile(tree, "", "exec") + + def test_negative_locations_for_compile(self): + # See https://github.com/python/cpython/issues/130775 + alias = ast.alias(name='traceback', lineno=0, col_offset=0) + for attrs in ( + {'lineno': -2, 'col_offset': 0}, + {'lineno': 0, 'col_offset': -2}, + {'lineno': 0, 'col_offset': -2, 'end_col_offset': -2}, + {'lineno': -2, 'end_lineno': -2, 'col_offset': 0}, + ): + with self.subTest(attrs=attrs): + tree = ast.Module(body=[ + ast.Import(names=[alias], **attrs) + ], type_ignores=[]) + + # It used to crash on this step: + compile(tree, "", "exec") + + # This also must not crash: + ast.parse(tree, optimize=2) + + def test_slice(self): + slc = ast.parse("x[::]").body[0].value.slice + self.assertIsNone(slc.upper) + self.assertIsNone(slc.lower) + self.assertIsNone(slc.step) + + def test_from_import(self): + im = ast.parse("from . import y").body[0] + self.assertIsNone(im.module) + + def test_non_interned_future_from_ast(self): + mod = ast.parse("from __future__ import division") + self.assertIsInstance(mod.body[0], ast.ImportFrom) + mod.body[0].module = " __future__ ".strip() + compile(mod, "", "exec") + + def test_alias(self): + im = ast.parse("from bar import y").body[0] + self.assertEqual(len(im.names), 1) + alias = im.names[0] + self.assertEqual(alias.name, "y") + self.assertIsNone(alias.asname) + self.assertEqual(alias.lineno, 1) + self.assertEqual(alias.end_lineno, 1) + self.assertEqual(alias.col_offset, 16) + self.assertEqual(alias.end_col_offset, 17) + + im = ast.parse("from bar import *").body[0] + alias = im.names[0] + self.assertEqual(alias.name, "*") + self.assertIsNone(alias.asname) + self.assertEqual(alias.lineno, 1) + self.assertEqual(alias.end_lineno, 1) + self.assertEqual(alias.col_offset, 16) + self.assertEqual(alias.end_col_offset, 17) + + im = ast.parse("from bar import y as z").body[0] + alias = im.names[0] + self.assertEqual(alias.name, "y") + self.assertEqual(alias.asname, "z") + self.assertEqual(alias.lineno, 1) + self.assertEqual(alias.end_lineno, 1) + self.assertEqual(alias.col_offset, 16) + self.assertEqual(alias.end_col_offset, 22) + + im = ast.parse("import bar as foo").body[0] + alias = im.names[0] + self.assertEqual(alias.name, "bar") + self.assertEqual(alias.asname, "foo") + self.assertEqual(alias.lineno, 1) + self.assertEqual(alias.end_lineno, 1) + self.assertEqual(alias.col_offset, 7) + self.assertEqual(alias.end_col_offset, 17) + + def test_base_classes(self): + self.assertTrue(issubclass(ast.For, ast.stmt)) + self.assertTrue(issubclass(ast.Name, ast.expr)) + self.assertTrue(issubclass(ast.stmt, ast.AST)) + self.assertTrue(issubclass(ast.expr, ast.AST)) + self.assertTrue(issubclass(ast.comprehension, ast.AST)) + self.assertTrue(issubclass(ast.Gt, ast.AST)) + + def test_import_deprecated(self): + ast = import_fresh_module("ast") + depr_regex = ( + r"ast\.{} is deprecated and will be removed in Python 3.14; " + r"use ast\.Constant instead" + ) + for name in "Num", "Str", "Bytes", "NameConstant", "Ellipsis": + with self.assertWarnsRegex(DeprecationWarning, depr_regex.format(name)): + getattr(ast, name) + + def test_field_attr_existence_deprecated(self): + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", "", DeprecationWarning) + from ast import Num, Str, Bytes, NameConstant, Ellipsis + + for name in ("Num", "Str", "Bytes", "NameConstant", "Ellipsis"): + item = getattr(ast, name) + if self._is_ast_node(name, item): + with self.subTest(item): + with self.assertWarns(DeprecationWarning): + x = item() + if isinstance(x, ast.AST): + self.assertIs(type(x._fields), tuple) + + @unittest.expectedFailure # TODO: RUSTPYTHON; type object 'Module' has no attribute '__annotations__' + def test_field_attr_existence(self): + for name, item in ast.__dict__.items(): + # These emit DeprecationWarnings + if name in {"Num", "Str", "Bytes", "NameConstant", "Ellipsis"}: + continue + # constructor has a different signature + if name == "Index": + continue + if self._is_ast_node(name, item): + x = self._construct_ast_class(item) + if isinstance(x, ast.AST): + self.assertIs(type(x._fields), tuple) + + def _construct_ast_class(self, cls): + kwargs = {} + for name, typ in cls.__annotations__.items(): + if typ is str: + kwargs[name] = "capybara" + elif typ is int: + kwargs[name] = 42 + elif typ is object: + kwargs[name] = b"capybara" + elif isinstance(typ, type) and issubclass(typ, ast.AST): + kwargs[name] = self._construct_ast_class(typ) + return cls(**kwargs) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_arguments(self): + x = ast.arguments() + self.assertEqual( + x._fields, + ( + "posonlyargs", + "args", + "vararg", + "kwonlyargs", + "kw_defaults", + "kwarg", + "defaults", + ), + ) + self.assertEqual( + x.__annotations__, + { + "posonlyargs": list[ast.arg], + "args": list[ast.arg], + "vararg": ast.arg | None, + "kwonlyargs": list[ast.arg], + "kw_defaults": list[ast.expr], + "kwarg": ast.arg | None, + "defaults": list[ast.expr], + }, + ) + + self.assertEqual(x.args, []) + self.assertIsNone(x.vararg) + + x = ast.arguments(*range(1, 8)) + self.assertEqual(x.args, 2) + self.assertEqual(x.vararg, 3) + + def test_field_attr_writable_deprecated(self): + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", "", DeprecationWarning) + x = ast.Num() + # We can assign to _fields + x._fields = 666 + self.assertEqual(x._fields, 666) + + def test_field_attr_writable(self): + x = ast.Constant(1) + # We can assign to _fields + x._fields = 666 + self.assertEqual(x._fields, 666) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_classattrs_deprecated(self): + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", "", DeprecationWarning) + from ast import Num, Str, Bytes, NameConstant, Ellipsis + + with warnings.catch_warnings(record=True) as wlog: + warnings.filterwarnings("always", "", DeprecationWarning) + x = ast.Num() + self.assertEqual(x._fields, ("value", "kind")) + + with self.assertRaises(AttributeError): + x.value + + with self.assertRaises(AttributeError): + x.n + + x = ast.Num(42) + self.assertEqual(x.value, 42) + self.assertEqual(x.n, 42) + + with self.assertRaises(AttributeError): + x.lineno + + with self.assertRaises(AttributeError): + x.foobar + + x = ast.Num(lineno=2) + self.assertEqual(x.lineno, 2) + + x = ast.Num(42, lineno=0) + self.assertEqual(x.lineno, 0) + self.assertEqual(x._fields, ("value", "kind")) + self.assertEqual(x.value, 42) + self.assertEqual(x.n, 42) + + self.assertRaises(TypeError, ast.Num, 1, None, 2) + self.assertRaises(TypeError, ast.Num, 1, None, 2, lineno=0) + + # Arbitrary keyword arguments are supported + self.assertEqual(ast.Num(1, foo="bar").foo, "bar") + + with self.assertRaisesRegex( + TypeError, "Num got multiple values for argument 'n'" + ): + ast.Num(1, n=2) + + self.assertEqual(ast.Num(42).n, 42) + self.assertEqual(ast.Num(4.25).n, 4.25) + self.assertEqual(ast.Num(4.25j).n, 4.25j) + self.assertEqual(ast.Str("42").s, "42") + self.assertEqual(ast.Bytes(b"42").s, b"42") + self.assertIs(ast.NameConstant(True).value, True) + self.assertIs(ast.NameConstant(False).value, False) + self.assertIs(ast.NameConstant(None).value, None) + + self.assertEqual( + [str(w.message) for w in wlog], + [ + "ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead", + "Constant.__init__ missing 1 required positional argument: 'value'. This will become " + "an error in Python 3.15.", + "Attribute n is deprecated and will be removed in Python 3.14; use value instead", + "ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead", + "Attribute n is deprecated and will be removed in Python 3.14; use value instead", + "ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead", + "Constant.__init__ missing 1 required positional argument: 'value'. This will become " + "an error in Python 3.15.", + "ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead", + "Attribute n is deprecated and will be removed in Python 3.14; use value instead", + "ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead", + "ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead", + "ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead", + "Constant.__init__ got an unexpected keyword argument 'foo'. Support for " + "arbitrary keyword arguments is deprecated and will be removed in Python " + "3.15.", + "ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead", + "Attribute n is deprecated and will be removed in Python 3.14; use value instead", + "ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead", + "Attribute n is deprecated and will be removed in Python 3.14; use value instead", + "ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead", + "Attribute n is deprecated and will be removed in Python 3.14; use value instead", + "ast.Str is deprecated and will be removed in Python 3.14; use ast.Constant instead", + "Attribute s is deprecated and will be removed in Python 3.14; use value instead", + "ast.Bytes is deprecated and will be removed in Python 3.14; use ast.Constant instead", + "Attribute s is deprecated and will be removed in Python 3.14; use value instead", + "ast.NameConstant is deprecated and will be removed in Python 3.14; use ast.Constant instead", + "ast.NameConstant is deprecated and will be removed in Python 3.14; use ast.Constant instead", + "ast.NameConstant is deprecated and will be removed in Python 3.14; use ast.Constant instead", + ], + ) + + @unittest.expectedFailure # TODO: RUSTPYTHON; DeprecationWarning not triggered + def test_classattrs(self): + with self.assertWarns(DeprecationWarning): + x = ast.Constant() + self.assertEqual(x._fields, ("value", "kind")) + + with self.assertRaises(AttributeError): + x.value + + x = ast.Constant(42) + self.assertEqual(x.value, 42) + + with self.assertRaises(AttributeError): + x.lineno + + with self.assertRaises(AttributeError): + x.foobar + + x = ast.Constant(lineno=2, value=3) + self.assertEqual(x.lineno, 2) + + x = ast.Constant(42, lineno=0) + self.assertEqual(x.lineno, 0) + self.assertEqual(x._fields, ("value", "kind")) + self.assertEqual(x.value, 42) + + self.assertRaises(TypeError, ast.Constant, 1, None, 2) + self.assertRaises(TypeError, ast.Constant, 1, None, 2, lineno=0) + + # Arbitrary keyword arguments are supported (but deprecated) + with self.assertWarns(DeprecationWarning): + self.assertEqual(ast.Constant(1, foo="bar").foo, "bar") + + with self.assertRaisesRegex( + TypeError, "Constant got multiple values for argument 'value'" + ): + ast.Constant(1, value=2) + + self.assertEqual(ast.Constant(42).value, 42) + self.assertEqual(ast.Constant(4.25).value, 4.25) + self.assertEqual(ast.Constant(4.25j).value, 4.25j) + self.assertEqual(ast.Constant("42").value, "42") + self.assertEqual(ast.Constant(b"42").value, b"42") + self.assertIs(ast.Constant(True).value, True) + self.assertIs(ast.Constant(False).value, False) + self.assertIs(ast.Constant(None).value, None) + self.assertIs(ast.Constant(...).value, ...) + + def test_realtype(self): + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", "", DeprecationWarning) + from ast import Num, Str, Bytes, NameConstant, Ellipsis + + with warnings.catch_warnings(record=True) as wlog: + warnings.filterwarnings("always", "", DeprecationWarning) + self.assertIs(type(ast.Num(42)), ast.Constant) + self.assertIs(type(ast.Num(4.25)), ast.Constant) + self.assertIs(type(ast.Num(4.25j)), ast.Constant) + self.assertIs(type(ast.Str("42")), ast.Constant) + self.assertIs(type(ast.Bytes(b"42")), ast.Constant) + self.assertIs(type(ast.NameConstant(True)), ast.Constant) + self.assertIs(type(ast.NameConstant(False)), ast.Constant) + self.assertIs(type(ast.NameConstant(None)), ast.Constant) + self.assertIs(type(ast.Ellipsis()), ast.Constant) + + self.assertEqual( + [str(w.message) for w in wlog], + [ + "ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead", + "ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead", + "ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead", + "ast.Str is deprecated and will be removed in Python 3.14; use ast.Constant instead", + "ast.Bytes is deprecated and will be removed in Python 3.14; use ast.Constant instead", + "ast.NameConstant is deprecated and will be removed in Python 3.14; use ast.Constant instead", + "ast.NameConstant is deprecated and will be removed in Python 3.14; use ast.Constant instead", + "ast.NameConstant is deprecated and will be removed in Python 3.14; use ast.Constant instead", + "ast.Ellipsis is deprecated and will be removed in Python 3.14; use ast.Constant instead", + ], + ) + + def test_isinstance(self): + from ast import Constant + + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", "", DeprecationWarning) + from ast import Num, Str, Bytes, NameConstant, Ellipsis + + cls_depr_msg = ( + "ast.{} is deprecated and will be removed in Python 3.14; " + "use ast.Constant instead" + ) + + assertNumDeprecated = partial( + self.assertWarnsRegex, DeprecationWarning, cls_depr_msg.format("Num") + ) + assertStrDeprecated = partial( + self.assertWarnsRegex, DeprecationWarning, cls_depr_msg.format("Str") + ) + assertBytesDeprecated = partial( + self.assertWarnsRegex, DeprecationWarning, cls_depr_msg.format("Bytes") + ) + assertNameConstantDeprecated = partial( + self.assertWarnsRegex, + DeprecationWarning, + cls_depr_msg.format("NameConstant"), + ) + assertEllipsisDeprecated = partial( + self.assertWarnsRegex, DeprecationWarning, cls_depr_msg.format("Ellipsis") + ) + + for arg in 42, 4.2, 4.2j: + with self.subTest(arg=arg): + with assertNumDeprecated(): + n = Num(arg) + with assertNumDeprecated(): + self.assertIsInstance(n, Num) + + with assertStrDeprecated(): + s = Str("42") + with assertStrDeprecated(): + self.assertIsInstance(s, Str) + + with assertBytesDeprecated(): + b = Bytes(b"42") + with assertBytesDeprecated(): + self.assertIsInstance(b, Bytes) + + for arg in True, False, None: + with self.subTest(arg=arg): + with assertNameConstantDeprecated(): + n = NameConstant(arg) + with assertNameConstantDeprecated(): + self.assertIsInstance(n, NameConstant) + + with assertEllipsisDeprecated(): + e = Ellipsis() + with assertEllipsisDeprecated(): + self.assertIsInstance(e, Ellipsis) + + for arg in 42, 4.2, 4.2j: + with self.subTest(arg=arg): + with assertNumDeprecated(): + self.assertIsInstance(Constant(arg), Num) + + with assertStrDeprecated(): + self.assertIsInstance(Constant("42"), Str) + + with assertBytesDeprecated(): + self.assertIsInstance(Constant(b"42"), Bytes) + + for arg in True, False, None: + with self.subTest(arg=arg): + with assertNameConstantDeprecated(): + self.assertIsInstance(Constant(arg), NameConstant) + + with assertEllipsisDeprecated(): + self.assertIsInstance(Constant(...), Ellipsis) + + with assertStrDeprecated(): + s = Str("42") + assertNumDeprecated(self.assertNotIsInstance, s, Num) + assertBytesDeprecated(self.assertNotIsInstance, s, Bytes) + + with assertNumDeprecated(): + n = Num(42) + assertStrDeprecated(self.assertNotIsInstance, n, Str) + assertNameConstantDeprecated(self.assertNotIsInstance, n, NameConstant) + assertEllipsisDeprecated(self.assertNotIsInstance, n, Ellipsis) + + with assertNameConstantDeprecated(): + n = NameConstant(True) + with assertNumDeprecated(): + self.assertNotIsInstance(n, Num) + + with assertNameConstantDeprecated(): + n = NameConstant(False) + with assertNumDeprecated(): + self.assertNotIsInstance(n, Num) + + for arg in "42", True, False: + with self.subTest(arg=arg): + with assertNumDeprecated(): + self.assertNotIsInstance(Constant(arg), Num) + + assertStrDeprecated(self.assertNotIsInstance, Constant(42), Str) + assertBytesDeprecated(self.assertNotIsInstance, Constant("42"), Bytes) + assertNameConstantDeprecated( + self.assertNotIsInstance, Constant(42), NameConstant + ) + assertEllipsisDeprecated(self.assertNotIsInstance, Constant(42), Ellipsis) + assertNumDeprecated(self.assertNotIsInstance, Constant(None), Num) + assertStrDeprecated(self.assertNotIsInstance, Constant(None), Str) + assertBytesDeprecated(self.assertNotIsInstance, Constant(None), Bytes) + assertNameConstantDeprecated( + self.assertNotIsInstance, Constant(1), NameConstant + ) + assertEllipsisDeprecated(self.assertNotIsInstance, Constant(None), Ellipsis) + + class S(str): + pass + + with assertStrDeprecated(): + self.assertIsInstance(Constant(S("42")), Str) + with assertNumDeprecated(): + self.assertNotIsInstance(Constant(S("42")), Num) + + @unittest.expectedFailure # TODO: RUSTPYTHON; will be removed in Python 3.14 + def test_constant_subclasses_deprecated(self): + with warnings.catch_warnings(): + warnings.filterwarnings("ignore", "", DeprecationWarning) + from ast import Num + + with warnings.catch_warnings(record=True) as wlog: + warnings.filterwarnings("always", "", DeprecationWarning) + + class N(ast.Num): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.z = "spam" + + class N2(ast.Num): + pass + + n = N(42) + self.assertEqual(n.n, 42) + self.assertEqual(n.z, "spam") + self.assertIs(type(n), N) + self.assertIsInstance(n, N) + self.assertIsInstance(n, ast.Num) + self.assertNotIsInstance(n, N2) + self.assertNotIsInstance(ast.Num(42), N) + n = N(n=42) + self.assertEqual(n.n, 42) + self.assertIs(type(n), N) + + self.assertEqual( + [str(w.message) for w in wlog], + [ + "Attribute n is deprecated and will be removed in Python 3.14; use value instead", + "Attribute n is deprecated and will be removed in Python 3.14; use value instead", + "ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead", + "ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead", + "Attribute n is deprecated and will be removed in Python 3.14; use value instead", + "Attribute n is deprecated and will be removed in Python 3.14; use value instead", + ], + ) + + def test_constant_subclasses(self): + class N(ast.Constant): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + self.z = "spam" + + class N2(ast.Constant): + pass + + n = N(42) + self.assertEqual(n.value, 42) + self.assertEqual(n.z, "spam") + self.assertEqual(type(n), N) + self.assertTrue(isinstance(n, N)) + self.assertTrue(isinstance(n, ast.Constant)) + self.assertFalse(isinstance(n, N2)) + self.assertFalse(isinstance(ast.Constant(42), N)) + n = N(value=42) + self.assertEqual(n.value, 42) + self.assertEqual(type(n), N) + + def test_module(self): + body = [ast.Constant(42)] + x = ast.Module(body, []) + self.assertEqual(x.body, body) + + @unittest.expectedFailure # TODO: RUSTPYTHON; DeprecationWarning not triggered + def test_nodeclasses(self): + # Zero arguments constructor explicitly allowed (but deprecated) + with self.assertWarns(DeprecationWarning): + x = ast.BinOp() + self.assertEqual(x._fields, ("left", "op", "right")) + + # Random attribute allowed too + x.foobarbaz = 5 + self.assertEqual(x.foobarbaz, 5) + + n1 = ast.Constant(1) + n3 = ast.Constant(3) + addop = ast.Add() + x = ast.BinOp(n1, addop, n3) + self.assertEqual(x.left, n1) + self.assertEqual(x.op, addop) + self.assertEqual(x.right, n3) + + x = ast.BinOp(1, 2, 3) + self.assertEqual(x.left, 1) + self.assertEqual(x.op, 2) + self.assertEqual(x.right, 3) + + x = ast.BinOp(1, 2, 3, lineno=0) + self.assertEqual(x.left, 1) + self.assertEqual(x.op, 2) + self.assertEqual(x.right, 3) + self.assertEqual(x.lineno, 0) + + # node raises exception when given too many arguments + self.assertRaises(TypeError, ast.BinOp, 1, 2, 3, 4) + # node raises exception when given too many arguments + self.assertRaises(TypeError, ast.BinOp, 1, 2, 3, 4, lineno=0) + + # can set attributes through kwargs too + x = ast.BinOp(left=1, op=2, right=3, lineno=0) + self.assertEqual(x.left, 1) + self.assertEqual(x.op, 2) + self.assertEqual(x.right, 3) + self.assertEqual(x.lineno, 0) + + # Random kwargs also allowed (but deprecated) + with self.assertWarns(DeprecationWarning): + x = ast.BinOp(1, 2, 3, foobarbaz=42) + self.assertEqual(x.foobarbaz, 42) + + def test_no_fields(self): + # this used to fail because Sub._fields was None + x = ast.Sub() + self.assertEqual(x._fields, ()) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_invalid_sum(self): + pos = dict(lineno=2, col_offset=3) + m = ast.Module([ast.Expr(ast.expr(**pos), **pos)], []) + with self.assertRaises(TypeError) as cm: + compile(m, "", "exec") + self.assertIn("but got ", "exec") + self.assertIn("identifier must be of type str", str(cm.exception)) + + def test_invalid_constant(self): + for invalid_constant in int, (1, 2, int), frozenset((1, 2, int)): + e = ast.Expression(body=ast.Constant(invalid_constant)) + ast.fix_missing_locations(e) + with self.assertRaisesRegex(TypeError, "invalid type in Constant: type"): + compile(e, "", "eval") + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_empty_yield_from(self): + # Issue 16546: yield from value is not optional. + empty_yield_from = ast.parse("def f():\n yield from g()") + empty_yield_from.body[0].body[0].value.value = None + with self.assertRaises(ValueError) as cm: + compile(empty_yield_from, "", "exec") + self.assertIn("field 'value' is required", str(cm.exception)) + + @support.cpython_only + def test_issue31592(self): + # There shouldn't be an assertion failure in case of a bad + # unicodedata.normalize(). + import unicodedata + + def bad_normalize(*args): + return None + + with support.swap_attr(unicodedata, "normalize", bad_normalize): + self.assertRaises(TypeError, ast.parse, "\u03d5") + + def test_issue18374_binop_col_offset(self): + tree = ast.parse("4+5+6+7") + parent_binop = tree.body[0].value + child_binop = parent_binop.left + grandchild_binop = child_binop.left + self.assertEqual(parent_binop.col_offset, 0) + self.assertEqual(parent_binop.end_col_offset, 7) + self.assertEqual(child_binop.col_offset, 0) + self.assertEqual(child_binop.end_col_offset, 5) + self.assertEqual(grandchild_binop.col_offset, 0) + self.assertEqual(grandchild_binop.end_col_offset, 3) + + tree = ast.parse("4+5-\\\n 6-7") + parent_binop = tree.body[0].value + child_binop = parent_binop.left + grandchild_binop = child_binop.left + self.assertEqual(parent_binop.col_offset, 0) + self.assertEqual(parent_binop.lineno, 1) + self.assertEqual(parent_binop.end_col_offset, 4) + self.assertEqual(parent_binop.end_lineno, 2) + + self.assertEqual(child_binop.col_offset, 0) + self.assertEqual(child_binop.lineno, 1) + self.assertEqual(child_binop.end_col_offset, 2) + self.assertEqual(child_binop.end_lineno, 2) + + self.assertEqual(grandchild_binop.col_offset, 0) + self.assertEqual(grandchild_binop.lineno, 1) + self.assertEqual(grandchild_binop.end_col_offset, 3) + self.assertEqual(grandchild_binop.end_lineno, 1) + + def test_issue39579_dotted_name_end_col_offset(self): + tree = ast.parse("@a.b.c\ndef f(): pass") + attr_b = tree.body[0].decorator_list[0].value + self.assertEqual(attr_b.end_col_offset, 4) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_ast_asdl_signature(self): + self.assertEqual( + ast.withitem.__doc__, "withitem(expr context_expr, expr? optional_vars)" + ) + self.assertEqual(ast.GtE.__doc__, "GtE") + self.assertEqual(ast.Name.__doc__, "Name(identifier id, expr_context ctx)") + self.assertEqual( + ast.cmpop.__doc__, + "cmpop = Eq | NotEq | Lt | LtE | Gt | GtE | Is | IsNot | In | NotIn", + ) + expressions = [f" | {node.__doc__}" for node in ast.expr.__subclasses__()] + expressions[0] = f"expr = {ast.expr.__subclasses__()[0].__doc__}" + self.assertCountEqual(ast.expr.__doc__.split("\n"), expressions) + + @unittest.expectedFailure # TODO: RUSTPYTHON; SyntaxError not raised + def test_positional_only_feature_version(self): + ast.parse("def foo(x, /): ...", feature_version=(3, 8)) + ast.parse("def bar(x=1, /): ...", feature_version=(3, 8)) + with self.assertRaises(SyntaxError): + ast.parse("def foo(x, /): ...", feature_version=(3, 7)) + with self.assertRaises(SyntaxError): + ast.parse("def bar(x=1, /): ...", feature_version=(3, 7)) + + ast.parse("lambda x, /: ...", feature_version=(3, 8)) + ast.parse("lambda x=1, /: ...", feature_version=(3, 8)) + with self.assertRaises(SyntaxError): + ast.parse("lambda x, /: ...", feature_version=(3, 7)) + with self.assertRaises(SyntaxError): + ast.parse("lambda x=1, /: ...", feature_version=(3, 7)) + + @unittest.expectedFailure # TODO: RUSTPYTHON; SyntaxError not raised + def test_assignment_expression_feature_version(self): + ast.parse("(x := 0)", feature_version=(3, 8)) + with self.assertRaises(SyntaxError): + ast.parse("(x := 0)", feature_version=(3, 7)) + + def test_conditional_context_managers_parse_with_low_feature_version(self): + # regression test for gh-115881 + ast.parse("with (x() if y else z()): ...", feature_version=(3, 8)) + + @unittest.expectedFailure # TODO: RUSTPYTHON; SyntaxError not raised + def test_exception_groups_feature_version(self): + code = dedent(""" + try: ... + except* Exception: ... + """) + ast.parse(code) + with self.assertRaises(SyntaxError): + ast.parse(code, feature_version=(3, 10)) + + @unittest.expectedFailure # TODO: RUSTPYTHON; SyntaxError not raised + def test_type_params_feature_version(self): + samples = [ + "type X = int", + "class X[T]: pass", + "def f[T](): pass", + ] + for sample in samples: + with self.subTest(sample): + ast.parse(sample) + with self.assertRaises(SyntaxError): + ast.parse(sample, feature_version=(3, 11)) + + @unittest.expectedFailure # TODO: RUSTPYTHON; SyntaxError not raised + def test_type_params_default_feature_version(self): + samples = [ + "type X[*Ts=int] = int", + "class X[T=int]: pass", + "def f[**P=int](): pass", + ] + for sample in samples: + with self.subTest(sample): + ast.parse(sample) + with self.assertRaises(SyntaxError): + ast.parse(sample, feature_version=(3, 12)) + + def test_invalid_major_feature_version(self): + with self.assertRaises(ValueError): + ast.parse("pass", feature_version=(2, 7)) + with self.assertRaises(ValueError): + ast.parse("pass", feature_version=(4, 0)) + + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised + def test_constant_as_name(self): + for constant in "True", "False", "None": + expr = ast.Expression(ast.Name(constant, ast.Load())) + ast.fix_missing_locations(expr) + with self.assertRaisesRegex( + ValueError, f"identifier field can't represent '{constant}' constant" + ): + compile(expr, "", "eval") + + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised + def test_constant_as_unicode_name(self): + constants = [ + ("True", b"Tru\xe1\xb5\x89"), + ("False", b"Fal\xc5\xbfe"), + ("None", b"N\xc2\xbane"), + ] + for constant in constants: + with self.assertRaisesRegex(ValueError, + f"identifier field can't represent '{constant[0]}' constant"): + ast.parse(constant[1], mode="eval") + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_precedence_enum(self): + class _Precedence(enum.IntEnum): + """Precedence table that originated from python grammar.""" + + NAMED_EXPR = enum.auto() # := + TUPLE = enum.auto() # , + YIELD = enum.auto() # 'yield', 'yield from' + TEST = enum.auto() # 'if'-'else', 'lambda' + OR = enum.auto() # 'or' + AND = enum.auto() # 'and' + NOT = enum.auto() # 'not' + CMP = enum.auto() # '<', '>', '==', '>=', '<=', '!=', + # 'in', 'not in', 'is', 'is not' + EXPR = enum.auto() + BOR = EXPR # '|' + BXOR = enum.auto() # '^' + BAND = enum.auto() # '&' + SHIFT = enum.auto() # '<<', '>>' + ARITH = enum.auto() # '+', '-' + TERM = enum.auto() # '*', '@', '/', '%', '//' + FACTOR = enum.auto() # unary '+', '-', '~' + POWER = enum.auto() # '**' + AWAIT = enum.auto() # 'await' + ATOM = enum.auto() + + def next(self): + try: + return self.__class__(self + 1) + except ValueError: + return self + + enum._test_simple_enum(_Precedence, ast._Precedence) + + @support.cpython_only + def test_ast_recursion_limit(self): + fail_depth = support.exceeds_recursion_limit() + crash_depth = 100_000 + success_depth = int(support.get_c_recursion_limit() * 0.8) + if _testinternalcapi is not None: + remaining = _testinternalcapi.get_c_recursion_remaining() + success_depth = min(success_depth, remaining) + + def check_limit(prefix, repeated): + expect_ok = prefix + repeated * success_depth + ast.parse(expect_ok) + for depth in (fail_depth, crash_depth): + broken = prefix + repeated * depth + details = "Compiling ({!r} + {!r} * {})".format(prefix, repeated, depth) + with self.assertRaises(RecursionError, msg=details): + with support.infinite_recursion(): + ast.parse(broken) + + check_limit("a", "()") + check_limit("a", ".b") + check_limit("a", "[0]") + check_limit("a", "*a") + + def test_null_bytes(self): + with self.assertRaises( + SyntaxError, msg="source code string cannot contain null bytes" + ): + ast.parse("a\0b") + + def assert_none_check(self, node: type[ast.AST], attr: str, source: str) -> None: + with self.subTest(f"{node.__name__}.{attr}"): + tree = ast.parse(source) + found = 0 + for child in ast.walk(tree): + if isinstance(child, node): + setattr(child, attr, None) + found += 1 + self.assertEqual(found, 1) + e = re.escape(f"field '{attr}' is required for {node.__name__}") + with self.assertRaisesRegex(ValueError, f"^{e}$"): + compile(tree, "", "exec") + + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: expected some sort of expr, but got None + def test_none_checks(self) -> None: + tests = [ + (ast.alias, "name", "import spam as SPAM"), + (ast.arg, "arg", "def spam(SPAM): spam"), + (ast.comprehension, "target", "[spam for SPAM in spam]"), + (ast.comprehension, "iter", "[spam for spam in SPAM]"), + (ast.keyword, "value", "spam(**SPAM)"), + (ast.match_case, "pattern", "match spam:\n case SPAM: spam"), + (ast.withitem, "context_expr", "with SPAM: spam"), + ] + for node, attr, source in tests: + self.assert_none_check(node, attr, source) + + +class CopyTests(unittest.TestCase): + """Test copying and pickling AST nodes.""" + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_pickling(self): + import pickle + + for protocol in range(pickle.HIGHEST_PROTOCOL + 1): + for code in exec_tests: + with self.subTest(code=code, protocol=protocol): + tree = compile(code, "?", "exec", 0x400) + ast2 = pickle.loads(pickle.dumps(tree, protocol)) + self.assertEqual(to_tuple(ast2), to_tuple(tree)) + + def test_copy_with_parents(self): + # gh-120108 + code = """ + ('',) + while i < n: + if ch == '': + ch = format[i] + if ch == '': + if freplace is None: + '' % getattr(object) + elif ch == '': + if zreplace is None: + if hasattr: + offset = object.utcoffset() + if offset is not None: + if offset.days < 0: + offset = -offset + h = divmod(timedelta(hours=0)) + if u: + zreplace = '' % (sign,) + elif s: + zreplace = '' % (sign,) + else: + zreplace = '' % (sign,) + elif ch == '': + if Zreplace is None: + Zreplace = '' + if hasattr(object): + s = object.tzname() + if s is not None: + Zreplace = s.replace('') + newformat.append(Zreplace) + else: + push('') + else: + push(ch) + + """ + tree = ast.parse(textwrap.dedent(code)) + for node in ast.walk(tree): + for child in ast.iter_child_nodes(node): + child.parent = node + try: + with support.infinite_recursion(200): + tree2 = copy.deepcopy(tree) + finally: + # Singletons like ast.Load() are shared; make sure we don't + # leave them mutated after this test. + for node in ast.walk(tree): + if hasattr(node, "parent"): + del node.parent + + for node in ast.walk(tree2): + for child in ast.iter_child_nodes(node): + if hasattr(child, "parent") and not isinstance( + child, + ( + ast.expr_context, + ast.boolop, + ast.unaryop, + ast.cmpop, + ast.operator, + ), + ): + self.assertEqual(to_tuple(child.parent), to_tuple(node)) + + +class ASTHelpers_Test(unittest.TestCase): + maxDiff = None + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_parse(self): + a = ast.parse("foo(1 + 1)") + b = compile("foo(1 + 1)", "", "exec", ast.PyCF_ONLY_AST) + self.assertEqual(ast.dump(a), ast.dump(b)) + + def test_parse_in_error(self): + try: + 1 / 0 + except Exception: + with self.assertRaises(SyntaxError) as e: + ast.literal_eval(r"'\U'") + self.assertIsNotNone(e.exception.__context__) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_dump(self): + node = ast.parse('spam(eggs, "and cheese")') + self.assertEqual( + ast.dump(node), + "Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load()), " + "args=[Name(id='eggs', ctx=Load()), Constant(value='and cheese')]))])", + ) + self.assertEqual( + ast.dump(node, annotate_fields=False), + "Module([Expr(Call(Name('spam', Load()), [Name('eggs', Load()), " + "Constant('and cheese')]))])", + ) + self.assertEqual( + ast.dump(node, include_attributes=True), + "Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load(), " + "lineno=1, col_offset=0, end_lineno=1, end_col_offset=4), " + "args=[Name(id='eggs', ctx=Load(), lineno=1, col_offset=5, " + "end_lineno=1, end_col_offset=9), Constant(value='and cheese', " + "lineno=1, col_offset=11, end_lineno=1, end_col_offset=23)], " + "lineno=1, col_offset=0, end_lineno=1, end_col_offset=24), " + "lineno=1, col_offset=0, end_lineno=1, end_col_offset=24)])", + ) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_dump_indent(self): + node = ast.parse('spam(eggs, "and cheese")') + self.assertEqual( + ast.dump(node, indent=3), + """\ +Module( + body=[ + Expr( + value=Call( + func=Name(id='spam', ctx=Load()), + args=[ + Name(id='eggs', ctx=Load()), + Constant(value='and cheese')]))])""", + ) + self.assertEqual( + ast.dump(node, annotate_fields=False, indent="\t"), + """\ +Module( +\t[ +\t\tExpr( +\t\t\tCall( +\t\t\t\tName('spam', Load()), +\t\t\t\t[ +\t\t\t\t\tName('eggs', Load()), +\t\t\t\t\tConstant('and cheese')]))])""", + ) + self.assertEqual( + ast.dump(node, include_attributes=True, indent=3), + """\ +Module( + body=[ + Expr( + value=Call( + func=Name( + id='spam', + ctx=Load(), + lineno=1, + col_offset=0, + end_lineno=1, + end_col_offset=4), + args=[ + Name( + id='eggs', + ctx=Load(), + lineno=1, + col_offset=5, + end_lineno=1, + end_col_offset=9), + Constant( + value='and cheese', + lineno=1, + col_offset=11, + end_lineno=1, + end_col_offset=23)], + lineno=1, + col_offset=0, + end_lineno=1, + end_col_offset=24), + lineno=1, + col_offset=0, + end_lineno=1, + end_col_offset=24)])""", + ) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_dump_incomplete(self): + node = ast.Raise(lineno=3, col_offset=4) + self.assertEqual(ast.dump(node), "Raise()") + self.assertEqual( + ast.dump(node, include_attributes=True), "Raise(lineno=3, col_offset=4)" + ) + node = ast.Raise(exc=ast.Name(id="e", ctx=ast.Load()), lineno=3, col_offset=4) + self.assertEqual(ast.dump(node), "Raise(exc=Name(id='e', ctx=Load()))") + self.assertEqual( + ast.dump(node, annotate_fields=False), "Raise(Name('e', Load()))" + ) + self.assertEqual( + ast.dump(node, include_attributes=True), + "Raise(exc=Name(id='e', ctx=Load()), lineno=3, col_offset=4)", + ) + self.assertEqual( + ast.dump(node, annotate_fields=False, include_attributes=True), + "Raise(Name('e', Load()), lineno=3, col_offset=4)", + ) + node = ast.Raise(cause=ast.Name(id="e", ctx=ast.Load())) + self.assertEqual(ast.dump(node), "Raise(cause=Name(id='e', ctx=Load()))") + self.assertEqual( + ast.dump(node, annotate_fields=False), "Raise(cause=Name('e', Load()))" + ) + # Arguments: + node = ast.arguments(args=[ast.arg("x")]) + self.assertEqual( + ast.dump(node, annotate_fields=False), + "arguments([], [arg('x')])", + ) + node = ast.arguments(posonlyargs=[ast.arg("x")]) + self.assertEqual( + ast.dump(node, annotate_fields=False), + "arguments([arg('x')])", + ) + node = ast.arguments(posonlyargs=[ast.arg("x")], kwonlyargs=[ast.arg("y")]) + self.assertEqual( + ast.dump(node, annotate_fields=False), + "arguments([arg('x')], kwonlyargs=[arg('y')])", + ) + node = ast.arguments(args=[ast.arg("x")], kwonlyargs=[ast.arg("y")]) + self.assertEqual( + ast.dump(node, annotate_fields=False), + "arguments([], [arg('x')], kwonlyargs=[arg('y')])", + ) + node = ast.arguments() + self.assertEqual( + ast.dump(node, annotate_fields=False), + "arguments()", + ) + # Classes: + node = ast.ClassDef( + "T", + [], + [ast.keyword("a", ast.Constant(None))], + [], + [ast.Name("dataclass", ctx=ast.Load())], + ) + self.assertEqual( + ast.dump(node), + "ClassDef(name='T', keywords=[keyword(arg='a', value=Constant(value=None))], decorator_list=[Name(id='dataclass', ctx=Load())])", + ) + self.assertEqual( + ast.dump(node, annotate_fields=False), + "ClassDef('T', [], [keyword('a', Constant(None))], [], [Name('dataclass', Load())])", + ) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_dump_show_empty(self): + def check_node(node, empty, full, **kwargs): + with self.subTest(show_empty=False): + self.assertEqual( + ast.dump(node, show_empty=False, **kwargs), + empty, + ) + with self.subTest(show_empty=True): + self.assertEqual( + ast.dump(node, show_empty=True, **kwargs), + full, + ) + + def check_text(code, empty, full, **kwargs): + check_node(ast.parse(code), empty, full, **kwargs) + + check_node( + ast.arguments(), + empty="arguments()", + full="arguments(posonlyargs=[], args=[], kwonlyargs=[], kw_defaults=[], defaults=[])", + ) + + check_node( + # Corner case: there are no real `Name` instances with `id=''`: + ast.Name(id="", ctx=ast.Load()), + empty="Name(id='', ctx=Load())", + full="Name(id='', ctx=Load())", + ) + + check_node( + ast.MatchSingleton(value=None), + empty="MatchSingleton(value=None)", + full="MatchSingleton(value=None)", + ) + + check_node( + ast.MatchSingleton(value=[]), + empty="MatchSingleton(value=[])", + full="MatchSingleton(value=[])", + ) + + check_node( + ast.Constant(value=None), + empty="Constant(value=None)", + full="Constant(value=None)", + ) + + check_node( + ast.Constant(value=[]), + empty="Constant(value=[])", + full="Constant(value=[])", + ) + + check_node( + ast.Constant(value=""), + empty="Constant(value='')", + full="Constant(value='')", + ) + + check_text( + "def a(b: int = 0, *, c): ...", + empty="Module(body=[FunctionDef(name='a', args=arguments(args=[arg(arg='b', annotation=Name(id='int', ctx=Load()))], kwonlyargs=[arg(arg='c')], kw_defaults=[None], defaults=[Constant(value=0)]), body=[Expr(value=Constant(value=Ellipsis))])])", + full="Module(body=[FunctionDef(name='a', args=arguments(posonlyargs=[], args=[arg(arg='b', annotation=Name(id='int', ctx=Load()))], kwonlyargs=[arg(arg='c')], kw_defaults=[None], defaults=[Constant(value=0)]), body=[Expr(value=Constant(value=Ellipsis))], decorator_list=[], type_params=[])], type_ignores=[])", + ) + + check_text( + "def a(b: int = 0, *, c): ...", + empty="Module(body=[FunctionDef(name='a', args=arguments(args=[arg(arg='b', annotation=Name(id='int', ctx=Load(), lineno=1, col_offset=9, end_lineno=1, end_col_offset=12), lineno=1, col_offset=6, end_lineno=1, end_col_offset=12)], kwonlyargs=[arg(arg='c', lineno=1, col_offset=21, end_lineno=1, end_col_offset=22)], kw_defaults=[None], defaults=[Constant(value=0, lineno=1, col_offset=15, end_lineno=1, end_col_offset=16)]), body=[Expr(value=Constant(value=Ellipsis, lineno=1, col_offset=25, end_lineno=1, end_col_offset=28), lineno=1, col_offset=25, end_lineno=1, end_col_offset=28)], lineno=1, col_offset=0, end_lineno=1, end_col_offset=28)])", + full="Module(body=[FunctionDef(name='a', args=arguments(posonlyargs=[], args=[arg(arg='b', annotation=Name(id='int', ctx=Load(), lineno=1, col_offset=9, end_lineno=1, end_col_offset=12), lineno=1, col_offset=6, end_lineno=1, end_col_offset=12)], kwonlyargs=[arg(arg='c', lineno=1, col_offset=21, end_lineno=1, end_col_offset=22)], kw_defaults=[None], defaults=[Constant(value=0, lineno=1, col_offset=15, end_lineno=1, end_col_offset=16)]), body=[Expr(value=Constant(value=Ellipsis, lineno=1, col_offset=25, end_lineno=1, end_col_offset=28), lineno=1, col_offset=25, end_lineno=1, end_col_offset=28)], decorator_list=[], type_params=[], lineno=1, col_offset=0, end_lineno=1, end_col_offset=28)], type_ignores=[])", + include_attributes=True, + ) + + check_text( + 'spam(eggs, "and cheese")', + empty="Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load()), args=[Name(id='eggs', ctx=Load()), Constant(value='and cheese')]))])", + full="Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load()), args=[Name(id='eggs', ctx=Load()), Constant(value='and cheese')], keywords=[]))], type_ignores=[])", + ) + + check_text( + 'spam(eggs, text="and cheese")', + empty="Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load()), args=[Name(id='eggs', ctx=Load())], keywords=[keyword(arg='text', value=Constant(value='and cheese'))]))])", + full="Module(body=[Expr(value=Call(func=Name(id='spam', ctx=Load()), args=[Name(id='eggs', ctx=Load())], keywords=[keyword(arg='text', value=Constant(value='and cheese'))]))], type_ignores=[])", + ) + + check_text( + "import _ast as ast; from module import sub", + empty="Module(body=[Import(names=[alias(name='_ast', asname='ast')]), ImportFrom(module='module', names=[alias(name='sub')], level=0)])", + full="Module(body=[Import(names=[alias(name='_ast', asname='ast')]), ImportFrom(module='module', names=[alias(name='sub')], level=0)], type_ignores=[])", + ) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_copy_location(self): + src = ast.parse("1 + 1", mode="eval") + src.body.right = ast.copy_location(ast.Constant(2), src.body.right) + self.assertEqual( + ast.dump(src, include_attributes=True), + "Expression(body=BinOp(left=Constant(value=1, lineno=1, col_offset=0, " + "end_lineno=1, end_col_offset=1), op=Add(), right=Constant(value=2, " + "lineno=1, col_offset=4, end_lineno=1, end_col_offset=5), lineno=1, " + "col_offset=0, end_lineno=1, end_col_offset=5))", + ) + func = ast.Name("spam", ast.Load()) + src = ast.Call( + col_offset=1, lineno=1, end_lineno=1, end_col_offset=1, func=func + ) + new = ast.copy_location(src, ast.Call(col_offset=None, lineno=None, func=func)) + self.assertIsNone(new.end_lineno) + self.assertIsNone(new.end_col_offset) + self.assertEqual(new.lineno, 1) + self.assertEqual(new.col_offset, 1) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_fix_missing_locations(self): + src = ast.parse('write("spam")') + src.body.append( + ast.Expr(ast.Call(ast.Name("spam", ast.Load()), [ast.Constant("eggs")], [])) + ) + self.assertEqual(src, ast.fix_missing_locations(src)) + self.maxDiff = None + self.assertEqual( + ast.dump(src, include_attributes=True), + "Module(body=[Expr(value=Call(func=Name(id='write', ctx=Load(), " + "lineno=1, col_offset=0, end_lineno=1, end_col_offset=5), " + "args=[Constant(value='spam', lineno=1, col_offset=6, end_lineno=1, " + "end_col_offset=12)], lineno=1, col_offset=0, end_lineno=1, " + "end_col_offset=13), lineno=1, col_offset=0, end_lineno=1, " + "end_col_offset=13), Expr(value=Call(func=Name(id='spam', ctx=Load(), " + "lineno=1, col_offset=0, end_lineno=1, end_col_offset=0), " + "args=[Constant(value='eggs', lineno=1, col_offset=0, end_lineno=1, " + "end_col_offset=0)], lineno=1, col_offset=0, end_lineno=1, " + "end_col_offset=0), lineno=1, col_offset=0, end_lineno=1, end_col_offset=0)])", + ) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_increment_lineno(self): + src = ast.parse("1 + 1", mode="eval") + self.assertEqual(ast.increment_lineno(src, n=3), src) + self.assertEqual( + ast.dump(src, include_attributes=True), + "Expression(body=BinOp(left=Constant(value=1, lineno=4, col_offset=0, " + "end_lineno=4, end_col_offset=1), op=Add(), right=Constant(value=1, " + "lineno=4, col_offset=4, end_lineno=4, end_col_offset=5), lineno=4, " + "col_offset=0, end_lineno=4, end_col_offset=5))", + ) + # issue10869: do not increment lineno of root twice + src = ast.parse("1 + 1", mode="eval") + self.assertEqual(ast.increment_lineno(src.body, n=3), src.body) + self.assertEqual( + ast.dump(src, include_attributes=True), + "Expression(body=BinOp(left=Constant(value=1, lineno=4, col_offset=0, " + "end_lineno=4, end_col_offset=1), op=Add(), right=Constant(value=1, " + "lineno=4, col_offset=4, end_lineno=4, end_col_offset=5), lineno=4, " + "col_offset=0, end_lineno=4, end_col_offset=5))", + ) + src = ast.Call( + func=ast.Name("test", ast.Load()), args=[], keywords=[], lineno=1 + ) + self.assertEqual(ast.increment_lineno(src).lineno, 2) + self.assertIsNone(ast.increment_lineno(src).end_lineno) + + @unittest.expectedFailure # TODO: RUSTPYTHON; IndexError: index out of range + def test_increment_lineno_on_module(self): + src = ast.parse( + dedent("""\ + a = 1 + b = 2 # type: ignore + c = 3 + d = 4 # type: ignore@tag + """), + type_comments=True, + ) + ast.increment_lineno(src, n=5) + self.assertEqual(src.type_ignores[0].lineno, 7) + self.assertEqual(src.type_ignores[1].lineno, 9) + self.assertEqual(src.type_ignores[1].tag, "@tag") + + def test_iter_fields(self): + node = ast.parse("foo()", mode="eval") + d = dict(ast.iter_fields(node.body)) + self.assertEqual(d.pop("func").id, "foo") + self.assertEqual(d, {"keywords": [], "args": []}) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_iter_child_nodes(self): + node = ast.parse("spam(23, 42, eggs='leek')", mode="eval") + self.assertEqual(len(list(ast.iter_child_nodes(node.body))), 4) + iterator = ast.iter_child_nodes(node.body) + self.assertEqual(next(iterator).id, "spam") + self.assertEqual(next(iterator).value, 23) + self.assertEqual(next(iterator).value, 42) + self.assertEqual( + ast.dump(next(iterator)), + "keyword(arg='eggs', value=Constant(value='leek'))", + ) + + def test_get_docstring(self): + node = ast.parse('"""line one\n line two"""') + self.assertEqual(ast.get_docstring(node), "line one\nline two") + + node = ast.parse('class foo:\n """line one\n line two"""') + self.assertEqual(ast.get_docstring(node.body[0]), "line one\nline two") + + node = ast.parse('def foo():\n """line one\n line two"""') + self.assertEqual(ast.get_docstring(node.body[0]), "line one\nline two") + + node = ast.parse('async def foo():\n """spam\n ham"""') + self.assertEqual(ast.get_docstring(node.body[0]), "spam\nham") + + node = ast.parse('async def foo():\n """spam\n ham"""') + self.assertEqual(ast.get_docstring(node.body[0], clean=False), "spam\n ham") + + node = ast.parse("x") + self.assertRaises(TypeError, ast.get_docstring, node.body[0]) + + def test_get_docstring_none(self): + self.assertIsNone(ast.get_docstring(ast.parse(""))) + node = ast.parse('x = "not docstring"') + self.assertIsNone(ast.get_docstring(node)) + node = ast.parse("def foo():\n pass") + self.assertIsNone(ast.get_docstring(node)) + + node = ast.parse("class foo:\n pass") + self.assertIsNone(ast.get_docstring(node.body[0])) + node = ast.parse('class foo:\n x = "not docstring"') + self.assertIsNone(ast.get_docstring(node.body[0])) + node = ast.parse("class foo:\n def bar(self): pass") + self.assertIsNone(ast.get_docstring(node.body[0])) + + node = ast.parse("def foo():\n pass") + self.assertIsNone(ast.get_docstring(node.body[0])) + node = ast.parse('def foo():\n x = "not docstring"') + self.assertIsNone(ast.get_docstring(node.body[0])) + + node = ast.parse("async def foo():\n pass") + self.assertIsNone(ast.get_docstring(node.body[0])) + node = ast.parse('async def foo():\n x = "not docstring"') + self.assertIsNone(ast.get_docstring(node.body[0])) + + node = ast.parse("async def foo():\n 42") + self.assertIsNone(ast.get_docstring(node.body[0])) + + def test_multi_line_docstring_col_offset_and_lineno_issue16806(self): + node = ast.parse( + '"""line one\nline two"""\n\n' + 'def foo():\n """line one\n line two"""\n\n' + ' def bar():\n """line one\n line two"""\n' + ' """line one\n line two"""\n' + '"""line one\nline two"""\n\n' + ) + self.assertEqual(node.body[0].col_offset, 0) + self.assertEqual(node.body[0].lineno, 1) + self.assertEqual(node.body[1].body[0].col_offset, 2) + self.assertEqual(node.body[1].body[0].lineno, 5) + self.assertEqual(node.body[1].body[1].body[0].col_offset, 4) + self.assertEqual(node.body[1].body[1].body[0].lineno, 9) + self.assertEqual(node.body[1].body[2].col_offset, 2) + self.assertEqual(node.body[1].body[2].lineno, 11) + self.assertEqual(node.body[2].col_offset, 0) + self.assertEqual(node.body[2].lineno, 13) + + def test_elif_stmt_start_position(self): + node = ast.parse("if a:\n pass\nelif b:\n pass\n") + elif_stmt = node.body[0].orelse[0] + self.assertEqual(elif_stmt.lineno, 3) + self.assertEqual(elif_stmt.col_offset, 0) + + def test_elif_stmt_start_position_with_else(self): + node = ast.parse("if a:\n pass\nelif b:\n pass\nelse:\n pass\n") + elif_stmt = node.body[0].orelse[0] + self.assertEqual(elif_stmt.lineno, 3) + self.assertEqual(elif_stmt.col_offset, 0) + + def test_starred_expr_end_position_within_call(self): + node = ast.parse("f(*[0, 1])") + starred_expr = node.body[0].value.args[0] + self.assertEqual(starred_expr.end_lineno, 1) + self.assertEqual(starred_expr.end_col_offset, 9) + + def test_literal_eval(self): + self.assertEqual(ast.literal_eval("[1, 2, 3]"), [1, 2, 3]) + self.assertEqual(ast.literal_eval('{"foo": 42}'), {"foo": 42}) + self.assertEqual(ast.literal_eval("(True, False, None)"), (True, False, None)) + self.assertEqual(ast.literal_eval("{1, 2, 3}"), {1, 2, 3}) + self.assertEqual(ast.literal_eval('b"hi"'), b"hi") + self.assertEqual(ast.literal_eval("set()"), set()) + self.assertRaises(ValueError, ast.literal_eval, "foo()") + self.assertEqual(ast.literal_eval("6"), 6) + self.assertEqual(ast.literal_eval("+6"), 6) + self.assertEqual(ast.literal_eval("-6"), -6) + self.assertEqual(ast.literal_eval("3.25"), 3.25) + self.assertEqual(ast.literal_eval("+3.25"), 3.25) + self.assertEqual(ast.literal_eval("-3.25"), -3.25) + self.assertEqual(repr(ast.literal_eval("-0.0")), "-0.0") + self.assertRaises(ValueError, ast.literal_eval, "++6") + self.assertRaises(ValueError, ast.literal_eval, "+True") + self.assertRaises(ValueError, ast.literal_eval, "2+3") + + @unittest.expectedFailure # TODO: RUSTPYTHON; SyntaxError not raised + def test_literal_eval_str_int_limit(self): + with support.adjust_int_max_str_digits(4000): + ast.literal_eval("3" * 4000) # no error + with self.assertRaises(SyntaxError) as err_ctx: + ast.literal_eval("3" * 4001) + self.assertIn("Exceeds the limit ", str(err_ctx.exception)) + self.assertIn(" Consider hexadecimal ", str(err_ctx.exception)) + + def test_literal_eval_complex(self): + # Issue #4907 + self.assertEqual(ast.literal_eval("6j"), 6j) + self.assertEqual(ast.literal_eval("-6j"), -6j) + self.assertEqual(ast.literal_eval("6.75j"), 6.75j) + self.assertEqual(ast.literal_eval("-6.75j"), -6.75j) + self.assertEqual(ast.literal_eval("3+6j"), 3 + 6j) + self.assertEqual(ast.literal_eval("-3+6j"), -3 + 6j) + self.assertEqual(ast.literal_eval("3-6j"), 3 - 6j) + self.assertEqual(ast.literal_eval("-3-6j"), -3 - 6j) + self.assertEqual(ast.literal_eval("3.25+6.75j"), 3.25 + 6.75j) + self.assertEqual(ast.literal_eval("-3.25+6.75j"), -3.25 + 6.75j) + self.assertEqual(ast.literal_eval("3.25-6.75j"), 3.25 - 6.75j) + self.assertEqual(ast.literal_eval("-3.25-6.75j"), -3.25 - 6.75j) + self.assertEqual(ast.literal_eval("(3+6j)"), 3 + 6j) + self.assertRaises(ValueError, ast.literal_eval, "-6j+3") + self.assertRaises(ValueError, ast.literal_eval, "-6j+3j") + self.assertRaises(ValueError, ast.literal_eval, "3+-6j") + self.assertRaises(ValueError, ast.literal_eval, "3+(0+6j)") + self.assertRaises(ValueError, ast.literal_eval, "-(3+6j)") + + def test_literal_eval_malformed_dict_nodes(self): + malformed = ast.Dict( + keys=[ast.Constant(1), ast.Constant(2)], values=[ast.Constant(3)] + ) + self.assertRaises(ValueError, ast.literal_eval, malformed) + malformed = ast.Dict( + keys=[ast.Constant(1)], values=[ast.Constant(2), ast.Constant(3)] + ) + self.assertRaises(ValueError, ast.literal_eval, malformed) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_literal_eval_trailing_ws(self): + self.assertEqual(ast.literal_eval(" -1"), -1) + self.assertEqual(ast.literal_eval("\t\t-1"), -1) + self.assertEqual(ast.literal_eval(" \t -1"), -1) + self.assertRaises(IndentationError, ast.literal_eval, "\n -1") + + def test_literal_eval_malformed_lineno(self): + msg = r"malformed node or string on line 3:" + with self.assertRaisesRegex(ValueError, msg): + ast.literal_eval("{'a': 1,\n'b':2,\n'c':++3,\n'd':4}") + + node = ast.UnaryOp(ast.UAdd(), ast.UnaryOp(ast.UAdd(), ast.Constant(6))) + self.assertIsNone(getattr(node, "lineno", None)) + msg = r"malformed node or string:" + with self.assertRaisesRegex(ValueError, msg): + ast.literal_eval(node) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_literal_eval_syntax_errors(self): + with self.assertRaisesRegex(SyntaxError, "unexpected indent"): + ast.literal_eval(r""" + \ + (\ + \ """) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_bad_integer(self): + # issue13436: Bad error message with invalid numeric values + body = [ + ast.ImportFrom( + module="time", + names=[ast.alias(name="sleep")], + level=None, + lineno=None, + col_offset=None, + ) + ] + mod = ast.Module(body, []) + with self.assertRaises(ValueError) as cm: + compile(mod, "test", "exec") + self.assertIn("invalid integer value: None", str(cm.exception)) + + def test_level_as_none(self): + body = [ + ast.ImportFrom( + module="time", + names=[ast.alias(name="sleep", lineno=0, col_offset=0)], + level=None, + lineno=0, + col_offset=0, + ) + ] + mod = ast.Module(body, []) + code = compile(mod, "test", "exec") + ns = {} + exec(code, ns) + self.assertIn("sleep", ns) + + @unittest.skip('TODO: RUSTPYTHON; crash') + def test_recursion_direct(self): + e = ast.UnaryOp(op=ast.Not(), lineno=0, col_offset=0, operand=ast.Constant(1)) + e.operand = e + with self.assertRaises(RecursionError): + with support.infinite_recursion(): + compile(ast.Expression(e), "", "eval") + + @unittest.skip('TODO: RUSTPYTHON; crash') + def test_recursion_indirect(self): + e = ast.UnaryOp(op=ast.Not(), lineno=0, col_offset=0, operand=ast.Constant(1)) + f = ast.UnaryOp(op=ast.Not(), lineno=0, col_offset=0, operand=ast.Constant(1)) + e.operand = f + f.operand = e + with self.assertRaises(RecursionError): + with support.infinite_recursion(): + compile(ast.Expression(e), "", "eval") + + +class ASTValidatorTests(unittest.TestCase): + def mod(self, mod, msg=None, mode="exec", *, exc=ValueError): + mod.lineno = mod.col_offset = 0 + ast.fix_missing_locations(mod) + if msg is None: + compile(mod, "", mode) + else: + with self.assertRaises(exc) as cm: + compile(mod, "", mode) + self.assertIn(msg, str(cm.exception)) + + def expr(self, node, msg=None, *, exc=ValueError): + mod = ast.Module([ast.Expr(node)], []) + self.mod(mod, msg, exc=exc) + + def stmt(self, stmt, msg=None): + mod = ast.Module([stmt], []) + self.mod(mod, msg) + + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised + def test_module(self): + m = ast.Interactive([ast.Expr(ast.Name("x", ast.Store()))]) + self.mod(m, "must have Load context", "single") + m = ast.Expression(ast.Name("x", ast.Store())) + self.mod(m, "must have Load context", "eval") + + def _check_arguments(self, fac, check): + def arguments( + args=None, + posonlyargs=None, + vararg=None, + kwonlyargs=None, + kwarg=None, + defaults=None, + kw_defaults=None, + ): + if args is None: + args = [] + if posonlyargs is None: + posonlyargs = [] + if kwonlyargs is None: + kwonlyargs = [] + if defaults is None: + defaults = [] + if kw_defaults is None: + kw_defaults = [] + args = ast.arguments( + args, posonlyargs, vararg, kwonlyargs, kw_defaults, kwarg, defaults + ) + return fac(args) + + args = [ast.arg("x", ast.Name("x", ast.Store()))] + check(arguments(args=args), "must have Load context") + check(arguments(posonlyargs=args), "must have Load context") + check(arguments(kwonlyargs=args), "must have Load context") + check( + arguments(defaults=[ast.Constant(3)]), "more positional defaults than args" + ) + check( + arguments(kw_defaults=[ast.Constant(4)]), + "length of kwonlyargs is not the same as kw_defaults", + ) + args = [ast.arg("x", ast.Name("x", ast.Load()))] + check( + arguments(args=args, defaults=[ast.Name("x", ast.Store())]), + "must have Load context", + ) + args = [ + ast.arg("a", ast.Name("x", ast.Load())), + ast.arg("b", ast.Name("y", ast.Load())), + ] + check( + arguments(kwonlyargs=args, kw_defaults=[None, ast.Name("x", ast.Store())]), + "must have Load context", + ) + + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised + def test_funcdef(self): + a = ast.arguments([], [], None, [], [], None, []) + f = ast.FunctionDef("x", a, [], [], None, None, []) + self.stmt(f, "empty body on FunctionDef") + f = ast.FunctionDef( + "x", a, [ast.Pass()], [ast.Name("x", ast.Store())], None, None, [] + ) + self.stmt(f, "must have Load context") + f = ast.FunctionDef( + "x", a, [ast.Pass()], [], ast.Name("x", ast.Store()), None, [] + ) + self.stmt(f, "must have Load context") + f = ast.FunctionDef("x", ast.arguments(), [ast.Pass()]) + self.stmt(f) + + def fac(args): + return ast.FunctionDef("x", args, [ast.Pass()], [], None, None, []) + + self._check_arguments(fac, self.stmt) + + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: class pattern defines no positional sub-patterns (__match_args__ missing) + def test_funcdef_pattern_matching(self): + # gh-104799: New fields on FunctionDef should be added at the end + def matcher(node): + match node: + case ast.FunctionDef( + "foo", + ast.arguments(args=[ast.arg("bar")]), + [ast.Pass()], + [ast.Name("capybara", ast.Load())], + ast.Name("pacarana", ast.Load()), + ): + return True + case _: + return False + + code = """ + @capybara + def foo(bar) -> pacarana: + pass + """ + source = ast.parse(textwrap.dedent(code)) + funcdef = source.body[0] + self.assertIsInstance(funcdef, ast.FunctionDef) + self.assertTrue(matcher(funcdef)) + + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised + def test_classdef(self): + def cls( + bases=None, keywords=None, body=None, decorator_list=None, type_params=None + ): + if bases is None: + bases = [] + if keywords is None: + keywords = [] + if body is None: + body = [ast.Pass()] + if decorator_list is None: + decorator_list = [] + if type_params is None: + type_params = [] + return ast.ClassDef( + "myclass", bases, keywords, body, decorator_list, type_params + ) + + self.stmt(cls(bases=[ast.Name("x", ast.Store())]), "must have Load context") + self.stmt( + cls(keywords=[ast.keyword("x", ast.Name("x", ast.Store()))]), + "must have Load context", + ) + self.stmt(cls(body=[]), "empty body on ClassDef") + self.stmt(cls(body=[None]), "None disallowed") + self.stmt( + cls(decorator_list=[ast.Name("x", ast.Store())]), "must have Load context" + ) + + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised + def test_delete(self): + self.stmt(ast.Delete([]), "empty targets on Delete") + self.stmt(ast.Delete([None]), "None disallowed") + self.stmt(ast.Delete([ast.Name("x", ast.Load())]), "must have Del context") + + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised + def test_assign(self): + self.stmt(ast.Assign([], ast.Constant(3)), "empty targets on Assign") + self.stmt(ast.Assign([None], ast.Constant(3)), "None disallowed") + self.stmt( + ast.Assign([ast.Name("x", ast.Load())], ast.Constant(3)), + "must have Store context", + ) + self.stmt( + ast.Assign([ast.Name("x", ast.Store())], ast.Name("y", ast.Store())), + "must have Load context", + ) + + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised + def test_augassign(self): + aug = ast.AugAssign( + ast.Name("x", ast.Load()), ast.Add(), ast.Name("y", ast.Load()) + ) + self.stmt(aug, "must have Store context") + aug = ast.AugAssign( + ast.Name("x", ast.Store()), ast.Add(), ast.Name("y", ast.Store()) + ) + self.stmt(aug, "must have Load context") + + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised + def test_for(self): + x = ast.Name("x", ast.Store()) + y = ast.Name("y", ast.Load()) + p = ast.Pass() + self.stmt(ast.For(x, y, [], []), "empty body on For") + self.stmt( + ast.For(ast.Name("x", ast.Load()), y, [p], []), "must have Store context" + ) + self.stmt( + ast.For(x, ast.Name("y", ast.Store()), [p], []), "must have Load context" + ) + e = ast.Expr(ast.Name("x", ast.Store())) + self.stmt(ast.For(x, y, [e], []), "must have Load context") + self.stmt(ast.For(x, y, [p], [e]), "must have Load context") + + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised + def test_while(self): + self.stmt(ast.While(ast.Constant(3), [], []), "empty body on While") + self.stmt( + ast.While(ast.Name("x", ast.Store()), [ast.Pass()], []), + "must have Load context", + ) + self.stmt( + ast.While( + ast.Constant(3), [ast.Pass()], [ast.Expr(ast.Name("x", ast.Store()))] + ), + "must have Load context", + ) + + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised + def test_if(self): + self.stmt(ast.If(ast.Constant(3), [], []), "empty body on If") + i = ast.If(ast.Name("x", ast.Store()), [ast.Pass()], []) + self.stmt(i, "must have Load context") + i = ast.If(ast.Constant(3), [ast.Expr(ast.Name("x", ast.Store()))], []) + self.stmt(i, "must have Load context") + i = ast.If( + ast.Constant(3), [ast.Pass()], [ast.Expr(ast.Name("x", ast.Store()))] + ) + self.stmt(i, "must have Load context") + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_with(self): + p = ast.Pass() + self.stmt(ast.With([], [p]), "empty items on With") + i = ast.withitem(ast.Constant(3), None) + self.stmt(ast.With([i], []), "empty body on With") + i = ast.withitem(ast.Name("x", ast.Store()), None) + self.stmt(ast.With([i], [p]), "must have Load context") + i = ast.withitem(ast.Constant(3), ast.Name("x", ast.Load())) + self.stmt(ast.With([i], [p]), "must have Store context") + + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised + def test_raise(self): + r = ast.Raise(None, ast.Constant(3)) + self.stmt(r, "Raise with cause but no exception") + r = ast.Raise(ast.Name("x", ast.Store()), None) + self.stmt(r, "must have Load context") + r = ast.Raise(ast.Constant(4), ast.Name("x", ast.Store())) + self.stmt(r, "must have Load context") + + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised + def test_try(self): + p = ast.Pass() + t = ast.Try([], [], [], [p]) + self.stmt(t, "empty body on Try") + t = ast.Try([ast.Expr(ast.Name("x", ast.Store()))], [], [], [p]) + self.stmt(t, "must have Load context") + t = ast.Try([p], [], [], []) + self.stmt(t, "Try has neither except handlers nor finalbody") + t = ast.Try([p], [], [p], [p]) + self.stmt(t, "Try has orelse but no except handlers") + t = ast.Try([p], [ast.ExceptHandler(None, "x", [])], [], []) + self.stmt(t, "empty body on ExceptHandler") + e = [ast.ExceptHandler(ast.Name("x", ast.Store()), "y", [p])] + self.stmt(ast.Try([p], e, [], []), "must have Load context") + e = [ast.ExceptHandler(None, "x", [p])] + t = ast.Try([p], e, [ast.Expr(ast.Name("x", ast.Store()))], [p]) + self.stmt(t, "must have Load context") + t = ast.Try([p], e, [p], [ast.Expr(ast.Name("x", ast.Store()))]) + self.stmt(t, "must have Load context") + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_try_star(self): + p = ast.Pass() + t = ast.TryStar([], [], [], [p]) + self.stmt(t, "empty body on TryStar") + t = ast.TryStar([ast.Expr(ast.Name("x", ast.Store()))], [], [], [p]) + self.stmt(t, "must have Load context") + t = ast.TryStar([p], [], [], []) + self.stmt(t, "TryStar has neither except handlers nor finalbody") + t = ast.TryStar([p], [], [p], [p]) + self.stmt(t, "TryStar has orelse but no except handlers") + t = ast.TryStar([p], [ast.ExceptHandler(None, "x", [])], [], []) + self.stmt(t, "empty body on ExceptHandler") + e = [ast.ExceptHandler(ast.Name("x", ast.Store()), "y", [p])] + self.stmt(ast.TryStar([p], e, [], []), "must have Load context") + e = [ast.ExceptHandler(None, "x", [p])] + t = ast.TryStar([p], e, [ast.Expr(ast.Name("x", ast.Store()))], [p]) + self.stmt(t, "must have Load context") + t = ast.TryStar([p], e, [p], [ast.Expr(ast.Name("x", ast.Store()))]) + self.stmt(t, "must have Load context") + + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised + def test_assert(self): + self.stmt( + ast.Assert(ast.Name("x", ast.Store()), None), "must have Load context" + ) + assrt = ast.Assert(ast.Name("x", ast.Load()), ast.Name("y", ast.Store())) + self.stmt(assrt, "must have Load context") + + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised + def test_import(self): + self.stmt(ast.Import([]), "empty names on Import") + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_importfrom(self): + imp = ast.ImportFrom(None, [ast.alias("x", None)], -42) + self.stmt(imp, "Negative ImportFrom level") + self.stmt(ast.ImportFrom(None, [], 0), "empty names on ImportFrom") + + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised + def test_global(self): + self.stmt(ast.Global([]), "empty names on Global") + + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised + def test_nonlocal(self): + self.stmt(ast.Nonlocal([]), "empty names on Nonlocal") + + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised + def test_expr(self): + e = ast.Expr(ast.Name("x", ast.Store())) + self.stmt(e, "must have Load context") + + @unittest.skip('TODO: RUSTPYTHON; called `Option::unwrap()` on a `None` value') + def test_boolop(self): + b = ast.BoolOp(ast.And(), []) + self.expr(b, "less than 2 values") + b = ast.BoolOp(ast.And(), [ast.Constant(3)]) + self.expr(b, "less than 2 values") + b = ast.BoolOp(ast.And(), [ast.Constant(4), None]) + self.expr(b, "None disallowed") + b = ast.BoolOp(ast.And(), [ast.Constant(4), ast.Name("x", ast.Store())]) + self.expr(b, "must have Load context") + + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised + def test_unaryop(self): + u = ast.UnaryOp(ast.Not(), ast.Name("x", ast.Store())) + self.expr(u, "must have Load context") + + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised + def test_lambda(self): + a = ast.arguments([], [], None, [], [], None, []) + self.expr(ast.Lambda(a, ast.Name("x", ast.Store())), "must have Load context") + + def fac(args): + return ast.Lambda(args, ast.Name("x", ast.Load())) + + self._check_arguments(fac, self.expr) + + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised + def test_ifexp(self): + l = ast.Name("x", ast.Load()) + s = ast.Name("y", ast.Store()) + for args in (s, l, l), (l, s, l), (l, l, s): + self.expr(ast.IfExp(*args), "must have Load context") + + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised + def test_dict(self): + d = ast.Dict([], [ast.Name("x", ast.Load())]) + self.expr(d, "same number of keys as values") + d = ast.Dict([ast.Name("x", ast.Load())], [None]) + self.expr(d, "None disallowed") + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_set(self): + self.expr(ast.Set([None]), "None disallowed") + s = ast.Set([ast.Name("x", ast.Store())]) + self.expr(s, "must have Load context") + + def _check_comprehension(self, fac): + self.expr(fac([]), "comprehension with no generators") + g = ast.comprehension( + ast.Name("x", ast.Load()), ast.Name("x", ast.Load()), [], 0 + ) + self.expr(fac([g]), "must have Store context") + g = ast.comprehension( + ast.Name("x", ast.Store()), ast.Name("x", ast.Store()), [], 0 + ) + self.expr(fac([g]), "must have Load context") + x = ast.Name("x", ast.Store()) + y = ast.Name("y", ast.Load()) + g = ast.comprehension(x, y, [None], 0) + self.expr(fac([g]), "None disallowed") + g = ast.comprehension(x, y, [ast.Name("x", ast.Store())], 0) + self.expr(fac([g]), "must have Load context") + + def _simple_comp(self, fac): + g = ast.comprehension( + ast.Name("x", ast.Store()), ast.Name("x", ast.Load()), [], 0 + ) + self.expr(fac(ast.Name("x", ast.Store()), [g]), "must have Load context") + + def wrap(gens): + return fac(ast.Name("x", ast.Store()), gens) + + self._check_comprehension(wrap) + + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised + def test_listcomp(self): + self._simple_comp(ast.ListComp) + + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised + def test_setcomp(self): + self._simple_comp(ast.SetComp) + + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised + def test_generatorexp(self): + self._simple_comp(ast.GeneratorExp) + + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised + def test_dictcomp(self): + g = ast.comprehension( + ast.Name("y", ast.Store()), ast.Name("p", ast.Load()), [], 0 + ) + c = ast.DictComp(ast.Name("x", ast.Store()), ast.Name("y", ast.Load()), [g]) + self.expr(c, "must have Load context") + c = ast.DictComp(ast.Name("x", ast.Load()), ast.Name("y", ast.Store()), [g]) + self.expr(c, "must have Load context") + + def factory(comps): + k = ast.Name("x", ast.Load()) + v = ast.Name("y", ast.Load()) + return ast.DictComp(k, v, comps) + + self._check_comprehension(factory) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_yield(self): + self.expr(ast.Yield(ast.Name("x", ast.Store())), "must have Load") + self.expr(ast.YieldFrom(ast.Name("x", ast.Store())), "must have Load") + + @unittest.skip("TODO: RUSTPYTHON; thread 'main' panicked") + def test_compare(self): + left = ast.Name("x", ast.Load()) + comp = ast.Compare(left, [ast.In()], []) + self.expr(comp, "no comparators") + comp = ast.Compare(left, [ast.In()], [ast.Constant(4), ast.Constant(5)]) + self.expr(comp, "different number of comparators and operands") + comp = ast.Compare(ast.Constant("blah"), [ast.In()], [left]) + self.expr(comp) + comp = ast.Compare(left, [ast.In()], [ast.Constant("blah")]) + self.expr(comp) + + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised + def test_call(self): + func = ast.Name("x", ast.Load()) + args = [ast.Name("y", ast.Load())] + keywords = [ast.keyword("w", ast.Name("z", ast.Load()))] + call = ast.Call(ast.Name("x", ast.Store()), args, keywords) + self.expr(call, "must have Load context") + call = ast.Call(func, [None], keywords) + self.expr(call, "None disallowed") + bad_keywords = [ast.keyword("w", ast.Name("z", ast.Store()))] + call = ast.Call(func, args, bad_keywords) + self.expr(call, "must have Load context") + + def test_num(self): + with warnings.catch_warnings(record=True) as wlog: + warnings.filterwarnings("ignore", "", DeprecationWarning) + from ast import Num + + with warnings.catch_warnings(record=True) as wlog: + warnings.filterwarnings("always", "", DeprecationWarning) + + class subint(int): + pass + + class subfloat(float): + pass + + class subcomplex(complex): + pass + + for obj in "0", "hello": + self.expr(ast.Num(obj)) + for obj in subint(), subfloat(), subcomplex(): + self.expr(ast.Num(obj), "invalid type", exc=TypeError) + + self.assertEqual( + [str(w.message) for w in wlog], + [ + "ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead", + "ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead", + "ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead", + "ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead", + "ast.Num is deprecated and will be removed in Python 3.14; use ast.Constant instead", + ], + ) + + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised + def test_attribute(self): + attr = ast.Attribute(ast.Name("x", ast.Store()), "y", ast.Load()) + self.expr(attr, "must have Load context") + + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised + def test_subscript(self): + sub = ast.Subscript(ast.Name("x", ast.Store()), ast.Constant(3), ast.Load()) + self.expr(sub, "must have Load context") + x = ast.Name("x", ast.Load()) + sub = ast.Subscript(x, ast.Name("y", ast.Store()), ast.Load()) + self.expr(sub, "must have Load context") + s = ast.Name("x", ast.Store()) + for args in (s, None, None), (None, s, None), (None, None, s): + sl = ast.Slice(*args) + self.expr(ast.Subscript(x, sl, ast.Load()), "must have Load context") + sl = ast.Tuple([], ast.Load()) + self.expr(ast.Subscript(x, sl, ast.Load())) + sl = ast.Tuple([s], ast.Load()) + self.expr(ast.Subscript(x, sl, ast.Load()), "must have Load context") + + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError not raised + def test_starred(self): + left = ast.List( + [ast.Starred(ast.Name("x", ast.Load()), ast.Store())], ast.Store() + ) + assign = ast.Assign([left], ast.Constant(4)) + self.stmt(assign, "must have Store context") + + def _sequence(self, fac): + self.expr(fac([None], ast.Load()), "None disallowed") + self.expr( + fac([ast.Name("x", ast.Store())], ast.Load()), "must have Load context" + ) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_list(self): + self._sequence(ast.List) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_tuple(self): + self._sequence(ast.Tuple) + + def test_nameconstant(self): + with warnings.catch_warnings(record=True) as wlog: + warnings.filterwarnings("ignore", "", DeprecationWarning) + from ast import NameConstant + + with warnings.catch_warnings(record=True) as wlog: + warnings.filterwarnings("always", "", DeprecationWarning) + self.expr(ast.NameConstant(4)) + + self.assertEqual( + [str(w.message) for w in wlog], + [ + "ast.NameConstant is deprecated and will be removed in Python 3.14; use ast.Constant instead", + ], + ) + + @unittest.expectedFailure # TODO: RUSTPYTHON + @support.requires_resource("cpu") + def test_stdlib_validates(self): + stdlib = os.path.dirname(ast.__file__) + tests = [fn for fn in os.listdir(stdlib) if fn.endswith(".py")] + tests.extend(["test/test_grammar.py", "test/test_unpack_ex.py"]) + for module in tests: + with self.subTest(module): + fn = os.path.join(stdlib, module) + with open(fn, "r", encoding="utf-8") as fp: + source = fp.read() + mod = ast.parse(source, fn) + compile(mod, fn, "exec") + + constant_1 = ast.Constant(1) + pattern_1 = ast.MatchValue(constant_1) + + constant_x = ast.Constant("x") + pattern_x = ast.MatchValue(constant_x) + + constant_true = ast.Constant(True) + pattern_true = ast.MatchSingleton(True) + + name_carter = ast.Name("carter", ast.Load()) + + _MATCH_PATTERNS = [ + ast.MatchValue( + ast.Attribute( + ast.Attribute(ast.Name("x", ast.Store()), "y", ast.Load()), + "z", + ast.Load(), + ) + ), + ast.MatchValue( + ast.Attribute( + ast.Attribute(ast.Name("x", ast.Load()), "y", ast.Store()), + "z", + ast.Load(), + ) + ), + ast.MatchValue(ast.Constant(...)), + ast.MatchValue(ast.Constant(True)), + ast.MatchValue(ast.Constant((1, 2, 3))), + ast.MatchSingleton("string"), + ast.MatchSequence([ast.MatchSingleton("string")]), + ast.MatchSequence([ast.MatchSequence([ast.MatchSingleton("string")])]), + ast.MatchMapping([constant_1, constant_true], [pattern_x]), + ast.MatchMapping( + [constant_true, constant_1], [pattern_x, pattern_1], rest="True" + ), + ast.MatchMapping( + [constant_true, ast.Starred(ast.Name("lol", ast.Load()), ast.Load())], + [pattern_x, pattern_1], + rest="legit", + ), + ast.MatchClass( + ast.Attribute(ast.Attribute(constant_x, "y", ast.Load()), "z", ast.Load()), + patterns=[], + kwd_attrs=[], + kwd_patterns=[], + ), + ast.MatchClass( + name_carter, patterns=[], kwd_attrs=["True"], kwd_patterns=[pattern_1] + ), + ast.MatchClass( + name_carter, patterns=[], kwd_attrs=[], kwd_patterns=[pattern_1] + ), + ast.MatchClass( + name_carter, + patterns=[ast.MatchSingleton("string")], + kwd_attrs=[], + kwd_patterns=[], + ), + ast.MatchClass( + name_carter, patterns=[ast.MatchStar()], kwd_attrs=[], kwd_patterns=[] + ), + ast.MatchClass( + name_carter, patterns=[], kwd_attrs=[], kwd_patterns=[ast.MatchStar()] + ), + ast.MatchClass( + constant_true, # invalid name + patterns=[], + kwd_attrs=["True"], + kwd_patterns=[pattern_1], + ), + ast.MatchSequence([ast.MatchStar("True")]), + ast.MatchAs(name="False"), + ast.MatchOr([]), + ast.MatchOr([pattern_1]), + ast.MatchOr([pattern_1, pattern_x, ast.MatchSingleton("xxx")]), + ast.MatchAs(name="_"), + ast.MatchStar(name="x"), + ast.MatchSequence([ast.MatchStar("_")]), + ast.MatchMapping([], [], rest="_"), + ] + + @unittest.skip("TODO: RUSTPYTHON; thread 'main' panicked") + def test_match_validation_pattern(self): + name_x = ast.Name("x", ast.Load()) + for pattern in self._MATCH_PATTERNS: + with self.subTest(ast.dump(pattern, indent=4)): + node = ast.Match( + subject=name_x, + cases=[ast.match_case(pattern=pattern, body=[ast.Pass()])], + ) + node = ast.fix_missing_locations(node) + module = ast.Module([node], []) + with self.assertRaises(ValueError): + compile(module, "", "exec") + + +class ConstantTests(unittest.TestCase): + """Tests on the ast.Constant node type.""" + + def compile_constant(self, value): + tree = ast.parse("x = 123") + + node = tree.body[0].value + new_node = ast.Constant(value=value) + ast.copy_location(new_node, node) + tree.body[0].value = new_node + + code = compile(tree, "", "exec") + + ns = {} + exec(code, ns) + return ns["x"] + + def test_validation(self): + with self.assertRaises(TypeError) as cm: + self.compile_constant([1, 2, 3]) + self.assertEqual(str(cm.exception), "got an invalid type in Constant: list") + + @unittest.expectedFailure # TODO: RUSTPYTHON; b'' is not b'' + def test_singletons(self): + for const in (None, False, True, Ellipsis, b"", frozenset()): + with self.subTest(const=const): + value = self.compile_constant(const) + self.assertIs(value, const) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_values(self): + nested_tuple = (1,) + nested_frozenset = frozenset({1}) + for level in range(3): + nested_tuple = (nested_tuple, 2) + nested_frozenset = frozenset({nested_frozenset, 2}) + values = ( + 123, + 123.0, + 123j, + "unicode", + b"bytes", + tuple("tuple"), + frozenset("frozenset"), + nested_tuple, + nested_frozenset, + ) + for value in values: + with self.subTest(value=value): + result = self.compile_constant(value) + self.assertEqual(result, value) + + @unittest.expectedFailure # TODO: RUSTPYTHON; SyntaxError: cannot assign to literal + def test_assign_to_constant(self): + tree = ast.parse("x = 1") + + target = tree.body[0].targets[0] + new_target = ast.Constant(value=1) + ast.copy_location(new_target, target) + tree.body[0].targets[0] = new_target + + with self.assertRaises(ValueError) as cm: + compile(tree, "string", "exec") + self.assertEqual( + str(cm.exception), + "expression which can't be assigned " "to in Store context", + ) + + def test_get_docstring(self): + tree = ast.parse("'docstring'\nx = 1") + self.assertEqual(ast.get_docstring(tree), "docstring") + + def get_load_const(self, tree): + # Compile to bytecode, disassemble and get parameter of LOAD_CONST + # instructions + co = compile(tree, "", "exec") + consts = [] + for instr in dis.get_instructions(co): + if instr.opname == "LOAD_CONST" or instr.opname == "RETURN_CONST": + consts.append(instr.argval) + return consts + + @support.cpython_only + def test_load_const(self): + consts = [None, True, False, 124, 2.0, 3j, "unicode", b"bytes", (1, 2, 3)] + + code = "\n".join(["x={!r}".format(const) for const in consts]) + code += "\nx = ..." + consts.extend((Ellipsis, None)) + + tree = ast.parse(code) + self.assertEqual(self.get_load_const(tree), consts) + + # Replace expression nodes with constants + for assign, const in zip(tree.body, consts): + assert isinstance(assign, ast.Assign), ast.dump(assign) + new_node = ast.Constant(value=const) + ast.copy_location(new_node, assign.value) + assign.value = new_node + + self.assertEqual(self.get_load_const(tree), consts) + + def test_literal_eval(self): + tree = ast.parse("1 + 2") + binop = tree.body[0].value + + new_left = ast.Constant(value=10) + ast.copy_location(new_left, binop.left) + binop.left = new_left + + new_right = ast.Constant(value=20j) + ast.copy_location(new_right, binop.right) + binop.right = new_right + + self.assertEqual(ast.literal_eval(binop), 10 + 20j) + + def test_string_kind(self): + c = ast.parse('"x"', mode="eval").body + self.assertEqual(c.value, "x") + self.assertEqual(c.kind, None) + + c = ast.parse('u"x"', mode="eval").body + self.assertEqual(c.value, "x") + self.assertEqual(c.kind, "u") + + c = ast.parse('r"x"', mode="eval").body + self.assertEqual(c.value, "x") + self.assertEqual(c.kind, None) + + c = ast.parse('b"x"', mode="eval").body + self.assertEqual(c.value, b"x") + self.assertEqual(c.kind, None) + + +class EndPositionTests(unittest.TestCase): + """Tests for end position of AST nodes. + + Testing end positions of nodes requires a bit of extra care + because of how LL parsers work. + """ + + def _check_end_pos(self, ast_node, end_lineno, end_col_offset): + self.assertEqual(ast_node.end_lineno, end_lineno) + self.assertEqual(ast_node.end_col_offset, end_col_offset) + + def _check_content(self, source, ast_node, content): + self.assertEqual(ast.get_source_segment(source, ast_node), content) + + def _parse_value(self, s): + # Use duck-typing to support both single expression + # and a right hand side of an assignment statement. + return ast.parse(s).body[0].value + + def test_lambda(self): + s = "lambda x, *y: None" + lam = self._parse_value(s) + self._check_content(s, lam.body, "None") + self._check_content(s, lam.args.args[0], "x") + self._check_content(s, lam.args.vararg, "y") + + def test_func_def(self): + s = dedent(""" + def func(x: int, + *args: str, + z: float = 0, + **kwargs: Any) -> bool: + return True + """).strip() + fdef = ast.parse(s).body[0] + self._check_end_pos(fdef, 5, 15) + self._check_content(s, fdef.body[0], "return True") + self._check_content(s, fdef.args.args[0], "x: int") + self._check_content(s, fdef.args.args[0].annotation, "int") + self._check_content(s, fdef.args.kwarg, "kwargs: Any") + self._check_content(s, fdef.args.kwarg.annotation, "Any") + + def test_call(self): + s = "func(x, y=2, **kw)" + call = self._parse_value(s) + self._check_content(s, call.func, "func") + self._check_content(s, call.keywords[0].value, "2") + self._check_content(s, call.keywords[1].value, "kw") + + def test_call_noargs(self): + s = "x[0]()" + call = self._parse_value(s) + self._check_content(s, call.func, "x[0]") + self._check_end_pos(call, 1, 6) + + def test_class_def(self): + s = dedent(""" + class C(A, B): + x: int = 0 + """).strip() + cdef = ast.parse(s).body[0] + self._check_end_pos(cdef, 2, 14) + self._check_content(s, cdef.bases[1], "B") + self._check_content(s, cdef.body[0], "x: int = 0") + + def test_class_kw(self): + s = "class S(metaclass=abc.ABCMeta): pass" + cdef = ast.parse(s).body[0] + self._check_content(s, cdef.keywords[0].value, "abc.ABCMeta") + + def test_multi_line_str(self): + s = dedent(''' + x = """Some multi-line text. + + It goes on starting from same indent.""" + ''').strip() + assign = ast.parse(s).body[0] + self._check_end_pos(assign, 3, 40) + self._check_end_pos(assign.value, 3, 40) + + def test_continued_str(self): + s = dedent(""" + x = "first part" \\ + "second part" + """).strip() + assign = ast.parse(s).body[0] + self._check_end_pos(assign, 2, 13) + self._check_end_pos(assign.value, 2, 13) + + def test_suites(self): + # We intentionally put these into the same string to check + # that empty lines are not part of the suite. + s = dedent(""" + while True: + pass + + if one(): + x = None + elif other(): + y = None + else: + z = None + + for x, y in stuff: + assert True + + try: + raise RuntimeError + except TypeError as e: + pass + + pass + """).strip() + mod = ast.parse(s) + while_loop = mod.body[0] + if_stmt = mod.body[1] + for_loop = mod.body[2] + try_stmt = mod.body[3] + pass_stmt = mod.body[4] + + self._check_end_pos(while_loop, 2, 8) + self._check_end_pos(if_stmt, 9, 12) + self._check_end_pos(for_loop, 12, 15) + self._check_end_pos(try_stmt, 17, 8) + self._check_end_pos(pass_stmt, 19, 4) + + self._check_content(s, while_loop.test, "True") + self._check_content(s, if_stmt.body[0], "x = None") + self._check_content(s, if_stmt.orelse[0].test, "other()") + self._check_content(s, for_loop.target, "x, y") + self._check_content(s, try_stmt.body[0], "raise RuntimeError") + self._check_content(s, try_stmt.handlers[0].type, "TypeError") + + def test_fstring(self): + s = 'x = f"abc {x + y} abc"' + fstr = self._parse_value(s) + binop = fstr.values[1].value + self._check_content(s, binop, "x + y") + + def test_fstring_multi_line(self): + s = dedent(''' + f"""Some multi-line text. + { + arg_one + + + arg_two + } + It goes on...""" + ''').strip() + fstr = self._parse_value(s) + binop = fstr.values[1].value + self._check_end_pos(binop, 5, 7) + self._check_content(s, binop.left, "arg_one") + self._check_content(s, binop.right, "arg_two") + + def test_import_from_multi_line(self): + s = dedent(""" + from x.y.z import ( + a, b, c as c + ) + """).strip() + imp = ast.parse(s).body[0] + self._check_end_pos(imp, 3, 1) + self._check_end_pos(imp.names[2], 2, 16) + + def test_slices(self): + s1 = "f()[1, 2] [0]" + s2 = "x[ a.b: c.d]" + sm = dedent(""" + x[ a.b: f () , + g () : c.d + ] + """).strip() + i1, i2, im = map(self._parse_value, (s1, s2, sm)) + self._check_content(s1, i1.value, "f()[1, 2]") + self._check_content(s1, i1.value.slice, "1, 2") + self._check_content(s2, i2.slice.lower, "a.b") + self._check_content(s2, i2.slice.upper, "c.d") + self._check_content(sm, im.slice.elts[0].upper, "f ()") + self._check_content(sm, im.slice.elts[1].lower, "g ()") + self._check_end_pos(im, 3, 3) + + def test_binop(self): + s = dedent(""" + (1 * 2 + (3 ) + + 4 + ) + """).strip() + binop = self._parse_value(s) + self._check_end_pos(binop, 2, 6) + self._check_content(s, binop.right, "4") + self._check_content(s, binop.left, "1 * 2 + (3 )") + self._check_content(s, binop.left.right, "3") + + def test_boolop(self): + s = dedent(""" + if (one_condition and + (other_condition or yet_another_one)): + pass + """).strip() + bop = ast.parse(s).body[0].test + self._check_end_pos(bop, 2, 44) + self._check_content(s, bop.values[1], "other_condition or yet_another_one") + + def test_tuples(self): + s1 = "x = () ;" + s2 = "x = 1 , ;" + s3 = "x = (1 , 2 ) ;" + sm = dedent(""" + x = ( + a, b, + ) + """).strip() + t1, t2, t3, tm = map(self._parse_value, (s1, s2, s3, sm)) + self._check_content(s1, t1, "()") + self._check_content(s2, t2, "1 ,") + self._check_content(s3, t3, "(1 , 2 )") + self._check_end_pos(tm, 3, 1) + + def test_attribute_spaces(self): + s = "func(x. y .z)" + call = self._parse_value(s) + self._check_content(s, call, s) + self._check_content(s, call.args[0], "x. y .z") + + def test_redundant_parenthesis(self): + s = "( ( ( a + b ) ) )" + v = ast.parse(s).body[0].value + self.assertEqual(type(v).__name__, "BinOp") + self._check_content(s, v, "a + b") + s2 = "await " + s + v = ast.parse(s2).body[0].value.value + self.assertEqual(type(v).__name__, "BinOp") + self._check_content(s2, v, "a + b") + + def test_trailers_with_redundant_parenthesis(self): + tests = ( + ("( ( ( a ) ) ) ( )", "Call"), + ("( ( ( a ) ) ) ( b )", "Call"), + ("( ( ( a ) ) ) [ b ]", "Subscript"), + ("( ( ( a ) ) ) . b", "Attribute"), + ) + for s, t in tests: + with self.subTest(s): + v = ast.parse(s).body[0].value + self.assertEqual(type(v).__name__, t) + self._check_content(s, v, s) + s2 = "await " + s + v = ast.parse(s2).body[0].value.value + self.assertEqual(type(v).__name__, t) + self._check_content(s2, v, s) + + def test_displays(self): + s1 = "[{}, {1, }, {1, 2,} ]" + s2 = "{a: b, f (): g () ,}" + c1 = self._parse_value(s1) + c2 = self._parse_value(s2) + self._check_content(s1, c1.elts[0], "{}") + self._check_content(s1, c1.elts[1], "{1, }") + self._check_content(s1, c1.elts[2], "{1, 2,}") + self._check_content(s2, c2.keys[1], "f ()") + self._check_content(s2, c2.values[1], "g ()") + + def test_comprehensions(self): + s = dedent(""" + x = [{x for x, y in stuff + if cond.x} for stuff in things] + """).strip() + cmp = self._parse_value(s) + self._check_end_pos(cmp, 2, 37) + self._check_content(s, cmp.generators[0].iter, "things") + self._check_content(s, cmp.elt.generators[0].iter, "stuff") + self._check_content(s, cmp.elt.generators[0].ifs[0], "cond.x") + self._check_content(s, cmp.elt.generators[0].target, "x, y") + + def test_yield_await(self): + s = dedent(""" + async def f(): + yield x + await y + """).strip() + fdef = ast.parse(s).body[0] + self._check_content(s, fdef.body[0].value, "yield x") + self._check_content(s, fdef.body[1].value, "await y") + + def test_source_segment_multi(self): + s_orig = dedent(""" + x = ( + a, b, + ) + () + """).strip() + s_tuple = dedent(""" + ( + a, b, + ) + """).strip() + binop = self._parse_value(s_orig) + self.assertEqual(ast.get_source_segment(s_orig, binop.left), s_tuple) + + def test_source_segment_padded(self): + s_orig = dedent(""" + class C: + def fun(self) -> None: + "ЖЖЖЖЖ" + """).strip() + s_method = " def fun(self) -> None:\n" ' "ЖЖЖЖЖ"' + cdef = ast.parse(s_orig).body[0] + self.assertEqual( + ast.get_source_segment(s_orig, cdef.body[0], padded=True), s_method + ) + + def test_source_segment_endings(self): + s = "v = 1\r\nw = 1\nx = 1\n\ry = 1\rz = 1\r\n" + v, w, x, y, z = ast.parse(s).body + self._check_content(s, v, "v = 1") + self._check_content(s, w, "w = 1") + self._check_content(s, x, "x = 1") + self._check_content(s, y, "y = 1") + self._check_content(s, z, "z = 1") + + def test_source_segment_tabs(self): + s = dedent(""" + class C: + \t\f def fun(self) -> None: + \t\f pass + """).strip() + s_method = " \t\f def fun(self) -> None:\n" " \t\f pass" + + cdef = ast.parse(s).body[0] + self.assertEqual(ast.get_source_segment(s, cdef.body[0], padded=True), s_method) + + def test_source_segment_newlines(self): + s = "def f():\n pass\ndef g():\r pass\r\ndef h():\r\n pass\r\n" + f, g, h = ast.parse(s).body + self._check_content(s, f, "def f():\n pass") + self._check_content(s, g, "def g():\r pass") + self._check_content(s, h, "def h():\r\n pass") + + s = "def f():\n a = 1\r b = 2\r\n c = 3\n" + f = ast.parse(s).body[0] + self._check_content(s, f, s.rstrip()) + + def test_source_segment_missing_info(self): + s = "v = 1\r\nw = 1\nx = 1\n\ry = 1\r\n" + v, w, x, y = ast.parse(s).body + del v.lineno + del w.end_lineno + del x.col_offset + del y.end_col_offset + self.assertIsNone(ast.get_source_segment(s, v)) + self.assertIsNone(ast.get_source_segment(s, w)) + self.assertIsNone(ast.get_source_segment(s, x)) + self.assertIsNone(ast.get_source_segment(s, y)) + + +class BaseNodeVisitorCases: + # Both `NodeVisitor` and `NodeTranformer` must raise these warnings: + def test_old_constant_nodes(self): + class Visitor(self.visitor_class): + def visit_Num(self, node): + log.append((node.lineno, "Num", node.n)) + + def visit_Str(self, node): + log.append((node.lineno, "Str", node.s)) + + def visit_Bytes(self, node): + log.append((node.lineno, "Bytes", node.s)) + + def visit_NameConstant(self, node): + log.append((node.lineno, "NameConstant", node.value)) + + def visit_Ellipsis(self, node): + log.append((node.lineno, "Ellipsis", ...)) + + mod = ast.parse( + dedent("""\ + i = 42 + f = 4.25 + c = 4.25j + s = 'string' + b = b'bytes' + t = True + n = None + e = ... + """) + ) + visitor = Visitor() + log = [] + with warnings.catch_warnings(record=True) as wlog: + warnings.filterwarnings("always", "", DeprecationWarning) + visitor.visit(mod) + self.assertEqual( + log, + [ + (1, "Num", 42), + (2, "Num", 4.25), + (3, "Num", 4.25j), + (4, "Str", "string"), + (5, "Bytes", b"bytes"), + (6, "NameConstant", True), + (7, "NameConstant", None), + (8, "Ellipsis", ...), + ], + ) + self.assertEqual( + [str(w.message) for w in wlog], + [ + "visit_Num is deprecated; add visit_Constant", + "Attribute n is deprecated and will be removed in Python 3.14; use value instead", + "visit_Num is deprecated; add visit_Constant", + "Attribute n is deprecated and will be removed in Python 3.14; use value instead", + "visit_Num is deprecated; add visit_Constant", + "Attribute n is deprecated and will be removed in Python 3.14; use value instead", + "visit_Str is deprecated; add visit_Constant", + "Attribute s is deprecated and will be removed in Python 3.14; use value instead", + "visit_Bytes is deprecated; add visit_Constant", + "Attribute s is deprecated and will be removed in Python 3.14; use value instead", + "visit_NameConstant is deprecated; add visit_Constant", + "visit_NameConstant is deprecated; add visit_Constant", + "visit_Ellipsis is deprecated; add visit_Constant", + ], + ) + + +class NodeVisitorTests(BaseNodeVisitorCases, unittest.TestCase): + visitor_class = ast.NodeVisitor + + +class NodeTransformerTests(ASTTestMixin, BaseNodeVisitorCases, unittest.TestCase): + visitor_class = ast.NodeTransformer + + def assertASTTransformation(self, tranformer_class, initial_code, expected_code): + initial_ast = ast.parse(dedent(initial_code)) + expected_ast = ast.parse(dedent(expected_code)) + + tranformer = tranformer_class() + result_ast = ast.fix_missing_locations(tranformer.visit(initial_ast)) + + self.assertASTEqual(result_ast, expected_ast) + + @unittest.expectedFailure # TODO: RUSTPYTHON; is not + def test_node_remove_single(self): + code = "def func(arg) -> SomeType: ..." + expected = "def func(arg): ..." + + # Since `FunctionDef.returns` is defined as a single value, we test + # the `if isinstance(old_value, AST):` branch here. + class SomeTypeRemover(ast.NodeTransformer): + def visit_Name(self, node: ast.Name): + self.generic_visit(node) + if node.id == "SomeType": + return None + return node + + self.assertASTTransformation(SomeTypeRemover, code, expected) + + def test_node_remove_from_list(self): + code = """ + def func(arg): + print(arg) + yield arg + """ + expected = """ + def func(arg): + print(arg) + """ + + # Since `FunctionDef.body` is defined as a list, we test + # the `if isinstance(old_value, list):` branch here. + class YieldRemover(ast.NodeTransformer): + def visit_Expr(self, node: ast.Expr): + self.generic_visit(node) + if isinstance(node.value, ast.Yield): + return None # Remove `yield` from a function + return node + + self.assertASTTransformation(YieldRemover, code, expected) + + @unittest.expectedFailure # TODO: RUSTPYTHON; is not + def test_node_return_list(self): + code = """ + class DSL(Base, kw1=True): ... + """ + expected = """ + class DSL(Base, kw1=True, kw2=True, kw3=False): ... + """ + + class ExtendKeywords(ast.NodeTransformer): + def visit_keyword(self, node: ast.keyword): + self.generic_visit(node) + if node.arg == "kw1": + return [ + node, + ast.keyword("kw2", ast.Constant(True)), + ast.keyword("kw3", ast.Constant(False)), + ] + return node + + self.assertASTTransformation(ExtendKeywords, code, expected) + + def test_node_mutate(self): + code = """ + def func(arg): + print(arg) + """ + expected = """ + def func(arg): + log(arg) + """ + + class PrintToLog(ast.NodeTransformer): + def visit_Call(self, node: ast.Call): + self.generic_visit(node) + if isinstance(node.func, ast.Name) and node.func.id == "print": + node.func.id = "log" + return node + + self.assertASTTransformation(PrintToLog, code, expected) + + @unittest.expectedFailure # TODO: RUSTPYTHON; is not + def test_node_replace(self): + code = """ + def func(arg): + print(arg) + """ + expected = """ + def func(arg): + logger.log(arg, debug=True) + """ + + class PrintToLog(ast.NodeTransformer): + def visit_Call(self, node: ast.Call): + self.generic_visit(node) + if isinstance(node.func, ast.Name) and node.func.id == "print": + return ast.Call( + func=ast.Attribute( + ast.Name("logger", ctx=ast.Load()), + attr="log", + ctx=ast.Load(), + ), + args=node.args, + keywords=[ast.keyword("debug", ast.Constant(True))], + ) + return node + + self.assertASTTransformation(PrintToLog, code, expected) + + +class ASTConstructorTests(unittest.TestCase): + """Test the autogenerated constructors for AST nodes.""" + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_FunctionDef(self): + args = ast.arguments() + self.assertEqual(args.args, []) + self.assertEqual(args.posonlyargs, []) + with self.assertWarnsRegex( + DeprecationWarning, + r"FunctionDef\.__init__ missing 1 required positional argument: 'name'", + ): + node = ast.FunctionDef(args=args) + self.assertFalse(hasattr(node, "name")) + self.assertEqual(node.decorator_list, []) + node = ast.FunctionDef(name="foo", args=args) + self.assertEqual(node.name, "foo") + self.assertEqual(node.decorator_list, []) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_expr_context(self): + name = ast.Name("x") + self.assertEqual(name.id, "x") + self.assertIsInstance(name.ctx, ast.Load) + + name2 = ast.Name("x", ast.Store()) + self.assertEqual(name2.id, "x") + self.assertIsInstance(name2.ctx, ast.Store) + + name3 = ast.Name("x", ctx=ast.Del()) + self.assertEqual(name3.id, "x") + self.assertIsInstance(name3.ctx, ast.Del) + + with self.assertWarnsRegex( + DeprecationWarning, + r"Name\.__init__ missing 1 required positional argument: 'id'", + ): + name3 = ast.Name() + + def test_custom_subclass_with_no_fields(self): + class NoInit(ast.AST): + pass + + obj = NoInit() + self.assertIsInstance(obj, NoInit) + self.assertEqual(obj.__dict__, {}) + + def test_fields_but_no_field_types(self): + class Fields(ast.AST): + _fields = ("a",) + + obj = Fields() + with self.assertRaises(AttributeError): + obj.a + obj = Fields(a=1) + self.assertEqual(obj.a, 1) + + def test_fields_and_types(self): + class FieldsAndTypes(ast.AST): + _fields = ("a",) + _field_types = {"a": int | None} + a: int | None = None + + obj = FieldsAndTypes() + self.assertIs(obj.a, None) + obj = FieldsAndTypes(a=1) + self.assertEqual(obj.a, 1) + + @unittest.expectedFailure # TODO: RUSTPYTHON; DeprecationWarning not triggered + def test_custom_attributes(self): + class MyAttrs(ast.AST): + _attributes = ("a", "b") + + obj = MyAttrs(a=1, b=2) + self.assertEqual(obj.a, 1) + self.assertEqual(obj.b, 2) + + with self.assertWarnsRegex( + DeprecationWarning, + r"MyAttrs.__init__ got an unexpected keyword argument 'c'.", + ): + obj = MyAttrs(c=3) + + @unittest.expectedFailure # TODO: RUSTPYTHON; DeprecationWarning not triggered + def test_fields_and_types_no_default(self): + class FieldsAndTypesNoDefault(ast.AST): + _fields = ("a",) + _field_types = {"a": int} + + with self.assertWarnsRegex( + DeprecationWarning, + r"FieldsAndTypesNoDefault\.__init__ missing 1 required positional argument: 'a'\.", + ): + obj = FieldsAndTypesNoDefault() + with self.assertRaises(AttributeError): + obj.a + obj = FieldsAndTypesNoDefault(a=1) + self.assertEqual(obj.a, 1) + + @unittest.expectedFailure # TODO: RUSTPYTHON; DeprecationWarning not triggered + def test_incomplete_field_types(self): + class MoreFieldsThanTypes(ast.AST): + _fields = ("a", "b") + _field_types = {"a": int | None} + a: int | None = None + b: int | None = None + + with self.assertWarnsRegex( + DeprecationWarning, + r"Field 'b' is missing from MoreFieldsThanTypes\._field_types", + ): + obj = MoreFieldsThanTypes() + self.assertIs(obj.a, None) + self.assertIs(obj.b, None) + + obj = MoreFieldsThanTypes(a=1, b=2) + self.assertEqual(obj.a, 1) + self.assertEqual(obj.b, 2) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_complete_field_types(self): + class _AllFieldTypes(ast.AST): + _fields = ("a", "b") + _field_types = {"a": int | None, "b": list[str]} + # This must be set explicitly + a: int | None = None + # This will add an implicit empty list default + b: list[str] + + obj = _AllFieldTypes() + self.assertIs(obj.a, None) + self.assertEqual(obj.b, []) + + +@support.cpython_only +class ModuleStateTests(unittest.TestCase): + # bpo-41194, bpo-41261, bpo-41631: The _ast module uses a global state. + + def check_ast_module(self): + # Check that the _ast module still works as expected + code = "x + 1" + filename = "" + mode = "eval" + + # Create _ast.AST subclasses instances + ast_tree = compile(code, filename, mode, flags=ast.PyCF_ONLY_AST) + + # Call PyAST_Check() + code = compile(ast_tree, filename, mode) + self.assertIsInstance(code, types.CodeType) + + def test_reload_module(self): + # bpo-41194: Importing the _ast module twice must not crash. + with support.swap_item(sys.modules, "_ast", None): + del sys.modules["_ast"] + import _ast as ast1 + + del sys.modules["_ast"] + import _ast as ast2 + + self.check_ast_module() + + # Unloading the two _ast module instances must not crash. + del ast1 + del ast2 + support.gc_collect() + + self.check_ast_module() + + def test_sys_modules(self): + # bpo-41631: Test reproducing a Mercurial crash when PyAST_Check() + # imported the _ast module internally. + lazy_mod = object() + + def my_import(name, *args, **kw): + sys.modules[name] = lazy_mod + return lazy_mod + + with support.swap_item(sys.modules, "_ast", None): + del sys.modules["_ast"] + + with support.swap_attr(builtins, "__import__", my_import): + # Test that compile() does not import the _ast module + self.check_ast_module() + self.assertNotIn("_ast", sys.modules) + + # Sanity check of the test itself + import _ast + + self.assertIs(_ast, lazy_mod) + + def test_subinterpreter(self): + # bpo-41631: Importing and using the _ast module in a subinterpreter + # must not crash. + code = dedent(""" + import _ast + import ast + import gc + import sys + import types + + # Create _ast.AST subclasses instances and call PyAST_Check() + ast_tree = compile('x+1', '', 'eval', + flags=ast.PyCF_ONLY_AST) + code = compile(ast_tree, 'string', 'eval') + if not isinstance(code, types.CodeType): + raise AssertionError + + # Unloading the _ast module must not crash. + del ast, _ast + del sys.modules['ast'], sys.modules['_ast'] + gc.collect() + """) + res = support.run_in_subinterp(code) + self.assertEqual(res, 0) + + +class ASTMainTests(unittest.TestCase): + # Tests `ast.main()` function. + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_cli_file_input(self): + code = "print(1, 2, 3)" + expected = ast.dump(ast.parse(code), indent=3) + + with os_helper.temp_dir() as tmp_dir: + filename = os.path.join(tmp_dir, "test_module.py") + with open(filename, "w", encoding="utf-8") as f: + f.write(code) + res, _ = script_helper.run_python_until_end("-m", "ast", filename) + + self.assertEqual(res.err, b"") + self.assertEqual(expected.splitlines(), res.out.decode("utf8").splitlines()) + self.assertEqual(res.rc, 0) + +def compare(left, right): + return ast.dump(left) == ast.dump(right) + +class ASTOptimizationTests(unittest.TestCase): + binop = { + "+": ast.Add(), + "-": ast.Sub(), + "*": ast.Mult(), + "/": ast.Div(), + "%": ast.Mod(), + "<<": ast.LShift(), + ">>": ast.RShift(), + "|": ast.BitOr(), + "^": ast.BitXor(), + "&": ast.BitAnd(), + "//": ast.FloorDiv(), + "**": ast.Pow(), + } + + unaryop = { + "~": ast.Invert(), + "+": ast.UAdd(), + "-": ast.USub(), + } + + def wrap_expr(self, expr): + return ast.Module(body=[ast.Expr(value=expr)]) + + def wrap_statement(self, statement): + return ast.Module(body=[statement]) + + def assert_ast(self, code, non_optimized_target, optimized_target): + + non_optimized_tree = ast.parse(code, optimize=-1) + optimized_tree = ast.parse(code, optimize=1) + + # Is a non-optimized tree equal to a non-optimized target? + self.assertTrue( + compare(non_optimized_tree, non_optimized_target), + f"{ast.dump(non_optimized_target)} must equal " + f"{ast.dump(non_optimized_tree)}", + ) + + # Is a optimized tree equal to a non-optimized target? + self.assertFalse( + compare(optimized_tree, non_optimized_target), + f"{ast.dump(non_optimized_target)} must not equal " + f"{ast.dump(non_optimized_tree)}" + ) + + # Is a optimized tree is equal to an optimized target? + self.assertTrue( + compare(optimized_tree, optimized_target), + f"{ast.dump(optimized_target)} must equal " + f"{ast.dump(optimized_tree)}", + ) + + def create_binop(self, operand, left=ast.Constant(1), right=ast.Constant(1)): + return ast.BinOp(left=left, op=self.binop[operand], right=right) + + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags + def test_folding_binop(self): + code = "1 %s 1" + operators = self.binop.keys() + + for op in operators: + result_code = code % op + non_optimized_target = self.wrap_expr(self.create_binop(op)) + optimized_target = self.wrap_expr(ast.Constant(value=eval(result_code))) + + with self.subTest( + result_code=result_code, + non_optimized_target=non_optimized_target, + optimized_target=optimized_target + ): + self.assert_ast(result_code, non_optimized_target, optimized_target) + + # Multiplication of constant tuples must be folded + code = "(1,) * 3" + non_optimized_target = self.wrap_expr(self.create_binop("*", ast.Tuple(elts=[ast.Constant(value=1)]), ast.Constant(value=3))) + optimized_target = self.wrap_expr(ast.Constant(eval(code))) + + self.assert_ast(code, non_optimized_target, optimized_target) + + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags + def test_folding_unaryop(self): + code = "%s1" + operators = self.unaryop.keys() + + def create_unaryop(operand): + return ast.UnaryOp(op=self.unaryop[operand], operand=ast.Constant(1)) + + for op in operators: + result_code = code % op + non_optimized_target = self.wrap_expr(create_unaryop(op)) + optimized_target = self.wrap_expr(ast.Constant(eval(result_code))) + + with self.subTest( + result_code=result_code, + non_optimized_target=non_optimized_target, + optimized_target=optimized_target + ): + self.assert_ast(result_code, non_optimized_target, optimized_target) + + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags + def test_folding_not(self): + code = "not (1 %s (1,))" + operators = { + "in": ast.In(), + "is": ast.Is(), + } + opt_operators = { + "is": ast.IsNot(), + "in": ast.NotIn(), + } + + def create_notop(operand): + return ast.UnaryOp(op=ast.Not(), operand=ast.Compare( + left=ast.Constant(value=1), + ops=[operators[operand]], + comparators=[ast.Tuple(elts=[ast.Constant(value=1)])] + )) + + for op in operators.keys(): + result_code = code % op + non_optimized_target = self.wrap_expr(create_notop(op)) + optimized_target = self.wrap_expr( + ast.Compare(left=ast.Constant(1), ops=[opt_operators[op]], comparators=[ast.Constant(value=(1,))]) + ) + + with self.subTest( + result_code=result_code, + non_optimized_target=non_optimized_target, + optimized_target=optimized_target + ): + self.assert_ast(result_code, non_optimized_target, optimized_target) + + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags + def test_folding_format(self): + code = "'%s' % (a,)" + + non_optimized_target = self.wrap_expr( + ast.BinOp( + left=ast.Constant(value="%s"), + op=ast.Mod(), + right=ast.Tuple(elts=[ast.Name(id='a')])) + ) + optimized_target = self.wrap_expr( + ast.JoinedStr( + values=[ + ast.FormattedValue(value=ast.Name(id='a'), conversion=115) + ] + ) + ) + + self.assert_ast(code, non_optimized_target, optimized_target) + + + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags + def test_folding_tuple(self): + code = "(1,)" + + non_optimized_target = self.wrap_expr(ast.Tuple(elts=[ast.Constant(1)])) + optimized_target = self.wrap_expr(ast.Constant(value=(1,))) + + self.assert_ast(code, non_optimized_target, optimized_target) + + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags + def test_folding_comparator(self): + code = "1 %s %s1%s" + operators = [("in", ast.In()), ("not in", ast.NotIn())] + braces = [ + ("[", "]", ast.List, (1,)), + ("{", "}", ast.Set, frozenset({1})), + ] + for left, right, non_optimized_comparator, optimized_comparator in braces: + for op, node in operators: + non_optimized_target = self.wrap_expr(ast.Compare( + left=ast.Constant(1), ops=[node], + comparators=[non_optimized_comparator(elts=[ast.Constant(1)])] + )) + optimized_target = self.wrap_expr(ast.Compare( + left=ast.Constant(1), ops=[node], + comparators=[ast.Constant(value=optimized_comparator)] + )) + self.assert_ast(code % (op, left, right), non_optimized_target, optimized_target) + + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags + def test_folding_iter(self): + code = "for _ in %s1%s: pass" + braces = [ + ("[", "]", ast.List, (1,)), + ("{", "}", ast.Set, frozenset({1})), + ] + + for left, right, ast_cls, optimized_iter in braces: + non_optimized_target = self.wrap_statement(ast.For( + target=ast.Name(id="_", ctx=ast.Store()), + iter=ast_cls(elts=[ast.Constant(1)]), + body=[ast.Pass()] + )) + optimized_target = self.wrap_statement(ast.For( + target=ast.Name(id="_", ctx=ast.Store()), + iter=ast.Constant(value=optimized_iter), + body=[ast.Pass()] + )) + + self.assert_ast(code % (left, right), non_optimized_target, optimized_target) + + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags + def test_folding_subscript(self): + code = "(1,)[0]" + + non_optimized_target = self.wrap_expr( + ast.Subscript(value=ast.Tuple(elts=[ast.Constant(value=1)]), slice=ast.Constant(value=0)) + ) + optimized_target = self.wrap_expr(ast.Constant(value=1)) + + self.assert_ast(code, non_optimized_target, optimized_target) + + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags + def test_folding_type_param_in_function_def(self): + code = "def foo[%s = 1 + 1](): pass" + + unoptimized_binop = self.create_binop("+") + unoptimized_type_params = [ + ("T", "T", ast.TypeVar), + ("**P", "P", ast.ParamSpec), + ("*Ts", "Ts", ast.TypeVarTuple), + ] + + for type, name, type_param in unoptimized_type_params: + result_code = code % type + optimized_target = self.wrap_statement( + ast.FunctionDef( + name='foo', + args=ast.arguments(), + body=[ast.Pass()], + type_params=[type_param(name=name, default_value=ast.Constant(2))] + ) + ) + non_optimized_target = self.wrap_statement( + ast.FunctionDef( + name='foo', + args=ast.arguments(), + body=[ast.Pass()], + type_params=[type_param(name=name, default_value=unoptimized_binop)] + ) + ) + self.assert_ast(result_code, non_optimized_target, optimized_target) + + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags + def test_folding_type_param_in_class_def(self): + code = "class foo[%s = 1 + 1]: pass" + + unoptimized_binop = self.create_binop("+") + unoptimized_type_params = [ + ("T", "T", ast.TypeVar), + ("**P", "P", ast.ParamSpec), + ("*Ts", "Ts", ast.TypeVarTuple), + ] + + for type, name, type_param in unoptimized_type_params: + result_code = code % type + optimized_target = self.wrap_statement( + ast.ClassDef( + name='foo', + body=[ast.Pass()], + type_params=[type_param(name=name, default_value=ast.Constant(2))] + ) + ) + non_optimized_target = self.wrap_statement( + ast.ClassDef( + name='foo', + body=[ast.Pass()], + type_params=[type_param(name=name, default_value=unoptimized_binop)] + ) + ) + self.assert_ast(result_code, non_optimized_target, optimized_target) + + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: compile() unrecognized flags + def test_folding_type_param_in_type_alias(self): + code = "type foo[%s = 1 + 1] = 1" + + unoptimized_binop = self.create_binop("+") + unoptimized_type_params = [ + ("T", "T", ast.TypeVar), + ("**P", "P", ast.ParamSpec), + ("*Ts", "Ts", ast.TypeVarTuple), + ] + + for type, name, type_param in unoptimized_type_params: + result_code = code % type + optimized_target = self.wrap_statement( + ast.TypeAlias( + name=ast.Name(id='foo', ctx=ast.Store()), + type_params=[type_param(name=name, default_value=ast.Constant(2))], + value=ast.Constant(value=1), + ) + ) + non_optimized_target = self.wrap_statement( + ast.TypeAlias( + name=ast.Name(id='foo', ctx=ast.Store()), + type_params=[type_param(name=name, default_value=unoptimized_binop)], + value=ast.Constant(value=1), + ) + ) + self.assert_ast(result_code, non_optimized_target, optimized_target) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_ast/utils.py b/Lib/test/test_ast/utils.py new file mode 100644 index 0000000000..145e89ee94 --- /dev/null +++ b/Lib/test/test_ast/utils.py @@ -0,0 +1,15 @@ +def to_tuple(t): + if t is None or isinstance(t, (str, int, complex, float, bytes)) or t is Ellipsis: + return t + elif isinstance(t, list): + return [to_tuple(e) for e in t] + result = [t.__class__.__name__] + if hasattr(t, 'lineno') and hasattr(t, 'col_offset'): + result.append((t.lineno, t.col_offset)) + if hasattr(t, 'end_lineno') and hasattr(t, 'end_col_offset'): + result[-1] += (t.end_lineno, t.end_col_offset) + if t._fields is None: + return tuple(result) + for f in t._fields: + result.append(to_tuple(getattr(t, f))) + return tuple(result) diff --git a/Lib/test/test_asyncgen.py b/Lib/test/test_asyncgen.py index f97316adeb..7039bd7054 100644 --- a/Lib/test/test_asyncgen.py +++ b/Lib/test/test_asyncgen.py @@ -375,8 +375,6 @@ async def async_gen_wrapper(): self.compare_generators(sync_gen_wrapper(), async_gen_wrapper()) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_async_gen_api_01(self): async def gen(): yield 123 @@ -467,16 +465,12 @@ async def test_throw(): result = self.loop.run_until_complete(test_throw()) self.assertEqual(result, "completed") - # TODO: RUSTPYTHON, NameError: name 'anext' is not defined - @unittest.expectedFailure def test_async_generator_anext(self): async def agen(): yield 1 yield 2 self.check_async_iterator_anext(agen) - # TODO: RUSTPYTHON, NameError: name 'anext' is not defined - @unittest.expectedFailure def test_python_async_iterator_anext(self): class MyAsyncIter: """Asynchronously yield 1, then 2.""" @@ -492,8 +486,6 @@ async def __anext__(self): return self.yielded self.check_async_iterator_anext(MyAsyncIter) - # TODO: RUSTPYTHON, NameError: name 'anext' is not defined - @unittest.expectedFailure def test_python_async_iterator_types_coroutine_anext(self): import types class MyAsyncIterWithTypesCoro: @@ -523,8 +515,6 @@ async def consume(): res = self.loop.run_until_complete(consume()) self.assertEqual(res, [1, 2]) - # TODO: RUSTPYTHON, NameError: name 'aiter' is not defined - @unittest.expectedFailure def test_async_gen_aiter_class(self): results = [] class Gen: @@ -549,8 +539,6 @@ async def gen(): applied_twice = aiter(applied_once) self.assertIs(applied_once, applied_twice) - # TODO: RUSTPYTHON, NameError: name 'anext' is not defined - @unittest.expectedFailure def test_anext_bad_args(self): async def gen(): yield 1 @@ -571,8 +559,6 @@ async def call_with_kwarg(): with self.assertRaises(TypeError): self.loop.run_until_complete(call_with_kwarg()) - # TODO: RUSTPYTHON, NameError: name 'anext' is not defined - @unittest.expectedFailure def test_anext_bad_await(self): async def bad_awaitable(): class BadAwaitable: @@ -603,8 +589,6 @@ async def check_anext_returning_iterator(self, aiter_class): await awaitable return "completed" - # TODO: RUSTPYTHON, NameError: name 'anext' is not defined - @unittest.expectedFailure def test_anext_return_iterator(self): class WithIterAnext: def __aiter__(self): @@ -614,8 +598,6 @@ def __anext__(self): result = self.loop.run_until_complete(self.check_anext_returning_iterator(WithIterAnext)) self.assertEqual(result, "completed") - # TODO: RUSTPYTHON, NameError: name 'anext' is not defined - @unittest.expectedFailure def test_anext_return_generator(self): class WithGenAnext: def __aiter__(self): @@ -625,8 +607,6 @@ def __anext__(self): result = self.loop.run_until_complete(self.check_anext_returning_iterator(WithGenAnext)) self.assertEqual(result, "completed") - # TODO: RUSTPYTHON, NameError: name 'anext' is not defined - @unittest.expectedFailure def test_anext_await_raises(self): class RaisingAwaitable: def __await__(self): @@ -648,8 +628,6 @@ async def do_test(): result = self.loop.run_until_complete(do_test()) self.assertEqual(result, "completed") - # TODO: RUSTPYTHON, NameError: name 'anext' is not defined - @unittest.expectedFailure def test_anext_iter(self): @types.coroutine def _async_yield(v): @@ -1507,8 +1485,6 @@ async def main(): self.assertEqual(messages, []) - # TODO: RUSTPYTHON, ValueError: not enough values to unpack (expected 1, got 0) - @unittest.expectedFailure def test_async_gen_asyncio_shutdown_exception_01(self): messages = [] diff --git a/Lib/test/test_binascii.py b/Lib/test/test_binascii.py index 40a2ca9f76..cf11ffce7f 100644 --- a/Lib/test/test_binascii.py +++ b/Lib/test/test_binascii.py @@ -4,7 +4,8 @@ import binascii import array import re -from test.support import bigmemtest, _1G, _4G, warnings_helper +from test.support import bigmemtest, _1G, _4G +from test.support.hypothesis_helper import hypothesis # Note: "*_hex" functions are aliases for "(un)hexlify" @@ -27,6 +28,14 @@ class BinASCIITest(unittest.TestCase): def setUp(self): self.data = self.type2test(self.rawdata) + def assertConversion(self, original, converted, restored, **kwargs): + self.assertIsInstance(original, bytes) + self.assertIsInstance(converted, bytes) + self.assertIsInstance(restored, bytes) + if converted: + self.assertLess(max(converted), 128) + self.assertEqual(original, restored, msg=f'{self.type2test=} {kwargs=}') + def test_exceptions(self): # Check module exceptions self.assertTrue(issubclass(binascii.Error, Exception)) @@ -52,9 +61,7 @@ def test_returned_value(self): self.fail("{}/{} conversion raises {!r}".format(fb, fa, err)) self.assertEqual(res, raw, "{}/{} conversion: " "{!r} != {!r}".format(fb, fa, res, raw)) - self.assertIsInstance(res, bytes) - self.assertIsInstance(a, bytes) - self.assertLess(max(a), 128) + self.assertConversion(raw, a, res) self.assertIsInstance(binascii.crc_hqx(raw, 0), int) self.assertIsInstance(binascii.crc32(raw), int) @@ -110,6 +117,7 @@ def addnoise(line): # empty strings. TBD: shouldn't it raise an exception instead ? self.assertEqual(binascii.a2b_base64(self.type2test(fillers)), b'') + @unittest.expectedFailure # TODO: RUSTPYTHON def test_base64_strict_mode(self): # Test base64 with strict mode on def _assertRegexTemplate(assert_regex: str, data: bytes, non_strict_mode_expected_result: bytes): @@ -132,13 +140,21 @@ def assertLeadingPadding(data, non_strict_mode_expected_result: bytes): def assertDiscontinuousPadding(data, non_strict_mode_expected_result: bytes): _assertRegexTemplate(r'(?i)Discontinuous padding', data, non_strict_mode_expected_result) + def assertExcessPadding(data, non_strict_mode_expected_result: bytes): + _assertRegexTemplate(r'(?i)Excess padding', data, non_strict_mode_expected_result) + # Test excess data exceptions assertExcessData(b'ab==a', b'i') assertExcessData(b'ab===', b'i') + assertExcessData(b'ab====', b'i') assertExcessData(b'ab==:', b'i') assertExcessData(b'abc=a', b'i\xb7') assertExcessData(b'abc=:', b'i\xb7') assertExcessData(b'ab==\n', b'i') + assertExcessData(b'abc==', b'i\xb7') + assertExcessData(b'abc===', b'i\xb7') + assertExcessData(b'abc====', b'i\xb7') + assertExcessData(b'abc=====', b'i\xb7') # Test non-base64 data exceptions assertNonBase64Data(b'\nab==', b'i') @@ -150,8 +166,16 @@ def assertDiscontinuousPadding(data, non_strict_mode_expected_result: bytes): assertLeadingPadding(b'=', b'') assertLeadingPadding(b'==', b'') assertLeadingPadding(b'===', b'') + assertLeadingPadding(b'====', b'') + assertLeadingPadding(b'=====', b'') assertDiscontinuousPadding(b'ab=c=', b'i\xb7') assertDiscontinuousPadding(b'ab=ab==', b'i\xb6\x9b') + assertExcessPadding(b'abcd=', b'i\xb7\x1d') + assertExcessPadding(b'abcd==', b'i\xb7\x1d') + assertExcessPadding(b'abcd===', b'i\xb7\x1d') + assertExcessPadding(b'abcd====', b'i\xb7\x1d') + assertExcessPadding(b'abcd=====', b'i\xb7\x1d') + def test_base64errors(self): # Test base64 with invalid padding @@ -221,6 +245,15 @@ def test_uu(self): with self.assertRaises(TypeError): binascii.b2a_uu(b"", True) + @hypothesis.given( + binary=hypothesis.strategies.binary(max_size=45), + backtick=hypothesis.strategies.booleans(), + ) + def test_b2a_roundtrip(self, binary, backtick): + converted = binascii.b2a_uu(self.type2test(binary), backtick=backtick) + restored = binascii.a2b_uu(self.type2test(converted)) + self.assertConversion(binary, converted, restored, backtick=backtick) + def test_crc_hqx(self): crc = binascii.crc_hqx(self.type2test(b"Test the CRC-32 of"), 0) crc = binascii.crc_hqx(self.type2test(b" this string."), crc) @@ -258,6 +291,12 @@ def test_hex(self): self.assertEqual(binascii.hexlify(self.type2test(s)), t) self.assertEqual(binascii.unhexlify(self.type2test(t)), u) + @hypothesis.given(binary=hypothesis.strategies.binary()) + def test_hex_roundtrip(self, binary): + converted = binascii.hexlify(self.type2test(binary)) + restored = binascii.unhexlify(self.type2test(converted)) + self.assertConversion(binary, converted, restored) + def test_hex_separator(self): """Test that hexlify and b2a_hex are binary versions of bytes.hex.""" # Logic of separators is tested in test_bytes.py. This checks that @@ -372,6 +411,21 @@ def test_qp(self): self.assertEqual(b2a_qp(type2test(b'a.\n')), b'a.\n') self.assertEqual(b2a_qp(type2test(b'.a')[:-1]), b'=2E') + @hypothesis.given( + binary=hypothesis.strategies.binary(), + quotetabs=hypothesis.strategies.booleans(), + istext=hypothesis.strategies.booleans(), + header=hypothesis.strategies.booleans(), + ) + def test_b2a_qp_a2b_qp_round_trip(self, binary, quotetabs, istext, header): + converted = binascii.b2a_qp( + self.type2test(binary), + quotetabs=quotetabs, istext=istext, header=header, + ) + restored = binascii.a2b_qp(self.type2test(converted), header=header) + self.assertConversion(binary, converted, restored, + quotetabs=quotetabs, istext=istext, header=header) + def test_empty_string(self): # A test for SF bug #1022953. Make sure SystemError is not raised. empty = self.type2test(b'') @@ -427,6 +481,21 @@ def test_b2a_base64_newline(self): self.assertEqual(binascii.b2a_base64(b, newline=False), b'aGVsbG8=') + @hypothesis.given( + binary=hypothesis.strategies.binary(), + newline=hypothesis.strategies.booleans(), + ) + def test_base64_roundtrip(self, binary, newline): + converted = binascii.b2a_base64(self.type2test(binary), newline=newline) + restored = binascii.a2b_base64(self.type2test(converted)) + self.assertConversion(binary, converted, restored, newline=newline) + + def test_c_contiguity(self): + m = memoryview(bytearray(b'noncontig')) + noncontig_writable = m[::-2] + with self.assertRaises(BufferError): + binascii.b2a_hex(noncontig_writable) + class ArrayBinASCIITest(BinASCIITest): def type2test(self, s): diff --git a/Lib/test/test_builtin.py b/Lib/test/test_builtin.py index b4119305f9..7d8c7a5e01 100644 --- a/Lib/test/test_builtin.py +++ b/Lib/test/test_builtin.py @@ -153,8 +153,6 @@ def check_iter_pickle(self, it, seq, proto): it = pickle.loads(d) self.assertEqual(list(it), seq[1:]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_import(self): __import__('sys') __import__('time') @@ -1358,7 +1356,6 @@ def test_open_default_encoding(self): os.environ.clear() os.environ.update(old_environ) - @unittest.expectedFailureIfWindows('TODO: RUSTPYTHON Windows') @support.requires_subprocess() def test_open_non_inheritable(self): fileobj = open(__file__, encoding="utf-8") @@ -2402,8 +2399,6 @@ def test_type_nokwargs(self): with self.assertRaises(TypeError): type('a', (), dict={}) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_type_name(self): for name in 'A', '\xc4', '\U0001f40d', 'B.A', '42', '': with self.subTest(name=name): @@ -2453,8 +2448,6 @@ def test_type_qualname(self): A.__qualname__ = b'B' self.assertEqual(A.__qualname__, 'D.E') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_type_doc(self): for doc in 'x', '\xc4', '\U0001f40d', 'x\x00y', b'x', 42, None: A = type('A', (), {'__doc__': doc}) @@ -2488,8 +2481,6 @@ def test_bad_args(self): with self.assertRaises(TypeError): type('A', (int, str), {}) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_bad_slots(self): with self.assertRaises(TypeError): type('A', (), {'__slots__': b'x'}) diff --git a/Lib/test/test_bytes.py b/Lib/test/test_bytes.py index 3c634b6cac..8f01f89030 100644 --- a/Lib/test/test_bytes.py +++ b/Lib/test/test_bytes.py @@ -10,6 +10,7 @@ import sys import copy import functools +import operator import pickle import tempfile import textwrap @@ -46,6 +47,10 @@ def __index__(self): class BaseBytesTest: + def assertTypedEqual(self, actual, expected): + self.assertIs(type(actual), type(expected)) + self.assertEqual(actual, expected) + def test_basics(self): b = self.type2test() self.assertEqual(type(b), self.type2test) @@ -196,8 +201,7 @@ def test_constructor_value_errors(self): self.assertRaises(ValueError, self.type2test, [sys.maxsize+1]) self.assertRaises(ValueError, self.type2test, [10**100]) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON @bigaddrspacetest def test_constructor_overflow(self): size = MAX_Py_ssize_t @@ -321,8 +325,7 @@ def test_decode(self): # Default encoding is utf-8 self.assertEqual(self.type2test(b'\xe2\x98\x83').decode(), '\u2603') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_check_encoding_errors(self): # bpo-37388: bytes(str) and bytes.encode() must check encoding # and errors arguments in dev mode @@ -737,6 +740,37 @@ def check(fmt, vals, result): check(b'%i%b %*.*b', (10, b'3', 5, 3, b'abc',), b'103 abc') check(b'%c', b'a', b'a') + class PseudoFloat: + def __init__(self, value): + self.value = float(value) + def __int__(self): + return int(self.value) + + pi = PseudoFloat(3.1415) + + exceptions_params = [ + ('%x format: an integer is required, not float', b'%x', 3.14), + ('%X format: an integer is required, not float', b'%X', 2.11), + ('%o format: an integer is required, not float', b'%o', 1.79), + ('%x format: an integer is required, not PseudoFloat', b'%x', pi), + ('%x format: an integer is required, not complex', b'%x', 3j), + ('%X format: an integer is required, not complex', b'%X', 2j), + ('%o format: an integer is required, not complex', b'%o', 1j), + ('%u format: a real number is required, not complex', b'%u', 3j), + # See https://github.com/python/cpython/issues/130928 as for why + # the exception message contains '%d' instead of '%i'. + ('%d format: a real number is required, not complex', b'%i', 2j), + ('%d format: a real number is required, not complex', b'%d', 2j), + ( + r'%c requires an integer in range\(256\) or a single byte', + b'%c', pi + ), + ] + + for msg, format_bytes, value in exceptions_params: + with self.assertRaisesRegex(TypeError, msg): + operator.mod(format_bytes, value) + def test_imod(self): b = self.type2test(b'hello, %b!') orig = b @@ -936,8 +970,7 @@ def test_integer_arguments_out_of_byte_range(self): self.assertRaises(ValueError, method, 256) self.assertRaises(ValueError, method, 9999) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_find_etc_raise_correct_error_messages(self): # issue 11828 b = self.type2test(b'hello') @@ -957,8 +990,7 @@ def test_find_etc_raise_correct_error_messages(self): self.assertRaisesRegex(TypeError, r'\bendswith\b', b.endswith, x, None, None, None) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_free_after_iterating(self): test.support.check_free_after_iterating(self, iter, self.type2test) test.support.check_free_after_iterating(self, reversed, self.type2test) @@ -995,13 +1027,13 @@ def test_translate(self): self.assertEqual(c, b'hllo') def test_sq_item(self): - _testcapi = import_helper.import_module('_testcapi') + _testlimitedcapi = import_helper.import_module('_testlimitedcapi') obj = self.type2test((42,)) with self.assertRaises(IndexError): - _testcapi.sequence_getitem(obj, -2) + _testlimitedcapi.sequence_getitem(obj, -2) with self.assertRaises(IndexError): - _testcapi.sequence_getitem(obj, 1) - self.assertEqual(_testcapi.sequence_getitem(obj, 0), 42) + _testlimitedcapi.sequence_getitem(obj, 1) + self.assertEqual(_testlimitedcapi.sequence_getitem(obj, 0), 42) class BytesTest(BaseBytesTest, unittest.TestCase): @@ -1031,36 +1063,63 @@ def test_buffer_is_readonly(self): self.assertRaises(TypeError, f.readinto, b"") def test_custom(self): - class A: - def __bytes__(self): - return b'abc' - self.assertEqual(bytes(A()), b'abc') - class A: pass - self.assertRaises(TypeError, bytes, A()) - class A: - def __bytes__(self): - return None - self.assertRaises(TypeError, bytes, A()) - class A: + self.assertEqual(bytes(BytesSubclass(b'abc')), b'abc') + self.assertEqual(BytesSubclass(OtherBytesSubclass(b'abc')), + BytesSubclass(b'abc')) + self.assertEqual(bytes(WithBytes(b'abc')), b'abc') + self.assertEqual(BytesSubclass(WithBytes(b'abc')), BytesSubclass(b'abc')) + + class NoBytes: pass + self.assertRaises(TypeError, bytes, NoBytes()) + self.assertRaises(TypeError, bytes, WithBytes('abc')) + self.assertRaises(TypeError, bytes, WithBytes(None)) + class IndexWithBytes: def __bytes__(self): return b'a' def __index__(self): return 42 - self.assertEqual(bytes(A()), b'a') + self.assertEqual(bytes(IndexWithBytes()), b'a') # Issue #25766 - class A(str): + class StrWithBytes(str): + def __new__(cls, value): + self = str.__new__(cls, '\u20ac') + self.value = value + return self def __bytes__(self): - return b'abc' - self.assertEqual(bytes(A('\u20ac')), b'abc') - self.assertEqual(bytes(A('\u20ac'), 'iso8859-15'), b'\xa4') + return self.value + self.assertEqual(bytes(StrWithBytes(b'abc')), b'abc') + self.assertEqual(bytes(StrWithBytes(b'abc'), 'iso8859-15'), b'\xa4') + self.assertEqual(bytes(StrWithBytes(BytesSubclass(b'abc'))), b'abc') + self.assertEqual(BytesSubclass(StrWithBytes(b'abc')), BytesSubclass(b'abc')) + self.assertEqual(BytesSubclass(StrWithBytes(b'abc'), 'iso8859-15'), + BytesSubclass(b'\xa4')) + self.assertEqual(BytesSubclass(StrWithBytes(BytesSubclass(b'abc'))), + BytesSubclass(b'abc')) + self.assertEqual(BytesSubclass(StrWithBytes(OtherBytesSubclass(b'abc'))), + BytesSubclass(b'abc')) # Issue #24731 - class A: + self.assertTypedEqual(bytes(WithBytes(BytesSubclass(b'abc'))), BytesSubclass(b'abc')) + self.assertTypedEqual(BytesSubclass(WithBytes(BytesSubclass(b'abc'))), + BytesSubclass(b'abc')) + self.assertTypedEqual(BytesSubclass(WithBytes(OtherBytesSubclass(b'abc'))), + BytesSubclass(b'abc')) + + class BytesWithBytes(bytes): + def __new__(cls, value): + self = bytes.__new__(cls, b'\xa4') + self.value = value + return self def __bytes__(self): - return OtherBytesSubclass(b'abc') - self.assertEqual(bytes(A()), b'abc') - self.assertIs(type(bytes(A())), OtherBytesSubclass) - self.assertEqual(BytesSubclass(A()), b'abc') - self.assertIs(type(BytesSubclass(A())), BytesSubclass) + return self.value + self.assertTypedEqual(bytes(BytesWithBytes(b'abc')), b'abc') + self.assertTypedEqual(BytesSubclass(BytesWithBytes(b'abc')), + BytesSubclass(b'abc')) + self.assertTypedEqual(bytes(BytesWithBytes(BytesSubclass(b'abc'))), + BytesSubclass(b'abc')) + self.assertTypedEqual(BytesSubclass(BytesWithBytes(BytesSubclass(b'abc'))), + BytesSubclass(b'abc')) + self.assertTypedEqual(BytesSubclass(BytesWithBytes(OtherBytesSubclass(b'abc'))), + BytesSubclass(b'abc')) # Test PyBytes_FromFormat() def test_from_format(self): @@ -1233,6 +1292,8 @@ class SubBytes(bytes): class ByteArrayTest(BaseBytesTest, unittest.TestCase): type2test = bytearray + _testlimitedcapi = import_helper.import_module('_testlimitedcapi') + def test_getitem_error(self): b = bytearray(b'python') msg = "bytearray indices must be integers or slices" @@ -1325,47 +1386,73 @@ def by(s): self.assertEqual(re.findall(br"\w+", b), [by("Hello"), by("world")]) def test_setitem(self): - b = bytearray([1, 2, 3]) - b[1] = 100 - self.assertEqual(b, bytearray([1, 100, 3])) - b[-1] = 200 - self.assertEqual(b, bytearray([1, 100, 200])) - b[0] = Indexable(10) - self.assertEqual(b, bytearray([10, 100, 200])) - try: - b[3] = 0 - self.fail("Didn't raise IndexError") - except IndexError: - pass - try: - b[-10] = 0 - self.fail("Didn't raise IndexError") - except IndexError: - pass - try: - b[0] = 256 - self.fail("Didn't raise ValueError") - except ValueError: - pass - try: - b[0] = Indexable(-1) - self.fail("Didn't raise ValueError") - except ValueError: - pass - try: - b[0] = None - self.fail("Didn't raise TypeError") - except TypeError: - pass + def setitem_as_mapping(b, i, val): + b[i] = val + + def setitem_as_sequence(b, i, val): + self._testlimitedcapi.sequence_setitem(b, i, val) + + def do_tests(setitem): + b = bytearray([1, 2, 3]) + setitem(b, 1, 100) + self.assertEqual(b, bytearray([1, 100, 3])) + setitem(b, -1, 200) + self.assertEqual(b, bytearray([1, 100, 200])) + setitem(b, 0, Indexable(10)) + self.assertEqual(b, bytearray([10, 100, 200])) + try: + setitem(b, 3, 0) + self.fail("Didn't raise IndexError") + except IndexError: + pass + try: + setitem(b, -10, 0) + self.fail("Didn't raise IndexError") + except IndexError: + pass + try: + setitem(b, 0, 256) + self.fail("Didn't raise ValueError") + except ValueError: + pass + try: + setitem(b, 0, Indexable(-1)) + self.fail("Didn't raise ValueError") + except ValueError: + pass + try: + setitem(b, 0, object()) + self.fail("Didn't raise TypeError") + except TypeError: + pass + + with self.subTest("tp_as_mapping"): + do_tests(setitem_as_mapping) + + with self.subTest("tp_as_sequence"): + do_tests(setitem_as_sequence) def test_delitem(self): - b = bytearray(range(10)) - del b[0] - self.assertEqual(b, bytearray(range(1, 10))) - del b[-1] - self.assertEqual(b, bytearray(range(1, 9))) - del b[4] - self.assertEqual(b, bytearray([1, 2, 3, 4, 6, 7, 8])) + def del_as_mapping(b, i): + del b[i] + + def del_as_sequence(b, i): + self._testlimitedcapi.sequence_delitem(b, i) + + def do_tests(delete): + b = bytearray(range(10)) + delete(b, 0) + self.assertEqual(b, bytearray(range(1, 10))) + delete(b, -1) + self.assertEqual(b, bytearray(range(1, 9))) + delete(b, 4) + self.assertEqual(b, bytearray([1, 2, 3, 4, 6, 7, 8])) + + with self.subTest("tp_as_mapping"): + do_tests(del_as_mapping) + + with self.subTest("tp_as_sequence"): + do_tests(del_as_sequence) def test_setslice(self): b = bytearray(range(10)) @@ -1558,6 +1645,13 @@ def test_extend(self): a = bytearray(b'') a.extend([Indexable(ord('a'))]) self.assertEqual(a, b'a') + a = bytearray(b'abc') + self.assertRaisesRegex(TypeError, # Override for string. + "expected iterable of integers; got: 'str'", + a.extend, 'def') + self.assertRaisesRegex(TypeError, # But not for others. + "can't extend bytearray with float", + a.extend, 1.0) def test_remove(self): b = bytearray(b'hello') @@ -1747,6 +1841,10 @@ def test_repeat_after_setslice(self): self.assertEqual(b3, b'xcxcxc') def test_mutating_index(self): + # bytearray slice assignment can call into python code + # that reallocates the internal buffer + # See gh-91153 + class Boom: def __index__(self): b.clear() @@ -1758,10 +1856,42 @@ def __index__(self): b[0] = Boom() with self.subTest("tp_as_sequence"): - _testcapi = import_helper.import_module('_testcapi') b = bytearray(b'Now you see me...') with self.assertRaises(IndexError): - _testcapi.sequence_setitem(b, 0, Boom()) + self._testlimitedcapi.sequence_setitem(b, 0, Boom()) + + def test_mutating_index_inbounds(self): + # gh-91153 continued + # Ensure buffer is not broken even if length is correct + + class MutatesOnIndex: + def __init__(self): + self.ba = bytearray(0x180) + + def __index__(self): + self.ba.clear() + self.new_ba = bytearray(0x180) # to catch out-of-bounds writes + self.ba.extend([0] * 0x180) # to check bounds checks + return 0 + + with self.subTest("skip_bounds_safety"): + instance = MutatesOnIndex() + instance.ba[instance] = ord("?") + self.assertEqual(instance.ba[0], ord("?"), "Assigned bytearray not altered") + self.assertEqual(instance.new_ba, bytearray(0x180), "Wrong object altered") + + with self.subTest("skip_bounds_safety_capi"): + instance = MutatesOnIndex() + instance.ba[instance] = ord("?") + self._testlimitedcapi.sequence_setitem(instance.ba, instance, ord("?")) + self.assertEqual(instance.ba[0], ord("?"), "Assigned bytearray not altered") + self.assertEqual(instance.new_ba, bytearray(0x180), "Wrong object altered") + + with self.subTest("skip_bounds_safety_slice"): + instance = MutatesOnIndex() + instance.ba[instance:1] = [ord("?")] + self.assertEqual(instance.ba[0], ord("?"), "Assigned bytearray not altered") + self.assertEqual(instance.new_ba, bytearray(0x180), "Wrong object altered") class AssortedBytesTest(unittest.TestCase): @@ -1769,16 +1899,43 @@ class AssortedBytesTest(unittest.TestCase): # Test various combinations of bytes and bytearray # + def test_bytes_repr(self, f=repr): + self.assertEqual(f(b''), "b''") + self.assertEqual(f(b"abc"), "b'abc'") + self.assertEqual(f(bytes([92])), r"b'\\'") + self.assertEqual(f(bytes([0, 1, 254, 255])), r"b'\x00\x01\xfe\xff'") + self.assertEqual(f(b'\a\b\t\n\v\f\r'), r"b'\x07\x08\t\n\x0b\x0c\r'") + self.assertEqual(f(b'"'), """b'"'""") # '"' + self.assertEqual(f(b"'"), '''b"'"''') # "'" + self.assertEqual(f(b"'\""), r"""b'\'"'""") # '\'"' + self.assertEqual(f(b"\"'\""), r"""b'"\'"'""") # '"\'"' + self.assertEqual(f(b"'\"'"), r"""b'\'"\''""") # '\'"\'' + self.assertEqual(f(BytesSubclass(b"abc")), "b'abc'") + + def test_bytearray_repr(self, f=repr): + self.assertEqual(f(bytearray()), "bytearray(b'')") + self.assertEqual(f(bytearray(b'abc')), "bytearray(b'abc')") + self.assertEqual(f(bytearray([92])), r"bytearray(b'\\')") + self.assertEqual(f(bytearray([0, 1, 254, 255])), + r"bytearray(b'\x00\x01\xfe\xff')") + self.assertEqual(f(bytearray([7, 8, 9, 10, 11, 12, 13])), + r"bytearray(b'\x07\x08\t\n\x0b\x0c\r')") + self.assertEqual(f(bytearray(b'"')), """bytearray(b'"')""") # '"' + self.assertEqual(f(bytearray(b"'")), r'''bytearray(b"\'")''') # "\'" + self.assertEqual(f(bytearray(b"'\"")), r"""bytearray(b'\'"')""") # '\'"' + self.assertEqual(f(bytearray(b"\"'\"")), r"""bytearray(b'"\'"')""") # '"\'"' + self.assertEqual(f(bytearray(b'\'"\'')), r"""bytearray(b'\'"\'')""") # '\'"\'' + self.assertEqual(f(ByteArraySubclass(b"abc")), "ByteArraySubclass(b'abc')") + self.assertEqual(f(ByteArraySubclass.Nested(b"abc")), "Nested(b'abc')") + self.assertEqual(f(ByteArraySubclass.Ŭñıçöđë(b"abc")), "Ŭñıçöđë(b'abc')") + @check_bytes_warnings - def test_repr_str(self): - for f in str, repr: - self.assertEqual(f(bytearray()), "bytearray(b'')") - self.assertEqual(f(bytearray([0])), "bytearray(b'\\x00')") - self.assertEqual(f(bytearray([0, 1, 254, 255])), - "bytearray(b'\\x00\\x01\\xfe\\xff')") - self.assertEqual(f(b"abc"), "b'abc'") - self.assertEqual(f(b"'"), '''b"'"''') # ''' - self.assertEqual(f(b"'\""), r"""b'\'"'""") # ' + def test_bytes_str(self): + self.test_bytes_repr(str) + + @check_bytes_warnings + def test_bytearray_str(self): + self.test_bytearray_repr(str) @check_bytes_warnings def test_format(self): @@ -1831,15 +1988,6 @@ def test_from_bytearray(self): b = bytearray(buf) self.assertEqual(b, bytearray(sample)) - @check_bytes_warnings - def test_to_str(self): - self.assertEqual(str(b''), "b''") - self.assertEqual(str(b'x'), "b'x'") - self.assertEqual(str(b'\x80'), "b'\\x80'") - self.assertEqual(str(bytearray(b'')), "bytearray(b'')") - self.assertEqual(str(bytearray(b'x')), "bytearray(b'x')") - self.assertEqual(str(bytearray(b'\x80')), "bytearray(b'\\x80')") - def test_literal(self): tests = [ (b"Wonderful spam", "Wonderful spam"), @@ -1990,7 +2138,7 @@ def test_join(self): s3 = s1.join([b"abcd"]) self.assertIs(type(s3), self.basetype) - @unittest.skip("TODO: RUSTPYTHON, Fails on ByteArraySubclassWithSlotsTest") + @unittest.skip('TODO: RUSTPYTHON; Fails on ByteArraySubclassWithSlotsTest') def test_pickle(self): a = self.type2test(b"abcd") a.x = 10 @@ -2005,7 +2153,7 @@ def test_pickle(self): self.assertEqual(type(a.z), type(b.z)) self.assertFalse(hasattr(b, 'y')) - @unittest.skip("TODO: RUSTPYTHON, Fails on ByteArraySubclassWithSlotsTest") + @unittest.skip('TODO: RUSTPYTHON; Fails on ByteArraySubclassWithSlotsTest') def test_copy(self): a = self.type2test(b"abcd") a.x = 10 @@ -2049,7 +2197,10 @@ def __init__(me, *args, **kwargs): class ByteArraySubclass(bytearray): - pass + class Nested(bytearray): + pass + class Ŭñıçöđë(bytearray): + pass class ByteArraySubclassWithSlots(bytearray): __slots__ = ('x', 'y', '__dict__') @@ -2060,6 +2211,12 @@ class BytesSubclass(bytes): class OtherBytesSubclass(bytes): pass +class WithBytes: + def __init__(self, value): + self.value = value + def __bytes__(self): + return self.value + class ByteArraySubclassTest(SubclassTest, unittest.TestCase): basetype = bytearray type2test = ByteArraySubclass diff --git a/Lib/test/test_bz2.py b/Lib/test/test_bz2.py index dfc444cbbd..3617eba8e8 100644 --- a/Lib/test/test_bz2.py +++ b/Lib/test/test_bz2.py @@ -3,19 +3,19 @@ import array import unittest +import io from io import BytesIO, DEFAULT_BUFFER_SIZE import os import pickle import glob import tempfile -import pathlib import random import shutil import subprocess import threading from test.support import import_helper from test.support import threading_helper -from test.support.os_helper import unlink +from test.support.os_helper import unlink, FakePath import _compression import sys @@ -476,7 +476,6 @@ def testReadlinesNoNewline(self): self.assertEqual(xlines, [b'Test']) def testContextProtocol(self): - f = None with BZ2File(self.filename, "wb") as f: f.write(b"xxx") f = BZ2File(self.filename, "rb") @@ -537,26 +536,210 @@ def testMultiStreamOrdering(self): with BZ2File(self.filename) as bz2f: self.assertEqual(bz2f.read(), data1 + data2) + def testOpenFilename(self): + with BZ2File(self.filename, "wb") as f: + f.write(b'content') + self.assertEqual(f.name, self.filename) + self.assertIsInstance(f.fileno(), int) + self.assertEqual(f.mode, 'wb') + self.assertIs(f.readable(), False) + self.assertIs(f.writable(), True) + self.assertIs(f.seekable(), False) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + with self.assertRaises(ValueError): + f.name + self.assertRaises(ValueError, f.fileno) + self.assertEqual(f.mode, 'wb') + self.assertRaises(ValueError, f.readable) + self.assertRaises(ValueError, f.writable) + self.assertRaises(ValueError, f.seekable) + + with BZ2File(self.filename, "ab") as f: + f.write(b'appendix') + self.assertEqual(f.name, self.filename) + self.assertIsInstance(f.fileno(), int) + self.assertEqual(f.mode, 'wb') + self.assertIs(f.readable(), False) + self.assertIs(f.writable(), True) + self.assertIs(f.seekable(), False) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + with self.assertRaises(ValueError): + f.name + self.assertRaises(ValueError, f.fileno) + self.assertEqual(f.mode, 'wb') + self.assertRaises(ValueError, f.readable) + self.assertRaises(ValueError, f.writable) + self.assertRaises(ValueError, f.seekable) + + with BZ2File(self.filename, 'rb') as f: + self.assertEqual(f.read(), b'contentappendix') + self.assertEqual(f.name, self.filename) + self.assertIsInstance(f.fileno(), int) + self.assertEqual(f.mode, 'rb') + self.assertIs(f.readable(), True) + self.assertIs(f.writable(), False) + self.assertIs(f.seekable(), True) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + with self.assertRaises(ValueError): + f.name + self.assertRaises(ValueError, f.fileno) + self.assertEqual(f.mode, 'rb') + self.assertRaises(ValueError, f.readable) + self.assertRaises(ValueError, f.writable) + self.assertRaises(ValueError, f.seekable) + + def testOpenFileWithName(self): + with open(self.filename, 'wb') as raw: + with BZ2File(raw, 'wb') as f: + f.write(b'content') + self.assertEqual(f.name, raw.name) + self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.mode, 'wb') + self.assertIs(f.readable(), False) + self.assertIs(f.writable(), True) + self.assertIs(f.seekable(), False) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + with self.assertRaises(ValueError): + f.name + self.assertRaises(ValueError, f.fileno) + self.assertEqual(f.mode, 'wb') + self.assertRaises(ValueError, f.readable) + self.assertRaises(ValueError, f.writable) + self.assertRaises(ValueError, f.seekable) + + with open(self.filename, 'ab') as raw: + with BZ2File(raw, 'ab') as f: + f.write(b'appendix') + self.assertEqual(f.name, raw.name) + self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.mode, 'wb') + self.assertIs(f.readable(), False) + self.assertIs(f.writable(), True) + self.assertIs(f.seekable(), False) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + with self.assertRaises(ValueError): + f.name + self.assertRaises(ValueError, f.fileno) + self.assertEqual(f.mode, 'wb') + self.assertRaises(ValueError, f.readable) + self.assertRaises(ValueError, f.writable) + self.assertRaises(ValueError, f.seekable) + + with open(self.filename, 'rb') as raw: + with BZ2File(raw, 'rb') as f: + self.assertEqual(f.read(), b'contentappendix') + self.assertEqual(f.name, raw.name) + self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.mode, 'rb') + self.assertIs(f.readable(), True) + self.assertIs(f.writable(), False) + self.assertIs(f.seekable(), True) + self.assertIs(f.closed, False) + self.assertIs(f.closed, True) + with self.assertRaises(ValueError): + f.name + self.assertRaises(ValueError, f.fileno) + self.assertEqual(f.mode, 'rb') + self.assertRaises(ValueError, f.readable) + self.assertRaises(ValueError, f.writable) + self.assertRaises(ValueError, f.seekable) + + def testOpenFileWithoutName(self): + bio = BytesIO() + with BZ2File(bio, 'wb') as f: + f.write(b'content') + with self.assertRaises(AttributeError): + f.name + self.assertRaises(io.UnsupportedOperation, f.fileno) + self.assertEqual(f.mode, 'wb') + with self.assertRaises(ValueError): + f.name + self.assertRaises(ValueError, f.fileno) + + with BZ2File(bio, 'ab') as f: + f.write(b'appendix') + with self.assertRaises(AttributeError): + f.name + self.assertRaises(io.UnsupportedOperation, f.fileno) + self.assertEqual(f.mode, 'wb') + with self.assertRaises(ValueError): + f.name + self.assertRaises(ValueError, f.fileno) + + bio.seek(0) + with BZ2File(bio, 'rb') as f: + self.assertEqual(f.read(), b'contentappendix') + with self.assertRaises(AttributeError): + f.name + self.assertRaises(io.UnsupportedOperation, f.fileno) + self.assertEqual(f.mode, 'rb') + with self.assertRaises(ValueError): + f.name + self.assertRaises(ValueError, f.fileno) + + def testOpenFileWithIntName(self): + fd = os.open(self.filename, os.O_WRONLY | os.O_CREAT | os.O_TRUNC) + with open(fd, 'wb') as raw: + with BZ2File(raw, 'wb') as f: + f.write(b'content') + self.assertEqual(f.name, raw.name) + self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.mode, 'wb') + with self.assertRaises(ValueError): + f.name + self.assertRaises(ValueError, f.fileno) + + fd = os.open(self.filename, os.O_WRONLY | os.O_CREAT | os.O_APPEND) + with open(fd, 'ab') as raw: + with BZ2File(raw, 'ab') as f: + f.write(b'appendix') + self.assertEqual(f.name, raw.name) + self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.mode, 'wb') + with self.assertRaises(ValueError): + f.name + self.assertRaises(ValueError, f.fileno) + + fd = os.open(self.filename, os.O_RDONLY) + with open(fd, 'rb') as raw: + with BZ2File(raw, 'rb') as f: + self.assertEqual(f.read(), b'contentappendix') + self.assertEqual(f.name, raw.name) + self.assertEqual(f.fileno(), raw.fileno()) + self.assertEqual(f.mode, 'rb') + with self.assertRaises(ValueError): + f.name + self.assertRaises(ValueError, f.fileno) + def testOpenBytesFilename(self): str_filename = self.filename - try: - bytes_filename = str_filename.encode("ascii") - except UnicodeEncodeError: - self.skipTest("Temporary file name needs to be ASCII") + bytes_filename = os.fsencode(str_filename) with BZ2File(bytes_filename, "wb") as f: f.write(self.DATA) + self.assertEqual(f.name, bytes_filename) with BZ2File(bytes_filename, "rb") as f: self.assertEqual(f.read(), self.DATA) + self.assertEqual(f.name, bytes_filename) # Sanity check that we are actually operating on the right file. with BZ2File(str_filename, "rb") as f: self.assertEqual(f.read(), self.DATA) + self.assertEqual(f.name, str_filename) + # TODO: RUSTPYTHON + @unittest.expectedFailure def testOpenPathLikeFilename(self): - filename = pathlib.Path(self.filename) + filename = FakePath(self.filename) with BZ2File(filename, "wb") as f: f.write(self.DATA) + self.assertEqual(f.name, self.filename) with BZ2File(filename, "rb") as f: self.assertEqual(f.read(), self.DATA) + self.assertEqual(f.name, self.filename) def testDecompressLimited(self): """Decompressed data buffering should be limited""" @@ -577,6 +760,9 @@ def testReadBytesIO(self): with BZ2File(bio) as bz2f: self.assertRaises(TypeError, bz2f.read, float()) self.assertEqual(bz2f.read(), self.TEXT) + with self.assertRaises(AttributeError): + bz2.name + self.assertEqual(bz2f.mode, 'rb') self.assertFalse(bio.closed) def testPeekBytesIO(self): @@ -592,6 +778,9 @@ def testWriteBytesIO(self): with BZ2File(bio, "w") as bz2f: self.assertRaises(TypeError, bz2f.write) bz2f.write(self.TEXT) + with self.assertRaises(AttributeError): + bz2.name + self.assertEqual(bz2f.mode, 'wb') self.assertEqual(ext_decompress(bio.getvalue()), self.TEXT) self.assertFalse(bio.closed) diff --git a/Lib/test/test_call.py b/Lib/test/test_call.py index 3cb9659acb..86ba0aa4b6 100644 --- a/Lib/test/test_call.py +++ b/Lib/test/test_call.py @@ -1,14 +1,29 @@ -import datetime import unittest -from test.support import cpython_only +from test.support import (cpython_only, is_wasi, requires_limited_api, Py_DEBUG, + set_recursion_limit, skip_on_s390x, import_helper) try: import _testcapi except ImportError: _testcapi = None +try: + import _testlimitedcapi +except ImportError: + _testlimitedcapi = None import struct import collections import itertools import gc +import contextlib +import sys +import types + + +class BadStr(str): + def __eq__(self, other): + return True + def __hash__(self): + # Guaranteed different hash + return str.__hash__(self) ^ 3 class FunctionCalls(unittest.TestCase): @@ -26,124 +41,22 @@ def fn(**kw): self.assertIsInstance(res, dict) self.assertEqual(list(res.items()), expected) - -# The test cases here cover several paths through the function calling -# code. They depend on the METH_XXX flag that is used to define a C -# function, which can't be verified from Python. If the METH_XXX decl -# for a C function changes, these tests may not cover the right paths. - -class CFunctionCalls(unittest.TestCase): - - def test_varargs0(self): - self.assertRaises(TypeError, {}.__contains__) - - def test_varargs1(self): - {}.__contains__(0) - - def test_varargs2(self): - self.assertRaises(TypeError, {}.__contains__, 0, 1) - - def test_varargs0_ext(self): - try: - {}.__contains__(*()) - except TypeError: - pass - - def test_varargs1_ext(self): - {}.__contains__(*(0,)) - - def test_varargs2_ext(self): - try: - {}.__contains__(*(1, 2)) - except TypeError: - pass - else: - raise RuntimeError - - def test_varargs1_kw(self): - self.assertRaises(TypeError, {}.__contains__, x=2) - - def test_varargs2_kw(self): - self.assertRaises(TypeError, {}.__contains__, x=2, y=2) - - def test_oldargs0_0(self): - {}.keys() - - def test_oldargs0_1(self): - self.assertRaises(TypeError, {}.keys, 0) - - def test_oldargs0_2(self): - self.assertRaises(TypeError, {}.keys, 0, 1) - - def test_oldargs0_0_ext(self): - {}.keys(*()) - - def test_oldargs0_1_ext(self): - try: - {}.keys(*(0,)) - except TypeError: + def test_frames_are_popped_after_failed_calls(self): + # GH-93252: stuff blows up if we don't pop the new frame after + # recovering from failed calls: + def f(): pass - else: - raise RuntimeError - - def test_oldargs0_2_ext(self): - try: - {}.keys(*(1, 2)) - except TypeError: - pass - else: - raise RuntimeError - - def test_oldargs0_0_kw(self): - try: - {}.keys(x=2) - except TypeError: - pass - else: - raise RuntimeError - - def test_oldargs0_1_kw(self): - self.assertRaises(TypeError, {}.keys, x=2) - - def test_oldargs0_2_kw(self): - self.assertRaises(TypeError, {}.keys, x=2, y=2) - - def test_oldargs1_0(self): - self.assertRaises(TypeError, [].count) - - def test_oldargs1_1(self): - [].count(1) - - def test_oldargs1_2(self): - self.assertRaises(TypeError, [].count, 1, 2) - - def test_oldargs1_0_ext(self): - try: - [].count(*()) - except TypeError: - pass - else: - raise RuntimeError - - def test_oldargs1_1_ext(self): - [].count(*(1,)) - - def test_oldargs1_2_ext(self): - try: - [].count(*(1, 2)) - except TypeError: - pass - else: - raise RuntimeError - - def test_oldargs1_0_kw(self): - self.assertRaises(TypeError, [].count, x=2) - - def test_oldargs1_1_kw(self): - self.assertRaises(TypeError, [].count, {}, x=2) - - def test_oldargs1_2_kw(self): - self.assertRaises(TypeError, [].count, x=2, y=2) + class C: + def m(self): + pass + callables = [f, C.m, [].__len__] + for c in callables: + for _ in range(1000): + try: + c(None) + except TypeError: + pass + # BOOM! @cpython_only @@ -158,11 +71,12 @@ def test_varargs2(self): self.assertRaisesRegex(TypeError, msg, {}.__contains__, 0, 1) def test_varargs3(self): - msg = r"^from_bytes\(\) takes exactly 2 positional arguments \(3 given\)" + msg = r"^from_bytes\(\) takes at most 2 positional arguments \(3 given\)" self.assertRaisesRegex(TypeError, msg, int.from_bytes, b'a', 'little', False) def test_varargs1min(self): - msg = r"get expected at least 1 argument, got 0" + msg = (r"get\(\) takes at least 1 argument \(0 given\)|" + r"get expected at least 1 argument, got 0") self.assertRaisesRegex(TypeError, msg, {}.get) msg = r"expected 1 argument, got 0" @@ -173,11 +87,13 @@ def test_varargs2min(self): self.assertRaisesRegex(TypeError, msg, getattr) def test_varargs1max(self): - msg = r"input expected at most 1 argument, got 2" + msg = (r"input\(\) takes at most 1 argument \(2 given\)|" + r"input expected at most 1 argument, got 2") self.assertRaisesRegex(TypeError, msg, input, 1, 2) def test_varargs2max(self): - msg = r"get expected at most 2 arguments, got 3" + msg = (r"get\(\) takes at most 2 arguments \(3 given\)|" + r"get expected at most 2 arguments, got 3") self.assertRaisesRegex(TypeError, msg, {}.get, 1, 2, 3) def test_varargs1_kw(self): @@ -193,7 +109,7 @@ def test_varargs3_kw(self): self.assertRaisesRegex(TypeError, msg, bool, x=2) def test_varargs4_kw(self): - msg = r"^index\(\) takes no keyword arguments$" + msg = r"^(list[.])?index\(\) takes no keyword arguments$" self.assertRaisesRegex(TypeError, msg, [].index, x=2) def test_varargs5_kw(self): @@ -209,19 +125,19 @@ def test_varargs7_kw(self): self.assertRaisesRegex(TypeError, msg, next, x=2) def test_varargs8_kw(self): - msg = r"^pack\(\) takes no keyword arguments$" + msg = r"^_struct[.]pack\(\) takes no keyword arguments$" self.assertRaisesRegex(TypeError, msg, struct.pack, x=2) def test_varargs9_kw(self): - msg = r"^pack_into\(\) takes no keyword arguments$" + msg = r"^_struct[.]pack_into\(\) takes no keyword arguments$" self.assertRaisesRegex(TypeError, msg, struct.pack_into, x=2) def test_varargs10_kw(self): - msg = r"^index\(\) takes no keyword arguments$" + msg = r"^deque[.]index\(\) takes no keyword arguments$" self.assertRaisesRegex(TypeError, msg, collections.deque().index, x=2) def test_varargs11_kw(self): - msg = r"^pack\(\) takes no keyword arguments$" + msg = r"^Struct[.]pack\(\) takes no keyword arguments$" self.assertRaisesRegex(TypeError, msg, struct.Struct.pack, struct.Struct(""), x=2) def test_varargs12_kw(self): @@ -238,9 +154,9 @@ def test_varargs14_kw(self): itertools.product, 0, repeat=1, foo=2) def test_varargs15_kw(self): - msg = r"^ImportError\(\) takes at most 2 keyword arguments \(3 given\)$" + msg = r"^ImportError\(\) takes at most 3 keyword arguments \(4 given\)$" self.assertRaisesRegex(TypeError, msg, - ImportError, 0, name=1, path=2, foo=3) + ImportError, 0, name=1, path=2, name_from=3, foo=3) def test_varargs16_kw(self): msg = r"^min\(\) takes at most 2 keyword arguments \(3 given\)$" @@ -248,10 +164,22 @@ def test_varargs16_kw(self): min, 0, default=1, key=2, foo=3) def test_varargs17_kw(self): - msg = r"^print\(\) takes at most 4 keyword arguments \(5 given\)$" + msg = r"print\(\) got an unexpected keyword argument 'foo'$" self.assertRaisesRegex(TypeError, msg, print, 0, sep=1, end=2, file=3, flush=4, foo=5) + def test_varargs18_kw(self): + # _PyArg_UnpackKeywordsWithVararg() + msg = r"invalid keyword argument for print\(\)$" + with self.assertRaisesRegex(TypeError, msg): + print(0, 1, **{BadStr('foo'): ','}) + + def test_varargs19_kw(self): + # _PyArg_UnpackKeywords() + msg = r"invalid keyword argument for round\(\)$" + with self.assertRaisesRegex(TypeError, msg): + round(1.75, **{BadStr('foo'): 1}) + def test_oldargs0_1(self): msg = r"keys\(\) takes no arguments \(1 given\)" self.assertRaisesRegex(TypeError, msg, {}.keys, 0) @@ -288,6 +216,208 @@ def test_oldargs1_2_kw(self): msg = r"count\(\) takes no keyword arguments" self.assertRaisesRegex(TypeError, msg, [].count, x=2, y=2) + def test_object_not_callable(self): + msg = r"^'object' object is not callable$" + self.assertRaisesRegex(TypeError, msg, object()) + + def test_module_not_callable_no_suggestion_0(self): + msg = r"^'module' object is not callable$" + self.assertRaisesRegex(TypeError, msg, types.ModuleType("mod")) + + def test_module_not_callable_no_suggestion_1(self): + msg = r"^'module' object is not callable$" + mod = types.ModuleType("mod") + mod.mod = 42 + self.assertRaisesRegex(TypeError, msg, mod) + + def test_module_not_callable_no_suggestion_2(self): + msg = r"^'module' object is not callable$" + mod = types.ModuleType("mod") + del mod.__name__ + self.assertRaisesRegex(TypeError, msg, mod) + + def test_module_not_callable_no_suggestion_3(self): + msg = r"^'module' object is not callable$" + mod = types.ModuleType("mod") + mod.__name__ = 42 + self.assertRaisesRegex(TypeError, msg, mod) + + def test_module_not_callable_suggestion(self): + msg = r"^'module' object is not callable\. Did you mean: 'mod\.mod\(\.\.\.\)'\?$" + mod = types.ModuleType("mod") + mod.mod = lambda: ... + self.assertRaisesRegex(TypeError, msg, mod) + + +@unittest.skipIf(_testcapi is None, "requires _testcapi") +class TestCallingConventions(unittest.TestCase): + """Test calling using various C calling conventions (METH_*) from Python + + Subclasses test several kinds of functions (module-level, methods, + class methods static methods) using these attributes: + obj: the object that contains tested functions (as attributes) + expected_self: expected "self" argument to the C function + + The base class tests module-level functions. + """ + + def setUp(self): + self.obj = self.expected_self = _testcapi + + def test_varargs(self): + self.assertEqual( + self.obj.meth_varargs(1, 2, 3), + (self.expected_self, (1, 2, 3)), + ) + + def test_varargs_ext(self): + self.assertEqual( + self.obj.meth_varargs(*(1, 2, 3)), + (self.expected_self, (1, 2, 3)), + ) + + def test_varargs_error_kw(self): + msg = r"meth_varargs\(\) takes no keyword arguments" + self.assertRaisesRegex( + TypeError, msg, lambda: self.obj.meth_varargs(k=1), + ) + + def test_varargs_keywords(self): + self.assertEqual( + self.obj.meth_varargs_keywords(1, 2, a=3, b=4), + (self.expected_self, (1, 2), {'a': 3, 'b': 4}) + ) + + def test_varargs_keywords_ext(self): + self.assertEqual( + self.obj.meth_varargs_keywords(*[1, 2], **{'a': 3, 'b': 4}), + (self.expected_self, (1, 2), {'a': 3, 'b': 4}) + ) + + def test_o(self): + self.assertEqual(self.obj.meth_o(1), (self.expected_self, 1)) + + def test_o_ext(self): + self.assertEqual(self.obj.meth_o(*[1]), (self.expected_self, 1)) + + def test_o_error_no_arg(self): + msg = r"meth_o\(\) takes exactly one argument \(0 given\)" + self.assertRaisesRegex(TypeError, msg, self.obj.meth_o) + + def test_o_error_two_args(self): + msg = r"meth_o\(\) takes exactly one argument \(2 given\)" + self.assertRaisesRegex( + TypeError, msg, lambda: self.obj.meth_o(1, 2), + ) + + def test_o_error_ext(self): + msg = r"meth_o\(\) takes exactly one argument \(3 given\)" + self.assertRaisesRegex( + TypeError, msg, lambda: self.obj.meth_o(*(1, 2, 3)), + ) + + def test_o_error_kw(self): + msg = r"meth_o\(\) takes no keyword arguments" + self.assertRaisesRegex( + TypeError, msg, lambda: self.obj.meth_o(k=1), + ) + + def test_o_error_arg_kw(self): + msg = r"meth_o\(\) takes no keyword arguments" + self.assertRaisesRegex( + TypeError, msg, lambda: self.obj.meth_o(k=1), + ) + + def test_noargs(self): + self.assertEqual(self.obj.meth_noargs(), self.expected_self) + + def test_noargs_ext(self): + self.assertEqual(self.obj.meth_noargs(*[]), self.expected_self) + + def test_noargs_error_arg(self): + msg = r"meth_noargs\(\) takes no arguments \(1 given\)" + self.assertRaisesRegex( + TypeError, msg, lambda: self.obj.meth_noargs(1), + ) + + def test_noargs_error_arg2(self): + msg = r"meth_noargs\(\) takes no arguments \(2 given\)" + self.assertRaisesRegex( + TypeError, msg, lambda: self.obj.meth_noargs(1, 2), + ) + + def test_noargs_error_ext(self): + msg = r"meth_noargs\(\) takes no arguments \(3 given\)" + self.assertRaisesRegex( + TypeError, msg, lambda: self.obj.meth_noargs(*(1, 2, 3)), + ) + + def test_noargs_error_kw(self): + msg = r"meth_noargs\(\) takes no keyword arguments" + self.assertRaisesRegex( + TypeError, msg, lambda: self.obj.meth_noargs(k=1), + ) + + def test_fastcall(self): + self.assertEqual( + self.obj.meth_fastcall(1, 2, 3), + (self.expected_self, (1, 2, 3)), + ) + + def test_fastcall_ext(self): + self.assertEqual( + self.obj.meth_fastcall(*(1, 2, 3)), + (self.expected_self, (1, 2, 3)), + ) + + def test_fastcall_error_kw(self): + msg = r"meth_fastcall\(\) takes no keyword arguments" + self.assertRaisesRegex( + TypeError, msg, lambda: self.obj.meth_fastcall(k=1), + ) + + def test_fastcall_keywords(self): + self.assertEqual( + self.obj.meth_fastcall_keywords(1, 2, a=3, b=4), + (self.expected_self, (1, 2), {'a': 3, 'b': 4}) + ) + + def test_fastcall_keywords_ext(self): + self.assertEqual( + self.obj.meth_fastcall_keywords(*(1, 2), **{'a': 3, 'b': 4}), + (self.expected_self, (1, 2), {'a': 3, 'b': 4}) + ) + + +class TestCallingConventionsInstance(TestCallingConventions): + """Test calling instance methods using various calling conventions""" + + def setUp(self): + self.obj = self.expected_self = _testcapi.MethInstance() + + +class TestCallingConventionsClass(TestCallingConventions): + """Test calling class methods using various calling conventions""" + + def setUp(self): + self.obj = self.expected_self = _testcapi.MethClass + + +class TestCallingConventionsClassInstance(TestCallingConventions): + """Test calling class methods on instance""" + + def setUp(self): + self.obj = _testcapi.MethClass() + self.expected_self = _testcapi.MethClass + + +class TestCallingConventionsStatic(TestCallingConventions): + """Test calling static methods using various calling conventions""" + + def setUp(self): + self.obj = _testcapi.MethStatic() + self.expected_self = None + def pyfunc(arg1, arg2): return [arg1, arg2] @@ -315,14 +445,15 @@ def static_method(): PYTHON_INSTANCE = PythonClass() - -IGNORE_RESULT = object() +NULL_OR_EMPTY = object() -@cpython_only class FastCallTests(unittest.TestCase): + """Test calling using various callables from C + """ + # Test calls with positional arguments - CALLS_POSARGS = ( + CALLS_POSARGS = [ # (func, args: tuple, result) # Python function with 2 arguments @@ -341,31 +472,11 @@ class FastCallTests(unittest.TestCase): (PYTHON_INSTANCE.class_method, (), "classmethod"), (PYTHON_INSTANCE.static_method, (), "staticmethod"), - # C function: METH_NOARGS - (globals, (), IGNORE_RESULT), - - # C function: METH_O - (id, ("hello",), IGNORE_RESULT), - - # C function: METH_VARARGS - (dir, (1,), IGNORE_RESULT), - - # C function: METH_VARARGS | METH_KEYWORDS - (min, (5, 9), 5), - - # C function: METH_FASTCALL - (divmod, (1000, 33), (30, 10)), - - # C type static method: METH_FASTCALL | METH_CLASS - (int.from_bytes, (b'\x01\x00', 'little'), 1), - - # bpo-30524: Test that calling a C type static method with no argument - # doesn't crash (ignore the result): METH_FASTCALL | METH_CLASS - (datetime.datetime.now, (), IGNORE_RESULT), - ) + # C callables are added later + ] # Test calls with positional and keyword arguments - CALLS_KWARGS = ( + CALLS_KWARGS = [ # (func, args: tuple, kwargs: dict, result) # Python function with 2 arguments @@ -376,34 +487,57 @@ class FastCallTests(unittest.TestCase): (PYTHON_INSTANCE.method, (1,), {'arg2': 2}, [1, 2]), (PYTHON_INSTANCE.method, (), {'arg1': 1, 'arg2': 2}, [1, 2]), - # C function: METH_VARARGS | METH_KEYWORDS - (max, ([],), {'default': 9}, 9), - - # C type static method: METH_FASTCALL | METH_CLASS - (int.from_bytes, (b'\x01\x00',), {'byteorder': 'little'}, 1), - (int.from_bytes, (), {'bytes': b'\x01\x00', 'byteorder': 'little'}, 1), - ) + # C callables are added later + ] + + # Add all the calling conventions and variants of C callables + if _testcapi: + _instance = _testcapi.MethInstance() + for obj, expected_self in ( + (_testcapi, _testcapi), # module-level function + (_instance, _instance), # bound method + (_testcapi.MethClass, _testcapi.MethClass), # class method on class + (_testcapi.MethClass(), _testcapi.MethClass), # class method on inst. + (_testcapi.MethStatic, None), # static method + ): + CALLS_POSARGS.extend([ + (obj.meth_varargs, (1, 2), (expected_self, (1, 2))), + (obj.meth_varargs_keywords, + (1, 2), (expected_self, (1, 2), NULL_OR_EMPTY)), + (obj.meth_fastcall, (1, 2), (expected_self, (1, 2))), + (obj.meth_fastcall, (), (expected_self, ())), + (obj.meth_fastcall_keywords, + (1, 2), (expected_self, (1, 2), NULL_OR_EMPTY)), + (obj.meth_fastcall_keywords, + (), (expected_self, (), NULL_OR_EMPTY)), + (obj.meth_noargs, (), expected_self), + (obj.meth_o, (123, ), (expected_self, 123)), + ]) + + CALLS_KWARGS.extend([ + (obj.meth_varargs_keywords, + (1, 2), {'x': 'y'}, (expected_self, (1, 2), {'x': 'y'})), + (obj.meth_varargs_keywords, + (), {'x': 'y'}, (expected_self, (), {'x': 'y'})), + (obj.meth_varargs_keywords, + (1, 2), {}, (expected_self, (1, 2), NULL_OR_EMPTY)), + (obj.meth_fastcall_keywords, + (1, 2), {'x': 'y'}, (expected_self, (1, 2), {'x': 'y'})), + (obj.meth_fastcall_keywords, + (), {'x': 'y'}, (expected_self, (), {'x': 'y'})), + (obj.meth_fastcall_keywords, + (1, 2), {}, (expected_self, (1, 2), NULL_OR_EMPTY)), + ]) def check_result(self, result, expected): - if expected is IGNORE_RESULT: - return + if isinstance(expected, tuple) and expected[-1] is NULL_OR_EMPTY: + if result[-1] in ({}, None): + expected = (*expected[:-1], result[-1]) self.assertEqual(result, expected) - def test_fastcall(self): - # Test _PyObject_FastCall() - - for func, args, expected in self.CALLS_POSARGS: - with self.subTest(func=func, args=args): - result = _testcapi.pyobject_fastcall(func, args) - self.check_result(result, expected) - - if not args: - # args=NULL, nargs=0 - result = _testcapi.pyobject_fastcall(func, None) - self.check_result(result, expected) - + @unittest.skipIf(_testcapi is None, "requires _testcapi") def test_vectorcall_dict(self): - # Test _PyObject_FastCallDict() + # Test PyObject_VectorcallDict() for func, args, expected in self.CALLS_POSARGS: with self.subTest(func=func, args=args): @@ -411,26 +545,19 @@ def test_vectorcall_dict(self): result = _testcapi.pyobject_fastcalldict(func, args, None) self.check_result(result, expected) - # kwargs={} - result = _testcapi.pyobject_fastcalldict(func, args, {}) - self.check_result(result, expected) - if not args: # args=NULL, nargs=0, kwargs=NULL result = _testcapi.pyobject_fastcalldict(func, None, None) self.check_result(result, expected) - # args=NULL, nargs=0, kwargs={} - result = _testcapi.pyobject_fastcalldict(func, None, {}) - self.check_result(result, expected) - for func, args, kwargs, expected in self.CALLS_KWARGS: with self.subTest(func=func, args=args, kwargs=kwargs): result = _testcapi.pyobject_fastcalldict(func, args, kwargs) self.check_result(result, expected) + @unittest.skipIf(_testcapi is None, "requires _testcapi") def test_vectorcall(self): - # Test _PyObject_Vectorcall() + # Test PyObject_Vectorcall() for func, args, expected in self.CALLS_POSARGS: with self.subTest(func=func, args=args): @@ -458,6 +585,7 @@ def test_vectorcall(self): result = _testcapi.pyobject_vectorcall(func, args, kwnames) self.check_result(result, expected) + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: Expected type 'int' but 'IntWithDict' found. def test_fastcall_clearing_dict(self): # Test bpo-36907: the point of the test is just checking that this # does not crash. @@ -469,7 +597,7 @@ def __index__(self): self.kwargs.clear() gc.collect() return 0 - x = IntWithDict(dont_inherit=IntWithDict()) + x = IntWithDict(optimize=IntWithDict()) # We test the argument handling of "compile" here, the compilation # itself is not relevant. When we pass flags=x below, x.__index__() is # called, which changes the keywords dict. @@ -490,10 +618,12 @@ def testfunction_kw(self, *, kw): return self +ADAPTIVE_WARMUP_DELAY = 2 + + +@unittest.skipIf(_testcapi is None, "requires _testcapi") class TestPEP590(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_method_descriptor_flag(self): import functools cached = functools.lru_cache(1)(testfunction) @@ -508,26 +638,32 @@ def test_method_descriptor_flag(self): self.assertTrue(_testcapi.MethodDescriptorDerived.__flags__ & Py_TPFLAGS_METHOD_DESCRIPTOR) self.assertFalse(_testcapi.MethodDescriptorNopGet.__flags__ & Py_TPFLAGS_METHOD_DESCRIPTOR) - # Heap type should not inherit Py_TPFLAGS_METHOD_DESCRIPTOR + # Mutable heap types should not inherit Py_TPFLAGS_METHOD_DESCRIPTOR class MethodDescriptorHeap(_testcapi.MethodDescriptorBase): pass self.assertFalse(MethodDescriptorHeap.__flags__ & Py_TPFLAGS_METHOD_DESCRIPTOR) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_vectorcall_flag(self): self.assertTrue(_testcapi.MethodDescriptorBase.__flags__ & Py_TPFLAGS_HAVE_VECTORCALL) self.assertTrue(_testcapi.MethodDescriptorDerived.__flags__ & Py_TPFLAGS_HAVE_VECTORCALL) self.assertFalse(_testcapi.MethodDescriptorNopGet.__flags__ & Py_TPFLAGS_HAVE_VECTORCALL) self.assertTrue(_testcapi.MethodDescriptor2.__flags__ & Py_TPFLAGS_HAVE_VECTORCALL) - # Heap type should not inherit Py_TPFLAGS_HAVE_VECTORCALL + # Mutable heap types should inherit Py_TPFLAGS_HAVE_VECTORCALL, + # but should lose it when __call__ is overridden class MethodDescriptorHeap(_testcapi.MethodDescriptorBase): pass + self.assertTrue(MethodDescriptorHeap.__flags__ & Py_TPFLAGS_HAVE_VECTORCALL) + MethodDescriptorHeap.__call__ = print + self.assertFalse(MethodDescriptorHeap.__flags__ & Py_TPFLAGS_HAVE_VECTORCALL) + + # Mutable heap types should not inherit Py_TPFLAGS_HAVE_VECTORCALL if + # they define __call__ directly + class MethodDescriptorHeap(_testcapi.MethodDescriptorBase): + def __call__(self): + pass self.assertFalse(MethodDescriptorHeap.__flags__ & Py_TPFLAGS_HAVE_VECTORCALL) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_vectorcall_override(self): # Check that tp_call can correctly override vectorcall. # MethodDescriptorNopGet implements tp_call but it inherits from @@ -538,14 +674,64 @@ def test_vectorcall_override(self): f = _testcapi.MethodDescriptorNopGet() self.assertIs(f(*args), args) - # TODO: RUSTPYTHON - @unittest.expectedFailure + def test_vectorcall_override_on_mutable_class(self): + """Setting __call__ should disable vectorcall""" + TestType = _testcapi.make_vectorcall_class() + instance = TestType() + self.assertEqual(instance(), "tp_call") + instance.set_vectorcall(TestType) + self.assertEqual(instance(), "vectorcall") # assume vectorcall is used + TestType.__call__ = lambda self: "custom" + self.assertEqual(instance(), "custom") + + def test_vectorcall_override_with_subclass(self): + """Setting __call__ on a superclass should disable vectorcall""" + SuperType = _testcapi.make_vectorcall_class() + class DerivedType(SuperType): + pass + + instance = DerivedType() + + # Derived types with its own vectorcall should be unaffected + UnaffectedType1 = _testcapi.make_vectorcall_class(DerivedType) + UnaffectedType2 = _testcapi.make_vectorcall_class(SuperType) + + # Aside: Quickly check that the C helper actually made derived types + self.assertTrue(issubclass(UnaffectedType1, DerivedType)) + self.assertTrue(issubclass(UnaffectedType2, SuperType)) + + # Initial state: tp_call + self.assertEqual(instance(), "tp_call") + self.assertEqual(_testcapi.has_vectorcall_flag(SuperType), True) + self.assertEqual(_testcapi.has_vectorcall_flag(DerivedType), True) + self.assertEqual(_testcapi.has_vectorcall_flag(UnaffectedType1), True) + self.assertEqual(_testcapi.has_vectorcall_flag(UnaffectedType2), True) + + # Setting the vectorcall function + instance.set_vectorcall(SuperType) + + self.assertEqual(instance(), "vectorcall") + self.assertEqual(_testcapi.has_vectorcall_flag(SuperType), True) + self.assertEqual(_testcapi.has_vectorcall_flag(DerivedType), True) + self.assertEqual(_testcapi.has_vectorcall_flag(UnaffectedType1), True) + self.assertEqual(_testcapi.has_vectorcall_flag(UnaffectedType2), True) + + # Setting __call__ should remove vectorcall from all subclasses + SuperType.__call__ = lambda self: "custom" + + self.assertEqual(instance(), "custom") + self.assertEqual(_testcapi.has_vectorcall_flag(SuperType), False) + self.assertEqual(_testcapi.has_vectorcall_flag(DerivedType), False) + self.assertEqual(_testcapi.has_vectorcall_flag(UnaffectedType1), True) + self.assertEqual(_testcapi.has_vectorcall_flag(UnaffectedType2), True) + + def test_vectorcall(self): # Test a bunch of different ways to call objects: # 1. vectorcall using PyVectorcall_Call() # (only for objects that support vectorcall directly) # 2. normal call - # 3. vectorcall using _PyObject_Vectorcall() + # 3. vectorcall using PyObject_Vectorcall() # 4. call as bound method # 5. call using functools.partial @@ -616,6 +802,300 @@ def __call__(self, *args): self.assertEqual(expected, meth(*args1, **kwargs)) self.assertEqual(expected, wrapped(*args, **kwargs)) + def test_setvectorcall(self): + from _testcapi import function_setvectorcall + def f(num): return num + 1 + assert_equal = self.assertEqual + num = 10 + assert_equal(11, f(num)) + function_setvectorcall(f) + # make sure specializer is triggered by running > 50 times + for _ in range(10 * ADAPTIVE_WARMUP_DELAY): + assert_equal("overridden", f(num)) + + def test_setvectorcall_load_attr_specialization_skip(self): + from _testcapi import function_setvectorcall + + class X: + def __getattribute__(self, attr): + return attr + + assert_equal = self.assertEqual + x = X() + assert_equal("a", x.a) + function_setvectorcall(X.__getattribute__) + # make sure specialization doesn't trigger + # when vectorcall is overridden + for _ in range(ADAPTIVE_WARMUP_DELAY): + assert_equal("overridden", x.a) + + def test_setvectorcall_load_attr_specialization_deopt(self): + from _testcapi import function_setvectorcall + + class X: + def __getattribute__(self, attr): + return attr + + def get_a(x): + return x.a + + assert_equal = self.assertEqual + x = X() + # trigger LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN specialization + for _ in range(ADAPTIVE_WARMUP_DELAY): + assert_equal("a", get_a(x)) + function_setvectorcall(X.__getattribute__) + # make sure specialized LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN + # gets deopted due to overridden vectorcall + for _ in range(ADAPTIVE_WARMUP_DELAY): + assert_equal("overridden", get_a(x)) + + @requires_limited_api + def test_vectorcall_limited_incoming(self): + from _testcapi import pyobject_vectorcall + obj = _testlimitedcapi.LimitedVectorCallClass() + self.assertEqual(pyobject_vectorcall(obj, (), ()), "vectorcall called") + + @requires_limited_api + def test_vectorcall_limited_outgoing(self): + from _testlimitedcapi import call_vectorcall + + args_captured = [] + kwargs_captured = [] + + def f(*args, **kwargs): + args_captured.append(args) + kwargs_captured.append(kwargs) + return "success" + + self.assertEqual(call_vectorcall(f), "success") + self.assertEqual(args_captured, [("foo",)]) + self.assertEqual(kwargs_captured, [{"baz": "bar"}]) + + @requires_limited_api + def test_vectorcall_limited_outgoing_method(self): + from _testlimitedcapi import call_vectorcall_method + + args_captured = [] + kwargs_captured = [] + + class TestInstance: + def f(self, *args, **kwargs): + args_captured.append(args) + kwargs_captured.append(kwargs) + return "success" + + self.assertEqual(call_vectorcall_method(TestInstance()), "success") + self.assertEqual(args_captured, [("foo",)]) + self.assertEqual(kwargs_captured, [{"baz": "bar"}]) + +class A: + def method_two_args(self, x, y): + pass + + @staticmethod + def static_no_args(): + pass + + @staticmethod + def positional_only(arg, /): + pass + +@cpython_only +class TestErrorMessagesUseQualifiedName(unittest.TestCase): + + @contextlib.contextmanager + def check_raises_type_error(self, message): + with self.assertRaises(TypeError) as cm: + yield + self.assertEqual(str(cm.exception), message) + + def test_missing_arguments(self): + msg = "A.method_two_args() missing 1 required positional argument: 'y'" + with self.check_raises_type_error(msg): + A().method_two_args("x") + + def test_too_many_positional(self): + msg = "A.static_no_args() takes 0 positional arguments but 1 was given" + with self.check_raises_type_error(msg): + A.static_no_args("oops it's an arg") + + def test_positional_only_passed_as_keyword(self): + msg = "A.positional_only() got some positional-only arguments passed as keyword arguments: 'arg'" + with self.check_raises_type_error(msg): + A.positional_only(arg="x") + + def test_unexpected_keyword(self): + msg = "A.method_two_args() got an unexpected keyword argument 'bad'" + with self.check_raises_type_error(msg): + A().method_two_args(bad="x") + + def test_multiple_values(self): + msg = "A.method_two_args() got multiple values for argument 'x'" + with self.check_raises_type_error(msg): + A().method_two_args("x", "y", x="oops") + +@cpython_only +class TestErrorMessagesSuggestions(unittest.TestCase): + @contextlib.contextmanager + def check_suggestion_includes(self, message): + with self.assertRaises(TypeError) as cm: + yield + self.assertIn(f"Did you mean '{message}'?", str(cm.exception)) + + @contextlib.contextmanager + def check_suggestion_not_present(self): + with self.assertRaises(TypeError) as cm: + yield + self.assertNotIn("Did you mean", str(cm.exception)) + + def test_unexpected_keyword_suggestion_valid_positions(self): + def foo(blech=None, /, aaa=None, *args, late1=None): + pass + + cases = [ + ("blach", None), + ("aa", "aaa"), + ("orgs", None), + ("late11", "late1"), + ] + + for keyword, suggestion in cases: + with self.subTest(keyword): + ctx = self.check_suggestion_includes(suggestion) if suggestion else self.check_suggestion_not_present() + with ctx: + foo(**{keyword:None}) + + def test_unexpected_keyword_suggestion_kinds(self): + + def substitution(noise=None, more_noise=None, a = None, blech = None): + pass + + def elimination(noise = None, more_noise = None, a = None, blch = None): + pass + + def addition(noise = None, more_noise = None, a = None, bluchin = None): + pass + + def substitution_over_elimination(blach = None, bluc = None): + pass + + def substitution_over_addition(blach = None, bluchi = None): + pass + + def elimination_over_addition(bluc = None, blucha = None): + pass + + def case_change_over_substitution(BLuch=None, Luch = None, fluch = None): + pass + + for func, suggestion in [ + (addition, "bluchin"), + (substitution, "blech"), + (elimination, "blch"), + (addition, "bluchin"), + (substitution_over_elimination, "blach"), + (substitution_over_addition, "blach"), + (elimination_over_addition, "bluc"), + (case_change_over_substitution, "BLuch"), + ]: + with self.subTest(suggestion): + with self.check_suggestion_includes(suggestion): + func(bluch=None) + + def test_unexpected_keyword_suggestion_via_getargs(self): + with self.check_suggestion_includes("maxsplit"): + "foo".split(maxsplt=1) + + self.assertRaisesRegex( + TypeError, r"split\(\) got an unexpected keyword argument 'blech'$", + "foo".split, blech=1 + ) + with self.check_suggestion_not_present(): + "foo".split(blech=1) + with self.check_suggestion_not_present(): + "foo".split(more_noise=1, maxsplt=1) + + # Also test the vgetargskeywords path + with self.check_suggestion_includes("name"): + ImportError(namez="oops") + + self.assertRaisesRegex( + TypeError, r"ImportError\(\) got an unexpected keyword argument 'blech'$", + ImportError, blech=1 + ) + with self.check_suggestion_not_present(): + ImportError(blech=1) + with self.check_suggestion_not_present(): + ImportError(blech=1, namez="oops") + +@cpython_only +class TestRecursion(unittest.TestCase): + + @skip_on_s390x + @unittest.skipIf(is_wasi and Py_DEBUG, "requires deep stack") + @unittest.skipIf(_testcapi is None, "requires _testcapi") + def test_super_deep(self): + + def recurse(n): + if n: + recurse(n-1) + + def py_recurse(n, m): + if n: + py_recurse(n-1, m) + else: + c_py_recurse(m-1) + + def c_recurse(n): + if n: + _testcapi.pyobject_vectorcall(c_recurse, (n-1,), ()) + + def c_py_recurse(m): + if m: + _testcapi.pyobject_vectorcall(py_recurse, (1000, m), ()) + + with set_recursion_limit(100_000): + recurse(90_000) + with self.assertRaises(RecursionError): + recurse(101_000) + c_recurse(100) + with self.assertRaises(RecursionError): + c_recurse(90_000) + c_py_recurse(90) + with self.assertRaises(RecursionError): + c_py_recurse(100_000) + + +class TestFunctionWithManyArgs(unittest.TestCase): + def test_function_with_many_args(self): + for N in (10, 500, 1000): + with self.subTest(N=N): + args = ",".join([f"a{i}" for i in range(N)]) + src = f"def f({args}) : return a{N//2}" + l = {} + exec(src, {}, l) + self.assertEqual(l['f'](*range(N)), N//2) + + +@unittest.skipIf(_testcapi is None, 'need _testcapi') +class TestCAPI(unittest.TestCase): + def test_cfunction_call(self): + def func(*args, **kwargs): + return (args, kwargs) + + # PyCFunction_Call() was removed in Python 3.13 API, but was kept in + # the stable ABI. + def PyCFunction_Call(func, *args, **kwargs): + if kwargs: + return _testcapi.pycfunction_call(func, args, kwargs) + else: + return _testcapi.pycfunction_call(func, args) + + self.assertEqual(PyCFunction_Call(func), ((), {})) + self.assertEqual(PyCFunction_Call(func, 1, 2, 3), ((1, 2, 3), {})) + self.assertEqual(PyCFunction_Call(func, "arg", num=5), (("arg",), {'num': 5})) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_cmath.py b/Lib/test/test_cmath.py index 51dd2ecf5f..44f1b2da63 100644 --- a/Lib/test/test_cmath.py +++ b/Lib/test/test_cmath.py @@ -1,4 +1,5 @@ from test.support import requires_IEEE_754, cpython_only, import_helper +from test.support.testcase import ComplexesAreIdenticalMixin from test.test_math import parse_testfile, test_file import test.test_math as test_math import unittest @@ -49,7 +50,7 @@ (INF, NAN) ]] -class CMathTests(unittest.TestCase): +class CMathTests(ComplexesAreIdenticalMixin, unittest.TestCase): # list of all functions in cmath test_functions = [getattr(cmath, fname) for fname in [ 'acos', 'acosh', 'asin', 'asinh', 'atan', 'atanh', @@ -65,39 +66,6 @@ def setUp(self): def tearDown(self): self.test_values.close() - def assertFloatIdentical(self, x, y): - """Fail unless floats x and y are identical, in the sense that: - (1) both x and y are nans, or - (2) both x and y are infinities, with the same sign, or - (3) both x and y are zeros, with the same sign, or - (4) x and y are both finite and nonzero, and x == y - - """ - msg = 'floats {!r} and {!r} are not identical' - - if math.isnan(x) or math.isnan(y): - if math.isnan(x) and math.isnan(y): - return - elif x == y: - if x != 0.0: - return - # both zero; check that signs match - elif math.copysign(1.0, x) == math.copysign(1.0, y): - return - else: - msg += ': zeros have different signs' - self.fail(msg.format(x, y)) - - def assertComplexIdentical(self, x, y): - """Fail unless complex numbers x and y have equal values and signs. - - In particular, if x and y both have real (or imaginary) part - zero, but the zeros have different signs, this test will fail. - - """ - self.assertFloatIdentical(x.real, y.real) - self.assertFloatIdentical(x.imag, y.imag) - def rAssertAlmostEqual(self, a, b, rel_err = 2e-15, abs_err = 5e-323, msg=None): """Fail if the two floating-point numbers are not almost equal. @@ -299,12 +267,6 @@ def test_cmath_matches_math(self): for v in values: z = complex_fn(v) self.rAssertAlmostEqual(float_fn(v), z.real) - # TODO: RUSTPYTHON - # This line currently fails for acos and asin. - # cmath.asin/acos(0.2) should produce a real number, - # but imaginary part is 1.1102230246251565e-16 for both. - if fn in {"asin", "acos"}: - continue self.assertEqual(0., z.imag) # test two-argument version of log with various bases @@ -314,8 +276,7 @@ def test_cmath_matches_math(self): self.rAssertAlmostEqual(math.log(v, base), z.real) self.assertEqual(0., z.imag) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON @requires_IEEE_754 def test_specific_values(self): # Some tests need to be skipped on ancient OS X versions. @@ -563,25 +524,23 @@ def test_isinf(self): @requires_IEEE_754 def testTanhSign(self): for z in complex_zeros: - self.assertComplexIdentical(cmath.tanh(z), z) + self.assertComplexesAreIdentical(cmath.tanh(z), z) # The algorithm used for atan and atanh makes use of the system # log1p function; If that system function doesn't respect the sign # of zero, then atan and atanh will also have difficulties with # the sign of complex zeros. - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON @requires_IEEE_754 def testAtanSign(self): for z in complex_zeros: - self.assertComplexIdentical(cmath.atan(z), z) + self.assertComplexesAreIdentical(cmath.atan(z), z) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON @requires_IEEE_754 def testAtanhSign(self): for z in complex_zeros: - self.assertComplexIdentical(cmath.atanh(z), z) + self.assertComplexesAreIdentical(cmath.atanh(z), z) class IsCloseTests(test_math.IsCloseTests): @@ -624,8 +583,7 @@ def test_complex_near_zero(self): self.assertIsClose(0.001-0.001j, 0.001+0.001j, abs_tol=2e-03) self.assertIsNotClose(0.001-0.001j, 0.001+0.001j, abs_tol=1e-03) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_complex_special(self): self.assertIsNotClose(INF, INF*1j) self.assertIsNotClose(INF*1j, INF) diff --git a/Lib/test/test_cmd_line.py b/Lib/test/test_cmd_line.py index a7e4f6dd27..eb455ebaed 100644 --- a/Lib/test/test_cmd_line.py +++ b/Lib/test/test_cmd_line.py @@ -713,8 +713,6 @@ def run_xdev(self, *args, check_exitcode=True, xdev=True): self.assertEqual(proc.returncode, 0, proc) return proc.stdout.rstrip() - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_xdev(self): # sys.flags.dev_mode code = "import sys; print(sys.flags.dev_mode)" diff --git a/Lib/test/test_cmd_line_script.py b/Lib/test/test_cmd_line_script.py index 833dc6b15d..9fc240de81 100644 --- a/Lib/test/test_cmd_line_script.py +++ b/Lib/test/test_cmd_line_script.py @@ -225,8 +225,6 @@ def test_repl_stderr_flush(self): def test_repl_stderr_flush_separate_stderr(self): self.check_repl_stderr_flush(True) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_basic_script(self): with os_helper.temp_dir() as script_dir: script_name = _make_test_script(script_dir, 'script') @@ -249,8 +247,6 @@ def test_script_abspath(self): script_dir, None, importlib.machinery.SourceFileLoader) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_script_compiled(self): with os_helper.temp_dir() as script_dir: script_name = _make_test_script(script_dir, 'script') @@ -413,8 +409,6 @@ def test_issue8202(self): script_name, script_name, script_dir, 'test_pkg', importlib.machinery.SourceFileLoader) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_issue8202_dash_c_file_ignored(self): # Make sure a "-c" file in the current directory # does not alter the value of sys.path[0] @@ -715,8 +709,6 @@ def test_syntaxerror_null_bytes_in_multiline_string(self): ] ) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_consistent_sys_path_for_direct_execution(self): # This test case ensures that the following all give the same # sys.path configuration: diff --git a/Lib/test/test_code.py b/Lib/test/test_code.py index 1aceff4efc..804cce1dba 100644 --- a/Lib/test/test_code.py +++ b/Lib/test/test_code.py @@ -222,8 +222,6 @@ class List(list): obj = List([1, 2, 3]) self.assertEqual(obj[0], "Foreign getitem: 1") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_constructor(self): def func(): pass co = func.__code__ @@ -249,16 +247,12 @@ def func(): pass co.co_freevars, co.co_cellvars) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_qualname(self): self.assertEqual( CodeTest.test_qualname.__code__.co_qualname, CodeTest.test_qualname.__qualname__ ) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_replace(self): def func(): x = 1 @@ -299,8 +293,6 @@ def func2(): self.assertEqual(new_code.co_varnames, code2.co_varnames) self.assertEqual(new_code.co_nlocals, code2.co_nlocals) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_nlocals_mismatch(self): def func(): x = 1 @@ -349,8 +341,6 @@ def func(arg): newcode = code.replace(co_name="func") # Should not raise SystemError self.assertEqual(code, newcode) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_empty_linetable(self): def func(): pass @@ -470,8 +460,6 @@ def f(): # co_positions behavior when info is missing. - # TODO: RUSTPYTHON - @unittest.expectedFailure # @requires_debug_ranges() def test_co_positions_empty_linetable(self): def func(): @@ -482,8 +470,6 @@ def func(): self.assertIsNone(line) self.assertEqual(end_line, new_code.co_firstlineno + 1) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_code_equality(self): def f(): try: @@ -524,8 +510,6 @@ def test_code_hash_uses_order(self): self.assertNotEqual(c, swapped) self.assertNotEqual(hash(c), hash(swapped)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_code_hash_uses_bytecode(self): c = (lambda x, y: x + y).__code__ d = (lambda x, y: x * y).__code__ @@ -737,8 +721,6 @@ def check_positions(self, func): self.assertEqual(l1, l2) self.assertEqual(len(pos1), len(pos2)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_positions(self): self.check_positions(parse_location_table) self.check_positions(misshappen) @@ -753,8 +735,6 @@ def check_lines(self, func): self.assertEqual(l1, l2) self.assertEqual(len(lines1), len(lines2)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_lines(self): self.check_lines(parse_location_table) self.check_lines(misshappen) diff --git a/Lib/test/test_codeccallbacks.py b/Lib/test/test_codeccallbacks.py index bd1dbcd626..9ca02cea35 100644 --- a/Lib/test/test_codeccallbacks.py +++ b/Lib/test/test_codeccallbacks.py @@ -909,8 +909,6 @@ def handle(exc): self.assertEqual(exc.object, input) self.assertEqual(exc.reason, "surrogates not allowed") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_badregistercall(self): # enhance coverage of: # Modules/_codecsmodule.c::register_error() diff --git a/Lib/test/test_codecs.py b/Lib/test/test_codecs.py index a12e5893dc..736022599e 100644 --- a/Lib/test/test_codecs.py +++ b/Lib/test/test_codecs.py @@ -2,7 +2,6 @@ import contextlib import copy import io -import locale import pickle import sys import unittest @@ -13,9 +12,9 @@ from test.support import os_helper try: - import _testcapi + import _testlimitedcapi except ImportError: - _testcapi = None + _testlimitedcapi = None try: import _testinternalcapi except ImportError: @@ -389,8 +388,7 @@ def test_bug1098990_b(self): ill_formed_sequence_replace = "\ufffd" - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_lone_surrogates(self): self.assertRaises(UnicodeEncodeError, "\ud800".encode, self.encoding) self.assertEqual("[\uDC80]".encode(self.encoding, "backslashreplace"), @@ -466,14 +464,7 @@ class UTF32Test(ReadTest, unittest.TestCase): b'\x00\x00\x00s\x00\x00\x00p\x00\x00\x00a\x00\x00\x00m' b'\x00\x00\x00s\x00\x00\x00p\x00\x00\x00a\x00\x00\x00m') - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_readline(self): # TODO: RUSTPYTHON, remove when this passes - super().test_readline() # TODO: RUSTPYTHON, remove when this passes - - - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_only_one_bom(self): _,_,reader,writer = codecs.lookup(self.encoding) # encode some stream @@ -489,19 +480,17 @@ def test_only_one_bom(self): f = reader(s) self.assertEqual(f.read(), "spamspam") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_badbom(self): s = io.BytesIO(4*b"\xff") f = codecs.getreader(self.encoding)(s) - self.assertRaises(UnicodeError, f.read) + self.assertRaises(UnicodeDecodeError, f.read) s = io.BytesIO(8*b"\xff") f = codecs.getreader(self.encoding)(s) - self.assertRaises(UnicodeError, f.read) + self.assertRaises(UnicodeDecodeError, f.read) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_partial(self): self.check_partial( "\x00\xff\u0100\uffff\U00010000", @@ -533,30 +522,26 @@ def test_partial(self): ] ) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_handlers(self): self.assertEqual(('\ufffd', 1), codecs.utf_32_decode(b'\x01', 'replace', True)) self.assertEqual(('', 1), codecs.utf_32_decode(b'\x01', 'ignore', True)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_errors(self): self.assertRaises(UnicodeDecodeError, codecs.utf_32_decode, b"\xff", "strict", True) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_decoder_state(self): self.check_state_handling_decode(self.encoding, "spamspam", self.spamle) self.check_state_handling_decode(self.encoding, "spamspam", self.spambe) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_issue8941(self): # Issue #8941: insufficient result allocation when decoding into # surrogate pairs on UCS-2 builds. @@ -567,48 +552,40 @@ def test_issue8941(self): self.assertEqual('\U00010000' * 1024, codecs.utf_32_decode(encoded_be)[0]) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_bug1098990_a(self): - super().test_bug1098990_a() + return super().test_bug1098990_a() - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_bug1098990_b(self): - super().test_bug1098990_b() + return super().test_bug1098990_b() - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_bug1175396(self): - super().test_bug1175396() + return super().test_bug1175396() - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_incremental_surrogatepass(self): - super().test_incremental_surrogatepass() + return super().test_incremental_surrogatepass() - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_mixed_readline_and_read(self): - super().test_mixed_readline_and_read() + return super().test_mixed_readline_and_read() - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_readline(self): + return super().test_readline() + + @unittest.expectedFailure # TODO: RUSTPYTHON def test_readlinequeue(self): - super().test_readlinequeue() + return super().test_readlinequeue() class UTF32LETest(ReadTest, unittest.TestCase): encoding = "utf-32-le" ill_formed_sequence = b"\x80\xdc\x00\x00" - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_readline(self): # TODO: RUSTPYTHON, remove when this passes - super().test_readline() # TODO: RUSTPYTHON, remove when this passes - - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_partial(self): self.check_partial( "\x00\xff\u0100\uffff\U00010000", @@ -636,19 +613,16 @@ def test_partial(self): ] ) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_simple(self): self.assertEqual("\U00010203".encode(self.encoding), b"\x03\x02\x01\x00") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_errors(self): self.assertRaises(UnicodeDecodeError, codecs.utf_32_le_decode, b"\xff", "strict", True) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_issue8941(self): # Issue #8941: insufficient result allocation when decoding into # surrogate pairs on UCS-2 builds. @@ -656,48 +630,40 @@ def test_issue8941(self): self.assertEqual('\U00010000' * 1024, codecs.utf_32_le_decode(encoded)[0]) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_bug1098990_a(self): - super().test_bug1098990_a() + return super().test_bug1098990_a() - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_bug1098990_b(self): - super().test_bug1098990_b() + return super().test_bug1098990_b() - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_bug1175396(self): - super().test_bug1175396() + return super().test_bug1175396() - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_incremental_surrogatepass(self): - super().test_incremental_surrogatepass() + return super().test_incremental_surrogatepass() - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_mixed_readline_and_read(self): - super().test_mixed_readline_and_read() + return super().test_mixed_readline_and_read() - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_readline(self): + return super().test_readline() + + @unittest.expectedFailure # TODO: RUSTPYTHON def test_readlinequeue(self): - super().test_readlinequeue() + return super().test_readlinequeue() class UTF32BETest(ReadTest, unittest.TestCase): encoding = "utf-32-be" ill_formed_sequence = b"\x00\x00\xdc\x80" - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_readline(self): # TODO: RUSTPYTHON, remove when this passes - super().test_readline() # TODO: RUSTPYTHON, remove when this passes - - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_partial(self): self.check_partial( "\x00\xff\u0100\uffff\U00010000", @@ -725,19 +691,16 @@ def test_partial(self): ] ) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_simple(self): self.assertEqual("\U00010203".encode(self.encoding), b"\x00\x01\x02\x03") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_errors(self): self.assertRaises(UnicodeDecodeError, codecs.utf_32_be_decode, b"\xff", "strict", True) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_issue8941(self): # Issue #8941: insufficient result allocation when decoding into # surrogate pairs on UCS-2 builds. @@ -745,35 +708,33 @@ def test_issue8941(self): self.assertEqual('\U00010000' * 1024, codecs.utf_32_be_decode(encoded)[0]) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_bug1098990_a(self): - super().test_bug1098990_a() + return super().test_bug1098990_a() - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_bug1098990_b(self): - super().test_bug1098990_b() + return super().test_bug1098990_b() - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_bug1175396(self): - super().test_bug1175396() + return super().test_bug1175396() - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_incremental_surrogatepass(self): - super().test_incremental_surrogatepass() + return super().test_incremental_surrogatepass() - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_mixed_readline_and_read(self): - super().test_mixed_readline_and_read() + return super().test_mixed_readline_and_read() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_readline(self): + return super().test_readline() - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_readlinequeue(self): - super().test_readlinequeue() + return super().test_readlinequeue() class UTF16Test(ReadTest, unittest.TestCase): @@ -804,14 +765,13 @@ def test_only_one_bom(self): def test_badbom(self): s = io.BytesIO(b"\xff\xff") f = codecs.getreader(self.encoding)(s) - self.assertRaises(UnicodeError, f.read) + self.assertRaises(UnicodeDecodeError, f.read) s = io.BytesIO(b"\xff\xff\xff\xff") f = codecs.getreader(self.encoding)(s) - self.assertRaises(UnicodeError, f.read) + self.assertRaises(UnicodeDecodeError, f.read) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_partial(self): self.check_partial( "\x00\xff\u0100\uffff\U00010000", @@ -833,8 +793,7 @@ def test_partial(self): ] ) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_handlers(self): self.assertEqual(('\ufffd', 1), codecs.utf_16_decode(b'\x01', 'replace', True)) @@ -850,9 +809,7 @@ def test_decoder_state(self): "spamspam", self.spamle) self.check_state_handling_decode(self.encoding, "spamspam", self.spambe) - - # TODO: RUSTPYTHON - ValueError: invalid mode 'Ub' - @unittest.expectedFailure + def test_bug691291(self): # If encoding is not None, then # files are always opened in binary mode, even if no binary mode was @@ -864,22 +821,32 @@ def test_bug691291(self): self.addCleanup(os_helper.unlink, os_helper.TESTFN) with open(os_helper.TESTFN, 'wb') as fp: fp.write(s) - with warnings_helper.check_warnings(('', DeprecationWarning)): - reader = codecs.open(os_helper.TESTFN, 'U', encoding=self.encoding) - with reader: + with codecs.open(os_helper.TESTFN, 'r', + encoding=self.encoding) as reader: self.assertEqual(reader.read(), s1) - # TODO: RUSTPYTHON - @unittest.expectedFailure + def test_invalid_modes(self): + for mode in ('U', 'rU', 'r+U'): + with self.assertRaises(ValueError) as cm: + codecs.open(os_helper.TESTFN, mode, encoding=self.encoding) + self.assertIn('invalid mode', str(cm.exception)) + + for mode in ('rt', 'wt', 'at', 'r+t'): + with self.assertRaises(ValueError) as cm: + codecs.open(os_helper.TESTFN, mode, encoding=self.encoding) + self.assertIn("can't have text and binary mode at once", + str(cm.exception)) + + @unittest.expectedFailure # TODO: RUSTPYTHON def test_incremental_surrogatepass(self): - super().test_incremental_surrogatepass() + return super().test_incremental_surrogatepass() + class UTF16LETest(ReadTest, unittest.TestCase): encoding = "utf-16-le" ill_formed_sequence = b"\x80\xdc" - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_partial(self): self.check_partial( "\x00\xff\u0100\uffff\U00010000", @@ -899,8 +866,7 @@ def test_partial(self): ] ) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_errors(self): tests = [ (b'\xff', '\ufffd'), @@ -922,17 +888,15 @@ def test_nonbmp(self): self.assertEqual(b'\x00\xd8\x03\xde'.decode(self.encoding), "\U00010203") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_incremental_surrogatepass(self): - super().test_incremental_surrogatepass() + return super().test_incremental_surrogatepass() class UTF16BETest(ReadTest, unittest.TestCase): encoding = "utf-16-be" ill_formed_sequence = b"\xdc\x80" - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_partial(self): self.check_partial( "\x00\xff\u0100\uffff\U00010000", @@ -952,8 +916,7 @@ def test_partial(self): ] ) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_errors(self): tests = [ (b'\xff', '\ufffd'), @@ -975,10 +938,9 @@ def test_nonbmp(self): self.assertEqual(b'\xd8\x00\xde\x03'.decode(self.encoding), "\U00010203") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_incremental_surrogatepass(self): - super().test_incremental_surrogatepass() + return super().test_incremental_surrogatepass() class UTF8Test(ReadTest, unittest.TestCase): encoding = "utf-8" @@ -1025,8 +987,7 @@ def test_decode_error(self): self.assertEqual(data.decode(self.encoding, error_handler), expected) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_lone_surrogates(self): super().test_lone_surrogates() # not sure if this is making sense for @@ -1079,13 +1040,7 @@ def test_incremental_errors(self): class UTF7Test(ReadTest, unittest.TestCase): encoding = "utf-7" - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_readline(self): # TODO: RUSTPYTHON, remove when this passes - super().test_readline() # TODO: RUSTPYTHON, remove when this passes - - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_ascii(self): # Set D (directly encoded characters) set_d = ('ABCDEFGHIJKLMNOPQRSTUVWXYZ' @@ -1112,8 +1067,7 @@ def test_ascii(self): b'+AAAAAQACAAMABAAFAAYABwAIAAsADAAOAA8AEAARABIAEwAU' b'ABUAFgAXABgAGQAaABsAHAAdAB4AHwBcAH4Afw-') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_partial(self): self.check_partial( 'a+-b\x00c\x80d\u0100e\U00010000f', @@ -1153,8 +1107,7 @@ def test_partial(self): ] ) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_errors(self): tests = [ (b'\xffb', '\ufffdb'), @@ -1185,8 +1138,7 @@ def test_errors(self): raw, 'strict', True) self.assertEqual(raw.decode('utf-7', 'replace'), expected) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_nonbmp(self): self.assertEqual('\U000104A0'.encode(self.encoding), b'+2AHcoA-') self.assertEqual('\ud801\udca0'.encode(self.encoding), b'+2AHcoA-') @@ -1202,8 +1154,7 @@ def test_nonbmp(self): self.assertEqual(b'+IKwgrNgB3KA'.decode(self.encoding), '\u20ac\u20ac\U000104A0') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_lone_surrogates(self): tests = [ (b'a+2AE-b', 'a\ud801b'), @@ -1224,15 +1175,17 @@ def test_lone_surrogates(self): with self.subTest(raw=raw): self.assertEqual(raw.decode('utf-7', 'replace'), expected) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_bug1175396(self): - super().test_bug1175396() + return super().test_bug1175396() - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_incremental_surrogatepass(self): - super().test_incremental_surrogatepass() + return super().test_incremental_surrogatepass() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_readline(self): + return super().test_readline() class UTF16ExTest(unittest.TestCase): @@ -1357,8 +1310,7 @@ def test_raw(self): if b != b'\\': self.assertEqual(decode(b + b'0'), (b + b'0', 2)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_escape(self): decode = codecs.escape_decode check = coding_checker(self, decode) @@ -1379,28 +1331,43 @@ def test_escape(self): check(br"[\418]", b"[!8]") check(br"[\101]", b"[A]") check(br"[\1010]", b"[A0]") - check(br"[\501]", b"[A]") check(br"[\x41]", b"[A]") check(br"[\x410]", b"[A0]") + + @unittest.expectedFailure # TODO: RUSTPYTHON; DeprecationWarning not triggered + def test_warnings(self): + decode = codecs.escape_decode + check = coding_checker(self, decode) for i in range(97, 123): b = bytes([i]) if b not in b'abfnrtvx': - with self.assertWarns(DeprecationWarning): + with self.assertWarnsRegex(DeprecationWarning, + r"invalid escape sequence '\\%c'" % i): check(b"\\" + b, b"\\" + b) - with self.assertWarns(DeprecationWarning): + with self.assertWarnsRegex(DeprecationWarning, + r"invalid escape sequence '\\%c'" % (i-32)): check(b"\\" + b.upper(), b"\\" + b.upper()) - with self.assertWarns(DeprecationWarning): + with self.assertWarnsRegex(DeprecationWarning, + r"invalid escape sequence '\\8'"): check(br"\8", b"\\8") with self.assertWarns(DeprecationWarning): check(br"\9", b"\\9") - with self.assertWarns(DeprecationWarning): + with self.assertWarnsRegex(DeprecationWarning, + r"invalid escape sequence '\\\xfa'") as cm: check(b"\\\xfa", b"\\\xfa") for i in range(0o400, 0o1000): - with self.assertWarns(DeprecationWarning): + with self.assertWarnsRegex(DeprecationWarning, + r"invalid octal escape sequence '\\%o'" % i): check(rb'\%o' % i, bytes([i & 0o377])) - - # TODO: RUSTPYTHON - ValueError: not raised by escape_decode - @unittest.expectedFailure + + with self.assertWarnsRegex(DeprecationWarning, + r"invalid escape sequence '\\z'"): + self.assertEqual(decode(br'\x\z', 'ignore'), (b'\\z', 4)) + with self.assertWarnsRegex(DeprecationWarning, + r"invalid octal escape sequence '\\501'"): + self.assertEqual(decode(br'\x\501', 'ignore'), (b'A', 6)) + + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: not raised by escape_decode def test_errors(self): decode = codecs.escape_decode self.assertRaises(ValueError, decode, br"\x") @@ -1543,13 +1510,29 @@ def test_decode(self): def test_decode_invalid(self): testcases = [ - (b"xn--w&", "strict", UnicodeError()), + (b"xn--w&", "strict", UnicodeDecodeError("punycode", b"", 5, 6, "")), + (b"&egbpdaj6bu4bxfgehfvwxn", "strict", UnicodeDecodeError("punycode", b"", 0, 1, "")), + (b"egbpdaj6bu&4bx&fgehfvwxn", "strict", UnicodeDecodeError("punycode", b"", 10, 11, "")), + (b"egbpdaj6bu4bxfgehfvwxn&", "strict", UnicodeDecodeError("punycode", b"", 22, 23, "")), + (b"\xFFProprostnemluvesky-uyb24dma41a", "strict", UnicodeDecodeError("ascii", b"", 0, 1, "")), + (b"Pro\xFFprostnemluvesky-uyb24dma41a", "strict", UnicodeDecodeError("ascii", b"", 3, 4, "")), + (b"Proprost&nemluvesky-uyb24&dma41a", "strict", UnicodeDecodeError("punycode", b"", 25, 26, "")), + (b"Proprostnemluvesky&-&uyb24dma41a", "strict", UnicodeDecodeError("punycode", b"", 20, 21, "")), + (b"Proprostnemluvesky-&uyb24dma41a", "strict", UnicodeDecodeError("punycode", b"", 19, 20, "")), + (b"Proprostnemluvesky-uyb24d&ma41a", "strict", UnicodeDecodeError("punycode", b"", 25, 26, "")), + (b"Proprostnemluvesky-uyb24dma41a&", "strict", UnicodeDecodeError("punycode", b"", 30, 31, "")), (b"xn--w&", "ignore", "xn-"), ] for puny, errors, expected in testcases: with self.subTest(puny=puny, errors=errors): if isinstance(expected, Exception): - self.assertRaises(UnicodeError, puny.decode, "punycode", errors) + with self.assertRaises(UnicodeDecodeError) as cm: + puny.decode("punycode", errors) + exc = cm.exception + self.assertEqual(exc.encoding, expected.encoding) + self.assertEqual(exc.object, puny) + self.assertEqual(exc.start, expected.start) + self.assertEqual(exc.end, expected.end) else: self.assertEqual(puny.decode("punycode", errors), expected) @@ -1719,7 +1702,7 @@ def test_nameprep(self): orig = str(orig, "utf-8", "surrogatepass") if prepped is None: # Input contains prohibited characters - self.assertRaises(UnicodeError, nameprep, orig) + self.assertRaises(UnicodeEncodeError, nameprep, orig) else: prepped = str(prepped, "utf-8", "surrogatepass") try: @@ -1729,11 +1712,46 @@ def test_nameprep(self): class IDNACodecTest(unittest.TestCase): + + invalid_decode_testcases = [ + (b"\xFFpython.org", UnicodeDecodeError("idna", b"\xFFpython.org", 0, 1, "")), + (b"pyt\xFFhon.org", UnicodeDecodeError("idna", b"pyt\xFFhon.org", 3, 4, "")), + (b"python\xFF.org", UnicodeDecodeError("idna", b"python\xFF.org", 6, 7, "")), + (b"python.\xFForg", UnicodeDecodeError("idna", b"python.\xFForg", 7, 8, "")), + (b"python.o\xFFrg", UnicodeDecodeError("idna", b"python.o\xFFrg", 8, 9, "")), + (b"python.org\xFF", UnicodeDecodeError("idna", b"python.org\xFF", 10, 11, "")), + (b"xn--pythn-&mua.org", UnicodeDecodeError("idna", b"xn--pythn-&mua.org", 10, 11, "")), + (b"xn--pythn-m&ua.org", UnicodeDecodeError("idna", b"xn--pythn-m&ua.org", 11, 12, "")), + (b"xn--pythn-mua&.org", UnicodeDecodeError("idna", b"xn--pythn-mua&.org", 13, 14, "")), + ] + invalid_encode_testcases = [ + (f"foo.{'\xff'*60}", UnicodeEncodeError("idna", f"foo.{'\xff'*60}", 4, 64, "")), + ("あさ.\u034f", UnicodeEncodeError("idna", "あさ.\u034f", 3, 4, "")), + ] + def test_builtin_decode(self): self.assertEqual(str(b"python.org", "idna"), "python.org") self.assertEqual(str(b"python.org.", "idna"), "python.org.") self.assertEqual(str(b"xn--pythn-mua.org", "idna"), "pyth\xf6n.org") self.assertEqual(str(b"xn--pythn-mua.org.", "idna"), "pyth\xf6n.org.") + self.assertEqual(str(b"XN--pythn-mua.org.", "idna"), "pyth\xf6n.org.") + self.assertEqual(str(b"xN--pythn-mua.org.", "idna"), "pyth\xf6n.org.") + self.assertEqual(str(b"Xn--pythn-mua.org.", "idna"), "pyth\xf6n.org.") + self.assertEqual(str(b"bugs.xn--pythn-mua.org.", "idna"), + "bugs.pyth\xf6n.org.") + self.assertEqual(str(b"bugs.XN--pythn-mua.org.", "idna"), + "bugs.pyth\xf6n.org.") + + def test_builtin_decode_invalid(self): + for case, expected in self.invalid_decode_testcases: + with self.subTest(case=case, expected=expected): + with self.assertRaises(UnicodeDecodeError) as cm: + case.decode("idna") + exc = cm.exception + self.assertEqual(exc.encoding, expected.encoding) + self.assertEqual(exc.object, expected.object) + self.assertEqual(exc.start, expected.start, msg=f'reason: {exc.reason}') + self.assertEqual(exc.end, expected.end) def test_builtin_encode(self): self.assertEqual("python.org".encode("idna"), b"python.org") @@ -1741,10 +1759,21 @@ def test_builtin_encode(self): self.assertEqual("pyth\xf6n.org".encode("idna"), b"xn--pythn-mua.org") self.assertEqual("pyth\xf6n.org.".encode("idna"), b"xn--pythn-mua.org.") + def test_builtin_encode_invalid(self): + for case, expected in self.invalid_encode_testcases: + with self.subTest(case=case, expected=expected): + with self.assertRaises(UnicodeEncodeError) as cm: + case.encode("idna") + exc = cm.exception + self.assertEqual(exc.encoding, expected.encoding) + self.assertEqual(exc.object, expected.object) + self.assertEqual(exc.start, expected.start) + self.assertEqual(exc.end, expected.end) + def test_builtin_decode_length_limit(self): - with self.assertRaisesRegex(UnicodeError, "way too long"): + with self.assertRaisesRegex(UnicodeDecodeError, "way too long"): (b"xn--016c"+b"a"*1100).decode("idna") - with self.assertRaisesRegex(UnicodeError, "too long"): + with self.assertRaisesRegex(UnicodeDecodeError, "too long"): (b"xn--016c"+b"a"*70).decode("idna") def test_stream(self): @@ -1782,6 +1811,39 @@ def test_incremental_decode(self): self.assertEqual(decoder.decode(b"rg."), "org.") self.assertEqual(decoder.decode(b"", True), "") + def test_incremental_decode_invalid(self): + iterdecode_testcases = [ + (b"\xFFpython.org", UnicodeDecodeError("idna", b"\xFF", 0, 1, "")), + (b"pyt\xFFhon.org", UnicodeDecodeError("idna", b"pyt\xFF", 3, 4, "")), + (b"python\xFF.org", UnicodeDecodeError("idna", b"python\xFF", 6, 7, "")), + (b"python.\xFForg", UnicodeDecodeError("idna", b"\xFF", 0, 1, "")), + (b"python.o\xFFrg", UnicodeDecodeError("idna", b"o\xFF", 1, 2, "")), + (b"python.org\xFF", UnicodeDecodeError("idna", b"org\xFF", 3, 4, "")), + (b"xn--pythn-&mua.org", UnicodeDecodeError("idna", b"xn--pythn-&mua.", 10, 11, "")), + (b"xn--pythn-m&ua.org", UnicodeDecodeError("idna", b"xn--pythn-m&ua.", 11, 12, "")), + (b"xn--pythn-mua&.org", UnicodeDecodeError("idna", b"xn--pythn-mua&.", 13, 14, "")), + ] + for case, expected in iterdecode_testcases: + with self.subTest(case=case, expected=expected): + with self.assertRaises(UnicodeDecodeError) as cm: + list(codecs.iterdecode((bytes([c]) for c in case), "idna")) + exc = cm.exception + self.assertEqual(exc.encoding, expected.encoding) + self.assertEqual(exc.object, expected.object) + self.assertEqual(exc.start, expected.start) + self.assertEqual(exc.end, expected.end) + + decoder = codecs.getincrementaldecoder("idna")() + for case, expected in self.invalid_decode_testcases: + with self.subTest(case=case, expected=expected): + with self.assertRaises(UnicodeDecodeError) as cm: + decoder.decode(case) + exc = cm.exception + self.assertEqual(exc.encoding, expected.encoding) + self.assertEqual(exc.object, expected.object) + self.assertEqual(exc.start, expected.start) + self.assertEqual(exc.end, expected.end) + def test_incremental_encode(self): self.assertEqual( b"".join(codecs.iterencode("python.org", "idna")), @@ -1810,6 +1872,23 @@ def test_incremental_encode(self): self.assertEqual(encoder.encode("ample.org."), b"xn--xample-9ta.org.") self.assertEqual(encoder.encode("", True), b"") + def test_incremental_encode_invalid(self): + iterencode_testcases = [ + (f"foo.{'\xff'*60}", UnicodeEncodeError("idna", f"{'\xff'*60}", 0, 60, "")), + ("あさ.\u034f", UnicodeEncodeError("idna", "\u034f", 0, 1, "")), + ] + for case, expected in iterencode_testcases: + with self.subTest(case=case, expected=expected): + with self.assertRaises(UnicodeEncodeError) as cm: + list(codecs.iterencode(case, "idna")) + exc = cm.exception + self.assertEqual(exc.encoding, expected.encoding) + self.assertEqual(exc.object, expected.object) + self.assertEqual(exc.start, expected.start) + self.assertEqual(exc.end, expected.end) + + # codecs.getincrementalencoder.encode() does not throw an error + def test_errors(self): """Only supports "strict" error handler""" "python.org".encode("idna", "strict") @@ -1887,16 +1966,10 @@ def test_getwriter(self): self.assertRaises(TypeError, codecs.getwriter) self.assertRaises(LookupError, codecs.getwriter, "__spam__") + @support.run_with_locale('LC_CTYPE', 'tr_TR') def test_lookup_issue1813(self): # Issue #1813: under Turkish locales, lookup of some codecs failed # because 'I' is lowercased as "ı" (dotless i) - oldlocale = locale.setlocale(locale.LC_CTYPE) - self.addCleanup(locale.setlocale, locale.LC_CTYPE, oldlocale) - try: - locale.setlocale(locale.LC_CTYPE, 'tr_TR') - except locale.Error: - # Unsupported locale on this system - self.skipTest('test needs Turkish locale') c = codecs.lookup('ASCII') self.assertEqual(c.name, 'ascii') @@ -1949,6 +2022,76 @@ def test_file_closes_if_lookup_error_raised(self): file().close.assert_called() + def test_copy(self): + orig = codecs.lookup('utf-8') + dup = copy.copy(orig) + self.assertIsNot(dup, orig) + self.assertEqual(dup, orig) + self.assertTrue(orig._is_text_encoding) + self.assertEqual(dup.encode, orig.encode) + self.assertEqual(dup.name, orig.name) + self.assertEqual(dup.incrementalencoder, orig.incrementalencoder) + + # Test a CodecInfo with _is_text_encoding equal to false. + orig = codecs.lookup("base64") + dup = copy.copy(orig) + self.assertIsNot(dup, orig) + self.assertEqual(dup, orig) + self.assertFalse(orig._is_text_encoding) + self.assertEqual(dup.encode, orig.encode) + self.assertEqual(dup.name, orig.name) + self.assertEqual(dup.incrementalencoder, orig.incrementalencoder) + + def test_deepcopy(self): + orig = codecs.lookup('utf-8') + dup = copy.deepcopy(orig) + self.assertIsNot(dup, orig) + self.assertEqual(dup, orig) + self.assertTrue(orig._is_text_encoding) + self.assertEqual(dup.encode, orig.encode) + self.assertEqual(dup.name, orig.name) + self.assertEqual(dup.incrementalencoder, orig.incrementalencoder) + + # Test a CodecInfo with _is_text_encoding equal to false. + orig = codecs.lookup("base64") + dup = copy.deepcopy(orig) + self.assertIsNot(dup, orig) + self.assertEqual(dup, orig) + self.assertFalse(orig._is_text_encoding) + self.assertEqual(dup.encode, orig.encode) + self.assertEqual(dup.name, orig.name) + self.assertEqual(dup.incrementalencoder, orig.incrementalencoder) + + def test_pickle(self): + codec_info = codecs.lookup('utf-8') + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(protocol=proto): + pickled_codec_info = pickle.dumps(codec_info) + unpickled_codec_info = pickle.loads(pickled_codec_info) + self.assertIsNot(codec_info, unpickled_codec_info) + self.assertEqual(codec_info, unpickled_codec_info) + self.assertEqual(codec_info.name, unpickled_codec_info.name) + self.assertEqual( + codec_info.incrementalencoder, + unpickled_codec_info.incrementalencoder + ) + self.assertTrue(unpickled_codec_info._is_text_encoding) + + # Test a CodecInfo with _is_text_encoding equal to false. + codec_info = codecs.lookup('base64') + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(protocol=proto): + pickled_codec_info = pickle.dumps(codec_info) + unpickled_codec_info = pickle.loads(pickled_codec_info) + self.assertIsNot(codec_info, unpickled_codec_info) + self.assertEqual(codec_info, unpickled_codec_info) + self.assertEqual(codec_info.name, unpickled_codec_info.name) + self.assertEqual( + codec_info.incrementalencoder, + unpickled_codec_info.incrementalencoder + ) + self.assertFalse(unpickled_codec_info._is_text_encoding) + class StreamReaderTest(unittest.TestCase): @@ -1979,14 +2122,14 @@ class StreamWriterTest(unittest.TestCase): def setUp(self): self.writer = codecs.getwriter('utf-8') - + def test_copy(self): f = self.writer(Queue(b'')) with self.assertRaisesRegex(TypeError, 'StreamWriter'): copy.copy(f) with self.assertRaisesRegex(TypeError, 'StreamWriter'): copy.deepcopy(f) - + def test_pickle(self): for proto in range(pickle.HIGHEST_PROTOCOL + 1): with self.subTest(protocol=proto): @@ -2007,7 +2150,7 @@ def test_copy(self): copy.copy(f) with self.assertRaisesRegex(TypeError, 'StreamReaderWriter'): copy.deepcopy(f) - + def test_pickle(self): for proto in range(pickle.HIGHEST_PROTOCOL + 1): with self.subTest(protocol=proto): @@ -2148,8 +2291,7 @@ def test_basic(self): class BasicUnicodeTest(unittest.TestCase, MixInCheckStateHandling): - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_basics(self): s = "abc123" # all codecs should be able to encode these for encoding in all_unicode_encodings: @@ -2231,14 +2373,14 @@ def test_basics(self): "encoding=%r" % encoding) @support.cpython_only - @unittest.skipIf(_testcapi is None, 'need _testcapi module') + @unittest.skipIf(_testlimitedcapi is None, 'need _testlimitedcapi module') def test_basics_capi(self): s = "abc123" # all codecs should be able to encode these for encoding in all_unicode_encodings: if encoding not in broken_unicode_with_stateful: # check incremental decoder/encoder (fetched via the C API) try: - cencoder = _testcapi.codec_incrementalencoder(encoding) + cencoder = _testlimitedcapi.codec_incrementalencoder(encoding) except LookupError: # no IncrementalEncoder pass else: @@ -2247,7 +2389,7 @@ def test_basics_capi(self): for c in s: encodedresult += cencoder.encode(c) encodedresult += cencoder.encode("", True) - cdecoder = _testcapi.codec_incrementaldecoder(encoding) + cdecoder = _testlimitedcapi.codec_incrementaldecoder(encoding) decodedresult = "" for c in encodedresult: decodedresult += cdecoder.decode(bytes([c])) @@ -2258,19 +2400,18 @@ def test_basics_capi(self): if encoding not in ("idna", "mbcs"): # check incremental decoder/encoder with errors argument try: - cencoder = _testcapi.codec_incrementalencoder(encoding, "ignore") + cencoder = _testlimitedcapi.codec_incrementalencoder(encoding, "ignore") except LookupError: # no IncrementalEncoder pass else: encodedresult = b"".join(cencoder.encode(c) for c in s) - cdecoder = _testcapi.codec_incrementaldecoder(encoding, "ignore") + cdecoder = _testlimitedcapi.codec_incrementaldecoder(encoding, "ignore") decodedresult = "".join(cdecoder.decode(bytes([c])) for c in encodedresult) self.assertEqual(decodedresult, s, "encoding=%r" % encoding) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_seek(self): # all codecs should be able to encode these s = "%s\n%s\n" % (100*"abc123", 100*"def456") @@ -2286,8 +2427,7 @@ def test_seek(self): data = reader.read() self.assertEqual(s, data) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_bad_decode_args(self): for encoding in all_unicode_encodings: decoder = codecs.getdecoder(encoding) @@ -2295,8 +2435,7 @@ def test_bad_decode_args(self): if encoding not in ("idna", "punycode"): self.assertRaises(TypeError, decoder, 42) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_bad_encode_args(self): for encoding in all_unicode_encodings: encoder = codecs.getencoder(encoding) @@ -2308,8 +2447,7 @@ def test_encoding_map_type_initialized(self): table_type = type(cp1140.encoding_table) self.assertEqual(table_type, table_type) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_decoder_state(self): # Check that getstate() and setstate() handle the state properly u = "abc123" @@ -2320,8 +2458,7 @@ def test_decoder_state(self): class CharmapTest(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_decode_with_string_map(self): self.assertEqual( codecs.charmap_decode(b"\x00\x01\x02", "strict", "abc"), @@ -2377,8 +2514,7 @@ def test_decode_with_string_map(self): ("", len(allbytes)) ) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_decode_with_int2str_map(self): self.assertEqual( codecs.charmap_decode(b"\x00\x01\x02", "strict", @@ -2495,8 +2631,7 @@ def test_decode_with_int2str_map(self): b"\x00\x01\x02", "strict", {0: "A", 1: 'Bb', 2: 999999999} ) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_decode_with_int2int_map(self): a = ord('a') b = ord('b') @@ -2589,8 +2724,7 @@ def test_streamreaderwriter(self): class TypesTest(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_decode_unicode(self): # Most decoders don't accept unicode input decoders = [ @@ -2636,16 +2770,6 @@ class UnicodeEscapeTest(ReadTest, unittest.TestCase): test_lone_surrogates = None - # TODO: RUSTPYTHON, TypeError: Expected type 'str', not 'bytes' - @unittest.expectedFailure - def test_incremental_surrogatepass(self): # TODO: RUSTPYTHON, remove when this passes - super().test_incremental_surrogatepass() # TODO: RUSTPYTHON, remove when this passes - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_readline(self): # TODO: RUSTPYTHON, remove when this passes - super().test_readline() # TODO: RUSTPYTHON, remove when this passes - def test_empty(self): self.assertEqual(codecs.unicode_escape_encode(""), (b"", 0)) self.assertEqual(codecs.unicode_escape_decode(b""), ("", 0)) @@ -2677,8 +2801,6 @@ def test_escape_encode(self): check('\u20ac', br'\u20ac') check('\U0001d120', br'\U0001d120') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_escape_decode(self): decode = codecs.unicode_escape_decode check = coding_checker(self, decode) @@ -2703,24 +2825,41 @@ def test_escape_decode(self): check(br"[\x410]", "[A0]") check(br"\u20ac", "\u20ac") check(br"\U0001d120", "\U0001d120") + + @unittest.expectedFailure # TODO: RUSTPYTHON; DeprecationWarning not triggered + def test_decode_warnings(self): + decode = codecs.unicode_escape_decode + check = coding_checker(self, decode) for i in range(97, 123): b = bytes([i]) if b not in b'abfnrtuvx': - with self.assertWarns(DeprecationWarning): + with self.assertWarnsRegex(DeprecationWarning, + r"invalid escape sequence '\\%c'" % i): check(b"\\" + b, "\\" + chr(i)) if b.upper() not in b'UN': - with self.assertWarns(DeprecationWarning): + with self.assertWarnsRegex(DeprecationWarning, + r"invalid escape sequence '\\%c'" % (i-32)): check(b"\\" + b.upper(), "\\" + chr(i-32)) - with self.assertWarns(DeprecationWarning): + with self.assertWarnsRegex(DeprecationWarning, + r"invalid escape sequence '\\8'"): check(br"\8", "\\8") with self.assertWarns(DeprecationWarning): check(br"\9", "\\9") - with self.assertWarns(DeprecationWarning): + with self.assertWarnsRegex(DeprecationWarning, + r"invalid escape sequence '\\\xfa'") as cm: check(b"\\\xfa", "\\\xfa") for i in range(0o400, 0o1000): - with self.assertWarns(DeprecationWarning): + with self.assertWarnsRegex(DeprecationWarning, + r"invalid octal escape sequence '\\%o'" % i): check(rb'\%o' % i, chr(i)) + with self.assertWarnsRegex(DeprecationWarning, + r"invalid escape sequence '\\z'"): + self.assertEqual(decode(br'\x\z', 'ignore'), ('\\z', 4)) + with self.assertWarnsRegex(DeprecationWarning, + r"invalid octal escape sequence '\\501'"): + self.assertEqual(decode(br'\x\501', 'ignore'), ('\u0141', 6)) + def test_decode_errors(self): decode = codecs.unicode_escape_decode for c, d in (b'x', 2), (b'u', 4), (b'U', 4): @@ -2737,8 +2876,7 @@ def test_decode_errors(self): self.assertEqual(decode(br"\U00110000", "ignore"), ("", 10)) self.assertEqual(decode(br"\U00110000", "replace"), ("\ufffd", 10)) - # TODO: RUSTPYTHON, UnicodeDecodeError: ('unicodeescape', b'\\', 0, 1, '\\ at end of string') - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: '\x00\t\n\r\\' != '\x00\t\n\r' def test_partial(self): self.check_partial( "\x00\t\n\r\\\xff\uffff\U00010000", @@ -2778,21 +2916,19 @@ def test_partial(self): ] ) + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_incremental_surrogatepass(self): + return super().test_incremental_surrogatepass() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_readline(self): + return super().test_readline() + class RawUnicodeEscapeTest(ReadTest, unittest.TestCase): encoding = "raw-unicode-escape" test_lone_surrogates = None - # TODO: RUSTPYTHON, AssertionError: '\\' != '' - @unittest.expectedFailure - def test_incremental_surrogatepass(self): # TODO: RUSTPYTHON, remove when this passes - super().test_incremental_surrogatepass() # TODO: RUSTPYTHON, remove when this passes - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_readline(self): # TODO: RUSTPYTHON, remove when this passes - super().test_readline() # TODO: RUSTPYTHON, remove when this passes - def test_empty(self): self.assertEqual(codecs.raw_unicode_escape_encode(""), (b"", 0)) self.assertEqual(codecs.raw_unicode_escape_decode(b""), ("", 0)) @@ -2841,8 +2977,7 @@ def test_decode_errors(self): self.assertEqual(decode(br"\U00110000", "ignore"), ("", 10)) self.assertEqual(decode(br"\U00110000", "replace"), ("\ufffd", 10)) - # TODO: RUSTPYTHON, AssertionError: '\x00\t\n\r\\' != '\x00\t\n\r' - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_partial(self): self.check_partial( "\x00\t\n\r\\\xff\uffff\U00010000", @@ -2872,6 +3007,14 @@ def test_partial(self): ] ) + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_incremental_surrogatepass(self): + return super().test_incremental_surrogatepass() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_readline(self): + return super().test_readline() + class EscapeEncodeTest(unittest.TestCase): @@ -2914,8 +3057,7 @@ def test_ascii(self): self.assertEqual("foo\udc80bar".encode("ascii", "surrogateescape"), b"foo\x80bar") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_charmap(self): # bad byte: \xa5 is unmapped in iso-8859-3 self.assertEqual(b"foo\xa5bar".decode("iso-8859-3", "surrogateescape"), @@ -2930,8 +3072,7 @@ def test_latin1(self): class BomTest(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_seek0(self): data = "1234567890" tests = ("utf-16", @@ -3048,8 +3189,6 @@ def test_readline(self): sout = reader.readline() self.assertEqual(sout, b"\x80") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_buffer_api_usage(self): # We check all the transform codecs accept memoryview input # for encoding and decoding @@ -3112,10 +3251,8 @@ def test_binary_to_text_denylists_text_transforms(self): bad_input.decode("rot_13") self.assertIsNone(failure.exception.__cause__) - - # @unittest.skipUnless(zlib, "Requires zlib support") - # TODO: RUSTPYTHON, ^ restore once test passes - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.skipUnless(zlib, "Requires zlib support") def test_custom_zlib_error_is_noted(self): # Check zlib codec gives a good error for malformed input msg = "decoding with 'zlib_codec' codec failed" @@ -3123,8 +3260,7 @@ def test_custom_zlib_error_is_noted(self): codecs.decode(b"hello", "zlib_codec") self.assertEqual(msg, failure.exception.__notes__[0]) - # TODO: RUSTPYTHON - AttributeError: 'Error' object has no attribute '__notes__' - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; - AttributeError: 'Error' object has no attribute '__notes__' def test_custom_hex_error_is_noted(self): # Check hex codec gives a good error for malformed input import binascii @@ -3204,9 +3340,8 @@ def assertNoted(self, operation, exc_type, msg): def raise_obj(self, *args, **kwds): # Helper to dynamically change the object raised by a test codec raise self.obj_to_raise - - # TODO: RUSTPYTHON - @unittest.expectedFailure + + @unittest.expectedFailure # TODO: RUSTPYTHON def check_note(self, obj_to_raise, msg, exc_type=RuntimeError): self.obj_to_raise = obj_to_raise self.set_codec(self.raise_obj, self.raise_obj) @@ -3218,65 +3353,56 @@ def check_note(self, obj_to_raise, msg, exc_type=RuntimeError): b"bytes input".decode(self.codec_name) with self.assertNoted("decoding", exc_type, msg): codecs.decode(b"bytes input", self.codec_name) - - # TODO: RUSTPYTHON - @unittest.expectedFailure + + @unittest.expectedFailure # TODO: RUSTPYTHON def test_raise_by_type(self): self.check_note(RuntimeError, "") - - # TODO: RUSTPYTHON - @unittest.expectedFailure + + @unittest.expectedFailure # TODO: RUSTPYTHON def test_raise_by_value(self): msg = "This should be noted" self.check_note(RuntimeError(msg), msg) - - # TODO: RUSTPYTHON - @unittest.expectedFailure + + @unittest.expectedFailure # TODO: RUSTPYTHON def test_raise_grandchild_subclass_exact_size(self): msg = "This should be noted" class MyRuntimeError(RuntimeError): __slots__ = () self.check_note(MyRuntimeError(msg), msg, MyRuntimeError) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_raise_subclass_with_weakref_support(self): msg = "This should be noted" class MyRuntimeError(RuntimeError): pass self.check_note(MyRuntimeError(msg), msg, MyRuntimeError) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_init_override(self): class CustomInit(RuntimeError): def __init__(self): pass self.check_note(CustomInit, "") - - # TODO: RUSTPYTHON - @unittest.expectedFailure + + @unittest.expectedFailure # TODO: RUSTPYTHON def test_new_override(self): class CustomNew(RuntimeError): def __new__(cls): return super().__new__(cls) self.check_note(CustomNew, "") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_instance_attribute(self): msg = "This should be noted" exc = RuntimeError(msg) exc.attr = 1 self.check_note(exc, "^{}$".format(msg)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_non_str_arg(self): self.check_note(RuntimeError(1), "1") - - # TODO: RUSTPYTHON - @unittest.expectedFailure + + @unittest.expectedFailure # TODO: RUSTPYTHON def test_multiple_args(self): msg_re = r"^\('a', 'b', 'c'\)$" self.check_note(RuntimeError('a', 'b', 'c'), msg_re) @@ -3293,8 +3419,7 @@ def test_codec_lookup_failure(self): with self.assertRaisesRegex(LookupError, msg): codecs.decode(b"bytes input", self.codec_name) - - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; def test_unflagged_non_text_codec_handling(self): # The stdlib non-text codecs are now marked so they're # pre-emptively skipped by the text model related methods @@ -3330,22 +3455,20 @@ def decode_to_bytes(*args, **kwds): class CodePageTest(unittest.TestCase): CP_UTF8 = 65001 - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_invalid_code_page(self): self.assertRaises(ValueError, codecs.code_page_encode, -1, 'a') self.assertRaises(ValueError, codecs.code_page_decode, -1, b'a') self.assertRaises(OSError, codecs.code_page_encode, 123, 'a') self.assertRaises(OSError, codecs.code_page_decode, 123, b'a') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_code_page_name(self): self.assertRaisesRegex(UnicodeEncodeError, 'cp932', codecs.code_page_encode, 932, '\xff') self.assertRaisesRegex(UnicodeDecodeError, 'cp932', codecs.code_page_decode, 932, b'\x81\x00', 'strict', True) - self.assertRaisesRegex(UnicodeDecodeError, 'CP_UTF8', + self.assertRaisesRegex(UnicodeDecodeError, 'cp65001', codecs.code_page_decode, self.CP_UTF8, b'\xff', 'strict', True) def check_decode(self, cp, tests): @@ -3382,8 +3505,7 @@ def check_encode(self, cp, tests): self.assertRaises(UnicodeEncodeError, codecs.code_page_encode, cp, text, errors) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_cp932(self): self.check_encode(932, ( ('abc', 'strict', b'abc'), @@ -3417,8 +3539,7 @@ def test_cp932(self): (b'\x81\x00abc', 'backslashreplace', '\\x81\x00abc'), )) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_cp1252(self): self.check_encode(1252, ( ('abc', 'strict', b'abc'), @@ -3437,8 +3558,7 @@ def test_cp1252(self): (b'\xff', 'strict', '\xff'), )) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_cp_utf7(self): cp = 65000 self.check_encode(cp, ( @@ -3459,8 +3579,7 @@ def test_cp_utf7(self): (b'[\xff]', 'strict', '[\xff]'), )) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_multibyte_encoding(self): self.check_decode(932, ( (b'\x84\xe9\x80', 'ignore', '\u9a3e'), @@ -3475,8 +3594,7 @@ def test_multibyte_encoding(self): ('[\U0010ffff\uDC80]', 'replace', b'[\xf4\x8f\xbf\xbf?]'), )) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_code_page_decode_flags(self): # Issue #36312: For some code pages (e.g. UTF-7) flags for # MultiByteToWideChar() must be set to 0. @@ -3496,8 +3614,7 @@ def test_code_page_decode_flags(self): self.assertEqual(codecs.code_page_decode(42, b'abc'), ('\uf061\uf062\uf063', 3)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_incremental(self): decoded = codecs.code_page_decode(932, b'\x82', 'strict', False) self.assertEqual(decoded, ('', 0)) @@ -3682,7 +3799,7 @@ def test_seeking_write(self): self.assertEqual(sr.readline(), b'1\n') self.assertEqual(sr.readline(), b'abc\n') self.assertEqual(sr.readline(), b'789\n') - + def test_copy(self): bio = io.BytesIO() codec = codecs.lookup('ascii') @@ -3693,7 +3810,7 @@ def test_copy(self): copy.copy(sr) with self.assertRaisesRegex(TypeError, 'StreamRecoder'): copy.deepcopy(sr) - + def test_pickle(self): q = Queue(b'') codec = codecs.lookup('ascii') @@ -3860,8 +3977,7 @@ def test_rot13_func(self): class CodecNameNormalizationTest(unittest.TestCase): """Test codec name normalization""" - # TODO: RUSTPYTHON, AssertionError: Tuples differ: (1, 2, 3, 4) != (None, None, None, None) - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: Tuples differ: (1, 2, 3, 4) != (None, None, None, None) def test_codecs_lookup(self): FOUND = (1, 2, 3, 4) NOT_FOUND = (None, None, None, None) diff --git a/Lib/test/test_codeop.py b/Lib/test/test_codeop.py index 1036b970cd..c62e3748e6 100644 --- a/Lib/test/test_codeop.py +++ b/Lib/test/test_codeop.py @@ -227,6 +227,9 @@ def test_incomplete(self): ai("(x for x in") ai("(x for x in (") + ai('a = f"""') + ai('a = \\') + def test_invalid(self): ai = self.assertInvalid ai("a b") @@ -300,12 +303,11 @@ def test_warning(self): warnings.simplefilter('error', SyntaxWarning) compile_command(r"'\e'", symbol='exec') - # TODO: RUSTPYTHON - #def test_incomplete_warning(self): - # with warnings.catch_warnings(record=True) as w: - # warnings.simplefilter('always') - # self.assertIncomplete("'\\e' + (") - # self.assertEqual(w, []) + def test_incomplete_warning(self): + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter('always') + self.assertIncomplete("'\\e' + (") + self.assertEqual(w, []) # TODO: RUSTPYTHON @unittest.expectedFailure diff --git a/Lib/test/test_collections.py b/Lib/test/test_collections.py index 901f596cc3..7b11601fc4 100644 --- a/Lib/test/test_collections.py +++ b/Lib/test/test_collections.py @@ -1,5 +1,6 @@ """Unit tests for collections.py.""" +import array import collections import copy import doctest @@ -261,8 +262,7 @@ def __contains__(self, key): d = c.new_child(b=20, c=30) self.assertEqual(d.maps, [{'b': 20, 'c': 30}, {'a': 1, 'b': 2}]) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_union_operators(self): cm1 = ChainMap(dict(a=1, b=2), dict(c=3, d=4)) cm2 = ChainMap(dict(a=10, e=5), dict(b=20, d=4)) @@ -469,6 +469,7 @@ def test_module_parameter(self): NT = namedtuple('NT', ['x', 'y'], module=collections) self.assertEqual(NT.__module__, collections) + @unittest.expectedFailure # TODO: RUSTPYTHON def test_instance(self): Point = namedtuple('Point', 'x y') p = Point(11, 22) @@ -490,12 +491,8 @@ def test_instance(self): self.assertEqual(p._replace(x=1), (1, 22)) # test _replace method self.assertEqual(p._asdict(), dict(x=11, y=22)) # test _asdict method - try: + with self.assertRaises(TypeError): p._replace(x=1, error=2) - except ValueError: - pass - else: - self._fail('Did not detect an incorrect fieldname') # verify that field string can have commas Point = namedtuple('Point', 'x, y') @@ -547,7 +544,9 @@ def test_odd_sizes(self): self.assertEqual(Dot(1)._replace(d=999), (999,)) self.assertEqual(Dot(1)._fields, ('d',)) - n = support.EXCEEDS_RECURSION_LIMIT + @support.requires_resource('cpu') + def test_large_size(self): + n = support.exceeds_recursion_limit() names = list(set(''.join([choice(string.ascii_letters) for j in range(10)]) for i in range(n))) n = len(names) @@ -739,7 +738,7 @@ def validate_abstract_methods(self, abc, *names): stubs = methodstubs.copy() del stubs[name] C = type('C', (abc,), stubs) - self.assertRaises(TypeError, C, name) + self.assertRaises(TypeError, C) def validate_isinstance(self, abc, name): stub = lambda s, *args: 0 @@ -789,8 +788,7 @@ def _test_gen(): class TestOneTrickPonyABCs(ABCTestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_Awaitable(self): def gen(): yield @@ -843,8 +841,7 @@ class CoroLike: pass CoroLike = None support.gc_collect() # Kill CoroLike to clean-up ABCMeta cache - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_Coroutine(self): def gen(): yield @@ -952,8 +949,6 @@ def __aiter__(self): self.validate_abstract_methods(AsyncIterable, '__aiter__') self.validate_isinstance(AsyncIterable, '__aiter__') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_AsyncIterator(self): class AI: def __aiter__(self): @@ -972,7 +967,7 @@ class AnextOnly: async def __anext__(self): raise StopAsyncIteration self.assertNotIsInstance(AnextOnly(), AsyncIterator) - self.validate_abstract_methods(AsyncIterator, '__anext__', '__aiter__') + self.validate_abstract_methods(AsyncIterator, '__anext__') def test_Iterable(self): # Check some non-iterables @@ -1152,8 +1147,7 @@ class NonCol(ColImpl): self.assertFalse(issubclass(NonCol, Collection)) self.assertFalse(isinstance(NonCol(), Collection)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + def test_Iterator(self): non_samples = [None, 42, 3.14, 1j, b"", "", (), [], {}, set()] for x in non_samples: @@ -1170,7 +1164,7 @@ def test_Iterator(self): for x in samples: self.assertIsInstance(x, Iterator) self.assertTrue(issubclass(type(x), Iterator), repr(type(x))) - self.validate_abstract_methods(Iterator, '__next__', '__iter__') + self.validate_abstract_methods(Iterator, '__next__') # Issue 10565 class NextOnly: @@ -1850,14 +1844,11 @@ def test_Set_hash_matches_frozenset(self): fs = frozenset(s) self.assertEqual(hash(fs), Set._hash(fs), msg=s) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_Mapping(self): for sample in [dict]: self.assertIsInstance(sample(), Mapping) self.assertTrue(issubclass(sample, Mapping)) - self.validate_abstract_methods(Mapping, '__contains__', '__iter__', '__len__', - '__getitem__') + self.validate_abstract_methods(Mapping, '__iter__', '__len__', '__getitem__') class MyMapping(Mapping): def __len__(self): return 0 @@ -1868,13 +1859,11 @@ def __iter__(self): self.validate_comparison(MyMapping()) self.assertRaises(TypeError, reversed, MyMapping()) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_MutableMapping(self): for sample in [dict]: self.assertIsInstance(sample(), MutableMapping) self.assertTrue(issubclass(sample, MutableMapping)) - self.validate_abstract_methods(MutableMapping, '__contains__', '__iter__', '__len__', + self.validate_abstract_methods(MutableMapping, '__iter__', '__len__', '__getitem__', '__setitem__', '__delitem__') def test_MutableMapping_subclass(self): @@ -1904,8 +1893,6 @@ def test_MutableMapping_subclass(self): mymap['blue'] = 7 # Shouldn't affect 'z' self.assertEqual(z, {('orange', 3), ('red', 5)}) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_Sequence(self): for sample in [tuple, list, bytes, str]: self.assertIsInstance(sample(), Sequence) @@ -1915,8 +1902,7 @@ def test_Sequence(self): self.assertIsInstance(memoryview(b""), Sequence) self.assertTrue(issubclass(memoryview, Sequence)) self.assertTrue(issubclass(str, Sequence)) - self.validate_abstract_methods(Sequence, '__contains__', '__iter__', '__len__', - '__getitem__') + self.validate_abstract_methods(Sequence, '__len__', '__getitem__') def test_Sequence_mixins(self): class SequenceSubclass(Sequence): @@ -1975,10 +1961,7 @@ class X(ByteString): pass # No metaclass conflict class Z(ByteString, Awaitable): pass - # TODO: RUSTPYTHON - # Need to implement __buffer__ and __release_buffer__ - # https://docs.python.org/3.13/reference/datamodel.html#emulating-buffer-types - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; Need to implement __buffer__ and __release_buffer__ (https://docs.python.org/3.13/reference/datamodel.html#emulating-buffer-types) def test_Buffer(self): for sample in [bytes, bytearray, memoryview]: self.assertIsInstance(sample(b"x"), Buffer) @@ -1988,8 +1971,6 @@ def test_Buffer(self): self.assertFalse(issubclass(sample, Buffer)) self.validate_abstract_methods(Buffer, '__buffer__') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_MutableSequence(self): for sample in [tuple, str, bytes]: self.assertNotIsInstance(sample(), MutableSequence) @@ -1997,9 +1978,10 @@ def test_MutableSequence(self): for sample in [list, bytearray, deque]: self.assertIsInstance(sample(), MutableSequence) self.assertTrue(issubclass(sample, MutableSequence)) + self.assertTrue(issubclass(array.array, MutableSequence)) self.assertFalse(issubclass(str, MutableSequence)) - self.validate_abstract_methods(MutableSequence, '__contains__', '__iter__', - '__len__', '__getitem__', '__setitem__', '__delitem__', 'insert') + self.validate_abstract_methods(MutableSequence, '__len__', '__getitem__', + '__setitem__', '__delitem__', 'insert') def test_MutableSequence_mixins(self): # Test the mixins of MutableSequence by creating a minimal concrete @@ -2051,8 +2033,7 @@ def insert(self, index, value): self.assertEqual(len(mss), len(mss2)) self.assertEqual(list(mss), list(mss2)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_illegal_patma_flags(self): with self.assertRaises(TypeError): class Both(Collection): diff --git a/Lib/test/test_compile.py b/Lib/test/test_compile.py index 51c834d798..3ea09354c3 100644 --- a/Lib/test/test_compile.py +++ b/Lib/test/test_compile.py @@ -138,7 +138,6 @@ def f(x): def test_argument_order(self): self.assertRaises(SyntaxError, exec, 'def f(a=1, b): pass') - @unittest.skip("TODO: RUSTPYTHON, thread 'main' panicked at 'called `Result::unwrap()` on an `Err` value: ParseFloatError { kind: Invalid }'") def test_float_literals(self): # testing bad float literals self.assertRaises(SyntaxError, eval, "2e") @@ -201,6 +200,8 @@ def test_literals_with_leading_zeroes(self): self.assertEqual(eval("0o777"), 511) self.assertEqual(eval("-0o0000010"), -8) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_int_literals_too_long(self): n = 3000 source = f"a = 1\nb = 2\nc = {'3'*n}\nd = 4" @@ -274,6 +275,8 @@ def test_none_assignment(self): self.assertRaises(SyntaxError, compile, stmt, 'tmp', 'single') self.assertRaises(SyntaxError, compile, stmt, 'tmp', 'exec') + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_import(self): succeed = [ 'import sys', @@ -821,6 +824,8 @@ def continue_in_while(): self.assertEqual(None, opcodes[1].argval) self.assertEqual('RETURN_VALUE', opcodes[2].opname) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_consts_in_conditionals(self): def and_true(x): return True and x @@ -844,6 +849,8 @@ def or_false(x): self.assertIn('LOAD_', opcodes[-2].opname) self.assertEqual('RETURN_VALUE', opcodes[-1].opname) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_imported_load_method(self): sources = [ """\ @@ -886,6 +893,8 @@ def call(): line1 = call.__code__.co_firstlineno + 1 assert line1 not in [line for (_, _, line) in call.__code__.co_lines()] + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_lineno_after_implicit_return(self): TRUE = True # Don't use constant True or False, as compiler will remove test @@ -920,6 +929,8 @@ def save_caller_frame(): func(save_caller_frame) self.assertEqual(frame.f_lineno-frame.f_code.co_firstlineno, lastline) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_lineno_after_no_code(self): def no_code1(): "doc string" @@ -944,6 +955,8 @@ def get_code_lines(self, code): last_line = line return res + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_lineno_attribute(self): def load_attr(): return ( @@ -988,6 +1001,8 @@ def aug_store_attr(): code_lines = self.get_code_lines(func.__code__) self.assertEqual(lines, code_lines) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_line_number_genexp(self): def return_genexp(): @@ -1002,6 +1017,8 @@ def return_genexp(): code_lines = self.get_code_lines(genexp_code) self.assertEqual(genexp_lines, code_lines) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_line_number_implicit_return_after_async_for(self): async def test(aseq): @@ -1022,6 +1039,8 @@ def test_big_dict_literal(self): the_dict = "{" + ",".join(f"{x}:{x}" for x in range(dict_size)) + "}" self.assertEqual(len(eval(the_dict)), dict_size) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_redundant_jump_in_if_else_break(self): # Check if bytecode containing jumps that simply point to the next line # is generated around if-else-break style structures. See bpo-42615. @@ -1061,6 +1080,8 @@ def while_not_chained(a, b, c): for instr in dis.Bytecode(while_not_chained): self.assertNotEqual(instr.opname, "EXTENDED_ARG") + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_compare_positions(self): for opname, op in [ ("COMPARE_OP", "<"), @@ -1361,64 +1382,66 @@ def check_stack_size(self, code): max_size = math.ceil(math.log(len(code.co_code))) self.assertLessEqual(code.co_stacksize, max_size) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_and(self): self.check_stack_size("x and " * self.N + "x") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_or(self): self.check_stack_size("x or " * self.N + "x") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_and_or(self): self.check_stack_size("x and x or " * self.N + "x") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_chained_comparison(self): self.check_stack_size("x < " * self.N + "x") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_if_else(self): self.check_stack_size("x if x else " * self.N + "x") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_binop(self): self.check_stack_size("x + " * self.N + "x") + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_list(self): self.check_stack_size("[" + "x, " * self.N + "x]") + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_tuple(self): self.check_stack_size("(" + "x, " * self.N + "x)") + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_set(self): self.check_stack_size("{" + "x, " * self.N + "x}") + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_dict(self): self.check_stack_size("{" + "x:x, " * self.N + "x:x}") + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_func_args(self): self.check_stack_size("f(" + "x, " * self.N + ")") + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_func_kwargs(self): kwargs = (f'a{i}=x' for i in range(self.N)) self.check_stack_size("f(" + ", ".join(kwargs) + ")") + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_meth_args(self): self.check_stack_size("o.m(" + "x, " * self.N + ")") + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_meth_kwargs(self): kwargs = (f'a{i}=x' for i in range(self.N)) self.check_stack_size("o.m(" + ", ".join(kwargs) + ")") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_func_and(self): code = "def f(x):\n" code += " x and x\n" * self.N diff --git a/Lib/test/test_compileall.py b/Lib/test/test_compileall.py index a490b8a1d5..9fa3dbc47e 100644 --- a/Lib/test/test_compileall.py +++ b/Lib/test/test_compileall.py @@ -325,8 +325,6 @@ def _test_ddir_only(self, *, ddir, parallel=True): self.assertEqual(mod_code_obj.co_filename, expected_in) self.assertIn(f'"{expected_in}"', os.fsdecode(err)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_ddir_only_one_worker(self): """Recursive compile_dir ddir= contains package paths; bpo39769.""" return self._test_ddir_only(ddir="", parallel=False) @@ -336,8 +334,6 @@ def test_ddir_multiple_workers(self): """Recursive compile_dir ddir= contains package paths; bpo39769.""" return self._test_ddir_only(ddir="", parallel=True) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_ddir_empty_only_one_worker(self): """Recursive compile_dir ddir='' contains package paths; bpo39769.""" return self._test_ddir_only(ddir="", parallel=False) @@ -347,8 +343,6 @@ def test_ddir_empty_multiple_workers(self): """Recursive compile_dir ddir='' contains package paths; bpo39769.""" return self._test_ddir_only(ddir="", parallel=True) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_strip_only(self): fullpath = ["test", "build", "real", "path"] path = os.path.join(self.directory, *fullpath) @@ -408,8 +402,6 @@ def test_prepend_only(self): str(err, encoding=sys.getdefaultencoding()) ) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_strip_and_prepend(self): fullpath = ["test", "build", "real", "path"] path = os.path.join(self.directory, *fullpath) @@ -459,7 +451,6 @@ def test_multiple_optimization_levels(self): except Exception: pass - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") @os_helper.skip_unless_symlink def test_ignore_symlink_destination(self): # Create folders for allowed files, symlinks and prohibited area @@ -888,8 +879,6 @@ def test_workers_available_cores(self, compile_dir): self.assertTrue(compile_dir.called) self.assertEqual(compile_dir.call_args[-1]['workers'], 0) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_strip_and_prepend(self): fullpath = ["test", "build", "real", "path"] path = os.path.join(self.directory, *fullpath) @@ -933,7 +922,6 @@ def test_multiple_optimization_levels(self): except Exception: pass - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") @os_helper.skip_unless_symlink def test_ignore_symlink_destination(self): # Create folders for allowed files, symlinks and prohibited area diff --git a/Lib/test/test_complex.py b/Lib/test/test_complex.py index dd3c4f281a..86d075de8c 100644 --- a/Lib/test/test_complex.py +++ b/Lib/test/test_complex.py @@ -1,15 +1,19 @@ import unittest import sys from test import support -from test.test_grammar import (VALID_UNDERSCORE_LITERALS, - INVALID_UNDERSCORE_LITERALS) +from test.support.testcase import ComplexesAreIdenticalMixin +from test.support.numbers import ( + VALID_UNDERSCORE_LITERALS, + INVALID_UNDERSCORE_LITERALS, +) from random import random -from math import atan2, isnan, copysign +from math import isnan, copysign import operator INF = float("inf") NAN = float("nan") +DBL_MAX = sys.float_info.max # These tests ensure that complex math does the right thing ZERO_DIVISION = ( @@ -20,7 +24,28 @@ (1, 0+0j), ) -class ComplexTest(unittest.TestCase): +class WithIndex: + def __init__(self, value): + self.value = value + def __index__(self): + return self.value + +class WithFloat: + def __init__(self, value): + self.value = value + def __float__(self): + return self.value + +class ComplexSubclass(complex): + pass + +class WithComplex: + def __init__(self, value): + self.value = value + def __complex__(self): + return self.value + +class ComplexTest(ComplexesAreIdenticalMixin, unittest.TestCase): def assertAlmostEqual(self, a, b): if isinstance(a, complex): @@ -49,29 +74,6 @@ def assertCloseAbs(self, x, y, eps=1e-9): # check that relative difference < eps self.assertTrue(abs((x-y)/y) < eps) - def assertFloatsAreIdentical(self, x, y): - """assert that floats x and y are identical, in the sense that: - (1) both x and y are nans, or - (2) both x and y are infinities, with the same sign, or - (3) both x and y are zeros, with the same sign, or - (4) x and y are both finite and nonzero, and x == y - - """ - msg = 'floats {!r} and {!r} are not identical' - - if isnan(x) or isnan(y): - if isnan(x) and isnan(y): - return - elif x == y: - if x != 0.0: - return - # both zero; check that signs match - elif copysign(1.0, x) == copysign(1.0, y): - return - else: - msg += ': zeros have different signs' - self.fail(msg.format(x, y)) - def assertClose(self, x, y, eps=1e-9): """Return true iff complexes x and y "are close".""" self.assertCloseAbs(x.real, y.real, eps) @@ -303,6 +305,11 @@ def test_pow(self): except OverflowError: pass + # gh-113841: possible undefined division by 0 in _Py_c_pow() + x, y = 9j, 33j**3 + with self.assertRaises(OverflowError): + x**y + def test_pow_with_small_integer_exponents(self): # Check that small integer exponents are handled identically # regardless of their type. @@ -340,138 +347,93 @@ def test_boolcontext(self): def test_conjugate(self): self.assertClose(complex(5.3, 9.8).conjugate(), 5.3-9.8j) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_constructor(self): - class NS: - def __init__(self, value): self.value = value - def __complex__(self): return self.value - self.assertEqual(complex(NS(1+10j)), 1+10j) - self.assertRaises(TypeError, complex, NS(None)) - self.assertRaises(TypeError, complex, {}) - self.assertRaises(TypeError, complex, NS(1.5)) - self.assertRaises(TypeError, complex, NS(1)) - self.assertRaises(TypeError, complex, object()) - self.assertRaises(TypeError, complex, NS(4.25+0.5j), object()) - - self.assertAlmostEqual(complex("1+10j"), 1+10j) - self.assertAlmostEqual(complex(10), 10+0j) - self.assertAlmostEqual(complex(10.0), 10+0j) - self.assertAlmostEqual(complex(10), 10+0j) - self.assertAlmostEqual(complex(10+0j), 10+0j) - self.assertAlmostEqual(complex(1,10), 1+10j) - self.assertAlmostEqual(complex(1,10), 1+10j) - self.assertAlmostEqual(complex(1,10.0), 1+10j) - self.assertAlmostEqual(complex(1,10), 1+10j) - self.assertAlmostEqual(complex(1,10), 1+10j) - self.assertAlmostEqual(complex(1,10.0), 1+10j) - self.assertAlmostEqual(complex(1.0,10), 1+10j) - self.assertAlmostEqual(complex(1.0,10), 1+10j) - self.assertAlmostEqual(complex(1.0,10.0), 1+10j) - self.assertAlmostEqual(complex(3.14+0j), 3.14+0j) - self.assertAlmostEqual(complex(3.14), 3.14+0j) - self.assertAlmostEqual(complex(314), 314.0+0j) - self.assertAlmostEqual(complex(314), 314.0+0j) - self.assertAlmostEqual(complex(3.14+0j, 0j), 3.14+0j) - self.assertAlmostEqual(complex(3.14, 0.0), 3.14+0j) - self.assertAlmostEqual(complex(314, 0), 314.0+0j) - self.assertAlmostEqual(complex(314, 0), 314.0+0j) - self.assertAlmostEqual(complex(0j, 3.14j), -3.14+0j) - self.assertAlmostEqual(complex(0.0, 3.14j), -3.14+0j) - self.assertAlmostEqual(complex(0j, 3.14), 3.14j) - self.assertAlmostEqual(complex(0.0, 3.14), 3.14j) - self.assertAlmostEqual(complex("1"), 1+0j) - self.assertAlmostEqual(complex("1j"), 1j) - self.assertAlmostEqual(complex(), 0) - self.assertAlmostEqual(complex("-1"), -1) - self.assertAlmostEqual(complex("+1"), +1) - self.assertAlmostEqual(complex("(1+2j)"), 1+2j) - self.assertAlmostEqual(complex("(1.3+2.2j)"), 1.3+2.2j) - self.assertAlmostEqual(complex("3.14+1J"), 3.14+1j) - self.assertAlmostEqual(complex(" ( +3.14-6J )"), 3.14-6j) - self.assertAlmostEqual(complex(" ( +3.14-J )"), 3.14-1j) - self.assertAlmostEqual(complex(" ( +3.14+j )"), 3.14+1j) - self.assertAlmostEqual(complex("J"), 1j) - self.assertAlmostEqual(complex("( j )"), 1j) - self.assertAlmostEqual(complex("+J"), 1j) - self.assertAlmostEqual(complex("( -j)"), -1j) - self.assertAlmostEqual(complex('1e-500'), 0.0 + 0.0j) - self.assertAlmostEqual(complex('-1e-500j'), 0.0 - 0.0j) - self.assertAlmostEqual(complex('-1e-500+1e-500j'), -0.0 + 0.0j) - self.assertEqual(complex('1-1j'), 1.0 - 1j) - self.assertEqual(complex('1J'), 1j) - - class complex2(complex): pass - self.assertAlmostEqual(complex(complex2(1+1j)), 1+1j) - self.assertAlmostEqual(complex(real=17, imag=23), 17+23j) - self.assertAlmostEqual(complex(real=17+23j), 17+23j) - self.assertAlmostEqual(complex(real=17+23j, imag=23), 17+46j) - self.assertAlmostEqual(complex(real=1+2j, imag=3+4j), -3+5j) + def check(z, x, y): + self.assertIs(type(z), complex) + self.assertFloatsAreIdentical(z.real, x) + self.assertFloatsAreIdentical(z.imag, y) + + check(complex(), 0.0, 0.0) + check(complex(10), 10.0, 0.0) + check(complex(4.25), 4.25, 0.0) + check(complex(4.25+0j), 4.25, 0.0) + check(complex(4.25+0.5j), 4.25, 0.5) + check(complex(ComplexSubclass(4.25+0.5j)), 4.25, 0.5) + check(complex(WithComplex(4.25+0.5j)), 4.25, 0.5) + + check(complex(1, 10), 1.0, 10.0) + check(complex(1, 10.0), 1.0, 10.0) + check(complex(1, 4.25), 1.0, 4.25) + check(complex(1.0, 10), 1.0, 10.0) + check(complex(4.25, 10), 4.25, 10.0) + check(complex(1.0, 10.0), 1.0, 10.0) + check(complex(4.25, 0.5), 4.25, 0.5) + + check(complex(4.25+0j, 0), 4.25, 0.0) + check(complex(ComplexSubclass(4.25+0j), 0), 4.25, 0.0) + check(complex(WithComplex(4.25+0j), 0), 4.25, 0.0) + check(complex(4.25j, 0), 0.0, 4.25) + check(complex(0j, 4.25), 0.0, 4.25) + check(complex(0, 4.25+0j), 0.0, 4.25) + check(complex(0, ComplexSubclass(4.25+0j)), 0.0, 4.25) + with self.assertRaisesRegex(TypeError, + "second argument must be a number, not 'WithComplex'"): + complex(0, WithComplex(4.25+0j)) + check(complex(0.0, 4.25j), -4.25, 0.0) + check(complex(4.25+0j, 0j), 4.25, 0.0) + check(complex(4.25j, 0j), 0.0, 4.25) + check(complex(0j, 4.25+0j), 0.0, 4.25) + check(complex(0j, 4.25j), -4.25, 0.0) + + check(complex(real=4.25), 4.25, 0.0) + check(complex(real=4.25+0j), 4.25, 0.0) + check(complex(real=4.25+1.5j), 4.25, 1.5) + check(complex(imag=1.5), 0.0, 1.5) + check(complex(real=4.25, imag=1.5), 4.25, 1.5) + check(complex(4.25, imag=1.5), 4.25, 1.5) # check that the sign of a zero in the real or imaginary part - # is preserved when constructing from two floats. (These checks - # are harmless on systems without support for signed zeros.) - def split_zeros(x): - """Function that produces different results for 0. and -0.""" - return atan2(x, -1.) - - self.assertEqual(split_zeros(complex(1., 0.).imag), split_zeros(0.)) - self.assertEqual(split_zeros(complex(1., -0.).imag), split_zeros(-0.)) - self.assertEqual(split_zeros(complex(0., 1.).real), split_zeros(0.)) - self.assertEqual(split_zeros(complex(-0., 1.).real), split_zeros(-0.)) - - c = 3.14 + 1j - self.assertTrue(complex(c) is c) - del c - - self.assertRaises(TypeError, complex, "1", "1") - self.assertRaises(TypeError, complex, 1, "1") - - # SF bug 543840: complex(string) accepts strings with \0 - # Fixed in 2.3. - self.assertRaises(ValueError, complex, '1+1j\0j') - - self.assertRaises(TypeError, int, 5+3j) - self.assertRaises(TypeError, int, 5+3j) - self.assertRaises(TypeError, float, 5+3j) - self.assertRaises(ValueError, complex, "") - self.assertRaises(TypeError, complex, None) - self.assertRaisesRegex(TypeError, "not 'NoneType'", complex, None) - self.assertRaises(ValueError, complex, "\0") - self.assertRaises(ValueError, complex, "3\09") - self.assertRaises(TypeError, complex, "1", "2") - self.assertRaises(TypeError, complex, "1", 42) - self.assertRaises(TypeError, complex, 1, "2") - self.assertRaises(ValueError, complex, "1+") - self.assertRaises(ValueError, complex, "1+1j+1j") - self.assertRaises(ValueError, complex, "--") - self.assertRaises(ValueError, complex, "(1+2j") - self.assertRaises(ValueError, complex, "1+2j)") - self.assertRaises(ValueError, complex, "1+(2j)") - self.assertRaises(ValueError, complex, "(1+2j)123") - self.assertRaises(ValueError, complex, "x") - self.assertRaises(ValueError, complex, "1j+2") - self.assertRaises(ValueError, complex, "1e1ej") - self.assertRaises(ValueError, complex, "1e++1ej") - self.assertRaises(ValueError, complex, ")1+2j(") - self.assertRaisesRegex( - TypeError, + # is preserved when constructing from two floats. + for x in 1.0, -1.0: + for y in 0.0, -0.0: + check(complex(x, y), x, y) + check(complex(y, x), y, x) + + c = complex(4.25, 1.5) + self.assertIs(complex(c), c) + c2 = ComplexSubclass(c) + self.assertEqual(c2, c) + self.assertIs(type(c2), ComplexSubclass) + del c, c2 + + self.assertRaisesRegex(TypeError, "first argument must be a string or a number, not 'dict'", - complex, {1:2}, 1) - self.assertRaisesRegex( - TypeError, + complex, {}) + self.assertRaisesRegex(TypeError, + "first argument must be a string or a number, not 'NoneType'", + complex, None) + self.assertRaisesRegex(TypeError, + "first argument must be a string or a number, not 'dict'", + complex, {1:2}, 0) + self.assertRaisesRegex(TypeError, + "can't take second arg if first is a string", + complex, '1', 0) + self.assertRaisesRegex(TypeError, "second argument must be a number, not 'dict'", - complex, 1, {1:2}) - # the following three are accepted by Python 2.6 - self.assertRaises(ValueError, complex, "1..1j") - self.assertRaises(ValueError, complex, "1.11.1j") - self.assertRaises(ValueError, complex, "1e1.1j") - - # check that complex accepts long unicode strings - self.assertEqual(type(complex("1"*500)), complex) - # check whitespace processing - self.assertEqual(complex('\N{EM SPACE}(\N{EN SPACE}1+1j ) '), 1+1j) - # Invalid unicode string - # See bpo-34087 - self.assertRaises(ValueError, complex, '\u3053\u3093\u306b\u3061\u306f') + complex, 0, {1:2}) + self.assertRaisesRegex(TypeError, + "second arg can't be a string", + complex, 0, '1') + + self.assertRaises(TypeError, complex, WithComplex(1.5)) + self.assertRaises(TypeError, complex, WithComplex(1)) + self.assertRaises(TypeError, complex, WithComplex(None)) + self.assertRaises(TypeError, complex, WithComplex(4.25+0j), object()) + self.assertRaises(TypeError, complex, WithComplex(1.5), object()) + self.assertRaises(TypeError, complex, WithComplex(1), object()) + self.assertRaises(TypeError, complex, WithComplex(None), object()) class EvilExc(Exception): pass @@ -482,33 +444,33 @@ def __complex__(self): self.assertRaises(EvilExc, complex, evilcomplex()) - class float2: - def __init__(self, value): - self.value = value - def __float__(self): - return self.value - - self.assertAlmostEqual(complex(float2(42.)), 42) - self.assertAlmostEqual(complex(real=float2(17.), imag=float2(23.)), 17+23j) - self.assertRaises(TypeError, complex, float2(None)) - - class MyIndex: - def __init__(self, value): - self.value = value - def __index__(self): - return self.value - - self.assertAlmostEqual(complex(MyIndex(42)), 42.0+0.0j) - self.assertAlmostEqual(complex(123, MyIndex(42)), 123.0+42.0j) - self.assertRaises(OverflowError, complex, MyIndex(2**2000)) - self.assertRaises(OverflowError, complex, 123, MyIndex(2**2000)) + check(complex(WithFloat(4.25)), 4.25, 0.0) + check(complex(WithFloat(4.25), 1.5), 4.25, 1.5) + check(complex(1.5, WithFloat(4.25)), 1.5, 4.25) + self.assertRaises(TypeError, complex, WithFloat(42)) + self.assertRaises(TypeError, complex, WithFloat(42), 1.5) + self.assertRaises(TypeError, complex, 1.5, WithFloat(42)) + self.assertRaises(TypeError, complex, WithFloat(None)) + self.assertRaises(TypeError, complex, WithFloat(None), 1.5) + self.assertRaises(TypeError, complex, 1.5, WithFloat(None)) + + check(complex(WithIndex(42)), 42.0, 0.0) + check(complex(WithIndex(42), 1.5), 42.0, 1.5) + check(complex(1.5, WithIndex(42)), 1.5, 42.0) + self.assertRaises(OverflowError, complex, WithIndex(2**2000)) + self.assertRaises(OverflowError, complex, WithIndex(2**2000), 1.5) + self.assertRaises(OverflowError, complex, 1.5, WithIndex(2**2000)) + self.assertRaises(TypeError, complex, WithIndex(None)) + self.assertRaises(TypeError, complex, WithIndex(None), 1.5) + self.assertRaises(TypeError, complex, 1.5, WithIndex(None)) class MyInt: def __int__(self): return 42 self.assertRaises(TypeError, complex, MyInt()) - self.assertRaises(TypeError, complex, 123, MyInt()) + self.assertRaises(TypeError, complex, MyInt(), 1.5) + self.assertRaises(TypeError, complex, 1.5, MyInt()) class complex0(complex): """Test usage of __complex__() when inheriting from 'complex'""" @@ -528,9 +490,9 @@ class complex2(complex): def __complex__(self): return None - self.assertEqual(complex(complex0(1j)), 42j) + check(complex(complex0(1j)), 0.0, 42.0) with self.assertWarns(DeprecationWarning): - self.assertEqual(complex(complex1(1j)), 2j) + check(complex(complex1(1j)), 0.0, 2.0) self.assertRaises(TypeError, complex, complex2(1j)) def test___complex__(self): @@ -538,36 +500,93 @@ def test___complex__(self): self.assertEqual(z.__complex__(), z) self.assertEqual(type(z.__complex__()), complex) - class complex_subclass(complex): - pass - - z = complex_subclass(3 + 4j) + z = ComplexSubclass(3 + 4j) self.assertEqual(z.__complex__(), 3 + 4j) self.assertEqual(type(z.__complex__()), complex) @support.requires_IEEE_754 def test_constructor_special_numbers(self): - class complex2(complex): - pass for x in 0.0, -0.0, INF, -INF, NAN: for y in 0.0, -0.0, INF, -INF, NAN: with self.subTest(x=x, y=y): z = complex(x, y) self.assertFloatsAreIdentical(z.real, x) self.assertFloatsAreIdentical(z.imag, y) - z = complex2(x, y) - self.assertIs(type(z), complex2) + z = ComplexSubclass(x, y) + self.assertIs(type(z), ComplexSubclass) self.assertFloatsAreIdentical(z.real, x) self.assertFloatsAreIdentical(z.imag, y) - z = complex(complex2(x, y)) + z = complex(ComplexSubclass(x, y)) self.assertIs(type(z), complex) self.assertFloatsAreIdentical(z.real, x) self.assertFloatsAreIdentical(z.imag, y) - z = complex2(complex(x, y)) - self.assertIs(type(z), complex2) + z = ComplexSubclass(complex(x, y)) + self.assertIs(type(z), ComplexSubclass) self.assertFloatsAreIdentical(z.real, x) self.assertFloatsAreIdentical(z.imag, y) + def test_constructor_from_string(self): + def check(z, x, y): + self.assertIs(type(z), complex) + self.assertFloatsAreIdentical(z.real, x) + self.assertFloatsAreIdentical(z.imag, y) + + check(complex("1"), 1.0, 0.0) + check(complex("1j"), 0.0, 1.0) + check(complex("-1"), -1.0, 0.0) + check(complex("+1"), 1.0, 0.0) + check(complex("1+2j"), 1.0, 2.0) + check(complex("(1+2j)"), 1.0, 2.0) + check(complex("(1.5+4.25j)"), 1.5, 4.25) + check(complex("4.25+1J"), 4.25, 1.0) + check(complex(" ( +4.25-6J )"), 4.25, -6.0) + check(complex(" ( +4.25-J )"), 4.25, -1.0) + check(complex(" ( +4.25+j )"), 4.25, 1.0) + check(complex("J"), 0.0, 1.0) + check(complex("( j )"), 0.0, 1.0) + check(complex("+J"), 0.0, 1.0) + check(complex("( -j)"), 0.0, -1.0) + check(complex('1-1j'), 1.0, -1.0) + check(complex('1J'), 0.0, 1.0) + + check(complex('1e-500'), 0.0, 0.0) + check(complex('-1e-500j'), 0.0, -0.0) + check(complex('1e-500+1e-500j'), 0.0, 0.0) + check(complex('-1e-500+1e-500j'), -0.0, 0.0) + check(complex('1e-500-1e-500j'), 0.0, -0.0) + check(complex('-1e-500-1e-500j'), -0.0, -0.0) + + # SF bug 543840: complex(string) accepts strings with \0 + # Fixed in 2.3. + self.assertRaises(ValueError, complex, '1+1j\0j') + self.assertRaises(ValueError, complex, "") + self.assertRaises(ValueError, complex, "\0") + self.assertRaises(ValueError, complex, "3\09") + self.assertRaises(ValueError, complex, "1+") + self.assertRaises(ValueError, complex, "1+1j+1j") + self.assertRaises(ValueError, complex, "--") + self.assertRaises(ValueError, complex, "(1+2j") + self.assertRaises(ValueError, complex, "1+2j)") + self.assertRaises(ValueError, complex, "1+(2j)") + self.assertRaises(ValueError, complex, "(1+2j)123") + self.assertRaises(ValueError, complex, "x") + self.assertRaises(ValueError, complex, "1j+2") + self.assertRaises(ValueError, complex, "1e1ej") + self.assertRaises(ValueError, complex, "1e++1ej") + self.assertRaises(ValueError, complex, ")1+2j(") + # the following three are accepted by Python 2.6 + self.assertRaises(ValueError, complex, "1..1j") + self.assertRaises(ValueError, complex, "1.11.1j") + self.assertRaises(ValueError, complex, "1e1.1j") + + # check that complex accepts long unicode strings + self.assertIs(type(complex("1"*500)), complex) + # check whitespace processing + self.assertEqual(complex('\N{EM SPACE}(\N{EN SPACE}1+1j ) '), 1+1j) + # Invalid unicode string + # See bpo-34087 + self.assertRaises(ValueError, complex, '\u3053\u3093\u306b\u3061\u306f') + def test_constructor_negative_nans_from_string(self): self.assertEqual(copysign(1., complex("-nan").real), -1.) self.assertEqual(copysign(1., complex("-nanj").imag), -1.) @@ -589,7 +608,7 @@ def test_underscores(self): def test_hash(self): for x in range(-30, 30): self.assertEqual(hash(x), hash(complex(x, 0))) - x /= 3.0 # now check against floating point + x /= 3.0 # now check against floating-point self.assertEqual(hash(x), hash(complex(x, 0.))) self.assertNotEqual(hash(2000005 - 1j), -1) @@ -599,6 +618,8 @@ def test_abs(self): for num in nums: self.assertAlmostEqual((num.real**2 + num.imag**2) ** 0.5, abs(num)) + self.assertRaises(OverflowError, abs, complex(DBL_MAX, DBL_MAX)) + def test_repr_str(self): def test(v, expected, test_fn=self.assertEqual): test_fn(repr(v), expected) @@ -644,9 +665,6 @@ def test(v, expected, test_fn=self.assertEqual): test(complex(-0., -0.), "(-0-0j)") def test_pos(self): - class ComplexSubclass(complex): - pass - self.assertEqual(+(1+6j), 1+6j) self.assertEqual(+ComplexSubclass(1, 6), 1+6j) self.assertIs(type(+ComplexSubclass(1, 6)), complex) @@ -666,8 +684,8 @@ def test_getnewargs(self): def test_plus_minus_0j(self): # test that -0j and 0j literals are not identified z1, z2 = 0j, -0j - self.assertEqual(atan2(z1.imag, -1.), atan2(0., -1.)) - self.assertEqual(atan2(z2.imag, -1.), atan2(-0., -1.)) + self.assertFloatsAreIdentical(z1.imag, 0.0) + self.assertFloatsAreIdentical(z2.imag, -0.0) @support.requires_IEEE_754 def test_negated_imaginary_literal(self): @@ -702,8 +720,7 @@ def test_repr_roundtrip(self): for y in vals: z = complex(x, y) roundtrip = complex(repr(z)) - self.assertFloatsAreIdentical(z.real, roundtrip.real) - self.assertFloatsAreIdentical(z.imag, roundtrip.imag) + self.assertComplexesAreIdentical(z, roundtrip) # if we predefine some constants, then eval(repr(z)) should # also work, except that it might change the sign of zeros diff --git a/Lib/test/test_configparser.py b/Lib/test/test_configparser.py index 01e8e6c675..d793cc5890 100644 --- a/Lib/test/test_configparser.py +++ b/Lib/test/test_configparser.py @@ -2,10 +2,8 @@ import configparser import io import os -import pathlib import textwrap import unittest -import warnings from test import support from test.support import os_helper @@ -545,7 +543,7 @@ def test_parse_errors(self): "[Foo]\n wrong-indent\n") self.assertEqual(e.args, ('',)) # read_file on a real file - tricky = support.findfile("cfgparser.3") + tricky = support.findfile("cfgparser.3", subdir="configdata") if self.delimiters[0] == '=': error = configparser.ParsingError expected = (tricky,) @@ -648,6 +646,21 @@ def test_weird_errors(self): "'opt' in section 'Bar' already exists") self.assertEqual(e.args, ("Bar", "opt", "", None)) + def test_get_after_duplicate_option_error(self): + cf = self.newconfig() + ini = textwrap.dedent("""\ + [Foo] + x{equals}1 + y{equals}2 + y{equals}3 + """.format(equals=self.delimiters[0])) + if self.strict: + with self.assertRaises(configparser.DuplicateOptionError): + cf.read_string(ini) + else: + cf.read_string(ini) + self.assertEqual(cf.get('Foo', 'x'), '1') + def test_write(self): config_string = ( "[Long Line]\n" @@ -719,7 +732,7 @@ class mystr(str): def test_read_returns_file_list(self): if self.delimiters[0] != '=': self.skipTest('incompatible format') - file1 = support.findfile("cfgparser.1") + file1 = support.findfile("cfgparser.1", subdir="configdata") # check when we pass a mix of readable and non-readable files: cf = self.newconfig() parsed_files = cf.read([file1, "nonexistent-file"], encoding="utf-8") @@ -732,12 +745,12 @@ def test_read_returns_file_list(self): self.assertEqual(cf.get("Foo Bar", "foo"), "newbar") # check when we pass only a Path object: cf = self.newconfig() - parsed_files = cf.read(pathlib.Path(file1), encoding="utf-8") + parsed_files = cf.read(os_helper.FakePath(file1), encoding="utf-8") self.assertEqual(parsed_files, [file1]) self.assertEqual(cf.get("Foo Bar", "foo"), "newbar") # check when we passed both a filename and a Path object: cf = self.newconfig() - parsed_files = cf.read([pathlib.Path(file1), file1], encoding="utf-8") + parsed_files = cf.read([os_helper.FakePath(file1), file1], encoding="utf-8") self.assertEqual(parsed_files, [file1, file1]) self.assertEqual(cf.get("Foo Bar", "foo"), "newbar") # check when we pass only missing files: @@ -753,7 +766,7 @@ def test_read_returns_file_list(self): def test_read_returns_file_list_with_bytestring_path(self): if self.delimiters[0] != '=': self.skipTest('incompatible format') - file1_bytestring = support.findfile("cfgparser.1").encode() + file1_bytestring = support.findfile("cfgparser.1", subdir="configdata").encode() # check when passing an existing bytestring path cf = self.newconfig() parsed_files = cf.read(file1_bytestring, encoding="utf-8") @@ -909,9 +922,6 @@ def test_interpolation(self): if self.interpolation == configparser._UNSET: self.assertEqual(e.args, ("bar11", "Foo", "something %(with11)s lots of interpolation (11 steps)")) - elif isinstance(self.interpolation, configparser.LegacyInterpolation): - self.assertEqual(e.args, ("bar11", "Foo", - "something %(with11)s lots of interpolation (11 steps)")) def test_interpolation_missing_value(self): cf = self.get_interpolation_config() @@ -923,9 +933,6 @@ def test_interpolation_missing_value(self): if self.interpolation == configparser._UNSET: self.assertEqual(e.args, ('name', 'Interpolation Error', '%(reference)s', 'reference')) - elif isinstance(self.interpolation, configparser.LegacyInterpolation): - self.assertEqual(e.args, ('name', 'Interpolation Error', - '%(reference)s', 'reference')) def test_items(self): self.check_items_config([('default', ''), @@ -944,9 +951,6 @@ def test_safe_interpolation(self): self.assertEqual(cf.get("section", "ok"), "xxx/%s") if self.interpolation == configparser._UNSET: self.assertEqual(cf.get("section", "not_ok"), "xxx/xxx/%s") - elif isinstance(self.interpolation, configparser.LegacyInterpolation): - with self.assertRaises(TypeError): - cf.get("section", "not_ok") def test_set_malformatted_interpolation(self): cf = self.fromstring("[sect]\n" @@ -1027,31 +1031,6 @@ class CustomConfigParser(configparser.ConfigParser): cf.read_string(self.ini) self.assertMatchesIni(cf) - -class ConfigParserTestCaseLegacyInterpolation(ConfigParserTestCase): - config_class = configparser.ConfigParser - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - interpolation = configparser.LegacyInterpolation() - - def test_set_malformatted_interpolation(self): - cf = self.fromstring("[sect]\n" - "option1{eq}foo\n".format(eq=self.delimiters[0])) - - self.assertEqual(cf.get('sect', "option1"), "foo") - - cf.set("sect", "option1", "%foo") - self.assertEqual(cf.get('sect', "option1"), "%foo") - cf.set("sect", "option1", "foo%") - self.assertEqual(cf.get('sect', "option1"), "foo%") - cf.set("sect", "option1", "f%oo") - self.assertEqual(cf.get('sect', "option1"), "f%oo") - - # bug #5741: double percents are *not* malformed - cf.set("sect", "option2", "foo%%bar") - self.assertEqual(cf.get("sect", "option2"), "foo%%bar") - - class ConfigParserTestCaseInvalidInterpolationType(unittest.TestCase): def test_error_on_wrong_type_for_interpolation(self): for value in [configparser.ExtendedInterpolation, 42, "a string"]: @@ -1163,7 +1142,7 @@ class RawConfigParserTestSambaConf(CfgParserTestCaseClass, unittest.TestCase): empty_lines_in_values = False def test_reading(self): - smbconf = support.findfile("cfgparser.2") + smbconf = support.findfile("cfgparser.2", subdir="configdata") # check when we pass a mix of readable and non-readable files: cf = self.newconfig() parsed_files = cf.read([smbconf, "nonexistent-file"], encoding='utf-8') @@ -1351,6 +1330,47 @@ class ConfigParserTestCaseNoValue(ConfigParserTestCase): allow_no_value = True +class NoValueAndExtendedInterpolation(CfgParserTestCaseClass): + interpolation = configparser.ExtendedInterpolation() + allow_no_value = True + + def test_interpolation_with_allow_no_value(self): + config = textwrap.dedent(""" + [dummy] + a + b = ${a} + """) + cf = self.fromstring(config) + + self.assertIs(cf["dummy"]["a"], None) + self.assertEqual(cf["dummy"]["b"], "") + + def test_explicit_none(self): + config = textwrap.dedent(""" + [dummy] + a = None + b = ${a} + """) + cf = self.fromstring(config) + + self.assertEqual(cf["dummy"]["a"], "None") + self.assertEqual(cf["dummy"]["b"], "None") + + +class ConfigParserNoValueAndExtendedInterpolationTest( + NoValueAndExtendedInterpolation, + unittest.TestCase, +): + config_class = configparser.ConfigParser + + +class RawConfigParserNoValueAndExtendedInterpolationTest( + NoValueAndExtendedInterpolation, + unittest.TestCase, +): + config_class = configparser.RawConfigParser + + class ConfigParserTestCaseTrickyFile(CfgParserTestCaseClass, unittest.TestCase): config_class = configparser.ConfigParser delimiters = {'='} @@ -1358,7 +1378,7 @@ class ConfigParserTestCaseTrickyFile(CfgParserTestCaseClass, unittest.TestCase): allow_no_value = True def test_cfgparser_dot_3(self): - tricky = support.findfile("cfgparser.3") + tricky = support.findfile("cfgparser.3", subdir="configdata") cf = self.newconfig() self.assertEqual(len(cf.read(tricky, encoding='utf-8')), 1) self.assertEqual(cf.sections(), ['strange', @@ -1390,7 +1410,7 @@ def test_cfgparser_dot_3(self): self.assertEqual(cf.get('more interpolation', 'lets'), 'go shopping') def test_unicode_failure(self): - tricky = support.findfile("cfgparser.3") + tricky = support.findfile("cfgparser.3", subdir="configdata") cf = self.newconfig() with self.assertRaises(UnicodeDecodeError): cf.read(tricky, encoding='ascii') @@ -1491,7 +1511,7 @@ def fromstring(self, string, defaults=None): class FakeFile: def __init__(self): - file_path = support.findfile("cfgparser.1") + file_path = support.findfile("cfgparser.1", subdir="configdata") with open(file_path, encoding="utf-8") as f: self.lines = f.readlines() self.lines.reverse() @@ -1512,7 +1532,7 @@ def readline_generator(f): class ReadFileTestCase(unittest.TestCase): def test_file(self): - file_paths = [support.findfile("cfgparser.1")] + file_paths = [support.findfile("cfgparser.1", subdir="configdata")] try: file_paths.append(file_paths[0].encode('utf8')) except UnicodeEncodeError: @@ -1592,6 +1612,30 @@ def test_source_as_bytes(self): "'[badbad'" ) + def test_keys_without_value_with_extra_whitespace(self): + lines = [ + '[SECT]\n', + 'KEY1\n', + ' KEY2 = VAL2\n', # note the Space before the key! + ] + parser = configparser.ConfigParser( + comment_prefixes="", + allow_no_value=True, + strict=False, + delimiters=('=',), + interpolation=None, + ) + with self.assertRaises(configparser.MultilineContinuationError) as dse: + parser.read_file(lines) + self.assertEqual( + str(dse.exception), + "Key without value continued with an indented line.\n" + "file: '', line: 3\n" + "' KEY2 = VAL2\\n'" + ) + + + class CoverageOneHundredTestCase(unittest.TestCase): """Covers edge cases in the codebase.""" @@ -1638,14 +1682,6 @@ def test_interpolation_validation(self): self.assertEqual(str(cm.exception), "bad interpolation variable " "reference '%(()'") - def test_legacyinterpolation_deprecation(self): - with warnings.catch_warnings(record=True) as w: - warnings.simplefilter("always", DeprecationWarning) - configparser.LegacyInterpolation() - self.assertGreaterEqual(len(w), 1) - for warning in w: - self.assertIs(warning.category, DeprecationWarning) - def test_sectionproxy_repr(self): parser = configparser.ConfigParser() parser.read_string(""" @@ -2121,6 +2157,63 @@ def test_instance_assignment(self): self.assertEqual(cfg['two'].getlen('one'), 5) +class SectionlessTestCase(unittest.TestCase): + + def fromstring(self, string): + cfg = configparser.ConfigParser(allow_unnamed_section=True) + cfg.read_string(string) + return cfg + + def test_no_first_section(self): + cfg1 = self.fromstring(""" + a = 1 + b = 2 + [sect1] + c = 3 + """) + + self.assertEqual(set([configparser.UNNAMED_SECTION, 'sect1']), set(cfg1.sections())) + self.assertEqual('1', cfg1[configparser.UNNAMED_SECTION]['a']) + self.assertEqual('2', cfg1[configparser.UNNAMED_SECTION]['b']) + self.assertEqual('3', cfg1['sect1']['c']) + + output = io.StringIO() + cfg1.write(output) + cfg2 = self.fromstring(output.getvalue()) + + #self.assertEqual(set([configparser.UNNAMED_SECTION, 'sect1']), set(cfg2.sections())) + self.assertEqual('1', cfg2[configparser.UNNAMED_SECTION]['a']) + self.assertEqual('2', cfg2[configparser.UNNAMED_SECTION]['b']) + self.assertEqual('3', cfg2['sect1']['c']) + + def test_no_section(self): + cfg1 = self.fromstring(""" + a = 1 + b = 2 + """) + + self.assertEqual([configparser.UNNAMED_SECTION], cfg1.sections()) + self.assertEqual('1', cfg1[configparser.UNNAMED_SECTION]['a']) + self.assertEqual('2', cfg1[configparser.UNNAMED_SECTION]['b']) + + output = io.StringIO() + cfg1.write(output) + cfg2 = self.fromstring(output.getvalue()) + + self.assertEqual([configparser.UNNAMED_SECTION], cfg2.sections()) + self.assertEqual('1', cfg2[configparser.UNNAMED_SECTION]['a']) + self.assertEqual('2', cfg2[configparser.UNNAMED_SECTION]['b']) + + def test_multiple_configs(self): + cfg = configparser.ConfigParser(allow_unnamed_section=True) + cfg.read_string('a = 1') + cfg.read_string('b = 2') + + self.assertEqual([configparser.UNNAMED_SECTION], cfg.sections()) + self.assertEqual('1', cfg[configparser.UNNAMED_SECTION]['a']) + self.assertEqual('2', cfg[configparser.UNNAMED_SECTION]['b']) + + class MiscTestCase(unittest.TestCase): def test__all__(self): support.check__all__(self, configparser, not_exported={"Error"}) diff --git a/Lib/test/test_contains.py b/Lib/test/test_contains.py index c533311572..471d04a76c 100644 --- a/Lib/test/test_contains.py +++ b/Lib/test/test_contains.py @@ -36,7 +36,6 @@ def test_common_tests(self): self.assertRaises(TypeError, lambda: None in 'abc') - @unittest.skip("TODO: RUSTPYTHON, hangs") def test_builtin_sequence_types(self): # a collection of tests on builtin sequence types a = range(10) diff --git a/Lib/test/test_contextlib.py b/Lib/test/test_contextlib.py index 91764696ba..9bb3fd0179 100644 --- a/Lib/test/test_contextlib.py +++ b/Lib/test/test_contextlib.py @@ -24,6 +24,16 @@ def __exit__(self, *args): manager = DefaultEnter() self.assertIs(manager.__enter__(), manager) + def test_slots(self): + class DefaultContextManager(AbstractContextManager): + __slots__ = () + + def __exit__(self, *args): + super().__exit__(*args) + + with self.assertRaises(AttributeError): + DefaultContextManager().var = 42 + def test_exit_is_abstract(self): class MissingExit(AbstractContextManager): pass @@ -194,6 +204,7 @@ def whoo(): yield except RuntimeError: raise SyntaxError + ctx = whoo() ctx.__enter__() with self.assertRaises(SyntaxError): @@ -285,9 +296,11 @@ def woohoo(): yield except Exception as exc: raise RuntimeError(f'caught {exc}') from exc + with self.assertRaises(RuntimeError): with woohoo(): 1 / 0 + # If the context manager wrapped StopIteration in a RuntimeError, # we also unwrap it, because we can't tell whether the wrapping was # done by the generator machinery or by the generator itself. @@ -1143,7 +1156,7 @@ def first(): class TestExitStack(TestBaseExitStack, unittest.TestCase): exit_stack = ExitStack callback_error_internal_frames = [ - ('__exit__', 'raise exc_details[1]'), + ('__exit__', 'raise exc'), ('__exit__', 'if cb(*exc_details):'), ] @@ -1294,7 +1307,6 @@ def test_exception_groups(self): [KeyError("ke1"), KeyError("ke2")], ), ) - # Check handling of BaseExceptionGroup, using GeneratorExit so that # we don't accidentally discard a ctrl-c with KeyboardInterrupt. with suppress(GeneratorExit): @@ -1322,8 +1334,6 @@ def make_relative_path(self, *parts): *parts, ) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_simple(self): old_cwd = os.getcwd() target = self.make_relative_path('data') @@ -1333,12 +1343,10 @@ def test_simple(self): self.assertEqual(os.getcwd(), target) self.assertEqual(os.getcwd(), old_cwd) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_reentrant(self): old_cwd = os.getcwd() target1 = self.make_relative_path('data') - target2 = self.make_relative_path('ziptestdata') + target2 = self.make_relative_path('archivetestdata') self.assertNotIn(old_cwd, (target1, target2)) chdir1, chdir2 = chdir(target1), chdir(target2) @@ -1352,8 +1360,6 @@ def test_reentrant(self): self.assertEqual(os.getcwd(), target1) self.assertEqual(os.getcwd(), old_cwd) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_exception(self): old_cwd = os.getcwd() target = self.make_relative_path('data') diff --git a/Lib/test/test_contextlib_async.py b/Lib/test/test_contextlib_async.py new file mode 100644 index 0000000000..d7331c4d43 --- /dev/null +++ b/Lib/test/test_contextlib_async.py @@ -0,0 +1,746 @@ +import asyncio +from contextlib import ( + asynccontextmanager, AbstractAsyncContextManager, + AsyncExitStack, nullcontext, aclosing, contextmanager) +from test import support +import unittest +import traceback + +from test.test_contextlib import TestBaseExitStack + +support.requires_working_socket(module=True) + +def tearDownModule(): + asyncio.set_event_loop_policy(None) + + +class TestAbstractAsyncContextManager(unittest.IsolatedAsyncioTestCase): + + async def test_enter(self): + class DefaultEnter(AbstractAsyncContextManager): + async def __aexit__(self, *args): + await super().__aexit__(*args) + + manager = DefaultEnter() + self.assertIs(await manager.__aenter__(), manager) + + async with manager as context: + self.assertIs(manager, context) + + async def test_slots(self): + class DefaultAsyncContextManager(AbstractAsyncContextManager): + __slots__ = () + + async def __aexit__(self, *args): + await super().__aexit__(*args) + + with self.assertRaises(AttributeError): + manager = DefaultAsyncContextManager() + manager.var = 42 + + async def test_async_gen_propagates_generator_exit(self): + # A regression test for https://bugs.python.org/issue33786. + + @asynccontextmanager + async def ctx(): + yield + + async def gen(): + async with ctx(): + yield 11 + + g = gen() + async for val in g: + self.assertEqual(val, 11) + break + await g.aclose() + + def test_exit_is_abstract(self): + class MissingAexit(AbstractAsyncContextManager): + pass + + with self.assertRaises(TypeError): + MissingAexit() + + def test_structural_subclassing(self): + class ManagerFromScratch: + async def __aenter__(self): + return self + async def __aexit__(self, exc_type, exc_value, traceback): + return None + + self.assertTrue(issubclass(ManagerFromScratch, AbstractAsyncContextManager)) + + class DefaultEnter(AbstractAsyncContextManager): + async def __aexit__(self, *args): + await super().__aexit__(*args) + + self.assertTrue(issubclass(DefaultEnter, AbstractAsyncContextManager)) + + class NoneAenter(ManagerFromScratch): + __aenter__ = None + + self.assertFalse(issubclass(NoneAenter, AbstractAsyncContextManager)) + + class NoneAexit(ManagerFromScratch): + __aexit__ = None + + self.assertFalse(issubclass(NoneAexit, AbstractAsyncContextManager)) + + +class AsyncContextManagerTestCase(unittest.IsolatedAsyncioTestCase): + + async def test_contextmanager_plain(self): + state = [] + @asynccontextmanager + async def woohoo(): + state.append(1) + yield 42 + state.append(999) + async with woohoo() as x: + self.assertEqual(state, [1]) + self.assertEqual(x, 42) + state.append(x) + self.assertEqual(state, [1, 42, 999]) + + async def test_contextmanager_finally(self): + state = [] + @asynccontextmanager + async def woohoo(): + state.append(1) + try: + yield 42 + finally: + state.append(999) + with self.assertRaises(ZeroDivisionError): + async with woohoo() as x: + self.assertEqual(state, [1]) + self.assertEqual(x, 42) + state.append(x) + raise ZeroDivisionError() + self.assertEqual(state, [1, 42, 999]) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + async def test_contextmanager_traceback(self): + @asynccontextmanager + async def f(): + yield + + try: + async with f(): + 1/0 + except ZeroDivisionError as e: + frames = traceback.extract_tb(e.__traceback__) + + self.assertEqual(len(frames), 1) + self.assertEqual(frames[0].name, 'test_contextmanager_traceback') + self.assertEqual(frames[0].line, '1/0') + + # Repeat with RuntimeError (which goes through a different code path) + class RuntimeErrorSubclass(RuntimeError): + pass + + try: + async with f(): + raise RuntimeErrorSubclass(42) + except RuntimeErrorSubclass as e: + frames = traceback.extract_tb(e.__traceback__) + + self.assertEqual(len(frames), 1) + self.assertEqual(frames[0].name, 'test_contextmanager_traceback') + self.assertEqual(frames[0].line, 'raise RuntimeErrorSubclass(42)') + + class StopIterationSubclass(StopIteration): + pass + + class StopAsyncIterationSubclass(StopAsyncIteration): + pass + + for stop_exc in ( + StopIteration('spam'), + StopAsyncIteration('ham'), + StopIterationSubclass('spam'), + StopAsyncIterationSubclass('spam') + ): + with self.subTest(type=type(stop_exc)): + try: + async with f(): + raise stop_exc + except type(stop_exc) as e: + self.assertIs(e, stop_exc) + frames = traceback.extract_tb(e.__traceback__) + else: + self.fail(f'{stop_exc} was suppressed') + + self.assertEqual(len(frames), 1) + self.assertEqual(frames[0].name, 'test_contextmanager_traceback') + self.assertEqual(frames[0].line, 'raise stop_exc') + + async def test_contextmanager_no_reraise(self): + @asynccontextmanager + async def whee(): + yield + ctx = whee() + await ctx.__aenter__() + # Calling __aexit__ should not result in an exception + self.assertFalse(await ctx.__aexit__(TypeError, TypeError("foo"), None)) + + async def test_contextmanager_trap_yield_after_throw(self): + @asynccontextmanager + async def whoo(): + try: + yield + except: + yield + ctx = whoo() + await ctx.__aenter__() + with self.assertRaises(RuntimeError): + await ctx.__aexit__(TypeError, TypeError('foo'), None) + if support.check_impl_detail(cpython=True): + # The "gen" attribute is an implementation detail. + self.assertFalse(ctx.gen.ag_suspended) + + async def test_contextmanager_trap_no_yield(self): + @asynccontextmanager + async def whoo(): + if False: + yield + ctx = whoo() + with self.assertRaises(RuntimeError): + await ctx.__aenter__() + + async def test_contextmanager_trap_second_yield(self): + @asynccontextmanager + async def whoo(): + yield + yield + ctx = whoo() + await ctx.__aenter__() + with self.assertRaises(RuntimeError): + await ctx.__aexit__(None, None, None) + if support.check_impl_detail(cpython=True): + # The "gen" attribute is an implementation detail. + self.assertFalse(ctx.gen.ag_suspended) + + async def test_contextmanager_non_normalised(self): + @asynccontextmanager + async def whoo(): + try: + yield + except RuntimeError: + raise SyntaxError + + ctx = whoo() + await ctx.__aenter__() + with self.assertRaises(SyntaxError): + await ctx.__aexit__(RuntimeError, None, None) + + async def test_contextmanager_except(self): + state = [] + @asynccontextmanager + async def woohoo(): + state.append(1) + try: + yield 42 + except ZeroDivisionError as e: + state.append(e.args[0]) + self.assertEqual(state, [1, 42, 999]) + async with woohoo() as x: + self.assertEqual(state, [1]) + self.assertEqual(x, 42) + state.append(x) + raise ZeroDivisionError(999) + self.assertEqual(state, [1, 42, 999]) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + async def test_contextmanager_except_stopiter(self): + @asynccontextmanager + async def woohoo(): + yield + + class StopIterationSubclass(StopIteration): + pass + + class StopAsyncIterationSubclass(StopAsyncIteration): + pass + + for stop_exc in ( + StopIteration('spam'), + StopAsyncIteration('ham'), + StopIterationSubclass('spam'), + StopAsyncIterationSubclass('spam') + ): + with self.subTest(type=type(stop_exc)): + try: + async with woohoo(): + raise stop_exc + except Exception as ex: + self.assertIs(ex, stop_exc) + else: + self.fail(f'{stop_exc} was suppressed') + + async def test_contextmanager_wrap_runtimeerror(self): + @asynccontextmanager + async def woohoo(): + try: + yield + except Exception as exc: + raise RuntimeError(f'caught {exc}') from exc + + with self.assertRaises(RuntimeError): + async with woohoo(): + 1 / 0 + + # If the context manager wrapped StopAsyncIteration in a RuntimeError, + # we also unwrap it, because we can't tell whether the wrapping was + # done by the generator machinery or by the generator itself. + with self.assertRaises(StopAsyncIteration): + async with woohoo(): + raise StopAsyncIteration + + def _create_contextmanager_attribs(self): + def attribs(**kw): + def decorate(func): + for k,v in kw.items(): + setattr(func,k,v) + return func + return decorate + @asynccontextmanager + @attribs(foo='bar') + async def baz(spam): + """Whee!""" + yield + return baz + + def test_contextmanager_attribs(self): + baz = self._create_contextmanager_attribs() + self.assertEqual(baz.__name__,'baz') + self.assertEqual(baz.foo, 'bar') + + @support.requires_docstrings + def test_contextmanager_doc_attrib(self): + baz = self._create_contextmanager_attribs() + self.assertEqual(baz.__doc__, "Whee!") + + @support.requires_docstrings + async def test_instance_docstring_given_cm_docstring(self): + baz = self._create_contextmanager_attribs()(None) + self.assertEqual(baz.__doc__, "Whee!") + async with baz: + pass # suppress warning + + async def test_keywords(self): + # Ensure no keyword arguments are inhibited + @asynccontextmanager + async def woohoo(self, func, args, kwds): + yield (self, func, args, kwds) + async with woohoo(self=11, func=22, args=33, kwds=44) as target: + self.assertEqual(target, (11, 22, 33, 44)) + + async def test_recursive(self): + depth = 0 + ncols = 0 + + @asynccontextmanager + async def woohoo(): + nonlocal ncols + ncols += 1 + + nonlocal depth + before = depth + depth += 1 + yield + depth -= 1 + self.assertEqual(depth, before) + + @woohoo() + async def recursive(): + if depth < 10: + await recursive() + + await recursive() + + self.assertEqual(ncols, 10) + self.assertEqual(depth, 0) + + async def test_decorator(self): + entered = False + + @asynccontextmanager + async def context(): + nonlocal entered + entered = True + yield + entered = False + + @context() + async def test(): + self.assertTrue(entered) + + self.assertFalse(entered) + await test() + self.assertFalse(entered) + + async def test_decorator_with_exception(self): + entered = False + + @asynccontextmanager + async def context(): + nonlocal entered + try: + entered = True + yield + finally: + entered = False + + @context() + async def test(): + self.assertTrue(entered) + raise NameError('foo') + + self.assertFalse(entered) + with self.assertRaisesRegex(NameError, 'foo'): + await test() + self.assertFalse(entered) + + async def test_decorating_method(self): + + @asynccontextmanager + async def context(): + yield + + + class Test(object): + + @context() + async def method(self, a, b, c=None): + self.a = a + self.b = b + self.c = c + + # these tests are for argument passing when used as a decorator + test = Test() + await test.method(1, 2) + self.assertEqual(test.a, 1) + self.assertEqual(test.b, 2) + self.assertEqual(test.c, None) + + test = Test() + await test.method('a', 'b', 'c') + self.assertEqual(test.a, 'a') + self.assertEqual(test.b, 'b') + self.assertEqual(test.c, 'c') + + test = Test() + await test.method(a=1, b=2) + self.assertEqual(test.a, 1) + self.assertEqual(test.b, 2) + + +class AclosingTestCase(unittest.IsolatedAsyncioTestCase): + + @support.requires_docstrings + def test_instance_docs(self): + cm_docstring = aclosing.__doc__ + obj = aclosing(None) + self.assertEqual(obj.__doc__, cm_docstring) + + async def test_aclosing(self): + state = [] + class C: + async def aclose(self): + state.append(1) + x = C() + self.assertEqual(state, []) + async with aclosing(x) as y: + self.assertEqual(x, y) + self.assertEqual(state, [1]) + + async def test_aclosing_error(self): + state = [] + class C: + async def aclose(self): + state.append(1) + x = C() + self.assertEqual(state, []) + with self.assertRaises(ZeroDivisionError): + async with aclosing(x) as y: + self.assertEqual(x, y) + 1 / 0 + self.assertEqual(state, [1]) + + async def test_aclosing_bpo41229(self): + state = [] + + @contextmanager + def sync_resource(): + try: + yield + finally: + state.append(1) + + async def agenfunc(): + with sync_resource(): + yield -1 + yield -2 + + x = agenfunc() + self.assertEqual(state, []) + with self.assertRaises(ZeroDivisionError): + async with aclosing(x) as y: + self.assertEqual(x, y) + self.assertEqual(-1, await x.__anext__()) + 1 / 0 + self.assertEqual(state, [1]) + + +class TestAsyncExitStack(TestBaseExitStack, unittest.IsolatedAsyncioTestCase): + class SyncAsyncExitStack(AsyncExitStack): + @staticmethod + def run_coroutine(coro): + loop = asyncio.get_event_loop_policy().get_event_loop() + t = loop.create_task(coro) + t.add_done_callback(lambda f: loop.stop()) + loop.run_forever() + + exc = t.exception() + if not exc: + return t.result() + else: + context = exc.__context__ + + try: + raise exc + except: + exc.__context__ = context + raise exc + + def close(self): + return self.run_coroutine(self.aclose()) + + def __enter__(self): + return self.run_coroutine(self.__aenter__()) + + def __exit__(self, *exc_details): + return self.run_coroutine(self.__aexit__(*exc_details)) + + exit_stack = SyncAsyncExitStack + callback_error_internal_frames = [ + ('__exit__', 'return self.run_coroutine(self.__aexit__(*exc_details))'), + ('run_coroutine', 'raise exc'), + ('run_coroutine', 'raise exc'), + ('__aexit__', 'raise exc'), + ('__aexit__', 'cb_suppress = cb(*exc_details)'), + ] + + async def test_async_callback(self): + expected = [ + ((), {}), + ((1,), {}), + ((1,2), {}), + ((), dict(example=1)), + ((1,), dict(example=1)), + ((1,2), dict(example=1)), + ] + result = [] + async def _exit(*args, **kwds): + """Test metadata propagation""" + result.append((args, kwds)) + + async with AsyncExitStack() as stack: + for args, kwds in reversed(expected): + if args and kwds: + f = stack.push_async_callback(_exit, *args, **kwds) + elif args: + f = stack.push_async_callback(_exit, *args) + elif kwds: + f = stack.push_async_callback(_exit, **kwds) + else: + f = stack.push_async_callback(_exit) + self.assertIs(f, _exit) + for wrapper in stack._exit_callbacks: + self.assertIs(wrapper[1].__wrapped__, _exit) + self.assertNotEqual(wrapper[1].__name__, _exit.__name__) + self.assertIsNone(wrapper[1].__doc__, _exit.__doc__) + + self.assertEqual(result, expected) + + result = [] + async with AsyncExitStack() as stack: + with self.assertRaises(TypeError): + stack.push_async_callback(arg=1) + with self.assertRaises(TypeError): + self.exit_stack.push_async_callback(arg=2) + with self.assertRaises(TypeError): + stack.push_async_callback(callback=_exit, arg=3) + self.assertEqual(result, []) + + async def test_async_push(self): + exc_raised = ZeroDivisionError + async def _expect_exc(exc_type, exc, exc_tb): + self.assertIs(exc_type, exc_raised) + async def _suppress_exc(*exc_details): + return True + async def _expect_ok(exc_type, exc, exc_tb): + self.assertIsNone(exc_type) + self.assertIsNone(exc) + self.assertIsNone(exc_tb) + class ExitCM(object): + def __init__(self, check_exc): + self.check_exc = check_exc + async def __aenter__(self): + self.fail("Should not be called!") + async def __aexit__(self, *exc_details): + await self.check_exc(*exc_details) + + async with self.exit_stack() as stack: + stack.push_async_exit(_expect_ok) + self.assertIs(stack._exit_callbacks[-1][1], _expect_ok) + cm = ExitCM(_expect_ok) + stack.push_async_exit(cm) + self.assertIs(stack._exit_callbacks[-1][1].__self__, cm) + stack.push_async_exit(_suppress_exc) + self.assertIs(stack._exit_callbacks[-1][1], _suppress_exc) + cm = ExitCM(_expect_exc) + stack.push_async_exit(cm) + self.assertIs(stack._exit_callbacks[-1][1].__self__, cm) + stack.push_async_exit(_expect_exc) + self.assertIs(stack._exit_callbacks[-1][1], _expect_exc) + stack.push_async_exit(_expect_exc) + self.assertIs(stack._exit_callbacks[-1][1], _expect_exc) + 1/0 + + async def test_enter_async_context(self): + class TestCM(object): + async def __aenter__(self): + result.append(1) + async def __aexit__(self, *exc_details): + result.append(3) + + result = [] + cm = TestCM() + + async with AsyncExitStack() as stack: + @stack.push_async_callback # Registered first => cleaned up last + async def _exit(): + result.append(4) + self.assertIsNotNone(_exit) + await stack.enter_async_context(cm) + self.assertIs(stack._exit_callbacks[-1][1].__self__, cm) + result.append(2) + + self.assertEqual(result, [1, 2, 3, 4]) + + async def test_enter_async_context_errors(self): + class LacksEnterAndExit: + pass + class LacksEnter: + async def __aexit__(self, *exc_info): + pass + class LacksExit: + async def __aenter__(self): + pass + + async with self.exit_stack() as stack: + with self.assertRaisesRegex(TypeError, 'asynchronous context manager'): + await stack.enter_async_context(LacksEnterAndExit()) + with self.assertRaisesRegex(TypeError, 'asynchronous context manager'): + await stack.enter_async_context(LacksEnter()) + with self.assertRaisesRegex(TypeError, 'asynchronous context manager'): + await stack.enter_async_context(LacksExit()) + self.assertFalse(stack._exit_callbacks) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + async def test_async_exit_exception_chaining(self): + # Ensure exception chaining matches the reference behaviour + async def raise_exc(exc): + raise exc + + saved_details = None + async def suppress_exc(*exc_details): + nonlocal saved_details + saved_details = exc_details + return True + + try: + async with self.exit_stack() as stack: + stack.push_async_callback(raise_exc, IndexError) + stack.push_async_callback(raise_exc, KeyError) + stack.push_async_callback(raise_exc, AttributeError) + stack.push_async_exit(suppress_exc) + stack.push_async_callback(raise_exc, ValueError) + 1 / 0 + except IndexError as exc: + self.assertIsInstance(exc.__context__, KeyError) + self.assertIsInstance(exc.__context__.__context__, AttributeError) + # Inner exceptions were suppressed + self.assertIsNone(exc.__context__.__context__.__context__) + else: + self.fail("Expected IndexError, but no exception was raised") + # Check the inner exceptions + inner_exc = saved_details[1] + self.assertIsInstance(inner_exc, ValueError) + self.assertIsInstance(inner_exc.__context__, ZeroDivisionError) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + async def test_async_exit_exception_explicit_none_context(self): + # Ensure AsyncExitStack chaining matches actual nested `with` statements + # regarding explicit __context__ = None. + + class MyException(Exception): + pass + + @asynccontextmanager + async def my_cm(): + try: + yield + except BaseException: + exc = MyException() + try: + raise exc + finally: + exc.__context__ = None + + @asynccontextmanager + async def my_cm_with_exit_stack(): + async with self.exit_stack() as stack: + await stack.enter_async_context(my_cm()) + yield stack + + for cm in (my_cm, my_cm_with_exit_stack): + with self.subTest(): + try: + async with cm(): + raise IndexError() + except MyException as exc: + self.assertIsNone(exc.__context__) + else: + self.fail("Expected IndexError, but no exception was raised") + + async def test_instance_bypass_async(self): + class Example(object): pass + cm = Example() + cm.__aenter__ = object() + cm.__aexit__ = object() + stack = self.exit_stack() + with self.assertRaisesRegex(TypeError, 'asynchronous context manager'): + await stack.enter_async_context(cm) + stack.push_async_exit(cm) + self.assertIs(stack._exit_callbacks[-1][1], cm) + + +class TestAsyncNullcontext(unittest.IsolatedAsyncioTestCase): + async def test_async_nullcontext(self): + class C: + pass + c = C() + async with nullcontext(c) as c_in: + self.assertIs(c_in, c) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_copy.py b/Lib/test/test_copy.py index 2f9d8ed9b6..456767bbe0 100644 --- a/Lib/test/test_copy.py +++ b/Lib/test/test_copy.py @@ -961,8 +961,6 @@ class PointFromClass(NamedTuple): with self.assertRaisesRegex(TypeError, 'unexpected field name'): copy.replace(p, x=1, error=2) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_dataclass(self): from dataclasses import dataclass @dataclass diff --git a/Lib/test/test_csv.py b/Lib/test/test_csv.py index 9a1743da6d..b7f93d1bac 100644 --- a/Lib/test/test_csv.py +++ b/Lib/test/test_csv.py @@ -10,7 +10,7 @@ import gc import pickle from test import support -from test.support import warnings_helper, import_helper, check_disallow_instantiation +from test.support import import_helper, check_disallow_instantiation from itertools import permutations from textwrap import dedent from collections import OrderedDict @@ -28,14 +28,20 @@ class Test_Csv(unittest.TestCase): in TestDialectRegistry. """ def _test_arg_valid(self, ctor, arg): + ctor(arg) self.assertRaises(TypeError, ctor) self.assertRaises(TypeError, ctor, None) - self.assertRaises(TypeError, ctor, arg, bad_attr = 0) - self.assertRaises(TypeError, ctor, arg, delimiter = 0) - self.assertRaises(TypeError, ctor, arg, delimiter = 'XX') + self.assertRaises(TypeError, ctor, arg, bad_attr=0) + self.assertRaises(TypeError, ctor, arg, delimiter='') + self.assertRaises(TypeError, ctor, arg, escapechar='') + self.assertRaises(TypeError, ctor, arg, quotechar='') + self.assertRaises(TypeError, ctor, arg, delimiter='^^') + self.assertRaises(TypeError, ctor, arg, escapechar='^^') + self.assertRaises(TypeError, ctor, arg, quotechar='^^') self.assertRaises(csv.Error, ctor, arg, 'foo') self.assertRaises(TypeError, ctor, arg, delimiter=None) self.assertRaises(TypeError, ctor, arg, delimiter=1) + self.assertRaises(TypeError, ctor, arg, escapechar=1) self.assertRaises(TypeError, ctor, arg, quotechar=1) self.assertRaises(TypeError, ctor, arg, lineterminator=None) self.assertRaises(TypeError, ctor, arg, lineterminator=1) @@ -46,11 +52,46 @@ def _test_arg_valid(self, ctor, arg): quoting=csv.QUOTE_ALL, quotechar=None) self.assertRaises(TypeError, ctor, arg, quoting=csv.QUOTE_NONE, quotechar='') - + self.assertRaises(ValueError, ctor, arg, delimiter='\n') + self.assertRaises(ValueError, ctor, arg, escapechar='\n') + self.assertRaises(ValueError, ctor, arg, quotechar='\n') + self.assertRaises(ValueError, ctor, arg, delimiter='\r') + self.assertRaises(ValueError, ctor, arg, escapechar='\r') + self.assertRaises(ValueError, ctor, arg, quotechar='\r') + ctor(arg, delimiter=' ') + ctor(arg, escapechar=' ') + ctor(arg, quotechar=' ') + ctor(arg, delimiter='\t', skipinitialspace=True) + ctor(arg, escapechar='\t', skipinitialspace=True) + ctor(arg, quotechar='\t', skipinitialspace=True) + ctor(arg, delimiter=' ', skipinitialspace=True) + self.assertRaises(ValueError, ctor, arg, + escapechar=' ', skipinitialspace=True) + self.assertRaises(ValueError, ctor, arg, + quotechar=' ', skipinitialspace=True) + ctor(arg, delimiter='^') + ctor(arg, escapechar='^') + ctor(arg, quotechar='^') + self.assertRaises(ValueError, ctor, arg, delimiter='^', escapechar='^') + self.assertRaises(ValueError, ctor, arg, delimiter='^', quotechar='^') + self.assertRaises(ValueError, ctor, arg, escapechar='^', quotechar='^') + ctor(arg, delimiter='\x85') + ctor(arg, escapechar='\x85') + ctor(arg, quotechar='\x85') + ctor(arg, lineterminator='\x85') + self.assertRaises(ValueError, ctor, arg, + delimiter='\x85', lineterminator='\x85') + self.assertRaises(ValueError, ctor, arg, + escapechar='\x85', lineterminator='\x85') + self.assertRaises(ValueError, ctor, arg, + quotechar='\x85', lineterminator='\x85') + + @unittest.expectedFailure # TODO: RUSTPYTHON def test_reader_arg_valid(self): self._test_arg_valid(csv.reader, []) self.assertRaises(OSError, csv.reader, BadIterable()) + @unittest.expectedFailure # TODO: RUSTPYTHON def test_writer_arg_valid(self): self._test_arg_valid(csv.writer, StringIO()) class BadWriter: @@ -150,13 +191,8 @@ def _write_error_test(self, exc, fields, **kwargs): fileobj.seek(0) self.assertEqual(fileobj.read(), '') - # TODO: RUSTPYTHON ''\r\n to ""\r\n unsupported - @unittest.expectedFailure def test_write_arg_valid(self): self._write_error_test(csv.Error, None) - self._write_test((), '') - self._write_test([None], '""') - self._write_error_test(csv.Error, [None], quoting = csv.QUOTE_NONE) # Check that exceptions are passed up the chain self._write_error_test(OSError, BadIterable()) class BadList: @@ -170,15 +206,13 @@ class BadItem: def __str__(self): raise OSError self._write_error_test(OSError, [BadItem()]) - def test_write_bigfield(self): # This exercises the buffer realloc functionality bigstring = 'X' * 50000 self._write_test([bigstring,bigstring], '%s,%s' % \ (bigstring, bigstring)) - # TODO: RUSTPYTHON quoting style check is unsupported - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_write_quoting(self): self._write_test(['a',1,'p,q'], 'a,1,"p,q"') self._write_error_test(csv.Error, ['a',1,'p,q'], @@ -196,8 +230,7 @@ def test_write_quoting(self): self._write_test(['a','',None,1], '"a","",,"1"', quoting = csv.QUOTE_NOTNULL) - # TODO: RUSTPYTHON doublequote check is unsupported - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_write_escape(self): self._write_test(['a',1,'p,q'], 'a,1,"p,q"', escapechar='\\') @@ -229,8 +262,7 @@ def test_write_escape(self): self._write_test(['C\\', '6', '7', 'X"'], 'C\\\\,6,7,"X"""', escapechar='\\', quoting=csv.QUOTE_MINIMAL) - # TODO: RUSTPYTHON lineterminator double char unsupported - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_write_lineterminator(self): for lineterminator in '\r\n', '\n', '\r', '!@#', '\0': with self.subTest(lineterminator=lineterminator): @@ -238,12 +270,13 @@ def test_write_lineterminator(self): writer = csv.writer(sio, lineterminator=lineterminator) writer.writerow(['a', 'b']) writer.writerow([1, 2]) + writer.writerow(['\r', '\n']) self.assertEqual(sio.getvalue(), f'a,b{lineterminator}' - f'1,2{lineterminator}') + f'1,2{lineterminator}' + f'"\r","\n"{lineterminator}') - # TODO: RUSTPYTHON ''\r\n to ""\r\n unspported - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_write_iterable(self): self._write_test(iter(['a', 1, 'p,q']), 'a,1,"p,q"') self._write_test(iter(['a', 1, None]), 'a,1,') @@ -285,6 +318,51 @@ def test_writerows_with_none(self): fileobj.seek(0) self.assertEqual(fileobj.read(), 'a\r\n""\r\n') + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_write_empty_fields(self): + self._write_test((), '') + self._write_test([''], '""') + self._write_error_test(csv.Error, [''], quoting=csv.QUOTE_NONE) + self._write_test([''], '""', quoting=csv.QUOTE_STRINGS) + self._write_test([''], '""', quoting=csv.QUOTE_NOTNULL) + self._write_test([None], '""') + self._write_error_test(csv.Error, [None], quoting=csv.QUOTE_NONE) + self._write_error_test(csv.Error, [None], quoting=csv.QUOTE_STRINGS) + self._write_error_test(csv.Error, [None], quoting=csv.QUOTE_NOTNULL) + self._write_test(['', ''], ',') + self._write_test([None, None], ',') + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_write_empty_fields_space_delimiter(self): + self._write_test([''], '""', delimiter=' ', skipinitialspace=False) + self._write_test([''], '""', delimiter=' ', skipinitialspace=True) + self._write_test([None], '""', delimiter=' ', skipinitialspace=False) + self._write_test([None], '""', delimiter=' ', skipinitialspace=True) + + self._write_test(['', ''], ' ', delimiter=' ', skipinitialspace=False) + self._write_test(['', ''], '"" ""', delimiter=' ', skipinitialspace=True) + self._write_test([None, None], ' ', delimiter=' ', skipinitialspace=False) + self._write_test([None, None], '"" ""', delimiter=' ', skipinitialspace=True) + + self._write_test(['', ''], ' ', delimiter=' ', skipinitialspace=False, + quoting=csv.QUOTE_NONE) + self._write_error_test(csv.Error, ['', ''], + delimiter=' ', skipinitialspace=True, + quoting=csv.QUOTE_NONE) + for quoting in csv.QUOTE_STRINGS, csv.QUOTE_NOTNULL: + self._write_test(['', ''], '"" ""', delimiter=' ', skipinitialspace=False, + quoting=quoting) + self._write_test(['', ''], '"" ""', delimiter=' ', skipinitialspace=True, + quoting=quoting) + + for quoting in csv.QUOTE_NONE, csv.QUOTE_STRINGS, csv.QUOTE_NOTNULL: + self._write_test([None, None], ' ', delimiter=' ', skipinitialspace=False, + quoting=quoting) + self._write_error_test(csv.Error, [None, None], + delimiter=' ', skipinitialspace=True, + quoting=quoting) + def test_writerows_errors(self): with TemporaryFile("w+", encoding="utf-8", newline='') as fileobj: writer = csv.writer(fileobj) @@ -296,8 +374,7 @@ def _read_test(self, input, expect, **kwargs): result = list(reader) self.assertEqual(result, expect) - # TODO RUSTPYTHON strict mode is unsupported - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_read_oddinputs(self): self._read_test([], []) self._read_test([''], [[]]) @@ -308,17 +385,22 @@ def test_read_oddinputs(self): self.assertRaises(csv.Error, self._read_test, [b'abc'], None) + @unittest.expectedFailure # TODO: RUSTPYTHON def test_read_eol(self): - self._read_test(['a,b'], [['a','b']]) - self._read_test(['a,b\n'], [['a','b']]) - self._read_test(['a,b\r\n'], [['a','b']]) - self._read_test(['a,b\r'], [['a','b']]) - self.assertRaises(csv.Error, self._read_test, ['a,b\rc,d'], []) - self.assertRaises(csv.Error, self._read_test, ['a,b\nc,d'], []) - self.assertRaises(csv.Error, self._read_test, ['a,b\r\nc,d'], []) - - # TODO RUSTPYTHON double quote umimplement - @unittest.expectedFailure + self._read_test(['a,b', 'c,d'], [['a','b'], ['c','d']]) + self._read_test(['a,b\n', 'c,d\n'], [['a','b'], ['c','d']]) + self._read_test(['a,b\r\n', 'c,d\r\n'], [['a','b'], ['c','d']]) + self._read_test(['a,b\r', 'c,d\r'], [['a','b'], ['c','d']]) + + errmsg = "with newline=''" + with self.assertRaisesRegex(csv.Error, errmsg): + next(csv.reader(['a,b\rc,d'])) + with self.assertRaisesRegex(csv.Error, errmsg): + next(csv.reader(['a,b\nc,d'])) + with self.assertRaisesRegex(csv.Error, errmsg): + next(csv.reader(['a,b\r\nc,d'])) + + @unittest.expectedFailure # TODO: RUSTPYTHON def test_read_eof(self): self._read_test(['a,"'], [['a', '']]) self._read_test(['"a'], [['a']]) @@ -328,8 +410,7 @@ def test_read_eof(self): self.assertRaises(csv.Error, self._read_test, ['^'], [], escapechar='^', strict=True) - # TODO RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_read_nul(self): self._read_test(['\0'], [['\0']]) self._read_test(['a,\0b,c'], [['a', '\0b', 'c']]) @@ -342,8 +423,7 @@ def test_read_delimiter(self): self._read_test(['a;b;c'], [['a', 'b', 'c']], delimiter=';') self._read_test(['a\0b\0c'], [['a', 'b', 'c']], delimiter='\0') - # TODO RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_read_escape(self): self._read_test(['a,\\b,c'], [['a', 'b', 'c']], escapechar='\\') self._read_test(['a,b\\,c'], [['a', 'b,c']], escapechar='\\') @@ -356,8 +436,7 @@ def test_read_escape(self): self._read_test(['a,\\b,c'], [['a', '\\b', 'c']], escapechar=None) self._read_test(['a,\\b,c'], [['a', '\\b', 'c']]) - # TODO RUSTPYTHON escapechar unsupported - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_read_quoting(self): self._read_test(['1,",3,",5'], [['1', ',3,', '5']]) self._read_test(['1,",3,",5'], [['1', '"', '3', '"', '5']], @@ -367,17 +446,56 @@ def test_read_quoting(self): # will this fail where locale uses comma for decimals? self._read_test([',3,"5",7.3, 9'], [['', 3, '5', 7.3, 9]], quoting=csv.QUOTE_NONNUMERIC) + self._read_test([',3,"5",7.3, 9'], [[None, '3', '5', '7.3', ' 9']], + quoting=csv.QUOTE_NOTNULL) + self._read_test([',3,"5",7.3, 9'], [[None, 3, '5', 7.3, 9]], + quoting=csv.QUOTE_STRINGS) + + self._read_test([',,"",'], [['', '', '', '']]) + self._read_test([',,"",'], [['', '', '', '']], + quoting=csv.QUOTE_NONNUMERIC) + self._read_test([',,"",'], [[None, None, '', None]], + quoting=csv.QUOTE_NOTNULL) + self._read_test([',,"",'], [[None, None, '', None]], + quoting=csv.QUOTE_STRINGS) + self._read_test(['"a\nb", 7'], [['a\nb', ' 7']]) self.assertRaises(ValueError, self._read_test, ['abc,3'], [[]], quoting=csv.QUOTE_NONNUMERIC) + self.assertRaises(ValueError, self._read_test, + ['abc,3'], [[]], + quoting=csv.QUOTE_STRINGS) self._read_test(['1,@,3,@,5'], [['1', ',3,', '5']], quotechar='@') self._read_test(['1,\0,3,\0,5'], [['1', ',3,', '5']], quotechar='\0') + self._read_test(['1\\.5,\\.5,.5'], [[1.5, 0.5, 0.5]], + quoting=csv.QUOTE_NONNUMERIC, escapechar='\\') + self._read_test(['1\\.5,\\.5,"\\.5"'], [[1.5, 0.5, ".5"]], + quoting=csv.QUOTE_STRINGS, escapechar='\\') + @unittest.skip('TODO: RUSTPYTHON; slice index starts at 1 but ends at 0') def test_read_skipinitialspace(self): self._read_test(['no space, space, spaces,\ttab'], [['no space', 'space', 'spaces', '\ttab']], skipinitialspace=True) + self._read_test([' , , '], + [['', '', '']], + skipinitialspace=True) + self._read_test([' , , '], + [[None, None, None]], + skipinitialspace=True, quoting=csv.QUOTE_NOTNULL) + self._read_test([' , , '], + [[None, None, None]], + skipinitialspace=True, quoting=csv.QUOTE_STRINGS) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_read_space_delimiter(self): + self._read_test(['a b', ' a ', ' ', ''], + [['a', '', '', 'b'], ['', '', 'a', '', ''], ['', '', ''], []], + delimiter=' ', skipinitialspace=False) + self._read_test(['a b', ' a ', ' ', ''], + [['a', 'b'], ['a', ''], [''], []], + delimiter=' ', skipinitialspace=True) def test_read_bigfield(self): # This exercises the buffer realloc functionality and field size @@ -410,27 +528,47 @@ def test_read_linenum(self): self.assertRaises(StopIteration, next, r) self.assertEqual(r.line_num, 3) - # TODO: RUSTPYTHON only '\r\n' unsupported - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_roundtrip_quoteed_newlines(self): - with TemporaryFile("w+", encoding="utf-8", newline='') as fileobj: - writer = csv.writer(fileobj) - rows = [['a\nb','b'],['c','x\r\nd']] - writer.writerows(rows) - fileobj.seek(0) - for i, row in enumerate(csv.reader(fileobj)): - self.assertEqual(row, rows[i]) - - # TODO: RUSTPYTHON only '\r\n' unsupported - @unittest.expectedFailure + rows = [ + ['\na', 'b\nc', 'd\n'], + ['\re', 'f\rg', 'h\r'], + ['\r\ni', 'j\r\nk', 'l\r\n'], + ['\n\rm', 'n\n\ro', 'p\n\r'], + ['\r\rq', 'r\r\rs', 't\r\r'], + ['\n\nu', 'v\n\nw', 'x\n\n'], + ] + for lineterminator in '\r\n', '\n', '\r': + with self.subTest(lineterminator=lineterminator): + with TemporaryFile("w+", encoding="utf-8", newline='') as fileobj: + writer = csv.writer(fileobj, lineterminator=lineterminator) + writer.writerows(rows) + fileobj.seek(0) + for i, row in enumerate(csv.reader(fileobj)): + self.assertEqual(row, rows[i]) + + @unittest.expectedFailure # TODO: RUSTPYTHON def test_roundtrip_escaped_unquoted_newlines(self): - with TemporaryFile("w+", encoding="utf-8", newline='') as fileobj: - writer = csv.writer(fileobj,quoting=csv.QUOTE_NONE,escapechar="\\") - rows = [['a\nb','b'],['c','x\r\nd']] - writer.writerows(rows) - fileobj.seek(0) - for i, row in enumerate(csv.reader(fileobj,quoting=csv.QUOTE_NONE,escapechar="\\")): - self.assertEqual(row,rows[i]) + rows = [ + ['\na', 'b\nc', 'd\n'], + ['\re', 'f\rg', 'h\r'], + ['\r\ni', 'j\r\nk', 'l\r\n'], + ['\n\rm', 'n\n\ro', 'p\n\r'], + ['\r\rq', 'r\r\rs', 't\r\r'], + ['\n\nu', 'v\n\nw', 'x\n\n'], + ] + for lineterminator in '\r\n', '\n', '\r': + with self.subTest(lineterminator=lineterminator): + with TemporaryFile("w+", encoding="utf-8", newline='') as fileobj: + writer = csv.writer(fileobj, lineterminator=lineterminator, + quoting=csv.QUOTE_NONE, escapechar="\\") + writer.writerows(rows) + fileobj.seek(0) + for i, row in enumerate(csv.reader(fileobj, + quoting=csv.QUOTE_NONE, + escapechar="\\")): + self.assertEqual(row, rows[i]) + class TestDialectRegistry(unittest.TestCase): def test_registry_badargs(self): @@ -509,10 +647,10 @@ class space(csv.excel): escapechar = "\\" with TemporaryFile("w+", encoding="utf-8") as fileobj: - fileobj.write("abc def\nc1ccccc1 benzene\n") + fileobj.write("abc def\nc1ccccc1 benzene\n") fileobj.seek(0) reader = csv.reader(fileobj, dialect=space()) - self.assertEqual(next(reader), ["abc", "def"]) + self.assertEqual(next(reader), ["abc", "", "", "def"]) self.assertEqual(next(reader), ["c1ccccc1", "benzene"]) def compare_dialect_123(self, expected, *writeargs, **kwwriteargs): @@ -524,8 +662,7 @@ def compare_dialect_123(self, expected, *writeargs, **kwwriteargs): fileobj.seek(0) self.assertEqual(fileobj.read(), expected) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_dialect_apply(self): class testA(csv.excel): delimiter = "\t" @@ -556,21 +693,12 @@ class unspecified(): finally: csv.unregister_dialect('testC') - def test_bad_dialect(self): - # Unknown parameter - self.assertRaises(TypeError, csv.reader, [], bad_attr = 0) - # Bad values - self.assertRaises(TypeError, csv.reader, [], delimiter = None) - self.assertRaises(TypeError, csv.reader, [], quoting = -1) - self.assertRaises(TypeError, csv.reader, [], quoting = 100) - def test_copy(self): for name in csv.list_dialects(): dialect = csv.get_dialect(name) self.assertRaises(TypeError, copy.copy, dialect) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_pickle(self): for name in csv.list_dialects(): dialect = csv.get_dialect(name) @@ -657,8 +785,7 @@ def test_quoted_quote(self): '"I see," said the blind man', 'as he picked up his hammer and saw']]) - # Rustpython TODO - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_quoted_nl(self): input = '''\ 1,2,3,"""I see,"" @@ -699,21 +826,18 @@ class EscapedExcel(csv.excel): class TestEscapedExcel(TestCsvBase): dialect = EscapedExcel() - # TODO RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_escape_fieldsep(self): self.writerAssertEqual([['abc,def']], 'abc\\,def\r\n') - # TODO RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_read_escape_fieldsep(self): self.readerAssertEqual('abc\\,def\r\n', [['abc,def']]) class TestDialectUnix(TestCsvBase): dialect = 'unix' - # TODO RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_simple_writer(self): self.writerAssertEqual([[1, 'abc def', 'abc']], '"1","abc def","abc"\n') @@ -730,8 +854,7 @@ class TestQuotedEscapedExcel(TestCsvBase): def test_write_escape_fieldsep(self): self.writerAssertEqual([['abc,def']], '"abc,def"\r\n') - # TODO RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_read_escape_fieldsep(self): self.readerAssertEqual('"abc\\,def"\r\n', [['abc,def']]) @@ -928,8 +1051,7 @@ def test_read_multi(self): "s1": 'abc', "s2": 'def'}) - # TODO RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_read_with_blanks(self): reader = csv.DictReader(["1,2,abc,4,5,6\r\n","\r\n", "1,2,abc,4,5,6\r\n"], @@ -981,9 +1103,10 @@ def test_float_write(self): fileobj.seek(0) self.assertEqual(fileobj.read(), expected) + @unittest.expectedFailure # TODO: RUSTPYTHON def test_char_write(self): import array, string - a = array.array('u', string.ascii_letters) + a = array.array('w', string.ascii_letters) with TemporaryFile("w+", encoding="utf-8", newline='') as fileobj: writer = csv.writer(fileobj, dialect="excel") @@ -1007,6 +1130,12 @@ class mydialect(csv.Dialect): mydialect.quoting = None self.assertRaises(csv.Error, mydialect) + mydialect.quoting = 42 + with self.assertRaises(csv.Error) as cm: + mydialect() + self.assertEqual(str(cm.exception), + 'bad "quoting" value') + mydialect.doublequote = True mydialect.quoting = csv.QUOTE_ALL mydialect.quotechar = '"' @@ -1122,11 +1251,17 @@ class mydialect(csv.Dialect): self.assertEqual(str(cm.exception), '"lineterminator" must be a string') + @unittest.expectedFailure # TODO: RUSTPYTHON def test_invalid_chars(self): - def create_invalid(field_name, value): + def create_invalid(field_name, value, **kwargs): class mydialect(csv.Dialect): - pass + delimiter = ',' + quoting = csv.QUOTE_ALL + quotechar = '"' + lineterminator = '\r\n' setattr(mydialect, field_name, value) + for field_name, value in kwargs.items(): + setattr(mydialect, field_name, value) d = mydialect() for field_name in ("delimiter", "escapechar", "quotechar"): @@ -1135,6 +1270,11 @@ class mydialect(csv.Dialect): self.assertRaises(csv.Error, create_invalid, field_name, "abc") self.assertRaises(csv.Error, create_invalid, field_name, b'x') self.assertRaises(csv.Error, create_invalid, field_name, 5) + self.assertRaises(ValueError, create_invalid, field_name, "\n") + self.assertRaises(ValueError, create_invalid, field_name, "\r") + if field_name != "delimiter": + self.assertRaises(ValueError, create_invalid, field_name, " ", + skipinitialspace=True) class TestSniffer(unittest.TestCase): @@ -1451,8 +1591,7 @@ def test_ordered_dict_reader(self): class MiscTestCase(unittest.TestCase): def test__all__(self): - extra = {'__doc__', '__version__'} - support.check__all__(self, csv, ('csv', '_csv'), extra=extra) + support.check__all__(self, csv, ('csv', '_csv')) def test_subclassable(self): # issue 44089 diff --git a/Lib/test/test_ctypes.py b/Lib/test/test_ctypes.py deleted file mode 100644 index b0a12c9734..0000000000 --- a/Lib/test/test_ctypes.py +++ /dev/null @@ -1,10 +0,0 @@ -import unittest -from test.support.import_helper import import_module - - -ctypes_test = import_module('ctypes.test') - -load_tests = ctypes_test.load_tests - -if __name__ == "__main__": - unittest.main() diff --git a/Lib/test/test_ctypes/__init__.py b/Lib/test/test_ctypes/__init__.py new file mode 100644 index 0000000000..eb9126cbe1 --- /dev/null +++ b/Lib/test/test_ctypes/__init__.py @@ -0,0 +1,10 @@ +import os +from test import support +from test.support import import_helper + + +# skip tests if the _ctypes extension was not built +import_helper.import_module('ctypes') + +def load_tests(*args): + return support.load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_ctypes/__main__.py b/Lib/test/test_ctypes/__main__.py new file mode 100644 index 0000000000..3003d4db89 --- /dev/null +++ b/Lib/test/test_ctypes/__main__.py @@ -0,0 +1,4 @@ +from test.test_ctypes import load_tests +import unittest + +unittest.main() diff --git a/Lib/test/test_ctypes/_support.py b/Lib/test/test_ctypes/_support.py new file mode 100644 index 0000000000..e4c2b33825 --- /dev/null +++ b/Lib/test/test_ctypes/_support.py @@ -0,0 +1,24 @@ +# Some classes and types are not export to _ctypes module directly. + +import ctypes +from _ctypes import Structure, Union, _Pointer, Array, _SimpleCData, CFuncPtr + + +_CData = Structure.__base__ +assert _CData.__name__ == "_CData" + +class _X(Structure): + _fields_ = [("x", ctypes.c_int)] +CField = type(_X.x) + +# metaclasses +PyCStructType = type(Structure) +UnionType = type(Union) +PyCPointerType = type(_Pointer) +PyCArrayType = type(Array) +PyCSimpleType = type(_SimpleCData) +PyCFuncPtrType = type(CFuncPtr) + +# type flags +Py_TPFLAGS_DISALLOW_INSTANTIATION = 1 << 7 +Py_TPFLAGS_IMMUTABLETYPE = 1 << 8 diff --git a/Lib/test/test_ctypes/test_aligned_structures.py b/Lib/test/test_ctypes/test_aligned_structures.py new file mode 100644 index 0000000000..8e8ac42990 --- /dev/null +++ b/Lib/test/test_ctypes/test_aligned_structures.py @@ -0,0 +1,321 @@ +from ctypes import ( + c_char, c_uint32, c_uint16, c_ubyte, c_byte, alignment, sizeof, + BigEndianStructure, LittleEndianStructure, + BigEndianUnion, LittleEndianUnion, Structure +) +import struct +import unittest + + +class TestAlignedStructures(unittest.TestCase): + def test_aligned_string(self): + for base, e in ( + (LittleEndianStructure, "<"), + (BigEndianStructure, ">"), + ): + data = bytearray(struct.pack(f"{e}i12x16s", 7, b"hello world!")) + class Aligned(base): + _align_ = 16 + _fields_ = [ + ('value', c_char * 12) + ] + + class Main(base): + _fields_ = [ + ('first', c_uint32), + ('string', Aligned), + ] + + main = Main.from_buffer(data) + self.assertEqual(main.first, 7) + self.assertEqual(main.string.value, b'hello world!') + self.assertEqual(bytes(main.string), b'hello world!\0\0\0\0') + self.assertEqual(Main.string.offset, 16) + self.assertEqual(Main.string.size, 16) + self.assertEqual(alignment(main.string), 16) + self.assertEqual(alignment(main), 16) + + def test_aligned_structures(self): + for base, data in ( + (LittleEndianStructure, bytearray(b"\1\0\0\0\1\0\0\0\7\0\0\0")), + (BigEndianStructure, bytearray(b"\1\0\0\0\1\0\0\0\7\0\0\0")), + ): + class SomeBools(base): + _align_ = 4 + _fields_ = [ + ("bool1", c_ubyte), + ("bool2", c_ubyte), + ] + class Main(base): + _fields_ = [ + ("x", c_ubyte), + ("y", SomeBools), + ("z", c_ubyte), + ] + + main = Main.from_buffer(data) + self.assertEqual(alignment(SomeBools), 4) + self.assertEqual(alignment(main), 4) + self.assertEqual(alignment(main.y), 4) + self.assertEqual(Main.x.size, 1) + self.assertEqual(Main.y.offset, 4) + self.assertEqual(Main.y.size, 4) + self.assertEqual(main.y.bool1, True) + self.assertEqual(main.y.bool2, False) + self.assertEqual(Main.z.offset, 8) + self.assertEqual(main.z, 7) + + def test_oversized_structure(self): + data = bytearray(b"\0" * 8) + for base in (LittleEndianStructure, BigEndianStructure): + class SomeBoolsTooBig(base): + _align_ = 8 + _fields_ = [ + ("bool1", c_ubyte), + ("bool2", c_ubyte), + ("bool3", c_ubyte), + ] + class Main(base): + _fields_ = [ + ("y", SomeBoolsTooBig), + ("z", c_uint32), + ] + with self.assertRaises(ValueError) as ctx: + Main.from_buffer(data) + self.assertEqual( + ctx.exception.args[0], + 'Buffer size too small (4 instead of at least 8 bytes)' + ) + + def test_aligned_subclasses(self): + for base, e in ( + (LittleEndianStructure, "<"), + (BigEndianStructure, ">"), + ): + data = bytearray(struct.pack(f"{e}4i", 1, 2, 3, 4)) + class UnalignedSub(base): + x: c_uint32 + _fields_ = [ + ("x", c_uint32), + ] + + class AlignedStruct(UnalignedSub): + _align_ = 8 + _fields_ = [ + ("y", c_uint32), + ] + + class Main(base): + _fields_ = [ + ("a", c_uint32), + ("b", AlignedStruct) + ] + + main = Main.from_buffer(data) + self.assertEqual(alignment(main.b), 8) + self.assertEqual(alignment(main), 8) + self.assertEqual(sizeof(main.b), 8) + self.assertEqual(sizeof(main), 16) + self.assertEqual(main.a, 1) + self.assertEqual(main.b.x, 3) + self.assertEqual(main.b.y, 4) + self.assertEqual(Main.b.offset, 8) + self.assertEqual(Main.b.size, 8) + + def test_aligned_union(self): + for sbase, ubase, e in ( + (LittleEndianStructure, LittleEndianUnion, "<"), + (BigEndianStructure, BigEndianUnion, ">"), + ): + data = bytearray(struct.pack(f"{e}4i", 1, 2, 3, 4)) + class AlignedUnion(ubase): + _align_ = 8 + _fields_ = [ + ("a", c_uint32), + ("b", c_ubyte * 7), + ] + + class Main(sbase): + _fields_ = [ + ("first", c_uint32), + ("union", AlignedUnion), + ] + + main = Main.from_buffer(data) + self.assertEqual(main.first, 1) + self.assertEqual(main.union.a, 3) + self.assertEqual(bytes(main.union.b), data[8:-1]) + self.assertEqual(Main.union.offset, 8) + self.assertEqual(Main.union.size, 8) + self.assertEqual(alignment(main.union), 8) + self.assertEqual(alignment(main), 8) + + def test_aligned_struct_in_union(self): + for sbase, ubase, e in ( + (LittleEndianStructure, LittleEndianUnion, "<"), + (BigEndianStructure, BigEndianUnion, ">"), + ): + data = bytearray(struct.pack(f"{e}4i", 1, 2, 3, 4)) + class Sub(sbase): + _align_ = 8 + _fields_ = [ + ("x", c_uint32), + ("y", c_uint32), + ] + + class MainUnion(ubase): + _fields_ = [ + ("a", c_uint32), + ("b", Sub), + ] + + class Main(sbase): + _fields_ = [ + ("first", c_uint32), + ("union", MainUnion), + ] + + main = Main.from_buffer(data) + self.assertEqual(Main.first.size, 4) + self.assertEqual(alignment(main.union), 8) + self.assertEqual(alignment(main), 8) + self.assertEqual(Main.union.offset, 8) + self.assertEqual(Main.union.size, 8) + self.assertEqual(main.first, 1) + self.assertEqual(main.union.a, 3) + self.assertEqual(main.union.b.x, 3) + self.assertEqual(main.union.b.y, 4) + + def test_smaller_aligned_subclassed_union(self): + for sbase, ubase, e in ( + (LittleEndianStructure, LittleEndianUnion, "<"), + (BigEndianStructure, BigEndianUnion, ">"), + ): + data = bytearray(struct.pack(f"{e}H2xI", 1, 0xD60102D7)) + class SubUnion(ubase): + _align_ = 2 + _fields_ = [ + ("unsigned", c_ubyte), + ("signed", c_byte), + ] + + class MainUnion(SubUnion): + _fields_ = [ + ("num", c_uint32) + ] + + class Main(sbase): + _fields_ = [ + ("first", c_uint16), + ("union", MainUnion), + ] + + main = Main.from_buffer(data) + self.assertEqual(main.union.num, 0xD60102D7) + self.assertEqual(main.union.unsigned, data[4]) + self.assertEqual(main.union.signed, data[4] - 256) + self.assertEqual(alignment(main), 4) + self.assertEqual(alignment(main.union), 4) + self.assertEqual(Main.union.offset, 4) + self.assertEqual(Main.union.size, 4) + self.assertEqual(Main.first.size, 2) + + def test_larger_aligned_subclassed_union(self): + for ubase, e in ( + (LittleEndianUnion, "<"), + (BigEndianUnion, ">"), + ): + data = bytearray(struct.pack(f"{e}I4x", 0xD60102D6)) + class SubUnion(ubase): + _align_ = 8 + _fields_ = [ + ("unsigned", c_ubyte), + ("signed", c_byte), + ] + + class Main(SubUnion): + _fields_ = [ + ("num", c_uint32) + ] + + main = Main.from_buffer(data) + self.assertEqual(alignment(main), 8) + self.assertEqual(sizeof(main), 8) + self.assertEqual(main.num, 0xD60102D6) + self.assertEqual(main.unsigned, 0xD6) + self.assertEqual(main.signed, -42) + + def test_aligned_packed_structures(self): + for sbase, e in ( + (LittleEndianStructure, "<"), + (BigEndianStructure, ">"), + ): + data = bytearray(struct.pack(f"{e}B2H4xB", 1, 2, 3, 4)) + + class Inner(sbase): + _align_ = 8 + _fields_ = [ + ("x", c_uint16), + ("y", c_uint16), + ] + + class Main(sbase): + _pack_ = 1 + _fields_ = [ + ("a", c_ubyte), + ("b", Inner), + ("c", c_ubyte), + ] + + main = Main.from_buffer(data) + self.assertEqual(sizeof(main), 10) + self.assertEqual(Main.b.offset, 1) + # Alignment == 8 because _pack_ wins out. + self.assertEqual(alignment(main.b), 8) + # Size is still 8 though since inside this Structure, it will have + # effect. + self.assertEqual(sizeof(main.b), 8) + self.assertEqual(Main.c.offset, 9) + self.assertEqual(main.a, 1) + self.assertEqual(main.b.x, 2) + self.assertEqual(main.b.y, 3) + self.assertEqual(main.c, 4) + + def test_negative_align(self): + for base in (Structure, LittleEndianStructure, BigEndianStructure): + with ( + self.subTest(base=base), + self.assertRaisesRegex( + ValueError, + '_align_ must be a non-negative integer', + ) + ): + class MyStructure(base): + _align_ = -1 + _fields_ = [] + + def test_zero_align_no_fields(self): + for base in (Structure, LittleEndianStructure, BigEndianStructure): + with self.subTest(base=base): + class MyStructure(base): + _align_ = 0 + _fields_ = [] + + self.assertEqual(alignment(MyStructure), 1) + self.assertEqual(alignment(MyStructure()), 1) + + def test_zero_align_with_fields(self): + for base in (Structure, LittleEndianStructure, BigEndianStructure): + with self.subTest(base=base): + class MyStructure(base): + _align_ = 0 + _fields_ = [ + ("x", c_ubyte), + ] + + self.assertEqual(alignment(MyStructure), 1) + self.assertEqual(alignment(MyStructure()), 1) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/ctypes/test/test_anon.py b/Lib/test/test_ctypes/test_anon.py similarity index 97% rename from Lib/ctypes/test/test_anon.py rename to Lib/test/test_ctypes/test_anon.py index d378392ebe..b36397b510 100644 --- a/Lib/ctypes/test/test_anon.py +++ b/Lib/test/test_ctypes/test_anon.py @@ -1,6 +1,7 @@ import unittest import test.support -from ctypes import * +from ctypes import c_int, Union, Structure, sizeof + class AnonTest(unittest.TestCase): @@ -69,5 +70,6 @@ class Y(Structure): self.assertEqual(Y._.offset, sizeof(c_int)) self.assertEqual(Y.y.offset, sizeof(c_int) * 2) + if __name__ == "__main__": unittest.main() diff --git a/Lib/ctypes/test/test_array_in_pointer.py b/Lib/test/test_ctypes/test_array_in_pointer.py similarity index 93% rename from Lib/ctypes/test/test_array_in_pointer.py rename to Lib/test/test_ctypes/test_array_in_pointer.py index ca1edcf621..b7c96b2fa4 100644 --- a/Lib/ctypes/test/test_array_in_pointer.py +++ b/Lib/test/test_ctypes/test_array_in_pointer.py @@ -1,21 +1,24 @@ -import unittest -from ctypes import * -from binascii import hexlify +import binascii import re +import unittest +from ctypes import c_byte, Structure, POINTER, cast + def dump(obj): # helper function to dump memory contents in hex, with a hyphen # between the bytes. - h = hexlify(memoryview(obj)).decode() + h = binascii.hexlify(memoryview(obj)).decode() return re.sub(r"(..)", r"\1-", h)[:-1] class Value(Structure): _fields_ = [("val", c_byte)] + class Container(Structure): _fields_ = [("pvalues", POINTER(Value))] + class Test(unittest.TestCase): def test(self): # create an array of 4 values @@ -60,5 +63,6 @@ def test_2(self): ([1, 2, 3, 4], "01-02-03-04") ) + if __name__ == "__main__": unittest.main() diff --git a/Lib/ctypes/test/test_arrays.py b/Lib/test/test_ctypes/test_arrays.py similarity index 77% rename from Lib/ctypes/test/test_arrays.py rename to Lib/test/test_ctypes/test_arrays.py index 14603b7049..c80fdff5de 100644 --- a/Lib/ctypes/test/test_arrays.py +++ b/Lib/test/test_ctypes/test_arrays.py @@ -1,16 +1,50 @@ +import ctypes +import sys import unittest +from ctypes import (Structure, Array, ARRAY, sizeof, addressof, + create_string_buffer, create_unicode_buffer, + c_char, c_wchar, c_byte, c_ubyte, c_short, c_ushort, c_int, c_uint, + c_long, c_ulonglong, c_float, c_double, c_longdouble) from test.support import bigmemtest, _2G -import sys -from ctypes import * +from ._support import (_CData, PyCArrayType, Py_TPFLAGS_DISALLOW_INSTANTIATION, + Py_TPFLAGS_IMMUTABLETYPE) -from ctypes.test import need_symbol formats = "bBhHiIlLqQfd" formats = c_byte, c_ubyte, c_short, c_ushort, c_int, c_uint, \ c_long, c_ulonglong, c_float, c_double, c_longdouble + class ArrayTestCase(unittest.TestCase): + def test_inheritance_hierarchy(self): + self.assertEqual(Array.mro(), [Array, _CData, object]) + + self.assertEqual(PyCArrayType.__name__, "PyCArrayType") + self.assertEqual(type(PyCArrayType), type) + + def test_type_flags(self): + for cls in Array, PyCArrayType: + with self.subTest(cls=cls): + self.assertTrue(cls.__flags__ & Py_TPFLAGS_IMMUTABLETYPE) + self.assertFalse(cls.__flags__ & Py_TPFLAGS_DISALLOW_INSTANTIATION) + + def test_metaclass_details(self): + # Abstract classes (whose metaclass __init__ was not called) can't be + # instantiated directly + NewArray = PyCArrayType.__new__(PyCArrayType, 'NewArray', (Array,), {}) + for cls in Array, NewArray: + with self.subTest(cls=cls): + with self.assertRaisesRegex(TypeError, "abstract class"): + obj = cls() + + # Cannot call the metaclass __init__ more than once + class T(Array): + _type_ = c_int + _length_ = 13 + with self.assertRaisesRegex(SystemError, "already initialized"): + PyCArrayType.__init__(T, 'ptr', (), {}) + def test_simple(self): # create classes holding simple numeric types, and check # various properties. @@ -34,9 +68,9 @@ def test_simple(self): with self.assertRaises(IndexError): ia[-alen-1] # change the items - from operator import setitem new_values = list(range(42, 42+alen)) - [setitem(ia, n, new_values[n]) for n in range(alen)] + for n in range(alen): + ia[n] = new_values[n] values = [ia[i] for i in range(alen)] self.assertEqual(values, new_values) @@ -66,8 +100,8 @@ def test_simple(self): self.assertEqual(len(ca), 3) # cannot delete items - from operator import delitem - self.assertRaises(TypeError, delitem, ca, 0) + with self.assertRaises(TypeError): + del ca[0] def test_step_overflow(self): a = (c_int * 5)() @@ -117,7 +151,6 @@ def test_from_address(self): self.assertEqual(sz[1:4:2], b"o") self.assertEqual(sz.value, b"foo") - @need_symbol('create_unicode_buffer') def test_from_addressW(self): p = create_unicode_buffer("foo") sz = (c_wchar * 3).from_address(addressof(p)) @@ -178,10 +211,10 @@ def test_bad_subclass(self): class T(Array): pass with self.assertRaises(AttributeError): - class T(Array): + class T2(Array): _type_ = c_int with self.assertRaises(AttributeError): - class T(Array): + class T3(Array): _length_ = 13 def test_bad_length(self): @@ -190,15 +223,15 @@ class T(Array): _type_ = c_int _length_ = - sys.maxsize * 2 with self.assertRaises(ValueError): - class T(Array): + class T2(Array): _type_ = c_int _length_ = -1 with self.assertRaises(TypeError): - class T(Array): + class T3(Array): _type_ = c_int _length_ = 1.87 with self.assertRaises(OverflowError): - class T(Array): + class T4(Array): _type_ = c_int _length_ = sys.maxsize * 2 @@ -212,7 +245,7 @@ def test_empty_element_struct(self): class EmptyStruct(Structure): _fields_ = [] - obj = (EmptyStruct * 2)() # bpo37188: Floating point exception + obj = (EmptyStruct * 2)() # bpo37188: Floating-point exception self.assertEqual(sizeof(obj), 0) def test_empty_element_array(self): @@ -220,7 +253,7 @@ class EmptyArray(Array): _type_ = c_int _length_ = 0 - obj = (EmptyArray * 2)() # bpo37188: Floating point exception + obj = (EmptyArray * 2)() # bpo37188: Floating-point exception self.assertEqual(sizeof(obj), 0) def test_bpo36504_signed_int_overflow(self): @@ -234,5 +267,6 @@ def test_bpo36504_signed_int_overflow(self): def test_large_array(self, size): c_char * size + if __name__ == '__main__': unittest.main() diff --git a/Lib/ctypes/test/test_as_parameter.py b/Lib/test/test_ctypes/test_as_parameter.py similarity index 84% rename from Lib/ctypes/test/test_as_parameter.py rename to Lib/test/test_ctypes/test_as_parameter.py index 9c39179d2a..c5e1840b0e 100644 --- a/Lib/ctypes/test/test_as_parameter.py +++ b/Lib/test/test_ctypes/test_as_parameter.py @@ -1,24 +1,31 @@ +import ctypes import unittest -from ctypes import * -from ctypes.test import need_symbol -import _ctypes_test +from ctypes import (Structure, CDLL, CFUNCTYPE, + POINTER, pointer, byref, + c_short, c_int, c_long, c_longlong, + c_byte, c_wchar, c_float, c_double, + ArgumentError) +from test.support import import_helper +_ctypes_test = import_helper.import_module("_ctypes_test") + dll = CDLL(_ctypes_test.__file__) try: - CALLBACK_FUNCTYPE = WINFUNCTYPE -except NameError: + CALLBACK_FUNCTYPE = ctypes.WINFUNCTYPE +except AttributeError: # fake to enable this test on Linux CALLBACK_FUNCTYPE = CFUNCTYPE + class POINT(Structure): _fields_ = [("x", c_int), ("y", c_int)] + class BasicWrapTestCase(unittest.TestCase): def wrap(self, param): return param - @need_symbol('c_wchar') def test_wchar_parm(self): f = dll._testfunc_i_bhilfd f.argtypes = [c_byte, c_wchar, c_int, c_long, c_float, c_double] @@ -67,8 +74,6 @@ def callback(v): f(self.wrap(2**18), self.wrap(cb)) self.assertEqual(args, expected) - ################################################################ - def test_callbacks(self): f = dll._testfunc_callback_i_if f.restype = c_int @@ -77,7 +82,6 @@ def test_callbacks(self): MyCallback = CFUNCTYPE(c_int, c_int) def callback(value): - #print "called back with", value return value cb = MyCallback(callback) @@ -114,7 +118,6 @@ def test_callbacks_2(self): f.argtypes = [c_int, MyCallback] def callback(value): - #print "called back with", value self.assertEqual(type(value), int) return value @@ -122,9 +125,7 @@ def callback(value): result = f(self.wrap(-10), self.wrap(cb)) self.assertEqual(result, -18) - @need_symbol('c_longlong') def test_longlong_callbacks(self): - f = dll._testfunc_callback_q_qf f.restype = c_longlong @@ -192,29 +193,32 @@ class S8I(Structure): (9*2, 8*3, 7*4, 6*5, 5*6, 4*7, 3*8, 2*9)) def test_recursive_as_param(self): - from ctypes import c_int - - class A(object): + class A: pass a = A() a._as_parameter_ = a - with self.assertRaises(RecursionError): - c_int.from_param(a) - - -#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -class AsParamWrapper(object): + for c_type in ( + ctypes.c_wchar_p, + ctypes.c_char_p, + ctypes.c_void_p, + ctypes.c_int, # PyCSimpleType + POINT, # CDataType + ): + with self.subTest(c_type=c_type): + with self.assertRaises(RecursionError): + c_type.from_param(a) + + +class AsParamWrapper: def __init__(self, param): self._as_parameter_ = param class AsParamWrapperTestCase(BasicWrapTestCase): wrap = AsParamWrapper -#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ -class AsParamPropertyWrapper(object): +class AsParamPropertyWrapper: def __init__(self, param): self._param = param @@ -225,7 +229,17 @@ def getParameter(self): class AsParamPropertyWrapperTestCase(BasicWrapTestCase): wrap = AsParamPropertyWrapper -#~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +class AsParamNestedWrapperTestCase(BasicWrapTestCase): + """Test that _as_parameter_ is evaluated recursively. + + The _as_parameter_ attribute can be another object which + defines its own _as_parameter_ attribute. + """ + + def wrap(self, param): + return AsParamWrapper(AsParamWrapper(AsParamWrapper(param))) + if __name__ == '__main__': unittest.main() diff --git a/Lib/ctypes/test/test_bitfields.py b/Lib/test/test_ctypes/test_bitfields.py similarity index 95% rename from Lib/ctypes/test/test_bitfields.py rename to Lib/test/test_ctypes/test_bitfields.py index 66acd62e68..0332544b58 100644 --- a/Lib/ctypes/test/test_bitfields.py +++ b/Lib/test/test_ctypes/test_bitfields.py @@ -1,10 +1,14 @@ -from ctypes import * -from ctypes.test import need_symbol -from test import support -import unittest import os +import unittest +from ctypes import (CDLL, Structure, sizeof, POINTER, byref, alignment, + LittleEndianStructure, BigEndianStructure, + c_byte, c_ubyte, c_char, c_char_p, c_void_p, c_wchar, + c_uint32, c_uint64, + c_short, c_ushort, c_int, c_uint, c_long, c_ulong, c_longlong, c_ulonglong) +from test import support +from test.support import import_helper +_ctypes_test = import_helper.import_module("_ctypes_test") -import _ctypes_test class BITS(Structure): _fields_ = [("A", c_int, 1), @@ -28,8 +32,6 @@ class BITS(Structure): func = CDLL(_ctypes_test.__file__).unpack_bitfields func.argtypes = POINTER(BITS), c_char -##for n in "ABCDEFGHIMNOPQRS": -## print n, hex(getattr(BITS, n).size), getattr(BITS, n).offset class C_Test(unittest.TestCase): @@ -53,6 +55,7 @@ def test_shorts(self): setattr(b, name, i) self.assertEqual(getattr(b, name), func(byref(b), name.encode('ascii'))) + signed_int_types = (c_byte, c_short, c_int, c_long, c_longlong) unsigned_int_types = (c_ubyte, c_ushort, c_uint, c_ulong, c_ulonglong) int_types = unsigned_int_types + signed_int_types @@ -114,7 +117,6 @@ class X(Structure): x.a, x.b = 0, -1 self.assertEqual((c_typ, x.a, x.b, x.c), (c_typ, 0, 7, 0)) - def fail_fields(self, *fields): return self.get_except(type(Structure), "X", (), {"_fields_": fields}) @@ -140,7 +142,6 @@ class Dummy(Structure): result = self.fail_fields(("a", Dummy, 1)) self.assertEqual(result, (TypeError, 'bit fields not allowed for type Dummy')) - @need_symbol('c_wchar') def test_c_wchar(self): result = self.fail_fields(("a", c_wchar, 1)) self.assertEqual(result, @@ -192,7 +193,6 @@ class X(Structure): self.assertEqual(X.b.offset, sizeof(c_short)*1) self.assertEqual(X.c.offset, sizeof(c_short)*2) - def get_except(self, func, *args, **kw): try: func(*args, **kw) @@ -245,7 +245,6 @@ class Y(Structure): _anonymous_ = ["_"] _fields_ = [("_", X)] - @need_symbol('c_uint32') def test_uint32(self): class X(Structure): _fields_ = [("a", c_uint32, 32)] @@ -255,7 +254,6 @@ class X(Structure): x.a = 0xFDCBA987 self.assertEqual(x.a, 0xFDCBA987) - @need_symbol('c_uint64') def test_uint64(self): class X(Structure): _fields_ = [("a", c_uint64, 64)] @@ -265,7 +263,6 @@ class X(Structure): x.a = 0xFEDCBA9876543211 self.assertEqual(x.a, 0xFEDCBA9876543211) - @need_symbol('c_uint32') def test_uint32_swap_little_endian(self): # Issue #23319 class Little(LittleEndianStructure): @@ -279,7 +276,6 @@ class Little(LittleEndianStructure): x.c = 2 self.assertEqual(b, b'\xef\xcd\xab\x21') - @need_symbol('c_uint32') def test_uint32_swap_big_endian(self): # Issue #23319 class Big(BigEndianStructure): @@ -293,5 +289,6 @@ class Big(BigEndianStructure): x.c = 2 self.assertEqual(b, b'\xab\xcd\xef\x12') + if __name__ == "__main__": unittest.main() diff --git a/Lib/ctypes/test/test_buffers.py b/Lib/test/test_ctypes/test_buffers.py similarity index 94% rename from Lib/ctypes/test/test_buffers.py rename to Lib/test/test_ctypes/test_buffers.py index 15782be757..468f41eb7c 100644 --- a/Lib/ctypes/test/test_buffers.py +++ b/Lib/test/test_ctypes/test_buffers.py @@ -1,9 +1,9 @@ -from ctypes import * -from ctypes.test import need_symbol import unittest +from ctypes import (create_string_buffer, create_unicode_buffer, sizeof, + c_char, c_wchar) -class StringBufferTestCase(unittest.TestCase): +class StringBufferTestCase(unittest.TestCase): def test_buffer(self): b = create_string_buffer(32) self.assertEqual(len(b), 32) @@ -27,7 +27,6 @@ def test_buffer_interface(self): self.assertEqual(len(bytearray(create_string_buffer(0))), 0) self.assertEqual(len(bytearray(create_string_buffer(1))), 1) - @need_symbol('c_wchar') def test_unicode_buffer(self): b = create_unicode_buffer(32) self.assertEqual(len(b), 32) @@ -47,7 +46,6 @@ def test_unicode_buffer(self): self.assertRaises(TypeError, create_unicode_buffer, b"abc") - @need_symbol('c_wchar') def test_unicode_conversion(self): b = create_unicode_buffer("abc") self.assertEqual(len(b), 4) # trailing nul char @@ -60,7 +58,6 @@ def test_unicode_conversion(self): self.assertEqual(b[::2], "ac") self.assertEqual(b[::5], "a") - @need_symbol('c_wchar') def test_create_unicode_buffer_non_bmp(self): expected = 5 if sizeof(c_wchar) == 2 else 3 for s in '\U00010000\U00100000', '\U00010000\U0010ffff': diff --git a/Lib/ctypes/test/test_bytes.py b/Lib/test/test_ctypes/test_bytes.py similarity index 95% rename from Lib/ctypes/test/test_bytes.py rename to Lib/test/test_ctypes/test_bytes.py index 092ec5af05..fa11e1bbd4 100644 --- a/Lib/ctypes/test/test_bytes.py +++ b/Lib/test/test_ctypes/test_bytes.py @@ -1,7 +1,9 @@ """Test where byte objects are accepted""" -import unittest import sys -from ctypes import * +import unittest +from _ctypes import _SimpleCData +from ctypes import Structure, c_char, c_char_p, c_wchar, c_wchar_p + class BytesTest(unittest.TestCase): def test_c_char(self): @@ -55,7 +57,6 @@ class X(Structure): @unittest.skipUnless(sys.platform == "win32", 'Windows-specific test') def test_BSTR(self): - from _ctypes import _SimpleCData class BSTR(_SimpleCData): _type_ = "X" diff --git a/Lib/ctypes/test/test_byteswap.py b/Lib/test/test_ctypes/test_byteswap.py similarity index 92% rename from Lib/ctypes/test/test_byteswap.py rename to Lib/test/test_ctypes/test_byteswap.py index 7e98559dfb..78eff0392c 100644 --- a/Lib/ctypes/test/test_byteswap.py +++ b/Lib/test/test_ctypes/test_byteswap.py @@ -1,10 +1,21 @@ -import sys, unittest, struct, math, ctypes -from binascii import hexlify +import binascii +import ctypes +import math +import struct +import sys +import unittest +from ctypes import (Structure, Union, LittleEndianUnion, BigEndianUnion, + BigEndianStructure, LittleEndianStructure, + POINTER, sizeof, cast, + c_byte, c_ubyte, c_char, c_wchar, c_void_p, + c_short, c_ushort, c_int, c_uint, + c_long, c_ulong, c_longlong, c_ulonglong, + c_uint32, c_float, c_double) -from ctypes import * def bin(s): - return hexlify(memoryview(s)).decode().upper() + return binascii.hexlify(memoryview(s)).decode().upper() + # Each *simple* type that supports different byte orders has an # __ctype_be__ attribute that specifies the same type in BIG ENDIAN @@ -14,14 +25,6 @@ def bin(s): # For Structures and Unions, these types are created on demand. class Test(unittest.TestCase): - @unittest.skip('test disabled') - def test_X(self): - print(sys.byteorder, file=sys.stderr) - for i in range(32): - bits = BITS() - setattr(bits, "i%s" % i, 1) - dump(bits) - def test_slots(self): class BigPoint(BigEndianStructure): __slots__ = () @@ -360,5 +363,24 @@ class TestUnion(parent): self.assertEqual(s.point.x, 1) self.assertEqual(s.point.y, 2) + def test_build_struct_union_opposite_system_byteorder(self): + # gh-105102 + if sys.byteorder == "little": + _Structure = BigEndianStructure + _Union = BigEndianUnion + else: + _Structure = LittleEndianStructure + _Union = LittleEndianUnion + + class S1(_Structure): + _fields_ = [("a", c_byte), ("b", c_byte)] + + class U1(_Union): + _fields_ = [("s1", S1), ("ab", c_short)] + + class S2(_Structure): + _fields_ = [("u1", U1), ("c", c_byte)] + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_ctypes/test_c_simple_type_meta.py b/Lib/test/test_ctypes/test_c_simple_type_meta.py new file mode 100644 index 0000000000..eb77d6d778 --- /dev/null +++ b/Lib/test/test_ctypes/test_c_simple_type_meta.py @@ -0,0 +1,152 @@ +import unittest +import ctypes +from ctypes import POINTER, c_void_p + +from ._support import PyCSimpleType + + +class PyCSimpleTypeAsMetaclassTest(unittest.TestCase): + def tearDown(self): + # to not leak references, we must clean _pointer_type_cache + ctypes._reset_cache() + + def test_creating_pointer_in_dunder_new_1(self): + # Test metaclass whose instances are C types; when the type is + # created it automatically creates a pointer type for itself. + # The pointer type is also an instance of the metaclass. + # Such an implementation is used in `IUnknown` of the `comtypes` + # project. See gh-124520. + + class ct_meta(type): + def __new__(cls, name, bases, namespace): + self = super().__new__(cls, name, bases, namespace) + + # Avoid recursion: don't set up a pointer to + # a pointer (to a pointer...) + if bases == (c_void_p,): + # When creating PtrBase itself, the name + # is not yet available + return self + if issubclass(self, PtrBase): + return self + + if bases == (object,): + ptr_bases = (self, PtrBase) + else: + ptr_bases = (self, POINTER(bases[0])) + p = p_meta(f"POINTER({self.__name__})", ptr_bases, {}) + ctypes._pointer_type_cache[self] = p + return self + + class p_meta(PyCSimpleType, ct_meta): + pass + + class PtrBase(c_void_p, metaclass=p_meta): + pass + + class CtBase(object, metaclass=ct_meta): + pass + + class Sub(CtBase): + pass + + class Sub2(Sub): + pass + + self.assertIsInstance(POINTER(Sub2), p_meta) + self.assertTrue(issubclass(POINTER(Sub2), Sub2)) + self.assertTrue(issubclass(POINTER(Sub2), POINTER(Sub))) + self.assertTrue(issubclass(POINTER(Sub), POINTER(CtBase))) + + def test_creating_pointer_in_dunder_new_2(self): + # A simpler variant of the above, used in `CoClass` of the `comtypes` + # project. + + class ct_meta(type): + def __new__(cls, name, bases, namespace): + self = super().__new__(cls, name, bases, namespace) + if isinstance(self, p_meta): + return self + p = p_meta(f"POINTER({self.__name__})", (self, c_void_p), {}) + ctypes._pointer_type_cache[self] = p + return self + + class p_meta(PyCSimpleType, ct_meta): + pass + + class Core(object): + pass + + class CtBase(Core, metaclass=ct_meta): + pass + + class Sub(CtBase): + pass + + self.assertIsInstance(POINTER(Sub), p_meta) + self.assertTrue(issubclass(POINTER(Sub), Sub)) + + def test_creating_pointer_in_dunder_init_1(self): + class ct_meta(type): + def __init__(self, name, bases, namespace): + super().__init__(name, bases, namespace) + + # Avoid recursion. + # (See test_creating_pointer_in_dunder_new_1) + if bases == (c_void_p,): + return + if issubclass(self, PtrBase): + return + if bases == (object,): + ptr_bases = (self, PtrBase) + else: + ptr_bases = (self, POINTER(bases[0])) + p = p_meta(f"POINTER({self.__name__})", ptr_bases, {}) + ctypes._pointer_type_cache[self] = p + + class p_meta(PyCSimpleType, ct_meta): + pass + + class PtrBase(c_void_p, metaclass=p_meta): + pass + + class CtBase(object, metaclass=ct_meta): + pass + + class Sub(CtBase): + pass + + class Sub2(Sub): + pass + + self.assertIsInstance(POINTER(Sub2), p_meta) + self.assertTrue(issubclass(POINTER(Sub2), Sub2)) + self.assertTrue(issubclass(POINTER(Sub2), POINTER(Sub))) + self.assertTrue(issubclass(POINTER(Sub), POINTER(CtBase))) + + def test_creating_pointer_in_dunder_init_2(self): + class ct_meta(type): + def __init__(self, name, bases, namespace): + super().__init__(name, bases, namespace) + + # Avoid recursion. + # (See test_creating_pointer_in_dunder_new_2) + if isinstance(self, p_meta): + return + p = p_meta(f"POINTER({self.__name__})", (self, c_void_p), {}) + ctypes._pointer_type_cache[self] = p + + class p_meta(PyCSimpleType, ct_meta): + pass + + class Core(object): + pass + + class CtBase(Core, metaclass=ct_meta): + pass + + class Sub(CtBase): + pass + + self.assertIsInstance(POINTER(Sub), p_meta) + self.assertTrue(issubclass(POINTER(Sub), Sub)) diff --git a/Lib/ctypes/test/test_callbacks.py b/Lib/test/test_ctypes/test_callbacks.py similarity index 82% rename from Lib/ctypes/test/test_callbacks.py rename to Lib/test/test_ctypes/test_callbacks.py index 8f95a24443..8f483dfe1d 100644 --- a/Lib/ctypes/test/test_callbacks.py +++ b/Lib/test/test_ctypes/test_callbacks.py @@ -1,19 +1,25 @@ +import ctypes import functools +import gc +import math +import sys import unittest +from _ctypes import CTYPES_MAX_ARGCOUNT +from ctypes import (CDLL, cdll, Structure, CFUNCTYPE, + ArgumentError, POINTER, sizeof, + c_byte, c_ubyte, c_char, + c_short, c_ushort, c_int, c_uint, + c_long, c_longlong, c_ulonglong, c_ulong, + c_float, c_double, c_longdouble, py_object) +from ctypes.util import find_library from test import support +from test.support import import_helper +_ctypes_test = import_helper.import_module("_ctypes_test") -from ctypes import * -from ctypes.test import need_symbol -from _ctypes import CTYPES_MAX_ARGCOUNT -import _ctypes_test class Callbacks(unittest.TestCase): functype = CFUNCTYPE -## def tearDown(self): -## import gc -## gc.collect() - def callback(self, *args): self.got_args = args return args[-1] @@ -35,8 +41,6 @@ def check_type(self, typ, arg): self.assertEqual(self.got_args, (-3, arg)) self.assertEqual(result, arg) - ################ - def test_byte(self): self.check_type(c_byte, 42) self.check_type(c_byte, -42) @@ -65,18 +69,15 @@ def test_long(self): def test_ulong(self): self.check_type(c_ulong, 42) - @need_symbol('c_longlong') def test_longlong(self): self.check_type(c_longlong, 42) self.check_type(c_longlong, -42) - @need_symbol('c_ulonglong') def test_ulonglong(self): self.check_type(c_ulonglong, 42) def test_float(self): # only almost equal: double -> float -> double - import math self.check_type(c_float, math.e) self.check_type(c_float, -math.e) @@ -84,7 +85,6 @@ def test_double(self): self.check_type(c_double, 3.14) self.check_type(c_double, -3.14) - @need_symbol('c_longdouble') def test_longdouble(self): self.check_type(c_longdouble, 3.14) self.check_type(c_longdouble, -3.14) @@ -93,30 +93,21 @@ def test_char(self): self.check_type(c_char, b"x") self.check_type(c_char, b"a") - # disabled: would now (correctly) raise a RuntimeWarning about - # a memory leak. A callback function cannot return a non-integral - # C type without causing a memory leak. - @unittest.skip('test disabled') - def test_char_p(self): - self.check_type(c_char_p, "abc") - self.check_type(c_char_p, "def") - def test_pyobject(self): o = () - from sys import getrefcount as grc for o in (), [], object(): - initial = grc(o) + initial = sys.getrefcount(o) # This call leaks a reference to 'o'... self.check_type(py_object, o) - before = grc(o) + before = sys.getrefcount(o) # ...but this call doesn't leak any more. Where is the refcount? self.check_type(py_object, o) - after = grc(o) + after = sys.getrefcount(o) self.assertEqual((after, o), (before, o)) def test_unsupported_restype_1(self): # Only "fundamental" result types are supported for callback - # functions, the type must have a non-NULL stgdict->setfunc. + # functions, the type must have a non-NULL stginfo->setfunc. # POINTER(c_double), for example, is not supported. prototype = self.functype.__func__(POINTER(c_double)) @@ -130,12 +121,11 @@ def test_unsupported_restype_2(self): def test_issue_7959(self): proto = self.functype.__func__(None) - class X(object): + class X: def func(self): pass def __init__(self): self.v = proto(self.func) - import gc for i in range(32): X() gc.collect() @@ -144,21 +134,30 @@ def __init__(self): self.assertEqual(len(live), 0) def test_issue12483(self): - import gc class Nasty: def __del__(self): gc.collect() CFUNCTYPE(None)(lambda x=Nasty(): None) + @unittest.skipUnless(hasattr(ctypes, 'WINFUNCTYPE'), + 'ctypes.WINFUNCTYPE is required') + def test_i38748_stackCorruption(self): + callback_funcType = ctypes.WINFUNCTYPE(c_long, c_long, c_longlong) + @callback_funcType + def callback(a, b): + c = a + b + print(f"a={a}, b={b}, c={c}") + return c + dll = cdll[_ctypes_test.__file__] + with support.captured_stdout() as out: + # With no fix for i38748, the next line will raise OSError and cause the test to fail. + self.assertEqual(dll._test_i38748_runCallback(callback, 5, 10), 15) + self.assertEqual(out.getvalue(), "a=5, b=10, c=15\n") + +if hasattr(ctypes, 'WINFUNCTYPE'): + class StdcallCallbacks(Callbacks): + functype = ctypes.WINFUNCTYPE -@need_symbol('WINFUNCTYPE') -class StdcallCallbacks(Callbacks): - try: - functype = WINFUNCTYPE - except NameError: - pass - -################################################################ class SampleCallbacksTestCase(unittest.TestCase): @@ -183,7 +182,6 @@ def func(x): self.assertLess(diff, 0.01, "%s not less than 0.01" % diff) def test_issue_8959_a(self): - from ctypes.util import find_library libc_path = find_library("c") if not libc_path: self.skipTest('could not find libc') @@ -198,19 +196,21 @@ def cmp_func(a, b): libc.qsort(array, len(array), sizeof(c_int), cmp_func) self.assertEqual(array[:], [1, 5, 7, 33, 99]) - @need_symbol('WINFUNCTYPE') + @unittest.skipUnless(hasattr(ctypes, 'WINFUNCTYPE'), + 'ctypes.WINFUNCTYPE is required') def test_issue_8959_b(self): from ctypes.wintypes import BOOL, HWND, LPARAM global windowCount windowCount = 0 - @WINFUNCTYPE(BOOL, HWND, LPARAM) + @ctypes.WINFUNCTYPE(BOOL, HWND, LPARAM) def EnumWindowsCallbackFunc(hwnd, lParam): global windowCount windowCount += 1 return True #Allow windows to keep enumerating - windll.user32.EnumWindows(EnumWindowsCallbackFunc, 0) + user32 = ctypes.windll.user32 + user32.EnumWindows(EnumWindowsCallbackFunc, 0) def test_callback_register_int(self): # Issue #8275: buggy handling of callback args under Win64 @@ -324,9 +324,9 @@ def func(): self.assertIsInstance(cm.unraisable.exc_value, TypeError) self.assertEqual(cm.unraisable.err_msg, - "Exception ignored on converting result " - "of ctypes callback function") - self.assertIs(cm.unraisable.object, func) + f"Exception ignored on converting result " + f"of ctypes callback function {func!r}") + self.assertIsNone(cm.unraisable.object) if __name__ == '__main__': diff --git a/Lib/ctypes/test/test_cast.py b/Lib/test/test_ctypes/test_cast.py similarity index 92% rename from Lib/ctypes/test/test_cast.py rename to Lib/test/test_ctypes/test_cast.py index 6878f97328..db6bdc75ef 100644 --- a/Lib/ctypes/test/test_cast.py +++ b/Lib/test/test_ctypes/test_cast.py @@ -1,10 +1,11 @@ -from ctypes import * -from ctypes.test import need_symbol -import unittest import sys +import unittest +from ctypes import (Structure, Union, POINTER, cast, sizeof, addressof, + c_void_p, c_char_p, c_wchar_p, + c_byte, c_short, c_int) -class Test(unittest.TestCase): +class Test(unittest.TestCase): def test_array2pointer(self): array = (c_int * 3)(42, 17, 2) @@ -12,7 +13,7 @@ def test_array2pointer(self): ptr = cast(array, POINTER(c_int)) self.assertEqual([ptr[i] for i in range(3)], [42, 17, 2]) - if 2*sizeof(c_short) == sizeof(c_int): + if 2 * sizeof(c_short) == sizeof(c_int): ptr = cast(array, POINTER(c_short)) if sys.byteorder == "little": self.assertEqual([ptr[i] for i in range(6)], @@ -31,6 +32,8 @@ def test_address2pointer(self): ptr = cast(address, POINTER(c_int)) self.assertEqual([ptr[i] for i in range(3)], [42, 17, 2]) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_p2a_objects(self): array = (c_char_p * 5)() self.assertEqual(array._objects, None) @@ -76,11 +79,10 @@ def test_char_p(self): self.assertEqual(cast(cast(s, c_void_p), c_char_p).value, b"hiho") - @need_symbol('c_wchar_p') def test_wchar_p(self): s = c_wchar_p("hiho") self.assertEqual(cast(cast(s, c_void_p), c_wchar_p).value, - "hiho") + "hiho") def test_bad_type_arg(self): # The type argument must be a ctypes pointer type. @@ -95,5 +97,6 @@ class MyUnion(Union): _fields_ = [("a", c_int)] self.assertRaises(TypeError, cast, array, MyUnion) + if __name__ == "__main__": unittest.main() diff --git a/Lib/ctypes/test/test_cfuncs.py b/Lib/test/test_ctypes/test_cfuncs.py similarity index 93% rename from Lib/ctypes/test/test_cfuncs.py rename to Lib/test/test_ctypes/test_cfuncs.py index 09b06840bf..48330c4b0a 100644 --- a/Lib/ctypes/test/test_cfuncs.py +++ b/Lib/test/test_ctypes/test_cfuncs.py @@ -1,11 +1,13 @@ -# A lot of failures in these tests on Mac OS X. -# Byte order related? - +import ctypes import unittest -from ctypes import * -from ctypes.test import need_symbol +from ctypes import (CDLL, + c_byte, c_ubyte, c_char, + c_short, c_ushort, c_int, c_uint, + c_long, c_ulong, c_longlong, c_ulonglong, + c_float, c_double, c_longdouble) +from test.support import import_helper +_ctypes_test = import_helper.import_module("_ctypes_test") -import _ctypes_test class CFunctions(unittest.TestCase): _dll = CDLL(_ctypes_test.__file__) @@ -111,28 +113,24 @@ def test_ulong_plus(self): self.assertEqual(self._dll.tf_bL(b' ', 4294967295), 1431655765) self.assertEqual(self.U(), 4294967295) - @need_symbol('c_longlong') def test_longlong(self): self._dll.tf_q.restype = c_longlong self._dll.tf_q.argtypes = (c_longlong, ) self.assertEqual(self._dll.tf_q(-9223372036854775806), -3074457345618258602) self.assertEqual(self.S(), -9223372036854775806) - @need_symbol('c_longlong') def test_longlong_plus(self): self._dll.tf_bq.restype = c_longlong self._dll.tf_bq.argtypes = (c_byte, c_longlong) self.assertEqual(self._dll.tf_bq(0, -9223372036854775806), -3074457345618258602) self.assertEqual(self.S(), -9223372036854775806) - @need_symbol('c_ulonglong') def test_ulonglong(self): self._dll.tf_Q.restype = c_ulonglong self._dll.tf_Q.argtypes = (c_ulonglong, ) self.assertEqual(self._dll.tf_Q(18446744073709551615), 6148914691236517205) self.assertEqual(self.U(), 18446744073709551615) - @need_symbol('c_ulonglong') def test_ulonglong_plus(self): self._dll.tf_bQ.restype = c_ulonglong self._dll.tf_bQ.argtypes = (c_byte, c_ulonglong) @@ -163,14 +161,12 @@ def test_double_plus(self): self.assertEqual(self._dll.tf_bd(0, 42.), 14.) self.assertEqual(self.S(), 42) - @need_symbol('c_longdouble') def test_longdouble(self): self._dll.tf_D.restype = c_longdouble self._dll.tf_D.argtypes = (c_longdouble,) self.assertEqual(self._dll.tf_D(42.), 14.) self.assertEqual(self.S(), 42) - @need_symbol('c_longdouble') def test_longdouble_plus(self): self._dll.tf_bD.restype = c_longdouble self._dll.tf_bD.argtypes = (c_byte, c_longdouble) @@ -195,14 +191,11 @@ def test_void(self): self.assertEqual(self._dll.tv_i(-42), None) self.assertEqual(self.S(), -42) + # The following repeats the above tests with stdcall functions (where # they are available) -try: - WinDLL -except NameError: - def stdcall_dll(*_): pass -else: - class stdcall_dll(WinDLL): +if hasattr(ctypes, 'WinDLL'): + class stdcall_dll(ctypes.WinDLL): def __getattr__(self, name): if name[:2] == '__' and name[-2:] == '__': raise AttributeError(name) @@ -210,9 +203,9 @@ def __getattr__(self, name): setattr(self, name, func) return func -@need_symbol('WinDLL') -class stdcallCFunctions(CFunctions): - _dll = stdcall_dll(_ctypes_test.__file__) + class stdcallCFunctions(CFunctions): + _dll = stdcall_dll(_ctypes_test.__file__) + if __name__ == '__main__': unittest.main() diff --git a/Lib/ctypes/test/test_checkretval.py b/Lib/test/test_ctypes/test_checkretval.py similarity index 66% rename from Lib/ctypes/test/test_checkretval.py rename to Lib/test/test_ctypes/test_checkretval.py index e9567dc391..9d6bfdb845 100644 --- a/Lib/ctypes/test/test_checkretval.py +++ b/Lib/test/test_ctypes/test_checkretval.py @@ -1,7 +1,9 @@ +import ctypes import unittest +from ctypes import CDLL, c_int +from test.support import import_helper +_ctypes_test = import_helper.import_module("_ctypes_test") -from ctypes import * -from ctypes.test import need_symbol class CHECKED(c_int): def _check_retval_(value): @@ -9,11 +11,9 @@ def _check_retval_(value): return str(value.value) _check_retval_ = staticmethod(_check_retval_) -class Test(unittest.TestCase): +class Test(unittest.TestCase): def test_checkretval(self): - - import _ctypes_test dll = CDLL(_ctypes_test.__file__) self.assertEqual(42, dll._testfunc_p_p(42)) @@ -26,11 +26,12 @@ def test_checkretval(self): del dll._testfunc_p_p.restype self.assertEqual(42, dll._testfunc_p_p(42)) - @need_symbol('oledll') + @unittest.skipUnless(hasattr(ctypes, 'oledll'), + 'ctypes.oledll is required') def test_oledll(self): - self.assertRaises(OSError, - oledll.oleaut32.CreateTypeLib2, - 0, None, None) + oleaut32 = ctypes.oledll.oleaut32 + self.assertRaises(OSError, oleaut32.CreateTypeLib2, 0, None, None) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_ctypes/test_delattr.py b/Lib/test/test_ctypes/test_delattr.py new file mode 100644 index 0000000000..e80b5fa6ef --- /dev/null +++ b/Lib/test/test_ctypes/test_delattr.py @@ -0,0 +1,26 @@ +import unittest +from ctypes import Structure, c_char, c_int + + +class X(Structure): + _fields_ = [("foo", c_int)] + + +class TestCase(unittest.TestCase): + def test_simple(self): + with self.assertRaises(TypeError): + del c_int(42).value + + def test_chararray(self): + chararray = (c_char * 5)() + with self.assertRaises(TypeError): + del chararray.value + + def test_struct(self): + struct = X() + with self.assertRaises(TypeError): + del struct.foo + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_ctypes/test_dlerror.py b/Lib/test/test_ctypes/test_dlerror.py new file mode 100644 index 0000000000..cd87bad382 --- /dev/null +++ b/Lib/test/test_ctypes/test_dlerror.py @@ -0,0 +1,180 @@ +import _ctypes +import os +import platform +import sys +import test.support +import unittest +from ctypes import CDLL, c_int +from ctypes.util import find_library + + +FOO_C = r""" +#include + +/* This is a 'GNU indirect function' (IFUNC) that will be called by + dlsym() to resolve the symbol "foo" to an address. Typically, such + a function would return the address of an actual function, but it + can also just return NULL. For some background on IFUNCs, see + https://willnewton.name/uncategorized/using-gnu-indirect-functions. + + Adapted from Michael Kerrisk's answer: https://stackoverflow.com/a/53590014. +*/ + +asm (".type foo STT_GNU_IFUNC"); + +void *foo(void) +{ + write($DESCRIPTOR, "OK", 2); + return NULL; +} +""" + + +@unittest.skipUnless(sys.platform.startswith('linux'), + 'test requires GNU IFUNC support') +class TestNullDlsym(unittest.TestCase): + """GH-126554: Ensure that we catch NULL dlsym return values + + In rare cases, such as when using GNU IFUNCs, dlsym(), + the C function that ctypes' CDLL uses to get the address + of symbols, can return NULL. + + The objective way of telling if an error during symbol + lookup happened is to call glibc's dlerror() and check + for a non-NULL return value. + + However, there can be cases where dlsym() returns NULL + and dlerror() is also NULL, meaning that glibc did not + encounter any error. + + In the case of ctypes, we subjectively treat that as + an error, and throw a relevant exception. + + This test case ensures that we correctly enforce + this 'dlsym returned NULL -> throw Error' rule. + """ + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_null_dlsym(self): + import subprocess + import tempfile + + try: + retcode = subprocess.call(["gcc", "--version"], + stdout=subprocess.DEVNULL, + stderr=subprocess.DEVNULL) + except OSError: + self.skipTest("gcc is missing") + if retcode != 0: + self.skipTest("gcc --version failed") + + pipe_r, pipe_w = os.pipe() + self.addCleanup(os.close, pipe_r) + self.addCleanup(os.close, pipe_w) + + with tempfile.TemporaryDirectory() as d: + # Create a C file with a GNU Indirect Function (FOO_C) + # and compile it into a shared library. + srcname = os.path.join(d, 'foo.c') + dstname = os.path.join(d, 'libfoo.so') + with open(srcname, 'w') as f: + f.write(FOO_C.replace('$DESCRIPTOR', str(pipe_w))) + args = ['gcc', '-fPIC', '-shared', '-o', dstname, srcname] + p = subprocess.run(args, capture_output=True) + + if p.returncode != 0: + # IFUNC is not supported on all architectures. + if platform.machine() == 'x86_64': + # It should be supported here. Something else went wrong. + p.check_returncode() + else: + # IFUNC might not be supported on this machine. + self.skipTest(f"could not compile indirect function: {p}") + + # Case #1: Test 'PyCFuncPtr_FromDll' from Modules/_ctypes/_ctypes.c + L = CDLL(dstname) + with self.assertRaisesRegex(AttributeError, "function 'foo' not found"): + # Try accessing the 'foo' symbol. + # It should resolve via dlsym() to NULL, + # and since we subjectively treat NULL + # addresses as errors, we should get + # an error. + L.foo + + # Assert that the IFUNC was called + self.assertEqual(os.read(pipe_r, 2), b'OK') + + # Case #2: Test 'CDataType_in_dll_impl' from Modules/_ctypes/_ctypes.c + with self.assertRaisesRegex(ValueError, "symbol 'foo' not found"): + c_int.in_dll(L, "foo") + + # Assert that the IFUNC was called + self.assertEqual(os.read(pipe_r, 2), b'OK') + + # Case #3: Test 'py_dl_sym' from Modules/_ctypes/callproc.c + dlopen = test.support.get_attribute(_ctypes, 'dlopen') + dlsym = test.support.get_attribute(_ctypes, 'dlsym') + L = dlopen(dstname) + with self.assertRaisesRegex(OSError, "symbol 'foo' not found"): + dlsym(L, "foo") + + # Assert that the IFUNC was called + self.assertEqual(os.read(pipe_r, 2), b'OK') + + +@unittest.skipUnless(os.name != 'nt', 'test requires dlerror() calls') +class TestLocalization(unittest.TestCase): + + @staticmethod + def configure_locales(func): + return test.support.run_with_locale( + 'LC_ALL', + 'fr_FR.iso88591', 'ja_JP.sjis', 'zh_CN.gbk', + 'fr_FR.utf8', 'en_US.utf8', + '', + )(func) + + @classmethod + def setUpClass(cls): + cls.libc_filename = find_library("c") + if cls.libc_filename is None: + raise unittest.SkipTest('cannot find libc') + + @configure_locales + def test_localized_error_from_dll(self): + dll = CDLL(self.libc_filename) + with self.assertRaises(AttributeError): + dll.this_name_does_not_exist + + @configure_locales + def test_localized_error_in_dll(self): + dll = CDLL(self.libc_filename) + with self.assertRaises(ValueError): + c_int.in_dll(dll, 'this_name_does_not_exist') + + @unittest.skipUnless(hasattr(_ctypes, 'dlopen'), + 'test requires _ctypes.dlopen()') + @configure_locales + def test_localized_error_dlopen(self): + missing_filename = b'missing\xff.so' + # Depending whether the locale, we may encode '\xff' differently + # but we are only interested in avoiding a UnicodeDecodeError + # when reporting the dlerror() error message which contains + # the localized filename. + filename_pattern = r'missing.*?\.so' + with self.assertRaisesRegex(OSError, filename_pattern): + _ctypes.dlopen(missing_filename, 2) + + @unittest.skipUnless(hasattr(_ctypes, 'dlopen'), + 'test requires _ctypes.dlopen()') + @unittest.skipUnless(hasattr(_ctypes, 'dlsym'), + 'test requires _ctypes.dlsym()') + @configure_locales + def test_localized_error_dlsym(self): + dll = _ctypes.dlopen(self.libc_filename) + with self.assertRaises(OSError): + _ctypes.dlsym(dll, 'this_name_does_not_exist') + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/ctypes/test/test_errno.py b/Lib/test/test_ctypes/test_errno.py similarity index 71% rename from Lib/ctypes/test/test_errno.py rename to Lib/test/test_ctypes/test_errno.py index 3685164dde..65d99c1e49 100644 --- a/Lib/ctypes/test/test_errno.py +++ b/Lib/test/test_ctypes/test_errno.py @@ -1,14 +1,18 @@ -import unittest, os, errno +import ctypes +import errno +import os import threading - -from ctypes import * +import unittest +from ctypes import CDLL, c_int, c_char_p, c_wchar_p, get_errno, set_errno from ctypes.util import find_library + class Test(unittest.TestCase): def test_open(self): libc_name = find_library("c") if libc_name is None: - raise unittest.SkipTest("Unable to find C library") + self.skipTest("Unable to find C library") + libc = CDLL(libc_name, use_errno=True) if os.name == "nt": libc_open = libc._open @@ -44,33 +48,34 @@ def _worker(): @unittest.skipUnless(os.name == "nt", 'Test specific to Windows') def test_GetLastError(self): - dll = WinDLL("kernel32", use_last_error=True) + dll = ctypes.WinDLL("kernel32", use_last_error=True) GetModuleHandle = dll.GetModuleHandleA GetModuleHandle.argtypes = [c_wchar_p] self.assertEqual(0, GetModuleHandle("foo")) - self.assertEqual(get_last_error(), 126) + self.assertEqual(ctypes.get_last_error(), 126) - self.assertEqual(set_last_error(32), 126) - self.assertEqual(get_last_error(), 32) + self.assertEqual(ctypes.set_last_error(32), 126) + self.assertEqual(ctypes.get_last_error(), 32) def _worker(): - set_last_error(0) + ctypes.set_last_error(0) - dll = WinDLL("kernel32", use_last_error=False) + dll = ctypes.WinDLL("kernel32", use_last_error=False) GetModuleHandle = dll.GetModuleHandleW GetModuleHandle.argtypes = [c_wchar_p] GetModuleHandle("bar") - self.assertEqual(get_last_error(), 0) + self.assertEqual(ctypes.get_last_error(), 0) t = threading.Thread(target=_worker) t.start() t.join() - self.assertEqual(get_last_error(), 32) + self.assertEqual(ctypes.get_last_error(), 32) + + ctypes.set_last_error(0) - set_last_error(0) if __name__ == "__main__": unittest.main() diff --git a/Lib/ctypes/test/test_find.py b/Lib/test/test_ctypes/test_find.py similarity index 80% rename from Lib/ctypes/test/test_find.py rename to Lib/test/test_ctypes/test_find.py index 1ff9d019b1..85b28617d2 100644 --- a/Lib/ctypes/test/test_find.py +++ b/Lib/test/test_ctypes/test_find.py @@ -1,11 +1,12 @@ -import unittest -import unittest.mock import os.path import sys import test.support -from test.support import os_helper -from ctypes import * +import unittest +import unittest.mock +from ctypes import CDLL, RTLD_GLOBAL from ctypes.util import find_library +from test.support import os_helper + # On some systems, loading the OpenGL libraries needs the RTLD_GLOBAL mode. class Test_OpenGL_libs(unittest.TestCase): @@ -22,7 +23,7 @@ def setUpClass(cls): lib_glu = find_library("GLU") lib_gle = find_library("gle") - ## print, for debugging + # print, for debugging if test.support.verbose: print("OpenGL libraries:") for item in (("GL", lib_gl), @@ -36,11 +37,13 @@ def setUpClass(cls): cls.gl = CDLL(lib_gl, mode=RTLD_GLOBAL) except OSError: pass + if lib_glu: try: cls.glu = CDLL(lib_glu, RTLD_GLOBAL) except OSError: pass + if lib_gle: try: cls.gle = CDLL(lib_gle) @@ -122,6 +125,32 @@ def test_find_library_with_ld(self): unittest.mock.patch("ctypes.util._findLib_gcc", lambda *args: None): self.assertNotEqual(find_library('c'), None) + def test_gh114257(self): + self.assertIsNone(find_library("libc")) + + +@unittest.skipUnless(sys.platform == 'android', 'Test only valid for Android') +class FindLibraryAndroid(unittest.TestCase): + def test_find(self): + for name in [ + "c", "m", # POSIX + "z", # Non-POSIX, but present on Linux + "log", # Not present on Linux + ]: + with self.subTest(name=name): + path = find_library(name) + self.assertIsInstance(path, str) + self.assertEqual( + os.path.dirname(path), + "/system/lib64" if "64" in os.uname().machine + else "/system/lib") + self.assertEqual(os.path.basename(path), f"lib{name}.so") + self.assertTrue(os.path.isfile(path), path) + + for name in ["libc", "nonexistent"]: + with self.subTest(name=name): + self.assertIsNone(find_library(name)) + if __name__ == "__main__": unittest.main() diff --git a/Lib/ctypes/test/test_frombuffer.py b/Lib/test/test_ctypes/test_frombuffer.py similarity index 97% rename from Lib/ctypes/test/test_frombuffer.py rename to Lib/test/test_ctypes/test_frombuffer.py index 55c244356b..d4e161f864 100644 --- a/Lib/ctypes/test/test_frombuffer.py +++ b/Lib/test/test_ctypes/test_frombuffer.py @@ -1,7 +1,10 @@ -from ctypes import * import array import gc import unittest +from ctypes import (Structure, Union, Array, sizeof, + _Pointer, _SimpleCData, _CFuncPtr, + c_char, c_int) + class X(Structure): _fields_ = [("c_int", c_int)] @@ -9,6 +12,7 @@ class X(Structure): def __init__(self): self._init_called = True + class Test(unittest.TestCase): def test_from_buffer(self): a = array.array("i", range(16)) @@ -121,8 +125,6 @@ def test_from_buffer_copy_with_offset(self): (c_int * 1).from_buffer_copy(a, 16 * sizeof(c_int)) def test_abstract(self): - from ctypes import _Pointer, _SimpleCData, _CFuncPtr - self.assertRaises(TypeError, Array.from_buffer, bytearray(10)) self.assertRaises(TypeError, Structure.from_buffer, bytearray(10)) self.assertRaises(TypeError, Union.from_buffer, bytearray(10)) @@ -137,5 +139,6 @@ def test_abstract(self): self.assertRaises(TypeError, _Pointer.from_buffer_copy, b"123") self.assertRaises(TypeError, _SimpleCData.from_buffer_copy, b"123") + if __name__ == '__main__': unittest.main() diff --git a/Lib/ctypes/test/test_funcptr.py b/Lib/test/test_ctypes/test_funcptr.py similarity index 68% rename from Lib/ctypes/test/test_funcptr.py rename to Lib/test/test_ctypes/test_funcptr.py index e0b9b54e97..8362fb16d9 100644 --- a/Lib/ctypes/test/test_funcptr.py +++ b/Lib/test/test_ctypes/test_funcptr.py @@ -1,16 +1,41 @@ +import ctypes import unittest -from ctypes import * +from ctypes import (CDLL, Structure, CFUNCTYPE, sizeof, _CFuncPtr, + c_void_p, c_char_p, c_char, c_int, c_uint, c_long) +from test.support import import_helper +_ctypes_test = import_helper.import_module("_ctypes_test") +from ._support import (_CData, PyCFuncPtrType, Py_TPFLAGS_DISALLOW_INSTANTIATION, + Py_TPFLAGS_IMMUTABLETYPE) + try: - WINFUNCTYPE -except NameError: + WINFUNCTYPE = ctypes.WINFUNCTYPE +except AttributeError: # fake to enable this test on Linux WINFUNCTYPE = CFUNCTYPE -import _ctypes_test lib = CDLL(_ctypes_test.__file__) + class CFuncPtrTestCase(unittest.TestCase): + def test_inheritance_hierarchy(self): + self.assertEqual(_CFuncPtr.mro(), [_CFuncPtr, _CData, object]) + + self.assertEqual(PyCFuncPtrType.__name__, "PyCFuncPtrType") + self.assertEqual(type(PyCFuncPtrType), type) + + def test_type_flags(self): + for cls in _CFuncPtr, PyCFuncPtrType: + with self.subTest(cls=cls): + self.assertTrue(_CFuncPtr.__flags__ & Py_TPFLAGS_IMMUTABLETYPE) + self.assertFalse(_CFuncPtr.__flags__ & Py_TPFLAGS_DISALLOW_INSTANTIATION) + + def test_metaclass_details(self): + # Cannot call the metaclass __init__ more than once + CdeclCallback = CFUNCTYPE(c_int, c_int, c_int) + with self.assertRaisesRegex(SystemError, "already initialized"): + PyCFuncPtrType.__init__(CdeclCallback, 'ptr', (), {}) + def test_basic(self): X = WINFUNCTYPE(c_int, c_int, c_int) @@ -20,8 +45,8 @@ def func(*args): x = X(func) self.assertEqual(x.restype, c_int) self.assertEqual(x.argtypes, (c_int, c_int)) - self.assertEqual(sizeof(x), sizeof(c_voidp)) - self.assertEqual(sizeof(X), sizeof(c_voidp)) + self.assertEqual(sizeof(x), sizeof(c_void_p)) + self.assertEqual(sizeof(X), sizeof(c_void_p)) def test_first(self): StdCallback = WINFUNCTYPE(c_int, c_int, c_int) @@ -39,7 +64,7 @@ def func(a, b): # possible, as in C, to call cdecl functions with more parameters. #self.assertRaises(TypeError, c, 1, 2, 3) self.assertEqual(c(1, 2, 3, 4, 5, 6), 3) - if not WINFUNCTYPE is CFUNCTYPE: + if WINFUNCTYPE is not CFUNCTYPE: self.assertRaises(TypeError, s, 1, 2, 3) def test_structures(self): @@ -69,17 +94,9 @@ class WNDCLASS(Structure): WNDPROC_2 = WINFUNCTYPE(c_long, c_int, c_int, c_int, c_int) - # This is no longer true, now that WINFUNCTYPE caches created types internally. - ## # CFuncPtr subclasses are compared by identity, so this raises a TypeError: - ## self.assertRaises(TypeError, setattr, wndclass, - ## "lpfnWndProc", WNDPROC_2(wndproc)) - # instead: - self.assertIs(WNDPROC, WNDPROC_2) - # 'wndclass.lpfnWndProc' leaks 94 references. Why? self.assertEqual(wndclass.lpfnWndProc(1, 2, 3, 4), 10) - f = wndclass.lpfnWndProc del wndclass @@ -88,24 +105,14 @@ class WNDCLASS(Structure): self.assertEqual(f(10, 11, 12, 13), 46) def test_dllfunctions(self): - - def NoNullHandle(value): - if not value: - raise WinError() - return value - strchr = lib.my_strchr strchr.restype = c_char_p strchr.argtypes = (c_char_p, c_char) self.assertEqual(strchr(b"abcdefghi", b"b"), b"bcdefghi") self.assertEqual(strchr(b"abcdefghi", b"x"), None) - strtok = lib.my_strtok strtok.restype = c_char_p - # Neither of this does work: strtok changes the buffer it is passed -## strtok.argtypes = (c_char_p, c_char_p) -## strtok.argtypes = (c_string, c_char_p) def c_string(init): size = len(init) + 1 @@ -114,19 +121,14 @@ def c_string(init): s = b"a\nb\nc" b = c_string(s) -## b = (c_char * (len(s)+1))() -## b.value = s - -## b = c_string(s) self.assertEqual(strtok(b, b"\n"), b"a") self.assertEqual(strtok(None, b"\n"), b"b") self.assertEqual(strtok(None, b"\n"), b"c") self.assertEqual(strtok(None, b"\n"), None) def test_abstract(self): - from ctypes import _CFuncPtr - self.assertRaises(TypeError, _CFuncPtr, 13, "name", 42, "iid") + if __name__ == '__main__': unittest.main() diff --git a/Lib/ctypes/test/test_functions.py b/Lib/test/test_ctypes/test_functions.py similarity index 76% rename from Lib/ctypes/test/test_functions.py rename to Lib/test/test_ctypes/test_functions.py index fc571700ce..63e393f7b7 100644 --- a/Lib/ctypes/test/test_functions.py +++ b/Lib/test/test_ctypes/test_functions.py @@ -1,30 +1,36 @@ -""" -Here is probably the place to write the docs, since the test-cases -show how the type behave. +import ctypes +import sys +import unittest +from ctypes import (CDLL, Structure, Array, CFUNCTYPE, + byref, POINTER, pointer, ArgumentError, + c_char, c_wchar, c_byte, c_char_p, c_wchar_p, + c_short, c_int, c_long, c_longlong, c_void_p, + c_float, c_double, c_longdouble) +from test.support import import_helper +_ctypes_test = import_helper.import_module("_ctypes_test") +from _ctypes import _Pointer, _SimpleCData -Later... -""" - -from ctypes import * -from ctypes.test import need_symbol -import sys, unittest try: - WINFUNCTYPE -except NameError: + WINFUNCTYPE = ctypes.WINFUNCTYPE +except AttributeError: # fake to enable this test on Linux WINFUNCTYPE = CFUNCTYPE -import _ctypes_test dll = CDLL(_ctypes_test.__file__) if sys.platform == "win32": - windll = WinDLL(_ctypes_test.__file__) + windll = ctypes.WinDLL(_ctypes_test.__file__) + class POINT(Structure): _fields_ = [("x", c_int), ("y", c_int)] + + class RECT(Structure): _fields_ = [("left", c_int), ("top", c_int), ("right", c_int), ("bottom", c_int)] + + class FunctionTestCase(unittest.TestCase): def test_mro(self): @@ -40,21 +46,34 @@ class X(object, Array): _length_ = 5 _type_ = "i" - from _ctypes import _Pointer with self.assertRaises(TypeError): - class X(object, _Pointer): + class X2(object, _Pointer): pass - from _ctypes import _SimpleCData with self.assertRaises(TypeError): - class X(object, _SimpleCData): + class X3(object, _SimpleCData): _type_ = "i" with self.assertRaises(TypeError): - class X(object, Structure): + class X4(object, Structure): _fields_ = [] - @need_symbol('c_wchar') + def test_c_char_parm(self): + proto = CFUNCTYPE(c_int, c_char) + def callback(*args): + return 0 + + callback = proto(callback) + + self.assertEqual(callback(b"a"), 0) + + with self.assertRaises(ArgumentError) as cm: + callback(b"abc") + + self.assertEqual(str(cm.exception), + "argument 1: TypeError: one character bytes, " + "bytearray or integer expected") + def test_wchar_parm(self): f = dll._testfunc_i_bhilfd f.argtypes = [c_byte, c_wchar, c_int, c_long, c_float, c_double] @@ -62,7 +81,66 @@ def test_wchar_parm(self): self.assertEqual(result, 139) self.assertEqual(type(result), int) - @need_symbol('c_wchar') + with self.assertRaises(ArgumentError) as cm: + f(1, 2, 3, 4, 5.0, 6.0) + self.assertEqual(str(cm.exception), + "argument 2: TypeError: unicode string expected " + "instead of int instance") + + with self.assertRaises(ArgumentError) as cm: + f(1, "abc", 3, 4, 5.0, 6.0) + self.assertEqual(str(cm.exception), + "argument 2: TypeError: one character unicode string " + "expected") + + def test_c_char_p_parm(self): + """Test the error message when converting an incompatible type to c_char_p.""" + proto = CFUNCTYPE(c_int, c_char_p) + def callback(*args): + return 0 + + callback = proto(callback) + self.assertEqual(callback(b"abc"), 0) + + with self.assertRaises(ArgumentError) as cm: + callback(10) + + self.assertEqual(str(cm.exception), + "argument 1: TypeError: 'int' object cannot be " + "interpreted as ctypes.c_char_p") + + def test_c_wchar_p_parm(self): + """Test the error message when converting an incompatible type to c_wchar_p.""" + proto = CFUNCTYPE(c_int, c_wchar_p) + def callback(*args): + return 0 + + callback = proto(callback) + self.assertEqual(callback("abc"), 0) + + with self.assertRaises(ArgumentError) as cm: + callback(10) + + self.assertEqual(str(cm.exception), + "argument 1: TypeError: 'int' object cannot be " + "interpreted as ctypes.c_wchar_p") + + def test_c_void_p_parm(self): + """Test the error message when converting an incompatible type to c_void_p.""" + proto = CFUNCTYPE(c_int, c_void_p) + def callback(*args): + return 0 + + callback = proto(callback) + self.assertEqual(callback(5), 0) + + with self.assertRaises(ArgumentError) as cm: + callback(2.5) + + self.assertEqual(str(cm.exception), + "argument 1: TypeError: 'float' object cannot be " + "interpreted as ctypes.c_void_p") + def test_wchar_result(self): f = dll._testfunc_i_bhilfd f.argtypes = [c_byte, c_short, c_int, c_long, c_float, c_double] @@ -128,7 +206,6 @@ def test_doubleresult(self): self.assertEqual(result, -21) self.assertEqual(type(result), float) - @need_symbol('c_longdouble') def test_longdoubleresult(self): f = dll._testfunc_D_bhilfD f.argtypes = [c_byte, c_short, c_int, c_long, c_float, c_longdouble] @@ -141,7 +218,6 @@ def test_longdoubleresult(self): self.assertEqual(result, -21) self.assertEqual(type(result), float) - @need_symbol('c_longlong') def test_longlongresult(self): f = dll._testfunc_q_bhilfd f.restype = c_longlong @@ -200,7 +276,6 @@ def test_pointers(self): result = f(byref(c_int(99))) self.assertNotEqual(result.contents, 99) - ################################################################ def test_shorts(self): f = dll._testfunc_callback_i_if @@ -218,9 +293,6 @@ def callback(v): f(2**18, cb) self.assertEqual(args, expected) - ################################################################ - - def test_callbacks(self): f = dll._testfunc_callback_i_if f.restype = c_int @@ -229,7 +301,6 @@ def test_callbacks(self): MyCallback = CFUNCTYPE(c_int, c_int) def callback(value): - #print "called back with", value return value cb = MyCallback(callback) @@ -262,7 +333,6 @@ def test_callbacks_2(self): f.argtypes = [c_int, MyCallback] def callback(value): - #print "called back with", value self.assertEqual(type(value), int) return value @@ -270,7 +340,6 @@ def callback(value): result = f(-10, cb) self.assertEqual(result, -18) - @need_symbol('c_longlong') def test_longlong_callbacks(self): f = dll._testfunc_callback_q_qf @@ -371,7 +440,7 @@ class S8I(Structure): (9*2, 8*3, 7*4, 6*5, 5*6, 4*7, 3*8, 2*9)) def test_sf1651235(self): - # see https://www.python.org/sf/1651235 + # see https://bugs.python.org/issue1651235 proto = CFUNCTYPE(c_int, RECT, POINT) def callback(*args): @@ -380,5 +449,6 @@ def callback(*args): callback = proto(callback) self.assertRaises(ArgumentError, lambda: callback((1, 2, 3, 4), POINT())) + if __name__ == '__main__': unittest.main() diff --git a/Lib/ctypes/test/test_incomplete.py b/Lib/test/test_ctypes/test_incomplete.py similarity index 53% rename from Lib/ctypes/test/test_incomplete.py rename to Lib/test/test_ctypes/test_incomplete.py index 00c430ef53..9f859793d8 100644 --- a/Lib/ctypes/test/test_incomplete.py +++ b/Lib/test/test_ctypes/test_incomplete.py @@ -1,12 +1,14 @@ +import ctypes import unittest -from ctypes import * +import warnings +from ctypes import Structure, POINTER, pointer, c_char_p -################################################################ -# -# The incomplete pointer example from the tutorial -# -class MyTestCase(unittest.TestCase): +# The incomplete pointer example from the tutorial +class TestSetPointerType(unittest.TestCase): + def tearDown(self): + # to not leak references, we must clean _pointer_type_cache + ctypes._reset_cache() def test_incomplete_example(self): lpcell = POINTER("cell") @@ -14,7 +16,9 @@ class cell(Structure): _fields_ = [("name", c_char_p), ("next", lpcell)] - SetPointerType(lpcell, cell) + with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + ctypes.SetPointerType(lpcell, cell) c1 = cell() c1.name = b"foo" @@ -32,11 +36,15 @@ class cell(Structure): p = p.next[0] self.assertEqual(result, [b"foo", b"bar"] * 4) - # to not leak references, we must clean _pointer_type_cache - from ctypes import _pointer_type_cache - del _pointer_type_cache[cell] + def test_deprecation(self): + lpcell = POINTER("cell") + class cell(Structure): + _fields_ = [("name", c_char_p), + ("next", lpcell)] + + with self.assertWarns(DeprecationWarning): + ctypes.SetPointerType(lpcell, cell) -################################################################ if __name__ == '__main__': unittest.main() diff --git a/Lib/ctypes/test/test_init.py b/Lib/test/test_ctypes/test_init.py similarity index 96% rename from Lib/ctypes/test/test_init.py rename to Lib/test/test_ctypes/test_init.py index 75fad112a0..113425e582 100644 --- a/Lib/ctypes/test/test_init.py +++ b/Lib/test/test_ctypes/test_init.py @@ -1,5 +1,6 @@ -from ctypes import * import unittest +from ctypes import Structure, c_int + class X(Structure): _fields_ = [("a", c_int), @@ -15,6 +16,7 @@ def __init__(self): self.a = 9 self.b = 12 + class Y(Structure): _fields_ = [("x", X)] @@ -36,5 +38,6 @@ def test_get(self): self.assertEqual((y.x.a, y.x.b), (9, 12)) self.assertEqual(y.x.new_was_called, False) + if __name__ == "__main__": unittest.main() diff --git a/Lib/ctypes/test/test_internals.py b/Lib/test/test_ctypes/test_internals.py similarity index 84% rename from Lib/ctypes/test/test_internals.py rename to Lib/test/test_ctypes/test_internals.py index 271e3f57f8..292633aaa4 100644 --- a/Lib/ctypes/test/test_internals.py +++ b/Lib/test/test_ctypes/test_internals.py @@ -1,7 +1,4 @@ # This tests the internal _objects attribute -import unittest -from ctypes import * -from sys import getrefcount as grc # XXX This test must be reviewed for correctness!!! @@ -14,22 +11,29 @@ # # What about pointers? +import sys +import unittest +from ctypes import Structure, POINTER, c_char_p, c_int + + class ObjectsTestCase(unittest.TestCase): def assertSame(self, a, b): self.assertEqual(id(a), id(b)) def test_ints(self): i = 42000123 - refcnt = grc(i) + refcnt = sys.getrefcount(i) ci = c_int(i) - self.assertEqual(refcnt, grc(i)) + self.assertEqual(refcnt, sys.getrefcount(i)) self.assertEqual(ci._objects, None) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_c_char_p(self): - s = b"Hello, World" - refcnt = grc(s) + s = "Hello, World".encode("ascii") + refcnt = sys.getrefcount(s) cs = c_char_p(s) - self.assertEqual(refcnt + 1, grc(s)) + self.assertEqual(refcnt + 1, sys.getrefcount(s)) self.assertSame(cs._objects, s) def test_simple_struct(self): @@ -60,6 +64,8 @@ class Y(Structure): x1.a, x2.b = 42, 93 self.assertEqual(y._objects, {"0": {}, "1": {}}) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_xxx(self): class X(Structure): _fields_ = [("a", c_char_p), ("b", c_char_p)] @@ -78,9 +84,6 @@ class Y(Structure): y = Y() y.x = x self.assertEqual(y._objects, {"0": {"0": s1, "1": s2}}) -## x = y.x -## del y -## print x._b_base_._objects def test_ptr_struct(self): class X(Structure): @@ -92,9 +95,7 @@ class X(Structure): x = X() x.data = a -##XXX print x._objects -##XXX print x.data[0] -##XXX print x.data._objects + if __name__ == '__main__': unittest.main() diff --git a/Lib/ctypes/test/test_keeprefs.py b/Lib/test/test_ctypes/test_keeprefs.py similarity index 76% rename from Lib/ctypes/test/test_keeprefs.py rename to Lib/test/test_ctypes/test_keeprefs.py index 94c02573fa..5aa5b86fa4 100644 --- a/Lib/ctypes/test/test_keeprefs.py +++ b/Lib/test/test_ctypes/test_keeprefs.py @@ -1,5 +1,7 @@ -from ctypes import * import unittest +from ctypes import (Structure, POINTER, pointer, _pointer_type_cache, + c_char_p, c_int) + class SimpleTestCase(unittest.TestCase): def test_cint(self): @@ -10,6 +12,8 @@ def test_cint(self): x = c_int(99) self.assertEqual(x._objects, None) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_ccharp(self): x = c_char_p() self.assertEqual(x._objects, None) @@ -18,6 +22,7 @@ def test_ccharp(self): x = c_char_p(b"spam") self.assertEqual(x._objects, b"spam") + class StructureTestCase(unittest.TestCase): def test_cint_struct(self): class X(Structure): @@ -30,6 +35,8 @@ class X(Structure): x.b = 99 self.assertEqual(x._objects, None) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_ccharp_struct(self): class X(Structure): _fields_ = [("a", c_char_p), @@ -64,6 +71,7 @@ class RECT(Structure): r.lr = POINT() self.assertEqual(r._objects, {'0': {}, '1': {}}) + class ArrayTestCase(unittest.TestCase): def test_cint_array(self): INTARR = c_int * 3 @@ -87,43 +95,13 @@ class X(Structure): x.a = ia self.assertEqual(x._objects, {'1': {}}) + class PointerTestCase(unittest.TestCase): def test_p_cint(self): i = c_int(42) x = pointer(i) self.assertEqual(x._objects, {'1': i}) -class DeletePointerTestCase(unittest.TestCase): - @unittest.skip('test disabled') - def test_X(self): - class X(Structure): - _fields_ = [("p", POINTER(c_char_p))] - x = X() - i = c_char_p("abc def") - from sys import getrefcount as grc - print("2?", grc(i)) - x.p = pointer(i) - print("3?", grc(i)) - for i in range(320): - c_int(99) - x.p[0] - print(x.p[0]) -## del x -## print "2?", grc(i) -## del i - import gc - gc.collect() - for i in range(320): - c_int(99) - x.p[0] - print(x.p[0]) - print(x.p.contents) -## print x._objects - - x.p[0] = "spam spam" -## print x.p[0] - print("+" * 42) - print(x._objects) class PointerToStructure(unittest.TestCase): def test(self): @@ -137,17 +115,14 @@ class RECT(Structure): r.a = pointer(p1) r.b = pointer(p1) -## from pprint import pprint as pp -## pp(p1._objects) -## pp(r._objects) r.a[0].x = 42 r.a[0].y = 99 # to avoid leaking when tests are run several times # clean up the types left in the cache. - from ctypes import _pointer_type_cache del _pointer_type_cache[POINT] + if __name__ == "__main__": unittest.main() diff --git a/Lib/ctypes/test/test_libc.py b/Lib/test/test_ctypes/test_libc.py similarity index 78% rename from Lib/ctypes/test/test_libc.py rename to Lib/test/test_ctypes/test_libc.py index 56285b5ff8..7716100b08 100644 --- a/Lib/ctypes/test/test_libc.py +++ b/Lib/test/test_ctypes/test_libc.py @@ -1,20 +1,24 @@ +import math import unittest +from ctypes import (CDLL, CFUNCTYPE, POINTER, create_string_buffer, sizeof, + c_void_p, c_char, c_int, c_double, c_size_t) +from test.support import import_helper +_ctypes_test = import_helper.import_module("_ctypes_test") -from ctypes import * -import _ctypes_test lib = CDLL(_ctypes_test.__file__) + def three_way_cmp(x, y): """Return -1 if x < y, 0 if x == y and 1 if x > y""" return (x > y) - (x < y) + class LibTest(unittest.TestCase): def test_sqrt(self): lib.my_sqrt.argtypes = c_double, lib.my_sqrt.restype = c_double self.assertEqual(lib.my_sqrt(4.0), 2.0) - import math self.assertEqual(lib.my_sqrt(2.0), math.sqrt(2.0)) def test_qsort(self): @@ -29,5 +33,6 @@ def sort(a, b): lib.my_qsort(chars, len(chars)-1, sizeof(c_char), comparefunc(sort)) self.assertEqual(chars.raw, b" ,,aaaadmmmnpppsss\x00") + if __name__ == "__main__": unittest.main() diff --git a/Lib/ctypes/test/test_loading.py b/Lib/test/test_ctypes/test_loading.py similarity index 73% rename from Lib/ctypes/test/test_loading.py rename to Lib/test/test_ctypes/test_loading.py index ea892277c4..a4d54f676a 100644 --- a/Lib/ctypes/test/test_loading.py +++ b/Lib/test/test_ctypes/test_loading.py @@ -1,16 +1,20 @@ -from ctypes import * +import _ctypes +import ctypes import os import shutil import subprocess import sys -import unittest import test.support -from test.support import import_helper -from test.support import os_helper +import unittest +from ctypes import CDLL, cdll, addressof, c_void_p, c_char_p from ctypes.util import find_library +from test.support import import_helper, os_helper +_ctypes_test = import_helper.import_module("_ctypes_test") + libc_name = None + def setUpModule(): global libc_name if os.name == "nt": @@ -23,15 +27,22 @@ def setUpModule(): if test.support.verbose: print("libc_name is", libc_name) + class LoaderTest(unittest.TestCase): unknowndll = "xxrandomnamexx" def test_load(self): - if libc_name is None: - self.skipTest('could not find libc') - CDLL(libc_name) - CDLL(os.path.basename(libc_name)) + if libc_name is not None: + test_lib = libc_name + else: + if os.name == "nt": + test_lib = _ctypes_test.__file__ + else: + self.skipTest('could not find library to load') + CDLL(test_lib) + CDLL(os.path.basename(test_lib)) + CDLL(os_helper.FakePath(test_lib)) self.assertRaises(OSError, CDLL, self.unknowndll) def test_load_version(self): @@ -45,11 +56,15 @@ def test_load_version(self): self.assertRaises(OSError, cdll.LoadLibrary, self.unknowndll) def test_find(self): + found = False for name in ("c", "m"): lib = find_library(name) if lib: + found = True cdll.LoadLibrary(lib) CDLL(lib) + if not found: + self.skipTest("Could not find c and m libraries") @unittest.skipUnless(os.name == "nt", 'test specific to Windows') @@ -62,18 +77,17 @@ def test_load_library(self): print(find_library("user32")) if os.name == "nt": - windll.kernel32.GetModuleHandleW - windll["kernel32"].GetModuleHandleW - windll.LoadLibrary("kernel32").GetModuleHandleW - WinDLL("kernel32").GetModuleHandleW + ctypes.windll.kernel32.GetModuleHandleW + ctypes.windll["kernel32"].GetModuleHandleW + ctypes.windll.LoadLibrary("kernel32").GetModuleHandleW + ctypes.WinDLL("kernel32").GetModuleHandleW # embedded null character - self.assertRaises(ValueError, windll.LoadLibrary, "kernel32\0") + self.assertRaises(ValueError, ctypes.windll.LoadLibrary, "kernel32\0") @unittest.skipUnless(os.name == "nt", 'test specific to Windows') def test_load_ordinal_functions(self): - import _ctypes_test - dll = WinDLL(_ctypes_test.__file__) + dll = ctypes.WinDLL(_ctypes_test.__file__) # We load the same function both via ordinal and name func_ord = dll[2] func_name = dll.GetString @@ -86,16 +100,21 @@ def test_load_ordinal_functions(self): self.assertRaises(AttributeError, dll.__getitem__, 1234) + @unittest.skipUnless(os.name == "nt", 'Windows-specific test') + def test_load_without_name_and_with_handle(self): + handle = ctypes.windll.kernel32._handle + lib = ctypes.WinDLL(name=None, handle=handle) + self.assertIs(handle, lib._handle) + @unittest.skipUnless(os.name == "nt", 'Windows-specific test') def test_1703286_A(self): - from _ctypes import LoadLibrary, FreeLibrary # On winXP 64-bit, advapi32 loads at an address that does # NOT fit into a 32-bit integer. FreeLibrary must be able # to accept this address. - # These are tests for https://www.python.org/sf/1703286 - handle = LoadLibrary("advapi32") - FreeLibrary(handle) + # These are tests for https://bugs.python.org/issue1703286 + handle = _ctypes.LoadLibrary("advapi32") + _ctypes.FreeLibrary(handle) @unittest.skipUnless(os.name == "nt", 'Windows-specific test') def test_1703286_B(self): @@ -103,25 +122,33 @@ def test_1703286_B(self): # above, the (arbitrarily selected) CloseEventLog function # also has a high address. 'call_function' should accept # addresses so large. - from _ctypes import call_function - advapi32 = windll.advapi32 + + advapi32 = ctypes.windll.advapi32 # Calling CloseEventLog with a NULL argument should fail, # but the call should not segfault or so. self.assertEqual(0, advapi32.CloseEventLog(None)) - windll.kernel32.GetProcAddress.argtypes = c_void_p, c_char_p - windll.kernel32.GetProcAddress.restype = c_void_p - proc = windll.kernel32.GetProcAddress(advapi32._handle, - b"CloseEventLog") + + kernel32 = ctypes.windll.kernel32 + kernel32.GetProcAddress.argtypes = c_void_p, c_char_p + kernel32.GetProcAddress.restype = c_void_p + proc = kernel32.GetProcAddress(advapi32._handle, b"CloseEventLog") self.assertTrue(proc) + # This is the real test: call the function via 'call_function' - self.assertEqual(0, call_function(proc, (None,))) + self.assertEqual(0, _ctypes.call_function(proc, (None,))) + + @unittest.skipUnless(os.name == "nt", + 'test specific to Windows') + def test_load_hasattr(self): + # bpo-34816: shouldn't raise OSError + self.assertFalse(hasattr(ctypes.windll, 'test')) @unittest.skipUnless(os.name == "nt", 'test specific to Windows') def test_load_dll_with_flags(self): _sqlite3 = import_helper.import_module("_sqlite3") src = _sqlite3.__file__ - if src.lower().endswith("_d.pyd"): + if os.path.basename(src).partition(".")[0].lower().endswith("_d"): ext = "_d.dll" else: ext = ".dll" @@ -177,6 +204,5 @@ def should_fail(command): "WinDLL('_sqlite3.dll'); p.close()") - if __name__ == "__main__": unittest.main() diff --git a/Lib/ctypes/test/test_macholib.py b/Lib/test/test_ctypes/test_macholib.py similarity index 99% rename from Lib/ctypes/test/test_macholib.py rename to Lib/test/test_ctypes/test_macholib.py index bc75f1a05a..9d90617995 100644 --- a/Lib/ctypes/test/test_macholib.py +++ b/Lib/test/test_ctypes/test_macholib.py @@ -1,7 +1,3 @@ -import os -import sys -import unittest - # Bob Ippolito: # # Ok.. the code to find the filename for __getattr__ should look @@ -31,10 +27,15 @@ # # -bob +import os +import sys +import unittest + from ctypes.macholib.dyld import dyld_find from ctypes.macholib.dylib import dylib_info from ctypes.macholib.framework import framework_info + def find_lib(name): possible = ['lib'+name+'.dylib', name+'.dylib', name+'.framework/'+name] for dylib in possible: @@ -106,5 +107,6 @@ def test_framework_info(self): self.assertEqual(framework_info('P/F.framework/Versions/A/F_debug'), d('P', 'F.framework/Versions/A/F_debug', 'F', 'A', 'debug')) + if __name__ == "__main__": unittest.main() diff --git a/Lib/ctypes/test/test_memfunctions.py b/Lib/test/test_ctypes/test_memfunctions.py similarity index 87% rename from Lib/ctypes/test/test_memfunctions.py rename to Lib/test/test_ctypes/test_memfunctions.py index e784b9a706..112b27ba48 100644 --- a/Lib/ctypes/test/test_memfunctions.py +++ b/Lib/test/test_ctypes/test_memfunctions.py @@ -1,20 +1,23 @@ import sys -from test import support import unittest -from ctypes import * -from ctypes.test import need_symbol +from test import support +from ctypes import (POINTER, sizeof, cast, + create_string_buffer, string_at, + create_unicode_buffer, wstring_at, + memmove, memset, + c_char_p, c_byte, c_ubyte, c_wchar) + class MemFunctionsTest(unittest.TestCase): - @unittest.skip('test disabled') def test_overflow(self): # string_at and wstring_at must use the Python calling # convention (which acquires the GIL and checks the Python # error flag). Provoke an error and catch it; see also issue - # #3554: + # gh-47804. self.assertRaises((OverflowError, MemoryError, SystemError), - lambda: wstring_at(u"foo", sys.maxint - 1)) + lambda: wstring_at(u"foo", sys.maxsize - 1)) self.assertRaises((OverflowError, MemoryError, SystemError), - lambda: string_at("foo", sys.maxint - 1)) + lambda: string_at("foo", sys.maxsize - 1)) def test_memmove(self): # large buffers apparently increase the chance that the memory @@ -63,7 +66,6 @@ def test_string_at(self): self.assertEqual(string_at(b"foo bar", 7), b"foo bar") self.assertEqual(string_at(b"foo bar", 3), b"foo") - @need_symbol('create_unicode_buffer') def test_wstring_at(self): p = create_unicode_buffer("Hello, World") a = create_unicode_buffer(1000000) @@ -75,5 +77,6 @@ def test_wstring_at(self): self.assertEqual(wstring_at(a, 16), "Hello, World\0\0\0\0") self.assertEqual(wstring_at(a, 0), "") + if __name__ == "__main__": unittest.main() diff --git a/Lib/ctypes/test/test_numbers.py b/Lib/test/test_ctypes/test_numbers.py similarity index 89% rename from Lib/ctypes/test/test_numbers.py rename to Lib/test/test_ctypes/test_numbers.py index a5c661b0e9..29108a28ec 100644 --- a/Lib/ctypes/test/test_numbers.py +++ b/Lib/test/test_ctypes/test_numbers.py @@ -1,6 +1,13 @@ -from ctypes import * -import unittest +import array import struct +import sys +import unittest +from operator import truth +from ctypes import (byref, sizeof, alignment, + c_char, c_byte, c_ubyte, c_short, c_ushort, c_int, c_uint, + c_long, c_ulong, c_longlong, c_ulonglong, + c_float, c_double, c_longdouble, c_bool) + def valid_ranges(*types): # given a sequence of numeric types, collect their _type_ @@ -19,36 +26,18 @@ def valid_ranges(*types): result.append((min(a, b, c, d), max(a, b, c, d))) return result + ArgType = type(byref(c_int(0))) -unsigned_types = [c_ubyte, c_ushort, c_uint, c_ulong] +unsigned_types = [c_ubyte, c_ushort, c_uint, c_ulong, c_ulonglong] signed_types = [c_byte, c_short, c_int, c_long, c_longlong] - -bool_types = [] - +bool_types = [c_bool] float_types = [c_double, c_float] -try: - c_ulonglong - c_longlong -except NameError: - pass -else: - unsigned_types.append(c_ulonglong) - signed_types.append(c_longlong) - -try: - c_bool -except NameError: - pass -else: - bool_types.append(c_bool) - unsigned_ranges = valid_ranges(*unsigned_types) signed_ranges = valid_ranges(*signed_types) bool_values = [True, False, 0, 1, -1, 5000, 'test', [], [1]] -################################################################ class NumberTestCase(unittest.TestCase): @@ -71,7 +60,6 @@ def test_signed_values(self): self.assertEqual(t(h).value, h) def test_bool_values(self): - from operator import truth for t, v in zip(bool_types, bool_values): self.assertEqual(t(v).value, truth(v)) @@ -147,21 +135,20 @@ def test_alignments(self): # alignment of the type... self.assertEqual((code, alignment(t)), - (code, align)) + (code, align)) # and alignment of an instance self.assertEqual((code, alignment(t())), - (code, align)) + (code, align)) def test_int_from_address(self): - from array import array for t in signed_types + unsigned_types: # the array module doesn't support all format codes # (no 'q' or 'Q') try: - array(t._type_) + array.array(t._type_) except ValueError: continue - a = array(t._type_, [100]) + a = array.array(t._type_, [100]) # v now is an integer at an 'external' memory location v = t.from_address(a.buffer_info()[0]) @@ -174,9 +161,8 @@ def test_int_from_address(self): def test_float_from_address(self): - from array import array for t in float_types: - a = array(t._type_, [3.14]) + a = array.array(t._type_, [3.14]) v = t.from_address(a.buffer_info()[0]) self.assertEqual(v.value, a[0]) self.assertIs(type(v), t) @@ -185,10 +171,7 @@ def test_float_from_address(self): self.assertIs(type(v), t) def test_char_from_address(self): - from ctypes import c_char - from array import array - - a = array('b', [0]) + a = array.array('b', [0]) a[0] = ord('x') v = c_char.from_address(a.buffer_info()[0]) self.assertEqual(v.value, b'x') @@ -204,7 +187,6 @@ def test_init(self): self.assertRaises(TypeError, c_int, c_long(42)) def test_float_overflow(self): - import sys big_int = int(sys.float_info.max) * 2 for t in float_types + [c_longdouble]: self.assertRaises(OverflowError, t, big_int) diff --git a/Lib/ctypes/test/test_objects.py b/Lib/test/test_ctypes/test_objects.py similarity index 78% rename from Lib/ctypes/test/test_objects.py rename to Lib/test/test_ctypes/test_objects.py index 19e3dc1f2d..8db1cd873f 100644 --- a/Lib/ctypes/test/test_objects.py +++ b/Lib/test/test_ctypes/test_objects.py @@ -11,7 +11,7 @@ Here is an array of string pointers: ->>> from ctypes import * +>>> from ctypes import Structure, c_int, c_char_p >>> array = (c_char_p * 5)() >>> print(array._objects) None @@ -42,7 +42,7 @@ of 'x' ('_b_base_' is either None, or the root object owning the memory block): >>> print(x.array._b_base_) # doctest: +ELLIPSIS - + >>> >>> x.array[0] = b'spam spam spam' @@ -51,17 +51,17 @@ >>> x.array._b_base_._objects {'0:2': b'spam spam spam'} >>> - ''' -import unittest, doctest +import doctest +import unittest + -import ctypes.test.test_objects +def load_tests(loader, tests, pattern): + # TODO: RUSTPYTHON - doctest disabled due to null terminator in _objects + # tests.addTest(doctest.DocTestSuite()) + return tests -class TestCase(unittest.TestCase): - def test(self): - failures, tests = doctest.testmod(ctypes.test.test_objects) - self.assertFalse(failures, 'doctests failed, see output above') if __name__ == '__main__': - doctest.testmod(ctypes.test.test_objects) + unittest.main() diff --git a/Lib/ctypes/test/test_parameters.py b/Lib/test/test_ctypes/test_parameters.py similarity index 72% rename from Lib/ctypes/test/test_parameters.py rename to Lib/test/test_ctypes/test_parameters.py index 38af7ac13d..1a6ddb91a7 100644 --- a/Lib/ctypes/test/test_parameters.py +++ b/Lib/test/test_ctypes/test_parameters.py @@ -1,11 +1,25 @@ +import sys import unittest -from ctypes.test import need_symbol import test.support +from ctypes import (CDLL, PyDLL, ArgumentError, + Structure, Array, Union, + _Pointer, _SimpleCData, _CFuncPtr, + POINTER, pointer, byref, + c_void_p, c_char_p, c_wchar_p, py_object, + c_bool, + c_char, c_wchar, + c_byte, c_ubyte, + c_short, c_ushort, + c_int, c_uint, + c_long, c_ulong, + c_longlong, c_ulonglong, + c_float, c_double, c_longdouble) +from test.support import import_helper +_ctypes_test = import_helper.import_module("_ctypes_test") -class SimpleTypesTestCase(unittest.TestCase): +class SimpleTypesTestCase(unittest.TestCase): def setUp(self): - import ctypes try: from _ctypes import set_conversion_mode except ImportError: @@ -22,7 +36,6 @@ def tearDown(self): set_conversion_mode(*self.prev_conv_mode) def test_subclasses(self): - from ctypes import c_void_p, c_char_p # ctypes 0.9.5 and before did overwrite from_param in SimpleType_new class CVOIDP(c_void_p): def from_param(cls, value): @@ -37,10 +50,7 @@ def from_param(cls, value): self.assertEqual(CVOIDP.from_param("abc"), "abcabc") self.assertEqual(CCHARP.from_param("abc"), "abcabcabcabc") - @need_symbol('c_wchar_p') def test_subclasses_c_wchar_p(self): - from ctypes import c_wchar_p - class CWCHARP(c_wchar_p): def from_param(cls, value): return value * 3 @@ -50,8 +60,6 @@ def from_param(cls, value): # XXX Replace by c_char_p tests def test_cstrings(self): - from ctypes import c_char_p - # c_char_p.from_param on a Python String packs the string # into a cparam object s = b"123" @@ -67,10 +75,7 @@ def test_cstrings(self): a = c_char_p(b"123") self.assertIs(c_char_p.from_param(a), a) - @need_symbol('c_wchar_p') def test_cw_strings(self): - from ctypes import c_wchar_p - c_wchar_p.from_param("123") self.assertRaises(TypeError, c_wchar_p.from_param, 42) @@ -79,12 +84,27 @@ def test_cw_strings(self): pa = c_wchar_p.from_param(c_wchar_p("123")) self.assertEqual(type(pa), c_wchar_p) + def test_c_char(self): + with self.assertRaises(TypeError) as cm: + c_char.from_param(b"abc") + self.assertEqual(str(cm.exception), + "one character bytes, bytearray or integer expected") + + def test_c_wchar(self): + with self.assertRaises(TypeError) as cm: + c_wchar.from_param("abc") + self.assertEqual(str(cm.exception), + "one character unicode string expected") + + + with self.assertRaises(TypeError) as cm: + c_wchar.from_param(123) + self.assertEqual(str(cm.exception), + "unicode string expected instead of int instance") + def test_int_pointers(self): - from ctypes import c_short, c_uint, c_int, c_long, POINTER, pointer LPINT = POINTER(c_int) -## p = pointer(c_int(42)) -## x = LPINT.from_param(p) x = LPINT.from_param(pointer(c_int(42))) self.assertEqual(x.contents.value, 42) self.assertEqual(LPINT(c_int(42)).contents.value, 42) @@ -99,7 +119,6 @@ def test_int_pointers(self): def test_byref_pointer(self): # The from_param class method of POINTER(typ) classes accepts what is # returned by byref(obj), it type(obj) == typ - from ctypes import c_short, c_uint, c_int, c_long, POINTER, byref LPINT = POINTER(c_int) LPINT.from_param(byref(c_int(42))) @@ -111,7 +130,6 @@ def test_byref_pointer(self): def test_byref_pointerpointer(self): # See above - from ctypes import c_short, c_uint, c_int, c_long, pointer, POINTER, byref LPLPINT = POINTER(POINTER(c_int)) LPLPINT.from_param(byref(pointer(c_int(42)))) @@ -122,7 +140,6 @@ def test_byref_pointerpointer(self): self.assertRaises(TypeError, LPLPINT.from_param, byref(pointer(c_uint(22)))) def test_array_pointers(self): - from ctypes import c_short, c_uint, c_int, c_long, POINTER INTARRAY = c_int * 3 ia = INTARRAY() self.assertEqual(len(ia), 3) @@ -137,15 +154,12 @@ def test_array_pointers(self): self.assertRaises(TypeError, LPINT.from_param, c_uint*3) def test_noctypes_argtype(self): - import _ctypes_test - from ctypes import CDLL, c_void_p, ArgumentError - func = CDLL(_ctypes_test.__file__)._testfunc_p_p func.restype = c_void_p # TypeError: has no from_param method self.assertRaises(TypeError, setattr, func, "argtypes", (object,)) - class Adapter(object): + class Adapter: def from_param(cls, obj): return None @@ -153,7 +167,7 @@ def from_param(cls, obj): self.assertEqual(func(None), None) self.assertEqual(func(object()), None) - class Adapter(object): + class Adapter: def from_param(cls, obj): return obj @@ -162,7 +176,7 @@ def from_param(cls, obj): self.assertRaises(ArgumentError, func, object()) self.assertEqual(func(c_void_p(42)), 42) - class Adapter(object): + class Adapter: def from_param(cls, obj): raise ValueError(obj) @@ -171,9 +185,6 @@ def from_param(cls, obj): self.assertRaises(ArgumentError, func, 99) def test_abstract(self): - from ctypes import (Array, Structure, Union, _Pointer, - _SimpleCData, _CFuncPtr) - self.assertRaises(TypeError, Array.from_param, 42) self.assertRaises(TypeError, Structure.from_param, 42) self.assertRaises(TypeError, Union.from_param, 42) @@ -185,7 +196,6 @@ def test_abstract(self): def test_issue31311(self): # __setstate__ should neither raise a SystemError nor crash in case # of a bad __dict__. - from ctypes import Structure class BadStruct(Structure): @property @@ -202,27 +212,6 @@ def __dict__(self): WorseStruct().__setstate__({}, b'foo') def test_parameter_repr(self): - from ctypes import ( - c_bool, - c_char, - c_wchar, - c_byte, - c_ubyte, - c_short, - c_ushort, - c_int, - c_uint, - c_long, - c_ulong, - c_longlong, - c_ulonglong, - c_float, - c_double, - c_longdouble, - c_char_p, - c_wchar_p, - c_void_p, - ) self.assertRegex(repr(c_bool.from_param(True)), r"^$") self.assertEqual(repr(c_char.from_param(97)), "") self.assertRegex(repr(c_wchar.from_param('a')), r"^$") @@ -238,13 +227,62 @@ def test_parameter_repr(self): self.assertRegex(repr(c_ulonglong.from_param(20000)), r"^$") self.assertEqual(repr(c_float.from_param(1.5)), "") self.assertEqual(repr(c_double.from_param(1.5)), "") - self.assertEqual(repr(c_double.from_param(1e300)), "") + if sys.float_repr_style == 'short': + self.assertEqual(repr(c_double.from_param(1e300)), "") self.assertRegex(repr(c_longdouble.from_param(1.5)), r"^$") self.assertRegex(repr(c_char_p.from_param(b'hihi')), r"^$") self.assertRegex(repr(c_wchar_p.from_param('hihi')), r"^$") self.assertRegex(repr(c_void_p.from_param(0x12)), r"^$") -################################################################ + @test.support.cpython_only + def test_from_param_result_refcount(self): + # Issue #99952 + class X(Structure): + """This struct size is <= sizeof(void*).""" + _fields_ = [("a", c_void_p)] + + def __del__(self): + trace.append(4) + + @classmethod + def from_param(cls, value): + trace.append(2) + return cls() + + PyList_Append = PyDLL(_ctypes_test.__file__)._testfunc_pylist_append + PyList_Append.restype = c_int + PyList_Append.argtypes = [py_object, py_object, X] + + trace = [] + trace.append(1) + PyList_Append(trace, 3, "dummy") + trace.append(5) + + self.assertEqual(trace, [1, 2, 3, 4, 5]) + + class Y(Structure): + """This struct size is > sizeof(void*).""" + _fields_ = [("a", c_void_p), ("b", c_void_p)] + + def __del__(self): + trace.append(4) + + @classmethod + def from_param(cls, value): + trace.append(2) + return cls() + + PyList_Append = PyDLL(_ctypes_test.__file__)._testfunc_pylist_append + PyList_Append.restype = c_int + PyList_Append.argtypes = [py_object, py_object, Y] + + trace = [] + trace.append(1) + PyList_Append(trace, 3, "dummy") + trace.append(5) + + self.assertEqual(trace, [1, 2, 3, 4, 5]) + if __name__ == '__main__': unittest.main() diff --git a/Lib/ctypes/test/test_pep3118.py b/Lib/test/test_ctypes/test_pep3118.py similarity index 70% rename from Lib/ctypes/test/test_pep3118.py rename to Lib/test/test_ctypes/test_pep3118.py index efffc80a66..06b2ccecad 100644 --- a/Lib/ctypes/test/test_pep3118.py +++ b/Lib/test/test_ctypes/test_pep3118.py @@ -1,6 +1,13 @@ +import re +import sys import unittest -from ctypes import * -import re, sys +from ctypes import (CFUNCTYPE, POINTER, sizeof, Union, + Structure, LittleEndianStructure, BigEndianStructure, + c_char, c_byte, c_ubyte, + c_short, c_ushort, c_int, c_uint, + c_long, c_ulong, c_longlong, c_ulonglong, c_uint64, + c_bool, c_float, c_double, c_longdouble, py_object) + if sys.byteorder == "little": THIS_ENDIAN = "<" @@ -9,6 +16,7 @@ THIS_ENDIAN = ">" OTHER_ENDIAN = "<" + def normalize(format): # Remove current endian specifier and white space from a format # string @@ -17,65 +25,54 @@ def normalize(format): format = format.replace(OTHER_ENDIAN, THIS_ENDIAN) return re.sub(r"\s", "", format) -class Test(unittest.TestCase): +class Test(unittest.TestCase): def test_native_types(self): for tp, fmt, shape, itemtp in native_types: ob = tp() v = memoryview(ob) - try: - self.assertEqual(normalize(v.format), normalize(fmt)) - if shape: - self.assertEqual(len(v), shape[0]) - else: - self.assertEqual(len(v) * sizeof(itemtp), sizeof(ob)) - self.assertEqual(v.itemsize, sizeof(itemtp)) - self.assertEqual(v.shape, shape) - # XXX Issue #12851: PyCData_NewGetBuffer() must provide strides - # if requested. memoryview currently reconstructs missing - # stride information, so this assert will fail. - # self.assertEqual(v.strides, ()) - - # they are always read/write - self.assertFalse(v.readonly) - - if v.shape: - n = 1 - for dim in v.shape: - n = n * dim - self.assertEqual(n * v.itemsize, len(v.tobytes())) - except: - # so that we can see the failing type - print(tp) - raise + self.assertEqual(normalize(v.format), normalize(fmt)) + if shape: + self.assertEqual(len(v), shape[0]) + else: + self.assertRaises(TypeError, len, v) + self.assertEqual(v.itemsize, sizeof(itemtp)) + self.assertEqual(v.shape, shape) + # XXX Issue #12851: PyCData_NewGetBuffer() must provide strides + # if requested. memoryview currently reconstructs missing + # stride information, so this assert will fail. + # self.assertEqual(v.strides, ()) + + # they are always read/write + self.assertFalse(v.readonly) + + n = 1 + for dim in v.shape: + n = n * dim + self.assertEqual(n * v.itemsize, len(v.tobytes())) def test_endian_types(self): for tp, fmt, shape, itemtp in endian_types: ob = tp() v = memoryview(ob) - try: - self.assertEqual(v.format, fmt) - if shape: - self.assertEqual(len(v), shape[0]) - else: - self.assertEqual(len(v) * sizeof(itemtp), sizeof(ob)) - self.assertEqual(v.itemsize, sizeof(itemtp)) - self.assertEqual(v.shape, shape) - # XXX Issue #12851 - # self.assertEqual(v.strides, ()) - - # they are always read/write - self.assertFalse(v.readonly) - - if v.shape: - n = 1 - for dim in v.shape: - n = n * dim - self.assertEqual(n, len(v)) - except: - # so that we can see the failing type - print(tp) - raise + self.assertEqual(v.format, fmt) + if shape: + self.assertEqual(len(v), shape[0]) + else: + self.assertRaises(TypeError, len, v) + self.assertEqual(v.itemsize, sizeof(itemtp)) + self.assertEqual(v.shape, shape) + # XXX Issue #12851 + # self.assertEqual(v.strides, ()) + + # they are always read/write + self.assertFalse(v.readonly) + + n = 1 + for dim in v.shape: + n = n * dim + self.assertEqual(n * v.itemsize, len(v.tobytes())) + # define some structure classes @@ -86,6 +83,20 @@ class PackedPoint(Structure): _pack_ = 2 _fields_ = [("x", c_long), ("y", c_long)] +class PointMidPad(Structure): + _fields_ = [("x", c_byte), ("y", c_uint)] + +class PackedPointMidPad(Structure): + _pack_ = 2 + _fields_ = [("x", c_byte), ("y", c_uint64)] + +class PointEndPad(Structure): + _fields_ = [("x", c_uint), ("y", c_byte)] + +class PackedPointEndPad(Structure): + _pack_ = 2 + _fields_ = [("x", c_uint64), ("y", c_byte)] + class Point2(Structure): pass Point2._fields_ = [("x", c_long), ("y", c_long)] @@ -107,6 +118,7 @@ class Complete(Structure): PComplete = POINTER(Complete) Complete._fields_ = [("a", c_long)] + ################################################################ # # This table contains format strings as they look on little endian @@ -185,11 +197,14 @@ class Complete(Structure): ## structures and unions - (Point, "T{l:x:>l:y:}".replace('l', s_long), (), BEPoint), - (LEPoint, "T{l:x:>l:y:}".replace('l', s_long), (), POINTER(BEPoint)), (POINTER(LEPoint), "&T{= 0 return a + def c_wbuffer(init): n = len(init) + 1 return (c_wchar * n)(*init) -class CharPointersTestCase(unittest.TestCase): +class CharPointersTestCase(unittest.TestCase): def setUp(self): func = testdll._testfunc_p_p func.restype = c_long @@ -66,6 +72,32 @@ def test_paramflags(self): self.assertEqual(func(None), None) self.assertEqual(func(input=None), None) + def test_invalid_paramflags(self): + proto = CFUNCTYPE(c_int, c_char_p) + with self.assertRaises(ValueError): + func = proto(("myprintf", testdll), ((1, "fmt"), (1, "arg1"))) + + def test_invalid_setattr_argtypes(self): + proto = CFUNCTYPE(c_int, c_char_p) + func = proto(("myprintf", testdll), ((1, "fmt"),)) + + with self.assertRaisesRegex(TypeError, "_argtypes_ must be a sequence of types"): + func.argtypes = 123 + self.assertEqual(func.argtypes, (c_char_p,)) + + with self.assertRaisesRegex(ValueError, "paramflags must have the same length as argtypes"): + func.argtypes = (c_char_p, c_int) + self.assertEqual(func.argtypes, (c_char_p,)) + + def test_paramflags_outarg(self): + proto = CFUNCTYPE(c_int, c_char_p, c_int) + with self.assertRaisesRegex(TypeError, "must be a pointer type"): + func = proto(("myprintf", testdll), ((1, "fmt"), (2, "out"))) + + proto = CFUNCTYPE(c_int, c_char_p, c_void_p) + func = proto(("myprintf", testdll), ((1, "fmt"), (2, "out"))) + with self.assertRaisesRegex(TypeError, "must be a pointer type"): + func.argtypes = (c_char_p, c_int) def test_int_pointer_arg(self): func = testdll._testfunc_p_p @@ -100,7 +132,7 @@ def test_POINTER_c_char_arg(self): self.assertEqual(None, func(c_char_p(None))) self.assertEqual(b"123", func(c_char_p(b"123"))) - self.assertEqual(b"123", func(c_buffer(b"123"))) + self.assertEqual(b"123", func(create_string_buffer(b"123"))) ca = c_char(b"a") self.assertEqual(ord(b"a"), func(pointer(ca))[0]) self.assertEqual(ord(b"a"), func(byref(ca))[0]) @@ -115,7 +147,7 @@ def test_c_char_p_arg(self): self.assertEqual(None, func(c_char_p(None))) self.assertEqual(b"123", func(c_char_p(b"123"))) - self.assertEqual(b"123", func(c_buffer(b"123"))) + self.assertEqual(b"123", func(create_string_buffer(b"123"))) ca = c_char(b"a") self.assertEqual(ord(b"a"), func(pointer(ca))[0]) self.assertEqual(ord(b"a"), func(byref(ca))[0]) @@ -130,7 +162,7 @@ def test_c_void_p_arg(self): self.assertEqual(b"123", func(c_char_p(b"123"))) self.assertEqual(None, func(c_char_p(None))) - self.assertEqual(b"123", func(c_buffer(b"123"))) + self.assertEqual(b"123", func(create_string_buffer(b"123"))) ca = c_char(b"a") self.assertEqual(ord(b"a"), func(pointer(ca))[0]) self.assertEqual(ord(b"a"), func(byref(ca))[0]) @@ -139,7 +171,6 @@ def test_c_void_p_arg(self): func(pointer(c_int())) func((c_int * 3)()) - @need_symbol('c_wchar_p') def test_c_void_p_arg_with_c_wchar_p(self): func = testdll._testfunc_p_p func.restype = c_wchar_p @@ -161,9 +192,8 @@ class X: func.argtypes = None self.assertEqual(None, func(X())) -@need_symbol('c_wchar') -class WCharPointersTestCase(unittest.TestCase): +class WCharPointersTestCase(unittest.TestCase): def setUp(self): func = testdll._testfunc_p_p func.restype = c_int @@ -203,6 +233,7 @@ def test_c_wchar_p_arg(self): self.assertEqual("a", func(pointer(ca))[0]) self.assertEqual("a", func(byref(ca))[0]) + class ArrayTest(unittest.TestCase): def test(self): func = testdll._testfunc_ai8 @@ -216,7 +247,6 @@ def test(self): def func(): pass CFUNCTYPE(None, c_int * 3)(func) -################################################################ if __name__ == '__main__': unittest.main() diff --git a/Lib/ctypes/test/test_python_api.py b/Lib/test/test_ctypes/test_python_api.py similarity index 69% rename from Lib/ctypes/test/test_python_api.py rename to Lib/test/test_ctypes/test_python_api.py index 49571f97bb..2e68b35f8a 100644 --- a/Lib/ctypes/test/test_python_api.py +++ b/Lib/test/test_ctypes/test_python_api.py @@ -1,18 +1,14 @@ -from ctypes import * +import _ctypes +import sys import unittest from test import support +from ctypes import (pythonapi, POINTER, create_string_buffer, sizeof, + py_object, c_char_p, c_char, c_long, c_size_t) -################################################################ -# This section should be moved into ctypes\__init__.py, when it's ready. - -from _ctypes import PyObj_FromPtr - -################################################################ - -from sys import getrefcount as grc class PythonAPITestCase(unittest.TestCase): - + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_PyBytes_FromStringAndSize(self): PyBytes_FromStringAndSize = pythonapi.PyBytes_FromStringAndSize @@ -27,46 +23,49 @@ def test_PyString_FromString(self): pythonapi.PyBytes_FromString.argtypes = (c_char_p,) s = b"abc" - refcnt = grc(s) + refcnt = sys.getrefcount(s) pyob = pythonapi.PyBytes_FromString(s) - self.assertEqual(grc(s), refcnt) + self.assertEqual(sys.getrefcount(s), refcnt) self.assertEqual(s, pyob) del pyob - self.assertEqual(grc(s), refcnt) + self.assertEqual(sys.getrefcount(s), refcnt) @support.refcount_test def test_PyLong_Long(self): - ref42 = grc(42) + ref42 = sys.getrefcount(42) pythonapi.PyLong_FromLong.restype = py_object self.assertEqual(pythonapi.PyLong_FromLong(42), 42) - self.assertEqual(grc(42), ref42) + self.assertEqual(sys.getrefcount(42), ref42) pythonapi.PyLong_AsLong.argtypes = (py_object,) pythonapi.PyLong_AsLong.restype = c_long res = pythonapi.PyLong_AsLong(42) - self.assertEqual(grc(res), ref42 + 1) + # Small int refcnts don't change + self.assertEqual(sys.getrefcount(res), ref42) del res - self.assertEqual(grc(42), ref42) + self.assertEqual(sys.getrefcount(42), ref42) @support.refcount_test def test_PyObj_FromPtr(self): - s = "abc def ghi jkl" - ref = grc(s) + s = object() + ref = sys.getrefcount(s) # id(python-object) is the address - pyobj = PyObj_FromPtr(id(s)) + pyobj = _ctypes.PyObj_FromPtr(id(s)) self.assertIs(s, pyobj) - self.assertEqual(grc(s), ref + 1) + self.assertEqual(sys.getrefcount(s), ref + 1) del pyobj - self.assertEqual(grc(s), ref) + self.assertEqual(sys.getrefcount(s), ref) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_PyOS_snprintf(self): PyOS_snprintf = pythonapi.PyOS_snprintf PyOS_snprintf.argtypes = POINTER(c_char), c_size_t, c_char_p - buf = c_buffer(256) + buf = create_string_buffer(256) PyOS_snprintf(buf, sizeof(buf), b"Hello from %s", b"ctypes") self.assertEqual(buf.value, b"Hello from ctypes") @@ -81,5 +80,6 @@ def test_pyobject_repr(self): self.assertEqual(repr(py_object(42)), "py_object(42)") self.assertEqual(repr(py_object(object)), "py_object(%r)" % object) + if __name__ == "__main__": unittest.main() diff --git a/Lib/ctypes/test/test_random_things.py b/Lib/test/test_ctypes/test_random_things.py similarity index 73% rename from Lib/ctypes/test/test_random_things.py rename to Lib/test/test_ctypes/test_random_things.py index 2988e275cf..3908eca092 100644 --- a/Lib/ctypes/test/test_random_things.py +++ b/Lib/test/test_ctypes/test_random_things.py @@ -1,30 +1,34 @@ -from ctypes import * +import _ctypes import contextlib -from test import support -import unittest +import ctypes import sys +import unittest +from test import support +from ctypes import CFUNCTYPE, c_void_p, c_char_p, c_int, c_double def callback_func(arg): 42 / arg raise ValueError(arg) + @unittest.skipUnless(sys.platform == "win32", 'Windows-specific test') class call_function_TestCase(unittest.TestCase): # _ctypes.call_function is deprecated and private, but used by # Gary Bishp's readline module. If we have it, we must test it as well. def test(self): - from _ctypes import call_function - windll.kernel32.LoadLibraryA.restype = c_void_p - windll.kernel32.GetProcAddress.argtypes = c_void_p, c_char_p - windll.kernel32.GetProcAddress.restype = c_void_p + kernel32 = ctypes.windll.kernel32 + kernel32.LoadLibraryA.restype = c_void_p + kernel32.GetProcAddress.argtypes = c_void_p, c_char_p + kernel32.GetProcAddress.restype = c_void_p + + hdll = kernel32.LoadLibraryA(b"kernel32") + funcaddr = kernel32.GetProcAddress(hdll, b"GetModuleHandleA") - hdll = windll.kernel32.LoadLibraryA(b"kernel32") - funcaddr = windll.kernel32.GetProcAddress(hdll, b"GetModuleHandleA") + self.assertEqual(_ctypes.call_function(funcaddr, (None,)), + kernel32.GetModuleHandleA(None)) - self.assertEqual(call_function(funcaddr, (None,)), - windll.kernel32.GetModuleHandleA(None)) class CallbackTracbackTestCase(unittest.TestCase): # When an exception is raised in a ctypes callback function, the C @@ -47,9 +51,9 @@ def expect_unraisable(self, exc_type, exc_msg=None): if exc_msg is not None: self.assertEqual(str(cm.unraisable.exc_value), exc_msg) self.assertEqual(cm.unraisable.err_msg, - "Exception ignored on calling ctypes " - "callback function") - self.assertIs(cm.unraisable.object, callback_func) + f"Exception ignored on calling ctypes " + f"callback function {callback_func!r}") + self.assertIsNone(cm.unraisable.object) def test_ValueError(self): cb = CFUNCTYPE(c_int, c_int)(callback_func) @@ -66,6 +70,8 @@ def test_FloatDivisionError(self): with self.expect_unraisable(ZeroDivisionError): cb(0.0) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_TypeErrorDivisionError(self): cb = CFUNCTYPE(c_int, c_char_p)(callback_func) err_msg = "unsupported operand type(s) for /: 'int' and 'bytes'" diff --git a/Lib/ctypes/test/test_refcounts.py b/Lib/test/test_ctypes/test_refcounts.py similarity index 57% rename from Lib/ctypes/test/test_refcounts.py rename to Lib/test/test_ctypes/test_refcounts.py index 48958cd2a6..9e87cfc661 100644 --- a/Lib/ctypes/test/test_refcounts.py +++ b/Lib/test/test_ctypes/test_refcounts.py @@ -1,60 +1,59 @@ -import unittest -from test import support import ctypes import gc +import sys +import unittest +from test import support +from test.support import import_helper +from test.support import script_helper +_ctypes_test = import_helper.import_module("_ctypes_test") + MyCallback = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_int) OtherCallback = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_int, ctypes.c_ulonglong) -import _ctypes_test dll = ctypes.CDLL(_ctypes_test.__file__) -class RefcountTestCase(unittest.TestCase): +class RefcountTestCase(unittest.TestCase): @support.refcount_test def test_1(self): - from sys import getrefcount as grc - f = dll._testfunc_callback_i_if f.restype = ctypes.c_int f.argtypes = [ctypes.c_int, MyCallback] def callback(value): - #print "called back with", value return value - self.assertEqual(grc(callback), 2) + self.assertEqual(sys.getrefcount(callback), 2) cb = MyCallback(callback) - self.assertGreater(grc(callback), 2) + self.assertGreater(sys.getrefcount(callback), 2) result = f(-10, cb) self.assertEqual(result, -18) cb = None gc.collect() - self.assertEqual(grc(callback), 2) - + self.assertEqual(sys.getrefcount(callback), 2) @support.refcount_test def test_refcount(self): - from sys import getrefcount as grc def func(*args): pass # this is the standard refcount for func - self.assertEqual(grc(func), 2) + self.assertEqual(sys.getrefcount(func), 2) # the CFuncPtr instance holds at least one refcount on func: f = OtherCallback(func) - self.assertGreater(grc(func), 2) + self.assertGreater(sys.getrefcount(func), 2) # and may release it again del f - self.assertGreaterEqual(grc(func), 2) + self.assertGreaterEqual(sys.getrefcount(func), 2) # but now it must be gone gc.collect() - self.assertEqual(grc(func), 2) + self.assertEqual(sys.getrefcount(func), 2) class X(ctypes.Structure): _fields_ = [("a", OtherCallback)] @@ -62,32 +61,31 @@ class X(ctypes.Structure): x.a = OtherCallback(func) # the CFuncPtr instance holds at least one refcount on func: - self.assertGreater(grc(func), 2) + self.assertGreater(sys.getrefcount(func), 2) # and may release it again del x - self.assertGreaterEqual(grc(func), 2) + self.assertGreaterEqual(sys.getrefcount(func), 2) # and now it must be gone again gc.collect() - self.assertEqual(grc(func), 2) + self.assertEqual(sys.getrefcount(func), 2) f = OtherCallback(func) # the CFuncPtr instance holds at least one refcount on func: - self.assertGreater(grc(func), 2) + self.assertGreater(sys.getrefcount(func), 2) # create a cycle f.cycle = f del f gc.collect() - self.assertEqual(grc(func), 2) + self.assertEqual(sys.getrefcount(func), 2) + class AnotherLeak(unittest.TestCase): def test_callback(self): - import sys - proto = ctypes.CFUNCTYPE(ctypes.c_int, ctypes.c_int, ctypes.c_int) def func(a, b): return a * b * 2 @@ -112,5 +110,34 @@ def func(): for _ in range(10000): func() + +class ModuleIsolationTest(unittest.TestCase): + def test_finalize(self): + # check if gc_decref() succeeds + script = ( + "import ctypes;" + "import sys;" + "del sys.modules['_ctypes'];" + "import _ctypes;" + "exit()" + ) + script_helper.assert_python_ok("-c", script) + + +class PyObjectRestypeTest(unittest.TestCase): + def test_restype_py_object_with_null_return(self): + # Test that a function which returns a NULL PyObject * + # without setting an exception does not crash. + PyErr_Occurred = ctypes.pythonapi.PyErr_Occurred + PyErr_Occurred.argtypes = [] + PyErr_Occurred.restype = ctypes.py_object + + # At this point, there's no exception set, so PyErr_Occurred + # returns NULL. Given the restype is py_object, the + # ctypes machinery will raise a custom error. + with self.assertRaisesRegex(ValueError, "PyObject is NULL"): + PyErr_Occurred() + + if __name__ == '__main__': unittest.main() diff --git a/Lib/ctypes/test/test_repr.py b/Lib/test/test_ctypes/test_repr.py similarity index 80% rename from Lib/ctypes/test/test_repr.py rename to Lib/test/test_ctypes/test_repr.py index 60a2c80345..e7587984a9 100644 --- a/Lib/ctypes/test/test_repr.py +++ b/Lib/test/test_ctypes/test_repr.py @@ -1,5 +1,8 @@ -from ctypes import * import unittest +from ctypes import (c_byte, c_short, c_int, c_long, c_longlong, + c_ubyte, c_ushort, c_uint, c_ulong, c_ulonglong, + c_float, c_double, c_longdouble, c_bool, c_char) + subclasses = [] for base in [c_byte, c_short, c_int, c_long, c_longlong, @@ -9,11 +12,12 @@ class X(base): pass subclasses.append(X) + class X(c_char): pass -# This test checks if the __repr__ is correct for subclasses of simple types +# This test checks if the __repr__ is correct for subclasses of simple types class ReprTest(unittest.TestCase): def test_numbers(self): for typ in subclasses: @@ -25,5 +29,6 @@ def test_char(self): self.assertEqual("c_char(b'x')", repr(c_char(b'x'))) self.assertEqual(" +# for reference. +# +# Tests also work on POSIX -from ctypes import * +import unittest +from ctypes import POINTER, cast, c_int16 from ctypes import wintypes @@ -38,6 +40,22 @@ def test_variant_bool(self): vb.value = [] self.assertIs(vb.value, False) + def assertIsSigned(self, ctype): + self.assertLess(ctype(-1).value, 0) + + def assertIsUnsigned(self, ctype): + self.assertGreater(ctype(-1).value, 0) + + def test_signedness(self): + for ctype in (wintypes.BYTE, wintypes.WORD, wintypes.DWORD, + wintypes.BOOLEAN, wintypes.UINT, wintypes.ULONG): + with self.subTest(ctype=ctype): + self.assertIsUnsigned(ctype) + + for ctype in (wintypes.BOOL, wintypes.INT, wintypes.LONG): + with self.subTest(ctype=ctype): + self.assertIsSigned(ctype) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_dataclasses.py b/Lib/test/test_dataclasses/__init__.py similarity index 80% rename from Lib/test/test_dataclasses.py rename to Lib/test/test_dataclasses/__init__.py index 46430d3231..1b16da4264 100644 --- a/Lib/test/test_dataclasses.py +++ b/Lib/test/test_dataclasses/__init__.py @@ -5,20 +5,26 @@ from dataclasses import * import abc +import io import pickle import inspect import builtins import types +import weakref +import traceback import unittest from unittest.mock import Mock -from typing import ClassVar, Any, List, Union, Tuple, Dict, Generic, TypeVar, Optional, Protocol +from typing import ClassVar, Any, List, Union, Tuple, Dict, Generic, TypeVar, Optional, Protocol, DefaultDict from typing import get_type_hints -from collections import deque, OrderedDict, namedtuple +from collections import deque, OrderedDict, namedtuple, defaultdict +from copy import deepcopy from functools import total_ordering import typing # Needed for the string "typing.ClassVar[int]" to work as an annotation. import dataclasses # Needed for the string "dataclasses.InitVar[int]" to work as an annotation. +from test import support + # Just any custom exception we can catch. class CustomError(Exception): pass @@ -67,6 +73,50 @@ def test_field_repr(self): self.assertEqual(repr_output, expected_output) + def test_field_recursive_repr(self): + rec_field = field() + rec_field.type = rec_field + rec_field.name = "id" + repr_output = repr(rec_field) + + self.assertIn(",type=...,", repr_output) + + def test_recursive_annotation(self): + class C: + pass + + @dataclass + class D: + C: C = field() + + self.assertIn(",type=...,", repr(D.__dataclass_fields__["C"])) + + def test_dataclass_params_repr(self): + # Even though this is testing an internal implementation detail, + # it's testing a feature we want to make sure is correctly implemented + # for the sake of dataclasses itself + @dataclass(slots=True, frozen=True) + class Some: pass + + repr_output = repr(Some.__dataclass_params__) + expected_output = "_DataclassParams(init=True,repr=True," \ + "eq=True,order=False,unsafe_hash=False,frozen=True," \ + "match_args=True,kw_only=False," \ + "slots=True,weakref_slot=False)" + self.assertEqual(repr_output, expected_output) + + def test_dataclass_params_signature(self): + # Even though this is testing an internal implementation detail, + # it's testing a feature we want to make sure is correctly implemented + # for the sake of dataclasses itself + @dataclass + class Some: pass + + for param in inspect.signature(dataclass).parameters: + if param == 'cls': + continue + self.assertTrue(hasattr(Some.__dataclass_params__, param), msg=param) + def test_named_init_params(self): @dataclass class C: @@ -87,7 +137,7 @@ class C: # Non-defaults following defaults. with self.assertRaisesRegex(TypeError, "non-default argument 'y' follows " - "default argument"): + "default argument 'x'"): @dataclass class C: x: int = 0 @@ -96,7 +146,7 @@ class C: # A derived class adds a non-default field after a default one. with self.assertRaisesRegex(TypeError, "non-default argument 'y' follows " - "default argument"): + "default argument 'x'"): @dataclass class B: x: int = 0 @@ -109,7 +159,7 @@ class C(B): # a field which didn't use to have a default. with self.assertRaisesRegex(TypeError, "non-default argument 'y' follows " - "default argument"): + "default argument 'x'"): @dataclass class B: x: int @@ -230,6 +280,31 @@ class C: c = C('foo') self.assertEqual(c.object, 'foo') + def test_field_named_BUILTINS_frozen(self): + # gh-96151 + @dataclass(frozen=True) + class C: + BUILTINS: int + c = C(5) + self.assertEqual(c.BUILTINS, 5) + + def test_field_with_special_single_underscore_names(self): + # gh-98886 + + @dataclass + class X: + x: int = field(default_factory=lambda: 111) + _dflt_x: int = field(default_factory=lambda: 222) + + X() + + @dataclass + class Y: + y: int = field(default_factory=lambda: 111) + _HAS_DEFAULT_FACTORY: int = 222 + + assert Y(y=222).y == 222 + def test_field_named_like_builtin(self): # Attribute names can shadow built-in names # since code generation is used. @@ -501,6 +576,32 @@ class C: self.assertNotEqual(C(3), C(4, 10)) self.assertNotEqual(C(3, 10), C(4, 10)) + def test_no_unhashable_default(self): + # See bpo-44674. + class Unhashable: + __hash__ = None + + unhashable_re = 'mutable default .* for field a is not allowed' + with self.assertRaisesRegex(ValueError, unhashable_re): + @dataclass + class A: + a: dict = {} + + with self.assertRaisesRegex(ValueError, unhashable_re): + @dataclass + class A: + a: Any = Unhashable() + + # Make sure that the machinery looking for hashability is using the + # class's __hash__, not the instance's __hash__. + with self.assertRaisesRegex(ValueError, unhashable_re): + unhashable = Unhashable() + # This shouldn't make the variable hashable. + unhashable.__hash__ = lambda: 0 + @dataclass + class A: + a: Any = unhashable + def test_hash_field_rules(self): # Test all 6 cases of: # hash=True/False/None @@ -659,8 +760,8 @@ class Point: class Subclass(typ): pass with self.assertRaisesRegex(ValueError, - f"mutable default .*Subclass'>" - ' for field z is not allowed' + "mutable default .*Subclass'>" + " for field z is not allowed" ): @dataclass class Point: @@ -668,12 +769,12 @@ class Point: # Because this is a ClassVar, it can be mutable. @dataclass - class C: + class UsesMutableClassVar: z: ClassVar[typ] = typ() # Because this is a ClassVar, it can be mutable. @dataclass - class C: + class UsesMutableClassVarWithSubType: x: ClassVar[typ] = Subclass() def test_deliberately_mutable_defaults(self): @@ -990,6 +1091,65 @@ def __post_init__(cls): self.assertEqual((c.x, c.y), (3, 4)) self.assertTrue(C.flag) + def test_post_init_not_auto_added(self): + # See bpo-46757, which had proposed always adding __post_init__. As + # Raymond Hettinger pointed out, that would be a breaking change. So, + # add a test to make sure that the current behavior doesn't change. + + @dataclass + class A0: + pass + + @dataclass + class B0: + b_called: bool = False + def __post_init__(self): + self.b_called = True + + @dataclass + class C0(A0, B0): + c_called: bool = False + def __post_init__(self): + super().__post_init__() + self.c_called = True + + # Since A0 has no __post_init__, and one wasn't automatically added + # (because that's the rule: it's never added by @dataclass, it's only + # the class author that can add it), then B0.__post_init__ is called. + # Verify that. + c = C0() + self.assertTrue(c.b_called) + self.assertTrue(c.c_called) + + ###################################### + # Now, the same thing, except A1 defines __post_init__. + @dataclass + class A1: + def __post_init__(self): + pass + + @dataclass + class B1: + b_called: bool = False + def __post_init__(self): + self.b_called = True + + @dataclass + class C1(A1, B1): + c_called: bool = False + def __post_init__(self): + super().__post_init__() + self.c_called = True + + # This time, B1.__post_init__ isn't being called. This mimics what + # would happen if A1.__post_init__ had been automatically added, + # instead of manually added as we see here. This test isn't really + # needed, but I'm including it just to demonstrate the changed + # behavior when A1 does define __post_init__. + c = C1() + self.assertFalse(c.b_called) + self.assertTrue(c.c_called) + def test_class_var(self): # Make sure ClassVars are ignored in __init__, __repr__, etc. @dataclass @@ -1158,6 +1318,29 @@ def __post_init__(self, init_base, init_derived): c = C(10, 11, 50, 51) self.assertEqual(vars(c), {'x': 21, 'y': 101}) + def test_init_var_name_shadowing(self): + # Because dataclasses rely exclusively on `__annotations__` for + # handling InitVar and `__annotations__` preserves shadowed definitions, + # you can actually shadow an InitVar with a method or property. + # + # This only works when there is no default value; `dataclasses` uses the + # actual name (which will be bound to the shadowing method) for default + # values. + @dataclass + class C: + shadowed: InitVar[int] + _shadowed: int = field(init=False) + + def __post_init__(self, shadowed): + self._shadowed = shadowed * 2 + + @property + def shadowed(self): + return self._shadowed * 3 + + c = C(5) + self.assertEqual(c.shadowed, 30) + def test_default_factory(self): # Test a factory that returns a new list. @dataclass @@ -1365,6 +1548,24 @@ class A(types.GenericAlias): self.assertTrue(is_dataclass(type(a))) self.assertTrue(is_dataclass(a)) + def test_is_dataclass_inheritance(self): + @dataclass + class X: + y: int + + class Z(X): + pass + + self.assertTrue(is_dataclass(X), "X should be a dataclass") + self.assertTrue( + is_dataclass(Z), + "Z should be a dataclass because it inherits from X", + ) + z_instance = Z(y=5) + self.assertTrue( + is_dataclass(z_instance), + "z_instance should be a dataclass because it is an instance of Z", + ) def test_helper_fields_with_class_instance(self): # Check that we can call fields() on either a class or instance, @@ -1388,6 +1589,16 @@ class C: pass with self.assertRaisesRegex(TypeError, 'dataclass type or instance'): fields(C()) + def test_clean_traceback_from_fields_exception(self): + stdout = io.StringIO() + try: + fields(object) + except TypeError as exc: + traceback.print_exception(exc, file=stdout) + printed_traceback = stdout.getvalue() + self.assertNotIn("AttributeError", printed_traceback) + self.assertNotIn("__dataclass_fields__", printed_traceback) + def test_helper_asdict(self): # Basic tests for asdict(), it should return a new dictionary. @dataclass @@ -1565,6 +1776,23 @@ class C: self.assertIsNot(d['f'], t) self.assertEqual(d['f'].my_a(), 6) + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_helper_asdict_defaultdict(self): + # Ensure asdict() does not throw exceptions when a + # defaultdict is a member of a dataclass + @dataclass + class C: + mp: DefaultDict[str, List] + + dd = defaultdict(list) + dd["x"].append(12) + c = C(mp=dd) + d = asdict(c) + + self.assertEqual(d, {"mp": {"x": [12]}}) + self.assertTrue(d["mp"] is not c.mp) # make sure defaultdict is copied + def test_helper_astuple(self): # Basic tests for astuple(), it should return a new tuple. @dataclass @@ -1692,6 +1920,23 @@ class C: t = astuple(c, tuple_factory=list) self.assertEqual(t, ['outer', T(1, ['inner', T(11, 12, 13)], 2)]) + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_helper_astuple_defaultdict(self): + # Ensure astuple() does not throw exceptions when a + # defaultdict is a member of a dataclass + @dataclass + class C: + mp: DefaultDict[str, List] + + dd = defaultdict(list) + dd["x"].append(12) + c = C(mp=dd) + t = astuple(c) + + self.assertEqual(t, ({"x": [12]},)) + self.assertTrue(t[0] is not dd) # make sure defaultdict is copied + def test_dynamic_class_creation(self): cls_dict = {'__annotations__': {'x': int, 'y': int}, } @@ -2019,6 +2264,7 @@ def assertDocStrEqual(self, a, b): # whitespace stripped. self.assertEqual(a.replace(' ', ''), b.replace(' ', '')) + @support.requires_docstrings def test_existing_docstring_not_overridden(self): @dataclass class C: @@ -2100,13 +2346,25 @@ class C: self.assertDocStrEqual(C.__doc__, "C(x:collections.deque=)") + def test_docstring_with_no_signature(self): + # See https://github.com/python/cpython/issues/103449 + class Meta(type): + __call__ = dict + class Base(metaclass=Meta): + pass + + @dataclass + class C(Base): + pass + + self.assertDocStrEqual(C.__doc__, "C") + class TestInit(unittest.TestCase): def test_base_has_init(self): class B: def __init__(self): self.z = 100 - pass # Make sure that declaring this class doesn't raise an error. # The issue is that we can't override __init__ in our class, @@ -2129,12 +2387,12 @@ class C(B): self.assertEqual(c.z, 100) def test_no_init(self): - dataclass(init=False) + @dataclass(init=False) class C: i: int = 0 self.assertEqual(C().i, 0) - dataclass(init=False) + @dataclass(init=False) class C: i: int = 2 def __init__(self): @@ -2259,6 +2517,15 @@ def __repr__(self): class TestEq(unittest.TestCase): + def test_recursive_eq(self): + # Test a class with recursive child + @dataclass + class C: + recursive: object = ... + c = C() + c.recursive = c + self.assertEqual(c, c) + def test_no_eq(self): # Test a class with no __eq__ and eq=False. @dataclass(eq=False) @@ -2613,6 +2880,19 @@ class C: c.i = 5 self.assertEqual(c.i, 10) + def test_frozen_empty(self): + @dataclass(frozen=True) + class C: + pass + + c = C() + self.assertFalse(hasattr(c, 'i')) + with self.assertRaises(FrozenInstanceError): + c.i = 5 + self.assertFalse(hasattr(c, 'i')) + with self.assertRaises(FrozenInstanceError): + del c.i + def test_inherit(self): @dataclass(frozen=True) class C: @@ -2641,6 +2921,101 @@ class C: class D(C): j: int + def test_inherit_frozen_mutliple_inheritance(self): + @dataclass + class NotFrozen: + pass + + @dataclass(frozen=True) + class Frozen: + pass + + class NotDataclass: + pass + + for bases in ( + (NotFrozen, Frozen), + (Frozen, NotFrozen), + (Frozen, NotDataclass), + (NotDataclass, Frozen), + ): + with self.subTest(bases=bases): + with self.assertRaisesRegex( + TypeError, + 'cannot inherit non-frozen dataclass from a frozen one', + ): + @dataclass + class NotFrozenChild(*bases): + pass + + for bases in ( + (NotFrozen, Frozen), + (Frozen, NotFrozen), + (NotFrozen, NotDataclass), + (NotDataclass, NotFrozen), + ): + with self.subTest(bases=bases): + with self.assertRaisesRegex( + TypeError, + 'cannot inherit frozen dataclass from a non-frozen one', + ): + @dataclass(frozen=True) + class FrozenChild(*bases): + pass + + def test_inherit_frozen_mutliple_inheritance_regular_mixins(self): + @dataclass(frozen=True) + class Frozen: + pass + + class NotDataclass: + pass + + class C1(Frozen, NotDataclass): + pass + self.assertEqual(C1.__mro__, (C1, Frozen, NotDataclass, object)) + + class C2(NotDataclass, Frozen): + pass + self.assertEqual(C2.__mro__, (C2, NotDataclass, Frozen, object)) + + @dataclass(frozen=True) + class C3(Frozen, NotDataclass): + pass + self.assertEqual(C3.__mro__, (C3, Frozen, NotDataclass, object)) + + @dataclass(frozen=True) + class C4(NotDataclass, Frozen): + pass + self.assertEqual(C4.__mro__, (C4, NotDataclass, Frozen, object)) + + def test_multiple_frozen_dataclasses_inheritance(self): + @dataclass(frozen=True) + class FrozenA: + pass + + @dataclass(frozen=True) + class FrozenB: + pass + + class C1(FrozenA, FrozenB): + pass + self.assertEqual(C1.__mro__, (C1, FrozenA, FrozenB, object)) + + class C2(FrozenB, FrozenA): + pass + self.assertEqual(C2.__mro__, (C2, FrozenB, FrozenA, object)) + + @dataclass(frozen=True) + class C3(FrozenA, FrozenB): + pass + self.assertEqual(C3.__mro__, (C3, FrozenA, FrozenB, object)) + + @dataclass(frozen=True) + class C4(FrozenB, FrozenA): + pass + self.assertEqual(C4.__mro__, (C4, FrozenB, FrozenA, object)) + def test_inherit_nonfrozen_from_empty(self): @dataclass class C: @@ -2736,6 +3111,37 @@ class S(D): self.assertEqual(s.y, 10) self.assertEqual(s.cached, True) + with self.assertRaises(FrozenInstanceError): + del s.x + self.assertEqual(s.x, 3) + with self.assertRaises(FrozenInstanceError): + del s.y + self.assertEqual(s.y, 10) + del s.cached + self.assertFalse(hasattr(s, 'cached')) + with self.assertRaises(AttributeError) as cm: + del s.cached + self.assertNotIsInstance(cm.exception, FrozenInstanceError) + + def test_non_frozen_normal_derived_from_empty_frozen(self): + @dataclass(frozen=True) + class D: + pass + + class S(D): + pass + + s = S() + self.assertFalse(hasattr(s, 'x')) + s.x = 5 + self.assertEqual(s.x, 5) + + del s.x + self.assertFalse(hasattr(s, 'x')) + with self.assertRaises(AttributeError) as cm: + del s.x + self.assertNotIsInstance(cm.exception, FrozenInstanceError) + def test_overwriting_frozen(self): # frozen uses __setattr__ and __delattr__. with self.assertRaisesRegex(TypeError, @@ -2774,10 +3180,50 @@ class C: with self.assertRaisesRegex(TypeError, 'unhashable type'): hash(C({})) + def test_frozen_deepcopy_without_slots(self): + # see: https://github.com/python/cpython/issues/89683 + @dataclass(frozen=True, slots=False) + class C: + s: str + + c = C('hello') + self.assertEqual(deepcopy(c), c) + + def test_frozen_deepcopy_with_slots(self): + # see: https://github.com/python/cpython/issues/89683 + with self.subTest('generated __slots__'): + @dataclass(frozen=True, slots=True) + class C: + s: str + + c = C('hello') + self.assertEqual(deepcopy(c), c) + + with self.subTest('user-defined __slots__ and no __{get,set}state__'): + @dataclass(frozen=True, slots=False) + class C: + __slots__ = ('s',) + s: str + + # with user-defined slots, __getstate__ and __setstate__ are not + # automatically added, hence the error + err = r"^cannot\ assign\ to\ field\ 's'$" + self.assertRaisesRegex(FrozenInstanceError, err, deepcopy, C('')) + + with self.subTest('user-defined __slots__ and __{get,set}state__'): + @dataclass(frozen=True, slots=False) + class C: + __slots__ = ('s',) + __getstate__ = dataclasses._dataclass_getstate + __setstate__ = dataclasses._dataclass_setstate + + s: str + + c = C('hello') + self.assertEqual(deepcopy(c), c) + class TestSlots(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_simple(self): @dataclass class C: @@ -2819,8 +3265,6 @@ class Derived(Base): # We can add a new field to the derived instance. d.z = 10 - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_generated_slots(self): @dataclass(slots=True) class C: @@ -2845,23 +3289,58 @@ class C: x: int def test_generated_slots_value(self): - @dataclass(slots=True) - class Base: - x: int - self.assertEqual(Base.__slots__, ('x',)) + class Root: + __slots__ = {'x'} + + class Root2(Root): + __slots__ = {'k': '...', 'j': ''} + + class Root3(Root2): + __slots__ = ['h'] + + class Root4(Root3): + __slots__ = 'aa' @dataclass(slots=True) - class Delivered(Base): + class Base(Root4): y: int + j: str + h: str + + self.assertEqual(Base.__slots__, ('y', )) + + @dataclass(slots=True) + class Derived(Base): + aa: float + x: str + z: int + k: str + h: str - self.assertEqual(Delivered.__slots__, ('x', 'y')) + self.assertEqual(Derived.__slots__, ('z', )) @dataclass - class AnotherDelivered(Base): + class AnotherDerived(Base): z: int - self.assertTrue('__slots__' not in AnotherDelivered.__dict__) + self.assertNotIn('__slots__', AnotherDerived.__dict__) + + def test_cant_inherit_from_iterator_slots(self): + + class Root: + __slots__ = iter(['a']) + + class Root2(Root): + __slots__ = ('b', ) + + with self.assertRaisesRegex( + TypeError, + "^Slots of 'Root' cannot be determined" + ): + @dataclass(slots=True) + class C(Root2): + x: int def test_returns_new_class(self): class A: @@ -2900,6 +3379,74 @@ def test_frozen_pickle(self): self.assertIsNot(obj, p) self.assertEqual(obj, p) + @dataclass(frozen=True, slots=True) + class FrozenSlotsGetStateClass: + foo: str + bar: int + + getstate_called: bool = field(default=False, compare=False) + + def __getstate__(self): + object.__setattr__(self, 'getstate_called', True) + return [self.foo, self.bar] + + @dataclass(frozen=True, slots=True) + class FrozenSlotsSetStateClass: + foo: str + bar: int + + setstate_called: bool = field(default=False, compare=False) + + def __setstate__(self, state): + object.__setattr__(self, 'setstate_called', True) + object.__setattr__(self, 'foo', state[0]) + object.__setattr__(self, 'bar', state[1]) + + @dataclass(frozen=True, slots=True) + class FrozenSlotsAllStateClass: + foo: str + bar: int + + getstate_called: bool = field(default=False, compare=False) + setstate_called: bool = field(default=False, compare=False) + + def __getstate__(self): + object.__setattr__(self, 'getstate_called', True) + return [self.foo, self.bar] + + def __setstate__(self, state): + object.__setattr__(self, 'setstate_called', True) + object.__setattr__(self, 'foo', state[0]) + object.__setattr__(self, 'bar', state[1]) + + def test_frozen_slots_pickle_custom_state(self): + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(proto=proto): + obj = self.FrozenSlotsGetStateClass('a', 1) + dumped = pickle.dumps(obj, protocol=proto) + + self.assertTrue(obj.getstate_called) + self.assertEqual(obj, pickle.loads(dumped)) + + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(proto=proto): + obj = self.FrozenSlotsSetStateClass('a', 1) + obj2 = pickle.loads(pickle.dumps(obj, protocol=proto)) + + self.assertTrue(obj2.setstate_called) + self.assertEqual(obj, obj2) + + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(proto=proto): + obj = self.FrozenSlotsAllStateClass('a', 1) + dumped = pickle.dumps(obj, protocol=proto) + + self.assertTrue(obj.getstate_called) + + obj2 = pickle.loads(dumped) + self.assertTrue(obj2.setstate_called) + self.assertEqual(obj, obj2) + def test_slots_with_default_no_init(self): # Originally reported in bpo-44649. @dataclass(slots=True) @@ -2922,6 +3469,311 @@ class A: self.assertEqual(obj.a, 'a') self.assertEqual(obj.b, 'b') + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_slots_no_weakref(self): + @dataclass(slots=True) + class A: + # No weakref. + pass + + self.assertNotIn("__weakref__", A.__slots__) + a = A() + with self.assertRaisesRegex(TypeError, + "cannot create weak reference"): + weakref.ref(a) + with self.assertRaises(AttributeError): + a.__weakref__ + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_slots_weakref(self): + @dataclass(slots=True, weakref_slot=True) + class A: + a: int + + self.assertIn("__weakref__", A.__slots__) + a = A(1) + a_ref = weakref.ref(a) + + self.assertIs(a.__weakref__, a_ref) + + def test_slots_weakref_base_str(self): + class Base: + __slots__ = '__weakref__' + + @dataclass(slots=True) + class A(Base): + a: int + + # __weakref__ is in the base class, not A. But an A is still weakref-able. + self.assertIn("__weakref__", Base.__slots__) + self.assertNotIn("__weakref__", A.__slots__) + a = A(1) + weakref.ref(a) + + def test_slots_weakref_base_tuple(self): + # Same as test_slots_weakref_base, but use a tuple instead of a string + # in the base class. + class Base: + __slots__ = ('__weakref__',) + + @dataclass(slots=True) + class A(Base): + a: int + + # __weakref__ is in the base class, not A. But an A is still + # weakref-able. + self.assertIn("__weakref__", Base.__slots__) + self.assertNotIn("__weakref__", A.__slots__) + a = A(1) + weakref.ref(a) + + def test_weakref_slot_without_slot(self): + with self.assertRaisesRegex(TypeError, + "weakref_slot is True but slots is False"): + @dataclass(weakref_slot=True) + class A: + a: int + + def test_weakref_slot_make_dataclass(self): + A = make_dataclass('A', [('a', int),], slots=True, weakref_slot=True) + self.assertIn("__weakref__", A.__slots__) + a = A(1) + weakref.ref(a) + + # And make sure if raises if slots=True is not given. + with self.assertRaisesRegex(TypeError, + "weakref_slot is True but slots is False"): + B = make_dataclass('B', [('a', int),], weakref_slot=True) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_weakref_slot_subclass_weakref_slot(self): + @dataclass(slots=True, weakref_slot=True) + class Base: + field: int + + # A *can* also specify weakref_slot=True if it wants to (gh-93521) + @dataclass(slots=True, weakref_slot=True) + class A(Base): + ... + + # __weakref__ is in the base class, not A. But an instance of A + # is still weakref-able. + self.assertIn("__weakref__", Base.__slots__) + self.assertNotIn("__weakref__", A.__slots__) + a = A(1) + a_ref = weakref.ref(a) + self.assertIs(a.__weakref__, a_ref) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_weakref_slot_subclass_no_weakref_slot(self): + @dataclass(slots=True, weakref_slot=True) + class Base: + field: int + + @dataclass(slots=True) + class A(Base): + ... + + # __weakref__ is in the base class, not A. Even though A doesn't + # specify weakref_slot, it should still be weakref-able. + self.assertIn("__weakref__", Base.__slots__) + self.assertNotIn("__weakref__", A.__slots__) + a = A(1) + a_ref = weakref.ref(a) + self.assertIs(a.__weakref__, a_ref) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_weakref_slot_normal_base_weakref_slot(self): + class Base: + __slots__ = ('__weakref__',) + + @dataclass(slots=True, weakref_slot=True) + class A(Base): + field: int + + # __weakref__ is in the base class, not A. But an instance of + # A is still weakref-able. + self.assertIn("__weakref__", Base.__slots__) + self.assertNotIn("__weakref__", A.__slots__) + a = A(1) + a_ref = weakref.ref(a) + self.assertIs(a.__weakref__, a_ref) + + def test_dataclass_derived_weakref_slot(self): + class A: + pass + + @dataclass(slots=True, weakref_slot=True) + class B(A): + pass + + self.assertEqual(B.__slots__, ()) + B() + + def test_dataclass_derived_generic(self): + T = typing.TypeVar('T') + + @dataclass(slots=True, weakref_slot=True) + class A(typing.Generic[T]): + pass + self.assertEqual(A.__slots__, ('__weakref__',)) + self.assertTrue(A.__weakref__) + A() + + @dataclass(slots=True, weakref_slot=True) + class B[T2]: + pass + self.assertEqual(B.__slots__, ('__weakref__',)) + self.assertTrue(B.__weakref__) + B() + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_dataclass_derived_generic_from_base(self): + T = typing.TypeVar('T') + + class RawBase: ... + + @dataclass(slots=True, weakref_slot=True) + class C1(typing.Generic[T], RawBase): + pass + self.assertEqual(C1.__slots__, ()) + self.assertTrue(C1.__weakref__) + C1() + @dataclass(slots=True, weakref_slot=True) + class C2(RawBase, typing.Generic[T]): + pass + self.assertEqual(C2.__slots__, ()) + self.assertTrue(C2.__weakref__) + C2() + + @dataclass(slots=True, weakref_slot=True) + class D[T2](RawBase): + pass + self.assertEqual(D.__slots__, ()) + self.assertTrue(D.__weakref__) + D() + + def test_dataclass_derived_generic_from_slotted_base(self): + T = typing.TypeVar('T') + + class WithSlots: + __slots__ = ('a', 'b') + + @dataclass(slots=True, weakref_slot=True) + class E1(WithSlots, Generic[T]): + pass + self.assertEqual(E1.__slots__, ('__weakref__',)) + self.assertTrue(E1.__weakref__) + E1() + @dataclass(slots=True, weakref_slot=True) + class E2(Generic[T], WithSlots): + pass + self.assertEqual(E2.__slots__, ('__weakref__',)) + self.assertTrue(E2.__weakref__) + E2() + + @dataclass(slots=True, weakref_slot=True) + class F[T2](WithSlots): + pass + self.assertEqual(F.__slots__, ('__weakref__',)) + self.assertTrue(F.__weakref__) + F() + + def test_dataclass_derived_generic_from_slotted_base_with_weakref(self): + T = typing.TypeVar('T') + + class WithWeakrefSlot: + __slots__ = ('__weakref__',) + + @dataclass(slots=True, weakref_slot=True) + class G1(WithWeakrefSlot, Generic[T]): + pass + self.assertEqual(G1.__slots__, ()) + self.assertTrue(G1.__weakref__) + G1() + @dataclass(slots=True, weakref_slot=True) + class G2(Generic[T], WithWeakrefSlot): + pass + self.assertEqual(G2.__slots__, ()) + self.assertTrue(G2.__weakref__) + G2() + + @dataclass(slots=True, weakref_slot=True) + class H[T2](WithWeakrefSlot): + pass + self.assertEqual(H.__slots__, ()) + self.assertTrue(H.__weakref__) + H() + + def test_dataclass_slot_dict(self): + class WithDictSlot: + __slots__ = ('__dict__',) + + @dataclass(slots=True) + class A(WithDictSlot): ... + + self.assertEqual(A.__slots__, ()) + self.assertEqual(A().__dict__, {}) + A() + + @support.cpython_only + def test_dataclass_slot_dict_ctype(self): + # https://github.com/python/cpython/issues/123935 + from test.support import import_helper + # Skips test if `_testcapi` is not present: + _testcapi = import_helper.import_module('_testcapi') + + @dataclass(slots=True) + class HasDictOffset(_testcapi.HeapCTypeWithDict): + __dict__: dict = {} + self.assertNotEqual(_testcapi.HeapCTypeWithDict.__dictoffset__, 0) + self.assertEqual(HasDictOffset.__slots__, ()) + + @dataclass(slots=True) + class DoesNotHaveDictOffset(_testcapi.HeapCTypeWithWeakref): + __dict__: dict = {} + self.assertEqual(_testcapi.HeapCTypeWithWeakref.__dictoffset__, 0) + self.assertEqual(DoesNotHaveDictOffset.__slots__, ('__dict__',)) + + @support.cpython_only + def test_slots_with_wrong_init_subclass(self): + # TODO: This test is for a kinda-buggy behavior. + # Ideally, it should be fixed and `__init_subclass__` + # should be fully supported in the future versions. + # See https://github.com/python/cpython/issues/91126 + class WrongSuper: + def __init_subclass__(cls, arg): + pass + + with self.assertRaisesRegex( + TypeError, + "missing 1 required positional argument: 'arg'", + ): + @dataclass(slots=True) + class WithWrongSuper(WrongSuper, arg=1): + pass + + class CorrectSuper: + args = [] + def __init_subclass__(cls, arg="default"): + cls.args.append(arg) + + @dataclass(slots=True) + class WithCorrectSuper(CorrectSuper): + pass + + # __init_subclass__ is called twice: once for `WithCorrectSuper` + # and once for `WithCorrectSuper__slots__` new class + # that we create internally. + self.assertEqual(CorrectSuper.args, ["default", "default"]) + + class TestDescriptors(unittest.TestCase): def test_set_name(self): # See bpo-33141. @@ -3206,10 +4058,10 @@ class C: self.assertEqual(C(10).x, 10) def test_classvar_module_level_import(self): - from test import dataclass_module_1 - from test import dataclass_module_1_str - from test import dataclass_module_2 - from test import dataclass_module_2_str + from test.test_dataclasses import dataclass_module_1 + from test.test_dataclasses import dataclass_module_1_str + from test.test_dataclasses import dataclass_module_2 + from test.test_dataclasses import dataclass_module_2_str for m in (dataclass_module_1, dataclass_module_1_str, dataclass_module_2, dataclass_module_2_str, @@ -3247,7 +4099,7 @@ def test_classvar_module_level_import(self): self.assertNotIn('not_iv4', c.__dict__) def test_text_annotations(self): - from test import dataclass_textanno + from test.test_dataclasses import dataclass_textanno self.assertEqual( get_type_hints(dataclass_textanno.Bar), @@ -3258,6 +4110,15 @@ def test_text_annotations(self): 'return': type(None)}) +ByMakeDataClass = make_dataclass('ByMakeDataClass', [('x', int)]) +ManualModuleMakeDataClass = make_dataclass('ManualModuleMakeDataClass', + [('x', int)], + module=__name__) +WrongNameMakeDataclass = make_dataclass('Wrong', [('x', int)]) +WrongModuleMakeDataclass = make_dataclass('WrongModuleMakeDataclass', + [('x', int)], + module='custom') + class TestMakeDataclass(unittest.TestCase): def test_simple(self): C = make_dataclass('C', @@ -3367,6 +4228,36 @@ def test_no_types(self): 'y': int, 'z': 'typing.Any'}) + def test_module_attr(self): + self.assertEqual(ByMakeDataClass.__module__, __name__) + self.assertEqual(ByMakeDataClass(1).__module__, __name__) + self.assertEqual(WrongModuleMakeDataclass.__module__, "custom") + Nested = make_dataclass('Nested', []) + self.assertEqual(Nested.__module__, __name__) + self.assertEqual(Nested().__module__, __name__) + + def test_pickle_support(self): + for klass in [ByMakeDataClass, ManualModuleMakeDataClass]: + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(proto=proto): + self.assertEqual( + pickle.loads(pickle.dumps(klass, proto)), + klass, + ) + self.assertEqual( + pickle.loads(pickle.dumps(klass(1), proto)), + klass(1), + ) + + def test_cannot_be_pickled(self): + for klass in [WrongNameMakeDataclass, WrongModuleMakeDataclass]: + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(proto=proto): + with self.assertRaises(pickle.PickleError): + pickle.dumps(klass, proto) + with self.assertRaises(pickle.PickleError): + pickle.dumps(klass(1), proto) + def test_invalid_type_specification(self): for bad_field in [(), (1, 2, 3, 4), @@ -3448,9 +4339,9 @@ class C: self.assertEqual((c1.x, c1.y, c1.z, c1.t), (3, 2, 10, 100)) - with self.assertRaisesRegex(ValueError, 'init=False'): + with self.assertRaisesRegex(TypeError, 'init=False'): replace(c, x=3, z=20, t=50) - with self.assertRaisesRegex(ValueError, 'init=False'): + with self.assertRaisesRegex(TypeError, 'init=False'): replace(c, z=20) replace(c, x=3, z=20, t=50) @@ -3503,10 +4394,10 @@ class C: self.assertEqual((c1.x, c1.y), (5, 10)) # Trying to replace y is an error. - with self.assertRaisesRegex(ValueError, 'init=False'): + with self.assertRaisesRegex(TypeError, 'init=False'): replace(c, x=2, y=30) - with self.assertRaisesRegex(ValueError, 'init=False'): + with self.assertRaisesRegex(TypeError, 'init=False'): replace(c, y=30) def test_classvar(self): @@ -3539,8 +4430,8 @@ def __post_init__(self, y): c = C(1, 10) self.assertEqual(c.x, 10) - with self.assertRaisesRegex(ValueError, r"InitVar 'y' must be " - "specified with replace()"): + with self.assertRaisesRegex(TypeError, r"InitVar 'y' must be " + r"specified with replace\(\)"): replace(c, x=3) c = replace(c, x=3, y=5) self.assertEqual(c.x, 15) @@ -3670,6 +4561,7 @@ class Date(Ordered): self.assertFalse(inspect.isabstract(Date)) self.assertGreater(Date(2020,12,25), Date(2020,8,31)) + # TODO: RUSTPYTHON @unittest.expectedFailure def test_maintain_abc(self): class A(abc.ABC): @@ -3684,7 +4576,7 @@ class Date(A): day: 'int' self.assertTrue(inspect.isabstract(Date)) - msg = 'class Date with abstract method foo' + msg = "class Date without an implementation for abstract method 'foo'" self.assertRaisesRegex(TypeError, msg, Date) @@ -3930,7 +4822,7 @@ class A: # But this usage is okay, since it's not using KW_ONLY. @dataclass - class A: + class NoDuplicateKwOnlyAnnotation: a: int _: KW_ONLY b: int @@ -3938,13 +4830,13 @@ class A: # And if inheriting, it's okay. @dataclass - class A: + class BaseUsesKwOnly: a: int _: KW_ONLY b: int c: int @dataclass - class B(A): + class SubclassUsesKwOnly(BaseUsesKwOnly): _: KW_ONLY d: int @@ -4007,7 +4899,7 @@ class A: # Make sure we still check for non-kwarg non-defaults not following # defaults. - err_regex = "non-default argument 'z' follows default argument" + err_regex = "non-default argument 'z' follows default argument 'a'" with self.assertRaisesRegex(TypeError, err_regex): @dataclass class A: @@ -4030,4 +4922,4 @@ def test_make_dataclass(self): if __name__ == '__main__': - unittest.main() + unittest.main() \ No newline at end of file diff --git a/Lib/test/test_dataclasses/dataclass_module_1.py b/Lib/test/test_dataclasses/dataclass_module_1.py new file mode 100644 index 0000000000..87a33f8191 --- /dev/null +++ b/Lib/test/test_dataclasses/dataclass_module_1.py @@ -0,0 +1,32 @@ +#from __future__ import annotations +USING_STRINGS = False + +# dataclass_module_1.py and dataclass_module_1_str.py are identical +# except only the latter uses string annotations. + +import dataclasses +import typing + +T_CV2 = typing.ClassVar[int] +T_CV3 = typing.ClassVar + +T_IV2 = dataclasses.InitVar[int] +T_IV3 = dataclasses.InitVar + +@dataclasses.dataclass +class CV: + T_CV4 = typing.ClassVar + cv0: typing.ClassVar[int] = 20 + cv1: typing.ClassVar = 30 + cv2: T_CV2 + cv3: T_CV3 + not_cv4: T_CV4 # When using string annotations, this field is not recognized as a ClassVar. + +@dataclasses.dataclass +class IV: + T_IV4 = dataclasses.InitVar + iv0: dataclasses.InitVar[int] + iv1: dataclasses.InitVar + iv2: T_IV2 + iv3: T_IV3 + not_iv4: T_IV4 # When using string annotations, this field is not recognized as an InitVar. diff --git a/Lib/test/test_dataclasses/dataclass_module_1_str.py b/Lib/test/test_dataclasses/dataclass_module_1_str.py new file mode 100644 index 0000000000..6de490b7ad --- /dev/null +++ b/Lib/test/test_dataclasses/dataclass_module_1_str.py @@ -0,0 +1,32 @@ +from __future__ import annotations +USING_STRINGS = True + +# dataclass_module_1.py and dataclass_module_1_str.py are identical +# except only the latter uses string annotations. + +import dataclasses +import typing + +T_CV2 = typing.ClassVar[int] +T_CV3 = typing.ClassVar + +T_IV2 = dataclasses.InitVar[int] +T_IV3 = dataclasses.InitVar + +@dataclasses.dataclass +class CV: + T_CV4 = typing.ClassVar + cv0: typing.ClassVar[int] = 20 + cv1: typing.ClassVar = 30 + cv2: T_CV2 + cv3: T_CV3 + not_cv4: T_CV4 # When using string annotations, this field is not recognized as a ClassVar. + +@dataclasses.dataclass +class IV: + T_IV4 = dataclasses.InitVar + iv0: dataclasses.InitVar[int] + iv1: dataclasses.InitVar + iv2: T_IV2 + iv3: T_IV3 + not_iv4: T_IV4 # When using string annotations, this field is not recognized as an InitVar. diff --git a/Lib/test/test_dataclasses/dataclass_module_2.py b/Lib/test/test_dataclasses/dataclass_module_2.py new file mode 100644 index 0000000000..68fb733e29 --- /dev/null +++ b/Lib/test/test_dataclasses/dataclass_module_2.py @@ -0,0 +1,32 @@ +#from __future__ import annotations +USING_STRINGS = False + +# dataclass_module_2.py and dataclass_module_2_str.py are identical +# except only the latter uses string annotations. + +from dataclasses import dataclass, InitVar +from typing import ClassVar + +T_CV2 = ClassVar[int] +T_CV3 = ClassVar + +T_IV2 = InitVar[int] +T_IV3 = InitVar + +@dataclass +class CV: + T_CV4 = ClassVar + cv0: ClassVar[int] = 20 + cv1: ClassVar = 30 + cv2: T_CV2 + cv3: T_CV3 + not_cv4: T_CV4 # When using string annotations, this field is not recognized as a ClassVar. + +@dataclass +class IV: + T_IV4 = InitVar + iv0: InitVar[int] + iv1: InitVar + iv2: T_IV2 + iv3: T_IV3 + not_iv4: T_IV4 # When using string annotations, this field is not recognized as an InitVar. diff --git a/Lib/test/test_dataclasses/dataclass_module_2_str.py b/Lib/test/test_dataclasses/dataclass_module_2_str.py new file mode 100644 index 0000000000..b363d17c17 --- /dev/null +++ b/Lib/test/test_dataclasses/dataclass_module_2_str.py @@ -0,0 +1,32 @@ +from __future__ import annotations +USING_STRINGS = True + +# dataclass_module_2.py and dataclass_module_2_str.py are identical +# except only the latter uses string annotations. + +from dataclasses import dataclass, InitVar +from typing import ClassVar + +T_CV2 = ClassVar[int] +T_CV3 = ClassVar + +T_IV2 = InitVar[int] +T_IV3 = InitVar + +@dataclass +class CV: + T_CV4 = ClassVar + cv0: ClassVar[int] = 20 + cv1: ClassVar = 30 + cv2: T_CV2 + cv3: T_CV3 + not_cv4: T_CV4 # When using string annotations, this field is not recognized as a ClassVar. + +@dataclass +class IV: + T_IV4 = InitVar + iv0: InitVar[int] + iv1: InitVar + iv2: T_IV2 + iv3: T_IV3 + not_iv4: T_IV4 # When using string annotations, this field is not recognized as an InitVar. diff --git a/Lib/test/test_dataclasses/dataclass_textanno.py b/Lib/test/test_dataclasses/dataclass_textanno.py new file mode 100644 index 0000000000..3eb6c943d4 --- /dev/null +++ b/Lib/test/test_dataclasses/dataclass_textanno.py @@ -0,0 +1,12 @@ +from __future__ import annotations + +import dataclasses + + +class Foo: + pass + + +@dataclasses.dataclass +class Bar: + foo: Foo diff --git a/Lib/test/test_decimal.py b/Lib/test/test_decimal.py index 0493d6a41d..01b0c06196 100644 --- a/Lib/test/test_decimal.py +++ b/Lib/test/test_decimal.py @@ -24,6 +24,7 @@ with the corresponding argument. """ +import logging import math import os, sys import operator @@ -812,8 +813,6 @@ def test_explicit_context_create_from_float(self): x = random.expovariate(0.01) * (random.random() * 2.0 - 1.0) self.assertEqual(x, float(nc.create_decimal(x))) # roundtrip - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_unicode_digits(self): Decimal = self.decimal.Decimal @@ -831,6 +830,11 @@ class CExplicitConstructionTest(ExplicitConstructionTest, unittest.TestCase): class PyExplicitConstructionTest(ExplicitConstructionTest, unittest.TestCase): decimal = P + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_unicode_digits(self): # TODO(RUSTPYTHON): Remove this test when it pass + return super().test_unicode_digits() + class ImplicitConstructionTest: '''Unit tests for Implicit Construction cases of Decimal.''' @@ -916,8 +920,6 @@ class PyImplicitConstructionTest(ImplicitConstructionTest, unittest.TestCase): class FormatTest: '''Unit tests for the format function.''' - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_formatting(self): Decimal = self.decimal.Decimal @@ -1113,6 +1115,13 @@ def test_formatting(self): ('z>z6.1f', '-0.', 'zzz0.0'), ('x>z6.1f', '-0.', 'xxx0.0'), ('🖤>z6.1f', '-0.', '🖤🖤🖤0.0'), # multi-byte fill char + ('\x00>z6.1f', '-0.', '\x00\x00\x000.0'), # null fill char + + # issue 114563 ('z' format on F type in cdecimal) + ('z3,.10F', '-6.24E-323', '0.0000000000'), + + # issue 91060 ('#' format in cdecimal) + ('#', '0', '0.'), # issue 6850 ('a=-7.0', '0.12345', 'aaaa0.1'), @@ -1128,8 +1137,6 @@ def test_formatting(self): # bytes format argument self.assertRaises(TypeError, Decimal(1).__format__, b'-020') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_negative_zero_format_directed_rounding(self): with self.decimal.localcontext() as ctx: ctx.rounding = ROUND_CEILING @@ -1228,7 +1235,31 @@ def get_fmt(x, override=None, fmt='n'): self.assertEqual(get_fmt(Decimal('-1.5'), dotsep_wide, '020n'), '-0\u00b4000\u00b4000\u00b4000\u00b4001\u00bf5') - @run_with_locale('LC_ALL', 'ps_AF') + def test_deprecated_N_format(self): + Decimal = self.decimal.Decimal + h = Decimal('6.62607015e-34') + if self.decimal == C: + with self.assertWarns(DeprecationWarning) as cm: + r = format(h, 'N') + self.assertEqual(cm.filename, __file__) + self.assertEqual(r, format(h, 'n').upper()) + with self.assertWarns(DeprecationWarning) as cm: + r = format(h, '010.3N') + self.assertEqual(cm.filename, __file__) + self.assertEqual(r, format(h, '010.3n').upper()) + else: + self.assertRaises(ValueError, format, h, 'N') + self.assertRaises(ValueError, format, h, '010.3N') + with warnings_helper.check_no_warnings(self): + self.assertEqual(format(h, 'N>10.3'), 'NN6.63E-34') + self.assertEqual(format(h, 'N>10.3n'), 'NN6.63e-34') + self.assertEqual(format(h, 'N>10.3e'), 'N6.626e-34') + self.assertEqual(format(h, 'N>10.3f'), 'NNNNN0.000') + self.assertRaises(ValueError, format, h, '>Nf') + self.assertRaises(ValueError, format, h, '10Nf') + self.assertRaises(ValueError, format, h, 'Nx') + + @run_with_locale('LC_ALL', 'ps_AF', '') def test_wide_char_separator_decimal_point(self): # locale with wide char separator and decimal point Decimal = self.decimal.Decimal @@ -1911,8 +1942,6 @@ def hashit(d): x = 1100 ** 1248 self.assertEqual(hashit(Decimal(x)), hashit(x)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_hash_method_nan(self): Decimal = self.decimal.Decimal self.assertRaises(TypeError, hash, Decimal('sNaN')) @@ -2048,7 +2077,9 @@ def test_tonum_methods(self): #to quantize, which is already extensively tested test_triples = [ ('123.456', -4, '0E+4'), + ('-123.456', -4, '-0E+4'), ('123.456', -3, '0E+3'), + ('-123.456', -3, '-0E+3'), ('123.456', -2, '1E+2'), ('123.456', -1, '1.2E+2'), ('123.456', 0, '123'), @@ -2718,8 +2749,6 @@ def test_quantize(self): x = d.quantize(context=c, exp=Decimal("1e797"), rounding=ROUND_DOWN) self.assertEqual(x, Decimal('8.71E+799')) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_complex(self): Decimal = self.decimal.Decimal @@ -2894,6 +2923,7 @@ class CPythonAPItests(PythonAPItests, unittest.TestCase): class PyPythonAPItests(PythonAPItests, unittest.TestCase): decimal = P + class ContextAPItests: def test_none_args(self): @@ -3663,8 +3693,6 @@ def test_localcontextarg(self): self.assertIsNot(new_ctx, set_ctx, 'did not copy the context') self.assertIs(set_ctx, enter_ctx, '__enter__ returned wrong context') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_localcontext_kwargs(self): with self.decimal.localcontext( prec=10, rounding=ROUND_HALF_DOWN, @@ -3693,8 +3721,6 @@ def test_localcontext_kwargs(self): self.assertRaises(TypeError, self.decimal.localcontext, Emin="") self.assertRaises(TypeError, self.decimal.localcontext, Emax="") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_local_context_kwargs_does_not_overwrite_existing_argument(self): ctx = self.decimal.getcontext() orig_prec = ctx.prec @@ -4362,7 +4388,8 @@ def test_module_attributes(self): self.assertEqual(C.__version__, P.__version__) - self.assertEqual(dir(C), dir(P)) + self.assertLessEqual(set(dir(C)), set(dir(P))) + self.assertEqual([n for n in dir(C) if n[:2] != '__'], sorted(P.__all__)) def test_context_attributes(self): @@ -4438,6 +4465,15 @@ def test_implicit_context(self): self.assertIs(Decimal("NaN").fma(7, 1).is_nan(), True) # three arg power self.assertEqual(pow(Decimal(10), 2, 7), 2) + if self.decimal == C: + self.assertEqual(pow(10, Decimal(2), 7), 2) + self.assertEqual(pow(10, 2, Decimal(7)), 2) + else: + # XXX: Three-arg power doesn't use __rpow__. + self.assertRaises(TypeError, pow, 10, Decimal(2), 7) + # XXX: There is no special method to dispatch on the + # third arg of three-arg power. + self.assertRaises(TypeError, pow, 10, 2, Decimal(7)) # exp self.assertEqual(Decimal("1.01").exp(), 3) # is_normal @@ -4648,6 +4684,11 @@ def tearDown(self): sys.set_int_max_str_digits(self._previous_int_limit) super().tearDown() + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_implicit_context(self): # TODO(RUSTPYTHON): Remove this test when it pass + return super().test_implicit_context() + class PyFunctionality(unittest.TestCase): """Extra functionality in decimal.py""" @@ -4699,9 +4740,33 @@ def test_py_exact_power(self): c.prec = 1 x = Decimal("152587890625") ** Decimal('-0.5') + self.assertEqual(x, Decimal('3e-6')) + c.prec = 2 + x = Decimal("152587890625") ** Decimal('-0.5') + self.assertEqual(x, Decimal('2.6e-6')) + c.prec = 3 + x = Decimal("152587890625") ** Decimal('-0.5') + self.assertEqual(x, Decimal('2.56e-6')) + c.prec = 28 + x = Decimal("152587890625") ** Decimal('-0.5') + self.assertEqual(x, Decimal('2.56e-6')) + c.prec = 201 x = Decimal(2**578) ** Decimal("-0.5") + # See https://github.com/python/cpython/issues/118027 + # Testing for an exact power could appear to hang, in the Python + # version, as it attempted to compute 10**(MAX_EMAX + 1). + # Fixed via https://github.com/python/cpython/pull/118503. + c.prec = P.MAX_PREC + c.Emax = P.MAX_EMAX + c.Emin = P.MIN_EMIN + c.traps[P.Inexact] = 1 + D2 = Decimal(2) + # If the bug is still present, the next statement won't complete. + res = D2 ** 117 + self.assertEqual(res, 1 << 117) + def test_py_immutability_operations(self): # Do operations and check that it didn't change internal objects. Decimal = P.Decimal @@ -5625,6 +5690,25 @@ def __abs__(self): self.assertEqual(Decimal.from_float(cls(101.1)), Decimal.from_float(101.1)) + def test_c_immutable_types(self): + SignalDict = type(C.Context().flags) + SignalDictMixin = SignalDict.__bases__[0] + ContextManager = type(C.localcontext()) + types = ( + SignalDictMixin, + ContextManager, + C.Decimal, + C.Context, + ) + for tp in types: + with self.subTest(tp=tp): + with self.assertRaisesRegex(TypeError, "immutable"): + tp.foo = 1 + + def test_c_disallow_instantiation(self): + ContextManager = type(C.localcontext()) + check_disallow_instantiation(self, ContextManager) + def test_c_signaldict_segfault(self): # See gh-106263 for details. SignalDict = type(C.Context().flags) @@ -5655,6 +5739,20 @@ def test_c_signaldict_segfault(self): with self.assertRaisesRegex(ValueError, err_msg): sd.copy() + def test_format_fallback_capitals(self): + # Fallback to _pydecimal formatting (triggered by `#` format which + # is unsupported by mpdecimal) should honor the current context. + x = C.Decimal('6.09e+23') + self.assertEqual(format(x, '#'), '6.09E+23') + with C.localcontext(capitals=0): + self.assertEqual(format(x, '#'), '6.09e+23') + + def test_format_fallback_rounding(self): + y = C.Decimal('6.09') + self.assertEqual(format(y, '#.1f'), '6.1') + with C.localcontext(rounding=C.ROUND_DOWN): + self.assertEqual(format(y, '#.1f'), '6.0') + @requires_docstrings @requires_cdecimal class SignatureTest(unittest.TestCase): @@ -5818,13 +5916,17 @@ def load_tests(loader, tests, pattern): if TODO_TESTS is None: from doctest import DocTestSuite, IGNORE_EXCEPTION_DETAIL + orig_context = orig_sys_decimal.getcontext().copy() for mod in C, P: if not mod: continue def setUp(slf, mod=mod): sys.modules['decimal'] = mod - def tearDown(slf): + init(mod) + def tearDown(slf, mod=mod): sys.modules['decimal'] = orig_sys_decimal + mod.setcontext(ORIGINAL_CONTEXT[mod].copy()) + orig_sys_decimal.setcontext(orig_context.copy()) optionflags = IGNORE_EXCEPTION_DETAIL if mod is C else 0 sys.modules['decimal'] = mod tests.addTest(DocTestSuite(mod, setUp=setUp, tearDown=tearDown, @@ -5839,11 +5941,12 @@ def setUpModule(): TEST_ALL = ARITH if ARITH is not None else is_resource_enabled('decimal') def tearDownModule(): - if C: C.setcontext(ORIGINAL_CONTEXT[C]) - P.setcontext(ORIGINAL_CONTEXT[P]) + if C: C.setcontext(ORIGINAL_CONTEXT[C].copy()) + P.setcontext(ORIGINAL_CONTEXT[P].copy()) if not C: - warnings.warn('C tests skipped: no module named _decimal.', - UserWarning) + logging.getLogger(__name__).warning( + 'C tests skipped: no module named _decimal.' + ) if not orig_sys_decimal is sys.modules['decimal']: raise TestFailed("Internal error: unbalanced number of changes to " "sys.modules['decimal'].") diff --git a/Lib/test/test_decorators.py b/Lib/test/test_decorators.py index 739c9b3909..4da2c13a60 100644 --- a/Lib/test/test_decorators.py +++ b/Lib/test/test_decorators.py @@ -291,46 +291,7 @@ def bar(): return 42 self.assertEqual(bar(), 42) self.assertEqual(actions, expected_actions) - def test_wrapped_descriptor_inside_classmethod(self): - class BoundWrapper: - def __init__(self, wrapped): - self.__wrapped__ = wrapped - - def __call__(self, *args, **kwargs): - return self.__wrapped__(*args, **kwargs) - - class Wrapper: - def __init__(self, wrapped): - self.__wrapped__ = wrapped - - def __get__(self, instance, owner): - bound_function = self.__wrapped__.__get__(instance, owner) - return BoundWrapper(bound_function) - - def decorator(wrapped): - return Wrapper(wrapped) - - class Class: - @decorator - @classmethod - def inner(cls): - # This should already work. - return 'spam' - - @classmethod - @decorator - def outer(cls): - # Raised TypeError with a message saying that the 'Wrapper' - # object is not callable. - return 'eggs' - - self.assertEqual(Class.inner(), 'spam') - self.assertEqual(Class.outer(), 'eggs') - self.assertEqual(Class().inner(), 'spam') - self.assertEqual(Class().outer(), 'eggs') - - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_bound_function_inside_classmethod(self): class A: def foo(self, cls): @@ -341,91 +302,6 @@ class B: self.assertEqual(B.bar(), 'spam') - def test_wrapped_classmethod_inside_classmethod(self): - class MyClassMethod1: - def __init__(self, func): - self.func = func - - def __call__(self, cls): - if hasattr(self.func, '__get__'): - return self.func.__get__(cls, cls)() - return self.func(cls) - - def __get__(self, instance, owner=None): - if owner is None: - owner = type(instance) - return MethodType(self, owner) - - class MyClassMethod2: - def __init__(self, func): - if isinstance(func, classmethod): - func = func.__func__ - self.func = func - - def __call__(self, cls): - return self.func(cls) - - def __get__(self, instance, owner=None): - if owner is None: - owner = type(instance) - return MethodType(self, owner) - - for myclassmethod in [MyClassMethod1, MyClassMethod2]: - class A: - @myclassmethod - def f1(cls): - return cls - - @classmethod - @myclassmethod - def f2(cls): - return cls - - @myclassmethod - @classmethod - def f3(cls): - return cls - - @classmethod - @classmethod - def f4(cls): - return cls - - @myclassmethod - @MyClassMethod1 - def f5(cls): - return cls - - @myclassmethod - @MyClassMethod2 - def f6(cls): - return cls - - self.assertIs(A.f1(), A) - self.assertIs(A.f2(), A) - self.assertIs(A.f3(), A) - self.assertIs(A.f4(), A) - self.assertIs(A.f5(), A) - self.assertIs(A.f6(), A) - a = A() - self.assertIs(a.f1(), A) - self.assertIs(a.f2(), A) - self.assertIs(a.f3(), A) - self.assertIs(a.f4(), A) - self.assertIs(a.f5(), A) - self.assertIs(a.f6(), A) - - def f(cls): - return cls - - self.assertIs(myclassmethod(f).__get__(a)(), A) - self.assertIs(myclassmethod(f).__get__(a, A)(), A) - self.assertIs(myclassmethod(f).__get__(A, A)(), A) - self.assertIs(myclassmethod(f).__get__(A)(), type(A)) - self.assertIs(classmethod(f).__get__(a)(), A) - self.assertIs(classmethod(f).__get__(a, A)(), A) - self.assertIs(classmethod(f).__get__(A, A)(), A) - self.assertIs(classmethod(f).__get__(A)(), type(A)) class TestClassDecorators(unittest.TestCase): diff --git a/Lib/test/test_deque.py b/Lib/test/test_deque.py index 2b0144eb06..9f00e12edd 100644 --- a/Lib/test/test_deque.py +++ b/Lib/test/test_deque.py @@ -166,7 +166,7 @@ def test_contains(self): with self.assertRaises(RuntimeError): n in d - def test_contains_count_stop_crashes(self): + def test_contains_count_index_stop_crashes(self): class A: def __eq__(self, other): d.clear() @@ -178,6 +178,10 @@ def __eq__(self, other): with self.assertRaises(RuntimeError): _ = d.count(3) + d = deque([A()]) + with self.assertRaises(RuntimeError): + d.index(0) + def test_extend(self): d = deque('a') self.assertRaises(TypeError, d.extend, 1) diff --git a/Lib/test/test_descr.py b/Lib/test/test_descr.py index d1c83cc337..b3d973237d 100644 --- a/Lib/test/test_descr.py +++ b/Lib/test/test_descr.py @@ -15,6 +15,7 @@ from copy import deepcopy from contextlib import redirect_stdout from test import support +from test.support.testcase import ExtraAssertions try: import _testcapi @@ -403,15 +404,7 @@ def test_wrap_lenfunc_bad_cast(self): self.assertEqual(range(sys.maxsize).__len__(), sys.maxsize) -class ClassPropertiesAndMethods(unittest.TestCase): - - def assertHasAttr(self, obj, name): - self.assertTrue(hasattr(obj, name), - '%r has no attribute %r' % (obj, name)) - - def assertNotHasAttr(self, obj, name): - self.assertFalse(hasattr(obj, name), - '%r has unexpected attribute %r' % (obj, name)) +class ClassPropertiesAndMethods(unittest.TestCase, ExtraAssertions): def test_python_dicts(self): # Testing Python subclass of dict... @@ -479,8 +472,6 @@ def __getitem__(self, i): self.assertEqual(a[2], 102) self.assertEqual(a[100:200], (100,200)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_metaclass(self): # Testing metaclasses... class C(metaclass=type): @@ -991,8 +982,8 @@ class EditableScrollablePane(ScrollablePane,EditablePane): pass def test_mro_disagreement(self): # Testing error messages for MRO disagreement... - mro_err_msg = """Cannot create a consistent method resolution -order (MRO) for bases """ + mro_err_msg = ("Cannot create a consistent method resolution " + "order (MRO) for bases ") def raises(exc, expected, callable, *args): try: @@ -1111,8 +1102,7 @@ class MyFrozenSet(frozenset): with self.assertRaises(TypeError): frozenset().__class__ = MyFrozenSet - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_slots(self): # Testing __slots__... class C0(object): @@ -1201,10 +1191,9 @@ class C(object): pass else: self.fail("[''] slots not caught") - class C(object): + + class WithValidIdentifiers(object): __slots__ = ["a", "a_b", "_a", "A0123456789Z"] - # XXX(nnorwitz): was there supposed to be something tested - # from the class above? # Test a single string is not expanded as a sequence. class C(object): @@ -1318,7 +1307,7 @@ class X(object): # Inherit from object on purpose to check some backwards compatibility paths class X(object): __slots__ = "a" - with self.assertRaisesRegex(AttributeError, "'X' object has no attribute 'a'"): + with self.assertRaisesRegex(AttributeError, "'test.test_descr.ClassPropertiesAndMethods.test_slots..X' object has no attribute 'a'"): X().a # Test string subclass in `__slots__`, see gh-98783 @@ -1330,8 +1319,7 @@ class X(object): with self.assertRaisesRegex(AttributeError, "'X' object has no attribute 'a'"): X().a - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_slots_special(self): # Testing __dict__ and __weakref__ in __slots__... class D(object): @@ -1370,8 +1358,7 @@ class C2(D, W): a.foo = 42 self.assertEqual(a.__dict__, {"foo": 42}) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_slots_special2(self): # Testing __qualname__ and __classcell__ in __slots__ class Meta(type): @@ -1412,8 +1399,6 @@ class Q2: __qualname__ = object() __slots__ = ["__qualname__"] - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_slots_descriptor(self): # Issue2115: slot descriptors did not correctly check # the type of the given object @@ -1604,7 +1589,11 @@ def f(cls, arg): cm = classmethod(f) cm_dict = {'__annotations__': {}, - '__doc__': "f docstring", + '__doc__': ( + "f docstring" + if support.HAVE_PY_DOCSTRINGS + else None + ), '__module__': __name__, '__name__': 'f', '__qualname__': f.__qualname__} @@ -1676,8 +1665,6 @@ class SubSpam(spam.spamlist): pass spam_cm.__get__(None, list) self.assertEqual(str(cm.exception), expected_errmsg) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_staticmethods(self): # Testing static methods... class C(object): @@ -1695,10 +1682,10 @@ class D(C): self.assertEqual(d.foo(1), (d, 1)) self.assertEqual(D.foo(d, 1), (d, 1)) sm = staticmethod(None) - self.assertEqual(sm.__dict__, {'__doc__': None}) + self.assertEqual(sm.__dict__, {'__doc__': None.__doc__}) sm.x = 42 self.assertEqual(sm.x, 42) - self.assertEqual(sm.__dict__, {"x" : 42, '__doc__': None}) + self.assertEqual(sm.__dict__, {"x" : 42, '__doc__': None.__doc__}) del sm.x self.assertNotHasAttr(sm, "x") @@ -1800,7 +1787,6 @@ class D(C): self.assertEqual(b.foo, 3) self.assertEqual(b.__class__, D) - @unittest.expectedFailure def test_bad_new(self): self.assertRaises(TypeError, object.__new__) self.assertRaises(TypeError, object.__new__, '') @@ -1813,8 +1799,7 @@ class C(list): __new__ = object.__new__ self.assertRaises(TypeError, C) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_object_new(self): class A(object): pass @@ -1849,6 +1834,8 @@ def __init__(self, foo): object.__init__(A(3)) self.assertRaises(TypeError, object.__init__, A(3), 5) + @unittest.skip('TODO: RUSTPYTHON; This passes, but the `expectedFailure` here is from CPython, so this test is an "UNEXPECTED SUCCESS" (not good)') + @unittest.expectedFailure def test_restored_object_new(self): class A(object): def __new__(cls, *args, **kwargs): @@ -1872,8 +1859,7 @@ def __init__(self, foo): self.assertEqual(b.foo, 3) self.assertEqual(b.__class__, B) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_altmro(self): # Testing mro() and overriding it... class A(object): @@ -2052,8 +2038,7 @@ def test_methods_in_c(self): set_add.__get__(0) self.assertEqual(cm.exception.args[0], expected_errmsg) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_special_method_lookup(self): # The lookup of special methods bypasses __getattr__ and # __getattribute__, but they still can be descriptors. @@ -2256,8 +2241,7 @@ def __contains__(self, value): self.assertIn(i, p10) self.assertNotIn(10, p10) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_weakrefs(self): # Testing weak references... import weakref @@ -2289,8 +2273,7 @@ class Weak(object): self.assertEqual(r(), None) del r - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_properties(self): # Testing property... class C(object): @@ -2618,8 +2601,7 @@ def __getclass(self): dir(C()) # This used to segfault - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_supers(self): # Testing super... @@ -2732,8 +2714,7 @@ def test(klass): with self.assertRaises(TypeError): super(Base, kw=1) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_basic_inheritance(self): # Testing inheritance from basic types... @@ -3066,8 +3047,7 @@ class sublist(list): ## pass ## os_helper.unlink(os_helper.TESTFN) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_keywords(self): # Testing keyword args to basic type constructors ... with self.assertRaisesRegex(TypeError, 'keyword argument'): @@ -3270,8 +3250,7 @@ def __ge__(self, other): eval("x %s y" % op), "x=%d, y=%d" % (x, y)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_descrdoc(self): # Testing descriptor doc strings... from _io import FileIO @@ -3295,8 +3274,7 @@ class NewClass: self.assertEqual(NewClass.__doc__, 'object=None; type=NewClass') self.assertEqual(NewClass().__doc__, 'object=NewClass instance; type=NewClass') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_set_class(self): # Testing __class__ assignment... class C(object): pass @@ -3386,8 +3364,7 @@ def __del__(self): l = [A() for x in range(100)] del l - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_set_dict(self): # Testing __dict__ assignment... class C(object): pass @@ -3646,7 +3623,7 @@ def f(a): return a encoding='latin1', errors='replace') self.assertEqual(ba, b'abc\xbd?') - @unittest.skip("TODO: RUSTPYTHON, rustpython segmentation fault") + @unittest.skip('TODO: RUSTPYTHON; rustpython segmentation fault') def test_recursive_call(self): # Testing recursive __call__() by setting to instance of class... class A(object): @@ -3765,8 +3742,7 @@ def test_uninitialized_modules(self): m.foo = 1 self.assertEqual(m.__dict__, {"foo": 1}) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_funny_new(self): # Testing __new__ returning something unexpected... class C(object): @@ -3928,7 +3904,7 @@ def __del__(self): # it as a leak. del C.__del__ - @unittest.skip("TODO: RUSTPYTHON, rustpython segmentation fault") + @unittest.skip('TODO: RUSTPYTHON; rustpython segmentation fault') def test_slots_trash(self): # Testing slot trash... # Deallocating deeply nested slotted trash caused stack overflows @@ -3941,8 +3917,7 @@ def __init__(self, x): o = trash(o) del o - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_slots_multiple_inheritance(self): # SF bug 575229, multiple inheritance w/ slots dumps core class A(object): @@ -4087,8 +4062,7 @@ class E(D): else: self.fail("shouldn't be able to create inheritance cycles") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_builtin_bases(self): # Make sure all the builtin types can have their base queried without # segfaulting. See issue #5787. @@ -4133,8 +4107,7 @@ class D(C): else: self.fail("best_base calculation found wanting") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_unsubclassable_types(self): with self.assertRaises(TypeError): class X(type(None)): @@ -4167,8 +4140,7 @@ class X(object): with self.assertRaises(TypeError): X.__bases__ = type(None), O - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_mutable_bases_with_failing_mro(self): # Testing mutable bases with failing mro... class WorkOnce(type): @@ -4261,7 +4233,6 @@ class C(object): C.__name__ = 'D.E' self.assertEqual((C.__module__, C.__name__), (mod, 'D.E')) - @unittest.skip("TODO: RUSTPYTHON, rustpython hang") def test_evil_type_name(self): # A badly placed Py_DECREF in type_set_name led to arbitrary code # execution while the type structure was not in a sane state, and a @@ -4276,8 +4247,7 @@ class C: C.__name__ = Nasty("abc") C.__name__ = "normal" - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_subclass_right_op(self): # Testing correct dispatch of subclass overloading __r__... @@ -4412,8 +4382,7 @@ class D(C): self.assertIsInstance(a, C) # Baseline self.assertIsInstance(pa, C) # Test - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_proxy_super(self): # Testing super() for a proxy object... class Proxy(object): @@ -4437,8 +4406,7 @@ def f(self): p = Proxy(obj) self.assertEqual(C.__dict__["f"](p), "B.f->C.f") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_carloverre(self): # Testing prohibition of Carlo Verre's hack... try: @@ -4471,8 +4439,7 @@ class C(B, A): except TypeError: self.fail("setattr through direct base types should be legal") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_carloverre_multi_inherit_invalid(self): class A(type): def __setattr__(cls, key, value): @@ -4510,8 +4477,8 @@ class Oops(object): o = Oops() o.whatever = Provoker(o) del o - - @unittest.skip("TODO: RUSTPYTHON, rustpython segmentation fault") + + @unittest.skip('TODO: RUSTPYTHON; rustpython segmentation fault') @support.requires_resource('cpu') def test_wrapper_segfault(self): # SF 927248: deeply nested wrappers could cause stack overflow @@ -4586,8 +4553,7 @@ def assertNotOrderable(self, a, b): with self.assertRaises(TypeError): a >= b - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_method_wrapper(self): # Testing method-wrapper objects... # did not support any reflection before 2.5 @@ -4647,18 +4613,16 @@ def test_special_unbound_method_types(self): def test_not_implemented(self): # Testing NotImplemented... # all binary methods should be able to return a NotImplemented - import operator def specialmethod(self, other): return NotImplemented def check(expr, x, y): - try: - exec(expr, {'x': x, 'y': y, 'operator': operator}) - except TypeError: - pass - else: - self.fail("no TypeError from %r" % (expr,)) + with ( + self.subTest(expr=expr, x=x, y=y), + self.assertRaises(TypeError), + ): + exec(expr, {'x': x, 'y': y}) N1 = sys.maxsize + 1 # might trigger OverflowErrors instead of # TypeErrors @@ -4679,12 +4643,23 @@ def check(expr, x, y): ('__and__', 'x & y', 'x &= y'), ('__or__', 'x | y', 'x |= y'), ('__xor__', 'x ^ y', 'x ^= y')]: - rname = '__r' + name[2:] + # Defines 'left' magic method: A = type('A', (), {name: specialmethod}) a = A() check(expr, a, a) check(expr, a, N1) check(expr, a, N2) + # Defines 'right' magic method: + rname = '__r' + name[2:] + B = type('B', (), {rname: specialmethod}) + b = B() + check(expr, b, b) + check(expr, a, b) + check(expr, b, a) + check(expr, b, N1) + check(expr, b, N2) + check(expr, N1, b) + check(expr, N2, b) if iexpr: check(iexpr, a, a) check(iexpr, a, N1) @@ -4791,6 +4766,21 @@ class X(object): with self.assertRaises(AttributeError): del X.__abstractmethods__ + @unittest.skip('TODO: RUSTPYTHON; crash. "dict has non-string keys: [PyObject PyInt { value: 1 }]"') + def test_gh55664(self): + # gh-55664: issue a warning when the + # __dict__ of a class contains non-string keys + with self.assertWarnsRegex(RuntimeWarning, 'MyClass'): + MyClass = type('MyClass', (), {1: 2}) + + class meta(type): + def __new__(mcls, name, bases, ns): + ns[1] = 2 + return super().__new__(mcls, name, bases, ns) + + with self.assertWarnsRegex(RuntimeWarning, 'MyClass'): + MyClass = meta('MyClass', (), {}) + def test_proxy_call(self): class FakeStr: __class__ = str @@ -4814,24 +4804,24 @@ class Thing: thing = Thing() for i in range(20): with self.assertRaises(TypeError): - # PRECALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS + # CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS list.sort(thing) for i in range(20): with self.assertRaises(TypeError): - # PRECALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS + # CALL_METHOD_DESCRIPTOR_FAST_WITH_KEYWORDS str.split(thing) for i in range(20): with self.assertRaises(TypeError): - # PRECALL_NO_KW_METHOD_DESCRIPTOR_NOARGS + # CALL_METHOD_DESCRIPTOR_NOARGS str.upper(thing) for i in range(20): with self.assertRaises(TypeError): - # PRECALL_NO_KW_METHOD_DESCRIPTOR_FAST + # CALL_METHOD_DESCRIPTOR_FAST str.strip(thing) from collections import deque for i in range(20): with self.assertRaises(TypeError): - # PRECALL_NO_KW_METHOD_DESCRIPTOR_O + # CALL_METHOD_DESCRIPTOR_O deque.append(thing, thing) def test_repr_as_str(self): @@ -4865,8 +4855,7 @@ class A(int): with self.assertRaises(TypeError): a + a - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_slot_shadows_class_variable(self): with self.assertRaises(ValueError) as cm: class X: @@ -4875,8 +4864,7 @@ class X: m = str(cm.exception) self.assertEqual("'foo' in __slots__ conflicts with class variable", m) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_set_doc(self): class X: "elephant" @@ -4892,8 +4880,7 @@ class X: self.assertIn("cannot delete '__doc__' attribute of immutable type 'X'", str(cm.exception)) self.assertEqual(X.__doc__, "banana") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_qualname(self): descriptors = [str.lower, complex.real, float.real, int.__add__] types = ['method', 'member', 'getset', 'wrapper'] @@ -4926,8 +4913,7 @@ class Inside: self.assertEqual(Y.__qualname__, 'Y') self.assertEqual(Y.Inside.__qualname__, 'Y.Inside') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_qualname_dict(self): ns = {'__qualname__': 'some.name'} tp = type('Foo', (), ns) @@ -4938,8 +4924,7 @@ def test_qualname_dict(self): ns = {'__qualname__': 1} self.assertRaises(TypeError, type, 'Foo', (), ns) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_cycle_through_dict(self): # See bug #1469629 class X(dict): @@ -4955,8 +4940,6 @@ def __init__(self): for o in gc.get_objects(): self.assertIsNot(type(o), X) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_object_new_and_init_with_parameters(self): # See issue #1683368 class OverrideNeither: @@ -4977,8 +4960,6 @@ class OverrideBoth(OverrideNew, OverrideInit): self.assertRaises(TypeError, case, 1, 2, 3) self.assertRaises(TypeError, case, 1, 2, foo=3) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_subclassing_does_not_duplicate_dict_descriptors(self): class Base: pass @@ -5058,6 +5039,7 @@ def __new__(cls): cls.lst = [2**i for i in range(10000)] X.descr + @support.suppress_immortalization() def test_remove_subclass(self): # bpo-46417: when the last subclass of a type is deleted, # remove_subclass() clears the internal dictionary of subclasses: @@ -5075,8 +5057,22 @@ class Child(Parent): gc.collect() self.assertEqual(Parent.__subclasses__(), []) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_instance_method_get_behavior(self): + # test case for gh-113157 + + class A: + def meth(self): + return self + + class B: + pass + + a = A() + b = B() + b.meth = a.meth.__get__(b, B) + self.assertEqual(b.meth(), a) + def test_attr_raise_through_property(self): # test case for gh-103272 class A: @@ -5111,8 +5107,7 @@ def meth(self): pass self.C = C - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), 'trace function introduces __local__') def test_iter_keys(self): @@ -5121,11 +5116,12 @@ def test_iter_keys(self): self.assertNotIsInstance(it, list) keys = list(it) keys.sort() - self.assertEqual(keys, ['__dict__', '__doc__', '__module__', - '__weakref__', 'meth']) + self.assertEqual(keys, ['__dict__', '__doc__', '__firstlineno__', + '__module__', + '__static_attributes__', '__weakref__', + 'meth']) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 5 != 7 @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), 'trace function introduces __local__') def test_iter_values(self): @@ -5133,10 +5129,9 @@ def test_iter_values(self): it = self.C.__dict__.values() self.assertNotIsInstance(it, list) values = list(it) - self.assertEqual(len(values), 5) + self.assertEqual(len(values), 7) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), 'trace function introduces __local__') def test_iter_items(self): @@ -5145,8 +5140,10 @@ def test_iter_items(self): self.assertNotIsInstance(it, list) keys = [item[0] for item in it] keys.sort() - self.assertEqual(keys, ['__dict__', '__doc__', '__module__', - '__weakref__', 'meth']) + self.assertEqual(keys, ['__dict__', '__doc__', '__firstlineno__', + '__module__', + '__static_attributes__', '__weakref__', + 'meth']) def test_dict_type_with_metaclass(self): # Testing type of __dict__ when metaclass set... @@ -5196,7 +5193,7 @@ def __pow__(self, *args): class MiscTests(unittest.TestCase): - @unittest.skip("TODO: RUSTPYTHON, rustpython panicked at 'dict has non-string keys: [PyObject PyBaseObject]'") + @unittest.skip("TODO: RUSTPYTHON; rustpython panicked at 'dict has non-string keys: [PyObject PyBaseObject]'") def test_type_lookup_mro_reference(self): # Issue #14199: _PyType_Lookup() has to keep a strong reference to # the type MRO because it may be modified during the lookup, if @@ -5216,11 +5213,17 @@ class Base2(object): mykey = 'from Base2' mykey2 = 'from Base2' - X = type('X', (Base,), {MyKey(): 5}) + with self.assertWarnsRegex(RuntimeWarning, 'X'): + X = type('X', (Base,), {MyKey(): 5}) + + # Note that the access below uses getattr() rather than normally + # accessing the attribute. That is done to avoid the bytecode + # specializer activating on repeated runs of the test. + # mykey is read from Base - self.assertEqual(X.mykey, 'from Base') + self.assertEqual(getattr(X, 'mykey'), 'from Base') # mykey2 is read from Base2 because MyKey.__eq__ has set __bases__ - self.assertEqual(X.mykey2, 'from Base2') + self.assertEqual(getattr(X, 'mykey2'), 'from Base2') class PicklingTests(unittest.TestCase): @@ -5255,8 +5258,7 @@ def _check_reduce(self, proto, obj, args=(), kwargs={}, state=None, self.assertEqual(obj.__reduce_ex__(proto), reduce_value) self.assertEqual(obj.__reduce__(), reduce_value) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_reduce(self): protocols = range(pickle.HIGHEST_PROTOCOL + 1) args = (-101, "spam") @@ -5380,8 +5382,7 @@ class C16(list): for proto in protocols: self._check_reduce(proto, obj, listitems=list(obj)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_special_method_lookup(self): protocols = range(pickle.HIGHEST_PROTOCOL + 1) class Picky: @@ -5514,8 +5515,7 @@ class E(C): y = pickle_copier.copy(x) self._assert_is_copy(x, y) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_reduce_copying(self): # Tests pickling and copying new-style classes and objects. global C1 @@ -5640,7 +5640,7 @@ def __repr__(self): objcopy2 = deepcopy(objcopy) self._assert_is_copy(obj, objcopy2) - @unittest.skip("TODO: RUSTPYTHON") + @unittest.skip('TODO: RUSTPYTHON') def test_issue24097(self): # Slot name is freed inside __getattr__ and is later used. class S(str): # Not interned @@ -5781,8 +5781,7 @@ class B(A): class C(B): pass - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_reent_set_bases_tp_base_cycle(self): """ type_set_bases must check for an inheritance cycle not only through @@ -5819,8 +5818,7 @@ class B2(A): with self.assertRaises(TypeError): B1.__bases__ += () - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_tp_subclasses_cycle_in_update_slots(self): """ type_set_bases must check for reentrancy upon finishing its job @@ -5857,8 +5855,7 @@ class C(A): self.assertEqual(B1.__bases__, (C,)) self.assertEqual(C.__subclasses__(), [B1]) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_tp_subclasses_cycle_error_return_path(self): """ The same as test_tp_subclasses_cycle_in_update_slots, but tests @@ -5927,8 +5924,7 @@ def mro(cls): class A(metaclass=M): pass - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_disappearing_custom_mro(self): """ gh-92112: A custom mro() returning a result conflicting with diff --git a/Lib/test/test_dict.py b/Lib/test/test_dict.py index 4aa6f1089a..9598a7ab96 100644 --- a/Lib/test/test_dict.py +++ b/Lib/test/test_dict.py @@ -8,7 +8,7 @@ import unittest import weakref from test import support -from test.support import import_helper, C_RECURSION_LIMIT +from test.support import import_helper, get_c_recursion_limit class DictTest(unittest.TestCase): @@ -312,17 +312,34 @@ def __setitem__(self, key, value): self.assertRaises(Exc, baddict2.fromkeys, [1]) # test fast path for dictionary inputs + res = dict(zip(range(6), [0]*6)) d = dict(zip(range(6), range(6))) - self.assertEqual(dict.fromkeys(d, 0), dict(zip(range(6), [0]*6))) - + self.assertEqual(dict.fromkeys(d, 0), res) + # test fast path for set inputs + d = set(range(6)) + self.assertEqual(dict.fromkeys(d, 0), res) + # test slow path for other iterable inputs + d = list(range(6)) + self.assertEqual(dict.fromkeys(d, 0), res) + + # test fast path when object's constructor returns large non-empty dict class baddict3(dict): def __new__(cls): return d - d = {i : i for i in range(10)} + d = {i : i for i in range(1000)} res = d.copy() res.update(a=None, b=None, c=None) self.assertEqual(baddict3.fromkeys({"a", "b", "c"}), res) + # test slow path when object is a proper subclass of dict + class baddict4(dict): + def __init__(self): + dict.__init__(self, d) + d = {i : i for i in range(1000)} + res = d.copy() + res.update(a=None, b=None, c=None) + self.assertEqual(baddict4.fromkeys({"a", "b", "c"}), res) + def test_copy(self): d = {1: 1, 2: 2, 3: 3} self.assertIsNot(d.copy(), d) @@ -596,10 +613,9 @@ def __repr__(self): d = {1: BadRepr()} self.assertRaises(Exc, repr, d) - @unittest.skipIf(sys.platform == 'win32', 'TODO: RUSTPYTHON Windows') def test_repr_deep(self): d = {} - for i in range(C_RECURSION_LIMIT + 1): + for i in range(get_c_recursion_limit() + 1): d = {1: d} self.assertRaises(RecursionError, repr, d) @@ -994,6 +1010,18 @@ class MyDict(dict): pass self._tracked(MyDict()) + @support.cpython_only + def test_track_lazy_instance_dicts(self): + class C: + pass + o = C() + d = o.__dict__ + self._not_tracked(d) + o.untracked = 42 + self._not_tracked(d) + o.tracked = [] + self._tracked(d) + def make_shared_key_dict(self, n): class C: pass @@ -1108,10 +1136,8 @@ class C: a = C() a.x = 1 d = a.__dict__ - before_resize = sys.getsizeof(d) d[2] = 2 # split table is resized to a generic combined table - self.assertGreater(sys.getsizeof(d), before_resize) self.assertEqual(list(d), ['x', 2]) def test_iterator_pickling(self): @@ -1485,6 +1511,24 @@ def test_dict_items_result_gc_reversed(self): gc.collect() self.assertTrue(gc.is_tracked(next(it))) + def test_store_evilattr(self): + class EvilAttr: + def __init__(self, d): + self.d = d + + def __del__(self): + if 'attr' in self.d: + del self.d['attr'] + gc.collect() + + class Obj: + pass + + obj = Obj() + obj.__dict__ = {} + for _ in range(10): + obj.attr = EvilAttr(obj.__dict__) + def test_str_nonstr(self): # cpython uses a different lookup function if the dict only contains # `str` keys. Make sure the unoptimized path is used when a non-`str` @@ -1591,8 +1635,8 @@ class CAPITest(unittest.TestCase): # Test _PyDict_GetItem_KnownHash() @support.cpython_only def test_getitem_knownhash(self): - _testcapi = import_helper.import_module('_testcapi') - dict_getitem_knownhash = _testcapi.dict_getitem_knownhash + _testinternalcapi = import_helper.import_module('_testinternalcapi') + dict_getitem_knownhash = _testinternalcapi.dict_getitem_knownhash d = {'x': 1, 'y': 2, 'z': 3} self.assertEqual(dict_getitem_knownhash(d, 'x', hash('x')), 1) diff --git a/Lib/test/test_dictviews.py b/Lib/test/test_dictviews.py index 172b98aa68..667cccd6cd 100644 --- a/Lib/test/test_dictviews.py +++ b/Lib/test/test_dictviews.py @@ -1,9 +1,8 @@ import collections.abc import copy import pickle -import sys import unittest -from test.support import C_RECURSION_LIMIT +from test.support import get_c_recursion_limit class DictSetTest(unittest.TestCase): @@ -282,7 +281,7 @@ def test_recursive_repr(self): @unittest.expectedFailure def test_deeply_nested_repr(self): d = {} - for i in range(C_RECURSION_LIMIT//2 + 100): + for i in range(get_c_recursion_limit()//2 + 100): d = {42: d.values()} self.assertRaises(RecursionError, repr, d) diff --git a/Lib/test/test_difflib.py b/Lib/test/test_difflib.py index 0d669afe61..943d7a659b 100644 --- a/Lib/test/test_difflib.py +++ b/Lib/test/test_difflib.py @@ -29,6 +29,16 @@ def test_one_delete(self): ('delete', 40, 41, 40, 40), ('equal', 41, 81, 40, 80)]) + def test_opcode_caching(self): + sm = difflib.SequenceMatcher(None, 'b' * 100, 'a' + 'b' * 100) + opcode = sm.get_opcodes() + self.assertEqual(opcode, + [ ('insert', 0, 0, 0, 1), + ('equal', 0, 100, 1, 101)]) + # Implementation detail: opcodes are cached; + # `get_opcodes()` returns the same object + self.assertIs(opcode, sm.get_opcodes()) + def test_bjunk(self): sm = difflib.SequenceMatcher(isjunk=lambda x: x == ' ', a='a' * 40 + 'b' * 40, b='a' * 44 + 'b' * 40) @@ -186,6 +196,7 @@ def test_mdiff_catch_stop_iteration(self): the end""" class TestSFpatches(unittest.TestCase): + def test_html_diff(self): # Check SF patch 914575 for generating HTML differences f1a = ((patch914575_from1 + '123\n'*10)*3) @@ -272,6 +283,15 @@ def test_make_file_usascii_charset_with_nonascii_input(self): self.assertIn('ımplıcıt', output) + def test_one_insert(self): + m = difflib.Differ().compare('b' * 2, 'a' + 'b' * 2) + self.assertEqual(list(m), ['+ a', ' b', ' b']) + + def test_one_delete(self): + m = difflib.Differ().compare('a' + 'b' * 2, 'b' * 2) + self.assertEqual(list(m), ['- a', ' b', ' b']) + + class TestOutputFormat(unittest.TestCase): def test_tab_delimiter(self): args = ['one', 'two', 'Original', 'Current', @@ -546,6 +566,26 @@ def test_longest_match_with_popular_chars(self): self.assertFalse(self.longer_match_exists(a, b, match.size)) +class TestCloseMatches(unittest.TestCase): + # Happy paths are tested in the doctests of `difflib.get_close_matches`. + + def test_invalid_inputs(self): + self.assertRaises(ValueError, difflib.get_close_matches, "spam", ['egg'], n=0) + self.assertRaises(ValueError, difflib.get_close_matches, "spam", ['egg'], n=-1) + self.assertRaises(ValueError, difflib.get_close_matches, "spam", ['egg'], cutoff=1.1) + self.assertRaises(ValueError, difflib.get_close_matches, "spam", ['egg'], cutoff=-0.1) + + +class TestRestore(unittest.TestCase): + # Happy paths are tested in the doctests of `difflib.restore`. + + def test_invalid_input(self): + with self.assertRaises(ValueError): + ''.join(difflib.restore([], 0)) + with self.assertRaises(ValueError): + ''.join(difflib.restore([], 3)) + + def setUpModule(): difflib.HtmlDiff._default_prefix = 0 diff --git a/Lib/test/test_difflib_expect.html b/Lib/test/test_difflib_expect.html index 3e6a7b7a99..12091206a2 100644 --- a/Lib/test/test_difflib_expect.html +++ b/Lib/test/test_difflib_expect.html @@ -9,7 +9,7 @@ content="text/html; charset=utf-8" /> ' + self._run_check(s, [("starttag", "style", []), + ("data", content), + ("endtag", "style")]) + + @support.subTests('content', [ + '', + "", + '', + '', + '', + '\u2603', + '< /title>', + '', + '', + '', + '', + '', + ]) + def test_title_content(self, content): + source = f"{content}" + self._run_check(source, [ + ("starttag", "title", []), + ("data", content), + ("endtag", "title"), + ]) + + @support.subTests('content', [ + '', + "", + '', + '', + '', + '\u2603', + '< /textarea>', + '', + '', + '', + '', + ]) + def test_textarea_content(self, content): + source = f"" + self._run_check(source, [ + ("starttag", "textarea", []), + ("data", content), + ("endtag", "textarea"), + ]) + + @support.subTests('endtag', ['script', 'SCRIPT', 'script ', 'script\n', + 'script/', 'script foo=bar', 'script foo=">"']) + def test_script_closing_tag(self, endtag): # see issue #13358 # make sure that HTMLParser calls handle_data only once for each CDATA. - # The normal event collector normalizes the events in get_events, - # so we override it to return the original list of events. - class Collector(EventCollector): - def get_events(self): - return self.events - content = """ ¬-an-entity-ref;

''""" - for element in [' script', 'script ', ' script ', - '\nscript', 'script\n', '\nscript\n']: - element_lower = element.lower().strip() - s = '{content}{tail}' + self._run_check(s, [("starttag", "script", []), + ("data", content if end else content + tail)], + collector=EventCollectorNoNormalize(convert_charrefs=False)) + + @support.subTests('tail,end', [ + ('', False), + ('<', False), + ('', True), + ]) + def test_eof_in_title(self, tail, end): + s = f'Egg & Spam{tail}' + self._run_check(s, [("starttag", "title", []), + ("data", "Egg & Spam" + ('' if end else tail))], + collector=EventCollectorNoNormalize(convert_charrefs=True)) + self._run_check(s, [("starttag", "title", []), + ('data', 'Egg '), + ('entityref', 'amp'), + ('data', ' Spam' + ('' if end else tail))], + collector=EventCollectorNoNormalize(convert_charrefs=False)) def test_comments(self): html = ("<!-- I'm a valid comment -->" '<!--me too!-->' '<!------>' + '<!----->' '<!---->' + # abrupt-closing-of-empty-comment + '<!--->' + '<!-->' '<!----I have many hyphens---->' '<!-- I have a > in the middle -->' - '<!-- and I have -- in the middle! -->') + '<!-- and I have -- in the middle! -->' + '<!--incorrectly-closed-comment--!>' + '<!----!>' + '<!----!-->' + '<!---- >-->' + '<!---!>-->' + '<!--!>-->' + # nested-comment + '<!-- <!-- nested --> -->' + '<!--<!-->' + '<!--<!--!>' + ) expected = [('comment', " I'm a valid comment "), ('comment', 'me too!'), ('comment', '--'), + ('comment', '-'), + ('comment', ''), + ('comment', ''), ('comment', ''), ('comment', '--I have many hyphens--'), ('comment', ' I have a > in the middle '), - ('comment', ' and I have -- in the middle! ')] + ('comment', ' and I have -- in the middle! '), + ('comment', 'incorrectly-closed-comment'), + ('comment', ''), + ('comment', '--!'), + ('comment', '-- >'), + ('comment', '-!>'), + ('comment', '!>'), + ('comment', ' <!-- nested '), ('data', ' -->'), + ('comment', '<!'), + ('comment', '<!'), + ] self._run_check(html, expected) def test_condcoms(self): @@ -346,18 +525,16 @@ def test_convert_charrefs(self): collector = lambda: EventCollectorCharrefs() self.assertTrue(collector().convert_charrefs) charrefs = ['"', '"', '"', '"', '"', '"'] - # check charrefs in the middle of the text/attributes - expected = [('starttag', 'a', [('href', 'foo"zar')]), - ('data', 'a"z'), ('endtag', 'a')] + # check charrefs in the middle of the text + expected = [('starttag', 'a', []), ('data', 'a"z'), ('endtag', 'a')] for charref in charrefs: - self._run_check('<a href="foo{0}zar">a{0}z</a>'.format(charref), + self._run_check('<a>a{0}z</a>'.format(charref), expected, collector=collector()) - # check charrefs at the beginning/end of the text/attributes - expected = [('data', '"'), - ('starttag', 'a', [('x', '"'), ('y', '"X'), ('z', 'X"')]), + # check charrefs at the beginning/end of the text + expected = [('data', '"'), ('starttag', 'a', []), ('data', '"'), ('endtag', 'a'), ('data', '"')] for charref in charrefs: - self._run_check('{0}<a x="{0}" y="{0}X" z="X{0}">' + self._run_check('{0}<a>' '{0}</a>{0}'.format(charref), expected, collector=collector()) # check charrefs in <script>/<style> elements @@ -380,6 +557,35 @@ def test_convert_charrefs(self): self._run_check('no charrefs here', [('data', 'no charrefs here')], collector=collector()) + def test_convert_charrefs_in_attribute_values(self): + # default value for convert_charrefs is now True + collector = lambda: EventCollectorCharrefs() + self.assertTrue(collector().convert_charrefs) + + # always unescape terminated entity refs, numeric and hex char refs: + # - regardless whether they are at start, middle, end of attribute + # - or followed by alphanumeric, non-alphanumeric, or equals char + charrefs = ['¢', '¢', '¢', '¢', '¢'] + expected = [('starttag', 'a', + [('x', '¢'), ('x', 'z¢'), ('x', '¢z'), + ('x', 'z¢z'), ('x', '¢ z'), ('x', '¢=z')]), + ('endtag', 'a')] + for charref in charrefs: + self._run_check('<a x="{0}" x="z{0}" x="{0}z" ' + ' x="z{0}z" x="{0} z" x="{0}=z"></a>' + .format(charref), expected, collector=collector()) + + # only unescape unterminated entity matches if they are not followed by + # an alphanumeric or an equals sign + charref = '¢' + expected = [('starttag', 'a', + [('x', '¢'), ('x', 'z¢'), ('x', '¢z'), + ('x', 'z¢z'), ('x', '¢ z'), ('x', '¢=z')]), + ('endtag', 'a')] + self._run_check('<a x="{0}" x="z{0}" x="{0}z" ' + ' x="z{0}z" x="{0} z" x="{0}=z"></a>' + .format(charref), expected, collector=collector()) + # the remaining tests were for the "tolerant" parser (which is now # the default), and check various kind of broken markup def test_tolerant_parsing(self): @@ -391,28 +597,34 @@ def test_tolerant_parsing(self): ('data', '<'), ('starttag', 'bc<', [('a', None)]), ('endtag', 'html'), - ('data', '\n<img src="URL>'), - ('comment', '/img'), - ('endtag', 'html<')]) + ('data', '\n')]) def test_starttag_junk_chars(self): + self._run_check("<", [('data', '<')]) + self._run_check("<>", [('data', '<>')]) + self._run_check("< >", [('data', '< >')]) + self._run_check("< ", [('data', '< ')]) self._run_check("</>", []) + self._run_check("<$>", [('data', '<$>')]) self._run_check("</$>", [('comment', '$')]) self._run_check("</", [('data', '</')]) - self._run_check("</a", [('data', '</a')]) + self._run_check("</a", []) + self._run_check("</ a>", [('comment', ' a')]) + self._run_check("</ a", [('comment', ' a')]) self._run_check("<a<a>", [('starttag', 'a<a', [])]) self._run_check("</a<a>", [('endtag', 'a<a')]) - self._run_check("<!", [('data', '<!')]) - self._run_check("<a", [('data', '<a')]) - self._run_check("<a foo='bar'", [('data', "<a foo='bar'")]) - self._run_check("<a foo='bar", [('data', "<a foo='bar")]) - self._run_check("<a foo='>'", [('data', "<a foo='>'")]) - self._run_check("<a foo='>", [('data', "<a foo='>")]) + self._run_check("<!", [('comment', '')]) + self._run_check("<a", []) + self._run_check("<a foo='bar'", []) + self._run_check("<a foo='bar", []) + self._run_check("<a foo='>'", []) + self._run_check("<a foo='>", []) self._run_check("<a$>", [('starttag', 'a$', [])]) self._run_check("<a$b>", [('starttag', 'a$b', [])]) self._run_check("<a$b/>", [('startendtag', 'a$b', [])]) self._run_check("<a$b >", [('starttag', 'a$b', [])]) self._run_check("<a$b />", [('startendtag', 'a$b', [])]) + self._run_check("</a$b>", [('endtag', 'a$b')]) def test_slashes_in_starttag(self): self._run_check('<a foo="var"/>', [('startendtag', 'a', [('foo', 'var')])]) @@ -445,6 +657,10 @@ def test_slashes_in_starttag(self): ] self._run_check(html, expected) + def test_slashes_in_endtag(self): + self._run_check('</a/>', [('endtag', 'a')]) + self._run_check('</a foo="var"/>', [('endtag', 'a')]) + def test_declaration_junk_chars(self): self._run_check("<!DOCTYPE foo $ >", [('decl', 'DOCTYPE foo $ ')]) @@ -479,15 +695,11 @@ def test_invalid_end_tags(self): self._run_check(html, expected) def test_broken_invalid_end_tag(self): - # This is technically wrong (the "> shouldn't be included in the 'data') - # but is probably not worth fixing it (in addition to all the cases of - # the previous test, it would require a full attribute parsing). - # see #13993 html = '<b>This</b attr=">"> confuses the parser' expected = [('starttag', 'b', []), ('data', 'This'), ('endtag', 'b'), - ('data', '"> confuses the parser')] + ('data', ' confuses the parser')] self._run_check(html, expected) def test_correct_detection_of_start_tags(self): @@ -537,55 +749,165 @@ def test_EOF_in_charref(self): for html, expected in data: self._run_check(html, expected) - def test_broken_comments(self): - html = ('<! not really a comment >' + def test_eof_in_comments(self): + data = [ + ('<!--', [('comment', '')]), + ('<!---', [('comment', '')]), + ('<!----', [('comment', '')]), + ('<!-----', [('comment', '-')]), + ('<!------', [('comment', '--')]), + ('<!----!', [('comment', '')]), + ('<!---!', [('comment', '-!')]), + ('<!---!>', [('comment', '-!>')]), + ('<!--foo', [('comment', 'foo')]), + ('<!--foo-', [('comment', 'foo')]), + ('<!--foo--', [('comment', 'foo')]), + ('<!--foo--!', [('comment', 'foo')]), + ('<!--<!--', [('comment', '<!')]), + ('<!--<!--!', [('comment', '<!')]), + ] + for html, expected in data: + self._run_check(html, expected) + + def test_eof_in_declarations(self): + data = [ + ('<!', [('comment', '')]), + ('<!-', [('comment', '-')]), + ('<![', [('comment', '[')]), + ('<!DOCTYPE', [('decl', 'DOCTYPE')]), + ('<!DOCTYPE ', [('decl', 'DOCTYPE ')]), + ('<!DOCTYPE html', [('decl', 'DOCTYPE html')]), + ('<!DOCTYPE html ', [('decl', 'DOCTYPE html ')]), + ('<!DOCTYPE html PUBLIC', [('decl', 'DOCTYPE html PUBLIC')]), + ('<!DOCTYPE html PUBLIC "foo', [('decl', 'DOCTYPE html PUBLIC "foo')]), + ('<!DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01//EN" "foo', + [('decl', 'DOCTYPE html PUBLIC "-//W3C//DTD HTML 4.01//EN" "foo')]), + ] + for html, expected in data: + self._run_check(html, expected) + + @support.subTests('content', ['', 'x', 'x]', 'x]]']) + def test_eof_in_cdata(self, content): + self._run_check('<![CDATA[' + content, + [('unknown decl', 'CDATA[' + content)]) + self._run_check('<![CDATA[' + content, + [('comment', '[CDATA[' + content)], + collector=EventCollector(autocdata=True)) + self._run_check('<svg><text y="100"><![CDATA[' + content, + [('starttag', 'svg', []), + ('starttag', 'text', [('y', '100')]), + ('unknown decl', 'CDATA[' + content)]) + + def test_bogus_comments(self): + html = ('<!ELEMENT br EMPTY>' + '<! not really a comment >' '<! not a comment either -->' '<! -- close enough -->' '<!><!<-- this was an empty comment>' - '<!!! another bogus comment !!!>') + '<!!! another bogus comment !!!>' + # see #32876 + '<![with square brackets]!>' + '<![\nmultiline\nbogusness\n]!>' + '<![more brackets]-[and a hyphen]!>' + '<![cdata[should be uppercase]]>' + '<![CDATA [whitespaces are not ignored]]>' + '<![CDATA]]>' # required '[' after CDATA + ) expected = [ + ('comment', 'ELEMENT br EMPTY'), ('comment', ' not really a comment '), ('comment', ' not a comment either --'), ('comment', ' -- close enough --'), ('comment', ''), ('comment', '<-- this was an empty comment'), ('comment', '!! another bogus comment !!!'), + ('comment', '[with square brackets]!'), + ('comment', '[\nmultiline\nbogusness\n]!'), + ('comment', '[more brackets]-[and a hyphen]!'), + ('comment', '[cdata[should be uppercase]]'), + ('comment', '[CDATA [whitespaces are not ignored]]'), + ('comment', '[CDATA]]'), ] self._run_check(html, expected) def test_broken_condcoms(self): # these condcoms are missing the '--' after '<!' and before the '>' + # and they are considered bogus comments according to + # "8.2.4.42. Markup declaration open state" html = ('<![if !(IE)]>broken condcom<![endif]>' '<![if ! IE]><link href="favicon.tiff"/><![endif]>' '<![if !IE 6]><img src="firefox.png" /><![endif]>' '<![if !ie 6]><b>foo</b><![endif]>' '<![if (!IE)|(lt IE 9)]><img src="mammoth.bmp" /><![endif]>') - # According to the HTML5 specs sections "8.2.4.44 Bogus comment state" - # and "8.2.4.45 Markup declaration open state", comment tokens should - # be emitted instead of 'unknown decl', but calling unknown_decl - # provides more flexibility. - # See also Lib/_markupbase.py:parse_declaration expected = [ - ('unknown decl', 'if !(IE)'), + ('comment', '[if !(IE)]'), ('data', 'broken condcom'), - ('unknown decl', 'endif'), - ('unknown decl', 'if ! IE'), + ('comment', '[endif]'), + ('comment', '[if ! IE]'), ('startendtag', 'link', [('href', 'favicon.tiff')]), - ('unknown decl', 'endif'), - ('unknown decl', 'if !IE 6'), + ('comment', '[endif]'), + ('comment', '[if !IE 6]'), ('startendtag', 'img', [('src', 'firefox.png')]), - ('unknown decl', 'endif'), - ('unknown decl', 'if !ie 6'), + ('comment', '[endif]'), + ('comment', '[if !ie 6]'), ('starttag', 'b', []), ('data', 'foo'), ('endtag', 'b'), - ('unknown decl', 'endif'), - ('unknown decl', 'if (!IE)|(lt IE 9)'), + ('comment', '[endif]'), + ('comment', '[if (!IE)|(lt IE 9)]'), ('startendtag', 'img', [('src', 'mammoth.bmp')]), - ('unknown decl', 'endif') + ('comment', '[endif]') ] self._run_check(html, expected) + @support.subTests('content', [ + 'just some plain text', + '<!-- not a comment -->', + '¬-an-entity-ref;', + "<not a='start tag'>", + '', + '[[I have many brackets]]', + 'I have a > in the middle', + 'I have a ]] in the middle', + '] ]>', + ']] >', + ('\n' + ' if (a < b && a > b) {\n' + ' printf("[<marquee>How?</marquee>]");\n' + ' }\n'), + ]) + def test_cdata_section_content(self, content): + # See "13.2.5.42 Markup declaration open state", + # "13.2.5.69 CDATA section state", and issue bpo-32876. + html = f'<svg><text y="100"><![CDATA[{content}]]></text></svg>' + expected = [ + ('starttag', 'svg', []), + ('starttag', 'text', [('y', '100')]), + ('unknown decl', 'CDATA[' + content), + ('endtag', 'text'), + ('endtag', 'svg'), + ] + self._run_check(html, expected) + self._run_check(html, expected, collector=EventCollector(autocdata=True)) + + def test_cdata_section(self): + # See "13.2.5.42 Markup declaration open state". + html = ('<![CDATA[foo<br>bar]]>' + '<svg><text y="100"><![CDATA[foo<br>bar]]></text></svg>' + '<![CDATA[foo<br>bar]]>') + expected = [ + ('comment', '[CDATA[foo<br'), + ('data', 'bar]]>'), + ('starttag', 'svg', []), + ('starttag', 'text', [('y', '100')]), + ('unknown decl', 'CDATA[foo<br>bar'), + ('endtag', 'text'), + ('endtag', 'svg'), + ('comment', '[CDATA[foo<br'), + ('data', 'bar]]>'), + ] + self._run_check(html, expected, collector=EventCollector(autocdata=True)) + def test_convert_charrefs_dropped_text(self): # #23144: make sure that all the events are triggered when # convert_charrefs is True, even if we don't call .close() @@ -598,6 +920,26 @@ def test_convert_charrefs_dropped_text(self): ('endtag', 'a'), ('data', ' bar & baz')] ) + @support.requires_resource('cpu') + def test_eof_no_quadratic_complexity(self): + # Each of these examples used to take about an hour. + # Now they take a fraction of a second. + def check(source): + parser = html.parser.HTMLParser() + parser.feed(source) + parser.close() + n = 120_000 + check("<a " * n) + check("<a a=" * n) + check("</a " * 14 * n) + check("</a a=" * 11 * n) + check("<!--" * 4 * n) + check("<!" * 60 * n) + check("<?" * 19 * n) + check("</$" * 15 * n) + check("<![CDATA[" * 9 * n) + check("<!doctype" * 35 * n) + class AttributesTestCase(TestCaseBase): @@ -606,9 +948,15 @@ def test_attr_syntax(self): ("starttag", "a", [("b", "v"), ("c", "v"), ("d", "v"), ("e", None)]) ] self._run_check("""<a b='v' c="v" d=v e>""", output) - self._run_check("""<a b = 'v' c = "v" d = v e>""", output) - self._run_check("""<a\nb\n=\n'v'\nc\n=\n"v"\nd\n=\nv\ne>""", output) - self._run_check("""<a\tb\t=\t'v'\tc\t=\t"v"\td\t=\tv\te>""", output) + self._run_check("<a foo==bar>", [('starttag', 'a', [('foo', '=bar')])]) + self._run_check("<a foo =bar>", [('starttag', 'a', [('foo', 'bar')])]) + self._run_check("<a foo\t=bar>", [('starttag', 'a', [('foo', 'bar')])]) + self._run_check("<a foo\v=bar>", [('starttag', 'a', [('foo\v', 'bar')])]) + self._run_check("<a foo\xa0=bar>", [('starttag', 'a', [('foo\xa0', 'bar')])]) + self._run_check("<a foo= bar>", [('starttag', 'a', [('foo', 'bar')])]) + self._run_check("<a foo=\tbar>", [('starttag', 'a', [('foo', 'bar')])]) + self._run_check("<a foo=\vbar>", [('starttag', 'a', [('foo', '\vbar')])]) + self._run_check("<a foo=\xa0bar>", [('starttag', 'a', [('foo', '\xa0bar')])]) def test_attr_values(self): self._run_check("""<a b='xxx\n\txxx' c="yyy\t\nyyy" d='\txyz\n'>""", @@ -617,6 +965,10 @@ def test_attr_values(self): ("d", "\txyz\n")])]) self._run_check("""<a b='' c="">""", [("starttag", "a", [("b", ""), ("c", "")])]) + self._run_check("<a b=\tx c=\ny>", + [('starttag', 'a', [('b', 'x'), ('c', 'y')])]) + self._run_check("<a b=\v c=\xa0>", + [("starttag", "a", [("b", "\v"), ("c", "\xa0")])]) # Regression test for SF patch #669683. self._run_check("<e a=rgb(1,2,3)>", [("starttag", "e", [("a", "rgb(1,2,3)")])]) @@ -683,13 +1035,17 @@ def test_malformed_attributes(self): ) expected = [ ('starttag', 'a', [('href', "test'style='color:red;bad1'")]), - ('data', 'test - bad1'), ('endtag', 'a'), + ('data', 'test - bad1'), + ('endtag', 'a'), ('starttag', 'a', [('href', "test'+style='color:red;ba2'")]), - ('data', 'test - bad2'), ('endtag', 'a'), + ('data', 'test - bad2'), + ('endtag', 'a'), ('starttag', 'a', [('href', "test'\xa0style='color:red;bad3'")]), - ('data', 'test - bad3'), ('endtag', 'a'), + ('data', 'test - bad3'), + ('endtag', 'a'), ('starttag', 'a', [('href', "test'\xa0style='color:red;bad4'")]), - ('data', 'test - bad4'), ('endtag', 'a') + ('data', 'test - bad4'), + ('endtag', 'a'), ] self._run_check(html, expected) @@ -787,5 +1143,17 @@ def test_weird_chars_in_unquoted_attribute_values(self): ('starttag', 'form', [('action', 'bogus|&#()value')])]) + +class TestInheritance(unittest.TestCase): + + @patch("_markupbase.ParserBase.__init__") + @patch("_markupbase.ParserBase.reset") + def test_base_class_methods_called(self, super_reset_method, super_init_method): + with patch('_markupbase.ParserBase') as parser_base: + EventCollector() + super_init_method.assert_called_once() + super_reset_method.assert_called_once() + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_http_cookies.py b/Lib/test/test_http_cookies.py index 6072c7e15e..3e0b4d1d5c 100644 --- a/Lib/test/test_http_cookies.py +++ b/Lib/test/test_http_cookies.py @@ -1,13 +1,15 @@ # Simple test suite for http/cookies.py import copy -from test.support import run_unittest, run_doctest import unittest +import doctest from http import cookies import pickle +from test import support +from test.support.testcase import ExtraAssertions -class CookieTests(unittest.TestCase): +class CookieTests(unittest.TestCase, ExtraAssertions): def test_basic(self): cases = [ @@ -58,6 +60,90 @@ def test_basic(self): for k, v in sorted(case['dict'].items()): self.assertEqual(C[k].value, v) + def test_obsolete_rfc850_date_format(self): + # Test cases with different days and dates in obsolete RFC 850 format + test_cases = [ + # from RFC 850, change EST to GMT + # https://datatracker.ietf.org/doc/html/rfc850#section-2 + { + 'data': 'key=value; expires=Saturday, 01-Jan-83 00:00:00 GMT', + 'output': 'Saturday, 01-Jan-83 00:00:00 GMT' + }, + { + 'data': 'key=value; expires=Friday, 19-Nov-82 16:59:30 GMT', + 'output': 'Friday, 19-Nov-82 16:59:30 GMT' + }, + # from RFC 9110 + # https://www.rfc-editor.org/rfc/rfc9110.html#section-5.6.7-6 + { + 'data': 'key=value; expires=Sunday, 06-Nov-94 08:49:37 GMT', + 'output': 'Sunday, 06-Nov-94 08:49:37 GMT' + }, + # other test cases + { + 'data': 'key=value; expires=Wednesday, 09-Nov-94 08:49:37 GMT', + 'output': 'Wednesday, 09-Nov-94 08:49:37 GMT' + }, + { + 'data': 'key=value; expires=Friday, 11-Nov-94 08:49:37 GMT', + 'output': 'Friday, 11-Nov-94 08:49:37 GMT' + }, + { + 'data': 'key=value; expires=Monday, 14-Nov-94 08:49:37 GMT', + 'output': 'Monday, 14-Nov-94 08:49:37 GMT' + }, + ] + + for case in test_cases: + with self.subTest(data=case['data']): + C = cookies.SimpleCookie() + C.load(case['data']) + + # Extract the cookie name from the data string + cookie_name = case['data'].split('=')[0] + + # Check if the cookie is loaded correctly + self.assertIn(cookie_name, C) + self.assertEqual(C[cookie_name].get('expires'), case['output']) + + def test_unquote(self): + cases = [ + (r'a="b=\""', 'b="'), + (r'a="b=\\"', 'b=\\'), + (r'a="b=\="', 'b=='), + (r'a="b=\n"', 'b=n'), + (r'a="b=\042"', 'b="'), + (r'a="b=\134"', 'b=\\'), + (r'a="b=\377"', 'b=\xff'), + (r'a="b=\400"', 'b=400'), + (r'a="b=\42"', 'b=42'), + (r'a="b=\\042"', 'b=\\042'), + (r'a="b=\\134"', 'b=\\134'), + (r'a="b=\\\""', 'b=\\"'), + (r'a="b=\\\042"', 'b=\\"'), + (r'a="b=\134\""', 'b=\\"'), + (r'a="b=\134\042"', 'b=\\"'), + ] + for encoded, decoded in cases: + with self.subTest(encoded): + C = cookies.SimpleCookie() + C.load(encoded) + self.assertEqual(C['a'].value, decoded) + + @support.requires_resource('cpu') + def test_unquote_large(self): + #n = 10**6 + n = 10**4 # XXX: RUSTPYTHON; This takes more than 10 minutes to run. lower to 4 + for encoded in r'\\', r'\134': + with self.subTest(encoded): + data = 'a="b=' + encoded*n + ';"' + C = cookies.SimpleCookie() + C.load(data) + value = C['a'].value + self.assertEqual(value[:3], 'b=\\') + self.assertEqual(value[-2:], '\\;') + self.assertEqual(len(value), n + 3) + def test_load(self): C = cookies.SimpleCookie() C.load('Customer="WILE_E_COYOTE"; Version=1; Path=/acme') @@ -96,7 +182,7 @@ def test_special_attrs(self): C = cookies.SimpleCookie('Customer="WILE_E_COYOTE"') C['Customer']['expires'] = 0 # can't test exact output, it always depends on current date/time - self.assertTrue(C.output().endswith('GMT')) + self.assertEndsWith(C.output(), 'GMT') # loading 'expires' C = cookies.SimpleCookie() @@ -479,9 +565,11 @@ def test_repr(self): r'Set-Cookie: key=coded_val; ' r'expires=\w+, \d+ \w+ \d+ \d+:\d+:\d+ \w+') -def test_main(): - run_unittest(CookieTests, MorselTests) - run_doctest(cookies) + +def load_tests(loader, tests, pattern): + tests.addTest(doctest.DocTestSuite(cookies)) + return tests + if __name__ == '__main__': - test_main() + unittest.main() diff --git a/Lib/test/test_httplib.py b/Lib/test/test_httplib.py index 5a59f372ad..d4a6eefe32 100644 --- a/Lib/test/test_httplib.py +++ b/Lib/test/test_httplib.py @@ -22,11 +22,11 @@ here = os.path.dirname(__file__) # Self-signed cert file for 'localhost' -CERT_localhost = os.path.join(here, 'keycert.pem') +CERT_localhost = os.path.join(here, 'certdata/keycert.pem') # Self-signed cert file for 'fakehostname' -CERT_fakehostname = os.path.join(here, 'keycert2.pem') +CERT_fakehostname = os.path.join(here, 'certdata/keycert2.pem') # Self-signed cert file for self-signed.pythontest.net -CERT_selfsigned_pythontestdotnet = os.path.join(here, 'selfsigned_pythontestdotnet.pem') +CERT_selfsigned_pythontestdotnet = os.path.join(here, 'certdata/selfsigned_pythontestdotnet.pem') # constants for testing chunked encoding chunked_start = ( @@ -1733,8 +1733,6 @@ def test_networked_trusted_by_default_cert(self): h.close() self.assertIn('text/html', content_type) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_networked_good_cert(self): # We feed the server's cert as a validating cert import ssl @@ -1780,8 +1778,6 @@ def test_networked_bad_cert(self): h.request('GET', '/') self.assertEqual(exc_info.exception.reason, 'CERTIFICATE_VERIFY_FAILED') - # TODO: RUSTPYTHON - @unittest.expectedFailure @unittest.skipIf(sys.platform == 'darwin', 'Occasionally success on macOS') def test_local_unknown_cert(self): # The custom cert isn't known to the default trust bundle @@ -1805,8 +1801,6 @@ def test_local_good_hostname(self): self.addCleanup(resp.close) self.assertEqual(resp.status, 404) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_local_bad_hostname(self): # The (valid) cert doesn't validate the HTTP hostname import ssl diff --git a/Lib/test/test_import/__init__.py b/Lib/test/test_import/__init__.py index 89e5ec1534..51fafd89ce 100644 --- a/Lib/test/test_import/__init__.py +++ b/Lib/test/test_import/__init__.py @@ -457,6 +457,7 @@ def test_issue31492(self): with self.assertRaises(AttributeError): os.does_not_exist + @unittest.skipIf(sys.platform == 'win32', 'TODO: RUSTPYTHON; Flaky') @threading_helper.requires_working_threading() def test_concurrency(self): # bpo 38091: this is a hack to slow down the code that calls @@ -1380,8 +1381,6 @@ def test_crossreference2(self): self.assertIn('partially initialized module', errmsg) self.assertIn('circular import', errmsg) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_circular_from_import(self): with self.assertRaises(ImportError) as cm: import test.test_import.data.circular_imports.from_cycle1 @@ -1404,8 +1403,6 @@ def test_absolute_circular_submodule(self): str(cm.exception), ) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_unwritable_module(self): self.addCleanup(unload, "test.test_import.data.unwritable") self.addCleanup(unload, "test.test_import.data.unwritable.x") diff --git a/Lib/test/test_importlib/import_/test___package__.py b/Lib/test/test_importlib/import_/test___package__.py index 431faea5b4..7130c99a6f 100644 --- a/Lib/test/test_importlib/import_/test___package__.py +++ b/Lib/test/test_importlib/import_/test___package__.py @@ -56,8 +56,6 @@ def test_using___name__(self): '__path__': []}) self.assertEqual(module.__name__, 'pkg') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_warn_when_using___name__(self): with self.assertWarns(ImportWarning): self.import_module({'__name__': 'pkg.fake', '__path__': []}) @@ -75,8 +73,6 @@ def test_spec_fallback(self): module = self.import_module({'__spec__': FakeSpec('pkg.fake')}) self.assertEqual(module.__name__, 'pkg') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_warn_when_package_and_spec_disagree(self): # Raise a DeprecationWarning if __package__ != __spec__.parent. with self.assertWarns(DeprecationWarning): diff --git a/Lib/test/test_importlib/import_/test_helpers.py b/Lib/test/test_importlib/import_/test_helpers.py index 28cdc0e526..550f88d1d7 100644 --- a/Lib/test/test_importlib/import_/test_helpers.py +++ b/Lib/test/test_importlib/import_/test_helpers.py @@ -126,8 +126,6 @@ def test_gh86298_loader_is_none_and_spec_loader_is_none(self): ValueError, _bootstrap_external._bless_my_loader, bar.__dict__) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_gh86298_no_spec(self): bar = ModuleType('bar') bar.__loader__ = object() @@ -137,8 +135,6 @@ def test_gh86298_no_spec(self): DeprecationWarning, _bootstrap_external._bless_my_loader, bar.__dict__) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_gh86298_spec_is_none(self): bar = ModuleType('bar') bar.__loader__ = object() @@ -148,8 +144,6 @@ def test_gh86298_spec_is_none(self): DeprecationWarning, _bootstrap_external._bless_my_loader, bar.__dict__) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_gh86298_no_spec_loader(self): bar = ModuleType('bar') bar.__loader__ = object() @@ -159,8 +153,6 @@ def test_gh86298_no_spec_loader(self): DeprecationWarning, _bootstrap_external._bless_my_loader, bar.__dict__) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_gh86298_loader_and_spec_loader_disagree(self): bar = ModuleType('bar') bar.__loader__ = object() diff --git a/Lib/test/test_importlib/import_/test_meta_path.py b/Lib/test/test_importlib/import_/test_meta_path.py index 26e7b070b9..8689017ba4 100644 --- a/Lib/test/test_importlib/import_/test_meta_path.py +++ b/Lib/test/test_importlib/import_/test_meta_path.py @@ -30,8 +30,6 @@ def test_continuing(self): with util.import_state(meta_path=[first, second]): self.assertIs(self.__import__(mod_name), second.modules[mod_name]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_empty(self): # Raise an ImportWarning if sys.meta_path is empty. module_name = 'nothing' diff --git a/Lib/test/test_importlib/import_/test_path.py b/Lib/test/test_importlib/import_/test_path.py index 9cf3a77cb8..89b52fbd1e 100644 --- a/Lib/test/test_importlib/import_/test_path.py +++ b/Lib/test/test_importlib/import_/test_path.py @@ -68,8 +68,6 @@ def test_path_hooks(self): self.assertIn(path, sys.path_importer_cache) self.assertIs(sys.path_importer_cache[path], importer) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_empty_path_hooks(self): # Test that if sys.path_hooks is empty a warning is raised, # sys.path_importer_cache gets None set, and PathFinder returns None. diff --git a/Lib/test/test_importlib/source/test_file_loader.py b/Lib/test/test_importlib/source/test_file_loader.py index 9c85bd234f..fa0efee8da 100644 --- a/Lib/test/test_importlib/source/test_file_loader.py +++ b/Lib/test/test_importlib/source/test_file_loader.py @@ -236,7 +236,6 @@ def test_unloadable(self): warnings.simplefilter('ignore', DeprecationWarning) loader.load_module('bad name') - @unittest.skip("TODO: RUSTPYTHON; successful only for Frozen") @util.writes_bytecode_files def test_checked_hash_based_pyc(self): with util.create_modules('_temp') as mapping: @@ -293,7 +292,6 @@ def test_overridden_checked_hash_based_pyc(self): loader.exec_module(mod) self.assertEqual(mod.state, 'old') - @unittest.skip("TODO: RUSTPYTHON; successful only for Frozen") @util.writes_bytecode_files def test_unchecked_hash_based_pyc(self): with util.create_modules('_temp') as mapping: @@ -324,7 +322,6 @@ def test_unchecked_hash_based_pyc(self): data[8:16], ) - @unittest.skip("TODO: RUSTPYTHON; successful only for Frozen") @util.writes_bytecode_files def test_overridden_unchecked_hash_based_pyc(self): with util.create_modules('_temp') as mapping, \ @@ -362,7 +359,6 @@ def test_overridden_unchecked_hash_based_pyc(self): ) = util.test_both(SimpleTest, importlib=importlib, machinery=machinery, abc=importlib_abc, util=importlib_util) - class SourceDateEpochTestMeta(SourceDateEpochTestMeta, type(Source_SimpleTest)): pass @@ -672,31 +668,27 @@ def test_read_only_bytecode(self): os.chmod(bytecode_path, stat.S_IWUSR) -# TODO: RUSTPYTHON -# class SourceLoaderBadBytecodeTestPEP451( -# SourceLoaderBadBytecodeTest, BadBytecodeTestPEP451): -# pass - - -# (Frozen_SourceBadBytecodePEP451, -# Source_SourceBadBytecodePEP451 -# ) = util.test_both(SourceLoaderBadBytecodeTestPEP451, importlib=importlib, -# machinery=machinery, abc=importlib_abc, -# util=importlib_util) +class SourceLoaderBadBytecodeTestPEP451( + SourceLoaderBadBytecodeTest, BadBytecodeTestPEP451): + pass -# TODO: RUSTPYTHON -# class SourceLoaderBadBytecodeTestPEP302( -# SourceLoaderBadBytecodeTest, BadBytecodeTestPEP302): -# pass +(Frozen_SourceBadBytecodePEP451, + Source_SourceBadBytecodePEP451 + ) = util.test_both(SourceLoaderBadBytecodeTestPEP451, importlib=importlib, + machinery=machinery, abc=importlib_abc, + util=importlib_util) +class SourceLoaderBadBytecodeTestPEP302( + SourceLoaderBadBytecodeTest, BadBytecodeTestPEP302): + pass -# (Frozen_SourceBadBytecodePEP302, -# Source_SourceBadBytecodePEP302 -# ) = util.test_both(SourceLoaderBadBytecodeTestPEP302, importlib=importlib, -# machinery=machinery, abc=importlib_abc, -# util=importlib_util) +(Frozen_SourceBadBytecodePEP302, + Source_SourceBadBytecodePEP302 + ) = util.test_both(SourceLoaderBadBytecodeTestPEP302, importlib=importlib, + machinery=machinery, abc=importlib_abc, + util=importlib_util) class SourcelessLoaderBadBytecodeTest: @@ -772,30 +764,27 @@ def test_non_code_marshal(self): self._test_non_code_marshal(del_source=True) -# TODO: RUSTPYTHON -# class SourcelessLoaderBadBytecodeTestPEP451(SourcelessLoaderBadBytecodeTest, -# BadBytecodeTestPEP451): -# pass - - -# (Frozen_SourcelessBadBytecodePEP451, -# Source_SourcelessBadBytecodePEP451 -# ) = util.test_both(SourcelessLoaderBadBytecodeTestPEP451, importlib=importlib, -# machinery=machinery, abc=importlib_abc, -# util=importlib_util) +class SourcelessLoaderBadBytecodeTestPEP451(SourcelessLoaderBadBytecodeTest, + BadBytecodeTestPEP451): + pass -# class SourcelessLoaderBadBytecodeTestPEP302(SourcelessLoaderBadBytecodeTest, -# BadBytecodeTestPEP302): -# pass +(Frozen_SourcelessBadBytecodePEP451, + Source_SourcelessBadBytecodePEP451 + ) = util.test_both(SourcelessLoaderBadBytecodeTestPEP451, importlib=importlib, + machinery=machinery, abc=importlib_abc, + util=importlib_util) +class SourcelessLoaderBadBytecodeTestPEP302(SourcelessLoaderBadBytecodeTest, + BadBytecodeTestPEP302): + pass -# (Frozen_SourcelessBadBytecodePEP302, -# Source_SourcelessBadBytecodePEP302 -# ) = util.test_both(SourcelessLoaderBadBytecodeTestPEP302, importlib=importlib, -# machinery=machinery, abc=importlib_abc, -# util=importlib_util) +(Frozen_SourcelessBadBytecodePEP302, + Source_SourcelessBadBytecodePEP302 + ) = util.test_both(SourcelessLoaderBadBytecodeTestPEP302, importlib=importlib, + machinery=machinery, abc=importlib_abc, + util=importlib_util) if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_importlib/test_metadata_api.py b/Lib/test/test_importlib/test_metadata_api.py index 55c9f8007e..33c6e85ee9 100644 --- a/Lib/test/test_importlib/test_metadata_api.py +++ b/Lib/test/test_importlib/test_metadata_api.py @@ -139,8 +139,6 @@ def test_entry_points_missing_name(self): def test_entry_points_missing_group(self): assert entry_points(group='missing') == () - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_entry_points_allows_no_attributes(self): ep = entry_points().select(group='entries', name='main') with self.assertRaises(AttributeError): diff --git a/Lib/test/test_importlib/test_util.py b/Lib/test/test_importlib/test_util.py index 201f506911..87b7de9889 100644 --- a/Lib/test/test_importlib/test_util.py +++ b/Lib/test/test_importlib/test_util.py @@ -327,15 +327,6 @@ def test_incorporates_rn(self): ) = util.test_both(MagicNumberTests, util=importlib_util) -# TODO: RUSTPYTHON -@unittest.expectedFailure -def test_incorporates_rn_MONKEYPATCH(self): - self.assertTrue(self.util.MAGIC_NUMBER.endswith(b'\r\n')) - -# TODO: RUSTPYTHON -Frozen_MagicNumberTests.test_incorporates_rn = test_incorporates_rn_MONKEYPATCH - - class PEP3147Tests: """Tests of PEP 3147-related functions: cache_from_source and source_from_cache.""" diff --git a/Lib/test/test_inspect/__init__.py b/Lib/test/test_inspect/__init__.py new file mode 100644 index 0000000000..f2a39a3fe2 --- /dev/null +++ b/Lib/test/test_inspect/__init__.py @@ -0,0 +1,6 @@ +import os +from test import support + + +def load_tests(*args): + return support.load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_inspect/inspect_fodder.py b/Lib/test/test_inspect/inspect_fodder.py new file mode 100644 index 0000000000..febd54c86f --- /dev/null +++ b/Lib/test/test_inspect/inspect_fodder.py @@ -0,0 +1,120 @@ +# line 1 +'A module docstring.' + +import inspect +# line 5 + +# line 7 +def spam(a, /, b, c, d=3, e=4, f=5, *g, **h): + eggs(b + d, c + f) + +# line 11 +def eggs(x, y): + "A docstring." + global fr, st + fr = inspect.currentframe() + st = inspect.stack() + p = x + q = y / 0 + +# line 20 +class StupidGit: + """A longer, + + indented + + docstring.""" +# line 27 + + def abuse(self, a, b, c): + """Another + +\tdocstring + + containing + +\ttabs +\t + """ + self.argue(a, b, c) +# line 40 + def argue(self, a, b, c): + try: + spam(a, b, c) + except BaseException as e: + self.ex = e + self.tr = inspect.trace() + + @property + def contradiction(self): + 'The automatic gainsaying.' + pass + +# line 53 +class MalodorousPervert(StupidGit): + def abuse(self, a, b, c): + pass + + @property + def contradiction(self): + pass + +Tit = MalodorousPervert + +class ParrotDroppings: + pass + +class FesteringGob(MalodorousPervert, ParrotDroppings): + def abuse(self, a, b, c): + pass + + def _getter(self): + pass + contradiction = property(_getter) + +async def lobbest(grenade): + pass + +currentframe = inspect.currentframe() +try: + raise Exception() +except BaseException as e: + tb = e.__traceback__ + +class Callable: + def __call__(self, *args): + return args + + def as_method_of(self, obj): + from types import MethodType + return MethodType(self, obj) + +custom_method = Callable().as_method_of(42) +del Callable + +# line 95 +class WhichComments: + # line 97 + # before f + def f(self): + # line 100 + # start f + return 1 + # line 103 + # end f + # line 105 + # after f + + # before asyncf - line 108 + async def asyncf(self): + # start asyncf + return 2 + # end asyncf + # after asyncf - line 113 + # end of WhichComments - line 114 + # after WhichComments - line 115 + +# Test that getsource works on a line that includes +# a closing parenthesis with the opening paren being in another line +( +); after_closing = lambda: 1 diff --git a/Lib/test/test_inspect/inspect_fodder2.py b/Lib/test/test_inspect/inspect_fodder2.py new file mode 100644 index 0000000000..157e12167b --- /dev/null +++ b/Lib/test/test_inspect/inspect_fodder2.py @@ -0,0 +1,403 @@ +# line 1 +def wrap(foo=None): + def wrapper(func): + return func + return wrapper + +# line 7 +def replace(func): + def insteadfunc(): + print('hello') + return insteadfunc + +# line 13 +@wrap() +@wrap(wrap) +def wrapped(): + pass + +# line 19 +@replace +def gone(): + pass + +# line 24 +oll = lambda m: m + +# line 27 +tll = lambda g: g and \ +g and \ +g + +# line 32 +tlli = lambda d: d and \ + d + +# line 36 +def onelinefunc(): pass + +# line 39 +def manyargs(arg1, arg2, +arg3, arg4): pass + +# line 43 +def twolinefunc(m): return m and \ +m + +# line 47 +a = [None, + lambda x: x, + None] + +# line 52 +def setfunc(func): + globals()["anonymous"] = func +setfunc(lambda x, y: x*y) + +# line 57 +def with_comment(): # hello + world + +# line 61 +multiline_sig = [ + lambda x, \ + y: x+y, + None, + ] + +# line 68 +def func69(): + class cls70: + def func71(): + pass + return cls70 +extra74 = 74 + +# line 76 +def func77(): pass +(extra78, stuff78) = 'xy' +extra79 = 'stop' + +# line 81 +class cls82: + def func83(): pass +(extra84, stuff84) = 'xy' +extra85 = 'stop' + +# line 87 +def func88(): + # comment + return 90 + +# line 92 +def f(): + class X: + def g(): + "doc" + return 42 + return X +method_in_dynamic_class = f().g + +#line 101 +def keyworded(*arg1, arg2=1): + pass + +#line 105 +def annotated(arg1: list): + pass + +#line 109 +def keyword_only_arg(*, arg): + pass + +@wrap(lambda: None) +def func114(): + return 115 + +class ClassWithMethod: + def method(self): + pass + +from functools import wraps + +def decorator(func): + @wraps(func) + def fake(): + return 42 + return fake + +#line 129 +@decorator +def real(): + return 20 + +#line 134 +class cls135: + def func136(): + def func137(): + never_reached1 + never_reached2 + +# line 141 +class cls142: + a = """ +class cls149: + ... +""" + +# line 148 +class cls149: + + def func151(self): + pass + +''' +class cls160: + pass +''' + +# line 159 +class cls160: + + def func162(self): + pass + +# line 165 +class cls166: + a = ''' + class cls175: + ... + ''' + +# line 172 +class cls173: + + class cls175: + pass + +# line 178 +class cls179: + pass + +# line 182 +class cls183: + + class cls185: + + def func186(self): + pass + +def class_decorator(cls): + return cls + +# line 193 +@class_decorator +@class_decorator +class cls196: + + @class_decorator + @class_decorator + class cls200: + pass + +class cls203: + class cls204: + class cls205: + pass + class cls207: + class cls205: + pass + +# line 211 +def func212(): + class cls213: + pass + return cls213 + +# line 217 +class cls213: + def func219(self): + class cls220: + pass + return cls220 + +# line 224 +async def func225(): + class cls226: + pass + return cls226 + +# line 230 +class cls226: + async def func232(self): + class cls233: + pass + return cls233 + +if True: + class cls238: + class cls239: + '''if clause cls239''' +else: + class cls238: + class cls239: + '''else clause 239''' + pass + +#line 247 +def positional_only_arg(a, /): + pass + +#line 251 +def all_markers(a, b, /, c, d, *, e, f): + pass + +# line 255 +def all_markers_with_args_and_kwargs(a, b, /, c, d, *args, e, f, **kwargs): + pass + +#line 259 +def all_markers_with_defaults(a, b=1, /, c=2, d=3, *, e=4, f=5): + pass + +# line 263 +def deco_factory(**kwargs): + def deco(f): + @wraps(f) + def wrapper(*a, **kwd): + kwd.update(kwargs) + return f(*a, **kwd) + return wrapper + return deco + +@deco_factory(foo=(1 + 2), bar=lambda: 1) +def complex_decorated(foo=0, bar=lambda: 0): + return foo + bar() + +# line 276 +parenthesized_lambda = ( + lambda: ()) +parenthesized_lambda2 = [ + lambda: ()][0] +parenthesized_lambda3 = {0: + lambda: ()}[0] + +# line 285 +post_line_parenthesized_lambda1 = (lambda: () +) + +# line 289 +nested_lambda = ( + lambda right: [].map( + lambda length: ())) + +# line 294 +if True: + class cls296: + def f(): + pass +else: + class cls296: + def g(): + pass + +# line 304 +if False: + class cls310: + def f(): + pass +else: + class cls310: + def g(): + pass + +# line 314 +class ClassWithCodeObject: + import sys + code = sys._getframe(0).f_code + +import enum + +# line 321 +class enum322(enum.Enum): + A = 'a' + +# line 325 +class enum326(enum.IntEnum): + A = 1 + +# line 329 +class flag330(enum.Flag): + A = 1 + +# line 333 +class flag334(enum.IntFlag): + A = 1 + +# line 337 +simple_enum338 = enum.Enum('simple_enum338', 'A') +simple_enum339 = enum.IntEnum('simple_enum339', 'A') +simple_flag340 = enum.Flag('simple_flag340', 'A') +simple_flag341 = enum.IntFlag('simple_flag341', 'A') + +import typing + +# line 345 +class nt346(typing.NamedTuple): + x: int + y: int + +# line 350 +nt351 = typing.NamedTuple('nt351', (('x', int), ('y', int))) + +# line 353 +class td354(typing.TypedDict): + x: int + y: int + +# line 358 +td359 = typing.TypedDict('td359', (('x', int), ('y', int))) + +import dataclasses + +# line 363 +@dataclasses.dataclass +class dc364: + x: int + y: int + +# line 369 +dc370 = dataclasses.make_dataclass('dc370', (('x', int), ('y', int))) +dc371 = dataclasses.make_dataclass('dc370', (('x', int), ('y', int)), module=__name__) + +import inspect +import itertools + +# line 376 +ge377 = ( + inspect.currentframe() + for i in itertools.count() +) + +# line 382 +def func383(): + # line 384 + ge385 = ( + inspect.currentframe() + for i in itertools.count() + ) + return ge385 + +# line 391 +@decorator +# comment +def func394(): + return 395 + +# line 397 +@decorator + +def func400(): + return 401 + +pass # end of file diff --git a/Lib/test/test_inspect/inspect_stock_annotations.py b/Lib/test/test_inspect/inspect_stock_annotations.py new file mode 100644 index 0000000000..d115a25b65 --- /dev/null +++ b/Lib/test/test_inspect/inspect_stock_annotations.py @@ -0,0 +1,28 @@ +a:int=3 +b:str="foo" + +class MyClass: + a:int=4 + b:str="bar" + def __init__(self, a, b): + self.a = a + self.b = b + def __eq__(self, other): + return isinstance(other, MyClass) and self.a == other.a and self.b == other.b + +def function(a:int, b:str) -> MyClass: + return MyClass(a, b) + + +def function2(a:int, b:"str", c:MyClass) -> MyClass: + pass + + +def function3(a:"int", b:"str", c:"MyClass"): + pass + + +class UnannotatedClass: + pass + +def unannotated_function(a, b, c): pass diff --git a/Lib/test/test_inspect/inspect_stringized_annotations.py b/Lib/test/test_inspect/inspect_stringized_annotations.py new file mode 100644 index 0000000000..a56fb050ea --- /dev/null +++ b/Lib/test/test_inspect/inspect_stringized_annotations.py @@ -0,0 +1,34 @@ +from __future__ import annotations + +a:int=3 +b:str="foo" + +class MyClass: + a:int=4 + b:str="bar" + def __init__(self, a, b): + self.a = a + self.b = b + def __eq__(self, other): + return isinstance(other, MyClass) and self.a == other.a and self.b == other.b + +def function(a:int, b:str) -> MyClass: + return MyClass(a, b) + + +def function2(a:int, b:"str", c:MyClass) -> MyClass: + pass + + +def function3(a:"int", b:"str", c:"MyClass"): + pass + + +class UnannotatedClass: + pass + +def unannotated_function(a, b, c): pass + +class MyClassWithLocalAnnotations: + mytype = int + x: mytype diff --git a/Lib/test/test_inspect/inspect_stringized_annotations_2.py b/Lib/test/test_inspect/inspect_stringized_annotations_2.py new file mode 100644 index 0000000000..87206d5a64 --- /dev/null +++ b/Lib/test/test_inspect/inspect_stringized_annotations_2.py @@ -0,0 +1,3 @@ +from __future__ import annotations + +def foo(a, b, c): pass diff --git a/Lib/test/test_inspect/inspect_stringized_annotations_pep695.py b/Lib/test/test_inspect/inspect_stringized_annotations_pep695.py new file mode 100644 index 0000000000..39bfe2edb0 --- /dev/null +++ b/Lib/test/test_inspect/inspect_stringized_annotations_pep695.py @@ -0,0 +1,87 @@ +from __future__ import annotations +from typing import Callable, Unpack + + +class A[T, *Ts, **P]: + x: T + y: tuple[*Ts] + z: Callable[P, str] + + +class B[T, *Ts, **P]: + T = int + Ts = str + P = bytes + x: T + y: Ts + z: P + + +Eggs = int +Spam = str + + +class C[Eggs, **Spam]: + x: Eggs + y: Spam + + +def generic_function[T, *Ts, **P]( + x: T, *y: Unpack[Ts], z: P.args, zz: P.kwargs +) -> None: ... + + +def generic_function_2[Eggs, **Spam](x: Eggs, y: Spam): pass + + +class D: + Foo = int + Bar = str + + def generic_method[Foo, **Bar]( + self, x: Foo, y: Bar + ) -> None: ... + + def generic_method_2[Eggs, **Spam](self, x: Eggs, y: Spam): pass + + +# Eggs is `int` in globals, a TypeVar in type_params, and `str` in locals: +class E[Eggs]: + Eggs = str + x: Eggs + + + +def nested(): + from types import SimpleNamespace + from inspect import get_annotations + + Eggs = bytes + Spam = memoryview + + + class F[Eggs, **Spam]: + x: Eggs + y: Spam + + def generic_method[Eggs, **Spam](self, x: Eggs, y: Spam): pass + + + def generic_function[Eggs, **Spam](x: Eggs, y: Spam): pass + + + # Eggs is `int` in globals, `bytes` in the function scope, + # a TypeVar in the type_params, and `str` in locals: + class G[Eggs]: + Eggs = str + x: Eggs + + + return SimpleNamespace( + F=F, + F_annotations=get_annotations(F, eval_str=True), + F_meth_annotations=get_annotations(F.generic_method, eval_str=True), + G_annotations=get_annotations(G, eval_str=True), + generic_func=generic_function, + generic_func_annotations=get_annotations(generic_function, eval_str=True) + ) diff --git a/Lib/test/test_inspect/test_inspect.py b/Lib/test/test_inspect/test_inspect.py new file mode 100644 index 0000000000..8f87fa8571 --- /dev/null +++ b/Lib/test/test_inspect/test_inspect.py @@ -0,0 +1,6753 @@ +import asyncio +import builtins +import collections +import copy +import datetime +import functools +import gc +import importlib +import inspect +import io +import linecache +import os +import dis +from os.path import normcase +import pickle +import shutil +import stat +import sys +import subprocess +import time +import types +import tempfile +import textwrap +from typing import Unpack +import unicodedata +import unittest +import unittest.mock +import warnings +import weakref + +# XXX: RUSTPYTHON; skip _pickle tests if _pickle is not available +try: + import _pickle +except ImportError: + _pickle = None + + +try: + from concurrent.futures import ThreadPoolExecutor +except ImportError: + ThreadPoolExecutor = None + +from test.support import cpython_only, import_helper, suppress_immortalization +from test.support import MISSING_C_DOCSTRINGS, ALWAYS_EQ +# XXX: RUSTPYTHON; test.support is not updated yet +from test.support.import_helper import DirsOnSysPath #, ready_to_import +from test.support.os_helper import TESTFN, temp_cwd +from test.support.script_helper import assert_python_ok, assert_python_failure, kill_python +from test.support import has_subprocess_support, SuppressCrashReport +from test import support + +from test.test_inspect import inspect_fodder as mod +from test.test_inspect import inspect_fodder2 as mod2 +from test.test_inspect import inspect_stock_annotations +from test.test_inspect import inspect_stringized_annotations +from test.test_inspect import inspect_stringized_annotations_2 +from test.test_inspect import inspect_stringized_annotations_pep695 + + +# Functions tested in this suite: +# ismodule, isclass, ismethod, isfunction, istraceback, isframe, iscode, +# isbuiltin, isroutine, isgenerator, isgeneratorfunction, getmembers, +# getdoc, getfile, getmodule, getsourcefile, getcomments, getsource, +# getclasstree, getargvalues, formatargvalues, currentframe, +# stack, trace, ismethoddescriptor, isdatadescriptor, ismethodwrapper + +# NOTE: There are some additional tests relating to interaction with +# zipimport in the test_zipimport_support test module. + +modfile = mod.__file__ +if modfile.endswith(('c', 'o')): + modfile = modfile[:-1] + +# Normalize file names: on Windows, the case of file names of compiled +# modules depends on the path used to start the python executable. +modfile = normcase(modfile) + +def revise(filename, *args): + return (normcase(filename),) + args + +git = mod.StupidGit() + + +def tearDownModule(): + if support.has_socket_support: + asyncio.set_event_loop_policy(None) + + +def signatures_with_lexicographic_keyword_only_parameters(): + """ + Yields a whole bunch of functions with only keyword-only parameters, + where those parameters are always in lexicographically sorted order. + """ + parameters = ['a', 'bar', 'c', 'delta', 'ephraim', 'magical', 'yoyo', 'z'] + for i in range(1, 2**len(parameters)): + p = [] + bit = 1 + for j in range(len(parameters)): + if i & (bit << j): + p.append(parameters[j]) + fn_text = "def foo(*, " + ", ".join(p) + "): pass" + symbols = {} + exec(fn_text, symbols, symbols) + yield symbols['foo'] + + +def unsorted_keyword_only_parameters_fn(*, throw, out, the, baby, with_, + the_, bathwater): + pass + +unsorted_keyword_only_parameters = 'throw out the baby with_ the_ bathwater'.split() + +class IsTestBase(unittest.TestCase): + predicates = set([inspect.isbuiltin, inspect.isclass, inspect.iscode, + inspect.isframe, inspect.isfunction, inspect.ismethod, + inspect.ismodule, inspect.istraceback, + inspect.isgenerator, inspect.isgeneratorfunction, + inspect.iscoroutine, inspect.iscoroutinefunction, + inspect.isasyncgen, inspect.isasyncgenfunction, + inspect.ismethodwrapper]) + + def istest(self, predicate, exp): + obj = eval(exp) + self.assertTrue(predicate(obj), '%s(%s)' % (predicate.__name__, exp)) + + for other in self.predicates - set([predicate]): + if (predicate == inspect.isgeneratorfunction or \ + predicate == inspect.isasyncgenfunction or \ + predicate == inspect.iscoroutinefunction) and \ + other == inspect.isfunction: + continue + self.assertFalse(other(obj), 'not %s(%s)' % (other.__name__, exp)) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test__all__(self): + support.check__all__(self, inspect, not_exported=("modulesbyfile",)) + +def generator_function_example(self): + for i in range(2): + yield i + +async def async_generator_function_example(self): + async for i in range(2): + yield i + +async def coroutine_function_example(self): + return 'spam' + +@types.coroutine +def gen_coroutine_function_example(self): + yield + return 'spam' + +def meth_noargs(): pass +def meth_o(object, /): pass +def meth_self_noargs(self, /): pass +def meth_self_o(self, object, /): pass +def meth_type_noargs(type, /): pass +def meth_type_o(type, object, /): pass + +# Decorator decorator that returns a simple wrapped function +def identity_wrapper(func): + @functools.wraps(func) + def wrapped(*args, **kwargs): + return func(*args, **kwargs) + return wrapped + +# Original signature of the simple wrapped function returned by +# identity_wrapper(). +varargs_signature = ( + (('args', ..., ..., 'var_positional'), + ('kwargs', ..., ..., 'var_keyword')), + ..., +) + +# Decorator decorator that returns a simple descriptor +class custom_descriptor: + def __init__(self, func): + self.func = func + + def __get__(self, instance, owner): + return self.func.__get__(instance, owner) + + +class TestPredicates(IsTestBase): + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_excluding_predicates(self): + global tb + self.istest(inspect.isbuiltin, 'sys.exit') + self.istest(inspect.isbuiltin, '[].append') + self.istest(inspect.iscode, 'mod.spam.__code__') + try: + 1/0 + except Exception as e: + tb = e.__traceback__ + self.istest(inspect.isframe, 'tb.tb_frame') + self.istest(inspect.istraceback, 'tb') + if hasattr(types, 'GetSetDescriptorType'): + self.istest(inspect.isgetsetdescriptor, + 'type(tb.tb_frame).f_locals') + else: + self.assertFalse(inspect.isgetsetdescriptor(type(tb.tb_frame).f_locals)) + finally: + # Clear traceback and all the frames and local variables hanging to it. + tb = None + self.istest(inspect.isfunction, 'mod.spam') + self.istest(inspect.isfunction, 'mod.StupidGit.abuse') + self.istest(inspect.ismethod, 'git.argue') + self.istest(inspect.ismethod, 'mod.custom_method') + self.istest(inspect.ismodule, 'mod') + self.istest(inspect.ismethoddescriptor, 'int.__add__') + self.istest(inspect.isdatadescriptor, 'collections.defaultdict.default_factory') + self.istest(inspect.isgenerator, '(x for x in range(2))') + self.istest(inspect.isgeneratorfunction, 'generator_function_example') + self.istest(inspect.isasyncgen, + 'async_generator_function_example(1)') + self.istest(inspect.isasyncgenfunction, + 'async_generator_function_example') + + with warnings.catch_warnings(): + warnings.simplefilter("ignore") + self.istest(inspect.iscoroutine, 'coroutine_function_example(1)') + self.istest(inspect.iscoroutinefunction, 'coroutine_function_example') + + if hasattr(types, 'MemberDescriptorType'): + self.istest(inspect.ismemberdescriptor, 'datetime.timedelta.days') + else: + self.assertFalse(inspect.ismemberdescriptor(datetime.timedelta.days)) + self.istest(inspect.ismethodwrapper, "object().__str__") + self.istest(inspect.ismethodwrapper, "object().__eq__") + self.istest(inspect.ismethodwrapper, "object().__repr__") + self.assertFalse(inspect.ismethodwrapper(type)) + self.assertFalse(inspect.ismethodwrapper(int)) + self.assertFalse(inspect.ismethodwrapper(type("AnyClass", (), {}))) + + + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_iscoroutine(self): + async_gen_coro = async_generator_function_example(1) + gen_coro = gen_coroutine_function_example(1) + coro = coroutine_function_example(1) + + class PMClass: + async_generator_partialmethod_example = functools.partialmethod( + async_generator_function_example) + coroutine_partialmethod_example = functools.partialmethod( + coroutine_function_example) + gen_coroutine_partialmethod_example = functools.partialmethod( + gen_coroutine_function_example) + + # partialmethods on the class, bound to an instance + pm_instance = PMClass() + async_gen_coro_pmi = pm_instance.async_generator_partialmethod_example + gen_coro_pmi = pm_instance.gen_coroutine_partialmethod_example + coro_pmi = pm_instance.coroutine_partialmethod_example + + # partialmethods on the class, unbound but accessed via the class + async_gen_coro_pmc = PMClass.async_generator_partialmethod_example + gen_coro_pmc = PMClass.gen_coroutine_partialmethod_example + coro_pmc = PMClass.coroutine_partialmethod_example + + self.assertFalse( + inspect.iscoroutinefunction(gen_coroutine_function_example)) + self.assertFalse( + inspect.iscoroutinefunction( + functools.partial(functools.partial( + gen_coroutine_function_example)))) + self.assertFalse(inspect.iscoroutinefunction(gen_coro_pmi)) + self.assertFalse(inspect.iscoroutinefunction(gen_coro_pmc)) + self.assertFalse(inspect.iscoroutinefunction(inspect)) + self.assertFalse(inspect.iscoroutine(gen_coro)) + + self.assertTrue( + inspect.isgeneratorfunction(gen_coroutine_function_example)) + self.assertTrue( + inspect.isgeneratorfunction( + functools.partial(functools.partial( + gen_coroutine_function_example)))) + self.assertTrue(inspect.isgeneratorfunction(gen_coro_pmi)) + self.assertTrue(inspect.isgeneratorfunction(gen_coro_pmc)) + self.assertTrue(inspect.isgenerator(gen_coro)) + + async def _fn3(): + pass + + @inspect.markcoroutinefunction + def fn3(): + return _fn3() + + self.assertTrue(inspect.iscoroutinefunction(fn3)) + self.assertTrue( + inspect.iscoroutinefunction( + inspect.markcoroutinefunction(lambda: _fn3()) + ) + ) + + class Cl: + async def __call__(self): + pass + + self.assertFalse(inspect.iscoroutinefunction(Cl)) + # instances with async def __call__ are NOT recognised. + self.assertFalse(inspect.iscoroutinefunction(Cl())) + # unless explicitly marked. + self.assertTrue(inspect.iscoroutinefunction( + inspect.markcoroutinefunction(Cl()) + )) + + class Cl2: + @inspect.markcoroutinefunction + def __call__(self): + pass + + self.assertFalse(inspect.iscoroutinefunction(Cl2)) + # instances with marked __call__ are NOT recognised. + self.assertFalse(inspect.iscoroutinefunction(Cl2())) + # unless explicitly marked. + self.assertTrue(inspect.iscoroutinefunction( + inspect.markcoroutinefunction(Cl2()) + )) + + class Cl3: + @inspect.markcoroutinefunction + @classmethod + def do_something_classy(cls): + pass + + @inspect.markcoroutinefunction + @staticmethod + def do_something_static(): + pass + + self.assertTrue(inspect.iscoroutinefunction(Cl3.do_something_classy)) + self.assertTrue(inspect.iscoroutinefunction(Cl3.do_something_static)) + + self.assertFalse( + inspect.iscoroutinefunction(unittest.mock.Mock())) + self.assertTrue( + inspect.iscoroutinefunction(unittest.mock.AsyncMock())) + self.assertTrue( + inspect.iscoroutinefunction(coroutine_function_example)) + self.assertTrue( + inspect.iscoroutinefunction( + functools.partial(functools.partial( + coroutine_function_example)))) + self.assertTrue(inspect.iscoroutinefunction(coro_pmi)) + self.assertTrue(inspect.iscoroutinefunction(coro_pmc)) + self.assertTrue(inspect.iscoroutine(coro)) + + self.assertFalse( + inspect.isgeneratorfunction(unittest.mock.Mock())) + self.assertFalse( + inspect.isgeneratorfunction(unittest.mock.AsyncMock())) + self.assertFalse( + inspect.isgeneratorfunction(coroutine_function_example)) + self.assertFalse( + inspect.isgeneratorfunction( + functools.partial(functools.partial( + coroutine_function_example)))) + self.assertFalse(inspect.isgeneratorfunction(coro_pmi)) + self.assertFalse(inspect.isgeneratorfunction(coro_pmc)) + self.assertFalse(inspect.isgenerator(coro)) + + self.assertFalse( + inspect.isasyncgenfunction(unittest.mock.Mock())) + self.assertFalse( + inspect.isasyncgenfunction(unittest.mock.AsyncMock())) + self.assertFalse( + inspect.isasyncgenfunction(coroutine_function_example)) + self.assertTrue( + inspect.isasyncgenfunction(async_generator_function_example)) + self.assertTrue( + inspect.isasyncgenfunction( + functools.partial(functools.partial( + async_generator_function_example)))) + self.assertTrue(inspect.isasyncgenfunction(async_gen_coro_pmi)) + self.assertTrue(inspect.isasyncgenfunction(async_gen_coro_pmc)) + self.assertTrue(inspect.isasyncgen(async_gen_coro)) + + coro.close(); gen_coro.close(); # silence warnings + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_isawaitable(self): + def gen(): yield + self.assertFalse(inspect.isawaitable(gen())) + + coro = coroutine_function_example(1) + gen_coro = gen_coroutine_function_example(1) + + self.assertTrue(inspect.isawaitable(coro)) + self.assertTrue(inspect.isawaitable(gen_coro)) + + class Future: + def __await__(): + pass + self.assertTrue(inspect.isawaitable(Future())) + self.assertFalse(inspect.isawaitable(Future)) + + class NotFuture: pass + not_fut = NotFuture() + not_fut.__await__ = lambda: None + self.assertFalse(inspect.isawaitable(not_fut)) + + coro.close(); gen_coro.close() # silence warnings + + def test_isroutine(self): + # method + self.assertTrue(inspect.isroutine(git.argue)) + self.assertTrue(inspect.isroutine(mod.custom_method)) + self.assertTrue(inspect.isroutine([].count)) + # function + self.assertTrue(inspect.isroutine(mod.spam)) + self.assertTrue(inspect.isroutine(mod.StupidGit.abuse)) + # slot-wrapper + self.assertTrue(inspect.isroutine(object.__init__)) + self.assertTrue(inspect.isroutine(object.__str__)) + self.assertTrue(inspect.isroutine(object.__lt__)) + self.assertTrue(inspect.isroutine(int.__lt__)) + # method-wrapper + self.assertTrue(inspect.isroutine(object().__init__)) + self.assertTrue(inspect.isroutine(object().__str__)) + self.assertTrue(inspect.isroutine(object().__lt__)) + self.assertTrue(inspect.isroutine((42).__lt__)) + # method-descriptor + self.assertTrue(inspect.isroutine(str.join)) + self.assertTrue(inspect.isroutine(list.append)) + self.assertTrue(inspect.isroutine(''.join)) + self.assertTrue(inspect.isroutine([].append)) + # object + self.assertFalse(inspect.isroutine(object)) + self.assertFalse(inspect.isroutine(object())) + self.assertFalse(inspect.isroutine(str())) + # module + self.assertFalse(inspect.isroutine(mod)) + # type + self.assertFalse(inspect.isroutine(type)) + self.assertFalse(inspect.isroutine(int)) + self.assertFalse(inspect.isroutine(type('some_class', (), {}))) + # partial + self.assertFalse(inspect.isroutine(functools.partial(mod.spam))) + + def test_isroutine_singledispatch(self): + self.assertTrue(inspect.isroutine(functools.singledispatch(mod.spam))) + + class A: + @functools.singledispatchmethod + def method(self, arg): + pass + @functools.singledispatchmethod + @classmethod + def class_method(cls, arg): + pass + @functools.singledispatchmethod + @staticmethod + def static_method(arg): + pass + + self.assertTrue(inspect.isroutine(A.method)) + self.assertTrue(inspect.isroutine(A().method)) + self.assertTrue(inspect.isroutine(A.static_method)) + self.assertTrue(inspect.isroutine(A.class_method)) + + def test_isclass(self): + self.istest(inspect.isclass, 'mod.StupidGit') + self.assertTrue(inspect.isclass(list)) + + class CustomGetattr(object): + def __getattr__(self, attr): + return None + self.assertFalse(inspect.isclass(CustomGetattr())) + + def test_get_slot_members(self): + class C(object): + __slots__ = ("a", "b") + x = C() + x.a = 42 + members = dict(inspect.getmembers(x)) + self.assertIn('a', members) + self.assertNotIn('b', members) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_isabstract(self): + from abc import ABCMeta, abstractmethod + + class AbstractClassExample(metaclass=ABCMeta): + + @abstractmethod + def foo(self): + pass + + class ClassExample(AbstractClassExample): + def foo(self): + pass + + a = ClassExample() + + # Test general behaviour. + self.assertTrue(inspect.isabstract(AbstractClassExample)) + self.assertFalse(inspect.isabstract(ClassExample)) + self.assertFalse(inspect.isabstract(a)) + self.assertFalse(inspect.isabstract(int)) + self.assertFalse(inspect.isabstract(5)) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_isabstract_during_init_subclass(self): + from abc import ABCMeta, abstractmethod + isabstract_checks = [] + class AbstractChecker(metaclass=ABCMeta): + def __init_subclass__(cls): + isabstract_checks.append(inspect.isabstract(cls)) + class AbstractClassExample(AbstractChecker): + @abstractmethod + def foo(self): + pass + class ClassExample(AbstractClassExample): + def foo(self): + pass + self.assertEqual(isabstract_checks, [True, False]) + + isabstract_checks.clear() + class AbstractChild(AbstractClassExample): + pass + class AbstractGrandchild(AbstractChild): + pass + class ConcreteGrandchild(ClassExample): + pass + self.assertEqual(isabstract_checks, [True, True, False]) + + +class TestInterpreterStack(IsTestBase): + def __init__(self, *args, **kwargs): + unittest.TestCase.__init__(self, *args, **kwargs) + + git.abuse(7, 8, 9) + + def test_abuse_done(self): + self.istest(inspect.istraceback, 'git.ex.__traceback__') + self.istest(inspect.isframe, 'mod.fr') + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_stack(self): + self.assertTrue(len(mod.st) >= 5) + frame1, frame2, frame3, frame4, *_ = mod.st + frameinfo = revise(*frame1[1:]) + self.assertEqual(frameinfo, + (modfile, 16, 'eggs', [' st = inspect.stack()\n'], 0)) + self.assertEqual(frame1.positions, dis.Positions(16, 16, 9, 24)) + frameinfo = revise(*frame2[1:]) + self.assertEqual(frameinfo, + (modfile, 9, 'spam', [' eggs(b + d, c + f)\n'], 0)) + self.assertEqual(frame2.positions, dis.Positions(9, 9, 4, 22)) + frameinfo = revise(*frame3[1:]) + self.assertEqual(frameinfo, + (modfile, 43, 'argue', [' spam(a, b, c)\n'], 0)) + self.assertEqual(frame3.positions, dis.Positions(43, 43, 12, 25)) + frameinfo = revise(*frame4[1:]) + self.assertEqual(frameinfo, + (modfile, 39, 'abuse', [' self.argue(a, b, c)\n'], 0)) + self.assertEqual(frame4.positions, dis.Positions(39, 39, 8, 27)) + # Test named tuple fields + record = mod.st[0] + self.assertIs(record.frame, mod.fr) + self.assertEqual(record.lineno, 16) + self.assertEqual(record.filename, mod.__file__) + self.assertEqual(record.function, 'eggs') + self.assertIn('inspect.stack()', record.code_context[0]) + self.assertEqual(record.index, 0) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_trace(self): + self.assertEqual(len(git.tr), 3) + frame1, frame2, frame3, = git.tr + self.assertEqual(revise(*frame1[1:]), + (modfile, 43, 'argue', [' spam(a, b, c)\n'], 0)) + self.assertEqual(frame1.positions, dis.Positions(43, 43, 12, 25)) + self.assertEqual(revise(*frame2[1:]), + (modfile, 9, 'spam', [' eggs(b + d, c + f)\n'], 0)) + self.assertEqual(frame2.positions, dis.Positions(9, 9, 4, 22)) + self.assertEqual(revise(*frame3[1:]), + (modfile, 18, 'eggs', [' q = y / 0\n'], 0)) + self.assertEqual(frame3.positions, dis.Positions(18, 18, 8, 13)) + + def test_frame(self): + args, varargs, varkw, locals = inspect.getargvalues(mod.fr) + self.assertEqual(args, ['x', 'y']) + self.assertEqual(varargs, None) + self.assertEqual(varkw, None) + self.assertEqual(locals, {'x': 11, 'p': 11, 'y': 14}) + self.assertEqual(inspect.formatargvalues(args, varargs, varkw, locals), + '(x=11, y=14)') + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_previous_frame(self): + args, varargs, varkw, locals = inspect.getargvalues(mod.fr.f_back) + self.assertEqual(args, ['a', 'b', 'c', 'd', 'e', 'f']) + self.assertEqual(varargs, 'g') + self.assertEqual(varkw, 'h') + self.assertEqual(inspect.formatargvalues(args, varargs, varkw, locals), + '(a=7, b=8, c=9, d=3, e=4, f=5, *g=(), **h={})') + +class GetSourceBase(unittest.TestCase): + # Subclasses must override. + fodderModule = None + + def setUp(self): + with open(inspect.getsourcefile(self.fodderModule), encoding="utf-8") as fp: + self.source = fp.read() + + def sourcerange(self, top, bottom): + lines = self.source.split("\n") + return "\n".join(lines[top-1:bottom]) + ("\n" if bottom else "") + + def assertSourceEqual(self, obj, top, bottom): + self.assertEqual(inspect.getsource(obj), + self.sourcerange(top, bottom)) + +class SlotUser: + 'Docstrings for __slots__' + __slots__ = {'power': 'measured in kilowatts', + 'distance': 'measured in kilometers'} + +class TestRetrievingSourceCode(GetSourceBase): + fodderModule = mod + + def test_getclasses(self): + classes = inspect.getmembers(mod, inspect.isclass) + self.assertEqual(classes, + [('FesteringGob', mod.FesteringGob), + ('MalodorousPervert', mod.MalodorousPervert), + ('ParrotDroppings', mod.ParrotDroppings), + ('StupidGit', mod.StupidGit), + ('Tit', mod.MalodorousPervert), + ('WhichComments', mod.WhichComments), + ]) + tree = inspect.getclasstree([cls[1] for cls in classes]) + self.assertEqual(tree, + [(object, ()), + [(mod.ParrotDroppings, (object,)), + [(mod.FesteringGob, (mod.MalodorousPervert, + mod.ParrotDroppings)) + ], + (mod.StupidGit, (object,)), + [(mod.MalodorousPervert, (mod.StupidGit,)), + [(mod.FesteringGob, (mod.MalodorousPervert, + mod.ParrotDroppings)) + ] + ], + (mod.WhichComments, (object,),) + ] + ]) + tree = inspect.getclasstree([cls[1] for cls in classes], True) + self.assertEqual(tree, + [(object, ()), + [(mod.ParrotDroppings, (object,)), + (mod.StupidGit, (object,)), + [(mod.MalodorousPervert, (mod.StupidGit,)), + [(mod.FesteringGob, (mod.MalodorousPervert, + mod.ParrotDroppings)) + ] + ], + (mod.WhichComments, (object,),) + ] + ]) + + def test_getfunctions(self): + functions = inspect.getmembers(mod, inspect.isfunction) + self.assertEqual(functions, [('after_closing', mod.after_closing), + ('eggs', mod.eggs), + ('lobbest', mod.lobbest), + ('spam', mod.spam)]) + + @unittest.skipIf(sys.flags.optimize >= 2, + "Docstrings are omitted with -O2 and above") + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_getdoc(self): + self.assertEqual(inspect.getdoc(mod), 'A module docstring.') + self.assertEqual(inspect.getdoc(mod.StupidGit), + 'A longer,\n\nindented\n\ndocstring.') + self.assertEqual(inspect.getdoc(git.abuse), + 'Another\n\ndocstring\n\ncontaining\n\ntabs') + self.assertEqual(inspect.getdoc(SlotUser.power), + 'measured in kilowatts') + self.assertEqual(inspect.getdoc(SlotUser.distance), + 'measured in kilometers') + + @unittest.skipIf(sys.flags.optimize >= 2, + "Docstrings are omitted with -O2 and above") + def test_getdoc_inherited(self): + self.assertEqual(inspect.getdoc(mod.FesteringGob), + 'A longer,\n\nindented\n\ndocstring.') + self.assertEqual(inspect.getdoc(mod.FesteringGob.abuse), + 'Another\n\ndocstring\n\ncontaining\n\ntabs') + self.assertEqual(inspect.getdoc(mod.FesteringGob().abuse), + 'Another\n\ndocstring\n\ncontaining\n\ntabs') + self.assertEqual(inspect.getdoc(mod.FesteringGob.contradiction), + 'The automatic gainsaying.') + + @unittest.skipIf(MISSING_C_DOCSTRINGS, "test requires docstrings") + def test_finddoc(self): + finddoc = inspect._finddoc + self.assertEqual(finddoc(int), int.__doc__) + self.assertEqual(finddoc(int.to_bytes), int.to_bytes.__doc__) + self.assertEqual(finddoc(int().to_bytes), int.to_bytes.__doc__) + self.assertEqual(finddoc(int.from_bytes), int.from_bytes.__doc__) + self.assertEqual(finddoc(int.real), int.real.__doc__) + + cleandoc_testdata = [ + # first line should have different margin + (' An\n indented\n docstring.', 'An\nindented\n docstring.'), + # trailing whitespace are not removed. + (' An \n \n indented \n docstring. ', + 'An \n \nindented \n docstring. '), + # NUL is not termination. + ('doc\0string\n\n second\0line\n third\0line\0', + 'doc\0string\n\nsecond\0line\nthird\0line\0'), + # first line is lstrip()-ped. other lines are kept when no margin.[w: + (' ', ''), + # compiler.cleandoc() doesn't strip leading/trailing newlines + # to keep maximum backward compatibility. + # inspect.cleandoc() removes them. + ('\n\n\n first paragraph\n\n second paragraph\n\n', + '\n\n\nfirst paragraph\n\n second paragraph\n\n'), + (' \n \n \n ', '\n \n \n '), + ] + + def test_cleandoc(self): + func = inspect.cleandoc + for i, (input, expected) in enumerate(self.cleandoc_testdata): + # only inspect.cleandoc() strip \n + expected = expected.strip('\n') + with self.subTest(i=i): + self.assertEqual(func(input), expected) + + @cpython_only + def test_c_cleandoc(self): + try: + import _testinternalcapi + except ImportError: + return unittest.skip("requires _testinternalcapi") + func = _testinternalcapi.compiler_cleandoc + for i, (input, expected) in enumerate(self.cleandoc_testdata): + with self.subTest(i=i): + self.assertEqual(func(input), expected) + + def test_getcomments(self): + self.assertEqual(inspect.getcomments(mod), '# line 1\n') + self.assertEqual(inspect.getcomments(mod.StupidGit), '# line 20\n') + self.assertEqual(inspect.getcomments(mod2.cls160), '# line 159\n') + # If the object source file is not available, return None. + co = compile('x=1', '_non_existing_filename.py', 'exec') + self.assertIsNone(inspect.getcomments(co)) + # If the object has been defined in C, return None. + self.assertIsNone(inspect.getcomments(list)) + + def test_getmodule(self): + # Check actual module + self.assertEqual(inspect.getmodule(mod), mod) + # Check class (uses __module__ attribute) + self.assertEqual(inspect.getmodule(mod.StupidGit), mod) + # Check a method (no __module__ attribute, falls back to filename) + self.assertEqual(inspect.getmodule(mod.StupidGit.abuse), mod) + # Do it again (check the caching isn't broken) + self.assertEqual(inspect.getmodule(mod.StupidGit.abuse), mod) + # Check a builtin + self.assertEqual(inspect.getmodule(str), sys.modules["builtins"]) + # Check filename override + self.assertEqual(inspect.getmodule(None, modfile), mod) + + def test_getmodule_file_not_found(self): + # See bpo-45406 + def _getabsfile(obj, _filename): + raise FileNotFoundError('bad file') + with unittest.mock.patch('inspect.getabsfile', _getabsfile): + f = inspect.currentframe() + self.assertIsNone(inspect.getmodule(f)) + inspect.getouterframes(f) # smoke test + + def test_getframeinfo_get_first_line(self): + frame_info = inspect.getframeinfo(self.fodderModule.fr, 50) + self.assertEqual(frame_info.code_context[0], "# line 1\n") + self.assertEqual(frame_info.code_context[1], "'A module docstring.'\n") + + def test_getsource(self): + self.assertSourceEqual(git.abuse, 29, 39) + self.assertSourceEqual(mod.StupidGit, 21, 51) + self.assertSourceEqual(mod.lobbest, 75, 76) + self.assertSourceEqual(mod.after_closing, 120, 120) + + def test_getsourcefile(self): + self.assertEqual(normcase(inspect.getsourcefile(mod.spam)), modfile) + self.assertEqual(normcase(inspect.getsourcefile(git.abuse)), modfile) + fn = "_non_existing_filename_used_for_sourcefile_test.py" + co = compile("x=1", fn, "exec") + self.assertEqual(inspect.getsourcefile(co), None) + linecache.cache[co.co_filename] = (1, None, "None", co.co_filename) + try: + self.assertEqual(normcase(inspect.getsourcefile(co)), fn) + finally: + del linecache.cache[co.co_filename] + + def test_getsource_empty_file(self): + with temp_cwd() as cwd: + with open('empty_file.py', 'w'): + pass + sys.path.insert(0, cwd) + try: + import empty_file + self.assertEqual(inspect.getsource(empty_file), '\n') + self.assertEqual(inspect.getsourcelines(empty_file), (['\n'], 0)) + finally: + sys.path.remove(cwd) + + def test_getfile(self): + self.assertEqual(inspect.getfile(mod.StupidGit), mod.__file__) + + def test_getfile_builtin_module(self): + with self.assertRaises(TypeError) as e: + inspect.getfile(sys) + self.assertTrue(str(e.exception).startswith('<module')) + + def test_getfile_builtin_class(self): + with self.assertRaises(TypeError) as e: + inspect.getfile(int) + self.assertTrue(str(e.exception).startswith('<class')) + + def test_getfile_builtin_function_or_method(self): + with self.assertRaises(TypeError) as e_abs: + inspect.getfile(abs) + self.assertIn('expected, got', str(e_abs.exception)) + with self.assertRaises(TypeError) as e_append: + inspect.getfile(list.append) + self.assertIn('expected, got', str(e_append.exception)) + + @suppress_immortalization() + def test_getfile_class_without_module(self): + class CM(type): + @property + def __module__(cls): + raise AttributeError + class C(metaclass=CM): + pass + with self.assertRaises(TypeError): + inspect.getfile(C) + + def test_getfile_broken_repr(self): + class ErrorRepr: + def __repr__(self): + raise Exception('xyz') + er = ErrorRepr() + with self.assertRaises(TypeError): + inspect.getfile(er) + + def test_getmodule_recursion(self): + from types import ModuleType + name = '__inspect_dummy' + m = sys.modules[name] = ModuleType(name) + m.__file__ = "<string>" # hopefully not a real filename... + m.__loader__ = "dummy" # pretend the filename is understood by a loader + exec("def x(): pass", m.__dict__) + self.assertEqual(inspect.getsourcefile(m.x.__code__), '<string>') + del sys.modules[name] + inspect.getmodule(compile('a=10','','single')) + + def test_proceed_with_fake_filename(self): + '''doctest monkeypatches linecache to enable inspection''' + fn, source = '<test>', 'def x(): pass\n' + getlines = linecache.getlines + def monkey(filename, module_globals=None): + if filename == fn: + return source.splitlines(keepends=True) + else: + return getlines(filename, module_globals) + linecache.getlines = monkey + try: + ns = {} + exec(compile(source, fn, 'single'), ns) + inspect.getsource(ns["x"]) + finally: + linecache.getlines = getlines + + def test_getsource_on_code_object(self): + self.assertSourceEqual(mod.eggs.__code__, 12, 18) + + def test_getsource_on_generated_class(self): + A = type('A', (unittest.TestCase,), {}) + self.assertEqual(inspect.getsourcefile(A), __file__) + self.assertEqual(inspect.getfile(A), __file__) + self.assertIs(inspect.getmodule(A), sys.modules[__name__]) + self.assertRaises(OSError, inspect.getsource, A) + self.assertRaises(OSError, inspect.getsourcelines, A) + self.assertIsNone(inspect.getcomments(A)) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_getsource_on_class_without_firstlineno(self): + __firstlineno__ = 1 + class C: + nonlocal __firstlineno__ + self.assertRaises(OSError, inspect.getsource, C) + +class TestGetsourceStdlib(unittest.TestCase): + # Test Python implementations of the stdlib modules + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_getsource_stdlib_collections_abc(self): + import collections.abc + lines, lineno = inspect.getsourcelines(collections.abc.Sequence) + self.assertEqual(lines[0], 'class Sequence(Reversible, Collection):\n') + src = inspect.getsource(collections.abc.Sequence) + self.assertEqual(src.splitlines(True), lines) + + def test_getsource_stdlib_tomllib(self): + import tomllib + self.assertRaises(OSError, inspect.getsource, tomllib.TOMLDecodeError) + self.assertRaises(OSError, inspect.getsourcelines, tomllib.TOMLDecodeError) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_getsource_stdlib_abc(self): + # Pure Python implementation + abc = import_helper.import_fresh_module('abc', blocked=['_abc']) + with support.swap_item(sys.modules, 'abc', abc): + self.assertRaises(OSError, inspect.getsource, abc.ABCMeta) + self.assertRaises(OSError, inspect.getsourcelines, abc.ABCMeta) + # With C acceleration + import abc + try: + src = inspect.getsource(abc.ABCMeta) + lines, lineno = inspect.getsourcelines(abc.ABCMeta) + except OSError: + pass + else: + self.assertEqual(lines[0], ' class ABCMeta(type):\n') + self.assertEqual(src.splitlines(True), lines) + + def test_getsource_stdlib_decimal(self): + # Pure Python implementation + decimal = import_helper.import_fresh_module('decimal', blocked=['_decimal']) + with support.swap_item(sys.modules, 'decimal', decimal): + src = inspect.getsource(decimal.Decimal) + lines, lineno = inspect.getsourcelines(decimal.Decimal) + self.assertEqual(lines[0], 'class Decimal(object):\n') + self.assertEqual(src.splitlines(True), lines) + +class TestGetsourceInteractive(unittest.TestCase): + @support.force_not_colorized + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_getclasses_interactive(self): + # bpo-44648: simulate a REPL session; + # there is no `__file__` in the __main__ module + code = "import sys, inspect; \ + assert not hasattr(sys.modules['__main__'], '__file__'); \ + A = type('A', (), {}); \ + inspect.getsource(A)" + _, _, stderr = assert_python_failure("-c", code, __isolated=True) + self.assertIn(b'OSError: source code not available', stderr) + +class TestGettingSourceOfToplevelFrames(GetSourceBase): + fodderModule = mod + + def test_range_toplevel_frame(self): + self.maxDiff = None + self.assertSourceEqual(mod.currentframe, 1, None) + + def test_range_traceback_toplevel_frame(self): + self.assertSourceEqual(mod.tb, 1, None) + +class TestDecorators(GetSourceBase): + fodderModule = mod2 + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_wrapped_decorator(self): + self.assertSourceEqual(mod2.wrapped, 14, 17) + + def test_replacing_decorator(self): + self.assertSourceEqual(mod2.gone, 9, 10) + + def test_getsource_unwrap(self): + self.assertSourceEqual(mod2.real, 130, 132) + + def test_decorator_with_lambda(self): + self.assertSourceEqual(mod2.func114, 113, 115) + +class TestOneliners(GetSourceBase): + fodderModule = mod2 + def test_oneline_lambda(self): + # Test inspect.getsource with a one-line lambda function. + self.assertSourceEqual(mod2.oll, 25, 25) + + def test_threeline_lambda(self): + # Test inspect.getsource with a three-line lambda function, + # where the second and third lines are _not_ indented. + self.assertSourceEqual(mod2.tll, 28, 30) + + def test_twoline_indented_lambda(self): + # Test inspect.getsource with a two-line lambda function, + # where the second line _is_ indented. + self.assertSourceEqual(mod2.tlli, 33, 34) + + def test_parenthesized_multiline_lambda(self): + # Test inspect.getsource with a parenthesized multi-line lambda + # function. + self.assertSourceEqual(mod2.parenthesized_lambda, 279, 279) + self.assertSourceEqual(mod2.parenthesized_lambda2, 281, 281) + self.assertSourceEqual(mod2.parenthesized_lambda3, 283, 283) + + def test_post_line_parenthesized_lambda(self): + # Test inspect.getsource with a parenthesized multi-line lambda + # function. + self.assertSourceEqual(mod2.post_line_parenthesized_lambda1, 286, 287) + + def test_nested_lambda(self): + # Test inspect.getsource with a nested lambda function. + self.assertSourceEqual(mod2.nested_lambda, 291, 292) + + def test_onelinefunc(self): + # Test inspect.getsource with a regular one-line function. + self.assertSourceEqual(mod2.onelinefunc, 37, 37) + + def test_manyargs(self): + # Test inspect.getsource with a regular function where + # the arguments are on two lines and _not_ indented and + # the body on the second line with the last arguments. + self.assertSourceEqual(mod2.manyargs, 40, 41) + + def test_twolinefunc(self): + # Test inspect.getsource with a regular function where + # the body is on two lines, following the argument list and + # continued on the next line by a \\. + self.assertSourceEqual(mod2.twolinefunc, 44, 45) + + def test_lambda_in_list(self): + # Test inspect.getsource with a one-line lambda function + # defined in a list, indented. + self.assertSourceEqual(mod2.a[1], 49, 49) + + def test_anonymous(self): + # Test inspect.getsource with a lambda function defined + # as argument to another function. + self.assertSourceEqual(mod2.anonymous, 55, 55) + + def test_enum(self): + self.assertSourceEqual(mod2.enum322, 322, 323) + self.assertSourceEqual(mod2.enum326, 326, 327) + self.assertSourceEqual(mod2.flag330, 330, 331) + self.assertSourceEqual(mod2.flag334, 334, 335) + self.assertRaises(OSError, inspect.getsource, mod2.simple_enum338) + self.assertRaises(OSError, inspect.getsource, mod2.simple_enum339) + self.assertRaises(OSError, inspect.getsource, mod2.simple_flag340) + self.assertRaises(OSError, inspect.getsource, mod2.simple_flag341) + + def test_namedtuple(self): + self.assertSourceEqual(mod2.nt346, 346, 348) + self.assertRaises(OSError, inspect.getsource, mod2.nt351) + + def test_typeddict(self): + self.assertSourceEqual(mod2.td354, 354, 356) + self.assertRaises(OSError, inspect.getsource, mod2.td359) + + def test_dataclass(self): + self.assertSourceEqual(mod2.dc364, 364, 367) + self.assertRaises(OSError, inspect.getsource, mod2.dc370) + self.assertRaises(OSError, inspect.getsource, mod2.dc371) + +class TestBlockComments(GetSourceBase): + fodderModule = mod + + def test_toplevel_class(self): + self.assertSourceEqual(mod.WhichComments, 96, 114) + + def test_class_method(self): + self.assertSourceEqual(mod.WhichComments.f, 99, 104) + + def test_class_async_method(self): + self.assertSourceEqual(mod.WhichComments.asyncf, 109, 112) + +class TestBuggyCases(GetSourceBase): + fodderModule = mod2 + + def test_with_comment(self): + self.assertSourceEqual(mod2.with_comment, 58, 59) + + def test_multiline_sig(self): + self.assertSourceEqual(mod2.multiline_sig[0], 63, 64) + + def test_nested_class(self): + self.assertSourceEqual(mod2.func69().func71, 71, 72) + + def test_one_liner_followed_by_non_name(self): + self.assertSourceEqual(mod2.func77, 77, 77) + + def test_one_liner_dedent_non_name(self): + self.assertSourceEqual(mod2.cls82.func83, 83, 83) + + def test_with_comment_instead_of_docstring(self): + self.assertSourceEqual(mod2.func88, 88, 90) + + def test_method_in_dynamic_class(self): + self.assertSourceEqual(mod2.method_in_dynamic_class, 95, 97) + + # This should not skip for CPython, but might on a repackaged python where + # unicodedata is not an external module, or on pypy. + @unittest.skipIf(not hasattr(unicodedata, '__file__') or + unicodedata.__file__.endswith('.py'), + "unicodedata is not an external binary module") + def test_findsource_binary(self): + self.assertRaises(OSError, inspect.getsource, unicodedata) + self.assertRaises(OSError, inspect.findsource, unicodedata) + + def test_findsource_code_in_linecache(self): + lines = ["x=1"] + co = compile(lines[0], "_dynamically_created_file", "exec") + self.assertRaises(OSError, inspect.findsource, co) + self.assertRaises(OSError, inspect.getsource, co) + linecache.cache[co.co_filename] = (1, None, lines, co.co_filename) + try: + self.assertEqual(inspect.findsource(co), (lines,0)) + self.assertEqual(inspect.getsource(co), lines[0]) + finally: + del linecache.cache[co.co_filename] + + def test_findsource_without_filename(self): + for fname in ['', '<string>']: + co = compile('x=1', fname, "exec") + self.assertRaises(IOError, inspect.findsource, co) + self.assertRaises(IOError, inspect.getsource, co) + + def test_findsource_on_func_with_out_of_bounds_lineno(self): + mod_len = len(inspect.getsource(mod)) + src = '\n' * 2* mod_len + "def f(): pass" + co = compile(src, mod.__file__, "exec") + g, l = {}, {} + eval(co, g, l) + func = l['f'] + self.assertEqual(func.__code__.co_firstlineno, 1+2*mod_len) + with self.assertRaisesRegex(OSError, "lineno is out of bounds"): + inspect.findsource(func) + + def test_findsource_on_class_with_out_of_bounds_lineno(self): + mod_len = len(inspect.getsource(mod)) + src = '\n' * 2* mod_len + "class A: pass" + co = compile(src, mod.__file__, "exec") + g, l = {'__name__': mod.__name__}, {} + eval(co, g, l) + cls = l['A'] + self.assertEqual(cls.__firstlineno__, 1+2*mod_len) + with self.assertRaisesRegex(OSError, "lineno is out of bounds"): + inspect.findsource(cls) + + def test_getsource_on_method(self): + self.assertSourceEqual(mod2.ClassWithMethod.method, 118, 119) + + def test_getsource_on_class_code_object(self): + self.assertSourceEqual(mod2.ClassWithCodeObject.code, 315, 317) + + def test_nested_func(self): + self.assertSourceEqual(mod2.cls135.func136, 136, 139) + + def test_class_definition_in_multiline_string_definition(self): + self.assertSourceEqual(mod2.cls149, 149, 152) + + def test_class_definition_in_multiline_comment(self): + self.assertSourceEqual(mod2.cls160, 160, 163) + + def test_nested_class_definition_indented_string(self): + self.assertSourceEqual(mod2.cls173.cls175, 175, 176) + + def test_nested_class_definition(self): + self.assertSourceEqual(mod2.cls183, 183, 188) + self.assertSourceEqual(mod2.cls183.cls185, 185, 188) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_class_decorator(self): + self.assertSourceEqual(mod2.cls196, 194, 201) + self.assertSourceEqual(mod2.cls196.cls200, 198, 201) + + @support.requires_docstrings + def test_class_inside_conditional(self): + # We skip this test when docstrings are not present, + # because docstrings are one of the main factors of + # finding the correct class in the source code. + self.assertSourceEqual(mod2.cls238.cls239, 239, 240) + + def test_multiple_children_classes(self): + self.assertSourceEqual(mod2.cls203, 203, 209) + self.assertSourceEqual(mod2.cls203.cls204, 204, 206) + self.assertSourceEqual(mod2.cls203.cls204.cls205, 205, 206) + self.assertSourceEqual(mod2.cls203.cls207, 207, 209) + self.assertSourceEqual(mod2.cls203.cls207.cls205, 208, 209) + + def test_nested_class_definition_inside_function(self): + self.assertSourceEqual(mod2.func212(), 213, 214) + self.assertSourceEqual(mod2.cls213, 218, 222) + self.assertSourceEqual(mod2.cls213().func219(), 220, 221) + + def test_class_with_method_from_other_module(self): + with tempfile.TemporaryDirectory() as tempdir: + with open(os.path.join(tempdir, 'inspect_actual%spy' % os.extsep), + 'w', encoding='utf-8') as f: + f.write(textwrap.dedent(""" + import inspect_other + class A: + def f(self): + pass + class A: + def f(self): + pass # correct one + A.f = inspect_other.A.f + """)) + + with open(os.path.join(tempdir, 'inspect_other%spy' % os.extsep), + 'w', encoding='utf-8') as f: + f.write(textwrap.dedent(""" + class A: + def f(self): + pass + """)) + + with DirsOnSysPath(tempdir): + import inspect_actual + self.assertIn("correct", inspect.getsource(inspect_actual.A)) + # Remove the module from sys.modules to force it to be reloaded. + # This is necessary when the test is run multiple times. + sys.modules.pop("inspect_actual") + + @unittest.skipIf( + support.is_emscripten or support.is_wasi, + "socket.accept is broken" + ) + def test_nested_class_definition_inside_async_function(self): + import asyncio + self.addCleanup(asyncio.set_event_loop_policy, None) + self.assertSourceEqual(asyncio.run(mod2.func225()), 226, 227) + self.assertSourceEqual(mod2.cls226, 231, 235) + self.assertSourceEqual(mod2.cls226.func232, 232, 235) + self.assertSourceEqual(asyncio.run(mod2.cls226().func232()), 233, 234) + + def test_class_definition_same_name_diff_methods(self): + self.assertSourceEqual(mod2.cls296, 296, 298) + self.assertSourceEqual(mod2.cls310, 310, 312) + + def test_generator_expression(self): + self.assertSourceEqual(next(mod2.ge377), 377, 380) + self.assertSourceEqual(next(mod2.func383()), 385, 388) + + def test_comment_or_empty_line_after_decorator(self): + self.assertSourceEqual(mod2.func394, 392, 395) + self.assertSourceEqual(mod2.func400, 398, 401) + + +class TestNoEOL(GetSourceBase): + def setUp(self): + self.tempdir = TESTFN + '_dir' + os.mkdir(self.tempdir) + with open(os.path.join(self.tempdir, 'inspect_fodder3%spy' % os.extsep), + 'w', encoding='utf-8') as f: + f.write("class X:\n pass # No EOL") + with DirsOnSysPath(self.tempdir): + import inspect_fodder3 as mod3 + self.fodderModule = mod3 + super().setUp() + + def tearDown(self): + shutil.rmtree(self.tempdir) + + def test_class(self): + self.assertSourceEqual(self.fodderModule.X, 1, 2) + + +class TestComplexDecorator(GetSourceBase): + fodderModule = mod2 + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_parens_in_decorator(self): + self.assertSourceEqual(self.fodderModule.complex_decorated, 273, 275) + +class _BrokenDataDescriptor(object): + """ + A broken data descriptor. See bug #1785. + """ + def __get__(*args): + raise AttributeError("broken data descriptor") + + def __set__(*args): + raise RuntimeError + + def __getattr__(*args): + raise AttributeError("broken data descriptor") + + +class _BrokenMethodDescriptor(object): + """ + A broken method descriptor. See bug #1785. + """ + def __get__(*args): + raise AttributeError("broken method descriptor") + + def __getattr__(*args): + raise AttributeError("broken method descriptor") + + +# Helper for testing classify_class_attrs. +def attrs_wo_objs(cls): + return [t[:3] for t in inspect.classify_class_attrs(cls)] + + +class TestClassesAndFunctions(unittest.TestCase): + def test_newstyle_mro(self): + # The same w/ new-class MRO. + class A(object): pass + class B(A): pass + class C(A): pass + class D(B, C): pass + + expected = (D, B, C, A, object) + got = inspect.getmro(D) + self.assertEqual(expected, got) + + def assertFullArgSpecEquals(self, routine, args_e, varargs_e=None, + varkw_e=None, defaults_e=None, + posonlyargs_e=[], kwonlyargs_e=[], + kwonlydefaults_e=None, + ann_e={}): + args, varargs, varkw, defaults, kwonlyargs, kwonlydefaults, ann = \ + inspect.getfullargspec(routine) + self.assertEqual(args, args_e) + self.assertEqual(varargs, varargs_e) + self.assertEqual(varkw, varkw_e) + self.assertEqual(defaults, defaults_e) + self.assertEqual(kwonlyargs, kwonlyargs_e) + self.assertEqual(kwonlydefaults, kwonlydefaults_e) + self.assertEqual(ann, ann_e) + + def test_getfullargspec(self): + self.assertFullArgSpecEquals(mod2.keyworded, [], varargs_e='arg1', + kwonlyargs_e=['arg2'], + kwonlydefaults_e={'arg2':1}) + + self.assertFullArgSpecEquals(mod2.annotated, ['arg1'], + ann_e={'arg1' : list}) + self.assertFullArgSpecEquals(mod2.keyword_only_arg, [], + kwonlyargs_e=['arg']) + + self.assertFullArgSpecEquals(mod2.all_markers, ['a', 'b', 'c', 'd'], + kwonlyargs_e=['e', 'f']) + + self.assertFullArgSpecEquals(mod2.all_markers_with_args_and_kwargs, + ['a', 'b', 'c', 'd'], + varargs_e='args', + varkw_e='kwargs', + kwonlyargs_e=['e', 'f']) + + self.assertFullArgSpecEquals(mod2.all_markers_with_defaults, ['a', 'b', 'c', 'd'], + defaults_e=(1,2,3), + kwonlyargs_e=['e', 'f'], + kwonlydefaults_e={'e': 4, 'f': 5}) + + def test_argspec_api_ignores_wrapped(self): + # Issue 20684: low level introspection API must ignore __wrapped__ + @functools.wraps(mod.spam) + def ham(x, y): + pass + # Basic check + self.assertFullArgSpecEquals(ham, ['x', 'y']) + self.assertFullArgSpecEquals(functools.partial(ham), + ['x', 'y']) + + def test_getfullargspec_signature_attr(self): + def test(): + pass + spam_param = inspect.Parameter('spam', inspect.Parameter.POSITIONAL_ONLY) + test.__signature__ = inspect.Signature(parameters=(spam_param,)) + + self.assertFullArgSpecEquals(test, ['spam']) + + def test_getfullargspec_signature_annos(self): + def test(a:'spam') -> 'ham': pass + spec = inspect.getfullargspec(test) + self.assertEqual(test.__annotations__, spec.annotations) + + def test(): pass + spec = inspect.getfullargspec(test) + self.assertEqual(test.__annotations__, spec.annotations) + + @unittest.skipIf(MISSING_C_DOCSTRINGS, + "Signature information for builtins requires docstrings") + @unittest.skipIf(_pickle is None, "requires _pickle") + def test_getfullargspec_builtin_methods(self): + self.assertFullArgSpecEquals(_pickle.Pickler.dump, ['self', 'obj']) + + self.assertFullArgSpecEquals(_pickle.Pickler(io.BytesIO()).dump, ['self', 'obj']) + + self.assertFullArgSpecEquals( + os.stat, + args_e=['path'], + kwonlyargs_e=['dir_fd', 'follow_symlinks'], + kwonlydefaults_e={'dir_fd': None, 'follow_symlinks': True}) + + @cpython_only + @unittest.skipIf(MISSING_C_DOCSTRINGS, + "Signature information for builtins requires docstrings") + def test_getfullargspec_builtin_func(self): + _testcapi = import_helper.import_module("_testcapi") + builtin = _testcapi.docstring_with_signature_with_defaults + spec = inspect.getfullargspec(builtin) + self.assertEqual(spec.defaults[0], 'avocado') + + @cpython_only + @unittest.skipIf(MISSING_C_DOCSTRINGS, + "Signature information for builtins requires docstrings") + def test_getfullargspec_builtin_func_no_signature(self): + _testcapi = import_helper.import_module("_testcapi") + builtin = _testcapi.docstring_no_signature + with self.assertRaises(TypeError): + inspect.getfullargspec(builtin) + + cls = _testcapi.DocStringNoSignatureTest + obj = _testcapi.DocStringNoSignatureTest() + tests = [ + (_testcapi.docstring_no_signature_noargs, meth_noargs), + (_testcapi.docstring_no_signature_o, meth_o), + (cls.meth_noargs, meth_self_noargs), + (cls.meth_o, meth_self_o), + (obj.meth_noargs, meth_self_noargs), + (obj.meth_o, meth_self_o), + (cls.meth_noargs_class, meth_type_noargs), + (cls.meth_o_class, meth_type_o), + (cls.meth_noargs_static, meth_noargs), + (cls.meth_o_static, meth_o), + (cls.meth_noargs_coexist, meth_self_noargs), + (cls.meth_o_coexist, meth_self_o), + + (time.time, meth_noargs), + (str.lower, meth_self_noargs), + (''.lower, meth_self_noargs), + (set.add, meth_self_o), + (set().add, meth_self_o), + (set.__contains__, meth_self_o), + (set().__contains__, meth_self_o), + (datetime.datetime.__dict__['utcnow'], meth_type_noargs), + (datetime.datetime.utcnow, meth_type_noargs), + (dict.__dict__['__class_getitem__'], meth_type_o), + (dict.__class_getitem__, meth_type_o), + ] + try: + import _stat + except ImportError: + # if the _stat extension is not available, stat.S_IMODE() is + # implemented in Python, not in C + pass + else: + tests.append((stat.S_IMODE, meth_o)) + for builtin, template in tests: + with self.subTest(builtin): + self.assertEqual(inspect.getfullargspec(builtin), + inspect.getfullargspec(template)) + + def test_getfullargspec_definition_order_preserved_on_kwonly(self): + for fn in signatures_with_lexicographic_keyword_only_parameters(): + signature = inspect.getfullargspec(fn) + l = list(signature.kwonlyargs) + sorted_l = sorted(l) + self.assertTrue(l) + self.assertEqual(l, sorted_l) + signature = inspect.getfullargspec(unsorted_keyword_only_parameters_fn) + l = list(signature.kwonlyargs) + self.assertEqual(l, unsorted_keyword_only_parameters) + + def test_classify_newstyle(self): + class A(object): + + def s(): pass + s = staticmethod(s) + + def c(cls): pass + c = classmethod(c) + + def getp(self): pass + p = property(getp) + + def m(self): pass + + def m1(self): pass + + datablob = '1' + + dd = _BrokenDataDescriptor() + md = _BrokenMethodDescriptor() + + attrs = attrs_wo_objs(A) + + self.assertIn(('__new__', 'static method', object), attrs, + 'missing __new__') + self.assertIn(('__init__', 'method', object), attrs, 'missing __init__') + + self.assertIn(('s', 'static method', A), attrs, 'missing static method') + self.assertIn(('c', 'class method', A), attrs, 'missing class method') + self.assertIn(('p', 'property', A), attrs, 'missing property') + self.assertIn(('m', 'method', A), attrs, + 'missing plain method: %r' % attrs) + self.assertIn(('m1', 'method', A), attrs, 'missing plain method') + self.assertIn(('datablob', 'data', A), attrs, 'missing data') + self.assertIn(('md', 'method', A), attrs, 'missing method descriptor') + self.assertIn(('dd', 'data', A), attrs, 'missing data descriptor') + + class B(A): + + def m(self): pass + + attrs = attrs_wo_objs(B) + self.assertIn(('s', 'static method', A), attrs, 'missing static method') + self.assertIn(('c', 'class method', A), attrs, 'missing class method') + self.assertIn(('p', 'property', A), attrs, 'missing property') + self.assertIn(('m', 'method', B), attrs, 'missing plain method') + self.assertIn(('m1', 'method', A), attrs, 'missing plain method') + self.assertIn(('datablob', 'data', A), attrs, 'missing data') + self.assertIn(('md', 'method', A), attrs, 'missing method descriptor') + self.assertIn(('dd', 'data', A), attrs, 'missing data descriptor') + + + class C(A): + + def m(self): pass + def c(self): pass + + attrs = attrs_wo_objs(C) + self.assertIn(('s', 'static method', A), attrs, 'missing static method') + self.assertIn(('c', 'method', C), attrs, 'missing plain method') + self.assertIn(('p', 'property', A), attrs, 'missing property') + self.assertIn(('m', 'method', C), attrs, 'missing plain method') + self.assertIn(('m1', 'method', A), attrs, 'missing plain method') + self.assertIn(('datablob', 'data', A), attrs, 'missing data') + self.assertIn(('md', 'method', A), attrs, 'missing method descriptor') + self.assertIn(('dd', 'data', A), attrs, 'missing data descriptor') + + class D(B, C): + + def m1(self): pass + + attrs = attrs_wo_objs(D) + self.assertIn(('s', 'static method', A), attrs, 'missing static method') + self.assertIn(('c', 'method', C), attrs, 'missing plain method') + self.assertIn(('p', 'property', A), attrs, 'missing property') + self.assertIn(('m', 'method', B), attrs, 'missing plain method') + self.assertIn(('m1', 'method', D), attrs, 'missing plain method') + self.assertIn(('datablob', 'data', A), attrs, 'missing data') + self.assertIn(('md', 'method', A), attrs, 'missing method descriptor') + self.assertIn(('dd', 'data', A), attrs, 'missing data descriptor') + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_classify_builtin_types(self): + # Simple sanity check that all built-in types can have their + # attributes classified. + for name in dir(__builtins__): + builtin = getattr(__builtins__, name) + if isinstance(builtin, type): + inspect.classify_class_attrs(builtin) + + attrs = attrs_wo_objs(bool) + self.assertIn(('__new__', 'static method', bool), attrs, + 'missing __new__') + self.assertIn(('from_bytes', 'class method', int), attrs, + 'missing class method') + self.assertIn(('to_bytes', 'method', int), attrs, + 'missing plain method') + self.assertIn(('__add__', 'method', int), attrs, + 'missing plain method') + self.assertIn(('__and__', 'method', bool), attrs, + 'missing plain method') + + def test_classify_DynamicClassAttribute(self): + class Meta(type): + def __getattr__(self, name): + if name == 'ham': + return 'spam' + return super().__getattr__(name) + class VA(metaclass=Meta): + @types.DynamicClassAttribute + def ham(self): + return 'eggs' + should_find_dca = inspect.Attribute('ham', 'data', VA, VA.__dict__['ham']) + self.assertIn(should_find_dca, inspect.classify_class_attrs(VA)) + should_find_ga = inspect.Attribute('ham', 'data', Meta, 'spam') + self.assertIn(should_find_ga, inspect.classify_class_attrs(VA)) + + def test_classify_overrides_bool(self): + class NoBool(object): + def __eq__(self, other): + return NoBool() + + def __bool__(self): + raise NotImplementedError( + "This object does not specify a boolean value") + + class HasNB(object): + dd = NoBool() + + should_find_attr = inspect.Attribute('dd', 'data', HasNB, HasNB.dd) + self.assertIn(should_find_attr, inspect.classify_class_attrs(HasNB)) + + def test_classify_metaclass_class_attribute(self): + class Meta(type): + fish = 'slap' + def __dir__(self): + return ['__class__', '__module__', '__name__', 'fish'] + class Class(metaclass=Meta): + pass + should_find = inspect.Attribute('fish', 'data', Meta, 'slap') + self.assertIn(should_find, inspect.classify_class_attrs(Class)) + + def test_classify_VirtualAttribute(self): + class Meta(type): + def __dir__(cls): + return ['__class__', '__module__', '__name__', 'BOOM'] + def __getattr__(self, name): + if name =='BOOM': + return 42 + return super().__getattr(name) + class Class(metaclass=Meta): + pass + should_find = inspect.Attribute('BOOM', 'data', Meta, 42) + self.assertIn(should_find, inspect.classify_class_attrs(Class)) + + def test_classify_VirtualAttribute_multi_classes(self): + class Meta1(type): + def __dir__(cls): + return ['__class__', '__module__', '__name__', 'one'] + def __getattr__(self, name): + if name =='one': + return 1 + return super().__getattr__(name) + class Meta2(type): + def __dir__(cls): + return ['__class__', '__module__', '__name__', 'two'] + def __getattr__(self, name): + if name =='two': + return 2 + return super().__getattr__(name) + class Meta3(Meta1, Meta2): + def __dir__(cls): + return list(sorted(set(['__class__', '__module__', '__name__', 'three'] + + Meta1.__dir__(cls) + Meta2.__dir__(cls)))) + def __getattr__(self, name): + if name =='three': + return 3 + return super().__getattr__(name) + class Class1(metaclass=Meta1): + pass + class Class2(Class1, metaclass=Meta3): + pass + + should_find1 = inspect.Attribute('one', 'data', Meta1, 1) + should_find2 = inspect.Attribute('two', 'data', Meta2, 2) + should_find3 = inspect.Attribute('three', 'data', Meta3, 3) + cca = inspect.classify_class_attrs(Class2) + for sf in (should_find1, should_find2, should_find3): + self.assertIn(sf, cca) + + def test_classify_class_attrs_with_buggy_dir(self): + class M(type): + def __dir__(cls): + return ['__class__', '__name__', 'missing'] + class C(metaclass=M): + pass + attrs = [a[0] for a in inspect.classify_class_attrs(C)] + self.assertNotIn('missing', attrs) + + def test_getmembers_descriptors(self): + class A(object): + dd = _BrokenDataDescriptor() + md = _BrokenMethodDescriptor() + + def pred_wrapper(pred): + # A quick'n'dirty way to discard standard attributes of new-style + # classes. + class Empty(object): + pass + def wrapped(x): + if '__name__' in dir(x) and hasattr(Empty, x.__name__): + return False + return pred(x) + return wrapped + + ismethoddescriptor = pred_wrapper(inspect.ismethoddescriptor) + isdatadescriptor = pred_wrapper(inspect.isdatadescriptor) + + self.assertEqual(inspect.getmembers(A, ismethoddescriptor), + [('md', A.__dict__['md'])]) + self.assertEqual(inspect.getmembers(A, isdatadescriptor), + [('dd', A.__dict__['dd'])]) + + class B(A): + pass + + self.assertEqual(inspect.getmembers(B, ismethoddescriptor), + [('md', A.__dict__['md'])]) + self.assertEqual(inspect.getmembers(B, isdatadescriptor), + [('dd', A.__dict__['dd'])]) + + def test_getmembers_method(self): + class B: + def f(self): + pass + + self.assertIn(('f', B.f), inspect.getmembers(B)) + self.assertNotIn(('f', B.f), inspect.getmembers(B, inspect.ismethod)) + b = B() + self.assertIn(('f', b.f), inspect.getmembers(b)) + self.assertIn(('f', b.f), inspect.getmembers(b, inspect.ismethod)) + + def test_getmembers_custom_dir(self): + class CorrectDir: + def __init__(self, attr): + self.attr = attr + def method(self): + return self.attr + 1 + def __dir__(self): + return ['attr', 'method'] + + cd = CorrectDir(5) + self.assertEqual(inspect.getmembers(cd), [ + ('attr', 5), + ('method', cd.method), + ]) + self.assertEqual(inspect.getmembers(cd, inspect.ismethod), [ + ('method', cd.method), + ]) + + def test_getmembers_custom_broken_dir(self): + # inspect.getmembers calls `dir()` on the passed object inside. + # if `__dir__` mentions some non-existent attribute, + # we still need to return others correctly. + class BrokenDir: + existing = 1 + def method(self): + return self.existing + 1 + def __dir__(self): + return ['method', 'missing', 'existing'] + + bd = BrokenDir() + self.assertEqual(inspect.getmembers(bd), [ + ('existing', 1), + ('method', bd.method), + ]) + self.assertEqual(inspect.getmembers(bd, inspect.ismethod), [ + ('method', bd.method), + ]) + + def test_getmembers_custom_duplicated_dir(self): + # Duplicates in `__dir__` must not fail and return just one result. + class DuplicatedDir: + attr = 1 + def __dir__(self): + return ['attr', 'attr'] + + dd = DuplicatedDir() + self.assertEqual(inspect.getmembers(dd), [ + ('attr', 1), + ]) + + def test_getmembers_VirtualAttribute(self): + class M(type): + def __getattr__(cls, name): + if name == 'eggs': + return 'scrambled' + return super().__getattr__(name) + class A(metaclass=M): + @types.DynamicClassAttribute + def eggs(self): + return 'spam' + class B: + def __getattr__(self, attribute): + return None + self.assertIn(('eggs', 'scrambled'), inspect.getmembers(A)) + self.assertIn(('eggs', 'spam'), inspect.getmembers(A())) + b = B() + self.assertIn(('__getattr__', b.__getattr__), inspect.getmembers(b)) + + def test_getmembers_static(self): + class A: + @property + def name(self): + raise NotImplementedError + @types.DynamicClassAttribute + def eggs(self): + raise NotImplementedError + + a = A() + instance_members = inspect.getmembers_static(a) + class_members = inspect.getmembers_static(A) + self.assertIn(('name', inspect.getattr_static(a, 'name')), instance_members) + self.assertIn(('eggs', inspect.getattr_static(a, 'eggs')), instance_members) + self.assertIn(('name', inspect.getattr_static(A, 'name')), class_members) + self.assertIn(('eggs', inspect.getattr_static(A, 'eggs')), class_members) + + def test_getmembers_with_buggy_dir(self): + class M(type): + def __dir__(cls): + return ['__class__', '__name__', 'missing'] + class C(metaclass=M): + pass + attrs = [a[0] for a in inspect.getmembers(C)] + self.assertNotIn('missing', attrs) + + def test_get_annotations_with_stock_annotations(self): + def foo(a:int, b:str): pass + self.assertEqual(inspect.get_annotations(foo), {'a': int, 'b': str}) + + foo.__annotations__ = {'a': 'foo', 'b':'str'} + self.assertEqual(inspect.get_annotations(foo), {'a': 'foo', 'b': 'str'}) + + self.assertEqual(inspect.get_annotations(foo, eval_str=True, locals=locals()), {'a': foo, 'b': str}) + self.assertEqual(inspect.get_annotations(foo, eval_str=True, globals=locals()), {'a': foo, 'b': str}) + + isa = inspect_stock_annotations + self.assertEqual(inspect.get_annotations(isa), {'a': int, 'b': str}) + self.assertEqual(inspect.get_annotations(isa.MyClass), {'a': int, 'b': str}) + self.assertEqual(inspect.get_annotations(isa.function), {'a': int, 'b': str, 'return': isa.MyClass}) + self.assertEqual(inspect.get_annotations(isa.function2), {'a': int, 'b': 'str', 'c': isa.MyClass, 'return': isa.MyClass}) + self.assertEqual(inspect.get_annotations(isa.function3), {'a': 'int', 'b': 'str', 'c': 'MyClass'}) + self.assertEqual(inspect.get_annotations(inspect), {}) # inspect module has no annotations + self.assertEqual(inspect.get_annotations(isa.UnannotatedClass), {}) + self.assertEqual(inspect.get_annotations(isa.unannotated_function), {}) + + self.assertEqual(inspect.get_annotations(isa, eval_str=True), {'a': int, 'b': str}) + self.assertEqual(inspect.get_annotations(isa.MyClass, eval_str=True), {'a': int, 'b': str}) + self.assertEqual(inspect.get_annotations(isa.function, eval_str=True), {'a': int, 'b': str, 'return': isa.MyClass}) + self.assertEqual(inspect.get_annotations(isa.function2, eval_str=True), {'a': int, 'b': str, 'c': isa.MyClass, 'return': isa.MyClass}) + self.assertEqual(inspect.get_annotations(isa.function3, eval_str=True), {'a': int, 'b': str, 'c': isa.MyClass}) + self.assertEqual(inspect.get_annotations(inspect, eval_str=True), {}) + self.assertEqual(inspect.get_annotations(isa.UnannotatedClass, eval_str=True), {}) + self.assertEqual(inspect.get_annotations(isa.unannotated_function, eval_str=True), {}) + + self.assertEqual(inspect.get_annotations(isa, eval_str=False), {'a': int, 'b': str}) + self.assertEqual(inspect.get_annotations(isa.MyClass, eval_str=False), {'a': int, 'b': str}) + self.assertEqual(inspect.get_annotations(isa.function, eval_str=False), {'a': int, 'b': str, 'return': isa.MyClass}) + self.assertEqual(inspect.get_annotations(isa.function2, eval_str=False), {'a': int, 'b': 'str', 'c': isa.MyClass, 'return': isa.MyClass}) + self.assertEqual(inspect.get_annotations(isa.function3, eval_str=False), {'a': 'int', 'b': 'str', 'c': 'MyClass'}) + self.assertEqual(inspect.get_annotations(inspect, eval_str=False), {}) + self.assertEqual(inspect.get_annotations(isa.UnannotatedClass, eval_str=False), {}) + self.assertEqual(inspect.get_annotations(isa.unannotated_function, eval_str=False), {}) + + def times_three(fn): + @functools.wraps(fn) + def wrapper(a, b): + return fn(a*3, b*3) + return wrapper + + wrapped = times_three(isa.function) + self.assertEqual(wrapped(1, 'x'), isa.MyClass(3, 'xxx')) + self.assertIsNot(wrapped.__globals__, isa.function.__globals__) + self.assertEqual(inspect.get_annotations(wrapped), {'a': int, 'b': str, 'return': isa.MyClass}) + self.assertEqual(inspect.get_annotations(wrapped, eval_str=True), {'a': int, 'b': str, 'return': isa.MyClass}) + self.assertEqual(inspect.get_annotations(wrapped, eval_str=False), {'a': int, 'b': str, 'return': isa.MyClass}) + + def test_get_annotations_with_stringized_annotations(self): + isa = inspect_stringized_annotations + self.assertEqual(inspect.get_annotations(isa), {'a': 'int', 'b': 'str'}) + self.assertEqual(inspect.get_annotations(isa.MyClass), {'a': 'int', 'b': 'str'}) + self.assertEqual(inspect.get_annotations(isa.function), {'a': 'int', 'b': 'str', 'return': 'MyClass'}) + self.assertEqual(inspect.get_annotations(isa.function2), {'a': 'int', 'b': "'str'", 'c': 'MyClass', 'return': 'MyClass'}) + self.assertEqual(inspect.get_annotations(isa.function3), {'a': "'int'", 'b': "'str'", 'c': "'MyClass'"}) + self.assertEqual(inspect.get_annotations(isa.UnannotatedClass), {}) + self.assertEqual(inspect.get_annotations(isa.unannotated_function), {}) + + self.assertEqual(inspect.get_annotations(isa, eval_str=True), {'a': int, 'b': str}) + self.assertEqual(inspect.get_annotations(isa.MyClass, eval_str=True), {'a': int, 'b': str}) + self.assertEqual(inspect.get_annotations(isa.function, eval_str=True), {'a': int, 'b': str, 'return': isa.MyClass}) + self.assertEqual(inspect.get_annotations(isa.function2, eval_str=True), {'a': int, 'b': 'str', 'c': isa.MyClass, 'return': isa.MyClass}) + self.assertEqual(inspect.get_annotations(isa.function3, eval_str=True), {'a': 'int', 'b': 'str', 'c': 'MyClass'}) + self.assertEqual(inspect.get_annotations(isa.UnannotatedClass, eval_str=True), {}) + self.assertEqual(inspect.get_annotations(isa.unannotated_function, eval_str=True), {}) + + self.assertEqual(inspect.get_annotations(isa, eval_str=False), {'a': 'int', 'b': 'str'}) + self.assertEqual(inspect.get_annotations(isa.MyClass, eval_str=False), {'a': 'int', 'b': 'str'}) + self.assertEqual(inspect.get_annotations(isa.function, eval_str=False), {'a': 'int', 'b': 'str', 'return': 'MyClass'}) + self.assertEqual(inspect.get_annotations(isa.function2, eval_str=False), {'a': 'int', 'b': "'str'", 'c': 'MyClass', 'return': 'MyClass'}) + self.assertEqual(inspect.get_annotations(isa.function3, eval_str=False), {'a': "'int'", 'b': "'str'", 'c': "'MyClass'"}) + self.assertEqual(inspect.get_annotations(isa.UnannotatedClass, eval_str=False), {}) + self.assertEqual(inspect.get_annotations(isa.unannotated_function, eval_str=False), {}) + + isa2 = inspect_stringized_annotations_2 + self.assertEqual(inspect.get_annotations(isa2), {}) + self.assertEqual(inspect.get_annotations(isa2, eval_str=True), {}) + self.assertEqual(inspect.get_annotations(isa2, eval_str=False), {}) + + def times_three(fn): + @functools.wraps(fn) + def wrapper(a, b): + return fn(a*3, b*3) + return wrapper + + wrapped = times_three(isa.function) + self.assertEqual(wrapped(1, 'x'), isa.MyClass(3, 'xxx')) + self.assertIsNot(wrapped.__globals__, isa.function.__globals__) + self.assertEqual(inspect.get_annotations(wrapped), {'a': 'int', 'b': 'str', 'return': 'MyClass'}) + self.assertEqual(inspect.get_annotations(wrapped, eval_str=True), {'a': int, 'b': str, 'return': isa.MyClass}) + self.assertEqual(inspect.get_annotations(wrapped, eval_str=False), {'a': 'int', 'b': 'str', 'return': 'MyClass'}) + + # test that local namespace lookups work + self.assertEqual(inspect.get_annotations(isa.MyClassWithLocalAnnotations), {'x': 'mytype'}) + self.assertEqual(inspect.get_annotations(isa.MyClassWithLocalAnnotations, eval_str=True), {'x': int}) + + def test_pep695_generic_class_with_future_annotations(self): + ann_module695 = inspect_stringized_annotations_pep695 + A_annotations = inspect.get_annotations(ann_module695.A, eval_str=True) + A_type_params = ann_module695.A.__type_params__ + self.assertIs(A_annotations["x"], A_type_params[0]) + self.assertEqual(A_annotations["y"].__args__[0], Unpack[A_type_params[1]]) + self.assertIs(A_annotations["z"].__args__[0], A_type_params[2]) + + def test_pep695_generic_class_with_future_annotations_and_local_shadowing(self): + B_annotations = inspect.get_annotations( + inspect_stringized_annotations_pep695.B, eval_str=True + ) + self.assertEqual(B_annotations, {"x": int, "y": str, "z": bytes}) + + def test_pep695_generic_class_with_future_annotations_name_clash_with_global_vars(self): + ann_module695 = inspect_stringized_annotations_pep695 + C_annotations = inspect.get_annotations(ann_module695.C, eval_str=True) + self.assertEqual( + set(C_annotations.values()), + set(ann_module695.C.__type_params__) + ) + + def test_pep_695_generic_function_with_future_annotations(self): + ann_module695 = inspect_stringized_annotations_pep695 + generic_func_annotations = inspect.get_annotations( + ann_module695.generic_function, eval_str=True + ) + func_t_params = ann_module695.generic_function.__type_params__ + self.assertEqual( + generic_func_annotations.keys(), {"x", "y", "z", "zz", "return"} + ) + self.assertIs(generic_func_annotations["x"], func_t_params[0]) + self.assertEqual(generic_func_annotations["y"], Unpack[func_t_params[1]]) + self.assertIs(generic_func_annotations["z"].__origin__, func_t_params[2]) + self.assertIs(generic_func_annotations["zz"].__origin__, func_t_params[2]) + + def test_pep_695_generic_function_with_future_annotations_name_clash_with_global_vars(self): + self.assertEqual( + set( + inspect.get_annotations( + inspect_stringized_annotations_pep695.generic_function_2, + eval_str=True + ).values() + ), + set( + inspect_stringized_annotations_pep695.generic_function_2.__type_params__ + ) + ) + + def test_pep_695_generic_method_with_future_annotations(self): + ann_module695 = inspect_stringized_annotations_pep695 + generic_method_annotations = inspect.get_annotations( + ann_module695.D.generic_method, eval_str=True + ) + params = { + param.__name__: param + for param in ann_module695.D.generic_method.__type_params__ + } + self.assertEqual( + generic_method_annotations, + {"x": params["Foo"], "y": params["Bar"], "return": None} + ) + + def test_pep_695_generic_method_with_future_annotations_name_clash_with_global_vars(self): + self.assertEqual( + set( + inspect.get_annotations( + inspect_stringized_annotations_pep695.D.generic_method_2, + eval_str=True + ).values() + ), + set( + inspect_stringized_annotations_pep695.D.generic_method_2.__type_params__ + ) + ) + + def test_pep_695_generic_method_with_future_annotations_name_clash_with_global_and_local_vars(self): + self.assertEqual( + inspect.get_annotations( + inspect_stringized_annotations_pep695.E, eval_str=True + ), + {"x": str}, + ) + + def test_pep_695_generics_with_future_annotations_nested_in_function(self): + results = inspect_stringized_annotations_pep695.nested() + + self.assertEqual( + set(results.F_annotations.values()), + set(results.F.__type_params__) + ) + self.assertEqual( + set(results.F_meth_annotations.values()), + set(results.F.generic_method.__type_params__) + ) + self.assertNotEqual( + set(results.F_meth_annotations.values()), + set(results.F.__type_params__) + ) + self.assertEqual( + set(results.F_meth_annotations.values()).intersection(results.F.__type_params__), + set() + ) + + self.assertEqual(results.G_annotations, {"x": str}) + + self.assertEqual( + set(results.generic_func_annotations.values()), + set(results.generic_func.__type_params__) + ) + + +class TestFormatAnnotation(unittest.TestCase): + def test_typing_replacement(self): + from test.typinganndata.ann_module9 import A, ann, ann1 + self.assertEqual(inspect.formatannotation(ann), 'Union[List[str], int]') + self.assertEqual(inspect.formatannotation(ann1), 'Union[List[testModule.typing.A], int]') + + self.assertEqual(inspect.formatannotation(A, 'testModule.typing'), 'A') + self.assertEqual(inspect.formatannotation(A, 'other'), 'testModule.typing.A') + self.assertEqual( + inspect.formatannotation(ann1, 'testModule.typing'), + 'Union[List[testModule.typing.A], int]', + ) + + def test_formatannotationrelativeto(self): + from test.typinganndata.ann_module9 import A, ann1 + + # Builtin types: + self.assertEqual( + inspect.formatannotationrelativeto(object)(type), + 'type', + ) + + # Custom types: + self.assertEqual( + inspect.formatannotationrelativeto(None)(A), + 'testModule.typing.A', + ) + + class B: ... + B.__module__ = 'testModule.typing' + + self.assertEqual( + inspect.formatannotationrelativeto(B)(A), + 'A', + ) + + self.assertEqual( + inspect.formatannotationrelativeto(object)(A), + 'testModule.typing.A', + ) + + # Not an instance of "type": + self.assertEqual( + inspect.formatannotationrelativeto(A)(ann1), + 'Union[List[testModule.typing.A], int]', + ) + + +class TestIsMethodDescriptor(unittest.TestCase): + + def test_custom_descriptors(self): + class MethodDescriptor: + def __get__(self, *_): pass + class MethodDescriptorSub(MethodDescriptor): + pass + class DataDescriptorWithNoGet: + def __set__(self, *_): pass + class DataDescriptorWithGetSet: + def __get__(self, *_): pass + def __set__(self, *_): pass + class DataDescriptorWithGetDelete: + def __get__(self, *_): pass + def __delete__(self, *_): pass + class DataDescriptorSub(DataDescriptorWithNoGet, + DataDescriptorWithGetDelete): + pass + + # Custom method descriptors: + self.assertTrue( + inspect.ismethoddescriptor(MethodDescriptor()), + '__get__ and no __set__/__delete__ => method descriptor') + self.assertTrue( + inspect.ismethoddescriptor(MethodDescriptorSub()), + '__get__ (inherited) and no __set__/__delete__' + ' => method descriptor') + + # Custom data descriptors: + self.assertFalse( + inspect.ismethoddescriptor(DataDescriptorWithNoGet()), + '__set__ (and no __get__) => not a method descriptor') + self.assertFalse( + inspect.ismethoddescriptor(DataDescriptorWithGetSet()), + '__get__ and __set__ => not a method descriptor') + self.assertFalse( + inspect.ismethoddescriptor(DataDescriptorWithGetDelete()), + '__get__ and __delete__ => not a method descriptor') + self.assertFalse( + inspect.ismethoddescriptor(DataDescriptorSub()), + '__get__, __set__ and __delete__ => not a method descriptor') + + # Classes of descriptors (are *not* descriptors themselves): + self.assertFalse(inspect.ismethoddescriptor(MethodDescriptor)) + self.assertFalse(inspect.ismethoddescriptor(MethodDescriptorSub)) + self.assertFalse(inspect.ismethoddescriptor(DataDescriptorSub)) + + def test_builtin_descriptors(self): + builtin_slot_wrapper = int.__add__ # This one is mentioned in docs. + class Owner: + def instance_method(self): pass + @classmethod + def class_method(cls): pass + @staticmethod + def static_method(): pass + @property + def a_property(self): pass + class Slotermeyer: + __slots__ = 'a_slot', + def function(): + pass + a_lambda = lambda: None + + # Example builtin method descriptors: + self.assertTrue( + inspect.ismethoddescriptor(builtin_slot_wrapper), + 'a builtin slot wrapper is a method descriptor') + self.assertTrue( + inspect.ismethoddescriptor(Owner.__dict__['class_method']), + 'a classmethod object is a method descriptor') + self.assertTrue( + inspect.ismethoddescriptor(Owner.__dict__['static_method']), + 'a staticmethod object is a method descriptor') + + # Example builtin data descriptors: + self.assertFalse( + inspect.ismethoddescriptor(Owner.__dict__['a_property']), + 'a property is not a method descriptor') + self.assertFalse( + inspect.ismethoddescriptor(Slotermeyer.__dict__['a_slot']), + 'a slot is not a method descriptor') + + # `types.MethodType`/`types.FunctionType` instances (they *are* + # method descriptors, but `ismethoddescriptor()` explicitly + # excludes them): + self.assertFalse(inspect.ismethoddescriptor(Owner().instance_method)) + self.assertFalse(inspect.ismethoddescriptor(Owner().class_method)) + self.assertFalse(inspect.ismethoddescriptor(Owner().static_method)) + self.assertFalse(inspect.ismethoddescriptor(Owner.instance_method)) + self.assertFalse(inspect.ismethoddescriptor(Owner.class_method)) + self.assertFalse(inspect.ismethoddescriptor(Owner.static_method)) + self.assertFalse(inspect.ismethoddescriptor(function)) + self.assertFalse(inspect.ismethoddescriptor(a_lambda)) + self.assertFalse(inspect.ismethoddescriptor(functools.partial(function))) + + def test_descriptor_being_a_class(self): + class MethodDescriptorMeta(type): + def __get__(self, *_): pass + class ClassBeingMethodDescriptor(metaclass=MethodDescriptorMeta): + pass + # `ClassBeingMethodDescriptor` itself *is* a method descriptor, + # but it is *also* a class, and `ismethoddescriptor()` explicitly + # excludes classes. + self.assertFalse( + inspect.ismethoddescriptor(ClassBeingMethodDescriptor), + 'classes (instances of type) are explicitly excluded') + + def test_non_descriptors(self): + class Test: + pass + self.assertFalse(inspect.ismethoddescriptor(Test())) + self.assertFalse(inspect.ismethoddescriptor(Test)) + self.assertFalse(inspect.ismethoddescriptor([42])) + self.assertFalse(inspect.ismethoddescriptor(42)) + + +class TestIsDataDescriptor(unittest.TestCase): + + def test_custom_descriptors(self): + class NonDataDescriptor: + def __get__(self, value, type=None): pass + class DataDescriptor0: + def __set__(self, name, value): pass + class DataDescriptor1: + def __delete__(self, name): pass + class DataDescriptor2: + __set__ = None + self.assertFalse(inspect.isdatadescriptor(NonDataDescriptor()), + 'class with only __get__ not a data descriptor') + self.assertTrue(inspect.isdatadescriptor(DataDescriptor0()), + 'class with __set__ is a data descriptor') + self.assertTrue(inspect.isdatadescriptor(DataDescriptor1()), + 'class with __delete__ is a data descriptor') + self.assertTrue(inspect.isdatadescriptor(DataDescriptor2()), + 'class with __set__ = None is a data descriptor') + + def test_slot(self): + class Slotted: + __slots__ = 'foo', + self.assertTrue(inspect.isdatadescriptor(Slotted.foo), + 'a slot is a data descriptor') + + def test_property(self): + class Propertied: + @property + def a_property(self): + pass + self.assertTrue(inspect.isdatadescriptor(Propertied.a_property), + 'a property is a data descriptor') + + def test_functions(self): + class Test(object): + def instance_method(self): pass + @classmethod + def class_method(cls): pass + @staticmethod + def static_method(): pass + def function(): + pass + a_lambda = lambda: None + self.assertFalse(inspect.isdatadescriptor(Test().instance_method), + 'a instance method is not a data descriptor') + self.assertFalse(inspect.isdatadescriptor(Test().class_method), + 'a class method is not a data descriptor') + self.assertFalse(inspect.isdatadescriptor(Test().static_method), + 'a static method is not a data descriptor') + self.assertFalse(inspect.isdatadescriptor(function), + 'a function is not a data descriptor') + self.assertFalse(inspect.isdatadescriptor(a_lambda), + 'a lambda is not a data descriptor') + + +_global_ref = object() +class TestGetClosureVars(unittest.TestCase): + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_name_resolution(self): + # Basic test of the 4 different resolution mechanisms + def f(nonlocal_ref): + def g(local_ref): + print(local_ref, nonlocal_ref, _global_ref, unbound_ref) + return g + _arg = object() + nonlocal_vars = {"nonlocal_ref": _arg} + global_vars = {"_global_ref": _global_ref} + builtin_vars = {"print": print} + unbound_names = {"unbound_ref"} + expected = inspect.ClosureVars(nonlocal_vars, global_vars, + builtin_vars, unbound_names) + self.assertEqual(inspect.getclosurevars(f(_arg)), expected) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_generator_closure(self): + def f(nonlocal_ref): + def g(local_ref): + print(local_ref, nonlocal_ref, _global_ref, unbound_ref) + yield + return g + _arg = object() + nonlocal_vars = {"nonlocal_ref": _arg} + global_vars = {"_global_ref": _global_ref} + builtin_vars = {"print": print} + unbound_names = {"unbound_ref"} + expected = inspect.ClosureVars(nonlocal_vars, global_vars, + builtin_vars, unbound_names) + self.assertEqual(inspect.getclosurevars(f(_arg)), expected) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_method_closure(self): + class C: + def f(self, nonlocal_ref): + def g(local_ref): + print(local_ref, nonlocal_ref, _global_ref, unbound_ref) + return g + _arg = object() + nonlocal_vars = {"nonlocal_ref": _arg} + global_vars = {"_global_ref": _global_ref} + builtin_vars = {"print": print} + unbound_names = {"unbound_ref"} + expected = inspect.ClosureVars(nonlocal_vars, global_vars, + builtin_vars, unbound_names) + self.assertEqual(inspect.getclosurevars(C().f(_arg)), expected) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_attribute_same_name_as_global_var(self): + class C: + _global_ref = object() + def f(): + print(C._global_ref, _global_ref) + nonlocal_vars = {"C": C} + global_vars = {"_global_ref": _global_ref} + builtin_vars = {"print": print} + unbound_names = {"_global_ref"} + expected = inspect.ClosureVars(nonlocal_vars, global_vars, + builtin_vars, unbound_names) + self.assertEqual(inspect.getclosurevars(f), expected) + + def test_nonlocal_vars(self): + # More complex tests of nonlocal resolution + def _nonlocal_vars(f): + return inspect.getclosurevars(f).nonlocals + + def make_adder(x): + def add(y): + return x + y + return add + + def curry(func, arg1): + return lambda arg2: func(arg1, arg2) + + def less_than(a, b): + return a < b + + # The infamous Y combinator. + def Y(le): + def g(f): + return le(lambda x: f(f)(x)) + Y.g_ref = g + return g(g) + + def check_y_combinator(func): + self.assertEqual(_nonlocal_vars(func), {'f': Y.g_ref}) + + inc = make_adder(1) + add_two = make_adder(2) + greater_than_five = curry(less_than, 5) + + self.assertEqual(_nonlocal_vars(inc), {'x': 1}) + self.assertEqual(_nonlocal_vars(add_two), {'x': 2}) + self.assertEqual(_nonlocal_vars(greater_than_five), + {'arg1': 5, 'func': less_than}) + self.assertEqual(_nonlocal_vars((lambda x: lambda y: x + y)(3)), + {'x': 3}) + Y(check_y_combinator) + + def test_getclosurevars_empty(self): + def foo(): pass + _empty = inspect.ClosureVars({}, {}, {}, set()) + self.assertEqual(inspect.getclosurevars(lambda: True), _empty) + self.assertEqual(inspect.getclosurevars(foo), _empty) + + def test_getclosurevars_error(self): + class T: pass + self.assertRaises(TypeError, inspect.getclosurevars, 1) + self.assertRaises(TypeError, inspect.getclosurevars, list) + self.assertRaises(TypeError, inspect.getclosurevars, {}) + + def _private_globals(self): + code = """def f(): print(path)""" + ns = {} + exec(code, ns) + return ns["f"], ns + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_builtins_fallback(self): + f, ns = self._private_globals() + ns.pop("__builtins__", None) + expected = inspect.ClosureVars({}, {}, {"print":print}, {"path"}) + self.assertEqual(inspect.getclosurevars(f), expected) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_builtins_as_dict(self): + f, ns = self._private_globals() + ns["__builtins__"] = {"path":1} + expected = inspect.ClosureVars({}, {}, {"path":1}, {"print"}) + self.assertEqual(inspect.getclosurevars(f), expected) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_builtins_as_module(self): + f, ns = self._private_globals() + ns["__builtins__"] = os + expected = inspect.ClosureVars({}, {}, {"path":os.path}, {"print"}) + self.assertEqual(inspect.getclosurevars(f), expected) + + +class TestGetcallargsFunctions(unittest.TestCase): + + def assertEqualCallArgs(self, func, call_params_string, locs=None): + locs = dict(locs or {}, func=func) + r1 = eval('func(%s)' % call_params_string, None, locs) + r2 = eval('inspect.getcallargs(func, %s)' % call_params_string, None, + locs) + self.assertEqual(r1, r2) + + def assertEqualException(self, func, call_param_string, locs=None): + locs = dict(locs or {}, func=func) + try: + eval('func(%s)' % call_param_string, None, locs) + except Exception as e: + ex1 = e + else: + self.fail('Exception not raised') + try: + eval('inspect.getcallargs(func, %s)' % call_param_string, None, + locs) + except Exception as e: + ex2 = e + else: + self.fail('Exception not raised') + self.assertIs(type(ex1), type(ex2)) + self.assertEqual(str(ex1), str(ex2)) + del ex1, ex2 + + def makeCallable(self, signature): + """Create a function that returns its locals()""" + code = "lambda %s: locals()" + return eval(code % signature) + + def test_plain(self): + f = self.makeCallable('a, b=1') + self.assertEqualCallArgs(f, '2') + self.assertEqualCallArgs(f, '2, 3') + self.assertEqualCallArgs(f, 'a=2') + self.assertEqualCallArgs(f, 'b=3, a=2') + self.assertEqualCallArgs(f, '2, b=3') + # expand *iterable / **mapping + self.assertEqualCallArgs(f, '*(2,)') + self.assertEqualCallArgs(f, '*[2]') + self.assertEqualCallArgs(f, '*(2, 3)') + self.assertEqualCallArgs(f, '*[2, 3]') + self.assertEqualCallArgs(f, '**{"a":2}') + self.assertEqualCallArgs(f, 'b=3, **{"a":2}') + self.assertEqualCallArgs(f, '2, **{"b":3}') + self.assertEqualCallArgs(f, '**{"b":3, "a":2}') + # expand UserList / UserDict + self.assertEqualCallArgs(f, '*collections.UserList([2])') + self.assertEqualCallArgs(f, '*collections.UserList([2, 3])') + self.assertEqualCallArgs(f, '**collections.UserDict(a=2)') + self.assertEqualCallArgs(f, '2, **collections.UserDict(b=3)') + self.assertEqualCallArgs(f, 'b=2, **collections.UserDict(a=3)') + + def test_varargs(self): + f = self.makeCallable('a, b=1, *c') + self.assertEqualCallArgs(f, '2') + self.assertEqualCallArgs(f, '2, 3') + self.assertEqualCallArgs(f, '2, 3, 4') + self.assertEqualCallArgs(f, '*(2,3,4)') + self.assertEqualCallArgs(f, '2, *[3,4]') + self.assertEqualCallArgs(f, '2, 3, *collections.UserList([4])') + + def test_varkw(self): + f = self.makeCallable('a, b=1, **c') + self.assertEqualCallArgs(f, 'a=2') + self.assertEqualCallArgs(f, '2, b=3, c=4') + self.assertEqualCallArgs(f, 'b=3, a=2, c=4') + self.assertEqualCallArgs(f, 'c=4, **{"a":2, "b":3}') + self.assertEqualCallArgs(f, '2, c=4, **{"b":3}') + self.assertEqualCallArgs(f, 'b=2, **{"a":3, "c":4}') + self.assertEqualCallArgs(f, '**collections.UserDict(a=2, b=3, c=4)') + self.assertEqualCallArgs(f, '2, c=4, **collections.UserDict(b=3)') + self.assertEqualCallArgs(f, 'b=2, **collections.UserDict(a=3, c=4)') + + def test_varkw_only(self): + # issue11256: + f = self.makeCallable('**c') + self.assertEqualCallArgs(f, '') + self.assertEqualCallArgs(f, 'a=1') + self.assertEqualCallArgs(f, 'a=1, b=2') + self.assertEqualCallArgs(f, 'c=3, **{"a": 1, "b": 2}') + self.assertEqualCallArgs(f, '**collections.UserDict(a=1, b=2)') + self.assertEqualCallArgs(f, 'c=3, **collections.UserDict(a=1, b=2)') + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_keyword_only(self): + f = self.makeCallable('a=3, *, c, d=2') + self.assertEqualCallArgs(f, 'c=3') + self.assertEqualCallArgs(f, 'c=3, a=3') + self.assertEqualCallArgs(f, 'a=2, c=4') + self.assertEqualCallArgs(f, '4, c=4') + self.assertEqualException(f, '') + self.assertEqualException(f, '3') + self.assertEqualException(f, 'a=3') + self.assertEqualException(f, 'd=4') + + f = self.makeCallable('*, c, d=2') + self.assertEqualCallArgs(f, 'c=3') + self.assertEqualCallArgs(f, 'c=3, d=4') + self.assertEqualCallArgs(f, 'd=4, c=3') + + def test_multiple_features(self): + f = self.makeCallable('a, b=2, *f, **g') + self.assertEqualCallArgs(f, '2, 3, 7') + self.assertEqualCallArgs(f, '2, 3, x=8') + self.assertEqualCallArgs(f, '2, 3, x=8, *[(4,[5,6]), 7]') + self.assertEqualCallArgs(f, '2, x=8, *[3, (4,[5,6]), 7], y=9') + self.assertEqualCallArgs(f, 'x=8, *[2, 3, (4,[5,6])], y=9') + self.assertEqualCallArgs(f, 'x=8, *collections.UserList(' + '[2, 3, (4,[5,6])]), **{"y":9, "z":10}') + self.assertEqualCallArgs(f, '2, x=8, *collections.UserList([3, ' + '(4,[5,6])]), **collections.UserDict(' + 'y=9, z=10)') + + f = self.makeCallable('a, b=2, *f, x, y=99, **g') + self.assertEqualCallArgs(f, '2, 3, x=8') + self.assertEqualCallArgs(f, '2, 3, x=8, *[(4,[5,6]), 7]') + self.assertEqualCallArgs(f, '2, x=8, *[3, (4,[5,6]), 7], y=9, z=10') + self.assertEqualCallArgs(f, 'x=8, *[2, 3, (4,[5,6])], y=9, z=10') + self.assertEqualCallArgs(f, 'x=8, *collections.UserList(' + '[2, 3, (4,[5,6])]), q=0, **{"y":9, "z":10}') + self.assertEqualCallArgs(f, '2, x=8, *collections.UserList([3, ' + '(4,[5,6])]), q=0, **collections.UserDict(' + 'y=9, z=10)') + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_errors(self): + f0 = self.makeCallable('') + f1 = self.makeCallable('a, b') + f2 = self.makeCallable('a, b=1') + # f0 takes no arguments + self.assertEqualException(f0, '1') + self.assertEqualException(f0, 'x=1') + self.assertEqualException(f0, '1,x=1') + # f1 takes exactly 2 arguments + self.assertEqualException(f1, '') + self.assertEqualException(f1, '1') + self.assertEqualException(f1, 'a=2') + self.assertEqualException(f1, 'b=3') + # f2 takes at least 1 argument + self.assertEqualException(f2, '') + self.assertEqualException(f2, 'b=3') + for f in f1, f2: + # f1/f2 takes exactly/at most 2 arguments + self.assertEqualException(f, '2, 3, 4') + self.assertEqualException(f, '1, 2, 3, a=1') + self.assertEqualException(f, '2, 3, 4, c=5') + self.assertEqualException(f, '2, 3, 4, a=1, c=5') + # f got an unexpected keyword argument + self.assertEqualException(f, 'c=2') + self.assertEqualException(f, '2, c=3') + self.assertEqualException(f, '2, 3, c=4') + self.assertEqualException(f, '2, c=4, b=3') + self.assertEqualException(f, '**{u"\u03c0\u03b9": 4}') + # f got multiple values for keyword argument + self.assertEqualException(f, '1, a=2') + self.assertEqualException(f, '1, **{"a":2}') + self.assertEqualException(f, '1, 2, b=3') + self.assertEqualException(f, '1, c=3, a=2') + # issue11256: + f3 = self.makeCallable('**c') + self.assertEqualException(f3, '1, 2') + self.assertEqualException(f3, '1, 2, a=1, b=2') + f4 = self.makeCallable('*, a, b=0') + self.assertEqualException(f4, '1, 2') + self.assertEqualException(f4, '1, 2, a=1, b=2') + self.assertEqualException(f4, 'a=1, a=3') + self.assertEqualException(f4, 'a=1, c=3') + self.assertEqualException(f4, 'a=1, a=3, b=4') + self.assertEqualException(f4, 'a=1, b=2, a=3, b=4') + self.assertEqualException(f4, 'a=1, a=2, a=3, b=4') + + # issue #20816: getcallargs() fails to iterate over non-existent + # kwonlydefaults and raises a wrong TypeError + def f5(*, a): pass + with self.assertRaisesRegex(TypeError, + 'missing 1 required keyword-only'): + inspect.getcallargs(f5) + + + # issue20817: + def f6(a, b, c): + pass + with self.assertRaisesRegex(TypeError, "'a', 'b' and 'c'"): + inspect.getcallargs(f6) + + # bpo-33197 + with self.assertRaisesRegex(ValueError, + 'variadic keyword parameters cannot' + ' have default values'): + inspect.Parameter("foo", kind=inspect.Parameter.VAR_KEYWORD, + default=42) + with self.assertRaisesRegex(ValueError, + "value 5 is not a valid Parameter.kind"): + inspect.Parameter("bar", kind=5, default=42) + + with self.assertRaisesRegex(TypeError, + 'name must be a str, not a int'): + inspect.Parameter(123, kind=4) + +class TestGetcallargsMethods(TestGetcallargsFunctions): + + def setUp(self): + class Foo(object): + pass + self.cls = Foo + self.inst = Foo() + + def makeCallable(self, signature): + assert 'self' not in signature + mk = super(TestGetcallargsMethods, self).makeCallable + self.cls.method = mk('self, ' + signature) + return self.inst.method + +class TestGetcallargsUnboundMethods(TestGetcallargsMethods): + + def makeCallable(self, signature): + super(TestGetcallargsUnboundMethods, self).makeCallable(signature) + return self.cls.method + + def assertEqualCallArgs(self, func, call_params_string, locs=None): + return super(TestGetcallargsUnboundMethods, self).assertEqualCallArgs( + *self._getAssertEqualParams(func, call_params_string, locs)) + + def assertEqualException(self, func, call_params_string, locs=None): + return super(TestGetcallargsUnboundMethods, self).assertEqualException( + *self._getAssertEqualParams(func, call_params_string, locs)) + + def _getAssertEqualParams(self, func, call_params_string, locs=None): + assert 'inst' not in call_params_string + locs = dict(locs or {}, inst=self.inst) + return (func, 'inst,' + call_params_string, locs) + + +class TestGetattrStatic(unittest.TestCase): + + def test_basic(self): + class Thing(object): + x = object() + + thing = Thing() + self.assertEqual(inspect.getattr_static(thing, 'x'), Thing.x) + self.assertEqual(inspect.getattr_static(thing, 'x', None), Thing.x) + with self.assertRaises(AttributeError): + inspect.getattr_static(thing, 'y') + + self.assertEqual(inspect.getattr_static(thing, 'y', 3), 3) + + def test_inherited(self): + class Thing(object): + x = object() + class OtherThing(Thing): + pass + + something = OtherThing() + self.assertEqual(inspect.getattr_static(something, 'x'), Thing.x) + + def test_instance_attr(self): + class Thing(object): + x = 2 + def __init__(self, x): + self.x = x + thing = Thing(3) + self.assertEqual(inspect.getattr_static(thing, 'x'), 3) + del thing.x + self.assertEqual(inspect.getattr_static(thing, 'x'), 2) + + def test_property(self): + class Thing(object): + @property + def x(self): + raise AttributeError("I'm pretending not to exist") + thing = Thing() + self.assertEqual(inspect.getattr_static(thing, 'x'), Thing.x) + + def test_descriptor_raises_AttributeError(self): + class descriptor(object): + def __get__(*_): + raise AttributeError("I'm pretending not to exist") + desc = descriptor() + class Thing(object): + x = desc + thing = Thing() + self.assertEqual(inspect.getattr_static(thing, 'x'), desc) + + def test_classAttribute(self): + class Thing(object): + x = object() + + self.assertEqual(inspect.getattr_static(Thing, 'x'), Thing.x) + + def test_classVirtualAttribute(self): + class Thing(object): + @types.DynamicClassAttribute + def x(self): + return self._x + _x = object() + + self.assertEqual(inspect.getattr_static(Thing, 'x'), Thing.__dict__['x']) + + def test_inherited_classattribute(self): + class Thing(object): + x = object() + class OtherThing(Thing): + pass + + self.assertEqual(inspect.getattr_static(OtherThing, 'x'), Thing.x) + + def test_slots(self): + class Thing(object): + y = 'bar' + __slots__ = ['x'] + def __init__(self): + self.x = 'foo' + thing = Thing() + self.assertEqual(inspect.getattr_static(thing, 'x'), Thing.x) + self.assertEqual(inspect.getattr_static(thing, 'y'), 'bar') + + del thing.x + self.assertEqual(inspect.getattr_static(thing, 'x'), Thing.x) + + def test_metaclass(self): + class meta(type): + attr = 'foo' + class Thing(object, metaclass=meta): + pass + self.assertEqual(inspect.getattr_static(Thing, 'attr'), 'foo') + + class sub(meta): + pass + class OtherThing(object, metaclass=sub): + x = 3 + self.assertEqual(inspect.getattr_static(OtherThing, 'attr'), 'foo') + + class OtherOtherThing(OtherThing): + pass + # this test is odd, but it was added as it exposed a bug + self.assertEqual(inspect.getattr_static(OtherOtherThing, 'x'), 3) + + def test_no_dict_no_slots(self): + self.assertEqual(inspect.getattr_static(1, 'foo', None), None) + self.assertNotEqual(inspect.getattr_static('foo', 'lower'), None) + + def test_no_dict_no_slots_instance_member(self): + # returns descriptor + with open(__file__, encoding='utf-8') as handle: + self.assertEqual(inspect.getattr_static(handle, 'name'), type(handle).name) + + def test_inherited_slots(self): + # returns descriptor + class Thing(object): + __slots__ = ['x'] + def __init__(self): + self.x = 'foo' + + class OtherThing(Thing): + pass + # it would be nice if this worked... + # we get the descriptor instead of the instance attribute + self.assertEqual(inspect.getattr_static(OtherThing(), 'x'), Thing.x) + + def test_descriptor(self): + class descriptor(object): + def __get__(self, instance, owner): + return 3 + class Foo(object): + d = descriptor() + + foo = Foo() + + # for a non data descriptor we return the instance attribute + foo.__dict__['d'] = 1 + self.assertEqual(inspect.getattr_static(foo, 'd'), 1) + + # if the descriptor is a data-descriptor we should return the + # descriptor + descriptor.__set__ = lambda s, i, v: None + self.assertEqual(inspect.getattr_static(foo, 'd'), Foo.__dict__['d']) + + del descriptor.__set__ + descriptor.__delete__ = lambda s, i, o: None + self.assertEqual(inspect.getattr_static(foo, 'd'), Foo.__dict__['d']) + + def test_metaclass_with_descriptor(self): + class descriptor(object): + def __get__(self, instance, owner): + return 3 + class meta(type): + d = descriptor() + class Thing(object, metaclass=meta): + pass + self.assertEqual(inspect.getattr_static(Thing, 'd'), meta.__dict__['d']) + + + def test_class_as_property(self): + class Base(object): + foo = 3 + + class Something(Base): + executed = False + @property + def __class__(self): + self.executed = True + return object + + instance = Something() + self.assertEqual(inspect.getattr_static(instance, 'foo'), 3) + self.assertFalse(instance.executed) + self.assertEqual(inspect.getattr_static(Something, 'foo'), 3) + + def test_mro_as_property(self): + class Meta(type): + @property + def __mro__(self): + return (object,) + + class Base(object): + foo = 3 + + class Something(Base, metaclass=Meta): + pass + + self.assertEqual(inspect.getattr_static(Something(), 'foo'), 3) + self.assertEqual(inspect.getattr_static(Something, 'foo'), 3) + + def test_dict_as_property(self): + test = self + test.called = False + + class Foo(dict): + a = 3 + @property + def __dict__(self): + test.called = True + return {} + + foo = Foo() + foo.a = 4 + self.assertEqual(inspect.getattr_static(foo, 'a'), 3) + self.assertFalse(test.called) + + class Bar(Foo): pass + + bar = Bar() + bar.a = 5 + self.assertEqual(inspect.getattr_static(bar, 'a'), 3) + self.assertFalse(test.called) + + def test_mutated_mro(self): + test = self + test.called = False + + class Foo(dict): + a = 3 + @property + def __dict__(self): + test.called = True + return {} + + class Bar(dict): + a = 4 + + class Baz(Bar): pass + + baz = Baz() + self.assertEqual(inspect.getattr_static(baz, 'a'), 4) + Baz.__bases__ = (Foo,) + self.assertEqual(inspect.getattr_static(baz, 'a'), 3) + self.assertFalse(test.called) + + def test_custom_object_dict(self): + test = self + test.called = False + + class Custom(dict): + def get(self, key, default=None): + test.called = True + super().get(key, default) + + class Foo(object): + a = 3 + foo = Foo() + foo.__dict__ = Custom() + self.assertEqual(inspect.getattr_static(foo, 'a'), 3) + self.assertFalse(test.called) + + def test_metaclass_dict_as_property(self): + class Meta(type): + @property + def __dict__(self): + self.executed = True + + class Thing(metaclass=Meta): + executed = False + + def __init__(self): + self.spam = 42 + + instance = Thing() + self.assertEqual(inspect.getattr_static(instance, "spam"), 42) + self.assertFalse(Thing.executed) + + def test_module(self): + sentinel = object() + self.assertIsNot(inspect.getattr_static(sys, "version", sentinel), + sentinel) + + def test_metaclass_with_metaclass_with_dict_as_property(self): + class MetaMeta(type): + @property + def __dict__(self): + self.executed = True + return dict(spam=42) + + class Meta(type, metaclass=MetaMeta): + executed = False + + class Thing(metaclass=Meta): + pass + + with self.assertRaises(AttributeError): + inspect.getattr_static(Thing, "spam") + self.assertFalse(Thing.executed) + + def test_custom___getattr__(self): + test = self + test.called = False + + class Foo: + def __getattr__(self, attr): + test.called = True + return {} + + with self.assertRaises(AttributeError): + inspect.getattr_static(Foo(), 'whatever') + + self.assertFalse(test.called) + + def test_custom___getattribute__(self): + test = self + test.called = False + + class Foo: + def __getattribute__(self, attr): + test.called = True + return {} + + with self.assertRaises(AttributeError): + inspect.getattr_static(Foo(), 'really_could_be_anything') + + self.assertFalse(test.called) + + @suppress_immortalization() + def test_cache_does_not_cause_classes_to_persist(self): + # regression test for gh-118013: + # check that the internal _shadowed_dict cache does not cause + # dynamically created classes to have extended lifetimes even + # when no other strong references to those classes remain. + # Since these classes can themselves hold strong references to + # other objects, this can cause unexpected memory consumption. + class Foo: pass + Foo.instance = Foo() + weakref_to_class = weakref.ref(Foo) + inspect.getattr_static(Foo.instance, 'whatever', 'irrelevant') + del Foo + gc.collect() + self.assertIsNone(weakref_to_class()) + + +class TestGetGeneratorState(unittest.TestCase): + + def setUp(self): + def number_generator(): + for number in range(5): + yield number + self.generator = number_generator() + + def _generatorstate(self): + return inspect.getgeneratorstate(self.generator) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_created(self): + self.assertEqual(self._generatorstate(), inspect.GEN_CREATED) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_suspended(self): + next(self.generator) + self.assertEqual(self._generatorstate(), inspect.GEN_SUSPENDED) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_closed_after_exhaustion(self): + for i in self.generator: + pass + self.assertEqual(self._generatorstate(), inspect.GEN_CLOSED) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_closed_after_immediate_exception(self): + with self.assertRaises(RuntimeError): + self.generator.throw(RuntimeError) + self.assertEqual(self._generatorstate(), inspect.GEN_CLOSED) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_closed_after_close(self): + self.generator.close() + self.assertEqual(self._generatorstate(), inspect.GEN_CLOSED) + + def test_running(self): + # As mentioned on issue #10220, checking for the RUNNING state only + # makes sense inside the generator itself. + # The following generator checks for this by using the closure's + # reference to self and the generator state checking helper method + def running_check_generator(): + for number in range(5): + self.assertEqual(self._generatorstate(), inspect.GEN_RUNNING) + yield number + self.assertEqual(self._generatorstate(), inspect.GEN_RUNNING) + self.generator = running_check_generator() + # Running up to the first yield + next(self.generator) + # Running after the first yield + next(self.generator) + + def test_easy_debugging(self): + # repr() and str() of a generator state should contain the state name + names = 'GEN_CREATED GEN_RUNNING GEN_SUSPENDED GEN_CLOSED'.split() + for name in names: + state = getattr(inspect, name) + self.assertIn(name, repr(state)) + self.assertIn(name, str(state)) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_getgeneratorlocals(self): + def each(lst, a=None): + b=(1, 2, 3) + for v in lst: + if v == 3: + c = 12 + yield v + + numbers = each([1, 2, 3]) + self.assertEqual(inspect.getgeneratorlocals(numbers), + {'a': None, 'lst': [1, 2, 3]}) + next(numbers) + self.assertEqual(inspect.getgeneratorlocals(numbers), + {'a': None, 'lst': [1, 2, 3], 'v': 1, + 'b': (1, 2, 3)}) + next(numbers) + self.assertEqual(inspect.getgeneratorlocals(numbers), + {'a': None, 'lst': [1, 2, 3], 'v': 2, + 'b': (1, 2, 3)}) + next(numbers) + self.assertEqual(inspect.getgeneratorlocals(numbers), + {'a': None, 'lst': [1, 2, 3], 'v': 3, + 'b': (1, 2, 3), 'c': 12}) + try: + next(numbers) + except StopIteration: + pass + self.assertEqual(inspect.getgeneratorlocals(numbers), {}) + + def test_getgeneratorlocals_empty(self): + def yield_one(): + yield 1 + one = yield_one() + self.assertEqual(inspect.getgeneratorlocals(one), {}) + try: + next(one) + except StopIteration: + pass + self.assertEqual(inspect.getgeneratorlocals(one), {}) + + def test_getgeneratorlocals_error(self): + self.assertRaises(TypeError, inspect.getgeneratorlocals, 1) + self.assertRaises(TypeError, inspect.getgeneratorlocals, lambda x: True) + self.assertRaises(TypeError, inspect.getgeneratorlocals, set) + self.assertRaises(TypeError, inspect.getgeneratorlocals, (2,3)) + + +class TestGetCoroutineState(unittest.TestCase): + + def setUp(self): + @types.coroutine + def number_coroutine(): + for number in range(5): + yield number + async def coroutine(): + await number_coroutine() + self.coroutine = coroutine() + + def tearDown(self): + self.coroutine.close() + + def _coroutinestate(self): + return inspect.getcoroutinestate(self.coroutine) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_created(self): + self.assertEqual(self._coroutinestate(), inspect.CORO_CREATED) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_suspended(self): + self.coroutine.send(None) + self.assertEqual(self._coroutinestate(), inspect.CORO_SUSPENDED) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_closed_after_exhaustion(self): + while True: + try: + self.coroutine.send(None) + except StopIteration: + break + + self.assertEqual(self._coroutinestate(), inspect.CORO_CLOSED) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_closed_after_immediate_exception(self): + with self.assertRaises(RuntimeError): + self.coroutine.throw(RuntimeError) + self.assertEqual(self._coroutinestate(), inspect.CORO_CLOSED) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_closed_after_close(self): + self.coroutine.close() + self.assertEqual(self._coroutinestate(), inspect.CORO_CLOSED) + + def test_easy_debugging(self): + # repr() and str() of a coroutine state should contain the state name + names = 'CORO_CREATED CORO_RUNNING CORO_SUSPENDED CORO_CLOSED'.split() + for name in names: + state = getattr(inspect, name) + self.assertIn(name, repr(state)) + self.assertIn(name, str(state)) + + def test_getcoroutinelocals(self): + @types.coroutine + def gencoro(): + yield + + gencoro = gencoro() + async def func(a=None): + b = 'spam' + await gencoro + + coro = func() + self.assertEqual(inspect.getcoroutinelocals(coro), + {'a': None, 'gencoro': gencoro}) + coro.send(None) + self.assertEqual(inspect.getcoroutinelocals(coro), + {'a': None, 'gencoro': gencoro, 'b': 'spam'}) + + +@support.requires_working_socket() +class TestGetAsyncGenState(unittest.IsolatedAsyncioTestCase): + + def setUp(self): + async def number_asyncgen(): + for number in range(5): + yield number + self.asyncgen = number_asyncgen() + + async def asyncTearDown(self): + await self.asyncgen.aclose() + + def _asyncgenstate(self): + return inspect.getasyncgenstate(self.asyncgen) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_created(self): + self.assertEqual(self._asyncgenstate(), inspect.AGEN_CREATED) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + async def test_suspended(self): + value = await anext(self.asyncgen) + self.assertEqual(self._asyncgenstate(), inspect.AGEN_SUSPENDED) + self.assertEqual(value, 0) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + async def test_closed_after_exhaustion(self): + countdown = 7 + with self.assertRaises(StopAsyncIteration): + while countdown := countdown - 1: + await anext(self.asyncgen) + self.assertEqual(countdown, 1) + self.assertEqual(self._asyncgenstate(), inspect.AGEN_CLOSED) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + async def test_closed_after_immediate_exception(self): + with self.assertRaises(RuntimeError): + await self.asyncgen.athrow(RuntimeError) + self.assertEqual(self._asyncgenstate(), inspect.AGEN_CLOSED) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + async def test_running(self): + async def running_check_asyncgen(): + for number in range(5): + self.assertEqual(self._asyncgenstate(), inspect.AGEN_RUNNING) + yield number + self.assertEqual(self._asyncgenstate(), inspect.AGEN_RUNNING) + self.asyncgen = running_check_asyncgen() + # Running up to the first yield + await anext(self.asyncgen) + self.assertEqual(self._asyncgenstate(), inspect.AGEN_SUSPENDED) + # Running after the first yield + await anext(self.asyncgen) + self.assertEqual(self._asyncgenstate(), inspect.AGEN_SUSPENDED) + + def test_easy_debugging(self): + # repr() and str() of a asyncgen state should contain the state name + names = 'AGEN_CREATED AGEN_RUNNING AGEN_SUSPENDED AGEN_CLOSED'.split() + for name in names: + state = getattr(inspect, name) + self.assertIn(name, repr(state)) + self.assertIn(name, str(state)) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + async def test_getasyncgenlocals(self): + async def each(lst, a=None): + b=(1, 2, 3) + for v in lst: + if v == 3: + c = 12 + yield v + + numbers = each([1, 2, 3]) + self.assertEqual(inspect.getasyncgenlocals(numbers), + {'a': None, 'lst': [1, 2, 3]}) + await anext(numbers) + self.assertEqual(inspect.getasyncgenlocals(numbers), + {'a': None, 'lst': [1, 2, 3], 'v': 1, + 'b': (1, 2, 3)}) + await anext(numbers) + self.assertEqual(inspect.getasyncgenlocals(numbers), + {'a': None, 'lst': [1, 2, 3], 'v': 2, + 'b': (1, 2, 3)}) + await anext(numbers) + self.assertEqual(inspect.getasyncgenlocals(numbers), + {'a': None, 'lst': [1, 2, 3], 'v': 3, + 'b': (1, 2, 3), 'c': 12}) + with self.assertRaises(StopAsyncIteration): + await anext(numbers) + self.assertEqual(inspect.getasyncgenlocals(numbers), {}) + + async def test_getasyncgenlocals_empty(self): + async def yield_one(): + yield 1 + one = yield_one() + self.assertEqual(inspect.getasyncgenlocals(one), {}) + await anext(one) + self.assertEqual(inspect.getasyncgenlocals(one), {}) + with self.assertRaises(StopAsyncIteration): + await anext(one) + self.assertEqual(inspect.getasyncgenlocals(one), {}) + + def test_getasyncgenlocals_error(self): + self.assertRaises(TypeError, inspect.getasyncgenlocals, 1) + self.assertRaises(TypeError, inspect.getasyncgenlocals, lambda x: True) + self.assertRaises(TypeError, inspect.getasyncgenlocals, set) + self.assertRaises(TypeError, inspect.getasyncgenlocals, (2,3)) + + +class MySignature(inspect.Signature): + # Top-level to make it picklable; + # used in test_signature_object_pickle + pass + +class MyParameter(inspect.Parameter): + # Top-level to make it picklable; + # used in test_signature_object_pickle + pass + + + +class TestSignatureObject(unittest.TestCase): + @staticmethod + def signature(func, **kw): + sig = inspect.signature(func, **kw) + return (tuple((param.name, + (... if param.default is param.empty else param.default), + (... if param.annotation is param.empty + else param.annotation), + str(param.kind).lower()) + for param in sig.parameters.values()), + (... if sig.return_annotation is sig.empty + else sig.return_annotation)) + + def test_signature_object(self): + S = inspect.Signature + P = inspect.Parameter + + self.assertEqual(str(S()), '()') + self.assertEqual(repr(S().parameters), 'mappingproxy(OrderedDict())') + + def test(po, /, pk, pkd=100, *args, ko, kod=10, **kwargs): + pass + + sig = inspect.signature(test) + self.assertTrue(repr(sig).startswith('<Signature')) + self.assertTrue('(po, /, pk' in repr(sig)) + + # We need two functions, because it is impossible to represent + # all param kinds in a single one. + def test2(pod=42, /): + pass + + sig2 = inspect.signature(test2) + self.assertTrue(repr(sig2).startswith('<Signature')) + self.assertTrue('(pod=42, /)' in repr(sig2)) + + po = sig.parameters['po'] + pod = sig2.parameters['pod'] + pk = sig.parameters['pk'] + pkd = sig.parameters['pkd'] + args = sig.parameters['args'] + ko = sig.parameters['ko'] + kod = sig.parameters['kod'] + kwargs = sig.parameters['kwargs'] + + S((po, pk, args, ko, kwargs)) + S((po, pk, ko, kod)) + S((po, pod, ko)) + S((po, pod, kod)) + S((pod, ko, kod)) + S((pod, kod)) + S((pod, args, kod, kwargs)) + # keyword-only parameters without default values + # can follow keyword-only parameters with default values: + S((kod, ko)) + S((kod, ko, kwargs)) + S((args, kod, ko)) + + with self.assertRaisesRegex(ValueError, 'wrong parameter order'): + S((pk, po, args, ko, kwargs)) + + with self.assertRaisesRegex(ValueError, 'wrong parameter order'): + S((po, args, pk, ko, kwargs)) + + with self.assertRaisesRegex(ValueError, 'wrong parameter order'): + S((args, po, pk, ko, kwargs)) + + with self.assertRaisesRegex(ValueError, 'wrong parameter order'): + S((po, pk, args, kwargs, ko)) + + kwargs2 = kwargs.replace(name='args') + with self.assertRaisesRegex(ValueError, 'duplicate parameter name'): + S((po, pk, args, kwargs2, ko)) + + with self.assertRaisesRegex(ValueError, 'follows default argument'): + S((pod, po)) + + with self.assertRaisesRegex(ValueError, 'follows default argument'): + S((pod, pk)) + + with self.assertRaisesRegex(ValueError, 'follows default argument'): + S((po, pod, pk)) + + with self.assertRaisesRegex(ValueError, 'follows default argument'): + S((po, pkd, pk)) + + with self.assertRaisesRegex(ValueError, 'follows default argument'): + S((pkd, pk)) + + def test_signature_object_pickle(self): + def foo(a, b, *, c:1={}, **kw) -> {42:'ham'}: pass + foo_partial = functools.partial(foo, a=1) + + sig = inspect.signature(foo_partial) + + for ver in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(pickle_ver=ver, subclass=False): + sig_pickled = pickle.loads(pickle.dumps(sig, ver)) + self.assertEqual(sig, sig_pickled) + + # Test that basic sub-classing works + sig = inspect.signature(foo) + myparam = MyParameter(name='z', kind=inspect.Parameter.POSITIONAL_ONLY) + myparams = collections.OrderedDict(sig.parameters, a=myparam) + mysig = MySignature().replace(parameters=myparams.values(), + return_annotation=sig.return_annotation) + self.assertTrue(isinstance(mysig, MySignature)) + self.assertTrue(isinstance(mysig.parameters['z'], MyParameter)) + + for ver in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(pickle_ver=ver, subclass=True): + sig_pickled = pickle.loads(pickle.dumps(mysig, ver)) + self.assertEqual(mysig, sig_pickled) + self.assertTrue(isinstance(sig_pickled, MySignature)) + self.assertTrue(isinstance(sig_pickled.parameters['z'], + MyParameter)) + + def test_signature_immutability(self): + def test(a): + pass + sig = inspect.signature(test) + + with self.assertRaises(AttributeError): + sig.foo = 'bar' + + with self.assertRaises(TypeError): + sig.parameters['a'] = None + + def test_signature_on_noarg(self): + def test(): + pass + self.assertEqual(self.signature(test), ((), ...)) + + def test_signature_on_wargs(self): + def test(a, b:'foo') -> 123: + pass + self.assertEqual(self.signature(test), + ((('a', ..., ..., "positional_or_keyword"), + ('b', ..., 'foo', "positional_or_keyword")), + 123)) + + def test_signature_on_wkwonly(self): + def test(*, a:float, b:str) -> int: + pass + self.assertEqual(self.signature(test), + ((('a', ..., float, "keyword_only"), + ('b', ..., str, "keyword_only")), + int)) + + def test_signature_on_complex_args(self): + def test(a, b:'foo'=10, *args:'bar', spam:'baz', ham=123, **kwargs:int): + pass + self.assertEqual(self.signature(test), + ((('a', ..., ..., "positional_or_keyword"), + ('b', 10, 'foo', "positional_or_keyword"), + ('args', ..., 'bar', "var_positional"), + ('spam', ..., 'baz', "keyword_only"), + ('ham', 123, ..., "keyword_only"), + ('kwargs', ..., int, "var_keyword")), + ...)) + + def test_signature_without_self(self): + def test_args_only(*args): # NOQA + pass + + def test_args_kwargs_only(*args, **kwargs): # NOQA + pass + + class A: + @classmethod + def test_classmethod(*args): # NOQA + pass + + @staticmethod + def test_staticmethod(*args): # NOQA + pass + + f1 = functools.partialmethod((test_classmethod), 1) + f2 = functools.partialmethod((test_args_only), 1) + f3 = functools.partialmethod((test_staticmethod), 1) + f4 = functools.partialmethod((test_args_kwargs_only),1) + + self.assertEqual(self.signature(test_args_only), + ((('args', ..., ..., 'var_positional'),), ...)) + self.assertEqual(self.signature(test_args_kwargs_only), + ((('args', ..., ..., 'var_positional'), + ('kwargs', ..., ..., 'var_keyword')), ...)) + self.assertEqual(self.signature(A.f1), + ((('args', ..., ..., 'var_positional'),), ...)) + self.assertEqual(self.signature(A.f2), + ((('args', ..., ..., 'var_positional'),), ...)) + self.assertEqual(self.signature(A.f3), + ((('args', ..., ..., 'var_positional'),), ...)) + self.assertEqual(self.signature(A.f4), + ((('args', ..., ..., 'var_positional'), + ('kwargs', ..., ..., 'var_keyword')), ...)) + @cpython_only + @unittest.skipIf(MISSING_C_DOCSTRINGS, + "Signature information for builtins requires docstrings") + def test_signature_on_builtins(self): + _testcapi = import_helper.import_module("_testcapi") + + def test_unbound_method(o): + """Use this to test unbound methods (things that should have a self)""" + signature = inspect.signature(o) + self.assertTrue(isinstance(signature, inspect.Signature)) + self.assertEqual(list(signature.parameters.values())[0].name, 'self') + return signature + + def test_callable(o): + """Use this to test bound methods or normal callables (things that don't expect self)""" + signature = inspect.signature(o) + self.assertTrue(isinstance(signature, inspect.Signature)) + if signature.parameters: + self.assertNotEqual(list(signature.parameters.values())[0].name, 'self') + return signature + + signature = test_callable(_testcapi.docstring_with_signature_with_defaults) + def p(name): return signature.parameters[name].default + self.assertEqual(p('s'), 'avocado') + self.assertEqual(p('b'), b'bytes') + self.assertEqual(p('d'), 3.14) + self.assertEqual(p('i'), 35) + self.assertEqual(p('n'), None) + self.assertEqual(p('t'), True) + self.assertEqual(p('f'), False) + self.assertEqual(p('local'), 3) + self.assertEqual(p('sys'), sys.maxsize) + self.assertEqual(p('exp'), sys.maxsize - 1) + + test_callable(object) + + # normal method + # (PyMethodDescr_Type, "method_descriptor") + test_unbound_method(_pickle.Pickler.dump) + d = _pickle.Pickler(io.StringIO()) + test_callable(d.dump) + + # static method + test_callable(bytes.maketrans) + test_callable(b'abc'.maketrans) + + # class method + test_callable(dict.fromkeys) + test_callable({}.fromkeys) + + # wrapper around slot (PyWrapperDescr_Type, "wrapper_descriptor") + test_unbound_method(type.__call__) + test_unbound_method(int.__add__) + test_callable((3).__add__) + + # _PyMethodWrapper_Type + # support for 'method-wrapper' + test_callable(min.__call__) + + # This doesn't work now. + # (We don't have a valid signature for "type" in 3.4) + class ThisWorksNow: + __call__ = type + # TODO: Support type. + self.assertEqual(ThisWorksNow()(1), int) + self.assertEqual(ThisWorksNow()('A', (), {}).__name__, 'A') + with self.assertRaisesRegex(ValueError, "no signature found"): + test_callable(ThisWorksNow()) + + # Regression test for issue #20786 + test_unbound_method(dict.__delitem__) + test_unbound_method(property.__delete__) + + # Regression test for issue #20586 + test_callable(_testcapi.docstring_with_signature_but_no_doc) + + # Regression test for gh-104955 + method = bytearray.__release_buffer__ + sig = test_unbound_method(method) + self.assertEqual(list(sig.parameters), ['self', 'buffer']) + + @cpython_only + @unittest.skipIf(MISSING_C_DOCSTRINGS, + "Signature information for builtins requires docstrings") + def test_signature_on_decorated_builtins(self): + _testcapi = import_helper.import_module("_testcapi") + func = _testcapi.docstring_with_signature_with_defaults + + def decorator(func): + @functools.wraps(func) + def wrapper(*args, **kwargs) -> int: + return func(*args, **kwargs) + return wrapper + + decorated_func = decorator(func) + + self.assertEqual(inspect.signature(func), + inspect.signature(decorated_func)) + + def wrapper_like(*args, **kwargs) -> int: pass + self.assertEqual(inspect.signature(decorated_func, + follow_wrapped=False), + inspect.signature(wrapper_like)) + + @cpython_only + def test_signature_on_builtins_no_signature(self): + _testcapi = import_helper.import_module("_testcapi") + with self.assertRaisesRegex(ValueError, + 'no signature found for builtin'): + inspect.signature(_testcapi.docstring_no_signature) + + with self.assertRaisesRegex(ValueError, + 'no signature found for builtin'): + inspect.signature(str) + + cls = _testcapi.DocStringNoSignatureTest + obj = _testcapi.DocStringNoSignatureTest() + tests = [ + (_testcapi.docstring_no_signature_noargs, meth_noargs), + (_testcapi.docstring_no_signature_o, meth_o), + (cls.meth_noargs, meth_self_noargs), + (cls.meth_o, meth_self_o), + (obj.meth_noargs, meth_noargs), + (obj.meth_o, meth_o), + (cls.meth_noargs_class, meth_noargs), + (cls.meth_o_class, meth_o), + (cls.meth_noargs_static, meth_noargs), + (cls.meth_o_static, meth_o), + (cls.meth_noargs_coexist, meth_self_noargs), + (cls.meth_o_coexist, meth_self_o), + + (time.time, meth_noargs), + (str.lower, meth_self_noargs), + (''.lower, meth_noargs), + (set.add, meth_self_o), + (set().add, meth_o), + (set.__contains__, meth_self_o), + (set().__contains__, meth_o), + (datetime.datetime.__dict__['utcnow'], meth_type_noargs), + (datetime.datetime.utcnow, meth_noargs), + (dict.__dict__['__class_getitem__'], meth_type_o), + (dict.__class_getitem__, meth_o), + ] + try: + import _stat + except ImportError: + # if the _stat extension is not available, stat.S_IMODE() is + # implemented in Python, not in C + pass + else: + tests.append((stat.S_IMODE, meth_o)) + for builtin, template in tests: + with self.subTest(builtin): + self.assertEqual(inspect.signature(builtin), + inspect.signature(template)) + + @unittest.skipIf(MISSING_C_DOCSTRINGS, + "Signature information for builtins requires docstrings") + def test_signature_parsing_with_defaults(self): + _testcapi = import_helper.import_module("_testcapi") + meth = _testcapi.DocStringUnrepresentableSignatureTest.with_default + self.assertEqual(str(inspect.signature(meth)), '(self, /, x=1)') + + def test_signature_on_non_function(self): + with self.assertRaisesRegex(TypeError, 'is not a callable object'): + inspect.signature(42) + + def test_signature_from_functionlike_object(self): + def func(a,b, *args, kwonly=True, kwonlyreq, **kwargs): + pass + + class funclike: + # Has to be callable, and have correct + # __code__, __annotations__, __defaults__, __name__, + # and __kwdefaults__ attributes + + def __init__(self, func): + self.__name__ = func.__name__ + self.__code__ = func.__code__ + self.__annotations__ = func.__annotations__ + self.__defaults__ = func.__defaults__ + self.__kwdefaults__ = func.__kwdefaults__ + self.func = func + + def __call__(self, *args, **kwargs): + return self.func(*args, **kwargs) + + sig_func = inspect.Signature.from_callable(func) + + sig_funclike = inspect.Signature.from_callable(funclike(func)) + self.assertEqual(sig_funclike, sig_func) + + sig_funclike = inspect.signature(funclike(func)) + self.assertEqual(sig_funclike, sig_func) + + # If object is not a duck type of function, then + # signature will try to get a signature for its '__call__' + # method + fl = funclike(func) + del fl.__defaults__ + self.assertEqual(self.signature(fl), + ((('args', ..., ..., "var_positional"), + ('kwargs', ..., ..., "var_keyword")), + ...)) + + # Test with cython-like builtins: + _orig_isdesc = inspect.ismethoddescriptor + def _isdesc(obj): + if hasattr(obj, '_builtinmock'): + return True + return _orig_isdesc(obj) + + with unittest.mock.patch('inspect.ismethoddescriptor', _isdesc): + builtin_func = funclike(func) + # Make sure that our mock setup is working + self.assertFalse(inspect.ismethoddescriptor(builtin_func)) + builtin_func._builtinmock = True + self.assertTrue(inspect.ismethoddescriptor(builtin_func)) + self.assertEqual(inspect.signature(builtin_func), sig_func) + + def test_signature_functionlike_class(self): + # We only want to duck type function-like objects, + # not classes. + + def func(a,b, *args, kwonly=True, kwonlyreq, **kwargs): + pass + + class funclike: + def __init__(self, marker): + pass + + __name__ = func.__name__ + __code__ = func.__code__ + __annotations__ = func.__annotations__ + __defaults__ = func.__defaults__ + __kwdefaults__ = func.__kwdefaults__ + + self.assertEqual(str(inspect.signature(funclike)), '(marker)') + + def test_signature_on_method(self): + class Test: + def __init__(*args): + pass + def m1(self, arg1, arg2=1) -> int: + pass + def m2(*args): + pass + def __call__(*, a): + pass + + self.assertEqual(self.signature(Test().m1), + ((('arg1', ..., ..., "positional_or_keyword"), + ('arg2', 1, ..., "positional_or_keyword")), + int)) + + self.assertEqual(self.signature(Test().m2), + ((('args', ..., ..., "var_positional"),), + ...)) + + self.assertEqual(self.signature(Test), + ((('args', ..., ..., "var_positional"),), + ...)) + + with self.assertRaisesRegex(ValueError, 'invalid method signature'): + self.signature(Test()) + + def test_signature_wrapped_bound_method(self): + # Issue 24298 + class Test: + def m1(self, arg1, arg2=1) -> int: + pass + @functools.wraps(Test().m1) + def m1d(*args, **kwargs): + pass + self.assertEqual(self.signature(m1d), + ((('arg1', ..., ..., "positional_or_keyword"), + ('arg2', 1, ..., "positional_or_keyword")), + int)) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_signature_on_classmethod(self): + if not support.MISSING_C_DOCSTRINGS: + self.assertEqual(self.signature(classmethod), + ((('function', ..., ..., "positional_only"),), + ...)) + + class Test: + @classmethod + def foo(cls, arg1, *, arg2=1): + pass + + meth = Test().foo + self.assertEqual(self.signature(meth), + ((('arg1', ..., ..., "positional_or_keyword"), + ('arg2', 1, ..., "keyword_only")), + ...)) + + meth = Test.foo + self.assertEqual(self.signature(meth), + ((('arg1', ..., ..., "positional_or_keyword"), + ('arg2', 1, ..., "keyword_only")), + ...)) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_signature_on_staticmethod(self): + if not support.MISSING_C_DOCSTRINGS: + self.assertEqual(self.signature(staticmethod), + ((('function', ..., ..., "positional_only"),), + ...)) + + class Test: + @staticmethod + def foo(cls, *, arg): + pass + + meth = Test().foo + self.assertEqual(self.signature(meth), + ((('cls', ..., ..., "positional_or_keyword"), + ('arg', ..., ..., "keyword_only")), + ...)) + + meth = Test.foo + self.assertEqual(self.signature(meth), + ((('cls', ..., ..., "positional_or_keyword"), + ('arg', ..., ..., "keyword_only")), + ...)) + + def test_signature_on_partial(self): + from functools import partial + + def test(): + pass + + self.assertEqual(self.signature(partial(test)), ((), ...)) + + with self.assertRaisesRegex(ValueError, "has incorrect arguments"): + inspect.signature(partial(test, 1)) + + with self.assertRaisesRegex(ValueError, "has incorrect arguments"): + inspect.signature(partial(test, a=1)) + + def test(a, b, *, c, d): + pass + + self.assertEqual(self.signature(partial(test)), + ((('a', ..., ..., "positional_or_keyword"), + ('b', ..., ..., "positional_or_keyword"), + ('c', ..., ..., "keyword_only"), + ('d', ..., ..., "keyword_only")), + ...)) + + self.assertEqual(self.signature(partial(test, 1)), + ((('b', ..., ..., "positional_or_keyword"), + ('c', ..., ..., "keyword_only"), + ('d', ..., ..., "keyword_only")), + ...)) + + self.assertEqual(self.signature(partial(test, 1, c=2)), + ((('b', ..., ..., "positional_or_keyword"), + ('c', 2, ..., "keyword_only"), + ('d', ..., ..., "keyword_only")), + ...)) + + self.assertEqual(self.signature(partial(test, b=1, c=2)), + ((('a', ..., ..., "positional_or_keyword"), + ('b', 1, ..., "keyword_only"), + ('c', 2, ..., "keyword_only"), + ('d', ..., ..., "keyword_only")), + ...)) + + self.assertEqual(self.signature(partial(test, 0, b=1, c=2)), + ((('b', 1, ..., "keyword_only"), + ('c', 2, ..., "keyword_only"), + ('d', ..., ..., "keyword_only")), + ...)) + + self.assertEqual(self.signature(partial(test, a=1)), + ((('a', 1, ..., "keyword_only"), + ('b', ..., ..., "keyword_only"), + ('c', ..., ..., "keyword_only"), + ('d', ..., ..., "keyword_only")), + ...)) + + def test(a, *args, b, **kwargs): + pass + + self.assertEqual(self.signature(partial(test, 1)), + ((('args', ..., ..., "var_positional"), + ('b', ..., ..., "keyword_only"), + ('kwargs', ..., ..., "var_keyword")), + ...)) + + self.assertEqual(self.signature(partial(test, a=1)), + ((('a', 1, ..., "keyword_only"), + ('b', ..., ..., "keyword_only"), + ('kwargs', ..., ..., "var_keyword")), + ...)) + + self.assertEqual(self.signature(partial(test, 1, 2, 3)), + ((('args', ..., ..., "var_positional"), + ('b', ..., ..., "keyword_only"), + ('kwargs', ..., ..., "var_keyword")), + ...)) + + self.assertEqual(self.signature(partial(test, 1, 2, 3, test=True)), + ((('args', ..., ..., "var_positional"), + ('b', ..., ..., "keyword_only"), + ('kwargs', ..., ..., "var_keyword")), + ...)) + + self.assertEqual(self.signature(partial(test, 1, 2, 3, test=1, b=0)), + ((('args', ..., ..., "var_positional"), + ('b', 0, ..., "keyword_only"), + ('kwargs', ..., ..., "var_keyword")), + ...)) + + self.assertEqual(self.signature(partial(test, b=0)), + ((('a', ..., ..., "positional_or_keyword"), + ('args', ..., ..., "var_positional"), + ('b', 0, ..., "keyword_only"), + ('kwargs', ..., ..., "var_keyword")), + ...)) + + self.assertEqual(self.signature(partial(test, b=0, test=1)), + ((('a', ..., ..., "positional_or_keyword"), + ('args', ..., ..., "var_positional"), + ('b', 0, ..., "keyword_only"), + ('kwargs', ..., ..., "var_keyword")), + ...)) + + def test(a, b, c:int) -> 42: + pass + + sig = test.__signature__ = inspect.signature(test) + + self.assertEqual(self.signature(partial(partial(test, 1))), + ((('b', ..., ..., "positional_or_keyword"), + ('c', ..., int, "positional_or_keyword")), + 42)) + + self.assertEqual(self.signature(partial(partial(test, 1), 2)), + ((('c', ..., int, "positional_or_keyword"),), + 42)) + + def foo(a): + return a + _foo = partial(partial(foo, a=10), a=20) + self.assertEqual(self.signature(_foo), + ((('a', 20, ..., "keyword_only"),), + ...)) + # check that we don't have any side-effects in signature(), + # and the partial object is still functioning + self.assertEqual(_foo(), 20) + + def foo(a, b, c): + return a, b, c + _foo = partial(partial(foo, 1, b=20), b=30) + + self.assertEqual(self.signature(_foo), + ((('b', 30, ..., "keyword_only"), + ('c', ..., ..., "keyword_only")), + ...)) + self.assertEqual(_foo(c=10), (1, 30, 10)) + + def foo(a, b, c, *, d): + return a, b, c, d + _foo = partial(partial(foo, d=20, c=20), b=10, d=30) + self.assertEqual(self.signature(_foo), + ((('a', ..., ..., "positional_or_keyword"), + ('b', 10, ..., "keyword_only"), + ('c', 20, ..., "keyword_only"), + ('d', 30, ..., "keyword_only"), + ), + ...)) + ba = inspect.signature(_foo).bind(a=200, b=11) + self.assertEqual(_foo(*ba.args, **ba.kwargs), (200, 11, 20, 30)) + + def foo(a=1, b=2, c=3): + return a, b, c + _foo = partial(foo, c=13) # (a=1, b=2, *, c=13) + + ba = inspect.signature(_foo).bind(a=11) + self.assertEqual(_foo(*ba.args, **ba.kwargs), (11, 2, 13)) + + ba = inspect.signature(_foo).bind(11, 12) + self.assertEqual(_foo(*ba.args, **ba.kwargs), (11, 12, 13)) + + ba = inspect.signature(_foo).bind(11, b=12) + self.assertEqual(_foo(*ba.args, **ba.kwargs), (11, 12, 13)) + + ba = inspect.signature(_foo).bind(b=12) + self.assertEqual(_foo(*ba.args, **ba.kwargs), (1, 12, 13)) + + _foo = partial(_foo, b=10, c=20) + ba = inspect.signature(_foo).bind(12) + self.assertEqual(_foo(*ba.args, **ba.kwargs), (12, 10, 20)) + + + def foo(a, b, /, c, d, **kwargs): + pass + sig = inspect.signature(foo) + self.assertEqual(str(sig), '(a, b, /, c, d, **kwargs)') + + self.assertEqual(self.signature(partial(foo, 1)), + ((('b', ..., ..., 'positional_only'), + ('c', ..., ..., 'positional_or_keyword'), + ('d', ..., ..., 'positional_or_keyword'), + ('kwargs', ..., ..., 'var_keyword')), + ...)) + + self.assertEqual(self.signature(partial(foo, 1, 2)), + ((('c', ..., ..., 'positional_or_keyword'), + ('d', ..., ..., 'positional_or_keyword'), + ('kwargs', ..., ..., 'var_keyword')), + ...)) + + self.assertEqual(self.signature(partial(foo, 1, 2, 3)), + ((('d', ..., ..., 'positional_or_keyword'), + ('kwargs', ..., ..., 'var_keyword')), + ...)) + + self.assertEqual(self.signature(partial(foo, 1, 2, c=3)), + ((('c', 3, ..., 'keyword_only'), + ('d', ..., ..., 'keyword_only'), + ('kwargs', ..., ..., 'var_keyword')), + ...)) + + self.assertEqual(self.signature(partial(foo, 1, c=3)), + ((('b', ..., ..., 'positional_only'), + ('c', 3, ..., 'keyword_only'), + ('d', ..., ..., 'keyword_only'), + ('kwargs', ..., ..., 'var_keyword')), + ...)) + + def test_signature_on_partialmethod(self): + from functools import partialmethod + + class Spam: + def test(): + pass + ham = partialmethod(test) + + with self.assertRaisesRegex(ValueError, "has incorrect arguments"): + inspect.signature(Spam.ham) + + class Spam: + def test(it, a, *, c) -> 'spam': + pass + ham = partialmethod(test, c=1) + + self.assertEqual(self.signature(Spam.ham, eval_str=False), + ((('it', ..., ..., 'positional_or_keyword'), + ('a', ..., ..., 'positional_or_keyword'), + ('c', 1, ..., 'keyword_only')), + 'spam')) + + self.assertEqual(self.signature(Spam().ham, eval_str=False), + ((('a', ..., ..., 'positional_or_keyword'), + ('c', 1, ..., 'keyword_only')), + 'spam')) + + class Spam: + def test(self: 'anno', x): + pass + + g = partialmethod(test, 1) + + self.assertEqual(self.signature(Spam.g, eval_str=False), + ((('self', ..., 'anno', 'positional_or_keyword'),), + ...)) + + def test_signature_on_fake_partialmethod(self): + def foo(a): pass + foo.__partialmethod__ = 'spam' + self.assertEqual(str(inspect.signature(foo)), '(a)') + + def test_signature_on_decorated(self): + def decorator(func): + @functools.wraps(func) + def wrapper(*args, **kwargs) -> int: + return func(*args, **kwargs) + return wrapper + + class Foo: + @decorator + def bar(self, a, b): + pass + + bar = decorator(Foo().bar) + + self.assertEqual(self.signature(Foo.bar), + ((('self', ..., ..., "positional_or_keyword"), + ('a', ..., ..., "positional_or_keyword"), + ('b', ..., ..., "positional_or_keyword")), + ...)) + + self.assertEqual(self.signature(Foo().bar), + ((('a', ..., ..., "positional_or_keyword"), + ('b', ..., ..., "positional_or_keyword")), + ...)) + + self.assertEqual(self.signature(Foo.bar, follow_wrapped=False), + ((('args', ..., ..., "var_positional"), + ('kwargs', ..., ..., "var_keyword")), + ...)) # functools.wraps will copy __annotations__ + # from "func" to "wrapper", hence no + # return_annotation + + self.assertEqual(self.signature(bar), + ((('a', ..., ..., "positional_or_keyword"), + ('b', ..., ..., "positional_or_keyword")), + ...)) + + # Test that we handle method wrappers correctly + def decorator(func): + @functools.wraps(func) + def wrapper(*args, **kwargs) -> int: + return func(42, *args, **kwargs) + sig = inspect.signature(func) + new_params = tuple(sig.parameters.values())[1:] + wrapper.__signature__ = sig.replace(parameters=new_params) + return wrapper + + class Foo: + @decorator + def __call__(self, a, b): + pass + + self.assertEqual(self.signature(Foo.__call__), + ((('a', ..., ..., "positional_or_keyword"), + ('b', ..., ..., "positional_or_keyword")), + ...)) + + self.assertEqual(self.signature(Foo().__call__), + ((('b', ..., ..., "positional_or_keyword"),), + ...)) + + # Test we handle __signature__ partway down the wrapper stack + def wrapped_foo_call(): + pass + wrapped_foo_call.__wrapped__ = Foo.__call__ + + self.assertEqual(self.signature(wrapped_foo_call), + ((('a', ..., ..., "positional_or_keyword"), + ('b', ..., ..., "positional_or_keyword")), + ...)) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_signature_on_class(self): + class C: + def __init__(self, a): + pass + + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + class CM(type): + def __call__(cls, a): + pass + class C(metaclass=CM): + def __init__(self, b): + pass + + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('classmethod'): + class CM(type): + @classmethod + def __call__(cls, a): + return a + class C(metaclass=CM): + def __init__(self, b): + pass + + self.assertEqual(C(1), 1) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('staticmethod'): + class CM(type): + @staticmethod + def __call__(a): + return a + class C(metaclass=CM): + def __init__(self, b): + pass + + self.assertEqual(C(1), 1) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('MethodType'): + class A: + def call(self, a): + return a + class CM(type): + __call__ = A().call + class C(metaclass=CM): + def __init__(self, b): + pass + + self.assertEqual(C(1), 1) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('partial'): + class CM(type): + __call__ = functools.partial(lambda x, a: (x, a), 2) + class C(metaclass=CM): + def __init__(self, b): + pass + + with self.assertWarns(FutureWarning): + self.assertEqual(C(1), (2, 1)) + with self.assertWarns(FutureWarning): + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('partialmethod'): + class CM(type): + __call__ = functools.partialmethod(lambda self, x, a: (x, a), 2) + class C(metaclass=CM): + def __init__(self, b): + pass + + self.assertEqual(C(1), (2, 1)) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('BuiltinMethodType'): + class CM(type): + __call__ = ':'.join + class C(metaclass=CM): + def __init__(self, b): + pass + + self.assertEqual(C(['a', 'bc']), 'a:bc') + # BUG: Returns '<Signature (b)>' + with self.assertRaises(AssertionError): + self.assertEqual(self.signature(C), self.signature(''.join)) + + with self.subTest('MethodWrapperType'): + class CM(type): + __call__ = (2).__pow__ + class C(metaclass=CM): + def __init__(self, b): + pass + + self.assertEqual(C(3), 8) + self.assertEqual(C(3, 7), 1) + if not support.MISSING_C_DOCSTRINGS: + # BUG: Returns '<Signature (b)>' + with self.assertRaises(AssertionError): + self.assertEqual(self.signature(C), self.signature((0).__pow__)) + + class CM(type): + def __new__(mcls, name, bases, dct, *, foo=1): + return super().__new__(mcls, name, bases, dct) + class C(metaclass=CM): + def __init__(self, b): + pass + + self.assertEqual(self.signature(C), + ((('b', ..., ..., "positional_or_keyword"),), + ...)) + + self.assertEqual(self.signature(CM), + ((('name', ..., ..., "positional_or_keyword"), + ('bases', ..., ..., "positional_or_keyword"), + ('dct', ..., ..., "positional_or_keyword"), + ('foo', 1, ..., "keyword_only")), + ...)) + + class CMM(type): + def __new__(mcls, name, bases, dct, *, foo=1): + return super().__new__(mcls, name, bases, dct) + def __call__(cls, nm, bs, dt): + return type(nm, bs, dt) + class CM(type, metaclass=CMM): + def __new__(mcls, name, bases, dct, *, bar=2): + return super().__new__(mcls, name, bases, dct) + class C(metaclass=CM): + def __init__(self, b): + pass + + self.assertEqual(self.signature(CMM), + ((('name', ..., ..., "positional_or_keyword"), + ('bases', ..., ..., "positional_or_keyword"), + ('dct', ..., ..., "positional_or_keyword"), + ('foo', 1, ..., "keyword_only")), + ...)) + + self.assertEqual(self.signature(CM), + ((('nm', ..., ..., "positional_or_keyword"), + ('bs', ..., ..., "positional_or_keyword"), + ('dt', ..., ..., "positional_or_keyword")), + ...)) + + self.assertEqual(self.signature(C), + ((('b', ..., ..., "positional_or_keyword"),), + ...)) + + class CM(type): + def __init__(cls, name, bases, dct, *, bar=2): + return super().__init__(name, bases, dct) + class C(metaclass=CM): + def __init__(self, b): + pass + + self.assertEqual(self.signature(CM), + ((('name', ..., ..., "positional_or_keyword"), + ('bases', ..., ..., "positional_or_keyword"), + ('dct', ..., ..., "positional_or_keyword"), + ('bar', 2, ..., "keyword_only")), + ...)) + + def test_signature_on_class_with_wrapped_metaclass_call(self): + class CM(type): + @identity_wrapper + def __call__(cls, a): + pass + class C(metaclass=CM): + def __init__(self, b): + pass + + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('classmethod'): + class CM(type): + @classmethod + @identity_wrapper + def __call__(cls, a): + return a + class C(metaclass=CM): + def __init__(self, b): + pass + + self.assertEqual(C(1), 1) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('staticmethod'): + class CM(type): + @staticmethod + @identity_wrapper + def __call__(a): + return a + class C(metaclass=CM): + def __init__(self, b): + pass + + self.assertEqual(C(1), 1) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('MethodType'): + class A: + @identity_wrapper + def call(self, a): + return a + class CM(type): + __call__ = A().call + class C(metaclass=CM): + def __init__(self, b): + pass + + self.assertEqual(C(1), 1) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('descriptor'): + class CM(type): + @custom_descriptor + @identity_wrapper + def __call__(self, a): + return a + class C(metaclass=CM): + def __init__(self, b): + pass + + self.assertEqual(C(1), 1) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + self.assertEqual(self.signature(C.__call__), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + self.assertEqual(self.signature(C, follow_wrapped=False), + varargs_signature) + self.assertEqual(self.signature(C.__call__, follow_wrapped=False), + varargs_signature) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_signature_on_class_with_wrapped_init(self): + class C: + @identity_wrapper + def __init__(self, b): + pass + + C(1) # does not raise + self.assertEqual(self.signature(C), + ((('b', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('classmethod'): + class C: + @classmethod + @identity_wrapper + def __init__(cls, b): + pass + + C(1) # does not raise + self.assertEqual(self.signature(C), + ((('b', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('staticmethod'): + class C: + @staticmethod + @identity_wrapper + def __init__(b): + pass + + C(1) # does not raise + self.assertEqual(self.signature(C), + ((('b', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('MethodType'): + class A: + @identity_wrapper + def call(self, a): + pass + + class C: + __init__ = A().call + + C(1) # does not raise + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('partial'): + class C: + __init__ = functools.partial(identity_wrapper(lambda x, a: None), 2) + + with self.assertWarns(FutureWarning): + C(1) # does not raise + with self.assertWarns(FutureWarning): + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('partialmethod'): + class C: + @identity_wrapper + def _init(self, x, a): + self.a = (x, a) + __init__ = functools.partialmethod(_init, 2) + + self.assertEqual(C(1).a, (2, 1)) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('descriptor'): + class C: + @custom_descriptor + @identity_wrapper + def __init__(self, a): + pass + + C(1) # does not raise + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + self.assertEqual(self.signature(C.__init__), + ((('self', ..., ..., "positional_or_keyword"), + ('a', ..., ..., "positional_or_keyword")), + ...)) + + self.assertEqual(self.signature(C, follow_wrapped=False), + varargs_signature) + if support.MISSING_C_DOCSTRINGS: + self.assertRaisesRegex( + ValueError, "no signature found", + self.signature, C.__new__, follow_wrapped=False, + ) + else: + self.assertEqual(self.signature(C.__new__, follow_wrapped=False), + varargs_signature) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_signature_on_class_with_wrapped_new(self): + with self.subTest('FunctionType'): + class C: + @identity_wrapper + def __new__(cls, a): + return a + + self.assertEqual(C(1), 1) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('classmethod'): + class C: + @classmethod + @identity_wrapper + def __new__(cls, cls2, a): + return a + + self.assertEqual(C(1), 1) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('staticmethod'): + class C: + @staticmethod + @identity_wrapper + def __new__(cls, a): + return a + + self.assertEqual(C(1), 1) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('MethodType'): + class A: + @identity_wrapper + def call(self, cls, a): + return a + class C: + __new__ = A().call + + self.assertEqual(C(1), 1) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('partial'): + class C: + __new__ = functools.partial(identity_wrapper(lambda x, cls, a: (x, a)), 2) + + self.assertEqual(C(1), (2, 1)) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('partialmethod'): + class C: + __new__ = functools.partialmethod(identity_wrapper(lambda cls, x, a: (x, a)), 2) + + self.assertEqual(C(1), (2, 1)) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('descriptor'): + class C: + @custom_descriptor + @identity_wrapper + def __new__(cls, a): + return a + + self.assertEqual(C(1), 1) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + self.assertEqual(self.signature(C.__new__), + ((('cls', ..., ..., "positional_or_keyword"), + ('a', ..., ..., "positional_or_keyword")), + ...)) + + self.assertEqual(self.signature(C, follow_wrapped=False), + varargs_signature) + self.assertEqual(self.signature(C.__new__, follow_wrapped=False), + varargs_signature) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_signature_on_class_with_init(self): + class C: + def __init__(self, b): + pass + + C(1) # does not raise + self.assertEqual(self.signature(C), + ((('b', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('classmethod'): + class C: + @classmethod + def __init__(cls, b): + pass + + C(1) # does not raise + self.assertEqual(self.signature(C), + ((('b', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('staticmethod'): + class C: + @staticmethod + def __init__(b): + pass + + C(1) # does not raise + self.assertEqual(self.signature(C), + ((('b', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('MethodType'): + class A: + def call(self, a): + pass + class C: + __init__ = A().call + + C(1) # does not raise + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('partial'): + class C: + __init__ = functools.partial(lambda x, a: None, 2) + + with self.assertWarns(FutureWarning): + C(1) # does not raise + with self.assertWarns(FutureWarning): + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('partialmethod'): + class C: + def _init(self, x, a): + self.a = (x, a) + __init__ = functools.partialmethod(_init, 2) + + self.assertEqual(C(1).a, (2, 1)) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_signature_on_class_with_new(self): + with self.subTest('FunctionType'): + class C: + def __new__(cls, a): + return a + + self.assertEqual(C(1), 1) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('classmethod'): + class C: + @classmethod + def __new__(cls, cls2, a): + return a + + self.assertEqual(C(1), 1) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('staticmethod'): + class C: + @staticmethod + def __new__(cls, a): + return a + + self.assertEqual(C(1), 1) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('MethodType'): + class A: + def call(self, cls, a): + return a + class C: + __new__ = A().call + + self.assertEqual(C(1), 1) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('partial'): + class C: + __new__ = functools.partial(lambda x, cls, a: (x, a), 2) + + self.assertEqual(C(1), (2, 1)) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('partialmethod'): + class C: + __new__ = functools.partialmethod(lambda cls, x, a: (x, a), 2) + + self.assertEqual(C(1), (2, 1)) + self.assertEqual(self.signature(C), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('BuiltinMethodType'): + class C: + __new__ = str.__subclasscheck__ + + self.assertEqual(C(), False) + # TODO: Support BuiltinMethodType + # self.assertEqual(self.signature(C), ((), ...)) + self.assertRaises(ValueError, self.signature, C) + + with self.subTest('MethodWrapperType'): + class C: + __new__ = type.__or__.__get__(int, type) + + self.assertEqual(C(), C | int) + # TODO: Support MethodWrapperType + # self.assertEqual(self.signature(C), ((), ...)) + self.assertRaises(ValueError, self.signature, C) + + # TODO: Test ClassMethodDescriptorType + + with self.subTest('MethodDescriptorType'): + class C: + __new__ = type.__dict__['__subclasscheck__'] + + self.assertEqual(C(C), True) + self.assertEqual(self.signature(C), self.signature(C.__subclasscheck__)) + + with self.subTest('WrapperDescriptorType'): + class C: + __new__ = type.__or__ + + self.assertEqual(C(int), C | int) + # TODO: Support WrapperDescriptorType + # self.assertEqual(self.signature(C), self.signature(C.__or__)) + self.assertRaises(ValueError, self.signature, C) + + def test_signature_on_subclass(self): + class A: + def __new__(cls, a=1, *args, **kwargs): + return object.__new__(cls) + class B(A): + def __init__(self, b): + pass + class C(A): + def __new__(cls, a=1, b=2, *args, **kwargs): + return object.__new__(cls) + class D(A): + pass + + self.assertEqual(self.signature(B), + ((('b', ..., ..., "positional_or_keyword"),), + ...)) + self.assertEqual(self.signature(C), + ((('a', 1, ..., 'positional_or_keyword'), + ('b', 2, ..., 'positional_or_keyword'), + ('args', ..., ..., 'var_positional'), + ('kwargs', ..., ..., 'var_keyword')), + ...)) + self.assertEqual(self.signature(D), + ((('a', 1, ..., 'positional_or_keyword'), + ('args', ..., ..., 'var_positional'), + ('kwargs', ..., ..., 'var_keyword')), + ...)) + + def test_signature_on_generic_subclass(self): + from typing import Generic, TypeVar + + T = TypeVar('T') + + class A(Generic[T]): + def __init__(self, *, a: int) -> None: + pass + + self.assertEqual(self.signature(A), + ((('a', ..., int, 'keyword_only'),), + None)) + + @unittest.skipIf(MISSING_C_DOCSTRINGS, + "Signature information for builtins requires docstrings") + def test_signature_on_class_without_init(self): + # Test classes without user-defined __init__ or __new__ + class C: pass + self.assertEqual(str(inspect.signature(C)), '()') + class D(C): pass + self.assertEqual(str(inspect.signature(D)), '()') + + # Test meta-classes without user-defined __init__ or __new__ + class C(type): pass + class D(C): pass + self.assertEqual(C('A', (), {}).__name__, 'A') + # TODO: Support type. + with self.assertRaisesRegex(ValueError, "callable.*is not supported"): + self.assertEqual(inspect.signature(C), None) + self.assertEqual(D('A', (), {}).__name__, 'A') + with self.assertRaisesRegex(ValueError, "callable.*is not supported"): + self.assertEqual(inspect.signature(D), None) + + @unittest.skipIf(MISSING_C_DOCSTRINGS, + "Signature information for builtins requires docstrings") + @unittest.skipIf(_pickle is None, "requires _pickle") + def test_signature_on_builtin_class(self): + expected = ('(file, protocol=None, fix_imports=True, ' + 'buffer_callback=None)') + self.assertEqual(str(inspect.signature(_pickle.Pickler)), expected) + + class P(_pickle.Pickler): pass + class EmptyTrait: pass + class P2(EmptyTrait, P): pass + self.assertEqual(str(inspect.signature(P)), expected) + self.assertEqual(str(inspect.signature(P2)), expected) + + class P3(P2): + def __init__(self, spam): + pass + self.assertEqual(str(inspect.signature(P3)), '(spam)') + + class MetaP(type): + def __call__(cls, foo, bar): + pass + class P4(P2, metaclass=MetaP): + pass + self.assertEqual(str(inspect.signature(P4)), '(foo, bar)') + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_signature_on_callable_objects(self): + class Foo: + def __call__(self, a): + pass + + self.assertEqual(self.signature(Foo()), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + class Spam: + pass + with self.assertRaisesRegex(TypeError, "is not a callable object"): + inspect.signature(Spam()) + + class Bar(Spam, Foo): + pass + + self.assertEqual(self.signature(Bar()), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('classmethod'): + class C: + @classmethod + def __call__(cls, a): + pass + + self.assertEqual(self.signature(C()), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('staticmethod'): + class C: + @staticmethod + def __call__(a): + pass + + self.assertEqual(self.signature(C()), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('MethodType'): + class A: + def call(self, a): + return a + class C: + __call__ = A().call + + self.assertEqual(C()(1), 1) + self.assertEqual(self.signature(C()), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('partial'): + class C: + __call__ = functools.partial(lambda x, a: (x, a), 2) + + c = C() + with self.assertWarns(FutureWarning): + self.assertEqual(c(1), (2, 1)) + with self.assertWarns(FutureWarning): + self.assertEqual(self.signature(c), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('partialmethod'): + class C: + __call__ = functools.partialmethod(lambda self, x, a: (x, a), 2) + + self.assertEqual(C()(1), (2, 1)) + self.assertEqual(self.signature(C()), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + with self.subTest('BuiltinMethodType'): + class C: + __call__ = ':'.join + + self.assertEqual(C()(['a', 'bc']), 'a:bc') + self.assertEqual(self.signature(C()), self.signature(''.join)) + + with self.subTest('MethodWrapperType'): + class C: + __call__ = (2).__pow__ + + self.assertEqual(C()(3), 8) + if not support.MISSING_C_DOCSTRINGS: + self.assertEqual(self.signature(C()), self.signature((0).__pow__)) + + with self.subTest('ClassMethodDescriptorType'): + class C(dict): + __call__ = dict.__dict__['fromkeys'] + + res = C()([1, 2], 3) + self.assertEqual(res, {1: 3, 2: 3}) + self.assertEqual(type(res), C) + if not support.MISSING_C_DOCSTRINGS: + self.assertEqual(self.signature(C()), self.signature(dict.fromkeys)) + + with self.subTest('MethodDescriptorType'): + class C(str): + __call__ = str.join + + self.assertEqual(C(':')(['a', 'bc']), 'a:bc') + self.assertEqual(self.signature(C()), self.signature(''.join)) + + with self.subTest('WrapperDescriptorType'): + class C(int): + __call__ = int.__pow__ + + self.assertEqual(C(2)(3), 8) + if not support.MISSING_C_DOCSTRINGS: + self.assertEqual(self.signature(C()), self.signature((0).__pow__)) + + with self.subTest('MemberDescriptorType'): + class C: + __slots__ = '__call__' + c = C() + c.__call__ = lambda a: a + self.assertEqual(c(1), 1) + self.assertEqual(self.signature(c), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + + def test_signature_on_callable_objects_with_text_signature_attr(self): + class C: + __text_signature__ = '(a, /, b, c=True)' + def __call__(self, *args, **kwargs): + pass + + if not support.MISSING_C_DOCSTRINGS: + self.assertEqual(self.signature(C), ((), ...)) + self.assertEqual(self.signature(C()), + ((('a', ..., ..., "positional_only"), + ('b', ..., ..., "positional_or_keyword"), + ('c', True, ..., "positional_or_keyword"), + ), + ...)) + + c = C() + c.__text_signature__ = '(x, y)' + self.assertEqual(self.signature(c), + ((('x', ..., ..., "positional_or_keyword"), + ('y', ..., ..., "positional_or_keyword"), + ), + ...)) + + def test_signature_on_wrapper(self): + class Wrapper: + def __call__(self, b): + pass + wrapper = Wrapper() + wrapper.__wrapped__ = lambda a: None + self.assertEqual(self.signature(wrapper), + ((('a', ..., ..., "positional_or_keyword"),), + ...)) + # wrapper loop: + wrapper = Wrapper() + wrapper.__wrapped__ = wrapper + with self.assertRaisesRegex(ValueError, 'wrapper loop'): + self.signature(wrapper) + + def test_signature_on_lambdas(self): + self.assertEqual(self.signature((lambda a=10: a)), + ((('a', 10, ..., "positional_or_keyword"),), + ...)) + + def test_signature_on_mocks(self): + # https://github.com/python/cpython/issues/96127 + for mock in ( + unittest.mock.Mock(), + unittest.mock.AsyncMock(), + unittest.mock.MagicMock(), + ): + with self.subTest(mock=mock): + self.assertEqual(str(inspect.signature(mock)), '(*args, **kwargs)') + + def test_signature_on_noncallable_mocks(self): + for mock in ( + unittest.mock.NonCallableMock(), + unittest.mock.NonCallableMagicMock(), + ): + with self.subTest(mock=mock): + with self.assertRaises(TypeError): + inspect.signature(mock) + + def test_signature_equality(self): + def foo(a, *, b:int) -> float: pass + self.assertFalse(inspect.signature(foo) == 42) + self.assertTrue(inspect.signature(foo) != 42) + self.assertTrue(inspect.signature(foo) == ALWAYS_EQ) + self.assertFalse(inspect.signature(foo) != ALWAYS_EQ) + + def bar(a, *, b:int) -> float: pass + self.assertTrue(inspect.signature(foo) == inspect.signature(bar)) + self.assertFalse(inspect.signature(foo) != inspect.signature(bar)) + self.assertEqual( + hash(inspect.signature(foo)), hash(inspect.signature(bar))) + + def bar(a, *, b:int) -> int: pass + self.assertFalse(inspect.signature(foo) == inspect.signature(bar)) + self.assertTrue(inspect.signature(foo) != inspect.signature(bar)) + self.assertNotEqual( + hash(inspect.signature(foo)), hash(inspect.signature(bar))) + + def bar(a, *, b:int): pass + self.assertFalse(inspect.signature(foo) == inspect.signature(bar)) + self.assertTrue(inspect.signature(foo) != inspect.signature(bar)) + self.assertNotEqual( + hash(inspect.signature(foo)), hash(inspect.signature(bar))) + + def bar(a, *, b:int=42) -> float: pass + self.assertFalse(inspect.signature(foo) == inspect.signature(bar)) + self.assertTrue(inspect.signature(foo) != inspect.signature(bar)) + self.assertNotEqual( + hash(inspect.signature(foo)), hash(inspect.signature(bar))) + + def bar(a, *, c) -> float: pass + self.assertFalse(inspect.signature(foo) == inspect.signature(bar)) + self.assertTrue(inspect.signature(foo) != inspect.signature(bar)) + self.assertNotEqual( + hash(inspect.signature(foo)), hash(inspect.signature(bar))) + + def bar(a, b:int) -> float: pass + self.assertFalse(inspect.signature(foo) == inspect.signature(bar)) + self.assertTrue(inspect.signature(foo) != inspect.signature(bar)) + self.assertNotEqual( + hash(inspect.signature(foo)), hash(inspect.signature(bar))) + def spam(b:int, a) -> float: pass + self.assertFalse(inspect.signature(spam) == inspect.signature(bar)) + self.assertTrue(inspect.signature(spam) != inspect.signature(bar)) + self.assertNotEqual( + hash(inspect.signature(spam)), hash(inspect.signature(bar))) + + def foo(*, a, b, c): pass + def bar(*, c, b, a): pass + self.assertTrue(inspect.signature(foo) == inspect.signature(bar)) + self.assertFalse(inspect.signature(foo) != inspect.signature(bar)) + self.assertEqual( + hash(inspect.signature(foo)), hash(inspect.signature(bar))) + + def foo(*, a=1, b, c): pass + def bar(*, c, b, a=1): pass + self.assertTrue(inspect.signature(foo) == inspect.signature(bar)) + self.assertFalse(inspect.signature(foo) != inspect.signature(bar)) + self.assertEqual( + hash(inspect.signature(foo)), hash(inspect.signature(bar))) + + def foo(pos, *, a=1, b, c): pass + def bar(pos, *, c, b, a=1): pass + self.assertTrue(inspect.signature(foo) == inspect.signature(bar)) + self.assertFalse(inspect.signature(foo) != inspect.signature(bar)) + self.assertEqual( + hash(inspect.signature(foo)), hash(inspect.signature(bar))) + + def foo(pos, *, a, b, c): pass + def bar(pos, *, c, b, a=1): pass + self.assertFalse(inspect.signature(foo) == inspect.signature(bar)) + self.assertTrue(inspect.signature(foo) != inspect.signature(bar)) + self.assertNotEqual( + hash(inspect.signature(foo)), hash(inspect.signature(bar))) + + def foo(pos, *args, a=42, b, c, **kwargs:int): pass + def bar(pos, *args, c, b, a=42, **kwargs:int): pass + self.assertTrue(inspect.signature(foo) == inspect.signature(bar)) + self.assertFalse(inspect.signature(foo) != inspect.signature(bar)) + self.assertEqual( + hash(inspect.signature(foo)), hash(inspect.signature(bar))) + + def test_signature_hashable(self): + S = inspect.Signature + P = inspect.Parameter + + def foo(a): pass + foo_sig = inspect.signature(foo) + + manual_sig = S(parameters=[P('a', P.POSITIONAL_OR_KEYWORD)]) + + self.assertEqual(hash(foo_sig), hash(manual_sig)) + self.assertNotEqual(hash(foo_sig), + hash(manual_sig.replace(return_annotation='spam'))) + + def bar(a) -> 1: pass + self.assertNotEqual(hash(foo_sig), hash(inspect.signature(bar))) + + def foo(a={}): pass + with self.assertRaisesRegex(TypeError, 'unhashable type'): + hash(inspect.signature(foo)) + + def foo(a) -> {}: pass + with self.assertRaisesRegex(TypeError, 'unhashable type'): + hash(inspect.signature(foo)) + + def test_signature_str(self): + def foo(a:int=1, *, b, c=None, **kwargs) -> 42: + pass + self.assertEqual(str(inspect.signature(foo)), + '(a: int = 1, *, b, c=None, **kwargs) -> 42') + self.assertEqual(str(inspect.signature(foo)), + inspect.signature(foo).format()) + + def foo(a:int=1, *args, b, c=None, **kwargs) -> 42: + pass + self.assertEqual(str(inspect.signature(foo)), + '(a: int = 1, *args, b, c=None, **kwargs) -> 42') + self.assertEqual(str(inspect.signature(foo)), + inspect.signature(foo).format()) + + def foo(): + pass + self.assertEqual(str(inspect.signature(foo)), '()') + self.assertEqual(str(inspect.signature(foo)), + inspect.signature(foo).format()) + + def foo(a: list[str]) -> tuple[str, float]: + pass + self.assertEqual(str(inspect.signature(foo)), + '(a: list[str]) -> tuple[str, float]') + self.assertEqual(str(inspect.signature(foo)), + inspect.signature(foo).format()) + + from typing import Tuple + def foo(a: list[str]) -> Tuple[str, float]: + pass + self.assertEqual(str(inspect.signature(foo)), + '(a: list[str]) -> Tuple[str, float]') + self.assertEqual(str(inspect.signature(foo)), + inspect.signature(foo).format()) + + def test_signature_str_positional_only(self): + P = inspect.Parameter + S = inspect.Signature + + def test(a_po, /, *, b, **kwargs): + return a_po, kwargs + + self.assertEqual(str(inspect.signature(test)), + '(a_po, /, *, b, **kwargs)') + self.assertEqual(str(inspect.signature(test)), + inspect.signature(test).format()) + + test = S(parameters=[P('foo', P.POSITIONAL_ONLY)]) + self.assertEqual(str(test), '(foo, /)') + self.assertEqual(str(test), test.format()) + + test = S(parameters=[P('foo', P.POSITIONAL_ONLY), + P('bar', P.VAR_KEYWORD)]) + self.assertEqual(str(test), '(foo, /, **bar)') + self.assertEqual(str(test), test.format()) + + test = S(parameters=[P('foo', P.POSITIONAL_ONLY), + P('bar', P.VAR_POSITIONAL)]) + self.assertEqual(str(test), '(foo, /, *bar)') + self.assertEqual(str(test), test.format()) + + def test_signature_format(self): + from typing import Annotated, Literal + + def func(x: Annotated[int, 'meta'], y: Literal['a', 'b'], z: 'LiteralString'): + pass + + expected_singleline = "(x: Annotated[int, 'meta'], y: Literal['a', 'b'], z: 'LiteralString')" + expected_multiline = """( + x: Annotated[int, 'meta'], + y: Literal['a', 'b'], + z: 'LiteralString' +)""" + self.assertEqual( + inspect.signature(func).format(), + expected_singleline, + ) + self.assertEqual( + inspect.signature(func).format(max_width=None), + expected_singleline, + ) + self.assertEqual( + inspect.signature(func).format(max_width=len(expected_singleline)), + expected_singleline, + ) + self.assertEqual( + inspect.signature(func).format(max_width=len(expected_singleline) - 1), + expected_multiline, + ) + self.assertEqual( + inspect.signature(func).format(max_width=0), + expected_multiline, + ) + self.assertEqual( + inspect.signature(func).format(max_width=-1), + expected_multiline, + ) + + def test_signature_format_all_arg_types(self): + from typing import Annotated, Literal + + def func( + x: Annotated[int, 'meta'], + /, + y: Literal['a', 'b'], + *, + z: 'LiteralString', + **kwargs: object, + ) -> None: + pass + + expected_multiline = """( + x: Annotated[int, 'meta'], + /, + y: Literal['a', 'b'], + *, + z: 'LiteralString', + **kwargs: object +) -> None""" + self.assertEqual( + inspect.signature(func).format(max_width=-1), + expected_multiline, + ) + + def test_signature_replace_parameters(self): + def test(a, b) -> 42: + pass + + sig = inspect.signature(test) + parameters = sig.parameters + sig = sig.replace(parameters=list(parameters.values())[1:]) + self.assertEqual(list(sig.parameters), ['b']) + self.assertEqual(sig.parameters['b'], parameters['b']) + self.assertEqual(sig.return_annotation, 42) + sig = sig.replace(parameters=()) + self.assertEqual(dict(sig.parameters), {}) + + sig = inspect.signature(test) + parameters = sig.parameters + sig = copy.replace(sig, parameters=list(parameters.values())[1:]) + self.assertEqual(list(sig.parameters), ['b']) + self.assertEqual(sig.parameters['b'], parameters['b']) + self.assertEqual(sig.return_annotation, 42) + sig = copy.replace(sig, parameters=()) + self.assertEqual(dict(sig.parameters), {}) + + def test_signature_replace_anno(self): + def test() -> 42: + pass + + sig = inspect.signature(test) + sig = sig.replace(return_annotation=None) + self.assertIs(sig.return_annotation, None) + sig = sig.replace(return_annotation=sig.empty) + self.assertIs(sig.return_annotation, sig.empty) + sig = sig.replace(return_annotation=42) + self.assertEqual(sig.return_annotation, 42) + self.assertEqual(sig, inspect.signature(test)) + + sig = inspect.signature(test) + sig = copy.replace(sig, return_annotation=None) + self.assertIs(sig.return_annotation, None) + sig = copy.replace(sig, return_annotation=sig.empty) + self.assertIs(sig.return_annotation, sig.empty) + sig = copy.replace(sig, return_annotation=42) + self.assertEqual(sig.return_annotation, 42) + self.assertEqual(sig, inspect.signature(test)) + + def test_signature_replaced(self): + def test(): + pass + + spam_param = inspect.Parameter('spam', inspect.Parameter.POSITIONAL_ONLY) + sig = test.__signature__ = inspect.Signature(parameters=(spam_param,)) + self.assertEqual(sig, inspect.signature(test)) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_signature_on_mangled_parameters(self): + class Spam: + def foo(self, __p1:1=2, *, __p2:2=3): + pass + class Ham(Spam): + pass + + self.assertEqual(self.signature(Spam.foo), + ((('self', ..., ..., "positional_or_keyword"), + ('_Spam__p1', 2, 1, "positional_or_keyword"), + ('_Spam__p2', 3, 2, "keyword_only")), + ...)) + + self.assertEqual(self.signature(Spam.foo), + self.signature(Ham.foo)) + + def test_signature_from_callable_python_obj(self): + class MySignature(inspect.Signature): pass + def foo(a, *, b:1): pass + foo_sig = MySignature.from_callable(foo) + self.assertIsInstance(foo_sig, MySignature) + + @unittest.skipIf(MISSING_C_DOCSTRINGS, + "Signature information for builtins requires docstrings") + def test_signature_from_callable_class(self): + # A regression test for a class inheriting its signature from `object`. + class MySignature(inspect.Signature): pass + class foo: pass + foo_sig = MySignature.from_callable(foo) + self.assertIsInstance(foo_sig, MySignature) + + @unittest.skipIf(MISSING_C_DOCSTRINGS, + "Signature information for builtins requires docstrings") + @unittest.skipIf(_pickle is None, "requires _pickle") + def test_signature_from_callable_builtin_obj(self): + class MySignature(inspect.Signature): pass + sig = MySignature.from_callable(_pickle.Pickler) + self.assertIsInstance(sig, MySignature) + + def test_signature_definition_order_preserved_on_kwonly(self): + for fn in signatures_with_lexicographic_keyword_only_parameters(): + signature = inspect.signature(fn) + l = list(signature.parameters) + sorted_l = sorted(l) + self.assertTrue(l) + self.assertEqual(l, sorted_l) + signature = inspect.signature(unsorted_keyword_only_parameters_fn) + l = list(signature.parameters) + self.assertEqual(l, unsorted_keyword_only_parameters) + + def test_signater_parameters_is_ordered(self): + p1 = inspect.signature(lambda x, y: None).parameters + p2 = inspect.signature(lambda y, x: None).parameters + self.assertNotEqual(p1, p2) + + def test_signature_annotations_with_local_namespaces(self): + class Foo: ... + def func(foo: Foo) -> int: pass + def func2(foo: Foo, bar: 'Bar') -> int: pass + + for signature_func in (inspect.signature, inspect.Signature.from_callable): + with self.subTest(signature_func = signature_func): + sig1 = signature_func(func) + self.assertEqual(sig1.return_annotation, int) + self.assertEqual(sig1.parameters['foo'].annotation, Foo) + + sig2 = signature_func(func, locals=locals()) + self.assertEqual(sig2.return_annotation, int) + self.assertEqual(sig2.parameters['foo'].annotation, Foo) + + sig3 = signature_func(func2, globals={'Bar': int}, locals=locals()) + self.assertEqual(sig3.return_annotation, int) + self.assertEqual(sig3.parameters['foo'].annotation, Foo) + self.assertEqual(sig3.parameters['bar'].annotation, 'Bar') + + def test_signature_eval_str(self): + isa = inspect_stringized_annotations + sig = inspect.Signature + par = inspect.Parameter + PORK = inspect.Parameter.POSITIONAL_OR_KEYWORD + for signature_func in (inspect.signature, inspect.Signature.from_callable): + with self.subTest(signature_func = signature_func): + self.assertEqual( + signature_func(isa.MyClass), + sig( + parameters=( + par('a', PORK), + par('b', PORK), + ))) + self.assertEqual( + signature_func(isa.function), + sig( + return_annotation='MyClass', + parameters=( + par('a', PORK, annotation='int'), + par('b', PORK, annotation='str'), + ))) + self.assertEqual( + signature_func(isa.function2), + sig( + return_annotation='MyClass', + parameters=( + par('a', PORK, annotation='int'), + par('b', PORK, annotation="'str'"), + par('c', PORK, annotation="MyClass"), + ))) + self.assertEqual( + signature_func(isa.function3), + sig( + parameters=( + par('a', PORK, annotation="'int'"), + par('b', PORK, annotation="'str'"), + par('c', PORK, annotation="'MyClass'"), + ))) + + if not MISSING_C_DOCSTRINGS: + self.assertEqual(signature_func(isa.UnannotatedClass), sig()) + self.assertEqual(signature_func(isa.unannotated_function), + sig( + parameters=( + par('a', PORK), + par('b', PORK), + par('c', PORK), + ))) + + self.assertEqual( + signature_func(isa.MyClass, eval_str=True), + sig( + parameters=( + par('a', PORK), + par('b', PORK), + ))) + self.assertEqual( + signature_func(isa.function, eval_str=True), + sig( + return_annotation=isa.MyClass, + parameters=( + par('a', PORK, annotation=int), + par('b', PORK, annotation=str), + ))) + self.assertEqual( + signature_func(isa.function2, eval_str=True), + sig( + return_annotation=isa.MyClass, + parameters=( + par('a', PORK, annotation=int), + par('b', PORK, annotation='str'), + par('c', PORK, annotation=isa.MyClass), + ))) + self.assertEqual( + signature_func(isa.function3, eval_str=True), + sig( + parameters=( + par('a', PORK, annotation='int'), + par('b', PORK, annotation='str'), + par('c', PORK, annotation='MyClass'), + ))) + + globalns = {'int': float, 'str': complex} + localns = {'str': tuple, 'MyClass': dict} + with self.assertRaises(NameError): + signature_func(isa.function, eval_str=True, globals=globalns) + + self.assertEqual( + signature_func(isa.function, eval_str=True, locals=localns), + sig( + return_annotation=dict, + parameters=( + par('a', PORK, annotation=int), + par('b', PORK, annotation=tuple), + ))) + + self.assertEqual( + signature_func(isa.function, eval_str=True, globals=globalns, locals=localns), + sig( + return_annotation=dict, + parameters=( + par('a', PORK, annotation=float), + par('b', PORK, annotation=tuple), + ))) + + def test_signature_none_annotation(self): + class funclike: + # Has to be callable, and have correct + # __code__, __annotations__, __defaults__, __name__, + # and __kwdefaults__ attributes + + def __init__(self, func): + self.__name__ = func.__name__ + self.__code__ = func.__code__ + self.__annotations__ = func.__annotations__ + self.__defaults__ = func.__defaults__ + self.__kwdefaults__ = func.__kwdefaults__ + self.func = func + + def __call__(self, *args, **kwargs): + return self.func(*args, **kwargs) + + def foo(): pass + foo = funclike(foo) + foo.__annotations__ = None + for signature_func in (inspect.signature, inspect.Signature.from_callable): + with self.subTest(signature_func = signature_func): + self.assertEqual(signature_func(foo), inspect.Signature()) + self.assertEqual(inspect.get_annotations(foo), {}) + + def test_signature_as_str(self): + self.maxDiff = None + class S: + __signature__ = '(a, b=2)' + + self.assertEqual(self.signature(S), + ((('a', ..., ..., 'positional_or_keyword'), + ('b', 2, ..., 'positional_or_keyword')), + ...)) + + def test_signature_as_callable(self): + # __signature__ should be either a staticmethod or a bound classmethod + class S: + @classmethod + def __signature__(cls): + return '(a, b=2)' + + self.assertEqual(self.signature(S), + ((('a', ..., ..., 'positional_or_keyword'), + ('b', 2, ..., 'positional_or_keyword')), + ...)) + + class S: + @staticmethod + def __signature__(): + return '(a, b=2)' + + self.assertEqual(self.signature(S), + ((('a', ..., ..., 'positional_or_keyword'), + ('b', 2, ..., 'positional_or_keyword')), + ...)) + + def test_signature_on_derived_classes(self): + # gh-105080: Make sure that signatures are consistent on derived classes + + class B: + def __new__(self, *args, **kwargs): + return super().__new__(self) + def __init__(self, value): + self.value = value + + class D1(B): + def __init__(self, value): + super().__init__(value) + + class D2(D1): + pass + + self.assertEqual(inspect.signature(D2), inspect.signature(D1)) + + def test_signature_on_non_comparable(self): + class NoncomparableCallable: + def __call__(self, a): + pass + def __eq__(self, other): + 1/0 + self.assertEqual(self.signature(NoncomparableCallable()), + ((('a', ..., ..., 'positional_or_keyword'),), + ...)) + + +class TestParameterObject(unittest.TestCase): + def test_signature_parameter_kinds(self): + P = inspect.Parameter + self.assertTrue(P.POSITIONAL_ONLY < P.POSITIONAL_OR_KEYWORD < \ + P.VAR_POSITIONAL < P.KEYWORD_ONLY < P.VAR_KEYWORD) + + self.assertEqual(str(P.POSITIONAL_ONLY), 'POSITIONAL_ONLY') + self.assertTrue('POSITIONAL_ONLY' in repr(P.POSITIONAL_ONLY)) + + def test_signature_parameter_object(self): + p = inspect.Parameter('foo', default=10, + kind=inspect.Parameter.POSITIONAL_ONLY) + self.assertEqual(p.name, 'foo') + self.assertEqual(p.default, 10) + self.assertIs(p.annotation, p.empty) + self.assertEqual(p.kind, inspect.Parameter.POSITIONAL_ONLY) + + with self.assertRaisesRegex(ValueError, "value '123' is " + "not a valid Parameter.kind"): + inspect.Parameter('foo', default=10, kind='123') + + with self.assertRaisesRegex(ValueError, 'not a valid parameter name'): + inspect.Parameter('1', kind=inspect.Parameter.VAR_KEYWORD) + + with self.assertRaisesRegex(ValueError, 'not a valid parameter name'): + inspect.Parameter('from', kind=inspect.Parameter.VAR_KEYWORD) + + with self.assertRaisesRegex(TypeError, 'name must be a str'): + inspect.Parameter(None, kind=inspect.Parameter.VAR_KEYWORD) + + with self.assertRaisesRegex(ValueError, + 'is not a valid parameter name'): + inspect.Parameter('$', kind=inspect.Parameter.VAR_KEYWORD) + + with self.assertRaisesRegex(ValueError, + 'is not a valid parameter name'): + inspect.Parameter('.a', kind=inspect.Parameter.VAR_KEYWORD) + + with self.assertRaisesRegex(ValueError, 'cannot have default values'): + inspect.Parameter('a', default=42, + kind=inspect.Parameter.VAR_KEYWORD) + + with self.assertRaisesRegex(ValueError, 'cannot have default values'): + inspect.Parameter('a', default=42, + kind=inspect.Parameter.VAR_POSITIONAL) + + p = inspect.Parameter('a', default=42, + kind=inspect.Parameter.POSITIONAL_OR_KEYWORD) + with self.assertRaisesRegex(ValueError, 'cannot have default values'): + p.replace(kind=inspect.Parameter.VAR_POSITIONAL) + + self.assertTrue(repr(p).startswith('<Parameter')) + self.assertTrue('"a=42"' in repr(p)) + + def test_signature_parameter_hashable(self): + P = inspect.Parameter + foo = P('foo', kind=P.POSITIONAL_ONLY) + self.assertEqual(hash(foo), hash(P('foo', kind=P.POSITIONAL_ONLY))) + self.assertNotEqual(hash(foo), hash(P('foo', kind=P.POSITIONAL_ONLY, + default=42))) + self.assertNotEqual(hash(foo), + hash(foo.replace(kind=P.VAR_POSITIONAL))) + + def test_signature_parameter_equality(self): + P = inspect.Parameter + p = P('foo', default=42, kind=inspect.Parameter.KEYWORD_ONLY) + + self.assertTrue(p == p) + self.assertFalse(p != p) + self.assertFalse(p == 42) + self.assertTrue(p != 42) + self.assertTrue(p == ALWAYS_EQ) + self.assertFalse(p != ALWAYS_EQ) + + self.assertTrue(p == P('foo', default=42, + kind=inspect.Parameter.KEYWORD_ONLY)) + self.assertFalse(p != P('foo', default=42, + kind=inspect.Parameter.KEYWORD_ONLY)) + + def test_signature_parameter_replace(self): + p = inspect.Parameter('foo', default=42, + kind=inspect.Parameter.KEYWORD_ONLY) + + self.assertIsNot(p.replace(), p) + self.assertEqual(p.replace(), p) + self.assertIsNot(copy.replace(p), p) + self.assertEqual(copy.replace(p), p) + + p2 = p.replace(annotation=1) + self.assertEqual(p2.annotation, 1) + p2 = p2.replace(annotation=p2.empty) + self.assertEqual(p2, p) + p3 = copy.replace(p, annotation=1) + self.assertEqual(p3.annotation, 1) + p3 = copy.replace(p3, annotation=p3.empty) + self.assertEqual(p3, p) + + p2 = p2.replace(name='bar') + self.assertEqual(p2.name, 'bar') + self.assertNotEqual(p2, p) + p3 = copy.replace(p3, name='bar') + self.assertEqual(p3.name, 'bar') + self.assertNotEqual(p3, p) + + with self.assertRaisesRegex(ValueError, + 'name is a required attribute'): + p2 = p2.replace(name=p2.empty) + with self.assertRaisesRegex(ValueError, + 'name is a required attribute'): + p3 = copy.replace(p3, name=p3.empty) + + p2 = p2.replace(name='foo', default=None) + self.assertIs(p2.default, None) + self.assertNotEqual(p2, p) + p3 = copy.replace(p3, name='foo', default=None) + self.assertIs(p3.default, None) + self.assertNotEqual(p3, p) + + p2 = p2.replace(name='foo', default=p2.empty) + self.assertIs(p2.default, p2.empty) + p3 = copy.replace(p3, name='foo', default=p3.empty) + self.assertIs(p3.default, p3.empty) + + p2 = p2.replace(default=42, kind=p2.POSITIONAL_OR_KEYWORD) + self.assertEqual(p2.kind, p2.POSITIONAL_OR_KEYWORD) + self.assertNotEqual(p2, p) + p3 = copy.replace(p3, default=42, kind=p3.POSITIONAL_OR_KEYWORD) + self.assertEqual(p3.kind, p3.POSITIONAL_OR_KEYWORD) + self.assertNotEqual(p3, p) + + with self.assertRaisesRegex(ValueError, + "value <class 'inspect._empty'> " + "is not a valid Parameter.kind"): + p2 = p2.replace(kind=p2.empty) + with self.assertRaisesRegex(ValueError, + "value <class 'inspect._empty'> " + "is not a valid Parameter.kind"): + p3 = copy.replace(p3, kind=p3.empty) + + p2 = p2.replace(kind=p2.KEYWORD_ONLY) + self.assertEqual(p2, p) + p3 = copy.replace(p3, kind=p3.KEYWORD_ONLY) + self.assertEqual(p3, p) + + def test_signature_parameter_positional_only(self): + with self.assertRaisesRegex(TypeError, 'name must be a str'): + inspect.Parameter(None, kind=inspect.Parameter.POSITIONAL_ONLY) + + @cpython_only + def test_signature_parameter_implicit(self): + with self.assertRaisesRegex(ValueError, + 'implicit arguments must be passed as ' + 'positional or keyword arguments, ' + 'not positional-only'): + inspect.Parameter('.0', kind=inspect.Parameter.POSITIONAL_ONLY) + + param = inspect.Parameter( + '.0', kind=inspect.Parameter.POSITIONAL_OR_KEYWORD) + self.assertEqual(param.kind, inspect.Parameter.POSITIONAL_ONLY) + self.assertEqual(param.name, 'implicit0') + + def test_signature_parameter_immutability(self): + p = inspect.Parameter('spam', kind=inspect.Parameter.KEYWORD_ONLY) + + with self.assertRaises(AttributeError): + p.foo = 'bar' + + with self.assertRaises(AttributeError): + p.kind = 123 + + +class TestSignatureBind(unittest.TestCase): + @staticmethod + def call(func, *args, **kwargs): + sig = inspect.signature(func) + ba = sig.bind(*args, **kwargs) + # Prevent unexpected success of assertRaises(TypeError, ...) + try: + return func(*ba.args, **ba.kwargs) + except TypeError as e: + raise AssertionError from e + + def test_signature_bind_empty(self): + def test(): + return 42 + + self.assertEqual(self.call(test), 42) + with self.assertRaisesRegex(TypeError, 'too many positional arguments'): + self.call(test, 1) + with self.assertRaisesRegex(TypeError, 'too many positional arguments'): + self.call(test, 1, spam=10) + with self.assertRaisesRegex( + TypeError, "got an unexpected keyword argument 'spam'"): + + self.call(test, spam=1) + + def test_signature_bind_var(self): + def test(*args, **kwargs): + return args, kwargs + + self.assertEqual(self.call(test), ((), {})) + self.assertEqual(self.call(test, 1), ((1,), {})) + self.assertEqual(self.call(test, 1, 2), ((1, 2), {})) + self.assertEqual(self.call(test, foo='bar'), ((), {'foo': 'bar'})) + self.assertEqual(self.call(test, 1, foo='bar'), ((1,), {'foo': 'bar'})) + self.assertEqual(self.call(test, args=10), ((), {'args': 10})) + self.assertEqual(self.call(test, 1, 2, foo='bar'), + ((1, 2), {'foo': 'bar'})) + + def test_signature_bind_just_args(self): + def test(a, b, c): + return a, b, c + + self.assertEqual(self.call(test, 1, 2, 3), (1, 2, 3)) + + with self.assertRaisesRegex(TypeError, 'too many positional arguments'): + self.call(test, 1, 2, 3, 4) + + with self.assertRaisesRegex(TypeError, + "missing a required argument: 'b'"): + self.call(test, 1) + + with self.assertRaisesRegex(TypeError, + "missing a required argument: 'a'"): + self.call(test) + + def test(a, b, c=10): + return a, b, c + self.assertEqual(self.call(test, 1, 2, 3), (1, 2, 3)) + self.assertEqual(self.call(test, 1, 2), (1, 2, 10)) + + def test(a=1, b=2, c=3): + return a, b, c + self.assertEqual(self.call(test, a=10, c=13), (10, 2, 13)) + self.assertEqual(self.call(test, a=10), (10, 2, 3)) + self.assertEqual(self.call(test, b=10), (1, 10, 3)) + + def test_signature_bind_varargs_order(self): + def test(*args): + return args + + self.assertEqual(self.call(test), ()) + self.assertEqual(self.call(test, 1, 2, 3), (1, 2, 3)) + + def test_signature_bind_args_and_varargs(self): + def test(a, b, c=3, *args): + return a, b, c, args + + self.assertEqual(self.call(test, 1, 2, 3, 4, 5), (1, 2, 3, (4, 5))) + self.assertEqual(self.call(test, 1, 2), (1, 2, 3, ())) + self.assertEqual(self.call(test, b=1, a=2), (2, 1, 3, ())) + self.assertEqual(self.call(test, 1, b=2), (1, 2, 3, ())) + + with self.assertRaisesRegex(TypeError, + "multiple values for argument 'c'"): + self.call(test, 1, 2, 3, c=4) + + def test_signature_bind_just_kwargs(self): + def test(**kwargs): + return kwargs + + self.assertEqual(self.call(test), {}) + self.assertEqual(self.call(test, foo='bar', spam='ham'), + {'foo': 'bar', 'spam': 'ham'}) + + def test_signature_bind_args_and_kwargs(self): + def test(a, b, c=3, **kwargs): + return a, b, c, kwargs + + self.assertEqual(self.call(test, 1, 2), (1, 2, 3, {})) + self.assertEqual(self.call(test, 1, 2, foo='bar', spam='ham'), + (1, 2, 3, {'foo': 'bar', 'spam': 'ham'})) + self.assertEqual(self.call(test, b=2, a=1, foo='bar', spam='ham'), + (1, 2, 3, {'foo': 'bar', 'spam': 'ham'})) + self.assertEqual(self.call(test, a=1, b=2, foo='bar', spam='ham'), + (1, 2, 3, {'foo': 'bar', 'spam': 'ham'})) + self.assertEqual(self.call(test, 1, b=2, foo='bar', spam='ham'), + (1, 2, 3, {'foo': 'bar', 'spam': 'ham'})) + self.assertEqual(self.call(test, 1, b=2, c=4, foo='bar', spam='ham'), + (1, 2, 4, {'foo': 'bar', 'spam': 'ham'})) + self.assertEqual(self.call(test, 1, 2, 4, foo='bar'), + (1, 2, 4, {'foo': 'bar'})) + self.assertEqual(self.call(test, c=5, a=4, b=3), + (4, 3, 5, {})) + + def test_signature_bind_kwonly(self): + def test(*, foo): + return foo + with self.assertRaisesRegex(TypeError, + 'too many positional arguments'): + self.call(test, 1) + self.assertEqual(self.call(test, foo=1), 1) + + def test(a, *, foo=1, bar): + return foo + with self.assertRaisesRegex(TypeError, + "missing a required argument: 'bar'"): + self.call(test, 1) + + def test(foo, *, bar): + return foo, bar + self.assertEqual(self.call(test, 1, bar=2), (1, 2)) + self.assertEqual(self.call(test, bar=2, foo=1), (1, 2)) + + with self.assertRaisesRegex( + TypeError, "got an unexpected keyword argument 'spam'"): + + self.call(test, bar=2, foo=1, spam=10) + + with self.assertRaisesRegex(TypeError, + 'too many positional arguments'): + self.call(test, 1, 2) + + with self.assertRaisesRegex(TypeError, + 'too many positional arguments'): + self.call(test, 1, 2, bar=2) + + with self.assertRaisesRegex( + TypeError, "got an unexpected keyword argument 'spam'"): + + self.call(test, 1, bar=2, spam='ham') + + with self.assertRaisesRegex(TypeError, + "missing a required keyword-only " + "argument: 'bar'"): + self.call(test, 1) + + def test(foo, *, bar, **bin): + return foo, bar, bin + self.assertEqual(self.call(test, 1, bar=2), (1, 2, {})) + self.assertEqual(self.call(test, foo=1, bar=2), (1, 2, {})) + self.assertEqual(self.call(test, 1, bar=2, spam='ham'), + (1, 2, {'spam': 'ham'})) + self.assertEqual(self.call(test, spam='ham', foo=1, bar=2), + (1, 2, {'spam': 'ham'})) + with self.assertRaisesRegex(TypeError, + "missing a required argument: 'foo'"): + self.call(test, spam='ham', bar=2) + self.assertEqual(self.call(test, 1, bar=2, bin=1, spam=10), + (1, 2, {'bin': 1, 'spam': 10})) + + def test_signature_bind_arguments(self): + def test(a, *args, b, z=100, **kwargs): + pass + sig = inspect.signature(test) + ba = sig.bind(10, 20, b=30, c=40, args=50, kwargs=60) + # we won't have 'z' argument in the bound arguments object, as we didn't + # pass it to the 'bind' + self.assertEqual(tuple(ba.arguments.items()), + (('a', 10), ('args', (20,)), ('b', 30), + ('kwargs', {'c': 40, 'args': 50, 'kwargs': 60}))) + self.assertEqual(ba.kwargs, + {'b': 30, 'c': 40, 'args': 50, 'kwargs': 60}) + self.assertEqual(ba.args, (10, 20)) + + def test_signature_bind_positional_only(self): + def test(a_po, b_po, c_po=3, /, foo=42, *, bar=50, **kwargs): + return a_po, b_po, c_po, foo, bar, kwargs + + self.assertEqual(self.call(test, 1, 2, 4, 5, bar=6), + (1, 2, 4, 5, 6, {})) + + self.assertEqual(self.call(test, 1, 2), + (1, 2, 3, 42, 50, {})) + + self.assertEqual(self.call(test, 1, 2, foo=4, bar=5), + (1, 2, 3, 4, 5, {})) + + self.assertEqual(self.call(test, 1, 2, foo=4, bar=5, c_po=10), + (1, 2, 3, 4, 5, {'c_po': 10})) + + self.assertEqual(self.call(test, 1, 2, 30, c_po=31, foo=4, bar=5), + (1, 2, 30, 4, 5, {'c_po': 31})) + + self.assertEqual(self.call(test, 1, 2, 30, foo=4, bar=5, c_po=31), + (1, 2, 30, 4, 5, {'c_po': 31})) + + self.assertEqual(self.call(test, 1, 2, c_po=4), + (1, 2, 3, 42, 50, {'c_po': 4})) + + with self.assertRaisesRegex(TypeError, "missing a required positional-only argument: 'a_po'"): + self.call(test, a_po=1, b_po=2) + + def without_var_kwargs(c_po=3, d_po=4, /): + return c_po, d_po + + with self.assertRaisesRegex( + TypeError, + "positional-only arguments passed as keyword arguments: 'c_po, d_po'", + ): + self.call(without_var_kwargs, c_po=33, d_po=44) + + def test_signature_bind_with_self_arg(self): + # Issue #17071: one of the parameters is named "self + def test(a, self, b): + pass + sig = inspect.signature(test) + ba = sig.bind(1, 2, 3) + self.assertEqual(ba.args, (1, 2, 3)) + ba = sig.bind(1, self=2, b=3) + self.assertEqual(ba.args, (1, 2, 3)) + + def test_signature_bind_vararg_name(self): + def test(a, *args): + return a, args + sig = inspect.signature(test) + + with self.assertRaisesRegex( + TypeError, "got an unexpected keyword argument 'args'"): + + sig.bind(a=0, args=1) + + def test(*args, **kwargs): + return args, kwargs + self.assertEqual(self.call(test, args=1), ((), {'args': 1})) + + sig = inspect.signature(test) + ba = sig.bind(args=1) + self.assertEqual(ba.arguments, {'kwargs': {'args': 1}}) + + @cpython_only + def test_signature_bind_implicit_arg(self): + # Issue #19611: getcallargs should work with comprehensions + def make_set(): + return set(z * z for z in range(5)) + gencomp_code = make_set.__code__.co_consts[1] + gencomp_func = types.FunctionType(gencomp_code, {}) + + iterator = iter(range(5)) + self.assertEqual(set(self.call(gencomp_func, iterator)), {0, 1, 4, 9, 16}) + + def test_signature_bind_posonly_kwargs(self): + def foo(bar, /, **kwargs): + return bar, kwargs.get(bar) + + sig = inspect.signature(foo) + result = sig.bind("pos-only", bar="keyword") + + self.assertEqual(result.kwargs, {"bar": "keyword"}) + self.assertIn(("bar", "pos-only"), result.arguments.items()) + + +class TestBoundArguments(unittest.TestCase): + def test_signature_bound_arguments_unhashable(self): + def foo(a): pass + ba = inspect.signature(foo).bind(1) + + with self.assertRaisesRegex(TypeError, 'unhashable type'): + hash(ba) + + def test_signature_bound_arguments_equality(self): + def foo(a): pass + ba = inspect.signature(foo).bind(1) + self.assertTrue(ba == ba) + self.assertFalse(ba != ba) + self.assertTrue(ba == ALWAYS_EQ) + self.assertFalse(ba != ALWAYS_EQ) + + ba2 = inspect.signature(foo).bind(1) + self.assertTrue(ba == ba2) + self.assertFalse(ba != ba2) + + ba3 = inspect.signature(foo).bind(2) + self.assertFalse(ba == ba3) + self.assertTrue(ba != ba3) + ba3.arguments['a'] = 1 + self.assertTrue(ba == ba3) + self.assertFalse(ba != ba3) + + def bar(b): pass + ba4 = inspect.signature(bar).bind(1) + self.assertFalse(ba == ba4) + self.assertTrue(ba != ba4) + + def foo(*, a, b): pass + sig = inspect.signature(foo) + ba1 = sig.bind(a=1, b=2) + ba2 = sig.bind(b=2, a=1) + self.assertTrue(ba1 == ba2) + self.assertFalse(ba1 != ba2) + + def test_signature_bound_arguments_pickle(self): + def foo(a, b, *, c:1={}, **kw) -> {42:'ham'}: pass + sig = inspect.signature(foo) + ba = sig.bind(20, 30, z={}) + + for ver in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(pickle_ver=ver): + ba_pickled = pickle.loads(pickle.dumps(ba, ver)) + self.assertEqual(ba, ba_pickled) + + def test_signature_bound_arguments_repr(self): + def foo(a, b, *, c:1={}, **kw) -> {42:'ham'}: pass + sig = inspect.signature(foo) + ba = sig.bind(20, 30, z={}) + self.assertRegex(repr(ba), r'<BoundArguments \(a=20,.*\}\}\)>') + + def test_signature_bound_arguments_apply_defaults(self): + def foo(a, b=1, *args, c:1={}, **kw): pass + sig = inspect.signature(foo) + + ba = sig.bind(20) + ba.apply_defaults() + self.assertEqual( + list(ba.arguments.items()), + [('a', 20), ('b', 1), ('args', ()), ('c', {}), ('kw', {})]) + + # Make sure that we preserve the order: + # i.e. 'c' should be *before* 'kw'. + ba = sig.bind(10, 20, 30, d=1) + ba.apply_defaults() + self.assertEqual( + list(ba.arguments.items()), + [('a', 10), ('b', 20), ('args', (30,)), ('c', {}), ('kw', {'d':1})]) + + # Make sure that BoundArguments produced by bind_partial() + # are supported. + def foo(a, b): pass + sig = inspect.signature(foo) + ba = sig.bind_partial(20) + ba.apply_defaults() + self.assertEqual( + list(ba.arguments.items()), + [('a', 20)]) + + # Test no args + def foo(): pass + sig = inspect.signature(foo) + ba = sig.bind() + ba.apply_defaults() + self.assertEqual(list(ba.arguments.items()), []) + + # Make sure a no-args binding still acquires proper defaults. + def foo(a='spam'): pass + sig = inspect.signature(foo) + ba = sig.bind() + ba.apply_defaults() + self.assertEqual(list(ba.arguments.items()), [('a', 'spam')]) + + def test_signature_bound_arguments_arguments_type(self): + def foo(a): pass + ba = inspect.signature(foo).bind(1) + self.assertIs(type(ba.arguments), dict) + +class TestSignaturePrivateHelpers(unittest.TestCase): + def _strip_non_python_syntax(self, input, + clean_signature, self_parameter): + computed_clean_signature, \ + computed_self_parameter = \ + inspect._signature_strip_non_python_syntax(input) + self.assertEqual(computed_clean_signature, clean_signature) + self.assertEqual(computed_self_parameter, self_parameter) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_signature_strip_non_python_syntax(self): + self._strip_non_python_syntax( + "($module, /, path, mode, *, dir_fd=None, " + + "effective_ids=False,\n follow_symlinks=True)", + "(module, /, path, mode, *, dir_fd=None, " + + "effective_ids=False, follow_symlinks=True)", + 0) + + self._strip_non_python_syntax( + "($module, word, salt, /)", + "(module, word, salt, /)", + 0) + + self._strip_non_python_syntax( + "(x, y=None, z=None, /)", + "(x, y=None, z=None, /)", + None) + + self._strip_non_python_syntax( + "(x, y=None, z=None)", + "(x, y=None, z=None)", + None) + + self._strip_non_python_syntax( + "(x,\n y=None,\n z = None )", + "(x, y=None, z=None)", + None) + + self._strip_non_python_syntax( + "", + "", + None) + + self._strip_non_python_syntax( + None, + None, + None) + +class TestSignatureDefinitions(unittest.TestCase): + # This test case provides a home for checking that particular APIs + # have signatures available for introspection + + @staticmethod + def is_public(name): + return not name.startswith('_') or name.startswith('__') and name.endswith('__') + + @cpython_only + @unittest.skipIf(MISSING_C_DOCSTRINGS, + "Signature information for builtins requires docstrings") + def _test_module_has_signatures(self, module, + no_signature=(), unsupported_signature=(), + methods_no_signature={}, methods_unsupported_signature={}, + good_exceptions=()): + # This checks all builtin callables in CPython have signatures + # A few have signatures Signature can't yet handle, so we skip those + # since they will have to wait until PEP 457 adds the required + # introspection support to the inspect module + # Some others also haven't been converted yet for various other + # reasons, so we also skip those for the time being, but design + # the test to fail in order to indicate when it needs to be + # updated. + no_signature = no_signature or set() + # Check the signatures we expect to be there + ns = vars(module) + try: + names = set(module.__all__) + except AttributeError: + names = set(name for name in ns if self.is_public(name)) + for name, obj in sorted(ns.items()): + if name not in names: + continue + if not callable(obj): + continue + if (isinstance(obj, type) and + issubclass(obj, BaseException) and + name not in good_exceptions): + no_signature.add(name) + if name not in no_signature and name not in unsupported_signature: + with self.subTest('supported', builtin=name): + self.assertIsNotNone(inspect.signature(obj)) + if isinstance(obj, type): + with self.subTest(type=name): + self._test_builtin_methods_have_signatures(obj, + methods_no_signature.get(name, ()), + methods_unsupported_signature.get(name, ())) + # Check callables that haven't been converted don't claim a signature + # This ensures this test will start failing as more signatures are + # added, so the affected items can be moved into the scope of the + # regression test above + for name in no_signature: + with self.subTest('none', builtin=name): + obj = ns[name] + self.assertIsNone(obj.__text_signature__) + self.assertRaises(ValueError, inspect.signature, obj) + for name in unsupported_signature: + with self.subTest('unsupported', builtin=name): + obj = ns[name] + self.assertIsNotNone(obj.__text_signature__) + self.assertRaises(ValueError, inspect.signature, obj) + + def _test_builtin_methods_have_signatures(self, cls, no_signature, unsupported_signature): + ns = vars(cls) + for name in ns: + obj = getattr(cls, name, None) + if not callable(obj) or isinstance(obj, type): + continue + if name not in no_signature and name not in unsupported_signature: + with self.subTest('supported', method=name): + self.assertIsNotNone(inspect.signature(obj)) + for name in no_signature: + with self.subTest('none', method=name): + self.assertIsNone(getattr(cls, name).__text_signature__) + self.assertRaises(ValueError, inspect.signature, getattr(cls, name)) + for name in unsupported_signature: + with self.subTest('unsupported', method=name): + self.assertIsNotNone(getattr(cls, name).__text_signature__) + self.assertRaises(ValueError, inspect.signature, getattr(cls, name)) + + def test_builtins_have_signatures(self): + no_signature = {'type', 'super', 'bytearray', 'bytes', 'dict', 'int', 'str'} + # These need PEP 457 groups + needs_groups = {"range", "slice", "dir", "getattr", + "next", "iter", "vars"} + no_signature |= needs_groups + # These have unrepresentable parameter default values of NULL + unsupported_signature = {"anext"} + # These need *args support in Argument Clinic + needs_varargs = {"min", "max", "__build_class__"} + no_signature |= needs_varargs + + methods_no_signature = { + 'dict': {'update'}, + 'object': {'__class__'}, + } + methods_unsupported_signature = { + 'bytearray': {'count', 'endswith', 'find', 'hex', 'index', 'rfind', 'rindex', 'startswith'}, + 'bytes': {'count', 'endswith', 'find', 'hex', 'index', 'rfind', 'rindex', 'startswith'}, + 'dict': {'pop'}, + 'int': {'__round__'}, + 'memoryview': {'cast', 'hex'}, + 'str': {'count', 'endswith', 'find', 'index', 'maketrans', 'rfind', 'rindex', 'startswith'}, + } + self._test_module_has_signatures(builtins, + no_signature, unsupported_signature, + methods_no_signature, methods_unsupported_signature) + + def test_types_module_has_signatures(self): + unsupported_signature = {'CellType'} + methods_no_signature = { + 'AsyncGeneratorType': {'athrow'}, + 'CoroutineType': {'throw'}, + 'GeneratorType': {'throw'}, + } + self._test_module_has_signatures(types, + unsupported_signature=unsupported_signature, + methods_no_signature=methods_no_signature) + + def test_sys_module_has_signatures(self): + no_signature = {'getsizeof', 'set_asyncgen_hooks'} + no_signature |= {name for name in ['getobjects'] + if hasattr(sys, name)} + self._test_module_has_signatures(sys, no_signature) + + def test_abc_module_has_signatures(self): + import abc + self._test_module_has_signatures(abc) + + def test_atexit_module_has_signatures(self): + import atexit + self._test_module_has_signatures(atexit) + + def test_codecs_module_has_signatures(self): + import codecs + methods_no_signature = {'StreamReader': {'charbuffertype'}} + self._test_module_has_signatures(codecs, + methods_no_signature=methods_no_signature) + + def test_collections_module_has_signatures(self): + no_signature = {'OrderedDict', 'defaultdict'} + unsupported_signature = {'deque'} + methods_no_signature = { + 'OrderedDict': {'update'}, + } + methods_unsupported_signature = { + 'deque': {'index'}, + 'OrderedDict': {'pop'}, + 'UserString': {'maketrans'}, + } + self._test_module_has_signatures(collections, + no_signature, unsupported_signature, + methods_no_signature, methods_unsupported_signature) + + def test_collections_abc_module_has_signatures(self): + import collections.abc + self._test_module_has_signatures(collections.abc) + + def test_errno_module_has_signatures(self): + import errno + self._test_module_has_signatures(errno) + + def test_faulthandler_module_has_signatures(self): + import faulthandler + unsupported_signature = {'dump_traceback', 'dump_traceback_later', 'enable'} + unsupported_signature |= {name for name in ['register'] + if hasattr(faulthandler, name)} + self._test_module_has_signatures(faulthandler, unsupported_signature=unsupported_signature) + + def test_functools_module_has_signatures(self): + no_signature = {'reduce'} + self._test_module_has_signatures(functools, no_signature) + + def test_gc_module_has_signatures(self): + import gc + no_signature = {'set_threshold'} + self._test_module_has_signatures(gc, no_signature) + + def test_io_module_has_signatures(self): + methods_no_signature = { + 'BufferedRWPair': {'read', 'peek', 'read1', 'readinto', 'readinto1', 'write'}, + } + self._test_module_has_signatures(io, + methods_no_signature=methods_no_signature) + + def test_itertools_module_has_signatures(self): + import itertools + no_signature = {'islice', 'repeat'} + self._test_module_has_signatures(itertools, no_signature) + + def test_locale_module_has_signatures(self): + import locale + self._test_module_has_signatures(locale) + + def test_marshal_module_has_signatures(self): + import marshal + self._test_module_has_signatures(marshal) + + def test_operator_module_has_signatures(self): + import operator + self._test_module_has_signatures(operator) + + def test_os_module_has_signatures(self): + unsupported_signature = {'chmod', 'utime'} + unsupported_signature |= {name for name in + ['get_terminal_size', 'posix_spawn', 'posix_spawnp', + 'register_at_fork', 'startfile'] + if hasattr(os, name)} + self._test_module_has_signatures(os, unsupported_signature=unsupported_signature) + + def test_pwd_module_has_signatures(self): + pwd = import_helper.import_module('pwd') + self._test_module_has_signatures(pwd) + + def test_re_module_has_signatures(self): + import re + methods_no_signature = {'Match': {'group'}} + self._test_module_has_signatures(re, + methods_no_signature=methods_no_signature, + good_exceptions={'error', 'PatternError'}) + + def test_signal_module_has_signatures(self): + import signal + self._test_module_has_signatures(signal) + + def test_stat_module_has_signatures(self): + import stat + self._test_module_has_signatures(stat) + + def test_string_module_has_signatures(self): + import string + self._test_module_has_signatures(string) + + def test_symtable_module_has_signatures(self): + import symtable + self._test_module_has_signatures(symtable) + + def test_sysconfig_module_has_signatures(self): + import sysconfig + self._test_module_has_signatures(sysconfig) + + def test_threading_module_has_signatures(self): + import threading + self._test_module_has_signatures(threading) + self.assertIsNotNone(inspect.signature(threading.__excepthook__)) + + def test_thread_module_has_signatures(self): + import _thread + no_signature = {'RLock'} + self._test_module_has_signatures(_thread, no_signature) + + def test_time_module_has_signatures(self): + no_signature = { + 'asctime', 'ctime', 'get_clock_info', 'gmtime', 'localtime', + 'strftime', 'strptime' + } + no_signature |= {name for name in + ['clock_getres', 'clock_settime', 'clock_settime_ns', + 'pthread_getcpuclockid'] + if hasattr(time, name)} + self._test_module_has_signatures(time, no_signature) + + def test_tokenize_module_has_signatures(self): + import tokenize + self._test_module_has_signatures(tokenize) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_tracemalloc_module_has_signatures(self): + import tracemalloc + self._test_module_has_signatures(tracemalloc) + + def test_typing_module_has_signatures(self): + import typing + no_signature = {'ParamSpec', 'ParamSpecArgs', 'ParamSpecKwargs', + 'Text', 'TypeAliasType', 'TypeVar', 'TypeVarTuple'} + methods_no_signature = { + 'Generic': {'__class_getitem__', '__init_subclass__'}, + } + methods_unsupported_signature = { + 'Text': {'count', 'find', 'index', 'rfind', 'rindex', 'startswith', 'endswith', 'maketrans'}, + } + self._test_module_has_signatures(typing, no_signature, + methods_no_signature=methods_no_signature, + methods_unsupported_signature=methods_unsupported_signature) + + def test_warnings_module_has_signatures(self): + unsupported_signature = {'warn', 'warn_explicit'} + self._test_module_has_signatures(warnings, unsupported_signature=unsupported_signature) + + def test_weakref_module_has_signatures(self): + import weakref + no_signature = {'ReferenceType', 'ref'} + self._test_module_has_signatures(weakref, no_signature) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_python_function_override_signature(self): + def func(*args, **kwargs): + pass + func.__text_signature__ = '($self, a, b=1, *args, c, d=2, **kwargs)' + sig = inspect.signature(func) + self.assertIsNotNone(sig) + self.assertEqual(str(sig), '(self, /, a, b=1, *args, c, d=2, **kwargs)') + + func.__text_signature__ = '($self, a, b=1, /, *args, c, d=2, **kwargs)' + sig = inspect.signature(func) + self.assertEqual(str(sig), '(self, a, b=1, /, *args, c, d=2, **kwargs)') + + func.__text_signature__ = '(self, a=1+2, b=4-3, c=1 | 3 | 16)' + sig = inspect.signature(func) + self.assertEqual(str(sig), '(self, a=3, b=1, c=19)') + + func.__text_signature__ = '(self, a=1,\nb=2,\n\n\n c=3)' + sig = inspect.signature(func) + self.assertEqual(str(sig), '(self, a=1, b=2, c=3)') + + func.__text_signature__ = '(self, x=does_not_exist)' + with self.assertRaises(ValueError): + inspect.signature(func) + func.__text_signature__ = '(self, x=sys, y=inspect)' + with self.assertRaises(ValueError): + inspect.signature(func) + func.__text_signature__ = '(self, 123)' + with self.assertRaises(ValueError): + inspect.signature(func) + + @support.requires_docstrings + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_base_class_have_text_signature(self): + # see issue 43118 + from test.typinganndata.ann_module7 import BufferedReader + class MyBufferedReader(BufferedReader): + """buffer reader class.""" + + text_signature = BufferedReader.__text_signature__ + self.assertEqual(text_signature, '(raw, buffer_size=DEFAULT_BUFFER_SIZE)') + sig = inspect.signature(MyBufferedReader) + self.assertEqual(str(sig), '(raw, buffer_size=8192)') + + +class NTimesUnwrappable: + def __init__(self, n): + self.n = n + self._next = None + + @property + def __wrapped__(self): + if self.n <= 0: + raise Exception("Unwrapped too many times") + if self._next is None: + self._next = NTimesUnwrappable(self.n - 1) + return self._next + +class TestUnwrap(unittest.TestCase): + + def test_unwrap_one(self): + def func(a, b): + return a + b + wrapper = functools.lru_cache(maxsize=20)(func) + self.assertIs(inspect.unwrap(wrapper), func) + + def test_unwrap_several(self): + def func(a, b): + return a + b + wrapper = func + for __ in range(10): + @functools.wraps(wrapper) + def wrapper(): + pass + self.assertIsNot(wrapper.__wrapped__, func) + self.assertIs(inspect.unwrap(wrapper), func) + + def test_stop(self): + def func1(a, b): + return a + b + @functools.wraps(func1) + def func2(): + pass + @functools.wraps(func2) + def wrapper(): + pass + func2.stop_here = 1 + unwrapped = inspect.unwrap(wrapper, + stop=(lambda f: hasattr(f, "stop_here"))) + self.assertIs(unwrapped, func2) + + def test_cycle(self): + def func1(): pass + func1.__wrapped__ = func1 + with self.assertRaisesRegex(ValueError, 'wrapper loop'): + inspect.unwrap(func1) + + def func2(): pass + func2.__wrapped__ = func1 + func1.__wrapped__ = func2 + with self.assertRaisesRegex(ValueError, 'wrapper loop'): + inspect.unwrap(func1) + with self.assertRaisesRegex(ValueError, 'wrapper loop'): + inspect.unwrap(func2) + + def test_unhashable(self): + def func(): pass + func.__wrapped__ = None + class C: + __hash__ = None + __wrapped__ = func + self.assertIsNone(inspect.unwrap(C())) + + def test_recursion_limit(self): + obj = NTimesUnwrappable(sys.getrecursionlimit() + 1) + with self.assertRaisesRegex(ValueError, 'wrapper loop'): + inspect.unwrap(obj) + + def test_wrapped_descriptor(self): + self.assertIs(inspect.unwrap(NTimesUnwrappable), NTimesUnwrappable) + self.assertIs(inspect.unwrap(staticmethod), staticmethod) + self.assertIs(inspect.unwrap(classmethod), classmethod) + self.assertIs(inspect.unwrap(staticmethod(classmethod)), classmethod) + self.assertIs(inspect.unwrap(classmethod(staticmethod)), staticmethod) + + +class TestMain(unittest.TestCase): + def test_only_source(self): + module = importlib.import_module('unittest') + rc, out, err = assert_python_ok('-m', 'inspect', + 'unittest') + lines = out.decode().splitlines() + # ignore the final newline + self.assertEqual(lines[:-1], inspect.getsource(module).splitlines()) + self.assertEqual(err, b'') + + def test_custom_getattr(self): + def foo(): + pass + foo.__signature__ = 42 + with self.assertRaises(TypeError): + inspect.signature(foo) + + @unittest.skipIf(ThreadPoolExecutor is None, + 'threads required to test __qualname__ for source files') + def test_qualname_source(self): + rc, out, err = assert_python_ok('-m', 'inspect', + 'concurrent.futures:ThreadPoolExecutor') + lines = out.decode().splitlines() + # ignore the final newline + self.assertEqual(lines[:-1], + inspect.getsource(ThreadPoolExecutor).splitlines()) + self.assertEqual(err, b'') + + def test_builtins(self): + _, out, err = assert_python_failure('-m', 'inspect', + 'sys') + lines = err.decode().splitlines() + self.assertEqual(lines, ["Can't get info for builtin modules."]) + + def test_details(self): + module = importlib.import_module('unittest') + args = support.optim_args_from_interpreter_flags() + rc, out, err = assert_python_ok(*args, '-m', 'inspect', + 'unittest', '--details') + output = out.decode() + # Just a quick sanity check on the output + self.assertIn(module.__spec__.name, output) + self.assertIn(module.__name__, output) + self.assertIn(module.__spec__.origin, output) + self.assertIn(module.__file__, output) + self.assertIn(module.__spec__.cached, output) + self.assertIn(module.__cached__, output) + self.assertEqual(err, b'') + + +class TestReload(unittest.TestCase): + + src_before = textwrap.dedent("""\ +def foo(): + print("Bla") + """) + + src_after = textwrap.dedent("""\ +def foo(): + print("Oh no!") + """) + + def assertInspectEqual(self, path, source): + inspected_src = inspect.getsource(source) + with open(path, encoding='utf-8') as src: + self.assertEqual( + src.read().splitlines(True), + inspected_src.splitlines(True) + ) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_getsource_reload(self): + # see issue 1218234 + with ready_to_import('reload_bug', self.src_before) as (name, path): + module = importlib.import_module(name) + self.assertInspectEqual(path, module) + with open(path, 'w', encoding='utf-8') as src: + src.write(self.src_after) + self.assertInspectEqual(path, module) + + +class TestRepl(unittest.TestCase): + + def spawn_repl(self, *args, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **kw): + """Run the Python REPL with the given arguments. + + kw is extra keyword args to pass to subprocess.Popen. Returns a Popen + object. + """ + + # To run the REPL without using a terminal, spawn python with the command + # line option '-i' and the process name set to '<stdin>'. + # The directory of argv[0] must match the directory of the Python + # executable for the Popen() call to python to succeed as the directory + # path may be used by Py_GetPath() to build the default module search + # path. + stdin_fname = os.path.join(os.path.dirname(sys.executable), "<stdin>") + cmd_line = [stdin_fname, '-E', '-i'] + cmd_line.extend(args) + + # Set TERM=vt100, for the rationale see the comments in spawn_python() of + # test.support.script_helper. + env = kw.setdefault('env', dict(os.environ)) + env['TERM'] = 'vt100' + return subprocess.Popen(cmd_line, + executable=sys.executable, + text=True, + stdin=subprocess.PIPE, + stdout=stdout, stderr=stderr, + **kw) + + def run_on_interactive_mode(self, source): + """Spawn a new Python interpreter, pass the given + input source code from the stdin and return the + result back. If the interpreter exits non-zero, it + raises a ValueError.""" + + process = self.spawn_repl() + process.stdin.write(source) + output = kill_python(process) + + if process.returncode != 0: + raise ValueError("Process didn't exit properly.") + return output + + @unittest.skipIf(not has_subprocess_support, "test requires subprocess") + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_getsource(self): + output = self.run_on_interactive_mode(textwrap.dedent("""\ + def f(): + print(0) + return 1 + 2 + + import inspect + print(f"The source is: <<<{inspect.getsource(f)}>>>") + """)) + + expected = "The source is: <<<def f():\n print(0)\n return 1 + 2\n>>>" + self.assertIn(expected, output) + + + + +if __name__ == "__main__": + unittest.main() \ No newline at end of file diff --git a/Lib/test/test_int.py b/Lib/test/test_int.py index 7ac83288f4..1ab7a1fb6d 100644 --- a/Lib/test/test_int.py +++ b/Lib/test/test_int.py @@ -1,9 +1,22 @@ import sys +import time import unittest +# TODO: RUSTPYTHON +# This is one of the tests that we run on wasi. `unittest.mock` requires `_socket` +# which we don't have on wasi (yet). Also, every test here the needs `unittest.mock` +# is cpython specifc, so this import is redundent anyway. +# from unittest import mock from test import support -from test.test_grammar import (VALID_UNDERSCORE_LITERALS, - INVALID_UNDERSCORE_LITERALS) +from test.support.numbers import ( + VALID_UNDERSCORE_LITERALS, + INVALID_UNDERSCORE_LITERALS, +) + +try: + import _pylong +except ImportError: + _pylong = None L = [ ('0', 0), @@ -83,6 +96,7 @@ def test_basic(self): self.assertRaises(TypeError, int, 1, 12) + self.assertRaises(TypeError, int, "10", 2, 1) self.assertEqual(int('0o123', 0), 83) self.assertEqual(int('0x123', 16), 291) @@ -148,6 +162,8 @@ def test_basic(self): self.assertEqual(int(' 0O123 ', 0), 83) self.assertEqual(int(' 0X123 ', 0), 291) self.assertEqual(int(' 0B100 ', 0), 4) + with self.assertRaises(ValueError): + int('010', 0) # without base still base 10 self.assertEqual(int('0123'), 123) @@ -214,6 +230,26 @@ def test_basic(self): self.assertEqual(int('2br45qc', 35), 4294967297) self.assertEqual(int('1z141z5', 36), 4294967297) + def test_invalid_signs(self): + with self.assertRaises(ValueError): + int('+') + with self.assertRaises(ValueError): + int('-') + with self.assertRaises(ValueError): + int('- 1') + with self.assertRaises(ValueError): + int('+ 1') + with self.assertRaises(ValueError): + int(' + 1 ') + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_unicode(self): + self.assertEqual(int("१२३४५६७८९०1234567890"), 12345678901234567890) + self.assertEqual(int('١٢٣٤٥٦٧٨٩٠'), 1234567890) + self.assertEqual(int("१२३४५६७८९०1234567890", 0), 12345678901234567890) + self.assertEqual(int('١٢٣٤٥٦٧٨٩٠', 0), 1234567890) + def test_underscores(self): for lit in VALID_UNDERSCORE_LITERALS: if any(ch in lit for ch in '.eEjJ'): @@ -233,7 +269,7 @@ def test_underscores(self): self.assertRaises(ValueError, int, "1__00") self.assertRaises(ValueError, int, "100_") - # @support.cpython_only + @support.cpython_only def test_small_ints(self): # Bug #3236: Return small longs from PyLong_FromString self.assertIs(int('10'), 10) @@ -369,12 +405,14 @@ def __trunc__(self): class JustTrunc(base): def __trunc__(self): return 42 - self.assertEqual(int(JustTrunc()), 42) + with self.assertWarns(DeprecationWarning): + self.assertEqual(int(JustTrunc()), 42) class ExceptionalTrunc(base): def __trunc__(self): 1 / 0 - with self.assertRaises(ZeroDivisionError): + with self.assertRaises(ZeroDivisionError), \ + self.assertWarns(DeprecationWarning): int(ExceptionalTrunc()) for trunc_result_base in (object, Classic): @@ -385,7 +423,8 @@ def __index__(self): class TruncReturnsNonInt(base): def __trunc__(self): return Index() - self.assertEqual(int(TruncReturnsNonInt()), 42) + with self.assertWarns(DeprecationWarning): + self.assertEqual(int(TruncReturnsNonInt()), 42) class Intable(trunc_result_base): def __int__(self): @@ -394,7 +433,8 @@ def __int__(self): class TruncReturnsNonIndex(base): def __trunc__(self): return Intable() - self.assertEqual(int(TruncReturnsNonInt()), 42) + with self.assertWarns(DeprecationWarning): + self.assertEqual(int(TruncReturnsNonInt()), 42) class NonIntegral(trunc_result_base): def __trunc__(self): @@ -405,7 +445,8 @@ class TruncReturnsNonIntegral(base): def __trunc__(self): return NonIntegral() try: - int(TruncReturnsNonIntegral()) + with self.assertWarns(DeprecationWarning): + int(TruncReturnsNonIntegral()) except TypeError as e: self.assertEqual(str(e), "__trunc__ returned non-Integral" @@ -423,7 +464,8 @@ class TruncReturnsBadInt(base): def __trunc__(self): return BadInt() - with self.assertRaises(TypeError): + with self.assertRaises(TypeError), \ + self.assertWarns(DeprecationWarning): int(TruncReturnsBadInt()) def test_int_subclass_with_index(self): @@ -517,13 +559,16 @@ def __trunc__(self): self.assertIs(type(n), int) bad_int = TruncReturnsBadInt() - self.assertRaises(TypeError, int, bad_int) + with self.assertWarns(DeprecationWarning): + self.assertRaises(TypeError, int, bad_int) good_int = TruncReturnsIntSubclass() - n = int(good_int) + with self.assertWarns(DeprecationWarning): + n = int(good_int) self.assertEqual(n, 1) self.assertIs(type(n), int) - n = IntSubclass(good_int) + with self.assertWarns(DeprecationWarning): + n = IntSubclass(good_int) self.assertEqual(n, 1) self.assertIs(type(n), IntSubclass) @@ -568,5 +613,326 @@ def test_issue31619(self): self.assertEqual(int('1_2_3_4_5_6_7', 32), 1144132807) +class IntStrDigitLimitsTests(unittest.TestCase): + + int_class = int # Override this in subclasses to reuse the suite. + + def setUp(self): + super().setUp() + self._previous_limit = sys.get_int_max_str_digits() + sys.set_int_max_str_digits(2048) + + def tearDown(self): + sys.set_int_max_str_digits(self._previous_limit) + super().tearDown() + + def test_disabled_limit(self): + self.assertGreater(sys.get_int_max_str_digits(), 0) + self.assertLess(sys.get_int_max_str_digits(), 20_000) + with support.adjust_int_max_str_digits(0): + self.assertEqual(sys.get_int_max_str_digits(), 0) + i = self.int_class('1' * 20_000) + str(i) + self.assertGreater(sys.get_int_max_str_digits(), 0) + + def test_max_str_digits_edge_cases(self): + """Ignore the +/- sign and space padding.""" + int_class = self.int_class + maxdigits = sys.get_int_max_str_digits() + + int_class('1' * maxdigits) + int_class(' ' + '1' * maxdigits) + int_class('1' * maxdigits + ' ') + int_class('+' + '1' * maxdigits) + int_class('-' + '1' * maxdigits) + self.assertEqual(len(str(10 ** (maxdigits - 1))), maxdigits) + + def check(self, i, base=None): + with self.assertRaises(ValueError): + if base is None: + self.int_class(i) + else: + self.int_class(i, base) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_max_str_digits(self): + maxdigits = sys.get_int_max_str_digits() + + self.check('1' * (maxdigits + 1)) + self.check(' ' + '1' * (maxdigits + 1)) + self.check('1' * (maxdigits + 1) + ' ') + self.check('+' + '1' * (maxdigits + 1)) + self.check('-' + '1' * (maxdigits + 1)) + self.check('1' * (maxdigits + 1)) + + i = 10 ** maxdigits + with self.assertRaises(ValueError): + str(i) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_denial_of_service_prevented_int_to_str(self): + """Regression test: ensure we fail before performing O(N**2) work.""" + maxdigits = sys.get_int_max_str_digits() + assert maxdigits < 50_000, maxdigits # A test prerequisite. + + huge_int = int(f'0x{"c"*65_000}', base=16) # 78268 decimal digits. + digits = 78_268 + with ( + support.adjust_int_max_str_digits(digits), + support.CPUStopwatch() as sw_convert): + huge_decimal = str(huge_int) + self.assertEqual(len(huge_decimal), digits) + # Ensuring that we chose a slow enough conversion to measure. + # It takes 0.1 seconds on a Zen based cloud VM in an opt build. + # Some OSes have a low res 1/64s timer, skip if hard to measure. + if sw_convert.seconds < sw_convert.clock_info.resolution * 2: + raise unittest.SkipTest('"slow" conversion took only ' + f'{sw_convert.seconds} seconds.') + + # We test with the limit almost at the size needed to check performance. + # The performant limit check is slightly fuzzy, give it a some room. + with support.adjust_int_max_str_digits(int(.995 * digits)): + with ( + self.assertRaises(ValueError) as err, + support.CPUStopwatch() as sw_fail_huge): + str(huge_int) + self.assertIn('conversion', str(err.exception)) + self.assertLessEqual(sw_fail_huge.seconds, sw_convert.seconds/2) + + # Now we test that a conversion that would take 30x as long also fails + # in a similarly fast fashion. + extra_huge_int = int(f'0x{"c"*500_000}', base=16) # 602060 digits. + with ( + self.assertRaises(ValueError) as err, + support.CPUStopwatch() as sw_fail_extra_huge): + # If not limited, 8 seconds said Zen based cloud VM. + str(extra_huge_int) + self.assertIn('conversion', str(err.exception)) + self.assertLess(sw_fail_extra_huge.seconds, sw_convert.seconds/2) + + @unittest.skip('TODO: RUSTPYTHON; flaky test') + def test_denial_of_service_prevented_str_to_int(self): + """Regression test: ensure we fail before performing O(N**2) work.""" + maxdigits = sys.get_int_max_str_digits() + assert maxdigits < 100_000, maxdigits # A test prerequisite. + + digits = 133700 + huge = '8'*digits + with ( + support.adjust_int_max_str_digits(digits), + support.CPUStopwatch() as sw_convert): + int(huge) + # Ensuring that we chose a slow enough conversion to measure. + # It takes 0.1 seconds on a Zen based cloud VM in an opt build. + # Some OSes have a low res 1/64s timer, skip if hard to measure. + if sw_convert.seconds < sw_convert.clock_info.resolution * 2: + raise unittest.SkipTest('"slow" conversion took only ' + f'{sw_convert.seconds} seconds.') + + with support.adjust_int_max_str_digits(digits - 1): + with ( + self.assertRaises(ValueError) as err, + support.CPUStopwatch() as sw_fail_huge): + int(huge) + self.assertIn('conversion', str(err.exception)) + self.assertLessEqual(sw_fail_huge.seconds, sw_convert.seconds/2) + + # Now we test that a conversion that would take 30x as long also fails + # in a similarly fast fashion. + extra_huge = '7'*1_200_000 + with ( + self.assertRaises(ValueError) as err, + support.CPUStopwatch() as sw_fail_extra_huge): + # If not limited, 8 seconds in the Zen based cloud VM. + int(extra_huge) + self.assertIn('conversion', str(err.exception)) + self.assertLessEqual(sw_fail_extra_huge.seconds, sw_convert.seconds/2) + + def test_power_of_two_bases_unlimited(self): + """The limit does not apply to power of 2 bases.""" + maxdigits = sys.get_int_max_str_digits() + + for base in (2, 4, 8, 16, 32): + with self.subTest(base=base): + self.int_class('1' * (maxdigits + 1), base) + assert maxdigits < 100_000 + self.int_class('1' * 100_000, base) + + def test_underscores_ignored(self): + maxdigits = sys.get_int_max_str_digits() + + triples = maxdigits // 3 + s = '111' * triples + s_ = '1_11' * triples + self.int_class(s) # succeeds + self.int_class(s_) # succeeds + self.check(f'{s}111') + self.check(f'{s_}_111') + + def test_sign_not_counted(self): + int_class = self.int_class + max_digits = sys.get_int_max_str_digits() + s = '5' * max_digits + i = int_class(s) + pos_i = int_class(f'+{s}') + assert i == pos_i + neg_i = int_class(f'-{s}') + assert -pos_i == neg_i + str(pos_i) + str(neg_i) + + def _other_base_helper(self, base): + int_class = self.int_class + max_digits = sys.get_int_max_str_digits() + s = '2' * max_digits + i = int_class(s, base) + if base > 10: + with self.assertRaises(ValueError): + str(i) + elif base < 10: + str(i) + with self.assertRaises(ValueError) as err: + int_class(f'{s}1', base) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_int_from_other_bases(self): + base = 3 + with self.subTest(base=base): + self._other_base_helper(base) + base = 36 + with self.subTest(base=base): + self._other_base_helper(base) + + def test_int_max_str_digits_is_per_interpreter(self): + # Changing the limit in one interpreter does not change others. + code = """if 1: + # Subinterpreters maintain and enforce their own limit + import sys + sys.set_int_max_str_digits(2323) + try: + int('3'*3333) + except ValueError: + pass + else: + raise AssertionError('Expected a int max str digits ValueError.') + """ + with support.adjust_int_max_str_digits(4000): + before_value = sys.get_int_max_str_digits() + self.assertEqual(support.run_in_subinterp(code), 0, + 'subinterp code failure, check stderr.') + after_value = sys.get_int_max_str_digits() + self.assertEqual(before_value, after_value) + + +class IntSubclassStrDigitLimitsTests(IntStrDigitLimitsTests): + int_class = IntSubclass + + +class PyLongModuleTests(unittest.TestCase): + # Tests of the functions in _pylong.py. Those get used when the + # number of digits in the input values are large enough. + + def setUp(self): + super().setUp() + self._previous_limit = sys.get_int_max_str_digits() + sys.set_int_max_str_digits(0) + + def tearDown(self): + sys.set_int_max_str_digits(self._previous_limit) + super().tearDown() + + def _test_pylong_int_to_decimal(self, n, suffix): + s = str(n) + self.assertEqual(s[-10:], suffix) + s2 = str(-n) + self.assertEqual(s2, '-' + s) + s3 = '%d' % n + self.assertEqual(s3, s) + s4 = b'%d' % n + self.assertEqual(s4, s.encode('ascii')) + + def test_pylong_int_to_decimal(self): + self._test_pylong_int_to_decimal((1 << 100_000), '9883109376') + self._test_pylong_int_to_decimal((1 << 100_000) - 1, '9883109375') + self._test_pylong_int_to_decimal(10**30_000, '0000000000') + self._test_pylong_int_to_decimal(10**30_000 - 1, '9999999999') + self._test_pylong_int_to_decimal(3**60_000, '9313200001') + + @support.requires_resource('cpu') + def test_pylong_int_to_decimal_2(self): + self._test_pylong_int_to_decimal(2**1_000_000, '2747109376') + self._test_pylong_int_to_decimal(10**300_000, '0000000000') + self._test_pylong_int_to_decimal(3**600_000, '3132000001') + + def test_pylong_int_divmod(self): + n = (1 << 100_000) + a, b = divmod(n*3 + 1, n) + assert a == 3 and b == 1 + + def test_pylong_str_to_int(self): + v1 = 1 << 100_000 + s = str(v1) + v2 = int(s) + assert v1 == v2 + v3 = int(' -' + s) + assert -v1 == v3 + v4 = int(' +' + s + ' ') + assert v1 == v4 + with self.assertRaises(ValueError) as err: + int(s + 'z') + with self.assertRaises(ValueError) as err: + int(s + '_') + with self.assertRaises(ValueError) as err: + int('_' + s) + + @support.cpython_only # tests implementation details of CPython. + @unittest.skipUnless(_pylong, "_pylong module required") + # @mock.patch.object(_pylong, "int_to_decimal_string") # NOTE(RUSTPYTHON): See comment at top of file + def test_pylong_misbehavior_error_path_to_str( + self, mock_int_to_str): + with support.adjust_int_max_str_digits(20_000): + big_value = int('7'*19_999) + mock_int_to_str.return_value = None # not a str + with self.assertRaises(TypeError) as ctx: + str(big_value) + self.assertIn('_pylong.int_to_decimal_string did not', + str(ctx.exception)) + mock_int_to_str.side_effect = RuntimeError("testABC") + with self.assertRaises(RuntimeError): + str(big_value) + + @support.cpython_only # tests implementation details of CPython. + @unittest.skipUnless(_pylong, "_pylong module required") + # @mock.patch.object(_pylong, "int_from_string") # NOTE(RUSTPYTHON): See comment at top of file + def test_pylong_misbehavior_error_path_from_str( + self, mock_int_from_str): + big_value = '7'*19_999 + with support.adjust_int_max_str_digits(20_000): + mock_int_from_str.return_value = b'not an int' + with self.assertRaises(TypeError) as ctx: + int(big_value) + self.assertIn('_pylong.int_from_string did not', + str(ctx.exception)) + + mock_int_from_str.side_effect = RuntimeError("test123") + with self.assertRaises(RuntimeError): + int(big_value) + + def test_pylong_roundtrip(self): + from random import randrange, getrandbits + bits = 5000 + while bits <= 1_000_000: + bits += randrange(-100, 101) # break bitlength patterns + hibit = 1 << (bits - 1) + n = hibit | getrandbits(bits - 1) + assert n.bit_length() == bits + sn = str(n) + self.assertFalse(sn.startswith('0')) + self.assertEqual(n, int(sn)) + bits <<= 1 + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_io.py b/Lib/test/test_io.py index e91d454b72..0142208427 100644 --- a/Lib/test/test_io.py +++ b/Lib/test/test_io.py @@ -39,11 +39,9 @@ from test import support from test.support.script_helper import ( assert_python_ok, assert_python_failure, run_python_until_end) -from test.support import import_helper -from test.support import os_helper -from test.support import threading_helper -from test.support import warnings_helper -from test.support import skip_if_sanitizer +from test.support import ( + import_helper, is_apple, os_helper, threading_helper, warnings_helper, +) from test.support.os_helper import FakePath import codecs @@ -66,10 +64,6 @@ def byteslike(*pos, **kw): class EmptyStruct(ctypes.Structure): pass -# Does io.IOBase finalizer log the exception if the close() method fails? -# The exception is ignored silently by default in release build. -IOBASE_EMITS_UNRAISABLE = (support.Py_DEBUG or sys.flags.dev_mode) - def _default_chunk_size(): """Get the default TextIOWrapper chunk size""" @@ -631,10 +625,10 @@ def test_raw_bytes_io(self): self.read_ops(f, True) def test_large_file_ops(self): - # On Windows and Mac OSX this test consumes large resources; It takes - # a long time to build the >2 GiB file and takes >2 GiB of disk space - # therefore the resource must be enabled to run this test. - if sys.platform[:3] == 'win' or sys.platform == 'darwin': + # On Windows and Apple platforms this test consumes large resources; It + # takes a long time to build the >2 GiB file and takes >2 GiB of disk + # space therefore the resource must be enabled to run this test. + if sys.platform[:3] == 'win' or is_apple: support.requires( 'largefile', 'test requires %s bytes and a long time to run' % self.LARGE) @@ -645,11 +639,9 @@ def test_large_file_ops(self): def test_with_open(self): for bufsize in (0, 100): - f = None with self.open(os_helper.TESTFN, "wb", bufsize) as f: f.write(b"xxx") self.assertEqual(f.closed, True) - f = None try: with self.open(os_helper.TESTFN, "wb", bufsize) as f: 1/0 @@ -788,8 +780,8 @@ def test_closefd_attr(self): file = self.open(f.fileno(), "r", encoding="utf-8", closefd=False) self.assertEqual(file.buffer.raw.closefd, False) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.skipIf(sys.platform == 'win32', 'TODO: RUSTPYTHON; cyclic GC not supported, causes file locking') + @unittest.expectedFailure # TODO: RUSTPYTHON def test_garbage_collection(self): # FileIO objects are collected, and collecting them flushes # all data to disk. @@ -904,7 +896,7 @@ def test_bad_opener_negative_1(self): def badopener(fname, flags): return -1 with self.assertRaises(ValueError) as cm: - open('non-existent', 'r', opener=badopener) + self.open('non-existent', 'r', opener=badopener) self.assertEqual(str(cm.exception), 'opener returned -1') def test_bad_opener_other_negative(self): @@ -912,7 +904,7 @@ def test_bad_opener_other_negative(self): def badopener(fname, flags): return -2 with self.assertRaises(ValueError) as cm: - open('non-existent', 'r', opener=badopener) + self.open('non-existent', 'r', opener=badopener) self.assertEqual(str(cm.exception), 'opener returned -2') def test_opener_invalid_fd(self): @@ -1048,11 +1040,41 @@ def flush(self): # Silence destructor error R.flush = lambda self: None + @threading_helper.requires_working_threading() + def test_write_readline_races(self): + # gh-134908: Concurrent iteration over a file caused races + thread_count = 2 + write_count = 100 + read_count = 100 + + def writer(file, barrier): + barrier.wait() + for _ in range(write_count): + file.write("x") + + def reader(file, barrier): + barrier.wait() + for _ in range(read_count): + for line in file: + self.assertEqual(line, "") + + with self.open(os_helper.TESTFN, "w+") as f: + barrier = threading.Barrier(thread_count + 1) + reader = threading.Thread(target=reader, args=(f, barrier)) + writers = [threading.Thread(target=writer, args=(f, barrier)) + for _ in range(thread_count)] + with threading_helper.catch_threading_exception() as cm: + with threading_helper.start_threads(writers + [reader]): + pass + self.assertIsNone(cm.exc_type) + + self.assertEqual(os.stat(os_helper.TESTFN).st_size, + write_count * thread_count) + class CIOTest(IOTest): - # TODO: RUSTPYTHON, cyclic gc - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; cyclic gc def test_IOBase_finalize(self): # Issue #12149: segmentation fault on _PyIOBase_finalize when both a # class which inherits IOBase and an object of this class are caught @@ -1071,10 +1093,9 @@ def close(self): support.gc_collect() self.assertIsNone(wr(), wr) - # TODO: RUSTPYTHON, AssertionError: filter ('', ResourceWarning) did not catch any warning - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: filter ('', ResourceWarning) did not catch any warning def test_destructor(self): - super().test_destructor(self) + return super().test_destructor() @support.cpython_only class TestIOCTypes(unittest.TestCase): @@ -1165,9 +1186,32 @@ def test_disallow_instantiation(self): _io = self._io support.check_disallow_instantiation(self, _io._BytesIOBuffer) + def test_stringio_setstate(self): + # gh-127182: Calling __setstate__() with invalid arguments must not crash + obj = self._io.StringIO() + with self.assertRaisesRegex( + TypeError, + 'initial_value must be str or None, not int', + ): + obj.__setstate__((1, '', 0, {})) + + obj.__setstate__((None, '', 0, {})) # should not crash + self.assertEqual(obj.getvalue(), '') + + obj.__setstate__(('', '', 0, {})) + self.assertEqual(obj.getvalue(), '') + class PyIOTest(IOTest): pass + @unittest.expectedFailure # TODO: RUSTPYTHON; OSError: Negative file descriptor + def test_bad_opener_negative_1(): + return super().test_bad_opener_negative_1() + + @unittest.expectedFailure # TODO: RUSTPYTHON; OSError: Negative file descriptor + def test_bad_opener_other_negative(): + return super().test_bad_opener_other_negative() + @support.cpython_only class APIMismatchTest(unittest.TestCase): @@ -1175,7 +1219,7 @@ class APIMismatchTest(unittest.TestCase): def test_RawIOBase_io_in_pyio_match(self): """Test that pyio RawIOBase class has all c RawIOBase methods""" mismatch = support.detect_api_mismatch(pyio.RawIOBase, io.RawIOBase, - ignore=('__weakref__',)) + ignore=('__weakref__', '__static_attributes__')) self.assertEqual(mismatch, set(), msg='Python RawIOBase does not have all C RawIOBase methods') def test_RawIOBase_pyio_in_io_match(self): @@ -1244,6 +1288,7 @@ def _with(): # a ValueError. self.assertRaises(ValueError, _with) + @unittest.expectedFailure # TODO: RUSTPYTHON def test_error_through_destructor(self): # Test that the exception state is not modified by a destructor, # even if close() fails. @@ -1252,10 +1297,7 @@ def test_error_through_destructor(self): with self.assertRaises(AttributeError): self.tp(rawio).xyzzy - if not IOBASE_EMITS_UNRAISABLE: - self.assertIsNone(cm.unraisable) - elif cm.unraisable is not None: - self.assertEqual(cm.unraisable.exc_type, OSError) + self.assertEqual(cm.unraisable.exc_type, OSError) def test_repr(self): raw = self.MockRawIO() @@ -1271,11 +1313,9 @@ def test_recursive_repr(self): # Issue #25455 raw = self.MockRawIO() b = self.tp(raw) - with support.swap_attr(raw, 'name', b): - try: + with support.swap_attr(raw, 'name', b), support.infinite_recursion(25): + with self.assertRaises(RuntimeError): repr(b) # Should not crash - except RuntimeError: - pass def test_flush_error_on_close(self): # Test that buffered file is closed despite failed flush @@ -1356,6 +1396,28 @@ def test_readonly_attributes(self): with self.assertRaises(AttributeError): buf.raw = x + def test_pickling_subclass(self): + global MyBufferedIO + class MyBufferedIO(self.tp): + def __init__(self, raw, tag): + super().__init__(raw) + self.tag = tag + def __getstate__(self): + return self.tag, self.raw.getvalue() + def __setstate__(slf, state): + tag, value = state + slf.__init__(self.BytesIO(value), tag) + + raw = self.BytesIO(b'data') + buf = MyBufferedIO(raw, tag='ham') + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(protocol=proto): + pickled = pickle.dumps(buf, proto) + newbuf = pickle.loads(pickled) + self.assertEqual(newbuf.raw.getvalue(), b'data') + self.assertEqual(newbuf.tag, 'ham') + del MyBufferedIO + class SizeofTest: @@ -1717,20 +1779,6 @@ def test_seek_character_device_file(self): class CBufferedReaderTest(BufferedReaderTest, SizeofTest): tp = io.BufferedReader - @unittest.skip("TODO: RUSTPYTHON, fallible allocation") - @skip_if_sanitizer(memory=True, address=True, thread=True, - reason="sanitizer defaults to crashing " - "instead of returning NULL for malloc failure.") - def test_constructor(self): - BufferedReaderTest.test_constructor(self) - # The allocation can succeed on 32-bit builds, e.g. with more - # than 2 GiB RAM and a 64-bit kernel. - if sys.maxsize > 0x7FFFFFFF: - rawio = self.MockRawIO() - bufio = self.tp(rawio) - self.assertRaises((OverflowError, MemoryError, ValueError), - bufio.__init__, rawio, sys.maxsize) - def test_initialization(self): rawio = self.MockRawIO([b"abc"]) bufio = self.tp(rawio) @@ -1748,8 +1796,8 @@ def test_misbehaved_io_read(self): # checking this is not so easy. self.assertRaises(OSError, bufio.read, 10) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.skipIf(sys.platform == 'win32', 'TODO: RUSTPYTHON; cyclic GC not supported, causes file locking') + @unittest.expectedFailure # TODO: RUSTPYTHON def test_garbage_collection(self): # C BufferedReader objects are collected. # The Python version has __del__, so it ends into gc.garbage instead @@ -1766,40 +1814,44 @@ def test_garbage_collection(self): def test_args_error(self): # Issue #17275 with self.assertRaisesRegex(TypeError, "BufferedReader"): - self.tp(io.BytesIO(), 1024, 1024, 1024) + self.tp(self.BytesIO(), 1024, 1024, 1024) def test_bad_readinto_value(self): - rawio = io.BufferedReader(io.BytesIO(b"12")) + rawio = self.tp(self.BytesIO(b"12")) rawio.readinto = lambda buf: -1 bufio = self.tp(rawio) with self.assertRaises(OSError) as cm: bufio.readline() self.assertIsNone(cm.exception.__cause__) - # TODO: RUSTPYTHON, TypeError: 'bytes' object cannot be interpreted as an integer") - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: 'bytes' object cannot be interpreted as an integer") def test_bad_readinto_type(self): - rawio = io.BufferedReader(io.BytesIO(b"12")) + rawio = self.tp(self.BytesIO(b"12")) rawio.readinto = lambda buf: b'' bufio = self.tp(rawio) with self.assertRaises(OSError) as cm: bufio.readline() self.assertIsInstance(cm.exception.__cause__, TypeError) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_flush_error_on_close(self): - super().test_flush_error_on_close() + return super().test_flush_error_on_close() - # TODO: RUSTPYTHON, AssertionError: UnsupportedOperation not raised by truncate - @unittest.expectedFailure - def test_truncate_on_read_only(self): # TODO: RUSTPYTHON, remove when this passes - super().test_truncate_on_read_only() # TODO: RUSTPYTHON, remove when this passes - - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_seek_character_device_file(self): - super().test_seek_character_device_file() + return super().test_seek_character_device_file() + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: UnsupportedOperation not raised by truncate + def test_truncate_on_read_only(self): + return super().test_truncate_on_read_only() + + @unittest.skip('TODO: RUSTPYTHON; fallible allocation') + def test_constructor(self): + return super().test_constructor() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_pickling_subclass(self): + return super().test_pickling_subclass() class PyBufferedReaderTest(BufferedReaderTest): @@ -1909,8 +1961,7 @@ def _seekrel(bufio): def test_writes_and_truncates(self): self.check_writes(lambda bufio: bufio.truncate(bufio.tell())) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_write_non_blocking(self): raw = self.MockNonBlockWriterIO() bufio = self.tp(raw, 8) @@ -2107,20 +2158,6 @@ def test_slow_close_from_thread(self): class CBufferedWriterTest(BufferedWriterTest, SizeofTest): tp = io.BufferedWriter - @unittest.skip("TODO: RUSTPYTHON, fallible allocation") - @skip_if_sanitizer(memory=True, address=True, thread=True, - reason="sanitizer defaults to crashing " - "instead of returning NULL for malloc failure.") - def test_constructor(self): - BufferedWriterTest.test_constructor(self) - # The allocation can succeed on 32-bit builds, e.g. with more - # than 2 GiB RAM and a 64-bit kernel. - if sys.maxsize > 0x7FFFFFFF: - rawio = self.MockRawIO() - bufio = self.tp(rawio) - self.assertRaises((OverflowError, MemoryError, ValueError), - bufio.__init__, rawio, sys.maxsize) - def test_initialization(self): rawio = self.MockRawIO() bufio = self.tp(rawio) @@ -2131,8 +2168,8 @@ def test_initialization(self): self.assertRaises(ValueError, bufio.__init__, rawio, buffer_size=-1) self.assertRaises(ValueError, bufio.write, b"def") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.skipIf(sys.platform == 'win32', 'TODO: RUSTPYTHON; cyclic GC not supported, causes file locking') + @unittest.expectedFailure # TODO: RUSTPYTHON def test_garbage_collection(self): # C BufferedWriter objects are collected, and collecting them flushes # all data to disk. @@ -2155,11 +2192,17 @@ def test_args_error(self): with self.assertRaisesRegex(TypeError, "BufferedWriter"): self.tp(self.BytesIO(), 1024, 1024, 1024) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_flush_error_on_close(self): - super().test_flush_error_on_close() + return super().test_flush_error_on_close() + + @unittest.skip('TODO: RUSTPYTHON; fallible allocation') + def test_constructor(self): + return super().test_constructor() + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_pickling_subclass(self): + return super().test_pickling_subclass() class PyBufferedWriterTest(BufferedWriterTest): tp = pyio.BufferedWriter @@ -2637,22 +2680,8 @@ def test_interleaved_readline_write(self): class CBufferedRandomTest(BufferedRandomTest, SizeofTest): tp = io.BufferedRandom - @unittest.skip("TODO: RUSTPYTHON, fallible allocation") - @skip_if_sanitizer(memory=True, address=True, thread=True, - reason="sanitizer defaults to crashing " - "instead of returning NULL for malloc failure.") - def test_constructor(self): - BufferedRandomTest.test_constructor(self) - # The allocation can succeed on 32-bit builds, e.g. with more - # than 2 GiB RAM and a 64-bit kernel. - if sys.maxsize > 0x7FFFFFFF: - rawio = self.MockRawIO() - bufio = self.tp(rawio) - self.assertRaises((OverflowError, MemoryError, ValueError), - bufio.__init__, rawio, sys.maxsize) - - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.skipIf(sys.platform == 'win32', 'TODO: RUSTPYTHON; cyclic GC not supported, causes file locking') + @unittest.expectedFailure # TODO: RUSTPYTHON def test_garbage_collection(self): CBufferedReaderTest.test_garbage_collection(self) CBufferedWriterTest.test_garbage_collection(self) @@ -2662,20 +2691,25 @@ def test_args_error(self): with self.assertRaisesRegex(TypeError, "BufferedRandom"): self.tp(self.BytesIO(), 1024, 1024, 1024) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_flush_error_on_close(self): - super().test_flush_error_on_close() + return super().test_flush_error_on_close() - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_seek_character_device_file(self): - super().test_seek_character_device_file() + return super().test_seek_character_device_file() - # TODO: RUSTPYTHON; f.read1(1) returns b'a' - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; f.read1(1) returns b'a' def test_read1_after_write(self): - super().test_read1_after_write() + return super().test_read1_after_write() + + @unittest.skip('TODO: RUSTPYTHON; fallible allocation') + def test_constructor(self): + return super().test_constructor() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_pickling_subclass(self): + return super().test_pickling_subclass() class PyBufferedRandomTest(BufferedRandomTest): @@ -2935,11 +2969,16 @@ def test_recursive_repr(self): # Issue #25455 raw = self.BytesIO() t = self.TextIOWrapper(raw, encoding="utf-8") - with support.swap_attr(raw, 'name', t): - try: + with support.swap_attr(raw, 'name', t), support.infinite_recursion(25): + with self.assertRaises(RuntimeError): repr(t) # Should not crash - except RuntimeError: - pass + + def test_subclass_repr(self): + class TestSubclass(self.TextIOWrapper): + pass + + f = TestSubclass(self.StringIO()) + self.assertIn(TestSubclass.__name__, repr(f)) def test_line_buffering(self): r = self.BytesIO() @@ -3166,6 +3205,7 @@ def flush(self): support.gc_collect() self.assertEqual(record, [1, 2, 3]) + @unittest.expectedFailure # TODO: RUSTPYTHON def test_error_through_destructor(self): # Test that the exception state is not modified by a destructor, # even if close() fails. @@ -3174,10 +3214,7 @@ def test_error_through_destructor(self): with self.assertRaises(AttributeError): self.TextIOWrapper(rawio, encoding="utf-8").xyzzy - if not IOBASE_EMITS_UNRAISABLE: - self.assertIsNone(cm.unraisable) - elif cm.unraisable is not None: - self.assertEqual(cm.unraisable.exc_type, OSError) + self.assertEqual(cm.unraisable.exc_type, OSError) # Systematic tests of the text I/O API @@ -3327,8 +3364,7 @@ def test_seek_and_tell_with_data(data, min_pos=0): finally: StatefulIncrementalDecoder.codecEnabled = 0 - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_multibyte_seek_and_tell(self): f = self.open(os_helper.TESTFN, "w", encoding="euc_jp") f.write("AB\n\u3046\u3048\n") @@ -3344,8 +3380,7 @@ def test_multibyte_seek_and_tell(self): self.assertEqual(f.tell(), p1) f.close() - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_seek_with_encoder_state(self): f = self.open(os_helper.TESTFN, "w", encoding="euc_jis_2004") f.write("\u00e6\u0300") @@ -3359,8 +3394,7 @@ def test_seek_with_encoder_state(self): self.assertEqual(f.readline(), "\u00e6\u0300\u0300") f.close() - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_encoded_writes(self): data = "1234567890" tests = ("utf-16", @@ -3499,8 +3533,7 @@ def test_issue2282(self): self.assertEqual(buffer.seekable(), txt.seekable()) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_append_bom(self): # The BOM is not written again when appending to a non-empty file filename = os_helper.TESTFN @@ -3516,8 +3549,7 @@ def test_append_bom(self): with self.open(filename, 'rb') as f: self.assertEqual(f.read(), 'aaaxxx'.encode(charset)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_seek_bom(self): # Same test, but when seeking manually filename = os_helper.TESTFN @@ -3533,8 +3565,7 @@ def test_seek_bom(self): with self.open(filename, 'rb') as f: self.assertEqual(f.read(), 'bbbzzz'.encode(charset)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_seek_append_bom(self): # Same test, but first seek to the start and then to the end filename = os_helper.TESTFN @@ -3795,17 +3826,14 @@ def _check_create_at_shutdown(self, **kwargs): codecs.lookup('utf-8') class C: - def __init__(self): - self.buf = io.BytesIO() def __del__(self): - io.TextIOWrapper(self.buf, **{kwargs}) + io.TextIOWrapper(io.BytesIO(), **{kwargs}) print("ok") c = C() """.format(iomod=iomod, kwargs=kwargs) return assert_python_ok("-c", code) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_create_at_shutdown_without_encoding(self): rc, out, err = self._check_create_at_shutdown() if err: @@ -3815,8 +3843,7 @@ def test_create_at_shutdown_without_encoding(self): else: self.assertEqual("ok", out.decode().strip()) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_create_at_shutdown_with_encoding(self): rc, out, err = self._check_create_at_shutdown(encoding='utf-8', errors='strict') @@ -4042,6 +4069,28 @@ def test_issue35928(self): f.write(res) self.assertEqual(res + f.readline(), 'foo\nbar\n') + def test_pickling_subclass(self): + global MyTextIO + class MyTextIO(self.TextIOWrapper): + def __init__(self, raw, tag): + super().__init__(raw) + self.tag = tag + def __getstate__(self): + return self.tag, self.buffer.getvalue() + def __setstate__(slf, state): + tag, value = state + slf.__init__(self.BytesIO(value), tag) + + raw = self.BytesIO(b'data') + txt = MyTextIO(raw, 'ham') + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(protocol=proto): + pickled = pickle.dumps(txt, proto) + newtxt = pickle.loads(pickled) + self.assertEqual(newtxt.buffer.getvalue(), b'data') + self.assertEqual(newtxt.tag, 'ham') + del MyTextIO + class MemviewBytesIO(io.BytesIO): '''A BytesIO object whose read method returns memoryviews @@ -4066,98 +4115,7 @@ class CTextIOWrapperTest(TextIOWrapperTest): io = io shutdown_error = "LookupError: unknown encoding: ascii" - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_constructor(self): - super().test_constructor() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_detach(self): - super().test_detach() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_reconfigure_encoding_read(self): - super().test_reconfigure_encoding_read() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_reconfigure_line_buffering(self): - super().test_reconfigure_line_buffering() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_basic_io(self): - super().test_basic_io() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_telling(self): - super().test_telling() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_uninitialized(self): - super().test_uninitialized() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_non_text_encoding_codecs_are_rejected(self): - super().test_non_text_encoding_codecs_are_rejected() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_repr(self): - super().test_repr() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_newlines(self): - super().test_newlines() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_newlines_input(self): - super().test_newlines_input() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_reconfigure_write_through(self): - super().test_reconfigure_write_through() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_reconfigure_write_fromascii(self): - super().test_reconfigure_write_fromascii() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_reconfigure_write(self): - super().test_reconfigure_write() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_reconfigure_defaults(self): - super().test_reconfigure_defaults() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_reconfigure_newline(self): - super().test_reconfigure_newline() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_reconfigure_errors(self): - super().test_reconfigure_errors() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_reconfigure_locale(self): - super().test_reconfigure_locale() - - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_initialization(self): r = self.BytesIO(b"\xc3\xa9\n\n") b = self.BufferedReader(r, 1000) @@ -4168,8 +4126,8 @@ def test_initialization(self): t = self.TextIOWrapper.__new__(self.TextIOWrapper) self.assertRaises(Exception, repr, t) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.skipIf(sys.platform == 'win32', 'TODO: RUSTPYTHON; cyclic GC not supported, causes file locking') + @unittest.expectedFailure # TODO: RUSTPYTHON def test_garbage_collection(self): # C TextIOWrapper objects are collected, and collecting them flushes # all data to disk. @@ -4233,20 +4191,115 @@ def write(self, data): t.write("x"*chunk_size) self.assertEqual([b"abcdef", b"ghi", b"x"*chunk_size], buf._write_stack) + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_issue119506(self): + chunk_size = 8192 + + class MockIO(self.MockRawIO): + written = False + def write(self, data): + if not self.written: + self.written = True + t.write("middle") + return super().write(data) + + buf = MockIO() + t = self.TextIOWrapper(buf) + t.write("abc") + t.write("def") + # writing data which size >= chunk_size cause flushing buffer before write. + t.write("g" * chunk_size) + t.flush() + + self.assertEqual([b"abcdef", b"middle", b"g"*chunk_size], + buf._write_stack) + + def test_basic_io(self): + return super().test_basic_io() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_constructor(self): + return super().test_constructor() + + def test_detach(self): + return super().test_detach() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_newlines(self): + return super().test_newlines() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_newlines_input(self): + return super().test_newlines_input() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_non_text_encoding_codecs_are_rejected(self): + return super().test_non_text_encoding_codecs_are_rejected() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_reconfigure_defaults(self): + return super().test_reconfigure_defaults() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_reconfigure_encoding_read(self): + return super().test_reconfigure_encoding_read() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_reconfigure_errors(self): + return super().test_reconfigure_errors() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_reconfigure_line_buffering(self): + return super().test_reconfigure_line_buffering() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_reconfigure_locale(self): + return super().test_reconfigure_locale() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_reconfigure_newline(self): + return super().test_reconfigure_newline() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_reconfigure_write(self): + return super().test_reconfigure_write() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_reconfigure_write_fromascii(self): + return super().test_reconfigure_write_fromascii() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_reconfigure_write_through(self): + return super().test_reconfigure_write_through() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_repr(self): + return super().test_repr() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_uninitialized(self): + return super().test_uninitialized() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_recursive_repr(self): + return super().test_recursive_repr() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_pickling_subclass(self): + return super().test_pickling_subclass() + class PyTextIOWrapperTest(TextIOWrapperTest): io = pyio shutdown_error = "LookupError: unknown encoding: ascii" - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: ValueError not raised def test_constructor(self): - super().test_constructor() + return super().test_constructor() - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_newlines(self): - super().test_newlines() + return super().test_newlines() class IncrementalNewlineDecoderTest(unittest.TestCase): @@ -4326,8 +4379,7 @@ def _decode_bytewise(s): self.assertEqual(decoder.decode(input), "abc") self.assertEqual(decoder.newlines, None) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_newline_decoder(self): encodings = ( # None meaning the IncrementalNewlineDecoder takes unicode input @@ -4489,8 +4541,7 @@ def test_io_after_close(self): self.assertRaises(ValueError, f.writelines, []) self.assertRaises(ValueError, next, f) - # TODO: RUSTPYTHON, cyclic gc - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; cyclic gc def test_blockingioerror(self): # Various BlockingIOError issues class C(str): @@ -4538,15 +4589,14 @@ def test_abc_inheritance_official(self): self._check_abc_inheritance(io) def _check_warn_on_dealloc(self, *args, **kwargs): - f = open(*args, **kwargs) + f = self.open(*args, **kwargs) r = repr(f) with self.assertWarns(ResourceWarning) as cm: f = None support.gc_collect() self.assertIn(r, str(cm.warning.args[0])) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_warn_on_dealloc(self): self._check_warn_on_dealloc(os_helper.TESTFN, "wb", buffering=0) self._check_warn_on_dealloc(os_helper.TESTFN, "wb") @@ -4569,10 +4619,9 @@ def cleanup_fds(): r, w = os.pipe() fds += r, w with warnings_helper.check_no_resource_warning(self): - open(r, *args, closefd=False, **kwargs) + self.open(r, *args, closefd=False, **kwargs) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON @unittest.skipUnless(hasattr(os, "pipe"), "requires os.pipe()") def test_warn_on_dealloc_fd(self): self._check_warn_on_dealloc_fd("rb", buffering=0) @@ -4602,16 +4651,14 @@ def test_pickling(self): with self.assertRaisesRegex(TypeError, msg): pickle.dumps(f, protocol) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON @unittest.skipIf( support.is_emscripten, "fstat() of a pipe fd is not supported" ) def test_nonblock_pipe_write_bigbuf(self): self._test_nonblock_pipe_write(16*1024) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON @unittest.skipIf( support.is_emscripten, "fstat() of a pipe fd is not supported" ) @@ -4733,8 +4780,7 @@ def test_check_encoding_errors(self): proc = assert_python_failure('-X', 'dev', '-c', code) self.assertEqual(proc.rc, 10, proc) - # TODO: RUSTPYTHON, AssertionError: 0 != 2 - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 0 != 2 def test_check_encoding_warning(self): # PEP 597: Raise warning when encoding is not specified # and sys.flags.warn_default_encoding is set. @@ -4758,8 +4804,7 @@ def test_check_encoding_warning(self): self.assertTrue( warnings[1].startswith(b"<string>:8: EncodingWarning: ")) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_text_encoding(self): # PEP 597, bpo-47000. io.text_encoding() returns "locale" or "utf-8" # based on sys.flags.utf8_mode @@ -4837,10 +4882,9 @@ def test_daemon_threads_shutdown_stdout_deadlock(self): def test_daemon_threads_shutdown_stderr_deadlock(self): self.check_daemon_threads_shutdown_deadlock('stderr') - # TODO: RUSTPYTHON, AssertionError: 22 != 10 : _PythonRunResult(rc=22, out=b'', err=b'') - @unittest.expectedFailure - def test_check_encoding_errors(self): # TODO: RUSTPYTHON, remove when this passes - super().test_check_encoding_errors() # TODO: RUSTPYTHON, remove when this passes + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: 22 != 10 : _PythonRunResult(rc=22, out=b'', err=b'') + def test_check_encoding_errors(self): + return super().test_check_encoding_errors() class PyMiscIOTest(MiscIOTest): @@ -5014,16 +5058,14 @@ def alarm_handler(sig, frame): os.close(w) os.close(r) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON @requires_alarm @support.requires_resource('walltime') def test_interrupted_read_retry_buffered(self): self.check_interrupted_read_retry(lambda x: x.decode('latin1'), mode="rb") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON @requires_alarm @support.requires_resource('walltime') def test_interrupted_read_retry_text(self): @@ -5098,15 +5140,13 @@ def alarm2(sig, frame): if e.errno != errno.EBADF: raise - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON @requires_alarm @support.requires_resource('walltime') def test_interrupted_write_retry_buffered(self): self.check_interrupted_write_retry(b"x", mode="wb") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON @requires_alarm @support.requires_resource('walltime') def test_interrupted_write_retry_text(self): diff --git a/Lib/test/test_isinstance.py b/Lib/test/test_isinstance.py index de80e47209..95a119ba68 100644 --- a/Lib/test/test_isinstance.py +++ b/Lib/test/test_isinstance.py @@ -352,9 +352,7 @@ def blowstack(fxn, arg, compare_to): # Make sure that calling isinstance with a deeply nested tuple for its # argument will raise RecursionError eventually. tuple_arg = (compare_to,) - # XXX: RUSTPYTHON; support.exceeds_recursion_limit() is not available yet. - import sys - for cnt in range(sys.getrecursionlimit()+5): + for cnt in range(support.exceeds_recursion_limit()): tuple_arg = (tuple_arg,) fxn(arg, tuple_arg) diff --git a/Lib/test/test_itertools.py b/Lib/test/test_itertools.py index 8709948b92..03dadb71f7 100644 --- a/Lib/test/test_itertools.py +++ b/Lib/test/test_itertools.py @@ -1,5 +1,6 @@ import doctest import unittest +import itertools from test import support from test.support import threading_helper, script_helper from itertools import * @@ -29,10 +30,11 @@ def inner(self): with warnings.catch_warnings(): warnings.simplefilter("ignore", category=DeprecationWarning) testfunc(self) - with warnings.catch_warnings(): - warnings.simplefilter("error", category=DeprecationWarning) - with self.assertRaises((DeprecationWarning, AssertionError, SystemError)): - testfunc(self) + # XXX: RUSTPYTHON; Patch to make tests pass. It will be removed once 3.14 is released anyway. + # with warnings.catch_warnings(): + # warnings.simplefilter("error", category=DeprecationWarning) + # with self.assertRaises((DeprecationWarning, AssertionError, SystemError)): + # testfunc(self) return inner @@ -144,8 +146,7 @@ def expand(it, i=0): c = expand(compare[took:]) self.assertEqual(a, c); - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; [7, 7, 8, 10] != <itertools.accumulate object at 0xb4000073f32b4480> @pickle_deprecated def test_accumulate(self): self.assertEqual(list(accumulate(range(10))), # one positional arg @@ -189,7 +190,11 @@ def test_batched(self): [('A', 'B'), ('C', 'D'), ('E', 'F'), ('G',)]) self.assertEqual(list(batched('ABCDEFG', 1)), [('A',), ('B',), ('C',), ('D',), ('E',), ('F',), ('G',)]) + self.assertEqual(list(batched('ABCDEF', 2, strict=True)), + [('A', 'B'), ('C', 'D'), ('E', 'F')]) + with self.assertRaises(ValueError): # Incomplete batch when strict + list(batched('ABCDEFG', 3, strict=True)) with self.assertRaises(TypeError): # Too few arguments list(batched('ABCDEFG')) with self.assertRaises(TypeError): @@ -243,8 +248,6 @@ def test_chain_from_iterable(self): self.assertRaises(TypeError, list, chain.from_iterable([2, 3])) self.assertEqual(list(islice(chain.from_iterable(repeat(range(5))), 2)), [0, 1]) - # TODO: RUSTPYTHON - @unittest.expectedFailure @pickle_deprecated def test_chain_reducible(self): for oper in [copy.deepcopy] + picklecopiers: @@ -259,8 +262,7 @@ def test_chain_reducible(self): for proto in range(pickle.HIGHEST_PROTOCOL + 1): self.pickletest(proto, chain('abc', 'def'), compare=list('abcdef')) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON @pickle_deprecated def test_chain_setstate(self): self.assertRaises(TypeError, chain().__setstate__, ()) @@ -275,8 +277,7 @@ def test_chain_setstate(self): it.__setstate__((iter(['abc', 'def']), iter(['ghi']))) self.assertEqual(list(it), ['ghi', 'a', 'b', 'c', 'd', 'e', 'f']) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON @pickle_deprecated def test_combinations(self): self.assertRaises(TypeError, combinations, 'abc') # missing r argument @@ -368,8 +369,7 @@ def test_combinations_tuple_reuse(self): self.assertEqual(len(set(map(id, combinations('abcde', 3)))), 1) self.assertNotEqual(len(set(map(id, list(combinations('abcde', 3))))), 1) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON @pickle_deprecated def test_combinations_with_replacement(self): cwr = combinations_with_replacement @@ -459,8 +459,7 @@ def test_combinations_with_replacement_tuple_reuse(self): self.assertEqual(len(set(map(id, cwr('abcde', 3)))), 1) self.assertNotEqual(len(set(map(id, list(cwr('abcde', 3))))), 1) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON @pickle_deprecated def test_permutations(self): self.assertRaises(TypeError, permutations) # too few arguments @@ -568,8 +567,6 @@ def test_combinatorics(self): self.assertEqual(comb, list(filter(set(perm).__contains__, cwr))) # comb: cwr that is a perm self.assertEqual(comb, sorted(set(cwr) & set(perm))) # comb: both a cwr and a perm - # TODO: RUSTPYTHON - @unittest.expectedFailure @pickle_deprecated def test_compress(self): self.assertEqual(list(compress(data='ABCDEF', selectors=[1,0,1,0,1,1])), list('ACEF')) @@ -604,8 +601,6 @@ def test_compress(self): next(testIntermediate) self.assertEqual(list(op(testIntermediate)), list(result2)) - # TODO: RUSTPYTHON - @unittest.expectedFailure @pickle_deprecated def test_count(self): self.assertEqual(lzip('abc',count()), [('a', 0), ('b', 1), ('c', 2)]) @@ -615,6 +610,8 @@ def test_count(self): self.assertEqual(take(2, zip('abc',count(-3))), [('a', -3), ('b', -2)]) self.assertRaises(TypeError, count, 2, 3, 4) self.assertRaises(TypeError, count, 'a') + self.assertEqual(take(3, count(maxsize)), + [maxsize, maxsize + 1, maxsize + 2]) self.assertEqual(take(10, count(maxsize-5)), list(range(maxsize-5, maxsize+5))) self.assertEqual(take(10, count(-maxsize-5)), @@ -637,6 +634,15 @@ def test_count(self): self.assertEqual(next(c), -8) self.assertEqual(repr(count(10.25)), 'count(10.25)') self.assertEqual(repr(count(10.0)), 'count(10.0)') + + self.assertEqual(repr(count(maxsize)), f'count({maxsize})') + c = count(maxsize - 1) + self.assertEqual(repr(c), f'count({maxsize - 1})') + next(c) # c is now at masize + self.assertEqual(repr(c), f'count({maxsize})') + next(c) + self.assertEqual(repr(c), f'count({maxsize + 1})') + self.assertEqual(type(next(count(10.0))), float) for i in (-sys.maxsize-5, -sys.maxsize+5 ,-10, -1, 0, 10, sys.maxsize-5, sys.maxsize+5): # Test repr @@ -655,10 +661,9 @@ def test_count(self): #check proper internal error handling for large "step' sizes count(1, maxsize+5); sys.exc_info() - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; 'count(10.5)' != 'count(10.5, 1.0)' @pickle_deprecated - def test_count_with_stride(self): + def test_count_with_step(self): self.assertEqual(lzip('abc',count(2,3)), [('a', 2), ('b', 5), ('c', 8)]) self.assertEqual(lzip('abc',count(start=2,step=3)), [('a', 2), ('b', 5), ('c', 8)]) @@ -672,6 +677,12 @@ def test_count_with_stride(self): self.assertEqual(take(20, count(-maxsize-15, 3)), take(20, range(-maxsize-15,-maxsize+100, 3))) self.assertEqual(take(3, count(10, maxsize+5)), list(range(10, 10+3*(maxsize+5), maxsize+5))) + self.assertEqual(take(3, count(maxsize, 2)), + [maxsize, maxsize + 2, maxsize + 4]) + self.assertEqual(take(3, count(maxsize, maxsize)), + [maxsize, 2 * maxsize, 3 * maxsize]) + self.assertEqual(take(3, count(-maxsize, maxsize)), + [-maxsize, 0, maxsize]) self.assertEqual(take(3, count(2, 1.25)), [2, 3.25, 4.5]) self.assertEqual(take(3, count(2, 3.25-4j)), [2, 5.25-4j, 8.5-8j]) self.assertEqual(take(3, count(Decimal('1.1'), Decimal('.1'))), @@ -713,6 +724,42 @@ def test_count_with_stride(self): for proto in range(pickle.HIGHEST_PROTOCOL + 1): self.pickletest(proto, count(i, j)) + c = count(maxsize -2, 2) + self.assertEqual(repr(c), f'count({maxsize - 2}, 2)') + next(c) # c is now at masize + self.assertEqual(repr(c), f'count({maxsize}, 2)') + next(c) + self.assertEqual(repr(c), f'count({maxsize + 2}, 2)') + + c = count(maxsize + 1, -1) + self.assertEqual(repr(c), f'count({maxsize + 1}, -1)') + next(c) # c is now at masize + self.assertEqual(repr(c), f'count({maxsize}, -1)') + next(c) + self.assertEqual(repr(c), f'count({maxsize - 1}, -1)') + + @threading_helper.requires_working_threading() + def test_count_threading(self, step=1): + # this test verifies multithreading consistency, which is + # mostly for testing builds without GIL, but nice to test anyway + count_to = 10_000 + num_threads = 10 + c = count(step=step) + def counting_thread(): + for i in range(count_to): + next(c) + threads = [] + for i in range(num_threads): + thread = threading.Thread(target=counting_thread) + thread.start() + threads.append(thread) + for thread in threads: + thread.join() + self.assertEqual(next(c), count_to * num_threads * step) + + def test_count_with_step_threading(self): + self.test_count_threading(step=5) + def test_cycle(self): self.assertEqual(take(10, cycle('abc')), list('abcabcabca')) self.assertEqual(list(cycle('')), []) @@ -720,8 +767,7 @@ def test_cycle(self): self.assertRaises(TypeError, cycle, 5) self.assertEqual(list(islice(cycle(gen3()),10)), [0,1,2,0,1,2,0,1,2,0]) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON @pickle_deprecated def test_cycle_copy_pickle(self): # check copy, deepcopy, pickle @@ -759,8 +805,7 @@ def test_cycle_copy_pickle(self): d = pickle.loads(p) # rebuild the cycle object self.assertEqual(take(20, d), list('cdeabcdeabcdeabcdeab')) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON @pickle_deprecated def test_cycle_unpickle_compat(self): testcases = [ @@ -793,8 +838,7 @@ def test_cycle_unpickle_compat(self): it = pickle.loads(t) self.assertEqual(take(10, it), [2, 3, 1, 2, 3, 1, 2, 3, 1, 2]) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON @pickle_deprecated def test_cycle_setstate(self): # Verify both modes for restoring state @@ -832,8 +876,7 @@ def test_cycle_setstate(self): self.assertRaises(TypeError, cycle('').__setstate__, ()) self.assertRaises(TypeError, cycle('').__setstate__, ([],)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON @pickle_deprecated def test_groupby(self): # Check whether it accepts arguments correctly @@ -992,8 +1035,6 @@ def test_filter(self): c = filter(isEven, range(6)) self.pickletest(proto, c) - # TODO: RUSTPYTHON - @unittest.expectedFailure @pickle_deprecated def test_filterfalse(self): self.assertEqual(list(filterfalse(isEven, range(6))), [1,3,5]) @@ -1101,8 +1142,6 @@ def test_zip_longest_tuple_reuse(self): ids = list(map(id, list(zip_longest('abc', 'def')))) self.assertEqual(len(dict.fromkeys(ids)), len(ids)) - # TODO: RUSTPYTHON - @unittest.expectedFailure @pickle_deprecated def test_zip_longest_pickling(self): for proto in range(pickle.HIGHEST_PROTOCOL + 1): @@ -1292,12 +1331,16 @@ def product1(*args, **kwds): else: return - def product2(*args, **kwds): + def product2(*iterables, repeat=1): 'Pure python version used in docs' - pools = list(map(tuple, args)) * kwds.get('repeat', 1) + if repeat < 0: + raise ValueError('repeat argument cannot be negative') + pools = [tuple(pool) for pool in iterables] * repeat + result = [[]] for pool in pools: result = [x+[y] for x in result for y in pool] + for prod in result: yield tuple(prod) @@ -1322,8 +1365,6 @@ def test_product_tuple_reuse(self): self.assertEqual(len(set(map(id, product('abc', 'def')))), 1) self.assertNotEqual(len(set(map(id, list(product('abc', 'def'))))), 1) - # TODO: RUSTPYTHON - @unittest.expectedFailure @pickle_deprecated def test_product_pickling(self): # check copy, deepcopy, pickle @@ -1340,8 +1381,7 @@ def test_product_pickling(self): for proto in range(pickle.HIGHEST_PROTOCOL + 1): self.pickletest(proto, product(*args)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON @pickle_deprecated def test_product_issue_25021(self): # test that indices are properly clamped to the length of the tuples @@ -1353,8 +1393,6 @@ def test_product_issue_25021(self): p.__setstate__((0, 0, 0x1000)) # will access tuple element 1 if not clamped self.assertRaises(StopIteration, next, p) - # TODO: RUSTPYTHON - @unittest.expectedFailure @pickle_deprecated def test_repeat(self): self.assertEqual(list(repeat(object='a', times=3)), ['a', 'a', 'a']) @@ -1388,8 +1426,7 @@ def test_repeat_with_negative_times(self): self.assertEqual(repr(repeat('a', times=-1)), "repeat('a', 0)") self.assertEqual(repr(repeat('a', times=-2)), "repeat('a', 0)") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON @pickle_deprecated def test_map(self): self.assertEqual(list(map(operator.pow, range(3), range(1,7))), @@ -1421,8 +1458,6 @@ def test_map(self): c = map(tupleize, 'abc', count()) self.pickletest(proto, c) - # TODO: RUSTPYTHON - @unittest.expectedFailure @pickle_deprecated def test_starmap(self): self.assertEqual(list(starmap(operator.pow, zip(range(3), range(1,7)))), @@ -1451,8 +1486,7 @@ def test_starmap(self): c = starmap(operator.pow, zip(range(3), range(1,7))) self.pickletest(proto, c) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON @pickle_deprecated def test_islice(self): for args in [ # islice(args) should agree with range(args) @@ -1548,8 +1582,6 @@ def __index__(self): self.assertEqual(list(islice(range(100), IntLike(10), IntLike(50), IntLike(5))), list(range(10,50,5))) - # TODO: RUSTPYTHON - @unittest.expectedFailure @pickle_deprecated def test_takewhile(self): data = [1, 3, 5, 20, 2, 4, 6, 8] @@ -1571,8 +1603,7 @@ def test_takewhile(self): for proto in range(pickle.HIGHEST_PROTOCOL + 1): self.pickletest(proto, takewhile(underten, data)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON @pickle_deprecated def test_dropwhile(self): data = [1, 3, 5, 20, 2, 4, 6, 8] @@ -1591,8 +1622,7 @@ def test_dropwhile(self): for proto in range(pickle.HIGHEST_PROTOCOL + 1): self.pickletest(proto, dropwhile(underten, data)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON @pickle_deprecated def test_tee(self): n = 200 @@ -1660,10 +1690,11 @@ def test_tee(self): self.assertEqual(len(result), n) self.assertEqual([list(x) for x in result], [list('abc')]*n) - # tee pass-through to copyable iterator + # tee objects are independent (see bug gh-123884) a, b = tee('abc') c, d = tee(a) - self.assertTrue(a is c) + e, f = tee(c) + self.assertTrue(len({a, b, c, d, e, f}) == 6) # test tee_new t1, t2 = tee('abc') @@ -1761,8 +1792,6 @@ def test_tee_del_backward(self): del forward, backward raise - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_tee_reenter(self): class I: first = True @@ -1778,7 +1807,7 @@ def __next__(self): with self.assertRaisesRegex(RuntimeError, "tee"): next(a) - @unittest.skip("TODO: RUSTPYTHON, hangs") + @unittest.skip("TODO: RUSTPYTHON; , hangs") @threading_helper.requires_working_threading() def test_tee_concurrent(self): start = threading.Event() @@ -1868,6 +1897,13 @@ def test_zip_longest_result_gc(self): gc.collect() self.assertTrue(gc.is_tracked(next(it))) + @support.cpython_only + def test_pairwise_result_gc(self): + # Ditto for pairwise. + it = pairwise([None, None]) + gc.collect() + self.assertTrue(gc.is_tracked(next(it))) + @support.cpython_only def test_immutable_types(self): from itertools import _grouper, _tee, _tee_dataobject @@ -1906,8 +1942,6 @@ class TestExamples(unittest.TestCase): def test_accumulate(self): self.assertEqual(list(accumulate([1,2,3,4,5])), [1, 3, 6, 10, 15]) - # TODO: RUSTPYTHON - @unittest.expectedFailure @pickle_deprecated def test_accumulate_reducible(self): # check copy, deepcopy, pickle @@ -1924,8 +1958,7 @@ def test_accumulate_reducible(self): self.assertEqual(list(copy.deepcopy(it)), accumulated[1:]) self.assertEqual(list(copy.copy(it)), accumulated[1:]) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON @pickle_deprecated def test_accumulate_reducible_none(self): # Issue #25718: total is None @@ -2084,6 +2117,172 @@ def test_islice_recipe(self): self.assertEqual(next(c), 3) + def test_tee_recipe(self): + + # Begin tee() recipe ########################################### + + def tee(iterable, n=2): + if n < 0: + raise ValueError + if n == 0: + return () + iterator = _tee(iterable) + result = [iterator] + for _ in range(n - 1): + result.append(_tee(iterator)) + return tuple(result) + + class _tee: + + def __init__(self, iterable): + it = iter(iterable) + if isinstance(it, _tee): + self.iterator = it.iterator + self.link = it.link + else: + self.iterator = it + self.link = [None, None] + + def __iter__(self): + return self + + def __next__(self): + link = self.link + if link[1] is None: + link[0] = next(self.iterator) + link[1] = [None, None] + value, self.link = link + return value + + # End tee() recipe ############################################# + + n = 200 + + a, b = tee([]) # test empty iterator + self.assertEqual(list(a), []) + self.assertEqual(list(b), []) + + a, b = tee(irange(n)) # test 100% interleaved + self.assertEqual(lzip(a,b), lzip(range(n), range(n))) + + a, b = tee(irange(n)) # test 0% interleaved + self.assertEqual(list(a), list(range(n))) + self.assertEqual(list(b), list(range(n))) + + a, b = tee(irange(n)) # test dealloc of leading iterator + for i in range(100): + self.assertEqual(next(a), i) + del a + self.assertEqual(list(b), list(range(n))) + + a, b = tee(irange(n)) # test dealloc of trailing iterator + for i in range(100): + self.assertEqual(next(a), i) + del b + self.assertEqual(list(a), list(range(100, n))) + + for j in range(5): # test randomly interleaved + order = [0]*n + [1]*n + random.shuffle(order) + lists = ([], []) + its = tee(irange(n)) + for i in order: + value = next(its[i]) + lists[i].append(value) + self.assertEqual(lists[0], list(range(n))) + self.assertEqual(lists[1], list(range(n))) + + # test argument format checking + self.assertRaises(TypeError, tee) + self.assertRaises(TypeError, tee, 3) + self.assertRaises(TypeError, tee, [1,2], 'x') + self.assertRaises(TypeError, tee, [1,2], 3, 'x') + + # tee object should be instantiable + a, b = tee('abc') + c = type(a)('def') + self.assertEqual(list(c), list('def')) + + # test long-lagged and multi-way split + a, b, c = tee(range(2000), 3) + for i in range(100): + self.assertEqual(next(a), i) + self.assertEqual(list(b), list(range(2000))) + self.assertEqual([next(c), next(c)], list(range(2))) + self.assertEqual(list(a), list(range(100,2000))) + self.assertEqual(list(c), list(range(2,2000))) + + # test invalid values of n + self.assertRaises(TypeError, tee, 'abc', 'invalid') + self.assertRaises(ValueError, tee, [], -1) + + for n in range(5): + result = tee('abc', n) + self.assertEqual(type(result), tuple) + self.assertEqual(len(result), n) + self.assertEqual([list(x) for x in result], [list('abc')]*n) + + # tee objects are independent (see bug gh-123884) + a, b = tee('abc') + c, d = tee(a) + e, f = tee(c) + self.assertTrue(len({a, b, c, d, e, f}) == 6) + + # test tee_new + t1, t2 = tee('abc') + tnew = type(t1) + self.assertRaises(TypeError, tnew) + self.assertRaises(TypeError, tnew, 10) + t3 = tnew(t1) + self.assertTrue(list(t1) == list(t2) == list(t3) == list('abc')) + + # test that tee objects are weak referencable + a, b = tee(range(10)) + p = weakref.proxy(a) + self.assertEqual(getattr(p, '__class__'), type(b)) + del a + gc.collect() # For PyPy or other GCs. + self.assertRaises(ReferenceError, getattr, p, '__class__') + + ans = list('abc') + long_ans = list(range(10000)) + + # Tests not applicable to the tee() recipe + if False: + # check copy + a, b = tee('abc') + self.assertEqual(list(copy.copy(a)), ans) + self.assertEqual(list(copy.copy(b)), ans) + a, b = tee(list(range(10000))) + self.assertEqual(list(copy.copy(a)), long_ans) + self.assertEqual(list(copy.copy(b)), long_ans) + + # check partially consumed copy + a, b = tee('abc') + take(2, a) + take(1, b) + self.assertEqual(list(copy.copy(a)), ans[2:]) + self.assertEqual(list(copy.copy(b)), ans[1:]) + self.assertEqual(list(a), ans[2:]) + self.assertEqual(list(b), ans[1:]) + a, b = tee(range(10000)) + take(100, a) + take(60, b) + self.assertEqual(list(copy.copy(a)), long_ans[100:]) + self.assertEqual(list(copy.copy(b)), long_ans[60:]) + self.assertEqual(list(a), long_ans[100:]) + self.assertEqual(list(b), long_ans[60:]) + + # Issue 13454: Crash when deleting backward iterator from tee() + forward, backward = tee(repeat(None, 2000)) # 20000000 + try: + any(forward) # exhaust the iterator + del backward + except: + del forward, backward + raise + + class TestGC(unittest.TestCase): def makecycle(self, iterator, container): @@ -2570,8 +2769,7 @@ def __eq__(self, other): class SubclassWithKwargsTest(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_keywords_in_subclass(self): # count is not subclassable... testcases = [ @@ -2666,7 +2864,7 @@ def test_permutations_sizeof(self): def load_tests(loader, tests, pattern): - tests.addTest(doctest.DocTestSuite()) + tests.addTest(doctest.DocTestSuite(itertools)) return tests diff --git a/Lib/test/test_json/test_decode.py b/Lib/test/test_json/test_decode.py index e48e36c54b..f07f7d5533 100644 --- a/Lib/test/test_json/test_decode.py +++ b/Lib/test/test_json/test_decode.py @@ -2,21 +2,50 @@ from io import StringIO from collections import OrderedDict from test.test_json import PyTest, CTest +from test import support -import unittest +import unittest # XXX: RUSTPYTHON; importing to be able to skip tests class TestDecode: def test_decimal(self): rval = self.loads('1.1', parse_float=decimal.Decimal) - self.assertTrue(isinstance(rval, decimal.Decimal)) + self.assertIsInstance(rval, decimal.Decimal) self.assertEqual(rval, decimal.Decimal('1.1')) def test_float(self): rval = self.loads('1', parse_int=float) - self.assertTrue(isinstance(rval, float)) + self.assertIsInstance(rval, float) self.assertEqual(rval, 1.0) + # TODO: RUSTPYTHON + @unittest.skip("TODO: RUSTPYTHON; called `Result::unwrap()` on an `Err` value: ParseFloatError { kind: Invalid }") + def test_nonascii_digits_rejected(self): + # JSON specifies only ascii digits, see gh-125687 + for num in ["1\uff10", "0.\uff10", "0e\uff10"]: + with self.assertRaises(self.JSONDecodeError): + self.loads(num) + + def test_bytes(self): + self.assertEqual(self.loads(b"1"), 1) + + def test_parse_constant(self): + for constant, expected in [ + ("Infinity", "INFINITY"), + ("-Infinity", "-INFINITY"), + ("NaN", "NAN"), + ]: + self.assertEqual( + self.loads(constant, parse_constant=str.upper), expected + ) + + def test_constant_invalid_case(self): + for constant in [ + "nan", "NAN", "naN", "infinity", "INFINITY", "inFiniTy" + ]: + with self.assertRaises(self.JSONDecodeError): + self.loads(constant) + def test_empty_objects(self): self.assertEqual(self.loads('{}'), {}) self.assertEqual(self.loads('[]'), []) @@ -89,7 +118,8 @@ def test_string_with_utf8_bom(self): self.json.load(StringIO(bom_json)) self.assertIn('BOM', str(cm.exception)) # make sure that the BOM is not detected in the middle of a string - bom_in_str = '"{}"'.format(''.encode('utf-8-sig').decode('utf-8')) + bom = ''.encode('utf-8-sig').decode('utf-8') + bom_in_str = f'"{bom}"' self.assertEqual(self.loads(bom_in_str), '\ufeff') self.assertEqual(self.json.load(StringIO(bom_in_str)), '\ufeff') @@ -97,10 +127,23 @@ def test_negative_index(self): d = self.json.JSONDecoder() self.assertRaises(ValueError, d.raw_decode, 'a'*42, -50000) + def test_limit_int(self): + maxdigits = 5000 + with support.adjust_int_max_str_digits(maxdigits): + self.loads('1' * maxdigits) + with self.assertRaises(ValueError): + self.loads('1' * (maxdigits + 1)) + + class TestPyDecode(TestDecode, PyTest): pass -# TODO: RUSTPYTHON -class TestCDecode(TestDecode, CTest): # pass + +class TestCDecode(TestDecode, CTest): # TODO: RUSTPYTHON @unittest.expectedFailure def test_keys_reuse(self): return super().test_keys_reuse() + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_limit_int(self): + return super().test_limit_int() diff --git a/Lib/test/test_json/test_default.py b/Lib/test/test_json/test_default.py index 9b8325e9c3..3ce16684a0 100644 --- a/Lib/test/test_json/test_default.py +++ b/Lib/test/test_json/test_default.py @@ -1,3 +1,4 @@ +import collections from test.test_json import PyTest, CTest @@ -7,6 +8,16 @@ def test_default(self): self.dumps(type, default=repr), self.dumps(repr(type))) + def test_ordereddict(self): + od = collections.OrderedDict(a=1, b=2, c=3, d=4) + od.move_to_end('b') + self.assertEqual( + self.dumps(od), + '{"a": 1, "c": 3, "d": 4, "b": 2}') + self.assertEqual( + self.dumps(od, sort_keys=True), + '{"a": 1, "b": 2, "c": 3, "d": 4}') + class TestPyDefault(TestDefault, PyTest): pass class TestCDefault(TestDefault, CTest): pass diff --git a/Lib/test/test_json/test_dump.py b/Lib/test/test_json/test_dump.py index 13b4002078..3947075400 100644 --- a/Lib/test/test_json/test_dump.py +++ b/Lib/test/test_json/test_dump.py @@ -22,6 +22,14 @@ def test_dump_skipkeys(self): self.assertIn('valid_key', o) self.assertNotIn(b'invalid_key', o) + def test_dump_skipkeys_indent_empty(self): + v = {b'invalid_key': False} + self.assertEqual(self.json.dumps(v, skipkeys=True, indent=4), '{}') + + def test_skipkeys_indent(self): + v = {b'invalid_key': False, 'valid_key': True} + self.assertEqual(self.json.dumps(v, skipkeys=True, indent=4), '{\n "valid_key": true\n}') + def test_encode_truefalse(self): self.assertEqual(self.dumps( {True: False, False: True}, sort_keys=True), diff --git a/Lib/test/test_json/test_encode_basestring_ascii.py b/Lib/test/test_json/test_encode_basestring_ascii.py index 4bbc6c7148..6a39b72a09 100644 --- a/Lib/test/test_json/test_encode_basestring_ascii.py +++ b/Lib/test/test_json/test_encode_basestring_ascii.py @@ -23,8 +23,7 @@ def test_encode_basestring_ascii(self): for input_string, expect in CASES: result = self.json.encoder.encode_basestring_ascii(input_string) self.assertEqual(result, expect, - '{0!r} != {1!r} for {2}({3!r})'.format( - result, expect, fname, input_string)) + f'{result!r} != {expect!r} for {fname}({input_string!r})') def test_ordered_dict(self): # See issue 6105 diff --git a/Lib/test/test_json/test_fail.py b/Lib/test/test_json/test_fail.py index dcadc1c51c..7a85665c81 100644 --- a/Lib/test/test_json/test_fail.py +++ b/Lib/test/test_json/test_fail.py @@ -1,75 +1,76 @@ -import unittest from test.test_json import PyTest, CTest +import unittest # XXX: RUSTPYTHON; importing to be able to skip tests + # 2007-10-05 JSONDOCS = [ - # http://json.org/JSON_checker/test/fail1.json + # https://json.org/JSON_checker/test/fail1.json '"A JSON payload should be an object or array, not a string."', - # http://json.org/JSON_checker/test/fail2.json + # https://json.org/JSON_checker/test/fail2.json '["Unclosed array"', - # http://json.org/JSON_checker/test/fail3.json + # https://json.org/JSON_checker/test/fail3.json '{unquoted_key: "keys must be quoted"}', - # http://json.org/JSON_checker/test/fail4.json + # https://json.org/JSON_checker/test/fail4.json '["extra comma",]', - # http://json.org/JSON_checker/test/fail5.json + # https://json.org/JSON_checker/test/fail5.json '["double extra comma",,]', - # http://json.org/JSON_checker/test/fail6.json + # https://json.org/JSON_checker/test/fail6.json '[ , "<-- missing value"]', - # http://json.org/JSON_checker/test/fail7.json + # https://json.org/JSON_checker/test/fail7.json '["Comma after the close"],', - # http://json.org/JSON_checker/test/fail8.json + # https://json.org/JSON_checker/test/fail8.json '["Extra close"]]', - # http://json.org/JSON_checker/test/fail9.json + # https://json.org/JSON_checker/test/fail9.json '{"Extra comma": true,}', - # http://json.org/JSON_checker/test/fail10.json + # https://json.org/JSON_checker/test/fail10.json '{"Extra value after close": true} "misplaced quoted value"', - # http://json.org/JSON_checker/test/fail11.json + # https://json.org/JSON_checker/test/fail11.json '{"Illegal expression": 1 + 2}', - # http://json.org/JSON_checker/test/fail12.json + # https://json.org/JSON_checker/test/fail12.json '{"Illegal invocation": alert()}', - # http://json.org/JSON_checker/test/fail13.json + # https://json.org/JSON_checker/test/fail13.json '{"Numbers cannot have leading zeroes": 013}', - # http://json.org/JSON_checker/test/fail14.json + # https://json.org/JSON_checker/test/fail14.json '{"Numbers cannot be hex": 0x14}', - # http://json.org/JSON_checker/test/fail15.json + # https://json.org/JSON_checker/test/fail15.json '["Illegal backslash escape: \\x15"]', - # http://json.org/JSON_checker/test/fail16.json + # https://json.org/JSON_checker/test/fail16.json '[\\naked]', - # http://json.org/JSON_checker/test/fail17.json + # https://json.org/JSON_checker/test/fail17.json '["Illegal backslash escape: \\017"]', - # http://json.org/JSON_checker/test/fail18.json + # https://json.org/JSON_checker/test/fail18.json '[[[[[[[[[[[[[[[[[[[["Too deep"]]]]]]]]]]]]]]]]]]]]', - # http://json.org/JSON_checker/test/fail19.json + # https://json.org/JSON_checker/test/fail19.json '{"Missing colon" null}', - # http://json.org/JSON_checker/test/fail20.json + # https://json.org/JSON_checker/test/fail20.json '{"Double colon":: null}', - # http://json.org/JSON_checker/test/fail21.json + # https://json.org/JSON_checker/test/fail21.json '{"Comma instead of colon", null}', - # http://json.org/JSON_checker/test/fail22.json + # https://json.org/JSON_checker/test/fail22.json '["Colon instead of comma": false]', - # http://json.org/JSON_checker/test/fail23.json + # https://json.org/JSON_checker/test/fail23.json '["Bad value", truth]', - # http://json.org/JSON_checker/test/fail24.json + # https://json.org/JSON_checker/test/fail24.json "['single quote']", - # http://json.org/JSON_checker/test/fail25.json + # https://json.org/JSON_checker/test/fail25.json '["\ttab\tcharacter\tin\tstring\t"]', - # http://json.org/JSON_checker/test/fail26.json + # https://json.org/JSON_checker/test/fail26.json '["tab\\ character\\ in\\ string\\ "]', - # http://json.org/JSON_checker/test/fail27.json + # https://json.org/JSON_checker/test/fail27.json '["line\nbreak"]', - # http://json.org/JSON_checker/test/fail28.json + # https://json.org/JSON_checker/test/fail28.json '["line\\\nbreak"]', - # http://json.org/JSON_checker/test/fail29.json + # https://json.org/JSON_checker/test/fail29.json '[0e]', - # http://json.org/JSON_checker/test/fail30.json + # https://json.org/JSON_checker/test/fail30.json '[0e+]', - # http://json.org/JSON_checker/test/fail31.json + # https://json.org/JSON_checker/test/fail31.json '[0e+-1]', - # http://json.org/JSON_checker/test/fail32.json + # https://json.org/JSON_checker/test/fail32.json '{"Comma instead if closing brace": true,', - # http://json.org/JSON_checker/test/fail33.json + # https://json.org/JSON_checker/test/fail33.json '["mismatch"}', - # http://code.google.com/p/simplejson/issues/detail?id=3 + # https://code.google.com/archive/p/simplejson/issues/3 '["A\u001FZ control characters in string"]', ] @@ -90,7 +91,7 @@ def test_failures(self): except self.JSONDecodeError: pass else: - self.fail("Expected failure for fail{0}.json: {1!r}".format(idx, doc)) + self.fail(f"Expected failure for fail{idx}.json: {doc!r}") def test_non_string_keys_dict(self): data = {'a' : 1, (1, 2) : 2} @@ -144,11 +145,11 @@ def test_unexpected_data(self): ('{"spam":[}', 'Expecting value', 9), ('[42:', "Expecting ',' delimiter", 3), ('[42 "spam"', "Expecting ',' delimiter", 4), - ('[42,]', 'Expecting value', 4), + ('[42,]', "Illegal trailing comma before end of array", 3), ('{"spam":[42}', "Expecting ',' delimiter", 11), ('["]', 'Unterminated string starting at', 1), ('["spam":', "Expecting ',' delimiter", 7), - ('["spam",]', 'Expecting value', 8), + ('["spam",]', "Illegal trailing comma before end of array", 7), ('{:', 'Expecting property name enclosed in double quotes', 1), ('{,', 'Expecting property name enclosed in double quotes', 1), ('{42', 'Expecting property name enclosed in double quotes', 1), @@ -160,7 +161,9 @@ def test_unexpected_data(self): ('[{"spam":]', 'Expecting value', 9), ('{"spam":42 "ham"', "Expecting ',' delimiter", 11), ('[{"spam":42]', "Expecting ',' delimiter", 11), - ('{"spam":42,}', 'Expecting property name enclosed in double quotes', 11), + ('{"spam":42,}', "Illegal trailing comma before end of object", 10), + ('{"spam":42 , }', "Illegal trailing comma before end of object", 11), + ('[123 , ]', "Illegal trailing comma before end of array", 6), ] for data, msg, idx in test_cases: with self.assertRaises(self.JSONDecodeError) as cm: @@ -217,10 +220,9 @@ def test_linecol(self): (line, col, idx)) class TestPyFail(TestFail, PyTest): pass -class TestCFail(TestFail, CTest): pass -# TODO: RUSTPYTHON + class TestCFail(TestFail, CTest): # TODO: RUSTPYTHON @unittest.expectedFailure def test_failures(self): - super().test_failures() + return super().test_failures() diff --git a/Lib/test/test_json/test_float.py b/Lib/test/test_json/test_float.py index d0c7214334..61540a3a02 100644 --- a/Lib/test/test_json/test_float.py +++ b/Lib/test/test_json/test_float.py @@ -26,7 +26,8 @@ def test_allow_nan(self): res = self.loads(out) self.assertEqual(len(res), 1) self.assertNotEqual(res[0], res[0]) - self.assertRaises(ValueError, self.dumps, [val], allow_nan=False) + msg = f'Out of range float values are not JSON compliant: {val}' + self.assertRaisesRegex(ValueError, msg, self.dumps, [val], allow_nan=False) class TestPyFloat(TestFloat, PyTest): pass diff --git a/Lib/test/test_json/test_pass1.py b/Lib/test/test_json/test_pass1.py index 15e64b0aea..26bf3cdbd7 100644 --- a/Lib/test/test_json/test_pass1.py +++ b/Lib/test/test_json/test_pass1.py @@ -1,7 +1,7 @@ from test.test_json import PyTest, CTest -# from http://json.org/JSON_checker/test/pass1.json +# from https://json.org/JSON_checker/test/pass1.json JSON = r''' [ "JSON Test Pattern pass1", diff --git a/Lib/test/test_json/test_pass2.py b/Lib/test/test_json/test_pass2.py index 35075249e3..9340de665a 100644 --- a/Lib/test/test_json/test_pass2.py +++ b/Lib/test/test_json/test_pass2.py @@ -1,7 +1,7 @@ from test.test_json import PyTest, CTest -# from http://json.org/JSON_checker/test/pass2.json +# from https://json.org/JSON_checker/test/pass2.json JSON = r''' [[[[[[[[[[[[[[[[[[["Not too deep"]]]]]]]]]]]]]]]]]]] ''' diff --git a/Lib/test/test_json/test_pass3.py b/Lib/test/test_json/test_pass3.py index cd0cf170d2..0adccc1c2a 100644 --- a/Lib/test/test_json/test_pass3.py +++ b/Lib/test/test_json/test_pass3.py @@ -1,7 +1,7 @@ from test.test_json import PyTest, CTest -# from http://json.org/JSON_checker/test/pass3.json +# from https://json.org/JSON_checker/test/pass3.json JSON = r''' { "JSON Test Pattern pass3": { diff --git a/Lib/test/test_json/test_recursion.py b/Lib/test/test_json/test_recursion.py index 9919d7fbe5..59f6f2c4b1 100644 --- a/Lib/test/test_json/test_recursion.py +++ b/Lib/test/test_json/test_recursion.py @@ -1,6 +1,8 @@ from test import support from test.test_json import PyTest, CTest +import unittest # XXX: RUSTPYTHON; importing to be able to skip tests + class JSONTestObject: pass @@ -65,7 +67,8 @@ def default(self, o): else: self.fail("didn't raise ValueError on default recursion") - + # TODO: RUSTPYTHON + @unittest.skip("TODO: RUSTPYTHON; crashes") def test_highly_nested_objects_decoding(self): # test that loading highly-nested objects doesn't segfault when C # accelerations are used. See #12017 @@ -79,16 +82,17 @@ def test_highly_nested_objects_decoding(self): with support.infinite_recursion(): self.loads('[' * 100000 + '1' + ']' * 100000) + @support.requires_resource('cpu') def test_highly_nested_objects_encoding(self): # See #12051 l, d = [], {} for x in range(100000): l, d = [l], {'k':d} with self.assertRaises(RecursionError): - with support.infinite_recursion(): + with support.infinite_recursion(5000): self.dumps(l) with self.assertRaises(RecursionError): - with support.infinite_recursion(): + with support.infinite_recursion(5000): self.dumps(d) def test_endless_recursion(self): @@ -99,7 +103,7 @@ def default(self, o): return [o] with self.assertRaises(RecursionError): - with support.infinite_recursion(): + with support.infinite_recursion(1000): EndlessJSONEncoder(check_circular=False).encode(5j) diff --git a/Lib/test/test_json/test_scanstring.py b/Lib/test/test_json/test_scanstring.py index af4bb3a639..a5c46bb64b 100644 --- a/Lib/test/test_json/test_scanstring.py +++ b/Lib/test/test_json/test_scanstring.py @@ -1,7 +1,7 @@ -import unittest import sys from test.test_json import PyTest, CTest +import unittest # XXX: RUSTPYTHON; importing to be able to skip tests class TestScanstring: def test_scanstring(self): @@ -117,6 +117,11 @@ def test_bad_escapes(self): '"\\u012z"', '"\\u0x12"', '"\\u0X12"', + '"\\u{0}"'.format("\uff10" * 4), + '"\\u 123"', + '"\\u-123"', + '"\\u+123"', + '"\\u1_23"', '"\\ud834\\"', '"\\ud834\\u"', '"\\ud834\\ud"', @@ -128,6 +133,11 @@ def test_bad_escapes(self): '"\\ud834\\udd2z"', '"\\ud834\\u0x20"', '"\\ud834\\u0X20"', + '"\\ud834\\u{0}"'.format("\uff10" * 4), + '"\\ud834\\u 123"', + '"\\ud834\\u-123"', + '"\\ud834\\u+123"', + '"\\ud834\\u1_23"', ] for s in bad_escapes: with self.assertRaises(self.JSONDecodeError, msg=s): diff --git a/Lib/test/test_json/test_speedups.py b/Lib/test/test_json/test_speedups.py index 66ce1d1b33..ada9672912 100644 --- a/Lib/test/test_json/test_speedups.py +++ b/Lib/test/test_json/test_speedups.py @@ -1,6 +1,7 @@ from test.test_json import CTest -import unittest +import unittest # XXX: RUSTPYTHON; importing to be able to skip tests + class BadBool: def __bool__(self): @@ -39,7 +40,7 @@ def test_make_encoder(self): b"\xCD\x7D\x3D\x4E\x12\x4C\xF9\x79\xD7\x52\xBA\x82\xF2\x27\x4A\x7D\xA0\xCA\x75", None) - # TODO: RUSTPYTHON, TypeError: 'NoneType' object is not callable + # TODO: RUSTPYTHON; TypeError: 'NoneType' object is not callable @unittest.expectedFailure def test_bad_str_encoder(self): # Issue #31505: There shouldn't be an assertion failure in case @@ -73,7 +74,7 @@ def test_bad_markers_argument_to_encoder(self): self.json.encoder.c_make_encoder(1, None, None, None, ': ', ', ', False, False, False) - # TODO: RUSTPYTHON, translate the encoder to Rust + # TODO: RUSTPYTHON; ZeroDivisionError not raised by test @unittest.expectedFailure def test_bad_bool_args(self): def test(name): diff --git a/Lib/test/test_json/test_tool.py b/Lib/test/test_json/test_tool.py index 1d7fca6efb..2b63810d53 100644 --- a/Lib/test/test_json/test_tool.py +++ b/Lib/test/test_json/test_tool.py @@ -10,6 +10,7 @@ from test.support.script_helper import assert_python_ok +@support.requires_subprocess() class TestTool(unittest.TestCase): data = """ diff --git a/Lib/test/test_json/test_unicode.py b/Lib/test/test_json/test_unicode.py index e5882a1ef4..4bdb607e7d 100644 --- a/Lib/test/test_json/test_unicode.py +++ b/Lib/test/test_json/test_unicode.py @@ -1,8 +1,9 @@ -import unittest import codecs from collections import OrderedDict from test.test_json import PyTest, CTest +import unittest # XXX: RUSTPYTHON; importing to be able to skip tests + class TestUnicode: # test_encoding1 and test_encoding2 from 2.x are irrelevant (only str @@ -21,12 +22,17 @@ def test_encoding4(self): def test_encoding5(self): u = '\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}' j = self.dumps(u, ensure_ascii=False) - self.assertEqual(j, '"{0}"'.format(u)) + self.assertEqual(j, f'"{u}"') def test_encoding6(self): u = '\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}' j = self.dumps([u], ensure_ascii=False) - self.assertEqual(j, '["{0}"]'.format(u)) + self.assertEqual(j, f'["{u}"]') + + def test_encoding7(self): + u = '\N{GREEK SMALL LETTER ALPHA}\N{GREEK CAPITAL LETTER OMEGA}' + j = self.dumps(u + "\n", ensure_ascii=False) + self.assertEqual(j, f'"{u}\\n"') def test_big_unicode_encode(self): u = '\U0001d120' @@ -35,13 +41,13 @@ def test_big_unicode_encode(self): def test_big_unicode_decode(self): u = 'z\U0001d120x' - self.assertEqual(self.loads('"' + u + '"'), u) + self.assertEqual(self.loads(f'"{u}"'), u) self.assertEqual(self.loads('"z\\ud834\\udd20x"'), u) def test_unicode_decode(self): for i in range(0, 0xd7ff): u = chr(i) - s = '"\\u{0:04x}"'.format(i) + s = f'"\\u{i:04x}"' self.assertEqual(self.loads(s), u) def test_unicode_preservation(self): diff --git a/Lib/test/test_keywordonlyarg.py b/Lib/test/test_keywordonlyarg.py index 9e9c6651dd..e41e7c051f 100644 --- a/Lib/test/test_keywordonlyarg.py +++ b/Lib/test/test_keywordonlyarg.py @@ -58,8 +58,7 @@ def testSyntaxForManyArguments(self): fundef = "def f(*, %s):\n pass\n" % ', '.join('i%d' % i for i in range(300)) compile(fundef, "<test>", "single") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def testTooManyPositionalErrorMessage(self): def f(a, b=None, *, c=None): pass @@ -158,8 +157,7 @@ def test_issue13343(self): # used to fail with a SystemError. lambda *, k1=unittest: None - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_mangling(self): class X: def f(self, *, __a=42): @@ -174,7 +172,7 @@ def f(v=a, x=b, *, y=c, z=d): pass self.assertEqual(str(err.exception), "name 'b' is not defined") with self.assertRaises(NameError) as err: - f = lambda v=a, x=b, *, y=c, z=d: None + g = lambda v=a, x=b, *, y=c, z=d: None self.assertEqual(str(err.exception), "name 'b' is not defined") diff --git a/Lib/test/test_kqueue.py b/Lib/test/test_kqueue.py index 998fd9d464..e94edcbc10 100644 --- a/Lib/test/test_kqueue.py +++ b/Lib/test/test_kqueue.py @@ -5,6 +5,7 @@ import os import select import socket +from test import support import time import unittest @@ -256,6 +257,23 @@ def test_fd_non_inheritable(self): self.addCleanup(kqueue.close) self.assertEqual(os.get_inheritable(kqueue.fileno()), False) + @support.requires_fork() + def test_fork(self): + # gh-110395: kqueue objects must be closed after fork + kqueue = select.kqueue() + if (pid := os.fork()) == 0: + try: + self.assertTrue(kqueue.closed) + with self.assertRaisesRegex(ValueError, "closed kqueue"): + kqueue.fileno() + except: + os._exit(1) + finally: + os._exit(0) + else: + support.wait_process(pid, exitcode=0) + self.assertFalse(kqueue.closed) # child done, we're still open. + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_list.py b/Lib/test/test_list.py index c82bf5067d..ed061384f1 100644 --- a/Lib/test/test_list.py +++ b/Lib/test/test_list.py @@ -1,6 +1,8 @@ import sys +import textwrap from test import list_tests from test.support import cpython_only +from test.support.script_helper import assert_python_ok import pickle import unittest @@ -46,8 +48,7 @@ def test_keyword_args(self): with self.assertRaisesRegex(TypeError, 'keyword argument'): list(sequence=[]) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_keywords_in_subclass(self): class subclass(list): pass @@ -98,8 +99,12 @@ def imul(a, b): a *= b self.assertRaises((MemoryError, OverflowError), mul, lst, n) self.assertRaises((MemoryError, OverflowError), imul, lst, n) - # TODO: RUSTPYTHON - @unittest.skip("Crashes on windows debug build") + def test_empty_slice(self): + x = [] + x[:] = x + self.assertEqual(x, []) + + @unittest.skip("TODO: RUSTPYTHON; crash") def test_list_resize_overflow(self): # gh-97616: test new_allocated * sizeof(PyObject*) overflow # check in list_resize() @@ -113,13 +118,27 @@ def test_list_resize_overflow(self): with self.assertRaises((MemoryError, OverflowError)): lst *= size + @unittest.skip("TODO: RUSTPYTHON; hangs") + def test_repr_mutate(self): + class Obj: + @staticmethod + def __repr__(): + try: + mylist.pop() + except IndexError: + pass + return 'obj' + + mylist = [Obj() for _ in range(5)] + self.assertEqual(repr(mylist), '[obj, obj, obj]') + def test_repr_large(self): # Check the repr of large list objects def check(n): l = [0] * n s = repr(l) self.assertEqual(s, - '[' + ', '.join(['0'] * n) + ']') + '[' + ', '.join(['0'] * n) + ']') check(10) # check our checking code check(1000000) @@ -208,7 +227,6 @@ class L(list): pass with self.assertRaises(TypeError): (3,) + L([1,2]) - # TODO: RUSTPYTHON @unittest.skip("TODO: RUSTPYTHON; hang") def test_equal_operator_modifying_operand(self): # test fix for seg fault reported in bpo-38588 part 2. @@ -235,7 +253,6 @@ def __eq__(self, other): list4 = [1] self.assertFalse(list3 == list4) - # TODO: RUSTPYTHON @unittest.skip("TODO: RUSTPYTHON; hang") def test_lt_operator_modifying_operand(self): # See gh-120298 @@ -302,6 +319,35 @@ def __eq__(self, other): lst = [X(), X()] X() in lst + def test_tier2_invalidates_iterator(self): + # GH-121012 + for _ in range(100): + a = [1, 2, 3] + it = iter(a) + for _ in it: + pass + a.append(4) + self.assertEqual(list(it), []) + + def test_deopt_from_append_list(self): + # gh-132011: it used to crash, because + # of `CALL_LIST_APPEND` specialization failure. + code = textwrap.dedent(""" + l = [] + def lappend(l, x, y): + l.append((x, y)) + for x in range(3): + lappend(l, None, None) + try: + lappend(list, None, None) + except TypeError: + pass + else: + raise AssertionError + """) + + rc, _, _ = assert_python_ok("-c", code) + self.assertEqual(rc, 0) if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_listcomps.py b/Lib/test/test_listcomps.py index ad1c5053a3..1380c08d28 100644 --- a/Lib/test/test_listcomps.py +++ b/Lib/test/test_listcomps.py @@ -177,7 +177,7 @@ def test_references___class___defined(self): res = [__class__ for x in [1]] """ self._check_in_scopes( - code, outputs={"res": [2]}, scopes=["module", "function"]) + code, outputs={"res": [2]}, scopes=["module", "function"]) self._check_in_scopes(code, raises=NameError, scopes=["class"]) def test_references___class___enclosing(self): @@ -648,11 +648,18 @@ def test_exception_in_post_comp_call(self): """ self._check_in_scopes(code, {"value": [1, None]}) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_frame_locals(self): code = """ - val = [sys._getframe().f_locals for a in [0]][0]["a"] + val = "a" in [sys._getframe().f_locals for a in [0]][0] """ import sys + self._check_in_scopes(code, {"val": False}, ns={"sys": sys}) + + code = """ + val = [sys._getframe().f_locals["a"] for a in [0]][0] + """ self._check_in_scopes(code, {"val": 0}, ns={"sys": sys}) def _recursive_replace(self, maybe_code): @@ -736,7 +743,7 @@ def iter_raises(): for func, expected in [(init_raises, "BrokenIter(init_raises=True)"), (next_raises, "BrokenIter(next_raises=True)"), (iter_raises, "BrokenIter(iter_raises=True)"), - ]: + ]: with self.subTest(func): exc = func() f = traceback.extract_tb(exc.__traceback__)[0] diff --git a/Lib/test/test_locale.py b/Lib/test/test_locale.py index b0d7998559..71d03f3a3f 100644 --- a/Lib/test/test_locale.py +++ b/Lib/test/test_locale.py @@ -1,8 +1,11 @@ from decimal import Decimal from test.support import verbose, is_android, is_emscripten, is_wasi from test.support.warnings_helper import check_warnings +from test.support.import_helper import import_fresh_module +from unittest import mock import unittest import locale +import os import sys import codecs @@ -342,8 +345,7 @@ def setUp(self): enc = codecs.lookup(locale.getencoding() or 'ascii').name if enc not in ('utf-8', 'iso8859-1', 'cp1252'): raise unittest.SkipTest('encoding not suitable') - if enc != 'iso8859-1' and (sys.platform == 'darwin' or is_android or - sys.platform.startswith('freebsd')): + if enc != 'iso8859-1' and is_android: raise unittest.SkipTest('wcscoll/wcsxfrm have known bugs') BaseLocalizedTest.setUp(self) @@ -353,6 +355,8 @@ def setUp(self): is_emscripten or is_wasi, "musl libc issue on Emscripten/WASI, bpo-46390" ) + @unittest.skipIf(sys.platform.startswith("netbsd"), + "gh-124108: NetBSD doesn't support UTF-8 for LC_COLLATE") def test_strcoll_with_diacritic(self): self.assertLess(locale.strcoll('à', 'b'), 0) @@ -362,6 +366,8 @@ def test_strcoll_with_diacritic(self): is_emscripten or is_wasi, "musl libc issue on Emscripten/WASI, bpo-46390" ) + @unittest.skipIf(sys.platform.startswith("netbsd"), + "gh-124108: NetBSD doesn't support UTF-8 for LC_COLLATE") def test_strxfrm_with_diacritic(self): self.assertLess(locale.strxfrm('à'), locale.strxfrm('b')) @@ -382,6 +388,10 @@ def test_c(self): self.check('c', 'C') self.check('posix', 'C') + def test_c_utf8(self): + self.check('c.utf8', 'C.UTF-8') + self.check('C.UTF-8', 'C.UTF-8') + def test_english(self): self.check('en', 'en_US.ISO8859-1') self.check('EN', 'en_US.ISO8859-1') @@ -477,6 +487,54 @@ def test_japanese(self): self.check('jp_jp', 'ja_JP.eucJP') +class TestRealLocales(unittest.TestCase): + def setUp(self): + oldlocale = locale.setlocale(locale.LC_CTYPE) + self.addCleanup(locale.setlocale, locale.LC_CTYPE, oldlocale) + + def test_getsetlocale_issue1813(self): + # Issue #1813: setting and getting the locale under a Turkish locale + try: + locale.setlocale(locale.LC_CTYPE, 'tr_TR') + except locale.Error: + # Unsupported locale on this system + self.skipTest('test needs Turkish locale') + loc = locale.getlocale(locale.LC_CTYPE) + if verbose: + print('testing with %a' % (loc,), end=' ', flush=True) + try: + locale.setlocale(locale.LC_CTYPE, loc) + except locale.Error as exc: + # bpo-37945: setlocale(LC_CTYPE) fails with getlocale(LC_CTYPE) + # and the tr_TR locale on Windows. getlocale() builds a locale + # which is not recognize by setlocale(). + self.skipTest(f"setlocale(LC_CTYPE, {loc!r}) failed: {exc!r}") + self.assertEqual(loc, locale.getlocale(locale.LC_CTYPE)) + + @unittest.skipUnless(os.name == 'nt', 'requires Windows') + def test_setlocale_long_encoding(self): + with self.assertRaises(locale.Error): + locale.setlocale(locale.LC_CTYPE, 'English.%016d' % 1252) + locale.setlocale(locale.LC_CTYPE, 'English.%015d' % 1252) + loc = locale.setlocale(locale.LC_ALL) + self.assertIn('.1252', loc) + loc2 = loc.replace('.1252', '.%016d' % 1252, 1) + with self.assertRaises(locale.Error): + locale.setlocale(locale.LC_ALL, loc2) + loc2 = loc.replace('.1252', '.%015d' % 1252, 1) + locale.setlocale(locale.LC_ALL, loc2) + + # gh-137273: Debug assertion failure on Windows for long encoding. + with self.assertRaises(locale.Error): + locale.setlocale(locale.LC_CTYPE, 'en_US.' + 'x'*16) + locale.setlocale(locale.LC_CTYPE, 'en_US.UTF-8') + loc = locale.setlocale(locale.LC_ALL) + self.assertIn('.UTF-8', loc) + loc2 = loc.replace('.UTF-8', '.' + 'x'*16, 1) + with self.assertRaises(locale.Error): + locale.setlocale(locale.LC_ALL, loc2) + + class TestMiscellaneous(unittest.TestCase): def test_defaults_UTF8(self): # Issue #18378: on (at least) macOS setting LC_CTYPE to "UTF-8" is @@ -523,6 +581,15 @@ def test_getencoding(self): # make sure it is valid codecs.lookup(enc) + def test_getencoding_fallback(self): + # When _locale.getencoding() is missing, locale.getencoding() uses + # the Python filesystem + encoding = 'FALLBACK_ENCODING' + with mock.patch.object(sys, 'getfilesystemencoding', + return_value=encoding): + locale_fallback = import_fresh_module('locale', blocked=['_locale']) + self.assertEqual(locale_fallback.getencoding(), encoding) + def test_getpreferredencoding(self): # Invoke getpreferredencoding to make sure it does not cause exceptions. enc = locale.getpreferredencoding() @@ -546,27 +613,6 @@ def test_setlocale_category(self): # crasher from bug #7419 self.assertRaises(locale.Error, locale.setlocale, 12345) - def test_getsetlocale_issue1813(self): - # Issue #1813: setting and getting the locale under a Turkish locale - oldlocale = locale.setlocale(locale.LC_CTYPE) - self.addCleanup(locale.setlocale, locale.LC_CTYPE, oldlocale) - try: - locale.setlocale(locale.LC_CTYPE, 'tr_TR') - except locale.Error: - # Unsupported locale on this system - self.skipTest('test needs Turkish locale') - loc = locale.getlocale(locale.LC_CTYPE) - if verbose: - print('testing with %a' % (loc,), end=' ', flush=True) - try: - locale.setlocale(locale.LC_CTYPE, loc) - except locale.Error as exc: - # bpo-37945: setlocale(LC_CTYPE) fails with getlocale(LC_CTYPE) - # and the tr_TR locale on Windows. getlocale() builds a locale - # which is not recognize by setlocale(). - self.skipTest(f"setlocale(LC_CTYPE, {loc!r}) failed: {exc!r}") - self.assertEqual(loc, locale.getlocale(locale.LC_CTYPE)) - def test_invalid_locale_format_in_localetuple(self): with self.assertRaises(TypeError): locale.setlocale(locale.LC_ALL, b'fi_FI') diff --git a/Lib/test/test_logging.py b/Lib/test/test_logging.py index 370685f1c6..9004e9ed74 100644 --- a/Lib/test/test_logging.py +++ b/Lib/test/test_logging.py @@ -100,8 +100,7 @@ def setUp(self): self._threading_key = threading_helper.threading_setup() logger_dict = logging.getLogger().manager.loggerDict - logging._acquireLock() - try: + with logging._lock: self.saved_handlers = logging._handlers.copy() self.saved_handler_list = logging._handlerList[:] self.saved_loggers = saved_loggers = logger_dict.copy() @@ -111,8 +110,6 @@ def setUp(self): for name in saved_loggers: logger_states[name] = getattr(saved_loggers[name], 'disabled', None) - finally: - logging._releaseLock() # Set two unused loggers self.logger1 = logging.getLogger("\xab\xd7\xbb") @@ -146,8 +143,7 @@ def tearDown(self): self.root_logger.removeHandler(h) h.close() self.root_logger.setLevel(self.original_logging_level) - logging._acquireLock() - try: + with logging._lock: logging._levelToName.clear() logging._levelToName.update(self.saved_level_to_name) logging._nameToLevel.clear() @@ -164,8 +160,6 @@ def tearDown(self): for name in self.logger_states: if logger_states[name] is not None: self.saved_loggers[name].disabled = logger_states[name] - finally: - logging._releaseLock() self.doCleanups() threading_helper.threading_cleanup(*self._threading_key) @@ -182,7 +176,7 @@ def assert_log_lines(self, expected_values, stream=None, pat=None): match = pat.search(actual) if not match: self.fail("Log line does not match expected pattern:\n" + - actual) + actual) self.assertEqual(tuple(match.groups()), expected) s = stream.read() if s: @@ -588,7 +582,7 @@ def test_specific_filters(self): ('Effusive', '17'), ('Terse', '18'), ('Silent', '20'), - ]) + ]) finally: if specific_filter: self.root_logger.removeFilter(specific_filter) @@ -613,7 +607,7 @@ def test_name(self): def test_builtin_handlers(self): # We can't actually *use* too many handlers in the tests, # but we can try instantiating them with various options - if sys.platform in ('linux', 'darwin'): + if sys.platform in ('linux', 'android', 'darwin'): for existing in (True, False): fn = make_temp_file() if not existing: @@ -673,11 +667,11 @@ def test_pathlike_objects(self): os.unlink(fn) pfn = os_helper.FakePath(fn) cases = ( - (logging.FileHandler, (pfn, 'w')), - (logging.handlers.RotatingFileHandler, (pfn, 'a')), - (logging.handlers.TimedRotatingFileHandler, (pfn, 'h')), - ) - if sys.platform in ('linux', 'darwin'): + (logging.FileHandler, (pfn, 'w')), + (logging.handlers.RotatingFileHandler, (pfn, 'a')), + (logging.handlers.TimedRotatingFileHandler, (pfn, 'h')), + ) + if sys.platform in ('linux', 'android', 'darwin'): cases += ((logging.handlers.WatchedFileHandler, (pfn, 'w')),) for cls, args in cases: h = cls(*args, encoding="utf-8") @@ -751,11 +745,8 @@ def __init__(self): stream=open('/dev/null', 'wt', encoding='utf-8')) def emit(self, record): - self.sub_handler.acquire() - try: + with self.sub_handler.lock: self.sub_handler.emit(record) - finally: - self.sub_handler.release() self.assertEqual(len(logging._handlers), 0) refed_h = _OurHandler() @@ -771,33 +762,26 @@ def emit(self, record): fork_happened__release_locks_and_end_thread = threading.Event() def lock_holder_thread_fn(): - logging._acquireLock() - try: - refed_h.acquire() - try: - # Tell the main thread to do the fork. - locks_held__ready_to_fork.set() - - # If the deadlock bug exists, the fork will happen - # without dealing with the locks we hold, deadlocking - # the child. - - # Wait for a successful fork or an unreasonable amount of - # time before releasing our locks. To avoid a timing based - # test we'd need communication from os.fork() as to when it - # has actually happened. Given this is a regression test - # for a fixed issue, potentially less reliably detecting - # regression via timing is acceptable for simplicity. - # The test will always take at least this long. :( - fork_happened__release_locks_and_end_thread.wait(0.5) - finally: - refed_h.release() - finally: - logging._releaseLock() + with logging._lock, refed_h.lock: + # Tell the main thread to do the fork. + locks_held__ready_to_fork.set() + + # If the deadlock bug exists, the fork will happen + # without dealing with the locks we hold, deadlocking + # the child. + + # Wait for a successful fork or an unreasonable amount of + # time before releasing our locks. To avoid a timing based + # test we'd need communication from os.fork() as to when it + # has actually happened. Given this is a regression test + # for a fixed issue, potentially less reliably detecting + # regression via timing is acceptable for simplicity. + # The test will always take at least this long. :( + fork_happened__release_locks_and_end_thread.wait(0.5) lock_holder_thread = threading.Thread( - target=lock_holder_thread_fn, - name='test_post_fork_child_no_deadlock lock holder') + target=lock_holder_thread_fn, + name='test_post_fork_child_no_deadlock lock holder') lock_holder_thread.start() locks_held__ready_to_fork.wait() @@ -1131,8 +1115,7 @@ class SMTPHandlerTest(BaseTest): # bpo-14314, bpo-19665, bpo-34092: don't wait forever TIMEOUT = support.LONG_TIMEOUT - # TODO: RUSTPYTHON - @unittest.skip(reason="Hangs RustPython") + @unittest.skip("TODO: RUSTPYTHON; hangs") def test_basic(self): sockmap = {} server = TestSMTPServer((socket_helper.HOST, 0), self.process_message, 0.001, @@ -1657,7 +1640,7 @@ def test_config4_ok(self): logging.exception("just testing") sys.stdout.seek(0) self.assertEqual(output.getvalue(), - "ERROR:root:just testing\nGot a [RuntimeError]\n") + "ERROR:root:just testing\nGot a [RuntimeError]\n") # Original logger output is empty self.assert_log_lines([]) @@ -2170,8 +2153,7 @@ def handle_request(self, request): request.end_headers() self.handled.set() - # TODO: RUSTPYTHON - @unittest.skip("RUSTPYTHON") + @unittest.skip('TODO: RUSTPYTHON; flaky test') def test_output(self): # The log message sent to the HTTPHandler is properly received. logger = logging.getLogger("http") @@ -2195,7 +2177,7 @@ def test_output(self): sslctx = None context = None self.server = server = TestHTTPServer(addr, self.handle_request, - 0.01, sslctx=sslctx) + 0.01, sslctx=sslctx) server.start() server.ready.wait() host = 'localhost:%d' % server.server_port @@ -2212,7 +2194,8 @@ def test_output(self): self.handled.clear() msg = "sp\xe4m" logger.error(msg) - self.handled.wait() + handled = self.handled.wait(support.SHORT_TIMEOUT) + self.assertTrue(handled, "HTTP request timed out") self.assertEqual(self.log_data.path, '/frob') self.assertEqual(self.command, method) if method == 'GET': @@ -2253,7 +2236,7 @@ def _assertTruesurvival(self): dead.append(repr_) if dead: self.fail("%d objects should have survived " - "but have been destroyed: %s" % (len(dead), ", ".join(dead))) + "but have been destroyed: %s" % (len(dead), ", ".join(dead))) def test_persistent_loggers(self): # Logger objects are persistent and retain their configuration, even @@ -2351,7 +2334,7 @@ def test_warnings(self): s = a_file.getvalue() a_file.close() self.assertEqual(s, - "dummy.py:42: UserWarning: Explicit\n Dummy line\n") + "dummy.py:42: UserWarning: Explicit\n Dummy line\n") def test_warnings_no_handlers(self): with warnings.catch_warnings(): @@ -2632,7 +2615,7 @@ class ConfigDictTest(BaseTest): }, 'root' : { 'level' : 'NOTSET', - 'handlers' : ['hand1'], + 'handlers' : ['hand1'], }, } @@ -2666,7 +2649,7 @@ class ConfigDictTest(BaseTest): }, 'root' : { 'level' : 'NOTSET', - 'handlers' : ['hand1'], + 'handlers' : ['hand1'], }, } @@ -3277,7 +3260,7 @@ def format(self, record): 'h1' : { 'class': 'logging.FileHandler', }, - # key is before depended on handlers to test that deferred config works + # key is before depended on handlers to test that deferred config works 'ah' : { 'class': 'logging.handlers.QueueHandler', 'handlers': ['h1'] @@ -3355,7 +3338,7 @@ def test_config4_ok(self): logging.exception("just testing") sys.stdout.seek(0) self.assertEqual(output.getvalue(), - "ERROR:root:just testing\nGot a [RuntimeError]\n") + "ERROR:root:just testing\nGot a [RuntimeError]\n") # Original logger output is empty self.assert_log_lines([]) @@ -3370,7 +3353,7 @@ def test_config4a_ok(self): logging.exception("just testing") sys.stdout.seek(0) self.assertEqual(output.getvalue(), - "ERROR:root:just testing\nGot a [RuntimeError]\n") + "ERROR:root:just testing\nGot a [RuntimeError]\n") # Original logger output is empty self.assert_log_lines([]) @@ -3770,7 +3753,28 @@ def test_baseconfig(self): d = { 'atuple': (1, 2, 3), 'alist': ['a', 'b', 'c'], - 'adict': {'d': 'e', 'f': 3 }, + 'adict': { + 'd': 'e', 'f': 3 , + 'alpha numeric 1 with spaces' : 5, + 'alpha numeric 1 %( - © ©ß¯' : 9, + 'alpha numeric ] 1 with spaces' : 15, + 'alpha ]] numeric 1 %( - © ©ß¯]' : 19, + ' alpha [ numeric 1 %( - © ©ß¯] ' : 11, + ' alpha ' : 32, + '' : 10, + 'nest4' : { + 'd': 'e', 'f': 3 , + 'alpha numeric 1 with spaces' : 5, + 'alpha numeric 1 %( - © ©ß¯' : 9, + '' : 10, + 'somelist' : ('g', ('h', 'i'), 'j'), + 'somedict' : { + 'a' : 1, + 'a with 1 and space' : 3, + 'a with ( and space' : 4, + } + } + }, 'nest1': ('g', ('h', 'i'), 'j'), 'nest2': ['k', ['l', 'm'], 'n'], 'nest3': ['o', 'cfg://alist', 'p'], @@ -3782,11 +3786,36 @@ def test_baseconfig(self): self.assertEqual(bc.convert('cfg://nest2[1][1]'), 'm') self.assertEqual(bc.convert('cfg://adict.d'), 'e') self.assertEqual(bc.convert('cfg://adict[f]'), 3) + self.assertEqual(bc.convert('cfg://adict[alpha numeric 1 with spaces]'), 5) + self.assertEqual(bc.convert('cfg://adict[alpha numeric 1 %( - © ©ß¯]'), 9) + self.assertEqual(bc.convert('cfg://adict[]'), 10) + self.assertEqual(bc.convert('cfg://adict.nest4.d'), 'e') + self.assertEqual(bc.convert('cfg://adict.nest4[d]'), 'e') + self.assertEqual(bc.convert('cfg://adict[nest4].d'), 'e') + self.assertEqual(bc.convert('cfg://adict[nest4][f]'), 3) + self.assertEqual(bc.convert('cfg://adict[nest4][alpha numeric 1 with spaces]'), 5) + self.assertEqual(bc.convert('cfg://adict[nest4][alpha numeric 1 %( - © ©ß¯]'), 9) + self.assertEqual(bc.convert('cfg://adict[nest4][]'), 10) + self.assertEqual(bc.convert('cfg://adict[nest4][somelist][0]'), 'g') + self.assertEqual(bc.convert('cfg://adict[nest4][somelist][1][0]'), 'h') + self.assertEqual(bc.convert('cfg://adict[nest4][somelist][1][1]'), 'i') + self.assertEqual(bc.convert('cfg://adict[nest4][somelist][2]'), 'j') + self.assertEqual(bc.convert('cfg://adict[nest4].somedict.a'), 1) + self.assertEqual(bc.convert('cfg://adict[nest4].somedict[a]'), 1) + self.assertEqual(bc.convert('cfg://adict[nest4].somedict[a with 1 and space]'), 3) + self.assertEqual(bc.convert('cfg://adict[nest4].somedict[a with ( and space]'), 4) + self.assertEqual(bc.convert('cfg://adict.nest4.somelist[1][1]'), 'i') + self.assertEqual(bc.convert('cfg://adict.nest4.somelist[2]'), 'j') + self.assertEqual(bc.convert('cfg://adict.nest4.somedict.a'), 1) + self.assertEqual(bc.convert('cfg://adict.nest4.somedict[a]'), 1) v = bc.convert('cfg://nest3') self.assertEqual(v.pop(1), ['a', 'b', 'c']) self.assertRaises(KeyError, bc.convert, 'cfg://nosuch') self.assertRaises(ValueError, bc.convert, 'cfg://!') self.assertRaises(KeyError, bc.convert, 'cfg://adict[2]') + self.assertRaises(KeyError, bc.convert, 'cfg://adict[alpha numeric ] 1 with spaces]') + self.assertRaises(ValueError, bc.convert, 'cfg://adict[ alpha ]] numeric 1 %( - © ©ß¯] ]') + self.assertRaises(ValueError, bc.convert, 'cfg://adict[ alpha [ numeric 1 %( - © ©ß¯] ]') def test_namedtuple(self): # see bpo-39142 @@ -3992,7 +4021,7 @@ def test_config_reject_simple_queue_handler_multiprocessing_context(self): @skip_if_tsan_fork @support.requires_subprocess() @unittest.skipUnless(support.Py_DEBUG, "requires a debug build for testing" - "assertions in multiprocessing") + " assertions in multiprocessing") def test_config_queue_handler_multiprocessing_context(self): # regression test for gh-121723 if support.MS_WINDOWS: @@ -4029,8 +4058,8 @@ def _mpinit_issue121723(qspec, message_to_log): # log a message (this creates a record put in the queue) logging.getLogger().info(message_to_log) - # TODO: RustPython - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; ImportError: cannot import name 'SemLock' + @skip_if_tsan_fork @support.requires_subprocess() def test_multiprocessing_queues(self): # See gh-119819 @@ -4089,8 +4118,7 @@ def test_90195(self): # Logger should be enabled, since explicitly mentioned self.assertFalse(logger.disabled) - # TODO: RustPython - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; ImportError: cannot import name 'SemLock' def test_111615(self): # See gh-111615 import_helper.import_module('_multiprocessing') # see gh-113692 @@ -4208,7 +4236,7 @@ def filter(self, record): t = type(record) if t is not self.cls: msg = 'Unexpected LogRecord type %s, expected %s' % (t, - self.cls) + self.cls) raise TypeError(msg) return True @@ -4228,7 +4256,7 @@ def test_logrecord_class(self): logging.setLogRecordFactory(DerivedLogRecord) self.root_logger.error(self.next_message()) self.assert_log_lines([ - ('root', 'ERROR', '2'), + ('root', 'ERROR', '2'), ]) @@ -4276,8 +4304,6 @@ def test_formatting(self): self.assertEqual(formatted_msg, log_record.msg) self.assertEqual(formatted_msg, log_record.message) - @unittest.skipUnless(hasattr(logging.handlers, 'QueueListener'), - 'logging.handlers.QueueListener required for this test') def test_queue_listener(self): handler = TestHandler(support.Matcher()) listener = logging.handlers.QueueListener(self.queue, handler) @@ -4288,6 +4314,7 @@ def test_queue_listener(self): self.que_logger.critical(self.next_message()) finally: listener.stop() + listener.stop() # gh-114706 - ensure no crash if called again self.assertTrue(handler.matches(levelno=logging.WARNING, message='1')) self.assertTrue(handler.matches(levelno=logging.ERROR, message='2')) self.assertTrue(handler.matches(levelno=logging.CRITICAL, message='3')) @@ -4311,8 +4338,18 @@ def test_queue_listener(self): self.assertTrue(handler.matches(levelno=logging.CRITICAL, message='6')) handler.close() - @unittest.skipUnless(hasattr(logging.handlers, 'QueueListener'), - 'logging.handlers.QueueListener required for this test') + # doesn't hurt to call stop() more than once. + listener.stop() + self.assertIsNone(listener._thread) + + def test_queue_listener_multi_start(self): + handler = TestHandler(support.Matcher()) + listener = logging.handlers.QueueListener(self.queue, handler) + listener.start() + self.assertRaises(RuntimeError, listener.start) + listener.stop() + self.assertIsNone(listener._thread) + def test_queue_listener_with_StreamHandler(self): # Test that traceback and stack-info only appends once (bpo-34334, bpo-46755). listener = logging.handlers.QueueListener(self.queue, self.root_hdlr) @@ -4327,8 +4364,6 @@ def test_queue_listener_with_StreamHandler(self): self.assertEqual(self.stream.getvalue().strip().count('Traceback'), 1) self.assertEqual(self.stream.getvalue().strip().count('Stack'), 1) - @unittest.skipUnless(hasattr(logging.handlers, 'QueueListener'), - 'logging.handlers.QueueListener required for this test') def test_queue_listener_with_multiple_handlers(self): # Test that queue handler format doesn't affect other handler formats (bpo-35726). self.que_hdlr.setFormatter(self.root_formatter) @@ -4344,6 +4379,7 @@ def test_queue_listener_with_multiple_handlers(self): import multiprocessing from unittest.mock import patch + @skip_if_tsan_fork @threading_helper.requires_working_threading() class QueueListenerTest(BaseTest): """ @@ -4427,8 +4463,8 @@ def test_no_messages_in_queue_after_stop(self): expected = [[], [logging.handlers.QueueListener._sentinel]] self.assertIn(items, expected, 'Found unexpected messages in queue: %s' % ( - [m.msg if isinstance(m, logging.LogRecord) - else m for m in items])) + [m.msg if isinstance(m, logging.LogRecord) + else m for m in items])) def test_calls_task_done_after_stop(self): # Issue 36813: Make sure queue.join does not deadlock. @@ -4538,8 +4574,7 @@ def test_dollars(self): f = logging.Formatter('${asctime}--', style='$') self.assertTrue(f.usesTime()) - # TODO: RustPython - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; ValueError: Unexpected error parsing format string def test_format_validate(self): # Check correct formatting # Percentage style @@ -4713,8 +4748,6 @@ def test_defaults_parameter(self): def test_invalid_style(self): self.assertRaises(ValueError, logging.Formatter, None, None, 'x') - # TODO: RustPython - @unittest.expectedFailure def test_time(self): r = self.get_record() dt = datetime.datetime(1993, 4, 21, 8, 3, 0, 0, utc) @@ -4729,8 +4762,6 @@ def test_time(self): f.format(r) self.assertEqual(r.asctime, '1993-04-21 08:03:00,123') - # TODO: RustPython - @unittest.expectedFailure def test_default_msec_format_none(self): class NoMsecFormatter(logging.Formatter): default_msec_format = None @@ -4751,6 +4782,77 @@ def test_issue_89047(self): s = f.format(r) self.assertNotIn('.1000', s) + def test_msecs_has_no_floating_point_precision_loss(self): + # See issue gh-102402 + tests = ( + # time_ns is approx. 2023-03-04 04:25:20 UTC + # (time_ns, expected_msecs_value) + (1_677_902_297_100_000_000, 100.0), # exactly 100ms + (1_677_903_920_999_998_503, 999.0), # check truncating doesn't round + (1_677_903_920_000_998_503, 0.0), # check truncating doesn't round + (1_677_903_920_999_999_900, 0.0), # check rounding up + ) + for ns, want in tests: + with patch('time.time_ns') as patched_ns: + patched_ns.return_value = ns + record = logging.makeLogRecord({'msg': 'test'}) + with self.subTest(ns): + self.assertEqual(record.msecs, want) + self.assertEqual(record.created, ns / 1e9) + self.assertAlmostEqual(record.created - int(record.created), + record.msecs / 1e3, + delta=1e-3) + + def test_relativeCreated_has_higher_precision(self): + # See issue gh-102402. + # Run the code in the subprocess, because the time module should + # be patched before the first import of the logging package. + # Temporary unloading and re-importing the logging package has + # side effects (including registering the atexit callback and + # references leak). + start_ns = 1_677_903_920_000_998_503 # approx. 2023-03-04 04:25:20 UTC + offsets_ns = (200, 500, 12_354, 99_999, 1_677_903_456_999_123_456) + code = textwrap.dedent(f""" + start_ns = {start_ns!r} + offsets_ns = {offsets_ns!r} + start_monotonic_ns = start_ns - 1 + + import time + # Only time.time_ns needs to be patched for the current + # implementation, but patch also other functions to make + # the test less implementation depending. + old_time_ns = time.time_ns + old_time = time.time + old_monotonic_ns = time.monotonic_ns + old_monotonic = time.monotonic + time_ns_result = start_ns + time.time_ns = lambda: time_ns_result + time.time = lambda: time.time_ns()/1e9 + time.monotonic_ns = lambda: time_ns_result - start_monotonic_ns + time.monotonic = lambda: time.monotonic_ns()/1e9 + try: + import logging + + for offset_ns in offsets_ns: + # mock for log record creation + time_ns_result = start_ns + offset_ns + record = logging.makeLogRecord({{'msg': 'test'}}) + print(record.created, record.relativeCreated) + finally: + time.time_ns = old_time_ns + time.time = old_time + time.monotonic_ns = old_monotonic_ns + time.monotonic = old_monotonic + """) + rc, out, err = assert_python_ok("-c", code) + out = out.decode() + for offset_ns, line in zip(offsets_ns, out.splitlines(), strict=True): + with self.subTest(offset_ns=offset_ns): + created, relativeCreated = map(float, line.split()) + self.assertAlmostEqual(created, (start_ns + offset_ns) / 1e9, places=6) + # After PR gh-102412, precision (places) increases from 3 to 7 + self.assertAlmostEqual(relativeCreated, offset_ns / 1e6, places=7) + class TestBufferingFormatter(logging.BufferingFormatter): def formatHeader(self, records): @@ -4795,9 +4897,9 @@ def test_formatting(self): self.assertTrue(r.exc_text.endswith('\nRuntimeError: ' 'deliberate mistake')) self.assertTrue(r.stack_info.startswith('Stack (most recent ' - 'call last):\n')) + 'call last):\n')) self.assertTrue(r.stack_info.endswith('logging.exception(\'failed\', ' - 'stack_info=True)')) + 'stack_info=True)')) class LastResortTest(BaseTest): @@ -5061,8 +5163,7 @@ def __init__(self, name='MyLogger', level=logging.NOTSET): h.close() logging.setLoggerClass(logging.Logger) - # TODO: RustPython - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_logging_at_shutdown(self): # bpo-20037: Doing text I/O late at interpreter shutdown must not crash code = textwrap.dedent(""" @@ -5082,8 +5183,7 @@ def __del__(self): self.assertIn("exception in __del__", err) self.assertIn("ValueError: some error", err) - # TODO: RustPython - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_logging_at_shutdown_open(self): # bpo-26789: FileHandler keeps a reference to the builtin open() # function to be able to open or reopen the file during Python @@ -5175,7 +5275,7 @@ def _extract_logrecord_process_name(key, logMultiprocessing, conn=None): results = {'processName' : name, 'r1.processName': r1.processName, 'r2.processName': r2.processName, - } + } finally: logging.logMultiprocessing = prev_logMultiprocessing if conn: @@ -5183,6 +5283,7 @@ def _extract_logrecord_process_name(key, logMultiprocessing, conn=None): else: return results + @skip_if_tsan_fork def test_multiprocessing(self): support.skip_if_broken_multiprocessing_synchronize() multiprocessing_imported = 'multiprocessing' in sys.modules @@ -5333,7 +5434,7 @@ def test_strformatstyle(self): logging.error("Log an error") sys.stdout.seek(0) self.assertEqual(output.getvalue().strip(), - "ERROR:root:Log an error") + "ERROR:root:Log an error") def test_stringtemplatestyle(self): with support.captured_stdout() as output: @@ -5341,7 +5442,7 @@ def test_stringtemplatestyle(self): logging.error("Log an error") sys.stdout.seek(0) self.assertEqual(output.getvalue().strip(), - "ERROR:root:Log an error") + "ERROR:root:Log an error") def test_filename(self): @@ -5418,11 +5519,11 @@ def test_incompatible(self): handlers = [logging.StreamHandler()] stream = sys.stderr assertRaises(ValueError, logging.basicConfig, filename='test.log', - stream=stream) + stream=stream) assertRaises(ValueError, logging.basicConfig, filename='test.log', - handlers=handlers) + handlers=handlers) assertRaises(ValueError, logging.basicConfig, stream=stream, - handlers=handlers) + handlers=handlers) # Issue 23207: test for invalid kwargs assertRaises(ValueError, logging.basicConfig, loglevel=logging.INFO) # Should pop both filename and filemode even if filename is None @@ -5803,6 +5904,30 @@ def test_extra_not_merged_by_default(self): record = self.recording.records[0] self.assertFalse(hasattr(record, 'foo')) + def test_extra_merged(self): + self.adapter = logging.LoggerAdapter(logger=self.logger, + extra={'foo': '1'}, + merge_extra=True) + + self.adapter.critical('foo and bar should be here', extra={'bar': '2'}) + self.assertEqual(len(self.recording.records), 1) + record = self.recording.records[0] + self.assertTrue(hasattr(record, 'foo')) + self.assertTrue(hasattr(record, 'bar')) + self.assertEqual(record.foo, '1') + self.assertEqual(record.bar, '2') + + def test_extra_merged_log_call_has_precedence(self): + self.adapter = logging.LoggerAdapter(logger=self.logger, + extra={'foo': '1'}, + merge_extra=True) + + self.adapter.critical('foo shall be min', extra={'foo': '2'}) + self.assertEqual(len(self.recording.records), 1) + record = self.recording.records[0] + self.assertTrue(hasattr(record, 'foo')) + self.assertEqual(record.foo, '2') + class PrefixAdapter(logging.LoggerAdapter): prefix = 'Adapter' @@ -6112,14 +6237,14 @@ def test_should_not_rollover(self): # If maxBytes is zero rollover never occurs rh = logging.handlers.RotatingFileHandler( - self.fn, encoding="utf-8", maxBytes=0) + self.fn, encoding="utf-8", maxBytes=0) self.assertFalse(rh.shouldRollover(None)) rh.close() with open(self.fn, 'wb') as f: f.write(b'\n') rh = logging.handlers.RotatingFileHandler( - self.fn, encoding="utf-8", maxBytes=0) + self.fn, encoding="utf-8", maxBytes=0) self.assertFalse(rh.shouldRollover(None)) rh.close() @@ -6129,7 +6254,7 @@ def test_should_not_rollover_non_file(self): # We set maxBytes to 1 so that rollover would normally happen, except # for the check for regular files rh = logging.handlers.RotatingFileHandler( - os.devnull, encoding="utf-8", maxBytes=1) + os.devnull, encoding="utf-8", maxBytes=1) self.assertFalse(rh.shouldRollover(self.next_rec())) rh.close() @@ -6259,13 +6384,11 @@ def rotator(source, dest): rh.close() class TimedRotatingFileHandlerTest(BaseFileTest): - # TODO: RUSTPYTHON - @unittest.skip("OS dependent bug") @unittest.skipIf(support.is_wasi, "WASI does not have /dev/null.") def test_should_not_rollover(self): # See bpo-45401. Should only ever rollover regular files fh = logging.handlers.TimedRotatingFileHandler( - os.devnull, 'S', encoding="utf-8", backupCount=1) + os.devnull, 'S', encoding="utf-8", backupCount=1) time.sleep(1.1) # a little over a second ... r = logging.makeLogRecord({'msg': 'testing - device file'}) self.assertFalse(fh.shouldRollover(r)) @@ -6274,7 +6397,7 @@ def test_should_not_rollover(self): # other test methods added below def test_rollover(self): fh = logging.handlers.TimedRotatingFileHandler( - self.fn, 'S', encoding="utf-8", backupCount=1) + self.fn, 'S', encoding="utf-8", backupCount=1) fmt = logging.Formatter('%(asctime)s %(message)s') fh.setFormatter(fmt) r1 = logging.makeLogRecord({'msg': 'testing - initial'}) @@ -6314,8 +6437,6 @@ def test_rollover(self): print(tf.read()) self.assertTrue(found, msg=msg) - # TODO: RustPython - @unittest.skip("OS dependent bug") def test_rollover_at_midnight(self, weekly=False): os_helper.unlink(self.fn) now = datetime.datetime.now() @@ -6359,8 +6480,6 @@ def test_rollover_at_midnight(self, weekly=False): for i, line in enumerate(f): self.assertIn(f'testing1 {i}', line) - # TODO: RustPython - @unittest.skip("OS dependent bug") def test_rollover_at_weekday(self): self.test_rollover_at_midnight(weekly=True) @@ -6944,7 +7063,7 @@ def secs(**kw): ('MIDNIGHT', 60 * 60 * 24), # current time (epoch start) is a Thursday, W0 means Monday ('W0', secs(days=4, hours=24)), - ): + ): for interval in 1, 3: def test_compute_rollover(self, when=when, interval=interval, exp=exp): rh = logging.handlers.TimedRotatingFileHandler( @@ -6970,8 +7089,8 @@ def test_compute_rollover(self, when=when, interval=interval, exp=exp): currentSecond = t[5] # r is the number of seconds left between now and midnight r = logging.handlers._MIDNIGHT - ((currentHour * 60 + - currentMinute) * 60 + - currentSecond) + currentMinute) * 60 + + currentSecond) result = currentTime + r print('t: %s (%s)' % (t, rh.utc), file=sys.stderr) print('currentHour: %s' % currentHour, file=sys.stderr) diff --git a/Lib/test/test_long.py b/Lib/test/test_long.py index 2a38b133f1..dbcc85bd69 100644 --- a/Lib/test/test_long.py +++ b/Lib/test/test_long.py @@ -412,8 +412,7 @@ def check_float_conversion(self, n): "Got {}, expected {}.".format(n, actual, expected)) self.assertEqual(actual, expected, msg) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON @support.requires_IEEE_754 def test_float_conversion(self): @@ -824,8 +823,7 @@ def check_truediv(self, a, b, skip_small=True): self.assertEqual(expected, got, "Incorrectly rounded division {}/{}: " "expected {}, got {}".format(a, b, expected, got)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON @support.requires_IEEE_754 def test_correctly_rounded_true_division(self): # more stringent tests than those above, checking that the @@ -1326,17 +1324,22 @@ def equivalent_python(n, length, byteorder, signed=False): check(tests4, 'little', signed=False) self.assertRaises(OverflowError, (256).to_bytes, 1, 'big', signed=False) - self.assertRaises(OverflowError, (256).to_bytes, 1, 'big', signed=True) self.assertRaises(OverflowError, (256).to_bytes, 1, 'little', signed=False) - self.assertRaises(OverflowError, (256).to_bytes, 1, 'little', signed=True) + self.assertRaises(OverflowError, (128).to_bytes, 1, 'big', signed=True) + self.assertRaises(OverflowError, (128).to_bytes, 1, 'little', signed=True) + self.assertRaises(OverflowError, (-129).to_bytes, 1, 'big', signed=True) + self.assertRaises(OverflowError, (-129).to_bytes, 1, 'little', signed=True) self.assertRaises(OverflowError, (-1).to_bytes, 2, 'big', signed=False) self.assertRaises(OverflowError, (-1).to_bytes, 2, 'little', signed=False) self.assertEqual((0).to_bytes(0, 'big'), b'') + self.assertEqual((0).to_bytes(0, 'big', signed=True), b'') self.assertEqual((1).to_bytes(5, 'big'), b'\x00\x00\x00\x00\x01') self.assertEqual((0).to_bytes(5, 'big'), b'\x00\x00\x00\x00\x00') self.assertEqual((-1).to_bytes(5, 'big', signed=True), b'\xff\xff\xff\xff\xff') self.assertRaises(OverflowError, (1).to_bytes, 0, 'big') + self.assertRaises(OverflowError, (-1).to_bytes, 0, 'big', signed=True) + self.assertRaises(OverflowError, (-1).to_bytes, 0, 'little', signed=True) # gh-98783 class SubStr(str): @@ -1344,8 +1347,7 @@ class SubStr(str): self.assertEqual((0).to_bytes(1, SubStr('big')), b'\x00') self.assertEqual((0).to_bytes(0, SubStr('little')), b'') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_from_bytes(self): def check(tests, byteorder, signed=False): def equivalent_python(byte_array, byteorder, signed=False): @@ -1559,6 +1561,11 @@ def test_from_bytes_small(self): b = i.to_bytes(2, signed=True) self.assertIs(int.from_bytes(b, signed=True), i) + def test_is_integer(self): + self.assertTrue((-1).is_integer()) + self.assertTrue((0).is_integer()) + self.assertTrue((1).is_integer()) + def test_access_to_nonexistent_digit_0(self): # http://bugs.python.org/issue14630: A bug in _PyLong_Copy meant that # ob_digit[0] was being incorrectly accessed for instances of a @@ -1602,5 +1609,63 @@ def test_square(self): self.assertEqual(n**2, (1 << (2 * bitlen)) - (1 << (bitlen + 1)) + 1) + @unittest.expectedFailure # TODO: RUSTPYTHON + def test___sizeof__(self): + self.assertEqual(int.__itemsize__, sys.int_info.sizeof_digit) + + # Pairs (test_value, number of allocated digits) + test_values = [ + # We always allocate space for at least one digit, even for + # a value of zero; sys.getsizeof should reflect that. + (0, 1), + (1, 1), + (-1, 1), + (BASE-1, 1), + (1-BASE, 1), + (BASE, 2), + (-BASE, 2), + (BASE*BASE - 1, 2), + (BASE*BASE, 3), + ] + + for value, ndigits in test_values: + with self.subTest(value): + self.assertEqual( + value.__sizeof__(), + int.__basicsize__ + int.__itemsize__ * ndigits + ) + + # Same test for a subclass of int. + class MyInt(int): + pass + + self.assertEqual(MyInt.__itemsize__, sys.int_info.sizeof_digit) + + for value, ndigits in test_values: + with self.subTest(value): + self.assertEqual( + MyInt(value).__sizeof__(), + MyInt.__basicsize__ + MyInt.__itemsize__ * ndigits + ) + + # GH-117195 -- This shouldn't crash + object.__sizeof__(1) + + def test_hash(self): + # gh-136599 + self.assertEqual(hash(-1), -2) + self.assertEqual(hash(0), 0) + self.assertEqual(hash(10), 10) + + self.assertEqual(hash(sys.hash_info.modulus - 2), sys.hash_info.modulus - 2) + self.assertEqual(hash(sys.hash_info.modulus - 1), sys.hash_info.modulus - 1) + self.assertEqual(hash(sys.hash_info.modulus), 0) + self.assertEqual(hash(sys.hash_info.modulus + 1), 1) + + self.assertEqual(hash(-sys.hash_info.modulus - 2), -2) + self.assertEqual(hash(-sys.hash_info.modulus - 1), -2) + self.assertEqual(hash(-sys.hash_info.modulus), 0) + self.assertEqual(hash(-sys.hash_info.modulus + 1), -sys.hash_info.modulus + 1) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_math.py b/Lib/test/test_math.py index 16d12f9688..1a4d257586 100644 --- a/Lib/test/test_math.py +++ b/Lib/test/test_math.py @@ -1202,6 +1202,12 @@ def testLdexp(self): self.assertEqual(math.ldexp(NINF, n), NINF) self.assertTrue(math.isnan(math.ldexp(NAN, n))) + @requires_IEEE_754 + def testLdexp_denormal(self): + # Denormal output incorrectly rounded (truncated) + # on some Windows. + self.assertEqual(math.ldexp(6993274598585239, -1126), 1e-323) + def testLog(self): self.assertRaises(TypeError, math.log) self.assertRaises(TypeError, math.log, 1, 2, 3) @@ -2014,8 +2020,7 @@ def test_exceptions(self): else: self.fail("sqrt(-1) didn't raise ValueError") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON @requires_IEEE_754 def test_testfile(self): # Some tests need to be skipped on ancient OS X versions. @@ -2640,9 +2645,10 @@ def test_fma_nan_results(self): # If any input is a NaN, the result should be a NaN, too. for a, b in itertools.product(values, repeat=2): - self.assertIsNaN(math.fma(math.nan, a, b)) - self.assertIsNaN(math.fma(a, math.nan, b)) - self.assertIsNaN(math.fma(a, b, math.nan)) + with self.subTest(a=a, b=b): + self.assertIsNaN(math.fma(math.nan, a, b)) + self.assertIsNaN(math.fma(a, math.nan, b)) + self.assertIsNaN(math.fma(a, b, math.nan)) def test_fma_infinities(self): # Cases involving infinite inputs or results. @@ -2654,86 +2660,93 @@ def test_fma_infinities(self): for c in non_nans: for infinity in [math.inf, -math.inf]: for zero in [0.0, -0.0]: - with self.assertRaises(ValueError): - math.fma(infinity, zero, c) - with self.assertRaises(ValueError): - math.fma(zero, infinity, c) + with self.subTest(c=c, infinity=infinity, zero=zero): + with self.assertRaises(ValueError): + math.fma(infinity, zero, c) + with self.assertRaises(ValueError): + math.fma(zero, infinity, c) # ValueError when a*b and c both infinite of opposite signs. for b in positives: - with self.assertRaises(ValueError): - math.fma(math.inf, b, -math.inf) - with self.assertRaises(ValueError): - math.fma(math.inf, -b, math.inf) - with self.assertRaises(ValueError): - math.fma(-math.inf, -b, -math.inf) - with self.assertRaises(ValueError): - math.fma(-math.inf, b, math.inf) - with self.assertRaises(ValueError): - math.fma(b, math.inf, -math.inf) - with self.assertRaises(ValueError): - math.fma(-b, math.inf, math.inf) - with self.assertRaises(ValueError): - math.fma(-b, -math.inf, -math.inf) - with self.assertRaises(ValueError): - math.fma(b, -math.inf, math.inf) + with self.subTest(b=b): + with self.assertRaises(ValueError): + math.fma(math.inf, b, -math.inf) + with self.assertRaises(ValueError): + math.fma(math.inf, -b, math.inf) + with self.assertRaises(ValueError): + math.fma(-math.inf, -b, -math.inf) + with self.assertRaises(ValueError): + math.fma(-math.inf, b, math.inf) + with self.assertRaises(ValueError): + math.fma(b, math.inf, -math.inf) + with self.assertRaises(ValueError): + math.fma(-b, math.inf, math.inf) + with self.assertRaises(ValueError): + math.fma(-b, -math.inf, -math.inf) + with self.assertRaises(ValueError): + math.fma(b, -math.inf, math.inf) # Infinite result when a*b and c both infinite of the same sign. for b in positives: - self.assertEqual(math.fma(math.inf, b, math.inf), math.inf) - self.assertEqual(math.fma(math.inf, -b, -math.inf), -math.inf) - self.assertEqual(math.fma(-math.inf, -b, math.inf), math.inf) - self.assertEqual(math.fma(-math.inf, b, -math.inf), -math.inf) - self.assertEqual(math.fma(b, math.inf, math.inf), math.inf) - self.assertEqual(math.fma(-b, math.inf, -math.inf), -math.inf) - self.assertEqual(math.fma(-b, -math.inf, math.inf), math.inf) - self.assertEqual(math.fma(b, -math.inf, -math.inf), -math.inf) + with self.subTest(b=b): + self.assertEqual(math.fma(math.inf, b, math.inf), math.inf) + self.assertEqual(math.fma(math.inf, -b, -math.inf), -math.inf) + self.assertEqual(math.fma(-math.inf, -b, math.inf), math.inf) + self.assertEqual(math.fma(-math.inf, b, -math.inf), -math.inf) + self.assertEqual(math.fma(b, math.inf, math.inf), math.inf) + self.assertEqual(math.fma(-b, math.inf, -math.inf), -math.inf) + self.assertEqual(math.fma(-b, -math.inf, math.inf), math.inf) + self.assertEqual(math.fma(b, -math.inf, -math.inf), -math.inf) # Infinite result when a*b finite, c infinite. for a, b in itertools.product(finites, finites): - self.assertEqual(math.fma(a, b, math.inf), math.inf) - self.assertEqual(math.fma(a, b, -math.inf), -math.inf) + with self.subTest(b=b): + self.assertEqual(math.fma(a, b, math.inf), math.inf) + self.assertEqual(math.fma(a, b, -math.inf), -math.inf) # Infinite result when a*b infinite, c finite. for b, c in itertools.product(positives, finites): - self.assertEqual(math.fma(math.inf, b, c), math.inf) - self.assertEqual(math.fma(-math.inf, b, c), -math.inf) - self.assertEqual(math.fma(-math.inf, -b, c), math.inf) - self.assertEqual(math.fma(math.inf, -b, c), -math.inf) + with self.subTest(b=b, c=c): + self.assertEqual(math.fma(math.inf, b, c), math.inf) + self.assertEqual(math.fma(-math.inf, b, c), -math.inf) + self.assertEqual(math.fma(-math.inf, -b, c), math.inf) + self.assertEqual(math.fma(math.inf, -b, c), -math.inf) - self.assertEqual(math.fma(b, math.inf, c), math.inf) - self.assertEqual(math.fma(b, -math.inf, c), -math.inf) - self.assertEqual(math.fma(-b, -math.inf, c), math.inf) - self.assertEqual(math.fma(-b, math.inf, c), -math.inf) + self.assertEqual(math.fma(b, math.inf, c), math.inf) + self.assertEqual(math.fma(b, -math.inf, c), -math.inf) + self.assertEqual(math.fma(-b, -math.inf, c), math.inf) + self.assertEqual(math.fma(-b, math.inf, c), -math.inf) # gh-73468: On some platforms, libc fma() doesn't implement IEE 754-2008 # properly: it doesn't use the right sign when the result is zero. @unittest.skipIf( - sys.platform.startswith(("freebsd", "wasi", "netbsd")) - or (sys.platform == "android" and platform.machine() == "x86_64"), + sys.platform.startswith(("freebsd", "wasi", "netbsd", "emscripten")) + or (sys.platform == "android" and platform.machine() == "x86_64") + or support.linked_to_musl(), # gh-131032 f"this platform doesn't implement IEE 754-2008 properly") def test_fma_zero_result(self): nonnegative_finites = [0.0, 1e-300, 2.3, 1e300] # Zero results from exact zero inputs. for b in nonnegative_finites: - self.assertIsPositiveZero(math.fma(0.0, b, 0.0)) - self.assertIsPositiveZero(math.fma(0.0, b, -0.0)) - self.assertIsNegativeZero(math.fma(0.0, -b, -0.0)) - self.assertIsPositiveZero(math.fma(0.0, -b, 0.0)) - self.assertIsPositiveZero(math.fma(-0.0, -b, 0.0)) - self.assertIsPositiveZero(math.fma(-0.0, -b, -0.0)) - self.assertIsNegativeZero(math.fma(-0.0, b, -0.0)) - self.assertIsPositiveZero(math.fma(-0.0, b, 0.0)) - - self.assertIsPositiveZero(math.fma(b, 0.0, 0.0)) - self.assertIsPositiveZero(math.fma(b, 0.0, -0.0)) - self.assertIsNegativeZero(math.fma(-b, 0.0, -0.0)) - self.assertIsPositiveZero(math.fma(-b, 0.0, 0.0)) - self.assertIsPositiveZero(math.fma(-b, -0.0, 0.0)) - self.assertIsPositiveZero(math.fma(-b, -0.0, -0.0)) - self.assertIsNegativeZero(math.fma(b, -0.0, -0.0)) - self.assertIsPositiveZero(math.fma(b, -0.0, 0.0)) + with self.subTest(b=b): + self.assertIsPositiveZero(math.fma(0.0, b, 0.0)) + self.assertIsPositiveZero(math.fma(0.0, b, -0.0)) + self.assertIsNegativeZero(math.fma(0.0, -b, -0.0)) + self.assertIsPositiveZero(math.fma(0.0, -b, 0.0)) + self.assertIsPositiveZero(math.fma(-0.0, -b, 0.0)) + self.assertIsPositiveZero(math.fma(-0.0, -b, -0.0)) + self.assertIsNegativeZero(math.fma(-0.0, b, -0.0)) + self.assertIsPositiveZero(math.fma(-0.0, b, 0.0)) + + self.assertIsPositiveZero(math.fma(b, 0.0, 0.0)) + self.assertIsPositiveZero(math.fma(b, 0.0, -0.0)) + self.assertIsNegativeZero(math.fma(-b, 0.0, -0.0)) + self.assertIsPositiveZero(math.fma(-b, 0.0, 0.0)) + self.assertIsPositiveZero(math.fma(-b, -0.0, 0.0)) + self.assertIsPositiveZero(math.fma(-b, -0.0, -0.0)) + self.assertIsNegativeZero(math.fma(b, -0.0, -0.0)) + self.assertIsPositiveZero(math.fma(b, -0.0, 0.0)) # Exact zero result from nonzero inputs. self.assertIsPositiveZero(math.fma(2.0, 2.0, -4.0)) @@ -2839,12 +2852,14 @@ def test_random(self): '0x1.f5467b1911fd6p-2', '0x1.b5cee3225caa5p-1'), ] for a_hex, b_hex, c_hex, expected_hex in test_values: - a = float.fromhex(a_hex) - b = float.fromhex(b_hex) - c = float.fromhex(c_hex) - expected = float.fromhex(expected_hex) - self.assertEqual(math.fma(a, b, c), expected) - self.assertEqual(math.fma(b, a, c), expected) + with self.subTest(a_hex=a_hex, b_hex=b_hex, c_hex=c_hex, + expected_hex=expected_hex): + a = float.fromhex(a_hex) + b = float.fromhex(b_hex) + c = float.fromhex(c_hex) + expected = float.fromhex(expected_hex) + self.assertEqual(math.fma(a, b, c), expected) + self.assertEqual(math.fma(b, a, c), expected) # Custom assertions. def assertIsNaN(self, value): diff --git a/Lib/test/test_math_property.py b/Lib/test/test_math_property.py new file mode 100644 index 0000000000..7d51aa17b4 --- /dev/null +++ b/Lib/test/test_math_property.py @@ -0,0 +1,41 @@ +import functools +import unittest +from math import isnan, nextafter +from test.support import requires_IEEE_754 +from test.support.hypothesis_helper import hypothesis + +floats = hypothesis.strategies.floats +integers = hypothesis.strategies.integers + + +def assert_equal_float(x, y): + assert isnan(x) and isnan(y) or x == y + + +def via_reduce(x, y, steps): + return functools.reduce(nextafter, [y] * steps, x) + + +class NextafterTests(unittest.TestCase): + @requires_IEEE_754 + @hypothesis.given( + x=floats(), + y=floats(), + steps=integers(min_value=0, max_value=2**16)) + def test_count(self, x, y, steps): + assert_equal_float(via_reduce(x, y, steps), + nextafter(x, y, steps=steps)) + + @requires_IEEE_754 + @hypothesis.given( + x=floats(), + y=floats(), + a=integers(min_value=0), + b=integers(min_value=0)) + def test_addition_commutes(self, x, y, a, b): + first = nextafter(x, y, steps=a) + second = nextafter(first, y, steps=b) + combined = nextafter(x, y, steps=a+b) + hypothesis.note(f"{first} -> {second} == {combined}") + + assert_equal_float(second, combined) diff --git a/Lib/test/test_memoryio.py b/Lib/test/test_memoryio.py index 5b695f167a..07d9d38d6e 100644 --- a/Lib/test/test_memoryio.py +++ b/Lib/test/test_memoryio.py @@ -724,8 +724,6 @@ class CBytesIOTest(PyBytesIOTest): ioclass = io.BytesIO UnsupportedOperation = io.UnsupportedOperation - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_bytes_array(self): super().test_bytes_array() @@ -739,8 +737,6 @@ def test_flags(self): def test_getbuffer(self): super().test_getbuffer() - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_init(self): super().test_init() @@ -770,8 +766,6 @@ def test_relative_seek(self): def test_seek(self): super().test_seek() - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_subclassing(self): super().test_subclassing() @@ -884,8 +878,6 @@ def test_detach(self): def test_flags(self): super().test_flags() - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_init(self): super().test_init() @@ -940,8 +932,6 @@ def test_relative_seek(self): def test_seek(self): super().test_seek() - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_textio_properties(self): super().test_textio_properties() @@ -1046,8 +1036,6 @@ def test_newlines_property(self): def test_relative_seek(self): super().test_relative_seek() - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_textio_properties(self): super().test_textio_properties() diff --git a/Lib/test/test_mmap.py b/Lib/test/test_mmap.py index ed96be53cc..f80df63bb8 100644 --- a/Lib/test/test_mmap.py +++ b/Lib/test/test_mmap.py @@ -618,6 +618,7 @@ def test_non_ascii_byte(self): self.assertEqual(m.read_byte(), b) m.close() + @unittest.expectedFailure # TODO: RUSTPYTHON @unittest.skipUnless(os.name == 'nt', 'requires Windows') def test_tagname(self): data1 = b"0123456789" @@ -867,6 +868,7 @@ def test_resize_fails_if_mapping_held_elsewhere(self): finally: f.close() + @unittest.expectedFailure # TODO: RUSTPYTHON @unittest.skipUnless(os.name == 'nt', 'requires Windows') def test_resize_succeeds_with_error_for_second_named_mapping(self): """If a more than one mapping exists of the same name, none of them can diff --git a/Lib/test/test_multiprocessing_fork/__init__.py b/Lib/test/test_multiprocessing_fork/__init__.py index aa1fff50b2..b35e82879d 100644 --- a/Lib/test/test_multiprocessing_fork/__init__.py +++ b/Lib/test/test_multiprocessing_fork/__init__.py @@ -12,5 +12,8 @@ if sys.platform == 'darwin': raise unittest.SkipTest("test may crash on macOS (bpo-33725)") +if support.check_sanitizer(thread=True): + raise unittest.SkipTest("TSAN doesn't support threads after fork") + def load_tests(*args): return support.load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_netrc.py b/Lib/test/test_netrc.py index 573d636de9..9d720f6271 100644 --- a/Lib/test/test_netrc.py +++ b/Lib/test/test_netrc.py @@ -1,10 +1,6 @@ import netrc, os, unittest, sys, textwrap -from test.support import os_helper, run_unittest - -try: - import pwd -except ImportError: - pwd = None +from test import support +from test.support import os_helper temp_filename = os_helper.TESTFN @@ -269,9 +265,14 @@ def test_comment_at_end_of_machine_line_pass_has_hash(self): machine bar.domain.com login foo password pass """, '#pass') + @unittest.skipUnless(support.is_wasi, 'WASI only test') + def test_security_on_WASI(self): + self.assertFalse(netrc._can_security_check()) + self.assertEqual(netrc._getpwuid(0), 'uid 0') + self.assertEqual(netrc._getpwuid(123456), 'uid 123456') @unittest.skipUnless(os.name == 'posix', 'POSIX only test') - @unittest.skipIf(pwd is None, 'security check requires pwd module') + @unittest.skipUnless(hasattr(os, 'getuid'), "os.getuid is required") @os_helper.skip_unless_working_chmod def test_security(self): # This test is incomplete since we are normally not run as root and @@ -308,8 +309,6 @@ def test_security(self): self.assertEqual(nrc.hosts['foo.domain.com'], ('anonymous', '', 'pass')) -def test_main(): - run_unittest(NetrcTestCase) if __name__ == "__main__": - test_main() + unittest.main() diff --git a/Lib/test/test_ntpath.py b/Lib/test/test_ntpath.py index 7609ecea79..e1982dfd0b 100644 --- a/Lib/test/test_ntpath.py +++ b/Lib/test/test_ntpath.py @@ -2,9 +2,12 @@ import ntpath import os import string +import subprocess import sys import unittest import warnings +from ntpath import ALLOW_MISSING +from test import support from test.support import cpython_only, os_helper from test.support import TestFailed, is_emscripten from test.support.os_helper import FakePath @@ -76,6 +79,10 @@ def tester(fn, wantResult): %(str(fn), str(wantResult), repr(gotResult))) +def _parameterize(*parameters): + return support.subTests('kwargs', parameters, _do_cleanups=True) + + class NtpathTestCase(unittest.TestCase): def assertPathEqual(self, path1, path2): if path1 == path2 or _norm(path1) == _norm(path2): @@ -123,6 +130,22 @@ def test_splitdrive(self): tester('ntpath.splitdrive("//?/UNC/server/share/dir")', ("//?/UNC/server/share", "/dir")) + def test_splitdrive_invalid_paths(self): + splitdrive = ntpath.splitdrive + self.assertEqual(splitdrive('\\\\ser\x00ver\\sha\x00re\\di\x00r'), + ('\\\\ser\x00ver\\sha\x00re', '\\di\x00r')) + self.assertEqual(splitdrive(b'\\\\ser\x00ver\\sha\x00re\\di\x00r'), + (b'\\\\ser\x00ver\\sha\x00re', b'\\di\x00r')) + self.assertEqual(splitdrive("\\\\\udfff\\\udffe\\\udffd"), + ('\\\\\udfff\\\udffe', '\\\udffd')) + if sys.platform == 'win32': + self.assertRaises(UnicodeDecodeError, splitdrive, b'\\\\\xff\\share\\dir') + self.assertRaises(UnicodeDecodeError, splitdrive, b'\\\\server\\\xff\\dir') + self.assertRaises(UnicodeDecodeError, splitdrive, b'\\\\server\\share\\\xff') + else: + self.assertEqual(splitdrive(b'\\\\\xff\\\xfe\\\xfd'), + (b'\\\\\xff\\\xfe', b'\\\xfd')) + def test_splitroot(self): tester("ntpath.splitroot('')", ('', '', '')) tester("ntpath.splitroot('foo')", ('', '', 'foo')) @@ -213,6 +236,22 @@ def test_splitroot(self): tester('ntpath.splitroot(" :/foo")', (" :", "/", "foo")) tester('ntpath.splitroot("/:/foo")', ("", "/", ":/foo")) + def test_splitroot_invalid_paths(self): + splitroot = ntpath.splitroot + self.assertEqual(splitroot('\\\\ser\x00ver\\sha\x00re\\di\x00r'), + ('\\\\ser\x00ver\\sha\x00re', '\\', 'di\x00r')) + self.assertEqual(splitroot(b'\\\\ser\x00ver\\sha\x00re\\di\x00r'), + (b'\\\\ser\x00ver\\sha\x00re', b'\\', b'di\x00r')) + self.assertEqual(splitroot("\\\\\udfff\\\udffe\\\udffd"), + ('\\\\\udfff\\\udffe', '\\', '\udffd')) + if sys.platform == 'win32': + self.assertRaises(UnicodeDecodeError, splitroot, b'\\\\\xff\\share\\dir') + self.assertRaises(UnicodeDecodeError, splitroot, b'\\\\server\\\xff\\dir') + self.assertRaises(UnicodeDecodeError, splitroot, b'\\\\server\\share\\\xff') + else: + self.assertEqual(splitroot(b'\\\\\xff\\\xfe\\\xfd'), + (b'\\\\\xff\\\xfe', b'\\', b'\xfd')) + def test_split(self): tester('ntpath.split("c:\\foo\\bar")', ('c:\\foo', 'bar')) tester('ntpath.split("\\\\conky\\mountpoint\\foo\\bar")', @@ -225,11 +264,34 @@ def test_split(self): tester('ntpath.split("c:/")', ('c:/', '')) tester('ntpath.split("//conky/mountpoint/")', ('//conky/mountpoint/', '')) + def test_split_invalid_paths(self): + split = ntpath.split + self.assertEqual(split('c:\\fo\x00o\\ba\x00r'), + ('c:\\fo\x00o', 'ba\x00r')) + self.assertEqual(split(b'c:\\fo\x00o\\ba\x00r'), + (b'c:\\fo\x00o', b'ba\x00r')) + self.assertEqual(split('c:\\\udfff\\\udffe'), + ('c:\\\udfff', '\udffe')) + if sys.platform == 'win32': + self.assertRaises(UnicodeDecodeError, split, b'c:\\\xff\\bar') + self.assertRaises(UnicodeDecodeError, split, b'c:\\foo\\\xff') + else: + self.assertEqual(split(b'c:\\\xff\\\xfe'), + (b'c:\\\xff', b'\xfe')) + def test_isabs(self): + tester('ntpath.isabs("foo\\bar")', 0) + tester('ntpath.isabs("foo/bar")', 0) tester('ntpath.isabs("c:\\")', 1) + tester('ntpath.isabs("c:\\foo\\bar")', 1) + tester('ntpath.isabs("c:/foo/bar")', 1) tester('ntpath.isabs("\\\\conky\\mountpoint\\")', 1) - tester('ntpath.isabs("\\foo")', 1) - tester('ntpath.isabs("\\foo\\bar")', 1) + + # gh-44626: paths with only a drive or root are not absolute. + tester('ntpath.isabs("\\foo\\bar")', 0) + tester('ntpath.isabs("/foo/bar")', 0) + tester('ntpath.isabs("c:foo\\bar")', 0) + tester('ntpath.isabs("c:foo/bar")', 0) # gh-96290: normal UNC paths and device paths without trailing backslashes tester('ntpath.isabs("\\\\conky\\mountpoint")', 1) @@ -255,6 +317,7 @@ def test_join(self): tester('ntpath.join("a", "b", "c")', 'a\\b\\c') tester('ntpath.join("a\\", "b", "c")', 'a\\b\\c') tester('ntpath.join("a", "b\\", "c")', 'a\\b\\c') + tester('ntpath.join("a", "b", "c\\")', 'a\\b\\c\\') tester('ntpath.join("a", "b", "\\c")', '\\c') tester('ntpath.join("d:\\", "\\pleep")', 'd:\\pleep') tester('ntpath.join("d:\\", "a", "b")', 'd:\\a\\b') @@ -312,6 +375,40 @@ def test_join(self): tester("ntpath.join('\\\\computer\\', 'share')", '\\\\computer\\share') tester("ntpath.join('\\\\computer\\share\\', 'a')", '\\\\computer\\share\\a') tester("ntpath.join('\\\\computer\\share\\a\\', 'b')", '\\\\computer\\share\\a\\b') + # Second part is anchored, so that the first part is ignored. + tester("ntpath.join('a', 'Z:b', 'c')", 'Z:b\\c') + tester("ntpath.join('a', 'Z:\\b', 'c')", 'Z:\\b\\c') + tester("ntpath.join('a', '\\\\b\\c', 'd')", '\\\\b\\c\\d') + # Second part has a root but not drive. + tester("ntpath.join('a', '\\b', 'c')", '\\b\\c') + tester("ntpath.join('Z:/a', '/b', 'c')", 'Z:\\b\\c') + tester("ntpath.join('//?/Z:/a', '/b', 'c')", '\\\\?\\Z:\\b\\c') + tester("ntpath.join('D:a', './c:b')", 'D:a\\.\\c:b') + tester("ntpath.join('D:/a', './c:b')", 'D:\\a\\.\\c:b') + + def test_normcase(self): + normcase = ntpath.normcase + self.assertEqual(normcase(''), '') + self.assertEqual(normcase(b''), b'') + self.assertEqual(normcase('ABC'), 'abc') + self.assertEqual(normcase(b'ABC'), b'abc') + self.assertEqual(normcase('\xc4\u0141\u03a8'), '\xe4\u0142\u03c8') + expected = '\u03c9\u2126' if sys.platform == 'win32' else '\u03c9\u03c9' + self.assertEqual(normcase('\u03a9\u2126'), expected) + if sys.platform == 'win32' or sys.getfilesystemencoding() == 'utf-8': + self.assertEqual(normcase('\xc4\u0141\u03a8'.encode()), + '\xe4\u0142\u03c8'.encode()) + self.assertEqual(normcase('\u03a9\u2126'.encode()), + expected.encode()) + + def test_normcase_invalid_paths(self): + normcase = ntpath.normcase + self.assertEqual(normcase('abc\x00def'), 'abc\x00def') + self.assertEqual(normcase(b'abc\x00def'), b'abc\x00def') + self.assertEqual(normcase('\udfff'), '\udfff') + if sys.platform == 'win32': + path = b'ABC' + bytes(range(128, 256)) + self.assertEqual(normcase(path), path.lower()) def test_normpath(self): tester("ntpath.normpath('A//////././//.//B')", r'A\B') @@ -327,13 +424,18 @@ def test_normpath(self): tester("ntpath.normpath('..')", r'..') tester("ntpath.normpath('.')", r'.') + tester("ntpath.normpath('c:.')", 'c:') tester("ntpath.normpath('')", r'.') tester("ntpath.normpath('/')", '\\') tester("ntpath.normpath('c:/')", 'c:\\') tester("ntpath.normpath('/../.././..')", '\\') tester("ntpath.normpath('c:/../../..')", 'c:\\') + tester("ntpath.normpath('/./a/b')", r'\a\b') + tester("ntpath.normpath('c:/./a/b')", r'c:\a\b') tester("ntpath.normpath('../.././..')", r'..\..\..') tester("ntpath.normpath('K:../.././..')", r'K:..\..\..') + tester("ntpath.normpath('./a/b')", r'a\b') + tester("ntpath.normpath('c:./a/b')", r'c:a\b') tester("ntpath.normpath('C:////a/b')", r'C:\a\b') tester("ntpath.normpath('//machine/share//a/b')", r'\\machine\share\a\b') @@ -354,6 +456,22 @@ def test_normpath(self): tester("ntpath.normpath('\\\\foo\\')", '\\\\foo\\') tester("ntpath.normpath('\\\\foo')", '\\\\foo') tester("ntpath.normpath('\\\\')", '\\\\') + tester("ntpath.normpath('//?/UNC/server/share/..')", '\\\\?\\UNC\\server\\share\\') + + def test_normpath_invalid_paths(self): + normpath = ntpath.normpath + self.assertEqual(normpath('fo\x00o'), 'fo\x00o') + self.assertEqual(normpath(b'fo\x00o'), b'fo\x00o') + self.assertEqual(normpath('fo\x00o\\..\\bar'), 'bar') + self.assertEqual(normpath(b'fo\x00o\\..\\bar'), b'bar') + self.assertEqual(normpath('\udfff'), '\udfff') + self.assertEqual(normpath('\udfff\\..\\foo'), 'foo') + if sys.platform == 'win32': + self.assertRaises(UnicodeDecodeError, normpath, b'\xff') + self.assertRaises(UnicodeDecodeError, normpath, b'\xff\\..\\foo') + else: + self.assertEqual(normpath(b'\xff'), b'\xff') + self.assertEqual(normpath(b'\xff\\..\\foo'), b'foo') def test_realpath_curdir(self): expected = ntpath.normpath(os.getcwd()) @@ -363,6 +481,27 @@ def test_realpath_curdir(self): tester("ntpath.realpath('.\\.')", expected) tester("ntpath.realpath('\\'.join(['.'] * 100))", expected) + def test_realpath_curdir_strict(self): + expected = ntpath.normpath(os.getcwd()) + tester("ntpath.realpath('.', strict=True)", expected) + tester("ntpath.realpath('./.', strict=True)", expected) + tester("ntpath.realpath('/'.join(['.'] * 100), strict=True)", expected) + tester("ntpath.realpath('.\\.', strict=True)", expected) + tester("ntpath.realpath('\\'.join(['.'] * 100), strict=True)", expected) + + def test_realpath_curdir_missing_ok(self): + expected = ntpath.normpath(os.getcwd()) + tester("ntpath.realpath('.', strict=ALLOW_MISSING)", + expected) + tester("ntpath.realpath('./.', strict=ALLOW_MISSING)", + expected) + tester("ntpath.realpath('/'.join(['.'] * 100), strict=ALLOW_MISSING)", + expected) + tester("ntpath.realpath('.\\.', strict=ALLOW_MISSING)", + expected) + tester("ntpath.realpath('\\'.join(['.'] * 100), strict=ALLOW_MISSING)", + expected) + def test_realpath_pardir(self): expected = ntpath.normpath(os.getcwd()) tester("ntpath.realpath('..')", ntpath.dirname(expected)) @@ -375,28 +514,59 @@ def test_realpath_pardir(self): tester("ntpath.realpath('\\'.join(['..'] * 50))", ntpath.splitdrive(expected)[0] + '\\') + def test_realpath_pardir_strict(self): + expected = ntpath.normpath(os.getcwd()) + tester("ntpath.realpath('..', strict=True)", ntpath.dirname(expected)) + tester("ntpath.realpath('../..', strict=True)", + ntpath.dirname(ntpath.dirname(expected))) + tester("ntpath.realpath('/'.join(['..'] * 50), strict=True)", + ntpath.splitdrive(expected)[0] + '\\') + tester("ntpath.realpath('..\\..', strict=True)", + ntpath.dirname(ntpath.dirname(expected))) + tester("ntpath.realpath('\\'.join(['..'] * 50), strict=True)", + ntpath.splitdrive(expected)[0] + '\\') + + def test_realpath_pardir_missing_ok(self): + expected = ntpath.normpath(os.getcwd()) + tester("ntpath.realpath('..', strict=ALLOW_MISSING)", + ntpath.dirname(expected)) + tester("ntpath.realpath('../..', strict=ALLOW_MISSING)", + ntpath.dirname(ntpath.dirname(expected))) + tester("ntpath.realpath('/'.join(['..'] * 50), strict=ALLOW_MISSING)", + ntpath.splitdrive(expected)[0] + '\\') + tester("ntpath.realpath('..\\..', strict=ALLOW_MISSING)", + ntpath.dirname(ntpath.dirname(expected))) + tester("ntpath.realpath('\\'.join(['..'] * 50), strict=ALLOW_MISSING)", + ntpath.splitdrive(expected)[0] + '\\') + @os_helper.skip_unless_symlink @unittest.skipUnless(HAVE_GETFINALPATHNAME, 'need _getfinalpathname') - def test_realpath_basic(self): + @_parameterize({}, {'strict': True}, {'strict': ALLOW_MISSING}) + def test_realpath_basic(self, kwargs): ABSTFN = ntpath.abspath(os_helper.TESTFN) open(ABSTFN, "wb").close() self.addCleanup(os_helper.unlink, ABSTFN) self.addCleanup(os_helper.unlink, ABSTFN + "1") os.symlink(ABSTFN, ABSTFN + "1") - self.assertPathEqual(ntpath.realpath(ABSTFN + "1"), ABSTFN) - self.assertPathEqual(ntpath.realpath(os.fsencode(ABSTFN + "1")), + self.assertPathEqual(ntpath.realpath(ABSTFN + "1", **kwargs), ABSTFN) + self.assertPathEqual(ntpath.realpath(os.fsencode(ABSTFN + "1"), **kwargs), os.fsencode(ABSTFN)) # gh-88013: call ntpath.realpath with binary drive name may raise a # TypeError. The drive should not exist to reproduce the bug. drives = {f"{c}:\\" for c in string.ascii_uppercase} - set(os.listdrives()) d = drives.pop().encode() - self.assertEqual(ntpath.realpath(d), d) + self.assertEqual(ntpath.realpath(d, strict=False), d) # gh-106242: Embedded nulls and non-strict fallback to abspath - self.assertEqual(ABSTFN + "\0spam", - ntpath.realpath(os_helper.TESTFN + "\0spam", strict=False)) + if kwargs: + with self.assertRaises(OSError): + ntpath.realpath(os_helper.TESTFN + "\0spam", + **kwargs) + else: + self.assertEqual(ABSTFN + "\0spam", + ntpath.realpath(os_helper.TESTFN + "\0spam", **kwargs)) @os_helper.skip_unless_symlink @unittest.skipUnless(HAVE_GETFINALPATHNAME, 'need _getfinalpathname') @@ -408,22 +578,78 @@ def test_realpath_strict(self): self.addCleanup(os_helper.unlink, ABSTFN) self.assertRaises(FileNotFoundError, ntpath.realpath, ABSTFN, strict=True) self.assertRaises(FileNotFoundError, ntpath.realpath, ABSTFN + "2", strict=True) + + @unittest.skipUnless(HAVE_GETFINALPATHNAME, 'need _getfinalpathname') + def test_realpath_invalid_paths(self): + realpath = ntpath.realpath + ABSTFN = ntpath.abspath(os_helper.TESTFN) + ABSTFNb = os.fsencode(ABSTFN) + path = ABSTFN + '\x00' + # gh-106242: Embedded nulls and non-strict fallback to abspath + self.assertEqual(realpath(path, strict=False), path) # gh-106242: Embedded nulls should raise OSError (not ValueError) - self.assertRaises(OSError, ntpath.realpath, ABSTFN + "\0spam", strict=True) + self.assertRaises(OSError, realpath, path, strict=True) + self.assertRaises(OSError, realpath, path, strict=ALLOW_MISSING) + path = ABSTFNb + b'\x00' + self.assertEqual(realpath(path, strict=False), path) + self.assertRaises(OSError, realpath, path, strict=True) + self.assertRaises(OSError, realpath, path, strict=ALLOW_MISSING) + path = ABSTFN + '\\nonexistent\\x\x00' + self.assertEqual(realpath(path, strict=False), path) + self.assertRaises(OSError, realpath, path, strict=True) + self.assertRaises(OSError, realpath, path, strict=ALLOW_MISSING) + path = ABSTFNb + b'\\nonexistent\\x\x00' + self.assertEqual(realpath(path, strict=False), path) + self.assertRaises(OSError, realpath, path, strict=True) + self.assertRaises(OSError, realpath, path, strict=ALLOW_MISSING) + path = ABSTFN + '\x00\\..' + self.assertEqual(realpath(path, strict=False), os.getcwd()) + self.assertEqual(realpath(path, strict=True), os.getcwd()) + self.assertEqual(realpath(path, strict=ALLOW_MISSING), os.getcwd()) + path = ABSTFNb + b'\x00\\..' + self.assertEqual(realpath(path, strict=False), os.getcwdb()) + self.assertEqual(realpath(path, strict=True), os.getcwdb()) + self.assertEqual(realpath(path, strict=ALLOW_MISSING), os.getcwdb()) + path = ABSTFN + '\\nonexistent\\x\x00\\..' + self.assertEqual(realpath(path, strict=False), ABSTFN + '\\nonexistent') + self.assertRaises(OSError, realpath, path, strict=True) + self.assertEqual(realpath(path, strict=ALLOW_MISSING), ABSTFN + '\\nonexistent') + path = ABSTFNb + b'\\nonexistent\\x\x00\\..' + self.assertEqual(realpath(path, strict=False), ABSTFNb + b'\\nonexistent') + self.assertRaises(OSError, realpath, path, strict=True) + self.assertEqual(realpath(path, strict=ALLOW_MISSING), ABSTFNb + b'\\nonexistent') + + @unittest.skipUnless(HAVE_GETFINALPATHNAME, 'need _getfinalpathname') + @_parameterize({}, {'strict': True}, {'strict': ALLOW_MISSING}) + def test_realpath_invalid_unicode_paths(self, kwargs): + realpath = ntpath.realpath + ABSTFN = ntpath.abspath(os_helper.TESTFN) + ABSTFNb = os.fsencode(ABSTFN) + path = ABSTFNb + b'\xff' + self.assertRaises(UnicodeDecodeError, realpath, path, **kwargs) + self.assertRaises(UnicodeDecodeError, realpath, path, **kwargs) + path = ABSTFNb + b'\\nonexistent\\\xff' + self.assertRaises(UnicodeDecodeError, realpath, path, **kwargs) + self.assertRaises(UnicodeDecodeError, realpath, path, **kwargs) + path = ABSTFNb + b'\xff\\..' + self.assertRaises(UnicodeDecodeError, realpath, path, **kwargs) + self.assertRaises(UnicodeDecodeError, realpath, path, **kwargs) + path = ABSTFNb + b'\\nonexistent\\\xff\\..' + self.assertRaises(UnicodeDecodeError, realpath, path, **kwargs) + self.assertRaises(UnicodeDecodeError, realpath, path, **kwargs) @os_helper.skip_unless_symlink @unittest.skipUnless(HAVE_GETFINALPATHNAME, 'need _getfinalpathname') - def test_realpath_relative(self): + @_parameterize({}, {'strict': True}, {'strict': ALLOW_MISSING}) + def test_realpath_relative(self, kwargs): ABSTFN = ntpath.abspath(os_helper.TESTFN) open(ABSTFN, "wb").close() self.addCleanup(os_helper.unlink, ABSTFN) self.addCleanup(os_helper.unlink, ABSTFN + "1") os.symlink(ABSTFN, ntpath.relpath(ABSTFN + "1")) - self.assertPathEqual(ntpath.realpath(ABSTFN + "1"), ABSTFN) + self.assertPathEqual(ntpath.realpath(ABSTFN + "1", **kwargs), ABSTFN) - # TODO: RUSTPYTHON - @unittest.expectedFailure @os_helper.skip_unless_symlink @unittest.skipUnless(HAVE_GETFINALPATHNAME, 'need _getfinalpathname') def test_realpath_broken_symlinks(self): @@ -574,7 +800,62 @@ def test_realpath_symlink_loops_strict(self): @os_helper.skip_unless_symlink @unittest.skipUnless(HAVE_GETFINALPATHNAME, 'need _getfinalpathname') - def test_realpath_symlink_prefix(self): + def test_realpath_symlink_loops_raise(self): + # Symlink loops raise OSError in ALLOW_MISSING mode + ABSTFN = ntpath.abspath(os_helper.TESTFN) + self.addCleanup(os_helper.unlink, ABSTFN) + self.addCleanup(os_helper.unlink, ABSTFN + "1") + self.addCleanup(os_helper.unlink, ABSTFN + "2") + self.addCleanup(os_helper.unlink, ABSTFN + "y") + self.addCleanup(os_helper.unlink, ABSTFN + "c") + self.addCleanup(os_helper.unlink, ABSTFN + "a") + self.addCleanup(os_helper.unlink, ABSTFN + "x") + + os.symlink(ABSTFN, ABSTFN) + self.assertRaises(OSError, ntpath.realpath, ABSTFN, strict=ALLOW_MISSING) + + os.symlink(ABSTFN + "1", ABSTFN + "2") + os.symlink(ABSTFN + "2", ABSTFN + "1") + self.assertRaises(OSError, ntpath.realpath, ABSTFN + "1", + strict=ALLOW_MISSING) + self.assertRaises(OSError, ntpath.realpath, ABSTFN + "2", + strict=ALLOW_MISSING) + self.assertRaises(OSError, ntpath.realpath, ABSTFN + "1\\x", + strict=ALLOW_MISSING) + + # Windows eliminates '..' components before resolving links; + # realpath is not expected to raise if this removes the loop. + self.assertPathEqual(ntpath.realpath(ABSTFN + "1\\.."), + ntpath.dirname(ABSTFN)) + self.assertPathEqual(ntpath.realpath(ABSTFN + "1\\..\\x"), + ntpath.dirname(ABSTFN) + "\\x") + + os.symlink(ABSTFN + "x", ABSTFN + "y") + self.assertPathEqual(ntpath.realpath(ABSTFN + "1\\..\\" + + ntpath.basename(ABSTFN) + "y"), + ABSTFN + "x") + self.assertRaises( + OSError, ntpath.realpath, + ABSTFN + "1\\..\\" + ntpath.basename(ABSTFN) + "1", + strict=ALLOW_MISSING) + + os.symlink(ntpath.basename(ABSTFN) + "a\\b", ABSTFN + "a") + self.assertRaises(OSError, ntpath.realpath, ABSTFN + "a", + strict=ALLOW_MISSING) + + os.symlink("..\\" + ntpath.basename(ntpath.dirname(ABSTFN)) + + "\\" + ntpath.basename(ABSTFN) + "c", ABSTFN + "c") + self.assertRaises(OSError, ntpath.realpath, ABSTFN + "c", + strict=ALLOW_MISSING) + + # Test using relative path as well. + self.assertRaises(OSError, ntpath.realpath, ntpath.basename(ABSTFN), + strict=ALLOW_MISSING) + + @os_helper.skip_unless_symlink + @unittest.skipUnless(HAVE_GETFINALPATHNAME, 'need _getfinalpathname') + @_parameterize({}, {'strict': True}, {'strict': ALLOW_MISSING}) + def test_realpath_symlink_prefix(self, kwargs): ABSTFN = ntpath.abspath(os_helper.TESTFN) self.addCleanup(os_helper.unlink, ABSTFN + "3") self.addCleanup(os_helper.unlink, "\\\\?\\" + ABSTFN + "3.") @@ -589,9 +870,9 @@ def test_realpath_symlink_prefix(self): f.write(b'1') os.symlink("\\\\?\\" + ABSTFN + "3.", ABSTFN + "3.link") - self.assertPathEqual(ntpath.realpath(ABSTFN + "3link"), + self.assertPathEqual(ntpath.realpath(ABSTFN + "3link", **kwargs), ABSTFN + "3") - self.assertPathEqual(ntpath.realpath(ABSTFN + "3.link"), + self.assertPathEqual(ntpath.realpath(ABSTFN + "3.link", **kwargs), "\\\\?\\" + ABSTFN + "3.") # Resolved paths should be usable to open target files @@ -601,14 +882,17 @@ def test_realpath_symlink_prefix(self): self.assertEqual(f.read(), b'1') # When the prefix is included, it is not stripped - self.assertPathEqual(ntpath.realpath("\\\\?\\" + ABSTFN + "3link"), + self.assertPathEqual(ntpath.realpath("\\\\?\\" + ABSTFN + "3link", **kwargs), "\\\\?\\" + ABSTFN + "3") - self.assertPathEqual(ntpath.realpath("\\\\?\\" + ABSTFN + "3.link"), + self.assertPathEqual(ntpath.realpath("\\\\?\\" + ABSTFN + "3.link", **kwargs), "\\\\?\\" + ABSTFN + "3.") @unittest.skipUnless(HAVE_GETFINALPATHNAME, 'need _getfinalpathname') def test_realpath_nul(self): tester("ntpath.realpath('NUL')", r'\\.\NUL') + tester("ntpath.realpath('NUL', strict=False)", r'\\.\NUL') + tester("ntpath.realpath('NUL', strict=True)", r'\\.\NUL') + tester("ntpath.realpath('NUL', strict=ALLOW_MISSING)", r'\\.\NUL') @unittest.skipUnless(HAVE_GETFINALPATHNAME, 'need _getfinalpathname') @unittest.skipUnless(HAVE_GETSHORTPATHNAME, 'need _getshortpathname') @@ -632,14 +916,66 @@ def test_realpath_cwd(self): self.assertPathEqual(test_file_long, ntpath.realpath(test_file_short)) - with os_helper.change_cwd(test_dir_long): - self.assertPathEqual(test_file_long, ntpath.realpath("file.txt")) - with os_helper.change_cwd(test_dir_long.lower()): - self.assertPathEqual(test_file_long, ntpath.realpath("file.txt")) - with os_helper.change_cwd(test_dir_short): - self.assertPathEqual(test_file_long, ntpath.realpath("file.txt")) + for kwargs in {}, {'strict': True}, {'strict': ALLOW_MISSING}: + with self.subTest(**kwargs): + with os_helper.change_cwd(test_dir_long): + self.assertPathEqual( + test_file_long, + ntpath.realpath("file.txt", **kwargs)) + with os_helper.change_cwd(test_dir_long.lower()): + self.assertPathEqual( + test_file_long, + ntpath.realpath("file.txt", **kwargs)) + with os_helper.change_cwd(test_dir_short): + self.assertPathEqual( + test_file_long, + ntpath.realpath("file.txt", **kwargs)) + + @unittest.skipUnless(HAVE_GETFINALPATHNAME, 'need _getfinalpathname') + def test_realpath_permission(self): + # Test whether python can resolve the real filename of a + # shortened file name even if it does not have permission to access it. + ABSTFN = ntpath.realpath(os_helper.TESTFN) + + os_helper.unlink(ABSTFN) + os_helper.rmtree(ABSTFN) + os.mkdir(ABSTFN) + self.addCleanup(os_helper.rmtree, ABSTFN) + + test_file = ntpath.join(ABSTFN, "LongFileName123.txt") + test_file_short = ntpath.join(ABSTFN, "LONGFI~1.TXT") + + with open(test_file, "wb") as f: + f.write(b"content") + # Automatic generation of short names may be disabled on + # NTFS volumes for the sake of performance. + # They're not supported at all on ReFS and exFAT. + p = subprocess.run( + # Try to set the short name manually. + ['fsutil.exe', 'file', 'setShortName', test_file, 'LONGFI~1.TXT'], + creationflags=subprocess.DETACHED_PROCESS + ) + + if p.returncode: + raise unittest.SkipTest('failed to set short name') + + try: + self.assertPathEqual(test_file, ntpath.realpath(test_file_short)) + except AssertionError: + raise unittest.SkipTest('the filesystem seems to lack support for short filenames') + + # Deny the right to [S]YNCHRONIZE on the file to + # force nt._getfinalpathname to fail with ERROR_ACCESS_DENIED. + p = subprocess.run( + ['icacls.exe', test_file, '/deny', '*S-1-5-32-545:(S)'], + creationflags=subprocess.DETACHED_PROCESS + ) + + if p.returncode: + raise unittest.SkipTest('failed to deny access to the test file') + + self.assertPathEqual(test_file, ntpath.realpath(test_file_short)) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; ValueError: illegal environment variable name") def test_expandvars(self): with os_helper.EnvironmentVarGuard() as env: env.clear() @@ -666,7 +1002,6 @@ def test_expandvars(self): tester('ntpath.expandvars("\'%foo%\'%bar")', "\'%foo%\'%bar") tester('ntpath.expandvars("bar\'%foo%")', "bar\'%foo%") - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; ValueError: illegal environment variable name") @unittest.skipUnless(os_helper.FS_NONASCII, 'need os_helper.FS_NONASCII') def test_expandvars_nonascii(self): def check(value, expected): @@ -687,7 +1022,6 @@ def check(value, expected): check('%spam%bar', '%sbar' % nonascii) check('%{}%bar'.format(nonascii), 'ham%sbar' % nonascii) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_expanduser(self): tester('ntpath.expanduser("test")', 'test') @@ -748,6 +1082,7 @@ def test_abspath(self): tester('ntpath.abspath("C:\\spam. . .")', "C:\\spam") tester('ntpath.abspath("C:/nul")', "\\\\.\\nul") tester('ntpath.abspath("C:\\nul")', "\\\\.\\nul") + self.assertTrue(ntpath.isabs(ntpath.abspath("C:spam"))) tester('ntpath.abspath("//..")', "\\\\") tester('ntpath.abspath("//../")', "\\\\..\\") tester('ntpath.abspath("//../..")', "\\\\..\\") @@ -781,6 +1116,26 @@ def test_abspath(self): drive, _ = ntpath.splitdrive(cwd_dir) tester('ntpath.abspath("/abc/")', drive + "\\abc") + def test_abspath_invalid_paths(self): + abspath = ntpath.abspath + if sys.platform == 'win32': + self.assertEqual(abspath("C:\x00"), ntpath.join(abspath("C:"), "\x00")) + self.assertEqual(abspath(b"C:\x00"), ntpath.join(abspath(b"C:"), b"\x00")) + self.assertEqual(abspath("\x00:spam"), "\x00:\\spam") + self.assertEqual(abspath(b"\x00:spam"), b"\x00:\\spam") + self.assertEqual(abspath('c:\\fo\x00o'), 'c:\\fo\x00o') + self.assertEqual(abspath(b'c:\\fo\x00o'), b'c:\\fo\x00o') + self.assertEqual(abspath('c:\\fo\x00o\\..\\bar'), 'c:\\bar') + self.assertEqual(abspath(b'c:\\fo\x00o\\..\\bar'), b'c:\\bar') + self.assertEqual(abspath('c:\\\udfff'), 'c:\\\udfff') + self.assertEqual(abspath('c:\\\udfff\\..\\foo'), 'c:\\foo') + if sys.platform == 'win32': + self.assertRaises(UnicodeDecodeError, abspath, b'c:\\\xff') + self.assertRaises(UnicodeDecodeError, abspath, b'c:\\\xff\\..\\foo') + else: + self.assertEqual(abspath(b'c:\\\xff'), b'c:\\\xff') + self.assertEqual(abspath(b'c:\\\xff\\..\\foo'), b'c:\\foo') + def test_relpath(self): tester('ntpath.relpath("a")', 'a') tester('ntpath.relpath(ntpath.abspath("a"))', 'a') @@ -809,43 +1164,47 @@ def test_commonpath(self): def check(paths, expected): tester(('ntpath.commonpath(%r)' % paths).replace('\\\\', '\\'), expected) - def check_error(exc, paths): - self.assertRaises(exc, ntpath.commonpath, paths) - self.assertRaises(exc, ntpath.commonpath, - [os.fsencode(p) for p in paths]) - + def check_error(paths, expected): + self.assertRaisesRegex(ValueError, expected, ntpath.commonpath, paths) + self.assertRaisesRegex(ValueError, expected, ntpath.commonpath, paths[::-1]) + self.assertRaisesRegex(ValueError, expected, ntpath.commonpath, + [os.fsencode(p) for p in paths]) + self.assertRaisesRegex(ValueError, expected, ntpath.commonpath, + [os.fsencode(p) for p in paths[::-1]]) + + self.assertRaises(TypeError, ntpath.commonpath, None) self.assertRaises(ValueError, ntpath.commonpath, []) - check_error(ValueError, ['C:\\Program Files', 'Program Files']) - check_error(ValueError, ['C:\\Program Files', 'C:Program Files']) - check_error(ValueError, ['\\Program Files', 'Program Files']) - check_error(ValueError, ['Program Files', 'C:\\Program Files']) - check(['C:\\Program Files'], 'C:\\Program Files') - check(['C:\\Program Files', 'C:\\Program Files'], 'C:\\Program Files') - check(['C:\\Program Files\\', 'C:\\Program Files'], - 'C:\\Program Files') - check(['C:\\Program Files\\', 'C:\\Program Files\\'], - 'C:\\Program Files') - check(['C:\\\\Program Files', 'C:\\Program Files\\\\'], - 'C:\\Program Files') - check(['C:\\.\\Program Files', 'C:\\Program Files\\.'], - 'C:\\Program Files') - check(['C:\\', 'C:\\bin'], 'C:\\') - check(['C:\\Program Files', 'C:\\bin'], 'C:\\') - check(['C:\\Program Files', 'C:\\Program Files\\Bar'], - 'C:\\Program Files') - check(['C:\\Program Files\\Foo', 'C:\\Program Files\\Bar'], - 'C:\\Program Files') - check(['C:\\Program Files', 'C:\\Projects'], 'C:\\') - check(['C:\\Program Files\\', 'C:\\Projects'], 'C:\\') - - check(['C:\\Program Files\\Foo', 'C:/Program Files/Bar'], - 'C:\\Program Files') - check(['C:\\Program Files\\Foo', 'c:/program files/bar'], - 'C:\\Program Files') - check(['c:/program files/bar', 'C:\\Program Files\\Foo'], - 'c:\\program files') - - check_error(ValueError, ['C:\\Program Files', 'D:\\Program Files']) + self.assertRaises(ValueError, ntpath.commonpath, iter([])) + + # gh-117381: Logical error messages + check_error(['C:\\Foo', 'C:Foo'], "Can't mix absolute and relative paths") + check_error(['C:\\Foo', '\\Foo'], "Paths don't have the same drive") + check_error(['C:\\Foo', 'Foo'], "Paths don't have the same drive") + check_error(['C:Foo', '\\Foo'], "Paths don't have the same drive") + check_error(['C:Foo', 'Foo'], "Paths don't have the same drive") + check_error(['\\Foo', 'Foo'], "Can't mix rooted and not-rooted paths") + + check(['C:\\Foo'], 'C:\\Foo') + check(['C:\\Foo', 'C:\\Foo'], 'C:\\Foo') + check(['C:\\Foo\\', 'C:\\Foo'], 'C:\\Foo') + check(['C:\\Foo\\', 'C:\\Foo\\'], 'C:\\Foo') + check(['C:\\\\Foo', 'C:\\Foo\\\\'], 'C:\\Foo') + check(['C:\\.\\Foo', 'C:\\Foo\\.'], 'C:\\Foo') + check(['C:\\', 'C:\\baz'], 'C:\\') + check(['C:\\Bar', 'C:\\baz'], 'C:\\') + check(['C:\\Foo', 'C:\\Foo\\Baz'], 'C:\\Foo') + check(['C:\\Foo\\Bar', 'C:\\Foo\\Baz'], 'C:\\Foo') + check(['C:\\Bar', 'C:\\Baz'], 'C:\\') + check(['C:\\Bar\\', 'C:\\Baz'], 'C:\\') + + check(['C:\\Foo\\Bar', 'C:/Foo/Baz'], 'C:\\Foo') + check(['C:\\Foo\\Bar', 'c:/foo/baz'], 'C:\\Foo') + check(['c:/foo/bar', 'C:\\Foo\\Baz'], 'c:\\foo') + + # gh-117381: Logical error messages + check_error(['C:\\Foo', 'D:\\Foo'], "Paths don't have the same drive") + check_error(['C:\\Foo', 'D:Foo'], "Paths don't have the same drive") + check_error(['C:Foo', 'D:Foo'], "Paths don't have the same drive") check(['spam'], 'spam') check(['spam', 'spam'], 'spam') @@ -859,20 +1218,16 @@ def check_error(exc, paths): check([''], '') check(['', 'spam\\alot'], '') - check_error(ValueError, ['', '\\spam\\alot']) - - self.assertRaises(TypeError, ntpath.commonpath, - [b'C:\\Program Files', 'C:\\Program Files\\Foo']) - self.assertRaises(TypeError, ntpath.commonpath, - [b'C:\\Program Files', 'Program Files\\Foo']) - self.assertRaises(TypeError, ntpath.commonpath, - [b'Program Files', 'C:\\Program Files\\Foo']) - self.assertRaises(TypeError, ntpath.commonpath, - ['C:\\Program Files', b'C:\\Program Files\\Foo']) - self.assertRaises(TypeError, ntpath.commonpath, - ['C:\\Program Files', b'Program Files\\Foo']) - self.assertRaises(TypeError, ntpath.commonpath, - ['Program Files', b'C:\\Program Files\\Foo']) + + # gh-117381: Logical error messages + check_error(['', '\\spam\\alot'], "Can't mix rooted and not-rooted paths") + + self.assertRaises(TypeError, ntpath.commonpath, [b'C:\\Foo', 'C:\\Foo\\Baz']) + self.assertRaises(TypeError, ntpath.commonpath, [b'C:\\Foo', 'Foo\\Baz']) + self.assertRaises(TypeError, ntpath.commonpath, [b'Foo', 'C:\\Foo\\Baz']) + self.assertRaises(TypeError, ntpath.commonpath, ['C:\\Foo', b'C:\\Foo\\Baz']) + self.assertRaises(TypeError, ntpath.commonpath, ['C:\\Foo', b'Foo\\Baz']) + self.assertRaises(TypeError, ntpath.commonpath, ['Foo', b'C:\\Foo\\Baz']) @unittest.skipIf(is_emscripten, "Emscripten cannot fstat unnamed files.") def test_sameopenfile(self): @@ -924,6 +1279,74 @@ def test_ismount(self): self.assertTrue(ntpath.ismount(b"\\\\localhost\\c$")) self.assertTrue(ntpath.ismount(b"\\\\localhost\\c$\\")) + def test_ismount_invalid_paths(self): + ismount = ntpath.ismount + self.assertFalse(ismount("c:\\\udfff")) + if sys.platform == 'win32': + self.assertRaises(ValueError, ismount, "c:\\\x00") + self.assertRaises(ValueError, ismount, b"c:\\\x00") + self.assertRaises(UnicodeDecodeError, ismount, b"c:\\\xff") + else: + self.assertFalse(ismount("c:\\\x00")) + self.assertFalse(ismount(b"c:\\\x00")) + self.assertFalse(ismount(b"c:\\\xff")) + + def test_isreserved(self): + self.assertFalse(ntpath.isreserved('')) + self.assertFalse(ntpath.isreserved('.')) + self.assertFalse(ntpath.isreserved('..')) + self.assertFalse(ntpath.isreserved('/')) + self.assertFalse(ntpath.isreserved('/foo/bar')) + # A name that ends with a space or dot is reserved. + self.assertTrue(ntpath.isreserved('foo.')) + self.assertTrue(ntpath.isreserved('foo ')) + # ASCII control characters are reserved. + self.assertTrue(ntpath.isreserved('\foo')) + # Wildcard characters, colon, and pipe are reserved. + self.assertTrue(ntpath.isreserved('foo*bar')) + self.assertTrue(ntpath.isreserved('foo?bar')) + self.assertTrue(ntpath.isreserved('foo"bar')) + self.assertTrue(ntpath.isreserved('foo<bar')) + self.assertTrue(ntpath.isreserved('foo>bar')) + self.assertTrue(ntpath.isreserved('foo:bar')) + self.assertTrue(ntpath.isreserved('foo|bar')) + # Case-insensitive DOS-device names are reserved. + self.assertTrue(ntpath.isreserved('nul')) + self.assertTrue(ntpath.isreserved('aux')) + self.assertTrue(ntpath.isreserved('prn')) + self.assertTrue(ntpath.isreserved('con')) + self.assertTrue(ntpath.isreserved('conin$')) + self.assertTrue(ntpath.isreserved('conout$')) + # COM/LPT + 1-9 or + superscript 1-3 are reserved. + self.assertTrue(ntpath.isreserved('COM1')) + self.assertTrue(ntpath.isreserved('LPT9')) + self.assertTrue(ntpath.isreserved('com\xb9')) + self.assertTrue(ntpath.isreserved('com\xb2')) + self.assertTrue(ntpath.isreserved('lpt\xb3')) + # DOS-device name matching ignores characters after a dot or + # a colon and also ignores trailing spaces. + self.assertTrue(ntpath.isreserved('NUL.txt')) + self.assertTrue(ntpath.isreserved('PRN ')) + self.assertTrue(ntpath.isreserved('AUX .txt')) + self.assertTrue(ntpath.isreserved('COM1:bar')) + self.assertTrue(ntpath.isreserved('LPT9 :bar')) + # DOS-device names are only matched at the beginning + # of a path component. + self.assertFalse(ntpath.isreserved('bar.com9')) + self.assertFalse(ntpath.isreserved('bar.lpt9')) + # The entire path is checked, except for the drive. + self.assertTrue(ntpath.isreserved('c:/bar/baz/NUL')) + self.assertTrue(ntpath.isreserved('c:/NUL/bar/baz')) + self.assertFalse(ntpath.isreserved('//./NUL')) + # Bytes are supported. + self.assertFalse(ntpath.isreserved(b'')) + self.assertFalse(ntpath.isreserved(b'.')) + self.assertFalse(ntpath.isreserved(b'..')) + self.assertFalse(ntpath.isreserved(b'/')) + self.assertFalse(ntpath.isreserved(b'/foo/bar')) + self.assertTrue(ntpath.isreserved(b'foo.')) + self.assertTrue(ntpath.isreserved(b'nul')) + def assertEqualCI(self, s1, s2): """Assert that two strings are equal ignoring case differences.""" self.assertEqual(s1.lower(), s2.lower()) @@ -974,6 +1397,13 @@ def test_isjunction(self): self.assertFalse(ntpath.isjunction('tmpdir')) self.assertPathEqual(ntpath.realpath('testjunc'), ntpath.realpath('tmpdir')) + def test_isfile_invalid_paths(self): + isfile = ntpath.isfile + self.assertIs(isfile('/tmp\udfffabcds'), False) + self.assertIs(isfile(b'/tmp\xffabcds'), False) + self.assertIs(isfile('/tmp\x00abcds'), False) + self.assertIs(isfile(b'/tmp\x00abcds'), False) + @unittest.skipIf(sys.platform != 'win32', "drive letters are a windows concept") def test_isfile_driveletter(self): drive = os.environ.get('SystemDrive') @@ -981,8 +1411,27 @@ def test_isfile_driveletter(self): raise unittest.SkipTest('SystemDrive is not defined or malformed') self.assertFalse(os.path.isfile('\\\\.\\' + drive)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.skipUnless(hasattr(os, 'pipe'), "need os.pipe()") + def test_isfile_anonymous_pipe(self): + pr, pw = os.pipe() + try: + self.assertFalse(ntpath.isfile(pr)) + finally: + os.close(pr) + os.close(pw) + + @unittest.skipIf(sys.platform != 'win32', "windows only") + def test_isfile_named_pipe(self): + import _winapi + named_pipe = f'//./PIPE/python_isfile_test_{os.getpid()}' + h = _winapi.CreateNamedPipe(named_pipe, + _winapi.PIPE_ACCESS_INBOUND, + 0, 1, 0, 0, 0, 0) + try: + self.assertFalse(ntpath.isfile(named_pipe)) + finally: + _winapi.CloseHandle(h) + @unittest.skipIf(sys.platform != 'win32', "windows only") def test_con_device(self): self.assertFalse(os.path.isfile(r"\\.\CON")) @@ -996,17 +1445,23 @@ def test_fast_paths_in_use(self): # There are fast paths of these functions implemented in posixmodule.c. # Confirm that they are being used, and not the Python fallbacks in # genericpath.py. + self.assertTrue(os.path.splitroot is nt._path_splitroot_ex) + self.assertFalse(inspect.isfunction(os.path.splitroot)) + self.assertTrue(os.path.normpath is nt._path_normpath) + self.assertFalse(inspect.isfunction(os.path.normpath)) self.assertTrue(os.path.isdir is nt._path_isdir) self.assertFalse(inspect.isfunction(os.path.isdir)) self.assertTrue(os.path.isfile is nt._path_isfile) self.assertFalse(inspect.isfunction(os.path.isfile)) self.assertTrue(os.path.islink is nt._path_islink) self.assertFalse(inspect.isfunction(os.path.islink)) + self.assertTrue(os.path.isjunction is nt._path_isjunction) + self.assertFalse(inspect.isfunction(os.path.isjunction)) self.assertTrue(os.path.exists is nt._path_exists) self.assertFalse(inspect.isfunction(os.path.exists)) + self.assertTrue(os.path.lexists is nt._path_lexists) + self.assertFalse(inspect.isfunction(os.path.lexists)) - # TODO: RUSTPYTHON - @unittest.expectedFailure @unittest.skipIf(os.name != 'nt', "Dev Drives only exist on Win32") def test_isdevdrive(self): # Result may be True or False, but shouldn't raise @@ -1032,18 +1487,6 @@ class NtCommonTest(test_genericpath.CommonTest, unittest.TestCase): pathmodule = ntpath attributes = ['relpath'] - # TODO: RUSTPYTHON - if sys.platform == "win32": - # TODO: RUSTPYTHON, ValueError: illegal environment variable name - @unittest.expectedFailure - def test_expandvars(self): # TODO: RUSTPYTHON; remove when done - super().test_expandvars() - - # TODO: RUSTPYTHON, ValueError: illegal environment variable name - @unittest.expectedFailure - def test_expandvars_nonascii(self): # TODO: RUSTPYTHON; remove when done - super().test_expandvars_nonascii() - class PathLikeTests(NtpathTestCase): @@ -1059,12 +1502,8 @@ def setUp(self): def _check_function(self, func): self.assertPathEqual(func(self.file_path), func(self.file_name)) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; AssertionError: 'ωω' != 'ωΩ'") def test_path_normcase(self): self._check_function(self.path.normcase) - if sys.platform == 'win32': - self.assertEqual(ntpath.normcase('\u03a9\u2126'), 'ωΩ') - self.assertEqual(ntpath.normcase('abc\x00def'), 'abc\x00def') def test_path_isabs(self): self._check_function(self.path.isabs) diff --git a/Lib/test/test_opcache.py b/Lib/test/test_opcache.py index 8f1f8764b1..aeb4de6caa 100644 --- a/Lib/test/test_opcache.py +++ b/Lib/test/test_opcache.py @@ -1,4 +1,61 @@ +import copy +import pickle +import dis +import threading +import types import unittest +from test.support import threading_helper, check_impl_detail, requires_specialization +from test.support.import_helper import import_module + +# Skip this module on other interpreters, it is cpython specific: +if check_impl_detail(cpython=False): + raise unittest.SkipTest('implementation detail specific to cpython') + +_testinternalcapi = import_module("_testinternalcapi") + + +def disabling_optimizer(func): + def wrapper(*args, **kwargs): + if not hasattr(_testinternalcapi, "get_optimizer"): + return func(*args, **kwargs) + old_opt = _testinternalcapi.get_optimizer() + _testinternalcapi.set_optimizer(None) + try: + return func(*args, **kwargs) + finally: + _testinternalcapi.set_optimizer(old_opt) + + return wrapper + + +class TestBase(unittest.TestCase): + def assert_specialized(self, f, opname): + instructions = dis.get_instructions(f, adaptive=True) + opnames = {instruction.opname for instruction in instructions} + self.assertIn(opname, opnames) + + +class TestLoadSuperAttrCache(unittest.TestCase): + def test_descriptor_not_double_executed_on_spec_fail(self): + calls = [] + class Descriptor: + def __get__(self, instance, owner): + calls.append((instance, owner)) + return lambda: 1 + + class C: + d = Descriptor() + + class D(C): + def f(self): + return super().d() + + d = D() + + self.assertEqual(d.f(), 1) # warmup + calls.clear() + self.assertEqual(d.f(), 1) # try to specialize + self.assertEqual(calls, [(d, D)]) class TestLoadAttrCache(unittest.TestCase): @@ -177,8 +234,6 @@ def f(): for _ in range(1025): self.assertFalse(f()) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_load_shadowing_slot_should_raise_type_error(self): class Class: __slots__ = ("slot",) @@ -197,8 +252,6 @@ def f(o): with self.assertRaises(TypeError): f(o) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_store_shadowing_slot_should_raise_type_error(self): class Class: __slots__ = ("slot",) @@ -216,7 +269,6 @@ def f(o): with self.assertRaises(TypeError): f(o) - @unittest.skip("TODO: RUSTPYTHON") def test_load_borrowed_slot_should_not_crash(self): class Class: __slots__ = ("slot",) @@ -233,7 +285,6 @@ def f(o): with self.assertRaises(TypeError): f(o) - @unittest.skip("TODO: RUSTPYTHON") def test_store_borrowed_slot_should_not_crash(self): class Class: __slots__ = ("slot",) @@ -435,6 +486,693 @@ def f(): self.assertFalse(f()) +class TestCallCache(TestBase): + def test_too_many_defaults_0(self): + def f(): + pass + + f.__defaults__ = (None,) + for _ in range(1025): + f() + + def test_too_many_defaults_1(self): + def f(x): + pass + + f.__defaults__ = (None, None) + for _ in range(1025): + f(None) + f() + + def test_too_many_defaults_2(self): + def f(x, y): + pass + + f.__defaults__ = (None, None, None) + for _ in range(1025): + f(None, None) + f(None) + f() + + @disabling_optimizer + @requires_specialization + def test_assign_init_code(self): + class MyClass: + def __init__(self): + pass + + def instantiate(): + return MyClass() + + # Trigger specialization + for _ in range(1025): + instantiate() + self.assert_specialized(instantiate, "CALL_ALLOC_AND_ENTER_INIT") + + def count_args(self, *args): + self.num_args = len(args) + + # Set MyClass.__init__.__code__ to a code object that is incompatible + # (uses varargs) with the current specialization + MyClass.__init__.__code__ = count_args.__code__ + instantiate() + + +@threading_helper.requires_working_threading() +@requires_specialization +class TestRacesDoNotCrash(TestBase): + # Careful with these. Bigger numbers have a higher chance of catching bugs, + # but you can also burn through a *ton* of type/dict/function versions: + ITEMS = 1000 + LOOPS = 4 + WARMUPS = 2 + WRITERS = 2 + + @disabling_optimizer + def assert_races_do_not_crash( + self, opname, get_items, read, write, *, check_items=False + ): + # This might need a few dozen loops in some cases: + for _ in range(self.LOOPS): + items = get_items() + # Reset: + if check_items: + for item in items: + item.__code__ = item.__code__.replace() + else: + read.__code__ = read.__code__.replace() + # Specialize: + for _ in range(self.WARMUPS): + read(items) + if check_items: + for item in items: + self.assert_specialized(item, opname) + else: + self.assert_specialized(read, opname) + # Create writers: + writers = [] + for _ in range(self.WRITERS): + writer = threading.Thread(target=write, args=[items]) + writers.append(writer) + # Run: + for writer in writers: + writer.start() + read(items) # BOOM! + for writer in writers: + writer.join() + + def test_binary_subscr_getitem(self): + def get_items(): + class C: + __getitem__ = lambda self, item: None + + items = [] + for _ in range(self.ITEMS): + item = C() + items.append(item) + return items + + def read(items): + for item in items: + try: + item[None] + except TypeError: + pass + + def write(items): + for item in items: + try: + del item.__getitem__ + except AttributeError: + pass + type(item).__getitem__ = lambda self, item: None + + opname = "BINARY_SUBSCR_GETITEM" + self.assert_races_do_not_crash(opname, get_items, read, write) + + def test_binary_subscr_list_int(self): + def get_items(): + items = [] + for _ in range(self.ITEMS): + item = [None] + items.append(item) + return items + + def read(items): + for item in items: + try: + item[0] + except IndexError: + pass + + def write(items): + for item in items: + item.clear() + item.append(None) + + opname = "BINARY_SUBSCR_LIST_INT" + self.assert_races_do_not_crash(opname, get_items, read, write) + + def test_for_iter_gen(self): + def get_items(): + def g(): + yield + yield + + items = [] + for _ in range(self.ITEMS): + item = g() + items.append(item) + return items + + def read(items): + for item in items: + try: + for _ in item: + break + except ValueError: + pass + + def write(items): + for item in items: + try: + for _ in item: + break + except ValueError: + pass + + opname = "FOR_ITER_GEN" + self.assert_races_do_not_crash(opname, get_items, read, write) + + def test_for_iter_list(self): + def get_items(): + items = [] + for _ in range(self.ITEMS): + item = [None] + items.append(item) + return items + + def read(items): + for item in items: + for item in item: + break + + def write(items): + for item in items: + item.clear() + item.append(None) + + opname = "FOR_ITER_LIST" + self.assert_races_do_not_crash(opname, get_items, read, write) + + def test_load_attr_class(self): + def get_items(): + class C: + a = object() + + items = [] + for _ in range(self.ITEMS): + item = C + items.append(item) + return items + + def read(items): + for item in items: + try: + item.a + except AttributeError: + pass + + def write(items): + for item in items: + try: + del item.a + except AttributeError: + pass + item.a = object() + + opname = "LOAD_ATTR_CLASS" + self.assert_races_do_not_crash(opname, get_items, read, write) + + def test_load_attr_getattribute_overridden(self): + def get_items(): + class C: + __getattribute__ = lambda self, name: None + + items = [] + for _ in range(self.ITEMS): + item = C() + items.append(item) + return items + + def read(items): + for item in items: + try: + item.a + except AttributeError: + pass + + def write(items): + for item in items: + try: + del item.__getattribute__ + except AttributeError: + pass + type(item).__getattribute__ = lambda self, name: None + + opname = "LOAD_ATTR_GETATTRIBUTE_OVERRIDDEN" + self.assert_races_do_not_crash(opname, get_items, read, write) + + def test_load_attr_instance_value(self): + def get_items(): + class C: + pass + + items = [] + for _ in range(self.ITEMS): + item = C() + item.a = None + items.append(item) + return items + + def read(items): + for item in items: + item.a + + def write(items): + for item in items: + item.__dict__[None] = None + + opname = "LOAD_ATTR_INSTANCE_VALUE" + self.assert_races_do_not_crash(opname, get_items, read, write) + + def test_load_attr_method_lazy_dict(self): + def get_items(): + class C(Exception): + m = lambda self: None + + items = [] + for _ in range(self.ITEMS): + item = C() + items.append(item) + return items + + def read(items): + for item in items: + try: + item.m() + except AttributeError: + pass + + def write(items): + for item in items: + try: + del item.m + except AttributeError: + pass + type(item).m = lambda self: None + + opname = "LOAD_ATTR_METHOD_LAZY_DICT" + self.assert_races_do_not_crash(opname, get_items, read, write) + + def test_load_attr_method_no_dict(self): + def get_items(): + class C: + __slots__ = () + m = lambda self: None + + items = [] + for _ in range(self.ITEMS): + item = C() + items.append(item) + return items + + def read(items): + for item in items: + try: + item.m() + except AttributeError: + pass + + def write(items): + for item in items: + try: + del item.m + except AttributeError: + pass + type(item).m = lambda self: None + + opname = "LOAD_ATTR_METHOD_NO_DICT" + self.assert_races_do_not_crash(opname, get_items, read, write) + + def test_load_attr_method_with_values(self): + def get_items(): + class C: + m = lambda self: None + + items = [] + for _ in range(self.ITEMS): + item = C() + items.append(item) + return items + + def read(items): + for item in items: + try: + item.m() + except AttributeError: + pass + + def write(items): + for item in items: + try: + del item.m + except AttributeError: + pass + type(item).m = lambda self: None + + opname = "LOAD_ATTR_METHOD_WITH_VALUES" + self.assert_races_do_not_crash(opname, get_items, read, write) + + def test_load_attr_module(self): + def get_items(): + items = [] + for _ in range(self.ITEMS): + item = types.ModuleType("<item>") + items.append(item) + return items + + def read(items): + for item in items: + try: + item.__name__ + except AttributeError: + pass + + def write(items): + for item in items: + d = item.__dict__.copy() + item.__dict__.clear() + item.__dict__.update(d) + + opname = "LOAD_ATTR_MODULE" + self.assert_races_do_not_crash(opname, get_items, read, write) + + def test_load_attr_property(self): + def get_items(): + class C: + a = property(lambda self: None) + + items = [] + for _ in range(self.ITEMS): + item = C() + items.append(item) + return items + + def read(items): + for item in items: + try: + item.a + except AttributeError: + pass + + def write(items): + for item in items: + try: + del type(item).a + except AttributeError: + pass + type(item).a = property(lambda self: None) + + opname = "LOAD_ATTR_PROPERTY" + self.assert_races_do_not_crash(opname, get_items, read, write) + + def test_load_attr_with_hint(self): + def get_items(): + class C: + pass + + items = [] + for _ in range(self.ITEMS): + item = C() + item.a = None + # Resize into a combined unicode dict: + for i in range(29): + setattr(item, f"_{i}", None) + items.append(item) + return items + + def read(items): + for item in items: + item.a + + def write(items): + for item in items: + item.__dict__[None] = None + + opname = "LOAD_ATTR_WITH_HINT" + self.assert_races_do_not_crash(opname, get_items, read, write) + + def test_load_global_module(self): + def get_items(): + items = [] + for _ in range(self.ITEMS): + item = eval("lambda: x", {"x": None}) + items.append(item) + return items + + def read(items): + for item in items: + item() + + def write(items): + for item in items: + item.__globals__[None] = None + + opname = "LOAD_GLOBAL_MODULE" + self.assert_races_do_not_crash( + opname, get_items, read, write, check_items=True + ) + + def test_store_attr_instance_value(self): + def get_items(): + class C: + pass + + items = [] + for _ in range(self.ITEMS): + item = C() + items.append(item) + return items + + def read(items): + for item in items: + item.a = None + + def write(items): + for item in items: + item.__dict__[None] = None + + opname = "STORE_ATTR_INSTANCE_VALUE" + self.assert_races_do_not_crash(opname, get_items, read, write) + + def test_store_attr_with_hint(self): + def get_items(): + class C: + pass + + items = [] + for _ in range(self.ITEMS): + item = C() + # Resize into a combined unicode dict: + for i in range(29): + setattr(item, f"_{i}", None) + items.append(item) + return items + + def read(items): + for item in items: + item.a = None + + def write(items): + for item in items: + item.__dict__[None] = None + + opname = "STORE_ATTR_WITH_HINT" + self.assert_races_do_not_crash(opname, get_items, read, write) + + def test_store_subscr_list_int(self): + def get_items(): + items = [] + for _ in range(self.ITEMS): + item = [None] + items.append(item) + return items + + def read(items): + for item in items: + try: + item[0] = None + except IndexError: + pass + + def write(items): + for item in items: + item.clear() + item.append(None) + + opname = "STORE_SUBSCR_LIST_INT" + self.assert_races_do_not_crash(opname, get_items, read, write) + + def test_unpack_sequence_list(self): + def get_items(): + items = [] + for _ in range(self.ITEMS): + item = [None] + items.append(item) + return items + + def read(items): + for item in items: + try: + [_] = item + except ValueError: + pass + + def write(items): + for item in items: + item.clear() + item.append(None) + + opname = "UNPACK_SEQUENCE_LIST" + self.assert_races_do_not_crash(opname, get_items, read, write) + +class C: + pass + +@requires_specialization +class TestInstanceDict(unittest.TestCase): + + def setUp(self): + c = C() + c.a, c.b, c.c = 0,0,0 + + def test_values_on_instance(self): + c = C() + c.a = 1 + C().b = 2 + c.c = 3 + self.assertEqual( + _testinternalcapi.get_object_dict_values(c), + (1, '<NULL>', 3) + ) + + def test_dict_materialization(self): + c = C() + c.a = 1 + c.b = 2 + c.__dict__ + self.assertEqual(c.__dict__, {"a":1, "b": 2}) + + def test_dict_dematerialization(self): + c = C() + c.a = 1 + c.b = 2 + c.__dict__ + for _ in range(100): + c.a + self.assertEqual( + _testinternalcapi.get_object_dict_values(c), + (1, 2, '<NULL>') + ) + + def test_dict_dematerialization_multiple_refs(self): + c = C() + c.a = 1 + c.b = 2 + d = c.__dict__ + for _ in range(100): + c.a + self.assertIs(c.__dict__, d) + + def test_dict_dematerialization_copy(self): + c = C() + c.a = 1 + c.b = 2 + c2 = copy.copy(c) + for _ in range(100): + c.a + c2.a + self.assertEqual( + _testinternalcapi.get_object_dict_values(c), + (1, 2, '<NULL>') + ) + self.assertEqual( + _testinternalcapi.get_object_dict_values(c2), + (1, 2, '<NULL>') + ) + c3 = copy.deepcopy(c) + for _ in range(100): + c.a + c3.a + self.assertEqual( + _testinternalcapi.get_object_dict_values(c), + (1, 2, '<NULL>') + ) + #NOTE -- c3.__dict__ does not de-materialize + + def test_dict_dematerialization_pickle(self): + c = C() + c.a = 1 + c.b = 2 + c2 = pickle.loads(pickle.dumps(c)) + for _ in range(100): + c.a + c2.a + self.assertEqual( + _testinternalcapi.get_object_dict_values(c), + (1, 2, '<NULL>') + ) + self.assertEqual( + _testinternalcapi.get_object_dict_values(c2), + (1, 2, '<NULL>') + ) + + def test_dict_dematerialization_subclass(self): + class D(dict): pass + c = C() + c.a = 1 + c.b = 2 + c.__dict__ = D(c.__dict__) + for _ in range(100): + c.a + self.assertIs( + _testinternalcapi.get_object_dict_values(c), + None + ) + self.assertEqual( + c.__dict__, + {'a':1, 'b':2} + ) + + def test_store_attr_with_hint(self): + # gh-133441: Regression test for STORE_ATTR_WITH_HINT bytecode + class Node: + def __init__(self): + self.parents = {} + + def __setstate__(self, data_dict): + self.__dict__ = data_dict + self.parents = {} + + class Dict(dict): + pass + + obj = Node() + obj.__setstate__({'parents': {}}) + obj.__setstate__({'parents': {}}) + obj.__setstate__(Dict({'parents': {}})) + + if __name__ == "__main__": - import unittest unittest.main() diff --git a/Lib/test/test_operator.py b/Lib/test/test_operator.py index 1db738d228..05b7a7462d 100644 --- a/Lib/test/test_operator.py +++ b/Lib/test/test_operator.py @@ -1,6 +1,9 @@ import unittest +import inspect import pickle import sys +from decimal import Decimal +from fractions import Fraction from test import support from test.support import import_helper @@ -508,6 +511,44 @@ def __getitem__(self, other): return 5 # so that C is a sequence self.assertEqual(operator.ixor (c, 5), "ixor") self.assertEqual(operator.iconcat (c, c), "iadd") + def test_iconcat_without_getitem(self): + operator = self.module + + msg = "'int' object can't be concatenated" + with self.assertRaisesRegex(TypeError, msg): + operator.iconcat(1, 0.5) + + def test_index(self): + operator = self.module + class X: + def __index__(self): + return 1 + + self.assertEqual(operator.index(X()), 1) + self.assertEqual(operator.index(0), 0) + self.assertEqual(operator.index(1), 1) + self.assertEqual(operator.index(2), 2) + with self.assertRaises((AttributeError, TypeError)): + operator.index(1.5) + with self.assertRaises((AttributeError, TypeError)): + operator.index(Fraction(3, 7)) + with self.assertRaises((AttributeError, TypeError)): + operator.index(Decimal(1)) + with self.assertRaises((AttributeError, TypeError)): + operator.index(None) + + def test_not_(self): + operator = self.module + class C: + def __bool__(self): + raise SyntaxError + self.assertRaises(TypeError, operator.not_) + self.assertRaises(SyntaxError, operator.not_, C()) + self.assertFalse(operator.not_(5)) + self.assertFalse(operator.not_([0])) + self.assertTrue(operator.not_(0)) + self.assertTrue(operator.not_([])) + def test_length_hint(self): operator = self.module class X(object): @@ -533,6 +574,13 @@ def __length_hint__(self): with self.assertRaises(LookupError): operator.length_hint(X(LookupError)) + class Y: pass + + msg = "'str' object cannot be interpreted as an integer" + with self.assertRaisesRegex(TypeError, msg): + operator.length_hint(X(2), "abc") + self.assertEqual(operator.length_hint(Y(), 10), 10) + def test_call(self): operator = self.module @@ -555,6 +603,31 @@ def test_dunder_is_original(self): if dunder: self.assertIs(dunder, orig) + @support.requires_docstrings + def test_attrgetter_signature(self): + operator = self.module + sig = inspect.signature(operator.attrgetter) + self.assertEqual(str(sig), '(attr, /, *attrs)') + sig = inspect.signature(operator.attrgetter('x', 'z', 'y')) + self.assertEqual(str(sig), '(obj, /)') + + @support.requires_docstrings + def test_itemgetter_signature(self): + operator = self.module + sig = inspect.signature(operator.itemgetter) + self.assertEqual(str(sig), '(item, /, *items)') + sig = inspect.signature(operator.itemgetter(2, 3, 5)) + self.assertEqual(str(sig), '(obj, /)') + + @support.requires_docstrings + def test_methodcaller_signature(self): + operator = self.module + sig = inspect.signature(operator.methodcaller) + self.assertEqual(str(sig), '(name, /, *args, **kwargs)') + sig = inspect.signature(operator.methodcaller('foo', 2, y=3)) + self.assertEqual(str(sig), '(obj, /)') + + class PyOperatorTestCase(OperatorTestCase, unittest.TestCase): module = py_operator @@ -562,6 +635,21 @@ class PyOperatorTestCase(OperatorTestCase, unittest.TestCase): class COperatorTestCase(OperatorTestCase, unittest.TestCase): module = c_operator + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_attrgetter_signature(self): + super().test_attrgetter_signature() + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_itemgetter_signature(self): + super().test_itemgetter_signature() + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_methodcaller_signature(self): + super().test_methodcaller_signature() + class OperatorPickleTestCase: def copy(self, obj, proto): diff --git a/Lib/test/test_optparse.py b/Lib/test/test_optparse.py index 28b2744623..c68214961e 100644 --- a/Lib/test/test_optparse.py +++ b/Lib/test/test_optparse.py @@ -15,7 +15,7 @@ from io import StringIO from test import support from test.support import os_helper - +from test.support.i18n_helper import TestTranslationsBase, update_translation_snapshots import optparse from optparse import make_option, Option, \ @@ -614,9 +614,9 @@ def test_float_default(self): self.parser.add_option( "-p", "--prob", help="blow up with probability PROB [default: %default]") - self.parser.set_defaults(prob=0.43) + self.parser.set_defaults(prob=0.25) expected_help = self.help_prefix + \ - " -p PROB, --prob=PROB blow up with probability PROB [default: 0.43]\n" + " -p PROB, --prob=PROB blow up with probability PROB [default: 0.25]\n" self.assertHelp(self.parser, expected_help) def test_alt_expand(self): @@ -1656,5 +1656,14 @@ def test__all__(self): support.check__all__(self, optparse, not_exported=not_exported) +class TestTranslations(TestTranslationsBase): + def test_translations(self): + self.assertMsgidsEqual(optparse) + + if __name__ == '__main__': + # To regenerate translation snapshots + if len(sys.argv) > 1 and sys.argv[1] == '--snapshot-update': + update_translation_snapshots(optparse) + sys.exit(0) unittest.main() diff --git a/Lib/test/test_os.py b/Lib/test/test_os.py index 3b19d3d3cd..4755aef080 100644 --- a/Lib/test/test_os.py +++ b/Lib/test/test_os.py @@ -14,6 +14,7 @@ import os import pickle import select +import selectors import shutil import signal import socket @@ -33,7 +34,7 @@ from test.support import import_helper from test.support import os_helper from test.support import socket_helper -from test.support import set_recursion_limit +from test.support import infinite_recursion from test.support import warnings_helper from platform import win32_is_iot @@ -55,8 +56,10 @@ except (ImportError, AttributeError): all_users = [] try: + import _testcapi from _testcapi import INT_MAX, PY_SSIZE_T_MAX except ImportError: + _testcapi = None INT_MAX = PY_SSIZE_T_MAX = sys.maxsize try: @@ -184,7 +187,7 @@ def test_access(self): os.close(f) self.assertTrue(os.access(os_helper.TESTFN, os.W_OK)) - @unittest.skipIf(sys.platform == "win32", "TODO: RUSTPYTHON, BrokenPipeError: (32, 'The process cannot access the file because it is being used by another process. (os error 32)')") + @unittest.skipIf(sys.platform == 'win32', "TODO: RUSTPYTHON; BrokenPipeError: (32, 'The process cannot access the file because it is being used by another process. (os error 32)')") @unittest.skipIf( support.is_emscripten, "Test is unstable under Emscripten." ) @@ -711,7 +714,6 @@ def check_file_attributes(self, result): self.assertTrue(isinstance(result.st_file_attributes, int)) self.assertTrue(0 <= result.st_file_attributes <= 0xFFFFFFFF) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON os.stat return value doesnt have st_file_attributes attribute") @unittest.skipUnless(sys.platform == "win32", "st_file_attributes is Win32 specific") def test_file_attributes(self): @@ -733,7 +735,6 @@ def test_file_attributes(self): result.st_file_attributes & stat.FILE_ATTRIBUTE_DIRECTORY, stat.FILE_ATTRIBUTE_DIRECTORY) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON os.stat (PermissionError: [Errno 5] Access is denied.)") @unittest.skipUnless(sys.platform == "win32", "Win32 specific tests") def test_access_denied(self): # Default to FindFirstFile WIN32_FIND_DATA when access is @@ -756,7 +757,6 @@ def test_access_denied(self): self.assertNotEqual(result.st_size, 0) self.assertTrue(os.path.isfile(fname)) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON os.stat (PermissionError: [Errno 1] Incorrect function.)") @unittest.skipUnless(sys.platform == "win32", "Win32 specific tests") def test_stat_block_device(self): # bpo-38030: os.stat fails for block devices @@ -814,7 +814,6 @@ def _test_utime(self, set_time, filename=None): self.assertEqual(st.st_atime_ns, atime_ns) self.assertEqual(st.st_mtime_ns, mtime_ns) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (AssertionError: 2.002003 != 1.002003 within 1e-06 delta (1.0000000000000002 difference))") def test_utime(self): def set_time(filename, ns): # test the ns keyword parameter @@ -826,12 +825,12 @@ def ns_to_sec(ns): # Convert a number of nanosecond (int) to a number of seconds (float). # Round towards infinity by adding 0.5 nanosecond to avoid rounding # issue, os.utime() rounds towards minus infinity. - # XXX: RUSTCPYTHON os.utime() use `[Duration::from_secs_f64](https://doc.rust-lang.org/std/time/struct.Duration.html#method.try_from_secs_f64)` + # XXX: RUSTPYTHON os.utime() use `[Duration::from_secs_f64](https://doc.rust-lang.org/std/time/struct.Duration.html#method.try_from_secs_f64)` # return (ns * 1e-9) + 0.5e-9 return (ns * 1e-9) def test_utime_by_indexed(self): - # pass times as floating point seconds as the second indexed parameter + # pass times as floating-point seconds as the second indexed parameter def set_time(filename, ns): atime_ns, mtime_ns = ns atime = self.ns_to_sec(atime_ns) @@ -880,7 +879,6 @@ def set_time(filename, ns): os.utime(name, dir_fd=dirfd, ns=ns) self._test_utime(set_time) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (AssertionError: 2.002003 != 1.002003 within 1e-06 delta (1.0000000000000002 difference))") def test_utime_directory(self): def set_time(filename, ns): # test calling os.utime() on a directory @@ -909,20 +907,25 @@ def _test_utime_current(self, set_time): self.assertAlmostEqual(st.st_mtime, current, delta=delta, msg=msg) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (AssertionError: 3359485824.516508 != 1679742912.516503 within 0.05 delta (1679742912.000005 difference) : st_time=3359485824.516508, current=1679742912.516503, dt=1679742912.000005)") def test_utime_current(self): def set_time(filename): # Set to the current time in the new way os.utime(self.fname) self._test_utime_current(set_time) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (AssertionError: 3359485824.5186944 != 1679742912.5186892 within 0.05 delta (1679742912.0000052 difference) : st_time=3359485824.5186944, current=1679742912.5186892, dt=1679742912.0000052)") def test_utime_current_old(self): def set_time(filename): # Set to the current time in the old explicit way. os.utime(self.fname, None) self._test_utime_current(set_time) + def test_utime_nonexistent(self): + now = time.time() + filename = 'nonexistent' + with self.assertRaises(FileNotFoundError) as cm: + os.utime(filename, (now, now)) + self.assertEqual(cm.exception.filename, filename) + def get_file_system(self, path): if sys.platform == 'win32': root = os.path.splitdrive(os.path.abspath(path))[0] + '\\' @@ -936,7 +939,6 @@ def get_file_system(self, path): return buf.value # return None if the filesystem is unknown - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (ModuleNotFoundError: No module named '_ctypes')") def test_large_time(self): # Many filesystems are limited to the year 2038. At least, the test # pass with NTFS filesystem. @@ -947,7 +949,6 @@ def test_large_time(self): os.utime(self.fname, (large, large)) self.assertEqual(os.stat(self.fname).st_mtime, large) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (AssertionError: NotImplementedError not raised)") def test_utime_invalid_arguments(self): # seconds and nanoseconds parameters are mutually exclusive with self.assertRaises(ValueError): @@ -1149,7 +1150,6 @@ def test_putenv_unsetenv(self): stdout=subprocess.PIPE, text=True) self.assertEqual(proc.stdout.rstrip(), repr(None)) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (AssertionError: ValueError not raised by putenv)") # On OS X < 10.6, unsetenv() doesn't return a value (bpo-13415). @support.requires_mac_ver(10, 6) def test_putenv_unsetenv_error(self): @@ -1574,7 +1574,7 @@ def test_walk_many_open_files(self): def test_walk_above_recursion_limit(self): depth = 50 os.makedirs(os.path.join(self.walk_path, *(['d'] * depth))) - with set_recursion_limit(depth - 5): + with infinite_recursion(depth - 5): all = list(self.walk(self.walk_path)) sub2_path = self.sub2_tree[0] @@ -1671,6 +1671,9 @@ def test_yields_correct_dir_fd(self): @unittest.skipIf( support.is_emscripten, "Cannot dup stdout on Emscripten" ) + @unittest.skipIf( + support.is_android, "dup return value is unpredictable on Android" + ) def test_fd_leak(self): # Since we're opening a lot of FDs, we must be careful to avoid leaks: # we both check that calling fwalk() a large number of times doesn't @@ -1684,10 +1687,29 @@ def test_fd_leak(self): self.addCleanup(os.close, newfd) self.assertEqual(newfd, minfd) + @unittest.skipIf( + support.is_emscripten, "Cannot dup stdout on Emscripten" + ) + @unittest.skipIf( + support.is_android, "dup return value is unpredictable on Android" + ) + def test_fd_finalization(self): + # Check that close()ing the fwalk() generator closes FDs + def getfd(): + fd = os.dup(1) + os.close(fd) + return fd + for topdown in (False, True): + old_fd = getfd() + it = self.fwalk(os_helper.TESTFN, topdown=topdown) + self.assertEqual(getfd(), old_fd) + next(it) + self.assertGreater(getfd(), old_fd) + it.close() + self.assertEqual(getfd(), old_fd) + # fwalk() keeps file descriptors open test_walk_many_open_files = None - # fwalk() still uses recursion - test_walk_above_recursion_limit = None class BytesWalkTests(WalkTests): @@ -1703,6 +1725,18 @@ def walk(self, top, **kwargs): bdirs[:] = list(map(os.fsencode, dirs)) bfiles[:] = list(map(os.fsencode, files)) + @unittest.expectedFailure # TODO: RUSTPYTHON; WalkTests doesn't have these methods + def test_compare_to_walk(self): + return super().test_compare_to_walk() + + @unittest.expectedFailure # TODO: RUSTPYTHON; WalkTests doesn't have these methods + def test_dir_fd(self): + return super().test_dir_fd() + + @unittest.expectedFailure # TODO: RUSTPYTHON; WalkTests doesn't have these methods + def test_yields_correct_dir_fd(self): + return super().test_yields_correct_dir_fd() + @unittest.skipUnless(hasattr(os, 'fwalk'), "Test needs os.fwalk()") class BytesFwalkTests(FwalkTests): """Tests for os.walk() with bytes.""" @@ -1715,26 +1749,6 @@ def fwalk(self, top='.', *args, **kwargs): bdirs[:] = list(map(os.fsencode, dirs)) bfiles[:] = list(map(os.fsencode, files)) - # TODO: RUSTPYTHON (TypeError: Can't mix strings and bytes in path components) - @unittest.expectedFailure - def test_compare_to_walk(self): - super().test_compare_to_walk() - - # TODO: RUSTPYTHON (TypeError: Can't mix strings and bytes in path components) - @unittest.expectedFailure - def test_dir_fd(self): - super().test_dir_fd() - - # TODO: RUSTPYTHON (TypeError: Can't mix strings and bytes in path components) - @unittest.expectedFailure - def test_yields_correct_dir_fd(self): - super().test_yields_correct_dir_fd() - - # TODO: RUSTPYTHON (TypeError: Can't mix strings and bytes in path components) - @unittest.expectedFailure - def test_walk_bottom_up(self): - super().test_walk_bottom_up() - class MakedirTests(unittest.TestCase): def setUp(self): @@ -1771,7 +1785,6 @@ def test_mode(self): self.assertEqual(os.stat(path).st_mode & 0o777, 0o555) self.assertEqual(os.stat(parent).st_mode & 0o777, 0o775) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON os.umask not implemented yet for all platforms") @unittest.skipIf( support.is_emscripten or support.is_wasi, "Emscripten's/WASI's umask is a stub." @@ -1790,7 +1803,6 @@ def test_exist_ok_existing_directory(self): # Issue #25583: A drive root could raise PermissionError on Windows os.makedirs(os.path.abspath('/'), exist_ok=True) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON os.umask not implemented yet for all platforms") @unittest.skipIf( support.is_emscripten or support.is_wasi, "Emscripten's/WASI's umask is a stub." @@ -1832,6 +1844,19 @@ def test_exist_ok_existing_regular_file(self): self.assertRaises(OSError, os.makedirs, path, exist_ok=True) os.remove(path) + @unittest.skipUnless(os.name == 'nt', "requires Windows") + def test_win32_mkdir_700(self): + base = os_helper.TESTFN + path = os.path.abspath(os.path.join(os_helper.TESTFN, 'dir')) + os.mkdir(path, mode=0o700) + out = subprocess.check_output(["cacls.exe", path, "/s"], encoding="oem") + os.rmdir(path) + out = out.strip().rsplit(" ", 1)[1] + self.assertEqual( + out, + '"D:P(A;OICI;FA;;;SY)(A;OICI;FA;;;BA)(A;OICI;FA;;;OW)"', + ) + def tearDown(self): path = os.path.join(os_helper.TESTFN, 'dir1', 'dir2', 'dir3', 'dir4', 'dir5', 'dir6') @@ -2064,7 +2089,6 @@ def test_urandom_failure(self): """ assert_python_ok('-c', code) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON on Windows (ModuleNotFoundError: No module named 'os')") def test_urandom_fd_closed(self): # Issue #21207: urandom() should reopen its fd to /dev/urandom if # closed. @@ -2079,7 +2103,6 @@ def test_urandom_fd_closed(self): """ rc, out, err = assert_python_ok('-Sc', code) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (ModuleNotFoundError: No module named 'os'") def test_urandom_fd_reopened(self): # Issue #21207: urandom() should detect its fd to /dev/urandom # changed to something else, and reopen it. @@ -2167,7 +2190,6 @@ def test_execv_with_bad_arglist(self): self.assertRaises(ValueError, os.execv, 'notepad', ('',)) self.assertRaises(ValueError, os.execv, 'notepad', ['']) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON os.execve not implemented yet for all platforms") def test_execvpe_with_bad_arglist(self): self.assertRaises(ValueError, os.execvpe, 'notepad', [], None) self.assertRaises(ValueError, os.execvpe, 'notepad', [], {}) @@ -2227,7 +2249,6 @@ def test_internal_execvpe_str(self): if os.name != "nt": self._test_internal_execvpe(bytes) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON os.execve not implemented yet for all platforms") def test_execve_invalid_env(self): args = [sys.executable, '-c', 'pass'] @@ -2249,7 +2270,6 @@ def test_execve_invalid_env(self): with self.assertRaises(ValueError): os.execve(args[0], args, newenv) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON os.execve not implemented yet for all platforms") @unittest.skipUnless(sys.platform == "win32", "Win32-specific test") def test_execve_with_empty_path(self): # bpo-32890: Check GetLastError() misuse @@ -2301,12 +2321,15 @@ def test_chmod(self): class TestInvalidFD(unittest.TestCase): singles = ["fchdir", "dup", "fdatasync", "fstat", "fstatvfs", "fsync", "tcgetpgrp", "ttyname"] + singles_fildes = {"fchdir", "fdatasync", "fsync"} #singles.append("close") #We omit close because it doesn't raise an exception on some platforms def get_single(f): def helper(self): if hasattr(os, f): self.check(getattr(os, f)) + if f in self.singles_fildes: + self.check_bool(getattr(os, f)) return helper for f in singles: locals()["test_"+f] = get_single(f) @@ -2320,9 +2343,17 @@ def check(self, f, *args, **kwargs): self.fail("%r didn't raise an OSError with a bad file descriptor" % f) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (AssertionError: <function fdopen at 0x1caa19ae6c0> didn't raise an OSError with a bad file descriptor)") + def check_bool(self, f, *args, **kwargs): + with warnings.catch_warnings(): + warnings.simplefilter("error", RuntimeWarning) + for fd in False, True: + with self.assertRaises(RuntimeWarning): + f(fd, *args, **kwargs) + + @unittest.expectedFailure # TODO: RUSTPYTHON def test_fdopen(self): self.check(os.fdopen, encoding="utf-8") + self.check_bool(os.fdopen, encoding="utf-8") @unittest.skipUnless(hasattr(os, 'isatty'), 'test needs os.isatty()') def test_isatty(self): @@ -2376,27 +2407,30 @@ def test_fchmod(self): def test_fchown(self): self.check(os.fchown, -1, -1) + @unittest.expectedFailure # TODO: RUSTPYTHON @unittest.skipUnless(hasattr(os, 'fpathconf'), 'test needs os.fpathconf()') @unittest.skipIf( support.is_emscripten or support.is_wasi, "musl libc issue on Emscripten/WASI, bpo-46390" ) def test_fpathconf(self): + self.assertIn("PC_NAME_MAX", os.pathconf_names) self.check(os.pathconf, "PC_NAME_MAX") self.check(os.fpathconf, "PC_NAME_MAX") + self.check_bool(os.pathconf, "PC_NAME_MAX") + self.check_bool(os.fpathconf, "PC_NAME_MAX") - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (AssertionError: <builtin_function_or_method object at 0x1f330cd8e60> didn't raise an OSError with a bad file descriptor)") + @unittest.expectedFailure # TODO: RUSTPYTHON @unittest.skipUnless(hasattr(os, 'ftruncate'), 'test needs os.ftruncate()') def test_ftruncate(self): self.check(os.truncate, 0) self.check(os.ftruncate, 0) + self.check_bool(os.truncate, 0) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (OSError: [Errno 18] There are no more files.)") @unittest.skipUnless(hasattr(os, 'lseek'), 'test needs os.lseek()') def test_lseek(self): self.check(os.lseek, 0, 0) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (OSError: [Errno 18] There are no more files.)") @unittest.skipUnless(hasattr(os, 'read'), 'test needs os.read()') def test_read(self): self.check(os.read, 1) @@ -2410,7 +2444,6 @@ def test_readv(self): def test_tcsetpgrpt(self): self.check(os.tcsetpgrp, 0) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (OSError: [Errno 18] There are no more files.)") @unittest.skipUnless(hasattr(os, 'write'), 'test needs os.write()') def test_write(self): self.check(os.write, b" ") @@ -2419,7 +2452,6 @@ def test_write(self): def test_writev(self): self.check(os.writev, [b'abc']) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON os.get_inheritable not implemented yet for all platforms") @support.requires_subprocess() def test_inheritable(self): self.check(os.get_inheritable) @@ -2431,6 +2463,14 @@ def test_blocking(self): self.check(os.get_blocking) self.check(os.set_blocking, True) + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_fchdir(self): + return super().test_fchdir() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_fsync(self): + return super().test_fsync() + @unittest.skipUnless(hasattr(os, 'link'), 'requires os.link') class LinkTests(unittest.TestCase): @@ -2590,8 +2630,10 @@ def test_listdir(self): # test listdir without arguments current_directory = os.getcwd() try: - os.chdir(os.sep) - self.assertEqual(set(os.listdir()), set(os.listdir(os.sep))) + # The root directory is not readable on Android, so use a directory + # we created ourselves. + os.chdir(self.dir) + self.assertEqual(set(os.listdir()), expected) finally: os.chdir(current_directory) @@ -2669,12 +2711,10 @@ def _kill(self, sig): os.kill(proc.pid, sig) self.assertEqual(proc.wait(), sig) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (ModuleNotFoundError: No module named '_ctypes')") def test_kill_sigterm(self): # SIGTERM doesn't mean anything special, but make sure it works self._kill(signal.SIGTERM) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (ModuleNotFoundError: No module named '_ctypes')") def test_kill_int(self): # os.kill on Windows can take an int which gets set as the exit code self._kill(100) @@ -2733,8 +2773,7 @@ def test_CTRL_C_EVENT(self): self._kill_with_event(signal.CTRL_C_EVENT, "CTRL_C_EVENT") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON @support.requires_subprocess() def test_CTRL_BREAK_EVENT(self): self._kill_with_event(signal.CTRL_BREAK_EVENT, "CTRL_BREAK_EVENT") @@ -3088,7 +3127,6 @@ def tearDown(self): if os.path.lexists(self.junction): os.unlink(self.junction) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (AttributeError: module '_winapi' has no attribute 'CreateJunction')") def test_create_junction(self): _winapi.CreateJunction(self.junction_target, self.junction) self.assertTrue(os.path.lexists(self.junction)) @@ -3102,7 +3140,6 @@ def test_create_junction(self): self.assertEqual(os.path.normcase("\\\\?\\" + self.junction_target), os.path.normcase(os.readlink(self.junction))) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (AttributeError: module '_winapi' has no attribute 'CreateJunction')") def test_unlink_removes_junction(self): _winapi.CreateJunction(self.junction_target, self.junction) self.assertTrue(os.path.exists(self.junction)) @@ -3116,7 +3153,8 @@ class Win32NtTests(unittest.TestCase): def test_getfinalpathname_handles(self): nt = import_helper.import_module('nt') ctypes = import_helper.import_module('ctypes') - import ctypes.wintypes + # Ruff false positive -- it thinks we're redefining `ctypes` here + import ctypes.wintypes # noqa: F811 kernel = ctypes.WinDLL('Kernel32.dll', use_last_error=True) kernel.GetCurrentProcess.restype = ctypes.wintypes.HANDLE @@ -3161,7 +3199,6 @@ def test_getfinalpathname_handles(self): self.assertEqual(0, handle_delta) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON os.stat (PermissionError: [Errno 5] Access is denied.)") @support.requires_subprocess() def test_stat_unlink_race(self): # bpo-46785: the implementation of os.stat() falls back to reading @@ -3229,9 +3266,8 @@ def test_stat_inaccessible_file(self): self.skipTest("Unable to create inaccessible file") def cleanup(): - # Give delete permission. We are the file owner, so we can do this - # even though we removed all permissions earlier. - subprocess.check_output([ICACLS, filename, "/grant", "Everyone:(D)"], + # Give delete permission to the owner (us) + subprocess.check_output([ICACLS, filename, "/grant", "*WD:(D)"], stderr=subprocess.STDOUT) os.unlink(filename) @@ -3372,7 +3408,6 @@ def test_waitstatus_to_exitcode(self): with self.assertRaises(TypeError): os.waitstatus_to_exitcode(0.0) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON os.spawnv not implemented yet for all platforms") @unittest.skipUnless(sys.platform == 'win32', 'win32-specific test') def test_waitpid_windows(self): # bpo-40138: test os.waitpid() and os.waitstatus_to_exitcode() @@ -3381,7 +3416,6 @@ def test_waitpid_windows(self): code = f'import _winapi; _winapi.ExitProcess({STATUS_CONTROL_C_EXIT})' self.check_waitpid(code, exitcode=STATUS_CONTROL_C_EXIT) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (OverflowError: Python int too large to convert to Rust i32)") @unittest.skipUnless(sys.platform == 'win32', 'win32-specific test') def test_waitstatus_to_exitcode_windows(self): max_exitcode = 2 ** 32 - 1 @@ -3454,6 +3488,7 @@ def test_spawnl(self): self.assertEqual(exitcode, self.exitcode) @requires_os_func('spawnle') + @unittest.skipIf(sys.platform == 'win32', "TODO: RUSTPYTHON; fix spawnve on Windows") def test_spawnle(self): program, args = self.create_args(with_env=True) exitcode = os.spawnle(os.P_WAIT, program, *args, self.env) @@ -3482,6 +3517,7 @@ def test_spawnv(self): self.assertEqual(exitcode, self.exitcode) @requires_os_func('spawnve') + @unittest.skipIf(sys.platform == 'win32', "TODO: RUSTPYTHON; fix spawnve on Windows") def test_spawnve(self): program, args = self.create_args(with_env=True) exitcode = os.spawnve(os.P_WAIT, program, args, self.env) @@ -3505,8 +3541,7 @@ def test_nowait(self): pid = os.spawnv(os.P_NOWAIT, program, args) support.wait_process(pid, exitcode=self.exitcode) - # TODO: RUSTPYTHON fix spawnv bytes - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; fix spawnv bytes @requires_os_func('spawnve') def test_spawnve_bytes(self): # Test bytes handling in parse_arglist and parse_envlist (#28114) @@ -3591,6 +3626,7 @@ def _test_invalid_env(self, spawn): self.assertEqual(exitcode, 0) @requires_os_func('spawnve') + @unittest.skipIf(sys.platform == 'win32', "TODO: RUSTPYTHON; fix spawnve on Windows") def test_spawnve_invalid_env(self): self._test_invalid_env(os.spawnve) @@ -3637,9 +3673,8 @@ def test_set_get_priority(self): class TestSendfile(unittest.IsolatedAsyncioTestCase): DATA = b"12345abcde" * 16 * 1024 # 160 KiB - SUPPORT_HEADERS_TRAILERS = not sys.platform.startswith("linux") and \ - not sys.platform.startswith("solaris") and \ - not sys.platform.startswith("sunos") + SUPPORT_HEADERS_TRAILERS = ( + not sys.platform.startswith(("linux", "android", "solaris", "sunos"))) requires_headers_trailers = unittest.skipUnless(SUPPORT_HEADERS_TRAILERS, 'requires headers and trailers support') requires_32b = unittest.skipUnless(sys.maxsize < 2**32, @@ -3912,10 +3947,10 @@ def _check_xattrs_str(self, s, getxattr, setxattr, removexattr, listxattr, **kwa xattr.remove("user.test") self.assertEqual(set(listxattr(fn)), xattr) self.assertEqual(getxattr(fn, s("user.test2"), **kwargs), b"foo") - setxattr(fn, s("user.test"), b"a"*1024, **kwargs) - self.assertEqual(getxattr(fn, s("user.test"), **kwargs), b"a"*1024) + setxattr(fn, s("user.test"), b"a"*256, **kwargs) + self.assertEqual(getxattr(fn, s("user.test"), **kwargs), b"a"*256) removexattr(fn, s("user.test"), **kwargs) - many = sorted("user.test{}".format(i) for i in range(100)) + many = sorted("user.test{}".format(i) for i in range(32)) for thing in many: setxattr(fn, thing, b"x", **kwargs) self.assertEqual(set(listxattr(fn)), set(init_xattr) | set(many)) @@ -3962,7 +3997,12 @@ def test_does_not_crash(self): try: size = os.get_terminal_size() except OSError as e: - if sys.platform == "win32" or e.errno in (errno.EINVAL, errno.ENOTTY): + known_errnos = [errno.EINVAL, errno.ENOTTY] + if sys.platform == "android": + # The Android testbed redirects the native stdout to a pipe, + # which returns a different error code. + known_errnos.append(errno.EACCES) + if sys.platform == "win32" or e.errno in known_errnos: # Under win32 a generic OSError can be thrown if the # handle cannot be retrieved self.skipTest("failed to query terminal size") @@ -3971,6 +4011,7 @@ def test_does_not_crash(self): self.assertGreaterEqual(size.columns, 0) self.assertGreaterEqual(size.lines, 0) + @support.requires_subprocess() def test_stty_match(self): """Check if stty returns the same results @@ -4112,6 +4153,333 @@ def test_eventfd_select(self): self.assertEqual((rfd, wfd, xfd), ([fd], [], [])) os.eventfd_read(fd) +@unittest.skipUnless(hasattr(os, 'timerfd_create'), 'requires os.timerfd_create') +@unittest.skipIf(sys.platform == "android", "gh-124873: Test is flaky on Android") +@support.requires_linux_version(2, 6, 30) +class TimerfdTests(unittest.TestCase): + # gh-126112: Use 10 ms to tolerate slow buildbots + CLOCK_RES_PLACES = 2 # 10 ms + CLOCK_RES = 10 ** -CLOCK_RES_PLACES + CLOCK_RES_NS = 10 ** (9 - CLOCK_RES_PLACES) + + def timerfd_create(self, *args, **kwargs): + fd = os.timerfd_create(*args, **kwargs) + self.assertGreaterEqual(fd, 0) + self.assertFalse(os.get_inheritable(fd)) + self.addCleanup(os.close, fd) + return fd + + def read_count_signaled(self, fd): + # read 8 bytes + data = os.read(fd, 8) + return int.from_bytes(data, byteorder=sys.byteorder) + + def test_timerfd_initval(self): + fd = self.timerfd_create(time.CLOCK_REALTIME) + + initial_expiration = 0.25 + interval = 0.125 + + # 1st call + next_expiration, interval2 = os.timerfd_settime(fd, initial=initial_expiration, interval=interval) + self.assertAlmostEqual(interval2, 0.0, places=self.CLOCK_RES_PLACES) + self.assertAlmostEqual(next_expiration, 0.0, places=self.CLOCK_RES_PLACES) + + # 2nd call + next_expiration, interval2 = os.timerfd_settime(fd, initial=initial_expiration, interval=interval) + self.assertAlmostEqual(interval2, interval, places=self.CLOCK_RES_PLACES) + self.assertAlmostEqual(next_expiration, initial_expiration, places=self.CLOCK_RES_PLACES) + + # timerfd_gettime + next_expiration, interval2 = os.timerfd_gettime(fd) + self.assertAlmostEqual(interval2, interval, places=self.CLOCK_RES_PLACES) + self.assertAlmostEqual(next_expiration, initial_expiration, places=self.CLOCK_RES_PLACES) + + def test_timerfd_non_blocking(self): + fd = self.timerfd_create(time.CLOCK_REALTIME, flags=os.TFD_NONBLOCK) + + # 0.1 second later + initial_expiration = 0.1 + os.timerfd_settime(fd, initial=initial_expiration, interval=0) + + # read() raises OSError with errno is EAGAIN for non-blocking timer. + with self.assertRaises(OSError) as ctx: + self.read_count_signaled(fd) + self.assertEqual(ctx.exception.errno, errno.EAGAIN) + + # Wait more than 0.1 seconds + time.sleep(initial_expiration + 0.1) + + # confirm if timerfd is readable and read() returns 1 as bytes. + self.assertEqual(self.read_count_signaled(fd), 1) + + @unittest.skipIf(sys.platform.startswith('netbsd'), + "gh-131263: Skip on NetBSD due to system freeze " + "with negative timer values") + def test_timerfd_negative(self): + one_sec_in_nsec = 10**9 + fd = self.timerfd_create(time.CLOCK_REALTIME) + + test_flags = [0, os.TFD_TIMER_ABSTIME] + if hasattr(os, 'TFD_TIMER_CANCEL_ON_SET'): + test_flags.append(os.TFD_TIMER_ABSTIME | os.TFD_TIMER_CANCEL_ON_SET) + + # Any of 'initial' and 'interval' is negative value. + for initial, interval in ( (-1, 0), (1, -1), (-1, -1), (-0.1, 0), (1, -0.1), (-0.1, -0.1)): + for flags in test_flags: + with self.subTest(flags=flags, initial=initial, interval=interval): + with self.assertRaises(OSError) as context: + os.timerfd_settime(fd, flags=flags, initial=initial, interval=interval) + self.assertEqual(context.exception.errno, errno.EINVAL) + + with self.assertRaises(OSError) as context: + initial_ns = int( one_sec_in_nsec * initial ) + interval_ns = int( one_sec_in_nsec * interval ) + os.timerfd_settime_ns(fd, flags=flags, initial=initial_ns, interval=interval_ns) + self.assertEqual(context.exception.errno, errno.EINVAL) + + def test_timerfd_interval(self): + fd = self.timerfd_create(time.CLOCK_REALTIME) + + # 1 second + initial_expiration = 1 + # 0.5 second + interval = 0.5 + + os.timerfd_settime(fd, initial=initial_expiration, interval=interval) + + # timerfd_gettime + next_expiration, interval2 = os.timerfd_gettime(fd) + self.assertAlmostEqual(interval2, interval, places=self.CLOCK_RES_PLACES) + self.assertAlmostEqual(next_expiration, initial_expiration, places=self.CLOCK_RES_PLACES) + + count = 3 + t = time.perf_counter() + for _ in range(count): + self.assertEqual(self.read_count_signaled(fd), 1) + t = time.perf_counter() - t + + total_time = initial_expiration + interval * (count - 1) + self.assertGreater(t, total_time - self.CLOCK_RES) + + # wait 3.5 time of interval + time.sleep( (count+0.5) * interval) + self.assertEqual(self.read_count_signaled(fd), count) + + def test_timerfd_TFD_TIMER_ABSTIME(self): + fd = self.timerfd_create(time.CLOCK_REALTIME) + + now = time.clock_gettime(time.CLOCK_REALTIME) + + # 1 second later from now. + offset = 1 + initial_expiration = now + offset + # not interval timer + interval = 0 + + os.timerfd_settime(fd, flags=os.TFD_TIMER_ABSTIME, initial=initial_expiration, interval=interval) + + # timerfd_gettime + # Note: timerfd_gettime returns relative values even if TFD_TIMER_ABSTIME is specified. + next_expiration, interval2 = os.timerfd_gettime(fd) + self.assertAlmostEqual(interval2, interval, places=self.CLOCK_RES_PLACES) + self.assertAlmostEqual(next_expiration, offset, places=self.CLOCK_RES_PLACES) + + t = time.perf_counter() + count_signaled = self.read_count_signaled(fd) + t = time.perf_counter() - t + self.assertEqual(count_signaled, 1) + + self.assertGreater(t, offset - self.CLOCK_RES) + + def test_timerfd_select(self): + fd = self.timerfd_create(time.CLOCK_REALTIME, flags=os.TFD_NONBLOCK) + + rfd, wfd, xfd = select.select([fd], [fd], [fd], 0) + self.assertEqual((rfd, wfd, xfd), ([], [], [])) + + # 0.25 second + initial_expiration = 0.25 + # every 0.125 second + interval = 0.125 + + os.timerfd_settime(fd, initial=initial_expiration, interval=interval) + + count = 3 + t = time.perf_counter() + for _ in range(count): + rfd, wfd, xfd = select.select([fd], [fd], [fd], initial_expiration + interval) + self.assertEqual((rfd, wfd, xfd), ([fd], [], [])) + self.assertEqual(self.read_count_signaled(fd), 1) + t = time.perf_counter() - t + + total_time = initial_expiration + interval * (count - 1) + self.assertGreater(t, total_time - self.CLOCK_RES) + + def check_timerfd_poll(self, nanoseconds): + fd = self.timerfd_create(time.CLOCK_REALTIME, flags=os.TFD_NONBLOCK) + + selector = selectors.DefaultSelector() + selector.register(fd, selectors.EVENT_READ) + self.addCleanup(selector.close) + + sec_to_nsec = 10 ** 9 + # 0.25 second + initial_expiration_ns = sec_to_nsec // 4 + # every 0.125 second + interval_ns = sec_to_nsec // 8 + + if nanoseconds: + os.timerfd_settime_ns(fd, + initial=initial_expiration_ns, + interval=interval_ns) + else: + os.timerfd_settime(fd, + initial=initial_expiration_ns / sec_to_nsec, + interval=interval_ns / sec_to_nsec) + + count = 3 + if nanoseconds: + t = time.perf_counter_ns() + else: + t = time.perf_counter() + for i in range(count): + timeout_margin_ns = interval_ns + if i == 0: + timeout_ns = initial_expiration_ns + interval_ns + timeout_margin_ns + else: + timeout_ns = interval_ns + timeout_margin_ns + + ready = selector.select(timeout_ns / sec_to_nsec) + self.assertEqual(len(ready), 1, ready) + event = ready[0][1] + self.assertEqual(event, selectors.EVENT_READ) + + self.assertEqual(self.read_count_signaled(fd), 1) + + total_time = initial_expiration_ns + interval_ns * (count - 1) + if nanoseconds: + dt = time.perf_counter_ns() - t + self.assertGreater(dt, total_time - self.CLOCK_RES_NS) + else: + dt = time.perf_counter() - t + self.assertGreater(dt, total_time / sec_to_nsec - self.CLOCK_RES) + selector.unregister(fd) + + def test_timerfd_poll(self): + self.check_timerfd_poll(False) + + def test_timerfd_ns_poll(self): + self.check_timerfd_poll(True) + + def test_timerfd_ns_initval(self): + one_sec_in_nsec = 10**9 + limit_error = one_sec_in_nsec // 10**3 + fd = self.timerfd_create(time.CLOCK_REALTIME) + + # 1st call + initial_expiration_ns = 0 + interval_ns = one_sec_in_nsec // 1000 + next_expiration_ns, interval_ns2 = os.timerfd_settime_ns(fd, initial=initial_expiration_ns, interval=interval_ns) + self.assertEqual(interval_ns2, 0) + self.assertEqual(next_expiration_ns, 0) + + # 2nd call + next_expiration_ns, interval_ns2 = os.timerfd_settime_ns(fd, initial=initial_expiration_ns, interval=interval_ns) + self.assertEqual(interval_ns2, interval_ns) + self.assertEqual(next_expiration_ns, initial_expiration_ns) + + # timerfd_gettime + next_expiration_ns, interval_ns2 = os.timerfd_gettime_ns(fd) + self.assertEqual(interval_ns2, interval_ns) + self.assertLessEqual(next_expiration_ns, initial_expiration_ns) + + self.assertAlmostEqual(next_expiration_ns, initial_expiration_ns, delta=limit_error) + + def test_timerfd_ns_interval(self): + one_sec_in_nsec = 10**9 + limit_error = one_sec_in_nsec // 10**3 + fd = self.timerfd_create(time.CLOCK_REALTIME) + + # 1 second + initial_expiration_ns = one_sec_in_nsec + # every 0.5 second + interval_ns = one_sec_in_nsec // 2 + + os.timerfd_settime_ns(fd, initial=initial_expiration_ns, interval=interval_ns) + + # timerfd_gettime + next_expiration_ns, interval_ns2 = os.timerfd_gettime_ns(fd) + self.assertEqual(interval_ns2, interval_ns) + self.assertLessEqual(next_expiration_ns, initial_expiration_ns) + + count = 3 + t = time.perf_counter_ns() + for _ in range(count): + self.assertEqual(self.read_count_signaled(fd), 1) + t = time.perf_counter_ns() - t + + total_time_ns = initial_expiration_ns + interval_ns * (count - 1) + self.assertGreater(t, total_time_ns - self.CLOCK_RES_NS) + + # wait 3.5 time of interval + time.sleep( (count+0.5) * interval_ns / one_sec_in_nsec) + self.assertEqual(self.read_count_signaled(fd), count) + + + def test_timerfd_ns_TFD_TIMER_ABSTIME(self): + one_sec_in_nsec = 10**9 + limit_error = one_sec_in_nsec // 10**3 + fd = self.timerfd_create(time.CLOCK_REALTIME) + + now_ns = time.clock_gettime_ns(time.CLOCK_REALTIME) + + # 1 second later from now. + offset_ns = one_sec_in_nsec + initial_expiration_ns = now_ns + offset_ns + # not interval timer + interval_ns = 0 + + os.timerfd_settime_ns(fd, flags=os.TFD_TIMER_ABSTIME, initial=initial_expiration_ns, interval=interval_ns) + + # timerfd_gettime + # Note: timerfd_gettime returns relative values even if TFD_TIMER_ABSTIME is specified. + next_expiration_ns, interval_ns2 = os.timerfd_gettime_ns(fd) + self.assertLess(abs(interval_ns2 - interval_ns), limit_error) + self.assertLess(abs(next_expiration_ns - offset_ns), limit_error) + + t = time.perf_counter_ns() + count_signaled = self.read_count_signaled(fd) + t = time.perf_counter_ns() - t + self.assertEqual(count_signaled, 1) + + self.assertGreater(t, offset_ns - self.CLOCK_RES_NS) + + def test_timerfd_ns_select(self): + one_sec_in_nsec = 10**9 + + fd = self.timerfd_create(time.CLOCK_REALTIME, flags=os.TFD_NONBLOCK) + + rfd, wfd, xfd = select.select([fd], [fd], [fd], 0) + self.assertEqual((rfd, wfd, xfd), ([], [], [])) + + # 0.25 second + initial_expiration_ns = one_sec_in_nsec // 4 + # every 0.125 second + interval_ns = one_sec_in_nsec // 8 + + os.timerfd_settime_ns(fd, initial=initial_expiration_ns, interval=interval_ns) + + count = 3 + t = time.perf_counter_ns() + for _ in range(count): + rfd, wfd, xfd = select.select([fd], [fd], [fd], (initial_expiration_ns + interval_ns) / 1e9 ) + self.assertEqual((rfd, wfd, xfd), ([fd], [], [])) + self.assertEqual(self.read_count_signaled(fd), 1) + t = time.perf_counter_ns() - t + + total_time_ns = initial_expiration_ns + interval_ns * (count - 1) + self.assertGreater(t, total_time_ns - self.CLOCK_RES_NS) class OSErrorTests(unittest.TestCase): def setUp(self): @@ -4134,8 +4502,6 @@ class Str(str): self.filenames = self.bytes_filenames + self.unicode_filenames - # TODO: RUSTPYTHON (AssertionError: b'@test_22106_tmp\xe7w\xf0' is not b'@test_22106_tmp\xe7w\xf0' : <built-in function chdir>) - @unittest.expectedFailure def test_oserror_filename(self): funcs = [ (self.filenames, os.chdir,), @@ -4188,19 +4554,46 @@ def test_oserror_filename(self): self.fail(f"No exception thrown by {func}") class CPUCountTests(unittest.TestCase): + def check_cpu_count(self, cpus): + if cpus is None: + self.skipTest("Could not determine the number of CPUs") + + self.assertIsInstance(cpus, int) + self.assertGreater(cpus, 0) + def test_cpu_count(self): cpus = os.cpu_count() - if cpus is not None: - self.assertIsInstance(cpus, int) - self.assertGreater(cpus, 0) - else: + self.check_cpu_count(cpus) + + def test_process_cpu_count(self): + cpus = os.process_cpu_count() + self.assertLessEqual(cpus, os.cpu_count()) + self.check_cpu_count(cpus) + + @unittest.skipUnless(hasattr(os, 'sched_setaffinity'), + "don't have sched affinity support") + def test_process_cpu_count_affinity(self): + affinity1 = os.process_cpu_count() + if affinity1 is None: self.skipTest("Could not determine the number of CPUs") + # Disable one CPU + mask = os.sched_getaffinity(0) + if len(mask) <= 1: + self.skipTest(f"sched_getaffinity() returns less than " + f"2 CPUs: {sorted(mask)}") + self.addCleanup(os.sched_setaffinity, 0, list(mask)) + mask.pop() + os.sched_setaffinity(0, mask) + + # test process_cpu_count() + affinity2 = os.process_cpu_count() + self.assertEqual(affinity2, affinity1 - 1) + # FD inheritance check is only useful for systems with process support. @support.requires_subprocess() class FDInheritanceTests(unittest.TestCase): - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON os.get_inheritable not implemented yet for all platforms") def test_get_set_inheritable(self): fd = os.open(__file__, os.O_RDONLY) self.addCleanup(os.close, fd) @@ -4245,7 +4638,6 @@ def test_get_set_inheritable_o_path(self): os.set_inheritable(fd, False) self.assertEqual(os.get_inheritable(fd), False) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON os.get_inheritable not implemented yet for all platforms") def test_get_set_inheritable_badf(self): fd = os_helper.make_bad_fd() @@ -4261,7 +4653,6 @@ def test_get_set_inheritable_badf(self): os.set_inheritable(fd, False) self.assertEqual(ctx.exception.errno, errno.EBADF) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON os.get_inheritable not implemented yet for all platforms") def test_open(self): fd = os.open(__file__, os.O_RDONLY) self.addCleanup(os.close, fd) @@ -4275,7 +4666,6 @@ def test_pipe(self): self.assertEqual(os.get_inheritable(rfd), False) self.assertEqual(os.get_inheritable(wfd), False) - @unittest.skipIf(sys.platform == 'win32', "TODO: RUSTPYTHON; os.dup on windows") def test_dup(self): fd1 = os.open(__file__, os.O_RDONLY) self.addCleanup(os.close, fd1) @@ -4284,13 +4674,11 @@ def test_dup(self): self.addCleanup(os.close, fd2) self.assertEqual(os.get_inheritable(fd2), False) - @unittest.skipIf(sys.platform == 'win32', "TODO: RUSTPYTHON; os.dup on windows") def test_dup_standard_stream(self): fd = os.dup(1) self.addCleanup(os.close, fd) self.assertGreater(fd, 0) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON os.dup not implemented yet for all platforms") @unittest.skipUnless(sys.platform == 'win32', 'win32-specific test') def test_dup_nul(self): # os.dup() was creating inheritable fds for character files. @@ -4317,13 +4705,105 @@ def test_dup2(self): self.assertEqual(os.dup2(fd, fd3, inheritable=False), fd3) self.assertFalse(os.get_inheritable(fd3)) - @unittest.skipUnless(hasattr(os, 'openpty'), "need os.openpty()") +@unittest.skipUnless(hasattr(os, 'openpty'), "need os.openpty()") +class PseudoterminalTests(unittest.TestCase): + def open_pty(self): + """Open a pty fd-pair, and schedule cleanup for it""" + main_fd, second_fd = os.openpty() + self.addCleanup(os.close, main_fd) + self.addCleanup(os.close, second_fd) + return main_fd, second_fd + def test_openpty(self): - master_fd, slave_fd = os.openpty() - self.addCleanup(os.close, master_fd) - self.addCleanup(os.close, slave_fd) - self.assertEqual(os.get_inheritable(master_fd), False) - self.assertEqual(os.get_inheritable(slave_fd), False) + main_fd, second_fd = self.open_pty() + self.assertEqual(os.get_inheritable(main_fd), False) + self.assertEqual(os.get_inheritable(second_fd), False) + + @unittest.skipUnless(hasattr(os, 'ptsname'), "need os.ptsname()") + @unittest.skipUnless(hasattr(os, 'O_RDWR'), "need os.O_RDWR") + @unittest.skipUnless(hasattr(os, 'O_NOCTTY'), "need os.O_NOCTTY") + def test_open_via_ptsname(self): + main_fd, second_fd = self.open_pty() + second_path = os.ptsname(main_fd) + reopened_second_fd = os.open(second_path, os.O_RDWR|os.O_NOCTTY) + self.addCleanup(os.close, reopened_second_fd) + os.write(reopened_second_fd, b'foo') + self.assertEqual(os.read(main_fd, 3), b'foo') + + @unittest.skipUnless(hasattr(os, 'posix_openpt'), "need os.posix_openpt()") + @unittest.skipUnless(hasattr(os, 'grantpt'), "need os.grantpt()") + @unittest.skipUnless(hasattr(os, 'unlockpt'), "need os.unlockpt()") + @unittest.skipUnless(hasattr(os, 'ptsname'), "need os.ptsname()") + @unittest.skipUnless(hasattr(os, 'O_RDWR'), "need os.O_RDWR") + @unittest.skipUnless(hasattr(os, 'O_NOCTTY'), "need os.O_NOCTTY") + def test_posix_pty_functions(self): + mother_fd = os.posix_openpt(os.O_RDWR|os.O_NOCTTY) + self.addCleanup(os.close, mother_fd) + os.grantpt(mother_fd) + os.unlockpt(mother_fd) + son_path = os.ptsname(mother_fd) + son_fd = os.open(son_path, os.O_RDWR|os.O_NOCTTY) + self.addCleanup(os.close, son_fd) + self.assertEqual(os.ptsname(mother_fd), os.ttyname(son_fd)) + + @unittest.skipUnless(hasattr(os, 'spawnl'), "need os.spawnl()") + @support.requires_subprocess() + def test_pipe_spawnl(self): + # gh-77046: On Windows, os.pipe() file descriptors must be created with + # _O_NOINHERIT to make them non-inheritable. UCRT has no public API to + # get (_osfile(fd) & _O_NOINHERIT), so use a functional test. + # + # Make sure that fd is not inherited by a child process created by + # os.spawnl(): get_osfhandle() and dup() must fail with EBADF. + + fd, fd2 = os.pipe() + self.addCleanup(os.close, fd) + self.addCleanup(os.close, fd2) + + code = textwrap.dedent(f""" + import errno + import os + import test.support + try: + import msvcrt + except ImportError: + msvcrt = None + + fd = {fd} + + with test.support.SuppressCrashReport(): + if msvcrt is not None: + try: + handle = msvcrt.get_osfhandle(fd) + except OSError as exc: + if exc.errno != errno.EBADF: + raise + # get_osfhandle(fd) failed with EBADF as expected + else: + raise Exception("get_osfhandle() must fail") + + try: + fd3 = os.dup(fd) + except OSError as exc: + if exc.errno != errno.EBADF: + raise + # os.dup(fd) failed with EBADF as expected + else: + os.close(fd3) + raise Exception("dup must fail") + """) + + filename = os_helper.TESTFN + self.addCleanup(os_helper.unlink, os_helper.TESTFN) + with open(filename, "w") as fp: + print(code, file=fp, end="") + + executable = sys.executable + cmd = [executable, filename] + if os.name == "nt" and " " in cmd[0]: + cmd[0] = f'"{cmd[0]}"' + exitcode = os.spawnl(os.P_WAIT, executable, *cmd) + self.assertEqual(exitcode, 0) class PathTConverterTests(unittest.TestCase): @@ -4338,8 +4818,6 @@ class PathTConverterTests(unittest.TestCase): ('open', False, (os.O_RDONLY,), getattr(os, 'close', None)), ] - # TODO: RUSTPYTHON (AssertionError: TypeError not raised) - @unittest.expectedFailure def test_path_t_converter(self): str_filename = os_helper.TESTFN if os.name == 'nt': @@ -4424,13 +4902,9 @@ def setUp(self): self.addCleanup(os_helper.rmtree, self.path) os.mkdir(self.path) - # TODO: RUSTPYTHON (AssertionError: TypeError not raised by DirEntry) - @unittest.expectedFailure def test_uninstantiable(self): self.assertRaises(TypeError, os.DirEntry) - # TODO: RUSTPYTHON (pickle.PicklingError: Can't pickle <class '_os.DirEntry'>: it's not found as _os.DirEntry) - @unittest.expectedFailure def test_unpickable(self): filename = create_file(os.path.join(self.path, "file.txt"), b'python') entry = [entry for entry in os.scandir(self.path)].pop() @@ -4475,8 +4949,6 @@ def assert_stat_equal(self, stat1, stat2, skip_fields): else: self.assertEqual(stat1, stat2) - # TODO: RUSTPYTHON (AssertionError: TypeError not raised by ScandirIter) - @unittest.expectedFailure def test_uninstantiable(self): scandir_iter = os.scandir(self.path) self.assertRaises(TypeError, type(scandir_iter)) @@ -4519,9 +4991,9 @@ def check_entry(self, entry, name, is_dir, is_file, is_symlink): entry_lstat, os.name == 'nt') - @unittest.skipIf(sys.platform == "linux", "TODO: RUSTPYTHON, flaky test") + @unittest.skipIf(sys.platform == 'linux', 'TODO: RUSTPYTHON; flaky test') def test_attributes(self): - link = hasattr(os, 'link') + link = os_helper.can_hardlink() symlink = os_helper.can_symlink() dirname = os.path.join(self.path, "dir") @@ -4619,7 +5091,6 @@ def test_fspath_protocol_bytes(self): self.assertEqual(fspath, os.path.join(os.fsencode(self.path),bytes_filename)) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON entry.is_dir() is False") def test_removed_dir(self): path = os.path.join(self.path, 'dir') @@ -4642,7 +5113,6 @@ def test_removed_dir(self): self.assertRaises(FileNotFoundError, entry.stat) self.assertRaises(FileNotFoundError, entry.stat, follow_symlinks=False) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON entry.is_file() is False") def test_removed_file(self): entry = self.create_file_entry() os.unlink(entry.path) @@ -4703,8 +5173,7 @@ def test_bytes_like(self): with self.assertRaises(TypeError): os.scandir(path_bytes) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON @unittest.skipUnless(os.listdir in os.supports_fd, 'fd support for listdir required for this test.') def test_fd(self): @@ -4731,7 +5200,6 @@ def test_fd(self): st = os.stat(entry.name, dir_fd=fd, follow_symlinks=False) self.assertEqual(entry.stat(follow_symlinks=False), st) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON (AssertionError: FileNotFoundError not raised by scandir)") @unittest.skipIf(support.is_wasi, "WASI maps '' to cwd") def test_empty_path(self): self.assertRaises(FileNotFoundError, os.scandir, '') @@ -4787,8 +5255,7 @@ def test_context_manager_exception(self): with self.check_no_resource_warning(): del iterator - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_resource_warning(self): self.create_file("file.txt") self.create_file("file2.txt") @@ -4861,6 +5328,51 @@ class A(os.PathLike): def test_pathlike_class_getitem(self): self.assertIsInstance(os.PathLike[bytes], types.GenericAlias) + def test_pathlike_subclass_slots(self): + class A(os.PathLike): + __slots__ = () + def __fspath__(self): + return '' + self.assertFalse(hasattr(A(), '__dict__')) + + def test_fspath_set_to_None(self): + class Foo: + __fspath__ = None + + class Bar: + def __fspath__(self): + return 'bar' + + class Baz(Bar): + __fspath__ = None + + good_error_msg = ( + r"expected str, bytes or os.PathLike object, not {}".format + ) + + with self.assertRaisesRegex(TypeError, good_error_msg("Foo")): + self.fspath(Foo()) + + self.assertEqual(self.fspath(Bar()), 'bar') + + with self.assertRaisesRegex(TypeError, good_error_msg("Baz")): + self.fspath(Baz()) + + with self.assertRaisesRegex(TypeError, good_error_msg("Foo")): + open(Foo()) + + with self.assertRaisesRegex(TypeError, good_error_msg("Baz")): + open(Baz()) + + other_good_error_msg = ( + r"should be string, bytes or os.PathLike, not {}".format + ) + + with self.assertRaisesRegex(TypeError, other_good_error_msg("Foo")): + os.rename(Foo(), "foooo") + + with self.assertRaisesRegex(TypeError, other_good_error_msg("Baz")): + os.rename(Baz(), "bazzz") class TimesTests(unittest.TestCase): def test_times(self): @@ -4891,12 +5403,14 @@ def test_fork(self): support.wait_process(pid, exitcode=0) """ assert_python_ok("-c", code) - assert_python_ok("-c", code, PYTHONMALLOC="malloc_debug") + if support.Py_GIL_DISABLED: + assert_python_ok("-c", code, PYTHONMALLOC="mimalloc_debug") + else: + assert_python_ok("-c", code, PYTHONMALLOC="malloc_debug") - # TODO: RUSTPYTHON; requires _testcapi - @unittest.expectedFailure - @unittest.skipUnless(sys.platform in ("linux", "darwin"), + @unittest.skipUnless(sys.platform in ("linux", "android", "darwin"), "Only Linux and macOS detect this today.") + @unittest.skipIf(_testcapi is None, "requires _testcapi") def test_fork_warns_when_non_python_thread_exists(self): code = """if 1: import os, threading, warnings @@ -4923,8 +5437,7 @@ def test_fork_warns_when_non_python_thread_exists(self): self.assertEqual(err.decode("utf-8"), "") self.assertEqual(out.decode("utf-8"), "") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_fork_at_finalization(self): code = """if 1: import atexit diff --git a/Lib/test/test_patma.py b/Lib/test/test_patma.py new file mode 100644 index 0000000000..847ca001e4 --- /dev/null +++ b/Lib/test/test_patma.py @@ -0,0 +1,3572 @@ +import array +import collections +import dataclasses +import dis +import enum +import inspect +import sys +import unittest + + +@dataclasses.dataclass +class Point: + x: int + y: int + + +class TestCompiler(unittest.TestCase): + + def test_refleaks(self): + # Hunting for leaks using -R doesn't catch leaks in the compiler itself, + # just the code under test. This test ensures that if there are leaks in + # the pattern compiler, those runs will fail: + with open(__file__) as file: + compile(file.read(), __file__, "exec") + + +class TestInheritance(unittest.TestCase): + + @staticmethod + def check_sequence_then_mapping(x): + match x: + case [*_]: + return "seq" + case {}: + return "map" + + @staticmethod + def check_mapping_then_sequence(x): + match x: + case {}: + return "map" + case [*_]: + return "seq" + + def test_multiple_inheritance_mapping(self): + class C: + pass + class M1(collections.UserDict, collections.abc.Sequence): + pass + class M2(C, collections.UserDict, collections.abc.Sequence): + pass + class M3(collections.UserDict, C, list): + pass + class M4(dict, collections.abc.Sequence, C): + pass + self.assertEqual(self.check_sequence_then_mapping(M1()), "map") + self.assertEqual(self.check_sequence_then_mapping(M2()), "map") + self.assertEqual(self.check_sequence_then_mapping(M3()), "map") + self.assertEqual(self.check_sequence_then_mapping(M4()), "map") + self.assertEqual(self.check_mapping_then_sequence(M1()), "map") + self.assertEqual(self.check_mapping_then_sequence(M2()), "map") + self.assertEqual(self.check_mapping_then_sequence(M3()), "map") + self.assertEqual(self.check_mapping_then_sequence(M4()), "map") + + def test_multiple_inheritance_sequence(self): + class C: + pass + class S1(collections.UserList, collections.abc.Mapping): + pass + class S2(C, collections.UserList, collections.abc.Mapping): + pass + class S3(list, C, collections.abc.Mapping): + pass + class S4(collections.UserList, dict, C): + pass + self.assertEqual(self.check_sequence_then_mapping(S1()), "seq") + self.assertEqual(self.check_sequence_then_mapping(S2()), "seq") + self.assertEqual(self.check_sequence_then_mapping(S3()), "seq") + self.assertEqual(self.check_sequence_then_mapping(S4()), "seq") + self.assertEqual(self.check_mapping_then_sequence(S1()), "seq") + self.assertEqual(self.check_mapping_then_sequence(S2()), "seq") + self.assertEqual(self.check_mapping_then_sequence(S3()), "seq") + self.assertEqual(self.check_mapping_then_sequence(S4()), "seq") + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_late_registration_mapping(self): + class Parent: + pass + class ChildPre(Parent): + pass + class GrandchildPre(ChildPre): + pass + collections.abc.Mapping.register(Parent) + class ChildPost(Parent): + pass + class GrandchildPost(ChildPost): + pass + self.assertEqual(self.check_sequence_then_mapping(Parent()), "map") + self.assertEqual(self.check_sequence_then_mapping(ChildPre()), "map") + self.assertEqual(self.check_sequence_then_mapping(GrandchildPre()), "map") + self.assertEqual(self.check_sequence_then_mapping(ChildPost()), "map") + self.assertEqual(self.check_sequence_then_mapping(GrandchildPost()), "map") + self.assertEqual(self.check_mapping_then_sequence(Parent()), "map") + self.assertEqual(self.check_mapping_then_sequence(ChildPre()), "map") + self.assertEqual(self.check_mapping_then_sequence(GrandchildPre()), "map") + self.assertEqual(self.check_mapping_then_sequence(ChildPost()), "map") + self.assertEqual(self.check_mapping_then_sequence(GrandchildPost()), "map") + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_late_registration_sequence(self): + class Parent: + pass + class ChildPre(Parent): + pass + class GrandchildPre(ChildPre): + pass + collections.abc.Sequence.register(Parent) + class ChildPost(Parent): + pass + class GrandchildPost(ChildPost): + pass + self.assertEqual(self.check_sequence_then_mapping(Parent()), "seq") + self.assertEqual(self.check_sequence_then_mapping(ChildPre()), "seq") + self.assertEqual(self.check_sequence_then_mapping(GrandchildPre()), "seq") + self.assertEqual(self.check_sequence_then_mapping(ChildPost()), "seq") + self.assertEqual(self.check_sequence_then_mapping(GrandchildPost()), "seq") + self.assertEqual(self.check_mapping_then_sequence(Parent()), "seq") + self.assertEqual(self.check_mapping_then_sequence(ChildPre()), "seq") + self.assertEqual(self.check_mapping_then_sequence(GrandchildPre()), "seq") + self.assertEqual(self.check_mapping_then_sequence(ChildPost()), "seq") + self.assertEqual(self.check_mapping_then_sequence(GrandchildPost()), "seq") + + +class TestPatma(unittest.TestCase): + + def test_patma_000(self): + match 0: + case 0: + x = True + self.assertIs(x, True) + + def test_patma_001(self): + match 0: + case 0 if False: + x = False + case 0 if True: + x = True + self.assertIs(x, True) + + def test_patma_002(self): + match 0: + case 0: + x = True + case 0: + x = False + self.assertIs(x, True) + + def test_patma_003(self): + x = False + match 0: + case 0 | 1 | 2 | 3: + x = True + self.assertIs(x, True) + + def test_patma_004(self): + x = False + match 1: + case 0 | 1 | 2 | 3: + x = True + self.assertIs(x, True) + + def test_patma_005(self): + x = False + match 2: + case 0 | 1 | 2 | 3: + x = True + self.assertIs(x, True) + + def test_patma_006(self): + x = False + match 3: + case 0 | 1 | 2 | 3: + x = True + self.assertIs(x, True) + + def test_patma_007(self): + x = False + match 4: + case 0 | 1 | 2 | 3: + x = True + self.assertIs(x, False) + + def test_patma_008(self): + x = 0 + class A: + y = 1 + match x: + case A.y as z: + pass + self.assertEqual(x, 0) + self.assertEqual(A.y, 1) + + def test_patma_009(self): + class A: + B = 0 + match 0: + case x if x: + z = 0 + case _ as y if y == x and y: + z = 1 + case A.B: + z = 2 + self.assertEqual(A.B, 0) + self.assertEqual(x, 0) + self.assertEqual(y, 0) + self.assertEqual(z, 2) + + def test_patma_010(self): + match (): + case []: + x = 0 + self.assertEqual(x, 0) + + def test_patma_011(self): + match (0, 1, 2): + case [*x]: + y = 0 + self.assertEqual(x, [0, 1, 2]) + self.assertEqual(y, 0) + + def test_patma_012(self): + match (0, 1, 2): + case [0, *x]: + y = 0 + self.assertEqual(x, [1, 2]) + self.assertEqual(y, 0) + + def test_patma_013(self): + match (0, 1, 2): + case [0, 1, *x,]: + y = 0 + self.assertEqual(x, [2]) + self.assertEqual(y, 0) + + def test_patma_014(self): + match (0, 1, 2): + case [0, 1, 2, *x]: + y = 0 + self.assertEqual(x, []) + self.assertEqual(y, 0) + + def test_patma_015(self): + match (0, 1, 2): + case [*x, 2,]: + y = 0 + self.assertEqual(x, [0, 1]) + self.assertEqual(y, 0) + + def test_patma_016(self): + match (0, 1, 2): + case [*x, 1, 2]: + y = 0 + self.assertEqual(x, [0]) + self.assertEqual(y, 0) + + def test_patma_017(self): + match (0, 1, 2): + case [*x, 0, 1, 2,]: + y = 0 + self.assertEqual(x, []) + self.assertEqual(y, 0) + + def test_patma_018(self): + match (0, 1, 2): + case [0, *x, 2]: + y = 0 + self.assertEqual(x, [1]) + self.assertEqual(y, 0) + + def test_patma_019(self): + match (0, 1, 2): + case [0, 1, *x, 2,]: + y = 0 + self.assertEqual(x, []) + self.assertEqual(y, 0) + + def test_patma_020(self): + match (0, 1, 2): + case [0, *x, 1, 2]: + y = 0 + self.assertEqual(x, []) + self.assertEqual(y, 0) + + def test_patma_021(self): + match (0, 1, 2): + case [*x,]: + y = 0 + self.assertEqual(x, [0, 1, 2]) + self.assertEqual(y, 0) + + def test_patma_022(self): + x = {} + match x: + case {}: + y = 0 + self.assertEqual(x, {}) + self.assertEqual(y, 0) + + def test_patma_023(self): + x = {0: 0} + match x: + case {}: + y = 0 + self.assertEqual(x, {0: 0}) + self.assertEqual(y, 0) + + def test_patma_024(self): + x = {} + y = None + match x: + case {0: 0}: + y = 0 + self.assertEqual(x, {}) + self.assertIs(y, None) + + def test_patma_025(self): + x = {0: 0} + match x: + case {0: (0 | 1 | 2 as z)}: + y = 0 + self.assertEqual(x, {0: 0}) + self.assertEqual(y, 0) + self.assertEqual(z, 0) + + def test_patma_026(self): + x = {0: 1} + match x: + case {0: (0 | 1 | 2 as z)}: + y = 0 + self.assertEqual(x, {0: 1}) + self.assertEqual(y, 0) + self.assertEqual(z, 1) + + def test_patma_027(self): + x = {0: 2} + match x: + case {0: (0 | 1 | 2 as z)}: + y = 0 + self.assertEqual(x, {0: 2}) + self.assertEqual(y, 0) + self.assertEqual(z, 2) + + def test_patma_028(self): + x = {0: 3} + y = None + match x: + case {0: (0 | 1 | 2 as z)}: + y = 0 + self.assertEqual(x, {0: 3}) + self.assertIs(y, None) + + def test_patma_029(self): + x = {} + y = None + match x: + case {0: [1, 2, {}]}: + y = 0 + case {0: [1, 2, {}], 1: [[]]}: + y = 1 + case []: + y = 2 + self.assertEqual(x, {}) + self.assertIs(y, None) + + def test_patma_030(self): + x = {False: (True, 2.0, {})} + match x: + case {0: [1, 2, {}]}: + y = 0 + case {0: [1, 2, {}], 1: [[]]}: + y = 1 + case []: + y = 2 + self.assertEqual(x, {False: (True, 2.0, {})}) + self.assertEqual(y, 0) + + def test_patma_031(self): + x = {False: (True, 2.0, {}), 1: [[]], 2: 0} + match x: + case {0: [1, 2, {}]}: + y = 0 + case {0: [1, 2, {}], 1: [[]]}: + y = 1 + case []: + y = 2 + self.assertEqual(x, {False: (True, 2.0, {}), 1: [[]], 2: 0}) + self.assertEqual(y, 0) + + def test_patma_032(self): + x = {False: (True, 2.0, {}), 1: [[]], 2: 0} + match x: + case {0: [1, 2]}: + y = 0 + case {0: [1, 2, {}], 1: [[]]}: + y = 1 + case []: + y = 2 + self.assertEqual(x, {False: (True, 2.0, {}), 1: [[]], 2: 0}) + self.assertEqual(y, 1) + + def test_patma_033(self): + x = [] + match x: + case {0: [1, 2, {}]}: + y = 0 + case {0: [1, 2, {}], 1: [[]]}: + y = 1 + case []: + y = 2 + self.assertEqual(x, []) + self.assertEqual(y, 2) + + def test_patma_034(self): + x = {0: 0} + match x: + case {0: [1, 2, {}]}: + y = 0 + case {0: ([1, 2, {}] | False)} | {1: [[]]} | {0: [1, 2, {}]} | [] | "X" | {}: + y = 1 + case []: + y = 2 + self.assertEqual(x, {0: 0}) + self.assertEqual(y, 1) + + def test_patma_035(self): + x = {0: 0} + match x: + case {0: [1, 2, {}]}: + y = 0 + case {0: [1, 2, {}] | True} | {1: [[]]} | {0: [1, 2, {}]} | [] | "X" | {}: + y = 1 + case []: + y = 2 + self.assertEqual(x, {0: 0}) + self.assertEqual(y, 1) + + def test_patma_036(self): + x = 0 + match x: + case 0 | 1 | 2: + y = 0 + self.assertEqual(x, 0) + self.assertEqual(y, 0) + + def test_patma_037(self): + x = 1 + match x: + case 0 | 1 | 2: + y = 0 + self.assertEqual(x, 1) + self.assertEqual(y, 0) + + def test_patma_038(self): + x = 2 + match x: + case 0 | 1 | 2: + y = 0 + self.assertEqual(x, 2) + self.assertEqual(y, 0) + + def test_patma_039(self): + x = 3 + y = None + match x: + case 0 | 1 | 2: + y = 0 + self.assertEqual(x, 3) + self.assertIs(y, None) + + def test_patma_040(self): + x = 0 + match x: + case (0 as z) | (1 as z) | (2 as z) if z == x % 2: + y = 0 + self.assertEqual(x, 0) + self.assertEqual(y, 0) + self.assertEqual(z, 0) + + def test_patma_041(self): + x = 1 + match x: + case (0 as z) | (1 as z) | (2 as z) if z == x % 2: + y = 0 + self.assertEqual(x, 1) + self.assertEqual(y, 0) + self.assertEqual(z, 1) + + def test_patma_042(self): + x = 2 + y = None + match x: + case (0 as z) | (1 as z) | (2 as z) if z == x % 2: + y = 0 + self.assertEqual(x, 2) + self.assertIs(y, None) + self.assertEqual(z, 2) + + def test_patma_043(self): + x = 3 + y = None + match x: + case (0 as z) | (1 as z) | (2 as z) if z == x % 2: + y = 0 + self.assertEqual(x, 3) + self.assertIs(y, None) + + def test_patma_044(self): + x = () + match x: + case []: + y = 0 + self.assertEqual(x, ()) + self.assertEqual(y, 0) + + def test_patma_045(self): + x = () + match x: + case (): + y = 0 + self.assertEqual(x, ()) + self.assertEqual(y, 0) + + def test_patma_046(self): + x = (0,) + match x: + case [0]: + y = 0 + self.assertEqual(x, (0,)) + self.assertEqual(y, 0) + + def test_patma_047(self): + x = ((),) + match x: + case [[]]: + y = 0 + self.assertEqual(x, ((),)) + self.assertEqual(y, 0) + + def test_patma_048(self): + x = [0, 1] + match x: + case [0, 1] | [1, 0]: + y = 0 + self.assertEqual(x, [0, 1]) + self.assertEqual(y, 0) + + def test_patma_049(self): + x = [1, 0] + match x: + case [0, 1] | [1, 0]: + y = 0 + self.assertEqual(x, [1, 0]) + self.assertEqual(y, 0) + + def test_patma_050(self): + x = [0, 0] + y = None + match x: + case [0, 1] | [1, 0]: + y = 0 + self.assertEqual(x, [0, 0]) + self.assertIs(y, None) + + def test_patma_051(self): + w = None + x = [1, 0] + match x: + case [(0 as w)]: + y = 0 + case [z] | [1, (0 | 1 as z)] | [z]: + y = 1 + self.assertIs(w, None) + self.assertEqual(x, [1, 0]) + self.assertEqual(y, 1) + self.assertEqual(z, 0) + + def test_patma_052(self): + x = [1, 0] + match x: + case [0]: + y = 0 + case [1, 0] if (x := x[:0]): + y = 1 + case [1, 0]: + y = 2 + self.assertEqual(x, []) + self.assertEqual(y, 2) + + def test_patma_053(self): + x = {0} + y = None + match x: + case [0]: + y = 0 + self.assertEqual(x, {0}) + self.assertIs(y, None) + + def test_patma_054(self): + x = set() + y = None + match x: + case []: + y = 0 + self.assertEqual(x, set()) + self.assertIs(y, None) + + def test_patma_055(self): + x = iter([1, 2, 3]) + y = None + match x: + case []: + y = 0 + self.assertEqual([*x], [1, 2, 3]) + self.assertIs(y, None) + + def test_patma_056(self): + x = {} + y = None + match x: + case []: + y = 0 + self.assertEqual(x, {}) + self.assertIs(y, None) + + def test_patma_057(self): + x = {0: False, 1: True} + y = None + match x: + case [0, 1]: + y = 0 + self.assertEqual(x, {0: False, 1: True}) + self.assertIs(y, None) + + def test_patma_058(self): + x = 0 + match x: + case 0: + y = 0 + self.assertEqual(x, 0) + self.assertEqual(y, 0) + + def test_patma_059(self): + x = 0 + y = None + match x: + case False: + y = 0 + self.assertEqual(x, 0) + self.assertEqual(y, None) + + def test_patma_060(self): + x = 0 + y = None + match x: + case 1: + y = 0 + self.assertEqual(x, 0) + self.assertIs(y, None) + + def test_patma_061(self): + x = 0 + y = None + match x: + case None: + y = 0 + self.assertEqual(x, 0) + self.assertIs(y, None) + + def test_patma_062(self): + x = 0 + match x: + case 0: + y = 0 + case 0: + y = 1 + self.assertEqual(x, 0) + self.assertEqual(y, 0) + + def test_patma_063(self): + x = 0 + y = None + match x: + case 1: + y = 0 + case 1: + y = 1 + self.assertEqual(x, 0) + self.assertIs(y, None) + + def test_patma_064(self): + x = "x" + match x: + case "x": + y = 0 + case "y": + y = 1 + self.assertEqual(x, "x") + self.assertEqual(y, 0) + + def test_patma_065(self): + x = "x" + match x: + case "y": + y = 0 + case "x": + y = 1 + self.assertEqual(x, "x") + self.assertEqual(y, 1) + + def test_patma_066(self): + x = "x" + match x: + case "": + y = 0 + case "x": + y = 1 + self.assertEqual(x, "x") + self.assertEqual(y, 1) + + def test_patma_067(self): + x = b"x" + match x: + case b"y": + y = 0 + case b"x": + y = 1 + self.assertEqual(x, b"x") + self.assertEqual(y, 1) + + def test_patma_068(self): + x = 0 + match x: + case 0 if False: + y = 0 + case 0: + y = 1 + self.assertEqual(x, 0) + self.assertEqual(y, 1) + + def test_patma_069(self): + x = 0 + y = None + match x: + case 0 if 0: + y = 0 + case 0 if 0: + y = 1 + self.assertEqual(x, 0) + self.assertIs(y, None) + + def test_patma_070(self): + x = 0 + match x: + case 0 if True: + y = 0 + case 0 if True: + y = 1 + self.assertEqual(x, 0) + self.assertEqual(y, 0) + + def test_patma_071(self): + x = 0 + match x: + case 0 if 1: + y = 0 + case 0 if 1: + y = 1 + self.assertEqual(x, 0) + self.assertEqual(y, 0) + + def test_patma_072(self): + x = 0 + match x: + case 0 if True: + y = 0 + case 0 if True: + y = 1 + y = 2 + self.assertEqual(x, 0) + self.assertEqual(y, 2) + + def test_patma_073(self): + x = 0 + match x: + case 0 if 0: + y = 0 + case 0 if 1: + y = 1 + y = 2 + self.assertEqual(x, 0) + self.assertEqual(y, 2) + + def test_patma_074(self): + x = 0 + y = None + match x: + case 0 if not (x := 1): + y = 0 + case 1: + y = 1 + self.assertEqual(x, 1) + self.assertIs(y, None) + + def test_patma_075(self): + x = "x" + match x: + case ["x"]: + y = 0 + case "x": + y = 1 + self.assertEqual(x, "x") + self.assertEqual(y, 1) + + def test_patma_076(self): + x = b"x" + match x: + case [b"x"]: + y = 0 + case ["x"]: + y = 1 + case [120]: + y = 2 + case b"x": + y = 4 + self.assertEqual(x, b"x") + self.assertEqual(y, 4) + + def test_patma_077(self): + x = bytearray(b"x") + y = None + match x: + case [120]: + y = 0 + case 120: + y = 1 + self.assertEqual(x, b"x") + self.assertIs(y, None) + + def test_patma_078(self): + x = "" + match x: + case []: + y = 0 + case [""]: + y = 1 + case "": + y = 2 + self.assertEqual(x, "") + self.assertEqual(y, 2) + + def test_patma_079(self): + x = "xxx" + match x: + case ["x", "x", "x"]: + y = 0 + case ["xxx"]: + y = 1 + case "xxx": + y = 2 + self.assertEqual(x, "xxx") + self.assertEqual(y, 2) + + def test_patma_080(self): + x = b"xxx" + match x: + case [120, 120, 120]: + y = 0 + case [b"xxx"]: + y = 1 + case b"xxx": + y = 2 + self.assertEqual(x, b"xxx") + self.assertEqual(y, 2) + + def test_patma_081(self): + x = 0 + match x: + case 0 if not (x := 1): + y = 0 + case (0 as z): + y = 1 + self.assertEqual(x, 1) + self.assertEqual(y, 1) + self.assertEqual(z, 0) + + def test_patma_082(self): + x = 0 + match x: + case (1 as z) if not (x := 1): + y = 0 + case 0: + y = 1 + self.assertEqual(x, 0) + self.assertEqual(y, 1) + + def test_patma_083(self): + x = 0 + match x: + case (0 as z): + y = 0 + self.assertEqual(x, 0) + self.assertEqual(y, 0) + self.assertEqual(z, 0) + + def test_patma_084(self): + x = 0 + y = None + match x: + case (1 as z): + y = 0 + self.assertEqual(x, 0) + self.assertIs(y, None) + + def test_patma_085(self): + x = 0 + y = None + match x: + case (0 as z) if (w := 0): + y = 0 + self.assertEqual(w, 0) + self.assertEqual(x, 0) + self.assertIs(y, None) + self.assertEqual(z, 0) + + def test_patma_086(self): + x = 0 + match x: + case ((0 as w) as z): + y = 0 + self.assertEqual(w, 0) + self.assertEqual(x, 0) + self.assertEqual(y, 0) + self.assertEqual(z, 0) + + def test_patma_087(self): + x = 0 + match x: + case (0 | 1) | 2: + y = 0 + self.assertEqual(x, 0) + self.assertEqual(y, 0) + + def test_patma_088(self): + x = 1 + match x: + case (0 | 1) | 2: + y = 0 + self.assertEqual(x, 1) + self.assertEqual(y, 0) + + def test_patma_089(self): + x = 2 + match x: + case (0 | 1) | 2: + y = 0 + self.assertEqual(x, 2) + self.assertEqual(y, 0) + + def test_patma_090(self): + x = 3 + y = None + match x: + case (0 | 1) | 2: + y = 0 + self.assertEqual(x, 3) + self.assertIs(y, None) + + def test_patma_091(self): + x = 0 + match x: + case 0 | (1 | 2): + y = 0 + self.assertEqual(x, 0) + self.assertEqual(y, 0) + + def test_patma_092(self): + x = 1 + match x: + case 0 | (1 | 2): + y = 0 + self.assertEqual(x, 1) + self.assertEqual(y, 0) + + def test_patma_093(self): + x = 2 + match x: + case 0 | (1 | 2): + y = 0 + self.assertEqual(x, 2) + self.assertEqual(y, 0) + + def test_patma_094(self): + x = 3 + y = None + match x: + case 0 | (1 | 2): + y = 0 + self.assertEqual(x, 3) + self.assertIs(y, None) + + def test_patma_095(self): + x = 0 + match x: + case -0: + y = 0 + self.assertEqual(x, 0) + self.assertEqual(y, 0) + + def test_patma_096(self): + x = 0 + match x: + case -0.0: + y = 0 + self.assertEqual(x, 0) + self.assertEqual(y, 0) + + def test_patma_097(self): + x = 0 + match x: + case -0j: + y = 0 + self.assertEqual(x, 0) + self.assertEqual(y, 0) + + def test_patma_098(self): + x = 0 + match x: + case -0.0j: + y = 0 + self.assertEqual(x, 0) + self.assertEqual(y, 0) + + def test_patma_099(self): + x = -1 + match x: + case -1: + y = 0 + self.assertEqual(x, -1) + self.assertEqual(y, 0) + + def test_patma_100(self): + x = -1.5 + match x: + case -1.5: + y = 0 + self.assertEqual(x, -1.5) + self.assertEqual(y, 0) + + def test_patma_101(self): + x = -1j + match x: + case -1j: + y = 0 + self.assertEqual(x, -1j) + self.assertEqual(y, 0) + + def test_patma_102(self): + x = -1.5j + match x: + case -1.5j: + y = 0 + self.assertEqual(x, -1.5j) + self.assertEqual(y, 0) + + def test_patma_103(self): + x = 0 + match x: + case 0 + 0j: + y = 0 + self.assertEqual(x, 0) + self.assertEqual(y, 0) + + def test_patma_104(self): + x = 0 + match x: + case 0 - 0j: + y = 0 + self.assertEqual(x, 0) + self.assertEqual(y, 0) + + def test_patma_105(self): + x = 0 + match x: + case -0 + 0j: + y = 0 + self.assertEqual(x, 0) + self.assertEqual(y, 0) + + def test_patma_106(self): + x = 0 + match x: + case -0 - 0j: + y = 0 + self.assertEqual(x, 0) + self.assertEqual(y, 0) + + def test_patma_107(self): + x = 0.25 + 1.75j + match x: + case 0.25 + 1.75j: + y = 0 + self.assertEqual(x, 0.25 + 1.75j) + self.assertEqual(y, 0) + + def test_patma_108(self): + x = 0.25 - 1.75j + match x: + case 0.25 - 1.75j: + y = 0 + self.assertEqual(x, 0.25 - 1.75j) + self.assertEqual(y, 0) + + def test_patma_109(self): + x = -0.25 + 1.75j + match x: + case -0.25 + 1.75j: + y = 0 + self.assertEqual(x, -0.25 + 1.75j) + self.assertEqual(y, 0) + + def test_patma_110(self): + x = -0.25 - 1.75j + match x: + case -0.25 - 1.75j: + y = 0 + self.assertEqual(x, -0.25 - 1.75j) + self.assertEqual(y, 0) + + def test_patma_111(self): + class A: + B = 0 + x = 0 + match x: + case A.B: + y = 0 + self.assertEqual(A.B, 0) + self.assertEqual(x, 0) + self.assertEqual(y, 0) + + def test_patma_112(self): + class A: + class B: + C = 0 + x = 0 + match x: + case A.B.C: + y = 0 + self.assertEqual(A.B.C, 0) + self.assertEqual(x, 0) + self.assertEqual(y, 0) + + def test_patma_113(self): + class A: + class B: + C = 0 + D = 1 + x = 1 + match x: + case A.B.C: + y = 0 + case A.B.D: + y = 1 + self.assertEqual(A.B.C, 0) + self.assertEqual(A.B.D, 1) + self.assertEqual(x, 1) + self.assertEqual(y, 1) + + def test_patma_114(self): + class A: + class B: + class C: + D = 0 + x = 0 + match x: + case A.B.C.D: + y = 0 + self.assertEqual(A.B.C.D, 0) + self.assertEqual(x, 0) + self.assertEqual(y, 0) + + def test_patma_115(self): + class A: + class B: + class C: + D = 0 + E = 1 + x = 1 + match x: + case A.B.C.D: + y = 0 + case A.B.C.E: + y = 1 + self.assertEqual(A.B.C.D, 0) + self.assertEqual(A.B.C.E, 1) + self.assertEqual(x, 1) + self.assertEqual(y, 1) + + def test_patma_116(self): + match = case = 0 + match match: + case case: + x = 0 + self.assertEqual(match, 0) + self.assertEqual(case, 0) + self.assertEqual(x, 0) + + def test_patma_117(self): + match = case = 0 + match case: + case match: + x = 0 + self.assertEqual(match, 0) + self.assertEqual(case, 0) + self.assertEqual(x, 0) + + def test_patma_118(self): + x = [] + match x: + case [*_, _]: + y = 0 + case []: + y = 1 + self.assertEqual(x, []) + self.assertEqual(y, 1) + + def test_patma_119(self): + x = collections.defaultdict(int) + match x: + case {0: 0}: + y = 0 + case {}: + y = 1 + self.assertEqual(x, {}) + self.assertEqual(y, 1) + + def test_patma_120(self): + x = collections.defaultdict(int) + match x: + case {0: 0}: + y = 0 + case {**z}: + y = 1 + self.assertEqual(x, {}) + self.assertEqual(y, 1) + self.assertEqual(z, {}) + + def test_patma_121(self): + match (): + case (): + x = 0 + self.assertEqual(x, 0) + + def test_patma_122(self): + match (0, 1, 2): + case (*x,): + y = 0 + self.assertEqual(x, [0, 1, 2]) + self.assertEqual(y, 0) + + def test_patma_123(self): + match (0, 1, 2): + case 0, *x: + y = 0 + self.assertEqual(x, [1, 2]) + self.assertEqual(y, 0) + + def test_patma_124(self): + match (0, 1, 2): + case (0, 1, *x,): + y = 0 + self.assertEqual(x, [2]) + self.assertEqual(y, 0) + + def test_patma_125(self): + match (0, 1, 2): + case 0, 1, 2, *x: + y = 0 + self.assertEqual(x, []) + self.assertEqual(y, 0) + + def test_patma_126(self): + match (0, 1, 2): + case *x, 2,: + y = 0 + self.assertEqual(x, [0, 1]) + self.assertEqual(y, 0) + + def test_patma_127(self): + match (0, 1, 2): + case (*x, 1, 2): + y = 0 + self.assertEqual(x, [0]) + self.assertEqual(y, 0) + + def test_patma_128(self): + match (0, 1, 2): + case *x, 0, 1, 2,: + y = 0 + self.assertEqual(x, []) + self.assertEqual(y, 0) + + def test_patma_129(self): + match (0, 1, 2): + case (0, *x, 2): + y = 0 + self.assertEqual(x, [1]) + self.assertEqual(y, 0) + + def test_patma_130(self): + match (0, 1, 2): + case 0, 1, *x, 2,: + y = 0 + self.assertEqual(x, []) + self.assertEqual(y, 0) + + def test_patma_131(self): + match (0, 1, 2): + case (0, *x, 1, 2): + y = 0 + self.assertEqual(x, []) + self.assertEqual(y, 0) + + def test_patma_132(self): + match (0, 1, 2): + case *x,: + y = 0 + self.assertEqual(x, [0, 1, 2]) + self.assertEqual(y, 0) + + def test_patma_133(self): + x = collections.defaultdict(int, {0: 1}) + match x: + case {1: 0}: + y = 0 + case {0: 0}: + y = 1 + case {}: + y = 2 + self.assertEqual(x, {0: 1}) + self.assertEqual(y, 2) + + def test_patma_134(self): + x = collections.defaultdict(int, {0: 1}) + match x: + case {1: 0}: + y = 0 + case {0: 0}: + y = 1 + case {**z}: + y = 2 + self.assertEqual(x, {0: 1}) + self.assertEqual(y, 2) + self.assertEqual(z, {0: 1}) + + def test_patma_135(self): + x = collections.defaultdict(int, {0: 1}) + match x: + case {1: 0}: + y = 0 + case {0: 0}: + y = 1 + case {0: _, **z}: + y = 2 + self.assertEqual(x, {0: 1}) + self.assertEqual(y, 2) + self.assertEqual(z, {}) + + def test_patma_136(self): + x = {0: 1} + match x: + case {1: 0}: + y = 0 + case {0: 0}: + y = 0 + case {}: + y = 1 + self.assertEqual(x, {0: 1}) + self.assertEqual(y, 1) + + def test_patma_137(self): + x = {0: 1} + match x: + case {1: 0}: + y = 0 + case {0: 0}: + y = 0 + case {**z}: + y = 1 + self.assertEqual(x, {0: 1}) + self.assertEqual(y, 1) + self.assertEqual(z, {0: 1}) + + def test_patma_138(self): + x = {0: 1} + match x: + case {1: 0}: + y = 0 + case {0: 0}: + y = 0 + case {0: _, **z}: + y = 1 + self.assertEqual(x, {0: 1}) + self.assertEqual(y, 1) + self.assertEqual(z, {}) + + def test_patma_139(self): + x = False + match x: + case bool(z): + y = 0 + self.assertIs(x, False) + self.assertEqual(y, 0) + self.assertIs(z, x) + + def test_patma_140(self): + x = True + match x: + case bool(z): + y = 0 + self.assertIs(x, True) + self.assertEqual(y, 0) + self.assertIs(z, x) + + def test_patma_141(self): + x = bytearray() + match x: + case bytearray(z): + y = 0 + self.assertEqual(x, bytearray()) + self.assertEqual(y, 0) + self.assertIs(z, x) + + def test_patma_142(self): + x = b"" + match x: + case bytes(z): + y = 0 + self.assertEqual(x, b"") + self.assertEqual(y, 0) + self.assertIs(z, x) + + def test_patma_143(self): + x = {} + match x: + case dict(z): + y = 0 + self.assertEqual(x, {}) + self.assertEqual(y, 0) + self.assertIs(z, x) + + def test_patma_144(self): + x = 0.0 + match x: + case float(z): + y = 0 + self.assertEqual(x, 0.0) + self.assertEqual(y, 0) + self.assertIs(z, x) + + def test_patma_145(self): + x = frozenset() + match x: + case frozenset(z): + y = 0 + self.assertEqual(x, frozenset()) + self.assertEqual(y, 0) + self.assertIs(z, x) + + def test_patma_146(self): + x = 0 + match x: + case int(z): + y = 0 + self.assertEqual(x, 0) + self.assertEqual(y, 0) + self.assertIs(z, x) + + def test_patma_147(self): + x = [] + match x: + case list(z): + y = 0 + self.assertEqual(x, []) + self.assertEqual(y, 0) + self.assertIs(z, x) + + def test_patma_148(self): + x = set() + match x: + case set(z): + y = 0 + self.assertEqual(x, set()) + self.assertEqual(y, 0) + self.assertIs(z, x) + + def test_patma_149(self): + x = "" + match x: + case str(z): + y = 0 + self.assertEqual(x, "") + self.assertEqual(y, 0) + self.assertIs(z, x) + + def test_patma_150(self): + x = () + match x: + case tuple(z): + y = 0 + self.assertEqual(x, ()) + self.assertEqual(y, 0) + self.assertIs(z, x) + + def test_patma_151(self): + x = 0 + match x,: + case y,: + z = 0 + self.assertEqual(x, 0) + self.assertIs(y, x) + self.assertIs(z, 0) + + def test_patma_152(self): + w = 0 + x = 0 + match w, x: + case y, z: + v = 0 + self.assertEqual(w, 0) + self.assertEqual(x, 0) + self.assertIs(y, w) + self.assertIs(z, x) + self.assertEqual(v, 0) + + def test_patma_153(self): + x = 0 + match w := x,: + case y as v,: + z = 0 + self.assertEqual(x, 0) + self.assertIs(y, x) + self.assertEqual(z, 0) + self.assertIs(w, x) + self.assertIs(v, y) + + def test_patma_154(self): + x = 0 + y = None + match x: + case 0 if x: + y = 0 + self.assertEqual(x, 0) + self.assertIs(y, None) + + def test_patma_155(self): + x = 0 + y = None + match x: + case 1e1000: + y = 0 + self.assertEqual(x, 0) + self.assertIs(y, None) + + def test_patma_156(self): + x = 0 + match x: + case z: + y = 0 + self.assertEqual(x, 0) + self.assertEqual(y, 0) + self.assertIs(z, x) + + def test_patma_157(self): + x = 0 + y = None + match x: + case _ if x: + y = 0 + self.assertEqual(x, 0) + self.assertIs(y, None) + + def test_patma_158(self): + x = 0 + match x: + case -1e1000: + y = 0 + case 0: + y = 1 + self.assertEqual(x, 0) + self.assertEqual(y, 1) + + def test_patma_159(self): + x = 0 + match x: + case 0 if not x: + y = 0 + case 1: + y = 1 + self.assertEqual(x, 0) + self.assertEqual(y, 0) + + def test_patma_160(self): + x = 0 + z = None + match x: + case 0: + y = 0 + case z if x: + y = 1 + self.assertEqual(x, 0) + self.assertEqual(y, 0) + self.assertIs(z, None) + + def test_patma_161(self): + x = 0 + match x: + case 0: + y = 0 + case _: + y = 1 + self.assertEqual(x, 0) + self.assertEqual(y, 0) + + def test_patma_162(self): + x = 0 + match x: + case 1 if x: + y = 0 + case 0: + y = 1 + self.assertEqual(x, 0) + self.assertEqual(y, 1) + + def test_patma_163(self): + x = 0 + y = None + match x: + case 1: + y = 0 + case 1 if not x: + y = 1 + self.assertEqual(x, 0) + self.assertIs(y, None) + + def test_patma_164(self): + x = 0 + match x: + case 1: + y = 0 + case z: + y = 1 + self.assertEqual(x, 0) + self.assertEqual(y, 1) + self.assertIs(z, x) + + def test_patma_165(self): + x = 0 + match x: + case 1 if x: + y = 0 + case _: + y = 1 + self.assertEqual(x, 0) + self.assertEqual(y, 1) + + def test_patma_166(self): + x = 0 + match x: + case z if not z: + y = 0 + case 0 if x: + y = 1 + self.assertEqual(x, 0) + self.assertEqual(y, 0) + self.assertIs(z, x) + + def test_patma_167(self): + x = 0 + match x: + case z if not z: + y = 0 + case 1: + y = 1 + self.assertEqual(x, 0) + self.assertEqual(y, 0) + self.assertIs(z, x) + + def test_patma_168(self): + x = 0 + match x: + case z if not x: + y = 0 + case z: + y = 1 + self.assertEqual(x, 0) + self.assertEqual(y, 0) + self.assertIs(z, x) + + def test_patma_169(self): + x = 0 + match x: + case z if not z: + y = 0 + case _ if x: + y = 1 + self.assertEqual(x, 0) + self.assertEqual(y, 0) + self.assertIs(z, x) + + def test_patma_170(self): + x = 0 + match x: + case _ if not x: + y = 0 + case 0: + y = 1 + self.assertEqual(x, 0) + self.assertEqual(y, 0) + + def test_patma_171(self): + x = 0 + y = None + match x: + case _ if x: + y = 0 + case 1: + y = 1 + self.assertEqual(x, 0) + self.assertIs(y, None) + + def test_patma_172(self): + x = 0 + z = None + match x: + case _ if not x: + y = 0 + case z if not x: + y = 1 + self.assertEqual(x, 0) + self.assertEqual(y, 0) + self.assertIs(z, None) + + def test_patma_173(self): + x = 0 + match x: + case _ if not x: + y = 0 + case _: + y = 1 + self.assertEqual(x, 0) + self.assertEqual(y, 0) + + def test_patma_174(self): + def http_error(status): + match status: + case 400: + return "Bad request" + case 401: + return "Unauthorized" + case 403: + return "Forbidden" + case 404: + return "Not found" + case 418: + return "I'm a teapot" + case _: + return "Something else" + self.assertEqual(http_error(400), "Bad request") + self.assertEqual(http_error(401), "Unauthorized") + self.assertEqual(http_error(403), "Forbidden") + self.assertEqual(http_error(404), "Not found") + self.assertEqual(http_error(418), "I'm a teapot") + self.assertEqual(http_error(123), "Something else") + self.assertEqual(http_error("400"), "Something else") + self.assertEqual(http_error(401 | 403 | 404), "Something else") # 407 + + def test_patma_175(self): + def http_error(status): + match status: + case 400: + return "Bad request" + case 401 | 403 | 404: + return "Not allowed" + case 418: + return "I'm a teapot" + self.assertEqual(http_error(400), "Bad request") + self.assertEqual(http_error(401), "Not allowed") + self.assertEqual(http_error(403), "Not allowed") + self.assertEqual(http_error(404), "Not allowed") + self.assertEqual(http_error(418), "I'm a teapot") + self.assertIs(http_error(123), None) + self.assertIs(http_error("400"), None) + self.assertIs(http_error(401 | 403 | 404), None) # 407 + + def test_patma_176(self): + def whereis(point): + match point: + case (0, 0): + return "Origin" + case (0, y): + return f"Y={y}" + case (x, 0): + return f"X={x}" + case (x, y): + return f"X={x}, Y={y}" + case _: + return "Not a point" + self.assertEqual(whereis((0, 0)), "Origin") + self.assertEqual(whereis((0, -1.0)), "Y=-1.0") + self.assertEqual(whereis(("X", 0)), "X=X") + self.assertEqual(whereis((None, 1j)), "X=None, Y=1j") + self.assertEqual(whereis(42), "Not a point") + + def test_patma_177(self): + def whereis(point): + match point: + case Point(0, 0): + return "Origin" + case Point(0, y): + return f"Y={y}" + case Point(x, 0): + return f"X={x}" + case Point(): + return "Somewhere else" + case _: + return "Not a point" + self.assertEqual(whereis(Point(1, 0)), "X=1") + self.assertEqual(whereis(Point(0, 0)), "Origin") + self.assertEqual(whereis(10), "Not a point") + self.assertEqual(whereis(Point(False, False)), "Origin") + self.assertEqual(whereis(Point(0, -1.0)), "Y=-1.0") + self.assertEqual(whereis(Point("X", 0)), "X=X") + self.assertEqual(whereis(Point(None, 1j)), "Somewhere else") + self.assertEqual(whereis(Point), "Not a point") + self.assertEqual(whereis(42), "Not a point") + + def test_patma_178(self): + def whereis(point): + match point: + case Point(1, var): + return var + self.assertEqual(whereis(Point(1, 0)), 0) + self.assertIs(whereis(Point(0, 0)), None) + + def test_patma_179(self): + def whereis(point): + match point: + case Point(1, y=var): + return var + self.assertEqual(whereis(Point(1, 0)), 0) + self.assertIs(whereis(Point(0, 0)), None) + + def test_patma_180(self): + def whereis(point): + match point: + case Point(x=1, y=var): + return var + self.assertEqual(whereis(Point(1, 0)), 0) + self.assertIs(whereis(Point(0, 0)), None) + + def test_patma_181(self): + def whereis(point): + match point: + case Point(y=var, x=1): + return var + self.assertEqual(whereis(Point(1, 0)), 0) + self.assertIs(whereis(Point(0, 0)), None) + + def test_patma_182(self): + def whereis(points): + match points: + case []: + return "No points" + case [Point(0, 0)]: + return "The origin" + case [Point(x, y)]: + return f"Single point {x}, {y}" + case [Point(0, y1), Point(0, y2)]: + return f"Two on the Y axis at {y1}, {y2}" + case _: + return "Something else" + self.assertEqual(whereis([]), "No points") + self.assertEqual(whereis([Point(0, 0)]), "The origin") + self.assertEqual(whereis([Point(0, 1)]), "Single point 0, 1") + self.assertEqual(whereis([Point(0, 0), Point(0, 0)]), "Two on the Y axis at 0, 0") + self.assertEqual(whereis([Point(0, 1), Point(0, 1)]), "Two on the Y axis at 1, 1") + self.assertEqual(whereis([Point(0, 0), Point(1, 0)]), "Something else") + self.assertEqual(whereis([Point(0, 0), Point(0, 0), Point(0, 0)]), "Something else") + self.assertEqual(whereis([Point(0, 1), Point(0, 1), Point(0, 1)]), "Something else") + + def test_patma_183(self): + def whereis(point): + match point: + case Point(x, y) if x == y: + return f"Y=X at {x}" + case Point(x, y): + return "Not on the diagonal" + self.assertEqual(whereis(Point(0, 0)), "Y=X at 0") + self.assertEqual(whereis(Point(0, False)), "Y=X at 0") + self.assertEqual(whereis(Point(False, 0)), "Y=X at False") + self.assertEqual(whereis(Point(-1 - 1j, -1 - 1j)), "Y=X at (-1-1j)") + self.assertEqual(whereis(Point("X", "X")), "Y=X at X") + self.assertEqual(whereis(Point("X", "x")), "Not on the diagonal") + + def test_patma_184(self): + class Seq(collections.abc.Sequence): + __getitem__ = None + def __len__(self): + return 0 + match Seq(): + case []: + y = 0 + self.assertEqual(y, 0) + + def test_patma_185(self): + class Seq(collections.abc.Sequence): + __getitem__ = None + def __len__(self): + return 42 + match Seq(): + case [*_]: + y = 0 + self.assertEqual(y, 0) + + def test_patma_186(self): + class Seq(collections.abc.Sequence): + def __getitem__(self, i): + return i + def __len__(self): + return 42 + match Seq(): + case [x, *_, y]: + z = 0 + self.assertEqual(x, 0) + self.assertEqual(y, 41) + self.assertEqual(z, 0) + + def test_patma_187(self): + w = range(10) + match w: + case [x, y, *rest]: + z = 0 + self.assertEqual(w, range(10)) + self.assertEqual(x, 0) + self.assertEqual(y, 1) + self.assertEqual(z, 0) + self.assertEqual(rest, list(range(2, 10))) + + def test_patma_188(self): + w = range(100) + match w: + case (x, y, *rest): + z = 0 + self.assertEqual(w, range(100)) + self.assertEqual(x, 0) + self.assertEqual(y, 1) + self.assertEqual(z, 0) + self.assertEqual(rest, list(range(2, 100))) + + def test_patma_189(self): + w = range(1000) + match w: + case x, y, *rest: + z = 0 + self.assertEqual(w, range(1000)) + self.assertEqual(x, 0) + self.assertEqual(y, 1) + self.assertEqual(z, 0) + self.assertEqual(rest, list(range(2, 1000))) + + def test_patma_190(self): + w = range(1 << 10) + match w: + case [x, y, *_]: + z = 0 + self.assertEqual(w, range(1 << 10)) + self.assertEqual(x, 0) + self.assertEqual(y, 1) + self.assertEqual(z, 0) + + def test_patma_191(self): + w = range(1 << 20) + match w: + case (x, y, *_): + z = 0 + self.assertEqual(w, range(1 << 20)) + self.assertEqual(x, 0) + self.assertEqual(y, 1) + self.assertEqual(z, 0) + + def test_patma_192(self): + w = range(1 << 30) + match w: + case x, y, *_: + z = 0 + self.assertEqual(w, range(1 << 30)) + self.assertEqual(x, 0) + self.assertEqual(y, 1) + self.assertEqual(z, 0) + + def test_patma_193(self): + x = {"bandwidth": 0, "latency": 1} + match x: + case {"bandwidth": b, "latency": l}: + y = 0 + self.assertEqual(x, {"bandwidth": 0, "latency": 1}) + self.assertIs(b, x["bandwidth"]) + self.assertIs(l, x["latency"]) + self.assertEqual(y, 0) + + def test_patma_194(self): + x = {"bandwidth": 0, "latency": 1, "key": "value"} + match x: + case {"latency": l, "bandwidth": b}: + y = 0 + self.assertEqual(x, {"bandwidth": 0, "latency": 1, "key": "value"}) + self.assertIs(l, x["latency"]) + self.assertIs(b, x["bandwidth"]) + self.assertEqual(y, 0) + + def test_patma_195(self): + x = {"bandwidth": 0, "latency": 1, "key": "value"} + match x: + case {"bandwidth": b, "latency": l, **rest}: + y = 0 + self.assertEqual(x, {"bandwidth": 0, "latency": 1, "key": "value"}) + self.assertIs(b, x["bandwidth"]) + self.assertIs(l, x["latency"]) + self.assertEqual(rest, {"key": "value"}) + self.assertEqual(y, 0) + + def test_patma_196(self): + x = {"bandwidth": 0, "latency": 1} + match x: + case {"latency": l, "bandwidth": b, **rest}: + y = 0 + self.assertEqual(x, {"bandwidth": 0, "latency": 1}) + self.assertIs(l, x["latency"]) + self.assertIs(b, x["bandwidth"]) + self.assertEqual(rest, {}) + self.assertEqual(y, 0) + + def test_patma_197(self): + w = [Point(-1, 0), Point(1, 2)] + match w: + case (Point(x1, y1), Point(x2, y2) as p2): + z = 0 + self.assertEqual(w, [Point(-1, 0), Point(1, 2)]) + self.assertIs(x1, w[0].x) + self.assertIs(y1, w[0].y) + self.assertIs(p2, w[1]) + self.assertIs(x2, w[1].x) + self.assertIs(y2, w[1].y) + self.assertIs(z, 0) + + def test_patma_198(self): + class Color(enum.Enum): + RED = 0 + GREEN = 1 + BLUE = 2 + def f(color): + match color: + case Color.RED: + return "I see red!" + case Color.GREEN: + return "Grass is green" + case Color.BLUE: + return "I'm feeling the blues :(" + self.assertEqual(f(Color.RED), "I see red!") + self.assertEqual(f(Color.GREEN), "Grass is green") + self.assertEqual(f(Color.BLUE), "I'm feeling the blues :(") + self.assertIs(f(Color), None) + self.assertIs(f(0), None) + self.assertIs(f(1), None) + self.assertIs(f(2), None) + self.assertIs(f(3), None) + self.assertIs(f(False), None) + self.assertIs(f(True), None) + self.assertIs(f(2+0j), None) + self.assertIs(f(3.0), None) + + def test_patma_199(self): + class Color(int, enum.Enum): + RED = 0 + GREEN = 1 + BLUE = 2 + def f(color): + match color: + case Color.RED: + return "I see red!" + case Color.GREEN: + return "Grass is green" + case Color.BLUE: + return "I'm feeling the blues :(" + self.assertEqual(f(Color.RED), "I see red!") + self.assertEqual(f(Color.GREEN), "Grass is green") + self.assertEqual(f(Color.BLUE), "I'm feeling the blues :(") + self.assertIs(f(Color), None) + self.assertEqual(f(0), "I see red!") + self.assertEqual(f(1), "Grass is green") + self.assertEqual(f(2), "I'm feeling the blues :(") + self.assertIs(f(3), None) + self.assertEqual(f(False), "I see red!") + self.assertEqual(f(True), "Grass is green") + self.assertEqual(f(2+0j), "I'm feeling the blues :(") + self.assertIs(f(3.0), None) + + def test_patma_200(self): + class Class: + __match_args__ = ("a", "b") + c = Class() + c.a = 0 + c.b = 1 + match c: + case Class(x, y): + z = 0 + self.assertIs(x, c.a) + self.assertIs(y, c.b) + self.assertEqual(z, 0) + + def test_patma_201(self): + class Class: + __match_args__ = ("a", "b") + c = Class() + c.a = 0 + c.b = 1 + match c: + case Class(x, b=y): + z = 0 + self.assertIs(x, c.a) + self.assertIs(y, c.b) + self.assertEqual(z, 0) + + def test_patma_202(self): + class Parent: + __match_args__ = "a", "b" + class Child(Parent): + __match_args__ = ("c", "d") + c = Child() + c.a = 0 + c.b = 1 + match c: + case Parent(x, y): + z = 0 + self.assertIs(x, c.a) + self.assertIs(y, c.b) + self.assertEqual(z, 0) + + def test_patma_203(self): + class Parent: + __match_args__ = ("a", "b") + class Child(Parent): + __match_args__ = "c", "d" + c = Child() + c.a = 0 + c.b = 1 + match c: + case Parent(x, b=y): + z = 0 + self.assertIs(x, c.a) + self.assertIs(y, c.b) + self.assertEqual(z, 0) + + def test_patma_204(self): + def f(w): + match w: + case 42: + out = locals() + del out["w"] + return out + self.assertEqual(f(42), {}) + self.assertIs(f(0), None) + self.assertEqual(f(42.0), {}) + self.assertIs(f("42"), None) + + def test_patma_205(self): + def f(w): + match w: + case 42.0: + out = locals() + del out["w"] + return out + self.assertEqual(f(42.0), {}) + self.assertEqual(f(42), {}) + self.assertIs(f(0.0), None) + self.assertIs(f(0), None) + + def test_patma_206(self): + def f(w): + match w: + case 1 | 2 | 3: + out = locals() + del out["w"] + return out + self.assertEqual(f(1), {}) + self.assertEqual(f(2), {}) + self.assertEqual(f(3), {}) + self.assertEqual(f(3.0), {}) + self.assertIs(f(0), None) + self.assertIs(f(4), None) + self.assertIs(f("1"), None) + + def test_patma_207(self): + def f(w): + match w: + case [1, 2] | [3, 4]: + out = locals() + del out["w"] + return out + self.assertEqual(f([1, 2]), {}) + self.assertEqual(f([3, 4]), {}) + self.assertIs(f(42), None) + self.assertIs(f([2, 3]), None) + self.assertIs(f([1, 2, 3]), None) + self.assertEqual(f([1, 2.0]), {}) + + def test_patma_208(self): + def f(w): + match w: + case x: + out = locals() + del out["w"] + return out + self.assertEqual(f(42), {"x": 42}) + self.assertEqual(f((1, 2)), {"x": (1, 2)}) + self.assertEqual(f(None), {"x": None}) + + def test_patma_209(self): + def f(w): + match w: + case _: + out = locals() + del out["w"] + return out + self.assertEqual(f(42), {}) + self.assertEqual(f(None), {}) + self.assertEqual(f((1, 2)), {}) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_patma_210(self): + def f(w): + match w: + case (x, y, z): + out = locals() + del out["w"] + return out + self.assertEqual(f((1, 2, 3)), {"x": 1, "y": 2, "z": 3}) + self.assertIs(f((1, 2)), None) + self.assertIs(f((1, 2, 3, 4)), None) + self.assertIs(f(123), None) + self.assertIs(f("abc"), None) + self.assertIs(f(b"abc"), None) + self.assertEqual(f(array.array("b", b"abc")), {'x': 97, 'y': 98, 'z': 99}) + self.assertEqual(f(memoryview(b"abc")), {"x": 97, "y": 98, "z": 99}) + self.assertIs(f(bytearray(b"abc")), None) + + def test_patma_211(self): + def f(w): + match w: + case {"x": x, "y": "y", "z": z}: + out = locals() + del out["w"] + return out + self.assertEqual(f({"x": "x", "y": "y", "z": "z"}), {"x": "x", "z": "z"}) + self.assertEqual(f({"x": "x", "y": "y", "z": "z", "a": "a"}), {"x": "x", "z": "z"}) + self.assertIs(f(({"x": "x", "y": "yy", "z": "z", "a": "a"})), None) + self.assertIs(f(({"x": "x", "y": "y"})), None) + + def test_patma_212(self): + def f(w): + match w: + case Point(int(xx), y="hello"): + out = locals() + del out["w"] + return out + self.assertEqual(f(Point(42, "hello")), {"xx": 42}) + + def test_patma_213(self): + def f(w): + match w: + case (p, q) as x: + out = locals() + del out["w"] + return out + self.assertEqual(f((1, 2)), {"p": 1, "q": 2, "x": (1, 2)}) + self.assertEqual(f([1, 2]), {"p": 1, "q": 2, "x": [1, 2]}) + self.assertIs(f(12), None) + self.assertIs(f((1, 2, 3)), None) + + def test_patma_214(self): + def f(): + match 42: + case 42: + return locals() + self.assertEqual(set(f()), set()) + + def test_patma_215(self): + def f(): + match 1: + case 1 | 2 | 3: + return locals() + self.assertEqual(set(f()), set()) + + def test_patma_216(self): + def f(): + match ...: + case _: + return locals() + self.assertEqual(set(f()), set()) + + def test_patma_217(self): + def f(): + match ...: + case abc: + return locals() + self.assertEqual(set(f()), {"abc"}) + + def test_patma_218(self): + def f(): + match ..., ...: + case a, b: + return locals() + self.assertEqual(set(f()), {"a", "b"}) + + def test_patma_219(self): + def f(): + match {"k": ..., "l": ...}: + case {"k": a, "l": b}: + return locals() + self.assertEqual(set(f()), {"a", "b"}) + + def test_patma_220(self): + def f(): + match Point(..., ...): + case Point(x, y=y): + return locals() + self.assertEqual(set(f()), {"x", "y"}) + + def test_patma_221(self): + def f(): + match ...: + case b as a: + return locals() + self.assertEqual(set(f()), {"a", "b"}) + + def test_patma_222(self): + def f(x): + match x: + case _: + return 0 + self.assertEqual(f(0), 0) + self.assertEqual(f(1), 0) + self.assertEqual(f(2), 0) + self.assertEqual(f(3), 0) + + def test_patma_223(self): + def f(x): + match x: + case 0: + return 0 + self.assertEqual(f(0), 0) + self.assertIs(f(1), None) + self.assertIs(f(2), None) + self.assertIs(f(3), None) + + def test_patma_224(self): + def f(x): + match x: + case 0: + return 0 + case _: + return 1 + self.assertEqual(f(0), 0) + self.assertEqual(f(1), 1) + self.assertEqual(f(2), 1) + self.assertEqual(f(3), 1) + + def test_patma_225(self): + def f(x): + match x: + case 0: + return 0 + case 1: + return 1 + self.assertEqual(f(0), 0) + self.assertEqual(f(1), 1) + self.assertIs(f(2), None) + self.assertIs(f(3), None) + + def test_patma_226(self): + def f(x): + match x: + case 0: + return 0 + case 1: + return 1 + case _: + return 2 + self.assertEqual(f(0), 0) + self.assertEqual(f(1), 1) + self.assertEqual(f(2), 2) + self.assertEqual(f(3), 2) + + def test_patma_227(self): + def f(x): + match x: + case 0: + return 0 + case 1: + return 1 + case 2: + return 2 + self.assertEqual(f(0), 0) + self.assertEqual(f(1), 1) + self.assertEqual(f(2), 2) + self.assertIs(f(3), None) + + def test_patma_228(self): + match(): + case(): + x = 0 + self.assertEqual(x, 0) + + def test_patma_229(self): + x = 0 + match(x): + case(x): + y = 0 + self.assertEqual(x, 0) + self.assertEqual(y, 0) + + def test_patma_230(self): + x = 0 + match x: + case False: + y = 0 + case 0: + y = 1 + self.assertEqual(x, 0) + self.assertEqual(y, 1) + + def test_patma_231(self): + x = 1 + match x: + case True: + y = 0 + case 1: + y = 1 + self.assertEqual(x, 1) + self.assertEqual(y, 1) + + def test_patma_232(self): + class Eq: + def __eq__(self, other): + return True + x = eq = Eq() + # None + y = None + match x: + case None: + y = 0 + self.assertIs(x, eq) + self.assertEqual(y, None) + # True + y = None + match x: + case True: + y = 0 + self.assertIs(x, eq) + self.assertEqual(y, None) + # False + y = None + match x: + case False: + y = 0 + self.assertIs(x, eq) + self.assertEqual(y, None) + + def test_patma_233(self): + x = False + match x: + case False: + y = 0 + self.assertIs(x, False) + self.assertEqual(y, 0) + + def test_patma_234(self): + x = True + match x: + case True: + y = 0 + self.assertIs(x, True) + self.assertEqual(y, 0) + + def test_patma_235(self): + x = None + match x: + case None: + y = 0 + self.assertIs(x, None) + self.assertEqual(y, 0) + + def test_patma_236(self): + x = 0 + match x: + case (0 as w) as z: + y = 0 + self.assertEqual(w, 0) + self.assertEqual(x, 0) + self.assertEqual(y, 0) + self.assertEqual(z, 0) + + def test_patma_237(self): + x = 0 + match x: + case (0 as w) as z: + y = 0 + self.assertEqual(w, 0) + self.assertEqual(x, 0) + self.assertEqual(y, 0) + self.assertEqual(z, 0) + + def test_patma_238(self): + x = ((0, 1), (2, 3)) + match x: + case ((a as b, c as d) as e) as w, ((f as g, h) as i) as z: + y = 0 + self.assertEqual(a, 0) + self.assertEqual(b, 0) + self.assertEqual(c, 1) + self.assertEqual(d, 1) + self.assertEqual(e, (0, 1)) + self.assertEqual(f, 2) + self.assertEqual(g, 2) + self.assertEqual(h, 3) + self.assertEqual(i, (2, 3)) + self.assertEqual(w, (0, 1)) + self.assertEqual(x, ((0, 1), (2, 3))) + self.assertEqual(y, 0) + self.assertEqual(z, (2, 3)) + + def test_patma_239(self): + x = collections.UserDict({0: 1, 2: 3}) + match x: + case {2: 3}: + y = 0 + self.assertEqual(x, {0: 1, 2: 3}) + self.assertEqual(y, 0) + + def test_patma_240(self): + x = collections.UserDict({0: 1, 2: 3}) + match x: + case {2: 3, **z}: + y = 0 + self.assertEqual(x, {0: 1, 2: 3}) + self.assertEqual(y, 0) + self.assertEqual(z, {0: 1}) + + # TODO: RUSTPYTHON + # def test_patma_241(self): + # x = [[{0: 0}]] + # match x: + # case list([({-0-0j: int(real=0+0j, imag=0-0j) | (1) as z},)]): + # y = 0 + # self.assertEqual(x, [[{0: 0}]]) + # self.assertEqual(y, 0) + # self.assertEqual(z, 0) + + def test_patma_242(self): + x = range(3) + match x: + case [y, *_, z]: + w = 0 + self.assertEqual(w, 0) + self.assertEqual(x, range(3)) + self.assertEqual(y, 0) + self.assertEqual(z, 2) + + def test_patma_243(self): + x = range(3) + match x: + case [_, *_, y]: + z = 0 + self.assertEqual(x, range(3)) + self.assertEqual(y, 2) + self.assertEqual(z, 0) + + def test_patma_244(self): + x = range(3) + match x: + case [*_, y]: + z = 0 + self.assertEqual(x, range(3)) + self.assertEqual(y, 2) + self.assertEqual(z, 0) + + def test_patma_245(self): + x = {"y": 1} + match x: + case {"y": (0 as y) | (1 as y)}: + z = 0 + self.assertEqual(x, {"y": 1}) + self.assertEqual(y, 1) + self.assertEqual(z, 0) + + def test_patma_246(self): + def f(x): + match x: + case ((a, b, c, d, e, f, g, h, i, 9) | + (h, g, i, a, b, d, e, c, f, 10) | + (g, b, a, c, d, -5, e, h, i, f) | + (-1, d, f, b, g, e, i, a, h, c)): + w = 0 + out = locals() + del out["x"] + return out + alts = [ + dict(a=0, b=1, c=2, d=3, e=4, f=5, g=6, h=7, i=8, w=0), + dict(h=1, g=2, i=3, a=4, b=5, d=6, e=7, c=8, f=9, w=0), + dict(g=0, b=-1, a=-2, c=-3, d=-4, e=-6, h=-7, i=-8, f=-9, w=0), + dict(d=-2, f=-3, b=-4, g=-5, e=-6, i=-7, a=-8, h=-9, c=-10, w=0), + dict(), + ] + self.assertEqual(f(range(10)), alts[0]) + self.assertEqual(f(range(1, 11)), alts[1]) + self.assertEqual(f(range(0, -10, -1)), alts[2]) + self.assertEqual(f(range(-1, -11, -1)), alts[3]) + self.assertEqual(f(range(10, 20)), alts[4]) + + def test_patma_247(self): + def f(x): + match x: + case [y, (a, b, c, d, e, f, g, h, i, 9) | + (h, g, i, a, b, d, e, c, f, 10) | + (g, b, a, c, d, -5, e, h, i, f) | + (-1, d, f, b, g, e, i, a, h, c), z]: + w = 0 + out = locals() + del out["x"] + return out + alts = [ + dict(a=0, b=1, c=2, d=3, e=4, f=5, g=6, h=7, i=8, w=0, y=False, z=True), + dict(h=1, g=2, i=3, a=4, b=5, d=6, e=7, c=8, f=9, w=0, y=False, z=True), + dict(g=0, b=-1, a=-2, c=-3, d=-4, e=-6, h=-7, i=-8, f=-9, w=0, y=False, z=True), + dict(d=-2, f=-3, b=-4, g=-5, e=-6, i=-7, a=-8, h=-9, c=-10, w=0, y=False, z=True), + dict(), + ] + self.assertEqual(f((False, range(10), True)), alts[0]) + self.assertEqual(f((False, range(1, 11), True)), alts[1]) + self.assertEqual(f((False, range(0, -10, -1), True)), alts[2]) + self.assertEqual(f((False, range(-1, -11, -1), True)), alts[3]) + self.assertEqual(f((False, range(10, 20), True)), alts[4]) + + def test_patma_248(self): + class C(dict): + @staticmethod + def get(key, default=None): + return 'bar' + + x = C({'foo': 'bar'}) + match x: + case {'foo': bar}: + y = bar + + self.assertEqual(y, 'bar') + + def test_patma_249(self): + class C: + __attr = "eggs" # mangled to _C__attr + _Outer__attr = "bacon" + class Outer: + def f(self, x): + match x: + # looks up __attr, not _C__attr or _Outer__attr + case C(__attr=y): + return y + c = C() + setattr(c, "__attr", "spam") # setattr is needed because we're in a class scope + self.assertEqual(Outer().f(c), "spam") + + def test_patma_250(self): + def f(x): + match x: + case {"foo": y} if y >= 0: + return True + case {"foo": y} if y < 0: + return False + + self.assertIs(f({"foo": 1}), True) + self.assertIs(f({"foo": -1}), False) + + def test_patma_251(self): + def f(v, x): + match v: + case x.attr if x.attr >= 0: + return True + case x.attr if x.attr < 0: + return False + case _: + return None + + class X: + def __init__(self, attr): + self.attr = attr + + self.assertIs(f(1, X(1)), True) + self.assertIs(f(-1, X(-1)), False) + self.assertIs(f(1, X(-1)), None) + + def test_patma_252(self): + # Side effects must be possible in guards: + effects = [] + def lt(x, y): + effects.append((x, y)) + return x < y + + res = None + match {"foo": 1}: + case {"foo": x} if lt(x, 0): + res = 0 + case {"foo": x} if lt(x, 1): + res = 1 + case {"foo": x} if lt(x, 2): + res = 2 + + self.assertEqual(res, 2) + self.assertEqual(effects, [(1, 0), (1, 1), (1, 2)]) + + def test_patma_253(self): + def f(v): + match v: + case [x] | x: + return x + + self.assertEqual(f(1), 1) + self.assertEqual(f([1]), 1) + + def test_patma_254(self): + def f(v): + match v: + case {"x": x} | x: + return x + + self.assertEqual(f(1), 1) + self.assertEqual(f({"x": 1}), 1) + + def test_patma_255(self): + x = [] + match x: + case [] as z if z.append(None): + y = 0 + case [None]: + y = 1 + self.assertEqual(x, [None]) + self.assertEqual(y, 1) + self.assertIs(z, x) + + def test_patma_runtime_checkable_protocol(self): + # Runtime-checkable protocol + from typing import Protocol, runtime_checkable + + @runtime_checkable + class P(Protocol): + x: int + y: int + + class A: + def __init__(self, x: int, y: int): + self.x = x + self.y = y + + class B(A): ... + + for cls in (A, B): + with self.subTest(cls=cls.__name__): + inst = cls(1, 2) + w = 0 + match inst: + case P() as p: + self.assertIsInstance(p, cls) + self.assertEqual(p.x, 1) + self.assertEqual(p.y, 2) + w = 1 + self.assertEqual(w, 1) + + q = 0 + match inst: + case P(x=x, y=y): + self.assertEqual(x, 1) + self.assertEqual(y, 2) + q = 1 + self.assertEqual(q, 1) + + + def test_patma_generic_protocol(self): + # Runtime-checkable generic protocol + from typing import Generic, TypeVar, Protocol, runtime_checkable + + T = TypeVar('T') # not using PEP695 to be able to backport changes + + @runtime_checkable + class P(Protocol[T]): + a: T + b: T + + class A: + def __init__(self, x: int, y: int): + self.x = x + self.y = y + + class G(Generic[T]): + def __init__(self, x: T, y: T): + self.x = x + self.y = y + + for cls in (A, G): + with self.subTest(cls=cls.__name__): + inst = cls(1, 2) + w = 0 + match inst: + case P(): + w = 1 + self.assertEqual(w, 0) + + def test_patma_protocol_with_match_args(self): + # Runtime-checkable protocol with `__match_args__` + from typing import Protocol, runtime_checkable + + # Used to fail before + # https://github.com/python/cpython/issues/110682 + @runtime_checkable + class P(Protocol): + __match_args__ = ('x', 'y') + x: int + y: int + + class A: + def __init__(self, x: int, y: int): + self.x = x + self.y = y + + class B(A): ... + + for cls in (A, B): + with self.subTest(cls=cls.__name__): + inst = cls(1, 2) + w = 0 + match inst: + case P() as p: + self.assertIsInstance(p, cls) + self.assertEqual(p.x, 1) + self.assertEqual(p.y, 2) + w = 1 + self.assertEqual(w, 1) + + q = 0 + match inst: + case P(x=x, y=y): + self.assertEqual(x, 1) + self.assertEqual(y, 2) + q = 1 + self.assertEqual(q, 1) + + j = 0 + match inst: + case P(x=1, y=2): + j = 1 + self.assertEqual(j, 1) + + g = 0 + match inst: + case P(x, y): + self.assertEqual(x, 1) + self.assertEqual(y, 2) + g = 1 + self.assertEqual(g, 1) + + h = 0 + match inst: + case P(1, 2): + h = 1 + self.assertEqual(h, 1) + + +class TestSyntaxErrors(unittest.TestCase): + + def assert_syntax_error(self, code: str): + with self.assertRaises(SyntaxError): + compile(inspect.cleandoc(code), "<test>", "exec") + + def test_alternative_patterns_bind_different_names_0(self): + self.assert_syntax_error(""" + match ...: + case "a" | a: + pass + """) + + def test_alternative_patterns_bind_different_names_1(self): + self.assert_syntax_error(""" + match ...: + case [a, [b] | [c] | [d]]: + pass + """) + + + def test_attribute_name_repeated_in_class_pattern(self): + self.assert_syntax_error(""" + match ...: + case Class(a=_, a=_): + pass + """) + + def test_imaginary_number_required_in_complex_literal_0(self): + self.assert_syntax_error(""" + match ...: + case 0+0: + pass + """) + + def test_imaginary_number_required_in_complex_literal_1(self): + self.assert_syntax_error(""" + match ...: + case {0+0: _}: + pass + """) + + def test_invalid_syntax_0(self): + self.assert_syntax_error(""" + match ...: + case {**rest, "key": value}: + pass + """) + + def test_invalid_syntax_1(self): + self.assert_syntax_error(""" + match ...: + case {"first": first, **rest, "last": last}: + pass + """) + + def test_invalid_syntax_2(self): + self.assert_syntax_error(""" + match ...: + case {**_}: + pass + """) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_invalid_syntax_3(self): + self.assert_syntax_error(""" + match ...: + case 42 as _: + pass + """) + + def test_len1_tuple_sequence_pattern_comma(self): + # correct syntax would be `case(*x,):` + self.assert_syntax_error(""" + match ...: + case (*x): + pass + """) + + def test_mapping_pattern_keys_may_only_match_literals_and_attribute_lookups(self): + self.assert_syntax_error(""" + match ...: + case {f"": _}: + pass + """) + + def test_multiple_assignments_to_name_in_pattern_0(self): + self.assert_syntax_error(""" + match ...: + case a, a: + pass + """) + + def test_multiple_assignments_to_name_in_pattern_1(self): + self.assert_syntax_error(""" + match ...: + case {"k": a, "l": a}: + pass + """) + + def test_multiple_assignments_to_name_in_pattern_2(self): + self.assert_syntax_error(""" + match ...: + case MyClass(x, x): + pass + """) + + def test_multiple_assignments_to_name_in_pattern_3(self): + self.assert_syntax_error(""" + match ...: + case MyClass(x=x, y=x): + pass + """) + + def test_multiple_assignments_to_name_in_pattern_4(self): + self.assert_syntax_error(""" + match ...: + case MyClass(x, y=x): + pass + """) + + def test_multiple_assignments_to_name_in_pattern_5(self): + self.assert_syntax_error(""" + match ...: + case a as a: + pass + """) + + def test_multiple_starred_names_in_sequence_pattern_0(self): + self.assert_syntax_error(""" + match ...: + case *a, b, *c, d, *e: + pass + """) + + def test_multiple_starred_names_in_sequence_pattern_1(self): + self.assert_syntax_error(""" + match ...: + case a, *b, c, *d, e: + pass + """) + + def test_name_capture_makes_remaining_patterns_unreachable_0(self): + self.assert_syntax_error(""" + match ...: + case a | "a": + pass + """) + + def test_name_capture_makes_remaining_patterns_unreachable_1(self): + self.assert_syntax_error(""" + match 42: + case x: + pass + case y: + pass + """) + + def test_name_capture_makes_remaining_patterns_unreachable_2(self): + self.assert_syntax_error(""" + match ...: + case x | [_ as x] if x: + pass + """) + + def test_name_capture_makes_remaining_patterns_unreachable_3(self): + self.assert_syntax_error(""" + match ...: + case x: + pass + case [x] if x: + pass + """) + + def test_name_capture_makes_remaining_patterns_unreachable_4(self): + self.assert_syntax_error(""" + match ...: + case x: + pass + case _: + pass + """) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_patterns_may_only_match_literals_and_attribute_lookups_0(self): + self.assert_syntax_error(""" + match ...: + case f"": + pass + """) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_patterns_may_only_match_literals_and_attribute_lookups_1(self): + self.assert_syntax_error(""" + match ...: + case f"{x}": + pass + """) + + def test_real_number_required_in_complex_literal_0(self): + self.assert_syntax_error(""" + match ...: + case 0j+0: + pass + """) + + def test_real_number_required_in_complex_literal_1(self): + self.assert_syntax_error(""" + match ...: + case 0j+0j: + pass + """) + + def test_real_number_required_in_complex_literal_2(self): + self.assert_syntax_error(""" + match ...: + case {0j+0: _}: + pass + """) + + def test_real_number_required_in_complex_literal_3(self): + self.assert_syntax_error(""" + match ...: + case {0j+0j: _}: + pass + """) + + def test_real_number_multiple_ops(self): + self.assert_syntax_error(""" + match ...: + case 0 + 0j + 0: + pass + """) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_real_number_wrong_ops(self): + for op in ["*", "/", "@", "**", "%", "//"]: + with self.subTest(op=op): + self.assert_syntax_error(f""" + match ...: + case 0 {op} 0j: + pass + """) + self.assert_syntax_error(f""" + match ...: + case 0j {op} 0: + pass + """) + self.assert_syntax_error(f""" + match ...: + case -0j {op} 0: + pass + """) + self.assert_syntax_error(f""" + match ...: + case 0j {op} -0: + pass + """) + + def test_wildcard_makes_remaining_patterns_unreachable_0(self): + self.assert_syntax_error(""" + match ...: + case _ | _: + pass + """) + + def test_wildcard_makes_remaining_patterns_unreachable_1(self): + self.assert_syntax_error(""" + match ...: + case (_ as x) | [x]: + pass + """) + + def test_wildcard_makes_remaining_patterns_unreachable_2(self): + self.assert_syntax_error(""" + match ...: + case _ | _ if condition(): + pass + """) + + def test_wildcard_makes_remaining_patterns_unreachable_3(self): + self.assert_syntax_error(""" + match ...: + case _: + pass + case None: + pass + """) + + def test_wildcard_makes_remaining_patterns_unreachable_4(self): + self.assert_syntax_error(""" + match ...: + case (None | _) | _: + pass + """) + + def test_wildcard_makes_remaining_patterns_unreachable_5(self): + self.assert_syntax_error(""" + match ...: + case _ | (True | False): + pass + """) + + def test_mapping_pattern_duplicate_key(self): + self.assert_syntax_error(""" + match ...: + case {"a": _, "a": _}: + pass + """) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_mapping_pattern_duplicate_key_edge_case0(self): + self.assert_syntax_error(""" + match ...: + case {0: _, False: _}: + pass + """) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_mapping_pattern_duplicate_key_edge_case1(self): + self.assert_syntax_error(""" + match ...: + case {0: _, 0.0: _}: + pass + """) + + def test_mapping_pattern_duplicate_key_edge_case2(self): + self.assert_syntax_error(""" + match ...: + case {0: _, -0: _}: + pass + """) + + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_mapping_pattern_duplicate_key_edge_case3(self): + self.assert_syntax_error(""" + match ...: + case {0: _, 0j: _}: + pass + """) + +class TestTypeErrors(unittest.TestCase): + + def test_accepts_positional_subpatterns_0(self): + class Class: + __match_args__ = () + x = Class() + y = z = None + with self.assertRaises(TypeError): + match x: + case Class(y): + z = 0 + self.assertIs(y, None) + self.assertIs(z, None) + + def test_accepts_positional_subpatterns_1(self): + x = range(10) + y = None + with self.assertRaises(TypeError): + match x: + case range(10): + y = 0 + self.assertEqual(x, range(10)) + self.assertIs(y, None) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_got_multiple_subpatterns_for_attribute_0(self): + class Class: + __match_args__ = ("a", "a") + a = None + x = Class() + w = y = z = None + with self.assertRaises(TypeError): + match x: + case Class(y, z): + w = 0 + self.assertIs(w, None) + self.assertIs(y, None) + self.assertIs(z, None) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_got_multiple_subpatterns_for_attribute_1(self): + class Class: + __match_args__ = ("a",) + a = None + x = Class() + w = y = z = None + with self.assertRaises(TypeError): + match x: + case Class(y, a=z): + w = 0 + self.assertIs(w, None) + self.assertIs(y, None) + self.assertIs(z, None) + + def test_match_args_elements_must_be_strings(self): + class Class: + __match_args__ = (None,) + x = Class() + y = z = None + with self.assertRaises(TypeError): + match x: + case Class(y): + z = 0 + self.assertIs(y, None) + self.assertIs(z, None) + + def test_match_args_must_be_a_tuple_0(self): + class Class: + __match_args__ = None + x = Class() + y = z = None + with self.assertRaises(TypeError): + match x: + case Class(y): + z = 0 + self.assertIs(y, None) + self.assertIs(z, None) + + def test_match_args_must_be_a_tuple_1(self): + class Class: + __match_args__ = "XYZ" + x = Class() + y = z = None + with self.assertRaises(TypeError): + match x: + case Class(y): + z = 0 + self.assertIs(y, None) + self.assertIs(z, None) + + def test_match_args_must_be_a_tuple_2(self): + class Class: + __match_args__ = ["spam", "eggs"] + spam = 0 + eggs = 1 + x = Class() + w = y = z = None + with self.assertRaises(TypeError): + match x: + case Class(y, z): + w = 0 + self.assertIs(w, None) + self.assertIs(y, None) + self.assertIs(z, None) + + def test_class_pattern_not_type(self): + w = None + with self.assertRaises(TypeError): + match 1: + case max(0, 1): + w = 0 + self.assertIsNone(w) + + def test_regular_protocol(self): + from typing import Protocol + class P(Protocol): ... + msg = ( + 'Instance and class checks can only be used ' + 'with @runtime_checkable protocols' + ) + w = None + with self.assertRaisesRegex(TypeError, msg): + match 1: + case P(): + w = 0 + self.assertIsNone(w) + + def test_positional_patterns_with_regular_protocol(self): + from typing import Protocol + class P(Protocol): + x: int # no `__match_args__` + y: int + class A: + x = 1 + y = 2 + w = None + with self.assertRaises(TypeError): + match A(): + case P(x, y): + w = 0 + self.assertIsNone(w) + + +class TestValueErrors(unittest.TestCase): + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_mapping_pattern_checks_duplicate_key_1(self): + class Keys: + KEY = "a" + x = {"a": 0, "b": 1} + w = y = z = None + with self.assertRaises(ValueError): + match x: + case {Keys.KEY: y, "a": z}: + w = 0 + self.assertIs(w, None) + self.assertIs(y, None) + self.assertIs(z, None) + +class TestSourceLocations(unittest.TestCase): + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_jump_threading(self): + # See gh-123048 + def f(): + x = 0 + v = 1 + match v: + case 1: + if x < 0: + x = 1 + case 2: + if x < 0: + x = 1 + x += 1 + + for inst in dis.get_instructions(f): + if inst.opcode in dis.hasjump: + self.assertIsNotNone(inst.positions.lineno, "jump without location") + +class TestTracing(unittest.TestCase): + + @staticmethod + def _trace(func, *args, **kwargs): + actual_linenos = [] + + def trace(frame, event, arg): + if event == "line" and frame.f_code.co_name == func.__name__: + assert arg is None + relative_lineno = frame.f_lineno - func.__code__.co_firstlineno + actual_linenos.append(relative_lineno) + return trace + + old_trace = sys.gettrace() + sys.settrace(trace) + try: + func(*args, **kwargs) + finally: + sys.settrace(old_trace) + return actual_linenos + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_default_wildcard(self): + def f(command): # 0 + match command.split(): # 1 + case ["go", direction] if direction in "nesw": # 2 + return f"go {direction}" # 3 + case ["go", _]: # 4 + return "no go" # 5 + case _: # 6 + return "default" # 7 + + self.assertListEqual(self._trace(f, "go n"), [1, 2, 3]) + self.assertListEqual(self._trace(f, "go x"), [1, 2, 4, 5]) + self.assertListEqual(self._trace(f, "spam"), [1, 2, 4, 6, 7]) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_default_capture(self): + def f(command): # 0 + match command.split(): # 1 + case ["go", direction] if direction in "nesw": # 2 + return f"go {direction}" # 3 + case ["go", _]: # 4 + return "no go" # 5 + case x: # 6 + return x # 7 + + self.assertListEqual(self._trace(f, "go n"), [1, 2, 3]) + self.assertListEqual(self._trace(f, "go x"), [1, 2, 4, 5]) + self.assertListEqual(self._trace(f, "spam"), [1, 2, 4, 6, 7]) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_no_default(self): + def f(command): # 0 + match command.split(): # 1 + case ["go", direction] if direction in "nesw": # 2 + return f"go {direction}" # 3 + case ["go", _]: # 4 + return "no go" # 5 + + self.assertListEqual(self._trace(f, "go n"), [1, 2, 3]) + self.assertListEqual(self._trace(f, "go x"), [1, 2, 4, 5]) + self.assertListEqual(self._trace(f, "spam"), [1, 2, 4]) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_only_default_wildcard(self): + def f(command): # 0 + match command.split(): # 1 + case _: # 2 + return "default" # 3 + + self.assertListEqual(self._trace(f, "go n"), [1, 2, 3]) + self.assertListEqual(self._trace(f, "go x"), [1, 2, 3]) + self.assertListEqual(self._trace(f, "spam"), [1, 2, 3]) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_only_default_capture(self): + def f(command): # 0 + match command.split(): # 1 + case x: # 2 + return x # 3 + + self.assertListEqual(self._trace(f, "go n"), [1, 2, 3]) + self.assertListEqual(self._trace(f, "go x"), [1, 2, 3]) + self.assertListEqual(self._trace(f, "spam"), [1, 2, 3]) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_unreachable_code(self): + def f(command): # 0 + match command: # 1 + case 1: # 2 + if False: # 3 + return 1 # 4 + case _: # 5 + if False: # 6 + return 0 # 7 + + self.assertListEqual(self._trace(f, 1), [1, 2, 3]) + self.assertListEqual(self._trace(f, 0), [1, 2, 5, 6]) + + def test_parser_deeply_nested_patterns(self): + # Deeply nested patterns can cause exponential backtracking when parsing. + # See gh-93671 for more information. + + levels = 100 + + patterns = [ + "A" + "(" * levels + ")" * levels, + "{1:" * levels + "1" + "}" * levels, + "[" * levels + "1" + "]" * levels, + ] + + for pattern in patterns: + with self.subTest(pattern): + code = inspect.cleandoc(""" + match None: + case {}: + pass + """.format(pattern)) + compile(code, "<string>", "exec") + + +if __name__ == "__main__": + """ + # From inside environment using this Python, with pyperf installed: + sudo $(which pyperf) system tune && \ + $(which python) -m test.test_patma --rigorous; \ + sudo $(which pyperf) system reset + """ + import pyperf + + + class PerfPatma(TestPatma): + + def assertEqual(*_, **__): + pass + + def assertIs(*_, **__): + pass + + def assertRaises(*_, **__): + assert False, "this test should be a method of a different class!" + + def run_perf(self, count): + tests = [] + for attr in vars(TestPatma): + if attr.startswith("test_"): + tests.append(getattr(self, attr)) + tests *= count + start = pyperf.perf_counter() + for test in tests: + test() + return pyperf.perf_counter() - start + + + runner = pyperf.Runner() + runner.bench_time_func("patma", PerfPatma().run_perf) diff --git a/Lib/test/test_pickle.py b/Lib/test/test_pickle.py index e01ddcf0a8..ea51b9d091 100644 --- a/Lib/test/test_pickle.py +++ b/Lib/test/test_pickle.py @@ -16,6 +16,7 @@ from test.pickletester import AbstractHookTests from test.pickletester import AbstractUnpickleTests +from test.pickletester import AbstractPicklingErrorTests from test.pickletester import AbstractPickleTests from test.pickletester import AbstractPickleModuleTests from test.pickletester import AbstractPersistentPicklerTests @@ -40,15 +41,15 @@ class PyPickleTests(AbstractPickleModuleTests, unittest.TestCase): Pickler = pickle._Pickler Unpickler = pickle._Unpickler - # TODO: RUSTPYTHON, AttributeError: module 'pickle' has no attribute 'PickleBuffer' + # TODO: RUSTPYTHON @unittest.expectedFailure - def test_dump_load_oob_buffers(self): # TODO: RUSTPYTHON, remove when this passes - super().test_dump_load_oob_buffers() # TODO: RUSTPYTHON, remove when this passes + def test_dump_load_oob_buffers(self): # TODO(RUSTPYTHON): Remove this test when it passes + return super().test_dump_load_oob_buffers() - # TODO: RUSTPYTHON, AttributeError: module 'pickle' has no attribute 'PickleBuffer' + # TODO: RUSTPYTHON @unittest.expectedFailure - def test_dumps_loads_oob_buffers(self): # TODO: RUSTPYTHON, remove when this passes - super().test_dumps_loads_oob_buffers() # TODO: RUSTPYTHON, remove when this passes + def test_dumps_loads_oob_buffers(self): # TODO(RUSTPYTHON): Remove this test when it passes + return super().test_dumps_loads_oob_buffers() class PyUnpicklerTests(AbstractUnpickleTests, unittest.TestCase): @@ -59,71 +60,63 @@ class PyUnpicklerTests(AbstractUnpickleTests, unittest.TestCase): AttributeError, ValueError, struct.error, IndexError, ImportError) - # TODO: RUSTPYTHON, AssertionError: ValueError not raised - @unittest.expectedFailure - def test_badly_escaped_string(self): # TODO: RUSTPYTHON, remove when this passes - super().test_badly_escaped_string() # TODO: RUSTPYTHON, remove when this passes - - # TODO: RUSTPYTHON, AssertionError - @unittest.expectedFailure - def test_correctly_quoted_string(self): # TODO: RUSTPYTHON, remove when this passes - super().test_correctly_quoted_string() # TODO: RUSTPYTHON, remove when this passes - - # TODO: RUSTPYTHON, AssertionError - @unittest.expectedFailure - def test_load_python2_str_as_bytes(self): # TODO: RUSTPYTHON, remove when this passes - super().test_load_python2_str_as_bytes() # TODO: RUSTPYTHON, remove when this passes - def loads(self, buf, **kwds): f = io.BytesIO(buf) u = self.unpickler(f, **kwds) return u.load() + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_badly_escaped_string(self): # TODO(RUSTPYTHON): Remove this test when it passes + return super().test_badly_escaped_string() -class PyPicklerTests(AbstractPickleTests, unittest.TestCase): + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_correctly_quoted_string(self): # TODO(RUSTPYTHON): Remove this test when it passes + return super().test_correctly_quoted_string() + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_load_python2_str_as_bytes(self): # TODO(RUSTPYTHON): Remove this test when it passes + return super().test_load_python2_str_as_bytes() + + +class PyPicklingErrorTests(AbstractPicklingErrorTests, unittest.TestCase): pickler = pickle._Pickler - unpickler = pickle._Unpickler - # TODO: RUSTPYTHON, AttributeError: module 'pickle' has no attribute 'PickleBuffer' - @unittest.expectedFailure - def test_buffer_callback_error(self): # TODO: RUSTPYTHON, remove when this passes - super().test_buffer_callback_error() # TODO: RUSTPYTHON, remove when this passes + def dumps(self, arg, proto=None, **kwargs): + f = io.BytesIO() + p = self.pickler(f, proto, **kwargs) + p.dump(arg) + f.seek(0) + return bytes(f.read()) - # TODO: RUSTPYTHON, AttributeError: module 'pickle' has no attribute 'PickleBuffer' + # TODO: RUSTPYTHON @unittest.expectedFailure - def test_buffers_error(self): # TODO: RUSTPYTHON, remove when this passes - super().test_buffers_error() # TODO: RUSTPYTHON, remove when this passes + def test_picklebuffer_error(self): # TODO(RUSTPYTHON): Remove this test when it passes + return super().test_picklebuffer_error() - # TODO: RUSTPYTHON, AssertionError + # TODO: RUSTPYTHON @unittest.expectedFailure - def test_complex_newobj_ex(self): # TODO: RUSTPYTHON, remove when this passes - super().test_complex_newobj_ex() # TODO: RUSTPYTHON, remove when this passes + def test_bad_getattr(self): # TODO(RUSTPYTHON): Remove this test when it passes + return super().test_bad_getattr() - # TODO: RUSTPYTHON, TypeError: cannot pickle 'method' object + # TODO: RUSTPYTHON @unittest.expectedFailure - def test_in_band_buffers(self): # TODO: RUSTPYTHON, remove when this passes - super().test_in_band_buffers() # TODO: RUSTPYTHON, remove when this passes + def test_buffer_callback_error(self): # TODO(RUSTPYTHON): Remove this test when it passes + return super().test_buffer_callback_error() - # TODO: RUSTPYTHON, AttributeError: module 'pickle' has no attribute 'PickleBuffer' + # TODO: RUSTPYTHON @unittest.expectedFailure - def test_oob_buffers(self): # TODO: RUSTPYTHON, remove when this passes - super().test_oob_buffers() # TODO: RUSTPYTHON, remove when this passes + def test_non_continuous_buffer(self): # TODO(RUSTPYTHON): Remove this test when it passes + return super().test_non_continuous_buffer() - # TODO: RUSTPYTHON, AttributeError: module 'pickle' has no attribute 'PickleBuffer' - @unittest.expectedFailure - def test_oob_buffers_writable_to_readonly(self): # TODO: RUSTPYTHON, remove when this passes - super().test_oob_buffers_writable_to_readonly() # TODO: RUSTPYTHON, remove when this passes - # TODO: RUSTPYTHON, TypeError: Expected type 'bytes', not 'bytearray' - @unittest.expectedFailure - def test_optional_frames(self): # TODO: RUSTPYTHON, remove when this passes - super().test_optional_frames() # TODO: RUSTPYTHON, remove when this passes +class PyPicklerTests(AbstractPickleTests, unittest.TestCase): - # TODO: RUSTPYTHON, AttributeError: module 'pickle' has no attribute 'PickleBuffer' - @unittest.expectedFailure - def test_picklebuffer_error(self): # TODO: RUSTPYTHON, remove when this passes - super().test_picklebuffer_error() # TODO: RUSTPYTHON, remove when this passes + pickler = pickle._Pickler + unpickler = pickle._Unpickler def dumps(self, arg, proto=None, **kwargs): f = io.BytesIO() @@ -137,69 +130,99 @@ def loads(self, buf, **kwds): u = self.unpickler(f, **kwds) return u.load() + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_c_methods(self): # TODO(RUSTPYTHON): Remove this test when it passes + return super().test_c_methods() -class InMemoryPickleTests(AbstractPickleTests, AbstractUnpickleTests, - BigmemPickleTests, unittest.TestCase): + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_complex_newobj_ex(self): # TODO(RUSTPYTHON): Remove this test when it passes + return super().test_complex_newobj_ex() - bad_stack_errors = (pickle.UnpicklingError, IndexError) - truncated_errors = (pickle.UnpicklingError, EOFError, - AttributeError, ValueError, - struct.error, IndexError, ImportError) + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_py_methods(self): # TODO(RUSTPYTHON): Remove this test when it passes + return super().test_py_methods() + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_buffers_error(self): # TODO(RUSTPYTHON): Remove this test when it passes + return super().test_buffers_error() - # TODO: RUSTPYTHON, AssertionError: ValueError not raised + # TODO: RUSTPYTHON @unittest.expectedFailure - def test_badly_escaped_string(self): # TODO: RUSTPYTHON, remove when this passes - super().test_badly_escaped_string() # TODO: RUSTPYTHON, remove when this passes + def test_builtin_functions(self): # TODO(RUSTPYTHON): Remove this test when it passes + return super().test_builtin_functions() - # TODO: RUSTPYTHON, AttributeError: module 'pickle' has no attribute 'PickleBuffer' + # TODO: RUSTPYTHON @unittest.expectedFailure - def test_buffer_callback_error(self): # TODO: RUSTPYTHON, remove when this passes - super().test_buffer_callback_error() # TODO: RUSTPYTHON, remove when this passes + def test_bytearray_memoization(self): # TODO(RUSTPYTHON): Remove this test when it passes + return super().test_bytearray_memoization() - # TODO: RUSTPYTHON, AttributeError: module 'pickle' has no attribute 'PickleBuffer' + # TODO: RUSTPYTHON @unittest.expectedFailure - def test_buffers_error(self): # TODO: RUSTPYTHON, remove when this passes - super().test_buffers_error() # TODO: RUSTPYTHON, remove when this passes + def test_bytes_memoization(self): # TODO(RUSTPYTHON): Remove this test when it passes + return super().test_bytes_memoization() - # TODO: RUSTPYTHON, AssertionError + # TODO: RUSTPYTHON @unittest.expectedFailure - def test_complex_newobj_ex(self): # TODO: RUSTPYTHON, remove when this passes - super().test_complex_newobj_ex() # TODO: RUSTPYTHON, remove when this passes + def test_in_band_buffers(self): # TODO(RUSTPYTHON): Remove this test when it passes + return super().test_in_band_buffers() - # TODO: RUSTPYTHON, AssertionError + # TODO: RUSTPYTHON @unittest.expectedFailure - def test_correctly_quoted_string(self): # TODO: RUSTPYTHON, remove when this passes - super().test_correctly_quoted_string() # TODO: RUSTPYTHON, remove when this passes + def test_oob_buffers(self): # TODO(RUSTPYTHON): Remove this test when it passes + return super().test_oob_buffers() - # TODO: RUSTPYTHON, TypeError: cannot pickle 'method' object + # TODO: RUSTPYTHON @unittest.expectedFailure - def test_in_band_buffers(self): # TODO: RUSTPYTHON, remove when this passes - super().test_in_band_buffers() # TODO: RUSTPYTHON, remove when this passes + def test_oob_buffers_writable_to_readonly(self): # TODO(RUSTPYTHON): Remove this test when it passes + return super().test_oob_buffers_writable_to_readonly() - # TODO: RUSTPYTHON, AssertionError + # TODO: RUSTPYTHON @unittest.expectedFailure - def test_load_python2_str_as_bytes(self): # TODO: RUSTPYTHON, remove when this passes - super().test_load_python2_str_as_bytes() # TODO: RUSTPYTHON, remove when this passes + def test_buffers_error(self): # TODO(RUSTPYTHON): Remove this test when it passes + return super().test_buffers_error() - # TODO: RUSTPYTHON, AttributeError: module 'pickle' has no attribute 'PickleBuffer' + # TODO: RUSTPYTHON @unittest.expectedFailure - def test_oob_buffers(self): # TODO: RUSTPYTHON, remove when this passes - super().test_oob_buffers() # TODO: RUSTPYTHON, remove when this passes + def test_builtin_functions(self): # TODO(RUSTPYTHON): Remove this test when it passes + return super().test_builtin_functions() - # TODO: RUSTPYTHON, AttributeError: module 'pickle' has no attribute 'PickleBuffer' + # TODO: RUSTPYTHON @unittest.expectedFailure - def test_oob_buffers_writable_to_readonly(self): # TODO: RUSTPYTHON, remove when this passes - super().test_oob_buffers_writable_to_readonly() # TODO: RUSTPYTHON, remove when this passes + def test_bytearray_memoization(self): # TODO(RUSTPYTHON): Remove this test when it passes + return super().test_bytearray_memoization() - # TODO: RUSTPYTHON, TypeError: Expected type 'bytes', not 'bytearray' + # TODO: RUSTPYTHON @unittest.expectedFailure - def test_optional_frames(self): # TODO: RUSTPYTHON, remove when this passes - super().test_optional_frames() # TODO: RUSTPYTHON, remove when this passes + def test_bytes_memoization(self): # TODO(RUSTPYTHON): Remove this test when it passes + return super().test_bytes_memoization() - # TODO: RUSTPYTHON, AttributeError: module 'pickle' has no attribute 'PickleBuffer' + # TODO: RUSTPYTHON @unittest.expectedFailure - def test_picklebuffer_error(self): # TODO: RUSTPYTHON, remove when this passes - super().test_picklebuffer_error() # TODO: RUSTPYTHON, remove when this passes + def test_in_band_buffers(self): # TODO(RUSTPYTHON): Remove this test when it passes + return super().test_in_band_buffers() + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_oob_buffers(self): # TODO(RUSTPYTHON): Remove this test when it passes + return super().test_oob_buffers() + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_oob_buffers_writable_to_readonly(self): # TODO(RUSTPYTHON): Remove this test when it passes + return super().test_oob_buffers_writable_to_readonly() + + +class InMemoryPickleTests(AbstractPickleTests, AbstractUnpickleTests, + BigmemPickleTests, unittest.TestCase): + + bad_stack_errors = (pickle.UnpicklingError, IndexError) + truncated_errors = (pickle.UnpicklingError, EOFError, + AttributeError, ValueError, + struct.error, IndexError, ImportError) def dumps(self, arg, protocol=None, **kwargs): return pickle.dumps(arg, protocol, **kwargs) @@ -208,7 +231,73 @@ def loads(self, buf, **kwds): return pickle.loads(buf, **kwds) test_framed_write_sizes_with_delayed_writer = None + test_find_class = None + test_custom_find_class = None + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_c_methods(self): # TODO(RUSTPYTHON): Remove this test when it passes + return super().test_c_methods() + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_complex_newobj_ex(self): # TODO(RUSTPYTHON): Remove this test when it passes + return super().test_complex_newobj_ex() + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_badly_escaped_string(self): # TODO(RUSTPYTHON): Remove this test when it passes + return super().test_badly_escaped_string() + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_correctly_quoted_string(self): # TODO(RUSTPYTHON): Remove this test when it passes + return super().test_correctly_quoted_string() + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_load_python2_str_as_bytes(self): # TODO(RUSTPYTHON): Remove this test when it passes + return super().test_load_python2_str_as_bytes() + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_py_methods(self): # TODO(RUSTPYTHON): Remove this test when it passes + return super().test_py_methods() + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_oob_buffers_writable_to_readonly(self): # TODO(RUSTPYTHON): Remove this test when it passes + return super().test_oob_buffers_writable_to_readonly() + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_buffers_error(self): # TODO(RUSTPYTHON): Remove this test when it passes + return super().test_buffers_error() + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_builtin_functions(self): # TODO(RUSTPYTHON): Remove this test when it passes + return super().test_builtin_functions() + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_bytearray_memoization(self): # TODO(RUSTPYTHON): Remove this test when it passes + return super().test_bytearray_memoization() + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_bytes_memoization(self): # TODO(RUSTPYTHON): Remove this test when it passes + return super().test_bytes_memoization() + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_in_band_buffers(self): # TODO(RUSTPYTHON): Remove this test when it passes + return super().test_in_band_buffers() + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_oob_buffers(self): # TODO(RUSTPYTHON): Remove this test when it passes + return super().test_oob_buffers() class PersistentPicklerUnpicklerMixin(object): @@ -242,6 +331,7 @@ class PyIdPersPicklerTests(AbstractIdentityPersistentPicklerTests, pickler = pickle._Pickler unpickler = pickle._Unpickler + persistent_load_error = pickle.UnpicklingError @support.cpython_only def test_pickler_reference_cycle(self): @@ -296,7 +386,6 @@ class DispatchTable: support.gc_collect() self.assertIsNone(table_ref()) - @support.cpython_only def test_unpickler_reference_cycle(self): def check(Unpickler): @@ -326,6 +415,111 @@ def persistent_load(pid): return pid check(PersUnpickler) + def test_pickler_super(self): + class PersPickler(self.pickler): + def persistent_id(subself, obj): + called.append(obj) + self.assertIsNone(super().persistent_id(obj)) + return obj + + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + f = io.BytesIO() + pickler = PersPickler(f, proto) + called = [] + pickler.dump('abc') + self.assertEqual(called, ['abc']) + self.assertEqual(self.loads(f.getvalue()), 'abc') + + def test_unpickler_super(self): + class PersUnpickler(self.unpickler): + def persistent_load(subself, pid): + called.append(pid) + with self.assertRaises(self.persistent_load_error): + super().persistent_load(pid) + return pid + + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + unpickler = PersUnpickler(io.BytesIO(self.dumps('abc', proto))) + called = [] + self.assertEqual(unpickler.load(), 'abc') + self.assertEqual(called, ['abc']) + + def test_pickler_instance_attribute(self): + def persistent_id(obj): + called.append(obj) + return obj + + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + f = io.BytesIO() + pickler = self.pickler(f, proto) + called = [] + old_persistent_id = pickler.persistent_id + pickler.persistent_id = persistent_id + self.assertEqual(pickler.persistent_id, persistent_id) + pickler.dump('abc') + self.assertEqual(called, ['abc']) + self.assertEqual(self.loads(f.getvalue()), 'abc') + del pickler.persistent_id + self.assertEqual(pickler.persistent_id, old_persistent_id) + + def test_unpickler_instance_attribute(self): + def persistent_load(pid): + called.append(pid) + return pid + + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + unpickler = self.unpickler(io.BytesIO(self.dumps('abc', proto))) + called = [] + old_persistent_load = unpickler.persistent_load + unpickler.persistent_load = persistent_load + self.assertEqual(unpickler.persistent_load, persistent_load) + self.assertEqual(unpickler.load(), 'abc') + self.assertEqual(called, ['abc']) + del unpickler.persistent_load + self.assertEqual(unpickler.persistent_load, old_persistent_load) + + def test_pickler_super_instance_attribute(self): + class PersPickler(self.pickler): + def persistent_id(subself, obj): + raise AssertionError('should never be called') + def _persistent_id(subself, obj): + called.append(obj) + self.assertIsNone(super().persistent_id(obj)) + return obj + + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + f = io.BytesIO() + pickler = PersPickler(f, proto) + called = [] + old_persistent_id = pickler.persistent_id + pickler.persistent_id = pickler._persistent_id + self.assertEqual(pickler.persistent_id, pickler._persistent_id) + pickler.dump('abc') + self.assertEqual(called, ['abc']) + self.assertEqual(self.loads(f.getvalue()), 'abc') + del pickler.persistent_id + self.assertEqual(pickler.persistent_id, old_persistent_id) + + def test_unpickler_super_instance_attribute(self): + class PersUnpickler(self.unpickler): + def persistent_load(subself, pid): + raise AssertionError('should never be called') + def _persistent_load(subself, pid): + called.append(pid) + with self.assertRaises(self.persistent_load_error): + super().persistent_load(pid) + return pid + + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + unpickler = PersUnpickler(io.BytesIO(self.dumps('abc', proto))) + called = [] + old_persistent_load = unpickler.persistent_load + unpickler.persistent_load = unpickler._persistent_load + self.assertEqual(unpickler.persistent_load, unpickler._persistent_load) + self.assertEqual(unpickler.load(), 'abc') + self.assertEqual(called, ['abc']) + del unpickler.persistent_load + self.assertEqual(unpickler.persistent_load, old_persistent_load) class PyPicklerUnpicklerObjectTests(AbstractPicklerUnpicklerObjectTests, unittest.TestCase): @@ -365,6 +559,9 @@ class CUnpicklerTests(PyUnpicklerTests): bad_stack_errors = (pickle.UnpicklingError,) truncated_errors = (pickle.UnpicklingError,) + class CPicklingErrorTests(PyPicklingErrorTests): + pickler = _pickle.Pickler + class CPicklerTests(PyPicklerTests): pickler = _pickle.Pickler unpickler = _pickle.Unpickler @@ -376,6 +573,7 @@ class CPersPicklerTests(PyPersPicklerTests): class CIdPersPicklerTests(PyIdPersPicklerTests): pickler = _pickle.Pickler unpickler = _pickle.Unpickler + persistent_load_error = _pickle.UnpicklingError class CDumpPickle_LoadPickle(PyPicklerTests): pickler = _pickle.Pickler @@ -499,7 +697,9 @@ def recurse(deep): check_unpickler(recurse(1), 32, 20) check_unpickler(recurse(20), 32, 20) check_unpickler(recurse(50), 64, 60) - check_unpickler(recurse(100), 128, 140) + if not (support.is_wasi and support.Py_DEBUG): + # stack depth too shallow in pydebug WASI. + check_unpickler(recurse(100), 128, 140) u = unpickler(io.BytesIO(pickle.dumps('a', 0)), encoding='ASCII', errors='strict') @@ -642,6 +842,8 @@ def test_reverse_name_mapping(self): module, name = mapping(module, name) self.assertEqual((module, name), (module3, name3)) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_exceptions(self): self.assertEqual(mapping('exceptions', 'StandardError'), ('builtins', 'Exception')) @@ -659,13 +861,12 @@ def test_exceptions(self): if exc in (BlockingIOError, ResourceWarning, StopAsyncIteration, + PythonFinalizationError, RecursionError, EncodingWarning, BaseExceptionGroup, - ExceptionGroup): - continue - # TODO: RUSTPYTHON: fix name mapping for _IncompleteInputError - if exc is _IncompleteInputError: + ExceptionGroup, + _IncompleteInputError): continue if exc is not OSError and issubclass(exc, OSError): self.assertEqual(reverse_mapping('builtins', name), @@ -692,9 +893,8 @@ def test_multiprocessing_exceptions(self): self.assertEqual(mapping('multiprocessing', name), ('multiprocessing.context', name)) - def load_tests(loader, tests, pattern): - tests.addTest(doctest.DocTestSuite()) + tests.addTest(doctest.DocTestSuite(pickle)) return tests diff --git a/Lib/test/test_picklebuffer.py b/Lib/test/test_picklebuffer.py new file mode 100644 index 0000000000..a14f6a86b4 --- /dev/null +++ b/Lib/test/test_picklebuffer.py @@ -0,0 +1,179 @@ +"""Unit tests for the PickleBuffer object. + +Pickling tests themselves are in pickletester.py. +""" + +import gc +# TODO: RUSTPYTHON; Implment PickleBuffer +try: + from pickle import PickleBuffer +except ImportError: + PickleBuffer = None +import weakref +import unittest + +from test.support import import_helper + + +class B(bytes): + pass + + +class PickleBufferTest(unittest.TestCase): + + def check_memoryview(self, pb, equiv): + with memoryview(pb) as m: + with memoryview(equiv) as expected: + self.assertEqual(m.nbytes, expected.nbytes) + self.assertEqual(m.readonly, expected.readonly) + self.assertEqual(m.itemsize, expected.itemsize) + self.assertEqual(m.shape, expected.shape) + self.assertEqual(m.strides, expected.strides) + self.assertEqual(m.c_contiguous, expected.c_contiguous) + self.assertEqual(m.f_contiguous, expected.f_contiguous) + self.assertEqual(m.format, expected.format) + self.assertEqual(m.tobytes(), expected.tobytes()) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_constructor_failure(self): + with self.assertRaises(TypeError): + PickleBuffer() + with self.assertRaises(TypeError): + PickleBuffer("foo") + # Released memoryview fails taking a buffer + m = memoryview(b"foo") + m.release() + with self.assertRaises(ValueError): + PickleBuffer(m) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_basics(self): + pb = PickleBuffer(b"foo") + self.assertEqual(b"foo", bytes(pb)) + with memoryview(pb) as m: + self.assertTrue(m.readonly) + + pb = PickleBuffer(bytearray(b"foo")) + self.assertEqual(b"foo", bytes(pb)) + with memoryview(pb) as m: + self.assertFalse(m.readonly) + m[0] = 48 + self.assertEqual(b"0oo", bytes(pb)) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_release(self): + pb = PickleBuffer(b"foo") + pb.release() + with self.assertRaises(ValueError) as raises: + memoryview(pb) + self.assertIn("operation forbidden on released PickleBuffer object", + str(raises.exception)) + # Idempotency + pb.release() + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_cycle(self): + b = B(b"foo") + pb = PickleBuffer(b) + b.cycle = pb + wpb = weakref.ref(pb) + del b, pb + gc.collect() + self.assertIsNone(wpb()) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_ndarray_2d(self): + # C-contiguous + ndarray = import_helper.import_module("_testbuffer").ndarray + arr = ndarray(list(range(12)), shape=(4, 3), format='<i') + self.assertTrue(arr.c_contiguous) + self.assertFalse(arr.f_contiguous) + pb = PickleBuffer(arr) + self.check_memoryview(pb, arr) + # Non-contiguous + arr = arr[::2] + self.assertFalse(arr.c_contiguous) + self.assertFalse(arr.f_contiguous) + pb = PickleBuffer(arr) + self.check_memoryview(pb, arr) + # F-contiguous + arr = ndarray(list(range(12)), shape=(3, 4), strides=(4, 12), format='<i') + self.assertTrue(arr.f_contiguous) + self.assertFalse(arr.c_contiguous) + pb = PickleBuffer(arr) + self.check_memoryview(pb, arr) + + # Tests for PickleBuffer.raw() + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def check_raw(self, obj, equiv): + pb = PickleBuffer(obj) + with pb.raw() as m: + self.assertIsInstance(m, memoryview) + self.check_memoryview(m, equiv) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_raw(self): + for obj in (b"foo", bytearray(b"foo")): + with self.subTest(obj=obj): + self.check_raw(obj, obj) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_raw_ndarray(self): + # 1-D, contiguous + ndarray = import_helper.import_module("_testbuffer").ndarray + arr = ndarray(list(range(3)), shape=(3,), format='<h') + equiv = b"\x00\x00\x01\x00\x02\x00" + self.check_raw(arr, equiv) + # 2-D, C-contiguous + arr = ndarray(list(range(6)), shape=(2, 3), format='<h') + equiv = b"\x00\x00\x01\x00\x02\x00\x03\x00\x04\x00\x05\x00" + self.check_raw(arr, equiv) + # 2-D, F-contiguous + arr = ndarray(list(range(6)), shape=(2, 3), strides=(2, 4), + format='<h') + # Note this is different from arr.tobytes() + equiv = b"\x00\x00\x01\x00\x02\x00\x03\x00\x04\x00\x05\x00" + self.check_raw(arr, equiv) + # 0-D + arr = ndarray(456, shape=(), format='<i') + equiv = b'\xc8\x01\x00\x00' + self.check_raw(arr, equiv) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def check_raw_non_contiguous(self, obj): + pb = PickleBuffer(obj) + with self.assertRaisesRegex(BufferError, "non-contiguous"): + pb.raw() + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_raw_non_contiguous(self): + # 1-D + ndarray = import_helper.import_module("_testbuffer").ndarray + arr = ndarray(list(range(6)), shape=(6,), format='<i')[::2] + self.check_raw_non_contiguous(arr) + # 2-D + arr = ndarray(list(range(12)), shape=(4, 3), format='<i')[::2] + self.check_raw_non_contiguous(arr) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_raw_released(self): + pb = PickleBuffer(b"foo") + pb.release() + with self.assertRaises(ValueError) as raises: + pb.raw() + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_pickletools.py b/Lib/test/test_pickletools.py index f3b9dccb0d..492f57cce2 100644 --- a/Lib/test/test_pickletools.py +++ b/Lib/test/test_pickletools.py @@ -1,3 +1,4 @@ +import io import pickle import pickletools from test import support @@ -7,52 +8,6 @@ class OptimizedPickleTests(AbstractPickleTests, unittest.TestCase): - # TODO: RUSTPYTHON, AttributeError: module 'pickle' has no attribute 'PickleBuffer' - @unittest.expectedFailure - def test_buffer_callback_error(self): # TODO: RUSTPYTHON, remove when this passes - super().test_buffer_callback_error() - - # TODO: RUSTPYTHON, AttributeError: module 'pickle' has no attribute 'PickleBuffer' - @unittest.expectedFailure - def test_buffers_error(self): # TODO: RUSTPYTHON, remove when this passes - super().test_buffers_error() - - def test_compat_pickle(self): # TODO: RUSTPYTHON, remove when this passes - super().test_compat_pickle() - - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_complex_newobj_ex(self): # TODO: RUSTPYTHON, remove when this passes - super().test_complex_newobj_ex() - - # TODO: RUSTPYTHON, TypeError: cannot pickle 'method' object - @unittest.expectedFailure - def test_in_band_buffers(self): # TODO: RUSTPYTHON, remove when this passes - super().test_in_band_buffers() - - def test_notimplemented(self): # TODO: RUSTPYTHON, remove when this passes - super().test_notimplemented() - - # TODO: RUSTPYTHON, AttributeError: module 'pickle' has no attribute 'PickleBuffer' - @unittest.expectedFailure - def test_oob_buffers(self): # TODO: RUSTPYTHON, remove when this passes - super().test_oob_buffers() - - # TODO: RUSTPYTHON, AttributeError: module 'pickle' has no attribute 'PickleBuffer' - @unittest.expectedFailure - def test_oob_buffers_writable_to_readonly(self): # TODO: RUSTPYTHON, remove when this passes - super().test_oob_buffers_writable_to_readonly() - - # TODO: RUSTPYTHON, TypeError: Expected type 'bytes', not 'bytearray' - @unittest.expectedFailure - def test_optional_frames(self): # TODO: RUSTPYTHON, remove when this passes - super().test_optional_frames() - - # TODO: RUSTPYTHON, AttributeError: module 'pickle' has no attribute 'PickleBuffer' - @unittest.expectedFailure - def test_picklebuffer_error(self): # TODO: RUSTPYTHON, remove when this passes - super().test_picklebuffer_error() - def dumps(self, arg, proto=None, **kwargs): return pickletools.optimize(pickle.dumps(arg, proto, **kwargs)) @@ -107,6 +62,446 @@ def test_optimize_binput_and_memoize(self): self.assertIs(unpickled2[1], unpickled2[2]) self.assertNotIn(pickle.BINPUT, pickled2) + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_buffers_error(self): # TODO(RUSTPYTHON): Remove this test when it passes + return super().test_buffers_error() + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_builtin_functions(self): # TODO(RUSTPYTHON): Remove this test when it passes + return super().test_builtin_functions() + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_bytearray_memoization(self): # TODO(RUSTPYTHON): Remove this test when it passes + return super().test_bytearray_memoization() + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_bytes_memoization(self): # TODO(RUSTPYTHON): Remove this test when it passes + return super().test_bytes_memoization() + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_in_band_buffers(self): # TODO(RUSTPYTHON): Remove this test when it passes + return super().test_in_band_buffers() + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_oob_buffers(self): # TODO(RUSTPYTHON): Remove this test when it passes + return super().test_oob_buffers() + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_oob_buffers_writable_to_readonly(self): # TODO(RUSTPYTHON): Remove this test when it passes + return super().test_oob_buffers_writable_to_readonly() + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_py_methods(self): # TODO(RUSTPYTHON): Remove this test when it passes + return super().test_py_methods() + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_complex_newobj_ex(self): # TODO(RUSTPYTHON): Remove this test when it passes + return super().test_complex_newobj_ex() + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_c_methods(self): # TODO(RUSTPYTHON): Remove this test when it passes + return super().test_c_methods() + + +class SimpleReader: + def __init__(self, data): + self.data = data + self.pos = 0 + + def read(self, n): + data = self.data[self.pos: self.pos + n] + self.pos += n + return data + + def readline(self): + nl = self.data.find(b'\n', self.pos) + 1 + if not nl: + nl = len(self.data) + data = self.data[self.pos: nl] + self.pos = nl + return data + + +class GenopsTests(unittest.TestCase): + def test_genops(self): + it = pickletools.genops(b'(I123\nK\x12J\x12\x34\x56\x78t.') + self.assertEqual([(item[0].name,) + item[1:] for item in it], [ + ('MARK', None, 0), + ('INT', 123, 1), + ('BININT1', 0x12, 6), + ('BININT', 0x78563412, 8), + ('TUPLE', None, 13), + ('STOP', None, 14), + ]) + + def test_from_file(self): + f = io.BytesIO(b'prefix(I123\nK\x12J\x12\x34\x56\x78t.suffix') + self.assertEqual(f.read(6), b'prefix') + it = pickletools.genops(f) + self.assertEqual([(item[0].name,) + item[1:] for item in it], [ + ('MARK', None, 6), + ('INT', 123, 7), + ('BININT1', 0x12, 12), + ('BININT', 0x78563412, 14), + ('TUPLE', None, 19), + ('STOP', None, 20), + ]) + self.assertEqual(f.read(), b'suffix') + + def test_without_pos(self): + f = SimpleReader(b'(I123\nK\x12J\x12\x34\x56\x78t.') + it = pickletools.genops(f) + self.assertEqual([(item[0].name,) + item[1:] for item in it], [ + ('MARK', None, None), + ('INT', 123, None), + ('BININT1', 0x12, None), + ('BININT', 0x78563412, None), + ('TUPLE', None, None), + ('STOP', None, None), + ]) + + def test_no_stop(self): + it = pickletools.genops(b'N') + item = next(it) + self.assertEqual(item[0].name, 'NONE') + with self.assertRaisesRegex(ValueError, + 'pickle exhausted before seeing STOP'): + next(it) + + def test_truncated_data(self): + it = pickletools.genops(b'I123') + with self.assertRaisesRegex(ValueError, + 'no newline found when trying to read stringnl'): + next(it) + it = pickletools.genops(b'J\x12\x34') + with self.assertRaisesRegex(ValueError, + 'not enough data in stream to read int4'): + next(it) + + def test_unknown_opcode(self): + it = pickletools.genops(b'N\xff') + item = next(it) + self.assertEqual(item[0].name, 'NONE') + with self.assertRaisesRegex(ValueError, + r"at position 1, opcode b'\\xff' unknown"): + next(it) + + def test_unknown_opcode_without_pos(self): + f = SimpleReader(b'N\xff') + it = pickletools.genops(f) + item = next(it) + self.assertEqual(item[0].name, 'NONE') + with self.assertRaisesRegex(ValueError, + r"at position <unknown>, opcode b'\\xff' unknown"): + next(it) + + +class DisTests(unittest.TestCase): + maxDiff = None + + def check_dis(self, data, expected, **kwargs): + out = io.StringIO() + pickletools.dis(data, out=out, **kwargs) + self.assertEqual(out.getvalue(), expected) + + def check_dis_error(self, data, expected, expected_error, **kwargs): + out = io.StringIO() + with self.assertRaisesRegex(ValueError, expected_error): + pickletools.dis(data, out=out, **kwargs) + self.assertEqual(out.getvalue(), expected) + + def test_mark(self): + self.check_dis(b'(N(tl.', '''\ + 0: ( MARK + 1: N NONE + 2: ( MARK + 3: t TUPLE (MARK at 2) + 4: l LIST (MARK at 0) + 5: . STOP +highest protocol among opcodes = 0 +''') + + def test_indentlevel(self): + self.check_dis(b'(N(tl.', '''\ + 0: ( MARK + 1: N NONE + 2: ( MARK + 3: t TUPLE (MARK at 2) + 4: l LIST (MARK at 0) + 5: . STOP +highest protocol among opcodes = 0 +''', indentlevel=2) + + def test_mark_without_pos(self): + self.check_dis(SimpleReader(b'(N(tl.'), '''\ +( MARK +N NONE +( MARK +t TUPLE (MARK at unknown opcode offset) +l LIST (MARK at unknown opcode offset) +. STOP +highest protocol among opcodes = 0 +''') + + def test_no_mark(self): + self.check_dis_error(b'Nt.', '''\ + 0: N NONE + 1: t TUPLE no MARK exists on stack +''', 'no MARK exists on stack') + + def test_put(self): + self.check_dis(b'Np0\nq\x01r\x02\x00\x00\x00\x94.', '''\ + 0: N NONE + 1: p PUT 0 + 4: q BINPUT 1 + 6: r LONG_BINPUT 2 + 11: \\x94 MEMOIZE (as 3) + 12: . STOP +highest protocol among opcodes = 4 +''') + + def test_put_redefined(self): + self.check_dis_error(b'Np1\np1\n.', '''\ + 0: N NONE + 1: p PUT 1 + 4: p PUT 1 +''', 'memo key 1 already defined') + self.check_dis_error(b'Np1\nq\x01.', '''\ + 0: N NONE + 1: p PUT 1 + 4: q BINPUT 1 +''', 'memo key 1 already defined') + self.check_dis_error(b'Np1\nr\x01\x00\x00\x00.', '''\ + 0: N NONE + 1: p PUT 1 + 4: r LONG_BINPUT 1 +''', 'memo key 1 already defined') + self.check_dis_error(b'Np1\n\x94.', '''\ + 0: N NONE + 1: p PUT 1 + 4: \\x94 MEMOIZE (as 1) +''', 'memo key None already defined') + + def test_put_empty_stack(self): + self.check_dis_error(b'p0\n', '''\ + 0: p PUT 0 +''', "stack is empty -- can't store into memo") + + def test_put_markobject(self): + self.check_dis_error(b'(p0\n', '''\ + 0: ( MARK + 1: p PUT 0 +''', "can't store markobject in the memo") + + def test_get(self): + self.check_dis(b'(Np1\ng1\nh\x01j\x01\x00\x00\x00t.', '''\ + 0: ( MARK + 1: N NONE + 2: p PUT 1 + 5: g GET 1 + 8: h BINGET 1 + 10: j LONG_BINGET 1 + 15: t TUPLE (MARK at 0) + 16: . STOP +highest protocol among opcodes = 1 +''') + + def test_get_without_put(self): + self.check_dis_error(b'g1\n.', '''\ + 0: g GET 1 +''', 'memo key 1 has never been stored into') + self.check_dis_error(b'h\x01.', '''\ + 0: h BINGET 1 +''', 'memo key 1 has never been stored into') + self.check_dis_error(b'j\x01\x00\x00\x00.', '''\ + 0: j LONG_BINGET 1 +''', 'memo key 1 has never been stored into') + + def test_memo(self): + memo = {} + self.check_dis(b'Np1\n.', '''\ + 0: N NONE + 1: p PUT 1 + 4: . STOP +highest protocol among opcodes = 0 +''', memo=memo) + self.check_dis(b'g1\n.', '''\ + 0: g GET 1 + 3: . STOP +highest protocol among opcodes = 0 +''', memo=memo) + + def test_mark_pop(self): + self.check_dis(b'(N00N.', '''\ + 0: ( MARK + 1: N NONE + 2: 0 POP + 3: 0 POP (MARK at 0) + 4: N NONE + 5: . STOP +highest protocol among opcodes = 0 +''') + + def test_too_small_stack(self): + self.check_dis_error(b'a', '''\ + 0: a APPEND +''', 'tries to pop 2 items from stack with only 0 items') + self.check_dis_error(b']a', '''\ + 0: ] EMPTY_LIST + 1: a APPEND +''', 'tries to pop 2 items from stack with only 1 items') + + def test_no_stop(self): + self.check_dis_error(b'N', '''\ + 0: N NONE +''', 'pickle exhausted before seeing STOP') + + def test_truncated_data(self): + self.check_dis_error(b'NI123', '''\ + 0: N NONE +''', 'no newline found when trying to read stringnl') + self.check_dis_error(b'NJ\x12\x34', '''\ + 0: N NONE +''', 'not enough data in stream to read int4') + + def test_unknown_opcode(self): + self.check_dis_error(b'N\xff', '''\ + 0: N NONE +''', r"at position 1, opcode b'\\xff' unknown") + + def test_stop_not_empty_stack(self): + self.check_dis_error(b']N.', '''\ + 0: ] EMPTY_LIST + 1: N NONE + 2: . STOP +highest protocol among opcodes = 1 +''', r'stack not empty after STOP: \[list\]') + + def test_annotate(self): + self.check_dis(b'(Nt.', '''\ + 0: ( MARK Push markobject onto the stack. + 1: N NONE Push None on the stack. + 2: t TUPLE (MARK at 0) Build a tuple out of the topmost stack slice, after markobject. + 3: . STOP Stop the unpickling machine. +highest protocol among opcodes = 0 +''', annotate=1) + self.check_dis(b'(Nt.', '''\ + 0: ( MARK Push markobject onto the stack. + 1: N NONE Push None on the stack. + 2: t TUPLE (MARK at 0) Build a tuple out of the topmost stack slice, after markobject. + 3: . STOP Stop the unpickling machine. +highest protocol among opcodes = 0 +''', annotate=20) + self.check_dis(b'(((((((ttttttt.', '''\ + 0: ( MARK Push markobject onto the stack. + 1: ( MARK Push markobject onto the stack. + 2: ( MARK Push markobject onto the stack. + 3: ( MARK Push markobject onto the stack. + 4: ( MARK Push markobject onto the stack. + 5: ( MARK Push markobject onto the stack. + 6: ( MARK Push markobject onto the stack. + 7: t TUPLE (MARK at 6) Build a tuple out of the topmost stack slice, after markobject. + 8: t TUPLE (MARK at 5) Build a tuple out of the topmost stack slice, after markobject. + 9: t TUPLE (MARK at 4) Build a tuple out of the topmost stack slice, after markobject. + 10: t TUPLE (MARK at 3) Build a tuple out of the topmost stack slice, after markobject. + 11: t TUPLE (MARK at 2) Build a tuple out of the topmost stack slice, after markobject. + 12: t TUPLE (MARK at 1) Build a tuple out of the topmost stack slice, after markobject. + 13: t TUPLE (MARK at 0) Build a tuple out of the topmost stack slice, after markobject. + 14: . STOP Stop the unpickling machine. +highest protocol among opcodes = 0 +''', annotate=20) + + def test_string(self): + self.check_dis(b"S'abc'\n.", '''\ + 0: S STRING 'abc' + 7: . STOP +highest protocol among opcodes = 0 +''') + self.check_dis(b'S"abc"\n.', '''\ + 0: S STRING 'abc' + 7: . STOP +highest protocol among opcodes = 0 +''') + self.check_dis(b"S'\xc3\xb5'\n.", '''\ + 0: S STRING '\\xc3\\xb5' + 6: . STOP +highest protocol among opcodes = 0 +''') + + def test_string_without_quotes(self): + self.check_dis_error(b"Sabc'\n.", '', + 'no string quotes around b"abc\'"') + self.check_dis_error(b'Sabc"\n.', '', + "no string quotes around b'abc\"'") + self.check_dis_error(b"S'abc\n.", '', + '''strinq quote b"'" not found at both ends of b"'abc"''') + self.check_dis_error(b'S"abc\n.', '', + r"""strinq quote b'"' not found at both ends of b'"abc'""") + self.check_dis_error(b"S'abc\"\n.", '', + r"""strinq quote b"'" not found at both ends of b'\\'abc"'""") + self.check_dis_error(b"S\"abc'\n.", '', + r"""strinq quote b'"' not found at both ends of b'"abc\\''""") + + def test_binstring(self): + self.check_dis(b"T\x03\x00\x00\x00abc.", '''\ + 0: T BINSTRING 'abc' + 8: . STOP +highest protocol among opcodes = 1 +''') + self.check_dis(b"T\x02\x00\x00\x00\xc3\xb5.", '''\ + 0: T BINSTRING '\\xc3\\xb5' + 7: . STOP +highest protocol among opcodes = 1 +''') + + def test_short_binstring(self): + self.check_dis(b"U\x03abc.", '''\ + 0: U SHORT_BINSTRING 'abc' + 5: . STOP +highest protocol among opcodes = 1 +''') + self.check_dis(b"U\x02\xc3\xb5.", '''\ + 0: U SHORT_BINSTRING '\\xc3\\xb5' + 4: . STOP +highest protocol among opcodes = 1 +''') + + def test_global(self): + self.check_dis(b"cmodule\nname\n.", '''\ + 0: c GLOBAL 'module name' + 13: . STOP +highest protocol among opcodes = 0 +''') + self.check_dis(b"cm\xc3\xb6dule\nn\xc3\xa4me\n.", '''\ + 0: c GLOBAL 'm\xf6dule n\xe4me' + 15: . STOP +highest protocol among opcodes = 0 +''') + + def test_inst(self): + self.check_dis(b"(imodule\nname\n.", '''\ + 0: ( MARK + 1: i INST 'module name' (MARK at 0) + 14: . STOP +highest protocol among opcodes = 0 +''') + + def test_persid(self): + self.check_dis(b"Pabc\n.", '''\ + 0: P PERSID 'abc' + 5: . STOP +highest protocol among opcodes = 0 +''') class MiscTestCase(unittest.TestCase): def test__all__(self): diff --git a/Lib/test/test_posix.py b/Lib/test/test_posix.py index 30d6b6d3c3..a4809a2379 100644 --- a/Lib/test/test_posix.py +++ b/Lib/test/test_posix.py @@ -1,17 +1,19 @@ "Test posix functions" from test import support -from test.support import import_helper +from test.support import is_apple from test.support import os_helper from test.support import warnings_helper from test.support.script_helper import assert_python_ok +import copy import errno import sys import signal import time import os import platform +import pickle import stat import tempfile import unittest @@ -411,8 +413,10 @@ def test_posix_fallocate(self): # issue33655: Also ignore EINVAL on *BSD since ZFS is also # often used there. if inst.errno == errno.EINVAL and sys.platform.startswith( - ('sunos', 'freebsd', 'netbsd', 'openbsd', 'gnukfreebsd')): + ('sunos', 'freebsd', 'openbsd', 'gnukfreebsd')): raise unittest.SkipTest("test may fail on ZFS filesystems") + elif inst.errno == errno.EOPNOTSUPP and sys.platform.startswith("netbsd"): + raise unittest.SkipTest("test may fail on FFS filesystems") else: raise finally: @@ -565,8 +569,38 @@ def test_dup(self): @unittest.skipUnless(hasattr(posix, 'confstr'), 'test needs posix.confstr()') def test_confstr(self): - self.assertRaises(ValueError, posix.confstr, "CS_garbage") - self.assertEqual(len(posix.confstr("CS_PATH")) > 0, True) + with self.assertRaisesRegex( + ValueError, "unrecognized configuration name" + ): + posix.confstr("CS_garbage") + + with self.assertRaisesRegex( + TypeError, "configuration names must be strings or integers" + ): + posix.confstr(1.23) + + path = posix.confstr("CS_PATH") + self.assertGreater(len(path), 0) + self.assertEqual(posix.confstr(posix.confstr_names["CS_PATH"]), path) + + @unittest.expectedFailureIf(sys.platform in ('darwin', 'linux'), '''TODO: RUSTPYTHON; AssertionError: "configuration names must be strings or integers" does not match "Expected type 'str' but 'float' found."''') + @unittest.skipUnless(hasattr(posix, 'sysconf'), + 'test needs posix.sysconf()') + def test_sysconf(self): + with self.assertRaisesRegex( + ValueError, "unrecognized configuration name" + ): + posix.sysconf("SC_garbage") + + with self.assertRaisesRegex( + TypeError, "configuration names must be strings or integers" + ): + posix.sysconf(1.23) + + arg_max = posix.sysconf("SC_ARG_MAX") + self.assertGreater(arg_max, 0) + self.assertEqual( + posix.sysconf(posix.sysconf_names["SC_ARG_MAX"]), arg_max) @unittest.skipUnless(hasattr(posix, 'dup2'), 'test needs posix.dup2()') @@ -703,7 +737,8 @@ def test_makedev(self): self.assertEqual(posix.major(dev), major) self.assertRaises(TypeError, posix.major, float(dev)) self.assertRaises(TypeError, posix.major) - self.assertRaises((ValueError, OverflowError), posix.major, -1) + for x in -2, 2**64, -2**63-1: + self.assertRaises((ValueError, OverflowError), posix.major, x) minor = posix.minor(dev) self.assertIsInstance(minor, int) @@ -711,13 +746,23 @@ def test_makedev(self): self.assertEqual(posix.minor(dev), minor) self.assertRaises(TypeError, posix.minor, float(dev)) self.assertRaises(TypeError, posix.minor) - self.assertRaises((ValueError, OverflowError), posix.minor, -1) + for x in -2, 2**64, -2**63-1: + self.assertRaises((ValueError, OverflowError), posix.minor, x) self.assertEqual(posix.makedev(major, minor), dev) self.assertRaises(TypeError, posix.makedev, float(major), minor) self.assertRaises(TypeError, posix.makedev, major, float(minor)) self.assertRaises(TypeError, posix.makedev, major) self.assertRaises(TypeError, posix.makedev) + for x in -2, 2**32, 2**64, -2**63-1: + self.assertRaises((ValueError, OverflowError), posix.makedev, x, minor) + self.assertRaises((ValueError, OverflowError), posix.makedev, major, x) + + if sys.platform == 'linux' and not support.linked_to_musl(): + NODEV = -1 + self.assertEqual(posix.major(NODEV), NODEV) + self.assertEqual(posix.minor(NODEV), NODEV) + self.assertEqual(posix.makedev(NODEV, NODEV), NODEV) def _test_all_chown_common(self, chown_func, first_param, stat_func): """Common code for chown, fchown and lchown tests.""" @@ -781,9 +826,10 @@ def check_stat(uid, gid): check_stat(uid, gid) self.assertRaises(OSError, chown_func, first_param, 0, -1) check_stat(uid, gid) - if 0 not in os.getgroups(): - self.assertRaises(OSError, chown_func, first_param, -1, 0) - check_stat(uid, gid) + if hasattr(os, 'getgroups'): + if 0 not in os.getgroups(): + self.assertRaises(OSError, chown_func, first_param, -1, 0) + check_stat(uid, gid) # test illegal types for t in str, float: self.assertRaises(TypeError, chown_func, first_param, t(uid), gid) @@ -936,6 +982,7 @@ def test_utime(self): posix.utime(os_helper.TESTFN, (now, now)) def check_chmod(self, chmod_func, target, **kwargs): + closefd = not isinstance(target, int) mode = os.stat(target).st_mode try: new_mode = mode & ~(stat.S_IWOTH | stat.S_IWGRP | stat.S_IWUSR) @@ -943,7 +990,7 @@ def check_chmod(self, chmod_func, target, **kwargs): self.assertEqual(os.stat(target).st_mode, new_mode) if stat.S_ISREG(mode): try: - with open(target, 'wb+'): + with open(target, 'wb+', closefd=closefd): pass except PermissionError: pass @@ -951,10 +998,10 @@ def check_chmod(self, chmod_func, target, **kwargs): chmod_func(target, new_mode, **kwargs) self.assertEqual(os.stat(target).st_mode, new_mode) if stat.S_ISREG(mode): - with open(target, 'wb+'): + with open(target, 'wb+', closefd=closefd): pass finally: - posix.chmod(target, mode) + chmod_func(target, mode) @os_helper.skip_unless_working_chmod def test_chmod_file(self): @@ -971,6 +1018,13 @@ def test_chmod_dir(self): target = self.tempdir() self.check_chmod(posix.chmod, target) + @unittest.skipIf(sys.platform in ('darwin', 'linux'), 'TODO: RUSTPYTHON; crash') + @os_helper.skip_unless_working_chmod + def test_fchmod_file(self): + with open(os_helper.TESTFN, 'wb+') as f: + self.check_chmod(posix.fchmod, f.fileno()) + self.check_chmod(posix.chmod, f.fileno()) + @unittest.skipUnless(hasattr(posix, 'lchmod'), 'test needs os.lchmod()') def test_lchmod_file(self): self.check_chmod(posix.lchmod, os_helper.TESTFN) @@ -1019,8 +1073,9 @@ def test_chmod_file_symlink(self): self.check_lchmod_link(posix.chmod, target, link) else: self.check_chmod_link(posix.chmod, target, link) - self.check_chmod_link(posix.chmod, target, link, follow_symlinks=True) + self.check_chmod_link(posix.chmod, target, link, follow_symlinks=True) + @unittest.skipIf(sys.platform == 'win32', 'TODO: RUSTPYTHON; flaky') @os_helper.skip_unless_symlink def test_chmod_dir_symlink(self): target = self.tempdir() @@ -1031,7 +1086,7 @@ def test_chmod_dir_symlink(self): self.check_lchmod_link(posix.chmod, target, link) else: self.check_chmod_link(posix.chmod, target, link) - self.check_chmod_link(posix.chmod, target, link, follow_symlinks=True) + self.check_chmod_link(posix.chmod, target, link, follow_symlinks=True) @unittest.skipUnless(hasattr(posix, 'lchmod'), 'test needs os.lchmod()') @os_helper.skip_unless_symlink @@ -1249,8 +1304,8 @@ def test_sched_priority(self): self.assertIsInstance(lo, int) self.assertIsInstance(hi, int) self.assertGreaterEqual(hi, lo) - # OSX evidently just returns 15 without checking the argument. - if sys.platform != "darwin": + # Apple platforms return 15 without checking the argument. + if not is_apple: self.assertRaises(OSError, posix.sched_get_priority_min, -23) self.assertRaises(OSError, posix.sched_get_priority_max, -23) @@ -1262,9 +1317,10 @@ def test_get_and_set_scheduler_and_param(self): self.assertIn(mine, possible_schedulers) try: parent = posix.sched_getscheduler(os.getppid()) - except OSError as e: - if e.errno != errno.EPERM: - raise + except PermissionError: + # POSIX specifies EPERM, but Android returns EACCES. Both errno + # values are mapped to PermissionError. + pass else: self.assertIn(parent, possible_schedulers) self.assertRaises(OSError, posix.sched_getscheduler, -1) @@ -1279,9 +1335,8 @@ def test_get_and_set_scheduler_and_param(self): try: posix.sched_setscheduler(0, mine, param) posix.sched_setparam(0, param) - except OSError as e: - if e.errno != errno.EPERM: - raise + except PermissionError: + pass self.assertRaises(OSError, posix.sched_setparam, -1, param) self.assertRaises(OSError, posix.sched_setscheduler, -1, mine, param) @@ -1295,6 +1350,26 @@ def test_get_and_set_scheduler_and_param(self): param = posix.sched_param(sched_priority=-large) self.assertRaises(OverflowError, posix.sched_setparam, 0, param) + @unittest.expectedFailureIf(sys.platform == 'linux', "TODO: RUSTPYTHON; TypeError: cannot pickle 'sched_param' object") + @requires_sched + def test_sched_param(self): + param = posix.sched_param(1) + for proto in range(pickle.HIGHEST_PROTOCOL+1): + newparam = pickle.loads(pickle.dumps(param, proto)) + self.assertEqual(newparam, param) + newparam = copy.copy(param) + self.assertIsNot(newparam, param) + self.assertEqual(newparam, param) + newparam = copy.deepcopy(param) + self.assertIsNot(newparam, param) + self.assertEqual(newparam, param) + newparam = copy.replace(param) + self.assertIsNot(newparam, param) + self.assertEqual(newparam, param) + newparam = copy.replace(param, sched_priority=0) + self.assertNotEqual(newparam, param) + self.assertEqual(newparam.sched_priority, 0) + @unittest.skipUnless(hasattr(posix, "sched_rr_get_interval"), "no function") def test_sched_rr_get_interval(self): try: @@ -1495,6 +1570,7 @@ def test_chown_dir_fd(self): with self.prepare_file() as (dir_fd, name, fullname): posix.chown(name, os.getuid(), os.getgid(), dir_fd=dir_fd) + @unittest.expectedFailureIf(sys.platform in ('darwin', 'linux'), 'TODO: RUSTPYTHON; AssertionError: RuntimeWarning not triggered') @unittest.skipUnless(os.stat in os.supports_dir_fd, "test needs dir_fd support in os.stat()") def test_stat_dir_fd(self): with self.prepare() as (dir_fd, name, fullname): @@ -1515,6 +1591,13 @@ def test_stat_dir_fd(self): self.assertRaises(OverflowError, posix.stat, name, dir_fd=10**20) + for fd in False, True: + with self.assertWarnsRegex(RuntimeWarning, + 'bool is used as a file descriptor') as cm: + with self.assertRaises(OSError): + posix.stat('nonexisting', dir_fd=fd) + self.assertEqual(cm.filename, __file__) + @unittest.skipUnless(os.utime in os.supports_dir_fd, "test needs dir_fd support in os.utime()") def test_utime_dir_fd(self): with self.prepare_file() as (dir_fd, name, fullname): @@ -1890,7 +1973,7 @@ def test_setsigdef_wrong_type(self): [sys.executable, "-c", "pass"], os.environ, setsigdef=[signal.NSIG, signal.NSIG+1]) - @unittest.expectedFailure + @unittest.expectedFailureIf(sys.platform in ('darwin', 'linux'), 'TODO: RUSTPYTHON; NotImplementedError: scheduler parameter is not yet implemented') @requires_sched @unittest.skipIf(sys.platform.startswith(('freebsd', 'netbsd')), "bpo-34685: test can fail on BSD") @@ -1911,10 +1994,15 @@ def test_setscheduler_only_param(self): ) support.wait_process(pid, exitcode=0) - @unittest.expectedFailure + @unittest.expectedFailureIf(sys.platform in ('darwin', 'linux'), 'TODO: RUSTPYTHON; NotImplementedError: scheduler parameter is not yet implemented') @requires_sched @unittest.skipIf(sys.platform.startswith(('freebsd', 'netbsd')), "bpo-34685: test can fail on BSD") + @unittest.skipIf(platform.libc_ver()[0] == 'glibc' and + os.sched_getscheduler(0) in [ + os.SCHED_BATCH, + os.SCHED_IDLE], + "Skip test due to glibc posix_spawn policy") def test_setscheduler_with_policy(self): policy = os.sched_getscheduler(0) priority = os.sched_get_priority_min(policy) @@ -1993,8 +2081,7 @@ def test_open_file(self): with open(outfile, encoding="utf-8") as f: self.assertEqual(f.read(), 'hello') - # TODO: RUSTPYTHON: FileNotFoundError: [Errno 2] No such file or directory (os error 2): '@test_55144_tmp' -> 'None' - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; the rust runtime reopens closed stdio fds at startup, so this test fails, even though POSIX_SPAWN_CLOSE does actually have an effect def test_close_file(self): closefile = os_helper.TESTFN self.addCleanup(os_helper.unlink, closefile) @@ -2034,11 +2121,13 @@ def test_dup2(self): @unittest.skipUnless(hasattr(os, 'posix_spawn'), "test needs os.posix_spawn") +@support.requires_subprocess() class TestPosixSpawn(unittest.TestCase, _PosixSpawnMixin): spawn_func = getattr(posix, 'posix_spawn', None) @unittest.skipUnless(hasattr(os, 'posix_spawnp'), "test needs os.posix_spawnp") +@support.requires_subprocess() class TestPosixSpawnP(unittest.TestCase, _PosixSpawnMixin): spawn_func = getattr(posix, 'posix_spawnp', None) @@ -2115,6 +2204,13 @@ def test_stat(self): with self.assertRaisesRegex(NotImplementedError, "dir_fd unavailable"): os.stat("file", dir_fd=0) + def test_ptsname_r(self): + self._verify_available("HAVE_PTSNAME_R") + if self.mac_ver >= (10, 13, 4): + self.assertIn("HAVE_PTSNAME_R", posix._have_functions) + else: + self.assertNotIn("HAVE_PTSNAME_R", posix._have_functions) + def test_access(self): self._verify_available("HAVE_FACCESSAT") if self.mac_ver >= (10, 10): diff --git a/Lib/test/test_posixpath.py b/Lib/test/test_posixpath.py index 4890bcc75e..0dc0211ead 100644 --- a/Lib/test/test_posixpath.py +++ b/Lib/test/test_posixpath.py @@ -1,12 +1,16 @@ +import inspect import os import posixpath +import random import sys import unittest -from posixpath import realpath, abspath, dirname, basename +from functools import partial +from posixpath import realpath, abspath, dirname, basename, ALLOW_MISSING +from test import support from test import test_genericpath from test.support import import_helper from test.support import os_helper -from test.support.os_helper import FakePath +from test.support.os_helper import FakePath, TESTFN from unittest import mock try: @@ -18,7 +22,7 @@ # An absolute path to a temporary filename for testing. We can't rely on TESTFN # being an absolute path, so we need this. -ABSTFN = abspath(os_helper.TESTFN) +ABSTFN = abspath(TESTFN) def skip_if_ABSTFN_contains_backslash(test): """ @@ -30,35 +34,40 @@ def skip_if_ABSTFN_contains_backslash(test): msg = "ABSTFN is not a posix path - tests fail" return [test, unittest.skip(msg)(test)][found_backslash] -def safe_rmdir(dirname): - try: - os.rmdir(dirname) - except OSError: - pass + +def _parameterize(*parameters): + return support.subTests('kwargs', parameters) + class PosixPathTest(unittest.TestCase): def setUp(self): - self.tearDown() - - def tearDown(self): for suffix in ["", "1", "2"]: - os_helper.unlink(os_helper.TESTFN + suffix) - safe_rmdir(os_helper.TESTFN + suffix) + self.assertFalse(posixpath.lexists(ABSTFN + suffix)) def test_join(self): - self.assertEqual(posixpath.join("/foo", "bar", "/bar", "baz"), - "/bar/baz") - self.assertEqual(posixpath.join("/foo", "bar", "baz"), "/foo/bar/baz") - self.assertEqual(posixpath.join("/foo/", "bar/", "baz/"), - "/foo/bar/baz/") - - self.assertEqual(posixpath.join(b"/foo", b"bar", b"/bar", b"baz"), - b"/bar/baz") - self.assertEqual(posixpath.join(b"/foo", b"bar", b"baz"), - b"/foo/bar/baz") - self.assertEqual(posixpath.join(b"/foo/", b"bar/", b"baz/"), - b"/foo/bar/baz/") + fn = posixpath.join + self.assertEqual(fn("/foo", "bar", "/bar", "baz"), "/bar/baz") + self.assertEqual(fn("/foo", "bar", "baz"), "/foo/bar/baz") + self.assertEqual(fn("/foo/", "bar/", "baz/"), "/foo/bar/baz/") + + self.assertEqual(fn(b"/foo", b"bar", b"/bar", b"baz"), b"/bar/baz") + self.assertEqual(fn(b"/foo", b"bar", b"baz"), b"/foo/bar/baz") + self.assertEqual(fn(b"/foo/", b"bar/", b"baz/"), b"/foo/bar/baz/") + + self.assertEqual(fn("a", ""), "a/") + self.assertEqual(fn("a", "", ""), "a/") + self.assertEqual(fn("a", "b"), "a/b") + self.assertEqual(fn("a", "b/"), "a/b/") + self.assertEqual(fn("a/", "b"), "a/b") + self.assertEqual(fn("a/", "b/"), "a/b/") + self.assertEqual(fn("a", "b/c", "d"), "a/b/c/d") + self.assertEqual(fn("a", "b//c", "d"), "a/b//c/d") + self.assertEqual(fn("a", "b/c/", "d"), "a/b/c/d") + self.assertEqual(fn("/a", "b"), "/a/b") + self.assertEqual(fn("/a/", "b"), "/a/b") + self.assertEqual(fn("a", "/b", "c"), "/b/c") + self.assertEqual(fn("a", "/b", "/c"), "/c") def test_split(self): self.assertEqual(posixpath.split("/foo/bar"), ("/foo", "bar")) @@ -180,27 +189,29 @@ def test_dirname(self): self.assertEqual(posixpath.dirname(b"////foo"), b"////") self.assertEqual(posixpath.dirname(b"//foo//bar"), b"//foo") - @unittest.expectedFailureIf(os.name == "nt", "TODO: RUSTPYTHON") def test_islink(self): - self.assertIs(posixpath.islink(os_helper.TESTFN + "1"), False) - self.assertIs(posixpath.lexists(os_helper.TESTFN + "2"), False) + self.assertIs(posixpath.islink(TESTFN + "1"), False) + self.assertIs(posixpath.lexists(TESTFN + "2"), False) - with open(os_helper.TESTFN + "1", "wb") as f: + self.addCleanup(os_helper.unlink, TESTFN + "1") + with open(TESTFN + "1", "wb") as f: f.write(b"foo") - self.assertIs(posixpath.islink(os_helper.TESTFN + "1"), False) + self.assertIs(posixpath.islink(TESTFN + "1"), False) if os_helper.can_symlink(): - os.symlink(os_helper.TESTFN + "1", os_helper.TESTFN + "2") - self.assertIs(posixpath.islink(os_helper.TESTFN + "2"), True) - os.remove(os_helper.TESTFN + "1") - self.assertIs(posixpath.islink(os_helper.TESTFN + "2"), True) - self.assertIs(posixpath.exists(os_helper.TESTFN + "2"), False) - self.assertIs(posixpath.lexists(os_helper.TESTFN + "2"), True) - - self.assertIs(posixpath.islink(os_helper.TESTFN + "\udfff"), False) - self.assertIs(posixpath.islink(os.fsencode(os_helper.TESTFN) + b"\xff"), False) - self.assertIs(posixpath.islink(os_helper.TESTFN + "\x00"), False) - self.assertIs(posixpath.islink(os.fsencode(os_helper.TESTFN) + b"\x00"), False) + self.addCleanup(os_helper.unlink, TESTFN + "2") + os.symlink(TESTFN + "1", TESTFN + "2") + self.assertIs(posixpath.islink(TESTFN + "2"), True) + os.remove(TESTFN + "1") + self.assertIs(posixpath.islink(TESTFN + "2"), True) + self.assertIs(posixpath.exists(TESTFN + "2"), False) + self.assertIs(posixpath.lexists(TESTFN + "2"), True) + + def test_islink_invalid_paths(self): + self.assertIs(posixpath.islink(TESTFN + "\udfff"), False) + self.assertIs(posixpath.islink(os.fsencode(TESTFN) + b"\xff"), False) + self.assertIs(posixpath.islink(TESTFN + "\x00"), False) + self.assertIs(posixpath.islink(os.fsencode(TESTFN) + b"\x00"), False) def test_ismount(self): self.assertIs(posixpath.ismount("/"), True) @@ -215,14 +226,14 @@ def test_ismount_non_existent(self): os.mkdir(ABSTFN) self.assertIs(posixpath.ismount(ABSTFN), False) finally: - safe_rmdir(ABSTFN) + os_helper.rmdir(ABSTFN) + def test_ismount_invalid_paths(self): self.assertIs(posixpath.ismount('/\udfff'), False) self.assertIs(posixpath.ismount(b'/\xff'), False) self.assertIs(posixpath.ismount('/\x00'), False) self.assertIs(posixpath.ismount(b'/\x00'), False) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") @os_helper.skip_unless_symlink def test_ismount_symlinks(self): # Symlinks are never mountpoints. @@ -230,7 +241,7 @@ def test_ismount_symlinks(self): os.symlink("/", ABSTFN) self.assertIs(posixpath.ismount(ABSTFN), False) finally: - os.unlink(ABSTFN) + os_helper.unlink(ABSTFN) @unittest.skipIf(posix is None, "Test requires posix module") def test_ismount_different_device(self): @@ -275,6 +286,16 @@ def fake_lstat(path): def test_isjunction(self): self.assertFalse(posixpath.isjunction(ABSTFN)) + @unittest.skipIf(sys.platform == 'win32', "Fast paths are not for win32") + @support.cpython_only + def test_fast_paths_in_use(self): + # There are fast paths of these functions implemented in posixmodule.c. + # Confirm that they are being used, and not the Python fallbacks + self.assertTrue(os.path.splitroot is posix._path_splitroot_ex) + self.assertFalse(inspect.isfunction(os.path.splitroot)) + self.assertTrue(os.path.normpath is posix._path_normpath) + self.assertFalse(inspect.isfunction(os.path.normpath)) + def test_expanduser(self): self.assertEqual(posixpath.expanduser("foo"), "foo") self.assertEqual(posixpath.expanduser(b"foo"), b"foo") @@ -336,12 +357,38 @@ def test_expanduser_pwd(self): for path in ('~', '~/.local', '~vstinner/'): self.assertEqual(posixpath.expanduser(path), path) + @unittest.skipIf(sys.platform == "vxworks", + "no home directory on VxWorks") + def test_expanduser_pwd2(self): + pwd = import_helper.import_module('pwd') + getpwall = support.get_attribute(pwd, 'getpwall') + names = [entry.pw_name for entry in getpwall()] + maxusers = 1000 if support.is_resource_enabled('cpu') else 100 + if len(names) > maxusers: + # Select random names, half of them with non-ASCII name, + # if available. + random.shuffle(names) + names.sort(key=lambda name: name.isascii()) + del names[maxusers//2:-maxusers//2] + for name in names: + # gh-121200: pw_dir can be different between getpwall() and + # getpwnam(), so use getpwnam() pw_dir as expanduser() does. + entry = pwd.getpwnam(name) + home = entry.pw_dir + home = home.rstrip('/') or '/' + + with self.subTest(name=name, pw_dir=entry.pw_dir): + self.assertEqual(posixpath.expanduser('~' + name), home) + self.assertEqual(posixpath.expanduser(os.fsencode('~' + name)), + os.fsencode(home)) + NORMPATH_CASES = [ ("", "."), ("/", "/"), ("/.", "/"), ("/./", "/"), ("/.//.", "/"), + ("/./foo/bar", "/foo/bar"), ("/foo", "/foo"), ("/foo/bar", "/foo/bar"), ("//", "//"), @@ -351,6 +398,7 @@ def test_expanduser_pwd(self): ("///..//./foo/.//bar", "/foo/bar"), (".", "."), (".//.", "."), + ("./foo/bar", "foo/bar"), ("..", ".."), ("../", ".."), ("../foo", "../foo"), @@ -400,32 +448,35 @@ def test_normpath(self): self.assertEqual(result, expected) @skip_if_ABSTFN_contains_backslash - def test_realpath_curdir(self): - self.assertEqual(realpath('.'), os.getcwd()) - self.assertEqual(realpath('./.'), os.getcwd()) - self.assertEqual(realpath('/'.join(['.'] * 100)), os.getcwd()) + @_parameterize({}, {'strict': True}, {'strict': ALLOW_MISSING}) + def test_realpath_curdir(self, kwargs): + self.assertEqual(realpath('.', **kwargs), os.getcwd()) + self.assertEqual(realpath('./.', **kwargs), os.getcwd()) + self.assertEqual(realpath('/'.join(['.'] * 100), **kwargs), os.getcwd()) - self.assertEqual(realpath(b'.'), os.getcwdb()) - self.assertEqual(realpath(b'./.'), os.getcwdb()) - self.assertEqual(realpath(b'/'.join([b'.'] * 100)), os.getcwdb()) + self.assertEqual(realpath(b'.', **kwargs), os.getcwdb()) + self.assertEqual(realpath(b'./.', **kwargs), os.getcwdb()) + self.assertEqual(realpath(b'/'.join([b'.'] * 100), **kwargs), os.getcwdb()) @skip_if_ABSTFN_contains_backslash - def test_realpath_pardir(self): - self.assertEqual(realpath('..'), dirname(os.getcwd())) - self.assertEqual(realpath('../..'), dirname(dirname(os.getcwd()))) - self.assertEqual(realpath('/'.join(['..'] * 100)), '/') + @_parameterize({}, {'strict': True}, {'strict': ALLOW_MISSING}) + def test_realpath_pardir(self, kwargs): + self.assertEqual(realpath('..', **kwargs), dirname(os.getcwd())) + self.assertEqual(realpath('../..', **kwargs), dirname(dirname(os.getcwd()))) + self.assertEqual(realpath('/'.join(['..'] * 100), **kwargs), '/') - self.assertEqual(realpath(b'..'), dirname(os.getcwdb())) - self.assertEqual(realpath(b'../..'), dirname(dirname(os.getcwdb()))) - self.assertEqual(realpath(b'/'.join([b'..'] * 100)), b'/') + self.assertEqual(realpath(b'..', **kwargs), dirname(os.getcwdb())) + self.assertEqual(realpath(b'../..', **kwargs), dirname(dirname(os.getcwdb()))) + self.assertEqual(realpath(b'/'.join([b'..'] * 100), **kwargs), b'/') @os_helper.skip_unless_symlink @skip_if_ABSTFN_contains_backslash - def test_realpath_basic(self): + @_parameterize({}, {'strict': ALLOW_MISSING}) + def test_realpath_basic(self, kwargs): # Basic operation. try: os.symlink(ABSTFN+"1", ABSTFN) - self.assertEqual(realpath(ABSTFN), ABSTFN+"1") + self.assertEqual(realpath(ABSTFN, **kwargs), ABSTFN+"1") finally: os_helper.unlink(ABSTFN) @@ -441,15 +492,122 @@ def test_realpath_strict(self): finally: os_helper.unlink(ABSTFN) + def test_realpath_invalid_paths(self): + path = '/\x00' + self.assertRaises(ValueError, realpath, path, strict=False) + self.assertRaises(ValueError, realpath, path, strict=True) + self.assertRaises(ValueError, realpath, path, strict=ALLOW_MISSING) + path = b'/\x00' + self.assertRaises(ValueError, realpath, path, strict=False) + self.assertRaises(ValueError, realpath, path, strict=True) + self.assertRaises(ValueError, realpath, path, strict=ALLOW_MISSING) + path = '/nonexistent/x\x00' + self.assertRaises(ValueError, realpath, path, strict=False) + self.assertRaises(FileNotFoundError, realpath, path, strict=True) + self.assertRaises(ValueError, realpath, path, strict=ALLOW_MISSING) + path = b'/nonexistent/x\x00' + self.assertRaises(ValueError, realpath, path, strict=False) + self.assertRaises(FileNotFoundError, realpath, path, strict=True) + self.assertRaises(ValueError, realpath, path, strict=ALLOW_MISSING) + path = '/\x00/..' + self.assertRaises(ValueError, realpath, path, strict=False) + self.assertRaises(ValueError, realpath, path, strict=True) + self.assertRaises(ValueError, realpath, path, strict=ALLOW_MISSING) + path = b'/\x00/..' + self.assertRaises(ValueError, realpath, path, strict=False) + self.assertRaises(ValueError, realpath, path, strict=True) + self.assertRaises(ValueError, realpath, path, strict=ALLOW_MISSING) + + path = '/nonexistent/x\x00/..' + self.assertRaises(ValueError, realpath, path, strict=False) + self.assertRaises(FileNotFoundError, realpath, path, strict=True) + self.assertRaises(ValueError, realpath, path, strict=ALLOW_MISSING) + path = b'/nonexistent/x\x00/..' + self.assertRaises(ValueError, realpath, path, strict=False) + self.assertRaises(FileNotFoundError, realpath, path, strict=True) + self.assertRaises(ValueError, realpath, path, strict=ALLOW_MISSING) + + path = '/\udfff' + if sys.platform == 'win32': + self.assertEqual(realpath(path, strict=False), path) + self.assertRaises(FileNotFoundError, realpath, path, strict=True) + self.assertEqual(realpath(path, strict=ALLOW_MISSING), path) + else: + self.assertRaises(UnicodeEncodeError, realpath, path, strict=False) + self.assertRaises(UnicodeEncodeError, realpath, path, strict=True) + self.assertRaises(UnicodeEncodeError, realpath, path, strict=ALLOW_MISSING) + path = '/nonexistent/\udfff' + if sys.platform == 'win32': + self.assertEqual(realpath(path, strict=False), path) + self.assertEqual(realpath(path, strict=ALLOW_MISSING), path) + else: + self.assertRaises(UnicodeEncodeError, realpath, path, strict=False) + self.assertRaises(UnicodeEncodeError, realpath, path, strict=ALLOW_MISSING) + self.assertRaises(FileNotFoundError, realpath, path, strict=True) + path = '/\udfff/..' + if sys.platform == 'win32': + self.assertEqual(realpath(path, strict=False), '/') + self.assertRaises(FileNotFoundError, realpath, path, strict=True) + self.assertEqual(realpath(path, strict=ALLOW_MISSING), '/') + else: + self.assertRaises(UnicodeEncodeError, realpath, path, strict=False) + self.assertRaises(UnicodeEncodeError, realpath, path, strict=True) + self.assertRaises(UnicodeEncodeError, realpath, path, strict=ALLOW_MISSING) + path = '/nonexistent/\udfff/..' + if sys.platform == 'win32': + self.assertEqual(realpath(path, strict=False), '/nonexistent') + self.assertEqual(realpath(path, strict=ALLOW_MISSING), '/nonexistent') + else: + self.assertRaises(UnicodeEncodeError, realpath, path, strict=False) + self.assertRaises(UnicodeEncodeError, realpath, path, strict=ALLOW_MISSING) + self.assertRaises(FileNotFoundError, realpath, path, strict=True) + + path = b'/\xff' + if sys.platform == 'win32': + self.assertRaises(UnicodeDecodeError, realpath, path, strict=False) + self.assertRaises(UnicodeDecodeError, realpath, path, strict=True) + self.assertRaises(UnicodeDecodeError, realpath, path, strict=ALLOW_MISSING) + else: + self.assertEqual(realpath(path, strict=False), path) + if support.is_wasi: + self.assertRaises(OSError, realpath, path, strict=True) + self.assertRaises(OSError, realpath, path, strict=ALLOW_MISSING) + else: + self.assertRaises(FileNotFoundError, realpath, path, strict=True) + self.assertEqual(realpath(path, strict=ALLOW_MISSING), path) + path = b'/nonexistent/\xff' + if sys.platform == 'win32': + self.assertRaises(UnicodeDecodeError, realpath, path, strict=False) + self.assertRaises(UnicodeDecodeError, realpath, path, strict=ALLOW_MISSING) + else: + self.assertEqual(realpath(path, strict=False), path) + if support.is_wasi: + self.assertRaises(OSError, realpath, path, strict=True) + self.assertRaises(OSError, realpath, path, strict=ALLOW_MISSING) + else: + self.assertRaises(FileNotFoundError, realpath, path, strict=True) + @os_helper.skip_unless_symlink @skip_if_ABSTFN_contains_backslash - def test_realpath_relative(self): + @_parameterize({}, {'strict': ALLOW_MISSING}) + def test_realpath_relative(self, kwargs): try: os.symlink(posixpath.relpath(ABSTFN+"1"), ABSTFN) - self.assertEqual(realpath(ABSTFN), ABSTFN+"1") + self.assertEqual(realpath(ABSTFN, **kwargs), ABSTFN+"1") finally: os_helper.unlink(ABSTFN) + @os_helper.skip_unless_symlink + @skip_if_ABSTFN_contains_backslash + @_parameterize({}, {'strict': ALLOW_MISSING}) + def test_realpath_missing_pardir(self, kwargs): + try: + os.symlink(TESTFN + "1", TESTFN) + self.assertEqual( + realpath("nonexistent/../" + TESTFN, **kwargs), ABSTFN + "1") + finally: + os_helper.unlink(TESTFN) + @os_helper.skip_unless_symlink @skip_if_ABSTFN_contains_backslash def test_realpath_symlink_loops(self): @@ -469,7 +627,7 @@ def test_realpath_symlink_loops(self): self.assertEqual(realpath(ABSTFN+"1/../x"), dirname(ABSTFN) + "/x") os.symlink(ABSTFN+"x", ABSTFN+"y") self.assertEqual(realpath(ABSTFN+"1/../" + basename(ABSTFN) + "y"), - ABSTFN + "y") + ABSTFN + "x") self.assertEqual(realpath(ABSTFN+"1/../" + basename(ABSTFN) + "1"), ABSTFN + "1") @@ -493,37 +651,38 @@ def test_realpath_symlink_loops(self): @os_helper.skip_unless_symlink @skip_if_ABSTFN_contains_backslash - def test_realpath_symlink_loops_strict(self): + @_parameterize({'strict': True}, {'strict': ALLOW_MISSING}) + def test_realpath_symlink_loops_strict(self, kwargs): # Bug #43757, raise OSError if we get into an infinite symlink loop in - # strict mode. + # the strict modes. try: os.symlink(ABSTFN, ABSTFN) - self.assertRaises(OSError, realpath, ABSTFN, strict=True) + self.assertRaises(OSError, realpath, ABSTFN, **kwargs) os.symlink(ABSTFN+"1", ABSTFN+"2") os.symlink(ABSTFN+"2", ABSTFN+"1") - self.assertRaises(OSError, realpath, ABSTFN+"1", strict=True) - self.assertRaises(OSError, realpath, ABSTFN+"2", strict=True) + self.assertRaises(OSError, realpath, ABSTFN+"1", **kwargs) + self.assertRaises(OSError, realpath, ABSTFN+"2", **kwargs) - self.assertRaises(OSError, realpath, ABSTFN+"1/x", strict=True) - self.assertRaises(OSError, realpath, ABSTFN+"1/..", strict=True) - self.assertRaises(OSError, realpath, ABSTFN+"1/../x", strict=True) + self.assertRaises(OSError, realpath, ABSTFN+"1/x", **kwargs) + self.assertRaises(OSError, realpath, ABSTFN+"1/..", **kwargs) + self.assertRaises(OSError, realpath, ABSTFN+"1/../x", **kwargs) os.symlink(ABSTFN+"x", ABSTFN+"y") self.assertRaises(OSError, realpath, - ABSTFN+"1/../" + basename(ABSTFN) + "y", strict=True) + ABSTFN+"1/../" + basename(ABSTFN) + "y", **kwargs) self.assertRaises(OSError, realpath, - ABSTFN+"1/../" + basename(ABSTFN) + "1", strict=True) + ABSTFN+"1/../" + basename(ABSTFN) + "1", **kwargs) os.symlink(basename(ABSTFN) + "a/b", ABSTFN+"a") - self.assertRaises(OSError, realpath, ABSTFN+"a", strict=True) + self.assertRaises(OSError, realpath, ABSTFN+"a", **kwargs) os.symlink("../" + basename(dirname(ABSTFN)) + "/" + basename(ABSTFN) + "c", ABSTFN+"c") - self.assertRaises(OSError, realpath, ABSTFN+"c", strict=True) + self.assertRaises(OSError, realpath, ABSTFN+"c", **kwargs) # Test using relative path as well. with os_helper.change_cwd(dirname(ABSTFN)): - self.assertRaises(OSError, realpath, basename(ABSTFN), strict=True) + self.assertRaises(OSError, realpath, basename(ABSTFN), **kwargs) finally: os_helper.unlink(ABSTFN) os_helper.unlink(ABSTFN+"1") @@ -534,28 +693,30 @@ def test_realpath_symlink_loops_strict(self): @os_helper.skip_unless_symlink @skip_if_ABSTFN_contains_backslash - def test_realpath_repeated_indirect_symlinks(self): + @_parameterize({}, {'strict': True}, {'strict': ALLOW_MISSING}) + def test_realpath_repeated_indirect_symlinks(self, kwargs): # Issue #6975. try: os.mkdir(ABSTFN) os.symlink('../' + basename(ABSTFN), ABSTFN + '/self') os.symlink('self/self/self', ABSTFN + '/link') - self.assertEqual(realpath(ABSTFN + '/link'), ABSTFN) + self.assertEqual(realpath(ABSTFN + '/link', **kwargs), ABSTFN) finally: os_helper.unlink(ABSTFN + '/self') os_helper.unlink(ABSTFN + '/link') - safe_rmdir(ABSTFN) + os_helper.rmdir(ABSTFN) @os_helper.skip_unless_symlink @skip_if_ABSTFN_contains_backslash - def test_realpath_deep_recursion(self): + @_parameterize({}, {'strict': True}, {'strict': ALLOW_MISSING}) + def test_realpath_deep_recursion(self, kwargs): depth = 10 try: os.mkdir(ABSTFN) for i in range(depth): os.symlink('/'.join(['%d' % i] * 10), ABSTFN + '/%d' % (i + 1)) os.symlink('.', ABSTFN + '/0') - self.assertEqual(realpath(ABSTFN + '/%d' % depth), ABSTFN) + self.assertEqual(realpath(ABSTFN + '/%d' % depth, **kwargs), ABSTFN) # Test using relative path as well. with os_helper.change_cwd(ABSTFN): @@ -563,11 +724,12 @@ def test_realpath_deep_recursion(self): finally: for i in range(depth + 1): os_helper.unlink(ABSTFN + '/%d' % i) - safe_rmdir(ABSTFN) + os_helper.rmdir(ABSTFN) @os_helper.skip_unless_symlink @skip_if_ABSTFN_contains_backslash - def test_realpath_resolve_parents(self): + @_parameterize({}, {'strict': ALLOW_MISSING}) + def test_realpath_resolve_parents(self, kwargs): # We also need to resolve any symlinks in the parents of a relative # path passed to realpath. E.g.: current working directory is # /usr/doc with 'doc' being a symlink to /usr/share/doc. We call @@ -578,15 +740,17 @@ def test_realpath_resolve_parents(self): os.symlink(ABSTFN + "/y", ABSTFN + "/k") with os_helper.change_cwd(ABSTFN + "/k"): - self.assertEqual(realpath("a"), ABSTFN + "/y/a") + self.assertEqual(realpath("a", **kwargs), + ABSTFN + "/y/a") finally: os_helper.unlink(ABSTFN + "/k") - safe_rmdir(ABSTFN + "/y") - safe_rmdir(ABSTFN) + os_helper.rmdir(ABSTFN + "/y") + os_helper.rmdir(ABSTFN) @os_helper.skip_unless_symlink @skip_if_ABSTFN_contains_backslash - def test_realpath_resolve_before_normalizing(self): + @_parameterize({}, {'strict': True}, {'strict': ALLOW_MISSING}) + def test_realpath_resolve_before_normalizing(self, kwargs): # Bug #990669: Symbolic links should be resolved before we # normalize the path. E.g.: if we have directories 'a', 'k' and 'y' # in the following hierarchy: @@ -601,20 +765,21 @@ def test_realpath_resolve_before_normalizing(self): os.symlink(ABSTFN + "/k/y", ABSTFN + "/link-y") # Absolute path. - self.assertEqual(realpath(ABSTFN + "/link-y/.."), ABSTFN + "/k") + self.assertEqual(realpath(ABSTFN + "/link-y/..", **kwargs), ABSTFN + "/k") # Relative path. with os_helper.change_cwd(dirname(ABSTFN)): - self.assertEqual(realpath(basename(ABSTFN) + "/link-y/.."), + self.assertEqual(realpath(basename(ABSTFN) + "/link-y/..", **kwargs), ABSTFN + "/k") finally: os_helper.unlink(ABSTFN + "/link-y") - safe_rmdir(ABSTFN + "/k/y") - safe_rmdir(ABSTFN + "/k") - safe_rmdir(ABSTFN) + os_helper.rmdir(ABSTFN + "/k/y") + os_helper.rmdir(ABSTFN + "/k") + os_helper.rmdir(ABSTFN) @os_helper.skip_unless_symlink @skip_if_ABSTFN_contains_backslash - def test_realpath_resolve_first(self): + @_parameterize({}, {'strict': True}, {'strict': ALLOW_MISSING}) + def test_realpath_resolve_first(self, kwargs): # Bug #1213894: The first component of the path, if not absolute, # must be resolved too. @@ -624,17 +789,192 @@ def test_realpath_resolve_first(self): os.symlink(ABSTFN, ABSTFN + "link") with os_helper.change_cwd(dirname(ABSTFN)): base = basename(ABSTFN) - self.assertEqual(realpath(base + "link"), ABSTFN) - self.assertEqual(realpath(base + "link/k"), ABSTFN + "/k") + self.assertEqual(realpath(base + "link", **kwargs), ABSTFN) + self.assertEqual(realpath(base + "link/k", **kwargs), ABSTFN + "/k") finally: os_helper.unlink(ABSTFN + "link") - safe_rmdir(ABSTFN + "/k") - safe_rmdir(ABSTFN) + os_helper.rmdir(ABSTFN + "/k") + os_helper.rmdir(ABSTFN) + + @os_helper.skip_unless_symlink + @skip_if_ABSTFN_contains_backslash + @unittest.skipIf(os.chmod not in os.supports_follow_symlinks, "Can't set symlink permissions") + @unittest.skipIf(sys.platform != "darwin", "only macOS requires read permission to readlink()") + def test_realpath_unreadable_symlink(self): + try: + os.symlink(ABSTFN+"1", ABSTFN) + os.chmod(ABSTFN, 0o000, follow_symlinks=False) + self.assertEqual(realpath(ABSTFN), ABSTFN) + self.assertEqual(realpath(ABSTFN + '/foo'), ABSTFN + '/foo') + self.assertEqual(realpath(ABSTFN + '/../foo'), dirname(ABSTFN) + '/foo') + self.assertEqual(realpath(ABSTFN + '/foo/..'), ABSTFN) + finally: + os.chmod(ABSTFN, 0o755, follow_symlinks=False) + os_helper.unlink(ABSTFN) + + @os_helper.skip_unless_symlink + @skip_if_ABSTFN_contains_backslash + @unittest.skipIf(os.chmod not in os.supports_follow_symlinks, "Can't set symlink permissions") + @unittest.skipIf(sys.platform != "darwin", "only macOS requires read permission to readlink()") + @_parameterize({'strict': True}, {'strict': ALLOW_MISSING}) + def test_realpath_unreadable_symlink_strict(self, kwargs): + try: + os.symlink(ABSTFN+"1", ABSTFN) + os.chmod(ABSTFN, 0o000, follow_symlinks=False) + with self.assertRaises(PermissionError): + realpath(ABSTFN, **kwargs) + with self.assertRaises(PermissionError): + realpath(ABSTFN + '/foo', **kwargs), + with self.assertRaises(PermissionError): + realpath(ABSTFN + '/../foo', **kwargs) + with self.assertRaises(PermissionError): + realpath(ABSTFN + '/foo/..', **kwargs) + finally: + os.chmod(ABSTFN, 0o755, follow_symlinks=False) + os.unlink(ABSTFN) + + @skip_if_ABSTFN_contains_backslash + @os_helper.skip_unless_symlink + def test_realpath_unreadable_directory(self): + try: + os.mkdir(ABSTFN) + os.mkdir(ABSTFN + '/k') + os.chmod(ABSTFN, 0o000) + self.assertEqual(realpath(ABSTFN, strict=False), ABSTFN) + self.assertEqual(realpath(ABSTFN, strict=True), ABSTFN) + self.assertEqual(realpath(ABSTFN, strict=ALLOW_MISSING), ABSTFN) + + try: + os.stat(ABSTFN) + except PermissionError: + pass + else: + self.skipTest('Cannot block permissions') + + self.assertEqual(realpath(ABSTFN + '/k', strict=False), + ABSTFN + '/k') + self.assertRaises(PermissionError, realpath, ABSTFN + '/k', + strict=True) + self.assertRaises(PermissionError, realpath, ABSTFN + '/k', + strict=ALLOW_MISSING) + + self.assertEqual(realpath(ABSTFN + '/missing', strict=False), + ABSTFN + '/missing') + self.assertRaises(PermissionError, realpath, ABSTFN + '/missing', + strict=True) + self.assertRaises(PermissionError, realpath, ABSTFN + '/missing', + strict=ALLOW_MISSING) + finally: + os.chmod(ABSTFN, 0o755) + os_helper.rmdir(ABSTFN + '/k') + os_helper.rmdir(ABSTFN) + + @skip_if_ABSTFN_contains_backslash + def test_realpath_nonterminal_file(self): + try: + with open(ABSTFN, 'w') as f: + f.write('test_posixpath wuz ere') + self.assertEqual(realpath(ABSTFN, strict=False), ABSTFN) + self.assertEqual(realpath(ABSTFN, strict=True), ABSTFN) + self.assertEqual(realpath(ABSTFN, strict=ALLOW_MISSING), ABSTFN) + + self.assertEqual(realpath(ABSTFN + "/", strict=False), ABSTFN) + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/", strict=True) + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/", + strict=ALLOW_MISSING) + + self.assertEqual(realpath(ABSTFN + "/.", strict=False), ABSTFN) + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/.", strict=True) + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/.", + strict=ALLOW_MISSING) + + self.assertEqual(realpath(ABSTFN + "/..", strict=False), dirname(ABSTFN)) + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/..", strict=True) + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/..", + strict=ALLOW_MISSING) + + self.assertEqual(realpath(ABSTFN + "/subdir", strict=False), ABSTFN + "/subdir") + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/subdir", strict=True) + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/subdir", + strict=ALLOW_MISSING) + finally: + os_helper.unlink(ABSTFN) + + @os_helper.skip_unless_symlink + @skip_if_ABSTFN_contains_backslash + def test_realpath_nonterminal_symlink_to_file(self): + try: + with open(ABSTFN + "1", 'w') as f: + f.write('test_posixpath wuz ere') + os.symlink(ABSTFN + "1", ABSTFN) + self.assertEqual(realpath(ABSTFN, strict=False), ABSTFN + "1") + self.assertEqual(realpath(ABSTFN, strict=True), ABSTFN + "1") + self.assertEqual(realpath(ABSTFN, strict=ALLOW_MISSING), ABSTFN + "1") + + self.assertEqual(realpath(ABSTFN + "/", strict=False), ABSTFN + "1") + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/", strict=True) + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/", + strict=ALLOW_MISSING) + + self.assertEqual(realpath(ABSTFN + "/.", strict=False), ABSTFN + "1") + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/.", strict=True) + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/.", + strict=ALLOW_MISSING) + + self.assertEqual(realpath(ABSTFN + "/..", strict=False), dirname(ABSTFN)) + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/..", strict=True) + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/..", + strict=ALLOW_MISSING) + + self.assertEqual(realpath(ABSTFN + "/subdir", strict=False), ABSTFN + "1/subdir") + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/subdir", strict=True) + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/subdir", + strict=ALLOW_MISSING) + finally: + os_helper.unlink(ABSTFN) + os_helper.unlink(ABSTFN + "1") + + @os_helper.skip_unless_symlink + @skip_if_ABSTFN_contains_backslash + def test_realpath_nonterminal_symlink_to_symlinks_to_file(self): + try: + with open(ABSTFN + "2", 'w') as f: + f.write('test_posixpath wuz ere') + os.symlink(ABSTFN + "2", ABSTFN + "1") + os.symlink(ABSTFN + "1", ABSTFN) + self.assertEqual(realpath(ABSTFN, strict=False), ABSTFN + "2") + self.assertEqual(realpath(ABSTFN, strict=True), ABSTFN + "2") + self.assertEqual(realpath(ABSTFN, strict=True), ABSTFN + "2") + + self.assertEqual(realpath(ABSTFN + "/", strict=False), ABSTFN + "2") + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/", strict=True) + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/", + strict=ALLOW_MISSING) + + self.assertEqual(realpath(ABSTFN + "/.", strict=False), ABSTFN + "2") + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/.", strict=True) + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/.", + strict=ALLOW_MISSING) + + self.assertEqual(realpath(ABSTFN + "/..", strict=False), dirname(ABSTFN)) + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/..", strict=True) + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/..", + strict=ALLOW_MISSING) + + self.assertEqual(realpath(ABSTFN + "/subdir", strict=False), ABSTFN + "2/subdir") + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/subdir", strict=True) + self.assertRaises(NotADirectoryError, realpath, ABSTFN + "/subdir", + strict=ALLOW_MISSING) + finally: + os_helper.unlink(ABSTFN) + os_helper.unlink(ABSTFN + "1") + os_helper.unlink(ABSTFN + "2") def test_relpath(self): (real_getcwd, os.getcwd) = (os.getcwd, lambda: r"/home/user/bar") try: curdir = os.path.split(os.getcwd())[-1] + self.assertRaises(TypeError, posixpath.relpath, None) self.assertRaises(ValueError, posixpath.relpath, "") self.assertEqual(posixpath.relpath("a"), "a") self.assertEqual(posixpath.relpath(posixpath.abspath("a")), "a") @@ -697,7 +1037,9 @@ def check_error(exc, paths): self.assertRaises(exc, posixpath.commonpath, [os.fsencode(p) for p in paths]) + self.assertRaises(TypeError, posixpath.commonpath, None) self.assertRaises(ValueError, posixpath.commonpath, []) + self.assertRaises(ValueError, posixpath.commonpath, iter([])) check_error(ValueError, ['/usr', 'usr']) check_error(ValueError, ['usr', '/usr']) @@ -742,62 +1084,22 @@ def check_error(exc, paths): ['usr/lib/', b'/usr/lib/python3']) +# TODO: RUSTPYTHON @unittest.skip("TODO: RUSTPYTHON, flaky tests") class PosixCommonTest(test_genericpath.CommonTest, unittest.TestCase): pathmodule = posixpath attributes = ['relpath', 'samefile', 'sameopenfile', 'samestat'] - # TODO: RUSTPYTHON - if os.name == "posix" and os.getenv("CI"): - @unittest.expectedFailure - def test_exists(self): - super().test_exists() - - # TODO: RUSTPYTHON - import sys - @unittest.skipIf(sys.platform.startswith("linux") and os.getenv("CI"), "TODO: RUSTPYTHON, flaky test") - def test_filetime(self): - super().test_filetime() - - # TODO: RUSTPYTHON - if sys.platform.startswith("linux"): - @unittest.expectedFailure - def test_nonascii_abspath(self): - super().test_nonascii_abspath() - - # TODO: RUSTPYTHON - if os.name == "nt": - @unittest.expectedFailure - def test_samefile(self): - super().test_samefile() - - # TODO: RUSTPYTHON - if os.name == "nt": - @unittest.expectedFailure - def test_samefile_on_link(self): - super().test_samefile_on_link() - - # TODO: RUSTPYTHON - if os.name == "nt": - @unittest.expectedFailure - def test_samestat(self): - super().test_samestat() - - # TODO: RUSTPYTHON - if os.name == "nt": - @unittest.expectedFailure - def test_samestat_on_link(self): - super().test_samestat_on_link() - +# TODO: RUSTPYTHON @unittest.skipIf(os.getenv("CI"), "TODO: RUSTPYTHON, FileExistsError: (17, 'File exists (os error 17)')") class PathLikeTests(unittest.TestCase): path = posixpath def setUp(self): - self.file_name = os_helper.TESTFN - self.file_path = FakePath(os_helper.TESTFN) + self.file_name = TESTFN + self.file_path = FakePath(TESTFN) self.addCleanup(os_helper.unlink, self.file_name) with open(self.file_name, 'xb', 0) as file: file.write(b"test_posixpath.PathLikeTests") @@ -854,9 +1156,12 @@ def test_path_normpath(self): def test_path_abspath(self): self.assertPathEqual(self.path.abspath) - def test_path_realpath(self): + @_parameterize({}, {'strict': True}, {'strict': ALLOW_MISSING}) + def test_path_realpath(self, kwargs): self.assertPathEqual(self.path.realpath) + self.assertPathEqual(partial(self.path.realpath, **kwargs)) + def test_path_relpath(self): self.assertPathEqual(self.path.relpath) diff --git a/Lib/test/test_pow.py b/Lib/test/test_pow.py index 5cea9ceb20..eeb482ec4b 100644 --- a/Lib/test/test_pow.py +++ b/Lib/test/test_pow.py @@ -19,12 +19,11 @@ def powtest(self, type): self.assertEqual(pow(2, i), pow2) if i != 30 : pow2 = pow2*2 - for othertype in (int,): - for i in list(range(-10, 0)) + list(range(1, 10)): - ii = type(i) - for j in range(1, 11): - jj = -othertype(j) - pow(ii, jj) + for i in list(range(-10, 0)) + list(range(1, 10)): + ii = type(i) + inv = pow(ii, -1) # inverse of ii + for jj in range(-10, 0): + self.assertAlmostEqual(pow(ii, jj), pow(inv, -jj)) for othertype in int, float: for i in range(1, 100): diff --git a/Lib/test/test_pprint.py b/Lib/test/test_pprint.py index 4e6fed1ab9..ace75561f2 100644 --- a/Lib/test/test_pprint.py +++ b/Lib/test/test_pprint.py @@ -374,7 +374,7 @@ def __new__(cls, celsius_degrees): return super().__new__(Temperature, celsius_degrees) def __repr__(self): kelvin_degrees = self + 273.15 - return f"{kelvin_degrees}°K" + return f"{kelvin_degrees:.2f}°K" self.assertEqual(pprint.pformat(Temperature(1000)), '1273.15°K') def test_sorted_dict(self): diff --git a/Lib/test/test_print.py b/Lib/test/test_print.py index 8445a501cf..6107b7032f 100644 --- a/Lib/test/test_print.py +++ b/Lib/test/test_print.py @@ -129,14 +129,24 @@ def flush(self): raise RuntimeError self.assertRaises(RuntimeError, print, 1, file=noflush(), flush=True) + def test_gh130163(self): + class X: + def __str__(self): + sys.stdout = StringIO() + support.gc_collect() + return 'foo' + + with support.swap_attr(sys, 'stdout', None): + sys.stdout = StringIO() # the only reference + print(X()) # should not crash + class TestPy2MigrationHint(unittest.TestCase): """Test that correct hint is produced analogous to Python3 syntax, if print statement is executed as in Python 2. """ - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_normal_string(self): python2_print_str = 'print "Hello World"' with self.assertRaises(SyntaxError) as context: @@ -145,8 +155,7 @@ def test_normal_string(self): self.assertIn("Missing parentheses in call to 'print'. Did you mean print(...)", str(context.exception)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_string_with_soft_space(self): python2_print_str = 'print "Hello World",' with self.assertRaises(SyntaxError) as context: @@ -155,8 +164,7 @@ def test_string_with_soft_space(self): self.assertIn("Missing parentheses in call to 'print'. Did you mean print(...)", str(context.exception)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_string_with_excessive_whitespace(self): python2_print_str = 'print "Hello World", ' with self.assertRaises(SyntaxError) as context: @@ -165,8 +173,7 @@ def test_string_with_excessive_whitespace(self): self.assertIn("Missing parentheses in call to 'print'. Did you mean print(...)", str(context.exception)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_string_with_leading_whitespace(self): python2_print_str = '''if 1: print "Hello World" @@ -180,9 +187,7 @@ def test_string_with_leading_whitespace(self): # bpo-32685: Suggestions for print statement should be proper when # it is in the same line as the header of a compound statement # and/or followed by a semicolon - - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_string_with_semicolon(self): python2_print_str = 'print p;' with self.assertRaises(SyntaxError) as context: @@ -191,8 +196,7 @@ def test_string_with_semicolon(self): self.assertIn("Missing parentheses in call to 'print'. Did you mean print(...)", str(context.exception)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_string_in_loop_on_same_line(self): python2_print_str = 'for i in s: print i' with self.assertRaises(SyntaxError) as context: @@ -201,8 +205,7 @@ def test_string_in_loop_on_same_line(self): self.assertIn("Missing parentheses in call to 'print'. Did you mean print(...)", str(context.exception)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_stream_redirection_hint_for_py2_migration(self): # Test correct hint produced for Py2 redirection syntax with self.assertRaises(TypeError) as context: diff --git a/Lib/test/test_property.py b/Lib/test/test_property.py index 8411e903b1..cea241b0f2 100644 --- a/Lib/test/test_property.py +++ b/Lib/test/test_property.py @@ -183,26 +183,76 @@ def test_refleaks_in___init__(self): fake_prop.__init__('fget', 'fset', 'fdel', 'doc') self.assertAlmostEqual(gettotalrefcount() - refs_before, 0, delta=10) - @unittest.skipIf(sys.flags.optimize >= 2, - "Docstrings are omitted with -O2 and above") - def test_class_property(self): - class A: - @classmethod - @property - def __doc__(cls): - return 'A doc for %r' % cls.__name__ - self.assertEqual(A.__doc__, "A doc for 'A'") + @support.refcount_test + def test_gh_115618(self): + # Py_XDECREF() was improperly called for None argument + # in property methods. + gettotalrefcount = support.get_attribute(sys, 'gettotalrefcount') + prop = property() + refs_before = gettotalrefcount() + for i in range(100): + prop = prop.getter(None) + self.assertIsNone(prop.fget) + for i in range(100): + prop = prop.setter(None) + self.assertIsNone(prop.fset) + for i in range(100): + prop = prop.deleter(None) + self.assertIsNone(prop.fdel) + self.assertAlmostEqual(gettotalrefcount() - refs_before, 0, delta=10) + + def test_property_name(self): + def getter(self): + return 42 + + def setter(self, value): + pass - @unittest.skipIf(sys.flags.optimize >= 2, - "Docstrings are omitted with -O2 and above") - def test_class_property_override(self): class A: - """First""" - @classmethod @property - def __doc__(cls): - return 'Second' - self.assertEqual(A.__doc__, 'Second') + def foo(self): + return 1 + + @foo.setter + def oof(self, value): + pass + + bar = property(getter) + baz = property(None, setter) + + self.assertEqual(A.foo.__name__, 'foo') + self.assertEqual(A.oof.__name__, 'oof') + self.assertEqual(A.bar.__name__, 'bar') + self.assertEqual(A.baz.__name__, 'baz') + + A.quux = property(getter) + self.assertEqual(A.quux.__name__, 'getter') + A.quux.__name__ = 'myquux' + self.assertEqual(A.quux.__name__, 'myquux') + self.assertEqual(A.bar.__name__, 'bar') # not affected + A.quux.__name__ = None + self.assertIsNone(A.quux.__name__) + + with self.assertRaisesRegex( + AttributeError, "'property' object has no attribute '__name__'" + ): + property(None, setter).__name__ + + with self.assertRaisesRegex( + AttributeError, "'property' object has no attribute '__name__'" + ): + property(1).__name__ + + class Err: + def __getattr__(self, attr): + raise RuntimeError('fail') + + p = property(Err()) + with self.assertRaisesRegex(RuntimeError, 'fail'): + p.__name__ + + p.__name__ = 'not_fail' + self.assertEqual(p.__name__, 'not_fail') def test_property_set_name_incorrect_args(self): p = property() @@ -236,25 +286,110 @@ class A: class PropertySub(property): """This is a subclass of property""" +class PropertySubWoDoc(property): + pass + class PropertySubSlots(property): """This is a subclass of property that defines __slots__""" __slots__ = () class PropertySubclassTests(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure + @support.requires_docstrings def test_slots_docstring_copy_exception(self): - try: + # A special case error that we preserve despite the GH-98963 behavior + # that would otherwise silently ignore this error. + # This came from commit b18500d39d791c879e9904ebac293402b4a7cd34 + # as part of https://bugs.python.org/issue5890 which allowed docs to + # be set via property subclasses in the first place. + with self.assertRaises(AttributeError): class Foo(object): @PropertySubSlots def spam(self): """Trying to copy this docstring will raise an exception""" return 1 - except AttributeError: + + def test_property_with_slots_no_docstring(self): + # https://github.com/python/cpython/issues/98963#issuecomment-1574413319 + class slotted_prop(property): + __slots__ = ("foo",) + + p = slotted_prop() # no AttributeError + self.assertIsNone(getattr(p, "__doc__", None)) + + def undocumented_getter(): + return 4 + + p = slotted_prop(undocumented_getter) # New in 3.12: no AttributeError + self.assertIsNone(getattr(p, "__doc__", None)) + + @unittest.skipIf(sys.flags.optimize >= 2, + "Docstrings are omitted with -O2 and above") + def test_property_with_slots_docstring_silently_dropped(self): + # https://github.com/python/cpython/issues/98963#issuecomment-1574413319 + class slotted_prop(property): + __slots__ = ("foo",) + + p = slotted_prop(doc="what's up") # no AttributeError + self.assertIsNone(p.__doc__) + + def documented_getter(): + """getter doc.""" + return 4 + + # Historical behavior: A docstring from a getter always raises. + # (matches test_slots_docstring_copy_exception above). + with self.assertRaises(AttributeError): + p = slotted_prop(documented_getter) + + @unittest.skipIf(sys.flags.optimize >= 2, + "Docstrings are omitted with -O2 and above") + def test_property_with_slots_and_doc_slot_docstring_present(self): + # https://github.com/python/cpython/issues/98963#issuecomment-1574413319 + class slotted_prop(property): + __slots__ = ("foo", "__doc__") + + p = slotted_prop(doc="what's up") + self.assertEqual("what's up", p.__doc__) # new in 3.12: This gets set. + + def documented_getter(): + """what's up getter doc?""" + return 4 + + p = slotted_prop(documented_getter) + self.assertEqual("what's up getter doc?", p.__doc__) + + @unittest.skipIf(sys.flags.optimize >= 2, + "Docstrings are omitted with -O2 and above") + def test_issue41287(self): + + self.assertEqual(PropertySub.__doc__, "This is a subclass of property", + "Docstring of `property` subclass is ignored") + + doc = PropertySub(None, None, None, "issue 41287 is fixed").__doc__ + self.assertEqual(doc, "issue 41287 is fixed", + "Subclasses of `property` ignores `doc` constructor argument") + + def getter(x): + """Getter docstring""" + + def getter_wo_doc(x): pass - else: - raise Exception("AttributeError not raised") + + for ps in property, PropertySub, PropertySubWoDoc: + doc = ps(getter, None, None, "issue 41287 is fixed").__doc__ + self.assertEqual(doc, "issue 41287 is fixed", + "Getter overrides explicit property docstring (%s)" % ps.__name__) + + doc = ps(getter, None, None, None).__doc__ + self.assertEqual(doc, "Getter docstring", "Getter docstring is not picked-up (%s)" % ps.__name__) + + doc = ps(getter_wo_doc, None, None, "issue 41287 is fixed").__doc__ + self.assertEqual(doc, "issue 41287 is fixed", + "Getter overrides explicit property docstring (%s)" % ps.__name__) + + doc = ps(getter_wo_doc, None, None, None).__doc__ + self.assertIsNone(doc, "Property class doc appears in instance __doc__ (%s)" % ps.__name__) @unittest.skipIf(sys.flags.optimize >= 2, "Docstrings are omitted with -O2 and above") @@ -268,6 +403,100 @@ def spam(self): Foo.spam.__doc__, "spam wrapped in property subclass") + @unittest.skipIf(sys.flags.optimize >= 2, + "Docstrings are omitted with -O2 and above") + def test_docstring_copy2(self): + """ + Property tries to provide the best docstring it finds for its instances. + If a user-provided docstring is available, it is preserved on copies. + If no docstring is available during property creation, the property + will utilize the docstring from the getter if available. + """ + def getter1(self): + return 1 + def getter2(self): + """doc 2""" + return 2 + def getter3(self): + """doc 3""" + return 3 + + # Case-1: user-provided doc is preserved in copies + # of property with undocumented getter + p = property(getter1, None, None, "doc-A") + + p2 = p.getter(getter2) + self.assertEqual(p.__doc__, "doc-A") + self.assertEqual(p2.__doc__, "doc-A") + + # Case-2: user-provided doc is preserved in copies + # of property with documented getter + p = property(getter2, None, None, "doc-A") + + p2 = p.getter(getter3) + self.assertEqual(p.__doc__, "doc-A") + self.assertEqual(p2.__doc__, "doc-A") + + # Case-3: with no user-provided doc new getter doc + # takes precedence + p = property(getter2, None, None, None) + + p2 = p.getter(getter3) + self.assertEqual(p.__doc__, "doc 2") + self.assertEqual(p2.__doc__, "doc 3") + + # Case-4: A user-provided doc is assigned after property construction + # with documented getter. The doc IS NOT preserved. + # It's an odd behaviour, but it's a strange enough + # use case with no easy solution. + p = property(getter2, None, None, None) + p.__doc__ = "user" + p2 = p.getter(getter3) + self.assertEqual(p.__doc__, "user") + self.assertEqual(p2.__doc__, "doc 3") + + # Case-5: A user-provided doc is assigned after property construction + # with UNdocumented getter. The doc IS preserved. + p = property(getter1, None, None, None) + p.__doc__ = "user" + p2 = p.getter(getter2) + self.assertEqual(p.__doc__, "user") + self.assertEqual(p2.__doc__, "user") + + @unittest.skipIf(sys.flags.optimize >= 2, + "Docstrings are omitted with -O2 and above") + def test_prefer_explicit_doc(self): + # Issue 25757: subclasses of property lose docstring + self.assertEqual(property(doc="explicit doc").__doc__, "explicit doc") + self.assertEqual(PropertySub(doc="explicit doc").__doc__, "explicit doc") + + class Foo: + spam = PropertySub(doc="spam explicit doc") + + @spam.getter + def spam(self): + """ignored as doc already set""" + return 1 + + def _stuff_getter(self): + """ignored as doc set directly""" + stuff = PropertySub(doc="stuff doc argument", fget=_stuff_getter) + + #self.assertEqual(Foo.spam.__doc__, "spam explicit doc") + self.assertEqual(Foo.stuff.__doc__, "stuff doc argument") + + def test_property_no_doc_on_getter(self): + # If a property's getter has no __doc__ then the property's doc should + # be None; test that this is consistent with subclasses as well; see + # GH-2487 + class NoDoc: + @property + def __doc__(self): + raise AttributeError + + self.assertEqual(property(NoDoc()).__doc__, None) + self.assertEqual(PropertySub(NoDoc()).__doc__, None) + @unittest.skipIf(sys.flags.optimize >= 2, "Docstrings are omitted with -O2 and above") def test_property_setter_copies_getter_docstring(self): @@ -360,7 +589,6 @@ class cls: foo = property() - class PropertyUnreachableAttributeNoName(_PropertyUnreachableAttribute, unittest.TestCase): msg_format = r"^property of 'PropertyUnreachableAttributeNoName\.cls' object {}$" diff --git a/Lib/test/test_py_compile.py b/Lib/test/test_py_compile.py index 750afc1de7..54302eba4d 100644 --- a/Lib/test/test_py_compile.py +++ b/Lib/test/test_py_compile.py @@ -78,7 +78,6 @@ def test_absolute_path(self): self.assertTrue(os.path.exists(self.pyc_path)) self.assertFalse(os.path.exists(self.cache_path)) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_do_not_overwrite_symlinks(self): # In the face of a cfile argument being a symlink, bail out. # Issue #17222 @@ -267,7 +266,6 @@ def test_bad_syntax_with_quiet(self): self.assertEqual(stdout, b'') self.assertEqual(stderr, b'') - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_file_not_exists(self): should_not_exists = os.path.join(os.path.dirname(__file__), 'should_not_exists.py') rc, stdout, stderr = self.pycompilecmd_failure(self.source_path, should_not_exists) diff --git a/Lib/test/test_pyclbr.py b/Lib/test/test_pyclbr.py new file mode 100644 index 0000000000..ad26fe1dba --- /dev/null +++ b/Lib/test/test_pyclbr.py @@ -0,0 +1,272 @@ +''' + Test cases for pyclbr.py + Nick Mathewson +''' + +import sys +from textwrap import dedent +from types import FunctionType, MethodType, BuiltinFunctionType +import pyclbr +from unittest import TestCase, main as unittest_main +from test.test_importlib import util as test_importlib_util +import warnings +from test.support.testcase import ExtraAssertions + +import unittest # TODO: RUSTPYTHON + + +StaticMethodType = type(staticmethod(lambda: None)) +ClassMethodType = type(classmethod(lambda c: None)) + +# Here we test the python class browser code. +# +# The main function in this suite, 'testModule', compares the output +# of pyclbr with the introspected members of a module. Because pyclbr +# is imperfect (as designed), testModule is called with a set of +# members to ignore. + +class PyclbrTest(TestCase, ExtraAssertions): + + def assertListEq(self, l1, l2, ignore): + ''' succeed iff {l1} - {ignore} == {l2} - {ignore} ''' + missing = (set(l1) ^ set(l2)) - set(ignore) + if missing: + print("l1=%r\nl2=%r\nignore=%r" % (l1, l2, ignore), file=sys.stderr) + self.fail("%r missing" % missing.pop()) + + def assertHaskey(self, obj, key, ignore): + ''' succeed iff key in obj or key in ignore. ''' + if key in ignore: return + if key not in obj: + print("***",key, file=sys.stderr) + self.assertIn(key, obj) + + def assertEqualsOrIgnored(self, a, b, ignore): + ''' succeed iff a == b or a in ignore or b in ignore ''' + if a not in ignore and b not in ignore: + self.assertEqual(a, b) + + def checkModule(self, moduleName, module=None, ignore=()): + ''' succeed iff pyclbr.readmodule_ex(modulename) corresponds + to the actual module object, module. Any identifiers in + ignore are ignored. If no module is provided, the appropriate + module is loaded with __import__.''' + + ignore = set(ignore) | set(['object']) + + if module is None: + # Import it. + # ('<silly>' is to work around an API silliness in __import__) + module = __import__(moduleName, globals(), {}, ['<silly>']) + + dict = pyclbr.readmodule_ex(moduleName) + + def ismethod(oclass, obj, name): + classdict = oclass.__dict__ + if isinstance(obj, MethodType): + # could be a classmethod + if (not isinstance(classdict[name], ClassMethodType) or + obj.__self__ is not oclass): + return False + elif not isinstance(obj, FunctionType): + return False + + objname = obj.__name__ + if objname.startswith("__") and not objname.endswith("__"): + if stripped_typename := oclass.__name__.lstrip('_'): + objname = f"_{stripped_typename}{objname}" + return objname == name + + # Make sure the toplevel functions and classes are the same. + for name, value in dict.items(): + if name in ignore: + continue + self.assertHasAttr(module, name, ignore) + py_item = getattr(module, name) + if isinstance(value, pyclbr.Function): + self.assertIsInstance(py_item, (FunctionType, BuiltinFunctionType)) + if py_item.__module__ != moduleName: + continue # skip functions that came from somewhere else + self.assertEqual(py_item.__module__, value.module) + else: + self.assertIsInstance(py_item, type) + if py_item.__module__ != moduleName: + continue # skip classes that came from somewhere else + + real_bases = [base.__name__ for base in py_item.__bases__] + pyclbr_bases = [ getattr(base, 'name', base) + for base in value.super ] + + try: + self.assertListEq(real_bases, pyclbr_bases, ignore) + except: + print("class=%s" % py_item, file=sys.stderr) + raise + + actualMethods = [] + for m in py_item.__dict__.keys(): + if ismethod(py_item, getattr(py_item, m), m): + actualMethods.append(m) + + if stripped_typename := name.lstrip('_'): + foundMethods = [] + for m in value.methods.keys(): + if m.startswith('__') and not m.endswith('__'): + foundMethods.append(f"_{stripped_typename}{m}") + else: + foundMethods.append(m) + else: + foundMethods = list(value.methods.keys()) + + try: + self.assertListEq(foundMethods, actualMethods, ignore) + self.assertEqual(py_item.__module__, value.module) + + self.assertEqualsOrIgnored(py_item.__name__, value.name, + ignore) + # can't check file or lineno + except: + print("class=%s" % py_item, file=sys.stderr) + raise + + # Now check for missing stuff. + def defined_in(item, module): + if isinstance(item, type): + return item.__module__ == module.__name__ + if isinstance(item, FunctionType): + return item.__globals__ is module.__dict__ + return False + for name in dir(module): + item = getattr(module, name) + if isinstance(item, (type, FunctionType)): + if defined_in(item, module): + self.assertHaskey(dict, name, ignore) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_easy(self): + self.checkModule('pyclbr') + # XXX: Metaclasses are not supported + # self.checkModule('ast') + self.checkModule('doctest', ignore=("TestResults", "_SpoofOut", + "DocTestCase", '_DocTestSuite')) + self.checkModule('difflib', ignore=("Match",)) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_cases(self): + # see test.pyclbr_input for the rationale behind the ignored symbols + self.checkModule('test.pyclbr_input', ignore=['om', 'f']) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_nested(self): + mb = pyclbr + # Set arguments for descriptor creation and _creat_tree call. + m, p, f, t, i = 'test', '', 'test.py', {}, None + source = dedent("""\ + def f0(): + def f1(a,b,c): + def f2(a=1, b=2, c=3): pass + return f1(a,b,d) + class c1: pass + class C0: + "Test class." + def F1(): + "Method." + return 'return' + class C1(): + class C2: + "Class nested within nested class." + def F3(): return 1+1 + + """) + actual = mb._create_tree(m, p, f, source, t, i) + + # Create descriptors, linked together, and expected dict. + f0 = mb.Function(m, 'f0', f, 1, end_lineno=5) + f1 = mb._nest_function(f0, 'f1', 2, 4) + f2 = mb._nest_function(f1, 'f2', 3, 3) + c1 = mb._nest_class(f0, 'c1', 5, 5) + C0 = mb.Class(m, 'C0', None, f, 6, end_lineno=14) + F1 = mb._nest_function(C0, 'F1', 8, 10) + C1 = mb._nest_class(C0, 'C1', 11, 14) + C2 = mb._nest_class(C1, 'C2', 12, 14) + F3 = mb._nest_function(C2, 'F3', 14, 14) + expected = {'f0':f0, 'C0':C0} + + def compare(parent1, children1, parent2, children2): + """Return equality of tree pairs. + + Each parent,children pair define a tree. The parents are + assumed equal. Comparing the children dictionaries as such + does not work due to comparison by identity and double + linkage. We separate comparing string and number attributes + from comparing the children of input children. + """ + self.assertEqual(children1.keys(), children2.keys()) + for ob in children1.values(): + self.assertIs(ob.parent, parent1) + for ob in children2.values(): + self.assertIs(ob.parent, parent2) + for key in children1.keys(): + o1, o2 = children1[key], children2[key] + t1 = type(o1), o1.name, o1.file, o1.module, o1.lineno, o1.end_lineno + t2 = type(o2), o2.name, o2.file, o2.module, o2.lineno, o2.end_lineno + self.assertEqual(t1, t2) + if type(o1) is mb.Class: + self.assertEqual(o1.methods, o2.methods) + # Skip superclasses for now as not part of example + compare(o1, o1.children, o2, o2.children) + + compare(None, actual, None, expected) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_others(self): + cm = self.checkModule + + # These were once some of the longest modules. + cm('random', ignore=('Random',)) # from _random import Random as CoreGenerator + cm('pickle', ignore=('partial', 'PickleBuffer')) + with warnings.catch_warnings(): + warnings.simplefilter('ignore', DeprecationWarning) + cm('sre_parse', ignore=('dump', 'groups', 'pos')) # from sre_constants import *; property + cm( + 'pdb', + # pyclbr does not handle elegantly `typing` or properties + ignore=('Union', '_ModuleTarget', '_ScriptTarget', '_ZipTarget'), + ) + cm('pydoc', ignore=('input', 'output',)) # properties + + # Tests for modules inside packages + cm('email.parser') + cm('test.test_pyclbr') + + +class ReadmoduleTests(TestCase): + + def setUp(self): + self._modules = pyclbr._modules.copy() + + def tearDown(self): + pyclbr._modules = self._modules + + + def test_dotted_name_not_a_package(self): + # test ImportError is raised when the first part of a dotted name is + # not a package. + # + # Issue #14798. + self.assertRaises(ImportError, pyclbr.readmodule_ex, 'asyncio.foo') + + def test_module_has_no_spec(self): + module_name = "doesnotexist" + assert module_name not in pyclbr._modules + with test_importlib_util.uncache(module_name): + with self.assertRaises(ModuleNotFoundError): + pyclbr.readmodule_ex(module_name) + + +if __name__ == "__main__": + unittest_main() diff --git a/Lib/test/test_pyexpat.py b/Lib/test/test_pyexpat.py index 33e1ffb836..015e749726 100644 --- a/Lib/test/test_pyexpat.py +++ b/Lib/test/test_pyexpat.py @@ -1,13 +1,14 @@ # XXX TypeErrors on calling handlers, or on bad return values from a # handler, are obscure and unhelpful. -from io import BytesIO import os -import platform import sys import sysconfig import unittest import traceback +from io import BytesIO +from test import support +from test.support import os_helper from xml.parsers import expat from xml.parsers.expat import errors @@ -19,32 +20,24 @@ class SetAttributeTest(unittest.TestCase): def setUp(self): self.parser = expat.ParserCreate(namespace_separator='!') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_buffer_text(self): self.assertIs(self.parser.buffer_text, False) for x in 0, 1, 2, 0: self.parser.buffer_text = x self.assertIs(self.parser.buffer_text, bool(x)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_namespace_prefixes(self): self.assertIs(self.parser.namespace_prefixes, False) for x in 0, 1, 2, 0: self.parser.namespace_prefixes = x self.assertIs(self.parser.namespace_prefixes, bool(x)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_ordered_attributes(self): self.assertIs(self.parser.ordered_attributes, False) for x in 0, 1, 2, 0: self.parser.ordered_attributes = x self.assertIs(self.parser.ordered_attributes, bool(x)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_specified_attributes(self): self.assertIs(self.parser.specified_attributes, False) for x in 0, 1, 2, 0: @@ -234,8 +227,7 @@ def _verify_parse_output(self, operations): for operation, expected_operation in zip(operations, expected_operations): self.assertEqual(operation, expected_operation) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_parse_bytes(self): out = self.Outputter() parser = expat.ParserCreate(namespace_separator='!') @@ -248,8 +240,6 @@ def test_parse_bytes(self): # Issue #6697. self.assertRaises(AttributeError, getattr, parser, '\uD800') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_parse_str(self): out = self.Outputter() parser = expat.ParserCreate(namespace_separator='!') @@ -260,8 +250,6 @@ def test_parse_str(self): operations = out.out self._verify_parse_output(operations) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_parse_file(self): # Try parsing a file out = self.Outputter() @@ -274,8 +262,7 @@ def test_parse_file(self): operations = out.out self._verify_parse_output(operations) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_parse_again(self): parser = expat.ParserCreate() file = BytesIO(data) @@ -289,8 +276,7 @@ def test_parse_again(self): expat.errors.XML_ERROR_FINISHED) class NamespaceSeparatorTest(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_legal(self): # Tests that make sure we get errors when the namespace_separator value # is illegal, and that we don't for good values: @@ -298,15 +284,12 @@ def test_legal(self): expat.ParserCreate(namespace_separator=None) expat.ParserCreate(namespace_separator=' ') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_illegal(self): - try: + with self.assertRaisesRegex(TypeError, + r"ParserCreate\(\) argument (2|'namespace_separator') " + r"must be str or None, not int"): expat.ParserCreate(namespace_separator=42) - self.fail() - except TypeError as e: - self.assertEqual(str(e), - "ParserCreate() argument 'namespace_separator' must be str or None, not int") try: expat.ParserCreate(namespace_separator='too long') @@ -328,9 +311,7 @@ def test_zero_length(self): class InterningTest(unittest.TestCase): - - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test(self): # Test the interning machinery. p = expat.ParserCreate() @@ -346,8 +327,7 @@ def collector(name, *args): # L should have the same string repeated over and over. self.assertTrue(tag is entry) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_issue9402(self): # create an ExternalEntityParserCreate with buffer text class ExternalOutputter: @@ -405,8 +385,7 @@ def test_default_to_disabled(self): parser = expat.ParserCreate() self.assertFalse(parser.buffer_text) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_buffering_enabled(self): # Make sure buffering is turned on self.assertTrue(self.parser.buffer_text) @@ -414,8 +393,7 @@ def test_buffering_enabled(self): self.assertEqual(self.stuff, ['123'], "buffered text not properly collapsed") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test1(self): # XXX This test exposes more detail of Expat's text chunking than we # XXX like, but it tests what we need to concisely. @@ -425,23 +403,20 @@ def test1(self): ["<a>", "1", "<b>", "2", "\n", "3", "<c>", "4\n5"], "buffering control not reacting as expected") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test2(self): self.parser.Parse(b"<a>1<b/><2><c/> \n 3</a>", True) self.assertEqual(self.stuff, ["1<2> \n 3"], "buffered text not properly collapsed") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test3(self): self.setHandlers(["StartElementHandler"]) self.parser.Parse(b"<a>1<b/>2<c/>3</a>", True) self.assertEqual(self.stuff, ["<a>", "1", "<b>", "2", "<c>", "3"], "buffered text not properly split") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test4(self): self.setHandlers(["StartElementHandler", "EndElementHandler"]) self.parser.CharacterDataHandler = None @@ -449,16 +424,14 @@ def test4(self): self.assertEqual(self.stuff, ["<a>", "<b>", "</b>", "<c>", "</c>", "</a>"]) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test5(self): self.setHandlers(["StartElementHandler", "EndElementHandler"]) self.parser.Parse(b"<a>1<b></b>2<c/>3</a>", True) self.assertEqual(self.stuff, ["<a>", "1", "<b>", "</b>", "2", "<c>", "</c>", "3", "</a>"]) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test6(self): self.setHandlers(["CommentHandler", "EndElementHandler", "StartElementHandler"]) @@ -467,8 +440,7 @@ def test6(self): ["<a>", "1", "<b>", "</b>", "2", "<c>", "</c>", "345", "</a>"], "buffered text not properly split") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test7(self): self.setHandlers(["CommentHandler", "EndElementHandler", "StartElementHandler"]) @@ -482,35 +454,59 @@ def test7(self): # Test handling of exception from callback: class HandlerExceptionTest(unittest.TestCase): def StartElementHandler(self, name, attrs): - raise RuntimeError(name) + raise RuntimeError(f'StartElementHandler: <{name}>') def check_traceback_entry(self, entry, filename, funcname): - self.assertEqual(os.path.basename(entry[0]), filename) - self.assertEqual(entry[2], funcname) + self.assertEqual(os.path.basename(entry.filename), filename) + self.assertEqual(entry.name, funcname) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @support.cpython_only def test_exception(self): + # gh-66652: test _PyTraceback_Add() used by pyexpat.c to inject frames + + # Change the current directory to the Python source code directory + # if it is available. + src_dir = sysconfig.get_config_var('abs_builddir') + if src_dir: + have_source = os.path.isdir(src_dir) + else: + have_source = False + if have_source: + with os_helper.change_cwd(src_dir): + self._test_exception(have_source) + else: + self._test_exception(have_source) + + def _test_exception(self, have_source): + # Use path relative to the current directory which should be the Python + # source code directory (if it is available). + PYEXPAT_C = os.path.join('Modules', 'pyexpat.c') + parser = expat.ParserCreate() parser.StartElementHandler = self.StartElementHandler try: parser.Parse(b"<a><b><c/></b></a>", True) - self.fail() - except RuntimeError as e: - self.assertEqual(e.args[0], 'a', - "Expected RuntimeError for element 'a', but" + \ - " found %r" % e.args[0]) - # Check that the traceback contains the relevant line in pyexpat.c - entries = traceback.extract_tb(e.__traceback__) - self.assertEqual(len(entries), 3) - self.check_traceback_entry(entries[0], - "test_pyexpat.py", "test_exception") - self.check_traceback_entry(entries[1], - "pyexpat.c", "StartElement") - self.check_traceback_entry(entries[2], - "test_pyexpat.py", "StartElementHandler") - if sysconfig.is_python_build() and not (sys.platform == 'win32' and platform.machine() == 'ARM'): - self.assertIn('call_with_frame("StartElement"', entries[1][3]) + + self.fail("the parser did not raise RuntimeError") + except RuntimeError as exc: + self.assertEqual(exc.args[0], 'StartElementHandler: <a>', exc) + entries = traceback.extract_tb(exc.__traceback__) + + self.assertEqual(len(entries), 3, entries) + self.check_traceback_entry(entries[0], + "test_pyexpat.py", "_test_exception") + self.check_traceback_entry(entries[1], + os.path.basename(PYEXPAT_C), + "StartElement") + self.check_traceback_entry(entries[2], + "test_pyexpat.py", "StartElementHandler") + + # Check that the traceback contains the relevant line in + # Modules/pyexpat.c. Skip the test if Modules/pyexpat.c is not + # available. + if have_source and os.path.exists(PYEXPAT_C): + self.assertIn('call_with_frame("StartElement"', + entries[1].line) # Test Current* members: @@ -533,8 +529,7 @@ def check_pos(self, event): 'Expected position %s, got position %s' %(pos, expected)) self.upto += 1 - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test(self): self.parser = expat.ParserCreate() self.parser.StartElementHandler = self.StartElementHandler @@ -548,9 +543,9 @@ def test(self): class sf1296433Test(unittest.TestCase): - + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: Expected type 'str' but 'bytes' found. def test_parse_only_xml_data(self): - # http://python.org/sf/1296433 + # https://bugs.python.org/issue1296433 # xml = "<?xml version='1.0' encoding='iso8859'?><s>%s</s>" % ('a' * 1025) # this one doesn't crash @@ -565,25 +560,22 @@ def handler(text): parser = expat.ParserCreate() parser.CharacterDataHandler = handler - self.assertRaises(Exception, parser.Parse, xml.encode('iso8859')) + self.assertRaises(SpecificException, parser.Parse, xml.encode('iso8859')) class ChardataBufferTest(unittest.TestCase): """ test setting of chardata buffer size """ - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_1025_bytes(self): self.assertEqual(self.small_buffer_test(1025), 2) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_1000_bytes(self): self.assertEqual(self.small_buffer_test(1000), 1) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_wrong_size(self): parser = expat.ParserCreate() parser.buffer_text = 1 @@ -596,8 +588,7 @@ def test_wrong_size(self): with self.assertRaises(TypeError): parser.buffer_size = 512.0 - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_unchanged_size(self): xml1 = b"<?xml version='1.0' encoding='iso8859'?><s>" + b'a' * 512 xml2 = b'a'*512 + b'</s>' @@ -620,8 +611,8 @@ def test_unchanged_size(self): parser.Parse(xml2) self.assertEqual(self.n, 2) - # TODO: RUSTPYTHON - @unittest.expectedFailure + + @unittest.expectedFailure # TODO: RUSTPYTHON def test_disabling_buffer(self): xml1 = b"<?xml version='1.0' encoding='iso8859'?><a>" + b'a' * 512 xml2 = b'b' * 1024 @@ -666,8 +657,7 @@ def small_buffer_test(self, buffer_len): parser.Parse(xml) return self.n - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_change_size_1(self): xml1 = b"<?xml version='1.0' encoding='iso8859'?><a><s>" + b'a' * 1024 xml2 = b'aaa</s><s>' + b'a' * 1025 + b'</s></a>' @@ -684,8 +674,7 @@ def test_change_size_1(self): parser.Parse(xml2, True) self.assertEqual(self.n, 2) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_change_size_2(self): xml1 = b"<?xml version='1.0' encoding='iso8859'?><a>a<s>" + b'a' * 1023 xml2 = b'aaa</s><s>' + b'a' * 1025 + b'</s></a>' @@ -703,9 +692,7 @@ def test_change_size_2(self): self.assertEqual(self.n, 4) class MalformedInputTest(unittest.TestCase): - - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test1(self): xml = b"\0\r\n" parser = expat.ParserCreate() @@ -715,8 +702,7 @@ def test1(self): except expat.ExpatError as e: self.assertEqual(str(e), 'unclosed token: line 2, column 0') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test2(self): # \xc2\x85 is UTF-8 encoded U+0085 (NEXT LINE) xml = b"<?xml version\xc2\x85='1.0'?>\r\n" @@ -726,16 +712,13 @@ def test2(self): parser.Parse(xml, True) class ErrorMessageTest(unittest.TestCase): - - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_codes(self): # verify mapping of errors.codes and errors.messages self.assertEqual(errors.XML_ERROR_SYNTAX, errors.messages[errors.codes[errors.XML_ERROR_SYNTAX]]) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_expaterror(self): xml = b'<' parser = expat.ParserCreate() @@ -751,9 +734,7 @@ class ForeignDTDTests(unittest.TestCase): """ Tests for the UseForeignDTD method of expat parser objects. """ - - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_use_foreign_dtd(self): """ If UseForeignDTD is passed True and a document without an external @@ -782,8 +763,7 @@ def resolve_entity(context, base, system_id, public_id): parser.Parse(b"<?xml version='1.0'?><element/>") self.assertEqual(handler_call_args, [(None, None)]) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_ignore_use_foreign_dtd(self): """ If UseForeignDTD is passed True and a document with an external @@ -804,5 +784,101 @@ def resolve_entity(context, base, system_id, public_id): self.assertEqual(handler_call_args, [("bar", "baz")]) +class ParentParserLifetimeTest(unittest.TestCase): + """ + Subparsers make use of their parent XML_Parser inside of Expat. + As a result, parent parsers need to outlive subparsers. + + See https://github.com/python/cpython/issues/139400. + """ + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'xmlparser' object has no attribute 'ExternalEntityParserCreate' + def test_parent_parser_outlives_its_subparsers__single(self): + parser = expat.ParserCreate() + subparser = parser.ExternalEntityParserCreate(None) + + # Now try to cause garbage collection of the parent parser + # while it's still being referenced by a related subparser. + del parser + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'xmlparser' object has no attribute 'ExternalEntityParserCreate' + def test_parent_parser_outlives_its_subparsers__multiple(self): + parser = expat.ParserCreate() + subparser_one = parser.ExternalEntityParserCreate(None) + subparser_two = parser.ExternalEntityParserCreate(None) + + # Now try to cause garbage collection of the parent parser + # while it's still being referenced by a related subparser. + del parser + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'xmlparser' object has no attribute 'ExternalEntityParserCreate' + def test_parent_parser_outlives_its_subparsers__chain(self): + parser = expat.ParserCreate() + subparser = parser.ExternalEntityParserCreate(None) + subsubparser = subparser.ExternalEntityParserCreate(None) + + # Now try to cause garbage collection of the parent parsers + # while they are still being referenced by a related subparser. + del parser + del subparser + + +class ReparseDeferralTest(unittest.TestCase): + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'xmlparser' object has no attribute 'GetReparseDeferralEnabled' + def test_getter_setter_round_trip(self): + parser = expat.ParserCreate() + enabled = (expat.version_info >= (2, 6, 0)) + + self.assertIs(parser.GetReparseDeferralEnabled(), enabled) + parser.SetReparseDeferralEnabled(False) + self.assertIs(parser.GetReparseDeferralEnabled(), False) + parser.SetReparseDeferralEnabled(True) + self.assertIs(parser.GetReparseDeferralEnabled(), enabled) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'xmlparser' object has no attribute 'GetReparseDeferralEnabled' + def test_reparse_deferral_enabled(self): + if expat.version_info < (2, 6, 0): + self.skipTest(f'Expat {expat.version_info} does not ' + 'support reparse deferral') + + started = [] + + def start_element(name, _): + started.append(name) + + parser = expat.ParserCreate() + parser.StartElementHandler = start_element + self.assertTrue(parser.GetReparseDeferralEnabled()) + + for chunk in (b'<doc', b'/>'): + parser.Parse(chunk, False) + + # The key test: Have handlers already fired? Expecting: no. + self.assertEqual(started, []) + + parser.Parse(b'', True) + + self.assertEqual(started, ['doc']) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'xmlparser' object has no attribute 'SetReparseDeferralEnabled' + def test_reparse_deferral_disabled(self): + started = [] + + def start_element(name, _): + started.append(name) + + parser = expat.ParserCreate() + parser.StartElementHandler = start_element + if expat.version_info >= (2, 6, 0): + parser.SetReparseDeferralEnabled(False) + self.assertFalse(parser.GetReparseDeferralEnabled()) + + for chunk in (b'<doc', b'/>'): + parser.Parse(chunk, False) + + # The key test: Have handlers already fired? Expecting: yes. + self.assertEqual(started, ['doc']) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_quopri.py b/Lib/test/test_quopri.py index 715544c8a9..152d1858dc 100644 --- a/Lib/test/test_quopri.py +++ b/Lib/test/test_quopri.py @@ -3,6 +3,7 @@ import sys, io, subprocess import quopri +from test import support ENCSAMPLE = b"""\ @@ -180,6 +181,7 @@ def test_decode_header(self): for p, e in self.HSTRINGS: self.assertEqual(quopri.decodestring(e, header=True), p) + @support.requires_subprocess() def test_scriptencode(self): (p, e) = self.STRINGS[-1] process = subprocess.Popen([sys.executable, "-mquopri"], @@ -196,6 +198,7 @@ def test_scriptencode(self): self.assertEqual(cout[i], e[i]) self.assertEqual(cout, e) + @support.requires_subprocess() def test_scriptdecode(self): (p, e) = self.STRINGS[-1] process = subprocess.Popen([sys.executable, "-mquopri", "-d"], diff --git a/Lib/test/test_raise.py b/Lib/test/test_raise.py index 94f42c84f1..53fce0a501 100644 --- a/Lib/test/test_raise.py +++ b/Lib/test/test_raise.py @@ -185,6 +185,17 @@ def test_class_cause(self): else: self.fail("No exception raised") + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: 'classmethod' object is not callable + def test_class_cause_nonexception_result(self): + # See https://github.com/python/cpython/issues/140530. + class ConstructMortal(BaseException): + def __new__(*args, **kwargs): + return ["mortal value"] + + msg = ".*should have returned an instance of BaseException.*" + with self.assertRaisesRegex(TypeError, msg): + raise IndexError from ConstructMortal + def test_instance_cause(self): cause = KeyError() try: @@ -233,8 +244,7 @@ class TestTracebackType(unittest.TestCase): def raiser(self): raise ValueError - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_attrs(self): try: self.raiser() @@ -270,8 +280,6 @@ def test_attrs(self): tb.tb_next = new_tb self.assertIs(tb.tb_next, new_tb) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_constructor(self): other_tb = get_tb() frame = sys._getframe() diff --git a/Lib/test/test_regrtest.py b/Lib/test/test_regrtest.py index fc56ec4afc..d9ae2f3548 100644 --- a/Lib/test/test_regrtest.py +++ b/Lib/test/test_regrtest.py @@ -820,8 +820,6 @@ def test_fromfile(self): output = self.run_tests('--fromfile', filename) self.check_executed_tests(output, tests) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_interrupted(self): code = TEST_INTERRUPTED test = self.create_test('sigint', code=code) @@ -839,8 +837,6 @@ def test_slowest(self): % (self.TESTNAME_REGEX, len(tests))) self.check_line(output, regex) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_slowest_interrupted(self): # Issue #25373: test --slowest with an interrupted test code = TEST_INTERRUPTED diff --git a/Lib/test/test_reprlib.py b/Lib/test/test_reprlib.py index 396be4b104..738b48f562 100644 --- a/Lib/test/test_reprlib.py +++ b/Lib/test/test_reprlib.py @@ -176,8 +176,6 @@ def test_instance(self): self.assertTrue(s.endswith(">")) self.assertIn(s.find("..."), [12, 13]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_lambda(self): r = repr(lambda x: x) self.assertTrue(r.startswith("<function ReprTests.test_lambda.<locals>.<lambda"), r) diff --git a/Lib/test/test_richcmp.py b/Lib/test/test_richcmp.py index 58729a9fea..5f449cdc05 100644 --- a/Lib/test/test_richcmp.py +++ b/Lib/test/test_richcmp.py @@ -221,6 +221,7 @@ def do(bad): self.assertRaises(Exc, func, Bad()) @support.no_tracing + @support.infinite_recursion(25) def test_recursion(self): # Check that comparison for recursive objects fails gracefully from collections import UserList diff --git a/Lib/test/test_rlcompleter.py b/Lib/test/test_rlcompleter.py index 0dc1080ca3..ffadfee276 100644 --- a/Lib/test/test_rlcompleter.py +++ b/Lib/test/test_rlcompleter.py @@ -2,6 +2,7 @@ from unittest.mock import patch import builtins import rlcompleter +from test.support import MISSING_C_DOCSTRINGS class CompleteMe: """ Trivial class used in testing rlcompleter.Completer. """ @@ -40,21 +41,41 @@ def test_global_matches(self): # test with a customized namespace self.assertEqual(self.completer.global_matches('CompleteM'), - ['CompleteMe(']) + ['CompleteMe(' if MISSING_C_DOCSTRINGS else 'CompleteMe()']) self.assertEqual(self.completer.global_matches('eg'), ['egg(']) # XXX: see issue5256 self.assertEqual(self.completer.global_matches('CompleteM'), - ['CompleteMe(']) + ['CompleteMe(' if MISSING_C_DOCSTRINGS else 'CompleteMe()']) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_attr_matches(self): # test with builtins namespace self.assertEqual(self.stdcompleter.attr_matches('str.s'), ['str.{}('.format(x) for x in dir(str) if x.startswith('s')]) self.assertEqual(self.stdcompleter.attr_matches('tuple.foospamegg'), []) - expected = sorted({'None.%s%s' % (x, '(' if x != '__doc__' else '') - for x in dir(None)}) + + def create_expected_for_none(): + if not MISSING_C_DOCSTRINGS: + parentheses = ('__init_subclass__', '__class__') + else: + # When `--without-doc-strings` is used, `__class__` + # won't have a known signature. + parentheses = ('__init_subclass__',) + + items = set() + for x in dir(None): + if x in parentheses: + items.add(f'None.{x}()') + elif x == '__doc__': + items.add(f'None.{x}') + else: + items.add(f'None.{x}(') + return sorted(items) + + expected = create_expected_for_none() self.assertEqual(self.stdcompleter.attr_matches('None.'), expected) self.assertEqual(self.stdcompleter.attr_matches('None._'), expected) self.assertEqual(self.stdcompleter.attr_matches('None.__'), expected) @@ -64,7 +85,7 @@ def test_attr_matches(self): ['CompleteMe.spam']) self.assertEqual(self.completer.attr_matches('Completeme.egg'), []) self.assertEqual(self.completer.attr_matches('CompleteMe.'), - ['CompleteMe.mro(', 'CompleteMe.spam']) + ['CompleteMe.mro()', 'CompleteMe.spam']) self.assertEqual(self.completer.attr_matches('CompleteMe._'), ['CompleteMe._ham']) matches = self.completer.attr_matches('CompleteMe.__') @@ -81,17 +102,41 @@ def test_attr_matches(self): if x.startswith('s')]) def test_excessive_getattr(self): - # Ensure getattr() is invoked no more than once per attribute + """Ensure getattr() is invoked no more than once per attribute""" + + # note the special case for @property methods below; that is why + # we use __dir__ and __getattr__ in class Foo to create a "magic" + # class attribute 'bar'. This forces `getattr` to call __getattr__ + # (which is doesn't necessarily do). class Foo: calls = 0 + bar = '' + def __getattribute__(self, name): + if name == 'bar': + self.calls += 1 + return None + return super().__getattribute__(name) + + f = Foo() + completer = rlcompleter.Completer(dict(f=f)) + self.assertEqual(completer.complete('f.b', 0), 'f.bar') + self.assertEqual(f.calls, 1) + + def test_property_method_not_called(self): + class Foo: + _bar = 0 + property_called = False + @property def bar(self): - self.calls += 1 - return None + self.property_called = True + return self._bar + f = Foo() completer = rlcompleter.Completer(dict(f=f)) self.assertEqual(completer.complete('f.b', 0), 'f.bar') - self.assertEqual(f.calls, 1) + self.assertFalse(f.property_called) + def test_uncreated_attr(self): # Attributes like properties and slots should be completed even when @@ -114,6 +159,9 @@ def test_complete(self): self.assertEqual(completer.complete('el', 0), 'elif ') self.assertEqual(completer.complete('el', 1), 'else') self.assertEqual(completer.complete('tr', 0), 'try:') + self.assertEqual(completer.complete('_', 0), '_') + self.assertEqual(completer.complete('match', 0), 'match ') + self.assertEqual(completer.complete('case', 0), 'case ') def test_duplicate_globals(self): namespace = { @@ -134,7 +182,7 @@ def test_duplicate_globals(self): # No opening bracket "(" because we overrode the built-in class self.assertEqual(completer.complete('memoryview', 0), 'memoryview') self.assertIsNone(completer.complete('memoryview', 1)) - self.assertEqual(completer.complete('Ellipsis', 0), 'Ellipsis(') + self.assertEqual(completer.complete('Ellipsis', 0), 'Ellipsis()') self.assertIsNone(completer.complete('Ellipsis', 1)) if __name__ == '__main__': diff --git a/Lib/test/test_runpy.py b/Lib/test/test_runpy.py index b2fee6207b..c1e255e7af 100644 --- a/Lib/test/test_runpy.py +++ b/Lib/test/test_runpy.py @@ -12,9 +12,16 @@ import textwrap import unittest import warnings -from test.support import no_tracing, verbose +from test.support import ( + force_not_colorized_test_class, + infinite_recursion, + no_tracing, + requires_resource, + requires_subprocess, + verbose, +) from test.support.import_helper import forget, make_legacy_pyc, unload -from test.support.os_helper import create_empty_file, temp_dir +from test.support.os_helper import create_empty_file, temp_dir, FakePath from test.support.script_helper import make_script, make_zip_script @@ -656,13 +663,14 @@ def test_basic_script(self): self._check_script(script_name, "<run_path>", script_name, script_name, expect_spec=False) - def test_basic_script_with_path_object(self): + def test_basic_script_with_pathlike_object(self): with temp_dir() as script_dir: mod_name = 'script' - script_name = pathlib.Path(self._make_test_script(script_dir, - mod_name)) - self._check_script(script_name, "<run_path>", script_name, - script_name, expect_spec=False) + script_name = self._make_test_script(script_dir, mod_name) + self._check_script(FakePath(script_name), "<run_path>", + script_name, + script_name, + expect_spec=False) def test_basic_script_no_suffix(self): with temp_dir() as script_dir: @@ -734,6 +742,7 @@ def test_zipfile_error(self): self._check_import_error(zip_name, msg) @no_tracing + @requires_resource('cpu') def test_main_recursion_error(self): with temp_dir() as script_dir, temp_dir() as dummy_dir: mod_name = '__main__' @@ -741,10 +750,10 @@ def test_main_recursion_error(self): "runpy.run_path(%r)\n") % dummy_dir script_name = self._make_test_script(script_dir, mod_name, source) zip_name, fname = make_zip_script(script_dir, 'test_zip', script_name) - self.assertRaises(RecursionError, run_path, zip_name) + with infinite_recursion(25): + self.assertRaises(RecursionError, run_path, zip_name) - # TODO: RUSTPYTHON, detect encoding comments in files - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; detect encoding comments in files def test_encoding(self): with temp_dir() as script_dir: filename = os.path.join(script_dir, 'script.py') @@ -757,6 +766,7 @@ def test_encoding(self): self.assertEqual(result['s'], "non-ASCII: h\xe9") +@force_not_colorized_test_class class TestExit(unittest.TestCase): STATUS_CONTROL_C_EXIT = 0xC000013A EXPECTED_CODE = ( @@ -783,16 +793,17 @@ def run(self, *args, **kwargs): ) super().run(*args, **kwargs) - def assertSigInt(self, *args, **kwargs): - proc = subprocess.run(*args, **kwargs, text=True, stderr=subprocess.PIPE) - self.assertTrue(proc.stderr.endswith("\nKeyboardInterrupt\n")) + @requires_subprocess() + def assertSigInt(self, cmd, *args, **kwargs): + # Use -E to ignore PYTHONSAFEPATH + cmd = [sys.executable, '-E', *cmd] + proc = subprocess.run(cmd, *args, **kwargs, text=True, stderr=subprocess.PIPE) + self.assertTrue(proc.stderr.endswith("\nKeyboardInterrupt\n"), proc.stderr) self.assertEqual(proc.returncode, self.EXPECTED_CODE) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_pymain_run_file(self): - self.assertSigInt([sys.executable, self.ham]) + self.assertSigInt([self.ham]) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_pymain_run_file_runpy_run_module(self): tmp = self.ham.parent run_module = tmp / "run_module.py" @@ -804,9 +815,8 @@ def test_pymain_run_file_runpy_run_module(self): """ ) ) - self.assertSigInt([sys.executable, run_module], cwd=tmp) + self.assertSigInt([run_module], cwd=tmp) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_pymain_run_file_runpy_run_module_as_main(self): tmp = self.ham.parent run_module_as_main = tmp / "run_module_as_main.py" @@ -818,28 +828,24 @@ def test_pymain_run_file_runpy_run_module_as_main(self): """ ) ) - self.assertSigInt([sys.executable, run_module_as_main], cwd=tmp) + self.assertSigInt([run_module_as_main], cwd=tmp) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_pymain_run_command_run_module(self): self.assertSigInt( - [sys.executable, "-c", "import runpy; runpy.run_module('ham')"], + ["-c", "import runpy; runpy.run_module('ham')"], cwd=self.ham.parent, ) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_pymain_run_command(self): - self.assertSigInt([sys.executable, "-c", "import ham"], cwd=self.ham.parent) + self.assertSigInt(["-c", "import ham"], cwd=self.ham.parent) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_pymain_run_stdin(self): - self.assertSigInt([sys.executable], input="import ham", cwd=self.ham.parent) + self.assertSigInt([], input="import ham", cwd=self.ham.parent) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_pymain_run_module(self): ham = self.ham - self.assertSigInt([sys.executable, "-m", ham.stem], cwd=ham.parent) + self.assertSigInt(["-m", ham.stem], cwd=ham.parent) if __name__ == "__main__": diff --git a/Lib/test/test_scope.py b/Lib/test/test_scope.py index 29a4ac3c16..662d8eefbc 100644 --- a/Lib/test/test_scope.py +++ b/Lib/test/test_scope.py @@ -177,6 +177,57 @@ def bar(): self.assertEqual(foo(a=42), 50) self.assertEqual(foo(), 25) + def testCellIsArgAndEscapes(self): + # We need to be sure that a cell passed in as an arg still + # gets wrapped in a new cell if the arg escapes into an + # inner function (closure). + + def external(): + value = 42 + def inner(): + return value + cell, = inner.__closure__ + return cell + cell_ext = external() + + def spam(arg): + def eggs(): + return arg + return eggs + + eggs = spam(cell_ext) + cell_closure, = eggs.__closure__ + cell_eggs = eggs() + + self.assertIs(cell_eggs, cell_ext) + self.assertIsNot(cell_eggs, cell_closure) + + def testCellIsLocalAndEscapes(self): + # We need to be sure that a cell bound to a local still + # gets wrapped in a new cell if the local escapes into an + # inner function (closure). + + def external(): + value = 42 + def inner(): + return value + cell, = inner.__closure__ + return cell + cell_ext = external() + + def spam(arg): + cell = arg + def eggs(): + return cell + return eggs + + eggs = spam(cell_ext) + cell_closure, = eggs.__closure__ + cell_eggs = eggs() + + self.assertIs(cell_eggs, cell_ext) + self.assertIsNot(cell_eggs, cell_closure) + def testRecursion(self): def f(x): @@ -641,10 +692,7 @@ def dec(self): self.assertEqual(c.dec(), 1) self.assertEqual(c.dec(), 0) - # TODO: RUSTPYTHON, figure out how to communicate that `y = 9` should be - # stored as a global rather than a STORE_NAME, even when - # the `global y` is in a nested subscope - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; figure out how to communicate that `y = 9` should be stored as a global rather than a STORE_NAME, even when the `global y` is in a nested subscope def testGlobalInParallelNestedFunctions(self): # A symbol table bug leaked the global statement from one # function to other nested functions in the same block. @@ -763,6 +811,30 @@ def dig(self): gc_collect() # For PyPy or other GCs. self.assertIsNone(ref()) + def test_multiple_nesting(self): + # Regression test for https://github.com/python/cpython/issues/121863 + class MultiplyNested: + def f1(self): + __arg = 1 + class D: + def g(self, __arg): + return __arg + return D().g(_MultiplyNested__arg=2) + + def f2(self): + __arg = 1 + class D: + def g(self, __arg): + return __arg + return D().g + + inst = MultiplyNested() + with self.assertRaises(TypeError): + inst.f1() + + closure = inst.f2() + with self.assertRaises(TypeError): + closure(_MultiplyNested__arg=2) if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_selectors.py b/Lib/test/test_selectors.py index af730a8dec..643775597c 100644 --- a/Lib/test/test_selectors.py +++ b/Lib/test/test_selectors.py @@ -6,8 +6,7 @@ import socket import sys from test import support -from test.support import os_helper -from test.support import socket_helper +from test.support import is_apple, os_helper, socket_helper from time import sleep import unittest import unittest.mock @@ -132,7 +131,6 @@ def test_unregister_after_fd_close_and_reuse(self): s.unregister(r) s.unregister(w) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_unregister_after_socket_close(self): s = self.SELECTOR() self.addCleanup(s.close) @@ -224,6 +222,8 @@ def test_close(self): self.assertRaises(RuntimeError, s.get_key, wr) self.assertRaises(KeyError, mapping.__getitem__, rd) self.assertRaises(KeyError, mapping.__getitem__, wr) + self.assertEqual(mapping.get(rd), None) + self.assertEqual(mapping.get(wr), None) def test_get_key(self): s = self.SELECTOR() @@ -242,13 +242,17 @@ def test_get_map(self): self.addCleanup(s.close) rd, wr = self.make_socketpair() + sentinel = object() keys = s.get_map() self.assertFalse(keys) self.assertEqual(len(keys), 0) self.assertEqual(list(keys), []) + self.assertEqual(keys.get(rd), None) + self.assertEqual(keys.get(rd, sentinel), sentinel) key = s.register(rd, selectors.EVENT_READ, "data") self.assertIn(rd, keys) + self.assertEqual(key, keys.get(rd)) self.assertEqual(key, keys[rd]) self.assertEqual(len(keys), 1) self.assertEqual(list(keys), [rd.fileno()]) @@ -521,7 +525,7 @@ def test_above_fd_setsize(self): try: fds = s.select() except OSError as e: - if e.errno == errno.EINVAL and sys.platform == 'darwin': + if e.errno == errno.EINVAL and is_apple: # unexplainable errors on macOS don't need to fail the test self.skipTest("Invalid argument error calling poll()") raise diff --git a/Lib/test/test_setcomps.py b/Lib/test/test_setcomps.py index ecc4fffec0..e8c0c33e98 100644 --- a/Lib/test/test_setcomps.py +++ b/Lib/test/test_setcomps.py @@ -1,3 +1,10 @@ +import doctest +import traceback +import unittest + +from test.support import BrokenIter + + doctests = """ ########### Tests mostly copied from test_listcomps.py ############ @@ -144,24 +151,50 @@ """ +class SetComprehensionTest(unittest.TestCase): + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'FrameSummary' object has no attribute 'end_lineno' + def test_exception_locations(self): + # The location of an exception raised from __init__ or + # __next__ should should be the iterator expression + + def init_raises(): + try: + {x for x in BrokenIter(init_raises=True)} + except Exception as e: + return e + + def next_raises(): + try: + {x for x in BrokenIter(next_raises=True)} + except Exception as e: + return e + + def iter_raises(): + try: + {x for x in BrokenIter(iter_raises=True)} + except Exception as e: + return e + + for func, expected in [(init_raises, "BrokenIter(init_raises=True)"), + (next_raises, "BrokenIter(next_raises=True)"), + (iter_raises, "BrokenIter(iter_raises=True)"), + ]: + with self.subTest(func): + exc = func() + f = traceback.extract_tb(exc.__traceback__)[0] + indent = 16 + co = func.__code__ + self.assertEqual(f.lineno, co.co_firstlineno + 2) + self.assertEqual(f.end_lineno, co.co_firstlineno + 2) + self.assertEqual(f.line[f.colno - indent : f.end_colno - indent], + expected) __test__ = {'doctests' : doctests} -def test_main(verbose=None): - import sys - from test import support - from test import test_setcomps - support.run_doctest(test_setcomps, verbose) - - # verify reference counting - if verbose and hasattr(sys, "gettotalrefcount"): - import gc - counts = [None] * 5 - for i in range(len(counts)): - support.run_doctest(test_setcomps, verbose) - gc.collect() - counts[i] = sys.gettotalrefcount() - print(counts) +def load_tests(loader, tests, pattern): + tests.addTest(doctest.DocTestSuite()) + return tests + if __name__ == "__main__": - test_main(verbose=True) + unittest.main() diff --git a/Lib/test/test_shlex.py b/Lib/test/test_shlex.py index c3d632a64c..baabccf19f 100644 --- a/Lib/test/test_shlex.py +++ b/Lib/test/test_shlex.py @@ -3,7 +3,6 @@ import shlex import string import unittest -from unittest import mock # The original test data set was from shellwords, by Hartmut Goebel. @@ -162,18 +161,17 @@ def oldSplit(self, s): tok = lex.get_token() return ret - @mock.patch('sys.stdin', io.StringIO()) - def testSplitNoneDeprecation(self): - with self.assertWarns(DeprecationWarning): + def testSplitNone(self): + with self.assertRaises(ValueError): shlex.split(None) - # TODO: RUSTPYTHON + # TODO: RUSTPYTHON; ValueError: Error Retrieving Value @unittest.expectedFailure def testSplitPosix(self): """Test data splitting with posix parser""" self.splitTest(self.posix_data, comments=True) - # TODO: RUSTPYTHON + # TODO: RUSTPYTHON; ValueError: Error Retrieving Value @unittest.expectedFailure def testCompat(self): """Test compatibility interface""" @@ -315,7 +313,7 @@ def testEmptyStringHandling(self): s = shlex.shlex("'')abc", punctuation_chars=True) self.assertEqual(list(s), expected) - # TODO: RUSTPYTHON + # TODO: RUSTPYTHON; ValueError: Error Retrieving Value @unittest.expectedFailure def testUnicodeHandling(self): """Test punctuation_chars and whitespace_split handle unicode.""" @@ -356,7 +354,7 @@ def testJoin(self): joined = shlex.join(split_command) self.assertEqual(joined, command) - # TODO: RUSTPYTHON + # TODO: RUSTPYTHON; ValueError: Error Retrieving Value @unittest.expectedFailure def testJoinRoundtrip(self): all_data = self.data + self.posix_data diff --git a/Lib/test/test_shutil.py b/Lib/test/test_shutil.py index b64ccb37a5..73ad81e871 100644 --- a/Lib/test/test_shutil.py +++ b/Lib/test/test_shutil.py @@ -23,7 +23,6 @@ unregister_unpack_format, get_unpack_formats, SameFileError, _GiveupOnFastCopy) import tarfile -import warnings import zipfile try: import posix @@ -71,18 +70,17 @@ def wrap(*args, **kwargs): os.rename = builtin_rename return wrap -def write_file(path, content, binary=False): +def create_file(path, content=b''): """Write *content* to a file located at *path*. If *path* is a tuple instead of a string, os.path.join will be used to - make a path. If *binary* is true, the file will be opened in binary - mode. + make a path. """ if isinstance(path, tuple): path = os.path.join(*path) - mode = 'wb' if binary else 'w' - encoding = None if binary else "utf-8" - with open(path, mode, encoding=encoding) as fp: + if isinstance(content, str): + content = content.encode() + with open(path, 'xb') as fp: fp.write(content) def write_test_file(path, size): @@ -191,12 +189,12 @@ def test_rmtree_works_on_bytes(self): tmp = self.mkdtemp() victim = os.path.join(tmp, 'killme') os.mkdir(victim) - write_file(os.path.join(victim, 'somefile'), 'foo') + create_file(os.path.join(victim, 'somefile'), 'foo') victim = os.fsencode(victim) self.assertIsInstance(victim, bytes) shutil.rmtree(victim) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") + @unittest.skipIf(sys.platform == 'win32', 'TODO: RUSTPYTHON; flaky') @os_helper.skip_unless_symlink def test_rmtree_fails_on_symlink_onerror(self): tmp = self.mkdtemp() @@ -216,7 +214,6 @@ def onerror(*args): self.assertEqual(errors[0][1], link) self.assertIsInstance(errors[0][2][1], OSError) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") @os_helper.skip_unless_symlink def test_rmtree_fails_on_symlink_onexc(self): tmp = self.mkdtemp() @@ -236,7 +233,6 @@ def onexc(*args): self.assertEqual(errors[0][1], link) self.assertIsInstance(errors[0][2], OSError) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") @os_helper.skip_unless_symlink def test_rmtree_works_on_symlinks(self): tmp = self.mkdtemp() @@ -246,7 +242,7 @@ def test_rmtree_works_on_symlinks(self): for d in dir1, dir2, dir3: os.mkdir(d) file1 = os.path.join(tmp, 'file1') - write_file(file1, 'foo') + create_file(file1, 'foo') link1 = os.path.join(dir1, 'link1') os.symlink(dir2, link1) link2 = os.path.join(dir1, 'link2') @@ -259,7 +255,6 @@ def test_rmtree_works_on_symlinks(self): self.assertTrue(os.path.exists(dir3)) self.assertTrue(os.path.exists(file1)) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") @unittest.skipUnless(_winapi, 'only relevant on Windows') def test_rmtree_fails_on_junctions_onerror(self): tmp = self.mkdtemp() @@ -280,7 +275,6 @@ def onerror(*args): self.assertEqual(errors[0][1], link) self.assertIsInstance(errors[0][2][1], OSError) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") @unittest.skipUnless(_winapi, 'only relevant on Windows') def test_rmtree_fails_on_junctions_onexc(self): tmp = self.mkdtemp() @@ -301,7 +295,6 @@ def onexc(*args): self.assertEqual(errors[0][1], link) self.assertIsInstance(errors[0][2], OSError) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") @unittest.skipUnless(_winapi, 'only relevant on Windows') def test_rmtree_works_on_junctions(self): tmp = self.mkdtemp() @@ -311,7 +304,7 @@ def test_rmtree_works_on_junctions(self): for d in dir1, dir2, dir3: os.mkdir(d) file1 = os.path.join(tmp, 'file1') - write_file(file1, 'foo') + create_file(file1, 'foo') link1 = os.path.join(dir1, 'link1') _winapi.CreateJunction(dir2, link1) link2 = os.path.join(dir1, 'link2') @@ -324,7 +317,7 @@ def test_rmtree_works_on_junctions(self): self.assertTrue(os.path.exists(dir3)) self.assertTrue(os.path.exists(file1)) - def test_rmtree_errors_onerror(self): + def test_rmtree_errors(self): # filename is guaranteed not to exist filename = tempfile.mktemp(dir=self.mkdtemp()) self.assertRaises(FileNotFoundError, shutil.rmtree, filename) @@ -333,8 +326,8 @@ def test_rmtree_errors_onerror(self): # existing file tmpdir = self.mkdtemp() - write_file((tmpdir, "tstfile"), "") filename = os.path.join(tmpdir, "tstfile") + create_file(filename) with self.assertRaises(NotADirectoryError) as cm: shutil.rmtree(filename) self.assertEqual(cm.exception.filename, filename) @@ -342,6 +335,19 @@ def test_rmtree_errors_onerror(self): # test that ignore_errors option is honored shutil.rmtree(filename, ignore_errors=True) self.assertTrue(os.path.exists(filename)) + + self.assertRaises(TypeError, shutil.rmtree, None) + self.assertRaises(TypeError, shutil.rmtree, None, ignore_errors=True) + exc = TypeError if shutil.rmtree.avoids_symlink_attacks else NotImplementedError + with self.assertRaises(exc): + shutil.rmtree(filename, dir_fd='invalid') + with self.assertRaises(exc): + shutil.rmtree(filename, dir_fd='invalid', ignore_errors=True) + + def test_rmtree_errors_onerror(self): + tmpdir = self.mkdtemp() + filename = os.path.join(tmpdir, "tstfile") + create_file(filename) errors = [] def onerror(*args): errors.append(args) @@ -357,23 +363,9 @@ def onerror(*args): self.assertEqual(errors[1][2][1].filename, filename) def test_rmtree_errors_onexc(self): - # filename is guaranteed not to exist - filename = tempfile.mktemp(dir=self.mkdtemp()) - self.assertRaises(FileNotFoundError, shutil.rmtree, filename) - # test that ignore_errors option is honored - shutil.rmtree(filename, ignore_errors=True) - - # existing file tmpdir = self.mkdtemp() - write_file((tmpdir, "tstfile"), "") filename = os.path.join(tmpdir, "tstfile") - with self.assertRaises(NotADirectoryError) as cm: - shutil.rmtree(filename) - self.assertEqual(cm.exception.filename, filename) - self.assertTrue(os.path.exists(filename)) - # test that ignore_errors option is honored - shutil.rmtree(filename, ignore_errors=True) - self.assertTrue(os.path.exists(filename)) + create_file(filename) errors = [] def onexc(*args): errors.append(args) @@ -555,7 +547,7 @@ def raiser(fn, *args, **kwargs): os.lstat = raiser os.mkdir(TESTFN) - write_file((TESTFN, 'foo'), 'foo') + create_file((TESTFN, 'foo'), 'foo') shutil.rmtree(TESTFN) finally: os.lstat = orig_lstat @@ -628,7 +620,7 @@ def test_rmtree_with_dir_fd(self): self.addCleanup(os.close, dir_fd) os.mkdir(fullname) os.mkdir(os.path.join(fullname, 'subdir')) - write_file(os.path.join(fullname, 'subdir', 'somefile'), 'foo') + create_file(os.path.join(fullname, 'subdir', 'somefile'), 'foo') self.assertTrue(os.path.exists(fullname)) shutil.rmtree(victim, dir_fd=dir_fd) self.assertFalse(os.path.exists(fullname)) @@ -661,7 +653,6 @@ def test_rmtree_on_symlink(self): finally: shutil.rmtree(TESTFN, ignore_errors=True) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") @unittest.skipUnless(_winapi, 'only relevant on Windows') def test_rmtree_on_junction(self): os.mkdir(TESTFN) @@ -669,7 +660,7 @@ def test_rmtree_on_junction(self): src = os.path.join(TESTFN, 'cheese') dst = os.path.join(TESTFN, 'shop') os.mkdir(src) - open(os.path.join(src, 'spam'), 'wb').close() + create_file(os.path.join(src, 'spam')) _winapi.CreateJunction(src, dst) self.assertRaises(OSError, shutil.rmtree, dst) shutil.rmtree(dst, ignore_errors=True) @@ -693,6 +684,73 @@ def test_rmtree_on_named_pipe(self): shutil.rmtree(TESTFN) self.assertFalse(os.path.exists(TESTFN)) + @unittest.skipIf(sys.platform[:6] == 'cygwin', + "This test can't be run on Cygwin (issue #1071513).") + @os_helper.skip_if_dac_override + @os_helper.skip_unless_working_chmod + def test_rmtree_deleted_race_condition(self): + # bpo-37260 + # + # Test that a file or a directory deleted after it is enumerated + # by scandir() but before unlink() or rmdr() is called doesn't + # generate any errors. + def _onexc(fn, path, exc): + assert fn in (os.rmdir, os.unlink) + if not isinstance(exc, PermissionError): + raise + # Make the parent and the children writeable. + for p, mode in zip(paths, old_modes): + os.chmod(p, mode) + # Remove other dirs except one. + keep = next(p for p in dirs if p != path) + for p in dirs: + if p != keep: + os.rmdir(p) + # Remove other files except one. + keep = next(p for p in files if p != path) + for p in files: + if p != keep: + os.unlink(p) + + os.mkdir(TESTFN) + paths = [TESTFN] + [os.path.join(TESTFN, f'child{i}') + for i in range(6)] + dirs = paths[1::2] + files = paths[2::2] + for path in dirs: + os.mkdir(path) + for path in files: + create_file(path) + + old_modes = [os.stat(path).st_mode for path in paths] + + # Make the parent and the children non-writeable. + new_mode = stat.S_IREAD|stat.S_IEXEC + for path in reversed(paths): + os.chmod(path, new_mode) + + try: + shutil.rmtree(TESTFN, onexc=_onexc) + except: + # Test failed, so cleanup artifacts. + for path, mode in zip(paths, old_modes): + try: + os.chmod(path, mode) + except OSError: + pass + shutil.rmtree(TESTFN) + raise + + def test_rmtree_above_recursion_limit(self): + recursion_limit = 40 + # directory_depth > recursion_limit + directory_depth = recursion_limit + 10 + base = os.path.join(TESTFN, *(['d'] * directory_depth)) + os.makedirs(base) + + with support.infinite_recursion(recursion_limit): + shutil.rmtree(TESTFN) + class TestCopyTree(BaseTest, unittest.TestCase): @@ -701,9 +759,9 @@ def test_copytree_simple(self): dst_dir = os.path.join(self.mkdtemp(), 'destination') self.addCleanup(shutil.rmtree, src_dir) self.addCleanup(shutil.rmtree, os.path.dirname(dst_dir)) - write_file((src_dir, 'test.txt'), '123') + create_file((src_dir, 'test.txt'), '123') os.mkdir(os.path.join(src_dir, 'test_dir')) - write_file((src_dir, 'test_dir', 'test.txt'), '456') + create_file((src_dir, 'test_dir', 'test.txt'), '456') shutil.copytree(src_dir, dst_dir) self.assertTrue(os.path.isfile(os.path.join(dst_dir, 'test.txt'))) @@ -721,11 +779,11 @@ def test_copytree_dirs_exist_ok(self): self.addCleanup(shutil.rmtree, src_dir) self.addCleanup(shutil.rmtree, dst_dir) - write_file((src_dir, 'nonexisting.txt'), '123') + create_file((src_dir, 'nonexisting.txt'), '123') os.mkdir(os.path.join(src_dir, 'existing_dir')) os.mkdir(os.path.join(dst_dir, 'existing_dir')) - write_file((dst_dir, 'existing_dir', 'existing.txt'), 'will be replaced') - write_file((src_dir, 'existing_dir', 'existing.txt'), 'has been replaced') + create_file((dst_dir, 'existing_dir', 'existing.txt'), 'will be replaced') + create_file((src_dir, 'existing_dir', 'existing.txt'), 'has been replaced') shutil.copytree(src_dir, dst_dir, dirs_exist_ok=True) self.assertTrue(os.path.isfile(os.path.join(dst_dir, 'nonexisting.txt'))) @@ -740,7 +798,6 @@ def test_copytree_dirs_exist_ok(self): with self.assertRaises(FileExistsError): shutil.copytree(src_dir, dst_dir, dirs_exist_ok=False) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") @os_helper.skip_unless_symlink def test_copytree_symlinks(self): tmp_dir = self.mkdtemp() @@ -749,7 +806,7 @@ def test_copytree_symlinks(self): sub_dir = os.path.join(src_dir, 'sub') os.mkdir(src_dir) os.mkdir(sub_dir) - write_file((src_dir, 'file.txt'), 'foo') + create_file((src_dir, 'file.txt'), 'foo') src_link = os.path.join(sub_dir, 'link') dst_link = os.path.join(dst_dir, 'sub/link') os.symlink(os.path.join(src_dir, 'file.txt'), @@ -780,16 +837,16 @@ def test_copytree_with_exclude(self): src_dir = self.mkdtemp() try: dst_dir = join(self.mkdtemp(), 'destination') - write_file((src_dir, 'test.txt'), '123') - write_file((src_dir, 'test.tmp'), '123') + create_file((src_dir, 'test.txt'), '123') + create_file((src_dir, 'test.tmp'), '123') os.mkdir(join(src_dir, 'test_dir')) - write_file((src_dir, 'test_dir', 'test.txt'), '456') + create_file((src_dir, 'test_dir', 'test.txt'), '456') os.mkdir(join(src_dir, 'test_dir2')) - write_file((src_dir, 'test_dir2', 'test.txt'), '456') + create_file((src_dir, 'test_dir2', 'test.txt'), '456') os.mkdir(join(src_dir, 'test_dir2', 'subdir')) os.mkdir(join(src_dir, 'test_dir2', 'subdir2')) - write_file((src_dir, 'test_dir2', 'subdir', 'test.txt'), '456') - write_file((src_dir, 'test_dir2', 'subdir2', 'test.py'), '456') + create_file((src_dir, 'test_dir2', 'subdir', 'test.txt'), '456') + create_file((src_dir, 'test_dir2', 'subdir2', 'test.py'), '456') # testing glob-like patterns try: @@ -848,7 +905,7 @@ def test_copytree_arg_types_of_ignore(self): os.mkdir(join(src_dir)) os.mkdir(join(src_dir, 'test_dir')) os.mkdir(os.path.join(src_dir, 'test_dir', 'subdir')) - write_file((src_dir, 'test_dir', 'subdir', 'test.txt'), '456') + create_file((src_dir, 'test_dir', 'subdir', 'test.txt'), '456') invokations = [] @@ -867,7 +924,7 @@ def _ignore(src, names): 'test.txt'))) dst_dir = join(self.mkdtemp(), 'destination') - shutil.copytree(pathlib.Path(src_dir), dst_dir, ignore=_ignore) + shutil.copytree(FakePath(src_dir), dst_dir, ignore=_ignore) self.assertTrue(exists(join(dst_dir, 'test_dir', 'subdir', 'test.txt'))) @@ -888,9 +945,9 @@ def test_copytree_retains_permissions(self): self.addCleanup(shutil.rmtree, tmp_dir) os.chmod(src_dir, 0o777) - write_file((src_dir, 'permissive.txt'), '123') + create_file((src_dir, 'permissive.txt'), '123') os.chmod(os.path.join(src_dir, 'permissive.txt'), 0o777) - write_file((src_dir, 'restrictive.txt'), '456') + create_file((src_dir, 'restrictive.txt'), '456') os.chmod(os.path.join(src_dir, 'restrictive.txt'), 0o600) restrictive_subdir = tempfile.mkdtemp(dir=src_dir) self.addCleanup(os_helper.rmtree, restrictive_subdir) @@ -933,8 +990,7 @@ def custom_cpfun(a, b): flag = [] src = self.mkdtemp() dst = tempfile.mktemp(dir=self.mkdtemp()) - with open(os.path.join(src, 'foo'), 'w', encoding='utf-8') as f: - f.close() + create_file(os.path.join(src, 'foo')) shutil.copytree(src, dst, copy_function=custom_cpfun) self.assertEqual(len(flag), 1) @@ -969,9 +1025,9 @@ def test_copytree_named_pipe(self): def test_copytree_special_func(self): src_dir = self.mkdtemp() dst_dir = os.path.join(self.mkdtemp(), 'destination') - write_file((src_dir, 'test.txt'), '123') + create_file((src_dir, 'test.txt'), '123') os.mkdir(os.path.join(src_dir, 'test_dir')) - write_file((src_dir, 'test_dir', 'test.txt'), '456') + create_file((src_dir, 'test_dir', 'test.txt'), '456') copied = [] def _copy(src, dst): @@ -980,12 +1036,11 @@ def _copy(src, dst): shutil.copytree(src_dir, dst_dir, copy_function=_copy) self.assertEqual(len(copied), 2) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") @os_helper.skip_unless_symlink def test_copytree_dangling_symlinks(self): src_dir = self.mkdtemp() valid_file = os.path.join(src_dir, 'test.txt') - write_file(valid_file, 'abc') + create_file(valid_file, 'abc') dir_a = os.path.join(src_dir, 'dir_a') os.mkdir(dir_a) for d in src_dir, dir_a: @@ -1008,14 +1063,12 @@ def test_copytree_dangling_symlinks(self): shutil.copytree(src_dir, dst_dir, symlinks=True) self.assertIn('test.txt', os.listdir(dst_dir)) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") @os_helper.skip_unless_symlink def test_copytree_symlink_dir(self): src_dir = self.mkdtemp() dst_dir = os.path.join(self.mkdtemp(), 'destination') os.mkdir(os.path.join(src_dir, 'real_dir')) - with open(os.path.join(src_dir, 'real_dir', 'test.txt'), 'wb'): - pass + create_file(os.path.join(src_dir, 'real_dir', 'test.txt')) os.symlink(os.path.join(src_dir, 'real_dir'), os.path.join(src_dir, 'link_to_dir'), target_is_directory=True) @@ -1035,7 +1088,7 @@ def test_copytree_return_value(self): dst_dir = src_dir + "dest" self.addCleanup(shutil.rmtree, dst_dir, True) src = os.path.join(src_dir, 'foo') - write_file(src, 'foo') + create_file(src, 'foo') rv = shutil.copytree(src_dir, dst_dir) self.assertEqual(['foo'], os.listdir(rv)) @@ -1047,7 +1100,7 @@ def test_copytree_subdirectory(self): dst_dir = os.path.join(src_dir, "somevendor", "1.0") os.makedirs(src_dir) src = os.path.join(src_dir, 'pol') - write_file(src, 'pol') + create_file(src, 'pol') rv = shutil.copytree(src_dir, dst_dir) self.assertEqual(['pol'], os.listdir(rv)) @@ -1055,7 +1108,6 @@ class TestCopy(BaseTest, unittest.TestCase): ### shutil.copymode - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") @os_helper.skip_unless_symlink def test_copymode_follow_symlinks(self): tmp_dir = self.mkdtemp() @@ -1063,8 +1115,8 @@ def test_copymode_follow_symlinks(self): dst = os.path.join(tmp_dir, 'bar') src_link = os.path.join(tmp_dir, 'baz') dst_link = os.path.join(tmp_dir, 'quux') - write_file(src, 'foo') - write_file(dst, 'foo') + create_file(src, 'foo') + create_file(dst, 'foo') os.symlink(src, src_link) os.symlink(dst, dst_link) os.chmod(src, stat.S_IRWXU|stat.S_IRWXG) @@ -1087,36 +1139,34 @@ def test_copymode_follow_symlinks(self): shutil.copymode(src_link, dst_link) self.assertEqual(os.stat(src).st_mode, os.stat(dst).st_mode) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") - @unittest.skipUnless(hasattr(os, 'lchmod') or os.name == 'nt', 'requires os.lchmod') + @unittest.skipUnless(hasattr(os, 'lchmod'), 'requires os.lchmod') @os_helper.skip_unless_symlink def test_copymode_symlink_to_symlink(self): - _lchmod = os.chmod if os.name == 'nt' else os.lchmod tmp_dir = self.mkdtemp() src = os.path.join(tmp_dir, 'foo') dst = os.path.join(tmp_dir, 'bar') src_link = os.path.join(tmp_dir, 'baz') dst_link = os.path.join(tmp_dir, 'quux') - write_file(src, 'foo') - write_file(dst, 'foo') + create_file(src, 'foo') + create_file(dst, 'foo') os.symlink(src, src_link) os.symlink(dst, dst_link) os.chmod(src, stat.S_IRWXU|stat.S_IRWXG) os.chmod(dst, stat.S_IRWXU) - _lchmod(src_link, stat.S_IRWXO|stat.S_IRWXG) + os.lchmod(src_link, stat.S_IRWXO|stat.S_IRWXG) # link to link - _lchmod(dst_link, stat.S_IRWXO) + os.lchmod(dst_link, stat.S_IRWXO) old_mode = os.stat(dst).st_mode shutil.copymode(src_link, dst_link, follow_symlinks=False) self.assertEqual(os.lstat(src_link).st_mode, os.lstat(dst_link).st_mode) self.assertEqual(os.stat(dst).st_mode, old_mode) # src link - use chmod - _lchmod(dst_link, stat.S_IRWXO) + os.lchmod(dst_link, stat.S_IRWXO) shutil.copymode(src_link, dst, follow_symlinks=False) self.assertEqual(os.stat(src).st_mode, os.stat(dst).st_mode) # dst link - use chmod - _lchmod(dst_link, stat.S_IRWXO) + os.lchmod(dst_link, stat.S_IRWXO) shutil.copymode(src, dst_link, follow_symlinks=False) self.assertEqual(os.stat(src).st_mode, os.stat(dst).st_mode) @@ -1128,15 +1178,14 @@ def test_copymode_symlink_to_symlink_wo_lchmod(self): dst = os.path.join(tmp_dir, 'bar') src_link = os.path.join(tmp_dir, 'baz') dst_link = os.path.join(tmp_dir, 'quux') - write_file(src, 'foo') - write_file(dst, 'foo') + create_file(src, 'foo') + create_file(dst, 'foo') os.symlink(src, src_link) os.symlink(dst, dst_link) shutil.copymode(src_link, dst_link, follow_symlinks=False) # silent fail ### shutil.copystat - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") @os_helper.skip_unless_symlink def test_copystat_symlinks(self): tmp_dir = self.mkdtemp() @@ -1144,23 +1193,21 @@ def test_copystat_symlinks(self): dst = os.path.join(tmp_dir, 'bar') src_link = os.path.join(tmp_dir, 'baz') dst_link = os.path.join(tmp_dir, 'qux') - write_file(src, 'foo') + create_file(src, 'foo') src_stat = os.stat(src) os.utime(src, (src_stat.st_atime, src_stat.st_mtime - 42.0)) # ensure different mtimes - write_file(dst, 'bar') + create_file(dst, 'bar') self.assertNotEqual(os.stat(src).st_mtime, os.stat(dst).st_mtime) os.symlink(src, src_link) os.symlink(dst, dst_link) if hasattr(os, 'lchmod'): os.lchmod(src_link, stat.S_IRWXO) - elif os.name == 'nt': - os.chmod(src_link, stat.S_IRWXO) if hasattr(os, 'lchflags') and hasattr(stat, 'UF_NODUMP'): os.lchflags(src_link, stat.UF_NODUMP) src_link_stat = os.lstat(src_link) # follow - if hasattr(os, 'lchmod') or os.name == 'nt': + if hasattr(os, 'lchmod'): shutil.copystat(src_link, dst_link, follow_symlinks=True) self.assertNotEqual(src_link_stat.st_mode, os.stat(dst).st_mode) # don't follow @@ -1171,7 +1218,7 @@ def test_copystat_symlinks(self): # The modification times may be truncated in the new file. self.assertLessEqual(getattr(src_link_stat, attr), getattr(dst_link_stat, attr) + 1) - if hasattr(os, 'lchmod') or os.name == 'nt': + if hasattr(os, 'lchmod'): self.assertEqual(src_link_stat.st_mode, dst_link_stat.st_mode) if hasattr(os, 'lchflags') and hasattr(src_link_stat, 'st_flags'): self.assertEqual(src_link_stat.st_flags, dst_link_stat.st_flags) @@ -1188,8 +1235,8 @@ def test_copystat_handles_harmless_chflags_errors(self): tmpdir = self.mkdtemp() file1 = os.path.join(tmpdir, 'file1') file2 = os.path.join(tmpdir, 'file2') - write_file(file1, 'xxx') - write_file(file2, 'xxx') + create_file(file1, 'xxx') + create_file(file2, 'xxx') def make_chflags_raiser(err): ex = OSError() @@ -1215,9 +1262,9 @@ def _chflags_raiser(path, flags, *, follow_symlinks=True): def test_copyxattr(self): tmp_dir = self.mkdtemp() src = os.path.join(tmp_dir, 'foo') - write_file(src, 'foo') + create_file(src, 'foo') dst = os.path.join(tmp_dir, 'bar') - write_file(dst, 'bar') + create_file(dst, 'bar') # no xattr == no problem shutil._copyxattr(src, dst) @@ -1231,7 +1278,7 @@ def test_copyxattr(self): os.getxattr(dst, 'user.foo')) # check errors don't affect other attrs os.remove(dst) - write_file(dst, 'bar') + create_file(dst, 'bar') os_error = OSError(errno.EPERM, 'EPERM') def _raise_on_user_foo(fname, attr, val, **kwargs): @@ -1261,15 +1308,15 @@ def _raise_on_src(fname, *, follow_symlinks=True): # test that shutil.copystat copies xattrs src = os.path.join(tmp_dir, 'the_original') srcro = os.path.join(tmp_dir, 'the_original_ro') - write_file(src, src) - write_file(srcro, srcro) + create_file(src, src) + create_file(srcro, srcro) os.setxattr(src, 'user.the_value', b'fiddly') os.setxattr(srcro, 'user.the_value', b'fiddly') os.chmod(srcro, 0o444) dst = os.path.join(tmp_dir, 'the_copy') dstro = os.path.join(tmp_dir, 'the_copy_ro') - write_file(dst, dst) - write_file(dstro, dstro) + create_file(dst, dst) + create_file(dstro, dstro) shutil.copystat(src, dst) shutil.copystat(srcro, dstro) self.assertEqual(os.getxattr(dst, 'user.the_value'), b'fiddly') @@ -1285,13 +1332,13 @@ def test_copyxattr_symlinks(self): tmp_dir = self.mkdtemp() src = os.path.join(tmp_dir, 'foo') src_link = os.path.join(tmp_dir, 'baz') - write_file(src, 'foo') + create_file(src, 'foo') os.symlink(src, src_link) os.setxattr(src, 'trusted.foo', b'42') os.setxattr(src_link, 'trusted.foo', b'43', follow_symlinks=False) dst = os.path.join(tmp_dir, 'bar') dst_link = os.path.join(tmp_dir, 'qux') - write_file(dst, 'bar') + create_file(dst, 'bar') os.symlink(dst, dst_link) shutil._copyxattr(src_link, dst_link, follow_symlinks=False) self.assertEqual(os.getxattr(dst_link, 'trusted.foo', follow_symlinks=False), b'43') @@ -1304,7 +1351,7 @@ def test_copyxattr_symlinks(self): def _copy_file(self, method): fname = 'test.txt' tmpdir = self.mkdtemp() - write_file((tmpdir, fname), 'xxx') + create_file((tmpdir, fname), 'xxx') file1 = os.path.join(tmpdir, fname) tmpdir2 = self.mkdtemp() method(file1, tmpdir2) @@ -1317,14 +1364,13 @@ def test_copy(self): self.assertTrue(os.path.exists(file2)) self.assertEqual(os.stat(file1).st_mode, os.stat(file2).st_mode) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") @os_helper.skip_unless_symlink def test_copy_symlinks(self): tmp_dir = self.mkdtemp() src = os.path.join(tmp_dir, 'foo') dst = os.path.join(tmp_dir, 'bar') src_link = os.path.join(tmp_dir, 'baz') - write_file(src, 'foo') + create_file(src, 'foo') os.symlink(src, src_link) if hasattr(os, 'lchmod'): os.lchmod(src_link, stat.S_IRWXU | stat.S_IRWXO) @@ -1360,14 +1406,13 @@ def test_copy2(self): self.assertEqual(getattr(file1_stat, 'st_flags'), getattr(file2_stat, 'st_flags')) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") @os_helper.skip_unless_symlink def test_copy2_symlinks(self): tmp_dir = self.mkdtemp() src = os.path.join(tmp_dir, 'foo') dst = os.path.join(tmp_dir, 'bar') src_link = os.path.join(tmp_dir, 'baz') - write_file(src, 'foo') + create_file(src, 'foo') os.symlink(src, src_link) if hasattr(os, 'lchmod'): os.lchmod(src_link, stat.S_IRWXU | stat.S_IRWXO) @@ -1401,7 +1446,7 @@ def test_copy2_xattr(self): tmp_dir = self.mkdtemp() src = os.path.join(tmp_dir, 'foo') dst = os.path.join(tmp_dir, 'bar') - write_file(src, 'foo') + create_file(src, 'foo') os.setxattr(src, 'user.foo', b'42') shutil.copy2(src, dst) self.assertEqual( @@ -1415,7 +1460,7 @@ def test_copy_return_value(self): src_dir = self.mkdtemp() dst_dir = self.mkdtemp() src = os.path.join(src_dir, 'foo') - write_file(src, 'foo') + create_file(src, 'foo') rv = fn(src, dst_dir) self.assertEqual(rv, os.path.join(dst_dir, 'foo')) rv = fn(src, os.path.join(dst_dir, 'bar')) @@ -1432,7 +1477,7 @@ def _test_copy_dir(self, copy_func): src_file = os.path.join(src_dir, 'foo') dir2 = self.mkdtemp() dst = os.path.join(src_dir, 'does_not_exist/') - write_file(src_file, 'foo') + create_file(src_file, 'foo') if sys.platform == "win32": err = PermissionError else: @@ -1445,7 +1490,6 @@ def _test_copy_dir(self, copy_func): ### shutil.copyfile - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") @os_helper.skip_unless_symlink def test_copyfile_symlinks(self): tmp_dir = self.mkdtemp() @@ -1453,7 +1497,7 @@ def test_copyfile_symlinks(self): dst = os.path.join(tmp_dir, 'dst') dst_link = os.path.join(tmp_dir, 'dst_link') link = os.path.join(tmp_dir, 'link') - write_file(src, 'foo') + create_file(src, 'foo') os.symlink(src, link) # don't follow shutil.copyfile(link, dst_link, follow_symlinks=False) @@ -1470,8 +1514,7 @@ def test_dont_copy_file_onto_link_to_itself(self): src = os.path.join(TESTFN, 'cheese') dst = os.path.join(TESTFN, 'shop') try: - with open(src, 'w', encoding='utf-8') as f: - f.write('cheddar') + create_file(src, 'cheddar') try: os.link(src, dst) except PermissionError as e: @@ -1490,8 +1533,7 @@ def test_dont_copy_file_onto_symlink_to_itself(self): src = os.path.join(TESTFN, 'cheese') dst = os.path.join(TESTFN, 'shop') try: - with open(src, 'w', encoding='utf-8') as f: - f.write('cheddar') + create_file(src, 'cheddar') # Using `src` here would mean we end up with a symlink pointing # to TESTFN/TESTFN/cheese, while it should point at # TESTFN/cheese. @@ -1526,7 +1568,7 @@ def test_copyfile_return_value(self): dst_dir = self.mkdtemp() dst_file = os.path.join(dst_dir, 'bar') src_file = os.path.join(src_dir, 'foo') - write_file(src_file, 'foo') + create_file(src_file, 'foo') rv = shutil.copyfile(src_file, dst_file) self.assertTrue(os.path.exists(rv)) self.assertEqual(read_file(src_file), read_file(dst_file)) @@ -1536,7 +1578,7 @@ def test_copyfile_same_file(self): # are the same. src_dir = self.mkdtemp() src_file = os.path.join(src_dir, 'foo') - write_file(src_file, 'foo') + create_file(src_file, 'foo') self.assertRaises(SameFileError, shutil.copyfile, src_file, src_file) # But Error should work too, to stay backward compatible. self.assertRaises(Error, shutil.copyfile, src_file, src_file) @@ -1553,7 +1595,7 @@ def test_copyfile_nonexistent_dir(self): src_dir = self.mkdtemp() src_file = os.path.join(src_dir, 'foo') dst = os.path.join(src_dir, 'does_not_exist/') - write_file(src_file, 'foo') + create_file(src_file, 'foo') self.assertRaises(FileNotFoundError, shutil.copyfile, src_file, dst) def test_copyfile_copy_dir(self): @@ -1564,7 +1606,7 @@ def test_copyfile_copy_dir(self): src_file = os.path.join(src_dir, 'foo') dir2 = self.mkdtemp() dst = os.path.join(src_dir, 'does_not_exist/') - write_file(src_file, 'foo') + create_file(src_file, 'foo') if sys.platform == "win32": err = PermissionError else: @@ -1579,42 +1621,6 @@ class TestArchives(BaseTest, unittest.TestCase): ### shutil.make_archive - @support.requires_zlib() - def test_make_tarball(self): - # creating something to tar - root_dir, base_dir = self._create_files('') - - tmpdir2 = self.mkdtemp() - # force shutil to create the directory - os.rmdir(tmpdir2) - # working with relative paths - work_dir = os.path.dirname(tmpdir2) - rel_base_name = os.path.join(os.path.basename(tmpdir2), 'archive') - - with os_helper.change_cwd(work_dir), no_chdir: - base_name = os.path.abspath(rel_base_name) - tarball = make_archive(rel_base_name, 'gztar', root_dir, '.') - - # check if the compressed tarball was created - self.assertEqual(tarball, base_name + '.tar.gz') - self.assertTrue(os.path.isfile(tarball)) - self.assertTrue(tarfile.is_tarfile(tarball)) - with tarfile.open(tarball, 'r:gz') as tf: - self.assertCountEqual(tf.getnames(), - ['.', './sub', './sub2', - './file1', './file2', './sub/file3']) - - # trying an uncompressed one - with os_helper.change_cwd(work_dir), no_chdir: - tarball = make_archive(rel_base_name, 'tar', root_dir, '.') - self.assertEqual(tarball, base_name + '.tar') - self.assertTrue(os.path.isfile(tarball)) - self.assertTrue(tarfile.is_tarfile(tarball)) - with tarfile.open(tarball, 'r') as tf: - self.assertCountEqual(tf.getnames(), - ['.', './sub', './sub2', - './file1', './file2', './sub/file3']) - def _tarinfo(self, path): with tarfile.open(path) as tar: names = tar.getnames() @@ -1626,15 +1632,101 @@ def _create_files(self, base_dir='dist'): root_dir = self.mkdtemp() dist = os.path.join(root_dir, base_dir) os.makedirs(dist, exist_ok=True) - write_file((dist, 'file1'), 'xxx') - write_file((dist, 'file2'), 'xxx') + create_file((dist, 'file1'), 'xxx') + create_file((dist, 'file2'), 'xxx') os.mkdir(os.path.join(dist, 'sub')) - write_file((dist, 'sub', 'file3'), 'xxx') + create_file((dist, 'sub', 'file3'), 'xxx') os.mkdir(os.path.join(dist, 'sub2')) if base_dir: - write_file((root_dir, 'outer'), 'xxx') + create_file((root_dir, 'outer'), 'xxx') return root_dir, base_dir + @support.requires_zlib() + def test_make_tarfile(self): + root_dir, base_dir = self._create_files() + # Test without base_dir. + with os_helper.temp_cwd(), no_chdir: + base_name = os.path.join('dst', 'archive') + archive = make_archive(base_name, 'tar', root_dir) + # check if the compressed tarball was created + self.assertEqual(archive, os.path.abspath(base_name) + '.tar') + self.assertTrue(os.path.isfile(archive)) + self.assertTrue(tarfile.is_tarfile(archive)) + with tarfile.open(archive, 'r') as tf: + self.assertCountEqual(tf.getnames(), + ['.', './dist', './dist/sub', './dist/sub2', + './dist/file1', './dist/file2', './dist/sub/file3', + './outer']) + + # Test with base_dir. + with os_helper.temp_cwd(), no_chdir: + base_name = os.path.join('dst2', 'archive') + archive = make_archive(base_name, 'tar', root_dir, base_dir) + self.assertEqual(archive, os.path.abspath(base_name) + '.tar') + # check if the uncompressed tarball was created + self.assertTrue(os.path.isfile(archive)) + self.assertTrue(tarfile.is_tarfile(archive)) + with tarfile.open(archive, 'r') as tf: + self.assertCountEqual(tf.getnames(), + ['dist', 'dist/sub', 'dist/sub2', + 'dist/file1', 'dist/file2', 'dist/sub/file3']) + + # Test with multi-component base_dir. + with os_helper.temp_cwd(), no_chdir: + base_name = os.path.join('dst3', 'archive') + archive = make_archive(base_name, 'tar', root_dir, + os.path.join(base_dir, 'sub')) + self.assertEqual(archive, os.path.abspath(base_name) + '.tar') + self.assertTrue(os.path.isfile(archive)) + self.assertTrue(tarfile.is_tarfile(archive)) + with tarfile.open(archive, 'r') as tf: + self.assertCountEqual(tf.getnames(), + ['dist/sub', 'dist/sub/file3']) + + @support.requires_zlib() + def test_make_tarfile_without_rootdir(self): + root_dir, base_dir = self._create_files() + # Test without base_dir. + base_name = os.path.join(self.mkdtemp(), 'dst', 'archive') + base_name = os.path.relpath(base_name, root_dir) + with os_helper.change_cwd(root_dir), no_chdir: + archive = make_archive(base_name, 'gztar') + self.assertEqual(archive, base_name + '.tar.gz') + self.assertTrue(os.path.isfile(archive)) + self.assertTrue(tarfile.is_tarfile(archive)) + with tarfile.open(archive, 'r:gz') as tf: + self.assertCountEqual(tf.getnames(), + ['.', './dist', './dist/sub', './dist/sub2', + './dist/file1', './dist/file2', './dist/sub/file3', + './outer']) + + # Test with base_dir. + with os_helper.change_cwd(root_dir), no_chdir: + base_name = os.path.join('dst', 'archive') + archive = make_archive(base_name, 'tar', base_dir=base_dir) + self.assertEqual(archive, base_name + '.tar') + self.assertTrue(os.path.isfile(archive)) + self.assertTrue(tarfile.is_tarfile(archive)) + with tarfile.open(archive, 'r') as tf: + self.assertCountEqual(tf.getnames(), + ['dist', 'dist/sub', 'dist/sub2', + 'dist/file1', 'dist/file2', 'dist/sub/file3']) + + def test_make_tarfile_with_explicit_curdir(self): + # Test with base_dir=os.curdir. + root_dir, base_dir = self._create_files() + with os_helper.temp_cwd(), no_chdir: + base_name = os.path.join('dst', 'archive') + archive = make_archive(base_name, 'tar', root_dir, os.curdir) + self.assertEqual(archive, os.path.abspath(base_name) + '.tar') + self.assertTrue(os.path.isfile(archive)) + self.assertTrue(tarfile.is_tarfile(archive)) + with tarfile.open(archive, 'r') as tf: + self.assertCountEqual(tf.getnames(), + ['.', './dist', './dist/sub', './dist/sub2', + './dist/file1', './dist/file2', './dist/sub/file3', + './outer']) + @support.requires_zlib() @unittest.skipUnless(shutil.which('tar'), 'Need the tar command to run') @@ -1684,40 +1776,89 @@ def test_tarfile_vs_tar(self): @support.requires_zlib() def test_make_zipfile(self): - # creating something to zip root_dir, base_dir = self._create_files() + # Test without base_dir. + with os_helper.temp_cwd(), no_chdir: + base_name = os.path.join('dst', 'archive') + archive = make_archive(base_name, 'zip', root_dir) + self.assertEqual(archive, os.path.abspath(base_name) + '.zip') + self.assertTrue(os.path.isfile(archive)) + self.assertTrue(zipfile.is_zipfile(archive)) + with zipfile.ZipFile(archive) as zf: + self.assertCountEqual(zf.namelist(), + ['dist/', 'dist/sub/', 'dist/sub2/', + 'dist/file1', 'dist/file2', 'dist/sub/file3', + 'outer']) + + # Test with base_dir. + with os_helper.temp_cwd(), no_chdir: + base_name = os.path.join('dst2', 'archive') + archive = make_archive(base_name, 'zip', root_dir, base_dir) + self.assertEqual(archive, os.path.abspath(base_name) + '.zip') + self.assertTrue(os.path.isfile(archive)) + self.assertTrue(zipfile.is_zipfile(archive)) + with zipfile.ZipFile(archive) as zf: + self.assertCountEqual(zf.namelist(), + ['dist/', 'dist/sub/', 'dist/sub2/', + 'dist/file1', 'dist/file2', 'dist/sub/file3']) + + # Test with multi-component base_dir. + with os_helper.temp_cwd(), no_chdir: + base_name = os.path.join('dst3', 'archive') + archive = make_archive(base_name, 'zip', root_dir, + os.path.join(base_dir, 'sub')) + self.assertEqual(archive, os.path.abspath(base_name) + '.zip') + self.assertTrue(os.path.isfile(archive)) + self.assertTrue(zipfile.is_zipfile(archive)) + with zipfile.ZipFile(archive) as zf: + self.assertCountEqual(zf.namelist(), + ['dist/sub/', 'dist/sub/file3']) - tmpdir2 = self.mkdtemp() - # force shutil to create the directory - os.rmdir(tmpdir2) - # working with relative paths - work_dir = os.path.dirname(tmpdir2) - rel_base_name = os.path.join(os.path.basename(tmpdir2), 'archive') - - with os_helper.change_cwd(work_dir), no_chdir: - base_name = os.path.abspath(rel_base_name) - res = make_archive(rel_base_name, 'zip', root_dir) + @support.requires_zlib() + def test_make_zipfile_without_rootdir(self): + root_dir, base_dir = self._create_files() + # Test without base_dir. + base_name = os.path.join(self.mkdtemp(), 'dst', 'archive') + base_name = os.path.relpath(base_name, root_dir) + with os_helper.change_cwd(root_dir), no_chdir: + archive = make_archive(base_name, 'zip') + self.assertEqual(archive, base_name + '.zip') + self.assertTrue(os.path.isfile(archive)) + self.assertTrue(zipfile.is_zipfile(archive)) + with zipfile.ZipFile(archive) as zf: + self.assertCountEqual(zf.namelist(), + ['dist/', 'dist/sub/', 'dist/sub2/', + 'dist/file1', 'dist/file2', 'dist/sub/file3', + 'outer']) + + # Test with base_dir. + root_dir, base_dir = self._create_files() + with os_helper.change_cwd(root_dir), no_chdir: + base_name = os.path.join('dst', 'archive') + archive = make_archive(base_name, 'zip', base_dir=base_dir) + self.assertEqual(archive, base_name + '.zip') + self.assertTrue(os.path.isfile(archive)) + self.assertTrue(zipfile.is_zipfile(archive)) + with zipfile.ZipFile(archive) as zf: + self.assertCountEqual(zf.namelist(), + ['dist/', 'dist/sub/', 'dist/sub2/', + 'dist/file1', 'dist/file2', 'dist/sub/file3']) - self.assertEqual(res, base_name + '.zip') - self.assertTrue(os.path.isfile(res)) - self.assertTrue(zipfile.is_zipfile(res)) - with zipfile.ZipFile(res) as zf: - self.assertCountEqual(zf.namelist(), - ['dist/', 'dist/sub/', 'dist/sub2/', - 'dist/file1', 'dist/file2', 'dist/sub/file3', - 'outer']) - - with os_helper.change_cwd(work_dir), no_chdir: - base_name = os.path.abspath(rel_base_name) - res = make_archive(rel_base_name, 'zip', root_dir, base_dir) - - self.assertEqual(res, base_name + '.zip') - self.assertTrue(os.path.isfile(res)) - self.assertTrue(zipfile.is_zipfile(res)) - with zipfile.ZipFile(res) as zf: - self.assertCountEqual(zf.namelist(), - ['dist/', 'dist/sub/', 'dist/sub2/', - 'dist/file1', 'dist/file2', 'dist/sub/file3']) + @support.requires_zlib() + def test_make_zipfile_with_explicit_curdir(self): + # Test with base_dir=os.curdir. + root_dir, base_dir = self._create_files() + with os_helper.temp_cwd(), no_chdir: + base_name = os.path.join('dst', 'archive') + archive = make_archive(base_name, 'zip', root_dir, os.curdir) + self.assertEqual(archive, os.path.abspath(base_name) + '.zip') + self.assertTrue(os.path.isfile(archive)) + self.assertTrue(zipfile.is_zipfile(archive)) + with zipfile.ZipFile(archive) as zf: + self.assertCountEqual(zf.namelist(), + ['dist/', 'dist/sub/', 'dist/sub2/', + 'dist/file1', 'dist/file2', 'dist/sub/file3', + 'outer']) @support.requires_zlib() @unittest.skipUnless(shutil.which('zip'), @@ -1766,7 +1907,10 @@ def test_unzip_zipfile(self): subprocess.check_output(zip_cmd, stderr=subprocess.STDOUT) except subprocess.CalledProcessError as exc: details = exc.output.decode(errors="replace") - if 'unrecognized option: t' in details: + if any(message in details for message in [ + 'unrecognized option: t', # BusyBox + 'invalid option -- t', # Android + ]): self.skipTest("unzip doesn't support -t") msg = "{}\n\n**Unzip Output**\n{}" self.fail(msg.format(exc, details)) @@ -1887,17 +2031,19 @@ def archiver(base_name, base_dir, **kw): unregister_archive_format('xxx') def test_make_tarfile_in_curdir(self): - # Issue #21280 + # Issue #21280: Test with the archive in the current directory. root_dir = self.mkdtemp() with os_helper.change_cwd(root_dir), no_chdir: + # root_dir must be None, so the archive path is relative. self.assertEqual(make_archive('test', 'tar'), 'test.tar') self.assertTrue(os.path.isfile('test.tar')) @support.requires_zlib() def test_make_zipfile_in_curdir(self): - # Issue #21280 + # Issue #21280: Test with the archive in the current directory. root_dir = self.mkdtemp() with os_helper.change_cwd(root_dir), no_chdir: + # root_dir must be None, so the archive path is relative. self.assertEqual(make_archive('test', 'zip'), 'test.zip') self.assertTrue(os.path.isfile('test.zip')) @@ -1918,10 +2064,11 @@ def test_register_archive_format(self): self.assertNotIn('xxx', formats) def test_make_tarfile_rootdir_nodir(self): - # GH-99203 + # GH-99203: Test with root_dir is not a real directory. self.addCleanup(os_helper.unlink, f'{TESTFN}.tar') for dry_run in (False, True): with self.subTest(dry_run=dry_run): + # root_dir does not exist. tmp_dir = self.mkdtemp() nonexisting_file = os.path.join(tmp_dir, 'nonexisting') with self.assertRaises(FileNotFoundError) as cm: @@ -1930,6 +2077,7 @@ def test_make_tarfile_rootdir_nodir(self): self.assertEqual(cm.exception.filename, nonexisting_file) self.assertFalse(os.path.exists(f'{TESTFN}.tar')) + # root_dir is a file. tmp_fd, tmp_file = tempfile.mkstemp(dir=tmp_dir) os.close(tmp_fd) with self.assertRaises(NotADirectoryError) as cm: @@ -1940,10 +2088,11 @@ def test_make_tarfile_rootdir_nodir(self): @support.requires_zlib() def test_make_zipfile_rootdir_nodir(self): - # GH-99203 + # GH-99203: Test with root_dir is not a real directory. self.addCleanup(os_helper.unlink, f'{TESTFN}.zip') for dry_run in (False, True): with self.subTest(dry_run=dry_run): + # root_dir does not exist. tmp_dir = self.mkdtemp() nonexisting_file = os.path.join(tmp_dir, 'nonexisting') with self.assertRaises(FileNotFoundError) as cm: @@ -1952,6 +2101,7 @@ def test_make_zipfile_rootdir_nodir(self): self.assertEqual(cm.exception.filename, nonexisting_file) self.assertFalse(os.path.exists(f'{TESTFN}.zip')) + # root_dir is a file. tmp_fd, tmp_file = tempfile.mkstemp(dir=tmp_dir) os.close(tmp_fd) with self.assertRaises(NotADirectoryError) as cm: @@ -1966,7 +2116,7 @@ def check_unpack_archive(self, format, **kwargs): self.check_unpack_archive_with_converter( format, lambda path: path, **kwargs) self.check_unpack_archive_with_converter( - format, pathlib.Path, **kwargs) + format, FakePath, **kwargs) self.check_unpack_archive_with_converter(format, FakePath, **kwargs) def check_unpack_archive_with_converter(self, format, converter, **kwargs): @@ -2071,7 +2221,9 @@ def test_disk_usage(self): def test_chown(self): dirname = self.mkdtemp() filename = tempfile.mktemp(dir=dirname) - write_file(filename, 'testing chown function') + linkname = os.path.join(dirname, "chown_link") + create_file(filename, 'testing chown function') + os.symlink(filename, linkname) with self.assertRaises(ValueError): shutil.chown(filename) @@ -2092,7 +2244,7 @@ def test_chown(self): gid = os.getgid() def check_chown(path, uid=None, gid=None): - s = os.stat(filename) + s = os.stat(path) if uid is not None: self.assertEqual(uid, s.st_uid) if gid is not None: @@ -2128,41 +2280,76 @@ def check_chown(path, uid=None, gid=None): shutil.chown(dirname, user, group) check_chown(dirname, uid, gid) + dirfd = os.open(dirname, os.O_RDONLY) + self.addCleanup(os.close, dirfd) + basename = os.path.basename(filename) + baselinkname = os.path.basename(linkname) + shutil.chown(basename, uid, gid, dir_fd=dirfd) + check_chown(filename, uid, gid) + shutil.chown(basename, uid, dir_fd=dirfd) + check_chown(filename, uid) + shutil.chown(basename, group=gid, dir_fd=dirfd) + check_chown(filename, gid=gid) + shutil.chown(basename, uid, gid, dir_fd=dirfd, follow_symlinks=True) + check_chown(filename, uid, gid) + shutil.chown(basename, uid, gid, dir_fd=dirfd, follow_symlinks=False) + check_chown(filename, uid, gid) + shutil.chown(linkname, uid, follow_symlinks=True) + check_chown(filename, uid) + shutil.chown(baselinkname, group=gid, dir_fd=dirfd, follow_symlinks=False) + check_chown(filename, gid=gid) + shutil.chown(baselinkname, uid, gid, dir_fd=dirfd, follow_symlinks=True) + check_chown(filename, uid, gid) + + with self.assertRaises(TypeError): + shutil.chown(filename, uid, dir_fd=dirname) + + with self.assertRaises(FileNotFoundError): + shutil.chown('missingfile', uid, gid, dir_fd=dirfd) + + with self.assertRaises(ValueError): + shutil.chown(filename, dir_fd=dirfd) + +@support.requires_subprocess() class TestWhich(BaseTest, unittest.TestCase): def setUp(self): - self.temp_dir = self.mkdtemp(prefix="Tmp") + temp_dir = self.mkdtemp(prefix="Tmp") + base_dir = os.path.join(temp_dir, TESTFN + '-basedir') + os.mkdir(base_dir) + self.dir = os.path.join(base_dir, TESTFN + '-dir') + os.mkdir(self.dir) + self.other_dir = os.path.join(base_dir, TESTFN + '-dir2') + os.mkdir(self.other_dir) # Give the temp_file an ".exe" suffix for all. # It's needed on Windows and not harmful on other platforms. - self.temp_file = tempfile.NamedTemporaryFile(dir=self.temp_dir, - prefix="Tmp", - suffix=".Exe") - os.chmod(self.temp_file.name, stat.S_IXUSR) - self.addCleanup(self.temp_file.close) - self.dir, self.file = os.path.split(self.temp_file.name) + self.file = TESTFN + '.Exe' + self.filepath = os.path.join(self.dir, self.file) + self.create_file(self.filepath) self.env_path = self.dir self.curdir = os.curdir self.ext = ".EXE" - def to_text_type(self, s): - ''' - In this class we're testing with str, so convert s to a str - ''' - if isinstance(s, bytes): - return s.decode() - return s + to_text_type = staticmethod(os.fsdecode) + + def create_file(self, path): + create_file(path) + os.chmod(path, 0o755) + + def assertNormEqual(self, actual, expected): + self.assertEqual(os.path.normcase(actual), os.path.normcase(expected)) def test_basic(self): # Given an EXE in a directory, it should be returned. rv = shutil.which(self.file, path=self.dir) - self.assertEqual(rv, self.temp_file.name) + self.assertEqual(rv, self.filepath) def test_absolute_cmd(self): # When given the fully qualified path to an executable that exists, # it should be returned. - rv = shutil.which(self.temp_file.name, path=self.temp_dir) - self.assertEqual(rv, self.temp_file.name) + rv = shutil.which(self.filepath, path=self.other_dir) + self.assertEqual(rv, self.filepath) def test_relative_cmd(self): # When given the relative path with a directory part to an executable @@ -2170,7 +2357,7 @@ def test_relative_cmd(self): base_dir, tail_dir = os.path.split(self.dir) relpath = os.path.join(tail_dir, self.file) with os_helper.change_cwd(path=base_dir): - rv = shutil.which(relpath, path=self.temp_dir) + rv = shutil.which(relpath, path=self.other_dir) self.assertEqual(rv, relpath) # But it shouldn't be searched in PATH directories (issue #16957). with os_helper.change_cwd(path=self.dir): @@ -2181,9 +2368,8 @@ def test_relative_cmd(self): "test is for non win32") def test_cwd_non_win32(self): # Issue #16957 - base_dir = os.path.dirname(self.dir) with os_helper.change_cwd(path=self.dir): - rv = shutil.which(self.file, path=base_dir) + rv = shutil.which(self.file, path=self.other_dir) # non-win32: shouldn't match in the current directory. self.assertIsNone(rv) @@ -2193,57 +2379,32 @@ def test_cwd_win32(self): base_dir = os.path.dirname(self.dir) with os_helper.change_cwd(path=self.dir): with unittest.mock.patch('shutil._win_path_needs_curdir', return_value=True): - rv = shutil.which(self.file, path=base_dir) + rv = shutil.which(self.file, path=self.other_dir) # Current directory implicitly on PATH self.assertEqual(rv, os.path.join(self.curdir, self.file)) with unittest.mock.patch('shutil._win_path_needs_curdir', return_value=False): - rv = shutil.which(self.file, path=base_dir) + rv = shutil.which(self.file, path=self.other_dir) # Current directory not on PATH self.assertIsNone(rv) @unittest.skipUnless(sys.platform == "win32", "test is for win32") def test_cwd_win32_added_before_all_other_path(self): - base_dir = pathlib.Path(os.fsdecode(self.dir)) - - elsewhere_in_path_dir = base_dir / 'dir1' - elsewhere_in_path_dir.mkdir() - match_elsewhere_in_path = elsewhere_in_path_dir / 'hello.exe' - match_elsewhere_in_path.touch() - - exe_in_cwd = base_dir / 'hello.exe' - exe_in_cwd.touch() - - with os_helper.change_cwd(path=base_dir): - with unittest.mock.patch('shutil._win_path_needs_curdir', return_value=True): - rv = shutil.which('hello.exe', path=elsewhere_in_path_dir) - - self.assertEqual(os.path.abspath(rv), os.path.abspath(exe_in_cwd)) - - @unittest.skipUnless(sys.platform == "win32", - "test is for win32") - def test_pathext_match_before_path_full_match(self): - base_dir = pathlib.Path(os.fsdecode(self.dir)) - dir1 = base_dir / 'dir1' - dir2 = base_dir / 'dir2' - dir1.mkdir() - dir2.mkdir() - - pathext_match = dir1 / 'hello.com.exe' - path_match = dir2 / 'hello.com' - pathext_match.touch() - path_match.touch() - - test_path = os.pathsep.join([str(dir1), str(dir2)]) - assert os.path.basename(shutil.which( - 'hello.com', path=test_path, mode = os.F_OK - )).lower() == 'hello.com.exe' + other_file_path = os.path.join(self.other_dir, self.file) + self.create_file(other_file_path) + with unittest.mock.patch('shutil._win_path_needs_curdir', return_value=True): + with os_helper.change_cwd(path=self.dir): + rv = shutil.which(self.file, path=self.other_dir) + self.assertEqual(rv, os.path.join(self.curdir, self.file)) + with os_helper.change_cwd(path=self.other_dir): + rv = shutil.which(self.file, path=self.dir) + self.assertEqual(rv, os.path.join(self.curdir, self.file)) @os_helper.skip_if_dac_override def test_non_matching_mode(self): # Set the file read-only and ask for writeable files. - os.chmod(self.temp_file.name, stat.S_IREAD) - if os.access(self.temp_file.name, os.W_OK): + os.chmod(self.filepath, stat.S_IREAD) + if os.access(self.filepath, os.W_OK): self.skipTest("can't set the file read-only") rv = shutil.which(self.file, path=self.dir, mode=os.W_OK) self.assertIsNone(rv) @@ -2265,13 +2426,13 @@ def test_pathext_checking(self): # Ask for the file without the ".exe" extension, then ensure that # it gets found properly with the extension. rv = shutil.which(self.file[:-4], path=self.dir) - self.assertEqual(rv, self.temp_file.name[:-4] + self.ext) + self.assertEqual(rv, self.filepath[:-4] + self.ext) def test_environ_path(self): with os_helper.EnvironmentVarGuard() as env: env['PATH'] = self.env_path rv = shutil.which(self.file) - self.assertEqual(rv, self.temp_file.name) + self.assertEqual(rv, self.filepath) def test_environ_path_empty(self): # PATH='': no match @@ -2285,12 +2446,9 @@ def test_environ_path_empty(self): self.assertIsNone(rv) def test_environ_path_cwd(self): - expected_cwd = os.path.basename(self.temp_file.name) + expected_cwd = self.file if sys.platform == "win32": - curdir = os.curdir - if isinstance(expected_cwd, bytes): - curdir = os.fsencode(curdir) - expected_cwd = os.path.join(curdir, expected_cwd) + expected_cwd = os.path.join(self.curdir, expected_cwd) # PATH=':': explicitly looks in the current directory with os_helper.EnvironmentVarGuard() as env: @@ -2308,21 +2466,21 @@ def test_environ_path_cwd(self): def test_environ_path_missing(self): with os_helper.EnvironmentVarGuard() as env: - env.pop('PATH', None) + del env['PATH'] # without confstr with unittest.mock.patch('os.confstr', side_effect=ValueError, \ create=True), \ support.swap_attr(os, 'defpath', self.dir): rv = shutil.which(self.file) - self.assertEqual(rv, self.temp_file.name) + self.assertEqual(rv, self.filepath) # with confstr with unittest.mock.patch('os.confstr', return_value=self.dir, \ create=True), \ support.swap_attr(os, 'defpath', ''): rv = shutil.which(self.file) - self.assertEqual(rv, self.temp_file.name) + self.assertEqual(rv, self.filepath) def test_empty_path(self): base_dir = os.path.dirname(self.dir) @@ -2334,56 +2492,94 @@ def test_empty_path(self): def test_empty_path_no_PATH(self): with os_helper.EnvironmentVarGuard() as env: - env.pop('PATH', None) + del env['PATH'] rv = shutil.which(self.file) self.assertIsNone(rv) @unittest.skipUnless(sys.platform == "win32", 'test specific to Windows') def test_pathext(self): - ext = self.to_text_type(".xyz") - temp_filexyz = tempfile.NamedTemporaryFile(dir=self.temp_dir, - prefix=self.to_text_type("Tmp2"), suffix=ext) - os.chmod(temp_filexyz.name, stat.S_IXUSR) - self.addCleanup(temp_filexyz.close) - - # strip path and extension - program = os.path.basename(temp_filexyz.name) - program = os.path.splitext(program)[0] - + ext = '.xyz' + cmd = self.to_text_type(TESTFN2) + cmdext = cmd + self.to_text_type(ext) + filepath = os.path.join(self.dir, cmdext) + self.create_file(filepath) with os_helper.EnvironmentVarGuard() as env: - env['PATHEXT'] = ext if isinstance(ext, str) else ext.decode() - rv = shutil.which(program, path=self.temp_dir) - self.assertEqual(rv, temp_filexyz.name) + env['PATHEXT'] = ext + self.assertEqual(shutil.which(cmd, path=self.dir), filepath) + self.assertEqual(shutil.which(cmdext, path=self.dir), filepath) # Issue 40592: See https://bugs.python.org/issue40592 @unittest.skipUnless(sys.platform == "win32", 'test specific to Windows') def test_pathext_with_empty_str(self): - ext = self.to_text_type(".xyz") - temp_filexyz = tempfile.NamedTemporaryFile(dir=self.temp_dir, - prefix=self.to_text_type("Tmp2"), suffix=ext) - self.addCleanup(temp_filexyz.close) + ext = '.xyz' + cmd = self.to_text_type(TESTFN2) + cmdext = cmd + self.to_text_type(ext) + filepath = os.path.join(self.dir, cmdext) + self.create_file(filepath) + with os_helper.EnvironmentVarGuard() as env: + env['PATHEXT'] = ext + ';' # note the ; + self.assertEqual(shutil.which(cmd, path=self.dir), filepath) + self.assertEqual(shutil.which(cmdext, path=self.dir), filepath) - # strip path and extension - program = os.path.basename(temp_filexyz.name) - program = os.path.splitext(program)[0] + @unittest.skipUnless(sys.platform == "win32", 'test specific to Windows') + def test_pathext_with_multidot_extension(self): + ext = '.foo.bar' + cmd = self.to_text_type(TESTFN2) + cmdext = cmd + self.to_text_type(ext) + filepath = os.path.join(self.dir, cmdext) + self.create_file(filepath) + with os_helper.EnvironmentVarGuard() as env: + env['PATHEXT'] = ext + self.assertEqual(shutil.which(cmd, path=self.dir), filepath) + self.assertEqual(shutil.which(cmdext, path=self.dir), filepath) + + @unittest.skipUnless(sys.platform == "win32", 'test specific to Windows') + def test_pathext_with_null_extension(self): + cmd = self.to_text_type(TESTFN2) + cmddot = cmd + self.to_text_type('.') + filepath = os.path.join(self.dir, cmd) + self.create_file(filepath) + with os_helper.EnvironmentVarGuard() as env: + env['PATHEXT'] = '.xyz' + self.assertIsNone(shutil.which(cmd, path=self.dir)) + self.assertIsNone(shutil.which(cmddot, path=self.dir)) + env['PATHEXT'] = '.xyz;.' # note the . + self.assertEqual(shutil.which(cmd, path=self.dir), filepath) + self.assertEqual(shutil.which(cmddot, path=self.dir), + filepath + self.to_text_type('.')) + env['PATHEXT'] = '.xyz;..' # multiple dots + self.assertEqual(shutil.which(cmd, path=self.dir), filepath) + self.assertEqual(shutil.which(cmddot, path=self.dir), + filepath + self.to_text_type('.')) + @unittest.skipUnless(sys.platform == "win32", 'test specific to Windows') + def test_pathext_extension_ends_with_dot(self): + ext = '.xyz' + cmd = self.to_text_type(TESTFN2) + cmdext = cmd + self.to_text_type(ext) + dot = self.to_text_type('.') + filepath = os.path.join(self.dir, cmdext) + self.create_file(filepath) with os_helper.EnvironmentVarGuard() as env: - env['PATHEXT'] = f"{ext if isinstance(ext, str) else ext.decode()};" # note the ; - rv = shutil.which(program, path=self.temp_dir) - self.assertEqual(rv, temp_filexyz.name) + env['PATHEXT'] = ext + '.' + self.assertEqual(shutil.which(cmd, path=self.dir), filepath) # cmd.exe hangs here + self.assertEqual(shutil.which(cmdext, path=self.dir), filepath) + self.assertIsNone(shutil.which(cmd + dot, path=self.dir)) + self.assertIsNone(shutil.which(cmdext + dot, path=self.dir)) # See GH-75586 @unittest.skipUnless(sys.platform == "win32", 'test specific to Windows') def test_pathext_applied_on_files_in_path(self): + ext = '.xyz' + cmd = self.to_text_type(TESTFN2) + cmdext = cmd + self.to_text_type(ext) + filepath = os.path.join(self.dir, cmdext) + self.create_file(filepath) with os_helper.EnvironmentVarGuard() as env: - env["PATH"] = self.temp_dir if isinstance(self.temp_dir, str) else self.temp_dir.decode() - env["PATHEXT"] = ".test" - - test_path = os.path.join(self.temp_dir, self.to_text_type("test_program.test")) - open(test_path, 'w').close() - os.chmod(test_path, 0o755) - - self.assertEqual(shutil.which(self.to_text_type("test_program")), test_path) + env["PATH"] = os.fsdecode(self.dir) + env["PATHEXT"] = ext + self.assertEqual(shutil.which(cmd), filepath) + self.assertEqual(shutil.which(cmdext), filepath) # See GH-75586 @unittest.skipUnless(sys.platform == "win32", 'test specific to Windows') @@ -2399,49 +2595,107 @@ def test_win_path_needs_curdir(self): self.assertFalse(shutil._win_path_needs_curdir('dontcare', os.X_OK)) need_curdir_mock.assert_called_once_with('dontcare') - # See GH-109590 @unittest.skipUnless(sys.platform == "win32", 'test specific to Windows') - def test_pathext_preferred_for_execute(self): - with os_helper.EnvironmentVarGuard() as env: - env["PATH"] = self.temp_dir if isinstance(self.temp_dir, str) else self.temp_dir.decode() - env["PATHEXT"] = ".test" - - exe = os.path.join(self.temp_dir, self.to_text_type("test.exe")) - open(exe, 'w').close() - os.chmod(exe, 0o755) - - # default behavior allows a direct match if nothing in PATHEXT matches - self.assertEqual(shutil.which(self.to_text_type("test.exe")), exe) - - dot_test = os.path.join(self.temp_dir, self.to_text_type("test.exe.test")) - open(dot_test, 'w').close() - os.chmod(dot_test, 0o755) + def test_same_dir_with_pathext_extension(self): + cmd = self.file # with .exe extension + # full match + self.assertNormEqual(shutil.which(cmd, path=self.dir), self.filepath) + self.assertNormEqual(shutil.which(cmd, path=self.dir, mode=os.F_OK), + self.filepath) + + cmd2 = cmd + self.to_text_type('.com') # with .exe.com extension + other_file_path = os.path.join(self.dir, cmd2) + self.create_file(other_file_path) + + # full match + self.assertNormEqual(shutil.which(cmd, path=self.dir), self.filepath) + self.assertNormEqual(shutil.which(cmd, path=self.dir, mode=os.F_OK), + self.filepath) + self.assertNormEqual(shutil.which(cmd2, path=self.dir), other_file_path) + self.assertNormEqual(shutil.which(cmd2, path=self.dir, mode=os.F_OK), + other_file_path) - # now we have a PATHEXT match, so it take precedence - self.assertEqual(shutil.which(self.to_text_type("test.exe")), dot_test) - - # but if we don't use os.X_OK we don't change the order based off PATHEXT - # and therefore get the direct match. - self.assertEqual(shutil.which(self.to_text_type("test.exe"), mode=os.F_OK), exe) - - # See GH-109590 @unittest.skipUnless(sys.platform == "win32", 'test specific to Windows') - def test_pathext_given_extension_preferred(self): - with os_helper.EnvironmentVarGuard() as env: - env["PATH"] = self.temp_dir if isinstance(self.temp_dir, str) else self.temp_dir.decode() - env["PATHEXT"] = ".exe2;.exe" - - exe = os.path.join(self.temp_dir, self.to_text_type("test.exe")) - open(exe, 'w').close() - os.chmod(exe, 0o755) + def test_same_dir_without_pathext_extension(self): + cmd = self.file[:-4] # without .exe extension + # pathext match + self.assertNormEqual(shutil.which(cmd, path=self.dir), self.filepath) + self.assertNormEqual(shutil.which(cmd, path=self.dir, mode=os.F_OK), + self.filepath) + + # without extension + other_file_path = os.path.join(self.dir, cmd) + self.create_file(other_file_path) + + # pathext match if mode contains X_OK + self.assertNormEqual(shutil.which(cmd, path=self.dir), self.filepath) + # full match + self.assertNormEqual(shutil.which(cmd, path=self.dir, mode=os.F_OK), + other_file_path) + self.assertNormEqual(shutil.which(self.file, path=self.dir), self.filepath) + self.assertNormEqual(shutil.which(self.file, path=self.dir, mode=os.F_OK), + self.filepath) - exe2 = os.path.join(self.temp_dir, self.to_text_type("test.exe2")) - open(exe2, 'w').close() - os.chmod(exe2, 0o755) + @unittest.skipUnless(sys.platform == "win32", 'test specific to Windows') + def test_dir_order_with_pathext_extension(self): + cmd = self.file # with .exe extension + search_path = os.pathsep.join([os.fsdecode(self.other_dir), + os.fsdecode(self.dir)]) + # full match in the second directory + self.assertNormEqual(shutil.which(cmd, path=search_path), self.filepath) + self.assertNormEqual(shutil.which(cmd, path=search_path, mode=os.F_OK), + self.filepath) + + cmd2 = cmd + self.to_text_type('.com') # with .exe.com extension + other_file_path = os.path.join(self.other_dir, cmd2) + self.create_file(other_file_path) + + # pathext match in the first directory + self.assertNormEqual(shutil.which(cmd, path=search_path), other_file_path) + self.assertNormEqual(shutil.which(cmd, path=search_path, mode=os.F_OK), + other_file_path) + # full match in the first directory + self.assertNormEqual(shutil.which(cmd2, path=search_path), other_file_path) + self.assertNormEqual(shutil.which(cmd2, path=search_path, mode=os.F_OK), + other_file_path) + + # full match in the first directory + search_path = os.pathsep.join([os.fsdecode(self.dir), + os.fsdecode(self.other_dir)]) + self.assertEqual(shutil.which(cmd, path=search_path), self.filepath) + self.assertEqual(shutil.which(cmd, path=search_path, mode=os.F_OK), + self.filepath) - # even though .exe2 is preferred in PATHEXT, we matched directly to test.exe - self.assertEqual(shutil.which(self.to_text_type("test.exe")), exe) - self.assertEqual(shutil.which(self.to_text_type("test")), exe2) + @unittest.skipUnless(sys.platform == "win32", 'test specific to Windows') + def test_dir_order_without_pathext_extension(self): + cmd = self.file[:-4] # without .exe extension + search_path = os.pathsep.join([os.fsdecode(self.other_dir), + os.fsdecode(self.dir)]) + # pathext match in the second directory + self.assertNormEqual(shutil.which(cmd, path=search_path), self.filepath) + self.assertNormEqual(shutil.which(cmd, path=search_path, mode=os.F_OK), + self.filepath) + + # without extension + other_file_path = os.path.join(self.other_dir, cmd) + self.create_file(other_file_path) + + # pathext match in the second directory + self.assertNormEqual(shutil.which(cmd, path=search_path), self.filepath) + # full match in the first directory + self.assertNormEqual(shutil.which(cmd, path=search_path, mode=os.F_OK), + other_file_path) + # full match in the second directory + self.assertNormEqual(shutil.which(self.file, path=search_path), self.filepath) + self.assertNormEqual(shutil.which(self.file, path=search_path, mode=os.F_OK), + self.filepath) + + # pathext match in the first directory + search_path = os.pathsep.join([os.fsdecode(self.dir), + os.fsdecode(self.other_dir)]) + self.assertNormEqual(shutil.which(cmd, path=search_path), self.filepath) + self.assertNormEqual(shutil.which(cmd, path=search_path, mode=os.F_OK), + self.filepath) class TestWhichBytes(TestWhich): @@ -2449,18 +2703,12 @@ def setUp(self): TestWhich.setUp(self) self.dir = os.fsencode(self.dir) self.file = os.fsencode(self.file) - self.temp_file.name = os.fsencode(self.temp_file.name) - self.temp_dir = os.fsencode(self.temp_dir) + self.filepath = os.fsencode(self.filepath) + self.other_dir = os.fsencode(self.other_dir) self.curdir = os.fsencode(self.curdir) self.ext = os.fsencode(self.ext) - def to_text_type(self, s): - ''' - In this class we're testing with bytes, so convert s to a bytes - ''' - if isinstance(s, str): - return s.encode() - return s + to_text_type = staticmethod(os.fsencode) class TestMove(BaseTest, unittest.TestCase): @@ -2471,8 +2719,7 @@ def setUp(self): self.dst_dir = self.mkdtemp() self.src_file = os.path.join(self.src_dir, filename) self.dst_file = os.path.join(self.dst_dir, filename) - with open(self.src_file, "wb") as f: - f.write(b"spam") + create_file(self.src_file, b"spam") def _check_move_file(self, src, dst, real_dst): with open(src, "rb") as f: @@ -2498,12 +2745,12 @@ def test_move_file_to_dir(self): def test_move_file_to_dir_pathlike_src(self): # Move a pathlike file to another location on the same filesystem. - src = pathlib.Path(self.src_file) + src = FakePath(self.src_file) self._check_move_file(src, self.dst_dir, self.dst_file) def test_move_file_to_dir_pathlike_dst(self): # Move a file to another pathlike location on the same filesystem. - dst = pathlib.Path(self.dst_dir) + dst = FakePath(self.dst_dir) self._check_move_file(self.src_file, dst, self.dst_file) @mock_rename @@ -2550,8 +2797,7 @@ def test_move_dir_altsep_to_dir(self): def test_existing_file_inside_dest_dir(self): # A file with the same name inside the destination dir already exists. - with open(self.dst_file, "wb"): - pass + create_file(self.dst_file) self.assertRaises(shutil.Error, shutil.move, self.src_file, self.dst_dir) def test_dont_move_dir_in_itself(self): @@ -2583,7 +2829,6 @@ def test_destinsrc_false_positive(self): finally: os_helper.rmtree(TESTFN) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") @os_helper.skip_unless_symlink @mock_rename def test_move_file_symlink(self): @@ -2593,7 +2838,6 @@ def test_move_file_symlink(self): self.assertTrue(os.path.islink(self.dst_file)) self.assertTrue(os.path.samefile(self.src_file, self.dst_file)) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") @os_helper.skip_unless_symlink @mock_rename def test_move_file_symlink_to_dir(self): @@ -2605,7 +2849,6 @@ def test_move_file_symlink_to_dir(self): self.assertTrue(os.path.islink(final_link)) self.assertTrue(os.path.samefile(self.src_file, final_link)) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") @os_helper.skip_unless_symlink @mock_rename def test_move_dangling_symlink(self): @@ -2617,7 +2860,6 @@ def test_move_dangling_symlink(self): self.assertTrue(os.path.islink(dst_link)) self.assertEqual(os.path.realpath(src), os.path.realpath(dst_link)) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") @os_helper.skip_unless_symlink @mock_rename def test_move_dir_symlink(self): @@ -2693,12 +2935,10 @@ def _test_move_symlink_to_dir_into_dir(self, dst): self.assertTrue(os.path.samefile(self.dst_dir, dst_link)) self.assertTrue(os.path.exists(src)) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") @os_helper.skip_unless_symlink def test_move_symlink_to_dir_into_dir(self): self._test_move_symlink_to_dir_into_dir(self.dst_dir) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") @os_helper.skip_unless_symlink def test_move_symlink_to_dir_into_symlink_to_dir(self): dst = os.path.join(self.src_dir, 'otherlinktodir') @@ -2972,8 +3212,7 @@ def test_empty_file(self): dstname = TESTFN + 'dst' self.addCleanup(lambda: os_helper.unlink(srcname)) self.addCleanup(lambda: os_helper.unlink(dstname)) - with open(srcname, "wb"): - pass + create_file(srcname) with open(srcname, "rb") as src: with open(dstname, "wb") as dst: @@ -3096,7 +3335,7 @@ def test_blocksize_arg(self): self.assertEqual(blocksize, os.path.getsize(TESTFN)) # ...unless we're dealing with a small file. os_helper.unlink(TESTFN2) - write_file(TESTFN2, b"hello", binary=True) + create_file(TESTFN2, b"hello") self.addCleanup(os_helper.unlink, TESTFN2 + '3') self.assertRaises(ZeroDivisionError, shutil.copyfile, TESTFN2, TESTFN2 + '3') @@ -3168,6 +3407,7 @@ def test_bad_environ(self): self.assertGreaterEqual(size.lines, 0) @unittest.skipUnless(os.isatty(sys.__stdout__.fileno()), "not on tty") + @support.requires_subprocess() @unittest.skipUnless(hasattr(os, 'get_terminal_size'), 'need os.get_terminal_size()') def test_stty_match(self): @@ -3185,8 +3425,7 @@ def test_stty_match(self): expected = (int(size[1]), int(size[0])) # reversed order with os_helper.EnvironmentVarGuard() as env: - del env['LINES'] - del env['COLUMNS'] + env.unset('LINES', 'COLUMNS') actual = shutil.get_terminal_size() self.assertEqual(expected, actual) @@ -3194,8 +3433,7 @@ def test_stty_match(self): @unittest.skipIf(support.is_wasi, "WASI has no /dev/null") def test_fallback(self): with os_helper.EnvironmentVarGuard() as env: - del env['LINES'] - del env['COLUMNS'] + env.unset('LINES', 'COLUMNS') # sys.__stdout__ has no fileno() with support.swap_attr(sys, '__stdout__', None): diff --git a/Lib/test/test_signal.py b/Lib/test/test_signal.py index e05467f776..cb744b41e8 100644 --- a/Lib/test/test_signal.py +++ b/Lib/test/test_signal.py @@ -24,8 +24,6 @@ class GenericTests(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_enums(self): for name in dir(signal): sig = getattr(signal, name) @@ -82,8 +80,6 @@ def trivial_signal_handler(self, *args): def create_handler_with_partial(self, argument): return functools.partial(self.trivial_signal_handler, argument) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_out_of_range_signal_number_raises_error(self): self.assertRaises(ValueError, signal.getsignal, 4242) @@ -126,23 +122,17 @@ def __repr__(self): self.assertEqual(signal.getsignal(signal.SIGHUP), hup) self.assertEqual(0, argument.repr_count) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_strsignal(self): self.assertIn("Interrupt", signal.strsignal(signal.SIGINT)) self.assertIn("Terminated", signal.strsignal(signal.SIGTERM)) self.assertIn("Hangup", signal.strsignal(signal.SIGHUP)) - # TODO: RUSTPYTHON - @unittest.expectedFailure # Issue 3864, unknown if this affects earlier versions of freebsd also def test_interprocess_signal(self): dirname = os.path.dirname(__file__) script = os.path.join(dirname, 'signalinterproctester.py') assert_python_ok(script) - # TODO: RUSTPYTHON - @unittest.expectedFailure @unittest.skipUnless( hasattr(signal, "valid_signals"), "requires signal.valid_signals" @@ -190,8 +180,6 @@ def test_keyboard_interrupt_exit_code(self): @unittest.skipUnless(sys.platform == "win32", "Windows specific") class WindowsSignalTests(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_valid_signals(self): s = signal.valid_signals() self.assertIsInstance(s, set) @@ -224,8 +212,6 @@ def test_issue9324(self): with self.assertRaises(ValueError): signal.signal(7, handler) - # TODO: RUSTPYTHON - @unittest.expectedFailure @unittest.skipUnless(sys.executable, "sys.executable required.") @support.requires_subprocess() def test_keyboard_interrupt_exit_code(self): @@ -253,13 +239,11 @@ def test_invalid_call(self): with self.assertRaises(TypeError): signal.set_wakeup_fd(signal.SIGINT, False) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_invalid_fd(self): fd = os_helper.make_bad_fd() self.assertRaises((ValueError, OSError), signal.set_wakeup_fd, fd) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") @unittest.skipUnless(support.has_socket_support, "needs working sockets.") def test_invalid_socket(self): sock = socket.socket() @@ -268,7 +252,6 @@ def test_invalid_socket(self): self.assertRaises((ValueError, OSError), signal.set_wakeup_fd, fd) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") # Emscripten does not support fstat on pipes yet. # https://github.com/emscripten-core/emscripten/issues/16414 @unittest.skipIf(support.is_emscripten, "Emscripten cannot fstat pipes.") @@ -290,7 +273,6 @@ def test_set_wakeup_fd_result(self): self.assertEqual(signal.set_wakeup_fd(-1), w2) self.assertEqual(signal.set_wakeup_fd(-1), -1) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") @unittest.skipIf(support.is_emscripten, "Emscripten cannot fstat pipes.") @unittest.skipUnless(support.has_socket_support, "needs working sockets.") def test_set_wakeup_fd_socket_result(self): @@ -777,8 +759,6 @@ def handler(signum, frame): % (exitcode, stdout)) return (exitcode == 3) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_without_siginterrupt(self): # If a signal handler is installed and siginterrupt is not called # at all, when that signal arrives, it interrupts a syscall that's in @@ -786,8 +766,6 @@ def test_without_siginterrupt(self): interrupted = self.readpipe_interrupted(None) self.assertTrue(interrupted) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_siginterrupt_on(self): # If a signal handler is installed and siginterrupt is called with # a true value for the second argument, when that signal arrives, it @@ -840,8 +818,6 @@ def sig_prof(self, *args): self.hndl_called = True signal.setitimer(signal.ITIMER_PROF, 0) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_itimer_exc(self): # XXX I'm assuming -1 is an invalid itimer, but maybe some platform # defines it ? @@ -851,16 +827,12 @@ def test_itimer_exc(self): self.assertRaises(signal.ItimerError, signal.setitimer, signal.ITIMER_REAL, -1) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_itimer_real(self): self.itimer = signal.ITIMER_REAL signal.setitimer(self.itimer, 1.0) signal.pause() self.assertEqual(self.hndl_called, True) - # TODO: RUSTPYTHON - @unittest.expectedFailure # Issue 3864, unknown if this affects earlier versions of freebsd also @unittest.skipIf(sys.platform in ('netbsd5',), 'itimer not reliable (does not mix well with threading) on some BSDs.') @@ -881,8 +853,6 @@ def test_itimer_virtual(self): # and the handler should have been called self.assertEqual(self.hndl_called, True) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_itimer_prof(self): self.itimer = signal.ITIMER_PROF signal.signal(signal.SIGPROF, self.sig_prof) @@ -900,8 +870,6 @@ def test_itimer_prof(self): # and the handler should have been called self.assertEqual(self.hndl_called, True) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_setitimer_tiny(self): # bpo-30807: C setitimer() takes a microsecond-resolution interval. # Check that float -> timeval conversion doesn't round @@ -1442,8 +1410,6 @@ def cycle_handlers(): class RaiseSignalTest(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_sigint(self): with self.assertRaises(KeyboardInterrupt): signal.raise_signal(signal.SIGINT) @@ -1462,8 +1428,6 @@ def test_invalid_argument(self): else: raise - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_handler(self): is_ok = False def handler(a, b): diff --git a/Lib/test/test_site.py b/Lib/test/test_site.py index 4ff187674e..df279bd965 100644 --- a/Lib/test/test_site.py +++ b/Lib/test/test_site.py @@ -7,10 +7,12 @@ import unittest import test.support from test import support +from test.support.script_helper import assert_python_ok from test.support import os_helper from test.support import socket_helper from test.support import captured_stderr from test.support.os_helper import TESTFN, EnvironmentVarGuard +from test.support.script_helper import spawn_python, kill_python import ast import builtins import glob @@ -23,6 +25,7 @@ import sys import sysconfig import tempfile +from textwrap import dedent import urllib.error import urllib.request from unittest import mock @@ -220,8 +223,6 @@ def test_addsitedir_hidden_flags(self): finally: pth_file.cleanup() - # TODO: RUSTPYTHON - @unittest.expectedFailure @unittest.skipUnless(sys.platform == 'win32', 'test needs Windows') @support.requires_subprocess() def test_addsitedir_hidden_file_attribute(self): @@ -330,14 +331,14 @@ def test_getsitepackages(self): self.assertEqual(len(dirs), 2) wanted = os.path.join('xoxo', sys.platlibdir, # XXX: RUSTPYTHON - 'rustpython%d.%d' % sys.version_info[:2], + f'rustpython{sysconfig._get_python_version_abi()}', 'site-packages') self.assertEqual(dirs[0], wanted) else: self.assertEqual(len(dirs), 1) wanted = os.path.join('xoxo', 'lib', # XXX: RUSTPYTHON - 'rustpython%d.%d' % sys.version_info[:2], + f'rustpython{sysconfig._get_python_version_abi()}', 'site-packages') self.assertEqual(dirs[-1], wanted) else: @@ -358,9 +359,7 @@ def test_no_home_directory(self): with EnvironmentVarGuard() as environ, \ mock.patch('os.path.expanduser', lambda path: path): - - del environ['PYTHONUSERBASE'] - del environ['APPDATA'] + environ.unset('PYTHONUSERBASE', 'APPDATA') user_base = site.getuserbase() self.assertTrue(user_base.startswith('~' + os.sep), @@ -382,6 +381,19 @@ def test_no_home_directory(self): mock_addsitedir.assert_not_called() self.assertFalse(known_paths) + def test_gethistoryfile(self): + filename = 'file' + rc, out, err = assert_python_ok('-c', + f'import site; assert site.gethistoryfile() == "{filename}"', + PYTHON_HISTORY=filename) + self.assertEqual(rc, 0) + + # Check that PYTHON_HISTORY is ignored in isolated mode. + rc, out, err = assert_python_ok('-I', '-c', + f'import site; assert site.gethistoryfile() != "{filename}"', + PYTHON_HISTORY=filename) + self.assertEqual(rc, 0) + def test_trace(self): message = "bla-bla-bla" for verbose, out in (True, message + "\n"), (False, ""): @@ -509,6 +521,44 @@ def test_sitecustomize_executed(self): else: self.fail("sitecustomize not imported automatically") + @support.requires_subprocess() + def test_customization_modules_on_startup(self): + mod_names = [ + 'sitecustomize' + ] + + if site.ENABLE_USER_SITE: + mod_names.append('usercustomize') + + temp_dir = tempfile.mkdtemp() + self.addCleanup(os_helper.rmtree, temp_dir) + + with EnvironmentVarGuard() as environ: + environ['PYTHONPATH'] = temp_dir + + for module_name in mod_names: + os_helper.rmtree(temp_dir) + os.mkdir(temp_dir) + + customize_path = os.path.join(temp_dir, f'{module_name}.py') + eyecatcher = f'EXECUTED_{module_name}' + + with open(customize_path, 'w') as f: + f.write(f'print("{eyecatcher}")') + + output = subprocess.check_output([sys.executable, '-c', '""']) + self.assertIn(eyecatcher, output.decode('utf-8')) + + # -S blocks any site-packages + output = subprocess.check_output([sys.executable, '-S', '-c', '""']) + self.assertNotIn(eyecatcher, output.decode('utf-8')) + + # -s blocks user site-packages + if 'usercustomize' == module_name: + output = subprocess.check_output([sys.executable, '-s', '-c', '""']) + self.assertNotIn(eyecatcher, output.decode('utf-8')) + + @unittest.skipUnless(hasattr(urllib.request, "HTTPSHandler"), 'need SSL support to download license') @test.support.requires_resource('network') @@ -531,8 +581,7 @@ def test_license_exists_at_url(self): class StartupImportTests(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON @support.requires_subprocess() def test_startup_imports(self): # Get sys.path in isolated mode (python3 -I) @@ -665,8 +714,7 @@ def _get_pth_lines(self, libpath: str, *, import_site: bool): pth_lines.append('import site') return pth_lines - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON @support.requires_subprocess() def test_underpth_basic(self): pth_lines = ['#.', '# ..', *sys.path, '.', '..'] @@ -686,8 +734,7 @@ def test_underpth_basic(self): "sys.path is incorrect" ) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON @support.requires_subprocess() def test_underpth_nosite_file(self): libpath = test.support.STDLIB_DIR @@ -712,8 +759,7 @@ def test_underpth_nosite_file(self): "sys.path is incorrect" ) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON @support.requires_subprocess() def test_underpth_file(self): libpath = test.support.STDLIB_DIR @@ -734,8 +780,7 @@ def test_underpth_file(self): )], env=env) self.assertTrue(rc, "sys.path is incorrect") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON @support.requires_subprocess() def test_underpth_dll_file(self): libpath = test.support.STDLIB_DIR @@ -757,5 +802,107 @@ def test_underpth_dll_file(self): self.assertTrue(rc, "sys.path is incorrect") +class CommandLineTests(unittest.TestCase): + def exists(self, path): + if path is not None and os.path.isdir(path): + return "exists" + else: + return "doesn't exist" + + def get_excepted_output(self, *args): + if len(args) == 0: + user_base = site.getuserbase() + user_site = site.getusersitepackages() + output = io.StringIO() + output.write("sys.path = [\n") + for dir in sys.path: + output.write(" %r,\n" % (dir,)) + output.write("]\n") + output.write(f"USER_BASE: {user_base} ({self.exists(user_base)})\n") + output.write(f"USER_SITE: {user_site} ({self.exists(user_site)})\n") + output.write(f"ENABLE_USER_SITE: {site.ENABLE_USER_SITE}\n") + return 0, dedent(output.getvalue()).strip() + + buffer = [] + if '--user-base' in args: + buffer.append(site.getuserbase()) + if '--user-site' in args: + buffer.append(site.getusersitepackages()) + + if buffer: + return_code = 3 + if site.ENABLE_USER_SITE: + return_code = 0 + elif site.ENABLE_USER_SITE is False: + return_code = 1 + elif site.ENABLE_USER_SITE is None: + return_code = 2 + output = os.pathsep.join(buffer) + return return_code, os.path.normpath(dedent(output).strip()) + else: + return 10, None + + def invoke_command_line(self, *args): + args = ["-m", "site", *args] + + with EnvironmentVarGuard() as env: + env["PYTHONUTF8"] = "1" + env["PYTHONIOENCODING"] = "utf-8" + proc = spawn_python(*args, text=True, env=env, + encoding='utf-8', errors='replace') + + output = kill_python(proc) + return_code = proc.returncode + return return_code, os.path.normpath(dedent(output).strip()) + + @support.requires_subprocess() + def test_no_args(self): + return_code, output = self.invoke_command_line() + excepted_return_code, _ = self.get_excepted_output() + self.assertEqual(return_code, excepted_return_code) + lines = output.splitlines() + self.assertEqual(lines[0], "sys.path = [") + self.assertEqual(lines[-4], "]") + excepted_base = f"USER_BASE: '{site.getuserbase()}'" +\ + f" ({self.exists(site.getuserbase())})" + self.assertEqual(lines[-3], excepted_base) + excepted_site = f"USER_SITE: '{site.getusersitepackages()}'" +\ + f" ({self.exists(site.getusersitepackages())})" + self.assertEqual(lines[-2], excepted_site) + self.assertEqual(lines[-1], f"ENABLE_USER_SITE: {site.ENABLE_USER_SITE}") + + @support.requires_subprocess() + def test_unknown_args(self): + return_code, output = self.invoke_command_line("--unknown-arg") + excepted_return_code, _ = self.get_excepted_output("--unknown-arg") + self.assertEqual(return_code, excepted_return_code) + self.assertIn('[--user-base] [--user-site]', output) + + @support.requires_subprocess() + def test_base_arg(self): + return_code, output = self.invoke_command_line("--user-base") + excepted = self.get_excepted_output("--user-base") + excepted_return_code, excepted_output = excepted + self.assertEqual(return_code, excepted_return_code) + self.assertEqual(output, excepted_output) + + @support.requires_subprocess() + def test_site_arg(self): + return_code, output = self.invoke_command_line("--user-site") + excepted = self.get_excepted_output("--user-site") + excepted_return_code, excepted_output = excepted + self.assertEqual(return_code, excepted_return_code) + self.assertEqual(output, excepted_output) + + @support.requires_subprocess() + def test_both_args(self): + return_code, output = self.invoke_command_line("--user-base", + "--user-site") + excepted = self.get_excepted_output("--user-base", "--user-site") + excepted_return_code, excepted_output = excepted + self.assertEqual(return_code, excepted_return_code) + self.assertEqual(output, excepted_output) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_slice.py b/Lib/test/test_slice.py index 53d4c77616..6de7e73c39 100644 --- a/Lib/test/test_slice.py +++ b/Lib/test/test_slice.py @@ -286,8 +286,7 @@ def test_deepcopy(self): self.assertIsNot(s.stop, c.stop) self.assertIsNot(s.step, c.step) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_cycle(self): class myobj(): pass o = myobj() diff --git a/Lib/test/test_smtplib.py b/Lib/test/test_smtplib.py index 9b787950fc..ade0dc1308 100644 --- a/Lib/test/test_smtplib.py +++ b/Lib/test/test_smtplib.py @@ -17,6 +17,7 @@ import threading import unittest +import unittest.mock as mock from test import support, mock_socket from test.support import hashlib_helper from test.support import socket_helper @@ -350,7 +351,7 @@ def testVRFY(self): timeout=support.LOOPBACK_TIMEOUT) self.addCleanup(smtp.close) expected = (252, b'Cannot VRFY user, but will accept message ' + \ - b'and attempt delivery') + b'and attempt delivery') self.assertEqual(smtp.vrfy('nobody@nowhere.com'), expected) self.assertEqual(smtp.verify('nobody@nowhere.com'), expected) smtp.quit() @@ -371,7 +372,7 @@ def testHELP(self): timeout=support.LOOPBACK_TIMEOUT) self.addCleanup(smtp.close) self.assertEqual(smtp.help(), b'Supported commands: EHLO HELO MAIL ' + \ - b'RCPT DATA RSET NOOP QUIT VRFY') + b'RCPT DATA RSET NOOP QUIT VRFY') smtp.quit() def testSend(self): @@ -527,7 +528,7 @@ def testSendMessageWithAddresses(self): smtp.quit() # make sure the Bcc header is still in the message. self.assertEqual(m['Bcc'], 'John Root <root@localhost>, "Dinsdale" ' - '<warped@silly.walks.com>') + '<warped@silly.walks.com>') self.client_evt.set() self.serv_evt.wait() @@ -766,7 +767,7 @@ def tearDown(self): def testFailingHELO(self): self.assertRaises(smtplib.SMTPConnectError, smtplib.SMTP, - HOST, self.port, 'localhost', 3) + HOST, self.port, 'localhost', 3) class TooLongLineTests(unittest.TestCase): @@ -804,14 +805,14 @@ def testLineTooLong(self): sim_users = {'Mr.A@somewhere.com':'John A', 'Ms.B@xn--fo-fka.com':'Sally B', 'Mrs.C@somewhereesle.com':'Ruth C', - } + } sim_auth = ('Mr.A@somewhere.com', 'somepassword') sim_cram_md5_challenge = ('PENCeUxFREJoU0NnbmhNWitOMjNGNn' 'dAZWx3b29kLmlubm9zb2Z0LmNvbT4=') sim_lists = {'list-1':['Mr.A@somewhere.com','Mrs.C@somewhereesle.com'], 'list-2':['Ms.B@xn--fo-fka.com',], - } + } # Simulated SMTP channel & server class ResponseException(Exception): pass @@ -830,6 +831,7 @@ class SimSMTPChannel(smtpd.SMTPChannel): def __init__(self, extra_features, *args, **kw): self._extrafeatures = ''.join( [ "250-{0}\r\n".format(x) for x in extra_features ]) + self.all_received_lines = [] super(SimSMTPChannel, self).__init__(*args, **kw) # AUTH related stuff. It would be nice if support for this were in smtpd. @@ -844,6 +846,7 @@ def found_terminator(self): self.smtp_state = self.COMMAND self.push('%s %s' % (e.smtp_code, e.smtp_error)) return + self.all_received_lines.append(self.received_lines) super().found_terminator() @@ -924,11 +927,14 @@ def _auth_cram_md5(self, arg=None): except ValueError as e: self.push('535 Splitting response {!r} into user and password ' 'failed: {}'.format(logpass, e)) - return False - valid_hashed_pass = hmac.HMAC( - sim_auth[1].encode('ascii'), - self._decode_base64(sim_cram_md5_challenge).encode('ascii'), - 'md5').hexdigest() + return + pwd = sim_auth[1].encode('ascii') + msg = self._decode_base64(sim_cram_md5_challenge).encode('ascii') + try: + valid_hashed_pass = hmac.HMAC(pwd, msg, 'md5').hexdigest() + except ValueError: + self.push('504 CRAM-MD5 is not supported') + return self._authenticated(user, hashed_pass == valid_hashed_pass) # end AUTH related stuff. @@ -1170,8 +1176,7 @@ def auth_buggy(challenge=None): finally: smtp.close() - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON @hashlib_helper.requires_hashdigest('md5', openssl=True) def testAUTH_CRAM_MD5(self): self.serv.add_feature("AUTH CRAM-MD5") @@ -1181,8 +1186,39 @@ def testAUTH_CRAM_MD5(self): self.assertEqual(resp, (235, b'Authentication Succeeded')) smtp.close() - # TODO: RUSTPYTHON - @unittest.expectedFailure + @mock.patch("hmac.HMAC") + @mock.patch("smtplib._have_cram_md5_support", False) + def testAUTH_CRAM_MD5_blocked(self, hmac_constructor): + # CRAM-MD5 is the only "known" method by the server, + # but it is not supported by the client. In particular, + # no challenge will ever be sent. + self.serv.add_feature("AUTH CRAM-MD5") + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', + timeout=support.LOOPBACK_TIMEOUT) + self.addCleanup(smtp.close) + msg = re.escape("No suitable authentication method found.") + with self.assertRaisesRegex(smtplib.SMTPException, msg): + smtp.login(sim_auth[0], sim_auth[1]) + hmac_constructor.assert_not_called() # call has been bypassed + + @mock.patch("smtplib._have_cram_md5_support", False) + def testAUTH_CRAM_MD5_blocked_and_fallback(self): + # Test that PLAIN is tried after CRAM-MD5 failed + self.serv.add_feature("AUTH CRAM-MD5 PLAIN") + smtp = smtplib.SMTP(HOST, self.port, local_hostname='localhost', + timeout=support.LOOPBACK_TIMEOUT) + self.addCleanup(smtp.close) + with ( + mock.patch.object(smtp, "auth_cram_md5") as smtp_auth_cram_md5, + mock.patch.object( + smtp, "auth_plain", wraps=smtp.auth_plain + ) as smtp_auth_plain + ): + resp = smtp.login(sim_auth[0], sim_auth[1]) + smtp_auth_plain.assert_called_once() + smtp_auth_cram_md5.assert_not_called() # no call to HMAC constructor + self.assertEqual(resp, (235, b'Authentication Succeeded')) + @hashlib_helper.requires_hashdigest('md5', openssl=True) def testAUTH_multiple(self): # Test that multiple authentication methods are tried. @@ -1193,8 +1229,7 @@ def testAUTH_multiple(self): self.assertEqual(resp, (235, b'Authentication Succeeded')) smtp.close() - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_auth_function(self): supported = {'PLAIN', 'LOGIN'} try: @@ -1354,6 +1389,18 @@ def test_name_field_not_included_in_envelop_addresses(self): self.assertEqual(self.serv._addresses['from'], 'michael@example.com') self.assertEqual(self.serv._addresses['tos'], ['rene@example.com']) + def test_lowercase_mail_from_rcpt_to(self): + m = 'A test message' + smtp = smtplib.SMTP( + HOST, self.port, local_hostname='localhost', + timeout=support.LOOPBACK_TIMEOUT) + self.addCleanup(smtp.close) + + smtp.sendmail('John', 'Sally', m) + + self.assertIn(['mail from:<John> size=14'], self.serv._SMTPchannel.all_received_lines) + self.assertIn(['rcpt to:<Sally>'], self.serv._SMTPchannel.all_received_lines) + class SimSMTPUTF8Server(SimSMTPServer): @@ -1372,7 +1419,7 @@ def handle_accepted(self, conn, addr): ) def process_message(self, peer, mailfrom, rcpttos, data, mail_options=None, - rcpt_options=None): + rcpt_options=None): self.last_peer = peer self.last_mailfrom = mailfrom self.last_rcpttos = rcpttos diff --git a/Lib/test/test_smtpnet.py b/Lib/test/test_smtpnet.py index be25e961f7..d765746987 100644 --- a/Lib/test/test_smtpnet.py +++ b/Lib/test/test_smtpnet.py @@ -2,6 +2,7 @@ from test import support from test.support import import_helper from test.support import socket_helper +import os import smtplib import socket @@ -9,6 +10,8 @@ support.requires("network") +SMTP_TEST_SERVER = os.getenv('CPYTHON_TEST_SMTP_SERVER', 'smtp.gmail.com') + def check_ssl_verifiy(host, port): context = ssl.create_default_context() with socket.create_connection((host, port)) as sock: @@ -22,7 +25,7 @@ def check_ssl_verifiy(host, port): class SmtpTest(unittest.TestCase): - testServer = 'smtp.gmail.com' + testServer = SMTP_TEST_SERVER remotePort = 587 def test_connect_starttls(self): @@ -44,7 +47,7 @@ def test_connect_starttls(self): class SmtpSSLTest(unittest.TestCase): - testServer = 'smtp.gmail.com' + testServer = SMTP_TEST_SERVER remotePort = 465 def test_connect(self): @@ -87,4 +90,4 @@ def test_connect_using_sslcontext_verified(self): if __name__ == "__main__": - unittest.main() \ No newline at end of file + unittest.main() diff --git a/Lib/test/test_socket.py b/Lib/test/test_socket.py index ea544f6afa..d5a35a3253 100644 --- a/Lib/test/test_socket.py +++ b/Lib/test/test_socket.py @@ -2052,8 +2052,6 @@ def test_socket_fileno_requires_socket_fd(self): fileno=afile.fileno()) self.assertEqual(cm.exception.errno, errno.ENOTSOCK) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_addressfamily_enum(self): import _socket, enum CheckedAddressFamily = enum._old_convert_( @@ -2063,8 +2061,6 @@ def test_addressfamily_enum(self): ) enum._test_simple_enum(CheckedAddressFamily, socket.AddressFamily) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_socketkind_enum(self): import _socket, enum CheckedSocketKind = enum._old_convert_( @@ -2074,8 +2070,6 @@ def test_socketkind_enum(self): ) enum._test_simple_enum(CheckedSocketKind, socket.SocketKind) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_msgflag_enum(self): import _socket, enum CheckedMsgFlag = enum._old_convert_( @@ -2085,8 +2079,6 @@ def test_msgflag_enum(self): ) enum._test_simple_enum(CheckedMsgFlag, socket.MsgFlag) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_addressinfo_enum(self): import _socket, enum CheckedAddressInfo = enum._old_convert_( @@ -5360,8 +5352,6 @@ def test_create_connection(self): expected_errnos = socket_helper.get_socket_conn_refused_errs() self.assertIn(cm.exception.errno, expected_errnos) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_create_connection_all_errors(self): port = socket_helper.find_unused_port() try: diff --git a/Lib/test/test_socketserver.py b/Lib/test/test_socketserver.py index 113f959ff2..cdbf341b9c 100644 --- a/Lib/test/test_socketserver.py +++ b/Lib/test/test_socketserver.py @@ -178,13 +178,11 @@ def dgram_examine(self, proto, addr): buf += data self.assertEqual(buf, TEST_STR) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; AssertionError: -1 != 18446744073709551615") def test_TCPServer(self): self.run_server(socketserver.TCPServer, socketserver.StreamRequestHandler, self.stream_examine) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; AssertionError: -1 != 18446744073709551615") def test_ThreadingTCPServer(self): self.run_server(socketserver.ThreadingTCPServer, socketserver.StreamRequestHandler, @@ -217,13 +215,11 @@ def test_ForkingUnixStreamServer(self): socketserver.StreamRequestHandler, self.stream_examine) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; AssertionError: -1 != 18446744073709551615") def test_UDPServer(self): self.run_server(socketserver.UDPServer, socketserver.DatagramRequestHandler, self.dgram_examine) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; AssertionError: -1 != 18446744073709551615") def test_ThreadingUDPServer(self): self.run_server(socketserver.ThreadingUDPServer, socketserver.DatagramRequestHandler, @@ -298,7 +294,6 @@ def test_tcpserver_bind_leak(self): socketserver.TCPServer((HOST, -1), socketserver.StreamRequestHandler) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; AssertionError: -1 != 18446744073709551615") def test_context_manager(self): with socketserver.TCPServer((HOST, 0), socketserver.StreamRequestHandler) as server: diff --git a/Lib/test/test_sqlite3/test_dbapi.py b/Lib/test/test_sqlite3/test_dbapi.py index 56cd8f8a68..9a95c489a3 100644 --- a/Lib/test/test_sqlite3/test_dbapi.py +++ b/Lib/test/test_sqlite3/test_dbapi.py @@ -346,8 +346,6 @@ def test_extended_error_code_on_exception(self): sqlite.SQLITE_CONSTRAINT_CHECK) self.assertEqual(exc.sqlite_errorname, "SQLITE_CONSTRAINT_CHECK") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_disallow_instantiation(self): cx = sqlite.connect(":memory:") check_disallow_instantiation(self, type(cx("select 1"))) @@ -575,8 +573,6 @@ def test_connection_reinit(self): self.assertTrue(all(isinstance(r, sqlite.Row) for r in rows)) self.assertEqual([r[0] for r in rows], ["2", "3"]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_connection_bad_reinit(self): cx = sqlite.connect(":memory:") with cx: @@ -769,8 +765,6 @@ def test_execute_illegal_sql(self): with self.assertRaises(sqlite.OperationalError): self.cu.execute("select asdf") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_execute_multiple_statements(self): msg = "You can only execute one statement at a time" dataset = ( @@ -793,8 +787,6 @@ def test_execute_multiple_statements(self): with self.assertRaisesRegex(sqlite.ProgrammingError, msg): self.cu.execute(query) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_execute_with_appended_comments(self): dataset = ( "select 1; -- foo bar", @@ -842,15 +834,11 @@ def test_execute_wrong_no_of_args1(self): with self.assertRaises(sqlite.ProgrammingError): self.cu.execute("insert into test(id) values (?)", (17, "Egon")) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_execute_wrong_no_of_args2(self): # too little parameters with self.assertRaises(sqlite.ProgrammingError): self.cu.execute("insert into test(id) values (?)") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_execute_wrong_no_of_args3(self): # no parameters, parameters are needed with self.assertRaises(sqlite.ProgrammingError): @@ -912,15 +900,11 @@ def __missing__(self, key): row = self.cu.fetchone() self.assertEqual(row[0], "foo") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_execute_dict_mapping_too_little_args(self): self.cu.execute("insert into test(name) values ('foo')") with self.assertRaises(sqlite.ProgrammingError): self.cu.execute("select name from test where name=:name and id=:id", {"name": "foo"}) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_execute_dict_mapping_no_args(self): self.cu.execute("insert into test(name) values ('foo')") with self.assertRaises(sqlite.ProgrammingError): @@ -963,8 +947,6 @@ def test_rowcount_update_returning(self): self.assertEqual(self.cu.fetchone()[0], 1) self.assertEqual(self.cu.rowcount, 1) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_rowcount_prefixed_with_comment(self): # gh-79579: rowcount is updated even if query is prefixed with comments self.cu.execute(""" @@ -1085,8 +1067,6 @@ def test_fetchmany(self): res = self.cu.fetchmany(100) self.assertEqual(res, []) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_fetchmany_kw_arg(self): """Checks if fetchmany works with keyword arguments""" self.cu.execute("select name from test") @@ -1330,8 +1310,6 @@ def test_blob_set_item_with_offset(self): expected = b"This blob data string is exactly fifty bytes long." self.assertEqual(self.blob.read(), expected) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_blob_set_slice_buffer_object(self): from array import array self.blob[0:5] = memoryview(b"12345") @@ -1359,22 +1337,16 @@ def test_blob_get_slice_negative_index(self): def test_blob_get_slice_with_skip(self): self.assertEqual(self.blob[0:10:2], b"ti lb") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_blob_set_slice(self): self.blob[0:5] = b"12345" expected = b"12345" + self.data[5:] actual = self.cx.execute("select b from test").fetchone()[0] self.assertEqual(actual, expected) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_blob_set_empty_slice(self): self.blob[0:0] = b"" self.assertEqual(self.blob[:], self.data) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_blob_set_slice_with_skip(self): self.blob[0:10:2] = b"12345" actual = self.cx.execute("select b from test").fetchone()[0] @@ -1427,8 +1399,6 @@ def test_blob_set_item_error(self): with self.assertRaisesRegex(ValueError, "must be in range"): self.blob[0] = 2**65 - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_blob_set_slice_error(self): with self.assertRaisesRegex(IndexError, "wrong size"): self.blob[5:10] = b"a" @@ -1441,8 +1411,6 @@ def test_blob_set_slice_error(self): with self.assertRaises(BufferError): self.blob[5:10] = memoryview(b"abcde")[::2] - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_blob_sequence_not_supported(self): with self.assertRaisesRegex(TypeError, "unsupported operand"): self.blob + self.blob @@ -1470,8 +1438,6 @@ class DummyException(Exception): raise DummyException("reraised") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_blob_closed(self): with memory_database() as cx: cx.execute("create table test(b blob)") @@ -1501,8 +1467,6 @@ def test_blob_closed(self): with self.assertRaisesRegex(sqlite.ProgrammingError, msg): blob[0] = b"" - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_blob_closed_db_read(self): with memory_database() as cx: cx.execute("create table test(b blob)") @@ -1661,8 +1625,6 @@ def test_cursor_executescript_with_null_characters(self): insert into a(i) values (5); """) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_cursor_executescript_with_surrogates(self): con = sqlite.connect(":memory:") cur = con.cursor() @@ -1766,8 +1728,6 @@ def progress(): pass with self.assertRaises(sqlite.ProgrammingError): con.set_progress_handler(progress, 100) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_closed_call(self): con = sqlite.connect(":memory:") con.close() @@ -1775,8 +1735,6 @@ def test_closed_call(self): con() class ClosedCurTests(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_closed(self): con = sqlite.connect(":memory:") cur = con.cursor() diff --git a/Lib/test/test_sqlite3/test_factory.py b/Lib/test/test_sqlite3/test_factory.py index e52c10fe94..c13a748152 100644 --- a/Lib/test/test_sqlite3/test_factory.py +++ b/Lib/test/test_sqlite3/test_factory.py @@ -241,8 +241,6 @@ def test_sqlite_row_hash_cmp(self): self.assertEqual(hash(row_1), hash(row_2)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_sqlite_row_as_sequence(self): """ Checks if the row object can act like a sequence """ self.con.row_factory = sqlite.Row diff --git a/Lib/test/test_sqlite3/test_regression.py b/Lib/test/test_sqlite3/test_regression.py index dfcf3b11f5..d746be647c 100644 --- a/Lib/test/test_sqlite3/test_regression.py +++ b/Lib/test/test_sqlite3/test_regression.py @@ -150,8 +150,6 @@ def __conform__(self, protocol): with self.assertRaises(IndexError): con.execute("insert into foo(bar, baz) values (?, ?)", parameters) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_error_msg_decode_error(self): # When porting the module to Python 3.0, the error message about # decoding errors disappeared. This verifies they're back again. @@ -197,8 +195,6 @@ def __del__(self): con.isolation_level = value self.assertEqual(con.isolation_level, "DEFERRED") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_cursor_constructor_call_check(self): """ Verifies that cursor methods check whether base class __init__ was @@ -223,8 +219,6 @@ def test_str_subclass(self): class MyStr(str): pass self.con.execute("select ?", (MyStr("abc"),)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_connection_constructor_call_check(self): """ Verifies that connection methods check whether base class __init__ was @@ -265,8 +259,6 @@ def test_connection_call(self): """ self.assertRaises(TypeError, self.con, b"select 1") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_collation(self): def collation_cb(a, b): return 1 @@ -329,8 +321,6 @@ def test_invalid_isolation_level_type(self): sqlite.connect, ":memory:", isolation_level=123) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_null_character(self): # Issue #21147 cur = self.con.cursor() @@ -343,8 +333,6 @@ def test_null_character(self): self.assertRaisesRegex(sqlite.ProgrammingError, "null char", cur.execute, query) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_surrogates(self): con = sqlite.connect(":memory:") self.assertRaises(UnicodeEncodeError, con, "select '\ud8ff'") @@ -353,8 +341,6 @@ def test_surrogates(self): self.assertRaises(UnicodeEncodeError, cur.execute, "select '\ud8ff'") self.assertRaises(UnicodeEncodeError, cur.execute, "select '\udcff'") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_large_sql(self): msg = "query string is too large" with memory_database() as cx, cx_limit(cx) as lim: @@ -415,8 +401,6 @@ def callback(*args): del ref support.gc_collect() - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_del_isolation_level_segfault(self): with self.assertRaises(AttributeError): del self.con.isolation_level diff --git a/Lib/test/test_sqlite3/test_types.py b/Lib/test/test_sqlite3/test_types.py index 6cbf99d6ea..6231882351 100644 --- a/Lib/test/test_sqlite3/test_types.py +++ b/Lib/test/test_sqlite3/test_types.py @@ -433,14 +433,10 @@ def test_missing_adapter(self): with self.assertRaises(sqlite.ProgrammingError): sqlite.adapt(1.) # No float adapter registered - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_missing_protocol(self): with self.assertRaises(sqlite.ProgrammingError): sqlite.adapt(1, None) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_defect_proto(self): class DefectProto(): def __adapt__(self): @@ -448,8 +444,6 @@ def __adapt__(self): with self.assertRaises(sqlite.ProgrammingError): sqlite.adapt(1., DefectProto) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_defect_self_adapt(self): class DefectSelfAdapt(float): def __conform__(self, _): diff --git a/Lib/test/test_sqlite3/test_userfunctions.py b/Lib/test/test_sqlite3/test_userfunctions.py index 7b092365d4..e8b98a66a5 100644 --- a/Lib/test/test_sqlite3/test_userfunctions.py +++ b/Lib/test/test_sqlite3/test_userfunctions.py @@ -354,8 +354,6 @@ def test_return_non_contiguous_blob(self): cur = self.con.execute("select return_noncont_blob()") cur.fetchone() - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_param_surrogates(self): self.assertRaisesRegex(UnicodeEncodeError, "surrogates not allowed", self.con.execute, "select spam(?)", diff --git a/Lib/test/test_ssl.py b/Lib/test/test_ssl.py new file mode 100644 index 0000000000..09d35a77a1 --- /dev/null +++ b/Lib/test/test_ssl.py @@ -0,0 +1,5567 @@ +# Test the support for SSL and sockets + +import sys +import unittest +import unittest.mock +from ast import literal_eval +from threading import Thread +from test import support +from test.support import import_helper +from test.support import os_helper +from test.support import socket_helper +from test.support import threading_helper +from test.support import warnings_helper +from test.support import asyncore +import array +import re +import socket +import select +import struct +import time +import enum +import gc +import http.client +import os +import errno +import pprint +import urllib.request +import threading +import traceback +import weakref +import platform +import sysconfig +import functools +from contextlib import nullcontext +try: + import ctypes +except ImportError: + ctypes = None + + +ssl = import_helper.import_module("ssl") +import _ssl + +from ssl import Purpose, TLSVersion, _TLSContentType, _TLSMessageType, _TLSAlertType + +Py_DEBUG_WIN32 = support.Py_DEBUG and sys.platform == 'win32' + +PROTOCOLS = sorted(ssl._PROTOCOL_NAMES) +HOST = socket_helper.HOST +IS_OPENSSL_3_0_0 = ssl.OPENSSL_VERSION_INFO >= (3, 0, 0) +PY_SSL_DEFAULT_CIPHERS = sysconfig.get_config_var('PY_SSL_DEFAULT_CIPHERS') + +PROTOCOL_TO_TLS_VERSION = {} +for proto, ver in ( + ("PROTOCOL_SSLv3", "SSLv3"), + ("PROTOCOL_TLSv1", "TLSv1"), + ("PROTOCOL_TLSv1_1", "TLSv1_1"), +): + try: + proto = getattr(ssl, proto) + ver = getattr(ssl.TLSVersion, ver) + except AttributeError: + continue + PROTOCOL_TO_TLS_VERSION[proto] = ver + +def data_file(*name): + return os.path.join(os.path.dirname(__file__), "certdata", *name) + +# The custom key and certificate files used in test_ssl are generated +# using Lib/test/certdata/make_ssl_certs.py. +# Other certificates are simply fetched from the internet servers they +# are meant to authenticate. + +CERTFILE = data_file("keycert.pem") +BYTES_CERTFILE = os.fsencode(CERTFILE) +ONLYCERT = data_file("ssl_cert.pem") +ONLYKEY = data_file("ssl_key.pem") +BYTES_ONLYCERT = os.fsencode(ONLYCERT) +BYTES_ONLYKEY = os.fsencode(ONLYKEY) +CERTFILE_PROTECTED = data_file("keycert.passwd.pem") +ONLYKEY_PROTECTED = data_file("ssl_key.passwd.pem") +KEY_PASSWORD = "somepass" +CAPATH = data_file("capath") +BYTES_CAPATH = os.fsencode(CAPATH) +CAFILE_NEURONIO = data_file("capath", "4e1295a3.0") +CAFILE_CACERT = data_file("capath", "5ed36f99.0") + +CERTFILE_INFO = { + 'issuer': ((('countryName', 'XY'),), + (('localityName', 'Castle Anthrax'),), + (('organizationName', 'Python Software Foundation'),), + (('commonName', 'localhost'),)), + 'notAfter': 'Jan 24 04:21:36 2043 GMT', + 'notBefore': 'Nov 25 04:21:36 2023 GMT', + 'serialNumber': '53E14833F7546C29256DD0F034F776C5E983004C', + 'subject': ((('countryName', 'XY'),), + (('localityName', 'Castle Anthrax'),), + (('organizationName', 'Python Software Foundation'),), + (('commonName', 'localhost'),)), + 'subjectAltName': (('DNS', 'localhost'),), + 'version': 3 +} + +# empty CRL +CRLFILE = data_file("revocation.crl") + +# Two keys and certs signed by the same CA (for SNI tests) +SIGNED_CERTFILE = data_file("keycert3.pem") +SINGED_CERTFILE_ONLY = data_file("cert3.pem") +SIGNED_CERTFILE_HOSTNAME = 'localhost' + +SIGNED_CERTFILE_INFO = { + 'OCSP': ('http://testca.pythontest.net/testca/ocsp/',), + 'caIssuers': ('http://testca.pythontest.net/testca/pycacert.cer',), + 'crlDistributionPoints': ('http://testca.pythontest.net/testca/revocation.crl',), + 'issuer': ((('countryName', 'XY'),), + (('organizationName', 'Python Software Foundation CA'),), + (('commonName', 'our-ca-server'),)), + 'notAfter': 'Oct 28 14:23:16 2037 GMT', + 'notBefore': 'Aug 29 14:23:16 2018 GMT', + 'serialNumber': 'CB2D80995A69525C', + 'subject': ((('countryName', 'XY'),), + (('localityName', 'Castle Anthrax'),), + (('organizationName', 'Python Software Foundation'),), + (('commonName', 'localhost'),)), + 'subjectAltName': (('DNS', 'localhost'),), + 'version': 3 +} + +SIGNED_CERTFILE2 = data_file("keycert4.pem") +SIGNED_CERTFILE2_HOSTNAME = 'fakehostname' +SIGNED_CERTFILE_ECC = data_file("keycertecc.pem") +SIGNED_CERTFILE_ECC_HOSTNAME = 'localhost-ecc' + +# A custom testcase, extracted from `rfc5280::aki::leaf-missing-aki` in x509-limbo: +# The leaf (server) certificate has no AKI, which is forbidden under RFC 5280. +# See: https://x509-limbo.com/testcases/rfc5280/#rfc5280akileaf-missing-aki +LEAF_MISSING_AKI_CERTFILE = data_file("leaf-missing-aki.keycert.pem") +LEAF_MISSING_AKI_CERTFILE_HOSTNAME = "example.com" +LEAF_MISSING_AKI_CA = data_file("leaf-missing-aki.ca.pem") + +# Same certificate as pycacert.pem, but without extra text in file +SIGNING_CA = data_file("capath", "ceff1710.0") +# cert with all kinds of subject alt names +ALLSANFILE = data_file("allsans.pem") +IDNSANSFILE = data_file("idnsans.pem") +NOSANFILE = data_file("nosan.pem") +NOSAN_HOSTNAME = 'localhost' + +REMOTE_HOST = "self-signed.pythontest.net" + +EMPTYCERT = data_file("nullcert.pem") +BADCERT = data_file("badcert.pem") +NONEXISTINGCERT = data_file("XXXnonexisting.pem") +BADKEY = data_file("badkey.pem") +NOKIACERT = data_file("nokia.pem") +NULLBYTECERT = data_file("nullbytecert.pem") +TALOS_INVALID_CRLDP = data_file("talos-2019-0758.pem") + +DHFILE = data_file("ffdh3072.pem") +BYTES_DHFILE = os.fsencode(DHFILE) + +# Not defined in all versions of OpenSSL +OP_NO_COMPRESSION = getattr(ssl, "OP_NO_COMPRESSION", 0) +OP_SINGLE_DH_USE = getattr(ssl, "OP_SINGLE_DH_USE", 0) +OP_SINGLE_ECDH_USE = getattr(ssl, "OP_SINGLE_ECDH_USE", 0) +OP_CIPHER_SERVER_PREFERENCE = getattr(ssl, "OP_CIPHER_SERVER_PREFERENCE", 0) +OP_ENABLE_MIDDLEBOX_COMPAT = getattr(ssl, "OP_ENABLE_MIDDLEBOX_COMPAT", 0) + +# Ubuntu has patched OpenSSL and changed behavior of security level 2 +# see https://bugs.python.org/issue41561#msg389003 +def is_ubuntu(): + try: + # Assume that any references of "ubuntu" implies Ubuntu-like distro + # The workaround is not required for 18.04, but doesn't hurt either. + with open("/etc/os-release", encoding="utf-8") as f: + return "ubuntu" in f.read() + except FileNotFoundError: + return False + +if is_ubuntu(): + def seclevel_workaround(*ctxs): + """"Lower security level to '1' and allow all ciphers for TLS 1.0/1""" + for ctx in ctxs: + if ( + hasattr(ctx, "minimum_version") and + ctx.minimum_version <= ssl.TLSVersion.TLSv1_1 and + ctx.security_level > 1 + ): + ctx.set_ciphers("@SECLEVEL=1:ALL") +else: + def seclevel_workaround(*ctxs): + pass + + +def has_tls_protocol(protocol): + """Check if a TLS protocol is available and enabled + + :param protocol: enum ssl._SSLMethod member or name + :return: bool + """ + if isinstance(protocol, str): + assert protocol.startswith('PROTOCOL_') + protocol = getattr(ssl, protocol, None) + if protocol is None: + return False + if protocol in { + ssl.PROTOCOL_TLS, ssl.PROTOCOL_TLS_SERVER, + ssl.PROTOCOL_TLS_CLIENT + }: + # auto-negotiate protocols are always available + return True + name = protocol.name + return has_tls_version(name[len('PROTOCOL_'):]) + + +@functools.lru_cache +def has_tls_version(version): + """Check if a TLS/SSL version is enabled + + :param version: TLS version name or ssl.TLSVersion member + :return: bool + """ + if isinstance(version, str): + version = ssl.TLSVersion.__members__[version] + + # check compile time flags like ssl.HAS_TLSv1_2 + if not getattr(ssl, f'HAS_{version.name}'): + return False + + if IS_OPENSSL_3_0_0 and version < ssl.TLSVersion.TLSv1_2: + # bpo43791: 3.0.0-alpha14 fails with TLSV1_ALERT_INTERNAL_ERROR + return False + + # check runtime and dynamic crypto policy settings. A TLS version may + # be compiled in but disabled by a policy or config option. + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + if ( + hasattr(ctx, 'minimum_version') and + ctx.minimum_version != ssl.TLSVersion.MINIMUM_SUPPORTED and + version < ctx.minimum_version + ): + return False + if ( + hasattr(ctx, 'maximum_version') and + ctx.maximum_version != ssl.TLSVersion.MAXIMUM_SUPPORTED and + version > ctx.maximum_version + ): + return False + + return True + + +def requires_tls_version(version): + """Decorator to skip tests when a required TLS version is not available + + :param version: TLS version name or ssl.TLSVersion member + :return: + """ + def decorator(func): + @functools.wraps(func) + def wrapper(*args, **kw): + if not has_tls_version(version): + raise unittest.SkipTest(f"{version} is not available.") + else: + return func(*args, **kw) + return wrapper + return decorator + + +def handle_error(prefix): + exc_format = ' '.join(traceback.format_exception(sys.exception())) + if support.verbose: + sys.stdout.write(prefix + exc_format) + + +def utc_offset(): #NOTE: ignore issues like #1647654 + # local time = utc time + utc offset + if time.daylight and time.localtime().tm_isdst > 0: + return -time.altzone # seconds + return -time.timezone + + +ignore_deprecation = warnings_helper.ignore_warnings( + category=DeprecationWarning +) + + +def test_wrap_socket(sock, *, + cert_reqs=ssl.CERT_NONE, ca_certs=None, + ciphers=None, certfile=None, keyfile=None, + **kwargs): + if not kwargs.get("server_side"): + kwargs["server_hostname"] = SIGNED_CERTFILE_HOSTNAME + context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + else: + context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + if cert_reqs is not None: + if cert_reqs == ssl.CERT_NONE: + context.check_hostname = False + context.verify_mode = cert_reqs + if ca_certs is not None: + context.load_verify_locations(ca_certs) + if certfile is not None or keyfile is not None: + context.load_cert_chain(certfile, keyfile) + if ciphers is not None: + context.set_ciphers(ciphers) + return context.wrap_socket(sock, **kwargs) + + +USE_SAME_TEST_CONTEXT = False +_TEST_CONTEXT = None + +def testing_context(server_cert=SIGNED_CERTFILE, *, server_chain=True): + """Create context + + client_context, server_context, hostname = testing_context() + """ + global _TEST_CONTEXT + if USE_SAME_TEST_CONTEXT: + if _TEST_CONTEXT is not None: + return _TEST_CONTEXT + + if server_cert == SIGNED_CERTFILE: + hostname = SIGNED_CERTFILE_HOSTNAME + elif server_cert == SIGNED_CERTFILE2: + hostname = SIGNED_CERTFILE2_HOSTNAME + elif server_cert == NOSANFILE: + hostname = NOSAN_HOSTNAME + else: + raise ValueError(server_cert) + + client_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + client_context.load_verify_locations(SIGNING_CA) + + server_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + server_context.load_cert_chain(server_cert) + if server_chain: + server_context.load_verify_locations(SIGNING_CA) + + if USE_SAME_TEST_CONTEXT: + if _TEST_CONTEXT is not None: + _TEST_CONTEXT = client_context, server_context, hostname + + return client_context, server_context, hostname + + +class BasicSocketTests(unittest.TestCase): + + def test_constants(self): + ssl.CERT_NONE + ssl.CERT_OPTIONAL + ssl.CERT_REQUIRED + ssl.OP_CIPHER_SERVER_PREFERENCE + ssl.OP_SINGLE_DH_USE + ssl.OP_SINGLE_ECDH_USE + ssl.OP_NO_COMPRESSION + self.assertEqual(ssl.HAS_SNI, True) + self.assertEqual(ssl.HAS_ECDH, True) + self.assertEqual(ssl.HAS_TLSv1_2, True) + self.assertEqual(ssl.HAS_TLSv1_3, True) + ssl.OP_NO_SSLv2 + ssl.OP_NO_SSLv3 + ssl.OP_NO_TLSv1 + ssl.OP_NO_TLSv1_3 + ssl.OP_NO_TLSv1_1 + ssl.OP_NO_TLSv1_2 + self.assertEqual(ssl.PROTOCOL_TLS, ssl.PROTOCOL_SSLv23) + + def test_options(self): + # gh-106687: SSL options values are unsigned integer (uint64_t) + for name in dir(ssl): + if not name.startswith('OP_'): + continue + with self.subTest(option=name): + value = getattr(ssl, name) + self.assertGreaterEqual(value, 0, f"ssl.{name}") + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_ssl_types(self): + ssl_types = [ + _ssl._SSLContext, + _ssl._SSLSocket, + _ssl.MemoryBIO, + _ssl.Certificate, + _ssl.SSLSession, + _ssl.SSLError, + ] + for ssl_type in ssl_types: + with self.subTest(ssl_type=ssl_type): + with self.assertRaisesRegex(TypeError, "immutable type"): + ssl_type.value = None + support.check_disallow_instantiation(self, _ssl.Certificate) + + def test_private_init(self): + with self.assertRaisesRegex(TypeError, "public constructor"): + with socket.socket() as s: + ssl.SSLSocket(s) + + def test_str_for_enums(self): + # Make sure that the PROTOCOL_* constants have enum-like string + # reprs. + proto = ssl.PROTOCOL_TLS_CLIENT + self.assertEqual(repr(proto), '<_SSLMethod.PROTOCOL_TLS_CLIENT: %r>' % proto.value) + self.assertEqual(str(proto), str(proto.value)) + ctx = ssl.SSLContext(proto) + self.assertIs(ctx.protocol, proto) + + def test_random(self): + v = ssl.RAND_status() + if support.verbose: + sys.stdout.write("\n RAND_status is %d (%s)\n" + % (v, (v and "sufficient randomness") or + "insufficient randomness")) + + if v: + data = ssl.RAND_bytes(16) + self.assertEqual(len(data), 16) + else: + self.assertRaises(ssl.SSLError, ssl.RAND_bytes, 16) + + # negative num is invalid + self.assertRaises(ValueError, ssl.RAND_bytes, -5) + + ssl.RAND_add("this is a random string", 75.0) + ssl.RAND_add(b"this is a random bytes object", 75.0) + ssl.RAND_add(bytearray(b"this is a random bytearray object"), 75.0) + + def test_parse_cert(self): + # note that this uses an 'unofficial' function in _ssl.c, + # provided solely for this test, to exercise the certificate + # parsing code + self.assertEqual( + ssl._ssl._test_decode_cert(CERTFILE), + CERTFILE_INFO + ) + self.assertEqual( + ssl._ssl._test_decode_cert(SIGNED_CERTFILE), + SIGNED_CERTFILE_INFO + ) + + # Issue #13034: the subjectAltName in some certificates + # (notably projects.developer.nokia.com:443) wasn't parsed + p = ssl._ssl._test_decode_cert(NOKIACERT) + if support.verbose: + sys.stdout.write("\n" + pprint.pformat(p) + "\n") + self.assertEqual(p['subjectAltName'], + (('DNS', 'projects.developer.nokia.com'), + ('DNS', 'projects.forum.nokia.com')) + ) + # extra OCSP and AIA fields + self.assertEqual(p['OCSP'], ('http://ocsp.verisign.com',)) + self.assertEqual(p['caIssuers'], + ('http://SVRIntl-G3-aia.verisign.com/SVRIntlG3.cer',)) + self.assertEqual(p['crlDistributionPoints'], + ('http://SVRIntl-G3-crl.verisign.com/SVRIntlG3.crl',)) + + def test_parse_cert_CVE_2019_5010(self): + p = ssl._ssl._test_decode_cert(TALOS_INVALID_CRLDP) + if support.verbose: + sys.stdout.write("\n" + pprint.pformat(p) + "\n") + self.assertEqual( + p, + { + 'issuer': ( + (('countryName', 'UK'),), (('commonName', 'cody-ca'),)), + 'notAfter': 'Jun 14 18:00:58 2028 GMT', + 'notBefore': 'Jun 18 18:00:58 2018 GMT', + 'serialNumber': '02', + 'subject': ((('countryName', 'UK'),), + (('commonName', + 'codenomicon-vm-2.test.lal.cisco.com'),)), + 'subjectAltName': ( + ('DNS', 'codenomicon-vm-2.test.lal.cisco.com'),), + 'version': 3 + } + ) + + def test_parse_cert_CVE_2013_4238(self): + p = ssl._ssl._test_decode_cert(NULLBYTECERT) + if support.verbose: + sys.stdout.write("\n" + pprint.pformat(p) + "\n") + subject = ((('countryName', 'US'),), + (('stateOrProvinceName', 'Oregon'),), + (('localityName', 'Beaverton'),), + (('organizationName', 'Python Software Foundation'),), + (('organizationalUnitName', 'Python Core Development'),), + (('commonName', 'null.python.org\x00example.org'),), + (('emailAddress', 'python-dev@python.org'),)) + self.assertEqual(p['subject'], subject) + self.assertEqual(p['issuer'], subject) + if ssl._OPENSSL_API_VERSION >= (0, 9, 8): + san = (('DNS', 'altnull.python.org\x00example.com'), + ('email', 'null@python.org\x00user@example.org'), + ('URI', 'http://null.python.org\x00http://example.org'), + ('IP Address', '192.0.2.1'), + ('IP Address', '2001:DB8:0:0:0:0:0:1')) + else: + # OpenSSL 0.9.7 doesn't support IPv6 addresses in subjectAltName + san = (('DNS', 'altnull.python.org\x00example.com'), + ('email', 'null@python.org\x00user@example.org'), + ('URI', 'http://null.python.org\x00http://example.org'), + ('IP Address', '192.0.2.1'), + ('IP Address', '<invalid>')) + + self.assertEqual(p['subjectAltName'], san) + + def test_parse_all_sans(self): + p = ssl._ssl._test_decode_cert(ALLSANFILE) + self.assertEqual(p['subjectAltName'], + ( + ('DNS', 'allsans'), + ('othername', '<unsupported>'), + ('othername', '<unsupported>'), + ('email', 'user@example.org'), + ('DNS', 'www.example.org'), + ('DirName', + ((('countryName', 'XY'),), + (('localityName', 'Castle Anthrax'),), + (('organizationName', 'Python Software Foundation'),), + (('commonName', 'dirname example'),))), + ('URI', 'https://www.python.org/'), + ('IP Address', '127.0.0.1'), + ('IP Address', '0:0:0:0:0:0:0:1'), + ('Registered ID', '1.2.3.4.5') + ) + ) + + def test_DER_to_PEM(self): + with open(CAFILE_CACERT, 'r') as f: + pem = f.read() + d1 = ssl.PEM_cert_to_DER_cert(pem) + p2 = ssl.DER_cert_to_PEM_cert(d1) + d2 = ssl.PEM_cert_to_DER_cert(p2) + self.assertEqual(d1, d2) + if not p2.startswith(ssl.PEM_HEADER + '\n'): + self.fail("DER-to-PEM didn't include correct header:\n%r\n" % p2) + if not p2.endswith('\n' + ssl.PEM_FOOTER + '\n'): + self.fail("DER-to-PEM didn't include correct footer:\n%r\n" % p2) + + def test_openssl_version(self): + n = ssl.OPENSSL_VERSION_NUMBER + t = ssl.OPENSSL_VERSION_INFO + s = ssl.OPENSSL_VERSION + self.assertIsInstance(n, int) + self.assertIsInstance(t, tuple) + self.assertIsInstance(s, str) + # Some sanity checks follow + # >= 1.1.1 + self.assertGreaterEqual(n, 0x10101000) + # < 4.0 + self.assertLess(n, 0x40000000) + major, minor, fix, patch, status = t + self.assertGreaterEqual(major, 1) + self.assertLess(major, 4) + self.assertGreaterEqual(minor, 0) + self.assertLess(minor, 256) + self.assertGreaterEqual(fix, 0) + self.assertLess(fix, 256) + self.assertGreaterEqual(patch, 0) + self.assertLessEqual(patch, 63) + self.assertGreaterEqual(status, 0) + self.assertLessEqual(status, 15) + + libressl_ver = f"LibreSSL {major:d}" + if major >= 3: + # 3.x uses 0xMNN00PP0L + openssl_ver = f"OpenSSL {major:d}.{minor:d}.{patch:d}" + else: + openssl_ver = f"OpenSSL {major:d}.{minor:d}.{fix:d}" + self.assertTrue( + s.startswith((openssl_ver, libressl_ver, "AWS-LC")), + (s, t, hex(n)) + ) + + @support.cpython_only + def test_refcycle(self): + # Issue #7943: an SSL object doesn't create reference cycles with + # itself. + s = socket.socket(socket.AF_INET) + ss = test_wrap_socket(s) + wr = weakref.ref(ss) + with warnings_helper.check_warnings(("", ResourceWarning)): + del ss + self.assertEqual(wr(), None) + + def test_wrapped_unconnected(self): + # Methods on an unconnected SSLSocket propagate the original + # OSError raise by the underlying socket object. + s = socket.socket(socket.AF_INET) + with test_wrap_socket(s) as ss: + self.assertRaises(OSError, ss.recv, 1) + self.assertRaises(OSError, ss.recv_into, bytearray(b'x')) + self.assertRaises(OSError, ss.recvfrom, 1) + self.assertRaises(OSError, ss.recvfrom_into, bytearray(b'x'), 1) + self.assertRaises(OSError, ss.send, b'x') + self.assertRaises(OSError, ss.sendto, b'x', ('0.0.0.0', 0)) + self.assertRaises(NotImplementedError, ss.dup) + self.assertRaises(NotImplementedError, ss.sendmsg, + [b'x'], (), 0, ('0.0.0.0', 0)) + self.assertRaises(NotImplementedError, ss.recvmsg, 100) + self.assertRaises(NotImplementedError, ss.recvmsg_into, + [bytearray(100)]) + + def test_timeout(self): + # Issue #8524: when creating an SSL socket, the timeout of the + # original socket should be retained. + for timeout in (None, 0.0, 5.0): + s = socket.socket(socket.AF_INET) + s.settimeout(timeout) + with test_wrap_socket(s) as ss: + self.assertEqual(timeout, ss.gettimeout()) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_openssl111_deprecations(self): + options = [ + ssl.OP_NO_TLSv1, + ssl.OP_NO_TLSv1_1, + ssl.OP_NO_TLSv1_2, + ssl.OP_NO_TLSv1_3 + ] + protocols = [ + ssl.PROTOCOL_TLSv1, + ssl.PROTOCOL_TLSv1_1, + ssl.PROTOCOL_TLSv1_2, + ssl.PROTOCOL_TLS + ] + versions = [ + ssl.TLSVersion.SSLv3, + ssl.TLSVersion.TLSv1, + ssl.TLSVersion.TLSv1_1, + ] + + for option in options: + with self.subTest(option=option): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + with self.assertWarns(DeprecationWarning) as cm: + ctx.options |= option + self.assertEqual( + 'ssl.OP_NO_SSL*/ssl.OP_NO_TLS* options are deprecated', + str(cm.warning) + ) + + for protocol in protocols: + if not has_tls_protocol(protocol): + continue + with self.subTest(protocol=protocol): + with self.assertWarns(DeprecationWarning) as cm: + ssl.SSLContext(protocol) + self.assertEqual( + f'ssl.{protocol.name} is deprecated', + str(cm.warning) + ) + + for version in versions: + if not has_tls_version(version): + continue + with self.subTest(version=version): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + with self.assertWarns(DeprecationWarning) as cm: + ctx.minimum_version = version + version_text = '%s.%s' % (version.__class__.__name__, version.name) + self.assertEqual( + f'ssl.{version_text} is deprecated', + str(cm.warning) + ) + + def bad_cert_test(self, certfile): + """Check that trying to use the given client certificate fails""" + certfile = os.path.join(os.path.dirname(__file__) or os.curdir, + "certdata", certfile) + sock = socket.socket() + self.addCleanup(sock.close) + with self.assertRaises(ssl.SSLError): + test_wrap_socket(sock, + certfile=certfile) + + def test_empty_cert(self): + """Wrapping with an empty cert file""" + self.bad_cert_test("nullcert.pem") + + def test_malformed_cert(self): + """Wrapping with a badly formatted certificate (syntax error)""" + self.bad_cert_test("badcert.pem") + + def test_malformed_key(self): + """Wrapping with a badly formatted key (syntax error)""" + self.bad_cert_test("badkey.pem") + + def test_server_side(self): + # server_hostname doesn't work for server sockets + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + with socket.socket() as sock: + self.assertRaises(ValueError, ctx.wrap_socket, sock, True, + server_hostname="some.hostname") + + def test_unknown_channel_binding(self): + # should raise ValueError for unknown type + s = socket.create_server(('127.0.0.1', 0)) + c = socket.socket(socket.AF_INET) + c.connect(s.getsockname()) + with test_wrap_socket(c, do_handshake_on_connect=False) as ss: + with self.assertRaises(ValueError): + ss.get_channel_binding("unknown-type") + s.close() + + @unittest.skipUnless("tls-unique" in ssl.CHANNEL_BINDING_TYPES, + "'tls-unique' channel binding not available") + def test_tls_unique_channel_binding(self): + # unconnected should return None for known type + s = socket.socket(socket.AF_INET) + with test_wrap_socket(s) as ss: + self.assertIsNone(ss.get_channel_binding("tls-unique")) + # the same for server-side + s = socket.socket(socket.AF_INET) + with test_wrap_socket(s, server_side=True, certfile=CERTFILE) as ss: + self.assertIsNone(ss.get_channel_binding("tls-unique")) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_dealloc_warn(self): + ss = test_wrap_socket(socket.socket(socket.AF_INET)) + r = repr(ss) + with self.assertWarns(ResourceWarning) as cm: + ss = None + support.gc_collect() + self.assertIn(r, str(cm.warning.args[0])) + + def test_get_default_verify_paths(self): + paths = ssl.get_default_verify_paths() + self.assertEqual(len(paths), 6) + self.assertIsInstance(paths, ssl.DefaultVerifyPaths) + + with os_helper.EnvironmentVarGuard() as env: + env["SSL_CERT_DIR"] = CAPATH + env["SSL_CERT_FILE"] = CERTFILE + paths = ssl.get_default_verify_paths() + self.assertEqual(paths.cafile, CERTFILE) + self.assertEqual(paths.capath, CAPATH) + + @unittest.skipUnless(sys.platform == "win32", "Windows specific") + def test_enum_certificates(self): + self.assertTrue(ssl.enum_certificates("CA")) + self.assertTrue(ssl.enum_certificates("ROOT")) + + self.assertRaises(TypeError, ssl.enum_certificates) + self.assertRaises(WindowsError, ssl.enum_certificates, "") + + trust_oids = set() + for storename in ("CA", "ROOT"): + store = ssl.enum_certificates(storename) + self.assertIsInstance(store, list) + for element in store: + self.assertIsInstance(element, tuple) + self.assertEqual(len(element), 3) + cert, enc, trust = element + self.assertIsInstance(cert, bytes) + self.assertIn(enc, {"x509_asn", "pkcs_7_asn"}) + self.assertIsInstance(trust, (frozenset, set, bool)) + if isinstance(trust, (frozenset, set)): + trust_oids.update(trust) + + serverAuth = "1.3.6.1.5.5.7.3.1" + self.assertIn(serverAuth, trust_oids) + + @unittest.skipUnless(sys.platform == "win32", "Windows specific") + def test_enum_crls(self): + self.assertTrue(ssl.enum_crls("CA")) + self.assertRaises(TypeError, ssl.enum_crls) + self.assertRaises(WindowsError, ssl.enum_crls, "") + + crls = ssl.enum_crls("CA") + self.assertIsInstance(crls, list) + for element in crls: + self.assertIsInstance(element, tuple) + self.assertEqual(len(element), 2) + self.assertIsInstance(element[0], bytes) + self.assertIn(element[1], {"x509_asn", "pkcs_7_asn"}) + + + def test_asn1object(self): + expected = (129, 'serverAuth', 'TLS Web Server Authentication', + '1.3.6.1.5.5.7.3.1') + + val = ssl._ASN1Object('1.3.6.1.5.5.7.3.1') + self.assertEqual(val, expected) + self.assertEqual(val.nid, 129) + self.assertEqual(val.shortname, 'serverAuth') + self.assertEqual(val.longname, 'TLS Web Server Authentication') + self.assertEqual(val.oid, '1.3.6.1.5.5.7.3.1') + self.assertIsInstance(val, ssl._ASN1Object) + self.assertRaises(ValueError, ssl._ASN1Object, 'serverAuth') + + val = ssl._ASN1Object.fromnid(129) + self.assertEqual(val, expected) + self.assertIsInstance(val, ssl._ASN1Object) + self.assertRaises(ValueError, ssl._ASN1Object.fromnid, -1) + with self.assertRaisesRegex(ValueError, "unknown NID 100000"): + ssl._ASN1Object.fromnid(100000) + for i in range(1000): + try: + obj = ssl._ASN1Object.fromnid(i) + except ValueError: + pass + else: + self.assertIsInstance(obj.nid, int) + self.assertIsInstance(obj.shortname, str) + self.assertIsInstance(obj.longname, str) + self.assertIsInstance(obj.oid, (str, type(None))) + + val = ssl._ASN1Object.fromname('TLS Web Server Authentication') + self.assertEqual(val, expected) + self.assertIsInstance(val, ssl._ASN1Object) + self.assertEqual(ssl._ASN1Object.fromname('serverAuth'), expected) + self.assertEqual(ssl._ASN1Object.fromname('1.3.6.1.5.5.7.3.1'), + expected) + with self.assertRaisesRegex(ValueError, "unknown object 'serverauth'"): + ssl._ASN1Object.fromname('serverauth') + + def test_purpose_enum(self): + val = ssl._ASN1Object('1.3.6.1.5.5.7.3.1') + self.assertIsInstance(ssl.Purpose.SERVER_AUTH, ssl._ASN1Object) + self.assertEqual(ssl.Purpose.SERVER_AUTH, val) + self.assertEqual(ssl.Purpose.SERVER_AUTH.nid, 129) + self.assertEqual(ssl.Purpose.SERVER_AUTH.shortname, 'serverAuth') + self.assertEqual(ssl.Purpose.SERVER_AUTH.oid, + '1.3.6.1.5.5.7.3.1') + + val = ssl._ASN1Object('1.3.6.1.5.5.7.3.2') + self.assertIsInstance(ssl.Purpose.CLIENT_AUTH, ssl._ASN1Object) + self.assertEqual(ssl.Purpose.CLIENT_AUTH, val) + self.assertEqual(ssl.Purpose.CLIENT_AUTH.nid, 130) + self.assertEqual(ssl.Purpose.CLIENT_AUTH.shortname, 'clientAuth') + self.assertEqual(ssl.Purpose.CLIENT_AUTH.oid, + '1.3.6.1.5.5.7.3.2') + + def test_unsupported_dtls(self): + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + self.addCleanup(s.close) + with self.assertRaises(NotImplementedError) as cx: + test_wrap_socket(s, cert_reqs=ssl.CERT_NONE) + self.assertEqual(str(cx.exception), "only stream sockets are supported") + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + with self.assertRaises(NotImplementedError) as cx: + ctx.wrap_socket(s) + self.assertEqual(str(cx.exception), "only stream sockets are supported") + + def cert_time_ok(self, timestring, timestamp): + self.assertEqual(ssl.cert_time_to_seconds(timestring), timestamp) + + def cert_time_fail(self, timestring): + with self.assertRaises(ValueError): + ssl.cert_time_to_seconds(timestring) + + @unittest.skipUnless(utc_offset(), + 'local time needs to be different from UTC') + def test_cert_time_to_seconds_timezone(self): + # Issue #19940: ssl.cert_time_to_seconds() returns wrong + # results if local timezone is not UTC + self.cert_time_ok("May 9 00:00:00 2007 GMT", 1178668800.0) + self.cert_time_ok("Jan 5 09:34:43 2018 GMT", 1515144883.0) + + def test_cert_time_to_seconds(self): + timestring = "Jan 5 09:34:43 2018 GMT" + ts = 1515144883.0 + self.cert_time_ok(timestring, ts) + # accept keyword parameter, assert its name + self.assertEqual(ssl.cert_time_to_seconds(cert_time=timestring), ts) + # accept both %e and %d (space or zero generated by strftime) + self.cert_time_ok("Jan 05 09:34:43 2018 GMT", ts) + # case-insensitive + self.cert_time_ok("JaN 5 09:34:43 2018 GmT", ts) + self.cert_time_fail("Jan 5 09:34 2018 GMT") # no seconds + self.cert_time_fail("Jan 5 09:34:43 2018") # no GMT + self.cert_time_fail("Jan 5 09:34:43 2018 UTC") # not GMT timezone + self.cert_time_fail("Jan 35 09:34:43 2018 GMT") # invalid day + self.cert_time_fail("Jon 5 09:34:43 2018 GMT") # invalid month + self.cert_time_fail("Jan 5 24:00:00 2018 GMT") # invalid hour + self.cert_time_fail("Jan 5 09:60:43 2018 GMT") # invalid minute + + newyear_ts = 1230768000.0 + # leap seconds + self.cert_time_ok("Dec 31 23:59:60 2008 GMT", newyear_ts) + # same timestamp + self.cert_time_ok("Jan 1 00:00:00 2009 GMT", newyear_ts) + + self.cert_time_ok("Jan 5 09:34:59 2018 GMT", 1515144899) + # allow 60th second (even if it is not a leap second) + self.cert_time_ok("Jan 5 09:34:60 2018 GMT", 1515144900) + # allow 2nd leap second for compatibility with time.strptime() + self.cert_time_ok("Jan 5 09:34:61 2018 GMT", 1515144901) + self.cert_time_fail("Jan 5 09:34:62 2018 GMT") # invalid seconds + + # no special treatment for the special value: + # 99991231235959Z (rfc 5280) + self.cert_time_ok("Dec 31 23:59:59 9999 GMT", 253402300799.0) + + @support.run_with_locale('LC_ALL', '') + def test_cert_time_to_seconds_locale(self): + # `cert_time_to_seconds()` should be locale independent + + def local_february_name(): + return time.strftime('%b', (1, 2, 3, 4, 5, 6, 0, 0, 0)) + + if local_february_name().lower() == 'feb': + self.skipTest("locale-specific month name needs to be " + "different from C locale") + + # locale-independent + self.cert_time_ok("Feb 9 00:00:00 2007 GMT", 1170979200.0) + self.cert_time_fail(local_february_name() + " 9 00:00:00 2007 GMT") + + def test_connect_ex_error(self): + server = socket.socket(socket.AF_INET) + self.addCleanup(server.close) + port = socket_helper.bind_port(server) # Reserve port but don't listen + s = test_wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_REQUIRED) + self.addCleanup(s.close) + rc = s.connect_ex((HOST, port)) + # Issue #19919: Windows machines or VMs hosted on Windows + # machines sometimes return EWOULDBLOCK. + errors = ( + errno.ECONNREFUSED, errno.EHOSTUNREACH, errno.ETIMEDOUT, + errno.EWOULDBLOCK, + ) + self.assertIn(rc, errors) + + def test_read_write_zero(self): + # empty reads and writes now work, bpo-42854, bpo-31711 + client_context, server_context, hostname = testing_context() + server = ThreadedEchoServer(context=server_context) + with server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + s.connect((HOST, server.port)) + self.assertEqual(s.recv(0), b"") + self.assertEqual(s.send(b""), 0) + + +class ContextTests(unittest.TestCase): + + def test_constructor(self): + for protocol in PROTOCOLS: + if has_tls_protocol(protocol): + with warnings_helper.check_warnings(): + ctx = ssl.SSLContext(protocol) + self.assertEqual(ctx.protocol, protocol) + with warnings_helper.check_warnings(): + ctx = ssl.SSLContext() + self.assertEqual(ctx.protocol, ssl.PROTOCOL_TLS) + self.assertRaises(ValueError, ssl.SSLContext, -1) + self.assertRaises(ValueError, ssl.SSLContext, 42) + + def test_ciphers(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ctx.set_ciphers("ALL") + ctx.set_ciphers("DEFAULT") + with self.assertRaisesRegex(ssl.SSLError, "No cipher can be selected"): + ctx.set_ciphers("^$:,;?*'dorothyx") + + @unittest.skipUnless(PY_SSL_DEFAULT_CIPHERS == 1, + "Test applies only to Python default ciphers") + def test_python_ciphers(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ciphers = ctx.get_ciphers() + for suite in ciphers: + name = suite['name'] + self.assertNotIn("PSK", name) + self.assertNotIn("SRP", name) + self.assertNotIn("MD5", name) + self.assertNotIn("RC4", name) + self.assertNotIn("3DES", name) + + def test_get_ciphers(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ctx.set_ciphers('AESGCM') + names = set(d['name'] for d in ctx.get_ciphers()) + expected = { + 'AES128-GCM-SHA256', + 'ECDHE-ECDSA-AES128-GCM-SHA256', + 'ECDHE-RSA-AES128-GCM-SHA256', + 'DHE-RSA-AES128-GCM-SHA256', + 'AES256-GCM-SHA384', + 'ECDHE-ECDSA-AES256-GCM-SHA384', + 'ECDHE-RSA-AES256-GCM-SHA384', + 'DHE-RSA-AES256-GCM-SHA384', + } + intersection = names.intersection(expected) + self.assertGreaterEqual( + len(intersection), 2, f"\ngot: {sorted(names)}\nexpected: {sorted(expected)}" + ) + + def test_options(self): + # Test default SSLContext options + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + # OP_ALL | OP_NO_SSLv2 | OP_NO_SSLv3 is the default value + default = (ssl.OP_ALL | ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3) + # SSLContext also enables these by default + default |= (OP_NO_COMPRESSION | OP_CIPHER_SERVER_PREFERENCE | + OP_SINGLE_DH_USE | OP_SINGLE_ECDH_USE | + OP_ENABLE_MIDDLEBOX_COMPAT) + self.assertEqual(default, ctx.options) + + # disallow TLSv1 + with warnings_helper.check_warnings(): + ctx.options |= ssl.OP_NO_TLSv1 + self.assertEqual(default | ssl.OP_NO_TLSv1, ctx.options) + + # allow TLSv1 + with warnings_helper.check_warnings(): + ctx.options = (ctx.options & ~ssl.OP_NO_TLSv1) + self.assertEqual(default, ctx.options) + + # clear all options + ctx.options = 0 + # Ubuntu has OP_NO_SSLv3 forced on by default + self.assertEqual(0, ctx.options & ~ssl.OP_NO_SSLv3) + + # invalid options + with self.assertRaises(OverflowError): + ctx.options = -1 + with self.assertRaises(OverflowError): + ctx.options = 2 ** 100 + with self.assertRaises(TypeError): + ctx.options = "abc" + + def test_verify_mode_protocol(self): + with warnings_helper.check_warnings(): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS) + # Default value + self.assertEqual(ctx.verify_mode, ssl.CERT_NONE) + ctx.verify_mode = ssl.CERT_OPTIONAL + self.assertEqual(ctx.verify_mode, ssl.CERT_OPTIONAL) + ctx.verify_mode = ssl.CERT_REQUIRED + self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED) + ctx.verify_mode = ssl.CERT_NONE + self.assertEqual(ctx.verify_mode, ssl.CERT_NONE) + with self.assertRaises(TypeError): + ctx.verify_mode = None + with self.assertRaises(ValueError): + ctx.verify_mode = 42 + + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + self.assertEqual(ctx.verify_mode, ssl.CERT_NONE) + self.assertFalse(ctx.check_hostname) + + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED) + self.assertTrue(ctx.check_hostname) + + def test_hostname_checks_common_name(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + self.assertTrue(ctx.hostname_checks_common_name) + if ssl.HAS_NEVER_CHECK_COMMON_NAME: + ctx.hostname_checks_common_name = True + self.assertTrue(ctx.hostname_checks_common_name) + ctx.hostname_checks_common_name = False + self.assertFalse(ctx.hostname_checks_common_name) + ctx.hostname_checks_common_name = True + self.assertTrue(ctx.hostname_checks_common_name) + else: + with self.assertRaises(AttributeError): + ctx.hostname_checks_common_name = True + + @unittest.expectedFailure # TODO: RUSTPYTHON + @ignore_deprecation + def test_min_max_version(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + # OpenSSL default is MINIMUM_SUPPORTED, however some vendors like + # Fedora override the setting to TLS 1.0. + minimum_range = { + # stock OpenSSL + ssl.TLSVersion.MINIMUM_SUPPORTED, + # Fedora 29 uses TLS 1.0 by default + ssl.TLSVersion.TLSv1, + # RHEL 8 uses TLS 1.2 by default + ssl.TLSVersion.TLSv1_2 + } + maximum_range = { + # stock OpenSSL + ssl.TLSVersion.MAXIMUM_SUPPORTED, + # Fedora 32 uses TLS 1.3 by default + ssl.TLSVersion.TLSv1_3 + } + + self.assertIn( + ctx.minimum_version, minimum_range + ) + self.assertIn( + ctx.maximum_version, maximum_range + ) + + ctx.minimum_version = ssl.TLSVersion.TLSv1_1 + ctx.maximum_version = ssl.TLSVersion.TLSv1_2 + self.assertEqual( + ctx.minimum_version, ssl.TLSVersion.TLSv1_1 + ) + self.assertEqual( + ctx.maximum_version, ssl.TLSVersion.TLSv1_2 + ) + + ctx.minimum_version = ssl.TLSVersion.MINIMUM_SUPPORTED + ctx.maximum_version = ssl.TLSVersion.TLSv1 + self.assertEqual( + ctx.minimum_version, ssl.TLSVersion.MINIMUM_SUPPORTED + ) + self.assertEqual( + ctx.maximum_version, ssl.TLSVersion.TLSv1 + ) + + ctx.maximum_version = ssl.TLSVersion.MAXIMUM_SUPPORTED + self.assertEqual( + ctx.maximum_version, ssl.TLSVersion.MAXIMUM_SUPPORTED + ) + + ctx.maximum_version = ssl.TLSVersion.MINIMUM_SUPPORTED + self.assertIn( + ctx.maximum_version, + {ssl.TLSVersion.TLSv1, ssl.TLSVersion.TLSv1_1, ssl.TLSVersion.SSLv3} + ) + + ctx.minimum_version = ssl.TLSVersion.MAXIMUM_SUPPORTED + self.assertIn( + ctx.minimum_version, + {ssl.TLSVersion.TLSv1_2, ssl.TLSVersion.TLSv1_3} + ) + + with self.assertRaises(ValueError): + ctx.minimum_version = 42 + + if has_tls_protocol(ssl.PROTOCOL_TLSv1_1): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLSv1_1) + + self.assertIn( + ctx.minimum_version, minimum_range + ) + self.assertEqual( + ctx.maximum_version, ssl.TLSVersion.MAXIMUM_SUPPORTED + ) + with self.assertRaises(ValueError): + ctx.minimum_version = ssl.TLSVersion.MINIMUM_SUPPORTED + with self.assertRaises(ValueError): + ctx.maximum_version = ssl.TLSVersion.TLSv1 + + @unittest.skipUnless( + hasattr(ssl.SSLContext, 'security_level'), + "requires OpenSSL >= 1.1.0" + ) + def test_security_level(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + # The default security callback allows for levels between 0-5 + # with OpenSSL defaulting to 1, however some vendors override the + # default value (e.g. Debian defaults to 2) + security_level_range = { + 0, + 1, # OpenSSL default + 2, # Debian + 3, + 4, + 5, + } + self.assertIn(ctx.security_level, security_level_range) + + def test_verify_flags(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + # default value + tf = getattr(ssl, "VERIFY_X509_TRUSTED_FIRST", 0) + self.assertEqual(ctx.verify_flags, ssl.VERIFY_DEFAULT | tf) + ctx.verify_flags = ssl.VERIFY_CRL_CHECK_LEAF + self.assertEqual(ctx.verify_flags, ssl.VERIFY_CRL_CHECK_LEAF) + ctx.verify_flags = ssl.VERIFY_CRL_CHECK_CHAIN + self.assertEqual(ctx.verify_flags, ssl.VERIFY_CRL_CHECK_CHAIN) + ctx.verify_flags = ssl.VERIFY_DEFAULT + self.assertEqual(ctx.verify_flags, ssl.VERIFY_DEFAULT) + ctx.verify_flags = ssl.VERIFY_ALLOW_PROXY_CERTS + self.assertEqual(ctx.verify_flags, ssl.VERIFY_ALLOW_PROXY_CERTS) + # supports any value + ctx.verify_flags = ssl.VERIFY_CRL_CHECK_LEAF | ssl.VERIFY_X509_STRICT + self.assertEqual(ctx.verify_flags, + ssl.VERIFY_CRL_CHECK_LEAF | ssl.VERIFY_X509_STRICT) + with self.assertRaises(TypeError): + ctx.verify_flags = None + + def test_load_cert_chain(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + # Combined key and cert in a single file + ctx.load_cert_chain(CERTFILE, keyfile=None) + ctx.load_cert_chain(CERTFILE, keyfile=CERTFILE) + self.assertRaises(TypeError, ctx.load_cert_chain, keyfile=CERTFILE) + with self.assertRaises(OSError) as cm: + ctx.load_cert_chain(NONEXISTINGCERT) + self.assertEqual(cm.exception.errno, errno.ENOENT) + with self.assertRaisesRegex(ssl.SSLError, "PEM (lib|routines)"): + ctx.load_cert_chain(BADCERT) + with self.assertRaisesRegex(ssl.SSLError, "PEM (lib|routines)"): + ctx.load_cert_chain(EMPTYCERT) + # Separate key and cert + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + ctx.load_cert_chain(ONLYCERT, ONLYKEY) + ctx.load_cert_chain(certfile=ONLYCERT, keyfile=ONLYKEY) + ctx.load_cert_chain(certfile=BYTES_ONLYCERT, keyfile=BYTES_ONLYKEY) + with self.assertRaisesRegex(ssl.SSLError, "PEM (lib|routines)"): + ctx.load_cert_chain(ONLYCERT) + with self.assertRaisesRegex(ssl.SSLError, "PEM (lib|routines)"): + ctx.load_cert_chain(ONLYKEY) + with self.assertRaisesRegex(ssl.SSLError, "PEM (lib|routines)"): + ctx.load_cert_chain(certfile=ONLYKEY, keyfile=ONLYCERT) + # Mismatching key and cert + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + # Allow for flexible libssl error messages. + regex = re.compile(r"""( + key values mismatch # OpenSSL + | + KEY_VALUES_MISMATCH # AWS-LC + )""", re.X) + with self.assertRaisesRegex(ssl.SSLError, regex): + ctx.load_cert_chain(CAFILE_CACERT, ONLYKEY) + # Password protected key and cert + ctx.load_cert_chain(CERTFILE_PROTECTED, password=KEY_PASSWORD) + ctx.load_cert_chain(CERTFILE_PROTECTED, password=KEY_PASSWORD.encode()) + ctx.load_cert_chain(CERTFILE_PROTECTED, + password=bytearray(KEY_PASSWORD.encode())) + ctx.load_cert_chain(ONLYCERT, ONLYKEY_PROTECTED, KEY_PASSWORD) + ctx.load_cert_chain(ONLYCERT, ONLYKEY_PROTECTED, KEY_PASSWORD.encode()) + ctx.load_cert_chain(ONLYCERT, ONLYKEY_PROTECTED, + bytearray(KEY_PASSWORD.encode())) + with self.assertRaisesRegex(TypeError, "should be a string"): + ctx.load_cert_chain(CERTFILE_PROTECTED, password=True) + with self.assertRaises(ssl.SSLError): + ctx.load_cert_chain(CERTFILE_PROTECTED, password="badpass") + with self.assertRaisesRegex(ValueError, "cannot be longer"): + # openssl has a fixed limit on the password buffer. + # PEM_BUFSIZE is generally set to 1kb. + # Return a string larger than this. + ctx.load_cert_chain(CERTFILE_PROTECTED, password=b'a' * 102400) + # Password callback + def getpass_unicode(): + return KEY_PASSWORD + def getpass_bytes(): + return KEY_PASSWORD.encode() + def getpass_bytearray(): + return bytearray(KEY_PASSWORD.encode()) + def getpass_badpass(): + return "badpass" + def getpass_huge(): + return b'a' * (1024 * 1024) + def getpass_bad_type(): + return 9 + def getpass_exception(): + raise Exception('getpass error') + class GetPassCallable: + def __call__(self): + return KEY_PASSWORD + def getpass(self): + return KEY_PASSWORD + ctx.load_cert_chain(CERTFILE_PROTECTED, password=getpass_unicode) + ctx.load_cert_chain(CERTFILE_PROTECTED, password=getpass_bytes) + ctx.load_cert_chain(CERTFILE_PROTECTED, password=getpass_bytearray) + ctx.load_cert_chain(CERTFILE_PROTECTED, password=GetPassCallable()) + ctx.load_cert_chain(CERTFILE_PROTECTED, + password=GetPassCallable().getpass) + with self.assertRaises(ssl.SSLError): + ctx.load_cert_chain(CERTFILE_PROTECTED, password=getpass_badpass) + with self.assertRaisesRegex(ValueError, "cannot be longer"): + ctx.load_cert_chain(CERTFILE_PROTECTED, password=getpass_huge) + with self.assertRaisesRegex(TypeError, "must return a string"): + ctx.load_cert_chain(CERTFILE_PROTECTED, password=getpass_bad_type) + with self.assertRaisesRegex(Exception, "getpass error"): + ctx.load_cert_chain(CERTFILE_PROTECTED, password=getpass_exception) + # Make sure the password function isn't called if it isn't needed + ctx.load_cert_chain(CERTFILE, password=getpass_exception) + + @threading_helper.requires_working_threading() + def test_load_cert_chain_thread_safety(self): + # gh-134698: _ssl detaches the thread state (and as such, + # releases the GIL and critical sections) around expensive + # OpenSSL calls. Unfortunately, OpenSSL structures aren't + # thread-safe, so executing these calls concurrently led + # to crashes. + ctx = ssl.create_default_context() + + def race(): + ctx.load_cert_chain(CERTFILE) + + threads = [threading.Thread(target=race) for _ in range(8)] + with threading_helper.catch_threading_exception() as cm: + with threading_helper.start_threads(threads): + pass + + self.assertIsNone(cm.exc_value) + + def test_load_verify_locations(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + ctx.load_verify_locations(CERTFILE) + ctx.load_verify_locations(cafile=CERTFILE, capath=None) + ctx.load_verify_locations(BYTES_CERTFILE) + ctx.load_verify_locations(cafile=BYTES_CERTFILE, capath=None) + self.assertRaises(TypeError, ctx.load_verify_locations) + self.assertRaises(TypeError, ctx.load_verify_locations, None, None, None) + with self.assertRaises(OSError) as cm: + ctx.load_verify_locations(NONEXISTINGCERT) + self.assertEqual(cm.exception.errno, errno.ENOENT) + with self.assertRaisesRegex(ssl.SSLError, "PEM (lib|routines)"): + ctx.load_verify_locations(BADCERT) + ctx.load_verify_locations(CERTFILE, CAPATH) + ctx.load_verify_locations(CERTFILE, capath=BYTES_CAPATH) + + # Issue #10989: crash if the second argument type is invalid + self.assertRaises(TypeError, ctx.load_verify_locations, None, True) + + def test_load_verify_cadata(self): + # test cadata + with open(CAFILE_CACERT) as f: + cacert_pem = f.read() + cacert_der = ssl.PEM_cert_to_DER_cert(cacert_pem) + with open(CAFILE_NEURONIO) as f: + neuronio_pem = f.read() + neuronio_der = ssl.PEM_cert_to_DER_cert(neuronio_pem) + + # test PEM + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + self.assertEqual(ctx.cert_store_stats()["x509_ca"], 0) + ctx.load_verify_locations(cadata=cacert_pem) + self.assertEqual(ctx.cert_store_stats()["x509_ca"], 1) + ctx.load_verify_locations(cadata=neuronio_pem) + self.assertEqual(ctx.cert_store_stats()["x509_ca"], 2) + # cert already in hash table + ctx.load_verify_locations(cadata=neuronio_pem) + self.assertEqual(ctx.cert_store_stats()["x509_ca"], 2) + + # combined + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + combined = "\n".join((cacert_pem, neuronio_pem)) + ctx.load_verify_locations(cadata=combined) + self.assertEqual(ctx.cert_store_stats()["x509_ca"], 2) + + # with junk around the certs + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + combined = ["head", cacert_pem, "other", neuronio_pem, "again", + neuronio_pem, "tail"] + ctx.load_verify_locations(cadata="\n".join(combined)) + self.assertEqual(ctx.cert_store_stats()["x509_ca"], 2) + + # test DER + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ctx.load_verify_locations(cadata=cacert_der) + ctx.load_verify_locations(cadata=neuronio_der) + self.assertEqual(ctx.cert_store_stats()["x509_ca"], 2) + # cert already in hash table + ctx.load_verify_locations(cadata=cacert_der) + self.assertEqual(ctx.cert_store_stats()["x509_ca"], 2) + + # combined + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + combined = b"".join((cacert_der, neuronio_der)) + ctx.load_verify_locations(cadata=combined) + self.assertEqual(ctx.cert_store_stats()["x509_ca"], 2) + + # error cases + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + self.assertRaises(TypeError, ctx.load_verify_locations, cadata=object) + + with self.assertRaisesRegex( + ssl.SSLError, + "no start line: cadata does not contain a certificate" + ): + ctx.load_verify_locations(cadata="broken") + with self.assertRaisesRegex( + ssl.SSLError, + "not enough data: cadata does not contain a certificate" + ): + ctx.load_verify_locations(cadata=b"broken") + with self.assertRaises(ssl.SSLError): + ctx.load_verify_locations(cadata=cacert_der + b"A") + + def test_load_dh_params(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + try: + ctx.load_dh_params(DHFILE) + except RuntimeError: + if Py_DEBUG_WIN32: + self.skipTest("not supported on Win32 debug build") + raise + if os.name != 'nt': + ctx.load_dh_params(BYTES_DHFILE) + self.assertRaises(TypeError, ctx.load_dh_params) + self.assertRaises(TypeError, ctx.load_dh_params, None) + with self.assertRaises(FileNotFoundError) as cm: + ctx.load_dh_params(NONEXISTINGCERT) + self.assertEqual(cm.exception.errno, errno.ENOENT) + with self.assertRaises(ssl.SSLError) as cm: + ctx.load_dh_params(CERTFILE) + + def test_session_stats(self): + for proto in {ssl.PROTOCOL_TLS_CLIENT, ssl.PROTOCOL_TLS_SERVER}: + ctx = ssl.SSLContext(proto) + self.assertEqual(ctx.session_stats(), { + 'number': 0, + 'connect': 0, + 'connect_good': 0, + 'connect_renegotiate': 0, + 'accept': 0, + 'accept_good': 0, + 'accept_renegotiate': 0, + 'hits': 0, + 'misses': 0, + 'timeouts': 0, + 'cache_full': 0, + }) + + def test_set_default_verify_paths(self): + # There's not much we can do to test that it acts as expected, + # so just check it doesn't crash or raise an exception. + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ctx.set_default_verify_paths() + + @unittest.skipUnless(ssl.HAS_ECDH, "ECDH disabled on this OpenSSL build") + def test_set_ecdh_curve(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + ctx.set_ecdh_curve("prime256v1") + ctx.set_ecdh_curve(b"prime256v1") + self.assertRaises(TypeError, ctx.set_ecdh_curve) + self.assertRaises(TypeError, ctx.set_ecdh_curve, None) + self.assertRaises(ValueError, ctx.set_ecdh_curve, "foo") + self.assertRaises(ValueError, ctx.set_ecdh_curve, b"foo") + + def test_sni_callback(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + + # set_servername_callback expects a callable, or None + self.assertRaises(TypeError, ctx.set_servername_callback) + self.assertRaises(TypeError, ctx.set_servername_callback, 4) + self.assertRaises(TypeError, ctx.set_servername_callback, "") + self.assertRaises(TypeError, ctx.set_servername_callback, ctx) + + def dummycallback(sock, servername, ctx): + pass + ctx.set_servername_callback(None) + ctx.set_servername_callback(dummycallback) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_sni_callback_refcycle(self): + # Reference cycles through the servername callback are detected + # and cleared. + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + def dummycallback(sock, servername, ctx, cycle=ctx): + pass + ctx.set_servername_callback(dummycallback) + wr = weakref.ref(ctx) + del ctx, dummycallback + gc.collect() + self.assertIs(wr(), None) + + def test_cert_store_stats(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + self.assertEqual(ctx.cert_store_stats(), + {'x509_ca': 0, 'crl': 0, 'x509': 0}) + ctx.load_cert_chain(CERTFILE) + self.assertEqual(ctx.cert_store_stats(), + {'x509_ca': 0, 'crl': 0, 'x509': 0}) + ctx.load_verify_locations(CERTFILE) + self.assertEqual(ctx.cert_store_stats(), + {'x509_ca': 0, 'crl': 0, 'x509': 1}) + ctx.load_verify_locations(CAFILE_CACERT) + self.assertEqual(ctx.cert_store_stats(), + {'x509_ca': 1, 'crl': 0, 'x509': 2}) + + def test_get_ca_certs(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + self.assertEqual(ctx.get_ca_certs(), []) + # CERTFILE is not flagged as X509v3 Basic Constraints: CA:TRUE + ctx.load_verify_locations(CERTFILE) + self.assertEqual(ctx.get_ca_certs(), []) + # but CAFILE_CACERT is a CA cert + ctx.load_verify_locations(CAFILE_CACERT) + self.assertEqual(ctx.get_ca_certs(), + [{'issuer': ((('organizationName', 'Root CA'),), + (('organizationalUnitName', 'http://www.cacert.org'),), + (('commonName', 'CA Cert Signing Authority'),), + (('emailAddress', 'support@cacert.org'),)), + 'notAfter': 'Mar 29 12:29:49 2033 GMT', + 'notBefore': 'Mar 30 12:29:49 2003 GMT', + 'serialNumber': '00', + 'crlDistributionPoints': ('https://www.cacert.org/revoke.crl',), + 'subject': ((('organizationName', 'Root CA'),), + (('organizationalUnitName', 'http://www.cacert.org'),), + (('commonName', 'CA Cert Signing Authority'),), + (('emailAddress', 'support@cacert.org'),)), + 'version': 3}]) + + with open(CAFILE_CACERT) as f: + pem = f.read() + der = ssl.PEM_cert_to_DER_cert(pem) + self.assertEqual(ctx.get_ca_certs(True), [der]) + + def test_load_default_certs(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ctx.load_default_certs() + + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ctx.load_default_certs(ssl.Purpose.SERVER_AUTH) + ctx.load_default_certs() + + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ctx.load_default_certs(ssl.Purpose.CLIENT_AUTH) + + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + self.assertRaises(TypeError, ctx.load_default_certs, None) + self.assertRaises(TypeError, ctx.load_default_certs, 'SERVER_AUTH') + + @unittest.skipIf(sys.platform == "win32", "not-Windows specific") + def test_load_default_certs_env(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + with os_helper.EnvironmentVarGuard() as env: + env["SSL_CERT_DIR"] = CAPATH + env["SSL_CERT_FILE"] = CERTFILE + ctx.load_default_certs() + self.assertEqual(ctx.cert_store_stats(), {"crl": 0, "x509": 1, "x509_ca": 0}) + + @unittest.skipUnless(sys.platform == "win32", "Windows specific") + @unittest.skipIf(support.Py_DEBUG, + "Debug build does not share environment between CRTs") + def test_load_default_certs_env_windows(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ctx.load_default_certs() + stats = ctx.cert_store_stats() + + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + with os_helper.EnvironmentVarGuard() as env: + env["SSL_CERT_DIR"] = CAPATH + env["SSL_CERT_FILE"] = CERTFILE + ctx.load_default_certs() + stats["x509"] += 1 + self.assertEqual(ctx.cert_store_stats(), stats) + + def _assert_context_options(self, ctx): + self.assertEqual(ctx.options & ssl.OP_NO_SSLv2, ssl.OP_NO_SSLv2) + if OP_NO_COMPRESSION != 0: + self.assertEqual(ctx.options & OP_NO_COMPRESSION, + OP_NO_COMPRESSION) + if OP_SINGLE_DH_USE != 0: + self.assertEqual(ctx.options & OP_SINGLE_DH_USE, + OP_SINGLE_DH_USE) + if OP_SINGLE_ECDH_USE != 0: + self.assertEqual(ctx.options & OP_SINGLE_ECDH_USE, + OP_SINGLE_ECDH_USE) + if OP_CIPHER_SERVER_PREFERENCE != 0: + self.assertEqual(ctx.options & OP_CIPHER_SERVER_PREFERENCE, + OP_CIPHER_SERVER_PREFERENCE) + self.assertEqual(ctx.options & ssl.OP_LEGACY_SERVER_CONNECT, + 0 if IS_OPENSSL_3_0_0 else ssl.OP_LEGACY_SERVER_CONNECT) + + def test_create_default_context(self): + ctx = ssl.create_default_context() + + self.assertEqual(ctx.protocol, ssl.PROTOCOL_TLS_CLIENT) + self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED) + self.assertEqual(ctx.verify_flags & ssl.VERIFY_X509_PARTIAL_CHAIN, + ssl.VERIFY_X509_PARTIAL_CHAIN) + self.assertEqual(ctx.verify_flags & ssl.VERIFY_X509_STRICT, + ssl.VERIFY_X509_STRICT) + self.assertTrue(ctx.check_hostname) + self._assert_context_options(ctx) + + with open(SIGNING_CA) as f: + cadata = f.read() + ctx = ssl.create_default_context(cafile=SIGNING_CA, capath=CAPATH, + cadata=cadata) + self.assertEqual(ctx.protocol, ssl.PROTOCOL_TLS_CLIENT) + self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED) + self._assert_context_options(ctx) + + ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH) + self.assertEqual(ctx.protocol, ssl.PROTOCOL_TLS_SERVER) + self.assertEqual(ctx.verify_mode, ssl.CERT_NONE) + self._assert_context_options(ctx) + + def test__create_stdlib_context(self): + ctx = ssl._create_stdlib_context() + self.assertEqual(ctx.protocol, ssl.PROTOCOL_TLS_CLIENT) + self.assertEqual(ctx.verify_mode, ssl.CERT_NONE) + self.assertFalse(ctx.check_hostname) + self._assert_context_options(ctx) + + if has_tls_protocol(ssl.PROTOCOL_TLSv1): + with warnings_helper.check_warnings(): + ctx = ssl._create_stdlib_context(ssl.PROTOCOL_TLSv1) + self.assertEqual(ctx.protocol, ssl.PROTOCOL_TLSv1) + self.assertEqual(ctx.verify_mode, ssl.CERT_NONE) + self._assert_context_options(ctx) + + with warnings_helper.check_warnings(): + ctx = ssl._create_stdlib_context( + ssl.PROTOCOL_TLSv1_2, + cert_reqs=ssl.CERT_REQUIRED, + check_hostname=True + ) + self.assertEqual(ctx.protocol, ssl.PROTOCOL_TLSv1_2) + self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED) + self.assertTrue(ctx.check_hostname) + self._assert_context_options(ctx) + + ctx = ssl._create_stdlib_context(purpose=ssl.Purpose.CLIENT_AUTH) + self.assertEqual(ctx.protocol, ssl.PROTOCOL_TLS_SERVER) + self.assertEqual(ctx.verify_mode, ssl.CERT_NONE) + self._assert_context_options(ctx) + + def test_check_hostname(self): + with warnings_helper.check_warnings(): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS) + self.assertFalse(ctx.check_hostname) + self.assertEqual(ctx.verify_mode, ssl.CERT_NONE) + + # Auto set CERT_REQUIRED + ctx.check_hostname = True + self.assertTrue(ctx.check_hostname) + self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED) + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_REQUIRED + self.assertFalse(ctx.check_hostname) + self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED) + + # Changing verify_mode does not affect check_hostname + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + ctx.check_hostname = False + self.assertFalse(ctx.check_hostname) + self.assertEqual(ctx.verify_mode, ssl.CERT_NONE) + # Auto set + ctx.check_hostname = True + self.assertTrue(ctx.check_hostname) + self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED) + + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_OPTIONAL + ctx.check_hostname = False + self.assertFalse(ctx.check_hostname) + self.assertEqual(ctx.verify_mode, ssl.CERT_OPTIONAL) + # keep CERT_OPTIONAL + ctx.check_hostname = True + self.assertTrue(ctx.check_hostname) + self.assertEqual(ctx.verify_mode, ssl.CERT_OPTIONAL) + + # Cannot set CERT_NONE with check_hostname enabled + with self.assertRaises(ValueError): + ctx.verify_mode = ssl.CERT_NONE + ctx.check_hostname = False + self.assertFalse(ctx.check_hostname) + ctx.verify_mode = ssl.CERT_NONE + self.assertEqual(ctx.verify_mode, ssl.CERT_NONE) + + def test_context_client_server(self): + # PROTOCOL_TLS_CLIENT has sane defaults + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + self.assertTrue(ctx.check_hostname) + self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED) + + # PROTOCOL_TLS_SERVER has different but also sane defaults + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + self.assertFalse(ctx.check_hostname) + self.assertEqual(ctx.verify_mode, ssl.CERT_NONE) + + def test_context_custom_class(self): + class MySSLSocket(ssl.SSLSocket): + pass + + class MySSLObject(ssl.SSLObject): + pass + + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + ctx.sslsocket_class = MySSLSocket + ctx.sslobject_class = MySSLObject + + with ctx.wrap_socket(socket.socket(), server_side=True) as sock: + self.assertIsInstance(sock, MySSLSocket) + obj = ctx.wrap_bio(ssl.MemoryBIO(), ssl.MemoryBIO(), server_side=True) + self.assertIsInstance(obj, MySSLObject) + + def test_num_tickest(self): + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + self.assertEqual(ctx.num_tickets, 2) + ctx.num_tickets = 1 + self.assertEqual(ctx.num_tickets, 1) + ctx.num_tickets = 0 + self.assertEqual(ctx.num_tickets, 0) + with self.assertRaises(ValueError): + ctx.num_tickets = -1 + with self.assertRaises(TypeError): + ctx.num_tickets = None + + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + self.assertEqual(ctx.num_tickets, 2) + with self.assertRaises(ValueError): + ctx.num_tickets = 1 + + +class SSLErrorTests(unittest.TestCase): + + def test_str(self): + # The str() of a SSLError doesn't include the errno + e = ssl.SSLError(1, "foo") + self.assertEqual(str(e), "foo") + self.assertEqual(e.errno, 1) + # Same for a subclass + e = ssl.SSLZeroReturnError(1, "foo") + self.assertEqual(str(e), "foo") + self.assertEqual(e.errno, 1) + + def test_lib_reason(self): + # Test the library and reason attributes + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + try: + with self.assertRaises(ssl.SSLError) as cm: + ctx.load_dh_params(CERTFILE) + except RuntimeError: + if Py_DEBUG_WIN32: + self.skipTest("not supported on Win32 debug build") + raise + + self.assertEqual(cm.exception.library, 'PEM') + regex = "(NO_START_LINE|UNSUPPORTED_PUBLIC_KEY_TYPE)" + self.assertRegex(cm.exception.reason, regex) + s = str(cm.exception) + self.assertTrue("NO_START_LINE" in s, s) + + def test_subclass(self): + # Check that the appropriate SSLError subclass is raised + # (this only tests one of them) + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + with socket.create_server(("127.0.0.1", 0)) as s: + c = socket.create_connection(s.getsockname()) + c.setblocking(False) + with ctx.wrap_socket(c, False, do_handshake_on_connect=False) as c: + with self.assertRaises(ssl.SSLWantReadError) as cm: + c.do_handshake() + s = str(cm.exception) + self.assertTrue(s.startswith("The operation did not complete (read)"), s) + # For compatibility + self.assertEqual(cm.exception.errno, ssl.SSL_ERROR_WANT_READ) + + + def test_bad_server_hostname(self): + ctx = ssl.create_default_context() + with self.assertRaises(ValueError): + ctx.wrap_bio(ssl.MemoryBIO(), ssl.MemoryBIO(), + server_hostname="") + with self.assertRaises(ValueError): + ctx.wrap_bio(ssl.MemoryBIO(), ssl.MemoryBIO(), + server_hostname=".example.org") + with self.assertRaises(TypeError): + ctx.wrap_bio(ssl.MemoryBIO(), ssl.MemoryBIO(), + server_hostname="example.org\x00evil.com") + + +class MemoryBIOTests(unittest.TestCase): + + def test_read_write(self): + bio = ssl.MemoryBIO() + bio.write(b'foo') + self.assertEqual(bio.read(), b'foo') + self.assertEqual(bio.read(), b'') + bio.write(b'foo') + bio.write(b'bar') + self.assertEqual(bio.read(), b'foobar') + self.assertEqual(bio.read(), b'') + bio.write(b'baz') + self.assertEqual(bio.read(2), b'ba') + self.assertEqual(bio.read(1), b'z') + self.assertEqual(bio.read(1), b'') + + def test_eof(self): + bio = ssl.MemoryBIO() + self.assertFalse(bio.eof) + self.assertEqual(bio.read(), b'') + self.assertFalse(bio.eof) + bio.write(b'foo') + self.assertFalse(bio.eof) + bio.write_eof() + self.assertFalse(bio.eof) + self.assertEqual(bio.read(2), b'fo') + self.assertFalse(bio.eof) + self.assertEqual(bio.read(1), b'o') + self.assertTrue(bio.eof) + self.assertEqual(bio.read(), b'') + self.assertTrue(bio.eof) + + def test_pending(self): + bio = ssl.MemoryBIO() + self.assertEqual(bio.pending, 0) + bio.write(b'foo') + self.assertEqual(bio.pending, 3) + for i in range(3): + bio.read(1) + self.assertEqual(bio.pending, 3-i-1) + for i in range(3): + bio.write(b'x') + self.assertEqual(bio.pending, i+1) + bio.read() + self.assertEqual(bio.pending, 0) + + def test_buffer_types(self): + bio = ssl.MemoryBIO() + bio.write(b'foo') + self.assertEqual(bio.read(), b'foo') + bio.write(bytearray(b'bar')) + self.assertEqual(bio.read(), b'bar') + bio.write(memoryview(b'baz')) + self.assertEqual(bio.read(), b'baz') + m = memoryview(bytearray(b'noncontig')) + noncontig_writable = m[::-2] + with self.assertRaises(BufferError): + bio.write(memoryview(noncontig_writable)) + + def test_error_types(self): + bio = ssl.MemoryBIO() + self.assertRaises(TypeError, bio.write, 'foo') + self.assertRaises(TypeError, bio.write, None) + self.assertRaises(TypeError, bio.write, True) + self.assertRaises(TypeError, bio.write, 1) + + +class SSLObjectTests(unittest.TestCase): + def test_private_init(self): + bio = ssl.MemoryBIO() + with self.assertRaisesRegex(TypeError, "public constructor"): + ssl.SSLObject(bio, bio) + + def test_unwrap(self): + client_ctx, server_ctx, hostname = testing_context() + c_in = ssl.MemoryBIO() + c_out = ssl.MemoryBIO() + s_in = ssl.MemoryBIO() + s_out = ssl.MemoryBIO() + client = client_ctx.wrap_bio(c_in, c_out, server_hostname=hostname) + server = server_ctx.wrap_bio(s_in, s_out, server_side=True) + + # Loop on the handshake for a bit to get it settled + for _ in range(5): + try: + client.do_handshake() + except ssl.SSLWantReadError: + pass + if c_out.pending: + s_in.write(c_out.read()) + try: + server.do_handshake() + except ssl.SSLWantReadError: + pass + if s_out.pending: + c_in.write(s_out.read()) + # Now the handshakes should be complete (don't raise WantReadError) + client.do_handshake() + server.do_handshake() + + # Now if we unwrap one side unilaterally, it should send close-notify + # and raise WantReadError: + with self.assertRaises(ssl.SSLWantReadError): + client.unwrap() + + # But server.unwrap() does not raise, because it reads the client's + # close-notify: + s_in.write(c_out.read()) + server.unwrap() + + # And now that the client gets the server's close-notify, it doesn't + # raise either. + c_in.write(s_out.read()) + client.unwrap() + +class SimpleBackgroundTests(unittest.TestCase): + """Tests that connect to a simple server running in the background""" + + def setUp(self): + self.server_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + self.server_context.load_cert_chain(SIGNED_CERTFILE) + server = ThreadedEchoServer(context=self.server_context) + self.enterContext(server) + self.server_addr = (HOST, server.port) + + def test_connect(self): + with test_wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_NONE) as s: + s.connect(self.server_addr) + self.assertEqual({}, s.getpeercert()) + self.assertFalse(s.server_side) + + # this should succeed because we specify the root cert + with test_wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_REQUIRED, + ca_certs=SIGNING_CA) as s: + s.connect(self.server_addr) + self.assertTrue(s.getpeercert()) + self.assertFalse(s.server_side) + + def test_connect_fail(self): + # This should fail because we have no verification certs. Connection + # failure crashes ThreadedEchoServer, so run this in an independent + # test method. + s = test_wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_REQUIRED) + self.addCleanup(s.close) + # Allow for flexible libssl error messages. + regex = re.compile(r"""( + certificate verify failed # OpenSSL + | + CERTIFICATE_VERIFY_FAILED # AWS-LC + )""", re.X) + self.assertRaisesRegex(ssl.SSLError, regex, + s.connect, self.server_addr) + + def test_connect_ex(self): + # Issue #11326: check connect_ex() implementation + s = test_wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_REQUIRED, + ca_certs=SIGNING_CA) + self.addCleanup(s.close) + self.assertEqual(0, s.connect_ex(self.server_addr)) + self.assertTrue(s.getpeercert()) + + def test_non_blocking_connect_ex(self): + # Issue #11326: non-blocking connect_ex() should allow handshake + # to proceed after the socket gets ready. + s = test_wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_REQUIRED, + ca_certs=SIGNING_CA, + do_handshake_on_connect=False) + self.addCleanup(s.close) + s.setblocking(False) + rc = s.connect_ex(self.server_addr) + # EWOULDBLOCK under Windows, EINPROGRESS elsewhere + self.assertIn(rc, (0, errno.EINPROGRESS, errno.EWOULDBLOCK)) + # Wait for connect to finish + select.select([], [s], [], 5.0) + # Non-blocking handshake + while True: + try: + s.do_handshake() + break + except ssl.SSLWantReadError: + select.select([s], [], [], 5.0) + except ssl.SSLWantWriteError: + select.select([], [s], [], 5.0) + # SSL established + self.assertTrue(s.getpeercert()) + + def test_connect_with_context(self): + # Same as test_connect, but with a separately created context + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + with ctx.wrap_socket(socket.socket(socket.AF_INET)) as s: + s.connect(self.server_addr) + self.assertEqual({}, s.getpeercert()) + # Same with a server hostname + with ctx.wrap_socket(socket.socket(socket.AF_INET), + server_hostname="dummy") as s: + s.connect(self.server_addr) + ctx.verify_mode = ssl.CERT_REQUIRED + # This should succeed because we specify the root cert + ctx.load_verify_locations(SIGNING_CA) + with ctx.wrap_socket(socket.socket(socket.AF_INET)) as s: + s.connect(self.server_addr) + cert = s.getpeercert() + self.assertTrue(cert) + + def test_connect_with_context_fail(self): + # This should fail because we have no verification certs. Connection + # failure crashes ThreadedEchoServer, so run this in an independent + # test method. + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + s = ctx.wrap_socket( + socket.socket(socket.AF_INET), + server_hostname=SIGNED_CERTFILE_HOSTNAME + ) + self.addCleanup(s.close) + # Allow for flexible libssl error messages. + regex = re.compile(r"""( + certificate verify failed # OpenSSL + | + CERTIFICATE_VERIFY_FAILED # AWS-LC + )""", re.X) + self.assertRaisesRegex(ssl.SSLError, regex, + s.connect, self.server_addr) + + def test_connect_capath(self): + # Verify server certificates using the `capath` argument + # NOTE: the subject hashing algorithm has been changed between + # OpenSSL 0.9.8n and 1.0.0, as a result the capath directory must + # contain both versions of each certificate (same content, different + # filename) for this test to be portable across OpenSSL releases. + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ctx.load_verify_locations(capath=CAPATH) + with ctx.wrap_socket(socket.socket(socket.AF_INET), + server_hostname=SIGNED_CERTFILE_HOSTNAME) as s: + s.connect(self.server_addr) + cert = s.getpeercert() + self.assertTrue(cert) + + # Same with a bytes `capath` argument + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ctx.load_verify_locations(capath=BYTES_CAPATH) + with ctx.wrap_socket(socket.socket(socket.AF_INET), + server_hostname=SIGNED_CERTFILE_HOSTNAME) as s: + s.connect(self.server_addr) + cert = s.getpeercert() + self.assertTrue(cert) + + def test_connect_cadata(self): + with open(SIGNING_CA) as f: + pem = f.read() + der = ssl.PEM_cert_to_DER_cert(pem) + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ctx.load_verify_locations(cadata=pem) + with ctx.wrap_socket(socket.socket(socket.AF_INET), + server_hostname=SIGNED_CERTFILE_HOSTNAME) as s: + s.connect(self.server_addr) + cert = s.getpeercert() + self.assertTrue(cert) + + # same with DER + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ctx.load_verify_locations(cadata=der) + with ctx.wrap_socket(socket.socket(socket.AF_INET), + server_hostname=SIGNED_CERTFILE_HOSTNAME) as s: + s.connect(self.server_addr) + cert = s.getpeercert() + self.assertTrue(cert) + + @unittest.skipIf(os.name == "nt", "Can't use a socket as a file under Windows") + def test_makefile_close(self): + # Issue #5238: creating a file-like object with makefile() shouldn't + # delay closing the underlying "real socket" (here tested with its + # file descriptor, hence skipping the test under Windows). + ss = test_wrap_socket(socket.socket(socket.AF_INET)) + ss.connect(self.server_addr) + fd = ss.fileno() + f = ss.makefile() + f.close() + # The fd is still open + os.read(fd, 0) + # Closing the SSL socket should close the fd too + ss.close() + gc.collect() + with self.assertRaises(OSError) as e: + os.read(fd, 0) + self.assertEqual(e.exception.errno, errno.EBADF) + + def test_non_blocking_handshake(self): + s = socket.socket(socket.AF_INET) + s.connect(self.server_addr) + s.setblocking(False) + s = test_wrap_socket(s, + cert_reqs=ssl.CERT_NONE, + do_handshake_on_connect=False) + self.addCleanup(s.close) + count = 0 + while True: + try: + count += 1 + s.do_handshake() + break + except ssl.SSLWantReadError: + select.select([s], [], []) + except ssl.SSLWantWriteError: + select.select([], [s], []) + if support.verbose: + sys.stdout.write("\nNeeded %d calls to do_handshake() to establish session.\n" % count) + + def test_get_server_certificate(self): + _test_get_server_certificate(self, *self.server_addr, cert=SIGNING_CA) + + def test_get_server_certificate_sni(self): + host, port = self.server_addr + server_names = [] + + # We store servername_cb arguments to make sure they match the host + def servername_cb(ssl_sock, server_name, initial_context): + server_names.append(server_name) + self.server_context.set_servername_callback(servername_cb) + + pem = ssl.get_server_certificate((host, port)) + if not pem: + self.fail("No server certificate on %s:%s!" % (host, port)) + + pem = ssl.get_server_certificate((host, port), ca_certs=SIGNING_CA) + if not pem: + self.fail("No server certificate on %s:%s!" % (host, port)) + if support.verbose: + sys.stdout.write("\nVerified certificate for %s:%s is\n%s\n" % (host, port, pem)) + + self.assertEqual(server_names, [host, host]) + + def test_get_server_certificate_fail(self): + # Connection failure crashes ThreadedEchoServer, so run this in an + # independent test method + _test_get_server_certificate_fail(self, *self.server_addr) + + def test_get_server_certificate_timeout(self): + def servername_cb(ssl_sock, server_name, initial_context): + time.sleep(0.2) + self.server_context.set_servername_callback(servername_cb) + + with self.assertRaises(socket.timeout): + ssl.get_server_certificate(self.server_addr, ca_certs=SIGNING_CA, + timeout=0.1) + + def test_ciphers(self): + with test_wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_NONE, ciphers="ALL") as s: + s.connect(self.server_addr) + with test_wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_NONE, ciphers="DEFAULT") as s: + s.connect(self.server_addr) + # Error checking can happen at instantiation or when connecting + with self.assertRaisesRegex(ssl.SSLError, "No cipher can be selected"): + with socket.socket(socket.AF_INET) as sock: + s = test_wrap_socket(sock, + cert_reqs=ssl.CERT_NONE, ciphers="^$:,;?*'dorothyx") + s.connect(self.server_addr) + + def test_get_ca_certs_capath(self): + # capath certs are loaded on request + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ctx.load_verify_locations(capath=CAPATH) + self.assertEqual(ctx.get_ca_certs(), []) + with ctx.wrap_socket(socket.socket(socket.AF_INET), + server_hostname='localhost') as s: + s.connect(self.server_addr) + cert = s.getpeercert() + self.assertTrue(cert) + self.assertEqual(len(ctx.get_ca_certs()), 1) + + def test_context_setget(self): + # Check that the context of a connected socket can be replaced. + ctx1 = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ctx1.load_verify_locations(capath=CAPATH) + ctx2 = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ctx2.load_verify_locations(capath=CAPATH) + s = socket.socket(socket.AF_INET) + with ctx1.wrap_socket(s, server_hostname='localhost') as ss: + ss.connect(self.server_addr) + self.assertIs(ss.context, ctx1) + self.assertIs(ss._sslobj.context, ctx1) + ss.context = ctx2 + self.assertIs(ss.context, ctx2) + self.assertIs(ss._sslobj.context, ctx2) + + def ssl_io_loop(self, sock, incoming, outgoing, func, *args, **kwargs): + # A simple IO loop. Call func(*args) depending on the error we get + # (WANT_READ or WANT_WRITE) move data between the socket and the BIOs. + timeout = kwargs.get('timeout', support.SHORT_TIMEOUT) + count = 0 + for _ in support.busy_retry(timeout): + errno = None + count += 1 + try: + ret = func(*args) + except ssl.SSLError as e: + if e.errno not in (ssl.SSL_ERROR_WANT_READ, + ssl.SSL_ERROR_WANT_WRITE): + raise + errno = e.errno + # Get any data from the outgoing BIO irrespective of any error, and + # send it to the socket. + buf = outgoing.read() + sock.sendall(buf) + # If there's no error, we're done. For WANT_READ, we need to get + # data from the socket and put it in the incoming BIO. + if errno is None: + break + elif errno == ssl.SSL_ERROR_WANT_READ: + buf = sock.recv(32768) + if buf: + incoming.write(buf) + else: + incoming.write_eof() + if support.verbose: + sys.stdout.write("Needed %d calls to complete %s().\n" + % (count, func.__name__)) + return ret + + def test_bio_handshake(self): + sock = socket.socket(socket.AF_INET) + self.addCleanup(sock.close) + sock.connect(self.server_addr) + incoming = ssl.MemoryBIO() + outgoing = ssl.MemoryBIO() + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + self.assertTrue(ctx.check_hostname) + self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED) + ctx.load_verify_locations(SIGNING_CA) + sslobj = ctx.wrap_bio(incoming, outgoing, False, + SIGNED_CERTFILE_HOSTNAME) + self.assertIs(sslobj._sslobj.owner, sslobj) + self.assertIsNone(sslobj.cipher()) + self.assertIsNone(sslobj.version()) + self.assertIsNone(sslobj.shared_ciphers()) + self.assertRaises(ValueError, sslobj.getpeercert) + # tls-unique is not defined for TLSv1.3 + # https://datatracker.ietf.org/doc/html/rfc8446#appendix-C.5 + if 'tls-unique' in ssl.CHANNEL_BINDING_TYPES and sslobj.version() != "TLSv1.3": + self.assertIsNone(sslobj.get_channel_binding('tls-unique')) + self.ssl_io_loop(sock, incoming, outgoing, sslobj.do_handshake) + self.assertTrue(sslobj.cipher()) + self.assertIsNone(sslobj.shared_ciphers()) + self.assertIsNotNone(sslobj.version()) + self.assertTrue(sslobj.getpeercert()) + if 'tls-unique' in ssl.CHANNEL_BINDING_TYPES and sslobj.version() != "TLSv1.3": + self.assertTrue(sslobj.get_channel_binding('tls-unique')) + try: + self.ssl_io_loop(sock, incoming, outgoing, sslobj.unwrap) + except ssl.SSLSyscallError: + # If the server shuts down the TCP connection without sending a + # secure shutdown message, this is reported as SSL_ERROR_SYSCALL + pass + self.assertRaises(ssl.SSLError, sslobj.write, b'foo') + + def test_bio_read_write_data(self): + sock = socket.socket(socket.AF_INET) + self.addCleanup(sock.close) + sock.connect(self.server_addr) + incoming = ssl.MemoryBIO() + outgoing = ssl.MemoryBIO() + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + ctx.check_hostname = False + ctx.verify_mode = ssl.CERT_NONE + sslobj = ctx.wrap_bio(incoming, outgoing, False) + self.ssl_io_loop(sock, incoming, outgoing, sslobj.do_handshake) + req = b'FOO\n' + self.ssl_io_loop(sock, incoming, outgoing, sslobj.write, req) + buf = self.ssl_io_loop(sock, incoming, outgoing, sslobj.read, 1024) + self.assertEqual(buf, b'foo\n') + self.ssl_io_loop(sock, incoming, outgoing, sslobj.unwrap) + + def test_transport_eof(self): + client_context, server_context, hostname = testing_context() + with socket.socket(socket.AF_INET) as sock: + sock.connect(self.server_addr) + incoming = ssl.MemoryBIO() + outgoing = ssl.MemoryBIO() + sslobj = client_context.wrap_bio(incoming, outgoing, + server_hostname=hostname) + self.ssl_io_loop(sock, incoming, outgoing, sslobj.do_handshake) + + # Simulate EOF from the transport. + incoming.write_eof() + self.assertRaises(ssl.SSLEOFError, sslobj.read) + + +@support.requires_resource('network') +class NetworkedTests(unittest.TestCase): + + def test_timeout_connect_ex(self): + # Issue #12065: on a timeout, connect_ex() should return the original + # errno (mimicking the behaviour of non-SSL sockets). + with socket_helper.transient_internet(REMOTE_HOST): + s = test_wrap_socket(socket.socket(socket.AF_INET), + cert_reqs=ssl.CERT_REQUIRED, + do_handshake_on_connect=False) + self.addCleanup(s.close) + s.settimeout(0.0000001) + rc = s.connect_ex((REMOTE_HOST, 443)) + if rc == 0: + self.skipTest("REMOTE_HOST responded too quickly") + elif rc == errno.ENETUNREACH: + self.skipTest("Network unreachable.") + self.assertIn(rc, (errno.EAGAIN, errno.EWOULDBLOCK)) + + @unittest.skipUnless(socket_helper.IPV6_ENABLED, 'Needs IPv6') + @support.requires_resource('walltime') + def test_get_server_certificate_ipv6(self): + with socket_helper.transient_internet('ipv6.google.com'): + _test_get_server_certificate(self, 'ipv6.google.com', 443) + _test_get_server_certificate_fail(self, 'ipv6.google.com', 443) + + +def _test_get_server_certificate(test, host, port, cert=None): + pem = ssl.get_server_certificate((host, port)) + if not pem: + test.fail("No server certificate on %s:%s!" % (host, port)) + + pem = ssl.get_server_certificate((host, port), ca_certs=cert) + if not pem: + test.fail("No server certificate on %s:%s!" % (host, port)) + if support.verbose: + sys.stdout.write("\nVerified certificate for %s:%s is\n%s\n" % (host, port ,pem)) + +def _test_get_server_certificate_fail(test, host, port): + with warnings_helper.check_no_resource_warning(test): + try: + pem = ssl.get_server_certificate((host, port), ca_certs=CERTFILE) + except ssl.SSLError as x: + #should fail + if support.verbose: + sys.stdout.write("%s\n" % x) + else: + test.fail("Got server certificate %s for %s:%s!" % (pem, host, port)) + + +from test.ssl_servers import make_https_server + +class ThreadedEchoServer(threading.Thread): + + class ConnectionHandler(threading.Thread): + + """A mildly complicated class, because we want it to work both + with and without the SSL wrapper around the socket connection, so + that we can test the STARTTLS functionality.""" + + def __init__(self, server, connsock, addr): + self.server = server + self.running = False + self.sock = connsock + self.addr = addr + self.sock.setblocking(True) + self.sslconn = None + threading.Thread.__init__(self) + self.daemon = True + + def wrap_conn(self): + try: + self.sslconn = self.server.context.wrap_socket( + self.sock, server_side=True) + self.server.selected_alpn_protocols.append(self.sslconn.selected_alpn_protocol()) + except (ConnectionResetError, BrokenPipeError, ConnectionAbortedError) as e: + # We treat ConnectionResetError as though it were an + # SSLError - OpenSSL on Ubuntu abruptly closes the + # connection when asked to use an unsupported protocol. + # + # BrokenPipeError is raised in TLS 1.3 mode, when OpenSSL + # tries to send session tickets after handshake. + # https://github.com/openssl/openssl/issues/6342 + # + # ConnectionAbortedError is raised in TLS 1.3 mode, when OpenSSL + # tries to send session tickets after handshake when using WinSock. + self.server.conn_errors.append(str(e)) + if self.server.chatty: + handle_error("\n server: bad connection attempt from " + repr(self.addr) + ":\n") + self.running = False + self.close() + return False + except (ssl.SSLError, OSError) as e: + # OSError may occur with wrong protocols, e.g. both + # sides use PROTOCOL_TLS_SERVER. + # + # XXX Various errors can have happened here, for example + # a mismatching protocol version, an invalid certificate, + # or a low-level bug. This should be made more discriminating. + # + # bpo-31323: Store the exception as string to prevent + # a reference leak: server -> conn_errors -> exception + # -> traceback -> self (ConnectionHandler) -> server + self.server.conn_errors.append(str(e)) + if self.server.chatty: + handle_error("\n server: bad connection attempt from " + repr(self.addr) + ":\n") + + # bpo-44229, bpo-43855, bpo-44237, and bpo-33450: + # Ignore spurious EPROTOTYPE returned by write() on macOS. + # See also http://erickt.github.io/blog/2014/11/19/adventures-in-debugging-a-potential-osx-kernel-bug/ + if e.errno != errno.EPROTOTYPE and sys.platform != "darwin": + self.running = False + self.close() + return False + else: + self.server.shared_ciphers.append(self.sslconn.shared_ciphers()) + if self.server.context.verify_mode == ssl.CERT_REQUIRED: + cert = self.sslconn.getpeercert() + if support.verbose and self.server.chatty: + sys.stdout.write(" client cert is " + pprint.pformat(cert) + "\n") + cert_binary = self.sslconn.getpeercert(True) + if support.verbose and self.server.chatty: + if cert_binary is None: + sys.stdout.write(" client did not provide a cert\n") + else: + sys.stdout.write(f" cert binary is {len(cert_binary)}b\n") + cipher = self.sslconn.cipher() + if support.verbose and self.server.chatty: + sys.stdout.write(" server: connection cipher is now " + str(cipher) + "\n") + return True + + def read(self): + if self.sslconn: + return self.sslconn.read() + else: + return self.sock.recv(1024) + + def write(self, bytes): + if self.sslconn: + return self.sslconn.write(bytes) + else: + return self.sock.send(bytes) + + def close(self): + if self.sslconn: + self.sslconn.close() + else: + self.sock.close() + + def run(self): + self.running = True + if not self.server.starttls_server: + if not self.wrap_conn(): + return + while self.running: + try: + msg = self.read() + stripped = msg.strip() + if not stripped: + # eof, so quit this handler + self.running = False + try: + self.sock = self.sslconn.unwrap() + except OSError: + # Many tests shut the TCP connection down + # without an SSL shutdown. This causes + # unwrap() to raise OSError with errno=0! + pass + else: + self.sslconn = None + self.close() + elif stripped == b'over': + if support.verbose and self.server.connectionchatty: + sys.stdout.write(" server: client closed connection\n") + self.close() + return + elif (self.server.starttls_server and + stripped == b'STARTTLS'): + if support.verbose and self.server.connectionchatty: + sys.stdout.write(" server: read STARTTLS from client, sending OK...\n") + self.write(b"OK\n") + if not self.wrap_conn(): + return + elif (self.server.starttls_server and self.sslconn + and stripped == b'ENDTLS'): + if support.verbose and self.server.connectionchatty: + sys.stdout.write(" server: read ENDTLS from client, sending OK...\n") + self.write(b"OK\n") + self.sock = self.sslconn.unwrap() + self.sslconn = None + if support.verbose and self.server.connectionchatty: + sys.stdout.write(" server: connection is now unencrypted...\n") + elif stripped == b'CB tls-unique': + if support.verbose and self.server.connectionchatty: + sys.stdout.write(" server: read CB tls-unique from client, sending our CB data...\n") + data = self.sslconn.get_channel_binding("tls-unique") + self.write(repr(data).encode("us-ascii") + b"\n") + elif stripped == b'PHA': + if support.verbose and self.server.connectionchatty: + sys.stdout.write(" server: initiating post handshake auth\n") + try: + self.sslconn.verify_client_post_handshake() + except ssl.SSLError as e: + self.write(repr(e).encode("us-ascii") + b"\n") + else: + self.write(b"OK\n") + elif stripped == b'HASCERT': + if self.sslconn.getpeercert() is not None: + self.write(b'TRUE\n') + else: + self.write(b'FALSE\n') + elif stripped == b'GETCERT': + cert = self.sslconn.getpeercert() + self.write(repr(cert).encode("us-ascii") + b"\n") + elif stripped == b'VERIFIEDCHAIN': + certs = self.sslconn._sslobj.get_verified_chain() + self.write(len(certs).to_bytes(1, "big") + b"\n") + elif stripped == b'UNVERIFIEDCHAIN': + certs = self.sslconn._sslobj.get_unverified_chain() + self.write(len(certs).to_bytes(1, "big") + b"\n") + else: + if (support.verbose and + self.server.connectionchatty): + ctype = (self.sslconn and "encrypted") or "unencrypted" + sys.stdout.write(" server: read %r (%s), sending back %r (%s)...\n" + % (msg, ctype, msg.lower(), ctype)) + self.write(msg.lower()) + except OSError as e: + # handles SSLError and socket errors + if isinstance(e, ConnectionError): + # OpenSSL 1.1.1 sometimes raises + # ConnectionResetError when connection is not + # shut down gracefully. + if self.server.chatty and support.verbose: + print(f" Connection reset by peer: {self.addr}") + + self.close() + self.running = False + return + if self.server.chatty and support.verbose: + handle_error("Test server failure:\n") + try: + self.write(b"ERROR\n") + except OSError: + pass + self.close() + self.running = False + + def __init__(self, certificate=None, ssl_version=None, + certreqs=None, cacerts=None, + chatty=True, connectionchatty=False, starttls_server=False, + alpn_protocols=None, + ciphers=None, context=None): + if context: + self.context = context + else: + self.context = ssl.SSLContext(ssl_version + if ssl_version is not None + else ssl.PROTOCOL_TLS_SERVER) + self.context.verify_mode = (certreqs if certreqs is not None + else ssl.CERT_NONE) + if cacerts: + self.context.load_verify_locations(cacerts) + if certificate: + self.context.load_cert_chain(certificate) + if alpn_protocols: + self.context.set_alpn_protocols(alpn_protocols) + if ciphers: + self.context.set_ciphers(ciphers) + self.chatty = chatty + self.connectionchatty = connectionchatty + self.starttls_server = starttls_server + self.sock = socket.socket() + self.port = socket_helper.bind_port(self.sock) + self.flag = None + self.active = False + self.selected_alpn_protocols = [] + self.shared_ciphers = [] + self.conn_errors = [] + threading.Thread.__init__(self) + self.daemon = True + self._in_context = False + + def __enter__(self): + if self._in_context: + raise ValueError('Re-entering ThreadedEchoServer context') + self._in_context = True + self.start(threading.Event()) + self.flag.wait() + return self + + def __exit__(self, *args): + assert self._in_context + self._in_context = False + self.stop() + self.join() + + def start(self, flag=None): + if not self._in_context: + raise ValueError( + 'ThreadedEchoServer must be used as a context manager') + self.flag = flag + threading.Thread.start(self) + + def run(self): + if not self._in_context: + raise ValueError( + 'ThreadedEchoServer must be used as a context manager') + self.sock.settimeout(1.0) + self.sock.listen(5) + self.active = True + if self.flag: + # signal an event + self.flag.set() + while self.active: + try: + newconn, connaddr = self.sock.accept() + if support.verbose and self.chatty: + sys.stdout.write(' server: new connection from ' + + repr(connaddr) + '\n') + handler = self.ConnectionHandler(self, newconn, connaddr) + handler.start() + handler.join() + except TimeoutError as e: + if support.verbose: + sys.stdout.write(f' connection timeout {e!r}\n') + except KeyboardInterrupt: + self.stop() + except BaseException as e: + if support.verbose and self.chatty: + sys.stdout.write( + ' connection handling failed: ' + repr(e) + '\n') + + self.close() + + def close(self): + if self.sock is not None: + self.sock.close() + self.sock = None + + def stop(self): + self.active = False + +class AsyncoreEchoServer(threading.Thread): + + # this one's based on asyncore.dispatcher + + class EchoServer (asyncore.dispatcher): + + class ConnectionHandler(asyncore.dispatcher_with_send): + + def __init__(self, conn, certfile): + self.socket = test_wrap_socket(conn, server_side=True, + certfile=certfile, + do_handshake_on_connect=False) + asyncore.dispatcher_with_send.__init__(self, self.socket) + self._ssl_accepting = True + self._do_ssl_handshake() + + def readable(self): + if isinstance(self.socket, ssl.SSLSocket): + while self.socket.pending() > 0: + self.handle_read_event() + return True + + def _do_ssl_handshake(self): + try: + self.socket.do_handshake() + except (ssl.SSLWantReadError, ssl.SSLWantWriteError): + return + except ssl.SSLEOFError: + return self.handle_close() + except ssl.SSLError: + raise + except OSError as err: + if err.args[0] == errno.ECONNABORTED: + return self.handle_close() + else: + self._ssl_accepting = False + + def handle_read(self): + if self._ssl_accepting: + self._do_ssl_handshake() + else: + data = self.recv(1024) + if support.verbose: + sys.stdout.write(" server: read %s from client\n" % repr(data)) + if not data: + self.close() + else: + self.send(data.lower()) + + def handle_close(self): + self.close() + if support.verbose: + sys.stdout.write(" server: closed connection %s\n" % self.socket) + + def handle_error(self): + raise + + def __init__(self, certfile): + self.certfile = certfile + sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + self.port = socket_helper.bind_port(sock, '') + asyncore.dispatcher.__init__(self, sock) + self.listen(5) + + def handle_accepted(self, sock_obj, addr): + if support.verbose: + sys.stdout.write(" server: new connection from %s:%s\n" %addr) + self.ConnectionHandler(sock_obj, self.certfile) + + def handle_error(self): + raise + + def __init__(self, certfile): + self.flag = None + self.active = False + self.server = self.EchoServer(certfile) + self.port = self.server.port + threading.Thread.__init__(self) + self.daemon = True + + def __str__(self): + return "<%s %s>" % (self.__class__.__name__, self.server) + + def __enter__(self): + self.start(threading.Event()) + self.flag.wait() + return self + + def __exit__(self, *args): + if support.verbose: + sys.stdout.write(" cleanup: stopping server.\n") + self.stop() + if support.verbose: + sys.stdout.write(" cleanup: joining server thread.\n") + self.join() + if support.verbose: + sys.stdout.write(" cleanup: successfully joined.\n") + # make sure that ConnectionHandler is removed from socket_map + asyncore.close_all(ignore_all=True) + + def start (self, flag=None): + self.flag = flag + threading.Thread.start(self) + + def run(self): + self.active = True + if self.flag: + self.flag.set() + while self.active: + try: + asyncore.loop(1) + except: + pass + + def stop(self): + self.active = False + self.server.close() + +def server_params_test(client_context, server_context, indata=b"FOO\n", + chatty=True, connectionchatty=False, sni_name=None, + session=None): + """ + Launch a server, connect a client to it and try various reads + and writes. + """ + stats = {} + server = ThreadedEchoServer(context=server_context, + chatty=chatty, + connectionchatty=False) + with server: + with client_context.wrap_socket(socket.socket(), + server_hostname=sni_name, session=session) as s: + s.connect((HOST, server.port)) + for arg in [indata, bytearray(indata), memoryview(indata)]: + if connectionchatty: + if support.verbose: + sys.stdout.write( + " client: sending %r...\n" % indata) + s.write(arg) + outdata = s.read() + if connectionchatty: + if support.verbose: + sys.stdout.write(" client: read %r\n" % outdata) + if outdata != indata.lower(): + raise AssertionError( + "bad data <<%r>> (%d) received; expected <<%r>> (%d)\n" + % (outdata[:20], len(outdata), + indata[:20].lower(), len(indata))) + s.write(b"over\n") + if connectionchatty: + if support.verbose: + sys.stdout.write(" client: closing connection.\n") + stats.update({ + 'compression': s.compression(), + 'cipher': s.cipher(), + 'peercert': s.getpeercert(), + 'client_alpn_protocol': s.selected_alpn_protocol(), + 'version': s.version(), + 'session_reused': s.session_reused, + 'session': s.session, + }) + s.close() + stats['server_alpn_protocols'] = server.selected_alpn_protocols + stats['server_shared_ciphers'] = server.shared_ciphers + return stats + +def try_protocol_combo(server_protocol, client_protocol, expect_success, + certsreqs=None, server_options=0, client_options=0): + """ + Try to SSL-connect using *client_protocol* to *server_protocol*. + If *expect_success* is true, assert that the connection succeeds, + if it's false, assert that the connection fails. + Also, if *expect_success* is a string, assert that it is the protocol + version actually used by the connection. + """ + if certsreqs is None: + certsreqs = ssl.CERT_NONE + certtype = { + ssl.CERT_NONE: "CERT_NONE", + ssl.CERT_OPTIONAL: "CERT_OPTIONAL", + ssl.CERT_REQUIRED: "CERT_REQUIRED", + }[certsreqs] + if support.verbose: + formatstr = (expect_success and " %s->%s %s\n") or " {%s->%s} %s\n" + sys.stdout.write(formatstr % + (ssl.get_protocol_name(client_protocol), + ssl.get_protocol_name(server_protocol), + certtype)) + + with warnings_helper.check_warnings(): + # ignore Deprecation warnings + client_context = ssl.SSLContext(client_protocol) + client_context.options |= client_options + server_context = ssl.SSLContext(server_protocol) + server_context.options |= server_options + + min_version = PROTOCOL_TO_TLS_VERSION.get(client_protocol, None) + if (min_version is not None + # SSLContext.minimum_version is only available on recent OpenSSL + # (setter added in OpenSSL 1.1.0, getter added in OpenSSL 1.1.1) + and hasattr(server_context, 'minimum_version') + and server_protocol == ssl.PROTOCOL_TLS + and server_context.minimum_version > min_version + ): + # If OpenSSL configuration is strict and requires more recent TLS + # version, we have to change the minimum to test old TLS versions. + with warnings_helper.check_warnings(): + server_context.minimum_version = min_version + + # NOTE: we must enable "ALL" ciphers on the client, otherwise an + # SSLv23 client will send an SSLv3 hello (rather than SSLv2) + # starting from OpenSSL 1.0.0 (see issue #8322). + if client_context.protocol == ssl.PROTOCOL_TLS: + client_context.set_ciphers("ALL") + + seclevel_workaround(server_context, client_context) + + for ctx in (client_context, server_context): + ctx.verify_mode = certsreqs + ctx.load_cert_chain(SIGNED_CERTFILE) + ctx.load_verify_locations(SIGNING_CA) + try: + stats = server_params_test(client_context, server_context, + chatty=False, connectionchatty=False) + # Protocol mismatch can result in either an SSLError, or a + # "Connection reset by peer" error. + except ssl.SSLError: + if expect_success: + raise + except OSError as e: + if expect_success or e.errno != errno.ECONNRESET: + raise + else: + if not expect_success: + raise AssertionError( + "Client protocol %s succeeded with server protocol %s!" + % (ssl.get_protocol_name(client_protocol), + ssl.get_protocol_name(server_protocol))) + elif (expect_success is not True + and expect_success != stats['version']): + raise AssertionError("version mismatch: expected %r, got %r" + % (expect_success, stats['version'])) + + +def supports_kx_alias(ctx, aliases): + for cipher in ctx.get_ciphers(): + for alias in aliases: + if f"Kx={alias}" in cipher['description']: + return True + return False + + +class ThreadedTests(unittest.TestCase): + + @support.requires_resource('walltime') + def test_echo(self): + """Basic test of an SSL client connecting to a server""" + if support.verbose: + sys.stdout.write("\n") + + client_context, server_context, hostname = testing_context() + + with self.subTest(client=ssl.PROTOCOL_TLS_CLIENT, server=ssl.PROTOCOL_TLS_SERVER): + server_params_test(client_context=client_context, + server_context=server_context, + chatty=True, connectionchatty=True, + sni_name=hostname) + + client_context.check_hostname = False + with self.subTest(client=ssl.PROTOCOL_TLS_SERVER, server=ssl.PROTOCOL_TLS_CLIENT): + with self.assertRaises(ssl.SSLError) as e: + server_params_test(client_context=server_context, + server_context=client_context, + chatty=True, connectionchatty=True, + sni_name=hostname) + self.assertIn( + 'Cannot create a client socket with a PROTOCOL_TLS_SERVER context', + str(e.exception) + ) + + with self.subTest(client=ssl.PROTOCOL_TLS_SERVER, server=ssl.PROTOCOL_TLS_SERVER): + with self.assertRaises(ssl.SSLError) as e: + server_params_test(client_context=server_context, + server_context=server_context, + chatty=True, connectionchatty=True) + self.assertIn( + 'Cannot create a client socket with a PROTOCOL_TLS_SERVER context', + str(e.exception) + ) + + with self.subTest(client=ssl.PROTOCOL_TLS_CLIENT, server=ssl.PROTOCOL_TLS_CLIENT): + with self.assertRaises(ssl.SSLError) as e: + server_params_test(client_context=server_context, + server_context=client_context, + chatty=True, connectionchatty=True) + self.assertIn( + 'Cannot create a client socket with a PROTOCOL_TLS_SERVER context', + str(e.exception)) + + @unittest.skipUnless(support.Py_GIL_DISABLED, "test is only useful if the GIL is disabled") + def test_ssl_in_multiple_threads(self): + # See GH-124984: OpenSSL is not thread safe. + threads = [] + + global USE_SAME_TEST_CONTEXT + USE_SAME_TEST_CONTEXT = True + try: + for func in ( + self.test_echo, + self.test_alpn_protocols, + self.test_getpeercert, + self.test_crl_check, + functools.partial( + self.test_check_hostname_idn, + warnings_filters=False, # gh-126483 + ), + self.test_wrong_cert_tls12, + self.test_wrong_cert_tls13, + ): + # Be careful with the number of threads here. + # Too many can result in failing tests. + for num in range(5): + with self.subTest(func=func, num=num): + threads.append(Thread(target=func)) + + with threading_helper.catch_threading_exception() as cm: + for thread in threads: + with self.subTest(thread=thread): + thread.start() + + for thread in threads: + with self.subTest(thread=thread): + thread.join() + if cm.exc_value is not None: + # Some threads can skip their test + if not isinstance(cm.exc_value, unittest.SkipTest): + raise cm.exc_value + finally: + USE_SAME_TEST_CONTEXT = False + + def test_getpeercert(self): + if support.verbose: + sys.stdout.write("\n") + + client_context, server_context, hostname = testing_context() + server = ThreadedEchoServer(context=server_context, chatty=False) + with server: + with client_context.wrap_socket(socket.socket(), + do_handshake_on_connect=False, + server_hostname=hostname) as s: + s.connect((HOST, server.port)) + # getpeercert() raise ValueError while the handshake isn't + # done. + with self.assertRaises(ValueError): + s.getpeercert() + s.do_handshake() + cert = s.getpeercert() + self.assertTrue(cert, "Can't get peer certificate.") + cipher = s.cipher() + if support.verbose: + sys.stdout.write(pprint.pformat(cert) + '\n') + sys.stdout.write("Connection cipher is " + str(cipher) + '.\n') + if 'subject' not in cert: + self.fail("No subject field in certificate: %s." % + pprint.pformat(cert)) + if ((('organizationName', 'Python Software Foundation'),) + not in cert['subject']): + self.fail( + "Missing or invalid 'organizationName' field in certificate subject; " + "should be 'Python Software Foundation'.") + self.assertIn('notBefore', cert) + self.assertIn('notAfter', cert) + before = ssl.cert_time_to_seconds(cert['notBefore']) + after = ssl.cert_time_to_seconds(cert['notAfter']) + self.assertLess(before, after) + + def test_crl_check(self): + if support.verbose: + sys.stdout.write("\n") + + client_context, server_context, hostname = testing_context() + + tf = getattr(ssl, "VERIFY_X509_TRUSTED_FIRST", 0) + self.assertEqual(client_context.verify_flags, ssl.VERIFY_DEFAULT | tf) + + # VERIFY_DEFAULT should pass + server = ThreadedEchoServer(context=server_context, chatty=True) + with server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + s.connect((HOST, server.port)) + cert = s.getpeercert() + self.assertTrue(cert, "Can't get peer certificate.") + + # VERIFY_CRL_CHECK_LEAF without a loaded CRL file fails + client_context.verify_flags |= ssl.VERIFY_CRL_CHECK_LEAF + + server = ThreadedEchoServer(context=server_context, chatty=True) + # Allow for flexible libssl error messages. + regex = re.compile(r"""( + certificate verify failed # OpenSSL + | + CERTIFICATE_VERIFY_FAILED # AWS-LC + )""", re.X) + with server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + with self.assertRaisesRegex(ssl.SSLError, regex): + s.connect((HOST, server.port)) + + # now load a CRL file. The CRL file is signed by the CA. + client_context.load_verify_locations(CRLFILE) + + server = ThreadedEchoServer(context=server_context, chatty=True) + with server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + s.connect((HOST, server.port)) + cert = s.getpeercert() + self.assertTrue(cert, "Can't get peer certificate.") + + def test_check_hostname(self): + if support.verbose: + sys.stdout.write("\n") + + client_context, server_context, hostname = testing_context() + + # correct hostname should verify + server = ThreadedEchoServer(context=server_context, chatty=True) + with server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + s.connect((HOST, server.port)) + cert = s.getpeercert() + self.assertTrue(cert, "Can't get peer certificate.") + + # incorrect hostname should raise an exception + server = ThreadedEchoServer(context=server_context, chatty=True) + # Allow for flexible libssl error messages. + regex = re.compile(r"""( + certificate verify failed # OpenSSL + | + CERTIFICATE_VERIFY_FAILED # AWS-LC + )""", re.X) + with server: + with client_context.wrap_socket(socket.socket(), + server_hostname="invalid") as s: + with self.assertRaisesRegex(ssl.CertificateError, regex): + s.connect((HOST, server.port)) + + # missing server_hostname arg should cause an exception, too + server = ThreadedEchoServer(context=server_context, chatty=True) + with server: + with socket.socket() as s: + with self.assertRaisesRegex(ValueError, + "check_hostname requires server_hostname"): + client_context.wrap_socket(s) + + @unittest.skipUnless( + ssl.HAS_NEVER_CHECK_COMMON_NAME, "test requires hostname_checks_common_name" + ) + def test_hostname_checks_common_name(self): + client_context, server_context, hostname = testing_context() + assert client_context.hostname_checks_common_name + client_context.hostname_checks_common_name = False + + # default cert has a SAN + server = ThreadedEchoServer(context=server_context, chatty=True) + with server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + s.connect((HOST, server.port)) + + client_context, server_context, hostname = testing_context(NOSANFILE) + client_context.hostname_checks_common_name = False + server = ThreadedEchoServer(context=server_context, chatty=True) + with server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + with self.assertRaises(ssl.SSLCertVerificationError): + s.connect((HOST, server.port)) + + def test_ecc_cert(self): + client_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + client_context.load_verify_locations(SIGNING_CA) + client_context.set_ciphers('ECDHE:ECDSA:!NULL:!aRSA') + hostname = SIGNED_CERTFILE_ECC_HOSTNAME + + server_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + # load ECC cert + server_context.load_cert_chain(SIGNED_CERTFILE_ECC) + + # correct hostname should verify + server = ThreadedEchoServer(context=server_context, chatty=True) + with server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + s.connect((HOST, server.port)) + cert = s.getpeercert() + self.assertTrue(cert, "Can't get peer certificate.") + cipher = s.cipher()[0].split('-') + self.assertTrue(cipher[:2], ('ECDHE', 'ECDSA')) + + @unittest.skipUnless(IS_OPENSSL_3_0_0, + "test requires RFC 5280 check added in OpenSSL 3.0+") + def test_verify_strict(self): + # verification fails by default, since the server cert is non-conforming + client_context = ssl.create_default_context() + client_context.load_verify_locations(LEAF_MISSING_AKI_CA) + hostname = LEAF_MISSING_AKI_CERTFILE_HOSTNAME + + server_context = ssl.create_default_context(purpose=Purpose.CLIENT_AUTH) + server_context.load_cert_chain(LEAF_MISSING_AKI_CERTFILE) + server = ThreadedEchoServer(context=server_context, chatty=True) + with server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + with self.assertRaises(ssl.SSLError): + s.connect((HOST, server.port)) + + # explicitly disabling VERIFY_X509_STRICT allows it to succeed + client_context = ssl.create_default_context() + client_context.load_verify_locations(LEAF_MISSING_AKI_CA) + client_context.verify_flags &= ~ssl.VERIFY_X509_STRICT + + server_context = ssl.create_default_context(purpose=Purpose.CLIENT_AUTH) + server_context.load_cert_chain(LEAF_MISSING_AKI_CERTFILE) + server = ThreadedEchoServer(context=server_context, chatty=True) + with server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + s.connect((HOST, server.port)) + cert = s.getpeercert() + self.assertTrue(cert, "Can't get peer certificate.") + + def test_dual_rsa_ecc(self): + client_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + client_context.load_verify_locations(SIGNING_CA) + # TODO: fix TLSv1.3 once SSLContext can restrict signature + # algorithms. + client_context.maximum_version = ssl.TLSVersion.TLSv1_2 + # only ECDSA certs + client_context.set_ciphers('ECDHE:ECDSA:!NULL:!aRSA') + hostname = SIGNED_CERTFILE_ECC_HOSTNAME + + server_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + # load ECC and RSA key/cert pairs + server_context.load_cert_chain(SIGNED_CERTFILE_ECC) + server_context.load_cert_chain(SIGNED_CERTFILE) + + # correct hostname should verify + server = ThreadedEchoServer(context=server_context, chatty=True) + with server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + s.connect((HOST, server.port)) + cert = s.getpeercert() + self.assertTrue(cert, "Can't get peer certificate.") + cipher = s.cipher()[0].split('-') + self.assertTrue(cipher[:2], ('ECDHE', 'ECDSA')) + + def test_check_hostname_idn(self, warnings_filters=True): + if support.verbose: + sys.stdout.write("\n") + + server_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + server_context.load_cert_chain(IDNSANSFILE) + + context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + context.verify_mode = ssl.CERT_REQUIRED + context.check_hostname = True + context.load_verify_locations(SIGNING_CA) + + # correct hostname should verify, when specified in several + # different ways + idn_hostnames = [ + ('könig.idn.pythontest.net', + 'xn--knig-5qa.idn.pythontest.net'), + ('xn--knig-5qa.idn.pythontest.net', + 'xn--knig-5qa.idn.pythontest.net'), + (b'xn--knig-5qa.idn.pythontest.net', + 'xn--knig-5qa.idn.pythontest.net'), + + ('königsgäßchen.idna2003.pythontest.net', + 'xn--knigsgsschen-lcb0w.idna2003.pythontest.net'), + ('xn--knigsgsschen-lcb0w.idna2003.pythontest.net', + 'xn--knigsgsschen-lcb0w.idna2003.pythontest.net'), + (b'xn--knigsgsschen-lcb0w.idna2003.pythontest.net', + 'xn--knigsgsschen-lcb0w.idna2003.pythontest.net'), + + # ('königsgäßchen.idna2008.pythontest.net', + # 'xn--knigsgchen-b4a3dun.idna2008.pythontest.net'), + ('xn--knigsgchen-b4a3dun.idna2008.pythontest.net', + 'xn--knigsgchen-b4a3dun.idna2008.pythontest.net'), + (b'xn--knigsgchen-b4a3dun.idna2008.pythontest.net', + 'xn--knigsgchen-b4a3dun.idna2008.pythontest.net'), + + ] + for server_hostname, expected_hostname in idn_hostnames: + server = ThreadedEchoServer(context=server_context, chatty=True) + with server: + with context.wrap_socket(socket.socket(), + server_hostname=server_hostname) as s: + self.assertEqual(s.server_hostname, expected_hostname) + s.connect((HOST, server.port)) + cert = s.getpeercert() + self.assertEqual(s.server_hostname, expected_hostname) + self.assertTrue(cert, "Can't get peer certificate.") + + # incorrect hostname should raise an exception + server = ThreadedEchoServer(context=server_context, chatty=True) + with server: + with context.wrap_socket(socket.socket(), + server_hostname="python.example.org") as s: + with self.assertRaises(ssl.CertificateError): + s.connect((HOST, server.port)) + with ( + ThreadedEchoServer(context=server_context, chatty=True) as server, + ( + warnings_helper.check_no_resource_warning(self) + if warnings_filters + else nullcontext() + ), + self.assertRaises(UnicodeError), + ): + context.wrap_socket(socket.socket(), server_hostname='.pythontest.net') + + with ( + ThreadedEchoServer(context=server_context, chatty=True) as server, + ( + warnings_helper.check_no_resource_warning(self) + if warnings_filters + else nullcontext() + ), + self.assertRaises(UnicodeDecodeError), + ): + context.wrap_socket( + socket.socket(), + server_hostname=b'k\xf6nig.idn.pythontest.net', + ) + + def test_wrong_cert_tls12(self): + """Connecting when the server rejects the client's certificate + + Launch a server with CERT_REQUIRED, and check that trying to + connect to it with a wrong client certificate fails. + """ + client_context, server_context, hostname = testing_context() + # load client cert that is not signed by trusted CA + client_context.load_cert_chain(CERTFILE) + # require TLS client authentication + server_context.verify_mode = ssl.CERT_REQUIRED + # TLS 1.3 has different handshake + client_context.maximum_version = ssl.TLSVersion.TLSv1_2 + + server = ThreadedEchoServer( + context=server_context, chatty=True, connectionchatty=True, + ) + + with server, \ + client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + try: + # Expect either an SSL error about the server rejecting + # the connection, or a low-level connection reset (which + # sometimes happens on Windows) + s.connect((HOST, server.port)) + except ssl.SSLError as e: + if support.verbose: + sys.stdout.write("\nSSLError is %r\n" % e) + except OSError as e: + if e.errno != errno.ECONNRESET: + raise + if support.verbose: + sys.stdout.write("\nsocket.error is %r\n" % e) + else: + self.fail("Use of invalid cert should have failed!") + + @requires_tls_version('TLSv1_3') + def test_wrong_cert_tls13(self): + client_context, server_context, hostname = testing_context() + # load client cert that is not signed by trusted CA + client_context.load_cert_chain(CERTFILE) + server_context.verify_mode = ssl.CERT_REQUIRED + server_context.minimum_version = ssl.TLSVersion.TLSv1_3 + client_context.minimum_version = ssl.TLSVersion.TLSv1_3 + + server = ThreadedEchoServer( + context=server_context, chatty=True, connectionchatty=True, + ) + with server, \ + client_context.wrap_socket(socket.socket(), + server_hostname=hostname, + suppress_ragged_eofs=False) as s: + s.connect((HOST, server.port)) + with self.assertRaisesRegex( + OSError, + 'alert unknown ca|EOF occurred|TLSV1_ALERT_UNKNOWN_CA|' + 'closed by the remote host|Connection reset by peer|' + 'Broken pipe' + ): + # TLS 1.3 perform client cert exchange after handshake + s.write(b'data') + s.read(1000) + s.write(b'should have failed already') + s.read(1000) + + def test_rude_shutdown(self): + """A brutal shutdown of an SSL server should raise an OSError + in the client when attempting handshake. + """ + listener_ready = threading.Event() + listener_gone = threading.Event() + + s = socket.socket() + port = socket_helper.bind_port(s, HOST) + + # `listener` runs in a thread. It sits in an accept() until + # the main thread connects. Then it rudely closes the socket, + # and sets Event `listener_gone` to let the main thread know + # the socket is gone. + def listener(): + s.listen() + listener_ready.set() + newsock, addr = s.accept() + newsock.close() + s.close() + listener_gone.set() + + def connector(): + listener_ready.wait() + with socket.socket() as c: + c.connect((HOST, port)) + listener_gone.wait() + try: + ssl_sock = test_wrap_socket(c) + except OSError: + pass + else: + self.fail('connecting to closed SSL socket should have failed') + + t = threading.Thread(target=listener) + t.start() + try: + connector() + finally: + t.join() + + def test_ssl_cert_verify_error(self): + if support.verbose: + sys.stdout.write("\n") + + server_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + server_context.load_cert_chain(SIGNED_CERTFILE) + + context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + + server = ThreadedEchoServer(context=server_context, chatty=True) + with server: + with context.wrap_socket(socket.socket(), + server_hostname=SIGNED_CERTFILE_HOSTNAME) as s: + try: + s.connect((HOST, server.port)) + self.fail("Expected connection failure") + except ssl.SSLError as e: + msg = 'unable to get local issuer certificate' + self.assertIsInstance(e, ssl.SSLCertVerificationError) + self.assertEqual(e.verify_code, 20) + self.assertEqual(e.verify_message, msg) + # Allow for flexible libssl error messages. + regex = f"({msg}|CERTIFICATE_VERIFY_FAILED)" + self.assertRegex(repr(e), regex) + regex = re.compile(r"""( + certificate verify failed # OpenSSL + | + CERTIFICATE_VERIFY_FAILED # AWS-LC + )""", re.X) + self.assertRegex(repr(e), regex) + + def test_PROTOCOL_TLS(self): + """Connecting to an SSLv23 server with various client options""" + if support.verbose: + sys.stdout.write("\n") + if has_tls_version('SSLv3'): + try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_SSLv3, False) + try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_TLS, True) + if has_tls_version('TLSv1'): + try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_TLSv1, 'TLSv1') + + if has_tls_version('SSLv3'): + try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_SSLv3, False, ssl.CERT_OPTIONAL) + try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_TLS, True, ssl.CERT_OPTIONAL) + if has_tls_version('TLSv1'): + try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_TLSv1, 'TLSv1', ssl.CERT_OPTIONAL) + + if has_tls_version('SSLv3'): + try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_SSLv3, False, ssl.CERT_REQUIRED) + try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_TLS, True, ssl.CERT_REQUIRED) + if has_tls_version('TLSv1'): + try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_TLSv1, 'TLSv1', ssl.CERT_REQUIRED) + + # Server with specific SSL options + if has_tls_version('SSLv3'): + try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_SSLv3, False, + server_options=ssl.OP_NO_SSLv3) + # Will choose TLSv1 + try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_TLS, True, + server_options=ssl.OP_NO_SSLv2 | ssl.OP_NO_SSLv3) + if has_tls_version('TLSv1'): + try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_TLSv1, False, + server_options=ssl.OP_NO_TLSv1) + + @requires_tls_version('SSLv3') + def test_protocol_sslv3(self): + """Connecting to an SSLv3 server with various client options""" + if support.verbose: + sys.stdout.write("\n") + try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv3, 'SSLv3') + try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv3, 'SSLv3', ssl.CERT_OPTIONAL) + try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_SSLv3, 'SSLv3', ssl.CERT_REQUIRED) + try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_TLS, False, + client_options=ssl.OP_NO_SSLv3) + try_protocol_combo(ssl.PROTOCOL_SSLv3, ssl.PROTOCOL_TLSv1, False) + + @requires_tls_version('TLSv1') + def test_protocol_tlsv1(self): + """Connecting to a TLSv1 server with various client options""" + if support.verbose: + sys.stdout.write("\n") + try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1, 'TLSv1') + try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1, 'TLSv1', ssl.CERT_OPTIONAL) + try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1, 'TLSv1', ssl.CERT_REQUIRED) + if has_tls_version('SSLv3'): + try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_SSLv3, False) + try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLS, False, + client_options=ssl.OP_NO_TLSv1) + + @requires_tls_version('TLSv1_1') + def test_protocol_tlsv1_1(self): + """Connecting to a TLSv1.1 server with various client options. + Testing against older TLS versions.""" + if support.verbose: + sys.stdout.write("\n") + try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_TLSv1_1, 'TLSv1.1') + if has_tls_version('SSLv3'): + try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_SSLv3, False) + try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_TLS, False, + client_options=ssl.OP_NO_TLSv1_1) + + try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_TLSv1_1, 'TLSv1.1') + try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_TLSv1_2, False) + try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_TLSv1_1, False) + + @requires_tls_version('TLSv1_2') + def test_protocol_tlsv1_2(self): + """Connecting to a TLSv1.2 server with various client options. + Testing against older TLS versions.""" + if support.verbose: + sys.stdout.write("\n") + try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_TLSv1_2, 'TLSv1.2', + server_options=ssl.OP_NO_SSLv3|ssl.OP_NO_SSLv2, + client_options=ssl.OP_NO_SSLv3|ssl.OP_NO_SSLv2,) + if has_tls_version('SSLv3'): + try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_SSLv3, False) + try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_TLS, False, + client_options=ssl.OP_NO_TLSv1_2) + + try_protocol_combo(ssl.PROTOCOL_TLS, ssl.PROTOCOL_TLSv1_2, 'TLSv1.2') + if has_tls_protocol(ssl.PROTOCOL_TLSv1): + try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_TLSv1, False) + try_protocol_combo(ssl.PROTOCOL_TLSv1, ssl.PROTOCOL_TLSv1_2, False) + if has_tls_protocol(ssl.PROTOCOL_TLSv1_1): + try_protocol_combo(ssl.PROTOCOL_TLSv1_2, ssl.PROTOCOL_TLSv1_1, False) + try_protocol_combo(ssl.PROTOCOL_TLSv1_1, ssl.PROTOCOL_TLSv1_2, False) + + def test_starttls(self): + """Switching from clear text to encrypted and back again.""" + msgs = (b"msg 1", b"MSG 2", b"STARTTLS", b"MSG 3", b"msg 4", b"ENDTLS", b"msg 5", b"msg 6") + + server = ThreadedEchoServer(CERTFILE, + starttls_server=True, + chatty=True, + connectionchatty=True) + wrapped = False + with server: + s = socket.socket() + s.setblocking(True) + s.connect((HOST, server.port)) + if support.verbose: + sys.stdout.write("\n") + for indata in msgs: + if support.verbose: + sys.stdout.write( + " client: sending %r...\n" % indata) + if wrapped: + conn.write(indata) + outdata = conn.read() + else: + s.send(indata) + outdata = s.recv(1024) + msg = outdata.strip().lower() + if indata == b"STARTTLS" and msg.startswith(b"ok"): + # STARTTLS ok, switch to secure mode + if support.verbose: + sys.stdout.write( + " client: read %r from server, starting TLS...\n" + % msg) + conn = test_wrap_socket(s) + wrapped = True + elif indata == b"ENDTLS" and msg.startswith(b"ok"): + # ENDTLS ok, switch back to clear text + if support.verbose: + sys.stdout.write( + " client: read %r from server, ending TLS...\n" + % msg) + s = conn.unwrap() + wrapped = False + else: + if support.verbose: + sys.stdout.write( + " client: read %r from server\n" % msg) + if support.verbose: + sys.stdout.write(" client: closing connection.\n") + if wrapped: + conn.write(b"over\n") + else: + s.send(b"over\n") + if wrapped: + conn.close() + else: + s.close() + + def test_socketserver(self): + """Using socketserver to create and manage SSL connections.""" + server = make_https_server(self, certfile=SIGNED_CERTFILE) + # try to connect + if support.verbose: + sys.stdout.write('\n') + # Get this test file itself: + with open(__file__, 'rb') as f: + d1 = f.read() + d2 = '' + # now fetch the same data from the HTTPS server + url = f'https://localhost:{server.port}/test_ssl.py' + context = ssl.create_default_context(cafile=SIGNING_CA) + f = urllib.request.urlopen(url, context=context) + try: + dlen = f.info().get("content-length") + if dlen and (int(dlen) > 0): + d2 = f.read(int(dlen)) + if support.verbose: + sys.stdout.write( + " client: read %d bytes from remote server '%s'\n" + % (len(d2), server)) + finally: + f.close() + self.assertEqual(d1, d2) + + def test_asyncore_server(self): + """Check the example asyncore integration.""" + if support.verbose: + sys.stdout.write("\n") + + indata = b"FOO\n" + server = AsyncoreEchoServer(CERTFILE) + with server: + s = test_wrap_socket(socket.socket()) + s.connect(('127.0.0.1', server.port)) + if support.verbose: + sys.stdout.write( + " client: sending %r...\n" % indata) + s.write(indata) + outdata = s.read() + if support.verbose: + sys.stdout.write(" client: read %r\n" % outdata) + if outdata != indata.lower(): + self.fail( + "bad data <<%r>> (%d) received; expected <<%r>> (%d)\n" + % (outdata[:20], len(outdata), + indata[:20].lower(), len(indata))) + s.write(b"over\n") + if support.verbose: + sys.stdout.write(" client: closing connection.\n") + s.close() + if support.verbose: + sys.stdout.write(" client: connection closed.\n") + + def test_recv_send(self): + """Test recv(), send() and friends.""" + if support.verbose: + sys.stdout.write("\n") + + server = ThreadedEchoServer(CERTFILE, + certreqs=ssl.CERT_NONE, + ssl_version=ssl.PROTOCOL_TLS_SERVER, + cacerts=CERTFILE, + chatty=True, + connectionchatty=False) + with server: + s = test_wrap_socket(socket.socket(), + server_side=False, + certfile=CERTFILE, + ca_certs=CERTFILE, + cert_reqs=ssl.CERT_NONE) + s.connect((HOST, server.port)) + # helper methods for standardising recv* method signatures + def _recv_into(): + b = bytearray(b"\0"*100) + count = s.recv_into(b) + return b[:count] + + def _recvfrom_into(): + b = bytearray(b"\0"*100) + count, addr = s.recvfrom_into(b) + return b[:count] + + # (name, method, expect success?, *args, return value func) + send_methods = [ + ('send', s.send, True, [], len), + ('sendto', s.sendto, False, ["some.address"], len), + ('sendall', s.sendall, True, [], lambda x: None), + ] + # (name, method, whether to expect success, *args) + recv_methods = [ + ('recv', s.recv, True, []), + ('recvfrom', s.recvfrom, False, ["some.address"]), + ('recv_into', _recv_into, True, []), + ('recvfrom_into', _recvfrom_into, False, []), + ] + data_prefix = "PREFIX_" + + for (meth_name, send_meth, expect_success, args, + ret_val_meth) in send_methods: + indata = (data_prefix + meth_name).encode('ascii') + try: + ret = send_meth(indata, *args) + msg = "sending with {}".format(meth_name) + self.assertEqual(ret, ret_val_meth(indata), msg=msg) + outdata = s.read() + if outdata != indata.lower(): + self.fail( + "While sending with <<{name:s}>> bad data " + "<<{outdata:r}>> ({nout:d}) received; " + "expected <<{indata:r}>> ({nin:d})\n".format( + name=meth_name, outdata=outdata[:20], + nout=len(outdata), + indata=indata[:20], nin=len(indata) + ) + ) + except ValueError as e: + if expect_success: + self.fail( + "Failed to send with method <<{name:s}>>; " + "expected to succeed.\n".format(name=meth_name) + ) + if not str(e).startswith(meth_name): + self.fail( + "Method <<{name:s}>> failed with unexpected " + "exception message: {exp:s}\n".format( + name=meth_name, exp=e + ) + ) + + for meth_name, recv_meth, expect_success, args in recv_methods: + indata = (data_prefix + meth_name).encode('ascii') + try: + s.send(indata) + outdata = recv_meth(*args) + if outdata != indata.lower(): + self.fail( + "While receiving with <<{name:s}>> bad data " + "<<{outdata:r}>> ({nout:d}) received; " + "expected <<{indata:r}>> ({nin:d})\n".format( + name=meth_name, outdata=outdata[:20], + nout=len(outdata), + indata=indata[:20], nin=len(indata) + ) + ) + except ValueError as e: + if expect_success: + self.fail( + "Failed to receive with method <<{name:s}>>; " + "expected to succeed.\n".format(name=meth_name) + ) + if not str(e).startswith(meth_name): + self.fail( + "Method <<{name:s}>> failed with unexpected " + "exception message: {exp:s}\n".format( + name=meth_name, exp=e + ) + ) + # consume data + s.read() + + # read(-1, buffer) is supported, even though read(-1) is not + data = b"data" + s.send(data) + buffer = bytearray(len(data)) + self.assertEqual(s.read(-1, buffer), len(data)) + self.assertEqual(buffer, data) + + # sendall accepts bytes-like objects + if ctypes is not None: + ubyte = ctypes.c_ubyte * len(data) + byteslike = ubyte.from_buffer_copy(data) + s.sendall(byteslike) + self.assertEqual(s.read(), data) + + # Make sure sendmsg et al are disallowed to avoid + # inadvertent disclosure of data and/or corruption + # of the encrypted data stream + self.assertRaises(NotImplementedError, s.dup) + self.assertRaises(NotImplementedError, s.sendmsg, [b"data"]) + self.assertRaises(NotImplementedError, s.recvmsg, 100) + self.assertRaises(NotImplementedError, + s.recvmsg_into, [bytearray(100)]) + s.write(b"over\n") + + self.assertRaises(ValueError, s.recv, -1) + self.assertRaises(ValueError, s.read, -1) + + s.close() + + def test_recv_zero(self): + server = ThreadedEchoServer(CERTFILE) + self.enterContext(server) + s = socket.create_connection((HOST, server.port)) + self.addCleanup(s.close) + s = test_wrap_socket(s, suppress_ragged_eofs=False) + self.addCleanup(s.close) + + # recv/read(0) should return no data + s.send(b"data") + self.assertEqual(s.recv(0), b"") + self.assertEqual(s.read(0), b"") + self.assertEqual(s.read(), b"data") + + # Should not block if the other end sends no data + s.setblocking(False) + self.assertEqual(s.recv(0), b"") + self.assertEqual(s.recv_into(bytearray()), 0) + + def test_recv_into_buffer_protocol_len(self): + server = ThreadedEchoServer(CERTFILE) + self.enterContext(server) + s = socket.create_connection((HOST, server.port)) + self.addCleanup(s.close) + s = test_wrap_socket(s, suppress_ragged_eofs=False) + self.addCleanup(s.close) + + s.send(b"data") + buf = array.array('I', [0, 0]) + self.assertEqual(s.recv_into(buf), 4) + self.assertEqual(bytes(buf)[:4], b"data") + + class B(bytearray): + def __len__(self): + 1/0 + s.send(b"data") + buf = B(6) + self.assertEqual(s.recv_into(buf), 4) + self.assertEqual(bytes(buf), b"data\0\0") + + def test_nonblocking_send(self): + server = ThreadedEchoServer(CERTFILE, + certreqs=ssl.CERT_NONE, + ssl_version=ssl.PROTOCOL_TLS_SERVER, + cacerts=CERTFILE, + chatty=True, + connectionchatty=False) + with server: + s = test_wrap_socket(socket.socket(), + server_side=False, + certfile=CERTFILE, + ca_certs=CERTFILE, + cert_reqs=ssl.CERT_NONE) + s.connect((HOST, server.port)) + s.setblocking(False) + + # If we keep sending data, at some point the buffers + # will be full and the call will block + buf = bytearray(8192) + def fill_buffer(): + while True: + s.send(buf) + self.assertRaises((ssl.SSLWantWriteError, + ssl.SSLWantReadError), fill_buffer) + + # Now read all the output and discard it + s.setblocking(True) + s.close() + + def test_handshake_timeout(self): + # Issue #5103: SSL handshake must respect the socket timeout + server = socket.socket(socket.AF_INET) + host = "127.0.0.1" + port = socket_helper.bind_port(server) + started = threading.Event() + finish = False + + def serve(): + server.listen() + started.set() + conns = [] + while not finish: + r, w, e = select.select([server], [], [], 0.1) + if server in r: + # Let the socket hang around rather than having + # it closed by garbage collection. + conns.append(server.accept()[0]) + for sock in conns: + sock.close() + + t = threading.Thread(target=serve) + t.start() + started.wait() + + try: + try: + c = socket.socket(socket.AF_INET) + c.settimeout(0.2) + c.connect((host, port)) + # Will attempt handshake and time out + self.assertRaisesRegex(TimeoutError, "timed out", + test_wrap_socket, c) + finally: + c.close() + try: + c = socket.socket(socket.AF_INET) + c = test_wrap_socket(c) + c.settimeout(0.2) + # Will attempt handshake and time out + self.assertRaisesRegex(TimeoutError, "timed out", + c.connect, (host, port)) + finally: + c.close() + finally: + finish = True + t.join() + server.close() + + def test_server_accept(self): + # Issue #16357: accept() on a SSLSocket created through + # SSLContext.wrap_socket(). + client_ctx, server_ctx, hostname = testing_context() + server = socket.socket(socket.AF_INET) + host = "127.0.0.1" + port = socket_helper.bind_port(server) + server = server_ctx.wrap_socket(server, server_side=True) + self.assertTrue(server.server_side) + + evt = threading.Event() + remote = None + peer = None + def serve(): + nonlocal remote, peer + server.listen() + # Block on the accept and wait on the connection to close. + evt.set() + remote, peer = server.accept() + remote.send(remote.recv(4)) + + t = threading.Thread(target=serve) + t.start() + # Client wait until server setup and perform a connect. + evt.wait() + client = client_ctx.wrap_socket( + socket.socket(), server_hostname=hostname + ) + client.connect((hostname, port)) + client.send(b'data') + client.recv() + client_addr = client.getsockname() + client.close() + t.join() + remote.close() + server.close() + # Sanity checks. + self.assertIsInstance(remote, ssl.SSLSocket) + self.assertEqual(peer, client_addr) + + def test_getpeercert_enotconn(self): + context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + context.check_hostname = False + with context.wrap_socket(socket.socket()) as sock: + with self.assertRaises(OSError) as cm: + sock.getpeercert() + self.assertEqual(cm.exception.errno, errno.ENOTCONN) + + def test_do_handshake_enotconn(self): + context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + context.check_hostname = False + with context.wrap_socket(socket.socket()) as sock: + with self.assertRaises(OSError) as cm: + sock.do_handshake() + self.assertEqual(cm.exception.errno, errno.ENOTCONN) + + def test_no_shared_ciphers(self): + client_context, server_context, hostname = testing_context() + # OpenSSL enables all TLS 1.3 ciphers, enforce TLS 1.2 for test + client_context.maximum_version = ssl.TLSVersion.TLSv1_2 + # Force different suites on client and server + client_context.set_ciphers("AES128") + server_context.set_ciphers("AES256") + with ThreadedEchoServer(context=server_context) as server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + with self.assertRaises(OSError): + s.connect((HOST, server.port)) + self.assertIn("NO_SHARED_CIPHER", server.conn_errors[0]) + + def test_version_basic(self): + """ + Basic tests for SSLSocket.version(). + More tests are done in the test_protocol_*() methods. + """ + context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + context.check_hostname = False + context.verify_mode = ssl.CERT_NONE + with ThreadedEchoServer(CERTFILE, + ssl_version=ssl.PROTOCOL_TLS_SERVER, + chatty=False) as server: + with context.wrap_socket(socket.socket()) as s: + self.assertIs(s.version(), None) + self.assertIs(s._sslobj, None) + s.connect((HOST, server.port)) + self.assertEqual(s.version(), 'TLSv1.3') + self.assertIs(s._sslobj, None) + self.assertIs(s.version(), None) + + @requires_tls_version('TLSv1_3') + def test_tls1_3(self): + client_context, server_context, hostname = testing_context() + client_context.minimum_version = ssl.TLSVersion.TLSv1_3 + with ThreadedEchoServer(context=server_context) as server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + s.connect((HOST, server.port)) + self.assertIn(s.cipher()[0], { + 'TLS_AES_256_GCM_SHA384', + 'TLS_CHACHA20_POLY1305_SHA256', + 'TLS_AES_128_GCM_SHA256', + }) + self.assertEqual(s.version(), 'TLSv1.3') + + @requires_tls_version('TLSv1_2') + @requires_tls_version('TLSv1') + @ignore_deprecation + def test_min_max_version_tlsv1_2(self): + client_context, server_context, hostname = testing_context() + # client TLSv1.0 to 1.2 + client_context.minimum_version = ssl.TLSVersion.TLSv1 + client_context.maximum_version = ssl.TLSVersion.TLSv1_2 + # server only TLSv1.2 + server_context.minimum_version = ssl.TLSVersion.TLSv1_2 + server_context.maximum_version = ssl.TLSVersion.TLSv1_2 + + with ThreadedEchoServer(context=server_context) as server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + s.connect((HOST, server.port)) + self.assertEqual(s.version(), 'TLSv1.2') + + @requires_tls_version('TLSv1_1') + @ignore_deprecation + def test_min_max_version_tlsv1_1(self): + client_context, server_context, hostname = testing_context() + # client 1.0 to 1.2, server 1.0 to 1.1 + client_context.minimum_version = ssl.TLSVersion.TLSv1 + client_context.maximum_version = ssl.TLSVersion.TLSv1_2 + server_context.minimum_version = ssl.TLSVersion.TLSv1 + server_context.maximum_version = ssl.TLSVersion.TLSv1_1 + seclevel_workaround(client_context, server_context) + + with ThreadedEchoServer(context=server_context) as server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + s.connect((HOST, server.port)) + self.assertEqual(s.version(), 'TLSv1.1') + + @requires_tls_version('TLSv1_2') + @requires_tls_version('TLSv1') + @ignore_deprecation + def test_min_max_version_mismatch(self): + client_context, server_context, hostname = testing_context() + # client 1.0, server 1.2 (mismatch) + server_context.maximum_version = ssl.TLSVersion.TLSv1_2 + server_context.minimum_version = ssl.TLSVersion.TLSv1_2 + client_context.maximum_version = ssl.TLSVersion.TLSv1 + client_context.minimum_version = ssl.TLSVersion.TLSv1 + seclevel_workaround(client_context, server_context) + + with ThreadedEchoServer(context=server_context) as server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + with self.assertRaises(ssl.SSLError) as e: + s.connect((HOST, server.port)) + self.assertRegex(str(e.exception), "(alert|ALERT)") + + @requires_tls_version('SSLv3') + def test_min_max_version_sslv3(self): + client_context, server_context, hostname = testing_context() + server_context.minimum_version = ssl.TLSVersion.SSLv3 + client_context.minimum_version = ssl.TLSVersion.SSLv3 + client_context.maximum_version = ssl.TLSVersion.SSLv3 + seclevel_workaround(client_context, server_context) + + with ThreadedEchoServer(context=server_context) as server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + s.connect((HOST, server.port)) + self.assertEqual(s.version(), 'SSLv3') + + def test_default_ecdh_curve(self): + # Issue #21015: elliptic curve-based Diffie Hellman key exchange + # should be enabled by default on SSL contexts. + client_context, server_context, hostname = testing_context() + # TLSv1.3 defaults to PFS key agreement and no longer has KEA in + # cipher name. + client_context.maximum_version = ssl.TLSVersion.TLSv1_2 + # Prior to OpenSSL 1.0.0, ECDH ciphers have to be enabled + # explicitly using the 'ECCdraft' cipher alias. Otherwise, + # our default cipher list should prefer ECDH-based ciphers + # automatically. + with ThreadedEchoServer(context=server_context) as server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + s.connect((HOST, server.port)) + self.assertIn("ECDH", s.cipher()[0]) + + @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.skipUnless("tls-unique" in ssl.CHANNEL_BINDING_TYPES, + "'tls-unique' channel binding not available") + def test_tls_unique_channel_binding(self): + """Test tls-unique channel binding.""" + if support.verbose: + sys.stdout.write("\n") + + client_context, server_context, hostname = testing_context() + + # tls-unique is not defined for TLSv1.3 + # https://datatracker.ietf.org/doc/html/rfc8446#appendix-C.5 + client_context.maximum_version = ssl.TLSVersion.TLSv1_2 + + server = ThreadedEchoServer(context=server_context, + chatty=True, + connectionchatty=False) + + with server: + with client_context.wrap_socket( + socket.socket(), + server_hostname=hostname) as s: + s.connect((HOST, server.port)) + # get the data + cb_data = s.get_channel_binding("tls-unique") + if support.verbose: + sys.stdout.write( + " got channel binding data: {0!r}\n".format(cb_data)) + + # check if it is sane + self.assertIsNotNone(cb_data) + if s.version() == 'TLSv1.3': + self.assertEqual(len(cb_data), 48) + else: + self.assertEqual(len(cb_data), 12) # True for TLSv1 + + # and compare with the peers version + s.write(b"CB tls-unique\n") + peer_data_repr = s.read().strip() + self.assertEqual(peer_data_repr, + repr(cb_data).encode("us-ascii")) + + # now, again + with client_context.wrap_socket( + socket.socket(), + server_hostname=hostname) as s: + s.connect((HOST, server.port)) + new_cb_data = s.get_channel_binding("tls-unique") + if support.verbose: + sys.stdout.write( + "got another channel binding data: {0!r}\n".format( + new_cb_data) + ) + # is it really unique + self.assertNotEqual(cb_data, new_cb_data) + self.assertIsNotNone(cb_data) + if s.version() == 'TLSv1.3': + self.assertEqual(len(cb_data), 48) + else: + self.assertEqual(len(cb_data), 12) # True for TLSv1 + s.write(b"CB tls-unique\n") + peer_data_repr = s.read().strip() + self.assertEqual(peer_data_repr, + repr(new_cb_data).encode("us-ascii")) + + def test_compression(self): + client_context, server_context, hostname = testing_context() + stats = server_params_test(client_context, server_context, + chatty=True, connectionchatty=True, + sni_name=hostname) + if support.verbose: + sys.stdout.write(" got compression: {!r}\n".format(stats['compression'])) + self.assertIn(stats['compression'], { None, 'ZLIB', 'RLE' }) + + @unittest.skipUnless(hasattr(ssl, 'OP_NO_COMPRESSION'), + "ssl.OP_NO_COMPRESSION needed for this test") + def test_compression_disabled(self): + client_context, server_context, hostname = testing_context() + client_context.options |= ssl.OP_NO_COMPRESSION + server_context.options |= ssl.OP_NO_COMPRESSION + stats = server_params_test(client_context, server_context, + chatty=True, connectionchatty=True, + sni_name=hostname) + self.assertIs(stats['compression'], None) + + def test_legacy_server_connect(self): + client_context, server_context, hostname = testing_context() + client_context.options |= ssl.OP_LEGACY_SERVER_CONNECT + server_params_test(client_context, server_context, + chatty=True, connectionchatty=True, + sni_name=hostname) + + def test_no_legacy_server_connect(self): + client_context, server_context, hostname = testing_context() + client_context.options &= ~ssl.OP_LEGACY_SERVER_CONNECT + server_params_test(client_context, server_context, + chatty=True, connectionchatty=True, + sni_name=hostname) + + def test_dh_params(self): + # Check we can get a connection with ephemeral finite-field + # Diffie-Hellman (if supported). + client_context, server_context, hostname = testing_context() + dhe_aliases = {"ADH", "EDH", "DHE"} + if not (supports_kx_alias(client_context, dhe_aliases) + and supports_kx_alias(server_context, dhe_aliases)): + self.skipTest("libssl doesn't support ephemeral DH") + # test scenario needs TLS <= 1.2 + client_context.maximum_version = ssl.TLSVersion.TLSv1_2 + try: + server_context.load_dh_params(DHFILE) + except RuntimeError: + if Py_DEBUG_WIN32: + self.skipTest("not supported on Win32 debug build") + raise + server_context.set_ciphers("kEDH") + server_context.maximum_version = ssl.TLSVersion.TLSv1_2 + stats = server_params_test(client_context, server_context, + chatty=True, connectionchatty=True, + sni_name=hostname) + cipher = stats["cipher"][0] + parts = cipher.split("-") + if not dhe_aliases.intersection(parts): + self.fail("Non-DH key exchange: " + cipher[0]) + + def test_ecdh_curve(self): + # server secp384r1, client auto + client_context, server_context, hostname = testing_context() + + server_context.set_ecdh_curve("secp384r1") + server_context.set_ciphers("ECDHE:!eNULL:!aNULL") + server_context.minimum_version = ssl.TLSVersion.TLSv1_2 + stats = server_params_test(client_context, server_context, + chatty=True, connectionchatty=True, + sni_name=hostname) + + # server auto, client secp384r1 + client_context, server_context, hostname = testing_context() + client_context.set_ecdh_curve("secp384r1") + server_context.set_ciphers("ECDHE:!eNULL:!aNULL") + server_context.minimum_version = ssl.TLSVersion.TLSv1_2 + stats = server_params_test(client_context, server_context, + chatty=True, connectionchatty=True, + sni_name=hostname) + + # server / client curve mismatch + client_context, server_context, hostname = testing_context() + client_context.set_ecdh_curve("prime256v1") + server_context.set_ecdh_curve("secp384r1") + server_context.set_ciphers("ECDHE:!eNULL:!aNULL") + server_context.minimum_version = ssl.TLSVersion.TLSv1_2 + with self.assertRaises(ssl.SSLError): + server_params_test(client_context, server_context, + chatty=True, connectionchatty=True, + sni_name=hostname) + + def test_selected_alpn_protocol(self): + # selected_alpn_protocol() is None unless ALPN is used. + client_context, server_context, hostname = testing_context() + stats = server_params_test(client_context, server_context, + chatty=True, connectionchatty=True, + sni_name=hostname) + self.assertIs(stats['client_alpn_protocol'], None) + + def test_selected_alpn_protocol_if_server_uses_alpn(self): + # selected_alpn_protocol() is None unless ALPN is used by the client. + client_context, server_context, hostname = testing_context() + server_context.set_alpn_protocols(['foo', 'bar']) + stats = server_params_test(client_context, server_context, + chatty=True, connectionchatty=True, + sni_name=hostname) + self.assertIs(stats['client_alpn_protocol'], None) + + def test_alpn_protocols(self): + server_protocols = ['foo', 'bar', 'milkshake'] + protocol_tests = [ + (['foo', 'bar'], 'foo'), + (['bar', 'foo'], 'foo'), + (['milkshake'], 'milkshake'), + (['http/3.0', 'http/4.0'], None) + ] + for client_protocols, expected in protocol_tests: + client_context, server_context, hostname = testing_context() + server_context.set_alpn_protocols(server_protocols) + client_context.set_alpn_protocols(client_protocols) + + try: + stats = server_params_test(client_context, + server_context, + chatty=True, + connectionchatty=True, + sni_name=hostname) + except ssl.SSLError as e: + stats = e + + msg = "failed trying %s (s) and %s (c).\n" \ + "was expecting %s, but got %%s from the %%s" \ + % (str(server_protocols), str(client_protocols), + str(expected)) + client_result = stats['client_alpn_protocol'] + self.assertEqual(client_result, expected, + msg % (client_result, "client")) + server_result = stats['server_alpn_protocols'][-1] \ + if len(stats['server_alpn_protocols']) else 'nothing' + self.assertEqual(server_result, expected, + msg % (server_result, "server")) + + def test_npn_protocols(self): + assert not ssl.HAS_NPN + + def sni_contexts(self): + server_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + server_context.load_cert_chain(SIGNED_CERTFILE) + other_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + other_context.load_cert_chain(SIGNED_CERTFILE2) + client_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + client_context.load_verify_locations(SIGNING_CA) + return server_context, other_context, client_context + + def check_common_name(self, stats, name): + cert = stats['peercert'] + self.assertIn((('commonName', name),), cert['subject']) + + def test_sni_callback(self): + calls = [] + server_context, other_context, client_context = self.sni_contexts() + + client_context.check_hostname = False + + def servername_cb(ssl_sock, server_name, initial_context): + calls.append((server_name, initial_context)) + if server_name is not None: + ssl_sock.context = other_context + server_context.set_servername_callback(servername_cb) + + stats = server_params_test(client_context, server_context, + chatty=True, + sni_name='supermessage') + # The hostname was fetched properly, and the certificate was + # changed for the connection. + self.assertEqual(calls, [("supermessage", server_context)]) + # CERTFILE4 was selected + self.check_common_name(stats, 'fakehostname') + + calls = [] + # The callback is called with server_name=None + stats = server_params_test(client_context, server_context, + chatty=True, + sni_name=None) + self.assertEqual(calls, [(None, server_context)]) + self.check_common_name(stats, SIGNED_CERTFILE_HOSTNAME) + + # Check disabling the callback + calls = [] + server_context.set_servername_callback(None) + + stats = server_params_test(client_context, server_context, + chatty=True, + sni_name='notfunny') + # Certificate didn't change + self.check_common_name(stats, SIGNED_CERTFILE_HOSTNAME) + self.assertEqual(calls, []) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_sni_callback_alert(self): + # Returning a TLS alert is reflected to the connecting client + server_context, other_context, client_context = self.sni_contexts() + + def cb_returning_alert(ssl_sock, server_name, initial_context): + return ssl.ALERT_DESCRIPTION_ACCESS_DENIED + server_context.set_servername_callback(cb_returning_alert) + with self.assertRaises(ssl.SSLError) as cm: + stats = server_params_test(client_context, server_context, + chatty=False, + sni_name='supermessage') + self.assertEqual(cm.exception.reason, 'TLSV1_ALERT_ACCESS_DENIED') + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_sni_callback_raising(self): + # Raising fails the connection with a TLS handshake failure alert. + server_context, other_context, client_context = self.sni_contexts() + + def cb_raising(ssl_sock, server_name, initial_context): + 1/0 + server_context.set_servername_callback(cb_raising) + + with support.catch_unraisable_exception() as catch: + with self.assertRaises(ssl.SSLError) as cm: + stats = server_params_test(client_context, server_context, + chatty=False, + sni_name='supermessage') + + # Allow for flexible libssl error messages. + regex = "(SSLV3_ALERT_HANDSHAKE_FAILURE|NO_PRIVATE_VALUE)" + self.assertRegex(cm.exception.reason, regex) + self.assertEqual(catch.unraisable.exc_type, ZeroDivisionError) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_sni_callback_wrong_return_type(self): + # Returning the wrong return type terminates the TLS connection + # with an internal error alert. + server_context, other_context, client_context = self.sni_contexts() + + def cb_wrong_return_type(ssl_sock, server_name, initial_context): + return "foo" + server_context.set_servername_callback(cb_wrong_return_type) + + with support.catch_unraisable_exception() as catch: + with self.assertRaises(ssl.SSLError) as cm: + stats = server_params_test(client_context, server_context, + chatty=False, + sni_name='supermessage') + + + self.assertEqual(cm.exception.reason, 'TLSV1_ALERT_INTERNAL_ERROR') + self.assertEqual(catch.unraisable.exc_type, TypeError) + + def test_shared_ciphers(self): + client_context, server_context, hostname = testing_context() + client_context.set_ciphers("AES128:AES256") + server_context.set_ciphers("AES256:eNULL") + expected_algs = [ + "AES256", "AES-256", + # TLS 1.3 ciphers are always enabled + "TLS_CHACHA20", "TLS_AES", + ] + + stats = server_params_test(client_context, server_context, + sni_name=hostname) + ciphers = stats['server_shared_ciphers'][0] + self.assertGreater(len(ciphers), 0) + for name, tls_version, bits in ciphers: + if not any(alg in name for alg in expected_algs): + self.fail(name) + + def test_read_write_after_close_raises_valuerror(self): + client_context, server_context, hostname = testing_context() + server = ThreadedEchoServer(context=server_context, chatty=False) + + with server: + s = client_context.wrap_socket(socket.socket(), + server_hostname=hostname) + s.connect((HOST, server.port)) + s.close() + + self.assertRaises(ValueError, s.read, 1024) + self.assertRaises(ValueError, s.write, b'hello') + + def test_sendfile(self): + TEST_DATA = b"x" * 512 + with open(os_helper.TESTFN, 'wb') as f: + f.write(TEST_DATA) + self.addCleanup(os_helper.unlink, os_helper.TESTFN) + client_context, server_context, hostname = testing_context() + server = ThreadedEchoServer(context=server_context, chatty=False) + with server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + s.connect((HOST, server.port)) + with open(os_helper.TESTFN, 'rb') as file: + s.sendfile(file) + self.assertEqual(s.recv(1024), TEST_DATA) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_session(self): + client_context, server_context, hostname = testing_context() + # TODO: sessions aren't compatible with TLSv1.3 yet + client_context.maximum_version = ssl.TLSVersion.TLSv1_2 + + # first connection without session + stats = server_params_test(client_context, server_context, + sni_name=hostname) + session = stats['session'] + self.assertTrue(session.id) + self.assertGreater(session.time, 0) + self.assertGreater(session.timeout, 0) + self.assertTrue(session.has_ticket) + self.assertGreater(session.ticket_lifetime_hint, 0) + self.assertFalse(stats['session_reused']) + sess_stat = server_context.session_stats() + self.assertEqual(sess_stat['accept'], 1) + self.assertEqual(sess_stat['hits'], 0) + + # reuse session + stats = server_params_test(client_context, server_context, + session=session, sni_name=hostname) + sess_stat = server_context.session_stats() + self.assertEqual(sess_stat['accept'], 2) + self.assertEqual(sess_stat['hits'], 1) + self.assertTrue(stats['session_reused']) + session2 = stats['session'] + self.assertEqual(session2.id, session.id) + self.assertEqual(session2, session) + self.assertIsNot(session2, session) + self.assertGreaterEqual(session2.time, session.time) + self.assertGreaterEqual(session2.timeout, session.timeout) + + # another one without session + stats = server_params_test(client_context, server_context, + sni_name=hostname) + self.assertFalse(stats['session_reused']) + session3 = stats['session'] + self.assertNotEqual(session3.id, session.id) + self.assertNotEqual(session3, session) + sess_stat = server_context.session_stats() + self.assertEqual(sess_stat['accept'], 3) + self.assertEqual(sess_stat['hits'], 1) + + # reuse session again + stats = server_params_test(client_context, server_context, + session=session, sni_name=hostname) + self.assertTrue(stats['session_reused']) + session4 = stats['session'] + self.assertEqual(session4.id, session.id) + self.assertEqual(session4, session) + self.assertGreaterEqual(session4.time, session.time) + self.assertGreaterEqual(session4.timeout, session.timeout) + sess_stat = server_context.session_stats() + self.assertEqual(sess_stat['accept'], 4) + self.assertEqual(sess_stat['hits'], 2) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_session_handling(self): + client_context, server_context, hostname = testing_context() + client_context2, _, _ = testing_context() + + # TODO: session reuse does not work with TLSv1.3 + client_context.maximum_version = ssl.TLSVersion.TLSv1_2 + client_context2.maximum_version = ssl.TLSVersion.TLSv1_2 + + server = ThreadedEchoServer(context=server_context, chatty=False) + with server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + # session is None before handshake + self.assertEqual(s.session, None) + self.assertEqual(s.session_reused, None) + s.connect((HOST, server.port)) + session = s.session + self.assertTrue(session) + with self.assertRaises(TypeError) as e: + s.session = object + self.assertEqual(str(e.exception), 'Value is not a SSLSession.') + + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + s.connect((HOST, server.port)) + # cannot set session after handshake + with self.assertRaises(ValueError) as e: + s.session = session + self.assertEqual(str(e.exception), + 'Cannot set session after handshake.') + + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + # can set session before handshake and before the + # connection was established + s.session = session + s.connect((HOST, server.port)) + self.assertEqual(s.session.id, session.id) + self.assertEqual(s.session, session) + self.assertEqual(s.session_reused, True) + + with client_context2.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + # cannot re-use session with a different SSLContext + with self.assertRaises(ValueError) as e: + s.session = session + s.connect((HOST, server.port)) + self.assertEqual(str(e.exception), + 'Session refers to a different SSLContext.') + + @requires_tls_version('TLSv1_2') + @unittest.skipUnless(ssl.HAS_PSK, 'TLS-PSK disabled on this OpenSSL build') + def test_psk(self): + psk = bytes.fromhex('deadbeef') + + client_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + client_context.check_hostname = False + client_context.verify_mode = ssl.CERT_NONE + client_context.maximum_version = ssl.TLSVersion.TLSv1_2 + client_context.set_ciphers('PSK') + client_context.set_psk_client_callback(lambda hint: (None, psk)) + + server_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + server_context.maximum_version = ssl.TLSVersion.TLSv1_2 + server_context.set_ciphers('PSK') + server_context.set_psk_server_callback(lambda identity: psk) + + # correct PSK should connect + server = ThreadedEchoServer(context=server_context) + with server: + with client_context.wrap_socket(socket.socket()) as s: + s.connect((HOST, server.port)) + + # incorrect PSK should fail + incorrect_psk = bytes.fromhex('cafebabe') + client_context.set_psk_client_callback(lambda hint: (None, incorrect_psk)) + server = ThreadedEchoServer(context=server_context) + with server: + with client_context.wrap_socket(socket.socket()) as s: + with self.assertRaises(ssl.SSLError): + s.connect((HOST, server.port)) + + # identity_hint and client_identity should be sent to the other side + identity_hint = 'identity-hint' + client_identity = 'client-identity' + + def client_callback(hint): + self.assertEqual(hint, identity_hint) + return client_identity, psk + + def server_callback(identity): + self.assertEqual(identity, client_identity) + return psk + + client_context.set_psk_client_callback(client_callback) + server_context.set_psk_server_callback(server_callback, identity_hint) + server = ThreadedEchoServer(context=server_context) + with server: + with client_context.wrap_socket(socket.socket()) as s: + s.connect((HOST, server.port)) + + # adding client callback to server or vice versa raises an exception + with self.assertRaisesRegex(ssl.SSLError, 'Cannot add PSK server callback'): + client_context.set_psk_server_callback(server_callback, identity_hint) + with self.assertRaisesRegex(ssl.SSLError, 'Cannot add PSK client callback'): + server_context.set_psk_client_callback(client_callback) + + # test with UTF-8 identities + identity_hint = '身份暗示' # Translation: "Identity hint" + client_identity = '客户身份' # Translation: "Customer identity" + + client_context.set_psk_client_callback(client_callback) + server_context.set_psk_server_callback(server_callback, identity_hint) + server = ThreadedEchoServer(context=server_context) + with server: + with client_context.wrap_socket(socket.socket()) as s: + s.connect((HOST, server.port)) + + @requires_tls_version('TLSv1_3') + @unittest.skipUnless(ssl.HAS_PSK, 'TLS-PSK disabled on this OpenSSL build') + def test_psk_tls1_3(self): + psk = bytes.fromhex('deadbeef') + identity_hint = 'identity-hint' + client_identity = 'client-identity' + + def client_callback(hint): + # identity_hint is not sent to the client in TLS 1.3 + self.assertIsNone(hint) + return client_identity, psk + + def server_callback(identity): + self.assertEqual(identity, client_identity) + return psk + + client_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + client_context.check_hostname = False + client_context.verify_mode = ssl.CERT_NONE + client_context.minimum_version = ssl.TLSVersion.TLSv1_3 + client_context.set_ciphers('PSK') + client_context.set_psk_client_callback(client_callback) + + server_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + server_context.minimum_version = ssl.TLSVersion.TLSv1_3 + server_context.set_ciphers('PSK') + server_context.set_psk_server_callback(server_callback, identity_hint) + + server = ThreadedEchoServer(context=server_context) + with server: + with client_context.wrap_socket(socket.socket()) as s: + s.connect((HOST, server.port)) + + @unittest.skip("TODO: rustpython") + def test_thread_recv_while_main_thread_sends(self): + # GH-137583: Locking was added to calls to send() and recv() on SSL + # socket objects. This seemed fine at the surface level because those + # calls weren't re-entrant, but recv() calls would implicitly mimick + # holding a lock by blocking until it received data. This means that + # if a thread started to infinitely block until data was received, calls + # to send() would deadlock, because it would wait forever on the lock + # that the recv() call held. + data = b"1" * 1024 + event = threading.Event() + def background(sock): + event.set() + received = sock.recv(len(data)) + self.assertEqual(received, data) + + client_context, server_context, hostname = testing_context() + server = ThreadedEchoServer(context=server_context) + with server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as sock: + sock.connect((HOST, server.port)) + sock.settimeout(1) + sock.setblocking(1) + # Ensure that the server is ready to accept requests + sock.sendall(b"123") + self.assertEqual(sock.recv(3), b"123") + with threading_helper.catch_threading_exception() as cm: + thread = threading.Thread(target=background, + args=(sock,), daemon=True) + thread.start() + event.wait() + sock.sendall(data) + thread.join() + if cm.exc_value is not None: + raise cm.exc_value + + +@unittest.skipUnless(has_tls_version('TLSv1_3'), "Test needs TLS 1.3") +class TestPostHandshakeAuth(unittest.TestCase): + def test_pha_setter(self): + protocols = [ + ssl.PROTOCOL_TLS_SERVER, ssl.PROTOCOL_TLS_CLIENT + ] + for protocol in protocols: + ctx = ssl.SSLContext(protocol) + self.assertEqual(ctx.post_handshake_auth, False) + + ctx.post_handshake_auth = True + self.assertEqual(ctx.post_handshake_auth, True) + + ctx.verify_mode = ssl.CERT_REQUIRED + self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED) + self.assertEqual(ctx.post_handshake_auth, True) + + ctx.post_handshake_auth = False + self.assertEqual(ctx.verify_mode, ssl.CERT_REQUIRED) + self.assertEqual(ctx.post_handshake_auth, False) + + ctx.verify_mode = ssl.CERT_OPTIONAL + ctx.post_handshake_auth = True + self.assertEqual(ctx.verify_mode, ssl.CERT_OPTIONAL) + self.assertEqual(ctx.post_handshake_auth, True) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_pha_required(self): + client_context, server_context, hostname = testing_context() + server_context.post_handshake_auth = True + server_context.verify_mode = ssl.CERT_REQUIRED + client_context.post_handshake_auth = True + client_context.load_cert_chain(SIGNED_CERTFILE) + + server = ThreadedEchoServer(context=server_context, chatty=False) + with server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + s.connect((HOST, server.port)) + s.write(b'HASCERT') + self.assertEqual(s.recv(1024), b'FALSE\n') + s.write(b'PHA') + self.assertEqual(s.recv(1024), b'OK\n') + s.write(b'HASCERT') + self.assertEqual(s.recv(1024), b'TRUE\n') + # PHA method just returns true when cert is already available + s.write(b'PHA') + self.assertEqual(s.recv(1024), b'OK\n') + s.write(b'GETCERT') + cert_text = s.recv(4096).decode('us-ascii') + self.assertIn('Python Software Foundation CA', cert_text) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_pha_required_nocert(self): + client_context, server_context, hostname = testing_context() + server_context.post_handshake_auth = True + server_context.verify_mode = ssl.CERT_REQUIRED + client_context.post_handshake_auth = True + + def msg_cb(conn, direction, version, content_type, msg_type, data): + if support.verbose and content_type == _TLSContentType.ALERT: + info = (conn, direction, version, content_type, msg_type, data) + sys.stdout.write(f"TLS: {info!r}\n") + + server_context._msg_callback = msg_cb + client_context._msg_callback = msg_cb + + server = ThreadedEchoServer(context=server_context, chatty=True) + with server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname, + suppress_ragged_eofs=False) as s: + s.connect((HOST, server.port)) + s.write(b'PHA') + # test sometimes fails with EOF error. Test passes as long as + # server aborts connection with an error. + with self.assertRaisesRegex( + OSError, + ('certificate required' + '|EOF occurred' + '|closed by the remote host' + '|Connection reset by peer' + '|Broken pipe') + ): + # receive CertificateRequest + data = s.recv(1024) + self.assertEqual(data, b'OK\n') + + # send empty Certificate + Finish + s.write(b'HASCERT') + + # receive alert + s.recv(1024) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_pha_optional(self): + if support.verbose: + sys.stdout.write("\n") + + client_context, server_context, hostname = testing_context() + server_context.post_handshake_auth = True + server_context.verify_mode = ssl.CERT_REQUIRED + client_context.post_handshake_auth = True + client_context.load_cert_chain(SIGNED_CERTFILE) + + # check CERT_OPTIONAL + server_context.verify_mode = ssl.CERT_OPTIONAL + server = ThreadedEchoServer(context=server_context, chatty=False) + with server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + s.connect((HOST, server.port)) + s.write(b'HASCERT') + self.assertEqual(s.recv(1024), b'FALSE\n') + s.write(b'PHA') + self.assertEqual(s.recv(1024), b'OK\n') + s.write(b'HASCERT') + self.assertEqual(s.recv(1024), b'TRUE\n') + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_pha_optional_nocert(self): + if support.verbose: + sys.stdout.write("\n") + + client_context, server_context, hostname = testing_context() + server_context.post_handshake_auth = True + server_context.verify_mode = ssl.CERT_OPTIONAL + client_context.post_handshake_auth = True + + server = ThreadedEchoServer(context=server_context, chatty=False) + with server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + s.connect((HOST, server.port)) + s.write(b'HASCERT') + self.assertEqual(s.recv(1024), b'FALSE\n') + s.write(b'PHA') + self.assertEqual(s.recv(1024), b'OK\n') + # optional doesn't fail when client does not have a cert + s.write(b'HASCERT') + self.assertEqual(s.recv(1024), b'FALSE\n') + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_pha_no_pha_client(self): + client_context, server_context, hostname = testing_context() + server_context.post_handshake_auth = True + server_context.verify_mode = ssl.CERT_REQUIRED + client_context.load_cert_chain(SIGNED_CERTFILE) + + server = ThreadedEchoServer(context=server_context, chatty=False) + with server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + s.connect((HOST, server.port)) + with self.assertRaisesRegex(ssl.SSLError, 'not server'): + s.verify_client_post_handshake() + s.write(b'PHA') + self.assertIn(b'extension not received', s.recv(1024)) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_pha_no_pha_server(self): + # server doesn't have PHA enabled, cert is requested in handshake + client_context, server_context, hostname = testing_context() + server_context.verify_mode = ssl.CERT_REQUIRED + client_context.post_handshake_auth = True + client_context.load_cert_chain(SIGNED_CERTFILE) + + server = ThreadedEchoServer(context=server_context, chatty=False) + with server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + s.connect((HOST, server.port)) + s.write(b'HASCERT') + self.assertEqual(s.recv(1024), b'TRUE\n') + # PHA doesn't fail if there is already a cert + s.write(b'PHA') + self.assertEqual(s.recv(1024), b'OK\n') + s.write(b'HASCERT') + self.assertEqual(s.recv(1024), b'TRUE\n') + + def test_pha_not_tls13(self): + # TLS 1.2 + client_context, server_context, hostname = testing_context() + server_context.verify_mode = ssl.CERT_REQUIRED + client_context.maximum_version = ssl.TLSVersion.TLSv1_2 + client_context.post_handshake_auth = True + client_context.load_cert_chain(SIGNED_CERTFILE) + + server = ThreadedEchoServer(context=server_context, chatty=False) + with server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + s.connect((HOST, server.port)) + # PHA fails for TLS != 1.3 + s.write(b'PHA') + self.assertIn(b'WRONG_SSL_VERSION', s.recv(1024)) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_bpo37428_pha_cert_none(self): + # verify that post_handshake_auth does not implicitly enable cert + # validation. + hostname = SIGNED_CERTFILE_HOSTNAME + client_context = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + client_context.post_handshake_auth = True + client_context.load_cert_chain(SIGNED_CERTFILE) + # no cert validation and CA on client side + client_context.check_hostname = False + client_context.verify_mode = ssl.CERT_NONE + + server_context = ssl.SSLContext(ssl.PROTOCOL_TLS_SERVER) + server_context.load_cert_chain(SIGNED_CERTFILE) + server_context.load_verify_locations(SIGNING_CA) + server_context.post_handshake_auth = True + server_context.verify_mode = ssl.CERT_REQUIRED + + server = ThreadedEchoServer(context=server_context, chatty=False) + with server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + s.connect((HOST, server.port)) + s.write(b'HASCERT') + self.assertEqual(s.recv(1024), b'FALSE\n') + s.write(b'PHA') + self.assertEqual(s.recv(1024), b'OK\n') + s.write(b'HASCERT') + self.assertEqual(s.recv(1024), b'TRUE\n') + # server cert has not been validated + self.assertEqual(s.getpeercert(), {}) + + def test_internal_chain_client(self): + client_context, server_context, hostname = testing_context( + server_chain=False + ) + server = ThreadedEchoServer(context=server_context, chatty=False) + with server: + with client_context.wrap_socket( + socket.socket(), + server_hostname=hostname + ) as s: + s.connect((HOST, server.port)) + vc = s._sslobj.get_verified_chain() + self.assertEqual(len(vc), 2) + ee, ca = vc + uvc = s._sslobj.get_unverified_chain() + self.assertEqual(len(uvc), 1) + + self.assertEqual(ee, uvc[0]) + self.assertEqual(hash(ee), hash(uvc[0])) + self.assertEqual(repr(ee), repr(uvc[0])) + + self.assertNotEqual(ee, ca) + self.assertNotEqual(hash(ee), hash(ca)) + self.assertNotEqual(repr(ee), repr(ca)) + self.assertNotEqual(ee.get_info(), ca.get_info()) + self.assertIn("CN=localhost", repr(ee)) + self.assertIn("CN=our-ca-server", repr(ca)) + + pem = ee.public_bytes(_ssl.ENCODING_PEM) + der = ee.public_bytes(_ssl.ENCODING_DER) + self.assertIsInstance(pem, str) + self.assertIn("-----BEGIN CERTIFICATE-----", pem) + self.assertIsInstance(der, bytes) + self.assertEqual( + ssl.PEM_cert_to_DER_cert(pem), der + ) + + def test_certificate_chain(self): + client_context, server_context, hostname = testing_context( + server_chain=False + ) + server = ThreadedEchoServer(context=server_context, chatty=False) + + with open(SIGNING_CA) as f: + expected_ca_cert = ssl.PEM_cert_to_DER_cert(f.read()) + + with open(SINGED_CERTFILE_ONLY) as f: + expected_ee_cert = ssl.PEM_cert_to_DER_cert(f.read()) + + with server: + with client_context.wrap_socket( + socket.socket(), + server_hostname=hostname + ) as s: + s.connect((HOST, server.port)) + vc = s.get_verified_chain() + self.assertEqual(len(vc), 2) + + ee, ca = vc + self.assertIsInstance(ee, bytes) + self.assertIsInstance(ca, bytes) + self.assertEqual(expected_ca_cert, ca) + self.assertEqual(expected_ee_cert, ee) + + uvc = s.get_unverified_chain() + self.assertEqual(len(uvc), 1) + self.assertIsInstance(uvc[0], bytes) + + self.assertEqual(ee, uvc[0]) + self.assertNotEqual(ee, ca) + + def test_internal_chain_server(self): + client_context, server_context, hostname = testing_context() + client_context.load_cert_chain(SIGNED_CERTFILE) + server_context.verify_mode = ssl.CERT_REQUIRED + server_context.maximum_version = ssl.TLSVersion.TLSv1_2 + + server = ThreadedEchoServer(context=server_context, chatty=False) + with server: + with client_context.wrap_socket( + socket.socket(), + server_hostname=hostname + ) as s: + s.connect((HOST, server.port)) + s.write(b'VERIFIEDCHAIN\n') + res = s.recv(1024) + self.assertEqual(res, b'\x02\n') + s.write(b'UNVERIFIEDCHAIN\n') + res = s.recv(1024) + self.assertEqual(res, b'\x02\n') + + +HAS_KEYLOG = hasattr(ssl.SSLContext, 'keylog_filename') +requires_keylog = unittest.skipUnless( + HAS_KEYLOG, 'test requires OpenSSL 1.1.1 with keylog callback') + +class TestSSLDebug(unittest.TestCase): + + def keylog_lines(self, fname=os_helper.TESTFN): + with open(fname) as f: + return len(list(f)) + + @requires_keylog + def test_keylog_defaults(self): + self.addCleanup(os_helper.unlink, os_helper.TESTFN) + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + self.assertEqual(ctx.keylog_filename, None) + + self.assertFalse(os.path.isfile(os_helper.TESTFN)) + try: + ctx.keylog_filename = os_helper.TESTFN + except RuntimeError: + if Py_DEBUG_WIN32: + self.skipTest("not supported on Win32 debug build") + raise + self.assertEqual(ctx.keylog_filename, os_helper.TESTFN) + self.assertTrue(os.path.isfile(os_helper.TESTFN)) + self.assertEqual(self.keylog_lines(), 1) + + ctx.keylog_filename = None + self.assertEqual(ctx.keylog_filename, None) + + with self.assertRaises((IsADirectoryError, PermissionError)): + # Windows raises PermissionError + ctx.keylog_filename = os.path.dirname( + os.path.abspath(os_helper.TESTFN)) + + with self.assertRaises(TypeError): + ctx.keylog_filename = 1 + + @requires_keylog + def test_keylog_filename(self): + self.addCleanup(os_helper.unlink, os_helper.TESTFN) + client_context, server_context, hostname = testing_context() + + try: + client_context.keylog_filename = os_helper.TESTFN + except RuntimeError: + if Py_DEBUG_WIN32: + self.skipTest("not supported on Win32 debug build") + raise + + server = ThreadedEchoServer(context=server_context, chatty=False) + with server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + s.connect((HOST, server.port)) + # header, 5 lines for TLS 1.3 + self.assertEqual(self.keylog_lines(), 6) + + client_context.keylog_filename = None + server_context.keylog_filename = os_helper.TESTFN + server = ThreadedEchoServer(context=server_context, chatty=False) + with server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + s.connect((HOST, server.port)) + self.assertGreaterEqual(self.keylog_lines(), 11) + + client_context.keylog_filename = os_helper.TESTFN + server_context.keylog_filename = os_helper.TESTFN + server = ThreadedEchoServer(context=server_context, chatty=False) + with server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + s.connect((HOST, server.port)) + self.assertGreaterEqual(self.keylog_lines(), 21) + + client_context.keylog_filename = None + server_context.keylog_filename = None + + @requires_keylog + @unittest.skipIf(sys.flags.ignore_environment, + "test is not compatible with ignore_environment") + def test_keylog_env(self): + self.addCleanup(os_helper.unlink, os_helper.TESTFN) + with unittest.mock.patch.dict(os.environ): + os.environ['SSLKEYLOGFILE'] = os_helper.TESTFN + self.assertEqual(os.environ['SSLKEYLOGFILE'], os_helper.TESTFN) + + ctx = ssl.SSLContext(ssl.PROTOCOL_TLS_CLIENT) + self.assertEqual(ctx.keylog_filename, None) + + try: + ctx = ssl.create_default_context() + except RuntimeError: + if Py_DEBUG_WIN32: + self.skipTest("not supported on Win32 debug build") + raise + self.assertEqual(ctx.keylog_filename, os_helper.TESTFN) + + ctx = ssl._create_stdlib_context() + self.assertEqual(ctx.keylog_filename, os_helper.TESTFN) + + def test_msg_callback(self): + client_context, server_context, hostname = testing_context() + + def msg_cb(conn, direction, version, content_type, msg_type, data): + pass + + self.assertIs(client_context._msg_callback, None) + client_context._msg_callback = msg_cb + self.assertIs(client_context._msg_callback, msg_cb) + with self.assertRaises(TypeError): + client_context._msg_callback = object() + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_msg_callback_tls12(self): + client_context, server_context, hostname = testing_context() + client_context.maximum_version = ssl.TLSVersion.TLSv1_2 + + msg = [] + + def msg_cb(conn, direction, version, content_type, msg_type, data): + self.assertIsInstance(conn, ssl.SSLSocket) + self.assertIsInstance(data, bytes) + self.assertIn(direction, {'read', 'write'}) + msg.append((direction, version, content_type, msg_type)) + + client_context._msg_callback = msg_cb + + server = ThreadedEchoServer(context=server_context, chatty=False) + with server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + s.connect((HOST, server.port)) + + self.assertIn( + ("read", TLSVersion.TLSv1_2, _TLSContentType.HANDSHAKE, + _TLSMessageType.SERVER_KEY_EXCHANGE), + msg + ) + self.assertIn( + ("write", TLSVersion.TLSv1_2, _TLSContentType.CHANGE_CIPHER_SPEC, + _TLSMessageType.CHANGE_CIPHER_SPEC), + msg + ) + + def test_msg_callback_deadlock_bpo43577(self): + client_context, server_context, hostname = testing_context() + server_context2 = testing_context()[1] + + def msg_cb(conn, direction, version, content_type, msg_type, data): + pass + + def sni_cb(sock, servername, ctx): + sock.context = server_context2 + + server_context._msg_callback = msg_cb + server_context.sni_callback = sni_cb + + server = ThreadedEchoServer(context=server_context, chatty=False) + with server: + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + s.connect((HOST, server.port)) + with client_context.wrap_socket(socket.socket(), + server_hostname=hostname) as s: + s.connect((HOST, server.port)) + + +def set_socket_so_linger_on_with_zero_timeout(sock): + sock.setsockopt(socket.SOL_SOCKET, socket.SO_LINGER, struct.pack('ii', 1, 0)) + + +class TestPreHandshakeClose(unittest.TestCase): + """Verify behavior of close sockets with received data before to the handshake. + """ + + class SingleConnectionTestServerThread(threading.Thread): + + def __init__(self, *, name, call_after_accept, timeout=None): + self.call_after_accept = call_after_accept + self.received_data = b'' # set by .run() + self.wrap_error = None # set by .run() + self.listener = None # set by .start() + self.port = None # set by .start() + if timeout is None: + self.timeout = support.SHORT_TIMEOUT + else: + self.timeout = timeout + super().__init__(name=name) + + def __enter__(self): + self.start() + return self + + def __exit__(self, *args): + try: + if self.listener: + self.listener.close() + except OSError: + pass + self.join() + self.wrap_error = None # avoid dangling references + + def start(self): + self.ssl_ctx = ssl.create_default_context(ssl.Purpose.CLIENT_AUTH) + self.ssl_ctx.verify_mode = ssl.CERT_REQUIRED + self.ssl_ctx.load_verify_locations(cafile=ONLYCERT) + self.ssl_ctx.load_cert_chain(certfile=ONLYCERT, keyfile=ONLYKEY) + self.listener = socket.socket() + self.port = socket_helper.bind_port(self.listener) + self.listener.settimeout(self.timeout) + self.listener.listen(1) + super().start() + + def run(self): + try: + conn, address = self.listener.accept() + except TimeoutError: + # on timeout, just close the listener + return + finally: + self.listener.close() + + with conn: + if self.call_after_accept(conn): + return + try: + tls_socket = self.ssl_ctx.wrap_socket(conn, server_side=True) + except OSError as err: # ssl.SSLError inherits from OSError + self.wrap_error = err + else: + try: + self.received_data = tls_socket.recv(400) + except OSError: + pass # closed, protocol error, etc. + + def non_linux_skip_if_other_okay_error(self, err): + if sys.platform in ("linux", "android"): + return # Expect the full test setup to always work on Linux. + if (isinstance(err, ConnectionResetError) or + (isinstance(err, OSError) and err.errno == errno.EINVAL) or + re.search('wrong.version.number', str(getattr(err, "reason", "")), re.I)): + # On Windows the TCP RST leads to a ConnectionResetError + # (ECONNRESET) which Linux doesn't appear to surface to userspace. + # If wrap_socket() winds up on the "if connected:" path and doing + # the actual wrapping... we get an SSLError from OpenSSL. Typically + # WRONG_VERSION_NUMBER. While appropriate, neither is the scenario + # we're specifically trying to test. The way this test is written + # is known to work on Linux. We'll skip it anywhere else that it + # does not present as doing so. + try: + self.skipTest(f"Could not recreate conditions on {sys.platform}:" + f" {err=}") + finally: + # gh-108342: Explicitly break the reference cycle + err = None + + # If maintaining this conditional winds up being a problem. + # just turn this into an unconditional skip anything but Linux. + # The important thing is that our CI has the logic covered. + + def test_preauth_data_to_tls_server(self): + server_accept_called = threading.Event() + ready_for_server_wrap_socket = threading.Event() + + def call_after_accept(unused): + server_accept_called.set() + if not ready_for_server_wrap_socket.wait(support.SHORT_TIMEOUT): + raise RuntimeError("wrap_socket event never set, test may fail.") + return False # Tell the server thread to continue. + + server = self.SingleConnectionTestServerThread( + call_after_accept=call_after_accept, + name="preauth_data_to_tls_server") + self.enterContext(server) # starts it & unittest.TestCase stops it. + + with socket.socket() as client: + client.connect(server.listener.getsockname()) + # This forces an immediate connection close via RST on .close(). + set_socket_so_linger_on_with_zero_timeout(client) + client.setblocking(False) + + server_accept_called.wait() + client.send(b"DELETE /data HTTP/1.0\r\n\r\n") + client.close() # RST + + ready_for_server_wrap_socket.set() + server.join() + + wrap_error = server.wrap_error + server.wrap_error = None + try: + self.assertEqual(b"", server.received_data) + self.assertIsInstance(wrap_error, OSError) # All platforms. + self.non_linux_skip_if_other_okay_error(wrap_error) + self.assertIsInstance(wrap_error, ssl.SSLError) + self.assertIn("before TLS handshake with data", wrap_error.args[1]) + self.assertIn("before TLS handshake with data", wrap_error.reason) + self.assertNotEqual(0, wrap_error.args[0]) + self.assertIsNone(wrap_error.library, msg="attr must exist") + finally: + # gh-108342: Explicitly break the reference cycle + wrap_error = None + server = None + + def test_preauth_data_to_tls_client(self): + server_can_continue_with_wrap_socket = threading.Event() + client_can_continue_with_wrap_socket = threading.Event() + + def call_after_accept(conn_to_client): + if not server_can_continue_with_wrap_socket.wait(support.SHORT_TIMEOUT): + print("ERROR: test client took too long") + + # This forces an immediate connection close via RST on .close(). + set_socket_so_linger_on_with_zero_timeout(conn_to_client) + conn_to_client.send( + b"HTTP/1.0 307 Temporary Redirect\r\n" + b"Location: https://example.com/someone-elses-server\r\n" + b"\r\n") + conn_to_client.close() # RST + client_can_continue_with_wrap_socket.set() + return True # Tell the server to stop. + + server = self.SingleConnectionTestServerThread( + call_after_accept=call_after_accept, + name="preauth_data_to_tls_client") + self.enterContext(server) # starts it & unittest.TestCase stops it. + # Redundant; call_after_accept sets SO_LINGER on the accepted conn. + set_socket_so_linger_on_with_zero_timeout(server.listener) + + with socket.socket() as client: + client.connect(server.listener.getsockname()) + server_can_continue_with_wrap_socket.set() + + if not client_can_continue_with_wrap_socket.wait(support.SHORT_TIMEOUT): + self.fail("test server took too long") + ssl_ctx = ssl.create_default_context() + try: + tls_client = ssl_ctx.wrap_socket( + client, server_hostname="localhost") + except OSError as err: # SSLError inherits from OSError + wrap_error = err + received_data = b"" + else: + wrap_error = None + received_data = tls_client.recv(400) + tls_client.close() + + server.join() + try: + self.assertEqual(b"", received_data) + self.assertIsInstance(wrap_error, OSError) # All platforms. + self.non_linux_skip_if_other_okay_error(wrap_error) + self.assertIsInstance(wrap_error, ssl.SSLError) + self.assertIn("before TLS handshake with data", wrap_error.args[1]) + self.assertIn("before TLS handshake with data", wrap_error.reason) + self.assertNotEqual(0, wrap_error.args[0]) + self.assertIsNone(wrap_error.library, msg="attr must exist") + finally: + # gh-108342: Explicitly break the reference cycle + with warnings_helper.check_no_resource_warning(self): + wrap_error = None + server = None + + def test_https_client_non_tls_response_ignored(self): + server_responding = threading.Event() + + class SynchronizedHTTPSConnection(http.client.HTTPSConnection): + def connect(self): + # Call clear text HTTP connect(), not the encrypted HTTPS (TLS) + # connect(): wrap_socket() is called manually below. + http.client.HTTPConnection.connect(self) + + # Wait for our fault injection server to have done its thing. + if not server_responding.wait(support.SHORT_TIMEOUT) and support.verbose: + sys.stdout.write("server_responding event never set.") + self.sock = self._context.wrap_socket( + self.sock, server_hostname=self.host) + + def call_after_accept(conn_to_client): + # This forces an immediate connection close via RST on .close(). + set_socket_so_linger_on_with_zero_timeout(conn_to_client) + conn_to_client.send( + b"HTTP/1.0 402 Payment Required\r\n" + b"\r\n") + conn_to_client.close() # RST + server_responding.set() + return True # Tell the server to stop. + + timeout = 2.0 + server = self.SingleConnectionTestServerThread( + call_after_accept=call_after_accept, + name="non_tls_http_RST_responder", + timeout=timeout) + self.enterContext(server) # starts it & unittest.TestCase stops it. + # Redundant; call_after_accept sets SO_LINGER on the accepted conn. + set_socket_so_linger_on_with_zero_timeout(server.listener) + + connection = SynchronizedHTTPSConnection( + server.listener.getsockname()[0], + port=server.port, + context=ssl.create_default_context(), + timeout=timeout, + ) + + # There are lots of reasons this raises as desired, long before this + # test was added. Sending the request requires a successful TLS wrapped + # socket; that fails if the connection is broken. It may seem pointless + # to test this. It serves as an illustration of something that we never + # want to happen... properly not happening. + with warnings_helper.check_no_resource_warning(self), \ + self.assertRaises(OSError): + connection.request("HEAD", "/test", headers={"Host": "localhost"}) + response = connection.getresponse() + + server.join() + + +class TestEnumerations(unittest.TestCase): + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_tlsversion(self): + class CheckedTLSVersion(enum.IntEnum): + MINIMUM_SUPPORTED = _ssl.PROTO_MINIMUM_SUPPORTED + SSLv3 = _ssl.PROTO_SSLv3 + TLSv1 = _ssl.PROTO_TLSv1 + TLSv1_1 = _ssl.PROTO_TLSv1_1 + TLSv1_2 = _ssl.PROTO_TLSv1_2 + TLSv1_3 = _ssl.PROTO_TLSv1_3 + MAXIMUM_SUPPORTED = _ssl.PROTO_MAXIMUM_SUPPORTED + enum._test_simple_enum(CheckedTLSVersion, TLSVersion) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_tlscontenttype(self): + class Checked_TLSContentType(enum.IntEnum): + """Content types (record layer) + + See RFC 8446, section B.1 + """ + CHANGE_CIPHER_SPEC = 20 + ALERT = 21 + HANDSHAKE = 22 + APPLICATION_DATA = 23 + # pseudo content types + HEADER = 0x100 + INNER_CONTENT_TYPE = 0x101 + enum._test_simple_enum(Checked_TLSContentType, _TLSContentType) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_tlsalerttype(self): + class Checked_TLSAlertType(enum.IntEnum): + """Alert types for TLSContentType.ALERT messages + + See RFC 8466, section B.2 + """ + CLOSE_NOTIFY = 0 + UNEXPECTED_MESSAGE = 10 + BAD_RECORD_MAC = 20 + DECRYPTION_FAILED = 21 + RECORD_OVERFLOW = 22 + DECOMPRESSION_FAILURE = 30 + HANDSHAKE_FAILURE = 40 + NO_CERTIFICATE = 41 + BAD_CERTIFICATE = 42 + UNSUPPORTED_CERTIFICATE = 43 + CERTIFICATE_REVOKED = 44 + CERTIFICATE_EXPIRED = 45 + CERTIFICATE_UNKNOWN = 46 + ILLEGAL_PARAMETER = 47 + UNKNOWN_CA = 48 + ACCESS_DENIED = 49 + DECODE_ERROR = 50 + DECRYPT_ERROR = 51 + EXPORT_RESTRICTION = 60 + PROTOCOL_VERSION = 70 + INSUFFICIENT_SECURITY = 71 + INTERNAL_ERROR = 80 + INAPPROPRIATE_FALLBACK = 86 + USER_CANCELED = 90 + NO_RENEGOTIATION = 100 + MISSING_EXTENSION = 109 + UNSUPPORTED_EXTENSION = 110 + CERTIFICATE_UNOBTAINABLE = 111 + UNRECOGNIZED_NAME = 112 + BAD_CERTIFICATE_STATUS_RESPONSE = 113 + BAD_CERTIFICATE_HASH_VALUE = 114 + UNKNOWN_PSK_IDENTITY = 115 + CERTIFICATE_REQUIRED = 116 + NO_APPLICATION_PROTOCOL = 120 + enum._test_simple_enum(Checked_TLSAlertType, _TLSAlertType) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_tlsmessagetype(self): + class Checked_TLSMessageType(enum.IntEnum): + """Message types (handshake protocol) + + See RFC 8446, section B.3 + """ + HELLO_REQUEST = 0 + CLIENT_HELLO = 1 + SERVER_HELLO = 2 + HELLO_VERIFY_REQUEST = 3 + NEWSESSION_TICKET = 4 + END_OF_EARLY_DATA = 5 + HELLO_RETRY_REQUEST = 6 + ENCRYPTED_EXTENSIONS = 8 + CERTIFICATE = 11 + SERVER_KEY_EXCHANGE = 12 + CERTIFICATE_REQUEST = 13 + SERVER_DONE = 14 + CERTIFICATE_VERIFY = 15 + CLIENT_KEY_EXCHANGE = 16 + FINISHED = 20 + CERTIFICATE_URL = 21 + CERTIFICATE_STATUS = 22 + SUPPLEMENTAL_DATA = 23 + KEY_UPDATE = 24 + NEXT_PROTO = 67 + MESSAGE_HASH = 254 + CHANGE_CIPHER_SPEC = 0x0101 + enum._test_simple_enum(Checked_TLSMessageType, _TLSMessageType) + + def test_sslmethod(self): + Checked_SSLMethod = enum._old_convert_( + enum.IntEnum, '_SSLMethod', 'ssl', + lambda name: name.startswith('PROTOCOL_') and name != 'PROTOCOL_SSLv23', + source=ssl._ssl, + ) + # This member is assigned dynamically in `ssl.py`: + Checked_SSLMethod.PROTOCOL_SSLv23 = Checked_SSLMethod.PROTOCOL_TLS + enum._test_simple_enum(Checked_SSLMethod, ssl._SSLMethod) + + def test_options(self): + CheckedOptions = enum._old_convert_( + enum.IntFlag, 'Options', 'ssl', + lambda name: name.startswith('OP_'), + source=ssl._ssl, + ) + enum._test_simple_enum(CheckedOptions, ssl.Options) + + def test_alertdescription(self): + CheckedAlertDescription = enum._old_convert_( + enum.IntEnum, 'AlertDescription', 'ssl', + lambda name: name.startswith('ALERT_DESCRIPTION_'), + source=ssl._ssl, + ) + enum._test_simple_enum(CheckedAlertDescription, ssl.AlertDescription) + + def test_sslerrornumber(self): + Checked_SSLErrorNumber = enum._old_convert_( + enum.IntEnum, 'SSLErrorNumber', 'ssl', + lambda name: name.startswith('SSL_ERROR_'), + source=ssl._ssl, + ) + enum._test_simple_enum(Checked_SSLErrorNumber, ssl.SSLErrorNumber) + + def test_verifyflags(self): + CheckedVerifyFlags = enum._old_convert_( + enum.IntFlag, 'VerifyFlags', 'ssl', + lambda name: name.startswith('VERIFY_'), + source=ssl._ssl, + ) + enum._test_simple_enum(CheckedVerifyFlags, ssl.VerifyFlags) + + def test_verifymode(self): + CheckedVerifyMode = enum._old_convert_( + enum.IntEnum, 'VerifyMode', 'ssl', + lambda name: name.startswith('CERT_'), + source=ssl._ssl, + ) + enum._test_simple_enum(CheckedVerifyMode, ssl.VerifyMode) + + +def setUpModule(): + if support.verbose: + plats = { + 'Mac': platform.mac_ver, + 'Windows': platform.win32_ver, + } + for name, func in plats.items(): + plat = func() + if plat and plat[0]: + plat = '%s %r' % (name, plat) + break + else: + plat = repr(platform.platform()) + print("test_ssl: testing with %r %r" % + (ssl.OPENSSL_VERSION, ssl.OPENSSL_VERSION_INFO)) + print(" under %s" % plat) + print(" HAS_SNI = %r" % ssl.HAS_SNI) + print(" OP_ALL = 0x%8x" % ssl.OP_ALL) + try: + print(" OP_NO_TLSv1_1 = 0x%8x" % ssl.OP_NO_TLSv1_1) + except AttributeError: + pass + + for filename in [ + CERTFILE, BYTES_CERTFILE, + ONLYCERT, ONLYKEY, BYTES_ONLYCERT, BYTES_ONLYKEY, + SIGNED_CERTFILE, SIGNED_CERTFILE2, SIGNING_CA, + BADCERT, BADKEY, EMPTYCERT]: + if not os.path.exists(filename): + raise support.TestFailed("Can't read certificate file %r" % filename) + + thread_info = threading_helper.threading_setup() + unittest.addModuleCleanup(threading_helper.threading_cleanup, *thread_info) + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_stat.py b/Lib/test/test_stat.py index b6e9c24a80..49013a4bcd 100644 --- a/Lib/test/test_stat.py +++ b/Lib/test/test_stat.py @@ -2,8 +2,7 @@ import os import socket import sys -from test.support import os_helper -from test.support import socket_helper +from test.support import is_apple, os_helper, socket_helper from test.support.import_helper import import_fresh_module from test.support.os_helper import TESTFN @@ -15,8 +14,10 @@ class TestFilemode: statmod = None file_flags = {'SF_APPEND', 'SF_ARCHIVED', 'SF_IMMUTABLE', 'SF_NOUNLINK', - 'SF_SNAPSHOT', 'UF_APPEND', 'UF_COMPRESSED', 'UF_HIDDEN', - 'UF_IMMUTABLE', 'UF_NODUMP', 'UF_NOUNLINK', 'UF_OPAQUE'} + 'SF_SNAPSHOT', 'SF_SETTABLE', 'SF_RESTRICTED', 'SF_FIRMLINK', + 'SF_DATALESS', 'UF_APPEND', 'UF_COMPRESSED', 'UF_HIDDEN', + 'UF_IMMUTABLE', 'UF_NODUMP', 'UF_NOUNLINK', 'UF_OPAQUE', + 'UF_SETTABLE', 'UF_TRACKED', 'UF_DATAVAULT'} formats = {'S_IFBLK', 'S_IFCHR', 'S_IFDIR', 'S_IFIFO', 'S_IFLNK', 'S_IFREG', 'S_IFSOCK', 'S_IFDOOR', 'S_IFPORT', 'S_IFWHT'} @@ -113,6 +114,7 @@ def assertS_IS(self, name, mode): else: self.assertFalse(func(mode)) + @os_helper.skip_unless_working_chmod def test_mode(self): with open(TESTFN, 'w'): pass @@ -121,8 +123,11 @@ def test_mode(self): st_mode, modestr = self.get_mode() self.assertEqual(modestr, '-rwx------') self.assertS_IS("REG", st_mode) - self.assertEqual(self.statmod.S_IMODE(st_mode), + imode = self.statmod.S_IMODE(st_mode) + self.assertEqual(imode, self.statmod.S_IRWXU) + self.assertEqual(self.statmod.filemode(imode), + '?rwx------') os.chmod(TESTFN, 0o070) st_mode, modestr = self.get_mode() @@ -144,13 +149,21 @@ def test_mode(self): self.assertEqual(modestr, '-r--r--r--') self.assertEqual(self.statmod.S_IMODE(st_mode), 0o444) else: + os.chmod(TESTFN, 0o500) + st_mode, modestr = self.get_mode() + self.assertEqual(modestr[:3], '-r-') + self.assertS_IS("REG", st_mode) + self.assertEqual(self.statmod.S_IMODE(st_mode), 0o444) + os.chmod(TESTFN, 0o700) st_mode, modestr = self.get_mode() self.assertEqual(modestr[:3], '-rw') self.assertS_IS("REG", st_mode) self.assertEqual(self.statmod.S_IFMT(st_mode), self.statmod.S_IFREG) + self.assertEqual(self.statmod.S_IMODE(st_mode), 0o666) + @os_helper.skip_unless_working_chmod def test_directory(self): os.mkdir(TESTFN) os.chmod(TESTFN, 0o700) @@ -161,7 +174,7 @@ def test_directory(self): else: self.assertEqual(modestr[0], 'd') - @unittest.skipUnless(hasattr(os, 'symlink'), 'os.symlink not available') + @os_helper.skip_unless_symlink def test_link(self): try: os.symlink(os.getcwd(), TESTFN) @@ -227,6 +240,18 @@ def test_module_attributes(self): self.assertTrue(callable(func)) self.assertEqual(func(0), 0) + def test_flags_consistent(self): + self.assertFalse(self.statmod.UF_SETTABLE & self.statmod.SF_SETTABLE) + + for flag in self.file_flags: + if flag.startswith("UF"): + self.assertTrue(getattr(self.statmod, flag) & self.statmod.UF_SETTABLE, f"{flag} not in UF_SETTABLE") + elif is_apple and self.statmod is c_stat and flag == 'SF_DATALESS': + self.assertTrue(self.statmod.SF_DATALESS & self.statmod.SF_SYNTHETIC, "SF_DATALESS not in SF_SYNTHETIC") + self.assertFalse(self.statmod.SF_DATALESS & self.statmod.SF_SETTABLE, "SF_DATALESS in SF_SETTABLE") + else: + self.assertTrue(getattr(self.statmod, flag) & self.statmod.SF_SETTABLE, f"{flag} notin SF_SETTABLE") + @unittest.skipUnless(sys.platform == "win32", "FILE_ATTRIBUTE_* constants are Win32 specific") def test_file_attribute_constants(self): @@ -235,25 +260,75 @@ def test_file_attribute_constants(self): modvalue = getattr(self.statmod, key) self.assertEqual(value, modvalue, key) + @unittest.skipUnless(sys.platform == "darwin", "macOS system check") + def test_macosx_attribute_values(self): + self.assertEqual(self.statmod.UF_SETTABLE, 0x0000ffff) + self.assertEqual(self.statmod.UF_NODUMP, 0x00000001) + self.assertEqual(self.statmod.UF_IMMUTABLE, 0x00000002) + self.assertEqual(self.statmod.UF_APPEND, 0x00000004) + self.assertEqual(self.statmod.UF_OPAQUE, 0x00000008) + self.assertEqual(self.statmod.UF_COMPRESSED, 0x00000020) + self.assertEqual(self.statmod.UF_TRACKED, 0x00000040) + self.assertEqual(self.statmod.UF_DATAVAULT, 0x00000080) + self.assertEqual(self.statmod.UF_HIDDEN, 0x00008000) + if self.statmod is c_stat: + self.assertEqual(self.statmod.SF_SUPPORTED, 0x009f0000) + self.assertEqual(self.statmod.SF_SETTABLE, 0x3fff0000) + self.assertEqual(self.statmod.SF_SYNTHETIC, 0xc0000000) + else: + self.assertEqual(self.statmod.SF_SETTABLE, 0xffff0000) + self.assertEqual(self.statmod.SF_ARCHIVED, 0x00010000) + self.assertEqual(self.statmod.SF_IMMUTABLE, 0x00020000) + self.assertEqual(self.statmod.SF_APPEND, 0x00040000) + self.assertEqual(self.statmod.SF_RESTRICTED, 0x00080000) + self.assertEqual(self.statmod.SF_NOUNLINK, 0x00100000) + self.assertEqual(self.statmod.SF_FIRMLINK, 0x00800000) + self.assertEqual(self.statmod.SF_DATALESS, 0x40000000) + + self.assertFalse(isinstance(self.statmod.S_IFMT, int)) + self.assertEqual(self.statmod.S_IFIFO, 0o010000) + self.assertEqual(self.statmod.S_IFCHR, 0o020000) + self.assertEqual(self.statmod.S_IFDIR, 0o040000) + self.assertEqual(self.statmod.S_IFBLK, 0o060000) + self.assertEqual(self.statmod.S_IFREG, 0o100000) + self.assertEqual(self.statmod.S_IFLNK, 0o120000) + self.assertEqual(self.statmod.S_IFSOCK, 0o140000) + + if self.statmod is c_stat: + self.assertEqual(self.statmod.S_IFWHT, 0o160000) + + self.assertEqual(self.statmod.S_IRWXU, 0o000700) + self.assertEqual(self.statmod.S_IRUSR, 0o000400) + self.assertEqual(self.statmod.S_IWUSR, 0o000200) + self.assertEqual(self.statmod.S_IXUSR, 0o000100) + self.assertEqual(self.statmod.S_IRWXG, 0o000070) + self.assertEqual(self.statmod.S_IRGRP, 0o000040) + self.assertEqual(self.statmod.S_IWGRP, 0o000020) + self.assertEqual(self.statmod.S_IXGRP, 0o000010) + self.assertEqual(self.statmod.S_IRWXO, 0o000007) + self.assertEqual(self.statmod.S_IROTH, 0o000004) + self.assertEqual(self.statmod.S_IWOTH, 0o000002) + self.assertEqual(self.statmod.S_IXOTH, 0o000001) + self.assertEqual(self.statmod.S_ISUID, 0o004000) + self.assertEqual(self.statmod.S_ISGID, 0o002000) + self.assertEqual(self.statmod.S_ISVTX, 0o001000) + + self.assertFalse(hasattr(self.statmod, "S_ISTXT")) + self.assertEqual(self.statmod.S_IREAD, self.statmod.S_IRUSR) + self.assertEqual(self.statmod.S_IWRITE, self.statmod.S_IWUSR) + self.assertEqual(self.statmod.S_IEXEC, self.statmod.S_IXUSR) + + + +@unittest.skipIf(c_stat is None, 'need _stat extension') class TestFilemodeCStat(TestFilemode, unittest.TestCase): statmod = c_stat - # TODO: RUSTPYTHON - if sys.platform == "win32": - @unittest.expectedFailure - def test_link(self): - super().test_link() class TestFilemodePyStat(TestFilemode, unittest.TestCase): statmod = py_stat - # TODO: RUSTPYTHON - if sys.platform == "win32": - @unittest.expectedFailure - def test_link(self): - super().test_link() - if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_statistics.py b/Lib/test/test_statistics.py index 8fcbcf3540..9c2714e99d 100644 --- a/Lib/test/test_statistics.py +++ b/Lib/test/test_statistics.py @@ -1,4 +1,4 @@ -"""Test suite for statistics module, including helper NumericTestCase and +x = """Test suite for statistics module, including helper NumericTestCase and approx_equal function. """ @@ -9,13 +9,14 @@ import copy import decimal import doctest +import itertools import math import pickle import random import sys import unittest from test import support -from test.support import import_helper +from test.support import import_helper, requires_IEEE_754 from decimal import Decimal from fractions import Fraction @@ -27,6 +28,12 @@ # === Helper functions and class === +# Test copied from Lib/test/test_math.py +# detect evidence of double-rounding: fsum is not always correctly +# rounded on machines that suffer from double rounding. +x, y = 1e16, 2.9999 # use temporary values to defeat peephole optimizer +HAVE_DOUBLE_ROUNDING = (x + y == 1e16 + 4) + def sign(x): """Return -1.0 for negatives, including -0.0, otherwise +1.0.""" return math.copysign(1, x) @@ -691,14 +698,6 @@ def test_check_all(self): 'missing name "%s" in __all__' % name) -class DocTests(unittest.TestCase): - @unittest.skipIf(sys.flags.optimize >= 2, - "Docstrings are omitted with -OO and above") - def test_doc_tests(self): - failed, tried = doctest.testmod(statistics, optionflags=doctest.ELLIPSIS) - self.assertGreater(tried, 0) - self.assertEqual(failed, 0) - class StatisticsErrorTest(unittest.TestCase): def test_has_exception(self): errmsg = ( @@ -1039,50 +1038,6 @@ def test_error_msg(self): self.assertEqual(errmsg, msg) -class FindLteqTest(unittest.TestCase): - # Test _find_lteq private function. - - def test_invalid_input_values(self): - for a, x in [ - ([], 1), - ([1, 2], 3), - ([1, 3], 2) - ]: - with self.subTest(a=a, x=x): - with self.assertRaises(ValueError): - statistics._find_lteq(a, x) - - def test_locate_successfully(self): - for a, x, expected_i in [ - ([1, 1, 1, 2, 3], 1, 0), - ([0, 1, 1, 1, 2, 3], 1, 1), - ([1, 2, 3, 3, 3], 3, 2) - ]: - with self.subTest(a=a, x=x): - self.assertEqual(expected_i, statistics._find_lteq(a, x)) - - -class FindRteqTest(unittest.TestCase): - # Test _find_rteq private function. - - def test_invalid_input_values(self): - for a, l, x in [ - ([1], 2, 1), - ([1, 3], 0, 2) - ]: - with self.assertRaises(ValueError): - statistics._find_rteq(a, l, x) - - def test_locate_successfully(self): - for a, l, x, expected_i in [ - ([1, 1, 1, 2, 3], 0, 1, 2), - ([0, 1, 1, 1, 2, 3], 0, 1, 3), - ([1, 2, 3, 3, 3], 0, 3, 4) - ]: - with self.subTest(a=a, l=l, x=x): - self.assertEqual(expected_i, statistics._find_rteq(a, l, x)) - - # === Tests for public functions === class UnivariateCommonMixin: @@ -1117,7 +1072,7 @@ def test_no_inplace_modifications(self): def test_order_doesnt_matter(self): # Test that the order of data points doesn't change the result. - # CAUTION: due to floating point rounding errors, the result actually + # CAUTION: due to floating-point rounding errors, the result actually # may depend on the order. Consider this test representing an ideal. # To avoid this test failing, only test with exact values such as ints # or Fractions. @@ -1210,6 +1165,9 @@ def __pow__(self, other): def __add__(self, other): return type(self)(super().__add__(other)) __radd__ = __add__ + def __mul__(self, other): + return type(self)(super().__mul__(other)) + __rmul__ = __mul__ return (float, Decimal, Fraction, MyFloat) def test_types_conserved(self): @@ -1782,6 +1740,12 @@ def test_repeated_single_value(self): data = [x]*count self.assertEqual(self.func(data), float(x)) + def test_single_value(self): + # Override method from AverageMixin. + # Average of a single value is the value as a float. + for x in (23, 42.5, 1.3e15, Fraction(15, 19), Decimal('0.28')): + self.assertEqual(self.func([x]), float(x)) + def test_odd_fractions(self): # Test median_grouped works with an odd number of Fractions. F = Fraction @@ -1961,6 +1925,27 @@ def test_special_values(self): with self.assertRaises(ValueError): fmean([Inf, -Inf]) + def test_weights(self): + fmean = statistics.fmean + StatisticsError = statistics.StatisticsError + self.assertEqual( + fmean([10, 10, 10, 50], [0.25] * 4), + fmean([10, 10, 10, 50])) + self.assertEqual( + fmean([10, 10, 20], [0.25, 0.25, 0.50]), + fmean([10, 10, 20, 20])) + self.assertEqual( # inputs are iterators + fmean(iter([10, 10, 20]), iter([0.25, 0.25, 0.50])), + fmean([10, 10, 20, 20])) + with self.assertRaises(StatisticsError): + fmean([10, 20, 30], [1, 2]) # unequal lengths + with self.assertRaises(StatisticsError): + fmean(iter([10, 20, 30]), iter([1, 2])) # unequal lengths + with self.assertRaises(StatisticsError): + fmean([10, 20], [-1, 1]) # sum of weights is zero + with self.assertRaises(StatisticsError): + fmean(iter([10, 20]), iter([-1, 1])) # sum of weights is zero + # === Tests for variances and standard deviations === @@ -2137,6 +2122,104 @@ def test_center_not_at_mean(self): self.assertEqual(self.func(data), 2.5) self.assertEqual(self.func(data, mu=0.5), 6.5) +class TestSqrtHelpers(unittest.TestCase): + + def test_integer_sqrt_of_frac_rto(self): + for n, m in itertools.product(range(100), range(1, 1000)): + r = statistics._integer_sqrt_of_frac_rto(n, m) + self.assertIsInstance(r, int) + if r*r*m == n: + # Root is exact + continue + # Inexact, so the root should be odd + self.assertEqual(r&1, 1) + # Verify correct rounding + self.assertTrue(m * (r - 1)**2 < n < m * (r + 1)**2) + + @requires_IEEE_754 + @support.requires_resource('cpu') + def test_float_sqrt_of_frac(self): + + def is_root_correctly_rounded(x: Fraction, root: float) -> bool: + if not x: + return root == 0.0 + + # Extract adjacent representable floats + r_up: float = math.nextafter(root, math.inf) + r_down: float = math.nextafter(root, -math.inf) + assert r_down < root < r_up + + # Convert to fractions for exact arithmetic + frac_root: Fraction = Fraction(root) + half_way_up: Fraction = (frac_root + Fraction(r_up)) / 2 + half_way_down: Fraction = (frac_root + Fraction(r_down)) / 2 + + # Check a closed interval. + # Does not test for a midpoint rounding rule. + return half_way_down ** 2 <= x <= half_way_up ** 2 + + randrange = random.randrange + + for i in range(60_000): + numerator: int = randrange(10 ** randrange(50)) + denonimator: int = randrange(10 ** randrange(50)) + 1 + with self.subTest(numerator=numerator, denonimator=denonimator): + x: Fraction = Fraction(numerator, denonimator) + root: float = statistics._float_sqrt_of_frac(numerator, denonimator) + self.assertTrue(is_root_correctly_rounded(x, root)) + + # Verify that corner cases and error handling match math.sqrt() + self.assertEqual(statistics._float_sqrt_of_frac(0, 1), 0.0) + with self.assertRaises(ValueError): + statistics._float_sqrt_of_frac(-1, 1) + with self.assertRaises(ValueError): + statistics._float_sqrt_of_frac(1, -1) + + # Error handling for zero denominator matches that for Fraction(1, 0) + with self.assertRaises(ZeroDivisionError): + statistics._float_sqrt_of_frac(1, 0) + + # The result is well defined if both inputs are negative + self.assertEqual(statistics._float_sqrt_of_frac(-2, -1), statistics._float_sqrt_of_frac(2, 1)) + + def test_decimal_sqrt_of_frac(self): + root: Decimal + numerator: int + denominator: int + + for root, numerator, denominator in [ + (Decimal('0.4481904599041192673635338663'), 200874688349065940678243576378, 1000000000000000000000000000000), # No adj + (Decimal('0.7924949131383786609961759598'), 628048187350206338833590574929, 1000000000000000000000000000000), # Adj up + (Decimal('0.8500554152289934068192208727'), 722594208960136395984391238251, 1000000000000000000000000000000), # Adj down + ]: + with decimal.localcontext(decimal.DefaultContext): + self.assertEqual(statistics._decimal_sqrt_of_frac(numerator, denominator), root) + + # Confirm expected root with a quad precision decimal computation + with decimal.localcontext(decimal.DefaultContext) as ctx: + ctx.prec *= 4 + high_prec_ratio = Decimal(numerator) / Decimal(denominator) + ctx.rounding = decimal.ROUND_05UP + high_prec_root = high_prec_ratio.sqrt() + with decimal.localcontext(decimal.DefaultContext): + target_root = +high_prec_root + self.assertEqual(root, target_root) + + # Verify that corner cases and error handling match Decimal.sqrt() + self.assertEqual(statistics._decimal_sqrt_of_frac(0, 1), 0.0) + with self.assertRaises(decimal.InvalidOperation): + statistics._decimal_sqrt_of_frac(-1, 1) + with self.assertRaises(decimal.InvalidOperation): + statistics._decimal_sqrt_of_frac(1, -1) + + # Error handling for zero denominator matches that for Fraction(1, 0) + with self.assertRaises(ZeroDivisionError): + statistics._decimal_sqrt_of_frac(1, 0) + + # The result is well defined if both inputs are negative + self.assertEqual(statistics._decimal_sqrt_of_frac(-2, -1), statistics._decimal_sqrt_of_frac(2, 1)) + + class TestStdev(VarianceStdevMixin, NumericTestCase): # Tests for sample standard deviation. def setUp(self): @@ -2151,7 +2234,7 @@ def test_compare_to_variance(self): # Test that stdev is, in fact, the square root of variance. data = [random.uniform(-2, 9) for _ in range(1000)] expected = math.sqrt(statistics.variance(data)) - self.assertEqual(self.func(data), expected) + self.assertAlmostEqual(self.func(data), expected) def test_center_not_at_mean(self): data = (1.0, 2.0) @@ -2219,10 +2302,12 @@ def test_error_cases(self): StatisticsError = statistics.StatisticsError with self.assertRaises(StatisticsError): geometric_mean([]) # empty input - with self.assertRaises(StatisticsError): - geometric_mean([3.5, 0.0, 5.25]) # zero input with self.assertRaises(StatisticsError): geometric_mean([3.5, -4.0, 5.25]) # negative input + with self.assertRaises(StatisticsError): + geometric_mean([0.0, -4.0, 5.25]) # negative input with zero + with self.assertRaises(StatisticsError): + geometric_mean([3.5, -math.inf, 5.25]) # negative infinity with self.assertRaises(StatisticsError): geometric_mean(iter([])) # empty iterator with self.assertRaises(TypeError): @@ -2245,6 +2330,203 @@ def test_special_values(self): with self.assertRaises(ValueError): geometric_mean([Inf, -Inf]) + # Cases with zero + self.assertEqual(geometric_mean([3, 0.0, 5]), 0.0) # Any zero gives a zero + self.assertEqual(geometric_mean([3, -0.0, 5]), 0.0) # Negative zero allowed + self.assertTrue(math.isnan(geometric_mean([0, NaN]))) # NaN beats zero + self.assertTrue(math.isnan(geometric_mean([0, Inf]))) # Because 0.0 * Inf -> NaN + + def test_mixed_int_and_float(self): + # Regression test for b.p.o. issue #28327 + geometric_mean = statistics.geometric_mean + expected_mean = 3.80675409583932 + values = [ + [2, 3, 5, 7], + [2, 3, 5, 7.0], + [2, 3, 5.0, 7.0], + [2, 3.0, 5.0, 7.0], + [2.0, 3.0, 5.0, 7.0], + ] + for v in values: + with self.subTest(v=v): + actual_mean = geometric_mean(v) + self.assertAlmostEqual(actual_mean, expected_mean, places=5) + + +class TestKDE(unittest.TestCase): + + @support.requires_resource('cpu') + def test_kde(self): + kde = statistics.kde + StatisticsError = statistics.StatisticsError + + kernels = ['normal', 'gauss', 'logistic', 'sigmoid', 'rectangular', + 'uniform', 'triangular', 'parabolic', 'epanechnikov', + 'quartic', 'biweight', 'triweight', 'cosine'] + + sample = [-2.1, -1.3, -0.4, 1.9, 5.1, 6.2] + + # The approximate integral of a PDF should be close to 1.0 + + def integrate(func, low, high, steps=10_000): + "Numeric approximation of a definite function integral." + dx = (high - low) / steps + midpoints = (low + (i + 1/2) * dx for i in range(steps)) + return sum(map(func, midpoints)) * dx + + for kernel in kernels: + with self.subTest(kernel=kernel): + f_hat = kde(sample, h=1.5, kernel=kernel) + area = integrate(f_hat, -20, 20) + self.assertAlmostEqual(area, 1.0, places=4) + + # Check CDF against an integral of the PDF + + data = [3, 5, 10, 12] + h = 2.3 + x = 10.5 + for kernel in kernels: + with self.subTest(kernel=kernel): + cdf = kde(data, h, kernel, cumulative=True) + f_hat = kde(data, h, kernel) + area = integrate(f_hat, -20, x, 100_000) + self.assertAlmostEqual(cdf(x), area, places=4) + + # Check error cases + + with self.assertRaises(StatisticsError): + kde([], h=1.0) # Empty dataset + with self.assertRaises(TypeError): + kde(['abc', 'def'], 1.5) # Non-numeric data + with self.assertRaises(TypeError): + kde(iter(sample), 1.5) # Data is not a sequence + with self.assertRaises(StatisticsError): + kde(sample, h=0.0) # Zero bandwidth + with self.assertRaises(StatisticsError): + kde(sample, h=-1.0) # Negative bandwidth + with self.assertRaises(TypeError): + kde(sample, h='str') # Wrong bandwidth type + with self.assertRaises(StatisticsError): + kde(sample, h=1.0, kernel='bogus') # Invalid kernel + with self.assertRaises(TypeError): + kde(sample, 1.0, 'gauss', True) # Positional cumulative argument + + # Test name and docstring of the generated function + + h = 1.5 + kernel = 'cosine' + f_hat = kde(sample, h, kernel) + self.assertEqual(f_hat.__name__, 'pdf') + self.assertIn(kernel, f_hat.__doc__) + self.assertIn(repr(h), f_hat.__doc__) + + # Test closed interval for the support boundaries. + # In particular, 'uniform' should non-zero at the boundaries. + + f_hat = kde([0], 1.0, 'uniform') + self.assertEqual(f_hat(-1.0), 1/2) + self.assertEqual(f_hat(1.0), 1/2) + + # Test online updates to data + + data = [1, 2] + f_hat = kde(data, 5.0, 'triangular') + self.assertEqual(f_hat(100), 0.0) + data.append(100) + self.assertGreater(f_hat(100), 0.0) + + def test_kde_kernel_invcdfs(self): + kernel_invcdfs = statistics._kernel_invcdfs + kde = statistics.kde + + # Verify that cdf / invcdf will round trip + xarr = [i/100 for i in range(-100, 101)] + for kernel, invcdf in kernel_invcdfs.items(): + with self.subTest(kernel=kernel): + cdf = kde([0.0], h=1.0, kernel=kernel, cumulative=True) + for x in xarr: + self.assertAlmostEqual(invcdf(cdf(x)), x, places=5) + + @support.requires_resource('cpu') + def test_kde_random(self): + kde_random = statistics.kde_random + StatisticsError = statistics.StatisticsError + kernels = ['normal', 'gauss', 'logistic', 'sigmoid', 'rectangular', + 'uniform', 'triangular', 'parabolic', 'epanechnikov', + 'quartic', 'biweight', 'triweight', 'cosine'] + sample = [-2.1, -1.3, -0.4, 1.9, 5.1, 6.2] + + # Smoke test + + for kernel in kernels: + with self.subTest(kernel=kernel): + rand = kde_random(sample, h=1.5, kernel=kernel) + selections = [rand() for i in range(10)] + + # Check error cases + + with self.assertRaises(StatisticsError): + kde_random([], h=1.0) # Empty dataset + with self.assertRaises(TypeError): + kde_random(['abc', 'def'], 1.5) # Non-numeric data + with self.assertRaises(TypeError): + kde_random(iter(sample), 1.5) # Data is not a sequence + with self.assertRaises(StatisticsError): + kde_random(sample, h=-1.0) # Zero bandwidth + with self.assertRaises(StatisticsError): + kde_random(sample, h=0.0) # Negative bandwidth + with self.assertRaises(TypeError): + kde_random(sample, h='str') # Wrong bandwidth type + with self.assertRaises(StatisticsError): + kde_random(sample, h=1.0, kernel='bogus') # Invalid kernel + + # Test name and docstring of the generated function + + h = 1.5 + kernel = 'cosine' + rand = kde_random(sample, h, kernel) + self.assertEqual(rand.__name__, 'rand') + self.assertIn(kernel, rand.__doc__) + self.assertIn(repr(h), rand.__doc__) + + # Approximate distribution test: Compare a random sample to the expected distribution + + data = [-2.1, -1.3, -0.4, 1.9, 5.1, 6.2, 7.8, 14.3, 15.1, 15.3, 15.8, 17.0] + xarr = [x / 10 for x in range(-100, 250)] + # TODO: RUSTPYTHON - n originally 1_000_000 in CPython implementation but sorting is slow + n = 30_000 + h = 1.75 + dx = 0.1 + + def p_observed(x): + # P(x <= X < x+dx) + i = bisect.bisect_left(big_sample, x) + j = bisect.bisect_left(big_sample, x + dx) + return (j - i) / len(big_sample) + + def p_expected(x): + # P(x <= X < x+dx) + return F_hat(x + dx) - F_hat(x) + + for kernel in kernels: + with self.subTest(kernel=kernel): + + rand = kde_random(data, h, kernel, seed=8675309**2) + big_sample = sorted([rand() for i in range(n)]) + F_hat = statistics.kde(data, h, kernel, cumulative=True) + + for x in xarr: + # TODO : RUSTPYTHON - abs_tol=0.0005 in CPython implementation but smaller `n` increases variance + self.assertTrue(math.isclose(p_observed(x), p_expected(x), abs_tol=0.005)) + + # Test online updates to data + + data = [1, 2] + rand = kde_random(data, 5, 'triangular') + self.assertLess(max([rand() for i in range(5000)]), 10) + data.append(100) + self.assertGreater(max(rand() for i in range(5000)), 10) + class TestQuantiles(unittest.TestCase): @@ -2355,6 +2637,11 @@ def f(x): data = random.choices(range(100), k=k) q1, q2, q3 = quantiles(data, method='inclusive') self.assertEqual(q2, statistics.median(data)) + # Base case with a single data point: When estimating quantiles from + # a sample, we want to be able to add one sample point at a time, + # getting increasingly better estimates. + self.assertEqual(quantiles([10], n=4), [10.0, 10.0, 10.0]) + self.assertEqual(quantiles([10], n=4, method='exclusive'), [10.0, 10.0, 10.0]) def test_equal_inputs(self): quantiles = statistics.quantiles @@ -2405,7 +2692,7 @@ def test_error_cases(self): with self.assertRaises(ValueError): quantiles([10, 20, 30], method='X') # method is unknown with self.assertRaises(StatisticsError): - quantiles([10], n=4) # not enough data points + quantiles([], n=4) # not enough data points with self.assertRaises(TypeError): quantiles([10, None, 30], n=4) # data is non-numeric @@ -2464,6 +2751,95 @@ def test_different_scales(self): self.assertAlmostEqual(statistics.correlation(x, y), 1) self.assertAlmostEqual(statistics.covariance(x, y), 0.1) + def test_sqrtprod_helper_function_fundamentals(self): + # Verify that results are close to sqrt(x * y) + for i in range(100): + x = random.expovariate() + y = random.expovariate() + expected = math.sqrt(x * y) + actual = statistics._sqrtprod(x, y) + with self.subTest(x=x, y=y, expected=expected, actual=actual): + self.assertAlmostEqual(expected, actual) + + x, y, target = 0.8035720646477457, 0.7957468097636939, 0.7996498651651661 + self.assertEqual(statistics._sqrtprod(x, y), target) + self.assertNotEqual(math.sqrt(x * y), target) + + # Test that range extremes avoid underflow and overflow + smallest = sys.float_info.min * sys.float_info.epsilon + self.assertEqual(statistics._sqrtprod(smallest, smallest), smallest) + biggest = sys.float_info.max + self.assertEqual(statistics._sqrtprod(biggest, biggest), biggest) + + # Check special values and the sign of the result + special_values = [0.0, -0.0, 1.0, -1.0, 4.0, -4.0, + math.nan, -math.nan, math.inf, -math.inf] + for x, y in itertools.product(special_values, repeat=2): + try: + expected = math.sqrt(x * y) + except ValueError: + expected = 'ValueError' + try: + actual = statistics._sqrtprod(x, y) + except ValueError: + actual = 'ValueError' + with self.subTest(x=x, y=y, expected=expected, actual=actual): + if isinstance(expected, str) and expected == 'ValueError': + self.assertEqual(actual, 'ValueError') + continue + self.assertIsInstance(actual, float) + if math.isnan(expected): + self.assertTrue(math.isnan(actual)) + continue + self.assertEqual(actual, expected) + self.assertEqual(sign(actual), sign(expected)) + + @requires_IEEE_754 + @unittest.skipIf(HAVE_DOUBLE_ROUNDING, + "accuracy not guaranteed on machines with double rounding") + @support.cpython_only # Allow for a weaker sumprod() implmentation + def test_sqrtprod_helper_function_improved_accuracy(self): + # Test a known example where accuracy is improved + x, y, target = 0.8035720646477457, 0.7957468097636939, 0.7996498651651661 + self.assertEqual(statistics._sqrtprod(x, y), target) + self.assertNotEqual(math.sqrt(x * y), target) + + def reference_value(x: float, y: float) -> float: + x = decimal.Decimal(x) + y = decimal.Decimal(y) + with decimal.localcontext() as ctx: + ctx.prec = 200 + return float((x * y).sqrt()) + + # Verify that the new function with improved accuracy + # agrees with a reference value more often than old version. + new_agreements = 0 + old_agreements = 0 + for i in range(10_000): + x = random.expovariate() + y = random.expovariate() + new = statistics._sqrtprod(x, y) + old = math.sqrt(x * y) + ref = reference_value(x, y) + new_agreements += (new == ref) + old_agreements += (old == ref) + self.assertGreater(new_agreements, old_agreements) + + def test_correlation_spearman(self): + # https://statistics.laerd.com/statistical-guides/spearmans-rank-order-correlation-statistical-guide-2.php + # Compare with: + # >>> import scipy.stats.mstats + # >>> scipy.stats.mstats.spearmanr(reading, mathematics) + # SpearmanrResult(correlation=0.6686960980480712, pvalue=0.03450954165178532) + # And Wolfram Alpha gives: 0.668696 + # https://www.wolframalpha.com/input?i=SpearmanRho%5B%7B56%2C+75%2C+45%2C+71%2C+61%2C+64%2C+58%2C+80%2C+76%2C+61%7D%2C+%7B66%2C+70%2C+40%2C+60%2C+65%2C+56%2C+59%2C+77%2C+67%2C+63%7D%5D + reading = [56, 75, 45, 71, 61, 64, 58, 80, 76, 61] + mathematics = [66, 70, 40, 60, 65, 56, 59, 77, 67, 63] + self.assertAlmostEqual(statistics.correlation(reading, mathematics, method='ranked'), + 0.6686960980480712) + + with self.assertRaises(ValueError): + statistics.correlation(reading, mathematics, method='bad_method') class TestLinearRegression(unittest.TestCase): @@ -2487,6 +2863,22 @@ def test_results(self): self.assertAlmostEqual(intercept, true_intercept) self.assertAlmostEqual(slope, true_slope) + def test_proportional(self): + x = [10, 20, 30, 40] + y = [180, 398, 610, 799] + slope, intercept = statistics.linear_regression(x, y, proportional=True) + self.assertAlmostEqual(slope, 20 + 1/150) + self.assertEqual(intercept, 0.0) + + def test_float_output(self): + x = [Fraction(2, 3), Fraction(3, 4)] + y = [Fraction(4, 5), Fraction(5, 6)] + slope, intercept = statistics.linear_regression(x, y) + self.assertTrue(isinstance(slope, float)) + self.assertTrue(isinstance(intercept, float)) + slope, intercept = statistics.linear_regression(x, y, proportional=True) + self.assertTrue(isinstance(slope, float)) + self.assertTrue(isinstance(intercept, float)) class TestNormalDist: @@ -2497,8 +2889,6 @@ class TestNormalDist: # inaccurate. There isn't much we can do about this short of # implementing our own implementations from scratch. - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_slots(self): nd = self.module.NormalDist(300, 23) with self.assertRaises(TypeError): @@ -2640,6 +3030,7 @@ def test_cdf(self): self.assertTrue(math.isnan(X.cdf(float('NaN')))) @support.skip_if_pgo_task + @support.requires_resource('cpu') def test_inv_cdf(self): NormalDist = self.module.NormalDist @@ -2697,9 +3088,10 @@ def test_inv_cdf(self): iq.inv_cdf(1.0) # p is one with self.assertRaises(self.module.StatisticsError): iq.inv_cdf(1.1) # p over one - with self.assertRaises(self.module.StatisticsError): - iq = NormalDist(100, 0) # sigma is zero - iq.inv_cdf(0.5) + + # Supported case: + iq = NormalDist(100, 0) # sigma is zero + self.assertEqual(iq.inv_cdf(0.5), 100) # Special values self.assertTrue(math.isnan(Z.inv_cdf(float('NaN')))) @@ -2882,14 +3274,19 @@ def __init__(self, mu, sigma): nd = NormalDist(100, 15) self.assertNotEqual(nd, lnd) - def test_pickle_and_copy(self): + def test_copy(self): nd = self.module.NormalDist(37.5, 5.625) nd1 = copy.copy(nd) self.assertEqual(nd, nd1) nd2 = copy.deepcopy(nd) self.assertEqual(nd, nd2) - nd3 = pickle.loads(pickle.dumps(nd)) - self.assertEqual(nd, nd3) + + def test_pickle(self): + nd = self.module.NormalDist(37.5, 5.625) + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + with self.subTest(proto=proto): + pickled = pickle.loads(pickle.dumps(nd, protocol=proto)) + self.assertEqual(nd, pickled) def test_hashability(self): ND = self.module.NormalDist @@ -2911,7 +3308,7 @@ def setUp(self): def tearDown(self): sys.modules['statistics'] = statistics - + @unittest.skipUnless(c_statistics, 'requires _statistics') class TestNormalDistC(unittest.TestCase, TestNormalDist): @@ -2928,6 +3325,8 @@ def tearDown(self): def load_tests(loader, tests, ignore): """Used for doctest/unittest integration.""" tests.addTests(doctest.DocTestSuite()) + if sys.float_repr_style == 'short': + tests.addTests(doctest.DocTestSuite(statistics)) return tests diff --git a/Lib/test/test_unicode.py b/Lib/test/test_str.py similarity index 93% rename from Lib/test/test_unicode.py rename to Lib/test/test_str.py index 1a8a8f7ee9..9d43a33cd9 100644 --- a/Lib/test/test_unicode.py +++ b/Lib/test/test_str.py @@ -7,6 +7,7 @@ """ import _string import codecs +import datetime import itertools import operator import pickle @@ -55,8 +56,22 @@ def duplicate_string(text): class StrSubclass(str): pass -class UnicodeTest(string_tests.CommonTest, - string_tests.MixinStrUnicodeUserStringTest, +class OtherStrSubclass(str): + pass + +class WithStr: + def __init__(self, value): + self.value = value + def __str__(self): + return self.value + +class WithRepr: + def __init__(self, value): + self.value = value + def __repr__(self): + return self.value + +class StrTest(string_tests.StringLikeTest, string_tests.MixinStrUnicodeTest, unittest.TestCase): @@ -84,6 +99,10 @@ def __repr__(self): self.assertEqual(realresult, result) self.assertTrue(object is not realresult) + def assertTypedEqual(self, actual, expected): + self.assertIs(type(actual), type(expected)) + self.assertEqual(actual, expected) + def test_literals(self): self.assertEqual('\xff', '\u00ff') self.assertEqual('\uffff', '\U0000ffff') @@ -93,6 +112,7 @@ def test_literals(self): # raw strings should not have unicode escapes self.assertNotEqual(r"\u0020", " ") + @unittest.expectedFailure # TODO: RUSTPYTHON def test_ascii(self): self.assertEqual(ascii('abc'), "'abc'") self.assertEqual(ascii('ab\\c'), "'ab\\\\c'") @@ -128,10 +148,13 @@ def test_ascii(self): self.assertEqual(ascii("\U00010000" * 39 + "\uffff" * 4096), ascii("\U00010000" * 39 + "\uffff" * 4096)) - class WrongRepr: - def __repr__(self): - return b'byte-repr' - self.assertRaises(TypeError, ascii, WrongRepr()) + self.assertTypedEqual(ascii('\U0001f40d'), r"'\U0001f40d'") + self.assertTypedEqual(ascii(StrSubclass('abc')), "'abc'") + self.assertTypedEqual(ascii(WithRepr('<abc>')), '<abc>') + self.assertTypedEqual(ascii(WithRepr(StrSubclass('<abc>'))), StrSubclass('<abc>')) + self.assertTypedEqual(ascii(WithRepr('<\U0001f40d>')), r'<\U0001f40d>') + self.assertTypedEqual(ascii(WithRepr(StrSubclass('<\U0001f40d>'))), r'<\U0001f40d>') + self.assertRaises(TypeError, ascii, WithRepr(b'byte-repr')) def test_repr(self): # Test basic sanity of repr() @@ -169,10 +192,13 @@ def test_repr(self): self.assertEqual(repr("\U00010000" * 39 + "\uffff" * 4096), repr("\U00010000" * 39 + "\uffff" * 4096)) - class WrongRepr: - def __repr__(self): - return b'byte-repr' - self.assertRaises(TypeError, repr, WrongRepr()) + self.assertTypedEqual(repr('\U0001f40d'), "'\U0001f40d'") + self.assertTypedEqual(repr(StrSubclass('abc')), "'abc'") + self.assertTypedEqual(repr(WithRepr('<abc>')), '<abc>') + self.assertTypedEqual(repr(WithRepr(StrSubclass('<abc>'))), StrSubclass('<abc>')) + self.assertTypedEqual(repr(WithRepr('<\U0001f40d>')), '<\U0001f40d>') + self.assertTypedEqual(repr(WithRepr(StrSubclass('<\U0001f40d>'))), StrSubclass('<\U0001f40d>')) + self.assertRaises(TypeError, repr, WithRepr(b'byte-repr')) def test_iterators(self): # Make sure unicode objects have an __iter__ method @@ -213,7 +239,7 @@ def test_pickle_iterator(self): self.assertEqual(case, pickled) def test_count(self): - string_tests.CommonTest.test_count(self) + string_tests.StringLikeTest.test_count(self) # check mixed argument types self.checkequalnofix(3, 'aaa', 'count', 'a') self.checkequalnofix(0, 'aaa', 'count', 'b') @@ -243,7 +269,7 @@ class MyStr(str): self.checkequal(3, MyStr('aaa'), 'count', 'a') def test_find(self): - string_tests.CommonTest.test_find(self) + string_tests.StringLikeTest.test_find(self) # test implementation details of the memchr fast path self.checkequal(100, 'a' * 100 + '\u0102', 'find', '\u0102') self.checkequal(-1, 'a' * 100 + '\u0102', 'find', '\u0201') @@ -288,7 +314,7 @@ def test_find(self): self.checkequal(-1, '\u0102' * 100, 'find', '\u0102\U00100304') def test_rfind(self): - string_tests.CommonTest.test_rfind(self) + string_tests.StringLikeTest.test_rfind(self) # test implementation details of the memrchr fast path self.checkequal(0, '\u0102' + 'a' * 100 , 'rfind', '\u0102') self.checkequal(-1, '\u0102' + 'a' * 100 , 'rfind', '\u0201') @@ -329,7 +355,7 @@ def test_rfind(self): self.checkequal(-1, '\u0102' * 100, 'rfind', '\U00100304\u0102') def test_index(self): - string_tests.CommonTest.test_index(self) + string_tests.StringLikeTest.test_index(self) self.checkequalnofix(0, 'abcdefghiabc', 'index', '') self.checkequalnofix(3, 'abcdefghiabc', 'index', 'def') self.checkequalnofix(0, 'abcdefghiabc', 'index', 'abc') @@ -353,7 +379,7 @@ def test_index(self): self.assertRaises(ValueError, ('\u0102' * 100).index, '\u0102\U00100304') def test_rindex(self): - string_tests.CommonTest.test_rindex(self) + string_tests.StringLikeTest.test_rindex(self) self.checkequalnofix(12, 'abcdefghiabc', 'rindex', '') self.checkequalnofix(3, 'abcdefghiabc', 'rindex', 'def') self.checkequalnofix(9, 'abcdefghiabc', 'rindex', 'abc') @@ -449,7 +475,7 @@ def test_maketrans_translate(self): self.assertRaises(TypeError, 'abababc'.translate, 'abc', 'xyz') def test_split(self): - string_tests.CommonTest.test_split(self) + string_tests.StringLikeTest.test_split(self) # test mixed kinds for left, right in ('ba', '\u0101\u0100', '\U00010301\U00010300'): @@ -466,7 +492,7 @@ def test_split(self): left + delim * 2 + right, 'split', delim *2) def test_rsplit(self): - string_tests.CommonTest.test_rsplit(self) + string_tests.StringLikeTest.test_rsplit(self) # test mixed kinds for left, right in ('ba', 'юё', '\u0101\u0100', '\U00010301\U00010300'): left *= 9 @@ -486,7 +512,7 @@ def test_rsplit(self): left + right, 'rsplit', None) def test_partition(self): - string_tests.MixinStrUnicodeUserStringTest.test_partition(self) + string_tests.StringLikeTest.test_partition(self) # test mixed kinds self.checkequal(('ABCDEFGH', '', ''), 'ABCDEFGH', 'partition', '\u4200') for left, right in ('ba', '\u0101\u0100', '\U00010301\U00010300'): @@ -503,7 +529,7 @@ def test_partition(self): left + delim * 2 + right, 'partition', delim * 2) def test_rpartition(self): - string_tests.MixinStrUnicodeUserStringTest.test_rpartition(self) + string_tests.StringLikeTest.test_rpartition(self) # test mixed kinds self.checkequal(('', '', 'ABCDEFGH'), 'ABCDEFGH', 'rpartition', '\u4200') for left, right in ('ba', '\u0101\u0100', '\U00010301\U00010300'): @@ -520,7 +546,7 @@ def test_rpartition(self): left + delim * 2 + right, 'rpartition', delim * 2) def test_join(self): - string_tests.MixinStrUnicodeUserStringTest.test_join(self) + string_tests.StringLikeTest.test_join(self) class MyWrapper: def __init__(self, sval): self.sval = sval @@ -539,7 +565,7 @@ def __str__(self): return self.sval self.checkraises(TypeError, ' ', 'join', [1, 2, 3]) self.checkraises(TypeError, ' ', 'join', ['1', '2', 3]) - @unittest.skip("TODO: RUSTPYTHON, oom handling") + @unittest.skip('TODO: RUSTPYTHON; oom handling') @unittest.skipIf(sys.maxsize > 2**32, 'needs too much memory on a 64-bit platform') def test_join_overflow(self): @@ -548,7 +574,7 @@ def test_join_overflow(self): self.assertRaises(OverflowError, ''.join, seq) def test_replace(self): - string_tests.CommonTest.test_replace(self) + string_tests.StringLikeTest.test_replace(self) # method call forwarded from str implementation because of unicode argument self.checkequalnofix('one@two!three!', 'one!two!three!', 'replace', '!', '@', 1) @@ -768,8 +794,7 @@ def test_isdecimal(self): for ch in ['\U0001D7F6', '\U00011066', '\U000104A0']: self.assertTrue(ch.isdecimal(), '{!a} is decimal.'.format(ch)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_isdigit(self): super().test_isdigit() self.checkequalnofix(True, '\u2460', 'isdigit') @@ -831,6 +856,15 @@ def test_isprintable(self): self.assertTrue('\U0001F46F'.isprintable()) self.assertFalse('\U000E0020'.isprintable()) + @support.requires_resource('cpu') + def test_isprintable_invariant(self): + for codepoint in range(sys.maxunicode + 1): + char = chr(codepoint) + category = unicodedata.category(char) + self.assertEqual(char.isprintable(), + category[0] not in ('C', 'Z') + or char == ' ') + def test_surrogates(self): for s in ('a\uD800b\uDFFF', 'a\uDFFFb\uD800', 'a\uD800b\uDFFFa', 'a\uDFFFb\uD800a'): @@ -859,7 +893,7 @@ def test_surrogates(self): def test_lower(self): - string_tests.CommonTest.test_lower(self) + string_tests.StringLikeTest.test_lower(self) self.assertEqual('\U00010427'.lower(), '\U0001044F') self.assertEqual('\U00010427\U00010427'.lower(), '\U0001044F\U0001044F') @@ -890,7 +924,7 @@ def test_casefold(self): self.assertEqual('\u00b5'.casefold(), '\u03bc') def test_upper(self): - string_tests.CommonTest.test_upper(self) + string_tests.StringLikeTest.test_upper(self) self.assertEqual('\U0001044F'.upper(), '\U00010427') self.assertEqual('\U0001044F\U0001044F'.upper(), '\U00010427\U00010427') @@ -906,10 +940,9 @@ def test_upper(self): self.assertEqual('\U0008fffe'.upper(), '\U0008fffe') self.assertEqual('\u2177'.upper(), '\u2167') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_capitalize(self): - string_tests.CommonTest.test_capitalize(self) + string_tests.StringLikeTest.test_capitalize(self) self.assertEqual('\U0001044F'.capitalize(), '\U00010427') self.assertEqual('\U0001044F\U0001044F'.capitalize(), '\U00010427\U0001044F') @@ -925,8 +958,7 @@ def test_capitalize(self): self.assertEqual('finnish'.capitalize(), 'Finnish') self.assertEqual('A\u0345\u03a3'.capitalize(), 'A\u0345\u03c2') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_title(self): super().test_title() self.assertEqual('\U0001044F'.title(), '\U00010427') @@ -944,10 +976,9 @@ def test_title(self): self.assertEqual('A\u03a3 \u1fa1xy'.title(), 'A\u03c2 \u1fa9xy') self.assertEqual('A\u03a3A'.title(), 'A\u03c3a') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_swapcase(self): - string_tests.CommonTest.test_swapcase(self) + string_tests.StringLikeTest.test_swapcase(self) self.assertEqual('\U0001044F'.swapcase(), '\U00010427') self.assertEqual('\U00010427'.swapcase(), '\U0001044F') self.assertEqual('\U0001044F\U0001044F'.swapcase(), @@ -973,7 +1004,7 @@ def test_swapcase(self): self.assertEqual('\u1fd2'.swapcase(), '\u0399\u0308\u0300') def test_center(self): - string_tests.CommonTest.test_center(self) + string_tests.StringLikeTest.test_center(self) self.assertEqual('x'.center(2, '\U0010FFFF'), 'x\U0010FFFF') self.assertEqual('x'.center(3, '\U0010FFFF'), @@ -1045,8 +1076,7 @@ def test_issue18183(self): '\U00100000'.ljust(3, '\U00010000') '\U00100000'.rjust(3, '\U00010000') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_format(self): self.assertEqual(''.format(), '') self.assertEqual('a'.format(), 'a') @@ -1208,10 +1238,10 @@ def __repr__(self): self.assertEqual('{0:\x00^6}'.format(3), '\x00\x003\x00\x00\x00') self.assertEqual('{0:<6}'.format(3), '3 ') - self.assertEqual('{0:\x00<6}'.format(3.14), '3.14\x00\x00') - self.assertEqual('{0:\x01<6}'.format(3.14), '3.14\x01\x01') - self.assertEqual('{0:\x00^6}'.format(3.14), '\x003.14\x00') - self.assertEqual('{0:^6}'.format(3.14), ' 3.14 ') + self.assertEqual('{0:\x00<6}'.format(3.25), '3.25\x00\x00') + self.assertEqual('{0:\x01<6}'.format(3.25), '3.25\x01\x01') + self.assertEqual('{0:\x00^6}'.format(3.25), '\x003.25\x00') + self.assertEqual('{0:^6}'.format(3.25), ' 3.25 ') self.assertEqual('{0:\x00<12}'.format(3+2.0j), '(3+2j)\x00\x00\x00\x00\x00\x00') self.assertEqual('{0:\x01<12}'.format(3+2.0j), '(3+2j)\x01\x01\x01\x01\x01\x01') @@ -1430,21 +1460,19 @@ def __getitem__(self, key): self.assertRaises(TypeError, '{a}'.format_map, []) self.assertRaises(ZeroDivisionError, '{a}'.format_map, BadMapping()) - @unittest.skip("TODO: RUSTPYTHON, killed for chewing up RAM") + @unittest.skip('TODO: RUSTPYTHON; killed for chewing up RAM') def test_format_huge_precision(self): format_string = ".{}f".format(sys.maxsize + 1) with self.assertRaises(ValueError): result = format(2.34, format_string) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_format_huge_width(self): format_string = "{}f".format(sys.maxsize + 1) with self.assertRaises(ValueError): result = format(2.34, format_string) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_format_huge_item_number(self): format_string = "{{{}:.6f}}".format(sys.maxsize + 1) with self.assertRaises(ValueError): @@ -1480,10 +1508,9 @@ def __format__(self, spec): self.assertEqual('{:{f}}{g}{}'.format(1, 3, g='g', f=2), ' 1g3') self.assertEqual('{f:{}}{}{g}'.format(2, 4, f=1, g='g'), ' 14g') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_formatting(self): - string_tests.MixinStrUnicodeUserStringTest.test_formatting(self) + string_tests.StringLikeTest.test_formatting(self) # Testing Unicode formatting strings... self.assertEqual("%s, %s" % ("abc", "abc"), 'abc, abc') self.assertEqual("%s, %s, %i, %f, %5.2f" % ("abc", "abc", 1, 2, 3), 'abc, abc, 1, 2.000000, 3.00') @@ -1659,7 +1686,7 @@ def test_startswith_endswith_errors(self): self.assertIn('str', exc) self.assertIn('tuple', exc) - @support.run_with_locale('LC_ALL', 'de_DE', 'fr_FR') + @support.run_with_locale('LC_ALL', 'de_DE', 'fr_FR', '') def test_format_float(self): # should not format with a comma, but always with C locale self.assertEqual('1.0', '%.1f' % 1.0) @@ -1730,10 +1757,7 @@ def __str__(self): 'character buffers are decoded to unicode' ) - self.assertRaises(TypeError, str, 42, 42, 42) - - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_constructor_keyword_args(self): """Pass various keyword argument combinations to the constructor.""" # The object argument can be passed as a keyword. @@ -1743,8 +1767,7 @@ def test_constructor_keyword_args(self): self.assertEqual(str(b'foo', errors='strict'), 'foo') # not "b'foo'" self.assertEqual(str(object=b'foo', errors='strict'), 'foo') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_constructor_defaults(self): """Check the constructor argument defaults.""" # The object argument defaults to '' or b''. @@ -1756,8 +1779,7 @@ def test_constructor_defaults(self): # The errors argument defaults to strict. self.assertRaises(UnicodeDecodeError, str, utf8_cent, encoding='ascii') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_codecs_utf7(self): utfTests = [ ('A\u2262\u0391.', b'A+ImIDkQ.'), # RFC2152 example @@ -1910,6 +1932,12 @@ def test_utf8_decode_invalid_sequences(self): self.assertRaises(UnicodeDecodeError, (b'\xF4'+cb+b'\xBF\xBF').decode, 'utf-8') + def test_issue127903(self): + # gh-127903: ``_copy_characters`` crashes on DEBUG builds when + # there is nothing to copy. + d = datetime.datetime(2013, 11, 10, 14, 20, 59) + self.assertEqual(d.strftime('%z'), '') + def test_issue8271(self): # Issue #8271: during the decoding of an invalid UTF-8 byte sequence, # only the start byte and the continuation byte(s) are now considered @@ -2261,8 +2289,7 @@ def test_codecs_errors(self): self.assertRaises(ValueError, complex, "\ud800") self.assertRaises(ValueError, complex, "\udf00") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_codecs(self): # Encoding self.assertEqual('hello'.encode('ascii'), b'hello') @@ -2392,32 +2419,40 @@ def test_ucs4(self): else: self.fail("Should have raised UnicodeDecodeError") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_conversion(self): # Make sure __str__() works properly - class ObjectToStr: - def __str__(self): - return "foo" - - class StrSubclassToStr(str): - def __str__(self): - return "foo" - - class StrSubclassToStrSubclass(str): - def __new__(cls, content=""): - return str.__new__(cls, 2*content) - def __str__(self): + class StrWithStr(str): + def __new__(cls, value): + self = str.__new__(cls, "") + self.value = value return self + def __str__(self): + return self.value - self.assertEqual(str(ObjectToStr()), "foo") - self.assertEqual(str(StrSubclassToStr("bar")), "foo") - s = str(StrSubclassToStrSubclass("foo")) - self.assertEqual(s, "foofoo") - self.assertIs(type(s), StrSubclassToStrSubclass) - s = StrSubclass(StrSubclassToStrSubclass("foo")) - self.assertEqual(s, "foofoo") - self.assertIs(type(s), StrSubclass) + self.assertTypedEqual(str(WithStr('abc')), 'abc') + self.assertTypedEqual(str(WithStr(StrSubclass('abc'))), StrSubclass('abc')) + self.assertTypedEqual(StrSubclass(WithStr('abc')), StrSubclass('abc')) + self.assertTypedEqual(StrSubclass(WithStr(StrSubclass('abc'))), + StrSubclass('abc')) + self.assertTypedEqual(StrSubclass(WithStr(OtherStrSubclass('abc'))), + StrSubclass('abc')) + + self.assertTypedEqual(str(StrWithStr('abc')), 'abc') + self.assertTypedEqual(str(StrWithStr(StrSubclass('abc'))), StrSubclass('abc')) + self.assertTypedEqual(StrSubclass(StrWithStr('abc')), StrSubclass('abc')) + self.assertTypedEqual(StrSubclass(StrWithStr(StrSubclass('abc'))), + StrSubclass('abc')) + self.assertTypedEqual(StrSubclass(StrWithStr(OtherStrSubclass('abc'))), + StrSubclass('abc')) + + self.assertTypedEqual(str(WithRepr('<abc>')), '<abc>') + self.assertTypedEqual(str(WithRepr(StrSubclass('<abc>'))), StrSubclass('<abc>')) + self.assertTypedEqual(StrSubclass(WithRepr('<abc>')), StrSubclass('<abc>')) + self.assertTypedEqual(StrSubclass(WithRepr(StrSubclass('<abc>'))), + StrSubclass('<abc>')) + self.assertTypedEqual(StrSubclass(WithRepr(OtherStrSubclass('<abc>'))), + StrSubclass('<abc>')) def test_unicode_repr(self): class s1: @@ -2433,7 +2468,7 @@ def test_printable_repr(self): # This test only affects 32-bit platforms because expandtabs can only take # an int as the max value, not a 64-bit C long. If expandtabs is changed # to take a 64-bit long, this test should apply to all platforms. - @unittest.skip("TODO: RUSTPYTHON, oom handling") + @unittest.skip('TODO: RUSTPYTHON; oom handling') @unittest.skipIf(sys.maxsize > (1 << 32) or struct.calcsize('P') != 4, 'only applies to 32-bit platforms') def test_expandtabs_overflows_gracefully(self): @@ -2444,7 +2479,7 @@ def test_expandtabs_optimization(self): s = 'abc' self.assertIs(s.expandtabs(), s) - @unittest.skip("TODO: RUSTPYTHON, aborted: memory allocation of 9223372036854775759 bytes failed") + @unittest.skip('TODO: RUSTPYTHON; aborted: memory allocation of 9223372036854775759 bytes failed') def test_raiseMemError(self): asciifields = "nnb" compactfields = asciifields + "nP" @@ -2584,14 +2619,12 @@ def test_compare(self): self.assertTrue(astral >= bmp2) self.assertFalse(astral >= astral2) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_free_after_iterating(self): support.check_free_after_iterating(self, iter, str) support.check_free_after_iterating(self, reversed, str) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_check_encoding_errors(self): # bpo-37388: str(bytes) and str.decode() must check encoding and errors # arguments in dev mode @@ -2652,6 +2685,48 @@ def test_check_encoding_errors(self): proc = assert_python_failure('-X', 'dev', '-c', code) self.assertEqual(proc.rc, 10, proc) + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_str_invalid_call(self): + # too many args + with self.assertRaisesRegex(TypeError, r"str expected at most 3 arguments, got 4"): + str("too", "many", "argu", "ments") + with self.assertRaisesRegex(TypeError, r"str expected at most 3 arguments, got 4"): + str(1, "", "", 1) + + # no such kw arg + with self.assertRaisesRegex(TypeError, r"str\(\) got an unexpected keyword argument 'test'"): + str(test=1) + + # 'encoding' must be str + with self.assertRaisesRegex(TypeError, r"str\(\) argument 'encoding' must be str, not int"): + str(1, 1) + with self.assertRaisesRegex(TypeError, r"str\(\) argument 'encoding' must be str, not int"): + str(1, encoding=1) + with self.assertRaisesRegex(TypeError, r"str\(\) argument 'encoding' must be str, not bytes"): + str(b"x", b"ascii") + with self.assertRaisesRegex(TypeError, r"str\(\) argument 'encoding' must be str, not bytes"): + str(b"x", encoding=b"ascii") + + # 'errors' must be str + with self.assertRaisesRegex(TypeError, r"str\(\) argument 'encoding' must be str, not int"): + str(1, 1, 1) + with self.assertRaisesRegex(TypeError, r"str\(\) argument 'errors' must be str, not int"): + str(1, errors=1) + with self.assertRaisesRegex(TypeError, r"str\(\) argument 'errors' must be str, not int"): + str(1, "", errors=1) + with self.assertRaisesRegex(TypeError, r"str\(\) argument 'errors' must be str, not bytes"): + str(b"x", "ascii", b"strict") + with self.assertRaisesRegex(TypeError, r"str\(\) argument 'errors' must be str, not bytes"): + str(b"x", "ascii", errors=b"strict") + + # both positional and kwarg + with self.assertRaisesRegex(TypeError, r"argument for str\(\) given by name \('encoding'\) and position \(2\)"): + str(b"x", "utf-8", encoding="ascii") + with self.assertRaisesRegex(TypeError, r"str\(\) takes at most 3 arguments \(4 given\)"): + str(b"x", "utf-8", "ignore", encoding="ascii") + with self.assertRaisesRegex(TypeError, r"str\(\) takes at most 3 arguments \(4 given\)"): + str(b"x", "utf-8", "strict", errors="ignore") + class StringModuleTest(unittest.TestCase): def test_formatter_parser(self): diff --git a/Lib/test/test_strftime.py b/Lib/test/test_strftime.py index f5024d8e6d..be43c49e40 100644 --- a/Lib/test/test_strftime.py +++ b/Lib/test/test_strftime.py @@ -184,7 +184,6 @@ class Y1900Tests(unittest.TestCase): a date before 1900 is passed with a format string containing "%y" """ - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_y_before_1900(self): # Issue #13674, #19634 t = (1899, 1, 1, 0, 0, 0, 0, 0, 0) diff --git a/Lib/test/test_string_literals.py b/Lib/test/test_string_literals.py index 537c8fc5c8..c6b2ffb9de 100644 --- a/Lib/test/test_string_literals.py +++ b/Lib/test/test_string_literals.py @@ -105,32 +105,95 @@ def test_eval_str_incomplete(self): self.assertRaises(SyntaxError, eval, r""" '\U000000' """) self.assertRaises(SyntaxError, eval, r""" '\U0000000' """) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_eval_str_invalid_escape(self): for b in range(1, 128): if b in b"""\n\r"'01234567NU\\abfnrtuvx""": continue - with self.assertWarns(DeprecationWarning): + with self.assertWarns(SyntaxWarning): self.assertEqual(eval(r"'\%c'" % b), '\\' + chr(b)) with warnings.catch_warnings(record=True) as w: - warnings.simplefilter('always', category=DeprecationWarning) + warnings.simplefilter('always', category=SyntaxWarning) eval("'''\n\\z'''") self.assertEqual(len(w), 1) + self.assertEqual(str(w[0].message), r"invalid escape sequence '\z'") self.assertEqual(w[0].filename, '<string>') - self.assertEqual(w[0].lineno, 1) + self.assertEqual(w[0].lineno, 2) with warnings.catch_warnings(record=True) as w: - warnings.simplefilter('error', category=DeprecationWarning) + warnings.simplefilter('error', category=SyntaxWarning) with self.assertRaises(SyntaxError) as cm: eval("'''\n\\z'''") exc = cm.exception self.assertEqual(w, []) + self.assertEqual(exc.msg, r"invalid escape sequence '\z'") self.assertEqual(exc.filename, '<string>') - self.assertEqual(exc.lineno, 1) + self.assertEqual(exc.lineno, 2) + self.assertEqual(exc.offset, 1) + + # Check that the warning is raised only once if there are syntax errors + + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter('always', category=SyntaxWarning) + with self.assertRaises(SyntaxError) as cm: + eval("'\\e' $") + exc = cm.exception + self.assertEqual(len(w), 1) + self.assertEqual(w[0].category, SyntaxWarning) + self.assertRegex(str(w[0].message), 'invalid escape sequence') + self.assertEqual(w[0].filename, '<string>') + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_eval_str_invalid_octal_escape(self): + for i in range(0o400, 0o1000): + with self.assertWarns(SyntaxWarning): + self.assertEqual(eval(r"'\%o'" % i), chr(i)) + + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter('always', category=SyntaxWarning) + eval("'''\n\\407'''") + self.assertEqual(len(w), 1) + self.assertEqual(str(w[0].message), + r"invalid octal escape sequence '\407'") + self.assertEqual(w[0].filename, '<string>') + self.assertEqual(w[0].lineno, 2) + + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter('error', category=SyntaxWarning) + with self.assertRaises(SyntaxError) as cm: + eval("'''\n\\407'''") + exc = cm.exception + self.assertEqual(w, []) + self.assertEqual(exc.msg, r"invalid octal escape sequence '\407'") + self.assertEqual(exc.filename, '<string>') + self.assertEqual(exc.lineno, 2) self.assertEqual(exc.offset, 1) + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_invalid_escape_locations_with_offset(self): + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter('error', category=SyntaxWarning) + with self.assertRaises(SyntaxError) as cm: + eval("\"'''''''''''''''''''''invalid\\ Escape\"") + exc = cm.exception + self.assertEqual(w, []) + self.assertEqual(exc.msg, r"invalid escape sequence '\ '") + self.assertEqual(exc.filename, '<string>') + self.assertEqual(exc.lineno, 1) + self.assertEqual(exc.offset, 30) + + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter('error', category=SyntaxWarning) + with self.assertRaises(SyntaxError) as cm: + eval("\"''Incorrect \\ logic?\"") + exc = cm.exception + self.assertEqual(w, []) + self.assertEqual(exc.msg, r"invalid escape sequence '\ '") + self.assertEqual(exc.filename, '<string>') + self.assertEqual(exc.lineno, 1) + self.assertEqual(exc.offset, 14) + def test_eval_str_raw(self): self.assertEqual(eval(""" r'x' """), 'x') self.assertEqual(eval(r""" r'\x01' """), '\\' + 'x01') @@ -157,30 +220,56 @@ def test_eval_bytes_incomplete(self): self.assertRaises(SyntaxError, eval, r""" b'\x' """) self.assertRaises(SyntaxError, eval, r""" b'\x0' """) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_eval_bytes_invalid_escape(self): for b in range(1, 128): if b in b"""\n\r"'01234567\\abfnrtvx""": continue - with self.assertWarns(DeprecationWarning): + with self.assertWarns(SyntaxWarning): self.assertEqual(eval(r"b'\%c'" % b), b'\\' + bytes([b])) with warnings.catch_warnings(record=True) as w: - warnings.simplefilter('always', category=DeprecationWarning) + warnings.simplefilter('always', category=SyntaxWarning) eval("b'''\n\\z'''") self.assertEqual(len(w), 1) + self.assertEqual(str(w[0].message), r"invalid escape sequence '\z'") self.assertEqual(w[0].filename, '<string>') - self.assertEqual(w[0].lineno, 1) + self.assertEqual(w[0].lineno, 2) with warnings.catch_warnings(record=True) as w: - warnings.simplefilter('error', category=DeprecationWarning) + warnings.simplefilter('error', category=SyntaxWarning) with self.assertRaises(SyntaxError) as cm: eval("b'''\n\\z'''") exc = cm.exception self.assertEqual(w, []) + self.assertEqual(exc.msg, r"invalid escape sequence '\z'") self.assertEqual(exc.filename, '<string>') - self.assertEqual(exc.lineno, 1) + self.assertEqual(exc.lineno, 2) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_eval_bytes_invalid_octal_escape(self): + for i in range(0o400, 0o1000): + with self.assertWarns(SyntaxWarning): + self.assertEqual(eval(r"b'\%o'" % i), bytes([i & 0o377])) + + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter('always', category=SyntaxWarning) + eval("b'''\n\\407'''") + self.assertEqual(len(w), 1) + self.assertEqual(str(w[0].message), + r"invalid octal escape sequence '\407'") + self.assertEqual(w[0].filename, '<string>') + self.assertEqual(w[0].lineno, 2) + + with warnings.catch_warnings(record=True) as w: + warnings.simplefilter('error', category=SyntaxWarning) + with self.assertRaises(SyntaxError) as cm: + eval("b'''\n\\407'''") + exc = cm.exception + self.assertEqual(w, []) + self.assertEqual(exc.msg, r"invalid octal escape sequence '\407'") + self.assertEqual(exc.filename, '<string>') + self.assertEqual(exc.lineno, 2) def test_eval_bytes_raw(self): self.assertEqual(eval(""" br'x' """), b'x') @@ -217,6 +306,13 @@ def test_eval_str_u(self): self.assertRaises(SyntaxError, eval, """ bu'' """) self.assertRaises(SyntaxError, eval, """ ub'' """) + def test_uppercase_prefixes(self): + self.assertEqual(eval(""" B'x' """), b'x') + self.assertEqual(eval(r""" R'\x01' """), r'\x01') + self.assertEqual(eval(r""" BR'\x01' """), br'\x01') + self.assertEqual(eval(""" F'{1+1}' """), f'{1+1}') + self.assertEqual(eval(r""" U'\U0001d120' """), u'\U0001d120') + def check_encoding(self, encoding, extra=""): modname = "xx_" + encoding.replace("-", "_") fn = os.path.join(self.tmpdir, modname + ".py") diff --git a/Lib/test/test_strtod.py b/Lib/test/test_strtod.py index b8b7a9d502..45fdcca92e 100644 --- a/Lib/test/test_strtod.py +++ b/Lib/test/test_strtod.py @@ -146,7 +146,7 @@ def test_short_halfway_cases(self): digits *= 5 exponent -= 1 - @unittest.skip("TODO: RUSTPYTHON, fails on debug mode, flaky in release mode") + @unittest.skip('TODO: RUSTPYTHON; fails on debug mode, flaky in release mode') def test_halfway_cases(self): # test halfway cases for the round-half-to-even rule for i in range(100 * TEST_SIZE): @@ -173,8 +173,7 @@ def test_halfway_cases(self): s = '{}e{}'.format(digits, exponent) self.check_strtod(s) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_boundaries(self): # boundaries expressed as triples (n, e, u), where # n*10**e is an approximation to the boundary value and @@ -195,8 +194,7 @@ def test_boundaries(self): u *= 10 e -= 1 - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_underflow_boundary(self): # test values close to 2**-1075, the underflow boundary; similar # to boundary_tests, except that the random error doesn't scale @@ -208,8 +206,7 @@ def test_underflow_boundary(self): s = '{}e{}'.format(digits, exponent) self.check_strtod(s) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_bigcomp(self): for ndigs in 5, 10, 14, 15, 16, 17, 18, 19, 20, 40, 41, 50: dig10 = 10**ndigs @@ -219,8 +216,7 @@ def test_bigcomp(self): s = '{}e{}'.format(digits, exponent) self.check_strtod(s) - # TODO: RUSTPYTHON, Incorrectly rounded str->float conversion for -07e-321 - @unittest.skip("TODO: RUSTPYTHON; flaky test") + @unittest.skip('TODO: RUSTPYTHON; flaky test') def test_parsing(self): # make '0' more likely to be chosen than other digits digits = '000000123456789' @@ -288,8 +284,7 @@ def negative_exp(n): self.assertEqual(float(negative_exp(20000)), 1.0) self.assertEqual(float(negative_exp(30000)), 1.0) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_particular(self): # inputs that produced crashes or incorrectly rounded results with # previous versions of dtoa.c, for various reasons diff --git a/Lib/test/test_structseq.py b/Lib/test/test_structseq.py index 41095f63ad..a9fe193028 100644 --- a/Lib/test/test_structseq.py +++ b/Lib/test/test_structseq.py @@ -75,8 +75,6 @@ def test_cmp(self): self.assertTrue(t1 >= t2) self.assertTrue(not (t1 != t2)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_fields(self): t = time.gmtime() self.assertEqual(len(t), t.n_sequence_fields) @@ -129,8 +127,6 @@ def test_match_args(self): 'tm_sec', 'tm_wday', 'tm_yday', 'tm_isdst') self.assertEqual(time.struct_time.__match_args__, expected_args) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_match_args_with_unnamed_fields(self): expected_args = ('st_mode', 'st_ino', 'st_dev', 'st_nlink', 'st_uid', 'st_gid', 'st_size') diff --git a/Lib/test/test_subprocess.py b/Lib/test/test_subprocess.py index 5a75971be6..3917c0a76d 100644 --- a/Lib/test/test_subprocess.py +++ b/Lib/test/test_subprocess.py @@ -857,7 +857,6 @@ def test_one_environment_variable(self): self.assertEqual(p.returncode, 0) self.assertEqual(stdout.strip(), b"fruit=orange") - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON, null byte is not checked") def test_invalid_cmd(self): # null character in the command name cmd = sys.executable + '\0' @@ -1016,7 +1015,6 @@ def test_communicate_returns(self): self.assertEqual(stdout, None) self.assertEqual(stderr, None) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_communicate_pipe_buf(self): # communicate() with writes larger than pipe_buf # This test will probably deadlock rather than fail, if @@ -1337,7 +1335,6 @@ def _test_bufsize_equal_one(self, line, expected, universal_newlines): self.assertEqual(p.returncode, 0) self.assertEqual(read_line, expected) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_bufsize_equal_one_text_mode(self): # line is flushed in text mode with bufsize=1. # we should get the full line in return @@ -1773,8 +1770,7 @@ def test_run_with_pathlike_path_and_arguments(self): res = subprocess.run(args) self.assertEqual(res.returncode, 57) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.skipIf(mswindows, 'TODO: RUSTPYTHON; Flakey') @unittest.skipUnless(mswindows, "Maybe test trigger a leak on Ubuntu") def test_run_with_an_empty_env(self): # gh-105436: fix subprocess.run(..., env={}) broken on Windows @@ -1835,8 +1831,6 @@ def test_run_with_shell_timeout_and_capture_output(self): msg="TimeoutExpired was delayed! Bad traceback:\n```\n" f"{stacks}```") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_encoding_warning(self): code = textwrap.dedent("""\ from subprocess import * @@ -1883,8 +1877,6 @@ def _get_chdir_exception(self): self._nonexistent_dir) return desired_exception - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_exception_cwd(self): """Test error in the child raised in the parent for a bad cwd.""" desired_exception = self._get_chdir_exception() @@ -1900,8 +1892,6 @@ def test_exception_cwd(self): else: self.fail("Expected OSError: %s" % desired_exception) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_exception_bad_executable(self): """Test error in the child raised in the parent for a bad executable.""" desired_exception = self._get_chdir_exception() @@ -1917,8 +1907,6 @@ def test_exception_bad_executable(self): else: self.fail("Expected OSError: %s" % desired_exception) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_exception_bad_args_0(self): """Test error in the child raised in the parent for a bad args[0].""" desired_exception = self._get_chdir_exception() @@ -2256,8 +2244,6 @@ def test_CalledProcessError_str_non_zero(self): error_string = str(err) self.assertIn("non-zero exit status 2.", error_string) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_preexec(self): # DISCLAIMER: Setting environment variables is *not* a good use # of a preexec_fn. This is merely a test. @@ -2269,8 +2255,6 @@ def test_preexec(self): with p: self.assertEqual(p.stdout.read(), b"apple") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_preexec_exception(self): def raise_it(): raise ValueError("What if two swallows carried a coconut?") @@ -2312,8 +2296,6 @@ def _execute_child(self, *args, **kwargs): for fd in devzero_fds: os.close(fd) - # TODO: RUSTPYTHON - @unittest.expectedFailure @unittest.skipIf(not os.path.exists("/dev/zero"), "/dev/zero required.") def test_preexec_errpipe_does_not_double_close_pipes(self): """Issue16140: Don't double close pipes on preexec error.""" @@ -2351,8 +2333,6 @@ def test_preexec_gc_module_failure(self): if not enabled: gc.disable() - # TODO: RUSTPYTHON - @unittest.expectedFailure @unittest.skipIf( sys.platform == 'darwin', 'setrlimit() seems to fail on OS X') def test_preexec_fork_failure(self): @@ -2763,8 +2743,6 @@ def test_swap_std_fds_with_one_closed(self): for to_fds in itertools.permutations(range(3), 2): self._check_swap_std_fds_with_one_closed(from_fds, to_fds) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_surrogates_error_message(self): def prepare(): raise ValueError("surrogate:\uDCff") @@ -3240,8 +3218,6 @@ def test_leak_fast_process_del_killed(self): else: self.assertNotIn(ident, [id(o) for o in subprocess._active]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_close_fds_after_preexec(self): fd_status = support.findfile("fd_status.py", subdir="subprocessdata") @@ -3565,8 +3541,6 @@ def test_close_fds(self): close_fds=True) self.assertEqual(rc, 47) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_close_fds_with_stdio(self): import msvcrt @@ -3649,8 +3623,6 @@ def test_shell_string(self): with p: self.assertIn(b"physalis", p.stdout.read()) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_shell_encodings(self): # Run command through the shell (string) for enc in ['ansi', 'oem']: @@ -3869,28 +3841,20 @@ def with_spaces(self, *args, **kwargs): "2 [%r, 'ab cd']" % self.fname ) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_shell_string_with_spaces(self): # call() function with string argument with spaces on Windows self.with_spaces('"%s" "%s" "%s"' % (sys.executable, self.fname, "ab cd"), shell=1) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_shell_sequence_with_spaces(self): # call() function with sequence argument with spaces on Windows self.with_spaces([sys.executable, self.fname, "ab cd"], shell=1) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_noshell_string_with_spaces(self): # call() function with string argument with spaces on Windows self.with_spaces('"%s" "%s" "%s"' % (sys.executable, self.fname, "ab cd")) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_noshell_sequence_with_spaces(self): # call() function with sequence argument with spaces on Windows self.with_spaces([sys.executable, self.fname, "ab cd"]) diff --git a/Lib/test/test_sundry.py b/Lib/test/test_sundry.py index 90af9da8f9..f4a8d434ed 100644 --- a/Lib/test/test_sundry.py +++ b/Lib/test/test_sundry.py @@ -1,14 +1,11 @@ """Do a minimal test of all the modules that aren't otherwise tested.""" import importlib -import platform -import sys from test import support from test.support import import_helper from test.support import warnings_helper import unittest class TestUntestedModules(unittest.TestCase): - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_untested_modules_can_be_imported(self): untested = ('encodings',) with warnings_helper.check_warnings(quiet=True): @@ -21,31 +18,6 @@ def test_untested_modules_can_be_imported(self): self.fail('{} has tests even though test_sundry claims ' 'otherwise'.format(name)) - import distutils.bcppcompiler - import distutils.ccompiler - import distutils.cygwinccompiler - import distutils.filelist - import distutils.text_file - import distutils.unixccompiler - - import distutils.command.bdist_dumb - if sys.platform.startswith('win') and not platform.win32_is_iot(): - import distutils.command.bdist_msi - import distutils.command.bdist - import distutils.command.bdist_rpm - import distutils.command.build_clib - import distutils.command.build_ext - import distutils.command.build - import distutils.command.clean - import distutils.command.config - import distutils.command.install_data - import distutils.command.install_egg_info - import distutils.command.install_headers - import distutils.command.install_lib - import distutils.command.register - import distutils.command.sdist - import distutils.command.upload - import html.entities try: @@ -54,5 +26,6 @@ def test_untested_modules_can_be_imported(self): if support.verbose: print("skipping tty") + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_super.py b/Lib/test/test_super.py index ca8dc6e869..76eda799da 100644 --- a/Lib/test/test_super.py +++ b/Lib/test/test_super.py @@ -1,6 +1,13 @@ """Unit tests for zero-argument super() & related machinery.""" +import textwrap +import threading import unittest +from unittest.mock import patch +from test.support import import_helper, threading_helper + + +ADAPTIVE_WARMUP_DELAY = 2 class A: @@ -84,41 +91,43 @@ def nested(): self.assertEqual(E().f(), 'AE') - # SyntaxError - # def test_various___class___pathologies(self): - # # See issue #12370 - # class X(A): - # def f(self): - # return super().f() - # __class__ = 413 - # x = X() - # self.assertEqual(x.f(), 'A') - # self.assertEqual(x.__class__, 413) - # class X: - # x = __class__ - # def f(): - # __class__ - # self.assertIs(X.x, type(self)) - # with self.assertRaises(NameError) as e: - # exec("""class X: - # __class__ - # def f(): - # __class__""", globals(), {}) - # self.assertIs(type(e.exception), NameError) # Not UnboundLocalError - # class X: - # global __class__ - # __class__ = 42 - # def f(): - # __class__ - # self.assertEqual(globals()["__class__"], 42) - # del globals()["__class__"] - # self.assertNotIn("__class__", X.__dict__) - # class X: - # nonlocal __class__ - # __class__ = 42 - # def f(): - # __class__ - # self.assertEqual(__class__, 42) + # TODO: RUSTPYTHON; SyntaxError: name '__class__' is assigned to before global declaration + ''' + def test_various___class___pathologies(self): + # See issue #12370 + class X(A): + def f(self): + return super().f() + __class__ = 413 + x = X() + self.assertEqual(x.f(), 'A') + self.assertEqual(x.__class__, 413) + class X: + x = __class__ + def f(): + __class__ + self.assertIs(X.x, type(self)) + with self.assertRaises(NameError) as e: + exec("""class X: + __class__ + def f(): + __class__""", globals(), {}) + self.assertIs(type(e.exception), NameError) # Not UnboundLocalError + class X: + global __class__ + __class__ = 42 + def f(): + __class__ + self.assertEqual(globals()["__class__"], 42) + del globals()["__class__"] + self.assertNotIn("__class__", X.__dict__) + class X: + nonlocal __class__ + __class__ = 42 + def f(): + __class__ + self.assertEqual(__class__, 42) + ''' def test___class___instancemethod(self): # See issue #14857 @@ -162,8 +171,6 @@ def f(): self.assertIs(test_class, A) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test___class___delayed(self): # See issue #23722 test_namespace = None @@ -184,8 +191,7 @@ def f(): B = type("B", (), test_namespace) self.assertIs(B.f(), B) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test___class___mro(self): # See issue #23722 test_class = None @@ -203,8 +209,7 @@ def f(): self.assertIs(test_class, A) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test___classcell___expected_behaviour(self): # See issue #23722 class Meta(type): @@ -236,8 +241,6 @@ def f(self): with self.assertRaises(AttributeError): WithClassRef.__classcell__ - # TODO: RUSTPYTHON - @unittest.expectedFailure def test___classcell___missing(self): # See issue #23722 # Some metaclasses may not pass the original namespace to type.__new__ @@ -292,17 +295,28 @@ def f(self): def test_obscure_super_errors(self): def f(): super() - self.assertRaises(RuntimeError, f) + with self.assertRaisesRegex(RuntimeError, r"no arguments"): + f() + + class C: + def f(): + super() + with self.assertRaisesRegex(RuntimeError, r"no arguments"): + C.f() + def f(x): del x super() - self.assertRaises(RuntimeError, f, None) + with self.assertRaisesRegex(RuntimeError, r"arg\[0\] deleted"): + f(None) + class X: def f(x): nonlocal __class__ del __class__ super() - self.assertRaises(RuntimeError, X().f) + with self.assertRaisesRegex(RuntimeError, r"empty __class__ cell"): + X().f() def test_cell_as_self(self): class X: @@ -326,6 +340,213 @@ def test_super_init_leaks(self): for i in range(1000): super.__init__(sp, int, i) + def test_super_argcount(self): + with self.assertRaisesRegex(TypeError, "expected at most"): + super(int, int, int) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: "argument 1 must be a type" does not match "Expected type 'type' but 'int' found." + def test_super_argtype(self): + with self.assertRaisesRegex(TypeError, "argument 1 must be a type"): + super(1, int) + + def test_shadowed_global(self): + source = textwrap.dedent( + """ + class super: + msg = "truly super" + + class C: + def method(self): + return super().msg + """, + ) + with import_helper.ready_to_import(name="shadowed_super", source=source): + import shadowed_super + self.assertEqual(shadowed_super.C().method(), "truly super") + import_helper.unload("shadowed_super") + + def test_shadowed_local(self): + class super: + msg = "quite super" + + class C: + def method(self): + return super().msg + + self.assertEqual(C().method(), "quite super") + + def test_shadowed_dynamic(self): + class MySuper: + msg = "super super" + + class C: + def method(self): + return super().msg + + with patch(f"{__name__}.super", MySuper) as m: + self.assertEqual(C().method(), "super super") + + def test_shadowed_dynamic_two_arg(self): + call_args = [] + class MySuper: + def __init__(self, *args): + call_args.append(args) + msg = "super super" + + class C: + def method(self): + return super(1, 2).msg + + with patch(f"{__name__}.super", MySuper) as m: + self.assertEqual(C().method(), "super super") + self.assertEqual(call_args, [(1, 2)]) + + def test_attribute_error(self): + class C: + def method(self): + return super().msg + + with self.assertRaisesRegex(AttributeError, "'super' object has no attribute 'msg'"): + C().method() + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: "argument 1 must be a type" does not match "Expected type 'type' but 'int' found." + def test_bad_first_arg(self): + class C: + def method(self): + return super(1, self).method() + + with self.assertRaisesRegex(TypeError, "argument 1 must be a type"): + C().method() + + def test_supercheck_fail(self): + class C: + def method(self, type_, obj): + return super(type_, obj).method() + + c = C() + err_msg = ( + r"super\(type, obj\): obj \({} {}\) is not " + r"an instance or subtype of type \({}\)." + ) + + cases = ( + (int, c, int.__name__, C.__name__, "instance of"), + # obj is instance of type + (C, list(), C.__name__, list.__name__, "instance of"), + # obj is type itself + (C, list, C.__name__, list.__name__, "type"), + ) + + for case in cases: + with self.subTest(case=case): + type_, obj, type_str, obj_str, instance_or_type = case + regex = err_msg.format(instance_or_type, obj_str, type_str) + + with self.assertRaisesRegex(TypeError, regex): + c.method(type_, obj) + + def test_super___class__(self): + class C: + def method(self): + return super().__class__ + + self.assertEqual(C().method(), super) + + @unittest.expectedFailure # TODO: RUSTPYTHON; TypeError: type 'super' is not an acceptable base type + def test_super_subclass___class__(self): + class mysuper(super): + pass + + class C: + def method(self): + return mysuper(C, self).__class__ + + self.assertEqual(C().method(), mysuper) + + def test_unusual_getattro(self): + class MyType(type): + pass + + def test(name): + mytype = MyType(name, (MyType,), {}) + super(MyType, type(mytype)).__setattr__(mytype, "bar", 1) + self.assertEqual(mytype.bar, 1) + + for _ in range(ADAPTIVE_WARMUP_DELAY): + test("foo1") + + def test_reassigned_new(self): + class A: + def __new__(cls): + pass + + def __init_subclass__(cls): + if "__new__" not in cls.__dict__: + cls.__new__ = cls.__new__ + + class B(A): + pass + + class C(B): + def __new__(cls): + return super().__new__(cls) + + for _ in range(ADAPTIVE_WARMUP_DELAY): + C() + + def test_mixed_staticmethod_hierarchy(self): + # This test is just a desugared version of `test_reassigned_new` + class A: + @staticmethod + def some(cls, *args, **kwargs): + self.assertFalse(args) + self.assertFalse(kwargs) + + class B(A): + def some(cls, *args, **kwargs): + return super().some(cls, *args, **kwargs) + + class C(B): + @staticmethod + def some(cls): + return super().some(cls) + + for _ in range(ADAPTIVE_WARMUP_DELAY): + C.some(C) + + @threading_helper.requires_working_threading() + def test___class___modification_multithreaded(self): + """ Note: this test isn't actually testing anything on its own. + It requires a sys audithook to be set to crash on older Python. + This should be the case anyways as our test suite sets + an audit hook. + """ + + class Foo: + pass + + class Bar: + pass + + thing = Foo() + def work(): + foo = thing + for _ in range(200): + foo.__class__ = Bar + type(foo) + foo.__class__ = Foo + type(foo) + + + threads = [] + for _ in range(6): + thread = threading.Thread(target=work) + thread.start() + threads.append(thread) + + for thread in threads: + thread.join() + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_support.py b/Lib/test/test_support.py index 4381a82b6b..7c8380498e 100644 --- a/Lib/test/test_support.py +++ b/Lib/test/test_support.py @@ -310,7 +310,6 @@ def test_temp_cwd__name_none(self): def test_sortdict(self): self.assertEqual(support.sortdict({3:3, 2:2, 1:1}), "{1: 1, 2: 2, 3: 3}") - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON; actual c fds on windows") def test_make_bad_fd(self): fd = os_helper.make_bad_fd() with self.assertRaises(OSError) as cm: diff --git a/Lib/test/test_symtable.py b/Lib/test/test_symtable.py index ca62b9d685..8f1a80a524 100644 --- a/Lib/test/test_symtable.py +++ b/Lib/test/test_symtable.py @@ -1,9 +1,13 @@ """ Test the API of the symtable module. """ + +import textwrap import symtable import unittest +from test import support +from test.support import os_helper TEST_CODE = """ @@ -11,7 +15,7 @@ glob = 42 some_var = 12 -some_non_assigned_global_var = 11 +some_non_assigned_global_var: int some_assigned_global_var = 11 class Mine: @@ -40,6 +44,129 @@ def foo(): def namespace_test(): pass def namespace_test(): pass + +type Alias = int +type GenericAlias[T] = list[T] + +def generic_spam[T](a): + pass + +class GenericMine[T: int, U: (int, str) = int]: + pass +""" + +TEST_COMPLEX_CLASS_CODE = """ +# The following symbols are defined in ComplexClass +# without being introduced by a 'global' statement. +glob_unassigned_meth: Any +glob_unassigned_meth_pep_695: Any + +glob_unassigned_async_meth: Any +glob_unassigned_async_meth_pep_695: Any + +def glob_assigned_meth(): pass +def glob_assigned_meth_pep_695[T](): pass + +async def glob_assigned_async_meth(): pass +async def glob_assigned_async_meth_pep_695[T](): pass + +# The following symbols are defined in ComplexClass after +# being introduced by a 'global' statement (and therefore +# are not considered as local symbols of ComplexClass). +glob_unassigned_meth_ignore: Any +glob_unassigned_meth_pep_695_ignore: Any + +glob_unassigned_async_meth_ignore: Any +glob_unassigned_async_meth_pep_695_ignore: Any + +def glob_assigned_meth_ignore(): pass +def glob_assigned_meth_pep_695_ignore[T](): pass + +async def glob_assigned_async_meth_ignore(): pass +async def glob_assigned_async_meth_pep_695_ignore[T](): pass + +class ComplexClass: + a_var = 1234 + a_genexpr = (x for x in []) + a_lambda = lambda x: x + + type a_type_alias = int + type a_type_alias_pep_695[T] = list[T] + + class a_class: pass + class a_class_pep_695[T]: pass + + def a_method(self): pass + def a_method_pep_695[T](self): pass + + async def an_async_method(self): pass + async def an_async_method_pep_695[T](self): pass + + @classmethod + def a_classmethod(cls): pass + @classmethod + def a_classmethod_pep_695[T](self): pass + + @classmethod + async def an_async_classmethod(cls): pass + @classmethod + async def an_async_classmethod_pep_695[T](self): pass + + @staticmethod + def a_staticmethod(): pass + @staticmethod + def a_staticmethod_pep_695[T](self): pass + + @staticmethod + async def an_async_staticmethod(): pass + @staticmethod + async def an_async_staticmethod_pep_695[T](self): pass + + # These ones will be considered as methods because of the 'def' although + # they are *not* valid methods at runtime since they are not decorated + # with @staticmethod. + def a_fakemethod(): pass + def a_fakemethod_pep_695[T](): pass + + async def an_async_fakemethod(): pass + async def an_async_fakemethod_pep_695[T](): pass + + # Check that those are still considered as methods + # since they are not using the 'global' keyword. + def glob_unassigned_meth(): pass + def glob_unassigned_meth_pep_695[T](): pass + + async def glob_unassigned_async_meth(): pass + async def glob_unassigned_async_meth_pep_695[T](): pass + + def glob_assigned_meth(): pass + def glob_assigned_meth_pep_695[T](): pass + + async def glob_assigned_async_meth(): pass + async def glob_assigned_async_meth_pep_695[T](): pass + + # The following are not picked as local symbols because they are not + # visible by the class at runtime (this is equivalent to having the + # definitions outside of the class). + global glob_unassigned_meth_ignore + def glob_unassigned_meth_ignore(): pass + global glob_unassigned_meth_pep_695_ignore + def glob_unassigned_meth_pep_695_ignore[T](): pass + + global glob_unassigned_async_meth_ignore + async def glob_unassigned_async_meth_ignore(): pass + global glob_unassigned_async_meth_pep_695_ignore + async def glob_unassigned_async_meth_pep_695_ignore[T](): pass + + global glob_assigned_meth_ignore + def glob_assigned_meth_ignore(): pass + global glob_assigned_meth_pep_695_ignore + def glob_assigned_meth_pep_695_ignore[T](): pass + + global glob_assigned_async_meth_ignore + async def glob_assigned_async_meth_ignore(): pass + global glob_assigned_async_meth_pep_695_ignore + async def glob_assigned_async_meth_pep_695_ignore[T](): pass """ @@ -54,18 +181,46 @@ class SymtableTest(unittest.TestCase): top = symtable.symtable(TEST_CODE, "?", "exec") # These correspond to scopes in TEST_CODE Mine = find_block(top, "Mine") + a_method = find_block(Mine, "a_method") spam = find_block(top, "spam") internal = find_block(spam, "internal") other_internal = find_block(spam, "other_internal") foo = find_block(top, "foo") + Alias = find_block(top, "Alias") + GenericAlias = find_block(top, "GenericAlias") + # XXX: RUSTPYTHON + # GenericAlias_inner = find_block(GenericAlias, "GenericAlias") + generic_spam = find_block(top, "generic_spam") + # XXX: RUSTPYTHON + # generic_spam_inner = find_block(generic_spam, "generic_spam") + GenericMine = find_block(top, "GenericMine") + # XXX: RUSTPYTHON + # GenericMine_inner = find_block(GenericMine, "GenericMine") + # XXX: RUSTPYTHON + # T = find_block(GenericMine, "T") + # XXX: RUSTPYTHON + # U = find_block(GenericMine, "U") + + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_type(self): self.assertEqual(self.top.get_type(), "module") self.assertEqual(self.Mine.get_type(), "class") self.assertEqual(self.a_method.get_type(), "function") self.assertEqual(self.spam.get_type(), "function") self.assertEqual(self.internal.get_type(), "function") + self.assertEqual(self.foo.get_type(), "function") + self.assertEqual(self.Alias.get_type(), "type alias") + self.assertEqual(self.GenericAlias.get_type(), "type parameters") + self.assertEqual(self.GenericAlias_inner.get_type(), "type alias") + self.assertEqual(self.generic_spam.get_type(), "type parameters") + self.assertEqual(self.generic_spam_inner.get_type(), "function") + self.assertEqual(self.GenericMine.get_type(), "type parameters") + self.assertEqual(self.GenericMine_inner.get_type(), "class") + self.assertEqual(self.T.get_type(), "type variable") + self.assertEqual(self.U.get_type(), "type variable") # TODO: RUSTPYTHON @unittest.expectedFailure @@ -75,6 +230,11 @@ def test_id(self): self.assertGreater(self.a_method.get_id(), 0) self.assertGreater(self.spam.get_id(), 0) self.assertGreater(self.internal.get_id(), 0) + self.assertGreater(self.foo.get_id(), 0) + self.assertGreater(self.Alias.get_id(), 0) + self.assertGreater(self.GenericAlias.get_id(), 0) + self.assertGreater(self.generic_spam.get_id(), 0) + self.assertGreater(self.GenericMine.get_id(), 0) def test_optimized(self): self.assertFalse(self.top.is_optimized()) @@ -106,6 +266,8 @@ def test_function_info(self): self.assertEqual(sorted(func.get_globals()), ["bar", "glob", "some_assigned_global_var"]) self.assertEqual(self.internal.get_frees(), ("x",)) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_globals(self): self.assertTrue(self.spam.lookup("glob").is_global()) self.assertFalse(self.spam.lookup("glob").is_declared_global()) @@ -126,6 +288,8 @@ def test_nonlocal(self): expected = ("some_var",) self.assertEqual(self.other_internal.get_nonlocals(), expected) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_local(self): self.assertTrue(self.spam.lookup("x").is_local()) self.assertFalse(self.spam.lookup("bar").is_local()) @@ -133,9 +297,13 @@ def test_local(self): self.assertTrue(self.top.lookup("some_non_assigned_global_var").is_local()) self.assertTrue(self.top.lookup("some_assigned_global_var").is_local()) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_free(self): self.assertTrue(self.internal.lookup("x").is_free()) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_referenced(self): self.assertTrue(self.internal.lookup("x").is_referenced()) self.assertTrue(self.spam.lookup("internal").is_referenced()) @@ -167,6 +335,10 @@ def test_namespaces(self): self.assertEqual(len(ns_test.get_namespaces()), 2) self.assertRaises(ValueError, ns_test.get_namespace) + ns_test_2 = self.top.lookup("glob") + self.assertEqual(len(ns_test_2.get_namespaces()), 0) + self.assertRaises(ValueError, ns_test_2.get_namespace) + def test_assigned(self): self.assertTrue(self.spam.lookup("x").is_assigned()) self.assertTrue(self.spam.lookup("bar").is_assigned()) @@ -174,6 +346,8 @@ def test_assigned(self): self.assertTrue(self.Mine.lookup("a_method").is_assigned()) self.assertFalse(self.internal.lookup("x").is_assigned()) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_annotated(self): st1 = symtable.symtable('def f():\n x: int\n', 'test', 'exec') st2 = st1.get_children()[0] @@ -199,6 +373,8 @@ def test_annotated(self): ' x: int', 'test', 'exec') + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_imported(self): self.assertTrue(self.top.lookup("sys").is_imported()) @@ -210,9 +386,79 @@ def test_name(self): # TODO: RUSTPYTHON @unittest.expectedFailure - def test_class_info(self): + def test_class_get_methods(self): self.assertEqual(self.Mine.get_methods(), ('a_method',)) + top = symtable.symtable(TEST_COMPLEX_CLASS_CODE, "?", "exec") + this = find_block(top, "ComplexClass") + + self.assertEqual(this.get_methods(), ( + 'a_method', 'a_method_pep_695', + 'an_async_method', 'an_async_method_pep_695', + 'a_classmethod', 'a_classmethod_pep_695', + 'an_async_classmethod', 'an_async_classmethod_pep_695', + 'a_staticmethod', 'a_staticmethod_pep_695', + 'an_async_staticmethod', 'an_async_staticmethod_pep_695', + 'a_fakemethod', 'a_fakemethod_pep_695', + 'an_async_fakemethod', 'an_async_fakemethod_pep_695', + 'glob_unassigned_meth', 'glob_unassigned_meth_pep_695', + 'glob_unassigned_async_meth', 'glob_unassigned_async_meth_pep_695', + 'glob_assigned_meth', 'glob_assigned_meth_pep_695', + 'glob_assigned_async_meth', 'glob_assigned_async_meth_pep_695', + )) + + # Test generator expressions that are of type TYPE_FUNCTION + # but will not be reported by get_methods() since they are + # not functions per se. + # + # Other kind of comprehensions such as list, set or dict + # expressions do not have the TYPE_FUNCTION type. + + def check_body(body, expected_methods): + indented = textwrap.indent(body, ' ' * 4) + top = symtable.symtable(f"class A:\n{indented}", "?", "exec") + this = find_block(top, "A") + self.assertEqual(this.get_methods(), expected_methods) + + # statements with 'genexpr' inside it + GENEXPRS = ( + 'x = (x for x in [])', + 'x = (x async for x in [])', + 'type x[genexpr = (x for x in [])] = (x for x in [])', + 'type x[genexpr = (x async for x in [])] = (x async for x in [])', + 'genexpr = (x for x in [])', + 'genexpr = (x async for x in [])', + 'type genexpr[genexpr = (x for x in [])] = (x for x in [])', + 'type genexpr[genexpr = (x async for x in [])] = (x async for x in [])', + ) + + for gen in GENEXPRS: + # test generator expression + with self.subTest(gen=gen): + check_body(gen, ()) + + # test generator expression + variable named 'genexpr' + with self.subTest(gen=gen, isvar=True): + check_body('\n'.join((gen, 'genexpr = 1')), ()) + check_body('\n'.join(('genexpr = 1', gen)), ()) + + for paramlist in ('()', '(x)', '(x, y)', '(z: T)'): + for func in ( + f'def genexpr{paramlist}:pass', + f'async def genexpr{paramlist}:pass', + f'def genexpr[T]{paramlist}:pass', + f'async def genexpr[T]{paramlist}:pass', + ): + with self.subTest(func=func): + # test function named 'genexpr' + check_body(func, ('genexpr',)) + + for gen in GENEXPRS: + with self.subTest(gen=gen, func=func): + # test generator expression + function named 'genexpr' + check_body('\n'.join((gen, func)), ('genexpr',)) + check_body('\n'.join((func, gen)), ('genexpr',)) + # TODO: RUSTPYTHON @unittest.expectedFailure def test_filename_correct(self): @@ -230,10 +476,9 @@ def checkfilename(brokencode, offset): checkfilename("def f(x): foo)(", 14) # parse-time checkfilename("def f(x): global x", 11) # symtable-build-time symtable.symtable("pass", b"spam", "exec") - with self.assertWarns(DeprecationWarning), \ - self.assertRaises(TypeError): + with self.assertRaises(TypeError): symtable.symtable("pass", bytearray(b"spam"), "exec") - with self.assertWarns(DeprecationWarning): + with self.assertRaises(TypeError): symtable.symtable("pass", memoryview(b"spam"), "exec") with self.assertRaises(TypeError): symtable.symtable("pass", list(b"spam"), "exec") @@ -258,12 +503,71 @@ def test_bytes(self): top = symtable.symtable(code, "?", "exec") self.assertIsNotNone(find_block(top, "\u017d")) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_symtable_repr(self): self.assertEqual(str(self.top), "<SymbolTable for module ?>") self.assertEqual(str(self.spam), "<Function SymbolTable for spam in ?>") + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_symbol_repr(self): + self.assertEqual(repr(self.spam.lookup("glob")), + "<symbol 'glob': GLOBAL_IMPLICIT, USE>") + self.assertEqual(repr(self.spam.lookup("bar")), + "<symbol 'bar': GLOBAL_EXPLICIT, DEF_GLOBAL|DEF_LOCAL>") + self.assertEqual(repr(self.spam.lookup("a")), + "<symbol 'a': LOCAL, DEF_PARAM>") + self.assertEqual(repr(self.spam.lookup("internal")), + "<symbol 'internal': LOCAL, USE|DEF_LOCAL>") + self.assertEqual(repr(self.spam.lookup("other_internal")), + "<symbol 'other_internal': LOCAL, DEF_LOCAL>") + self.assertEqual(repr(self.internal.lookup("x")), + "<symbol 'x': FREE, USE>") + self.assertEqual(repr(self.other_internal.lookup("some_var")), + "<symbol 'some_var': FREE, USE|DEF_NONLOCAL|DEF_LOCAL>") + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_symtable_entry_repr(self): + expected = f"<symtable entry top({self.top.get_id()}), line {self.top.get_lineno()}>" + self.assertEqual(repr(self.top._table), expected) + + +class CommandLineTest(unittest.TestCase): + maxDiff = None + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_file(self): + filename = os_helper.TESTFN + self.addCleanup(os_helper.unlink, filename) + with open(filename, 'w') as f: + f.write(TEST_CODE) + with support.captured_stdout() as stdout: + symtable.main([filename]) + out = stdout.getvalue() + self.assertIn('\n\n', out) + self.assertNotIn('\n\n\n', out) + lines = out.splitlines() + self.assertIn(f"symbol table for module from file {filename!r}:", lines) + self.assertIn(" local symbol 'glob': def_local", lines) + self.assertIn(" global_implicit symbol 'glob': use", lines) + self.assertIn(" local symbol 'spam': def_local", lines) + self.assertIn(" symbol table for function 'spam':", lines) + + def test_stdin(self): + with support.captured_stdin() as stdin: + stdin.write(TEST_CODE) + stdin.seek(0) + with support.captured_stdout() as stdout: + symtable.main([]) + out = stdout.getvalue() + stdin.seek(0) + with support.captured_stdout() as stdout: + symtable.main(['-']) + self.assertEqual(stdout.getvalue(), out) + lines = out.splitlines() + self.assertIn("symbol table for module from file '<stdin>':", lines) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_syntax.py b/Lib/test/test_syntax.py index 7e46773047..29dd04995e 100644 --- a/Lib/test/test_syntax.py +++ b/Lib/test/test_syntax.py @@ -259,6 +259,36 @@ Traceback (most recent call last): SyntaxError: invalid syntax +Comprehensions without 'in' keyword: + +>>> [x for x if range(1)] +Traceback (most recent call last): +SyntaxError: 'in' expected after for-loop variables + +>>> tuple(x for x if range(1)) +Traceback (most recent call last): +SyntaxError: 'in' expected after for-loop variables + +>>> [x for x() in a] +Traceback (most recent call last): +SyntaxError: cannot assign to function call + +>>> [x for a, b, (c + 1, d()) in y] +Traceback (most recent call last): +SyntaxError: cannot assign to expression + +>>> [x for a, b, (c + 1, d()) if y] +Traceback (most recent call last): +SyntaxError: 'in' expected after for-loop variables + +>>> [x for x+1 in y] +Traceback (most recent call last): +SyntaxError: cannot assign to expression + +>>> [x for x+1, x() in y] +Traceback (most recent call last): +SyntaxError: cannot assign to expression + Comprehensions creating tuples without parentheses should produce a specialized error message: @@ -322,6 +352,13 @@ Traceback (most recent call last): SyntaxError: invalid syntax +# But prefixes of soft keywords should +# still raise specialized errors + +>>> (mat x) +Traceback (most recent call last): +SyntaxError: invalid syntax. Perhaps you forgot a comma? + From compiler_complex_args(): >>> def f(None=1): @@ -334,7 +371,12 @@ >>> def f(x, y=1, z): ... pass Traceback (most recent call last): -SyntaxError: non-default argument follows default argument +SyntaxError: parameter without a default follows parameter with a default + +>>> def f(x, /, y=1, z): +... pass +Traceback (most recent call last): +SyntaxError: parameter without a default follows parameter with a default >>> def f(x, None): ... pass @@ -555,8 +597,16 @@ Traceback (most recent call last): SyntaxError: expected default value expression -# TODO: RUSTPYTHON NameError: name 'PyCF_TYPE_COMMENTS' is not defined ->>> import ast; ast.parse(''' # doctest: +SKIP +>>> lambda a,d=3,c: None +Traceback (most recent call last): +SyntaxError: parameter without a default follows parameter with a default + +>>> lambda a,/,d=3,c: None +Traceback (most recent call last): +SyntaxError: parameter without a default follows parameter with a default + +>>> # TODO: RUSTPYTHON +>>> import ast; ast.parse(''' # doctest: +SKIP ... def f( ... *, # type: int ... a, # type: int @@ -582,46 +632,31 @@ >>> L = range(10) >>> f(x for x in L) [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] - -# TODO: RUSTPYTHON does not raise. ->>> f(x for x in L, 1) # doctest: +SKIP +>>> f(x for x in L, 1) Traceback (most recent call last): SyntaxError: Generator expression must be parenthesized - -# TODO: RUSTPYTHON does not raise. ->>> f(x for x in L, y=1) # doctest: +SKIP +>>> f(x for x in L, y=1) Traceback (most recent call last): SyntaxError: Generator expression must be parenthesized - -# TODO: RUSTPYTHON does not raise. ->>> f(x for x in L, *[]) # doctest: +SKIP +>>> f(x for x in L, *[]) Traceback (most recent call last): SyntaxError: Generator expression must be parenthesized - -# TODO: RUSTPYTHON does not raise. ->>> f(x for x in L, **{}) # doctest: +SKIP +>>> f(x for x in L, **{}) Traceback (most recent call last): SyntaxError: Generator expression must be parenthesized - -# TODO: RUSTPYTHON does not raise. ->>> f(L, x for x in L) # doctest: +SKIP +>>> f(L, x for x in L) Traceback (most recent call last): SyntaxError: Generator expression must be parenthesized - -# TODO: RUSTPYTHON does not raise. ->>> f(x for x in L, y for y in L) # doctest: +SKIP +>>> f(x for x in L, y for y in L) Traceback (most recent call last): SyntaxError: Generator expression must be parenthesized - -# TODO: RUSTPYTHON does not raise. ->>> f(x for x in L,) # doctest: +SKIP +>>> f(x for x in L,) Traceback (most recent call last): SyntaxError: Generator expression must be parenthesized >>> f((x for x in L), 1) [0, 1, 2, 3, 4, 5, 6, 7, 8, 9] - -# TODO: RUSTPYTHON TypeError: metaclass conflict: the metaclass of a derived class must be a (non-strict) subclass of the metaclasses of all its bases ->>> class C(x for x in L): # doctest: +SKIP +>>> # TODO: RUSTPYTHON +>>> class C(x for x in L): # doctest: +SKIP ... pass Traceback (most recent call last): SyntaxError: invalid syntax @@ -742,8 +777,8 @@ >>> f(x.y=1) Traceback (most recent call last): SyntaxError: expression cannot contain assignment, perhaps you meant "=="? -# TODO: RUSTPYTHON ->>> f((x)=2) # doctest: +SKIP +>>> # TODO: RUSTPYTHON +>>> f((x)=2) # doctest: +SKIP Traceback (most recent call last): SyntaxError: expression cannot contain assignment, perhaps you meant "=="? >>> f(True=1) @@ -758,11 +793,31 @@ >>> f(__debug__=1) Traceback (most recent call last): SyntaxError: cannot assign to __debug__ - -# TODO: RUSTPYTHON NameError: name '__annotations__' is not defined ->>> __debug__: int # doctest: +SKIP +>>> # TODO: RUSTPYTHON +>>> __debug__: int # doctest: +SKIP Traceback (most recent call last): SyntaxError: cannot assign to __debug__ +>>> f(a=) +Traceback (most recent call last): +SyntaxError: expected argument value expression +>>> f(a, b, c=) +Traceback (most recent call last): +SyntaxError: expected argument value expression +>>> f(a, b, c=, d) +Traceback (most recent call last): +SyntaxError: expected argument value expression +>>> f(*args=[0]) +Traceback (most recent call last): +SyntaxError: cannot assign to iterable argument unpacking +>>> f(a, b, *args=[0]) +Traceback (most recent call last): +SyntaxError: cannot assign to iterable argument unpacking +>>> f(**kwargs={'a': 1}) +Traceback (most recent call last): +SyntaxError: cannot assign to keyword argument unpacking +>>> f(a, b, *args, **kwargs={'a': 1}) +Traceback (most recent call last): +SyntaxError: cannot assign to keyword argument unpacking More set_context(): @@ -990,11 +1045,26 @@ Traceback (most recent call last): SyntaxError: expected ':' + >>> def f[T]() + ... pass + Traceback (most recent call last): + SyntaxError: expected ':' + >>> class A ... pass Traceback (most recent call last): SyntaxError: expected ':' + >>> class A[T] + ... pass + Traceback (most recent call last): + SyntaxError: expected ':' + + >>> class A[T]() + ... pass + Traceback (most recent call last): + SyntaxError: expected ':' + >>> class R&D: ... pass Traceback (most recent call last): @@ -1154,6 +1224,22 @@ Traceback (most recent call last): SyntaxError: expected '(' + >>> def f -> int: + Traceback (most recent call last): + SyntaxError: expected '(' + + >>> async def f -> int: # type: int + Traceback (most recent call last): + SyntaxError: expected '(' + + >>> async def f[T]: + Traceback (most recent call last): + SyntaxError: expected '(' + + >>> def f[T] -> str: + Traceback (most recent call last): + SyntaxError: expected '(' + Parenthesized arguments in function definitions >>> def f(x, (y, z), w): @@ -1432,11 +1518,21 @@ Traceback (most recent call last): IndentationError: expected an indented block after function definition on line 1 + >>> def foo[T](x, /, y, *, z=2): + ... pass + Traceback (most recent call last): + IndentationError: expected an indented block after function definition on line 1 + >>> class Blech(A): ... pass Traceback (most recent call last): IndentationError: expected an indented block after class definition on line 1 + >>> class Blech[T](A): + ... pass + Traceback (most recent call last): + IndentationError: expected an indented block after class definition on line 1 + >>> match something: ... pass Traceback (most recent call last): @@ -1469,14 +1565,16 @@ Check that an multiple exception types with missing parentheses raise a custom exception - >>> try: + >>> # TODO: RUSTPYTHON + >>> try: # doctest: +SKIP ... pass ... except A, B: ... pass Traceback (most recent call last): SyntaxError: multiple exception types must be parenthesized - >>> try: + >>> # TODO: RUSTPYTHON + >>> try: # doctest: +SKIP ... pass ... except A, B, C: ... pass @@ -1591,30 +1689,113 @@ Traceback (most recent call last): SyntaxError: trailing comma not allowed without surrounding parentheses +>>> import a from b +Traceback (most recent call last): +SyntaxError: Did you mean to use 'from ... import ...' instead? + +>>> import a.y.z from b.y.z +Traceback (most recent call last): +SyntaxError: Did you mean to use 'from ... import ...' instead? + +>>> import a from b as bar +Traceback (most recent call last): +SyntaxError: Did you mean to use 'from ... import ...' instead? + +>>> import a.y.z from b.y.z as bar +Traceback (most recent call last): +SyntaxError: Did you mean to use 'from ... import ...' instead? + +>>> import a, b,c from b +Traceback (most recent call last): +SyntaxError: Did you mean to use 'from ... import ...' instead? + +>>> import a.y.z, b.y.z, c.y.z from b.y.z +Traceback (most recent call last): +SyntaxError: Did you mean to use 'from ... import ...' instead? + +>>> import a,b,c from b as bar +Traceback (most recent call last): +SyntaxError: Did you mean to use 'from ... import ...' instead? + +>>> import a.y.z, b.y.z, c.y.z from b.y.z as bar +Traceback (most recent call last): +SyntaxError: Did you mean to use 'from ... import ...' instead? + # Check that we dont raise the "trailing comma" error if there is more # input to the left of the valid part that we parsed. ->>> from t import x,y, and 3 +>>> from t import x,y, and 3 Traceback (most recent call last): SyntaxError: invalid syntax -# TODO: RUSTPYTHON nothing raised. ->>> (): int # doctest: +SKIP +>>> from i import +Traceback (most recent call last): +SyntaxError: Expected one or more names after 'import' + +>>> from .. import +Traceback (most recent call last): +SyntaxError: Expected one or more names after 'import' + +>>> import +Traceback (most recent call last): +SyntaxError: Expected one or more names after 'import' + +>>> (): int Traceback (most recent call last): SyntaxError: only single target (not tuple) can be annotated -# TODO: RUSTPYTHON nothing raised. ->>> []: int # doctest: +SKIP +>>> []: int Traceback (most recent call last): SyntaxError: only single target (not list) can be annotated -# TODO: RUSTPYTHON nothing raised. ->>> (()): int # doctest: +SKIP +>>> (()): int Traceback (most recent call last): SyntaxError: only single target (not tuple) can be annotated -# TODO: RUSTPYTHON nothing raised. ->>> ([]): int # doctest: +SKIP +>>> ([]): int Traceback (most recent call last): SyntaxError: only single target (not list) can be annotated +# 'not' after operators: + +>>> 3 + not 3 +Traceback (most recent call last): +SyntaxError: 'not' after an operator must be parenthesized + +>>> 3 * not 3 +Traceback (most recent call last): +SyntaxError: 'not' after an operator must be parenthesized + +>>> + not 3 +Traceback (most recent call last): +SyntaxError: 'not' after an operator must be parenthesized + +>>> - not 3 +Traceback (most recent call last): +SyntaxError: 'not' after an operator must be parenthesized + +>>> ~ not 3 +Traceback (most recent call last): +SyntaxError: 'not' after an operator must be parenthesized + +>>> 3 + - not 3 +Traceback (most recent call last): +SyntaxError: 'not' after an operator must be parenthesized + +>>> 3 + not -1 +Traceback (most recent call last): +SyntaxError: 'not' after an operator must be parenthesized + +# Check that we don't introduce misleading errors +>>> not 1 */ 2 +Traceback (most recent call last): +SyntaxError: invalid syntax + +>>> not 1 + +Traceback (most recent call last): +SyntaxError: invalid syntax + +>>> not + 1 + +Traceback (most recent call last): +SyntaxError: invalid syntax + Corner-cases that used to fail to raise the correct error: >>> def f(*, x=lambda __debug__:0): pass @@ -1635,8 +1816,7 @@ Corner-cases that used to crash: - # TODO: RUSTPYTHON nothing raised. - >>> def f(**__debug__): pass # doctest: +SKIP + >>> def f(**__debug__): pass Traceback (most recent call last): SyntaxError: cannot assign to __debug__ @@ -1650,8 +1830,8 @@ Invalid pattern matching constructs: - # TODO: RUSTPYTHON nothing raised. - >>> match ...: # doctest: +SKIP + >>> # TODO: RUSTPYTHON + >>> match ...: # doctest: +SKIP ... case 42 as _: ... ... Traceback (most recent call last): @@ -1751,22 +1931,22 @@ >>> A[*(1:2)] Traceback (most recent call last): ... - SyntaxError: invalid syntax + SyntaxError: Invalid star expression >>> A[*(1:2)] = 1 Traceback (most recent call last): ... - SyntaxError: invalid syntax + SyntaxError: Invalid star expression >>> del A[*(1:2)] Traceback (most recent call last): ... - SyntaxError: invalid syntax + SyntaxError: Invalid star expression A[*:] and A[:*] >>> A[*:] Traceback (most recent call last): ... - SyntaxError: invalid syntax + SyntaxError: Invalid star expression >>> A[:*] Traceback (most recent call last): ... @@ -1777,7 +1957,7 @@ >>> A[*] Traceback (most recent call last): ... - SyntaxError: invalid syntax + SyntaxError: Invalid star expression A[**] @@ -1829,10 +2009,246 @@ def f(x: *b) Traceback (most recent call last): ... SyntaxError: invalid syntax + +Invalid bytes literals: + + >>> b"Ā" + Traceback (most recent call last): + ... + b"Ā" + ^^^ + SyntaxError: bytes can only contain ASCII literal characters + + >>> b"абвгде" + Traceback (most recent call last): + ... + b"абвгде" + ^^^^^^^^ + SyntaxError: bytes can only contain ASCII literal characters + + >>> b"abc ъющый" # first 3 letters are ascii + Traceback (most recent call last): + ... + b"abc ъющый" + ^^^^^^^^^^^ + SyntaxError: bytes can only contain ASCII literal characters + +Invalid expressions in type scopes: + + >>> type A[] = int + Traceback (most recent call last): + ... + SyntaxError: Type parameter list cannot be empty + + >>> class A[]: ... + Traceback (most recent call last): + ... + SyntaxError: Type parameter list cannot be empty + + >>> def some[](): ... + Traceback (most recent call last): + ... + SyntaxError: Type parameter list cannot be empty + + >>> def some[]() + Traceback (most recent call last): + ... + SyntaxError: Type parameter list cannot be empty + + >>> async def some[]: # type: int + Traceback (most recent call last): + ... + SyntaxError: Type parameter list cannot be empty + + >>> def f[T: (x:=3)](): pass + Traceback (most recent call last): + ... + SyntaxError: named expression cannot be used within a TypeVar bound + + >>> def f[T: ((x:= 3), int)](): pass + Traceback (most recent call last): + ... + SyntaxError: named expression cannot be used within a TypeVar constraint + + >>> def f[T = ((x:=3))](): pass + Traceback (most recent call last): + ... + SyntaxError: named expression cannot be used within a TypeVar default + + >>> async def f[T: (x:=3)](): pass + Traceback (most recent call last): + ... + SyntaxError: named expression cannot be used within a TypeVar bound + + >>> async def f[T: ((x:= 3), int)](): pass + Traceback (most recent call last): + ... + SyntaxError: named expression cannot be used within a TypeVar constraint + + >>> async def f[T = ((x:=3))](): pass + Traceback (most recent call last): + ... + SyntaxError: named expression cannot be used within a TypeVar default + + >>> type A[T: (x:=3)] = int + Traceback (most recent call last): + ... + SyntaxError: named expression cannot be used within a TypeVar bound + + >>> type A[T: ((x:= 3), int)] = int + Traceback (most recent call last): + ... + SyntaxError: named expression cannot be used within a TypeVar constraint + + >>> type A[T = ((x:=3))] = int + Traceback (most recent call last): + ... + SyntaxError: named expression cannot be used within a TypeVar default + + >>> def f[T: (yield)](): pass + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a TypeVar bound + + >>> def f[T: (int, (yield))](): pass + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a TypeVar constraint + + >>> def f[T = (yield)](): pass + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a TypeVar default + + >>> def f[*Ts = (yield)](): pass + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a TypeVarTuple default + + >>> def f[**P = [(yield), int]](): pass + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a ParamSpec default + + >>> type A[T: (yield 3)] = int + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a TypeVar bound + + >>> type A[T: (int, (yield 3))] = int + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a TypeVar constraint + + >>> type A[T = (yield 3)] = int + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a TypeVar default + + >>> type A[T: (await 3)] = int + Traceback (most recent call last): + ... + SyntaxError: await expression cannot be used within a TypeVar bound + + >>> type A[T: (yield from [])] = int + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a TypeVar bound + + >>> class A[T: (yield 3)]: pass + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a TypeVar bound + + >>> class A[T: (int, (yield 3))]: pass + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a TypeVar constraint + + >>> class A[T = (yield)]: pass + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a TypeVar default + + >>> class A[*Ts = (yield)]: pass + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a TypeVarTuple default + + >>> class A[**P = [(yield), int]]: pass + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a ParamSpec default + + >>> # TODO: RUSTPYTHON + >>> type A = (x := 3) # doctest: +SKIP + Traceback (most recent call last): + ... + SyntaxError: named expression cannot be used within a type alias + + >>> type A = (yield 3) + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a type alias + + >>> type A = (await 3) + Traceback (most recent call last): + ... + SyntaxError: await expression cannot be used within a type alias + + >>> type A = (yield from []) + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within a type alias + + >>> class A[T]((x := 3)): ... + Traceback (most recent call last): + ... + SyntaxError: named expression cannot be used within the definition of a generic + + >>> class A[T]((yield 3)): ... + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within the definition of a generic + + >>> class A[T]((await 3)): ... + Traceback (most recent call last): + ... + SyntaxError: await expression cannot be used within the definition of a generic + + >>> class A[T]((yield from [])): ... + Traceback (most recent call last): + ... + SyntaxError: yield expression cannot be used within the definition of a generic + + >>> f(**x, *y) + Traceback (most recent call last): + SyntaxError: iterable argument unpacking follows keyword argument unpacking + + >>> f(**x, *) + Traceback (most recent call last): + SyntaxError: Invalid star expression + + >>> f(x, *:) + Traceback (most recent call last): + SyntaxError: Invalid star expression + + >>> f(x, *) + Traceback (most recent call last): + SyntaxError: Invalid star expression + + >>> f(x = 5, *) + Traceback (most recent call last): + SyntaxError: Invalid star expression + + >>> f(x = 5, *:) + Traceback (most recent call last): + SyntaxError: Invalid star expression """ import re import doctest +import textwrap import unittest from test import support @@ -1869,8 +2285,7 @@ def _check_error(self, code, errtext, else: self.fail("compile() did not raise SyntaxError") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_expression_with_assignment(self): self._check_error( "print(end1 + end2 = ' ')", @@ -1878,16 +2293,14 @@ def test_expression_with_assignment(self): offset=7 ) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_curly_brace_after_primary_raises_immediately(self): self._check_error("f{}", "invalid syntax", mode="single") def test_assign_call(self): self._check_error("f() = 1", "assign") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_assign_del(self): self._check_error("del (,)", "invalid syntax") self._check_error("del 1", "cannot delete literal") @@ -1939,9 +2352,6 @@ def error2(): """ self._check_error(source, "parameter and nonlocal", lineno=3) - def test_break_outside_loop(self): - self._check_error("break", "outside loop") - def test_yield_outside_function(self): self._check_error("if 0: yield", "outside function") self._check_error("if 0: yield\nelse: x=1", "outside function") @@ -1970,22 +2380,28 @@ def test_return_outside_function(self): "outside function") def test_break_outside_loop(self): - self._check_error("if 0: break", "outside loop") - self._check_error("if 0: break\nelse: x=1", "outside loop") - self._check_error("if 1: pass\nelse: break", "outside loop") - self._check_error("class C:\n if 0: break", "outside loop") + msg = "outside loop" + self._check_error("break", msg, lineno=1) + self._check_error("if 0: break", msg, lineno=1) + self._check_error("if 0: break\nelse: x=1", msg, lineno=1) + self._check_error("if 1: pass\nelse: break", msg, lineno=2) + self._check_error("class C:\n if 0: break", msg, lineno=2) self._check_error("class C:\n if 1: pass\n else: break", - "outside loop") + msg, lineno=3) + self._check_error("with object() as obj:\n break", + msg, lineno=2) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_continue_outside_loop(self): - self._check_error("if 0: continue", "not properly in loop") - self._check_error("if 0: continue\nelse: x=1", "not properly in loop") - self._check_error("if 1: pass\nelse: continue", "not properly in loop") - self._check_error("class C:\n if 0: continue", "not properly in loop") + msg = "not properly in loop" + self._check_error("if 0: continue", msg, lineno=1) + self._check_error("if 0: continue\nelse: x=1", msg, lineno=1) + self._check_error("if 1: pass\nelse: continue", msg, lineno=2) + self._check_error("class C:\n if 0: continue", msg, lineno=2) self._check_error("class C:\n if 1: pass\n else: continue", - "not properly in loop") + msg, lineno=3) + self._check_error("with object() as obj:\n continue", + msg, lineno=2) def test_unexpected_indent(self): self._check_error("foo()\n bar()\n", "unexpected indent", @@ -2000,35 +2416,42 @@ def test_bad_outdent(self): "unindent does not match .* level", subclass=IndentationError) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_kwargs_last(self): self._check_error("int(base=10, '2')", "positional argument follows keyword argument") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_kwargs_last2(self): self._check_error("int(**{'base': 10}, '2')", "positional argument follows " "keyword argument unpacking") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_kwargs_last3(self): self._check_error("int(**{'base': 10}, *['2'])", "iterable argument unpacking follows " "keyword argument unpacking") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_generator_in_function_call(self): self._check_error("foo(x, y for y in range(3) for z in range(2) if z , p)", "Generator expression must be parenthesized", lineno=1, end_lineno=1, offset=11, end_offset=53) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_except_then_except_star(self): + self._check_error("try: pass\nexcept ValueError: pass\nexcept* TypeError: pass", + r"cannot have both 'except' and 'except\*' on the same 'try'", + lineno=3, end_lineno=3, offset=1, end_offset=8) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_except_star_then_except(self): + self._check_error("try: pass\nexcept* ValueError: pass\nexcept TypeError: pass", + r"cannot have both 'except' and 'except\*' on the same 'try'", + lineno=3, end_lineno=3, offset=1, end_offset=7) + + @unittest.expectedFailure # TODO: RUSTPYTHON def test_empty_line_after_linecont(self): # See issue-40847 s = r"""\ @@ -2073,6 +2496,25 @@ def test_continuation_bad_indentation(self): self.assertRaises(IndentationError, exec, code) + @support.cpython_only + def test_disallowed_type_param_names(self): + # See gh-128632 + + self._check_error(f"class A[__classdict__]: pass", + f"reserved name '__classdict__' cannot be used for type parameter") + self._check_error(f"def f[__classdict__](): pass", + f"reserved name '__classdict__' cannot be used for type parameter") + self._check_error(f"type T[__classdict__] = tuple[__classdict__]", + f"reserved name '__classdict__' cannot be used for type parameter") + + # These compilations are here to make sure __class__, __classcell__ and __classdictcell__ + # don't break in the future like __classdict__ did in this case. + for name in ('__class__', '__classcell__', '__classdictcell__'): + compile(f""" +class A: + class B[{name}]: pass + """, "<testcase>", mode="exec") + @support.cpython_only def test_nested_named_except_blocks(self): code = "" @@ -2083,6 +2525,58 @@ def test_nested_named_except_blocks(self): code += f"{' '*4*12}pass" self._check_error(code, "too many statically nested blocks") + @support.cpython_only + def test_with_statement_many_context_managers(self): + # See gh-113297 + + def get_code(n): + code = textwrap.dedent(""" + def bug(): + with ( + a + """) + for i in range(n): + code += f" as a{i}, a\n" + code += "): yield a" + return code + + CO_MAXBLOCKS = 21 # static nesting limit of the compiler + MAX_MANAGERS = CO_MAXBLOCKS - 1 # One for the StopIteration block + + for n in range(MAX_MANAGERS): + with self.subTest(f"within range: {n=}"): + compile(get_code(n), "<string>", "exec") + + for n in range(MAX_MANAGERS, MAX_MANAGERS + 5): + with self.subTest(f"out of range: {n=}"): + self._check_error(get_code(n), "too many statically nested blocks") + + @support.cpython_only + def test_async_with_statement_many_context_managers(self): + # See gh-116767 + + def get_code(n): + code = [ textwrap.dedent(""" + async def bug(): + async with ( + a + """) ] + for i in range(n): + code.append(f" as a{i}, a\n") + code.append("): yield a") + return "".join(code) + + CO_MAXBLOCKS = 21 # static nesting limit of the compiler + MAX_MANAGERS = CO_MAXBLOCKS - 1 # One for the StopIteration block + + for n in range(MAX_MANAGERS): + with self.subTest(f"within range: {n=}"): + compile(get_code(n), "<string>", "exec") + + for n in range(MAX_MANAGERS, MAX_MANAGERS + 5): + with self.subTest(f"out of range: {n=}"): + self._check_error(get_code(n), "too many statically nested blocks") + def test_barry_as_flufl_with_syntax_errors(self): # The "barry_as_flufl" rule can produce some "bugs-at-a-distance" if # is reading the wrong token in the presence of syntax errors later @@ -2100,8 +2594,7 @@ def func2(): """ self._check_error(code, "expected ':'") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_invalid_line_continuation_error_position(self): self._check_error(r"a = 3 \ 4", "unexpected character after line continuation character", @@ -2113,8 +2606,7 @@ def test_invalid_line_continuation_error_position(self): "unexpected character after line continuation character", lineno=3, offset=4) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_invalid_line_continuation_left_recursive(self): # Check bpo-42218: SyntaxErrors following left-recursive rules # (t_primary_raw in this case) need to be tested explicitly @@ -2123,8 +2615,7 @@ def test_invalid_line_continuation_left_recursive(self): self._check_error("A.\u03bc\\\n", "unexpected EOF while parsing") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_error_parenthesis(self): for paren in "([{": self._check_error(paren + "1 + 2", f"\\{paren}' was never closed") @@ -2135,10 +2626,39 @@ def test_error_parenthesis(self): for paren in ")]}": self._check_error(paren + "1 + 2", f"unmatched '\\{paren}'") - # TODO: RUSTPYTHON - @unittest.expectedFailure + # Some more complex examples: + code = """\ +func( + a=["unclosed], # Need a quote in this comment: " + b=2, +) +""" + self._check_error(code, "parenthesis '\\)' does not match opening parenthesis '\\['") + + self._check_error("match y:\n case e(e=v,v,", " was never closed") + + # Examples with dencodings + s = b'# coding=latin\n(aaaaaaaaaaaaaaaaa\naaaaaaaaaaa\xb5' + self._check_error(s, r"'\(' was never closed") + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_error_string_literal(self): + + self._check_error("'blech", r"unterminated string literal \(.*\)$") + self._check_error('"blech', r"unterminated string literal \(.*\)$") + self._check_error( + r'"blech\"', r"unterminated string literal \(.*\); perhaps you escaped the end quote" + ) + self._check_error( + r'r"blech\"', r"unterminated string literal \(.*\); perhaps you escaped the end quote" + ) + self._check_error("'''blech", "unterminated triple-quoted string literal") + self._check_error('"""blech', "unterminated triple-quoted string literal") + + @unittest.expectedFailure # TODO: RUSTPYTHON def test_invisible_characters(self): self._check_error('print\x17("Hello")', "invalid non-printable character") + self._check_error(b"with(0,,):\n\x01", "invalid non-printable character") def test_match_call_does_not_raise_syntax_error(self): code = """ @@ -2158,8 +2678,7 @@ def case(x): """ compile(code, "<string>", "exec") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_multiline_compiler_error_points_to_the_end(self): self._check_error( "call(\na=1,\na=1\n)", @@ -2198,7 +2717,8 @@ def test_syntax_error_on_deeply_nested_blocks(self): while 20: while 21: while 22: - break + while 23: + break """ self._check_error(source, "too many statically nested blocks") @@ -2207,7 +2727,7 @@ def test_error_on_parser_stack_overflow(self): source = "-" * 100000 + "4" for mode in ["exec", "eval", "single"]: with self.subTest(mode=mode): - with self.assertRaises(MemoryError): + with self.assertRaisesRegex(MemoryError, r"too complex"): compile(source, "<string>", mode) @support.cpython_only diff --git a/Lib/test/test_sysconfig.py b/Lib/test/test_sysconfig.py index 6db1442980..82c11bdf7e 100644 --- a/Lib/test/test_sysconfig.py +++ b/Lib/test/test_sysconfig.py @@ -1,24 +1,35 @@ +import platform +import re import unittest import sys import os import subprocess import shutil +import json +import textwrap from copy import copy from test.support import ( - captured_stdout, PythonSymlink, requires_subprocess, is_wasi + captured_stdout, + is_apple_mobile, + is_wasi, + PythonSymlink, + requires_subprocess, ) from test.support.import_helper import import_module from test.support.os_helper import (TESTFN, unlink, skip_unless_symlink, change_cwd) -from test.support.warnings_helper import check_warnings +from test.support.venv import VirtualEnvironment import sysconfig from sysconfig import (get_paths, get_platform, get_config_vars, get_path, get_path_names, _INSTALL_SCHEMES, get_default_scheme, get_scheme_names, get_config_var, - _expand_vars, _get_preferred_schemes, _main) + _expand_vars, _get_preferred_schemes) +from sysconfig.__main__ import _main, _parse_makefile +import _imp import _osx_support +import _sysconfig HAS_USER_BASE = sysconfig._HAS_USER_BASE @@ -41,6 +52,7 @@ def setUp(self): self.name = os.name self.platform = sys.platform self.version = sys.version + self._framework = sys._framework self.sep = os.sep self.join = os.path.join self.isabs = os.path.isabs @@ -64,6 +76,7 @@ def tearDown(self): os.name = self.name sys.platform = self.platform sys.version = self.version + sys._framework = self._framework os.sep = self.sep os.path.join = self.join os.path.isabs = self.isabs @@ -91,6 +104,12 @@ def _cleanup_testfn(self): elif os.path.isdir(path): shutil.rmtree(path) + def venv(self, **venv_create_args): + return VirtualEnvironment.from_tmpdir( + prefix=f'{self.id()}-venv-', + **venv_create_args, + ) + def test_get_path_names(self): self.assertEqual(get_path_names(), sysconfig._SCHEME_KEYS) @@ -137,35 +156,37 @@ def test_get_preferred_schemes(self): # Mac, framework build. os.name = 'posix' sys.platform = 'darwin' - sys._framework = True + sys._framework = "MyPython" self.assertIsInstance(schemes, dict) self.assertEqual(set(schemes), expected_schemes) - # NOTE: RUSTPYTHON this is hardcoded to 'python', we're set up for failure. - @unittest.expectedFailure def test_posix_venv_scheme(self): # The following directories were hardcoded in the venv module # before bpo-45413, here we assert the posix_venv scheme does not regress binpath = 'bin' incpath = 'include' libpath = os.path.join('lib', - 'python%d.%d' % sys.version_info[:2], + # XXX: RUSTPYTHON + f'rustpython{sysconfig._get_python_version_abi()}', 'site-packages') - # Resolve the paths in prefix - binpath = os.path.join(sys.prefix, binpath) - incpath = os.path.join(sys.prefix, incpath) - libpath = os.path.join(sys.prefix, libpath) + # Resolve the paths in an imaginary venv/ directory + binpath = os.path.join('venv', binpath) + incpath = os.path.join('venv', incpath) + libpath = os.path.join('venv', libpath) + + # Mimic the venv module, set all bases to the venv directory + bases = ('base', 'platbase', 'installed_base', 'installed_platbase') + vars = {base: 'venv' for base in bases} - self.assertEqual(binpath, sysconfig.get_path('scripts', scheme='posix_venv')) - self.assertEqual(libpath, sysconfig.get_path('purelib', scheme='posix_venv')) + self.assertEqual(binpath, sysconfig.get_path('scripts', scheme='posix_venv', vars=vars)) + self.assertEqual(libpath, sysconfig.get_path('purelib', scheme='posix_venv', vars=vars)) # The include directory on POSIX isn't exactly the same as before, # but it is "within" - sysconfig_includedir = sysconfig.get_path('include', scheme='posix_venv') + sysconfig_includedir = sysconfig.get_path('include', scheme='posix_venv', vars=vars) self.assertTrue(sysconfig_includedir.startswith(incpath + os.sep)) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_nt_venv_scheme(self): # The following directories were hardcoded in the venv module # before bpo-45413, here we assert the posix_venv scheme does not regress @@ -173,14 +194,19 @@ def test_nt_venv_scheme(self): incpath = 'Include' libpath = os.path.join('Lib', 'site-packages') - # Resolve the paths in prefix - binpath = os.path.join(sys.prefix, binpath) - incpath = os.path.join(sys.prefix, incpath) - libpath = os.path.join(sys.prefix, libpath) + # Resolve the paths in an imaginary venv\ directory + venv = 'venv' + binpath = os.path.join(venv, binpath) + incpath = os.path.join(venv, incpath) + libpath = os.path.join(venv, libpath) + + # Mimic the venv module, set all bases to the venv directory + bases = ('base', 'platbase', 'installed_base', 'installed_platbase') + vars = {base: 'venv' for base in bases} - self.assertEqual(binpath, sysconfig.get_path('scripts', scheme='nt_venv')) - self.assertEqual(incpath, sysconfig.get_path('include', scheme='nt_venv')) - self.assertEqual(libpath, sysconfig.get_path('purelib', scheme='nt_venv')) + self.assertEqual(binpath, sysconfig.get_path('scripts', scheme='nt_venv', vars=vars)) + self.assertEqual(incpath, sysconfig.get_path('include', scheme='nt_venv', vars=vars)) + self.assertEqual(libpath, sysconfig.get_path('purelib', scheme='nt_venv', vars=vars)) def test_venv_scheme(self): if sys.platform == 'win32': @@ -216,6 +242,11 @@ def test_get_config_vars(self): self.assertTrue(cvars) def test_get_platform(self): + # Check the actual platform returns something reasonable. + actual_platform = get_platform() + self.assertIsInstance(actual_platform, str) + self.assertTrue(actual_platform) + # windows XP, 32bits os.name = 'nt' sys.version = ('2.4.4 (#71, Oct 18 2006, 08:34:43) ' @@ -321,6 +352,13 @@ def test_get_platform(self): self.assertEqual(get_platform(), 'macosx-10.4-%s' % arch) + for macver in range(11, 16): + _osx_support._remove_original_values(get_config_vars()) + get_config_vars()['CFLAGS'] = ('-fno-strict-overflow -Wsign-compare -Wunreachable-code' + '-arch arm64 -fno-common -dynamic -DNDEBUG -g -O3 -Wall') + get_config_vars()['MACOSX_DEPLOYMENT_TARGET'] = f"{macver}.0" + self.assertEqual(get_platform(), 'macosx-%d.0-arm64' % macver) + # linux debian sarge os.name = 'posix' sys.version = ('2.3.5 (#1, Jul 4 2007, 17:28:59) ' @@ -331,11 +369,27 @@ def test_get_platform(self): self.assertEqual(get_platform(), 'linux-i686') + # Android + os.name = 'posix' + sys.platform = 'android' + get_config_vars()['ANDROID_API_LEVEL'] = 9 + for machine, abi in { + 'x86_64': 'x86_64', + 'i686': 'x86', + 'aarch64': 'arm64_v8a', + 'armv7l': 'armeabi_v7a', + }.items(): + with self.subTest(machine): + self._set_uname(('Linux', 'localhost', '3.18.91+', + '#1 Tue Jan 9 20:35:43 UTC 2018', machine)) + self.assertEqual(get_platform(), f'android-9-{abi}') + # XXX more platforms to tests here - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON @unittest.skipIf(is_wasi, "Incompatible with WASI mapdir and OOT builds") + @unittest.skipIf(is_apple_mobile, + f"{sys.platform} doesn't distribute header files in the runtime environment") def test_get_config_h_filename(self): config_h = sysconfig.get_config_h_filename() self.assertTrue(os.path.isfile(config_h), config_h) @@ -381,8 +435,8 @@ def test_user_similar(self): if name == 'platlib': # Replace "/lib64/python3.11/site-packages" suffix # with "/lib/python3.11/site-packages". - py_version_short = sysconfig.get_python_version() - suffix = f'python{py_version_short}/site-packages' + py_version_abi = sysconfig._get_python_version_abi() + suffix = f'python{py_version_abi}/site-packages' expected = expected.replace(f'/{sys.platlibdir}/{suffix}', f'/lib/{suffix}') self.assertEqual(user_path, expected) @@ -393,8 +447,6 @@ def test_main(self): _main() self.assertTrue(len(output.getvalue().split('\n')) > 0) - # TODO: RUSTPYTHON - @unittest.expectedFailure @unittest.skipIf(sys.platform == "win32", "Does not apply to Windows") def test_ldshared_value(self): ldflags = sysconfig.get_config_var('LDFLAGS') @@ -402,6 +454,31 @@ def test_ldshared_value(self): self.assertIn(ldflags, ldshared) + @unittest.skipIf(not _imp.extension_suffixes(), "stub loader has no suffixes") + def test_soabi(self): + soabi = sysconfig.get_config_var('SOABI') + self.assertIn(soabi, _imp.extension_suffixes()[0]) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_library(self): + library = sysconfig.get_config_var('LIBRARY') + ldlibrary = sysconfig.get_config_var('LDLIBRARY') + major, minor = sys.version_info[:2] + if sys.platform == 'win32': + self.assertTrue(library.startswith(f'python{major}{minor}')) + self.assertTrue(library.endswith('.dll')) + self.assertEqual(library, ldlibrary) + elif is_apple_mobile: + framework = sysconfig.get_config_var('PYTHONFRAMEWORK') + self.assertEqual(ldlibrary, f"{framework}.framework/{framework}") + else: + self.assertTrue(library.startswith(f'libpython{major}.{minor}')) + self.assertTrue(library.endswith('.a')) + if sys.platform == 'darwin' and sys._framework: + self.skipTest('gh-110824: skip LDLIBRARY test for framework build') + else: + self.assertTrue(ldlibrary.startswith(f'libpython{major}.{minor}')) + @unittest.skipUnless(sys.platform == "darwin", "test only relevant on MacOSX") @requires_subprocess() def test_platform_in_subprocess(self): @@ -446,8 +523,10 @@ def test_platform_in_subprocess(self): self.assertEqual(status, 0) self.assertEqual(my_platform, test_platform) - @unittest.expectedFailureIf(sys.platform != "win32", "TODO: RUSTPYTHON") + @unittest.expectedFailureIf(sys.platform != 'win32', 'TODO: RUSTPYTHON') @unittest.skipIf(is_wasi, "Incompatible with WASI mapdir and OOT builds") + @unittest.skipIf(is_apple_mobile, + f"{sys.platform} doesn't include config folder at runtime") def test_srcdir(self): # See Issues #15322, #15364. srcdir = sysconfig.get_config_var('srcdir') @@ -460,11 +539,15 @@ def test_srcdir(self): # should be a full source checkout. Python_h = os.path.join(srcdir, 'Include', 'Python.h') self.assertTrue(os.path.exists(Python_h), Python_h) - # <srcdir>/PC/pyconfig.h always exists even if unused on POSIX. - pyconfig_h = os.path.join(srcdir, 'PC', 'pyconfig.h') + # <srcdir>/PC/pyconfig.h.in always exists even if unused + pyconfig_h = os.path.join(srcdir, 'PC', 'pyconfig.h.in') self.assertTrue(os.path.exists(pyconfig_h), pyconfig_h) pyconfig_h_in = os.path.join(srcdir, 'pyconfig.h.in') self.assertTrue(os.path.exists(pyconfig_h_in), pyconfig_h_in) + if os.name == 'nt': + # <executable dir>/pyconfig.h exists on Windows in a build tree + pyconfig_h = os.path.join(sys.executable, '..', 'pyconfig.h') + self.assertTrue(os.path.exists(pyconfig_h), pyconfig_h) elif os.name == 'posix': makefile_dir = os.path.dirname(sysconfig.get_makefile_filename()) # Issue #19340: srcdir has been realpath'ed already @@ -479,23 +562,16 @@ def test_srcdir_independent_of_cwd(self): srcdir2 = sysconfig.get_config_var('srcdir') self.assertEqual(srcdir, srcdir2) - # TODO: RUSTPYTHON - @unittest.expectedFailure @unittest.skipIf(sysconfig.get_config_var('EXT_SUFFIX') is None, 'EXT_SUFFIX required for this test') + @unittest.skipIf(not _imp.extension_suffixes(), "stub loader has no suffixes") def test_EXT_SUFFIX_in_vars(self): - import _imp - if not _imp.extension_suffixes(): - self.skipTest("stub loader has no suffixes") vars = sysconfig.get_config_vars() self.assertEqual(vars['EXT_SUFFIX'], _imp.extension_suffixes()[0]) - @unittest.skipUnless(sys.platform == 'linux' and - hasattr(sys.implementation, '_multiarch'), - 'multiarch-specific test') - def test_triplet_in_ext_suffix(self): + @unittest.skipUnless(sys.platform == 'linux', 'Linux-specific test') + def test_linux_ext_suffix(self): ctypes = import_module('ctypes') - import platform, re machine = platform.machine() suffix = sysconfig.get_config_var('EXT_SUFFIX') if re.match('(aarch64|arm|mips|ppc|powerpc|s390|sparc)', machine): @@ -508,20 +584,102 @@ def test_triplet_in_ext_suffix(self): self.assertTrue(suffix.endswith(expected_suffixes), f'unexpected suffix {suffix!r}') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.skipUnless(sys.platform == 'android', 'Android-specific test') + def test_android_ext_suffix(self): + machine = platform.machine() + suffix = sysconfig.get_config_var('EXT_SUFFIX') + expected_triplet = { + "x86_64": "x86_64-linux-android", + "i686": "i686-linux-android", + "aarch64": "aarch64-linux-android", + "armv7l": "arm-linux-androideabi", + }[machine] + self.assertTrue(suffix.endswith(f"-{expected_triplet}.so"), + f"{machine=}, {suffix=}") + @unittest.skipUnless(sys.platform == 'darwin', 'OS X-specific test') def test_osx_ext_suffix(self): suffix = sysconfig.get_config_var('EXT_SUFFIX') self.assertTrue(suffix.endswith('-darwin.so'), suffix) + @unittest.expectedFailure # TODO: RUSTPYTHON + @requires_subprocess() + def test_config_vars_depend_on_site_initialization(self): + script = textwrap.dedent(""" + import sysconfig + + config_vars = sysconfig.get_config_vars() + + import json + print(json.dumps(config_vars, indent=2)) + """) + + with self.venv() as venv: + site_config_vars = json.loads(venv.run('-c', script).stdout) + no_site_config_vars = json.loads(venv.run('-S', '-c', script).stdout) + + self.assertNotEqual(site_config_vars, no_site_config_vars) + # With the site initialization, the virtual environment should be enabled. + self.assertEqual(site_config_vars['base'], venv.prefix) + self.assertEqual(site_config_vars['platbase'], venv.prefix) + #self.assertEqual(site_config_vars['prefix'], venv.prefix) # # FIXME: prefix gets overwriten by _init_posix + # Without the site initialization, the virtual environment should be disabled. + self.assertEqual(no_site_config_vars['base'], site_config_vars['installed_base']) + self.assertEqual(no_site_config_vars['platbase'], site_config_vars['installed_platbase']) + + @unittest.expectedFailure # TODO: RUSTPYTHON + @requires_subprocess() + def test_config_vars_recalculation_after_site_initialization(self): + script = textwrap.dedent(""" + import sysconfig + + before = sysconfig.get_config_vars() + + import site + site.main() + + after = sysconfig.get_config_vars() + + import json + print(json.dumps({'before': before, 'after': after}, indent=2)) + """) + + with self.venv() as venv: + config_vars = json.loads(venv.run('-S', '-c', script).stdout) + + self.assertNotEqual(config_vars['before'], config_vars['after']) + self.assertEqual(config_vars['after']['base'], venv.prefix) + #self.assertEqual(config_vars['after']['prefix'], venv.prefix) # FIXME: prefix gets overwriten by _init_posix + #self.assertEqual(config_vars['after']['exec_prefix'], venv.prefix) # FIXME: exec_prefix gets overwriten by _init_posix + + @unittest.expectedFailure # TODO: RUSTPYTHON + @requires_subprocess() + def test_paths_depend_on_site_initialization(self): + script = textwrap.dedent(""" + import sysconfig + + paths = sysconfig.get_paths() + + import json + print(json.dumps(paths, indent=2)) + """) + + with self.venv() as venv: + site_paths = json.loads(venv.run('-c', script).stdout) + no_site_paths = json.loads(venv.run('-S', '-c', script).stdout) + + self.assertNotEqual(site_paths, no_site_paths) + + class MakefileTests(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON @unittest.skipIf(sys.platform.startswith('win'), 'Test is not Windows compatible') @unittest.skipIf(is_wasi, "Incompatible with WASI mapdir and OOT builds") + @unittest.skipIf(is_apple_mobile, + f"{sys.platform} doesn't include config folder at runtime") def test_get_makefile_filename(self): makefile = sysconfig.get_makefile_filename() self.assertTrue(os.path.isfile(makefile), makefile) @@ -536,7 +694,7 @@ def test_parse_makefile(self): print("var5=dollar$$5", file=makefile) print("var6=${var3}/lib/python3.5/config-$(VAR2)$(var5)" "-x86_64-linux-gnu", file=makefile) - vars = sysconfig._parse_makefile(TESTFN) + vars = _parse_makefile(TESTFN) self.assertEqual(vars, { 'var1': 'ab42', 'VAR2': 'b42', @@ -547,5 +705,26 @@ def test_parse_makefile(self): }) +class DeprecationTests(unittest.TestCase): + def deprecated(self, removal_version, deprecation_msg=None, error=Exception, error_msg=None): + if sys.version_info >= removal_version: + return self.assertRaises(error, msg=error_msg) + else: + return self.assertWarns(DeprecationWarning, msg=deprecation_msg) + + def test_is_python_build_check_home(self): + with self.deprecated( + removal_version=(3, 15), + deprecation_msg=( + 'The check_home argument of sysconfig.is_python_build is ' + 'deprecated and its value is ignored. ' + 'It will be removed in Python 3.15.' + ), + error=TypeError, + error_msg="is_python_build() takes 0 positional arguments but 1 were given", + ): + sysconfig.is_python_build('foo') + + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_tabnanny.py b/Lib/test/test_tabnanny.py index 74d3a71a35..aa71166a38 100644 --- a/Lib/test/test_tabnanny.py +++ b/Lib/test/test_tabnanny.py @@ -14,7 +14,7 @@ findfile) from test.support.os_helper import unlink -import unittest # XXX: RUSTPYTHON +import unittest # TODO: RUSTPYTHON SOURCE_CODES = { @@ -112,9 +112,10 @@ def test_errprint(self): for args, expected in tests: with self.subTest(arguments=args, expected=expected): - with captured_stderr() as stderr: - tabnanny.errprint(*args) - self.assertEqual(stderr.getvalue() , expected) + with self.assertRaises(SystemExit): + with captured_stderr() as stderr: + tabnanny.errprint(*args) + self.assertEqual(stderr.getvalue() , expected) class TestNannyNag(TestCase): @@ -199,23 +200,25 @@ def test_correct_directory(self): with TemporaryPyFile(SOURCE_CODES["error_free"], directory=tmp_dir): self.verify_tabnanny_check(tmp_dir) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_when_wrong_indented(self): """A python source code file eligible for raising `IndentationError`.""" with TemporaryPyFile(SOURCE_CODES["wrong_indented"]) as file_path: err = ('unindent does not match any outer indentation level' ' (<tokenize>, line 3)\n') err = f"{file_path!r}: Indentation Error: {err}" - self.verify_tabnanny_check(file_path, err=err) + with self.assertRaises(SystemExit): + self.verify_tabnanny_check(file_path, err=err) def test_when_tokenize_tokenerror(self): """A python source code file eligible for raising 'tokenize.TokenError'.""" with TemporaryPyFile(SOURCE_CODES["incomplete_expression"]) as file_path: err = "('EOF in multi-line statement', (7, 0))\n" err = f"{file_path!r}: Token Error: {err}" - self.verify_tabnanny_check(file_path, err=err) + with self.assertRaises(SystemExit): + self.verify_tabnanny_check(file_path, err=err) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_when_nannynag_error_verbose(self): """A python source code file eligible for raising `tabnanny.NannyNag`. @@ -223,29 +226,28 @@ def test_when_nannynag_error_verbose(self): """ with TemporaryPyFile(SOURCE_CODES["nannynag_errored"]) as file_path: out = f"{file_path!r}: *** Line 3: trouble in tab city! ***\n" - out += "offending line: '\\tprint(\"world\")\\n'\n" - out += "indent not equal e.g. at tab size 1\n" + out += "offending line: '\\tprint(\"world\")'\n" + out += "inconsistent use of tabs and spaces in indentation\n" tabnanny.verbose = 1 self.verify_tabnanny_check(file_path, out=out) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_when_nannynag_error(self): """A python source code file eligible for raising `tabnanny.NannyNag`.""" with TemporaryPyFile(SOURCE_CODES["nannynag_errored"]) as file_path: - out = f"{file_path} 3 '\\tprint(\"world\")\\n'\n" + out = f"{file_path} 3 '\\tprint(\"world\")'\n" self.verify_tabnanny_check(file_path, out=out) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_when_no_file(self): """A python file which does not exist actually in system.""" path = 'no_file.py' err = (f"{path!r}: I/O Error: [Errno {errno.ENOENT}] " f"{os.strerror(errno.ENOENT)}: {path!r}\n") - self.verify_tabnanny_check(path, err=err) + with self.assertRaises(SystemExit): + self.verify_tabnanny_check(path, err=err) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_errored_directory(self): """Directory containing wrongly indented python source code files.""" with tempfile.TemporaryDirectory() as tmp_dir: @@ -259,7 +261,8 @@ def test_errored_directory(self): err = ('unindent does not match any outer indentation level' ' (<tokenize>, line 3)\n') err = f"{e_file!r}: Indentation Error: {err}" - self.verify_tabnanny_check(tmp_dir, err=err) + with self.assertRaises(SystemExit): + self.verify_tabnanny_check(tmp_dir, err=err) class TestProcessTokens(TestCase): @@ -295,9 +298,12 @@ def test_with_errored_codes_samples(self): class TestCommandLine(TestCase): """Tests command line interface of `tabnanny`.""" - def validate_cmd(self, *args, stdout="", stderr="", partial=False): + def validate_cmd(self, *args, stdout="", stderr="", partial=False, expect_failure=False): """Common function to assert the behaviour of command line interface.""" - _, out, err = script_helper.assert_python_ok('-m', 'tabnanny', *args) + if expect_failure: + _, out, err = script_helper.assert_python_failure('-m', 'tabnanny', *args) + else: + _, out, err = script_helper.assert_python_ok('-m', 'tabnanny', *args) # Note: The `splitlines()` will solve the problem of CRLF(\r) added # by OS Windows. out = os.fsdecode(out) @@ -319,8 +325,8 @@ def test_with_errored_file(self): with TemporaryPyFile(SOURCE_CODES["wrong_indented"]) as file_path: stderr = f"{file_path!r}: Indentation Error: " stderr += ('unindent does not match any outer indentation level' - ' (<tokenize>, line 3)') - self.validate_cmd(file_path, stderr=stderr) + ' (<string>, line 3)') + self.validate_cmd(file_path, stderr=stderr, expect_failure=True) def test_with_error_free_file(self): """Should not display anything if python file is correctly indented.""" @@ -331,7 +337,7 @@ def test_command_usage(self): """Should display usage on no arguments.""" path = findfile('tabnanny.py') stderr = f"Usage: {path} [-v] file_or_directory ..." - self.validate_cmd(stderr=stderr) + self.validate_cmd(stderr=stderr, expect_failure=True) def test_quiet_flag(self): """Should display less when quite mode is on.""" @@ -339,18 +345,22 @@ def test_quiet_flag(self): stdout = f"{file_path}\n" self.validate_cmd("-q", file_path, stdout=stdout) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_verbose_mode(self): """Should display more error information if verbose mode is on.""" with TemporaryPyFile(SOURCE_CODES["nannynag_errored"]) as path: stdout = textwrap.dedent( - "offending line: '\\tprint(\"world\")\\n'" + "offending line: '\\tprint(\"world\")'" ).strip() self.validate_cmd("-v", path, stdout=stdout, partial=True) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_double_verbose_mode(self): """Should display detailed error information if double verbose is on.""" with TemporaryPyFile(SOURCE_CODES["nannynag_errored"]) as path: stdout = textwrap.dedent( - "offending line: '\\tprint(\"world\")\\n'" + "offending line: '\\tprint(\"world\")'" ).strip() self.validate_cmd("-vv", path, stdout=stdout, partial=True) diff --git a/Lib/test/test_tarfile.py b/Lib/test/test_tarfile.py index 0fed89c773..635a1c1c85 100644 --- a/Lib/test/test_tarfile.py +++ b/Lib/test/test_tarfile.py @@ -258,8 +258,6 @@ class ListTest(ReadTest, unittest.TestCase): def setUp(self): self.tar = tarfile.open(self.tarname, mode=self.mode) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_list(self): tio = io.TextIOWrapper(io.BytesIO(), 'ascii', newline='\n') with support.swap_attr(sys, 'stdout', tio): @@ -297,8 +295,6 @@ def conv(b): self.assertNotIn(b'link to', out) self.assertNotIn(b'->', out) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_list_verbose(self): tio = io.TextIOWrapper(io.BytesIO(), 'ascii', newline='\n') with support.swap_attr(sys, 'stdout', tio): @@ -323,8 +319,6 @@ def test_list_verbose(self): self.assertIn(b'pax' + (b'/123' * 125) + b'/longlink link to pax' + (b'/123' * 125) + b'/longname', out) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_list_members(self): tio = io.TextIOWrapper(io.BytesIO(), 'ascii', newline='\n') def members(tar): @@ -628,7 +622,6 @@ def test_extract_hardlink(self): data = f.read() self.assertEqual(sha256sum(data), sha256_regtype) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_extractall(self): # Test if extractall() correctly restores directory permissions # and times (see issue1735). @@ -659,7 +652,6 @@ def format_mtime(mtime): tar.close() os_helper.rmtree(DIR) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_extract_directory(self): dirtype = "ustar/dirtype" DIR = os.path.join(TEMPDIR, "extractdir") @@ -675,7 +667,6 @@ def test_extract_directory(self): finally: os_helper.rmtree(DIR) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_extractall_pathlike_name(self): DIR = pathlib.Path(TEMPDIR) / "extractall" with os_helper.temp_dir(DIR), \ @@ -686,7 +677,6 @@ def test_extractall_pathlike_name(self): path = DIR / tarinfo.name self.assertEqual(os.path.getmtime(path), tarinfo.mtime) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_extract_pathlike_name(self): dirtype = "ustar/dirtype" DIR = pathlib.Path(TEMPDIR) / "extractall" @@ -1276,7 +1266,6 @@ def test_gettarinfo_pathlike_name(self): self.assertEqual(tarinfo.name, tarinfo2.name) self.assertEqual(tarinfo.size, 3) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") @unittest.skipUnless(hasattr(os, "link"), "Missing hardlink implementation") def test_link_size(self): @@ -1301,7 +1290,6 @@ def test_link_size(self): os_helper.unlink(target) os_helper.unlink(link) - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") @os_helper.skip_unless_symlink def test_symlink_size(self): path = os.path.join(TEMPDIR, "symlink") @@ -1849,7 +1837,6 @@ def test_add_twice(self): self.assertEqual(tarinfo.type, tarfile.REGTYPE, "add file as regular failed") - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def test_add_hardlink(self): tarinfo = self.tar.gettarinfo(self.bar) self.assertEqual(tarinfo.type, tarfile.LNKTYPE, diff --git a/Lib/test/test_tempfile.py b/Lib/test/test_tempfile.py index 913897de40..5674839cd4 100644 --- a/Lib/test/test_tempfile.py +++ b/Lib/test/test_tempfile.py @@ -1187,8 +1187,6 @@ def test_properties(self): with self.assertRaises(AttributeError): f.errors - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_text_mode(self): # Creating a SpooledTemporaryFile with a text mode should produce # a file object reading and writing (Unicode) text strings. @@ -1221,8 +1219,6 @@ def test_text_mode(self): self.assertEqual(f.encoding, "utf-8") self.assertEqual(f.errors, "strict") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_text_newline_and_encoding(self): f = tempfile.SpooledTemporaryFile(mode='w+', max_size=10, newline='', encoding='utf-8', diff --git a/Lib/test/test_threading.py b/Lib/test/test_threading.py index 94f21d1c38..8f86792e82 100644 --- a/Lib/test/test_threading.py +++ b/Lib/test/test_threading.py @@ -229,6 +229,7 @@ def f(mutex): # PyThreadState_SetAsyncExc() is a CPython-only gimmick, not (currently) # exposed at the Python level. This test relies on ctypes to get at it. + @unittest.skip("TODO: RUSTPYTHON; expects @cpython_only") def test_PyThreadState_SetAsyncExc(self): ctypes = import_module("ctypes") @@ -332,6 +333,7 @@ def fail_new_thread(*args): finally: threading._start_new_thread = _start_new_thread + @unittest.skip("TODO: RUSTPYTHON; ctypes.pythonapi is not supported") def test_finalize_running_thread(self): # Issue 1402: the PyGILState_Ensure / _Release functions may be called # very late on python exit: on deallocation of a running thread for diff --git a/Lib/test/test_threading_local.py b/Lib/test/test_threading_local.py index 3443e3875d..99052de4c7 100644 --- a/Lib/test/test_threading_local.py +++ b/Lib/test/test_threading_local.py @@ -3,8 +3,8 @@ from doctest import DocTestSuite from test import support from test.support import threading_helper +from test.support.import_helper import import_module import weakref -import gc # Modules under test import _thread @@ -12,6 +12,9 @@ import _threading_local +threading_helper.requires_working_threading(module=True) + + class Weak(object): pass @@ -23,7 +26,7 @@ def target(local, weaklist): class BaseLocalTest: - @unittest.skip("TODO: RUSTPYTHON, flaky test") + @unittest.skip('TODO: RUSTPYTHON; flaky test') def test_local_refs(self): self._local_refs(20) self._local_refs(50) @@ -182,8 +185,7 @@ class LocalSubclass(self._local): """To test that subclasses behave properly.""" self._test_dict_attribute(LocalSubclass) - # TODO: RUSTPYTHON, cycle detection/collection - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON; cycle detection/collection def test_cycle_collection(self): class X: pass @@ -197,35 +199,57 @@ class X: self.assertIsNone(wr()) + def test_threading_local_clear_race(self): + # See https://github.com/python/cpython/issues/100892 + + _testcapi = import_module('_testcapi') + _testcapi.call_in_temporary_c_thread(lambda: None, False) + + for _ in range(1000): + _ = threading.local() + + _testcapi.join_temporary_c_thread() + + @support.cpython_only + def test_error(self): + class Loop(self._local): + attr = 1 + + # Trick the "if name == '__dict__':" test of __setattr__() + # to always be true + class NameCompareTrue: + def __eq__(self, other): + return True + + loop = Loop() + with self.assertRaisesRegex(AttributeError, 'Loop.*read-only'): + loop.__setattr__(NameCompareTrue(), 2) + + class ThreadLocalTest(unittest.TestCase, BaseLocalTest): _local = _thread._local - # TODO: RUSTPYTHON, __new__ vs __init__ cooperation - @unittest.expectedFailure - def test_arguments(): - super().test_arguments() - + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: TypeError not raised by _local + def test_arguments(self): + return super().test_arguments() class PyThreadingLocalTest(unittest.TestCase, BaseLocalTest): _local = _threading_local.local -def test_main(): - suite = unittest.TestSuite() - suite.addTest(DocTestSuite('_threading_local')) - suite.addTest(unittest.makeSuite(ThreadLocalTest)) - suite.addTest(unittest.makeSuite(PyThreadingLocalTest)) +def load_tests(loader, tests, pattern): + tests.addTest(DocTestSuite('_threading_local')) local_orig = _threading_local.local def setUp(test): _threading_local.local = _thread._local def tearDown(test): _threading_local.local = local_orig - suite.addTest(DocTestSuite('_threading_local', - setUp=setUp, tearDown=tearDown) - ) + tests.addTests(DocTestSuite('_threading_local', + setUp=setUp, tearDown=tearDown) + ) + return tests - support.run_unittest(suite) if __name__ == '__main__': - test_main() + unittest.main() diff --git a/Lib/test/test_time.py b/Lib/test/test_time.py index 3886aae934..31a2a920d9 100644 --- a/Lib/test/test_time.py +++ b/Lib/test/test_time.py @@ -284,8 +284,6 @@ def test_strptime_bytes(self): self.assertRaises(TypeError, time.strptime, b'2009', "%Y") self.assertRaises(TypeError, time.strptime, '2009', b'%Y') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_strptime_exception_context(self): # check that this doesn't chain exceptions needlessly (see #17572) with self.assertRaises(ValueError) as e: diff --git a/Lib/test/test_trace.py b/Lib/test/test_trace.py index a551a17fc1..a90c9d0baa 100644 --- a/Lib/test/test_trace.py +++ b/Lib/test/test_trace.py @@ -1,11 +1,12 @@ import os from pickle import dump import sys -from test.support import captured_stdout +from test.support import captured_stdout, requires_resource, requires_gil_enabled from test.support.os_helper import (TESTFN, rmtree, unlink) from test.support.script_helper import assert_python_ok, assert_python_failure import textwrap import unittest +from types import FunctionType import trace from trace import Trace @@ -197,9 +198,7 @@ def test_trace_list_comprehension(self): firstlineno_called = get_firstlineno(traced_doubler) expected = { (self.my_py_filename, firstlineno_calling + 1): 1, - # List comprehensions work differently in 3.x, so the count - # below changed compared to 2.x. - (self.my_py_filename, firstlineno_calling + 2): 12, + (self.my_py_filename, firstlineno_calling + 2): 11, (self.my_py_filename, firstlineno_calling + 3): 1, (self.my_py_filename, firstlineno_called + 1): 10, } @@ -251,7 +250,7 @@ def setUp(self): self.my_py_filename = fix_ext_py(__file__) self.addCleanup(sys.settrace, sys.gettrace()) - # TODO: RUSTPYTHON, KeyError: ('Lib/test/test_trace.py', 43) + # TODO: RUSTPYTHON @unittest.expectedFailure def test_exec_counts(self): self.tracer = Trace(count=1, trace=0, countfuncs=0, countcallers=0) @@ -287,7 +286,7 @@ def tearDown(self): if self._saved_tracefunc is not None: sys.settrace(self._saved_tracefunc) - # TODO: RUSTPYTHON, gc + # TODO: RUSTPYTHON @unittest.expectedFailure def test_simple_caller(self): self.tracer.runfunc(traced_func_simple_caller, 1) @@ -306,7 +305,7 @@ def test_arg_errors(self): with self.assertRaises(TypeError): self.tracer.runfunc() - # TODO: RUSTPYTHON, gc + # TODO: RUSTPYTHON @unittest.expectedFailure def test_loop_caller_importing(self): self.tracer.runfunc(traced_func_importing_caller, 1) @@ -320,10 +319,11 @@ def test_loop_caller_importing(self): } self.assertEqual(self.tracer.results().calledfuncs, expected) - # TODO: RUSTPYTHON, gc + # TODO: RUSTPYTHON @unittest.expectedFailure @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), 'pre-existing trace function throws off measurements') + @requires_gil_enabled("gh-117783: immortalization of types affects traced method names") def test_inst_method_calling(self): obj = TracedClass(20) self.tracer.runfunc(obj.inst_method_calling, 1) @@ -335,7 +335,7 @@ def test_inst_method_calling(self): } self.assertEqual(self.tracer.results().calledfuncs, expected) - # TODO: RUSTPYTHON, gc + # TODO: RUSTPYTHON @unittest.expectedFailure def test_traced_decorated_function(self): self.tracer.runfunc(traced_decorated_function) @@ -357,9 +357,11 @@ def setUp(self): self.tracer = Trace(count=0, trace=0, countcallers=1) self.filemod = my_file_and_modname() + # TODO: RUSTPYTHON + @unittest.expectedFailure @unittest.skipIf(hasattr(sys, 'gettrace') and sys.gettrace(), 'pre-existing trace function throws off measurements') - @unittest.skip("TODO: RUSTPYTHON, Error in atexit._run_exitfuncs") + @requires_gil_enabled("gh-117783: immortalization of types affects traced method names") def test_loop_caller_importing(self): self.tracer.runfunc(traced_func_importing_caller, 1) @@ -387,15 +389,21 @@ def tearDown(self): rmtree(TESTFN) unlink(TESTFN) - def _coverage(self, tracer, - cmd='import test.support, test.test_pprint;' - 'test.support.run_unittest(test.test_pprint.QueryTestCase)'): + DEFAULT_SCRIPT = '''if True: + import unittest + from test.test_pprint import QueryTestCase + loader = unittest.TestLoader() + tests = loader.loadTestsFromTestCase(QueryTestCase) + tests(unittest.TestResult()) + ''' + def _coverage(self, tracer, cmd=DEFAULT_SCRIPT): tracer.run(cmd) r = tracer.results() r.write_results(show_missing=True, summary=True, coverdir=TESTFN) # TODO: RUSTPYTHON @unittest.expectedFailure + @requires_resource('cpu') def test_coverage(self): tracer = trace.Trace(trace=0, count=1) with captured_stdout() as stdout: @@ -412,7 +420,7 @@ def test_coverage_ignore(self): libpath = os.path.normpath(os.path.dirname(os.path.dirname(__file__))) # sys.prefix does not work when running from a checkout tracer = trace.Trace(ignoredirs=[sys.base_prefix, sys.base_exec_prefix, - libpath], trace=0, count=1) + libpath] + sys.path, trace=0, count=1) with captured_stdout() as stdout: self._coverage(tracer) if os.path.exists(TESTFN): @@ -592,5 +600,31 @@ def test_run_as_module(self): assert_python_failure('-m', 'trace', '-l', '--module', 'not_a_module_zzz') +class TestTrace(unittest.TestCase): + def setUp(self): + self.addCleanup(sys.settrace, sys.gettrace()) + self.tracer = Trace(count=0, trace=1) + self.filemod = my_file_and_modname() + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_no_source_file(self): + filename = "<unknown>" + co = traced_func_linear.__code__ + co = co.replace(co_filename=filename) + f = FunctionType(co, globals()) + + with captured_stdout() as out: + self.tracer.runfunc(f, 2, 3) + + out = out.getvalue().splitlines() + firstlineno = get_firstlineno(f) + self.assertIn(f" --- modulename: {self.filemod[1]}, funcname: {f.__code__.co_name}", out[0]) + self.assertIn(f"{filename}({firstlineno + 1})", out[1]) + self.assertIn(f"{filename}({firstlineno + 2})", out[2]) + self.assertIn(f"{filename}({firstlineno + 3})", out[3]) + self.assertIn(f"{filename}({firstlineno + 4})", out[4]) + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_traceback.py b/Lib/test/test_traceback.py index 30a13587a2..9d95903d52 100644 --- a/Lib/test/test_traceback.py +++ b/Lib/test/test_traceback.py @@ -4,26 +4,54 @@ from io import StringIO import linecache import sys +import types import inspect +import builtins import unittest +import unittest.mock import re +import tempfile +import random +import string from test import support -from test.support import Error, captured_output, cpython_only, ALWAYS_EQ +import shutil +from test.support import (Error, captured_output, cpython_only, ALWAYS_EQ, + requires_debug_ranges, has_no_debug_ranges, + requires_subprocess) from test.support.os_helper import TESTFN, unlink -from test.support.script_helper import assert_python_ok -import textwrap +from test.support.script_helper import assert_python_ok, assert_python_failure +from test.support.import_helper import forget +from test.support import force_not_colorized, force_not_colorized_test_class +import json +import textwrap import traceback +from functools import partial +from pathlib import Path +import _colorize +MODULE_PREFIX = f'{__name__}.' if __name__ == '__main__' else '' test_code = namedtuple('code', ['co_filename', 'co_name']) +test_code.co_positions = lambda _: iter([(6, 6, 0, 0)]) test_frame = namedtuple('frame', ['f_code', 'f_globals', 'f_locals']) -test_tb = namedtuple('tb', ['tb_frame', 'tb_lineno', 'tb_next']) +test_tb = namedtuple('tb', ['tb_frame', 'tb_lineno', 'tb_next', 'tb_lasti']) + + +LEVENSHTEIN_DATA_FILE = Path(__file__).parent / 'levenshtein_examples.json' class TracebackCases(unittest.TestCase): # For now, a very minimal set of tests. I want to be sure that # formatting of SyntaxErrors works based on changes for 2.1. + def setUp(self): + super().setUp() + self.colorize = _colorize.COLORIZE + _colorize.COLORIZE = False + + def tearDown(self): + super().tearDown() + _colorize.COLORIZE = self.colorize def get_exception_format(self, func, exc): try: @@ -99,6 +127,75 @@ def test_nocaret(self): self.assertEqual(len(err), 3) self.assertEqual(err[1].strip(), "bad syntax") + @force_not_colorized + def test_no_caret_with_no_debug_ranges_flag(self): + # Make sure that if `-X no_debug_ranges` is used, there are no carets + # in the traceback. + try: + with open(TESTFN, 'w') as f: + f.write("x = 1 / 0\n") + + _, _, stderr = assert_python_failure( + '-X', 'no_debug_ranges', TESTFN) + + lines = stderr.splitlines() + self.assertEqual(len(lines), 4) + self.assertEqual(lines[0], b'Traceback (most recent call last):') + self.assertIn(b'line 1, in <module>', lines[1]) + self.assertEqual(lines[2], b' x = 1 / 0') + self.assertEqual(lines[3], b'ZeroDivisionError: division by zero') + finally: + unlink(TESTFN) + + def test_no_caret_with_no_debug_ranges_flag_python_traceback(self): + code = textwrap.dedent(""" + import traceback + try: + x = 1 / 0 + except ZeroDivisionError: + traceback.print_exc() + """) + try: + with open(TESTFN, 'w') as f: + f.write(code) + + _, _, stderr = assert_python_ok( + '-X', 'no_debug_ranges', TESTFN) + + lines = stderr.splitlines() + self.assertEqual(len(lines), 4) + self.assertEqual(lines[0], b'Traceback (most recent call last):') + self.assertIn(b'line 4, in <module>', lines[1]) + self.assertEqual(lines[2], b' x = 1 / 0') + self.assertEqual(lines[3], b'ZeroDivisionError: division by zero') + finally: + unlink(TESTFN) + + def test_recursion_error_during_traceback(self): + code = textwrap.dedent(""" + import sys + from weakref import ref + + sys.setrecursionlimit(15) + + def f(): + ref(lambda: 0, []) + f() + + try: + f() + except RecursionError: + pass + """) + try: + with open(TESTFN, 'w') as f: + f.write(code) + + rc, _, _ = assert_python_ok(TESTFN) + self.assertEqual(rc, 0) + finally: + unlink(TESTFN) + # TODO: RUSTPYTHON @unittest.expectedFailure def test_bad_indentation(self): @@ -121,21 +218,222 @@ def test_base_exception(self): lst = traceback.format_exception_only(e.__class__, e) self.assertEqual(lst, ['KeyboardInterrupt\n']) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_format_exception_only_bad__str__(self): class X(Exception): def __str__(self): 1/0 err = traceback.format_exception_only(X, X()) self.assertEqual(len(err), 1) - str_value = '<unprintable %s object>' % X.__name__ + str_value = '<exception str() failed>' if X.__module__ in ('__main__', 'builtins'): str_name = X.__qualname__ else: str_name = '.'.join([X.__module__, X.__qualname__]) self.assertEqual(err[0], "%s: %s\n" % (str_name, str_value)) + def test_format_exception_group_without_show_group(self): + eg = ExceptionGroup('A', [ValueError('B')]) + err = traceback.format_exception_only(eg) + self.assertEqual(err, ['ExceptionGroup: A (1 sub-exception)\n']) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_format_exception_group(self): + eg = ExceptionGroup('A', [ValueError('B')]) + err = traceback.format_exception_only(eg, show_group=True) + self.assertEqual(err, [ + 'ExceptionGroup: A (1 sub-exception)\n', + ' ValueError: B\n', + ]) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_format_base_exception_group(self): + eg = BaseExceptionGroup('A', [BaseException('B')]) + err = traceback.format_exception_only(eg, show_group=True) + self.assertEqual(err, [ + 'BaseExceptionGroup: A (1 sub-exception)\n', + ' BaseException: B\n', + ]) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_format_exception_group_with_note(self): + exc = ValueError('B') + exc.add_note('Note') + eg = ExceptionGroup('A', [exc]) + err = traceback.format_exception_only(eg, show_group=True) + self.assertEqual(err, [ + 'ExceptionGroup: A (1 sub-exception)\n', + ' ValueError: B\n', + ' Note\n', + ]) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_format_exception_group_explicit_class(self): + eg = ExceptionGroup('A', [ValueError('B')]) + err = traceback.format_exception_only(ExceptionGroup, eg, show_group=True) + self.assertEqual(err, [ + 'ExceptionGroup: A (1 sub-exception)\n', + ' ValueError: B\n', + ]) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_format_exception_group_multiple_exceptions(self): + eg = ExceptionGroup('A', [ValueError('B'), TypeError('C')]) + err = traceback.format_exception_only(eg, show_group=True) + self.assertEqual(err, [ + 'ExceptionGroup: A (2 sub-exceptions)\n', + ' ValueError: B\n', + ' TypeError: C\n', + ]) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_format_exception_group_multiline_messages(self): + eg = ExceptionGroup('A\n1', [ValueError('B\n2')]) + err = traceback.format_exception_only(eg, show_group=True) + self.assertEqual(err, [ + 'ExceptionGroup: A\n1 (1 sub-exception)\n', + ' ValueError: B\n', + ' 2\n', + ]) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_format_exception_group_multiline2_messages(self): + exc = ValueError('B\n\n2\n') + exc.add_note('\nC\n\n3') + eg = ExceptionGroup('A\n\n1\n', [exc, IndexError('D')]) + err = traceback.format_exception_only(eg, show_group=True) + self.assertEqual(err, [ + 'ExceptionGroup: A\n\n1\n (2 sub-exceptions)\n', + ' ValueError: B\n', + ' \n', + ' 2\n', + ' \n', + ' \n', # first char of `note` + ' C\n', + ' \n', + ' 3\n', # note ends + ' IndexError: D\n', + ]) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_format_exception_group_syntax_error(self): + exc = SyntaxError("error", ("x.py", 23, None, "bad syntax")) + eg = ExceptionGroup('A\n1', [exc]) + err = traceback.format_exception_only(eg, show_group=True) + self.assertEqual(err, [ + 'ExceptionGroup: A\n1 (1 sub-exception)\n', + ' File "x.py", line 23\n', + ' bad syntax\n', + ' SyntaxError: error\n', + ]) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_format_exception_group_nested_with_notes(self): + exc = IndexError('D') + exc.add_note('Note\nmultiline') + eg = ExceptionGroup('A', [ + ValueError('B'), + ExceptionGroup('C', [exc, LookupError('E')]), + TypeError('F'), + ]) + err = traceback.format_exception_only(eg, show_group=True) + self.assertEqual(err, [ + 'ExceptionGroup: A (3 sub-exceptions)\n', + ' ValueError: B\n', + ' ExceptionGroup: C (2 sub-exceptions)\n', + ' IndexError: D\n', + ' Note\n', + ' multiline\n', + ' LookupError: E\n', + ' TypeError: F\n', + ]) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_format_exception_group_with_tracebacks(self): + def f(): + try: + 1 / 0 + except ZeroDivisionError as e: + return e + + def g(): + try: + raise TypeError('g') + except TypeError as e: + return e + + eg = ExceptionGroup('A', [ + f(), + ExceptionGroup('B', [g()]), + ]) + err = traceback.format_exception_only(eg, show_group=True) + self.assertEqual(err, [ + 'ExceptionGroup: A (2 sub-exceptions)\n', + ' ZeroDivisionError: division by zero\n', + ' ExceptionGroup: B (1 sub-exception)\n', + ' TypeError: g\n', + ]) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_format_exception_group_with_cause(self): + def f(): + try: + try: + 1 / 0 + except ZeroDivisionError: + raise ValueError(0) + except Exception as e: + return e + + eg = ExceptionGroup('A', [f()]) + err = traceback.format_exception_only(eg, show_group=True) + self.assertEqual(err, [ + 'ExceptionGroup: A (1 sub-exception)\n', + ' ValueError: 0\n', + ]) + # TODO: RUSTPYTHON @unittest.expectedFailure + def test_format_exception_group_syntax_error_with_custom_values(self): + # See https://github.com/python/cpython/issues/128894 + for exc in [ + SyntaxError('error', 'abcd'), + SyntaxError('error', [None] * 4), + SyntaxError('error', (1, 2, 3, 4)), + SyntaxError('error', (1, 2, 3, 4)), + SyntaxError('error', (1, 'a', 'b', 2)), + # with end_lineno and end_offset: + SyntaxError('error', 'abcdef'), + SyntaxError('error', [None] * 6), + SyntaxError('error', (1, 2, 3, 4, 5, 6)), + SyntaxError('error', (1, 'a', 'b', 2, 'c', 'd')), + ]: + with self.subTest(exc=exc): + err = traceback.format_exception_only(exc, show_group=True) + # Should not raise an exception: + if exc.lineno is not None: + self.assertEqual(len(err), 2) + self.assertTrue(err[0].startswith(' File')) + else: + self.assertEqual(len(err), 1) + self.assertEqual(err[-1], 'SyntaxError: error\n') + + # TODO: RUSTPYTHON; IndexError: index out of range + @unittest.expectedFailure + @requires_subprocess() + @force_not_colorized def test_encoded_file(self): # Test that tracebacks are correctly printed for encoded source files: # - correct line number (Issue2384) @@ -183,9 +481,10 @@ def do_test(firstlines, message, charset, lineno): self.assertTrue(stdout[2].endswith(err_line), "Invalid traceback line: {0!r} instead of {1!r}".format( stdout[2], err_line)) - self.assertTrue(stdout[3] == err_msg, + actual_err_msg = stdout[3] + self.assertTrue(actual_err_msg == err_msg, "Invalid error message: {0!r} instead of {1!r}".format( - stdout[3], err_msg)) + actual_err_msg, err_msg)) do_test("", "foo", "ascii", 3) for charset in ("ascii", "iso-8859-1", "utf-8", "GBK"): @@ -217,15 +516,15 @@ class PrintExceptionAtExit(object): def __init__(self): try: x = 1 / 0 - except Exception: - self.exc_info = sys.exc_info() - # self.exc_info[1] (traceback) contains frames: + except Exception as e: + self.exc = e + # self.exc.__traceback__ contains frames: # explicitly clear the reference to self in the current # frame to break a reference cycle self = None def __del__(self): - traceback.print_exception(*self.exc_info) + traceback.print_exception(self.exc) # Keep a reference in the module namespace to call the destructor # when the module is unloaded @@ -234,6 +533,8 @@ def __del__(self): rc, stdout, stderr = assert_python_ok('-c', code) expected = [b'Traceback (most recent call last):', b' File "<string>", line 8, in __init__', + b' x = 1 / 0', + b' ^^^^^', b'ZeroDivisionError: division by zero'] self.assertEqual(stderr.splitlines(), expected) @@ -249,6 +550,16 @@ def test_print_exception_exc(self): traceback.print_exception(Exception("projector"), file=output) self.assertEqual(output.getvalue(), "Exception: projector\n") + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_print_last(self): + with support.swap_attr(sys, 'last_exc', ValueError(42)): + output = StringIO() + traceback.print_last(file=output) + self.assertEqual(output.getvalue(), "ValueError: 42\n") + + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_format_exception_exc(self): e = Exception("projector") output = traceback.format_exception(e) @@ -257,7 +568,7 @@ def test_format_exception_exc(self): traceback.format_exception(e.__class__, e) with self.assertRaisesRegex(ValueError, 'Both or neither'): traceback.format_exception(e.__class__, tb=e.__traceback__) - with self.assertRaisesRegex(TypeError, 'positional-only'): + with self.assertRaisesRegex(TypeError, 'required positional argument'): traceback.format_exception(exc=e) def test_format_exception_only_exc(self): @@ -287,194 +598,1502 @@ def test_exception_is_None(self): self.assertEqual( traceback.format_exception_only(None, None), [NONE_EXC_STRING]) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_signatures(self): self.assertEqual( str(inspect.signature(traceback.print_exception)), ('(exc, /, value=<implicit>, tb=<implicit>, ' - 'limit=None, file=None, chain=True)')) + 'limit=None, file=None, chain=True, **kwargs)')) self.assertEqual( str(inspect.signature(traceback.format_exception)), ('(exc, /, value=<implicit>, tb=<implicit>, limit=None, ' - 'chain=True)')) + 'chain=True, **kwargs)')) self.assertEqual( str(inspect.signature(traceback.format_exception_only)), - '(exc, /, value=<implicit>)') - + '(exc, /, value=<implicit>, *, show_group=False, **kwargs)') -class TracebackFormatTests(unittest.TestCase): - def some_exception(self): - raise KeyError('blah') - - @cpython_only - def check_traceback_format(self, cleanup_func=None): - from _testcapi import traceback_print +class PurePythonExceptionFormattingMixin: + def get_exception(self, callable, slice_start=0, slice_end=-1): try: - self.some_exception() - except KeyError: - type_, value, tb = sys.exc_info() - if cleanup_func is not None: - # Clear the inner frames, not this one - cleanup_func(tb.tb_next) - traceback_fmt = 'Traceback (most recent call last):\n' + \ - ''.join(traceback.format_tb(tb)) - file_ = StringIO() - traceback_print(tb, file_) - python_fmt = file_.getvalue() - # Call all _tb and _exc functions - with captured_output("stderr") as tbstderr: - traceback.print_tb(tb) - tbfile = StringIO() - traceback.print_tb(tb, file=tbfile) - with captured_output("stderr") as excstderr: - traceback.print_exc() - excfmt = traceback.format_exc() - excfile = StringIO() - traceback.print_exc(file=excfile) + callable() + except BaseException: + return traceback.format_exc().splitlines()[slice_start:slice_end] else: - raise Error("unable to create test traceback string") + self.fail("No exception thrown.") - # Make sure that Python and the traceback module format the same thing - self.assertEqual(traceback_fmt, python_fmt) - # Now verify the _tb func output - self.assertEqual(tbstderr.getvalue(), tbfile.getvalue()) - # Now verify the _exc func output - self.assertEqual(excstderr.getvalue(), excfile.getvalue()) - self.assertEqual(excfmt, excfile.getvalue()) + callable_line = get_exception.__code__.co_firstlineno + 2 - # Make sure that the traceback is properly indented. - tb_lines = python_fmt.splitlines() - self.assertEqual(len(tb_lines), 5) - banner = tb_lines[0] - location, source_line = tb_lines[-2:] - self.assertTrue(banner.startswith('Traceback')) - self.assertTrue(location.startswith(' File')) - self.assertTrue(source_line.startswith(' raise')) - def test_traceback_format(self): - self.check_traceback_format() +class CAPIExceptionFormattingMixin: + LEGACY = 0 - def test_traceback_format_with_cleared_frames(self): - # Check that traceback formatting also works with a clear()ed frame - def cleanup_tb(tb): - tb.tb_frame.clear() - self.check_traceback_format(cleanup_tb) + def get_exception(self, callable, slice_start=0, slice_end=-1): + from _testcapi import exception_print + try: + callable() + self.fail("No exception thrown.") + except Exception as e: + with captured_output("stderr") as tbstderr: + exception_print(e, self.LEGACY) + return tbstderr.getvalue().splitlines()[slice_start:slice_end] - def test_stack_format(self): - # Verify _stack functions. Note we have to use _getframe(1) to - # compare them without this frame appearing in the output - with captured_output("stderr") as ststderr: - traceback.print_stack(sys._getframe(1)) - stfile = StringIO() - traceback.print_stack(sys._getframe(1), file=stfile) - self.assertEqual(ststderr.getvalue(), stfile.getvalue()) + callable_line = get_exception.__code__.co_firstlineno + 3 - stfmt = traceback.format_stack(sys._getframe(1)) +class CAPIExceptionFormattingLegacyMixin(CAPIExceptionFormattingMixin): + LEGACY = 1 - self.assertEqual(ststderr.getvalue(), "".join(stfmt)) +# @requires_debug_ranges() # XXX: RUSTPYTHON patch +class TracebackErrorLocationCaretTestBase: + """ + Tests for printing code error expressions as part of PEP 657 + """ + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_basic_caret(self): + # NOTE: In caret tests, "if True:" is used as a way to force indicator + # display, since the raising expression spans only part of the line. + def f(): + if True: raise ValueError("basic caret tests") - def test_print_stack(self): - def prn(): - traceback.print_stack() - with captured_output("stderr") as stderr: - prn() - lineno = prn.__code__.co_firstlineno - self.assertEqual(stderr.getvalue().splitlines()[-4:], [ - ' File "%s", line %d, in test_print_stack' % (__file__, lineno+3), - ' prn()', - ' File "%s", line %d, in prn' % (__file__, lineno+1), - ' traceback.print_stack()', - ]) + lineno_f = f.__code__.co_firstlineno + expected_f = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + ' callable()\n' + ' ~~~~~~~~^^\n' + f' File "{__file__}", line {lineno_f+1}, in f\n' + ' if True: raise ValueError("basic caret tests")\n' + ' ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n' + ) + result_lines = self.get_exception(f) + self.assertEqual(result_lines, expected_f.splitlines()) - # issue 26823 - Shrink recursive tracebacks - def _check_recursive_traceback_display(self, render_exc): - # Always show full diffs when this test fails - # Note that rearranging things may require adjusting - # the relative line numbers in the expected tracebacks - self.maxDiff = None + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_line_with_unicode(self): + # Make sure that even if a line contains multi-byte unicode characters + # the correct carets are printed. + def f_with_unicode(): + if True: raise ValueError("Ĥellö Wörld") + + lineno_f = f_with_unicode.__code__.co_firstlineno + expected_f = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + ' callable()\n' + ' ~~~~~~~~^^\n' + f' File "{__file__}", line {lineno_f+1}, in f_with_unicode\n' + ' if True: raise ValueError("Ĥellö Wörld")\n' + ' ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n' + ) + result_lines = self.get_exception(f_with_unicode) + self.assertEqual(result_lines, expected_f.splitlines()) - # Check hitting the recursion limit - def f(): - f() + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_caret_in_type_annotation(self): + def f_with_type(): + def foo(a: THIS_DOES_NOT_EXIST ) -> int: + return 0 - with captured_output("stderr") as stderr_f: - try: - f() - except RecursionError: - render_exc() - else: - self.fail("no recursion occurred") + lineno_f = f_with_type.__code__.co_firstlineno + expected_f = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + ' callable()\n' + ' ~~~~~~~~^^\n' + f' File "{__file__}", line {lineno_f+1}, in f_with_type\n' + ' def foo(a: THIS_DOES_NOT_EXIST ) -> int:\n' + ' ^^^^^^^^^^^^^^^^^^^\n' + ) + result_lines = self.get_exception(f_with_type) + self.assertEqual(result_lines, expected_f.splitlines()) - lineno_f = f.__code__.co_firstlineno - result_f = ( + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_caret_multiline_expression(self): + # Make sure no carets are printed for expressions spanning multiple + # lines. + def f_with_multiline(): + if True: raise ValueError( + "error over multiple lines" + ) + + lineno_f = f_with_multiline.__code__.co_firstlineno + expected_f = ( 'Traceback (most recent call last):\n' - f' File "{__file__}", line {lineno_f+5}, in _check_recursive_traceback_display\n' - ' f()\n' - f' File "{__file__}", line {lineno_f+1}, in f\n' - ' f()\n' - f' File "{__file__}", line {lineno_f+1}, in f\n' - ' f()\n' - f' File "{__file__}", line {lineno_f+1}, in f\n' - ' f()\n' - # XXX: The following line changes depending on whether the tests - # are run through the interactive interpreter or with -m - # It also varies depending on the platform (stack size) - # Fortunately, we don't care about exactness here, so we use regex - r' \[Previous line repeated (\d+) more times\]' '\n' - 'RecursionError: maximum recursion depth exceeded\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + ' callable()\n' + ' ~~~~~~~~^^\n' + f' File "{__file__}", line {lineno_f+1}, in f_with_multiline\n' + ' if True: raise ValueError(\n' + ' ^^^^^^^^^^^^^^^^^\n' + ' "error over multiple lines"\n' + ' ^^^^^^^^^^^^^^^^^^^^^^^^^^^\n' + ' )\n' + ' ^' ) + result_lines = self.get_exception(f_with_multiline) + self.assertEqual(result_lines, expected_f.splitlines()) - expected = result_f.splitlines() - actual = stderr_f.getvalue().splitlines() + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_caret_multiline_expression_syntax_error(self): + # Make sure an expression spanning multiple lines that has + # a syntax error is correctly marked with carets. + code = textwrap.dedent(""" + def foo(*args, **kwargs): + pass - # Check the output text matches expectations - # 2nd last line contains the repetition count - self.assertEqual(actual[:-2], expected[:-2]) - self.assertRegex(actual[-2], expected[-2]) - # last line can have additional text appended - self.assertIn(expected[-1], actual[-1]) + a, b, c = 1, 2, 3 - # Check the recursion count is roughly as expected - rec_limit = sys.getrecursionlimit() - self.assertIn(int(re.search(r"\d+", actual[-2]).group()), range(rec_limit-60, rec_limit)) + foo(a, z + for z in + range(10), b, c) + """) - # Check a known (limited) number of recursive invocations - def g(count=10): - if count: - return g(count-1) - raise ValueError + def f_with_multiline(): + # Need to defer the compilation until in self.get_exception(..) + return compile(code, "?", "exec") - with captured_output("stderr") as stderr_g: - try: - g() - except ValueError: - render_exc() - else: - self.fail("no value error was raised") + lineno_f = f_with_multiline.__code__.co_firstlineno - lineno_g = g.__code__.co_firstlineno - result_g = ( - f' File "{__file__}", line {lineno_g+2}, in g\n' - ' return g(count-1)\n' - f' File "{__file__}", line {lineno_g+2}, in g\n' - ' return g(count-1)\n' - f' File "{__file__}", line {lineno_g+2}, in g\n' - ' return g(count-1)\n' - ' [Previous line repeated 7 more times]\n' - f' File "{__file__}", line {lineno_g+3}, in g\n' - ' raise ValueError\n' - 'ValueError\n' - ) - tb_line = ( + expected_f = ( 'Traceback (most recent call last):\n' - f' File "{__file__}", line {lineno_g+7}, in _check_recursive_traceback_display\n' - ' g()\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + ' callable()\n' + ' ~~~~~~~~^^\n' + f' File "{__file__}", line {lineno_f+2}, in f_with_multiline\n' + ' return compile(code, "?", "exec")\n' + ' File "?", line 7\n' + ' foo(a, z\n' + ' ^' + ) + + result_lines = self.get_exception(f_with_multiline) + self.assertEqual(result_lines, expected_f.splitlines()) + + # Check custom error messages covering multiple lines + code = textwrap.dedent(""" + dummy_call( + "dummy value" + foo="bar", ) - expected = (tb_line + result_g).splitlines() - actual = stderr_g.getvalue().splitlines() - self.assertEqual(actual, expected) + """) + + def f_with_multiline(): + # Need to defer the compilation until in self.get_exception(..) + return compile(code, "?", "exec") + + lineno_f = f_with_multiline.__code__.co_firstlineno + + expected_f = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + ' callable()\n' + ' ~~~~~~~~^^\n' + f' File "{__file__}", line {lineno_f+2}, in f_with_multiline\n' + ' return compile(code, "?", "exec")\n' + ' File "?", line 3\n' + ' "dummy value"\n' + ' ^^^^^^^^^^^^^' + ) + + result_lines = self.get_exception(f_with_multiline) + self.assertEqual(result_lines, expected_f.splitlines()) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_caret_multiline_expression_bin_op(self): + # Make sure no carets are printed for expressions spanning multiple + # lines. + def f_with_multiline(): + return ( + 2 + 1 / + 0 + ) + + lineno_f = f_with_multiline.__code__.co_firstlineno + expected_f = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + ' callable()\n' + ' ~~~~~~~~^^\n' + f' File "{__file__}", line {lineno_f+2}, in f_with_multiline\n' + ' 2 + 1 /\n' + ' ~~^\n' + ' 0\n' + ' ~' + ) + result_lines = self.get_exception(f_with_multiline) + self.assertEqual(result_lines, expected_f.splitlines()) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_caret_for_binary_operators(self): + def f_with_binary_operator(): + divisor = 20 + return 10 + divisor / 0 + 30 + + lineno_f = f_with_binary_operator.__code__.co_firstlineno + expected_error = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + ' callable()\n' + ' ~~~~~~~~^^\n' + f' File "{__file__}", line {lineno_f+2}, in f_with_binary_operator\n' + ' return 10 + divisor / 0 + 30\n' + ' ~~~~~~~~^~~\n' + ) + result_lines = self.get_exception(f_with_binary_operator) + self.assertEqual(result_lines, expected_error.splitlines()) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_caret_for_binary_operators_with_unicode(self): + def f_with_binary_operator(): + áóí = 20 + return 10 + áóí / 0 + 30 + + lineno_f = f_with_binary_operator.__code__.co_firstlineno + expected_error = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + ' callable()\n' + ' ~~~~~~~~^^\n' + f' File "{__file__}", line {lineno_f+2}, in f_with_binary_operator\n' + ' return 10 + áóí / 0 + 30\n' + ' ~~~~^~~\n' + ) + result_lines = self.get_exception(f_with_binary_operator) + self.assertEqual(result_lines, expected_error.splitlines()) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_caret_for_binary_operators_two_char(self): + def f_with_binary_operator(): + divisor = 20 + return 10 + divisor // 0 + 30 + + lineno_f = f_with_binary_operator.__code__.co_firstlineno + expected_error = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + ' callable()\n' + ' ~~~~~~~~^^\n' + f' File "{__file__}", line {lineno_f+2}, in f_with_binary_operator\n' + ' return 10 + divisor // 0 + 30\n' + ' ~~~~~~~~^^~~\n' + ) + result_lines = self.get_exception(f_with_binary_operator) + self.assertEqual(result_lines, expected_error.splitlines()) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_caret_for_binary_operators_with_spaces_and_parenthesis(self): + def f_with_binary_operator(): + a = 1 + b = c = "" + return ( a ) +b + c + + lineno_f = f_with_binary_operator.__code__.co_firstlineno + expected_error = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + ' callable()\n' + ' ~~~~~~~~^^\n' + f' File "{__file__}", line {lineno_f+3}, in f_with_binary_operator\n' + ' return ( a ) +b + c\n' + ' ~~~~~~~~~~^~\n' + ) + result_lines = self.get_exception(f_with_binary_operator) + self.assertEqual(result_lines, expected_error.splitlines()) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_caret_for_binary_operators_multiline(self): + def f_with_binary_operator(): + b = 1 + c = "" + a = b \ + +\ + c # test + return a + + lineno_f = f_with_binary_operator.__code__.co_firstlineno + expected_error = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + ' callable()\n' + ' ~~~~~~~~^^\n' + f' File "{__file__}", line {lineno_f+3}, in f_with_binary_operator\n' + ' a = b \\\n' + ' ~~~~~~\n' + ' +\\\n' + ' ^~\n' + ' c # test\n' + ' ~\n' + ) + result_lines = self.get_exception(f_with_binary_operator) + self.assertEqual(result_lines, expected_error.splitlines()) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_caret_for_binary_operators_multiline_two_char(self): + def f_with_binary_operator(): + b = 1 + c = "" + a = ( + (b # test + + ) \ + # + + << (c # test + \ + ) # test + ) + return a + + lineno_f = f_with_binary_operator.__code__.co_firstlineno + expected_error = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + ' callable()\n' + ' ~~~~~~~~^^\n' + f' File "{__file__}", line {lineno_f+4}, in f_with_binary_operator\n' + ' (b # test +\n' + ' ~~~~~~~~~~~~\n' + ' ) \\\n' + ' ~~~~\n' + ' # +\n' + ' ~~~\n' + ' << (c # test\n' + ' ^^~~~~~~~~~~~\n' + ' \\\n' + ' ~\n' + ' ) # test\n' + ' ~\n' + ) + result_lines = self.get_exception(f_with_binary_operator) + self.assertEqual(result_lines, expected_error.splitlines()) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_caret_for_binary_operators_multiline_with_unicode(self): + def f_with_binary_operator(): + b = 1 + a = ("ááá" + + "áá") + b + return a + + lineno_f = f_with_binary_operator.__code__.co_firstlineno + expected_error = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + ' callable()\n' + ' ~~~~~~~~^^\n' + f' File "{__file__}", line {lineno_f+2}, in f_with_binary_operator\n' + ' a = ("ááá" +\n' + ' ~~~~~~~~\n' + ' "áá") + b\n' + ' ~~~~~~^~~\n' + ) + result_lines = self.get_exception(f_with_binary_operator) + self.assertEqual(result_lines, expected_error.splitlines()) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_caret_for_subscript(self): + def f_with_subscript(): + some_dict = {'x': {'y': None}} + return some_dict['x']['y']['z'] + + lineno_f = f_with_subscript.__code__.co_firstlineno + expected_error = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + ' callable()\n' + ' ~~~~~~~~^^\n' + f' File "{__file__}", line {lineno_f+2}, in f_with_subscript\n' + " return some_dict['x']['y']['z']\n" + ' ~~~~~~~~~~~~~~~~~~~^^^^^\n' + ) + result_lines = self.get_exception(f_with_subscript) + self.assertEqual(result_lines, expected_error.splitlines()) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_caret_for_subscript_unicode(self): + def f_with_subscript(): + some_dict = {'ó': {'á': {'í': {'theta': 1}}}} + return some_dict['ó']['á']['í']['beta'] + + lineno_f = f_with_subscript.__code__.co_firstlineno + expected_error = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + ' callable()\n' + ' ~~~~~~~~^^\n' + f' File "{__file__}", line {lineno_f+2}, in f_with_subscript\n' + " return some_dict['ó']['á']['í']['beta']\n" + ' ~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^^^\n' + ) + result_lines = self.get_exception(f_with_subscript) + self.assertEqual(result_lines, expected_error.splitlines()) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_caret_for_subscript_with_spaces_and_parenthesis(self): + def f_with_binary_operator(): + a = [] + b = c = 1 + return b [ a ] + c + + lineno_f = f_with_binary_operator.__code__.co_firstlineno + expected_error = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + ' callable()\n' + ' ~~~~~~~~^^\n' + f' File "{__file__}", line {lineno_f+3}, in f_with_binary_operator\n' + ' return b [ a ] + c\n' + ' ~~~~~~^^^^^^^^^\n' + ) + result_lines = self.get_exception(f_with_binary_operator) + self.assertEqual(result_lines, expected_error.splitlines()) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_caret_for_subscript_multiline(self): + def f_with_subscript(): + bbbbb = {} + ccc = 1 + ddd = 2 + b = bbbbb \ + [ ccc # test + + + ddd \ + + ] # test + return b + + lineno_f = f_with_subscript.__code__.co_firstlineno + expected_error = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + ' callable()\n' + ' ~~~~~~~~^^\n' + f' File "{__file__}", line {lineno_f+4}, in f_with_subscript\n' + ' b = bbbbb \\\n' + ' ~~~~~~~\n' + ' [ ccc # test\n' + ' ^^^^^^^^^^^^^\n' + ' \n' + ' \n' + ' + ddd \\\n' + ' ^^^^^^^^\n' + ' \n' + ' \n' + ' ] # test\n' + ' ^\n' + ) + result_lines = self.get_exception(f_with_subscript) + self.assertEqual(result_lines, expected_error.splitlines()) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_caret_for_call(self): + def f_with_call(): + def f1(a): + def f2(b): + raise RuntimeError("fail") + return f2 + return f1("x")("y")("z") + + lineno_f = f_with_call.__code__.co_firstlineno + expected_error = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + ' callable()\n' + ' ~~~~~~~~^^\n' + f' File "{__file__}", line {lineno_f+5}, in f_with_call\n' + ' return f1("x")("y")("z")\n' + ' ~~~~~~~^^^^^\n' + f' File "{__file__}", line {lineno_f+3}, in f2\n' + ' raise RuntimeError("fail")\n' + ) + result_lines = self.get_exception(f_with_call) + self.assertEqual(result_lines, expected_error.splitlines()) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_caret_for_call_unicode(self): + def f_with_call(): + def f1(a): + def f2(b): + raise RuntimeError("fail") + return f2 + return f1("ó")("á") + + lineno_f = f_with_call.__code__.co_firstlineno + expected_error = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + ' callable()\n' + ' ~~~~~~~~^^\n' + f' File "{__file__}", line {lineno_f+5}, in f_with_call\n' + ' return f1("ó")("á")\n' + ' ~~~~~~~^^^^^\n' + f' File "{__file__}", line {lineno_f+3}, in f2\n' + ' raise RuntimeError("fail")\n' + ) + result_lines = self.get_exception(f_with_call) + self.assertEqual(result_lines, expected_error.splitlines()) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_caret_for_call_with_spaces_and_parenthesis(self): + def f_with_binary_operator(): + def f(a): + raise RuntimeError("fail") + return f ( "x" ) + 2 + + lineno_f = f_with_binary_operator.__code__.co_firstlineno + expected_error = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + ' callable()\n' + ' ~~~~~~~~^^\n' + f' File "{__file__}", line {lineno_f+3}, in f_with_binary_operator\n' + ' return f ( "x" ) + 2\n' + ' ~~~~~~^^^^^^^^^^^\n' + f' File "{__file__}", line {lineno_f+2}, in f\n' + ' raise RuntimeError("fail")\n' + ) + result_lines = self.get_exception(f_with_binary_operator) + self.assertEqual(result_lines, expected_error.splitlines()) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_caret_for_call_multiline(self): + def f_with_call(): + class C: + def y(self, a): + def f(b): + raise RuntimeError("fail") + return f + def g(x): + return C() + a = (g(1).y)( + 2 + )(3)(4) + return a + + lineno_f = f_with_call.__code__.co_firstlineno + expected_error = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + ' callable()\n' + ' ~~~~~~~~^^\n' + f' File "{__file__}", line {lineno_f+8}, in f_with_call\n' + ' a = (g(1).y)(\n' + ' ~~~~~~~~~\n' + ' 2\n' + ' ~\n' + ' )(3)(4)\n' + ' ~^^^\n' + f' File "{__file__}", line {lineno_f+4}, in f\n' + ' raise RuntimeError("fail")\n' + ) + result_lines = self.get_exception(f_with_call) + self.assertEqual(result_lines, expected_error.splitlines()) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_many_lines(self): + def f(): + x = 1 + if True: x += ( + "a" + + "a" + ) # test + + lineno_f = f.__code__.co_firstlineno + expected_error = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + ' callable()\n' + ' ~~~~~~~~^^\n' + f' File "{__file__}", line {lineno_f+2}, in f\n' + ' if True: x += (\n' + ' ^^^^^^\n' + ' ...<2 lines>...\n' + ' ) # test\n' + ' ^\n' + ) + result_lines = self.get_exception(f) + self.assertEqual(result_lines, expected_error.splitlines()) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_many_lines_no_caret(self): + def f(): + x = 1 + x += ( + "a" + + "a" + ) + + lineno_f = f.__code__.co_firstlineno + expected_error = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + ' callable()\n' + ' ~~~~~~~~^^\n' + f' File "{__file__}", line {lineno_f+2}, in f\n' + ' x += (\n' + ' ...<2 lines>...\n' + ' )\n' + ) + result_lines = self.get_exception(f) + self.assertEqual(result_lines, expected_error.splitlines()) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_many_lines_binary_op(self): + def f_with_binary_operator(): + b = 1 + c = "a" + a = ( + b + + b + ) + ( + c + + c + + c + ) + return a + + lineno_f = f_with_binary_operator.__code__.co_firstlineno + expected_error = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + ' callable()\n' + ' ~~~~~~~~^^\n' + f' File "{__file__}", line {lineno_f+3}, in f_with_binary_operator\n' + ' a = (\n' + ' ~\n' + ' b +\n' + ' ~~~\n' + ' b\n' + ' ~\n' + ' ) + (\n' + ' ~~^~~\n' + ' c +\n' + ' ~~~\n' + ' ...<2 lines>...\n' + ' )\n' + ' ~\n' + ) + result_lines = self.get_exception(f_with_binary_operator) + self.assertEqual(result_lines, expected_error.splitlines()) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_traceback_specialization_with_syntax_error(self): + bytecode = compile("1 / 0 / 1 / 2\n", TESTFN, "exec") + + with open(TESTFN, "w") as file: + # make the file's contents invalid + file.write("1 $ 0 / 1 / 2\n") + self.addCleanup(unlink, TESTFN) + + func = partial(exec, bytecode) + result_lines = self.get_exception(func) + + lineno_f = bytecode.co_firstlineno + expected_error = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + ' callable()\n' + ' ~~~~~~~~^^\n' + f' File "{TESTFN}", line {lineno_f}, in <module>\n' + " 1 $ 0 / 1 / 2\n" + ' ^^^^^\n' + ) + self.assertEqual(result_lines, expected_error.splitlines()) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_traceback_very_long_line(self): + source = "if True: " + "a" * 256 + bytecode = compile(source, TESTFN, "exec") + + with open(TESTFN, "w") as file: + file.write(source) + self.addCleanup(unlink, TESTFN) + + func = partial(exec, bytecode) + result_lines = self.get_exception(func) + + lineno_f = bytecode.co_firstlineno + expected_error = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + ' callable()\n' + ' ~~~~~~~~^^\n' + f' File "{TESTFN}", line {lineno_f}, in <module>\n' + f' {source}\n' + f' {" "*len("if True: ") + "^"*256}\n' + ) + self.assertEqual(result_lines, expected_error.splitlines()) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_secondary_caret_not_elided(self): + # Always show a line's indicators if they include the secondary character. + def f_with_subscript(): + some_dict = {'x': {'y': None}} + some_dict['x']['y']['z'] + + lineno_f = f_with_subscript.__code__.co_firstlineno + expected_error = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + ' callable()\n' + ' ~~~~~~~~^^\n' + f' File "{__file__}", line {lineno_f+2}, in f_with_subscript\n' + " some_dict['x']['y']['z']\n" + ' ~~~~~~~~~~~~~~~~~~~^^^^^\n' + ) + result_lines = self.get_exception(f_with_subscript) + self.assertEqual(result_lines, expected_error.splitlines()) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_caret_exception_group(self): + # Notably, this covers whether indicators handle margin strings correctly. + # (Exception groups use margin strings to display vertical indicators.) + # The implementation must account for both "indent" and "margin" offsets. + + def exc(): + if True: raise ExceptionGroup("eg", [ValueError(1), TypeError(2)]) + + expected_error = ( + f' + Exception Group Traceback (most recent call last):\n' + f' | File "{__file__}", line {self.callable_line}, in get_exception\n' + f' | callable()\n' + f' | ~~~~~~~~^^\n' + f' | File "{__file__}", line {exc.__code__.co_firstlineno + 1}, in exc\n' + f' | if True: raise ExceptionGroup("eg", [ValueError(1), TypeError(2)])\n' + f' | ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n' + f' | ExceptionGroup: eg (2 sub-exceptions)\n' + f' +-+---------------- 1 ----------------\n' + f' | ValueError: 1\n' + f' +---------------- 2 ----------------\n' + f' | TypeError: 2\n') + + result_lines = self.get_exception(exc) + self.assertEqual(result_lines, expected_error.splitlines()) + + def assertSpecialized(self, func, expected_specialization): + result_lines = self.get_exception(func) + specialization_line = result_lines[-1] + self.assertEqual(specialization_line.lstrip(), expected_specialization) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_specialization_variations(self): + self.assertSpecialized(lambda: 1/0, + "~^~") + self.assertSpecialized(lambda: 1/0/3, + "~^~") + self.assertSpecialized(lambda: 1 / 0, + "~~^~~") + self.assertSpecialized(lambda: 1 / 0 / 3, + "~~^~~") + self.assertSpecialized(lambda: 1/ 0, + "~^~~") + self.assertSpecialized(lambda: 1/ 0/3, + "~^~~") + self.assertSpecialized(lambda: 1 / 0, + "~~~~~^~~~") + self.assertSpecialized(lambda: 1 / 0 / 5, + "~~~~~^~~~") + self.assertSpecialized(lambda: 1 /0, + "~~^~") + self.assertSpecialized(lambda: 1//0, + "~^^~") + self.assertSpecialized(lambda: 1//0//4, + "~^^~") + self.assertSpecialized(lambda: 1 // 0, + "~~^^~~") + self.assertSpecialized(lambda: 1 // 0 // 4, + "~~^^~~") + self.assertSpecialized(lambda: 1 //0, + "~~^^~") + self.assertSpecialized(lambda: 1// 0, + "~^^~~") + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_decorator_application_lineno_correct(self): + def dec_error(func): + raise TypeError + def dec_fine(func): + return func + def applydecs(): + @dec_error + @dec_fine + def g(): pass + result_lines = self.get_exception(applydecs) + lineno_applydescs = applydecs.__code__.co_firstlineno + lineno_dec_error = dec_error.__code__.co_firstlineno + expected_error = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + ' callable()\n' + ' ~~~~~~~~^^\n' + f' File "{__file__}", line {lineno_applydescs + 1}, in applydecs\n' + ' @dec_error\n' + ' ^^^^^^^^^\n' + f' File "{__file__}", line {lineno_dec_error + 1}, in dec_error\n' + ' raise TypeError\n' + ) + self.assertEqual(result_lines, expected_error.splitlines()) + + def applydecs_class(): + @dec_error + @dec_fine + class A: pass + result_lines = self.get_exception(applydecs_class) + lineno_applydescs_class = applydecs_class.__code__.co_firstlineno + expected_error = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + ' callable()\n' + ' ~~~~~~~~^^\n' + f' File "{__file__}", line {lineno_applydescs_class + 1}, in applydecs_class\n' + ' @dec_error\n' + ' ^^^^^^^^^\n' + f' File "{__file__}", line {lineno_dec_error + 1}, in dec_error\n' + ' raise TypeError\n' + ) + self.assertEqual(result_lines, expected_error.splitlines()) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_multiline_method_call_a(self): + def f(): + (None + .method + )() + actual = self.get_exception(f) + expected = [ + "Traceback (most recent call last):", + f" File \"{__file__}\", line {self.callable_line}, in get_exception", + " callable()", + " ~~~~~~~~^^", + f" File \"{__file__}\", line {f.__code__.co_firstlineno + 2}, in f", + " .method", + " ^^^^^^", + ] + self.assertEqual(actual, expected) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_multiline_method_call_b(self): + def f(): + (None. + method + )() + actual = self.get_exception(f) + expected = [ + "Traceback (most recent call last):", + f" File \"{__file__}\", line {self.callable_line}, in get_exception", + " callable()", + " ~~~~~~~~^^", + f" File \"{__file__}\", line {f.__code__.co_firstlineno + 2}, in f", + " method", + ] + self.assertEqual(actual, expected) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_multiline_method_call_c(self): + def f(): + (None + . method + )() + actual = self.get_exception(f) + expected = [ + "Traceback (most recent call last):", + f" File \"{__file__}\", line {self.callable_line}, in get_exception", + " callable()", + " ~~~~~~~~^^", + f" File \"{__file__}\", line {f.__code__.co_firstlineno + 2}, in f", + " . method", + " ^^^^^^", + ] + self.assertEqual(actual, expected) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_wide_characters_unicode_with_problematic_byte_offset(self): + def f(): + width + + actual = self.get_exception(f) + expected = [ + "Traceback (most recent call last):", + f" File \"{__file__}\", line {self.callable_line}, in get_exception", + " callable()", + " ~~~~~~~~^^", + f" File \"{__file__}\", line {f.__code__.co_firstlineno + 1}, in f", + " width", + ] + self.assertEqual(actual, expected) + + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_byte_offset_with_wide_characters_middle(self): + def f(): + width = 1 + raise ValueError(width) + + actual = self.get_exception(f) + expected = [ + "Traceback (most recent call last):", + f" File \"{__file__}\", line {self.callable_line}, in get_exception", + " callable()", + " ~~~~~~~~^^", + f" File \"{__file__}\", line {f.__code__.co_firstlineno + 2}, in f", + " raise ValueError(width)", + ] + self.assertEqual(actual, expected) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_byte_offset_multiline(self): + def f(): + www = 1 + th = 0 + + print(1, www( + th)) + + actual = self.get_exception(f) + expected = [ + "Traceback (most recent call last):", + f" File \"{__file__}\", line {self.callable_line}, in get_exception", + " callable()", + " ~~~~~~~~^^", + f" File \"{__file__}\", line {f.__code__.co_firstlineno + 4}, in f", + f" print(1, www(", + f" ~~~~~~^", + f" th))", + f" ^^^^^", + ] + self.assertEqual(actual, expected) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_byte_offset_with_wide_characters_term_highlight(self): + def f(): + 说明说明 = 1 + şçöğıĤellö = 0 # not wide but still non-ascii + return 说明说明 / şçöğıĤellö + + actual = self.get_exception(f) + expected = [ + f"Traceback (most recent call last):", + f" File \"{__file__}\", line {self.callable_line}, in get_exception", + f" callable()", + f" ~~~~~~~~^^", + f" File \"{__file__}\", line {f.__code__.co_firstlineno + 3}, in f", + f" return 说明说明 / şçöğıĤellö", + f" ~~~~~~~~~^~~~~~~~~~~~", + ] + self.assertEqual(actual, expected) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_byte_offset_with_emojis_term_highlight(self): + def f(): + return "✨🐍" + func_说明说明("📗🚛", + "📗🚛") + "🐍" + + actual = self.get_exception(f) + expected = [ + f"Traceback (most recent call last):", + f" File \"{__file__}\", line {self.callable_line}, in get_exception", + f" callable()", + f" ~~~~~~~~^^", + f" File \"{__file__}\", line {f.__code__.co_firstlineno + 1}, in f", + f' return "✨🐍" + func_说明说明("📗🚛",', + f" ^^^^^^^^^^^^^", + ] + self.assertEqual(actual, expected) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_byte_offset_wide_chars_subscript(self): + def f(): + my_dct = { + "✨🚛✨": { + "说明": { + "🐍🐍🐍": None + } + } + } + return my_dct["✨🚛✨"]["说明"]["🐍"]["说明"]["🐍🐍"] + + actual = self.get_exception(f) + expected = [ + f"Traceback (most recent call last):", + f" File \"{__file__}\", line {self.callable_line}, in get_exception", + f" callable()", + f" ~~~~~~~~^^", + f" File \"{__file__}\", line {f.__code__.co_firstlineno + 8}, in f", + f' return my_dct["✨🚛✨"]["说明"]["🐍"]["说明"]["🐍🐍"]', + f" ~~~~~~~~~~~~~~~~~~~~~~~~^^^^^^", + ] + self.assertEqual(actual, expected) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_memory_error(self): + def f(): + raise MemoryError() + + actual = self.get_exception(f) + expected = ['Traceback (most recent call last):', + f' File "{__file__}", line {self.callable_line}, in get_exception', + ' callable()', + ' ~~~~~~~~^^', + f' File "{__file__}", line {f.__code__.co_firstlineno + 1}, in f', + ' raise MemoryError()'] + self.assertEqual(actual, expected) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_anchors_for_simple_return_statements_are_elided(self): + def g(): + 1/0 + + def f(): + return g() + + result_lines = self.get_exception(f) + expected = ['Traceback (most recent call last):', + f" File \"{__file__}\", line {self.callable_line}, in get_exception", + " callable()", + " ~~~~~~~~^^", + f" File \"{__file__}\", line {f.__code__.co_firstlineno + 1}, in f", + " return g()", + f" File \"{__file__}\", line {g.__code__.co_firstlineno + 1}, in g", + " 1/0", + " ~^~" + ] + self.assertEqual(result_lines, expected) + + def g(): + 1/0 + + def f(): + return g() + 1 + + result_lines = self.get_exception(f) + expected = ['Traceback (most recent call last):', + f" File \"{__file__}\", line {self.callable_line}, in get_exception", + " callable()", + " ~~~~~~~~^^", + f" File \"{__file__}\", line {f.__code__.co_firstlineno + 1}, in f", + " return g() + 1", + " ~^^", + f" File \"{__file__}\", line {g.__code__.co_firstlineno + 1}, in g", + " 1/0", + " ~^~" + ] + self.assertEqual(result_lines, expected) + + def g(*args): + 1/0 + + def f(): + return g(1, + 2, 4, + 5) + + result_lines = self.get_exception(f) + expected = ['Traceback (most recent call last):', + f" File \"{__file__}\", line {self.callable_line}, in get_exception", + " callable()", + " ~~~~~~~~^^", + f" File \"{__file__}\", line {f.__code__.co_firstlineno + 1}, in f", + " return g(1,", + " 2, 4,", + " 5)", + f" File \"{__file__}\", line {g.__code__.co_firstlineno + 1}, in g", + " 1/0", + " ~^~" + ] + self.assertEqual(result_lines, expected) + + def g(*args): + 1/0 + + def f(): + return g(1, + 2, 4, + 5) + 1 + + result_lines = self.get_exception(f) + expected = ['Traceback (most recent call last):', + f" File \"{__file__}\", line {self.callable_line}, in get_exception", + " callable()", + " ~~~~~~~~^^", + f" File \"{__file__}\", line {f.__code__.co_firstlineno + 1}, in f", + " return g(1,", + " ~^^^", + " 2, 4,", + " ^^^^^", + " 5) + 1", + " ^^", + f" File \"{__file__}\", line {g.__code__.co_firstlineno + 1}, in g", + " 1/0", + " ~^~" + ] + self.assertEqual(result_lines, expected) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_anchors_for_simple_assign_statements_are_elided(self): + def g(): + 1/0 + + def f(): + x = g() + + result_lines = self.get_exception(f) + expected = ['Traceback (most recent call last):', + f" File \"{__file__}\", line {self.callable_line}, in get_exception", + " callable()", + " ~~~~~~~~^^", + f" File \"{__file__}\", line {f.__code__.co_firstlineno + 1}, in f", + " x = g()", + f" File \"{__file__}\", line {g.__code__.co_firstlineno + 1}, in g", + " 1/0", + " ~^~" + ] + self.assertEqual(result_lines, expected) + + def g(*args): + 1/0 + + def f(): + x = g(1, + 2, 3, + 4) + + result_lines = self.get_exception(f) + expected = ['Traceback (most recent call last):', + f" File \"{__file__}\", line {self.callable_line}, in get_exception", + " callable()", + " ~~~~~~~~^^", + f" File \"{__file__}\", line {f.__code__.co_firstlineno + 1}, in f", + " x = g(1,", + " 2, 3,", + " 4)", + f" File \"{__file__}\", line {g.__code__.co_firstlineno + 1}, in g", + " 1/0", + " ~^~" + ] + self.assertEqual(result_lines, expected) + + def g(): + 1/0 + + def f(): + x = y = g() + + result_lines = self.get_exception(f) + expected = ['Traceback (most recent call last):', + f" File \"{__file__}\", line {self.callable_line}, in get_exception", + " callable()", + " ~~~~~~~~^^", + f" File \"{__file__}\", line {f.__code__.co_firstlineno + 1}, in f", + " x = y = g()", + " ~^^", + f" File \"{__file__}\", line {g.__code__.co_firstlineno + 1}, in g", + " 1/0", + " ~^~" + ] + self.assertEqual(result_lines, expected) + + def g(*args): + 1/0 + + def f(): + x = y = g(1, + 2, 3, + 4) + + result_lines = self.get_exception(f) + expected = ['Traceback (most recent call last):', + f" File \"{__file__}\", line {self.callable_line}, in get_exception", + " callable()", + " ~~~~~~~~^^", + f" File \"{__file__}\", line {f.__code__.co_firstlineno + 1}, in f", + " x = y = g(1,", + " ~^^^", + " 2, 3,", + " ^^^^^", + " 4)", + " ^^", + f" File \"{__file__}\", line {g.__code__.co_firstlineno + 1}, in g", + " 1/0", + " ~^~" + ] + self.assertEqual(result_lines, expected) + + +# @requires_debug_ranges() # XXX: RUSTPYTHON patch +@force_not_colorized_test_class +class PurePythonTracebackErrorCaretTests( + PurePythonExceptionFormattingMixin, + TracebackErrorLocationCaretTestBase, + unittest.TestCase, +): + """ + Same set of tests as above using the pure Python implementation of + traceback printing in traceback.py. + """ + + +@cpython_only +# @requires_debug_ranges() # XXX: RUSTPYTHON patch +@force_not_colorized_test_class +class CPythonTracebackErrorCaretTests( + CAPIExceptionFormattingMixin, + TracebackErrorLocationCaretTestBase, + unittest.TestCase, +): + """ + Same set of tests as above but with Python's internal traceback printing. + """ + +@cpython_only +# @requires_debug_ranges() # XXX: RUSTPYTHON patch +@force_not_colorized_test_class +class CPythonTracebackLegacyErrorCaretTests( + CAPIExceptionFormattingLegacyMixin, + TracebackErrorLocationCaretTestBase, + unittest.TestCase, +): + """ + Same set of tests as above but with Python's legacy internal traceback printing. + """ + + +class TracebackFormatMixin: + DEBUG_RANGES = True + + def some_exception(self): + raise KeyError('blah') + + def _filter_debug_ranges(self, expected): + return [line for line in expected if not set(line.strip()) <= set("^~")] + + def _maybe_filter_debug_ranges(self, expected): + if not self.DEBUG_RANGES: + return self._filter_debug_ranges(expected) + return expected + + @cpython_only + def check_traceback_format(self, cleanup_func=None): + from _testcapi import traceback_print + try: + self.some_exception() + except KeyError as e: + tb = e.__traceback__ + if cleanup_func is not None: + # Clear the inner frames, not this one + cleanup_func(tb.tb_next) + traceback_fmt = 'Traceback (most recent call last):\n' + \ + ''.join(traceback.format_tb(tb)) + # clear caret lines from traceback_fmt since internal API does + # not emit them + traceback_fmt = "\n".join( + self._filter_debug_ranges(traceback_fmt.splitlines()) + ) + "\n" + file_ = StringIO() + traceback_print(tb, file_) + python_fmt = file_.getvalue() + # Call all _tb and _exc functions + with captured_output("stderr") as tbstderr: + traceback.print_tb(tb) + tbfile = StringIO() + traceback.print_tb(tb, file=tbfile) + with captured_output("stderr") as excstderr: + traceback.print_exc() + excfmt = traceback.format_exc() + excfile = StringIO() + traceback.print_exc(file=excfile) + else: + raise Error("unable to create test traceback string") + + # Make sure that Python and the traceback module format the same thing + self.assertEqual(traceback_fmt, python_fmt) + # Now verify the _tb func output + self.assertEqual(tbstderr.getvalue(), tbfile.getvalue()) + # Now verify the _exc func output + self.assertEqual(excstderr.getvalue(), excfile.getvalue()) + self.assertEqual(excfmt, excfile.getvalue()) + + # Make sure that the traceback is properly indented. + tb_lines = python_fmt.splitlines() + banner = tb_lines[0] + self.assertEqual(len(tb_lines), 5) + location, source_line = tb_lines[-2], tb_lines[-1] + self.assertTrue(banner.startswith('Traceback')) + self.assertTrue(location.startswith(' File')) + self.assertTrue(source_line.startswith(' raise')) + + def test_traceback_format(self): + self.check_traceback_format() + + def test_traceback_format_with_cleared_frames(self): + # Check that traceback formatting also works with a clear()ed frame + def cleanup_tb(tb): + tb.tb_frame.clear() + self.check_traceback_format(cleanup_tb) + + def test_stack_format(self): + # Verify _stack functions. Note we have to use _getframe(1) to + # compare them without this frame appearing in the output + with captured_output("stderr") as ststderr: + traceback.print_stack(sys._getframe(1)) + stfile = StringIO() + traceback.print_stack(sys._getframe(1), file=stfile) + self.assertEqual(ststderr.getvalue(), stfile.getvalue()) + + stfmt = traceback.format_stack(sys._getframe(1)) + + self.assertEqual(ststderr.getvalue(), "".join(stfmt)) + + def test_print_stack(self): + def prn(): + traceback.print_stack() + with captured_output("stderr") as stderr: + prn() + lineno = prn.__code__.co_firstlineno + self.assertEqual(stderr.getvalue().splitlines()[-4:], [ + ' File "%s", line %d, in test_print_stack' % (__file__, lineno+3), + ' prn()', + ' File "%s", line %d, in prn' % (__file__, lineno+1), + ' traceback.print_stack()', + ]) + + # issue 26823 - Shrink recursive tracebacks + def _check_recursive_traceback_display(self, render_exc): + # Always show full diffs when this test fails + # Note that rearranging things may require adjusting + # the relative line numbers in the expected tracebacks + self.maxDiff = None + + # Check hitting the recursion limit + def f(): + f() + + with captured_output("stderr") as stderr_f: + try: + f() + except RecursionError: + render_exc() + else: + self.fail("no recursion occurred") + + lineno_f = f.__code__.co_firstlineno + result_f = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {lineno_f+5}, in _check_recursive_traceback_display\n' + ' f()\n' + ' ~^^\n' + f' File "{__file__}", line {lineno_f+1}, in f\n' + ' f()\n' + ' ~^^\n' + f' File "{__file__}", line {lineno_f+1}, in f\n' + ' f()\n' + ' ~^^\n' + f' File "{__file__}", line {lineno_f+1}, in f\n' + ' f()\n' + ' ~^^\n' + # XXX: The following line changes depending on whether the tests + # are run through the interactive interpreter or with -m + # It also varies depending on the platform (stack size) + # Fortunately, we don't care about exactness here, so we use regex + r' \[Previous line repeated (\d+) more times\]' '\n' + 'RecursionError: maximum recursion depth exceeded\n' + ) + + expected = self._maybe_filter_debug_ranges(result_f.splitlines()) + actual = stderr_f.getvalue().splitlines() + + # Check the output text matches expectations + # 2nd last line contains the repetition count + self.assertEqual(actual[:-2], expected[:-2]) + self.assertRegex(actual[-2], expected[-2]) + # last line can have additional text appended + self.assertIn(expected[-1], actual[-1]) + + # Check the recursion count is roughly as expected + rec_limit = sys.getrecursionlimit() + self.assertIn(int(re.search(r"\d+", actual[-2]).group()), range(rec_limit-60, rec_limit)) + + # Check a known (limited) number of recursive invocations + def g(count=10): + if count: + return g(count-1) + 1 + raise ValueError + + with captured_output("stderr") as stderr_g: + try: + g() + except ValueError: + render_exc() + else: + self.fail("no value error was raised") + + lineno_g = g.__code__.co_firstlineno + result_g = ( + f' File "{__file__}", line {lineno_g+2}, in g\n' + ' return g(count-1) + 1\n' + ' ~^^^^^^^^^\n' + f' File "{__file__}", line {lineno_g+2}, in g\n' + ' return g(count-1) + 1\n' + ' ~^^^^^^^^^\n' + f' File "{__file__}", line {lineno_g+2}, in g\n' + ' return g(count-1) + 1\n' + ' ~^^^^^^^^^\n' + ' [Previous line repeated 7 more times]\n' + f' File "{__file__}", line {lineno_g+3}, in g\n' + ' raise ValueError\n' + 'ValueError\n' + ) + tb_line = ( + 'Traceback (most recent call last):\n' + f' File "{__file__}", line {lineno_g+7}, in _check_recursive_traceback_display\n' + ' g()\n' + ' ~^^\n' + ) + expected = self._maybe_filter_debug_ranges((tb_line + result_g).splitlines()) + actual = stderr_g.getvalue().splitlines() + self.assertEqual(actual, expected) # Check 2 different repetitive sections def h(count=10): @@ -495,6 +2114,7 @@ def h(count=10): 'Traceback (most recent call last):\n' f' File "{__file__}", line {lineno_h+7}, in _check_recursive_traceback_display\n' ' h()\n' + ' ~^^\n' f' File "{__file__}", line {lineno_h+2}, in h\n' ' return h(count-1)\n' f' File "{__file__}", line {lineno_h+2}, in h\n' @@ -504,8 +2124,9 @@ def h(count=10): ' [Previous line repeated 7 more times]\n' f' File "{__file__}", line {lineno_h+3}, in h\n' ' g()\n' + ' ~^^\n' ) - expected = (result_h + result_g).splitlines() + expected = self._maybe_filter_debug_ranges((result_h + result_g).splitlines()) actual = stderr_h.getvalue().splitlines() self.assertEqual(actual, expected) @@ -519,21 +2140,25 @@ def h(count=10): self.fail("no error raised") result_g = ( f' File "{__file__}", line {lineno_g+2}, in g\n' - ' return g(count-1)\n' + ' return g(count-1) + 1\n' + ' ~^^^^^^^^^\n' f' File "{__file__}", line {lineno_g+2}, in g\n' - ' return g(count-1)\n' + ' return g(count-1) + 1\n' + ' ~^^^^^^^^^\n' f' File "{__file__}", line {lineno_g+2}, in g\n' - ' return g(count-1)\n' + ' return g(count-1) + 1\n' + ' ~^^^^^^^^^\n' f' File "{__file__}", line {lineno_g+3}, in g\n' ' raise ValueError\n' 'ValueError\n' ) tb_line = ( 'Traceback (most recent call last):\n' - f' File "{__file__}", line {lineno_g+71}, in _check_recursive_traceback_display\n' + f' File "{__file__}", line {lineno_g+77}, in _check_recursive_traceback_display\n' ' g(traceback._RECURSIVE_CUTOFF)\n' + ' ~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n' ) - expected = (tb_line + result_g).splitlines() + expected = self._maybe_filter_debug_ranges((tb_line + result_g).splitlines()) actual = stderr_g.getvalue().splitlines() self.assertEqual(actual, expected) @@ -547,11 +2172,14 @@ def h(count=10): self.fail("no error raised") result_g = ( f' File "{__file__}", line {lineno_g+2}, in g\n' - ' return g(count-1)\n' + ' return g(count-1) + 1\n' + ' ~^^^^^^^^^\n' f' File "{__file__}", line {lineno_g+2}, in g\n' - ' return g(count-1)\n' + ' return g(count-1) + 1\n' + ' ~^^^^^^^^^\n' f' File "{__file__}", line {lineno_g+2}, in g\n' - ' return g(count-1)\n' + ' return g(count-1) + 1\n' + ' ~^^^^^^^^^\n' ' [Previous line repeated 1 more time]\n' f' File "{__file__}", line {lineno_g+3}, in g\n' ' raise ValueError\n' @@ -559,23 +2187,25 @@ def h(count=10): ) tb_line = ( 'Traceback (most recent call last):\n' - f' File "{__file__}", line {lineno_g+99}, in _check_recursive_traceback_display\n' + f' File "{__file__}", line {lineno_g+109}, in _check_recursive_traceback_display\n' ' g(traceback._RECURSIVE_CUTOFF + 1)\n' + ' ~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^\n' ) - expected = (tb_line + result_g).splitlines() + expected = self._maybe_filter_debug_ranges((tb_line + result_g).splitlines()) actual = stderr_g.getvalue().splitlines() self.assertEqual(actual, expected) - def test_recursive_traceback_python(self): - self._check_recursive_traceback_display(traceback.print_exc) - - @cpython_only - def test_recursive_traceback_cpython_internal(self): - from _testcapi import exception_print - def render_exc(): - exc_type, exc_value, exc_tb = sys.exc_info() - exception_print(exc_value) - self._check_recursive_traceback_display(render_exc) + # TODO: RUSTPYTHON + @unittest.expectedFailure + # @requires_debug_ranges() # XXX: RUSTPYTHON patch + def test_recursive_traceback(self): + if self.DEBUG_RANGES: + self._check_recursive_traceback_display(traceback.print_exc) + else: + from _testcapi import exception_print + def render_exc(): + exception_print(sys.exception()) + self._check_recursive_traceback_display(render_exc) def test_format_stack(self): def fmt(): @@ -604,8 +2234,8 @@ def __eq__(self, other): except UnhashableException: try: raise ex1 - except UnhashableException: - exc_type, exc_val, exc_tb = sys.exc_info() + except UnhashableException as e: + exc_val = e with captured_output("stderr") as stderr_f: exception_print(exc_val) @@ -616,6 +2246,53 @@ def __eq__(self, other): self.assertIn('UnhashableException: ex2', tb[3]) self.assertIn('UnhashableException: ex1', tb[10]) + def deep_eg(self): + e = TypeError(1) + for i in range(2000): + e = ExceptionGroup('eg', [e]) + return e + + @cpython_only + def test_exception_group_deep_recursion_capi(self): + from _testcapi import exception_print + LIMIT = 75 + eg = self.deep_eg() + with captured_output("stderr") as stderr_f: + with support.infinite_recursion(max_depth=LIMIT): + exception_print(eg) + output = stderr_f.getvalue() + self.assertIn('ExceptionGroup', output) + self.assertLessEqual(output.count('ExceptionGroup'), LIMIT) + + def test_exception_group_deep_recursion_traceback(self): + LIMIT = 75 + eg = self.deep_eg() + with captured_output("stderr") as stderr_f: + with support.infinite_recursion(max_depth=LIMIT): + traceback.print_exception(type(eg), eg, eg.__traceback__) + output = stderr_f.getvalue() + self.assertIn('ExceptionGroup', output) + self.assertLessEqual(output.count('ExceptionGroup'), LIMIT) + + @cpython_only + def test_print_exception_bad_type_capi(self): + from _testcapi import exception_print + with captured_output("stderr") as stderr: + with support.catch_unraisable_exception(): + exception_print(42) + self.assertEqual( + stderr.getvalue(), + ('TypeError: print_exception(): ' + 'Exception expected for value, int found\n') + ) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_print_exception_bad_type_python(self): + msg = "Exception expected for value, int found" + with self.assertRaisesRegex(TypeError, msg): + traceback.print_exception(42) + cause_message = ( "\nThe above exception was the direct cause " @@ -628,24 +2305,49 @@ def __eq__(self, other): boundaries = re.compile( '(%s|%s)' % (re.escape(cause_message), re.escape(context_message))) +@force_not_colorized_test_class +class TestTracebackFormat(unittest.TestCase, TracebackFormatMixin): + pass + +@cpython_only +@force_not_colorized_test_class +class TestFallbackTracebackFormat(unittest.TestCase, TracebackFormatMixin): + DEBUG_RANGES = False + def setUp(self) -> None: + self.original_unraisable_hook = sys.unraisablehook + sys.unraisablehook = lambda *args: None + self.original_hook = traceback._print_exception_bltin + traceback._print_exception_bltin = lambda *args: 1/0 + return super().setUp() + + def tearDown(self) -> None: + traceback._print_exception_bltin = self.original_hook + sys.unraisablehook = self.original_unraisable_hook + return super().tearDown() class BaseExceptionReportingTests: def get_exception(self, exception_or_callable): - if isinstance(exception_or_callable, Exception): + if isinstance(exception_or_callable, BaseException): return exception_or_callable try: exception_or_callable() except Exception as e: return e + callable_line = get_exception.__code__.co_firstlineno + 4 + def zero_div(self): 1/0 # In zero_div def check_zero_div(self, msg): lines = msg.splitlines() - self.assertTrue(lines[-3].startswith(' File')) - self.assertIn('1/0 # In zero_div', lines[-2]) + if has_no_debug_ranges(): + self.assertTrue(lines[-3].startswith(' File')) + self.assertIn('1/0 # In zero_div', lines[-2]) + else: + self.assertTrue(lines[-4].startswith(' File')) + self.assertIn('1/0 # In zero_div', lines[-3]) self.assertTrue(lines[-1].startswith('ZeroDivisionError'), lines[-1]) def test_simple(self): @@ -654,12 +2356,18 @@ def test_simple(self): except ZeroDivisionError as _: e = _ lines = self.get_report(e).splitlines() - self.assertEqual(len(lines), 4) + if has_no_debug_ranges(): + self.assertEqual(len(lines), 4) + self.assertTrue(lines[3].startswith('ZeroDivisionError')) + else: + self.assertEqual(len(lines), 5) + self.assertTrue(lines[4].startswith('ZeroDivisionError')) self.assertTrue(lines[0].startswith('Traceback')) self.assertTrue(lines[1].startswith(' File')) self.assertIn('1/0 # Marker', lines[2]) - self.assertTrue(lines[3].startswith('ZeroDivisionError')) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_cause(self): def inner_raise(): try: @@ -692,16 +2400,16 @@ def test_context_suppression(self): try: try: raise Exception - except: + except Exception: raise ZeroDivisionError from None except ZeroDivisionError as _: e = _ lines = self.get_report(e).splitlines() self.assertEqual(len(lines), 4) + self.assertTrue(lines[3].startswith('ZeroDivisionError')) self.assertTrue(lines[0].startswith('Traceback')) self.assertTrue(lines[1].startswith(' File')) self.assertIn('ZeroDivisionError from None', lines[2]) - self.assertTrue(lines[3].startswith('ZeroDivisionError')) def test_cause_and_context(self): # When both a cause and a context are set, only the cause should be @@ -793,50 +2501,661 @@ def test_message_none(self): err = self.get_report(Exception('')) self.assertIn('Exception\n', err) - def test_exception_modulename_not_unicode(self): - class X(Exception): - def __str__(self): - return "I am X" - - X.__module__ = 42 - - err = self.get_report(X()) - exp = f'<unknown>.{X.__qualname__}: I am X\n' - self.assertEqual(exp, err) - + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_syntax_error_various_offsets(self): for offset in range(-5, 10): for add in [0, 2]: - text = " "*add + "text%d" % offset + text = " " * add + "text%d" % offset expected = [' File "file.py", line 1'] if offset < 1: expected.append(" %s" % text.lstrip()) elif offset <= 6: expected.append(" %s" % text.lstrip()) - expected.append(" %s^" % (" "*(offset-1))) + # Set the caret length to match the length of the text minus the offset. + caret_length = max(1, len(text.lstrip()) - offset + 1) + expected.append(" %s%s" % (" " * (offset - 1), "^" * caret_length)) else: + caret_length = max(1, len(text.lstrip()) - 4) expected.append(" %s" % text.lstrip()) - expected.append(" %s^" % (" "*5)) + expected.append(" %s%s" % (" " * 5, "^" * caret_length)) expected.append("SyntaxError: msg") expected.append("") - err = self.get_report(SyntaxError("msg", ("file.py", 1, offset+add, text))) + err = self.get_report(SyntaxError("msg", ("file.py", 1, offset + add, text))) exp = "\n".join(expected) self.assertEqual(exp, err) - def test_format_exception_only_qualname(self): + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_exception_with_note(self): + e = ValueError(123) + vanilla = self.get_report(e) + + e.add_note('My Note') + self.assertEqual(self.get_report(e), vanilla + 'My Note\n') + + del e.__notes__ + e.add_note('') + self.assertEqual(self.get_report(e), vanilla + '\n') + + del e.__notes__ + e.add_note('Your Note') + self.assertEqual(self.get_report(e), vanilla + 'Your Note\n') + + del e.__notes__ + self.assertEqual(self.get_report(e), vanilla) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_exception_with_invalid_notes(self): + e = ValueError(123) + vanilla = self.get_report(e) + + # non-sequence __notes__ + class BadThing: + def __str__(self): + return 'bad str' + + def __repr__(self): + return 'bad repr' + + # unprintable, non-sequence __notes__ + class Unprintable: + def __repr__(self): + raise ValueError('bad value') + + e.__notes__ = BadThing() + notes_repr = 'bad repr' + self.assertEqual(self.get_report(e), vanilla + notes_repr + '\n') + + e.__notes__ = Unprintable() + err_msg = '<__notes__ repr() failed>' + self.assertEqual(self.get_report(e), vanilla + err_msg + '\n') + + # non-string item in the __notes__ sequence + e.__notes__ = [BadThing(), 'Final Note'] + bad_note = 'bad str' + self.assertEqual(self.get_report(e), vanilla + bad_note + '\nFinal Note\n') + + # unprintable, non-string item in the __notes__ sequence + e.__notes__ = [Unprintable(), 'Final Note'] + err_msg = '<note str() failed>' + self.assertEqual(self.get_report(e), vanilla + err_msg + '\nFinal Note\n') + + e.__notes__ = "please do not explode me" + err_msg = "'please do not explode me'" + self.assertEqual(self.get_report(e), vanilla + err_msg + '\n') + + e.__notes__ = b"please do not show me as numbers" + err_msg = "b'please do not show me as numbers'" + self.assertEqual(self.get_report(e), vanilla + err_msg + '\n') + + # an exception with a broken __getattr__ raising a non expected error + class BrokenException(Exception): + broken = False + def __getattr__(self, name): + if self.broken: + raise ValueError(f'no {name}') + + e = BrokenException(123) + vanilla = self.get_report(e) + e.broken = True + self.assertEqual( + self.get_report(e), + vanilla + "Ignored error getting __notes__: ValueError('no __notes__')\n") + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_exception_with_multiple_notes(self): + for e in [ValueError(42), SyntaxError('bad syntax')]: + with self.subTest(e=e): + vanilla = self.get_report(e) + + e.add_note('Note 1') + e.add_note('Note 2') + e.add_note('Note 3') + + self.assertEqual( + self.get_report(e), + vanilla + 'Note 1\n' + 'Note 2\n' + 'Note 3\n') + + del e.__notes__ + e.add_note('Note 4') + del e.__notes__ + e.add_note('Note 5') + e.add_note('Note 6') + + self.assertEqual( + self.get_report(e), + vanilla + 'Note 5\n' + 'Note 6\n') + + def test_exception_qualname(self): class A: class B: class X(Exception): def __str__(self): return "I am X" - pass + err = self.get_report(A.B.X()) str_value = 'I am X' str_name = '.'.join([A.B.X.__module__, A.B.X.__qualname__]) exp = "%s: %s\n" % (str_name, str_value) + self.assertEqual(exp, MODULE_PREFIX + err) + + def test_exception_modulename(self): + class X(Exception): + def __str__(self): + return "I am X" + + for modulename in '__main__', 'builtins', 'some_module': + X.__module__ = modulename + with self.subTest(modulename=modulename): + err = self.get_report(X()) + str_value = 'I am X' + if modulename in ['builtins', '__main__']: + str_name = X.__qualname__ + else: + str_name = '.'.join([X.__module__, X.__qualname__]) + exp = "%s: %s\n" % (str_name, str_value) + self.assertEqual(exp, err) + + def test_exception_angle_bracketed_filename(self): + src = textwrap.dedent(""" + try: + raise ValueError(42) + except Exception as e: + exc = e + """) + + code = compile(src, "<does not exist>", "exec") + g, l = {}, {} + exec(code, g, l) + err = self.get_report(l['exc']) + exp = ' File "<does not exist>", line 3, in <module>\nValueError: 42\n' + self.assertIn(exp, err) + + def test_exception_modulename_not_unicode(self): + class X(Exception): + def __str__(self): + return "I am X" + + X.__module__ = 42 + + err = self.get_report(X()) + exp = f'<unknown>.{X.__qualname__}: I am X\n' self.assertEqual(exp, err) + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_exception_bad__str__(self): + class X(Exception): + def __str__(self): + 1/0 + err = self.get_report(X()) + str_value = '<exception str() failed>' + str_name = '.'.join([X.__module__, X.__qualname__]) + self.assertEqual(MODULE_PREFIX + err, f"{str_name}: {str_value}\n") + + + # #### Exception Groups #### + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_exception_group_basic(self): + def exc(): + raise ExceptionGroup("eg", [ValueError(1), TypeError(2)]) + + expected = ( + f' + Exception Group Traceback (most recent call last):\n' + f' | File "{__file__}", line {self.callable_line}, in get_exception\n' + f' | exception_or_callable()\n' + f' | ~~~~~~~~~~~~~~~~~~~~~^^\n' + f' | File "{__file__}", line {exc.__code__.co_firstlineno + 1}, in exc\n' + f' | raise ExceptionGroup("eg", [ValueError(1), TypeError(2)])\n' + f' | ExceptionGroup: eg (2 sub-exceptions)\n' + f' +-+---------------- 1 ----------------\n' + f' | ValueError: 1\n' + f' +---------------- 2 ----------------\n' + f' | TypeError: 2\n' + f' +------------------------------------\n') + + report = self.get_report(exc) + self.assertEqual(report, expected) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_exception_group_cause(self): + def exc(): + EG = ExceptionGroup + try: + raise EG("eg1", [ValueError(1), TypeError(2)]) + except Exception as e: + raise EG("eg2", [ValueError(3), TypeError(4)]) from e + + expected = (f' + Exception Group Traceback (most recent call last):\n' + f' | File "{__file__}", line {exc.__code__.co_firstlineno + 3}, in exc\n' + f' | raise EG("eg1", [ValueError(1), TypeError(2)])\n' + f' | ExceptionGroup: eg1 (2 sub-exceptions)\n' + f' +-+---------------- 1 ----------------\n' + f' | ValueError: 1\n' + f' +---------------- 2 ----------------\n' + f' | TypeError: 2\n' + f' +------------------------------------\n' + f'\n' + f'The above exception was the direct cause of the following exception:\n' + f'\n' + f' + Exception Group Traceback (most recent call last):\n' + f' | File "{__file__}", line {self.callable_line}, in get_exception\n' + f' | exception_or_callable()\n' + f' | ~~~~~~~~~~~~~~~~~~~~~^^\n' + f' | File "{__file__}", line {exc.__code__.co_firstlineno + 5}, in exc\n' + f' | raise EG("eg2", [ValueError(3), TypeError(4)]) from e\n' + f' | ExceptionGroup: eg2 (2 sub-exceptions)\n' + f' +-+---------------- 1 ----------------\n' + f' | ValueError: 3\n' + f' +---------------- 2 ----------------\n' + f' | TypeError: 4\n' + f' +------------------------------------\n') + + report = self.get_report(exc) + self.assertEqual(report, expected) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_exception_group_context_with_context(self): + def exc(): + EG = ExceptionGroup + try: + try: + raise EG("eg1", [ValueError(1), TypeError(2)]) + except EG: + raise EG("eg2", [ValueError(3), TypeError(4)]) + except EG: + raise ImportError(5) + + expected = ( + f' + Exception Group Traceback (most recent call last):\n' + f' | File "{__file__}", line {exc.__code__.co_firstlineno + 4}, in exc\n' + f' | raise EG("eg1", [ValueError(1), TypeError(2)])\n' + f' | ExceptionGroup: eg1 (2 sub-exceptions)\n' + f' +-+---------------- 1 ----------------\n' + f' | ValueError: 1\n' + f' +---------------- 2 ----------------\n' + f' | TypeError: 2\n' + f' +------------------------------------\n' + f'\n' + f'During handling of the above exception, another exception occurred:\n' + f'\n' + f' + Exception Group Traceback (most recent call last):\n' + f' | File "{__file__}", line {exc.__code__.co_firstlineno + 6}, in exc\n' + f' | raise EG("eg2", [ValueError(3), TypeError(4)])\n' + f' | ExceptionGroup: eg2 (2 sub-exceptions)\n' + f' +-+---------------- 1 ----------------\n' + f' | ValueError: 3\n' + f' +---------------- 2 ----------------\n' + f' | TypeError: 4\n' + f' +------------------------------------\n' + f'\n' + f'During handling of the above exception, another exception occurred:\n' + f'\n' + f'Traceback (most recent call last):\n' + f' File "{__file__}", line {self.callable_line}, in get_exception\n' + f' exception_or_callable()\n' + f' ~~~~~~~~~~~~~~~~~~~~~^^\n' + f' File "{__file__}", line {exc.__code__.co_firstlineno + 8}, in exc\n' + f' raise ImportError(5)\n' + f'ImportError: 5\n') + + report = self.get_report(exc) + self.assertEqual(report, expected) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_exception_group_nested(self): + def exc(): + EG = ExceptionGroup + VE = ValueError + TE = TypeError + try: + try: + raise EG("nested", [TE(2), TE(3)]) + except Exception as e: + exc = e + raise EG("eg", [VE(1), exc, VE(4)]) + except EG: + raise EG("top", [VE(5)]) + + expected = (f' + Exception Group Traceback (most recent call last):\n' + f' | File "{__file__}", line {exc.__code__.co_firstlineno + 9}, in exc\n' + f' | raise EG("eg", [VE(1), exc, VE(4)])\n' + f' | ExceptionGroup: eg (3 sub-exceptions)\n' + f' +-+---------------- 1 ----------------\n' + f' | ValueError: 1\n' + f' +---------------- 2 ----------------\n' + f' | Exception Group Traceback (most recent call last):\n' + f' | File "{__file__}", line {exc.__code__.co_firstlineno + 6}, in exc\n' + f' | raise EG("nested", [TE(2), TE(3)])\n' + f' | ExceptionGroup: nested (2 sub-exceptions)\n' + f' +-+---------------- 1 ----------------\n' + f' | TypeError: 2\n' + f' +---------------- 2 ----------------\n' + f' | TypeError: 3\n' + f' +------------------------------------\n' + f' +---------------- 3 ----------------\n' + f' | ValueError: 4\n' + f' +------------------------------------\n' + f'\n' + f'During handling of the above exception, another exception occurred:\n' + f'\n' + f' + Exception Group Traceback (most recent call last):\n' + f' | File "{__file__}", line {self.callable_line}, in get_exception\n' + f' | exception_or_callable()\n' + f' | ~~~~~~~~~~~~~~~~~~~~~^^\n' + f' | File "{__file__}", line {exc.__code__.co_firstlineno + 11}, in exc\n' + f' | raise EG("top", [VE(5)])\n' + f' | ExceptionGroup: top (1 sub-exception)\n' + f' +-+---------------- 1 ----------------\n' + f' | ValueError: 5\n' + f' +------------------------------------\n') + + report = self.get_report(exc) + self.assertEqual(report, expected) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_exception_group_width_limit(self): + excs = [] + for i in range(1000): + excs.append(ValueError(i)) + eg = ExceptionGroup('eg', excs) + + expected = (' | ExceptionGroup: eg (1000 sub-exceptions)\n' + ' +-+---------------- 1 ----------------\n' + ' | ValueError: 0\n' + ' +---------------- 2 ----------------\n' + ' | ValueError: 1\n' + ' +---------------- 3 ----------------\n' + ' | ValueError: 2\n' + ' +---------------- 4 ----------------\n' + ' | ValueError: 3\n' + ' +---------------- 5 ----------------\n' + ' | ValueError: 4\n' + ' +---------------- 6 ----------------\n' + ' | ValueError: 5\n' + ' +---------------- 7 ----------------\n' + ' | ValueError: 6\n' + ' +---------------- 8 ----------------\n' + ' | ValueError: 7\n' + ' +---------------- 9 ----------------\n' + ' | ValueError: 8\n' + ' +---------------- 10 ----------------\n' + ' | ValueError: 9\n' + ' +---------------- 11 ----------------\n' + ' | ValueError: 10\n' + ' +---------------- 12 ----------------\n' + ' | ValueError: 11\n' + ' +---------------- 13 ----------------\n' + ' | ValueError: 12\n' + ' +---------------- 14 ----------------\n' + ' | ValueError: 13\n' + ' +---------------- 15 ----------------\n' + ' | ValueError: 14\n' + ' +---------------- ... ----------------\n' + ' | and 985 more exceptions\n' + ' +------------------------------------\n') + + report = self.get_report(eg) + self.assertEqual(report, expected) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_exception_group_depth_limit(self): + exc = TypeError('bad type') + for i in range(1000): + exc = ExceptionGroup( + f'eg{i}', + [ValueError(i), exc, ValueError(-i)]) + + expected = (' | ExceptionGroup: eg999 (3 sub-exceptions)\n' + ' +-+---------------- 1 ----------------\n' + ' | ValueError: 999\n' + ' +---------------- 2 ----------------\n' + ' | ExceptionGroup: eg998 (3 sub-exceptions)\n' + ' +-+---------------- 1 ----------------\n' + ' | ValueError: 998\n' + ' +---------------- 2 ----------------\n' + ' | ExceptionGroup: eg997 (3 sub-exceptions)\n' + ' +-+---------------- 1 ----------------\n' + ' | ValueError: 997\n' + ' +---------------- 2 ----------------\n' + ' | ExceptionGroup: eg996 (3 sub-exceptions)\n' + ' +-+---------------- 1 ----------------\n' + ' | ValueError: 996\n' + ' +---------------- 2 ----------------\n' + ' | ExceptionGroup: eg995 (3 sub-exceptions)\n' + ' +-+---------------- 1 ----------------\n' + ' | ValueError: 995\n' + ' +---------------- 2 ----------------\n' + ' | ExceptionGroup: eg994 (3 sub-exceptions)\n' + ' +-+---------------- 1 ----------------\n' + ' | ValueError: 994\n' + ' +---------------- 2 ----------------\n' + ' | ExceptionGroup: eg993 (3 sub-exceptions)\n' + ' +-+---------------- 1 ----------------\n' + ' | ValueError: 993\n' + ' +---------------- 2 ----------------\n' + ' | ExceptionGroup: eg992 (3 sub-exceptions)\n' + ' +-+---------------- 1 ----------------\n' + ' | ValueError: 992\n' + ' +---------------- 2 ----------------\n' + ' | ExceptionGroup: eg991 (3 sub-exceptions)\n' + ' +-+---------------- 1 ----------------\n' + ' | ValueError: 991\n' + ' +---------------- 2 ----------------\n' + ' | ExceptionGroup: eg990 (3 sub-exceptions)\n' + ' +-+---------------- 1 ----------------\n' + ' | ValueError: 990\n' + ' +---------------- 2 ----------------\n' + ' | ... (max_group_depth is 10)\n' + ' +---------------- 3 ----------------\n' + ' | ValueError: -990\n' + ' +------------------------------------\n' + ' +---------------- 3 ----------------\n' + ' | ValueError: -991\n' + ' +------------------------------------\n' + ' +---------------- 3 ----------------\n' + ' | ValueError: -992\n' + ' +------------------------------------\n' + ' +---------------- 3 ----------------\n' + ' | ValueError: -993\n' + ' +------------------------------------\n' + ' +---------------- 3 ----------------\n' + ' | ValueError: -994\n' + ' +------------------------------------\n' + ' +---------------- 3 ----------------\n' + ' | ValueError: -995\n' + ' +------------------------------------\n' + ' +---------------- 3 ----------------\n' + ' | ValueError: -996\n' + ' +------------------------------------\n' + ' +---------------- 3 ----------------\n' + ' | ValueError: -997\n' + ' +------------------------------------\n' + ' +---------------- 3 ----------------\n' + ' | ValueError: -998\n' + ' +------------------------------------\n' + ' +---------------- 3 ----------------\n' + ' | ValueError: -999\n' + ' +------------------------------------\n') + + report = self.get_report(exc) + self.assertEqual(report, expected) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_exception_group_with_notes(self): + def exc(): + try: + excs = [] + for msg in ['bad value', 'terrible value']: + try: + raise ValueError(msg) + except ValueError as e: + e.add_note(f'the {msg}') + excs.append(e) + raise ExceptionGroup("nested", excs) + except ExceptionGroup as e: + e.add_note(('>> Multi line note\n' + '>> Because I am such\n' + '>> an important exception.\n' + '>> empty lines work too\n' + '\n' + '(that was an empty line)')) + raise + + expected = (f' + Exception Group Traceback (most recent call last):\n' + f' | File "{__file__}", line {self.callable_line}, in get_exception\n' + f' | exception_or_callable()\n' + f' | ~~~~~~~~~~~~~~~~~~~~~^^\n' + f' | File "{__file__}", line {exc.__code__.co_firstlineno + 9}, in exc\n' + f' | raise ExceptionGroup("nested", excs)\n' + f' | ExceptionGroup: nested (2 sub-exceptions)\n' + f' | >> Multi line note\n' + f' | >> Because I am such\n' + f' | >> an important exception.\n' + f' | >> empty lines work too\n' + f' | \n' + f' | (that was an empty line)\n' + f' +-+---------------- 1 ----------------\n' + f' | Traceback (most recent call last):\n' + f' | File "{__file__}", line {exc.__code__.co_firstlineno + 5}, in exc\n' + f' | raise ValueError(msg)\n' + f' | ValueError: bad value\n' + f' | the bad value\n' + f' +---------------- 2 ----------------\n' + f' | Traceback (most recent call last):\n' + f' | File "{__file__}", line {exc.__code__.co_firstlineno + 5}, in exc\n' + f' | raise ValueError(msg)\n' + f' | ValueError: terrible value\n' + f' | the terrible value\n' + f' +------------------------------------\n') + + report = self.get_report(exc) + self.assertEqual(report, expected) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_exception_group_with_multiple_notes(self): + def exc(): + try: + excs = [] + for msg in ['bad value', 'terrible value']: + try: + raise ValueError(msg) + except ValueError as e: + e.add_note(f'the {msg}') + e.add_note(f'Goodbye {msg}') + excs.append(e) + raise ExceptionGroup("nested", excs) + except ExceptionGroup as e: + e.add_note(('>> Multi line note\n' + '>> Because I am such\n' + '>> an important exception.\n' + '>> empty lines work too\n' + '\n' + '(that was an empty line)')) + e.add_note('Goodbye!') + raise + + expected = (f' + Exception Group Traceback (most recent call last):\n' + f' | File "{__file__}", line {self.callable_line}, in get_exception\n' + f' | exception_or_callable()\n' + f' | ~~~~~~~~~~~~~~~~~~~~~^^\n' + f' | File "{__file__}", line {exc.__code__.co_firstlineno + 10}, in exc\n' + f' | raise ExceptionGroup("nested", excs)\n' + f' | ExceptionGroup: nested (2 sub-exceptions)\n' + f' | >> Multi line note\n' + f' | >> Because I am such\n' + f' | >> an important exception.\n' + f' | >> empty lines work too\n' + f' | \n' + f' | (that was an empty line)\n' + f' | Goodbye!\n' + f' +-+---------------- 1 ----------------\n' + f' | Traceback (most recent call last):\n' + f' | File "{__file__}", line {exc.__code__.co_firstlineno + 5}, in exc\n' + f' | raise ValueError(msg)\n' + f' | ValueError: bad value\n' + f' | the bad value\n' + f' | Goodbye bad value\n' + f' +---------------- 2 ----------------\n' + f' | Traceback (most recent call last):\n' + f' | File "{__file__}", line {exc.__code__.co_firstlineno + 5}, in exc\n' + f' | raise ValueError(msg)\n' + f' | ValueError: terrible value\n' + f' | the terrible value\n' + f' | Goodbye terrible value\n' + f' +------------------------------------\n') + + report = self.get_report(exc) + self.assertEqual(report, expected) + + # TODO: RUSTPYTHON + ''' + def test_exception_group_wrapped_naked(self): + # See gh-128799 + + def exc(): + try: + raise Exception(42) + except* Exception as e: + raise + expected = (f' + Exception Group Traceback (most recent call last):\n' + f' | File "{__file__}", line {self.callable_line}, in get_exception\n' + f' | exception_or_callable()\n' + f' | ~~~~~~~~~~~~~~~~~~~~~^^\n' + f' | File "{__file__}", line {exc.__code__.co_firstlineno + 3}, in exc\n' + f' | except* Exception as e:\n' + f' | raise\n' + f' | ExceptionGroup: (1 sub-exception)\n' + f' +-+---------------- 1 ----------------\n' + f' | Traceback (most recent call last):\n' + f' | File "{__file__}", line {exc.__code__.co_firstlineno + 2}, in exc\n' + f' | raise Exception(42)\n' + f' | Exception: 42\n' + f' +------------------------------------\n') + + report = self.get_report(exc) + self.assertEqual(report, expected) + + def test_KeyboardInterrupt_at_first_line_of_frame(self): + # see GH-93249 + def f(): + return sys._getframe() + + tb_next = None + frame = f() + lasti = 0 + lineno = f.__code__.co_firstlineno + tb = types.TracebackType(tb_next, frame, lasti, lineno) + + exc = KeyboardInterrupt() + exc.__traceback__ = tb + + expected = (f'Traceback (most recent call last):\n' + f' File "{__file__}", line {lineno}, in f\n' + f' def f():\n' + f'\n' + f'KeyboardInterrupt\n') + + report = self.get_report(exc) + # remove trailing writespace: + report = '\n'.join([l.rstrip() for l in report.split('\n')]) + self.assertEqual(report, expected) + ''' + +@force_not_colorized_test_class class PyExcReportingTests(BaseExceptionReportingTests, unittest.TestCase): # # This checks reporting through the 'traceback' module, with both @@ -853,6 +3172,7 @@ def get_report(self, e): return s +@force_not_colorized_test_class class CExcReportingTests(BaseExceptionReportingTests, unittest.TestCase): # # This checks built-in reporting by the interpreter. @@ -935,8 +3255,8 @@ def assertEqualExcept(actual, expected, ignore): def test_extract_tb(self): try: self.last_raises5() - except Exception: - exc_type, exc_value, tb = sys.exc_info() + except Exception as e: + tb = e.__traceback__ def extract(**kwargs): return traceback.extract_tb(tb, **kwargs) @@ -962,12 +3282,12 @@ def extract(**kwargs): def test_format_exception(self): try: self.last_raises5() - except Exception: - exc_type, exc_value, tb = sys.exc_info() + except Exception as e: + exc = e # [1:-1] to exclude "Traceback (...)" header and # exception type and value def extract(**kwargs): - return traceback.format_exception(exc_type, exc_value, tb, **kwargs)[1:-1] + return traceback.format_exception(exc, **kwargs)[1:-1] with support.swap_attr(sys, 'tracebacklimit', 1000): nolim = extract() @@ -1007,8 +3327,8 @@ def inner(): try: outer() - except: - type_, value, tb = sys.exc_info() + except BaseException as e: + tb = e.__traceback__ # Initial assertion: there's one local in the inner frame. inner_frame = tb.tb_next.tb_next.tb_next.tb_frame @@ -1051,10 +3371,12 @@ def test_basics(self): self.assertNotEqual(f, object()) self.assertEqual(f, ALWAYS_EQ) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_lazy_lines(self): linecache.clearcache() f = traceback.FrameSummary("f", 1, "dummy", lookup_line=False) - self.assertEqual(None, f._line) + self.assertEqual(None, f._lines) linecache.lazycache("f", globals()) self.assertEqual( '"""Test cases for traceback module"""', @@ -1086,8 +3408,8 @@ def deeper(): def test_walk_tb(self): try: 1/0 - except Exception: - _, _, tb = sys.exc_info() + except Exception as e: + tb = e.__traceback__ s = list(traceback.walk_tb(tb)) self.assertEqual(len(s), 1) @@ -1171,23 +3493,145 @@ def some_inner(k, v): ' v = 4\n' % (__file__, some_inner.__code__.co_firstlineno + 3) ], s.format()) -class TestTracebackException(unittest.TestCase): + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_custom_format_frame(self): + class CustomStackSummary(traceback.StackSummary): + def format_frame_summary(self, frame_summary, colorize=False): + return f'{frame_summary.filename}:{frame_summary.lineno}' - def test_smoke(self): - try: + def some_inner(): + return CustomStackSummary.extract( + traceback.walk_stack(None), limit=1) + + s = some_inner() + self.assertEqual( + s.format(), + [f'{__file__}:{some_inner.__code__.co_firstlineno + 1}']) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_dropping_frames(self): + def f(): 1/0 - except Exception: - exc_info = sys.exc_info() - exc = traceback.TracebackException(*exc_info) + + def g(): + try: + f() + except Exception as e: + return e.__traceback__ + + tb = g() + + class Skip_G(traceback.StackSummary): + def format_frame_summary(self, frame_summary, colorize=False): + if frame_summary.name == 'g': + return None + return super().format_frame_summary(frame_summary) + + stack = Skip_G.extract( + traceback.walk_tb(tb)).format() + + self.assertEqual(len(stack), 1) + lno = f.__code__.co_firstlineno + 1 + self.assertEqual( + stack[0], + f' File "{__file__}", line {lno}, in f\n 1/0\n' + ) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_summary_should_show_carets(self): + # See: https://github.com/python/cpython/issues/122353 + + # statement to execute and to get a ZeroDivisionError for a traceback + statement = "abcdef = 1 / 0 and 2.0" + colno = statement.index('1 / 0') + end_colno = colno + len('1 / 0') + + # Actual line to use when rendering the traceback + # and whose AST will be extracted (it will be empty). + cached_line = '# this line will be used during rendering' + self.addCleanup(unlink, TESTFN) + with open(TESTFN, "w") as file: + file.write(cached_line) + linecache.updatecache(TESTFN, {}) + + try: + exec(compile(statement, TESTFN, "exec")) + except ZeroDivisionError as exc: + # This is the simplest way to create a StackSummary + # whose FrameSummary items have their column offsets. + s = traceback.TracebackException.from_exception(exc).stack + self.assertIsInstance(s, traceback.StackSummary) + with unittest.mock.patch.object(s, '_should_show_carets', + wraps=s._should_show_carets) as ff: + self.assertEqual(len(s), 2) + self.assertListEqual( + s.format_frame_summary(s[1]).splitlines(), + [ + f' File "{TESTFN}", line 1, in <module>', + f' {cached_line}' + ] + ) + ff.assert_called_with(colno, end_colno, [cached_line], None) + +class Unrepresentable: + def __repr__(self) -> str: + raise Exception("Unrepresentable") + + +# Used in test_dont_swallow_cause_or_context_of_falsey_exception and +# test_dont_swallow_subexceptions_of_falsey_exceptiongroup. +class FalseyException(Exception): + def __bool__(self): + return False + + +class FalseyExceptionGroup(ExceptionGroup): + def __bool__(self): + return False + + +class TestTracebackException(unittest.TestCase): + def do_test_smoke(self, exc, expected_type_str): + try: + raise exc + except Exception as e: + exc_obj = e + exc = traceback.TracebackException.from_exception(e) expected_stack = traceback.StackSummary.extract( - traceback.walk_tb(exc_info[2])) + traceback.walk_tb(e.__traceback__)) self.assertEqual(None, exc.__cause__) self.assertEqual(None, exc.__context__) self.assertEqual(False, exc.__suppress_context__) self.assertEqual(expected_stack, exc.stack) - self.assertEqual(exc_info[0], exc.exc_type) - self.assertEqual(str(exc_info[1]), str(exc)) + with self.assertWarns(DeprecationWarning): + self.assertEqual(type(exc_obj), exc.exc_type) + self.assertEqual(expected_type_str, exc.exc_type_str) + self.assertEqual(str(exc_obj), str(exc)) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_smoke_builtin(self): + self.do_test_smoke(ValueError(42), 'ValueError') + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_smoke_user_exception(self): + class MyException(Exception): + pass + + if __name__ == '__main__': + expected = ('TestTracebackException.' + 'test_smoke_user_exception.<locals>.MyException') + else: + expected = ('test.test_traceback.TestTracebackException.' + 'test_smoke_user_exception.<locals>.MyException') + self.do_test_smoke(MyException('bad things happened'), expected) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_from_exception(self): # Check all the parameters are accepted. def foo(): @@ -1195,9 +3639,10 @@ def foo(): try: foo() except Exception as e: - exc_info = sys.exc_info() + exc_obj = e + tb = e.__traceback__ self.expected_stack = traceback.StackSummary.extract( - traceback.walk_tb(exc_info[2]), limit=1, lookup_lines=False, + traceback.walk_tb(tb), limit=1, lookup_lines=False, capture_locals=True) self.exc = traceback.TracebackException.from_exception( e, limit=1, lookup_lines=False, capture_locals=True) @@ -1207,50 +3652,60 @@ def foo(): self.assertEqual(None, exc.__context__) self.assertEqual(False, exc.__suppress_context__) self.assertEqual(expected_stack, exc.stack) - self.assertEqual(exc_info[0], exc.exc_type) - self.assertEqual(str(exc_info[1]), str(exc)) + with self.assertWarns(DeprecationWarning): + self.assertEqual(type(exc_obj), exc.exc_type) + self.assertEqual(type(exc_obj).__name__, exc.exc_type_str) + self.assertEqual(str(exc_obj), str(exc)) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_cause(self): try: try: 1/0 finally: - exc_info_context = sys.exc_info() - exc_context = traceback.TracebackException(*exc_info_context) + exc = sys.exception() + exc_context = traceback.TracebackException.from_exception(exc) cause = Exception("cause") raise Exception("uh oh") from cause - except Exception: - exc_info = sys.exc_info() - exc = traceback.TracebackException(*exc_info) + except Exception as e: + exc_obj = e + exc = traceback.TracebackException.from_exception(e) expected_stack = traceback.StackSummary.extract( - traceback.walk_tb(exc_info[2])) + traceback.walk_tb(e.__traceback__)) exc_cause = traceback.TracebackException(Exception, cause, None) self.assertEqual(exc_cause, exc.__cause__) self.assertEqual(exc_context, exc.__context__) self.assertEqual(True, exc.__suppress_context__) self.assertEqual(expected_stack, exc.stack) - self.assertEqual(exc_info[0], exc.exc_type) - self.assertEqual(str(exc_info[1]), str(exc)) + with self.assertWarns(DeprecationWarning): + self.assertEqual(type(exc_obj), exc.exc_type) + self.assertEqual(type(exc_obj).__name__, exc.exc_type_str) + self.assertEqual(str(exc_obj), str(exc)) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_context(self): try: try: 1/0 finally: - exc_info_context = sys.exc_info() - exc_context = traceback.TracebackException(*exc_info_context) + exc = sys.exception() + exc_context = traceback.TracebackException.from_exception(exc) raise Exception("uh oh") - except Exception: - exc_info = sys.exc_info() - exc = traceback.TracebackException(*exc_info) + except Exception as e: + exc_obj = e + exc = traceback.TracebackException.from_exception(e) expected_stack = traceback.StackSummary.extract( - traceback.walk_tb(exc_info[2])) + traceback.walk_tb(e.__traceback__)) self.assertEqual(None, exc.__cause__) self.assertEqual(exc_context, exc.__context__) self.assertEqual(False, exc.__suppress_context__) self.assertEqual(expected_stack, exc.stack) - self.assertEqual(exc_info[0], exc.exc_type) - self.assertEqual(str(exc_info[1]), str(exc)) + with self.assertWarns(DeprecationWarning): + self.assertEqual(type(exc_obj), exc.exc_type) + self.assertEqual(type(exc_obj).__name__, exc.exc_type_str) + self.assertEqual(str(exc_obj), str(exc)) # TODO: RUSTPYTHON @unittest.expectedFailure @@ -1258,17 +3713,17 @@ def test_long_context_chain(self): def f(): try: 1/0 - except: + except ZeroDivisionError: f() try: f() - except RecursionError: - exc_info = sys.exc_info() + except RecursionError as e: + exc_obj = e else: self.fail("Exception not raised") - te = traceback.TracebackException(*exc_info) + te = traceback.TracebackException.from_exception(exc_obj) res = list(te.format()) # many ZeroDiv errors followed by the RecursionError @@ -1279,6 +3734,8 @@ def f(): self.assertIn( "RecursionError: maximum recursion depth exceeded", res[-1]) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_compact_with_cause(self): try: try: @@ -1286,58 +3743,77 @@ def test_compact_with_cause(self): finally: cause = Exception("cause") raise Exception("uh oh") from cause - except Exception: - exc_info = sys.exc_info() - exc = traceback.TracebackException(*exc_info, compact=True) + except Exception as e: + exc_obj = e + exc = traceback.TracebackException.from_exception(exc_obj, compact=True) expected_stack = traceback.StackSummary.extract( - traceback.walk_tb(exc_info[2])) + traceback.walk_tb(exc_obj.__traceback__)) exc_cause = traceback.TracebackException(Exception, cause, None) self.assertEqual(exc_cause, exc.__cause__) self.assertEqual(None, exc.__context__) self.assertEqual(True, exc.__suppress_context__) self.assertEqual(expected_stack, exc.stack) - self.assertEqual(exc_info[0], exc.exc_type) - self.assertEqual(str(exc_info[1]), str(exc)) + with self.assertWarns(DeprecationWarning): + self.assertEqual(type(exc_obj), exc.exc_type) + self.assertEqual(type(exc_obj).__name__, exc.exc_type_str) + self.assertEqual(str(exc_obj), str(exc)) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_compact_no_cause(self): try: try: 1/0 finally: - exc_info_context = sys.exc_info() - exc_context = traceback.TracebackException(*exc_info_context) + exc = sys.exception() + exc_context = traceback.TracebackException.from_exception(exc) raise Exception("uh oh") - except Exception: - exc_info = sys.exc_info() - exc = traceback.TracebackException(*exc_info, compact=True) + except Exception as e: + exc_obj = e + exc = traceback.TracebackException.from_exception(e, compact=True) expected_stack = traceback.StackSummary.extract( - traceback.walk_tb(exc_info[2])) + traceback.walk_tb(exc_obj.__traceback__)) self.assertEqual(None, exc.__cause__) self.assertEqual(exc_context, exc.__context__) self.assertEqual(False, exc.__suppress_context__) self.assertEqual(expected_stack, exc.stack) - self.assertEqual(exc_info[0], exc.exc_type) - self.assertEqual(str(exc_info[1]), str(exc)) + with self.assertWarns(DeprecationWarning): + self.assertEqual(type(exc_obj), exc.exc_type) + self.assertEqual(type(exc_obj).__name__, exc.exc_type_str) + self.assertEqual(str(exc_obj), str(exc)) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_no_save_exc_type(self): + try: + 1/0 + except Exception as e: + exc = e + + te = traceback.TracebackException.from_exception( + exc, save_exc_type=False) + with self.assertWarns(DeprecationWarning): + self.assertIsNone(te.exc_type) def test_no_refs_to_exception_and_traceback_objects(self): try: 1/0 - except Exception: - exc_info = sys.exc_info() + except Exception as e: + exc_obj = e - refcnt1 = sys.getrefcount(exc_info[1]) - refcnt2 = sys.getrefcount(exc_info[2]) - exc = traceback.TracebackException(*exc_info) - self.assertEqual(sys.getrefcount(exc_info[1]), refcnt1) - self.assertEqual(sys.getrefcount(exc_info[2]), refcnt2) + refcnt1 = sys.getrefcount(exc_obj) + refcnt2 = sys.getrefcount(exc_obj.__traceback__) + exc = traceback.TracebackException.from_exception(exc_obj) + self.assertEqual(sys.getrefcount(exc_obj), refcnt1) + self.assertEqual(sys.getrefcount(exc_obj.__traceback__), refcnt2) def test_comparison_basic(self): try: 1/0 - except Exception: - exc_info = sys.exc_info() - exc = traceback.TracebackException(*exc_info) - exc2 = traceback.TracebackException(*exc_info) + except Exception as e: + exc_obj = e + exc = traceback.TracebackException.from_exception(exc_obj) + exc2 = traceback.TracebackException.from_exception(exc_obj) self.assertIsNot(exc, exc2) self.assertEqual(exc, exc2) self.assertNotEqual(exc, object()) @@ -1349,7 +3825,7 @@ def test_comparison_params_variations(self): def raise_exc(): try: raise ValueError('bad value') - except: + except ValueError: raise def raise_with_locals(): @@ -1358,28 +3834,28 @@ def raise_with_locals(): try: raise_with_locals() - except Exception: - exc_info = sys.exc_info() + except Exception as e: + exc_obj = e - exc = traceback.TracebackException(*exc_info) - exc1 = traceback.TracebackException(*exc_info, limit=10) - exc2 = traceback.TracebackException(*exc_info, limit=2) + exc = traceback.TracebackException.from_exception(exc_obj) + exc1 = traceback.TracebackException.from_exception(exc_obj, limit=10) + exc2 = traceback.TracebackException.from_exception(exc_obj, limit=2) self.assertEqual(exc, exc1) # limit=10 gets all frames self.assertNotEqual(exc, exc2) # limit=2 truncates the output # locals change the output - exc3 = traceback.TracebackException(*exc_info, capture_locals=True) + exc3 = traceback.TracebackException.from_exception(exc_obj, capture_locals=True) self.assertNotEqual(exc, exc3) # there are no locals in the innermost frame - exc4 = traceback.TracebackException(*exc_info, limit=-1) - exc5 = traceback.TracebackException(*exc_info, limit=-1, capture_locals=True) + exc4 = traceback.TracebackException.from_exception(exc_obj, limit=-1) + exc5 = traceback.TracebackException.from_exception(exc_obj, limit=-1, capture_locals=True) self.assertEqual(exc4, exc5) # there are locals in the next-to-innermost frame - exc6 = traceback.TracebackException(*exc_info, limit=-2) - exc7 = traceback.TracebackException(*exc_info, limit=-2, capture_locals=True) + exc6 = traceback.TracebackException.from_exception(exc_obj, limit=-2) + exc7 = traceback.TracebackException.from_exception(exc_obj, limit=-2, capture_locals=True) self.assertNotEqual(exc6, exc7) def test_comparison_equivalent_exceptions_are_equal(self): @@ -1387,8 +3863,8 @@ def test_comparison_equivalent_exceptions_are_equal(self): for _ in range(2): try: 1/0 - except: - excs.append(traceback.TracebackException(*sys.exc_info())) + except Exception as e: + excs.append(traceback.TracebackException.from_exception(e)) self.assertEqual(excs[0], excs[1]) self.assertEqual(list(excs[0].format()), list(excs[1].format())) @@ -1404,9 +3880,9 @@ def __eq__(self, other): except UnhashableException: try: raise ex1 - except UnhashableException: - exc_info = sys.exc_info() - exc = traceback.TracebackException(*exc_info) + except UnhashableException as e: + exc_obj = e + exc = traceback.TracebackException.from_exception(exc_obj) formatted = list(exc.format()) self.assertIn('UnhashableException: ex2\n', formatted[2]) self.assertIn('UnhashableException: ex1\n', formatted[6]) @@ -1419,11 +3895,10 @@ def recurse(n): 1/0 try: recurse(10) - except Exception: - exc_info = sys.exc_info() - exc = traceback.TracebackException(*exc_info, limit=5) + except Exception as e: + exc = traceback.TracebackException.from_exception(e, limit=5) expected_stack = traceback.StackSummary.extract( - traceback.walk_tb(exc_info[2]), limit=5) + traceback.walk_tb(e.__traceback__), limit=5) self.assertEqual(expected_stack, exc.stack) def test_lookup_lines(self): @@ -1431,29 +3906,32 @@ def test_lookup_lines(self): e = Exception("uh oh") c = test_code('/foo.py', 'method') f = test_frame(c, None, None) - tb = test_tb(f, 6, None) + tb = test_tb(f, 6, None, 0) exc = traceback.TracebackException(Exception, e, tb, lookup_lines=False) self.assertEqual(linecache.cache, {}) linecache.updatecache('/foo.py', globals()) self.assertEqual(exc.stack[0].line, "import sys") + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_locals(self): linecache.updatecache('/foo.py', globals()) e = Exception("uh oh") c = test_code('/foo.py', 'method') - f = test_frame(c, globals(), {'something': 1, 'other': 'string'}) - tb = test_tb(f, 6, None) + f = test_frame(c, globals(), {'something': 1, 'other': 'string', 'unrepresentable': Unrepresentable()}) + tb = test_tb(f, 6, None, 0) exc = traceback.TracebackException( Exception, e, tb, capture_locals=True) self.assertEqual( - exc.stack[0].locals, {'something': '1', 'other': "'string'"}) + exc.stack[0].locals, + {'something': '1', 'other': "'string'", 'unrepresentable': '<local repr() failed>'}) def test_no_locals(self): linecache.updatecache('/foo.py', globals()) e = Exception("uh oh") c = test_code('/foo.py', 'method') f = test_frame(c, globals(), {'something': 1}) - tb = test_tb(f, 6, None) + tb = test_tb(f, 6, None, 0) exc = traceback.TracebackException(Exception, e, tb) self.assertEqual(exc.stack[0].locals, None) @@ -1463,6 +3941,949 @@ def test_traceback_header(self): exc = traceback.TracebackException(Exception, Exception("haven"), None) self.assertEqual(list(exc.format()), ["Exception: haven\n"]) + # @requires_debug_ranges() # XXX: RUSTPYTHON patch + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_print(self): + def f(): + x = 12 + try: + x/0 + except Exception as e: + return e + exc = traceback.TracebackException.from_exception(f(), capture_locals=True) + output = StringIO() + exc.print(file=output) + self.assertEqual( + output.getvalue().split('\n')[-5:], + [' x/0', + ' ~^~', + ' x = 12', + 'ZeroDivisionError: division by zero', + '']) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_dont_swallow_cause_or_context_of_falsey_exception(self): + # see gh-132308: Ensure that __cause__ or __context__ attributes of exceptions + # that evaluate as falsey are included in the output. For falsey term, + # see https://docs.python.org/3/library/stdtypes.html#truth-value-testing. + + try: + raise FalseyException from KeyError + except FalseyException as e: + self.assertIn(cause_message, traceback.format_exception(e)) + + try: + try: + 1/0 + except ZeroDivisionError: + raise FalseyException + except FalseyException as e: + self.assertIn(context_message, traceback.format_exception(e)) + + +class TestTracebackException_ExceptionGroups(unittest.TestCase): + def setUp(self): + super().setUp() + self.eg = self._get_exception_group() + + def _get_exception_group(self): + def f(): + 1/0 + + def g(v): + raise ValueError(v) + + self.lno_f = f.__code__.co_firstlineno + self.lno_g = g.__code__.co_firstlineno + + try: + try: + try: + f() + except Exception as e: + exc1 = e + try: + g(42) + except Exception as e: + exc2 = e + raise ExceptionGroup("eg1", [exc1, exc2]) + except ExceptionGroup as e: + exc3 = e + try: + g(24) + except Exception as e: + exc4 = e + raise ExceptionGroup("eg2", [exc3, exc4]) + except ExceptionGroup as eg: + return eg + self.fail('Exception Not Raised') + + def test_exception_group_construction(self): + eg = self.eg + teg1 = traceback.TracebackException(type(eg), eg, eg.__traceback__) + teg2 = traceback.TracebackException.from_exception(eg) + self.assertIsNot(teg1, teg2) + self.assertEqual(teg1, teg2) + + def test_exception_group_format_exception_only(self): + teg = traceback.TracebackException.from_exception(self.eg) + formatted = ''.join(teg.format_exception_only()).split('\n') + expected = "ExceptionGroup: eg2 (2 sub-exceptions)\n".split('\n') + + self.assertEqual(formatted, expected) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_exception_group_format_exception_onlyi_recursive(self): + teg = traceback.TracebackException.from_exception(self.eg) + formatted = ''.join(teg.format_exception_only(show_group=True)).split('\n') + expected = [ + 'ExceptionGroup: eg2 (2 sub-exceptions)', + ' ExceptionGroup: eg1 (2 sub-exceptions)', + ' ZeroDivisionError: division by zero', + ' ValueError: 42', + ' ValueError: 24', + '' + ] + + self.assertEqual(formatted, expected) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_exception_group_format(self): + teg = traceback.TracebackException.from_exception(self.eg) + + formatted = ''.join(teg.format()).split('\n') + lno_f = self.lno_f + lno_g = self.lno_g + + expected = [ + f' + Exception Group Traceback (most recent call last):', + f' | File "{__file__}", line {lno_g+23}, in _get_exception_group', + f' | raise ExceptionGroup("eg2", [exc3, exc4])', + f' | ExceptionGroup: eg2 (2 sub-exceptions)', + f' +-+---------------- 1 ----------------', + f' | Exception Group Traceback (most recent call last):', + f' | File "{__file__}", line {lno_g+16}, in _get_exception_group', + f' | raise ExceptionGroup("eg1", [exc1, exc2])', + f' | ExceptionGroup: eg1 (2 sub-exceptions)', + f' +-+---------------- 1 ----------------', + f' | Traceback (most recent call last):', + f' | File "{__file__}", line {lno_g+9}, in _get_exception_group', + f' | f()', + f' | ~^^', + f' | File "{__file__}", line {lno_f+1}, in f', + f' | 1/0', + f' | ~^~', + f' | ZeroDivisionError: division by zero', + f' +---------------- 2 ----------------', + f' | Traceback (most recent call last):', + f' | File "{__file__}", line {lno_g+13}, in _get_exception_group', + f' | g(42)', + f' | ~^^^^', + f' | File "{__file__}", line {lno_g+1}, in g', + f' | raise ValueError(v)', + f' | ValueError: 42', + f' +------------------------------------', + f' +---------------- 2 ----------------', + f' | Traceback (most recent call last):', + f' | File "{__file__}", line {lno_g+20}, in _get_exception_group', + f' | g(24)', + f' | ~^^^^', + f' | File "{__file__}", line {lno_g+1}, in g', + f' | raise ValueError(v)', + f' | ValueError: 24', + f' +------------------------------------', + f''] + + self.assertEqual(formatted, expected) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_max_group_width(self): + excs1 = [] + excs2 = [] + for i in range(3): + excs1.append(ValueError(i)) + for i in range(10): + excs2.append(TypeError(i)) + + EG = ExceptionGroup + eg = EG('eg', [EG('eg1', excs1), EG('eg2', excs2)]) + + teg = traceback.TracebackException.from_exception(eg, max_group_width=2) + formatted = ''.join(teg.format()).split('\n') + + expected = [ + ' | ExceptionGroup: eg (2 sub-exceptions)', + ' +-+---------------- 1 ----------------', + ' | ExceptionGroup: eg1 (3 sub-exceptions)', + ' +-+---------------- 1 ----------------', + ' | ValueError: 0', + ' +---------------- 2 ----------------', + ' | ValueError: 1', + ' +---------------- ... ----------------', + ' | and 1 more exception', + ' +------------------------------------', + ' +---------------- 2 ----------------', + ' | ExceptionGroup: eg2 (10 sub-exceptions)', + ' +-+---------------- 1 ----------------', + ' | TypeError: 0', + ' +---------------- 2 ----------------', + ' | TypeError: 1', + ' +---------------- ... ----------------', + ' | and 8 more exceptions', + ' +------------------------------------', + ''] + + self.assertEqual(formatted, expected) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_max_group_depth(self): + exc = TypeError('bad type') + for i in range(3): + exc = ExceptionGroup('exc', [ValueError(-i), exc, ValueError(i)]) + + teg = traceback.TracebackException.from_exception(exc, max_group_depth=2) + formatted = ''.join(teg.format()).split('\n') + + expected = [ + ' | ExceptionGroup: exc (3 sub-exceptions)', + ' +-+---------------- 1 ----------------', + ' | ValueError: -2', + ' +---------------- 2 ----------------', + ' | ExceptionGroup: exc (3 sub-exceptions)', + ' +-+---------------- 1 ----------------', + ' | ValueError: -1', + ' +---------------- 2 ----------------', + ' | ... (max_group_depth is 2)', + ' +---------------- 3 ----------------', + ' | ValueError: 1', + ' +------------------------------------', + ' +---------------- 3 ----------------', + ' | ValueError: 2', + ' +------------------------------------', + ''] + + self.assertEqual(formatted, expected) + + def test_comparison(self): + try: + raise self.eg + except ExceptionGroup as e: + exc = e + for _ in range(5): + try: + raise exc + except Exception as e: + exc_obj = e + exc = traceback.TracebackException.from_exception(exc_obj) + exc2 = traceback.TracebackException.from_exception(exc_obj) + exc3 = traceback.TracebackException.from_exception(exc_obj, limit=300) + ne = traceback.TracebackException.from_exception(exc_obj, limit=3) + self.assertIsNot(exc, exc2) + self.assertEqual(exc, exc2) + self.assertEqual(exc, exc3) + self.assertNotEqual(exc, ne) + self.assertNotEqual(exc, object()) + self.assertEqual(exc, ALWAYS_EQ) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_dont_swallow_subexceptions_of_falsey_exceptiongroup(self): + # see gh-132308: Ensure that subexceptions of exception groups + # that evaluate as falsey are displayed in the output. For falsey term, + # see https://docs.python.org/3/library/stdtypes.html#truth-value-testing. + + try: + raise FalseyExceptionGroup("Gih", (KeyError(), NameError())) + except Exception as ee: + str_exc = ''.join(traceback.format_exception(ee)) + self.assertIn('+---------------- 1 ----------------', str_exc) + self.assertIn('+---------------- 2 ----------------', str_exc) + + # Test with a falsey exception, in last position, as sub-exceptions. + msg = 'bool' + try: + raise FalseyExceptionGroup("Gah", (KeyError(), FalseyException(msg))) + except Exception as ee: + str_exc = traceback.format_exception(ee) + self.assertIn(f'{FalseyException.__name__}: {msg}', str_exc[-2]) + + +global_for_suggestions = None + + +class SuggestionFormattingTestBase: + def get_suggestion(self, obj, attr_name=None): + if attr_name is not None: + def callable(): + getattr(obj, attr_name) + else: + callable = obj + + result_lines = self.get_exception( + callable, slice_start=-1, slice_end=None + ) + return result_lines[0] + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_getattr_suggestions(self): + class Substitution: + noise = more_noise = a = bc = None + blech = None + + class Elimination: + noise = more_noise = a = bc = None + blch = None + + class Addition: + noise = more_noise = a = bc = None + bluchin = None + + class SubstitutionOverElimination: + blach = None + bluc = None + + class SubstitutionOverAddition: + blach = None + bluchi = None + + class EliminationOverAddition: + blucha = None + bluc = None + + class CaseChangeOverSubstitution: + Luch = None + fluch = None + BLuch = None + + for cls, suggestion in [ + (Addition, "'bluchin'?"), + (Substitution, "'blech'?"), + (Elimination, "'blch'?"), + (Addition, "'bluchin'?"), + (SubstitutionOverElimination, "'blach'?"), + (SubstitutionOverAddition, "'blach'?"), + (EliminationOverAddition, "'bluc'?"), + (CaseChangeOverSubstitution, "'BLuch'?"), + ]: + actual = self.get_suggestion(cls(), 'bluch') + self.assertIn(suggestion, actual) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_getattr_suggestions_underscored(self): + class A: + bluch = None + + self.assertIn("'bluch'", self.get_suggestion(A(), 'blach')) + self.assertIn("'bluch'", self.get_suggestion(A(), '_luch')) + self.assertIn("'bluch'", self.get_suggestion(A(), '_bluch')) + + class B: + _bluch = None + def method(self, name): + getattr(self, name) + + self.assertIn("'_bluch'", self.get_suggestion(B(), '_blach')) + self.assertIn("'_bluch'", self.get_suggestion(B(), '_luch')) + self.assertNotIn("'_bluch'", self.get_suggestion(B(), 'bluch')) + + self.assertIn("'_bluch'", self.get_suggestion(partial(B().method, '_blach'))) + self.assertIn("'_bluch'", self.get_suggestion(partial(B().method, '_luch'))) + self.assertIn("'_bluch'", self.get_suggestion(partial(B().method, 'bluch'))) + + def test_getattr_suggestions_do_not_trigger_for_long_attributes(self): + class A: + blech = None + + actual = self.get_suggestion(A(), 'somethingverywrong') + self.assertNotIn("blech", actual) + + def test_getattr_error_bad_suggestions_do_not_trigger_for_small_names(self): + class MyClass: + vvv = mom = w = id = pytho = None + + for name in ("b", "v", "m", "py"): + with self.subTest(name=name): + actual = self.get_suggestion(MyClass, name) + self.assertNotIn("Did you mean", actual) + self.assertNotIn("'vvv", actual) + self.assertNotIn("'mom'", actual) + self.assertNotIn("'id'", actual) + self.assertNotIn("'w'", actual) + self.assertNotIn("'pytho'", actual) + + def test_getattr_suggestions_do_not_trigger_for_big_dicts(self): + class A: + blech = None + # A class with a very big __dict__ will not be considered + # for suggestions. + for index in range(2000): + setattr(A, f"index_{index}", None) + + actual = self.get_suggestion(A(), 'bluch') + self.assertNotIn("blech", actual) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_getattr_suggestions_no_args(self): + class A: + blech = None + def __getattr__(self, attr): + raise AttributeError() + + actual = self.get_suggestion(A(), 'bluch') + self.assertIn("blech", actual) + + class A: + blech = None + def __getattr__(self, attr): + raise AttributeError + + actual = self.get_suggestion(A(), 'bluch') + self.assertIn("blech", actual) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_getattr_suggestions_invalid_args(self): + class NonStringifyClass: + __str__ = None + __repr__ = None + + class A: + blech = None + def __getattr__(self, attr): + raise AttributeError(NonStringifyClass()) + + class B: + blech = None + def __getattr__(self, attr): + raise AttributeError("Error", 23) + + class C: + blech = None + def __getattr__(self, attr): + raise AttributeError(23) + + for cls in [A, B, C]: + actual = self.get_suggestion(cls(), 'bluch') + self.assertIn("blech", actual) + + def test_getattr_suggestions_for_same_name(self): + class A: + def __dir__(self): + return ['blech'] + actual = self.get_suggestion(A(), 'blech') + self.assertNotIn("Did you mean", actual) + + def test_attribute_error_with_failing_dict(self): + class T: + bluch = 1 + def __dir__(self): + raise AttributeError("oh no!") + + actual = self.get_suggestion(T(), 'blich') + self.assertNotIn("blech", actual) + self.assertNotIn("oh no!", actual) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_attribute_error_with_non_string_candidates(self): + class T: + bluch = 1 + + instance = T() + instance.__dict__[0] = 1 + actual = self.get_suggestion(instance, 'blich') + self.assertIn("bluch", actual) + + def test_attribute_error_with_bad_name(self): + def raise_attribute_error_with_bad_name(): + raise AttributeError(name=12, obj=23) + + result_lines = self.get_exception( + raise_attribute_error_with_bad_name, slice_start=-1, slice_end=None + ) + self.assertNotIn("?", result_lines[-1]) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_attribute_error_inside_nested_getattr(self): + class A: + bluch = 1 + + class B: + def __getattribute__(self, attr): + a = A() + return a.blich + + actual = self.get_suggestion(B(), 'something') + self.assertIn("Did you mean", actual) + self.assertIn("bluch", actual) + + def make_module(self, code): + tmpdir = Path(tempfile.mkdtemp()) + self.addCleanup(shutil.rmtree, tmpdir) + + sys.path.append(str(tmpdir)) + self.addCleanup(sys.path.pop) + + mod_name = ''.join(random.choices(string.ascii_letters, k=16)) + module = tmpdir / (mod_name + ".py") + module.write_text(code) + + return mod_name + + def get_import_from_suggestion(self, code, name): + modname = self.make_module(code) + + def callable(): + try: + exec(f"from {modname} import {name}") + except ImportError as e: + raise e from None + except Exception as e: + self.fail(f"Expected ImportError but got {type(e)}") + self.addCleanup(forget, modname) + + result_lines = self.get_exception( + callable, slice_start=-1, slice_end=None + ) + return result_lines[0] + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_import_from_suggestions(self): + substitution = textwrap.dedent("""\ + noise = more_noise = a = bc = None + blech = None + """) + + elimination = textwrap.dedent(""" + noise = more_noise = a = bc = None + blch = None + """) + + addition = textwrap.dedent(""" + noise = more_noise = a = bc = None + bluchin = None + """) + + substitutionOverElimination = textwrap.dedent(""" + blach = None + bluc = None + """) + + substitutionOverAddition = textwrap.dedent(""" + blach = None + bluchi = None + """) + + eliminationOverAddition = textwrap.dedent(""" + blucha = None + bluc = None + """) + + caseChangeOverSubstitution = textwrap.dedent(""" + Luch = None + fluch = None + BLuch = None + """) + + for code, suggestion in [ + (addition, "'bluchin'?"), + (substitution, "'blech'?"), + (elimination, "'blch'?"), + (addition, "'bluchin'?"), + (substitutionOverElimination, "'blach'?"), + (substitutionOverAddition, "'blach'?"), + (eliminationOverAddition, "'bluc'?"), + (caseChangeOverSubstitution, "'BLuch'?"), + ]: + actual = self.get_import_from_suggestion(code, 'bluch') + self.assertIn(suggestion, actual) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_import_from_suggestions_underscored(self): + code = "bluch = None" + self.assertIn("'bluch'", self.get_import_from_suggestion(code, 'blach')) + self.assertIn("'bluch'", self.get_import_from_suggestion(code, '_luch')) + self.assertIn("'bluch'", self.get_import_from_suggestion(code, '_bluch')) + + code = "_bluch = None" + self.assertIn("'_bluch'", self.get_import_from_suggestion(code, '_blach')) + self.assertIn("'_bluch'", self.get_import_from_suggestion(code, '_luch')) + self.assertNotIn("'_bluch'", self.get_import_from_suggestion(code, 'bluch')) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_import_from_suggestions_non_string(self): + modWithNonStringAttr = textwrap.dedent("""\ + globals()[0] = 1 + bluch = 1 + """) + self.assertIn("'bluch'", self.get_import_from_suggestion(modWithNonStringAttr, 'blech')) + + def test_import_from_suggestions_do_not_trigger_for_long_attributes(self): + code = "blech = None" + + actual = self.get_suggestion(code, 'somethingverywrong') + self.assertNotIn("blech", actual) + + def test_import_from_error_bad_suggestions_do_not_trigger_for_small_names(self): + code = "vvv = mom = w = id = pytho = None" + + for name in ("b", "v", "m", "py"): + with self.subTest(name=name): + actual = self.get_import_from_suggestion(code, name) + self.assertNotIn("Did you mean", actual) + self.assertNotIn("'vvv'", actual) + self.assertNotIn("'mom'", actual) + self.assertNotIn("'id'", actual) + self.assertNotIn("'w'", actual) + self.assertNotIn("'pytho'", actual) + + def test_import_from_suggestions_do_not_trigger_for_big_namespaces(self): + # A module with lots of names will not be considered for suggestions. + chunks = [f"index_{index} = " for index in range(200)] + chunks.append(" None") + code = " ".join(chunks) + actual = self.get_import_from_suggestion(code, 'bluch') + self.assertNotIn("blech", actual) + + def test_import_from_error_with_bad_name(self): + def raise_attribute_error_with_bad_name(): + raise ImportError(name=12, obj=23, name_from=11) + + result_lines = self.get_exception( + raise_attribute_error_with_bad_name, slice_start=-1, slice_end=None + ) + self.assertNotIn("?", result_lines[-1]) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_name_error_suggestions(self): + def Substitution(): + noise = more_noise = a = bc = None + blech = None + print(bluch) + + def Elimination(): + noise = more_noise = a = bc = None + blch = None + print(bluch) + + def Addition(): + noise = more_noise = a = bc = None + bluchin = None + print(bluch) + + def SubstitutionOverElimination(): + blach = None + bluc = None + print(bluch) + + def SubstitutionOverAddition(): + blach = None + bluchi = None + print(bluch) + + def EliminationOverAddition(): + blucha = None + bluc = None + print(bluch) + + for func, suggestion in [(Substitution, "'blech'?"), + (Elimination, "'blch'?"), + (Addition, "'bluchin'?"), + (EliminationOverAddition, "'blucha'?"), + (SubstitutionOverElimination, "'blach'?"), + (SubstitutionOverAddition, "'blach'?")]: + actual = self.get_suggestion(func) + self.assertIn(suggestion, actual) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_name_error_suggestions_from_globals(self): + def func(): + print(global_for_suggestio) + actual = self.get_suggestion(func) + self.assertIn("'global_for_suggestions'?", actual) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_name_error_suggestions_from_builtins(self): + def func(): + print(ZeroDivisionErrrrr) + actual = self.get_suggestion(func) + self.assertIn("'ZeroDivisionError'?", actual) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_name_error_suggestions_from_builtins_when_builtins_is_module(self): + def func(): + custom_globals = globals().copy() + custom_globals["__builtins__"] = builtins + print(eval("ZeroDivisionErrrrr", custom_globals)) + actual = self.get_suggestion(func) + self.assertIn("'ZeroDivisionError'?", actual) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_name_error_suggestions_with_non_string_candidates(self): + def func(): + abc = 1 + custom_globals = globals().copy() + custom_globals[0] = 1 + print(eval("abv", custom_globals, locals())) + actual = self.get_suggestion(func) + self.assertIn("abc", actual) + + def test_name_error_suggestions_do_not_trigger_for_long_names(self): + def func(): + somethingverywronghehehehehehe = None + print(somethingverywronghe) + actual = self.get_suggestion(func) + self.assertNotIn("somethingverywronghehe", actual) + + def test_name_error_bad_suggestions_do_not_trigger_for_small_names(self): + + def f_b(): + vvv = mom = w = id = pytho = None + b + + def f_v(): + vvv = mom = w = id = pytho = None + v + + def f_m(): + vvv = mom = w = id = pytho = None + m + + def f_py(): + vvv = mom = w = id = pytho = None + py + + for name, func in (("b", f_b), ("v", f_v), ("m", f_m), ("py", f_py)): + with self.subTest(name=name): + actual = self.get_suggestion(func) + self.assertNotIn("you mean", actual) + self.assertNotIn("vvv", actual) + self.assertNotIn("mom", actual) + self.assertNotIn("'id'", actual) + self.assertNotIn("'w'", actual) + self.assertNotIn("'pytho'", actual) + + def test_name_error_suggestions_do_not_trigger_for_too_many_locals(self): + def func(): + # Mutating locals() is unreliable, so we need to do it by hand + a1 = a2 = a3 = a4 = a5 = a6 = a7 = a8 = a9 = a10 = \ + a11 = a12 = a13 = a14 = a15 = a16 = a17 = a18 = a19 = a20 = \ + a21 = a22 = a23 = a24 = a25 = a26 = a27 = a28 = a29 = a30 = \ + a31 = a32 = a33 = a34 = a35 = a36 = a37 = a38 = a39 = a40 = \ + a41 = a42 = a43 = a44 = a45 = a46 = a47 = a48 = a49 = a50 = \ + a51 = a52 = a53 = a54 = a55 = a56 = a57 = a58 = a59 = a60 = \ + a61 = a62 = a63 = a64 = a65 = a66 = a67 = a68 = a69 = a70 = \ + a71 = a72 = a73 = a74 = a75 = a76 = a77 = a78 = a79 = a80 = \ + a81 = a82 = a83 = a84 = a85 = a86 = a87 = a88 = a89 = a90 = \ + a91 = a92 = a93 = a94 = a95 = a96 = a97 = a98 = a99 = a100 = \ + a101 = a102 = a103 = a104 = a105 = a106 = a107 = a108 = a109 = a110 = \ + a111 = a112 = a113 = a114 = a115 = a116 = a117 = a118 = a119 = a120 = \ + a121 = a122 = a123 = a124 = a125 = a126 = a127 = a128 = a129 = a130 = \ + a131 = a132 = a133 = a134 = a135 = a136 = a137 = a138 = a139 = a140 = \ + a141 = a142 = a143 = a144 = a145 = a146 = a147 = a148 = a149 = a150 = \ + a151 = a152 = a153 = a154 = a155 = a156 = a157 = a158 = a159 = a160 = \ + a161 = a162 = a163 = a164 = a165 = a166 = a167 = a168 = a169 = a170 = \ + a171 = a172 = a173 = a174 = a175 = a176 = a177 = a178 = a179 = a180 = \ + a181 = a182 = a183 = a184 = a185 = a186 = a187 = a188 = a189 = a190 = \ + a191 = a192 = a193 = a194 = a195 = a196 = a197 = a198 = a199 = a200 = \ + a201 = a202 = a203 = a204 = a205 = a206 = a207 = a208 = a209 = a210 = \ + a211 = a212 = a213 = a214 = a215 = a216 = a217 = a218 = a219 = a220 = \ + a221 = a222 = a223 = a224 = a225 = a226 = a227 = a228 = a229 = a230 = \ + a231 = a232 = a233 = a234 = a235 = a236 = a237 = a238 = a239 = a240 = \ + a241 = a242 = a243 = a244 = a245 = a246 = a247 = a248 = a249 = a250 = \ + a251 = a252 = a253 = a254 = a255 = a256 = a257 = a258 = a259 = a260 = \ + a261 = a262 = a263 = a264 = a265 = a266 = a267 = a268 = a269 = a270 = \ + a271 = a272 = a273 = a274 = a275 = a276 = a277 = a278 = a279 = a280 = \ + a281 = a282 = a283 = a284 = a285 = a286 = a287 = a288 = a289 = a290 = \ + a291 = a292 = a293 = a294 = a295 = a296 = a297 = a298 = a299 = a300 = \ + a301 = a302 = a303 = a304 = a305 = a306 = a307 = a308 = a309 = a310 = \ + a311 = a312 = a313 = a314 = a315 = a316 = a317 = a318 = a319 = a320 = \ + a321 = a322 = a323 = a324 = a325 = a326 = a327 = a328 = a329 = a330 = \ + a331 = a332 = a333 = a334 = a335 = a336 = a337 = a338 = a339 = a340 = \ + a341 = a342 = a343 = a344 = a345 = a346 = a347 = a348 = a349 = a350 = \ + a351 = a352 = a353 = a354 = a355 = a356 = a357 = a358 = a359 = a360 = \ + a361 = a362 = a363 = a364 = a365 = a366 = a367 = a368 = a369 = a370 = \ + a371 = a372 = a373 = a374 = a375 = a376 = a377 = a378 = a379 = a380 = \ + a381 = a382 = a383 = a384 = a385 = a386 = a387 = a388 = a389 = a390 = \ + a391 = a392 = a393 = a394 = a395 = a396 = a397 = a398 = a399 = a400 = \ + a401 = a402 = a403 = a404 = a405 = a406 = a407 = a408 = a409 = a410 = \ + a411 = a412 = a413 = a414 = a415 = a416 = a417 = a418 = a419 = a420 = \ + a421 = a422 = a423 = a424 = a425 = a426 = a427 = a428 = a429 = a430 = \ + a431 = a432 = a433 = a434 = a435 = a436 = a437 = a438 = a439 = a440 = \ + a441 = a442 = a443 = a444 = a445 = a446 = a447 = a448 = a449 = a450 = \ + a451 = a452 = a453 = a454 = a455 = a456 = a457 = a458 = a459 = a460 = \ + a461 = a462 = a463 = a464 = a465 = a466 = a467 = a468 = a469 = a470 = \ + a471 = a472 = a473 = a474 = a475 = a476 = a477 = a478 = a479 = a480 = \ + a481 = a482 = a483 = a484 = a485 = a486 = a487 = a488 = a489 = a490 = \ + a491 = a492 = a493 = a494 = a495 = a496 = a497 = a498 = a499 = a500 = \ + a501 = a502 = a503 = a504 = a505 = a506 = a507 = a508 = a509 = a510 = \ + a511 = a512 = a513 = a514 = a515 = a516 = a517 = a518 = a519 = a520 = \ + a521 = a522 = a523 = a524 = a525 = a526 = a527 = a528 = a529 = a530 = \ + a531 = a532 = a533 = a534 = a535 = a536 = a537 = a538 = a539 = a540 = \ + a541 = a542 = a543 = a544 = a545 = a546 = a547 = a548 = a549 = a550 = \ + a551 = a552 = a553 = a554 = a555 = a556 = a557 = a558 = a559 = a560 = \ + a561 = a562 = a563 = a564 = a565 = a566 = a567 = a568 = a569 = a570 = \ + a571 = a572 = a573 = a574 = a575 = a576 = a577 = a578 = a579 = a580 = \ + a581 = a582 = a583 = a584 = a585 = a586 = a587 = a588 = a589 = a590 = \ + a591 = a592 = a593 = a594 = a595 = a596 = a597 = a598 = a599 = a600 = \ + a601 = a602 = a603 = a604 = a605 = a606 = a607 = a608 = a609 = a610 = \ + a611 = a612 = a613 = a614 = a615 = a616 = a617 = a618 = a619 = a620 = \ + a621 = a622 = a623 = a624 = a625 = a626 = a627 = a628 = a629 = a630 = \ + a631 = a632 = a633 = a634 = a635 = a636 = a637 = a638 = a639 = a640 = \ + a641 = a642 = a643 = a644 = a645 = a646 = a647 = a648 = a649 = a650 = \ + a651 = a652 = a653 = a654 = a655 = a656 = a657 = a658 = a659 = a660 = \ + a661 = a662 = a663 = a664 = a665 = a666 = a667 = a668 = a669 = a670 = \ + a671 = a672 = a673 = a674 = a675 = a676 = a677 = a678 = a679 = a680 = \ + a681 = a682 = a683 = a684 = a685 = a686 = a687 = a688 = a689 = a690 = \ + a691 = a692 = a693 = a694 = a695 = a696 = a697 = a698 = a699 = a700 = \ + a701 = a702 = a703 = a704 = a705 = a706 = a707 = a708 = a709 = a710 = \ + a711 = a712 = a713 = a714 = a715 = a716 = a717 = a718 = a719 = a720 = \ + a721 = a722 = a723 = a724 = a725 = a726 = a727 = a728 = a729 = a730 = \ + a731 = a732 = a733 = a734 = a735 = a736 = a737 = a738 = a739 = a740 = \ + a741 = a742 = a743 = a744 = a745 = a746 = a747 = a748 = a749 = a750 = \ + a751 = a752 = a753 = a754 = a755 = a756 = a757 = a758 = a759 = a760 = \ + a761 = a762 = a763 = a764 = a765 = a766 = a767 = a768 = a769 = a770 = \ + a771 = a772 = a773 = a774 = a775 = a776 = a777 = a778 = a779 = a780 = \ + a781 = a782 = a783 = a784 = a785 = a786 = a787 = a788 = a789 = a790 = \ + a791 = a792 = a793 = a794 = a795 = a796 = a797 = a798 = a799 = a800 \ + = None + print(a0) + + actual = self.get_suggestion(func) + self.assertNotRegex(actual, r"NameError.*a1") + + def test_name_error_with_custom_exceptions(self): + def func(): + blech = None + raise NameError() + + actual = self.get_suggestion(func) + self.assertNotIn("blech", actual) + + def func(): + blech = None + raise NameError + + actual = self.get_suggestion(func) + self.assertNotIn("blech", actual) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_name_error_with_instance(self): + class A: + def __init__(self): + self.blech = None + def foo(self): + blich = 1 + x = blech + + instance = A() + actual = self.get_suggestion(instance.foo) + self.assertIn("self.blech", actual) + + def test_unbound_local_error_with_instance(self): + class A: + def __init__(self): + self.blech = None + def foo(self): + blich = 1 + x = blech + blech = 1 + + instance = A() + actual = self.get_suggestion(instance.foo) + self.assertNotIn("self.blech", actual) + + def test_unbound_local_error_with_side_effect(self): + # gh-132385 + class A: + def __getattr__(self, key): + if key == 'foo': + raise AttributeError('foo') + if key == 'spam': + raise ValueError('spam') + + def bar(self): + foo + def baz(self): + spam + + suggestion = self.get_suggestion(A().bar) + self.assertNotIn('self.', suggestion) + self.assertIn("'foo'", suggestion) + + suggestion = self.get_suggestion(A().baz) + self.assertNotIn('self.', suggestion) + self.assertIn("'spam'", suggestion) + + def test_unbound_local_error_does_not_match(self): + def func(): + something = 3 + print(somethong) + somethong = 3 + + actual = self.get_suggestion(func) + self.assertNotIn("something", actual) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_name_error_for_stdlib_modules(self): + def func(): + stream = io.StringIO() + + actual = self.get_suggestion(func) + self.assertIn("forget to import 'io'", actual) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_name_error_for_private_stdlib_modules(self): + def func(): + stream = _io.StringIO() + + actual = self.get_suggestion(func) + self.assertIn("forget to import '_io'", actual) + + + +class PurePythonSuggestionFormattingTests( + PurePythonExceptionFormattingMixin, + SuggestionFormattingTestBase, + unittest.TestCase, +): + """ + Same set of tests as above using the pure Python implementation of + traceback printing in traceback.py. + """ + + +@cpython_only +class CPythonSuggestionFormattingTests( + CAPIExceptionFormattingMixin, + SuggestionFormattingTestBase, + unittest.TestCase, +): + """ + Same set of tests as above but with Python's internal traceback printing. + """ + class MiscTest(unittest.TestCase): @@ -1477,6 +4898,231 @@ def test_all(self): expected.add(name) self.assertCountEqual(traceback.__all__, expected) + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_levenshtein_distance(self): + # copied from _testinternalcapi.test_edit_cost + # to also exercise the Python implementation + + def CHECK(a, b, expected): + actual = traceback._levenshtein_distance(a, b, 4044) + self.assertEqual(actual, expected) + + CHECK("", "", 0) + CHECK("", "a", 2) + CHECK("a", "A", 1) + CHECK("Apple", "Aple", 2) + CHECK("Banana", "B@n@n@", 6) + CHECK("Cherry", "Cherry!", 2) + CHECK("---0---", "------", 2) + CHECK("abc", "y", 6) + CHECK("aa", "bb", 4) + CHECK("aaaaa", "AAAAA", 5) + CHECK("wxyz", "wXyZ", 2) + CHECK("wxyz", "wXyZ123", 8) + CHECK("Python", "Java", 12) + CHECK("Java", "C#", 8) + CHECK("AbstractFoobarManager", "abstract_foobar_manager", 3+2*2) + CHECK("CPython", "PyPy", 10) + CHECK("CPython", "pypy", 11) + CHECK("AttributeError", "AttributeErrop", 2) + CHECK("AttributeError", "AttributeErrorTests", 10) + CHECK("ABA", "AAB", 4) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + @support.requires_resource('cpu') + def test_levenshtein_distance_short_circuit(self): + if not LEVENSHTEIN_DATA_FILE.is_file(): + self.fail( + f"{LEVENSHTEIN_DATA_FILE} is missing." + f" Run `make regen-test-levenshtein`" + ) + + with LEVENSHTEIN_DATA_FILE.open("r") as f: + examples = json.load(f) + for a, b, expected in examples: + res1 = traceback._levenshtein_distance(a, b, 1000) + self.assertEqual(res1, expected, msg=(a, b)) + + for threshold in [expected, expected + 1, expected + 2]: + # big enough thresholds shouldn't change the result + res2 = traceback._levenshtein_distance(a, b, threshold) + self.assertEqual(res2, expected, msg=(a, b, threshold)) + + for threshold in range(expected): + # for small thresholds, the only piece of information + # we receive is "strings not close enough". + res3 = traceback._levenshtein_distance(a, b, threshold) + self.assertGreater(res3, threshold, msg=(a, b, threshold)) + + @cpython_only + def test_suggestions_extension(self): + # Check that the C extension is available + import _suggestions + + self.assertEqual( + _suggestions._generate_suggestions( + ["hello", "world"], + "hell" + ), + "hello" + ) + self.assertEqual( + _suggestions._generate_suggestions( + ["hovercraft"], + "eels" + ), + None + ) + + # gh-131936: _generate_suggestions() doesn't accept list subclasses + class MyList(list): + pass + + with self.assertRaises(TypeError): + _suggestions._generate_suggestions(MyList(), "") + + + + +class TestColorizedTraceback(unittest.TestCase): + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_colorized_traceback(self): + def foo(*args): + x = {'a':{'b': None}} + y = x['a']['b']['c'] + + def baz2(*args): + return (lambda *args: foo(*args))(1,2,3,4) + + def baz1(*args): + return baz2(1,2,3,4) + + def bar(): + return baz1(1, + 2,3 + ,4) + try: + bar() + except Exception as e: + exc = traceback.TracebackException.from_exception( + e, capture_locals=True + ) + lines = "".join(exc.format(colorize=True)) + red = _colorize.ANSIColors.RED + boldr = _colorize.ANSIColors.BOLD_RED + reset = _colorize.ANSIColors.RESET + self.assertIn("y = " + red + "x['a']['b']" + reset + boldr + "['c']" + reset, lines) + self.assertIn("return " + red + "(lambda *args: foo(*args))" + reset + boldr + "(1,2,3,4)" + reset, lines) + self.assertIn("return (lambda *args: " + red + "foo" + reset + boldr + "(*args)" + reset + ")(1,2,3,4)", lines) + self.assertIn("return baz2(1,2,3,4)", lines) + self.assertIn("return baz1(1,\n 2,3\n ,4)", lines) + self.assertIn(red + "bar" + reset + boldr + "()" + reset, lines) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_colorized_syntax_error(self): + try: + compile("a $ b", "<string>", "exec") + except SyntaxError as e: + exc = traceback.TracebackException.from_exception( + e, capture_locals=True + ) + actual = "".join(exc.format(colorize=True)) + red = _colorize.ANSIColors.RED + magenta = _colorize.ANSIColors.MAGENTA + boldm = _colorize.ANSIColors.BOLD_MAGENTA + boldr = _colorize.ANSIColors.BOLD_RED + reset = _colorize.ANSIColors.RESET + expected = "".join([ + f' File {magenta}"<string>"{reset}, line {magenta}1{reset}\n', + f' a {boldr}${reset} b\n', + f' {boldr}^{reset}\n', + f'{boldm}SyntaxError{reset}: {magenta}invalid syntax{reset}\n'] + ) + self.assertIn(expected, actual) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_colorized_traceback_is_the_default(self): + def foo(): + 1/0 + + from _testcapi import exception_print + try: + foo() + self.fail("No exception thrown.") + except Exception as e: + with captured_output("stderr") as tbstderr: + with unittest.mock.patch('_colorize.can_colorize', return_value=True): + exception_print(e) + actual = tbstderr.getvalue().splitlines() + + red = _colorize.ANSIColors.RED + boldr = _colorize.ANSIColors.BOLD_RED + magenta = _colorize.ANSIColors.MAGENTA + boldm = _colorize.ANSIColors.BOLD_MAGENTA + reset = _colorize.ANSIColors.RESET + lno_foo = foo.__code__.co_firstlineno + expected = ['Traceback (most recent call last):', + f' File {magenta}"{__file__}"{reset}, ' + f'line {magenta}{lno_foo+5}{reset}, in {magenta}test_colorized_traceback_is_the_default{reset}', + f' {red}foo{reset+boldr}(){reset}', + f' {red}~~~{reset+boldr}^^{reset}', + f' File {magenta}"{__file__}"{reset}, ' + f'line {magenta}{lno_foo+1}{reset}, in {magenta}foo{reset}', + f' {red}1{reset+boldr}/{reset+red}0{reset}', + f' {red}~{reset+boldr}^{reset+red}~{reset}', + f'{boldm}ZeroDivisionError{reset}: {magenta}division by zero{reset}'] + self.assertEqual(actual, expected) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_colorized_traceback_from_exception_group(self): + def foo(): + exceptions = [] + try: + 1 / 0 + except ZeroDivisionError as inner_exc: + exceptions.append(inner_exc) + raise ExceptionGroup("test", exceptions) + + try: + foo() + except Exception as e: + exc = traceback.TracebackException.from_exception( + e, capture_locals=True + ) + + red = _colorize.ANSIColors.RED + boldr = _colorize.ANSIColors.BOLD_RED + magenta = _colorize.ANSIColors.MAGENTA + boldm = _colorize.ANSIColors.BOLD_MAGENTA + reset = _colorize.ANSIColors.RESET + lno_foo = foo.__code__.co_firstlineno + actual = "".join(exc.format(colorize=True)).splitlines() + expected = [f" + Exception Group Traceback (most recent call last):", + f' | File {magenta}"{__file__}"{reset}, line {magenta}{lno_foo+9}{reset}, in {magenta}test_colorized_traceback_from_exception_group{reset}', + f' | {red}foo{reset}{boldr}(){reset}', + f' | {red}~~~{reset}{boldr}^^{reset}', + f" | e = ExceptionGroup('test', [ZeroDivisionError('division by zero')])", + f" | foo = {foo}", + f' | self = <{__name__}.TestColorizedTraceback testMethod=test_colorized_traceback_from_exception_group>', + f' | File {magenta}"{__file__}"{reset}, line {magenta}{lno_foo+6}{reset}, in {magenta}foo{reset}', + f' | raise ExceptionGroup("test", exceptions)', + f" | exceptions = [ZeroDivisionError('division by zero')]", + f' | {boldm}ExceptionGroup{reset}: {magenta}test (1 sub-exception){reset}', + f' +-+---------------- 1 ----------------', + f' | Traceback (most recent call last):', + f' | File {magenta}"{__file__}"{reset}, line {magenta}{lno_foo+3}{reset}, in {magenta}foo{reset}', + f' | {red}1 {reset}{boldr}/{reset}{red} 0{reset}', + f' | {red}~~{reset}{boldr}^{reset}{red}~~{reset}', + f" | exceptions = [ZeroDivisionError('division by zero')]", + f' | {boldm}ZeroDivisionError{reset}: {magenta}division by zero{reset}', + f' +------------------------------------'] + self.assertEqual(actual, expected) if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_tuple.py b/Lib/test/test_tuple.py index d2a2ed310b..153df0e52d 100644 --- a/Lib/test/test_tuple.py +++ b/Lib/test/test_tuple.py @@ -42,6 +42,35 @@ def test_keyword_args(self): with self.assertRaisesRegex(TypeError, 'keyword argument'): tuple(sequence=()) + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_keywords_in_subclass(self): + class subclass(tuple): + pass + u = subclass([1, 2]) + self.assertIs(type(u), subclass) + self.assertEqual(list(u), [1, 2]) + with self.assertRaises(TypeError): + subclass(sequence=()) + + class subclass_with_init(tuple): + def __init__(self, arg, newarg=None): + self.newarg = newarg + u = subclass_with_init([1, 2], newarg=3) + self.assertIs(type(u), subclass_with_init) + self.assertEqual(list(u), [1, 2]) + self.assertEqual(u.newarg, 3) + + class subclass_with_new(tuple): + def __new__(cls, arg, newarg=None): + self = super().__new__(cls, arg) + self.newarg = newarg + return self + u = subclass_with_new([1, 2], newarg=3) + self.assertIs(type(u), subclass_with_new) + self.assertEqual(list(u), [1, 2]) + self.assertEqual(u.newarg, 3) + def test_truth(self): super().test_truth() self.assertTrue(not ()) diff --git a/Lib/test/test_types.py b/Lib/test/test_types.py index 59dc9814fb..b57fdf35fb 100644 --- a/Lib/test/test_types.py +++ b/Lib/test/test_types.py @@ -591,12 +591,11 @@ def test_format_spec_errors(self): for code in 'xXobns': self.assertRaises(ValueError, format, 0, ',' + code) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_internal_sizes(self): self.assertGreater(object.__basicsize__, 0) self.assertGreater(tuple.__itemsize__, 0) + @unittest.expectedFailure # TODO: RUSTPYTHON def test_slot_wrapper_types(self): self.assertIsInstance(object.__init__, types.WrapperDescriptorType) self.assertIsInstance(object.__str__, types.WrapperDescriptorType) @@ -611,6 +610,7 @@ def test_dunder_get_signature(self): # gh-93021: Second parameter is optional self.assertIs(sig.parameters["owner"].default, None) + @unittest.expectedFailure # TODO: RUSTPYTHON def test_method_wrapper_types(self): self.assertIsInstance(object().__init__, types.MethodWrapperType) self.assertIsInstance(object().__str__, types.MethodWrapperType) @@ -836,8 +836,6 @@ def test_union_parameter_chaining(self): self.assertEqual((list[T] | list[S])[int, T], list[int] | list[T]) self.assertEqual((list[T] | list[S])[int, int], list[int]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_union_parameter_substitution(self): def eq(actual, expected, typed=True): self.assertEqual(actual, expected) @@ -2052,8 +2050,6 @@ async def corofunc(): else: self.fail('StopIteration was expected') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_gen(self): def gen_func(): yield 1 diff --git a/Lib/test/test_typing.py b/Lib/test/test_typing.py index a048c39cc9..74ab94eb5c 100644 --- a/Lib/test/test_typing.py +++ b/Lib/test/test_typing.py @@ -25,6 +25,7 @@ from typing import assert_type, cast, runtime_checkable from typing import get_type_hints from typing import get_origin, get_args, get_protocol_members +from typing import override from typing import is_typeddict, is_protocol from typing import reveal_type from typing import dataclass_transform @@ -48,8 +49,8 @@ from test.support.testcase import ExtraAssertions from test.typinganndata import ann_module695, mod_generics_cache, _typed_dict_helper -# TODO: RUSTPYTHON -import unittest +import unittest # XXX: RUSTPYTHON + CANNOT_SUBCLASS_TYPE = 'Cannot subclass special typing classes' NOT_A_BASE_TYPE = "type 'typing.%s' is not an acceptable base type" @@ -619,6 +620,7 @@ def test_constructor(self): self.assertIs(T.__contravariant__, False) self.assertIs(T.__infer_variance__, True) + class TypeParameterDefaultsTests(BaseTestCase): def test_typevar(self): T = TypeVar('T', default=int) @@ -804,13 +806,12 @@ def test_pickle(self): self.assertEqual(z.__default__, typevar.__default__) - def template_replace(templates: list[str], replacements: dict[str, list[str]]) -> list[tuple[str]]: """Renders templates with possible combinations of replacements. Example 1: Suppose that: templates = ["dog_breed are awesome", "dog_breed are cool"] - replacements = ["dog_breed": ["Huskies", "Beagles"]] + replacements = {"dog_breed": ["Huskies", "Beagles"]} Then we would return: [ ("Huskies are awesome", "Huskies are cool"), @@ -906,7 +907,6 @@ def test_no_duplicates_if_replacement_not_in_templates(self): self.assertEqual(actual, expected) - class GenericAliasSubstitutionTests(BaseTestCase): """Tests for type variable substitution in generic aliases. @@ -966,8 +966,7 @@ class C(Generic[T]): pass ) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_two_parameters(self): T1 = TypeVar('T1') T2 = TypeVar('T2') @@ -1065,13 +1064,14 @@ class C(Generic[T1, T2, T3]): pass eval(expected_str) ) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_variadic_parameters(self): T1 = TypeVar('T1') T2 = TypeVar('T2') Ts = TypeVarTuple('Ts') + class C(Generic[*Ts]): pass + generics = ['C', 'tuple', 'Tuple'] tuple_types = ['tuple', 'Tuple'] @@ -1169,12 +1169,10 @@ def test_variadic_parameters(self): - - class UnpackTests(BaseTestCase): def test_accepts_single_type(self): - # (*tuple[int],) + (*tuple[int],) Unpack[Tuple[int]] def test_dir(self): @@ -1228,8 +1226,6 @@ class New[*Ts]: ... self.assertEqual(PartNew[Unpack[tuple[str]]].__args__, (int, str)) self.assertEqual(PartNew[Unpack[Tuple[str]]].__args__, (int, str)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_unpack_wrong_type(self): Ts = TypeVarTuple("Ts") class Gen[*Ts]: ... @@ -1241,6 +1237,7 @@ class Gen[*Ts]: ... with self.assertRaisesRegex(TypeError, bad_unpack_param): PartGen[Unpack[List[int]]] + class TypeVarTupleTests(BaseTestCase): def test_name(self): @@ -1627,7 +1624,6 @@ def func3(*args: '*CustomVariadic[int, str]'): pass self.assertEqual(gth(func3, localns={'CustomVariadic': CustomVariadic}), {'args': Unpack[CustomVariadic[int, str]]}) - def test_tuple_args_are_correct(self): Ts = TypeVarTuple('Ts') @@ -1807,7 +1803,6 @@ def g(*args: *Ts): pass self.assertEqual(f.__annotations__, {'args': Unpack[Ts]}) self.assertEqual(g.__annotations__, {'args': (*Ts,)[0]}) - def test_variadic_args_with_ellipsis_annotations_are_correct(self): def a(*args: *tuple[int, ...]): pass self.assertEqual(a.__annotations__, @@ -1817,7 +1812,6 @@ def b(*args: Unpack[Tuple[int, ...]]): pass self.assertEqual(b.__annotations__, {'args': Unpack[Tuple[int, ...]]}) - def test_concatenation_in_variadic_args_annotations_are_correct(self): Ts = TypeVarTuple('Ts') @@ -1948,13 +1942,13 @@ class D(Generic[Unpack[Ts]]): pass self.assertNotEqual(C[*Ts1], C[*Ts2]) self.assertNotEqual(D[Unpack[Ts1]], D[Unpack[Ts2]]) + class TypeVarTuplePicklingTests(BaseTestCase): # These are slightly awkward tests to run, because TypeVarTuples are only # picklable if defined in the global scope. We therefore need to push # various things defined in these tests into the global scope with `global` # statements at the start of each test. - # TODO: RUSTPYTHON @all_pickle_protocols def test_pickling_then_unpickling_results_in_same_identity(self, proto): global global_Ts1 # See explanation at start of class. @@ -1998,7 +1992,6 @@ def test_pickling_then_unpickling_tuple_with_typevartuple_equality( self.assertEqual(t, t2) - class UnionTests(BaseTestCase): def test_basics(self): @@ -2616,7 +2609,6 @@ def test_errors(self): with self.assertRaisesRegex(TypeError, "few arguments for"): C1[int] - class TypingCallableTests(BaseCallableTests, BaseTestCase): Callable = typing.Callable @@ -2885,8 +2877,6 @@ def method(self) -> int: ... self.assertNotIsSubclass(NotImpl, Foo) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_pep695_generics_can_be_runtime_checkable(self): @runtime_checkable class HasX(Protocol): @@ -2905,7 +2895,6 @@ def __init__(self, y): self.assertIsInstance(Bar(1), HasX) self.assertNotIsInstance(Capybara('a'), HasX) - def test_everything_implements_empty_protocol(self): @runtime_checkable class Empty(Protocol): @@ -2950,8 +2939,23 @@ class E(C, BP): pass self.assertNotIsInstance(D(), E) self.assertNotIsInstance(E(), D) - # TODO: RUSTPYTHON - @unittest.expectedFailure + def test_inheritance_from_object(self): + # Inheritance from object is specifically allowed, unlike other nominal classes + class P(Protocol, object): + x: int + + self.assertEqual(typing.get_protocol_members(P), {'x'}) + + class OldGeneric(Protocol, Generic[T], object): + y: T + + self.assertEqual(typing.get_protocol_members(OldGeneric), {'y'}) + + class NewGeneric[T](Protocol, object): + z: T + + self.assertEqual(typing.get_protocol_members(NewGeneric), {'z'}) + def test_no_instantiation(self): class P(Protocol): pass @@ -2995,7 +2999,6 @@ class C: pass P.__init__(c, 1) self.assertEqual(c.x, 1) - def test_concrete_class_inheriting_init_from_protocol(self): class P(Protocol): x: int @@ -3152,7 +3155,6 @@ def x(self): ... with self.assertRaisesRegex(TypeError, only_classes_allowed): issubclass(1, BadPG) - def test_implicit_issubclass_between_two_protocols(self): @runtime_checkable class CallableMembersProto(Protocol): @@ -3180,255 +3182,1364 @@ class MixedMembersProto1(Protocol): meth: Callable[[], None] def meth2(self): ... - class MixedMembersProto2(Protocol): - def meth(self): ... - meth2: Callable[[str, int], bool] + class MixedMembersProto2(Protocol): + def meth(self): ... + meth2: Callable[[str, int], bool] + + for proto in ( + IdenticalProto, SupersetProto, NonCallableMembersProto, + NonCallableMembersSupersetProto, MixedMembersProto1, MixedMembersProto2 + ): + with self.subTest(proto=proto.__name__): + self.assertIsSubclass(proto, CallableMembersProto) + + # These two shouldn't be considered subclasses of CallableMembersProto, however, + # since they don't have the `meth` protocol member + + class EmptyProtocol(Protocol): ... + class UnrelatedProtocol(Protocol): + def wut(self): ... + + self.assertNotIsSubclass(EmptyProtocol, CallableMembersProto) + self.assertNotIsSubclass(UnrelatedProtocol, CallableMembersProto) + + # These aren't protocols at all (despite having annotations), + # so they should only be considered subclasses of CallableMembersProto + # if they *actually have an attribute* matching the `meth` member + # (just having an annotation is insufficient) + + class AnnotatedButNotAProtocol: + meth: Callable[[], None] + + class NotAProtocolButAnImplicitSubclass: + def meth(self): pass + + class NotAProtocolButAnImplicitSubclass2: + meth: Callable[[], None] + def meth(self): pass + + class NotAProtocolButAnImplicitSubclass3: + meth: Callable[[], None] + meth2: Callable[[int, str], bool] + def meth(self): pass + def meth2(self, x, y): return True + + self.assertNotIsSubclass(AnnotatedButNotAProtocol, CallableMembersProto) + self.assertIsSubclass(NotAProtocolButAnImplicitSubclass, CallableMembersProto) + self.assertIsSubclass(NotAProtocolButAnImplicitSubclass2, CallableMembersProto) + self.assertIsSubclass(NotAProtocolButAnImplicitSubclass3, CallableMembersProto) + + @unittest.skip('TODO: RUSTPYTHON; (no gc)') + def test_isinstance_checks_not_at_whim_of_gc(self): + self.addCleanup(gc.enable) + gc.disable() + + with self.assertRaisesRegex( + TypeError, + "Protocols can only inherit from other protocols" + ): + class Foo(collections.abc.Mapping, Protocol): + pass + + self.assertNotIsInstance([], collections.abc.Mapping) + + def test_issubclass_and_isinstance_on_Protocol_itself(self): + class C: + def x(self): pass + + self.assertNotIsSubclass(object, Protocol) + self.assertNotIsInstance(object(), Protocol) + + self.assertNotIsSubclass(str, Protocol) + self.assertNotIsInstance('foo', Protocol) + + self.assertNotIsSubclass(C, Protocol) + self.assertNotIsInstance(C(), Protocol) + + only_classes_allowed = r"issubclass\(\) arg 1 must be a class" + + with self.assertRaisesRegex(TypeError, only_classes_allowed): + issubclass(1, Protocol) + with self.assertRaisesRegex(TypeError, only_classes_allowed): + issubclass('foo', Protocol) + with self.assertRaisesRegex(TypeError, only_classes_allowed): + issubclass(C(), Protocol) + + T = TypeVar('T') + + @runtime_checkable + class EmptyProtocol(Protocol): pass + + @runtime_checkable + class SupportsStartsWith(Protocol): + def startswith(self, x: str) -> bool: ... + + @runtime_checkable + class SupportsX(Protocol[T]): + def x(self): ... + + for proto in EmptyProtocol, SupportsStartsWith, SupportsX: + with self.subTest(proto=proto.__name__): + self.assertIsSubclass(proto, Protocol) + + # gh-105237 / PR #105239: + # check that the presence of Protocol subclasses + # where `issubclass(X, <subclass>)` evaluates to True + # doesn't influence the result of `issubclass(X, Protocol)` + + self.assertIsSubclass(object, EmptyProtocol) + self.assertIsInstance(object(), EmptyProtocol) + self.assertNotIsSubclass(object, Protocol) + self.assertNotIsInstance(object(), Protocol) + + self.assertIsSubclass(str, SupportsStartsWith) + self.assertIsInstance('foo', SupportsStartsWith) + self.assertNotIsSubclass(str, Protocol) + self.assertNotIsInstance('foo', Protocol) + + self.assertIsSubclass(C, SupportsX) + self.assertIsInstance(C(), SupportsX) + self.assertNotIsSubclass(C, Protocol) + self.assertNotIsInstance(C(), Protocol) + + def test_protocols_issubclass_non_callable(self): + class C: + x = 1 + + @runtime_checkable + class PNonCall(Protocol): + x = 1 + + non_callable_members_illegal = ( + "Protocols with non-method members don't support issubclass()" + ) + + with self.assertRaisesRegex(TypeError, non_callable_members_illegal): + issubclass(C, PNonCall) + + self.assertIsInstance(C(), PNonCall) + PNonCall.register(C) + + with self.assertRaisesRegex(TypeError, non_callable_members_illegal): + issubclass(C, PNonCall) + + self.assertIsInstance(C(), PNonCall) + + # check that non-protocol subclasses are not affected + class D(PNonCall): ... + + self.assertNotIsSubclass(C, D) + self.assertNotIsInstance(C(), D) + D.register(C) + self.assertIsSubclass(C, D) + self.assertIsInstance(C(), D) + + with self.assertRaisesRegex(TypeError, non_callable_members_illegal): + issubclass(D, PNonCall) + + def test_no_weird_caching_with_issubclass_after_isinstance(self): + @runtime_checkable + class Spam(Protocol): + x: int + + class Eggs: + def __init__(self) -> None: + self.x = 42 + + self.assertIsInstance(Eggs(), Spam) + + # gh-104555: If we didn't override ABCMeta.__subclasscheck__ in _ProtocolMeta, + # TypeError wouldn't be raised here, + # as the cached result of the isinstance() check immediately above + # would mean the issubclass() call would short-circuit + # before we got to the "raise TypeError" line + with self.assertRaisesRegex( + TypeError, + "Protocols with non-method members don't support issubclass()" + ): + issubclass(Eggs, Spam) + + def test_no_weird_caching_with_issubclass_after_isinstance_2(self): + @runtime_checkable + class Spam(Protocol): + x: int + + class Eggs: ... + + self.assertNotIsInstance(Eggs(), Spam) + + # gh-104555: If we didn't override ABCMeta.__subclasscheck__ in _ProtocolMeta, + # TypeError wouldn't be raised here, + # as the cached result of the isinstance() check immediately above + # would mean the issubclass() call would short-circuit + # before we got to the "raise TypeError" line + with self.assertRaisesRegex( + TypeError, + "Protocols with non-method members don't support issubclass()" + ): + issubclass(Eggs, Spam) + + def test_no_weird_caching_with_issubclass_after_isinstance_3(self): + @runtime_checkable + class Spam(Protocol): + x: int + + class Eggs: + def __getattr__(self, attr): + if attr == "x": + return 42 + raise AttributeError(attr) + + self.assertNotIsInstance(Eggs(), Spam) + + # gh-104555: If we didn't override ABCMeta.__subclasscheck__ in _ProtocolMeta, + # TypeError wouldn't be raised here, + # as the cached result of the isinstance() check immediately above + # would mean the issubclass() call would short-circuit + # before we got to the "raise TypeError" line + with self.assertRaisesRegex( + TypeError, + "Protocols with non-method members don't support issubclass()" + ): + issubclass(Eggs, Spam) + + def test_no_weird_caching_with_issubclass_after_isinstance_pep695(self): + @runtime_checkable + class Spam[T](Protocol): + x: T + + class Eggs[T]: + def __init__(self, x: T) -> None: + self.x = x + + self.assertIsInstance(Eggs(42), Spam) + + # gh-104555: If we didn't override ABCMeta.__subclasscheck__ in _ProtocolMeta, + # TypeError wouldn't be raised here, + # as the cached result of the isinstance() check immediately above + # would mean the issubclass() call would short-circuit + # before we got to the "raise TypeError" line + with self.assertRaisesRegex( + TypeError, + "Protocols with non-method members don't support issubclass()" + ): + issubclass(Eggs, Spam) + + def test_protocols_isinstance(self): + T = TypeVar('T') + + @runtime_checkable + class P(Protocol): + def meth(x): ... + + @runtime_checkable + class PG(Protocol[T]): + def meth(x): ... + + @runtime_checkable + class WeirdProto(Protocol): + meth = str.maketrans + + @runtime_checkable + class WeirdProto2(Protocol): + meth = lambda *args, **kwargs: None + + class CustomCallable: + def __call__(self, *args, **kwargs): + pass + + @runtime_checkable + class WeirderProto(Protocol): + meth = CustomCallable() + + class BadP(Protocol): + def meth(x): ... + + class BadPG(Protocol[T]): + def meth(x): ... + + class C: + def meth(x): ... + + class C2: + def __init__(self): + self.meth = lambda: None + + for klass in C, C2: + for proto in P, PG, WeirdProto, WeirdProto2, WeirderProto: + with self.subTest(klass=klass.__name__, proto=proto.__name__): + self.assertIsInstance(klass(), proto) + + no_subscripted_generics = "Subscripted generics cannot be used with class and instance checks" + + with self.assertRaisesRegex(TypeError, no_subscripted_generics): + isinstance(C(), PG[T]) + with self.assertRaisesRegex(TypeError, no_subscripted_generics): + isinstance(C(), PG[C]) + + only_runtime_checkable_msg = ( + "Instance and class checks can only be used " + "with @runtime_checkable protocols" + ) + + with self.assertRaisesRegex(TypeError, only_runtime_checkable_msg): + isinstance(C(), BadP) + with self.assertRaisesRegex(TypeError, only_runtime_checkable_msg): + isinstance(C(), BadPG) + + def test_protocols_isinstance_properties_and_descriptors(self): + class C: + @property + def attr(self): + return 42 + + class CustomDescriptor: + def __get__(self, obj, objtype=None): + return 42 + + class D: + attr = CustomDescriptor() + + # Check that properties set on superclasses + # are still found by the isinstance() logic + class E(C): ... + class F(D): ... + + class Empty: ... + + T = TypeVar('T') + + @runtime_checkable + class P(Protocol): + @property + def attr(self): ... + + @runtime_checkable + class P1(Protocol): + attr: int + + @runtime_checkable + class PG(Protocol[T]): + @property + def attr(self): ... + + @runtime_checkable + class PG1(Protocol[T]): + attr: T + + @runtime_checkable + class MethodP(Protocol): + def attr(self): ... + + @runtime_checkable + class MethodPG(Protocol[T]): + def attr(self) -> T: ... + + for protocol_class in P, P1, PG, PG1, MethodP, MethodPG: + for klass in C, D, E, F: + with self.subTest( + klass=klass.__name__, + protocol_class=protocol_class.__name__ + ): + self.assertIsInstance(klass(), protocol_class) + + with self.subTest(klass="Empty", protocol_class=protocol_class.__name__): + self.assertNotIsInstance(Empty(), protocol_class) + + class BadP(Protocol): + @property + def attr(self): ... + + class BadP1(Protocol): + attr: int + + class BadPG(Protocol[T]): + @property + def attr(self): ... + + class BadPG1(Protocol[T]): + attr: T + + cases = ( + PG[T], PG[C], PG1[T], PG1[C], MethodPG[T], + MethodPG[C], BadP, BadP1, BadPG, BadPG1 + ) + + for obj in cases: + for klass in C, D, E, F, Empty: + with self.subTest(klass=klass.__name__, obj=obj): + with self.assertRaises(TypeError): + isinstance(klass(), obj) + + def test_protocols_isinstance_not_fooled_by_custom_dir(self): + @runtime_checkable + class HasX(Protocol): + x: int + + class CustomDirWithX: + x = 10 + def __dir__(self): + return [] + + class CustomDirWithoutX: + def __dir__(self): + return ["x"] + + self.assertIsInstance(CustomDirWithX(), HasX) + self.assertNotIsInstance(CustomDirWithoutX(), HasX) + + def test_protocols_isinstance_attribute_access_with_side_effects(self): + class C: + @property + def attr(self): + raise AttributeError('no') + + class CustomDescriptor: + def __get__(self, obj, objtype=None): + raise RuntimeError("NO") + + class D: + attr = CustomDescriptor() + + # Check that properties set on superclasses + # are still found by the isinstance() logic + class E(C): ... + class F(D): ... + + class WhyWouldYouDoThis: + def __getattr__(self, name): + raise RuntimeError("wut") + + T = TypeVar('T') + + @runtime_checkable + class P(Protocol): + @property + def attr(self): ... + + @runtime_checkable + class P1(Protocol): + attr: int + + @runtime_checkable + class PG(Protocol[T]): + @property + def attr(self): ... + + @runtime_checkable + class PG1(Protocol[T]): + attr: T + + @runtime_checkable + class MethodP(Protocol): + def attr(self): ... + + @runtime_checkable + class MethodPG(Protocol[T]): + def attr(self) -> T: ... + + for protocol_class in P, P1, PG, PG1, MethodP, MethodPG: + for klass in C, D, E, F: + with self.subTest( + klass=klass.__name__, + protocol_class=protocol_class.__name__ + ): + self.assertIsInstance(klass(), protocol_class) + + with self.subTest( + klass="WhyWouldYouDoThis", + protocol_class=protocol_class.__name__ + ): + self.assertNotIsInstance(WhyWouldYouDoThis(), protocol_class) + + def test_protocols_isinstance___slots__(self): + # As per the consensus in https://github.com/python/typing/issues/1367, + # this is desirable behaviour + @runtime_checkable + class HasX(Protocol): + x: int + + class HasNothingButSlots: + __slots__ = ("x",) + + self.assertIsInstance(HasNothingButSlots(), HasX) + + def test_protocols_isinstance_py36(self): + class APoint: + def __init__(self, x, y, label): + self.x = x + self.y = y + self.label = label + + class BPoint: + label = 'B' + + def __init__(self, x, y): + self.x = x + self.y = y + + class C: + def __init__(self, attr): + self.attr = attr + + def meth(self, arg): + return 0 + + class Bad: pass + + self.assertIsInstance(APoint(1, 2, 'A'), Point) + self.assertIsInstance(BPoint(1, 2), Point) + self.assertNotIsInstance(MyPoint(), Point) + self.assertIsInstance(BPoint(1, 2), Position) + self.assertIsInstance(Other(), Proto) + self.assertIsInstance(Concrete(), Proto) + self.assertIsInstance(C(42), Proto) + self.assertNotIsInstance(Bad(), Proto) + self.assertNotIsInstance(Bad(), Point) + self.assertNotIsInstance(Bad(), Position) + self.assertNotIsInstance(Bad(), Concrete) + self.assertNotIsInstance(Other(), Concrete) + self.assertIsInstance(NT(1, 2), Position) + + def test_protocols_isinstance_init(self): + T = TypeVar('T') + + @runtime_checkable + class P(Protocol): + x = 1 + + @runtime_checkable + class PG(Protocol[T]): + x = 1 + + class C: + def __init__(self, x): + self.x = x + + self.assertIsInstance(C(1), P) + self.assertIsInstance(C(1), PG) + + def test_protocols_isinstance_monkeypatching(self): + @runtime_checkable + class HasX(Protocol): + x: int + + class Foo: ... + + f = Foo() + self.assertNotIsInstance(f, HasX) + f.x = 42 + self.assertIsInstance(f, HasX) + del f.x + self.assertNotIsInstance(f, HasX) + + def test_protocol_checks_after_subscript(self): + class P(Protocol[T]): pass + class C(P[T]): pass + class Other1: pass + class Other2: pass + CA = C[Any] + + self.assertNotIsInstance(Other1(), C) + self.assertNotIsSubclass(Other2, C) + + class D1(C[Any]): pass + class D2(C[Any]): pass + CI = C[int] + + self.assertIsInstance(D1(), C) + self.assertIsSubclass(D2, C) + + def test_protocols_support_register(self): + @runtime_checkable + class P(Protocol): + x = 1 + + class PM(Protocol): + def meth(self): pass + + class D(PM): pass + + class C: pass + + D.register(C) + P.register(C) + self.assertIsInstance(C(), P) + self.assertIsInstance(C(), D) + + def test_none_on_non_callable_doesnt_block_implementation(self): + @runtime_checkable + class P(Protocol): + x = 1 + + class A: + x = 1 + + class B(A): + x = None + + class C: + def __init__(self): + self.x = None + + self.assertIsInstance(B(), P) + self.assertIsInstance(C(), P) + + def test_none_on_callable_blocks_implementation(self): + @runtime_checkable + class P(Protocol): + def x(self): ... + + class A: + def x(self): ... + + class B(A): + x = None + + class C: + def __init__(self): + self.x = None + + self.assertNotIsInstance(B(), P) + self.assertNotIsInstance(C(), P) + + def test_non_protocol_subclasses(self): + class P(Protocol): + x = 1 + + @runtime_checkable + class PR(Protocol): + def meth(self): pass + + class NonP(P): + x = 1 + + class NonPR(PR): pass + + class C(metaclass=abc.ABCMeta): + x = 1 + + class D(metaclass=abc.ABCMeta): + def meth(self): pass + + self.assertNotIsInstance(C(), NonP) + self.assertNotIsInstance(D(), NonPR) + self.assertNotIsSubclass(C, NonP) + self.assertNotIsSubclass(D, NonPR) + self.assertIsInstance(NonPR(), PR) + self.assertIsSubclass(NonPR, PR) + + self.assertNotIn("__protocol_attrs__", vars(NonP)) + self.assertNotIn("__protocol_attrs__", vars(NonPR)) + self.assertNotIn("__non_callable_proto_members__", vars(NonP)) + self.assertNotIn("__non_callable_proto_members__", vars(NonPR)) + + self.assertEqual(get_protocol_members(P), {"x"}) + self.assertEqual(get_protocol_members(PR), {"meth"}) + + # the returned object should be immutable, + # and should be a different object to the original attribute + # to prevent users from (accidentally or deliberately) + # mutating the attribute on the original class + self.assertIsInstance(get_protocol_members(P), frozenset) + self.assertIsNot(get_protocol_members(P), P.__protocol_attrs__) + self.assertIsInstance(get_protocol_members(PR), frozenset) + self.assertIsNot(get_protocol_members(PR), P.__protocol_attrs__) + + acceptable_extra_attrs = { + '_is_protocol', '_is_runtime_protocol', '__parameters__', + '__init__', '__annotations__', '__subclasshook__', + } + self.assertLessEqual(vars(NonP).keys(), vars(C).keys() | acceptable_extra_attrs) + self.assertLessEqual( + vars(NonPR).keys(), vars(D).keys() | acceptable_extra_attrs + ) + + def test_custom_subclasshook(self): + class P(Protocol): + x = 1 + + class OKClass: pass + + class BadClass: + x = 1 + + class C(P): + @classmethod + def __subclasshook__(cls, other): + return other.__name__.startswith("OK") + + self.assertIsInstance(OKClass(), C) + self.assertNotIsInstance(BadClass(), C) + self.assertIsSubclass(OKClass, C) + self.assertNotIsSubclass(BadClass, C) + + def test_custom_subclasshook_2(self): + @runtime_checkable + class HasX(Protocol): + # The presence of a non-callable member + # would mean issubclass() checks would fail with TypeError + # if it weren't for the custom `__subclasshook__` method + x = 1 + + @classmethod + def __subclasshook__(cls, other): + return hasattr(other, 'x') + + class Empty: pass + + class ImplementsHasX: + x = 1 + + self.assertIsInstance(ImplementsHasX(), HasX) + self.assertNotIsInstance(Empty(), HasX) + self.assertIsSubclass(ImplementsHasX, HasX) + self.assertNotIsSubclass(Empty, HasX) + + # isinstance() and issubclass() checks against this still raise TypeError, + # despite the presence of the custom __subclasshook__ method, + # as it's not decorated with @runtime_checkable + class NotRuntimeCheckable(Protocol): + @classmethod + def __subclasshook__(cls, other): + return hasattr(other, 'x') + + must_be_runtime_checkable = ( + "Instance and class checks can only be used " + "with @runtime_checkable protocols" + ) + + with self.assertRaisesRegex(TypeError, must_be_runtime_checkable): + issubclass(object, NotRuntimeCheckable) + with self.assertRaisesRegex(TypeError, must_be_runtime_checkable): + isinstance(object(), NotRuntimeCheckable) + + def test_issubclass_fails_correctly(self): + @runtime_checkable + class NonCallableMembers(Protocol): + x = 1 + + class NotRuntimeCheckable(Protocol): + def callable_member(self) -> int: ... + + @runtime_checkable + class RuntimeCheckable(Protocol): + def callable_member(self) -> int: ... + + class C: pass + + # These three all exercise different code paths, + # but should result in the same error message: + for protocol in NonCallableMembers, NotRuntimeCheckable, RuntimeCheckable: + with self.subTest(proto_name=protocol.__name__): + with self.assertRaisesRegex( + TypeError, r"issubclass\(\) arg 1 must be a class" + ): + issubclass(C(), protocol) + + def test_defining_generic_protocols(self): + T = TypeVar('T') + S = TypeVar('S') + + @runtime_checkable + class PR(Protocol[T, S]): + def meth(self): pass + + class P(PR[int, T], Protocol[T]): + y = 1 + + with self.assertRaises(TypeError): + PR[int] + with self.assertRaises(TypeError): + P[int, str] + + class C(PR[int, T]): pass + + self.assertIsInstance(C[str](), C) + + def test_defining_generic_protocols_old_style(self): + T = TypeVar('T') + S = TypeVar('S') + + @runtime_checkable + class PR(Protocol, Generic[T, S]): + def meth(self): pass + + class P(PR[int, str], Protocol): + y = 1 + + with self.assertRaises(TypeError): + issubclass(PR[int, str], PR) + self.assertIsSubclass(P, PR) + with self.assertRaises(TypeError): + PR[int] + + class P1(Protocol, Generic[T]): + def bar(self, x: T) -> str: ... + + class P2(Generic[T], Protocol): + def bar(self, x: T) -> str: ... + + @runtime_checkable + class PSub(P1[str], Protocol): + x = 1 + + class Test: + x = 1 + + def bar(self, x: str) -> str: + return x + + self.assertIsInstance(Test(), PSub) + + def test_pep695_generic_protocol_callable_members(self): + @runtime_checkable + class Foo[T](Protocol): + def meth(self, x: T) -> None: ... + + class Bar[T]: + def meth(self, x: T) -> None: ... + + self.assertIsInstance(Bar(), Foo) + self.assertIsSubclass(Bar, Foo) + + @runtime_checkable + class SupportsTrunc[T](Protocol): + def __trunc__(self) -> T: ... + + self.assertIsInstance(0.0, SupportsTrunc) + self.assertIsSubclass(float, SupportsTrunc) + + def test_init_called(self): + T = TypeVar('T') + + class P(Protocol[T]): pass + + class C(P[T]): + def __init__(self): + self.test = 'OK' + + self.assertEqual(C[int]().test, 'OK') + + class B: + def __init__(self): + self.test = 'OK' + + class D1(B, P[T]): + pass + + self.assertEqual(D1[int]().test, 'OK') + + class D2(P[T], B): + pass + + self.assertEqual(D2[int]().test, 'OK') + + def test_new_called(self): + T = TypeVar('T') + + class P(Protocol[T]): pass + + class C(P[T]): + def __new__(cls, *args): + self = super().__new__(cls, *args) + self.test = 'OK' + return self + + self.assertEqual(C[int]().test, 'OK') + with self.assertRaises(TypeError): + C[int](42) + with self.assertRaises(TypeError): + C[int](a=42) + + def test_protocols_bad_subscripts(self): + T = TypeVar('T') + S = TypeVar('S') + with self.assertRaises(TypeError): + class P(Protocol[T, T]): pass + with self.assertRaises(TypeError): + class Q(Protocol[int]): pass + with self.assertRaises(TypeError): + class R(Protocol[T], Protocol[S]): pass + with self.assertRaises(TypeError): + class S(typing.Mapping[T, S], Protocol[T]): pass + + def test_generic_protocols_repr(self): + T = TypeVar('T') + S = TypeVar('S') + + class P(Protocol[T, S]): pass + + self.assertTrue(repr(P[T, S]).endswith('P[~T, ~S]')) + self.assertTrue(repr(P[int, str]).endswith('P[int, str]')) + + def test_generic_protocols_eq(self): + T = TypeVar('T') + S = TypeVar('S') + + class P(Protocol[T, S]): pass + + self.assertEqual(P, P) + self.assertEqual(P[int, T], P[int, T]) + self.assertEqual(P[T, T][Tuple[T, S]][int, str], + P[Tuple[int, str], Tuple[int, str]]) + + def test_generic_protocols_special_from_generic(self): + T = TypeVar('T') + + class P(Protocol[T]): pass + + self.assertEqual(P.__parameters__, (T,)) + self.assertEqual(P[int].__parameters__, ()) + self.assertEqual(P[int].__args__, (int,)) + self.assertIs(P[int].__origin__, P) + + def test_generic_protocols_special_from_protocol(self): + @runtime_checkable + class PR(Protocol): + x = 1 + + class P(Protocol): + def meth(self): + pass + + T = TypeVar('T') + + class PG(Protocol[T]): + x = 1 + + def meth(self): + pass + + self.assertTrue(P._is_protocol) + self.assertTrue(PR._is_protocol) + self.assertTrue(PG._is_protocol) + self.assertFalse(P._is_runtime_protocol) + self.assertTrue(PR._is_runtime_protocol) + self.assertTrue(PG[int]._is_protocol) + self.assertEqual(typing._get_protocol_attrs(P), {'meth'}) + self.assertEqual(typing._get_protocol_attrs(PR), {'x'}) + self.assertEqual(frozenset(typing._get_protocol_attrs(PG)), + frozenset({'x', 'meth'})) + + def test_no_runtime_deco_on_nominal(self): + with self.assertRaises(TypeError): + @runtime_checkable + class C: pass + + class Proto(Protocol): + x = 1 + + with self.assertRaises(TypeError): + @runtime_checkable + class Concrete(Proto): + pass + + def test_none_treated_correctly(self): + @runtime_checkable + class P(Protocol): + x = None # type: int + + class B(object): pass + + self.assertNotIsInstance(B(), P) + + class C: + x = 1 + + class D: + x = None + + self.assertIsInstance(C(), P) + self.assertIsInstance(D(), P) + + class CI: + def __init__(self): + self.x = 1 + + class DI: + def __init__(self): + self.x = None + + self.assertIsInstance(CI(), P) + self.assertIsInstance(DI(), P) + + def test_protocols_in_unions(self): + class P(Protocol): + x = None # type: int + + Alias = typing.Union[typing.Iterable, P] + Alias2 = typing.Union[P, typing.Iterable] + self.assertEqual(Alias, Alias2) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_protocols_pickleable(self): + global P, CP # pickle wants to reference the class by name + T = TypeVar('T') + + @runtime_checkable + class P(Protocol[T]): + x = 1 + + class CP(P[int]): + pass + + c = CP() + c.foo = 42 + c.bar = 'abc' + for proto in range(pickle.HIGHEST_PROTOCOL + 1): + z = pickle.dumps(c, proto) + x = pickle.loads(z) + self.assertEqual(x.foo, 42) + self.assertEqual(x.bar, 'abc') + self.assertEqual(x.x, 1) + self.assertEqual(x.__dict__, {'foo': 42, 'bar': 'abc'}) + s = pickle.dumps(P, proto) + D = pickle.loads(s) + + class E: + x = 1 + + self.assertIsInstance(E(), D) + + def test_runtime_checkable_with_match_args(self): + @runtime_checkable + class P_regular(Protocol): + x: int + y: int + + @runtime_checkable + class P_match(Protocol): + __match_args__ = ('x', 'y') + x: int + y: int + + class Regular: + def __init__(self, x: int, y: int): + self.x = x + self.y = y + + class WithMatch: + __match_args__ = ('x', 'y', 'z') + def __init__(self, x: int, y: int, z: int): + self.x = x + self.y = y + self.z = z + + class Nope: ... + + self.assertIsInstance(Regular(1, 2), P_regular) + self.assertIsInstance(Regular(1, 2), P_match) + self.assertIsInstance(WithMatch(1, 2, 3), P_regular) + self.assertIsInstance(WithMatch(1, 2, 3), P_match) + self.assertNotIsInstance(Nope(), P_regular) + self.assertNotIsInstance(Nope(), P_match) + + def test_supports_int(self): + self.assertIsSubclass(int, typing.SupportsInt) + self.assertNotIsSubclass(str, typing.SupportsInt) - for proto in ( - IdenticalProto, SupersetProto, NonCallableMembersProto, - NonCallableMembersSupersetProto, MixedMembersProto1, MixedMembersProto2 - ): - with self.subTest(proto=proto.__name__): - self.assertIsSubclass(proto, CallableMembersProto) + def test_supports_float(self): + self.assertIsSubclass(float, typing.SupportsFloat) + self.assertNotIsSubclass(str, typing.SupportsFloat) - # These two shouldn't be considered subclasses of CallableMembersProto, however, - # since they don't have the `meth` protocol member + def test_supports_complex(self): - class EmptyProtocol(Protocol): ... - class UnrelatedProtocol(Protocol): - def wut(self): ... + class C: + def __complex__(self): + return 0j - self.assertNotIsSubclass(EmptyProtocol, CallableMembersProto) - self.assertNotIsSubclass(UnrelatedProtocol, CallableMembersProto) + self.assertIsSubclass(complex, typing.SupportsComplex) + self.assertIsSubclass(C, typing.SupportsComplex) + self.assertNotIsSubclass(str, typing.SupportsComplex) - # These aren't protocols at all (despite having annotations), - # so they should only be considered subclasses of CallableMembersProto - # if they *actually have an attribute* matching the `meth` member - # (just having an annotation is insufficient) + def test_supports_bytes(self): - class AnnotatedButNotAProtocol: - meth: Callable[[], None] + class B: + def __bytes__(self): + return b'' + + self.assertIsSubclass(bytes, typing.SupportsBytes) + self.assertIsSubclass(B, typing.SupportsBytes) + self.assertNotIsSubclass(str, typing.SupportsBytes) + + def test_supports_abs(self): + self.assertIsSubclass(float, typing.SupportsAbs) + self.assertIsSubclass(int, typing.SupportsAbs) + self.assertNotIsSubclass(str, typing.SupportsAbs) + + def test_supports_round(self): + issubclass(float, typing.SupportsRound) + self.assertIsSubclass(float, typing.SupportsRound) + self.assertIsSubclass(int, typing.SupportsRound) + self.assertNotIsSubclass(str, typing.SupportsRound) + + def test_reversible(self): + self.assertIsSubclass(list, typing.Reversible) + self.assertNotIsSubclass(int, typing.Reversible) + + def test_supports_index(self): + self.assertIsSubclass(int, typing.SupportsIndex) + self.assertNotIsSubclass(str, typing.SupportsIndex) + + def test_bundled_protocol_instance_works(self): + self.assertIsInstance(0, typing.SupportsAbs) + class C1(typing.SupportsInt): + def __int__(self) -> int: + return 42 + class C2(C1): + pass + c = C2() + self.assertIsInstance(c, C1) - class NotAProtocolButAnImplicitSubclass: - def meth(self): pass + def test_collections_protocols_allowed(self): + @runtime_checkable + class Custom(collections.abc.Iterable, Protocol): + def close(self): ... - class NotAProtocolButAnImplicitSubclass2: - meth: Callable[[], None] - def meth(self): pass + class A: pass + class B: + def __iter__(self): + return [] + def close(self): + return 0 - class NotAProtocolButAnImplicitSubclass3: - meth: Callable[[], None] - meth2: Callable[[int, str], bool] - def meth(self): pass - def meth2(self, x, y): return True + self.assertIsSubclass(B, Custom) + self.assertNotIsSubclass(A, Custom) - self.assertNotIsSubclass(AnnotatedButNotAProtocol, CallableMembersProto) - self.assertIsSubclass(NotAProtocolButAnImplicitSubclass, CallableMembersProto) - self.assertIsSubclass(NotAProtocolButAnImplicitSubclass2, CallableMembersProto) - self.assertIsSubclass(NotAProtocolButAnImplicitSubclass3, CallableMembersProto) + @runtime_checkable + class ReleasableBuffer(collections.abc.Buffer, Protocol): + def __release_buffer__(self, mv: memoryview) -> None: ... - # TODO: RUSTPYTHON - @unittest.skip("TODO: RUSTPYTHON (no gc)") - def test_isinstance_checks_not_at_whim_of_gc(self): - self.addCleanup(gc.enable) - gc.disable() + class C: pass + class D: + def __buffer__(self, flags: int) -> memoryview: + return memoryview(b'') + def __release_buffer__(self, mv: memoryview) -> None: + pass - with self.assertRaisesRegex( - TypeError, - "Protocols can only inherit from other protocols" - ): - class Foo(collections.abc.Mapping, Protocol): + self.assertIsSubclass(D, ReleasableBuffer) + self.assertIsInstance(D(), ReleasableBuffer) + self.assertNotIsSubclass(C, ReleasableBuffer) + self.assertNotIsInstance(C(), ReleasableBuffer) + + def test_builtin_protocol_allowlist(self): + with self.assertRaises(TypeError): + class CustomProtocol(TestCase, Protocol): pass - self.assertNotIsInstance([], collections.abc.Mapping) + class CustomContextManager(typing.ContextManager, Protocol): + pass - def test_issubclass_and_isinstance_on_Protocol_itself(self): - class C: - def x(self): pass + class CustomAsyncIterator(typing.AsyncIterator, Protocol): + pass - self.assertNotIsSubclass(object, Protocol) - self.assertNotIsInstance(object(), Protocol) + def test_non_runtime_protocol_isinstance_check(self): + class P(Protocol): + x: int - self.assertNotIsSubclass(str, Protocol) - self.assertNotIsInstance('foo', Protocol) + with self.assertRaisesRegex(TypeError, "@runtime_checkable"): + isinstance(1, P) - self.assertNotIsSubclass(C, Protocol) - self.assertNotIsInstance(C(), Protocol) + def test_super_call_init(self): + class P(Protocol): + x: int - only_classes_allowed = r"issubclass\(\) arg 1 must be a class" + class Foo(P): + def __init__(self): + super().__init__() - with self.assertRaisesRegex(TypeError, only_classes_allowed): - issubclass(1, Protocol) - with self.assertRaisesRegex(TypeError, only_classes_allowed): - issubclass('foo', Protocol) - with self.assertRaisesRegex(TypeError, only_classes_allowed): - issubclass(C(), Protocol) + Foo() # Previously triggered RecursionError - T = TypeVar('T') + def test_get_protocol_members(self): + with self.assertRaisesRegex(TypeError, "not a Protocol"): + get_protocol_members(object) + with self.assertRaisesRegex(TypeError, "not a Protocol"): + get_protocol_members(object()) + with self.assertRaisesRegex(TypeError, "not a Protocol"): + get_protocol_members(Protocol) + with self.assertRaisesRegex(TypeError, "not a Protocol"): + get_protocol_members(Generic) - @runtime_checkable - class EmptyProtocol(Protocol): pass + class P(Protocol): + a: int + def b(self) -> str: ... + @property + def c(self) -> int: ... - @runtime_checkable - class SupportsStartsWith(Protocol): - def startswith(self, x: str) -> bool: ... + self.assertEqual(get_protocol_members(P), {'a', 'b', 'c'}) + self.assertIsInstance(get_protocol_members(P), frozenset) + self.assertIsNot(get_protocol_members(P), P.__protocol_attrs__) - @runtime_checkable - class SupportsX(Protocol[T]): - def x(self): ... + class Concrete: + a: int + def b(self) -> str: return "capybara" + @property + def c(self) -> int: return 5 - for proto in EmptyProtocol, SupportsStartsWith, SupportsX: - with self.subTest(proto=proto.__name__): - self.assertIsSubclass(proto, Protocol) + with self.assertRaisesRegex(TypeError, "not a Protocol"): + get_protocol_members(Concrete) + with self.assertRaisesRegex(TypeError, "not a Protocol"): + get_protocol_members(Concrete()) - # gh-105237 / PR #105239: - # check that the presence of Protocol subclasses - # where `issubclass(X, <subclass>)` evaluates to True - # doesn't influence the result of `issubclass(X, Protocol)` + class ConcreteInherit(P): + a: int = 42 + def b(self) -> str: return "capybara" + @property + def c(self) -> int: return 5 - self.assertIsSubclass(object, EmptyProtocol) - self.assertIsInstance(object(), EmptyProtocol) - self.assertNotIsSubclass(object, Protocol) - self.assertNotIsInstance(object(), Protocol) + with self.assertRaisesRegex(TypeError, "not a Protocol"): + get_protocol_members(ConcreteInherit) + with self.assertRaisesRegex(TypeError, "not a Protocol"): + get_protocol_members(ConcreteInherit()) - self.assertIsSubclass(str, SupportsStartsWith) - self.assertIsInstance('foo', SupportsStartsWith) - self.assertNotIsSubclass(str, Protocol) - self.assertNotIsInstance('foo', Protocol) + def test_is_protocol(self): + self.assertTrue(is_protocol(Proto)) + self.assertTrue(is_protocol(Point)) + self.assertFalse(is_protocol(Concrete)) + self.assertFalse(is_protocol(Concrete())) + self.assertFalse(is_protocol(Generic)) + self.assertFalse(is_protocol(object)) - self.assertIsSubclass(C, SupportsX) - self.assertIsInstance(C(), SupportsX) - self.assertNotIsSubclass(C, Protocol) - self.assertNotIsInstance(C(), Protocol) + # Protocol is not itself a protocol + self.assertFalse(is_protocol(Protocol)) - def test_protocols_issubclass_non_callable(self): - class C: - x = 1 + def test_interaction_with_isinstance_checks_on_superclasses_with_ABCMeta(self): + # Ensure the cache is empty, or this test won't work correctly + collections.abc.Sized._abc_registry_clear() - @runtime_checkable - class PNonCall(Protocol): - x = 1 + class Foo(collections.abc.Sized, Protocol): pass - non_callable_members_illegal = ( - "Protocols with non-method members don't support issubclass()" - ) + # gh-105144: this previously raised TypeError + # if a Protocol subclass of Sized had been created + # before any isinstance() checks against Sized + self.assertNotIsInstance(1, collections.abc.Sized) - with self.assertRaisesRegex(TypeError, non_callable_members_illegal): - issubclass(C, PNonCall) + def test_interaction_with_isinstance_checks_on_superclasses_with_ABCMeta_2(self): + # Ensure the cache is empty, or this test won't work correctly + collections.abc.Sized._abc_registry_clear() - self.assertIsInstance(C(), PNonCall) - PNonCall.register(C) + class Foo(typing.Sized, Protocol): pass - with self.assertRaisesRegex(TypeError, non_callable_members_illegal): - issubclass(C, PNonCall) + # gh-105144: this previously raised TypeError + # if a Protocol subclass of Sized had been created + # before any isinstance() checks against Sized + self.assertNotIsInstance(1, typing.Sized) - self.assertIsInstance(C(), PNonCall) + def test_empty_protocol_decorated_with_final(self): + @final + @runtime_checkable + class EmptyProtocol(Protocol): ... - # check that non-protocol subclasses are not affected - class D(PNonCall): ... + self.assertIsSubclass(object, EmptyProtocol) + self.assertIsInstance(object(), EmptyProtocol) - self.assertNotIsSubclass(C, D) - self.assertNotIsInstance(C(), D) - D.register(C) - self.assertIsSubclass(C, D) - self.assertIsInstance(C(), D) + def test_protocol_decorated_with_final_callable_members(self): + @final + @runtime_checkable + class ProtocolWithMethod(Protocol): + def startswith(self, string: str) -> bool: ... - with self.assertRaisesRegex(TypeError, non_callable_members_illegal): - issubclass(D, PNonCall) + self.assertIsSubclass(str, ProtocolWithMethod) + self.assertNotIsSubclass(int, ProtocolWithMethod) + self.assertIsInstance('foo', ProtocolWithMethod) + self.assertNotIsInstance(42, ProtocolWithMethod) - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_no_weird_caching_with_issubclass_after_isinstance(self): + def test_protocol_decorated_with_final_noncallable_members(self): + @final @runtime_checkable - class Spam(Protocol): + class ProtocolWithNonCallableMember(Protocol): x: int - class Eggs: - def __init__(self) -> None: - self.x = 42 + class Foo: + x = 42 - self.assertIsInstance(Eggs(), Spam) + only_callable_members_please = ( + r"Protocols with non-method members don't support issubclass()" + ) - # gh-104555: If we didn't override ABCMeta.__subclasscheck__ in _ProtocolMeta, - # TypeError wouldn't be raised here, - # as the cached result of the isinstance() check immediately above - # would mean the issubclass() call would short-circuit - # before we got to the "raise TypeError" line - with self.assertRaisesRegex( - TypeError, - "Protocols with non-method members don't support issubclass()" - ): - issubclass(Eggs, Spam) + with self.assertRaisesRegex(TypeError, only_callable_members_please): + issubclass(Foo, ProtocolWithNonCallableMember) - def test_no_weird_caching_with_issubclass_after_isinstance_2(self): + with self.assertRaisesRegex(TypeError, only_callable_members_please): + issubclass(int, ProtocolWithNonCallableMember) + + self.assertIsInstance(Foo(), ProtocolWithNonCallableMember) + self.assertNotIsInstance(42, ProtocolWithNonCallableMember) + + def test_protocol_decorated_with_final_mixed_members(self): + @final @runtime_checkable - class Spam(Protocol): + class ProtocolWithMixedMembers(Protocol): x: int + def method(self) -> None: ... - class Eggs: ... + class Foo: + x = 42 + def method(self) -> None: ... - self.assertNotIsInstance(Eggs(), Spam) + only_callable_members_please = ( + r"Protocols with non-method members don't support issubclass()" + ) - # gh-104555: If we didn't override ABCMeta.__subclasscheck__ in _ProtocolMeta, - # TypeError wouldn't be raised here, - # as the cached result of the isinstance() check immediately above - # would mean the issubclass() call would short-circuit - # before we got to the "raise TypeError" line - with self.assertRaisesRegex( - TypeError, - "Protocols with non-method members don't support issubclass()" - ): - issubclass(Eggs, Spam) + with self.assertRaisesRegex(TypeError, only_callable_members_please): + issubclass(Foo, ProtocolWithMixedMembers) - def test_no_weird_caching_with_issubclass_after_isinstance_3(self): + with self.assertRaisesRegex(TypeError, only_callable_members_please): + issubclass(int, ProtocolWithMixedMembers) + + self.assertIsInstance(Foo(), ProtocolWithMixedMembers) + self.assertNotIsInstance(42, ProtocolWithMixedMembers) + + def test_protocol_issubclass_error_message(self): @runtime_checkable - class Spam(Protocol): - x: int + class Vec2D(Protocol): + x: float + y: float - class Eggs: - def __getattr__(self, attr): - if attr == "x": - return 42 - raise AttributeError(attr) + def square_norm(self) -> float: + return self.x ** 2 + self.y ** 2 - self.assertNotIsInstance(Eggs(), Spam) + self.assertEqual(Vec2D.__protocol_attrs__, {'x', 'y', 'square_norm'}) + expected_error_message = ( + "Protocols with non-method members don't support issubclass()." + " Non-method members: 'x', 'y'." + ) + with self.assertRaisesRegex(TypeError, re.escape(expected_error_message)): + issubclass(int, Vec2D) - # gh-104555: If we didn't override ABCMeta.__subclasscheck__ in _ProtocolMeta, - # TypeError wouldn't be raised here, - # as the cached result of the isinstance() check immediately above - # would mean the issubclass() call would short-circuit - # before we got to the "raise TypeError" line - with self.assertRaisesRegex( - TypeError, - "Protocols with non-method members don't support issubclass()" - ): - issubclass(Eggs, Spam) + def test_nonruntime_protocol_interaction_with_evil_classproperty(self): + class classproperty: + def __get__(self, instance, type): + raise RuntimeError("NO") - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_no_weird_caching_with_issubclass_after_isinstance_pep695(self): - @runtime_checkable - class Spam[T](Protocol): - x: T + class Commentable(Protocol): + evil = classproperty() - class Eggs[T]: - def __init__(self, x: T) -> None: - self.x = x + # recognised as a protocol attr, + # but not actually accessed by the protocol metaclass + # (which would raise RuntimeError) for non-runtime protocols. + # See gh-113320 + self.assertEqual(get_protocol_members(Commentable), {"evil"}) - self.assertIsInstance(Eggs(42), Spam) + def test_runtime_protocol_interaction_with_evil_classproperty(self): + class CustomError(Exception): pass - # gh-104555: If we didn't override ABCMeta.__subclasscheck__ in _ProtocolMeta, - # TypeError wouldn't be raised here, - # as the cached result of the isinstance() check immediately above - # would mean the issubclass() call would short-circuit - # before we got to the "raise TypeError" line - with self.assertRaisesRegex( - TypeError, - "Protocols with non-method members don't support issubclass()" - ): - issubclass(Eggs, Spam) - - # FIXME(arihant2math): start more porting from test_protocols_isinstance + class classproperty: + def __get__(self, instance, type): + raise CustomError + + with self.assertRaises(TypeError) as cm: + @runtime_checkable + class Commentable(Protocol): + evil = classproperty() + + exc = cm.exception + self.assertEqual( + exc.args[0], + "Failed to determine whether protocol member 'evil' is a method member" + ) + self.assertIs(type(exc.__cause__), CustomError) class GenericTests(BaseTestCase): @@ -3888,8 +4999,6 @@ def test_pep695_generic_class_with_future_annotations(self): # should not have changed as a result of the get_type_hints() calls! self.assertEqual(ann_module695.__dict__, original_globals) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_pep695_generic_class_with_future_annotations_and_local_shadowing(self): hints_for_B = get_type_hints(ann_module695.B) self.assertEqual(hints_for_B, {"x": int, "y": str, "z": bytes}) @@ -3935,8 +5044,6 @@ def test_pep_695_generic_method_with_future_annotations_name_clash_with_global_v set(ann_module695.D.generic_method_2.__type_params__) ) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_pep_695_generics_with_future_annotations_nested_in_function(self): results = ann_module695.nested() @@ -4040,8 +5147,7 @@ def test_all_repr_eq_any(self): self.assertNotEqual(repr(base), '') self.assertEqual(base, base) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_pickle(self): global C # pickle wants to reference the class by name T = TypeVar('T') @@ -4163,8 +5269,6 @@ def test_weakref_all(self): for t in things: self.assertEqual(weakref.ref(t)(), t) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_parameterized_slots(self): T = TypeVar('T') class C(Generic[T]): @@ -4184,8 +5288,6 @@ def foo(x: C['C']): ... self.assertEqual(get_type_hints(foo, globals(), locals())['x'], C[C]) self.assertEqual(copy(C[int]), deepcopy(C[int])) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_parameterized_slots_dict(self): T = TypeVar('T') class D(Generic[T]): @@ -4506,8 +5608,6 @@ def __init__(self, arg): self.assertEqual(c.from_a, 'foo') self.assertEqual(c.from_c, 'foo') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_new_no_args(self): class A(Generic[T]): @@ -4613,6 +5713,24 @@ class A: with self.assertRaises(TypeError): a[int] + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: ".+__typing_subst__.+tuple.+int.*" does not match "'TypeAliasType' object is not subscriptable" + def test_return_non_tuple_while_unpacking(self): + # GH-138497: GenericAlias objects didn't ensure that __typing_subst__ actually + # returned a tuple + class EvilTypeVar: + __typing_is_unpacked_typevartuple__ = True + def __typing_prepare_subst__(*_): + return None # any value + def __typing_subst__(*_): + return 42 # not tuple + + evil = EvilTypeVar() + # Create a dummy TypeAlias that will be given the evil generic from + # above. + type type_alias[*_] = 0 + with self.assertRaisesRegex(TypeError, ".+__typing_subst__.+tuple.+int.*"): + type_alias[evil][0] + class ClassVarTests(BaseTestCase): @@ -4659,7 +5777,6 @@ def test_no_isinstance(self): with self.assertRaises(TypeError): issubclass(int, ClassVar) - class FinalTests(BaseTestCase): def test_basics(self): @@ -4716,8 +5833,7 @@ def test_final_unmodified(self): def func(x): ... self.assertIs(func, final(func)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_dunder_final(self): @final def func(): ... @@ -4788,8 +5904,6 @@ def cached(self): ... class OverrideDecoratorTests(BaseTestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_override(self): class Base: def normal_method(self): ... @@ -4848,8 +5962,6 @@ def static_method_bad_order(): self.assertIs(False, hasattr(Base.static_method_good_order, "__override__")) self.assertIs(False, hasattr(Base.static_method_bad_order, "__override__")) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_property(self): class Base: @property @@ -4876,8 +5988,6 @@ def wrong(self) -> int: self.assertFalse(hasattr(Child.wrong, "__override__")) self.assertFalse(hasattr(Child.wrong.fset, "__override__")) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_silent_failure(self): class CustomProp: __slots__ = ('fget',) @@ -4895,8 +6005,6 @@ def some(self): self.assertEqual(WithOverride.some, 1) self.assertFalse(hasattr(WithOverride.some, "__override__")) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_multiple_decorators(self): def with_wraps(f): # similar to `lru_cache` definition @wraps(f) @@ -4955,7 +6063,6 @@ def test_errors(self): self.assertIs(assert_type(arg, 'hello'), arg) - # We need this to make sure that `@no_type_check` respects `__module__` attr: from test.typinganndata import ann_module8 @@ -4969,6 +6076,7 @@ class NoTypeCheck_WithFunction: class ForwardRefTests(BaseTestCase): + def test_basics(self): class Node(Generic[T]): @@ -5357,8 +6465,6 @@ class A: some.__no_type_check__ self.assertEqual(get_type_hints(some), {'args': int, 'return': int}) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_no_type_check_lambda(self): @no_type_check class A: @@ -5579,11 +6685,11 @@ def test_overload_registry_repeated(self): self.assertEqual(list(get_overloads(impl)), overloads) + from test.typinganndata import ( ann_module, ann_module2, ann_module3, ann_module5, ann_module6, ) - T_a = TypeVar('T_a') class AwaitableWrapper(typing.Awaitable[T_a]): @@ -5616,7 +6722,6 @@ async def __aenter__(self) -> int: async def __aexit__(self, etype, eval, tb): return None - class A: y: float class B(A): @@ -5710,10 +6815,9 @@ class WeirdlyQuotedMovie(TypedDict): title: Annotated['Annotated[Required[str], "foobar"]', "another level"] year: NotRequired['Annotated[int, 2000]'] -# TODO: RUSTPYTHON -# class HasForeignBaseClass(mod_generics_cache.A): -# some_xrepr: 'XRepr' -# other_a: 'mod_generics_cache.A' +class HasForeignBaseClass(mod_generics_cache.A): + some_xrepr: 'XRepr' + other_a: 'mod_generics_cache.A' async def g_with(am: typing.AsyncContextManager[int]): x: int @@ -5727,7 +6831,6 @@ async def g_with(am: typing.AsyncContextManager[int]): gth = get_type_hints - class ForRefExample: @ann_module.dec def func(self: 'ForRefExample'): @@ -5749,8 +6852,7 @@ def test_get_type_hints_from_various_objects(self): with self.assertRaises(TypeError): gth(None) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_get_type_hints_modules(self): ann_module_type_hints = {1: 2, 'f': Tuple[int, int], 'x': int, 'y': str, 'u': int | float} self.assertEqual(gth(ann_module), ann_module_type_hints) @@ -5766,8 +6868,6 @@ def test_get_type_hints_modules_forwardref(self): 'default_b': Optional[mod_generics_cache.B]} self.assertEqual(gth(mod_generics_cache), mgc_hints) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_get_type_hints_classes(self): self.assertEqual(gth(ann_module.C), # gth will find the right globalns {'y': Optional[ann_module.C]}) @@ -6003,8 +7103,7 @@ def test_top_level_class_var(self): ): get_type_hints(ann_module6) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_get_type_hints_typeddict(self): self.assertEqual(get_type_hints(TotalMovie), {'title': str, 'year': int}) self.assertEqual(get_type_hints(TotalMovie, include_extras=True), { @@ -6058,7 +7157,6 @@ def h(x: collections.abc.Callable[P, int]): ... self.assertEqual(get_type_hints(h), {'x': collections.abc.Callable[P, int]}) - class GetUtilitiesTestCase(TestCase): def test_get_origin(self): T = TypeVar('T') @@ -6160,7 +7258,6 @@ class C(Generic[T]): pass self.assertEqual(get_args(Unpack[tuple[Unpack[Ts]]]), (tuple[Unpack[Ts]],)) - class CollectionsAbcTests(BaseTestCase): def test_hashable(self): @@ -6653,8 +7750,6 @@ def manager(): self.assertIsInstance(cm, typing.ContextManager) self.assertNotIsInstance(42, typing.ContextManager) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_contextmanager_type_params(self): cm1 = typing.ContextManager[int] self.assertEqual(get_args(cm1), (int, bool | None)) @@ -6820,7 +7915,6 @@ class ProUserId(UserId): ... - class NamedTupleTests(BaseTestCase): class NestedEmployee(NamedTuple): name: str @@ -6940,8 +8034,6 @@ class Y(Generic[T], NamedTuple): with self.assertRaises(TypeError): G[int, str] - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_generic_pep695(self): class X[T](NamedTuple): x: T @@ -7052,8 +8144,7 @@ class CNT(NamedTuple): self.assertEqual(struct.__annotations__, {}) self.assertIsInstance(struct(), struct) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_namedtuple_errors(self): with self.assertRaises(TypeError): NamedTuple.__new__() @@ -7141,8 +8232,7 @@ class Bar(NamedTuple): self.assertIsInstance(bar.attr, Vanilla) self.assertEqual(bar.attr.name, "attr") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_setname_raises_the_same_as_on_other_classes(self): class CustomException(BaseException): pass @@ -7313,6 +8403,22 @@ class TD2(TD1): self.assertIs(TD2.__total__, True) + def test_total_with_assigned_value(self): + class TD(TypedDict): + __total__ = "some_value" + + self.assertIs(TD.__total__, True) + + class TD2(TypedDict, total=True): + __total__ = "some_value" + + self.assertIs(TD2.__total__, True) + + class TD3(TypedDict, total=False): + __total__ = "some value" + + self.assertIs(TD3.__total__, False) + def test_optional_keys(self): class Point2Dor3D(Point2D, total=False): z: int @@ -7542,16 +8648,14 @@ class NewGeneric[T](TypedDict): # The TypedDict constructor is not itself a TypedDict self.assertIs(is_typeddict(TypedDict), False) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_get_type_hints(self): self.assertEqual( get_type_hints(Bar), {'a': typing.Optional[int], 'b': int} ) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_get_type_hints_generic(self): self.assertEqual( get_type_hints(BarGeneric), @@ -7566,8 +8670,6 @@ class FooBarGeneric(BarGeneric[int]): {'a': typing.Optional[T], 'b': int, 'c': str} ) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_pep695_generic_typeddict(self): class A[T](TypedDict): a: T @@ -7686,8 +8788,7 @@ class WithImplicitAny(B): with self.assertRaises(TypeError): WithImplicitAny[str] - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_non_generic_subscript(self): # For backward compatibility, subscription works # on arbitrary TypedDict types. @@ -7815,8 +8916,7 @@ class Child(Base): self.assertEqual(Child.__readonly_keys__, frozenset()) self.assertEqual(Child.__mutable_keys__, frozenset({'a'})) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_combine_qualifiers(self): class AllTheThings(TypedDict): a: Annotated[Required[ReadOnly[int]], "why not"] @@ -8010,8 +9110,7 @@ def test_repr(self): self.assertEqual(repr(Match[str]), 'typing.Match[str]') self.assertEqual(repr(Match[bytes]), 'typing.Match[bytes]') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_cannot_subclass(self): with self.assertRaisesRegex( TypeError, @@ -8027,7 +9126,6 @@ class B(typing.Pattern): pass - class AnnotatedTests(BaseTestCase): def test_new(self): @@ -8535,7 +9633,6 @@ def test_cannot_subscript(self): TypeAlias[int] - class ParamSpecTests(BaseTestCase): def test_basic_plain(self): @@ -8570,8 +9667,6 @@ def test_valid_uses(self): self.assertEqual(C4.__args__, (P, T)) self.assertEqual(C4.__parameters__, (P, T)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_args_kwargs(self): P = ParamSpec('P') P_2 = ParamSpec('P_2') @@ -8591,7 +9686,6 @@ def test_args_kwargs(self): self.assertEqual(repr(P.args), "P.args") self.assertEqual(repr(P.kwargs), "P.kwargs") - def test_stringized(self): P = ParamSpec('P') class C(Generic[P]): @@ -8933,7 +10027,6 @@ class G(P.args): pass class H(P.kwargs): pass - class ConcatenateTests(BaseTestCase): def test_basics(self): P = ParamSpec('P') @@ -9000,7 +10093,6 @@ def test_var_substitution(self): self.assertEqual(C[Concatenate[str, P2]], Concatenate[int, str, P2]) self.assertEqual(C[...], Concatenate[int, ...]) - class TypeGuardTests(BaseTestCase): def test_basics(self): TypeGuard[int] # OK @@ -9106,6 +10198,7 @@ def test_no_isinstance(self): class SpecialAttrsTests(BaseTestCase): + def test_special_attrs(self): cls_to_check = { # ABC classes @@ -9236,8 +10329,7 @@ def test_special_attrs(self): TypeName = typing.NewType('SpecialAttrsTests.TypeName', Any) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_special_attrs2(self): # Forward refs provide a different introspection API. __name__ and # __qualname__ make little sense for forward refs as they can store @@ -9404,8 +10496,6 @@ class CustomerModel(ModelBase, init=False): class NoDefaultTests(BaseTestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_pickling(self): for proto in range(pickle.HIGHEST_PROTOCOL + 1): s = pickle.dumps(NoDefault, proto) @@ -9431,8 +10521,7 @@ def test_no_call(self): with self.assertRaises(TypeError): NoDefault() - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_no_attributes(self): with self.assertRaises(AttributeError): NoDefault.foo = 3 @@ -9508,8 +10597,7 @@ class TypeIterationTests(BaseTestCase): Annotated[T, ''], ) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_cannot_iterate(self): expected_error_regex = "object is not iterable" for test_type in self._UNITERABLE_TYPES: diff --git a/Lib/test/test_ucn.py b/Lib/test/test_ucn.py index f6d69540b9..69b58da020 100644 --- a/Lib/test/test_ucn.py +++ b/Lib/test/test_ucn.py @@ -10,6 +10,7 @@ import ast import unittest import unicodedata +import urllib.error from test import support from http.client import HTTPException @@ -115,8 +116,7 @@ def test_misc_symbols(self): self.checkletter("HALFWIDTH KATAKANA SEMI-VOICED SOUND MARK", "\uFF9F") self.checkletter("FULLWIDTH LATIN SMALL LETTER A", "\uFF41") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_aliases(self): # Check that the aliases defined in the NameAliases.txt file work. # This should be updated when new aliases are added or the file @@ -143,8 +143,6 @@ def test_aliases(self): with self.assertRaises(KeyError): unicodedata.ucd_3_2_0.lookup(alias) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_aliases_names_in_pua_range(self): # We are storing aliases in the PUA 15, but their names shouldn't leak for cp in range(0xf0000, 0xf0100): @@ -152,8 +150,6 @@ def test_aliases_names_in_pua_range(self): unicodedata.name(chr(cp)) self.assertEqual(str(cm.exception), 'no such name') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_named_sequences_names_in_pua_range(self): # We are storing named seq in the PUA 15, but their names shouldn't leak for cp in range(0xf0100, 0xf0fff): @@ -161,8 +157,7 @@ def test_named_sequences_names_in_pua_range(self): unicodedata.name(chr(cp)) self.assertEqual(str(cm.exception), 'no such name') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_named_sequences_sample(self): # Check a few named sequences. See #12753. sequences = [ @@ -179,6 +174,7 @@ def test_named_sequences_sample(self): with self.assertRaises(KeyError): unicodedata.ucd_3_2_0.lookup(seqname) + @unittest.expectedFailure # TODO: RUSTPYTHON def test_named_sequences_full(self): # Check all the named sequences def check_version(testfile): @@ -189,23 +185,25 @@ def check_version(testfile): try: testdata = support.open_urlresource(url, encoding="utf-8", check=check_version) - except (OSError, HTTPException): - self.skipTest("Could not retrieve " + url) - self.addCleanup(testdata.close) - for line in testdata: - line = line.strip() - if not line or line.startswith('#'): - continue - seqname, codepoints = line.split(';') - codepoints = ''.join(chr(int(cp, 16)) for cp in codepoints.split()) - self.assertEqual(unicodedata.lookup(seqname), codepoints) - with self.assertRaises(SyntaxError): - self.checkletter(seqname, None) - with self.assertRaises(KeyError): - unicodedata.ucd_3_2_0.lookup(seqname) - - # TODO: RUSTPYTHON - @unittest.expectedFailure + except urllib.error.HTTPError as exc: + exc.close() + self.skipTest(f"Could not retrieve {url}: {exc!r}") + except (OSError, HTTPException) as exc: + self.skipTest(f"Could not retrieve {url}: {exc!r}") + with testdata: + for line in testdata: + line = line.strip() + if not line or line.startswith('#'): + continue + seqname, codepoints = line.split(';') + codepoints = ''.join(chr(int(cp, 16)) for cp in codepoints.split()) + self.assertEqual(unicodedata.lookup(seqname), codepoints) + with self.assertRaises(SyntaxError): + self.checkletter(seqname, None) + with self.assertRaises(KeyError): + unicodedata.ucd_3_2_0.lookup(seqname) + + @unittest.expectedFailure # TODO: RUSTPYTHON def test_errors(self): self.assertRaises(TypeError, unicodedata.name) self.assertRaises(TypeError, unicodedata.name, 'xx') diff --git a/Lib/test/test_unicode_file.py b/Lib/test/test_unicode_file.py index 80c22c6cdd..fe25bfe9f8 100644 --- a/Lib/test/test_unicode_file.py +++ b/Lib/test/test_unicode_file.py @@ -110,7 +110,7 @@ def _test_single(self, filename): os.unlink(filename) self.assertTrue(not os.path.exists(filename)) # and again with os.open. - f = os.open(filename, os.O_CREAT) + f = os.open(filename, os.O_CREAT | os.O_WRONLY) os.close(f) try: self._do_single(filename) diff --git a/Lib/test/test_unicode_file_functions.py b/Lib/test/test_unicode_file_functions.py index 47619c8807..25c16e3a0b 100644 --- a/Lib/test/test_unicode_file_functions.py +++ b/Lib/test/test_unicode_file_functions.py @@ -5,7 +5,7 @@ import unittest import warnings from unicodedata import normalize -from test.support import os_helper +from test.support import is_apple, os_helper from test import support @@ -23,13 +23,13 @@ '10_\u1fee\u1ffd', ] -# Mac OS X decomposes Unicode names, using Normal Form D. +# Apple platforms decompose Unicode names, using Normal Form D. # http://developer.apple.com/mac/library/qa/qa2001/qa1173.html # "However, most volume formats do not follow the exact specification for # these normal forms. For example, HFS Plus uses a variant of Normal Form D # in which U+2000 through U+2FFF, U+F900 through U+FAFF, and U+2F800 through # U+2FAFF are not decomposed." -if sys.platform != 'darwin': +if not is_apple: filenames.extend([ # Specific code points: NFC(fn), NFD(fn), NFKC(fn) and NFKD(fn) all different '11_\u0385\u03d3\u03d4', @@ -119,11 +119,11 @@ def test_open(self): os.stat(name) self._apply_failure(os.listdir, name, self._listdir_failure) - # Skip the test on darwin, because darwin does normalize the filename to + # Skip the test on Apple platforms, because they don't normalize the filename to # NFD (a variant of Unicode NFD form). Normalize the filename to NFC, NFKC, # NFKD in Python is useless, because darwin will normalize it later and so # open(), os.stat(), etc. don't raise any exception. - @unittest.skipIf(sys.platform == 'darwin', 'irrelevant test on Mac OS X') + @unittest.skipIf(is_apple, 'irrelevant test on Apple platforms') @unittest.skipIf( support.is_emscripten or support.is_wasi, "test fails on Emscripten/WASI when host platform is macOS." @@ -142,10 +142,10 @@ def test_normalize(self): self._apply_failure(os.remove, name) self._apply_failure(os.listdir, name) - # Skip the test on darwin, because darwin uses a normalization different + # Skip the test on Apple platforms, because they use a normalization different # than Python NFD normalization: filenames are different even if we use # Python NFD normalization. - @unittest.skipIf(sys.platform == 'darwin', 'irrelevant test on Mac OS X') + @unittest.skipIf(is_apple, 'irrelevant test on Apple platforms') def test_listdir(self): sf0 = set(self.files) with warnings.catch_warnings(): diff --git a/Lib/test/test_unicode_identifiers.py b/Lib/test/test_unicode_identifiers.py index d7a0ece253..60cfdaabe8 100644 --- a/Lib/test/test_unicode_identifiers.py +++ b/Lib/test/test_unicode_identifiers.py @@ -21,7 +21,7 @@ def test_non_bmp_normalized(self): @unittest.expectedFailure def test_invalid(self): try: - from test import badsyntax_3131 + from test.tokenizedata import badsyntax_3131 except SyntaxError as err: self.assertEqual(str(err), "invalid character '€' (U+20AC) (badsyntax_3131.py, line 2)") diff --git a/Lib/test/test_unicodedata.py b/Lib/test/test_unicodedata.py index 29da4a25a3..e14123aaa6 100644 --- a/Lib/test/test_unicodedata.py +++ b/Lib/test/test_unicodedata.py @@ -11,15 +11,20 @@ import sys import unicodedata import unittest -from test.support import (open_urlresource, requires_resource, script_helper, - cpython_only, check_disallow_instantiation, - ResourceDenied) +from test.support import ( + open_urlresource, + requires_resource, + script_helper, + cpython_only, + check_disallow_instantiation, + force_not_colorized, +) class UnicodeMethodsTest(unittest.TestCase): # update this, if the database changes - expectedchecksum = '4739770dd4d0e5f1b1677accfc3552ed3c8ef326' + expectedchecksum = '63aa77dcb36b0e1df082ee2a6071caeda7f0955e' # TODO: RUSTPYTHON @unittest.expectedFailure @@ -74,7 +79,8 @@ class UnicodeFunctionsTest(UnicodeDatabaseTest): # Update this if the database changes. Make sure to do a full rebuild # (e.g. 'make distclean && make') to get the correct checksum. - expectedchecksum = '98d602e1f69d5c5bb8a5910c40bbbad4e18e8370' + expectedchecksum = '232affd2a50ec4bd69d2482aa0291385cbdefaba' + # TODO: RUSTPYTHON @unittest.expectedFailure @requires_resource('cpu') @@ -94,6 +100,8 @@ def test_function_checksum(self): self.db.decomposition(char), str(self.db.mirrored(char)), str(self.db.combining(char)), + unicodedata.east_asian_width(char), + self.db.name(char, ""), ] h.update(''.join(data).encode("ascii")) result = h.hexdigest() @@ -106,6 +114,28 @@ def test_name_inverse_lookup(self): if looked_name := self.db.name(char, None): self.assertEqual(self.db.lookup(looked_name), char) + def test_no_names_in_pua(self): + puas = [*range(0xe000, 0xf8ff), + *range(0xf0000, 0xfffff), + *range(0x100000, 0x10ffff)] + for i in puas: + char = chr(i) + self.assertRaises(ValueError, self.db.name, char) + + # TODO: RUSTPYTHON; LookupError: undefined character name 'LATIN SMLL LETR A' + @unittest.expectedFailure + def test_lookup_nonexistant(self): + # just make sure that lookup can fail + for nonexistant in [ + "LATIN SMLL LETR A", + "OPEN HANDS SIGHS", + "DREGS", + "HANDBUG", + "MODIFIER LETTER CYRILLIC SMALL QUESTION MARK", + "???", + ]: + self.assertRaises(KeyError, self.db.lookup, nonexistant) + # TODO: RUSTPYTHON @unittest.expectedFailure def test_digit(self): @@ -245,6 +275,25 @@ def test_east_asian_width(self): self.assertEqual(eaw('\u2010'), 'A') self.assertEqual(eaw('\U00020000'), 'W') + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_east_asian_width_unassigned(self): + eaw = self.db.east_asian_width + # unassigned + for char in '\u0530\u0ecf\u10c6\u20fc\uaaca\U000107bd\U000115f2': + self.assertEqual(eaw(char), 'N') + self.assertIs(self.db.name(char, None), None) + + # unassigned but reserved for CJK + for char in '\uFA6E\uFADA\U0002A6E0\U0002FA20\U0003134B\U0003FFFD': + self.assertEqual(eaw(char), 'W') + self.assertIs(self.db.name(char, None), None) + + # private use areas + for char in '\uE000\uF800\U000F0000\U000FFFEE\U00100000\U0010FFF0': + self.assertEqual(eaw(char), 'A') + self.assertIs(self.db.name(char, None), None) + # TODO: RUSTPYTHON @unittest.expectedFailure def test_east_asian_width_9_0_changes(self): @@ -260,6 +309,7 @@ def test_disallow_instantiation(self): # TODO: RUSTPYTHON @unittest.expectedFailure + @force_not_colorized def test_failed_import_during_compiling(self): # Issue 4367 # Decoding \N escapes requires the unicodedata module. If it can't be @@ -322,6 +372,7 @@ def test_ucd_510(self): self.assertTrue("\u1d79".upper()=='\ua77d') self.assertTrue(".".upper()=='.') + @requires_resource('cpu') def test_bug_5828(self): self.assertEqual("\u1d79".lower(), "\u1d79") # Only U+0000 should have U+0000 as its upper/lower/titlecase variant @@ -363,7 +414,9 @@ def unistr(data): data = [int(x, 16) for x in data.split(" ")] return "".join([chr(x) for x in data]) + @unittest.expectedFailure # TODO: RUSTPYTHON @requires_resource('network') + @requires_resource('cpu') def test_normalization(self): TESTDATAFILE = "NormalizationTest.txt" TESTDATAURL = f"http://www.pythontest.net/unicode/{unicodedata.unidata_version}/{TESTDATAFILE}" diff --git a/Lib/test/test_unittest.py b/Lib/test/test_unittest.py deleted file mode 100644 index 1079c7df2e..0000000000 --- a/Lib/test/test_unittest.py +++ /dev/null @@ -1,16 +0,0 @@ -import unittest.test - -from test import support - - -def load_tests(*_): - # used by unittest - return unittest.test.suite() - - -def tearDownModule(): - support.reap_children() - - -if __name__ == "__main__": - unittest.main() diff --git a/Lib/test/test_unittest/__init__.py b/Lib/test/test_unittest/__init__.py new file mode 100644 index 0000000000..bc502ef32d --- /dev/null +++ b/Lib/test/test_unittest/__init__.py @@ -0,0 +1,6 @@ +import os.path +from test.support import load_package_tests + + +def load_tests(*args): + return load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/test/test_unittest/__main__.py b/Lib/test/test_unittest/__main__.py new file mode 100644 index 0000000000..40a23a297e --- /dev/null +++ b/Lib/test/test_unittest/__main__.py @@ -0,0 +1,4 @@ +from . import load_tests +import unittest + +unittest.main() diff --git a/Lib/unittest/test/_test_warnings.py b/Lib/test/test_unittest/_test_warnings.py similarity index 84% rename from Lib/unittest/test/_test_warnings.py rename to Lib/test/test_unittest/_test_warnings.py index 5cbfb532ad..08b846ee47 100644 --- a/Lib/unittest/test/_test_warnings.py +++ b/Lib/test/test_unittest/_test_warnings.py @@ -18,17 +18,6 @@ def warnfun(): warnings.warn('rw', RuntimeWarning) class TestWarnings(unittest.TestCase): - # unittest warnings will be printed at most once per type (max one message - # for the fail* methods, and one for the assert* methods) - def test_assert(self): - self.assertEquals(2+2, 4) - self.assertEquals(2*2, 4) - self.assertEquals(2**2, 4) - - def test_fail(self): - self.failUnless(1) - self.failUnless(True) - def test_other_unittest(self): self.assertAlmostEqual(2+2, 4) self.assertNotAlmostEqual(4+4, 2) diff --git a/Lib/unittest/test/dummy.py b/Lib/test/test_unittest/dummy.py similarity index 100% rename from Lib/unittest/test/dummy.py rename to Lib/test/test_unittest/dummy.py diff --git a/Lib/unittest/test/support.py b/Lib/test/test_unittest/support.py similarity index 92% rename from Lib/unittest/test/support.py rename to Lib/test/test_unittest/support.py index 529265304f..8c97bf5c72 100644 --- a/Lib/unittest/test/support.py +++ b/Lib/test/test_unittest/support.py @@ -136,3 +136,19 @@ def addSuccess(self, test): def wasSuccessful(self): return True + + +class BufferedWriter: + def __init__(self): + self.result = '' + self.buffer = '' + + def write(self, arg): + self.buffer += arg + + def flush(self): + self.result += self.buffer + self.buffer = '' + + def getvalue(self): + return self.result diff --git a/Lib/unittest/test/test_assertions.py b/Lib/test/test_unittest/test_assertions.py similarity index 96% rename from Lib/unittest/test/test_assertions.py rename to Lib/test/test_unittest/test_assertions.py index a0db3423b8..1dec947ea7 100644 --- a/Lib/unittest/test/test_assertions.py +++ b/Lib/test/test_unittest/test_assertions.py @@ -271,20 +271,11 @@ def testAssertDictEqual(self): r"\+ \{'key': 'value'\}$", r"\+ \{'key': 'value'\} : oops$"]) - def testAssertDictContainsSubset(self): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - - self.assertMessages('assertDictContainsSubset', ({'key': 'value'}, {}), - ["^Missing: 'key'$", "^oops$", - "^Missing: 'key'$", - "^Missing: 'key' : oops$"]) - def testAssertMultiLineEqual(self): self.assertMessages('assertMultiLineEqual', ("", "foo"), - [r"\+ foo$", "^oops$", - r"\+ foo$", - r"\+ foo : oops$"]) + [r"\+ foo\n$", "^oops$", + r"\+ foo\n$", + r"\+ foo\n : oops$"]) def testAssertLess(self): self.assertMessages('assertLess', (2, 1), @@ -395,6 +386,16 @@ def testAssertWarns(self): '^UserWarning not triggered$', '^UserWarning not triggered : oops$']) + def test_assertNotWarns(self): + def warn_future(): + warnings.warn('xyz', FutureWarning, stacklevel=2) + self.assertMessagesCM('_assertNotWarns', (FutureWarning,), + warn_future, + ['^FutureWarning triggered$', + '^oops$', + '^FutureWarning triggered$', + '^FutureWarning triggered : oops$']) + def testAssertWarnsRegex(self): # test error not raised self.assertMessagesCM('assertWarnsRegex', (UserWarning, 'unused regex'), diff --git a/Lib/unittest/test/test_async_case.py b/Lib/test/test_unittest/test_async_case.py similarity index 96% rename from Lib/unittest/test/test_async_case.py rename to Lib/test/test_unittest/test_async_case.py index 98ec4f3345..1d2b87f161 100644 --- a/Lib/unittest/test/test_async_case.py +++ b/Lib/test/test_unittest/test_async_case.py @@ -37,11 +37,9 @@ async def __aenter__(self): pass -# TODO: RUSTPYTHON; used by following test suite -# VAR = contextvars.ContextVar('VAR', default=()) +VAR = contextvars.ContextVar('VAR', default=()) -@unittest.skip("TODO: RUSTPYTHON; requires sys.get_coroutine_origin_tracking_depth()") class TestAsyncCase(unittest.TestCase): maxDiff = None @@ -50,6 +48,8 @@ def setUp(self): # starting a new event loop self.addCleanup(support.gc_collect) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_full_cycle(self): expected = ['setUp', 'asyncSetUp', @@ -296,6 +296,8 @@ async def on_cleanup2(self): test.doCleanups() self.assertEqual(events, ['asyncSetUp', 'test', 'asyncTearDown', 'cleanup2', 'cleanup1']) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_deprecation_of_return_val_from_test(self): # Issue 41322 - deprecate return of value that is not None from a test class Nothing: @@ -486,5 +488,21 @@ async def test_demo1(self): result = test.run() self.assertTrue(result.wasSuccessful()) + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_loop_factory(self): + asyncio.set_event_loop_policy(None) + + class TestCase1(unittest.IsolatedAsyncioTestCase): + loop_factory = asyncio.EventLoop + + async def test_demo1(self): + pass + + test = TestCase1('test_demo1') + result = test.run() + self.assertTrue(result.wasSuccessful()) + self.assertIsNone(support.maybe_get_event_loop_policy()) + if __name__ == "__main__": unittest.main() diff --git a/Lib/unittest/test/test_break.py b/Lib/test/test_unittest/test_break.py similarity index 98% rename from Lib/unittest/test/test_break.py rename to Lib/test/test_unittest/test_break.py index 33cbdd2661..1da98af3e7 100644 --- a/Lib/unittest/test/test_break.py +++ b/Lib/test/test_unittest/test_break.py @@ -236,6 +236,7 @@ def __init__(self, catchbreak): self.testRunner = FakeRunner self.test = test self.result = None + self.durations = None p = Program(False) p.runTests() @@ -244,7 +245,8 @@ def __init__(self, catchbreak): 'verbosity': verbosity, 'failfast': failfast, 'tb_locals': False, - 'warnings': None})]) + 'warnings': None, + 'durations': None})]) self.assertEqual(FakeRunner.runArgs, [test]) self.assertEqual(p.result, result) @@ -259,7 +261,8 @@ def __init__(self, catchbreak): 'verbosity': verbosity, 'failfast': failfast, 'tb_locals': False, - 'warnings': None})]) + 'warnings': None, + 'durations': None})]) self.assertEqual(FakeRunner.runArgs, [test]) self.assertEqual(p.result, result) diff --git a/Lib/unittest/test/test_case.py b/Lib/test/test_unittest/test_case.py similarity index 96% rename from Lib/unittest/test/test_case.py rename to Lib/test/test_unittest/test_case.py index e2547f4777..ae6a2b94db 100644 --- a/Lib/unittest/test/test_case.py +++ b/Lib/test/test_unittest/test_case.py @@ -15,7 +15,7 @@ import unittest -from unittest.test.support import ( +from test.test_unittest.support import ( TestEquality, TestHashing, LoggingResult, LegacyLoggingResult, ResultWithNoStartTestRunStopTestRun ) @@ -304,7 +304,8 @@ def defaultTestResult(self): def test(self): pass - Foo('test').run() + with self.assertWarns(RuntimeWarning): + Foo('test').run() def test_deprecation_of_return_val_from_test(self): # Issue 41322 - deprecate return of value that is not None from a test @@ -709,36 +710,6 @@ def testAssertIn(self): self.assertRaises(self.failureException, self.assertNotIn, 'cow', animals) - def testAssertDictContainsSubset(self): - with warnings.catch_warnings(): - warnings.simplefilter("ignore", DeprecationWarning) - - self.assertDictContainsSubset({}, {}) - self.assertDictContainsSubset({}, {'a': 1}) - self.assertDictContainsSubset({'a': 1}, {'a': 1}) - self.assertDictContainsSubset({'a': 1}, {'a': 1, 'b': 2}) - self.assertDictContainsSubset({'a': 1, 'b': 2}, {'a': 1, 'b': 2}) - - with self.assertRaises(self.failureException): - self.assertDictContainsSubset({1: "one"}, {}) - - with self.assertRaises(self.failureException): - self.assertDictContainsSubset({'a': 2}, {'a': 1}) - - with self.assertRaises(self.failureException): - self.assertDictContainsSubset({'c': 1}, {'a': 1}) - - with self.assertRaises(self.failureException): - self.assertDictContainsSubset({'a': 1, 'c': 1}, {'a': 1}) - - with self.assertRaises(self.failureException): - self.assertDictContainsSubset({'a': 1, 'c': 1}, {'a': 1}) - - one = ''.join(chr(i) for i in range(255)) - # this used to cause a UnicodeDecodeError constructing the failure msg - with self.assertRaises(self.failureException): - self.assertDictContainsSubset({'foo': one}, {'foo': '\uFFFD'}) - def testAssertEqual(self): equal_pairs = [ ((), ()), @@ -1161,6 +1132,8 @@ def testAssertMultiLineEqual(self): # need to remove the first line of the error message error = str(e).split('\n', 1)[1] self.assertEqual(sample_text_error, error) + else: + self.fail(f'{self.failureException} not raised') def testAssertEqualSingleLine(self): sample_text = "laden swallows fly slowly" @@ -1177,6 +1150,74 @@ def testAssertEqualSingleLine(self): # need to remove the first line of the error message error = str(e).split('\n', 1)[1] self.assertEqual(sample_text_error, error) + else: + self.fail(f'{self.failureException} not raised') + + def testAssertEqualwithEmptyString(self): + '''Verify when there is an empty string involved, the diff output + does not treat the empty string as a single empty line. It should + instead be handled as a non-line. + ''' + sample_text = '' + revised_sample_text = 'unladen swallows fly quickly' + sample_text_error = '''\ ++ unladen swallows fly quickly +''' + try: + self.assertEqual(sample_text, revised_sample_text) + except self.failureException as e: + # need to remove the first line of the error message + error = str(e).split('\n', 1)[1] + self.assertEqual(sample_text_error, error) + else: + self.fail(f'{self.failureException} not raised') + + def testAssertEqualMultipleLinesMissingNewlineTerminator(self): + '''Verifying format of diff output from assertEqual involving strings + with multiple lines, but missing the terminating newline on both. + ''' + sample_text = 'laden swallows\nfly sloely' + revised_sample_text = 'laden swallows\nfly slowly' + sample_text_error = '''\ + laden swallows +- fly sloely +? ^ ++ fly slowly +? ^ +''' + try: + self.assertEqual(sample_text, revised_sample_text) + except self.failureException as e: + # need to remove the first line of the error message + error = str(e).split('\n', 1)[1] + self.assertEqual(sample_text_error, error) + else: + self.fail(f'{self.failureException} not raised') + + def testAssertEqualMultipleLinesMismatchedNewlinesTerminators(self): + '''Verifying format of diff output from assertEqual involving strings + with multiple lines and mismatched newlines. The output should + include a - on it's own line to indicate the newline difference + between the two strings + ''' + sample_text = 'laden swallows\nfly sloely\n' + revised_sample_text = 'laden swallows\nfly slowly' + sample_text_error = '''\ + laden swallows +- fly sloely +? ^ ++ fly slowly +? ^ +-\x20 +''' + try: + self.assertEqual(sample_text, revised_sample_text) + except self.failureException as e: + # need to remove the first line of the error message + error = str(e).split('\n', 1)[1] + self.assertEqual(sample_text_error, error) + else: + self.fail(f'{self.failureException} not raised') def testEqualityBytesWarning(self): if sys.flags.bytes_warning: @@ -1801,45 +1842,18 @@ def testAssertNoLogsYieldsNone(self): pass self.assertIsNone(value) - def testDeprecatedMethodNames(self): - """ - Test that the deprecated methods raise a DeprecationWarning. See #9424. - """ - old = ( - (self.failIfEqual, (3, 5)), - (self.assertNotEquals, (3, 5)), - (self.failUnlessEqual, (3, 3)), - (self.assertEquals, (3, 3)), - (self.failUnlessAlmostEqual, (2.0, 2.0)), - (self.assertAlmostEquals, (2.0, 2.0)), - (self.failIfAlmostEqual, (3.0, 5.0)), - (self.assertNotAlmostEquals, (3.0, 5.0)), - (self.failUnless, (True,)), - (self.assert_, (True,)), - (self.failUnlessRaises, (TypeError, lambda _: 3.14 + 'spam')), - (self.failIf, (False,)), - (self.assertDictContainsSubset, (dict(a=1, b=2), dict(a=1, b=2, c=3))), - (self.assertRaisesRegexp, (KeyError, 'foo', lambda: {}['foo'])), - (self.assertRegexpMatches, ('bar', 'bar')), - ) - for meth, args in old: - with self.assertWarns(DeprecationWarning): - meth(*args) - - # disable this test for now. When the version where the fail* methods will - # be removed is decided, re-enable it and update the version - def _testDeprecatedFailMethods(self): - """Test that the deprecated fail* methods get removed in 3.x""" - if sys.version_info[:2] < (3, 3): - return + def testDeprecatedFailMethods(self): + """Test that the deprecated fail* methods get removed in 3.12""" deprecated_names = [ 'failIfEqual', 'failUnlessEqual', 'failUnlessAlmostEqual', 'failIfAlmostEqual', 'failUnless', 'failUnlessRaises', 'failIf', - 'assertDictContainsSubset', + 'assertNotEquals', 'assertEquals', 'assertAlmostEquals', + 'assertNotAlmostEquals', 'assert_', 'assertDictContainsSubset', + 'assertRaisesRegexp', 'assertRegexpMatches' ] for deprecated_name in deprecated_names: with self.assertRaises(AttributeError): - getattr(self, deprecated_name) # remove these in 3.x + getattr(self, deprecated_name) def testDeepcopy(self): # Issue: 5660 @@ -1956,7 +1970,7 @@ def testNoCycles(self): del case self.assertFalse(wr()) - # TODO: RUSTPYTHON; destructors + # TODO: RUSTPYTHON @unittest.expectedFailure def test_no_exception_leak(self): # Issue #19880: TestCase.run() should not keep a reference diff --git a/Lib/unittest/test/test_discovery.py b/Lib/test/test_unittest/test_discovery.py similarity index 96% rename from Lib/unittest/test/test_discovery.py rename to Lib/test/test_unittest/test_discovery.py index 3b58786ec1..9e231ff8d2 100644 --- a/Lib/unittest/test/test_discovery.py +++ b/Lib/test/test_unittest/test_discovery.py @@ -6,11 +6,10 @@ import pickle from test import support from test.support import import_helper -import test.test_importlib.util import unittest import unittest.mock -import unittest.test +import test.test_unittest class TestableTestProgram(unittest.TestProgram): @@ -365,6 +364,8 @@ def __eq__(self, other): [(loader, [], 'test*.py'), (loader, [], 'test*.py')]) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_discover(self): loader = unittest.TestLoader() @@ -407,10 +408,36 @@ def _find_tests(start_dir, pattern): top_level_dir = os.path.abspath('/foo/bar') start_dir = os.path.abspath('/foo/bar/baz') self.assertEqual(suite, "['tests']") - self.assertEqual(loader._top_level_dir, top_level_dir) + self.assertEqual(loader._top_level_dir, os.path.abspath('/foo')) self.assertEqual(_find_tests_args, [(start_dir, 'pattern')]) self.assertIn(top_level_dir, sys.path) + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_discover_should_not_persist_top_level_dir_between_calls(self): + original_isfile = os.path.isfile + original_isdir = os.path.isdir + original_sys_path = sys.path[:] + def restore(): + os.path.isfile = original_isfile + os.path.isdir = original_isdir + sys.path[:] = original_sys_path + self.addCleanup(restore) + + os.path.isfile = lambda path: True + os.path.isdir = lambda path: True + loader = unittest.TestLoader() + loader.suiteClass = str + dir = '/foo/bar' + top_level_dir = '/foo' + + loader.discover(dir, top_level_dir=top_level_dir) + self.assertEqual(loader._top_level_dir, None) + + loader._top_level_dir = dir2 = '/previous/dir' + loader.discover(dir, top_level_dir=top_level_dir) + self.assertEqual(loader._top_level_dir, dir2) + def test_discover_start_dir_is_package_calls_package_load_tests(self): # This test verifies that the package load_tests in a package is indeed # invoked when the start_dir is a package (and not the top level). @@ -789,7 +816,7 @@ def test_discovery_from_dotted_path(self): loader = unittest.TestLoader() tests = [self] - expectedPath = os.path.abspath(os.path.dirname(unittest.test.__file__)) + expectedPath = os.path.abspath(os.path.dirname(test.test_unittest.__file__)) self.wasRun = False def _find_tests(start_dir, pattern): @@ -797,7 +824,7 @@ def _find_tests(start_dir, pattern): self.assertEqual(start_dir, expectedPath) return tests loader._find_tests = _find_tests - suite = loader.discover('unittest.test') + suite = loader.discover('test.test_unittest') self.assertTrue(self.wasRun) self.assertEqual(suite._tests, tests) @@ -826,6 +853,8 @@ def restore(): 'as dotted module names') def test_discovery_failed_discovery(self): + from test.test_importlib import util + loader = unittest.TestLoader() package = types.ModuleType('package') @@ -837,7 +866,7 @@ def _import(packagename, *args, **kwargs): # Since loader.discover() can modify sys.path, restore it when done. with import_helper.DirsOnSysPath(): # Make sure to remove 'package' from sys.modules when done. - with test.test_importlib.util.uncache('package'): + with util.uncache('package'): with self.assertRaises(TypeError) as cm: loader.discover('package') self.assertEqual(str(cm.exception), diff --git a/Lib/unittest/test/test_functiontestcase.py b/Lib/test/test_unittest/test_functiontestcase.py similarity index 99% rename from Lib/unittest/test/test_functiontestcase.py rename to Lib/test/test_unittest/test_functiontestcase.py index 4971729880..2ebed9564a 100644 --- a/Lib/unittest/test/test_functiontestcase.py +++ b/Lib/test/test_unittest/test_functiontestcase.py @@ -1,6 +1,6 @@ import unittest -from unittest.test.support import LoggingResult +from test.test_unittest.support import LoggingResult class Test_FunctionTestCase(unittest.TestCase): diff --git a/Lib/unittest/test/test_loader.py b/Lib/test/test_unittest/test_loader.py similarity index 88% rename from Lib/unittest/test/test_loader.py rename to Lib/test/test_unittest/test_loader.py index de2268cda9..9881335318 100644 --- a/Lib/unittest/test/test_loader.py +++ b/Lib/test/test_unittest/test_loader.py @@ -1,29 +1,9 @@ import functools import sys import types -import warnings import unittest -# Decorator used in the deprecation tests to reset the warning registry for -# test isolation and reproducibility. -def warningregistry(func): - def wrapper(*args, **kws): - missing = [] - saved = getattr(warnings, '__warningregistry__', missing).copy() - try: - return func(*args, **kws) - finally: - if saved is missing: - try: - del warnings.__warningregistry__ - except AttributeError: - pass - else: - warnings.__warningregistry__ = saved - return wrapper - - class Test_TestLoader(unittest.TestCase): ### Basic object tests @@ -102,6 +82,24 @@ def runTest(self): self.assertIsInstance(suite, loader.suiteClass) self.assertEqual(list(suite), [Foo('runTest')]) + # "Do not load any tests from `TestCase` class itself." + def test_loadTestsFromTestCase__from_TestCase(self): + loader = unittest.TestLoader() + + suite = loader.loadTestsFromTestCase(unittest.TestCase) + self.assertIsInstance(suite, loader.suiteClass) + self.assertEqual(list(suite), []) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + # "Do not load any tests from `FunctionTestCase` class." + def test_loadTestsFromTestCase__from_FunctionTestCase(self): + loader = unittest.TestLoader() + + suite = loader.loadTestsFromTestCase(unittest.FunctionTestCase) + self.assertIsInstance(suite, loader.suiteClass) + self.assertEqual(list(suite), []) + ################################################################ ### /Tests for TestLoader.loadTestsFromTestCase @@ -123,6 +121,21 @@ def test(self): expected = [loader.suiteClass([MyTestCase('test')])] self.assertEqual(list(suite), expected) + # TODO: RUSTPYTHON + @unittest.expectedFailure + # "This test ensures that internal `TestCase` subclasses are not loaded" + def test_loadTestsFromModule__TestCase_subclass_internals(self): + # See https://github.com/python/cpython/issues/84867 + m = types.ModuleType('m') + # Simulate imported names: + m.TestCase = unittest.TestCase + m.FunctionTestCase = unittest.FunctionTestCase + + loader = unittest.TestLoader() + suite = loader.loadTestsFromModule(m) + self.assertIsInstance(suite, loader.suiteClass) + self.assertEqual(list(suite), []) + # "This method searches `module` for classes derived from TestCase" # # What happens if no tests are found (no TestCase instances)? @@ -174,9 +187,10 @@ class NotAModule(object): self.assertEqual(list(suite), reference) - # Check that loadTestsFromModule honors (or not) a module + # TODO: RUSTPYTHON + @unittest.expectedFailure + # Check that loadTestsFromModule honors a module # with a load_tests function. - @warningregistry def test_loadTestsFromModule__load_tests(self): m = types.ModuleType('m') class MyTestCase(unittest.TestCase): @@ -195,126 +209,13 @@ def load_tests(loader, tests, pattern): suite = loader.loadTestsFromModule(m) self.assertIsInstance(suite, unittest.TestSuite) self.assertEqual(load_tests_args, [loader, suite, None]) - # With Python 3.5, the undocumented and unofficial use_load_tests is - # ignored (and deprecated). - load_tests_args = [] - with warnings.catch_warnings(record=False): - warnings.simplefilter('ignore') - suite = loader.loadTestsFromModule(m, use_load_tests=False) - self.assertEqual(load_tests_args, [loader, suite, None]) - @warningregistry - def test_loadTestsFromModule__use_load_tests_deprecated_positional(self): - m = types.ModuleType('m') - class MyTestCase(unittest.TestCase): - def test(self): - pass - m.testcase_1 = MyTestCase - - load_tests_args = [] - def load_tests(loader, tests, pattern): - self.assertIsInstance(tests, unittest.TestSuite) - load_tests_args.extend((loader, tests, pattern)) - return tests - m.load_tests = load_tests - # The method still works. - loader = unittest.TestLoader() - # use_load_tests=True as a positional argument. - with warnings.catch_warnings(record=True) as w: - warnings.simplefilter('always') - suite = loader.loadTestsFromModule(m, False) - self.assertIsInstance(suite, unittest.TestSuite) - # load_tests was still called because use_load_tests is deprecated - # and ignored. - self.assertEqual(load_tests_args, [loader, suite, None]) - # We got a warning. - self.assertIs(w[-1].category, DeprecationWarning) - self.assertEqual(str(w[-1].message), - 'use_load_tests is deprecated and ignored') - - @warningregistry - def test_loadTestsFromModule__use_load_tests_deprecated_keyword(self): - m = types.ModuleType('m') - class MyTestCase(unittest.TestCase): - def test(self): - pass - m.testcase_1 = MyTestCase - - load_tests_args = [] - def load_tests(loader, tests, pattern): - self.assertIsInstance(tests, unittest.TestSuite) - load_tests_args.extend((loader, tests, pattern)) - return tests - m.load_tests = load_tests - # The method still works. - loader = unittest.TestLoader() - with warnings.catch_warnings(record=True) as w: - warnings.simplefilter('always') - suite = loader.loadTestsFromModule(m, use_load_tests=False) - self.assertIsInstance(suite, unittest.TestSuite) - # load_tests was still called because use_load_tests is deprecated - # and ignored. - self.assertEqual(load_tests_args, [loader, suite, None]) - # We got a warning. - self.assertIs(w[-1].category, DeprecationWarning) - self.assertEqual(str(w[-1].message), - 'use_load_tests is deprecated and ignored') - - @warningregistry - def test_loadTestsFromModule__too_many_positional_args(self): - m = types.ModuleType('m') - class MyTestCase(unittest.TestCase): - def test(self): - pass - m.testcase_1 = MyTestCase - - load_tests_args = [] - def load_tests(loader, tests, pattern): - self.assertIsInstance(tests, unittest.TestSuite) - load_tests_args.extend((loader, tests, pattern)) - return tests - m.load_tests = load_tests - loader = unittest.TestLoader() - with self.assertRaises(TypeError) as cm, \ - warnings.catch_warnings(record=True) as w: - warnings.simplefilter('always') - loader.loadTestsFromModule(m, False, 'testme.*') - # We still got the deprecation warning. - self.assertIs(w[-1].category, DeprecationWarning) - self.assertEqual(str(w[-1].message), - 'use_load_tests is deprecated and ignored') - # We also got a TypeError for too many positional arguments. - self.assertEqual(type(cm.exception), TypeError) - self.assertEqual( - str(cm.exception), - 'loadTestsFromModule() takes 1 positional argument but 3 were given') - - @warningregistry - def test_loadTestsFromModule__use_load_tests_other_bad_keyword(self): - m = types.ModuleType('m') - class MyTestCase(unittest.TestCase): - def test(self): - pass - m.testcase_1 = MyTestCase - - load_tests_args = [] - def load_tests(loader, tests, pattern): - self.assertIsInstance(tests, unittest.TestSuite) - load_tests_args.extend((loader, tests, pattern)) - return tests - m.load_tests = load_tests - loader = unittest.TestLoader() - with warnings.catch_warnings(): - warnings.simplefilter('ignore') - with self.assertRaises(TypeError) as cm: - loader.loadTestsFromModule( - m, use_load_tests=False, very_bad=True, worse=False) - self.assertEqual(type(cm.exception), TypeError) - # The error message names the first bad argument alphabetically, - # however use_load_tests (which sorts first) is ignored. - self.assertEqual( - str(cm.exception), - "loadTestsFromModule() got an unexpected keyword argument 'very_bad'") + # In Python 3.12, the undocumented and unofficial use_load_tests has + # been removed. + with self.assertRaises(TypeError): + loader.loadTestsFromModule(m, False) + with self.assertRaises(TypeError): + loader.loadTestsFromModule(m, use_load_tests=False) def test_loadTestsFromModule__pattern(self): m = types.ModuleType('m') @@ -716,7 +617,7 @@ def test_loadTestsFromName__module_not_loaded(self): # We're going to try to load this module as a side-effect, so it # better not be loaded before we try. # - module_name = 'unittest.test.dummy' + module_name = 'test.test_unittest.dummy' sys.modules.pop(module_name, None) loader = unittest.TestLoader() @@ -844,7 +745,7 @@ def test_loadTestsFromNames__unknown_attr_name(self): loader = unittest.TestLoader() suite = loader.loadTestsFromNames( - ['unittest.loader.sdasfasfasdf', 'unittest.test.dummy']) + ['unittest.loader.sdasfasfasdf', 'test.test_unittest.dummy']) error, test = self.check_deferred_error(loader, list(suite)[0]) expected = "module 'unittest.loader' has no attribute 'sdasfasfasdf'" self.assertIn( @@ -1141,7 +1042,7 @@ def test_loadTestsFromNames__module_not_loaded(self): # We're going to try to load this module as a side-effect, so it # better not be loaded before we try. # - module_name = 'unittest.test.dummy' + module_name = 'test.test_unittest.dummy' sys.modules.pop(module_name, None) loader = unittest.TestLoader() @@ -1604,39 +1505,6 @@ def test(self): pass def reverse_three_way_cmp(a, b): return unittest.util.three_way_cmp(b, a) - def test_getTestCaseNames(self): - with self.assertWarns(DeprecationWarning) as w: - tests = unittest.getTestCaseNames(self.MyTestCase, - prefix='check', sortUsing=self.reverse_three_way_cmp, - testNamePatterns=None) - self.assertEqual(w.filename, __file__) - self.assertEqual(tests, ['check_2', 'check_1']) - - def test_makeSuite(self): - with self.assertWarns(DeprecationWarning) as w: - suite = unittest.makeSuite(self.MyTestCase, - prefix='check', sortUsing=self.reverse_three_way_cmp, - suiteClass=self.MyTestSuite) - self.assertEqual(w.filename, __file__) - self.assertIsInstance(suite, self.MyTestSuite) - expected = self.MyTestSuite([self.MyTestCase('check_2'), - self.MyTestCase('check_1')]) - self.assertEqual(suite, expected) - - def test_findTestCases(self): - m = types.ModuleType('m') - m.testcase_1 = self.MyTestCase - - with self.assertWarns(DeprecationWarning) as w: - suite = unittest.findTestCases(m, - prefix='check', sortUsing=self.reverse_three_way_cmp, - suiteClass=self.MyTestSuite) - self.assertEqual(w.filename, __file__) - self.assertIsInstance(suite, self.MyTestSuite) - expected = [self.MyTestSuite([self.MyTestCase('check_2'), - self.MyTestCase('check_1')])] - self.assertEqual(list(suite), expected) - if __name__ == "__main__": unittest.main() diff --git a/Lib/unittest/test/test_program.py b/Lib/test/test_unittest/test_program.py similarity index 80% rename from Lib/unittest/test/test_program.py rename to Lib/test/test_unittest/test_program.py index 26a8550af8..8256aaeb8c 100644 --- a/Lib/unittest/test/test_program.py +++ b/Lib/test/test_unittest/test_program.py @@ -1,21 +1,20 @@ -import io - import os import sys import subprocess from test import support import unittest -import unittest.test -from unittest.test.test_result import BufferedWriter +import test.test_unittest +from test.test_unittest.test_result import BufferedWriter +@support.force_not_colorized_test_class class Test_TestProgram(unittest.TestCase): def test_discovery_from_dotted_path(self): loader = unittest.TestLoader() tests = [self] - expectedPath = os.path.abspath(os.path.dirname(unittest.test.__file__)) + expectedPath = os.path.abspath(os.path.dirname(test.test_unittest.__file__)) self.wasRun = False def _find_tests(start_dir, pattern): @@ -23,7 +22,7 @@ def _find_tests(start_dir, pattern): self.assertEqual(start_dir, expectedPath) return tests loader._find_tests = _find_tests - suite = loader.discover('unittest.test') + suite = loader.discover('test.test_unittest') self.assertTrue(self.wasRun) self.assertEqual(suite._tests, tests) @@ -73,15 +72,22 @@ def testExpectedFailure(self): def testUnexpectedSuccess(self): pass - class FooBarLoader(unittest.TestLoader): - """Test loader that returns a suite containing FooBar.""" + class Empty(unittest.TestCase): + pass + + class TestLoader(unittest.TestLoader): + """Test loader that returns a suite containing the supplied testcase.""" + + def __init__(self, testcase): + self.testcase = testcase + def loadTestsFromModule(self, module): return self.suiteClass( - [self.loadTestsFromTestCase(Test_TestProgram.FooBar)]) + [self.loadTestsFromTestCase(self.testcase)]) def loadTestsFromNames(self, names, module): return self.suiteClass( - [self.loadTestsFromTestCase(Test_TestProgram.FooBar)]) + [self.loadTestsFromTestCase(self.testcase)]) def test_defaultTest_with_string(self): class FakeRunner(object): @@ -93,10 +99,10 @@ def run(self, test): sys.argv = ['faketest'] runner = FakeRunner() program = unittest.TestProgram(testRunner=runner, exit=False, - defaultTest='unittest.test', - testLoader=self.FooBarLoader()) + defaultTest='test.test_unittest', + testLoader=self.TestLoader(self.FooBar)) sys.argv = old_argv - self.assertEqual(('unittest.test',), program.testNames) + self.assertEqual(('test.test_unittest',), program.testNames) def test_defaultTest_with_iterable(self): class FakeRunner(object): @@ -109,10 +115,10 @@ def run(self, test): runner = FakeRunner() program = unittest.TestProgram( testRunner=runner, exit=False, - defaultTest=['unittest.test', 'unittest.test2'], - testLoader=self.FooBarLoader()) + defaultTest=['test.test_unittest', 'test.test_unittest2'], + testLoader=self.TestLoader(self.FooBar)) sys.argv = old_argv - self.assertEqual(['unittest.test', 'unittest.test2'], + self.assertEqual(['test.test_unittest', 'test.test_unittest2'], program.testNames) def test_NonExit(self): @@ -120,7 +126,7 @@ def test_NonExit(self): program = unittest.main(exit=False, argv=["foobar"], testRunner=unittest.TextTestRunner(stream=stream), - testLoader=self.FooBarLoader()) + testLoader=self.TestLoader(self.FooBar)) self.assertTrue(hasattr(program, 'result')) out = stream.getvalue() self.assertIn('\nFAIL: testFail ', out) @@ -132,13 +138,13 @@ def test_NonExit(self): def test_Exit(self): stream = BufferedWriter() - self.assertRaises( - SystemExit, - unittest.main, - argv=["foobar"], - testRunner=unittest.TextTestRunner(stream=stream), - exit=True, - testLoader=self.FooBarLoader()) + with self.assertRaises(SystemExit) as cm: + unittest.main( + argv=["foobar"], + testRunner=unittest.TextTestRunner(stream=stream), + exit=True, + testLoader=self.TestLoader(self.FooBar)) + self.assertEqual(cm.exception.code, 1) out = stream.getvalue() self.assertIn('\nFAIL: testFail ', out) self.assertIn('\nERROR: testError ', out) @@ -149,12 +155,11 @@ def test_Exit(self): def test_ExitAsDefault(self): stream = BufferedWriter() - self.assertRaises( - SystemExit, - unittest.main, - argv=["foobar"], - testRunner=unittest.TextTestRunner(stream=stream), - testLoader=self.FooBarLoader()) + with self.assertRaises(SystemExit): + unittest.main( + argv=["foobar"], + testRunner=unittest.TextTestRunner(stream=stream), + testLoader=self.TestLoader(self.FooBar)) out = stream.getvalue() self.assertIn('\nFAIL: testFail ', out) self.assertIn('\nERROR: testError ', out) @@ -163,6 +168,29 @@ def test_ExitAsDefault(self): 'expected failures=1, unexpected successes=1)\n') self.assertTrue(out.endswith(expected)) + def test_ExitSkippedSuite(self): + stream = BufferedWriter() + with self.assertRaises(SystemExit) as cm: + unittest.main( + argv=["foobar", "-k", "testSkipped"], + testRunner=unittest.TextTestRunner(stream=stream), + testLoader=self.TestLoader(self.FooBar)) + self.assertEqual(cm.exception.code, 0) + out = stream.getvalue() + expected = '\n\nOK (skipped=1)\n' + self.assertTrue(out.endswith(expected)) + + def test_ExitEmptySuite(self): + stream = BufferedWriter() + with self.assertRaises(SystemExit) as cm: + unittest.main( + argv=["empty"], + testRunner=unittest.TextTestRunner(stream=stream), + testLoader=self.TestLoader(self.Empty)) + self.assertEqual(cm.exception.code, 5) + out = stream.getvalue() + self.assertIn('\nNO TESTS RAN\n', out) + class InitialisableProgram(unittest.TestProgram): exit = False @@ -286,6 +314,7 @@ def testRunTestsRunnerClass(self): program.failfast = 'failfast' program.buffer = 'buffer' program.warnings = 'warnings' + program.durations = '5' program.runTests() @@ -293,7 +322,8 @@ def testRunTestsRunnerClass(self): 'failfast': 'failfast', 'buffer': 'buffer', 'tb_locals': False, - 'warnings': 'warnings'}) + 'warnings': 'warnings', + 'durations': '5'}) self.assertEqual(FakeRunner.test, 'test') self.assertIs(program.result, RESULT) @@ -322,7 +352,8 @@ def test_locals(self): 'failfast': False, 'tb_locals': True, 'verbosity': 1, - 'warnings': None}) + 'warnings': None, + 'durations': None}) def testRunTestsOldRunnerClass(self): program = self.program @@ -335,6 +366,7 @@ def testRunTestsOldRunnerClass(self): program.failfast = 'failfast' program.buffer = 'buffer' program.test = 'test' + program.durations = '0' program.runTests() @@ -358,6 +390,7 @@ def fakeInstallHandler(): program = self.program program.catchbreak = True + program.durations = None program.testRunner = FakeRunner @@ -427,8 +460,8 @@ def _join(name): def testParseArgsAbsolutePathsThatCannotBeConverted(self): program = self.program - # even on Windows '/...' is considered absolute by os.path.abspath - argv = ['progname', '/foo/bar/baz.py', '/green/red.py'] + drive = os.path.splitdrive(os.getcwd())[0] + argv = ['progname', f'{drive}/foo/bar/baz.py', f'{drive}/green/red.py'] self._patch_isfile(argv) program.createTests = lambda: None @@ -452,6 +485,8 @@ def testParseArgsSelectedTestNames(self): self.assertEqual(program.testNamePatterns, ['*foo*', '*bar*', '*pat*']) + # TODO: RUSTPYTHON + @unittest.expectedFailureIf(sys.platform != 'win32', "TODO: RUSTPYTHON") def testSelectedTestNamesFunctionalTest(self): def run_unittest(args): # Use -E to ignore PYTHONSAFEPATH env var @@ -463,14 +498,14 @@ def run_unittest(args): return stderr.decode() t = '_test_warnings' - self.assertIn('Ran 7 tests', run_unittest([t])) - self.assertIn('Ran 7 tests', run_unittest(['-k', 'TestWarnings', t])) - self.assertIn('Ran 7 tests', run_unittest(['discover', '-p', '*_test*', '-k', 'TestWarnings'])) - self.assertIn('Ran 2 tests', run_unittest(['-k', 'f', t])) - self.assertIn('Ran 7 tests', run_unittest(['-k', 't', t])) - self.assertIn('Ran 3 tests', run_unittest(['-k', '*t', t])) - self.assertIn('Ran 7 tests', run_unittest(['-k', '*test_warnings.*Warning*', t])) - self.assertIn('Ran 1 test', run_unittest(['-k', '*test_warnings.*warning*', t])) + self.assertIn('Ran 5 tests', run_unittest([t])) + self.assertIn('Ran 5 tests', run_unittest(['-k', 'TestWarnings', t])) + self.assertIn('Ran 5 tests', run_unittest(['discover', '-p', '*_test*', '-k', 'TestWarnings'])) + self.assertIn('Ran 1 test ', run_unittest(['-k', 'f', t])) + self.assertIn('Ran 5 tests', run_unittest(['-k', 't', t])) + self.assertIn('Ran 2 tests', run_unittest(['-k', '*t', t])) + self.assertIn('Ran 5 tests', run_unittest(['-k', '*test_warnings.*Warning*', t])) + self.assertIn('Ran 1 test ', run_unittest(['-k', '*test_warnings.*warning*', t])) if __name__ == '__main__': diff --git a/Lib/unittest/test/test_result.py b/Lib/test/test_unittest/test_result.py similarity index 98% rename from Lib/unittest/test/test_result.py rename to Lib/test/test_unittest/test_result.py index 9320b0a44b..4d552d54e9 100644 --- a/Lib/unittest/test/test_result.py +++ b/Lib/test/test_unittest/test_result.py @@ -1,12 +1,12 @@ import io import sys import textwrap - -from test.support import warnings_helper, captured_stdout, captured_stderr - import traceback import unittest from unittest.util import strclass +from test.support import warnings_helper +from test.support import captured_stdout, force_not_colorized_test_class +from test.test_unittest.support import BufferedWriter class MockTraceback(object): @@ -33,22 +33,7 @@ def bad_cleanup2(): raise ValueError('bad cleanup2') -class BufferedWriter: - def __init__(self): - self.result = '' - self.buffer = '' - - def write(self, arg): - self.buffer += arg - - def flush(self): - self.result += self.buffer - self.buffer = '' - - def getvalue(self): - return self.result - - +@force_not_colorized_test_class class Test_TestResult(unittest.TestCase): # Note: there are not separate tests for TestResult.wasSuccessful(), # TestResult.errors, TestResult.failures, TestResult.testsRun or @@ -465,12 +450,14 @@ def testFailFastSetByRunner(self): stream = BufferedWriter() runner = unittest.TextTestRunner(stream=stream, failfast=True) def test(result): + result.testsRun += 1 self.assertTrue(result.failfast) result = runner.run(test) stream.flush() self.assertTrue(stream.getvalue().endswith('\n\nOK\n')) +@force_not_colorized_test_class class Test_TextTestResult(unittest.TestCase): maxDiff = None @@ -505,8 +492,8 @@ def testGetSubTestDescriptionWithoutDocstringAndParams(self): '(' + __name__ + '.Test_TextTestResult.testGetSubTestDescriptionWithoutDocstringAndParams) ' '(<subtest>)') - def testGetSubTestDescriptionForFalsyValues(self): - expected = 'testGetSubTestDescriptionForFalsyValues (%s.Test_TextTestResult.testGetSubTestDescriptionForFalsyValues) [%s]' + def testGetSubTestDescriptionForFalseValues(self): + expected = 'testGetSubTestDescriptionForFalseValues (%s.Test_TextTestResult.testGetSubTestDescriptionForFalseValues) [%s]' result = unittest.TextTestResult(None, True, 1) for arg in [0, None, []]: with self.subTest(arg): @@ -772,6 +759,7 @@ def testFoo(self): runner.run(Test('testFoo')) +@force_not_colorized_test_class class TestOutputBuffering(unittest.TestCase): def setUp(self): diff --git a/Lib/unittest/test/test_runner.py b/Lib/test/test_unittest/test_runner.py similarity index 89% rename from Lib/unittest/test/test_runner.py rename to Lib/test/test_unittest/test_runner.py index 50d9ed2fd3..a652c71513 100644 --- a/Lib/unittest/test/test_runner.py +++ b/Lib/test/test_unittest/test_runner.py @@ -8,8 +8,11 @@ import unittest from unittest.case import _Outcome -from unittest.test.support import (LoggingResult, - ResultWithNoStartTestRunStopTestRun) +from test.test_unittest.support import ( + BufferedWriter, + LoggingResult, + ResultWithNoStartTestRunStopTestRun, +) def resultFactory(*_): @@ -21,6 +24,13 @@ def getRunner(): stream=io.StringIO()) +class CustomError(Exception): + pass + +# For test output compat: +CustomErrorRepr = f"{__name__ + '.' if __name__ != '__main__' else ''}CustomError" + + def runTests(*cases): suite = unittest.TestSuite() for case in cases: @@ -43,7 +53,7 @@ def cleanup(ordering, blowUp=False): ordering.append('cleanup_good') else: ordering.append('cleanup_exc') - raise Exception('CleanUpExc') + raise CustomError('CleanUpExc') class TestCM: @@ -96,6 +106,7 @@ def cleanup2(*args, **kwargs): self.assertTrue(test.doCleanups()) self.assertEqual(cleanups, [(2, (), {}), (1, (1, 2, 3), dict(four='hello', five='goodbye'))]) + @support.force_not_colorized def testCleanUpWithErrors(self): class TestableTest(unittest.TestCase): def testNothing(self): @@ -105,8 +116,8 @@ def testNothing(self): result = unittest.TestResult() outcome = test._outcome = _Outcome(result=result) - CleanUpExc = Exception('foo') - exc2 = Exception('bar') + CleanUpExc = CustomError('foo') + exc2 = CustomError('bar') def cleanup1(): raise CleanUpExc @@ -122,10 +133,10 @@ def cleanup2(): (_, msg2), (_, msg1) = result.errors self.assertIn('in cleanup1', msg1) self.assertIn('raise CleanUpExc', msg1) - self.assertIn('Exception: foo', msg1) + self.assertIn(f'{CustomErrorRepr}: foo', msg1) self.assertIn('in cleanup2', msg2) self.assertIn('raise exc2', msg2) - self.assertIn('Exception: bar', msg2) + self.assertIn(f'{CustomErrorRepr}: bar', msg2) def testCleanupInRun(self): blowUp = False @@ -136,7 +147,7 @@ def setUp(self): ordering.append('setUp') test.addCleanup(cleanup2) if blowUp: - raise Exception('foo') + raise CustomError('foo') def testNothing(self): ordering.append('test') @@ -239,6 +250,7 @@ def testNothing(self): self.assertEqual(test._cleanups, []) +@support.force_not_colorized_test_class class TestClassCleanup(unittest.TestCase): def test_addClassCleanUp(self): class TestableTest(unittest.TestCase): @@ -277,7 +289,7 @@ def setUpClass(cls): ordering.append('setUpClass') cls.addClassCleanup(cleanup, ordering) if blowUp: - raise Exception() + raise CustomError() def testNothing(self): ordering.append('test') @classmethod @@ -303,7 +315,7 @@ def setUpClass(cls): ordering.append('setUpClass') cls.addClassCleanup(cleanup, ordering) if blowUp: - raise Exception() + raise CustomError() def testNothing(self): ordering.append('test') @classmethod @@ -343,7 +355,7 @@ def tearDownClass(cls): ordering = [] blowUp = True suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestableTest) - with self.assertRaises(Exception) as cm: + with self.assertRaises(CustomError) as cm: suite.debug() self.assertEqual(str(cm.exception), 'CleanUpExc') self.assertEqual(ordering, @@ -363,10 +375,10 @@ def testNothing(self): @classmethod def tearDownClass(cls): ordering.append('tearDownClass') - raise Exception('TearDownClassExc') + raise CustomError('TearDownClassExc') suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestableTest) - with self.assertRaises(Exception) as cm: + with self.assertRaises(CustomError) as cm: suite.debug() self.assertEqual(str(cm.exception), 'TearDownClassExc') self.assertEqual(ordering, ['setUpClass', 'test', 'tearDownClass']) @@ -376,7 +388,7 @@ def tearDownClass(cls): ordering = [] blowUp = True suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestableTest) - with self.assertRaises(Exception) as cm: + with self.assertRaises(CustomError) as cm: suite.debug() self.assertEqual(str(cm.exception), 'TearDownClassExc') self.assertEqual(ordering, ['setUpClass', 'test', 'tearDownClass']) @@ -389,16 +401,22 @@ def testNothing(self): pass def cleanup1(): - raise Exception('cleanup1') + raise CustomError('cleanup1') def cleanup2(): - raise Exception('cleanup2') + raise CustomError('cleanup2') TestableTest.addClassCleanup(cleanup1) TestableTest.addClassCleanup(cleanup2) - with self.assertRaises(Exception) as e: - TestableTest.doClassCleanups() - self.assertEqual(e, 'cleanup1') + TestableTest.doClassCleanups() + + self.assertEqual(len(TestableTest.tearDown_exceptions), 2) + + e1, e2 = TestableTest.tearDown_exceptions + self.assertIsInstance(e1[1], CustomError) + self.assertEqual(str(e1[1]), 'cleanup2') + self.assertIsInstance(e2[1], CustomError) + self.assertEqual(str(e2[1]), 'cleanup1') def test_with_errors_addCleanUp(self): ordering = [] @@ -418,7 +436,7 @@ def tearDownClass(cls): result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: CleanUpExc') + f'{CustomErrorRepr}: CleanUpExc') self.assertEqual(ordering, ['setUpClass', 'setUp', 'cleanup_exc', 'tearDownClass', 'cleanup_good']) @@ -441,7 +459,7 @@ def tearDownClass(cls): result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: CleanUpExc') + f'{CustomErrorRepr}: CleanUpExc') self.assertEqual(ordering, ['setUpClass', 'setUp', 'test', 'cleanup_good', 'tearDownClass', 'cleanup_exc']) @@ -457,11 +475,11 @@ def setUpClass(cls): ordering.append('setUpClass') cls.addClassCleanup(cleanup, ordering, blowUp=True) if class_blow_up: - raise Exception('ClassExc') + raise CustomError('ClassExc') def setUp(self): ordering.append('setUp') if method_blow_up: - raise Exception('MethodExc') + raise CustomError('MethodExc') def testNothing(self): ordering.append('test') @classmethod @@ -470,7 +488,7 @@ def tearDownClass(cls): result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: CleanUpExc') + f'{CustomErrorRepr}: CleanUpExc') self.assertEqual(ordering, ['setUpClass', 'setUp', 'test', 'tearDownClass', 'cleanup_exc']) @@ -480,9 +498,9 @@ def tearDownClass(cls): method_blow_up = False result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: ClassExc') + f'{CustomErrorRepr}: ClassExc') self.assertEqual(result.errors[1][1].splitlines()[-1], - 'Exception: CleanUpExc') + f'{CustomErrorRepr}: CleanUpExc') self.assertEqual(ordering, ['setUpClass', 'cleanup_exc']) @@ -491,9 +509,9 @@ def tearDownClass(cls): method_blow_up = True result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: MethodExc') + f'{CustomErrorRepr}: MethodExc') self.assertEqual(result.errors[1][1].splitlines()[-1], - 'Exception: CleanUpExc') + f'{CustomErrorRepr}: CleanUpExc') self.assertEqual(ordering, ['setUpClass', 'setUp', 'tearDownClass', 'cleanup_exc']) @@ -510,11 +528,11 @@ def testNothing(self): @classmethod def tearDownClass(cls): ordering.append('tearDownClass') - raise Exception('TearDownExc') + raise CustomError('TearDownExc') result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: TearDownExc') + f'{CustomErrorRepr}: TearDownExc') self.assertEqual(ordering, ['setUpClass', 'test', 'tearDownClass', 'cleanup_good']) @@ -574,7 +592,18 @@ def test(self): 'inner setup', 'inner test', 'inner cleanup', 'end outer test', 'outer cleanup']) + def test_run_empty_suite_error_message(self): + class EmptyTest(unittest.TestCase): + pass + suite = unittest.defaultTestLoader.loadTestsFromTestCase(EmptyTest) + runner = getRunner() + runner.run(suite) + + self.assertIn("\nNO TESTS RAN\n", runner.stream.getvalue()) + + +@support.force_not_colorized_test_class class TestModuleCleanUp(unittest.TestCase): def test_add_and_do_ModuleCleanup(self): module_cleanups = [] @@ -607,7 +636,7 @@ def module_cleanup_good(*args, **kwargs): module_cleanups.append((3, args, kwargs)) def module_cleanup_bad(*args, **kwargs): - raise Exception('CleanUpExc') + raise CustomError('CleanUpExc') class Module(object): unittest.addModuleCleanup(module_cleanup_good, 1, 2, 3, @@ -617,7 +646,7 @@ class Module(object): [(module_cleanup_good, (1, 2, 3), dict(four='hello', five='goodbye')), (module_cleanup_bad, (), {})]) - with self.assertRaises(Exception) as e: + with self.assertRaises(CustomError) as e: unittest.case.doModuleCleanups() self.assertEqual(str(e.exception), 'CleanUpExc') self.assertEqual(unittest.case._module_cleanups, []) @@ -646,7 +675,7 @@ def setUpModule(): ordering.append('setUpModule') unittest.addModuleCleanup(cleanup, ordering) if blowUp: - raise Exception('setUpModule Exc') + raise CustomError('setUpModule Exc') @staticmethod def tearDownModule(): ordering.append('tearDownModule') @@ -666,7 +695,7 @@ def tearDownClass(cls): result = runTests(TestableTest) self.assertEqual(ordering, ['setUpModule', 'cleanup_good']) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: setUpModule Exc') + f'{CustomErrorRepr}: setUpModule Exc') ordering = [] blowUp = False @@ -686,7 +715,7 @@ def setUpModule(): ordering.append('setUpModule') unittest.addModuleCleanup(cleanup, ordering) if blowUp: - raise Exception() + raise CustomError() @staticmethod def tearDownModule(): ordering.append('tearDownModule') @@ -697,7 +726,7 @@ def setUpModule(): ordering.append('setUpModule2') unittest.addModuleCleanup(cleanup, ordering) if blowUp2: - raise Exception() + raise CustomError() @staticmethod def tearDownModule(): ordering.append('tearDownModule2') @@ -786,7 +815,7 @@ def setUpModule(): @staticmethod def tearDownModule(): ordering.append('tearDownModule') - raise Exception('CleanUpExc') + raise CustomError('CleanUpExc') class TestableTest(unittest.TestCase): @classmethod @@ -802,7 +831,7 @@ def tearDownClass(cls): sys.modules['Module'] = Module result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: CleanUpExc') + f'{CustomErrorRepr}: CleanUpExc') self.assertEqual(ordering, ['setUpModule', 'setUpClass', 'test', 'tearDownClass', 'tearDownModule', 'cleanup_good']) @@ -842,7 +871,7 @@ def tearDownClass(cls): ordering = [] blowUp = True suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestableTest) - with self.assertRaises(Exception) as cm: + with self.assertRaises(CustomError) as cm: suite.debug() self.assertEqual(str(cm.exception), 'CleanUpExc') self.assertEqual(ordering, ['setUpModule', 'setUpClass', 'test', @@ -860,7 +889,7 @@ def setUpModule(): @staticmethod def tearDownModule(): ordering.append('tearDownModule') - raise Exception('TearDownModuleExc') + raise CustomError('TearDownModuleExc') class TestableTest(unittest.TestCase): @classmethod @@ -875,7 +904,7 @@ def tearDownClass(cls): TestableTest.__module__ = 'Module' sys.modules['Module'] = Module suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestableTest) - with self.assertRaises(Exception) as cm: + with self.assertRaises(CustomError) as cm: suite.debug() self.assertEqual(str(cm.exception), 'TearDownModuleExc') self.assertEqual(ordering, ['setUpModule', 'setUpClass', 'test', @@ -886,7 +915,7 @@ def tearDownClass(cls): ordering = [] blowUp = True suite = unittest.defaultTestLoader.loadTestsFromTestCase(TestableTest) - with self.assertRaises(Exception) as cm: + with self.assertRaises(CustomError) as cm: suite.debug() self.assertEqual(str(cm.exception), 'TearDownModuleExc') self.assertEqual(ordering, ['setUpModule', 'setUpClass', 'test', @@ -965,7 +994,7 @@ def tearDownClass(cls): result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: CleanUpExc') + f'{CustomErrorRepr}: CleanUpExc') self.assertEqual(ordering, ['setUpModule', 'setUpClass', 'test', 'tearDownClass', 'cleanup_exc', 'tearDownModule', 'cleanup_good']) @@ -995,7 +1024,7 @@ def tearDown(self): result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: CleanUpExc') + f'{CustomErrorRepr}: CleanUpExc') self.assertEqual(ordering, ['setUpModule', 'setUp', 'test', 'tearDown', 'cleanup_exc', 'tearDownModule', 'cleanup_good']) @@ -1011,7 +1040,7 @@ def setUpModule(): ordering.append('setUpModule') unittest.addModuleCleanup(cleanup, ordering, blowUp=True) if module_blow_up: - raise Exception('ModuleExc') + raise CustomError('ModuleExc') @staticmethod def tearDownModule(): ordering.append('tearDownModule') @@ -1021,11 +1050,11 @@ class TestableTest(unittest.TestCase): def setUpClass(cls): ordering.append('setUpClass') if class_blow_up: - raise Exception('ClassExc') + raise CustomError('ClassExc') def setUp(self): ordering.append('setUp') if method_blow_up: - raise Exception('MethodExc') + raise CustomError('MethodExc') def testNothing(self): ordering.append('test') @classmethod @@ -1037,7 +1066,7 @@ def tearDownClass(cls): result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: CleanUpExc') + f'{CustomErrorRepr}: CleanUpExc') self.assertEqual(ordering, ['setUpModule', 'setUpClass', 'setUp', 'test', 'tearDownClass', 'tearDownModule', @@ -1049,9 +1078,9 @@ def tearDownClass(cls): method_blow_up = False result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: ModuleExc') + f'{CustomErrorRepr}: ModuleExc') self.assertEqual(result.errors[1][1].splitlines()[-1], - 'Exception: CleanUpExc') + f'{CustomErrorRepr}: CleanUpExc') self.assertEqual(ordering, ['setUpModule', 'cleanup_exc']) ordering = [] @@ -1060,9 +1089,9 @@ def tearDownClass(cls): method_blow_up = False result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: ClassExc') + f'{CustomErrorRepr}: ClassExc') self.assertEqual(result.errors[1][1].splitlines()[-1], - 'Exception: CleanUpExc') + f'{CustomErrorRepr}: CleanUpExc') self.assertEqual(ordering, ['setUpModule', 'setUpClass', 'tearDownModule', 'cleanup_exc']) @@ -1072,9 +1101,9 @@ def tearDownClass(cls): method_blow_up = True result = runTests(TestableTest) self.assertEqual(result.errors[0][1].splitlines()[-1], - 'Exception: MethodExc') + f'{CustomErrorRepr}: MethodExc') self.assertEqual(result.errors[1][1].splitlines()[-1], - 'Exception: CleanUpExc') + f'{CustomErrorRepr}: CleanUpExc') self.assertEqual(ordering, ['setUpModule', 'setUpClass', 'setUp', 'tearDownClass', 'tearDownModule', 'cleanup_exc']) @@ -1176,6 +1205,7 @@ def test_init(self): self.assertTrue(runner.descriptions) self.assertEqual(runner.resultclass, unittest.TextTestResult) self.assertFalse(runner.tb_locals) + self.assertIsNone(runner.durations) def test_multiple_inheritance(self): class AResult(unittest.TestResult): @@ -1267,7 +1297,7 @@ def _makeResult(self): expected = ['startTestRun', 'stopTestRun'] self.assertEqual(events, expected) - # TODO: RUSTPYTHON; fix pickling with io objects + # TODO: RUSTPYTHON @unittest.expectedFailure def test_pickle_unpickle(self): # Issue #7197: a TextTestRunner should be (un)pickleable. This is @@ -1293,6 +1323,7 @@ def MockResultClass(*args): expectedresult = (runner.stream, DESCRIPTIONS, VERBOSITY) self.assertEqual(runner._makeResult(), expectedresult) + @support.force_not_colorized @support.requires_subprocess() def test_warnings(self): """ @@ -1305,8 +1336,6 @@ def get_parse_out_err(p): return [b.splitlines() for b in p.communicate()] opts = dict(stdout=subprocess.PIPE, stderr=subprocess.PIPE, cwd=os.path.dirname(__file__)) - ae_msg = b'Please use assertEqual instead.' - at_msg = b'Please use assertTrue instead.' # no args -> all the warnings are printed, unittest warnings only once p = subprocess.Popen([sys.executable, '-E', '_test_warnings.py'], **opts) @@ -1314,11 +1343,11 @@ def get_parse_out_err(p): out, err = get_parse_out_err(p) self.assertIn(b'OK', err) # check that the total number of warnings in the output is correct - self.assertEqual(len(out), 12) + self.assertEqual(len(out), 10) # check that the numbers of the different kind of warnings is correct for msg in [b'dw', b'iw', b'uw']: self.assertEqual(out.count(msg), 3) - for msg in [ae_msg, at_msg, b'rw']: + for msg in [b'rw']: self.assertEqual(out.count(msg), 1) args_list = ( @@ -1345,11 +1374,9 @@ def get_parse_out_err(p): with p: out, err = get_parse_out_err(p) self.assertIn(b'OK', err) - self.assertEqual(len(out), 14) + self.assertEqual(len(out), 12) for msg in [b'dw', b'iw', b'uw', b'rw']: self.assertEqual(out.count(msg), 3) - for msg in [ae_msg, at_msg]: - self.assertEqual(out.count(msg), 1) def testStdErrLookedUpAtInstantiationTime(self): # see issue 10786 @@ -1368,6 +1395,65 @@ def testSpecifiedStreamUsed(self): runner = unittest.TextTestRunner(f) self.assertTrue(runner.stream.stream is f) + def test_durations(self): + def run(test, *, expect_durations=True): + stream = BufferedWriter() + runner = unittest.TextTestRunner(stream=stream, durations=5, verbosity=2) + result = runner.run(test) + self.assertEqual(result.durations, 5) + stream.flush() + text = stream.getvalue() + regex = r"\n\d+.\d\d\ds" + if expect_durations: + self.assertEqual(len(result.collectedDurations), 1) + self.assertIn('Slowest test durations', text) + self.assertRegex(text, regex) + else: + self.assertEqual(len(result.collectedDurations), 0) + self.assertNotIn('Slowest test durations', text) + self.assertNotRegex(text, regex) + + # success + class Foo(unittest.TestCase): + def test_1(self): + pass + + run(Foo('test_1'), expect_durations=True) + + # failure + class Foo(unittest.TestCase): + def test_1(self): + self.assertEqual(0, 1) + + run(Foo('test_1'), expect_durations=True) + + # error + class Foo(unittest.TestCase): + def test_1(self): + 1 / 0 + + run(Foo('test_1'), expect_durations=True) + + + # error in setUp and tearDown + class Foo(unittest.TestCase): + def setUp(self): + 1 / 0 + tearDown = setUp + def test_1(self): + pass + + run(Foo('test_1'), expect_durations=True) + + # skip (expect no durations) + class Foo(unittest.TestCase): + @unittest.skip("reason") + def test_1(self): + pass + + run(Foo('test_1'), expect_durations=False) + + if __name__ == "__main__": unittest.main() diff --git a/Lib/unittest/test/test_setups.py b/Lib/test/test_unittest/test_setups.py similarity index 100% rename from Lib/unittest/test/test_setups.py rename to Lib/test/test_unittest/test_setups.py diff --git a/Lib/unittest/test/test_skipping.py b/Lib/test/test_unittest/test_skipping.py similarity index 99% rename from Lib/unittest/test/test_skipping.py rename to Lib/test/test_unittest/test_skipping.py index 64ceeae37e..f146dcac18 100644 --- a/Lib/unittest/test/test_skipping.py +++ b/Lib/test/test_unittest/test_skipping.py @@ -1,6 +1,6 @@ import unittest -from unittest.test.support import LoggingResult +from test.test_unittest.support import LoggingResult class Test_TestSkipping(unittest.TestCase): diff --git a/Lib/unittest/test/test_suite.py b/Lib/test/test_unittest/test_suite.py similarity index 99% rename from Lib/unittest/test/test_suite.py rename to Lib/test/test_unittest/test_suite.py index 0551a16996..ca52ee9d9c 100644 --- a/Lib/unittest/test/test_suite.py +++ b/Lib/test/test_unittest/test_suite.py @@ -3,7 +3,7 @@ import gc import sys import weakref -from unittest.test.support import LoggingResult, TestEquality +from test.test_unittest.support import LoggingResult, TestEquality ### Support code for Test_TestSuite diff --git a/Lib/test/test_unittest/test_util.py b/Lib/test/test_unittest/test_util.py new file mode 100644 index 0000000000..d590a33393 --- /dev/null +++ b/Lib/test/test_unittest/test_util.py @@ -0,0 +1,33 @@ +import unittest +from unittest.util import safe_repr, sorted_list_difference, unorderable_list_difference + + +class TestUtil(unittest.TestCase): + def test_safe_repr(self): + class RaisingRepr: + def __repr__(self): + raise ValueError("Invalid repr()") + + class LongRepr: + def __repr__(self): + return 'x' * 100 + + safe_repr(RaisingRepr()) + self.assertEqual(safe_repr('foo'), "'foo'") + self.assertEqual(safe_repr(LongRepr(), short=True), 'x'*80 + ' [truncated]...') + + def test_sorted_list_difference(self): + self.assertEqual(sorted_list_difference([], []), ([], [])) + self.assertEqual(sorted_list_difference([1, 2], [2, 3]), ([1], [3])) + self.assertEqual(sorted_list_difference([1, 2], [1, 3]), ([2], [3])) + self.assertEqual(sorted_list_difference([1, 1, 1], [1, 2, 3]), ([], [2, 3])) + self.assertEqual(sorted_list_difference([4], [1, 2, 3, 4]), ([], [1, 2, 3])) + self.assertEqual(sorted_list_difference([1, 1], [2]), ([1], [2])) + self.assertEqual(sorted_list_difference([2], [1, 1]), ([2], [1])) + self.assertEqual(sorted_list_difference([1, 2], [1, 1]), ([2], [])) + + def test_unorderable_list_difference(self): + self.assertEqual(unorderable_list_difference([], []), ([], [])) + self.assertEqual(unorderable_list_difference([1, 2], []), ([2, 1], [])) + self.assertEqual(unorderable_list_difference([], [1, 2]), ([], [1, 2])) + self.assertEqual(unorderable_list_difference([1, 2], [1, 3]), ([2], [3])) diff --git a/Lib/test/test_unittest/testmock/__init__.py b/Lib/test/test_unittest/testmock/__init__.py new file mode 100644 index 0000000000..bc502ef32d --- /dev/null +++ b/Lib/test/test_unittest/testmock/__init__.py @@ -0,0 +1,6 @@ +import os.path +from test.support import load_package_tests + + +def load_tests(*args): + return load_package_tests(os.path.dirname(__file__), *args) diff --git a/Lib/unittest/test/testmock/__main__.py b/Lib/test/test_unittest/testmock/__main__.py similarity index 86% rename from Lib/unittest/test/testmock/__main__.py rename to Lib/test/test_unittest/testmock/__main__.py index 45c633a4ee..1e3068b0dd 100644 --- a/Lib/unittest/test/testmock/__main__.py +++ b/Lib/test/test_unittest/testmock/__main__.py @@ -6,7 +6,7 @@ def load_tests(loader, standard_tests, pattern): # top level directory cached on loader instance this_dir = os.path.dirname(__file__) pattern = pattern or "test*.py" - # We are inside unittest.test.testmock, so the top-level is three notches up + # We are inside test.test_unittest.testmock, so the top-level is three notches up top_level_dir = os.path.dirname(os.path.dirname(os.path.dirname(this_dir))) package_tests = loader.discover(start_dir=this_dir, pattern=pattern, top_level_dir=top_level_dir) diff --git a/Lib/unittest/test/testmock/support.py b/Lib/test/test_unittest/testmock/support.py similarity index 57% rename from Lib/unittest/test/testmock/support.py rename to Lib/test/test_unittest/testmock/support.py index 49986d65dc..6c535b7944 100644 --- a/Lib/unittest/test/testmock/support.py +++ b/Lib/test/test_unittest/testmock/support.py @@ -14,3 +14,14 @@ def wibble(self): pass class X(object): pass + +# A standin for weurkzeug.local.LocalProxy - issue 119600 +def _inaccessible(*args, **kwargs): + raise AttributeError + + +class OpaqueProxy: + __getattribute__ = _inaccessible + + +g = OpaqueProxy() diff --git a/Lib/unittest/test/testmock/testasync.py b/Lib/test/test_unittest/testmock/testasync.py similarity index 94% rename from Lib/unittest/test/testmock/testasync.py rename to Lib/test/test_unittest/testmock/testasync.py index 90ea72d82b..06ec7ea204 100644 --- a/Lib/unittest/test/testmock/testasync.py +++ b/Lib/test/test_unittest/testmock/testasync.py @@ -218,10 +218,6 @@ def test_create_autospec_instance(self): with self.assertRaises(RuntimeError): create_autospec(async_func, instance=True) - @unittest.skip('Broken test from https://bugs.python.org/issue37251') - def test_create_autospec_awaitable_class(self): - self.assertIsInstance(create_autospec(AwaitableClass), AsyncMock) - def test_create_autospec(self): spec = create_autospec(async_func_args) awaitable = spec(1, 2, c=3) @@ -236,7 +232,9 @@ async def main(): run(main()) self.assertTrue(iscoroutinefunction(spec)) + self.assertTrue(inspect.iscoroutinefunction(spec)) self.assertTrue(asyncio.iscoroutine(awaitable)) + self.assertTrue(inspect.iscoroutine(awaitable)) self.assertEqual(spec.await_count, 1) self.assertEqual(spec.await_args, call(1, 2, c=3)) self.assertEqual(spec.await_args_list, [call(1, 2, c=3)]) @@ -248,6 +246,25 @@ async def main(): with self.assertRaises(AssertionError): spec.assert_any_await(e=1) + def test_autospec_checks_signature(self): + spec = create_autospec(async_func_args) + # signature is not checked when called + awaitable = spec() + self.assertListEqual(spec.mock_calls, []) + + async def main(): + await awaitable + + # but it is checked when awaited + with self.assertRaises(TypeError): + run(main()) + + # _checksig_ raises before running or awaiting the mock + self.assertListEqual(spec.mock_calls, []) + self.assertEqual(spec.await_count, 0) + self.assertIsNone(spec.await_args) + self.assertEqual(spec.await_args_list, []) + spec.assert_not_awaited() def test_patch_with_autospec(self): @@ -257,7 +274,9 @@ async def test_async(): self.assertIsInstance(mock_method.mock, AsyncMock) self.assertTrue(iscoroutinefunction(mock_method)) + self.assertTrue(inspect.iscoroutinefunction(mock_method)) self.assertTrue(asyncio.iscoroutine(awaitable)) + self.assertTrue(inspect.iscoroutine(awaitable)) self.assertTrue(inspect.isawaitable(awaitable)) # Verify the default values during mock setup @@ -308,6 +327,21 @@ def test_spec_normal_methods_on_class_with_mock_seal(self): with self.assertRaises(AttributeError): mock.async_method + def test_spec_async_attributes_instance(self): + async_instance = AsyncClass() + async_instance.async_func_attr = async_func + async_instance.later_async_func_attr = normal_func + + mock_async_instance = Mock(spec_set=async_instance) + + async_instance.later_async_func_attr = async_func + + self.assertIsInstance(mock_async_instance.async_func_attr, AsyncMock) + # only the shape of the spec at the time of mock construction matters + self.assertNotIsInstance(mock_async_instance.later_async_func_attr, AsyncMock) + + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_spec_mock_type_kw(self): def inner_test(mock_type): async_mock = mock_type(spec=async_func) @@ -322,6 +356,8 @@ def inner_test(mock_type): with self.subTest(f"test spec kwarg with {mock_type}"): inner_test(mock_type) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_spec_mock_type_positional(self): def inner_test(mock_type): async_mock = mock_type(async_func) @@ -427,9 +463,10 @@ async def addition(self, var): pass self.assertEqual(output, 10) async def test_add_side_effect_exception(self): + class CustomError(Exception): pass async def addition(var): pass - mock = AsyncMock(addition, side_effect=Exception('err')) - with self.assertRaises(Exception): + mock = AsyncMock(addition, side_effect=CustomError('side-effect')) + with self.assertRaisesRegex(CustomError, 'side-effect'): await mock(5) async def test_add_side_effect_coroutine(self): @@ -699,8 +736,8 @@ def __aiter__(self): pass async def __anext__(self): pass - # TODO: RUSTPYTHON; async for - """ + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_aiter_set_return_value(self): mock_iter = AsyncMock(name="tester") mock_iter.__aiter__.return_value = [1, 2, 3] @@ -708,7 +745,6 @@ async def main(): return [i async for i in mock_iter] result = run(main()) self.assertEqual(result, [1, 2, 3]) - """ def test_mock_aiter_and_anext_asyncmock(self): def inner_test(mock_type): @@ -726,8 +762,9 @@ def inner_test(mock_type): with self.subTest(f"test aiter and anext corourtine with {mock_type}"): inner_test(mock_type) - # TODO: RUSTPYTHON; async for - """ + + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_mock_async_for(self): async def iterate(iterator): accumulator = [] @@ -761,7 +798,7 @@ def test_set_return_value_iter(mock_type): with self.subTest(f"set return_value iterator with {mock_type}"): test_set_return_value_iter(mock_type) - """ + class AsyncMockAssert(unittest.TestCase): def setUp(self): @@ -773,6 +810,8 @@ async def _runnable_test(self, *args, **kwargs): async def _await_coroutine(self, coroutine): return await coroutine + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_assert_called_but_not_awaited(self): mock = AsyncMock(AsyncClass) with assertNeverAwaited(self): @@ -813,6 +852,8 @@ def test_assert_called_and_awaited_at_same_time(self): self.mock.assert_called_once() self.mock.assert_awaited_once() + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_assert_called_twice_and_awaited_once(self): mock = AsyncMock(AsyncClass) coroutine = mock.async_method() @@ -827,6 +868,8 @@ def test_assert_called_twice_and_awaited_once(self): mock.async_method.assert_awaited() mock.async_method.assert_awaited_once() + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_assert_called_once_and_awaited_twice(self): mock = AsyncMock(AsyncClass) coroutine = mock.async_method() @@ -851,6 +894,8 @@ def test_assert_awaited_but_not_called(self): with self.assertRaises(AssertionError): self.mock.assert_called() + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_assert_has_calls_not_awaits(self): kalls = [call('foo')] with assertNeverAwaited(self): @@ -859,6 +904,8 @@ def test_assert_has_calls_not_awaits(self): with self.assertRaises(AssertionError): self.mock.assert_has_awaits(kalls) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_assert_has_mock_calls_on_async_mock_no_spec(self): with assertNeverAwaited(self): self.mock() @@ -872,6 +919,8 @@ def test_assert_has_mock_calls_on_async_mock_no_spec(self): mock_kalls = ([call(), call('foo'), call('baz')]) self.assertEqual(self.mock.mock_calls, mock_kalls) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_assert_has_mock_calls_on_async_mock_with_spec(self): a_class_mock = AsyncMock(AsyncClass) with assertNeverAwaited(self): @@ -887,6 +936,8 @@ def test_assert_has_mock_calls_on_async_mock_with_spec(self): self.assertEqual(a_class_mock.async_method.mock_calls, method_kalls) self.assertEqual(a_class_mock.mock_calls, mock_kalls) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_async_method_calls_recorded(self): with assertNeverAwaited(self): self.mock.something(3, fish=None) @@ -902,6 +953,8 @@ def test_async_method_calls_recorded(self): [("something", (6,), {'cake': sentinel.Cake})], "method calls not recorded correctly") + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_async_arg_lists(self): def assert_attrs(mock): names = ('call_args_list', 'method_calls', 'mock_calls') diff --git a/Lib/unittest/test/testmock/testcallable.py b/Lib/test/test_unittest/testmock/testcallable.py similarity index 98% rename from Lib/unittest/test/testmock/testcallable.py rename to Lib/test/test_unittest/testmock/testcallable.py index 5eadc00704..ca88511f63 100644 --- a/Lib/unittest/test/testmock/testcallable.py +++ b/Lib/test/test_unittest/testmock/testcallable.py @@ -3,7 +3,7 @@ # http://www.voidspace.org.uk/python/mock/ import unittest -from unittest.test.testmock.support import is_instance, X, SomeClass +from test.test_unittest.testmock.support import is_instance, X, SomeClass from unittest.mock import ( Mock, MagicMock, NonCallableMagicMock, diff --git a/Lib/unittest/test/testmock/testhelpers.py b/Lib/test/test_unittest/testmock/testhelpers.py similarity index 96% rename from Lib/unittest/test/testmock/testhelpers.py rename to Lib/test/test_unittest/testmock/testhelpers.py index 9e7ec5d62d..c83068beb1 100644 --- a/Lib/unittest/test/testmock/testhelpers.py +++ b/Lib/test/test_unittest/testmock/testhelpers.py @@ -43,6 +43,8 @@ def test_any_and_datetime(self): mock.assert_called_with(ANY, foo=ANY) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_any_mock_calls_comparison_order(self): mock = Mock() class Foo(object): @@ -952,6 +954,24 @@ def __getattr__(self, attribute): self.assertFalse(hasattr(autospec, '__name__')) + def test_autospec_signature_staticmethod(self): + class Foo: + @staticmethod + def static_method(a, b=10, *, c): pass + + mock = create_autospec(Foo.__dict__['static_method']) + self.assertEqual(inspect.signature(Foo.static_method), inspect.signature(mock)) + + + def test_autospec_signature_classmethod(self): + class Foo: + @classmethod + def class_method(cls, a, b=10, *, c): pass + + mock = create_autospec(Foo.__dict__['class_method']) + self.assertEqual(inspect.signature(Foo.class_method), inspect.signature(mock)) + + def test_spec_inspect_signature(self): def myfunc(x, y): pass @@ -1077,7 +1097,7 @@ def test_propertymock(self): p.stop() - def test_propertymock_returnvalue(self): + def test_propertymock_bare(self): m = MagicMock() p = PropertyMock() type(m).foo = p @@ -1088,6 +1108,35 @@ def test_propertymock_returnvalue(self): self.assertNotIsInstance(returned, PropertyMock) + def test_propertymock_returnvalue(self): + m = MagicMock() + p = PropertyMock(return_value=42) + type(m).foo = p + + returned = m.foo + p.assert_called_once_with() + self.assertEqual(returned, 42) + self.assertNotIsInstance(returned, PropertyMock) + + + def test_propertymock_side_effect(self): + m = MagicMock() + p = PropertyMock(side_effect=ValueError) + type(m).foo = p + + with self.assertRaises(ValueError): + m.foo + p.assert_called_once_with() + + + def test_propertymock_attach(self): + m = Mock() + p = PropertyMock() + type(m).foo = p + m.attach_mock(p, 'foo') + self.assertEqual(m.mock_calls, []) + + class TestCallablePredicate(unittest.TestCase): def test_type(self): diff --git a/Lib/unittest/test/testmock/testmagicmethods.py b/Lib/test/test_unittest/testmock/testmagicmethods.py similarity index 91% rename from Lib/unittest/test/testmock/testmagicmethods.py rename to Lib/test/test_unittest/testmock/testmagicmethods.py index a4feae7e9d..a8b52ce487 100644 --- a/Lib/unittest/test/testmock/testmagicmethods.py +++ b/Lib/test/test_unittest/testmock/testmagicmethods.py @@ -331,6 +331,45 @@ def test_magic_methods_fspath(self): self.assertEqual(os.fspath(mock), expected_path) mock.__fspath__.assert_called_once() + def test_magic_mock_does_not_reset_magic_returns(self): + # https://github.com/python/cpython/issues/123934 + for reset in (True, False): + with self.subTest(reset=reset): + mm = MagicMock() + self.assertIs(type(mm.__str__()), str) + mm.__str__.assert_called_once() + + self.assertIs(type(mm.__hash__()), int) + mm.__hash__.assert_called_once() + + for _ in range(3): + # Repeat reset several times to be sure: + mm.reset_mock(return_value=reset) + + self.assertIs(type(mm.__str__()), str) + mm.__str__.assert_called_once() + + self.assertIs(type(mm.__hash__()), int) + mm.__hash__.assert_called_once() + + def test_magic_mock_resets_manual_mocks(self): + mm = MagicMock() + mm.__iter__ = MagicMock(return_value=iter([1])) + mm.custom = MagicMock(return_value=2) + self.assertEqual(list(iter(mm)), [1]) + self.assertEqual(mm.custom(), 2) + + mm.reset_mock(return_value=True) + self.assertEqual(list(iter(mm)), []) + self.assertIsInstance(mm.custom(), MagicMock) + + def test_magic_mock_resets_manual_mocks_empty_iter(self): + mm = MagicMock() + mm.__iter__.return_value = [] + self.assertEqual(list(iter(mm)), []) + + mm.reset_mock(return_value=True) + self.assertEqual(list(iter(mm)), []) def test_magic_methods_and_spec(self): class Iterable(object): diff --git a/Lib/unittest/test/testmock/testmock.py b/Lib/test/test_unittest/testmock/testmock.py similarity index 91% rename from Lib/unittest/test/testmock/testmock.py rename to Lib/test/test_unittest/testmock/testmock.py index eaae22e854..e1b108f81e 100644 --- a/Lib/unittest/test/testmock/testmock.py +++ b/Lib/test/test_unittest/testmock/testmock.py @@ -5,7 +5,7 @@ from test.support import ALWAYS_EQ import unittest -from unittest.test.testmock.support import is_instance +from test.test_unittest.testmock.support import is_instance from unittest import mock from unittest.mock import ( call, DEFAULT, patch, sentinel, @@ -38,6 +38,17 @@ def cmeth(cls, a, b, c, d=None): pass def smeth(a, b, c, d=None): pass +class SomethingElse(object): + def __init__(self): + self._instance = None + + @property + def instance(self): + if not self._instance: + self._instance = 'object' + return self._instance + + class Typos(): autospect = None auto_spec = None @@ -104,6 +115,24 @@ def f(): pass with self.assertRaises(TypeError): mock() + def test_create_autospec_should_be_configurable_by_kwargs(self): + """If kwargs are given to configure mock, the function must configure + the parent mock during initialization.""" + mocked_result = 'mocked value' + class_mock = create_autospec(spec=Something, **{ + 'return_value.meth.side_effect': [ValueError, DEFAULT], + 'return_value.meth.return_value': mocked_result}) + with self.assertRaises(ValueError): + class_mock().meth(a=None, b=None, c=None) + self.assertEqual(class_mock().meth(a=None, b=None, c=None), mocked_result) + # Only the parent mock should be configurable because the user will + # pass kwargs with respect to the parent mock. + self.assertEqual(class_mock().return_value.meth.side_effect, None) + + def test_create_autospec_correctly_handles_name(self): + class X: ... + mock = create_autospec(X, spec_set=True, name="Y") + self.assertEqual(mock._mock_name, "Y") def test_repr(self): mock = Mock(name='foo') @@ -234,6 +263,73 @@ class B(object): with mock.patch('builtins.open', mock.mock_open()): mock.mock_open() # should still be valid with open() mocked + def test_create_autospec_wraps_class(self): + """Autospec a class with wraps & test if the call is passed to the + wrapped object.""" + result = "real result" + + class Result: + def get_result(self): + return result + class_mock = create_autospec(spec=Result, wraps=Result) + # Have to reassign the return_value to DEFAULT to return the real + # result (actual instance of "Result") when the mock is called. + class_mock.return_value = mock.DEFAULT + self.assertEqual(class_mock().get_result(), result) + # Autospec should also wrap child attributes of parent. + self.assertEqual(class_mock.get_result._mock_wraps, Result.get_result) + + def test_create_autospec_instance_wraps_class(self): + """Autospec a class instance with wraps & test if the call is passed + to the wrapped object.""" + result = "real result" + + class Result: + @staticmethod + def get_result(): + """This is a static method because when the mocked instance of + 'Result' will call this method, it won't be able to consume + 'self' argument.""" + return result + instance_mock = create_autospec(spec=Result, instance=True, wraps=Result) + # Have to reassign the return_value to DEFAULT to return the real + # result from "Result.get_result" when the mocked instance of "Result" + # calls "get_result". + instance_mock.get_result.return_value = mock.DEFAULT + self.assertEqual(instance_mock.get_result(), result) + # Autospec should also wrap child attributes of the instance. + self.assertEqual(instance_mock.get_result._mock_wraps, Result.get_result) + + def test_create_autospec_wraps_function_type(self): + """Autospec a function or a method with wraps & test if the call is + passed to the wrapped object.""" + result = "real result" + + class Result: + def get_result(self): + return result + func_mock = create_autospec(spec=Result.get_result, wraps=Result.get_result) + self.assertEqual(func_mock(Result()), result) + + def test_explicit_return_value_even_if_mock_wraps_object(self): + """If the mock has an explicit return_value set then calls are not + passed to the wrapped object and the return_value is returned instead. + """ + def my_func(): + return None + func_mock = create_autospec(spec=my_func, wraps=my_func) + return_value = "explicit return value" + func_mock.return_value = return_value + self.assertEqual(func_mock(), return_value) + + def test_explicit_parent(self): + parent = Mock() + mock1 = Mock(parent=parent, return_value=None) + mock1(1, 2, 3) + mock2 = Mock(parent=parent, return_value=None) + mock2(4, 5, 6) + + self.assertEqual(parent.mock_calls, [call(1, 2, 3), call(4, 5, 6)]) def test_reset_mock(self): parent = Mock() @@ -603,6 +699,14 @@ def test_wraps_calls(self): real = Mock() mock = Mock(wraps=real) + # If "Mock" wraps an object, just accessing its + # "return_value" ("NonCallableMock.__get_return_value") should not + # trigger its descriptor ("NonCallableMock.__set_return_value") so + # the default "return_value" should always be "sentinel.DEFAULT". + self.assertEqual(mock.return_value, DEFAULT) + # It will not be "sentinel.DEFAULT" if the mock is not wrapping any + # object. + self.assertNotEqual(real.return_value, DEFAULT) self.assertEqual(mock(), real()) real.reset_mock() @@ -1039,7 +1143,7 @@ def test_assert_called_with_failure_message(self): actual = 'not called.' expected = "mock(1, '2', 3, bar='foo')" - message = 'expected call not found.\nExpected: %s\nActual: %s' + message = 'expected call not found.\nExpected: %s\n Actual: %s' self.assertRaisesWithMsg( AssertionError, message % (expected, actual), mock.assert_called_with, 1, '2', 3, bar='foo' @@ -1054,7 +1158,7 @@ def test_assert_called_with_failure_message(self): for meth in asserters: actual = "foo(1, '2', 3, foo='foo')" expected = "foo(1, '2', 3, bar='foo')" - message = 'expected call not found.\nExpected: %s\nActual: %s' + message = 'expected call not found.\nExpected: %s\n Actual: %s' self.assertRaisesWithMsg( AssertionError, message % (expected, actual), meth, 1, '2', 3, bar='foo' @@ -1064,7 +1168,7 @@ def test_assert_called_with_failure_message(self): for meth in asserters: actual = "foo(1, '2', 3, foo='foo')" expected = "foo(bar='foo')" - message = 'expected call not found.\nExpected: %s\nActual: %s' + message = 'expected call not found.\nExpected: %s\n Actual: %s' self.assertRaisesWithMsg( AssertionError, message % (expected, actual), meth, bar='foo' @@ -1074,7 +1178,7 @@ def test_assert_called_with_failure_message(self): for meth in asserters: actual = "foo(1, '2', 3, foo='foo')" expected = "foo(1, 2, 3)" - message = 'expected call not found.\nExpected: %s\nActual: %s' + message = 'expected call not found.\nExpected: %s\n Actual: %s' self.assertRaisesWithMsg( AssertionError, message % (expected, actual), meth, 1, 2, 3 @@ -1084,7 +1188,7 @@ def test_assert_called_with_failure_message(self): for meth in asserters: actual = "foo(1, '2', 3, foo='foo')" expected = "foo()" - message = 'expected call not found.\nExpected: %s\nActual: %s' + message = 'expected call not found.\nExpected: %s\n Actual: %s' self.assertRaisesWithMsg( AssertionError, message % (expected, actual), meth ) @@ -1528,25 +1632,33 @@ def f(x=None): pass mock = Mock(spec=f) mock(1) - with self.assertRaisesRegex( - AssertionError, - '^{}$'.format( - re.escape('Calls not found.\n' - 'Expected: [call()]\n' - 'Actual: [call(1)]'))) as cm: + with self.assertRaises(AssertionError) as cm: mock.assert_has_calls([call()]) + self.assertEqual(str(cm.exception), + 'Calls not found.\n' + 'Expected: [call()]\n' + ' Actual: [call(1)]' + ) self.assertIsNone(cm.exception.__cause__) + uncalled_mock = Mock() + with self.assertRaises(AssertionError) as cm: + uncalled_mock.assert_has_calls([call()]) + self.assertEqual(str(cm.exception), + 'Calls not found.\n' + 'Expected: [call()]\n' + ' Actual: []' + ) + self.assertIsNone(cm.exception.__cause__) - with self.assertRaisesRegex( - AssertionError, - '^{}$'.format( - re.escape( - 'Error processing expected calls.\n' - "Errors: [None, TypeError('too many positional arguments')]\n" - "Expected: [call(), call(1, 2)]\n" - 'Actual: [call(1)]'))) as cm: + with self.assertRaises(AssertionError) as cm: mock.assert_has_calls([call(), call(1, 2)]) + self.assertEqual(str(cm.exception), + 'Error processing expected calls.\n' + "Errors: [None, TypeError('too many positional arguments')]\n" + 'Expected: [call(), call(1, 2)]\n' + ' Actual: [call(1)]' + ) self.assertIsInstance(cm.exception.__cause__, TypeError) def test_assert_any_call(self): @@ -1645,21 +1757,41 @@ def test_mock_unsafe(self): m.aseert_foo_call() with self.assertRaisesRegex(AttributeError, msg): m.assrt_foo_call() + with self.assertRaisesRegex(AttributeError, msg): + m.called_once_with() + with self.assertRaisesRegex(AttributeError, msg): + m.called_once() + with self.assertRaisesRegex(AttributeError, msg): + m.has_calls() + + class Foo(object): + def called_once(self): pass + + def has_calls(self): pass + + m = Mock(spec=Foo) + m.called_once() + m.has_calls() + + m.called_once.assert_called_once() + m.has_calls.assert_called_once() + m = Mock(unsafe=True) m.assert_foo_call() m.assret_foo_call() m.asert_foo_call() m.aseert_foo_call() m.assrt_foo_call() + m.called_once() + m.called_once_with() + m.has_calls() # gh-100739 def test_mock_safe_with_spec(self): class Foo(object): - def assert_bar(self): - pass + def assert_bar(self): pass - def assertSome(self): - pass + def assertSome(self): pass m = Mock(spec=Foo) m.assert_bar() @@ -2231,7 +2363,7 @@ def test_misspelled_arguments(self): class Foo(): one = 'one' # patch, patch.object and create_autospec need to check for misspelled - # arguments explicitly and throw a RuntimError if found. + # arguments explicitly and throw a RuntimeError if found. with self.assertRaises(RuntimeError): with patch(f'{__name__}.Something.meth', autospect=True): pass with self.assertRaises(RuntimeError): @@ -2273,6 +2405,26 @@ class Foo(): f'{__name__}.Typos', autospect=True, set_spec=True, auto_spec=True): pass + def test_property_not_called_with_spec_mock(self): + obj = SomethingElse() + self.assertIsNone(obj._instance, msg='before mock') + mock = Mock(spec=obj) + self.assertIsNone(obj._instance, msg='after mock') + self.assertEqual('object', obj.instance) + + def test_decorated_async_methods_with_spec_mock(self): + class Foo(): + @classmethod + async def class_method(cls): + pass + @staticmethod + async def static_method(): + pass + async def method(self): + pass + mock = Mock(spec=Foo) + for m in (mock.method, mock.class_method, mock.static_method): + self.assertIsInstance(m, AsyncMock) if __name__ == '__main__': unittest.main() diff --git a/Lib/unittest/test/testmock/testpatch.py b/Lib/test/test_unittest/testmock/testpatch.py similarity index 89% rename from Lib/unittest/test/testmock/testpatch.py rename to Lib/test/test_unittest/testmock/testpatch.py index 8ab63a1317..62c6221f77 100644 --- a/Lib/unittest/test/testmock/testpatch.py +++ b/Lib/test/test_unittest/testmock/testpatch.py @@ -7,9 +7,11 @@ from collections import OrderedDict import unittest -from unittest.test.testmock import support -from unittest.test.testmock.support import SomeClass, is_instance +import test +from test.test_unittest.testmock import support +from test.test_unittest.testmock.support import SomeClass, is_instance +from test.support.import_helper import DirsOnSysPath from test.test_importlib.util import uncache from unittest.mock import ( NonCallableMock, CallableMixin, sentinel, @@ -669,7 +671,7 @@ def test_patch_dict_decorator_resolution(self): # the new dictionary during function call original = support.target.copy() - @patch.dict('unittest.test.testmock.support.target', {'bar': 'BAR'}) + @patch.dict('test.test_unittest.testmock.support.target', {'bar': 'BAR'}) def test(): self.assertEqual(support.target, {'foo': 'BAZ', 'bar': 'BAR'}) @@ -743,6 +745,54 @@ def test_stop_idempotent(self): self.assertIsNone(patcher.stop()) + def test_exit_idempotent(self): + patcher = patch(foo_name, 'bar', 3) + with patcher: + patcher.stop() + + + def test_second_start_failure(self): + patcher = patch(foo_name, 'bar', 3) + patcher.start() + try: + self.assertRaises(RuntimeError, patcher.start) + finally: + patcher.stop() + + + def test_second_enter_failure(self): + patcher = patch(foo_name, 'bar', 3) + with patcher: + self.assertRaises(RuntimeError, patcher.start) + + + def test_second_start_after_stop(self): + patcher = patch(foo_name, 'bar', 3) + patcher.start() + patcher.stop() + patcher.start() + patcher.stop() + + + def test_property_setters(self): + mock_object = Mock() + mock_bar = mock_object.bar + patcher = patch.object(mock_object, 'bar', 'x') + with patcher: + self.assertEqual(patcher.is_local, False) + self.assertIs(patcher.target, mock_object) + self.assertEqual(patcher.temp_original, mock_bar) + patcher.is_local = True + patcher.target = mock_bar + patcher.temp_original = mock_object + self.assertEqual(patcher.is_local, True) + self.assertIs(patcher.target, mock_bar) + self.assertEqual(patcher.temp_original, mock_object) + # if changes are left intact, they may lead to disruption as shown below (it might be what someone needs though) + self.assertEqual(mock_bar.bar, mock_object) + self.assertEqual(mock_object.bar, 'x') + + def test_patchobject_start_stop(self): original = something patcher = patch.object(PTModule, 'something', 'foo') @@ -996,6 +1046,36 @@ def test_autospec_classmethod(self): method.assert_called_once_with() + def test_autospec_staticmethod_signature(self): + # Patched methods which are decorated with @staticmethod should have the same signature + class Foo: + @staticmethod + def static_method(a, b=10, *, c): pass + + Foo.static_method(1, 2, c=3) + + with patch.object(Foo, 'static_method', autospec=True) as method: + method(1, 2, c=3) + self.assertRaises(TypeError, method) + self.assertRaises(TypeError, method, 1) + self.assertRaises(TypeError, method, 1, 2, 3, c=4) + + + def test_autospec_classmethod_signature(self): + # Patched methods which are decorated with @classmethod should have the same signature + class Foo: + @classmethod + def class_method(cls, a, b=10, *, c): pass + + Foo.class_method(1, 2, c=3) + + with patch.object(Foo, 'class_method', autospec=True) as method: + method(1, 2, c=3) + self.assertRaises(TypeError, method) + self.assertRaises(TypeError, method, 1) + self.assertRaises(TypeError, method, 1, 2, 3, c=4) + + def test_autospec_with_new(self): patcher = patch('%s.function' % __name__, new=3, autospec=True) self.assertRaises(TypeError, patcher.start) @@ -1066,7 +1146,7 @@ def test_new_callable_patch(self): self.assertIsNot(m1, m2) for mock in m1, m2: - self.assertNotCallable(m1) + self.assertNotCallable(mock) def test_new_callable_patch_object(self): @@ -1079,7 +1159,7 @@ def test_new_callable_patch_object(self): self.assertIsNot(m1, m2) for mock in m1, m2: - self.assertNotCallable(m1) + self.assertNotCallable(mock) def test_new_callable_keyword_arguments(self): @@ -1614,7 +1694,7 @@ def test_patch_with_spec_mock_repr(self): def test_patch_nested_autospec_repr(self): - with patch('unittest.test.testmock.support', autospec=True) as m: + with patch('test.test_unittest.testmock.support', autospec=True) as m: self.assertIn(" name='support.SomeClass.wibble()'", repr(m.SomeClass.wibble())) self.assertIn(" name='support.SomeClass().wibble()'", @@ -1698,6 +1778,75 @@ def test(mock): 'exception traceback not propagated') + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_name_resolution_import_rebinding(self): + # Currently mock.patch uses pkgutil.resolve_name(), but repeat + # similar tests just for the case. + # The same data is also used for testing import in test_import and + # pkgutil.resolve_name() in test_pkgutil. + path = os.path.join(os.path.dirname(test.__file__), 'test_import', 'data') + def check(name): + p = patch(name) + p.start() + p.stop() + def check_error(name): + p = patch(name) + self.assertRaises(AttributeError, p.start) + with uncache('package3', 'package3.submodule'), DirsOnSysPath(path): + check('package3.submodule.A.attr') + check_error('package3.submodule.B.attr') + with uncache('package3', 'package3.submodule'), DirsOnSysPath(path): + check('package3.submodule:A.attr') + check_error('package3.submodule:B.attr') + with uncache('package3', 'package3.submodule'), DirsOnSysPath(path): + check('package3:submodule.B.attr') + check_error('package3:submodule.A.attr') + check('package3.submodule.A.attr') + check_error('package3.submodule.B.attr') + check('package3:submodule.B.attr') + check_error('package3:submodule.A.attr') + with uncache('package3', 'package3.submodule'), DirsOnSysPath(path): + check('package3:submodule.B.attr') + check_error('package3:submodule.A.attr') + check('package3.submodule:A.attr') + check_error('package3.submodule:B.attr') + check('package3:submodule.B.attr') + check_error('package3:submodule.A.attr') + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_name_resolution_import_rebinding2(self): + path = os.path.join(os.path.dirname(test.__file__), 'test_import', 'data') + def check(name): + p = patch(name) + p.start() + p.stop() + def check_error(name): + p = patch(name) + self.assertRaises(AttributeError, p.start) + with uncache('package4', 'package4.submodule'), DirsOnSysPath(path): + check('package4.submodule.A.attr') + check_error('package4.submodule.B.attr') + with uncache('package4', 'package4.submodule'), DirsOnSysPath(path): + check('package4.submodule:A.attr') + check_error('package4.submodule:B.attr') + with uncache('package4', 'package4.submodule'), DirsOnSysPath(path): + check('package4:submodule.B.attr') + check_error('package4:submodule.A.attr') + check('package4.submodule.A.attr') + check_error('package4.submodule.B.attr') + check('package4:submodule.A.attr') + check_error('package4:submodule.B.attr') + with uncache('package4', 'package4.submodule'), DirsOnSysPath(path): + check('package4:submodule.B.attr') + check_error('package4:submodule.A.attr') + check('package4.submodule:A.attr') + check_error('package4.submodule:B.attr') + check('package4:submodule.A.attr') + check_error('package4:submodule.B.attr') + + def test_create_and_specs(self): for kwarg in ('spec', 'spec_set', 'autospec'): p = patch('%s.doesnotexist' % __name__, create=True, @@ -1867,6 +2016,8 @@ def test_patch_and_patch_dict_stopall(self): self.assertEqual(dic2, origdic2) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_special_attrs(self): def foo(x=0): """TEST""" @@ -1882,7 +2033,7 @@ def foo(x=0): with patch.object(foo, '__module__', "testpatch2"): self.assertEqual(foo.__module__, "testpatch2") - self.assertEqual(foo.__module__, 'unittest.test.testmock.testpatch') + self.assertEqual(foo.__module__, __name__) with patch.object(foo, '__annotations__', dict([('s', 1, )])): self.assertEqual(foo.__annotations__, dict([('s', 1, )])) @@ -1917,16 +2068,16 @@ def test_dotted_but_module_not_loaded(self): # This exercises the AttributeError branch of _dot_lookup. # make sure it's there - import unittest.test.testmock.support + import test.test_unittest.testmock.support # now make sure it's not: with patch.dict('sys.modules'): - del sys.modules['unittest.test.testmock.support'] - del sys.modules['unittest.test.testmock'] - del sys.modules['unittest.test'] - del sys.modules['unittest'] + del sys.modules['test.test_unittest.testmock.support'] + del sys.modules['test.test_unittest.testmock'] + del sys.modules['test.test_unittest'] + del sys.modules['test'] # now make sure we can patch based on a dotted path: - @patch('unittest.test.testmock.support.X') + @patch('test.test_unittest.testmock.support.X') def test(mock): pass test() @@ -1943,11 +2094,18 @@ class Foo: def test_cant_set_kwargs_when_passing_a_mock(self): - @patch('unittest.test.testmock.support.X', new=object(), x=1) + @patch('test.test_unittest.testmock.support.X', new=object(), x=1) def test(): pass with self.assertRaises(TypeError): test() + def test_patch_proxy_object(self): + @patch("test.test_unittest.testmock.support.g", new_callable=MagicMock()) + def test(_): + pass + + test() + if __name__ == '__main__': unittest.main() diff --git a/Lib/unittest/test/testmock/testsealable.py b/Lib/test/test_unittest/testmock/testsealable.py similarity index 96% rename from Lib/unittest/test/testmock/testsealable.py rename to Lib/test/test_unittest/testmock/testsealable.py index daba2b49b4..8bf98cfa56 100644 --- a/Lib/unittest/test/testmock/testsealable.py +++ b/Lib/test/test_unittest/testmock/testsealable.py @@ -175,15 +175,12 @@ def test_seal_with_autospec(self): # https://bugs.python.org/issue45156 class Foo: foo = 0 - def bar1(self): - return 1 - def bar2(self): - return 2 + def bar1(self): pass + def bar2(self): pass class Baz: baz = 3 - def ban(self): - return 4 + def ban(self): pass for spec_set in (True, False): with self.subTest(spec_set=spec_set): @@ -200,6 +197,9 @@ def ban(self): self.assertIsInstance(foo.Baz.baz, mock.NonCallableMagicMock) self.assertIsInstance(foo.Baz.ban, mock.MagicMock) + # see gh-91803 + self.assertIsInstance(foo.bar2(), mock.MagicMock) + self.assertEqual(foo.bar1(), 'a') foo.bar1.return_value = 'new_a' self.assertEqual(foo.bar1(), 'new_a') @@ -212,7 +212,7 @@ def ban(self): with self.assertRaises(AttributeError): foo.bar = 1 with self.assertRaises(AttributeError): - foo.bar2() + foo.bar2().x foo.bar2.return_value = 'bar2' self.assertEqual(foo.bar2(), 'bar2') diff --git a/Lib/unittest/test/testmock/testsentinel.py b/Lib/test/test_unittest/testmock/testsentinel.py similarity index 100% rename from Lib/unittest/test/testmock/testsentinel.py rename to Lib/test/test_unittest/testmock/testsentinel.py diff --git a/Lib/test/test_unittest/testmock/testthreadingmock.py b/Lib/test/test_unittest/testmock/testthreadingmock.py new file mode 100644 index 0000000000..a02b532ed4 --- /dev/null +++ b/Lib/test/test_unittest/testmock/testthreadingmock.py @@ -0,0 +1,201 @@ +import time +import unittest +import concurrent.futures + +from test.support import threading_helper +from unittest.mock import patch, ThreadingMock + + +threading_helper.requires_working_threading(module=True) + +VERY_SHORT_TIMEOUT = 0.1 + + +class Something: + def method_1(self): + pass # pragma: no cover + + def method_2(self): + pass # pragma: no cover + + +class TestThreadingMock(unittest.TestCase): + def _call_after_delay(self, func, /, *args, **kwargs): + time.sleep(kwargs.pop("delay")) + func(*args, **kwargs) + + def setUp(self): + self._executor = concurrent.futures.ThreadPoolExecutor(max_workers=5) + + def tearDown(self): + self._executor.shutdown() + + def run_async(self, func, /, *args, delay=0, **kwargs): + self._executor.submit( + self._call_after_delay, func, *args, **kwargs, delay=delay + ) + + def _make_mock(self, *args, **kwargs): + return ThreadingMock(*args, **kwargs) + + def test_spec(self): + waitable_mock = self._make_mock(spec=Something) + + with patch(f"{__name__}.Something", waitable_mock) as m: + something = m() + + self.assertIsInstance(something.method_1, ThreadingMock) + self.assertIsInstance(something.method_1().method_2(), ThreadingMock) + + with self.assertRaises(AttributeError): + m.test + + def test_side_effect(self): + waitable_mock = self._make_mock() + + with patch(f"{__name__}.Something", waitable_mock): + something = Something() + something.method_1.side_effect = [1] + + self.assertEqual(something.method_1(), 1) + + def test_instance_check(self): + waitable_mock = self._make_mock() + + with patch(f"{__name__}.Something", waitable_mock): + something = Something() + + self.assertIsInstance(something.method_1, ThreadingMock) + self.assertIsInstance(something.method_1().method_2(), ThreadingMock) + + def test_dynamic_child_mocks_are_threading_mocks(self): + waitable_mock = self._make_mock() + self.assertIsInstance(waitable_mock.child, ThreadingMock) + + def test_dynamic_child_mocks_inherit_timeout(self): + mock1 = self._make_mock() + self.assertIs(mock1._mock_wait_timeout, None) + mock2 = self._make_mock(timeout=2) + self.assertEqual(mock2._mock_wait_timeout, 2) + mock3 = self._make_mock(timeout=3) + self.assertEqual(mock3._mock_wait_timeout, 3) + + self.assertIs(mock1.child._mock_wait_timeout, None) + self.assertEqual(mock2.child._mock_wait_timeout, 2) + self.assertEqual(mock3.child._mock_wait_timeout, 3) + + self.assertEqual(mock2.really().__mul__().complex._mock_wait_timeout, 2) + + def test_no_name_clash(self): + waitable_mock = self._make_mock() + waitable_mock._event = "myevent" + waitable_mock.event = "myevent" + waitable_mock.timeout = "mytimeout" + waitable_mock("works") + waitable_mock.wait_until_called() + waitable_mock.wait_until_any_call_with("works") + + def test_patch(self): + waitable_mock = self._make_mock(spec=Something) + + with patch(f"{__name__}.Something", waitable_mock): + something = Something() + something.method_1() + something.method_1.wait_until_called() + + def test_wait_already_called_success(self): + waitable_mock = self._make_mock(spec=Something) + waitable_mock.method_1() + waitable_mock.method_1.wait_until_called() + waitable_mock.method_1.wait_until_any_call_with() + waitable_mock.method_1.assert_called() + + def test_wait_until_called_success(self): + waitable_mock = self._make_mock(spec=Something) + self.run_async(waitable_mock.method_1, delay=VERY_SHORT_TIMEOUT) + waitable_mock.method_1.wait_until_called() + + def test_wait_until_called_method_timeout(self): + waitable_mock = self._make_mock(spec=Something) + with self.assertRaises(AssertionError): + waitable_mock.method_1.wait_until_called(timeout=VERY_SHORT_TIMEOUT) + + def test_wait_until_called_instance_timeout(self): + waitable_mock = self._make_mock(spec=Something, timeout=VERY_SHORT_TIMEOUT) + with self.assertRaises(AssertionError): + waitable_mock.method_1.wait_until_called() + + def test_wait_until_called_global_timeout(self): + with patch.object(ThreadingMock, "DEFAULT_TIMEOUT"): + ThreadingMock.DEFAULT_TIMEOUT = VERY_SHORT_TIMEOUT + waitable_mock = self._make_mock(spec=Something) + with self.assertRaises(AssertionError): + waitable_mock.method_1.wait_until_called() + + def test_wait_until_any_call_with_success(self): + waitable_mock = self._make_mock() + self.run_async(waitable_mock, delay=VERY_SHORT_TIMEOUT) + waitable_mock.wait_until_any_call_with() + + def test_wait_until_any_call_with_instance_timeout(self): + waitable_mock = self._make_mock(timeout=VERY_SHORT_TIMEOUT) + with self.assertRaises(AssertionError): + waitable_mock.wait_until_any_call_with() + + def test_wait_until_any_call_global_timeout(self): + with patch.object(ThreadingMock, "DEFAULT_TIMEOUT"): + ThreadingMock.DEFAULT_TIMEOUT = VERY_SHORT_TIMEOUT + waitable_mock = self._make_mock() + with self.assertRaises(AssertionError): + waitable_mock.wait_until_any_call_with() + + def test_wait_until_any_call_positional(self): + waitable_mock = self._make_mock(timeout=VERY_SHORT_TIMEOUT) + waitable_mock.method_1(1, 2, 3) + waitable_mock.method_1.wait_until_any_call_with(1, 2, 3) + with self.assertRaises(AssertionError): + waitable_mock.method_1.wait_until_any_call_with(2, 3, 1) + with self.assertRaises(AssertionError): + waitable_mock.method_1.wait_until_any_call_with() + + def test_wait_until_any_call_kw(self): + waitable_mock = self._make_mock(timeout=VERY_SHORT_TIMEOUT) + waitable_mock.method_1(a=1, b=2) + waitable_mock.method_1.wait_until_any_call_with(a=1, b=2) + with self.assertRaises(AssertionError): + waitable_mock.method_1.wait_until_any_call_with(a=2, b=1) + with self.assertRaises(AssertionError): + waitable_mock.method_1.wait_until_any_call_with() + + def test_magic_methods_success(self): + waitable_mock = self._make_mock() + str(waitable_mock) + waitable_mock.__str__.wait_until_called() + waitable_mock.__str__.assert_called() + + def test_reset_mock_resets_wait(self): + m = self._make_mock(timeout=VERY_SHORT_TIMEOUT) + + with self.assertRaises(AssertionError): + m.wait_until_called() + with self.assertRaises(AssertionError): + m.wait_until_any_call_with() + m() + m.wait_until_called() + m.wait_until_any_call_with() + m.assert_called_once() + + m.reset_mock() + + with self.assertRaises(AssertionError): + m.wait_until_called() + with self.assertRaises(AssertionError): + m.wait_until_any_call_with() + m() + m.wait_until_called() + m.wait_until_any_call_with() + m.assert_called_once() + + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/unittest/test/testmock/testwith.py b/Lib/test/test_unittest/testmock/testwith.py similarity index 97% rename from Lib/unittest/test/testmock/testwith.py rename to Lib/test/test_unittest/testmock/testwith.py index c74d49a63c..56cb16394f 100644 --- a/Lib/unittest/test/testmock/testwith.py +++ b/Lib/test/test_unittest/testmock/testwith.py @@ -1,7 +1,7 @@ import unittest from warnings import catch_warnings -from unittest.test.testmock.support import is_instance +from test.test_unittest.testmock.support import is_instance from unittest.mock import MagicMock, Mock, patch, sentinel, mock_open, call @@ -158,7 +158,7 @@ def test_mock_open_context_manager(self): f.read() expected_calls = [call('foo'), call().__enter__(), call().read(), - call().__exit__(None, None, None)] + call().__exit__(None, None, None), call().close()] self.assertEqual(mock.mock_calls, expected_calls) self.assertIs(f, handle) @@ -172,9 +172,9 @@ def test_mock_open_context_manager_multiple_times(self): expected_calls = [ call('foo'), call().__enter__(), call().read(), - call().__exit__(None, None, None), + call().__exit__(None, None, None), call().close(), call('bar'), call().__enter__(), call().read(), - call().__exit__(None, None, None)] + call().__exit__(None, None, None), call().close()] self.assertEqual(mock.mock_calls, expected_calls) def test_explicit_mock(self): diff --git a/Lib/test/test_univnewlines.py b/Lib/test/test_univnewlines.py index b905491878..ed2e0970ba 100644 --- a/Lib/test/test_univnewlines.py +++ b/Lib/test/test_univnewlines.py @@ -4,7 +4,6 @@ import unittest import os import sys -from test import support from test.support import os_helper diff --git a/Lib/test/test_urllib2.py b/Lib/test/test_urllib2.py index 5a02b5db8e..399c94213a 100644 --- a/Lib/test/test_urllib2.py +++ b/Lib/test/test_urllib2.py @@ -641,8 +641,6 @@ def test_raise(self): self.assertRaises(urllib.error.URLError, o.open, req) self.assertEqual(o.calls, [(handlers[0], "http_open", (req,), {})]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_http_error(self): # XXX http_error_default # http errors are a special case @@ -666,8 +664,6 @@ def test_http_error(self): self.assertEqual((handler, method_name), got[:2]) self.assertEqual(args, got[2]) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_processors(self): # *_request / *_response methods get called appropriately o = OpenerDirector() @@ -874,8 +870,6 @@ def test_file(self): self.assertEqual(req.type, "ftp") self.assertEqual(req.type == "ftp", ftp) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_http(self): h = urllib.request.AbstractHTTPHandler() @@ -1136,8 +1130,6 @@ def test_fixpath_in_weirdurls(self): self.assertEqual(newreq.host, 'www.python.org') self.assertEqual(newreq.selector, '') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_errors(self): h = urllib.request.HTTPErrorProcessor() o = h.parent = MockOpener() @@ -1163,8 +1155,6 @@ def test_errors(self): self.assertEqual(o.proto, "http") # o.error called self.assertEqual(o.args, (req, r, 502, "Bad gateway", {})) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_cookies(self): cj = MockCookieJar() h = urllib.request.HTTPCookieProcessor(cj) @@ -1291,8 +1281,6 @@ def test_relative_redirect(self): MockHeaders({"location": valid_url})) self.assertEqual(o.req.get_full_url(), valid_url) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_cookie_redirect(self): # cookies shouldn't leak into redirected requests from http.cookiejar import CookieJar @@ -1308,8 +1296,6 @@ def test_cookie_redirect(self): o.open("http://www.example.com/") self.assertFalse(hh.req.has_header("Cookie")) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_redirect_fragment(self): redirected_url = 'http://www.example.com/index.html#OK\r\n\r\n' hh = MockHTTPHandler(302, 'Location: ' + redirected_url) @@ -1374,8 +1360,6 @@ def http_open(self, req): request = handler.last_buf self.assertTrue(request.startswith(expected), repr(request)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_proxy(self): u = "proxy.example.com:3128" for d in dict(http=u), dict(HTTP=u): @@ -1420,8 +1404,6 @@ def test_proxy_no_proxy_all(self): self.assertEqual(req.host, "www.python.org") del os.environ['no_proxy'] - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_proxy_https(self): o = OpenerDirector() ph = urllib.request.ProxyHandler(dict(https="proxy.example.com:3128")) @@ -1509,8 +1491,6 @@ def check_basic_auth(self, headers, realm): "http://acme.example.com/protected", "http://acme.example.com/protected") - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_basic_auth(self): realm = "realm2@example.com" realm2 = "realm2@example.com" @@ -1556,8 +1536,6 @@ def test_basic_auth(self): for challenge in challenges] self.check_basic_auth(headers, realm) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_proxy_basic_auth(self): opener = OpenerDirector() ph = urllib.request.ProxyHandler(dict(http="proxy.example.com:3128")) @@ -1575,8 +1553,6 @@ def test_proxy_basic_auth(self): "proxy.example.com:3128", ) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_basic_and_digest_auth_handlers(self): # HTTPDigestAuthHandler raised an exception if it couldn't handle a 40* # response (http://python.org/sf/1479302), where it should instead @@ -1684,8 +1660,6 @@ def _test_basic_auth(self, opener, auth_handler, auth_header, self.assertEqual(len(http_handler.requests), 1) self.assertFalse(http_handler.requests[0].has_header(auth_header)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_basic_prior_auth_auto_send(self): # Assume already authenticated if is_authenticated=True # for APIs like Github that don't return 401 @@ -1713,8 +1687,6 @@ def test_basic_prior_auth_auto_send(self): # expect request to be sent with auth header self.assertTrue(http_handler.has_auth_header) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_basic_prior_auth_send_after_first_success(self): # Auto send auth header after authentication is successful once diff --git a/Lib/test/test_urllib2_localnet.py b/Lib/test/test_urllib2_localnet.py index 3d5ca2faab..2c54ef85b4 100644 --- a/Lib/test/test_urllib2_localnet.py +++ b/Lib/test/test_urllib2_localnet.py @@ -19,9 +19,9 @@ here = os.path.dirname(__file__) # Self-signed cert file for 'localhost' -CERT_localhost = os.path.join(here, 'keycert.pem') +CERT_localhost = os.path.join(here, 'certdata', 'keycert.pem') # Self-signed cert file for 'fakehostname' -CERT_fakehostname = os.path.join(here, 'keycert2.pem') +CERT_fakehostname = os.path.join(here, 'certdata', 'keycert2.pem') # Loopback http server infrastructure @@ -568,8 +568,6 @@ def test_200_with_parameters(self): self.assertEqual(data, expected_response) self.assertEqual(handler.requests, ["/bizarre", b"get=with_feeling"]) - # TODO: RUSTPYTHON - @unittest.expectedFailure @unittest.skipIf(os.name == "nt", "TODO: RUSTPYTHON, ValueError: illegal environment variable name") def test_https(self): handler = self.start_https_server() @@ -577,8 +575,6 @@ def test_https(self): data = self.urlopen("https://localhost:%s/bizarre" % handler.port, context=context) self.assertEqual(data, b"we care a bit") - # TODO: RUSTPYTHON - @unittest.expectedFailure @unittest.skipIf(os.name == "nt", "TODO: RUSTPYTHON, ValueError: illegal environment variable name") def test_https_with_cafile(self): handler = self.start_https_server(certfile=CERT_localhost) @@ -606,8 +602,6 @@ def test_https_with_cadefault(self): self.urlopen("https://localhost:%s/bizarre" % handler.port, cadefault=True) - # TODO: RUSTPYTHON - @unittest.expectedFailure @unittest.skipIf(os.name == "nt", "TODO: RUSTPYTHON, ValueError: illegal environment variable name") def test_https_sni(self): if ssl is None: diff --git a/Lib/test/test_urllib2net.py b/Lib/test/test_urllib2net.py index a7e7c9f0b9..41f170a6ad 100644 --- a/Lib/test/test_urllib2net.py +++ b/Lib/test/test_urllib2net.py @@ -1,10 +1,14 @@ +import contextlib import errno +import sysconfig import unittest +from unittest import mock from test import support from test.support import os_helper from test.support import socket_helper from test.support import ResourceDenied from test.test_urllib2 import sanepathname2url +from test.support.warnings_helper import check_no_resource_warning import os import socket @@ -29,13 +33,6 @@ def wrapped(*args, **kwargs): return _retry_thrice(func, exc, *args, **kwargs) return wrapped -# bpo-35411: FTP tests of test_urllib2net randomly fail -# with "425 Security: Bad IP connecting" on Travis CI -skip_ftp_test_on_travis = unittest.skipIf('TRAVIS' in os.environ, - 'bpo-35411: skip FTP test ' - 'on Travis CI') - - # Connecting to remote hosts is flaky. Make it more robust by retrying # the connection several times. _urlopen_with_retry = _wrap_with_retry_thrice(urllib.request.urlopen, @@ -140,15 +137,54 @@ def setUp(self): # XXX The rest of these tests aren't very good -- they don't check much. # They do sometimes catch some major disasters, though. - @skip_ftp_test_on_travis + @support.requires_resource('walltime') def test_ftp(self): + # Testing the same URL twice exercises the caching in CacheFTPHandler urls = [ + 'ftp://www.pythontest.net/README', 'ftp://www.pythontest.net/README', ('ftp://www.pythontest.net/non-existent-file', None, urllib.error.URLError), ] self._test_urls(urls, self._extra_handlers()) + @support.requires_resource('walltime') + @unittest.skipIf(sysconfig.get_platform() == 'linux-ppc64le', + 'leaks on PPC64LE (gh-140691)') + def test_ftp_no_leak(self): + # gh-140691: When the data connection (but not control connection) + # cannot be made established, we shouldn't leave an open socket object. + + class MockError(OSError): + pass + + orig_create_connection = socket.create_connection + def patched_create_connection(address, *args, **kwargs): + """Simulate REJECTing connections to ports other than 21""" + host, port = address + if port != 21: + raise MockError() + return orig_create_connection(address, *args, **kwargs) + + url = 'ftp://www.pythontest.net/README' + entry = url, None, urllib.error.URLError + no_cache_handlers = [urllib.request.FTPHandler()] + cache_handlers = self._extra_handlers() + with mock.patch('socket.create_connection', patched_create_connection): + with check_no_resource_warning(self): + # Try without CacheFTPHandler + self._test_urls([entry], handlers=no_cache_handlers, + retry=False) + with check_no_resource_warning(self): + # Try with CacheFTPHandler (uncached) + self._test_urls([entry], cache_handlers, retry=False) + with check_no_resource_warning(self): + # Try with CacheFTPHandler (cached) + self._test_urls([entry], cache_handlers, retry=False) + # Try without the mock: the handler should not use a closed connection + with check_no_resource_warning(self): + self._test_urls([url], cache_handlers, retry=False) + def test_file(self): TESTFN = os_helper.TESTFN f = open(TESTFN, 'w') @@ -202,6 +238,7 @@ def test_urlwithfrag(self): self.assertEqual(res.geturl(), "http://www.pythontest.net/index.html#frag") + @support.requires_resource('walltime') def test_redirect_url_withfrag(self): redirect_url_with_frag = "http://www.pythontest.net/redir/with_frag/" with socket_helper.transient_internet(redirect_url_with_frag): @@ -260,18 +297,16 @@ def _test_urls(self, urls, handlers, retry=True): else: req = expected_err = None + if expected_err: + context = self.assertRaises(expected_err) + else: + context = contextlib.nullcontext() + with socket_helper.transient_internet(url): - try: + f = None + with context: f = urlopen(url, req, support.INTERNET_TIMEOUT) - # urllib.error.URLError is a subclass of OSError - except OSError as err: - if expected_err: - msg = ("Didn't get expected error(s) %s for %s %s, got %s: %s" % - (expected_err, url, req, type(err), err)) - self.assertIsInstance(err, expected_err, msg) - else: - raise - else: + if f is not None: try: with time_out, \ socket_peer_reset, \ @@ -340,7 +375,7 @@ def test_http_timeout(self): FTP_HOST = 'ftp://www.pythontest.net/' - @skip_ftp_test_on_travis + @support.requires_resource('walltime') def test_ftp_basic(self): self.assertIsNone(socket.getdefaulttimeout()) with socket_helper.transient_internet(self.FTP_HOST, timeout=None): @@ -348,7 +383,6 @@ def test_ftp_basic(self): self.addCleanup(u.close) self.assertIsNone(u.fp.fp.raw._sock.gettimeout()) - @skip_ftp_test_on_travis def test_ftp_default_timeout(self): self.assertIsNone(socket.getdefaulttimeout()) with socket_helper.transient_internet(self.FTP_HOST): @@ -360,7 +394,7 @@ def test_ftp_default_timeout(self): socket.setdefaulttimeout(None) self.assertEqual(u.fp.fp.raw._sock.gettimeout(), 60) - @skip_ftp_test_on_travis + @support.requires_resource('walltime') def test_ftp_no_timeout(self): self.assertIsNone(socket.getdefaulttimeout()) with socket_helper.transient_internet(self.FTP_HOST): @@ -372,7 +406,7 @@ def test_ftp_no_timeout(self): socket.setdefaulttimeout(None) self.assertIsNone(u.fp.fp.raw._sock.gettimeout()) - @skip_ftp_test_on_travis + @support.requires_resource('walltime') def test_ftp_timeout(self): with socket_helper.transient_internet(self.FTP_HOST): u = _urlopen_with_retry(self.FTP_HOST, timeout=60) diff --git a/Lib/test/test_urllibnet.py b/Lib/test/test_urllibnet.py index 773101ce41..6733fe9c6e 100644 --- a/Lib/test/test_urllibnet.py +++ b/Lib/test/test_urllibnet.py @@ -2,6 +2,7 @@ from test import support from test.support import os_helper from test.support import socket_helper +from test.support.testcase import ExtraAssertions import contextlib import socket @@ -34,7 +35,7 @@ def testURLread(self): f.read() -class urlopenNetworkTests(unittest.TestCase): +class urlopenNetworkTests(unittest.TestCase, ExtraAssertions): """Tests urllib.request.urlopen using the network. These tests are not exhaustive. Assuming that testing using files does a @@ -70,8 +71,7 @@ def test_basic(self): with self.urlopen(self.url) as open_url: for attr in ("read", "readline", "readlines", "fileno", "close", "info", "geturl"): - self.assertTrue(hasattr(open_url, attr), "object returned from " - "urlopen lacks the %s attribute" % attr) + self.assertHasAttr(open_url, attr) self.assertTrue(open_url.read(), "calling 'read' failed") def test_readlines(self): @@ -109,6 +109,7 @@ def test_getcode(self): open_url.close() self.assertEqual(code, 404) + @support.requires_resource('walltime') def test_bad_address(self): # Make sure proper exception is raised when connecting to a bogus # address. @@ -191,6 +192,7 @@ def test_header(self): logo = "http://www.pythontest.net/" + @support.requires_resource('walltime') def test_data_header(self): with self.urlretrieve(self.logo) as (file_location, fileheaders): datevalue = fileheaders.get('Date') diff --git a/Lib/test/test_userdict.py b/Lib/test/test_userdict.py index 483910aaa4..61e79f553e 100644 --- a/Lib/test/test_userdict.py +++ b/Lib/test/test_userdict.py @@ -1,6 +1,6 @@ # Check every path through every method of UserDict -from test import mapping_tests +from test import mapping_tests, support import unittest import collections @@ -213,6 +213,11 @@ class G(collections.UserDict): else: self.fail("g[42] didn't raise KeyError") + # Decorate existing test with recursion limit, because + # the test is for C structure, but `UserDict` is a Python structure. + test_repr_deep = support.infinite_recursion(25)( + mapping_tests.TestHashMappingProtocol.test_repr_deep, + ) if __name__ == "__main__": diff --git a/Lib/test/test_userlist.py b/Lib/test/test_userlist.py index 1ed67dac80..312702c8e3 100644 --- a/Lib/test/test_userlist.py +++ b/Lib/test/test_userlist.py @@ -3,6 +3,8 @@ from collections import UserList from test import list_tests import unittest +from test import support + class UserListTest(list_tests.CommonTest): type2test = UserList @@ -65,5 +67,11 @@ def test_userlist_copy(self): self.assertEqual(u, v) self.assertEqual(type(u), type(v)) + # Decorate existing test with recursion limit, because + # the test is for C structure, but `UserList` is a Python structure. + test_repr_deep = support.infinite_recursion(25)( + list_tests.CommonTest.test_repr_deep, + ) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_userstring.py b/Lib/test/test_userstring.py index 51b4f6041e..74df52f541 100644 --- a/Lib/test/test_userstring.py +++ b/Lib/test/test_userstring.py @@ -7,8 +7,7 @@ from collections import UserString class UserStringTest( - string_tests.CommonTest, - string_tests.MixinStrUnicodeUserStringTest, + string_tests.StringLikeTest, unittest.TestCase ): diff --git a/Lib/test/test_uuid.py b/Lib/test/test_uuid.py index 069221ae47..4aa15f6993 100644 --- a/Lib/test/test_uuid.py +++ b/Lib/test/test_uuid.py @@ -1,6 +1,7 @@ import unittest from test import support from test.support import import_helper +from test.support.script_helper import assert_python_ok import builtins import contextlib import copy @@ -32,8 +33,7 @@ def get_command_stdout(command, args): class BaseTestUUID: uuid = None - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_safe_uuid_enum(self): class CheckedSafeUUID(enum.Enum): safe = 0 @@ -775,10 +775,37 @@ def test_cli_uuid5_ouputted_with_valid_namespace_and_name(self): class TestUUIDWithoutExtModule(BaseTestUUID, unittest.TestCase): uuid = py_uuid + @unittest.skipUnless(c_uuid, 'requires the C _uuid module') class TestUUIDWithExtModule(BaseTestUUID, unittest.TestCase): uuid = c_uuid + def check_has_stable_libuuid_extractable_node(self): + if not self.uuid._has_stable_extractable_node: + self.skipTest("libuuid cannot deduce MAC address") + + @unittest.skipUnless(os.name == 'posix', 'POSIX only') + def test_unix_getnode_from_libuuid(self): + self.check_has_stable_libuuid_extractable_node() + script = 'import uuid; print(uuid._unix_getnode())' + _, n_a, _ = assert_python_ok('-c', script) + _, n_b, _ = assert_python_ok('-c', script) + n_a, n_b = n_a.decode().strip(), n_b.decode().strip() + self.assertTrue(n_a.isdigit()) + self.assertTrue(n_b.isdigit()) + self.assertEqual(n_a, n_b) + + @unittest.skipUnless(os.name == 'nt', 'Windows only') + def test_windows_getnode_from_libuuid(self): + self.check_has_stable_libuuid_extractable_node() + script = 'import uuid; print(uuid._windll_getnode())' + _, n_a, _ = assert_python_ok('-c', script) + _, n_b, _ = assert_python_ok('-c', script) + n_a, n_b = n_a.decode().strip(), n_b.decode().strip() + self.assertTrue(n_a.isdigit()) + self.assertTrue(n_b.isdigit()) + self.assertEqual(n_a, n_b) + class BaseTestInternals: _uuid = py_uuid diff --git a/Lib/test/test_venv.py b/Lib/test/test_venv.py index 94d626598b..19b1207053 100644 --- a/Lib/test/test_venv.py +++ b/Lib/test/test_venv.py @@ -5,21 +5,31 @@ Licensed to the PSF under a contributor agreement. """ +import contextlib import ensurepip import os import os.path +import pathlib import re import shutil import struct import subprocess import sys +import sysconfig import tempfile -from test.support import (captured_stdout, captured_stderr, requires_zlib, - skip_if_broken_multiprocessing_synchronize) -from test.support.os_helper import (can_symlink, EnvironmentVarGuard, rmtree) +import shlex +from test.support import (captured_stdout, captured_stderr, + skip_if_broken_multiprocessing_synchronize, verbose, + requires_subprocess, is_android, is_apple_mobile, + is_emscripten, is_wasi, + requires_venv_with_pip, TEST_HOME_DIR, + requires_resource, copy_python_src_ignore) +from test.support.os_helper import (can_symlink, EnvironmentVarGuard, rmtree, + TESTFN, FakePath) +from test.support.testcase import ExtraAssertions import unittest import venv -from unittest.mock import patch +from unittest.mock import patch, Mock try: import ctypes @@ -33,18 +43,29 @@ or sys._base_executable != sys.executable, 'cannot run venv.create from within a venv on this platform') +if is_android or is_apple_mobile or is_emscripten or is_wasi: + raise unittest.SkipTest("venv is not available on this platform") + +@requires_subprocess() def check_output(cmd, encoding=None): p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, - encoding=encoding) + env={**os.environ, "PYTHONHOME": ""}) out, err = p.communicate() if p.returncode: + if verbose and err: + print(err.decode(encoding or 'utf-8', 'backslashreplace')) raise subprocess.CalledProcessError( p.returncode, cmd, out, err) + if encoding: + return ( + out.decode(encoding, 'backslashreplace'), + err.decode(encoding, 'backslashreplace'), + ) return out, err -class BaseTest(unittest.TestCase): +class BaseTest(unittest.TestCase, ExtraAssertions): """Base class for venv tests.""" maxDiff = 80 * 50 @@ -56,7 +77,7 @@ def setUp(self): self.include = 'Include' else: self.bindir = 'bin' - self.lib = ('lib', 'python%d.%d' % sys.version_info[:2]) + self.lib = ('lib', f'python{sysconfig._get_python_version_abi()}') self.include = 'include' executable = sys._base_executable self.exe = os.path.split(executable)[-1] @@ -70,6 +91,13 @@ def setUp(self): def tearDown(self): rmtree(self.env_dir) + def envpy(self, *, real_env_dir=False): + if real_env_dir: + env_dir = os.path.realpath(self.env_dir) + else: + env_dir = self.env_dir + return os.path.join(env_dir, self.bindir, self.exe) + def run_with_capture(self, func, *args, **kwargs): with captured_stdout() as output: with captured_stderr() as error: @@ -91,12 +119,27 @@ def isdir(self, *args): fn = self.get_env_file(*args) self.assertTrue(os.path.isdir(fn)) - def test_defaults(self): + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_defaults_with_str_path(self): """ - Test the create function with default arguments. + Test the create function with default arguments and a str path. """ rmtree(self.env_dir) self.run_with_capture(venv.create, self.env_dir) + self._check_output_of_default_create() + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_defaults_with_pathlike(self): + """ + Test the create function with default arguments and a path-like path. + """ + rmtree(self.env_dir) + self.run_with_capture(venv.create, FakePath(self.env_dir)) + self._check_output_of_default_create() + + def _check_output_of_default_create(self): self.isdir(self.bindir) self.isdir(self.include) self.isdir(*self.lib) @@ -112,6 +155,12 @@ def test_defaults(self): executable = sys._base_executable path = os.path.dirname(executable) self.assertIn('home = %s' % path, data) + self.assertIn('executable = %s' % + os.path.realpath(sys.executable), data) + copies = '' if os.name=='nt' else ' --copies' + cmd = (f'command = {sys.executable} -m venv{copies} --without-pip ' + f'--without-scm-ignore-files {self.env_dir}') + self.assertIn(cmd, data) fn = self.get_env_file(self.bindir, self.exe) if not os.path.exists(fn): # diagnostics for Windows buildbot failures bd = self.get_env_file(self.bindir) @@ -119,6 +168,39 @@ def test_defaults(self): print(' %r' % os.listdir(bd)) self.assertTrue(os.path.exists(fn), 'File %r should exist.' % fn) + def test_config_file_command_key(self): + options = [ + (None, None, None), # Default case. + ('--copies', 'symlinks', False), + ('--without-pip', 'with_pip', False), + ('--system-site-packages', 'system_site_packages', True), + ('--clear', 'clear', True), + ('--upgrade', 'upgrade', True), + ('--upgrade-deps', 'upgrade_deps', True), + ('--prompt="foobar"', 'prompt', 'foobar'), + ('--without-scm-ignore-files', 'scm_ignore_files', frozenset()), + ] + for opt, attr, value in options: + with self.subTest(opt=opt, attr=attr, value=value): + rmtree(self.env_dir) + if not attr: + kwargs = {} + else: + kwargs = {attr: value} + b = venv.EnvBuilder(**kwargs) + b.upgrade_dependencies = Mock() # avoid pip command to upgrade deps + b._setup_pip = Mock() # avoid pip setup + self.run_with_capture(b.create, self.env_dir) + data = self.get_text_file_contents('pyvenv.cfg') + if not attr or opt.endswith('git'): + for opt in ('--system-site-packages', '--clear', '--upgrade', + '--upgrade-deps', '--prompt'): + self.assertNotRegex(data, rf'command = .* {opt}') + elif os.name=='nt' and attr=='symlinks': + pass + else: + self.assertRegex(data, rf'command = .* {opt}') + def test_prompt(self): env_name = os.path.split(self.env_dir)[1] @@ -127,7 +209,7 @@ def test_prompt(self): self.run_with_capture(builder.create, self.env_dir) context = builder.ensure_directories(self.env_dir) data = self.get_text_file_contents('pyvenv.cfg') - self.assertEqual(context.prompt, '(%s) ' % env_name) + self.assertEqual(context.prompt, env_name) self.assertNotIn("prompt = ", data) rmtree(self.env_dir) @@ -135,7 +217,7 @@ def test_prompt(self): self.run_with_capture(builder.create, self.env_dir) context = builder.ensure_directories(self.env_dir) data = self.get_text_file_contents('pyvenv.cfg') - self.assertEqual(context.prompt, '(My prompt) ') + self.assertEqual(context.prompt, 'My prompt') self.assertIn("prompt = 'My prompt'\n", data) rmtree(self.env_dir) @@ -144,13 +226,19 @@ def test_prompt(self): self.run_with_capture(builder.create, self.env_dir) context = builder.ensure_directories(self.env_dir) data = self.get_text_file_contents('pyvenv.cfg') - self.assertEqual(context.prompt, '(%s) ' % cwd) + self.assertEqual(context.prompt, cwd) self.assertIn("prompt = '%s'\n" % cwd, data) def test_upgrade_dependencies(self): builder = venv.EnvBuilder() - bin_path = 'Scripts' if sys.platform == 'win32' else 'bin' + bin_path = 'bin' python_exe = os.path.split(sys.executable)[1] + if sys.platform == 'win32': + bin_path = 'Scripts' + if os.path.normcase(os.path.splitext(python_exe)[0]).endswith('_d'): + python_exe = 'python_d.exe' + else: + python_exe = 'python.exe' with tempfile.TemporaryDirectory() as fake_env_dir: expect_exe = os.path.normcase( os.path.join(fake_env_dir, bin_path, python_exe) @@ -158,7 +246,7 @@ def test_upgrade_dependencies(self): if sys.platform == 'win32': expect_exe = os.path.normcase(os.path.realpath(expect_exe)) - def pip_cmd_checker(cmd): + def pip_cmd_checker(cmd, **kwargs): cmd[0] = os.path.normcase(cmd[0]) self.assertEqual( cmd, @@ -169,12 +257,11 @@ def pip_cmd_checker(cmd): 'install', '--upgrade', 'pip', - 'setuptools' ] ) fake_context = builder.ensure_directories(fake_env_dir) - with patch('venv.subprocess.check_call', pip_cmd_checker): + with patch('venv.subprocess.check_output', pip_cmd_checker): builder.upgrade_dependencies(fake_context) @requireVenvCreate @@ -185,8 +272,7 @@ def test_prefixes(self): # check a venv's prefixes rmtree(self.env_dir) self.run_with_capture(venv.create, self.env_dir) - envpy = os.path.join(self.env_dir, self.bindir, self.exe) - cmd = [envpy, '-c', None] + cmd = [self.envpy(), '-c', None] for prefix, expected in ( ('prefix', self.env_dir), ('exec_prefix', self.env_dir), @@ -194,7 +280,76 @@ def test_prefixes(self): ('base_exec_prefix', sys.base_exec_prefix)): cmd[2] = 'import sys; print(sys.%s)' % prefix out, err = check_output(cmd) - self.assertEqual(out.strip(), expected.encode()) + self.assertEqual(pathlib.Path(out.strip().decode()), + pathlib.Path(expected), prefix) + + @requireVenvCreate + def test_sysconfig(self): + """ + Test that the sysconfig functions work in a virtual environment. + """ + rmtree(self.env_dir) + self.run_with_capture(venv.create, self.env_dir, symlinks=False) + cmd = [self.envpy(), '-c', None] + for call, expected in ( + # installation scheme + ('get_preferred_scheme("prefix")', 'venv'), + ('get_default_scheme()', 'venv'), + # build environment + ('is_python_build()', str(sysconfig.is_python_build())), + ('get_makefile_filename()', sysconfig.get_makefile_filename()), + ('get_config_h_filename()', sysconfig.get_config_h_filename()), + ('get_config_var("Py_GIL_DISABLED")', + str(sysconfig.get_config_var("Py_GIL_DISABLED")))): + with self.subTest(call): + cmd[2] = 'import sysconfig; print(sysconfig.%s)' % call + out, err = check_output(cmd, encoding='utf-8') + self.assertEqual(out.strip(), expected, err) + for attr, expected in ( + ('executable', self.envpy()), + # Usually compare to sys.executable, but if we're running in our own + # venv then we really need to compare to our base executable + ('_base_executable', sys._base_executable), + ): + with self.subTest(attr): + cmd[2] = f'import sys; print(sys.{attr})' + out, err = check_output(cmd, encoding='utf-8') + self.assertEqual(out.strip(), expected, err) + + @requireVenvCreate + @unittest.skipUnless(can_symlink(), 'Needs symlinks') + def test_sysconfig_symlinks(self): + """ + Test that the sysconfig functions work in a virtual environment. + """ + rmtree(self.env_dir) + self.run_with_capture(venv.create, self.env_dir, symlinks=True) + cmd = [self.envpy(), '-c', None] + for call, expected in ( + # installation scheme + ('get_preferred_scheme("prefix")', 'venv'), + ('get_default_scheme()', 'venv'), + # build environment + ('is_python_build()', str(sysconfig.is_python_build())), + ('get_makefile_filename()', sysconfig.get_makefile_filename()), + ('get_config_h_filename()', sysconfig.get_config_h_filename()), + ('get_config_var("Py_GIL_DISABLED")', + str(sysconfig.get_config_var("Py_GIL_DISABLED")))): + with self.subTest(call): + cmd[2] = 'import sysconfig; print(sysconfig.%s)' % call + out, err = check_output(cmd, encoding='utf-8') + self.assertEqual(out.strip(), expected, err) + for attr, expected in ( + ('executable', self.envpy()), + # Usually compare to sys.executable, but if we're running in our own + # venv then we really need to compare to our base executable + # HACK: Test fails on POSIX with unversioned binary (PR gh-113033) + #('_base_executable', sys._base_executable), + ): + with self.subTest(attr): + cmd[2] = f'import sys; print(sys.{attr})' + out, err = check_output(cmd, encoding='utf-8') + self.assertEqual(out.strip(), expected, err) if sys.platform == 'win32': ENV_SUBDIRS = ( @@ -259,6 +414,8 @@ def test_unoverwritable_fails(self): self.assertRaises((ValueError, OSError), venv.create, self.env_dir) self.clear_directory(self.env_dir) + # TODO: RUSTPYTHON + @unittest.expectedFailure def test_upgrade(self): """ Test upgrading an existing environment directory. @@ -321,8 +478,7 @@ def test_executable(self): """ rmtree(self.env_dir) self.run_with_capture(venv.create, self.env_dir) - envpy = os.path.join(os.path.realpath(self.env_dir), - self.bindir, self.exe) + envpy = self.envpy(real_env_dir=True) out, err = check_output([envpy, '-c', 'import sys; print(sys.executable)']) self.assertEqual(out.strip(), envpy.encode()) @@ -335,12 +491,89 @@ def test_executable_symlinks(self): rmtree(self.env_dir) builder = venv.EnvBuilder(clear=True, symlinks=True) builder.create(self.env_dir) - envpy = os.path.join(os.path.realpath(self.env_dir), - self.bindir, self.exe) + envpy = self.envpy(real_env_dir=True) out, err = check_output([envpy, '-c', 'import sys; print(sys.executable)']) self.assertEqual(out.strip(), envpy.encode()) + # gh-124651: test quoted strings + @unittest.skipIf(os.name == 'nt', 'contains invalid characters on Windows') + def test_special_chars_bash(self): + """ + Test that the template strings are quoted properly (bash) + """ + rmtree(self.env_dir) + bash = shutil.which('bash') + if bash is None: + self.skipTest('bash required for this test') + env_name = '"\';&&$e|\'"' + env_dir = os.path.join(os.path.realpath(self.env_dir), env_name) + builder = venv.EnvBuilder(clear=True) + builder.create(env_dir) + activate = os.path.join(env_dir, self.bindir, 'activate') + test_script = os.path.join(self.env_dir, 'test_special_chars.sh') + with open(test_script, "w") as f: + f.write(f'source {shlex.quote(activate)}\n' + 'python -c \'import sys; print(sys.executable)\'\n' + 'python -c \'import os; print(os.environ["VIRTUAL_ENV"])\'\n' + 'deactivate\n') + out, err = check_output([bash, test_script]) + lines = out.splitlines() + self.assertTrue(env_name.encode() in lines[0]) + self.assertEndsWith(lines[1], env_name.encode()) + + # gh-124651: test quoted strings + @unittest.skipIf(os.name == 'nt', 'contains invalid characters on Windows') + @unittest.skipIf(sys.platform.startswith('netbsd'), + "NetBSD csh fails with quoted special chars; see gh-139308") + def test_special_chars_csh(self): + """ + Test that the template strings are quoted properly (csh) + """ + rmtree(self.env_dir) + csh = shutil.which('tcsh') or shutil.which('csh') + if csh is None: + self.skipTest('csh required for this test') + env_name = '"\';&&$e|\'"' + env_dir = os.path.join(os.path.realpath(self.env_dir), env_name) + builder = venv.EnvBuilder(clear=True) + builder.create(env_dir) + activate = os.path.join(env_dir, self.bindir, 'activate.csh') + test_script = os.path.join(self.env_dir, 'test_special_chars.csh') + with open(test_script, "w") as f: + f.write(f'source {shlex.quote(activate)}\n' + 'python -c \'import sys; print(sys.executable)\'\n' + 'python -c \'import os; print(os.environ["VIRTUAL_ENV"])\'\n' + 'deactivate\n') + out, err = check_output([csh, test_script]) + lines = out.splitlines() + self.assertTrue(env_name.encode() in lines[0]) + self.assertEndsWith(lines[1], env_name.encode()) + + # gh-124651: test quoted strings on Windows + @unittest.skipUnless(os.name == 'nt', 'only relevant on Windows') + def test_special_chars_windows(self): + """ + Test that the template strings are quoted properly on Windows + """ + rmtree(self.env_dir) + env_name = "'&&^$e" + env_dir = os.path.join(os.path.realpath(self.env_dir), env_name) + builder = venv.EnvBuilder(clear=True) + builder.create(env_dir) + activate = os.path.join(env_dir, self.bindir, 'activate.bat') + test_batch = os.path.join(self.env_dir, 'test_special_chars.bat') + with open(test_batch, "w") as f: + f.write('@echo off\n' + f'"{activate}" & ' + f'{self.exe} -c "import sys; print(sys.executable)" & ' + f'{self.exe} -c "import os; print(os.environ[\'VIRTUAL_ENV\'])" & ' + 'deactivate') + out, err = check_output([test_batch]) + lines = out.splitlines() + self.assertTrue(env_name.encode() in lines[0]) + self.assertEndsWith(lines[1], env_name.encode()) + @unittest.skipUnless(os.name == 'nt', 'only relevant on Windows') def test_unicode_in_batch_file(self): """ @@ -351,13 +584,27 @@ def test_unicode_in_batch_file(self): builder = venv.EnvBuilder(clear=True) builder.create(env_dir) activate = os.path.join(env_dir, self.bindir, 'activate.bat') - envpy = os.path.join(env_dir, self.bindir, self.exe) out, err = check_output( [activate, '&', self.exe, '-c', 'print(0)'], encoding='oem', ) self.assertEqual(out.strip(), '0') + @unittest.skipUnless(os.name == 'nt' and can_symlink(), + 'symlinks on Windows') + def test_failed_symlink(self): + """ + Test handling of failed symlinks on Windows. + """ + rmtree(self.env_dir) + env_dir = os.path.join(os.path.realpath(self.env_dir), 'venv') + with patch('os.symlink') as mock_symlink: + mock_symlink.side_effect = OSError() + builder = venv.EnvBuilder(clear=True, symlinks=True) + _, err = self.run_with_capture(builder.create, env_dir) + filepath_regex = r"'[A-Z]:\\\\(?:[^\\\\]+\\\\)*[^\\\\]+'" + self.assertRegex(err, rf"Unable to symlink {filepath_regex} to {filepath_regex}") + @requireVenvCreate def test_multiprocessing(self): """ @@ -370,15 +617,25 @@ def test_multiprocessing(self): rmtree(self.env_dir) self.run_with_capture(venv.create, self.env_dir) - envpy = os.path.join(os.path.realpath(self.env_dir), - self.bindir, self.exe) - out, err = check_output([envpy, '-c', + out, err = check_output([self.envpy(real_env_dir=True), '-c', 'from multiprocessing import Pool; ' 'pool = Pool(1); ' 'print(pool.apply_async("Python".lower).get(3)); ' 'pool.terminate()']) self.assertEqual(out.strip(), "python".encode()) + @requireVenvCreate + def test_multiprocessing_recursion(self): + """ + Test that the multiprocessing is able to spawn itself + """ + skip_if_broken_multiprocessing_synchronize() + + rmtree(self.env_dir) + self.run_with_capture(venv.create, self.env_dir) + script = os.path.join(TEST_HOME_DIR, '_test_venv_multiprocessing.py') + subprocess.check_call([self.envpy(real_env_dir=True), "-I", script]) + @unittest.skipIf(os.name == 'nt', 'not relevant on Windows') def test_deactivate_with_strict_bash_opts(self): bash = shutil.which("bash") @@ -404,19 +661,250 @@ def test_macos_env(self): builder = venv.EnvBuilder() builder.create(self.env_dir) - envpy = os.path.join(os.path.realpath(self.env_dir), - self.bindir, self.exe) - out, err = check_output([envpy, '-c', + out, err = check_output([self.envpy(real_env_dir=True), '-c', 'import os; print("__PYVENV_LAUNCHER__" in os.environ)']) self.assertEqual(out.strip(), 'False'.encode()) + def test_pathsep_error(self): + """ + Test that venv creation fails when the target directory contains + the path separator. + """ + rmtree(self.env_dir) + bad_itempath = self.env_dir + os.pathsep + self.assertRaises(ValueError, venv.create, bad_itempath) + self.assertRaises(ValueError, venv.create, FakePath(bad_itempath)) + + @unittest.skipIf(os.name == 'nt', 'not relevant on Windows') + @requireVenvCreate + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_zippath_from_non_installed_posix(self): + """ + Test that when create venv from non-installed python, the zip path + value is as expected. + """ + rmtree(self.env_dir) + # First try to create a non-installed python. It's not a real full + # functional non-installed python, but enough for this test. + platlibdir = sys.platlibdir + non_installed_dir = os.path.realpath(tempfile.mkdtemp()) + self.addCleanup(rmtree, non_installed_dir) + bindir = os.path.join(non_installed_dir, self.bindir) + os.mkdir(bindir) + shutil.copy2(sys.executable, bindir) + libdir = os.path.join(non_installed_dir, platlibdir, self.lib[1]) + os.makedirs(libdir) + landmark = os.path.join(libdir, "os.py") + abi_thread = "t" if sysconfig.get_config_var("Py_GIL_DISABLED") else "" + stdlib_zip = f"python{sys.version_info.major}{sys.version_info.minor}{abi_thread}" + zip_landmark = os.path.join(non_installed_dir, + platlibdir, + stdlib_zip) + additional_pythonpath_for_non_installed = [] + + # Copy stdlib files to the non-installed python so venv can + # correctly calculate the prefix. + for eachpath in sys.path: + if eachpath.endswith(".zip"): + if os.path.isfile(eachpath): + shutil.copyfile( + eachpath, + os.path.join(non_installed_dir, platlibdir)) + elif os.path.isfile(os.path.join(eachpath, "os.py")): + names = os.listdir(eachpath) + ignored_names = copy_python_src_ignore(eachpath, names) + for name in names: + if name in ignored_names: + continue + if name == "site-packages": + continue + fn = os.path.join(eachpath, name) + if os.path.isfile(fn): + shutil.copy(fn, libdir) + elif os.path.isdir(fn): + shutil.copytree(fn, os.path.join(libdir, name), + ignore=copy_python_src_ignore) + else: + additional_pythonpath_for_non_installed.append( + eachpath) + cmd = [os.path.join(non_installed_dir, self.bindir, self.exe), + "-m", + "venv", + "--without-pip", + "--without-scm-ignore-files", + self.env_dir] + # Our fake non-installed python is not fully functional because + # it cannot find the extensions. Set PYTHONPATH so it can run the + # venv module correctly. + pythonpath = os.pathsep.join( + additional_pythonpath_for_non_installed) + # For python built with shared enabled. We need to set + # LD_LIBRARY_PATH so the non-installed python can find and link + # libpython.so + ld_library_path = sysconfig.get_config_var("LIBDIR") + if not ld_library_path or sysconfig.is_python_build(): + ld_library_path = os.path.abspath(os.path.dirname(sys.executable)) + if sys.platform == 'darwin': + ld_library_path_env = "DYLD_LIBRARY_PATH" + else: + ld_library_path_env = "LD_LIBRARY_PATH" + child_env = { + "PYTHONPATH": pythonpath, + ld_library_path_env: ld_library_path, + } + if asan_options := os.environ.get("ASAN_OPTIONS"): + # prevent https://github.com/python/cpython/issues/104839 + child_env["ASAN_OPTIONS"] = asan_options + subprocess.check_call(cmd, env=child_env) + # Now check the venv created from the non-installed python has + # correct zip path in pythonpath. + cmd = [self.envpy(), '-S', '-c', 'import sys; print(sys.path)'] + out, err = check_output(cmd) + self.assertTrue(zip_landmark.encode() in out) + + @requireVenvCreate + def test_activate_shell_script_has_no_dos_newlines(self): + """ + Test that the `activate` shell script contains no CR LF. + This is relevant for Cygwin, as the Windows build might have + converted line endings accidentally. + """ + venv_dir = pathlib.Path(self.env_dir) + rmtree(venv_dir) + [[scripts_dir], *_] = self.ENV_SUBDIRS + script_path = venv_dir / scripts_dir / "activate" + venv.create(venv_dir) + with open(script_path, 'rb') as script: + for i, line in enumerate(script, 1): + error_message = f"CR LF found in line {i}" + self.assertFalse(line.endswith(b'\r\n'), error_message) + + @requireVenvCreate + def test_scm_ignore_files_git(self): + """ + Test that a .gitignore file is created when "git" is specified. + The file should contain a `*\n` line. + """ + self.run_with_capture(venv.create, self.env_dir, + scm_ignore_files={'git'}) + file_lines = self.get_text_file_contents('.gitignore').splitlines() + self.assertIn('*', file_lines) + + @requireVenvCreate + def test_create_scm_ignore_files_multiple(self): + """ + Test that ``scm_ignore_files`` can work with multiple SCMs. + """ + bzrignore_name = ".bzrignore" + contents = "# For Bazaar.\n*\n" + + class BzrEnvBuilder(venv.EnvBuilder): + def create_bzr_ignore_file(self, context): + gitignore_path = os.path.join(context.env_dir, bzrignore_name) + with open(gitignore_path, 'w', encoding='utf-8') as file: + file.write(contents) + + builder = BzrEnvBuilder(scm_ignore_files={'git', 'bzr'}) + self.run_with_capture(builder.create, self.env_dir) + + gitignore_lines = self.get_text_file_contents('.gitignore').splitlines() + self.assertIn('*', gitignore_lines) + + bzrignore = self.get_text_file_contents(bzrignore_name) + self.assertEqual(bzrignore, contents) + + @requireVenvCreate + def test_create_scm_ignore_files_empty(self): + """ + Test that no default ignore files are created when ``scm_ignore_files`` + is empty. + """ + # scm_ignore_files is set to frozenset() by default. + self.run_with_capture(venv.create, self.env_dir) + with self.assertRaises(FileNotFoundError): + self.get_text_file_contents('.gitignore') + + self.assertIn("--without-scm-ignore-files", + self.get_text_file_contents('pyvenv.cfg')) + + @requireVenvCreate + def test_cli_with_scm_ignore_files(self): + """ + Test that default SCM ignore files are created by default via the CLI. + """ + self.run_with_capture(venv.main, ['--without-pip', self.env_dir]) + + gitignore_lines = self.get_text_file_contents('.gitignore').splitlines() + self.assertIn('*', gitignore_lines) + + @requireVenvCreate + def test_cli_without_scm_ignore_files(self): + """ + Test that ``--without-scm-ignore-files`` doesn't create SCM ignore files. + """ + args = ['--without-pip', '--without-scm-ignore-files', self.env_dir] + self.run_with_capture(venv.main, args) + + with self.assertRaises(FileNotFoundError): + self.get_text_file_contents('.gitignore') + + def test_venv_same_path(self): + same_path = venv.EnvBuilder._same_path + if sys.platform == 'win32': + # Case-insensitive, and handles short/long names + tests = [ + (True, TESTFN, TESTFN), + (True, TESTFN.lower(), TESTFN.upper()), + ] + import _winapi + # ProgramFiles is the most reliable path that will have short/long + progfiles = os.getenv('ProgramFiles') + if progfiles: + tests = [ + *tests, + (True, progfiles, progfiles), + (True, _winapi.GetShortPathName(progfiles), _winapi.GetLongPathName(progfiles)), + ] + else: + # Just a simple case-sensitive comparison + tests = [ + (True, TESTFN, TESTFN), + (False, TESTFN.lower(), TESTFN.upper()), + ] + for r, path1, path2 in tests: + with self.subTest(f"{path1}-{path2}"): + if r: + self.assertTrue(same_path(path1, path2)) + else: + self.assertFalse(same_path(path1, path2)) + + # gh-126084: venvwlauncher should run pythonw, not python + @requireVenvCreate + @unittest.skipUnless(os.name == 'nt', 'only relevant on Windows') + def test_venvwlauncher(self): + """ + Test that the GUI launcher runs the GUI python. + """ + rmtree(self.env_dir) + venv.create(self.env_dir) + exename = self.exe + # Retain the debug suffix if present + if "python" in exename and not "pythonw" in exename: + exename = exename.replace("python", "pythonw") + envpyw = os.path.join(self.env_dir, self.bindir, exename) + try: + subprocess.check_call([envpyw, "-c", "import sys; " + "assert sys._base_executable.endswith('%s')" % exename]) + except subprocess.CalledProcessError: + self.fail("venvwlauncher.exe did not run %s" % exename) + + @requireVenvCreate class EnsurePipTest(BaseTest): """Test venv module installation of pip.""" def assert_pip_not_installed(self): - envpy = os.path.join(os.path.realpath(self.env_dir), - self.bindir, self.exe) - out, err = check_output([envpy, '-c', + out, err = check_output([self.envpy(real_env_dir=True), '-c', 'try:\n import pip\nexcept ImportError:\n print("OK")']) # We force everything to text, so unittest gives the detailed diff # if we get unexpected results @@ -478,20 +966,14 @@ def do_test_with_pip(self, system_site_packages): # Actually run the create command with all that unhelpful # config in place to ensure we ignore it - try: + with self.nicer_error(): self.run_with_capture(venv.create, self.env_dir, system_site_packages=system_site_packages, with_pip=True) - except subprocess.CalledProcessError as exc: - # The output this produces can be a little hard to read, - # but at least it has all the details - details = exc.output.decode(errors="replace") - msg = "{}\n\n**Subprocess Output**\n{}" - self.fail(msg.format(exc, details)) # Ensure pip is available in the virtual environment - envpy = os.path.join(os.path.realpath(self.env_dir), self.bindir, self.exe) # Ignore DeprecationWarning since pip code is not part of Python - out, err = check_output([envpy, '-W', 'ignore::DeprecationWarning', + out, err = check_output([self.envpy(real_env_dir=True), + '-W', 'ignore::DeprecationWarning', '-W', 'ignore::ImportWarning', '-I', '-m', 'pip', '--version']) # We force everything to text, so unittest gives the detailed diff @@ -508,13 +990,14 @@ def do_test_with_pip(self, system_site_packages): # Check the private uninstall command provided for the Windows # installers works (at least in a virtual environment) with EnvironmentVarGuard() as envvars: - # It seems ensurepip._uninstall calls subprocesses which do not - # inherit the interpreter settings. - envvars["PYTHONWARNINGS"] = "ignore" - out, err = check_output([envpy, - '-W', 'ignore::DeprecationWarning', - '-W', 'ignore::ImportWarning', '-I', - '-m', 'ensurepip._uninstall']) + with self.nicer_error(): + # It seems ensurepip._uninstall calls subprocesses which do not + # inherit the interpreter settings. + envvars["PYTHONWARNINGS"] = "ignore" + out, err = check_output([self.envpy(real_env_dir=True), + '-W', 'ignore::DeprecationWarning', + '-W', 'ignore::ImportWarning', '-I', + '-m', 'ensurepip._uninstall']) # We force everything to text, so unittest gives the detailed diff # if we get unexpected results err = err.decode("latin-1") # Force to text, prevent decoding errors @@ -527,25 +1010,51 @@ def do_test_with_pip(self, system_site_packages): err = re.sub("^(WARNING: )?The directory .* or its parent directory " "is not owned or is not writable by the current user.*$", "", err, flags=re.MULTILINE) + # Ignore warning about missing optional module: + try: + import ssl + except ImportError: + err = re.sub( + "^WARNING: Disabling truststore since ssl support is missing$", + "", + err, flags=re.MULTILINE) self.assertEqual(err.rstrip(), "") # Being fairly specific regarding the expected behaviour for the # initial bundling phase in Python 3.4. If the output changes in # future pip versions, this test can likely be relaxed further. out = out.decode("latin-1") # Force to text, prevent decoding errors self.assertIn("Successfully uninstalled pip", out) - self.assertIn("Successfully uninstalled setuptools", out) # Check pip is now gone from the virtual environment. This only # applies in the system_site_packages=False case, because in the # other case, pip may still be available in the system site-packages if not system_site_packages: self.assert_pip_not_installed() - # Issue #26610: pip/pep425tags.py requires ctypes - @unittest.skipUnless(ctypes, 'pip requires ctypes') - @requires_zlib() + @contextlib.contextmanager + def nicer_error(self): + """ + Capture output from a failed subprocess for easier debugging. + + The output this handler produces can be a little hard to read, + but at least it has all the details. + """ + try: + yield + except subprocess.CalledProcessError as exc: + out = (exc.output or b'').decode(errors="replace") + err = (exc.stderr or b'').decode(errors="replace") + self.fail( + f"{exc}\n\n" + f"**Subprocess Output**\n{out}\n\n" + f"**Subprocess Error**\n{err}" + ) + + @requires_venv_with_pip() + @requires_resource('cpu') def test_with_pip(self): self.do_test_with_pip(False) self.do_test_with_pip(True) + if __name__ == "__main__": unittest.main() diff --git a/Lib/test/test_wait3.py b/Lib/test/test_wait3.py new file mode 100644 index 0000000000..eae885a613 --- /dev/null +++ b/Lib/test/test_wait3.py @@ -0,0 +1,53 @@ +"""This test checks for correct wait3() behavior. +""" + +import os +import subprocess +import sys +import unittest +from test.fork_wait import ForkWait +from test import support + +if not support.has_fork_support: + raise unittest.SkipTest("requires working os.fork()") + +if not hasattr(os, 'wait3'): + raise unittest.SkipTest("os.wait3 not defined") + +class Wait3Test(ForkWait): + def wait_impl(self, cpid, *, exitcode): + # This many iterations can be required, since some previously run + # tests (e.g. test_ctypes) could have spawned a lot of children + # very quickly. + for _ in support.sleeping_retry(support.SHORT_TIMEOUT): + # wait3() shouldn't hang, but some of the buildbots seem to hang + # in the forking tests. This is an attempt to fix the problem. + spid, status, rusage = os.wait3(os.WNOHANG) + if spid == cpid: + break + + self.assertEqual(spid, cpid) + self.assertEqual(os.waitstatus_to_exitcode(status), exitcode) + self.assertTrue(rusage) + + def test_wait3_rusage_initialized(self): + # Ensure a successful wait3() call where no child was ready to report + # its exit status does not return uninitialized memory in the rusage + # structure. See bpo-36279. + args = [sys.executable, '-c', 'import sys; sys.stdin.read()'] + proc = subprocess.Popen(args, stdin=subprocess.PIPE) + try: + pid, status, rusage = os.wait3(os.WNOHANG) + self.assertEqual(0, pid) + self.assertEqual(0, status) + self.assertEqual(0, sum(rusage)) + finally: + proc.stdin.close() + proc.wait() + + +def tearDownModule(): + support.reap_children() + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_wait4.py b/Lib/test/test_wait4.py new file mode 100644 index 0000000000..67afab1d6f --- /dev/null +++ b/Lib/test/test_wait4.py @@ -0,0 +1,38 @@ +"""This test checks for correct wait4() behavior. +""" + +import os +import sys +import unittest +from test.fork_wait import ForkWait +from test import support + +# If either of these do not exist, skip this test. +if not support.has_fork_support: + raise unittest.SkipTest("requires working os.fork()") + +support.get_attribute(os, 'wait4') + + +class Wait4Test(ForkWait): + def wait_impl(self, cpid, *, exitcode): + option = os.WNOHANG + if sys.platform.startswith('aix'): + # Issue #11185: wait4 is broken on AIX and will always return 0 + # with WNOHANG. + option = 0 + for _ in support.sleeping_retry(support.SHORT_TIMEOUT): + # wait4() shouldn't hang, but some of the buildbots seem to hang + # in the forking tests. This is an attempt to fix the problem. + spid, status, rusage = os.wait4(cpid, option) + if spid == cpid: + break + self.assertEqual(spid, cpid) + self.assertEqual(os.waitstatus_to_exitcode(status), exitcode) + self.assertTrue(rusage) + +def tearDownModule(): + support.reap_children() + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_warnings/__init__.py b/Lib/test/test_warnings/__init__.py new file mode 100644 index 0000000000..123d51b77e --- /dev/null +++ b/Lib/test/test_warnings/__init__.py @@ -0,0 +1,1979 @@ +from contextlib import contextmanager +import linecache +import os +import importlib +import inspect +from io import StringIO +import re +import sys +import textwrap +import types +from typing import overload, get_overloads +import unittest +from test import support +from test.support import import_helper +from test.support import os_helper +from test.support import warnings_helper +from test.support import force_not_colorized +from test.support.script_helper import assert_python_ok, assert_python_failure + +from test.test_warnings.data import package_helper +from test.test_warnings.data import stacklevel as warning_tests + +import warnings as original_warnings +from warnings import deprecated + + +py_warnings = import_helper.import_fresh_module('warnings', + blocked=['_warnings']) +c_warnings = import_helper.import_fresh_module('warnings', + fresh=['_warnings']) + +@contextmanager +def warnings_state(module): + """Use a specific warnings implementation in warning_tests.""" + global __warningregistry__ + for to_clear in (sys, warning_tests): + try: + to_clear.__warningregistry__.clear() + except AttributeError: + pass + try: + __warningregistry__.clear() + except NameError: + pass + original_warnings = warning_tests.warnings + original_filters = module.filters + try: + module.filters = original_filters[:] + module.simplefilter("once") + warning_tests.warnings = module + yield + finally: + warning_tests.warnings = original_warnings + module.filters = original_filters + + +class TestWarning(Warning): + pass + + +class BaseTest: + + """Basic bookkeeping required for testing.""" + + def setUp(self): + self.old_unittest_module = unittest.case.warnings + # The __warningregistry__ needs to be in a pristine state for tests + # to work properly. + if '__warningregistry__' in globals(): + del globals()['__warningregistry__'] + if hasattr(warning_tests, '__warningregistry__'): + del warning_tests.__warningregistry__ + if hasattr(sys, '__warningregistry__'): + del sys.__warningregistry__ + # The 'warnings' module must be explicitly set so that the proper + # interaction between _warnings and 'warnings' can be controlled. + sys.modules['warnings'] = self.module + # Ensure that unittest.TestCase.assertWarns() uses the same warnings + # module than warnings.catch_warnings(). Otherwise, + # warnings.catch_warnings() will be unable to remove the added filter. + unittest.case.warnings = self.module + super(BaseTest, self).setUp() + + def tearDown(self): + sys.modules['warnings'] = original_warnings + unittest.case.warnings = self.old_unittest_module + super(BaseTest, self).tearDown() + +class PublicAPITests(BaseTest): + + """Ensures that the correct values are exposed in the + public API. + """ + + def test_module_all_attribute(self): + self.assertTrue(hasattr(self.module, '__all__')) + target_api = ["warn", "warn_explicit", "showwarning", + "formatwarning", "filterwarnings", "simplefilter", + "resetwarnings", "catch_warnings", "deprecated"] + self.assertSetEqual(set(self.module.__all__), + set(target_api)) + +class CPublicAPITests(PublicAPITests, unittest.TestCase): + module = c_warnings + +class PyPublicAPITests(PublicAPITests, unittest.TestCase): + module = py_warnings + +class FilterTests(BaseTest): + + """Testing the filtering functionality.""" + + def test_error(self): + with original_warnings.catch_warnings(module=self.module) as w: + self.module.resetwarnings() + self.module.filterwarnings("error", category=UserWarning) + self.assertRaises(UserWarning, self.module.warn, + "FilterTests.test_error") + + def test_error_after_default(self): + with original_warnings.catch_warnings(module=self.module) as w: + self.module.resetwarnings() + message = "FilterTests.test_ignore_after_default" + def f(): + self.module.warn(message, UserWarning) + + with support.captured_stderr() as stderr: + f() + stderr = stderr.getvalue() + self.assertIn("UserWarning: FilterTests.test_ignore_after_default", + stderr) + self.assertIn("self.module.warn(message, UserWarning)", + stderr) + + self.module.filterwarnings("error", category=UserWarning) + self.assertRaises(UserWarning, f) + + def test_ignore(self): + with original_warnings.catch_warnings(record=True, + module=self.module) as w: + self.module.resetwarnings() + self.module.filterwarnings("ignore", category=UserWarning) + self.module.warn("FilterTests.test_ignore", UserWarning) + self.assertEqual(len(w), 0) + self.assertEqual(list(__warningregistry__), ['version']) + + def test_ignore_after_default(self): + with original_warnings.catch_warnings(record=True, + module=self.module) as w: + self.module.resetwarnings() + message = "FilterTests.test_ignore_after_default" + def f(): + self.module.warn(message, UserWarning) + f() + self.module.filterwarnings("ignore", category=UserWarning) + f() + f() + self.assertEqual(len(w), 1) + + def test_always(self): + with original_warnings.catch_warnings(record=True, + module=self.module) as w: + self.module.resetwarnings() + self.module.filterwarnings("always", category=UserWarning) + message = "FilterTests.test_always" + def f(): + self.module.warn(message, UserWarning) + f() + self.assertEqual(len(w), 1) + self.assertEqual(w[-1].message.args[0], message) + f() + self.assertEqual(len(w), 2) + self.assertEqual(w[-1].message.args[0], message) + + def test_always_after_default(self): + with original_warnings.catch_warnings(record=True, + module=self.module) as w: + self.module.resetwarnings() + message = "FilterTests.test_always_after_ignore" + def f(): + self.module.warn(message, UserWarning) + f() + self.assertEqual(len(w), 1) + self.assertEqual(w[-1].message.args[0], message) + f() + self.assertEqual(len(w), 1) + self.module.filterwarnings("always", category=UserWarning) + f() + self.assertEqual(len(w), 2) + self.assertEqual(w[-1].message.args[0], message) + f() + self.assertEqual(len(w), 3) + self.assertEqual(w[-1].message.args[0], message) + + def test_default(self): + with original_warnings.catch_warnings(record=True, + module=self.module) as w: + self.module.resetwarnings() + self.module.filterwarnings("default", category=UserWarning) + message = UserWarning("FilterTests.test_default") + for x in range(2): + self.module.warn(message, UserWarning) + if x == 0: + self.assertEqual(w[-1].message, message) + del w[:] + elif x == 1: + self.assertEqual(len(w), 0) + else: + raise ValueError("loop variant unhandled") + + def test_module(self): + with original_warnings.catch_warnings(record=True, + module=self.module) as w: + self.module.resetwarnings() + self.module.filterwarnings("module", category=UserWarning) + message = UserWarning("FilterTests.test_module") + self.module.warn(message, UserWarning) + self.assertEqual(w[-1].message, message) + del w[:] + self.module.warn(message, UserWarning) + self.assertEqual(len(w), 0) + + def test_once(self): + with original_warnings.catch_warnings(record=True, + module=self.module) as w: + self.module.resetwarnings() + self.module.filterwarnings("once", category=UserWarning) + message = UserWarning("FilterTests.test_once") + self.module.warn_explicit(message, UserWarning, "__init__.py", + 42) + self.assertEqual(w[-1].message, message) + del w[:] + self.module.warn_explicit(message, UserWarning, "__init__.py", + 13) + self.assertEqual(len(w), 0) + self.module.warn_explicit(message, UserWarning, "test_warnings2.py", + 42) + self.assertEqual(len(w), 0) + + def test_module_globals(self): + with original_warnings.catch_warnings(record=True, + module=self.module) as w: + self.module.simplefilter("always", UserWarning) + + # bpo-33509: module_globals=None must not crash + self.module.warn_explicit('msg', UserWarning, "filename", 42, + module_globals=None) + self.assertEqual(len(w), 1) + + # Invalid module_globals type + with self.assertRaises(TypeError): + self.module.warn_explicit('msg', UserWarning, "filename", 42, + module_globals=True) + self.assertEqual(len(w), 1) + + # Empty module_globals + self.module.warn_explicit('msg', UserWarning, "filename", 42, + module_globals={}) + self.assertEqual(len(w), 2) + + def test_inheritance(self): + with original_warnings.catch_warnings(module=self.module) as w: + self.module.resetwarnings() + self.module.filterwarnings("error", category=Warning) + self.assertRaises(UserWarning, self.module.warn, + "FilterTests.test_inheritance", UserWarning) + + def test_ordering(self): + with original_warnings.catch_warnings(record=True, + module=self.module) as w: + self.module.resetwarnings() + self.module.filterwarnings("ignore", category=UserWarning) + self.module.filterwarnings("error", category=UserWarning, + append=True) + del w[:] + try: + self.module.warn("FilterTests.test_ordering", UserWarning) + except UserWarning: + self.fail("order handling for actions failed") + self.assertEqual(len(w), 0) + + def test_filterwarnings(self): + # Test filterwarnings(). + # Implicitly also tests resetwarnings(). + with original_warnings.catch_warnings(record=True, + module=self.module) as w: + self.module.filterwarnings("error", "", Warning, "", 0) + self.assertRaises(UserWarning, self.module.warn, 'convert to error') + + self.module.resetwarnings() + text = 'handle normally' + self.module.warn(text) + self.assertEqual(str(w[-1].message), text) + self.assertIs(w[-1].category, UserWarning) + + self.module.filterwarnings("ignore", "", Warning, "", 0) + text = 'filtered out' + self.module.warn(text) + self.assertNotEqual(str(w[-1].message), text) + + self.module.resetwarnings() + self.module.filterwarnings("error", "hex*", Warning, "", 0) + self.assertRaises(UserWarning, self.module.warn, 'hex/oct') + text = 'nonmatching text' + self.module.warn(text) + self.assertEqual(str(w[-1].message), text) + self.assertIs(w[-1].category, UserWarning) + + def test_message_matching(self): + with original_warnings.catch_warnings(record=True, + module=self.module) as w: + self.module.simplefilter("ignore", UserWarning) + self.module.filterwarnings("error", "match", UserWarning) + self.assertRaises(UserWarning, self.module.warn, "match") + self.assertRaises(UserWarning, self.module.warn, "match prefix") + self.module.warn("suffix match") + self.assertEqual(w, []) + self.module.warn("something completely different") + self.assertEqual(w, []) + + def test_mutate_filter_list(self): + class X: + def match(self, a): + L[:] = [] + + L = [("default",X(),UserWarning,X(),0) for i in range(2)] + with original_warnings.catch_warnings(record=True, + module=self.module) as w: + self.module.filters = L + self.module.warn_explicit(UserWarning("b"), None, "f.py", 42) + self.assertEqual(str(w[-1].message), "b") + + def test_filterwarnings_duplicate_filters(self): + with original_warnings.catch_warnings(module=self.module): + self.module.resetwarnings() + self.module.filterwarnings("error", category=UserWarning) + self.assertEqual(len(self.module.filters), 1) + self.module.filterwarnings("ignore", category=UserWarning) + self.module.filterwarnings("error", category=UserWarning) + self.assertEqual( + len(self.module.filters), 2, + "filterwarnings inserted duplicate filter" + ) + self.assertEqual( + self.module.filters[0][0], "error", + "filterwarnings did not promote filter to " + "the beginning of list" + ) + + def test_simplefilter_duplicate_filters(self): + with original_warnings.catch_warnings(module=self.module): + self.module.resetwarnings() + self.module.simplefilter("error", category=UserWarning) + self.assertEqual(len(self.module.filters), 1) + self.module.simplefilter("ignore", category=UserWarning) + self.module.simplefilter("error", category=UserWarning) + self.assertEqual( + len(self.module.filters), 2, + "simplefilter inserted duplicate filter" + ) + self.assertEqual( + self.module.filters[0][0], "error", + "simplefilter did not promote filter to the beginning of list" + ) + + def test_append_duplicate(self): + with original_warnings.catch_warnings(module=self.module, + record=True) as w: + self.module.resetwarnings() + self.module.simplefilter("ignore") + self.module.simplefilter("error", append=True) + self.module.simplefilter("ignore", append=True) + self.module.warn("test_append_duplicate", category=UserWarning) + self.assertEqual(len(self.module.filters), 2, + "simplefilter inserted duplicate filter" + ) + self.assertEqual(len(w), 0, + "appended duplicate changed order of filters" + ) + + def test_argument_validation(self): + with self.assertRaises(ValueError): + self.module.filterwarnings(action='foo') + with self.assertRaises(TypeError): + self.module.filterwarnings('ignore', message=0) + with self.assertRaises(TypeError): + self.module.filterwarnings('ignore', category=0) + with self.assertRaises(TypeError): + self.module.filterwarnings('ignore', category=int) + with self.assertRaises(TypeError): + self.module.filterwarnings('ignore', module=0) + with self.assertRaises(TypeError): + self.module.filterwarnings('ignore', lineno=int) + with self.assertRaises(ValueError): + self.module.filterwarnings('ignore', lineno=-1) + with self.assertRaises(ValueError): + self.module.simplefilter(action='foo') + with self.assertRaises(TypeError): + self.module.simplefilter('ignore', lineno=int) + with self.assertRaises(ValueError): + self.module.simplefilter('ignore', lineno=-1) + + def test_catchwarnings_with_simplefilter_ignore(self): + with original_warnings.catch_warnings(module=self.module): + self.module.resetwarnings() + self.module.simplefilter("error") + with self.module.catch_warnings( + module=self.module, action="ignore" + ): + self.module.warn("This will be ignored") + + def test_catchwarnings_with_simplefilter_error(self): + with original_warnings.catch_warnings(module=self.module): + self.module.resetwarnings() + with self.module.catch_warnings( + module=self.module, action="error", category=FutureWarning + ): + with support.captured_stderr() as stderr: + error_msg = "Other types of warnings are not errors" + self.module.warn(error_msg) + self.assertRaises(FutureWarning, + self.module.warn, FutureWarning("msg")) + stderr = stderr.getvalue() + self.assertIn(error_msg, stderr) + +class CFilterTests(FilterTests, unittest.TestCase): + module = c_warnings + +class PyFilterTests(FilterTests, unittest.TestCase): + module = py_warnings + + +class WarnTests(BaseTest): + + """Test warnings.warn() and warnings.warn_explicit().""" + + def test_message(self): + with original_warnings.catch_warnings(record=True, + module=self.module) as w: + self.module.simplefilter("once") + for i in range(4): + text = 'multi %d' %i # Different text on each call. + self.module.warn(text) + self.assertEqual(str(w[-1].message), text) + self.assertIs(w[-1].category, UserWarning) + + # Issue 3639 + def test_warn_nonstandard_types(self): + # warn() should handle non-standard types without issue. + for ob in (Warning, None, 42): + with original_warnings.catch_warnings(record=True, + module=self.module) as w: + self.module.simplefilter("once") + self.module.warn(ob) + # Don't directly compare objects since + # ``Warning() != Warning()``. + self.assertEqual(str(w[-1].message), str(UserWarning(ob))) + + def test_filename(self): + with warnings_state(self.module): + with original_warnings.catch_warnings(record=True, + module=self.module) as w: + warning_tests.inner("spam1") + self.assertEqual(os.path.basename(w[-1].filename), + "stacklevel.py") + warning_tests.outer("spam2") + self.assertEqual(os.path.basename(w[-1].filename), + "stacklevel.py") + + def test_stacklevel(self): + # Test stacklevel argument + # make sure all messages are different, so the warning won't be skipped + with warnings_state(self.module): + with original_warnings.catch_warnings(record=True, + module=self.module) as w: + warning_tests.inner("spam3", stacklevel=1) + self.assertEqual(os.path.basename(w[-1].filename), + "stacklevel.py") + warning_tests.outer("spam4", stacklevel=1) + self.assertEqual(os.path.basename(w[-1].filename), + "stacklevel.py") + + warning_tests.inner("spam5", stacklevel=2) + self.assertEqual(os.path.basename(w[-1].filename), + "__init__.py") + warning_tests.outer("spam6", stacklevel=2) + self.assertEqual(os.path.basename(w[-1].filename), + "stacklevel.py") + warning_tests.outer("spam6.5", stacklevel=3) + self.assertEqual(os.path.basename(w[-1].filename), + "__init__.py") + + warning_tests.inner("spam7", stacklevel=9999) + self.assertEqual(os.path.basename(w[-1].filename), + "<sys>") + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_stacklevel_import(self): + # Issue #24305: With stacklevel=2, module-level warnings should work. + import_helper.unload('test.test_warnings.data.import_warning') + with warnings_state(self.module): + with original_warnings.catch_warnings(record=True, + module=self.module) as w: + self.module.simplefilter('always') + import test.test_warnings.data.import_warning + self.assertEqual(len(w), 1) + self.assertEqual(w[0].filename, __file__) + + def test_skip_file_prefixes(self): + with warnings_state(self.module): + with original_warnings.catch_warnings(record=True, + module=self.module) as w: + self.module.simplefilter('always') + + # Warning never attributed to the data/ package. + package_helper.inner_api( + "inner_api", stacklevel=2, + warnings_module=warning_tests.warnings) + self.assertEqual(w[-1].filename, __file__) + warning_tests.package("package api", stacklevel=2) + self.assertEqual(w[-1].filename, __file__) + self.assertEqual(w[-2].filename, w[-1].filename) + # Low stacklevels are overridden to 2 behavior. + warning_tests.package("package api 1", stacklevel=1) + self.assertEqual(w[-1].filename, __file__) + warning_tests.package("package api 0", stacklevel=0) + self.assertEqual(w[-1].filename, __file__) + warning_tests.package("package api -99", stacklevel=-99) + self.assertEqual(w[-1].filename, __file__) + + # The stacklevel still goes up out of the package. + warning_tests.package("prefix02", stacklevel=3) + self.assertIn("unittest", w[-1].filename) + + def test_skip_file_prefixes_type_errors(self): + with warnings_state(self.module): + warn = warning_tests.warnings.warn + with self.assertRaises(TypeError): + warn("msg", skip_file_prefixes=[]) + with self.assertRaises(TypeError): + warn("msg", skip_file_prefixes=(b"bytes",)) + with self.assertRaises(TypeError): + warn("msg", skip_file_prefixes="a sequence of strs") + + def test_exec_filename(self): + filename = "<warnings-test>" + codeobj = compile(("import warnings\n" + "warnings.warn('hello', UserWarning)"), + filename, "exec") + with original_warnings.catch_warnings(record=True) as w: + self.module.simplefilter("always", category=UserWarning) + exec(codeobj) + self.assertEqual(w[0].filename, filename) + + def test_warn_explicit_non_ascii_filename(self): + with original_warnings.catch_warnings(record=True, + module=self.module) as w: + self.module.resetwarnings() + self.module.filterwarnings("always", category=UserWarning) + filenames = ["nonascii\xe9\u20ac"] + if not support.is_emscripten: + # JavaScript does not like surrogates. + # Invalid UTF-8 leading byte 0x80 encountered when + # deserializing a UTF-8 string in wasm memory to a JS + # string! + filenames.append("surrogate\udc80") + for filename in filenames: + try: + os.fsencode(filename) + except UnicodeEncodeError: + continue + self.module.warn_explicit("text", UserWarning, filename, 1) + self.assertEqual(w[-1].filename, filename) + + def test_warn_explicit_type_errors(self): + # warn_explicit() should error out gracefully if it is given objects + # of the wrong types. + # lineno is expected to be an integer. + self.assertRaises(TypeError, self.module.warn_explicit, + None, UserWarning, None, None) + # Either 'message' needs to be an instance of Warning or 'category' + # needs to be a subclass. + self.assertRaises(TypeError, self.module.warn_explicit, + None, None, None, 1) + # 'registry' must be a dict or None. + self.assertRaises((TypeError, AttributeError), + self.module.warn_explicit, + None, Warning, None, 1, registry=42) + + def test_bad_str(self): + # issue 6415 + # Warnings instance with a bad format string for __str__ should not + # trigger a bus error. + class BadStrWarning(Warning): + """Warning with a bad format string for __str__.""" + def __str__(self): + return ("A bad formatted string %(err)" % + {"err" : "there is no %(err)s"}) + + with self.assertRaises(ValueError): + self.module.warn(BadStrWarning()) + + def test_warning_classes(self): + class MyWarningClass(Warning): + pass + + class NonWarningSubclass: + pass + + # passing a non-subclass of Warning should raise a TypeError + with self.assertRaises(TypeError) as cm: + self.module.warn('bad warning category', '') + self.assertIn('category must be a Warning subclass, not ', + str(cm.exception)) + + with self.assertRaises(TypeError) as cm: + self.module.warn('bad warning category', NonWarningSubclass) + self.assertIn('category must be a Warning subclass, not ', + str(cm.exception)) + + # check that warning instances also raise a TypeError + with self.assertRaises(TypeError) as cm: + self.module.warn('bad warning category', MyWarningClass()) + self.assertIn('category must be a Warning subclass, not ', + str(cm.exception)) + + with original_warnings.catch_warnings(module=self.module): + self.module.resetwarnings() + self.module.filterwarnings('default') + with self.assertWarns(MyWarningClass) as cm: + self.module.warn('good warning category', MyWarningClass) + self.assertEqual('good warning category', str(cm.warning)) + + with self.assertWarns(UserWarning) as cm: + self.module.warn('good warning category', None) + self.assertEqual('good warning category', str(cm.warning)) + + with self.assertWarns(MyWarningClass) as cm: + self.module.warn('good warning category', MyWarningClass) + self.assertIsInstance(cm.warning, Warning) + + def check_module_globals(self, module_globals): + with original_warnings.catch_warnings(module=self.module, record=True) as w: + self.module.filterwarnings('default') + self.module.warn_explicit( + 'eggs', UserWarning, 'bar', 1, + module_globals=module_globals) + self.assertEqual(len(w), 1) + self.assertEqual(w[0].category, UserWarning) + self.assertEqual(str(w[0].message), 'eggs') + + def check_module_globals_error(self, module_globals, errmsg, errtype=ValueError): + if self.module is py_warnings: + self.check_module_globals(module_globals) + return + with original_warnings.catch_warnings(module=self.module, record=True) as w: + self.module.filterwarnings('always') + with self.assertRaisesRegex(errtype, re.escape(errmsg)): + self.module.warn_explicit( + 'eggs', UserWarning, 'bar', 1, + module_globals=module_globals) + self.assertEqual(len(w), 0) + + def check_module_globals_deprecated(self, module_globals, msg): + if self.module is py_warnings: + self.check_module_globals(module_globals) + return + with original_warnings.catch_warnings(module=self.module, record=True) as w: + self.module.filterwarnings('always') + self.module.warn_explicit( + 'eggs', UserWarning, 'bar', 1, + module_globals=module_globals) + self.assertEqual(len(w), 2) + self.assertEqual(w[0].category, DeprecationWarning) + self.assertEqual(str(w[0].message), msg) + self.assertEqual(w[1].category, UserWarning) + self.assertEqual(str(w[1].message), 'eggs') + + def test_gh86298_no_loader_and_no_spec(self): + self.check_module_globals({'__name__': 'bar'}) + + def test_gh86298_loader_is_none_and_no_spec(self): + self.check_module_globals({'__name__': 'bar', '__loader__': None}) + + def test_gh86298_no_loader_and_spec_is_none(self): + self.check_module_globals_error( + {'__name__': 'bar', '__spec__': None}, + 'Module globals is missing a __spec__.loader') + + def test_gh86298_loader_is_none_and_spec_is_none(self): + self.check_module_globals_error( + {'__name__': 'bar', '__loader__': None, '__spec__': None}, + 'Module globals is missing a __spec__.loader') + + def test_gh86298_loader_is_none_and_spec_loader_is_none(self): + self.check_module_globals_error( + {'__name__': 'bar', '__loader__': None, + '__spec__': types.SimpleNamespace(loader=None)}, + 'Module globals is missing a __spec__.loader') + + def test_gh86298_no_spec(self): + self.check_module_globals_deprecated( + {'__name__': 'bar', '__loader__': object()}, + 'Module globals is missing a __spec__.loader') + + def test_gh86298_spec_is_none(self): + self.check_module_globals_deprecated( + {'__name__': 'bar', '__loader__': object(), '__spec__': None}, + 'Module globals is missing a __spec__.loader') + + def test_gh86298_no_spec_loader(self): + self.check_module_globals_deprecated( + {'__name__': 'bar', '__loader__': object(), + '__spec__': types.SimpleNamespace()}, + 'Module globals is missing a __spec__.loader') + + def test_gh86298_loader_and_spec_loader_disagree(self): + self.check_module_globals_deprecated( + {'__name__': 'bar', '__loader__': object(), + '__spec__': types.SimpleNamespace(loader=object())}, + 'Module globals; __loader__ != __spec__.loader') + + def test_gh86298_no_loader_and_no_spec_loader(self): + self.check_module_globals_error( + {'__name__': 'bar', '__spec__': types.SimpleNamespace()}, + 'Module globals is missing a __spec__.loader', AttributeError) + + def test_gh86298_no_loader_with_spec_loader_okay(self): + self.check_module_globals( + {'__name__': 'bar', + '__spec__': types.SimpleNamespace(loader=object())}) + +class CWarnTests(WarnTests, unittest.TestCase): + module = c_warnings + + # TODO: RUSTPYTHON + @unittest.expectedFailure + # As an early adopter, we sanity check the + # test.import_helper.import_fresh_module utility function + def test_accelerated(self): + self.assertIsNot(original_warnings, self.module) + self.assertFalse(hasattr(self.module.warn, '__code__')) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_gh86298_no_loader_and_spec_is_none(self): + return super().test_gh86298_no_loader_and_spec_is_none() + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_gh86298_loader_is_none_and_spec_is_none(self): + return super().test_gh86298_loader_is_none_and_spec_is_none() + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_gh86298_loader_is_none_and_spec_loader_is_none(self): + return super().test_gh86298_loader_is_none_and_spec_loader_is_none() + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_gh86298_no_spec(self): + return super().test_gh86298_no_spec() + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_gh86298_spec_is_none(self): + return super().test_gh86298_spec_is_none() + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_gh86298_no_spec_loader(self): + return super().test_gh86298_no_spec_loader() + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_gh86298_loader_and_spec_loader_disagree(self): + return super().test_gh86298_loader_and_spec_loader_disagree() + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_gh86298_no_loader_and_no_spec_loader(self): + return super().test_gh86298_no_loader_and_no_spec_loader() + +class PyWarnTests(WarnTests, unittest.TestCase): + module = py_warnings + + # As an early adopter, we sanity check the + # test.import_helper.import_fresh_module utility function + def test_pure_python(self): + self.assertIsNot(original_warnings, self.module) + self.assertTrue(hasattr(self.module.warn, '__code__')) + + +class WCmdLineTests(BaseTest): + + def test_improper_input(self): + # Uses the private _setoption() function to test the parsing + # of command-line warning arguments + with original_warnings.catch_warnings(module=self.module): + self.assertRaises(self.module._OptionError, + self.module._setoption, '1:2:3:4:5:6') + self.assertRaises(self.module._OptionError, + self.module._setoption, 'bogus::Warning') + self.assertRaises(self.module._OptionError, + self.module._setoption, 'ignore:2::4:-5') + with self.assertRaises(self.module._OptionError): + self.module._setoption('ignore::123') + with self.assertRaises(self.module._OptionError): + self.module._setoption('ignore::123abc') + with self.assertRaises(self.module._OptionError): + self.module._setoption('ignore::===') + with self.assertRaisesRegex(self.module._OptionError, 'Wärning'): + self.module._setoption('ignore::Wärning') + self.module._setoption('error::Warning::0') + self.assertRaises(UserWarning, self.module.warn, 'convert to error') + + def test_import_from_module(self): + with original_warnings.catch_warnings(module=self.module): + self.module._setoption('ignore::Warning') + with self.assertRaises(self.module._OptionError): + self.module._setoption('ignore::TestWarning') + with self.assertRaises(self.module._OptionError): + self.module._setoption('ignore::test.test_warnings.bogus') + self.module._setoption('error::test.test_warnings.TestWarning') + with self.assertRaises(TestWarning): + self.module.warn('test warning', TestWarning) + + +class CWCmdLineTests(WCmdLineTests, unittest.TestCase): + module = c_warnings + + +class PyWCmdLineTests(WCmdLineTests, unittest.TestCase): + module = py_warnings + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_improper_option(self): + # Same as above, but check that the message is printed out when + # the interpreter is executed. This also checks that options are + # actually parsed at all. + rc, out, err = assert_python_ok("-Wxxx", "-c", "pass") + self.assertIn(b"Invalid -W option ignored: invalid action: 'xxx'", err) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_warnings_bootstrap(self): + # Check that the warnings module does get loaded when -W<some option> + # is used (see issue #10372 for an example of silent bootstrap failure). + rc, out, err = assert_python_ok("-Wi", "-c", + "import sys; sys.modules['warnings'].warn('foo', RuntimeWarning)") + # '-Wi' was observed + self.assertFalse(out.strip()) + self.assertNotIn(b'RuntimeWarning', err) + + +class _WarningsTests(BaseTest, unittest.TestCase): + + """Tests specific to the _warnings module.""" + + module = c_warnings + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_filter(self): + # Everything should function even if 'filters' is not in warnings. + with original_warnings.catch_warnings(module=self.module) as w: + self.module.filterwarnings("error", "", Warning, "", 0) + self.assertRaises(UserWarning, self.module.warn, + 'convert to error') + del self.module.filters + self.assertRaises(UserWarning, self.module.warn, + 'convert to error') + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_onceregistry(self): + # Replacing or removing the onceregistry should be okay. + global __warningregistry__ + message = UserWarning('onceregistry test') + try: + original_registry = self.module.onceregistry + __warningregistry__ = {} + with original_warnings.catch_warnings(record=True, + module=self.module) as w: + self.module.resetwarnings() + self.module.filterwarnings("once", category=UserWarning) + self.module.warn_explicit(message, UserWarning, "file", 42) + self.assertEqual(w[-1].message, message) + del w[:] + self.module.warn_explicit(message, UserWarning, "file", 42) + self.assertEqual(len(w), 0) + # Test the resetting of onceregistry. + self.module.onceregistry = {} + __warningregistry__ = {} + self.module.warn('onceregistry test') + self.assertEqual(w[-1].message.args, message.args) + # Removal of onceregistry is okay. + del w[:] + del self.module.onceregistry + __warningregistry__ = {} + self.module.warn_explicit(message, UserWarning, "file", 42) + self.assertEqual(len(w), 0) + finally: + self.module.onceregistry = original_registry + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_default_action(self): + # Replacing or removing defaultaction should be okay. + message = UserWarning("defaultaction test") + original = self.module.defaultaction + try: + with original_warnings.catch_warnings(record=True, + module=self.module) as w: + self.module.resetwarnings() + registry = {} + self.module.warn_explicit(message, UserWarning, "<test>", 42, + registry=registry) + self.assertEqual(w[-1].message, message) + self.assertEqual(len(w), 1) + # One actual registry key plus the "version" key + self.assertEqual(len(registry), 2) + self.assertIn("version", registry) + del w[:] + # Test removal. + del self.module.defaultaction + __warningregistry__ = {} + registry = {} + self.module.warn_explicit(message, UserWarning, "<test>", 43, + registry=registry) + self.assertEqual(w[-1].message, message) + self.assertEqual(len(w), 1) + self.assertEqual(len(registry), 2) + del w[:] + # Test setting. + self.module.defaultaction = "ignore" + __warningregistry__ = {} + registry = {} + self.module.warn_explicit(message, UserWarning, "<test>", 44, + registry=registry) + self.assertEqual(len(w), 0) + finally: + self.module.defaultaction = original + + def test_showwarning_missing(self): + # Test that showwarning() missing is okay. + text = 'del showwarning test' + with original_warnings.catch_warnings(module=self.module): + self.module.filterwarnings("always", category=UserWarning) + del self.module.showwarning + with support.captured_output('stderr') as stream: + self.module.warn(text) + result = stream.getvalue() + self.assertIn(text, result) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_showwarnmsg_missing(self): + # Test that _showwarnmsg() missing is okay. + text = 'del _showwarnmsg test' + with original_warnings.catch_warnings(module=self.module): + self.module.filterwarnings("always", category=UserWarning) + + show = self.module._showwarnmsg + try: + del self.module._showwarnmsg + with support.captured_output('stderr') as stream: + self.module.warn(text) + result = stream.getvalue() + finally: + self.module._showwarnmsg = show + self.assertIn(text, result) + + def test_showwarning_not_callable(self): + with original_warnings.catch_warnings(module=self.module): + self.module.filterwarnings("always", category=UserWarning) + self.module.showwarning = print + with support.captured_output('stdout'): + self.module.warn('Warning!') + self.module.showwarning = 23 + self.assertRaises(TypeError, self.module.warn, "Warning!") + + def test_show_warning_output(self): + # With showwarning() missing, make sure that output is okay. + text = 'test show_warning' + with original_warnings.catch_warnings(module=self.module): + self.module.filterwarnings("always", category=UserWarning) + del self.module.showwarning + with support.captured_output('stderr') as stream: + warning_tests.inner(text) + result = stream.getvalue() + self.assertEqual(result.count('\n'), 2, + "Too many newlines in %r" % result) + first_line, second_line = result.split('\n', 1) + expected_file = os.path.splitext(warning_tests.__file__)[0] + '.py' + first_line_parts = first_line.rsplit(':', 3) + path, line, warning_class, message = first_line_parts + line = int(line) + self.assertEqual(expected_file, path) + self.assertEqual(warning_class, ' ' + UserWarning.__name__) + self.assertEqual(message, ' ' + text) + expected_line = ' ' + linecache.getline(path, line).strip() + '\n' + assert expected_line + self.assertEqual(second_line, expected_line) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_filename_none(self): + # issue #12467: race condition if a warning is emitted at shutdown + globals_dict = globals() + oldfile = globals_dict['__file__'] + try: + catch = original_warnings.catch_warnings(record=True, + module=self.module) + with catch as w: + self.module.filterwarnings("always", category=UserWarning) + globals_dict['__file__'] = None + original_warnings.warn('test', UserWarning) + self.assertTrue(len(w)) + finally: + globals_dict['__file__'] = oldfile + + def test_stderr_none(self): + rc, stdout, stderr = assert_python_ok("-c", + "import sys; sys.stderr = None; " + "import warnings; warnings.simplefilter('always'); " + "warnings.warn('Warning!')") + self.assertEqual(stdout, b'') + self.assertNotIn(b'Warning!', stderr) + self.assertNotIn(b'Error', stderr) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_issue31285(self): + # warn_explicit() should neither raise a SystemError nor cause an + # assertion failure, in case the return value of get_source() has a + # bad splitlines() method. + get_source_called = [] + def get_module_globals(*, splitlines_ret_val): + class BadSource(str): + def splitlines(self): + return splitlines_ret_val + + class BadLoader: + def get_source(self, fullname): + get_source_called.append(splitlines_ret_val) + return BadSource('spam') + + loader = BadLoader() + spec = importlib.machinery.ModuleSpec('foobar', loader) + return {'__loader__': loader, + '__spec__': spec, + '__name__': 'foobar'} + + + wmod = self.module + with original_warnings.catch_warnings(module=wmod): + wmod.filterwarnings('default', category=UserWarning) + + linecache.clearcache() + with support.captured_stderr() as stderr: + wmod.warn_explicit( + 'foo', UserWarning, 'bar', 1, + module_globals=get_module_globals(splitlines_ret_val=42)) + self.assertIn('UserWarning: foo', stderr.getvalue()) + self.assertEqual(get_source_called, [42]) + + linecache.clearcache() + with support.swap_attr(wmod, '_showwarnmsg', None): + del wmod._showwarnmsg + with support.captured_stderr() as stderr: + wmod.warn_explicit( + 'eggs', UserWarning, 'bar', 1, + module_globals=get_module_globals(splitlines_ret_val=[42])) + self.assertIn('UserWarning: eggs', stderr.getvalue()) + self.assertEqual(get_source_called, [42, [42]]) + linecache.clearcache() + + @support.cpython_only + def test_issue31411(self): + # warn_explicit() shouldn't raise a SystemError in case + # warnings.onceregistry isn't a dictionary. + wmod = self.module + with original_warnings.catch_warnings(module=wmod): + wmod.filterwarnings('once') + with support.swap_attr(wmod, 'onceregistry', None): + with self.assertRaises(TypeError): + wmod.warn_explicit('foo', Warning, 'bar', 1, registry=None) + + @support.cpython_only + def test_issue31416(self): + # warn_explicit() shouldn't cause an assertion failure in case of a + # bad warnings.filters or warnings.defaultaction. + wmod = self.module + with original_warnings.catch_warnings(module=wmod): + wmod.filters = [(None, None, Warning, None, 0)] + with self.assertRaises(TypeError): + wmod.warn_explicit('foo', Warning, 'bar', 1) + + wmod.filters = [] + with support.swap_attr(wmod, 'defaultaction', None), \ + self.assertRaises(TypeError): + wmod.warn_explicit('foo', Warning, 'bar', 1) + + @support.cpython_only + def test_issue31566(self): + # warn() shouldn't cause an assertion failure in case of a bad + # __name__ global. + with original_warnings.catch_warnings(module=self.module): + self.module.filterwarnings('error', category=UserWarning) + with support.swap_item(globals(), '__name__', b'foo'), \ + support.swap_item(globals(), '__file__', None): + self.assertRaises(UserWarning, self.module.warn, 'bar') + + +class WarningsDisplayTests(BaseTest): + + """Test the displaying of warnings and the ability to overload functions + related to displaying warnings.""" + + def test_formatwarning(self): + message = "msg" + category = Warning + file_name = os.path.splitext(warning_tests.__file__)[0] + '.py' + line_num = 5 + file_line = linecache.getline(file_name, line_num).strip() + format = "%s:%s: %s: %s\n %s\n" + expect = format % (file_name, line_num, category.__name__, message, + file_line) + self.assertEqual(expect, self.module.formatwarning(message, + category, file_name, line_num)) + # Test the 'line' argument. + file_line += " for the win!" + expect = format % (file_name, line_num, category.__name__, message, + file_line) + self.assertEqual(expect, self.module.formatwarning(message, + category, file_name, line_num, file_line)) + + def test_showwarning(self): + file_name = os.path.splitext(warning_tests.__file__)[0] + '.py' + line_num = 3 + expected_file_line = linecache.getline(file_name, line_num).strip() + message = 'msg' + category = Warning + file_object = StringIO() + expect = self.module.formatwarning(message, category, file_name, + line_num) + self.module.showwarning(message, category, file_name, line_num, + file_object) + self.assertEqual(file_object.getvalue(), expect) + # Test 'line' argument. + expected_file_line += "for the win!" + expect = self.module.formatwarning(message, category, file_name, + line_num, expected_file_line) + file_object = StringIO() + self.module.showwarning(message, category, file_name, line_num, + file_object, expected_file_line) + self.assertEqual(expect, file_object.getvalue()) + + def test_formatwarning_override(self): + # bpo-35178: Test that a custom formatwarning function gets the 'line' + # argument as a positional argument, and not only as a keyword argument + def myformatwarning(message, category, filename, lineno, text): + return f'm={message}:c={category}:f={filename}:l={lineno}:t={text}' + + file_name = os.path.splitext(warning_tests.__file__)[0] + '.py' + line_num = 3 + file_line = linecache.getline(file_name, line_num).strip() + message = 'msg' + category = Warning + file_object = StringIO() + expected = f'm={message}:c={category}:f={file_name}:l={line_num}' + \ + f':t={file_line}' + with support.swap_attr(self.module, 'formatwarning', myformatwarning): + self.module.showwarning(message, category, file_name, line_num, + file_object, file_line) + self.assertEqual(file_object.getvalue(), expected) + + +class CWarningsDisplayTests(WarningsDisplayTests, unittest.TestCase): + module = c_warnings + +class PyWarningsDisplayTests(WarningsDisplayTests, unittest.TestCase): + module = py_warnings + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_tracemalloc(self): + self.addCleanup(os_helper.unlink, os_helper.TESTFN) + + with open(os_helper.TESTFN, 'w', encoding="utf-8") as fp: + fp.write(textwrap.dedent(""" + def func(): + f = open(__file__, "rb") + # Emit ResourceWarning + f = None + + func() + """)) + + def run(*args): + res = assert_python_ok(*args, PYTHONIOENCODING='utf-8') + stderr = res.err.decode('utf-8', 'replace') + stderr = '\n'.join(stderr.splitlines()) + + # normalize newlines + stderr = re.sub('<.*>', '<...>', stderr) + return stderr + + # tracemalloc disabled + filename = os.path.abspath(os_helper.TESTFN) + stderr = run('-Wd', os_helper.TESTFN) + expected = textwrap.dedent(f''' + {filename}:5: ResourceWarning: unclosed file <...> + f = None + ResourceWarning: Enable tracemalloc to get the object allocation traceback + ''').strip() + self.assertEqual(stderr, expected) + + # tracemalloc enabled + stderr = run('-Wd', '-X', 'tracemalloc=2', os_helper.TESTFN) + expected = textwrap.dedent(f''' + {filename}:5: ResourceWarning: unclosed file <...> + f = None + Object allocated at (most recent call last): + File "{filename}", lineno 7 + func() + File "{filename}", lineno 3 + f = open(__file__, "rb") + ''').strip() + self.assertEqual(stderr, expected) + + +class CatchWarningTests(BaseTest): + + """Test catch_warnings().""" + + def test_catch_warnings_restore(self): + wmod = self.module + orig_filters = wmod.filters + orig_showwarning = wmod.showwarning + # Ensure both showwarning and filters are restored when recording + with wmod.catch_warnings(module=wmod, record=True): + wmod.filters = wmod.showwarning = object() + self.assertIs(wmod.filters, orig_filters) + self.assertIs(wmod.showwarning, orig_showwarning) + # Same test, but with recording disabled + with wmod.catch_warnings(module=wmod, record=False): + wmod.filters = wmod.showwarning = object() + self.assertIs(wmod.filters, orig_filters) + self.assertIs(wmod.showwarning, orig_showwarning) + + def test_catch_warnings_recording(self): + wmod = self.module + # Ensure warnings are recorded when requested + with wmod.catch_warnings(module=wmod, record=True) as w: + self.assertEqual(w, []) + self.assertIs(type(w), list) + wmod.simplefilter("always") + wmod.warn("foo") + self.assertEqual(str(w[-1].message), "foo") + wmod.warn("bar") + self.assertEqual(str(w[-1].message), "bar") + self.assertEqual(str(w[0].message), "foo") + self.assertEqual(str(w[1].message), "bar") + del w[:] + self.assertEqual(w, []) + # Ensure warnings are not recorded when not requested + orig_showwarning = wmod.showwarning + with wmod.catch_warnings(module=wmod, record=False) as w: + self.assertIsNone(w) + self.assertIs(wmod.showwarning, orig_showwarning) + + def test_catch_warnings_reentry_guard(self): + wmod = self.module + # Ensure catch_warnings is protected against incorrect usage + x = wmod.catch_warnings(module=wmod, record=True) + self.assertRaises(RuntimeError, x.__exit__) + with x: + self.assertRaises(RuntimeError, x.__enter__) + # Same test, but with recording disabled + x = wmod.catch_warnings(module=wmod, record=False) + self.assertRaises(RuntimeError, x.__exit__) + with x: + self.assertRaises(RuntimeError, x.__enter__) + + def test_catch_warnings_defaults(self): + wmod = self.module + orig_filters = wmod.filters + orig_showwarning = wmod.showwarning + # Ensure default behaviour is not to record warnings + with wmod.catch_warnings(module=wmod) as w: + self.assertIsNone(w) + self.assertIs(wmod.showwarning, orig_showwarning) + self.assertIsNot(wmod.filters, orig_filters) + self.assertIs(wmod.filters, orig_filters) + if wmod is sys.modules['warnings']: + # Ensure the default module is this one + with wmod.catch_warnings() as w: + self.assertIsNone(w) + self.assertIs(wmod.showwarning, orig_showwarning) + self.assertIsNot(wmod.filters, orig_filters) + self.assertIs(wmod.filters, orig_filters) + + def test_record_override_showwarning_before(self): + # Issue #28835: If warnings.showwarning() was overridden, make sure + # that catch_warnings(record=True) overrides it again. + text = "This is a warning" + wmod = self.module + my_log = [] + + def my_logger(message, category, filename, lineno, file=None, line=None): + nonlocal my_log + my_log.append(message) + + # Override warnings.showwarning() before calling catch_warnings() + with support.swap_attr(wmod, 'showwarning', my_logger): + with wmod.catch_warnings(module=wmod, record=True) as log: + self.assertIsNot(wmod.showwarning, my_logger) + + wmod.simplefilter("always") + wmod.warn(text) + + self.assertIs(wmod.showwarning, my_logger) + + self.assertEqual(len(log), 1, log) + self.assertEqual(log[0].message.args[0], text) + self.assertEqual(my_log, []) + + def test_record_override_showwarning_inside(self): + # Issue #28835: It is possible to override warnings.showwarning() + # in the catch_warnings(record=True) context manager. + text = "This is a warning" + wmod = self.module + my_log = [] + + def my_logger(message, category, filename, lineno, file=None, line=None): + nonlocal my_log + my_log.append(message) + + with wmod.catch_warnings(module=wmod, record=True) as log: + wmod.simplefilter("always") + wmod.showwarning = my_logger + wmod.warn(text) + + self.assertEqual(len(my_log), 1, my_log) + self.assertEqual(my_log[0].args[0], text) + self.assertEqual(log, []) + + def test_check_warnings(self): + # Explicit tests for the test.support convenience wrapper + wmod = self.module + if wmod is not sys.modules['warnings']: + self.skipTest('module to test is not loaded warnings module') + with warnings_helper.check_warnings(quiet=False) as w: + self.assertEqual(w.warnings, []) + wmod.simplefilter("always") + wmod.warn("foo") + self.assertEqual(str(w.message), "foo") + wmod.warn("bar") + self.assertEqual(str(w.message), "bar") + self.assertEqual(str(w.warnings[0].message), "foo") + self.assertEqual(str(w.warnings[1].message), "bar") + w.reset() + self.assertEqual(w.warnings, []) + + with warnings_helper.check_warnings(): + # defaults to quiet=True without argument + pass + with warnings_helper.check_warnings(('foo', UserWarning)): + wmod.warn("foo") + + with self.assertRaises(AssertionError): + with warnings_helper.check_warnings(('', RuntimeWarning)): + # defaults to quiet=False with argument + pass + with self.assertRaises(AssertionError): + with warnings_helper.check_warnings(('foo', RuntimeWarning)): + wmod.warn("foo") + +class CCatchWarningTests(CatchWarningTests, unittest.TestCase): + module = c_warnings + +class PyCatchWarningTests(CatchWarningTests, unittest.TestCase): + module = py_warnings + + +class EnvironmentVariableTests(BaseTest): + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_single_warning(self): + rc, stdout, stderr = assert_python_ok("-c", + "import sys; sys.stdout.write(str(sys.warnoptions))", + PYTHONWARNINGS="ignore::DeprecationWarning", + PYTHONDEVMODE="") + self.assertEqual(stdout, b"['ignore::DeprecationWarning']") + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_comma_separated_warnings(self): + rc, stdout, stderr = assert_python_ok("-c", + "import sys; sys.stdout.write(str(sys.warnoptions))", + PYTHONWARNINGS="ignore::DeprecationWarning,ignore::UnicodeWarning", + PYTHONDEVMODE="") + self.assertEqual(stdout, + b"['ignore::DeprecationWarning', 'ignore::UnicodeWarning']") + + # TODO: RUSTPYTHON + @unittest.expectedFailure + @force_not_colorized + def test_envvar_and_command_line(self): + rc, stdout, stderr = assert_python_ok("-Wignore::UnicodeWarning", "-c", + "import sys; sys.stdout.write(str(sys.warnoptions))", + PYTHONWARNINGS="ignore::DeprecationWarning", + PYTHONDEVMODE="") + self.assertEqual(stdout, + b"['ignore::DeprecationWarning', 'ignore::UnicodeWarning']") + + # TODO: RUSTPYTHON + @unittest.expectedFailure + @force_not_colorized + def test_conflicting_envvar_and_command_line(self): + rc, stdout, stderr = assert_python_failure("-Werror::DeprecationWarning", "-c", + "import sys, warnings; sys.stdout.write(str(sys.warnoptions)); " + "warnings.warn('Message', DeprecationWarning)", + PYTHONWARNINGS="default::DeprecationWarning", + PYTHONDEVMODE="") + self.assertEqual(stdout, + b"['default::DeprecationWarning', 'error::DeprecationWarning']") + self.assertEqual(stderr.splitlines(), + [b"Traceback (most recent call last):", + b" File \"<string>\", line 1, in <module>", + b' import sys, warnings; sys.stdout.write(str(sys.warnoptions)); warnings.w' + b"arn('Message', DeprecationWarning)", + b' ~~~~~~~~~~' + b'~~~^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^', + b"DeprecationWarning: Message"]) + + def test_default_filter_configuration(self): + pure_python_api = self.module is py_warnings + if support.Py_DEBUG: + expected_default_filters = [] + else: + if pure_python_api: + main_module_filter = re.compile("__main__") + else: + main_module_filter = "__main__" + expected_default_filters = [ + ('default', None, DeprecationWarning, main_module_filter, 0), + ('ignore', None, DeprecationWarning, None, 0), + ('ignore', None, PendingDeprecationWarning, None, 0), + ('ignore', None, ImportWarning, None, 0), + ('ignore', None, ResourceWarning, None, 0), + ] + expected_output = [str(f).encode() for f in expected_default_filters] + + if pure_python_api: + # Disable the warnings acceleration module in the subprocess + code = "import sys; sys.modules.pop('warnings', None); sys.modules['_warnings'] = None; " + else: + code = "" + code += "import warnings; [print(f) for f in warnings.filters]" + + rc, stdout, stderr = assert_python_ok("-c", code, __isolated=True) + stdout_lines = [line.strip() for line in stdout.splitlines()] + self.maxDiff = None + self.assertEqual(stdout_lines, expected_output) + + + # TODO: RUSTPYTHON + @unittest.expectedFailure + @unittest.skipUnless(sys.getfilesystemencoding() != 'ascii', + 'requires non-ascii filesystemencoding') + def test_nonascii(self): + PYTHONWARNINGS="ignore:DeprecationWarning" + os_helper.FS_NONASCII + rc, stdout, stderr = assert_python_ok("-c", + "import sys; sys.stdout.write(str(sys.warnoptions))", + PYTHONIOENCODING="utf-8", + PYTHONWARNINGS=PYTHONWARNINGS, + PYTHONDEVMODE="") + self.assertEqual(stdout, str([PYTHONWARNINGS]).encode()) + +class CEnvironmentVariableTests(EnvironmentVariableTests, unittest.TestCase): + module = c_warnings + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_default_filter_configuration(self): + # XXX: RUSTPYHTON; remove the entire function when fixed + super().test_default_filter_configuration() + + +class PyEnvironmentVariableTests(EnvironmentVariableTests, unittest.TestCase): + module = py_warnings + + +class _DeprecatedTest(BaseTest, unittest.TestCase): + + """Test _deprecated().""" + + module = original_warnings + + def test_warning(self): + version = (3, 11, 0, "final", 0) + test = [(4, 12), (4, 11), (4, 0), (3, 12)] + for remove in test: + msg = rf".*test_warnings.*{remove[0]}\.{remove[1]}" + filter = msg, DeprecationWarning + with self.subTest(remove=remove): + with warnings_helper.check_warnings(filter, quiet=False): + self.module._deprecated("test_warnings", remove=remove, + _version=version) + + version = (3, 11, 0, "alpha", 0) + msg = r".*test_warnings.*3\.11" + with warnings_helper.check_warnings((msg, DeprecationWarning), quiet=False): + self.module._deprecated("test_warnings", remove=(3, 11), + _version=version) + + def test_RuntimeError(self): + version = (3, 11, 0, "final", 0) + test = [(2, 0), (2, 12), (3, 10)] + for remove in test: + with self.subTest(remove=remove): + with self.assertRaises(RuntimeError): + self.module._deprecated("test_warnings", remove=remove, + _version=version) + for level in ["beta", "candidate", "final"]: + version = (3, 11, 0, level, 0) + with self.subTest(releaselevel=level): + with self.assertRaises(RuntimeError): + self.module._deprecated("test_warnings", remove=(3, 11), + _version=version) + + +class BootstrapTest(unittest.TestCase): + + def test_issue_8766(self): + # "import encodings" emits a warning whereas the warnings is not loaded + # or not completely loaded (warnings imports indirectly encodings by + # importing linecache) yet + with os_helper.temp_cwd() as cwd, os_helper.temp_cwd('encodings'): + # encodings loaded by initfsencoding() + assert_python_ok('-c', 'pass', PYTHONPATH=cwd) + + # Use -W to load warnings module at startup + assert_python_ok('-c', 'pass', '-W', 'always', PYTHONPATH=cwd) + + +class FinalizationTest(unittest.TestCase): + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_finalization(self): + # Issue #19421: warnings.warn() should not crash + # during Python finalization + code = """ +import warnings +warn = warnings.warn + +class A: + def __del__(self): + warn("test") + +a=A() + """ + rc, out, err = assert_python_ok("-c", code) + self.assertEqual(err.decode().rstrip(), + '<string>:7: UserWarning: test') + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_late_resource_warning(self): + # Issue #21925: Emitting a ResourceWarning late during the Python + # shutdown must be logged. + + expected = b"<sys>:0: ResourceWarning: unclosed file " + + # don't import the warnings module + # (_warnings will try to import it) + code = "f = open(%a)" % __file__ + rc, out, err = assert_python_ok("-Wd", "-c", code) + self.assertTrue(err.startswith(expected), ascii(err)) + + # import the warnings module + code = "import warnings; f = open(%a)" % __file__ + rc, out, err = assert_python_ok("-Wd", "-c", code) + self.assertTrue(err.startswith(expected), ascii(err)) + + +class DeprecatedTests(unittest.TestCase): + def test_dunder_deprecated(self): + @deprecated("A will go away soon") + class A: + pass + + self.assertEqual(A.__deprecated__, "A will go away soon") + self.assertIsInstance(A, type) + + @deprecated("b will go away soon") + def b(): + pass + + self.assertEqual(b.__deprecated__, "b will go away soon") + self.assertIsInstance(b, types.FunctionType) + + @overload + @deprecated("no more ints") + def h(x: int) -> int: ... + @overload + def h(x: str) -> str: ... + def h(x): + return x + + overloads = get_overloads(h) + self.assertEqual(len(overloads), 2) + self.assertEqual(overloads[0].__deprecated__, "no more ints") + + def test_class(self): + @deprecated("A will go away soon") + class A: + pass + + with self.assertWarnsRegex(DeprecationWarning, "A will go away soon"): + A() + with self.assertWarnsRegex(DeprecationWarning, "A will go away soon"): + with self.assertRaises(TypeError): + A(42) + + def test_class_with_init(self): + @deprecated("HasInit will go away soon") + class HasInit: + def __init__(self, x): + self.x = x + + with self.assertWarnsRegex(DeprecationWarning, "HasInit will go away soon"): + instance = HasInit(42) + self.assertEqual(instance.x, 42) + + def test_class_with_new(self): + has_new_called = False + + @deprecated("HasNew will go away soon") + class HasNew: + def __new__(cls, x): + nonlocal has_new_called + has_new_called = True + return super().__new__(cls) + + def __init__(self, x) -> None: + self.x = x + + with self.assertWarnsRegex(DeprecationWarning, "HasNew will go away soon"): + instance = HasNew(42) + self.assertEqual(instance.x, 42) + self.assertTrue(has_new_called) + + def test_class_with_inherited_new(self): + new_base_called = False + + class NewBase: + def __new__(cls, x): + nonlocal new_base_called + new_base_called = True + return super().__new__(cls) + + def __init__(self, x) -> None: + self.x = x + + @deprecated("HasInheritedNew will go away soon") + class HasInheritedNew(NewBase): + pass + + with self.assertWarnsRegex(DeprecationWarning, "HasInheritedNew will go away soon"): + instance = HasInheritedNew(42) + self.assertEqual(instance.x, 42) + self.assertTrue(new_base_called) + + def test_class_with_new_but_no_init(self): + new_called = False + + @deprecated("HasNewNoInit will go away soon") + class HasNewNoInit: + def __new__(cls, x): + nonlocal new_called + new_called = True + obj = super().__new__(cls) + obj.x = x + return obj + + with self.assertWarnsRegex(DeprecationWarning, "HasNewNoInit will go away soon"): + instance = HasNewNoInit(42) + self.assertEqual(instance.x, 42) + self.assertTrue(new_called) + + def test_mixin_class(self): + @deprecated("Mixin will go away soon") + class Mixin: + pass + + class Base: + def __init__(self, a) -> None: + self.a = a + + with self.assertWarnsRegex(DeprecationWarning, "Mixin will go away soon"): + class Child(Base, Mixin): + pass + + instance = Child(42) + self.assertEqual(instance.a, 42) + + def test_do_not_shadow_user_arguments(self): + new_called = False + new_called_cls = None + + @deprecated("MyMeta will go away soon") + class MyMeta(type): + def __new__(mcs, name, bases, attrs, cls=None): + nonlocal new_called, new_called_cls + new_called = True + new_called_cls = cls + return super().__new__(mcs, name, bases, attrs) + + with self.assertWarnsRegex(DeprecationWarning, "MyMeta will go away soon"): + class Foo(metaclass=MyMeta, cls='haha'): + pass + + self.assertTrue(new_called) + self.assertEqual(new_called_cls, 'haha') + + def test_existing_init_subclass(self): + @deprecated("C will go away soon") + class C: + def __init_subclass__(cls) -> None: + cls.inited = True + + with self.assertWarnsRegex(DeprecationWarning, "C will go away soon"): + C() + + with self.assertWarnsRegex(DeprecationWarning, "C will go away soon"): + class D(C): + pass + + self.assertTrue(D.inited) + self.assertIsInstance(D(), D) # no deprecation + + def test_existing_init_subclass_in_base(self): + class Base: + def __init_subclass__(cls, x) -> None: + cls.inited = x + + @deprecated("C will go away soon") + class C(Base, x=42): + pass + + self.assertEqual(C.inited, 42) + + with self.assertWarnsRegex(DeprecationWarning, "C will go away soon"): + C() + + with self.assertWarnsRegex(DeprecationWarning, "C will go away soon"): + class D(C, x=3): + pass + + self.assertEqual(D.inited, 3) + + def test_init_subclass_has_correct_cls(self): + init_subclass_saw = None + + @deprecated("Base will go away soon") + class Base: + def __init_subclass__(cls) -> None: + nonlocal init_subclass_saw + init_subclass_saw = cls + + self.assertIsNone(init_subclass_saw) + + with self.assertWarnsRegex(DeprecationWarning, "Base will go away soon"): + class C(Base): + pass + + self.assertIs(init_subclass_saw, C) + + def test_init_subclass_with_explicit_classmethod(self): + init_subclass_saw = None + + @deprecated("Base will go away soon") + class Base: + @classmethod + def __init_subclass__(cls) -> None: + nonlocal init_subclass_saw + init_subclass_saw = cls + + self.assertIsNone(init_subclass_saw) + + with self.assertWarnsRegex(DeprecationWarning, "Base will go away soon"): + class C(Base): + pass + + self.assertIs(init_subclass_saw, C) + + def test_function(self): + @deprecated("b will go away soon") + def b(): + pass + + with self.assertWarnsRegex(DeprecationWarning, "b will go away soon"): + b() + + def test_method(self): + class Capybara: + @deprecated("x will go away soon") + def x(self): + pass + + instance = Capybara() + with self.assertWarnsRegex(DeprecationWarning, "x will go away soon"): + instance.x() + + def test_property(self): + class Capybara: + @property + @deprecated("x will go away soon") + def x(self): + pass + + @property + def no_more_setting(self): + return 42 + + @no_more_setting.setter + @deprecated("no more setting") + def no_more_setting(self, value): + pass + + instance = Capybara() + with self.assertWarnsRegex(DeprecationWarning, "x will go away soon"): + instance.x + + with py_warnings.catch_warnings(): + py_warnings.simplefilter("error") + self.assertEqual(instance.no_more_setting, 42) + + with self.assertWarnsRegex(DeprecationWarning, "no more setting"): + instance.no_more_setting = 42 + + def test_category(self): + @deprecated("c will go away soon", category=RuntimeWarning) + def c(): + pass + + with self.assertWarnsRegex(RuntimeWarning, "c will go away soon"): + c() + + def test_turn_off_warnings(self): + @deprecated("d will go away soon", category=None) + def d(): + pass + + with py_warnings.catch_warnings(): + py_warnings.simplefilter("error") + d() + + def test_only_strings_allowed(self): + with self.assertRaisesRegex( + TypeError, + "Expected an object of type str for 'message', not 'type'" + ): + @deprecated + class Foo: ... + + with self.assertRaisesRegex( + TypeError, + "Expected an object of type str for 'message', not 'function'" + ): + @deprecated + def foo(): ... + + def test_no_retained_references_to_wrapper_instance(self): + @deprecated('depr') + def d(): pass + + self.assertFalse(any( + isinstance(cell.cell_contents, deprecated) for cell in d.__closure__ + )) + + def test_inspect(self): + @deprecated("depr") + def sync(): + pass + + @deprecated("depr") + async def coro(): + pass + + class Cls: + @deprecated("depr") + def sync(self): + pass + + @deprecated("depr") + async def coro(self): + pass + + self.assertFalse(inspect.iscoroutinefunction(sync)) + self.assertTrue(inspect.iscoroutinefunction(coro)) + self.assertFalse(inspect.iscoroutinefunction(Cls.sync)) + self.assertTrue(inspect.iscoroutinefunction(Cls.coro)) + + def test_inspect_class_signature(self): + class Cls1: # no __init__ or __new__ + pass + + class Cls2: # __new__ only + def __new__(cls, x, y): + return super().__new__(cls) + + class Cls3: # __init__ only + def __init__(self, x, y): + pass + + class Cls4: # __new__ and __init__ + def __new__(cls, x, y): + return super().__new__(cls) + + def __init__(self, x, y): + pass + + class Cls5(Cls1): # inherits no __init__ or __new__ + pass + + class Cls6(Cls2): # inherits __new__ only + pass + + class Cls7(Cls3): # inherits __init__ only + pass + + class Cls8(Cls4): # inherits __new__ and __init__ + pass + + # The `@deprecated` decorator will update the class in-place. + # Test the child classes first. + for cls in reversed((Cls1, Cls2, Cls3, Cls4, Cls5, Cls6, Cls7, Cls8)): + with self.subTest(f'class {cls.__name__} signature'): + try: + original_signature = inspect.signature(cls) + except ValueError: + original_signature = None + try: + original_new_signature = inspect.signature(cls.__new__) + except ValueError: + original_new_signature = None + + deprecated_cls = deprecated("depr")(cls) + + try: + deprecated_signature = inspect.signature(deprecated_cls) + except ValueError: + deprecated_signature = None + self.assertEqual(original_signature, deprecated_signature) + + try: + deprecated_new_signature = inspect.signature(deprecated_cls.__new__) + except ValueError: + deprecated_new_signature = None + self.assertEqual(original_new_signature, deprecated_new_signature) + + +def setUpModule(): + py_warnings.onceregistry.clear() + c_warnings.onceregistry.clear() + + +tearDownModule = setUpModule + +if __name__ == "__main__": + unittest.main() diff --git a/Lib/test/test_warnings/__main__.py b/Lib/test/test_warnings/__main__.py new file mode 100644 index 0000000000..44e52ec070 --- /dev/null +++ b/Lib/test/test_warnings/__main__.py @@ -0,0 +1,3 @@ +import unittest + +unittest.main('test.test_warnings') diff --git a/Lib/test/test_warnings/data/import_warning.py b/Lib/test/test_warnings/data/import_warning.py new file mode 100644 index 0000000000..32daec1140 --- /dev/null +++ b/Lib/test/test_warnings/data/import_warning.py @@ -0,0 +1,3 @@ +import warnings + +warnings.warn('module-level warning', DeprecationWarning, stacklevel=2) diff --git a/Lib/test/test_warnings/data/package_helper.py b/Lib/test/test_warnings/data/package_helper.py new file mode 100644 index 0000000000..c22a4f6405 --- /dev/null +++ b/Lib/test/test_warnings/data/package_helper.py @@ -0,0 +1,10 @@ +# helper to the helper for testing skip_file_prefixes. + +import os + +package_path = os.path.dirname(__file__) + +def inner_api(message, *, stacklevel, warnings_module): + warnings_module.warn( + message, stacklevel=stacklevel, + skip_file_prefixes=(package_path,)) diff --git a/Lib/test/test_warnings/data/stacklevel.py b/Lib/test/test_warnings/data/stacklevel.py new file mode 100644 index 0000000000..c6dd24733b --- /dev/null +++ b/Lib/test/test_warnings/data/stacklevel.py @@ -0,0 +1,15 @@ +# Helper module for testing stacklevel and skip_file_prefixes arguments +# of warnings.warn() + +import warnings +from test.test_warnings.data import package_helper + +def outer(message, stacklevel=1): + inner(message, stacklevel) + +def inner(message, stacklevel=1): + warnings.warn(message, stacklevel=stacklevel) + +def package(message, *, stacklevel): + package_helper.inner_api(message, stacklevel=stacklevel, + warnings_module=warnings) diff --git a/Lib/test/test_weakref.py b/Lib/test/test_weakref.py index 7d204f3c4c..f47c17b723 100644 --- a/Lib/test/test_weakref.py +++ b/Lib/test/test_weakref.py @@ -1,5 +1,6 @@ import gc import sys +import doctest import unittest import collections import weakref @@ -9,10 +10,14 @@ import threading import time import random +import textwrap from test import support -from test.support import script_helper, ALWAYS_EQ +from test.support import script_helper, ALWAYS_EQ, suppress_immortalization from test.support import gc_collect +from test.support import import_helper +from test.support import threading_helper +from test.support import is_wasi, Py_DEBUG # Used in ReferencesTestCase.test_ref_created_during_del() . ref_from_del = None @@ -77,7 +82,7 @@ def callback(self, ref): @contextlib.contextmanager -def collect_in_thread(period=0.0001): +def collect_in_thread(period=0.005): """ Ensure GC collections happen in a different thread, at a high frequency. """ @@ -114,6 +119,49 @@ def test_basic_ref(self): del o repr(wr) + @support.cpython_only + def test_ref_repr(self): + obj = C() + ref = weakref.ref(obj) + regex = ( + rf"<weakref at 0x[0-9a-fA-F]+; " + rf"to '{'' if __name__ == '__main__' else C.__module__ + '.'}{C.__qualname__}' " + rf"at 0x[0-9a-fA-F]+>" + ) + self.assertRegex(repr(ref), regex) + + obj = None + gc_collect() + self.assertRegex(repr(ref), + rf'<weakref at 0x[0-9a-fA-F]+; dead>') + + # test type with __name__ + class WithName: + @property + def __name__(self): + return "custom_name" + + obj2 = WithName() + ref2 = weakref.ref(obj2) + regex = ( + rf"<weakref at 0x[0-9a-fA-F]+; " + rf"to '{'' if __name__ == '__main__' else WithName.__module__ + '.'}" + rf"{WithName.__qualname__}' " + rf"at 0x[0-9a-fA-F]+ +\(custom_name\)>" + ) + self.assertRegex(repr(ref2), regex) + + def test_repr_failure_gh99184(self): + class MyConfig(dict): + def __getattr__(self, x): + return self[x] + + obj = MyConfig(offset=5) + obj_weakref = weakref.ref(obj) + + self.assertIn('MyConfig', repr(obj_weakref)) + self.assertIn('MyConfig', str(obj_weakref)) + def test_basic_callback(self): self.check_basic_callback(C) self.check_basic_callback(create_function) @@ -121,7 +169,7 @@ def test_basic_callback(self): @support.cpython_only def test_cfunction(self): - import _testcapi + _testcapi = import_helper.import_module("_testcapi") create_cfunction = _testcapi.create_cfunction f = create_cfunction() wr = weakref.ref(f) @@ -182,6 +230,22 @@ def check(proxy): self.assertRaises(ReferenceError, bool, ref3) self.assertEqual(self.cbcalled, 2) + @support.cpython_only + def test_proxy_repr(self): + obj = C() + ref = weakref.proxy(obj, self.callback) + regex = ( + rf"<weakproxy at 0x[0-9a-fA-F]+; " + rf"to '{'' if __name__ == '__main__' else C.__module__ + '.'}{C.__qualname__}' " + rf"at 0x[0-9a-fA-F]+>" + ) + self.assertRegex(repr(ref), regex) + + obj = None + gc_collect() + self.assertRegex(repr(ref), + rf'<weakproxy at 0x[0-9a-fA-F]+; dead>') + def check_basic_ref(self, factory): o = factory() ref = weakref.ref(o) @@ -613,7 +677,8 @@ class C(object): # deallocation of c2. del c2 - def test_callback_in_cycle_1(self): + @suppress_immortalization() + def test_callback_in_cycle(self): import gc class J(object): @@ -653,40 +718,11 @@ def acallback(self, ignore): del I, J, II gc.collect() - def test_callback_in_cycle_2(self): + def test_callback_reachable_one_way(self): import gc - # This is just like test_callback_in_cycle_1, except that II is an - # old-style class. The symptom is different then: an instance of an - # old-style class looks in its own __dict__ first. 'J' happens to - # get cleared from I.__dict__ before 'wr', and 'J' was never in II's - # __dict__, so the attribute isn't found. The difference is that - # the old-style II doesn't have a NULL __mro__ (it doesn't have any - # __mro__), so no segfault occurs. Instead it got: - # test_callback_in_cycle_2 (__main__.ReferencesTestCase) ... - # Exception exceptions.AttributeError: - # "II instance has no attribute 'J'" in <bound method II.acallback - # of <?.II instance at 0x00B9B4B8>> ignored - - class J(object): - pass - - class II: - def acallback(self, ignore): - self.J - - I = II() - I.J = J - I.wr = weakref.ref(J, I.acallback) - - del I, J, II - gc.collect() - - def test_callback_in_cycle_3(self): - import gc - - # This one broke the first patch that fixed the last two. In this - # case, the objects reachable from the callback aren't also reachable + # This one broke the first patch that fixed the previous test. In this case, + # the objects reachable from the callback aren't also reachable # from the object (c1) *triggering* the callback: you can get to # c1 from c2, but not vice-versa. The result was that c2's __dict__ # got tp_clear'ed by the time the c2.cb callback got invoked. @@ -706,10 +742,10 @@ def cb(self, ignore): del c1, c2 gc.collect() - def test_callback_in_cycle_4(self): + def test_callback_different_classes(self): import gc - # Like test_callback_in_cycle_3, except c2 and c1 have different + # Like test_callback_reachable_one_way, except c2 and c1 have different # classes. c2's class (C) isn't reachable from c1 then, so protecting # objects reachable from the dying object (c1) isn't enough to stop # c2's class (C) from getting tp_clear'ed before c2.cb is invoked. @@ -736,6 +772,7 @@ class D: # TODO: RUSTPYTHON @unittest.expectedFailure + @suppress_immortalization() def test_callback_in_cycle_resurrection(self): import gc @@ -869,8 +906,6 @@ def __del__(self): w = Target() - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_init(self): # Issue 3634 # <weakref to class>.__init__() doesn't check errors correctly @@ -879,6 +914,7 @@ def test_init(self): # No exception should be raised here gc.collect() + @suppress_immortalization() def test_classes(self): # Check that classes are weakrefable. class A(object): @@ -958,6 +994,7 @@ def test_hashing(self): self.assertEqual(hash(a), hash(42)) self.assertRaises(TypeError, hash, b) + @unittest.skipIf(is_wasi and Py_DEBUG, "requires deep stack") def test_trashcan_16602(self): # Issue #16602: when a weakref's target was part of a long # deallocation chain, the trashcan mechanism could delay clearing @@ -1015,6 +1052,31 @@ def __del__(self): pass del x support.gc_collect() + @support.cpython_only + def test_no_memory_when_clearing(self): + # gh-118331: Make sure we do not raise an exception from the destructor + # when clearing weakrefs if allocating the intermediate tuple fails. + code = textwrap.dedent(""" + import _testcapi + import weakref + + class TestObj: + pass + + def callback(obj): + pass + + obj = TestObj() + # The choice of 50 is arbitrary, but must be large enough to ensure + # the allocation won't be serviced by the free list. + wrs = [weakref.ref(obj, callback) for _ in range(50)] + _testcapi.set_nomemory(0) + del obj + """).strip() + res, _ = script_helper.run_python_until_end("-c", code) + stderr = res.err.decode("ascii", "backslashreplace") + self.assertNotRegex(stderr, "_Py_Dealloc: Deallocator of type 'TestObj'") + class SubclassableWeakrefTestCase(TestBase): @@ -1068,8 +1130,6 @@ class MyRef(weakref.ref): self.assertIn(r1, refs) self.assertIn(r2, refs) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_subclass_refs_with_slots(self): class MyRef(weakref.ref): __slots__ = "slot1", "slot2" @@ -1267,6 +1327,12 @@ class MappingTestCase(TestBase): COUNT = 10 + if support.check_sanitizer(thread=True) and support.Py_GIL_DISABLED: + # Reduce iteration count to get acceptable latency + NUM_THREADED_ITERATIONS = 1000 + else: + NUM_THREADED_ITERATIONS = 100000 + def check_len_cycles(self, dict_type, cons): N = 20 items = [RefCycle() for i in range(N)] @@ -1898,34 +1964,56 @@ def test_make_weak_keyed_dict_repr(self): dict = weakref.WeakKeyDictionary() self.assertRegex(repr(dict), '<WeakKeyDictionary at 0x.*>') + @threading_helper.requires_working_threading() def test_threaded_weak_valued_setdefault(self): d = weakref.WeakValueDictionary() with collect_in_thread(): - for i in range(100000): + for i in range(self.NUM_THREADED_ITERATIONS): x = d.setdefault(10, RefCycle()) self.assertIsNot(x, None) # we never put None in there! del x + @threading_helper.requires_working_threading() def test_threaded_weak_valued_pop(self): d = weakref.WeakValueDictionary() with collect_in_thread(): - for i in range(100000): + for i in range(self.NUM_THREADED_ITERATIONS): d[10] = RefCycle() x = d.pop(10, 10) self.assertIsNot(x, None) # we never put None in there! + @threading_helper.requires_working_threading() def test_threaded_weak_valued_consistency(self): # Issue #28427: old keys should not remove new values from # WeakValueDictionary when collecting from another thread. d = weakref.WeakValueDictionary() with collect_in_thread(): - for i in range(200000): + for i in range(2 * self.NUM_THREADED_ITERATIONS): o = RefCycle() d[10] = o # o is still alive, so the dict can't be empty self.assertEqual(len(d), 1) o = None # lose ref + @support.cpython_only + def test_weak_valued_consistency(self): + # A single-threaded, deterministic repro for issue #28427: old keys + # should not remove new values from WeakValueDictionary. This relies on + # an implementation detail of CPython's WeakValueDictionary (its + # underlying dictionary of KeyedRefs) to reproduce the issue. + d = weakref.WeakValueDictionary() + with support.disable_gc(): + d[10] = RefCycle() + # Keep the KeyedRef alive after it's replaced so that GC will invoke + # the callback. + wr = d.data[10] + # Replace the value with something that isn't cyclic garbage + o = RefCycle() + d[10] = o + # Trigger GC, which will invoke the callback for `wr` + gc.collect() + self.assertEqual(len(d), 1) + def check_threaded_weak_dict_copy(self, type_, deepcopy): # `type_` should be either WeakKeyDictionary or WeakValueDictionary. # `deepcopy` should be either True or False. @@ -1987,22 +2075,28 @@ def pop_and_collect(lst): if exc: raise exc[0] + @threading_helper.requires_working_threading() def test_threaded_weak_key_dict_copy(self): # Issue #35615: Weakref keys or values getting GC'ed during dict # copying should not result in a crash. self.check_threaded_weak_dict_copy(weakref.WeakKeyDictionary, False) + @threading_helper.requires_working_threading() + @support.requires_resource('cpu') def test_threaded_weak_key_dict_deepcopy(self): # Issue #35615: Weakref keys or values getting GC'ed during dict # copying should not result in a crash. self.check_threaded_weak_dict_copy(weakref.WeakKeyDictionary, True) @unittest.skip("TODO: RUSTPYTHON; occasionally crash (Exit code -6)") + @threading_helper.requires_working_threading() def test_threaded_weak_value_dict_copy(self): # Issue #35615: Weakref keys or values getting GC'ed during dict # copying should not result in a crash. self.check_threaded_weak_dict_copy(weakref.WeakValueDictionary, False) + @threading_helper.requires_working_threading() + @support.requires_resource('cpu') def test_threaded_weak_value_dict_deepcopy(self): # Issue #35615: Weakref keys or values getting GC'ed during dict # copying should not result in a crash. @@ -2195,6 +2289,19 @@ def test_atexit(self): self.assertTrue(b'ZeroDivisionError' in err) +class ModuleTestCase(unittest.TestCase): + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_names(self): + for name in ('ReferenceType', 'ProxyType', 'CallableProxyType', + 'WeakMethod', 'WeakSet', 'WeakKeyDictionary', 'WeakValueDictionary'): + obj = getattr(weakref, name) + if name != 'WeakSet': + self.assertEqual(obj.__module__, 'weakref') + self.assertEqual(obj.__name__, name) + self.assertEqual(obj.__qualname__, name) + + libreftest = """ Doctest for examples in the library reference: weakref.rst >>> from test.support import gc_collect @@ -2283,19 +2390,11 @@ def test_atexit(self): __test__ = {'libreftest' : libreftest} -def test_main(): - support.run_unittest( - ReferencesTestCase, - WeakMethodTestCase, - MappingTestCase, - WeakValueDictionaryTestCase, - WeakKeyDictionaryTestCase, - SubclassableWeakrefTestCase, - FinalizeTestCase, - ) +def load_tests(loader, tests, pattern): # TODO: RUSTPYTHON - # support.run_doctest(sys.modules[__name__]) + # tests.addTest(doctest.DocTestSuite()) + return tests if __name__ == "__main__": - test_main() + unittest.main() diff --git a/Lib/test/test_weakset.py b/Lib/test/test_weakset.py index 6c4cb05a5f..5e8cacc09d 100644 --- a/Lib/test/test_weakset.py +++ b/Lib/test/test_weakset.py @@ -44,7 +44,7 @@ def setUp(self): def test_methods(self): weaksetmethods = dir(WeakSet) for method in dir(set): - if method == 'test_c_api' or method.startswith('_'): + if method.startswith('_'): continue self.assertIn(method, weaksetmethods, "WeakSet missing method " + method) @@ -458,8 +458,6 @@ def test_abc(self): self.assertIsInstance(self.s, Set) self.assertIsInstance(self.s, MutableSet) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_copying(self): for cls in WeakSet, WeakSetWithSlots: s = cls(self.items) diff --git a/Lib/test/test_winapi.py b/Lib/test/test_winapi.py index 7a33f90698..99b5a0dfd1 100644 --- a/Lib/test/test_winapi.py +++ b/Lib/test/test_winapi.py @@ -77,41 +77,27 @@ def _events_waitany_test(self, n): evts[i] = old_evt - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_few_events_waitall(self): self._events_waitall_test(16) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_many_events_waitall(self): self._events_waitall_test(256) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_max_events_waitall(self): self._events_waitall_test(MAXIMUM_BATCHED_WAIT_OBJECTS) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_few_events_waitany(self): self._events_waitany_test(16) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_many_events_waitany(self): self._events_waitany_test(256) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_max_events_waitany(self): self._events_waitany_test(MAXIMUM_BATCHED_WAIT_OBJECTS) class WinAPITests(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_getlongpathname(self): testfn = pathlib.Path(os.getenv("ProgramFiles")).parents[-1] / "PROGRA~1" if not os.path.isdir(testfn): @@ -128,8 +114,6 @@ def test_getlongpathname(self): candidates = set(testfn.parent.glob("Progra*")) self.assertIn(pathlib.Path(actual), candidates) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_getshortpathname(self): testfn = pathlib.Path(os.getenv("ProgramFiles")) if not os.path.isdir(testfn): @@ -144,8 +128,6 @@ def test_getshortpathname(self): # Should contain "PROGRA~" but we can't predict the number self.assertIsNotNone(re.match(r".\:\\PROGRA~\d", actual.upper()), actual) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_namedpipe(self): pipe_name = rf"\\.\pipe\LOCAL\{os_helper.TESTFN}" diff --git a/Lib/test/test_winreg.py b/Lib/test/test_winreg.py new file mode 100644 index 0000000000..924a962781 --- /dev/null +++ b/Lib/test/test_winreg.py @@ -0,0 +1,525 @@ +# Test the windows specific win32reg module. +# Only win32reg functions not hit here: FlushKey, LoadKey and SaveKey + +import gc +import os, sys, errno +import threading +import unittest +from platform import machine, win32_edition +from test.support import cpython_only, import_helper + +# Do this first so test will be skipped if module doesn't exist +import_helper.import_module('winreg', required_on=['win']) +# Now import everything +from winreg import * + +try: + REMOTE_NAME = sys.argv[sys.argv.index("--remote")+1] +except (IndexError, ValueError): + REMOTE_NAME = None + +# tuple of (major, minor) +WIN_VER = sys.getwindowsversion()[:2] +# Some tests should only run on 64-bit architectures where WOW64 will be. +WIN64_MACHINE = True if machine() == "AMD64" else False + +# Starting with Windows 7 and Windows Server 2008 R2, WOW64 no longer uses +# registry reflection and formerly reflected keys are shared instead. +# Windows 7 and Windows Server 2008 R2 are version 6.1. Due to this, some +# tests are only valid up until 6.1 +HAS_REFLECTION = True if WIN_VER < (6, 1) else False + +# Use a per-process key to prevent concurrent test runs (buildbot!) from +# stomping on each other. +test_key_base = "Python Test Key [%d] - Delete Me" % (os.getpid(),) +test_key_name = "SOFTWARE\\" + test_key_base +# On OS'es that support reflection we should test with a reflected key +test_reflect_key_name = "SOFTWARE\\Classes\\" + test_key_base + +test_data = [ + ("Int Value", 45, REG_DWORD), + ("Qword Value", 0x1122334455667788, REG_QWORD), + ("String Val", "A string value", REG_SZ), + ("StringExpand", "The path is %path%", REG_EXPAND_SZ), + ("Multi-string", ["Lots", "of", "string", "values"], REG_MULTI_SZ), + ("Multi-nul", ["", "", "", ""], REG_MULTI_SZ), + ("Raw Data", b"binary\x00data", REG_BINARY), + ("Big String", "x"*(2**14-1), REG_SZ), + ("Big Binary", b"x"*(2**14), REG_BINARY), + # Two and three kanjis, meaning: "Japan" and "Japanese". + ("Japanese 日本", "日本語", REG_SZ), +] + + +@cpython_only +class HeapTypeTests(unittest.TestCase): + def test_have_gc(self): + self.assertTrue(gc.is_tracked(HKEYType)) + + def test_immutable(self): + with self.assertRaisesRegex(TypeError, "immutable"): + HKEYType.foo = "bar" + + +class BaseWinregTests(unittest.TestCase): + + def setUp(self): + # Make sure that the test key is absent when the test + # starts. + self.delete_tree(HKEY_CURRENT_USER, test_key_name) + + def delete_tree(self, root, subkey): + try: + hkey = OpenKey(root, subkey, 0, KEY_ALL_ACCESS) + except OSError: + # subkey does not exist + return + while True: + try: + subsubkey = EnumKey(hkey, 0) + except OSError: + # no more subkeys + break + self.delete_tree(hkey, subsubkey) + CloseKey(hkey) + DeleteKey(root, subkey) + + def _write_test_data(self, root_key, subkeystr="sub_key", + CreateKey=CreateKey): + # Set the default value for this key. + SetValue(root_key, test_key_name, REG_SZ, "Default value") + key = CreateKey(root_key, test_key_name) + self.assertTrue(key.handle != 0) + # Create a sub-key + sub_key = CreateKey(key, subkeystr) + # Give the sub-key some named values + + for value_name, value_data, value_type in test_data: + SetValueEx(sub_key, value_name, 0, value_type, value_data) + + # Check we wrote as many items as we thought. + nkeys, nvalues, since_mod = QueryInfoKey(key) + self.assertEqual(nkeys, 1, "Not the correct number of sub keys") + self.assertEqual(nvalues, 1, "Not the correct number of values") + nkeys, nvalues, since_mod = QueryInfoKey(sub_key) + self.assertEqual(nkeys, 0, "Not the correct number of sub keys") + self.assertEqual(nvalues, len(test_data), + "Not the correct number of values") + # Close this key this way... + # (but before we do, copy the key as an integer - this allows + # us to test that the key really gets closed). + int_sub_key = int(sub_key) + CloseKey(sub_key) + try: + QueryInfoKey(int_sub_key) + self.fail("It appears the CloseKey() function does " + "not close the actual key!") + except OSError: + pass + # ... and close that key that way :-) + int_key = int(key) + key.Close() + try: + QueryInfoKey(int_key) + self.fail("It appears the key.Close() function " + "does not close the actual key!") + except OSError: + pass + def _read_test_data(self, root_key, subkeystr="sub_key", OpenKey=OpenKey): + # Check we can get default value for this key. + val = QueryValue(root_key, test_key_name) + self.assertEqual(val, "Default value", + "Registry didn't give back the correct value") + + key = OpenKey(root_key, test_key_name) + # Read the sub-keys + with OpenKey(key, subkeystr) as sub_key: + # Check I can enumerate over the values. + index = 0 + while 1: + try: + data = EnumValue(sub_key, index) + except OSError: + break + self.assertEqual(data in test_data, True, + "Didn't read back the correct test data") + index = index + 1 + self.assertEqual(index, len(test_data), + "Didn't read the correct number of items") + # Check I can directly access each item + for value_name, value_data, value_type in test_data: + read_val, read_typ = QueryValueEx(sub_key, value_name) + self.assertEqual(read_val, value_data, + "Could not directly read the value") + self.assertEqual(read_typ, value_type, + "Could not directly read the value") + sub_key.Close() + # Enumerate our main key. + read_val = EnumKey(key, 0) + self.assertEqual(read_val, subkeystr, "Read subkey value wrong") + try: + EnumKey(key, 1) + self.fail("Was able to get a second key when I only have one!") + except OSError: + pass + + key.Close() + + def _delete_test_data(self, root_key, subkeystr="sub_key"): + key = OpenKey(root_key, test_key_name, 0, KEY_ALL_ACCESS) + sub_key = OpenKey(key, subkeystr, 0, KEY_ALL_ACCESS) + # It is not necessary to delete the values before deleting + # the key (although subkeys must not exist). We delete them + # manually just to prove we can :-) + for value_name, value_data, value_type in test_data: + DeleteValue(sub_key, value_name) + + nkeys, nvalues, since_mod = QueryInfoKey(sub_key) + self.assertEqual(nkeys, 0, "subkey not empty before delete") + self.assertEqual(nvalues, 0, "subkey not empty before delete") + sub_key.Close() + DeleteKey(key, subkeystr) + + try: + # Shouldn't be able to delete it twice! + DeleteKey(key, subkeystr) + self.fail("Deleting the key twice succeeded") + except OSError: + pass + key.Close() + DeleteKey(root_key, test_key_name) + # Opening should now fail! + try: + key = OpenKey(root_key, test_key_name) + self.fail("Could open the non-existent key") + except OSError: # Use this error name this time + pass + + def _test_all(self, root_key, subkeystr="sub_key"): + self._write_test_data(root_key, subkeystr) + self._read_test_data(root_key, subkeystr) + self._delete_test_data(root_key, subkeystr) + + def _test_named_args(self, key, sub_key): + with CreateKeyEx(key=key, sub_key=sub_key, reserved=0, + access=KEY_ALL_ACCESS) as ckey: + self.assertTrue(ckey.handle != 0) + + with OpenKeyEx(key=key, sub_key=sub_key, reserved=0, + access=KEY_ALL_ACCESS) as okey: + self.assertTrue(okey.handle != 0) + + +class LocalWinregTests(BaseWinregTests): + + def test_registry_works(self): + self._test_all(HKEY_CURRENT_USER) + self._test_all(HKEY_CURRENT_USER, "日本-subkey") + + def test_registry_works_extended_functions(self): + # Substitute the regular CreateKey and OpenKey calls with their + # extended counterparts. + # Note: DeleteKeyEx is not used here because it is platform dependent + cke = lambda key, sub_key: CreateKeyEx(key, sub_key, 0, KEY_ALL_ACCESS) + self._write_test_data(HKEY_CURRENT_USER, CreateKey=cke) + + oke = lambda key, sub_key: OpenKeyEx(key, sub_key, 0, KEY_READ) + self._read_test_data(HKEY_CURRENT_USER, OpenKey=oke) + + self._delete_test_data(HKEY_CURRENT_USER) + + def test_named_arguments(self): + self._test_named_args(HKEY_CURRENT_USER, test_key_name) + # Use the regular DeleteKey to clean up + # DeleteKeyEx takes named args and is tested separately + DeleteKey(HKEY_CURRENT_USER, test_key_name) + + def test_connect_registry_to_local_machine_works(self): + # perform minimal ConnectRegistry test which just invokes it + h = ConnectRegistry(None, HKEY_LOCAL_MACHINE) + self.assertNotEqual(h.handle, 0) + h.Close() + self.assertEqual(h.handle, 0) + + def test_nonexistent_remote_registry(self): + connect = lambda: ConnectRegistry("abcdefghijkl", HKEY_CURRENT_USER) + self.assertRaises(OSError, connect) + + def testExpandEnvironmentStrings(self): + r = ExpandEnvironmentStrings("%windir%\\test") + self.assertEqual(type(r), str) + self.assertEqual(r, os.environ["windir"] + "\\test") + + def test_context_manager(self): + # ensure that the handle is closed if an exception occurs + try: + with ConnectRegistry(None, HKEY_LOCAL_MACHINE) as h: + self.assertNotEqual(h.handle, 0) + raise OSError + except OSError: + self.assertEqual(h.handle, 0) + + def test_changing_value(self): + # Issue2810: A race condition in 2.6 and 3.1 may cause + # EnumValue or QueryValue to raise "WindowsError: More data is + # available" + done = False + + class VeryActiveThread(threading.Thread): + def run(self): + with CreateKey(HKEY_CURRENT_USER, test_key_name) as key: + use_short = True + long_string = 'x'*2000 + while not done: + s = 'x' if use_short else long_string + use_short = not use_short + SetValue(key, 'changing_value', REG_SZ, s) + + thread = VeryActiveThread() + thread.start() + try: + with CreateKey(HKEY_CURRENT_USER, + test_key_name+'\\changing_value') as key: + for _ in range(1000): + num_subkeys, num_values, t = QueryInfoKey(key) + for i in range(num_values): + name = EnumValue(key, i) + QueryValue(key, name[0]) + finally: + done = True + thread.join() + DeleteKey(HKEY_CURRENT_USER, test_key_name+'\\changing_value') + DeleteKey(HKEY_CURRENT_USER, test_key_name) + + def test_long_key(self): + # Issue2810, in 2.6 and 3.1 when the key name was exactly 256 + # characters, EnumKey raised "WindowsError: More data is + # available" + name = 'x'*256 + try: + with CreateKey(HKEY_CURRENT_USER, test_key_name) as key: + SetValue(key, name, REG_SZ, 'x') + num_subkeys, num_values, t = QueryInfoKey(key) + EnumKey(key, 0) + finally: + DeleteKey(HKEY_CURRENT_USER, '\\'.join((test_key_name, name))) + DeleteKey(HKEY_CURRENT_USER, test_key_name) + + def test_dynamic_key(self): + # Issue2810, when the value is dynamically generated, these + # raise "WindowsError: More data is available" in 2.6 and 3.1 + try: + EnumValue(HKEY_PERFORMANCE_DATA, 0) + except OSError as e: + if e.errno in (errno.EPERM, errno.EACCES): + self.skipTest("access denied to registry key " + "(are you running in a non-interactive session?)") + raise + QueryValueEx(HKEY_PERFORMANCE_DATA, "") + + # Reflection requires XP x64/Vista at a minimum. XP doesn't have this stuff + # or DeleteKeyEx so make sure their use raises NotImplementedError + @unittest.skipUnless(WIN_VER < (5, 2), "Requires Windows XP") + def test_reflection_unsupported(self): + try: + with CreateKey(HKEY_CURRENT_USER, test_key_name) as ck: + self.assertNotEqual(ck.handle, 0) + + key = OpenKey(HKEY_CURRENT_USER, test_key_name) + self.assertNotEqual(key.handle, 0) + + with self.assertRaises(NotImplementedError): + DisableReflectionKey(key) + with self.assertRaises(NotImplementedError): + EnableReflectionKey(key) + with self.assertRaises(NotImplementedError): + QueryReflectionKey(key) + with self.assertRaises(NotImplementedError): + DeleteKeyEx(HKEY_CURRENT_USER, test_key_name) + finally: + DeleteKey(HKEY_CURRENT_USER, test_key_name) + + def test_setvalueex_value_range(self): + # Test for Issue #14420, accept proper ranges for SetValueEx. + # Py2Reg, which gets called by SetValueEx, was using PyLong_AsLong, + # thus raising OverflowError. The implementation now uses + # PyLong_AsUnsignedLong to match DWORD's size. + try: + with CreateKey(HKEY_CURRENT_USER, test_key_name) as ck: + self.assertNotEqual(ck.handle, 0) + SetValueEx(ck, "test_name", None, REG_DWORD, 0x80000000) + finally: + DeleteKey(HKEY_CURRENT_USER, test_key_name) + + def test_setvalueex_negative_one_check(self): + # Test for Issue #43984, check -1 was not set by SetValueEx. + # Py2Reg, which gets called by SetValueEx, wasn't checking return + # value by PyLong_AsUnsignedLong, thus setting -1 as value in the registry. + # The implementation now checks PyLong_AsUnsignedLong return value to assure + # the value set was not -1. + try: + with CreateKey(HKEY_CURRENT_USER, test_key_name) as ck: + with self.assertRaises(OverflowError): + SetValueEx(ck, "test_name_dword", None, REG_DWORD, -1) + SetValueEx(ck, "test_name_qword", None, REG_QWORD, -1) + self.assertRaises(FileNotFoundError, QueryValueEx, ck, "test_name_dword") + self.assertRaises(FileNotFoundError, QueryValueEx, ck, "test_name_qword") + + finally: + DeleteKey(HKEY_CURRENT_USER, test_key_name) + + def test_queryvalueex_return_value(self): + # Test for Issue #16759, return unsigned int from QueryValueEx. + # Reg2Py, which gets called by QueryValueEx, was returning a value + # generated by PyLong_FromLong. The implementation now uses + # PyLong_FromUnsignedLong to match DWORD's size. + try: + with CreateKey(HKEY_CURRENT_USER, test_key_name) as ck: + self.assertNotEqual(ck.handle, 0) + test_val = 0x80000000 + SetValueEx(ck, "test_name", None, REG_DWORD, test_val) + ret_val, ret_type = QueryValueEx(ck, "test_name") + self.assertEqual(ret_type, REG_DWORD) + self.assertEqual(ret_val, test_val) + finally: + DeleteKey(HKEY_CURRENT_USER, test_key_name) + + def test_setvalueex_crash_with_none_arg(self): + # Test for Issue #21151, segfault when None is passed to SetValueEx + try: + with CreateKey(HKEY_CURRENT_USER, test_key_name) as ck: + self.assertNotEqual(ck.handle, 0) + test_val = None + SetValueEx(ck, "test_name", 0, REG_BINARY, test_val) + ret_val, ret_type = QueryValueEx(ck, "test_name") + self.assertEqual(ret_type, REG_BINARY) + self.assertEqual(ret_val, test_val) + finally: + DeleteKey(HKEY_CURRENT_USER, test_key_name) + + def test_read_string_containing_null(self): + # Test for issue 25778: REG_SZ should not contain null characters + try: + with CreateKey(HKEY_CURRENT_USER, test_key_name) as ck: + self.assertNotEqual(ck.handle, 0) + test_val = "A string\x00 with a null" + SetValueEx(ck, "test_name", 0, REG_SZ, test_val) + ret_val, ret_type = QueryValueEx(ck, "test_name") + self.assertEqual(ret_type, REG_SZ) + self.assertEqual(ret_val, "A string") + finally: + DeleteKey(HKEY_CURRENT_USER, test_key_name) + + +@unittest.skipUnless(REMOTE_NAME, "Skipping remote registry tests") +class RemoteWinregTests(BaseWinregTests): + + def test_remote_registry_works(self): + remote_key = ConnectRegistry(REMOTE_NAME, HKEY_CURRENT_USER) + self._test_all(remote_key) + + +@unittest.skipUnless(WIN64_MACHINE, "x64 specific registry tests") +class Win64WinregTests(BaseWinregTests): + + def test_named_arguments(self): + self._test_named_args(HKEY_CURRENT_USER, test_key_name) + # Clean up and also exercise the named arguments + DeleteKeyEx(key=HKEY_CURRENT_USER, sub_key=test_key_name, + access=KEY_ALL_ACCESS, reserved=0) + + @unittest.skipIf(win32_edition() in ('WindowsCoreHeadless', 'IoTEdgeOS'), "APIs not available on WindowsCoreHeadless") + def test_reflection_functions(self): + # Test that we can call the query, enable, and disable functions + # on a key which isn't on the reflection list with no consequences. + with OpenKey(HKEY_LOCAL_MACHINE, "Software") as key: + # HKLM\Software is redirected but not reflected in all OSes + self.assertTrue(QueryReflectionKey(key)) + self.assertIsNone(EnableReflectionKey(key)) + self.assertIsNone(DisableReflectionKey(key)) + self.assertTrue(QueryReflectionKey(key)) + + @unittest.skipUnless(HAS_REFLECTION, "OS doesn't support reflection") + def test_reflection(self): + # Test that we can create, open, and delete keys in the 32-bit + # area. Because we are doing this in a key which gets reflected, + # test the differences of 32 and 64-bit keys before and after the + # reflection occurs (ie. when the created key is closed). + try: + with CreateKeyEx(HKEY_CURRENT_USER, test_reflect_key_name, 0, + KEY_ALL_ACCESS | KEY_WOW64_32KEY) as created_key: + self.assertNotEqual(created_key.handle, 0) + + # The key should now be available in the 32-bit area + with OpenKey(HKEY_CURRENT_USER, test_reflect_key_name, 0, + KEY_ALL_ACCESS | KEY_WOW64_32KEY) as key: + self.assertNotEqual(key.handle, 0) + + # Write a value to what currently is only in the 32-bit area + SetValueEx(created_key, "", 0, REG_SZ, "32KEY") + + # The key is not reflected until created_key is closed. + # The 64-bit version of the key should not be available yet. + open_fail = lambda: OpenKey(HKEY_CURRENT_USER, + test_reflect_key_name, 0, + KEY_READ | KEY_WOW64_64KEY) + self.assertRaises(OSError, open_fail) + + # Now explicitly open the 64-bit version of the key + with OpenKey(HKEY_CURRENT_USER, test_reflect_key_name, 0, + KEY_ALL_ACCESS | KEY_WOW64_64KEY) as key: + self.assertNotEqual(key.handle, 0) + # Make sure the original value we set is there + self.assertEqual("32KEY", QueryValue(key, "")) + # Set a new value, which will get reflected to 32-bit + SetValueEx(key, "", 0, REG_SZ, "64KEY") + + # Reflection uses a "last-writer wins policy, so the value we set + # on the 64-bit key should be the same on 32-bit + with OpenKey(HKEY_CURRENT_USER, test_reflect_key_name, 0, + KEY_READ | KEY_WOW64_32KEY) as key: + self.assertEqual("64KEY", QueryValue(key, "")) + finally: + DeleteKeyEx(HKEY_CURRENT_USER, test_reflect_key_name, + KEY_WOW64_32KEY, 0) + + @unittest.skipUnless(HAS_REFLECTION, "OS doesn't support reflection") + def test_disable_reflection(self): + # Make use of a key which gets redirected and reflected + try: + with CreateKeyEx(HKEY_CURRENT_USER, test_reflect_key_name, 0, + KEY_ALL_ACCESS | KEY_WOW64_32KEY) as created_key: + # QueryReflectionKey returns whether or not the key is disabled + disabled = QueryReflectionKey(created_key) + self.assertEqual(type(disabled), bool) + # HKCU\Software\Classes is reflected by default + self.assertFalse(disabled) + + DisableReflectionKey(created_key) + self.assertTrue(QueryReflectionKey(created_key)) + + # The key is now closed and would normally be reflected to the + # 64-bit area, but let's make sure that didn't happen. + open_fail = lambda: OpenKeyEx(HKEY_CURRENT_USER, + test_reflect_key_name, 0, + KEY_READ | KEY_WOW64_64KEY) + self.assertRaises(OSError, open_fail) + + # Make sure the 32-bit key is actually there + with OpenKeyEx(HKEY_CURRENT_USER, test_reflect_key_name, 0, + KEY_READ | KEY_WOW64_32KEY) as key: + self.assertNotEqual(key.handle, 0) + finally: + DeleteKeyEx(HKEY_CURRENT_USER, test_reflect_key_name, + KEY_WOW64_32KEY, 0) + + def test_exception_numbers(self): + with self.assertRaises(FileNotFoundError) as ctx: + QueryValue(HKEY_CLASSES_ROOT, 'some_value_that_does_not_exist') + + +if __name__ == "__main__": + if not REMOTE_NAME: + print("Remote registry calls can be tested using", + "'test_winreg.py --remote \\\\machine_name'") + unittest.main() diff --git a/Lib/test/test_with.py b/Lib/test/test_with.py index 07522bda6a..b321dac6c6 100644 --- a/Lib/test/test_with.py +++ b/Lib/test/test_with.py @@ -5,6 +5,7 @@ __email__ = "mbland at acm dot org" import sys +import traceback import unittest from collections import deque from contextlib import _GeneratorContextManager, contextmanager, nullcontext @@ -79,11 +80,11 @@ def __exit__(self, *exc_info): try: if mgr.__exit__(*ex): ex = (None, None, None) - except: - ex = sys.exc_info() + except BaseException as e: + ex = (type(e), e, e.__traceback__) self.entered = None if ex is not exc_info: - raise ex[0](ex[1]).with_traceback(ex[2]) + raise ex class MockNested(Nested): @@ -170,7 +171,10 @@ def __exit__(self, *args): def shouldThrow(): ct = EnterThrows() self.foo = None - with ct as self.foo: + # Ruff complains that we're redefining `self.foo` here, + # but the whole point of the test is to check that `self.foo` + # is *not* redefined (because `__enter__` raises) + with ct as self.foo: # noqa: F811 pass self.assertRaises(RuntimeError, shouldThrow) self.assertEqual(self.foo, None) @@ -251,7 +255,6 @@ def testInlineGeneratorBoundSyntax(self): self.assertAfterWithGeneratorInvariantsNoError(foo) def testInlineGeneratorBoundToExistingVariable(self): - foo = None with mock_contextmanager_generator() as foo: self.assertInWithGeneratorInvariants(foo) self.assertAfterWithGeneratorInvariantsNoError(foo) @@ -749,5 +752,49 @@ def testEnterReturnsTuple(self): self.assertEqual(10, b1) self.assertEqual(20, b2) + @unittest.expectedFailure # TODO: RUSTPYTHON; AttributeError: 'FrameSummary' object has no attribute 'end_lineno' + def testExceptionLocation(self): + # The location of an exception raised from + # __init__, __enter__ or __exit__ of a context + # manager should be just the context manager expression, + # pinpointing the precise context manager in case there + # is more than one. + + def init_raises(): + try: + with self.Dummy(), self.InitRaises() as cm, self.Dummy() as d: + pass + except Exception as e: + return e + + def enter_raises(): + try: + with self.EnterRaises(), self.Dummy() as d: + pass + except Exception as e: + return e + + def exit_raises(): + try: + with self.ExitRaises(), self.Dummy() as d: + pass + except Exception as e: + return e + + for func, expected in [(init_raises, "self.InitRaises()"), + (enter_raises, "self.EnterRaises()"), + (exit_raises, "self.ExitRaises()"), + ]: + with self.subTest(func): + exc = func() + f = traceback.extract_tb(exc.__traceback__)[0] + indent = 16 + co = func.__code__ + self.assertEqual(f.lineno, co.co_firstlineno + 2) + self.assertEqual(f.end_lineno, co.co_firstlineno + 2) + self.assertEqual(f.line[f.colno - indent : f.end_colno - indent], + expected) + + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_xml_dom_xmlbuilder.py b/Lib/test/test_xml_dom_xmlbuilder.py new file mode 100644 index 0000000000..5282e806e4 --- /dev/null +++ b/Lib/test/test_xml_dom_xmlbuilder.py @@ -0,0 +1,92 @@ +import io +import unittest +from http import client +from test.test_httplib import FakeSocket +from unittest import mock +from xml.dom import getDOMImplementation, minidom, xmlbuilder + +SMALL_SAMPLE = b"""<?xml version="1.0"?> +<html xmlns="http://www.w3.org/1999/xhtml" xmlns:xdc="http://www.xml.com/books"> +<!-- A comment --> +<title>Introduction to XSL +


+

A. Namespace

+""" + + +class XMLBuilderTest(unittest.TestCase): + def test_entity_resolver(self): + body = ( + b"HTTP/1.1 200 OK\r\nContent-Type: text/xml; charset=utf-8\r\n\r\n" + + SMALL_SAMPLE + ) + + sock = FakeSocket(body) + response = client.HTTPResponse(sock) + response.begin() + attrs = {"open.return_value": response} + opener = mock.Mock(**attrs) + + resolver = xmlbuilder.DOMEntityResolver() + + with mock.patch("urllib.request.build_opener") as mock_build: + mock_build.return_value = opener + source = resolver.resolveEntity(None, "http://example.com/2000/svg") + + self.assertIsInstance(source, xmlbuilder.DOMInputSource) + self.assertIsNone(source.publicId) + self.assertEqual(source.systemId, "http://example.com/2000/svg") + self.assertEqual(source.baseURI, "http://example.com/2000/") + self.assertEqual(source.encoding, "utf-8") + self.assertIs(source.byteStream, response) + + self.assertIsNone(source.characterStream) + self.assertIsNone(source.stringData) + + def test_builder(self): + imp = getDOMImplementation() + self.assertIsInstance(imp, xmlbuilder.DOMImplementationLS) + + builder = imp.createDOMBuilder(imp.MODE_SYNCHRONOUS, None) + self.assertIsInstance(builder, xmlbuilder.DOMBuilder) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_parse_uri(self): + body = ( + b"HTTP/1.1 200 OK\r\nContent-Type: text/xml; charset=utf-8\r\n\r\n" + + SMALL_SAMPLE + ) + + sock = FakeSocket(body) + response = client.HTTPResponse(sock) + response.begin() + attrs = {"open.return_value": response} + opener = mock.Mock(**attrs) + + with mock.patch("urllib.request.build_opener") as mock_build: + mock_build.return_value = opener + + imp = getDOMImplementation() + builder = imp.createDOMBuilder(imp.MODE_SYNCHRONOUS, None) + document = builder.parseURI("http://example.com/2000/svg") + + self.assertIsInstance(document, minidom.Document) + self.assertEqual(len(document.childNodes), 1) + + # TODO: RUSTPYTHON + @unittest.expectedFailure + def test_parse_with_systemId(self): + response = io.BytesIO(SMALL_SAMPLE) + + with mock.patch("urllib.request.urlopen") as mock_open: + mock_open.return_value = response + + imp = getDOMImplementation() + source = imp.createDOMInputSource() + builder = imp.createDOMBuilder(imp.MODE_SYNCHRONOUS, None) + source.systemId = "http://example.com/2000/svg" + document = builder.parse(source) + + self.assertIsInstance(document, minidom.Document) + self.assertEqual(len(document.childNodes), 1) diff --git a/Lib/test/test_xml_etree.py b/Lib/test/test_xml_etree.py index 1a681d5a7c..11c617b5f3 100644 --- a/Lib/test/test_xml_etree.py +++ b/Lib/test/test_xml_etree.py @@ -13,13 +13,16 @@ import operator import os import pickle +import pyexpat import sys import textwrap import types import unittest +import unittest.mock as mock import warnings import weakref +from contextlib import nullcontext from functools import partial from itertools import product, islice from test import support @@ -120,6 +123,21 @@ """ +def is_python_implementation(): + assert ET is not None, "ET must be initialized" + assert pyET is not None, "pyET must be initialized" + return ET is pyET + + +def equal_wrapper(cls): + """Mock cls.__eq__ to check whether it has been called or not. + + The behaviour of cls.__eq__ (side-effects included) is left as is. + """ + eq = cls.__eq__ + return mock.patch.object(cls, "__eq__", autospec=True, wraps=eq) + + def checkwarnings(*filters, quiet=False): def decorator(test): def newtest(*args, **kwargs): @@ -200,28 +218,36 @@ class ElementTreeTest(unittest.TestCase): def serialize_check(self, elem, expected): self.assertEqual(serialize(elem), expected) + def test_constructor(self): + # Test constructor behavior. + + with self.assertRaises(TypeError): + tree = ET.ElementTree("") + with self.assertRaises(TypeError): + tree = ET.ElementTree(ET.ElementTree()) + + def test_setroot(self): + # Test _setroot behavior. + + tree = ET.ElementTree() + element = ET.Element("tag") + tree._setroot(element) + self.assertEqual(tree.getroot().tag, "tag") + self.assertEqual(tree.getroot(), element) + + # Test behavior with an invalid root element + + tree = ET.ElementTree() + with self.assertRaises(TypeError): + tree._setroot("") + with self.assertRaises(TypeError): + tree._setroot(ET.ElementTree()) + with self.assertRaises(TypeError): + tree._setroot(None) + def test_interface(self): # Test element tree interface. - def check_string(string): - len(string) - for char in string: - self.assertEqual(len(char), 1, - msg="expected one-character string, got %r" % char) - new_string = string + "" - new_string = string + " " - string[:0] - - def check_mapping(mapping): - len(mapping) - keys = mapping.keys() - items = mapping.items() - for key in keys: - item = mapping[key] - mapping["key"] = "value" - self.assertEqual(mapping["key"], "value", - msg="expected value string, got %r" % mapping["key"]) - def check_element(element): self.assertTrue(ET.iselement(element), msg="not an element") direlem = dir(element) @@ -231,12 +257,12 @@ def check_element(element): self.assertIn(attr, direlem, msg='no %s visible by dir' % attr) - check_string(element.tag) - check_mapping(element.attrib) + self.assertIsInstance(element.tag, str) + self.assertIsInstance(element.attrib, dict) if element.text is not None: - check_string(element.text) + self.assertIsInstance(element.text, str) if element.tail is not None: - check_string(element.tail) + self.assertIsInstance(element.tail, str) for elem in element: check_element(elem) @@ -313,8 +339,7 @@ def test_set_attribute(self): element.attrib = {'A': 'B', 'C': 'D'} self.assertEqual(element.attrib, {'A': 'B', 'C': 'D'}) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_simpleops(self): # Basic method sanity checks. @@ -359,8 +384,7 @@ def test_simpleops(self): self.serialize_check(element, '') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_cdata(self): # Test CDATA handling (etc). @@ -371,8 +395,7 @@ def test_cdata(self): self.serialize_check(ET.XML(""), 'hello') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_file_init(self): stringfile = io.BytesIO(SAMPLE_XML.encode("utf-8")) tree = ET.ElementTree(file=stringfile) @@ -384,14 +407,14 @@ def test_file_init(self): self.assertEqual(tree.find("element/../empty-element").tag, 'empty-element') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_path_cache(self): # Check that the path cache behaves sanely. from xml.etree import ElementPath elem = ET.XML(SAMPLE_XML) + ElementPath._cache.clear() for i in range(10): ET.ElementTree(elem).find('./'+str(i)) cache_len_10 = len(ElementPath._cache) for i in range(10): ET.ElementTree(elem).find('./'+str(i)) @@ -401,8 +424,7 @@ def test_path_cache(self): for i in range(600): ET.ElementTree(elem).find('./'+str(i)) self.assertLess(len(ElementPath._cache), 500) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_copy(self): # Test copy handling (etc). @@ -415,8 +437,6 @@ def test_copy(self): self.serialize_check(e2, 'hello') self.serialize_check(e3, 'hello') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_attrib(self): # Test attribute handling. @@ -493,8 +513,7 @@ def test_makeelement(self): elem[:] = tuple([subelem]) self.serialize_check(elem, '') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_parsefile(self): # Test parsing from file. @@ -540,8 +559,7 @@ def test_parsefile(self): ' \n' '') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_parseliteral(self): element = ET.XML("text") self.assertEqual(ET.tostring(element, encoding='unicode'), @@ -564,15 +582,16 @@ def test_parseliteral(self): self.assertEqual(len(ids), 1) self.assertEqual(ids["body"].tag, 'body') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_iterparse(self): # Test iterparse interface. iterparse = ET.iterparse context = iterparse(SIMPLE_XMLFILE) + self.assertIsNone(context.root) action, elem = next(context) + self.assertIsNone(context.root) self.assertEqual((action, elem.tag), ('end', 'element')) self.assertEqual([(action, elem.tag) for action, elem in context], [ ('end', 'element'), @@ -589,6 +608,17 @@ def test_iterparse(self): ('end', '{namespace}root'), ]) + with open(SIMPLE_XMLFILE, 'rb') as source: + context = iterparse(source) + action, elem = next(context) + self.assertEqual((action, elem.tag), ('end', 'element')) + self.assertEqual([(action, elem.tag) for action, elem in context], [ + ('end', 'element'), + ('end', 'empty-element'), + ('end', 'root'), + ]) + self.assertEqual(context.root.tag, 'root') + events = () context = iterparse(SIMPLE_XMLFILE, events) self.assertEqual([(action, elem.tag) for action, elem in context], []) @@ -680,12 +710,82 @@ def test_iterparse(self): # Not exhausting the iterator still closes the resource (bpo-43292) with warnings_helper.check_no_resource_warning(self): - it = iterparse(TESTFN) + it = iterparse(SIMPLE_XMLFILE) del it + with warnings_helper.check_no_resource_warning(self): + it = iterparse(SIMPLE_XMLFILE) + it.close() + del it + + with warnings_helper.check_no_resource_warning(self): + it = iterparse(SIMPLE_XMLFILE) + action, elem = next(it) + self.assertEqual((action, elem.tag), ('end', 'element')) + del it, elem + + with warnings_helper.check_no_resource_warning(self): + it = iterparse(SIMPLE_XMLFILE) + action, elem = next(it) + it.close() + self.assertEqual((action, elem.tag), ('end', 'element')) + del it, elem + with self.assertRaises(FileNotFoundError): iterparse("nonexistent") + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_iterparse_close(self): + iterparse = ET.iterparse + + it = iterparse(SIMPLE_XMLFILE) + it.close() + with self.assertRaises(StopIteration): + next(it) + it.close() # idempotent + + with open(SIMPLE_XMLFILE, 'rb') as source: + it = iterparse(source) + it.close() + self.assertFalse(source.closed) + with self.assertRaises(StopIteration): + next(it) + it.close() # idempotent + + it = iterparse(SIMPLE_XMLFILE) + action, elem = next(it) + self.assertEqual((action, elem.tag), ('end', 'element')) + it.close() + with self.assertRaises(StopIteration): + next(it) + it.close() # idempotent + + with open(SIMPLE_XMLFILE, 'rb') as source: + it = iterparse(source) + action, elem = next(it) + self.assertEqual((action, elem.tag), ('end', 'element')) + it.close() + self.assertFalse(source.closed) + with self.assertRaises(StopIteration): + next(it) + it.close() # idempotent + + it = iterparse(SIMPLE_XMLFILE) + list(it) + it.close() + with self.assertRaises(StopIteration): + next(it) + it.close() # idempotent + + with open(SIMPLE_XMLFILE, 'rb') as source: + it = iterparse(source) + list(it) + it.close() + self.assertFalse(source.closed) + with self.assertRaises(StopIteration): + next(it) + it.close() # idempotent + def test_writefile(self): elem = ET.Element("tag") elem.text = "text" @@ -703,8 +803,7 @@ def test_writefile(self): elem[0] = ET.PI("key", "value") self.serialize_check(elem, 'textsubtext') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_custom_builder(self): # Test parser w. custom builder. @@ -766,8 +865,7 @@ def end_ns(self, prefix): ('end-ns', ''), ]) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_custom_builder_only_end_ns(self): class Builder(list): def end_ns(self, prefix): @@ -790,8 +888,7 @@ def end_ns(self, prefix): ('end-ns', ''), ]) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_initialize_parser_without_target(self): # Explicit None parser = ET.XMLParser(target=None) @@ -801,8 +898,7 @@ def test_initialize_parser_without_target(self): parser2 = ET.XMLParser() self.assertIsInstance(parser2.target, ET.TreeBuilder) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_children(self): # Test Element children iteration @@ -840,16 +936,14 @@ def test_children(self): elem.clear() self.assertEqual(list(elem), []) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_writestring(self): elem = ET.XML("text") self.assertEqual(ET.tostring(elem), b'text') elem = ET.fromstring("text") self.assertEqual(ET.tostring(elem), b'text') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_indent(self): elem = ET.XML("") ET.indent(elem) @@ -894,8 +988,7 @@ def test_indent(self): b'' ) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_indent_space(self): elem = ET.XML("

pre
post

text

") ET.indent(elem, space='\t') @@ -921,8 +1014,7 @@ def test_indent_space(self): b'' ) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_indent_space_caching(self): elem = ET.XML("

par

text


") ET.indent(elem) @@ -939,8 +1031,7 @@ def test_indent_space_caching(self): len({id(el.tail) for el in elem.iter()}), ) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_indent_level(self): elem = ET.XML("

pre
post

text

") with self.assertRaises(ValueError): @@ -973,8 +1064,7 @@ def test_indent_level(self): b' ' ) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_tostring_default_namespace(self): elem = ET.XML('') self.assertEqual( @@ -986,8 +1076,7 @@ def test_tostring_default_namespace(self): '' ) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_tostring_default_namespace_different_namespace(self): elem = ET.XML('') self.assertEqual( @@ -995,16 +1084,14 @@ def test_tostring_default_namespace_different_namespace(self): '' ) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_tostring_default_namespace_original_no_namespace(self): elem = ET.XML('') EXPECTED_MSG = '^cannot use non-qualified names with default_namespace option$' with self.assertRaisesRegex(ValueError, EXPECTED_MSG): ET.tostring(elem, encoding='unicode', default_namespace='foobar') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_tostring_no_xml_declaration(self): elem = ET.XML('') self.assertEqual( @@ -1012,8 +1099,7 @@ def test_tostring_no_xml_declaration(self): '' ) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_tostring_xml_declaration(self): elem = ET.XML('') self.assertEqual( @@ -1021,8 +1107,7 @@ def test_tostring_xml_declaration(self): b"\n" ) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_tostring_xml_declaration_unicode_encoding(self): elem = ET.XML('') self.assertEqual( @@ -1030,8 +1115,7 @@ def test_tostring_xml_declaration_unicode_encoding(self): "\n" ) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_tostring_xml_declaration_cases(self): elem = ET.XML('ø') TESTCASES = [ @@ -1076,8 +1160,7 @@ def test_tostring_xml_declaration_cases(self): expected_retval ) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_tostringlist_default_namespace(self): elem = ET.XML('') self.assertEqual( @@ -1089,8 +1172,7 @@ def test_tostringlist_default_namespace(self): '' ) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_tostringlist_xml_declaration(self): elem = ET.XML('') self.assertEqual( @@ -1110,8 +1192,7 @@ def test_tostringlist_xml_declaration(self): self.assertRegex(stringlist[0], r"^<\?xml version='1.0' encoding='.+'?>") self.assertEqual(['', '', ''], stringlist[1:]) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_encoding(self): def check(encoding, body=''): xml = ("%s" % @@ -1171,8 +1252,7 @@ def bxml(encoding): self.assertRaises(ValueError, ET.XML, xml('undefined').encode('ascii')) self.assertRaises(LookupError, ET.XML, xml('xxx').encode('ascii')) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_methods(self): # Test serialization methods. @@ -1188,8 +1268,7 @@ def test_methods(self): '\n') self.assertEqual(serialize(e, method="text"), '1 < 2\n') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_issue18347(self): e = ET.XML('text') self.assertEqual(serialize(e), @@ -1197,8 +1276,7 @@ def test_issue18347(self): self.assertEqual(serialize(e, method="html"), 'text') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_entity(self): # Test entity handling. @@ -1236,8 +1314,7 @@ def test_entity(self): self.assertEqual(str(cm.exception), 'undefined entity &entity;: line 4, column 10') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_namespace(self): # Test namespace issues. @@ -1336,8 +1413,7 @@ def test_qname(self): self.assertNotEqual(q1, 'ns:tag') self.assertEqual(q1, '{ns}tag') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_doctype_public(self): # Test PUBLIC doctype. @@ -1404,8 +1480,6 @@ def check(p, expected, namespaces=None): {'': 'http://www.w3.org/2001/XMLSchema', 'ns': 'http://www.w3.org/2001/XMLSchema'}) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_processinginstruction(self): # Test ProcessingInstruction directly @@ -1422,13 +1496,13 @@ def test_processinginstruction(self): b"\n" b"\xe3?>") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_html_empty_elems_serialization(self): # issue 15970 # from http://www.w3.org/TR/html401/index/elements.html - for element in ['AREA', 'BASE', 'BASEFONT', 'BR', 'COL', 'FRAME', 'HR', - 'IMG', 'INPUT', 'ISINDEX', 'LINK', 'META', 'PARAM']: + for element in ['AREA', 'BASE', 'BASEFONT', 'BR', 'COL', 'EMBED', 'FRAME', + 'HR', 'IMG', 'INPUT', 'ISINDEX', 'LINK', 'META', 'PARAM', + 'SOURCE', 'TRACK', 'WBR']: for elem in [element, element.lower()]: expected = '<%s>' % elem serialized = serialize(ET.XML('<%s />' % elem), method='html') @@ -1453,8 +1527,7 @@ def test_tree_write_attribute_order(self): self.assertEqual(serialize(root, method='html'), '') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_attlist_default(self): # Test default attribute values; See BPO 42151. root = ET.fromstring(ATTLIST_XML) @@ -1464,12 +1537,14 @@ def test_attlist_default(self): class XMLPullParserTest(unittest.TestCase): - def _feed(self, parser, data, chunk_size=None): + def _feed(self, parser, data, chunk_size=None, flush=False): if chunk_size is None: parser.feed(data) else: for i in range(0, len(data), chunk_size): parser.feed(data[i:i+chunk_size]) + if flush: + parser.flush() def assert_events(self, parser, expected, max_events=None): self.assertEqual( @@ -1487,33 +1562,41 @@ def assert_event_tags(self, parser, expected, max_events=None): self.assertEqual([(action, elem.tag) for action, elem in events], expected) - # TODO: RUSTPYTHON - @unittest.expectedFailure - def test_simple_xml(self): - for chunk_size in (None, 1, 5): - with self.subTest(chunk_size=chunk_size): - parser = ET.XMLPullParser() - self.assert_event_tags(parser, []) - self._feed(parser, "\n", chunk_size) - self.assert_event_tags(parser, []) - self._feed(parser, - "\n text\n", chunk_size) - self.assert_event_tags(parser, [('end', 'element')]) - self._feed(parser, "texttail\n", chunk_size) - self._feed(parser, "\n", chunk_size) - self.assert_event_tags(parser, [ - ('end', 'element'), - ('end', 'empty-element'), - ]) - self._feed(parser, "\n", chunk_size) - self.assert_event_tags(parser, [('end', 'root')]) - self.assertIsNone(parser.close()) + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_simple_xml(self, chunk_size=None, flush=False): + parser = ET.XMLPullParser() + self.assert_event_tags(parser, []) + self._feed(parser, "\n", chunk_size, flush) + self.assert_event_tags(parser, []) + self._feed(parser, + "\n text\n", chunk_size, flush) + self.assert_event_tags(parser, [('end', 'element')]) + self._feed(parser, "texttail\n", chunk_size, flush) + self._feed(parser, "\n", chunk_size, flush) + self.assert_event_tags(parser, [ + ('end', 'element'), + ('end', 'empty-element'), + ]) + self._feed(parser, "\n", chunk_size, flush) + self.assert_event_tags(parser, [('end', 'root')]) + self.assertIsNone(parser.close()) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_simple_xml_chunk_1(self): + self.test_simple_xml(chunk_size=1, flush=True) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_simple_xml_chunk_5(self): + self.test_simple_xml(chunk_size=5, flush=True) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_simple_xml_chunk_22(self): + self.test_simple_xml(chunk_size=22) + + @unittest.expectedFailure # TODO: RUSTPYTHON def test_feed_while_iterating(self): parser = ET.XMLPullParser() it = parser.read_events() @@ -1526,8 +1609,7 @@ def test_feed_while_iterating(self): with self.assertRaises(StopIteration): next(it) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_simple_xml_with_ns(self): parser = ET.XMLPullParser() self.assert_event_tags(parser, []) @@ -1549,8 +1631,7 @@ def test_simple_xml_with_ns(self): self.assert_event_tags(parser, [('end', '{namespace}root')]) self.assertIsNone(parser.close()) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_ns_events(self): parser = ET.XMLPullParser(events=('start-ns', 'end-ns')) self._feed(parser, "\n") @@ -1566,8 +1647,7 @@ def test_ns_events(self): self.assertEqual(list(parser.read_events()), [('end-ns', None)]) self.assertIsNone(parser.close()) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_ns_events_start(self): parser = ET.XMLPullParser(events=('start-ns', 'start', 'end')) self._feed(parser, "\n") @@ -1591,8 +1671,7 @@ def test_ns_events_start(self): ('end', '{abc}tag'), ]) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_ns_events_start_end(self): parser = ET.XMLPullParser(events=('start-ns', 'start', 'end', 'end-ns')) self._feed(parser, "\n") @@ -1620,8 +1699,7 @@ def test_ns_events_start_end(self): ('end-ns', None), ]) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_events(self): parser = ET.XMLPullParser(events=()) self._feed(parser, "\n") @@ -1668,8 +1746,7 @@ def test_events(self): self._feed(parser, "") self.assertIsNone(parser.close()) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_events_comment(self): parser = ET.XMLPullParser(events=('start', 'comment', 'end')) self._feed(parser, "\n") @@ -1689,8 +1766,7 @@ def test_events_comment(self): self._feed(parser, "\n") self.assert_events(parser, [('comment', (ET.Comment, ' text here '))]) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_events_pi(self): parser = ET.XMLPullParser(events=('start', 'pi', 'end')) self._feed(parser, "\n") @@ -1699,8 +1775,7 @@ def test_events_pi(self): self._feed(parser, "\n") self.assert_events(parser, [('pi', (ET.PI, 'pitarget some text '))]) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_events_sequence(self): # Test that events can be some sequence that's not just a tuple or list eventset = {'end', 'start'} @@ -1720,12 +1795,63 @@ def __next__(self): self._feed(parser, "bar") self.assert_event_tags(parser, [('start', 'foo'), ('end', 'foo')]) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_unknown_event(self): with self.assertRaises(ValueError): ET.XMLPullParser(events=('start', 'end', 'bogus')) + @unittest.expectedFailure # TODO: RUSTPYTHON + @unittest.skipIf(pyexpat.version_info < (2, 6, 0), + f'Expat {pyexpat.version_info} does not ' + 'support reparse deferral') + def test_flush_reparse_deferral_enabled(self): + parser = ET.XMLPullParser(events=('start', 'end')) + + for chunk in (""): + parser.feed(chunk) + + self.assert_event_tags(parser, []) # i.e. no elements started + if ET is pyET: + self.assertTrue(parser._parser._parser.GetReparseDeferralEnabled()) + + parser.flush() + + self.assert_event_tags(parser, [('start', 'doc')]) + if ET is pyET: + self.assertTrue(parser._parser._parser.GetReparseDeferralEnabled()) + + parser.feed("") + parser.close() + + self.assert_event_tags(parser, [('end', 'doc')]) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_flush_reparse_deferral_disabled(self): + parser = ET.XMLPullParser(events=('start', 'end')) + + for chunk in (""): + parser.feed(chunk) + + if pyexpat.version_info >= (2, 6, 0): + if not ET is pyET: + self.skipTest(f'XMLParser.(Get|Set)ReparseDeferralEnabled ' + 'methods not available in C') + parser._parser._parser.SetReparseDeferralEnabled(False) + self.assert_event_tags(parser, []) # i.e. no elements started + + if ET is pyET: + self.assertFalse(parser._parser._parser.GetReparseDeferralEnabled()) + + parser.flush() + + self.assert_event_tags(parser, [('start', 'doc')]) + if ET is pyET: + self.assertFalse(parser._parser._parser.GetReparseDeferralEnabled()) + + parser.feed("") + parser.close() + + self.assert_event_tags(parser, [('end', 'doc')]) # # xinclude tests (samples from appendix C of the xinclude specification) @@ -1882,8 +2008,7 @@ def _my_loader(self, href, parse): else: return None - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_xinclude_default(self): from xml.etree import ElementInclude doc = self.xinclude_loader('default.xml') @@ -1898,8 +2023,7 @@ def test_xinclude_default(self): '\n' '') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_xinclude(self): from xml.etree import ElementInclude @@ -1964,8 +2088,7 @@ def test_xinclude(self): ' \n' '') # C5 - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_xinclude_repeated(self): from xml.etree import ElementInclude @@ -1973,8 +2096,7 @@ def test_xinclude_repeated(self): ElementInclude.include(document, self.xinclude_loader) self.assertEqual(1+4*2, len(document.findall(".//p"))) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_xinclude_failures(self): from xml.etree import ElementInclude @@ -2079,8 +2201,7 @@ def check(elem): elem.set("123", 123) check(elem) # attribute value - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_bug_xmltoolkit25(self): # typo in ElementTree.findtext @@ -2089,8 +2210,7 @@ def test_bug_xmltoolkit25(self): self.assertEqual(tree.findtext("tag"), 'text') self.assertEqual(tree.findtext("section/tag"), 'subtext') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_bug_xmltoolkit28(self): # .//tag causes exceptions @@ -2098,8 +2218,7 @@ def test_bug_xmltoolkit28(self): self.assertEqual(summarize_list(tree.findall(".//thead")), []) self.assertEqual(summarize_list(tree.findall(".//tbody")), ['tbody']) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_bug_xmltoolkitX1(self): # dump() doesn't flush the output buffer @@ -2108,8 +2227,7 @@ def test_bug_xmltoolkitX1(self): ET.dump(tree) self.assertEqual(stdout.getvalue(), '
\n') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_bug_xmltoolkit39(self): # non-ascii element and attribute names doesn't work @@ -2135,8 +2253,7 @@ def test_bug_xmltoolkit39(self): self.assertEqual(ET.tostring(tree, "utf-8"), b'') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_bug_xmltoolkit54(self): # problems handling internally defined entities @@ -2146,8 +2263,7 @@ def test_bug_xmltoolkit54(self): b'') self.assertEqual(serialize(e), '\u8230') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_bug_xmltoolkit55(self): # make sure we're reporting the first error, not the last @@ -2157,8 +2273,7 @@ def test_bug_xmltoolkit55(self): self.assertEqual(str(cm.exception), 'undefined entity &ldots;: line 1, column 36') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_bug_xmltoolkit60(self): # Handle crash in stream source. @@ -2168,8 +2283,7 @@ def read(self, x): self.assertRaises(OSError, ET.parse, ExceptionFile()) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_bug_xmltoolkit62(self): # Don't crash when using custom entities. @@ -2187,8 +2301,7 @@ def test_bug_xmltoolkit62(self): self.assertEqual(t.find('.//paragraph').text, 'A new cultivar of Begonia plant named \u2018BCT9801BEG\u2019.') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON @unittest.skipIf(sys.gettrace(), "Skips under coverage.") def test_bug_xmltoolkit63(self): # Check reference leak. @@ -2204,8 +2317,7 @@ def xmltoolkit63(): xmltoolkit63() self.assertEqual(sys.getrefcount(None), count) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_bug_200708_newline(self): # Preserve newlines in attributes. @@ -2217,8 +2329,7 @@ def test_bug_200708_newline(self): self.assertEqual(ET.tostring(ET.XML(ET.tostring(e))), b'') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_bug_200708_close(self): # Test default builder. parser = ET.XMLParser() # default @@ -2256,8 +2367,6 @@ def test_bug_200709_default_namespace(self): self.assertEqual(str(cm.exception), 'cannot use non-qualified names with default_namespace option') - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_bug_200709_register_namespace(self): e = ET.Element("{http://namespace.invalid/does/not/exist/}title") self.assertEqual(ET.tostring(e), @@ -2313,8 +2422,7 @@ def test_bug_1534630(self): e = bob.close() self.assertEqual(serialize(e), '') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_issue6233(self): e = ET.XML(b"" b't\xc3\xa3g') @@ -2327,8 +2435,7 @@ def test_issue6233(self): b"\n" b'tãg') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_issue6565(self): elem = ET.XML("") self.assertEqual(summarize_list(elem), ['tag']) @@ -2374,8 +2481,7 @@ def __bool__(self): self.assertIsInstance(e[0].tail, str) self.assertEqual(e[0].tail, 'changed') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_lost_elem(self): # Issue #25902: Borrowed element can disappear class Tag: @@ -2401,8 +2507,7 @@ def check_expat224_utf8_bug(self, text): root = ET.XML(xml) self.assertEqual(root.get('b'), text.decode('utf-8')) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_expat224_utf8_bug(self): # bpo-31170: Expat 2.2.3 had a bug in its UTF-8 decoder. # Check that Expat 2.2.4 fixed the bug. @@ -2415,8 +2520,7 @@ def test_expat224_utf8_bug(self): text = b'x' + b'\xc3\xa0' * 1024 self.check_expat224_utf8_bug(text) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_expat224_utf8_bug_file(self): with open(UTF8_BUG_XMLFILE, 'rb') as fp: raw = fp.read() @@ -2434,6 +2538,22 @@ def test_39495_treebuilder_start(self): self.assertRaises(TypeError, ET.TreeBuilder().start, "tag") self.assertRaises(TypeError, ET.TreeBuilder().start, "tag", None) + def test_issue123213_correct_extend_exception(self): + # Does not hide the internal exception when extending the element + self.assertRaises(ZeroDivisionError, ET.Element('tag').extend, + (1/0 for i in range(2))) + + # Still raises the TypeError when extending with a non-iterable + self.assertRaises(TypeError, ET.Element('tag').extend, None) + + # Preserves the TypeError message when extending with a generator + def f(): + raise TypeError("mymessage") + + self.assertRaisesRegex( + TypeError, 'mymessage', + ET.Element('tag').extend, (f() for i in range(2))) + # -------------------------------------------------------------------- @@ -2468,35 +2588,6 @@ def test___init__(self): self.assertIsNot(element_foo.attrib, attrib) self.assertNotEqual(element_foo.attrib, attrib) - def test_copy(self): - # Only run this test if Element.copy() is defined. - if "copy" not in dir(ET.Element): - raise unittest.SkipTest("Element.copy() not present") - - element_foo = ET.Element("foo", { "zix": "wyp" }) - element_foo.append(ET.Element("bar", { "baz": "qix" })) - - with self.assertWarns(DeprecationWarning): - element_foo2 = element_foo.copy() - - # elements are not the same - self.assertIsNot(element_foo2, element_foo) - - # string attributes are equal - self.assertEqual(element_foo2.tag, element_foo.tag) - self.assertEqual(element_foo2.text, element_foo.text) - self.assertEqual(element_foo2.tail, element_foo.tail) - - # number of children is the same - self.assertEqual(len(element_foo2), len(element_foo)) - - # children are the same - for (child1, child2) in itertools.zip_longest(element_foo, element_foo2): - self.assertIs(child1, child2) - - # attrib is a copy - self.assertEqual(element_foo2.attrib, element_foo.attrib) - def test___copy__(self): element_foo = ET.Element("foo", { "zix": "wyp" }) element_foo.append(ET.Element("bar", { "baz": "qix" })) @@ -2579,8 +2670,7 @@ def __deepcopy__(self, memo): e[:] = [E('bar')] self.assertRaises(TypeError, copy.deepcopy, e) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_cyclic_gc(self): class Dummy: pass @@ -2653,8 +2743,7 @@ def test_pickle(self): self.assertEqual(len(e2), 2) self.assertEqualElements(e, e2) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_pickle_issue18997(self): for proto in range(2, pickle.HIGHEST_PROTOCOL + 1): for dumper, loader in product(self.modules, repeat=2): @@ -2662,8 +2751,7 @@ def test_pickle_issue18997(self): 4 """ e1 = dumper.fromstring(XMLTEXT) - if hasattr(e1, '__getstate__'): - self.assertEqual(e1.__getstate__()['tag'], 'group') + self.assertEqual(e1.__getstate__()['tag'], 'group') e2 = self.pickleRoundTrip(e1, 'xml.etree.ElementTree', dumper, loader, proto) self.assertEqual(e2.tag, 'group') @@ -2671,6 +2759,7 @@ def test_pickle_issue18997(self): class BadElementTest(ElementTestCase, unittest.TestCase): + def test_extend_mutable_list(self): class X: @property @@ -2709,20 +2798,170 @@ class Y(X, ET.Element): e = ET.Element('foo') e.extend(L) - @unittest.skip("TODO: RUSTPYTHON, hangs") - def test_remove_with_mutating(self): - class X(ET.Element): + def test_remove_with_clear_assume_missing(self): + # gh-126033: Check that a concurrent clear() for an assumed-to-be + # missing element does not make the interpreter crash. + self.do_test_remove_with_clear(raises=True) + + def test_remove_with_clear_assume_existing(self): + # gh-126033: Check that a concurrent clear() for an assumed-to-be + # existing element does not make the interpreter crash. + self.do_test_remove_with_clear(raises=False) + + def do_test_remove_with_clear(self, *, raises): + + # Until the discrepency between "del root[:]" and "root.clear()" is + # resolved, we need to keep two tests. Previously, using "del root[:]" + # did not crash with the reproducer of gh-126033 while "root.clear()" + # did. + + class E(ET.Element): + """Local class to be able to mock E.__eq__ for introspection.""" + + class X(E): def __eq__(self, o): - del e[:] - return False - e = ET.Element('foo') - e.extend([X('bar')]) - self.assertRaises(ValueError, e.remove, ET.Element('baz')) + del root[:] + return not raises - e = ET.Element('foo') - e.extend([ET.Element('bar')]) - self.assertRaises(ValueError, e.remove, X('baz')) + class Y(E): + def __eq__(self, o): + root.clear() + return not raises + if raises: + get_checker_context = lambda: self.assertRaises(ValueError) + else: + get_checker_context = nullcontext + + self.assertIs(E.__eq__, object.__eq__) + + for Z, side_effect in [(X, 'del root[:]'), (Y, 'root.clear()')]: + self.enterContext(self.subTest(side_effect=side_effect)) + + # test removing R() from [U()] + for R, U, description in [ + (E, Z, "remove missing E() from [Z()]"), + (Z, E, "remove missing Z() from [E()]"), + (Z, Z, "remove missing Z() from [Z()]"), + ]: + with self.subTest(description): + root = E('top') + root.extend([U('one')]) + with get_checker_context(): + root.remove(R('missing')) + + # test removing R() from [U(), V()] + cases = self.cases_for_remove_missing_with_mutations(E, Z) + for R, U, V, description in cases: + with self.subTest(description): + root = E('top') + root.extend([U('one'), V('two')]) + with get_checker_context(): + root.remove(R('missing')) + + # Test removing root[0] from [Z()]. + # + # Since we call root.remove() with root[0], Z.__eq__() + # will not be called (we branch on the fast Py_EQ path). + with self.subTest("remove root[0] from [Z()]"): + root = E('top') + root.append(Z('rem')) + with equal_wrapper(E) as f, equal_wrapper(Z) as g: + root.remove(root[0]) + f.assert_not_called() + g.assert_not_called() + + # Test removing root[1] (of type R) from [U(), R()]. + is_special = is_python_implementation() and raises and Z is Y + if is_python_implementation() and raises and Z is Y: + # In pure Python, using root.clear() sets the children + # list to [] without calling list.clear(). + # + # For this reason, the call to root.remove() first + # checks root[0] and sets the children list to [] + # since either root[0] or root[1] is an evil element. + # + # Since checking root[1] still uses the old reference + # to the children list, PyObject_RichCompareBool() branches + # to the fast Py_EQ path and Y.__eq__() is called exactly + # once (when checking root[0]). + continue + else: + cases = self.cases_for_remove_existing_with_mutations(E, Z) + for R, U, description in cases: + with self.subTest(description): + root = E('top') + root.extend([U('one'), R('rem')]) + with get_checker_context(): + root.remove(root[1]) + + def test_remove_with_mutate_root_assume_missing(self): + # gh-126033: Check that a concurrent mutation for an assumed-to-be + # missing element does not make the interpreter crash. + self.do_test_remove_with_mutate_root(raises=True) + + def test_remove_with_mutate_root_assume_existing(self): + # gh-126033: Check that a concurrent mutation for an assumed-to-be + # existing element does not make the interpreter crash. + self.do_test_remove_with_mutate_root(raises=False) + + def do_test_remove_with_mutate_root(self, *, raises): + E = ET.Element + + class Z(E): + def __eq__(self, o): + del root[0] + return not raises + + if raises: + get_checker_context = lambda: self.assertRaises(ValueError) + else: + get_checker_context = nullcontext + + # test removing R() from [U(), V()] + cases = self.cases_for_remove_missing_with_mutations(E, Z) + for R, U, V, description in cases: + with self.subTest(description): + root = E('top') + root.extend([U('one'), V('two')]) + with get_checker_context(): + root.remove(R('missing')) + + # test removing root[1] (of type R) from [U(), R()] + cases = self.cases_for_remove_existing_with_mutations(E, Z) + for R, U, description in cases: + with self.subTest(description): + root = E('top') + root.extend([U('one'), R('rem')]) + with get_checker_context(): + root.remove(root[1]) + + def cases_for_remove_missing_with_mutations(self, E, Z): + # Cases for removing R() from [U(), V()]. + # The case U = V = R = E is not interesting as there is no mutation. + for U, V in [(E, Z), (Z, E), (Z, Z)]: + description = (f"remove missing {E.__name__}() from " + f"[{U.__name__}(), {V.__name__}()]") + yield E, U, V, description + + for U, V in [(E, E), (E, Z), (Z, E), (Z, Z)]: + description = (f"remove missing {Z.__name__}() from " + f"[{U.__name__}(), {V.__name__}()]") + yield Z, U, V, description + + def cases_for_remove_existing_with_mutations(self, E, Z): + # Cases for removing root[1] (of type R) from [U(), R()]. + # The case U = R = E is not interesting as there is no mutation. + for U, R, description in [ + (E, Z, "remove root[1] from [E(), Z()]"), + (Z, E, "remove root[1] from [Z(), E()]"), + (Z, Z, "remove root[1] from [Z(), Z()]"), + ]: + description = (f"remove root[1] (of type {R.__name__}) " + f"from [{U.__name__}(), {R.__name__}()]") + yield R, U, description + + @support.infinite_recursion(25) def test_recursive_repr(self): # Issue #25455 e = ET.Element('foo') @@ -2821,21 +3060,83 @@ def element_factory(x, y): del b gc_collect() + def test_deepcopy_clear(self): + # Prevent crashes when __deepcopy__() clears the children list. + # See https://github.com/python/cpython/issues/133009. + class X(ET.Element): + def __deepcopy__(self, memo): + root.clear() + return self + + root = ET.Element('a') + evil = X('x') + root.extend([evil, ET.Element('y')]) + if is_python_implementation(): + # Mutating a list over which we iterate raises an error. + self.assertRaises(RuntimeError, copy.deepcopy, root) + else: + c = copy.deepcopy(root) + # In the C implementation, we can still copy the evil element. + self.assertListEqual(list(c), [evil]) + + def test_deepcopy_grow(self): + # Prevent crashes when __deepcopy__() mutates the children list. + # See https://github.com/python/cpython/issues/133009. + a = ET.Element('a') + b = ET.Element('b') + c = ET.Element('c') + + class X(ET.Element): + def __deepcopy__(self, memo): + root.append(a) + root.append(b) + return self + + root = ET.Element('top') + evil1, evil2 = X('1'), X('2') + root.extend([evil1, c, evil2]) + children = list(copy.deepcopy(root)) + # mock deep copies + self.assertIs(children[0], evil1) + self.assertIs(children[2], evil2) + # true deep copies + self.assertEqual(children[1].tag, c.tag) + self.assertEqual([c.tag for c in children[3:]], + [a.tag, b.tag, a.tag, b.tag]) + -class MutatingElementPath(str): +class MutationDeleteElementPath(str): def __new__(cls, elem, *args): self = str.__new__(cls, *args) self.elem = elem return self + def __eq__(self, o): del self.elem[:] return True -MutatingElementPath.__hash__ = str.__hash__ + + __hash__ = str.__hash__ + + +class MutationClearElementPath(str): + def __new__(cls, elem, *args): + self = str.__new__(cls, *args) + self.elem = elem + return self + + def __eq__(self, o): + self.elem.clear() + return True + + __hash__ = str.__hash__ + class BadElementPath(str): def __eq__(self, o): raise 1/0 -BadElementPath.__hash__ = str.__hash__ + + __hash__ = str.__hash__ + class BadElementPathTest(ElementTestCase, unittest.TestCase): def setUp(self): @@ -2850,9 +3151,11 @@ def tearDown(self): super().tearDown() def test_find_with_mutating(self): - e = ET.Element('foo') - e.extend([ET.Element('bar')]) - e.find(MutatingElementPath(e, 'x')) + for cls in [MutationDeleteElementPath, MutationClearElementPath]: + with self.subTest(cls): + e = ET.Element('foo') + e.extend([ET.Element('bar')]) + e.find(cls(e, 'x')) def test_find_with_error(self): e = ET.Element('foo') @@ -2863,9 +3166,11 @@ def test_find_with_error(self): pass def test_findtext_with_mutating(self): - e = ET.Element('foo') - e.extend([ET.Element('bar')]) - e.findtext(MutatingElementPath(e, 'x')) + for cls in [MutationDeleteElementPath, MutationClearElementPath]: + with self.subTest(cls): + e = ET.Element('foo') + e.extend([ET.Element('bar')]) + e.findtext(cls(e, 'x')) def test_findtext_with_error(self): e = ET.Element('foo') @@ -2875,10 +3180,26 @@ def test_findtext_with_error(self): except ZeroDivisionError: pass + def test_findtext_with_falsey_text_attribute(self): + root_elem = ET.Element('foo') + sub_elem = ET.SubElement(root_elem, 'bar') + falsey = ["", 0, False, [], (), {}] + for val in falsey: + sub_elem.text = val + self.assertEqual(root_elem.findtext('./bar'), val) + + def test_findtext_with_none_text_attribute(self): + root_elem = ET.Element('foo') + sub_elem = ET.SubElement(root_elem, 'bar') + sub_elem.text = None + self.assertEqual(root_elem.findtext('./bar'), '') + def test_findall_with_mutating(self): - e = ET.Element('foo') - e.extend([ET.Element('bar')]) - e.findall(MutatingElementPath(e, 'x')) + for cls in [MutationDeleteElementPath, MutationClearElementPath]: + with self.subTest(cls): + e = ET.Element('foo') + e.extend([ET.Element('bar')]) + e.findall(cls(e, 'x')) def test_findall_with_error(self): e = ET.Element('foo') @@ -2946,8 +3267,7 @@ class MyElement(ET.Element): class ElementFindTest(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_find_simple(self): e = ET.XML(SAMPLE_XML) self.assertEqual(e.find('tag').tag, 'tag') @@ -2971,8 +3291,7 @@ def test_find_simple(self): # Issue #16922 self.assertEqual(ET.XML('').findtext('empty'), '') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_find_xpath(self): LINEAR_XML = ''' @@ -2995,8 +3314,7 @@ def test_find_xpath(self): self.assertRaisesRegex(SyntaxError, 'XPath', e.find, './tag[last()-0]') self.assertRaisesRegex(SyntaxError, 'XPath', e.find, './tag[last()+1]') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_findall(self): e = ET.XML(SAMPLE_XML) e[2] = ET.XML(SAMPLE_SECTION) @@ -3109,8 +3427,7 @@ def test_findall(self): self.assertEqual(summarize_list(e.findall(".//tag[. = 'subtext']")), ['tag', 'tag']) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_test_find_with_ns(self): e = ET.XML(SAMPLE_XML_NS) self.assertEqual(summarize_list(e.findall('tag')), []) @@ -3121,8 +3438,7 @@ def test_test_find_with_ns(self): summarize_list(e.findall(".//{http://effbot.org/ns}tag")), ['{http://effbot.org/ns}tag'] * 3) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_findall_different_nsmaps(self): root = ET.XML('''
@@ -3140,8 +3456,7 @@ def test_findall_different_nsmaps(self): self.assertEqual(len(root.findall(".//xx:b", namespaces=nsmap)), 2) self.assertEqual(len(root.findall(".//b", namespaces=nsmap)), 1) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_findall_wildcard(self): root = ET.XML(''' @@ -3186,15 +3501,13 @@ def test_findall_wildcard(self): self.assertEqual(summarize_list(root.findall(".//{}b")), summarize_list(root.findall(".//b"))) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_bad_find(self): e = ET.XML(SAMPLE_XML) with self.assertRaisesRegex(SyntaxError, 'cannot use absolute path'): e.findall('/tag') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_find_through_ElementTree(self): e = ET.XML(SAMPLE_XML) self.assertEqual(ET.ElementTree(e).find('tag').tag, 'tag') @@ -3214,8 +3527,7 @@ class ElementIterTest(unittest.TestCase): def _ilist(self, elem, tag=None): return summarize_list(elem.iter(tag)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_basic(self): doc = ET.XML("this is a paragraph...") self.assertEqual(self._ilist(doc), ['html', 'body', 'i']) @@ -3233,8 +3545,7 @@ def test_basic(self): # With an explicit parser too (issue #9708) sourcefile = serialize(doc, to_string=False) parser = ET.XMLParser(target=ET.TreeBuilder()) - self.assertEqual(next(ET.iterparse(sourcefile, parser=parser))[0], - 'end') + self.assertEqual(next(ET.iterparse(sourcefile, parser=parser))[0], 'end') tree = ET.ElementTree(None) self.assertRaises(AttributeError, tree.iter) @@ -3265,8 +3576,7 @@ def test_corners(self): del a[1] self.assertEqual(self._ilist(a), ['a', 'd']) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_iter_by_tag(self): doc = ET.XML(''' @@ -3331,8 +3641,7 @@ def _check_sample1_element(self, e): self.assertEqual(child.tail, 'tail') self.assertEqual(child.attrib, {}) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_dummy_builder(self): class BaseDummyBuilder: def close(self): @@ -3380,8 +3689,7 @@ def test_treebuilder_pi(self): self.assertEqual(b.pi('target'), (len('target'), None)) self.assertEqual(b.pi('pitarget', ' text '), (len('pitarget'), ' text ')) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_late_tail(self): # Issue #37399: The tail of an ignored comment could overwrite the text before it. class TreeBuilderSubclass(ET.TreeBuilder): @@ -3406,8 +3714,7 @@ class TreeBuilderSubclass(ET.TreeBuilder): a = parser.close() self.assertEqual(a.text, "texttail") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_late_tail_mix_pi_comments(self): # Issue #37399: The tail of an ignored comment could overwrite the text before it. # Test appending tails to comments/pis. @@ -3444,16 +3751,14 @@ class TreeBuilderSubclass(ET.TreeBuilder): self.assertEqual(a[0].tail, 'tail') self.assertEqual(a.text, "text\n") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_treebuilder_elementfactory_none(self): parser = ET.XMLParser(target=ET.TreeBuilder(element_factory=None)) parser.feed(self.sample1) e = parser.close() self._check_sample1_element(e) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_subclass(self): class MyTreeBuilder(ET.TreeBuilder): def foobar(self, x): @@ -3468,8 +3773,7 @@ def foobar(self, x): e = parser.close() self._check_sample1_element(e) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_subclass_comment_pi(self): class MyTreeBuilder(ET.TreeBuilder): def foobar(self, x): @@ -3485,8 +3789,7 @@ def foobar(self, x): e = parser.close() self._check_sample1_element(e) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_element_factory(self): lst = [] def myfactory(tag, attrib): @@ -3510,15 +3813,13 @@ def _check_element_factory_class(self, cls): self.assertIsInstance(e, cls) self._check_sample1_element(e) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_element_factory_subclass(self): class MyElement(ET.Element): pass self._check_element_factory_class(MyElement) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_element_factory_pure_python_subclass(self): # Mimic SimpleTAL's behaviour (issue #16089): both versions of # TreeBuilder should be able to cope with a subclass of the @@ -3532,8 +3833,7 @@ class MyElement(base, ValueError): pass self._check_element_factory_class(MyElement) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_doctype(self): class DoctypeParser: _doctype = None @@ -3551,8 +3851,7 @@ def close(self): ('html', '-//W3C//DTD XHTML 1.0 Transitional//EN', 'http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd')) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_builder_lookup_errors(self): class RaisingBuilder: def __init__(self, raise_in=None, what=ValueError): @@ -3593,16 +3892,14 @@ def _check_sample_element(self, e): self.assertEqual(e[0].tag, 'line') self.assertEqual(e[0].text, '22') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_constructor_args(self): parser2 = ET.XMLParser(encoding='utf-8', target=ET.TreeBuilder()) parser2.feed(self.sample1) self._check_sample_element(parser2.close()) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_subclass(self): class MyParser(ET.XMLParser): pass @@ -3610,8 +3907,7 @@ class MyParser(ET.XMLParser): parser.feed(self.sample1) self._check_sample_element(parser.close()) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_doctype_warning(self): with warnings.catch_warnings(): warnings.simplefilter('error', DeprecationWarning) @@ -3619,8 +3915,7 @@ def test_doctype_warning(self): parser.feed(self.sample2) parser.close() - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_subclass_doctype(self): _doctype = None class MyParserWithDoctype(ET.XMLParser): @@ -3651,8 +3946,7 @@ def doctype(self, name, pubid, system): ('html', '-//W3C//DTD XHTML 1.0 Transitional//EN', 'http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd')) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_inherited_doctype(self): '''Ensure that ordinary usage is not deprecated (Issue 19176)''' with warnings.catch_warnings(): @@ -3664,8 +3958,7 @@ class MyParserWithoutDoctype(ET.XMLParser): parser.feed(self.sample2) parser.close() - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_parse_string(self): parser = ET.XMLParser(target=ET.TreeBuilder()) parser.feed(self.sample3) @@ -3676,8 +3969,7 @@ def test_parse_string(self): class NamespaceParseTest(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_find_with_namespace(self): nsmap = {'h': 'hello', 'f': 'foo'} doc = ET.fromstring(SAMPLE_XML_NS_ELEMS) @@ -3836,10 +4128,25 @@ def test_setslice_negative_steps(self): e[1::-sys.maxsize<<64] = [ET.Element('d')] self.assertEqual(self._subelem_tags(e), ['a0', 'd', 'a2', 'a3']) + def test_issue123213_setslice_exception(self): + e = ET.Element('tag') + # Does not hide the internal exception when assigning to the element + with self.assertRaises(ZeroDivisionError): + e[:1] = (1/0 for i in range(2)) + + # Still raises the TypeError when assigning with a non-iterable + with self.assertRaises(TypeError): + e[:1] = None + + # Preserve the original TypeError message when assigning. + def f(): + raise TypeError("mymessage") + + with self.assertRaisesRegex(TypeError, 'mymessage'): + e[:1] = (f() for i in range(2)) class IOTest(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_encoding(self): # Test encoding issues. elem = ET.Element("tag") @@ -3909,8 +4216,7 @@ def test_encoding(self): ("\n" "" % enc).encode(enc)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_write_to_filename(self): self.addCleanup(os_helper.unlink, TESTFN) tree = ET.ElementTree(ET.XML('''\xf8''')) @@ -3918,8 +4224,7 @@ def test_write_to_filename(self): with open(TESTFN, 'rb') as f: self.assertEqual(f.read(), b'''ø''') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_write_to_filename_with_encoding(self): self.addCleanup(os_helper.unlink, TESTFN) tree = ET.ElementTree(ET.XML('''\xf8''')) @@ -3933,8 +4238,7 @@ def test_write_to_filename_with_encoding(self): b'''\n''' b'''\xf8''')) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_write_to_filename_as_unicode(self): self.addCleanup(os_helper.unlink, TESTFN) with open(TESTFN, 'w') as f: @@ -3946,8 +4250,7 @@ def test_write_to_filename_as_unicode(self): with open(TESTFN, 'rb') as f: self.assertEqual(f.read(), b"\xc3\xb8") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_write_to_text_file(self): self.addCleanup(os_helper.unlink, TESTFN) tree = ET.ElementTree(ET.XML('''\xf8''')) @@ -3969,8 +4272,7 @@ def test_write_to_text_file(self): with open(TESTFN, 'rb') as f: self.assertEqual(f.read(), b'''\xf8''') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_write_to_binary_file(self): self.addCleanup(os_helper.unlink, TESTFN) tree = ET.ElementTree(ET.XML('''\xf8''')) @@ -3980,8 +4282,7 @@ def test_write_to_binary_file(self): with open(TESTFN, 'rb') as f: self.assertEqual(f.read(), b'''ø''') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_write_to_binary_file_with_encoding(self): self.addCleanup(os_helper.unlink, TESTFN) tree = ET.ElementTree(ET.XML('''\xf8''')) @@ -3999,8 +4300,7 @@ def test_write_to_binary_file_with_encoding(self): b'''\n''' b'''\xf8''') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_write_to_binary_file_with_bom(self): self.addCleanup(os_helper.unlink, TESTFN) tree = ET.ElementTree(ET.XML('''\xf8''')) @@ -4021,32 +4321,28 @@ def test_write_to_binary_file_with_bom(self): '''\n''' '''\xf8'''.encode("utf-16")) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_read_from_stringio(self): tree = ET.ElementTree() stream = io.StringIO('''''') tree.parse(stream) self.assertEqual(tree.getroot().tag, 'site') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_write_to_stringio(self): tree = ET.ElementTree(ET.XML('''\xf8''')) stream = io.StringIO() tree.write(stream, encoding='unicode') self.assertEqual(stream.getvalue(), '''\xf8''') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_read_from_bytesio(self): tree = ET.ElementTree() raw = io.BytesIO(b'''''') tree.parse(raw) self.assertEqual(tree.getroot().tag, 'site') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_write_to_bytesio(self): tree = ET.ElementTree(ET.XML('''\xf8''')) raw = io.BytesIO() @@ -4056,8 +4352,7 @@ def test_write_to_bytesio(self): class dummy: pass - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_read_from_user_text_reader(self): stream = io.StringIO('''''') reader = self.dummy() @@ -4066,8 +4361,7 @@ def test_read_from_user_text_reader(self): tree.parse(reader) self.assertEqual(tree.getroot().tag, 'site') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_write_to_user_text_writer(self): tree = ET.ElementTree(ET.XML('''\xf8''')) stream = io.StringIO() @@ -4076,8 +4370,7 @@ def test_write_to_user_text_writer(self): tree.write(writer, encoding='unicode') self.assertEqual(stream.getvalue(), '''\xf8''') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_read_from_user_binary_reader(self): raw = io.BytesIO(b'''''') reader = self.dummy() @@ -4087,8 +4380,7 @@ def test_read_from_user_binary_reader(self): self.assertEqual(tree.getroot().tag, 'site') tree = ET.ElementTree() - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_write_to_user_binary_writer(self): tree = ET.ElementTree(ET.XML('''\xf8''')) raw = io.BytesIO() @@ -4097,8 +4389,7 @@ def test_write_to_user_binary_writer(self): tree.write(writer) self.assertEqual(raw.getvalue(), b'''ø''') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_write_to_user_binary_writer_with_bom(self): tree = ET.ElementTree(ET.XML('''''')) raw = io.BytesIO() @@ -4111,8 +4402,7 @@ def test_write_to_user_binary_writer_with_bom(self): '''\n''' ''''''.encode("utf-16")) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_tostringlist_invariant(self): root = ET.fromstring('foo') self.assertEqual( @@ -4122,8 +4412,7 @@ def test_tostringlist_invariant(self): ET.tostring(root, 'utf-16'), b''.join(ET.tostringlist(root, 'utf-16'))) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_short_empty_elements(self): root = ET.fromstring('abc') self.assertEqual( @@ -4147,15 +4436,13 @@ def _get_error(self, s): except ET.ParseError as e: return e - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_error_position(self): self.assertEqual(self._get_error('foo').position, (1, 0)) self.assertEqual(self._get_error('&foo;').position, (1, 5)) self.assertEqual(self._get_error('foobar<').position, (1, 6)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_error_code(self): import xml.parsers.expat.errors as ERRORS self.assertEqual(self._get_error('foo').code, @@ -4163,10 +4450,9 @@ def test_error_code(self): class KeywordArgsTest(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure # Test various issues with keyword arguments passed to ET.Element # constructor and methods + @unittest.expectedFailure # TODO: RUSTPYTHON def test_issue14818(self): x = ET.XML("foo") self.assertEqual(x.find('a', None), @@ -4201,12 +4487,11 @@ def test_issue14818(self): # -------------------------------------------------------------------- class NoAcceleratorTest(unittest.TestCase): - def setUp(self): - if not pyET: + @classmethod + def setUpClass(cls): + if ET is not pyET: raise unittest.SkipTest('only for the Python version') - # TODO: RUSTPYTHON - @unittest.expectedFailure # Test that the C accelerator was not imported for pyET def test_correct_import_pyET(self): # The type of methods defined in Python code is types.FunctionType, @@ -4215,6 +4500,26 @@ def test_correct_import_pyET(self): self.assertIsInstance(pyET.Element.__init__, types.FunctionType) self.assertIsInstance(pyET.XMLParser.__init__, types.FunctionType) +# -------------------------------------------------------------------- + +class BoolTest(unittest.TestCase): + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_warning(self): + e = ET.fromstring('') + msg = ( + r"Testing an element's truth value will always return True in " + r"future versions. " + r"Use specific 'len\(elem\)' or 'elem is not None' test instead.") + with self.assertWarnsRegex(DeprecationWarning, msg): + result = bool(e) + # Emulate prior behavior for now + self.assertIs(result, False) + + # Element with children + ET.SubElement(e, 'b') + with self.assertWarnsRegex(DeprecationWarning, msg): + new_result = bool(e) + self.assertIs(new_result, True) # -------------------------------------------------------------------- @@ -4228,8 +4533,7 @@ class C14NTest(unittest.TestCase): # # simple roundtrip tests (from c14n.py) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_simple_roundtrip(self): # Basics self.assertEqual(c14n_roundtrip(""), '') @@ -4270,8 +4574,7 @@ def test_simple_roundtrip(self): xml = '' self.assertEqual(c14n_roundtrip(xml), xml) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_c14n_exclusion(self): xml = textwrap.dedent("""\ @@ -4352,8 +4655,7 @@ def test_c14n_exclusion(self): # note that this uses generated C14N versions of the standard ET.write # output, not roundtripped C14N (see above). - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_xml_c14n2(self): datadir = findfile("c14n-20", subdir="xmltestdata") full_path = partial(os.path.join, datadir) @@ -4456,8 +4758,7 @@ def get_option(config, option_name, default=None): # -------------------------------------------------------------------- - -def test_main(module=None): +def setUpModule(module=None): # When invoked without a module, runs the Python ET tests by loading pyET. # Otherwise, uses the given module as the ET. global pyET @@ -4469,62 +4770,30 @@ def test_main(module=None): global ET ET = module - test_classes = [ - ModuleTest, - ElementSlicingTest, - BasicElementTest, - BadElementTest, - BadElementPathTest, - ElementTreeTest, - IOTest, - ParseErrorTest, - XIncludeTest, - ElementTreeTypeTest, - ElementFindTest, - ElementIterTest, - TreeBuilderTest, - XMLParserTest, - XMLPullParserTest, - BugsTest, - KeywordArgsTest, - C14NTest, - ] - - # These tests will only run for the pure-Python version that doesn't import - # _elementtree. We can't use skipUnless here, because pyET is filled in only - # after the module is loaded. - if pyET is not ET: - test_classes.extend([ - NoAcceleratorTest, - ]) + # don't interfere with subsequent tests + def cleanup(): + global ET, pyET + ET = pyET = None + unittest.addModuleCleanup(cleanup) # Provide default namespace mapping and path cache. from xml.etree import ElementPath nsmap = ET.register_namespace._namespace_map # Copy the default namespace mapping nsmap_copy = nsmap.copy() + unittest.addModuleCleanup(nsmap.update, nsmap_copy) + unittest.addModuleCleanup(nsmap.clear) + # Copy the path cache (should be empty) path_cache = ElementPath._cache + unittest.addModuleCleanup(setattr, ElementPath, "_cache", path_cache) ElementPath._cache = path_cache.copy() + # Align the Comment/PI factories. if hasattr(ET, '_set_factories'): old_factories = ET._set_factories(ET.Comment, ET.PI) - else: - old_factories = None - - try: - support.run_unittest(*test_classes) - finally: - from xml.etree import ElementPath - # Restore mapping and path cache - nsmap.clear() - nsmap.update(nsmap_copy) - ElementPath._cache = path_cache - if old_factories is not None: - ET._set_factories(*old_factories) - # don't interfere with subsequent tests - ET = pyET = None + unittest.addModuleCleanup(ET._set_factories, *old_factories) if __name__ == '__main__': - test_main() + unittest.main() diff --git a/Lib/test/test_xml_etree_c.py b/Lib/test/test_xml_etree_c.py new file mode 100644 index 0000000000..3a0fc572f4 --- /dev/null +++ b/Lib/test/test_xml_etree_c.py @@ -0,0 +1,278 @@ +# xml.etree test for cElementTree +import io +import struct +from test import support +from test.support.import_helper import import_fresh_module +import types +import unittest + +cET = import_fresh_module('xml.etree.ElementTree', + fresh=['_elementtree']) +cET_alias = import_fresh_module('xml.etree.cElementTree', + fresh=['_elementtree', 'xml.etree'], + deprecated=True) + + +@unittest.skipUnless(cET, 'requires _elementtree') +class MiscTests(unittest.TestCase): + # Issue #8651. + @support.bigmemtest(size=support._2G + 100, memuse=1, dry_run=False) + def test_length_overflow(self, size): + data = b'x' * size + parser = cET.XMLParser() + try: + self.assertRaises(OverflowError, parser.feed, data) + finally: + data = None + + def test_del_attribute(self): + element = cET.Element('tag') + + element.tag = 'TAG' + with self.assertRaises(AttributeError): + del element.tag + self.assertEqual(element.tag, 'TAG') + + with self.assertRaises(AttributeError): + del element.text + self.assertIsNone(element.text) + element.text = 'TEXT' + with self.assertRaises(AttributeError): + del element.text + self.assertEqual(element.text, 'TEXT') + + with self.assertRaises(AttributeError): + del element.tail + self.assertIsNone(element.tail) + element.tail = 'TAIL' + with self.assertRaises(AttributeError): + del element.tail + self.assertEqual(element.tail, 'TAIL') + + with self.assertRaises(AttributeError): + del element.attrib + self.assertEqual(element.attrib, {}) + element.attrib = {'A': 'B', 'C': 'D'} + with self.assertRaises(AttributeError): + del element.attrib + self.assertEqual(element.attrib, {'A': 'B', 'C': 'D'}) + + def test_trashcan(self): + # If this test fails, it will most likely die via segfault. + e = root = cET.Element('root') + for i in range(200000): + e = cET.SubElement(e, 'x') + del e + del root + support.gc_collect() + + def test_parser_ref_cycle(self): + # bpo-31499: xmlparser_dealloc() crashed with a segmentation fault when + # xmlparser_gc_clear() was called previously by the garbage collector, + # when the parser was part of a reference cycle. + + def parser_ref_cycle(): + parser = cET.XMLParser() + # Create a reference cycle using an exception to keep the frame + # alive, so the parser will be destroyed by the garbage collector + try: + raise ValueError + except ValueError as exc: + err = exc + + # Create a parser part of reference cycle + parser_ref_cycle() + # Trigger an explicit garbage collection to break the reference cycle + # and so destroy the parser + support.gc_collect() + + def test_bpo_31728(self): + # A crash or an assertion failure shouldn't happen, in case garbage + # collection triggers a call to clear() or a reading of text or tail, + # while a setter or clear() or __setstate__() is already running. + elem = cET.Element('elem') + class X: + def __del__(self): + elem.text + elem.tail + elem.clear() + + elem.text = X() + elem.clear() # shouldn't crash + + elem.tail = X() + elem.clear() # shouldn't crash + + elem.text = X() + elem.text = X() # shouldn't crash + elem.clear() + + elem.tail = X() + elem.tail = X() # shouldn't crash + elem.clear() + + elem.text = X() + elem.__setstate__({'tag': 42}) # shouldn't cause an assertion failure + elem.clear() + + elem.tail = X() + elem.__setstate__({'tag': 42}) # shouldn't cause an assertion failure + + @support.cpython_only + def test_uninitialized_parser(self): + # The interpreter shouldn't crash in case of calling methods or + # accessing attributes of uninitialized XMLParser objects. + parser = cET.XMLParser.__new__(cET.XMLParser) + self.assertRaises(ValueError, parser.close) + self.assertRaises(ValueError, parser.feed, 'foo') + class MockFile: + def read(*args): + return '' + self.assertRaises(ValueError, parser._parse_whole, MockFile()) + self.assertRaises(ValueError, parser._setevents, None) + self.assertIsNone(parser.entity) + self.assertIsNone(parser.target) + + def test_setstate_leaks(self): + # Test reference leaks + elem = cET.Element.__new__(cET.Element) + for i in range(100): + elem.__setstate__({'tag': 'foo', 'attrib': {'bar': 42}, + '_children': [cET.Element('child')], + 'text': 'text goes here', + 'tail': 'opposite of head'}) + + self.assertEqual(elem.tag, 'foo') + self.assertEqual(elem.text, 'text goes here') + self.assertEqual(elem.tail, 'opposite of head') + self.assertEqual(list(elem.attrib.items()), [('bar', 42)]) + self.assertEqual(len(elem), 1) + self.assertEqual(elem[0].tag, 'child') + + def test_iterparse_leaks(self): + # Test reference leaks in TreeBuilder (issue #35502). + # The test is written to be executed in the hunting reference leaks + # mode. + XML = '' + parser = cET.iterparse(io.StringIO(XML)) + next(parser) + del parser + support.gc_collect() + + def test_xmlpullparser_leaks(self): + # Test reference leaks in TreeBuilder (issue #35502). + # The test is written to be executed in the hunting reference leaks + # mode. + XML = '' + parser = cET.XMLPullParser() + parser.feed(XML) + del parser + support.gc_collect() + + def test_dict_disappearing_during_get_item(self): + # test fix for seg fault reported in issue 27946 + class X: + def __hash__(self): + e.attrib = {} # this frees e->extra->attrib + [{i: i} for i in range(1000)] # exhaust the dict keys cache + return 13 + + e = cET.Element("elem", {1: 2}) + r = e.get(X()) + self.assertIsNone(r) + + @support.cpython_only + def test_immutable_types(self): + root = cET.fromstring('') + dataset = ( + cET.Element, + cET.TreeBuilder, + cET.XMLParser, + type(root.iter()), + ) + for tp in dataset: + with self.subTest(tp=tp): + with self.assertRaisesRegex(TypeError, "immutable"): + tp.foo = 1 + + @support.cpython_only + def test_disallow_instantiation(self): + root = cET.fromstring('') + iter_type = type(root.iter()) + support.check_disallow_instantiation(self, iter_type) + + +@unittest.skipUnless(cET, 'requires _elementtree') +class TestAliasWorking(unittest.TestCase): + # Test that the cET alias module is alive + def test_alias_working(self): + e = cET_alias.Element('foo') + self.assertEqual(e.tag, 'foo') + + +@unittest.skipUnless(cET, 'requires _elementtree') +@support.cpython_only +class TestAcceleratorImported(unittest.TestCase): + # Test that the C accelerator was imported, as expected + def test_correct_import_cET(self): + # SubElement is a function so it retains _elementtree as its module. + self.assertEqual(cET.SubElement.__module__, '_elementtree') + + def test_correct_import_cET_alias(self): + self.assertEqual(cET_alias.SubElement.__module__, '_elementtree') + + def test_parser_comes_from_C(self): + # The type of methods defined in Python code is types.FunctionType, + # while the type of methods defined inside _elementtree is + # + self.assertNotIsInstance(cET.Element.__init__, types.FunctionType) + + +@unittest.skipUnless(cET, 'requires _elementtree') +@support.cpython_only +class SizeofTest(unittest.TestCase): + def setUp(self): + self.elementsize = support.calcobjsize('5P') + # extra + self.extra = struct.calcsize('PnnP4P') + + check_sizeof = support.check_sizeof + + def test_element(self): + e = cET.Element('a') + self.check_sizeof(e, self.elementsize) + + def test_element_with_attrib(self): + e = cET.Element('a', href='about:') + self.check_sizeof(e, self.elementsize + self.extra) + + def test_element_with_children(self): + e = cET.Element('a') + for i in range(5): + cET.SubElement(e, 'span') + # should have space for 8 children now + self.check_sizeof(e, self.elementsize + self.extra + + struct.calcsize('8P')) + + +def install_tests(): + # Test classes should have __module__ referring to this module. + from test import test_xml_etree + for name, base in vars(test_xml_etree).items(): + if isinstance(base, type) and issubclass(base, unittest.TestCase): + class Temp(base): + pass + Temp.__name__ = Temp.__qualname__ = name + Temp.__module__ = __name__ + assert name not in globals() + globals()[name] = Temp + +install_tests() + +def setUpModule(): + from test import test_xml_etree + test_xml_etree.setUpModule(module=cET) + + +if __name__ == '__main__': + unittest.main() diff --git a/Lib/test/test_xmlrpc.py b/Lib/test/test_xmlrpc.py index 25a4be0e03..8684e042be 100644 --- a/Lib/test/test_xmlrpc.py +++ b/Lib/test/test_xmlrpc.py @@ -25,6 +25,8 @@ except ImportError: gzip = None +support.requires_working_socket(module=True) + alist = [{'astring': 'foo@bar.baz.spam', 'afloat': 7283.43, 'anint': 2**20, @@ -45,15 +47,13 @@ class XMLRPCTestCase(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_dump_load(self): dump = xmlrpclib.dumps((alist,)) load = xmlrpclib.loads(dump) self.assertEqual(alist, load[0][0]) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_dump_bare_datetime(self): # This checks that an unwrapped datetime.date object can be handled # by the marshalling code. This can't be done via test_dump_load() @@ -88,8 +88,7 @@ def test_dump_bare_datetime(self): self.assertIsNone(m) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_datetime_before_1900(self): # same as before but with a date before 1900 dt = datetime.datetime(1, 2, 10, 11, 41, 23) @@ -108,8 +107,7 @@ def test_datetime_before_1900(self): self.assertIs(type(newdt), xmlrpclib.DateTime) self.assertIsNone(m) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_bug_1164912 (self): d = xmlrpclib.DateTime() ((new_d,), dummy) = xmlrpclib.loads(xmlrpclib.dumps((d,), @@ -120,8 +118,7 @@ def test_bug_1164912 (self): s = xmlrpclib.dumps((new_d,), methodresponse=True) self.assertIsInstance(s, str) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_newstyle_class(self): class T(object): pass @@ -187,8 +184,7 @@ def dummy_write(s): m.dump_double(xmlrpclib.MAXINT + 42, dummy_write) m.dump_double(xmlrpclib.MININT - 42, dummy_write) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_dump_none(self): value = alist + [None] arg1 = (alist + [None],) @@ -197,8 +193,7 @@ def test_dump_none(self): xmlrpclib.loads(strg)[0][0]) self.assertRaises(TypeError, xmlrpclib.dumps, (arg1,)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_dump_encoding(self): value = {'key\u20ac\xa4': 'value\u20ac\xa4'} @@ -220,8 +215,7 @@ def test_dump_encoding(self): self.assertEqual(xmlrpclib.loads(strg)[0][0], value) self.assertEqual(xmlrpclib.loads(strg)[1], methodname) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_dump_bytes(self): sample = b"my dog has fleas" self.assertEqual(sample, xmlrpclib.Binary(sample)) @@ -241,8 +235,7 @@ def test_dump_bytes(self): self.assertIs(type(newvalue), xmlrpclib.Binary) self.assertIsNone(m) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_loads_unsupported(self): ResponseError = xmlrpclib.ResponseError data = '' @@ -265,8 +258,7 @@ def check_loads(self, s, value, **kwargs): self.assertIs(type(newvalue), type(value)) self.assertIsNone(m) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_load_standard_types(self): check = self.check_loads check('string', 'string') @@ -294,8 +286,7 @@ def test_load_standard_types(self): 'a1' '', {'a': 1, 'b': 2}) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_load_extension_types(self): check = self.check_loads check('', None) @@ -309,6 +300,17 @@ def test_load_extension_types(self): check('9876543210.0123456789', decimal.Decimal('9876543210.0123456789')) + @unittest.expectedFailure # TODO: RUSTPYTHON; NameError: name 'expat' is not defined + def test_limit_int(self): + check = self.check_loads + maxdigits = 5000 + with support.adjust_int_max_str_digits(maxdigits): + s = '1' * (maxdigits + 1) + with self.assertRaises(ValueError): + check(f'{s}', None) + with self.assertRaises(ValueError): + check(f'{s}', None) + def test_get_host_info(self): # see bug #3613, this raised a TypeError transp = xmlrpc.client.Transport() @@ -330,8 +332,7 @@ def test_ssl_presence(self): except OSError: self.assertTrue(has_ssl) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_keepalive_disconnect(self): class RequestHandler(http.server.BaseHTTPRequestHandler): protocol_version = "HTTP/1.1" @@ -471,8 +472,7 @@ def test_repr(self): self.assertEqual(repr(f), "") self.assertEqual(repr(f), str(f)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_dump_fault(self): f = xmlrpclib.Fault(42, 'Test Fault') s = xmlrpclib.dumps((f,)) @@ -518,10 +518,16 @@ def test_time_struct(self): self.assertEqual(str(t), time.strftime("%Y%m%dT%H:%M:%S", d)) def test_datetime_datetime(self): + # naive (no tzinfo) d = datetime.datetime(2007,1,2,3,4,5) t = xmlrpclib.DateTime(d) self.assertEqual(str(t), '20070102T03:04:05') + # aware (with tzinfo): the timezone is ignored + d = datetime.datetime(2023, 6, 12, 13, 30, tzinfo=datetime.UTC) + t = xmlrpclib.DateTime(d) + self.assertEqual(str(t), '20230612T13:30:00') + def test_repr(self): d = datetime.datetime(2007,1,2,3,4,5) t = xmlrpclib.DateTime(d) @@ -814,8 +820,7 @@ def tearDown(self): xmlrpc.server.SimpleXMLRPCServer._send_traceback_header = False class SimpleServerTestCase(BaseServerTestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_simple1(self): try: p = xmlrpclib.ServerProxy(URL) @@ -826,8 +831,7 @@ def test_simple1(self): # protocol error; provide additional information in test output self.fail("%s\n%s" % (e, getattr(e, "headers", ""))) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_nonascii(self): start_string = 'P\N{LATIN SMALL LETTER Y WITH CIRCUMFLEX}t' end_string = 'h\N{LATIN SMALL LETTER O WITH HORN}n' @@ -841,8 +845,7 @@ def test_nonascii(self): # protocol error; provide additional information in test output self.fail("%s\n%s" % (e, getattr(e, "headers", ""))) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_client_encoding(self): start_string = '\u20ac' end_string = '\xa4' @@ -857,8 +860,7 @@ def test_client_encoding(self): # protocol error; provide additional information in test output self.fail("%s\n%s" % (e, getattr(e, "headers", ""))) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_nonascii_methodname(self): try: p = xmlrpclib.ServerProxy(URL, encoding='ascii') @@ -879,8 +881,7 @@ def test_404(self): self.assertEqual(response.status, 404) self.assertEqual(response.reason, 'Not Found') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_introspection1(self): expected_methods = set(['pow', 'div', 'my_function', 'add', 'têšt', 'system.listMethods', 'system.methodHelp', @@ -897,8 +898,7 @@ def test_introspection1(self): self.fail("%s\n%s" % (e, getattr(e, "headers", ""))) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_introspection2(self): try: # test _methodHelp() @@ -911,8 +911,7 @@ def test_introspection2(self): # protocol error; provide additional information in test output self.fail("%s\n%s" % (e, getattr(e, "headers", ""))) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON @make_request_and_skipIf(sys.flags.optimize >= 2, "Docstrings are omitted with -O2 and above") def test_introspection3(self): @@ -927,8 +926,7 @@ def test_introspection3(self): # protocol error; provide additional information in test output self.fail("%s\n%s" % (e, getattr(e, "headers", ""))) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_introspection4(self): # the SimpleXMLRPCServer doesn't support signatures, but # at least check that we can try making the call @@ -942,8 +940,7 @@ def test_introspection4(self): # protocol error; provide additional information in test output self.fail("%s\n%s" % (e, getattr(e, "headers", ""))) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_multicall(self): try: p = xmlrpclib.ServerProxy(URL) @@ -961,8 +958,7 @@ def test_multicall(self): # protocol error; provide additional information in test output self.fail("%s\n%s" % (e, getattr(e, "headers", ""))) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_non_existing_multicall(self): try: p = xmlrpclib.ServerProxy(URL) @@ -984,8 +980,7 @@ def test_non_existing_multicall(self): # protocol error; provide additional information in test output self.fail("%s\n%s" % (e, getattr(e, "headers", ""))) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_dotted_attribute(self): # Raises an AttributeError because private methods are not allowed. self.assertRaises(AttributeError, @@ -996,16 +991,14 @@ def test_dotted_attribute(self): # This avoids waiting for the socket timeout. self.test_simple1() - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_allow_dotted_names_true(self): # XXX also need allow_dotted_names_false test. server = xmlrpclib.ServerProxy("http://%s:%d/RPC2" % (ADDR, PORT)) data = server.Fixture.getData() self.assertEqual(data, '42') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_unicode_host(self): server = xmlrpclib.ServerProxy("http://%s:%d/RPC2" % (ADDR, PORT)) self.assertEqual(server.add("a", "\xe9"), "a\xe9") @@ -1020,8 +1013,7 @@ def test_partial_post(self): 'Accept-Encoding: identity\r\n' 'Content-Length: 0\r\n\r\n'.encode('ascii')) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_context_manager(self): with xmlrpclib.ServerProxy(URL) as server: server.add(2, 3) @@ -1030,8 +1022,7 @@ def test_context_manager(self): self.assertEqual(server('transport')._connection, (None, None)) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_context_manager_method_error(self): try: with xmlrpclib.ServerProxy(URL) as server: @@ -1047,8 +1038,7 @@ class SimpleServerEncodingTestCase(BaseServerTestCase): def threadFunc(evt, numrequests, requestHandler=None, encoding=None): http_server(evt, numrequests, requestHandler, 'iso-8859-15') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_server_encoding(self): start_string = '\u20ac' end_string = '\xa4' @@ -1067,70 +1057,68 @@ def test_server_encoding(self): class MultiPathServerTestCase(BaseServerTestCase): threadFunc = staticmethod(http_multi_server) request_count = 2 - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_path1(self): p = xmlrpclib.ServerProxy(URL+"/foo") self.assertEqual(p.pow(6,8), 6**8) self.assertRaises(xmlrpclib.Fault, p.add, 6, 8) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_path2(self): p = xmlrpclib.ServerProxy(URL+"/foo/bar") self.assertEqual(p.add(6,8), 6+8) self.assertRaises(xmlrpclib.Fault, p.pow, 6, 8) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON + @support.requires_resource('walltime') def test_path3(self): p = xmlrpclib.ServerProxy(URL+"/is/broken") self.assertRaises(xmlrpclib.Fault, p.add, 6, 8) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON + @support.requires_resource('walltime') def test_invalid_path(self): p = xmlrpclib.ServerProxy(URL+"/invalid") self.assertRaises(xmlrpclib.Fault, p.add, 6, 8) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON + @support.requires_resource('walltime') def test_path_query_fragment(self): p = xmlrpclib.ServerProxy(URL+"/foo?k=v#frag") self.assertEqual(p.test(), "/foo?k=v#frag") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON + @support.requires_resource('walltime') def test_path_fragment(self): p = xmlrpclib.ServerProxy(URL+"/foo#frag") self.assertEqual(p.test(), "/foo#frag") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON + @support.requires_resource('walltime') def test_path_query(self): p = xmlrpclib.ServerProxy(URL+"/foo?k=v") self.assertEqual(p.test(), "/foo?k=v") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON + @support.requires_resource('walltime') def test_empty_path(self): p = xmlrpclib.ServerProxy(URL) self.assertEqual(p.test(), "/RPC2") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON + @support.requires_resource('walltime') def test_root_path(self): p = xmlrpclib.ServerProxy(URL + "/") self.assertEqual(p.test(), "/") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON + @support.requires_resource('walltime') def test_empty_path_query(self): p = xmlrpclib.ServerProxy(URL + "?k=v") self.assertEqual(p.test(), "?k=v") - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON + @support.requires_resource('walltime') def test_empty_path_fragment(self): p = xmlrpclib.ServerProxy(URL + "#frag") self.assertEqual(p.test(), "#frag") @@ -1163,8 +1151,7 @@ def setUp(self): #A test case that verifies that a server using the HTTP/1.1 keep-alive mechanism #does indeed serve subsequent requests on the same connection class KeepaliveServerTestCase1(BaseKeepaliveServerTestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_two(self): p = xmlrpclib.ServerProxy(URL) #do three requests. @@ -1181,9 +1168,9 @@ def test_two(self): self.assertGreaterEqual(len(self.RequestHandler.myRequests[-1]), 2) -@unittest.skip("TODO: RUSTPYTHON, appears to hang") #test special attribute access on the serverproxy, through the __call__ #function. +@unittest.skip("TODO: RUSTPYTHON, appears to hang") class KeepaliveServerTestCase2(BaseKeepaliveServerTestCase): #ask for two keepalive requests to be handled. request_count=2 @@ -1248,8 +1235,7 @@ def send_content(self, connection, body): def setUp(self): BaseServerTestCase.setUp(self) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_gzip_request(self): t = self.Transport() t.encode_threshold = None @@ -1273,8 +1259,7 @@ def test_bad_gzip_request(self): p.pow(6, 8) p("close")() - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_gzip_response(self): t = self.Transport() p = xmlrpclib.ServerProxy(URL, transport=t) @@ -1333,8 +1318,7 @@ def assertContainsAdditionalHeaders(self, headers, additional): for key, value in additional.items(): self.assertEqual(headers.get(key), value) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_header(self): p = xmlrpclib.ServerProxy(URL, headers=[('X-Test', 'foo')]) self.assertEqual(p.pow(6, 8), 6**8) @@ -1342,8 +1326,7 @@ def test_header(self): headers = self.RequestHandler.test_headers self.assertContainsAdditionalHeaders(headers, {'X-Test': 'foo'}) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_header_many(self): p = xmlrpclib.ServerProxy( URL, headers=[('X-Test', 'foo'), ('X-Test-Second', 'bar')]) @@ -1353,8 +1336,7 @@ def test_header_many(self): self.assertContainsAdditionalHeaders( headers, {'X-Test': 'foo', 'X-Test-Second': 'bar'}) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_header_empty(self): p = xmlrpclib.ServerProxy(URL, headers=[]) self.assertEqual(p.pow(6, 8), 6**8) @@ -1362,8 +1344,7 @@ def test_header_empty(self): headers = self.RequestHandler.test_headers self.assertContainsAdditionalHeaders(headers, {}) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_header_tuple(self): p = xmlrpclib.ServerProxy(URL, headers=(('X-Test', 'foo'),)) self.assertEqual(p.pow(6, 8), 6**8) @@ -1371,8 +1352,7 @@ def test_header_tuple(self): headers = self.RequestHandler.test_headers self.assertContainsAdditionalHeaders(headers, {'X-Test': 'foo'}) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_header_items(self): p = xmlrpclib.ServerProxy(URL, headers={'X-Test': 'foo'}.items()) self.assertEqual(p.pow(6, 8), 6**8) @@ -1431,8 +1411,7 @@ def tearDown(self): default_class = http.client.HTTPMessage xmlrpc.server.SimpleXMLRPCRequestHandler.MessageClass = default_class - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_basic(self): # check that flag is false by default flagval = xmlrpc.server.SimpleXMLRPCServer._send_traceback_header @@ -1527,8 +1506,7 @@ def test_cgi_get(self): self.assertEqual(message, 'Bad Request') - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_cgi_xmlrpc_response(self): data = """ @@ -1574,8 +1552,7 @@ def test_cgi_xmlrpc_response(self): class UseBuiltinTypesTestCase(unittest.TestCase): - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_use_builtin_types(self): # SimpleXMLRPCDispatcher.__init__ accepts use_builtin_types, which # makes all dispatch of binary data as bytes instances, and all diff --git a/Lib/test/test_yield_from.py b/Lib/test/test_yield_from.py index 97acfd5413..88fa1b88c9 100644 --- a/Lib/test/test_yield_from.py +++ b/Lib/test/test_yield_from.py @@ -538,8 +538,7 @@ def g(): "finishing g", ]) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_broken_getattr_handling(self): """ Test subiterator with a broken getattr implementation @@ -787,8 +786,7 @@ def outer(): repr(value), ]) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_throwing_GeneratorExit_into_subgen_that_returns(self): """ Test throwing GeneratorExit into a subgenerator that @@ -819,8 +817,7 @@ def g(): "Enter f", ]) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_throwing_GeneratorExit_into_subgenerator_that_yields(self): """ Test throwing GeneratorExit into a subgenerator that @@ -887,8 +884,7 @@ def g(): yield from () self.assertRaises(StopIteration, next, g()) - # TODO: RUSTPYTHON - @unittest.expectedFailure + @unittest.expectedFailure # TODO: RUSTPYTHON def test_delegating_generators_claim_to_be_running(self): # Check with basic iteration def one(): @@ -904,6 +900,7 @@ def two(): yield 2 g1 = one() self.assertEqual(list(g1), [0, 1, 2, 3]) + # Check with send g1 = one() res = [next(g1)] @@ -913,6 +910,9 @@ def two(): except StopIteration: pass self.assertEqual(res, [0, 1, 2, 3]) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: Lists differ: [0, 1, 2] != [0, 1, 2, 3] + def test_delegating_generators_claim_to_be_running_with_throw(self): # Check with throw class MyErr(Exception): pass @@ -949,8 +949,10 @@ def two(): except: self.assertEqual(res, [0, 1, 2, 3]) raise + + def test_delegating_generators_claim_to_be_running_with_close(self): # Check with close - class MyIt(object): + class MyIt: def __iter__(self): return self def __next__(self): @@ -1057,6 +1059,538 @@ def outer(): g.send((1, 2, 3, 4)) self.assertEqual(v, (1, 2, 3, 4)) +class TestInterestingEdgeCases(unittest.TestCase): + + def assert_stop_iteration(self, iterator): + with self.assertRaises(StopIteration) as caught: + next(iterator) + self.assertIsNone(caught.exception.value) + self.assertIsNone(caught.exception.__context__) + + def assert_generator_raised_stop_iteration(self): + return self.assertRaisesRegex(RuntimeError, r"^generator raised StopIteration$") + + def assert_generator_ignored_generator_exit(self): + return self.assertRaisesRegex(RuntimeError, r"^generator ignored GeneratorExit$") + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_close_and_throw_work(self): + + yielded_first = object() + yielded_second = object() + returned = object() + + def inner(): + yield yielded_first + yield yielded_second + return returned + + def outer(): + return (yield from inner()) + + with self.subTest("close"): + g = outer() + self.assertIs(next(g), yielded_first) + g.close() + self.assert_stop_iteration(g) + + with self.subTest("throw GeneratorExit"): + g = outer() + self.assertIs(next(g), yielded_first) + thrown = GeneratorExit() + with self.assertRaises(GeneratorExit) as caught: + g.throw(thrown) + self.assertIs(caught.exception, thrown) + self.assertIsNone(caught.exception.__context__) + self.assert_stop_iteration(g) + + with self.subTest("throw StopIteration"): + g = outer() + self.assertIs(next(g), yielded_first) + thrown = StopIteration() + # PEP 479: + with self.assert_generator_raised_stop_iteration() as caught: + g.throw(thrown) + self.assertIs(caught.exception.__context__, thrown) + self.assertIsNone(caught.exception.__context__.__context__) + self.assert_stop_iteration(g) + + with self.subTest("throw BaseException"): + g = outer() + self.assertIs(next(g), yielded_first) + thrown = BaseException() + with self.assertRaises(BaseException) as caught: + g.throw(thrown) + self.assertIs(caught.exception, thrown) + self.assertIsNone(caught.exception.__context__) + self.assert_stop_iteration(g) + + with self.subTest("throw Exception"): + g = outer() + self.assertIs(next(g), yielded_first) + thrown = Exception() + with self.assertRaises(Exception) as caught: + g.throw(thrown) + self.assertIs(caught.exception, thrown) + self.assertIsNone(caught.exception.__context__) + self.assert_stop_iteration(g) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: GeneratorExit() is not GeneratorExit() + def test_close_and_throw_raise_generator_exit(self): + + yielded_first = object() + yielded_second = object() + returned = object() + + def inner(): + try: + yield yielded_first + yield yielded_second + return returned + finally: + raise raised + + def outer(): + return (yield from inner()) + + with self.subTest("close"): + g = outer() + self.assertIs(next(g), yielded_first) + raised = GeneratorExit() + # GeneratorExit is suppressed. This is consistent with PEP 342: + # https://peps.python.org/pep-0342/#new-generator-method-close + g.close() + self.assert_stop_iteration(g) + + with self.subTest("throw GeneratorExit"): + g = outer() + self.assertIs(next(g), yielded_first) + raised = GeneratorExit() + thrown = GeneratorExit() + with self.assertRaises(GeneratorExit) as caught: + g.throw(thrown) + # The raised GeneratorExit is suppressed, but the thrown one + # propagates. This is consistent with PEP 380: + # https://peps.python.org/pep-0380/#proposal + self.assertIs(caught.exception, thrown) + self.assertIsNone(caught.exception.__context__) + self.assert_stop_iteration(g) + + with self.subTest("throw StopIteration"): + g = outer() + self.assertIs(next(g), yielded_first) + raised = GeneratorExit() + thrown = StopIteration() + with self.assertRaises(GeneratorExit) as caught: + g.throw(thrown) + self.assertIs(caught.exception, raised) + self.assertIs(caught.exception.__context__, thrown) + self.assertIsNone(caught.exception.__context__.__context__) + self.assert_stop_iteration(g) + + with self.subTest("throw BaseException"): + g = outer() + self.assertIs(next(g), yielded_first) + raised = GeneratorExit() + thrown = BaseException() + with self.assertRaises(GeneratorExit) as caught: + g.throw(thrown) + self.assertIs(caught.exception, raised) + self.assertIs(caught.exception.__context__, thrown) + self.assertIsNone(caught.exception.__context__.__context__) + self.assert_stop_iteration(g) + + with self.subTest("throw Exception"): + g = outer() + self.assertIs(next(g), yielded_first) + raised = GeneratorExit() + thrown = Exception() + with self.assertRaises(GeneratorExit) as caught: + g.throw(thrown) + self.assertIs(caught.exception, raised) + self.assertIs(caught.exception.__context__, thrown) + self.assertIsNone(caught.exception.__context__.__context__) + self.assert_stop_iteration(g) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: RuntimeError not raised + def test_close_and_throw_raise_stop_iteration(self): + + yielded_first = object() + yielded_second = object() + returned = object() + + def inner(): + try: + yield yielded_first + yield yielded_second + return returned + finally: + raise raised + + def outer(): + return (yield from inner()) + + with self.subTest("close"): + g = outer() + self.assertIs(next(g), yielded_first) + raised = StopIteration() + # PEP 479: + with self.assert_generator_raised_stop_iteration() as caught: + g.close() + self.assertIs(caught.exception.__context__, raised) + self.assertIsInstance(caught.exception.__context__.__context__, GeneratorExit) + self.assertIsNone(caught.exception.__context__.__context__.__context__) + self.assert_stop_iteration(g) + + with self.subTest("throw GeneratorExit"): + g = outer() + self.assertIs(next(g), yielded_first) + raised = StopIteration() + thrown = GeneratorExit() + # PEP 479: + with self.assert_generator_raised_stop_iteration() as caught: + g.throw(thrown) + self.assertIs(caught.exception.__context__, raised) + # This isn't the same GeneratorExit as thrown! It's the one created + # by calling inner.close(): + self.assertIsInstance(caught.exception.__context__.__context__, GeneratorExit) + self.assertIsNone(caught.exception.__context__.__context__.__context__) + self.assert_stop_iteration(g) + + with self.subTest("throw StopIteration"): + g = outer() + self.assertIs(next(g), yielded_first) + raised = StopIteration() + thrown = StopIteration() + # PEP 479: + with self.assert_generator_raised_stop_iteration() as caught: + g.throw(thrown) + self.assertIs(caught.exception.__context__, raised) + self.assertIs(caught.exception.__context__.__context__, thrown) + self.assertIsNone(caught.exception.__context__.__context__.__context__) + self.assert_stop_iteration(g) + + with self.subTest("throw BaseException"): + g = outer() + self.assertIs(next(g), yielded_first) + raised = StopIteration() + thrown = BaseException() + # PEP 479: + with self.assert_generator_raised_stop_iteration() as caught: + g.throw(thrown) + self.assertIs(caught.exception.__context__, raised) + self.assertIs(caught.exception.__context__.__context__, thrown) + self.assertIsNone(caught.exception.__context__.__context__.__context__) + self.assert_stop_iteration(g) + + with self.subTest("throw Exception"): + g = outer() + self.assertIs(next(g), yielded_first) + raised = StopIteration() + thrown = Exception() + # PEP 479: + with self.assert_generator_raised_stop_iteration() as caught: + g.throw(thrown) + self.assertIs(caught.exception.__context__, raised) + self.assertIs(caught.exception.__context__.__context__, thrown) + self.assertIsNone(caught.exception.__context__.__context__.__context__) + self.assert_stop_iteration(g) + + def test_close_and_throw_raise_base_exception(self): + + yielded_first = object() + yielded_second = object() + returned = object() + + def inner(): + try: + yield yielded_first + yield yielded_second + return returned + finally: + raise raised + + def outer(): + return (yield from inner()) + + with self.subTest("close"): + g = outer() + self.assertIs(next(g), yielded_first) + raised = BaseException() + with self.assertRaises(BaseException) as caught: + g.close() + self.assertIs(caught.exception, raised) + self.assertIsInstance(caught.exception.__context__, GeneratorExit) + self.assertIsNone(caught.exception.__context__.__context__) + self.assert_stop_iteration(g) + + with self.subTest("throw GeneratorExit"): + g = outer() + self.assertIs(next(g), yielded_first) + raised = BaseException() + thrown = GeneratorExit() + with self.assertRaises(BaseException) as caught: + g.throw(thrown) + self.assertIs(caught.exception, raised) + # This isn't the same GeneratorExit as thrown! It's the one created + # by calling inner.close(): + self.assertIsInstance(caught.exception.__context__, GeneratorExit) + self.assertIsNone(caught.exception.__context__.__context__) + self.assert_stop_iteration(g) + + with self.subTest("throw StopIteration"): + g = outer() + self.assertIs(next(g), yielded_first) + raised = BaseException() + thrown = StopIteration() + with self.assertRaises(BaseException) as caught: + g.throw(thrown) + self.assertIs(caught.exception, raised) + self.assertIs(caught.exception.__context__, thrown) + self.assertIsNone(caught.exception.__context__.__context__) + self.assert_stop_iteration(g) + + with self.subTest("throw BaseException"): + g = outer() + self.assertIs(next(g), yielded_first) + raised = BaseException() + thrown = BaseException() + with self.assertRaises(BaseException) as caught: + g.throw(thrown) + self.assertIs(caught.exception, raised) + self.assertIs(caught.exception.__context__, thrown) + self.assertIsNone(caught.exception.__context__.__context__) + self.assert_stop_iteration(g) + + with self.subTest("throw Exception"): + g = outer() + self.assertIs(next(g), yielded_first) + raised = BaseException() + thrown = Exception() + with self.assertRaises(BaseException) as caught: + g.throw(thrown) + self.assertIs(caught.exception, raised) + self.assertIs(caught.exception.__context__, thrown) + self.assertIsNone(caught.exception.__context__.__context__) + self.assert_stop_iteration(g) + + def test_close_and_throw_raise_exception(self): + + yielded_first = object() + yielded_second = object() + returned = object() + + def inner(): + try: + yield yielded_first + yield yielded_second + return returned + finally: + raise raised + + def outer(): + return (yield from inner()) + + with self.subTest("close"): + g = outer() + self.assertIs(next(g), yielded_first) + raised = Exception() + with self.assertRaises(Exception) as caught: + g.close() + self.assertIs(caught.exception, raised) + self.assertIsInstance(caught.exception.__context__, GeneratorExit) + self.assertIsNone(caught.exception.__context__.__context__) + self.assert_stop_iteration(g) + + with self.subTest("throw GeneratorExit"): + g = outer() + self.assertIs(next(g), yielded_first) + raised = Exception() + thrown = GeneratorExit() + with self.assertRaises(Exception) as caught: + g.throw(thrown) + self.assertIs(caught.exception, raised) + # This isn't the same GeneratorExit as thrown! It's the one created + # by calling inner.close(): + self.assertIsInstance(caught.exception.__context__, GeneratorExit) + self.assertIsNone(caught.exception.__context__.__context__) + self.assert_stop_iteration(g) + + with self.subTest("throw StopIteration"): + g = outer() + self.assertIs(next(g), yielded_first) + raised = Exception() + thrown = StopIteration() + with self.assertRaises(Exception) as caught: + g.throw(thrown) + self.assertIs(caught.exception, raised) + self.assertIs(caught.exception.__context__, thrown) + self.assertIsNone(caught.exception.__context__.__context__) + self.assert_stop_iteration(g) + + with self.subTest("throw BaseException"): + g = outer() + self.assertIs(next(g), yielded_first) + raised = Exception() + thrown = BaseException() + with self.assertRaises(Exception) as caught: + g.throw(thrown) + self.assertIs(caught.exception, raised) + self.assertIs(caught.exception.__context__, thrown) + self.assertIsNone(caught.exception.__context__.__context__) + self.assert_stop_iteration(g) + + with self.subTest("throw Exception"): + g = outer() + self.assertIs(next(g), yielded_first) + raised = Exception() + thrown = Exception() + with self.assertRaises(Exception) as caught: + g.throw(thrown) + self.assertIs(caught.exception, raised) + self.assertIs(caught.exception.__context__, thrown) + self.assertIsNone(caught.exception.__context__.__context__) + self.assert_stop_iteration(g) + + @unittest.expectedFailure # TODO: RUSTPYTHON; AssertionError: None is not StopIteration() + def test_close_and_throw_yield(self): + + yielded_first = object() + yielded_second = object() + returned = object() + + def inner(): + try: + yield yielded_first + finally: + yield yielded_second + return returned + + def outer(): + return (yield from inner()) + + with self.subTest("close"): + g = outer() + self.assertIs(next(g), yielded_first) + # No chaining happens. This is consistent with PEP 342: + # https://peps.python.org/pep-0342/#new-generator-method-close + with self.assert_generator_ignored_generator_exit() as caught: + g.close() + self.assertIsNone(caught.exception.__context__) + self.assert_stop_iteration(g) + + with self.subTest("throw GeneratorExit"): + g = outer() + self.assertIs(next(g), yielded_first) + thrown = GeneratorExit() + # No chaining happens. This is consistent with PEP 342: + # https://peps.python.org/pep-0342/#new-generator-method-close + with self.assert_generator_ignored_generator_exit() as caught: + g.throw(thrown) + self.assertIsNone(caught.exception.__context__) + self.assert_stop_iteration(g) + + with self.subTest("throw StopIteration"): + g = outer() + self.assertIs(next(g), yielded_first) + thrown = StopIteration() + self.assertEqual(g.throw(thrown), yielded_second) + # PEP 479: + with self.assert_generator_raised_stop_iteration() as caught: + next(g) + self.assertIs(caught.exception.__context__, thrown) + self.assertIsNone(caught.exception.__context__.__context__) + self.assert_stop_iteration(g) + + with self.subTest("throw BaseException"): + g = outer() + self.assertIs(next(g), yielded_first) + thrown = BaseException() + self.assertEqual(g.throw(thrown), yielded_second) + with self.assertRaises(BaseException) as caught: + next(g) + self.assertIs(caught.exception, thrown) + self.assertIsNone(caught.exception.__context__) + self.assert_stop_iteration(g) + + with self.subTest("throw Exception"): + g = outer() + self.assertIs(next(g), yielded_first) + thrown = Exception() + self.assertEqual(g.throw(thrown), yielded_second) + with self.assertRaises(Exception) as caught: + next(g) + self.assertIs(caught.exception, thrown) + self.assertIsNone(caught.exception.__context__) + self.assert_stop_iteration(g) + + @unittest.expectedFailure # TODO: RUSTPYTHON + def test_close_and_throw_return(self): + + yielded_first = object() + yielded_second = object() + returned = object() + + def inner(): + try: + yield yielded_first + yield yielded_second + finally: + return returned + + def outer(): + return (yield from inner()) + + with self.subTest("close"): + g = outer() + self.assertIs(next(g), yielded_first) + # StopIteration is suppressed. This is consistent with PEP 342: + # https://peps.python.org/pep-0342/#new-generator-method-close + g.close() + self.assert_stop_iteration(g) + + with self.subTest("throw GeneratorExit"): + g = outer() + self.assertIs(next(g), yielded_first) + thrown = GeneratorExit() + # StopIteration is suppressed. This is consistent with PEP 342: + # https://peps.python.org/pep-0342/#new-generator-method-close + with self.assertRaises(GeneratorExit) as caught: + g.throw(thrown) + self.assertIs(caught.exception, thrown) + self.assertIsNone(caught.exception.__context__) + self.assert_stop_iteration(g) + + with self.subTest("throw StopIteration"): + g = outer() + self.assertIs(next(g), yielded_first) + thrown = StopIteration() + with self.assertRaises(StopIteration) as caught: + g.throw(thrown) + self.assertIs(caught.exception.value, returned) + self.assertIsNone(caught.exception.__context__) + self.assert_stop_iteration(g) + + with self.subTest("throw BaseException"): + g = outer() + self.assertIs(next(g), yielded_first) + thrown = BaseException() + with self.assertRaises(StopIteration) as caught: + g.throw(thrown) + self.assertIs(caught.exception.value, returned) + self.assertIsNone(caught.exception.__context__) + self.assert_stop_iteration(g) + + with self.subTest("throw Exception"): + g = outer() + self.assertIs(next(g), yielded_first) + thrown = Exception() + with self.assertRaises(StopIteration) as caught: + g.throw(thrown) + self.assertIs(caught.exception.value, returned) + self.assertIsNone(caught.exception.__context__) + self.assert_stop_iteration(g) + if __name__ == '__main__': unittest.main() diff --git a/Lib/test/test_zipapp.py b/Lib/test/test_zipapp.py index 69f2e55d56..ad13283962 100644 --- a/Lib/test/test_zipapp.py +++ b/Lib/test/test_zipapp.py @@ -9,6 +9,7 @@ import zipapp import zipfile from test.support import requires_zlib +from test.support import os_helper from unittest.mock import patch @@ -54,6 +55,22 @@ def test_create_archive_with_subdirs(self): self.assertIn('foo/', z.namelist()) self.assertIn('bar/', z.namelist()) + def test_create_sorted_archive(self): + # Test that zipapps order their files by name + source = self.tmpdir / 'source' + source.mkdir() + (source / 'zed.py').touch() + (source / 'bin').mkdir() + (source / 'bin' / 'qux').touch() + (source / 'bin' / 'baz').touch() + (source / '__main__.py').touch() + target = io.BytesIO() + zipapp.create_archive(str(source), target) + target.seek(0) + with zipfile.ZipFile(target, 'r') as zf: + self.assertEqual(zf.namelist(), + ["__main__.py", "bin/", "bin/baz", "bin/qux", "zed.py"]) + def test_create_archive_with_filter(self): # Test packing a directory and using filter to specify # which files to include. @@ -72,6 +89,30 @@ def skip_pyc_files(path): self.assertIn('test.py', z.namelist()) self.assertNotIn('test.pyc', z.namelist()) + def test_create_archive_self_insertion(self): + # When creating an archive, we shouldn't + # include the archive in the list of files to add. + source = self.tmpdir + (source / '__main__.py').touch() + (source / 'test.py').touch() + target = self.tmpdir / 'target.pyz' + + zipapp.create_archive(source, target) + with zipfile.ZipFile(target, 'r') as z: + self.assertEqual(len(z.namelist()), 2) + self.assertIn('__main__.py', z.namelist()) + self.assertIn('test.py', z.namelist()) + + def test_target_overwrites_source_file(self): + # The target cannot be one of the files to add. + source = self.tmpdir + (source / '__main__.py').touch() + target = source / 'target.pyz' + target.touch() + + with self.assertRaises(zipapp.ZipAppError): + zipapp.create_archive(source, target) + def test_create_archive_filter_exclude_dir(self): # Test packing a directory and using a filter to exclude a # subdirectory (ensures that the path supplied to include @@ -248,14 +289,15 @@ def test_write_shebang_to_fileobj(self): zipapp.create_archive(str(target), new_target, interpreter='python2.7') self.assertTrue(new_target.getvalue().startswith(b'#!python2.7\n')) - def test_read_from_pathobj(self): - # Test that we can copy an archive using a pathlib.Path object + def test_read_from_pathlike_obj(self): + # Test that we can copy an archive using a path-like object # for the source. source = self.tmpdir / 'source' source.mkdir() (source / '__main__.py').touch() - target1 = self.tmpdir / 'target1.pyz' - target2 = self.tmpdir / 'target2.pyz' + source = os_helper.FakePath(str(source)) + target1 = os_helper.FakePath(str(self.tmpdir / 'target1.pyz')) + target2 = os_helper.FakePath(str(self.tmpdir / 'target2.pyz')) zipapp.create_archive(source, target1, interpreter='python') zipapp.create_archive(target1, target2, interpreter='python2.7') self.assertEqual(zipapp.get_interpreter(target2), 'python2.7') @@ -301,6 +343,7 @@ def test_content_of_copied_archive(self): # (Unix only) tests that archives with shebang lines are made executable @unittest.skipIf(sys.platform == 'win32', 'Windows does not support an executable bit') + @os_helper.skip_unless_working_chmod def test_shebang_is_executable(self): # Test that an archive with a shebang line is made executable. source = self.tmpdir / 'source' diff --git a/Lib/test/test_zipfile.py b/Lib/test/test_zipfile.py index 43178ca26b..e7705590cd 100644 --- a/Lib/test/test_zipfile.py +++ b/Lib/test/test_zipfile.py @@ -123,8 +123,6 @@ def zip_test(self, f, compression, compresslevel=None): # Check that testzip doesn't raise an exception zipfp.testzip() - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_basic(self): for f in get_files(self): self.zip_test(f, self.compression) @@ -394,8 +392,6 @@ def test_repr(self): self.assertIn('[closed]', repr(zipopen)) self.assertIn('[closed]', repr(zipfp)) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_compresslevel_basic(self): for f in get_files(self): self.zip_test(f, self.compression, compresslevel=9) @@ -751,8 +747,6 @@ def zip_test(self, f, compression): # Check that testzip doesn't raise an exception zipfp.testzip() - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_basic(self): for f in get_files(self): self.zip_test(f, self.compression) @@ -1752,8 +1746,6 @@ def test_empty_file_raises_BadZipFile(self): fp.write("short file") self.assertRaises(zipfile.BadZipFile, zipfile.ZipFile, TESTFN) - # TODO: RUSTPYTHON - @unittest.expectedFailure def test_negative_central_directory_offset_raises_BadZipFile(self): # Zip file containing an empty EOCD record buffer = bytearray(b'PK\x05\x06' + b'\0'*18) diff --git a/Lib/test/test_zipfile64.py b/Lib/test/test_zipfile64.py index 0947013afb..2e1affe025 100644 --- a/Lib/test/test_zipfile64.py +++ b/Lib/test/test_zipfile64.py @@ -11,7 +11,7 @@ 'test requires loads of disk-space bytes and a long time to run' ) -import zipfile, os, unittest +import zipfile, unittest import time import sys @@ -32,10 +32,6 @@ def setUp(self): line_gen = ("Test of zipfile line %d." % i for i in range(1000000)) self.data = '\n'.join(line_gen).encode('ascii') - # And write it to a file. - with open(TESTFN, "wb") as fp: - fp.write(self.data) - def zipTest(self, f, compression): # Create the ZIP archive. with zipfile.ZipFile(f, "w", compression) as zipfp: @@ -67,6 +63,9 @@ def zipTest(self, f, compression): (num, filecount)), file=sys.__stdout__) sys.__stdout__.flush() + # Check that testzip thinks the archive is valid + self.assertIsNone(zipfp.testzip()) + def testStored(self): # Try the temp file first. If we do TESTFN2 first, then it hogs # gigabytes of disk space for the duration of the test. @@ -85,9 +84,7 @@ def testDeflated(self): self.zipTest(TESTFN2, zipfile.ZIP_DEFLATED) def tearDown(self): - for fname in TESTFN, TESTFN2: - if os.path.exists(fname): - os.remove(fname) + os_helper.unlink(TESTFN2) class OtherTests(unittest.TestCase): diff --git a/Lib/test/test_zipimport.py b/Lib/test/test_zipimport.py index 488a67e80f..b291d53016 100644 --- a/Lib/test/test_zipimport.py +++ b/Lib/test/test_zipimport.py @@ -730,7 +730,6 @@ def testTraceback(self): @unittest.skipIf(os_helper.TESTFN_UNENCODABLE is None, "need an unencodable filename") - @unittest.expectedFailureIfWindows("TODO: RUSTPYTHON") def testUnencodable(self): filename = os_helper.TESTFN_UNENCODABLE + ".zip" self.addCleanup(os_helper.unlink, filename) diff --git a/Lib/test/translationdata/argparse/msgids.txt b/Lib/test/translationdata/argparse/msgids.txt new file mode 100644 index 0000000000..97416aad52 --- /dev/null +++ b/Lib/test/translationdata/argparse/msgids.txt @@ -0,0 +1,40 @@ + (default: %(default)s) +%(heading)s: +%(prog)s: error: %(message)s\n +%(prog)s: warning: %(message)s\n +%r is not callable +'required' is an invalid argument for positionals +.__call__() not defined +ambiguous option: %(option)s could match %(matches)s +argument "-" with mode %r +argument %(argument_name)s: %(message)s +argument '%(argument_name)s' is deprecated +can't open '%(filename)s': %(error)s +cannot have multiple subparser arguments +cannot merge actions - two groups are named %r +command '%(parser_name)s' is deprecated +conflicting subparser alias: %s +conflicting subparser: %s +dest= is required for options like %r +expected at least one argument +expected at most one argument +expected one argument +ignored explicit argument %r +invalid %(type)s value: %(value)r +invalid choice: %(value)r (choose from %(choices)s) +invalid conflict_resolution value: %r +invalid option string %(option)r: must start with a character %(prefix_chars)r +mutually exclusive arguments must be optional +not allowed with argument %s +one of the arguments %s is required +option '%(option)s' is deprecated +options +positional arguments +show program's version number and exit +show this help message and exit +subcommands +the following arguments are required: %s +unexpected option string: %s +unknown parser %(parser_name)r (choices: %(choices)s) +unrecognized arguments: %s +usage: \ No newline at end of file diff --git a/Lib/test/translationdata/getopt/msgids.txt b/Lib/test/translationdata/getopt/msgids.txt new file mode 100644 index 0000000000..1ffab1f31a --- /dev/null +++ b/Lib/test/translationdata/getopt/msgids.txt @@ -0,0 +1,6 @@ +option -%s not recognized +option -%s requires argument +option --%s must not have an argument +option --%s not a unique prefix +option --%s not recognized +option --%s requires argument \ No newline at end of file diff --git a/Lib/test/translationdata/optparse/msgids.txt b/Lib/test/translationdata/optparse/msgids.txt new file mode 100644 index 0000000000..ac5317c736 --- /dev/null +++ b/Lib/test/translationdata/optparse/msgids.txt @@ -0,0 +1,14 @@ +%prog [options] +%s option does not take a value +Options +Usage +Usage: %s\n +ambiguous option: %s (%s?) +complex +floating-point +integer +no such option: %s +option %s: invalid %s value: %r +option %s: invalid choice: %r (choose from %s) +show program's version number and exit +show this help message and exit \ No newline at end of file diff --git a/Lib/textwrap.py b/Lib/textwrap.py index 841de9baec..7ca393d1c3 100644 --- a/Lib/textwrap.py +++ b/Lib/textwrap.py @@ -63,10 +63,7 @@ class TextWrapper: Append to the last line of truncated text. """ - unicode_whitespace_trans = {} - uspace = ord(' ') - for x in _whitespace: - unicode_whitespace_trans[ord(x)] = uspace + unicode_whitespace_trans = dict.fromkeys(map(ord, _whitespace), ord(' ')) # This funky little regex is just the trick for splitting # text up into word-wrappable chunks. E.g. @@ -479,13 +476,19 @@ def indent(text, prefix, predicate=None): consist solely of whitespace characters. """ if predicate is None: - def predicate(line): - return line.strip() - - def prefixed_lines(): - for line in text.splitlines(True): - yield (prefix + line if predicate(line) else line) - return ''.join(prefixed_lines()) + # str.splitlines(True) doesn't produce empty string. + # ''.splitlines(True) => [] + # 'foo\n'.splitlines(True) => ['foo\n'] + # So we can use just `not s.isspace()` here. + predicate = lambda s: not s.isspace() + + prefixed_lines = [] + for line in text.splitlines(True): + if predicate(line): + prefixed_lines.append(prefix) + prefixed_lines.append(line) + + return ''.join(prefixed_lines) if __name__ == "__main__": diff --git a/Lib/trace.py b/Lib/trace.py index 213e46517d..64fc8037e3 100755 --- a/Lib/trace.py +++ b/Lib/trace.py @@ -49,6 +49,7 @@ """ __all__ = ['Trace', 'CoverageResults'] +import io import linecache import os import sys @@ -201,7 +202,8 @@ def update(self, other): for key in other_callers: callers[key] = 1 - def write_results(self, show_missing=True, summary=False, coverdir=None): + def write_results(self, show_missing=True, summary=False, coverdir=None, *, + ignore_missing_files=False): """ Write the coverage results. @@ -210,6 +212,9 @@ def write_results(self, show_missing=True, summary=False, coverdir=None): :param coverdir: If None, the results of each module are placed in its directory, otherwise it is included in the directory specified. + :param ignore_missing_files: If True, counts for files that no longer + exist are silently ignored. Otherwise, a missing file + will raise a FileNotFoundError. """ if self.calledfuncs: print() @@ -252,13 +257,15 @@ def write_results(self, show_missing=True, summary=False, coverdir=None): if filename.endswith(".pyc"): filename = filename[:-1] + if ignore_missing_files and not os.path.isfile(filename): + continue + if coverdir is None: dir = os.path.dirname(os.path.abspath(filename)) modulename = _modname(filename) else: dir = coverdir - if not os.path.exists(dir): - os.makedirs(dir) + os.makedirs(dir, exist_ok=True) modulename = _fullmodname(filename) # If desired, get a list of the line numbers which represent @@ -277,7 +284,6 @@ def write_results(self, show_missing=True, summary=False, coverdir=None): percent = int(100 * n_hits / n_lines) sums[modulename] = n_lines, percent, modulename, filename - if summary and sums: print("lines cov% module (path)") for m in sorted(sums): @@ -559,8 +565,12 @@ def localtrace_trace_and_count(self, frame, why, arg): if self.start_time: print('%.2f' % (_time() - self.start_time), end=' ') bname = os.path.basename(filename) - print("%s(%d): %s" % (bname, lineno, - linecache.getline(filename, lineno)), end='') + line = linecache.getline(filename, lineno) + print("%s(%d)" % (bname, lineno), end='') + if line: + print(": ", line, end='') + else: + print() return self.localtrace def localtrace_trace(self, frame, why, arg): @@ -572,8 +582,12 @@ def localtrace_trace(self, frame, why, arg): if self.start_time: print('%.2f' % (_time() - self.start_time), end=' ') bname = os.path.basename(filename) - print("%s(%d): %s" % (bname, lineno, - linecache.getline(filename, lineno)), end='') + line = linecache.getline(filename, lineno) + print("%s(%d)" % (bname, lineno), end='') + if line: + print(": ", line, end='') + else: + print() return self.localtrace def localtrace_count(self, frame, why, arg): @@ -716,7 +730,7 @@ def parse_ignore_dir(s): sys.argv = [opts.progname, *opts.arguments] sys.path[0] = os.path.dirname(opts.progname) - with open(opts.progname, 'rb') as fp: + with io.open_code(opts.progname) as fp: code = compile(fp.read(), opts.progname, 'exec') # try to emulate __main__ namespace as much as possible globs = { diff --git a/Lib/types.py b/Lib/types.py index 4dab6ddce0..b036a85068 100644 --- a/Lib/types.py +++ b/Lib/types.py @@ -1,6 +1,7 @@ """ Define names for built-in types that aren't directly accessible as a builtin. """ + import sys # Iterators in Python aren't a matter of type but of protocol. A large @@ -52,17 +53,14 @@ def _m(self): pass try: raise TypeError -except TypeError: - tb = sys.exc_info()[2] - TracebackType = type(tb) - FrameType = type(tb.tb_frame) - tb = None; del tb +except TypeError as exc: + TracebackType = type(exc.__traceback__) + FrameType = type(exc.__traceback__.tb_frame) -# For Jython, the following two types are identical GetSetDescriptorType = type(FunctionType.__code__) MemberDescriptorType = type(FunctionType.__globals__) -del sys, _f, _g, _C, _c, _ag # Not for export +del sys, _f, _g, _C, _c, _ag, _cell_factory # Not for export # Provide a PEP 3115 compliant mechanism for class creation @@ -82,7 +80,7 @@ def resolve_bases(bases): updated = False shift = 0 for i, base in enumerate(bases): - if isinstance(base, type) and not isinstance(base, GenericAlias): + if isinstance(base, type): continue if not hasattr(base, "__mro_entries__"): continue @@ -146,6 +144,35 @@ def _calculate_meta(meta, bases): "of the metaclasses of all its bases") return winner + +def get_original_bases(cls, /): + """Return the class's "original" bases prior to modification by `__mro_entries__`. + + Examples:: + + from typing import TypeVar, Generic, NamedTuple, TypedDict + + T = TypeVar("T") + class Foo(Generic[T]): ... + class Bar(Foo[int], float): ... + class Baz(list[str]): ... + Eggs = NamedTuple("Eggs", [("a", int), ("b", str)]) + Spam = TypedDict("Spam", {"a": int, "b": str}) + + assert get_original_bases(Bar) == (Foo[int], float) + assert get_original_bases(Baz) == (list[str],) + assert get_original_bases(Eggs) == (NamedTuple,) + assert get_original_bases(Spam) == (TypedDict,) + assert get_original_bases(int) == (object,) + """ + try: + return cls.__dict__.get("__orig_bases__", cls.__bases__) + except AttributeError: + raise TypeError( + f"Expected an instance of type, not {type(cls).__name__!r}" + ) from None + + class DynamicClassAttribute: """Route attribute access on a class to __getattr__. @@ -158,7 +185,7 @@ class DynamicClassAttribute: attributes on the class with the same name. (Enum used this between Python versions 3.4 - 3.9 .) - Subclass from this to use a different method of accessing virtual atributes + Subclass from this to use a different method of accessing virtual attributes and still be treated properly by the inspect module. (Enum uses this since Python 3.10 .) @@ -305,4 +332,11 @@ def wrapped(*args, **kwargs): NoneType = type(None) NotImplementedType = type(NotImplemented) +def __getattr__(name): + if name == 'CapsuleType': + import _socket + return type(_socket.CAPI) + raise AttributeError(f"module {__name__!r} has no attribute {name!r}") + __all__ = [n for n in globals() if n[:1] != '_'] +__all__ += ['CapsuleType'] diff --git a/Lib/unittest/case.py b/Lib/unittest/case.py index c4aa2d7721..36daa61fa3 100644 --- a/Lib/unittest/case.py +++ b/Lib/unittest/case.py @@ -9,6 +9,7 @@ import collections import contextlib import traceback +import time import types from . import result @@ -331,6 +332,23 @@ def __exit__(self, exc_type, exc_value, tb): self._raiseFailure("{} not triggered".format(exc_name)) +class _AssertNotWarnsContext(_AssertWarnsContext): + + def __exit__(self, exc_type, exc_value, tb): + self.warnings_manager.__exit__(exc_type, exc_value, tb) + if exc_type is not None: + # let unexpected exceptions pass through + return + try: + exc_name = self.expected.__name__ + except AttributeError: + exc_name = str(self.expected) + for m in self.warnings: + w = m.message + if isinstance(w, self.expected): + self._raiseFailure(f"{exc_name} triggered") + + class _OrderedChainMap(collections.ChainMap): def __iter__(self): seen = set() @@ -572,6 +590,15 @@ def _addUnexpectedSuccess(self, result): else: addUnexpectedSuccess(self) + def _addDuration(self, result, elapsed): + try: + addDuration = result.addDuration + except AttributeError: + warnings.warn("TestResult has no addDuration method", + RuntimeWarning) + else: + addDuration(self, elapsed) + def _callSetUp(self): self.setUp() @@ -612,6 +639,7 @@ def run(self, result=None): getattr(testMethod, "__unittest_expecting_failure__", False) ) outcome = _Outcome(result) + start_time = time.perf_counter() try: self._outcome = outcome @@ -625,6 +653,7 @@ def run(self, result=None): with outcome.testPartExecutor(self): self._callTearDown() self.doCleanups() + self._addDuration(result, (time.perf_counter() - start_time)) if outcome.success: if expecting_failure: @@ -799,6 +828,11 @@ def assertWarns(self, expected_warning, *args, **kwargs): context = _AssertWarnsContext(expected_warning, self) return context.handle('assertWarns', args, kwargs) + def _assertNotWarns(self, expected_warning, *args, **kwargs): + """The opposite of assertWarns. Private due to low demand.""" + context = _AssertNotWarnsContext(expected_warning, self) + return context.handle('_assertNotWarns', args, kwargs) + def assertLogs(self, logger=None, level=None): """Fail unless a log message of level *level* or higher is emitted on *logger_name* or its children. If omitted, *level* defaults to @@ -1171,35 +1205,6 @@ def assertDictEqual(self, d1, d2, msg=None): standardMsg = self._truncateMessage(standardMsg, diff) self.fail(self._formatMessage(msg, standardMsg)) - def assertDictContainsSubset(self, subset, dictionary, msg=None): - """Checks whether dictionary is a superset of subset.""" - warnings.warn('assertDictContainsSubset is deprecated', - DeprecationWarning) - missing = [] - mismatched = [] - for key, value in subset.items(): - if key not in dictionary: - missing.append(key) - elif value != dictionary[key]: - mismatched.append('%s, expected: %s, actual: %s' % - (safe_repr(key), safe_repr(value), - safe_repr(dictionary[key]))) - - if not (missing or mismatched): - return - - standardMsg = '' - if missing: - standardMsg = 'Missing: %s' % ','.join(safe_repr(m) for m in - missing) - if mismatched: - if standardMsg: - standardMsg += '; ' - standardMsg += 'Mismatched values: %s' % ','.join(mismatched) - - self.fail(self._formatMessage(msg, standardMsg)) - - def assertCountEqual(self, first, second, msg=None): """Asserts that two iterables have the same elements, the same number of times, without regard to order. @@ -1234,19 +1239,34 @@ def assertCountEqual(self, first, second, msg=None): def assertMultiLineEqual(self, first, second, msg=None): """Assert that two multi-line strings are equal.""" - self.assertIsInstance(first, str, 'First argument is not a string') - self.assertIsInstance(second, str, 'Second argument is not a string') + self.assertIsInstance(first, str, "First argument is not a string") + self.assertIsInstance(second, str, "Second argument is not a string") if first != second: - # don't use difflib if the strings are too long + # Don't use difflib if the strings are too long if (len(first) > self._diffThreshold or len(second) > self._diffThreshold): self._baseAssertEqual(first, second, msg) - firstlines = first.splitlines(keepends=True) - secondlines = second.splitlines(keepends=True) - if len(firstlines) == 1 and first.strip('\r\n') == first: - firstlines = [first + '\n'] - secondlines = [second + '\n'] + + # Append \n to both strings if either is missing the \n. + # This allows the final ndiff to show the \n difference. The + # exception here is if the string is empty, in which case no + # \n should be added + first_presplit = first + second_presplit = second + if first and second: + if first[-1] != '\n' or second[-1] != '\n': + first_presplit += '\n' + second_presplit += '\n' + elif second and second[-1] != '\n': + second_presplit += '\n' + elif first and first[-1] != '\n': + first_presplit += '\n' + + firstlines = first_presplit.splitlines(keepends=True) + secondlines = second_presplit.splitlines(keepends=True) + + # Generate the message and diff, then raise the exception standardMsg = '%s != %s' % _common_shorten_repr(first, second) diff = '\n' + ''.join(difflib.ndiff(firstlines, secondlines)) standardMsg = self._truncateMessage(standardMsg, diff) @@ -1363,27 +1383,6 @@ def assertNotRegex(self, text, unexpected_regex, msg=None): raise self.failureException(msg) - def _deprecate(original_func): - def deprecated_func(*args, **kwargs): - warnings.warn( - 'Please use {0} instead.'.format(original_func.__name__), - DeprecationWarning, 2) - return original_func(*args, **kwargs) - return deprecated_func - - # see #9424 - failUnlessEqual = assertEquals = _deprecate(assertEqual) - failIfEqual = assertNotEquals = _deprecate(assertNotEqual) - failUnlessAlmostEqual = assertAlmostEquals = _deprecate(assertAlmostEqual) - failIfAlmostEqual = assertNotAlmostEquals = _deprecate(assertNotAlmostEqual) - failUnless = assert_ = _deprecate(assertTrue) - failUnlessRaises = _deprecate(assertRaises) - failIf = _deprecate(assertFalse) - assertRaisesRegexp = _deprecate(assertRaisesRegex) - assertRegexpMatches = _deprecate(assertRegex) - assertNotRegexpMatches = _deprecate(assertNotRegex) - - class FunctionTestCase(TestCase): """A test case that wraps a test function. diff --git a/Lib/unittest/main.py b/Lib/unittest/main.py index 046fbd3a45..c3869de3f6 100644 --- a/Lib/unittest/main.py +++ b/Lib/unittest/main.py @@ -3,12 +3,12 @@ import sys import argparse import os -import warnings from . import loader, runner from .signals import installHandler __unittest = True +_NO_TESTS_EXITCODE = 5 MAIN_EXAMPLES = """\ Examples: @@ -66,7 +66,8 @@ class TestProgram(object): def __init__(self, module='__main__', defaultTest=None, argv=None, testRunner=None, testLoader=loader.defaultTestLoader, exit=True, verbosity=1, failfast=None, catchbreak=None, - buffer=None, warnings=None, *, tb_locals=False): + buffer=None, warnings=None, *, tb_locals=False, + durations=None): if isinstance(module, str): self.module = __import__(module) for part in module.split('.')[1:]: @@ -82,6 +83,7 @@ def __init__(self, module='__main__', defaultTest=None, argv=None, self.verbosity = verbosity self.buffer = buffer self.tb_locals = tb_locals + self.durations = durations if warnings is None and not sys.warnoptions: # even if DeprecationWarnings are ignored by default # print them anyway unless other warnings settings are @@ -101,16 +103,6 @@ def __init__(self, module='__main__', defaultTest=None, argv=None, self.parseArgs(argv) self.runTests() - def usageExit(self, msg=None): - warnings.warn("TestProgram.usageExit() is deprecated and will be" - " removed in Python 3.13", DeprecationWarning) - if msg: - print(msg) - if self._discovery_parser is None: - self._initArgParsers() - self._print_help() - sys.exit(2) - def _print_help(self, *args, **kwargs): if self.module is None: print(self._main_parser.format_help()) @@ -178,6 +170,9 @@ def _getParentArgParser(self): parser.add_argument('--locals', dest='tb_locals', action='store_true', help='Show local variables in tracebacks') + parser.add_argument('--durations', dest='durations', type=int, + default=None, metavar="N", + help='Show the N slowest test cases (N=0 for all)') if self.failfast is None: parser.add_argument('-f', '--failfast', dest='failfast', action='store_true', @@ -258,9 +253,10 @@ def runTests(self): failfast=self.failfast, buffer=self.buffer, warnings=self.warnings, - tb_locals=self.tb_locals) + tb_locals=self.tb_locals, + durations=self.durations) except TypeError: - # didn't accept the tb_locals argument + # didn't accept the tb_locals or durations argument testRunner = self.testRunner(verbosity=self.verbosity, failfast=self.failfast, buffer=self.buffer, @@ -273,6 +269,12 @@ def runTests(self): testRunner = self.testRunner self.result = testRunner.run(self.test) if self.exit: - sys.exit(not self.result.wasSuccessful()) + if self.result.testsRun == 0 and len(self.result.skipped) == 0: + sys.exit(_NO_TESTS_EXITCODE) + elif self.result.wasSuccessful(): + sys.exit(0) + else: + sys.exit(1) + main = TestProgram diff --git a/Lib/unittest/mock.py b/Lib/unittest/mock.py index fa0bd9131a..6cec61ff35 100644 --- a/Lib/unittest/mock.py +++ b/Lib/unittest/mock.py @@ -14,6 +14,7 @@ 'call', 'create_autospec', 'AsyncMock', + 'ThreadingMock', 'FILTER_DIR', 'NonCallableMock', 'NonCallableMagicMock', @@ -32,6 +33,7 @@ import builtins import pkgutil from asyncio import iscoroutinefunction +import threading from types import CodeType, ModuleType, MethodType from unittest.util import safe_repr from functools import wraps, partial @@ -98,6 +100,12 @@ def _get_signature_object(func, as_instance, eat_self): func = func.__init__ # Skip the `self` argument in __init__ eat_self = True + elif isinstance(func, (classmethod, staticmethod)): + if isinstance(func, classmethod): + # Skip the `cls` argument of a class method + eat_self = True + # Use the original decorated method to extract the correct function signature + func = func.__func__ elif not isinstance(func, FunctionTypes): # If we really want to model an instance of the passed type, # __call__ should be looked up, not __init__. @@ -198,6 +206,28 @@ def checksig(*args, **kwargs): _setup_func(funcopy, mock, sig) return funcopy +def _set_async_signature(mock, original, instance=False, is_async_mock=False): + # creates an async function with signature (*args, **kwargs) that delegates to a + # mock. It still does signature checking by calling a lambda with the same + # signature as the original. + + skipfirst = isinstance(original, type) + func, sig = _get_signature_object(original, instance, skipfirst) + def checksig(*args, **kwargs): + sig.bind(*args, **kwargs) + _copy_func_details(func, checksig) + + name = original.__name__ + context = {'_checksig_': checksig, 'mock': mock} + src = """async def %s(*args, **kwargs): + _checksig_(*args, **kwargs) + return await mock(*args, **kwargs)""" % name + exec (src, context) + funcopy = context[name] + _setup_func(funcopy, mock, sig) + _setup_async_mock(funcopy) + return funcopy + def _setup_func(funcopy, mock, sig): funcopy.mock = mock @@ -411,15 +441,18 @@ class NonCallableMock(Base): # necessary. _lock = RLock() - def __new__(cls, /, *args, **kw): + def __new__( + cls, spec=None, wraps=None, name=None, spec_set=None, + parent=None, _spec_state=None, _new_name='', _new_parent=None, + _spec_as_instance=False, _eat_self=None, unsafe=False, **kwargs + ): # every instance has its own class # so we can create magic methods on the # class without stomping on other mocks bases = (cls,) if not issubclass(cls, AsyncMockMixin): # Check if spec is an async object or function - bound_args = _MOCK_SIG.bind_partial(cls, *args, **kw).arguments - spec_arg = bound_args.get('spec_set', bound_args.get('spec')) + spec_arg = spec_set or spec if spec_arg is not None and _is_async_obj(spec_arg): bases = (AsyncMockMixin, cls) new = type(cls.__name__, bases, {'__doc__': cls.__doc__}) @@ -505,10 +538,6 @@ def _mock_add_spec(self, spec, spec_set, _spec_as_instance=False, _spec_signature = None _spec_asyncs = [] - for attr in dir(spec): - if iscoroutinefunction(getattr(spec, attr, None)): - _spec_asyncs.append(attr) - if spec is not None and not _is_list(spec): if isinstance(spec, type): _spec_class = spec @@ -518,7 +547,19 @@ def _mock_add_spec(self, spec, spec_set, _spec_as_instance=False, _spec_as_instance, _eat_self) _spec_signature = res and res[1] - spec = dir(spec) + spec_list = dir(spec) + + for attr in spec_list: + static_attr = inspect.getattr_static(spec, attr, None) + unwrapped_attr = static_attr + try: + unwrapped_attr = inspect.unwrap(unwrapped_attr) + except ValueError: + pass + if iscoroutinefunction(unwrapped_attr): + _spec_asyncs.append(attr) + + spec = spec_list __dict__ = self.__dict__ __dict__['_spec_class'] = _spec_class @@ -532,7 +573,7 @@ def __get_return_value(self): if self._mock_delegate is not None: ret = self._mock_delegate.return_value - if ret is DEFAULT: + if ret is DEFAULT and self._mock_wraps is None: ret = self._get_child_mock( _new_parent=self, _new_name='()' ) @@ -587,7 +628,9 @@ def __set_side_effect(self, value): side_effect = property(__get_side_effect, __set_side_effect) - def reset_mock(self, visited=None,*, return_value=False, side_effect=False): + def reset_mock(self, visited=None, *, + return_value: bool = False, + side_effect: bool = False): "Restore the mock object to its initial state." if visited is None: visited = [] @@ -648,7 +691,7 @@ def __getattr__(self, name): elif _is_magic(name): raise AttributeError(name) if not self._mock_unsafe and (not self._mock_methods or name not in self._mock_methods): - if name.startswith(('assert', 'assret', 'asert', 'aseert', 'assrt')): + if name.startswith(('assert', 'assret', 'asert', 'aseert', 'assrt')) or name in _ATTRIB_DENY_LIST: raise AttributeError( f"{name!r} is not a valid assertion. Use a spec " f"for the mock if {name!r} is meant to be an attribute.") @@ -789,6 +832,9 @@ def __setattr__(self, name, value): mock_name = f'{self._extract_mock_name()}.{name}' raise AttributeError(f'Cannot set {mock_name}') + if isinstance(value, PropertyMock): + self.__dict__[name] = value + return return object.__setattr__(self, name, value) @@ -816,7 +862,7 @@ def _format_mock_call_signature(self, args, kwargs): def _format_mock_failure_message(self, args, kwargs, action='call'): - message = 'expected %s not found.\nExpected: %s\nActual: %s' + message = 'expected %s not found.\nExpected: %s\n Actual: %s' expected_string = self._format_mock_call_signature(args, kwargs) call_args = self.call_args actual_string = self._format_mock_call_signature(*call_args) @@ -919,7 +965,7 @@ def assert_called_with(self, /, *args, **kwargs): if self.call_args is None: expected = self._format_mock_call_signature(args, kwargs) actual = 'not called.' - error_message = ('expected call not found.\nExpected: %s\nActual: %s' + error_message = ('expected call not found.\nExpected: %s\n Actual: %s' % (expected, actual)) raise AssertionError(error_message) @@ -969,8 +1015,8 @@ def assert_has_calls(self, calls, any_order=False): for e in expected]) raise AssertionError( f'{problem}\n' - f'Expected: {_CallList(calls)}' - f'{self._calls_repr(prefix="Actual").rstrip(".")}' + f'Expected: {_CallList(calls)}\n' + f' Actual: {safe_repr(self.mock_calls)}' ) from cause return @@ -1044,7 +1090,7 @@ def _get_child_mock(self, /, **kw): return klass(**kw) - def _calls_repr(self, prefix="Calls"): + def _calls_repr(self): """Renders self.mock_calls as a string. Example: "\nCalls: [call(1), call(2)]." @@ -1054,10 +1100,15 @@ def _calls_repr(self, prefix="Calls"): """ if not self.mock_calls: return "" - return f"\n{prefix}: {safe_repr(self.mock_calls)}." + return f"\nCalls: {safe_repr(self.mock_calls)}." -_MOCK_SIG = inspect.signature(NonCallableMock.__init__) +# Denylist for forbidden attribute names in safe mode +_ATTRIB_DENY_LIST = frozenset({ + name.removeprefix("assert_") + for name in dir(NonCallableMock) + if name.startswith("assert_") +}) class _AnyComparer(list): @@ -1188,6 +1239,9 @@ def _execute_mock_call(self, /, *args, **kwargs): if self._mock_return_value is not DEFAULT: return self.return_value + if self._mock_delegate and self._mock_delegate.return_value is not DEFAULT: + return self.return_value + if self._mock_wraps is not None: return self._mock_wraps(*args, **kwargs) @@ -1229,9 +1283,11 @@ class or instance) that acts as the specification for the mock object. If `return_value` attribute. * `unsafe`: By default, accessing any attribute whose name starts with - *assert*, *assret*, *asert*, *aseert* or *assrt* will raise an - AttributeError. Passing `unsafe=True` will allow access to - these attributes. + *assert*, *assret*, *asert*, *aseert*, or *assrt* raises an AttributeError. + Additionally, an AttributeError is raised when accessing + attributes that match the name of an assertion method without the prefix + `assert_`, e.g. accessing `called_once` instead of `assert_called_once`. + Passing `unsafe=True` will allow access to these attributes. * `wraps`: Item for the mock object to wrap. If `wraps` is not None then calling the Mock will pass the call through to the wrapped object @@ -1303,6 +1359,7 @@ def __init__( self.autospec = autospec self.kwargs = kwargs self.additional_patchers = [] + self.is_started = False def copy(self): @@ -1415,6 +1472,9 @@ def get_original(self): def __enter__(self): """Perform the patch.""" + if self.is_started: + raise RuntimeError("Patch is already started") + new, spec, spec_set = self.new, self.spec, self.spec_set autospec, kwargs = self.autospec, self.kwargs new_callable = self.new_callable @@ -1457,13 +1517,12 @@ def __enter__(self): if isinstance(original, type): # If we're patching out a class and there is a spec inherit = True - if spec is None and _is_async_obj(original): - Klass = AsyncMock - else: - Klass = MagicMock - _kwargs = {} + + # Determine the Klass to use if new_callable is not None: Klass = new_callable + elif spec is None and _is_async_obj(original): + Klass = AsyncMock elif spec is not None or spec_set is not None: this_spec = spec if spec_set is not None: @@ -1476,7 +1535,12 @@ def __enter__(self): Klass = AsyncMock elif not_callable: Klass = NonCallableMagicMock + else: + Klass = MagicMock + else: + Klass = MagicMock + _kwargs = {} if spec is not None: _kwargs['spec'] = spec if spec_set is not None: @@ -1542,6 +1606,7 @@ def __enter__(self): self.temp_original = original self.is_local = local self._exit_stack = contextlib.ExitStack() + self.is_started = True try: setattr(self.target, self.attribute, new_attr) if self.attribute_name is not None: @@ -1561,6 +1626,9 @@ def __enter__(self): def __exit__(self, *exc_info): """Undo the patch.""" + if not self.is_started: + return + if self.is_local and self.temp_original is not DEFAULT: setattr(self.target, self.attribute, self.temp_original) else: @@ -1577,6 +1645,7 @@ def __exit__(self, *exc_info): del self.target exit_stack = self._exit_stack del self._exit_stack + self.is_started = False return exit_stack.__exit__(*exc_info) @@ -2138,10 +2207,8 @@ def mock_add_spec(self, spec, spec_set=False): class AsyncMagicMixin(MagicMixin): - def __init__(self, /, *args, **kw): - self._mock_set_magics() # make magic work for kwargs in init - _safe_super(AsyncMagicMixin, self).__init__(*args, **kw) - self._mock_set_magics() # fix magic broken by upper level init + pass + class MagicMock(MagicMixin, Mock): """ @@ -2163,6 +2230,17 @@ def mock_add_spec(self, spec, spec_set=False): self._mock_add_spec(spec, spec_set) self._mock_set_magics() + def reset_mock(self, /, *args, return_value: bool = False, **kwargs): + if ( + return_value + and self._mock_name + and _is_magic(self._mock_name) + ): + # Don't reset return values for magic methods, + # otherwise `m.__str__` will start + # to return `MagicMock` instances, instead of `str` instances. + return_value = False + super().reset_mock(*args, return_value=return_value, **kwargs) class MagicProxy(Base): @@ -2183,6 +2261,13 @@ def __get__(self, obj, _type=None): return self.create_mock() +try: + _CODE_SIG = inspect.signature(partial(CodeType.__init__, None)) + _CODE_ATTRS = dir(CodeType) +except ValueError: + _CODE_SIG = None + + class AsyncMockMixin(Base): await_count = _delegating_property('await_count') await_args = _delegating_property('await_args') @@ -2200,8 +2285,21 @@ def __init__(self, /, *args, **kwargs): self.__dict__['_mock_await_count'] = 0 self.__dict__['_mock_await_args'] = None self.__dict__['_mock_await_args_list'] = _CallList() - code_mock = NonCallableMock(spec_set=CodeType) - code_mock.co_flags = inspect.CO_COROUTINE + if _CODE_SIG: + code_mock = NonCallableMock(spec_set=_CODE_ATTRS) + code_mock.__dict__["_spec_class"] = CodeType + code_mock.__dict__["_spec_signature"] = _CODE_SIG + else: + code_mock = NonCallableMock(spec_set=CodeType) + code_mock.co_flags = ( + inspect.CO_COROUTINE + + inspect.CO_VARARGS + + inspect.CO_VARKEYWORDS + ) + code_mock.co_argcount = 0 + code_mock.co_varnames = ('args', 'kwargs') + code_mock.co_posonlyargcount = 0 + code_mock.co_kwonlyargcount = 0 self.__dict__['__code__'] = code_mock self.__dict__['__name__'] = 'AsyncMock' self.__dict__['__defaults__'] = tuple() @@ -2679,6 +2777,12 @@ def create_autospec(spec, spec_set=False, instance=False, _parent=None, if not unsafe: _check_spec_arg_typos(kwargs) + _name = kwargs.pop('name', _name) + _new_name = _name + if _parent is None: + # for a top level object no _new_name should be set + _new_name = '' + _kwargs.update(kwargs) Klass = MagicMock @@ -2696,31 +2800,28 @@ def create_autospec(spec, spec_set=False, instance=False, _parent=None, elif is_type and instance and not _instance_callable(spec): Klass = NonCallableMagicMock - _name = _kwargs.pop('name', _name) - - _new_name = _name - if _parent is None: - # for a top level object no _new_name should be set - _new_name = '' - mock = Klass(parent=_parent, _new_parent=_parent, _new_name=_new_name, name=_name, **_kwargs) if isinstance(spec, FunctionTypes): # should only happen at the top level because we don't # recurse for functions - mock = _set_signature(mock, spec) if is_async_func: - _setup_async_mock(mock) + mock = _set_async_signature(mock, spec) + else: + mock = _set_signature(mock, spec) else: _check_signature(spec, mock, is_type, instance) if _parent is not None and not instance: _parent._mock_children[_name] = mock + # Pop wraps from kwargs because it must not be passed to configure_mock. + wrapped = kwargs.pop('wraps', None) if is_type and not instance and 'return_value' not in kwargs: mock.return_value = create_autospec(spec, spec_set, instance=True, - _name='()', _parent=mock) + _name='()', _parent=mock, + wraps=wrapped) for entry in dir(spec): if _is_magic(entry): @@ -2741,9 +2842,12 @@ def create_autospec(spec, spec_set=False, instance=False, _parent=None, except AttributeError: continue - kwargs = {'spec': original} + child_kwargs = {'spec': original} + # Wrap child attributes also. + if wrapped and hasattr(wrapped, entry): + child_kwargs.update(wraps=original) if spec_set: - kwargs = {'spec_set': original} + child_kwargs = {'spec_set': original} if not isinstance(original, FunctionTypes): new = _SpecState(original, spec_set, mock, entry, instance) @@ -2754,15 +2858,15 @@ def create_autospec(spec, spec_set=False, instance=False, _parent=None, parent = mock.mock skipfirst = _must_skip(spec, entry, is_type) - kwargs['_eat_self'] = skipfirst + child_kwargs['_eat_self'] = skipfirst if iscoroutinefunction(original): child_klass = AsyncMock else: child_klass = MagicMock new = child_klass(parent=parent, name=entry, _new_name=entry, - _new_parent=parent, - **kwargs) + _new_parent=parent, **child_kwargs) mock._mock_children[entry] = new + new.return_value = child_klass() _check_signature(original, new, skipfirst=skipfirst) # so functions created with _set_signature become instance attributes, @@ -2771,6 +2875,11 @@ def create_autospec(spec, spec_set=False, instance=False, _parent=None, # setting as an instance attribute? if isinstance(new, FunctionTypes): setattr(mock, entry, new) + # kwargs are passed with respect to the parent mock so, they are not used + # for creating return_value of the parent mock. So, this condition + # should be true only for the parent mock if kwargs are given. + if _is_instance_mock(mock) and kwargs: + mock.configure_mock(**kwargs) return mock @@ -2876,6 +2985,9 @@ def _next_side_effect(): return handle.readline.return_value return next(_state[0]) + def _exit_side_effect(exctype, excinst, exctb): + handle.close() + global file_spec if file_spec is None: import _io @@ -2902,6 +3014,7 @@ def _next_side_effect(): handle.readlines.side_effect = _readlines_side_effect handle.__iter__.side_effect = _iter_side_effect handle.__next__.side_effect = _next_side_effect + handle.__exit__.side_effect = _exit_side_effect def reset_data(*args, **kwargs): _state[0] = _to_stream(read_data) @@ -2934,6 +3047,96 @@ def __set__(self, obj, val): self(val) +_timeout_unset = sentinel.TIMEOUT_UNSET + +class ThreadingMixin(Base): + + DEFAULT_TIMEOUT = None + + def _get_child_mock(self, /, **kw): + if isinstance(kw.get("parent"), ThreadingMixin): + kw["timeout"] = kw["parent"]._mock_wait_timeout + elif isinstance(kw.get("_new_parent"), ThreadingMixin): + kw["timeout"] = kw["_new_parent"]._mock_wait_timeout + return super()._get_child_mock(**kw) + + def __init__(self, *args, timeout=_timeout_unset, **kwargs): + super().__init__(*args, **kwargs) + if timeout is _timeout_unset: + timeout = self.DEFAULT_TIMEOUT + self.__dict__["_mock_event"] = threading.Event() # Event for any call + self.__dict__["_mock_calls_events"] = [] # Events for each of the calls + self.__dict__["_mock_calls_events_lock"] = threading.Lock() + self.__dict__["_mock_wait_timeout"] = timeout + + def reset_mock(self, /, *args, **kwargs): + """ + See :func:`.Mock.reset_mock()` + """ + super().reset_mock(*args, **kwargs) + self.__dict__["_mock_event"] = threading.Event() + self.__dict__["_mock_calls_events"] = [] + + def __get_event(self, expected_args, expected_kwargs): + with self._mock_calls_events_lock: + for args, kwargs, event in self._mock_calls_events: + if (args, kwargs) == (expected_args, expected_kwargs): + return event + new_event = threading.Event() + self._mock_calls_events.append((expected_args, expected_kwargs, new_event)) + return new_event + + def _mock_call(self, *args, **kwargs): + ret_value = super()._mock_call(*args, **kwargs) + + call_event = self.__get_event(args, kwargs) + call_event.set() + + self._mock_event.set() + + return ret_value + + def wait_until_called(self, *, timeout=_timeout_unset): + """Wait until the mock object is called. + + `timeout` - time to wait for in seconds, waits forever otherwise. + Defaults to the constructor provided timeout. + Use None to block undefinetively. + """ + if timeout is _timeout_unset: + timeout = self._mock_wait_timeout + if not self._mock_event.wait(timeout=timeout): + msg = (f"{self._mock_name or 'mock'} was not called before" + f" timeout({timeout}).") + raise AssertionError(msg) + + def wait_until_any_call_with(self, *args, **kwargs): + """Wait until the mock object is called with given args. + + Waits for the timeout in seconds provided in the constructor. + """ + event = self.__get_event(args, kwargs) + if not event.wait(timeout=self._mock_wait_timeout): + expected_string = self._format_mock_call_signature(args, kwargs) + raise AssertionError(f'{expected_string} call not found') + + +class ThreadingMock(ThreadingMixin, MagicMixin, Mock): + """ + A mock that can be used to wait until on calls happening + in a different thread. + + The constructor can take a `timeout` argument which + controls the timeout in seconds for all `wait` calls of the mock. + + You can change the default timeout of all instances via the + `ThreadingMock.DEFAULT_TIMEOUT` attribute. + + If no timeout is set, it will block undefinetively. + """ + pass + + def seal(mock): """Disable the automatic generation of child mocks. diff --git a/Lib/unittest/result.py b/Lib/unittest/result.py index 5ca4c23238..3ace0a5b7b 100644 --- a/Lib/unittest/result.py +++ b/Lib/unittest/result.py @@ -43,6 +43,7 @@ def __init__(self, stream=None, descriptions=None, verbosity=None): self.skipped = [] self.expectedFailures = [] self.unexpectedSuccesses = [] + self.collectedDurations = [] self.shouldStop = False self.buffer = False self.tb_locals = False @@ -157,6 +158,17 @@ def addUnexpectedSuccess(self, test): """Called when a test was expected to fail, but succeed.""" self.unexpectedSuccesses.append(test) + def addDuration(self, test, elapsed): + """Called when a test finished to run, regardless of its outcome. + *test* is the test case corresponding to the test method. + *elapsed* is the time represented in seconds, and it includes the + execution of cleanup functions. + """ + # support for a TextTestRunner using an old TestResult class + if hasattr(self, "collectedDurations"): + # Pass test repr and not the test object itself to avoid resources leak + self.collectedDurations.append((str(test), elapsed)) + def wasSuccessful(self): """Tells whether or not this result was a success.""" # The hasattr check is for test_result's OldResult test. That diff --git a/Lib/unittest/runner.py b/Lib/unittest/runner.py index cb452c7ade..2bcadf0c99 100644 --- a/Lib/unittest/runner.py +++ b/Lib/unittest/runner.py @@ -35,13 +35,16 @@ class TextTestResult(result.TestResult): separator1 = '=' * 70 separator2 = '-' * 70 - def __init__(self, stream, descriptions, verbosity): + def __init__(self, stream, descriptions, verbosity, *, durations=None): + """Construct a TextTestResult. Subclasses should accept **kwargs + to ensure compatibility as the interface changes.""" super(TextTestResult, self).__init__(stream, descriptions, verbosity) self.stream = stream self.showAll = verbosity > 1 self.dots = verbosity == 1 self.descriptions = descriptions self._newline = True + self.durations = durations def getDescription(self, test): doc_first_line = test.shortDescription() @@ -168,7 +171,7 @@ class TextTestRunner(object): def __init__(self, stream=None, descriptions=True, verbosity=1, failfast=False, buffer=False, resultclass=None, warnings=None, - *, tb_locals=False): + *, tb_locals=False, durations=None): """Construct a TextTestRunner. Subclasses should accept **kwargs to ensure compatibility as the @@ -182,12 +185,41 @@ def __init__(self, stream=None, descriptions=True, verbosity=1, self.failfast = failfast self.buffer = buffer self.tb_locals = tb_locals + self.durations = durations self.warnings = warnings if resultclass is not None: self.resultclass = resultclass def _makeResult(self): - return self.resultclass(self.stream, self.descriptions, self.verbosity) + try: + return self.resultclass(self.stream, self.descriptions, + self.verbosity, durations=self.durations) + except TypeError: + # didn't accept the durations argument + return self.resultclass(self.stream, self.descriptions, + self.verbosity) + + def _printDurations(self, result): + if not result.collectedDurations: + return + ls = sorted(result.collectedDurations, key=lambda x: x[1], + reverse=True) + if self.durations > 0: + ls = ls[:self.durations] + self.stream.writeln("Slowest test durations") + if hasattr(result, 'separator2'): + self.stream.writeln(result.separator2) + hidden = False + for test, elapsed in ls: + if self.verbosity < 2 and elapsed < 0.001: + hidden = True + continue + self.stream.writeln("%-10s %s" % ("%.3fs" % elapsed, test)) + if hidden: + self.stream.writeln("\n(durations < 0.001s were hidden; " + "use -v to show these durations)") + else: + self.stream.writeln("") def run(self, test): "Run the given test case or test suite." @@ -200,15 +232,6 @@ def run(self, test): if self.warnings: # if self.warnings is set, use it to filter all the warnings warnings.simplefilter(self.warnings) - # if the filter is 'default' or 'always', special-case the - # warnings from the deprecated unittest methods to show them - # no more than once per module, because they can be fairly - # noisy. The -Wd and -Wa flags can be used to bypass this - # only when self.warnings is None. - if self.warnings in ['default', 'always']: - warnings.filterwarnings('module', - category=DeprecationWarning, - message=r'Please use assert\w+ instead.') startTime = time.perf_counter() startTestRun = getattr(result, 'startTestRun', None) if startTestRun is not None: @@ -222,8 +245,12 @@ def run(self, test): stopTime = time.perf_counter() timeTaken = stopTime - startTime result.printErrors() + if self.durations is not None: + self._printDurations(result) + if hasattr(result, 'separator2'): self.stream.writeln(result.separator2) + run = result.testsRun self.stream.writeln("Ran %d test%s in %.3fs" % (run, run != 1 and "s" or "", timeTaken)) @@ -247,6 +274,8 @@ def run(self, test): infos.append("failures=%d" % failed) if errored: infos.append("errors=%d" % errored) + elif run == 0 and not skipped: + self.stream.write("NO TESTS RAN") else: self.stream.write("OK") if skipped: diff --git a/Lib/unittest/test/__init__.py b/Lib/unittest/test/__init__.py deleted file mode 100644 index 143f4ab5a3..0000000000 --- a/Lib/unittest/test/__init__.py +++ /dev/null @@ -1,25 +0,0 @@ -import os -import sys -import unittest - - -here = os.path.dirname(__file__) -loader = unittest.defaultTestLoader - -def suite(): - suite = unittest.TestSuite() - for fn in os.listdir(here): - if fn.startswith("test") and fn.endswith(".py"): - modname = "unittest.test." + fn[:-3] - try: - __import__(modname) - except unittest.SkipTest: - continue - module = sys.modules[modname] - suite.addTest(loader.loadTestsFromModule(module)) - suite.addTest(loader.loadTestsFromName('unittest.test.testmock')) - return suite - - -if __name__ == "__main__": - unittest.main(defaultTest="suite") diff --git a/Lib/unittest/test/__main__.py b/Lib/unittest/test/__main__.py deleted file mode 100644 index 44d0591e84..0000000000 --- a/Lib/unittest/test/__main__.py +++ /dev/null @@ -1,18 +0,0 @@ -import os -import unittest - - -def load_tests(loader, standard_tests, pattern): - # top level directory cached on loader instance - this_dir = os.path.dirname(__file__) - pattern = pattern or "test_*.py" - # We are inside unittest.test, so the top-level is two notches up - top_level_dir = os.path.dirname(os.path.dirname(this_dir)) - package_tests = loader.discover(start_dir=this_dir, pattern=pattern, - top_level_dir=top_level_dir) - standard_tests.addTests(package_tests) - return standard_tests - - -if __name__ == '__main__': - unittest.main() diff --git a/Lib/unittest/test/testmock/__init__.py b/Lib/unittest/test/testmock/__init__.py deleted file mode 100644 index 661b577259..0000000000 --- a/Lib/unittest/test/testmock/__init__.py +++ /dev/null @@ -1,19 +0,0 @@ -import os -import sys -import unittest - - -here = os.path.dirname(__file__) -loader = unittest.defaultTestLoader - -def load_tests(*args): - suite = unittest.TestSuite() - # TODO: RUSTPYTHON; allow objects to be mocked better - return suite - for fn in os.listdir(here): - if fn.startswith("test") and fn.endswith(".py"): - modname = "unittest.test.testmock." + fn[:-3] - __import__(modname) - module = sys.modules[modname] - suite.addTest(loader.loadTestsFromModule(module)) - return suite diff --git a/Lib/uuid.py b/Lib/uuid.py index c286eac38e..55f46eb510 100644 --- a/Lib/uuid.py +++ b/Lib/uuid.py @@ -572,39 +572,43 @@ def _netstat_getnode(): try: import _uuid _generate_time_safe = getattr(_uuid, "generate_time_safe", None) + _has_stable_extractable_node = getattr(_uuid, "has_stable_extractable_node", False) _UuidCreate = getattr(_uuid, "UuidCreate", None) except ImportError: _uuid = None _generate_time_safe = None + _has_stable_extractable_node = False _UuidCreate = None def _unix_getnode(): """Get the hardware address on Unix using the _uuid extension module.""" - if _generate_time_safe: + if _generate_time_safe and _has_stable_extractable_node: uuid_time, _ = _generate_time_safe() return UUID(bytes=uuid_time).node def _windll_getnode(): """Get the hardware address on Windows using the _uuid extension module.""" - if _UuidCreate: + if _UuidCreate and _has_stable_extractable_node: uuid_bytes = _UuidCreate() return UUID(bytes_le=uuid_bytes).node def _random_getnode(): """Get a random node ID.""" - # RFC 4122, $4.1.6 says "For systems with no IEEE address, a randomly or - # pseudo-randomly generated value may be used; see Section 4.5. The - # multicast bit must be set in such addresses, in order that they will - # never conflict with addresses obtained from network cards." + # RFC 9562, §6.10-3 says that + # + # Implementations MAY elect to obtain a 48-bit cryptographic-quality + # random number as per Section 6.9 to use as the Node ID. [...] [and] + # implementations MUST set the least significant bit of the first octet + # of the Node ID to 1. This bit is the unicast or multicast bit, which + # will never be set in IEEE 802 addresses obtained from network cards. # # The "multicast bit" of a MAC address is defined to be "the least # significant bit of the first octet". This works out to be the 41st bit # counting from 1 being the least significant bit, or 1<<40. # # See https://en.wikipedia.org/w/index.php?title=MAC_address&oldid=1128764812#Universal_vs._local_(U/L_bit) - import random - return random.getrandbits(48) | (1 << 40) + return int.from_bytes(os.urandom(6)) | (1 << 40) # _OS_GETTERS, when known, are targeted for a specific OS or platform. diff --git a/Lib/venv/__init__.py b/Lib/venv/__init__.py index 6f1af294ae..f7a6d26140 100644 --- a/Lib/venv/__init__.py +++ b/Lib/venv/__init__.py @@ -11,9 +11,10 @@ import sys import sysconfig import types +import shlex -CORE_VENV_DEPS = ('pip', 'setuptools') +CORE_VENV_DEPS = ('pip',) logger = logging.getLogger(__name__) @@ -41,20 +42,24 @@ class EnvBuilder: environment :param prompt: Alternative terminal prefix for the environment. :param upgrade_deps: Update the base venv modules to the latest on PyPI + :param scm_ignore_files: Create ignore files for the SCMs specified by the + iterable. """ def __init__(self, system_site_packages=False, clear=False, symlinks=False, upgrade=False, with_pip=False, prompt=None, - upgrade_deps=False): + upgrade_deps=False, *, scm_ignore_files=frozenset()): self.system_site_packages = system_site_packages self.clear = clear self.symlinks = symlinks self.upgrade = upgrade self.with_pip = with_pip + self.orig_prompt = prompt if prompt == '.': # see bpo-38901 prompt = os.path.basename(os.getcwd()) self.prompt = prompt self.upgrade_deps = upgrade_deps + self.scm_ignore_files = frozenset(map(str.lower, scm_ignore_files)) def create(self, env_dir): """ @@ -65,6 +70,8 @@ def create(self, env_dir): """ env_dir = os.path.abspath(env_dir) context = self.ensure_directories(env_dir) + for scm in self.scm_ignore_files: + getattr(self, f"create_{scm}_ignore_file")(context) # See issue 24875. We need system_site_packages to be False # until after pip is installed. true_system_site_packages = self.system_site_packages @@ -92,6 +99,42 @@ def clear_directory(self, path): elif os.path.isdir(fn): shutil.rmtree(fn) + def _venv_path(self, env_dir, name): + vars = { + 'base': env_dir, + 'platbase': env_dir, + 'installed_base': env_dir, + 'installed_platbase': env_dir, + } + return sysconfig.get_path(name, scheme='venv', vars=vars) + + @classmethod + def _same_path(cls, path1, path2): + """Check whether two paths appear the same. + + Whether they refer to the same file is irrelevant; we're testing for + whether a human reader would look at the path string and easily tell + that they're the same file. + """ + if sys.platform == 'win32': + if os.path.normcase(path1) == os.path.normcase(path2): + return True + # gh-90329: Don't display a warning for short/long names + import _winapi + try: + path1 = _winapi.GetLongPathName(os.fsdecode(path1)) + except OSError: + pass + try: + path2 = _winapi.GetLongPathName(os.fsdecode(path2)) + except OSError: + pass + if os.path.normcase(path1) == os.path.normcase(path2): + return True + return False + else: + return path1 == path2 + def ensure_directories(self, env_dir): """ Create the directories for the environment. @@ -106,31 +149,38 @@ def create_if_needed(d): elif os.path.islink(d) or os.path.isfile(d): raise ValueError('Unable to create directory %r' % d) + if os.pathsep in os.fspath(env_dir): + raise ValueError(f'Refusing to create a venv in {env_dir} because ' + f'it contains the PATH separator {os.pathsep}.') if os.path.exists(env_dir) and self.clear: self.clear_directory(env_dir) context = types.SimpleNamespace() context.env_dir = env_dir context.env_name = os.path.split(env_dir)[1] - prompt = self.prompt if self.prompt is not None else context.env_name - context.prompt = '(%s) ' % prompt + context.prompt = self.prompt if self.prompt is not None else context.env_name create_if_needed(env_dir) executable = sys._base_executable + if not executable: # see gh-96861 + raise ValueError('Unable to determine path to the running ' + 'Python interpreter. Provide an explicit path or ' + 'check that your PATH environment variable is ' + 'correctly set.') dirname, exename = os.path.split(os.path.abspath(executable)) + if sys.platform == 'win32': + # Always create the simplest name in the venv. It will either be a + # link back to executable, or a copy of the appropriate launcher + _d = '_d' if os.path.splitext(exename)[0].endswith('_d') else '' + exename = f'python{_d}.exe' context.executable = executable context.python_dir = dirname context.python_exe = exename - if sys.platform == 'win32': - binname = 'Scripts' - incpath = 'Include' - libpath = os.path.join(env_dir, 'Lib', 'site-packages') - else: - binname = 'bin' - incpath = 'include' - libpath = os.path.join(env_dir, 'lib', - 'python%d.%d' % sys.version_info[:2], - 'site-packages') - context.inc_path = path = os.path.join(env_dir, incpath) - create_if_needed(path) + binpath = self._venv_path(env_dir, 'scripts') + incpath = self._venv_path(env_dir, 'include') + libpath = self._venv_path(env_dir, 'purelib') + + context.inc_path = incpath + create_if_needed(incpath) + context.lib_path = libpath create_if_needed(libpath) # Issue 21197: create lib64 as a symlink to lib on 64-bit non-OS X POSIX if ((sys.maxsize > 2**32) and (os.name == 'posix') and @@ -138,8 +188,8 @@ def create_if_needed(d): link_path = os.path.join(env_dir, 'lib64') if not os.path.exists(link_path): # Issue #21643 os.symlink('lib', link_path) - context.bin_path = binpath = os.path.join(env_dir, binname) - context.bin_name = binname + context.bin_path = binpath + context.bin_name = os.path.relpath(binpath, env_dir) context.env_exe = os.path.join(binpath, exename) create_if_needed(binpath) # Assign and update the command to use when launching the newly created @@ -149,7 +199,7 @@ def create_if_needed(d): # bpo-45337: Fix up env_exec_cmd to account for file system redirections. # Some redirects only apply to CreateFile and not CreateProcess real_env_exe = os.path.realpath(context.env_exe) - if os.path.normcase(real_env_exe) != os.path.normcase(context.env_exe): + if not self._same_path(real_env_exe, context.env_exe): logger.warning('Actual environment location may have moved due to ' 'redirects, links or junctions.\n' ' Requested location: "%s"\n' @@ -178,86 +228,84 @@ def create_configuration(self, context): f.write('version = %d.%d.%d\n' % sys.version_info[:3]) if self.prompt is not None: f.write(f'prompt = {self.prompt!r}\n') - - if os.name != 'nt': - def symlink_or_copy(self, src, dst, relative_symlinks_ok=False): - """ - Try symlinking a file, and if that fails, fall back to copying. - """ - force_copy = not self.symlinks - if not force_copy: - try: - if not os.path.islink(dst): # can't link to itself! - if relative_symlinks_ok: - assert os.path.dirname(src) == os.path.dirname(dst) - os.symlink(os.path.basename(src), dst) - else: - os.symlink(src, dst) - except Exception: # may need to use a more specific exception - logger.warning('Unable to symlink %r to %r', src, dst) - force_copy = True - if force_copy: - shutil.copyfile(src, dst) - else: - def symlink_or_copy(self, src, dst, relative_symlinks_ok=False): - """ - Try symlinking a file, and if that fails, fall back to copying. - """ - bad_src = os.path.lexists(src) and not os.path.exists(src) - if self.symlinks and not bad_src and not os.path.islink(dst): - try: + f.write('executable = %s\n' % os.path.realpath(sys.executable)) + args = [] + nt = os.name == 'nt' + if nt and self.symlinks: + args.append('--symlinks') + if not nt and not self.symlinks: + args.append('--copies') + if not self.with_pip: + args.append('--without-pip') + if self.system_site_packages: + args.append('--system-site-packages') + if self.clear: + args.append('--clear') + if self.upgrade: + args.append('--upgrade') + if self.upgrade_deps: + args.append('--upgrade-deps') + if self.orig_prompt is not None: + args.append(f'--prompt="{self.orig_prompt}"') + if not self.scm_ignore_files: + args.append('--without-scm-ignore-files') + + args.append(context.env_dir) + args = ' '.join(args) + f.write(f'command = {sys.executable} -m venv {args}\n') + + def symlink_or_copy(self, src, dst, relative_symlinks_ok=False): + """ + Try symlinking a file, and if that fails, fall back to copying. + (Unused on Windows, because we can't just copy a failed symlink file: we + switch to a different set of files instead.) + """ + assert os.name != 'nt' + force_copy = not self.symlinks + if not force_copy: + try: + if not os.path.islink(dst): # can't link to itself! if relative_symlinks_ok: assert os.path.dirname(src) == os.path.dirname(dst) os.symlink(os.path.basename(src), dst) else: os.symlink(src, dst) - return - except Exception: # may need to use a more specific exception - logger.warning('Unable to symlink %r to %r', src, dst) - - # On Windows, we rewrite symlinks to our base python.exe into - # copies of venvlauncher.exe - basename, ext = os.path.splitext(os.path.basename(src)) - srcfn = os.path.join(os.path.dirname(__file__), - "scripts", - "nt", - basename + ext) - # Builds or venv's from builds need to remap source file - # locations, as we do not put them into Lib/venv/scripts - if sysconfig.is_python_build(True) or not os.path.isfile(srcfn): - if basename.endswith('_d'): - ext = '_d' + ext - basename = basename[:-2] - if basename == 'python': - basename = 'venvlauncher' - elif basename == 'pythonw': - basename = 'venvwlauncher' - src = os.path.join(os.path.dirname(src), basename + ext) - else: - src = srcfn - if not os.path.exists(src): - if not bad_src: - logger.warning('Unable to copy %r', src) - return - + except Exception: # may need to use a more specific exception + logger.warning('Unable to symlink %r to %r', src, dst) + force_copy = True + if force_copy: shutil.copyfile(src, dst) - def setup_python(self, context): + def create_git_ignore_file(self, context): """ - Set up a Python executable in the environment. + Create a .gitignore file in the environment directory. - :param context: The information for the environment creation request - being processed. + The contents of the file cause the entire environment directory to be + ignored by git. """ - binpath = context.bin_path - path = context.env_exe - copier = self.symlink_or_copy - dirname = context.python_dir - if os.name != 'nt': + gitignore_path = os.path.join(context.env_dir, '.gitignore') + with open(gitignore_path, 'w', encoding='utf-8') as file: + file.write('# Created by venv; ' + 'see https://docs.python.org/3/library/venv.html\n') + file.write('*\n') + + if os.name != 'nt': + def setup_python(self, context): + """ + Set up a Python executable in the environment. + + :param context: The information for the environment creation request + being processed. + """ + binpath = context.bin_path + path = context.env_exe + copier = self.symlink_or_copy + dirname = context.python_dir copier(context.executable, path) if not os.path.islink(path): os.chmod(path, 0o755) - for suffix in ('python', 'python3', f'python3.{sys.version_info[1]}'): + for suffix in ('python', 'python3', + f'python3.{sys.version_info[1]}'): path = os.path.join(binpath, suffix) if not os.path.exists(path): # Issue 18807: make copies if @@ -265,32 +313,107 @@ def setup_python(self, context): copier(context.env_exe, path, relative_symlinks_ok=True) if not os.path.islink(path): os.chmod(path, 0o755) - else: - if self.symlinks: - # For symlinking, we need a complete copy of the root directory - # If symlinks fail, you'll get unnecessary copies of files, but - # we assume that if you've opted into symlinks on Windows then - # you know what you're doing. - suffixes = [ - f for f in os.listdir(dirname) if - os.path.normcase(os.path.splitext(f)[1]) in ('.exe', '.dll') - ] - if sysconfig.is_python_build(True): - suffixes = [ - f for f in suffixes if - os.path.normcase(f).startswith(('python', 'vcruntime')) - ] + + else: + def setup_python(self, context): + """ + Set up a Python executable in the environment. + + :param context: The information for the environment creation request + being processed. + """ + binpath = context.bin_path + dirname = context.python_dir + exename = os.path.basename(context.env_exe) + exe_stem = os.path.splitext(exename)[0] + exe_d = '_d' if os.path.normcase(exe_stem).endswith('_d') else '' + if sysconfig.is_python_build(): + scripts = dirname else: - suffixes = {'python.exe', 'python_d.exe', 'pythonw.exe', 'pythonw_d.exe'} - base_exe = os.path.basename(context.env_exe) - suffixes.add(base_exe) + scripts = os.path.join(os.path.dirname(__file__), + 'scripts', 'nt') + if not sysconfig.get_config_var("Py_GIL_DISABLED"): + python_exe = os.path.join(dirname, f'python{exe_d}.exe') + pythonw_exe = os.path.join(dirname, f'pythonw{exe_d}.exe') + link_sources = { + 'python.exe': python_exe, + f'python{exe_d}.exe': python_exe, + 'pythonw.exe': pythonw_exe, + f'pythonw{exe_d}.exe': pythonw_exe, + } + python_exe = os.path.join(scripts, f'venvlauncher{exe_d}.exe') + pythonw_exe = os.path.join(scripts, f'venvwlauncher{exe_d}.exe') + copy_sources = { + 'python.exe': python_exe, + f'python{exe_d}.exe': python_exe, + 'pythonw.exe': pythonw_exe, + f'pythonw{exe_d}.exe': pythonw_exe, + } + else: + exe_t = f'3.{sys.version_info[1]}t' + python_exe = os.path.join(dirname, f'python{exe_t}{exe_d}.exe') + pythonw_exe = os.path.join(dirname, f'pythonw{exe_t}{exe_d}.exe') + link_sources = { + 'python.exe': python_exe, + f'python{exe_d}.exe': python_exe, + f'python{exe_t}.exe': python_exe, + f'python{exe_t}{exe_d}.exe': python_exe, + 'pythonw.exe': pythonw_exe, + f'pythonw{exe_d}.exe': pythonw_exe, + f'pythonw{exe_t}.exe': pythonw_exe, + f'pythonw{exe_t}{exe_d}.exe': pythonw_exe, + } + python_exe = os.path.join(scripts, f'venvlaunchert{exe_d}.exe') + pythonw_exe = os.path.join(scripts, f'venvwlaunchert{exe_d}.exe') + copy_sources = { + 'python.exe': python_exe, + f'python{exe_d}.exe': python_exe, + f'python{exe_t}.exe': python_exe, + f'python{exe_t}{exe_d}.exe': python_exe, + 'pythonw.exe': pythonw_exe, + f'pythonw{exe_d}.exe': pythonw_exe, + f'pythonw{exe_t}.exe': pythonw_exe, + f'pythonw{exe_t}{exe_d}.exe': pythonw_exe, + } + + do_copies = True + if self.symlinks: + do_copies = False + # For symlinking, we need all the DLLs to be available alongside + # the executables. + link_sources.update({ + f: os.path.join(dirname, f) for f in os.listdir(dirname) + if os.path.normcase(f).startswith(('python', 'vcruntime')) + and os.path.normcase(os.path.splitext(f)[1]) == '.dll' + }) + + to_unlink = [] + for dest, src in link_sources.items(): + dest = os.path.join(binpath, dest) + try: + os.symlink(src, dest) + to_unlink.append(dest) + except OSError: + logger.warning('Unable to symlink %r to %r', src, dest) + do_copies = True + for f in to_unlink: + try: + os.unlink(f) + except OSError: + logger.warning('Failed to clean up symlink %r', + f) + logger.warning('Retrying with copies') + break - for suffix in suffixes: - src = os.path.join(dirname, suffix) - if os.path.lexists(src): - copier(src, os.path.join(binpath, suffix)) + if do_copies: + for dest, src in copy_sources.items(): + dest = os.path.join(binpath, dest) + try: + shutil.copy2(src, dest) + except OSError: + logger.warning('Unable to copy %r to %r', src, dest) - if sysconfig.is_python_build(True): + if sysconfig.is_python_build(): # copy init.tcl for root, dirs, files in os.walk(context.python_dir): if 'init.tcl' in files: @@ -303,14 +426,25 @@ def setup_python(self, context): shutil.copyfile(src, dst) break + def _call_new_python(self, context, *py_args, **kwargs): + """Executes the newly created Python using safe-ish options""" + # gh-98251: We do not want to just use '-I' because that masks + # legitimate user preferences (such as not writing bytecode). All we + # really need is to ensure that the path variables do not overrule + # normal venv handling. + args = [context.env_exec_cmd, *py_args] + kwargs['env'] = env = os.environ.copy() + env['VIRTUAL_ENV'] = context.env_dir + env.pop('PYTHONHOME', None) + env.pop('PYTHONPATH', None) + kwargs['cwd'] = context.env_dir + kwargs['executable'] = context.env_exec_cmd + subprocess.check_output(args, **kwargs) + def _setup_pip(self, context): """Installs or upgrades pip in a virtual environment""" - # We run ensurepip in isolated mode to avoid side effects from - # environment vars, the current directory and anything else - # intended for the global Python environment - cmd = [context.env_exec_cmd, '-Im', 'ensurepip', '--upgrade', - '--default-pip'] - subprocess.check_output(cmd, stderr=subprocess.STDOUT) + self._call_new_python(context, '-m', 'ensurepip', '--upgrade', + '--default-pip', stderr=subprocess.STDOUT) def setup_scripts(self, context): """ @@ -348,11 +482,41 @@ def replace_variables(self, text, context): :param context: The information for the environment creation request being processed. """ - text = text.replace('__VENV_DIR__', context.env_dir) - text = text.replace('__VENV_NAME__', context.env_name) - text = text.replace('__VENV_PROMPT__', context.prompt) - text = text.replace('__VENV_BIN_NAME__', context.bin_name) - text = text.replace('__VENV_PYTHON__', context.env_exe) + replacements = { + '__VENV_DIR__': context.env_dir, + '__VENV_NAME__': context.env_name, + '__VENV_PROMPT__': context.prompt, + '__VENV_BIN_NAME__': context.bin_name, + '__VENV_PYTHON__': context.env_exe, + } + + def quote_ps1(s): + """ + This should satisfy PowerShell quoting rules [1], unless the quoted + string is passed directly to Windows native commands [2]. + [1]: https://learn.microsoft.com/en-us/powershell/module/microsoft.powershell.core/about/about_quoting_rules + [2]: https://learn.microsoft.com/en-us/powershell/module/microsoft.powershell.core/about/about_parsing#passing-arguments-that-contain-quote-characters + """ + s = s.replace("'", "''") + return f"'{s}'" + + def quote_bat(s): + return s + + # gh-124651: need to quote the template strings properly + quote = shlex.quote + script_path = context.script_path + if script_path.endswith('.ps1'): + quote = quote_ps1 + elif script_path.endswith('.bat'): + quote = quote_bat + else: + # fallbacks to POSIX shell compliant quote + quote = shlex.quote + + replacements = {key: quote(s) for key, s in replacements.items()} + for key, quoted in replacements.items(): + text = text.replace(key, quoted) return text def install_scripts(self, context, path): @@ -370,15 +534,22 @@ def install_scripts(self, context, path): """ binpath = context.bin_path plen = len(path) + if os.name == 'nt': + def skip_file(f): + f = os.path.normcase(f) + return (f.startswith(('python', 'venv')) + and f.endswith(('.exe', '.pdb'))) + else: + def skip_file(f): + return False for root, dirs, files in os.walk(path): - if root == path: # at top-level, remove irrelevant dirs + if root == path: # at top-level, remove irrelevant dirs for d in dirs[:]: if d not in ('common', os.name): dirs.remove(d) - continue # ignore files in top level + continue # ignore files in top level for f in files: - if (os.name == 'nt' and f.startswith('python') - and f.endswith(('.exe', '.pdb'))): + if skip_file(f): continue srcfile = os.path.join(root, f) suffix = root[plen:].split(os.sep)[2:] @@ -389,116 +560,122 @@ def install_scripts(self, context, path): if not os.path.exists(dstdir): os.makedirs(dstdir) dstfile = os.path.join(dstdir, f) + if os.name == 'nt' and srcfile.endswith(('.exe', '.pdb')): + shutil.copy2(srcfile, dstfile) + continue with open(srcfile, 'rb') as f: data = f.read() - if not srcfile.endswith(('.exe', '.pdb')): - try: - data = data.decode('utf-8') - data = self.replace_variables(data, context) - data = data.encode('utf-8') - except UnicodeError as e: - data = None - logger.warning('unable to copy script %r, ' - 'may be binary: %s', srcfile, e) - if data is not None: + try: + context.script_path = srcfile + new_data = ( + self.replace_variables(data.decode('utf-8'), context) + .encode('utf-8') + ) + except UnicodeError as e: + logger.warning('unable to copy script %r, ' + 'may be binary: %s', srcfile, e) + continue + if new_data == data: + shutil.copy2(srcfile, dstfile) + else: with open(dstfile, 'wb') as f: - f.write(data) + f.write(new_data) shutil.copymode(srcfile, dstfile) def upgrade_dependencies(self, context): logger.debug( f'Upgrading {CORE_VENV_DEPS} packages in {context.bin_path}' ) - cmd = [context.env_exec_cmd, '-m', 'pip', 'install', '--upgrade'] - cmd.extend(CORE_VENV_DEPS) - subprocess.check_call(cmd) + self._call_new_python(context, '-m', 'pip', 'install', '--upgrade', + *CORE_VENV_DEPS) def create(env_dir, system_site_packages=False, clear=False, - symlinks=False, with_pip=False, prompt=None, upgrade_deps=False): + symlinks=False, with_pip=False, prompt=None, upgrade_deps=False, + *, scm_ignore_files=frozenset()): """Create a virtual environment in a directory.""" builder = EnvBuilder(system_site_packages=system_site_packages, clear=clear, symlinks=symlinks, with_pip=with_pip, - prompt=prompt, upgrade_deps=upgrade_deps) + prompt=prompt, upgrade_deps=upgrade_deps, + scm_ignore_files=scm_ignore_files) builder.create(env_dir) + def main(args=None): - compatible = True - if sys.version_info < (3, 3): - compatible = False - elif not hasattr(sys, 'base_prefix'): - compatible = False - if not compatible: - raise ValueError('This script is only for use with Python >= 3.3') + import argparse + + parser = argparse.ArgumentParser(prog=__name__, + description='Creates virtual Python ' + 'environments in one or ' + 'more target ' + 'directories.', + epilog='Once an environment has been ' + 'created, you may wish to ' + 'activate it, e.g. by ' + 'sourcing an activate script ' + 'in its bin directory.') + parser.add_argument('dirs', metavar='ENV_DIR', nargs='+', + help='A directory to create the environment in.') + parser.add_argument('--system-site-packages', default=False, + action='store_true', dest='system_site', + help='Give the virtual environment access to the ' + 'system site-packages dir.') + if os.name == 'nt': + use_symlinks = False else: - import argparse - - parser = argparse.ArgumentParser(prog=__name__, - description='Creates virtual Python ' - 'environments in one or ' - 'more target ' - 'directories.', - epilog='Once an environment has been ' - 'created, you may wish to ' - 'activate it, e.g. by ' - 'sourcing an activate script ' - 'in its bin directory.') - parser.add_argument('dirs', metavar='ENV_DIR', nargs='+', - help='A directory to create the environment in.') - parser.add_argument('--system-site-packages', default=False, - action='store_true', dest='system_site', - help='Give the virtual environment access to the ' - 'system site-packages dir.') - if os.name == 'nt': - use_symlinks = False - else: - use_symlinks = True - group = parser.add_mutually_exclusive_group() - group.add_argument('--symlinks', default=use_symlinks, - action='store_true', dest='symlinks', - help='Try to use symlinks rather than copies, ' - 'when symlinks are not the default for ' - 'the platform.') - group.add_argument('--copies', default=not use_symlinks, - action='store_false', dest='symlinks', - help='Try to use copies rather than symlinks, ' - 'even when symlinks are the default for ' - 'the platform.') - parser.add_argument('--clear', default=False, action='store_true', - dest='clear', help='Delete the contents of the ' - 'environment directory if it ' - 'already exists, before ' - 'environment creation.') - parser.add_argument('--upgrade', default=False, action='store_true', - dest='upgrade', help='Upgrade the environment ' - 'directory to use this version ' - 'of Python, assuming Python ' - 'has been upgraded in-place.') - parser.add_argument('--without-pip', dest='with_pip', - default=True, action='store_false', - help='Skips installing or upgrading pip in the ' - 'virtual environment (pip is bootstrapped ' - 'by default)') - parser.add_argument('--prompt', - help='Provides an alternative prompt prefix for ' - 'this environment.') - parser.add_argument('--upgrade-deps', default=False, action='store_true', - dest='upgrade_deps', - help='Upgrade core dependencies: {} to the latest ' - 'version in PyPI'.format( - ' '.join(CORE_VENV_DEPS))) - options = parser.parse_args(args) - if options.upgrade and options.clear: - raise ValueError('you cannot supply --upgrade and --clear together.') - builder = EnvBuilder(system_site_packages=options.system_site, - clear=options.clear, - symlinks=options.symlinks, - upgrade=options.upgrade, - with_pip=options.with_pip, - prompt=options.prompt, - upgrade_deps=options.upgrade_deps) - for d in options.dirs: - builder.create(d) + use_symlinks = True + group = parser.add_mutually_exclusive_group() + group.add_argument('--symlinks', default=use_symlinks, + action='store_true', dest='symlinks', + help='Try to use symlinks rather than copies, ' + 'when symlinks are not the default for ' + 'the platform.') + group.add_argument('--copies', default=not use_symlinks, + action='store_false', dest='symlinks', + help='Try to use copies rather than symlinks, ' + 'even when symlinks are the default for ' + 'the platform.') + parser.add_argument('--clear', default=False, action='store_true', + dest='clear', help='Delete the contents of the ' + 'environment directory if it ' + 'already exists, before ' + 'environment creation.') + parser.add_argument('--upgrade', default=False, action='store_true', + dest='upgrade', help='Upgrade the environment ' + 'directory to use this version ' + 'of Python, assuming Python ' + 'has been upgraded in-place.') + parser.add_argument('--without-pip', dest='with_pip', + default=True, action='store_false', + help='Skips installing or upgrading pip in the ' + 'virtual environment (pip is bootstrapped ' + 'by default)') + parser.add_argument('--prompt', + help='Provides an alternative prompt prefix for ' + 'this environment.') + parser.add_argument('--upgrade-deps', default=False, action='store_true', + dest='upgrade_deps', + help=f'Upgrade core dependencies ({", ".join(CORE_VENV_DEPS)}) ' + 'to the latest version in PyPI') + parser.add_argument('--without-scm-ignore-files', dest='scm_ignore_files', + action='store_const', const=frozenset(), + default=frozenset(['git']), + help='Skips adding SCM ignore files to the environment ' + 'directory (Git is supported by default).') + options = parser.parse_args(args) + if options.upgrade and options.clear: + raise ValueError('you cannot supply --upgrade and --clear together.') + builder = EnvBuilder(system_site_packages=options.system_site, + clear=options.clear, + symlinks=options.symlinks, + upgrade=options.upgrade, + with_pip=options.with_pip, + prompt=options.prompt, + upgrade_deps=options.upgrade_deps, + scm_ignore_files=options.scm_ignore_files) + for d in options.dirs: + builder.create(d) + if __name__ == '__main__': rc = 1 diff --git a/Lib/venv/__main__.py b/Lib/venv/__main__.py index 912423e4a7..88f55439dc 100644 --- a/Lib/venv/__main__.py +++ b/Lib/venv/__main__.py @@ -6,5 +6,5 @@ main() rc = 0 except Exception as e: - print('Error: %s' % e, file=sys.stderr) + print('Error:', e, file=sys.stderr) sys.exit(rc) diff --git a/Lib/venv/scripts/common/Activate.ps1 b/Lib/venv/scripts/common/Activate.ps1 index b49d77ba44..16ba5290fa 100644 --- a/Lib/venv/scripts/common/Activate.ps1 +++ b/Lib/venv/scripts/common/Activate.ps1 @@ -219,6 +219,8 @@ deactivate -nondestructive # that there is an activated venv. $env:VIRTUAL_ENV = $VenvDir +$env:VIRTUAL_ENV_PROMPT = $Prompt + if (-not $Env:VIRTUAL_ENV_DISABLE_PROMPT) { Write-Verbose "Setting prompt to '$Prompt'" @@ -233,7 +235,6 @@ if (-not $Env:VIRTUAL_ENV_DISABLE_PROMPT) { Write-Host -NoNewline -ForegroundColor Green "($_PYTHON_VENV_PROMPT_PREFIX) " _OLD_VIRTUAL_PROMPT } - $env:VIRTUAL_ENV_PROMPT = $Prompt } # Clear PYTHONHOME diff --git a/Lib/venv/scripts/common/activate b/Lib/venv/scripts/common/activate index 6fbc2b8801..70673a265d 100644 --- a/Lib/venv/scripts/common/activate +++ b/Lib/venv/scripts/common/activate @@ -1,5 +1,5 @@ # This file must be used with "source bin/activate" *from bash* -# you cannot run it directly +# You cannot run it directly deactivate () { # reset old environment variables @@ -14,12 +14,10 @@ deactivate () { unset _OLD_VIRTUAL_PYTHONHOME fi - # This should detect bash and zsh, which have a hash command that must - # be called to get it to forget past commands. Without forgetting - # past commands the $PATH changes we made may not be respected - if [ -n "${BASH:-}" -o -n "${ZSH_VERSION:-}" ] ; then - hash -r 2> /dev/null - fi + # Call hash to forget past locations. Without forgetting + # past locations the $PATH changes we made may not be respected. + # See "man bash" for more details. hash is usually a builtin of your shell + hash -r 2> /dev/null if [ -n "${_OLD_VIRTUAL_PS1:-}" ] ; then PS1="${_OLD_VIRTUAL_PS1:-}" @@ -38,13 +36,27 @@ deactivate () { # unset irrelevant variables deactivate nondestructive -VIRTUAL_ENV="__VENV_DIR__" -export VIRTUAL_ENV +# on Windows, a path can contain colons and backslashes and has to be converted: +case "$(uname)" in + CYGWIN*|MSYS*|MINGW*) + # transform D:\path\to\venv to /d/path/to/venv on MSYS and MINGW + # and to /cygdrive/d/path/to/venv on Cygwin + VIRTUAL_ENV=$(cygpath __VENV_DIR__) + export VIRTUAL_ENV + ;; + *) + # use the path as-is + export VIRTUAL_ENV=__VENV_DIR__ + ;; +esac _OLD_VIRTUAL_PATH="$PATH" -PATH="$VIRTUAL_ENV/__VENV_BIN_NAME__:$PATH" +PATH="$VIRTUAL_ENV/"__VENV_BIN_NAME__":$PATH" export PATH +VIRTUAL_ENV_PROMPT=__VENV_PROMPT__ +export VIRTUAL_ENV_PROMPT + # unset PYTHONHOME if set # this will fail if PYTHONHOME is set to the empty string (which is bad anyway) # could use `if (set -u; : $PYTHONHOME) ;` in bash @@ -55,15 +67,10 @@ fi if [ -z "${VIRTUAL_ENV_DISABLE_PROMPT:-}" ] ; then _OLD_VIRTUAL_PS1="${PS1:-}" - PS1="__VENV_PROMPT__${PS1:-}" + PS1="("__VENV_PROMPT__") ${PS1:-}" export PS1 - VIRTUAL_ENV_PROMPT="__VENV_PROMPT__" - export VIRTUAL_ENV_PROMPT fi -# This should detect bash and zsh, which have a hash command that must -# be called to get it to forget past commands. Without forgetting +# Call hash to forget past commands. Without forgetting # past commands the $PATH changes we made may not be respected -if [ -n "${BASH:-}" -o -n "${ZSH_VERSION:-}" ] ; then - hash -r 2> /dev/null -fi +hash -r 2> /dev/null diff --git a/Lib/venv/scripts/posix/activate.fish b/Lib/venv/scripts/common/activate.fish similarity index 76% rename from Lib/venv/scripts/posix/activate.fish rename to Lib/venv/scripts/common/activate.fish index e40a1d7148..284a7469c9 100644 --- a/Lib/venv/scripts/posix/activate.fish +++ b/Lib/venv/scripts/common/activate.fish @@ -1,5 +1,5 @@ # This file must be used with "source /bin/activate.fish" *from fish* -# (https://fishshell.com/); you cannot run it directly. +# (https://fishshell.com/). You cannot run it directly. function deactivate -d "Exit virtual environment and return to normal shell environment" # reset old environment variables @@ -13,10 +13,13 @@ function deactivate -d "Exit virtual environment and return to normal shell env end if test -n "$_OLD_FISH_PROMPT_OVERRIDE" - functions -e fish_prompt set -e _OLD_FISH_PROMPT_OVERRIDE - functions -c _old_fish_prompt fish_prompt - functions -e _old_fish_prompt + # prevents error when using nested fish instances (Issue #93858) + if functions -q _old_fish_prompt + functions -e fish_prompt + functions -c _old_fish_prompt fish_prompt + functions -e _old_fish_prompt + end end set -e VIRTUAL_ENV @@ -30,10 +33,11 @@ end # Unset irrelevant variables. deactivate nondestructive -set -gx VIRTUAL_ENV "__VENV_DIR__" +set -gx VIRTUAL_ENV __VENV_DIR__ set -gx _OLD_VIRTUAL_PATH $PATH -set -gx PATH "$VIRTUAL_ENV/__VENV_BIN_NAME__" $PATH +set -gx PATH "$VIRTUAL_ENV/"__VENV_BIN_NAME__ $PATH +set -gx VIRTUAL_ENV_PROMPT __VENV_PROMPT__ # Unset PYTHONHOME if set. if set -q PYTHONHOME @@ -53,7 +57,7 @@ if test -z "$VIRTUAL_ENV_DISABLE_PROMPT" set -l old_status $status # Output the venv prompt; color taken from the blue of the Python logo. - printf "%s%s%s" (set_color 4B8BBE) "__VENV_PROMPT__" (set_color normal) + printf "%s(%s)%s " (set_color 4B8BBE) __VENV_PROMPT__ (set_color normal) # Restore the return status of the previous command. echo "exit $old_status" | . @@ -62,5 +66,4 @@ if test -z "$VIRTUAL_ENV_DISABLE_PROMPT" end set -gx _OLD_FISH_PROMPT_OVERRIDE "$VIRTUAL_ENV" - set -gx VIRTUAL_ENV_PROMPT "__VENV_PROMPT__" end diff --git a/Lib/venv/scripts/nt/activate.bat b/Lib/venv/scripts/nt/activate.bat index 5daa45afc9..9ac5c20b47 100644 --- a/Lib/venv/scripts/nt/activate.bat +++ b/Lib/venv/scripts/nt/activate.bat @@ -8,15 +8,15 @@ if defined _OLD_CODEPAGE ( "%SystemRoot%\System32\chcp.com" 65001 > nul ) -set VIRTUAL_ENV=__VENV_DIR__ +set "VIRTUAL_ENV=__VENV_DIR__" if not defined PROMPT set PROMPT=$P$G if defined _OLD_VIRTUAL_PROMPT set PROMPT=%_OLD_VIRTUAL_PROMPT% if defined _OLD_VIRTUAL_PYTHONHOME set PYTHONHOME=%_OLD_VIRTUAL_PYTHONHOME% -set _OLD_VIRTUAL_PROMPT=%PROMPT% -set PROMPT=__VENV_PROMPT__%PROMPT% +set "_OLD_VIRTUAL_PROMPT=%PROMPT%" +set "PROMPT=(__VENV_PROMPT__) %PROMPT%" if defined PYTHONHOME set _OLD_VIRTUAL_PYTHONHOME=%PYTHONHOME% set PYTHONHOME= @@ -24,8 +24,8 @@ set PYTHONHOME= if defined _OLD_VIRTUAL_PATH set PATH=%_OLD_VIRTUAL_PATH% if not defined _OLD_VIRTUAL_PATH set _OLD_VIRTUAL_PATH=%PATH% -set PATH=%VIRTUAL_ENV%\__VENV_BIN_NAME__;%PATH% -set VIRTUAL_ENV_PROMPT=__VENV_PROMPT__ +set "PATH=%VIRTUAL_ENV%\__VENV_BIN_NAME__;%PATH%" +set "VIRTUAL_ENV_PROMPT=__VENV_PROMPT__" :END if defined _OLD_CODEPAGE ( diff --git a/Lib/venv/scripts/nt/venvlauncher.exe b/Lib/venv/scripts/nt/venvlauncher.exe new file mode 100644 index 0000000000..2439c22aa9 Binary files /dev/null and b/Lib/venv/scripts/nt/venvlauncher.exe differ diff --git a/Lib/venv/scripts/nt/venvlaunchert.exe b/Lib/venv/scripts/nt/venvlaunchert.exe new file mode 100644 index 0000000000..99f5f5e9fc Binary files /dev/null and b/Lib/venv/scripts/nt/venvlaunchert.exe differ diff --git a/Lib/venv/scripts/nt/venvwlauncher.exe b/Lib/venv/scripts/nt/venvwlauncher.exe new file mode 100644 index 0000000000..6c43c2e9d9 Binary files /dev/null and b/Lib/venv/scripts/nt/venvwlauncher.exe differ diff --git a/Lib/venv/scripts/nt/venvwlaunchert.exe b/Lib/venv/scripts/nt/venvwlaunchert.exe new file mode 100644 index 0000000000..74f40deb04 Binary files /dev/null and b/Lib/venv/scripts/nt/venvwlaunchert.exe differ diff --git a/Lib/venv/scripts/posix/activate.csh b/Lib/venv/scripts/posix/activate.csh index d6f697c55e..2a3fa83547 100644 --- a/Lib/venv/scripts/posix/activate.csh +++ b/Lib/venv/scripts/posix/activate.csh @@ -1,5 +1,6 @@ # This file must be used with "source bin/activate.csh" *from csh*. # You cannot run it directly. + # Created by Davide Di Blasi . # Ported to Python 3.3 venv by Andrew Svetlov @@ -8,17 +9,17 @@ alias deactivate 'test $?_OLD_VIRTUAL_PATH != 0 && setenv PATH "$_OLD_VIRTUAL_PA # Unset irrelevant variables. deactivate nondestructive -setenv VIRTUAL_ENV "__VENV_DIR__" +setenv VIRTUAL_ENV __VENV_DIR__ set _OLD_VIRTUAL_PATH="$PATH" -setenv PATH "$VIRTUAL_ENV/__VENV_BIN_NAME__:$PATH" +setenv PATH "$VIRTUAL_ENV/"__VENV_BIN_NAME__":$PATH" +setenv VIRTUAL_ENV_PROMPT __VENV_PROMPT__ set _OLD_VIRTUAL_PROMPT="$prompt" if (! "$?VIRTUAL_ENV_DISABLE_PROMPT") then - set prompt = "__VENV_PROMPT__$prompt" - setenv VIRTUAL_ENV_PROMPT "__VENV_PROMPT__" + set prompt = "("__VENV_PROMPT__") $prompt:q" endif alias pydoc python -m pydoc diff --git a/Lib/warnings.py b/Lib/warnings.py index 7d8c440012..f83aaf231e 100644 --- a/Lib/warnings.py +++ b/Lib/warnings.py @@ -5,7 +5,7 @@ __all__ = ["warn", "warn_explicit", "showwarning", "formatwarning", "filterwarnings", "simplefilter", - "resetwarnings", "catch_warnings"] + "resetwarnings", "catch_warnings", "deprecated"] def showwarning(message, category, filename, lineno, file=None, line=None): """Hook to write a warning to a file; replace if you like.""" @@ -58,15 +58,16 @@ def _formatwarnmsg_impl(msg): # catch Exception, not only ImportError and RecursionError. except Exception: # don't suggest to enable tracemalloc if it's not available - tracing = True + suggest_tracemalloc = False tb = None else: - tracing = tracemalloc.is_tracing() try: + suggest_tracemalloc = not tracemalloc.is_tracing() tb = tracemalloc.get_object_traceback(msg.source) except Exception: # When a warning is logged during Python shutdown, tracemalloc # and the import machinery don't work anymore + suggest_tracemalloc = False tb = None if tb is not None: @@ -85,7 +86,7 @@ def _formatwarnmsg_impl(msg): if line: line = line.strip() s += ' %s\n' % line - elif not tracing: + elif suggest_tracemalloc: s += (f'{category}: Enable tracemalloc to get the object ' f'allocation traceback\n') return s @@ -139,14 +140,18 @@ def filterwarnings(action, message="", category=Warning, module="", lineno=0, 'lineno' -- an integer line number, 0 matches all warnings 'append' -- if true, append to the list of filters """ - assert action in ("error", "ignore", "always", "default", "module", - "once"), "invalid action: %r" % (action,) - assert isinstance(message, str), "message must be a string" - assert isinstance(category, type), "category must be a class" - assert issubclass(category, Warning), "category must be a Warning subclass" - assert isinstance(module, str), "module must be a string" - assert isinstance(lineno, int) and lineno >= 0, \ - "lineno must be an int >= 0" + if action not in {"error", "ignore", "always", "default", "module", "once"}: + raise ValueError(f"invalid action: {action!r}") + if not isinstance(message, str): + raise TypeError("message must be a string") + if not isinstance(category, type) or not issubclass(category, Warning): + raise TypeError("category must be a Warning subclass") + if not isinstance(module, str): + raise TypeError("module must be a string") + if not isinstance(lineno, int): + raise TypeError("lineno must be an int") + if lineno < 0: + raise ValueError("lineno must be an int >= 0") if message or module: import re @@ -172,10 +177,12 @@ def simplefilter(action, category=Warning, lineno=0, append=False): 'lineno' -- an integer line number, 0 matches all warnings 'append' -- if true, append to the list of filters """ - assert action in ("error", "ignore", "always", "default", "module", - "once"), "invalid action: %r" % (action,) - assert isinstance(lineno, int) and lineno >= 0, \ - "lineno must be an int >= 0" + if action not in {"error", "ignore", "always", "default", "module", "once"}: + raise ValueError(f"invalid action: {action!r}") + if not isinstance(lineno, int): + raise TypeError("lineno must be an int") + if lineno < 0: + raise ValueError("lineno must be an int >= 0") _add_filter(action, None, category, None, lineno, append=append) def _add_filter(*item, append): @@ -269,22 +276,32 @@ def _getcategory(category): return cat +def _is_internal_filename(filename): + return 'importlib' in filename and '_bootstrap' in filename + + +def _is_filename_to_skip(filename, skip_file_prefixes): + return any(filename.startswith(prefix) for prefix in skip_file_prefixes) + + def _is_internal_frame(frame): """Signal whether the frame is an internal CPython implementation detail.""" - filename = frame.f_code.co_filename - return 'importlib' in filename and '_bootstrap' in filename + return _is_internal_filename(frame.f_code.co_filename) -def _next_external_frame(frame): - """Find the next frame that doesn't involve CPython internals.""" +def _next_external_frame(frame, skip_file_prefixes): + """Find the next frame that doesn't involve Python or user internals.""" frame = frame.f_back - while frame is not None and _is_internal_frame(frame): + while frame is not None and ( + _is_internal_filename(filename := frame.f_code.co_filename) or + _is_filename_to_skip(filename, skip_file_prefixes)): frame = frame.f_back return frame # Code typically replaced by _warnings -def warn(message, category=None, stacklevel=1, source=None): +def warn(message, category=None, stacklevel=1, source=None, + *, skip_file_prefixes=()): """Issue a warning, or maybe ignore it or raise an exception.""" # Check if message is already a Warning object if isinstance(message, Warning): @@ -295,6 +312,11 @@ def warn(message, category=None, stacklevel=1, source=None): if not (isinstance(category, type) and issubclass(category, Warning)): raise TypeError("category must be a Warning subclass, " "not '{:s}'".format(type(category).__name__)) + if not isinstance(skip_file_prefixes, tuple): + # The C version demands a tuple for implementation performance. + raise TypeError('skip_file_prefixes must be a tuple of strs.') + if skip_file_prefixes: + stacklevel = max(2, stacklevel) # Get context information try: if stacklevel <= 1 or _is_internal_frame(sys._getframe(1)): @@ -305,13 +327,13 @@ def warn(message, category=None, stacklevel=1, source=None): frame = sys._getframe(1) # Look for one frame less since the above line starts us off. for x in range(stacklevel-1): - frame = _next_external_frame(frame) + frame = _next_external_frame(frame, skip_file_prefixes) if frame is None: raise ValueError except ValueError: globals = sys.__dict__ - filename = "sys" - lineno = 1 + filename = "" + lineno = 0 else: globals = frame.f_globals filename = frame.f_code.co_filename @@ -391,7 +413,7 @@ def warn_explicit(message, category, filename, lineno, "Unrecognized action (%r) in warnings.filters:\n %s" % (action, item)) # Print message and context - msg = WarningMessage(message, category, filename, lineno, source) + msg = WarningMessage(message, category, filename, lineno, source=source) _showwarnmsg(msg) @@ -493,6 +515,139 @@ def __exit__(self, *exc_info): self._module._showwarnmsg_impl = self._showwarnmsg_impl +class deprecated: + """Indicate that a class, function or overload is deprecated. + + When this decorator is applied to an object, the type checker + will generate a diagnostic on usage of the deprecated object. + + Usage: + + @deprecated("Use B instead") + class A: + pass + + @deprecated("Use g instead") + def f(): + pass + + @overload + @deprecated("int support is deprecated") + def g(x: int) -> int: ... + @overload + def g(x: str) -> int: ... + + The warning specified by *category* will be emitted at runtime + on use of deprecated objects. For functions, that happens on calls; + for classes, on instantiation and on creation of subclasses. + If the *category* is ``None``, no warning is emitted at runtime. + The *stacklevel* determines where the + warning is emitted. If it is ``1`` (the default), the warning + is emitted at the direct caller of the deprecated object; if it + is higher, it is emitted further up the stack. + Static type checker behavior is not affected by the *category* + and *stacklevel* arguments. + + The deprecation message passed to the decorator is saved in the + ``__deprecated__`` attribute on the decorated object. + If applied to an overload, the decorator + must be after the ``@overload`` decorator for the attribute to + exist on the overload as returned by ``get_overloads()``. + + See PEP 702 for details. + + """ + def __init__( + self, + message: str, + /, + *, + category: type[Warning] | None = DeprecationWarning, + stacklevel: int = 1, + ) -> None: + if not isinstance(message, str): + raise TypeError( + f"Expected an object of type str for 'message', not {type(message).__name__!r}" + ) + self.message = message + self.category = category + self.stacklevel = stacklevel + + def __call__(self, arg, /): + # Make sure the inner functions created below don't + # retain a reference to self. + msg = self.message + category = self.category + stacklevel = self.stacklevel + if category is None: + arg.__deprecated__ = msg + return arg + elif isinstance(arg, type): + import functools + from types import MethodType + + original_new = arg.__new__ + + @functools.wraps(original_new) + def __new__(cls, /, *args, **kwargs): + if cls is arg: + warn(msg, category=category, stacklevel=stacklevel + 1) + if original_new is not object.__new__: + return original_new(cls, *args, **kwargs) + # Mirrors a similar check in object.__new__. + elif cls.__init__ is object.__init__ and (args or kwargs): + raise TypeError(f"{cls.__name__}() takes no arguments") + else: + return original_new(cls) + + arg.__new__ = staticmethod(__new__) + + original_init_subclass = arg.__init_subclass__ + # We need slightly different behavior if __init_subclass__ + # is a bound method (likely if it was implemented in Python) + if isinstance(original_init_subclass, MethodType): + original_init_subclass = original_init_subclass.__func__ + + @functools.wraps(original_init_subclass) + def __init_subclass__(*args, **kwargs): + warn(msg, category=category, stacklevel=stacklevel + 1) + return original_init_subclass(*args, **kwargs) + + arg.__init_subclass__ = classmethod(__init_subclass__) + # Or otherwise, which likely means it's a builtin such as + # object's implementation of __init_subclass__. + else: + @functools.wraps(original_init_subclass) + def __init_subclass__(*args, **kwargs): + warn(msg, category=category, stacklevel=stacklevel + 1) + return original_init_subclass(*args, **kwargs) + + arg.__init_subclass__ = __init_subclass__ + + arg.__deprecated__ = __new__.__deprecated__ = msg + __init_subclass__.__deprecated__ = msg + return arg + elif callable(arg): + import functools + import inspect + + @functools.wraps(arg) + def wrapper(*args, **kwargs): + warn(msg, category=category, stacklevel=stacklevel + 1) + return arg(*args, **kwargs) + + if inspect.iscoroutinefunction(arg): + wrapper = inspect.markcoroutinefunction(wrapper) + + arg.__deprecated__ = wrapper.__deprecated__ = msg + return wrapper + else: + raise TypeError( + "@deprecated decorator with non-None category must be applied to " + f"a class or callable, not {arg!r}" + ) + + _DEPRECATED_MSG = "{name!r} is deprecated and slated for removal in Python {remove}" def _deprecated(name, message=_DEPRECATED_MSG, *, remove, _version=sys.version_info): @@ -502,7 +657,7 @@ def _deprecated(name, message=_DEPRECATED_MSG, *, remove, _version=sys.version_i the current Python version or the same version but past the alpha. The *message* argument is formatted with *name* and *remove* as a Python - version (e.g. "3.11"). + version tuple (e.g. (3, 11)). """ remove_formatted = f"{remove[0]}.{remove[1]}" diff --git a/Lib/xml/dom/expatbuilder.py b/Lib/xml/dom/expatbuilder.py index 199c22d0af..7dd667bf3f 100644 --- a/Lib/xml/dom/expatbuilder.py +++ b/Lib/xml/dom/expatbuilder.py @@ -200,10 +200,7 @@ def parseFile(self, file): parser = self.getParser() first_buffer = True try: - while 1: - buffer = file.read(16*1024) - if not buffer: - break + while buffer := file.read(16*1024): parser.Parse(buffer, False) if first_buffer and self.document.documentElement: self._setup_subset(buffer) diff --git a/Lib/xml/dom/minidom.py b/Lib/xml/dom/minidom.py index d09ef5e7d0..db51f350ea 100644 --- a/Lib/xml/dom/minidom.py +++ b/Lib/xml/dom/minidom.py @@ -300,12 +300,28 @@ def _in_document(node): node = node.parentNode return False -def _write_data(writer, data): +def _write_data(writer, text, attr): "Writes datachars to writer." - if data: - data = data.replace("&", "&").replace("<", "<"). \ - replace("\"", """).replace(">", ">") - writer.write(data) + if not text: + return + # See the comments in ElementTree.py for behavior and + # implementation details. + if "&" in text: + text = text.replace("&", "&") + if "<" in text: + text = text.replace("<", "<") + if ">" in text: + text = text.replace(">", ">") + if attr: + if '"' in text: + text = text.replace('"', """) + if "\r" in text: + text = text.replace("\r", " ") + if "\n" in text: + text = text.replace("\n", " ") + if "\t" in text: + text = text.replace("\t", " ") + writer.write(text) def _get_elements_by_tagName_helper(parent, name, rc): for node in parent.childNodes: @@ -358,6 +374,8 @@ def __init__(self, qName, namespaceURI=EMPTY_NAMESPACE, localName=None, self._name = qName self.namespaceURI = namespaceURI self._prefix = prefix + if localName is not None: + self._localName = localName self.childNodes = NodeList() # Add the single child node that represents the value of the attr @@ -881,7 +899,7 @@ def writexml(self, writer, indent="", addindent="", newl=""): for a_name in attrs.keys(): writer.write(" %s=\"" % a_name) - _write_data(writer, attrs[a_name].value) + _write_data(writer, attrs[a_name].value, True) writer.write("\"") if self.childNodes: writer.write(">") @@ -1110,7 +1128,7 @@ def splitText(self, offset): return newText def writexml(self, writer, indent="", addindent="", newl=""): - _write_data(writer, "%s%s%s" % (indent, self.data, newl)) + _write_data(writer, "%s%s%s" % (indent, self.data, newl), False) # DOM Level 3 (WD 9 April 2002) diff --git a/Lib/xml/dom/xmlbuilder.py b/Lib/xml/dom/xmlbuilder.py index 8a20026349..a8852625a2 100644 --- a/Lib/xml/dom/xmlbuilder.py +++ b/Lib/xml/dom/xmlbuilder.py @@ -189,7 +189,7 @@ def parse(self, input): options.filter = self.filter options.errorHandler = self.errorHandler fp = input.byteStream - if fp is None and options.systemId: + if fp is None and input.systemId: import urllib.request fp = urllib.request.urlopen(input.systemId) return self._parse_bytestream(fp, options) @@ -247,10 +247,12 @@ def _create_opener(self): def _guess_media_encoding(self, source): info = source.byteStream.info() - if "Content-Type" in info: - for param in info.getplist(): - if param.startswith("charset="): - return param.split("=", 1)[1].lower() + # import email.message + # assert isinstance(info, email.message.Message) + charset = info.get_param('charset') + if charset is not None: + return charset.lower() + return None class DOMInputSource(object): diff --git a/Lib/xml/etree/ElementInclude.py b/Lib/xml/etree/ElementInclude.py index 40a9b22292..986e6c3bbe 100644 --- a/Lib/xml/etree/ElementInclude.py +++ b/Lib/xml/etree/ElementInclude.py @@ -79,8 +79,8 @@ class LimitedRecursiveIncludeError(FatalIncludeError): # @param parse Parse mode. Either "xml" or "text". # @param encoding Optional text encoding (UTF-8 by default for "text"). # @return The expanded resource. If the parse mode is "xml", this -# is an ElementTree instance. If the parse mode is "text", this -# is a Unicode string. If the loader fails, it can return None +# is an Element instance. If the parse mode is "text", this +# is a string. If the loader fails, it can return None # or raise an OSError exception. # @throws OSError If the loader fails to load the resource. @@ -98,7 +98,7 @@ def default_loader(href, parse, encoding=None): ## # Expand XInclude directives. # -# @param elem Root element. +# @param elem Root Element or any ElementTree of a tree to be expanded # @param loader Optional resource loader. If omitted, it defaults # to {@link default_loader}. If given, it should be a callable # that implements the same interface as default_loader. @@ -106,12 +106,13 @@ def default_loader(href, parse, encoding=None): # relative include file references. # @param max_depth The maximum number of recursive inclusions. # Limited to reduce the risk of malicious content explosion. -# Pass a negative value to disable the limitation. +# Pass None to disable the limitation. # @throws LimitedRecursiveIncludeError If the {@link max_depth} was exceeded. # @throws FatalIncludeError If the function fails to include a given # resource, or if the tree contains malformed XInclude elements. -# @throws IOError If the function fails to load a given resource. -# @returns the node or its replacement if it was an XInclude node +# @throws OSError If the function fails to load a given resource. +# @throws ValueError If negative {@link max_depth} is passed. +# @returns None. Modifies tree pointed by {@link elem} def include(elem, loader=None, base_url=None, max_depth=DEFAULT_MAX_INCLUSION_DEPTH): diff --git a/Lib/xml/etree/ElementPath.py b/Lib/xml/etree/ElementPath.py index cd3c354d08..dc6bd28c03 100644 --- a/Lib/xml/etree/ElementPath.py +++ b/Lib/xml/etree/ElementPath.py @@ -416,6 +416,8 @@ def findall(elem, path, namespaces=None): def findtext(elem, path, default=None, namespaces=None): try: elem = next(iterfind(elem, path, namespaces)) - return elem.text or "" + if elem.text is None: + return "" + return elem.text except StopIteration: return default diff --git a/Lib/xml/etree/ElementTree.py b/Lib/xml/etree/ElementTree.py index 2503d9ee76..9bb09ab540 100644 --- a/Lib/xml/etree/ElementTree.py +++ b/Lib/xml/etree/ElementTree.py @@ -99,6 +99,7 @@ import collections import collections.abc import contextlib +import weakref from . import ElementPath @@ -188,19 +189,6 @@ def makeelement(self, tag, attrib): """ return self.__class__(tag, attrib) - def copy(self): - """Return copy of current element. - - This creates a shallow copy. Subelements will be shared with the - original tree. - - """ - warnings.warn( - "elem.copy() is deprecated. Use copy.copy(elem) instead.", - DeprecationWarning - ) - return self.__copy__() - def __copy__(self): elem = self.makeelement(self.tag, self.attrib) elem.text = self.text @@ -213,9 +201,10 @@ def __len__(self): def __bool__(self): warnings.warn( - "The behavior of this method will change in future versions. " + "Testing an element's truth value will always return True in " + "future versions. " "Use specific 'len(elem)' or 'elem is not None' test instead.", - FutureWarning, stacklevel=2 + DeprecationWarning, stacklevel=2 ) return len(self._children) != 0 # emulate old behaviour, for now @@ -534,7 +523,9 @@ class ElementTree: """ def __init__(self, element=None, file=None): - # assert element is None or iselement(element) + if element is not None and not iselement(element): + raise TypeError('expected an Element, not %s' % + type(element).__name__) self._root = element # first node if file: self.parse(file) @@ -550,7 +541,9 @@ def _setroot(self, element): with the given element. Use with care! """ - # assert iselement(element) + if not iselement(element): + raise TypeError('expected an Element, not %s' + % type(element).__name__) self._root = element def parse(self, source, parser=None): @@ -579,10 +572,7 @@ def parse(self, source, parser=None): # it with chunks. self._root = parser._parse_whole(source) return self._root - while True: - data = source.read(65536) - if not data: - break + while data := source.read(65536): parser.feed(data) self._root = parser.close() return self._root @@ -719,6 +709,8 @@ def write(self, file_or_filename, of start/end tags """ + if self._root is None: + raise TypeError('ElementTree not initialized') if not method: method = "xml" elif method not in _serialize: @@ -911,13 +903,9 @@ def _serialize_xml(write, elem, qnames, namespaces, if elem.tail: write(_escape_cdata(elem.tail)) -HTML_EMPTY = ("area", "base", "basefont", "br", "col", "frame", "hr", - "img", "input", "isindex", "link", "meta", "param") - -try: - HTML_EMPTY = set(HTML_EMPTY) -except NameError: - pass +HTML_EMPTY = {"area", "base", "basefont", "br", "col", "embed", "frame", "hr", + "img", "input", "isindex", "link", "meta", "param", "source", + "track", "wbr"} def _serialize_html(write, elem, qnames, namespaces, **kwargs): tag = elem.tag @@ -1242,13 +1230,14 @@ def iterparse(source, events=None, parser=None): # parser argument of iterparse is removed, this can be killed. pullparser = XMLPullParser(events=events, _parser=parser) - def iterator(source): + if not hasattr(source, "read"): + source = open(source, "rb") + close_source = True + else: close_source = False + + def iterator(source): try: - if not hasattr(source, "read"): - source = open(source, "rb") - close_source = True - yield None while True: yield from pullparser.read_events() # load event buffer @@ -1258,18 +1247,30 @@ def iterator(source): pullparser.feed(data) root = pullparser._close_and_return_root() yield from pullparser.read_events() - it.root = root + it = wr() + if it is not None: + it.root = root finally: if close_source: source.close() + gen = iterator(source) class IterParseIterator(collections.abc.Iterator): - __next__ = iterator(source).__next__ + __next__ = gen.__next__ + def close(self): + if close_source: + source.close() + gen.close() + + def __del__(self): + # TODO: Emit a ResourceWarning if it was not explicitly closed. + # (When the close() method will be supported in all maintained Python versions.) + if close_source: + source.close() + it = IterParseIterator() it.root = None - del iterator, IterParseIterator - - next(it) + wr = weakref.ref(it) return it @@ -1325,6 +1326,11 @@ def read_events(self): else: yield event + def flush(self): + if self._parser is None: + raise ValueError("flush() called after end of stream") + self._parser.flush() + def XML(text, parser=None): """Parse XML document from string constant. @@ -1731,6 +1737,15 @@ def close(self): del self.parser, self._parser del self.target, self._target + def flush(self): + was_enabled = self.parser.GetReparseDeferralEnabled() + try: + self.parser.SetReparseDeferralEnabled(False) + self.parser.Parse(b"", False) + except self._error as v: + self._raiseerror(v) + finally: + self.parser.SetReparseDeferralEnabled(was_enabled) # -------------------------------------------------------------------- # C14N 2.0 diff --git a/Lib/xml/sax/__init__.py b/Lib/xml/sax/__init__.py index 17b75879eb..b657310207 100644 --- a/Lib/xml/sax/__init__.py +++ b/Lib/xml/sax/__init__.py @@ -60,11 +60,7 @@ def parseString(string, handler, errorHandler=ErrorHandler()): import os, sys if not sys.flags.ignore_environment and "PY_SAX_PARSER" in os.environ: default_parser_list = os.environ["PY_SAX_PARSER"].split(",") -del os - -_key = "python.xml.sax.parser" -if sys.platform[:4] == "java" and sys.registry.containsKey(_key): - default_parser_list = sys.registry.getProperty(_key).split(",") +del os, sys def make_parser(parser_list=()): @@ -93,15 +89,6 @@ def make_parser(parser_list=()): # --- Internal utility methods used by make_parser -if sys.platform[ : 4] == "java": - def _create_parser(parser_name): - from org.python.core import imp - drv_module = imp.importName(parser_name, 0, globals()) - return drv_module.create_parser() - -else: - def _create_parser(parser_name): - drv_module = __import__(parser_name,{},{},['create_parser']) - return drv_module.create_parser() - -del sys +def _create_parser(parser_name): + drv_module = __import__(parser_name,{},{},['create_parser']) + return drv_module.create_parser() diff --git a/Lib/xml/sax/_exceptions.py b/Lib/xml/sax/_exceptions.py index a9b2ba35c6..f292dc3a8e 100644 --- a/Lib/xml/sax/_exceptions.py +++ b/Lib/xml/sax/_exceptions.py @@ -1,8 +1,4 @@ """Different kinds of SAX Exceptions""" -import sys -if sys.platform[:4] == "java": - from java.lang import Exception -del sys # ===== SAXEXCEPTION ===== diff --git a/Lib/xml/sax/expatreader.py b/Lib/xml/sax/expatreader.py index e334ac9fea..ba3c1e9851 100644 --- a/Lib/xml/sax/expatreader.py +++ b/Lib/xml/sax/expatreader.py @@ -12,12 +12,6 @@ from xml.sax.handler import feature_string_interning from xml.sax.handler import property_xml_string, property_interning_dict -# xml.parsers.expat does not raise ImportError in Jython -import sys -if sys.platform[:4] == "java": - raise SAXReaderNotAvailable("expat not available in Java", None) -del sys - try: from xml.parsers import expat except ImportError: @@ -220,6 +214,20 @@ def feed(self, data, isFinal=False): # FIXME: when to invoke error()? self._err_handler.fatalError(exc) + def flush(self): + if self._parser is None: + return + + was_enabled = self._parser.GetReparseDeferralEnabled() + try: + self._parser.SetReparseDeferralEnabled(False) + self._parser.Parse(b"", False) + except expat.error as e: + exc = SAXParseException(expat.ErrorString(e.code), e, self) + self._err_handler.fatalError(exc) + finally: + self._parser.SetReparseDeferralEnabled(was_enabled) + def _close_source(self): source = self._source try: diff --git a/Lib/xml/sax/xmlreader.py b/Lib/xml/sax/xmlreader.py index 716f228404..e906121d23 100644 --- a/Lib/xml/sax/xmlreader.py +++ b/Lib/xml/sax/xmlreader.py @@ -120,10 +120,8 @@ def parse(self, source): file = source.getCharacterStream() if file is None: file = source.getByteStream() - buffer = file.read(self._bufsize) - while buffer: + while buffer := file.read(self._bufsize): self.feed(buffer) - buffer = file.read(self._bufsize) self.close() def feed(self, data): diff --git a/Lib/zipapp.py b/Lib/zipapp.py index ce77632516..4ffacc49fa 100644 --- a/Lib/zipapp.py +++ b/Lib/zipapp.py @@ -131,12 +131,35 @@ def create_archive(source, target=None, interpreter=None, main=None, elif not hasattr(target, 'write'): target = pathlib.Path(target) + # Create the list of files to add to the archive now, in case + # the target is being created in the source directory - we + # don't want the target being added to itself + files_to_add = sorted(source.rglob('*')) + + # The target cannot be in the list of files to add. If it were, we'd + # end up overwriting the source file and writing the archive into + # itself, which is an error. We therefore check for that case and + # provide a helpful message for the user. + + # Note that we only do a simple path equality check. This won't + # catch every case, but it will catch the common case where the + # source is the CWD and the target is a file in the CWD. More + # thorough checks don't provide enough value to justify the extra + # cost. + + # If target is a file-like object, it will simply fail to compare + # equal to any of the entries in files_to_add, so there's no need + # to add a special check for that. + if target in files_to_add: + raise ZipAppError( + f"The target archive {target} overwrites one of the source files.") + with _maybe_open(target, 'wb') as fd: _write_file_prefix(fd, interpreter) compression = (zipfile.ZIP_DEFLATED if compressed else zipfile.ZIP_STORED) with zipfile.ZipFile(fd, 'w', compression=compression) as z: - for child in source.rglob('*'): + for child in files_to_add: arcname = child.relative_to(source) if filter is None or filter(arcname): z.write(child, arcname.as_posix()) diff --git a/Lib/zoneinfo/_common.py b/Lib/zoneinfo/_common.py index 98cdfe37ca..03cc42149f 100644 --- a/Lib/zoneinfo/_common.py +++ b/Lib/zoneinfo/_common.py @@ -9,9 +9,13 @@ def load_tzdata(key): resource_name = components[-1] try: - return resources.files(package_name).joinpath(resource_name).open("rb") - except (ImportError, FileNotFoundError, UnicodeEncodeError): - # There are three types of exception that can be raised that all amount + path = resources.files(package_name).joinpath(resource_name) + # gh-85702: Prevent PermissionError on Windows + if path.is_dir(): + raise IsADirectoryError + return path.open("rb") + except (ImportError, FileNotFoundError, UnicodeEncodeError, IsADirectoryError): + # There are four types of exception that can be raised that all amount # to "we cannot find this key": # # ImportError: If package_name doesn't exist (e.g. if tzdata is not @@ -21,6 +25,7 @@ def load_tzdata(key): # (e.g. Europe/Krasnoy) # UnicodeEncodeError: If package_name or resource_name are not UTF-8, # such as keys containing a surrogate character. + # IsADirectoryError: If package_name without a resource_name specified. raise ZoneInfoNotFoundError(f"No time zone found with key {key}") diff --git a/Lib/zoneinfo/_zoneinfo.py b/Lib/zoneinfo/_zoneinfo.py index b77dc0ed39..3ffdb4c837 100644 --- a/Lib/zoneinfo/_zoneinfo.py +++ b/Lib/zoneinfo/_zoneinfo.py @@ -75,12 +75,12 @@ def _new_instance(cls, key): return obj @classmethod - def from_file(cls, fobj, /, key=None): + def from_file(cls, file_obj, /, key=None): obj = super().__new__(cls) obj._key = key obj._file_path = None - obj._load_file(fobj) - obj._file_repr = repr(fobj) + obj._load_file(file_obj) + obj._file_repr = repr(file_obj) # Disable pickling for objects created from files obj.__reduce__ = obj._file_reduce diff --git a/README.md b/README.md index ce5f02bee2..86d0738ec8 100644 --- a/README.md +++ b/README.md @@ -66,17 +66,11 @@ Welcome to the magnificent Rust Python interpreter >>>>> ``` -If you'd like to make https requests, you can enable the `ssl` feature, which -also lets you install the `pip` package manager. Note that on Windows, you may -need to install OpenSSL, or you can enable the `ssl-vendor` feature instead, -which compiles OpenSSL for you but requires a C compiler, perl, and `make`. -OpenSSL version 3 is expected and tested in CI. Older versions may not work. - -Once you've installed rustpython with SSL support, you can install pip by +You can install pip by running: ```bash -cargo install --git https://github.com/RustPython/RustPython --features ssl +cargo install --git https://github.com/RustPython/RustPython rustpython --install-pip ``` @@ -88,6 +82,13 @@ conda install rustpython -c conda-forge rustpython ``` +### SSL provider + +For HTTPS requests, `ssl-rustls` feature is enabled by default. You can replace it with `ssl-openssl` feature if your environment requires OpenSSL. +Note that to use OpenSSL on Windows, you may need to install OpenSSL, or you can enable the `ssl-vendor` feature instead, +which compiles OpenSSL for you but requires a C compiler, perl, and `make`. +OpenSSL version 3 is expected and tested in CI. Older versions may not work. + ### WASI You can compile RustPython to a standalone WebAssembly WASI module so it can run anywhere. diff --git a/architecture/architecture.md b/architecture/architecture.md index a59b6498bf..4b4110ee89 100644 --- a/architecture/architecture.md +++ b/architecture/architecture.md @@ -20,7 +20,43 @@ If, after reading this, you want to contribute to RustPython, take a look at the A high-level overview of the workings of RustPython is visible in the figure below, showing how Python source files are interpreted. -![overview.png](overview.png) +```mermaid +flowchart TB + SourceCode["🐍 Source code"] + + subgraph Interpreter["RustPython Interpreter"] + direction TB + + subgraph Parser["Parser"] + ParserBox["• Tokenize source code
• Validate tokens
• Create AST"] + end + + AST[["AST"]] + + subgraph Compiler["Compiler"] + CompilerBox["• Converts AST to bytecode"] + end + + Bytecode[["Bytecode"]] + + subgraph VM["VM"] + VMBox["• Executes bytecode given input"] + end + + Parser --> AST + AST --> Compiler + Compiler --> Bytecode + Bytecode --> VM + end + + SourceCode -------> Interpreter + + Input[["Input"]] + Output["Code gets executed"] + + Input --> VM + VM --> Output +``` Main architecture of RustPython. @@ -36,9 +72,9 @@ The main entry point of RustPython is located in `src/main.rs` and simply forwar For each of the three components, the entry point is as follows: -- Parser: The Parser is located in a separate project, [RustPython/Parser][10]. See the documentation there for more information. -- Compiler: `compile`, located in [`vm/src/vm/compile.rs`][11], this eventually forwards a call to [`compiler::compile`][12]. -- VM: `run_code_obj`, located in [`vm/src/vm/mod.rs`][13]. This creates a new frame in which the bytecode is executed. +- Parser: The Parser is located in a separate project, [ruff_python_parser][10]. See the documentation there for more information. +- Compiler: `compile`, located in [`crates/vm/src/vm/compile.rs`][11], this eventually forwards a call to [`compiler::compile`][12]. +- VM: `run_code_obj`, located in [`crates/vm/src/vm/mod.rs`][13]. This creates a new frame in which the bytecode is executed. ## Components @@ -46,7 +82,7 @@ Here we give a brief overview of each component and its function. For more detai ### Compiler -This component, implemented as the `rustpython-compiler/` package, is responsible for translating a Python source file into its equivalent bytecode representation. As an example, the following Python file: +This component, implemented as the `rustpython-compiler` package, is responsible for translating a Python source file into its equivalent bytecode representation. As an example, the following Python file: ```python def f(x): @@ -71,11 +107,11 @@ The Parser is the main sub-component of the compiler. All the functionality requ 1. Lexical Analysis 2. Parsing -The functionality for parsing resides in the RustPython/Parser project. See the documentation there for more information. +The functionality for parsing resides in the ruff_python_parser project in the astral-sh/ruff repository. See the documentation there for more information. ### VM -The Virtual Machine (VM) is responsible for executing the bytecode generated by the compiler. It is implemented in the `rustpython-vm/` package. The VM is currently implemented as a stack machine, meaning that it uses a stack to store intermediate results. In the `rustpython-vm/` package, additional sub-components are present, for example: +The Virtual Machine (VM) is responsible for executing the bytecode generated by the compiler. It is implemented in the `rustpython-vm` package. The VM is currently implemented as a stack machine, meaning that it uses a stack to store intermediate results. In the `rustpython-vm` package, additional sub-components are present, for example: - builtins: the built in objects of Python, such as `int` and `str`. - stdlib: parts of the standard library that contains built-in modules needed for the VM to function, such as `sys`. @@ -122,7 +158,7 @@ The RustPython executable/REPL (Read-Eval-Print-Loop) is implemented here, which Some things to note: - The CLI is defined in the [`run` function of `src/lib.rs`][16]. -- The interface and helper for the REPL are defined in this package, but the actual REPL can be found in `vm/src/readline.rs` +- The interface and helper for the REPL are defined in this package, but the actual REPL can be found in `crates/vm/src/readline.rs` ### WASM @@ -133,18 +169,18 @@ Crate for WebAssembly build, which compiles the RustPython package to a format t Integration and snippets that test for additional edge-cases, implementation specific functionality and bug report snippets. [1]: https://github.com/RustPython/RustPython -[2]: https://2021.desosa.nl/projects/rustpython/posts/vision/ +[2]: https://web.archive.org/web/20240723122357/https://2021.desosa.nl/projects/rustpython/posts/vision/ [3]: https://www.youtube.com/watch?v=nJDY9ASuiLc&t=213s [4]: https://rustpython.github.io/2020/04/02/thing-explainer-parser.html [5]: https://rustpython.github.io/demo/ [6]: https://rustpython.github.io/demo/notebook/ -[7]: https://github.com/RustPython/RustPython/blob/master/DEVELOPMENT.md +[7]: https://github.com/RustPython/RustPython/blob/main/DEVELOPMENT.md [8]: https://rustpython.github.io/guideline/2020/04/04/how-to-contribute-by-cpython-unittest.html -[9]: https://github.com/RustPython/RustPython/blob/0e24cf48c63ae4ca9f829e88142a987cab3a9966/src/lib.rs#L63 -[10]: https://github.com/RustPython/Parser -[11]: https://github.com/RustPython/RustPython/blob/0e24cf48c63ae4ca9f829e88142a987cab3a9966/vm/src/vm/compile.rs#LL10C17-L10C17 -[12]: https://github.com/RustPython/RustPython/blob/0e24cf48c63ae4ca9f829e88142a987cab3a9966/vm/src/vm/compile.rs#L26 -[13]: https://github.com/RustPython/RustPython/blob/0e24cf48c63ae4ca9f829e88142a987cab3a9966/vm/src/vm/mod.rs#L356 -[14]: https://github.com/RustPython/RustPython/blob/0e24cf48c63ae4ca9f829e88142a987cab3a9966/common/src/float_ops.rs +[9]: https://github.com/RustPython/RustPython/blob/d36a2cffdef2218f3264cef9145a1f781d474ea3/src/lib.rs#L72 +[10]: https://github.com/astral-sh/ruff/tree/2bffef59665ce7d2630dfd72ee99846663660db8/crates/ruff_python_parser +[11]: https://github.com/RustPython/RustPython/blob/d36a2cffdef2218f3264cef9145a1f781d474ea3/crates/vm/src/vm/compile.rs#L10 +[12]: https://github.com/RustPython/RustPython/blob/d36a2cffdef2218f3264cef9145a1f781d474ea3/crates/vm/src/vm/compile.rs#L26 +[13]: https://github.com/RustPython/RustPython/blob/d36a2cffdef2218f3264cef9145a1f781d474ea3/crates/vm/src/vm/mod.rs#L433 +[14]: https://github.com/RustPython/RustPython/blob/d36a2cffdef2218f3264cef9145a1f781d474ea3/crates/common/src/float_ops.rs [15]: https://rustpython.github.io/guideline/2020/04/04/how-to-contribute-by-cpython-unittest.html -[16]: https://github.com/RustPython/RustPython/blob/0e24cf48c63ae4ca9f829e88142a987cab3a9966/src/lib.rs#L63 +[16]: https://github.com/RustPython/RustPython/blob/d36a2cffdef2218f3264cef9145a1f781d474ea3/src/lib.rs#L72 diff --git a/architecture/overview.png b/architecture/overview.png deleted file mode 100644 index ce66706f3c..0000000000 Binary files a/architecture/overview.png and /dev/null differ diff --git a/benches/README.md b/benches/README.md index f6e5129617..4823751fd1 100644 --- a/benches/README.md +++ b/benches/README.md @@ -4,19 +4,26 @@ These are some files to determine performance of rustpython. ## Usage -Running `cargo bench` from the root of the repository will start the benchmarks. Once done there will be a graphical +Running `cargo bench` from the root of the repository will start the benchmarks. Once done there will be a graphical report under `target/criterion/report/index.html` that you can use use to view the results. -To view Python tracebacks during benchmarks, run `RUST_BACKTRACE=1 cargo bench`. You can also bench against a +`cargo bench` supports name matching to run a subset of the benchmarks. To +run only the sorted microbenchmark, you can run: + +```shell +cargo bench sorted +``` + +To view Python tracebacks during benchmarks, run `RUST_BACKTRACE=1 cargo bench`. You can also bench against a specific installed Python version by running: ```shell -$ PYTHON_SYS_EXECUTABLE=python3.7 cargo bench +PYTHON_SYS_EXECUTABLE=python3.13 cargo bench ``` ### Adding a benchmark -Simply adding a file to the `benchmarks/` directory will add it to the set of files benchmarked. Each file is tested +Simply adding a file to the `benchmarks/` directory will add it to the set of files benchmarked. Each file is tested in two ways: 1. The time to parse the file to AST @@ -24,8 +31,9 @@ in two ways: ### Adding a micro benchmark -Micro benchmarks are small snippets of code added under the `microbenchmarks/` directory. A microbenchmark file has +Micro benchmarks are small snippets of code added under the `microbenchmarks/` directory. A microbenchmark file has two sections: + 1. Optional setup code 2. The code to be benchmarked @@ -39,8 +47,8 @@ a_list = [1,2,3] len(a_list) ``` -Only `len(a_list)` will be timed. Setup or benchmarked code can optionally reference a variable called `ITERATIONS`. If -present then the benchmark code will be invoked 5 times with `ITERATIONS` set to a value between 100 and 1,000. For +Only `len(a_list)` will be timed. Setup or benchmarked code can optionally reference a variable called `ITERATIONS`. If +present then the benchmark code will be invoked 5 times with `ITERATIONS` set to a value between 100 and 1,000. For example: ```python @@ -49,7 +57,7 @@ obj = [i for i in range(ITERATIONS)] `ITERATIONS` can appear in both the setup code and the benchmark code. -## MacOS setup +## MacOS setup On MacOS you will need to add the following to a `.cargo/config` file: @@ -63,4 +71,4 @@ rustflags = [ ## Benchmark source -- https://benchmarksgame-team.pages.debian.net/benchmarksgame/program/nbody-python3-2.html +- diff --git a/benches/execution.rs b/benches/execution.rs index 7a7ba247e5..c2239b59d1 100644 --- a/benches/execution.rs +++ b/benches/execution.rs @@ -1,17 +1,15 @@ -use criterion::measurement::WallTime; use criterion::{ - Bencher, BenchmarkGroup, BenchmarkId, Criterion, Throughput, black_box, criterion_group, - criterion_main, + Bencher, BenchmarkGroup, BenchmarkId, Criterion, Throughput, criterion_group, criterion_main, + measurement::WallTime, }; use rustpython_compiler::Mode; use rustpython_vm::{Interpreter, PyResult, Settings}; -use std::collections::HashMap; -use std::path::Path; +use std::{collections::HashMap, hint::black_box, path::Path}; fn bench_cpython_code(b: &mut Bencher, source: &str) { let c_str_source_head = std::ffi::CString::new(source).unwrap(); let c_str_source = c_str_source_head.as_c_str(); - pyo3::Python::with_gil(|py| { + pyo3::Python::attach(|py| { b.iter(|| { let module = pyo3::types::PyModule::from_code(py, c_str_source, c"", c"") .expect("Error running source"); @@ -54,7 +52,7 @@ pub fn benchmark_file_parsing(group: &mut BenchmarkGroup, name: &str, }); group.bench_function(BenchmarkId::new("cpython", name), |b| { use pyo3::types::PyAnyMethods; - pyo3::Python::with_gil(|py| { + pyo3::Python::attach(|py| { let builtins = pyo3::types::PyModule::import(py, "builtins").expect("Failed to import builtins"); let compile = builtins.getattr("compile").expect("no compile in builtins"); diff --git a/benches/microbenchmarks.rs b/benches/microbenchmarks.rs index 5f04f4bbf8..98993b4154 100644 --- a/benches/microbenchmarks.rs +++ b/benches/microbenchmarks.rs @@ -37,7 +37,7 @@ pub struct MicroBenchmark { } fn bench_cpython_code(group: &mut BenchmarkGroup, bench: &MicroBenchmark) { - pyo3::Python::with_gil(|py| { + pyo3::Python::attach(|py| { let setup_name = format!("{}_setup", bench.name); let setup_code = cpy_compile_code(py, &bench.setup, &setup_name).unwrap(); @@ -101,7 +101,10 @@ fn cpy_compile_code<'a>( let builtins = pyo3::types::PyModule::import(py, "builtins").expect("Failed to import builtins"); let compile = builtins.getattr("compile").expect("no compile in builtins"); - compile.call1((code, name, "exec"))?.extract() + Ok(compile + .call1((code, name, "exec"))? + .cast_into() + .expect("compile() should return a code object")) } fn bench_rustpython_code(group: &mut BenchmarkGroup, bench: &MicroBenchmark) { diff --git a/benches/microbenchmarks/sorted.py b/benches/microbenchmarks/sorted.py new file mode 100644 index 0000000000..8e5e0beeb1 --- /dev/null +++ b/benches/microbenchmarks/sorted.py @@ -0,0 +1,9 @@ +from random import random, seed +seed(0) + +unsorted_list = [random() for _ in range(5 * ITERATIONS)] + +# --- + +# Setup code only runs once so do not modify in-place +sorted(unsorted_list) diff --git a/common/src/crt_fd.rs b/common/src/crt_fd.rs deleted file mode 100644 index 64d4df98a5..0000000000 --- a/common/src/crt_fd.rs +++ /dev/null @@ -1,112 +0,0 @@ -//! A module implementing an io type backed by the C runtime's file descriptors, i.e. what's -//! returned from libc::open, even on windows. - -use std::{cmp, ffi, io}; - -#[cfg(windows)] -use libc::commit as fsync; -#[cfg(windows)] -unsafe extern "C" { - #[link_name = "_chsize_s"] - fn ftruncate(fd: i32, len: i64) -> i32; -} -#[cfg(not(windows))] -use libc::{fsync, ftruncate}; - -// this is basically what CPython has for Py_off_t; windows uses long long -// for offsets, other platforms just use off_t -#[cfg(not(windows))] -pub type Offset = libc::off_t; -#[cfg(windows)] -pub type Offset = libc::c_longlong; - -#[inline] -fn cvt(ret: I, f: impl FnOnce(I) -> T) -> io::Result { - if ret < I::zero() { - Err(crate::os::last_os_error()) - } else { - Ok(f(ret)) - } -} - -const MAX_RW: usize = if cfg!(any(windows, target_vendor = "apple")) { - i32::MAX as usize -} else { - isize::MAX as usize -}; - -#[derive(Copy, Clone, PartialEq, Eq)] -#[repr(transparent)] -pub struct Fd(pub i32); - -impl Fd { - pub fn open(path: &ffi::CStr, flags: i32, mode: i32) -> io::Result { - cvt(unsafe { libc::open(path.as_ptr(), flags, mode) }, Fd) - } - - #[cfg(windows)] - pub fn wopen(path: &widestring::WideCStr, flags: i32, mode: i32) -> io::Result { - cvt( - unsafe { suppress_iph!(libc::wopen(path.as_ptr(), flags, mode)) }, - Fd, - ) - } - - #[cfg(all(any(unix, target_os = "wasi"), not(target_os = "redox")))] - pub fn openat(&self, path: &ffi::CStr, flags: i32, mode: i32) -> io::Result { - cvt( - unsafe { libc::openat(self.0, path.as_ptr(), flags, mode) }, - Fd, - ) - } - - pub fn fsync(&self) -> io::Result<()> { - cvt(unsafe { suppress_iph!(fsync(self.0)) }, drop) - } - - pub fn close(&self) -> io::Result<()> { - cvt(unsafe { suppress_iph!(libc::close(self.0)) }, drop) - } - - pub fn ftruncate(&self, len: Offset) -> io::Result<()> { - cvt(unsafe { suppress_iph!(ftruncate(self.0, len)) }, drop) - } - - #[cfg(windows)] - pub fn to_raw_handle(&self) -> io::Result { - unsafe extern "C" { - fn _get_osfhandle(fd: i32) -> libc::intptr_t; - } - let handle = unsafe { suppress_iph!(_get_osfhandle(self.0)) }; - if handle == -1 { - Err(io::Error::last_os_error()) - } else { - Ok(handle as _) - } - } -} - -impl io::Write for Fd { - fn write(&mut self, buf: &[u8]) -> io::Result { - let count = cmp::min(buf.len(), MAX_RW); - cvt( - unsafe { suppress_iph!(libc::write(self.0, buf.as_ptr() as _, count as _)) }, - |i| i as usize, - ) - } - - #[inline] - fn flush(&mut self) -> io::Result<()> { - Ok(()) - } -} - -impl io::Read for Fd { - fn read(&mut self, buf: &mut [u8]) -> io::Result { - let count = cmp::min(buf.len(), MAX_RW); - cvt( - unsafe { suppress_iph!(libc::read(self.0, buf.as_mut_ptr() as _, count as _)) }, - |i| i as usize, - ) - } -} diff --git a/common/src/int.rs b/common/src/int.rs deleted file mode 100644 index 9ec9e01498..0000000000 --- a/common/src/int.rs +++ /dev/null @@ -1,158 +0,0 @@ -use malachite_base::{num::conversion::traits::RoundingInto, rounding_modes::RoundingMode}; -use malachite_bigint::{BigInt, BigUint, Sign}; -use malachite_q::Rational; -use num_traits::{One, ToPrimitive, Zero}; - -pub fn true_div(numerator: &BigInt, denominator: &BigInt) -> f64 { - let rational = Rational::from_integers_ref(numerator.into(), denominator.into()); - match rational.rounding_into(RoundingMode::Nearest) { - // returned value is $t::MAX but still less than the original - (val, std::cmp::Ordering::Less) if val == f64::MAX => f64::INFINITY, - // returned value is $t::MIN but still greater than the original - (val, std::cmp::Ordering::Greater) if val == f64::MIN => f64::NEG_INFINITY, - (val, _) => val, - } -} - -pub fn float_to_ratio(value: f64) -> Option<(BigInt, BigInt)> { - let sign = match std::cmp::PartialOrd::partial_cmp(&value, &0.0)? { - std::cmp::Ordering::Less => Sign::Minus, - std::cmp::Ordering::Equal => return Some((BigInt::zero(), BigInt::one())), - std::cmp::Ordering::Greater => Sign::Plus, - }; - Rational::try_from(value).ok().map(|x| { - let (numer, denom) = x.into_numerator_and_denominator(); - ( - BigInt::from_biguint(sign, numer.into()), - BigUint::from(denom).into(), - ) - }) -} - -pub fn bytes_to_int(lit: &[u8], mut base: u32) -> Option { - // split sign - let mut lit = lit.trim_ascii(); - let sign = match lit.first()? { - b'+' => Some(Sign::Plus), - b'-' => Some(Sign::Minus), - _ => None, - }; - if sign.is_some() { - lit = &lit[1..]; - } - - // split radix - let first = *lit.first()?; - let has_radix = if first == b'0' { - match base { - 0 => { - if let Some(parsed) = lit.get(1).and_then(detect_base) { - base = parsed; - true - } else { - if let [_first, others @ .., last] = lit { - let is_zero = - others.iter().all(|&c| c == b'0' || c == b'_') && *last == b'0'; - if !is_zero { - return None; - } - } - return Some(BigInt::zero()); - } - } - 16 => lit.get(1).is_some_and(|&b| matches!(b, b'x' | b'X')), - 2 => lit.get(1).is_some_and(|&b| matches!(b, b'b' | b'B')), - 8 => lit.get(1).is_some_and(|&b| matches!(b, b'o' | b'O')), - _ => false, - } - } else { - if base == 0 { - base = 10; - } - false - }; - if has_radix { - lit = &lit[2..]; - if lit.first()? == &b'_' { - lit = &lit[1..]; - } - } - - // remove zeroes - let mut last = *lit.first()?; - if last == b'0' { - let mut count = 0; - for &cur in &lit[1..] { - if cur == b'_' { - if last == b'_' { - return None; - } - } else if cur != b'0' { - break; - }; - count += 1; - last = cur; - } - let prefix_last = lit[count]; - lit = &lit[count + 1..]; - if lit.is_empty() && prefix_last == b'_' { - return None; - } - } - - // validate - for c in lit { - let c = *c; - if !(c.is_ascii_alphanumeric() || c == b'_') { - return None; - } - - if c == b'_' && last == b'_' { - return None; - } - - last = c; - } - if last == b'_' { - return None; - } - - // parse - let number = if lit.is_empty() { - BigInt::zero() - } else { - let uint = BigUint::parse_bytes(lit, base)?; - BigInt::from_biguint(sign.unwrap_or(Sign::Plus), uint) - }; - Some(number) -} - -#[inline] -pub const fn detect_base(c: &u8) -> Option { - let base = match c { - b'x' | b'X' => 16, - b'b' | b'B' => 2, - b'o' | b'O' => 8, - _ => return None, - }; - Some(base) -} - -// num-bigint now returns Some(inf) for to_f64() in some cases, so just keep that the same for now -#[inline(always)] -pub fn bigint_to_finite_float(int: &BigInt) -> Option { - int.to_f64().filter(|f| f.is_finite()) -} - -#[test] -fn test_bytes_to_int() { - assert_eq!(bytes_to_int(&b"0b101"[..], 2).unwrap(), BigInt::from(5)); - assert_eq!(bytes_to_int(&b"0x_10"[..], 16).unwrap(), BigInt::from(16)); - assert_eq!(bytes_to_int(&b"0b"[..], 16).unwrap(), BigInt::from(11)); - assert_eq!(bytes_to_int(&b"+0b101"[..], 2).unwrap(), BigInt::from(5)); - assert_eq!(bytes_to_int(&b"0_0_0"[..], 10).unwrap(), BigInt::from(0)); - assert_eq!(bytes_to_int(&b"09_99"[..], 0), None); - assert_eq!(bytes_to_int(&b"000"[..], 0).unwrap(), BigInt::from(0)); - assert_eq!(bytes_to_int(&b"0_"[..], 0), None); - assert_eq!(bytes_to_int(&b"0_100"[..], 10).unwrap(), BigInt::from(100)); -} diff --git a/compiler/codegen/src/ir.rs b/compiler/codegen/src/ir.rs deleted file mode 100644 index 7acd9d7f6a..0000000000 --- a/compiler/codegen/src/ir.rs +++ /dev/null @@ -1,339 +0,0 @@ -use std::ops; - -use crate::IndexSet; -use crate::error::InternalError; -use ruff_source_file::{OneIndexed, SourceLocation}; -use rustpython_compiler_core::bytecode::{ - CodeFlags, CodeObject, CodeUnit, ConstantData, InstrDisplayContext, Instruction, Label, OpArg, -}; -// use rustpython_parser_core::source_code::{LineNumber, SourceLocation}; - -#[derive(Copy, Clone, PartialEq, Eq, Debug)] -pub struct BlockIdx(pub u32); -impl BlockIdx { - pub const NULL: Self = Self(u32::MAX); - const fn idx(self) -> usize { - self.0 as usize - } -} -impl ops::Index for [Block] { - type Output = Block; - fn index(&self, idx: BlockIdx) -> &Block { - &self[idx.idx()] - } -} -impl ops::IndexMut for [Block] { - fn index_mut(&mut self, idx: BlockIdx) -> &mut Block { - &mut self[idx.idx()] - } -} -impl ops::Index for Vec { - type Output = Block; - fn index(&self, idx: BlockIdx) -> &Block { - &self[idx.idx()] - } -} -impl ops::IndexMut for Vec { - fn index_mut(&mut self, idx: BlockIdx) -> &mut Block { - &mut self[idx.idx()] - } -} - -#[derive(Debug, Clone)] -pub struct InstructionInfo { - pub instr: Instruction, - pub arg: OpArg, - pub target: BlockIdx, - // pub range: TextRange, - pub location: SourceLocation, -} - -// spell-checker:ignore petgraph -// TODO: look into using petgraph for handling blocks and stuff? it's heavier than this, but it -// might enable more analysis/optimizations -#[derive(Debug)] -pub struct Block { - pub instructions: Vec, - pub next: BlockIdx, -} -impl Default for Block { - fn default() -> Self { - Self { - instructions: Vec::new(), - next: BlockIdx::NULL, - } - } -} - -pub struct CodeInfo { - pub flags: CodeFlags, - pub posonlyarg_count: u32, // Number of positional-only arguments - pub arg_count: u32, - pub kwonlyarg_count: u32, - pub source_path: String, - pub first_line_number: OneIndexed, - pub obj_name: String, // Name of the object that created this code object - - pub blocks: Vec, - pub current_block: BlockIdx, - pub constants: IndexSet, - pub name_cache: IndexSet, - pub varname_cache: IndexSet, - pub cellvar_cache: IndexSet, - pub freevar_cache: IndexSet, -} -impl CodeInfo { - pub fn finalize_code(mut self, optimize: u8) -> crate::InternalResult { - if optimize > 0 { - self.dce(); - } - - let max_stackdepth = self.max_stackdepth()?; - let cell2arg = self.cell2arg(); - - let Self { - flags, - posonlyarg_count, - arg_count, - kwonlyarg_count, - source_path, - first_line_number, - obj_name, - - mut blocks, - current_block: _, - constants, - name_cache, - varname_cache, - cellvar_cache, - freevar_cache, - } = self; - - let mut instructions = Vec::new(); - let mut locations = Vec::new(); - - let mut block_to_offset = vec![Label(0); blocks.len()]; - loop { - let mut num_instructions = 0; - for (idx, block) in iter_blocks(&blocks) { - block_to_offset[idx.idx()] = Label(num_instructions as u32); - for instr in &block.instructions { - num_instructions += instr.arg.instr_size() - } - } - - instructions.reserve_exact(num_instructions); - locations.reserve_exact(num_instructions); - - let mut recompile_extended_arg = false; - let mut next_block = BlockIdx(0); - while next_block != BlockIdx::NULL { - let block = &mut blocks[next_block]; - for info in &mut block.instructions { - let (op, arg, target) = (info.instr, &mut info.arg, info.target); - if target != BlockIdx::NULL { - let new_arg = OpArg(block_to_offset[target.idx()].0); - recompile_extended_arg |= new_arg.instr_size() != arg.instr_size(); - *arg = new_arg; - } - let (extras, lo_arg) = arg.split(); - locations.extend(std::iter::repeat_n(info.location.clone(), arg.instr_size())); - instructions.extend( - extras - .map(|byte| CodeUnit::new(Instruction::ExtendedArg, byte)) - .chain([CodeUnit { op, arg: lo_arg }]), - ); - } - next_block = block.next; - } - - if !recompile_extended_arg { - break; - } - - instructions.clear(); - locations.clear() - } - - Ok(CodeObject { - flags, - posonlyarg_count, - arg_count, - kwonlyarg_count, - source_path, - first_line_number: Some(first_line_number), - obj_name, - - max_stackdepth, - instructions: instructions.into_boxed_slice(), - locations: locations.into_boxed_slice(), - constants: constants.into_iter().collect(), - names: name_cache.into_iter().collect(), - varnames: varname_cache.into_iter().collect(), - cellvars: cellvar_cache.into_iter().collect(), - freevars: freevar_cache.into_iter().collect(), - cell2arg, - }) - } - - fn cell2arg(&self) -> Option> { - if self.cellvar_cache.is_empty() { - return None; - } - - let total_args = self.arg_count - + self.kwonlyarg_count - + self.flags.contains(CodeFlags::HAS_VARARGS) as u32 - + self.flags.contains(CodeFlags::HAS_VARKEYWORDS) as u32; - - let mut found_cellarg = false; - let cell2arg = self - .cellvar_cache - .iter() - .map(|var| { - self.varname_cache - .get_index_of(var) - // check that it's actually an arg - .filter(|i| *i < total_args as usize) - .map_or(-1, |i| { - found_cellarg = true; - i as i32 - }) - }) - .collect::>(); - - if found_cellarg { Some(cell2arg) } else { None } - } - - fn dce(&mut self) { - for block in &mut self.blocks { - let mut last_instr = None; - for (i, ins) in block.instructions.iter().enumerate() { - if ins.instr.unconditional_branch() { - last_instr = Some(i); - break; - } - } - if let Some(i) = last_instr { - block.instructions.truncate(i + 1); - } - } - } - - fn max_stackdepth(&self) -> crate::InternalResult { - let mut maxdepth = 0u32; - let mut stack = Vec::with_capacity(self.blocks.len()); - let mut start_depths = vec![u32::MAX; self.blocks.len()]; - start_depths[0] = 0; - stack.push(BlockIdx(0)); - const DEBUG: bool = false; - 'process_blocks: while let Some(block) = stack.pop() { - let mut depth = start_depths[block.idx()]; - if DEBUG { - eprintln!("===BLOCK {}===", block.0); - } - let block = &self.blocks[block]; - for ins in &block.instructions { - let instr = &ins.instr; - let effect = instr.stack_effect(ins.arg, false); - if DEBUG { - let display_arg = if ins.target == BlockIdx::NULL { - ins.arg - } else { - OpArg(ins.target.0) - }; - let instr_display = instr.display(display_arg, self); - eprint!("{instr_display}: {depth} {effect:+} => "); - } - let new_depth = depth.checked_add_signed(effect).ok_or({ - if effect < 0 { - InternalError::StackUnderflow - } else { - InternalError::StackOverflow - } - })?; - if DEBUG { - eprintln!("{new_depth}"); - } - if new_depth > maxdepth { - maxdepth = new_depth - } - // we don't want to worry about Break/Continue, they use unwinding to jump to - // their targets and as such the stack size is taken care of in frame.rs by setting - // it back to the level it was at when SetupLoop was run - if ins.target != BlockIdx::NULL - && !matches!( - instr, - Instruction::Continue { .. } | Instruction::Break { .. } - ) - { - let effect = instr.stack_effect(ins.arg, true); - let target_depth = depth.checked_add_signed(effect).ok_or({ - if effect < 0 { - InternalError::StackUnderflow - } else { - InternalError::StackOverflow - } - })?; - if target_depth > maxdepth { - maxdepth = target_depth - } - stackdepth_push(&mut stack, &mut start_depths, ins.target, target_depth); - } - depth = new_depth; - if instr.unconditional_branch() { - continue 'process_blocks; - } - } - stackdepth_push(&mut stack, &mut start_depths, block.next, depth); - } - if DEBUG { - eprintln!("DONE: {maxdepth}"); - } - Ok(maxdepth) - } -} - -impl InstrDisplayContext for CodeInfo { - type Constant = ConstantData; - fn get_constant(&self, i: usize) -> &ConstantData { - &self.constants[i] - } - fn get_name(&self, i: usize) -> &str { - self.name_cache[i].as_ref() - } - fn get_varname(&self, i: usize) -> &str { - self.varname_cache[i].as_ref() - } - fn get_cell_name(&self, i: usize) -> &str { - self.cellvar_cache - .get_index(i) - .unwrap_or_else(|| &self.freevar_cache[i - self.cellvar_cache.len()]) - .as_ref() - } -} - -fn stackdepth_push( - stack: &mut Vec, - start_depths: &mut [u32], - target: BlockIdx, - depth: u32, -) { - let block_depth = &mut start_depths[target.idx()]; - if *block_depth == u32::MAX || depth > *block_depth { - *block_depth = depth; - stack.push(target); - } -} - -fn iter_blocks(blocks: &[Block]) -> impl Iterator + '_ { - let mut next = BlockIdx(0); - std::iter::from_fn(move || { - if next == BlockIdx::NULL { - return None; - } - let (idx, b) = (next, &blocks[next]); - next = b.next; - Some((idx, b)) - }) -} diff --git a/compiler/codegen/src/snapshots/rustpython_codegen__compile__tests__if_ands.snap b/compiler/codegen/src/snapshots/rustpython_codegen__compile__tests__if_ands.snap deleted file mode 100644 index cf6223be1f..0000000000 --- a/compiler/codegen/src/snapshots/rustpython_codegen__compile__tests__if_ands.snap +++ /dev/null @@ -1,12 +0,0 @@ ---- -source: compiler/codegen/src/compile.rs -expression: "compile_exec(\"\\\nif True and False and False:\n pass\n\")" ---- - 1 0 LoadConst (True) - 1 JumpIfFalse (6) - 2 LoadConst (False) - 3 JumpIfFalse (6) - 4 LoadConst (False) - 5 JumpIfFalse (6) - - 2 >> 6 ReturnConst (None) diff --git a/compiler/codegen/src/snapshots/rustpython_codegen__compile__tests__if_mixed.snap b/compiler/codegen/src/snapshots/rustpython_codegen__compile__tests__if_mixed.snap deleted file mode 100644 index 332d0fa985..0000000000 --- a/compiler/codegen/src/snapshots/rustpython_codegen__compile__tests__if_mixed.snap +++ /dev/null @@ -1,14 +0,0 @@ ---- -source: compiler/codegen/src/compile.rs -expression: "compile_exec(\"\\\nif (True and False) or (False and True):\n pass\n\")" ---- - 1 0 LoadConst (True) - 1 JumpIfFalse (4) - 2 LoadConst (False) - 3 JumpIfTrue (8) - >> 4 LoadConst (False) - 5 JumpIfFalse (8) - 6 LoadConst (True) - 7 JumpIfFalse (8) - - 2 >> 8 ReturnConst (None) diff --git a/compiler/codegen/src/snapshots/rustpython_codegen__compile__tests__if_ors.snap b/compiler/codegen/src/snapshots/rustpython_codegen__compile__tests__if_ors.snap deleted file mode 100644 index d4a4d8f2c5..0000000000 --- a/compiler/codegen/src/snapshots/rustpython_codegen__compile__tests__if_ors.snap +++ /dev/null @@ -1,12 +0,0 @@ ---- -source: compiler/codegen/src/compile.rs -expression: "compile_exec(\"\\\nif True or False or False:\n pass\n\")" ---- - 1 0 LoadConst (True) - 1 JumpIfTrue (6) - 2 LoadConst (False) - 3 JumpIfTrue (6) - 4 LoadConst (False) - 5 JumpIfFalse (6) - - 2 >> 6 ReturnConst (None) diff --git a/compiler/codegen/src/snapshots/rustpython_codegen__compile__tests__nested_double_async_with.snap b/compiler/codegen/src/snapshots/rustpython_codegen__compile__tests__nested_double_async_with.snap deleted file mode 100644 index 36b00c567d..0000000000 --- a/compiler/codegen/src/snapshots/rustpython_codegen__compile__tests__nested_double_async_with.snap +++ /dev/null @@ -1,86 +0,0 @@ ---- -source: compiler/codegen/src/compile.rs -expression: "compile_exec(\"\\\nfor stop_exc in (StopIteration('spam'), StopAsyncIteration('ham')):\n with self.subTest(type=type(stop_exc)):\n try:\n async with egg():\n raise stop_exc\n except Exception as ex:\n self.assertIs(ex, stop_exc)\n else:\n self.fail(f'{stop_exc} was suppressed')\n\")" ---- - 1 0 SetupLoop - 1 LoadNameAny (0, StopIteration) - 2 LoadConst ("spam") - 3 CallFunctionPositional(1) - 4 LoadNameAny (1, StopAsyncIteration) - 5 LoadConst ("ham") - 6 CallFunctionPositional(1) - 7 BuildTuple (2) - 8 GetIter - >> 9 ForIter (71) - 10 StoreLocal (2, stop_exc) - - 2 11 LoadNameAny (3, self) - 12 LoadMethod (4, subTest) - 13 LoadNameAny (5, type) - 14 LoadNameAny (2, stop_exc) - 15 CallFunctionPositional(1) - 16 LoadConst (("type")) - 17 CallMethodKeyword (1) - 18 SetupWith (68) - 19 Pop - - 3 20 SetupExcept (40) - - 4 21 LoadNameAny (6, egg) - 22 CallFunctionPositional(0) - 23 BeforeAsyncWith - 24 GetAwaitable - 25 LoadConst (None) - 26 YieldFrom - 27 SetupAsyncWith (33) - 28 Pop - - 5 29 LoadNameAny (2, stop_exc) - 30 Raise (Raise) - - 4 31 PopBlock - 32 EnterFinally - >> 33 WithCleanupStart - 34 GetAwaitable - 35 LoadConst (None) - 36 YieldFrom - 37 WithCleanupFinish - 38 PopBlock - 39 Jump (57) - >> 40 Duplicate - - 6 41 LoadNameAny (7, Exception) - 42 TestOperation (ExceptionMatch) - 43 JumpIfFalse (56) - 44 StoreLocal (8, ex) - - 7 45 LoadNameAny (3, self) - 46 LoadMethod (9, assertIs) - 47 LoadNameAny (8, ex) - 48 LoadNameAny (2, stop_exc) - 49 CallMethodPositional (2) - 50 Pop - 51 PopException - 52 LoadConst (None) - 53 StoreLocal (8, ex) - 54 DeleteLocal (8, ex) - 55 Jump (66) - >> 56 Raise (Reraise) - - 9 >> 57 LoadNameAny (3, self) - 58 LoadMethod (10, fail) - 59 LoadConst ("") - 60 LoadNameAny (2, stop_exc) - 61 FormatValue (None) - 62 LoadConst (" was suppressed") - 63 BuildString (2) - 64 CallMethodPositional (1) - 65 Pop - - 2 >> 66 PopBlock - 67 EnterFinally - >> 68 WithCleanupStart - 69 WithCleanupFinish - 70 Jump (9) - >> 71 PopBlock - 72 ReturnConst (None) diff --git a/compiler/codegen/Cargo.toml b/crates/codegen/Cargo.toml similarity index 81% rename from compiler/codegen/Cargo.toml rename to crates/codegen/Cargo.toml index 479b0b29f6..ce7e8d74f5 100644 --- a/compiler/codegen/Cargo.toml +++ b/crates/codegen/Cargo.toml @@ -8,17 +8,12 @@ rust-version.workspace = true repository.workspace = true license.workspace = true - [dependencies] -# rustpython-ast = { workspace = true, features=["unparse", "constant-optimization"] } -# rustpython-parser-core = { workspace = true } rustpython-compiler-core = { workspace = true } -rustpython-compiler-source = {workspace = true } rustpython-literal = {workspace = true } rustpython-wtf8 = { workspace = true } ruff_python_ast = { workspace = true } ruff_text_size = { workspace = true } -ruff_source_file = { workspace = true } ahash = { workspace = true } bitflags = { workspace = true } diff --git a/compiler/codegen/src/compile.rs b/crates/codegen/src/compile.rs similarity index 58% rename from compiler/codegen/src/compile.rs rename to crates/codegen/src/compile.rs index 4180b79712..7909e92425 100644 --- a/compiler/codegen/src/compile.rs +++ b/crates/codegen/src/compile.rs @@ -5,14 +5,16 @@ //! //! +// spell-checker:ignore starunpack subscripter + #![deny(clippy::cast_possible_truncation)] use crate::{ - IndexSet, ToPythonName, - error::{CodegenError, CodegenErrorType, PatternUnreachableReason}, + IndexMap, IndexSet, ToPythonName, + error::{CodegenError, CodegenErrorType, InternalError, PatternUnreachableReason}, ir::{self, BlockIdx}, - symboltable::{self, SymbolFlags, SymbolScope, SymbolTable}, - unparse::unparse_expr, + symboltable::{self, CompilerScope, SymbolFlags, SymbolScope, SymbolTable}, + unparse::UnparseExpr, }; use itertools::Itertools; use malachite_bigint::BigInt; @@ -20,29 +22,54 @@ use num_complex::Complex; use num_traits::{Num, ToPrimitive}; use ruff_python_ast::{ Alias, Arguments, BoolOp, CmpOp, Comprehension, ConversionFlag, DebugText, Decorator, DictItem, - ExceptHandler, ExceptHandlerExceptHandler, Expr, ExprAttribute, ExprBoolOp, ExprFString, - ExprList, ExprName, ExprStarred, ExprSubscript, ExprTuple, ExprUnaryOp, FString, - FStringElement, FStringElements, FStringFlags, FStringPart, Identifier, Int, Keyword, - MatchCase, ModExpression, ModModule, Operator, Parameters, Pattern, PatternMatchAs, - PatternMatchClass, PatternMatchOr, PatternMatchSequence, PatternMatchSingleton, - PatternMatchStar, PatternMatchValue, Singleton, Stmt, StmtExpr, TypeParam, TypeParamParamSpec, - TypeParamTypeVar, TypeParamTypeVarTuple, TypeParams, UnaryOp, WithItem, + ExceptHandler, ExceptHandlerExceptHandler, Expr, ExprAttribute, ExprBoolOp, ExprContext, + ExprFString, ExprList, ExprName, ExprSlice, ExprStarred, ExprSubscript, ExprTuple, ExprUnaryOp, + FString, FStringFlags, FStringPart, Identifier, Int, InterpolatedStringElement, + InterpolatedStringElements, Keyword, MatchCase, ModExpression, ModModule, Operator, Parameters, + Pattern, PatternMatchAs, PatternMatchClass, PatternMatchMapping, PatternMatchOr, + PatternMatchSequence, PatternMatchSingleton, PatternMatchStar, PatternMatchValue, Singleton, + Stmt, StmtExpr, TypeParam, TypeParamParamSpec, TypeParamTypeVar, TypeParamTypeVarTuple, + TypeParams, UnaryOp, WithItem, + visitor::{Visitor, walk_expr}, }; -use ruff_source_file::OneIndexed; use ruff_text_size::{Ranged, TextRange}; -use rustpython_wtf8::Wtf8Buf; -// use rustpython_ast::located::{self as located_ast, Located}; use rustpython_compiler_core::{ - Mode, + Mode, OneIndexed, PositionEncoding, SourceFile, SourceLocation, bytecode::{ - self, Arg as OpArgMarker, BinaryOperator, CodeObject, ComparisonOperator, ConstantData, - Instruction, OpArg, OpArgType, UnpackExArgs, + self, Arg as OpArgMarker, BinaryOperator, BuildSliceArgCount, CodeObject, + ComparisonOperator, ConstantData, ConvertValueOparg, Instruction, Invert, OpArg, OpArgType, + UnpackExArgs, }, }; -use rustpython_compiler_source::SourceCode; -// use rustpython_parser_core::source_code::{LineNumber, SourceLocation}; -use crate::error::InternalError; -use std::borrow::Cow; +use rustpython_wtf8::Wtf8Buf; +use std::{borrow::Cow, collections::HashSet}; + +const MAXBLOCKS: usize = 20; + +#[derive(Debug, Clone, Copy)] +pub enum FBlockType { + WhileLoop, + ForLoop, + TryExcept, + FinallyTry, + FinallyEnd, + With, + AsyncWith, + HandlerCleanup, + PopValue, + ExceptionHandler, + ExceptionGroupHandler, + AsyncComprehensionGenerator, + StopIteration, +} + +#[derive(Debug, Clone)] +pub struct FBlockInfo { + pub fb_type: FBlockType, + pub fb_block: BlockIdx, + pub fb_exit: BlockIdx, + // fb_datum is not needed in RustPython +} pub(crate) type InternalResult = Result; type CompileResult = Result; @@ -68,17 +95,15 @@ fn is_forbidden_name(name: &str) -> bool { } /// Main structure holding the state of compilation. -struct Compiler<'src> { +struct Compiler { code_stack: Vec, symbol_table_stack: Vec, - source_code: SourceCode<'src>, + source_file: SourceFile, // current_source_location: SourceLocation, current_source_range: TextRange, - qualified_path: Vec, done_with_future_stmts: DoneWithFuture, future_annotations: bool, ctx: CompileContext, - class_name: Option, opts: CompileOpts, in_annotation: bool, } @@ -98,7 +123,7 @@ pub struct CompileOpts { #[derive(Debug, Clone, Copy)] struct CompileContext { - loop_data: Option<(ir::BlockIdx, ir::BlockIdx)>, + loop_data: Option<(BlockIdx, BlockIdx)>, in_class: bool, func: FunctionContext, } @@ -124,34 +149,48 @@ enum ComprehensionType { Dict, } +fn validate_duplicate_params(params: &Parameters) -> Result<(), CodegenErrorType> { + let mut seen_params = HashSet::new(); + for param in params { + let param_name = param.name().as_str(); + if !seen_params.insert(param_name) { + return Err(CodegenErrorType::SyntaxError(format!( + r#"Duplicate parameter "{param_name}""# + ))); + } + } + + Ok(()) +} + /// Compile an Mod produced from ruff parser pub fn compile_top( ast: ruff_python_ast::Mod, - source_code: SourceCode<'_>, + source_file: SourceFile, mode: Mode, opts: CompileOpts, ) -> CompileResult { match ast { ruff_python_ast::Mod::Module(module) => match mode { - Mode::Exec | Mode::Eval => compile_program(&module, source_code, opts), - Mode::Single => compile_program_single(&module, source_code, opts), - Mode::BlockExpr => compile_block_expression(&module, source_code, opts), + Mode::Exec | Mode::Eval => compile_program(&module, source_file, opts), + Mode::Single => compile_program_single(&module, source_file, opts), + Mode::BlockExpr => compile_block_expression(&module, source_file, opts), }, - ruff_python_ast::Mod::Expression(expr) => compile_expression(&expr, source_code, opts), + ruff_python_ast::Mod::Expression(expr) => compile_expression(&expr, source_file, opts), } } /// Compile a standard Python program to bytecode pub fn compile_program( ast: &ModModule, - source_code: SourceCode<'_>, + source_file: SourceFile, opts: CompileOpts, ) -> CompileResult { - let symbol_table = SymbolTable::scan_program(ast, source_code.clone()) - .map_err(|e| e.into_codegen_error(source_code.path.to_owned()))?; - let mut compiler = Compiler::new(opts, source_code, "".to_owned()); + let symbol_table = SymbolTable::scan_program(ast, source_file.clone()) + .map_err(|e| e.into_codegen_error(source_file.name().to_owned()))?; + let mut compiler = Compiler::new(opts, source_file, "".to_owned()); compiler.compile_program(ast, symbol_table)?; - let code = compiler.pop_code_object(); + let code = compiler.exit_scope(); trace!("Compilation completed: {code:?}"); Ok(code) } @@ -159,42 +198,42 @@ pub fn compile_program( /// Compile a Python program to bytecode for the context of a REPL pub fn compile_program_single( ast: &ModModule, - source_code: SourceCode<'_>, + source_file: SourceFile, opts: CompileOpts, ) -> CompileResult { - let symbol_table = SymbolTable::scan_program(ast, source_code.clone()) - .map_err(|e| e.into_codegen_error(source_code.path.to_owned()))?; - let mut compiler = Compiler::new(opts, source_code, "".to_owned()); + let symbol_table = SymbolTable::scan_program(ast, source_file.clone()) + .map_err(|e| e.into_codegen_error(source_file.name().to_owned()))?; + let mut compiler = Compiler::new(opts, source_file, "".to_owned()); compiler.compile_program_single(&ast.body, symbol_table)?; - let code = compiler.pop_code_object(); + let code = compiler.exit_scope(); trace!("Compilation completed: {code:?}"); Ok(code) } pub fn compile_block_expression( ast: &ModModule, - source_code: SourceCode<'_>, + source_file: SourceFile, opts: CompileOpts, ) -> CompileResult { - let symbol_table = SymbolTable::scan_program(ast, source_code.clone()) - .map_err(|e| e.into_codegen_error(source_code.path.to_owned()))?; - let mut compiler = Compiler::new(opts, source_code, "".to_owned()); + let symbol_table = SymbolTable::scan_program(ast, source_file.clone()) + .map_err(|e| e.into_codegen_error(source_file.name().to_owned()))?; + let mut compiler = Compiler::new(opts, source_file, "".to_owned()); compiler.compile_block_expr(&ast.body, symbol_table)?; - let code = compiler.pop_code_object(); + let code = compiler.exit_scope(); trace!("Compilation completed: {code:?}"); Ok(code) } pub fn compile_expression( ast: &ModExpression, - source_code: SourceCode<'_>, + source_file: SourceFile, opts: CompileOpts, ) -> CompileResult { - let symbol_table = SymbolTable::scan_expr(ast, source_code.clone()) - .map_err(|e| e.into_codegen_error(source_code.path.to_owned()))?; - let mut compiler = Compiler::new(opts, source_code, "".to_owned()); + let symbol_table = SymbolTable::scan_expr(ast, source_file.clone()) + .map_err(|e| e.into_codegen_error(source_file.name().to_owned()))?; + let mut compiler = Compiler::new(opts, source_file, "".to_owned()); compiler.compile_eval(ast, symbol_table)?; - let code = compiler.pop_code_object(); + let code = compiler.exit_scope(); Ok(code) } @@ -213,26 +252,28 @@ macro_rules! emit { }; } -fn eprint_location(zelf: &Compiler<'_>) { +fn eprint_location(zelf: &Compiler) { let start = zelf - .source_code - .source_location(zelf.current_source_range.start()); + .source_file + .to_source_code() + .source_location(zelf.current_source_range.start(), PositionEncoding::Utf8); let end = zelf - .source_code - .source_location(zelf.current_source_range.end()); + .source_file + .to_source_code() + .source_location(zelf.current_source_range.end(), PositionEncoding::Utf8); eprintln!( "LOCATION: {} from {}:{} to {}:{}", - zelf.source_code.path.to_owned(), - start.row, - start.column, - end.row, - end.column + zelf.source_file.name(), + start.line, + start.character_offset, + end.line, + end.character_offset ); } /// Better traceback for internal error #[track_caller] -fn unwrap_internal(zelf: &Compiler<'_>, r: InternalResult) -> T { +fn unwrap_internal(zelf: &Compiler, r: InternalResult) -> T { if let Err(ref r_err) = r { eprintln!("=== CODEGEN PANIC INFO ==="); eprintln!("This IS an internal error: {r_err}"); @@ -242,7 +283,7 @@ fn unwrap_internal(zelf: &Compiler<'_>, r: InternalResult) -> T { r.unwrap() } -fn compiler_unwrap_option(zelf: &Compiler<'_>, o: Option) -> T { +fn compiler_unwrap_option(zelf: &Compiler, o: Option) -> T { if o.is_none() { eprintln!("=== CODEGEN PANIC INFO ==="); eprintln!("This IS an internal error, an option was unwrapped during codegen"); @@ -252,7 +293,7 @@ fn compiler_unwrap_option(zelf: &Compiler<'_>, o: Option) -> T { o.unwrap() } -// fn compiler_result_unwrap(zelf: &Compiler<'_>, result: Result) -> T { +// fn compiler_result_unwrap(zelf: &Compiler, result: Result) -> T { // if result.is_err() { // eprintln!("=== CODEGEN PANIC INFO ==="); // eprintln!("This IS an internal error, an result was unwrapped during codegen"); @@ -301,32 +342,47 @@ enum JumpOp { PopJumpIfFalse, } -impl<'src> Compiler<'src> { - fn new(opts: CompileOpts, source_code: SourceCode<'src>, code_name: String) -> Self { +/// Type of collection to build in starunpack_helper +#[derive(Debug, Clone, Copy, PartialEq)] +enum CollectionType { + Tuple, + List, + Set, +} + +impl Compiler { + fn new(opts: CompileOpts, source_file: SourceFile, code_name: String) -> Self { let module_code = ir::CodeInfo { flags: bytecode::CodeFlags::NEW_LOCALS, - posonlyarg_count: 0, - arg_count: 0, - kwonlyarg_count: 0, - source_path: source_code.path.to_owned(), - first_line_number: OneIndexed::MIN, - obj_name: code_name, - + source_path: source_file.name().to_owned(), + private: None, blocks: vec![ir::Block::default()], - current_block: ir::BlockIdx(0), - constants: IndexSet::default(), - name_cache: IndexSet::default(), - varname_cache: IndexSet::default(), - cellvar_cache: IndexSet::default(), - freevar_cache: IndexSet::default(), + current_block: BlockIdx::new(0), + metadata: ir::CodeUnitMetadata { + name: code_name.clone(), + qualname: Some(code_name), + consts: IndexSet::default(), + names: IndexSet::default(), + varnames: IndexSet::default(), + cellvars: IndexSet::default(), + freevars: IndexSet::default(), + fast_hidden: IndexMap::default(), + argcount: 0, + posonlyargcount: 0, + kwonlyargcount: 0, + firstlineno: OneIndexed::MIN, + }, + static_attributes: None, + in_inlined_comp: false, + fblock: Vec::with_capacity(MAXBLOCKS), + symbol_table_index: 0, // Module is always the first symbol table }; - Compiler { + Self { code_stack: vec![module_code], symbol_table_stack: Vec::new(), - source_code, + source_file, // current_source_location: SourceLocation::default(), current_source_range: TextRange::default(), - qualified_path: Vec::new(), done_with_future_stmts: DoneWithFuture::No, future_annotations: false, ctx: CompileContext { @@ -334,39 +390,243 @@ impl<'src> Compiler<'src> { in_class: false, func: FunctionContext::NoFunction, }, - class_name: None, opts, in_annotation: false, } } -} -impl Compiler<'_> { + /// Check if the slice is a two-element slice (no step) + // = is_two_element_slice + const fn is_two_element_slice(slice: &Expr) -> bool { + matches!(slice, Expr::Slice(s) if s.step.is_none()) + } + + /// Compile a slice expression + // = compiler_slice + fn compile_slice(&mut self, s: &ExprSlice) -> CompileResult { + // Compile lower + if let Some(lower) = &s.lower { + self.compile_expression(lower)?; + } else { + self.emit_load_const(ConstantData::None); + } + + // Compile upper + if let Some(upper) = &s.upper { + self.compile_expression(upper)?; + } else { + self.emit_load_const(ConstantData::None); + } + + Ok(match &s.step { + Some(step) => { + // Compile step if present + self.compile_expression(step)?; + BuildSliceArgCount::Three + } + None => BuildSliceArgCount::Two, + }) + } + + /// Compile a subscript expression + // = compiler_subscript + fn compile_subscript( + &mut self, + value: &Expr, + slice: &Expr, + ctx: ExprContext, + ) -> CompileResult<()> { + // 1. Check subscripter and index for Load context + // 2. VISIT value + // 3. Handle two-element slice specially + // 4. Otherwise VISIT slice and emit appropriate instruction + + // For Load context, CPython does some checks (we skip for now) + // if ctx == ExprContext::Load { + // check_subscripter(value); + // check_index(value, slice); + // } + + // VISIT(c, expr, e->v.Subscript.value) + self.compile_expression(value)?; + + // Handle two-element slice (for Load/Store, not Del) + if Self::is_two_element_slice(slice) && !matches!(ctx, ExprContext::Del) { + let argc = match slice { + Expr::Slice(s) => self.compile_slice(s)?, + _ => unreachable!("is_two_element_slice should only return true for Expr::Slice"), + }; + match ctx { + ExprContext::Load => { + // CPython uses BINARY_SLICE + emit!(self, Instruction::BuildSlice { argc }); + emit!(self, Instruction::Subscript); + } + ExprContext::Store => { + // CPython uses STORE_SLICE + emit!(self, Instruction::BuildSlice { argc }); + emit!(self, Instruction::StoreSubscript); + } + _ => unreachable!(), + } + } else { + // VISIT(c, expr, e->v.Subscript.slice) + self.compile_expression(slice)?; + + // Emit appropriate instruction based on context + match ctx { + ExprContext::Load => emit!(self, Instruction::Subscript), + ExprContext::Store => emit!(self, Instruction::StoreSubscript), + ExprContext::Del => emit!(self, Instruction::DeleteSubscript), + ExprContext::Invalid => { + return Err(self.error(CodegenErrorType::SyntaxError( + "Invalid expression context".to_owned(), + ))); + } + } + } + + Ok(()) + } + + /// Helper function for compiling tuples/lists/sets with starred expressions + /// + /// Parameters: + /// - elts: The elements to compile + /// - pushed: Number of items already on the stack + /// - collection_type: What type of collection to build (tuple, list, set) + /// + // = starunpack_helper in compile.c + fn starunpack_helper( + &mut self, + elts: &[Expr], + pushed: u32, + collection_type: CollectionType, + ) -> CompileResult<()> { + // Use RustPython's existing approach with BuildXFromTuples + let (size, unpack) = self.gather_elements(pushed, elts)?; + + if unpack { + // Has starred elements + match collection_type { + CollectionType::Tuple => { + if size > 1 || pushed > 0 { + emit!(self, Instruction::BuildTupleFromTuples { size }); + } + // If size == 1 and pushed == 0, the single tuple is already on the stack + } + CollectionType::List => { + emit!(self, Instruction::BuildListFromTuples { size }); + } + CollectionType::Set => { + emit!(self, Instruction::BuildSetFromTuples { size }); + } + } + } else { + // No starred elements + match collection_type { + CollectionType::Tuple => { + emit!(self, Instruction::BuildTuple { size }); + } + CollectionType::List => { + emit!(self, Instruction::BuildList { size }); + } + CollectionType::Set => { + emit!(self, Instruction::BuildSet { size }); + } + } + } + + Ok(()) + } + fn error(&mut self, error: CodegenErrorType) -> CodegenError { self.error_ranged(error, self.current_source_range) } + fn error_ranged(&mut self, error: CodegenErrorType, range: TextRange) -> CodegenError { - let location = self.source_code.source_location(range.start()); + let location = self + .source_file + .to_source_code() + .source_location(range.start(), PositionEncoding::Utf8); CodegenError { error, location: Some(location), - source_path: self.source_code.path.to_owned(), + source_path: self.source_file.name().to_owned(), } } + /// Get the SymbolTable for the current scope. + fn current_symbol_table(&self) -> &SymbolTable { + self.symbol_table_stack + .last() + .expect("symbol_table_stack is empty! This is a compiler bug.") + } + + /// Get the index of a free variable. + fn get_free_var_index(&mut self, name: &str) -> CompileResult { + let info = self.code_stack.last_mut().unwrap(); + let idx = info + .metadata + .freevars + .get_index_of(name) + .unwrap_or_else(|| info.metadata.freevars.insert_full(name.to_owned()).0); + Ok((idx + info.metadata.cellvars.len()).to_u32()) + } + + /// Get the index of a cell variable. + fn get_cell_var_index(&mut self, name: &str) -> CompileResult { + let info = self.code_stack.last_mut().unwrap(); + let idx = info + .metadata + .cellvars + .get_index_of(name) + .unwrap_or_else(|| info.metadata.cellvars.insert_full(name.to_owned()).0); + Ok(idx.to_u32()) + } + + /// Get the index of a local variable. + fn get_local_var_index(&mut self, name: &str) -> CompileResult { + let info = self.code_stack.last_mut().unwrap(); + let idx = info + .metadata + .varnames + .get_index_of(name) + .unwrap_or_else(|| info.metadata.varnames.insert_full(name.to_owned()).0); + Ok(idx.to_u32()) + } + + /// Get the index of a global name. + fn get_global_name_index(&mut self, name: &str) -> u32 { + let info = self.code_stack.last_mut().unwrap(); + let idx = info + .metadata + .names + .get_index_of(name) + .unwrap_or_else(|| info.metadata.names.insert_full(name.to_owned()).0); + idx.to_u32() + } + /// Push the next symbol table on to the stack fn push_symbol_table(&mut self) -> &SymbolTable { // Look up the next table contained in the scope of the current table - let table = self + let current_table = self .symbol_table_stack .last_mut() - .expect("no next symbol table") - .sub_tables - .remove(0); + .expect("no current symbol table"); + + if current_table.sub_tables.is_empty() { + panic!( + "push_symbol_table: no sub_tables available in {} (type: {:?})", + current_table.name, current_table.typ + ); + } + + let table = current_table.sub_tables.remove(0); + // Push the next table onto the stack - let last_idx = self.symbol_table_stack.len(); self.symbol_table_stack.push(table); - &self.symbol_table_stack[last_idx] + self.current_symbol_table() } /// Pop the current symbol table off the stack @@ -374,74 +634,276 @@ impl Compiler<'_> { self.symbol_table_stack.pop().expect("compiler bug") } - fn push_output( + /// Enter a new scope + // = compiler_enter_scope + fn enter_scope( &mut self, - flags: bytecode::CodeFlags, - posonlyarg_count: u32, - arg_count: u32, - kwonlyarg_count: u32, - obj_name: String, - ) { - let source_path = self.source_code.path.to_owned(); - let first_line_number = self.get_source_line_number(); + name: &str, + scope_type: CompilerScope, + key: usize, // In RustPython, we use the index in symbol_table_stack as key + lineno: u32, + ) -> CompileResult<()> { + // Create location + let location = SourceLocation { + line: OneIndexed::new(lineno as usize).unwrap_or(OneIndexed::MIN), + character_offset: OneIndexed::MIN, + }; - let table = self.push_symbol_table(); + // Allocate a new compiler unit - let cellvar_cache = table + // In Rust, we'll create the structure directly + let source_path = self.source_file.name().to_owned(); + + // Lookup symbol table entry using key (_PySymtable_Lookup) + let ste = match self.symbol_table_stack.get(key) { + Some(v) => v, + None => { + return Err(self.error(CodegenErrorType::SyntaxError( + "unknown symbol table entry".to_owned(), + ))); + } + }; + + // Use varnames from symbol table (already collected in definition order) + let varname_cache: IndexSet = ste.varnames.iter().cloned().collect(); + + // Build cellvars using dictbytype (CELL scope, sorted) + let mut cellvar_cache = IndexSet::default(); + let mut cell_names: Vec<_> = ste .symbols .iter() .filter(|(_, s)| s.scope == SymbolScope::Cell) - .map(|(var, _)| var.clone()) + .map(|(name, _)| name.clone()) .collect(); - let freevar_cache = table + cell_names.sort(); + for name in cell_names { + cellvar_cache.insert(name); + } + + // Handle implicit __class__ cell if needed + if ste.needs_class_closure { + // Cook up an implicit __class__ cell + debug_assert_eq!(scope_type, CompilerScope::Class); + cellvar_cache.insert("__class__".to_string()); + } + + // Handle implicit __classdict__ cell if needed + if ste.needs_classdict { + // Cook up an implicit __classdict__ cell + debug_assert_eq!(scope_type, CompilerScope::Class); + cellvar_cache.insert("__classdict__".to_string()); + } + + // Build freevars using dictbytype (FREE scope, offset by cellvars size) + let mut freevar_cache = IndexSet::default(); + let mut free_names: Vec<_> = ste .symbols .iter() .filter(|(_, s)| { s.scope == SymbolScope::Free || s.flags.contains(SymbolFlags::FREE_CLASS) }) - .map(|(var, _)| var.clone()) + .map(|(name, _)| name.clone()) .collect(); + free_names.sort(); + for name in free_names { + freevar_cache.insert(name); + } - let info = ir::CodeInfo { - flags, - posonlyarg_count, - arg_count, - kwonlyarg_count, - source_path, - first_line_number, - obj_name, + // Initialize u_metadata fields + let (flags, posonlyarg_count, arg_count, kwonlyarg_count) = match scope_type { + CompilerScope::Module => (bytecode::CodeFlags::empty(), 0, 0, 0), + CompilerScope::Class => (bytecode::CodeFlags::empty(), 0, 0, 0), + CompilerScope::Function | CompilerScope::AsyncFunction | CompilerScope::Lambda => ( + bytecode::CodeFlags::NEW_LOCALS | bytecode::CodeFlags::IS_OPTIMIZED, + 0, // Will be set later in enter_function + 0, // Will be set later in enter_function + 0, // Will be set later in enter_function + ), + CompilerScope::Comprehension => ( + bytecode::CodeFlags::NEW_LOCALS | bytecode::CodeFlags::IS_OPTIMIZED, + 0, + 1, // comprehensions take one argument (.0) + 0, + ), + CompilerScope::TypeParams => ( + bytecode::CodeFlags::NEW_LOCALS | bytecode::CodeFlags::IS_OPTIMIZED, + 0, + 0, + 0, + ), + }; + + // Get private name from parent scope + let private = if !self.code_stack.is_empty() { + self.code_stack.last().unwrap().private.clone() + } else { + None + }; + // Create the new compilation unit + let code_info = ir::CodeInfo { + flags, + source_path: source_path.clone(), + private, blocks: vec![ir::Block::default()], - current_block: ir::BlockIdx(0), - constants: IndexSet::default(), - name_cache: IndexSet::default(), - varname_cache: IndexSet::default(), - cellvar_cache, - freevar_cache, + current_block: BlockIdx::new(0), + metadata: ir::CodeUnitMetadata { + name: name.to_owned(), + qualname: None, // Will be set below + consts: IndexSet::default(), + names: IndexSet::default(), + varnames: varname_cache, + cellvars: cellvar_cache, + freevars: freevar_cache, + fast_hidden: IndexMap::default(), + argcount: arg_count, + posonlyargcount: posonlyarg_count, + kwonlyargcount: kwonlyarg_count, + firstlineno: OneIndexed::new(lineno as usize).unwrap_or(OneIndexed::MIN), + }, + static_attributes: if scope_type == CompilerScope::Class { + Some(IndexSet::default()) + } else { + None + }, + in_inlined_comp: false, + fblock: Vec::with_capacity(MAXBLOCKS), + symbol_table_index: key, + }; + + // Push the old compiler unit on the stack (like PyCapsule) + // This happens before setting qualname + self.code_stack.push(code_info); + + // Set qualname after pushing (uses compiler_set_qualname logic) + if scope_type != CompilerScope::Module { + self.set_qualname(); + } + + // Emit RESUME instruction + let _resume_loc = if scope_type == CompilerScope::Module { + // Module scope starts with lineno 0 + SourceLocation { + line: OneIndexed::MIN, + character_offset: OneIndexed::MIN, + } + } else { + location }; - self.code_stack.push(info); + + // Set the source range for the RESUME instruction + // For now, just use an empty range at the beginning + self.current_source_range = TextRange::default(); + emit!( + self, + Instruction::Resume { + arg: bytecode::ResumeType::AtFuncStart as u32 + } + ); + + if scope_type == CompilerScope::Module { + // This would be loc.lineno = -1 in CPython + // We handle this differently in RustPython + } + + Ok(()) + } + + fn push_output( + &mut self, + flags: bytecode::CodeFlags, + posonlyarg_count: u32, + arg_count: u32, + kwonlyarg_count: u32, + obj_name: String, + ) { + // First push the symbol table + let table = self.push_symbol_table(); + let scope_type = table.typ; + + // The key is the current position in the symbol table stack + let key = self.symbol_table_stack.len() - 1; + + // Get the line number + let lineno = self.get_source_line_number().get(); + + // Call enter_scope which does most of the work + if let Err(e) = self.enter_scope(&obj_name, scope_type, key, lineno.to_u32()) { + // In the current implementation, push_output doesn't return an error, + // so we panic here. This maintains the same behavior. + panic!("enter_scope failed: {e:?}"); + } + + // Override the values that push_output sets explicitly + // enter_scope sets default values based on scope_type, but push_output + // allows callers to specify exact values + if let Some(info) = self.code_stack.last_mut() { + info.flags = flags; + info.metadata.argcount = arg_count; + info.metadata.posonlyargcount = posonlyarg_count; + info.metadata.kwonlyargcount = kwonlyarg_count; + } } - fn pop_code_object(&mut self) -> CodeObject { - let table = self.pop_symbol_table(); - assert!(table.sub_tables.is_empty()); + // compiler_exit_scope + fn exit_scope(&mut self) -> CodeObject { + let _table = self.pop_symbol_table(); + + // Various scopes can have sub_tables: + // - TypeParams scope can have sub_tables (the function body's symbol table) + // - Module scope can have sub_tables (for TypeAlias scopes, nested functions, classes) + // - Function scope can have sub_tables (for nested functions, classes) + // - Class scope can have sub_tables (for nested classes, methods) + let pop = self.code_stack.pop(); let stack_top = compiler_unwrap_option(self, pop); + // No parent scope stack to maintain unwrap_internal(self, stack_top.finalize_code(self.opts.optimize)) } + /// Push a new fblock + // = compiler_push_fblock + fn push_fblock( + &mut self, + fb_type: FBlockType, + fb_block: BlockIdx, + fb_exit: BlockIdx, + ) -> CompileResult<()> { + let code = self.current_code_info(); + if code.fblock.len() >= MAXBLOCKS { + return Err(self.error(CodegenErrorType::SyntaxError( + "too many statically nested blocks".to_owned(), + ))); + } + code.fblock.push(FBlockInfo { + fb_type, + fb_block, + fb_exit, + }); + Ok(()) + } + + /// Pop an fblock + // = compiler_pop_fblock + fn pop_fblock(&mut self, _expected_type: FBlockType) -> FBlockInfo { + let code = self.current_code_info(); + // TODO: Add assertion to check expected type matches + // assert!(matches!(fblock.fb_type, expected_type)); + code.fblock.pop().expect("fblock stack underflow") + } + // could take impl Into>, but everything is borrowed from ast structs; we never // actually have a `String` to pass fn name(&mut self, name: &str) -> bytecode::NameIdx { - self._name_inner(name, |i| &mut i.name_cache) + self._name_inner(name, |i| &mut i.metadata.names) } fn varname(&mut self, name: &str) -> CompileResult { - if Compiler::is_forbidden_arg_name(name) { + if Self::is_forbidden_arg_name(name) { return Err(self.error(CodegenErrorType::SyntaxError(format!( "cannot assign to {name}", )))); } - Ok(self._name_inner(name, |i| &mut i.varname_cache)) + Ok(self._name_inner(name, |i| &mut i.metadata.varnames)) } fn _name_inner( &mut self, @@ -456,6 +918,99 @@ impl Compiler<'_> { .to_u32() } + /// Set the qualified name for the current code object + // = compiler_set_qualname + fn set_qualname(&mut self) -> String { + let qualname = self.make_qualname(); + self.current_code_info().metadata.qualname = Some(qualname.clone()); + qualname + } + fn make_qualname(&mut self) -> String { + let stack_size = self.code_stack.len(); + assert!(stack_size >= 1); + + let current_obj_name = self.current_code_info().metadata.name.clone(); + + // If we're at the module level (stack_size == 1), qualname is just the name + if stack_size <= 1 { + return current_obj_name; + } + + // Check parent scope + let mut parent_idx = stack_size - 2; + let mut parent = &self.code_stack[parent_idx]; + + // If parent is TypeParams scope, look at grandparent + // Check if parent is a type params scope by name pattern + if parent.metadata.name.starts_with(" self.symbol_table_stack.len() { + // We might be in a situation where symbol table isn't pushed yet + // In this case, check the parent symbol table + if let Some(parent_table) = self.symbol_table_stack.last() + && let Some(symbol) = parent_table.lookup(¤t_obj_name) + && symbol.scope == SymbolScope::GlobalExplicit + { + force_global = true; + } + } else if let Some(_current_table) = self.symbol_table_stack.last() { + // Mangle the name if necessary (for private names in classes) + let mangled_name = self.mangle(¤t_obj_name); + + // Look up in parent symbol table to check scope + if self.symbol_table_stack.len() >= 2 { + let parent_table = &self.symbol_table_stack[self.symbol_table_stack.len() - 2]; + if let Some(symbol) = parent_table.lookup(&mangled_name) + && symbol.scope == SymbolScope::GlobalExplicit + { + force_global = true; + } + } + } + + // Build the qualified name + if force_global { + // For global symbols, qualname is just the name + current_obj_name + } else { + // Check parent scope type + let parent_obj_name = &parent.metadata.name; + + // Determine if parent is a function-like scope + let is_function_parent = parent.flags.contains(bytecode::CodeFlags::IS_OPTIMIZED) + && !parent_obj_name.starts_with("<") // Not a special scope like , , etc. + && parent_obj_name != ""; // Not the module scope + + if is_function_parent { + // For functions, append . to parent qualname + // Use parent's qualname if available, otherwise use parent_obj_name + let parent_qualname = parent.metadata.qualname.as_ref().unwrap_or(parent_obj_name); + format!("{parent_qualname}..{current_obj_name}") + } else { + // For classes and other scopes, use parent's qualname directly + // Use parent's qualname if available, otherwise use parent_obj_name + let parent_qualname = parent.metadata.qualname.as_ref().unwrap_or(parent_obj_name); + if parent_qualname == "" { + // Module level, just use the name + current_obj_name + } else { + // Concatenate parent qualname with current name + format!("{parent_qualname}.{current_obj_name}") + } + } + } + } + fn compile_program( &mut self, body: &ModModule, @@ -501,7 +1056,14 @@ impl Compiler<'_> { for statement in body { if let Stmt::Expr(StmtExpr { value, .. }) = &statement { self.compile_expression(value)?; - emit!(self, Instruction::PrintExpr); + emit!( + self, + Instruction::CallIntrinsic1 { + func: bytecode::IntrinsicFunction1::Print + } + ); + + emit!(self, Instruction::PopTop); } else { self.compile_statement(statement)?; } @@ -509,8 +1071,15 @@ impl Compiler<'_> { if let Stmt::Expr(StmtExpr { value, .. }) = &last { self.compile_expression(value)?; - emit!(self, Instruction::Duplicate); - emit!(self, Instruction::PrintExpr); + emit!(self, Instruction::CopyItem { index: 1_u32 }); + emit!( + self, + Instruction::CallIntrinsic1 { + func: bytecode::IntrinsicFunction1::Print + } + ); + + emit!(self, Instruction::PopTop); } else { self.compile_statement(last)?; self.emit_load_const(ConstantData::None); @@ -535,12 +1104,12 @@ impl Compiler<'_> { if let Some(last_statement) = body.last() { match last_statement { Stmt::Expr(_) => { - self.current_block().instructions.pop(); // pop Instruction::Pop + self.current_block().instructions.pop(); // pop Instruction::PopTop } Stmt::FunctionDef(_) | Stmt::ClassDef(_) => { let pop_instructions = self.current_block().instructions.pop(); let store_inst = compiler_unwrap_option(self, pop_instructions); // pop Instruction::Store - emit!(self, Instruction::Duplicate); + emit!(self, Instruction::CopyItem { index: 1_u32 }); self.current_block().instructions.push(store_inst); } _ => self.emit_load_const(ConstantData::None), @@ -579,7 +1148,12 @@ impl Compiler<'_> { } fn mangle<'a>(&self, name: &'a str) -> Cow<'a, str> { - symboltable::mangle_name(self.class_name.as_deref(), name) + // Use private from current code unit for name mangling + let private = self + .code_stack + .last() + .and_then(|info| info.private.as_deref()); + symboltable::mangle_name(private, name) } fn check_forbidden_name(&mut self, name: &str, usage: NameUsage) -> CompileResult<()> { @@ -591,48 +1165,19 @@ impl Compiler<'_> { Err(self.error(CodegenErrorType::SyntaxError(format!("{msg} {name}")))) } + // = compiler_nameop fn compile_name(&mut self, name: &str, usage: NameUsage) -> CompileResult<()> { - let name = self.mangle(name); - - self.check_forbidden_name(&name, usage)?; - - let symbol_table = self.symbol_table_stack.last().unwrap(); - let symbol = unwrap_internal( - self, - symbol_table - .lookup(name.as_ref()) - .ok_or_else(|| InternalError::MissingSymbol(name.to_string())), - ); - let info = self.code_stack.last_mut().unwrap(); - let mut cache = &mut info.name_cache; - enum NameOpType { + enum NameOp { Fast, Global, Deref, - Local, - } - let op_typ = match symbol.scope { - SymbolScope::Local if self.ctx.in_func() => { - cache = &mut info.varname_cache; - NameOpType::Fast - } - SymbolScope::GlobalExplicit => NameOpType::Global, - SymbolScope::GlobalImplicit | SymbolScope::Unknown if self.ctx.in_func() => { - NameOpType::Global - } - SymbolScope::GlobalImplicit | SymbolScope::Unknown => NameOpType::Local, - SymbolScope::Local => NameOpType::Local, - SymbolScope::Free => { - cache = &mut info.freevar_cache; - NameOpType::Deref - } - SymbolScope::Cell => { - cache = &mut info.cellvar_cache; - NameOpType::Deref - } // TODO: is this right? - // SymbolScope::Unknown => NameOpType::Global, - }; + Name, + } + + let name = self.mangle(name); + self.check_forbidden_name(&name, usage)?; + // Special handling for __debug__ if NameUsage::Load == usage && name == "__debug__" { self.emit_load_const(ConstantData::Boolean { value: self.opts.optimize == 0, @@ -640,56 +1185,127 @@ impl Compiler<'_> { return Ok(()); } - let mut idx = cache - .get_index_of(name.as_ref()) - .unwrap_or_else(|| cache.insert_full(name.into_owned()).0); - if let SymbolScope::Free = symbol.scope { - idx += info.cellvar_cache.len(); - } - let op = match op_typ { - NameOpType::Fast => match usage { - NameUsage::Load => Instruction::LoadFast, - NameUsage::Store => Instruction::StoreFast, - NameUsage::Delete => Instruction::DeleteFast, - }, - NameOpType::Global => match usage { - NameUsage::Load => Instruction::LoadGlobal, - NameUsage::Store => Instruction::StoreGlobal, - NameUsage::Delete => Instruction::DeleteGlobal, - }, - NameOpType::Deref => match usage { - NameUsage::Load if !self.ctx.in_func() && self.ctx.in_class => { - Instruction::LoadClassDeref - } - NameUsage::Load => Instruction::LoadDeref, - NameUsage::Store => Instruction::StoreDeref, - NameUsage::Delete => Instruction::DeleteDeref, - }, - NameOpType::Local => match usage { - NameUsage::Load => Instruction::LoadNameAny, - NameUsage::Store => Instruction::StoreLocal, - NameUsage::Delete => Instruction::DeleteLocal, - }, - }; - self.emit_arg(idx.to_u32(), op); + // Determine the operation type based on symbol scope + let is_function_like = self.ctx.in_func(); - Ok(()) - } + // Look up the symbol, handling TypeParams scope specially + let (symbol_scope, _is_typeparams) = { + let current_table = self.current_symbol_table(); + let is_typeparams = current_table.typ == CompilerScope::TypeParams; - fn compile_statement(&mut self, statement: &Stmt) -> CompileResult<()> { - use ruff_python_ast::*; - trace!("Compiling {statement:?}"); - self.set_source_range(statement.range()); + // First try to find in current table + let symbol = current_table.lookup(name.as_ref()); - match &statement { - // we do this here because `from __future__` still executes that `from` statement at runtime, - // we still need to compile the ImportFrom down below - Stmt::ImportFrom(StmtImportFrom { module, names, .. }) - if module.as_ref().map(|id| id.as_str()) == Some("__future__") => - { - self.compile_future_features(names)? + // If not found and we're in TypeParams scope, try parent scope + let symbol = if symbol.is_none() && is_typeparams { + self.symbol_table_stack + .get(self.symbol_table_stack.len() - 2) // Try to get parent index + .expect("Symbol has no parent! This is a compiler bug.") + .lookup(name.as_ref()) + } else { + symbol + }; + + (symbol.map(|s| s.scope), is_typeparams) + }; + + let actual_scope = symbol_scope.ok_or_else(|| { + self.error(CodegenErrorType::SyntaxError(format!( + "The symbol '{name}' must be present in the symbol table" + ))) + })?; + + // Determine operation type based on scope + let op_type = match actual_scope { + SymbolScope::Free => NameOp::Deref, + SymbolScope::Cell => NameOp::Deref, + SymbolScope::Local => { + if is_function_like { + NameOp::Fast + } else { + NameOp::Name + } } - // ignore module-level doc comments + SymbolScope::GlobalImplicit => { + if is_function_like { + NameOp::Global + } else { + NameOp::Name + } + } + SymbolScope::GlobalExplicit => NameOp::Global, + SymbolScope::Unknown => NameOp::Name, + }; + + // Generate appropriate instructions based on operation type + match op_type { + NameOp::Deref => { + let idx = match actual_scope { + SymbolScope::Free => self.get_free_var_index(&name)?, + SymbolScope::Cell => self.get_cell_var_index(&name)?, + _ => unreachable!("Invalid scope for Deref operation"), + }; + + let op = match usage { + NameUsage::Load => { + // Special case for class scope + if self.ctx.in_class && !self.ctx.in_func() { + Instruction::LoadClassDeref + } else { + Instruction::LoadDeref + } + } + NameUsage::Store => Instruction::StoreDeref, + NameUsage::Delete => Instruction::DeleteDeref, + }; + self.emit_arg(idx, op); + } + NameOp::Fast => { + let idx = self.get_local_var_index(&name)?; + let op = match usage { + NameUsage::Load => Instruction::LoadFast, + NameUsage::Store => Instruction::StoreFast, + NameUsage::Delete => Instruction::DeleteFast, + }; + self.emit_arg(idx, op); + } + NameOp::Global => { + let idx = self.get_global_name_index(&name); + let op = match usage { + NameUsage::Load => Instruction::LoadGlobal, + NameUsage::Store => Instruction::StoreGlobal, + NameUsage::Delete => Instruction::DeleteGlobal, + }; + self.emit_arg(idx, op); + } + NameOp::Name => { + let idx = self.get_global_name_index(&name); + let op = match usage { + NameUsage::Load => Instruction::LoadNameAny, + NameUsage::Store => Instruction::StoreLocal, + NameUsage::Delete => Instruction::DeleteLocal, + }; + self.emit_arg(idx, op); + } + } + + Ok(()) + } + + fn compile_statement(&mut self, statement: &Stmt) -> CompileResult<()> { + use ruff_python_ast::*; + trace!("Compiling {statement:?}"); + self.set_source_range(statement.range()); + + match &statement { + // we do this here because `from __future__` still executes that `from` statement at runtime, + // we still need to compile the ImportFrom down below + Stmt::ImportFrom(StmtImportFrom { module, names, .. }) + if module.as_ref().map(|id| id.as_str()) == Some("__future__") => + { + self.compile_future_features(names)? + } + // ignore module-level doc comments Stmt::Expr(StmtExpr { value, .. }) if matches!(&**value, Expr::StringLiteral(..)) && matches!(self.done_with_future_stmts, DoneWithFuture::No) => @@ -747,8 +1363,6 @@ impl Compiler<'_> { .collect() }; - let module_idx = module.as_ref().map(|s| self.name(s.as_str())); - // from .... import (*fromlist) self.emit_load_const(ConstantData::Integer { value: (*level).into(), @@ -756,15 +1370,19 @@ impl Compiler<'_> { self.emit_load_const(ConstantData::Tuple { elements: from_list, }); - if let Some(idx) = module_idx { - emit!(self, Instruction::ImportName { idx }); - } else { - emit!(self, Instruction::ImportNameless); - } + + let module_name = module.as_ref().map_or("", |s| s.as_str()); + let module_idx = self.name(module_name); + emit!(self, Instruction::ImportName { idx: module_idx }); if import_star { // from .... import * - emit!(self, Instruction::ImportStar); + emit!( + self, + Instruction::CallIntrinsic1 { + func: bytecode::IntrinsicFunction1::ImportStar + } + ); } else { // from mod import a, b as c @@ -783,14 +1401,14 @@ impl Compiler<'_> { } // Pop module from stack: - emit!(self, Instruction::Pop); + emit!(self, Instruction::PopTop); } } Stmt::Expr(StmtExpr { value, .. }) => { self.compile_expression(value)?; // Pop result of stack, since we not use it: - emit!(self, Instruction::Pop); + emit!(self, Instruction::PopTop); } Stmt::Global(_) | Stmt::Nonlocal(_) => { // Handled during symbol table construction. @@ -892,7 +1510,7 @@ impl Compiler<'_> { .. }) => { if *is_star { - self.compile_try_star_statement(body, handlers, orelse, finalbody)? + self.compile_try_star_except(body, handlers, orelse, finalbody)? } else { self.compile_try_statement(body, handlers, orelse, finalbody)? } @@ -906,15 +1524,19 @@ impl Compiler<'_> { type_params, is_async, .. - }) => self.compile_function_def( - name.as_str(), - parameters, - body, - decorator_list, - returns.as_deref(), - *is_async, - type_params.as_deref(), - )?, + }) => { + validate_duplicate_params(parameters).map_err(|e| self.error(e))?; + + self.compile_function_def( + name.as_str(), + parameters, + body, + decorator_list, + returns.as_deref(), + *is_async, + type_params.as_deref(), + )? + } Stmt::ClassDef(StmtClassDef { name, body, @@ -956,32 +1578,102 @@ impl Compiler<'_> { self.switch_to_block(after_block); } } - Stmt::Break(_) => match self.ctx.loop_data { - Some((_, end)) => { - emit!(self, Instruction::Break { target: end }); - } - None => { - return Err( - self.error_ranged(CodegenErrorType::InvalidBreak, statement.range()) - ); - } - }, - Stmt::Continue(_) => match self.ctx.loop_data { - Some((start, _)) => { - emit!(self, Instruction::Continue { target: start }); + Stmt::Break(_) => { + // Find the innermost loop in fblock stack + // Error if we encounter ExceptionGroupHandler before finding a loop + let found_loop = { + let code = self.current_code_info(); + let mut result = Ok(None); + for i in (0..code.fblock.len()).rev() { + match code.fblock[i].fb_type { + FBlockType::WhileLoop | FBlockType::ForLoop => { + result = Ok(Some(code.fblock[i].fb_exit)); + break; + } + FBlockType::ExceptionGroupHandler => { + result = Err(()); + break; + } + _ => continue, + } + } + result + }; + + match found_loop { + Ok(Some(exit_block)) => { + emit!(self, Instruction::Break { target: exit_block }); + } + Ok(None) => { + return Err( + self.error_ranged(CodegenErrorType::InvalidBreak, statement.range()) + ); + } + Err(()) => { + return Err(self.error_ranged( + CodegenErrorType::BreakContinueReturnInExceptStar, + statement.range(), + )); + } } - None => { - return Err( - self.error_ranged(CodegenErrorType::InvalidContinue, statement.range()) - ); + } + Stmt::Continue(_) => { + // Find the innermost loop in fblock stack + // Error if we encounter ExceptionGroupHandler before finding a loop + let found_loop = { + let code = self.current_code_info(); + let mut result = Ok(None); + for i in (0..code.fblock.len()).rev() { + match code.fblock[i].fb_type { + FBlockType::WhileLoop | FBlockType::ForLoop => { + result = Ok(Some(code.fblock[i].fb_block)); + break; + } + FBlockType::ExceptionGroupHandler => { + result = Err(()); + break; + } + _ => continue, + } + } + result + }; + + match found_loop { + Ok(Some(loop_block)) => { + emit!(self, Instruction::Continue { target: loop_block }); + } + Ok(None) => { + return Err( + self.error_ranged(CodegenErrorType::InvalidContinue, statement.range()) + ); + } + Err(()) => { + return Err(self.error_ranged( + CodegenErrorType::BreakContinueReturnInExceptStar, + statement.range(), + )); + } } - }, + } Stmt::Return(StmtReturn { value, .. }) => { if !self.ctx.in_func() { return Err( self.error_ranged(CodegenErrorType::InvalidReturn, statement.range()) ); } + // Check if we're inside an except* block in the current function + { + let code = self.current_code_info(); + for block in code.fblock.iter().rev() { + if matches!(block.fb_type, FBlockType::ExceptionGroupHandler) { + return Err(self.error_ranged( + CodegenErrorType::BreakContinueReturnInExceptStar, + statement.range(), + )); + } + } + } match value { Some(v) => { if self.ctx.func == FunctionContext::AsyncFunction @@ -1008,7 +1700,7 @@ impl Compiler<'_> { for (i, target) in targets.iter().enumerate() { if i + 1 != targets.len() { - emit!(self, Instruction::Duplicate); + emit!(self, Instruction::CopyItem { index: 1_u32 }); } self.compile_store(target)?; } @@ -1047,33 +1739,43 @@ impl Compiler<'_> { // For PEP 695 syntax, we need to compile type_params first // so that they're available when compiling the value expression + // Push name first + self.emit_load_const(ConstantData::Str { + value: name_string.clone().into(), + }); + if let Some(type_params) = type_params { + // For TypeAlias, we need to use push_symbol_table to properly handle the TypeAlias scope self.push_symbol_table(); - // Compile type params first to define T1, T2, etc. + // Compile type params and push to stack self.compile_type_params(type_params)?; - // Stack now has type_params tuple at top + // Stack now has [name, type_params_tuple] // Compile value expression (can now see T1, T2) self.compile_expression(value)?; - // Stack: [type_params_tuple, value] - - // We need [value, type_params_tuple] for TypeAlias instruction - emit!(self, Instruction::Rotate2); + // Stack: [name, type_params_tuple, value] + // Pop the TypeAlias scope self.pop_symbol_table(); } else { - // No type params - push value first, then None (not empty tuple) - self.compile_expression(value)?; - // Push None for type_params (matching CPython) + // Push None for type_params self.emit_load_const(ConstantData::None); + // Stack: [name, None] + + // Compile value expression + self.compile_expression(value)?; + // Stack: [name, None, value] } - // Push name last - self.emit_load_const(ConstantData::Str { - value: name_string.clone().into(), - }); - emit!(self, Instruction::TypeAlias); + // Build tuple of 3 elements and call intrinsic + emit!(self, Instruction::BuildTuple { size: 3 }); + emit!( + self, + Instruction::CallIntrinsic1 { + func: bytecode::IntrinsicFunction1::TypeAlias + } + ); self.store_name(&name_string)?; } Stmt::IpyEscapeCommand(_) => todo!(), @@ -1091,10 +1793,10 @@ impl Compiler<'_> { let idx = self.name(attr.as_str()); emit!(self, Instruction::DeleteAttr { idx }); } - Expr::Subscript(ExprSubscript { value, slice, .. }) => { - self.compile_expression(value)?; - self.compile_expression(slice)?; - emit!(self, Instruction::DeleteSubscript); + Expr::Subscript(ExprSubscript { + value, slice, ctx, .. + }) => { + self.compile_subscript(value, slice, *ctx)?; } Expr::Tuple(ExprTuple { elts, .. }) | Expr::List(ExprList { elts, .. }) => { for element in elts { @@ -1109,26 +1811,7 @@ impl Compiler<'_> { Ok(()) } - fn enter_function( - &mut self, - name: &str, - parameters: &Parameters, - ) -> CompileResult { - let defaults: Vec<_> = std::iter::empty() - .chain(¶meters.posonlyargs) - .chain(¶meters.args) - .filter_map(|x| x.default.as_deref()) - .collect(); - let have_defaults = !defaults.is_empty(); - if have_defaults { - // Construct a tuple: - let size = defaults.len().to_u32(); - for element in &defaults { - self.compile_expression(element)?; - } - emit!(self, Instruction::BuildTuple { size }); - } - + fn enter_function(&mut self, name: &str, parameters: &Parameters) -> CompileResult<()> { // TODO: partition_in_place let mut kw_without_defaults = vec![]; let mut kw_with_defaults = vec![]; @@ -1140,31 +1823,6 @@ impl Compiler<'_> { } } - // let (kw_without_defaults, kw_with_defaults) = args.split_kwonlyargs(); - if !kw_with_defaults.is_empty() { - let default_kw_count = kw_with_defaults.len(); - for (arg, default) in kw_with_defaults.iter() { - self.emit_load_const(ConstantData::Str { - value: arg.name.as_str().into(), - }); - self.compile_expression(default)?; - } - emit!( - self, - Instruction::BuildMap { - size: default_kw_count.to_u32(), - } - ); - } - - let mut func_flags = bytecode::MakeFunctionFlags::empty(); - if have_defaults { - func_flags |= bytecode::MakeFunctionFlags::DEFAULTS; - } - if !kw_with_defaults.is_empty() { - func_flags |= bytecode::MakeFunctionFlags::KW_ONLY_DEFAULTS; - } - self.push_output( bytecode::CodeFlags::NEW_LOCALS | bytecode::CodeFlags::IS_OPTIMIZED, parameters.posonlyargs.len().to_u32(), @@ -1192,7 +1850,7 @@ impl Compiler<'_> { self.varname(name.name.as_str())?; } - Ok(func_flags) + Ok(()) } fn prepare_decorators(&mut self, decorator_list: &[Decorator]) -> CompileResult<()> { @@ -1209,6 +1867,49 @@ impl Compiler<'_> { } } + /// Compile type parameter bound or default in a separate scope and return closure + fn compile_type_param_bound_or_default( + &mut self, + expr: &Expr, + name: &str, + allow_starred: bool, + ) -> CompileResult<()> { + // Push the next symbol table onto the stack + self.push_symbol_table(); + + // Get the current symbol table + let key = self.symbol_table_stack.len() - 1; + let lineno = expr.range().start().to_u32(); + + // Enter scope with the type parameter name + self.enter_scope(name, CompilerScope::TypeParams, key, lineno)?; + + // Compile the expression + if allow_starred && matches!(expr, Expr::Starred(_)) { + if let Expr::Starred(starred) = expr { + self.compile_expression(&starred.value)?; + emit!(self, Instruction::UnpackSequence { size: 1 }); + } + } else { + self.compile_expression(expr)?; + } + + // Return value + emit!(self, Instruction::ReturnValue); + + // Exit scope and create closure + let code = self.exit_scope(); + // Note: exit_scope already calls pop_symbol_table, so we don't need to call it again + + // Create type params function with closure + self.make_closure(code, bytecode::MakeFunctionFlags::empty())?; + + // Call the function immediately + emit!(self, Instruction::CallFunctionPositional { nargs: 0 }); + + Ok(()) + } + /// Store each type parameter so it is accessible to the current scope, and leave a tuple of /// all the type parameters on the stack. fn compile_type_params(&mut self, type_params: &TypeParams) -> CompileResult<()> { @@ -1221,24 +1922,37 @@ impl Compiler<'_> { default, .. }) => { + self.emit_load_const(ConstantData::Str { + value: name.as_str().into(), + }); + if let Some(expr) = &bound { - self.compile_expression(expr)?; - self.emit_load_const(ConstantData::Str { - value: name.as_str().into(), - }); - emit!(self, Instruction::TypeVarWithBound); + let scope_name = if expr.is_tuple_expr() { + format!("") + } else { + format!("") + }; + self.compile_type_param_bound_or_default(expr, &scope_name, false)?; + + let intrinsic = if expr.is_tuple_expr() { + bytecode::IntrinsicFunction2::TypeVarWithConstraint + } else { + bytecode::IntrinsicFunction2::TypeVarWithBound + }; + emit!(self, Instruction::CallIntrinsic2 { func: intrinsic }); } else { - self.emit_load_const(ConstantData::Str { - value: name.as_str().into(), - }); - emit!(self, Instruction::TypeVar); + emit!( + self, + Instruction::CallIntrinsic1 { + func: bytecode::IntrinsicFunction1::TypeVar + } + ); } // Handle default value if present (PEP 695) if let Some(default_expr) = default { - // Compile the default expression - self.compile_expression(default_expr)?; - + let scope_name = format!(""); + self.compile_type_param_bound_or_default(default_expr, &scope_name, false)?; emit!( self, Instruction::CallIntrinsic2 { @@ -1247,20 +1961,24 @@ impl Compiler<'_> { ); } - emit!(self, Instruction::Duplicate); + emit!(self, Instruction::CopyItem { index: 1_u32 }); self.store_name(name.as_ref())?; } TypeParam::ParamSpec(TypeParamParamSpec { name, default, .. }) => { self.emit_load_const(ConstantData::Str { value: name.as_str().into(), }); - emit!(self, Instruction::ParamSpec); + emit!( + self, + Instruction::CallIntrinsic1 { + func: bytecode::IntrinsicFunction1::ParamSpec + } + ); // Handle default value if present (PEP 695) if let Some(default_expr) = default { - // Compile the default expression - self.compile_expression(default_expr)?; - + let scope_name = format!(""); + self.compile_type_param_bound_or_default(default_expr, &scope_name, false)?; emit!( self, Instruction::CallIntrinsic2 { @@ -1269,21 +1987,25 @@ impl Compiler<'_> { ); } - emit!(self, Instruction::Duplicate); + emit!(self, Instruction::CopyItem { index: 1_u32 }); self.store_name(name.as_ref())?; } TypeParam::TypeVarTuple(TypeParamTypeVarTuple { name, default, .. }) => { self.emit_load_const(ConstantData::Str { value: name.as_str().into(), }); - emit!(self, Instruction::TypeVarTuple); + emit!( + self, + Instruction::CallIntrinsic1 { + func: bytecode::IntrinsicFunction1::TypeVarTuple + } + ); // Handle default value if present (PEP 695) if let Some(default_expr) = default { - // Compile the default expression - self.compile_expression(default_expr)?; - - // Handle starred expression (*default) + // TypeVarTuple allows starred expressions + let scope_name = format!(""); + self.compile_type_param_bound_or_default(default_expr, &scope_name, true)?; emit!( self, Instruction::CallIntrinsic2 { @@ -1292,7 +2014,7 @@ impl Compiler<'_> { ); } - emit!(self, Instruction::Duplicate); + emit!(self, Instruction::CopyItem { index: 1_u32 }); self.store_name(name.as_ref())?; } }; @@ -1352,36 +2074,23 @@ impl Compiler<'_> { // check if this handler can handle the exception: if let Some(exc_type) = type_ { // Duplicate exception for test: - emit!(self, Instruction::Duplicate); + emit!(self, Instruction::CopyItem { index: 1_u32 }); // Check exception type: self.compile_expression(exc_type)?; - emit!( - self, - Instruction::TestOperation { - op: bytecode::TestOperator::ExceptionMatch, - } - ); - - // We cannot handle this exception type: - emit!( - self, - Instruction::JumpIfFalse { - target: next_handler, - } - ); + emit!(self, Instruction::JumpIfNotExcMatch(next_handler)); // We have a match, store in name (except x as y) if let Some(alias) = name { self.store_name(alias.as_str())? } else { // Drop exception from top of stack: - emit!(self, Instruction::Pop); + emit!(self, Instruction::PopTop); } } else { // Catch all! // Drop exception from top of stack: - emit!(self, Instruction::Pop); + emit!(self, Instruction::PopTop); } // Handler code: @@ -1390,7 +2099,7 @@ impl Compiler<'_> { // Delete the exception variable if it was bound if let Some(alias) = name { - // Set the variable to None before deleting (as CPython does) + // Set the variable to None before deleting self.emit_load_const(ConstantData::None); self.store_name(alias.as_str())?; self.compile_name(alias.as_str(), NameUsage::Delete)?; @@ -1444,46 +2153,357 @@ impl Compiler<'_> { Ok(()) } - fn compile_try_star_statement( + fn compile_try_star_except( &mut self, - _body: &[Stmt], - _handlers: &[ExceptHandler], - _orelse: &[Stmt], - _finalbody: &[Stmt], + body: &[Stmt], + handlers: &[ExceptHandler], + orelse: &[Stmt], + finalbody: &[Stmt], ) -> CompileResult<()> { - Err(self.error(CodegenErrorType::NotImplementedYet)) + // Simplified except* implementation using PrepReraiseStar intrinsic + // Stack layout during handler processing: [orig, list, rest] + let handler_block = self.new_block(); + let finally_block = self.new_block(); + let else_block = self.new_block(); + let end_block = self.new_block(); + let reraise_star_block = self.new_block(); + let reraise_block = self.new_block(); + + if !finalbody.is_empty() { + emit!( + self, + Instruction::SetupFinally { + handler: finally_block, + } + ); + } + + emit!( + self, + Instruction::SetupExcept { + handler: handler_block, + } + ); + self.compile_statements(body)?; + emit!(self, Instruction::PopBlock); + emit!(self, Instruction::Jump { target: else_block }); + + // Exception handler entry + self.switch_to_block(handler_block); + // Stack: [exc] + + // Create list for tracking exception results and copy orig + emit!(self, Instruction::BuildList { size: 0 }); + // Stack: [exc, []] + // CopyItem is 1-indexed: CopyItem(1)=TOS, CopyItem(2)=second from top + // With stack [exc, []], CopyItem(2) copies exc + emit!(self, Instruction::CopyItem { index: 2 }); + // Stack: [exc, [], exc_copy] + + // Now stack is: [orig, list, rest] + + let n = handlers.len(); + for (i, handler) in handlers.iter().enumerate() { + let ExceptHandler::ExceptHandler(ExceptHandlerExceptHandler { + type_, name, body, .. + }) = handler; + + let no_match_block = self.new_block(); + let next_block = self.new_block(); + + // Compile exception type + if let Some(exc_type) = type_ { + // Check for unparenthesized tuple + if let Expr::Tuple(ExprTuple { elts, range, .. }) = exc_type.as_ref() + && let Some(first) = elts.first() + && range.start().to_u32() == first.range().start().to_u32() + { + return Err(self.error(CodegenErrorType::SyntaxError( + "multiple exception types must be parenthesized".to_owned(), + ))); + } + self.compile_expression(exc_type)?; + } else { + return Err(self.error(CodegenErrorType::SyntaxError( + "except* must specify an exception type".to_owned(), + ))); + } + // Stack: [orig, list, rest, type] + + emit!(self, Instruction::CheckEgMatch); + // Stack: [orig, list, new_rest, match] + + // Check if match is not None (use identity check, not truthiness) + // CopyItem is 1-indexed: CopyItem(1) = TOS, CopyItem(2) = second from top + emit!(self, Instruction::CopyItem { index: 1 }); + self.emit_load_const(ConstantData::None); + emit!(self, Instruction::IsOp(bytecode::Invert::No)); // is None? + emit!( + self, + Instruction::PopJumpIfTrue { + target: no_match_block + } + ); + + // Handler matched + // Stack: [orig, list, new_rest, match] + let handler_except_block = self.new_block(); + let handler_done_block = self.new_block(); + + // Set matched exception as current exception for bare 'raise' + emit!(self, Instruction::SetExcInfo); + + // Store match to name if provided + if let Some(alias) = name { + // CopyItem(1) copies TOS (match) + emit!(self, Instruction::CopyItem { index: 1 }); + self.store_name(alias.as_str())?; + } + // Stack: [orig, list, new_rest, match] + + // Setup exception handler to catch 'raise' in handler body + emit!( + self, + Instruction::SetupExcept { + handler: handler_except_block, + } + ); + + // Push fblock to disallow break/continue/return in except* handler + self.push_fblock( + FBlockType::ExceptionGroupHandler, + handler_done_block, + end_block, + )?; + + // Execute handler body + self.compile_statements(body)?; + + // Handler body completed normally (didn't raise) + self.pop_fblock(FBlockType::ExceptionGroupHandler); + emit!(self, Instruction::PopBlock); + + // Cleanup name binding + if let Some(alias) = name { + self.emit_load_const(ConstantData::None); + self.store_name(alias.as_str())?; + self.compile_name(alias.as_str(), NameUsage::Delete)?; + } + + // Stack: [orig, list, new_rest, match] + // Pop match (handler consumed it) + emit!(self, Instruction::PopTop); + // Stack: [orig, list, new_rest] + + // Append None to list (exception was consumed, not reraised) + self.emit_load_const(ConstantData::None); + // Stack: [orig, list, new_rest, None] + emit!(self, Instruction::ListAppend { i: 1 }); + // Stack: [orig, list, new_rest] + + emit!( + self, + Instruction::Jump { + target: handler_done_block + } + ); + + // Handler raised an exception (bare 'raise' or other) + self.switch_to_block(handler_except_block); + // Stack: [orig, list, new_rest, match, raised_exc] + + // Cleanup name binding + if let Some(alias) = name { + self.emit_load_const(ConstantData::None); + self.store_name(alias.as_str())?; + self.compile_name(alias.as_str(), NameUsage::Delete)?; + } + + // Append raised_exc to list (the actual exception that was raised) + // Stack: [orig, list, new_rest, match, raised_exc] + // ListAppend(2): pop raised_exc, then append to list at stack[4-2-1]=stack[1] + emit!(self, Instruction::ListAppend { i: 2 }); + // Stack: [orig, list, new_rest, match] + + // Pop match (no longer needed) + emit!(self, Instruction::PopTop); + // Stack: [orig, list, new_rest] + + self.switch_to_block(handler_done_block); + // Stack: [orig, list, new_rest] + + emit!(self, Instruction::Jump { target: next_block }); + + // No match - pop match (None), keep rest unchanged + self.switch_to_block(no_match_block); + emit!(self, Instruction::PopTop); // pop match (None) + // Stack: [orig, list, new_rest] + + self.switch_to_block(next_block); + // Stack: [orig, list, rest] (rest may have been updated) + + // After last handler, append remaining rest to list + if i == n - 1 { + // Stack: [orig, list, rest] + // ListAppend(i) pops TOS, then accesses stack[len - i - 1] + // After pop, stack is [orig, list], len=2 + // We want list at index 1, so 2 - i - 1 = 1, i = 0 + emit!(self, Instruction::ListAppend { i: 0 }); + // Stack: [orig, list] + emit!( + self, + Instruction::Jump { + target: reraise_star_block + } + ); + } + } + + // Reraise star block + self.switch_to_block(reraise_star_block); + // Stack: [orig, list] + emit!( + self, + Instruction::CallIntrinsic2 { + func: bytecode::IntrinsicFunction2::PrepReraiseStar + } + ); + // Stack: [result] (exception to reraise or None) + + // Check if result is not None (use identity check, not truthiness) + emit!(self, Instruction::CopyItem { index: 1 }); + self.emit_load_const(ConstantData::None); + emit!(self, Instruction::IsOp(bytecode::Invert::Yes)); // is not None? + emit!( + self, + Instruction::PopJumpIfTrue { + target: reraise_block + } + ); + + // Nothing to reraise + emit!(self, Instruction::PopTop); + emit!(self, Instruction::PopException); + + if !finalbody.is_empty() { + emit!(self, Instruction::PopBlock); + emit!(self, Instruction::EnterFinally); + } + + emit!(self, Instruction::Jump { target: end_block }); + + // Reraise the result + self.switch_to_block(reraise_block); + // Don't call PopException before Raise - it truncates the stack and removes the result. + // When Raise is executed, the exception propagates through unwind_blocks which + // will properly handle the ExceptHandler block. + emit!( + self, + Instruction::Raise { + kind: bytecode::RaiseKind::Raise + } + ); + + // try-else path + self.switch_to_block(else_block); + self.compile_statements(orelse)?; + + if !finalbody.is_empty() { + emit!(self, Instruction::PopBlock); + emit!(self, Instruction::EnterFinally); + } + + emit!(self, Instruction::Jump { target: end_block }); + + self.switch_to_block(end_block); + if !finalbody.is_empty() { + self.switch_to_block(finally_block); + self.compile_statements(finalbody)?; + emit!(self, Instruction::EndFinally); + } + + Ok(()) } fn is_forbidden_arg_name(name: &str) -> bool { is_forbidden_name(name) } - #[allow(clippy::too_many_arguments)] - fn compile_function_def( + /// Compile default arguments + // = compiler_default_arguments + fn compile_default_arguments( + &mut self, + parameters: &Parameters, + ) -> CompileResult { + let mut funcflags = bytecode::MakeFunctionFlags::empty(); + + // Handle positional defaults + let defaults: Vec<_> = std::iter::empty() + .chain(¶meters.posonlyargs) + .chain(¶meters.args) + .filter_map(|x| x.default.as_deref()) + .collect(); + + if !defaults.is_empty() { + // Compile defaults and build tuple + for default in &defaults { + self.compile_expression(default)?; + } + emit!( + self, + Instruction::BuildTuple { + size: defaults.len().to_u32() + } + ); + funcflags |= bytecode::MakeFunctionFlags::DEFAULTS; + } + + // Handle keyword-only defaults + let mut kw_with_defaults = vec![]; + for kwonlyarg in ¶meters.kwonlyargs { + if let Some(default) = &kwonlyarg.default { + kw_with_defaults.push((&kwonlyarg.parameter, default)); + } + } + + if !kw_with_defaults.is_empty() { + // Compile kwdefaults and build dict + for (arg, default) in &kw_with_defaults { + self.emit_load_const(ConstantData::Str { + value: arg.name.as_str().into(), + }); + self.compile_expression(default)?; + } + emit!( + self, + Instruction::BuildMap { + size: kw_with_defaults.len().to_u32(), + } + ); + funcflags |= bytecode::MakeFunctionFlags::KW_ONLY_DEFAULTS; + } + + Ok(funcflags) + } + + /// Compile function body and create function object + // = compiler_function_body + fn compile_function_body( &mut self, name: &str, parameters: &Parameters, body: &[Stmt], - decorator_list: &[Decorator], - returns: Option<&Expr>, // TODO: use type hint somehow.. is_async: bool, - type_params: Option<&TypeParams>, + funcflags: bytecode::MakeFunctionFlags, ) -> CompileResult<()> { - self.prepare_decorators(decorator_list)?; - - // If there are type params, we need to push a special symbol table just for them - if type_params.is_some() { - self.push_symbol_table(); - } - - let mut func_flags = self.enter_function(name, parameters)?; + // Always enter function scope + self.enter_function(name, parameters)?; self.current_code_info() .flags .set(bytecode::CodeFlags::IS_COROUTINE, is_async); - // remember to restore self.ctx.in_loop to the original after the function is compiled + // Set up context let prev_ctx = self.ctx; - self.ctx = CompileContext { loop_data: None, in_class: prev_ctx.in_class, @@ -1494,53 +2514,58 @@ impl Compiler<'_> { }, }; - self.push_qualified_path(name); - let qualified_name = self.qualified_path.join("."); - self.push_qualified_path(""); + // Set qualname + self.set_qualname(); + // Handle docstring let (doc_str, body) = split_doc(body, &self.opts); - self.current_code_info() - .constants + .metadata + .consts .insert_full(ConstantData::None); + // Compile body statements self.compile_statements(body)?; - // Emit None at end: + // Emit None at end if needed match body.last() { - Some(Stmt::Return(_)) => { - // the last instruction is a ReturnValue already, we don't need to emit it - } + Some(Stmt::Return(_)) => {} _ => { self.emit_return_const(ConstantData::None); } } - let code = self.pop_code_object(); - self.qualified_path.pop(); - self.qualified_path.pop(); + // Exit scope and create function object + let code = self.exit_scope(); self.ctx = prev_ctx; - // Prepare generic type parameters: - if let Some(type_params) = type_params { - self.compile_type_params(type_params)?; - func_flags |= bytecode::MakeFunctionFlags::TYPE_PARAMS; - } - - // Prepare type annotations: - let mut num_annotations = 0; + // Create function object with closure + self.make_closure(code, funcflags)?; - // Return annotation: - if let Some(annotation) = returns { - // key: + // Handle docstring if present + if let Some(doc) = doc_str { + emit!(self, Instruction::CopyItem { index: 1_u32 }); self.emit_load_const(ConstantData::Str { - value: "return".into(), + value: doc.to_string().into(), }); - // value: - self.compile_annotation(annotation)?; - num_annotations += 1; + emit!(self, Instruction::Swap { index: 2 }); + let doc_attr = self.name("__doc__"); + emit!(self, Instruction::StoreAttr { idx: doc_attr }); } + Ok(()) + } + + /// Compile function annotations + // = compiler_visit_annotations + fn visit_annotations( + &mut self, + parameters: &Parameters, + returns: Option<&Expr>, + ) -> CompileResult { + let mut num_annotations = 0; + + // Handle parameter annotations let parameters_iter = std::iter::empty() .chain(¶meters.posonlyargs) .chain(¶meters.args) @@ -1548,6 +2573,7 @@ impl Compiler<'_> { .map(|x| &x.parameter) .chain(parameters.vararg.as_deref()) .chain(parameters.kwarg.as_deref()); + for param in parameters_iter { if let Some(annotation) = ¶m.annotation { self.emit_load_const(ConstantData::Str { @@ -1558,85 +2584,324 @@ impl Compiler<'_> { } } - if num_annotations > 0 { - func_flags |= bytecode::MakeFunctionFlags::ANNOTATIONS; + // Handle return annotation last + if let Some(annotation) = returns { + self.emit_load_const(ConstantData::Str { + value: "return".into(), + }); + self.compile_annotation(annotation)?; + num_annotations += 1; + } + + Ok(num_annotations) + } + + // = compiler_function + #[allow(clippy::too_many_arguments)] + fn compile_function_def( + &mut self, + name: &str, + parameters: &Parameters, + body: &[Stmt], + decorator_list: &[Decorator], + returns: Option<&Expr>, // TODO: use type hint somehow.. + is_async: bool, + type_params: Option<&TypeParams>, + ) -> CompileResult<()> { + self.prepare_decorators(decorator_list)?; + + // compile defaults and return funcflags + let funcflags = self.compile_default_arguments(parameters)?; + + let is_generic = type_params.is_some(); + let mut num_typeparam_args = 0; + + if is_generic { + // Count args to pass to type params scope + if funcflags.contains(bytecode::MakeFunctionFlags::DEFAULTS) { + num_typeparam_args += 1; + } + if funcflags.contains(bytecode::MakeFunctionFlags::KW_ONLY_DEFAULTS) { + num_typeparam_args += 1; + } + + // SWAP if we have both + if num_typeparam_args == 2 { + emit!(self, Instruction::Swap { index: 2 }); + } + + // Enter type params scope + let type_params_name = format!(""); + self.push_output( + bytecode::CodeFlags::IS_OPTIMIZED | bytecode::CodeFlags::NEW_LOCALS, + 0, + num_typeparam_args as u32, + 0, + type_params_name, + ); + + // Add parameter names to varnames for the type params scope + // These will be passed as arguments when the closure is called + let current_info = self.current_code_info(); + if funcflags.contains(bytecode::MakeFunctionFlags::DEFAULTS) { + current_info + .metadata + .varnames + .insert(".defaults".to_owned()); + } + if funcflags.contains(bytecode::MakeFunctionFlags::KW_ONLY_DEFAULTS) { + current_info + .metadata + .varnames + .insert(".kwdefaults".to_owned()); + } + + // Compile type parameters + self.compile_type_params(type_params.unwrap())?; + + // Load defaults/kwdefaults with LOAD_FAST + for i in 0..num_typeparam_args { + emit!(self, Instruction::LoadFast(i as u32)); + } + } + + // Compile annotations + let mut annotations_flag = bytecode::MakeFunctionFlags::empty(); + let num_annotations = self.visit_annotations(parameters, returns)?; + if num_annotations > 0 { + annotations_flag = bytecode::MakeFunctionFlags::ANNOTATIONS; + emit!( + self, + Instruction::BuildMap { + size: num_annotations, + } + ); + } + + // Compile function body + let final_funcflags = funcflags | annotations_flag; + self.compile_function_body(name, parameters, body, is_async, final_funcflags)?; + + // Handle type params if present + if is_generic { + // SWAP to get function on top + // Stack: [type_params_tuple, function] -> [function, type_params_tuple] + emit!(self, Instruction::Swap { index: 2 }); + + // Call INTRINSIC_SET_FUNCTION_TYPE_PARAMS + emit!( + self, + Instruction::CallIntrinsic2 { + func: bytecode::IntrinsicFunction2::SetFunctionTypeParams, + } + ); + + // Return the function object from type params scope + emit!(self, Instruction::ReturnValue); + + // Set argcount for type params scope + self.current_code_info().metadata.argcount = num_typeparam_args as u32; + + // Exit type params scope and create closure + let type_params_code = self.exit_scope(); + + // Make closure for type params code + self.make_closure(type_params_code, bytecode::MakeFunctionFlags::empty())?; + + // Call the closure + if num_typeparam_args > 0 { + emit!( + self, + Instruction::Swap { + index: (num_typeparam_args + 1) as u32 + } + ); + emit!( + self, + Instruction::CallFunctionPositional { + nargs: num_typeparam_args as u32 + } + ); + } else { + // No arguments, just call the closure + emit!(self, Instruction::CallFunctionPositional { nargs: 0 }); + } + } + + // Apply decorators + self.apply_decorators(decorator_list); + + // Store the function + self.store_name(name)?; + + Ok(()) + } + + /// Determines if a variable should be CELL or FREE type + // = get_ref_type + fn get_ref_type(&self, name: &str) -> Result { + // Special handling for __class__ and __classdict__ in class scope + if self.ctx.in_class && (name == "__class__" || name == "__classdict__") { + return Ok(SymbolScope::Cell); + } + + let table = self.symbol_table_stack.last().unwrap(); + match table.lookup(name) { + Some(symbol) => match symbol.scope { + SymbolScope::Cell => Ok(SymbolScope::Cell), + SymbolScope::Free => Ok(SymbolScope::Free), + _ if symbol.flags.contains(SymbolFlags::FREE_CLASS) => Ok(SymbolScope::Free), + _ => Err(CodegenErrorType::SyntaxError(format!( + "get_ref_type: invalid scope for '{name}'" + ))), + }, + None => Err(CodegenErrorType::SyntaxError(format!( + "get_ref_type: cannot find symbol '{name}'" + ))), + } + } + + /// Loads closure variables if needed and creates a function object + // = compiler_make_closure + fn make_closure( + &mut self, + code: CodeObject, + flags: bytecode::MakeFunctionFlags, + ) -> CompileResult<()> { + // Handle free variables (closure) + let has_freevars = !code.freevars.is_empty(); + if has_freevars { + // Build closure tuple by loading free variables + + for var in &code.freevars { + // Special case: If a class contains a method with a + // free variable that has the same name as a method, + // the name will be considered free *and* local in the + // class. It should be handled by the closure, as + // well as by the normal name lookup logic. + + // Get reference type using our get_ref_type function + let ref_type = self.get_ref_type(var).map_err(|e| self.error(e))?; + + // Get parent code info + let parent_code = self.code_stack.last().unwrap(); + let cellvars_len = parent_code.metadata.cellvars.len(); + + // Look up the variable index based on reference type + let idx = match ref_type { + SymbolScope::Cell => parent_code + .metadata + .cellvars + .get_index_of(var) + .or_else(|| { + parent_code + .metadata + .freevars + .get_index_of(var) + .map(|i| i + cellvars_len) + }) + .ok_or_else(|| { + self.error(CodegenErrorType::SyntaxError(format!( + "compiler_make_closure: cannot find '{var}' in parent vars", + ))) + })?, + SymbolScope::Free => parent_code + .metadata + .freevars + .get_index_of(var) + .map(|i| i + cellvars_len) + .or_else(|| parent_code.metadata.cellvars.get_index_of(var)) + .ok_or_else(|| { + self.error(CodegenErrorType::SyntaxError(format!( + "compiler_make_closure: cannot find '{var}' in parent vars", + ))) + })?, + _ => { + return Err(self.error(CodegenErrorType::SyntaxError(format!( + "compiler_make_closure: unexpected ref_type {ref_type:?} for '{var}'", + )))); + } + }; + + emit!(self, Instruction::LoadClosure(idx.to_u32())); + } + + // Build tuple of closure variables emit!( self, - Instruction::BuildMap { - size: num_annotations, + Instruction::BuildTuple { + size: code.freevars.len().to_u32(), } ); } - if self.build_closure(&code) { - func_flags |= bytecode::MakeFunctionFlags::CLOSURE; - } - - // Pop the special type params symbol table - if type_params.is_some() { - self.pop_symbol_table(); - } - + // load code object and create function self.emit_load_const(ConstantData::Code { code: Box::new(code), }); - self.emit_load_const(ConstantData::Str { - value: qualified_name.into(), - }); - // Turn code object into function object: - emit!(self, Instruction::MakeFunction(func_flags)); + // Create function with no flags + emit!(self, Instruction::MakeFunction); - if let Some(value) = doc_str { - emit!(self, Instruction::Duplicate); - self.emit_load_const(ConstantData::Str { - value: value.into(), - }); - emit!(self, Instruction::Rotate2); - let doc = self.name("__doc__"); - emit!(self, Instruction::StoreAttr { idx: doc }); + // Now set attributes one by one using SET_FUNCTION_ATTRIBUTE + // Note: The order matters! Values must be on stack before calling SET_FUNCTION_ATTRIBUTE + + // Set closure if needed + if has_freevars { + // Closure tuple is already on stack + emit!( + self, + Instruction::SetFunctionAttribute { + attr: bytecode::MakeFunctionFlags::CLOSURE + } + ); } - self.apply_decorators(decorator_list); + // Set annotations if present + if flags.contains(bytecode::MakeFunctionFlags::ANNOTATIONS) { + // Annotations dict is already on stack + emit!( + self, + Instruction::SetFunctionAttribute { + attr: bytecode::MakeFunctionFlags::ANNOTATIONS + } + ); + } - self.store_name(name) - } + // Set kwdefaults if present + if flags.contains(bytecode::MakeFunctionFlags::KW_ONLY_DEFAULTS) { + // kwdefaults dict is already on stack + emit!( + self, + Instruction::SetFunctionAttribute { + attr: bytecode::MakeFunctionFlags::KW_ONLY_DEFAULTS + } + ); + } - fn build_closure(&mut self, code: &CodeObject) -> bool { - if code.freevars.is_empty() { - return false; + // Set defaults if present + if flags.contains(bytecode::MakeFunctionFlags::DEFAULTS) { + // defaults tuple is already on stack + emit!( + self, + Instruction::SetFunctionAttribute { + attr: bytecode::MakeFunctionFlags::DEFAULTS + } + ); } - for var in &*code.freevars { - let table = self.symbol_table_stack.last().unwrap(); - let symbol = unwrap_internal( + + // Set type_params if present + if flags.contains(bytecode::MakeFunctionFlags::TYPE_PARAMS) { + // type_params tuple is already on stack + emit!( self, - table - .lookup(var) - .ok_or_else(|| InternalError::MissingSymbol(var.to_owned())), + Instruction::SetFunctionAttribute { + attr: bytecode::MakeFunctionFlags::TYPE_PARAMS + } ); - let parent_code = self.code_stack.last().unwrap(); - let vars = match symbol.scope { - SymbolScope::Free => &parent_code.freevar_cache, - SymbolScope::Cell => &parent_code.cellvar_cache, - _ if symbol.flags.contains(SymbolFlags::FREE_CLASS) => &parent_code.freevar_cache, - x => unreachable!( - "var {} in a {:?} should be free or cell but it's {:?}", - var, table.typ, x - ), - }; - let mut idx = vars.get_index_of(var).unwrap(); - if let SymbolScope::Free = symbol.scope { - idx += parent_code.cellvar_cache.len(); - } - emit!(self, Instruction::LoadClosure(idx.to_u32())) } - emit!( - self, - Instruction::BuildTuple { - size: code.freevars.len().to_u32(), - } - ); - true + + Ok(()) } // Python/compile.c find_ann @@ -1675,71 +2940,57 @@ impl Compiler<'_> { false } - fn compile_class_def( + /// Compile the class body into a code object + // = compiler_class_body + fn compile_class_body( &mut self, name: &str, body: &[Stmt], - decorator_list: &[Decorator], type_params: Option<&TypeParams>, - arguments: Option<&Arguments>, - ) -> CompileResult<()> { - self.prepare_decorators(decorator_list)?; - - let prev_ctx = self.ctx; - self.ctx = CompileContext { - func: FunctionContext::NoFunction, - in_class: true, - loop_data: None, - }; - - let prev_class_name = self.class_name.replace(name.to_owned()); - - // Check if the class is declared global - let symbol_table = self.symbol_table_stack.last().unwrap(); - let symbol = unwrap_internal( - self, - symbol_table - .lookup(name.as_ref()) - .ok_or_else(|| InternalError::MissingSymbol(name.to_owned())), - ); - let mut global_path_prefix = Vec::new(); - if symbol.scope == SymbolScope::GlobalExplicit { - global_path_prefix.append(&mut self.qualified_path); - } - self.push_qualified_path(name); - let qualified_name = self.qualified_path.join("."); + firstlineno: u32, + ) -> CompileResult { + // 1. Enter class scope + let key = self.symbol_table_stack.len(); + self.push_symbol_table(); + self.enter_scope(name, CompilerScope::Class, key, firstlineno)?; - // If there are type params, we need to push a special symbol table just for them - if let Some(type_params) = type_params { - self.push_symbol_table(); - // Compile type parameters and store as .type_params - self.compile_type_params(type_params)?; - let dot_type_params = self.name(".type_params"); - emit!(self, Instruction::StoreLocal(dot_type_params)); - } + // Set qualname using the new method + let qualname = self.set_qualname(); - self.push_output(bytecode::CodeFlags::empty(), 0, 0, 0, name.to_owned()); + // For class scopes, set u_private to the class name for name mangling + self.code_stack.last_mut().unwrap().private = Some(name.to_owned()); + // 2. Set up class namespace let (doc_str, body) = split_doc(body, &self.opts); + // Load (global) __name__ and store as __module__ let dunder_name = self.name("__name__"); emit!(self, Instruction::LoadGlobal(dunder_name)); let dunder_module = self.name("__module__"); emit!(self, Instruction::StoreLocal(dunder_module)); + + // Store __qualname__ self.emit_load_const(ConstantData::Str { - value: qualified_name.into(), + value: qualname.into(), }); - let qualname = self.name("__qualname__"); - emit!(self, Instruction::StoreLocal(qualname)); - self.load_docstring(doc_str); - let doc = self.name("__doc__"); - emit!(self, Instruction::StoreLocal(doc)); - // setup annotations - if Self::find_ann(body) { - emit!(self, Instruction::SetupAnnotation); + let qualname_name = self.name("__qualname__"); + emit!(self, Instruction::StoreLocal(qualname_name)); + + // Store __doc__ only if there's an explicit docstring + if let Some(doc) = doc_str { + self.emit_load_const(ConstantData::Str { value: doc.into() }); + let doc_name = self.name("__doc__"); + emit!(self, Instruction::StoreLocal(doc_name)); } - // Set __type_params__ from .type_params if we have type parameters (PEP 695) + // Store __firstlineno__ (new in Python 3.12+) + self.emit_load_const(ConstantData::Integer { + value: BigInt::from(firstlineno), + }); + let firstlineno_name = self.name("__firstlineno__"); + emit!(self, Instruction::StoreLocal(firstlineno_name)); + + // Set __type_params__ if we have type parameters if type_params.is_some() { // Load .type_params from enclosing scope let dot_type_params = self.name(".type_params"); @@ -1750,88 +3001,167 @@ impl Compiler<'_> { emit!(self, Instruction::StoreLocal(dunder_type_params)); } + // Setup annotations if needed + if Self::find_ann(body) { + emit!(self, Instruction::SetupAnnotation); + } + + // 3. Compile the class body self.compile_statements(body)?; + // 4. Handle __classcell__ if needed let classcell_idx = self .code_stack .last_mut() .unwrap() - .cellvar_cache + .metadata + .cellvars .iter() .position(|var| *var == "__class__"); if let Some(classcell_idx) = classcell_idx { emit!(self, Instruction::LoadClosure(classcell_idx.to_u32())); - emit!(self, Instruction::Duplicate); + emit!(self, Instruction::CopyItem { index: 1_u32 }); let classcell = self.name("__classcell__"); emit!(self, Instruction::StoreLocal(classcell)); } else { self.emit_load_const(ConstantData::None); } + // Return the class namespace self.emit_return_value(); - let code = self.pop_code_object(); + // Exit scope and return the code object + Ok(self.exit_scope()) + } - self.class_name = prev_class_name; - self.qualified_path.pop(); - self.qualified_path.append(global_path_prefix.as_mut()); - self.ctx = prev_ctx; + fn compile_class_def( + &mut self, + name: &str, + body: &[Stmt], + decorator_list: &[Decorator], + type_params: Option<&TypeParams>, + arguments: Option<&Arguments>, + ) -> CompileResult<()> { + self.prepare_decorators(decorator_list)?; - emit!(self, Instruction::LoadBuildClass); + let is_generic = type_params.is_some(); + let firstlineno = self.get_source_line_number().get().to_u32(); + + // Step 1: If generic, enter type params scope and compile type params + if is_generic { + let type_params_name = format!(""); + self.push_output( + bytecode::CodeFlags::IS_OPTIMIZED | bytecode::CodeFlags::NEW_LOCALS, + 0, + 0, + 0, + type_params_name, + ); - let mut func_flags = bytecode::MakeFunctionFlags::empty(); + // Set private name for name mangling + self.code_stack.last_mut().unwrap().private = Some(name.to_owned()); - // Prepare generic type parameters: - if type_params.is_some() { - // Load .type_params from the type params scope + // Compile type parameters and store as .type_params + self.compile_type_params(type_params.unwrap())?; let dot_type_params = self.name(".type_params"); - emit!(self, Instruction::LoadNameAny(dot_type_params)); - func_flags |= bytecode::MakeFunctionFlags::TYPE_PARAMS; + emit!(self, Instruction::StoreLocal(dot_type_params)); } - if self.build_closure(&code) { - func_flags |= bytecode::MakeFunctionFlags::CLOSURE; - } + // Step 2: Compile class body (always done, whether generic or not) + let prev_ctx = self.ctx; + self.ctx = CompileContext { + func: FunctionContext::NoFunction, + in_class: true, + loop_data: None, + }; + let class_code = self.compile_class_body(name, body, type_params, firstlineno)?; + self.ctx = prev_ctx; - self.emit_load_const(ConstantData::Code { - code: Box::new(code), - }); - self.emit_load_const(ConstantData::Str { value: name.into() }); + // Step 3: Generate the rest of the code for the call + if is_generic { + // Still in type params scope + let dot_type_params = self.name(".type_params"); + let dot_generic_base = self.name(".generic_base"); - // Turn code object into function object: - emit!(self, Instruction::MakeFunction(func_flags)); + // Create .generic_base + emit!(self, Instruction::LoadNameAny(dot_type_params)); + emit!( + self, + Instruction::CallIntrinsic1 { + func: bytecode::IntrinsicFunction1::SubscriptGeneric + } + ); + emit!(self, Instruction::StoreLocal(dot_generic_base)); - self.emit_load_const(ConstantData::Str { value: name.into() }); + // Generate class creation code + emit!(self, Instruction::LoadBuildClass); - // For PEP 695 classes: handle Generic base creation - if type_params.is_some() { - if let Some(arguments) = arguments { - // Has explicit bases - use them as is, don't add Generic - // CPython doesn't add Generic when explicit bases are present - let call = self.compile_call_inner(2, arguments)?; - self.compile_normal_call(call); + // Set up the class function with type params + let mut func_flags = bytecode::MakeFunctionFlags::empty(); + emit!(self, Instruction::LoadNameAny(dot_type_params)); + func_flags |= bytecode::MakeFunctionFlags::TYPE_PARAMS; + + // Create class function with closure + self.make_closure(class_code, func_flags)?; + self.emit_load_const(ConstantData::Str { value: name.into() }); + + // Compile original bases + let base_count = if let Some(arguments) = arguments { + for arg in &arguments.args { + self.compile_expression(arg)?; + } + arguments.args.len() } else { - // No explicit bases, add Generic[*type_params] as the only base - // Stack currently: [function, class_name] + 0 + }; - // Load .type_params for creating Generic base - let dot_type_params = self.name(".type_params"); - emit!(self, Instruction::LoadNameAny(dot_type_params)); + // Load .generic_base as the last base + emit!(self, Instruction::LoadNameAny(dot_generic_base)); - // Call INTRINSIC_SUBSCRIPT_GENERIC to create Generic[*type_params] + let nargs = 2 + u32::try_from(base_count).expect("too many base classes") + 1; // function, name, bases..., generic_base + + // Handle keyword arguments + if let Some(arguments) = arguments + && !arguments.keywords.is_empty() + { + for keyword in &arguments.keywords { + if let Some(name) = &keyword.arg { + self.emit_load_const(ConstantData::Str { + value: name.as_str().into(), + }); + } + self.compile_expression(&keyword.value)?; + } emit!( self, - Instruction::CallIntrinsic1 { - func: bytecode::IntrinsicFunction1::SubscriptGeneric + Instruction::CallFunctionKeyword { + nargs: nargs + + u32::try_from(arguments.keywords.len()) + .expect("too many keyword arguments") } ); - - // Call __build_class__ with 3 positional args: function, class_name, Generic[T] - emit!(self, Instruction::CallFunctionPositional { nargs: 3 }); + } else { + emit!(self, Instruction::CallFunctionPositional { nargs }); } + + // Return the created class + self.emit_return_value(); + + // Exit type params scope and wrap in function + let type_params_code = self.exit_scope(); + + // Execute the type params function + self.make_closure(type_params_code, bytecode::MakeFunctionFlags::empty())?; + emit!(self, Instruction::CallFunctionPositional { nargs: 0 }); } else { - // No type params, normal compilation + // Non-generic class: standard path + emit!(self, Instruction::LoadBuildClass); + + // Create class function with closure + self.make_closure(class_code, bytecode::MakeFunctionFlags::empty())?; + self.emit_load_const(ConstantData::Str { value: name.into() }); + let call = if let Some(arguments) = arguments { self.compile_call_inner(2, arguments)? } else { @@ -1840,27 +3170,11 @@ impl Compiler<'_> { self.compile_normal_call(call); } - // Pop the special type params symbol table - if type_params.is_some() { - self.pop_symbol_table(); - } - + // Step 4: Apply decorators and store (common to both paths) self.apply_decorators(decorator_list); - self.store_name(name) } - fn load_docstring(&mut self, doc_str: Option) { - // TODO: __doc__ must be default None and no bytecode unless it is Some - // Duplicate top of stack (the function or class object) - - // Doc string value: - self.emit_load_const(match doc_str { - Some(doc) => ConstantData::Str { value: doc.into() }, - None => ConstantData::None, // set docstring None if not declared - }); - } - fn compile_while(&mut self, test: &Expr, body: &[Stmt], orelse: &[Stmt]) -> CompileResult<()> { let while_block = self.new_block(); let else_block = self.new_block(); @@ -1869,6 +3183,9 @@ impl Compiler<'_> { emit!(self, Instruction::SetupLoop); self.switch_to_block(while_block); + // Push fblock for while loop + self.push_fblock(FBlockType::WhileLoop, while_block, after_block)?; + self.compile_jump_if(test, false, else_block)?; let was_in_loop = self.ctx.loop_data.replace((while_block, after_block)); @@ -1881,6 +3198,9 @@ impl Compiler<'_> { } ); self.switch_to_block(else_block); + + // Pop fblock + self.pop_fblock(FBlockType::WhileLoop); emit!(self, Instruction::PopBlock); self.compile_statements(orelse)?; self.switch_to_block(after_block); @@ -1909,6 +3229,12 @@ impl Compiler<'_> { emit!(self, Instruction::GetAwaitable); self.emit_load_const(ConstantData::None); emit!(self, Instruction::YieldFrom); + emit!( + self, + Instruction::Resume { + arg: bytecode::ResumeType::AfterAwait as u32 + } + ); emit!(self, Instruction::SetupAsyncWith { end: final_block }); } else { emit!(self, Instruction::SetupWith { end: final_block }); @@ -1920,7 +3246,7 @@ impl Compiler<'_> { self.compile_store(var)?; } None => { - emit!(self, Instruction::Pop); + emit!(self, Instruction::PopTop); } } final_block @@ -1950,6 +3276,12 @@ impl Compiler<'_> { emit!(self, Instruction::GetAwaitable); self.emit_load_const(ConstantData::None); emit!(self, Instruction::YieldFrom); + emit!( + self, + Instruction::Resume { + arg: bytecode::ResumeType::AfterAwait as u32 + } + ); } emit!(self, Instruction::WithCleanupFinish); @@ -1979,6 +3311,10 @@ impl Compiler<'_> { emit!(self, Instruction::GetAIter); self.switch_to_block(for_block); + + // Push fblock for async for loop + self.push_fblock(FBlockType::ForLoop, for_block, after_block)?; + emit!( self, Instruction::SetupExcept { @@ -1988,6 +3324,12 @@ impl Compiler<'_> { emit!(self, Instruction::GetANext); self.emit_load_const(ConstantData::None); emit!(self, Instruction::YieldFrom); + emit!( + self, + Instruction::Resume { + arg: bytecode::ResumeType::AfterAwait as u32 + } + ); self.compile_store(target)?; emit!(self, Instruction::PopBlock); } else { @@ -1995,6 +3337,10 @@ impl Compiler<'_> { emit!(self, Instruction::GetIter); self.switch_to_block(for_block); + + // Push fblock for for loop + self.push_fblock(FBlockType::ForLoop, for_block, after_block)?; + emit!(self, Instruction::ForIter { target: else_block }); // Start of loop iteration, set targets: @@ -2007,6 +3353,10 @@ impl Compiler<'_> { emit!(self, Instruction::Jump { target: for_block }); self.switch_to_block(else_block); + + // Pop fblock + self.pop_fblock(FBlockType::ForLoop); + if is_async { emit!(self, Instruction::EndAsyncFor); } @@ -2068,7 +3418,7 @@ impl Compiler<'_> { JumpOp::PopJumpIfFalse => { emit!( self, - Instruction::JumpIfFalse { + Instruction::PopJumpIfFalse { target: pc.fail_pop[pops] } ); @@ -2089,7 +3439,7 @@ impl Compiler<'_> { for &label in pc.fail_pop.iter().skip(1).rev() { self.switch_to_block(label); // Emit the POP instruction. - emit!(self, Instruction::Pop); + emit!(self, Instruction::PopTop); } // Finally, use the first label. self.switch_to_block(pc.fail_pop[0]); @@ -2101,6 +3451,11 @@ impl Compiler<'_> { /// Duplicate the effect of Python 3.10's ROT_* instructions using SWAPs. fn pattern_helper_rotate(&mut self, mut count: usize) -> CompileResult<()> { + // Rotate TOS (top of stack) to position `count` down + // This is done by a series of swaps + // For count=1, no rotation needed (already at top) + // For count=2, swap TOS with item 1 position down + // For count=3, swap TOS with item 2 positions down, then with item 1 position down while count > 1 { // Emit a SWAP instruction with the current count. emit!( @@ -2128,7 +3483,7 @@ impl Compiler<'_> { match n { // If no name is provided, simply pop the top of the stack. None => { - emit!(self, Instruction::Pop); + emit!(self, Instruction::PopTop); Ok(()) } Some(name) => { @@ -2237,12 +3592,12 @@ impl Compiler<'_> { // index = len(subject) - (size - i) emit!(self, Instruction::GetLen); self.emit_load_const(ConstantData::Integer { - value: (patterns.len() - 1).into(), + value: (patterns.len() - i).into(), }); // Subtract to compute the correct index. emit!( self, - Instruction::BinaryOperation { + Instruction::BinaryOp { op: BinaryOperator::Subtract } ); @@ -2254,7 +3609,7 @@ impl Compiler<'_> { } // Pop the subject off the stack. pc.on_top -= 1; - emit!(self, Instruction::Pop); + emit!(self, Instruction::PopTop); Ok(()) } @@ -2387,15 +3742,13 @@ impl Compiler<'_> { // Create a new tuple of attribute names. let mut attr_names = vec![]; - for name in kwd_attrs.iter() { + for name in &kwd_attrs { // Py_NewRef(name) is emulated by cloning the name into a PyObject. attr_names.push(ConstantData::Str { value: name.as_str().to_string().into(), }); } - use bytecode::TestOperator::*; - // Emit instructions: // 1. Load the new tuple of attribute names. self.emit_load_const(ConstantData::Tuple { @@ -2408,155 +3761,251 @@ impl Compiler<'_> { // 4. Load None. self.emit_load_const(ConstantData::None); // 5. Compare with IS_OP 1. - emit!(self, Instruction::TestOperation { op: IsNot }); + emit!(self, Instruction::IsOp(Invert::Yes)); // At this point the TOS is a tuple of (nargs + n_attrs) attributes (or None). pc.on_top += 1; self.jump_to_fail_pop(pc, JumpOp::PopJumpIfFalse)?; - // Unpack the tuple into (nargs + n_attrs) items. - let total = nargs + n_attrs; - emit!( - self, - Instruction::UnpackSequence { - size: u32::try_from(total).unwrap() - } - ); - pc.on_top += total; + // Unpack the tuple into (nargs + n_attrs) items. + let total = nargs + n_attrs; + emit!( + self, + Instruction::UnpackSequence { + size: u32::try_from(total).unwrap() + } + ); + pc.on_top += total; + pc.on_top -= 1; + + // Process each sub-pattern. + for subpattern in patterns.iter().chain(kwd_patterns.iter()) { + // Check if this is a true wildcard (underscore pattern without name binding) + let is_true_wildcard = match subpattern { + Pattern::MatchAs(match_as) => { + // Only consider it wildcard if both pattern and name are None (i.e., "_") + match_as.pattern.is_none() && match_as.name.is_none() + } + _ => subpattern.is_wildcard(), + }; + + // Decrement the on_top counter for each sub-pattern + pc.on_top -= 1; + + if is_true_wildcard { + emit!(self, Instruction::PopTop); + continue; // Don't compile wildcard patterns + } + + // Compile the subpattern without irrefutability checks. + self.compile_pattern_subpattern(subpattern, pc)?; + } + Ok(()) + } + + fn compile_pattern_mapping( + &mut self, + p: &PatternMatchMapping, + pc: &mut PatternContext, + ) -> CompileResult<()> { + let mapping = p; + let keys = &mapping.keys; + let patterns = &mapping.patterns; + let size = keys.len(); + let star_target = &mapping.rest; + + // Validate pattern count matches key count + if keys.len() != patterns.len() { + return Err(self.error(CodegenErrorType::SyntaxError(format!( + "keys ({}) / patterns ({}) length mismatch in mapping pattern", + keys.len(), + patterns.len() + )))); + } + + // Validate rest pattern: '_' cannot be used as a rest target + if let Some(rest) = star_target + && rest.as_str() == "_" + { + return Err(self.error(CodegenErrorType::SyntaxError("invalid syntax".to_string()))); + } + + // Step 1: Check if subject is a mapping + // Stack: [subject] + pc.on_top += 1; + + emit!(self, Instruction::MatchMapping); + // Stack: [subject, is_mapping] + + self.jump_to_fail_pop(pc, JumpOp::PopJumpIfFalse)?; + // Stack: [subject] + + // Special case: empty pattern {} with no rest + if size == 0 && star_target.is_none() { + // If the pattern is just "{}", we're done! Pop the subject + pc.on_top -= 1; + emit!(self, Instruction::PopTop); + return Ok(()); + } + + // Length check for patterns with keys + if size > 0 { + // Check if the mapping has at least 'size' keys + emit!(self, Instruction::GetLen); + self.emit_load_const(ConstantData::Integer { value: size.into() }); + // Stack: [subject, len, size] + emit!( + self, + Instruction::CompareOperation { + op: ComparisonOperator::GreaterOrEqual + } + ); + self.jump_to_fail_pop(pc, JumpOp::PopJumpIfFalse)?; + // Stack: [subject] + } + + // Check for overflow (INT_MAX < size - 1) + if size > (i32::MAX as usize + 1) { + return Err(self.error(CodegenErrorType::SyntaxError( + "too many sub-patterns in mapping pattern".to_string(), + ))); + } + #[allow(clippy::cast_possible_truncation)] + let size = size as u32; // checked right before + + // Step 2: If we have keys to match + if size > 0 { + // Validate and compile keys + let mut seen = HashSet::new(); + for key in keys { + let is_attribute = matches!(key, Expr::Attribute(_)); + let is_literal = matches!( + key, + Expr::NumberLiteral(_) + | Expr::StringLiteral(_) + | Expr::BytesLiteral(_) + | Expr::BooleanLiteral(_) + | Expr::NoneLiteral(_) + ); + let key_repr = if is_literal { + UnparseExpr::new(key, &self.source_file).to_string() + } else if is_attribute { + String::new() + } else { + return Err(self.error(CodegenErrorType::SyntaxError( + "mapping pattern keys may only match literals and attribute lookups" + .to_string(), + ))); + }; + + if !key_repr.is_empty() && seen.contains(&key_repr) { + return Err(self.error(CodegenErrorType::SyntaxError(format!( + "mapping pattern checks duplicate key ({key_repr})" + )))); + } + if !key_repr.is_empty() { + seen.insert(key_repr); + } + + self.compile_expression(key)?; + } + } + // Stack: [subject, key1, key2, ..., key_n] + + // Build tuple of keys (empty tuple if size==0) + emit!(self, Instruction::BuildTuple { size }); + // Stack: [subject, keys_tuple] + + // Match keys + emit!(self, Instruction::MatchKeys); + // Stack: [subject, keys_tuple, values_or_none] + pc.on_top += 2; // subject and keys_tuple are underneath + + // Check if match succeeded + emit!(self, Instruction::CopyItem { index: 1_u32 }); + // Stack: [subject, keys_tuple, values_tuple, values_tuple_copy] + + // Check if copy is None (consumes the copy like POP_JUMP_IF_NONE) + self.emit_load_const(ConstantData::None); + emit!(self, Instruction::IsOp(Invert::Yes)); + + // Stack: [subject, keys_tuple, values_tuple, bool] + self.jump_to_fail_pop(pc, JumpOp::PopJumpIfFalse)?; + // Stack: [subject, keys_tuple, values_tuple] + + // Unpack values (the original values_tuple) + emit!(self, Instruction::UnpackSequence { size }); + // Stack after unpack: [subject, keys_tuple, ...unpacked values...] + pc.on_top += size as usize; // Unpacked size values, tuple replaced by values pc.on_top -= 1; - // Process each sub-pattern. - for subpattern in patterns.iter().chain(kwd_patterns.iter()) { - // Decrement the on_top counter as each sub-pattern is processed - // (on_top should be zero at the end of the algorithm as a sanity check). + // Step 3: Process matched values + for i in 0..size { pc.on_top -= 1; - if subpattern.is_wildcard() { - emit!(self, Instruction::Pop); + self.compile_pattern_subpattern(&patterns[i as usize], pc)?; + } + + // After processing subpatterns, adjust on_top + // CPython: "Whatever happens next should consume the tuple of keys and the subject" + // Stack currently: [subject, keys_tuple, ...any captured values...] + pc.on_top -= 2; + + // Step 4: Handle rest pattern or cleanup + if let Some(rest_name) = star_target { + // Build rest dict for **rest pattern + // Stack: [subject, keys_tuple] + + // Build rest dict exactly + emit!(self, Instruction::BuildMap { size: 0 }); + // Stack: [subject, keys_tuple, {}] + emit!(self, Instruction::Swap { index: 3 }); + // Stack: [{}, keys_tuple, subject] + emit!(self, Instruction::DictUpdate { index: 2 }); + // Stack after DICT_UPDATE: [rest_dict, keys_tuple] + // DICT_UPDATE consumes source (subject) and leaves dict in place + + // Unpack keys and delete from rest_dict + emit!(self, Instruction::UnpackSequence { size }); + // Stack: [rest_dict, k1, k2, ..., kn] (if size==0, nothing pushed) + + // Delete each key from rest_dict (skipped when size==0) + // while (size) { COPY(1 + size--); SWAP(2); DELETE_SUBSCR } + let mut remaining = size; + while remaining > 0 { + // Copy rest_dict which is at position (1 + remaining) from TOS + emit!( + self, + Instruction::CopyItem { + index: 1 + remaining + } + ); + // Stack: [rest_dict, k1, ..., kn, rest_dict] + emit!(self, Instruction::Swap { index: 2 }); + // Stack: [rest_dict, k1, ..., kn-1, rest_dict, kn] + emit!(self, Instruction::DeleteSubscript); + // Stack: [rest_dict, k1, ..., kn-1] (removed kn from rest_dict) + remaining -= 1; } - // Compile the subpattern without irrefutability checks. - self.compile_pattern_subpattern(subpattern, pc)?; + // Stack: [rest_dict] (plus any previously stored values) + // pattern_helper_store_name will handle the rotation correctly + + // Store the rest dict + self.pattern_helper_store_name(Some(rest_name), pc)?; + + // After storing all values, pc.on_top should be 0 + // The values are rotated to the bottom for later storage + pc.on_top = 0; + } else { + // Non-rest pattern: just clean up the stack + + // Pop them as we're not using them + emit!(self, Instruction::PopTop); // Pop keys_tuple + emit!(self, Instruction::PopTop); // Pop subject } + Ok(()) } - // fn compile_pattern_mapping(&mut self, p: &PatternMatchMapping, pc: &mut PatternContext) -> CompileResult<()> { - // // Ensure the pattern is a mapping pattern. - // let mapping = p; // Extract MatchMapping-specific data. - // let keys = &mapping.keys; - // let patterns = &mapping.patterns; - // let size = keys.len(); - // let n_patterns = patterns.len(); - - // if size != n_patterns { - // panic!("keys ({}) / patterns ({}) length mismatch in mapping pattern", size, n_patterns); - // // return self.compiler_error( - // // &format!("keys ({}) / patterns ({}) length mismatch in mapping pattern", size, n_patterns) - // // ); - // } - - // // A double-star target is present if `rest` is set. - // let star_target = mapping.rest; - - // // Keep the subject on top during the mapping and length checks. - // pc.on_top += 1; - // emit!(self, Instruction::MatchMapping); - // self.jump_to_fail_pop(pc, JumpOp::PopJumpIfFalse)?; - - // // If the pattern is just "{}" (empty mapping) and there's no star target, - // // we're done—pop the subject. - // if size == 0 && star_target.is_none() { - // pc.on_top -= 1; - // emit!(self, Instruction::Pop); - // return Ok(()); - // } - - // // If there are any keys, perform a length check. - // if size != 0 { - // emit!(self, Instruction::GetLen); - // self.emit_load_const(ConstantData::Integer { value: size.into() }); - // emit!(self, Instruction::CompareOperation { op: ComparisonOperator::GreaterOrEqual }); - // self.jump_to_fail_pop(pc, JumpOp::PopJumpIfFalse)?; - // } - - // // Check that the number of sub-patterns is not absurd. - // if size.saturating_sub(1) > (i32::MAX as usize) { - // panic!("too many sub-patterns in mapping pattern"); - // // return self.compiler_error("too many sub-patterns in mapping pattern"); - // } - - // // Collect all keys into a set for duplicate checking. - // let mut seen = HashSet::new(); - - // // For each key, validate it and check for duplicates. - // for (i, key) in keys.iter().enumerate() { - // if let Some(key_val) = key.as_literal_expr() { - // let in_seen = seen.contains(&key_val); - // if in_seen { - // panic!("mapping pattern checks duplicate key: {:?}", key_val); - // // return self.compiler_error(format!("mapping pattern checks duplicate key: {:?}", key_val)); - // } - // seen.insert(key_val); - // } else if !key.is_attribute_expr() { - // panic!("mapping pattern keys may only match literals and attribute lookups"); - // // return self.compiler_error("mapping pattern keys may only match literals and attribute lookups"); - // } - - // // Visit the key expression. - // self.compile_expression(key)?; - // } - // // Drop the set (its resources will be freed automatically). - - // // Build a tuple of keys and emit MATCH_KEYS. - // emit!(self, Instruction::BuildTuple { size: size as u32 }); - // emit!(self, Instruction::MatchKeys); - // // Now, on top of the subject there are two new tuples: one of keys and one of values. - // pc.on_top += 2; - - // // Prepare for matching the values. - // emit!(self, Instruction::CopyItem { index: 1_u32 }); - // self.emit_load_const(ConstantData::None); - // // TODO: should be is - // emit!(self, Instruction::TestOperation::IsNot); - // self.jump_to_fail_pop(pc, JumpOp::PopJumpIfFalse)?; - - // // Unpack the tuple of values. - // emit!(self, Instruction::UnpackSequence { size: size as u32 }); - // pc.on_top += size.saturating_sub(1); - - // // Compile each subpattern in "subpattern" mode. - // for pattern in patterns { - // pc.on_top = pc.on_top.saturating_sub(1); - // self.compile_pattern_subpattern(pattern, pc)?; - // } - - // // Consume the tuple of keys and the subject. - // pc.on_top = pc.on_top.saturating_sub(2); - // if let Some(star_target) = star_target { - // // If we have a starred name, bind a dict of remaining items to it. - // // This sequence of instructions performs: - // // rest = dict(subject) - // // for key in keys: del rest[key] - // emit!(self, Instruction::BuildMap { size: 0 }); // Build an empty dict. - // emit!(self, Instruction::Swap(3)); // Rearrange stack: [empty, keys, subject] - // emit!(self, Instruction::DictUpdate { size: 2 }); // Update dict with subject. - // emit!(self, Instruction::UnpackSequence { size: size as u32 }); // Unpack keys. - // let mut remaining = size; - // while remaining > 0 { - // emit!(self, Instruction::CopyItem { index: 1 + remaining as u32 }); // Duplicate subject copy. - // emit!(self, Instruction::Swap { index: 2_u32 }); // Bring key to top. - // emit!(self, Instruction::DeleteSubscript); // Delete key from dict. - // remaining -= 1; - // } - // // Bind the dict to the starred target. - // self.pattern_helper_store_name(Some(&star_target), pc)?; - // } else { - // // No starred target: just pop the tuple of keys and the subject. - // emit!(self, Instruction::Pop); - // emit!(self, Instruction::Pop); - // } - // Ok(()) - // } - fn compile_pattern_or( &mut self, p: &PatternMatchOr, @@ -2619,7 +4068,7 @@ impl Compiler<'_> { pc.stores.insert(insert_pos + j, elem); } // Also perform the same rotation on the evaluation stack. - for _ in 0..(i_stores + 1) { + for _ in 0..=i_stores { self.pattern_helper_rotate(i_control + 1)?; } } @@ -2639,7 +4088,7 @@ impl Compiler<'_> { // In Rust, old_pc is a local clone, so we need not worry about that. // No alternative matched: pop the subject and fail. - emit!(self, Instruction::Pop); + emit!(self, Instruction::PopTop); self.jump_to_fail_pop(pc, JumpOp::Jump)?; // Use the label "end". @@ -2661,7 +4110,7 @@ impl Compiler<'_> { // Old context and control will be dropped automatically. // Finally, pop the copy of the subject. - emit!(self, Instruction::Pop); + emit!(self, Instruction::PopTop); Ok(()) } @@ -2735,7 +4184,7 @@ impl Compiler<'_> { pc.on_top -= 1; if only_wildcard { // Patterns like: [] / [_] / [_, _] / [*_] / [_, *_] / [_, _, *_] / etc. - emit!(self, Instruction::Pop); + emit!(self, Instruction::PopTop); } else if star_wildcard { self.pattern_helper_sequence_subscr(patterns, star.unwrap(), pc)?; } else { @@ -2774,12 +4223,7 @@ impl Compiler<'_> { Singleton::True => ConstantData::Boolean { value: true }, }); // Compare using the "Is" operator. - emit!( - self, - Instruction::CompareOperation { - op: bytecode::ComparisonOperator::Equal - } - ); + emit!(self, Instruction::IsOp(Invert::No)); // Jump to the failure label if the comparison is false. self.jump_to_fail_pop(pc, JumpOp::PopJumpIfFalse)?; Ok(()) @@ -2800,7 +4244,9 @@ impl Compiler<'_> { Pattern::MatchSequence(pattern_type) => { self.compile_pattern_sequence(pattern_type, pattern_context) } - // Pattern::MatchMapping(pattern_type) => self.compile_pattern_mapping(pattern_type, pattern_context), + Pattern::MatchMapping(pattern_type) => { + self.compile_pattern_mapping(pattern_type, pattern_context) + } Pattern::MatchClass(pattern_type) => { self.compile_pattern_class(pattern_type, pattern_context) } @@ -2813,11 +4259,6 @@ impl Compiler<'_> { Pattern::MatchOr(pattern_type) => { self.compile_pattern_or(pattern_type, pattern_context) } - _ => { - // The eprintln gives context as to which pattern type is not implemented. - eprintln!("not implemented pattern type: {pattern_type:?}"); - Err(self.error(CodegenErrorType::NotImplementedYet)) - } } } @@ -2853,16 +4294,21 @@ impl Compiler<'_> { self.compile_name(name, NameUsage::Store)?; } - if let Some(ref _guard) = m.guard { + if let Some(ref guard) = m.guard { self.ensure_fail_pop(pattern_context, 0)?; - // TODO: Fix compile jump if call - return Err(self.error(CodegenErrorType::NotImplementedYet)); - // Jump if the guard fails. We assume that patter_context.fail_pop[0] is the jump target. - // self.compile_jump_if(&m.pattern, &guard, pattern_context.fail_pop[0])?; + // Compile the guard expression + self.compile_expression(guard)?; + emit!(self, Instruction::ToBool); + emit!( + self, + Instruction::PopJumpIfFalse { + target: pattern_context.fail_pop[0] + } + ); } if i != case_count - 1 { - emit!(self, Instruction::Pop); + emit!(self, Instruction::PopTop); } self.compile_statements(&m.body)?; @@ -2873,13 +4319,16 @@ impl Compiler<'_> { if has_default { let m = &cases[num_cases - 1]; if num_cases == 1 { - emit!(self, Instruction::Pop); + emit!(self, Instruction::PopTop); } else { emit!(self, Instruction::Nop); } - if let Some(ref _guard) = m.guard { - // TODO: Fix compile jump if call - return Err(self.error(CodegenErrorType::NotImplementedYet)); + if let Some(ref guard) = m.guard { + // Compile guard and jump to end if false + self.compile_expression(guard)?; + emit!(self, Instruction::CopyItem { index: 1_u32 }); + emit!(self, Instruction::PopJumpIfFalse { target: end }); + emit!(self, Instruction::PopTop); } self.compile_statements(&m.body)?; } @@ -2893,100 +4342,104 @@ impl Compiler<'_> { Ok(()) } - fn compile_chained_comparison( + /// [CPython `compiler_addcompare`](https://github.com/python/cpython/blob/627894459a84be3488a1789919679c997056a03c/Python/compile.c#L2880-L2924) + fn compile_addcompare(&mut self, op: &CmpOp) { + use bytecode::ComparisonOperator::*; + match op { + CmpOp::Eq => emit!(self, Instruction::CompareOperation { op: Equal }), + CmpOp::NotEq => emit!(self, Instruction::CompareOperation { op: NotEqual }), + CmpOp::Lt => emit!(self, Instruction::CompareOperation { op: Less }), + CmpOp::LtE => emit!(self, Instruction::CompareOperation { op: LessOrEqual }), + CmpOp::Gt => emit!(self, Instruction::CompareOperation { op: Greater }), + CmpOp::GtE => { + emit!(self, Instruction::CompareOperation { op: GreaterOrEqual }) + } + CmpOp::In => emit!(self, Instruction::ContainsOp(Invert::No)), + CmpOp::NotIn => emit!(self, Instruction::ContainsOp(Invert::Yes)), + CmpOp::Is => emit!(self, Instruction::IsOp(Invert::No)), + CmpOp::IsNot => emit!(self, Instruction::IsOp(Invert::Yes)), + } + } + + /// Compile a chained comparison. + /// + /// ```py + /// a == b == c == d + /// ``` + /// + /// Will compile into (pseudo code): + /// + /// ```py + /// result = a == b + /// if result: + /// result = b == c + /// if result: + /// result = c == d + /// ``` + /// + /// # See Also + /// - [CPython `compiler_compare`](https://github.com/python/cpython/blob/627894459a84be3488a1789919679c997056a03c/Python/compile.c#L4678-L4717) + fn compile_compare( &mut self, left: &Expr, ops: &[CmpOp], - exprs: &[Expr], + comparators: &[Expr], ) -> CompileResult<()> { - assert!(!ops.is_empty()); - assert_eq!(exprs.len(), ops.len()); let (last_op, mid_ops) = ops.split_last().unwrap(); - let (last_val, mid_exprs) = exprs.split_last().unwrap(); - - use bytecode::ComparisonOperator::*; - use bytecode::TestOperator::*; - let compile_cmpop = |c: &mut Self, op: &CmpOp| match op { - CmpOp::Eq => emit!(c, Instruction::CompareOperation { op: Equal }), - CmpOp::NotEq => emit!(c, Instruction::CompareOperation { op: NotEqual }), - CmpOp::Lt => emit!(c, Instruction::CompareOperation { op: Less }), - CmpOp::LtE => emit!(c, Instruction::CompareOperation { op: LessOrEqual }), - CmpOp::Gt => emit!(c, Instruction::CompareOperation { op: Greater }), - CmpOp::GtE => { - emit!(c, Instruction::CompareOperation { op: GreaterOrEqual }) - } - CmpOp::In => emit!(c, Instruction::TestOperation { op: In }), - CmpOp::NotIn => emit!(c, Instruction::TestOperation { op: NotIn }), - CmpOp::Is => emit!(c, Instruction::TestOperation { op: Is }), - CmpOp::IsNot => emit!(c, Instruction::TestOperation { op: IsNot }), - }; - - // a == b == c == d - // compile into (pseudo code): - // result = a == b - // if result: - // result = b == c - // if result: - // result = c == d + let (last_comparator, mid_comparators) = comparators.split_last().unwrap(); // initialize lhs outside of loop self.compile_expression(left)?; - let end_blocks = if mid_exprs.is_empty() { - None - } else { - let break_block = self.new_block(); - let after_block = self.new_block(); - Some((break_block, after_block)) - }; + if mid_comparators.is_empty() { + self.compile_expression(last_comparator)?; + self.compile_addcompare(last_op); + + return Ok(()); + } + + let cleanup = self.new_block(); // for all comparisons except the last (as the last one doesn't need a conditional jump) - for (op, val) in mid_ops.iter().zip(mid_exprs) { - self.compile_expression(val)?; + for (op, comparator) in mid_ops.iter().zip(mid_comparators) { + self.compile_expression(comparator)?; + // store rhs for the next comparison in chain - emit!(self, Instruction::Duplicate); - emit!(self, Instruction::Rotate3); + emit!(self, Instruction::Swap { index: 2 }); + emit!(self, Instruction::CopyItem { index: 2 }); - compile_cmpop(self, op); + self.compile_addcompare(op); // if comparison result is false, we break with this value; if true, try the next one. - if let Some((break_block, _)) = end_blocks { - emit!( - self, - Instruction::JumpIfFalseOrPop { - target: break_block, - } - ); - } + /* + emit!(self, Instruction::CopyItem { index: 1 }); + // emit!(self, Instruction::ToBool); // TODO: Uncomment this + emit!(self, Instruction::PopJumpIfFalse { target: cleanup }); + emit!(self, Instruction::PopTop); + */ + + emit!(self, Instruction::JumpIfFalseOrPop { target: cleanup }); } - // handle the last comparison - self.compile_expression(last_val)?; - compile_cmpop(self, last_op); - - if let Some((break_block, after_block)) = end_blocks { - emit!( - self, - Instruction::Jump { - target: after_block, - } - ); + self.compile_expression(last_comparator)?; + self.compile_addcompare(last_op); - // early exit left us with stack: `rhs, comparison_result`. We need to clean up rhs. - self.switch_to_block(break_block); - emit!(self, Instruction::Rotate2); - emit!(self, Instruction::Pop); + let end = self.new_block(); + emit!(self, Instruction::Jump { target: end }); - self.switch_to_block(after_block); - } + // early exit left us with stack: `rhs, comparison_result`. We need to clean up rhs. + self.switch_to_block(cleanup); + emit!(self, Instruction::Swap { index: 2 }); + emit!(self, Instruction::PopTop); + self.switch_to_block(end); Ok(()) } fn compile_annotation(&mut self, annotation: &Expr) -> CompileResult<()> { if self.future_annotations { self.emit_load_const(ConstantData::Str { - value: unparse_expr(annotation, &self.source_code) + value: UnparseExpr::new(annotation, &self.source_file) .to_string() .into(), }); @@ -3042,7 +4495,7 @@ impl Compiler<'_> { emit!(self, Instruction::StoreSubscript); } else { // Drop annotation if not assigned to simple identifier. - emit!(self, Instruction::Pop); + emit!(self, Instruction::PopTop); } Ok(()) @@ -3051,10 +4504,10 @@ impl Compiler<'_> { fn compile_store(&mut self, target: &Expr) -> CompileResult<()> { match &target { Expr::Name(ExprName { id, .. }) => self.store_name(id.as_str())?, - Expr::Subscript(ExprSubscript { value, slice, .. }) => { - self.compile_expression(value)?; - self.compile_expression(slice)?; - emit!(self, Instruction::StoreSubscript); + Expr::Subscript(ExprSubscript { + value, slice, ctx, .. + }) => { + self.compile_subscript(value, slice, *ctx)?; } Expr::Attribute(ExprAttribute { value, attr, .. }) => { self.check_forbidden_name(attr.as_str(), NameUsage::Store)?; @@ -3135,10 +4588,18 @@ impl Compiler<'_> { self.compile_name(id, NameUsage::Load)?; AugAssignKind::Name { id } } - Expr::Subscript(ExprSubscript { value, slice, .. }) => { + Expr::Subscript(ExprSubscript { + value, + slice, + ctx: _, + .. + }) => { + // For augmented assignment, we need to load the value first + // But we can't use compile_subscript directly because we need DUP_TOP2 self.compile_expression(value)?; self.compile_expression(slice)?; - emit!(self, Instruction::Duplicate2); + emit!(self, Instruction::CopyItem { index: 2_u32 }); + emit!(self, Instruction::CopyItem { index: 2_u32 }); emit!(self, Instruction::Subscript); AugAssignKind::Subscript } @@ -3146,7 +4607,7 @@ impl Compiler<'_> { let attr = attr.as_str(); self.check_forbidden_name(attr, NameUsage::Store)?; self.compile_expression(value)?; - emit!(self, Instruction::Duplicate); + emit!(self, Instruction::CopyItem { index: 1_u32 }); let idx = self.name(attr); emit!(self, Instruction::LoadAttr { idx }); AugAssignKind::Attr { idx } @@ -3166,12 +4627,13 @@ impl Compiler<'_> { } AugAssignKind::Subscript => { // stack: CONTAINER SLICE RESULT - emit!(self, Instruction::Rotate3); + emit!(self, Instruction::Swap { index: 3 }); + emit!(self, Instruction::Swap { index: 2 }); emit!(self, Instruction::StoreSubscript); } AugAssignKind::Attr { idx } => { // stack: CONTAINER RESULT - emit!(self, Instruction::Rotate2); + emit!(self, Instruction::Swap { index: 2 }); emit!(self, Instruction::StoreAttr { idx }); } } @@ -3180,26 +4642,24 @@ impl Compiler<'_> { } fn compile_op(&mut self, op: &Operator, inplace: bool) { - let op = match op { - Operator::Add => bytecode::BinaryOperator::Add, - Operator::Sub => bytecode::BinaryOperator::Subtract, - Operator::Mult => bytecode::BinaryOperator::Multiply, - Operator::MatMult => bytecode::BinaryOperator::MatrixMultiply, - Operator::Div => bytecode::BinaryOperator::Divide, - Operator::FloorDiv => bytecode::BinaryOperator::FloorDivide, - Operator::Mod => bytecode::BinaryOperator::Modulo, - Operator::Pow => bytecode::BinaryOperator::Power, - Operator::LShift => bytecode::BinaryOperator::Lshift, - Operator::RShift => bytecode::BinaryOperator::Rshift, - Operator::BitOr => bytecode::BinaryOperator::Or, - Operator::BitXor => bytecode::BinaryOperator::Xor, - Operator::BitAnd => bytecode::BinaryOperator::And, + let bin_op = match op { + Operator::Add => BinaryOperator::Add, + Operator::Sub => BinaryOperator::Subtract, + Operator::Mult => BinaryOperator::Multiply, + Operator::MatMult => BinaryOperator::MatrixMultiply, + Operator::Div => BinaryOperator::TrueDivide, + Operator::FloorDiv => BinaryOperator::FloorDivide, + Operator::Mod => BinaryOperator::Remainder, + Operator::Pow => BinaryOperator::Power, + Operator::LShift => BinaryOperator::Lshift, + Operator::RShift => BinaryOperator::Rshift, + Operator::BitOr => BinaryOperator::Or, + Operator::BitXor => BinaryOperator::Xor, + Operator::BitAnd => BinaryOperator::And, }; - if inplace { - emit!(self, Instruction::BinaryOperationInplace { op }) - } else { - emit!(self, Instruction::BinaryOperation { op }) - } + + let op = if inplace { bin_op.as_inplace() } else { bin_op }; + emit!(self, Instruction::BinaryOp { op }) } /// Implement boolean short circuit evaluation logic. @@ -3214,7 +4674,7 @@ impl Compiler<'_> { &mut self, expression: &Expr, condition: bool, - target_block: ir::BlockIdx, + target_block: BlockIdx, ) -> CompileResult<()> { // Compile expression for test, and jump to label if false match &expression { @@ -3277,14 +4737,14 @@ impl Compiler<'_> { if condition { emit!( self, - Instruction::JumpIfTrue { + Instruction::PopJumpIfTrue { target: target_block, } ); } else { emit!( self, - Instruction::JumpIfFalse { + Instruction::PopJumpIfFalse { target: target_block, } ); @@ -3300,14 +4760,16 @@ impl Compiler<'_> { let after_block = self.new_block(); let (last_value, values) = values.split_last().unwrap(); + for value in values { self.compile_expression(value)?; + emit!(self, Instruction::CopyItem { index: 1_u32 }); match op { BoolOp::And => { emit!( self, - Instruction::JumpIfFalseOrPop { + Instruction::PopJumpIfFalse { target: after_block, } ); @@ -3315,12 +4777,14 @@ impl Compiler<'_> { BoolOp::Or => { emit!( self, - Instruction::JumpIfTrueOrPop { + Instruction::PopJumpIfTrue { target: after_block, } ); } } + + emit!(self, Instruction::PopTop); } // If all values did not qualify, take the value of the last value: @@ -3343,7 +4807,7 @@ impl Compiler<'_> { for item in unpacked { self.compile_expression(&item.value)?; - emit!(self, Instruction::DictUpdate); + emit!(self, Instruction::DictUpdate { index: 1 }); } Ok(()) @@ -3369,10 +4833,10 @@ impl Compiler<'_> { // Perform operation: self.compile_op(op, false); } - Expr::Subscript(ExprSubscript { value, slice, .. }) => { - self.compile_expression(value)?; - self.compile_expression(slice)?; - emit!(self, Instruction::Subscript); + Expr::Subscript(ExprSubscript { + value, slice, ctx, .. + }) => { + self.compile_subscript(value, slice, *ctx)?; } Expr::UnaryOp(ExprUnaryOp { op, operand, .. }) => { self.compile_expression(operand)?; @@ -3397,36 +4861,19 @@ impl Compiler<'_> { comparators, .. }) => { - self.compile_chained_comparison(left, ops, comparators)?; + self.compile_compare(left, ops, comparators)?; } // Expr::Constant(ExprConstant { value, .. }) => { // self.emit_load_const(compile_constant(value)); // } Expr::List(ExprList { elts, .. }) => { - let (size, unpack) = self.gather_elements(0, elts)?; - if unpack { - emit!(self, Instruction::BuildListFromTuples { size }); - } else { - emit!(self, Instruction::BuildList { size }); - } + self.starunpack_helper(elts, 0, CollectionType::List)?; } Expr::Tuple(ExprTuple { elts, .. }) => { - let (size, unpack) = self.gather_elements(0, elts)?; - if unpack { - if size > 1 { - emit!(self, Instruction::BuildTupleFromTuples { size }); - } - } else { - emit!(self, Instruction::BuildTuple { size }); - } + self.starunpack_helper(elts, 0, CollectionType::Tuple)?; } Expr::Set(ExprSet { elts, .. }) => { - let (size, unpack) = self.gather_elements(0, elts)?; - if unpack { - emit!(self, Instruction::BuildSetFromTuples { size }); - } else { - emit!(self, Instruction::BuildSet { size }); - } + self.starunpack_helper(elts, 0, CollectionType::Set)?; } Expr::Dict(ExprDict { items, .. }) => { self.compile_dict(items)?; @@ -3446,8 +4893,11 @@ impl Compiler<'_> { if let Some(step) = step { self.compile_expression(step)?; } - let step = step.is_some(); - emit!(self, Instruction::BuildSlice { step }); + let argc = match step { + Some(_) => BuildSliceArgCount::Three, + None => BuildSliceArgCount::Two, + }; + emit!(self, Instruction::BuildSlice { argc }); } Expr::Yield(ExprYield { value, .. }) => { if !self.ctx.in_func() { @@ -3459,6 +4909,12 @@ impl Compiler<'_> { Option::None => self.emit_load_const(ConstantData::None), }; emit!(self, Instruction::YieldValue); + emit!( + self, + Instruction::Resume { + arg: bytecode::ResumeType::AfterYield as u32 + } + ); } Expr::Await(ExprAwait { value, .. }) => { if self.ctx.func != FunctionContext::AsyncFunction { @@ -3468,6 +4924,12 @@ impl Compiler<'_> { emit!(self, Instruction::GetAwaitable); self.emit_load_const(ConstantData::None); emit!(self, Instruction::YieldFrom); + emit!( + self, + Instruction::Resume { + arg: bytecode::ResumeType::AfterAwait as u32 + } + ); } Expr::YieldFrom(ExprYieldFrom { value, .. }) => { match self.ctx.func { @@ -3484,16 +4946,76 @@ impl Compiler<'_> { emit!(self, Instruction::GetIter); self.emit_load_const(ConstantData::None); emit!(self, Instruction::YieldFrom); + emit!( + self, + Instruction::Resume { + arg: bytecode::ResumeType::AfterYieldFrom as u32 + } + ); } Expr::Name(ExprName { id, .. }) => self.load_name(id.as_str())?, Expr::Lambda(ExprLambda { parameters, body, .. }) => { - let prev_ctx = self.ctx; + let default_params = Parameters::default(); + let params = parameters.as_deref().unwrap_or(&default_params); + validate_duplicate_params(params).map_err(|e| self.error(e))?; + let prev_ctx = self.ctx; let name = "".to_owned(); - let mut func_flags = self - .enter_function(&name, parameters.as_deref().unwrap_or(&Default::default()))?; + + // Prepare defaults before entering function + let defaults: Vec<_> = std::iter::empty() + .chain(¶ms.posonlyargs) + .chain(¶ms.args) + .filter_map(|x| x.default.as_deref()) + .collect(); + let have_defaults = !defaults.is_empty(); + + if have_defaults { + let size = defaults.len().to_u32(); + for element in &defaults { + self.compile_expression(element)?; + } + emit!(self, Instruction::BuildTuple { size }); + } + + // Prepare keyword-only defaults + let mut kw_with_defaults = vec![]; + for kwonlyarg in ¶ms.kwonlyargs { + if let Some(default) = &kwonlyarg.default { + kw_with_defaults.push((&kwonlyarg.parameter, default)); + } + } + + let have_kwdefaults = !kw_with_defaults.is_empty(); + if have_kwdefaults { + let default_kw_count = kw_with_defaults.len(); + for (arg, default) in &kw_with_defaults { + self.emit_load_const(ConstantData::Str { + value: arg.name.as_str().into(), + }); + self.compile_expression(default)?; + } + emit!( + self, + Instruction::BuildMap { + size: default_kw_count.to_u32(), + } + ); + } + + self.enter_function(&name, params)?; + let mut func_flags = bytecode::MakeFunctionFlags::empty(); + if have_defaults { + func_flags |= bytecode::MakeFunctionFlags::DEFAULTS; + } + if have_kwdefaults { + func_flags |= bytecode::MakeFunctionFlags::KW_ONLY_DEFAULTS; + } + + // Set qualname for lambda + self.set_qualname(); self.ctx = CompileContext { loop_data: Option::None, @@ -3502,21 +5024,16 @@ impl Compiler<'_> { }; self.current_code_info() - .constants + .metadata + .consts .insert_full(ConstantData::None); self.compile_expression(body)?; self.emit_return_value(); - let code = self.pop_code_object(); - if self.build_closure(&code) { - func_flags |= bytecode::MakeFunctionFlags::CLOSURE; - } - self.emit_load_const(ConstantData::Code { - code: Box::new(code), - }); - self.emit_load_const(ConstantData::Str { value: name.into() }); - // Turn code object into function object: - emit!(self, Instruction::MakeFunction(func_flags)); + let code = self.exit_scope(); + + // Create lambda function with closure + self.make_closure(code, func_flags)?; self.ctx = prev_ctx; } @@ -3607,7 +5124,13 @@ impl Compiler<'_> { compiler.compile_comprehension_element(elt)?; compiler.mark_generator(); emit!(compiler, Instruction::YieldValue); - emit!(compiler, Instruction::Pop); + emit!( + compiler, + Instruction::Resume { + arg: bytecode::ResumeType::AfterYield as u32 + } + ); + emit!(compiler, Instruction::PopTop); Ok(()) }, @@ -3652,15 +5175,19 @@ impl Compiler<'_> { Expr::Named(ExprNamed { target, value, + node_index: _, range: _, }) => { self.compile_expression(value)?; - emit!(self, Instruction::Duplicate); + emit!(self, Instruction::CopyItem { index: 1_u32 }); self.compile_store(target)?; } Expr::FString(fstring) => { self.compile_expr_fstring(fstring)?; } + Expr::TString(_) => { + return Err(self.error(CodegenErrorType::NotImplementedYet)); + } Expr::StringLiteral(string) => { let value = string.value.to_str(); if value.contains(char::REPLACEMENT_CHARACTER) { @@ -3668,7 +5195,7 @@ impl Compiler<'_> { .value .iter() .map(|lit| { - let source = self.source_code.get_range(lit.range); + let source = self.source_file.slice(lit.range); crate::string_parser::parse_string_literal(source, lit.flags.into()) }) .collect(); @@ -3956,12 +5483,19 @@ impl Compiler<'_> { // Create magnificent function : self.push_output(flags, 1, 1, 0, name.to_owned()); + + // Mark that we're in an inlined comprehension + self.current_code_info().in_inlined_comp = true; + + // Set qualname for comprehension + self.set_qualname(); + let arg0 = self.varname(".0")?; let return_none = init_collection.is_none(); // Create empty object of proper type: if let Some(init_collection) = init_collection { - self._emit(init_collection, OpArg(0), ir::BlockIdx::NULL) + self._emit(init_collection, OpArg(0), BlockIdx::NULL) } let mut loop_labels = vec![]; @@ -3998,6 +5532,12 @@ impl Compiler<'_> { emit!(self, Instruction::GetANext); self.emit_load_const(ConstantData::None); emit!(self, Instruction::YieldFrom); + emit!( + self, + Instruction::Resume { + arg: bytecode::ResumeType::AfterAwait as u32 + } + ); self.compile_store(&generator.target)?; emit!(self, Instruction::PopBlock); } else { @@ -4037,25 +5577,12 @@ impl Compiler<'_> { self.emit_return_value(); // Fetch code for listcomp function: - let code = self.pop_code_object(); + let code = self.exit_scope(); self.ctx = prev_ctx; - let mut func_flags = bytecode::MakeFunctionFlags::empty(); - if self.build_closure(&code) { - func_flags |= bytecode::MakeFunctionFlags::CLOSURE; - } - - // List comprehension code: - self.emit_load_const(ConstantData::Code { - code: Box::new(code), - }); - - // List comprehension function name: - self.emit_load_const(ConstantData::Str { value: name.into() }); - - // Turn code object into function object: - emit!(self, Instruction::MakeFunction(func_flags)); + // Create comprehension function with closure + self.make_closure(code, bytecode::MakeFunctionFlags::empty())?; // Evaluate iterated item: self.compile_expression(&generators[0].iter)?; @@ -4076,6 +5603,12 @@ impl Compiler<'_> { emit!(self, Instruction::GetAwaitable); self.emit_load_const(ConstantData::None); emit!(self, Instruction::YieldFrom); + emit!( + self, + Instruction::Resume { + arg: bytecode::ResumeType::AfterAwait as u32 + } + ); } Ok(()) @@ -4103,9 +5636,12 @@ impl Compiler<'_> { } // Low level helper functions: - fn _emit(&mut self, instr: Instruction, arg: OpArg, target: ir::BlockIdx) { + fn _emit(&mut self, instr: Instruction, arg: OpArg, target: BlockIdx) { let range = self.current_source_range; - let location = self.source_code.source_location(range.start()); + let location = self + .source_file + .to_source_code() + .source_location(range.start(), PositionEncoding::Utf8); // TODO: insert source filename self.current_block().instructions.push(ir::InstructionInfo { instr, @@ -4117,7 +5653,7 @@ impl Compiler<'_> { } fn emit_no_arg(&mut self, ins: Instruction) { - self._emit(ins, OpArg::null(), ir::BlockIdx::NULL) + self._emit(ins, OpArg::null(), BlockIdx::NULL) } fn emit_arg>( @@ -4133,7 +5669,7 @@ impl Compiler<'_> { fn arg_constant(&mut self, constant: ConstantData) -> u32 { let info = self.current_code_info(); - info.constants.insert_full(constant).0.to_u32() + info.metadata.consts.insert_full(constant).0.to_u32() } fn emit_load_const(&mut self, constant: ConstantData) { @@ -4147,11 +5683,11 @@ impl Compiler<'_> { } fn emit_return_value(&mut self) { - if let Some(inst) = self.current_block().instructions.last_mut() { - if let Instruction::LoadConst { idx } = inst.instr { - inst.instr = Instruction::ReturnConst { idx }; - return; - } + if let Some(inst) = self.current_block().instructions.last_mut() + && let Instruction::LoadConst { idx } = inst.instr + { + inst.instr = Instruction::ReturnConst { idx }; + return; } emit!(self, Instruction::ReturnValue) } @@ -4165,25 +5701,25 @@ impl Compiler<'_> { &mut info.blocks[info.current_block] } - fn new_block(&mut self) -> ir::BlockIdx { + fn new_block(&mut self) -> BlockIdx { let code = self.current_code_info(); - let idx = ir::BlockIdx(code.blocks.len().to_u32()); + let idx = BlockIdx::new(code.blocks.len().to_u32()); code.blocks.push(ir::Block::default()); idx } - fn switch_to_block(&mut self, block: ir::BlockIdx) { + fn switch_to_block(&mut self, block: BlockIdx) { let code = self.current_code_info(); let prev = code.current_block; assert_ne!(prev, block, "recursive switching {prev:?} -> {block:?}"); assert_eq!( code.blocks[block].next, - ir::BlockIdx::NULL, + BlockIdx::NULL, "switching {prev:?} -> {block:?} to completed block" ); - let prev_block = &mut code.blocks[prev.0 as usize]; + let prev_block = &mut code.blocks[prev.idx()]; assert_eq!( - prev_block.next.0, + u32::from(prev_block.next), u32::MAX, "switching {prev:?} -> {block:?} from block that's already got a next" ); @@ -4196,138 +5732,46 @@ impl Compiler<'_> { } fn get_source_line_number(&mut self) -> OneIndexed { - self.source_code + self.source_file + .to_source_code() .line_index(self.current_source_range.start()) } - fn push_qualified_path(&mut self, name: &str) { - self.qualified_path.push(name.to_owned()); - } - fn mark_generator(&mut self) { self.current_code_info().flags |= bytecode::CodeFlags::IS_GENERATOR } /// Whether the expression contains an await expression and /// thus requires the function to be async. - /// Async with and async for are statements, so I won't check for them here + /// + /// Both: + /// ```py + /// async with: ... + /// async for: ... + /// ``` + /// are statements, so we won't check for them here fn contains_await(expression: &Expr) -> bool { - use ruff_python_ast::*; - - match &expression { - Expr::Call(ExprCall { - func, arguments, .. - }) => { - Self::contains_await(func) - || arguments.args.iter().any(Self::contains_await) - || arguments - .keywords - .iter() - .any(|kw| Self::contains_await(&kw.value)) - } - Expr::BoolOp(ExprBoolOp { values, .. }) => values.iter().any(Self::contains_await), - Expr::BinOp(ExprBinOp { left, right, .. }) => { - Self::contains_await(left) || Self::contains_await(right) - } - Expr::Subscript(ExprSubscript { value, slice, .. }) => { - Self::contains_await(value) || Self::contains_await(slice) - } - Expr::UnaryOp(ExprUnaryOp { operand, .. }) => Self::contains_await(operand), - Expr::Attribute(ExprAttribute { value, .. }) => Self::contains_await(value), - Expr::Compare(ExprCompare { - left, comparators, .. - }) => Self::contains_await(left) || comparators.iter().any(Self::contains_await), - Expr::List(ExprList { elts, .. }) => elts.iter().any(Self::contains_await), - Expr::Tuple(ExprTuple { elts, .. }) => elts.iter().any(Self::contains_await), - Expr::Set(ExprSet { elts, .. }) => elts.iter().any(Self::contains_await), - Expr::Dict(ExprDict { items, .. }) => items - .iter() - .flat_map(|item| &item.key) - .any(Self::contains_await), - Expr::Slice(ExprSlice { - lower, upper, step, .. - }) => { - lower.as_deref().is_some_and(Self::contains_await) - || upper.as_deref().is_some_and(Self::contains_await) - || step.as_deref().is_some_and(Self::contains_await) - } - Expr::Yield(ExprYield { value, .. }) => { - value.as_deref().is_some_and(Self::contains_await) - } - Expr::Await(ExprAwait { .. }) => true, - Expr::YieldFrom(ExprYieldFrom { value, .. }) => Self::contains_await(value), - Expr::Name(ExprName { .. }) => false, - Expr::Lambda(ExprLambda { body, .. }) => Self::contains_await(body), - Expr::ListComp(ExprListComp { - elt, generators, .. - }) => { - Self::contains_await(elt) - || generators.iter().any(|jen| Self::contains_await(&jen.iter)) - } - Expr::SetComp(ExprSetComp { - elt, generators, .. - }) => { - Self::contains_await(elt) - || generators.iter().any(|jen| Self::contains_await(&jen.iter)) - } - Expr::DictComp(ExprDictComp { - key, - value, - generators, - .. - }) => { - Self::contains_await(key) - || Self::contains_await(value) - || generators.iter().any(|jen| Self::contains_await(&jen.iter)) - } - Expr::Generator(ExprGenerator { - elt, generators, .. - }) => { - Self::contains_await(elt) - || generators.iter().any(|jen| Self::contains_await(&jen.iter)) - } - Expr::Starred(expr) => Self::contains_await(&expr.value), - Expr::If(ExprIf { - test, body, orelse, .. - }) => { - Self::contains_await(test) - || Self::contains_await(body) - || Self::contains_await(orelse) - } + #[derive(Default)] + struct AwaitVisitor { + found: bool, + } - Expr::Named(ExprNamed { - target, - value, - range: _, - }) => Self::contains_await(target) || Self::contains_await(value), - Expr::FString(ExprFString { value, range: _ }) => { - fn expr_element_contains_await bool>( - expr_element: &FStringExpressionElement, - contains_await: F, - ) -> bool { - contains_await(&expr_element.expression) - || expr_element - .format_spec - .iter() - .flat_map(|spec| spec.elements.expressions()) - .any(|element| expr_element_contains_await(element, contains_await)) + impl Visitor<'_> for AwaitVisitor { + fn visit_expr(&mut self, expr: &Expr) { + if self.found { + return; } - value.elements().any(|element| match element { - FStringElement::Expression(expr_element) => { - expr_element_contains_await(expr_element, Self::contains_await) - } - FStringElement::Literal(_) => false, - }) + match expr { + Expr::Await(_) => self.found = true, + _ => walk_expr(self, expr), + } } - Expr::StringLiteral(_) - | Expr::BytesLiteral(_) - | Expr::NumberLiteral(_) - | Expr::BooleanLiteral(_) - | Expr::NoneLiteral(_) - | Expr::EllipsisLiteral(_) - | Expr::IpyEscapeCommand(_) => false, } + + let mut visitor = AwaitVisitor::default(); + visitor.visit_expr(expression); + visitor.found } fn compile_expr_fstring(&mut self, fstring: &ExprFString) -> CompileResult<()> { @@ -4352,7 +5796,7 @@ impl Compiler<'_> { FStringPart::Literal(string) => { if string.value.contains(char::REPLACEMENT_CHARACTER) { // might have a surrogate literal; should reparse to be sure - let source = self.source_code.get_range(string.range); + let source = self.source_file.slice(string.range); let value = crate::string_parser::parse_string_literal(source, string.flags.into()); self.emit_load_const(ConstantData::Str { @@ -4376,16 +5820,16 @@ impl Compiler<'_> { fn compile_fstring_elements( &mut self, flags: FStringFlags, - fstring_elements: &FStringElements, + fstring_elements: &InterpolatedStringElements, ) -> CompileResult<()> { let mut element_count = 0; for element in fstring_elements { element_count += 1; match element { - FStringElement::Literal(string) => { + InterpolatedStringElement::Literal(string) => { if string.value.contains(char::REPLACEMENT_CHARACTER) { // might have a surrogate literal; should reparse to be sure - let source = self.source_code.get_range(string.range); + let source = self.source_file.slice(string.range); let value = crate::string_parser::parse_fstring_literal_element( source.into(), flags.into(), @@ -4399,45 +5843,54 @@ impl Compiler<'_> { }); } } - FStringElement::Expression(fstring_expr) => { - let mut conversion = fstring_expr.conversion; + InterpolatedStringElement::Interpolation(fstring_expr) => { + let mut conversion = match fstring_expr.conversion { + ConversionFlag::None => ConvertValueOparg::None, + ConversionFlag::Str => ConvertValueOparg::Str, + ConversionFlag::Repr => ConvertValueOparg::Repr, + ConversionFlag::Ascii => ConvertValueOparg::Ascii, + }; if let Some(DebugText { leading, trailing }) = &fstring_expr.debug_text { let range = fstring_expr.expression.range(); - let source = self.source_code.get_range(range); + let source = self.source_file.slice(range); let text = [leading, source, trailing].concat(); self.emit_load_const(ConstantData::Str { value: text.into() }); element_count += 1; + + // Match CPython behavior: If debug text is present, apply repr conversion. + // if no `format_spec` specified. + // See: https://github.com/python/cpython/blob/f61afca262d3a0aa6a8a501db0b1936c60858e35/Parser/action_helpers.c#L1456 + if matches!( + (conversion, &fstring_expr.format_spec), + (ConvertValueOparg::None, None) + ) { + conversion = ConvertValueOparg::Repr; + } } - match &fstring_expr.format_spec { - None => { - self.emit_load_const(ConstantData::Str { - value: Wtf8Buf::new(), - }); - // Match CPython behavior: If debug text is present, apply repr conversion. - // See: https://github.com/python/cpython/blob/f61afca262d3a0aa6a8a501db0b1936c60858e35/Parser/action_helpers.c#L1456 - if conversion == ConversionFlag::None - && fstring_expr.debug_text.is_some() - { - conversion = ConversionFlag::Repr; - } + self.compile_expression(&fstring_expr.expression)?; + + match conversion { + ConvertValueOparg::None => {} + ConvertValueOparg::Str + | ConvertValueOparg::Repr + | ConvertValueOparg::Ascii => { + emit!(self, Instruction::ConvertValue { oparg: conversion }) } + } + + match &fstring_expr.format_spec { Some(format_spec) => { self.compile_fstring_elements(flags, &format_spec.elements)?; + + emit!(self, Instruction::FormatWithSpec); + } + None => { + emit!(self, Instruction::FormatSimple); } } - - self.compile_expression(&fstring_expr.expression)?; - - let conversion = match conversion { - ConversionFlag::None => bytecode::ConversionFlag::None, - ConversionFlag::Str => bytecode::ConversionFlag::Str, - ConversionFlag::Ascii => bytecode::ConversionFlag::Ascii, - ConversionFlag::Repr => bytecode::ConversionFlag::Repr, - }; - emit!(self, Instruction::FormatValue { conversion }); } } } @@ -4464,30 +5917,27 @@ trait EmitArg { fn emit( self, f: impl FnOnce(OpArgMarker) -> Instruction, - ) -> (Instruction, OpArg, ir::BlockIdx); + ) -> (Instruction, OpArg, BlockIdx); } impl EmitArg for T { - fn emit( - self, - f: impl FnOnce(OpArgMarker) -> Instruction, - ) -> (Instruction, OpArg, ir::BlockIdx) { + fn emit(self, f: impl FnOnce(OpArgMarker) -> Instruction) -> (Instruction, OpArg, BlockIdx) { let (marker, arg) = OpArgMarker::new(self); - (f(marker), arg, ir::BlockIdx::NULL) + (f(marker), arg, BlockIdx::NULL) } } -impl EmitArg for ir::BlockIdx { +impl EmitArg for BlockIdx { fn emit( self, f: impl FnOnce(OpArgMarker) -> Instruction, - ) -> (Instruction, OpArg, ir::BlockIdx) { + ) -> (Instruction, OpArg, BlockIdx) { (f(OpArgMarker::marker()), OpArg::null(), self) } } /// Strips leading whitespace from a docstring. /// -/// The code has been ported from `_PyCompile_CleanDoc` in cpython. -/// `inspect.cleandoc` is also a good reference, but has a few incompatibilities. +/// `inspect.cleandoc` is a good reference, but has a few incompatibilities. +// = _PyCompile_CleanDoc fn clean_doc(doc: &str) -> String { let doc = expandtabs(doc, 8); // First pass: find minimum indentation of any non-blank lines @@ -4632,21 +6082,27 @@ mod ruff_tests { // f'{x}' let expr_x = Expr::Name(ExprName { + node_index: AtomicNodeIndex::NONE, range, id: Name::new("x"), ctx: ExprContext::Load, }); let not_present = &Expr::FString(ExprFString { + node_index: AtomicNodeIndex::NONE, range, value: FStringValue::single(FString { + node_index: AtomicNodeIndex::NONE, range, - elements: vec![FStringElement::Expression(FStringExpressionElement { - range, - expression: Box::new(expr_x), - debug_text: None, - conversion: ConversionFlag::None, - format_spec: None, - })] + elements: vec![InterpolatedStringElement::Interpolation( + InterpolatedElement { + node_index: AtomicNodeIndex::NONE, + range, + expression: Box::new(expr_x), + debug_text: None, + conversion: ConversionFlag::None, + format_spec: None, + }, + )] .into(), flags, }), @@ -4655,24 +6111,31 @@ mod ruff_tests { // f'{await x}' let expr_await_x = Expr::Await(ExprAwait { + node_index: AtomicNodeIndex::NONE, range, value: Box::new(Expr::Name(ExprName { + node_index: AtomicNodeIndex::NONE, range, id: Name::new("x"), ctx: ExprContext::Load, })), }); let present = &Expr::FString(ExprFString { + node_index: AtomicNodeIndex::NONE, range, value: FStringValue::single(FString { + node_index: AtomicNodeIndex::NONE, range, - elements: vec![FStringElement::Expression(FStringExpressionElement { - range, - expression: Box::new(expr_await_x), - debug_text: None, - conversion: ConversionFlag::None, - format_spec: None, - })] + elements: vec![InterpolatedStringElement::Interpolation( + InterpolatedElement { + node_index: AtomicNodeIndex::NONE, + range, + expression: Box::new(expr_await_x), + debug_text: None, + conversion: ConversionFlag::None, + format_spec: None, + }, + )] .into(), flags, }), @@ -4681,39 +6144,51 @@ mod ruff_tests { // f'{x:{await y}}' let expr_x = Expr::Name(ExprName { + node_index: AtomicNodeIndex::NONE, range, id: Name::new("x"), ctx: ExprContext::Load, }); let expr_await_y = Expr::Await(ExprAwait { + node_index: AtomicNodeIndex::NONE, range, value: Box::new(Expr::Name(ExprName { + node_index: AtomicNodeIndex::NONE, range, id: Name::new("y"), ctx: ExprContext::Load, })), }); let present = &Expr::FString(ExprFString { + node_index: AtomicNodeIndex::NONE, range, value: FStringValue::single(FString { + node_index: AtomicNodeIndex::NONE, range, - elements: vec![FStringElement::Expression(FStringExpressionElement { - range, - expression: Box::new(expr_x), - debug_text: None, - conversion: ConversionFlag::None, - format_spec: Some(Box::new(FStringFormatSpec { + elements: vec![InterpolatedStringElement::Interpolation( + InterpolatedElement { + node_index: AtomicNodeIndex::NONE, range, - elements: vec![FStringElement::Expression(FStringExpressionElement { + expression: Box::new(expr_x), + debug_text: None, + conversion: ConversionFlag::None, + format_spec: Some(Box::new(InterpolatedStringFormatSpec { + node_index: AtomicNodeIndex::NONE, range, - expression: Box::new(expr_await_y), - debug_text: None, - conversion: ConversionFlag::None, - format_spec: None, - })] - .into(), - })), - })] + elements: vec![InterpolatedStringElement::Interpolation( + InterpolatedElement { + node_index: AtomicNodeIndex::NONE, + range, + expression: Box::new(expr_await_y), + debug_text: None, + conversion: ConversionFlag::None, + format_spec: None, + }, + )] + .into(), + })), + }, + )] .into(), flags, }), @@ -4725,24 +6200,27 @@ mod ruff_tests { #[cfg(test)] mod tests { use super::*; + use rustpython_compiler_core::SourceFileBuilder; fn compile_exec(source: &str) -> CodeObject { let opts = CompileOpts::default(); - let source_code = SourceCode::new("source_path", source); - let parsed = - ruff_python_parser::parse(source_code.text, ruff_python_parser::Mode::Module.into()) - .unwrap(); + let source_file = SourceFileBuilder::new("source_path", source).finish(); + let parsed = ruff_python_parser::parse( + source_file.source_text(), + ruff_python_parser::Mode::Module.into(), + ) + .unwrap(); let ast = parsed.into_syntax(); let ast = match ast { ruff_python_ast::Mod::Module(stmts) => stmts, _ => unreachable!(), }; - let symbol_table = SymbolTable::scan_program(&ast, source_code.clone()) - .map_err(|e| e.into_codegen_error(source_code.path.to_owned())) + let symbol_table = SymbolTable::scan_program(&ast, source_file.clone()) + .map_err(|e| e.into_codegen_error(source_file.name().to_owned())) .unwrap(); - let mut compiler = Compiler::new(opts, source_code, "".to_owned()); + let mut compiler = Compiler::new(opts, source_file, "".to_owned()); compiler.compile_program(&ast, symbol_table).unwrap(); - compiler.pop_code_object() + compiler.exit_scope() } macro_rules! assert_dis_snapshot { diff --git a/compiler/codegen/src/error.rs b/crates/codegen/src/error.rs similarity index 93% rename from compiler/codegen/src/error.rs rename to crates/codegen/src/error.rs index 5e0ac12934..70e2f13f25 100644 --- a/compiler/codegen/src/error.rs +++ b/crates/codegen/src/error.rs @@ -1,4 +1,4 @@ -use ruff_source_file::SourceLocation; +use rustpython_compiler_core::SourceLocation; use std::fmt::{self, Display}; use thiserror::Error; @@ -88,6 +88,8 @@ pub enum CodegenErrorType { UnreachablePattern(PatternUnreachableReason), RepeatedAttributePattern, ConflictingNameBindPattern, + /// break/continue/return inside except* block + BreakContinueReturnInExceptStar, NotImplementedYet, // RustPython marker for unimplemented features } @@ -148,6 +150,12 @@ impl fmt::Display for CodegenErrorType { ConflictingNameBindPattern => { write!(f, "alternative patterns bind different names") } + BreakContinueReturnInExceptStar => { + write!( + f, + "'break', 'continue' and 'return' cannot appear in an except* block" + ) + } NotImplementedYet => { write!(f, "RustPython does not implement this feature yet") } diff --git a/crates/codegen/src/ir.rs b/crates/codegen/src/ir.rs new file mode 100644 index 0000000000..de0126f112 --- /dev/null +++ b/crates/codegen/src/ir.rs @@ -0,0 +1,543 @@ +use std::ops; + +use crate::{IndexMap, IndexSet, error::InternalError}; +use rustpython_compiler_core::{ + OneIndexed, SourceLocation, + bytecode::{ + CodeFlags, CodeObject, CodeUnit, CodeUnits, ConstantData, InstrDisplayContext, Instruction, + Label, OpArg, PyCodeLocationInfoKind, + }, +}; + +/// Metadata for a code unit +// = _PyCompile_CodeUnitMetadata +#[derive(Clone, Debug)] +pub struct CodeUnitMetadata { + pub name: String, // u_name (obj_name) + pub qualname: Option, // u_qualname + pub consts: IndexSet, // u_consts + pub names: IndexSet, // u_names + pub varnames: IndexSet, // u_varnames + pub cellvars: IndexSet, // u_cellvars + pub freevars: IndexSet, // u_freevars + pub fast_hidden: IndexMap, // u_fast_hidden + pub argcount: u32, // u_argcount + pub posonlyargcount: u32, // u_posonlyargcount + pub kwonlyargcount: u32, // u_kwonlyargcount + pub firstlineno: OneIndexed, // u_firstlineno +} +// use rustpython_parser_core::source_code::{LineNumber, SourceLocation}; + +#[derive(Copy, Clone, PartialEq, Eq, Debug)] +pub struct BlockIdx(u32); + +impl BlockIdx { + pub const NULL: Self = Self::new(u32::MAX); + + /// Creates a new instance of [`BlockIdx`] from a [`u32`]. + #[must_use] + pub const fn new(value: u32) -> Self { + Self(value) + } + + /// Returns the inner value as a [`usize`]. + #[must_use] + pub const fn idx(self) -> usize { + self.0 as usize + } +} + +impl From for u32 { + fn from(block_idx: BlockIdx) -> Self { + block_idx.0 + } +} + +impl ops::Index for [Block] { + type Output = Block; + + fn index(&self, idx: BlockIdx) -> &Block { + &self[idx.idx()] + } +} + +impl ops::IndexMut for [Block] { + fn index_mut(&mut self, idx: BlockIdx) -> &mut Block { + &mut self[idx.idx()] + } +} + +impl ops::Index for Vec { + type Output = Block; + + fn index(&self, idx: BlockIdx) -> &Block { + &self[idx.idx()] + } +} + +impl ops::IndexMut for Vec { + fn index_mut(&mut self, idx: BlockIdx) -> &mut Block { + &mut self[idx.idx()] + } +} + +#[derive(Debug, Clone)] +pub struct InstructionInfo { + pub instr: Instruction, + pub arg: OpArg, + pub target: BlockIdx, + // pub range: TextRange, + pub location: SourceLocation, + // TODO: end_location for debug ranges +} + +// spell-checker:ignore petgraph +// TODO: look into using petgraph for handling blocks and stuff? it's heavier than this, but it +// might enable more analysis/optimizations +#[derive(Debug)] +pub struct Block { + pub instructions: Vec, + pub next: BlockIdx, +} + +impl Default for Block { + fn default() -> Self { + Self { + instructions: Vec::new(), + next: BlockIdx::NULL, + } + } +} + +pub struct CodeInfo { + pub flags: CodeFlags, + pub source_path: String, + pub private: Option, // For private name mangling, mostly for class + + pub blocks: Vec, + pub current_block: BlockIdx, + + pub metadata: CodeUnitMetadata, + + // For class scopes: attributes accessed via self.X + pub static_attributes: Option>, + + // True if compiling an inlined comprehension + pub in_inlined_comp: bool, + + // Block stack for tracking nested control structures + pub fblock: Vec, + + // Reference to the symbol table for this scope + pub symbol_table_index: usize, +} + +impl CodeInfo { + pub fn finalize_code(mut self, optimize: u8) -> crate::InternalResult { + if optimize > 0 { + self.dce(); + } + + let max_stackdepth = self.max_stackdepth()?; + let cell2arg = self.cell2arg(); + + let Self { + flags, + source_path, + private: _, // private is only used during compilation + + mut blocks, + current_block: _, + metadata, + static_attributes: _, + in_inlined_comp: _, + fblock: _, + symbol_table_index: _, + } = self; + + let CodeUnitMetadata { + name: obj_name, + qualname, + consts: constants, + names: name_cache, + varnames: varname_cache, + cellvars: cellvar_cache, + freevars: freevar_cache, + fast_hidden: _, + argcount: arg_count, + posonlyargcount: posonlyarg_count, + kwonlyargcount: kwonlyarg_count, + firstlineno: first_line_number, + } = metadata; + + let mut instructions = Vec::new(); + let mut locations = Vec::new(); + + let mut block_to_offset = vec![Label(0); blocks.len()]; + loop { + let mut num_instructions = 0; + for (idx, block) in iter_blocks(&blocks) { + block_to_offset[idx.idx()] = Label(num_instructions as u32); + for instr in &block.instructions { + num_instructions += instr.arg.instr_size() + } + } + + instructions.reserve_exact(num_instructions); + locations.reserve_exact(num_instructions); + + let mut recompile_extended_arg = false; + let mut next_block = BlockIdx(0); + while next_block != BlockIdx::NULL { + let block = &mut blocks[next_block]; + for info in &mut block.instructions { + let (op, arg, target) = (info.instr, &mut info.arg, info.target); + if target != BlockIdx::NULL { + let new_arg = OpArg(block_to_offset[target.idx()].0); + recompile_extended_arg |= new_arg.instr_size() != arg.instr_size(); + *arg = new_arg; + } + let (extras, lo_arg) = arg.split(); + locations.extend(std::iter::repeat_n(info.location, arg.instr_size())); + instructions.extend( + extras + .map(|byte| CodeUnit::new(Instruction::ExtendedArg, byte)) + .chain([CodeUnit { op, arg: lo_arg }]), + ); + } + next_block = block.next; + } + + if !recompile_extended_arg { + break; + } + + instructions.clear(); + locations.clear() + } + + // Generate linetable from locations + let linetable = generate_linetable(&locations, first_line_number.get() as i32); + + Ok(CodeObject { + flags, + posonlyarg_count, + arg_count, + kwonlyarg_count, + source_path, + first_line_number: Some(first_line_number), + obj_name: obj_name.clone(), + qualname: qualname.unwrap_or(obj_name), + + max_stackdepth, + instructions: CodeUnits::from(instructions), + locations: locations.into_boxed_slice(), + constants: constants.into_iter().collect(), + names: name_cache.into_iter().collect(), + varnames: varname_cache.into_iter().collect(), + cellvars: cellvar_cache.into_iter().collect(), + freevars: freevar_cache.into_iter().collect(), + cell2arg, + linetable, + exceptiontable: Box::new([]), // TODO: Generate actual exception table + }) + } + + fn cell2arg(&self) -> Option> { + if self.metadata.cellvars.is_empty() { + return None; + } + + let total_args = self.metadata.argcount + + self.metadata.kwonlyargcount + + self.flags.contains(CodeFlags::HAS_VARARGS) as u32 + + self.flags.contains(CodeFlags::HAS_VARKEYWORDS) as u32; + + let mut found_cellarg = false; + let cell2arg = self + .metadata + .cellvars + .iter() + .map(|var| { + self.metadata + .varnames + .get_index_of(var) + // check that it's actually an arg + .filter(|i| *i < total_args as usize) + .map_or(-1, |i| { + found_cellarg = true; + i as i32 + }) + }) + .collect::>(); + + if found_cellarg { Some(cell2arg) } else { None } + } + + fn dce(&mut self) { + for block in &mut self.blocks { + let mut last_instr = None; + for (i, ins) in block.instructions.iter().enumerate() { + if ins.instr.unconditional_branch() { + last_instr = Some(i); + break; + } + } + if let Some(i) = last_instr { + block.instructions.truncate(i + 1); + } + } + } + + fn max_stackdepth(&self) -> crate::InternalResult { + let mut maxdepth = 0u32; + let mut stack = Vec::with_capacity(self.blocks.len()); + let mut start_depths = vec![u32::MAX; self.blocks.len()]; + start_depths[0] = 0; + stack.push(BlockIdx(0)); + const DEBUG: bool = false; + 'process_blocks: while let Some(block) = stack.pop() { + let mut depth = start_depths[block.idx()]; + if DEBUG { + eprintln!("===BLOCK {}===", block.0); + } + let block = &self.blocks[block]; + for ins in &block.instructions { + let instr = &ins.instr; + let effect = instr.stack_effect(ins.arg, false); + if DEBUG { + let display_arg = if ins.target == BlockIdx::NULL { + ins.arg + } else { + OpArg(ins.target.0) + }; + let instr_display = instr.display(display_arg, self); + eprint!("{instr_display}: {depth} {effect:+} => "); + } + let new_depth = depth.checked_add_signed(effect).ok_or({ + if effect < 0 { + InternalError::StackUnderflow + } else { + InternalError::StackOverflow + } + })?; + if DEBUG { + eprintln!("{new_depth}"); + } + if new_depth > maxdepth { + maxdepth = new_depth + } + // we don't want to worry about Break/Continue, they use unwinding to jump to + // their targets and as such the stack size is taken care of in frame.rs by setting + // it back to the level it was at when SetupLoop was run + if ins.target != BlockIdx::NULL + && !matches!( + instr, + Instruction::Continue { .. } | Instruction::Break { .. } + ) + { + let effect = instr.stack_effect(ins.arg, true); + let target_depth = depth.checked_add_signed(effect).ok_or({ + if effect < 0 { + InternalError::StackUnderflow + } else { + InternalError::StackOverflow + } + })?; + if target_depth > maxdepth { + maxdepth = target_depth + } + stackdepth_push(&mut stack, &mut start_depths, ins.target, target_depth); + } + depth = new_depth; + if instr.unconditional_branch() { + continue 'process_blocks; + } + } + // Only push next block if it's not NULL + if block.next != BlockIdx::NULL { + stackdepth_push(&mut stack, &mut start_depths, block.next, depth); + } + } + if DEBUG { + eprintln!("DONE: {maxdepth}"); + } + Ok(maxdepth) + } +} + +impl InstrDisplayContext for CodeInfo { + type Constant = ConstantData; + fn get_constant(&self, i: usize) -> &ConstantData { + &self.metadata.consts[i] + } + fn get_name(&self, i: usize) -> &str { + self.metadata.names[i].as_ref() + } + fn get_varname(&self, i: usize) -> &str { + self.metadata.varnames[i].as_ref() + } + fn get_cell_name(&self, i: usize) -> &str { + self.metadata + .cellvars + .get_index(i) + .unwrap_or_else(|| &self.metadata.freevars[i - self.metadata.cellvars.len()]) + .as_ref() + } +} + +fn stackdepth_push( + stack: &mut Vec, + start_depths: &mut [u32], + target: BlockIdx, + depth: u32, +) { + let block_depth = &mut start_depths[target.idx()]; + if *block_depth == u32::MAX || depth > *block_depth { + *block_depth = depth; + stack.push(target); + } +} + +fn iter_blocks(blocks: &[Block]) -> impl Iterator + '_ { + let mut next = BlockIdx(0); + std::iter::from_fn(move || { + if next == BlockIdx::NULL { + return None; + } + let (idx, b) = (next, &blocks[next]); + next = b.next; + Some((idx, b)) + }) +} + +/// Generate CPython 3.11+ format linetable from source locations +fn generate_linetable(locations: &[SourceLocation], first_line: i32) -> Box<[u8]> { + if locations.is_empty() { + return Box::new([]); + } + + let mut linetable = Vec::new(); + // Initialize prev_line to first_line + // The first entry's delta is relative to co_firstlineno + let mut prev_line = first_line; + let mut i = 0; + + while i < locations.len() { + let loc = &locations[i]; + + // Count consecutive instructions with the same location + let mut length = 1; + while i + length < locations.len() && locations[i + length] == locations[i] { + length += 1; + } + + // Process in chunks of up to 8 instructions + while length > 0 { + let entry_length = length.min(8); + + // Get line and column information + // SourceLocation always has row and column (both are OneIndexed) + let line = loc.line.get() as i32; + let col = loc.character_offset.to_zero_indexed() as i32; + + let line_delta = line - prev_line; + + // Choose the appropriate encoding based on line delta and column info + // Note: SourceLocation always has valid column, so we never get NO_COLUMNS case + if line_delta == 0 { + let end_col = col; // Use same column for end (no range info available) + + if col < 80 && end_col - col < 16 && end_col >= col { + // Short form (codes 0-9) for common cases + let code = (col / 8).min(9) as u8; // Short0 to Short9 + linetable.push(0x80 | (code << 3) | ((entry_length - 1) as u8)); + let col_byte = (((col % 8) as u8) << 4) | ((end_col - col) as u8 & 0xf); + linetable.push(col_byte); + } else if col < 128 && end_col < 128 { + // One-line form (code 10) for same line + linetable.push( + 0x80 | ((PyCodeLocationInfoKind::OneLine0 as u8) << 3) + | ((entry_length - 1) as u8), + ); + linetable.push(col as u8); + linetable.push(end_col as u8); + } else { + // Long form for columns >= 128 + linetable.push( + 0x80 | ((PyCodeLocationInfoKind::Long as u8) << 3) + | ((entry_length - 1) as u8), + ); + write_signed_varint(&mut linetable, 0); // line_delta = 0 + write_varint(&mut linetable, 0); // end_line delta = 0 + write_varint(&mut linetable, (col as u32) + 1); // column + 1 for encoding + write_varint(&mut linetable, (end_col as u32) + 1); // end_col + 1 + } + } else if line_delta > 0 && line_delta < 3 + /* && column.is_some() */ + { + // One-line form (codes 11-12) for line deltas 1-2 + let end_col = col; // Use same column for end + + if col < 128 && end_col < 128 { + let code = (PyCodeLocationInfoKind::OneLine0 as u8) + (line_delta as u8); // 11 for delta=1, 12 for delta=2 + linetable.push(0x80 | (code << 3) | ((entry_length - 1) as u8)); + linetable.push(col as u8); + linetable.push(end_col as u8); + } else { + // Long form for columns >= 128 or negative line delta + linetable.push( + 0x80 | ((PyCodeLocationInfoKind::Long as u8) << 3) + | ((entry_length - 1) as u8), + ); + write_signed_varint(&mut linetable, line_delta); + write_varint(&mut linetable, 0); // end_line delta = 0 + write_varint(&mut linetable, (col as u32) + 1); // column + 1 for encoding + write_varint(&mut linetable, (end_col as u32) + 1); // end_col + 1 + } + } else { + // Long form (code 14) for all other cases + // This handles: line_delta < 0, line_delta >= 3, or columns >= 128 + let end_col = col; // Use same column for end + linetable.push( + 0x80 | ((PyCodeLocationInfoKind::Long as u8) << 3) | ((entry_length - 1) as u8), + ); + write_signed_varint(&mut linetable, line_delta); + write_varint(&mut linetable, 0); // end_line delta = 0 + write_varint(&mut linetable, (col as u32) + 1); // column + 1 for encoding + write_varint(&mut linetable, (end_col as u32) + 1); // end_col + 1 + } + + prev_line = line; + length -= entry_length; + i += entry_length; + } + } + + linetable.into_boxed_slice() +} + +/// Write a variable-length unsigned integer (6-bit chunks) +/// Returns the number of bytes written +fn write_varint(buf: &mut Vec, mut val: u32) -> usize { + let start_len = buf.len(); + while val >= 64 { + buf.push(0x40 | (val & 0x3f) as u8); + val >>= 6; + } + buf.push(val as u8); + buf.len() - start_len +} + +/// Write a variable-length signed integer +/// Returns the number of bytes written +fn write_signed_varint(buf: &mut Vec, val: i32) -> usize { + let uval = if val < 0 { + // (unsigned int)(-val) has an undefined behavior for INT_MIN + // So we use (0 - val as u32) to handle it correctly + ((0u32.wrapping_sub(val as u32)) << 1) | 1 + } else { + (val as u32) << 1 + }; + write_varint(buf, uval) +} diff --git a/compiler/codegen/src/lib.rs b/crates/codegen/src/lib.rs similarity index 97% rename from compiler/codegen/src/lib.rs rename to crates/codegen/src/lib.rs index 9b444de994..291b57d7f6 100644 --- a/compiler/codegen/src/lib.rs +++ b/crates/codegen/src/lib.rs @@ -56,6 +56,7 @@ impl ToPythonName for Expr { Self::Starred { .. } => "starred", Self::Slice { .. } => "slice", Self::FString { .. } => "f-string expression", + Self::TString { .. } => "t-string expression", Self::Name { .. } => "name", Self::Lambda { .. } => "lambda", Self::If { .. } => "conditional expression", diff --git a/crates/codegen/src/snapshots/rustpython_codegen__compile__tests__if_ands.snap b/crates/codegen/src/snapshots/rustpython_codegen__compile__tests__if_ands.snap new file mode 100644 index 0000000000..4c9c29887e --- /dev/null +++ b/crates/codegen/src/snapshots/rustpython_codegen__compile__tests__if_ands.snap @@ -0,0 +1,12 @@ +--- +source: crates/codegen/src/compile.rs +expression: "compile_exec(\"\\\nif True and False and False:\n pass\n\")" +--- + 1 0 LOAD_CONST (True) + 1 POP_JUMP_IF_FALSE (6) + 2 LOAD_CONST (False) + 3 POP_JUMP_IF_FALSE (6) + 4 LOAD_CONST (False) + 5 POP_JUMP_IF_FALSE (6) + + 2 >> 6 RETURN_CONST (None) diff --git a/crates/codegen/src/snapshots/rustpython_codegen__compile__tests__if_mixed.snap b/crates/codegen/src/snapshots/rustpython_codegen__compile__tests__if_mixed.snap new file mode 100644 index 0000000000..a93479df96 --- /dev/null +++ b/crates/codegen/src/snapshots/rustpython_codegen__compile__tests__if_mixed.snap @@ -0,0 +1,14 @@ +--- +source: crates/codegen/src/compile.rs +expression: "compile_exec(\"\\\nif (True and False) or (False and True):\n pass\n\")" +--- + 1 0 LOAD_CONST (True) + 1 POP_JUMP_IF_FALSE (4) + 2 LOAD_CONST (False) + 3 POP_JUMP_IF_TRUE (8) + >> 4 LOAD_CONST (False) + 5 POP_JUMP_IF_FALSE (8) + 6 LOAD_CONST (True) + 7 POP_JUMP_IF_FALSE (8) + + 2 >> 8 RETURN_CONST (None) diff --git a/crates/codegen/src/snapshots/rustpython_codegen__compile__tests__if_ors.snap b/crates/codegen/src/snapshots/rustpython_codegen__compile__tests__if_ors.snap new file mode 100644 index 0000000000..37a862cc65 --- /dev/null +++ b/crates/codegen/src/snapshots/rustpython_codegen__compile__tests__if_ors.snap @@ -0,0 +1,12 @@ +--- +source: crates/codegen/src/compile.rs +expression: "compile_exec(\"\\\nif True or False or False:\n pass\n\")" +--- + 1 0 LOAD_CONST (True) + 1 POP_JUMP_IF_TRUE (6) + 2 LOAD_CONST (False) + 3 POP_JUMP_IF_TRUE (6) + 4 LOAD_CONST (False) + 5 POP_JUMP_IF_FALSE (6) + + 2 >> 6 RETURN_CONST (None) diff --git a/crates/codegen/src/snapshots/rustpython_codegen__compile__tests__nested_double_async_with.snap b/crates/codegen/src/snapshots/rustpython_codegen__compile__tests__nested_double_async_with.snap new file mode 100644 index 0000000000..6cd7d4d523 --- /dev/null +++ b/crates/codegen/src/snapshots/rustpython_codegen__compile__tests__nested_double_async_with.snap @@ -0,0 +1,86 @@ +--- +source: crates/codegen/src/compile.rs +expression: "compile_exec(\"\\\nfor stop_exc in (StopIteration('spam'), StopAsyncIteration('ham')):\n with self.subTest(type=type(stop_exc)):\n try:\n async with egg():\n raise stop_exc\n except Exception as ex:\n self.assertIs(ex, stop_exc)\n else:\n self.fail(f'{stop_exc} was suppressed')\n\")" +--- + 1 0 SETUP_LOOP + 1 LOAD_NAME_ANY (0, StopIteration) + 2 LOAD_CONST ("spam") + 3 CALL_FUNCTION_POSITIONAL(1) + 4 LOAD_NAME_ANY (1, StopAsyncIteration) + 5 LOAD_CONST ("ham") + 6 CALL_FUNCTION_POSITIONAL(1) + 7 BUILD_TUPLE (2) + 8 GET_ITER + >> 9 FOR_ITER (71) + 10 STORE_LOCAL (2, stop_exc) + + 2 11 LOAD_NAME_ANY (3, self) + 12 LOAD_METHOD (4, subTest) + 13 LOAD_NAME_ANY (5, type) + 14 LOAD_NAME_ANY (2, stop_exc) + 15 CALL_FUNCTION_POSITIONAL(1) + 16 LOAD_CONST (("type")) + 17 CALL_METHOD_KEYWORD (1) + 18 SETUP_WITH (68) + 19 POP_TOP + + 3 20 SETUP_EXCEPT (42) + + 4 21 LOAD_NAME_ANY (6, egg) + 22 CALL_FUNCTION_POSITIONAL(0) + 23 BEFORE_ASYNC_WITH + 24 GET_AWAITABLE + 25 LOAD_CONST (None) + 26 YIELD_FROM + 27 RESUME (3) + 28 SETUP_ASYNC_WITH (34) + 29 POP_TOP + + 5 30 LOAD_NAME_ANY (2, stop_exc) + 31 RAISE (Raise) + + 4 32 POP_BLOCK + 33 ENTER_FINALLY + >> 34 WITH_CLEANUP_START + 35 GET_AWAITABLE + 36 LOAD_CONST (None) + 37 YIELD_FROM + 38 RESUME (3) + 39 WITH_CLEANUP_FINISH + 40 POP_BLOCK + 41 JUMP (58) + >> 42 COPY (1) + + 6 43 LOAD_NAME_ANY (7, Exception) + 44 JUMP_IF_NOT_EXC_MATCH(57) + 45 STORE_LOCAL (8, ex) + + 7 46 LOAD_NAME_ANY (3, self) + 47 LOAD_METHOD (9, assertIs) + 48 LOAD_NAME_ANY (8, ex) + 49 LOAD_NAME_ANY (2, stop_exc) + 50 CALL_METHOD_POSITIONAL(2) + 51 POP_TOP + 52 POP_EXCEPTION + 53 LOAD_CONST (None) + 54 STORE_LOCAL (8, ex) + 55 DELETE_LOCAL (8, ex) + 56 JUMP (66) + >> 57 RAISE (Reraise) + + 9 >> 58 LOAD_NAME_ANY (3, self) + 59 LOAD_METHOD (10, fail) + 60 LOAD_NAME_ANY (2, stop_exc) + 61 FORMAT_SIMPLE + 62 LOAD_CONST (" was suppressed") + 63 BUILD_STRING (2) + 64 CALL_METHOD_POSITIONAL(1) + 65 POP_TOP + + 2 >> 66 POP_BLOCK + 67 ENTER_FINALLY + >> 68 WITH_CLEANUP_START + 69 WITH_CLEANUP_FINISH + 70 JUMP (9) + >> 71 POP_BLOCK + 72 RETURN_CONST (None) diff --git a/compiler/codegen/src/snapshots/rustpython_compiler_core__compile__tests__if_ands.snap b/crates/codegen/src/snapshots/rustpython_compiler_core__compile__tests__if_ands.snap similarity index 72% rename from compiler/codegen/src/snapshots/rustpython_compiler_core__compile__tests__if_ands.snap rename to crates/codegen/src/snapshots/rustpython_compiler_core__compile__tests__if_ands.snap index d80f10dfee..bc88cf2349 100644 --- a/compiler/codegen/src/snapshots/rustpython_compiler_core__compile__tests__if_ands.snap +++ b/crates/codegen/src/snapshots/rustpython_compiler_core__compile__tests__if_ands.snap @@ -3,11 +3,11 @@ source: compiler/src/compile.rs expression: "compile_exec(\"\\\nif True and False and False:\n pass\n\")" --- 1 0 LoadConst (True) - 1 JumpIfFalse (6) + 1 PopJumpIfFalse (6) 2 LoadConst (False) - 3 JumpIfFalse (6) + 3 PopJumpIfFalse (6) 4 LoadConst (False) - 5 JumpIfFalse (6) + 5 PopJumpIfFalse (6) 2 >> 6 LoadConst (None) 7 ReturnValue diff --git a/compiler/codegen/src/snapshots/rustpython_compiler_core__compile__tests__if_mixed.snap b/crates/codegen/src/snapshots/rustpython_compiler_core__compile__tests__if_mixed.snap similarity index 69% rename from compiler/codegen/src/snapshots/rustpython_compiler_core__compile__tests__if_mixed.snap rename to crates/codegen/src/snapshots/rustpython_compiler_core__compile__tests__if_mixed.snap index 0a9175bb12..b19cbb119d 100644 --- a/compiler/codegen/src/snapshots/rustpython_compiler_core__compile__tests__if_mixed.snap +++ b/crates/codegen/src/snapshots/rustpython_compiler_core__compile__tests__if_mixed.snap @@ -3,13 +3,13 @@ source: compiler/src/compile.rs expression: "compile_exec(\"\\\nif (True and False) or (False and True):\n pass\n\")" --- 1 0 LoadConst (True) - 1 JumpIfFalse (4) + 1 PopJumpIfFalse (4) 2 LoadConst (False) - 3 JumpIfTrue (8) + 3 PopJumpIfTrue (8) >> 4 LoadConst (False) - 5 JumpIfFalse (8) + 5 PopJumpIfFalse (8) 6 LoadConst (True) - 7 JumpIfFalse (8) + 7 PopJumpIfFalse (8) 2 >> 8 LoadConst (None) 9 ReturnValue diff --git a/compiler/codegen/src/snapshots/rustpython_compiler_core__compile__tests__if_ors.snap b/crates/codegen/src/snapshots/rustpython_compiler_core__compile__tests__if_ors.snap similarity index 72% rename from compiler/codegen/src/snapshots/rustpython_compiler_core__compile__tests__if_ors.snap rename to crates/codegen/src/snapshots/rustpython_compiler_core__compile__tests__if_ors.snap index 4b812639b2..3d1f5a1d6f 100644 --- a/compiler/codegen/src/snapshots/rustpython_compiler_core__compile__tests__if_ors.snap +++ b/crates/codegen/src/snapshots/rustpython_compiler_core__compile__tests__if_ors.snap @@ -3,11 +3,11 @@ source: compiler/src/compile.rs expression: "compile_exec(\"\\\nif True or False or False:\n pass\n\")" --- 1 0 LoadConst (True) - 1 JumpIfTrue (6) + 1 PopJumpIfTrue (6) 2 LoadConst (False) - 3 JumpIfTrue (6) + 3 PopJumpIfTrue (6) 4 LoadConst (False) - 5 JumpIfFalse (6) + 5 PopJumpIfFalse (6) 2 >> 6 LoadConst (None) 7 ReturnValue diff --git a/compiler/codegen/src/snapshots/rustpython_compiler_core__compile__tests__nested_double_async_with.snap b/crates/codegen/src/snapshots/rustpython_compiler_core__compile__tests__nested_double_async_with.snap similarity index 98% rename from compiler/codegen/src/snapshots/rustpython_compiler_core__compile__tests__nested_double_async_with.snap rename to crates/codegen/src/snapshots/rustpython_compiler_core__compile__tests__nested_double_async_with.snap index 79a1a86ad1..589f3210cf 100644 --- a/compiler/codegen/src/snapshots/rustpython_compiler_core__compile__tests__nested_double_async_with.snap +++ b/crates/codegen/src/snapshots/rustpython_compiler_core__compile__tests__nested_double_async_with.snap @@ -51,7 +51,7 @@ expression: "compile_exec(\"\\\nfor stop_exc in (StopIteration('spam'), StopAsyn 6 41 LoadNameAny (7, Exception) 42 TestOperation (ExceptionMatch) - 43 JumpIfFalse (53) + 43 PopJumpIfFalse (53) 44 StoreLocal (8, ex) 7 45 LoadNameAny (3, self) diff --git a/compiler/codegen/src/string_parser.rs b/crates/codegen/src/string_parser.rs similarity index 100% rename from compiler/codegen/src/string_parser.rs rename to crates/codegen/src/string_parser.rs diff --git a/compiler/codegen/src/symboltable.rs b/crates/codegen/src/symboltable.rs similarity index 81% rename from compiler/codegen/src/symboltable.rs rename to crates/codegen/src/symboltable.rs index 2949f39a9f..3c8454b9e2 100644 --- a/compiler/codegen/src/symboltable.rs +++ b/crates/codegen/src/symboltable.rs @@ -19,9 +19,7 @@ use ruff_python_ast::{ Stmt, TypeParam, TypeParamParamSpec, TypeParamTypeVar, TypeParamTypeVarTuple, TypeParams, }; use ruff_text_size::{Ranged, TextRange}; -use rustpython_compiler_source::{SourceCode, SourceLocation}; -// use rustpython_ast::{self as ast, located::Located}; -// use rustpython_parser_core::source_code::{LineNumber, SourceLocation}; +use rustpython_compiler_core::{PositionEncoding, SourceFile, SourceLocation}; use std::{borrow::Cow, fmt}; /// Captures all symbols in the current scope, and has a list of sub-scopes in this scope. @@ -31,7 +29,7 @@ pub struct SymbolTable { pub name: String, /// The type of symbol table - pub typ: SymbolTableType, + pub typ: CompilerScope, /// The line number in the source code where this symboltable begins. pub line_number: u32, @@ -45,10 +43,22 @@ pub struct SymbolTable { /// A list of sub-scopes in the order as found in the /// AST nodes. pub sub_tables: Vec, + + /// Variable names in definition order (parameters first, then locals) + pub varnames: Vec, + + /// Whether this class scope needs an implicit __class__ cell + pub needs_class_closure: bool, + + /// Whether this class scope needs an implicit __classdict__ cell + pub needs_classdict: bool, + + /// Whether this type param scope can see the parent class scope + pub can_see_class_scope: bool, } impl SymbolTable { - fn new(name: String, typ: SymbolTableType, line_number: u32, is_nested: bool) -> Self { + fn new(name: String, typ: CompilerScope, line_number: u32, is_nested: bool) -> Self { Self { name, typ, @@ -56,40 +66,49 @@ impl SymbolTable { is_nested, symbols: IndexMap::default(), sub_tables: vec![], + varnames: Vec::new(), + needs_class_closure: false, + needs_classdict: false, + can_see_class_scope: false, } } - pub fn scan_program( - program: &ModModule, - source_code: SourceCode<'_>, - ) -> SymbolTableResult { - let mut builder = SymbolTableBuilder::new(source_code); + pub fn scan_program(program: &ModModule, source_file: SourceFile) -> SymbolTableResult { + let mut builder = SymbolTableBuilder::new(source_file); builder.scan_statements(program.body.as_ref())?; builder.finish() } - pub fn scan_expr(expr: &ModExpression, source_code: SourceCode<'_>) -> SymbolTableResult { - let mut builder = SymbolTableBuilder::new(source_code); + pub fn scan_expr(expr: &ModExpression, source_file: SourceFile) -> SymbolTableResult { + let mut builder = SymbolTableBuilder::new(source_file); builder.scan_expression(expr.body.as_ref(), ExpressionContext::Load)?; builder.finish() } + + pub fn lookup(&self, name: &str) -> Option<&Symbol> { + self.symbols.get(name) + } } #[derive(Debug, Clone, Copy, PartialEq, Eq)] -pub enum SymbolTableType { +pub enum CompilerScope { Module, Class, Function, + AsyncFunction, + Lambda, Comprehension, TypeParams, } -impl fmt::Display for SymbolTableType { +impl fmt::Display for CompilerScope { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { match self { Self::Module => write!(f, "module"), Self::Class => write!(f, "class"), Self::Function => write!(f, "function"), + Self::AsyncFunction => write!(f, "async function"), + Self::Lambda => write!(f, "lambda"), Self::Comprehension => write!(f, "comprehension"), Self::TypeParams => write!(f, "type parameter"), // TODO missing types from the C implementation @@ -196,12 +215,6 @@ impl SymbolTableError { type SymbolTableResult = Result; -impl SymbolTable { - pub fn lookup(&self, name: &str) -> Option<&Symbol> { - self.symbols.get(name) - } -} - impl std::fmt::Debug for SymbolTable { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { write!( @@ -221,6 +234,30 @@ fn analyze_symbol_table(symbol_table: &mut SymbolTable) -> SymbolTableResult { analyzer.analyze_symbol_table(symbol_table) } +/* Drop __class__ and __classdict__ from free variables in class scope + and set the appropriate flags. Equivalent to CPython's drop_class_free(). + See: https://github.com/python/cpython/blob/main/Python/symtable.c#L884 +*/ +fn drop_class_free(symbol_table: &mut SymbolTable) { + // Check if __class__ is used as a free variable + if let Some(class_symbol) = symbol_table.symbols.get("__class__") + && class_symbol.scope == SymbolScope::Free + { + symbol_table.needs_class_closure = true; + // Note: In CPython, the symbol is removed from the free set, + // but in RustPython we handle this differently during code generation + } + + // Check if __classdict__ is used as a free variable + if let Some(classdict_symbol) = symbol_table.symbols.get("__classdict__") + && classdict_symbol.scope == SymbolScope::Free + { + symbol_table.needs_classdict = true; + // Note: In CPython, the symbol is removed from the free set, + // but in RustPython we handle this differently during code generation + } +} + type SymbolMap = IndexMap; mod stack { @@ -283,7 +320,7 @@ use stack::StackStack; #[derive(Default)] #[repr(transparent)] struct SymbolTableAnalyzer { - tables: StackStack<(SymbolMap, SymbolTableType)>, + tables: StackStack<(SymbolMap, CompilerScope)>, } impl SymbolTableAnalyzer { @@ -307,19 +344,25 @@ impl SymbolTableAnalyzer { for symbol in symbol_table.symbols.values_mut() { self.analyze_symbol(symbol, symbol_table.typ, sub_tables)?; } + + // Handle class-specific implicit cells (like CPython) + if symbol_table.typ == CompilerScope::Class { + drop_class_free(symbol_table); + } + Ok(()) } fn analyze_symbol( &mut self, symbol: &mut Symbol, - st_typ: SymbolTableType, + st_typ: CompilerScope, sub_tables: &[SymbolTable], ) -> SymbolTableResult { if symbol .flags .contains(SymbolFlags::ASSIGNED_IN_COMPREHENSION) - && st_typ == SymbolTableType::Comprehension + && st_typ == CompilerScope::Comprehension { // propagate symbol to next higher level that can hold it, // i.e., function or module. Comprehension is skipped and @@ -383,8 +426,8 @@ impl SymbolTableAnalyzer { fn found_in_outer_scope(&mut self, name: &str) -> Option { let mut decl_depth = None; for (i, (symbols, typ)) in self.tables.iter().rev().enumerate() { - if matches!(typ, SymbolTableType::Module) - || matches!(typ, SymbolTableType::Class if name != "__class__") + if matches!(typ, CompilerScope::Module) + || matches!(typ, CompilerScope::Class if name != "__class__") { continue; } @@ -406,7 +449,7 @@ impl SymbolTableAnalyzer { // decl_depth is the number of tables between the current one and // the one that declared the cell var for (table, typ) in self.tables.iter_mut().rev().take(decl_depth) { - if let SymbolTableType::Class = typ { + if let CompilerScope::Class = typ { if let Some(free_class) = table.get_mut(name) { free_class.flags.insert(SymbolFlags::FREE_CLASS) } else { @@ -431,12 +474,12 @@ impl SymbolTableAnalyzer { &self, sub_tables: &[SymbolTable], name: &str, - st_typ: SymbolTableType, + st_typ: CompilerScope, ) -> Option { sub_tables.iter().find_map(|st| { let sym = st.symbols.get(name)?; if sym.scope == SymbolScope::Free || sym.flags.contains(SymbolFlags::FREE_CLASS) { - if st_typ == SymbolTableType::Class && name != "__class__" { + if st_typ == CompilerScope::Class && name != "__class__" { None } else { Some(SymbolScope::Cell) @@ -477,10 +520,10 @@ impl SymbolTableAnalyzer { } match table_type { - SymbolTableType::Module => { + CompilerScope::Module => { symbol.scope = SymbolScope::GlobalImplicit; } - SymbolTableType::Class => { + CompilerScope::Class => { // named expressions are forbidden in comprehensions on class scope return Err(SymbolTableError { error: "assignment expression within a comprehension cannot be used in a class body".to_string(), @@ -488,7 +531,7 @@ impl SymbolTableAnalyzer { location: None, }); } - SymbolTableType::Function => { + CompilerScope::Function | CompilerScope::AsyncFunction | CompilerScope::Lambda => { if let Some(parent_symbol) = symbols.get_mut(&symbol.name) { if let SymbolScope::Unknown = parent_symbol.scope { // this information is new, as the assignment is done in inner scope @@ -506,7 +549,7 @@ impl SymbolTableAnalyzer { last.0.insert(cloned_sym.name.to_owned(), cloned_sym); } } - SymbolTableType::Comprehension => { + CompilerScope::Comprehension => { // TODO check for conflicts - requires more context information about variables match symbols.get_mut(&symbol.name) { Some(parent_symbol) => { @@ -537,7 +580,7 @@ impl SymbolTableAnalyzer { self.analyze_symbol_comprehension(symbol, parent_offset + 1)?; } - SymbolTableType::TypeParams => { + CompilerScope::TypeParams => { todo!("analyze symbol comprehension for type params"); } } @@ -557,14 +600,17 @@ enum SymbolUsage { AnnotationParameter, AssignedNamedExprInComprehension, Iter, + TypeParam, } -struct SymbolTableBuilder<'src> { +struct SymbolTableBuilder { class_name: Option, // Scope stack. tables: Vec, future_annotations: bool, - source_code: SourceCode<'src>, + source_file: SourceFile, + // Current scope's varnames being collected (temporary storage) + current_varnames: Vec, } /// Enum to indicate in what mode an expression @@ -580,45 +626,78 @@ enum ExpressionContext { IterDefinitionExp, } -impl<'src> SymbolTableBuilder<'src> { - fn new(source_code: SourceCode<'src>) -> Self { +impl SymbolTableBuilder { + fn new(source_file: SourceFile) -> Self { let mut this = Self { class_name: None, tables: vec![], future_annotations: false, - source_code, + source_file, + current_varnames: Vec::new(), }; - this.enter_scope("top", SymbolTableType::Module, 0); + this.enter_scope("top", CompilerScope::Module, 0); this } -} -impl SymbolTableBuilder<'_> { fn finish(mut self) -> Result { assert_eq!(self.tables.len(), 1); let mut symbol_table = self.tables.pop().unwrap(); + // Save varnames for the top-level module scope + symbol_table.varnames = self.current_varnames; analyze_symbol_table(&mut symbol_table)?; Ok(symbol_table) } - fn enter_scope(&mut self, name: &str, typ: SymbolTableType, line_number: u32) { + fn enter_scope(&mut self, name: &str, typ: CompilerScope, line_number: u32) { let is_nested = self .tables .last() - .map(|table| table.is_nested || table.typ == SymbolTableType::Function) + .map(|table| table.is_nested || table.typ == CompilerScope::Function) .unwrap_or(false); let table = SymbolTable::new(name.to_owned(), typ, line_number, is_nested); self.tables.push(table); + // Clear current_varnames for the new scope + self.current_varnames.clear(); + } + + fn enter_type_param_block(&mut self, name: &str, line_number: u32) -> SymbolTableResult { + // Check if we're in a class scope + let in_class = self + .tables + .last() + .is_some_and(|t| t.typ == CompilerScope::Class); + + self.enter_scope(name, CompilerScope::TypeParams, line_number); + + // If we're in a class, mark that this type param scope can see the class scope + if let Some(table) = self.tables.last_mut() { + table.can_see_class_scope = in_class; + + // Add __classdict__ as a USE symbol in type param scope if in class + if in_class { + self.register_name("__classdict__", SymbolUsage::Used, TextRange::default())?; + } + } + + // Register .type_params as a SET symbol (it will be converted to cell variable later) + self.register_name(".type_params", SymbolUsage::Assigned, TextRange::default())?; + + Ok(()) } /// Pop symbol table and add to sub table of parent table. fn leave_scope(&mut self) { - let table = self.tables.pop().unwrap(); + let mut table = self.tables.pop().unwrap(); + // Save the collected varnames to the symbol table + table.varnames = std::mem::take(&mut self.current_varnames); self.tables.last_mut().unwrap().sub_tables.push(table); } fn line_index_start(&self, range: TextRange) -> u32 { - self.source_code.line_index(range.start()).get() as _ + self.source_file + .to_source_code() + .line_index(range.start()) + .get() as _ } fn scan_statements(&mut self, statements: &[Stmt]) -> SymbolTableResult { @@ -654,15 +733,13 @@ impl SymbolTableBuilder<'_> { fn scan_statement(&mut self, statement: &Stmt) -> SymbolTableResult { use ruff_python_ast::*; - if let Stmt::ImportFrom(StmtImportFrom { module, names, .. }) = &statement { - if module.as_ref().map(|id| id.as_str()) == Some("__future__") { - for feature in names { - if &feature.name == "annotations" { - self.future_annotations = true; - } - } - } + if let Stmt::ImportFrom(StmtImportFrom { module, names, .. }) = &statement + && module.as_ref().map(|id| id.as_str()) == Some("__future__") + { + self.future_annotations = + self.future_annotations || names.iter().any(|future| &future.name == "annotations"); } + match &statement { Stmt::Global(StmtGlobal { names, .. }) => { for name in names { @@ -690,12 +767,10 @@ impl SymbolTableBuilder<'_> { self.scan_annotation(expression)?; } if let Some(type_params) = type_params { - self.enter_scope( + self.enter_type_param_block( &format!("", name.as_str()), - SymbolTableType::TypeParams, - // FIXME: line no - self.line_index_start(*range), - ); + self.line_index_start(type_params.range), + )?; self.scan_type_params(type_params)?; } self.enter_scope_with_parameters( @@ -716,18 +791,18 @@ impl SymbolTableBuilder<'_> { decorator_list, type_params, range, + node_index: _, }) => { if let Some(type_params) = type_params { - self.enter_scope( + self.enter_type_param_block( &format!("", name.as_str()), - SymbolTableType::TypeParams, self.line_index_start(type_params.range), - ); + )?; self.scan_type_params(type_params)?; } self.enter_scope( name.as_str(), - SymbolTableType::Class, + CompilerScope::Class, self.line_index_start(*range), ); let prev_class = self.class_name.replace(name.to_string()); @@ -834,6 +909,7 @@ impl SymbolTableBuilder<'_> { value, simple, range, + node_index: _, }) => { // https://github.com/python/cpython/blob/main/Python/symtable.c#L1233 match &**target { @@ -909,12 +985,10 @@ impl SymbolTableBuilder<'_> { .. }) => { if let Some(type_params) = type_params { - self.enter_scope( - // &name.to_string(), + self.enter_type_param_block( "TypeAlias", - SymbolTableType::TypeParams, self.line_index_start(type_params.range), - ); + )?; self.scan_type_params(type_params)?; self.scan_expression(value, ExpressionContext::Load)?; self.leave_scope(); @@ -956,6 +1030,27 @@ impl SymbolTableBuilder<'_> { context: ExpressionContext, ) -> SymbolTableResult { use ruff_python_ast::*; + + // Check for expressions not allowed in type parameters scope + if let Some(table) = self.tables.last() + && table.typ == CompilerScope::TypeParams + && let Some(keyword) = match expression { + Expr::Yield(_) | Expr::YieldFrom(_) => Some("yield"), + Expr::Await(_) => Some("await"), + Expr::Named(_) => Some("named"), + _ => None, + } + { + return Err(SymbolTableError { + error: format!("{keyword} expression cannot be used within a type parameter"), + location: Some( + self.source_file + .to_source_code() + .source_location(expression.range().start(), PositionEncoding::Utf8), + ), + }); + } + match expression { Expr::BinOp(ExprBinOp { left, @@ -994,7 +1089,11 @@ impl SymbolTableBuilder<'_> { }) => { self.scan_expression(value, ExpressionContext::Load)?; } - Expr::Dict(ExprDict { items, range: _ }) => { + Expr::Dict(ExprDict { + items, + node_index: _, + range: _, + }) => { for item in items { if let Some(key) = &item.key { self.scan_expression(key, context)?; @@ -1002,15 +1101,27 @@ impl SymbolTableBuilder<'_> { self.scan_expression(&item.value, context)?; } } - Expr::Await(ExprAwait { value, range: _ }) => { + Expr::Await(ExprAwait { + value, + node_index: _, + range: _, + }) => { self.scan_expression(value, context)?; } - Expr::Yield(ExprYield { value, range: _ }) => { + Expr::Yield(ExprYield { + value, + node_index: _, + range: _, + }) => { if let Some(expression) = value { self.scan_expression(expression, context)?; } } - Expr::YieldFrom(ExprYieldFrom { value, range: _ }) => { + Expr::YieldFrom(ExprYieldFrom { + value, + node_index: _, + range: _, + }) => { self.scan_expression(value, context)?; } Expr::UnaryOp(ExprUnaryOp { @@ -1032,6 +1143,7 @@ impl SymbolTableBuilder<'_> { lower, upper, step, + node_index: _, range: _, }) => { if let Some(lower) = lower { @@ -1056,6 +1168,7 @@ impl SymbolTableBuilder<'_> { elt, generators, range, + node_index: _, }) => { self.scan_comprehension("genexpr", elt, None, generators, *range)?; } @@ -1063,6 +1176,7 @@ impl SymbolTableBuilder<'_> { elt, generators, range, + node_index: _, }) => { self.scan_comprehension("genexpr", elt, None, generators, *range)?; } @@ -1071,12 +1185,14 @@ impl SymbolTableBuilder<'_> { value, generators, range, + node_index: _, }) => { self.scan_comprehension("genexpr", key, Some(value), generators, *range)?; } Expr::Call(ExprCall { func, arguments, + node_index: _, range: _, }) => { match context { @@ -1114,7 +1230,7 @@ impl SymbolTableBuilder<'_> { // Interesting stuff about the __class__ variable: // https://docs.python.org/3/reference/datamodel.html?highlight=__class__#creating-the-class-object if context == ExpressionContext::Load - && self.tables.last().unwrap().typ == SymbolTableType::Function + && self.tables.last().unwrap().typ == CompilerScope::Function && id == "super" { self.register_name("__class__", SymbolUsage::Used, *range)?; @@ -1123,6 +1239,7 @@ impl SymbolTableBuilder<'_> { Expr::Lambda(ExprLambda { body, parameters, + node_index: _, range: _, }) => { if let Some(parameters) = parameters { @@ -1134,7 +1251,7 @@ impl SymbolTableBuilder<'_> { } else { self.enter_scope( "lambda", - SymbolTableType::Function, + CompilerScope::Lambda, self.line_index_start(expression.range()), ); } @@ -1149,15 +1266,25 @@ impl SymbolTableBuilder<'_> { self.leave_scope(); } Expr::FString(ExprFString { value, .. }) => { - for expr in value.elements().filter_map(|x| x.as_expression()) { + for expr in value.elements().filter_map(|x| x.as_interpolation()) { self.scan_expression(&expr.expression, ExpressionContext::Load)?; if let Some(format_spec) = &expr.format_spec { - for element in format_spec.elements.expressions() { + for element in format_spec.elements.interpolations() { self.scan_expression(&element.expression, ExpressionContext::Load)? } } } } + Expr::TString(tstring) => { + return Err(SymbolTableError { + error: "not yet implemented".into(), + location: Some( + self.source_file + .to_source_code() + .source_location(tstring.range.start(), PositionEncoding::Utf8), + ), + }); + } // Constants Expr::StringLiteral(_) | Expr::BytesLiteral(_) @@ -1170,6 +1297,7 @@ impl SymbolTableBuilder<'_> { test, body, orelse, + node_index: _, range: _, }) => { self.scan_expression(test, ExpressionContext::Load)?; @@ -1181,13 +1309,14 @@ impl SymbolTableBuilder<'_> { target, value, range, + node_index: _, }) => { // named expressions are not allowed in the definition of // comprehension iterator definitions if let ExpressionContext::IterDefinitionExp = context { return Err(SymbolTableError { error: "assignment expression cannot be used in a comprehension iterable expression".to_string(), - location: Some(self.source_code.source_location(target.range().start())), + location: Some(self.source_file.to_source_code().source_location(target.range().start(), PositionEncoding::Utf8)), }); } @@ -1200,7 +1329,7 @@ impl SymbolTableBuilder<'_> { if let Expr::Name(ExprName { id, .. }) = &**target { let id = id.as_str(); let table = self.tables.last().unwrap(); - if table.typ == SymbolTableType::Comprehension { + if table.typ == CompilerScope::Comprehension { self.register_name( id, SymbolUsage::AssignedNamedExprInComprehension, @@ -1231,7 +1360,7 @@ impl SymbolTableBuilder<'_> { // Comprehensions are compiled as functions, so create a scope for them: self.enter_scope( scope_name, - SymbolTableType::Comprehension, + CompilerScope::Comprehension, self.line_index_start(range), ); @@ -1266,7 +1395,30 @@ impl SymbolTableBuilder<'_> { Ok(()) } + /// Scan type parameter bound or default in a separate scope + // = symtable_visit_type_param_bound_or_default + fn scan_type_param_bound_or_default(&mut self, expr: &Expr, name: &str) -> SymbolTableResult { + // Enter a new TypeParams scope for the bound/default expression + // This allows the expression to access outer scope symbols + let line_number = self.line_index_start(expr.range()); + self.enter_scope(name, CompilerScope::TypeParams, line_number); + + // Note: In CPython, can_see_class_scope is preserved in the new scope + // In RustPython, this is handled through the scope hierarchy + + // Scan the expression in this new scope + let result = self.scan_expression(expr, ExpressionContext::Load); + + // Exit the scope + self.leave_scope(); + + result + } + fn scan_type_params(&mut self, type_params: &TypeParams) -> SymbolTableResult { + // Register .type_params as a type parameter (automatically becomes cell variable) + self.register_name(".type_params", SymbolUsage::TypeParam, type_params.range)?; + // First register all type parameters for type_param in &type_params.type_params { match type_param { @@ -1274,26 +1426,54 @@ impl SymbolTableBuilder<'_> { name, bound, range: type_var_range, - .. + default, + node_index: _, }) => { - self.register_name(name.as_str(), SymbolUsage::Assigned, *type_var_range)?; + self.register_name(name.as_str(), SymbolUsage::TypeParam, *type_var_range)?; + + // Process bound in a separate scope if let Some(binding) = bound { - self.scan_expression(binding, ExpressionContext::Load)?; + let scope_name = if binding.is_tuple_expr() { + format!("") + } else { + format!("") + }; + self.scan_type_param_bound_or_default(binding, &scope_name)?; + } + + // Process default in a separate scope + if let Some(default_value) = default { + let scope_name = format!(""); + self.scan_type_param_bound_or_default(default_value, &scope_name)?; } } TypeParam::ParamSpec(TypeParamParamSpec { name, range: param_spec_range, - .. + default, + node_index: _, }) => { - self.register_name(name, SymbolUsage::Assigned, *param_spec_range)?; + self.register_name(name, SymbolUsage::TypeParam, *param_spec_range)?; + + // Process default in a separate scope + if let Some(default_value) = default { + let scope_name = format!(""); + self.scan_type_param_bound_or_default(default_value, &scope_name)?; + } } TypeParam::TypeVarTuple(TypeParamTypeVarTuple { name, range: type_var_tuple_range, - .. + default, + node_index: _, }) => { - self.register_name(name, SymbolUsage::Assigned, *type_var_tuple_range)?; + self.register_name(name, SymbolUsage::TypeParam, *type_var_tuple_range)?; + + // Process default in a separate scope + if let Some(default_value) = default { + let scope_name = format!(""); + self.scan_type_param_bound_or_default(default_value, &scope_name)?; + } } } } @@ -1394,7 +1574,7 @@ impl SymbolTableBuilder<'_> { self.scan_annotation(annotation)?; } - self.enter_scope(name, SymbolTableType::Function, line_number); + self.enter_scope(name, CompilerScope::Function, line_number); // Fill scope with parameter names: self.scan_parameters(¶meters.posonlyargs)?; @@ -1419,7 +1599,10 @@ impl SymbolTableBuilder<'_> { role: SymbolUsage, range: TextRange, ) -> SymbolTableResult { - let location = self.source_code.source_location(range.start()); + let location = self + .source_file + .to_source_code() + .source_location(range.start(), PositionEncoding::Utf8); let location = Some(location); let scope_depth = self.tables.len(); let table = self.tables.last_mut().unwrap(); @@ -1522,18 +1705,43 @@ impl SymbolTableBuilder<'_> { } SymbolUsage::Parameter => { flags.insert(SymbolFlags::PARAMETER); + // Parameters are always added to varnames first + let name_str = symbol.name.clone(); + if !self.current_varnames.contains(&name_str) { + self.current_varnames.push(name_str); + } } SymbolUsage::AnnotationParameter => { flags.insert(SymbolFlags::PARAMETER | SymbolFlags::ANNOTATED); + // Annotated parameters are also added to varnames + let name_str = symbol.name.clone(); + if !self.current_varnames.contains(&name_str) { + self.current_varnames.push(name_str); + } } SymbolUsage::AnnotationAssigned => { flags.insert(SymbolFlags::ASSIGNED | SymbolFlags::ANNOTATED); } SymbolUsage::Assigned => { flags.insert(SymbolFlags::ASSIGNED); + // Local variables (assigned) are added to varnames if they are local scope + // and not already in varnames + if symbol.scope == SymbolScope::Local { + let name_str = symbol.name.clone(); + if !self.current_varnames.contains(&name_str) { + self.current_varnames.push(name_str); + } + } } SymbolUsage::AssignedNamedExprInComprehension => { flags.insert(SymbolFlags::ASSIGNED | SymbolFlags::ASSIGNED_IN_COMPREHENSION); + // Named expressions in comprehensions might also be locals + if symbol.scope == SymbolScope::Local { + let name_str = symbol.name.clone(); + if !self.current_varnames.contains(&name_str) { + self.current_varnames.push(name_str); + } + } } SymbolUsage::Global => { symbol.scope = SymbolScope::GlobalExplicit; @@ -1544,6 +1752,11 @@ impl SymbolTableBuilder<'_> { SymbolUsage::Iter => { flags.insert(SymbolFlags::ITER); } + SymbolUsage::TypeParam => { + // Type parameters are always cell variables in their scope + symbol.scope = SymbolScope::Cell; + flags.insert(SymbolFlags::ASSIGNED); + } } // and even more checking diff --git a/compiler/codegen/src/unparse.rs b/crates/codegen/src/unparse.rs similarity index 89% rename from compiler/codegen/src/unparse.rs rename to crates/codegen/src/unparse.rs index 47e883da3a..74e35fd5e2 100644 --- a/compiler/codegen/src/unparse.rs +++ b/crates/codegen/src/unparse.rs @@ -1,14 +1,12 @@ -use ruff_python_ast as ruff; +use ruff_python_ast::{ + self as ruff, Arguments, BoolOp, Comprehension, ConversionFlag, Expr, Identifier, Operator, + Parameter, ParameterWithDefault, Parameters, +}; use ruff_text_size::Ranged; -use rustpython_compiler_source::SourceCode; +use rustpython_compiler_core::SourceFile; use rustpython_literal::escape::{AsciiEscape, UnicodeEscape}; use std::fmt::{self, Display as _}; -use ruff::{ - Arguments, BoolOp, Comprehension, ConversionFlag, Expr, Identifier, Operator, Parameter, - ParameterWithDefault, Parameters, -}; - mod precedence { macro_rules! precedence { ($($op:ident,)*) => { @@ -29,28 +27,33 @@ mod precedence { struct Unparser<'a, 'b, 'c> { f: &'b mut fmt::Formatter<'a>, - source: &'c SourceCode<'c>, + source: &'c SourceFile, } + impl<'a, 'b, 'c> Unparser<'a, 'b, 'c> { - const fn new(f: &'b mut fmt::Formatter<'a>, source: &'c SourceCode<'c>) -> Self { - Unparser { f, source } + const fn new(f: &'b mut fmt::Formatter<'a>, source: &'c SourceFile) -> Self { + Self { f, source } } fn p(&mut self, s: &str) -> fmt::Result { self.f.write_str(s) } + fn p_id(&mut self, s: &Identifier) -> fmt::Result { self.f.write_str(s.as_str()) } + fn p_if(&mut self, cond: bool, s: &str) -> fmt::Result { if cond { self.f.write_str(s)?; } Ok(()) } + fn p_delim(&mut self, first: &mut bool, s: &str) -> fmt::Result { self.p_if(!std::mem::take(first), s) } + fn write_fmt(&mut self, f: fmt::Arguments<'_>) -> fmt::Result { self.f.write_fmt(f) } @@ -82,6 +85,7 @@ impl<'a, 'b, 'c> Unparser<'a, 'b, 'c> { Expr::BoolOp(ruff::ExprBoolOp { op, values, + node_index: _, range: _range, }) => { let (op, prec) = op_prec!(bin, op, BoolOp, And("and", AND), Or("or", OR)); @@ -96,6 +100,7 @@ impl<'a, 'b, 'c> Unparser<'a, 'b, 'c> { Expr::Named(ruff::ExprNamed { target, value, + node_index: _, range: _range, }) => { group_if!(precedence::TUPLE, { @@ -108,6 +113,7 @@ impl<'a, 'b, 'c> Unparser<'a, 'b, 'c> { left, op, right, + node_index: _, range: _range, }) => { let right_associative = matches!(op, Operator::Pow); @@ -138,6 +144,7 @@ impl<'a, 'b, 'c> Unparser<'a, 'b, 'c> { Expr::UnaryOp(ruff::ExprUnaryOp { op, operand, + node_index: _, range: _range, }) => { let (op, prec) = op_prec!( @@ -157,6 +164,7 @@ impl<'a, 'b, 'c> Unparser<'a, 'b, 'c> { Expr::Lambda(ruff::ExprLambda { parameters, body, + node_index: _, range: _range, }) => { group_if!(precedence::TEST, { @@ -166,13 +174,14 @@ impl<'a, 'b, 'c> Unparser<'a, 'b, 'c> { } else { self.p("lambda")?; } - write!(self, ": {}", unparse_expr(body, self.source))?; + write!(self, ": {}", UnparseExpr::new(body, self.source))?; }) } Expr::If(ruff::ExprIf { test, body, orelse, + node_index: _, range: _range, }) => { group_if!(precedence::TEST, { @@ -185,6 +194,7 @@ impl<'a, 'b, 'c> Unparser<'a, 'b, 'c> { } Expr::Dict(ruff::ExprDict { items, + node_index: _, range: _range, }) => { self.p("{")?; @@ -192,7 +202,7 @@ impl<'a, 'b, 'c> Unparser<'a, 'b, 'c> { for item in items { self.p_delim(&mut first, ", ")?; if let Some(k) = &item.key { - write!(self, "{}: ", unparse_expr(k, self.source))?; + write!(self, "{}: ", UnparseExpr::new(k, self.source))?; } else { self.p("**")?; } @@ -202,6 +212,7 @@ impl<'a, 'b, 'c> Unparser<'a, 'b, 'c> { } Expr::Set(ruff::ExprSet { elts, + node_index: _, range: _range, }) => { self.p("{")?; @@ -215,6 +226,7 @@ impl<'a, 'b, 'c> Unparser<'a, 'b, 'c> { Expr::ListComp(ruff::ExprListComp { elt, generators, + node_index: _, range: _range, }) => { self.p("[")?; @@ -225,6 +237,7 @@ impl<'a, 'b, 'c> Unparser<'a, 'b, 'c> { Expr::SetComp(ruff::ExprSetComp { elt, generators, + node_index: _, range: _range, }) => { self.p("{")?; @@ -236,6 +249,7 @@ impl<'a, 'b, 'c> Unparser<'a, 'b, 'c> { key, value, generators, + node_index: _, range: _range, }) => { self.p("{")?; @@ -249,6 +263,7 @@ impl<'a, 'b, 'c> Unparser<'a, 'b, 'c> { parenthesized: _, elt, generators, + node_index: _, range: _range, }) => { self.p("(")?; @@ -258,6 +273,7 @@ impl<'a, 'b, 'c> Unparser<'a, 'b, 'c> { } Expr::Await(ruff::ExprAwait { value, + node_index: _, range: _range, }) => { group_if!(precedence::AWAIT, { @@ -267,24 +283,31 @@ impl<'a, 'b, 'c> Unparser<'a, 'b, 'c> { } Expr::Yield(ruff::ExprYield { value, + node_index: _, range: _range, }) => { if let Some(value) = value { - write!(self, "(yield {})", unparse_expr(value, self.source))?; + write!(self, "(yield {})", UnparseExpr::new(value, self.source))?; } else { self.p("(yield)")?; } } Expr::YieldFrom(ruff::ExprYieldFrom { value, + node_index: _, range: _range, }) => { - write!(self, "(yield from {})", unparse_expr(value, self.source))?; + write!( + self, + "(yield from {})", + UnparseExpr::new(value, self.source) + )?; } Expr::Compare(ruff::ExprCompare { left, ops, comparators, + node_index: _, range: _range, }) => { group_if!(precedence::CMP, { @@ -301,6 +324,7 @@ impl<'a, 'b, 'c> Unparser<'a, 'b, 'c> { Expr::Call(ruff::ExprCall { func, arguments: Arguments { args, keywords, .. }, + node_index: _, range: _range, }) => { self.unparse_expr(func, precedence::ATOM)?; @@ -310,6 +334,7 @@ impl<'a, 'b, 'c> Unparser<'a, 'b, 'c> { Expr::Generator(ruff::ExprGenerator { elt, generators, + node_index: _, range: _range, .. }), @@ -340,6 +365,7 @@ impl<'a, 'b, 'c> Unparser<'a, 'b, 'c> { self.p(")")?; } Expr::FString(ruff::ExprFString { value, .. }) => self.unparse_fstring(value)?, + Expr::TString(_) => self.p("t\"\"")?, Expr::StringLiteral(ruff::ExprStringLiteral { value, .. }) => { if value.is_unicode() { self.p("u")? @@ -354,7 +380,11 @@ impl<'a, 'b, 'c> Unparser<'a, 'b, 'c> { .fmt(self.f)? } Expr::NumberLiteral(ruff::ExprNumberLiteral { value, .. }) => { - const { assert!(f64::MAX_10_EXP == 308) }; + #[allow(clippy::correctness, clippy::assertions_on_constants)] + const { + assert!(f64::MAX_10_EXP == 308) + }; + let inf_str = "1e309"; match value { ruff::Number::Int(int) => int.fmt(self.f)?, @@ -428,6 +458,7 @@ impl<'a, 'b, 'c> Unparser<'a, 'b, 'c> { lower, upper, step, + node_index: _, range: _range, }) => { if let Some(lower) = lower { @@ -461,7 +492,7 @@ impl<'a, 'b, 'c> Unparser<'a, 'b, 'c> { if let Some(vararg) = &args.vararg { self.unparse_arg(vararg)?; } - for kwarg in args.kwonlyargs.iter() { + for kwarg in &args.kwonlyargs { self.p_delim(&mut first, ", ")?; self.unparse_function_arg(kwarg)?; } @@ -475,7 +506,7 @@ impl<'a, 'b, 'c> Unparser<'a, 'b, 'c> { fn unparse_function_arg(&mut self, arg: &ParameterWithDefault) -> fmt::Result { self.unparse_arg(&arg.parameter)?; if let Some(default) = &arg.default { - write!(self, "={}", unparse_expr(default, self.source))?; + write!(self, "={}", UnparseExpr::new(default, self.source))?; } Ok(()) } @@ -483,7 +514,7 @@ impl<'a, 'b, 'c> Unparser<'a, 'b, 'c> { fn unparse_arg(&mut self, arg: &Parameter) -> fmt::Result { self.p_id(&arg.name)?; if let Some(ann) = &arg.annotation { - write!(self, ": {}", unparse_expr(ann, self.source))?; + write!(self, ": {}", UnparseExpr::new(ann, self.source))?; } Ok(()) } @@ -506,7 +537,10 @@ impl<'a, 'b, 'c> Unparser<'a, 'b, 'c> { Ok(()) } - fn unparse_fstring_body(&mut self, elements: &[ruff::FStringElement]) -> fmt::Result { + fn unparse_fstring_body( + &mut self, + elements: &[ruff::InterpolatedStringElement], + ) -> fmt::Result { for elem in elements { self.unparse_fstring_elem(elem)?; } @@ -518,14 +552,14 @@ impl<'a, 'b, 'c> Unparser<'a, 'b, 'c> { val: &Expr, debug_text: Option<&ruff::DebugText>, conversion: ConversionFlag, - spec: Option<&ruff::FStringFormatSpec>, + spec: Option<&ruff::InterpolatedStringFormatSpec>, ) -> fmt::Result { let buffered = to_string_fmt(|f| { Unparser::new(f, self.source).unparse_expr(val, precedence::TEST + 1) }); if let Some(ruff::DebugText { leading, trailing }) = debug_text { self.p(leading)?; - self.p(self.source.get_range(val.range()))?; + self.p(self.source.slice(val.range()))?; self.p(trailing)?; } let brace = if buffered.starts_with('{') { @@ -555,9 +589,9 @@ impl<'a, 'b, 'c> Unparser<'a, 'b, 'c> { Ok(()) } - fn unparse_fstring_elem(&mut self, elem: &ruff::FStringElement) -> fmt::Result { + fn unparse_fstring_elem(&mut self, elem: &ruff::InterpolatedStringElement) -> fmt::Result { match elem { - ruff::FStringElement::Expression(ruff::FStringExpressionElement { + ruff::InterpolatedStringElement::Interpolation(ruff::InterpolatedElement { expression, debug_text, conversion, @@ -569,9 +603,10 @@ impl<'a, 'b, 'c> Unparser<'a, 'b, 'c> { *conversion, format_spec.as_deref(), ), - ruff::FStringElement::Literal(ruff::FStringLiteralElement { value, .. }) => { - self.unparse_fstring_str(value) - } + ruff::InterpolatedStringElement::Literal(ruff::InterpolatedStringLiteralElement { + value, + .. + }) => self.unparse_fstring_str(value), } } @@ -599,11 +634,13 @@ impl<'a, 'b, 'c> Unparser<'a, 'b, 'c> { pub struct UnparseExpr<'a> { expr: &'a Expr, - source: &'a SourceCode<'a>, + source: &'a SourceFile, } -pub const fn unparse_expr<'a>(expr: &'a Expr, source: &'a SourceCode<'a>) -> UnparseExpr<'a> { - UnparseExpr { expr, source } +impl<'a> UnparseExpr<'a> { + pub const fn new(expr: &'a Expr, source: &'a SourceFile) -> Self { + Self { expr, source } + } } impl fmt::Display for UnparseExpr<'_> { diff --git a/common/Cargo.toml b/crates/common/Cargo.toml similarity index 93% rename from common/Cargo.toml rename to crates/common/Cargo.toml index 94704d18c9..9fd7ea3880 100644 --- a/common/Cargo.toml +++ b/crates/common/Cargo.toml @@ -10,6 +10,7 @@ license.workspace = true [features] threading = ["parking_lot"] +wasm_js = ["getrandom/wasm_js"] [dependencies] rustpython-literal = { workspace = true } @@ -17,7 +18,6 @@ rustpython-wtf8 = { workspace = true } ascii = { workspace = true } bitflags = { workspace = true } -bstr = { workspace = true } cfg-if = { workspace = true } getrandom = { workspace = true } itertools = { workspace = true } @@ -25,7 +25,6 @@ libc = { workspace = true } malachite-bigint = { workspace = true } malachite-q = { workspace = true } malachite-base = { workspace = true } -memchr = { workspace = true } num-traits = { workspace = true } once_cell = { workspace = true } parking_lot = { workspace = true, optional = true } @@ -36,6 +35,9 @@ lock_api = "0.4" siphasher = "1" num-complex.workspace = true +[target.'cfg(unix)'.dependencies] +nix = { workspace = true } + [target.'cfg(windows)'.dependencies] widestring = { workspace = true } windows-sys = { workspace = true, features = [ diff --git a/common/src/atomic.rs b/crates/common/src/atomic.rs similarity index 100% rename from common/src/atomic.rs rename to crates/common/src/atomic.rs diff --git a/common/src/borrow.rs b/crates/common/src/borrow.rs similarity index 100% rename from common/src/borrow.rs rename to crates/common/src/borrow.rs diff --git a/common/src/boxvec.rs b/crates/common/src/boxvec.rs similarity index 99% rename from common/src/boxvec.rs rename to crates/common/src/boxvec.rs index 4f3928e56b..8687ba7f7f 100644 --- a/common/src/boxvec.rs +++ b/crates/common/src/boxvec.rs @@ -165,7 +165,7 @@ impl BoxVec { if index >= self.len() { None } else { - self.drain(index..index + 1).next() + self.drain(index..=index).next() } } diff --git a/common/src/cformat.rs b/crates/common/src/cformat.rs similarity index 100% rename from common/src/cformat.rs rename to crates/common/src/cformat.rs diff --git a/crates/common/src/crt_fd.rs b/crates/common/src/crt_fd.rs new file mode 100644 index 0000000000..7b8279adbe --- /dev/null +++ b/crates/common/src/crt_fd.rs @@ -0,0 +1,398 @@ +//! A module implementing an io type backed by the C runtime's file descriptors, i.e. what's +//! returned from libc::open, even on windows. + +use std::{cmp, ffi, fmt, io}; + +#[cfg(not(windows))] +use std::os::fd::{AsFd, AsRawFd, BorrowedFd, FromRawFd, IntoRawFd, OwnedFd, RawFd}; +#[cfg(windows)] +use std::os::windows::io::BorrowedHandle; + +mod c { + pub(super) use libc::*; + + #[cfg(windows)] + pub(super) use libc::commit as fsync; + #[cfg(windows)] + unsafe extern "C" { + #[link_name = "_chsize_s"] + pub(super) fn ftruncate(fd: i32, len: i64) -> i32; + } +} + +// this is basically what CPython has for Py_off_t; windows uses long long +// for offsets, other platforms just use off_t +#[cfg(not(windows))] +pub type Offset = c::off_t; +#[cfg(windows)] +pub type Offset = c::c_longlong; + +#[cfg(not(windows))] +pub type Raw = RawFd; +#[cfg(windows)] +pub type Raw = i32; + +#[inline] +fn cvt(ret: I) -> io::Result { + if ret < I::zero() { + // CRT functions set errno, not GetLastError(), so use errno_io_error + Err(crate::os::errno_io_error()) + } else { + Ok(ret) + } +} + +fn cvt_fd(ret: Raw) -> io::Result { + cvt(ret).map(|fd| unsafe { Owned::from_raw(fd) }) +} + +const MAX_RW: usize = if cfg!(any(windows, target_vendor = "apple")) { + i32::MAX as usize +} else { + isize::MAX as usize +}; + +#[cfg(not(windows))] +type OwnedInner = OwnedFd; +#[cfg(not(windows))] +type BorrowedInner<'fd> = BorrowedFd<'fd>; + +#[cfg(windows)] +mod win { + use super::*; + use std::marker::PhantomData; + use std::mem::ManuallyDrop; + + #[repr(transparent)] + pub(super) struct OwnedInner(i32); + + impl OwnedInner { + #[inline] + pub unsafe fn from_raw_fd(fd: Raw) -> Self { + Self(fd) + } + #[inline] + pub fn as_raw_fd(&self) -> Raw { + self.0 + } + #[inline] + pub fn into_raw_fd(self) -> Raw { + let me = ManuallyDrop::new(self); + me.0 + } + } + + impl Drop for OwnedInner { + #[inline] + fn drop(&mut self) { + let _ = _close(self.0); + } + } + + #[derive(Copy, Clone)] + #[repr(transparent)] + pub(super) struct BorrowedInner<'fd> { + fd: Raw, + _marker: PhantomData<&'fd Owned>, + } + + impl BorrowedInner<'_> { + #[inline] + pub const unsafe fn borrow_raw(fd: Raw) -> Self { + Self { + fd, + _marker: PhantomData, + } + } + #[inline] + pub fn as_raw_fd(&self) -> Raw { + self.fd + } + } +} + +#[cfg(windows)] +use self::win::{BorrowedInner, OwnedInner}; + +#[repr(transparent)] +pub struct Owned { + inner: OwnedInner, +} + +impl fmt::Debug for Owned { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_tuple("crt_fd::Owned") + .field(&self.as_raw()) + .finish() + } +} + +#[derive(Copy, Clone)] +#[repr(transparent)] +pub struct Borrowed<'fd> { + inner: BorrowedInner<'fd>, +} + +impl<'fd> PartialEq for Borrowed<'fd> { + fn eq(&self, other: &Self) -> bool { + self.as_raw() == other.as_raw() + } +} +impl<'fd> Eq for Borrowed<'fd> {} + +impl fmt::Debug for Borrowed<'_> { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + f.debug_tuple("crt_fd::Borrowed") + .field(&self.as_raw()) + .finish() + } +} + +impl Owned { + /// Create a `crt_fd::Owned` from a raw file descriptor. + /// + /// # Safety + /// + /// `fd` must be a valid file descriptor. + #[inline] + pub unsafe fn from_raw(fd: Raw) -> Self { + let inner = unsafe { OwnedInner::from_raw_fd(fd) }; + Self { inner } + } + + /// Create a `crt_fd::Owned` from a raw file descriptor. + /// + /// Returns an error if `fd` is -1. + /// + /// # Safety + /// + /// `fd` must be a valid file descriptor. + #[inline] + pub unsafe fn try_from_raw(fd: Raw) -> io::Result { + if fd == -1 { + Err(ebadf()) + } else { + Ok(unsafe { Self::from_raw(fd) }) + } + } + + #[inline] + pub fn borrow(&self) -> Borrowed<'_> { + unsafe { Borrowed::borrow_raw(self.as_raw()) } + } + + #[inline] + pub fn as_raw(&self) -> Raw { + self.inner.as_raw_fd() + } + + #[inline] + pub fn into_raw(self) -> Raw { + self.inner.into_raw_fd() + } + + pub fn leak<'fd>(self) -> Borrowed<'fd> { + unsafe { Borrowed::borrow_raw(self.into_raw()) } + } +} + +#[cfg(unix)] +impl From for OwnedFd { + fn from(fd: Owned) -> Self { + fd.inner + } +} + +#[cfg(unix)] +impl From for Owned { + fn from(fd: OwnedFd) -> Self { + Self { inner: fd } + } +} + +#[cfg(unix)] +impl AsFd for Owned { + fn as_fd(&self) -> BorrowedFd<'_> { + self.inner.as_fd() + } +} + +#[cfg(unix)] +impl AsRawFd for Owned { + fn as_raw_fd(&self) -> RawFd { + self.as_raw() + } +} + +#[cfg(unix)] +impl FromRawFd for Owned { + unsafe fn from_raw_fd(fd: RawFd) -> Self { + unsafe { Self::from_raw(fd) } + } +} + +#[cfg(unix)] +impl IntoRawFd for Owned { + fn into_raw_fd(self) -> RawFd { + self.into_raw() + } +} + +impl<'fd> Borrowed<'fd> { + /// Create a `crt_fd::Borrowed` from a raw file descriptor. + /// + /// # Safety + /// + /// `fd` must be a valid file descriptor. + #[inline] + pub const unsafe fn borrow_raw(fd: Raw) -> Self { + let inner = unsafe { BorrowedInner::borrow_raw(fd) }; + Self { inner } + } + + /// Create a `crt_fd::Borrowed` from a raw file descriptor. + /// + /// Returns an error if `fd` is -1. + /// + /// # Safety + /// + /// `fd` must be a valid file descriptor. + #[inline] + pub unsafe fn try_borrow_raw(fd: Raw) -> io::Result { + if fd == -1 { + Err(ebadf()) + } else { + Ok(unsafe { Self::borrow_raw(fd) }) + } + } + + #[inline] + pub fn as_raw(self) -> Raw { + self.inner.as_raw_fd() + } +} + +#[cfg(unix)] +impl<'fd> From> for BorrowedFd<'fd> { + fn from(fd: Borrowed<'fd>) -> Self { + fd.inner + } +} + +#[cfg(unix)] +impl<'fd> From> for Borrowed<'fd> { + fn from(fd: BorrowedFd<'fd>) -> Self { + Self { inner: fd } + } +} + +#[cfg(unix)] +impl AsFd for Borrowed<'_> { + fn as_fd(&self) -> BorrowedFd<'_> { + self.inner.as_fd() + } +} + +#[cfg(unix)] +impl AsRawFd for Borrowed<'_> { + fn as_raw_fd(&self) -> RawFd { + self.as_raw() + } +} + +#[inline] +fn ebadf() -> io::Error { + io::Error::from_raw_os_error(c::EBADF) +} + +pub fn open(path: &ffi::CStr, flags: i32, mode: i32) -> io::Result { + cvt_fd(unsafe { c::open(path.as_ptr(), flags, mode) }) +} + +#[cfg(windows)] +pub fn wopen(path: &widestring::WideCStr, flags: i32, mode: i32) -> io::Result { + cvt_fd(unsafe { suppress_iph!(c::wopen(path.as_ptr(), flags, mode)) }) +} + +#[cfg(all(any(unix, target_os = "wasi"), not(target_os = "redox")))] +pub fn openat(dir: Borrowed<'_>, path: &ffi::CStr, flags: i32, mode: i32) -> io::Result { + cvt_fd(unsafe { c::openat(dir.as_raw(), path.as_ptr(), flags, mode) }) +} + +pub fn fsync(fd: Borrowed<'_>) -> io::Result<()> { + cvt(unsafe { suppress_iph!(c::fsync(fd.as_raw())) })?; + Ok(()) +} + +fn _close(fd: Raw) -> io::Result<()> { + cvt(unsafe { suppress_iph!(c::close(fd)) })?; + Ok(()) +} + +pub fn close(fd: Owned) -> io::Result<()> { + _close(fd.into_raw()) +} + +pub fn ftruncate(fd: Borrowed<'_>, len: Offset) -> io::Result<()> { + cvt(unsafe { suppress_iph!(c::ftruncate(fd.as_raw(), len)) })?; + Ok(()) +} + +#[cfg(windows)] +pub fn as_handle(fd: Borrowed<'_>) -> io::Result> { + use windows_sys::Win32::Foundation::{HANDLE, INVALID_HANDLE_VALUE}; + unsafe extern "C" { + fn _get_osfhandle(fd: Borrowed<'_>) -> c::intptr_t; + } + let handle = unsafe { suppress_iph!(_get_osfhandle(fd)) }; + if handle as HANDLE == INVALID_HANDLE_VALUE { + // _get_osfhandle is a CRT function that sets errno, not GetLastError() + Err(crate::os::errno_io_error()) + } else { + Ok(unsafe { BorrowedHandle::borrow_raw(handle as _) }) + } +} + +fn _write(fd: Raw, buf: &[u8]) -> io::Result { + let count = cmp::min(buf.len(), MAX_RW); + let n = cvt(unsafe { suppress_iph!(c::write(fd, buf.as_ptr() as _, count as _)) })?; + Ok(n as usize) +} + +fn _read(fd: Raw, buf: &mut [u8]) -> io::Result { + let count = cmp::min(buf.len(), MAX_RW); + let n = cvt(unsafe { suppress_iph!(libc::read(fd, buf.as_mut_ptr() as _, count as _)) })?; + Ok(n as usize) +} + +pub fn write(fd: Borrowed<'_>, buf: &[u8]) -> io::Result { + _write(fd.as_raw(), buf) +} + +pub fn read(fd: Borrowed<'_>, buf: &mut [u8]) -> io::Result { + _read(fd.as_raw(), buf) +} + +macro_rules! impl_rw { + ($t:ty) => { + impl io::Write for $t { + fn write(&mut self, buf: &[u8]) -> io::Result { + _write(self.as_raw(), buf) + } + + #[inline] + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } + } + + impl io::Read for $t { + fn read(&mut self, buf: &mut [u8]) -> io::Result { + _read(self.as_raw(), buf) + } + } + }; +} + +impl_rw!(Owned); +impl_rw!(Borrowed<'_>); diff --git a/common/src/encodings.rs b/crates/common/src/encodings.rs similarity index 100% rename from common/src/encodings.rs rename to crates/common/src/encodings.rs diff --git a/common/src/fileutils.rs b/crates/common/src/fileutils.rs similarity index 80% rename from common/src/fileutils.rs rename to crates/common/src/fileutils.rs index e9a93947c1..a12c1cd82e 100644 --- a/common/src/fileutils.rs +++ b/crates/common/src/fileutils.rs @@ -8,12 +8,12 @@ pub use libc::stat as StatStruct; pub use windows::{StatStruct, fstat}; #[cfg(not(windows))] -pub fn fstat(fd: libc::c_int) -> std::io::Result { +pub fn fstat(fd: crate::crt_fd::Borrowed<'_>) -> std::io::Result { let mut stat = std::mem::MaybeUninit::uninit(); unsafe { - let ret = libc::fstat(fd, stat.as_mut_ptr()); + let ret = libc::fstat(fd.as_raw(), stat.as_mut_ptr()); if ret == -1 { - Err(crate::os::last_os_error()) + Err(crate::os::errno_io_error()) } else { Ok(stat.assume_init()) } @@ -22,15 +22,14 @@ pub fn fstat(fd: libc::c_int) -> std::io::Result { #[cfg(windows)] pub mod windows { - use crate::suppress_iph; + use crate::crt_fd; use crate::windows::ToWideString; use libc::{S_IFCHR, S_IFDIR, S_IFMT}; use std::ffi::{CString, OsStr, OsString}; - use std::os::windows::ffi::OsStrExt; + use std::os::windows::io::AsRawHandle; use std::sync::OnceLock; use windows_sys::Win32::Foundation::{ - BOOL, ERROR_INVALID_HANDLE, ERROR_NOT_SUPPORTED, FILETIME, FreeLibrary, HANDLE, - INVALID_HANDLE_VALUE, SetLastError, + ERROR_INVALID_HANDLE, ERROR_NOT_SUPPORTED, FILETIME, FreeLibrary, SetLastError, }; use windows_sys::Win32::Storage::FileSystem::{ BY_HANDLE_FILE_INFORMATION, FILE_ATTRIBUTE_DIRECTORY, FILE_ATTRIBUTE_READONLY, @@ -75,45 +74,31 @@ pub mod windows { pub fn update_st_mode_from_path(&mut self, path: &OsStr, attr: u32) { if attr & FILE_ATTRIBUTE_DIRECTORY == 0 { let file_extension = path - .encode_wide() - .collect::>() + .to_wide() .split(|&c| c == '.' as u16) .next_back() .and_then(|s| String::from_utf16(s).ok()); - if let Some(file_extension) = file_extension { - if file_extension.eq_ignore_ascii_case("exe") + if let Some(file_extension) = file_extension + && (file_extension.eq_ignore_ascii_case("exe") || file_extension.eq_ignore_ascii_case("bat") || file_extension.eq_ignore_ascii_case("cmd") - || file_extension.eq_ignore_ascii_case("com") - { - self.st_mode |= 0o111; - } + || file_extension.eq_ignore_ascii_case("com")) + { + self.st_mode |= 0o111; } } } } - unsafe extern "C" { - fn _get_osfhandle(fd: i32) -> libc::intptr_t; - } - - fn get_osfhandle(fd: i32) -> std::io::Result { - let ret = unsafe { suppress_iph!(_get_osfhandle(fd)) }; - if ret as HANDLE == INVALID_HANDLE_VALUE { - Err(crate::os::last_os_error()) - } else { - Ok(ret) - } - } - // _Py_fstat_noraise in cpython - pub fn fstat(fd: libc::c_int) -> std::io::Result { - let h = get_osfhandle(fd); + pub fn fstat(fd: crt_fd::Borrowed<'_>) -> std::io::Result { + let h = crt_fd::as_handle(fd); if h.is_err() { unsafe { SetLastError(ERROR_INVALID_HANDLE) }; } let h = h?; + let h = h.as_raw_handle(); // reset stat? let file_type = unsafe { GetFileType(h as _) }; @@ -312,7 +297,7 @@ pub mod windows { FILE_INFO_BY_NAME_CLASS, *mut libc::c_void, u32, - ) -> BOOL, + ) -> i32, >, > = OnceLock::new(); @@ -451,3 +436,93 @@ pub mod windows { } } } + +// _Py_fopen_obj in cpython (Python/fileutils.c:1757-1835) +// Open a file using std::fs::File and convert to FILE* +// Automatically handles path encoding and EINTR retries +pub fn fopen(path: &std::path::Path, mode: &str) -> std::io::Result<*mut libc::FILE> { + use std::ffi::CString; + use std::fs::File; + + // Currently only supports read mode + // Can be extended to support "wb", "w+b", etc. if needed + if mode != "rb" { + return Err(std::io::Error::new( + std::io::ErrorKind::InvalidInput, + format!("unsupported mode: {}", mode), + )); + } + + // Open file using std::fs::File (handles path encoding and EINTR automatically) + let file = File::open(path)?; + + #[cfg(windows)] + { + use std::os::windows::io::IntoRawHandle; + + // Convert File handle to CRT file descriptor + let handle = file.into_raw_handle(); + let fd = unsafe { libc::open_osfhandle(handle as isize, libc::O_RDONLY) }; + if fd == -1 { + return Err(std::io::Error::last_os_error()); + } + + // Convert fd to FILE* + let mode_cstr = CString::new(mode).unwrap(); + let fp = unsafe { libc::fdopen(fd, mode_cstr.as_ptr()) }; + if fp.is_null() { + unsafe { libc::close(fd) }; + return Err(std::io::Error::last_os_error()); + } + + // Set non-inheritable (Windows needs this explicitly) + if let Err(e) = set_inheritable(fd, false) { + unsafe { libc::fclose(fp) }; + return Err(e); + } + + Ok(fp) + } + + #[cfg(not(windows))] + { + use std::os::fd::IntoRawFd; + + // Convert File to raw fd + let fd = file.into_raw_fd(); + + // Convert fd to FILE* + let mode_cstr = CString::new(mode).unwrap(); + let fp = unsafe { libc::fdopen(fd, mode_cstr.as_ptr()) }; + if fp.is_null() { + unsafe { libc::close(fd) }; + return Err(std::io::Error::last_os_error()); + } + + // Unix: O_CLOEXEC is already set by File::open, so non-inheritable is automatic + Ok(fp) + } +} + +// set_inheritable in cpython (Python/fileutils.c:1443-1570) +// Set the inheritable flag of the specified file descriptor +// Only used on Windows; Unix automatically sets O_CLOEXEC +#[cfg(windows)] +fn set_inheritable(fd: libc::c_int, inheritable: bool) -> std::io::Result<()> { + use windows_sys::Win32::Foundation::{ + HANDLE, HANDLE_FLAG_INHERIT, INVALID_HANDLE_VALUE, SetHandleInformation, + }; + + let handle = unsafe { libc::get_osfhandle(fd) }; + if handle == INVALID_HANDLE_VALUE as isize { + return Err(std::io::Error::last_os_error()); + } + + let flags = if inheritable { HANDLE_FLAG_INHERIT } else { 0 }; + let result = unsafe { SetHandleInformation(handle as HANDLE, HANDLE_FLAG_INHERIT, flags) }; + if result == 0 { + return Err(std::io::Error::last_os_error()); + } + + Ok(()) +} diff --git a/common/src/float_ops.rs b/crates/common/src/float_ops.rs similarity index 100% rename from common/src/float_ops.rs rename to crates/common/src/float_ops.rs diff --git a/common/src/format.rs b/crates/common/src/format.rs similarity index 95% rename from common/src/format.rs rename to crates/common/src/format.rs index 92ab99a571..447ae575f4 100644 --- a/common/src/format.rs +++ b/crates/common/src/format.rs @@ -389,7 +389,7 @@ impl FormatSpec { fn insert_separator(mut magnitude_str: String, inter: i32, sep: char, sep_cnt: i32) -> String { let magnitude_len = magnitude_str.len() as i32; - for i in 1..sep_cnt + 1 { + for i in 1..=sep_cnt { magnitude_str.insert((magnitude_len - inter * i) as usize, sep); } magnitude_str @@ -424,7 +424,13 @@ impl FormatSpec { const fn get_separator_interval(&self) -> usize { match self.format_type { Some(FormatType::Binary | FormatType::Octal | FormatType::Hex(_)) => 4, - Some(FormatType::Decimal | FormatType::Number(_) | FormatType::FixedPoint(_)) => 3, + Some( + FormatType::Decimal + | FormatType::FixedPoint(_) + | FormatType::GeneralFormat(_) + | FormatType::Exponent(_) + | FormatType::Percentage, + ) => 3, None => 3, _ => panic!("Separators only valid for numbers!"), } @@ -436,8 +442,14 @@ impl FormatSpec { let sep = char::from(fg); let inter = self.get_separator_interval().try_into().unwrap(); let magnitude_len = magnitude_str.len(); - let width = self.width.unwrap_or(magnitude_len) as i32 - prefix.len() as i32; - let disp_digit_cnt = cmp::max(width, magnitude_len as i32); + let disp_digit_cnt = if self.fill == Some('0'.into()) + && self.align == Some(FormatAlign::AfterSign) + { + let width = self.width.unwrap_or(magnitude_len) as i32 - prefix.len() as i32; + cmp::max(width, magnitude_len as i32) + } else { + magnitude_len as i32 + }; Self::add_magnitude_separators_for_char(magnitude_str, inter, sep, disp_digit_cnt) } None => magnitude_str, @@ -748,7 +760,7 @@ impl FormatSpec { let inter = self.get_separator_interval().try_into().unwrap(); let len = magnitude_str.len() as i32; let separated_magnitude = - FormatSpec::add_magnitude_separators_for_char(magnitude_str, inter, sep, len); + Self::add_magnitude_separators_for_char(magnitude_str, inter, sep, len); Ok(separated_magnitude) } None => Ok(magnitude_str), @@ -1319,6 +1331,45 @@ mod tests { ); } + #[test] + fn test_format_int_width_and_grouping() { + // issue #5922: width + comma grouping should pad left, not inside the number + let spec = FormatSpec::parse("10,").unwrap(); + let result = spec.format_int(&BigInt::from(1234)).unwrap(); + assert_eq!(result, " 1,234"); // CPython 3.13.5 + } + + #[test] + fn test_format_int_padding_with_grouping() { + // CPython behavior: f'{1234:010,}' results in "00,001,234" + let spec1 = FormatSpec::parse("010,").unwrap(); + let result1 = spec1.format_int(&BigInt::from(1234)).unwrap(); + assert_eq!(result1, "00,001,234"); + + // CPython behavior: f'{-1234:010,}' results in "-0,001,234" + let spec2 = FormatSpec::parse("010,").unwrap(); + let result2 = spec2.format_int(&BigInt::from(-1234)).unwrap(); + assert_eq!(result2, "-0,001,234"); + + // CPython behavior: f'{-1234:=10,}' results in "- 1,234" + let spec3 = FormatSpec::parse("=10,").unwrap(); + let result3 = spec3.format_int(&BigInt::from(-1234)).unwrap(); + assert_eq!(result3, "- 1,234"); + + // CPython behavior: f'{1234:=10,}' results in " 1,234" (same as right-align for positive numbers) + let spec4 = FormatSpec::parse("=10,").unwrap(); + let result4 = spec4.format_int(&BigInt::from(1234)).unwrap(); + assert_eq!(result4, " 1,234"); + } + + #[test] + fn test_format_int_non_aftersign_zero_padding() { + // CPython behavior: f'{1234:0>10,}' results in "000001,234" + let spec = FormatSpec::parse("0>10,").unwrap(); + let result = spec.format_int(&BigInt::from(1234)).unwrap(); + assert_eq!(result, "000001,234"); + } + #[test] fn test_format_parse() { let expected = Ok(FormatString { diff --git a/common/src/hash.rs b/crates/common/src/hash.rs similarity index 100% rename from common/src/hash.rs rename to crates/common/src/hash.rs diff --git a/crates/common/src/int.rs b/crates/common/src/int.rs new file mode 100644 index 0000000000..ed09cc01a0 --- /dev/null +++ b/crates/common/src/int.rs @@ -0,0 +1,208 @@ +use malachite_base::{num::conversion::traits::RoundingInto, rounding_modes::RoundingMode}; +use malachite_bigint::{BigInt, BigUint, Sign}; +use malachite_q::Rational; +use num_traits::{One, ToPrimitive, Zero}; + +pub fn true_div(numerator: &BigInt, denominator: &BigInt) -> f64 { + let rational = Rational::from_integers_ref(numerator.into(), denominator.into()); + match rational.rounding_into(RoundingMode::Nearest) { + // returned value is $t::MAX but still less than the original + (val, std::cmp::Ordering::Less) if val == f64::MAX => f64::INFINITY, + // returned value is $t::MIN but still greater than the original + (val, std::cmp::Ordering::Greater) if val == f64::MIN => f64::NEG_INFINITY, + (val, _) => val, + } +} + +pub fn float_to_ratio(value: f64) -> Option<(BigInt, BigInt)> { + let sign = match std::cmp::PartialOrd::partial_cmp(&value, &0.0)? { + std::cmp::Ordering::Less => Sign::Minus, + std::cmp::Ordering::Equal => return Some((BigInt::zero(), BigInt::one())), + std::cmp::Ordering::Greater => Sign::Plus, + }; + Rational::try_from(value).ok().map(|x| { + let (numer, denom) = x.into_numerator_and_denominator(); + ( + BigInt::from_biguint(sign, numer.into()), + BigUint::from(denom).into(), + ) + }) +} + +#[derive(Debug, Eq, PartialEq)] +pub enum BytesToIntError { + InvalidLiteral { base: u32 }, + InvalidBase, + DigitLimit { got: usize, limit: usize }, +} + +// https://github.com/python/cpython/blob/4e665351082c50018fb31d80db25b4693057393e/Objects/longobject.c#L2977 +// https://github.com/python/cpython/blob/4e665351082c50018fb31d80db25b4693057393e/Objects/longobject.c#L2884 +pub fn bytes_to_int( + buf: &[u8], + mut base: u32, + digit_limit: usize, +) -> Result { + if base != 0 && !(2..=36).contains(&base) { + return Err(BytesToIntError::InvalidBase); + } + + let mut buf = buf.trim_ascii(); + + // split sign + let sign = match buf.first() { + Some(b'+') => Some(Sign::Plus), + Some(b'-') => Some(Sign::Minus), + None => return Err(BytesToIntError::InvalidLiteral { base }), + _ => None, + }; + + if sign.is_some() { + buf = &buf[1..]; + } + + let mut error_if_nonzero = false; + if base == 0 { + match (buf.first(), buf.get(1)) { + (Some(v), _) if *v != b'0' => base = 10, + (_, Some(b'x' | b'X')) => base = 16, + (_, Some(b'o' | b'O')) => base = 8, + (_, Some(b'b' | b'B')) => base = 2, + (_, _) => { + // "old" (C-style) octal literal, now invalid. it might still be zero though + base = 10; + error_if_nonzero = true; + } + } + } + + if error_if_nonzero { + if let [_first, others @ .., last] = buf { + let is_zero = *last == b'0' && others.iter().all(|&c| c == b'0' || c == b'_'); + if !is_zero { + return Err(BytesToIntError::InvalidLiteral { base }); + } + } + return Ok(BigInt::zero()); + } + + if buf.first().is_some_and(|&v| v == b'0') + && buf.get(1).is_some_and(|&v| { + (base == 16 && (v == b'x' || v == b'X')) + || (base == 8 && (v == b'o' || v == b'O')) + || (base == 2 && (v == b'b' || v == b'B')) + }) + { + buf = &buf[2..]; + + // One underscore allowed here + if buf.first().is_some_and(|&v| v == b'_') { + buf = &buf[1..]; + } + } + + // Reject empty strings + let mut prev = *buf + .first() + .ok_or(BytesToIntError::InvalidLiteral { base })?; + + // Leading underscore not allowed + if prev == b'_' || !prev.is_ascii_alphanumeric() { + return Err(BytesToIntError::InvalidLiteral { base }); + } + + // Verify all characters are digits and underscores + let mut digits = 1; + for &cur in buf.iter().skip(1) { + if cur == b'_' { + // Double underscore not allowed + if prev == b'_' { + return Err(BytesToIntError::InvalidLiteral { base }); + } + } else if cur.is_ascii_alphanumeric() { + digits += 1; + } else { + return Err(BytesToIntError::InvalidLiteral { base }); + } + + prev = cur; + } + + // Trailing underscore not allowed + if prev == b'_' { + return Err(BytesToIntError::InvalidLiteral { base }); + } + + if digit_limit > 0 && !base.is_power_of_two() && digits > digit_limit { + return Err(BytesToIntError::DigitLimit { + got: digits, + limit: digit_limit, + }); + } + + let uint = BigUint::parse_bytes(buf, base).ok_or(BytesToIntError::InvalidLiteral { base })?; + Ok(BigInt::from_biguint(sign.unwrap_or(Sign::Plus), uint)) +} + +// num-bigint now returns Some(inf) for to_f64() in some cases, so just keep that the same for now +#[inline(always)] +pub fn bigint_to_finite_float(int: &BigInt) -> Option { + int.to_f64().filter(|f| f.is_finite()) +} + +#[cfg(test)] +mod tests { + use super::*; + + const DIGIT_LIMIT: usize = 4300; // Default of Cpython + + #[test] + fn bytes_to_int_valid() { + for ((buf, base), expected) in [ + (("0b101", 2), BigInt::from(5)), + (("0x_10", 16), BigInt::from(16)), + (("0b", 16), BigInt::from(11)), + (("+0b101", 2), BigInt::from(5)), + (("0_0_0", 10), BigInt::from(0)), + (("000", 0), BigInt::from(0)), + (("0_100", 10), BigInt::from(100)), + ] { + assert_eq!( + bytes_to_int(buf.as_bytes(), base, DIGIT_LIMIT), + Ok(expected) + ); + } + } + + #[test] + fn bytes_to_int_invalid_literal() { + for ((buf, base), expected) in [ + (("09_99", 0), BytesToIntError::InvalidLiteral { base: 10 }), + (("0_", 0), BytesToIntError::InvalidLiteral { base: 10 }), + (("0_", 2), BytesToIntError::InvalidLiteral { base: 2 }), + ] { + assert_eq!( + bytes_to_int(buf.as_bytes(), base, DIGIT_LIMIT), + Err(expected) + ) + } + } + + #[test] + fn bytes_to_int_invalid_base() { + for base in [1, 37] { + assert_eq!( + bytes_to_int("012345".as_bytes(), base, DIGIT_LIMIT), + Err(BytesToIntError::InvalidBase) + ) + } + } + + #[test] + fn bytes_to_int_digit_limit() { + assert_eq!( + bytes_to_int("012345".as_bytes(), 10, 5), + Err(BytesToIntError::DigitLimit { got: 6, limit: 5 }) + ); + } +} diff --git a/common/src/lib.rs b/crates/common/src/lib.rs similarity index 100% rename from common/src/lib.rs rename to crates/common/src/lib.rs diff --git a/common/src/linked_list.rs b/crates/common/src/linked_list.rs similarity index 100% rename from common/src/linked_list.rs rename to crates/common/src/linked_list.rs diff --git a/common/src/lock.rs b/crates/common/src/lock.rs similarity index 100% rename from common/src/lock.rs rename to crates/common/src/lock.rs diff --git a/common/src/lock/cell_lock.rs b/crates/common/src/lock/cell_lock.rs similarity index 100% rename from common/src/lock/cell_lock.rs rename to crates/common/src/lock/cell_lock.rs diff --git a/common/src/lock/immutable_mutex.rs b/crates/common/src/lock/immutable_mutex.rs similarity index 100% rename from common/src/lock/immutable_mutex.rs rename to crates/common/src/lock/immutable_mutex.rs diff --git a/common/src/lock/thread_mutex.rs b/crates/common/src/lock/thread_mutex.rs similarity index 100% rename from common/src/lock/thread_mutex.rs rename to crates/common/src/lock/thread_mutex.rs diff --git a/common/src/macros.rs b/crates/common/src/macros.rs similarity index 100% rename from common/src/macros.rs rename to crates/common/src/macros.rs diff --git a/common/src/os.rs b/crates/common/src/os.rs similarity index 76% rename from common/src/os.rs rename to crates/common/src/os.rs index e298db462a..e77a81fd94 100644 --- a/common/src/os.rs +++ b/crates/common/src/os.rs @@ -1,7 +1,25 @@ // spell-checker:disable // TODO: we can move more os-specific bindings/interfaces from stdlib::{os, posix, nt} to here -use std::{io, str::Utf8Error}; +use std::{io, process::ExitCode, str::Utf8Error}; + +/// Convert exit code to std::process::ExitCode +/// +/// On Windows, this supports the full u32 range including STATUS_CONTROL_C_EXIT (0xC000013A). +/// On other platforms, only the lower 8 bits are used. +pub fn exit_code(code: u32) -> ExitCode { + #[cfg(windows)] + { + // For large exit codes like STATUS_CONTROL_C_EXIT (0xC000013A), + // we need to call std::process::exit() directly since ExitCode::from(u8) + // would truncate the value, and ExitCode::from_raw() is still unstable. + // FIXME: side effect in exit_code is not ideal. + if code > u8::MAX as u32 { + std::process::exit(code as i32) + } + } + ExitCode::from(code as u8) +} pub trait ErrorExt { fn posix_errno(&self) -> i32; @@ -19,46 +37,47 @@ impl ErrorExt for io::Error { } } +/// Get the last error from C runtime library functions (like _dup, _dup2, _fstat, etc.) +/// CRT functions set errno, not GetLastError(), so we need to read errno directly. #[cfg(windows)] -pub fn last_os_error() -> io::Error { - let err = io::Error::last_os_error(); - // FIXME: probably not ideal, we need a bigger dichotomy between GetLastError and errno - if err.raw_os_error() == Some(0) { - unsafe extern "C" { - fn _get_errno(pValue: *mut i32) -> i32; - } - let mut errno = 0; - unsafe { suppress_iph!(_get_errno(&mut errno)) }; - let errno = errno_to_winerror(errno); - io::Error::from_raw_os_error(errno) - } else { - err - } +pub fn errno_io_error() -> io::Error { + let errno: i32 = get_errno(); + let winerror = errno_to_winerror(errno); + io::Error::from_raw_os_error(winerror) } #[cfg(not(windows))] -pub fn last_os_error() -> io::Error { - io::Error::last_os_error() +pub fn errno_io_error() -> io::Error { + std::io::Error::last_os_error() } #[cfg(windows)] -pub fn last_posix_errno() -> i32 { - let err = io::Error::last_os_error(); - if err.raw_os_error() == Some(0) { - unsafe extern "C" { - fn _get_errno(pValue: *mut i32) -> i32; - } - let mut errno = 0; - unsafe { suppress_iph!(_get_errno(&mut errno)) }; - errno - } else { - err.posix_errno() +pub fn get_errno() -> i32 { + unsafe extern "C" { + fn _get_errno(pValue: *mut i32) -> i32; } + let mut errno = 0; + unsafe { suppress_iph!(_get_errno(&mut errno)) }; + errno } #[cfg(not(windows))] -pub fn last_posix_errno() -> i32 { - last_os_error().posix_errno() +pub fn get_errno() -> i32 { + std::io::Error::last_os_error().posix_errno() +} + +/// Set errno to the specified value. +#[cfg(windows)] +pub fn set_errno(value: i32) { + unsafe extern "C" { + fn _set_errno(value: i32) -> i32; + } + unsafe { suppress_iph!(_set_errno(value)) }; +} + +#[cfg(unix)] +pub fn set_errno(value: i32) { + nix::errno::Errno::from_raw(value).set(); } #[cfg(unix)] diff --git a/common/src/rand.rs b/crates/common/src/rand.rs similarity index 100% rename from common/src/rand.rs rename to crates/common/src/rand.rs diff --git a/common/src/rc.rs b/crates/common/src/rc.rs similarity index 100% rename from common/src/rc.rs rename to crates/common/src/rc.rs diff --git a/common/src/refcount.rs b/crates/common/src/refcount.rs similarity index 100% rename from common/src/refcount.rs rename to crates/common/src/refcount.rs diff --git a/common/src/static_cell.rs b/crates/common/src/static_cell.rs similarity index 100% rename from common/src/static_cell.rs rename to crates/common/src/static_cell.rs diff --git a/common/src/str.rs b/crates/common/src/str.rs similarity index 98% rename from common/src/str.rs rename to crates/common/src/str.rs index af30ed6dec..2d867130ed 100644 --- a/common/src/str.rs +++ b/crates/common/src/str.rs @@ -487,7 +487,7 @@ pub mod levenshtein { if a == b { CASE_COST } else { MOVE_COST } } - pub fn levenshtein_distance(a: &str, b: &str, max_cost: usize) -> usize { + pub fn levenshtein_distance(a: &[u8], b: &[u8], max_cost: usize) -> usize { thread_local! { #[allow(clippy::declare_interior_mutable_const)] static BUFFER: RefCell<[usize; MAX_STRING_SIZE]> = const { @@ -499,7 +499,7 @@ pub mod levenshtein { return 0; } - let (mut a_bytes, mut b_bytes) = (a.as_bytes(), b.as_bytes()); + let (mut a_bytes, mut b_bytes) = (a, b); let (mut a_begin, mut a_end) = (0usize, a.len()); let (mut b_begin, mut b_end) = (0usize, b.len()); @@ -533,10 +533,9 @@ pub mod levenshtein { return max_cost + 1; } - BUFFER.with(|buffer| { - let mut buffer = buffer.borrow_mut(); - for i in 0..a_end { - buffer[i] = (i + 1) * MOVE_COST; + BUFFER.with_borrow_mut(|buffer| { + for (i, x) in buffer.iter_mut().take(a_end).enumerate() { + *x = (i + 1) * MOVE_COST; } let mut result = 0usize; diff --git a/common/src/windows.rs b/crates/common/src/windows.rs similarity index 57% rename from common/src/windows.rs rename to crates/common/src/windows.rs index 4a922ce435..a934a118ec 100644 --- a/common/src/windows.rs +++ b/crates/common/src/windows.rs @@ -1,12 +1,17 @@ +use rustpython_wtf8::Wtf8; use std::{ ffi::{OsStr, OsString}, os::windows::ffi::{OsStrExt, OsStringExt}, }; +/// _MAX_ENV from Windows CRT stdlib.h - maximum environment variable size +pub const _MAX_ENV: usize = 32767; + pub trait ToWideString { fn to_wide(&self) -> Vec; fn to_wide_with_nul(&self) -> Vec; } + impl ToWideString for T where T: AsRef, @@ -19,6 +24,24 @@ where } } +impl ToWideString for OsStr { + fn to_wide(&self) -> Vec { + self.encode_wide().collect() + } + fn to_wide_with_nul(&self) -> Vec { + self.encode_wide().chain(Some(0)).collect() + } +} + +impl ToWideString for Wtf8 { + fn to_wide(&self) -> Vec { + self.encode_wide().collect() + } + fn to_wide_with_nul(&self) -> Vec { + self.encode_wide().chain(Some(0)).collect() + } +} + pub trait FromWideString where Self: Sized, diff --git a/compiler/core/Cargo.toml b/crates/compiler-core/Cargo.toml similarity index 81% rename from compiler/core/Cargo.toml rename to crates/compiler-core/Cargo.toml index 837f9e4866..f4e619b95a 100644 --- a/compiler/core/Cargo.toml +++ b/crates/compiler-core/Cargo.toml @@ -17,9 +17,8 @@ bitflags = { workspace = true } itertools = { workspace = true } malachite-bigint = { workspace = true } num-complex = { workspace = true } -serde = { workspace = true, optional = true, default-features = false, features = ["derive"] } -lz4_flex = "0.11" +lz4_flex = "0.12" [lints] -workspace = true \ No newline at end of file +workspace = true diff --git a/compiler/core/src/bytecode.rs b/crates/compiler-core/src/bytecode.rs similarity index 64% rename from compiler/core/src/bytecode.rs rename to crates/compiler-core/src/bytecode.rs index be55fe3502..8df5d9caf6 100644 --- a/compiler/core/src/bytecode.rs +++ b/crates/compiler-core/src/bytecode.rs @@ -1,27 +1,166 @@ //! Implement python as a virtual machine with bytecode. This module //! implements bytecode structure. +use crate::{ + marshal::MarshalError, + {OneIndexed, SourceLocation}, +}; use bitflags::bitflags; use itertools::Itertools; use malachite_bigint::BigInt; use num_complex::Complex64; -use ruff_source_file::{OneIndexed, SourceLocation}; use rustpython_wtf8::{Wtf8, Wtf8Buf}; -use std::marker::PhantomData; -use std::{collections::BTreeSet, fmt, hash, mem}; +use std::{collections::BTreeSet, fmt, hash, marker::PhantomData, mem, num::NonZeroU8, ops::Deref}; +/// Oparg values for [`Instruction::ConvertValue`]. +/// +/// ## See also +/// +/// - [CPython FVC_* flags](https://github.com/python/cpython/blob/8183fa5e3f78ca6ab862de7fb8b14f3d929421e0/Include/ceval.h#L129-L132) +#[repr(u8)] #[derive(Copy, Clone, Debug, Hash, PartialEq, Eq)] -#[repr(i8)] -#[allow(clippy::cast_possible_wrap)] -pub enum ConversionFlag { - /// No conversion - None = -1, // CPython uses -1 +pub enum ConvertValueOparg { + /// No conversion. + /// + /// ```python + /// f"{x}" + /// f"{x:4}" + /// ``` + None = 0, /// Converts by calling `str()`. - Str = b's' as i8, - /// Converts by calling `ascii()`. - Ascii = b'a' as i8, + /// + /// ```python + /// f"{x!s}" + /// f"{x!s:2}" + /// ``` + Str = 1, /// Converts by calling `repr()`. - Repr = b'r' as i8, + /// + /// ```python + /// f"{x!r}" + /// f"{x!r:2}" + /// ``` + Repr = 2, + /// Converts by calling `ascii()`. + /// + /// ```python + /// f"{x!a}" + /// f"{x!a:2}" + /// ``` + Ascii = 3, +} + +impl fmt::Display for ConvertValueOparg { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + let out = match self { + Self::Str => "1 (str)", + Self::Repr => "2 (repr)", + Self::Ascii => "3 (ascii)", + // We should never reach this. `FVC_NONE` are being handled by `Instruction::FormatSimple` + Self::None => "", + }; + + write!(f, "{out}") + } +} + +impl OpArgType for ConvertValueOparg { + #[inline] + fn from_op_arg(x: u32) -> Option { + Some(match x { + // Ruff `ConversionFlag::None` is `-1i8`, + // when its converted to `u8` its value is `u8::MAX` + 0 | 255 => Self::None, + 1 => Self::Str, + 2 => Self::Repr, + 3 => Self::Ascii, + _ => return None, + }) + } + + #[inline] + fn to_op_arg(self) -> u32 { + self as u32 + } +} + +/// Resume type for the RESUME instruction +#[derive(Copy, Clone, Debug, Hash, PartialEq, Eq)] +#[repr(u32)] +pub enum ResumeType { + AtFuncStart = 0, + AfterYield = 1, + AfterYieldFrom = 2, + AfterAwait = 3, +} + +/// CPython 3.11+ linetable location info codes +#[derive(Copy, Clone, Debug, PartialEq, Eq)] +#[repr(u8)] +pub enum PyCodeLocationInfoKind { + // Short forms are 0 to 9 + Short0 = 0, + Short1 = 1, + Short2 = 2, + Short3 = 3, + Short4 = 4, + Short5 = 5, + Short6 = 6, + Short7 = 7, + Short8 = 8, + Short9 = 9, + // One line forms are 10 to 12 + OneLine0 = 10, + OneLine1 = 11, + OneLine2 = 12, + NoColumns = 13, + Long = 14, + None = 15, +} + +impl PyCodeLocationInfoKind { + pub fn from_code(code: u8) -> Option { + match code { + 0 => Some(Self::Short0), + 1 => Some(Self::Short1), + 2 => Some(Self::Short2), + 3 => Some(Self::Short3), + 4 => Some(Self::Short4), + 5 => Some(Self::Short5), + 6 => Some(Self::Short6), + 7 => Some(Self::Short7), + 8 => Some(Self::Short8), + 9 => Some(Self::Short9), + 10 => Some(Self::OneLine0), + 11 => Some(Self::OneLine1), + 12 => Some(Self::OneLine2), + 13 => Some(Self::NoColumns), + 14 => Some(Self::Long), + 15 => Some(Self::None), + _ => Option::None, + } + } + + pub fn is_short(&self) -> bool { + (*self as u8) <= 9 + } + + pub fn short_column_group(&self) -> Option { + if self.is_short() { + Some(*self as u8) + } else { + Option::None + } + } + + pub fn one_line_delta(&self) -> Option { + match self { + Self::OneLine0 => Some(0), + Self::OneLine1 => Some(1), + Self::OneLine2 => Some(2), + _ => Option::None, + } + } } pub trait Constant: Sized { @@ -33,8 +172,10 @@ pub trait Constant: Sized { impl Constant for ConstantData { type Name = String; + fn borrow_constant(&self) -> BorrowedConstant<'_, Self> { use BorrowedConstant::*; + match self { Self::Integer { value } => Integer { value }, Self::Float { value } => Float { value: *value }, @@ -53,21 +194,28 @@ impl Constant for ConstantData { /// A Constant Bag pub trait ConstantBag: Sized + Copy { type Constant: Constant; + fn make_constant(&self, constant: BorrowedConstant<'_, C>) -> Self::Constant; + fn make_int(&self, value: BigInt) -> Self::Constant; + fn make_tuple(&self, elements: impl Iterator) -> Self::Constant; + fn make_code(&self, code: CodeObject) -> Self::Constant; + fn make_name(&self, name: &str) -> ::Name; } pub trait AsBag { type Bag: ConstantBag; + #[allow(clippy::wrong_self_convention)] fn as_bag(self) -> Self::Bag; } impl AsBag for Bag { type Bag = Self; + fn as_bag(self) -> Self { self } @@ -78,22 +226,27 @@ pub struct BasicBag; impl ConstantBag for BasicBag { type Constant = ConstantData; + fn make_constant(&self, constant: BorrowedConstant<'_, C>) -> Self::Constant { constant.to_owned() } + fn make_int(&self, value: BigInt) -> Self::Constant { ConstantData::Integer { value } } + fn make_tuple(&self, elements: impl Iterator) -> Self::Constant { ConstantData::Tuple { elements: elements.collect(), } } + fn make_code(&self, code: CodeObject) -> Self::Constant { ConstantData::Code { code: Box::new(code), } } + fn make_name(&self, name: &str) -> ::Name { name.to_owned() } @@ -103,24 +256,30 @@ impl ConstantBag for BasicBag { /// a code object. Also a module has a code object. #[derive(Clone)] pub struct CodeObject { - pub instructions: Box<[CodeUnit]>, + pub instructions: CodeUnits, pub locations: Box<[SourceLocation]>, pub flags: CodeFlags, + /// Number of positional-only arguments pub posonlyarg_count: u32, - // Number of positional-only arguments pub arg_count: u32, pub kwonlyarg_count: u32, pub source_path: C::Name, pub first_line_number: Option, pub max_stackdepth: u32, + /// Name of the object that created this code object pub obj_name: C::Name, - // Name of the object that created this code object + /// Qualified name of the object (like CPython's co_qualname) + pub qualname: C::Name, pub cell2arg: Option>, pub constants: Box<[C]>, pub names: Box<[C::Name]>, pub varnames: Box<[C::Name]>, pub cellvars: Box<[C::Name]>, pub freevars: Box<[C::Name]>, + /// Line number table (CPython 3.11+ format) + pub linetable: Box<[u8]>, + /// Exception handling table + pub exceptiontable: Box<[u8]>, } bitflags! { @@ -152,11 +311,19 @@ impl CodeFlags { #[derive(Copy, Clone, PartialEq, Eq)] #[repr(transparent)] pub struct OpArgByte(pub u8); + impl OpArgByte { pub const fn null() -> Self { Self(0) } } + +impl From for OpArgByte { + fn from(raw: u8) -> Self { + Self(raw) + } +} + impl fmt::Debug for OpArgByte { fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { self.0.fmt(f) @@ -167,6 +334,7 @@ impl fmt::Debug for OpArgByte { #[derive(Copy, Clone, Debug)] #[repr(transparent)] pub struct OpArg(pub u32); + impl OpArg { pub const fn null() -> Self { Self(0) @@ -174,7 +342,7 @@ impl OpArg { /// Returns how many CodeUnits a instruction with this op_arg will be encoded as #[inline] - pub fn instr_size(self) -> usize { + pub const fn instr_size(self) -> usize { (self.0 > 0xff) as usize + (self.0 > 0xff_ff) as usize + (self.0 > 0xff_ff_ff) as usize + 1 } @@ -193,6 +361,12 @@ impl OpArg { } } +impl From for OpArg { + fn from(raw: u32) -> Self { + Self(raw) + } +} + #[derive(Default, Copy, Clone)] #[repr(transparent)] pub struct OpArgState { @@ -208,19 +382,22 @@ impl OpArgState { } (ins.op, arg) } + #[inline(always)] pub fn extend(&mut self, arg: OpArgByte) -> OpArg { self.state = (self.state << 8) | u32::from(arg.0); OpArg(self.state) } + #[inline(always)] - pub fn reset(&mut self) { + pub const fn reset(&mut self) { self.state = 0 } } pub trait OpArgType: Copy { fn from_op_arg(x: u32) -> Option; + fn to_op_arg(self) -> u32; } @@ -229,6 +406,7 @@ impl OpArgType for u32 { fn from_op_arg(x: u32) -> Option { Some(x) } + #[inline(always)] fn to_op_arg(self) -> u32 { self @@ -240,6 +418,7 @@ impl OpArgType for bool { fn from_op_arg(x: u32) -> Option { Some(x != 0) } + #[inline(always)] fn to_op_arg(self) -> u32 { self as u32 @@ -252,6 +431,7 @@ macro_rules! op_arg_enum_impl { fn to_op_arg(self) -> u32 { self as u32 } + fn from_op_arg(x: u32) -> Option { Some(match u8::try_from(x).ok()? { $($value => Self::$var,)* @@ -280,13 +460,15 @@ pub struct Arg(PhantomData); impl Arg { #[inline] - pub fn marker() -> Self { + pub const fn marker() -> Self { Self(PhantomData) } + #[inline] pub fn new(arg: T) -> (Self, OpArg) { (Self(PhantomData), OpArg(arg.to_op_arg())) } + #[inline] pub fn new_single(arg: T) -> (Self, OpArgByte) where @@ -294,17 +476,20 @@ impl Arg { { (Self(PhantomData), OpArgByte(arg.into())) } + #[inline(always)] pub fn get(self, arg: OpArg) -> T { self.try_get(arg).unwrap() } + #[inline(always)] pub fn try_get(self, arg: OpArg) -> Option { T::from_op_arg(arg.0) } - #[inline(always)] + /// # Safety /// T::from_op_arg(self) must succeed + #[inline(always)] pub unsafe fn get_unchecked(self, arg: OpArg) -> T { // SAFETY: requirements forwarded from caller unsafe { T::from_op_arg(arg.0).unwrap_unchecked() } @@ -316,6 +501,7 @@ impl PartialEq for Arg { true } } + impl Eq for Arg {} impl fmt::Debug for Arg { @@ -335,6 +521,7 @@ impl OpArgType for Label { fn from_op_arg(x: u32) -> Option { Some(Self(x)) } + #[inline(always)] fn to_op_arg(self) -> u32 { self.0 @@ -347,23 +534,6 @@ impl fmt::Display for Label { } } -impl OpArgType for ConversionFlag { - #[inline] - fn from_op_arg(x: u32) -> Option { - match x as u8 { - b's' => Some(Self::Str), - b'a' => Some(Self::Ascii), - b'r' => Some(Self::Repr), - std::u8::MAX => Some(Self::None), - _ => None, - } - } - #[inline] - fn to_op_arg(self) -> u32 { - self as i8 as u8 as u32 - } -} - op_arg_enum!( /// The kind of Raise that occurred. #[derive(Copy, Clone, Debug, PartialEq, Eq)] @@ -380,27 +550,22 @@ op_arg_enum!( #[derive(Copy, Clone, Debug, PartialEq, Eq)] #[repr(u8)] pub enum IntrinsicFunction1 { - /// Import * special case - // ImportStar = 0, - /// Set stop iteration value - // StopAsyncIteration = 1, - /// Unary operators - // UnaryPositive = 2, - // UnaryNegative = 3, - // UnaryNot = 4, - // UnaryInvert = 5, - /// Exit init subclass - // ExitInitCheck = 6, - /// Create a new list from an iterator - // ListToTupleForCall = 7, + // Invalid = 0, + Print = 1, + /// Import * operation + ImportStar = 2, + // StopIterationError = 3, + // AsyncGenWrap = 4, + // UnaryPositive = 5, + /// Convert list to tuple + ListToTuple = 6, /// Type parameter related - // TypeVar = 8, - // TypeVarTuple = 9, - // ParamSpec = 10, + TypeVar = 7, + ParamSpec = 8, + TypeVarTuple = 9, /// Generic subscript for PEP 695 SubscriptGeneric = 10, - // TypeAlias = 12, - // TypeParams = 13, + TypeAlias = 11, } ); @@ -409,10 +574,10 @@ op_arg_enum!( #[derive(Copy, Clone, Debug, PartialEq, Eq)] #[repr(u8)] pub enum IntrinsicFunction2 { - // PrepReraiseS tar = 1, - // TypeVarWithBound = 2, - // TypeVarWithConstraints = 3, - // SetFunctionTypeParams = 4, + PrepReraiseStar = 1, + TypeVarWithBound = 2, + TypeVarWithConstraint = 3, + SetFunctionTypeParams = 4, /// Set default value for type parameter (PEP 695) SetTypeparamDefault = 5, } @@ -424,146 +589,245 @@ pub type NameIdx = u32; #[derive(Debug, Copy, Clone, PartialEq, Eq)] #[repr(u8)] pub enum Instruction { - Nop, - /// Importing by name - ImportName { - idx: Arg, + BeforeAsyncWith, + BinaryOp { + op: Arg, }, - /// Importing without name - ImportNameless, - /// Import * - ImportStar, - /// from ... import ... - ImportFrom { - idx: Arg, + BinarySubscript, + Break { + target: Arg