diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 11375e966b3..d2006f366f3 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -5,6 +5,7 @@ By submitting these contributions you agree for them to be dual-licensed under P Please consider adding the following to your pull request: - an entry for this PR in newsfragments - see [https://pyo3.rs/main/contributing.html#documenting-changes] - or start the PR title with `docs:` if this is a docs-only change to skip the check + - or start the PR title with `ci:` if this is a ci-only change to skip the check - docs to all new functions and / or detail in the guide - tests for all new or changed functions diff --git a/.github/workflows/benches.yml b/.github/workflows/benches.yml index 62c64d9d113..73005715515 100644 --- a/.github/workflows/benches.yml +++ b/.github/workflows/benches.yml @@ -20,7 +20,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: - python-version: '3.12' + python-version: '3.13' - uses: dtolnay/rust-toolchain@stable with: components: rust-src @@ -30,7 +30,7 @@ jobs: workspaces: | . pyo3-benches - continue-on-error: true + save-if: ${{ github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'CI-save-pr-cache') }} - uses: taiki-e/install-action@v2 with: diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5fb2124836e..60744cadbf7 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -19,6 +19,9 @@ on: MSRV: required: true type: string + verbose: + type: boolean + default: false jobs: build: @@ -27,32 +30,52 @@ jobs: if: ${{ !(startsWith(inputs.python-version, 'graalpy') && startsWith(inputs.os, 'windows')) }} steps: - uses: actions/checkout@v4 + with: + # For PRs, we need to run on the real PR head, not the resultant merge of the PR into the target branch. + # + # This is necessary for coverage reporting to make sense; we then get exactly the coverage change + # between the base branch and the real PR head. + # + # If it were run on the merge commit the problem is that the coverage potentially does not align + # with the commit diff, because the merge may affect line numbers. + ref: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} - name: Set up Python ${{ inputs.python-version }} uses: actions/setup-python@v5 with: python-version: ${{ inputs.python-version }} architecture: ${{ inputs.python-architecture }} - check-latest: ${{ startsWith(inputs.python-version, 'pypy') }} # PyPy can have FFI changes within Python versions, which creates pain in CI + # PyPy can have FFI changes within Python versions, which creates pain in CI + # 3.13.2 also had an ABI break so temporarily add this for 3.13 to ensure that we're using 3.13.3 + check-latest: ${{ startsWith(inputs.python-version, 'pypy') || startsWith(inputs.python-version, '3.13') }} - name: Install nox run: python -m pip install --upgrade pip && pip install nox - - if: inputs.python-version == 'graalpy24.1' - name: Install GraalPy virtualenv (only GraalPy 24.1) - run: python -m pip install 'git+https://github.com/oracle/graalpython#egg=graalpy_virtualenv_seeder&subdirectory=graalpy_virtualenv_seeder' - - name: Install Rust toolchain uses: dtolnay/rust-toolchain@master with: toolchain: ${{ inputs.rust }} targets: ${{ inputs.rust-target }} - # needed to correctly format errors, see #1865 - components: rust-src + # rust-src needed to correctly format errors, see #1865 + components: rust-src,llvm-tools-preview + + # On windows 32 bit, we are running on an x64 host, so we need to specifically set the target + # NB we don't do this for *all* jobs because it breaks coverage of proc macros to have an + # explicit target set. + - name: Set Rust target for Windows 32-bit + if: inputs.os == 'windows-latest' && inputs.python-architecture == 'x86' + shell: bash + run: | + echo "CARGO_BUILD_TARGET=i686-pc-windows-msvc" >> $GITHUB_ENV + + - name: Install zoneinfo backport for Python 3.7 / 3.8 + if: contains(fromJSON('["3.7", "3.8"]'), inputs.python-version) + run: python -m pip install backports.zoneinfo - uses: Swatinem/rust-cache@v2 with: - save-if: ${{ github.event_name != 'merge_group' }} + save-if: ${{ github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'CI-save-pr-cache') }} - if: inputs.os == 'ubuntu-latest' name: Prepare LD_LIBRARY_PATH (Ubuntu only) @@ -66,80 +89,12 @@ jobs: name: Ignore changed error messages when using trybuild run: echo "TRYBUILD=overwrite" >> "$GITHUB_ENV" - - if: inputs.rust == 'nightly' - name: Prepare to test on nightly rust - run: echo "MAYBE_NIGHTLY=nightly" >> "$GITHUB_ENV" - - - name: Build docs - run: nox -s docs - - - name: Build (no features) - if: ${{ !startsWith(inputs.python-version, 'graalpy') }} - run: cargo build --lib --tests --no-default-features - - # --no-default-features when used with `cargo build/test -p` doesn't seem to work! - - name: Build pyo3-build-config (no features) - run: | - cd pyo3-build-config - cargo build --no-default-features - - # Run tests (except on PyPy, because no embedding API). - - if: ${{ !startsWith(inputs.python-version, 'pypy') && !startsWith(inputs.python-version, 'graalpy') }} - name: Test (no features) - run: cargo test --no-default-features --lib --tests - - # --no-default-features when used with `cargo build/test -p` doesn't seem to work! - - name: Test pyo3-build-config (no features) - run: | - cd pyo3-build-config - cargo test --no-default-features - - - name: Build (all additive features) - if: ${{ !startsWith(inputs.python-version, 'graalpy') }} - run: cargo build --lib --tests --no-default-features --features "multiple-pymethods full $MAYBE_NIGHTLY" - - - if: ${{ startsWith(inputs.python-version, 'pypy') }} - name: Build PyPy (abi3-py39) - run: cargo build --lib --tests --no-default-features --features "multiple-pymethods abi3-py39 full $MAYBE_NIGHTLY" - - # Run tests (except on PyPy, because no embedding API). - - if: ${{ !startsWith(inputs.python-version, 'pypy') && !startsWith(inputs.python-version, 'graalpy') }} - name: Test - run: cargo test --no-default-features --features "full $MAYBE_NIGHTLY" - - # Repeat, with multiple-pymethods feature enabled (it's not entirely additive) - - if: ${{ !startsWith(inputs.python-version, 'pypy') && !startsWith(inputs.python-version, 'graalpy') }} - name: Test - run: cargo test --no-default-features --features "multiple-pymethods full $MAYBE_NIGHTLY" - - # Run tests again, but in abi3 mode - - if: ${{ !startsWith(inputs.python-version, 'pypy') && !startsWith(inputs.python-version, 'graalpy') }} - name: Test (abi3) - run: cargo test --no-default-features --features "multiple-pymethods abi3 full $MAYBE_NIGHTLY" - - # Run tests again, for abi3-py37 (the minimal Python version) - - if: ${{ (!startsWith(inputs.python-version, 'pypy') && !startsWith(inputs.python-version, 'graalpy')) && (inputs.python-version != '3.7') }} - name: Test (abi3-py37) - run: cargo test --no-default-features --features "multiple-pymethods abi3-py37 full $MAYBE_NIGHTLY" - - - name: Test proc-macro code - run: cargo test --manifest-path=pyo3-macros-backend/Cargo.toml - - - name: Test build config - run: cargo test --manifest-path=pyo3-build-config/Cargo.toml - - - name: Test python examples and tests - shell: bash - run: nox -s test-py - env: - CARGO_TARGET_DIR: ${{ github.workspace }}/target - - uses: dorny/paths-filter@v3 if: ${{ inputs.rust == 'stable' && !startsWith(inputs.python-version, 'graalpy') }} id: ffi-changes with: - base: ${{ github.event.pull_request.base.ref || github.event.merge_group.base_ref }} - ref: ${{ github.event.pull_request.head.ref || github.event.merge_group.head_ref }} + base: ${{ github.event.merge_group.base_ref }} + ref: ${{ github.event.merge_group.head_ref }} filters: | changed: - 'pyo3-ffi/**' @@ -152,9 +107,49 @@ jobs: if: ${{ endsWith(inputs.python-version, '-dev') || (steps.ffi-changes.outputs.changed == 'true' && inputs.rust == 'stable' && !startsWith(inputs.python-version, 'graalpy') && !(inputs.python-version == 'pypy3.9' && contains(inputs.os, 'windows'))) }} run: nox -s ffi-check + - if: ${{ github.event_name != 'merge_group' }} + name: Install cargo-llvm-cov + uses: taiki-e/install-action@cargo-llvm-cov + + - if: ${{ github.event_name != 'merge_group' }} + name: Prepare coverage environment + run: | + cargo llvm-cov clean --workspace --profraw-only + nox -s set-coverage-env + + - name: Build docs + run: nox -s docs + + - name: Run Rust tests + run: nox -s test-rust + + - name: Test python examples and tests + shell: bash + run: nox -s test-py + continue-on-error: ${{ endsWith(inputs.python-version, '-dev') }} + env: + CARGO_TARGET_DIR: ${{ github.workspace }}/target + + - if: ${{ github.event_name != 'merge_group' }} + name: Generate coverage report + run: cargo llvm-cov + --package=pyo3 + --package=pyo3-build-config + --package=pyo3-macros-backend + --package=pyo3-macros + --package=pyo3-ffi + report --codecov --output-path coverage.json + + - if: ${{ github.event_name != 'merge_group' }} + name: Upload coverage report + uses: codecov/codecov-action@v5 + with: + files: coverage.json + name: ${{ inputs.os }}/${{ inputs.python-version }}/${{ inputs.rust }} + token: ${{ secrets.CODECOV_TOKEN }} + env: - CARGO_TERM_VERBOSE: true - CARGO_BUILD_TARGET: ${{ inputs.rust-target }} + CARGO_TERM_VERBOSE: ${{ inputs.verbose }} RUST_BACKTRACE: 1 RUSTFLAGS: "-D warnings" RUSTDOCFLAGS: "-D warnings" diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml index b4c3b9892f3..798a8dd06f9 100644 --- a/.github/workflows/changelog.yml +++ b/.github/workflows/changelog.yml @@ -12,6 +12,6 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: - python-version: '3.12' + python-version: '3.13' - run: python -m pip install --upgrade pip && pip install nox - run: nox -s check-changelog diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 18367f353a0..428b78e5c9d 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -23,7 +23,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: - python-version: "3.12" + python-version: "3.13" - run: python -m pip install --upgrade pip && pip install nox - uses: dtolnay/rust-toolchain@stable with: @@ -37,11 +37,12 @@ jobs: runs-on: ubuntu-latest outputs: MSRV: ${{ steps.resolve-msrv.outputs.MSRV }} + verbose: ${{ runner.debug == '1' }} steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: - python-version: "3.12" + python-version: "3.13" - name: resolve MSRV id: resolve-msrv run: echo MSRV=`python -c 'import tomllib; print(tomllib.load(open("Cargo.toml", "rb"))["package"]["rust-version"])'` >> $GITHUB_OUTPUT @@ -54,7 +55,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: - python-version: "3.12" + python-version: "3.13" - uses: obi1kenobi/cargo-semver-checks-action@v2 check-msrv: @@ -68,10 +69,10 @@ jobs: components: rust-src - uses: actions/setup-python@v5 with: - python-version: "3.12" + python-version: "3.13" - uses: Swatinem/rust-cache@v2 with: - save-if: ${{ github.event_name != 'merge_group' }} + save-if: ${{ github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'CI-save-pr-cache') }} - run: python -m pip install --upgrade pip && pip install nox # This is a smoke test to confirm that CI will run on MSRV (including dev dependencies) - name: Check with MSRV package versions @@ -153,11 +154,11 @@ jobs: components: clippy,rust-src - uses: actions/setup-python@v5 with: - python-version: "3.12" + python-version: "3.13" architecture: ${{ matrix.platform.python-architecture }} - uses: Swatinem/rust-cache@v2 with: - save-if: ${{ github.event_name != 'merge_group' }} + save-if: ${{ github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'CI-save-pr-cache') }} - run: python -m pip install --upgrade pip && pip install nox - run: nox -s clippy-all env: @@ -175,13 +176,14 @@ jobs: rust: ${{ matrix.rust }} rust-target: ${{ matrix.platform.rust-target }} MSRV: ${{ needs.resolve.outputs.MSRV }} + verbose: ${{ needs.resolve.outputs.verbose == 'true' }} secrets: inherit strategy: # If one platform fails, allow the rest to keep testing if `CI-no-fail-fast` label is present fail-fast: ${{ !contains(github.event.pull_request.labels.*.name, 'CI-no-fail-fast') }} matrix: rust: [stable] - python-version: ["3.12"] + python-version: ["3.13"] platform: [ { os: "macos-latest", # first available arm macos runner @@ -218,13 +220,24 @@ jobs: # Test nightly Rust on PRs so that PR authors have a chance to fix nightly # failures, as nightly does not block merge. - rust: nightly - python-version: "3.12" + python-version: "3.13" platform: { os: "ubuntu-latest", python-architecture: "x64", rust-target: "x86_64-unknown-linux-gnu", } + # Also test free-threaded Python just for latest Python version, on ubuntu + # (run for all OSes on build-full) + - rust: stable + python-version: "3.13t" + platform: + { + os: "ubuntu-latest", + python-architecture: "x64", + rust-target: "x86_64-unknown-linux-gnu", + } + build-full: if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || github.event_name != 'pull_request' }} name: python${{ matrix.python-version }}-${{ matrix.platform.python-architecture }} ${{ matrix.platform.os }} rust-${{ matrix.rust }} @@ -237,6 +250,7 @@ jobs: rust: ${{ matrix.rust }} rust-target: ${{ matrix.platform.rust-target }} MSRV: ${{ needs.resolve.outputs.MSRV }} + verbose: ${{ needs.resolve.outputs.verbose == 'true' }} secrets: inherit strategy: # If one platform fails, allow the rest to keep testing if `CI-no-fail-fast` label is present @@ -252,11 +266,13 @@ jobs: "3.11", "3.12", "3.13", + "3.13t", + "3.14-dev", + "3.14t-dev", "pypy3.9", "pypy3.10", "pypy3.11", - "graalpy24.0", - "graalpy24.1", + "graalpy24.2", ] platform: [ @@ -279,7 +295,7 @@ jobs: include: # Test minimal supported Rust version - rust: ${{ needs.resolve.outputs.MSRV }} - python-version: "3.12" + python-version: "3.13" platform: { os: "ubuntu-latest", @@ -289,7 +305,7 @@ jobs: # Test the `nightly` feature - rust: nightly - python-version: "3.12" + python-version: "3.13" platform: { os: "ubuntu-latest", @@ -299,7 +315,7 @@ jobs: # Run rust beta to help catch toolchain regressions - rust: beta - python-version: "3.12" + python-version: "3.13" platform: { os: "ubuntu-latest", @@ -309,7 +325,7 @@ jobs: # Test 32-bit Windows and x64 macOS only with the latest Python version - rust: stable - python-version: "3.12" + python-version: "3.13" platform: { os: "windows-latest", @@ -317,7 +333,7 @@ jobs: rust-target: "i686-pc-windows-msvc", } - rust: stable - python-version: "3.12" + python-version: "3.13" platform: { os: "macos-13", @@ -408,12 +424,10 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: - # FIXME valgrind detects an issue with Python 3.12.5, needs investigation - # whether it's a PyO3 issue or upstream CPython. - python-version: "3.12.4" + python-version: "3.13" - uses: Swatinem/rust-cache@v2 with: - save-if: ${{ github.event_name != 'merge_group' }} + save-if: ${{ github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'CI-save-pr-cache') }} - uses: dtolnay/rust-toolchain@stable - uses: taiki-e/install-action@valgrind - run: python -m pip install --upgrade pip && pip install nox @@ -431,10 +445,10 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: - python-version: "3.12" + python-version: "3.13" - uses: Swatinem/rust-cache@v2 with: - save-if: ${{ github.event_name != 'merge_group' }} + save-if: ${{ github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'CI-save-pr-cache') }} - uses: dtolnay/rust-toolchain@nightly with: components: rust-src @@ -453,44 +467,15 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: - python-version: "3.12" + python-version: "3.13" - uses: Swatinem/rust-cache@v2 with: - save-if: ${{ github.event_name != 'merge_group' }} + save-if: ${{ github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'CI-save-pr-cache') }} - uses: dtolnay/rust-toolchain@nightly with: components: rust-src - run: cargo rustdoc --lib --no-default-features --features full,jiff-02 -Zunstable-options --config "build.rustdocflags=[\"--cfg\", \"docsrs\"]" - coverage: - if: ${{ github.event_name != 'merge_group' }} - needs: [fmt] - name: coverage ${{ matrix.os }} - strategy: - matrix: - os: ["windows-latest", "macos-latest", "ubuntu-latest"] - runs-on: ${{ matrix.os }} - steps: - - uses: actions/checkout@v4 - - uses: actions/setup-python@v5 - with: - python-version: "3.12" - - uses: Swatinem/rust-cache@v2 - with: - save-if: ${{ github.event_name != 'merge_group' }} - - uses: dtolnay/rust-toolchain@stable - with: - components: llvm-tools-preview,rust-src - - name: Install cargo-llvm-cov - uses: taiki-e/install-action@cargo-llvm-cov - - run: python -m pip install --upgrade pip && pip install nox - - run: nox -s coverage - - uses: codecov/codecov-action@v5 - with: - files: coverage.json - name: ${{ matrix.os }} - token: ${{ secrets.CODECOV_TOKEN }} - emscripten: name: emscripten if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || github.event_name != 'pull_request' }} @@ -500,7 +485,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: - # TODO bump emscripten builds to test on 3.12 + # TODO bump emscripten builds to test on 3.13 python-version: 3.11 id: setup-python - name: Install Rust toolchain @@ -520,14 +505,14 @@ jobs: key: emscripten-${{ hashFiles('emscripten/*') }}-${{ hashFiles('noxfile.py') }}-${{ steps.setup-python.outputs.python-path }} - uses: Swatinem/rust-cache@v2 with: - save-if: ${{ github.event_name != 'merge_group' }} + save-if: ${{ github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'CI-save-pr-cache') }} - name: Build if: steps.cache.outputs.cache-hit != 'true' run: nox -s build-emscripten - name: Test run: nox -s test-emscripten - uses: actions/cache/save@v4 - if: ${{ github.event_name != 'merge_group' }} + if: ${{ github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'CI-save-pr-cache') }} with: path: | .nox/emscripten @@ -541,14 +526,14 @@ jobs: - uses: actions/checkout@v4 - uses: Swatinem/rust-cache@v2 with: - save-if: ${{ github.event_name != 'merge_group' }} + save-if: ${{ github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'CI-save-pr-cache') }} - uses: dtolnay/rust-toolchain@stable with: components: rust-src - name: Install python3 standalone debug build with nox run: | - PBS_RELEASE="20241016" - PBS_PYTHON_VERSION="3.13.0" + PBS_RELEASE="20241219" + PBS_PYTHON_VERSION="3.13.1" PBS_ARCHIVE="cpython-${PBS_PYTHON_VERSION}+${PBS_RELEASE}-x86_64-unknown-linux-gnu-debug-full.tar.zst" wget "https://github.com/indygreg/python-build-standalone/releases/download/${PBS_RELEASE}/${PBS_ARCHIVE}" tar -I zstd -xf "${PBS_ARCHIVE}" @@ -577,43 +562,6 @@ jobs: echo PYO3_CONFIG_FILE=$PYO3_CONFIG_FILE >> $GITHUB_ENV - run: python3 -m nox -s test - test-free-threaded: - needs: [fmt] - name: Free threaded tests - ${{ matrix.os }} - runs-on: ${{ matrix.os }} - strategy: - matrix: - os: ["ubuntu-latest", "macos-latest", "windows-latest"] - steps: - - uses: actions/checkout@v4 - - uses: Swatinem/rust-cache@v2 - with: - save-if: ${{ github.event_name != 'merge_group' }} - - uses: dtolnay/rust-toolchain@stable - with: - components: rust-src - - uses: actions/setup-python@v5.5.0 - with: - python-version: "3.13t" - - name: Install cargo-llvm-cov - uses: taiki-e/install-action@cargo-llvm-cov - - run: python3 -m sysconfig - - run: python3 -m pip install --upgrade pip && pip install nox - - name: Prepare coverage environment - run: | - cargo llvm-cov clean --workspace --profraw-only - nox -s set-coverage-env - - run: nox -s ffi-check - - run: nox - - name: Generate coverage report - run: nox -s generate-coverage-report - - name: Upload coverage report - uses: codecov/codecov-action@v5 - with: - files: coverage.json - name: ${{ matrix.os }}-test-free-threaded - token: ${{ secrets.CODECOV_TOKEN }} - test-version-limits: needs: [fmt] if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || github.event_name != 'pull_request' }} @@ -622,10 +570,10 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: - python-version: "3.12" + python-version: "3.13" - uses: Swatinem/rust-cache@v2 with: - save-if: ${{ github.event_name != 'merge_group' }} + save-if: ${{ github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'CI-save-pr-cache') }} - uses: dtolnay/rust-toolchain@stable - run: python3 -m pip install --upgrade pip && pip install nox - run: python3 -m nox -s test-version-limits @@ -646,10 +594,10 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: - python-version: "3.12" + python-version: "3.13" - uses: Swatinem/rust-cache@v2 with: - save-if: ${{ github.event_name != 'merge_group' }} + save-if: ${{ github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'CI-save-pr-cache') }} - uses: dtolnay/rust-toolchain@master with: toolchain: stable @@ -672,17 +620,17 @@ jobs: # ubuntu "cross compile" to itself - os: "ubuntu-latest" target: "x86_64-unknown-linux-gnu" - flags: "-i python3.12" + flags: "-i python3.13" manylinux: auto # ubuntu x86_64 -> aarch64 - os: "ubuntu-latest" target: "aarch64-unknown-linux-gnu" - flags: "-i python3.12" + flags: "-i python3.13" manylinux: auto # ubuntu x86_64 -> windows x86_64 - os: "ubuntu-latest" target: "x86_64-pc-windows-gnu" - flags: "-i python3.12 --features generate-import-lib" + flags: "-i python3.13 --features generate-import-lib" # macos x86_64 -> aarch64 - os: "macos-13" # last x86_64 macos runners target: "aarch64-apple-darwin" @@ -692,16 +640,16 @@ jobs: # windows x86_64 -> aarch64 - os: "windows-latest" target: "aarch64-pc-windows-msvc" - flags: "-i python3.12 --features generate-import-lib" + flags: "-i python3.13 --features generate-import-lib" steps: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: - python-version: "3.12" + python-version: "3.13" - uses: Swatinem/rust-cache@v2 with: workspaces: examples/maturin-starter - save-if: ${{ github.event_name != 'merge_group' }} + save-if: ${{ github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'CI-save-pr-cache') }} key: ${{ matrix.target }} - name: Setup cross-compiler if: ${{ matrix.target == 'x86_64-pc-windows-gnu' }} @@ -727,11 +675,11 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: - python-version: "3.12" + python-version: "3.13" - uses: Swatinem/rust-cache@v2 with: workspaces: examples/maturin-starter - save-if: ${{ github.event_name != 'merge_group' }} + save-if: ${{ github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'CI-save-pr-cache') }} - uses: actions/cache/restore@v4 with: # https://github.com/PyO3/maturin/discussions/1953 @@ -749,7 +697,7 @@ jobs: cargo build --manifest-path examples/maturin-starter/Cargo.toml --features abi3 --target x86_64-pc-windows-gnu cargo xwin build --cross-compiler clang --manifest-path examples/maturin-starter/Cargo.toml --features abi3 --target x86_64-pc-windows-msvc # non-abi3 - export PYO3_CROSS_PYTHON_VERSION=3.12 + export PYO3_CROSS_PYTHON_VERSION=3.13 cargo build --manifest-path examples/maturin-starter/Cargo.toml --features generate-import-lib --target x86_64-pc-windows-gnu cargo xwin build --cross-compiler clang --manifest-path examples/maturin-starter/Cargo.toml --features generate-import-lib --target x86_64-pc-windows-msvc - if: ${{ github.ref == 'refs/heads/main' }} @@ -757,6 +705,54 @@ jobs: with: path: ~/.cache/cargo-xwin key: cargo-xwin-cache + + test-introspection: + needs: [fmt] + if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-test-introspection') || contains(github.event.pull_request.labels.*.name, 'CI-build-full') || github.event_name != 'pull_request' }} + strategy: + matrix: + platform: + [ + { + os: "macos-latest", + python-architecture: "arm64", + rust-target: "aarch64-apple-darwin", + }, + { + os: "ubuntu-latest", + python-architecture: "x64", + rust-target: "x86_64-unknown-linux-gnu", + }, + { + os: "windows-latest", + python-architecture: "x64", + rust-target: "x86_64-pc-windows-msvc", + }, + { + os: "windows-latest", + python-architecture: "x86", + rust-target: "i686-pc-windows-msvc", + }, + ] + runs-on: ${{ matrix.platform.os }} + steps: + - uses: actions/checkout@v4 + - uses: dtolnay/rust-toolchain@stable + with: + targets: ${{ matrix.platform.rust-target }} + components: rust-src + - uses: actions/setup-python@v5 + with: + python-version: "3.13" + architecture: ${{ matrix.platform.python-architecture }} + - uses: Swatinem/rust-cache@v2 + with: + save-if: ${{ github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'CI-save-pr-cache') }} + - run: python -m pip install --upgrade pip && pip install nox + - run: nox -s test-introspection + env: + CARGO_BUILD_TARGET: ${{ matrix.platform.rust-target }} + conclusion: needs: - fmt @@ -767,14 +763,13 @@ jobs: - valgrind - careful - docsrs - - coverage - emscripten - test-debug - - test-free-threaded - test-version-limits - check-feature-powerset - test-cross-compilation - test-cross-compilation-windows + - test-introspection if: always() runs-on: ubuntu-latest steps: diff --git a/.github/workflows/coverage-pr-base.yml b/.github/workflows/coverage-pr-base.yml index 33118697636..907baa01ed2 100644 --- a/.github/workflows/coverage-pr-base.yml +++ b/.github/workflows/coverage-pr-base.yml @@ -15,7 +15,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: - python-version: '3.12' + python-version: '3.13' - name: Set PR base on codecov run: | # fetch the merge commit between the PR base and head diff --git a/.github/workflows/gh-pages.yml b/.github/workflows/gh-pages.yml index 1dc5927b49e..56814be8715 100644 --- a/.github/workflows/gh-pages.yml +++ b/.github/workflows/gh-pages.yml @@ -24,7 +24,7 @@ jobs: - uses: actions/checkout@v4 - uses: actions/setup-python@v5 with: - python-version: '3.12' + python-version: '3.13' - uses: dtolnay/rust-toolchain@nightly - name: Setup mdBook diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml new file mode 100644 index 00000000000..3da11abdf7f --- /dev/null +++ b/.github/workflows/release.yaml @@ -0,0 +1,26 @@ +name: Release Rust Crate + +on: + push: + tags: + - "v*" + +jobs: + release: + runs-on: ubuntu-latest + environment: release + steps: + - name: Checkout repository + uses: actions/checkout@v4 + + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - run: python -m pip install --upgrade pip && pip install nox + + - uses: dtolnay/rust-toolchain@stable + + - name: Publish to crates.io + run: nox -s publish + env: + CARGO_REGISTRY_TOKEN: ${{ secrets.CARGO_REGISTRY_TOKEN }} diff --git a/CHANGELOG.md b/CHANGELOG.md index 1532d3a7e9f..b13c92caef9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,91 @@ To see unreleased changes, please see the [CHANGELOG on the main branch guide](h +## [0.25.0] - 2025-05-14 + +### Packaging + +- Support Python 3.14.0b1. [#4811](https://github.com/PyO3/pyo3/pull/4811) +- Bump supported GraalPy version to 24.2. [#5116](https://github.com/PyO3/pyo3/pull/5116) +- Add optional `bigdecimal` dependency to add conversions for `bigdecimal::BigDecimal`. [#5011](https://github.com/PyO3/pyo3/pull/5011) +- Add optional `time` dependency to add conversions for `time` types. [#5057](https://github.com/PyO3/pyo3/pull/5057) +- Remove `cfg-if` dependency. [#5110](https://github.com/PyO3/pyo3/pull/5110) +- Add optional `ordered_float` dependency to add conversions for `ordered_float::NotNan` and `ordered_float::OrderedFloat`. [#5114](https://github.com/PyO3/pyo3/pull/5114) + +### Added + +- Add initial type stub generation to the `experimental-inspect` feature. [#3977](https://github.com/PyO3/pyo3/pull/3977) +- Add `#[pyclass(generic)]` option to support runtime generic typing. [#4926](https://github.com/PyO3/pyo3/pull/4926) +- Implement `OnceExt` & `MutexExt` for `parking_lot` & `lock_api`. Use the new extension traits by enabling the `arc_lock`, `lock_api`, or `parking_lot` cargo features. [#5044](https://github.com/PyO3/pyo3/pull/5044) +- Implement `From`/`Into` for `Borrowed` -> `Py`. [#5054](https://github.com/PyO3/pyo3/pull/5054) +- Add `PyTzInfo` constructors. [#5055](https://github.com/PyO3/pyo3/pull/5055) +- Add FFI definition `PY_INVALID_STACK_EFFECT`. [#5064](https://github.com/PyO3/pyo3/pull/5064) +- Implement `AsRef>` for `Py`, `Bound` and `Borrowed`. [#5071](https://github.com/PyO3/pyo3/pull/5071) +- Add FFI definition `PyModule_Add` and `compat::PyModule_Add`. [#5085](https://github.com/PyO3/pyo3/pull/5085) +- Add FFI definitions `Py_HashBuffer`, `Py_HashPointer`, and `PyObject_GenericHash`. [#5086](https://github.com/PyO3/pyo3/pull/5086) +- Support `#[pymodule_export]` on `const` items in declarative modules. [#5096](https://github.com/PyO3/pyo3/pull/5096) +- Add `#[pyclass(immutable_type)]` option (on Python 3.14+ with `abi3`, or 3.10+ otherwise) for immutable type objects. [#5101](https://github.com/PyO3/pyo3/pull/5101) +- Support `#[pyo3(rename_all)]` support on `#[derive(IntoPyObject)]`. [#5112](https://github.com/PyO3/pyo3/pull/5112) +- Add `PyRange` wrapper. [#5117](https://github.com/PyO3/pyo3/pull/5117) + +### Changed + +- Enable use of `datetime` types with `abi3` feature enabled. [#4970](https://github.com/PyO3/pyo3/pull/4970) +- Deprecate `timezone_utc` in favor of `PyTzInfo::utc`. [#5055](https://github.com/PyO3/pyo3/pull/5055) +- Reduce visibility of some CPython implementation details: [#5064](https://github.com/PyO3/pyo3/pull/5064) + - The FFI definition `PyCodeObject` is now an opaque struct on all Python versions. + - The FFI definition `PyFutureFeatures` is now only defined up until Python 3.10 (it was present in CPython headers but unused in 3.11 and 3.12). +- Change `PyAnyMethods::is` to take `other: &Bound`. [#5071](https://github.com/PyO3/pyo3/pull/5071) +- Change `Py::is` to take `other: &Py`. [#5071](https://github.com/PyO3/pyo3/pull/5071) +- Change `PyVisit::call` to take `T: Into>>`. [#5071](https://github.com/PyO3/pyo3/pull/5071) +- Expose `PyDateTime_DATE_GET_TZINFO` and `PyDateTime_TIME_GET_TZINFO` on PyPy 3.10 and later. [#5079](https://github.com/PyO3/pyo3/pull/5079) +- Add `#[track_caller]` to `with_gil` and `with_gil_unchecked`. [#5109](https://github.com/PyO3/pyo3/pull/5109) +- Use `std::thread::park()` instead of `libc::pause()` or `sleep(9999999)`. [#5115](https://github.com/PyO3/pyo3/pull/5115) + +### Removed + +- Remove all functionality deprecated in PyO3 0.23. [#4982](https://github.com/PyO3/pyo3/pull/4982) +- Remove deprecated `IntoPy` and `ToPyObject` traits. [#5010](https://github.com/PyO3/pyo3/pull/5010) +- Remove private types from `pyo3-ffi` (i.e. starting with `_Py`) which are not referenced by public APIs: `_PyLocalMonitors`, `_Py_GlobalMonitors`, `_PyCoCached`, `_PyCoLineInstrumentationData`, `_PyCoMonitoringData`, `_PyCompilerSrcLocation`, `_PyErr_StackItem`. [#5064](https://github.com/PyO3/pyo3/pull/5064) +- Remove FFI definition `PyCode_GetNumFree` (PyO3 cannot support it due to knowledge of the code object). [#5064](https://github.com/PyO3/pyo3/pull/5064) +- Remove `AsPyPointer` trait. [#5071](https://github.com/PyO3/pyo3/pull/5071) +- Remove support for the deprecated string form of `from_py_with`. [#5097](https://github.com/PyO3/pyo3/pull/5097) +- Remove FFI definitions of private static variables: `_PyMethodWrapper_Type`, `_PyCoroWrapper_Type`, `_PyImport_FrozenBootstrap`, `_PyImport_FrozenStdlib`, `_PyImport_FrozenTest`, `_PyManagedBuffer_Type`, `_PySet_Dummy`, `_PyWeakref_ProxyType`, and `_PyWeakref_CallableProxyType`. [#5105](https://github.com/PyO3/pyo3/pull/5105) +- Remove FFI definitions `PyASCIIObjectState`, `PyUnicode_IS_ASCII`, `PyUnicode_IS_COMPACT`, and `PyUnicode_IS_COMPACT_ASCII` on Python 3.14 and newer. [#5133](https://github.com/PyO3/pyo3/pull/5133) + +### Fixed + +- Correctly pick up the shared state for conda-based Python installation when reading information from sysconfigdata. [#5037](https://github.com/PyO3/pyo3/pull/5037) +- Fix compile failure with `#[derive(IntoPyObject, FromPyObject)]` when using `#[pyo3()]` options recognised by only one of the two derives. [#5070](https://github.com/PyO3/pyo3/pull/5070) +- Fix various compile errors from missing FFI definitions using certain feature combinations on PyPy and GraalPy. [#5091](https://github.com/PyO3/pyo3/pull/5091) +- Fallback on `backports.zoneinfo` for python <3.9 when converting timezones into python. [#5120](https://github.com/PyO3/pyo3/pull/5120) + +## [0.24.2] - 2025-04-21 + +### Fixed + +- Fix `unused_imports` lint of `#[pyfunction]` and `#[pymethods]` expanded in `macro_rules` context. [#5030](https://github.com/PyO3/pyo3/pull/5030) +- Fix size of `PyCodeObject::_co_instrumentation_version` ffi struct member on Python 3.13 for systems where `uintptr_t` is not 64 bits. [#5048](https://github.com/PyO3/pyo3/pull/5048) +- Fix struct-type complex enum variant fields incorrectly exposing raw identifiers as `r#ident` in Python bindings. [#5050](https://github.com/PyO3/pyo3/pull/5050) + +## [0.24.1] - 2025-03-31 + +### Added + +- Add `abi3-py313` feature. [#4969](https://github.com/PyO3/pyo3/pull/4969) +- Add `PyAnyMethods::getattr_opt`. [#4978](https://github.com/PyO3/pyo3/pull/4978) +- Add `PyInt::new` constructor for all supported number types (i32, u32, i64, u64, isize, usize). [#4984](https://github.com/PyO3/pyo3/pull/4984) +- Add `pyo3::sync::with_critical_section2`. [#4992](https://github.com/PyO3/pyo3/pull/4992) +- Implement `PyCallArgs` for `Borrowed<'_, 'py, PyTuple>`, `&Bound<'py, PyTuple>`, and `&Py`. [#5013](https://github.com/PyO3/pyo3/pull/5013) + +### Fixed + +- Fix `is_type_of` for native types not using same specialized check as `is_type_of_bound`. [#4981](https://github.com/PyO3/pyo3/pull/4981) +- Fix `Probe` class naming issue with `#[pymethods]`. [#4988](https://github.com/PyO3/pyo3/pull/4988) +- Fix compile failure with required `#[pyfunction]` arguments taking `Option<&str>` and `Option<&T>` (for `#[pyclass]` types). [#5002](https://github.com/PyO3/pyo3/pull/5002) +- Fix `PyString::from_object` causing of bounds reads whith `encoding` and `errors` parameters which are not nul-terminated. [#5008](https://github.com/PyO3/pyo3/pull/5008) +- Fix compile error when additional options follow after `crate` for `#[pyfunction]`. [#5015](https://github.com/PyO3/pyo3/pull/5015) + ## [0.24.0] - 2025-03-09 ### Packaging @@ -17,6 +102,7 @@ To see unreleased changes, please see the [CHANGELOG on the main branch guide](h - Add supported CPython/PyPy versions to cargo package metadata. [#4756](https://github.com/PyO3/pyo3/pull/4756) - Bump `target-lexicon` dependency to 0.13. [#4822](https://github.com/PyO3/pyo3/pull/4822) - Add optional `jiff` dependency to add conversions for `jiff` datetime types. [#4823](https://github.com/PyO3/pyo3/pull/4823) +- Add optional `uuid` dependency to add conversions for `uuid::Uuid`. [#4864](https://github.com/PyO3/pyo3/pull/4864) - Bump minimum supported `inventory` version to 0.3.5. [#4954](https://github.com/PyO3/pyo3/pull/4954) ### Added @@ -25,7 +111,6 @@ To see unreleased changes, please see the [CHANGELOG on the main branch guide](h - Add `PyCallArgs` trait for passing arguments into the Python calling protocol. This enabled using a faster calling convention for certain types, improving performance. [#4768](https://github.com/PyO3/pyo3/pull/4768) - Add `#[pyo3(default = ...']` option for `#[derive(FromPyObject)]` to set a default value for extracted fields of named structs. [#4829](https://github.com/PyO3/pyo3/pull/4829) - Add `#[pyo3(into_py_with = ...)]` option for `#[derive(IntoPyObject, IntoPyObjectRef)]`. [#4850](https://github.com/PyO3/pyo3/pull/4850) -- Add uuid to/from python conversions. [#4864](https://github.com/PyO3/pyo3/pull/4864) - Add FFI definitions `PyThreadState_GetFrame` and `PyFrame_GetBack`. [#4866](https://github.com/PyO3/pyo3/pull/4866) - Optimize `last` for `BoundListIterator`, `BoundTupleIterator` and `BorrowedTupleIterator`. [#4878](https://github.com/PyO3/pyo3/pull/4878) - Optimize `Iterator::count()` for `PyDict`, `PyList`, `PyTuple` & `PySet`. [#4878](https://github.com/PyO3/pyo3/pull/4878) @@ -60,6 +145,7 @@ To see unreleased changes, please see the [CHANGELOG on the main branch guide](h ## [0.23.5] - 2025-02-22 + ### Packaging - Add support for PyPy3.11 [#4760](https://github.com/PyO3/pyo3/pull/4760) @@ -109,7 +195,6 @@ To see unreleased changes, please see the [CHANGELOG on the main branch guide](h - Fix unresolved symbol link failures on Windows when compiling for Python 3.13t using the `generate-import-lib` feature. [#4749](https://github.com/PyO3/pyo3/pull/4749) - Fix compile-time regression in PyO3 0.23.0 where changing `PYO3_CONFIG_FILE` would not reconfigure PyO3 for the new interpreter. [#4758](https://github.com/PyO3/pyo3/pull/4758) - ## [0.23.2] - 2024-11-25 ### Added @@ -2113,7 +2198,10 @@ Yanked - Initial release -[Unreleased]: https://github.com/pyo3/pyo3/compare/v0.24.0...HEAD +[Unreleased]: https://github.com/pyo3/pyo3/compare/v0.25.0...HEAD +[0.25.0]: https://github.com/pyo3/pyo3/compare/v0.24.2...v0.25.0 +[0.24.2]: https://github.com/pyo3/pyo3/compare/v0.24.1...v0.24.2 +[0.24.1]: https://github.com/pyo3/pyo3/compare/v0.24.0...v0.24.1 [0.24.0]: https://github.com/pyo3/pyo3/compare/v0.23.5...v0.24.0 [0.23.5]: https://github.com/pyo3/pyo3/compare/v0.23.4...v0.23.5 [0.23.4]: https://github.com/pyo3/pyo3/compare/v0.23.3...v0.23.4 diff --git a/Cargo.toml b/Cargo.toml index 84c6daba30e..55e4a29170c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3" -version = "0.24.0" +version = "0.25.0" description = "Bindings to Python interpreter" authors = ["PyO3 Project and Contributors "] readme = "README.md" @@ -15,16 +15,15 @@ edition = "2021" rust-version = "1.63" [dependencies] -cfg-if = "1.0" libc = "0.2.62" memoffset = "0.9" once_cell = "1.13" # ffi bindings to the python interpreter, split into a separate crate so they can be used independently -pyo3-ffi = { path = "pyo3-ffi", version = "=0.24.0" } +pyo3-ffi = { path = "pyo3-ffi", version = "=0.25.0" } # support crates for macros feature -pyo3-macros = { path = "pyo3-macros", version = "=0.24.0", optional = true } +pyo3-macros = { path = "pyo3-macros", version = "=0.25.0", optional = true } indoc = { version = "2.0.1", optional = true } unindent = { version = "0.2.1", optional = true } @@ -33,6 +32,7 @@ inventory = { version = "0.3.5", optional = true } # crate integrations that can be added using the eponymous features anyhow = { version = "1.0.1", optional = true } +bigdecimal = {version = "0.4", optional = true } chrono = { version = "0.4.25", default-features = false, optional = true } chrono-tz = { version = ">= 0.10, < 0.11", default-features = false, optional = true } either = { version = "1.9", optional = true } @@ -43,10 +43,14 @@ jiff-02 = { package = "jiff", version = "0.2", optional = true } num-bigint = { version = "0.4.2", optional = true } num-complex = { version = ">= 0.4.6, < 0.5", optional = true } num-rational = { version = "0.4.1", optional = true } +ordered-float = { version = "5.0.0", default-features = false, optional = true } rust_decimal = { version = "1.15", default-features = false, optional = true } +time = { version = "0.3.38", default-features = false, optional = true } serde = { version = "1.0", optional = true } smallvec = { version = "1.0", optional = true } uuid = { version = "1.11.0", optional = true } +lock_api = { version = "0.4", optional = true } +parking_lot = { version = "0.12", optional = true} [target.'cfg(not(target_has_atomic = "64"))'.dependencies] portable-atomic = "1.0" @@ -66,9 +70,10 @@ futures = "0.3.28" tempfile = "3.12.0" static_assertions = "1.1.0" uuid = { version = "1.10.0", features = ["v4"] } +parking_lot = { version = "0.12.3", features = ["arc_lock"]} [build-dependencies] -pyo3-build-config = { path = "pyo3-build-config", version = "=0.24.0", features = ["resolve-config"] } +pyo3-build-config = { path = "pyo3-build-config", version = "=0.25.0", features = ["resolve-config"] } [features] default = ["macros"] @@ -78,7 +83,7 @@ experimental-async = ["macros", "pyo3-macros/experimental-async"] # Enables pyo3::inspect module and additional type information on FromPyObject # and IntoPy traits -experimental-inspect = [] +experimental-inspect = ["pyo3-macros/experimental-inspect"] # Enables macros: #[pyclass], #[pymodule], #[pyfunction] etc. macros = ["pyo3-macros", "indoc", "unindent"] @@ -101,7 +106,8 @@ abi3-py39 = ["abi3-py310", "pyo3-build-config/abi3-py39", "pyo3-ffi/abi3-py39"] abi3-py310 = ["abi3-py311", "pyo3-build-config/abi3-py310", "pyo3-ffi/abi3-py310"] abi3-py311 = ["abi3-py312", "pyo3-build-config/abi3-py311", "pyo3-ffi/abi3-py311"] abi3-py312 = ["abi3-py313", "pyo3-build-config/abi3-py312", "pyo3-ffi/abi3-py312"] -abi3-py313 = ["abi3", "pyo3-build-config/abi3-py313", "pyo3-ffi/abi3-py313"] +abi3-py313 = ["abi3-py314", "pyo3-build-config/abi3-py313", "pyo3-ffi/abi3-py313"] +abi3-py314 = ["abi3", "pyo3-build-config/abi3-py314", "pyo3-ffi/abi3-py314"] # Automatically generates `python3.dll` import libraries for Windows targets. generate-import-lib = ["pyo3-ffi/generate-import-lib"] @@ -112,6 +118,9 @@ auto-initialize = [] # Enables `Clone`ing references to Python objects `Py` which panics if the GIL is not held. py-clone = [] +parking_lot = ["dep:parking_lot", "lock_api"] +arc_lock = ["lock_api", "lock_api/arc_lock", "parking_lot?/arc_lock"] + # Optimizes PyObject to Vec conversion and so on. nightly = [] @@ -121,6 +130,8 @@ full = [ "macros", # "multiple-pymethods", # Not supported by wasm "anyhow", + "arc_lock", + "bigdecimal", "chrono", "chrono-tz", "either", @@ -129,9 +140,12 @@ full = [ "eyre", "hashbrown", "indexmap", + "lock_api", "num-bigint", "num-complex", "num-rational", + "ordered-float", + "parking_lot", "py-clone", "rust_decimal", "serde", @@ -145,6 +159,7 @@ members = [ "pyo3-build-config", "pyo3-macros", "pyo3-macros-backend", + "pyo3-introspection", "pytests", "examples", ] diff --git a/Contributing.md b/Contributing.md index 054ef42fb88..40fcb1af030 100644 --- a/Contributing.md +++ b/Contributing.md @@ -179,6 +179,20 @@ Below are guidelines on what compatibility all PRs are expected to deliver for e PyO3 supports all officially supported Python versions, as well as the latest PyPy3 release. All of these versions are tested in CI. +#### Adding support for new CPython versions + +If you plan to add support for a pre-release version of CPython, here's a (non-exhaustive) checklist: + + - [ ] Wait until the last alpha release (usually alpha7), since ABI is not guranteed until the first beta release + - [ ] Add prelease_ver-dev (e.g. `3.14-dev`) to `‎.github/workflows/ci.yml`, and bump version in `noxfile.py`, `pyo3-ffi/Cargo.toml` under `max-version` within `[package.metadata.cpython]`, and `max` within `pyo3-ffi/build.rs` +- [ ] Add a new abi3-prerelease feature for the version (e.g. `abi3-py314`) + - In `pyo3-build-config/Cargo.toml`, set abi3-most_current_stable to ["abi3-prerelease"] and abi3-prerelease to ["abi3"] + - In `pyo3-ffi/Cargo.toml`, set abi3-most_current_stable to ["abi3-prerelease", "pyo3-build-config/abi3-most_current_stable"] and abi3-prerelease to ["abi3", "pyo3-build-config/abi3-prerelease"] + - In `Cargo.toml`, set abi3-most_current_stable to ["abi3-prerelease", "pyo3-ffi/abi3-most_current_stable"] and abi3-prerelease to ["abi3", "pyo3-ffi/abi3-prerelease"] + - [ ] Use `#[cfg(Py_prerelease])` (e.g. `#[cfg(Py_3_14)]`) and `#[cfg(not(Py_prerelease]))` to indicate changes between the stable branches of CPython and the pre-release + - [ ] Do not add a Rust binding to any function, struct, or global variable prefixed with `_` in CPython's headers + - [ ] Ping @ngoldbaum and @davidhewitt for assistance + ### Rust PyO3 aims to make use of up-to-date Rust language features to keep the implementation as efficient as possible. diff --git a/README.md b/README.md index 55558594495..47a74ec84a3 100644 --- a/README.md +++ b/README.md @@ -21,7 +21,7 @@ Requires Rust 1.63 or greater. PyO3 supports the following Python distributions: - CPython 3.7 or greater - PyPy 7.3 (Python 3.9+) - - GraalPy 24.0 or greater (Python 3.10+) + - GraalPy 24.2 or greater (Python 3.11+) You can use PyO3 to write a native Python module in Rust, or to embed Python in a Rust binary. The following sections explain each of these in turn. @@ -71,7 +71,7 @@ name = "string_sum" crate-type = ["cdylib"] [dependencies] -pyo3 = { version = "0.24.0", features = ["extension-module"] } +pyo3 = { version = "0.25.0", features = ["extension-module"] } ``` **`src/lib.rs`** @@ -140,7 +140,7 @@ Start a new project with `cargo new` and add `pyo3` to the `Cargo.toml` like th ```toml [dependencies.pyo3] -version = "0.24.0" +version = "0.25.0" features = ["auto-initialize"] ``` @@ -187,6 +187,8 @@ about this topic. - [bed-reader](https://github.com/fastlmm/bed-reader) _Read and write the PLINK BED format, simply and efficiently._ - Shows Rayon/ndarray::parallel (including capturing errors, controlling thread num), Python types to Rust generics, Github Actions +- [blake3-py](https://github.com/oconnor663/blake3-py) _Python bindings for the [BLAKE3](https://github.com/BLAKE3-team/BLAKE3) cryptographic hash function._ + - Parallelized [builds](https://github.com/oconnor663/blake3-py/blob/master/.github/workflows/dists.yml) on GitHub Actions for MacOS, Linux, Windows, including free-threaded 3.13t wheels. - [cellular_raza](https://cellular-raza.com) _A cellular agent-based simulation framework for building complex models from a clean slate._ - [connector-x](https://github.com/sfu-db/connector-x/tree/main/connectorx-python) _Fastest library to load data from DB to DataFrames in Rust and Python._ - [cryptography](https://github.com/pyca/cryptography/tree/main/src/rust) _Python cryptography library with some functionality in Rust._ @@ -209,6 +211,7 @@ about this topic. - [orjson](https://github.com/ijl/orjson) _Fast Python JSON library._ - [ormsgpack](https://github.com/aviramha/ormsgpack) _Fast Python msgpack library._ - [polars](https://github.com/pola-rs/polars) _Fast multi-threaded DataFrame library in Rust | Python | Node.js._ +- [pycrdt](https://github.com/jupyter-server/pycrdt) _Python bindings for the Rust CRDT implementation [Yrs](https://github.com/y-crdt/y-crdt)._ - [pydantic-core](https://github.com/pydantic/pydantic-core) _Core validation logic for pydantic written in Rust._ - [primp](https://github.com/deedy5/primp) _The fastest python HTTP client that can impersonate web browsers by mimicking their headers and TLS/JA3/JA4/HTTP2 fingerprints._ - [rateslib](https://github.com/attack68/rateslib) _A fixed income library for Python using Rust extensions._ diff --git a/Releasing.md b/Releasing.md index 545783c598c..d3a1b4cf8c4 100644 --- a/Releasing.md +++ b/Releasing.md @@ -44,9 +44,10 @@ Wait a couple of days in case anyone wants to hold up the release to add bugfixe ## 4. Put live To put live: -- 1. run `nox -s publish` to put live on crates.io -- 2. publish the release on Github -- 3. merge the release PR +- 1. merge the release PR +- 2. publish a release on GitHub targeting the release branch + +CI will automatically push to `crates.io`. ## 5. Tidy the main branch diff --git a/build.rs b/build.rs index 68a658bf285..fd28b03b79d 100644 --- a/build.rs +++ b/build.rs @@ -38,7 +38,7 @@ fn configure_pyo3() -> Result<()> { ensure_auto_initialize_ok(interpreter_config)?; for cfg in interpreter_config.build_script_outputs() { - println!("{}", cfg) + println!("{cfg}") } // Emit cfgs like `invalid_from_utf8_lint` diff --git a/examples/decorator/.template/pre-script.rhai b/examples/decorator/.template/pre-script.rhai index 1ad80e9afbe..80ebe3ce006 100644 --- a/examples/decorator/.template/pre-script.rhai +++ b/examples/decorator/.template/pre-script.rhai @@ -1,4 +1,4 @@ -variable::set("PYO3_VERSION", "0.24.0"); +variable::set("PYO3_VERSION", "0.25.0"); file::rename(".template/Cargo.toml", "Cargo.toml"); file::rename(".template/pyproject.toml", "pyproject.toml"); file::delete(".template"); diff --git a/examples/decorator/src/lib.rs b/examples/decorator/src/lib.rs index 4c5471c9945..69915ff8256 100644 --- a/examples/decorator/src/lib.rs +++ b/examples/decorator/src/lib.rs @@ -46,7 +46,7 @@ impl PyCounter { let new_count = self.count.fetch_add(1, Ordering::Relaxed); let name = self.wraps.getattr(py, "__name__")?; - println!("{} has been called {} time(s).", name, new_count); + println!("{name} has been called {new_count} time(s)."); // After doing something, we finally forward the call to the wrapped function let ret = self.wraps.call(py, args, kwargs)?; diff --git a/examples/maturin-starter/.template/pre-script.rhai b/examples/maturin-starter/.template/pre-script.rhai index 1ad80e9afbe..80ebe3ce006 100644 --- a/examples/maturin-starter/.template/pre-script.rhai +++ b/examples/maturin-starter/.template/pre-script.rhai @@ -1,4 +1,4 @@ -variable::set("PYO3_VERSION", "0.24.0"); +variable::set("PYO3_VERSION", "0.25.0"); file::rename(".template/Cargo.toml", "Cargo.toml"); file::rename(".template/pyproject.toml", "pyproject.toml"); file::delete(".template"); diff --git a/examples/plugin/.template/pre-script.rhai b/examples/plugin/.template/pre-script.rhai index ffd73d3a0fa..2a21f6b2e53 100644 --- a/examples/plugin/.template/pre-script.rhai +++ b/examples/plugin/.template/pre-script.rhai @@ -1,4 +1,4 @@ -variable::set("PYO3_VERSION", "0.24.0"); +variable::set("PYO3_VERSION", "0.25.0"); file::rename(".template/Cargo.toml", "Cargo.toml"); file::rename(".template/plugin_api/Cargo.toml", "plugin_api/Cargo.toml"); file::delete(".template"); diff --git a/examples/setuptools-rust-starter/.template/pre-script.rhai b/examples/setuptools-rust-starter/.template/pre-script.rhai index fd6e6775627..92eeabb3ea8 100644 --- a/examples/setuptools-rust-starter/.template/pre-script.rhai +++ b/examples/setuptools-rust-starter/.template/pre-script.rhai @@ -1,4 +1,4 @@ -variable::set("PYO3_VERSION", "0.24.0"); +variable::set("PYO3_VERSION", "0.25.0"); file::rename(".template/Cargo.toml", "Cargo.toml"); file::rename(".template/setup.cfg", "setup.cfg"); file::delete(".template"); diff --git a/examples/setuptools-rust-starter/noxfile.py b/examples/setuptools-rust-starter/noxfile.py index 9edab96254b..d3579912ea7 100644 --- a/examples/setuptools-rust-starter/noxfile.py +++ b/examples/setuptools-rust-starter/noxfile.py @@ -1,10 +1,11 @@ import nox +import sys @nox.session def python(session: nox.Session): - session.install("-rrequirements-dev.txt") - session.run_always( - "pip", "install", "-e", ".", "--no-build-isolation", env={"BUILD_DEBUG": "1"} - ) + if sys.version_info < (3, 9): + session.skip("Python 3.9 or later is required for setuptools-rust 1.11") + session.env["SETUPTOOLS_RUST_CARGO_PROFILE"] = "dev" + session.install(".[dev]") session.run("pytest") diff --git a/examples/setuptools-rust-starter/pyproject.toml b/examples/setuptools-rust-starter/pyproject.toml index d82653c1701..bc4c7d2bc98 100644 --- a/examples/setuptools-rust-starter/pyproject.toml +++ b/examples/setuptools-rust-starter/pyproject.toml @@ -1,2 +1,25 @@ [build-system] -requires = ["setuptools>=41.0.0", "wheel", "setuptools_rust>=1.0.0"] +requires = ["setuptools>=62.4", "setuptools_rust>=1.11"] + +[project] +name = "setuptools-rust-starter" +version = "0.1.0" +classifiers = [ + "License :: OSI Approved :: MIT License", + "Development Status :: 3 - Alpha", + "Intended Audience :: Developers", + "Programming Language :: Python", + "Programming Language :: Rust", + "Operating System :: POSIX", + "Operating System :: MacOS :: MacOS X", +] + +[project.optional-dependencies] +dev = ["pytest"] + +[tool.setuptools.packages.find] +include = ["setuptools_rust_starter"] + +[[tool.setuptools-rust.ext-modules]] +target = "setuptools_rust_starter._setuptools_rust_starter" +path = "Cargo.toml" diff --git a/examples/setuptools-rust-starter/setup.cfg b/examples/setuptools-rust-starter/setup.cfg deleted file mode 100644 index 95caeb757e6..00000000000 --- a/examples/setuptools-rust-starter/setup.cfg +++ /dev/null @@ -1,17 +0,0 @@ -[metadata] -name = setuptools-rust-starter -version = 0.1.0 -classifiers = - License :: OSI Approved :: MIT License - Development Status :: 3 - Alpha - Intended Audience :: Developers - Programming Language :: Python - Programming Language :: Rust - Operating System :: POSIX - Operating System :: MacOS :: MacOS X - -[options] -packages = - setuptools_rust_starter -include_package_data = True -zip_safe = False diff --git a/examples/setuptools-rust-starter/setup.py b/examples/setuptools-rust-starter/setup.py deleted file mode 100644 index 85d9e21d3a4..00000000000 --- a/examples/setuptools-rust-starter/setup.py +++ /dev/null @@ -1,13 +0,0 @@ -import os - -from setuptools import setup -from setuptools_rust import RustExtension - -setup( - rust_extensions=[ - RustExtension( - "setuptools_rust_starter._setuptools_rust_starter", - debug=os.environ.get("BUILD_DEBUG") == "1", - ) - ], -) diff --git a/examples/word-count/.template/pre-script.rhai b/examples/word-count/.template/pre-script.rhai index 1ad80e9afbe..80ebe3ce006 100644 --- a/examples/word-count/.template/pre-script.rhai +++ b/examples/word-count/.template/pre-script.rhai @@ -1,4 +1,4 @@ -variable::set("PYO3_VERSION", "0.24.0"); +variable::set("PYO3_VERSION", "0.25.0"); file::rename(".template/Cargo.toml", "Cargo.toml"); file::rename(".template/pyproject.toml", "pyproject.toml"); file::delete(".template"); diff --git a/examples/word-count/noxfile.py b/examples/word-count/noxfile.py index d64f210f3e5..c9dd6fce159 100644 --- a/examples/word-count/noxfile.py +++ b/examples/word-count/noxfile.py @@ -12,6 +12,5 @@ def test(session: nox.Session): @nox.session def bench(session: nox.Session): - session.env["MATURIN_PEP517_ARGS"] = "--profile=dev" session.install(".[dev]") session.run("pytest", "--benchmark-enable") diff --git a/guide/pyclass-parameters.md b/guide/pyclass-parameters.md index 7ebca2ec821..27b15118dd4 100644 --- a/guide/pyclass-parameters.md +++ b/guide/pyclass-parameters.md @@ -10,8 +10,10 @@ | `extends = BaseType` | Use a custom baseclass. Defaults to [`PyAny`][params-1] | | `freelist = N` | Implements a [free list][params-2] of size N. This can improve performance for types that are often created and deleted in quick succession. Profile your code to see whether `freelist` is right for you. | | `frozen` | Declares that your pyclass is immutable. It removes the borrow checker overhead when retrieving a shared reference to the Rust struct, but disables the ability to get a mutable reference. | +| `generic` | Implements runtime parametrization for the class following [PEP 560](https://peps.python.org/pep-0560/). | | `get_all` | Generates getters for all fields of the pyclass. | | `hash` | Implements `__hash__` using the `Hash` implementation of the underlying Rust datatype. | +| `immutable_type` | Makes the type object immutable. Supported on 3.14+ with the `abi3` feature active, or 3.10+ otherwise. | | `mapping` | Inform PyO3 that this class is a [`Mapping`][params-mapping], and so leave its implementation of sequence C-API slots empty. | | `module = "module_name"` | Python code will see the class as being defined in this module. Defaults to `builtins`. | | `name = "python_name"` | Sets the name that Python sees this class as. Defaults to the name of the Rust struct. | diff --git a/guide/src/async-await.md b/guide/src/async-await.md index 27574181804..1e0772b8a09 100644 --- a/guide/src/async-await.md +++ b/guide/src/async-await.md @@ -4,7 +4,7 @@ `#[pyfunction]` and `#[pymethods]` attributes also support `async fn`. -```rust +```rust,no_run # #![allow(dead_code)] # #[cfg(feature = "experimental-async")] { use std::{thread, time::Duration}; @@ -66,8 +66,8 @@ where fn poll(mut self: Pin<&mut Self>, cx: &mut Context<'_>) -> Poll { let waker = cx.waker(); - Python::with_gil(|gil| { - gil.allow_threads(|| pin!(&mut self.0).poll(&mut Context::from_waker(waker))) + Python::with_gil(|py| { + py.allow_threads(|| pin!(&mut self.0).poll(&mut Context::from_waker(waker))) }) } } @@ -77,7 +77,7 @@ where Cancellation on the Python side can be caught using [`CancelHandle`]({{#PYO3_DOCS_URL}}/pyo3/coroutine/struct.CancelHandle.html) type, by annotating a function parameter with `#[pyo3(cancel_handle)]`. -```rust +```rust,no_run # #![allow(dead_code)] # #[cfg(feature = "experimental-async")] { use futures::FutureExt; diff --git a/guide/src/building-and-distribution.md b/guide/src/building-and-distribution.md index d3474fedaf7..b9ea56916ba 100644 --- a/guide/src/building-and-distribution.md +++ b/guide/src/building-and-distribution.md @@ -86,7 +86,7 @@ Once built, symlink (or copy) and rename the shared library from Cargo's `target You can then open a Python shell in the output directory and you'll be able to run `import your_module`. -If you're packaging your library for redistribution, you should indicated the Python interpreter your library is compiled for by including the [platform tag](#platform-tags) in its name. This prevents incompatible interpreters from trying to import your library. If you're compiling for PyPy you *must* include the platform tag, or PyPy will ignore the module. +If you're packaging your library for redistribution, you should indicate the Python interpreter your library is compiled for by including the [platform tag](#platform-tags) in its name. This prevents incompatible interpreters from trying to import your library. If you're compiling for PyPy you *must* include the platform tag, or PyPy will ignore the module. #### Bazel builds diff --git a/guide/src/class.md b/guide/src/class.md index 90991328fe6..8772e857d7e 100644 --- a/guide/src/class.md +++ b/guide/src/class.md @@ -788,7 +788,7 @@ impl MyClass { To create a class attribute (also called [class variable][classattr]), a method without any arguments can be annotated with the `#[classattr]` attribute. -```rust +```rust,no_run # use pyo3::prelude::*; # #[pyclass] # struct MyClass {} @@ -812,7 +812,7 @@ class creation. If the class attribute is defined with `const` code only, one can also annotate associated constants: -```rust +```rust,no_run # use pyo3::prelude::*; # #[pyclass] # struct MyClass {} @@ -827,7 +827,7 @@ impl MyClass { Free functions defined using `#[pyfunction]` interact with classes through the same mechanisms as the self parameters of instance methods, i.e. they can take GIL-bound references, GIL-bound reference wrappers or GIL-indepedent references: -```rust +```rust,no_run # #![allow(dead_code)] # use pyo3::prelude::*; #[pyclass] @@ -866,7 +866,7 @@ fn print_refcnt(my_class: Py, py: Python<'_>) { Classes can also be passed by value if they can be cloned, i.e. they automatically implement `FromPyObject` if they implement `Clone`, e.g. via `#[derive(Clone)]`: -```rust +```rust,no_run # #![allow(dead_code)] # use pyo3::prelude::*; #[pyclass] @@ -890,7 +890,7 @@ Similar to `#[pyfunction]`, the `#[pyo3(signature = (...))]` attribute can be us The following example defines a class `MyClass` with a method `method`. This method has a signature that sets default values for `num` and `name`, and indicates that `py_args` should collect all extra positional arguments and `py_kwargs` all extra keyword arguments: -```rust +```rust,no_run # use pyo3::prelude::*; use pyo3::types::{PyDict, PyTuple}; # @@ -1410,13 +1410,6 @@ impl<'a, 'py> pyo3::impl_::extract_argument::PyFunctionArgument<'a, 'py, false> } } -#[allow(deprecated)] -impl pyo3::IntoPy for MyClass { - fn into_py(self, py: pyo3::Python<'_>) -> pyo3::PyObject { - pyo3::IntoPy::into_py(pyo3::Py::new(py, self).unwrap(), py) - } -} - impl pyo3::impl_::pyclass::PyClassImpl for MyClass { const IS_BASETYPE: bool = false; const IS_SUBCLASS: bool = false; diff --git a/guide/src/class/numeric.md b/guide/src/class/numeric.md index 4f73a44adab..124bb8d27a6 100644 --- a/guide/src/class/numeric.md +++ b/guide/src/class/numeric.md @@ -31,7 +31,7 @@ own extraction function, using the `#[pyo3(from_py_with = ...)]` attribute. Unfo doesn't provide a way to wrap Python integers out of the box, but we can do a Python call to mask it and cast it to an `i32`. -```rust +```rust,no_run # #![allow(dead_code)] use pyo3::prelude::*; @@ -44,7 +44,7 @@ fn wrap(obj: &Bound<'_, PyAny>) -> PyResult { ``` We also add documentation, via `///` comments, which are visible to Python users. -```rust +```rust,no_run # #![allow(dead_code)] use pyo3::prelude::*; @@ -70,7 +70,7 @@ impl Number { With that out of the way, let's implement some operators: -```rust +```rust,no_run use pyo3::exceptions::{PyZeroDivisionError, PyValueError}; # use pyo3::prelude::*; @@ -124,7 +124,7 @@ impl Number { ### Unary arithmetic operations -```rust +```rust,no_run # use pyo3::prelude::*; # # #[pyclass] @@ -152,7 +152,7 @@ impl Number { ### Support for the `complex()`, `int()` and `float()` built-in functions. -```rust +```rust,no_run # use pyo3::prelude::*; # # #[pyclass] @@ -415,7 +415,7 @@ Let's create that helper function. The signature has to be `fn(&Bound<'_, PyAny> - `&Bound<'_, PyAny>` represents a checked borrowed reference, so the pointer derived from it is valid (and not null). - Whenever we have borrowed references to Python objects in scope, it is guaranteed that the GIL is held. This reference is also where we can get a [`Python`] token to use in our call to [`PyErr::take`]. -```rust +```rust,no_run # #![allow(dead_code)] use std::os::raw::c_ulong; use pyo3::prelude::*; diff --git a/guide/src/class/object.md b/guide/src/class/object.md index 07f445aac60..3b9f1aafa40 100644 --- a/guide/src/class/object.md +++ b/guide/src/class/object.md @@ -2,7 +2,7 @@ Recall the `Number` class from the previous chapter: -```rust +```rust,no_run # #![allow(dead_code)] use pyo3::prelude::*; @@ -44,7 +44,7 @@ It can't even print an user-readable representation of itself! We can fix that b `__repr__` and `__str__` methods inside a `#[pymethods]` block. We do this by accessing the value contained inside `Number`. -```rust +```rust,no_run # use pyo3::prelude::*; # # #[pyclass] @@ -72,7 +72,7 @@ impl Number { To automatically generate the `__str__` implementation using a `Display` trait implementation, pass the `str` argument to `pyclass`. -```rust +```rust,no_run # use std::fmt::{Display, Formatter}; # use pyo3::prelude::*; # @@ -91,16 +91,16 @@ impl Display for Coordinate { } ``` -For convenience, a shorthand format string can be passed to `str` as `str=""` for **structs only**. It expands and is passed into the `format!` macro in the following ways: +For convenience, a shorthand format string can be passed to `str` as `str=""` for **structs only**. It expands and is passed into the `format!` macro in the following ways: * `"{x}"` -> `"{}", self.x` * `"{0}"` -> `"{}", self.0` * `"{x:?}"` -> `"{:?}", self.x` -*Note: Depending upon the format string you use, this may require implementation of the `Display` or `Debug` traits for the given Rust types.* +*Note: Depending upon the format string you use, this may require implementation of the `Display` or `Debug` traits for the given Rust types.* *Note: the pyclass args `name` and `rename_all` are incompatible with the shorthand format string and will raise a compile time error.* -```rust +```rust,no_run # use pyo3::prelude::*; # # #[allow(dead_code)] @@ -120,7 +120,7 @@ the subclass name. This is typically done in Python code by accessing `self.__class__.__name__`. In order to be able to access the Python type information *and* the Rust struct, we need to use a `Bound` as the `self` argument. -```rust +```rust,no_run # use pyo3::prelude::*; # use pyo3::types::PyString; # @@ -145,7 +145,7 @@ impl Number { Let's also implement hashing. We'll just hash the `i32`. For that we need a [`Hasher`]. The one provided by `std` is [`DefaultHasher`], which uses the [SipHash] algorithm. -```rust +```rust,no_run use std::collections::hash_map::DefaultHasher; // Required to call the `.hash` and `.finish` methods, which are defined on traits. @@ -171,7 +171,7 @@ This option is only available for `frozen` classes to prevent accidental hash ch an `__hash__` implementation for a mutable class, use the manual method from above. This option also requires `eq`: According to the [Python docs](https://docs.python.org/3/reference/datamodel.html#object.__hash__) "If a class does not define an `__eq__()` method it should not define a `__hash__()` operation either" -```rust +```rust,no_run # use pyo3::prelude::*; # # #[allow(dead_code)] @@ -195,7 +195,7 @@ struct Number(i32); > Types which should not be hashable can override this by setting `__hash__` to None. > This is the same mechanism as for a pure-Python class. This is done like so: > -> ```rust +> ```rust,no_run > # use pyo3::prelude::*; > #[pyclass] > struct NotHashable {} @@ -213,7 +213,7 @@ PyO3 supports the usual magic comparison methods available in Python such as `__ and so on. It is also possible to support all six operations at once with `__richcmp__`. This method will be called with a value of `CompareOp` depending on the operation. -```rust +```rust,no_run use pyo3::class::basic::CompareOp; # use pyo3::prelude::*; @@ -240,7 +240,7 @@ impl Number { If you obtain the result by comparing two Rust values, as in this example, you can take a shortcut using `CompareOp::matches`: -```rust +```rust,no_run use pyo3::class::basic::CompareOp; # use pyo3::prelude::*; @@ -289,7 +289,7 @@ impl Number { To implement `__eq__` using the Rust [`PartialEq`] trait implementation, the `eq` option can be used. -```rust +```rust,no_run # use pyo3::prelude::*; # # #[allow(dead_code)] @@ -300,7 +300,7 @@ struct Number(i32); To implement `__lt__`, `__le__`, `__gt__`, & `__ge__` using the Rust `PartialOrd` trait implementation, the `ord` option can be used. *Note: Requires `eq`.* -```rust +```rust,no_run # use pyo3::prelude::*; # # #[allow(dead_code)] @@ -313,7 +313,7 @@ struct Number(i32); We'll consider `Number` to be `True` if it is nonzero: -```rust +```rust,no_run # use pyo3::prelude::*; # # #[allow(dead_code)] @@ -330,7 +330,7 @@ impl Number { ### Final code -```rust +```rust,no_run use std::collections::hash_map::DefaultHasher; use std::hash::{Hash, Hasher}; diff --git a/guide/src/class/protocols.md b/guide/src/class/protocols.md index 8a361a1442e..44aa9199e36 100644 --- a/guide/src/class/protocols.md +++ b/guide/src/class/protocols.md @@ -50,7 +50,7 @@ given signatures should be interpreted as follows: Disabling Python's default hash By default, all `#[pyclass]` types have a default hash implementation from Python. Types which should not be hashable can override this by setting `__hash__` to `None`. This is the same mechanism as for a pure-Python class. This is done like so: - ```rust + ```rust,no_run # use pyo3::prelude::*; # #[pyclass] @@ -95,7 +95,7 @@ given signatures should be interpreted as follows: If you want to leave some operations unimplemented, you can return `py.NotImplemented()` for some of the operations: - ```rust + ```rust,no_run use pyo3::class::basic::CompareOp; use pyo3::types::PyNotImplemented; @@ -155,7 +155,7 @@ Returning `None` from `__next__` indicates that that there are no further items. Example: -```rust +```rust,no_run use pyo3::prelude::*; use std::sync::Mutex; @@ -181,7 +181,7 @@ In many cases you'll have a distinction between the type being iterated over only needs to implement `__iter__()` while the iterator must implement both `__iter__()` and `__next__()`. For example: -```rust +```rust,no_run # use pyo3::prelude::*; #[pyclass] @@ -274,7 +274,7 @@ Use the `#[pyclass(sequence)]` annotation to instruct PyO3 to fill the `sq_lengt can override this by setting `__contains__` to `None`. This is the same mechanism as for a pure-Python class. This is done like so: - ```rust + ```rust,no_run # use pyo3::prelude::*; # #[pyclass] @@ -430,7 +430,7 @@ cleared, as every cycle must contain at least one mutable reference. Example: -```rust +```rust,no_run use pyo3::prelude::*; use pyo3::PyTraverseError; use pyo3::gc::PyVisit; @@ -443,9 +443,7 @@ struct ClassWithGCSupport { #[pymethods] impl ClassWithGCSupport { fn __traverse__(&self, visit: PyVisit<'_>) -> Result<(), PyTraverseError> { - if let Some(obj) = &self.obj { - visit.call(obj)? - } + visit.call(&self.obj)?; Ok(()) } diff --git a/guide/src/class/thread-safety.md b/guide/src/class/thread-safety.md index 55c2a3caca8..d0cf9e10a7f 100644 --- a/guide/src/class/thread-safety.md +++ b/guide/src/class/thread-safety.md @@ -16,7 +16,7 @@ By default, `#[pyclass]` employs an ["interior mutability" pattern](../class.md# For example, the below simple class is thread-safe: -```rust +```rust,no_run # use pyo3::prelude::*; #[pyclass] @@ -47,7 +47,7 @@ To remove the possibility of having overlapping `&self` and `&mut self` referenc For example, a thread-safe version of the above `MyClass` using atomic integers would be as follows: -```rust +```rust,no_run # use pyo3::prelude::*; use std::sync::atomic::{AtomicI32, Ordering}; @@ -75,7 +75,7 @@ An alternative to atomic data structures is to use [locks](https://doc.rust-lang For example, a thread-safe version of the above `MyClass` using locks would be as follows: -```rust +```rust,no_run # use pyo3::prelude::*; use std::sync::Mutex; @@ -101,10 +101,10 @@ impl MyClass { } ``` -If you need to lock around state stored in the Python interpreter or otherwise call into the Python C API while a lock is held, you might find the `MutexExt` trait useful. It provides a `lock_py_attached` method for `std::sync::Mutex` that avoids deadlocks with the GIL or other global synchronization events in the interpreter. +If you need to lock around state stored in the Python interpreter or otherwise call into the Python C API while a lock is held, you might find the `MutexExt` trait useful. It provides a `lock_py_attached` method for `std::sync::Mutex` that avoids deadlocks with the GIL or other global synchronization events in the interpreter. Additionally, support for the `parking_lot` and `lock_api` synchronization libraries is gated behind the `parking_lot` and `lock_api` features. You can also enable the `arc_lock` feature if you need the `arc_lock` features of either library. ### Wrapping unsynchronized data In some cases, the data structures stored within a `#[pyclass]` may themselves not be thread-safe. Rust will therefore not implement `Send` and `Sync` on the `#[pyclass]` type. -To achieve thread-safety, a manual `Send` and `Sync` implementation is required which is `unsafe` and should only be done following careful review of the soundness of the implementation. Doing this for PyO3 types is no different than for any other Rust code, [the Rustonomicon](https://doc.rust-lang.org/nomicon/send-and-sync.html) has a great discussion on this. \ No newline at end of file +To achieve thread-safety, a manual `Send` and `Sync` implementation is required which is `unsafe` and should only be done following careful review of the soundness of the implementation. Doing this for PyO3 types is no different than for any other Rust code, [the Rustonomicon](https://doc.rust-lang.org/nomicon/send-and-sync.html) has a great discussion on this. diff --git a/guide/src/conversions/tables.md b/guide/src/conversions/tables.md index 531e48b8f73..23d6942b177 100644 --- a/guide/src/conversions/tables.md +++ b/guide/src/conversions/tables.md @@ -17,7 +17,7 @@ The table below contains the Python type and the corresponding function argument | `bytes` | `Vec`, `&[u8]`, `Cow<[u8]>` | `PyBytes` | | `bool` | `bool` | `PyBool` | | `int` | `i8`, `u8`, `i16`, `u16`, `i32`, `u32`, `i64`, `u64`, `i128`, `u128`, `isize`, `usize`, `num_bigint::BigInt`[^1], `num_bigint::BigUint`[^1] | `PyInt` | -| `float` | `f32`, `f64` | `PyFloat` | +| `float` | `f32`, `f64`, `ordered_float::NotNan`[^10], `ordered_float::OrderedFloat`[^10] | `PyFloat` | | `complex` | `num_complex::Complex`[^2] | `PyComplex` | | `fractions.Fraction`| `num_rational::Ratio`[^8] | - | | `list[T]` | `Vec` | `PyList` | @@ -36,8 +36,9 @@ The table below contains the Python type and the corresponding function argument | `datetime.tzinfo` | `chrono::FixedOffset`[^5], `chrono::Utc`[^5], `chrono_tz::TimeZone`[^6] | `PyTzInfo` | | `datetime.timedelta` | `Duration`, `chrono::Duration`[^5] | `PyDelta` | | `decimal.Decimal` | `rust_decimal::Decimal`[^7] | - | -| `ipaddress.IPv4Address` | `std::net::IpAddr`, `std::net::IpV4Addr` | - | -| `ipaddress.IPv6Address` | `std::net::IpAddr`, `std::net::IpV6Addr` | - | +| `decimal.Decimal` | `bigdecimal::BigDecimal`[^9] | - | +| `ipaddress.IPv4Address` | `std::net::IpAddr`, `std::net::Ipv4Addr` | - | +| `ipaddress.IPv6Address` | `std::net::IpAddr`, `std::net::Ipv6Addr` | - | | `os.PathLike ` | `PathBuf`, `Path` | `PyString` | | `pathlib.Path` | `PathBuf`, `Path` | `PyString` | | `typing.Optional[T]` | `Option` | - | @@ -116,3 +117,7 @@ Finally, the following Rust types are also able to convert to Python as return v [^7]: Requires the `rust_decimal` optional feature. [^8]: Requires the `num-rational` optional feature. + +[^9]: Requires the `bigdecimal` optional feature. + +[^10]: Requires the `ordered-float` optional feature. diff --git a/guide/src/conversions/traits.md b/guide/src/conversions/traits.md old mode 100755 new mode 100644 index 848dc041ef7..b1b6cd31127 --- a/guide/src/conversions/traits.md +++ b/guide/src/conversions/traits.md @@ -550,7 +550,7 @@ _without_ having a unique python type. `struct`s will turn into a `PyDict` using the field names as keys, tuple `struct`s will turn convert into `PyTuple` with the fields in declaration order. -```rust +```rust,no_run # #![allow(dead_code)] # use pyo3::prelude::*; # use std::collections::HashMap; @@ -574,7 +574,7 @@ For structs with a single field (newtype pattern) the `#[pyo3(transparent)]` opt forward the implementation to the inner type. -```rust +```rust,no_run # #![allow(dead_code)] # use pyo3::prelude::*; @@ -591,7 +591,7 @@ struct TransparentStruct<'py> { For `enum`s each variant is converted according to the rules for `struct`s above. -```rust +```rust,no_run # #![allow(dead_code)] # use pyo3::prelude::*; # use std::collections::HashMap; @@ -618,7 +618,7 @@ Additionally `IntoPyObject` can be derived for a reference to a struct or enum u - `#[derive(IntoPyObject)]` will invoke the function with `Cow::Owned` - `#[derive(IntoPyObjectRef)]` will invoke the function with `Cow::Borrowed` - ```rust + ```rust,no_run # use pyo3::prelude::*; # use pyo3::IntoPyObjectExt; # use std::borrow::Cow; @@ -642,7 +642,7 @@ Additionally `IntoPyObject` can be derived for a reference to a struct or enum u If the derive macro is not suitable for your use case, `IntoPyObject` can be implemented manually as demonstrated below. -```rust +```rust,no_run # use pyo3::prelude::*; # #[allow(dead_code)] struct MyPyObjectWrapper(PyObject); @@ -673,7 +673,7 @@ impl<'a, 'py> IntoPyObject<'py> for &'a MyPyObjectWrapper { `IntoPyObject::into_py_object` returns either `Bound` or `Borrowed` depending on the implementation for a concrete type. For example, the `IntoPyObject` implementation for `u32` produces a `Bound<'py, PyInt>` and the `bool` implementation produces a `Borrowed<'py, 'py, PyBool>`: -```rust +```rust,no_run use pyo3::prelude::*; use pyo3::IntoPyObject; use pyo3::types::{PyBool, PyInt}; @@ -700,7 +700,7 @@ In this example if we wanted to combine `ints_as_pyints` and `bools_as_pybool` i Instead, we can write a function that generically converts vectors of either integers or bools into a vector of `Py` using the [`BoundObject`] trait: -```rust +```rust,no_run # use pyo3::prelude::*; # use pyo3::BoundObject; # use pyo3::IntoPyObject; @@ -738,55 +738,7 @@ let vec_of_pyobjs: Vec> = Python::with_gil(|py| { In the example above we used `BoundObject::into_any` and `BoundObject::unbind` to manipulate the python types and smart pointers into the result type we wanted to produce from the function. -### `IntoPy` - -
- -⚠️ Warning: API update in progress 🛠️ - -PyO3 0.23 has introduced `IntoPyObject` as the new trait for to-python conversions. While `#[pymethods]` and `#[pyfunction]` contain a compatibility layer to allow `IntoPy` as a return type, all Python API have been migrated to use `IntoPyObject`. To migrate implement `IntoPyObject` for your type. -
- - -This trait defines the to-python conversion for a Rust type. It is usually implemented as -`IntoPy`, which is the trait needed for returning a value from `#[pyfunction]` and -`#[pymethods]`. - -All types in PyO3 implement this trait, as does a `#[pyclass]` which doesn't use `extends`. - -Occasionally you may choose to implement this for custom types which are mapped to Python types -_without_ having a unique python type. - -```rust -use pyo3::prelude::*; -# #[allow(dead_code)] -struct MyPyObjectWrapper(PyObject); - -#[allow(deprecated)] -impl IntoPy for MyPyObjectWrapper { - fn into_py(self, py: Python<'_>) -> PyObject { - self.0 - } -} -``` - -### The `ToPyObject` trait - -
- -⚠️ Warning: API update in progress 🛠️ - -PyO3 0.23 has introduced `IntoPyObject` as the new trait for to-python conversions. To migrate -implement `IntoPyObject` on a reference of your type (`impl<'py> IntoPyObject<'py> for &Type { ... }`). -
- -[`ToPyObject`] is a conversion trait that allows various objects to be -converted into [`PyObject`]. `IntoPy` serves the -same purpose, except that it consumes `self`. - -[`IntoPy`]: {{#PYO3_DOCS_URL}}/pyo3/conversion/trait.IntoPy.html [`FromPyObject`]: {{#PYO3_DOCS_URL}}/pyo3/conversion/trait.FromPyObject.html -[`ToPyObject`]: {{#PYO3_DOCS_URL}}/pyo3/conversion/trait.ToPyObject.html [`IntoPyObject`]: {{#PYO3_DOCS_URL}}/pyo3/conversion/trait.IntoPyObject.html [`IntoPyObjectExt`]: {{#PYO3_DOCS_URL}}/pyo3/conversion/trait.IntoPyObjectExt.html [`PyObject`]: {{#PYO3_DOCS_URL}}/pyo3/type.PyObject.html diff --git a/guide/src/ecosystem/logging.md b/guide/src/ecosystem/logging.md index da95c4a7cd2..e7d5c3c1e01 100644 --- a/guide/src/ecosystem/logging.md +++ b/guide/src/ecosystem/logging.md @@ -18,7 +18,7 @@ Python programs. Use [`pyo3_log::init`][init] to install the logger in its default configuration. It's also possible to tweak its configuration (mostly to tune its performance). -```rust +```rust,no_run use log::info; use pyo3::prelude::*; @@ -61,7 +61,7 @@ To have python logs be handled by Rust, one need only register a rust function t This has been implemented within the [pyo3-pylogger] crate. -```rust +```rust,no_run use log::{info, warn}; use pyo3::prelude::*; diff --git a/guide/src/ecosystem/tracing.md b/guide/src/ecosystem/tracing.md index 341d0759e96..c9fd355173a 100644 --- a/guide/src/ecosystem/tracing.md +++ b/guide/src/ecosystem/tracing.md @@ -37,7 +37,7 @@ implementation defined in and passed in from Python. There are many ways an extension module could integrate `pyo3-python-tracing-subscriber` but a simple one may look something like this: -```rust +```rust,no_run #[tracing::instrument] #[pyfunction] fn fibonacci(index: usize, use_memoized: bool) -> PyResult { diff --git a/guide/src/exception.md b/guide/src/exception.md index 8de04ba986a..2517ebde8b4 100644 --- a/guide/src/exception.md +++ b/guide/src/exception.md @@ -40,7 +40,7 @@ Python::with_gil(|py| { When using PyO3 to create an extension module, you can add the new exception to the module like this, so that it is importable from Python: -```rust +```rust,no_run use pyo3::prelude::*; use pyo3::exceptions::PyException; @@ -77,7 +77,7 @@ Python::with_gil(|py| { Python has an [`isinstance`](https://docs.python.org/3/library/functions.html#isinstance) method to check an object's type. In PyO3 every object has the [`PyAny::is_instance`] and [`PyAny::is_instance_of`] methods which do the same thing. -```rust +```rust,no_run use pyo3::prelude::*; use pyo3::types::{PyBool, PyList}; @@ -94,7 +94,7 @@ Python::with_gil(|py| { To check the type of an exception, you can similarly do: -```rust +```rust,no_run # use pyo3::exceptions::PyTypeError; # use pyo3::prelude::*; # Python::with_gil(|py| { @@ -109,7 +109,7 @@ It is possible to use an exception defined in Python code as a native Rust type. The `import_exception!` macro allows importing a specific exception class and defines a Rust type for that exception. -```rust +```rust,no_run #![allow(dead_code)] use pyo3::prelude::*; diff --git a/guide/src/faq.md b/guide/src/faq.md index 83089cf395e..8efe0b61bad 100644 --- a/guide/src/faq.md +++ b/guide/src/faq.md @@ -86,7 +86,7 @@ You can give the Python interpreter a chance to process the signal properly by c You may have a nested struct similar to this: -```rust +```rust,no_run # use pyo3::prelude::*; #[pyclass] #[derive(Clone)] @@ -126,7 +126,7 @@ b: This can be especially confusing if the field is mutable, as getting the field and then mutating it won't persist - you'll just get a fresh clone of the original on the next access. Unfortunately Python and Rust don't agree about ownership - if PyO3 gave out references to (possibly) temporary Rust objects to Python code, Python code could then keep that reference alive indefinitely. Therefore returning Rust objects requires cloning. If you don't want that cloning to happen, a workaround is to allocate the field on the Python heap and store a reference to that, by using [`Py<...>`]({{#PYO3_DOCS_URL}}/pyo3/struct.Py.html): -```rust +```rust,no_run # use pyo3::prelude::*; #[pyclass] struct Inner {/* fields omitted */} @@ -179,7 +179,7 @@ However, when the dependency is renamed, or your crate only indirectly depends on `pyo3`, you need to let the macro code know where to find the crate. This is done with the `crate` attribute: -```rust +```rust,no_run # use pyo3::prelude::*; # pub extern crate pyo3; # mod reexported { pub use ::pyo3; } diff --git a/guide/src/features.md b/guide/src/features.md index b48c138b287..450c1b4ea04 100644 --- a/guide/src/features.md +++ b/guide/src/features.md @@ -59,7 +59,9 @@ The feature has some unfinished refinements and performance improvements. To hel ### `experimental-inspect` -This feature adds the `pyo3::inspect` module, as well as `IntoPy::type_output` and `FromPyObject::type_input` APIs to produce Python type "annotations" for Rust types. +This feature adds to the built binaries introspection data that can be then retrieved using the `pyo3-introspection` crate to generate [type stubs](https://typing.readthedocs.io/en/latest/source/stubs.html). + +Also, this feature adds the `pyo3::inspect` module, as well as `IntoPy::type_output` and `FromPyObject::type_input` APIs to produce Python type "annotations" for Rust types. This is a first step towards adding first-class support for generating type annotations automatically in PyO3, however work is needed to finish this off. All feedback and offers of help welcome on [issue #2454](https://github.com/PyO3/pyo3/issues/2454). @@ -119,6 +121,13 @@ These features enable conversions between Python types and types from other Rust Adds a dependency on [anyhow](https://docs.rs/anyhow). Enables a conversion from [anyhow](https://docs.rs/anyhow)’s [`Error`](https://docs.rs/anyhow/latest/anyhow/struct.Error.html) type to [`PyErr`]({{#PYO3_DOCS_URL}}/pyo3/struct.PyErr.html), for easy error handling. +### `arc_lock` + +Enables Pyo3's `MutexExt` trait for all Mutexes that extend on [`lock_api::Mutex`](https://docs.rs/lock_api/latest/lock_api/struct.Mutex.html) and are wrapped in an [`Arc`](https://doc.rust-lang.org/std/sync/struct.Arc.html) type. Like [`Arc`](https://docs.rs/parking_lot/latest/parking_lot/type.Mutex.html#method.lock_arc) + +### `bigdecimal` +Adds a dependency on [bigdecimal](https://docs.rs/bigdecimal) and enables conversions into its [`BigDecimal`](https://docs.rs/bigdecimal/latest/bigdecimal/struct.BigDecimal.html) type. + ### `chrono` Adds a dependency on [chrono](https://docs.rs/chrono). Enables a conversion from [chrono](https://docs.rs/chrono)'s types to python: @@ -163,6 +172,10 @@ Adds a dependency on [jiff@0.2](https://docs.rs/jiff/0.2) and requires MSRV 1.70 - [Zoned](https://docs.rs/jiff/0.2/jiff/struct.Zoned.html) -> [`PyDateTime`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyDateTime.html) - [Timestamp](https://docs.rs/jiff/0.2/jiff/struct.Timestamp.html) -> [`PyDateTime`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyDateTime.html) +### `lock_api` + +Adds a dependency on [lock_api](https://docs.rs/lock_api) and enables Pyo3's `MutexExt` trait for all mutexes that extend on [`lock_api::Mutex`](https://docs.rs/lock_api/latest/lock_api/struct.Mutex.html) (like `parking_lot` or `spin`). + ### `num-bigint` Adds a dependency on [num-bigint](https://docs.rs/num-bigint) and enables conversions into its [`BigInt`](https://docs.rs/num-bigint/latest/num_bigint/struct.BigInt.html) and [`BigUint`](https://docs.rs/num-bigint/latest/num_bigint/struct.BigUint.html) types. @@ -175,16 +188,37 @@ Adds a dependency on [num-complex](https://docs.rs/num-complex) and enables conv Adds a dependency on [num-rational](https://docs.rs/num-rational) and enables conversions into its [`Ratio`](https://docs.rs/num-rational/latest/num_rational/struct.Ratio.html) type. +### `ordered-float` + +Adds a dependency on [ordered-float](https://docs.rs/ordered-float) and enables conversions between [ordered-float](https://docs.rs/ordered-float)'s types and Python: +- [NotNan](https://docs.rs/ordered-float/latest/ordered_float/struct.NotNan.html) -> [`PyFloat`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyFloat.html) +- [OrderedFloat](https://docs.rs/ordered-float/latest/ordered_float/struct.OrderedFloat.html) -> [`PyFloat`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyFloat.html) + +### `parking-lot` + +Adds a dependency on [parking_lot](https://docs.rs/parking_lot) and enables Pyo3's `OnceExt` & `MutexExt` traits for [`parking_lot::Once`](https://docs.rs/parking_lot/latest/parking_lot/struct.Once.html) and [`parking_lot::Mutex`](https://docs.rs/parking_lot/latest/parking_lot/type.Mutex.html) types. + ### `rust_decimal` Adds a dependency on [rust_decimal](https://docs.rs/rust_decimal) and enables conversions into its [`Decimal`](https://docs.rs/rust_decimal/latest/rust_decimal/struct.Decimal.html) type. +### `time` + +Adds a dependency on [time](https://docs.rs/time) and requires MSRV 1.67.1. Enables conversions between [time](https://docs.rs/time)'s types and Python: +- [Date](https://docs.rs/time/0.3.38/time/struct.Date.html) -> [`PyDate`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyDate.html) +- [Time](https://docs.rs/time/0.3.38/time/struct.Time.html) -> [`PyTime`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyTime.html) +- [OffsetDateTime](https://docs.rs/time/0.3.38/time/struct.OffsetDateTime.html) -> [`PyDateTime`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyDateTime.html) +- [PrimitiveDateTime](https://docs.rs/time/0.3.38/time/struct.PrimitiveDateTime.html) -> [`PyDateTime`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyDateTime.html) +- [Duration](https://docs.rs/time/0.3.38/time/struct.Duration.html) -> [`PyDelta`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyDelta.html) +- [UtcOffset](https://docs.rs/time/0.3.38/time/struct.UtcOffset.html) -> [`PyTzInfo`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyTzInfo.html) +- [UtcDateTime](https://docs.rs/time/0.3.38/time/struct.UtcDateTime.html) -> [`PyDateTime`]({{#PYO3_DOCS_URL}}/pyo3/types/struct.PyDateTime.html) + ### `serde` Enables (de)serialization of `Py` objects via [serde](https://serde.rs/). This allows to use [`#[derive(Serialize, Deserialize)`](https://serde.rs/derive.html) on structs that hold references to `#[pyclass]` instances -```rust +```rust,no_run # #[cfg(feature = "serde")] # #[allow(dead_code)] # mod serde_only { @@ -214,4 +248,4 @@ Adds a dependency on [smallvec](https://docs.rs/smallvec) and enables conversion ### `uuid` -Adds a dependency on [uuid](https://docs.rs/uuid) and enables conversions into its [`Uuid`](https://docs.rs/uuid/latest/uuid/struct.Uuid.html) type. \ No newline at end of file +Adds a dependency on [uuid](https://docs.rs/uuid) and enables conversions into its [`Uuid`](https://docs.rs/uuid/latest/uuid/struct.Uuid.html) type. diff --git a/guide/src/free-threading.md b/guide/src/free-threading.md index ffb95d240a1..6df9ac01cfb 100644 --- a/guide/src/free-threading.md +++ b/guide/src/free-threading.md @@ -72,7 +72,7 @@ thread-safe, then pass `gil_used = false` as a parameter to the `pymodule` procedural macro declaring the module or call `PyModule::gil_used` on a `PyModule` instance. For example: -```rust +```rust,no_run use pyo3::prelude::*; /// This module supports free-threaded Python @@ -85,7 +85,7 @@ fn my_extension(m: &Bound<'_, PyModule>) -> PyResult<()> { Or for a module that is set up without using the `pymodule` macro: -```rust +```rust,no_run use pyo3::prelude::*; # #[allow(dead_code)] @@ -208,9 +208,8 @@ The most straightforward way to trigger this problem is to use the Python [`pyclass`]({{#PYO3_DOCS_URL}}/pyo3/attr.pyclass.html) in multiple threads. For example, consider the following implementation: -```rust +```rust,no_run # use pyo3::prelude::*; -# fn main() { #[pyclass] #[derive(Default)] struct ThreadIter { @@ -229,7 +228,6 @@ impl ThreadIter { self.count } } -# } ``` And then if we do something like this in Python: @@ -309,7 +307,6 @@ extension traits. Here is an example of how to use [`OnceExt`] to enable single-initialization of a runtime cache holding a `Py`. ```rust -# fn main() { # use pyo3::prelude::*; use std::sync::Once; use pyo3::sync::OnceExt; @@ -331,7 +328,6 @@ Python::with_gil(|py| { cache.cache = Some(PyDict::new(py).unbind()); }); }); -# } ``` ### `GILProtected` is not exposed diff --git a/guide/src/function.md b/guide/src/function.md index 323bc9c8f87..a3121835215 100644 --- a/guide/src/function.md +++ b/guide/src/function.md @@ -4,7 +4,7 @@ The `#[pyfunction]` attribute is used to define a Python function from a Rust fu The following example defines a function called `double` in a Python module called `my_extension`: -```rust +```rust,no_run use pyo3::prelude::*; #[pyfunction] @@ -79,7 +79,7 @@ The `#[pyo3]` attribute can be used to modify properties of the generated Python The following example creates a function `pyfunction_with_module` which returns the containing module's name (i.e. `module_with_fn`): - ```rust + ```rust,no_run use pyo3::prelude::*; use pyo3::types::PyString; @@ -174,7 +174,7 @@ annotated with `#[pyfn]`. To simplify PyO3, it is expected that `#[pyfn]` may be An example of `#[pyfn]` is below: -```rust +```rust,no_run use pyo3::prelude::*; #[pymodule] @@ -191,7 +191,7 @@ fn my_extension(m: &Bound<'_, PyModule>) -> PyResult<()> { `#[pyfn(m)]` is just syntactic sugar for `#[pyfunction]`, and takes all the same options documented in the rest of this chapter. The code above is expanded to the following: -```rust +```rust,no_run use pyo3::prelude::*; #[pymodule] diff --git a/guide/src/function/signature.md b/guide/src/function/signature.md index 431cad87bfd..1e011a27840 100644 --- a/guide/src/function/signature.md +++ b/guide/src/function/signature.md @@ -10,7 +10,7 @@ This section of the guide goes into detail about use of the `#[pyo3(signature = For example, below is a function that accepts arbitrary keyword arguments (`**kwargs` in Python syntax) and returns the number that was passed: -```rust +```rust,no_run use pyo3::prelude::*; use pyo3::types::PyDict; @@ -38,7 +38,7 @@ Just like in Python, the following constructs can be part of the signature:: code unmodified. Example: -```rust +```rust,no_run # use pyo3::prelude::*; use pyo3::types::{PyDict, PyTuple}; # @@ -79,7 +79,7 @@ impl MyClass { Arguments of type `Python` must not be part of the signature: -```rust +```rust,no_run # #![allow(dead_code)] # use pyo3::prelude::*; #[pyfunction] @@ -108,7 +108,7 @@ num=-1 > Note: to use keywords like `struct` as a function argument, use "raw identifier" syntax `r#struct` in both the signature and the function definition: > -> ```rust +> ```rust,no_run > # #![allow(dead_code)] > # use pyo3::prelude::*; > #[pyfunction(signature = (r#struct = "foo"))] diff --git a/guide/src/getting-started.md b/guide/src/getting-started.md index e2cc040bbd7..99cffa25895 100644 --- a/guide/src/getting-started.md +++ b/guide/src/getting-started.md @@ -145,7 +145,7 @@ classifiers = [ After this you can setup Rust code to be available in Python as below; for example, you can place this code in `src/lib.rs`: -```rust +```rust,no_run use pyo3::prelude::*; /// Formats the sum of two numbers as string. diff --git a/guide/src/migration.md b/guide/src/migration.md index 35126dfcaef..3beaa5bc07c 100644 --- a/guide/src/migration.md +++ b/guide/src/migration.md @@ -3,9 +3,26 @@ This guide can help you upgrade code through breaking changes from one PyO3 version to the next. For a detailed list of all changes, see the [CHANGELOG](changelog.md). -## from 0.22.* to 0.23 +## from 0.24.* to 0.25 +### `AsPyPointer` removal
Click to expand +The `AsPyPointer` trait is mostly a leftover from the now removed gil-refs API. The last remaining uses were the GC API, namely `PyVisit::call`, and identity comparison (`PyAnyMethods::is` and `Py::is`). + +`PyVisit::call` has been updated to take `T: Into>>`, which allows for arguments of type `&Py`, `&Option>` and `Option<&Py>`. It is unlikely any changes are needed here to migrate. + +`PyAnyMethods::is`/`Py::is` has been updated to take `T: AsRef>>`. Additionally `AsRef>>` implementations were added for `Py`, `Bound` and `Borrowed`. Because of the existing `AsRef> for Bound` implementation this may cause inference issues in non-generic code. This can be easily migrated by switching to `as_any` instead of `as_ref` for these calls. +
+ +## from 0.23.* to 0.24 +
+Click to expand +There were no significant changes from 0.23 to 0.24 which required documenting in this guide. +
+ +## from 0.22.* to 0.23 +
+Click to expand PyO3 0.23 is a significant rework of PyO3's internals for two major improvements: - Support of Python 3.13's new freethreaded build (aka "3.13t") @@ -20,7 +37,7 @@ The sections below discuss the rationale and details of each change in more dept
### Free-threaded Python Support -
+
Click to expand PyO3 0.23 introduces initial support for the new free-threaded build of @@ -43,7 +60,7 @@ See [the guide section on free-threaded Python](free-threading.md) for more deta
### New `IntoPyObject` trait unifies to-Python conversions -
+
Click to expand PyO3 0.23 introduces a new `IntoPyObject` trait to convert Rust types into Python objects which replaces both `IntoPy` and `ToPyObject`. @@ -70,7 +87,7 @@ are deprecated and will be removed in a future PyO3 version. To implement the new trait you may use the new `IntoPyObject` and `IntoPyObjectRef` derive macros as below. -```rust +```rust,no_run # use pyo3::prelude::*; #[derive(IntoPyObject, IntoPyObjectRef)] struct Struct { @@ -103,7 +120,7 @@ impl ToPyObject for MyPyObjectWrapper { ``` After: -```rust +```rust,no_run # use pyo3::prelude::*; # #[allow(dead_code)] # struct MyPyObjectWrapper(PyObject); @@ -132,7 +149,7 @@ impl<'a, 'py> IntoPyObject<'py> for &'a MyPyObjectWrapper {
### To-Python conversions changed for byte collections (`Vec`, `[u8; N]` and `SmallVec<[u8; N]>`). -
+
Click to expand With the introduction of the `IntoPyObject` trait, PyO3's macros now prefer `IntoPyObject` implementations over `IntoPy` when producing Python values. This applies to `#[pyfunction]` and `#[pymethods]` return values and also fields accessed via `#[pyo3(get)]`. @@ -145,7 +162,7 @@ This change has an effect on functions and methods returning _byte_ collections In their new `IntoPyObject` implementation these will now turn into `PyBytes` rather than a `PyList`. All other `T`s are unaffected and still convert into a `PyList`. -```rust +```rust,no_run # #![allow(dead_code)] # use pyo3::prelude::*; #[pyfunction] @@ -170,7 +187,7 @@ This is purely additional and should just extend the possible return types.
### `gil-refs` feature removed -
+
Click to expand PyO3 0.23 completes the removal of the "GIL Refs" API in favour of the new "Bound" API introduced in PyO3 0.21. @@ -179,7 +196,7 @@ With the removal of the old API, many "Bound" API functions which had been intro Before: -```rust +```rust,ignore # #![allow(deprecated)] # use pyo3::prelude::*; # use pyo3::types::PyTuple; @@ -237,7 +254,7 @@ where After: -```rust +```rust,no_run # use pyo3::prelude::*; # use pyo3::types::{PyDict, IntoPyDict}; # use std::collections::HashMap; @@ -282,7 +299,7 @@ and unnoticed changes in behavior. With 0.24 this restriction will be lifted aga Before: -```rust +```rust,no_run # #![allow(deprecated, dead_code)] # use pyo3::prelude::*; #[pyfunction] @@ -293,7 +310,7 @@ fn increment(x: u64, amount: Option) -> u64 { After: -```rust +```rust,no_run # #![allow(dead_code)] # use pyo3::prelude::*; #[pyfunction] @@ -326,7 +343,7 @@ To migrate, place a `#[pyo3(eq, eq_int)]` attribute on simple enum classes. Before: -```rust +```rust,no_run # #![allow(deprecated, dead_code)] # use pyo3::prelude::*; #[pyclass] @@ -338,7 +355,7 @@ enum SimpleEnum { After: -```rust +```rust,no_run # #![allow(dead_code)] # use pyo3::prelude::*; #[pyclass(eq, eq_int)] @@ -528,7 +545,7 @@ impl PyClassIter { If returning `"done"` via `StopIteration` is not really required, this should be written as -```rust +```rust,no_run use pyo3::prelude::*; #[pyclass] @@ -553,7 +570,7 @@ This form also has additional benefits: It has already worked in previous PyO3 v Alternatively, the implementation can also be done as it would in Python itself, i.e. by "raising" a `StopIteration` exception -```rust +```rust,no_run use pyo3::prelude::*; use pyo3::exceptions::PyStopIteration; @@ -577,7 +594,7 @@ impl PyClassIter { Finally, an asynchronous iterator can directly return an awaitable without confusing wrapping -```rust +```rust,no_run use pyo3::prelude::*; #[pyclass] @@ -885,7 +902,7 @@ fn x_or_y(x: Option, y: u64) -> u64 { After: -```rust +```rust,no_run # #![allow(dead_code)] # use pyo3::prelude::*; @@ -915,7 +932,7 @@ fn add(a: u64, b: u64) -> u64 { After: -```rust +```rust,no_run # #![allow(dead_code)] # use pyo3::prelude::*; @@ -1105,7 +1122,7 @@ fn required_argument_after_option(x: Option, y: i32) {} After, specify the intended Python signature explicitly: -```rust +```rust,no_run # #![allow(dead_code)] # use pyo3::prelude::*; @@ -1237,7 +1254,7 @@ Python::with_gil(|py| { After, some type annotations may be necessary: -```rust +```rust,ignore # #![allow(deprecated)] # use pyo3::prelude::*; # @@ -1553,7 +1570,7 @@ impl PyObjectProtocol for MyClass { After: -```rust +```rust,no_run use pyo3::prelude::*; #[pyclass] @@ -1646,7 +1663,7 @@ let result: PyResult<()> = PyErr::new::("error message").into(); ``` After (also using the new reworked exception types; see the following section): -```rust +```rust,no_run # use pyo3::{PyResult, exceptions::PyTypeError}; let result: PyResult<()> = Err(PyTypeError::new_err("error message")); ``` @@ -1712,7 +1729,7 @@ impl FromPy for PyObject { ``` After -```rust +```rust,ignore # use pyo3::prelude::*; # #[allow(dead_code)] struct MyPyObjectWrapper(PyObject); @@ -1736,7 +1753,7 @@ let obj = PyObject::from_py(1.234, py); ``` After: -```rust +```rust,ignore # #![allow(deprecated)] # use pyo3::prelude::*; # Python::with_gil(|py| { @@ -1853,7 +1870,7 @@ There can be two fixes: ``` After: - ```rust + ```rust,no_run # #![allow(dead_code)] use pyo3::prelude::*; @@ -1951,7 +1968,7 @@ impl MyClass { ``` After: -```rust +```rust,no_run # use pyo3::prelude::*; #[pyclass] struct MyClass {} diff --git a/guide/src/module.md b/guide/src/module.md index 1e274c7c953..fa756e85920 100644 --- a/guide/src/module.md +++ b/guide/src/module.md @@ -2,7 +2,7 @@ You can create a module using `#[pymodule]`: -```rust +```rust,no_run use pyo3::prelude::*; #[pyfunction] @@ -23,7 +23,7 @@ module to Python. The module's name defaults to the name of the Rust function. You can override the module name by using `#[pyo3(name = "custom_name")]`: -```rust +```rust,no_run use pyo3::prelude::*; #[pyfunction] @@ -108,7 +108,7 @@ It is not necessary to add `#[pymodule]` on nested modules, which is only requir Another syntax based on Rust inline modules is also available to declare modules. For example: -```rust +```rust,no_run # mod declarative_module_test { use pyo3::prelude::*; @@ -124,6 +124,9 @@ mod my_extension { #[pymodule_export] use super::double; // Exports the double function as part of the module + #[pymodule_export] + const PI: f64 = std::f64::consts::PI; // Exports PI constant as part of the module + #[pyfunction] // This will be part of the module fn triple(x: usize) -> usize { x * 3 @@ -151,7 +154,7 @@ For nested modules, the name of the parent module is automatically added. In the following example, the `Unit` class will have for `module` `my_extension.submodule` because it is properly nested but the `Ext` class will have for `module` the default `builtins` because it not nested. -```rust +```rust,no_run # mod declarative_module_module_attr_test { use pyo3::prelude::*; diff --git a/guide/src/performance.md b/guide/src/performance.md index 5a57585c4a0..3c9b63b648a 100644 --- a/guide/src/performance.md +++ b/guide/src/performance.md @@ -6,7 +6,7 @@ To achieve the best possible performance, it is useful to be aware of several tr Pythonic API implemented using PyO3 are often polymorphic, i.e. they will accept `&Bound<'_, PyAny>` and try to turn this into multiple more concrete types to which the requested operation is applied. This often leads to chains of calls to `extract`, e.g. -```rust +```rust,no_run # #![allow(dead_code)] # use pyo3::prelude::*; # use pyo3::{exceptions::PyTypeError, types::PyList}; @@ -33,7 +33,7 @@ fn frobnicate<'py>(value: &Bound<'py, PyAny>) -> PyResult> { This suboptimal as the `FromPyObject` trait requires `extract` to have a `Result` return type. For native types like `PyList`, it faster to use `downcast` (which `extract` calls internally) when the error value is ignored. This avoids the costly conversion of a `PyDowncastError` to a `PyErr` required to fulfil the `FromPyObject` contract, i.e. -```rust +```rust,no_run # #![allow(dead_code)] # use pyo3::prelude::*; # use pyo3::{exceptions::PyTypeError, types::PyList}; @@ -59,7 +59,7 @@ Calling `Python::with_gil` is effectively a no-op when the GIL is already held, For example, instead of writing -```rust +```rust,no_run # #![allow(dead_code)] # use pyo3::prelude::*; # use pyo3::types::PyList; @@ -80,7 +80,7 @@ impl PartialEq for FooBound<'_> { use the more efficient -```rust +```rust,no_run # #![allow(dead_code)] # use pyo3::prelude::*; # use pyo3::types::PyList; diff --git a/guide/src/python-from-rust.md b/guide/src/python-from-rust.md index ebb7fa1f4da..7ab39161e70 100644 --- a/guide/src/python-from-rust.md +++ b/guide/src/python-from-rust.md @@ -17,7 +17,7 @@ are met. Its lifetime `'py` is a central part of PyO3's API. The `Python<'py>` token serves three purposes: -* It provides global APIs for the Python interpreter, such as [`py.eval_bound()`][eval] and [`py.import_bound()`][import]. +* It provides global APIs for the Python interpreter, such as [`py.eval()`][eval] and [`py.import()`][import]. * It can be passed to functions that require a proof of holding the GIL, such as [`Py::clone_ref`][clone_ref]. * Its lifetime `'py` is used to bind many of PyO3's types to the Python interpreter, such as [`Bound<'py, T>`][Bound]. @@ -45,6 +45,6 @@ Because of the lack of exclusive `&mut` references, PyO3's APIs for Python objec [obtaining-py]: {{#PYO3_DOCS_URL}}/pyo3/marker/struct.Python.html#obtaining-a-python-token [`pyo3::sync`]: {{#PYO3_DOCS_URL}}/pyo3/sync/index.html [eval]: {{#PYO3_DOCS_URL}}/pyo3/marker/struct.Python.html#method.eval -[import]: {{#PYO3_DOCS_URL}}/pyo3/marker/struct.Python.html#method.import_bound +[import]: {{#PYO3_DOCS_URL}}/pyo3/marker/struct.Python.html#method.import [clone_ref]: {{#PYO3_DOCS_URL}}/pyo3/prelude/struct.Py.html#method.clone_ref [Bound]: {{#PYO3_DOCS_URL}}/pyo3/struct.Bound.html diff --git a/guide/src/python-from-rust/function-calls.md b/guide/src/python-from-rust/function-calls.md index 2e5cf3c589e..b64c062eec7 100644 --- a/guide/src/python-from-rust/function-calls.md +++ b/guide/src/python-from-rust/function-calls.md @@ -59,13 +59,13 @@ fn main() -> PyResult<()> { ## Creating keyword arguments -For the `call` and `call_method` APIs, `kwargs` are `Option<&Bound<'py, PyDict>>`, so can either be `None` or `Some(&dict)`. You can use the [`IntoPyDict`]({{#PYO3_DOCS_URL}}/pyo3/types/trait.IntoPyDict.html) trait to convert other dict-like containers, e.g. `HashMap` or `BTreeMap`, as well as tuples with up to 10 elements and `Vec`s where each element is a two-element tuple. +For the `call` and `call_method` APIs, `kwargs` are `Option<&Bound<'py, PyDict>>`, so can either be `None` or `Some(&dict)`. You can use the [`IntoPyDict`]({{#PYO3_DOCS_URL}}/pyo3/types/trait.IntoPyDict.html) trait to convert other dict-like containers, e.g. `HashMap` or `BTreeMap`, as well as tuples with up to 10 elements and `Vec`s where each element is a two-element tuple. To pass keyword arguments of different types, construct a `PyDict` object. ```rust use pyo3::prelude::*; -use pyo3::types::IntoPyDict; +use pyo3::types::{PyDict, IntoPyDict}; use std::collections::HashMap; -use pyo3_ffi::c_str; +use pyo3::ffi::c_str; fn main() -> PyResult<()> { let key1 = "key1"; @@ -102,7 +102,13 @@ fn main() -> PyResult<()> { kwargs.insert(key1, 1); fun.call(py, (), Some(&kwargs.into_py_dict(py)?))?; + // pass arguments of different types as PyDict + let kwargs = PyDict::new(py); + kwargs.set_item(key1, val1)?; + kwargs.set_item(key2, "string")?; + fun.call(py, (), Some(&kwargs))?; + Ok(()) }) } -``` \ No newline at end of file +``` diff --git a/guide/src/python-typing-hints.md b/guide/src/python-typing-hints.md index 43a15a21a6e..eb39bb35fb0 100644 --- a/guide/src/python-typing-hints.md +++ b/guide/src/python-typing-hints.md @@ -41,7 +41,7 @@ As we can see, those are not full definitions containing implementation, but jus ### What do the PEPs say? -At the time of writing this documentation, the `pyi` files are referenced in three PEPs. +At the time of writing this documentation, the `pyi` files are referenced in four PEPs. [PEP8 - Style Guide for Python Code - #Function Annotations](https://www.python.org/dev/peps/pep-0008/#function-annotations) (last point) recommends all third party library creators to provide stub files as the source of knowledge about the package for type checker tools. @@ -55,6 +55,8 @@ It contains a specification for them (highly recommended reading, since it conta [PEP561 - Distributing and Packaging Type Information](https://www.python.org/dev/peps/pep-0561/) describes in detail how to build packages that will enable type checking. In particular it contains information about how the stub files must be distributed in order for type checkers to use them. +[PEP560 - Core support for typing module and generic types](https://www.python.org/dev/peps/pep-0560/) describes the details on how Python's type system internally supports generics, including both runtime behavior and integration with static type checkers. + ## How to do it? [PEP561](https://www.python.org/dev/peps/pep-0561/) recognizes three ways of distributing type information: @@ -165,3 +167,77 @@ class Car: :return: the name of the color our great algorithm thinks is the best for this car """ ``` + +### Supporting Generics + +Type annotations can also be made generic in Python. They are useful for working +with different types while maintaining type safety. Usually, generic classes +inherit from the `typing.Generic` metaclass. + +Take for example the following `.pyi` file that specifies a `Car` that can +accept multiple types of wheels: + +```python +from typing import Generic, TypeVar + +W = TypeVar('W') + +class Car(Generic[W]): + def __init__(self, wheels: list[W]) -> None: ... + + def get_wheels(self) -> list[W]: ... + + def change_wheel(self, wheel_number: int, wheel: W) -> None: ... +``` + +This way, the end-user can specify the type with variables such as `truck: Car[SteelWheel] = ...` +and `f1_car: Car[AlloyWheel] = ...`. + +There is also a special syntax for specifying generic types in Python 3.12+: + +```python +class Car[W]: + def __init__(self, wheels: list[W]) -> None: ... + + def get_wheels(self) -> list[W]: ... +``` + +#### Runtime Behaviour + +Stub files (`pyi`) are only useful for static type checkers and ignored at runtime. Therefore, +PyO3 classes do not inherit from `typing.Generic` even if specified in the stub files. + +This can cause some runtime issues, as annotating a variable like `f1_car: Car[AlloyWheel] = ...` +can make Python call magic methods that are not defined. + +To overcome this limitation, implementers can pass the `generic` parameter to `pyclass` in Rust: + +```rust ignore +#[pyclass(generic)] +``` + +#### Advanced Users + +`#[pyclass(generic)]` implements a very simple runtime behavior that accepts +any generic argument. Advanced users can opt to manually implement +[`__class_geitem__`](https://docs.python.org/3/reference/datamodel.html#emulating-generic-types) +for the generic class to have more control. + +```rust ignore +impl MyClass { + #[classmethod] + #[pyo3(signature = (key, /))] + pub fn __class_getitem__( + cls: &Bound<'_, PyType>, + key: &Bound<'_, PyAny>, + ) -> PyResult { + /* implementation details */ + } +} +``` + +Note that [`pyo3::types::PyGenericAlias`][pygenericalias] can be helfpul when implementing +`__class_geitem__` as it can create [`types.GenericAlias`][genericalias] objects from Rust. + +[pygenericalias]: {{#PYO3_DOCS_URL}}/pyo3/types/struct.pygenericalias +[genericalias]: https://docs.python.org/3/library/types.html#types.GenericAlias \ No newline at end of file diff --git a/guide/src/trait-bounds.md b/guide/src/trait-bounds.md index e1b8e82f1db..18476eeb9f6 100644 --- a/guide/src/trait-bounds.md +++ b/guide/src/trait-bounds.md @@ -22,7 +22,7 @@ Let's work with the following basic example of an implementation of a optimizati Let's say we have a function `solve` that operates on a model and mutates its state. The argument of the function can be any model that implements the `Model` trait : -```rust +```rust,no_run # #![allow(dead_code)] pub trait Model { fn set_variables(&mut self, inputs: &Vec); @@ -63,7 +63,7 @@ class Model: The following wrapper will call the Python model from Rust, using a struct to hold the model as a `PyAny` object: -```rust +```rust,no_run # #![allow(dead_code)] use pyo3::prelude::*; use pyo3::types::PyList; @@ -116,7 +116,7 @@ impl Model for UserModel { Now that this bit is implemented, let's expose the model wrapper to Python. Let's add the PyO3 annotations and add a constructor: -```rust +```rust,no_run # #![allow(dead_code)] # pub trait Model { # fn set_variables(&mut self, inputs: &Vec); @@ -161,7 +161,7 @@ That's a bummer! However, we can write a second wrapper around these functions to call them directly. This wrapper will also perform the type conversions between Python and Rust. -```rust +```rust,no_run # #![allow(dead_code)] # use pyo3::prelude::*; # use pyo3::types::PyList; @@ -328,7 +328,7 @@ Let's modify the code performing the type conversion to give a helpful error mes We used in our `get_results` method the following call that performs the type conversion: -```rust +```rust,no_run # #![allow(dead_code)] # use pyo3::prelude::*; # use pyo3::types::PyList; @@ -379,7 +379,7 @@ impl Model for UserModel { Let's break it down in order to perform better error handling: -```rust +```rust,no_run # #![allow(dead_code)] # use pyo3::prelude::*; # use pyo3::types::PyList; @@ -456,7 +456,7 @@ Because of this, we can write a function wrapper that takes the `UserModel`--whi It is also required to make the struct public. -```rust +```rust,no_run # #![allow(dead_code)] use pyo3::prelude::*; use pyo3::types::PyList; diff --git a/newsfragments/4969.added.md b/newsfragments/4969.added.md deleted file mode 100644 index 7a59e9aef6f..00000000000 --- a/newsfragments/4969.added.md +++ /dev/null @@ -1 +0,0 @@ -Added `abi3-py313` feature \ No newline at end of file diff --git a/newsfragments/4978.added.md b/newsfragments/4978.added.md deleted file mode 100644 index 5518cee89d8..00000000000 --- a/newsfragments/4978.added.md +++ /dev/null @@ -1 +0,0 @@ -Implement getattr_opt in `PyAnyMethods` \ No newline at end of file diff --git a/newsfragments/4981.fixed.md b/newsfragments/4981.fixed.md deleted file mode 100644 index 0ac31b19a11..00000000000 --- a/newsfragments/4981.fixed.md +++ /dev/null @@ -1 +0,0 @@ -Fix `is_type_of` for native types not using same specialized check as `is_type_of_bound`. diff --git a/newsfragments/4984.added.md b/newsfragments/4984.added.md deleted file mode 100644 index 63559617812..00000000000 --- a/newsfragments/4984.added.md +++ /dev/null @@ -1 +0,0 @@ -Added PyInt constructor for all supported number types (i32, u32, i64, u64, isize, usize) \ No newline at end of file diff --git a/newsfragments/4988.fixed.md b/newsfragments/4988.fixed.md deleted file mode 100644 index 1a050b905e3..00000000000 --- a/newsfragments/4988.fixed.md +++ /dev/null @@ -1 +0,0 @@ -Fix `Probe` class naming issue with `#[pymethods]` diff --git a/newsfragments/4992.added.md b/newsfragments/4992.added.md deleted file mode 100644 index 36cb9a4e08e..00000000000 --- a/newsfragments/4992.added.md +++ /dev/null @@ -1 +0,0 @@ -Add `pyo3::sync::with_critical_section2` binding diff --git a/newsfragments/5002.fixed.md b/newsfragments/5002.fixed.md deleted file mode 100644 index ad9b0091a79..00000000000 --- a/newsfragments/5002.fixed.md +++ /dev/null @@ -1 +0,0 @@ -Fix compile failure with required `#[pyfunction]` arguments taking `Option<&str>` and `Option<&T>` (for `#[pyclass]` types). diff --git a/newsfragments/5008.fixed.md b/newsfragments/5008.fixed.md deleted file mode 100644 index 6b431d6e437..00000000000 --- a/newsfragments/5008.fixed.md +++ /dev/null @@ -1 +0,0 @@ -Fix `PyString::from_object`, avoid out of bounds reads by null terminating the `encoding` and `errors` parameters \ No newline at end of file diff --git a/newsfragments/5013.added.md b/newsfragments/5013.added.md deleted file mode 100644 index 0becc3ee1e9..00000000000 --- a/newsfragments/5013.added.md +++ /dev/null @@ -1 +0,0 @@ -Implement `PyCallArgs` for `Borrowed<'_, 'py, PyTuple>`, `&Bound<'py, PyTuple>`, and `&Py`. diff --git a/newsfragments/5015.fixed.md b/newsfragments/5015.fixed.md deleted file mode 100644 index b1da048828f..00000000000 --- a/newsfragments/5015.fixed.md +++ /dev/null @@ -1 +0,0 @@ -Fixes compile error if more options followed after `crate` for `#[pyfunction]`. \ No newline at end of file diff --git a/noxfile.py b/noxfile.py index 61e8dee71fd..94e0f392044 100644 --- a/noxfile.py +++ b/noxfile.py @@ -19,7 +19,6 @@ List, Optional, Tuple, - Generator, ) @@ -72,6 +71,9 @@ def _supported_interpreter_versions( min_minor = int(min_version.split(".")[1]) max_minor = int(max_version.split(".")[1]) versions = [f"{major}.{minor}" for minor in range(min_minor, max_minor + 1)] + # Add free-threaded builds for 3.13+ + if python_impl == "cpython": + versions += [f"{major}.{minor}t" for minor in range(13, max_minor + 1)] return versions @@ -90,17 +92,49 @@ def test_rust(session: nox.Session): _run_cargo_test(session, package="pyo3-build-config") _run_cargo_test(session, package="pyo3-macros-backend") _run_cargo_test(session, package="pyo3-macros") - _run_cargo_test(session, package="pyo3-ffi") - _run_cargo_test(session) - # the free-threaded build ignores abi3, so we skip abi3 - # tests to avoid unnecessarily running the tests twice - if not FREE_THREADED_BUILD: - _run_cargo_test(session, features="abi3") - if "skip-full" not in session.posargs: - _run_cargo_test(session, features="full jiff-02") - if not FREE_THREADED_BUILD: - _run_cargo_test(session, features="abi3 full jiff-02") + extra_flags = [] + # pypy and graalpy don't have Py_Initialize APIs, so we can only + # build the main tests, not run them + if sys.implementation.name in ("pypy", "graalpy"): + extra_flags.append("--no-run") + + _run_cargo_test(session, package="pyo3-ffi", extra_flags=extra_flags) + + extra_flags.append("--no-default-features") + + for feature_set in _get_feature_sets(): + flags = extra_flags.copy() + print(feature_set) + + if feature_set is None or "full" not in feature_set: + # doctests require at least the macros feature, which is + # activated by the full feature set + # + # using `--all-targets` makes cargo run everything except doctests + flags.append("--all-targets") + + # We need to pass the feature set to the test command + # so that it can be used in the test code + # (e.g. for `#[cfg(feature = "abi3-py37")]`) + if feature_set and "abi3" in feature_set and FREE_THREADED_BUILD: + # free-threaded builds don't support abi3 yet + continue + + _run_cargo_test(session, features=feature_set, extra_flags=flags) + + if ( + feature_set + and "abi3" in feature_set + and "full" in feature_set + and sys.version_info >= (3, 7) + ): + # run abi3-py37 tests to check abi3 forward compatibility + _run_cargo_test( + session, + features=feature_set.replace("abi3", "abi3-py37"), + extra_flags=flags, + ) @nox.session(name="test-py", venv_backend="none") @@ -179,7 +213,8 @@ def _clippy(session: nox.Session, *, env: Dict[str, str] = None) -> bool: _run_cargo( session, "clippy", - *feature_set, + "--no-default-features", + *((f"--features={feature_set}",) if feature_set else ()), "--all-targets", "--workspace", "--", @@ -256,7 +291,8 @@ def _check(env: Dict[str, str]) -> None: _run_cargo( session, "check", - *feature_set, + "--no-default-features", + *((f"--features={feature_set}",) if feature_set else ()), "--all-targets", "--workspace", env=env, @@ -426,6 +462,10 @@ def docs(session: nox.Session) -> None: features = "full" + if get_rust_version()[:2] >= (1, 67): + # time needs MSRC 1.67+ + features += ",time" + if get_rust_version()[:2] >= (1, 70): # jiff needs MSRC 1.70+ features += ",jiff-02" @@ -486,6 +526,7 @@ def check_guide(session: nox.Session): "--include-fragments", str(PYO3_GUIDE_SRC), *remap_args, + "--accept=200,429", *session.posargs, ) # check external links in the docs @@ -497,6 +538,7 @@ def check_guide(session: nox.Session): *remap_args, f"--exclude=file://{PYO3_DOCS_TARGET}", "--exclude=http://www.adobe.com/", + "--accept=200,429", *session.posargs, ) @@ -573,6 +615,7 @@ def address_sanitizer(session: nox.Session): _IGNORE_CHANGELOG_PR_CATEGORIES = ( "release", "docs", + "ci", ) @@ -700,11 +743,11 @@ def test_version_limits(session: nox.Session): config_file.set("CPython", "3.6") _run_cargo(session, "check", env=env, expect_error=True) - assert "3.14" not in PY_VERSIONS - config_file.set("CPython", "3.14") + assert "3.15" not in PY_VERSIONS + config_file.set("CPython", "3.15") _run_cargo(session, "check", env=env, expect_error=True) - # 3.14 CPython should build with forward compatibility + # 3.15 CPython should build with forward compatibility env["PYO3_USE_ABI3_FORWARD_COMPATIBILITY"] = "1" _run_cargo(session, "check", env=env) @@ -724,7 +767,10 @@ def check_feature_powerset(session: nox.Session): cargo_toml = toml.loads((PYO3_DIR / "Cargo.toml").read_text()) - EXPECTED_ABI3_FEATURES = {f"abi3-py3{ver.split('.')[1]}" for ver in PY_VERSIONS} + # free-threaded builds do not support ABI3 (yet) + EXPECTED_ABI3_FEATURES = { + f"abi3-py3{ver.split('.')[1]}" for ver in PY_VERSIONS if not ver.endswith("t") + } EXCLUDED_FROM_FULL = { "nightly", @@ -812,8 +858,28 @@ def update_ui_tests(session: nox.Session): env["TRYBUILD"] = "overwrite" command = ["test", "--test", "test_compile_error"] _run_cargo(session, *command, env=env) - _run_cargo(session, *command, "--features=full,jiff-02", env=env) - _run_cargo(session, *command, "--features=abi3,full,jiff-02", env=env) + _run_cargo(session, *command, "--features=full,jiff-02,time", env=env) + _run_cargo(session, *command, "--features=abi3,full,jiff-02,time", env=env) + + +@nox.session(name="test-introspection") +def test_introspection(session: nox.Session): + session.install("maturin") + target = os.environ.get("CARGO_BUILD_TARGET") + for options in ([], ["--release"]): + if target is not None: + options += ("--target", target) + session.run_always("maturin", "develop", "-m", "./pytests/Cargo.toml", *options) + # We look for the built library + lib_file = None + for file in Path(session.virtualenv.location).rglob("pyo3_pytests.*"): + if file.is_file(): + lib_file = str(file.resolve()) + _run_cargo_test( + session, + package="pyo3-introspection", + env={"PYO3_PYTEST_LIB_PATH": lib_file}, + ) def _build_docs_for_ffi_check(session: nox.Session) -> None: @@ -839,6 +905,13 @@ def get_rust_version() -> Tuple[int, int, int, List[str]]: return (*map(int, version_number.split(".")), extra) +def is_rust_nightly() -> bool: + for line in _get_rust_info(): + if line.startswith(_RELEASE_LINE_START): + return line.strip().endswith("-nightly") + return False + + def _get_rust_default_target() -> str: for line in _get_rust_info(): if line.startswith(_HOST_LINE_START): @@ -846,30 +919,28 @@ def _get_rust_default_target() -> str: @lru_cache() -def _get_feature_sets() -> Generator[Tuple[str, ...], None, None]: - """Returns feature sets to use for clippy job""" +def _get_feature_sets() -> Tuple[Optional[str], ...]: + """Returns feature sets to use for Rust jobs""" cargo_target = os.getenv("CARGO_BUILD_TARGET", "") - yield from ( - ("--no-default-features",), - ( - "--no-default-features", - "--features=abi3", - ), - ) - features = "full" if "wasm32-wasip1" not in cargo_target: # multiple-pymethods not supported on wasm features += ",multiple-pymethods" + if get_rust_version()[:2] >= (1, 67): + # time needs MSRC 1.67+ + features += ",time" + if get_rust_version()[:2] >= (1, 70): # jiff needs MSRC 1.70+ features += ",jiff-02" - yield (f"--features={features}",) - yield (f"--features=abi3,{features}",) + if is_rust_nightly(): + features += ",nightly" + + return (None, "abi3", features, f"abi3,{features}") _RELEASE_LINE_START = "release: " @@ -932,21 +1003,27 @@ def _run_cargo_test( *, package: Optional[str] = None, features: Optional[str] = None, + env: Optional[Dict[str, str]] = None, + extra_flags: Optional[List[str]] = None, ) -> None: command = ["cargo"] if "careful" in session.posargs: # do explicit setup so failures in setup can be seen _run_cargo(session, "careful", "setup") command.append("careful") + command.extend(("test", "--no-fail-fast")) + if "release" in session.posargs: command.append("--release") if package: command.append(f"--package={package}") if features: command.append(f"--features={features}") + if extra_flags: + command.extend(extra_flags) - _run(session, *command, external=True) + _run(session, *command, external=True, env=env or {}) def _run_cargo_publish(session: nox.Session, *, package: str) -> None: @@ -993,6 +1070,11 @@ def set( self, implementation: str, version: str, build_flags: Iterable[str] = () ) -> None: """Set the contents of this config file to the given implementation and version.""" + if version.endswith("t"): + # Free threaded versions pass the support in config file through a flag + version = version[:-1] + build_flags = (*build_flags, "Py_GIL_DISABLED") + self._config_file.seek(0) self._config_file.truncate(0) self._config_file.write( diff --git a/pyo3-benches/benches/bench_intopyobject.rs b/pyo3-benches/benches/bench_intopyobject.rs index 42af893cd8a..0e1fbad1a57 100644 --- a/pyo3-benches/benches/bench_intopyobject.rs +++ b/pyo3-benches/benches/bench_intopyobject.rs @@ -46,15 +46,6 @@ fn byte_slice_into_pyobject_large(b: &mut Bencher<'_>) { bench_bytes_into_pyobject(b, &data); } -#[allow(deprecated)] -fn byte_slice_into_py(b: &mut Bencher<'_>) { - Python::with_gil(|py| { - let data = (0..u8::MAX).collect::>(); - let bytes = data.as_slice(); - b.iter_with_large_drop(|| black_box(bytes).into_py(py)); - }); -} - fn vec_into_pyobject(b: &mut Bencher<'_>) { Python::with_gil(|py| { let bytes = (0..u8::MAX).collect::>(); @@ -62,14 +53,6 @@ fn vec_into_pyobject(b: &mut Bencher<'_>) { }); } -#[allow(deprecated)] -fn vec_into_py(b: &mut Bencher<'_>) { - Python::with_gil(|py| { - let bytes = (0..u8::MAX).collect::>(); - b.iter_with_large_drop(|| black_box(&bytes).clone().into_py(py)); - }); -} - fn criterion_benchmark(c: &mut Criterion) { c.bench_function("bytes_new_small", bytes_new_small); c.bench_function("bytes_new_medium", bytes_new_medium); @@ -86,9 +69,7 @@ fn criterion_benchmark(c: &mut Criterion) { "byte_slice_into_pyobject_large", byte_slice_into_pyobject_large, ); - c.bench_function("byte_slice_into_py", byte_slice_into_py); c.bench_function("vec_into_pyobject", vec_into_pyobject); - c.bench_function("vec_into_py", vec_into_py); } criterion_group!(benches, criterion_benchmark); diff --git a/pyo3-benches/benches/bench_list.rs b/pyo3-benches/benches/bench_list.rs index 7a19452455e..f6df0cc4315 100644 --- a/pyo3-benches/benches/bench_list.rs +++ b/pyo3-benches/benches/bench_list.rs @@ -42,7 +42,7 @@ fn list_get_item(b: &mut Bencher<'_>) { fn list_nth(b: &mut Bencher<'_>) { Python::with_gil(|py| { const LEN: usize = 50; - let list = PyList::new_bound(py, 0..LEN); + let list = PyList::new(py, 0..LEN).unwrap(); let mut sum = 0; b.iter(|| { for i in 0..LEN { @@ -55,7 +55,7 @@ fn list_nth(b: &mut Bencher<'_>) { fn list_nth_back(b: &mut Bencher<'_>) { Python::with_gil(|py| { const LEN: usize = 50; - let list = PyList::new_bound(py, 0..LEN); + let list = PyList::new(py, 0..LEN).unwrap(); let mut sum = 0; b.iter(|| { for i in 0..LEN { diff --git a/pyo3-benches/benches/bench_tuple.rs b/pyo3-benches/benches/bench_tuple.rs index e235567e926..cba7573c9d2 100644 --- a/pyo3-benches/benches/bench_tuple.rs +++ b/pyo3-benches/benches/bench_tuple.rs @@ -115,13 +115,6 @@ fn tuple_to_list(b: &mut Bencher<'_>) { }); } -#[allow(deprecated)] -fn tuple_into_py(b: &mut Bencher<'_>) { - Python::with_gil(|py| { - b.iter(|| -> PyObject { (1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12).into_py(py) }); - }); -} - fn tuple_into_pyobject(b: &mut Bencher<'_>) { Python::with_gil(|py| { b.iter(|| { @@ -175,7 +168,6 @@ fn criterion_benchmark(c: &mut Criterion) { c.bench_function("sequence_from_tuple", sequence_from_tuple); c.bench_function("tuple_new_list", tuple_new_list); c.bench_function("tuple_to_list", tuple_to_list); - c.bench_function("tuple_into_py", tuple_into_py); c.bench_function("tuple_into_pyobject", tuple_into_pyobject); } diff --git a/pyo3-build-config/Cargo.toml b/pyo3-build-config/Cargo.toml index b8030cc4304..8070860a1e9 100644 --- a/pyo3-build-config/Cargo.toml +++ b/pyo3-build-config/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3-build-config" -version = "0.24.0" +version = "0.25.0" description = "Build configuration for the PyO3 ecosystem" authors = ["PyO3 Project and Contributors "] keywords = ["pyo3", "python", "cpython", "ffi"] @@ -38,7 +38,8 @@ abi3-py39 = ["abi3-py310"] abi3-py310 = ["abi3-py311"] abi3-py311 = ["abi3-py312"] abi3-py312 = ["abi3-py313"] -abi3-py313 = ["abi3"] +abi3-py313 = ["abi3-py314"] +abi3-py314 = ["abi3"] [package.metadata.docs.rs] features = ["resolve-config"] diff --git a/pyo3-build-config/src/errors.rs b/pyo3-build-config/src/errors.rs index 87c59a998b4..b11d02fd581 100644 --- a/pyo3-build-config/src/errors.rs +++ b/pyo3-build-config/src/errors.rs @@ -68,7 +68,7 @@ impl std::fmt::Display for ErrorReport<'_> { writeln!(f, "\ncaused by:")?; let mut index = 0; while let Some(some_source) = source { - writeln!(f, " - {}: {}", index, some_source)?; + writeln!(f, " - {index}: {some_source}")?; source = some_source.source(); index += 1; } diff --git a/pyo3-build-config/src/impl_.rs b/pyo3-build-config/src/impl_.rs index 2c4955dcc6f..9485b8a0394 100644 --- a/pyo3-build-config/src/impl_.rs +++ b/pyo3-build-config/src/impl_.rs @@ -58,7 +58,7 @@ pub fn cargo_env_var(var: &str) -> Option { /// the variable changes. pub fn env_var(var: &str) -> Option { if cfg!(feature = "resolve-config") { - println!("cargo:rerun-if-env-changed={}", var); + println!("cargo:rerun-if-env-changed={var}"); } #[cfg(test)] { @@ -180,7 +180,7 @@ impl InterpreterConfig { let mut out = vec![]; for i in MINIMUM_SUPPORTED_VERSION.minor..=self.version.minor { - out.push(format!("cargo:rustc-cfg=Py_3_{}", i)); + out.push(format!("cargo:rustc-cfg=Py_3_{i}")); } match self.implementation { @@ -199,7 +199,7 @@ impl InterpreterConfig { BuildFlag::Py_GIL_DISABLED => { out.push("cargo:rustc-cfg=Py_GIL_DISABLED".to_owned()) } - flag => out.push(format!("cargo:rustc-cfg=py_sys_config=\"{}\"", flag)), + flag => out.push(format!("cargo:rustc-cfg=py_sys_config=\"{flag}\"")), } } @@ -338,7 +338,7 @@ print("gil_disabled", get_config_var("Py_GIL_DISABLED")) let lib_dir = if cfg!(windows) { map.get("base_prefix") - .map(|base_prefix| format!("{}\\libs", base_prefix)) + .map(|base_prefix| format!("{base_prefix}\\libs")) } else { map.get("libdir").cloned() }; @@ -666,7 +666,7 @@ print("gil_disabled", get_config_var("Py_GIL_DISABLED")) write_option_line!(python_framework_prefix)?; write_line!(suppress_build_script_link_lines)?; for line in &self.extra_build_script_lines { - writeln!(writer, "extra_build_script_line={}", line) + writeln!(writer, "extra_build_script_line={line}") .context("failed to write extra_build_script_line")?; } Ok(()) @@ -853,7 +853,7 @@ fn is_abi3() -> bool { /// Must be called from a PyO3 crate build script. pub fn get_abi3_version() -> Option { let minor_version = (MINIMUM_SUPPORTED_VERSION.minor..=ABI3_MAX_MINOR) - .find(|i| cargo_env_var(&format!("CARGO_FEATURE_ABI3_PY3{}", i)).is_some()); + .find(|i| cargo_env_var(&format!("CARGO_FEATURE_ABI3_PY3{i}")).is_some()); minor_version.map(|minor| PythonVersion { major: 3, minor }) } @@ -1121,8 +1121,8 @@ pub enum BuildFlag { impl Display for BuildFlag { fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { match self { - BuildFlag::Other(flag) => write!(f, "{}", flag), - _ => write!(f, "{:?}", self), + BuildFlag::Other(flag) => write!(f, "{flag}"), + _ => write!(f, "{self:?}"), } } } @@ -1146,7 +1146,7 @@ impl FromStr for BuildFlag { /// PyO3 will pick these up and pass to rustc via `--cfg=py_sys_config={varname}`; /// this allows using them conditional cfg attributes in the .rs files, so /// -/// ```rust +/// ```rust,no_run /// #[cfg(py_sys_config="{varname}")] /// # struct Foo; /// ``` @@ -1202,7 +1202,7 @@ impl BuildFlags { for k in &BuildFlags::ALL { use std::fmt::Write; - writeln!(&mut script, "print(config.get('{}', '0'))", k).unwrap(); + writeln!(&mut script, "print(config.get('{k}', '0'))").unwrap(); } let stdout = run_python_script(interpreter.as_ref(), &script)?; @@ -1240,7 +1240,7 @@ impl Display for BuildFlags { } else { write!(f, ",")?; } - write!(f, "{}", flag)?; + write!(f, "{flag}")?; } Ok(()) } @@ -1306,6 +1306,10 @@ pub fn parse_sysconfigdata(sysconfigdata_path: impl AsRef) -> Result Result fn is_pypy_lib_dir(path: &str, v: &Option) -> bool { let pypy_version_pat = if let Some(v) = v { - format!("pypy{}", v) + format!("pypy{v}") } else { "pypy3.".into() }; @@ -1432,7 +1436,7 @@ fn is_pypy_lib_dir(path: &str, v: &Option) -> bool { fn is_graalpy_lib_dir(path: &str, v: &Option) -> bool { let graalpy_version_pat = if let Some(v) = v { - format!("graalpy{}", v) + format!("graalpy{v}") } else { "graalpy2".into() }; @@ -1441,7 +1445,7 @@ fn is_graalpy_lib_dir(path: &str, v: &Option) -> bool { fn is_cpython_lib_dir(path: &str, v: &Option) -> bool { let cpython_version_pat = if let Some(v) = v { - format!("python{}", v) + format!("python{v}") } else { "python3.".into() }; @@ -1755,7 +1759,7 @@ fn default_lib_name_unix( ) -> Result { match implementation { PythonImplementation::CPython => match ld_version { - Some(ld_version) => Ok(format!("python{}", ld_version)), + Some(ld_version) => Ok(format!("python{ld_version}")), None => { if version > PythonVersion::PY37 { // PEP 3149 ABI version tags are finally gone @@ -1772,7 +1776,7 @@ fn default_lib_name_unix( } }, PythonImplementation::PyPy => match ld_version { - Some(ld_version) => Ok(format!("pypy{}-c", ld_version)), + Some(ld_version) => Ok(format!("pypy{ld_version}-c")), None => Ok(format!("pypy{}.{}-c", version.major, version.minor)), }, diff --git a/pyo3-build-config/src/lib.rs b/pyo3-build-config/src/lib.rs index 68d597a8f2e..f47c16f425d 100644 --- a/pyo3-build-config/src/lib.rs +++ b/pyo3-build-config/src/lib.rs @@ -46,7 +46,7 @@ use target_lexicon::OperatingSystem; pub fn use_pyo3_cfgs() { print_expected_cfgs(); for cargo_command in get().build_script_outputs() { - println!("{}", cargo_command) + println!("{cargo_command}") } } @@ -102,12 +102,7 @@ fn _add_python_framework_link_args( ) { if matches!(triple.operating_system, OperatingSystem::Darwin(_)) && link_libpython { if let Some(framework_prefix) = interpreter_config.python_framework_prefix.as_ref() { - writeln!( - writer, - "cargo:rustc-link-arg=-Wl,-rpath,{}", - framework_prefix - ) - .unwrap(); + writeln!(writer, "cargo:rustc-link-arg=-Wl,-rpath,{framework_prefix}").unwrap(); } } } @@ -173,12 +168,12 @@ fn print_feature_cfg(minor_version_required: u32, cfg: &str) { let minor_version = rustc_minor_version().unwrap_or(0); if minor_version >= minor_version_required { - println!("cargo:rustc-cfg={}", cfg); + println!("cargo:rustc-cfg={cfg}"); } // rustc 1.80.0 stabilized `rustc-check-cfg` feature, don't emit before if minor_version >= 80 { - println!("cargo:rustc-check-cfg=cfg({})", cfg); + println!("cargo:rustc-check-cfg=cfg({cfg})"); } } @@ -221,8 +216,7 @@ pub fn print_expected_cfgs() { // allow `Py_3_*` cfgs from the minimum supported version up to the // maximum minor version (+1 for development for the next) - // FIXME: support cfg(Py_3_14) as well due to PyGILState_Ensure - for i in impl_::MINIMUM_SUPPORTED_VERSION.minor..=std::cmp::max(14, impl_::ABI3_MAX_MINOR + 1) { + for i in impl_::MINIMUM_SUPPORTED_VERSION.minor..=impl_::ABI3_MAX_MINOR + 1 { println!("cargo:rustc-check-cfg=cfg(Py_3_{i})"); } } diff --git a/pyo3-ffi-check/build.rs b/pyo3-ffi-check/build.rs index 67808888e1e..e7cfbe40df3 100644 --- a/pyo3-ffi-check/build.rs +++ b/pyo3-ffi-check/build.rs @@ -1,6 +1,24 @@ use std::env; use std::path::PathBuf; +#[derive(Debug)] +struct ParseCallbacks; + +impl bindgen::callbacks::ParseCallbacks for ParseCallbacks { + // these are anonymous fields and structs in CPython that we needed to + // invent names for. Bindgen seems to generate stable names, so we remap the + // automatically generated names to the names we invented in the FFI + fn item_name(&self, _original_item_name: &str) -> Option { + if _original_item_name == "_object__bindgen_ty_1__bindgen_ty_1" { + Some("PyObjectObFlagsAndRefcnt".into()) + } else if _original_item_name == "_object__bindgen_ty_1" { + Some("PyObjectObRefcnt".into()) + } else { + None + } + } +} + fn main() { let config = pyo3_build_config::get(); let python_include_dir = config @@ -29,6 +47,7 @@ fn main() { .header("wrapper.h") .clang_args(clang_args) .parse_callbacks(Box::new(bindgen::CargoCallbacks::new())) + .parse_callbacks(Box::new(ParseCallbacks)) // blocklist some values which apparently have conflicting definitions on unix .blocklist_item("FP_NORMAL") .blocklist_item("FP_SUBNORMAL") diff --git a/pyo3-ffi-check/macro/src/lib.rs b/pyo3-ffi-check/macro/src/lib.rs index 8438393b9eb..41092b9020e 100644 --- a/pyo3-ffi-check/macro/src/lib.rs +++ b/pyo3-ffi-check/macro/src/lib.rs @@ -9,11 +9,6 @@ const PY_3_12: PythonVersion = PythonVersion { minor: 12, }; -const PY_3_13: PythonVersion = PythonVersion { - major: 3, - minor: 13, -}; - /// Macro which expands to multiple macro calls, one per pyo3-ffi struct. #[proc_macro] pub fn for_all_structs(input: proc_macro::TokenStream) -> proc_macro::TokenStream { @@ -55,13 +50,6 @@ pub fn for_all_structs(input: proc_macro::TokenStream) -> proc_macro::TokenStrea .strip_suffix(".html") .unwrap(); - if struct_name == "PyConfig" && pyo3_build_config::get().version == PY_3_13 { - // https://github.com/python/cpython/issues/130940 - // PyConfig has an ABI break on Python 3.13.1 -> 3.13.2, waiting for advice - // how to proceed in PyO3. - continue; - } - let struct_ident = Ident::new(struct_name, Span::call_site()); output.extend(quote!(#macro_name!(#struct_ident);)); } diff --git a/pyo3-ffi/Cargo.toml b/pyo3-ffi/Cargo.toml index 16c8a3374cd..647b76fbdcd 100644 --- a/pyo3-ffi/Cargo.toml +++ b/pyo3-ffi/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3-ffi" -version = "0.24.0" +version = "0.25.0" description = "Python-API bindings for the PyO3 ecosystem" authors = ["PyO3 Project and Contributors "] keywords = ["pyo3", "python", "cpython", "ffi"] @@ -34,7 +34,8 @@ abi3-py39 = ["abi3-py310", "pyo3-build-config/abi3-py39"] abi3-py310 = ["abi3-py311", "pyo3-build-config/abi3-py310"] abi3-py311 = ["abi3-py312", "pyo3-build-config/abi3-py311"] abi3-py312 = ["abi3-py313", "pyo3-build-config/abi3-py312"] -abi3-py313 = ["abi3", "pyo3-build-config/abi3-py313"] +abi3-py313 = ["abi3-py314", "pyo3-build-config/abi3-py313"] +abi3-py314 = ["abi3", "pyo3-build-config/abi3-py314"] # Automatically generates `python3.dll` import libraries for Windows targets. generate-import-lib = ["pyo3-build-config/python3-dll-a"] @@ -43,14 +44,14 @@ generate-import-lib = ["pyo3-build-config/python3-dll-a"] paste = "1" [build-dependencies] -pyo3-build-config = { path = "../pyo3-build-config", version = "=0.24.0", features = ["resolve-config"] } +pyo3-build-config = { path = "../pyo3-build-config", version = "=0.25.0", features = ["resolve-config"] } [lints] workspace = true [package.metadata.cpython] min-version = "3.7" -max-version = "3.13" # inclusive +max-version = "3.14" # inclusive [package.metadata.pypy] min-version = "3.9" diff --git a/pyo3-ffi/README.md b/pyo3-ffi/README.md index 3fada2ffab6..09be0a06f66 100644 --- a/pyo3-ffi/README.md +++ b/pyo3-ffi/README.md @@ -41,13 +41,13 @@ name = "string_sum" crate-type = ["cdylib"] [dependencies.pyo3-ffi] -version = "0.24.0" +version = "0.25.0" features = ["extension-module"] [build-dependencies] # This is only necessary if you need to configure your build based on # the Python version or the compile-time configuration for the interpreter. -pyo3_build_config = "0.24.0" +pyo3_build_config = "0.25.0" ``` If you need to use conditional compilation based on Python version or how @@ -65,7 +65,7 @@ fn main() { ``` **`src/lib.rs`** -```rust +```rust,no_run use std::os::raw::{c_char, c_long}; use std::ptr; diff --git a/pyo3-ffi/build.rs b/pyo3-ffi/build.rs index 096614c7961..6776cd80476 100644 --- a/pyo3-ffi/build.rs +++ b/pyo3-ffi/build.rs @@ -17,7 +17,7 @@ const SUPPORTED_VERSIONS_CPYTHON: SupportedVersions = SupportedVersions { min: PythonVersion { major: 3, minor: 7 }, max: PythonVersion { major: 3, - minor: 13, + minor: 14, }, }; @@ -176,7 +176,7 @@ fn emit_link_config(interpreter_config: &InterpreterConfig) -> Result<()> { ); if let Some(lib_dir) = &interpreter_config.lib_dir { - println!("cargo:rustc-link-search=native={}", lib_dir); + println!("cargo:rustc-link-search=native={lib_dir}"); } Ok(()) @@ -207,12 +207,12 @@ fn configure_pyo3() -> Result<()> { } for cfg in interpreter_config.build_script_outputs() { - println!("{}", cfg) + println!("{cfg}") } // Extra lines come last, to support last write wins. for line in &interpreter_config.extra_build_script_lines { - println!("{}", line); + println!("{line}"); } // Emit cfgs like `invalid_from_utf8_lint` diff --git a/pyo3-ffi/src/abstract_.rs b/pyo3-ffi/src/abstract_.rs index a79ec43f271..84fb98a1b4e 100644 --- a/pyo3-ffi/src/abstract_.rs +++ b/pyo3-ffi/src/abstract_.rs @@ -25,6 +25,7 @@ pub unsafe fn PyObject_DelAttr(o: *mut PyObject, attr_name: *mut PyObject) -> c_ extern "C" { #[cfg(all( not(PyPy), + not(GraalPy), any(Py_3_10, all(not(Py_LIMITED_API), Py_3_9)) // Added to python in 3.9 but to limited API in 3.10 ))] #[cfg_attr(PyPy, link_name = "PyPyObject_CallNoArgs")] diff --git a/pyo3-ffi/src/bytearrayobject.rs b/pyo3-ffi/src/bytearrayobject.rs index 24a97bcc31b..d27dfa8b0ec 100644 --- a/pyo3-ffi/src/bytearrayobject.rs +++ b/pyo3-ffi/src/bytearrayobject.rs @@ -17,7 +17,7 @@ pub struct PyByteArrayObject { } #[cfg(any(PyPy, GraalPy, Py_LIMITED_API))] -opaque_struct!(PyByteArrayObject); +opaque_struct!(pub PyByteArrayObject); #[cfg_attr(windows, link(name = "pythonXY"))] extern "C" { diff --git a/pyo3-ffi/src/code.rs b/pyo3-ffi/src/code.rs index d28f68cded7..296b17f6aa4 100644 --- a/pyo3-ffi/src/code.rs +++ b/pyo3-ffi/src/code.rs @@ -1,4 +1,4 @@ // This header doesn't exist in CPython, but Include/cpython/code.h does. We add // this here so that PyCodeObject has a definition under the limited API. -opaque_struct!(PyCodeObject); +opaque_struct!(pub PyCodeObject); diff --git a/pyo3-ffi/src/compat/mod.rs b/pyo3-ffi/src/compat/mod.rs index 11f2912848e..044ea46762b 100644 --- a/pyo3-ffi/src/compat/mod.rs +++ b/pyo3-ffi/src/compat/mod.rs @@ -52,8 +52,10 @@ macro_rules! compat_function { mod py_3_10; mod py_3_13; +mod py_3_14; mod py_3_9; pub use self::py_3_10::*; pub use self::py_3_13::*; +pub use self::py_3_14::*; pub use self::py_3_9::*; diff --git a/pyo3-ffi/src/compat/py_3_10.rs b/pyo3-ffi/src/compat/py_3_10.rs index c6e8c2cb5ca..d962fb3bc7e 100644 --- a/pyo3-ffi/src/compat/py_3_10.rs +++ b/pyo3-ffi/src/compat/py_3_10.rs @@ -17,3 +17,29 @@ compat_function!( obj } ); + +compat_function!( + originally_defined_for(Py_3_10); + + #[inline] + pub unsafe fn PyModule_AddObjectRef( + module: *mut crate::PyObject, + name: *const std::os::raw::c_char, + value: *mut crate::PyObject, + ) -> std::os::raw::c_int { + if value.is_null() && crate::PyErr_Occurred().is_null() { + crate::PyErr_SetString( + crate::PyExc_SystemError, + c_str!("PyModule_AddObjectRef() must be called with an exception raised if value is NULL").as_ptr(), + ); + return -1; + } + + crate::Py_XINCREF(value); + let result = crate::PyModule_AddObject(module, name, value); + if result < 0 { + crate::Py_XDECREF(value); + } + result + } +); diff --git a/pyo3-ffi/src/compat/py_3_13.rs b/pyo3-ffi/src/compat/py_3_13.rs index 59289cb76ae..96c90e7eb66 100644 --- a/pyo3-ffi/src/compat/py_3_13.rs +++ b/pyo3-ffi/src/compat/py_3_13.rs @@ -104,3 +104,18 @@ compat_function!( crate::PyList_SetSlice(list, 0, crate::PY_SSIZE_T_MAX, std::ptr::null_mut()) } ); + +compat_function!( + originally_defined_for(Py_3_13); + + #[inline] + pub unsafe fn PyModule_Add( + module: *mut crate::PyObject, + name: *const std::os::raw::c_char, + value: *mut crate::PyObject, + ) -> std::os::raw::c_int { + let result = crate::compat::PyModule_AddObjectRef(module, name, value); + crate::Py_XDECREF(value); + result + } +); diff --git a/pyo3-ffi/src/compat/py_3_14.rs b/pyo3-ffi/src/compat/py_3_14.rs new file mode 100644 index 00000000000..6fdaef17488 --- /dev/null +++ b/pyo3-ffi/src/compat/py_3_14.rs @@ -0,0 +1,26 @@ +compat_function!( + originally_defined_for(all(Py_3_14, not(Py_LIMITED_API))); + + #[inline] + pub unsafe fn Py_HashBuffer( + ptr: *const std::ffi::c_void, + len: crate::Py_ssize_t, + ) -> crate::Py_hash_t { + #[cfg(not(any(Py_LIMITED_API, PyPy)))] + { + crate::_Py_HashBytes(ptr, len) + } + + #[cfg(any(Py_LIMITED_API, PyPy))] + { + let bytes = crate::PyBytes_FromStringAndSize(ptr as *const std::os::raw::c_char, len); + if bytes.is_null() { + -1 + } else { + let result = crate::PyObject_Hash(bytes); + crate::Py_DECREF(bytes); + result + } + } + } +); diff --git a/pyo3-ffi/src/cpython/bytesobject.rs b/pyo3-ffi/src/cpython/bytesobject.rs index 306702de25e..dd78e646a12 100644 --- a/pyo3-ffi/src/cpython/bytesobject.rs +++ b/pyo3-ffi/src/cpython/bytesobject.rs @@ -17,7 +17,7 @@ pub struct PyBytesObject { } #[cfg(any(PyPy, GraalPy, Py_LIMITED_API))] -opaque_struct!(PyBytesObject); +opaque_struct!(pub PyBytesObject); extern "C" { #[cfg_attr(PyPy, link_name = "_PyPyBytes_Resize")] diff --git a/pyo3-ffi/src/cpython/code.rs b/pyo3-ffi/src/cpython/code.rs index 230096ca378..3d47a1bc8c3 100644 --- a/pyo3-ffi/src/cpython/code.rs +++ b/pyo3-ffi/src/cpython/code.rs @@ -1,207 +1,43 @@ use crate::object::*; use crate::pyport::Py_ssize_t; -#[allow(unused_imports)] -use std::os::raw::{c_char, c_int, c_short, c_uchar, c_void}; +#[cfg(not(GraalPy))] +use std::os::raw::c_char; +use std::os::raw::{c_int, c_void}; #[cfg(not(any(PyPy, GraalPy)))] use std::ptr::addr_of_mut; -#[cfg(all(Py_3_8, not(any(PyPy, GraalPy)), not(Py_3_11)))] -opaque_struct!(_PyOpcache); +// skipped private _PY_MONITORING_LOCAL_EVENTS +// skipped private _PY_MONITORING_UNGROUPED_EVENTS +// skipped private _PY_MONITORING_EVENTS -#[cfg(Py_3_12)] -pub const _PY_MONITORING_LOCAL_EVENTS: usize = 10; -#[cfg(Py_3_12)] -pub const _PY_MONITORING_UNGROUPED_EVENTS: usize = 15; -#[cfg(Py_3_12)] -pub const _PY_MONITORING_EVENTS: usize = 17; +// skipped private _PyLocalMonitors +// skipped private _Py_GlobalMonitors -#[cfg(Py_3_12)] -#[repr(C)] -#[derive(Clone, Copy)] -pub struct _Py_LocalMonitors { - pub tools: [u8; if cfg!(Py_3_13) { - _PY_MONITORING_LOCAL_EVENTS - } else { - _PY_MONITORING_UNGROUPED_EVENTS - }], -} - -#[cfg(Py_3_12)] -#[repr(C)] -#[derive(Clone, Copy)] -pub struct _Py_GlobalMonitors { - pub tools: [u8; _PY_MONITORING_UNGROUPED_EVENTS], -} - -// skipped _Py_CODEUNIT +// skipped private _Py_CODEUNIT -// skipped _Py_OPCODE -// skipped _Py_OPARG +// skipped private _Py_OPCODE +// skipped private _Py_OPARG -// skipped _py_make_codeunit +// skipped private _py_make_codeunit -// skipped _py_set_opcode +// skipped private _py_set_opcode -// skipped _Py_MAKE_CODEUNIT -// skipped _Py_SET_OPCODE - -#[cfg(Py_3_12)] -#[repr(C)] -#[derive(Copy, Clone)] -pub struct _PyCoCached { - pub _co_code: *mut PyObject, - pub _co_varnames: *mut PyObject, - pub _co_cellvars: *mut PyObject, - pub _co_freevars: *mut PyObject, -} - -#[cfg(Py_3_12)] -#[repr(C)] -#[derive(Copy, Clone)] -pub struct _PyCoLineInstrumentationData { - pub original_opcode: u8, - pub line_delta: i8, -} +// skipped private _Py_MAKE_CODEUNIT +// skipped private _Py_SET_OPCODE -#[cfg(Py_3_12)] -#[repr(C)] -#[derive(Copy, Clone)] -pub struct _PyCoMonitoringData { - pub local_monitors: _Py_LocalMonitors, - pub active_monitors: _Py_LocalMonitors, - pub tools: *mut u8, - pub lines: *mut _PyCoLineInstrumentationData, - pub line_tools: *mut u8, - pub per_instruction_opcodes: *mut u8, - pub per_instruction_tools: *mut u8, -} - -#[cfg(all(not(any(PyPy, GraalPy)), not(Py_3_7)))] -opaque_struct!(PyCodeObject); - -#[cfg(all(not(any(PyPy, GraalPy)), Py_3_7, not(Py_3_8)))] -#[repr(C)] -pub struct PyCodeObject { - pub ob_base: PyObject, - pub co_argcount: c_int, - pub co_kwonlyargcount: c_int, - pub co_nlocals: c_int, - pub co_stacksize: c_int, - pub co_flags: c_int, - pub co_firstlineno: c_int, - pub co_code: *mut PyObject, - pub co_consts: *mut PyObject, - pub co_names: *mut PyObject, - pub co_varnames: *mut PyObject, - pub co_freevars: *mut PyObject, - pub co_cellvars: *mut PyObject, - pub co_cell2arg: *mut Py_ssize_t, - pub co_filename: *mut PyObject, - pub co_name: *mut PyObject, - pub co_lnotab: *mut PyObject, - pub co_zombieframe: *mut c_void, - pub co_weakreflist: *mut PyObject, - pub co_extra: *mut c_void, -} - -#[cfg(Py_3_13)] -opaque_struct!(_PyExecutorArray); - -#[cfg(all(not(any(PyPy, GraalPy)), Py_3_8, not(Py_3_11)))] -#[repr(C)] -pub struct PyCodeObject { - pub ob_base: PyObject, - pub co_argcount: c_int, - pub co_posonlyargcount: c_int, - pub co_kwonlyargcount: c_int, - pub co_nlocals: c_int, - pub co_stacksize: c_int, - pub co_flags: c_int, - pub co_firstlineno: c_int, - pub co_code: *mut PyObject, - pub co_consts: *mut PyObject, - pub co_names: *mut PyObject, - pub co_varnames: *mut PyObject, - pub co_freevars: *mut PyObject, - pub co_cellvars: *mut PyObject, - pub co_cell2arg: *mut Py_ssize_t, - pub co_filename: *mut PyObject, - pub co_name: *mut PyObject, - #[cfg(not(Py_3_10))] - pub co_lnotab: *mut PyObject, - #[cfg(Py_3_10)] - pub co_linetable: *mut PyObject, - pub co_zombieframe: *mut c_void, - pub co_weakreflist: *mut PyObject, - pub co_extra: *mut c_void, - pub co_opcache_map: *mut c_uchar, - pub co_opcache: *mut _PyOpcache, - pub co_opcache_flag: c_int, - pub co_opcache_size: c_uchar, -} +// skipped private _PyCoCached +// skipped private _PyCoLineInstrumentationData +// skipped private _PyCoMontoringData -#[cfg(all(not(any(PyPy, GraalPy)), Py_3_11))] -#[repr(C)] -pub struct PyCodeObject { - pub ob_base: PyVarObject, - pub co_consts: *mut PyObject, - pub co_names: *mut PyObject, - pub co_exceptiontable: *mut PyObject, - pub co_flags: c_int, - #[cfg(not(Py_3_12))] - pub co_warmup: c_int, +// skipped private _PyExecutorArray - pub co_argcount: c_int, - pub co_posonlyargcount: c_int, - pub co_kwonlyargcount: c_int, - pub co_stacksize: c_int, - pub co_firstlineno: c_int, - - pub co_nlocalsplus: c_int, - #[cfg(Py_3_12)] - pub co_framesize: c_int, - pub co_nlocals: c_int, - #[cfg(not(Py_3_12))] - pub co_nplaincellvars: c_int, - pub co_ncellvars: c_int, - pub co_nfreevars: c_int, - #[cfg(Py_3_12)] - pub co_version: u32, - - pub co_localsplusnames: *mut PyObject, - pub co_localspluskinds: *mut PyObject, - pub co_filename: *mut PyObject, - pub co_name: *mut PyObject, - pub co_qualname: *mut PyObject, - pub co_linetable: *mut PyObject, - pub co_weakreflist: *mut PyObject, - #[cfg(not(Py_3_12))] - pub _co_code: *mut PyObject, - #[cfg(not(Py_3_12))] - pub _co_linearray: *mut c_char, - #[cfg(Py_3_13)] - pub co_executors: *mut _PyExecutorArray, - #[cfg(Py_3_12)] - pub _co_cached: *mut _PyCoCached, - #[cfg(Py_3_12)] - pub _co_instrumentation_version: u64, - #[cfg(Py_3_12)] - pub _co_monitoring: *mut _PyCoMonitoringData, - pub _co_firsttraceable: c_int, - pub co_extra: *mut c_void, - pub co_code_adaptive: [c_char; 1], -} - -#[cfg(PyPy)] -#[repr(C)] -pub struct PyCodeObject { - pub ob_base: PyObject, - pub co_name: *mut PyObject, - pub co_filename: *mut PyObject, - pub co_argcount: c_int, - pub co_flags: c_int, -} +opaque_struct!( + #[doc = "A Python code object.\n"] + #[doc = "\n"] + #[doc = "`pyo3-ffi` does not expose the contents of this struct, as it has no stability guarantees."] + pub PyCodeObject +); /* Masks for co_flags */ pub const CO_OPTIMIZED: c_int = 0x0001; @@ -247,28 +83,14 @@ pub unsafe fn PyCode_Check(op: *mut PyObject) -> c_int { (Py_TYPE(op) == addr_of_mut!(PyCode_Type)) as c_int } -#[inline] -#[cfg(all(not(any(PyPy, GraalPy)), Py_3_10, not(Py_3_11)))] -pub unsafe fn PyCode_GetNumFree(op: *mut PyCodeObject) -> Py_ssize_t { - crate::PyTuple_GET_SIZE((*op).co_freevars) -} - -#[inline] -#[cfg(all(not(Py_3_10), Py_3_11, not(any(PyPy, GraalPy))))] -pub unsafe fn PyCode_GetNumFree(op: *mut PyCodeObject) -> c_int { - (*op).co_nfreevars -} - extern "C" { #[cfg(PyPy)] #[link_name = "PyPyCode_Check"] pub fn PyCode_Check(op: *mut PyObject) -> c_int; - - #[cfg(PyPy)] - #[link_name = "PyPyCode_GetNumFree"] - pub fn PyCode_GetNumFree(op: *mut PyCodeObject) -> Py_ssize_t; } +// skipped PyCode_GetNumFree (requires knowledge of code object layout) + extern "C" { #[cfg(not(GraalPy))] #[cfg_attr(PyPy, link_name = "PyPyCode_New")] diff --git a/pyo3-ffi/src/cpython/compile.rs b/pyo3-ffi/src/cpython/compile.rs index 79f06c92003..078b5e0d675 100644 --- a/pyo3-ffi/src/cpython/compile.rs +++ b/pyo3-ffi/src/cpython/compile.rs @@ -6,19 +6,21 @@ use crate::pyarena::*; use crate::pythonrun::*; #[cfg(not(any(PyPy, Py_3_10)))] use crate::PyCodeObject; +use crate::INT_MAX; #[cfg(not(any(PyPy, Py_3_10)))] use std::os::raw::c_char; use std::os::raw::c_int; -// skipped non-limited PyCF_MASK -// skipped non-limited PyCF_MASK_OBSOLETE -// skipped non-limited PyCF_SOURCE_IS_UTF8 -// skipped non-limited PyCF_DONT_IMPLY_DEDENT -// skipped non-limited PyCF_ONLY_AST -// skipped non-limited PyCF_IGNORE_COOKIE -// skipped non-limited PyCF_TYPE_COMMENTS -// skipped non-limited PyCF_ALLOW_TOP_LEVEL_AWAIT -// skipped non-limited PyCF_COMPILE_MASK +// skipped PyCF_MASK +// skipped PyCF_MASK_OBSOLETE +// skipped PyCF_SOURCE_IS_UTF8 +// skipped PyCF_DONT_IMPLY_DEDENT +// skipped PyCF_ONLY_AST +// skipped PyCF_IGNORE_COOKIE +// skipped PyCF_TYPE_COMMENTS +// skipped PyCF_ALLOW_TOP_LEVEL_AWAIT +// skipped PyCF_OPTIMIZED_AST +// skipped PyCF_COMPILE_MASK #[repr(C)] #[derive(Copy, Clone)] @@ -28,31 +30,23 @@ pub struct PyCompilerFlags { pub cf_feature_version: c_int, } -// skipped non-limited _PyCompilerFlags_INIT +// skipped _PyCompilerFlags_INIT -#[cfg(all(Py_3_12, not(any(Py_3_13, PyPy, GraalPy))))] -#[repr(C)] -#[derive(Copy, Clone)] -pub struct _PyCompilerSrcLocation { - pub lineno: c_int, - pub end_lineno: c_int, - pub col_offset: c_int, - pub end_col_offset: c_int, -} - -// skipped SRC_LOCATION_FROM_AST - -#[cfg(not(any(PyPy, GraalPy, Py_3_13)))] +// NB this type technically existed in the header until 3.13, when it was +// moved to the internal CPython headers. +// +// We choose not to expose it in the public API past 3.10, as it is +// not used in the public API past that point. +#[cfg(not(any(PyPy, GraalPy, Py_3_10)))] #[repr(C)] #[derive(Copy, Clone)] pub struct PyFutureFeatures { pub ff_features: c_int, - #[cfg(not(Py_3_12))] pub ff_lineno: c_int, - #[cfg(Py_3_12)] - pub ff_location: _PyCompilerSrcLocation, } +// FIXME: these constants should probably be &CStr, if they are used at all + pub const FUTURE_NESTED_SCOPES: &str = "nested_scopes"; pub const FUTURE_GENERATORS: &str = "generators"; pub const FUTURE_DIVISION: &str = "division"; @@ -62,13 +56,12 @@ pub const FUTURE_PRINT_FUNCTION: &str = "print_function"; pub const FUTURE_UNICODE_LITERALS: &str = "unicode_literals"; pub const FUTURE_BARRY_AS_BDFL: &str = "barry_as_FLUFL"; pub const FUTURE_GENERATOR_STOP: &str = "generator_stop"; -// skipped non-limited FUTURE_ANNOTATIONS +pub const FUTURE_ANNOTATIONS: &str = "annotations"; +#[cfg(not(any(PyPy, GraalPy, Py_3_10)))] extern "C" { - #[cfg(not(any(PyPy, Py_3_10)))] pub fn PyNode_Compile(arg1: *mut _node, arg2: *const c_char) -> *mut PyCodeObject; - #[cfg(not(any(PyPy, Py_3_10)))] pub fn PyAST_CompileEx( _mod: *mut _mod, filename: *const c_char, @@ -77,7 +70,6 @@ extern "C" { arena: *mut PyArena, ) -> *mut PyCodeObject; - #[cfg(not(any(PyPy, Py_3_10)))] pub fn PyAST_CompileObject( _mod: *mut _mod, filename: *mut PyObject, @@ -86,23 +78,20 @@ extern "C" { arena: *mut PyArena, ) -> *mut PyCodeObject; - #[cfg(not(any(PyPy, Py_3_10)))] pub fn PyFuture_FromAST(_mod: *mut _mod, filename: *const c_char) -> *mut PyFutureFeatures; - #[cfg(not(any(PyPy, Py_3_10)))] pub fn PyFuture_FromASTObject( _mod: *mut _mod, filename: *mut PyObject, ) -> *mut PyFutureFeatures; +} - // skipped non-limited _Py_Mangle - // skipped non-limited PY_INVALID_STACK_EFFECT +pub const PY_INVALID_STACK_EFFECT: c_int = INT_MAX; + +extern "C" { pub fn PyCompile_OpcodeStackEffect(opcode: c_int, oparg: c_int) -> c_int; #[cfg(Py_3_8)] pub fn PyCompile_OpcodeStackEffectWithJump(opcode: c_int, oparg: c_int, jump: c_int) -> c_int; - - // skipped non-limited _PyASTOptimizeState - // skipped non-limited _PyAST_Optimize } diff --git a/pyo3-ffi/src/cpython/critical_section.rs b/pyo3-ffi/src/cpython/critical_section.rs index 97b2f5e0559..808dba870c6 100644 --- a/pyo3-ffi/src/cpython/critical_section.rs +++ b/pyo3-ffi/src/cpython/critical_section.rs @@ -17,10 +17,10 @@ pub struct PyCriticalSection2 { } #[cfg(not(Py_GIL_DISABLED))] -opaque_struct!(PyCriticalSection); +opaque_struct!(pub PyCriticalSection); #[cfg(not(Py_GIL_DISABLED))] -opaque_struct!(PyCriticalSection2); +opaque_struct!(pub PyCriticalSection2); extern "C" { pub fn PyCriticalSection_Begin(c: *mut PyCriticalSection, op: *mut PyObject); diff --git a/pyo3-ffi/src/cpython/descrobject.rs b/pyo3-ffi/src/cpython/descrobject.rs index 1b5ee466c8e..7cef9bdbf42 100644 --- a/pyo3-ffi/src/cpython/descrobject.rs +++ b/pyo3-ffi/src/cpython/descrobject.rs @@ -69,10 +69,7 @@ pub struct PyWrapperDescrObject { pub d_wrapped: *mut c_void, } -#[cfg_attr(windows, link(name = "pythonXY"))] -extern "C" { - pub static mut _PyMethodWrapper_Type: PyTypeObject; -} +// skipped _PyMethodWrapper_Type // skipped non-limited PyDescr_NewWrapper // skipped non-limited PyDescr_IsData diff --git a/pyo3-ffi/src/cpython/dictobject.rs b/pyo3-ffi/src/cpython/dictobject.rs index 79dcbfdb62e..34b66a9699d 100644 --- a/pyo3-ffi/src/cpython/dictobject.rs +++ b/pyo3-ffi/src/cpython/dictobject.rs @@ -2,10 +2,10 @@ use crate::object::*; use crate::pyport::Py_ssize_t; use std::os::raw::c_int; -opaque_struct!(PyDictKeysObject); +opaque_struct!(pub PyDictKeysObject); #[cfg(Py_3_11)] -opaque_struct!(PyDictValues); +opaque_struct!(pub PyDictValues); #[cfg(not(GraalPy))] #[repr(C)] @@ -17,7 +17,10 @@ pub struct PyDictObject { Py_3_12, deprecated(note = "Deprecated in Python 3.12 and will be removed in the future.") )] + #[cfg(not(Py_3_14))] pub ma_version_tag: u64, + #[cfg(Py_3_14)] + _ma_watcher_tag: u64, pub ma_keys: *mut PyDictKeysObject, #[cfg(not(Py_3_11))] pub ma_values: *mut *mut PyObject, diff --git a/pyo3-ffi/src/cpython/frameobject.rs b/pyo3-ffi/src/cpython/frameobject.rs index e9b9c183f37..993e93c838b 100644 --- a/pyo3-ffi/src/cpython/frameobject.rs +++ b/pyo3-ffi/src/cpython/frameobject.rs @@ -53,7 +53,7 @@ pub struct PyFrameObject { } #[cfg(any(PyPy, GraalPy, Py_3_11))] -opaque_struct!(PyFrameObject); +opaque_struct!(pub PyFrameObject); // skipped _PyFrame_IsRunnable // skipped _PyFrame_IsExecuting diff --git a/pyo3-ffi/src/cpython/funcobject.rs b/pyo3-ffi/src/cpython/funcobject.rs index 25de30d57f7..cd2052de174 100644 --- a/pyo3-ffi/src/cpython/funcobject.rs +++ b/pyo3-ffi/src/cpython/funcobject.rs @@ -41,6 +41,8 @@ pub struct PyFunctionObject { pub func_weakreflist: *mut PyObject, pub func_module: *mut PyObject, pub func_annotations: *mut PyObject, + #[cfg(Py_3_14)] + pub func_annotate: *mut PyObject, #[cfg(Py_3_12)] pub func_typeparams: *mut PyObject, pub vectorcall: Option, diff --git a/pyo3-ffi/src/cpython/genobject.rs b/pyo3-ffi/src/cpython/genobject.rs index c9d419e3782..92f14d59e4b 100644 --- a/pyo3-ffi/src/cpython/genobject.rs +++ b/pyo3-ffi/src/cpython/genobject.rs @@ -1,13 +1,11 @@ use crate::object::*; use crate::PyFrameObject; -#[cfg(not(any(PyPy, GraalPy)))] -use crate::_PyErr_StackItem; -#[cfg(all(Py_3_11, not(any(PyPy, GraalPy))))] +#[cfg(all(Py_3_11, not(any(PyPy, GraalPy, Py_3_14))))] use std::os::raw::c_char; use std::os::raw::c_int; use std::ptr::addr_of_mut; -#[cfg(not(any(PyPy, GraalPy)))] +#[cfg(not(any(PyPy, GraalPy, Py_3_14)))] #[repr(C)] pub struct PyGenObject { pub ob_base: PyObject, @@ -20,7 +18,7 @@ pub struct PyGenObject { pub gi_weakreflist: *mut PyObject, pub gi_name: *mut PyObject, pub gi_qualname: *mut PyObject, - pub gi_exc_state: _PyErr_StackItem, + pub gi_exc_state: crate::cpython::pystate::_PyErr_StackItem, #[cfg(Py_3_11)] pub gi_origin_or_finalizer: *mut PyObject, #[cfg(Py_3_11)] @@ -35,6 +33,9 @@ pub struct PyGenObject { pub gi_iframe: [*mut PyObject; 1], } +#[cfg(all(Py_3_14, not(any(PyPy, GraalPy))))] +opaque_struct!(pub PyGenObject); + #[cfg_attr(windows, link(name = "pythonXY"))] extern "C" { pub static mut PyGen_Type: PyTypeObject; @@ -67,9 +68,10 @@ extern "C" { #[cfg_attr(windows, link(name = "pythonXY"))] extern "C" { pub static mut PyCoro_Type: PyTypeObject; - pub static mut _PyCoroWrapper_Type: PyTypeObject; } +// skipped _PyCoroWrapper_Type + #[inline] pub unsafe fn PyCoro_CheckExact(op: *mut PyObject) -> c_int { PyObject_TypeCheck(op, addr_of_mut!(PyCoro_Type)) diff --git a/pyo3-ffi/src/cpython/import.rs b/pyo3-ffi/src/cpython/import.rs index 697d68a419c..c8ef5ab487e 100644 --- a/pyo3-ffi/src/cpython/import.rs +++ b/pyo3-ffi/src/cpython/import.rs @@ -65,10 +65,8 @@ pub struct _frozen { extern "C" { #[cfg(not(PyPy))] pub static mut PyImport_FrozenModules: *const _frozen; - #[cfg(all(not(PyPy), Py_3_11))] - pub static mut _PyImport_FrozenBootstrap: *const _frozen; - #[cfg(all(not(PyPy), Py_3_11))] - pub static mut _PyImport_FrozenStdlib: *const _frozen; - #[cfg(all(not(PyPy), Py_3_11))] - pub static mut _PyImport_FrozenTest: *const _frozen; } + +// skipped _PyImport_FrozenBootstrap +// skipped _PyImport_FrozenStdlib +// skipped _PyImport_FrozenTest diff --git a/pyo3-ffi/src/cpython/initconfig.rs b/pyo3-ffi/src/cpython/initconfig.rs index 321d200e141..076981f1485 100644 --- a/pyo3-ffi/src/cpython/initconfig.rs +++ b/pyo3-ffi/src/cpython/initconfig.rs @@ -93,6 +93,8 @@ pub struct PyConfig { pub tracemalloc: c_int, #[cfg(Py_3_12)] pub perf_profiling: c_int, + #[cfg(Py_3_14)] + pub remote_debug: c_int, pub import_time: c_int, #[cfg(Py_3_11)] pub code_debug_ranges: c_int, @@ -141,10 +143,18 @@ pub struct PyConfig { pub safe_path: c_int, #[cfg(Py_3_12)] pub int_max_str_digits: c_int, + #[cfg(Py_3_14)] + pub thread_inherit_context: c_int, + #[cfg(Py_3_14)] + pub context_aware_warnings: c_int, + #[cfg(all(Py_3_14, target_os = "macos"))] + pub use_system_logger: c_int, #[cfg(Py_3_13)] pub cpu_count: c_int, #[cfg(Py_GIL_DISABLED)] pub enable_gil: c_int, + #[cfg(all(Py_3_14, Py_GIL_DISABLED))] + pub tlbc_enabled: c_int, pub pathconfig_warnings: c_int, #[cfg(Py_3_10)] pub program_name: *mut wchar_t, diff --git a/pyo3-ffi/src/cpython/mod.rs b/pyo3-ffi/src/cpython/mod.rs index f09d51d0e4e..adaf8bc8861 100644 --- a/pyo3-ffi/src/cpython/mod.rs +++ b/pyo3-ffi/src/cpython/mod.rs @@ -38,6 +38,7 @@ pub(crate) mod pythonrun; // skipped sysmodule.h pub(crate) mod floatobject; pub(crate) mod pyframe; +pub(crate) mod pyhash; pub(crate) mod tupleobject; pub(crate) mod unicodeobject; pub(crate) mod weakrefobject; @@ -73,6 +74,8 @@ pub use self::pydebug::*; pub use self::pyerrors::*; #[cfg(all(Py_3_11, not(PyPy)))] pub use self::pyframe::*; +#[cfg(any(not(PyPy), Py_3_13))] +pub use self::pyhash::*; #[cfg(all(Py_3_8, not(PyPy)))] pub use self::pylifecycle::*; pub use self::pymem::*; diff --git a/pyo3-ffi/src/cpython/object.rs b/pyo3-ffi/src/cpython/object.rs index 4e6932da789..26ef784dde1 100644 --- a/pyo3-ffi/src/cpython/object.rs +++ b/pyo3-ffi/src/cpython/object.rs @@ -312,8 +312,12 @@ pub struct PyHeapTypeObject { pub ht_module: *mut object::PyObject, #[cfg(all(Py_3_11, not(PyPy)))] _ht_tpname: *mut c_char, + #[cfg(Py_3_14)] + pub ht_token: *mut c_void, #[cfg(all(Py_3_11, not(PyPy)))] _spec_cache: _specialization_cache, + #[cfg(all(Py_GIL_DISABLED, Py_3_14))] + pub unique_id: Py_ssize_t, } impl Default for PyHeapTypeObject { diff --git a/pyo3-ffi/src/cpython/pyerrors.rs b/pyo3-ffi/src/cpython/pyerrors.rs index c6e10e5f07b..c9831669ac7 100644 --- a/pyo3-ffi/src/cpython/pyerrors.rs +++ b/pyo3-ffi/src/cpython/pyerrors.rs @@ -46,6 +46,8 @@ pub struct PySyntaxErrorObject { pub end_offset: *mut PyObject, pub text: *mut PyObject, pub print_file_and_line: *mut PyObject, + #[cfg(Py_3_14)] + pub metadata: *mut PyObject, } #[cfg(not(any(PyPy, GraalPy)))] diff --git a/pyo3-ffi/src/cpython/pyframe.rs b/pyo3-ffi/src/cpython/pyframe.rs index 5e1e16a7d08..f0c38be47be 100644 --- a/pyo3-ffi/src/cpython/pyframe.rs +++ b/pyo3-ffi/src/cpython/pyframe.rs @@ -1,2 +1,3 @@ +// NB used in `_PyEval_EvalFrameDefault`, maybe we remove this too. #[cfg(all(Py_3_11, not(PyPy)))] -opaque_struct!(_PyInterpreterFrame); +opaque_struct!(pub _PyInterpreterFrame); diff --git a/pyo3-ffi/src/cpython/pyhash.rs b/pyo3-ffi/src/cpython/pyhash.rs new file mode 100644 index 00000000000..b746018a38a --- /dev/null +++ b/pyo3-ffi/src/cpython/pyhash.rs @@ -0,0 +1,38 @@ +#[cfg(Py_3_14)] +use crate::Py_ssize_t; +#[cfg(Py_3_13)] +use crate::{PyObject, Py_hash_t}; +#[cfg(any(Py_3_13, not(PyPy)))] +use std::os::raw::c_void; +#[cfg(not(PyPy))] +use std::os::raw::{c_char, c_int}; + +#[cfg(not(PyPy))] +#[repr(C)] +#[derive(Copy, Clone)] +pub struct PyHash_FuncDef { + pub hash: + Option crate::Py_hash_t>, + pub name: *const c_char, + pub hash_bits: c_int, + pub seed_bits: c_int, +} + +#[cfg(not(PyPy))] +impl Default for PyHash_FuncDef { + #[inline] + fn default() -> Self { + unsafe { std::mem::zeroed() } + } +} + +extern "C" { + #[cfg(not(PyPy))] + pub fn PyHash_GetFuncDef() -> *mut PyHash_FuncDef; + #[cfg(Py_3_13)] + pub fn Py_HashPointer(ptr: *const c_void) -> Py_hash_t; + #[cfg(Py_3_13)] + pub fn PyObject_GenericHash(obj: *mut PyObject) -> Py_hash_t; + #[cfg(Py_3_14)] + pub fn Py_HashBuffer(ptr: *const c_void, len: Py_ssize_t) -> Py_hash_t; +} diff --git a/pyo3-ffi/src/cpython/pystate.rs b/pyo3-ffi/src/cpython/pystate.rs index 650cd6a1f7f..b8f6fd667b7 100644 --- a/pyo3-ffi/src/cpython/pystate.rs +++ b/pyo3-ffi/src/cpython/pystate.rs @@ -27,16 +27,18 @@ pub const PyTrace_OPCODE: c_int = 7; // skipped PyTraceInfo // skipped CFrame +/// Private structure used inline in `PyGenObject` #[cfg(not(PyPy))] #[repr(C)] #[derive(Clone, Copy)] +#[doc(hidden)] // TODO should be able to make pub(crate) after MSRV 1.74 pub struct _PyErr_StackItem { #[cfg(not(Py_3_11))] - pub exc_type: *mut PyObject, - pub exc_value: *mut PyObject, + exc_type: *mut PyObject, + exc_value: *mut PyObject, #[cfg(not(Py_3_11))] - pub exc_traceback: *mut PyObject, - pub previous_item: *mut _PyErr_StackItem, + exc_traceback: *mut PyObject, + previous_item: *mut _PyErr_StackItem, } // skipped _PyStackChunk diff --git a/pyo3-ffi/src/cpython/tupleobject.rs b/pyo3-ffi/src/cpython/tupleobject.rs index 9616d4372cc..dc1bf8e40d0 100644 --- a/pyo3-ffi/src/cpython/tupleobject.rs +++ b/pyo3-ffi/src/cpython/tupleobject.rs @@ -1,10 +1,14 @@ use crate::object::*; +#[cfg(Py_3_14)] +use crate::pyport::Py_hash_t; #[cfg(not(PyPy))] use crate::pyport::Py_ssize_t; #[repr(C)] pub struct PyTupleObject { pub ob_base: PyVarObject, + #[cfg(Py_3_14)] + pub ob_hash: Py_hash_t, pub ob_item: [*mut PyObject; 1], } diff --git a/pyo3-ffi/src/cpython/unicodeobject.rs b/pyo3-ffi/src/cpython/unicodeobject.rs index 3527a5aeadb..452c82e4c4b 100644 --- a/pyo3-ffi/src/cpython/unicodeobject.rs +++ b/pyo3-ffi/src/cpython/unicodeobject.rs @@ -31,11 +31,13 @@ use std::os::raw::{c_char, c_int, c_uint, c_void}; // skipped Py_UNICODE_LOW_SURROGATE // generated by bindgen v0.63.0 (with small adaptations) +#[cfg(not(Py_3_14))] #[repr(C)] struct BitfieldUnit { storage: Storage, } +#[cfg(not(Py_3_14))] impl BitfieldUnit { #[inline] pub const fn new(storage: Storage) -> Self { @@ -43,7 +45,7 @@ impl BitfieldUnit { } } -#[cfg(not(GraalPy))] +#[cfg(not(any(GraalPy, Py_3_14)))] impl BitfieldUnit where Storage: AsRef<[u8]> + AsMut<[u8]>, @@ -117,27 +119,33 @@ where } } -#[cfg(not(GraalPy))] +#[cfg(not(any(GraalPy, Py_3_14)))] const STATE_INTERNED_INDEX: usize = 0; -#[cfg(not(GraalPy))] +#[cfg(not(any(GraalPy, Py_3_14)))] const STATE_INTERNED_WIDTH: u8 = 2; -#[cfg(not(GraalPy))] +#[cfg(not(any(GraalPy, Py_3_14)))] const STATE_KIND_INDEX: usize = STATE_INTERNED_WIDTH as usize; -#[cfg(not(GraalPy))] +#[cfg(not(any(GraalPy, Py_3_14)))] const STATE_KIND_WIDTH: u8 = 3; -#[cfg(not(GraalPy))] +#[cfg(not(any(GraalPy, Py_3_14)))] const STATE_COMPACT_INDEX: usize = (STATE_INTERNED_WIDTH + STATE_KIND_WIDTH) as usize; -#[cfg(not(GraalPy))] +#[cfg(not(any(GraalPy, Py_3_14)))] const STATE_COMPACT_WIDTH: u8 = 1; -#[cfg(not(GraalPy))] +#[cfg(not(any(GraalPy, Py_3_14)))] const STATE_ASCII_INDEX: usize = (STATE_INTERNED_WIDTH + STATE_KIND_WIDTH + STATE_COMPACT_WIDTH) as usize; -#[cfg(not(GraalPy))] +#[cfg(not(any(GraalPy, Py_3_14)))] const STATE_ASCII_WIDTH: u8 = 1; +#[cfg(all(not(any(GraalPy, Py_3_14)), Py_3_12))] +const STATE_STATICALLY_ALLOCATED_INDEX: usize = + (STATE_INTERNED_WIDTH + STATE_KIND_WIDTH + STATE_COMPACT_WIDTH + STATE_ASCII_WIDTH) as usize; +#[cfg(all(not(any(GraalPy, Py_3_14)), Py_3_12))] +const STATE_STATICALLY_ALLOCATED_WIDTH: u8 = 1; + #[cfg(not(any(Py_3_12, GraalPy)))] const STATE_READY_INDEX: usize = (STATE_INTERNED_WIDTH + STATE_KIND_WIDTH + STATE_COMPACT_WIDTH + STATE_ASCII_WIDTH) as usize; @@ -153,15 +161,15 @@ const STATE_READY_WIDTH: u8 = 1; /// /// Memory layout of C bitfields is implementation defined, so these functions are still /// unsafe. Users must verify that they work as expected on the architectures they target. +#[cfg(not(Py_3_14))] #[repr(C)] -#[repr(align(4))] struct PyASCIIObjectState { bitfield_align: [u8; 0], bitfield: BitfieldUnit<[u8; 4usize]>, } // c_uint and u32 are not necessarily the same type on all targets / architectures -#[cfg(not(GraalPy))] +#[cfg(not(any(GraalPy, Py_3_14)))] #[allow(clippy::useless_transmute)] impl PyASCIIObjectState { #[inline] @@ -215,6 +223,26 @@ impl PyASCIIObjectState { .set(STATE_ASCII_INDEX, STATE_ASCII_WIDTH, val as u64) } + #[cfg(Py_3_12)] + #[inline] + unsafe fn statically_allocated(&self) -> c_uint { + std::mem::transmute(self.bitfield.get( + STATE_STATICALLY_ALLOCATED_INDEX, + STATE_STATICALLY_ALLOCATED_WIDTH, + ) as u32) + } + + #[cfg(Py_3_12)] + #[inline] + unsafe fn set_statically_allocated(&mut self, val: c_uint) { + let val: u32 = std::mem::transmute(val); + self.bitfield.set( + STATE_STATICALLY_ALLOCATED_INDEX, + STATE_STATICALLY_ALLOCATED_WIDTH, + val as u64, + ) + } + #[cfg(not(Py_3_12))] #[inline] unsafe fn ready(&self) -> c_uint { @@ -230,6 +258,7 @@ impl PyASCIIObjectState { } } +#[cfg(not(Py_3_14))] impl From for PyASCIIObjectState { #[inline] fn from(value: u32) -> Self { @@ -240,6 +269,7 @@ impl From for PyASCIIObjectState { } } +#[cfg(not(Py_3_14))] impl From for u32 { #[inline] fn from(value: PyASCIIObjectState) -> Self { @@ -258,19 +288,29 @@ pub struct PyASCIIObject { /// Rust doesn't expose bitfields. So we have accessor functions for /// retrieving values. /// + /// Before 3.12: /// unsigned int interned:2; // SSTATE_* constants. /// unsigned int kind:3; // PyUnicode_*_KIND constants. /// unsigned int compact:1; /// unsigned int ascii:1; /// unsigned int ready:1; /// unsigned int :24; + /// + /// 3.12, and 3.13 + /// unsigned int interned:2; // SSTATE_* constants. + /// unsigned int kind:3; // PyUnicode_*_KIND constants. + /// unsigned int compact:1; + /// unsigned int ascii:1; + /// unsigned int statically_allocated:1; + /// unsigned int :24; + /// on 3.14 and higher PyO3 doesn't access the internal state pub state: u32, #[cfg(not(Py_3_12))] pub wstr: *mut wchar_t, } /// Interacting with the bitfield is not actually well-defined, so we mark these APIs unsafe. -#[cfg(not(GraalPy))] +#[cfg(not(any(GraalPy, Py_3_14)))] impl PyASCIIObject { #[cfg_attr(not(Py_3_12), allow(rustdoc::broken_intra_doc_links))] // SSTATE_INTERNED_IMMORTAL_STATIC requires 3.12 /// Get the `interned` field of the [`PyASCIIObject`] state bitfield. @@ -347,6 +387,7 @@ impl PyASCIIObject { /// /// Calling this function with an argument that is neither `0` nor `1` is invalid. #[inline] + #[cfg(not(all(Py_3_14, Py_GIL_DISABLED)))] pub unsafe fn set_ascii(&mut self, val: c_uint) { let mut state = PyASCIIObjectState::from(self.state); state.set_ascii(val); @@ -372,6 +413,26 @@ impl PyASCIIObject { state.set_ready(val); self.state = u32::from(state); } + + /// Get the `statically_allocated` field of the [`PyASCIIObject`] state bitfield. + /// + /// Returns either `0` or `1`. + #[inline] + #[cfg(Py_3_12)] + pub unsafe fn statically_allocated(&self) -> c_uint { + PyASCIIObjectState::from(self.state).statically_allocated() + } + + /// Set the `statically_allocated` flag of the [`PyASCIIObject`] state bitfield. + /// + /// Calling this function with an argument that is neither `0` nor `1` is invalid. + #[inline] + #[cfg(Py_3_12)] + pub unsafe fn set_statically_allocated(&mut self, val: c_uint) { + let mut state = PyASCIIObjectState::from(self.state); + state.set_statically_allocated(val); + self.state = u32::from(state); + } } #[repr(C)] @@ -413,7 +474,7 @@ pub const SSTATE_INTERNED_IMMORTAL: c_uint = 2; #[cfg(Py_3_12)] pub const SSTATE_INTERNED_IMMORTAL_STATIC: c_uint = 3; -#[cfg(not(GraalPy))] +#[cfg(not(any(GraalPy, Py_3_14)))] #[inline] pub unsafe fn PyUnicode_IS_ASCII(op: *mut PyObject) -> c_uint { debug_assert!(crate::PyUnicode_Check(op) != 0); @@ -423,13 +484,13 @@ pub unsafe fn PyUnicode_IS_ASCII(op: *mut PyObject) -> c_uint { (*(op as *mut PyASCIIObject)).ascii() } -#[cfg(not(GraalPy))] +#[cfg(not(any(GraalPy, Py_3_14)))] #[inline] pub unsafe fn PyUnicode_IS_COMPACT(op: *mut PyObject) -> c_uint { (*(op as *mut PyASCIIObject)).compact() } -#[cfg(not(GraalPy))] +#[cfg(not(any(GraalPy, Py_3_14)))] #[inline] pub unsafe fn PyUnicode_IS_COMPACT_ASCII(op: *mut PyObject) -> c_uint { ((*(op as *mut PyASCIIObject)).ascii() != 0 && PyUnicode_IS_COMPACT(op) != 0).into() @@ -461,7 +522,13 @@ pub unsafe fn PyUnicode_4BYTE_DATA(op: *mut PyObject) -> *mut Py_UCS4 { PyUnicode_DATA(op) as *mut Py_UCS4 } -#[cfg(not(GraalPy))] +#[cfg(all(not(GraalPy), Py_3_14))] +extern "C" { + #[cfg_attr(PyPy, link_name = "PyPyUnicode_KIND")] + pub fn PyUnicode_KIND(op: *mut PyObject) -> c_uint; +} + +#[cfg(all(not(GraalPy), not(Py_3_14)))] #[inline] pub unsafe fn PyUnicode_KIND(op: *mut PyObject) -> c_uint { debug_assert!(crate::PyUnicode_Check(op) != 0); @@ -471,7 +538,7 @@ pub unsafe fn PyUnicode_KIND(op: *mut PyObject) -> c_uint { (*(op as *mut PyASCIIObject)).kind() } -#[cfg(not(GraalPy))] +#[cfg(not(any(GraalPy, Py_3_14)))] #[inline] pub unsafe fn _PyUnicode_COMPACT_DATA(op: *mut PyObject) -> *mut c_void { if PyUnicode_IS_ASCII(op) != 0 { @@ -489,7 +556,7 @@ pub unsafe fn _PyUnicode_NONCOMPACT_DATA(op: *mut PyObject) -> *mut c_void { (*(op as *mut PyUnicodeObject)).data.any } -#[cfg(not(any(GraalPy, PyPy)))] +#[cfg(not(any(GraalPy, PyPy, Py_3_14)))] #[inline] pub unsafe fn PyUnicode_DATA(op: *mut PyObject) -> *mut c_void { debug_assert!(crate::PyUnicode_Check(op) != 0); @@ -501,6 +568,13 @@ pub unsafe fn PyUnicode_DATA(op: *mut PyObject) -> *mut c_void { } } +#[cfg(Py_3_14)] +#[cfg(all(not(GraalPy), Py_3_14))] +extern "C" { + #[cfg_attr(PyPy, link_name = "PyPyUnicode_DATA")] + pub fn PyUnicode_DATA(op: *mut PyObject) -> *mut c_void; +} + // skipped PyUnicode_WRITE // skipped PyUnicode_READ // skipped PyUnicode_READ_CHAR diff --git a/pyo3-ffi/src/cpython/weakrefobject.rs b/pyo3-ffi/src/cpython/weakrefobject.rs index 88bb501bcc5..1c50c7a759f 100644 --- a/pyo3-ffi/src/cpython/weakrefobject.rs +++ b/pyo3-ffi/src/cpython/weakrefobject.rs @@ -1,3 +1,4 @@ +// NB publicly re-exported in `src/weakrefobject.rs` #[cfg(not(any(PyPy, GraalPy)))] pub struct _PyWeakReference { pub ob_base: crate::PyObject, diff --git a/pyo3-ffi/src/datetime.rs b/pyo3-ffi/src/datetime.rs index 7f2d7958364..4db1bdd16d1 100644 --- a/pyo3-ffi/src/datetime.rs +++ b/pyo3-ffi/src/datetime.rs @@ -487,7 +487,9 @@ extern "C" { pub fn PyDateTime_DATE_GET_MICROSECOND(o: *mut PyObject) -> c_int; #[link_name = "PyPyDateTime_GET_FOLD"] pub fn PyDateTime_DATE_GET_FOLD(o: *mut PyObject) -> c_int; - // skipped PyDateTime_DATE_GET_TZINFO (not in PyPy) + #[link_name = "PyPyDateTime_DATE_GET_TZINFO"] + #[cfg(Py_3_10)] + pub fn PyDateTime_DATE_GET_TZINFO(o: *mut PyObject) -> *mut PyObject; #[link_name = "PyPyDateTime_TIME_GET_HOUR"] pub fn PyDateTime_TIME_GET_HOUR(o: *mut PyObject) -> c_int; @@ -499,7 +501,9 @@ extern "C" { pub fn PyDateTime_TIME_GET_MICROSECOND(o: *mut PyObject) -> c_int; #[link_name = "PyPyDateTime_TIME_GET_FOLD"] pub fn PyDateTime_TIME_GET_FOLD(o: *mut PyObject) -> c_int; - // skipped PyDateTime_TIME_GET_TZINFO (not in PyPy) + #[link_name = "PyPyDateTime_TIME_GET_TZINFO"] + #[cfg(Py_3_10)] + pub fn PyDateTime_TIME_GET_TZINFO(o: *mut PyObject) -> *mut PyObject; #[link_name = "PyPyDateTime_DELTA_GET_DAYS"] pub fn PyDateTime_DELTA_GET_DAYS(o: *mut PyObject) -> c_int; diff --git a/pyo3-ffi/src/dictobject.rs b/pyo3-ffi/src/dictobject.rs index 710be80243f..e609352ac1b 100644 --- a/pyo3-ffi/src/dictobject.rs +++ b/pyo3-ffi/src/dictobject.rs @@ -118,4 +118,4 @@ extern "C" { #[cfg(any(PyPy, GraalPy, Py_LIMITED_API))] // TODO: remove (see https://github.com/PyO3/pyo3/pull/1341#issuecomment-751515985) -opaque_struct!(PyDictObject); +opaque_struct!(pub PyDictObject); diff --git a/pyo3-ffi/src/floatobject.rs b/pyo3-ffi/src/floatobject.rs index 65fc1d4c316..4e1d6476f58 100644 --- a/pyo3-ffi/src/floatobject.rs +++ b/pyo3-ffi/src/floatobject.rs @@ -4,7 +4,7 @@ use std::ptr::addr_of_mut; #[cfg(Py_LIMITED_API)] // TODO: remove (see https://github.com/PyO3/pyo3/pull/1341#issuecomment-751515985) -opaque_struct!(PyFloatObject); +opaque_struct!(pub PyFloatObject); #[cfg_attr(windows, link(name = "pythonXY"))] extern "C" { diff --git a/pyo3-ffi/src/lib.rs b/pyo3-ffi/src/lib.rs index b14fe1e8611..f78c918a8c5 100644 --- a/pyo3-ffi/src/lib.rs +++ b/pyo3-ffi/src/lib.rs @@ -129,7 +129,7 @@ //! ``` //! //! **`src/lib.rs`** -//! ```rust +//! ```rust,no_run //! use std::os::raw::{c_char, c_long}; //! use std::ptr; //! @@ -327,7 +327,8 @@ non_snake_case, non_upper_case_globals, clippy::upper_case_acronyms, - clippy::missing_safety_doc + clippy::missing_safety_doc, + clippy::ptr_eq )] #![warn(elided_lifetimes_in_paths, unused_lifetimes)] // This crate is a hand-maintained translation of CPython's headers, so requiring "unsafe" @@ -340,9 +341,10 @@ // model opaque types: // https://doc.rust-lang.org/nomicon/ffi.html#representing-opaque-structs macro_rules! opaque_struct { - ($name:ident) => { + ($(#[$attrs:meta])* $pub:vis $name:ident) => { + $(#[$attrs])* #[repr(C)] - pub struct $name([u8; 0]); + $pub struct $name([u8; 0]); }; } @@ -356,7 +358,7 @@ macro_rules! opaque_struct { /// /// Examples: /// -/// ```rust +/// ```rust,no_run /// use std::ffi::CStr; /// /// const HELLO: &CStr = pyo3_ffi::c_str!("hello"); @@ -446,6 +448,7 @@ pub use self::pystate::*; pub use self::pystrtod::*; pub use self::pythonrun::*; pub use self::rangeobject::*; +pub use self::refcount::*; pub use self::setobject::*; pub use self::sliceobject::*; pub use self::structseq::*; @@ -538,6 +541,7 @@ mod pystrtod; // skipped pythread.h // skipped pytime.h mod rangeobject; +mod refcount; mod setobject; mod sliceobject; mod structseq; diff --git a/pyo3-ffi/src/longobject.rs b/pyo3-ffi/src/longobject.rs index 68b4ecba540..eca0af3d0a5 100644 --- a/pyo3-ffi/src/longobject.rs +++ b/pyo3-ffi/src/longobject.rs @@ -4,7 +4,7 @@ use libc::size_t; use std::os::raw::{c_char, c_double, c_int, c_long, c_longlong, c_ulong, c_ulonglong, c_void}; use std::ptr::addr_of_mut; -opaque_struct!(PyLongObject); +opaque_struct!(pub PyLongObject); #[inline] pub unsafe fn PyLong_Check(op: *mut PyObject) -> c_int { diff --git a/pyo3-ffi/src/memoryobject.rs b/pyo3-ffi/src/memoryobject.rs index b7ef9e2ef1d..4e1e50c6c82 100644 --- a/pyo3-ffi/src/memoryobject.rs +++ b/pyo3-ffi/src/memoryobject.rs @@ -3,11 +3,10 @@ use crate::pyport::Py_ssize_t; use std::os::raw::{c_char, c_int}; use std::ptr::addr_of_mut; +// skipped _PyManagedBuffer_Type + #[cfg_attr(windows, link(name = "pythonXY"))] extern "C" { - #[cfg(not(Py_LIMITED_API))] - pub static mut _PyManagedBuffer_Type: PyTypeObject; - #[cfg_attr(PyPy, link_name = "PyPyMemoryView_Type")] pub static mut PyMemoryView_Type: PyTypeObject; } diff --git a/pyo3-ffi/src/modsupport.rs b/pyo3-ffi/src/modsupport.rs index 4a18d30f97c..56a68fe2613 100644 --- a/pyo3-ffi/src/modsupport.rs +++ b/pyo3-ffi/src/modsupport.rs @@ -36,6 +36,8 @@ extern "C" { pub fn Py_BuildValue(arg1: *const c_char, ...) -> *mut PyObject; // skipped Py_VaBuildValue + #[cfg(Py_3_13)] + pub fn PyModule_Add(module: *mut PyObject, name: *const c_char, value: *mut PyObject) -> c_int; #[cfg(Py_3_10)] #[cfg_attr(PyPy, link_name = "PyPyModule_AddObjectRef")] pub fn PyModule_AddObjectRef( diff --git a/pyo3-ffi/src/object.rs b/pyo3-ffi/src/object.rs index 087cd32920c..5fbf45db617 100644 --- a/pyo3-ffi/src/object.rs +++ b/pyo3-ffi/src/object.rs @@ -1,5 +1,7 @@ use crate::pyport::{Py_hash_t, Py_ssize_t}; #[cfg(Py_GIL_DISABLED)] +use crate::refcount; +#[cfg(Py_GIL_DISABLED)] use crate::PyMutex; #[cfg(Py_GIL_DISABLED)] use std::marker::PhantomPinned; @@ -7,81 +9,50 @@ use std::mem; use std::os::raw::{c_char, c_int, c_uint, c_ulong, c_void}; use std::ptr; #[cfg(Py_GIL_DISABLED)] -use std::sync::atomic::{AtomicIsize, AtomicU32, AtomicU8, Ordering::Relaxed}; +use std::sync::atomic::{AtomicIsize, AtomicU32, AtomicU8}; #[cfg(Py_LIMITED_API)] -opaque_struct!(PyTypeObject); +opaque_struct!(pub PyTypeObject); #[cfg(not(Py_LIMITED_API))] pub use crate::cpython::object::PyTypeObject; -#[cfg(Py_3_12)] -const _Py_IMMORTAL_REFCNT: Py_ssize_t = { - if cfg!(target_pointer_width = "64") { - c_uint::MAX as Py_ssize_t - } else { - // for 32-bit systems, use the lower 30 bits (see comment in CPython's object.h) - (c_uint::MAX >> 2) as Py_ssize_t - } -}; - -#[cfg(Py_GIL_DISABLED)] -const _Py_IMMORTAL_REFCNT_LOCAL: u32 = u32::MAX; - -#[allow(clippy::declare_interior_mutable_const)] -pub const PyObject_HEAD_INIT: PyObject = PyObject { - #[cfg(py_sys_config = "Py_TRACE_REFS")] - _ob_next: std::ptr::null_mut(), - #[cfg(py_sys_config = "Py_TRACE_REFS")] - _ob_prev: std::ptr::null_mut(), - #[cfg(Py_GIL_DISABLED)] - ob_tid: 0, - #[cfg(Py_GIL_DISABLED)] - _padding: 0, - #[cfg(Py_GIL_DISABLED)] - ob_mutex: PyMutex { - _bits: AtomicU8::new(0), - _pin: PhantomPinned, - }, - #[cfg(Py_GIL_DISABLED)] - ob_gc_bits: 0, - #[cfg(Py_GIL_DISABLED)] - ob_ref_local: AtomicU32::new(_Py_IMMORTAL_REFCNT_LOCAL), - #[cfg(Py_GIL_DISABLED)] - ob_ref_shared: AtomicIsize::new(0), - #[cfg(all(not(Py_GIL_DISABLED), Py_3_12))] - ob_refcnt: PyObjectObRefcnt { ob_refcnt: 1 }, - #[cfg(not(Py_3_12))] - ob_refcnt: 1, - #[cfg(PyPy)] - ob_pypy_link: 0, - ob_type: std::ptr::null_mut(), -}; - -// skipped PyObject_VAR_HEAD -// skipped Py_INVALID_SIZE - -// skipped private _Py_UNOWNED_TID +// skip PyObject_HEAD -#[cfg(Py_GIL_DISABLED)] -const _Py_REF_SHARED_SHIFT: isize = 2; -// skipped private _Py_REF_SHARED_FLAG_MASK - -// skipped private _Py_REF_SHARED_INIT -// skipped private _Py_REF_MAYBE_WEAKREF -// skipped private _Py_REF_QUEUED -// skipped private _Py_REF_MERGED +#[repr(C)] +#[derive(Copy, Clone)] +#[cfg(all(Py_3_14, not(Py_GIL_DISABLED), target_endian = "big"))] +/// This struct is anonymous in CPython, so the name was given by PyO3 because +/// Rust structs need a name. +pub struct PyObjectObFlagsAndRefcnt { + pub ob_flags: u16, + pub ob_overflow: u16, + pub ob_refcnt: u32, +} -// skipped private _Py_REF_SHARED +#[repr(C)] +#[derive(Copy, Clone)] +#[cfg(all(Py_3_14, not(Py_GIL_DISABLED), target_endian = "little"))] +/// This struct is anonymous in CPython, so the name was given by PyO3 because +/// Rust structs need a name. +pub struct PyObjectObFlagsAndRefcnt { + pub ob_refcnt: u32, + pub ob_overflow: u16, + pub ob_flags: u16, +} #[repr(C)] #[derive(Copy, Clone)] #[cfg(all(Py_3_12, not(Py_GIL_DISABLED)))] /// This union is anonymous in CPython, so the name was given by PyO3 because -/// Rust unions need a name. +/// Rust union need a name. pub union PyObjectObRefcnt { + #[cfg(all(target_pointer_width = "64", Py_3_14))] + pub ob_refcnt_full: crate::PY_INT64_T, + #[cfg(Py_3_14)] + pub refcnt_and_flags: PyObjectObFlagsAndRefcnt, pub ob_refcnt: Py_ssize_t, - #[cfg(target_pointer_width = "64")] + #[cfg(all(target_pointer_width = "64", not(Py_3_14)))] pub ob_refcnt_split: [crate::PY_UINT32_T; 2], } @@ -95,6 +66,9 @@ impl std::fmt::Debug for PyObjectObRefcnt { #[cfg(all(not(Py_3_12), not(Py_GIL_DISABLED)))] pub type PyObjectObRefcnt = Py_ssize_t; +// PyObject_HEAD_INIT comes before the PyObject definition in object.h +// but we put it after PyObject because HEAD_INIT uses PyObject + #[repr(C)] #[derive(Debug)] pub struct PyObject { @@ -104,8 +78,10 @@ pub struct PyObject { pub _ob_prev: *mut PyObject, #[cfg(Py_GIL_DISABLED)] pub ob_tid: libc::uintptr_t, - #[cfg(Py_GIL_DISABLED)] + #[cfg(all(Py_GIL_DISABLED, not(Py_3_14)))] pub _padding: u16, + #[cfg(all(Py_GIL_DISABLED, Py_3_14))] + pub ob_flags: u16, #[cfg(Py_GIL_DISABLED)] pub ob_mutex: PyMutex, // per-object lock #[cfg(Py_GIL_DISABLED)] @@ -121,7 +97,41 @@ pub struct PyObject { pub ob_type: *mut PyTypeObject, } -// skipped private _PyObject_CAST +#[allow(clippy::declare_interior_mutable_const)] +pub const PyObject_HEAD_INIT: PyObject = PyObject { + #[cfg(py_sys_config = "Py_TRACE_REFS")] + _ob_next: std::ptr::null_mut(), + #[cfg(py_sys_config = "Py_TRACE_REFS")] + _ob_prev: std::ptr::null_mut(), + #[cfg(Py_GIL_DISABLED)] + ob_tid: 0, + #[cfg(all(Py_GIL_DISABLED, Py_3_14))] + ob_flags: 0, + #[cfg(all(Py_GIL_DISABLED, not(Py_3_14)))] + _padding: 0, + #[cfg(Py_GIL_DISABLED)] + ob_mutex: PyMutex { + _bits: AtomicU8::new(0), + _pin: PhantomPinned, + }, + #[cfg(Py_GIL_DISABLED)] + ob_gc_bits: 0, + #[cfg(Py_GIL_DISABLED)] + ob_ref_local: AtomicU32::new(refcount::_Py_IMMORTAL_REFCNT_LOCAL), + #[cfg(Py_GIL_DISABLED)] + ob_ref_shared: AtomicIsize::new(0), + #[cfg(all(not(Py_GIL_DISABLED), Py_3_12))] + ob_refcnt: PyObjectObRefcnt { ob_refcnt: 1 }, + #[cfg(not(Py_3_12))] + ob_refcnt: 1, + #[cfg(PyPy)] + ob_pypy_link: 0, + ob_type: std::ptr::null_mut(), +}; + +// skipped _Py_UNOWNED_TID + +// skipped _PyObject_CAST #[repr(C)] #[derive(Debug)] @@ -150,41 +160,23 @@ extern "C" { pub fn Py_Is(x: *mut PyObject, y: *mut PyObject) -> c_int; } -// skipped private _Py_GetThreadLocal_Addr +// skipped _Py_GetThreadLocal_Addr -// skipped private _Py_ThreadId +// skipped _Py_ThreadID -// skipped private _Py_IsOwnedByCurrentThread +// skipped _Py_IsOwnedByCurrentThread -#[inline] -pub unsafe fn Py_REFCNT(ob: *mut PyObject) -> Py_ssize_t { - #[cfg(Py_GIL_DISABLED)] - { - let local = (*ob).ob_ref_local.load(Relaxed); - if local == _Py_IMMORTAL_REFCNT_LOCAL { - return _Py_IMMORTAL_REFCNT; - } - let shared = (*ob).ob_ref_shared.load(Relaxed); - local as Py_ssize_t + Py_ssize_t::from(shared >> _Py_REF_SHARED_SHIFT) - } - - #[cfg(all(not(Py_GIL_DISABLED), Py_3_12))] - { - (*ob).ob_refcnt.ob_refcnt - } - - #[cfg(all(not(Py_GIL_DISABLED), not(Py_3_12), not(GraalPy)))] - { - (*ob).ob_refcnt - } +#[cfg(GraalPy)] +extern "C" { + #[cfg(GraalPy)] + fn _Py_TYPE(arg1: *const PyObject) -> *mut PyTypeObject; - #[cfg(all(not(Py_GIL_DISABLED), not(Py_3_12), GraalPy))] - { - _Py_REFCNT(ob) - } + #[cfg(GraalPy)] + fn _Py_SIZE(arg1: *const PyObject) -> Py_ssize_t; } #[inline] +#[cfg(not(Py_3_14))] pub unsafe fn Py_TYPE(ob: *mut PyObject) -> *mut PyTypeObject { #[cfg(not(GraalPy))] return (*ob).ob_type; @@ -192,6 +184,15 @@ pub unsafe fn Py_TYPE(ob: *mut PyObject) -> *mut PyTypeObject { return _Py_TYPE(ob); } +#[cfg_attr(windows, link(name = "pythonXY"))] +#[cfg(Py_3_14)] +extern "C" { + #[cfg_attr(PyPy, link_name = "PyPy_TYPE")] + pub fn Py_TYPE(ob: *mut PyObject) -> *mut PyTypeObject; +} + +// skip _Py_TYPE compat shim + #[cfg_attr(windows, link(name = "pythonXY"))] extern "C" { #[cfg_attr(PyPy, link_name = "PyPyLong_Type")] @@ -212,29 +213,11 @@ pub unsafe fn Py_SIZE(ob: *mut PyObject) -> Py_ssize_t { _Py_SIZE(ob) } -#[inline(always)] -#[cfg(all(Py_3_12, not(Py_GIL_DISABLED)))] -unsafe fn _Py_IsImmortal(op: *mut PyObject) -> c_int { - #[cfg(target_pointer_width = "64")] - { - (((*op).ob_refcnt.ob_refcnt as crate::PY_INT32_T) < 0) as c_int - } - - #[cfg(target_pointer_width = "32")] - { - ((*op).ob_refcnt.ob_refcnt == _Py_IMMORTAL_REFCNT) as c_int - } -} - #[inline] pub unsafe fn Py_IS_TYPE(ob: *mut PyObject, tp: *mut PyTypeObject) -> c_int { (Py_TYPE(ob) == tp) as c_int } -// skipped _Py_SetRefCnt - -// skipped Py_SET_REFCNT - // skipped Py_SET_TYPE // skipped Py_SET_SIZE @@ -586,222 +569,6 @@ pub const Py_TPFLAGS_DEFAULT: c_ulong = if cfg!(Py_3_10) { pub const Py_TPFLAGS_HAVE_FINALIZE: c_ulong = 1; pub const Py_TPFLAGS_HAVE_VERSION_TAG: c_ulong = 1 << 18; -extern "C" { - #[cfg(all(py_sys_config = "Py_REF_DEBUG", not(Py_LIMITED_API)))] - fn _Py_NegativeRefcount(filename: *const c_char, lineno: c_int, op: *mut PyObject); - #[cfg(all(Py_3_12, py_sys_config = "Py_REF_DEBUG", not(Py_LIMITED_API)))] - fn _Py_INCREF_IncRefTotal(); - #[cfg(all(Py_3_12, py_sys_config = "Py_REF_DEBUG", not(Py_LIMITED_API)))] - fn _Py_DECREF_DecRefTotal(); - - #[cfg_attr(PyPy, link_name = "_PyPy_Dealloc")] - fn _Py_Dealloc(arg1: *mut PyObject); - - #[cfg_attr(PyPy, link_name = "PyPy_IncRef")] - #[cfg_attr(GraalPy, link_name = "_Py_IncRef")] - pub fn Py_IncRef(o: *mut PyObject); - #[cfg_attr(PyPy, link_name = "PyPy_DecRef")] - #[cfg_attr(GraalPy, link_name = "_Py_DecRef")] - pub fn Py_DecRef(o: *mut PyObject); - - #[cfg(all(Py_3_10, not(PyPy)))] - fn _Py_IncRef(o: *mut PyObject); - #[cfg(all(Py_3_10, not(PyPy)))] - fn _Py_DecRef(o: *mut PyObject); - - #[cfg(GraalPy)] - fn _Py_REFCNT(arg1: *const PyObject) -> Py_ssize_t; - - #[cfg(GraalPy)] - fn _Py_TYPE(arg1: *const PyObject) -> *mut PyTypeObject; - - #[cfg(GraalPy)] - fn _Py_SIZE(arg1: *const PyObject) -> Py_ssize_t; -} - -#[inline(always)] -pub unsafe fn Py_INCREF(op: *mut PyObject) { - // On limited API, the free-threaded build, or with refcount debugging, let the interpreter do refcounting - // TODO: reimplement the logic in the header in the free-threaded build, for a little bit of performance. - #[cfg(any( - Py_GIL_DISABLED, - Py_LIMITED_API, - py_sys_config = "Py_REF_DEBUG", - GraalPy - ))] - { - // _Py_IncRef was added to the ABI in 3.10; skips null checks - #[cfg(all(Py_3_10, not(PyPy)))] - { - _Py_IncRef(op); - } - - #[cfg(any(not(Py_3_10), PyPy))] - { - Py_IncRef(op); - } - } - - // version-specific builds are allowed to directly manipulate the reference count - #[cfg(not(any( - Py_GIL_DISABLED, - Py_LIMITED_API, - py_sys_config = "Py_REF_DEBUG", - GraalPy - )))] - { - #[cfg(all(Py_3_12, target_pointer_width = "64"))] - { - let cur_refcnt = (*op).ob_refcnt.ob_refcnt_split[crate::PY_BIG_ENDIAN]; - let new_refcnt = cur_refcnt.wrapping_add(1); - if new_refcnt == 0 { - return; - } - (*op).ob_refcnt.ob_refcnt_split[crate::PY_BIG_ENDIAN] = new_refcnt; - } - - #[cfg(all(Py_3_12, target_pointer_width = "32"))] - { - if _Py_IsImmortal(op) != 0 { - return; - } - (*op).ob_refcnt.ob_refcnt += 1 - } - - #[cfg(not(Py_3_12))] - { - (*op).ob_refcnt += 1 - } - - // Skipped _Py_INCREF_STAT_INC - if anyone wants this, please file an issue - // or submit a PR supporting Py_STATS build option and pystats.h - } -} - -#[inline(always)] -#[cfg_attr( - all(py_sys_config = "Py_REF_DEBUG", Py_3_12, not(Py_LIMITED_API)), - track_caller -)] -pub unsafe fn Py_DECREF(op: *mut PyObject) { - // On limited API, the free-threaded build, or with refcount debugging, let the interpreter do refcounting - // On 3.12+ we implement refcount debugging to get better assertion locations on negative refcounts - // TODO: reimplement the logic in the header in the free-threaded build, for a little bit of performance. - #[cfg(any( - Py_GIL_DISABLED, - Py_LIMITED_API, - all(py_sys_config = "Py_REF_DEBUG", not(Py_3_12)), - GraalPy - ))] - { - // _Py_DecRef was added to the ABI in 3.10; skips null checks - #[cfg(all(Py_3_10, not(PyPy)))] - { - _Py_DecRef(op); - } - - #[cfg(any(not(Py_3_10), PyPy))] - { - Py_DecRef(op); - } - } - - #[cfg(not(any( - Py_GIL_DISABLED, - Py_LIMITED_API, - all(py_sys_config = "Py_REF_DEBUG", not(Py_3_12)), - GraalPy - )))] - { - #[cfg(Py_3_12)] - if _Py_IsImmortal(op) != 0 { - return; - } - - // Skipped _Py_DECREF_STAT_INC - if anyone needs this, please file an issue - // or submit a PR supporting Py_STATS build option and pystats.h - - #[cfg(py_sys_config = "Py_REF_DEBUG")] - _Py_DECREF_DecRefTotal(); - - #[cfg(Py_3_12)] - { - (*op).ob_refcnt.ob_refcnt -= 1; - - #[cfg(py_sys_config = "Py_REF_DEBUG")] - if (*op).ob_refcnt.ob_refcnt < 0 { - let location = std::panic::Location::caller(); - let filename = std::ffi::CString::new(location.file()).unwrap(); - _Py_NegativeRefcount(filename.as_ptr(), location.line() as i32, op); - } - - if (*op).ob_refcnt.ob_refcnt == 0 { - _Py_Dealloc(op); - } - } - - #[cfg(not(Py_3_12))] - { - (*op).ob_refcnt -= 1; - - if (*op).ob_refcnt == 0 { - _Py_Dealloc(op); - } - } - } -} - -#[inline] -pub unsafe fn Py_CLEAR(op: *mut *mut PyObject) { - let tmp = *op; - if !tmp.is_null() { - *op = ptr::null_mut(); - Py_DECREF(tmp); - } -} - -#[inline] -pub unsafe fn Py_XINCREF(op: *mut PyObject) { - if !op.is_null() { - Py_INCREF(op) - } -} - -#[inline] -pub unsafe fn Py_XDECREF(op: *mut PyObject) { - if !op.is_null() { - Py_DECREF(op) - } -} - -extern "C" { - #[cfg(all(Py_3_10, Py_LIMITED_API, not(PyPy)))] - #[cfg_attr(docsrs, doc(cfg(Py_3_10)))] - pub fn Py_NewRef(obj: *mut PyObject) -> *mut PyObject; - #[cfg(all(Py_3_10, Py_LIMITED_API, not(PyPy)))] - #[cfg_attr(docsrs, doc(cfg(Py_3_10)))] - pub fn Py_XNewRef(obj: *mut PyObject) -> *mut PyObject; -} - -// macro _Py_NewRef not public; reimplemented directly inside Py_NewRef here -// macro _Py_XNewRef not public; reimplemented directly inside Py_XNewRef here - -#[cfg(all(Py_3_10, any(not(Py_LIMITED_API), PyPy)))] -#[cfg_attr(docsrs, doc(cfg(Py_3_10)))] -#[inline] -pub unsafe fn Py_NewRef(obj: *mut PyObject) -> *mut PyObject { - Py_INCREF(obj); - obj -} - -#[cfg(all(Py_3_10, any(not(Py_LIMITED_API), PyPy)))] -#[cfg_attr(docsrs, doc(cfg(Py_3_10)))] -#[inline] -pub unsafe fn Py_XNewRef(obj: *mut PyObject) -> *mut PyObject { - Py_XINCREF(obj); - obj -} - #[cfg(Py_3_13)] pub const Py_CONSTANT_NONE: c_uint = 0; #[cfg(Py_3_13)] @@ -942,4 +709,7 @@ extern "C" { arg1: *mut crate::PyTypeObject, arg2: *mut crate::PyModuleDef, ) -> *mut PyObject; + + #[cfg(Py_3_14)] + pub fn PyType_Freeze(tp: *mut crate::PyTypeObject) -> c_int; } diff --git a/pyo3-ffi/src/pyarena.rs b/pyo3-ffi/src/pyarena.rs index 87d5f28a7a5..1200de3df48 100644 --- a/pyo3-ffi/src/pyarena.rs +++ b/pyo3-ffi/src/pyarena.rs @@ -1 +1 @@ -opaque_struct!(PyArena); +opaque_struct!(pub PyArena); diff --git a/pyo3-ffi/src/pyframe.rs b/pyo3-ffi/src/pyframe.rs index 4dd3d2b31a5..1693b20b0af 100644 --- a/pyo3-ffi/src/pyframe.rs +++ b/pyo3-ffi/src/pyframe.rs @@ -6,7 +6,7 @@ use crate::PyFrameObject; use std::os::raw::c_int; #[cfg(Py_LIMITED_API)] -opaque_struct!(PyFrameObject); +opaque_struct!(pub PyFrameObject); extern "C" { pub fn PyFrame_GetLineNumber(f: *mut PyFrameObject) -> c_int; diff --git a/pyo3-ffi/src/pyhash.rs b/pyo3-ffi/src/pyhash.rs index 4f14e04a695..074278ddf3c 100644 --- a/pyo3-ffi/src/pyhash.rs +++ b/pyo3-ffi/src/pyhash.rs @@ -1,7 +1,5 @@ #[cfg(not(any(Py_LIMITED_API, PyPy)))] use crate::pyport::{Py_hash_t, Py_ssize_t}; -#[cfg(not(any(Py_LIMITED_API, PyPy, GraalPy)))] -use std::os::raw::c_char; #[cfg(not(any(Py_LIMITED_API, PyPy)))] use std::os::raw::c_void; @@ -22,29 +20,6 @@ pub const _PyHASH_MULTIPLIER: c_ulong = 1000003; // skipped non-limited _Py_HashSecret_t -#[cfg(not(any(Py_LIMITED_API, PyPy, GraalPy)))] -#[repr(C)] -#[derive(Copy, Clone)] -pub struct PyHash_FuncDef { - pub hash: Option Py_hash_t>, - pub name: *const c_char, - pub hash_bits: c_int, - pub seed_bits: c_int, -} - -#[cfg(not(any(Py_LIMITED_API, PyPy, GraalPy)))] -impl Default for PyHash_FuncDef { - #[inline] - fn default() -> Self { - unsafe { std::mem::zeroed() } - } -} - -extern "C" { - #[cfg(not(any(Py_LIMITED_API, PyPy, GraalPy)))] - pub fn PyHash_GetFuncDef() -> *mut PyHash_FuncDef; -} - // skipped Py_HASH_CUTOFF pub const Py_HASH_EXTERNAL: c_int = 0; diff --git a/pyo3-ffi/src/pyport.rs b/pyo3-ffi/src/pyport.rs index a144c67fb1b..e524831d80d 100644 --- a/pyo3-ffi/src/pyport.rs +++ b/pyo3-ffi/src/pyport.rs @@ -1,3 +1,8 @@ +// NB libc does not define this constant on all platforms, so we hard code it +// like CPython does. +// https://github.com/python/cpython/blob/d8b9011702443bb57579f8834f3effe58e290dfc/Include/pyport.h#L372 +pub const INT_MAX: std::os::raw::c_int = 2147483647; + pub type PY_UINT32_T = u32; pub type PY_UINT64_T = u64; @@ -11,8 +16,8 @@ pub type Py_ssize_t = ::libc::ssize_t; pub type Py_hash_t = Py_ssize_t; pub type Py_uhash_t = ::libc::size_t; -pub const PY_SSIZE_T_MIN: Py_ssize_t = isize::MIN as Py_ssize_t; -pub const PY_SSIZE_T_MAX: Py_ssize_t = isize::MAX as Py_ssize_t; +pub const PY_SSIZE_T_MIN: Py_ssize_t = Py_ssize_t::MIN; +pub const PY_SSIZE_T_MAX: Py_ssize_t = Py_ssize_t::MAX; #[cfg(target_endian = "big")] pub const PY_BIG_ENDIAN: usize = 1; diff --git a/pyo3-ffi/src/pystate.rs b/pyo3-ffi/src/pystate.rs index a6caf421ff6..cc16e554ca9 100644 --- a/pyo3-ffi/src/pystate.rs +++ b/pyo3-ffi/src/pystate.rs @@ -9,8 +9,8 @@ use std::os::raw::c_long; pub const MAX_CO_EXTRA_USERS: c_int = 255; -opaque_struct!(PyThreadState); -opaque_struct!(PyInterpreterState); +opaque_struct!(pub PyThreadState); +opaque_struct!(pub PyInterpreterState); extern "C" { #[cfg(not(PyPy))] @@ -80,17 +80,14 @@ pub enum PyGILState_STATE { PyGILState_UNLOCKED, } +#[cfg(not(Py_3_14))] struct HangThread; +#[cfg(not(Py_3_14))] impl Drop for HangThread { fn drop(&mut self) { loop { - #[cfg(target_family = "unix")] - unsafe { - libc::pause(); - } - #[cfg(not(target_family = "unix"))] - std::thread::sleep(std::time::Duration::from_secs(9_999_999)); + std::thread::park(); // Block forever. } } } diff --git a/pyo3-ffi/src/pythonrun.rs b/pyo3-ffi/src/pythonrun.rs index e7ea2d2efd0..80209b5875a 100644 --- a/pyo3-ffi/src/pythonrun.rs +++ b/pyo3-ffi/src/pythonrun.rs @@ -49,12 +49,12 @@ pub const PYOS_STACK_MARGIN: c_int = 2048; // skipped PyOS_CheckStack under Microsoft C #[cfg(not(any(PyPy, Py_LIMITED_API, Py_3_10)))] -opaque_struct!(_mod); +opaque_struct!(pub _mod); #[cfg(not(any(PyPy, Py_3_10)))] -opaque_struct!(symtable); +opaque_struct!(pub symtable); #[cfg(not(any(PyPy, Py_3_10)))] -opaque_struct!(_node); +opaque_struct!(pub _node); #[cfg(not(any(PyPy, Py_LIMITED_API, Py_3_10)))] #[cfg_attr(Py_3_9, deprecated(note = "Python 3.9"))] diff --git a/pyo3-ffi/src/refcount.rs b/pyo3-ffi/src/refcount.rs new file mode 100644 index 00000000000..fcb5f45be6a --- /dev/null +++ b/pyo3-ffi/src/refcount.rs @@ -0,0 +1,369 @@ +use crate::pyport::Py_ssize_t; +use crate::PyObject; +#[cfg(py_sys_config = "Py_REF_DEBUG")] +use std::os::raw::c_char; +#[cfg(Py_3_12)] +use std::os::raw::c_int; +#[cfg(all(Py_3_14, any(not(Py_GIL_DISABLED), target_pointer_width = "32")))] +use std::os::raw::c_long; +#[cfg(any(Py_GIL_DISABLED, all(Py_3_12, not(Py_3_14))))] +use std::os::raw::c_uint; +#[cfg(all(Py_3_14, not(Py_GIL_DISABLED)))] +use std::os::raw::c_ulong; +use std::ptr; +#[cfg(Py_GIL_DISABLED)] +use std::sync::atomic::Ordering::Relaxed; + +#[cfg(Py_3_14)] +const _Py_STATICALLY_ALLOCATED_FLAG: c_int = 1 << 7; + +#[cfg(all(Py_3_12, not(Py_3_14)))] +const _Py_IMMORTAL_REFCNT: Py_ssize_t = { + if cfg!(target_pointer_width = "64") { + c_uint::MAX as Py_ssize_t + } else { + // for 32-bit systems, use the lower 30 bits (see comment in CPython's object.h) + (c_uint::MAX >> 2) as Py_ssize_t + } +}; + +// comments in Python.h about the choices for these constants + +#[cfg(all(Py_3_14, not(Py_GIL_DISABLED)))] +const _Py_IMMORTAL_INITIAL_REFCNT: Py_ssize_t = { + if cfg!(target_pointer_width = "64") { + ((3 as c_ulong) << (30 as c_ulong)) as Py_ssize_t + } else { + ((5 as c_long) << (28 as c_long)) as Py_ssize_t + } +}; + +#[cfg(all(Py_3_14, not(Py_GIL_DISABLED)))] +const _Py_STATIC_IMMORTAL_INITIAL_REFCNT: Py_ssize_t = { + if cfg!(target_pointer_width = "64") { + _Py_IMMORTAL_INITIAL_REFCNT + | ((_Py_STATICALLY_ALLOCATED_FLAG as Py_ssize_t) << (32 as Py_ssize_t)) + } else { + ((7 as c_long) << (28 as c_long)) as Py_ssize_t + } +}; + +#[cfg(all(Py_3_14, target_pointer_width = "32"))] +const _Py_IMMORTAL_MINIMUM_REFCNT: Py_ssize_t = ((1 as c_long) << (30 as c_long)) as Py_ssize_t; + +#[cfg(all(Py_3_14, target_pointer_width = "32"))] +const _Py_STATIC_IMMORTAL_MINIMUM_REFCNT: Py_ssize_t = + ((6 as c_long) << (28 as c_long)) as Py_ssize_t; + +#[cfg(all(Py_3_14, Py_GIL_DISABLED))] +const _Py_IMMORTAL_INITIAL_REFCNT: Py_ssize_t = c_uint::MAX as Py_ssize_t; + +#[cfg(Py_GIL_DISABLED)] +pub(crate) const _Py_IMMORTAL_REFCNT_LOCAL: u32 = u32::MAX; + +#[cfg(Py_GIL_DISABLED)] +const _Py_REF_SHARED_SHIFT: isize = 2; +// skipped private _Py_REF_SHARED_FLAG_MASK + +// skipped private _Py_REF_SHARED_INIT +// skipped private _Py_REF_MAYBE_WEAKREF +// skipped private _Py_REF_QUEUED +// skipped private _Py_REF_MERGED + +// skipped private _Py_REF_SHARED + +extern "C" { + #[cfg(all(Py_3_14, Py_LIMITED_API))] + pub fn Py_REFCNT(ob: *mut PyObject) -> Py_ssize_t; +} + +#[cfg(not(all(Py_3_14, Py_LIMITED_API)))] +#[inline] +pub unsafe fn Py_REFCNT(ob: *mut PyObject) -> Py_ssize_t { + #[cfg(Py_GIL_DISABLED)] + { + let local = (*ob).ob_ref_local.load(Relaxed); + if local == _Py_IMMORTAL_REFCNT_LOCAL { + #[cfg(not(Py_3_14))] + return _Py_IMMORTAL_REFCNT; + #[cfg(Py_3_14)] + return _Py_IMMORTAL_INITIAL_REFCNT; + } + let shared = (*ob).ob_ref_shared.load(Relaxed); + local as Py_ssize_t + Py_ssize_t::from(shared >> _Py_REF_SHARED_SHIFT) + } + + #[cfg(all(Py_LIMITED_API, Py_3_14))] + { + Py_REFCNT(ob) + } + + #[cfg(all(not(Py_GIL_DISABLED), not(all(Py_LIMITED_API, Py_3_14)), Py_3_12))] + { + (*ob).ob_refcnt.ob_refcnt + } + + #[cfg(all(not(Py_GIL_DISABLED), not(Py_3_12), not(GraalPy)))] + { + (*ob).ob_refcnt + } + + #[cfg(all(not(Py_GIL_DISABLED), not(Py_3_12), GraalPy))] + { + _Py_REFCNT(ob) + } +} + +#[cfg(Py_3_12)] +#[inline(always)] +unsafe fn _Py_IsImmortal(op: *mut PyObject) -> c_int { + #[cfg(all(target_pointer_width = "64", not(Py_GIL_DISABLED)))] + { + (((*op).ob_refcnt.ob_refcnt as crate::PY_INT32_T) < 0) as c_int + } + + #[cfg(all(target_pointer_width = "32", not(Py_GIL_DISABLED)))] + { + #[cfg(not(Py_3_14))] + { + ((*op).ob_refcnt.ob_refcnt == _Py_IMMORTAL_REFCNT) as c_int + } + + #[cfg(Py_3_14)] + { + ((*op).ob_refcnt.ob_refcnt >= _Py_IMMORTAL_MINIMUM_REFCNT) as c_int + } + } + + #[cfg(Py_GIL_DISABLED)] + { + ((*op).ob_ref_local.load(Relaxed) == _Py_IMMORTAL_REFCNT_LOCAL) as c_int + } +} + +// skipped _Py_IsStaticImmortal + +// TODO: Py_SET_REFCNT + +extern "C" { + #[cfg(all(py_sys_config = "Py_REF_DEBUG", not(Py_LIMITED_API)))] + fn _Py_NegativeRefcount(filename: *const c_char, lineno: c_int, op: *mut PyObject); + #[cfg(all(Py_3_12, py_sys_config = "Py_REF_DEBUG", not(Py_LIMITED_API)))] + fn _Py_INCREF_IncRefTotal(); + #[cfg(all(Py_3_12, py_sys_config = "Py_REF_DEBUG", not(Py_LIMITED_API)))] + fn _Py_DECREF_DecRefTotal(); + + #[cfg_attr(PyPy, link_name = "_PyPy_Dealloc")] + fn _Py_Dealloc(arg1: *mut PyObject); + + #[cfg_attr(PyPy, link_name = "PyPy_IncRef")] + #[cfg_attr(GraalPy, link_name = "_Py_IncRef")] + pub fn Py_IncRef(o: *mut PyObject); + #[cfg_attr(PyPy, link_name = "PyPy_DecRef")] + #[cfg_attr(GraalPy, link_name = "_Py_DecRef")] + pub fn Py_DecRef(o: *mut PyObject); + + #[cfg(all(Py_3_10, not(PyPy)))] + fn _Py_IncRef(o: *mut PyObject); + #[cfg(all(Py_3_10, not(PyPy)))] + fn _Py_DecRef(o: *mut PyObject); + + #[cfg(GraalPy)] + fn _Py_REFCNT(arg1: *const PyObject) -> Py_ssize_t; +} + +#[inline(always)] +pub unsafe fn Py_INCREF(op: *mut PyObject) { + // On limited API, the free-threaded build, or with refcount debugging, let the interpreter do refcounting + // TODO: reimplement the logic in the header in the free-threaded build, for a little bit of performance. + #[cfg(any( + Py_GIL_DISABLED, + Py_LIMITED_API, + py_sys_config = "Py_REF_DEBUG", + GraalPy + ))] + { + // _Py_IncRef was added to the ABI in 3.10; skips null checks + #[cfg(all(Py_3_10, not(PyPy)))] + { + _Py_IncRef(op); + } + + #[cfg(any(not(Py_3_10), PyPy))] + { + Py_IncRef(op); + } + } + + // version-specific builds are allowed to directly manipulate the reference count + #[cfg(not(any( + Py_GIL_DISABLED, + Py_LIMITED_API, + py_sys_config = "Py_REF_DEBUG", + GraalPy + )))] + { + #[cfg(all(Py_3_14, target_pointer_width = "64"))] + { + let cur_refcnt = (*op).ob_refcnt.ob_refcnt; + if (cur_refcnt as i32) < 0 { + return; + } + (*op).ob_refcnt.ob_refcnt = cur_refcnt.wrapping_add(1); + } + + #[cfg(all(Py_3_12, not(Py_3_14), target_pointer_width = "64"))] + { + let cur_refcnt = (*op).ob_refcnt.ob_refcnt_split[crate::PY_BIG_ENDIAN]; + let new_refcnt = cur_refcnt.wrapping_add(1); + if new_refcnt == 0 { + return; + } + (*op).ob_refcnt.ob_refcnt_split[crate::PY_BIG_ENDIAN] = new_refcnt; + } + + #[cfg(all(Py_3_12, target_pointer_width = "32"))] + { + if _Py_IsImmortal(op) != 0 { + return; + } + (*op).ob_refcnt.ob_refcnt += 1 + } + + #[cfg(not(Py_3_12))] + { + (*op).ob_refcnt += 1 + } + + // Skipped _Py_INCREF_STAT_INC - if anyone wants this, please file an issue + // or submit a PR supporting Py_STATS build option and pystats.h + } +} + +// skipped _Py_DecRefShared +// skipped _Py_DecRefSharedDebug +// skipped _Py_MergeZeroLocalRefcount + +#[inline(always)] +#[cfg_attr( + all(py_sys_config = "Py_REF_DEBUG", Py_3_12, not(Py_LIMITED_API)), + track_caller +)] +pub unsafe fn Py_DECREF(op: *mut PyObject) { + // On limited API, the free-threaded build, or with refcount debugging, let the interpreter do refcounting + // On 3.12+ we implement refcount debugging to get better assertion locations on negative refcounts + // TODO: reimplement the logic in the header in the free-threaded build, for a little bit of performance. + #[cfg(any( + Py_GIL_DISABLED, + Py_LIMITED_API, + all(py_sys_config = "Py_REF_DEBUG", not(Py_3_12)), + GraalPy + ))] + { + // _Py_DecRef was added to the ABI in 3.10; skips null checks + #[cfg(all(Py_3_10, not(PyPy)))] + { + _Py_DecRef(op); + } + + #[cfg(any(not(Py_3_10), PyPy))] + { + Py_DecRef(op); + } + } + + #[cfg(not(any( + Py_GIL_DISABLED, + Py_LIMITED_API, + all(py_sys_config = "Py_REF_DEBUG", not(Py_3_12)), + GraalPy + )))] + { + #[cfg(Py_3_12)] + if _Py_IsImmortal(op) != 0 { + return; + } + + // Skipped _Py_DECREF_STAT_INC - if anyone needs this, please file an issue + // or submit a PR supporting Py_STATS build option and pystats.h + + #[cfg(py_sys_config = "Py_REF_DEBUG")] + _Py_DECREF_DecRefTotal(); + + #[cfg(Py_3_12)] + { + (*op).ob_refcnt.ob_refcnt -= 1; + + #[cfg(py_sys_config = "Py_REF_DEBUG")] + if (*op).ob_refcnt.ob_refcnt < 0 { + let location = std::panic::Location::caller(); + let filename = std::ffi::CString::new(location.file()).unwrap(); + _Py_NegativeRefcount(filename.as_ptr(), location.line() as i32, op); + } + + if (*op).ob_refcnt.ob_refcnt == 0 { + _Py_Dealloc(op); + } + } + + #[cfg(not(Py_3_12))] + { + (*op).ob_refcnt -= 1; + + if (*op).ob_refcnt == 0 { + _Py_Dealloc(op); + } + } + } +} + +#[inline] +pub unsafe fn Py_CLEAR(op: *mut *mut PyObject) { + let tmp = *op; + if !tmp.is_null() { + *op = ptr::null_mut(); + Py_DECREF(tmp); + } +} + +#[inline] +pub unsafe fn Py_XINCREF(op: *mut PyObject) { + if !op.is_null() { + Py_INCREF(op) + } +} + +#[inline] +pub unsafe fn Py_XDECREF(op: *mut PyObject) { + if !op.is_null() { + Py_DECREF(op) + } +} + +extern "C" { + #[cfg(all(Py_3_10, Py_LIMITED_API, not(PyPy)))] + #[cfg_attr(docsrs, doc(cfg(Py_3_10)))] + pub fn Py_NewRef(obj: *mut PyObject) -> *mut PyObject; + #[cfg(all(Py_3_10, Py_LIMITED_API, not(PyPy)))] + #[cfg_attr(docsrs, doc(cfg(Py_3_10)))] + pub fn Py_XNewRef(obj: *mut PyObject) -> *mut PyObject; +} + +// macro _Py_NewRef not public; reimplemented directly inside Py_NewRef here +// macro _Py_XNewRef not public; reimplemented directly inside Py_XNewRef here + +#[cfg(all(Py_3_10, any(not(Py_LIMITED_API), PyPy)))] +#[cfg_attr(docsrs, doc(cfg(Py_3_10)))] +#[inline] +pub unsafe fn Py_NewRef(obj: *mut PyObject) -> *mut PyObject { + Py_INCREF(obj); + obj +} + +#[cfg(all(Py_3_10, any(not(Py_LIMITED_API), PyPy)))] +#[cfg_attr(docsrs, doc(cfg(Py_3_10)))] +#[inline] +pub unsafe fn Py_XNewRef(obj: *mut PyObject) -> *mut PyObject { + Py_XINCREF(obj); + obj +} diff --git a/pyo3-ffi/src/setobject.rs b/pyo3-ffi/src/setobject.rs index 9d5351fc798..87e33e803f4 100644 --- a/pyo3-ffi/src/setobject.rs +++ b/pyo3-ffi/src/setobject.rs @@ -39,11 +39,7 @@ pub unsafe fn PySet_GET_SIZE(so: *mut PyObject) -> Py_ssize_t { (*so).used } -#[cfg(not(Py_LIMITED_API))] -#[cfg_attr(windows, link(name = "pythonXY"))] -extern "C" { - pub static mut _PySet_Dummy: *mut PyObject; -} +// skipped _PySet_Dummy extern "C" { #[cfg(not(Py_LIMITED_API))] diff --git a/pyo3-ffi/src/weakrefobject.rs b/pyo3-ffi/src/weakrefobject.rs index 305dc290fa8..88a1bf90314 100644 --- a/pyo3-ffi/src/weakrefobject.rs +++ b/pyo3-ffi/src/weakrefobject.rs @@ -4,16 +4,18 @@ use std::os::raw::c_int; use std::ptr::addr_of_mut; #[cfg(all(not(PyPy), Py_LIMITED_API, not(GraalPy)))] -opaque_struct!(PyWeakReference); +opaque_struct!(pub PyWeakReference); #[cfg(all(not(PyPy), not(Py_LIMITED_API), not(GraalPy)))] pub use crate::_PyWeakReference as PyWeakReference; #[cfg_attr(windows, link(name = "pythonXY"))] extern "C" { + // TODO: PyO3 is depending on this symbol in `reference.rs`, we should change this and + // remove the export as this is a private symbol. pub static mut _PyWeakref_RefType: PyTypeObject; - pub static mut _PyWeakref_ProxyType: PyTypeObject; - pub static mut _PyWeakref_CallableProxyType: PyTypeObject; + static mut _PyWeakref_ProxyType: PyTypeObject; + static mut _PyWeakref_CallableProxyType: PyTypeObject; #[cfg(PyPy)] #[link_name = "PyPyWeakref_CheckRef"] diff --git a/pyo3-introspection/Cargo.toml b/pyo3-introspection/Cargo.toml new file mode 100644 index 00000000000..98dfdebbd10 --- /dev/null +++ b/pyo3-introspection/Cargo.toml @@ -0,0 +1,18 @@ +[package] +name = "pyo3-introspection" +version = "0.25.0" +description = "Introspect dynamic libraries built with PyO3 to get metadata about the exported Python types" +authors = ["PyO3 Project and Contributors "] +homepage = "https://github.com/pyo3/pyo3" +repository = "https://github.com/pyo3/pyo3" +license = "MIT OR Apache-2.0" +edition = "2021" + +[dependencies] +anyhow = "1" +goblin = "0.9.0" +serde = { version = "1", features = ["derive"] } +serde_json = "1" + +[lints] +workspace = true diff --git a/pyo3-introspection/LICENSE-APACHE b/pyo3-introspection/LICENSE-APACHE new file mode 120000 index 00000000000..965b606f331 --- /dev/null +++ b/pyo3-introspection/LICENSE-APACHE @@ -0,0 +1 @@ +../LICENSE-APACHE \ No newline at end of file diff --git a/pyo3-introspection/LICENSE-MIT b/pyo3-introspection/LICENSE-MIT new file mode 120000 index 00000000000..76219eb72e8 --- /dev/null +++ b/pyo3-introspection/LICENSE-MIT @@ -0,0 +1 @@ +../LICENSE-MIT \ No newline at end of file diff --git a/pyo3-introspection/src/introspection.rs b/pyo3-introspection/src/introspection.rs new file mode 100644 index 00000000000..e4f49d5e0e3 --- /dev/null +++ b/pyo3-introspection/src/introspection.rs @@ -0,0 +1,318 @@ +use crate::model::{Argument, Arguments, Class, Function, Module, VariableLengthArgument}; +use anyhow::{bail, ensure, Context, Result}; +use goblin::elf::Elf; +use goblin::mach::load_command::CommandVariant; +use goblin::mach::symbols::{NO_SECT, N_SECT}; +use goblin::mach::{Mach, MachO, SingleArch}; +use goblin::pe::PE; +use goblin::Object; +use serde::Deserialize; +use std::collections::HashMap; +use std::fs; +use std::path::Path; + +/// Introspect a cdylib built with PyO3 and returns the definition of a Python module. +/// +/// This function currently supports the ELF (most *nix including Linux), Match-O (macOS) and PE (Windows) formats. +pub fn introspect_cdylib(library_path: impl AsRef, main_module_name: &str) -> Result { + let chunks = find_introspection_chunks_in_binary_object(library_path.as_ref())?; + parse_chunks(&chunks, main_module_name) +} + +/// Parses the introspection chunks found in the binary +fn parse_chunks(chunks: &[Chunk], main_module_name: &str) -> Result { + let chunks_by_id = chunks + .iter() + .map(|c| { + ( + match c { + Chunk::Module { id, .. } => id, + Chunk::Class { id, .. } => id, + Chunk::Function { id, .. } => id, + }, + c, + ) + }) + .collect::>(); + // We look for the root chunk + for chunk in chunks { + if let Chunk::Module { + name, + members, + id: _, + } = chunk + { + if name == main_module_name { + return convert_module(name, members, &chunks_by_id); + } + } + } + bail!("No module named {main_module_name} found") +} + +fn convert_module( + name: &str, + members: &[String], + chunks_by_id: &HashMap<&String, &Chunk>, +) -> Result { + let mut modules = Vec::new(); + let mut classes = Vec::new(); + let mut functions = Vec::new(); + for member in members { + if let Some(chunk) = chunks_by_id.get(member) { + match chunk { + Chunk::Module { + name, + members, + id: _, + } => { + modules.push(convert_module(name, members, chunks_by_id)?); + } + Chunk::Class { name, id: _ } => classes.push(Class { name: name.into() }), + Chunk::Function { + name, + id: _, + arguments, + } => functions.push(Function { + name: name.into(), + arguments: Arguments { + positional_only_arguments: arguments + .posonlyargs + .iter() + .map(convert_argument) + .collect(), + arguments: arguments.args.iter().map(convert_argument).collect(), + vararg: arguments + .vararg + .as_ref() + .map(convert_variable_length_argument), + keyword_only_arguments: arguments + .kwonlyargs + .iter() + .map(convert_argument) + .collect(), + kwarg: arguments + .kwarg + .as_ref() + .map(convert_variable_length_argument), + }, + }), + } + } + } + Ok(Module { + name: name.into(), + modules, + classes, + functions, + }) +} + +fn convert_argument(arg: &ChunkArgument) -> Argument { + Argument { + name: arg.name.clone(), + default_value: arg.default.clone(), + } +} + +fn convert_variable_length_argument(arg: &ChunkArgument) -> VariableLengthArgument { + VariableLengthArgument { + name: arg.name.clone(), + } +} + +fn find_introspection_chunks_in_binary_object(path: &Path) -> Result> { + let library_content = + fs::read(path).with_context(|| format!("Failed to read {}", path.display()))?; + match Object::parse(&library_content) + .context("The built library is not valid or not supported by our binary parser")? + { + Object::Elf(elf) => find_introspection_chunks_in_elf(&elf, &library_content), + Object::Mach(Mach::Binary(macho)) => { + find_introspection_chunks_in_macho(&macho, &library_content) + } + Object::Mach(Mach::Fat(multi_arch)) => { + for arch in &multi_arch { + match arch? { + SingleArch::MachO(macho) => { + return find_introspection_chunks_in_macho(&macho, &library_content) + } + SingleArch::Archive(_) => (), + } + } + bail!("No Mach-o chunk found in the multi-arch Mach-o container") + } + Object::PE(pe) => find_introspection_chunks_in_pe(&pe, &library_content), + _ => { + bail!("Only ELF, Mach-o and PE containers can be introspected") + } + } +} + +fn find_introspection_chunks_in_elf(elf: &Elf<'_>, library_content: &[u8]) -> Result> { + let mut chunks = Vec::new(); + for sym in &elf.syms { + if is_introspection_symbol(elf.strtab.get_at(sym.st_name).unwrap_or_default()) { + let section_header = &elf.section_headers[sym.st_shndx]; + let data_offset = sym.st_value + section_header.sh_offset - section_header.sh_addr; + chunks.push(read_symbol_value_with_ptr_and_len( + &library_content[usize::try_from(data_offset).context("File offset overflow")?..], + 0, + library_content, + elf.is_64, + )?); + } + } + Ok(chunks) +} + +fn find_introspection_chunks_in_macho( + macho: &MachO<'_>, + library_content: &[u8], +) -> Result> { + if !macho.little_endian { + bail!("Only little endian Mach-o binaries are supported"); + } + ensure!( + !macho.load_commands.iter().any(|command| { + matches!(command.command, CommandVariant::DyldChainedFixups(_)) + }), + "Mach-O binaries with fixup chains are not supported yet, to avoid using fixup chains, use `--codegen=link-arg=-no_fixup_chains` option." + ); + + let sections = macho + .segments + .sections() + .flatten() + .map(|t| t.map(|s| s.0)) + .collect::, _>>()?; + let mut chunks = Vec::new(); + for symbol in macho.symbols() { + let (name, nlist) = symbol?; + if nlist.is_global() + && nlist.get_type() == N_SECT + && nlist.n_sect != NO_SECT as usize + && is_introspection_symbol(name) + { + let section = §ions[nlist.n_sect - 1]; // Sections are counted from 1 + let data_offset = nlist.n_value + u64::from(section.offset) - section.addr; + chunks.push(read_symbol_value_with_ptr_and_len( + &library_content[usize::try_from(data_offset).context("File offset overflow")?..], + 0, + library_content, + macho.is_64, + )?); + } + } + Ok(chunks) +} + +fn find_introspection_chunks_in_pe(pe: &PE<'_>, library_content: &[u8]) -> Result> { + let rdata_data_section = pe + .sections + .iter() + .find(|section| section.name().unwrap_or_default() == ".rdata") + .context("No .rdata section found")?; + let rdata_shift = pe.image_base + + usize::try_from(rdata_data_section.virtual_address) + .context(".rdata virtual_address overflow")? + - usize::try_from(rdata_data_section.pointer_to_raw_data) + .context(".rdata pointer_to_raw_data overflow")?; + + let mut chunks = Vec::new(); + for export in &pe.exports { + if is_introspection_symbol(export.name.unwrap_or_default()) { + chunks.push(read_symbol_value_with_ptr_and_len( + &library_content[export.offset.context("No symbol offset")?..], + rdata_shift, + library_content, + pe.is_64, + )?); + } + } + Ok(chunks) +} + +fn read_symbol_value_with_ptr_and_len( + value_slice: &[u8], + shift: usize, + full_library_content: &[u8], + is_64: bool, +) -> Result { + let (ptr, len) = if is_64 { + let (ptr, len) = value_slice[..16].split_at(8); + let ptr = usize::try_from(u64::from_le_bytes( + ptr.try_into().context("Too short symbol value")?, + )) + .context("Pointer overflow")?; + let len = usize::try_from(u64::from_le_bytes( + len.try_into().context("Too short symbol value")?, + )) + .context("Length overflow")?; + (ptr, len) + } else { + let (ptr, len) = value_slice[..8].split_at(4); + let ptr = usize::try_from(u32::from_le_bytes( + ptr.try_into().context("Too short symbol value")?, + )) + .context("Pointer overflow")?; + let len = usize::try_from(u32::from_le_bytes( + len.try_into().context("Too short symbol value")?, + )) + .context("Length overflow")?; + (ptr, len) + }; + let chunk = &full_library_content[ptr - shift..ptr - shift + len]; + serde_json::from_slice(chunk).with_context(|| { + format!( + "Failed to parse introspection chunk: '{}'", + String::from_utf8_lossy(chunk) + ) + }) +} + +fn is_introspection_symbol(name: &str) -> bool { + name.strip_prefix('_') + .unwrap_or(name) + .starts_with("PYO3_INTROSPECTION_0_") +} + +#[derive(Deserialize)] +#[serde(tag = "type", rename_all = "lowercase")] +enum Chunk { + Module { + id: String, + name: String, + members: Vec, + }, + Class { + id: String, + name: String, + }, + Function { + id: String, + name: String, + arguments: ChunkArguments, + }, +} + +#[derive(Deserialize)] +struct ChunkArguments { + #[serde(default)] + posonlyargs: Vec, + #[serde(default)] + args: Vec, + #[serde(default)] + vararg: Option, + #[serde(default)] + kwonlyargs: Vec, + #[serde(default)] + kwarg: Option, +} + +#[derive(Deserialize)] +struct ChunkArgument { + name: String, + #[serde(default)] + default: Option, +} diff --git a/pyo3-introspection/src/lib.rs b/pyo3-introspection/src/lib.rs new file mode 100644 index 00000000000..22aac933e85 --- /dev/null +++ b/pyo3-introspection/src/lib.rs @@ -0,0 +1,8 @@ +//! Utilities to introspect cdylib built using PyO3 and generate [type stubs](https://typing.readthedocs.io/en/latest/source/stubs.html). + +pub use crate::introspection::introspect_cdylib; +pub use crate::stubs::module_stub_files; + +mod introspection; +pub mod model; +mod stubs; diff --git a/pyo3-introspection/src/model.rs b/pyo3-introspection/src/model.rs new file mode 100644 index 00000000000..7705a0006a4 --- /dev/null +++ b/pyo3-introspection/src/model.rs @@ -0,0 +1,45 @@ +#[derive(Debug, Eq, PartialEq, Clone, Hash)] +pub struct Module { + pub name: String, + pub modules: Vec, + pub classes: Vec, + pub functions: Vec, +} + +#[derive(Debug, Eq, PartialEq, Clone, Hash)] +pub struct Class { + pub name: String, +} + +#[derive(Debug, Eq, PartialEq, Clone, Hash)] +pub struct Function { + pub name: String, + pub arguments: Arguments, +} + +#[derive(Debug, Eq, PartialEq, Clone, Hash)] +pub struct Arguments { + /// Arguments before / + pub positional_only_arguments: Vec, + /// Regular arguments (between / and *) + pub arguments: Vec, + /// *vararg + pub vararg: Option, + /// Arguments after * + pub keyword_only_arguments: Vec, + /// **kwarg + pub kwarg: Option, +} + +#[derive(Debug, Eq, PartialEq, Clone, Hash)] +pub struct Argument { + pub name: String, + /// Default value as a Python expression + pub default_value: Option, +} + +/// A variable length argument ie. *vararg or **kwarg +#[derive(Debug, Eq, PartialEq, Clone, Hash)] +pub struct VariableLengthArgument { + pub name: String, +} diff --git a/pyo3-introspection/src/stubs.rs b/pyo3-introspection/src/stubs.rs new file mode 100644 index 00000000000..2312d7d37ac --- /dev/null +++ b/pyo3-introspection/src/stubs.rs @@ -0,0 +1,151 @@ +use crate::model::{Argument, Class, Function, Module, VariableLengthArgument}; +use std::collections::HashMap; +use std::path::{Path, PathBuf}; + +/// Generates the [type stubs](https://typing.readthedocs.io/en/latest/source/stubs.html) of a given module. +/// It returns a map between the file name and the file content. +/// The root module stubs will be in the `__init__.pyi` file and the submodules directory +/// in files with a relevant name. +pub fn module_stub_files(module: &Module) -> HashMap { + let mut output_files = HashMap::new(); + add_module_stub_files(module, Path::new(""), &mut output_files); + output_files +} + +fn add_module_stub_files( + module: &Module, + module_path: &Path, + output_files: &mut HashMap, +) { + output_files.insert(module_path.join("__init__.pyi"), module_stubs(module)); + for submodule in &module.modules { + if submodule.modules.is_empty() { + output_files.insert( + module_path.join(format!("{}.pyi", submodule.name)), + module_stubs(submodule), + ); + } else { + add_module_stub_files(submodule, &module_path.join(&submodule.name), output_files); + } + } +} + +/// Generates the module stubs to a String, not including submodules +fn module_stubs(module: &Module) -> String { + let mut elements = Vec::new(); + for class in &module.classes { + elements.push(class_stubs(class)); + } + for function in &module.functions { + elements.push(function_stubs(function)); + } + elements.push(String::new()); // last line jump + elements.join("\n") +} + +fn class_stubs(class: &Class) -> String { + format!("class {}: ...", class.name) +} + +fn function_stubs(function: &Function) -> String { + // Signature + let mut parameters = Vec::new(); + for argument in &function.arguments.positional_only_arguments { + parameters.push(argument_stub(argument)); + } + if !function.arguments.positional_only_arguments.is_empty() { + parameters.push("/".into()); + } + for argument in &function.arguments.arguments { + parameters.push(argument_stub(argument)); + } + if let Some(argument) = &function.arguments.vararg { + parameters.push(format!("*{}", variable_length_argument_stub(argument))); + } else if !function.arguments.keyword_only_arguments.is_empty() { + parameters.push("*".into()); + } + for argument in &function.arguments.keyword_only_arguments { + parameters.push(argument_stub(argument)); + } + if let Some(argument) = &function.arguments.kwarg { + parameters.push(format!("**{}", variable_length_argument_stub(argument))); + } + format!("def {}({}): ...", function.name, parameters.join(", ")) +} + +fn argument_stub(argument: &Argument) -> String { + let mut output = argument.name.clone(); + if let Some(default_value) = &argument.default_value { + output.push('='); + output.push_str(default_value); + } + output +} + +fn variable_length_argument_stub(argument: &VariableLengthArgument) -> String { + argument.name.clone() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::model::Arguments; + + #[test] + fn function_stubs_with_variable_length() { + let function = Function { + name: "func".into(), + arguments: Arguments { + positional_only_arguments: vec![Argument { + name: "posonly".into(), + default_value: None, + }], + arguments: vec![Argument { + name: "arg".into(), + default_value: None, + }], + vararg: Some(VariableLengthArgument { + name: "varargs".into(), + }), + keyword_only_arguments: vec![Argument { + name: "karg".into(), + default_value: None, + }], + kwarg: Some(VariableLengthArgument { + name: "kwarg".into(), + }), + }, + }; + assert_eq!( + "def func(posonly, /, arg, *varargs, karg, **kwarg): ...", + function_stubs(&function) + ) + } + + #[test] + fn function_stubs_without_variable_length() { + let function = Function { + name: "afunc".into(), + arguments: Arguments { + positional_only_arguments: vec![Argument { + name: "posonly".into(), + default_value: Some("1".into()), + }], + arguments: vec![Argument { + name: "arg".into(), + default_value: Some("True".into()), + }], + vararg: None, + keyword_only_arguments: vec![Argument { + name: "karg".into(), + default_value: Some("\"foo\"".into()), + }], + kwarg: None, + }, + }; + assert_eq!( + "def afunc(posonly=1, /, arg=True, *, karg=\"foo\"): ...", + function_stubs(&function) + ) + } +} diff --git a/pyo3-introspection/tests/test.rs b/pyo3-introspection/tests/test.rs new file mode 100644 index 00000000000..37070a53a13 --- /dev/null +++ b/pyo3-introspection/tests/test.rs @@ -0,0 +1,77 @@ +use anyhow::Result; +use pyo3_introspection::{introspect_cdylib, module_stub_files}; +use std::collections::HashMap; +use std::path::{Path, PathBuf}; +use std::{env, fs}; + +#[test] +fn pytests_stubs() -> Result<()> { + // We run the introspection + let binary = env::var_os("PYO3_PYTEST_LIB_PATH") + .expect("The PYO3_PYTEST_LIB_PATH constant must be set and target the pyo3-pytests cdylib"); + let module = introspect_cdylib(binary, "pyo3_pytests")?; + let actual_stubs = module_stub_files(&module); + + // We read the expected stubs + let expected_subs_dir = Path::new(env!("CARGO_MANIFEST_DIR")) + .parent() + .unwrap() + .join("pytests") + .join("stubs"); + let mut expected_subs = HashMap::new(); + add_dir_files( + &expected_subs_dir, + &expected_subs_dir.canonicalize()?, + &mut expected_subs, + )?; + + // We ensure we do not have extra generated files + for file_name in actual_stubs.keys() { + assert!( + expected_subs.contains_key(file_name), + "The generated file {} is not in the expected stubs directory pytests/stubs", + file_name.display() + ); + } + + // We ensure the expected files are generated properly + for (file_name, expected_file_content) in &expected_subs { + let actual_file_content = actual_stubs.get(file_name).unwrap_or_else(|| { + panic!( + "The expected stub file {} has not been generated", + file_name.display() + ) + }); + assert_eq!( + &expected_file_content.replace('\r', ""), // Windows compatibility + actual_file_content, + "The content of file {} is different", + file_name.display() + ) + } + + Ok(()) +} + +fn add_dir_files( + dir_path: &Path, + base_dir_path: &Path, + output: &mut HashMap, +) -> Result<()> { + for entry in fs::read_dir(dir_path)? { + let entry = entry?; + if entry.file_type()?.is_dir() { + add_dir_files(&entry.path(), base_dir_path, output)?; + } else { + output.insert( + entry + .path() + .canonicalize()? + .strip_prefix(base_dir_path)? + .into(), + fs::read_to_string(entry.path())?, + ); + } + } + Ok(()) +} diff --git a/pyo3-macros-backend/Cargo.toml b/pyo3-macros-backend/Cargo.toml index 91e0009f9f6..d626fa1ebf7 100644 --- a/pyo3-macros-backend/Cargo.toml +++ b/pyo3-macros-backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3-macros-backend" -version = "0.24.0" +version = "0.25.0" description = "Code generation for PyO3 package" authors = ["PyO3 Project and Contributors "] keywords = ["pyo3", "python", "cpython", "ffi"] @@ -17,7 +17,7 @@ rust-version = "1.63" [dependencies] heck = "0.5" proc-macro2 = { version = "1.0.60", default-features = false } -pyo3-build-config = { path = "../pyo3-build-config", version = "=0.24.0", features = ["resolve-config"] } +pyo3-build-config = { path = "../pyo3-build-config", version = "=0.25.0", features = ["resolve-config"] } quote = { version = "1", default-features = false } [dependencies.syn] @@ -26,10 +26,11 @@ default-features = false features = ["derive", "parsing", "printing", "clone-impls", "full", "extra-traits"] [build-dependencies] -pyo3-build-config = { path = "../pyo3-build-config", version = "=0.24.0" } +pyo3-build-config = { path = "../pyo3-build-config", version = "=0.25.0" } [lints] workspace = true [features] experimental-async = [] +experimental-inspect = [] diff --git a/pyo3-macros-backend/src/attributes.rs b/pyo3-macros-backend/src/attributes.rs old mode 100755 new mode 100644 index 19a12801065..ac3894d6419 --- a/pyo3-macros-backend/src/attributes.rs +++ b/pyo3-macros-backend/src/attributes.rs @@ -27,6 +27,7 @@ pub mod kw { syn::custom_keyword!(hash); syn::custom_keyword!(into_py_with); syn::custom_keyword!(item); + syn::custom_keyword!(immutable_type); syn::custom_keyword!(from_item_all); syn::custom_keyword!(mapping); syn::custom_keyword!(module); @@ -45,6 +46,7 @@ pub mod kw { syn::custom_keyword!(transparent); syn::custom_keyword!(unsendable); syn::custom_keyword!(weakref); + syn::custom_keyword!(generic); syn::custom_keyword!(gil_used); } @@ -350,37 +352,7 @@ impl ToTokens for OptionalKeywordAttribute { } } -#[derive(Debug, Clone)] -pub struct ExprPathWrap { - pub from_lit_str: bool, - pub expr_path: ExprPath, -} - -impl Parse for ExprPathWrap { - fn parse(input: ParseStream<'_>) -> Result { - match input.parse::() { - Ok(expr_path) => Ok(ExprPathWrap { - from_lit_str: false, - expr_path, - }), - Err(e) => match input.parse::>() { - Ok(LitStrValue(expr_path)) => Ok(ExprPathWrap { - from_lit_str: true, - expr_path, - }), - Err(_) => Err(e), - }, - } - } -} - -impl ToTokens for ExprPathWrap { - fn to_tokens(&self, tokens: &mut TokenStream) { - self.expr_path.to_tokens(tokens) - } -} - -pub type FromPyWithAttribute = KeywordAttribute; +pub type FromPyWithAttribute = KeywordAttribute; pub type IntoPyWithAttribute = KeywordAttribute; pub type DefaultAttribute = OptionalKeywordAttribute; diff --git a/pyo3-macros-backend/src/derive_attributes.rs b/pyo3-macros-backend/src/derive_attributes.rs new file mode 100644 index 00000000000..63328a107b4 --- /dev/null +++ b/pyo3-macros-backend/src/derive_attributes.rs @@ -0,0 +1,217 @@ +use crate::attributes::{ + self, get_pyo3_options, CrateAttribute, DefaultAttribute, FromPyWithAttribute, + IntoPyWithAttribute, RenameAllAttribute, +}; +use proc_macro2::Span; +use syn::parse::{Parse, ParseStream}; +use syn::spanned::Spanned; +use syn::{parenthesized, Attribute, LitStr, Result, Token}; + +/// Attributes for deriving `FromPyObject`/`IntoPyObject` scoped on containers. +pub enum ContainerAttribute { + /// Treat the Container as a Wrapper, operate directly on its field + Transparent(attributes::kw::transparent), + /// Force every field to be extracted from item of source Python object. + ItemAll(attributes::kw::from_item_all), + /// Change the name of an enum variant in the generated error message. + ErrorAnnotation(LitStr), + /// Change the path for the pyo3 crate + Crate(CrateAttribute), + /// Converts the field idents according to the [RenamingRule](attributes::RenamingRule) before extraction + RenameAll(RenameAllAttribute), +} + +impl Parse for ContainerAttribute { + fn parse(input: ParseStream<'_>) -> Result { + let lookahead = input.lookahead1(); + if lookahead.peek(attributes::kw::transparent) { + let kw: attributes::kw::transparent = input.parse()?; + Ok(ContainerAttribute::Transparent(kw)) + } else if lookahead.peek(attributes::kw::from_item_all) { + let kw: attributes::kw::from_item_all = input.parse()?; + Ok(ContainerAttribute::ItemAll(kw)) + } else if lookahead.peek(attributes::kw::annotation) { + let _: attributes::kw::annotation = input.parse()?; + let _: Token![=] = input.parse()?; + input.parse().map(ContainerAttribute::ErrorAnnotation) + } else if lookahead.peek(Token![crate]) { + input.parse().map(ContainerAttribute::Crate) + } else if lookahead.peek(attributes::kw::rename_all) { + input.parse().map(ContainerAttribute::RenameAll) + } else { + Err(lookahead.error()) + } + } +} + +#[derive(Default)] +pub struct ContainerAttributes { + /// Treat the Container as a Wrapper, operate directly on its field + pub transparent: Option, + /// Force every field to be extracted from item of source Python object. + pub from_item_all: Option, + /// Change the name of an enum variant in the generated error message. + pub annotation: Option, + /// Change the path for the pyo3 crate + pub krate: Option, + /// Converts the field idents according to the [RenamingRule](attributes::RenamingRule) before extraction + pub rename_all: Option, +} + +impl ContainerAttributes { + pub fn from_attrs(attrs: &[Attribute]) -> Result { + let mut options = ContainerAttributes::default(); + + for attr in attrs { + if let Some(pyo3_attrs) = get_pyo3_options(attr)? { + pyo3_attrs + .into_iter() + .try_for_each(|opt| options.set_option(opt))?; + } + } + Ok(options) + } + + fn set_option(&mut self, option: ContainerAttribute) -> syn::Result<()> { + macro_rules! set_option { + ($key:ident) => { + { + ensure_spanned!( + self.$key.is_none(), + $key.span() => concat!("`", stringify!($key), "` may only be specified once") + ); + self.$key = Some($key); + } + }; + } + + match option { + ContainerAttribute::Transparent(transparent) => set_option!(transparent), + ContainerAttribute::ItemAll(from_item_all) => set_option!(from_item_all), + ContainerAttribute::ErrorAnnotation(annotation) => set_option!(annotation), + ContainerAttribute::Crate(krate) => set_option!(krate), + ContainerAttribute::RenameAll(rename_all) => set_option!(rename_all), + } + Ok(()) + } +} + +#[derive(Clone, Debug)] +pub enum FieldGetter { + GetItem(attributes::kw::item, Option), + GetAttr(attributes::kw::attribute, Option), +} + +impl FieldGetter { + pub fn span(&self) -> Span { + match self { + FieldGetter::GetItem(item, _) => item.span, + FieldGetter::GetAttr(attribute, _) => attribute.span, + } + } +} + +pub enum FieldAttribute { + Getter(FieldGetter), + FromPyWith(FromPyWithAttribute), + IntoPyWith(IntoPyWithAttribute), + Default(DefaultAttribute), +} + +impl Parse for FieldAttribute { + fn parse(input: ParseStream<'_>) -> Result { + let lookahead = input.lookahead1(); + if lookahead.peek(attributes::kw::attribute) { + let attr_kw: attributes::kw::attribute = input.parse()?; + if input.peek(syn::token::Paren) { + let content; + let _ = parenthesized!(content in input); + let attr_name: LitStr = content.parse()?; + if !content.is_empty() { + return Err(content.error( + "expected at most one argument: `attribute` or `attribute(\"name\")`", + )); + } + ensure_spanned!( + !attr_name.value().is_empty(), + attr_name.span() => "attribute name cannot be empty" + ); + Ok(Self::Getter(FieldGetter::GetAttr(attr_kw, Some(attr_name)))) + } else { + Ok(Self::Getter(FieldGetter::GetAttr(attr_kw, None))) + } + } else if lookahead.peek(attributes::kw::item) { + let item_kw: attributes::kw::item = input.parse()?; + if input.peek(syn::token::Paren) { + let content; + let _ = parenthesized!(content in input); + let key = content.parse()?; + if !content.is_empty() { + return Err( + content.error("expected at most one argument: `item` or `item(key)`") + ); + } + Ok(Self::Getter(FieldGetter::GetItem(item_kw, Some(key)))) + } else { + Ok(Self::Getter(FieldGetter::GetItem(item_kw, None))) + } + } else if lookahead.peek(attributes::kw::from_py_with) { + input.parse().map(Self::FromPyWith) + } else if lookahead.peek(attributes::kw::into_py_with) { + input.parse().map(FieldAttribute::IntoPyWith) + } else if lookahead.peek(Token![default]) { + input.parse().map(Self::Default) + } else { + Err(lookahead.error()) + } + } +} + +#[derive(Clone, Debug, Default)] +pub struct FieldAttributes { + pub getter: Option, + pub from_py_with: Option, + pub into_py_with: Option, + pub default: Option, +} + +impl FieldAttributes { + /// Extract the field attributes. + pub fn from_attrs(attrs: &[Attribute]) -> Result { + let mut options = FieldAttributes::default(); + + for attr in attrs { + if let Some(pyo3_attrs) = get_pyo3_options(attr)? { + pyo3_attrs + .into_iter() + .try_for_each(|opt| options.set_option(opt))?; + } + } + Ok(options) + } + + fn set_option(&mut self, option: FieldAttribute) -> syn::Result<()> { + macro_rules! set_option { + ($key:ident) => { + set_option!($key, concat!("`", stringify!($key), "` may only be specified once")) + }; + ($key:ident, $msg: expr) => {{ + ensure_spanned!( + self.$key.is_none(), + $key.span() => $msg + ); + self.$key = Some($key); + }} + } + + match option { + FieldAttribute::Getter(getter) => { + set_option!(getter, "only one of `attribute` or `item` can be provided") + } + FieldAttribute::FromPyWith(from_py_with) => set_option!(from_py_with), + FieldAttribute::IntoPyWith(into_py_with) => set_option!(into_py_with), + FieldAttribute::Default(default) => set_option!(default), + } + Ok(()) + } +} diff --git a/pyo3-macros-backend/src/frompyobject.rs b/pyo3-macros-backend/src/frompyobject.rs index 68f95e794a8..2c288709f93 100644 --- a/pyo3-macros-backend/src/frompyobject.rs +++ b/pyo3-macros-backend/src/frompyobject.rs @@ -1,18 +1,11 @@ -use crate::attributes::{ - self, get_pyo3_options, CrateAttribute, DefaultAttribute, FromPyWithAttribute, - RenameAllAttribute, RenamingRule, -}; -use crate::utils::{self, deprecated_from_py_with, Ctx}; +use crate::attributes::{DefaultAttribute, FromPyWithAttribute, RenamingRule}; +use crate::derive_attributes::{ContainerAttributes, FieldAttributes, FieldGetter}; +use crate::utils::{self, Ctx}; use proc_macro2::TokenStream; use quote::{format_ident, quote, quote_spanned, ToTokens}; use syn::{ - ext::IdentExt, - parenthesized, - parse::{Parse, ParseStream}, - parse_quote, - punctuated::Punctuated, - spanned::Spanned, - Attribute, DataEnum, DeriveInput, Fields, Ident, LitStr, Result, Token, + ext::IdentExt, parse_quote, punctuated::Punctuated, spanned::Spanned, DataEnum, DeriveInput, + Fields, Ident, Result, Token, }; /// Describes derivation input of an enum. @@ -26,7 +19,11 @@ impl<'a> Enum<'a> { /// /// `data_enum` is the `syn` representation of the input enum, `ident` is the /// `Identifier` of the enum. - fn new(data_enum: &'a DataEnum, ident: &'a Ident, options: ContainerOptions) -> Result { + fn new( + data_enum: &'a DataEnum, + ident: &'a Ident, + options: ContainerAttributes, + ) -> Result { ensure_spanned!( !data_enum.variants.is_empty(), ident.span() => "cannot derive FromPyObject for empty enum" @@ -35,7 +32,7 @@ impl<'a> Enum<'a> { .variants .iter() .map(|variant| { - let mut variant_options = ContainerOptions::from_attrs(&variant.attrs)?; + let mut variant_options = ContainerAttributes::from_attrs(&variant.attrs)?; if let Some(rename_all) = &options.rename_all { ensure_spanned!( variant_options.rename_all.is_none(), @@ -149,7 +146,7 @@ impl<'a> Container<'a> { /// Construct a container based on fields, identifier and attributes. /// /// Fails if the variant has no fields or incompatible attributes. - fn new(fields: &'a Fields, path: syn::Path, options: ContainerOptions) -> Result { + fn new(fields: &'a Fields, path: syn::Path, options: ContainerAttributes) -> Result { let style = match fields { Fields::Unnamed(unnamed) if !unnamed.unnamed.is_empty() => { ensure_spanned!( @@ -160,7 +157,7 @@ impl<'a> Container<'a> { .unnamed .iter() .map(|field| { - let attrs = FieldPyO3Attributes::from_attrs(&field.attrs)?; + let attrs = FieldAttributes::from_attrs(&field.attrs)?; ensure_spanned!( attrs.getter.is_none(), field.span() => "`getter` is not permitted on tuple struct elements." @@ -180,7 +177,7 @@ impl<'a> Container<'a> { // explicit annotation. let field = tuple_fields.pop().unwrap(); ContainerType::TupleNewtype(field.from_py_with) - } else if options.transparent { + } else if options.transparent.is_some() { bail_spanned!( fields.span() => "transparent structs and variants can only have 1 field" ); @@ -197,17 +194,17 @@ impl<'a> Container<'a> { .ident .as_ref() .expect("Named fields should have identifiers"); - let mut attrs = FieldPyO3Attributes::from_attrs(&field.attrs)?; + let mut attrs = FieldAttributes::from_attrs(&field.attrs)?; if let Some(ref from_item_all) = options.from_item_all { - if let Some(replaced) = attrs.getter.replace(FieldGetter::GetItem(None)) + if let Some(replaced) = attrs.getter.replace(FieldGetter::GetItem(parse_quote!(item), None)) { match replaced { - FieldGetter::GetItem(Some(item_name)) => { - attrs.getter = Some(FieldGetter::GetItem(Some(item_name))); + FieldGetter::GetItem(item, Some(item_name)) => { + attrs.getter = Some(FieldGetter::GetItem(item, Some(item_name))); } - FieldGetter::GetItem(None) => bail_spanned!(from_item_all.span() => "Useless `item` - the struct is already annotated with `from_item_all`"), - FieldGetter::GetAttr(_) => bail_spanned!( + FieldGetter::GetItem(_, None) => bail_spanned!(from_item_all.span() => "Useless `item` - the struct is already annotated with `from_item_all`"), + FieldGetter::GetAttr(_, _) => bail_spanned!( from_item_all.span() => "The struct is already annotated with `from_item_all`, `attribute` is not allowed" ), } @@ -226,7 +223,7 @@ impl<'a> Container<'a> { bail_spanned!( fields.span() => "cannot derive FromPyObject for structs and variants with only default values" ) - } else if options.transparent { + } else if options.transparent.is_some() { ensure_spanned!( struct_fields.len() == 1, fields.span() => "transparent structs and variants can only have 1 field" @@ -304,13 +301,10 @@ impl<'a> Container<'a> { value: expr_path, }) = from_py_with { - let deprecation = deprecated_from_py_with(expr_path).unwrap_or_default(); - let extractor = quote_spanned! { kw.span => { let from_py_with: fn(_) -> _ = #expr_path; from_py_with } }; quote! { - #deprecation Ok(#self_ty { #ident: #pyo3_path::impl_::frompyobject::extract_struct_field_with(#extractor, obj, #struct_name, #field_name)? }) @@ -327,13 +321,10 @@ impl<'a> Container<'a> { value: expr_path, }) = from_py_with { - let deprecation = deprecated_from_py_with(expr_path).unwrap_or_default(); - let extractor = quote_spanned! { kw.span => { let from_py_with: fn(_) -> _ = #expr_path; from_py_with } }; quote! { - #deprecation #pyo3_path::impl_::frompyobject::extract_tuple_struct_field_with(#extractor, obj, #struct_name, 0).map(#self_ty) } } else { @@ -367,14 +358,7 @@ impl<'a> Container<'a> { }} }); - let deprecations = struct_fields - .iter() - .filter_map(|fields| fields.from_py_with.as_ref()) - .filter_map(|kw| deprecated_from_py_with(&kw.value)) - .collect::(); - quote!( - #deprecations match #pyo3_path::types::PyAnyMethods::extract(obj) { ::std::result::Result::Ok((#(#field_idents),*)) => ::std::result::Result::Ok(#self_ty(#(#fields),*)), ::std::result::Result::Err(err) => ::std::result::Result::Err(err), @@ -390,24 +374,28 @@ impl<'a> Container<'a> { for field in struct_fields { let ident = field.ident; let field_name = ident.unraw().to_string(); - let getter = match field.getter.as_ref().unwrap_or(&FieldGetter::GetAttr(None)) { - FieldGetter::GetAttr(Some(name)) => { + let getter = match field + .getter + .as_ref() + .unwrap_or(&FieldGetter::GetAttr(parse_quote!(attribute), None)) + { + FieldGetter::GetAttr(_, Some(name)) => { quote!(#pyo3_path::types::PyAnyMethods::getattr(obj, #pyo3_path::intern!(obj.py(), #name))) } - FieldGetter::GetAttr(None) => { + FieldGetter::GetAttr(_, None) => { let name = self .rename_rule .map(|rule| utils::apply_renaming_rule(rule, &field_name)); let name = name.as_deref().unwrap_or(&field_name); quote!(#pyo3_path::types::PyAnyMethods::getattr(obj, #pyo3_path::intern!(obj.py(), #name))) } - FieldGetter::GetItem(Some(syn::Lit::Str(key))) => { + FieldGetter::GetItem(_, Some(syn::Lit::Str(key))) => { quote!(#pyo3_path::types::PyAnyMethods::get_item(obj, #pyo3_path::intern!(obj.py(), #key))) } - FieldGetter::GetItem(Some(key)) => { + FieldGetter::GetItem(_, Some(key)) => { quote!(#pyo3_path::types::PyAnyMethods::get_item(obj, #key)) } - FieldGetter::GetItem(None) => { + FieldGetter::GetItem(_, None) => { let name = self .rename_rule .map(|rule| utils::apply_renaming_rule(rule, &field_name)); @@ -448,229 +436,7 @@ impl<'a> Container<'a> { fields.push(quote!(#ident: #extracted)); } - let d = struct_fields - .iter() - .filter_map(|field| field.from_py_with.as_ref()) - .filter_map(|kw| deprecated_from_py_with(&kw.value)) - .collect::(); - - quote!(#d ::std::result::Result::Ok(#self_ty{#fields})) - } -} - -#[derive(Default)] -struct ContainerOptions { - /// Treat the Container as a Wrapper, directly extract its fields from the input object. - transparent: bool, - /// Force every field to be extracted from item of source Python object. - from_item_all: Option, - /// Change the name of an enum variant in the generated error message. - annotation: Option, - /// Change the path for the pyo3 crate - krate: Option, - /// Converts the field idents according to the [RenamingRule] before extraction - rename_all: Option, -} - -/// Attributes for deriving FromPyObject scoped on containers. -enum ContainerPyO3Attribute { - /// Treat the Container as a Wrapper, directly extract its fields from the input object. - Transparent(attributes::kw::transparent), - /// Force every field to be extracted from item of source Python object. - ItemAll(attributes::kw::from_item_all), - /// Change the name of an enum variant in the generated error message. - ErrorAnnotation(LitStr), - /// Change the path for the pyo3 crate - Crate(CrateAttribute), - /// Converts the field idents according to the [RenamingRule] before extraction - RenameAll(RenameAllAttribute), -} - -impl Parse for ContainerPyO3Attribute { - fn parse(input: ParseStream<'_>) -> Result { - let lookahead = input.lookahead1(); - if lookahead.peek(attributes::kw::transparent) { - let kw: attributes::kw::transparent = input.parse()?; - Ok(ContainerPyO3Attribute::Transparent(kw)) - } else if lookahead.peek(attributes::kw::from_item_all) { - let kw: attributes::kw::from_item_all = input.parse()?; - Ok(ContainerPyO3Attribute::ItemAll(kw)) - } else if lookahead.peek(attributes::kw::annotation) { - let _: attributes::kw::annotation = input.parse()?; - let _: Token![=] = input.parse()?; - input.parse().map(ContainerPyO3Attribute::ErrorAnnotation) - } else if lookahead.peek(Token![crate]) { - input.parse().map(ContainerPyO3Attribute::Crate) - } else if lookahead.peek(attributes::kw::rename_all) { - input.parse().map(ContainerPyO3Attribute::RenameAll) - } else { - Err(lookahead.error()) - } - } -} - -impl ContainerOptions { - fn from_attrs(attrs: &[Attribute]) -> Result { - let mut options = ContainerOptions::default(); - - for attr in attrs { - if let Some(pyo3_attrs) = get_pyo3_options(attr)? { - for pyo3_attr in pyo3_attrs { - match pyo3_attr { - ContainerPyO3Attribute::Transparent(kw) => { - ensure_spanned!( - !options.transparent, - kw.span() => "`transparent` may only be provided once" - ); - options.transparent = true; - } - ContainerPyO3Attribute::ItemAll(kw) => { - ensure_spanned!( - options.from_item_all.is_none(), - kw.span() => "`from_item_all` may only be provided once" - ); - options.from_item_all = Some(kw); - } - ContainerPyO3Attribute::ErrorAnnotation(lit_str) => { - ensure_spanned!( - options.annotation.is_none(), - lit_str.span() => "`annotation` may only be provided once" - ); - options.annotation = Some(lit_str); - } - ContainerPyO3Attribute::Crate(path) => { - ensure_spanned!( - options.krate.is_none(), - path.span() => "`crate` may only be provided once" - ); - options.krate = Some(path); - } - ContainerPyO3Attribute::RenameAll(rename_all) => { - ensure_spanned!( - options.rename_all.is_none(), - rename_all.span() => "`rename_all` may only be provided once" - ); - options.rename_all = Some(rename_all); - } - } - } - } - } - Ok(options) - } -} - -/// Attributes for deriving FromPyObject scoped on fields. -#[derive(Clone, Debug)] -struct FieldPyO3Attributes { - getter: Option, - from_py_with: Option, - default: Option, -} - -#[derive(Clone, Debug)] -enum FieldGetter { - GetItem(Option), - GetAttr(Option), -} - -enum FieldPyO3Attribute { - Getter(FieldGetter), - FromPyWith(FromPyWithAttribute), - Default(DefaultAttribute), -} - -impl Parse for FieldPyO3Attribute { - fn parse(input: ParseStream<'_>) -> Result { - let lookahead = input.lookahead1(); - if lookahead.peek(attributes::kw::attribute) { - let _: attributes::kw::attribute = input.parse()?; - if input.peek(syn::token::Paren) { - let content; - let _ = parenthesized!(content in input); - let attr_name: LitStr = content.parse()?; - if !content.is_empty() { - return Err(content.error( - "expected at most one argument: `attribute` or `attribute(\"name\")`", - )); - } - ensure_spanned!( - !attr_name.value().is_empty(), - attr_name.span() => "attribute name cannot be empty" - ); - Ok(FieldPyO3Attribute::Getter(FieldGetter::GetAttr(Some( - attr_name, - )))) - } else { - Ok(FieldPyO3Attribute::Getter(FieldGetter::GetAttr(None))) - } - } else if lookahead.peek(attributes::kw::item) { - let _: attributes::kw::item = input.parse()?; - if input.peek(syn::token::Paren) { - let content; - let _ = parenthesized!(content in input); - let key = content.parse()?; - if !content.is_empty() { - return Err( - content.error("expected at most one argument: `item` or `item(key)`") - ); - } - Ok(FieldPyO3Attribute::Getter(FieldGetter::GetItem(Some(key)))) - } else { - Ok(FieldPyO3Attribute::Getter(FieldGetter::GetItem(None))) - } - } else if lookahead.peek(attributes::kw::from_py_with) { - input.parse().map(FieldPyO3Attribute::FromPyWith) - } else if lookahead.peek(Token![default]) { - input.parse().map(FieldPyO3Attribute::Default) - } else { - Err(lookahead.error()) - } - } -} - -impl FieldPyO3Attributes { - /// Extract the field attributes. - fn from_attrs(attrs: &[Attribute]) -> Result { - let mut getter = None; - let mut from_py_with = None; - let mut default = None; - - for attr in attrs { - if let Some(pyo3_attrs) = get_pyo3_options(attr)? { - for pyo3_attr in pyo3_attrs { - match pyo3_attr { - FieldPyO3Attribute::Getter(field_getter) => { - ensure_spanned!( - getter.is_none(), - attr.span() => "only one of `attribute` or `item` can be provided" - ); - getter = Some(field_getter); - } - FieldPyO3Attribute::FromPyWith(from_py_with_attr) => { - ensure_spanned!( - from_py_with.is_none(), - attr.span() => "`from_py_with` may only be provided once" - ); - from_py_with = Some(from_py_with_attr); - } - FieldPyO3Attribute::Default(default_attr) => { - ensure_spanned!( - default.is_none(), - attr.span() => "`default` may only be provided once" - ); - default = Some(default_attr); - } - } - } - } - } - - Ok(FieldPyO3Attributes { - getter, - from_py_with, - default, - }) + quote!(::std::result::Result::Ok(#self_ty{#fields})) } } @@ -693,7 +459,7 @@ fn verify_and_get_lifetime(generics: &syn::Generics) -> Result Foo(T)` /// adds `T: FromPyObject` on the derived implementation. pub fn build_derive_from_pyobject(tokens: &DeriveInput) -> Result { - let options = ContainerOptions::from_attrs(&tokens.attrs)?; + let options = ContainerAttributes::from_attrs(&tokens.attrs)?; let ctx = &Ctx::new(&options.krate, None); let Ctx { pyo3_path, .. } = &ctx; @@ -717,7 +483,7 @@ pub fn build_derive_from_pyobject(tokens: &DeriveInput) -> Result { let derives = match &tokens.data { syn::Data::Enum(en) => { - if options.transparent || options.annotation.is_some() { + if options.transparent.is_some() || options.annotation.is_some() { bail_spanned!(tokens.span() => "`transparent` or `annotation` is not supported \ at top level for enums"); } diff --git a/pyo3-macros-backend/src/intopyobject.rs b/pyo3-macros-backend/src/intopyobject.rs old mode 100755 new mode 100644 index 787d1dd6259..2532812f6e1 --- a/pyo3-macros-backend/src/intopyobject.rs +++ b/pyo3-macros-backend/src/intopyobject.rs @@ -1,173 +1,13 @@ -use crate::attributes::{self, get_pyo3_options, CrateAttribute, IntoPyWithAttribute}; -use crate::utils::Ctx; +use crate::attributes::{IntoPyWithAttribute, RenamingRule}; +use crate::derive_attributes::{ContainerAttributes, FieldAttributes}; +use crate::utils::{self, Ctx}; use proc_macro2::{Span, TokenStream}; -use quote::{format_ident, quote, quote_spanned}; +use quote::{format_ident, quote, quote_spanned, ToTokens}; use syn::ext::IdentExt; -use syn::parse::{Parse, ParseStream}; use syn::spanned::Spanned as _; -use syn::{ - parenthesized, parse_quote, Attribute, DataEnum, DeriveInput, Fields, Ident, Index, Result, - Token, -}; - -/// Attributes for deriving `IntoPyObject` scoped on containers. -enum ContainerPyO3Attribute { - /// Treat the Container as a Wrapper, directly convert its field into the output object. - Transparent(attributes::kw::transparent), - /// Change the path for the pyo3 crate - Crate(CrateAttribute), -} - -impl Parse for ContainerPyO3Attribute { - fn parse(input: ParseStream<'_>) -> Result { - let lookahead = input.lookahead1(); - if lookahead.peek(attributes::kw::transparent) { - let kw: attributes::kw::transparent = input.parse()?; - Ok(ContainerPyO3Attribute::Transparent(kw)) - } else if lookahead.peek(Token![crate]) { - input.parse().map(ContainerPyO3Attribute::Crate) - } else { - Err(lookahead.error()) - } - } -} - -#[derive(Default)] -struct ContainerOptions { - /// Treat the Container as a Wrapper, directly convert its field into the output object. - transparent: Option, - /// Change the path for the pyo3 crate - krate: Option, -} - -impl ContainerOptions { - fn from_attrs(attrs: &[Attribute]) -> Result { - let mut options = ContainerOptions::default(); - - for attr in attrs { - if let Some(pyo3_attrs) = get_pyo3_options(attr)? { - pyo3_attrs - .into_iter() - .try_for_each(|opt| options.set_option(opt))?; - } - } - Ok(options) - } - - fn set_option(&mut self, option: ContainerPyO3Attribute) -> syn::Result<()> { - macro_rules! set_option { - ($key:ident) => { - { - ensure_spanned!( - self.$key.is_none(), - $key.span() => concat!("`", stringify!($key), "` may only be specified once") - ); - self.$key = Some($key); - } - }; - } - - match option { - ContainerPyO3Attribute::Transparent(transparent) => set_option!(transparent), - ContainerPyO3Attribute::Crate(krate) => set_option!(krate), - } - Ok(()) - } -} - -#[derive(Debug, Clone)] -struct ItemOption { - field: Option, - span: Span, -} - -impl ItemOption { - fn span(&self) -> Span { - self.span - } -} - -enum FieldAttribute { - Item(ItemOption), - IntoPyWith(IntoPyWithAttribute), -} - -impl Parse for FieldAttribute { - fn parse(input: ParseStream<'_>) -> Result { - let lookahead = input.lookahead1(); - if lookahead.peek(attributes::kw::attribute) { - let attr: attributes::kw::attribute = input.parse()?; - bail_spanned!(attr.span => "`attribute` is not supported by `IntoPyObject`"); - } else if lookahead.peek(attributes::kw::item) { - let attr: attributes::kw::item = input.parse()?; - if input.peek(syn::token::Paren) { - let content; - let _ = parenthesized!(content in input); - let key = content.parse()?; - if !content.is_empty() { - return Err( - content.error("expected at most one argument: `item` or `item(key)`") - ); - } - Ok(FieldAttribute::Item(ItemOption { - field: Some(key), - span: attr.span, - })) - } else { - Ok(FieldAttribute::Item(ItemOption { - field: None, - span: attr.span, - })) - } - } else if lookahead.peek(attributes::kw::into_py_with) { - input.parse().map(FieldAttribute::IntoPyWith) - } else { - Err(lookahead.error()) - } - } -} - -#[derive(Clone, Debug, Default)] -struct FieldAttributes { - item: Option, - into_py_with: Option, -} - -impl FieldAttributes { - /// Extract the field attributes. - fn from_attrs(attrs: &[Attribute]) -> Result { - let mut options = FieldAttributes::default(); - - for attr in attrs { - if let Some(pyo3_attrs) = get_pyo3_options(attr)? { - pyo3_attrs - .into_iter() - .try_for_each(|opt| options.set_option(opt))?; - } - } - Ok(options) - } +use syn::{parse_quote, DataEnum, DeriveInput, Fields, Ident, Index, Result}; - fn set_option(&mut self, option: FieldAttribute) -> syn::Result<()> { - macro_rules! set_option { - ($key:ident) => { - { - ensure_spanned!( - self.$key.is_none(), - $key.span() => concat!("`", stringify!($key), "` may only be specified once") - ); - self.$key = Some($key); - } - }; - } - - match option { - FieldAttribute::Item(item) => set_option!(item), - FieldAttribute::IntoPyWith(into_py_with) => set_option!(into_py_with), - } - Ok(()) - } -} +struct ItemOption(Option); enum IntoPyObjectTypes { Transparent(syn::Type), @@ -225,6 +65,7 @@ struct Container<'a, const REF: bool> { path: syn::Path, receiver: Option, ty: ContainerType<'a>, + rename_rule: Option, } /// Construct a container based on fields, identifier and attributes. @@ -235,18 +76,22 @@ impl<'a, const REF: bool> Container<'a, REF> { receiver: Option, fields: &'a Fields, path: syn::Path, - options: ContainerOptions, + options: ContainerAttributes, ) -> Result { let style = match fields { Fields::Unnamed(unnamed) if !unnamed.unnamed.is_empty() => { + ensure_spanned!( + options.rename_all.is_none(), + options.rename_all.span() => "`rename_all` is useless on tuple structs and variants." + ); let mut tuple_fields = unnamed .unnamed .iter() .map(|field| { let attrs = FieldAttributes::from_attrs(&field.attrs)?; ensure_spanned!( - attrs.item.is_none(), - attrs.item.unwrap().span() => "`item` is not permitted on tuple struct elements." + attrs.getter.is_none(), + attrs.getter.unwrap().span() => "`item` and `attribute` are not permitted on tuple struct elements." ); Ok(TupleStructField { field, @@ -284,8 +129,12 @@ impl<'a, const REF: bool> Container<'a, REF> { let field = named.named.iter().next().unwrap(); let attrs = FieldAttributes::from_attrs(&field.attrs)?; ensure_spanned!( - attrs.item.is_none(), - attrs.item.unwrap().span() => "`transparent` structs may not have `item` for the inner field" + attrs.getter.is_none(), + attrs.getter.unwrap().span() => "`transparent` structs may not have `item` nor `attribute` for the inner field" + ); + ensure_spanned!( + options.rename_all.is_none(), + options.rename_all.span() => "`rename_all` is not permitted on `transparent` structs and variants" ); ensure_spanned!( attrs.into_py_with.is_none(), @@ -307,7 +156,12 @@ impl<'a, const REF: bool> Container<'a, REF> { Ok(NamedStructField { ident, field, - item: attrs.item, + item: attrs.getter.and_then(|getter| match getter { + crate::derive_attributes::FieldGetter::GetItem(_, lit) => { + Some(ItemOption(lit)) + } + crate::derive_attributes::FieldGetter::GetAttr(_, _) => None, + }), into_py_with: attrs.into_py_with, }) }) @@ -324,6 +178,7 @@ impl<'a, const REF: bool> Container<'a, REF> { path, receiver, ty: style, + rename_rule: options.rename_all.map(|v| v.value.rule), }; Ok(v) } @@ -407,9 +262,12 @@ impl<'a, const REF: bool> Container<'a, REF> { let key = f .item .as_ref() - .and_then(|item| item.field.as_ref()) - .map(|item| item.value()) - .unwrap_or_else(|| f.ident.unraw().to_string()); + .and_then(|item| item.0.as_ref()) + .map(|item| item.into_token_stream()) + .unwrap_or_else(|| { + let name = f.ident.unraw().to_string(); + self.rename_rule.map(|rule| utils::apply_renaming_rule(rule, &name)).unwrap_or(name).into_token_stream() + }); let value = Ident::new(&format!("arg{i}"), f.field.ty.span()); if let Some(expr_path) = f.into_py_with.as_ref().map(|i|&i.value) { @@ -519,7 +377,7 @@ impl<'a, const REF: bool> Enum<'a, REF> { .variants .iter() .map(|variant| { - let attrs = ContainerOptions::from_attrs(&variant.attrs)?; + let attrs = ContainerAttributes::from_attrs(&variant.attrs)?; let var_ident = &variant.ident; ensure_spanned!( @@ -582,7 +440,7 @@ fn verify_and_get_lifetime(generics: &syn::Generics) -> Option<&syn::LifetimePar } pub fn build_derive_into_pyobject(tokens: &DeriveInput) -> Result { - let options = ContainerOptions::from_attrs(&tokens.attrs)?; + let options = ContainerAttributes::from_attrs(&tokens.attrs)?; let ctx = &Ctx::new(&options.krate, None); let Ctx { pyo3_path, .. } = &ctx; @@ -614,6 +472,9 @@ pub fn build_derive_into_pyobject(tokens: &DeriveInput) -> Resu if options.transparent.is_some() { bail_spanned!(tokens.span() => "`transparent` is not supported at top level for enums"); } + if let Some(rename_all) = options.rename_all { + bail_spanned!(rename_all.span() => "`rename_all` is not supported at top level for enums"); + } let en = Enum::::new(en, &tokens.ident)?; en.build(ctx) } diff --git a/pyo3-macros-backend/src/introspection.rs b/pyo3-macros-backend/src/introspection.rs new file mode 100644 index 00000000000..4888417cb08 --- /dev/null +++ b/pyo3-macros-backend/src/introspection.rs @@ -0,0 +1,335 @@ +//! Generates introspection data i.e. JSON strings in the .pyo3i0 section. +//! +//! There is a JSON per PyO3 proc macro (pyclass, pymodule, pyfunction...). +//! +//! These JSON blobs can refer to each others via the _PYO3_INTROSPECTION_ID constants +//! providing unique ids for each element. +//! +//! The JSON blobs format must be synchronized with the `pyo3_introspection::introspection.rs::Chunk` +//! type that is used to parse them. + +use crate::method::{FnArg, RegularArg}; +use crate::pyfunction::FunctionSignature; +use crate::utils::PyO3CratePath; +use proc_macro2::{Span, TokenStream}; +use quote::{format_ident, quote, ToTokens}; +use std::borrow::Cow; +use std::collections::hash_map::DefaultHasher; +use std::collections::HashMap; +use std::hash::{Hash, Hasher}; +use std::mem::take; +use std::sync::atomic::{AtomicUsize, Ordering}; +use syn::{Attribute, Ident}; + +static GLOBAL_COUNTER_FOR_UNIQUE_NAMES: AtomicUsize = AtomicUsize::new(0); + +pub fn module_introspection_code<'a>( + pyo3_crate_path: &PyO3CratePath, + name: &str, + members: impl IntoIterator, + members_cfg_attrs: impl IntoIterator>, +) -> TokenStream { + IntrospectionNode::Map( + [ + ("type", IntrospectionNode::String("module".into())), + ("id", IntrospectionNode::IntrospectionId(None)), + ("name", IntrospectionNode::String(name.into())), + ( + "members", + IntrospectionNode::List( + members + .into_iter() + .zip(members_cfg_attrs) + .filter_map(|(member, attributes)| { + if attributes.is_empty() { + Some(IntrospectionNode::IntrospectionId(Some(member))) + } else { + None // TODO: properly interpret cfg attributes + } + }) + .collect(), + ), + ), + ] + .into(), + ) + .emit(pyo3_crate_path) +} + +pub fn class_introspection_code( + pyo3_crate_path: &PyO3CratePath, + ident: &Ident, + name: &str, +) -> TokenStream { + IntrospectionNode::Map( + [ + ("type", IntrospectionNode::String("class".into())), + ("id", IntrospectionNode::IntrospectionId(Some(ident))), + ("name", IntrospectionNode::String(name.into())), + ] + .into(), + ) + .emit(pyo3_crate_path) +} + +pub fn function_introspection_code( + pyo3_crate_path: &PyO3CratePath, + ident: &Ident, + name: &str, + signature: &FunctionSignature<'_>, +) -> TokenStream { + IntrospectionNode::Map( + [ + ("type", IntrospectionNode::String("function".into())), + ("id", IntrospectionNode::IntrospectionId(Some(ident))), + ("name", IntrospectionNode::String(name.into())), + ("arguments", arguments_introspection_data(signature)), + ] + .into(), + ) + .emit(pyo3_crate_path) +} + +fn arguments_introspection_data<'a>(signature: &'a FunctionSignature<'a>) -> IntrospectionNode<'a> { + let mut argument_desc = signature.arguments.iter().filter_map(|arg| { + if let FnArg::Regular(arg) = arg { + Some(arg) + } else { + None + } + }); + + let mut posonlyargs = Vec::new(); + let mut args = Vec::new(); + let mut vararg = None; + let mut kwonlyargs = Vec::new(); + let mut kwarg = None; + + for (i, param) in signature + .python_signature + .positional_parameters + .iter() + .enumerate() + { + let arg_desc = if let Some(arg_desc) = argument_desc.next() { + arg_desc + } else { + panic!("Less arguments than in python signature"); + }; + let arg = argument_introspection_data(param, arg_desc); + if i < signature.python_signature.positional_only_parameters { + posonlyargs.push(arg); + } else { + args.push(arg) + } + } + + if let Some(param) = &signature.python_signature.varargs { + vararg = Some(IntrospectionNode::Map( + [("name", IntrospectionNode::String(param.into()))].into(), + )); + } + + for (param, _) in &signature.python_signature.keyword_only_parameters { + let arg_desc = if let Some(arg_desc) = argument_desc.next() { + arg_desc + } else { + panic!("Less arguments than in python signature"); + }; + kwonlyargs.push(argument_introspection_data(param, arg_desc)); + } + + if let Some(param) = &signature.python_signature.kwargs { + kwarg = Some(IntrospectionNode::Map( + [ + ("name", IntrospectionNode::String(param.into())), + ("kind", IntrospectionNode::String("VAR_KEYWORD".into())), + ] + .into(), + )); + } + + let mut map = HashMap::new(); + if !posonlyargs.is_empty() { + map.insert("posonlyargs", IntrospectionNode::List(posonlyargs)); + } + if !args.is_empty() { + map.insert("args", IntrospectionNode::List(args)); + } + if let Some(vararg) = vararg { + map.insert("vararg", vararg); + } + if !kwonlyargs.is_empty() { + map.insert("kwonlyargs", IntrospectionNode::List(kwonlyargs)); + } + if let Some(kwarg) = kwarg { + map.insert("kwarg", kwarg); + } + IntrospectionNode::Map(map) +} + +fn argument_introspection_data<'a>( + name: &'a str, + desc: &'a RegularArg<'_>, +) -> IntrospectionNode<'a> { + let mut params: HashMap<_, _> = [("name", IntrospectionNode::String(name.into()))].into(); + if desc.default_value.is_some() { + params.insert( + "default", + IntrospectionNode::String(desc.default_value().into()), + ); + } + IntrospectionNode::Map(params) +} + +enum IntrospectionNode<'a> { + String(Cow<'a, str>), + IntrospectionId(Option<&'a Ident>), + Map(HashMap<&'static str, IntrospectionNode<'a>>), + List(Vec>), +} + +impl IntrospectionNode<'_> { + fn emit(self, pyo3_crate_path: &PyO3CratePath) -> TokenStream { + let mut content = ConcatenationBuilder::default(); + self.add_to_serialization(&mut content); + let content = content.into_token_stream(pyo3_crate_path); + + let static_name = format_ident!("PYO3_INTROSPECTION_0_{}", unique_element_id()); + // #[no_mangle] is required to make sure some linkers like Linux ones do not mangle the section name too. + quote! { + const _: () = { + #[used] + #[no_mangle] + static #static_name: &'static str = #content; + }; + } + } + + fn add_to_serialization(self, content: &mut ConcatenationBuilder) { + match self { + Self::String(string) => { + content.push_str_to_escape(&string); + } + Self::IntrospectionId(ident) => { + content.push_str("\""); + content.push_tokens(if let Some(ident) = ident { + quote! { #ident::_PYO3_INTROSPECTION_ID } + } else { + Ident::new("_PYO3_INTROSPECTION_ID", Span::call_site()).into_token_stream() + }); + content.push_str("\""); + } + Self::Map(map) => { + content.push_str("{"); + for (i, (key, value)) in map.into_iter().enumerate() { + if i > 0 { + content.push_str(","); + } + content.push_str_to_escape(key); + content.push_str(":"); + value.add_to_serialization(content); + } + content.push_str("}"); + } + Self::List(list) => { + content.push_str("["); + for (i, value) in list.into_iter().enumerate() { + if i > 0 { + content.push_str(","); + } + value.add_to_serialization(content); + } + content.push_str("]"); + } + } + } +} + +#[derive(Default)] +struct ConcatenationBuilder { + elements: Vec, + current_string: String, +} + +impl ConcatenationBuilder { + fn push_tokens(&mut self, token_stream: TokenStream) { + if !self.current_string.is_empty() { + self.elements.push(ConcatenationBuilderElement::String(take( + &mut self.current_string, + ))); + } + self.elements + .push(ConcatenationBuilderElement::TokenStream(token_stream)); + } + + fn push_str(&mut self, value: &str) { + self.current_string.push_str(value); + } + + fn push_str_to_escape(&mut self, value: &str) { + self.current_string.push('"'); + for c in value.chars() { + match c { + '\\' => self.current_string.push_str("\\\\"), + '"' => self.current_string.push_str("\\\""), + c => { + if c < char::from(32) { + panic!("ASCII chars below 32 are not allowed") + } else { + self.current_string.push(c); + } + } + } + } + self.current_string.push('"'); + } + + fn into_token_stream(self, pyo3_crate_path: &PyO3CratePath) -> TokenStream { + let mut elements = self.elements; + if !self.current_string.is_empty() { + elements.push(ConcatenationBuilderElement::String(self.current_string)); + } + + if let [ConcatenationBuilderElement::String(string)] = elements.as_slice() { + // We avoid the const_concat! macro if there is only a single string + return string.to_token_stream(); + } + + quote! { + #pyo3_crate_path::impl_::concat::const_concat!(#(#elements , )*) + } + } +} + +enum ConcatenationBuilderElement { + String(String), + TokenStream(TokenStream), +} + +impl ToTokens for ConcatenationBuilderElement { + fn to_tokens(&self, tokens: &mut TokenStream) { + match self { + Self::String(s) => s.to_tokens(tokens), + Self::TokenStream(ts) => ts.to_tokens(tokens), + } + } +} + +/// Generates a new unique identifier for linking introspection objects together +pub fn introspection_id_const() -> TokenStream { + let id = unique_element_id().to_string(); + quote! { + #[doc(hidden)] + pub const _PYO3_INTROSPECTION_ID: &'static str = #id; + } +} + +fn unique_element_id() -> u64 { + let mut hasher = DefaultHasher::new(); + format!("{:?}", Span::call_site()).hash(&mut hasher); // Distinguishes between call sites + GLOBAL_COUNTER_FOR_UNIQUE_NAMES + .fetch_add(1, Ordering::Relaxed) + .hash(&mut hasher); // If there are multiple elements in the same call site + hasher.finish() +} diff --git a/pyo3-macros-backend/src/lib.rs b/pyo3-macros-backend/src/lib.rs index 7893a94af98..8919830dda9 100644 --- a/pyo3-macros-backend/src/lib.rs +++ b/pyo3-macros-backend/src/lib.rs @@ -9,8 +9,11 @@ mod utils; mod attributes; +mod derive_attributes; mod frompyobject; mod intopyobject; +#[cfg(feature = "experimental-inspect")] +mod introspection; mod konst; mod method; mod module; diff --git a/pyo3-macros-backend/src/method.rs b/pyo3-macros-backend/src/method.rs index a1d7a95df35..0ab5135e93a 100644 --- a/pyo3-macros-backend/src/method.rs +++ b/pyo3-macros-backend/src/method.rs @@ -27,6 +27,52 @@ pub struct RegularArg<'a> { pub option_wrapped_type: Option<&'a syn::Type>, } +impl RegularArg<'_> { + pub fn default_value(&self) -> String { + if let Self { + default_value: Some(arg_default), + .. + } = self + { + match arg_default { + // literal values + syn::Expr::Lit(syn::ExprLit { lit, .. }) => match lit { + syn::Lit::Str(s) => s.token().to_string(), + syn::Lit::Char(c) => c.token().to_string(), + syn::Lit::Int(i) => i.base10_digits().to_string(), + syn::Lit::Float(f) => f.base10_digits().to_string(), + syn::Lit::Bool(b) => { + if b.value() { + "True".to_string() + } else { + "False".to_string() + } + } + _ => "...".to_string(), + }, + // None + syn::Expr::Path(syn::ExprPath { qself, path, .. }) + if qself.is_none() && path.is_ident("None") => + { + "None".to_string() + } + // others, unsupported yet so defaults to `...` + _ => "...".to_string(), + } + } else if let RegularArg { + option_wrapped_type: Some(..), + .. + } = self + { + // functions without a `#[pyo3(signature = (...))]` option + // will treat trailing `Option` arguments as having a default of `None` + "None".to_string() + } else { + "...".to_string() + } + } +} + /// Pythons *args argument #[derive(Clone, Debug)] pub struct VarargsArg<'a> { @@ -53,6 +99,7 @@ pub struct PyArg<'a> { pub ty: &'a syn::Type, } +#[allow(clippy::large_enum_variant)] // See #5039 #[derive(Clone, Debug)] pub enum FnArg<'a> { Regular(RegularArg<'a>), @@ -176,6 +223,14 @@ impl<'a> FnArg<'a> { } } } + + pub fn default_value(&self) -> String { + if let Self::Regular(args) = self { + args.default_value() + } else { + "...".to_string() + } + } } fn handle_argument_error(pat: &syn::Pat) -> syn::Error { @@ -592,10 +647,10 @@ impl<'a> FnSpec<'a> { .fold(first.span(), |s, next| s.join(next.span()).unwrap_or(s)); let span = span.join(last.span()).unwrap_or(span); // List all the attributes in the error message - let mut msg = format!("`{}` may not be combined with", first); + let mut msg = format!("`{first}` may not be combined with"); let mut is_first = true; for attr in &*rest { - msg.push_str(&format!(" `{}`", attr)); + msg.push_str(&format!(" `{attr}`")); if is_first { is_first = false; } else { @@ -605,7 +660,7 @@ impl<'a> FnSpec<'a> { if !rest.is_empty() { msg.push_str(" and"); } - msg.push_str(&format!(" `{}`", last)); + msg.push_str(&format!(" `{last}`")); bail_spanned!(span => msg) } }; diff --git a/pyo3-macros-backend/src/module.rs b/pyo3-macros-backend/src/module.rs index d9fac3cbd7b..860a3b6d857 100644 --- a/pyo3-macros-backend/src/module.rs +++ b/pyo3-macros-backend/src/module.rs @@ -1,5 +1,7 @@ //! Code generation for the function that initializes a python module and adds classes and function. +#[cfg(feature = "experimental-inspect")] +use crate::introspection::{introspection_id_const, module_introspection_code}; use crate::{ attributes::{ self, kw, take_attributes, take_pyo3_options, CrateAttribute, GILUsedAttribute, @@ -147,6 +149,8 @@ pub fn pymodule_module_impl( } let mut pymodule_init = None; + let mut module_consts = Vec::new(); + let mut module_consts_cfg_attrs = Vec::new(); for item in &mut *items { match item { @@ -166,7 +170,7 @@ pub fn pymodule_module_impl( Item::Fn(item_fn) => { ensure_spanned!( !has_attribute(&item_fn.attrs, "pymodule_export"), - item.span() => "`#[pymodule_export]` may only be used on `use` statements" + item.span() => "`#[pymodule_export]` may only be used on `use` or `const` statements" ); let is_pymodule_init = find_and_remove_attribute(&mut item_fn.attrs, "pymodule_init"); @@ -197,7 +201,7 @@ pub fn pymodule_module_impl( Item::Struct(item_struct) => { ensure_spanned!( !has_attribute(&item_struct.attrs, "pymodule_export"), - item.span() => "`#[pymodule_export]` may only be used on `use` statements" + item.span() => "`#[pymodule_export]` may only be used on `use` or `const` statements" ); if has_attribute(&item_struct.attrs, "pyclass") || has_attribute_with_namespace( @@ -225,7 +229,7 @@ pub fn pymodule_module_impl( Item::Enum(item_enum) => { ensure_spanned!( !has_attribute(&item_enum.attrs, "pymodule_export"), - item.span() => "`#[pymodule_export]` may only be used on `use` statements" + item.span() => "`#[pymodule_export]` may only be used on `use` or `const` statements" ); if has_attribute(&item_enum.attrs, "pyclass") || has_attribute_with_namespace(&item_enum.attrs, Some(pyo3_path), &["pyclass"]) @@ -249,7 +253,7 @@ pub fn pymodule_module_impl( Item::Mod(item_mod) => { ensure_spanned!( !has_attribute(&item_mod.attrs, "pymodule_export"), - item.span() => "`#[pymodule_export]` may only be used on `use` statements" + item.span() => "`#[pymodule_export]` may only be used on `use` or `const` statements" ); if has_attribute(&item_mod.attrs, "pymodule") || has_attribute_with_namespace(&item_mod.attrs, Some(pyo3_path), &["pymodule"]) @@ -276,67 +280,83 @@ pub fn pymodule_module_impl( Item::ForeignMod(item) => { ensure_spanned!( !has_attribute(&item.attrs, "pymodule_export"), - item.span() => "`#[pymodule_export]` may only be used on `use` statements" + item.span() => "`#[pymodule_export]` may only be used on `use` or `const` statements" ); } Item::Trait(item) => { ensure_spanned!( !has_attribute(&item.attrs, "pymodule_export"), - item.span() => "`#[pymodule_export]` may only be used on `use` statements" + item.span() => "`#[pymodule_export]` may only be used on `use` or `const` statements" ); } Item::Const(item) => { - ensure_spanned!( - !has_attribute(&item.attrs, "pymodule_export"), - item.span() => "`#[pymodule_export]` may only be used on `use` statements" - ); + if !find_and_remove_attribute(&mut item.attrs, "pymodule_export") { + continue; + } + + module_consts.push(item.ident.clone()); + module_consts_cfg_attrs.push(get_cfg_attributes(&item.attrs)); } Item::Static(item) => { ensure_spanned!( !has_attribute(&item.attrs, "pymodule_export"), - item.span() => "`#[pymodule_export]` may only be used on `use` statements" + item.span() => "`#[pymodule_export]` may only be used on `use` or `const` statements" ); } Item::Macro(item) => { ensure_spanned!( !has_attribute(&item.attrs, "pymodule_export"), - item.span() => "`#[pymodule_export]` may only be used on `use` statements" + item.span() => "`#[pymodule_export]` may only be used on `use` or `const` statements" ); } Item::ExternCrate(item) => { ensure_spanned!( !has_attribute(&item.attrs, "pymodule_export"), - item.span() => "`#[pymodule_export]` may only be used on `use` statements" + item.span() => "`#[pymodule_export]` may only be used on `use` or `const` statements" ); } Item::Impl(item) => { ensure_spanned!( !has_attribute(&item.attrs, "pymodule_export"), - item.span() => "`#[pymodule_export]` may only be used on `use` statements" + item.span() => "`#[pymodule_export]` may only be used on `use` or `const` statements" ); } Item::TraitAlias(item) => { ensure_spanned!( !has_attribute(&item.attrs, "pymodule_export"), - item.span() => "`#[pymodule_export]` may only be used on `use` statements" + item.span() => "`#[pymodule_export]` may only be used on `use` or `const` statements" ); } Item::Type(item) => { ensure_spanned!( !has_attribute(&item.attrs, "pymodule_export"), - item.span() => "`#[pymodule_export]` may only be used on `use` statements" + item.span() => "`#[pymodule_export]` may only be used on `use` or `const` statements" ); } Item::Union(item) => { ensure_spanned!( !has_attribute(&item.attrs, "pymodule_export"), - item.span() => "`#[pymodule_export]` may only be used on `use` statements" + item.span() => "`#[pymodule_export]` may only be used on `use` or `const` statements" ); } _ => (), } } + #[cfg(feature = "experimental-inspect")] + let introspection = module_introspection_code( + pyo3_path, + &name.to_string(), + &module_items, + &module_items_cfg_attrs, + ); + #[cfg(not(feature = "experimental-inspect"))] + let introspection = quote! {}; + #[cfg(feature = "experimental-inspect")] + let introspection_id = introspection_id_const(); + #[cfg(not(feature = "experimental-inspect"))] + let introspection_id = quote! {}; + let module_def = quote! {{ use #pyo3_path::impl_::pymodule as impl_; const INITIALIZER: impl_::ModuleInitializer = impl_::ModuleInitializer(__pyo3_pymodule); @@ -356,12 +376,16 @@ pub fn pymodule_module_impl( options.gil_used.map_or(true, |op| op.value.value), ); + let module_consts_names = module_consts.iter().map(|i| i.unraw().to_string()); + Ok(quote!( #(#attrs)* #vis #mod_token #ident { #(#items)* #initialization + #introspection + #introspection_id fn __pyo3_pymodule(module: &#pyo3_path::Bound<'_, #pyo3_path::types::PyModule>) -> #pyo3_path::PyResult<()> { use #pyo3_path::impl_::pymodule::PyAddToModule; @@ -369,6 +393,12 @@ pub fn pymodule_module_impl( #(#module_items_cfg_attrs)* #module_items::_PYO3_DEF.add_to_module(module)?; )* + + #( + #(#module_consts_cfg_attrs)* + #pyo3_path::types::PyModuleMethods::add(module, #module_consts_names, #module_consts)?; + )* + #pymodule_init ::std::result::Result::Ok(()) } @@ -401,6 +431,15 @@ pub fn pymodule_function_impl( options.gil_used.map_or(true, |op| op.value.value), ); + #[cfg(feature = "experimental-inspect")] + let introspection = module_introspection_code(pyo3_path, &name.to_string(), &[], &[]); + #[cfg(not(feature = "experimental-inspect"))] + let introspection = quote! {}; + #[cfg(feature = "experimental-inspect")] + let introspection_id = introspection_id_const(); + #[cfg(not(feature = "experimental-inspect"))] + let introspection_id = quote! {}; + // Module function called with optional Python<'_> marker as first arg, followed by the module. let mut module_args = Vec::new(); if function.sig.inputs.len() == 2 { @@ -413,6 +452,8 @@ pub fn pymodule_function_impl( #[doc(hidden)] #vis mod #ident { #initialization + #introspection + #introspection_id } // Generate the definition inside an anonymous function in the same scope as the original function - @@ -447,7 +488,7 @@ fn module_initialization( gil_used: bool, ) -> TokenStream { let Ctx { pyo3_path, .. } = ctx; - let pyinit_symbol = format!("PyInit_{}", name); + let pyinit_symbol = format!("PyInit_{name}"); let name = name.to_string(); let pyo3_name = LitCStr::new(CString::new(name).unwrap(), Span::call_site(), ctx); diff --git a/pyo3-macros-backend/src/params.rs b/pyo3-macros-backend/src/params.rs index ae7a6c916a8..087e97f500c 100644 --- a/pyo3-macros-backend/src/params.rs +++ b/pyo3-macros-backend/src/params.rs @@ -1,4 +1,4 @@ -use crate::utils::{deprecated_from_py_with, Ctx, TypeExt as _}; +use crate::utils::{Ctx, TypeExt as _}; use crate::{ attributes::FromPyWithAttribute, method::{FnArg, FnSpec, RegularArg}, @@ -62,9 +62,7 @@ pub fn impl_arg_params( .filter_map(|(i, arg)| { let from_py_with = &arg.from_py_with()?.value; let from_py_with_holder = format_ident!("from_py_with_{}", i); - let d = deprecated_from_py_with(from_py_with).unwrap_or_default(); Some(quote_spanned! { from_py_with.span() => - #d let #from_py_with_holder = #from_py_with; }) }) @@ -238,7 +236,10 @@ pub(crate) fn impl_regular_arg_param( // Use this macro inside this function, to ensure that all code generated here is associated // with the function argument - let use_probe = quote!(use #pyo3_path::impl_::pyclass::Probe as _;); + let use_probe = quote! { + #[allow(unused_imports)] + use #pyo3_path::impl_::pyclass::Probe as _; + }; macro_rules! quote_arg_span { ($($tokens:tt)*) => { quote_spanned!(arg.ty.span() => { #use_probe $($tokens)* }) } } diff --git a/pyo3-macros-backend/src/pyclass.rs b/pyo3-macros-backend/src/pyclass.rs index 7512d63c9fb..ab86138338b 100644 --- a/pyo3-macros-backend/src/pyclass.rs +++ b/pyo3-macros-backend/src/pyclass.rs @@ -14,6 +14,8 @@ use crate::attributes::{ FreelistAttribute, ModuleAttribute, NameAttribute, NameLitStr, RenameAllAttribute, StrFormatterAttribute, }; +#[cfg(feature = "experimental-inspect")] +use crate::introspection::{class_introspection_code, introspection_id_const}; use crate::konst::{ConstAttributes, ConstSpec}; use crate::method::{FnArg, FnSpec, PyArg, RegularArg}; use crate::pyfunction::ConstructorAttribute; @@ -23,7 +25,7 @@ use crate::pymethod::{ MethodAndSlotDef, PropertyType, SlotDef, __GETITEM__, __HASH__, __INT__, __LEN__, __REPR__, __RICHCMP__, __STR__, }; -use crate::pyversions::is_abi3_before; +use crate::pyversions::{is_abi3_before, is_py_before}; use crate::utils::{self, apply_renaming_rule, Ctx, LitCStr, PythonDoc}; use crate::PyFunctionOptions; @@ -69,6 +71,7 @@ pub struct PyClassPyO3Options { pub freelist: Option, pub frozen: Option, pub hash: Option, + pub immutable_type: Option, pub mapping: Option, pub module: Option, pub name: Option, @@ -80,6 +83,7 @@ pub struct PyClassPyO3Options { pub subclass: Option, pub unsendable: Option, pub weakref: Option, + pub generic: Option, } pub enum PyClassPyO3Option { @@ -92,6 +96,7 @@ pub enum PyClassPyO3Option { Frozen(kw::frozen), GetAll(kw::get_all), Hash(kw::hash), + ImmutableType(kw::immutable_type), Mapping(kw::mapping), Module(ModuleAttribute), Name(NameAttribute), @@ -103,6 +108,7 @@ pub enum PyClassPyO3Option { Subclass(kw::subclass), Unsendable(kw::unsendable), Weakref(kw::weakref), + Generic(kw::generic), } impl Parse for PyClassPyO3Option { @@ -126,6 +132,8 @@ impl Parse for PyClassPyO3Option { input.parse().map(PyClassPyO3Option::GetAll) } else if lookahead.peek(attributes::kw::hash) { input.parse().map(PyClassPyO3Option::Hash) + } else if lookahead.peek(attributes::kw::immutable_type) { + input.parse().map(PyClassPyO3Option::ImmutableType) } else if lookahead.peek(attributes::kw::mapping) { input.parse().map(PyClassPyO3Option::Mapping) } else if lookahead.peek(attributes::kw::module) { @@ -148,6 +156,8 @@ impl Parse for PyClassPyO3Option { input.parse().map(PyClassPyO3Option::Unsendable) } else if lookahead.peek(attributes::kw::weakref) { input.parse().map(PyClassPyO3Option::Weakref) + } else if lookahead.peek(attributes::kw::generic) { + input.parse().map(PyClassPyO3Option::Generic) } else { Err(lookahead.error()) } @@ -201,6 +211,13 @@ impl PyClassPyO3Options { PyClassPyO3Option::Freelist(freelist) => set_option!(freelist), PyClassPyO3Option::Frozen(frozen) => set_option!(frozen), PyClassPyO3Option::GetAll(get_all) => set_option!(get_all), + PyClassPyO3Option::ImmutableType(immutable_type) => { + ensure_spanned!( + !(is_py_before(3, 10) || is_abi3_before(3, 14)), + immutable_type.span() => "`immutable_type` requires Python >= 3.10 or >= 3.14 (ABI3)" + ); + set_option!(immutable_type) + } PyClassPyO3Option::Hash(hash) => set_option!(hash), PyClassPyO3Option::Mapping(mapping) => set_option!(mapping), PyClassPyO3Option::Module(module) => set_option!(module), @@ -219,6 +236,7 @@ impl PyClassPyO3Options { ); set_option!(weakref); } + PyClassPyO3Option::Generic(generic) => set_option!(generic), } Ok(()) } @@ -416,6 +434,21 @@ fn impl_class( } } + let mut default_methods = descriptors_to_items( + cls, + args.options.rename_all.as_ref(), + args.options.frozen, + field_options, + ctx, + )?; + + let (default_class_geitem, default_class_geitem_method) = + pyclass_class_geitem(&args.options, &syn::parse_quote!(#cls), ctx)?; + + if let Some(default_class_geitem_method) = default_class_geitem_method { + default_methods.push(default_class_geitem_method); + } + let (default_str, default_str_slot) = implement_pyclass_str(&args.options, &syn::parse_quote!(#cls), ctx); @@ -430,21 +463,9 @@ fn impl_class( slots.extend(default_hash_slot); slots.extend(default_str_slot); - let py_class_impl = PyClassImplsBuilder::new( - cls, - args, - methods_type, - descriptors_to_items( - cls, - args.options.rename_all.as_ref(), - args.options.frozen, - field_options, - ctx, - )?, - slots, - ) - .doc(doc) - .impl_all(ctx)?; + let py_class_impl = PyClassImplsBuilder::new(cls, args, methods_type, default_methods, slots) + .doc(doc) + .impl_all(ctx)?; Ok(quote! { impl #pyo3_path::types::DerefToPyAny for #cls {} @@ -459,6 +480,7 @@ fn impl_class( #default_richcmp #default_hash #default_str + #default_class_geitem } }) } @@ -501,6 +523,10 @@ pub fn build_py_enum( bail_spanned!(enum_.brace_token.span.join() => "#[pyclass] can't be used on enums without any variants"); } + if let Some(generic) = &args.options.generic { + bail_spanned!(generic.span() => "enums do not support #[pyclass(generic)]"); + } + let doc = utils::get_doc(&enum_.attrs, None, ctx); let enum_ = PyClassEnum::new(enum_)?; impl_enum(enum_, &args, doc, method_type, ctx) @@ -1027,35 +1053,6 @@ fn impl_complex_enum( ) .doc(doc); - // Need to customize the into_py impl so that it returns the variant PyClass - let enum_into_py_impl = { - let match_arms: Vec = variants - .iter() - .map(|variant| { - let variant_ident = variant.get_ident(); - let variant_cls = gen_complex_enum_variant_class_ident(cls, variant.get_ident()); - quote! { - #cls::#variant_ident { .. } => { - let pyclass_init = <#pyo3_path::PyClassInitializer as ::std::convert::From>::from(self).add_subclass(#variant_cls); - let variant_value = #pyo3_path::Py::new(py, pyclass_init).unwrap(); - #pyo3_path::IntoPy::into_py(variant_value, py) - } - } - }) - .collect(); - - quote! { - #[allow(deprecated)] - impl #pyo3_path::IntoPy<#pyo3_path::PyObject> for #cls { - fn into_py(self, py: #pyo3_path::Python) -> #pyo3_path::PyObject { - match self { - #(#match_arms)* - } - } - } - } - }; - let enum_into_pyobject_impl = { let match_arms = variants .iter() @@ -1091,11 +1088,11 @@ fn impl_complex_enum( let pyclass_impls: TokenStream = [ impl_builder.impl_pyclass(ctx), impl_builder.impl_extractext(ctx), - enum_into_py_impl, enum_into_pyobject_impl, impl_builder.impl_pyclassimpl(ctx)?, impl_builder.impl_add_to_module(ctx), impl_builder.impl_freelist(ctx), + impl_builder.impl_introspection(ctx), ] .into_iter() .collect(); @@ -1188,15 +1185,16 @@ fn impl_complex_enum_variant_cls( fn impl_complex_enum_variant_match_args( ctx @ Ctx { pyo3_path, .. }: &Ctx, variant_cls_type: &syn::Type, - field_names: &mut Vec, + field_names: &[Ident], ) -> syn::Result<(MethodAndMethodDef, syn::ImplItemFn)> { let ident = format_ident!("__match_args__"); + let field_names_unraw = field_names.iter().map(|name| name.unraw()); let mut match_args_impl: syn::ImplItemFn = { parse_quote! { #[classattr] fn #ident(py: #pyo3_path::Python<'_>) -> #pyo3_path::PyResult<#pyo3_path::Bound<'_, #pyo3_path::types::PyTuple>> { #pyo3_path::types::PyTuple::new::<&str, _>(py, [ - #(stringify!(#field_names),)* + #(stringify!(#field_names_unraw),)* ]) } } @@ -1257,7 +1255,7 @@ fn impl_complex_enum_struct_variant_cls( } let (variant_match_args, match_args_const_impl) = - impl_complex_enum_variant_match_args(ctx, &variant_cls_type, &mut field_names)?; + impl_complex_enum_variant_match_args(ctx, &variant_cls_type, &field_names)?; field_getters.push(variant_match_args); @@ -1432,7 +1430,7 @@ fn impl_complex_enum_tuple_variant_cls( slots.push(variant_getitem); let (variant_match_args, match_args_method_impl) = - impl_complex_enum_variant_match_args(ctx, &variant_cls_type, &mut field_names)?; + impl_complex_enum_variant_match_args(ctx, &variant_cls_type, &field_names)?; field_getters.push(variant_match_args); @@ -1495,7 +1493,7 @@ fn generate_default_protocol_slot( slot.generate_type_slot( &syn::parse_quote!(#cls), &spec, - &format!("__default_{}__", name), + &format!("__default_{name}__"), ctx, ) } @@ -1743,7 +1741,7 @@ fn complex_enum_variant_field_getter<'a>( let spec = FnSpec { tp: crate::method::FnType::Getter(self_type.clone()), name: field_name, - python_name: field_name.clone(), + python_name: field_name.unraw(), signature, convention: crate::method::CallingConvention::Noargs, text_signature: None, @@ -2023,6 +2021,46 @@ fn pyclass_hash( } } +fn pyclass_class_geitem( + options: &PyClassPyO3Options, + cls: &syn::Type, + ctx: &Ctx, +) -> Result<(Option, Option)> { + let Ctx { pyo3_path, .. } = ctx; + match options.generic { + Some(_) => { + let ident = format_ident!("__class_getitem__"); + let mut class_geitem_impl: syn::ImplItemFn = { + parse_quote! { + #[classmethod] + fn #ident<'py>( + cls: &#pyo3_path::Bound<'py, #pyo3_path::types::PyType>, + key: &#pyo3_path::Bound<'py, #pyo3_path::types::PyAny> + ) -> #pyo3_path::PyResult<#pyo3_path::Bound<'py, #pyo3_path::types::PyGenericAlias>> { + #pyo3_path::types::PyGenericAlias::new(cls.py(), cls.as_any(), key) + } + } + }; + + let spec = FnSpec::parse( + &mut class_geitem_impl.sig, + &mut class_geitem_impl.attrs, + Default::default(), + )?; + + let class_geitem_method = crate::pymethod::impl_py_method_def( + cls, + &spec, + &spec.get_doc(&class_geitem_impl.attrs, ctx), + Some(quote!(#pyo3_path::ffi::METH_CLASS)), + ctx, + )?; + Ok((Some(class_geitem_impl), Some(class_geitem_method))) + } + None => Ok((None, None)), + } +} + /// Implements most traits used by `#[pyclass]`. /// /// Specifically, it implements traits that only depend on class name, @@ -2063,17 +2101,17 @@ impl<'a> PyClassImplsBuilder<'a> { } fn impl_all(&self, ctx: &Ctx) -> Result { - let tokens = [ + Ok([ self.impl_pyclass(ctx), self.impl_extractext(ctx), self.impl_into_py(ctx), self.impl_pyclassimpl(ctx)?, self.impl_add_to_module(ctx), self.impl_freelist(ctx), + self.impl_introspection(ctx), ] .into_iter() - .collect(); - Ok(tokens) + .collect()) } fn impl_pyclass(&self, ctx: &Ctx) -> TokenStream { @@ -2139,13 +2177,6 @@ impl<'a> PyClassImplsBuilder<'a> { // If #cls is not extended type, we allow Self->PyObject conversion if attr.options.extends.is_none() { quote! { - #[allow(deprecated)] - impl #pyo3_path::IntoPy<#pyo3_path::PyObject> for #cls { - fn into_py(self, py: #pyo3_path::Python<'_>) -> #pyo3_path::PyObject { - #pyo3_path::IntoPy::into_py(#pyo3_path::Py::new(py, self).unwrap(), py) - } - } - impl<'py> #pyo3_path::conversion::IntoPyObject<'py> for #cls { type Target = Self; type Output = #pyo3_path::Bound<'py, >::Target>; @@ -2178,6 +2209,7 @@ impl<'a> PyClassImplsBuilder<'a> { let is_subclass = self.attr.options.extends.is_some(); let is_mapping: bool = self.attr.options.mapping.is_some(); let is_sequence: bool = self.attr.options.sequence.is_some(); + let is_immutable_type = self.attr.options.immutable_type.is_some(); ensure_spanned!( !(is_mapping && is_sequence), @@ -2311,6 +2343,7 @@ impl<'a> PyClassImplsBuilder<'a> { const IS_SUBCLASS: bool = #is_subclass; const IS_MAPPING: bool = #is_mapping; const IS_SEQUENCE: bool = #is_sequence; + const IS_IMMUTABLE_TYPE: bool = #is_immutable_type; type BaseType = #base; type ThreadChecker = #thread_checker; @@ -2416,6 +2449,26 @@ impl<'a> PyClassImplsBuilder<'a> { Vec::new() } } + + #[cfg(feature = "experimental-inspect")] + fn impl_introspection(&self, ctx: &Ctx) -> TokenStream { + let Ctx { pyo3_path, .. } = ctx; + let name = get_class_python_name(self.cls, self.attr).to_string(); + let ident = self.cls; + let static_introspection = class_introspection_code(pyo3_path, ident, &name); + let introspection_id = introspection_id_const(); + quote! { + #static_introspection + impl #ident { + #introspection_id + } + } + } + + #[cfg(not(feature = "experimental-inspect"))] + fn impl_introspection(&self, _ctx: &Ctx) -> TokenStream { + quote! {} + } } fn define_inventory_class(inventory_class_name: &syn::Ident, ctx: &Ctx) -> TokenStream { diff --git a/pyo3-macros-backend/src/pyfunction.rs b/pyo3-macros-backend/src/pyfunction.rs index c87492f095c..301819f42dd 100644 --- a/pyo3-macros-backend/src/pyfunction.rs +++ b/pyo3-macros-backend/src/pyfunction.rs @@ -1,3 +1,5 @@ +#[cfg(feature = "experimental-inspect")] +use crate::introspection::{function_introspection_code, introspection_id_const}; use crate::utils::Ctx; use crate::{ attributes::{ @@ -224,6 +226,18 @@ pub fn impl_wrap_pyfunction( FunctionSignature::from_arguments(arguments) }; + let vis = &func.vis; + let name = &func.sig.ident; + + #[cfg(feature = "experimental-inspect")] + let introspection = function_introspection_code(pyo3_path, name, &name.to_string(), &signature); + #[cfg(not(feature = "experimental-inspect"))] + let introspection = quote! {}; + #[cfg(feature = "experimental-inspect")] + let introspection_id = introspection_id_const(); + #[cfg(not(feature = "experimental-inspect"))] + let introspection_id = quote! {}; + let spec = method::FnSpec { tp, name: &func.sig.ident, @@ -235,21 +249,18 @@ pub fn impl_wrap_pyfunction( unsafety: func.sig.unsafety, }; - let vis = &func.vis; - let name = &func.sig.ident; - let wrapper_ident = format_ident!("__pyfunction_{}", spec.name); let wrapper = spec.get_wrapper_function(&wrapper_ident, None, ctx)?; let methoddef = spec.get_methoddef(wrapper_ident, &spec.get_doc(&func.attrs, ctx), ctx); let wrapped_pyfunction = quote! { - // Create a module with the same name as the `#[pyfunction]` - this way `use ` // will actually bring both the module and the function into scope. #[doc(hidden)] #vis mod #name { pub(crate) struct MakeDef; pub const _PYO3_DEF: #pyo3_path::impl_::pymethods::PyMethodDef = MakeDef::_PYO3_DEF; + #introspection_id } // Generate the definition inside an anonymous function in the same scope as the original function - @@ -263,6 +274,8 @@ pub fn impl_wrap_pyfunction( #[allow(non_snake_case)] #wrapper + + #introspection }; Ok(wrapped_pyfunction) } diff --git a/pyo3-macros-backend/src/pyfunction/signature.rs b/pyo3-macros-backend/src/pyfunction/signature.rs index deea3dfa052..fac1541bdf1 100644 --- a/pyo3-macros-backend/src/pyfunction/signature.rs +++ b/pyo3-macros-backend/src/pyfunction/signature.rs @@ -491,49 +491,11 @@ impl<'a> FunctionSignature<'a> { } fn default_value_for_parameter(&self, parameter: &str) -> String { - let mut default = "...".to_string(); if let Some(fn_arg) = self.arguments.iter().find(|arg| arg.name() == parameter) { - if let FnArg::Regular(RegularArg { - default_value: Some(arg_default), - .. - }) = fn_arg - { - match arg_default { - // literal values - syn::Expr::Lit(syn::ExprLit { lit, .. }) => match lit { - syn::Lit::Str(s) => default = s.token().to_string(), - syn::Lit::Char(c) => default = c.token().to_string(), - syn::Lit::Int(i) => default = i.base10_digits().to_string(), - syn::Lit::Float(f) => default = f.base10_digits().to_string(), - syn::Lit::Bool(b) => { - default = if b.value() { - "True".to_string() - } else { - "False".to_string() - } - } - _ => {} - }, - // None - syn::Expr::Path(syn::ExprPath { - qself: None, path, .. - }) if path.is_ident("None") => { - default = "None".to_string(); - } - // others, unsupported yet so defaults to `...` - _ => {} - } - } else if let FnArg::Regular(RegularArg { - option_wrapped_type: Some(..), - .. - }) = fn_arg - { - // functions without a `#[pyo3(signature = (...))]` option - // will treat trailing `Option` arguments as having a default of `None` - default = "None".to_string(); - } + fn_arg.default_value() + } else { + "...".to_string() } - default } pub fn text_signature(&self, self_argument: Option<&str>) -> String { diff --git a/pyo3-macros-backend/src/pymethod.rs b/pyo3-macros-backend/src/pymethod.rs index a1689e4e75c..15cce6f365f 100644 --- a/pyo3-macros-backend/src/pymethod.rs +++ b/pyo3-macros-backend/src/pymethod.rs @@ -4,8 +4,8 @@ use std::ffi::CString; use crate::attributes::{FromPyWithAttribute, NameAttribute, RenamingRule}; use crate::method::{CallingConvention, ExtractErrorMode, PyArg}; use crate::params::{impl_regular_arg_param, Holders}; -use crate::utils::{deprecated_from_py_with, PythonDoc, TypeExt as _}; use crate::utils::{Ctx, LitCStr}; +use crate::utils::{PythonDoc, TypeExt as _}; use crate::{ method::{FnArg, FnSpec, FnType, SelfType}, pyfunction::PyFunctionOptions, @@ -278,7 +278,7 @@ pub fn gen_py_method( } pub fn check_generic(sig: &syn::Signature) -> syn::Result<()> { - let err_msg = |typ| format!("Python functions cannot have generic {} parameters", typ); + let err_msg = |typ| format!("Python functions cannot have generic {typ} parameters"); for param in &sig.generics.params { match param { syn::GenericParam::Lifetime(_) => {} @@ -659,10 +659,8 @@ pub fn impl_py_setter_def( let (from_py_with, ident) = if let Some(from_py_with) = &value_arg.from_py_with().as_ref().map(|f| &f.value) { let ident = syn::Ident::new("from_py_with", from_py_with.span()); - let d = deprecated_from_py_with(from_py_with).unwrap_or_default(); ( quote_spanned! { from_py_with.span() => - #d let #ident = #from_py_with; }, ident, @@ -701,6 +699,7 @@ pub fn impl_py_setter_def( let holder = holders.push_holder(span); let ty = field.ty.clone().elide_lifetimes(); quote! { + #[allow(unused_imports)] use #pyo3_path::impl_::pyclass::Probe as _; let _val = #pyo3_path::impl_::extract_argument::extract_argument::< _, @@ -845,8 +844,6 @@ pub fn impl_py_getter_def( #ty, Offset, { #pyo3_path::impl_::pyclass::IsPyT::<#ty>::VALUE }, - { #pyo3_path::impl_::pyclass::IsToPyObject::<#ty>::VALUE }, - { #pyo3_path::impl_::pyclass::IsIntoPy::<#ty>::VALUE }, { #pyo3_path::impl_::pyclass::IsIntoPyObjectRef::<#ty>::VALUE }, { #pyo3_path::impl_::pyclass::IsIntoPyObject::<#ty>::VALUE }, > = unsafe { #pyo3_path::impl_::pyclass::PyClassGetterGenerator::new() }; @@ -1205,6 +1202,7 @@ fn extract_object( let holder = holders.push_holder(Span::call_site()); let ty = arg.ty().clone().elide_lifetimes(); quote! {{ + #[allow(unused_imports)] use #pyo3_path::impl_::pyclass::Probe as _; #pyo3_path::impl_::extract_argument::extract_argument::< _, @@ -1603,7 +1601,7 @@ fn extract_proto_arguments( if let FnArg::Py(..) = arg { args.push(quote! { py }); } else { - let ident = syn::Ident::new(&format!("arg{}", non_python_args), Span::call_site()); + let ident = syn::Ident::new(&format!("arg{non_python_args}"), Span::call_site()); let conversions = proto_args.get(non_python_args) .ok_or_else(|| err_spanned!(arg.ty().span() => format!("Expected at most {} non-python arguments", proto_args.len())))? .extract(&ident, arg, extract_error_mode, holders, ctx); diff --git a/pyo3-macros-backend/src/pyversions.rs b/pyo3-macros-backend/src/pyversions.rs index 4c0998667d8..3c5ac47fb84 100644 --- a/pyo3-macros-backend/src/pyversions.rs +++ b/pyo3-macros-backend/src/pyversions.rs @@ -2,5 +2,10 @@ use pyo3_build_config::PythonVersion; pub fn is_abi3_before(major: u8, minor: u8) -> bool { let config = pyo3_build_config::get(); - config.abi3 && config.version < PythonVersion { major, minor } + config.abi3 && !config.is_free_threaded() && config.version < PythonVersion { major, minor } +} + +pub fn is_py_before(major: u8, minor: u8) -> bool { + let config = pyo3_build_config::get(); + config.version < PythonVersion { major, minor } } diff --git a/pyo3-macros-backend/src/utils.rs b/pyo3-macros-backend/src/utils.rs index bdec23388df..09f86158834 100644 --- a/pyo3-macros-backend/src/utils.rs +++ b/pyo3-macros-backend/src/utils.rs @@ -1,6 +1,6 @@ -use crate::attributes::{CrateAttribute, ExprPathWrap, RenamingRule}; +use crate::attributes::{CrateAttribute, RenamingRule}; use proc_macro2::{Span, TokenStream}; -use quote::{quote, quote_spanned, ToTokens}; +use quote::{quote, ToTokens}; use std::ffi::CString; use syn::spanned::Spanned; use syn::{punctuated::Punctuated, Token}; @@ -324,20 +324,6 @@ pub(crate) fn has_attribute_with_namespace( }) } -pub(crate) fn deprecated_from_py_with(expr_path: &ExprPathWrap) -> Option { - let path = quote!(#expr_path).to_string(); - let msg = - format!("remove the quotes from the literal\n= help: use `{path}` instead of `\"{path}\"`"); - expr_path.from_lit_str.then(|| { - quote_spanned! { expr_path.span() => - #[deprecated(since = "0.24.0", note = #msg)] - #[allow(dead_code)] - const LIT_STR_DEPRECATION: () = (); - let _: () = LIT_STR_DEPRECATION; - } - }) -} - pub(crate) trait TypeExt { /// Replaces all explicit lifetimes in `self` with elided (`'_`) lifetimes /// diff --git a/pyo3-macros/Cargo.toml b/pyo3-macros/Cargo.toml index 25821b84e81..7df15c23f76 100644 --- a/pyo3-macros/Cargo.toml +++ b/pyo3-macros/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3-macros" -version = "0.24.0" +version = "0.25.0" description = "Proc macros for PyO3 package" authors = ["PyO3 Project and Contributors "] keywords = ["pyo3", "python", "cpython", "ffi"] @@ -17,12 +17,13 @@ proc-macro = true [features] multiple-pymethods = [] experimental-async = ["pyo3-macros-backend/experimental-async"] +experimental-inspect = ["pyo3-macros-backend/experimental-inspect"] [dependencies] proc-macro2 = { version = "1.0.60", default-features = false } quote = "1" syn = { version = "2", features = ["full", "extra-traits"] } -pyo3-macros-backend = { path = "../pyo3-macros-backend", version = "=0.24.0" } +pyo3-macros-backend = { path = "../pyo3-macros-backend", version = "=0.25.0" } [lints] workspace = true diff --git a/pyproject.toml b/pyproject.toml index d757c927f4f..adfe3a27348 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -3,7 +3,7 @@ [tool.towncrier] filename = "CHANGELOG.md" -version = "0.24.0" +version = "0.25.0" start_string = "\n" template = ".towncrier.template.md" title_format = "## [{version}] - {project_date}" diff --git a/pytests/Cargo.toml b/pytests/Cargo.toml index 1fee3093275..2763cdec2e3 100644 --- a/pytests/Cargo.toml +++ b/pytests/Cargo.toml @@ -8,7 +8,7 @@ publish = false rust-version = "1.63" [dependencies] -pyo3 = { path = "../", features = ["extension-module"] } +pyo3 = { path = "../", features = ["extension-module", "experimental-inspect"] } [build-dependencies] pyo3-build-config = { path = "../pyo3-build-config" } diff --git a/pytests/README.md b/pytests/README.md index 7ced072aa36..1016baa7209 100644 --- a/pytests/README.md +++ b/pytests/README.md @@ -2,6 +2,9 @@ An extension module built using PyO3, used to test and benchmark PyO3 from Python. +The `stubs` directory contains Python stubs used to test the automated stubs introspection. +To test them run `nox -s test-introspection`. + ## Testing This package is intended to be built using `maturin`. Once built, you can run the tests using `pytest`: diff --git a/pytests/pytest.ini b/pytests/pytest.ini new file mode 100644 index 00000000000..3d62037f722 --- /dev/null +++ b/pytests/pytest.ini @@ -0,0 +1,3 @@ +# see https://github.com/PyO3/pyo3/issues/5094 for details +[pytest] +filterwarnings = ignore::DeprecationWarning:pytest_asyncio.* \ No newline at end of file diff --git a/pytests/src/lib.rs b/pytests/src/lib.rs index b6c32230dac..4112c90400e 100644 --- a/pytests/src/lib.rs +++ b/pytests/src/lib.rs @@ -18,43 +18,51 @@ pub mod sequence; pub mod subclassing; #[pymodule(gil_used = false)] -fn pyo3_pytests(py: Python<'_>, m: &Bound<'_, PyModule>) -> PyResult<()> { - m.add_wrapped(wrap_pymodule!(awaitable::awaitable))?; - #[cfg(not(Py_LIMITED_API))] - m.add_wrapped(wrap_pymodule!(buf_and_str::buf_and_str))?; - m.add_wrapped(wrap_pymodule!(comparisons::comparisons))?; - #[cfg(not(Py_LIMITED_API))] - m.add_wrapped(wrap_pymodule!(datetime::datetime))?; - m.add_wrapped(wrap_pymodule!(dict_iter::dict_iter))?; - m.add_wrapped(wrap_pymodule!(enums::enums))?; - m.add_wrapped(wrap_pymodule!(misc::misc))?; - m.add_wrapped(wrap_pymodule!(objstore::objstore))?; - m.add_wrapped(wrap_pymodule!(othermod::othermod))?; - m.add_wrapped(wrap_pymodule!(path::path))?; - m.add_wrapped(wrap_pymodule!(pyclasses::pyclasses))?; - m.add_wrapped(wrap_pymodule!(pyfunctions::pyfunctions))?; - m.add_wrapped(wrap_pymodule!(sequence::sequence))?; - m.add_wrapped(wrap_pymodule!(subclassing::subclassing))?; +mod pyo3_pytests { + use super::*; + + #[pymodule_export] + use {pyclasses::pyclasses, pyfunctions::pyfunctions}; // Inserting to sys.modules allows importing submodules nicely from Python // e.g. import pyo3_pytests.buf_and_str as bas + #[pymodule_init] + fn init(m: &Bound<'_, PyModule>) -> PyResult<()> { + m.add_wrapped(wrap_pymodule!(awaitable::awaitable))?; + #[cfg(not(Py_LIMITED_API))] + m.add_wrapped(wrap_pymodule!(buf_and_str::buf_and_str))?; + m.add_wrapped(wrap_pymodule!(comparisons::comparisons))?; + #[cfg(not(Py_LIMITED_API))] + m.add_wrapped(wrap_pymodule!(datetime::datetime))?; + m.add_wrapped(wrap_pymodule!(dict_iter::dict_iter))?; + m.add_wrapped(wrap_pymodule!(enums::enums))?; + m.add_wrapped(wrap_pymodule!(misc::misc))?; + m.add_wrapped(wrap_pymodule!(objstore::objstore))?; + m.add_wrapped(wrap_pymodule!(othermod::othermod))?; + m.add_wrapped(wrap_pymodule!(path::path))?; + m.add_wrapped(wrap_pymodule!(sequence::sequence))?; + m.add_wrapped(wrap_pymodule!(subclassing::subclassing))?; + + // Inserting to sys.modules allows importing submodules nicely from Python + // e.g. import pyo3_pytests.buf_and_str as bas - let sys = PyModule::import(py, "sys")?; - let sys_modules = sys.getattr("modules")?.downcast_into::()?; - sys_modules.set_item("pyo3_pytests.awaitable", m.getattr("awaitable")?)?; - sys_modules.set_item("pyo3_pytests.buf_and_str", m.getattr("buf_and_str")?)?; - sys_modules.set_item("pyo3_pytests.comparisons", m.getattr("comparisons")?)?; - sys_modules.set_item("pyo3_pytests.datetime", m.getattr("datetime")?)?; - sys_modules.set_item("pyo3_pytests.dict_iter", m.getattr("dict_iter")?)?; - sys_modules.set_item("pyo3_pytests.enums", m.getattr("enums")?)?; - sys_modules.set_item("pyo3_pytests.misc", m.getattr("misc")?)?; - sys_modules.set_item("pyo3_pytests.objstore", m.getattr("objstore")?)?; - sys_modules.set_item("pyo3_pytests.othermod", m.getattr("othermod")?)?; - sys_modules.set_item("pyo3_pytests.path", m.getattr("path")?)?; - sys_modules.set_item("pyo3_pytests.pyclasses", m.getattr("pyclasses")?)?; - sys_modules.set_item("pyo3_pytests.pyfunctions", m.getattr("pyfunctions")?)?; - sys_modules.set_item("pyo3_pytests.sequence", m.getattr("sequence")?)?; - sys_modules.set_item("pyo3_pytests.subclassing", m.getattr("subclassing")?)?; + let sys = PyModule::import(m.py(), "sys")?; + let sys_modules = sys.getattr("modules")?.downcast_into::()?; + sys_modules.set_item("pyo3_pytests.awaitable", m.getattr("awaitable")?)?; + sys_modules.set_item("pyo3_pytests.buf_and_str", m.getattr("buf_and_str")?)?; + sys_modules.set_item("pyo3_pytests.comparisons", m.getattr("comparisons")?)?; + sys_modules.set_item("pyo3_pytests.datetime", m.getattr("datetime")?)?; + sys_modules.set_item("pyo3_pytests.dict_iter", m.getattr("dict_iter")?)?; + sys_modules.set_item("pyo3_pytests.enums", m.getattr("enums")?)?; + sys_modules.set_item("pyo3_pytests.misc", m.getattr("misc")?)?; + sys_modules.set_item("pyo3_pytests.objstore", m.getattr("objstore")?)?; + sys_modules.set_item("pyo3_pytests.othermod", m.getattr("othermod")?)?; + sys_modules.set_item("pyo3_pytests.path", m.getattr("path")?)?; + sys_modules.set_item("pyo3_pytests.pyclasses", m.getattr("pyclasses")?)?; + sys_modules.set_item("pyo3_pytests.pyfunctions", m.getattr("pyfunctions")?)?; + sys_modules.set_item("pyo3_pytests.sequence", m.getattr("sequence")?)?; + sys_modules.set_item("pyo3_pytests.subclassing", m.getattr("subclassing")?)?; - Ok(()) + Ok(()) + } } diff --git a/pytests/src/pyclasses.rs b/pytests/src/pyclasses.rs index 3af08c053cc..4c3398b6627 100644 --- a/pytests/src/pyclasses.rs +++ b/pytests/src/pyclasses.rs @@ -80,8 +80,7 @@ impl AssertingBaseClass { fn new(cls: &Bound<'_, PyType>, expected_type: Bound<'_, PyType>) -> PyResult { if !cls.is(&expected_type) { return Err(PyValueError::new_err(format!( - "{:?} != {:?}", - cls, expected_type + "{cls:?} != {expected_type:?}" ))); } Ok(Self) @@ -105,14 +104,12 @@ impl ClassWithDict { } #[pymodule(gil_used = false)] -pub fn pyclasses(m: &Bound<'_, PyModule>) -> PyResult<()> { - m.add_class::()?; - m.add_class::()?; - m.add_class::()?; - m.add_class::()?; - m.add_class::()?; +pub mod pyclasses { #[cfg(any(Py_3_10, not(Py_LIMITED_API)))] - m.add_class::()?; - - Ok(()) + #[pymodule_export] + use super::ClassWithDict; + #[pymodule_export] + use super::{ + AssertingBaseClass, ClassWithoutConstructor, EmptyClass, PyClassIter, PyClassThreadIter, + }; } diff --git a/pytests/src/pyfunctions.rs b/pytests/src/pyfunctions.rs index 024641d3d2e..19e30712909 100644 --- a/pytests/src/pyfunctions.rs +++ b/pytests/src/pyfunctions.rs @@ -67,13 +67,21 @@ fn args_kwargs<'py>( (args, kwargs) } -#[pymodule(gil_used = false)] -pub fn pyfunctions(m: &Bound<'_, PyModule>) -> PyResult<()> { - m.add_function(wrap_pyfunction!(none, m)?)?; - m.add_function(wrap_pyfunction!(simple, m)?)?; - m.add_function(wrap_pyfunction!(simple_args, m)?)?; - m.add_function(wrap_pyfunction!(simple_kwargs, m)?)?; - m.add_function(wrap_pyfunction!(simple_args_kwargs, m)?)?; - m.add_function(wrap_pyfunction!(args_kwargs, m)?)?; - Ok(()) +#[pyfunction(signature = (a, /, b))] +fn positional_only<'py>(a: Any<'py>, b: Any<'py>) -> (Any<'py>, Any<'py>) { + (a, b) +} + +#[pyfunction(signature = (a = false, b = 0, c = 0.0, d = ""))] +fn with_typed_args(a: bool, b: u64, c: f64, d: &str) -> (bool, u64, f64, &str) { + (a, b, c, d) +} + +#[pymodule] +pub mod pyfunctions { + #[pymodule_export] + use super::{ + args_kwargs, none, positional_only, simple, simple_args, simple_args_kwargs, simple_kwargs, + with_typed_args, + }; } diff --git a/pytests/stubs/__init__.pyi b/pytests/stubs/__init__.pyi new file mode 100644 index 00000000000..e69de29bb2d diff --git a/pytests/stubs/pyclasses.pyi b/pytests/stubs/pyclasses.pyi new file mode 100644 index 00000000000..2a36e0b4540 --- /dev/null +++ b/pytests/stubs/pyclasses.pyi @@ -0,0 +1,5 @@ +class AssertingBaseClass: ... +class ClassWithoutConstructor: ... +class EmptyClass: ... +class PyClassIter: ... +class PyClassThreadIter: ... diff --git a/pytests/stubs/pyfunctions.pyi b/pytests/stubs/pyfunctions.pyi new file mode 100644 index 00000000000..5fb5e6c474c --- /dev/null +++ b/pytests/stubs/pyfunctions.pyi @@ -0,0 +1,8 @@ +def args_kwargs(*args, **kwargs): ... +def none(): ... +def positional_only(a, /, b): ... +def simple(a, b=None, *, c=None): ... +def simple_args(a, b=None, *args, c=None): ... +def simple_args_kwargs(a, b=None, *args, c=None, **kwargs): ... +def simple_kwargs(a, b=None, c=None, **kwargs): ... +def with_typed_args(a=False, b=0, c=0.0, d=""): ... diff --git a/pytests/tests/test_pyfunctions.py b/pytests/tests/test_pyfunctions.py index c6fb448248b..b3897f289c6 100644 --- a/pytests/tests/test_pyfunctions.py +++ b/pytests/tests/test_pyfunctions.py @@ -1,3 +1,5 @@ +from typing import Tuple + from pyo3_pytests import pyfunctions @@ -58,7 +60,7 @@ def test_simple_kwargs_rs(benchmark): def simple_args_kwargs_py(a, b=None, *args, c=None, **kwargs): - return (a, b, args, c, kwargs) + return a, b, args, c, kwargs def test_simple_args_kwargs_py(benchmark): @@ -72,7 +74,7 @@ def test_simple_args_kwargs_rs(benchmark): def args_kwargs_py(*args, **kwargs): - return (args, kwargs) + return args, kwargs def test_args_kwargs_py(benchmark): @@ -83,3 +85,36 @@ def test_args_kwargs_rs(benchmark): rust = benchmark(pyfunctions.args_kwargs, 1, "foo", {1: 2}, bar=4, foo=10) py = args_kwargs_py(1, "foo", {1: 2}, bar=4, foo=10) assert rust == py + + +# TODO: the second argument should be positional-only +# but can't be without breaking tests on Python 3.7. +# See gh-5095. +def positional_only_py(a, b): + return a, b + + +def test_positional_only_py(benchmark): + benchmark(positional_only_py, 1, "foo") + + +def test_positional_only_rs(benchmark): + rust = benchmark(pyfunctions.positional_only, 1, "foo") + py = positional_only_py(1, "foo") + assert rust == py + + +def with_typed_args_py( + a: bool, b: int, c: float, d: str +) -> Tuple[bool, int, float, str]: + return a, b, c, d + + +def test_with_typed_args_py(benchmark): + benchmark(with_typed_args_py, True, 1, 1.2, "foo") + + +def test_with_typed_args_rs(benchmark): + rust = benchmark(pyfunctions.with_typed_args, True, 1, 1.2, "foo") + py = with_typed_args_py(True, 1, 1.2, "foo") + assert rust == py diff --git a/src/buffer.rs b/src/buffer.rs index 2d94681a5c7..6f74b698de7 100644 --- a/src/buffer.rs +++ b/src/buffer.rs @@ -224,13 +224,6 @@ impl PyBuffer { } } - /// Deprecated name for [`PyBuffer::get`]. - #[deprecated(since = "0.23.0", note = "renamed to `PyBuffer::get`")] - #[inline] - pub fn get_bound(obj: &Bound<'_, PyAny>) -> PyResult> { - Self::get(obj) - } - /// Gets the pointer to the start of the buffer memory. /// /// Warning: the buffer memory might be mutated by other Python functions, @@ -709,7 +702,7 @@ mod tests { buffer.0.suboffsets, buffer.0.internal ); - let debug_repr = format!("{:?}", buffer); + let debug_repr = format!("{buffer:?}"); assert_eq!(debug_repr, expected); }); } @@ -836,8 +829,7 @@ mod tests { assert_eq!( ElementType::from_format(cstr), expected, - "element from format &Cstr: {:?}", - cstr, + "element from format &Cstr: {cstr:?}", ); } } diff --git a/src/conversion.rs b/src/conversion.rs index 82ad4d84977..165175fae54 100644 --- a/src/conversion.rs +++ b/src/conversion.rs @@ -5,177 +5,9 @@ use crate::inspect::types::TypeInfo; use crate::pyclass::boolean_struct::False; use crate::types::any::PyAnyMethods; use crate::types::PyTuple; -use crate::{ - ffi, Borrowed, Bound, BoundObject, Py, PyAny, PyClass, PyErr, PyObject, PyRef, PyRefMut, Python, -}; +use crate::{Borrowed, Bound, BoundObject, Py, PyAny, PyClass, PyErr, PyRef, PyRefMut, Python}; use std::convert::Infallible; -/// Returns a borrowed pointer to a Python object. -/// -/// The returned pointer will be valid for as long as `self` is. It may be null depending on the -/// implementation. -/// -/// # Examples -/// -/// ```rust -/// use pyo3::prelude::*; -/// use pyo3::ffi; -/// -/// Python::with_gil(|py| { -/// let s = "foo".into_pyobject(py)?; -/// let ptr = s.as_ptr(); -/// -/// let is_really_a_pystring = unsafe { ffi::PyUnicode_CheckExact(ptr) }; -/// assert_eq!(is_really_a_pystring, 1); -/// # Ok::<_, PyErr>(()) -/// }) -/// # .unwrap(); -/// ``` -/// -/// # Safety -/// -/// For callers, it is your responsibility to make sure that the underlying Python object is not dropped too -/// early. For example, the following code will cause undefined behavior: -/// -/// ```rust,no_run -/// # use pyo3::prelude::*; -/// # use pyo3::ffi; -/// # -/// Python::with_gil(|py| { -/// // ERROR: calling `.as_ptr()` will throw away the temporary object and leave `ptr` dangling. -/// let ptr: *mut ffi::PyObject = 0xabad1dea_u32.into_pyobject(py)?.as_ptr(); -/// -/// let isnt_a_pystring = unsafe { -/// // `ptr` is dangling, this is UB -/// ffi::PyUnicode_CheckExact(ptr) -/// }; -/// # assert_eq!(isnt_a_pystring, 0); -/// # Ok::<_, PyErr>(()) -/// }) -/// # .unwrap(); -/// ``` -/// -/// This happens because the pointer returned by `as_ptr` does not carry any lifetime information -/// and the Python object is dropped immediately after the `0xabad1dea_u32.into_pyobject(py).as_ptr()` -/// expression is evaluated. To fix the problem, bind Python object to a local variable like earlier -/// to keep the Python object alive until the end of its scope. -/// -/// Implementors must ensure this returns a valid pointer to a Python object, which borrows a reference count from `&self`. -pub unsafe trait AsPyPointer { - /// Returns the underlying FFI pointer as a borrowed pointer. - fn as_ptr(&self) -> *mut ffi::PyObject; -} - -/// Conversion trait that allows various objects to be converted into `PyObject`. -#[deprecated( - since = "0.23.0", - note = "`ToPyObject` is going to be replaced by `IntoPyObject`. See the migration guide (https://pyo3.rs/v0.23.0/migration) for more information." -)] -pub trait ToPyObject { - /// Converts self into a Python object. - fn to_object(&self, py: Python<'_>) -> PyObject; -} - -/// Defines a conversion from a Rust type to a Python object. -/// -/// It functions similarly to std's [`Into`] trait, but requires a [GIL token](Python) -/// as an argument. Many functions and traits internal to PyO3 require this trait as a bound, -/// so a lack of this trait can manifest itself in different error messages. -/// -/// # Examples -/// ## With `#[pyclass]` -/// The easiest way to implement `IntoPy` is by exposing a struct as a native Python object -/// by annotating it with [`#[pyclass]`](crate::prelude::pyclass). -/// -/// ```rust -/// use pyo3::prelude::*; -/// -/// # #[allow(dead_code)] -/// #[pyclass] -/// struct Number { -/// #[pyo3(get, set)] -/// value: i32, -/// } -/// ``` -/// Python code will see this as an instance of the `Number` class with a `value` attribute. -/// -/// ## Conversion to a Python object -/// -/// However, it may not be desirable to expose the existence of `Number` to Python code. -/// `IntoPy` allows us to define a conversion to an appropriate Python object. -/// ```rust -/// #![allow(deprecated)] -/// use pyo3::prelude::*; -/// -/// # #[allow(dead_code)] -/// struct Number { -/// value: i32, -/// } -/// -/// impl IntoPy for Number { -/// fn into_py(self, py: Python<'_>) -> PyObject { -/// // delegates to i32's IntoPy implementation. -/// self.value.into_py(py) -/// } -/// } -/// ``` -/// Python code will see this as an `int` object. -/// -/// ## Dynamic conversion into Python objects. -/// It is also possible to return a different Python object depending on some condition. -/// This is useful for types like enums that can carry different types. -/// -/// ```rust -/// #![allow(deprecated)] -/// use pyo3::prelude::*; -/// -/// enum Value { -/// Integer(i32), -/// String(String), -/// None, -/// } -/// -/// impl IntoPy for Value { -/// fn into_py(self, py: Python<'_>) -> PyObject { -/// match self { -/// Self::Integer(val) => val.into_py(py), -/// Self::String(val) => val.into_py(py), -/// Self::None => py.None(), -/// } -/// } -/// } -/// # fn main() { -/// # Python::with_gil(|py| { -/// # let v = Value::Integer(73).into_py(py); -/// # let v = v.extract::(py).unwrap(); -/// # -/// # let v = Value::String("foo".into()).into_py(py); -/// # let v = v.extract::(py).unwrap(); -/// # -/// # let v = Value::None.into_py(py); -/// # let v = v.extract::>>(py).unwrap(); -/// # }); -/// # } -/// ``` -/// Python code will see this as any of the `int`, `string` or `None` objects. -#[cfg_attr( - diagnostic_namespace, - diagnostic::on_unimplemented( - message = "`{Self}` cannot be converted to a Python object", - note = "`IntoPy` is automatically implemented by the `#[pyclass]` macro", - note = "if you do not wish to have a corresponding Python type, implement it manually", - note = "if you do not own `{Self}` you can perform a manual conversion to one of the types in `pyo3::types::*`" - ) -)] -#[deprecated( - since = "0.23.0", - note = "`IntoPy` is going to be replaced by `IntoPyObject`. See the migration guide (https://pyo3.rs/v0.23.0/migration) for more information." -)] -pub trait IntoPy: Sized { - /// Performs the conversion. - fn into_py(self, py: Python<'_>) -> T; -} - /// Defines a conversion from a Rust type to a Python object, which may fail. /// /// This trait has `#[derive(IntoPyObject)]` to automatically implement it for simple types and @@ -541,16 +373,6 @@ where } } -/// Identity conversion: allows using existing `PyObject` instances where -/// `T: ToPyObject` is expected. -#[allow(deprecated)] -impl ToPyObject for &'_ T { - #[inline] - fn to_object(&self, py: Python<'_>) -> PyObject { - ::to_object(*self, py) - } -} - impl FromPyObject<'_> for T where T: PyClass + Clone, @@ -579,14 +401,6 @@ where } } -/// Converts `()` to an empty Python tuple. -#[allow(deprecated)] -impl IntoPy> for () { - fn into_py(self, py: Python<'_>) -> Py { - PyTuple::empty(py).unbind() - } -} - impl<'py> IntoPyObject<'py> for () { type Target = PyTuple; type Output = Bound<'py, Self::Target>; diff --git a/src/conversions/anyhow.rs b/src/conversions/anyhow.rs index d2cb3f3eb60..0bf346d835a 100644 --- a/src/conversions/anyhow.rs +++ b/src/conversions/anyhow.rs @@ -113,7 +113,7 @@ impl From for PyErr { Err(error) => error, }; } - PyRuntimeError::new_err(format!("{:?}", error)) + PyRuntimeError::new_err(format!("{error:?}")) } } @@ -141,7 +141,7 @@ mod test_anyhow { #[test] fn test_pyo3_exception_contents() { let err = h().unwrap_err(); - let expected_contents = format!("{:?}", err); + let expected_contents = format!("{err:?}"); let pyerr = PyErr::from(err); Python::with_gil(|py| { @@ -160,7 +160,7 @@ mod test_anyhow { #[test] fn test_pyo3_exception_contents2() { let err = k().unwrap_err(); - let expected_contents = format!("{:?}", err); + let expected_contents = format!("{err:?}"); let pyerr = PyErr::from(err); Python::with_gil(|py| { diff --git a/src/conversions/bigdecimal.rs b/src/conversions/bigdecimal.rs new file mode 100644 index 00000000000..129def772c8 --- /dev/null +++ b/src/conversions/bigdecimal.rs @@ -0,0 +1,213 @@ +#![cfg(feature = "bigdecimal")] +//! Conversions to and from [bigdecimal](https://docs.rs/bigdecimal)'s [`BigDecimal`] type. +//! +//! This is useful for converting Python's decimal.Decimal into and from a native Rust type. +//! +//! # Setup +//! +//! To use this feature, add to your **`Cargo.toml`**: +//! +//! ```toml +//! [dependencies] +#![doc = concat!("pyo3 = { version = \"", env!("CARGO_PKG_VERSION"), "\", features = [\"bigdecimal\"] }")] +//! bigdecimal = "0.4" +//! ``` +//! +//! Note that you must use a compatible version of bigdecimal and PyO3. +//! The required bigdecimal version may vary based on the version of PyO3. +//! +//! # Example +//! +//! Rust code to create a function that adds one to a BigDecimal +//! +//! ```rust +//! use bigdecimal::BigDecimal; +//! use pyo3::prelude::*; +//! +//! #[pyfunction] +//! fn add_one(d: BigDecimal) -> BigDecimal { +//! d + 1 +//! } +//! +//! #[pymodule] +//! fn my_module(m: &Bound<'_, PyModule>) -> PyResult<()> { +//! m.add_function(wrap_pyfunction!(add_one, m)?)?; +//! Ok(()) +//! } +//! ``` +//! +//! Python code that validates the functionality +//! +//! +//! ```python +//! from my_module import add_one +//! from decimal import Decimal +//! +//! d = Decimal("2") +//! value = add_one(d) +//! +//! assert d + 1 == value +//! ``` + +use std::str::FromStr; + +use crate::{ + exceptions::PyValueError, + sync::GILOnceCell, + types::{PyAnyMethods, PyStringMethods, PyType}, + Bound, FromPyObject, IntoPyObject, Py, PyAny, PyErr, PyResult, Python, +}; +use bigdecimal::BigDecimal; + +static DECIMAL_CLS: GILOnceCell> = GILOnceCell::new(); + +fn get_decimal_cls(py: Python<'_>) -> PyResult<&Bound<'_, PyType>> { + DECIMAL_CLS.import(py, "decimal", "Decimal") +} + +impl FromPyObject<'_> for BigDecimal { + fn extract_bound(obj: &Bound<'_, PyAny>) -> PyResult { + let py_str = &obj.str()?; + let rs_str = &py_str.to_cow()?; + BigDecimal::from_str(rs_str).map_err(|e| PyValueError::new_err(e.to_string())) + } +} + +impl<'py> IntoPyObject<'py> for BigDecimal { + type Target = PyAny; + + type Output = Bound<'py, Self::Target>; + + type Error = PyErr; + + fn into_pyobject(self, py: Python<'py>) -> Result { + let cls = get_decimal_cls(py)?; + cls.call1((self.to_string(),)) + } +} + +#[cfg(test)] +mod test_bigdecimal { + use super::*; + use crate::types::dict::PyDictMethods; + use crate::types::PyDict; + use std::ffi::CString; + + use crate::ffi; + use bigdecimal::{One, Zero}; + #[cfg(not(target_arch = "wasm32"))] + use proptest::prelude::*; + + macro_rules! convert_constants { + ($name:ident, $rs:expr, $py:literal) => { + #[test] + fn $name() { + Python::with_gil(|py| { + let rs_orig = $rs; + let rs_dec = rs_orig.clone().into_pyobject(py).unwrap(); + let locals = PyDict::new(py); + locals.set_item("rs_dec", &rs_dec).unwrap(); + // Checks if BigDecimal -> Python Decimal conversion is correct + py.run( + &CString::new(format!( + "import decimal\npy_dec = decimal.Decimal(\"{}\")\nassert py_dec == rs_dec", + $py + )) + .unwrap(), + None, + Some(&locals), + ) + .unwrap(); + // Checks if Python Decimal -> BigDecimal conversion is correct + let py_dec = locals.get_item("py_dec").unwrap().unwrap(); + let py_result: BigDecimal = py_dec.extract().unwrap(); + assert_eq!(rs_orig, py_result); + }) + } + }; + } + + convert_constants!(convert_zero, BigDecimal::zero(), "0"); + convert_constants!(convert_one, BigDecimal::one(), "1"); + convert_constants!(convert_neg_one, -BigDecimal::one(), "-1"); + convert_constants!(convert_two, BigDecimal::from(2), "2"); + convert_constants!(convert_ten, BigDecimal::from_str("10").unwrap(), "10"); + convert_constants!( + convert_one_hundred_point_one, + BigDecimal::from_str("100.1").unwrap(), + "100.1" + ); + convert_constants!( + convert_one_thousand, + BigDecimal::from_str("1000").unwrap(), + "1000" + ); + convert_constants!( + convert_scientific, + BigDecimal::from_str("1e10").unwrap(), + "1e10" + ); + + #[cfg(not(target_arch = "wasm32"))] + proptest! { + #[test] + fn test_roundtrip( + number in 0..28u32 + ) { + let num = BigDecimal::from(number); + Python::with_gil(|py| { + let rs_dec = num.clone().into_pyobject(py).unwrap(); + let locals = PyDict::new(py); + locals.set_item("rs_dec", &rs_dec).unwrap(); + py.run( + &CString::new(format!( + "import decimal\npy_dec = decimal.Decimal(\"{num}\")\nassert py_dec == rs_dec")).unwrap(), + None, Some(&locals)).unwrap(); + let roundtripped: BigDecimal = rs_dec.extract().unwrap(); + assert_eq!(num, roundtripped); + }) + } + + #[test] + fn test_integers(num in any::()) { + Python::with_gil(|py| { + let py_num = num.into_pyobject(py).unwrap(); + let roundtripped: BigDecimal = py_num.extract().unwrap(); + let rs_dec = BigDecimal::from(num); + assert_eq!(rs_dec, roundtripped); + }) + } + } + + #[test] + fn test_nan() { + Python::with_gil(|py| { + let locals = PyDict::new(py); + py.run( + ffi::c_str!("import decimal\npy_dec = decimal.Decimal(\"NaN\")"), + None, + Some(&locals), + ) + .unwrap(); + let py_dec = locals.get_item("py_dec").unwrap().unwrap(); + let roundtripped: Result = py_dec.extract(); + assert!(roundtripped.is_err()); + }) + } + + #[test] + fn test_infinity() { + Python::with_gil(|py| { + let locals = PyDict::new(py); + py.run( + ffi::c_str!("import decimal\npy_dec = decimal.Decimal(\"Infinity\")"), + None, + Some(&locals), + ) + .unwrap(); + let py_dec = locals.get_item("py_dec").unwrap().unwrap(); + let roundtripped: Result = py_dec.extract(); + assert!(roundtripped.is_err()); + }) + } +} diff --git a/src/conversions/chrono.rs b/src/conversions/chrono.rs index 342ed659e22..bf99951e459 100644 --- a/src/conversions/chrono.rs +++ b/src/conversions/chrono.rs @@ -46,47 +46,18 @@ use crate::exceptions::{PyTypeError, PyUserWarning, PyValueError}; #[cfg(Py_LIMITED_API)] use crate::intern; use crate::types::any::PyAnyMethods; -#[cfg(not(Py_LIMITED_API))] -use crate::types::datetime::timezone_from_offset; -#[cfg(Py_LIMITED_API)] -use crate::types::datetime_abi3::{check_type, timezone_utc, DatetimeTypes}; -#[cfg(Py_LIMITED_API)] -use crate::types::IntoPyDict; use crate::types::PyNone; +use crate::types::{PyDate, PyDateTime, PyDelta, PyTime, PyTzInfo, PyTzInfoAccess}; #[cfg(not(Py_LIMITED_API))] -use crate::types::{ - timezone_utc, PyDate, PyDateAccess, PyDateTime, PyDelta, PyDeltaAccess, PyTime, PyTimeAccess, - PyTzInfo, PyTzInfoAccess, -}; -use crate::{ffi, Bound, FromPyObject, IntoPyObjectExt, PyAny, PyErr, PyObject, PyResult, Python}; -#[allow(deprecated)] -use crate::{IntoPy, ToPyObject}; +use crate::types::{PyDateAccess, PyDeltaAccess, PyTimeAccess}; +use crate::{ffi, Borrowed, Bound, FromPyObject, IntoPyObjectExt, PyAny, PyErr, PyResult, Python}; use chrono::offset::{FixedOffset, Utc}; use chrono::{ DateTime, Datelike, Duration, LocalResult, NaiveDate, NaiveDateTime, NaiveTime, Offset, TimeZone, Timelike, }; -#[allow(deprecated)] -impl ToPyObject for Duration { - #[inline] - fn to_object(&self, py: Python<'_>) -> PyObject { - self.into_pyobject(py).unwrap().into_any().unbind() - } -} - -#[allow(deprecated)] -impl IntoPy for Duration { - #[inline] - fn into_py(self, py: Python<'_>) -> PyObject { - self.into_pyobject(py).unwrap().into_any().unbind() - } -} - impl<'py> IntoPyObject<'py> for Duration { - #[cfg(Py_LIMITED_API)] - type Target = PyAny; - #[cfg(not(Py_LIMITED_API))] type Target = PyDelta; type Output = Bound<'py, Self::Target>; type Error = PyErr; @@ -103,35 +74,22 @@ impl<'py> IntoPyObject<'py> for Duration { // This should never panic since we are just getting the fractional // part of the total microseconds, which should never overflow. .unwrap(); - - #[cfg(not(Py_LIMITED_API))] - { - // We do not need to check the days i64 to i32 cast from rust because - // python will panic with OverflowError. - // We pass true as the `normalize` parameter since we'd need to do several checks here to - // avoid that, and it shouldn't have a big performance impact. - // The seconds and microseconds cast should never overflow since it's at most the number of seconds per day - PyDelta::new( - py, - days.try_into().unwrap_or(i32::MAX), - secs.try_into()?, - micros.try_into()?, - true, - ) - } - - #[cfg(Py_LIMITED_API)] - { - DatetimeTypes::try_get(py) - .and_then(|dt| dt.timedelta.bind(py).call1((days, secs, micros))) - } + // We do not need to check the days i64 to i32 cast from rust because + // python will panic with OverflowError. + // We pass true as the `normalize` parameter since we'd need to do several checks here to + // avoid that, and it shouldn't have a big performance impact. + // The seconds and microseconds cast should never overflow since it's at most the number of seconds per day + PyDelta::new( + py, + days.try_into().unwrap_or(i32::MAX), + secs.try_into()?, + micros.try_into()?, + true, + ) } } impl<'py> IntoPyObject<'py> for &Duration { - #[cfg(Py_LIMITED_API)] - type Target = PyAny; - #[cfg(not(Py_LIMITED_API))] type Target = PyDelta; type Output = Bound<'py, Self::Target>; type Error = PyErr; @@ -144,13 +102,13 @@ impl<'py> IntoPyObject<'py> for &Duration { impl FromPyObject<'_> for Duration { fn extract_bound(ob: &Bound<'_, PyAny>) -> PyResult { + let delta = ob.downcast::()?; // Python size are much lower than rust size so we do not need bound checks. // 0 <= microseconds < 1000000 // 0 <= seconds < 3600*24 // -999999999 <= days <= 999999999 #[cfg(not(Py_LIMITED_API))] let (days, seconds, microseconds) = { - let delta = ob.downcast::()?; ( delta.get_days().into(), delta.get_seconds().into(), @@ -159,11 +117,11 @@ impl FromPyObject<'_> for Duration { }; #[cfg(Py_LIMITED_API)] let (days, seconds, microseconds) = { - check_type(ob, &DatetimeTypes::get(ob.py()).timedelta, "PyDelta")?; + let py = delta.py(); ( - ob.getattr(intern!(ob.py(), "days"))?.extract()?, - ob.getattr(intern!(ob.py(), "seconds"))?.extract()?, - ob.getattr(intern!(ob.py(), "microseconds"))?.extract()?, + delta.getattr(intern!(py, "days"))?.extract()?, + delta.getattr(intern!(py, "seconds"))?.extract()?, + delta.getattr(intern!(py, "microseconds"))?.extract()?, ) }; Ok( @@ -174,48 +132,18 @@ impl FromPyObject<'_> for Duration { } } -#[allow(deprecated)] -impl ToPyObject for NaiveDate { - #[inline] - fn to_object(&self, py: Python<'_>) -> PyObject { - self.into_pyobject(py).unwrap().into_any().unbind() - } -} - -#[allow(deprecated)] -impl IntoPy for NaiveDate { - #[inline] - fn into_py(self, py: Python<'_>) -> PyObject { - self.into_pyobject(py).unwrap().into_any().unbind() - } -} - impl<'py> IntoPyObject<'py> for NaiveDate { - #[cfg(Py_LIMITED_API)] - type Target = PyAny; - #[cfg(not(Py_LIMITED_API))] type Target = PyDate; type Output = Bound<'py, Self::Target>; type Error = PyErr; fn into_pyobject(self, py: Python<'py>) -> Result { let DateArgs { year, month, day } = (&self).into(); - #[cfg(not(Py_LIMITED_API))] - { - PyDate::new(py, year, month, day) - } - - #[cfg(Py_LIMITED_API)] - { - DatetimeTypes::try_get(py).and_then(|dt| dt.date.bind(py).call1((year, month, day))) - } + PyDate::new(py, year, month, day) } } impl<'py> IntoPyObject<'py> for &NaiveDate { - #[cfg(Py_LIMITED_API)] - type Target = PyAny; - #[cfg(not(Py_LIMITED_API))] type Target = PyDate; type Output = Bound<'py, Self::Target>; type Error = PyErr; @@ -228,39 +156,12 @@ impl<'py> IntoPyObject<'py> for &NaiveDate { impl FromPyObject<'_> for NaiveDate { fn extract_bound(ob: &Bound<'_, PyAny>) -> PyResult { - #[cfg(not(Py_LIMITED_API))] - { - let date = ob.downcast::()?; - py_date_to_naive_date(date) - } - #[cfg(Py_LIMITED_API)] - { - check_type(ob, &DatetimeTypes::get(ob.py()).date, "PyDate")?; - py_date_to_naive_date(ob) - } - } -} - -#[allow(deprecated)] -impl ToPyObject for NaiveTime { - #[inline] - fn to_object(&self, py: Python<'_>) -> PyObject { - self.into_pyobject(py).unwrap().into_any().unbind() - } -} - -#[allow(deprecated)] -impl IntoPy for NaiveTime { - #[inline] - fn into_py(self, py: Python<'_>) -> PyObject { - self.into_pyobject(py).unwrap().into_any().unbind() + let date = ob.downcast::()?; + py_date_to_naive_date(date) } } impl<'py> IntoPyObject<'py> for NaiveTime { - #[cfg(Py_LIMITED_API)] - type Target = PyAny; - #[cfg(not(Py_LIMITED_API))] type Target = PyTime; type Output = Bound<'py, Self::Target>; type Error = PyErr; @@ -274,13 +175,8 @@ impl<'py> IntoPyObject<'py> for NaiveTime { truncated_leap_second, } = (&self).into(); - #[cfg(not(Py_LIMITED_API))] let time = PyTime::new(py, hour, min, sec, micro, None)?; - #[cfg(Py_LIMITED_API)] - let time = DatetimeTypes::try_get(py) - .and_then(|dt| dt.time.bind(py).call1((hour, min, sec, micro)))?; - if truncated_leap_second { warn_truncated_leap_second(&time); } @@ -290,9 +186,6 @@ impl<'py> IntoPyObject<'py> for NaiveTime { } impl<'py> IntoPyObject<'py> for &NaiveTime { - #[cfg(Py_LIMITED_API)] - type Target = PyAny; - #[cfg(not(Py_LIMITED_API))] type Target = PyTime; type Output = Bound<'py, Self::Target>; type Error = PyErr; @@ -305,39 +198,12 @@ impl<'py> IntoPyObject<'py> for &NaiveTime { impl FromPyObject<'_> for NaiveTime { fn extract_bound(ob: &Bound<'_, PyAny>) -> PyResult { - #[cfg(not(Py_LIMITED_API))] - { - let time = ob.downcast::()?; - py_time_to_naive_time(time) - } - #[cfg(Py_LIMITED_API)] - { - check_type(ob, &DatetimeTypes::get(ob.py()).time, "PyTime")?; - py_time_to_naive_time(ob) - } - } -} - -#[allow(deprecated)] -impl ToPyObject for NaiveDateTime { - #[inline] - fn to_object(&self, py: Python<'_>) -> PyObject { - self.into_pyobject(py).unwrap().into_any().unbind() - } -} - -#[allow(deprecated)] -impl IntoPy for NaiveDateTime { - #[inline] - fn into_py(self, py: Python<'_>) -> PyObject { - self.into_pyobject(py).unwrap().into_any().unbind() + let time = ob.downcast::()?; + py_time_to_naive_time(time) } } impl<'py> IntoPyObject<'py> for NaiveDateTime { - #[cfg(Py_LIMITED_API)] - type Target = PyAny; - #[cfg(not(Py_LIMITED_API))] type Target = PyDateTime; type Output = Bound<'py, Self::Target>; type Error = PyErr; @@ -352,16 +218,8 @@ impl<'py> IntoPyObject<'py> for NaiveDateTime { truncated_leap_second, } = (&self.time()).into(); - #[cfg(not(Py_LIMITED_API))] let datetime = PyDateTime::new(py, year, month, day, hour, min, sec, micro, None)?; - #[cfg(Py_LIMITED_API)] - let datetime = DatetimeTypes::try_get(py).and_then(|dt| { - dt.datetime - .bind(py) - .call1((year, month, day, hour, min, sec, micro)) - })?; - if truncated_leap_second { warn_truncated_leap_second(&datetime); } @@ -371,9 +229,6 @@ impl<'py> IntoPyObject<'py> for NaiveDateTime { } impl<'py> IntoPyObject<'py> for &NaiveDateTime { - #[cfg(Py_LIMITED_API)] - type Target = PyAny; - #[cfg(not(Py_LIMITED_API))] type Target = PyDateTime; type Output = Bound<'py, Self::Target>; type Error = PyErr; @@ -386,18 +241,12 @@ impl<'py> IntoPyObject<'py> for &NaiveDateTime { impl FromPyObject<'_> for NaiveDateTime { fn extract_bound(dt: &Bound<'_, PyAny>) -> PyResult { - #[cfg(not(Py_LIMITED_API))] let dt = dt.downcast::()?; - #[cfg(Py_LIMITED_API)] - check_type(dt, &DatetimeTypes::get(dt.py()).datetime, "PyDateTime")?; // If the user tries to convert a timezone aware datetime into a naive one, // we return a hard error. We could silently remove tzinfo, or assume local timezone // and do a conversion, but better leave this decision to the user of the library. - #[cfg(not(Py_LIMITED_API))] let has_tzinfo = dt.get_tzinfo().is_some(); - #[cfg(Py_LIMITED_API)] - let has_tzinfo = !dt.getattr(intern!(dt.py(), "tzinfo"))?.is_none(); if has_tzinfo { return Err(PyTypeError::new_err("expected a datetime without tzinfo")); } @@ -407,31 +256,10 @@ impl FromPyObject<'_> for NaiveDateTime { } } -#[allow(deprecated)] -impl ToPyObject for DateTime { - fn to_object(&self, py: Python<'_>) -> PyObject { - // FIXME: convert to better timezone representation here than just convert to fixed offset - // See https://github.com/PyO3/pyo3/issues/3266 - let tz = self.offset().fix().to_object(py); - let tz = tz.bind(py).downcast().unwrap(); - naive_datetime_to_py_datetime(py, &self.naive_local(), Some(tz)) - } -} - -#[allow(deprecated)] -impl IntoPy for DateTime { - fn into_py(self, py: Python<'_>) -> PyObject { - self.to_object(py) - } -} - impl<'py, Tz: TimeZone> IntoPyObject<'py> for DateTime where Tz: IntoPyObject<'py>, { - #[cfg(Py_LIMITED_API)] - type Target = PyAny; - #[cfg(not(Py_LIMITED_API))] type Target = PyDateTime; type Output = Bound<'py, Self::Target>; type Error = PyErr; @@ -446,18 +274,12 @@ impl<'py, Tz: TimeZone> IntoPyObject<'py> for &DateTime where Tz: IntoPyObject<'py>, { - #[cfg(Py_LIMITED_API)] - type Target = PyAny; - #[cfg(not(Py_LIMITED_API))] type Target = PyDateTime; type Output = Bound<'py, Self::Target>; type Error = PyErr; fn into_pyobject(self, py: Python<'py>) -> Result { - let tz = self.timezone().into_bound_py_any(py)?; - - #[cfg(not(Py_LIMITED_API))] - let tz = tz.downcast()?; + let tz = self.timezone().into_bound_py_any(py)?.downcast_into()?; let DateArgs { year, month, day } = (&self.naive_local().date()).into(); let TimeArgs { @@ -473,17 +295,18 @@ where LocalResult::Ambiguous(_, latest) if self.offset().fix() == latest.fix() ); - #[cfg(not(Py_LIMITED_API))] - let datetime = - PyDateTime::new_with_fold(py, year, month, day, hour, min, sec, micro, Some(tz), fold)?; - - #[cfg(Py_LIMITED_API)] - let datetime = DatetimeTypes::try_get(py).and_then(|dt| { - dt.datetime.bind(py).call( - (year, month, day, hour, min, sec, micro, tz), - Some(&[("fold", fold as u8)].into_py_dict(py)?), - ) - })?; + let datetime = PyDateTime::new_with_fold( + py, + year, + month, + day, + hour, + min, + sec, + micro, + Some(&tz), + fold, + )?; if truncated_leap_second { warn_truncated_leap_second(&datetime); @@ -495,15 +318,8 @@ where impl FromPyObject<'py>> FromPyObject<'_> for DateTime { fn extract_bound(dt: &Bound<'_, PyAny>) -> PyResult> { - #[cfg(not(Py_LIMITED_API))] let dt = dt.downcast::()?; - #[cfg(Py_LIMITED_API)] - check_type(dt, &DatetimeTypes::get(dt.py()).datetime, "PyDateTime")?; - - #[cfg(not(Py_LIMITED_API))] let tzinfo = dt.get_tzinfo(); - #[cfg(Py_LIMITED_API)] - let tzinfo: Option> = dt.getattr(intern!(dt.py(), "tzinfo"))?.extract()?; let tz = if let Some(tzinfo) = tzinfo { tzinfo.extract()? @@ -529,57 +345,25 @@ impl FromPyObject<'py>> FromPyObject<'_> for DateTime Err(PyValueError::new_err(format!( - "The datetime {:?} contains an incompatible timezone", - dt + "The datetime {dt:?} contains an incompatible timezone" ))), } } } -#[allow(deprecated)] -impl ToPyObject for FixedOffset { - #[inline] - fn to_object(&self, py: Python<'_>) -> PyObject { - self.into_pyobject(py).unwrap().into_any().unbind() - } -} - -#[allow(deprecated)] -impl IntoPy for FixedOffset { - #[inline] - fn into_py(self, py: Python<'_>) -> PyObject { - self.into_pyobject(py).unwrap().into_any().unbind() - } -} - impl<'py> IntoPyObject<'py> for FixedOffset { - #[cfg(Py_LIMITED_API)] - type Target = PyAny; - #[cfg(not(Py_LIMITED_API))] type Target = PyTzInfo; type Output = Bound<'py, Self::Target>; type Error = PyErr; fn into_pyobject(self, py: Python<'py>) -> Result { let seconds_offset = self.local_minus_utc(); - #[cfg(not(Py_LIMITED_API))] - { - let td = PyDelta::new(py, 0, seconds_offset, 0, true)?; - timezone_from_offset(&td) - } - - #[cfg(Py_LIMITED_API)] - { - let td = Duration::seconds(seconds_offset.into()).into_pyobject(py)?; - DatetimeTypes::try_get(py).and_then(|dt| dt.timezone.bind(py).call1((td,))) - } + let td = PyDelta::new(py, 0, seconds_offset, 0, true)?; + PyTzInfo::fixed_offset(py, td) } } impl<'py> IntoPyObject<'py> for &FixedOffset { - #[cfg(Py_LIMITED_API)] - type Target = PyAny; - #[cfg(not(Py_LIMITED_API))] type Target = PyTzInfo; type Output = Bound<'py, Self::Target>; type Error = PyErr; @@ -596,10 +380,7 @@ impl FromPyObject<'_> for FixedOffset { /// Note that the conversion will result in precision lost in microseconds as chrono offset /// does not supports microseconds. fn extract_bound(ob: &Bound<'_, PyAny>) -> PyResult { - #[cfg(not(Py_LIMITED_API))] let ob = ob.downcast::()?; - #[cfg(Py_LIMITED_API)] - check_type(ob, &DatetimeTypes::get(ob.py()).tzinfo, "PyTzInfo")?; // Passing Python's None to the `utcoffset` function will only // work for timezones defined as fixed offsets in Python. @@ -609,8 +390,7 @@ impl FromPyObject<'_> for FixedOffset { let py_timedelta = ob.call_method1("utcoffset", (PyNone::get(ob.py()),))?; if py_timedelta.is_none() { return Err(PyTypeError::new_err(format!( - "{:?} is not a fixed offset timezone", - ob + "{ob:?} is not a fixed offset timezone" ))); } let total_seconds: Duration = py_timedelta.extract()?; @@ -621,48 +401,19 @@ impl FromPyObject<'_> for FixedOffset { } } -#[allow(deprecated)] -impl ToPyObject for Utc { - #[inline] - fn to_object(&self, py: Python<'_>) -> PyObject { - self.into_pyobject(py).unwrap().into_any().unbind() - } -} - -#[allow(deprecated)] -impl IntoPy for Utc { - #[inline] - fn into_py(self, py: Python<'_>) -> PyObject { - self.into_pyobject(py).unwrap().into_any().unbind() - } -} - impl<'py> IntoPyObject<'py> for Utc { - #[cfg(Py_LIMITED_API)] - type Target = PyAny; - #[cfg(not(Py_LIMITED_API))] type Target = PyTzInfo; - type Output = Bound<'py, Self::Target>; + type Output = Borrowed<'static, 'py, Self::Target>; type Error = PyErr; fn into_pyobject(self, py: Python<'py>) -> Result { - #[cfg(Py_LIMITED_API)] - { - Ok(timezone_utc(py).into_any()) - } - #[cfg(not(Py_LIMITED_API))] - { - Ok(timezone_utc(py)) - } + PyTzInfo::utc(py) } } impl<'py> IntoPyObject<'py> for &Utc { - #[cfg(Py_LIMITED_API)] - type Target = PyAny; - #[cfg(not(Py_LIMITED_API))] type Target = PyTzInfo; - type Output = Bound<'py, Self::Target>; + type Output = Borrowed<'static, 'py, Self::Target>; type Error = PyErr; #[inline] @@ -673,7 +424,7 @@ impl<'py> IntoPyObject<'py> for &Utc { impl FromPyObject<'_> for Utc { fn extract_bound(ob: &Bound<'_, PyAny>) -> PyResult { - let py_utc = timezone_utc(ob.py()); + let py_utc = PyTzInfo::utc(ob.py())?; if ob.eq(py_utc)? { Ok(Utc) } else { @@ -722,35 +473,6 @@ impl From<&NaiveTime> for TimeArgs { } } -fn naive_datetime_to_py_datetime( - py: Python<'_>, - naive_datetime: &NaiveDateTime, - #[cfg(not(Py_LIMITED_API))] tzinfo: Option<&Bound<'_, PyTzInfo>>, - #[cfg(Py_LIMITED_API)] tzinfo: Option<&Bound<'_, PyAny>>, -) -> PyObject { - let DateArgs { year, month, day } = (&naive_datetime.date()).into(); - let TimeArgs { - hour, - min, - sec, - micro, - truncated_leap_second, - } = (&naive_datetime.time()).into(); - #[cfg(not(Py_LIMITED_API))] - let datetime = PyDateTime::new(py, year, month, day, hour, min, sec, micro, tzinfo) - .expect("failed to construct datetime"); - #[cfg(Py_LIMITED_API)] - let datetime = DatetimeTypes::get(py) - .datetime - .bind(py) - .call1((year, month, day, hour, min, sec, micro, tzinfo)) - .expect("failed to construct datetime.datetime"); - if truncated_leap_second { - warn_truncated_leap_second(&datetime); - } - datetime.into() -} - fn warn_truncated_leap_second(obj: &Bound<'_, PyAny>) { let py = obj.py(); if let Err(e) = PyErr::warn( @@ -935,10 +657,7 @@ mod tests { let py_delta = new_py_datetime_ob(py, "timedelta", (py_days, py_seconds, py_ms)); assert!( delta.eq(&py_delta).unwrap(), - "{}: {} != {}", - name, - delta, - py_delta + "{name}: {delta} != {py_delta}" ); }); }; @@ -974,7 +693,7 @@ mod tests { Python::with_gil(|py| { let py_delta = new_py_datetime_ob(py, "timedelta", (py_days, py_seconds, py_ms)); let py_delta: Duration = py_delta.extract().unwrap(); - assert_eq!(py_delta, delta, "{}: {} != {}", name, py_delta, delta); + assert_eq!(py_delta, delta, "{name}: {py_delta} != {delta}"); }) }; @@ -1037,10 +756,7 @@ mod tests { assert_eq!( date.compare(&py_date).unwrap(), Ordering::Equal, - "{}: {} != {}", - name, - date, - py_date + "{name}: {date} != {py_date}" ); }) }; @@ -1058,7 +774,7 @@ mod tests { let py_date = new_py_datetime_ob(py, "date", (year, month, day)); let py_date: NaiveDate = py_date.extract().unwrap(); let date = NaiveDate::from_ymd_opt(year, month, day).unwrap(); - assert_eq!(py_date, date, "{}: {} != {}", name, date, py_date); + assert_eq!(py_date, date, "{name}: {date} != {py_date}"); }) }; @@ -1096,10 +812,7 @@ mod tests { assert_eq!( datetime.compare(&py_datetime).unwrap(), Ordering::Equal, - "{}: {} != {}", - name, - datetime, - py_datetime + "{name}: {datetime} != {py_datetime}" ); }; @@ -1139,10 +852,7 @@ mod tests { assert_eq!( datetime.compare(&py_datetime).unwrap(), Ordering::Equal, - "{}: {} != {}", - name, - datetime, - py_datetime + "{name}: {datetime} != {py_datetime}" ); }; @@ -1199,7 +909,7 @@ mod tests { let minute = 8; let second = 9; let micro = 999_999; - let tz_utc = timezone_utc(py); + let tz_utc = PyTzInfo::utc(py).unwrap(); let py_datetime = new_py_datetime_ob( py, "datetime", @@ -1329,13 +1039,7 @@ mod tests { .into_pyobject(py) .unwrap(); let py_time = new_py_datetime_ob(py, "time", (hour, minute, second, py_ms)); - assert!( - time.eq(&py_time).unwrap(), - "{}: {} != {}", - name, - time, - py_time - ); + assert!(time.eq(&py_time).unwrap(), "{name}: {time} != {py_time}"); }; check_time("regular", 3, 5, 7, 999_999, 999_999); @@ -1418,7 +1122,7 @@ mod tests { Python::with_gil(|py| { let globals = [("datetime", py.import("datetime").unwrap())].into_py_dict(py).unwrap(); - let code = format!("datetime.datetime.fromtimestamp({}).replace(tzinfo=datetime.timezone(datetime.timedelta(seconds={})))", timestamp, timedelta); + let code = format!("datetime.datetime.fromtimestamp({timestamp}).replace(tzinfo=datetime.timezone(datetime.timedelta(seconds={timedelta})))"); let t = py.eval(&CString::new(code).unwrap(), Some(&globals), None).unwrap(); // Get ISO 8601 string from python diff --git a/src/conversions/chrono_tz.rs b/src/conversions/chrono_tz.rs index 60a3bab4918..e8e5dc60e7a 100644 --- a/src/conversions/chrono_tz.rs +++ b/src/conversions/chrono_tz.rs @@ -37,45 +37,23 @@ use crate::conversion::IntoPyObject; use crate::exceptions::PyValueError; use crate::pybacked::PyBackedStr; -use crate::sync::GILOnceCell; -use crate::types::{any::PyAnyMethods, PyType}; -use crate::{intern, Bound, FromPyObject, Py, PyAny, PyErr, PyObject, PyResult, Python}; -#[allow(deprecated)] -use crate::{IntoPy, ToPyObject}; +use crate::types::{any::PyAnyMethods, PyTzInfo}; +use crate::{intern, Bound, FromPyObject, PyAny, PyErr, PyResult, Python}; use chrono_tz::Tz; use std::str::FromStr; -#[allow(deprecated)] -impl ToPyObject for Tz { - #[inline] - fn to_object(&self, py: Python<'_>) -> PyObject { - self.into_pyobject(py).unwrap().unbind() - } -} - -#[allow(deprecated)] -impl IntoPy for Tz { - #[inline] - fn into_py(self, py: Python<'_>) -> PyObject { - self.into_pyobject(py).unwrap().unbind() - } -} - impl<'py> IntoPyObject<'py> for Tz { - type Target = PyAny; + type Target = PyTzInfo; type Output = Bound<'py, Self::Target>; type Error = PyErr; fn into_pyobject(self, py: Python<'py>) -> Result { - static ZONE_INFO: GILOnceCell> = GILOnceCell::new(); - ZONE_INFO - .import(py, "zoneinfo", "ZoneInfo") - .and_then(|obj| obj.call1((self.name(),))) + PyTzInfo::timezone(py, self.name()) } } impl<'py> IntoPyObject<'py> for &Tz { - type Target = PyAny; + type Target = PyTzInfo; type Output = Bound<'py, Self::Target>; type Error = PyErr; @@ -99,6 +77,7 @@ impl FromPyObject<'_> for Tz { mod tests { use super::*; use crate::prelude::PyAnyMethods; + use crate::types::PyTzInfo; use crate::Python; use chrono::{DateTime, Utc}; use chrono_tz::Tz; @@ -170,8 +149,8 @@ mod tests { #[cfg(not(Py_GIL_DISABLED))] // https://github.com/python/cpython/issues/116738#issuecomment-2404360445 fn test_into_pyobject() { Python::with_gil(|py| { - let assert_eq = |l: Bound<'_, PyAny>, r: Bound<'_, PyAny>| { - assert!(l.eq(&r).unwrap(), "{:?} != {:?}", l, r); + let assert_eq = |l: Bound<'_, PyTzInfo>, r: Bound<'_, PyTzInfo>| { + assert!(l.eq(&r).unwrap(), "{l:?} != {r:?}"); }; assert_eq( @@ -186,11 +165,7 @@ mod tests { }); } - fn new_zoneinfo<'py>(py: Python<'py>, name: &str) -> Bound<'py, PyAny> { - zoneinfo_class(py).call1((name,)).unwrap() - } - - fn zoneinfo_class(py: Python<'_>) -> Bound<'_, PyAny> { - py.import("zoneinfo").unwrap().getattr("ZoneInfo").unwrap() + fn new_zoneinfo<'py>(py: Python<'py>, name: &str) -> Bound<'py, PyTzInfo> { + PyTzInfo::timezone(py, name).unwrap() } } diff --git a/src/conversions/either.rs b/src/conversions/either.rs index a514b1fde8d..7e8e3bfc2dc 100644 --- a/src/conversions/either.rs +++ b/src/conversions/either.rs @@ -48,28 +48,10 @@ use crate::inspect::types::TypeInfo; use crate::{ exceptions::PyTypeError, types::any::PyAnyMethods, Bound, FromPyObject, IntoPyObject, - IntoPyObjectExt, PyAny, PyErr, PyObject, PyResult, Python, + IntoPyObjectExt, PyAny, PyErr, PyResult, Python, }; -#[allow(deprecated)] -use crate::{IntoPy, ToPyObject}; use either::Either; -#[cfg_attr(docsrs, doc(cfg(feature = "either")))] -#[allow(deprecated)] -impl IntoPy for Either -where - L: IntoPy, - R: IntoPy, -{ - #[inline] - fn into_py(self, py: Python<'_>) -> PyObject { - match self { - Either::Left(l) => l.into_py(py), - Either::Right(r) => r.into_py(py), - } - } -} - #[cfg_attr(docsrs, doc(cfg(feature = "either")))] impl<'py, L, R> IntoPyObject<'py> for Either where @@ -106,22 +88,6 @@ where } } -#[cfg_attr(docsrs, doc(cfg(feature = "either")))] -#[allow(deprecated)] -impl ToPyObject for Either -where - L: ToPyObject, - R: ToPyObject, -{ - #[inline] - fn to_object(&self, py: Python<'_>) -> PyObject { - match self { - Either::Left(l) => l.to_object(py), - Either::Right(r) => r.to_object(py), - } - } -} - #[cfg_attr(docsrs, doc(cfg(feature = "either")))] impl<'py, L, R> FromPyObject<'py> for Either where diff --git a/src/conversions/eyre.rs b/src/conversions/eyre.rs index 42d7a12c872..4a501de9c69 100644 --- a/src/conversions/eyre.rs +++ b/src/conversions/eyre.rs @@ -119,7 +119,7 @@ impl From for PyErr { Err(error) => error, }; } - PyRuntimeError::new_err(format!("{:?}", error)) + PyRuntimeError::new_err(format!("{error:?}")) } } @@ -147,7 +147,7 @@ mod tests { #[test] fn test_pyo3_exception_contents() { let err = h().unwrap_err(); - let expected_contents = format!("{:?}", err); + let expected_contents = format!("{err:?}"); let pyerr = PyErr::from(err); Python::with_gil(|py| { @@ -166,7 +166,7 @@ mod tests { #[test] fn test_pyo3_exception_contents2() { let err = k().unwrap_err(); - let expected_contents = format!("{:?}", err); + let expected_contents = format!("{err:?}"); let pyerr = PyErr::from(err); Python::with_gil(|py| { diff --git a/src/conversions/hashbrown.rs b/src/conversions/hashbrown.rs index 0efe7f5161f..c302d28d2a1 100644 --- a/src/conversions/hashbrown.rs +++ b/src/conversions/hashbrown.rs @@ -22,47 +22,13 @@ use crate::{ any::PyAnyMethods, dict::PyDictMethods, frozenset::PyFrozenSetMethods, - set::{new_from_iter, try_new_from_iter, PySetMethods}, + set::{try_new_from_iter, PySetMethods}, PyDict, PyFrozenSet, PySet, }, - Bound, FromPyObject, PyAny, PyErr, PyObject, PyResult, Python, + Bound, FromPyObject, PyAny, PyErr, PyResult, Python, }; -#[allow(deprecated)] -use crate::{IntoPy, ToPyObject}; use std::{cmp, hash}; -#[allow(deprecated)] -impl ToPyObject for hashbrown::HashMap -where - K: hash::Hash + cmp::Eq + ToPyObject, - V: ToPyObject, - H: hash::BuildHasher, -{ - fn to_object(&self, py: Python<'_>) -> PyObject { - let dict = PyDict::new(py); - for (k, v) in self { - dict.set_item(k.to_object(py), v.to_object(py)).unwrap(); - } - dict.into_any().unbind() - } -} - -#[allow(deprecated)] -impl IntoPy for hashbrown::HashMap -where - K: hash::Hash + cmp::Eq + IntoPy, - V: IntoPy, - H: hash::BuildHasher, -{ - fn into_py(self, py: Python<'_>) -> PyObject { - let dict = PyDict::new(py); - for (k, v) in self { - dict.set_item(k.into_py(py), v.into_py(py)).unwrap(); - } - dict.into_any().unbind() - } -} - impl<'py, K, V, H> IntoPyObject<'py> for hashbrown::HashMap where K: IntoPyObject<'py> + cmp::Eq + hash::Hash, @@ -117,31 +83,6 @@ where } } -#[allow(deprecated)] -impl ToPyObject for hashbrown::HashSet -where - T: hash::Hash + Eq + ToPyObject, -{ - fn to_object(&self, py: Python<'_>) -> PyObject { - new_from_iter(py, self) - .expect("Failed to create Python set from hashbrown::HashSet") - .into() - } -} - -#[allow(deprecated)] -impl IntoPy for hashbrown::HashSet -where - K: IntoPy + Eq + hash::Hash, - S: hash::BuildHasher + Default, -{ - fn into_py(self, py: Python<'_>) -> PyObject { - new_from_iter(py, self.into_iter().map(|item| item.into_py(py))) - .expect("Failed to create Python set from hashbrown::HashSet") - .into() - } -} - impl<'py, K, H> IntoPyObject<'py> for hashbrown::HashSet where K: IntoPyObject<'py> + cmp::Eq + hash::Hash, diff --git a/src/conversions/indexmap.rs b/src/conversions/indexmap.rs index e3787e68091..d6995c14db2 100644 --- a/src/conversions/indexmap.rs +++ b/src/conversions/indexmap.rs @@ -89,43 +89,9 @@ use crate::conversion::IntoPyObject; use crate::types::*; -use crate::{Bound, FromPyObject, PyErr, PyObject, Python}; -#[allow(deprecated)] -use crate::{IntoPy, ToPyObject}; +use crate::{Bound, FromPyObject, PyErr, Python}; use std::{cmp, hash}; -#[allow(deprecated)] -impl ToPyObject for indexmap::IndexMap -where - K: hash::Hash + cmp::Eq + ToPyObject, - V: ToPyObject, - H: hash::BuildHasher, -{ - fn to_object(&self, py: Python<'_>) -> PyObject { - let dict = PyDict::new(py); - for (k, v) in self { - dict.set_item(k.to_object(py), v.to_object(py)).unwrap(); - } - dict.into_any().unbind() - } -} - -#[allow(deprecated)] -impl IntoPy for indexmap::IndexMap -where - K: hash::Hash + cmp::Eq + IntoPy, - V: IntoPy, - H: hash::BuildHasher, -{ - fn into_py(self, py: Python<'_>) -> PyObject { - let dict = PyDict::new(py); - for (k, v) in self { - dict.set_item(k.into_py(py), v.into_py(py)).unwrap(); - } - dict.into_any().unbind() - } -} - impl<'py, K, V, H> IntoPyObject<'py> for indexmap::IndexMap where K: IntoPyObject<'py> + cmp::Eq + hash::Hash, diff --git a/src/conversions/jiff.rs b/src/conversions/jiff.rs index 23ffddf99eb..814040e5fb4 100644 --- a/src/conversions/jiff.rs +++ b/src/conversions/jiff.rs @@ -48,27 +48,17 @@ //! ``` use crate::exceptions::{PyTypeError, PyValueError}; use crate::pybacked::PyBackedStr; -use crate::sync::GILOnceCell; +use crate::types::{PyAnyMethods, PyNone}; +use crate::types::{PyDate, PyDateTime, PyDelta, PyTime, PyTzInfo, PyTzInfoAccess}; #[cfg(not(Py_LIMITED_API))] -use crate::types::datetime::timezone_from_offset; -#[cfg(Py_LIMITED_API)] -use crate::types::datetime_abi3::{check_type, timezone_utc, DatetimeTypes}; -#[cfg(Py_LIMITED_API)] -use crate::types::IntoPyDict; -#[cfg(not(Py_LIMITED_API))] -use crate::types::{ - timezone_utc, PyDate, PyDateAccess, PyDateTime, PyDelta, PyDeltaAccess, PyTime, PyTimeAccess, - PyTzInfo, PyTzInfoAccess, -}; -use crate::types::{PyAnyMethods, PyNone, PyType}; -use crate::{intern, Bound, FromPyObject, IntoPyObject, Py, PyAny, PyErr, PyResult, Python}; +use crate::types::{PyDateAccess, PyDeltaAccess, PyTimeAccess}; +use crate::{intern, Bound, FromPyObject, IntoPyObject, PyAny, PyErr, PyResult, Python}; use jiff::civil::{Date, DateTime, Time}; use jiff::tz::{Offset, TimeZone}; use jiff::{SignedDuration, Span, Timestamp, Zoned}; #[cfg(feature = "jiff-02")] use jiff_02 as jiff; -#[cfg(not(Py_LIMITED_API))] fn datetime_to_pydatetime<'py>( py: Python<'py>, datetime: &DateTime, @@ -92,28 +82,6 @@ fn datetime_to_pydatetime<'py>( ) } -#[cfg(Py_LIMITED_API)] -fn datetime_to_pydatetime<'py>( - py: Python<'py>, - datetime: &DateTime, - fold: bool, - timezone: Option<&TimeZone>, -) -> PyResult> { - DatetimeTypes::try_get(py)?.datetime.bind(py).call( - ( - datetime.year(), - datetime.month(), - datetime.day(), - datetime.hour(), - datetime.minute(), - datetime.second(), - datetime.subsec_nanosecond() / 1000, - timezone, - ), - Some(&[("fold", fold as u8)].into_py_dict(py)?), - ) -} - #[cfg(not(Py_LIMITED_API))] fn pytime_to_time(time: &impl PyTimeAccess) -> PyResult