diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md index 2156c1d7322..820761ad89c 100644 --- a/.github/pull_request_template.md +++ b/.github/pull_request_template.md @@ -1,12 +1,21 @@ + diff --git a/.github/workflows/benches.yml b/.github/workflows/benches.yml index e14303cb8c6..7a1f92b024b 100644 --- a/.github/workflows/benches.yml +++ b/.github/workflows/benches.yml @@ -17,7 +17,7 @@ jobs: benchmarks: runs-on: ubuntu-24.04 steps: - - uses: actions/checkout@v6.0.3 + - uses: actions/checkout@v7.0.0 - uses: astral-sh/setup-uv@v7 with: save-cache: ${{ github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'CI-save-pr-cache') }} diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 5f816fc6b91..1f1a047521f 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -35,7 +35,7 @@ jobs: runs-on: ${{ inputs.os }} if: ${{ !(startsWith(inputs.python-version, 'graalpy') && startsWith(inputs.os, 'windows')) }} steps: - - uses: actions/checkout@v6.0.3 + - uses: actions/checkout@v7.0.0 with: ref: ${{ inputs.sha }} diff --git a/.github/workflows/changelog.yml b/.github/workflows/changelog.yml index e47c96f9cec..0ef6b0654bd 100644 --- a/.github/workflows/changelog.yml +++ b/.github/workflows/changelog.yml @@ -9,7 +9,7 @@ jobs: name: Check changelog entry runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6.0.3 + - uses: actions/checkout@v7.0.0 - uses: actions/setup-python@v6 with: python-version: '3.14' diff --git a/.github/workflows/ci-cache-warmup.yml b/.github/workflows/ci-cache-warmup.yml index a26af7e53bb..6bf9426595b 100644 --- a/.github/workflows/ci-cache-warmup.yml +++ b/.github/workflows/ci-cache-warmup.yml @@ -9,7 +9,7 @@ jobs: cross-compilation-windows: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6.0.3 + - uses: actions/checkout@v7.0.0 - uses: actions/setup-python@v6 with: python-version: "3.14" @@ -17,7 +17,7 @@ jobs: with: targets: x86_64-pc-windows-gnu,x86_64-pc-windows-msvc components: rust-src - - uses: actions/cache/restore@v5 + - uses: actions/cache/restore@v6 with: # https://github.com/PyO3/maturin/discussions/1953 path: ~/.cache/cargo-xwin @@ -28,7 +28,7 @@ jobs: sudo apt-get install -y mingw-w64 llvm pip install nox nox -s test-cross-compilation-windows - - uses: actions/cache/save@v5 + - uses: actions/cache/save@v6 with: path: ~/.cache/cargo-xwin key: cargo-xwin-cache diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 67a63c40e17..d6ee83d48b3 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -19,7 +19,7 @@ jobs: fmt: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6.0.3 + - uses: actions/checkout@v7.0.0 - uses: actions/setup-python@v6 with: python-version: "3.14" @@ -35,6 +35,8 @@ jobs: run: nox -s rumdl - name: Check `required-features` in Cargo.toml run: nox -s check-test-features + - name: Spell check + run: nox -s typos resolve: runs-on: ubuntu-latest @@ -51,7 +53,7 @@ jobs: # with the commit diff, because the merge may affect line numbers. coverage-sha: ${{ github.event_name == 'pull_request' && github.event.pull_request.head.sha || github.sha }} steps: - - uses: actions/checkout@v6.0.3 + - uses: actions/checkout@v7.0.0 - uses: actions/setup-python@v6 with: python-version: "3.14" @@ -64,7 +66,7 @@ jobs: needs: [fmt] runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6.0.3 + - uses: actions/checkout@v7.0.0 - uses: actions/setup-python@v6 with: python-version: "3.14" @@ -82,7 +84,7 @@ jobs: needs: [fmt, resolve] runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6.0.3 + - uses: actions/checkout@v7.0.0 - uses: dtolnay/rust-toolchain@master with: toolchain: ${{ needs.resolve.outputs.MSRV }} @@ -130,7 +132,7 @@ jobs: name: clippy/${{ matrix.target }}/${{ matrix.rust }} continue-on-error: ${{ matrix.rust != 'stable' }} steps: - - uses: actions/checkout@v6.0.3 + - uses: actions/checkout@v7.0.0 - uses: dtolnay/rust-toolchain@master with: toolchain: ${{ matrix.rust }} @@ -143,6 +145,29 @@ jobs: env: CARGO_BUILD_TARGET: ${{ matrix.target }} + check-nightly: + needs: [fmt] + runs-on: ubuntu-24.04-arm + strategy: + # If one platform fails, allow the rest to keep testing if `CI-no-fail-fast` label is present + fail-fast: ${{ !contains(github.event.pull_request.labels.*.name, 'CI-no-fail-fast') }} + matrix: + target: ["x86_64-unknown-linux-gnu"] + name: check-nightly/${{ matrix.target }}/${{ matrix.rust }} + continue-on-error: true + steps: + - uses: actions/checkout@v7.0.0 + - uses: dtolnay/rust-toolchain@nightly + with: + targets: ${{ matrix.target }} + components: clippy,rust-src + - uses: astral-sh/setup-uv@v7 + with: + save-cache: ${{ github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'CI-save-pr-cache') }} + - run: uvx nox -s check-all + env: + CARGO_BUILD_TARGET: ${{ matrix.target }} + build-pr: if: ${{ !contains(github.event.pull_request.labels.*.name, 'CI-build-full') && github.event_name == 'pull_request' }} name: python${{ matrix.python-version }}-${{ matrix.platform.python-architecture }} ${{ matrix.platform.os }} rust-${{ matrix.rust }} @@ -413,7 +438,7 @@ jobs: needs: [fmt] runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6.0.3 + - uses: actions/checkout@v7.0.0 - uses: actions/setup-python@v6 with: python-version: "3.14" @@ -434,7 +459,7 @@ jobs: needs: [fmt] runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6.0.3 + - uses: actions/checkout@v7.0.0 - uses: actions/setup-python@v6 with: python-version: "3.14" @@ -456,7 +481,7 @@ jobs: needs: [fmt] runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6.0.3 + - uses: actions/checkout@v7.0.0 - uses: actions/setup-python@v6 with: python-version: "3.14" @@ -474,11 +499,10 @@ jobs: needs: [fmt] runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6.0.3 - - uses: actions/setup-python@v6 + - uses: actions/checkout@v7.0.0 + - uses: astral-sh/setup-uv@v7 with: - python-version: 3.14 - id: setup-python + save-cache: ${{ needs.resolve.outputs.save-cache }} - name: Install Rust toolchain uses: dtolnay/rust-toolchain@stable with: @@ -487,42 +511,37 @@ jobs: - uses: actions/setup-node@v6 with: node-version: 24 - - run: python -m pip install --upgrade pip && pip install nox[uv] - - uses: actions/cache/restore@v5 + - uses: actions/cache/restore@v6 id: cache with: path: | .nox/emscripten - key: emscripten-${{ hashFiles('emscripten/*') }}-${{ hashFiles('noxfile.py') }}-${{ steps.setup-python.outputs.python-path }} + key: emscripten-${{ hashFiles('wasm/emscripten/*', 'wasm/common.mk') }}-${{ hashFiles('noxfile.py') }}-${{ env.UV_PYTHON }} - uses: Swatinem/rust-cache@v2 with: save-if: ${{ github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'CI-save-pr-cache') }} - name: Build if: steps.cache.outputs.cache-hit != 'true' - run: nox -s build-emscripten + run: uvx nox -s build-emscripten - name: Test - run: nox -s test-emscripten - - uses: actions/cache/save@v5 + run: uvx nox -s test-emscripten + - uses: actions/cache/save@v6 if: ${{ github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'CI-save-pr-cache') }} with: path: | .nox/emscripten - key: emscripten-${{ hashFiles('emscripten/*') }}-${{ hashFiles('noxfile.py') }}-${{ steps.setup-python.outputs.python-path }} + key: emscripten-${{ hashFiles('wasm/emscripten/*', 'wasm/common.mk') }}-${{ hashFiles('noxfile.py') }}-${{ env.UV_PYTHON }} wasm32-wasip1: name: wasm32-wasip1 if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || github.event_name != 'pull_request' }} needs: [fmt] runs-on: ubuntu-latest - env: - WASI_SDK_PATH: "/opt/wasi-sdk" - CPYTHON_PATH: "${{ github.workspace }}/wasi/cpython" steps: - - uses: actions/checkout@v6.0.3 - - uses: actions/setup-python@v6 + - uses: actions/checkout@v7.0.0 + - uses: astral-sh/setup-uv@v7 with: - python-version: 3.14 - id: setup-python + save-cache: ${{ needs.resolve.outputs.save-cache }} - name: Install Rust toolchain uses: dtolnay/rust-toolchain@stable with: @@ -531,66 +550,39 @@ jobs: - name: "Install wasmtime" uses: bytecodealliance/actions/wasmtime/setup@v1 - name: "Install WASI SDK" - run: | - mkdir ${{ env.WASI_SDK_PATH }} && \ - curl -s -S --location https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-24/wasi-sdk-24.0-x86_64-linux.tar.gz | \ - tar --strip-components 1 --directory ${{ env.WASI_SDK_PATH }} --extract --gunzip - $WASI_SDK_PATH/bin/clang --version - - uses: actions/cache/restore@v5 - id: cache-wasip1-python - with: - path: ${{ env.CPYTHON_PATH }}/cross-build/ - key: wasm32-wasip1-python - - uses: actions/checkout@v6.0.3 - with: - repository: python/cpython - ref: 3.14 - path: ${{ env.CPYTHON_PATH }} - fetch-depth: 1 - - name: Build - run: | - cd ${{ env.CPYTHON_PATH }} - cat >> Tools/wasm/wasi/config.site-wasm32-wasi <<'EOF' - - # Force-disable POSIX dynamic loading for WASI - ac_cv_func_dlopen=no - ac_cv_lib_dl_dlopen=no - EOF - python Tools/wasm/wasi build --quiet -- --config-cache - cp cross-build/wasm32-wasip1/libpython3.14.a \ - cross-build/wasm32-wasip1/Modules/_hacl/libHacl_HMAC.a \ - cross-build/wasm32-wasip1/Modules/_decimal/libmpdec/libmpdec.a \ - cross-build/wasm32-wasip1/Modules/expat/libexpat.a \ - cross-build/wasm32-wasip1/build/lib.wasi-wasm32-3.14/ + uses: bytecodealliance/setup-wasi-sdk-action@main + with: + version: "24" + # wasi sdk sets CC variables which break Python's configure script + # (it also sets WASI_SDK_PATH even without `add-to-path`, which is sufficient) + add-to-path: false + - uses: actions/cache/restore@v6 + id: cache + with: + path: | + .nox/wasi + key: wasi-${{ hashFiles('wasm/wasi/*', 'wasm/common.mk') }}-${{ hashFiles('noxfile.py') }}-${{ env.UV_PYTHON }} - uses: Swatinem/rust-cache@v2 with: save-if: ${{ github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'CI-save-pr-cache') }} + - name: Build + if: steps.cache.outputs.cache-hit != 'true' + run: uvx nox -s build-wasm - name: Test - env: - PYO3_CROSS_LIB_DIR: ${{ env.CPYTHON_PATH }}/cross-build/wasm32-wasip1/build/lib.wasi-wasm32-3.14/ - CARGO_BUILD_TARGET: wasm32-wasip1 - CARGO_TARGET_WASM32_WASIP1_RUNNER: wasmtime run --dir ${{ env.CPYTHON_PATH }}::/ --env PYTHONPATH=/lib - RUSTFLAGS: > - -C link-arg=-L${{ env.WASI_SDK_PATH }}/share/wasi-sysroot/lib/wasm32-wasi - -C link-arg=-lwasi-emulated-signal - -C link-arg=-lwasi-emulated-process-clocks - -C link-arg=-lwasi-emulated-getpid - -C link-arg=-lmpdec - -C link-arg=-lHacl_HMAC - -C link-arg=-lexpat - run: RUSTDOCFLAGS=$RUSTFLAGS cargo test - - uses: actions/cache/save@v5 + run: uvx nox -s test-wasm + - uses: actions/cache/save@v6 if: ${{ github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'CI-save-pr-cache') }} with: - path: ${{ env.CPYTHON_PATH }}/cross-build/ - key: ${{ steps.cache-wasip1-python.outputs.cache-primary-key }} + path: | + .nox/wasi + key: wasi-${{ hashFiles('wasm/wasi/*', 'wasm/common.mk') }}-${{ hashFiles('noxfile.py') }}-${{ env.UV_PYTHON }} test-debug: if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || github.event_name != 'pull_request' }} needs: [fmt] runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6.0.3 + - uses: actions/checkout@v7.0.0 - uses: Swatinem/rust-cache@v2 with: save-if: ${{ github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'CI-save-pr-cache') }} @@ -608,7 +600,7 @@ jobs: needs: [fmt] runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6.0.3 + - uses: actions/checkout@v7.0.0 - uses: Swatinem/rust-cache@v2 with: save-if: ${{ github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'CI-save-pr-cache') }} @@ -626,7 +618,7 @@ jobs: if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || github.event_name != 'pull_request' }} runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6.0.3 + - uses: actions/checkout@v7.0.0 with: ref: ${{ needs.resolve.outputs.coverage-sha }} - uses: astral-sh/setup-uv@v7 @@ -656,7 +648,7 @@ jobs: include: - rust: ${{ needs.resolve.outputs.MSRV }} steps: - - uses: actions/checkout@v6.0.3 + - uses: actions/checkout@v7.0.0 - uses: actions/setup-python@v6 with: python-version: "3.15-dev" @@ -695,15 +687,14 @@ jobs: # ubuntu x86_64 -> windows x86_64 - os: "ubuntu-latest" target: "x86_64-pc-windows-gnu" - # TODO: remove pyo3/generate-import-lib feature when maturin supports cross compiling to Windows without it - flags: "-i python3.13 --features pyo3/generate-import-lib" + flags: "-i python3.13" apt-packages: mingw-w64 llvm # windows x86_64 -> aarch64 - os: "windows-latest" target: "aarch64-pc-windows-msvc" - flags: "-i python3.13 --features pyo3/generate-import-lib" + flags: "-i python3.13" steps: - - uses: actions/checkout@v6.0.3 + - uses: actions/checkout@v7.0.0 with: ref: ${{ needs.resolve.outputs.coverage-sha }} - uses: astral-sh/setup-uv@v7 @@ -757,7 +748,7 @@ jobs: if: ${{ contains(github.event.pull_request.labels.*.name, 'CI-build-full') || github.event_name != 'pull_request' }} runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6.0.3 + - uses: actions/checkout@v7.0.0 with: ref: ${{ needs.resolve.outputs.coverage-sha }} - uses: astral-sh/setup-uv@v7 @@ -768,7 +759,7 @@ jobs: targets: x86_64-pc-windows-gnu,x86_64-pc-windows-msvc components: rust-src # load cache (prepared in ci-cache-warmup.yml) - - uses: actions/cache/restore@v5 + - uses: actions/cache/restore@v6 with: path: ~/.cache/cargo-xwin key: cargo-xwin-cache @@ -814,7 +805,7 @@ jobs: ] runs-on: ${{ matrix.platform.os }} steps: - - uses: actions/checkout@v6.0.3 + - uses: actions/checkout@v7.0.0 - uses: dtolnay/rust-toolchain@stable with: targets: ${{ matrix.platform.rust-target }} @@ -835,7 +826,7 @@ jobs: if: ${{ !contains(github.event.pull_request.labels.*.name, 'CI-build-full') && github.event_name == 'pull_request' }} runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6.0.3 + - uses: actions/checkout@v7.0.0 - uses: dtolnay/rust-toolchain@stable - uses: actions/setup-python@v6 with: @@ -852,7 +843,7 @@ jobs: matrix: checker: [mypy, pyrefly] steps: - - uses: actions/checkout@v6.0.3 + - uses: actions/checkout@v7.0.0 - uses: dtolnay/rust-toolchain@stable with: components: rust-src @@ -868,6 +859,7 @@ jobs: - fmt - check-msrv - clippy + - check-nightly - build-pr - build-full - valgrind diff --git a/.github/workflows/coverage-pr-base.yml b/.github/workflows/coverage-pr-base.yml index 8bc2a434fc7..6d45a0312a1 100644 --- a/.github/workflows/coverage-pr-base.yml +++ b/.github/workflows/coverage-pr-base.yml @@ -12,7 +12,7 @@ jobs: coverage-pr-base: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6.0.3 + - uses: actions/checkout@v7.0.0 - uses: actions/setup-python@v6 with: python-version: '3.14' diff --git a/.github/workflows/netlify-build.yml b/.github/workflows/netlify-build.yml index 1cf4a806b90..0eea2be60b0 100644 --- a/.github/workflows/netlify-build.yml +++ b/.github/workflows/netlify-build.yml @@ -19,7 +19,7 @@ jobs: guide-build: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6.0.3 + - uses: actions/checkout@v7.0.0 - uses: actions/setup-python@v6 with: python-version: "3.14" @@ -39,7 +39,7 @@ jobs: - name: Restore lychee cache id: restore-cache - uses: actions/cache/restore@v5 + uses: actions/cache/restore@v6 with: path: .lycheecache key: lychee-${{ github.run_id }} @@ -56,7 +56,7 @@ jobs: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - name: Save lychee cache - uses: actions/cache/save@v5 + uses: actions/cache/save@v6 if: ${{ github.ref == 'refs/heads/main' || contains(github.event.pull_request.labels.*.name, 'CI-save-pr-cache') }} with: path: .lycheecache diff --git a/.github/workflows/python-wheel.yml b/.github/workflows/python-wheel.yml index a6f19b5b385..bd08ed50c5f 100644 --- a/.github/workflows/python-wheel.yml +++ b/.github/workflows/python-wheel.yml @@ -19,13 +19,15 @@ jobs: matrix: target: [x86_64, x86, aarch64, armv7, s390x, ppc64le] steps: - - uses: actions/checkout@v6.0.3 + - uses: actions/checkout@v7.0.0 - uses: PyO3/maturin-action@v1 with: target: ${{ matrix.target }} args: --release --out dist --compatibility pypi manylinux: auto working-directory: pyo3-introspection + - uses: astral-sh/setup-uv@v8.1.0 + - run: uvx twine check pyo3-introspection/dist/* - uses: actions/upload-artifact@v7 with: name: wheels-linux-${{ matrix.target }} @@ -43,13 +45,15 @@ jobs: - runner: windows-11-arm target: aarch64 steps: - - uses: actions/checkout@v6.0.3 + - uses: actions/checkout@v7.0.0 - name: Build wheels uses: PyO3/maturin-action@v1 with: target: ${{ matrix.platform.target }} args: --release --out dist --compatibility pypi working-directory: pyo3-introspection + - uses: astral-sh/setup-uv@v8.1.0 + - run: uvx twine check pyo3-introspection/dist/* - uses: actions/upload-artifact@v7 with: name: wheels-windows-${{ matrix.platform.target }} @@ -65,12 +69,14 @@ jobs: - runner: macos-latest target: aarch64 steps: - - uses: actions/checkout@v6.0.3 + - uses: actions/checkout@v7.0.0 - uses: PyO3/maturin-action@v1 with: target: ${{ matrix.platform.target }} args: --release --out dist --compatibility pypi working-directory: pyo3-introspection + - uses: astral-sh/setup-uv@v8.1.0 + - run: uvx twine check pyo3-introspection/dist/* - uses: actions/upload-artifact@v7 with: name: wheels-macos-${{ matrix.platform.target }} @@ -79,12 +85,14 @@ jobs: pypi_sdist: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v6.0.3 + - uses: actions/checkout@v7.0.0 - uses: PyO3/maturin-action@v1 with: command: sdist args: --out dist working-directory: pyo3-introspection + - uses: astral-sh/setup-uv@v8.1.0 + - run: uvx twine check pyo3-introspection/dist/* - uses: actions/upload-artifact@v7 with: name: wheels-sdist @@ -105,3 +113,5 @@ jobs: path: pyo3-introspection/dist merge-multiple: true - uses: pypa/gh-action-pypi-publish@release/v1 + with: + packages-dir: pyo3-introspection/dist diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 09e2827ac15..a84dc721e87 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -17,7 +17,7 @@ jobs: runs-on: ubuntu-latest environment: release steps: - - uses: actions/checkout@df4cb1c069e1874edd31b4311f1884172cec0e10 # v6.0.3 + - uses: actions/checkout@9c091bb21b7c1c1d1991bb908d89e4e9dddfe3e0 # v7.0.0 with: # The tag to build or the tag received by the tag event ref: ${{ github.event.inputs.version || github.ref }} diff --git a/CHANGELOG.md b/CHANGELOG.md index 48598416018..9e688ead935 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -10,6 +10,33 @@ To see unreleased changes, please see the [CHANGELOG on the main branch guide](h +## [0.29.1] - 2026-08-02 + +### Changed + +- Use the inline definition of `Py_TYPE` in the unlimited API on 3.14+ [#6179](https://github.com/PyO3/pyo3/pull/6179) + +### Fixed + +- Fix incorrect pointer arithmetic in FFI definitions `PyObject_GET_WEAKREFS_LISTPTR` and `PyHeapType_GET_MEMBERS`. [#6145](https://github.com/PyO3/pyo3/pull/6145) +- Fix compilation error with `nightly` feature on PyPy and GraalPy due to `!Ungil` implementations for FFI types not available on those platforms. [#6146](https://github.com/PyO3/pyo3/pull/6146) +- Fix `append_to_inittab` and `PyInit_` internal module definition corruption on 32-bit and big-endian platforms on Python 3.15+. [#6150](https://github.com/PyO3/pyo3/pull/6150) +- Fix return value of `PyClassGuardMutSuper::as_super` being scoped to the full guard lifetime, now the `&mut` borrow of the `as_super()` call. [#6181](https://github.com/PyO3/pyo3/pull/6181) +- Fix builds for free-threaded interpreters older than 3.15 erroring with "cannot set a minimum Python version" when an `abi3t-py3*` feature is enabled and the configuration comes from `PYO3_CONFIG_FILE`, sysconfigdata or cross-compilation defaults. [#6192](https://github.com/PyO3/pyo3/pull/6192) +- Fix a memory leak when deallocating `#[pyclass(dict)]` instances with a populated `__dict__`. [#6198](https://github.com/PyO3/pyo3/pull/6198) +- Fix an abort inside a `#[pyclass]`'s GC traversal when the traversal is stopped early. [#6206](https://github.com/PyO3/pyo3/pull/6206) +- Fix reference cycles through the `__dict__` of a `#[pyclass(dict)]` never being collected. [#6206](https://github.com/PyO3/pyo3/pull/6206) +- Fix building on GraalPy 3.13. [#6208](https://github.com/PyO3/pyo3/pull/6208) +- Fix reference count leak of references to `#[pyclass]` type objects held by their instances on instance deallocation. [#6224](https://github.com/PyO3/pyo3/pull/6224) +- Fix FFI definitions `PyByteArray_GET_SIZE`, `PyList_GET_SIZE`, and `PySet_GET_SIZE` to use an atomic load for free-threaded Python. [#6230](https://github.com/PyO3/pyo3/pull/6230) +- Fix a memory leak on Python 3.11 and 3.12 where creating an instance of a `#[pyclass(dict)]` class leaked one empty dict per instance. [#6234](https://github.com/PyO3/pyo3/pull/6234) +- Fix `experimental-inspect` type stubs to emit the arguments of many magic methods as positional-only to match runtime behavior, rather than positional-or-keyword. [#6239](https://github.com/PyO3/pyo3/pull/6239) +- Fix `experimental-inspect` type stubs to emit the Python name rather than the Rust name for `#[pyfunction(name = "...")]`. [#6254](https://github.com/PyO3/pyo3/pull/6254) +- Fix `experimental-inspect` generating invalid internal JSON when `#[pymodule]` members are gated by `#[cfg]`. [#6255](https://github.com/PyO3/pyo3/pull/6255) +- Fix `__inplace_concat__` and `__inplace_repeat__` overriding `__concat__` and `__repeat__` when both were defined. [#6260](https://github.com/PyO3/pyo3/pull/6260) +- Fix conversion of out-of-range `time::Duration` values to return `OverflowError` instead of panicking. [#6266](https://github.com/PyO3/pyo3/pull/6266) +- Fix `experimental-inspect` type stubs padding blank lines inside indented docstrings. [#6270](https://github.com/PyO3/pyo3/pull/6270) + ## [0.29.0] - 2026-06-11 ### Packaging @@ -2655,7 +2682,9 @@ Yanked - Initial release -[Unreleased]: https://github.com/pyo3/pyo3/compare/v0.28.3...HEAD +[Unreleased]: https://github.com/pyo3/pyo3/compare/v0.29.1...HEAD +[0.29.1]: https://github.com/pyo3/pyo3/compare/v0.29.0...v0.29.1 +[0.29.0]: https://github.com/pyo3/pyo3/compare/v0.28.3...v0.29.0 [0.28.3]: https://github.com/pyo3/pyo3/compare/v0.28.2...v0.28.3 [0.28.2]: https://github.com/pyo3/pyo3/compare/v0.28.1...v0.28.2 [0.28.1]: https://github.com/pyo3/pyo3/compare/v0.28.0...v0.28.1 diff --git a/Cargo.toml b/Cargo.toml index 180d51a262d..4eb623f8ad5 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3" -version = "0.29.0" +version = "0.29.1" description = "Bindings to Python interpreter" authors = ["PyO3 Project and Contributors "] readme = "README.md" @@ -36,10 +36,10 @@ libc = "0.2.62" once_cell = "1.21" # ffi bindings to the python interpreter, split into a separate crate so they can be used independently -pyo3-ffi = { path = "pyo3-ffi", version = "=0.29.0" } +pyo3-ffi = { path = "pyo3-ffi", version = "=0.29.1" } # support crate for macros feature -pyo3-macros = { path = "pyo3-macros", version = "=0.29.0", optional = true } +pyo3-macros = { path = "pyo3-macros", version = "=0.29.1", optional = true } # support crate for multiple-pymethods feature inventory = { version = "0.3.5", optional = true } @@ -94,7 +94,7 @@ regex = "1.12.3" ctrlc = "3.5.2" [build-dependencies] -pyo3-build-config = { path = "pyo3-build-config", version = "=0.29.0" } +pyo3-build-config = { path = "pyo3-build-config", version = "=0.29.1" } [features] default = ["macros"] diff --git a/Contributing.md b/Contributing.md index 6d22e5f772c..2d330da1daf 100644 --- a/Contributing.md +++ b/Contributing.md @@ -1,6 +1,6 @@ # Contributing -Thank you for your interest in contributing to PyO3! All are welcome - please consider reading our [Code of Conduct](https://github.com/PyO3/pyo3/blob/main/Code-of-Conduct.md) to keep our community positive and inclusive. +Thank you for your interest in contributing to PyO3! All are welcome - please consider reading our [Code of Conduct] to keep our community positive and inclusive. If you are searching for ideas how to contribute, proceed to the ["Getting started contributing"](#getting-started-contributing) section. If you have found a specific issue to contribute to and need information about the development process, you may find the section ["Writing pull requests"](#writing-pull-requests) helpful. @@ -11,10 +11,34 @@ If you want to become familiar with the codebase, see Please join in with any part of PyO3 which interests you. We use GitHub issues to record all bugs and ideas. Feel free to request an issue to be assigned to you if you want to work on it. -You can browse the API of the non-public parts of PyO3 [here](https://pyo3.netlify.app/internal/doc/pyo3/index.html). +You can browse the API of the non-public parts of PyO3 [here](https://pyo3.rs/internal/doc/pyo3/index.html). The following sections also contain specific ideas on where to start contributing to PyO3. +## AI Contributions + +In alignment with our [Code of Conduct] we expect contributors using AI to respect the impact of their contributions on the PyO3 community. +A "contribution" includes pull requests, issues, comments, discussions, and any other interaction with the community. +See other projects such as FastAPI which describes ["Human Effort Denial of Service"](https://tiangolo.com/open-source/contributing/#human-effort-denial-of-service) and LLVM's discussion of ["Extractive Contributions"](https://llvm.org/docs/AIToolPolicy.html#extractive-contributions). + +We note that many have strong opinions on the use of AI, and the technical and social landscape is changing rapidly. +As a project PyO3 does not take a position on AI and its merits or issues; we accept that AI is a reality of current software development and wish to prioritise here the best outcomes for the PyO3 community. + +Please abide by the following guidelines for AI contributions: +- Do not submit raw AI output; you are expected to be able to use your own words to explain and reason about any contribution you make to the project. + Submitting raw AI output amplifies the time needed to read and reason about the output from just your own time to many PyO3 community members' time. +- Do not submit AI-generated solutions to issues labelled "easy" or "Good First Issue". + These issues are the best opportunity for new contributors to familiarise with the project and using AI to complete these adds value for nobody. +- Any code which you submit with AI assistance must clearly have sufficient ownership from you that you can assert full copyright in accordance with our MIT and Apache licensing. +- If you are not familiar enough with the project to own your AI-assisted contribution, consider using an issue or discussion to engage constructively with PyO3 maintainers. + We maintainers can always take over development (and use our own AI, if need be), if you do not have the time to build the necessary knowledge to own the solution yourself. + +Any contribution which looks like it was produced by AI without sufficient human effort to ensure its quality may be closed, deleted, and/or hidden at the PyO3 maintainers' discretion. +We will take into consideration (non-exhaustively) factors such as the size of the contribution and whether the GitHub account looks like it is engaging in widespread automated contributions, and whether we need to ban the offending GitHub account. +We prefer that authors disclose if a contribution is generated by AI so we can take that into account without guesswork. + +This policy may be updated in the future to reflect conventions in the open source ecosystem as they evolve. + ## Setting up a development environment To work and develop PyO3, you need Python & Rust installed on your system. @@ -32,8 +56,8 @@ The main nox commands we have implemented are: * `nox -s test` will run the full suite of recommended rust and python tests (>10 minutes) * `nox -s test-rust -- skip-full` will run a short suite of rust tests (2-3 minutes) -* `nox -s ruff` will check python linting and apply standard formatting rules -* `nox -s rustfmt` will check basic rust linting and apply standard formatting rules +* `nox -s ruff` will check python linting and formatting +* `nox -s rustfmt` will check rust formatting * `nox -s rumdl` will check the markdown in the guide * `nox -s clippy` will run clippy to make recommendations on rust style * `nox -s bench` will benchmark your rust code @@ -130,7 +154,7 @@ PRs are blocked from merging if CI is not successful. Formatting, linting and tests are checked for all Rust and Python code (the pipeline will abort early if formatting fails to save resources). In addition, all warnings in Rust code are disallowed (using `RUSTFLAGS="-D warnings"`). -Tests run with all supported Python versions with the latest stable Rust compiler, as well as for Python 3.9 with the minimum supported Rust version. +Tests run with all supported Python versions with the latest stable Rust compiler, as well as for the latest Python version with the minimum supported Rust version. If you are adding a new feature, you should add it to the `full` feature in our *Cargo.toml** so that it is tested in CI. @@ -164,17 +188,17 @@ PyO3 makes a lot of FFI calls to Python's C API using raw pointers. Where possib ```rust // dangerous -pyo3::ffi::Something(name.to_object(py).as_ptr()); +pyo3::ffi::Something(name.into_pyobject(py).as_ptr()); // because the following refactoring is a use-after-free error: -let name = name.to_object(py).as_ptr(); +let name = name.into_pyobject(py).as_ptr(); pyo3::ffi::Something(name) ``` -Instead, prefer to bind the safe owned `PyObject` wrapper before passing to ffi functions: +Instead, prefer to bind the safe owned smart pointer before passing to ffi functions: ```rust -let name: PyObject = name.to_object(py); +let name = name.into_pyobject(py); pyo3::ffi::Something(name.as_ptr()) // name will automatically be freed when it falls out of scope ``` @@ -189,7 +213,7 @@ Below are guidelines on what compatibility all PRs are expected to deliver for e ### Python -PyO3 supports all officially supported Python versions, as well as the latest PyPy3 release. All of these versions are tested in CI. +PyO3 supports all officially supported Python versions, as well as the latest PyPy3 and GraalPy releases. All of these versions are tested in CI. #### Adding support for new CPython versions @@ -234,7 +258,7 @@ cargo llvm-cov ``` - Then, generate an `lcov.info` file with ```shell -nox -s coverage -- lcov +nox -s coverage -- --lcov ``` You can install an IDE plugin to view the coverage. For example, if you use VSCode: - Add the [coverage-gutters](https://marketplace.visualstudio.com/items?itemName=ryanluker.vscode-coverage-gutters) plugin. @@ -266,3 +290,4 @@ In the meanwhile, some of our maintainers have personal GitHub sponsorship pages [nox]: https://github.com/theacodes/nox [pipx]: https://pipx.pypa.io/stable/ [ui_test]: https://github.com/oli-obk/ui_test +[Code of Conduct]: https://github.com/PyO3/pyo3/blob/main/Code-of-Conduct.md diff --git a/README.md b/README.md index 804fae9d7bd..c0b59450f39 100644 --- a/README.md +++ b/README.md @@ -71,7 +71,7 @@ name = "string_sum" crate-type = ["cdylib"] [dependencies] -pyo3 = "0.29.0" +pyo3 = "0.29.1" ``` **`src/lib.rs`** @@ -103,6 +103,8 @@ $ python '25' ``` +When checking runtime performance, run `maturin develop --release` to build with optimizations. + To make changes to the package, just edit the Rust source code and then re-run `maturin develop` to recompile. To run this all as a single copy-and-paste, use the bash script below (replace `string_sum` in the first command with the desired package name): @@ -137,7 +139,7 @@ Start a new project with `cargo new` and add `pyo3` to the `Cargo.toml` like th ```toml [dependencies.pyo3] -version = "0.29.0" +version = "0.29.1" # Enabling this cargo feature will cause PyO3 to start a Python interpreter on first call to `Python::attach` features = ["auto-initialize"] ``` @@ -219,10 +221,12 @@ about this topic. - [opendal](https://github.com/apache/opendal/tree/main/bindings/python) _A data access layer that allows users to easily and efficiently retrieve data from various storage services in a unified way._ - [orjson](https://github.com/ijl/orjson) _Fast Python JSON library._ - [ormsgpack](https://github.com/aviramha/ormsgpack) _Fast Python msgpack library._ +- [pdfcrate](https://github.com/ratazzi/pdfcrate) _An ergonomic, high-level PDF generation library for Rust and Python — a Prawn-style layout API for composing documents, not low-level PDF plumbing._ - [polars](https://github.com/pola-rs/polars) _Fast multi-threaded DataFrame library in Rust | Python | Node.js._ - [pycrdt](https://github.com/jupyter-server/pycrdt) _Python bindings for the Rust CRDT implementation [Yrs](https://github.com/y-crdt/y-crdt)._ - [pydantic-core](https://github.com/pydantic/pydantic-core) _Core validation logic for pydantic written in Rust._ - [primp](https://github.com/deedy5/primp) _The fastest python HTTP client that can impersonate web browsers by mimicking their headers and TLS/JA3/JA4/HTTP2 fingerprints._ +- [quebec](https://github.com/ratazzi/quebec) _A database-backed background job queue for Python, inspired by Rails' Solid Queue._ - [radiate](https://github.com/pkalivas/radiate): _A high-performance evolution engine for genetic programming and evolutionary algorithms._ - [rateslib](https://github.com/attack68/rateslib) _A fixed income library for Python using Rust extensions._ - [river](https://github.com/online-ml/river) _Online machine learning in python, the computationally heavy statistics algorithms are implemented in Rust._ @@ -230,6 +234,7 @@ about this topic. - [rust-python-coverage](https://github.com/cjermain/rust-python-coverage) _Example PyO3 project with automated test coverage for Rust and Python._ - [rnet](https://github.com/0x676e67/rnet) Asynchronous Python HTTP Client with Black Magic - [sail](https://github.com/lakehq/sail) _Unifying stream, batch, and AI workloads with Apache Spark compatibility._ +- [tibs](https://github.com/scott-griffiths/tibs) _A sleek Python library for binary data._ - [tiktoken](https://github.com/openai/tiktoken) _A fast BPE tokeniser for use with OpenAI's models._ - [tokenizers](https://github.com/huggingface/tokenizers/tree/main/bindings/python) _Python bindings to the Hugging Face tokenizers (NLP) written in Rust._ - [tzfpy](http://github.com/ringsaturn/tzfpy) _A fast package to convert longitude/latitude to timezone name._ diff --git a/emscripten/Makefile b/emscripten/Makefile deleted file mode 100644 index 4d2516e3fa0..00000000000 --- a/emscripten/Makefile +++ /dev/null @@ -1,85 +0,0 @@ -CURDIR=$(abspath .) - -# These three are passed in from nox. -BUILDROOT ?= $(CURDIR)/builddir -PYTHON ?= python3 -PYMAJORMINORMICRO ?= $(shell $(PYTHON) --version 2>&1 | awk '{print $$2}') - -export EMSDKDIR = $(PYTHONBUILD)/emsdk-cache - -PLATFORM=wasm32_emscripten -SYSCONFIGDATA_NAME=_sysconfigdata__$(PLATFORM) - -# Set version variables. -version_tuple := $(subst ., ,$(PYMAJORMINORMICRO:v%=%)) -PYMAJOR=$(word 1,$(version_tuple)) -PYMINOR=$(word 2,$(version_tuple)) -PYMICRO=$(word 3,$(version_tuple)) -PYVERSION=$(PYMAJORMINORMICRO) -PYMAJORMINOR=$(PYMAJOR).$(PYMINOR) - - -PYTHONURL=https://www.python.org/ftp/python/$(PYMAJORMINORMICRO)/Python-$(PYVERSION).tgz -# TODO: resume download once 3.14.4 ships with emscripten cache -# PYTHONTARBALL=$(BUILDROOT)/downloads/Python-$(PYVERSION).tgz -# PYTHONBUILD=$(BUILDROOT)/build/Python-$(PYVERSION) -PYTHONBUILD=$(BUILDROOT)/build/cpython - -PYTHONLIBDIR=$(BUILDROOT)/install/Python-$(PYVERSION)/lib - -CROSS_PYTHON=$(PYTHONBUILD)/cross-build/wasm32-emscripten/build/python/python.sh - -all: $(PYTHONLIBDIR)/libpython$(PYMAJORMINOR).a - -$(BUILDROOT)/.exists: - mkdir -p $(BUILDROOT) - touch $@ - -# TODO: use tarball once 3.14.4 ships with emscripten cache - -# $(PYTHONTARBALL): -# [ -d $(BUILDROOT)/downloads ] || mkdir -p $(BUILDROOT)/downloads -# wget -q -O $@ $(PYTHONURL) - -# $(PYTHONBUILD)/.patched: $(PYTHONTARBALL) -# [ -d $(PYTHONBUILD) ] || ( \ -# mkdir -p $(dir $(PYTHONBUILD));\ -# tar -C $(dir $(PYTHONBUILD)) -xf $(PYTHONTARBALL) \ -# ) -# touch $@ - -ifneq ($(PYMAJORMINOR),3.14) -$(error PYMAJORMINOR must be 3.14, got '$(PYMAJORMINOR)') -endif - -$(PYTHONBUILD)/.patched: $(BUILDROOT)/.exists - [ -d $(PYTHONBUILD) ] || ( \ - mkdir -p $(dir $(PYTHONBUILD));\ - git clone --depth 1 --branch 3.14 https://github.com/python/cpython $(PYTHONBUILD) \ - ) - touch $@ - -$(CROSS_PYTHON): $(PYTHONBUILD)/.patched - cd $(PYTHONBUILD) && \ - $(PYTHON) Tools/wasm/emscripten install-emscripten --quiet --emsdk-cache=$(EMSDKDIR) && \ - $(PYTHON) Tools/wasm/emscripten build --quiet --emsdk-cache=$(EMSDKDIR) - -$(PYTHONLIBDIR)/libpython$(PYMAJORMINOR).a: $(CROSS_PYTHON) - # Generate sysconfigdata - _PYTHON_SYSCONFIGDATA_NAME=$(SYSCONFIGDATA_NAME) _PYTHON_PROJECT_BASE=$(PYTHONBUILD)/cross-build/wasm32-emscripten/build/python $(CROSS_PYTHON) -m sysconfig --generate-posix-vars - cp `cat pybuilddir.txt`/$(SYSCONFIGDATA_NAME).py $(PYTHONBUILD)/Lib - - mkdir -p $(PYTHONLIBDIR) - # Make a static library for _hacl, for some reason these are missing from the build? - # source emsdk_env.sh to get emar in PATH, works best when done from the emsdk directory - EMSDK_ENV=$$(find $(PYTHONBUILD)/emsdk-cache -name 'emsdk_env.sh' | head -n 1) && \ - cd $$(dirname $$EMSDK_ENV) && \ - . $$EMSDK_ENV && \ - cd $(PYTHONBUILD)/cross-build/wasm32-emscripten/build/python && \ - emar rcs Modules/_hacl/libhacl.a Modules/_hacl/*.o - # Copy all .a libraries - find $(PYTHONBUILD)/cross-build/wasm32-emscripten/ -name '*.a' -exec cp {} $(PYTHONLIBDIR) \; - # Install Python stdlib - cp -r $(PYTHONBUILD)/Lib $(PYTHONLIBDIR)/python$(PYMAJORMINOR) -clean: - rm -rf $(BUILDROOT) diff --git a/examples/decorator/.template/pre-script.rhai b/examples/decorator/.template/pre-script.rhai index 7ffe1383756..e7a86412b03 100644 --- a/examples/decorator/.template/pre-script.rhai +++ b/examples/decorator/.template/pre-script.rhai @@ -1,4 +1,4 @@ -variable::set("PYO3_VERSION", "0.29.0"); +variable::set("PYO3_VERSION", "0.29.1"); file::rename(".template/Cargo.toml", "Cargo.toml"); file::rename(".template/pyproject.toml", "pyproject.toml"); file::delete(".template"); diff --git a/examples/maturin-starter/.template/pre-script.rhai b/examples/maturin-starter/.template/pre-script.rhai index 7ffe1383756..e7a86412b03 100644 --- a/examples/maturin-starter/.template/pre-script.rhai +++ b/examples/maturin-starter/.template/pre-script.rhai @@ -1,4 +1,4 @@ -variable::set("PYO3_VERSION", "0.29.0"); +variable::set("PYO3_VERSION", "0.29.1"); file::rename(".template/Cargo.toml", "Cargo.toml"); file::rename(".template/pyproject.toml", "pyproject.toml"); file::delete(".template"); diff --git a/examples/plugin/.template/pre-script.rhai b/examples/plugin/.template/pre-script.rhai index 384a86a0979..b83111fc0c3 100644 --- a/examples/plugin/.template/pre-script.rhai +++ b/examples/plugin/.template/pre-script.rhai @@ -1,4 +1,4 @@ -variable::set("PYO3_VERSION", "0.29.0"); +variable::set("PYO3_VERSION", "0.29.1"); file::rename(".template/Cargo.toml", "Cargo.toml"); file::rename(".template/plugin_api/Cargo.toml", "plugin_api/Cargo.toml"); file::delete(".template"); diff --git a/examples/setuptools-rust-starter/.template/pre-script.rhai b/examples/setuptools-rust-starter/.template/pre-script.rhai index de7ce10d1ae..692d05408c7 100644 --- a/examples/setuptools-rust-starter/.template/pre-script.rhai +++ b/examples/setuptools-rust-starter/.template/pre-script.rhai @@ -1,4 +1,4 @@ -variable::set("PYO3_VERSION", "0.29.0"); +variable::set("PYO3_VERSION", "0.29.1"); file::rename(".template/Cargo.toml", "Cargo.toml"); file::rename(".template/setup.cfg", "setup.cfg"); file::delete(".template"); diff --git a/examples/word-count/.template/pre-script.rhai b/examples/word-count/.template/pre-script.rhai index 7ffe1383756..e7a86412b03 100644 --- a/examples/word-count/.template/pre-script.rhai +++ b/examples/word-count/.template/pre-script.rhai @@ -1,4 +1,4 @@ -variable::set("PYO3_VERSION", "0.29.0"); +variable::set("PYO3_VERSION", "0.29.1"); file::rename(".template/Cargo.toml", "Cargo.toml"); file::rename(".template/pyproject.toml", "pyproject.toml"); file::delete(".template"); diff --git a/guide/src/building-and-distribution.md b/guide/src/building-and-distribution.md index 5d6efeeb681..8c9b29c9bf4 100644 --- a/guide/src/building-and-distribution.md +++ b/guide/src/building-and-distribution.md @@ -28,8 +28,8 @@ An example output of doing this is shown below: ```console $ PYO3_PRINT_CONFIG=1 cargo build - Compiling pyo3-ffi v0.29.0 (/Users/goldbaum/Documents/pyo3/pyo3-ffi) -error: failed to run custom build command for `pyo3-ffi v0.29.0 (/Users/goldbaum/Documents/pyo3/pyo3-ffi)` + Compiling pyo3-ffi v0.29.1 (/Users/goldbaum/Documents/pyo3/pyo3-ffi) +error: failed to run custom build command for `pyo3-ffi v0.29.1 (/Users/goldbaum/Documents/pyo3/pyo3-ffi)` Caused by: process didn't exit successfully: `/Users/goldbaum/Documents/pyo3/target/debug/build/pyo3-ffi-71f0882ba738a1f0/build-script-build` (exit status: 101) @@ -230,7 +230,7 @@ This makes binaries, tests, and examples "just work". However, Python extension modules on Unix must not link to libpython for [manylinux](https://www.python.org/dev/peps/pep-0513/) compliance. The downside of not linking to `libpython` is that binaries, tests, and examples (which usually embed Python) will fail to build. -As a result, PyO3 uses an envionment variable `PYO3_BUILD_EXTENSION_MODULE` to disable linking to `libpython`. +As a result, PyO3 uses an environment variable `PYO3_BUILD_EXTENSION_MODULE` to disable linking to `libpython`. This should only be set when building a library for distribution. `maturin >= 1.9.4` and `setuptools-rust >= 1.12` will set this for you automatically. diff --git a/guide/src/class.md b/guide/src/class.md index d0dcf262259..031a2a5d047 100644 --- a/guide/src/class.md +++ b/guide/src/class.md @@ -197,7 +197,7 @@ For arguments, see the [`Method arguments`](#method-arguments) section below. An initializer implements Python's `__init__` method. -It may be required when it's needed to control an object initalization flow on the Rust code. +It may be required when it's needed to control an object initialization flow on the Rust code. If possible handling this in `__new__` should be preferred, but in some cases, like subclassing native types, overwriting `__init__` might be necessary. For example, you define a class that extends `PyDict` and don't want that the original `__init__` method of `PyDict` been called. In this case by defining an own `__init__` method it's possible to stop initialization flow. diff --git a/guide/src/ecosystem/async-await.md b/guide/src/ecosystem/async-await.md index c772a316e42..4b291d675aa 100644 --- a/guide/src/ecosystem/async-await.md +++ b/guide/src/ecosystem/async-await.md @@ -4,7 +4,7 @@ *See the [dedicated documentation](../async-await.md)* If you are working with a Python library that makes use of async functions or wish to provide Python bindings for an async Rust library, [`pyo3-async-runtimes`](https://github.com/PyO3/pyo3-async-runtimes) likely has the tools you need. -It provides conversions between async functions in both Python and Rust and was designed with first-class support for popular Rust runtimes such as [`tokio`](https://tokio.rs/) and [`async-std`](https://async.rs/). +It provides conversions between async functions in both Python and Rust and was designed with first-class support for popular Rust runtimes such as [`tokio`](https://tokio.rs/) and [`async-std`](https://docs.rs/async-std/latest/async_std/). In addition, all async Python code runs on the default `asyncio` event loop, so `pyo3-async-runtimes` should work just fine with existing Python libraries. ## Additional Information diff --git a/guide/src/features.md b/guide/src/features.md index bff8d03c945..084b09c1fe2 100644 --- a/guide/src/features.md +++ b/guide/src/features.md @@ -15,7 +15,7 @@ Extensions targeting the stable ABI defined by Python 3.X can be imported by any There are two "flavors" of stable ABI: `abi3`, supported on Python 3.2 and newer but *not* the free-threaded builds and `abi3t`, supported on Python 3.15 and newer for both the GIL-enabled and free-threaded builds of Python. PyO3 supports building extensions targeting both flavors of stable ABI. -When PyO3 finds a "host" python interpreter and no `abi3` or `abi3t` feature is active, it will generate extensions targetting the version-specific ABI for the host Python version. +When PyO3 finds a "host" python interpreter and no `abi3` or `abi3t` feature is active, it will generate extensions targeting the version-specific ABI for the host Python version. For example, when ``pip`` or ``uv`` installs a Python package that includes Rust dependencies that depend on PyO3, the host interpreter is the interpreter running ``pip`` or the interpreter from the activated ``uv`` environment. To build for the stable ABIs, you must activate an `abi3` and/or an `abi3t` feature. diff --git a/guide/src/getting-started.md b/guide/src/getting-started.md index b7d6fb98046..f8b1d768842 100644 --- a/guide/src/getting-started.md +++ b/guide/src/getting-started.md @@ -192,6 +192,9 @@ $ python '25' ``` +By default, `maturin develop` builds your Rust code with Cargo's default [`dev` profile](https://doc.rust-lang.org/cargo/reference/profiles.html#default-profiles), which includes debug information and limited optimizations. +When checking runtime performance, run `maturin develop --release` to build with Cargo's `release` profile and enable optimizations. + For more instructions on how to use Python code from Rust, see the [Python from Rust](python-from-rust.md) page. ## Maturin Import Hook diff --git a/guide/src/glossary.md b/guide/src/glossary.md index 3a0065fd9ea..d3bc866c8c9 100644 --- a/guide/src/glossary.md +++ b/guide/src/glossary.md @@ -10,7 +10,7 @@ attached The [`Python::attach`]({{#PYO3_DOCS_URL}}/pyo3/marker/struct.Python.html#method.attach) method is used to attach the current thread to the Python interpreter and obtain a `Python` token which can be used to call Python APIs. extension module - : A Python module which is implemeted using native code (e.g. C, C++ or Rust) instead of Python. + : A Python module which is implemented using native code (e.g. C, C++ or Rust) instead of Python. See also [CPython's documentation on creating extension modules](https://docs.python.org/3/extending/extending.html). GIL-enabled Python diff --git a/guide/src/type-stub.md b/guide/src/type-stub.md index 3069c88434d..132dcf97382 100644 --- a/guide/src/type-stub.md +++ b/guide/src/type-stub.md @@ -63,8 +63,8 @@ class Class: @property def value(self) -> int: ... - def __eq__(self, other: Class) -> bool: ... - def __ne__(self, other: Class) -> bool: ... + def __eq__(self, value: object, /) -> bool: ... + def __ne__(self, value: object, /) -> bool: ... def list_of_int_identity(arg: "list[int]") -> "list[int]": ... ``` diff --git a/noxfile.py b/noxfile.py index d387150c0ca..c383dfe9eb2 100644 --- a/noxfile.py +++ b/noxfile.py @@ -42,7 +42,7 @@ except ImportError: requests = None -nox.options.sessions = ["test", "clippy", "rustfmt", "ruff", "rumdl", "docs"] +nox.options.sessions = ["test", "clippy", "rustfmt", "ruff", "rumdl", "docs", "typos"] PYO3_DIR = Path(__file__).parent PYO3_TARGET = Path(os.environ.get("CARGO_TARGET_DIR", PYO3_DIR / "target")).absolute() @@ -65,7 +65,7 @@ def _get_output(*args: str, env: Optional[Dict[str, str]] = None) -> str: def _parse_supported_interpreter_version( - python_impl: Literal["cpython", "pypy"], + python_impl: Literal["cpython", "pypy", "graalpy"], ) -> Tuple[str, str]: output = _get_output("cargo", "metadata", "--format-version=1", "--no-deps") cargo_packages = json.loads(output)["packages"] @@ -79,7 +79,7 @@ def _parse_supported_interpreter_version( def _supported_interpreter_versions( - python_impl: Literal["cpython", "pypy"], + python_impl: Literal["cpython", "pypy", "graalpy"], ) -> List[str]: min_version, max_version = _parse_supported_interpreter_version(python_impl) major = int(min_version.split(".")[0]) @@ -99,6 +99,7 @@ def _supported_interpreter_versions( p for p in PY_VERSIONS if p.endswith("t") and int(p.split(".")[1].strip("t")) > 14 ] PYPY_VERSIONS = _supported_interpreter_versions("pypy") +GRAALPY_VERSIONS = _supported_interpreter_versions("graalpy") @nox.session(venv_backend="none") @@ -231,9 +232,8 @@ def rustfmt(session: nox.Session): @nox.session(name="ruff") def ruff(session: nox.Session): - session.install("ruff") - _run(session, "ruff", "format", ".", "--check") - _run(session, "ruff", "check", ".") + _run(session, "uv", "run", "ruff", "format", ".", "--check") + _run(session, "uv", "run", "ruff", "check", ".") @nox.session(name="rumdl", venv_backend="none") @@ -247,6 +247,11 @@ def rumdl(session: nox.Session): ) +@nox.session(name="typos", venv_backend="none") +def typos(session: nox.Session): + _run(session, "uv", "run", "typos", *session.posargs, external=True) + + @nox.session(name="clippy", venv_backend="none") def clippy(session: nox.Session) -> bool: if not (_clippy(session) and _clippy_additional_workspaces(session)): @@ -421,7 +426,7 @@ def contributors(session: nox.Session) -> None: class EmscriptenInfo: def __init__(self): - self.emscripten_dir = PYO3_DIR / "emscripten" + self.emscripten_dir = PYO3_DIR / "wasm" / "emscripten" self.builddir = PYO3_DIR / ".nox/emscripten" self.builddir.mkdir(exist_ok=True, parents=True) @@ -498,6 +503,75 @@ def test_emscripten(session: nox.Session): ) +class WasiInfo: + def __init__(self): + self.wasi_dir = PYO3_DIR / "wasm" / "wasi" + self.builddir = PYO3_DIR / ".nox/wasi" + self.builddir.mkdir(exist_ok=True, parents=True) + + self.pyversion = sys.version.split()[0] + self.pymajor, self.pyminor = self.pyversion.split(".")[:2] + self.pymajorminor = f"{self.pymajor}.{self.pyminor}" + + # In CI the WASI SDK is installed by setup-wasi-sdk-action (sets WASI_SDK_PATH); + # otherwise the Makefile downloads it into the build dir. + wasi_sdk_env = os.environ.get("WASI_SDK_PATH") + self.wasi_sdk = ( + Path(wasi_sdk_env) if wasi_sdk_env else self.builddir / "wasi-sdk" + ) + self.cpython_dir = self.builddir / "build" / f"Python-{self.pyversion}" + crossbuild_dir = self.cpython_dir / "cross-build" / "wasm32-wasip1" + self.libdir = crossbuild_dir / "build" / f"lib.wasi-wasm32-{self.pymajorminor}" + + +@nox.session(name="build-wasm", venv_backend="none") +def build_wasm(session: nox.Session): + info = WasiInfo() + _run( + session, + "make", + "-C", + str(info.wasi_dir), + f"PYTHON={sys.executable}", + f"BUILDROOT={info.builddir}", + f"PYMAJORMINORMICRO={info.pyversion}", + external=True, + ) + + +@nox.session(name="test-wasm", venv_backend="none") +def test_wasm(session: nox.Session): + info = WasiInfo() + + target = "wasm32-wasip1" + + # if wasmtime was installed by build-wasm, this is where it would be; + # in CI it is installed by the wasmtime/setup action + session.env["PATH"] = ( + f"{info.builddir / 'wasmtime'}{os.pathsep}{os.environ['PATH']}" + ) + session.env["PYO3_CROSS_LIB_DIR"] = str(info.libdir) + session.env["CARGO_BUILD_TARGET"] = target + session.env["CARGO_TARGET_WASM32_WASIP1_RUNNER"] = ( + f"wasmtime run --dir {info.cpython_dir}::/ --env PYTHONPATH=/lib" + ) + session.env["RUSTFLAGS"] = " ".join( + [ + f"-C link-arg=-L{info.wasi_sdk}/share/wasi-sysroot/lib/wasm32-wasi", + "-C link-arg=-lwasi-emulated-signal", + "-C link-arg=-lwasi-emulated-process-clocks", + "-C link-arg=-lwasi-emulated-getpid", + "-C link-arg=-lmpdec", + "-C link-arg=-lHacl_HMAC", + "-C link-arg=-lexpat", + ] + ) + session.env["RUSTDOCFLAGS"] = session.env["RUSTFLAGS"] + _run(session, "rustup", "target", "add", target, "--toolchain", "stable") + + _run(session, "cargo", "test", *session.posargs, external=True) + + @nox.session(name="test-cross-compilation-windows") def test_cross_compilation_windows(session: nox.Session): session.install("cargo-xwin") @@ -539,8 +613,6 @@ def test_cross_compilation_windows(session: nox.Session): "build", "--manifest-path", "examples/maturin-starter/Cargo.toml", - "--features", - "pyo3/generate-import-lib", "--target", "x86_64-pc-windows-gnu", env=env, @@ -553,8 +625,6 @@ def test_cross_compilation_windows(session: nox.Session): "clang", "--manifest-path", "examples/maturin-starter/Cargo.toml", - "--features", - "pyo3/generate-import-lib", "--target", "x86_64-pc-windows-msvc", env=env, @@ -1900,6 +1970,9 @@ def _job_with_config(implementation, version): for version in PYPY_VERSIONS: _job_with_config("PyPy", version) + for version in GRAALPY_VERSIONS: + _job_with_config("GraalVM", version) + class _ConfigFile: def __init__(self, config_file) -> None: diff --git a/pyo3-benches/Cargo.toml b/pyo3-benches/Cargo.toml index 12d81ebe1b6..219b4c67fcc 100644 --- a/pyo3-benches/Cargo.toml +++ b/pyo3-benches/Cargo.toml @@ -13,7 +13,7 @@ pyo3 = { path = "../", features = ["auto-initialize", "full"] } pyo3-build-config = { path = "../pyo3-build-config" } [dev-dependencies] -codspeed-criterion-compat = "4.0" +codspeed-criterion-compat = "5.0" criterion = "0.8.0" num-bigint = "0.4.3" rust_decimal = { version = "1.0.0", default-features = false } diff --git a/pyo3-build-config/Cargo.toml b/pyo3-build-config/Cargo.toml index 39f750ef9cb..523c45034fe 100644 --- a/pyo3-build-config/Cargo.toml +++ b/pyo3-build-config/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3-build-config" -version = "0.29.0" +version = "0.29.1" description = "Build configuration for the PyO3 ecosystem" authors = ["PyO3 Project and Contributors "] keywords = ["pyo3", "python", "cpython", "ffi"] diff --git a/pyo3-build-config/src/impl_.rs b/pyo3-build-config/src/impl_.rs index 572ef4665bb..83e75863877 100644 --- a/pyo3-build-config/src/impl_.rs +++ b/pyo3-build-config/src/impl_.rs @@ -102,6 +102,63 @@ fn sanitize_stable_abi_version( } } +/// Selects which stable ABI (kind and minimum version) from the `abi3-py3*` +/// and `abi3t-py3*` features (if any) applies to the given interpreter. +/// +/// Interpreters which cannot target the requested stable ABI (e.g. +/// free-threaded CPython before 3.15) get a version-specific build instead. +/// A bare `abi3`/`abi3t` feature request resolves to the interpreter version. +fn applicable_stable_abi( + implementation: PythonImplementation, + version: PythonVersion, + gil_disabled: bool, + abi3_version: Option, + abi3t_version: Option, +) -> Option<(StableAbi, PythonVersion)> { + let exact = |requested: StableAbiVersion| match requested { + StableAbiVersion::Current => version, + StableAbiVersion::Target(target) => target, + }; + let abi3 = abi3_version.map(|v| (StableAbi::Abi3, exact(v))); + let abi3t = abi3t_version.map(|v| (StableAbi::Abi3t, exact(v))); + let selected = if version >= MINIMUM_SUPPORTED_VERSION_ABI3T { + match gil_disabled { + false => abi3t.or(abi3), + true => abi3t, + } + } else { + match gil_disabled { + false => abi3, + true => None, + } + }; + match implementation { + PythonImplementation::PyPy | PythonImplementation::GraalPy => { + selected.map(|(kind, _)| (kind, version)) + } + _ => selected, + } +} + +/// Like [`applicable_stable_abi`], but reads the `abi3-py3*`/`abi3t-py3*` +/// cargo features and keeps the interpreter version rather than the feature +/// minimum, so that `lib_name` matches the real libpython; +/// `apply_build_env` lowers the ABI to the feature minimum afterwards. +fn applicable_stable_abi_at_interpreter_version( + implementation: PythonImplementation, + version: PythonVersion, + gil_disabled: bool, +) -> Option<(StableAbi, PythonVersion)> { + applicable_stable_abi( + implementation, + version, + gil_disabled, + get_abi3_version(), + get_abi3t_version(), + ) + .map(|(kind, _)| (kind, version)) +} + /// Configuration needed by PyO3 to build for the correct Python implementation. /// /// The version and implementation fields correspond to the interpreter @@ -389,8 +446,8 @@ impl InterpreterConfig { fn from_interpreter( interpreter: impl AsRef, - abi3_version: Option, - abi3t_version: Option, + abi3_version: Option, + abi3t_version: Option, ) -> Result { const SCRIPT: &str = r#" # Allow the script to run on Python 2, so that nicer error can be printed later. @@ -498,27 +555,16 @@ print("gil_disabled", get_config_var("Py_GIL_DISABLED")) _ => panic!("Unknown Py_GIL_DISABLED value"), }; - let stable_abi_version = if !matches!( + let stable_abi = applicable_stable_abi( implementation, - PythonImplementation::PyPy | PythonImplementation::GraalPy - ) { - if version >= PythonVersion::PY315 { - match gil_disabled { - false => abi3t_version.or(abi3_version), - true => abi3t_version, - } - } else { - match gil_disabled { - false => abi3_version, - true => None, - } - } - } else { - None - }; + version, + gil_disabled, + abi3_version, + abi3t_version, + ); let target_abi = - PythonAbi::from_build_env(implementation, version, stable_abi_version, gil_disabled)?; + PythonAbi::from_stable_abi(implementation, version, stable_abi, gil_disabled)?; let cygwin = map["cygwin"].as_str() == "True"; @@ -610,7 +656,10 @@ print("gil_disabled", get_config_var("Py_GIL_DISABLED")) None => false, }; let cygwin = soabi.ends_with("cygwin"); - let target_abi = PythonAbi::from_build_env(implementation, version, None, gil_disabled)?; + let stable_abi = + applicable_stable_abi_at_interpreter_version(implementation, version, gil_disabled); + let target_abi = + PythonAbi::from_stable_abi(implementation, version, stable_abi, gil_disabled)?; let lib_name = default_lib_name_unix(target_abi, cygwin, sysconfigdata.get_value("LDVERSION"))?; let pointer_width = parse_key!(sysconfigdata, "SIZEOF_VOID_P") @@ -883,21 +932,19 @@ print("gil_disabled", get_config_var("Py_GIL_DISABLED")) } fn apply_build_env(mut self) -> Result { - let abi3_version = if self.target_abi.kind.is_free_threaded() - || matches!( - self.target_abi.implementation, - PythonImplementation::PyPy | PythonImplementation::GraalPy - ) { - None - } else { - get_abi3_version() - }; - self.target_abi = PythonAbi::from_build_env( - self.implementation, + // the host `implementation` may differ from the `target_abi` + // implementation; the recomputed ABI must stay on the target + let implementation = self.target_abi.implementation; + let gil_disabled = self.target_abi.kind().is_free_threaded(); + let stable_abi = applicable_stable_abi( + implementation, self.version, - exact_stable_abi_version(abi3_version.or(get_abi3t_version())), - self.target_abi.kind().is_free_threaded(), - )?; + gil_disabled, + get_abi3_version(), + get_abi3t_version(), + ); + self.target_abi = + PythonAbi::from_stable_abi(implementation, self.version, stable_abi, gil_disabled)?; Ok(self) } } @@ -1006,6 +1053,38 @@ impl FromStr for PythonAbi { } impl PythonAbi { + /// Constructs the ABI to target for an interpreter of `version`, given the + /// stable ABI kind and minimum Python version to target, if any. + /// + /// Callers decide whether a stable ABI applies to the interpreter; this + /// does not consult the `abi3`/`abi3t` cargo features. The minimum version + /// must not exceed the interpreter version. Without a stable ABI the + /// result is version-specific, free-threaded when `gil_disabled` is set. + fn from_stable_abi( + implementation: PythonImplementation, + version: PythonVersion, + stable_abi: Option<(StableAbi, PythonVersion)>, + gil_disabled: bool, + ) -> Result { + let builder = match stable_abi { + Some((kind, min_version)) => { + ensure!( + min_version <= version, + "cannot set a minimum Python version {} higher than the interpreter version {} \ + (the minimum Python version is implied by the {}-py3{} feature)", + min_version, + version, + kind, + min_version.minor + ); + PythonAbiBuilder::new(implementation, min_version).stable_abi(kind) + } + None if gil_disabled => PythonAbiBuilder::new(implementation, version).free_threaded(), + None => PythonAbiBuilder::new(implementation, version), + }; + builder.finalize() + } + pub fn from_build_env( implementation: PythonImplementation, version: PythonVersion, @@ -1330,6 +1409,7 @@ pub struct PythonVersion { } impl PythonVersion { + #[cfg(test)] pub(crate) const PY315: Self = PythonVersion { major: 3, minor: 15, @@ -2200,18 +2280,14 @@ fn default_cross_compile(cross_compile_config: &CrossCompileConfig) -> Result Result { /// /// Lowers the configured Python version to `abi3_version` or `abi3t_version` if required. fn get_host_interpreter( - abi3_version: Option, - abi3t_version: Option, + abi3_version: Option, + abi3t_version: Option, ) -> Result { let interpreter_path = find_interpreter()?; @@ -2563,10 +2639,7 @@ pub fn make_interpreter_config() -> Result { (abi3_version.is_none() && abi3t_version.is_none()) || require_libdir_for_target(&host); if have_python_interpreter() { - match get_host_interpreter( - exact_stable_abi_version(abi3_version), - exact_stable_abi_version(abi3t_version), - ) { + match get_host_interpreter(abi3_version, abi3t_version) { Ok(interpreter_config) => return Ok(interpreter_config), // Bail if the interpreter configuration is required to build. Err(e) if need_interpreter => return Err(e), @@ -3630,6 +3703,179 @@ mod tests { assert_eq!(config.version, host_version); } + #[test] + fn stable_abi_applicability() { + use PythonImplementation::*; + let abi3 = Some(StableAbiVersion::Target(PythonVersion::PY310)); + let abi3t = Some(StableAbiVersion::Target(PythonVersion::PY315)); + + // 3.14t cannot target any stable ABI, so the features are ignored + // rather than raising an error + assert_eq!( + applicable_stable_abi(CPython, PythonVersion::PY314, true, abi3, abi3t), + None + ); + // GIL-enabled below 3.15: only abi3 applies + assert_eq!( + applicable_stable_abi(CPython, PythonVersion::PY314, false, abi3, abi3t), + Some((StableAbi::Abi3, PythonVersion::PY310)) + ); + assert_eq!( + applicable_stable_abi(CPython, PythonVersion::PY314, false, None, abi3t), + None + ); + // 3.15+ GIL-enabled: abi3t preferred over abi3 + assert_eq!( + applicable_stable_abi(CPython, PythonVersion::PY315, false, abi3, abi3t), + Some((StableAbi::Abi3t, PythonVersion::PY315)) + ); + assert_eq!( + applicable_stable_abi(CPython, PythonVersion::PY315, false, abi3, None), + Some((StableAbi::Abi3, PythonVersion::PY310)) + ); + // 3.15+ free-threaded: only abi3t applies + assert_eq!( + applicable_stable_abi(CPython, PythonVersion::PY315, true, abi3, abi3t), + Some((StableAbi::Abi3t, PythonVersion::PY315)) + ); + assert_eq!( + applicable_stable_abi(CPython, PythonVersion::PY315, true, abi3, None), + None + ); + // a bare abi3/abi3t feature resolves to the interpreter version + assert_eq!( + applicable_stable_abi( + CPython, + PythonVersion::PY314, + false, + Some(StableAbiVersion::Current), + None + ), + Some((StableAbi::Abi3, PythonVersion::PY314)) + ); + assert_eq!( + applicable_stable_abi( + CPython, + PythonVersion::PY315, + true, + None, + Some(StableAbiVersion::Current) + ), + Some((StableAbi::Abi3t, PythonVersion::PY315)) + ); + // PyPy and GraalPy: the kind applies but the version is never lowered + assert_eq!( + applicable_stable_abi(PyPy, PythonVersion::PY311, false, abi3, abi3t), + Some((StableAbi::Abi3, PythonVersion::PY311)) + ); + assert_eq!( + applicable_stable_abi(GraalPy, PythonVersion::PY311, false, abi3, abi3t), + Some((StableAbi::Abi3, PythonVersion::PY311)) + ); + assert_eq!( + applicable_stable_abi(PyPy, PythonVersion::PY311, false, None, abi3t), + None + ); + } + + #[test] + fn apply_build_env_preserves_target_implementation() { + // the host `implementation` may differ from the `target_abi` + // implementation; recomputing the target ABI from the build + // environment must not switch it to the host's + let config = InterpreterConfig::from_reader( + "implementation=CPython\nversion=3.11\ntarget_abi=PyPy-gil_enabled-3.11".as_bytes(), + ) + .unwrap() + .apply_build_env() + .unwrap(); + assert_eq!( + config.target_abi.implementation(), + PythonImplementation::PyPy + ); + assert_eq!( + config.target_abi.kind(), + PythonAbiKind::VersionSpecific(GilUsed::GilEnabled) + ); + assert_eq!(config.target_abi.version(), PythonVersion::PY311); + } + + #[test] + fn python_abi_from_stable_abi() { + let implementation = PythonImplementation::CPython; + + // no stable ABI: version-specific, free-threaded per gil_disabled + let abi = + PythonAbi::from_stable_abi(implementation, PythonVersion::PY314, None, true).unwrap(); + assert_eq!( + abi.kind(), + PythonAbiKind::VersionSpecific(GilUsed::FreeThreaded) + ); + assert_eq!(abi.version(), PythonVersion::PY314); + + let abi = + PythonAbi::from_stable_abi(implementation, PythonVersion::PY314, None, false).unwrap(); + assert_eq!( + abi.kind(), + PythonAbiKind::VersionSpecific(GilUsed::GilEnabled) + ); + + // stable ABI: targets the minimum version + let abi = PythonAbi::from_stable_abi( + implementation, + PythonVersion::PY314, + Some((StableAbi::Abi3, PythonVersion::PY310)), + false, + ) + .unwrap(); + assert_eq!(abi.kind(), PythonAbiKind::Stable(StableAbi::Abi3)); + assert_eq!(abi.version(), PythonVersion::PY310); + + // a minimum above the interpreter version errors, naming the right feature + let error = PythonAbi::from_stable_abi( + implementation, + PythonVersion::PY314, + Some((StableAbi::Abi3t, PythonVersion::PY315)), + true, + ) + .unwrap_err(); + assert!(error.to_string().contains( + "cannot set a minimum Python version 3.15 higher than the interpreter version 3.14 \ + (the minimum Python version is implied by the abi3t-py315 feature)" + )); + } + + #[test] + fn config_file_applies_build_env() { + // no abi3/abi3t cargo features are set when running tests, so + // apply_build_env preserves the version-specific target ABI + let config = InterpreterConfig::from_reader( + "version=3.14\ntarget_abi=CPython-free_threaded-3.14\nbuild_flags=Py_GIL_DISABLED" + .as_bytes(), + ) + .unwrap() + .apply_build_env() + .unwrap(); + assert_eq!( + config.target_abi.kind(), + PythonAbiKind::VersionSpecific(GilUsed::FreeThreaded) + ); + assert_eq!(config.target_abi.version(), PythonVersion::PY314); + + // a stable ABI recorded in the config file is recomputed from the + // (unset) features, so the result is version-specific + let config = + InterpreterConfig::from_reader("version=3.12\ntarget_abi=CPython-abi3-3.10".as_bytes()) + .unwrap() + .apply_build_env() + .unwrap(); + assert_eq!( + config.target_abi.kind(), + PythonAbiKind::VersionSpecific(GilUsed::GilEnabled) + ); + assert_eq!(config.target_abi.version(), PythonVersion::PY312); + } + #[test] fn abi3_version_cannot_be_higher_than_interpreter() { if !have_python_interpreter() { @@ -3650,10 +3896,10 @@ mod tests { } let interpreter = get_host_interpreter( - Some(PythonVersion { + Some(StableAbiVersion::Target(PythonVersion { major: 3, minor: 45, - }), + })), None, ); if !host_free_threaded { @@ -3661,7 +3907,10 @@ mod tests { "cannot set a minimum Python version 3.45 higher than the interpreter version" )); if host_version >= PythonVersion::PY313 { - let interpreter = get_host_interpreter(Some(PythonVersion::PY313), None); + let interpreter = get_host_interpreter( + Some(StableAbiVersion::Target(PythonVersion::PY313)), + None, + ); assert_eq!( interpreter.unwrap().target_abi.version(), PythonVersion::PY313 @@ -3671,9 +3920,11 @@ mod tests { // If both features abi3 and abi3t features are active, the feature that "wins" depends on the host Python version if host_version >= PythonVersion::PY313 { - let interpreter = - get_host_interpreter(Some(PythonVersion::PY313), Some(PythonVersion::PY315)) - .unwrap(); + let interpreter = get_host_interpreter( + Some(StableAbiVersion::Target(PythonVersion::PY313)), + Some(StableAbiVersion::Target(PythonVersion::PY315)), + ) + .unwrap(); assert_eq!( interpreter.target_abi.version(), if host_version >= PythonVersion::PY315 { @@ -4002,7 +4253,7 @@ mod tests { .build_flags(flags) .finalize() .unwrap(); - // build flags win due to backward compatbility (abi3 feature is a no-op on ft builds) + // build flags win due to backward compatibility (abi3 feature is a no-op on ft builds) assert!(config.target_abi.kind() == PythonAbiKind::VersionSpecific(GilUsed::FreeThreaded)); // The reconciliation is order-independent: build_flags first, then stable_abi(Abi3) diff --git a/pyo3-ffi-check/build.rs b/pyo3-ffi-check/build.rs index b7b8fa3cbff..fc46f195389 100644 --- a/pyo3-ffi-check/build.rs +++ b/pyo3-ffi-check/build.rs @@ -18,7 +18,7 @@ fn main() { "--document-private-items", ]) .env("CARGO_TARGET_DIR", out_dir) - // forward target to the doc buid to ensure `--target` is honored + // forward target to the doc build to ensure `--target` is honored .env("CARGO_BUILD_TARGET", target) .status() .expect("failed to build definitions"); diff --git a/pyo3-ffi-check/macro/src/lib.rs b/pyo3-ffi-check/macro/src/lib.rs index 4f52e9a50cb..2c448640362 100644 --- a/pyo3-ffi-check/macro/src/lib.rs +++ b/pyo3-ffi-check/macro/src/lib.rs @@ -359,6 +359,7 @@ const MACRO_EXCLUSIONS: &[(&str, &str)] = &[ ("PyObject_GC_New", ""), ("PyObject_GC_NewVar", ""), ("PyObject_GC_Resize", ""), + ("PyObject_GET_WEAKREFS_LISTPTR", "not(Py_3_9)"), ("PyObject_IS_GC", "not(Py_3_9)"), ("PyObject_New", ""), ("PyObject_NewVar", ""), @@ -427,7 +428,7 @@ const MACRO_EXCLUSIONS: &[(&str, &str)] = &[ ("PyVectorcall_NARGS", "not(Py_3_12)"), ("Py_CLEAR", ""), ("Py_CompileString", "not(Py_3_10)"), - ("Py_CompileStringFlags", "not(PyPy)"), + ("Py_CompileStringFlags", "all(not(PyPy), not(Py_3_15))"), ("Py_DECREF", ""), ("Py_Ellipsis", ""), ("Py_False", ""), @@ -467,7 +468,6 @@ const EXCLUDED_SYMBOLS: &[&str] = &[ "PyCode_New", "PyCode_NewWithPosOnlyArgs", "PyCFunction_New", - "PyObject_GET_WEAKREFS_LISTPTR", "PyFrame_BlockSetup", "PySys_AddWarnOption", "PySys_AddWarnOptionUnicode", diff --git a/pyo3-ffi/Cargo.toml b/pyo3-ffi/Cargo.toml index 75dcf1fddef..abfbca8724b 100644 --- a/pyo3-ffi/Cargo.toml +++ b/pyo3-ffi/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3-ffi" -version = "0.29.0" +version = "0.29.1" description = "Python-API bindings for the PyO3 ecosystem" authors = ["PyO3 Project and Contributors "] keywords = ["pyo3", "python", "cpython", "ffi"] @@ -47,7 +47,7 @@ generate-import-lib = ["pyo3-build-config/generate-import-lib"] paste = "1" [build-dependencies] -pyo3-build-config = { path = "../pyo3-build-config", version = "=0.29.0" } +pyo3-build-config = { path = "../pyo3-build-config", version = "=0.29.1" } [lints] workspace = true @@ -59,3 +59,7 @@ max-version = "3.15" # inclusive [package.metadata.pypy] min-version = "3.11" max-version = "3.11" # inclusive + +[package.metadata.graalpy] +min-version = "3.12" +max-version = "3.13" # inclusive diff --git a/pyo3-ffi/README.md b/pyo3-ffi/README.md index 761b0918c0d..72b59cb6b97 100644 --- a/pyo3-ffi/README.md +++ b/pyo3-ffi/README.md @@ -12,7 +12,7 @@ Manual][capi] for up-to-date documentation. # Minimum supported Rust and Python versions -Requires Rust 1.63 or greater. +Requires Rust 1.83 or greater. `pyo3-ffi` supports the following Python distributions: - CPython 3.8 or greater @@ -41,12 +41,12 @@ name = "string_sum" crate-type = ["cdylib"] [dependencies] -pyo3-ffi = "0.29.0" +pyo3-ffi = "0.29.1" [build-dependencies] # This is only necessary if you need to configure your build based on # the Python version or the compile-time configuration for the interpreter. -pyo3_build_config = "0.29.0" +pyo3_build_config = "0.29.1" ``` If you need to use conditional compilation based on Python version or how diff --git a/pyo3-ffi/src/cpython/bytearrayobject.rs b/pyo3-ffi/src/cpython/bytearrayobject.rs index 8a922fcb6b5..52829b43fd8 100644 --- a/pyo3-ffi/src/cpython/bytearrayobject.rs +++ b/pyo3-ffi/src/cpython/bytearrayobject.rs @@ -1,5 +1,8 @@ +#[cfg(Py_GIL_DISABLED)] +use crate::cpython::pyatomic::_Py_atomic_load_ssize_relaxed; use crate::object::*; use crate::pyport::Py_ssize_t; +use crate::PyByteArray_Check; #[cfg(not(any(PyPy, GraalPy)))] use core::ffi::c_char; #[cfg(not(Py_3_9))] @@ -24,25 +27,26 @@ pub struct PyByteArrayObject { opaque_struct!(pub PyByteArrayObject); #[inline] -#[cfg(not(any(PyPy, GraalPy)))] -pub unsafe fn PyByteArray_AS_STRING(op: *mut PyObject) -> *mut c_char { - let byte_array = op as *mut PyByteArrayObject; - (*byte_array).ob_start +pub(crate) unsafe fn _PyByteArray_CAST(op: *mut PyObject) -> *mut PyByteArrayObject { + debug_assert_eq!(PyByteArray_Check(op), 1); + op.cast() } -/* #[inline] -#[cfg(Py_GIL_DISABLED)] -pub unsafe fn PyByteArray_GET_SIZE(op: *mut PyObject) -> Py_ssize_t { - let byte_array = op as *mut PyByteArrayObject; - // _Py_atomic_load_ssize_relaxed and _PyVarObject_CAST not implemented - // Insert Rust equivalent of the next line: - return _Py_atomic_load_ssize_relaxed(&(_PyVarObject_CAST(byte_array)->ob_size)); +#[cfg(not(any(PyPy, GraalPy)))] +pub unsafe fn PyByteArray_AS_STRING(op: *mut PyObject) -> *mut c_char { + (*_PyByteArray_CAST(op)).ob_start } -*/ #[inline] -#[cfg(not(Py_GIL_DISABLED))] pub unsafe fn PyByteArray_GET_SIZE(op: *mut PyObject) -> Py_ssize_t { - Py_SIZE(op) + let byte_array = _PyByteArray_CAST(op); + #[cfg(Py_GIL_DISABLED)] + { + _Py_atomic_load_ssize_relaxed(&raw const (*_PyVarObject_CAST(byte_array.cast())).ob_size) + } + #[cfg(not(Py_GIL_DISABLED))] + { + Py_SIZE(byte_array.cast()) + } } diff --git a/pyo3-ffi/src/cpython/dictobject.rs b/pyo3-ffi/src/cpython/dictobject.rs index 1bd75d8c6c5..df98a94c870 100644 --- a/pyo3-ffi/src/cpython/dictobject.rs +++ b/pyo3-ffi/src/cpython/dictobject.rs @@ -2,6 +2,8 @@ use crate::object::*; #[cfg(not(any(PyPy, GraalPy)))] use crate::pyport::Py_ssize_t; +#[cfg(all(GraalPy, Py_3_13))] +use crate::PyObject; #[cfg(all(not(PyPy), Py_3_13))] use core::ffi::c_char; diff --git a/pyo3-ffi/src/cpython/listobject.rs b/pyo3-ffi/src/cpython/listobject.rs index 694e6bc4290..d2f2b280a11 100644 --- a/pyo3-ffi/src/cpython/listobject.rs +++ b/pyo3-ffi/src/cpython/listobject.rs @@ -1,6 +1,10 @@ +#[cfg(Py_GIL_DISABLED)] +use crate::cpython::pyatomic::_Py_atomic_load_ssize_relaxed; use crate::object::*; #[cfg(not(PyPy))] use crate::pyport::Py_ssize_t; +#[cfg(not(PyPy))] +use crate::PyList_Check; #[cfg(not(PyPy))] #[repr(C)] @@ -15,26 +19,40 @@ pub struct PyListObject { pub ob_base: PyObject, } -// skipped _PyList_Extend -// skipped _PyList_DebugMallocStats -// skipped _PyList_CAST (used inline below) +#[inline] +#[cfg(not(PyPy))] +pub(crate) unsafe fn _PyList_CAST(op: *mut PyObject) -> *mut PyListObject { + debug_assert_eq!(PyList_Check(op), 1); + op.cast() +} + +#[inline] +#[cfg(not(PyPy))] +pub unsafe fn PyList_GET_SIZE(op: *mut PyObject) -> Py_ssize_t { + let list = _PyList_CAST(op); + #[cfg(Py_GIL_DISABLED)] + { + _Py_atomic_load_ssize_relaxed(&raw const (*_PyVarObject_CAST(list.cast())).ob_size) + } + #[cfg(not(Py_GIL_DISABLED))] + { + Py_SIZE(list.cast()) + } +} /// Macro, trading safety for speed #[inline] #[cfg(not(any(PyPy, GraalPy)))] pub unsafe fn PyList_GET_ITEM(op: *mut PyObject, i: Py_ssize_t) -> *mut PyObject { - *(*(op as *mut PyListObject)).ob_item.offset(i) + *(*_PyList_CAST(op)).ob_item.offset(i) } /// Macro, *only* to be used to fill in brand new lists #[inline] #[cfg(not(any(PyPy, GraalPy)))] pub unsafe fn PyList_SET_ITEM(op: *mut PyObject, i: Py_ssize_t, v: *mut PyObject) { - *(*(op as *mut PyListObject)).ob_item.offset(i) = v; + *(*_PyList_CAST(op)).ob_item.offset(i) = v; } -#[inline] -#[cfg(not(PyPy))] -pub unsafe fn PyList_GET_SIZE(op: *mut PyObject) -> Py_ssize_t { - Py_SIZE(op) -} +// skipped _PyList_Extend +// skipped _PyList_DebugMallocStats diff --git a/pyo3-ffi/src/cpython/mod.rs b/pyo3-ffi/src/cpython/mod.rs index 2705740d832..8fe53588384 100644 --- a/pyo3-ffi/src/cpython/mod.rs +++ b/pyo3-ffi/src/cpython/mod.rs @@ -32,6 +32,7 @@ pub(crate) mod marshal; pub(crate) mod methodobject; pub(crate) mod object; pub(crate) mod objimpl; +pub(crate) mod pyatomic; pub(crate) mod pydebug; pub(crate) mod pyerrors; #[cfg(not(PyPy))] @@ -43,6 +44,7 @@ pub(crate) mod pythonrun; pub(crate) mod floatobject; pub(crate) mod pyframe; pub(crate) mod pyhash; +pub(crate) mod setobject; pub(crate) mod traceback; pub(crate) mod tupleobject; pub(crate) mod unicodeobject; @@ -91,6 +93,7 @@ pub use self::pylifecycle::*; pub use self::pymem::*; pub use self::pystate::*; pub use self::pythonrun::*; +pub use self::setobject::*; pub use self::traceback::*; pub use self::tupleobject::*; pub use self::unicodeobject::*; diff --git a/pyo3-ffi/src/cpython/object.rs b/pyo3-ffi/src/cpython/object.rs index 735465a48b8..5cba19215fb 100644 --- a/pyo3-ffi/src/cpython/object.rs +++ b/pyo3-ffi/src/cpython/object.rs @@ -321,7 +321,7 @@ pub struct PyHeapTypeObject { #[cfg(not(Py_3_11))] pub unsafe fn PyHeapType_GET_MEMBERS(etype: *mut PyHeapTypeObject) -> *mut PyMemberDef { let py_type = object::Py_TYPE(etype as *mut object::PyObject); - let ptr = etype.offset((*py_type).tp_basicsize); + let ptr = etype.byte_offset((*py_type).tp_basicsize); ptr as *mut PyMemberDef } diff --git a/pyo3-ffi/src/cpython/objimpl.rs b/pyo3-ffi/src/cpython/objimpl.rs index 703a00e742d..ca01547b774 100644 --- a/pyo3-ffi/src/cpython/objimpl.rs +++ b/pyo3-ffi/src/cpython/objimpl.rs @@ -29,6 +29,16 @@ extern_libpython! { #[cfg(Py_3_9)] pub fn PyObject_IS_GC(o: *mut PyObject) -> c_int; + + #[cfg(all(Py_3_9, not(any(PyPy, GraalPy))))] + pub fn PyObject_GET_WEAKREFS_LISTPTR(o: *mut PyObject) -> *mut *mut PyObject; +} + +#[inline] +#[cfg(not(Py_3_9))] +pub unsafe fn PyObject_GET_WEAKREFS_LISTPTR(o: *mut PyObject) -> *mut *mut PyObject { + let weaklistoffset = (*Py_TYPE(o)).tp_weaklistoffset; + o.byte_offset(weaklistoffset) as *mut *mut PyObject } #[inline] @@ -49,10 +59,4 @@ pub unsafe fn PyType_SUPPORTS_WEAKREFS(t: *mut PyTypeObject) -> c_int { ((*t).tp_weaklistoffset > 0) as c_int } -#[inline] -pub unsafe fn PyObject_GET_WEAKREFS_LISTPTR(o: *mut PyObject) -> *mut *mut PyObject { - let weaklistoffset = (*Py_TYPE(o)).tp_weaklistoffset; - o.offset(weaklistoffset) as *mut *mut PyObject -} - // skipped PyUnstable_Object_GC_NewWithExtraData diff --git a/pyo3-ffi/src/cpython/pyatomic.rs b/pyo3-ffi/src/cpython/pyatomic.rs new file mode 100644 index 00000000000..2b132850c62 --- /dev/null +++ b/pyo3-ffi/src/cpython/pyatomic.rs @@ -0,0 +1,165 @@ +use crate::pyport::Py_ssize_t; +use core::sync::atomic::{AtomicIsize, Ordering}; + +// skipped _Py_atomic_add_int +// skipped _Py_atomic_add_int8 +// skipped _Py_atomic_add_int16 +// skipped _Py_atomic_add_int32 +// skipped _Py_atomic_add_int64 +// skipped _Py_atomic_add_intptr +// skipped _Py_atomic_add_uint +// skipped _Py_atomic_add_uint8 +// skipped _Py_atomic_add_uint16 +// skipped _Py_atomic_add_uint32 +// skipped _Py_atomic_add_uint64 +// skipped _Py_atomic_add_uintptr +// skipped _Py_atomic_add_ssize + +// skipped _Py_atomic_compare_exchange_int +// skipped _Py_atomic_compare_exchange_int8 +// skipped _Py_atomic_compare_exchange_int16 +// skipped _Py_atomic_compare_exchange_int32 +// skipped _Py_atomic_compare_exchange_int64 +// skipped _Py_atomic_compare_exchange_intptr +// skipped _Py_atomic_compare_exchange_uint +// skipped _Py_atomic_compare_exchange_uint8 +// skipped _Py_atomic_compare_exchange_uint16 +// skipped _Py_atomic_compare_exchange_uint32 +// skipped _Py_atomic_compare_exchange_uint64 +// skipped _Py_atomic_compare_exchange_uintptr +// skipped _Py_atomic_compare_exchange_ssize +// skipped _Py_atomic_compare_exchange_ptr + +// skipped _Py_atomic_exchange_int +// skipped _Py_atomic_exchange_int8 +// skipped _Py_atomic_exchange_int16 +// skipped _Py_atomic_exchange_int32 +// skipped _Py_atomic_exchange_int64 +// skipped _Py_atomic_exchange_intptr +// skipped _Py_atomic_exchange_uint +// skipped _Py_atomic_exchange_uint8 +// skipped _Py_atomic_exchange_uint16 +// skipped _Py_atomic_exchange_uint32 +// skipped _Py_atomic_exchange_uint64 +// skipped _Py_atomic_exchange_uintptr +// skipped _Py_atomic_exchange_ssize +// skipped _Py_atomic_exchange_ptr + +// skipped _Py_atomic_and_uint8 +// skipped _Py_atomic_and_uint16 +// skipped _Py_atomic_and_uint32 +// skipped _Py_atomic_and_uint64 +// skipped _Py_atomic_and_uintptr + +// skipped _Py_atomic_or_uint8 +// skipped _Py_atomic_or_uint16 +// skipped _Py_atomic_or_uint32 +// skipped _Py_atomic_or_uint64 +// skipped _Py_atomic_or_uintptr + +// skipped _Py_atomic_load_int +// skipped _Py_atomic_load_int8 +// skipped _Py_atomic_load_int16 +// skipped _Py_atomic_load_int32 +// skipped _Py_atomic_load_int64 +// skipped _Py_atomic_load_intptr +// skipped _Py_atomic_load_uint8 +// skipped _Py_atomic_load_uint16 +// skipped _Py_atomic_load_uint32 +// skipped _Py_atomic_load_uint64 +// skipped _Py_atomic_load_uintptr +// skipped _Py_atomic_load_uint +// skipped _Py_atomic_load_ssize +// skipped _Py_atomic_load_ptr + +// skipped _Py_atomic_load_int_relaxed +// skipped _Py_atomic_load_char_relaxed +// skipped _Py_atomic_load_uchar_relaxed +// skipped _Py_atomic_load_short_relaxed +// skipped _Py_atomic_load_ushort_relaxed +// skipped _Py_atomic_load_long_relaxed +// skipped _Py_atomic_load_double_relaxed +// skipped _Py_atomic_load_llong_relaxed +// skipped _Py_atomic_load_int8_relaxed +// skipped _Py_atomic_load_int16_relaxed +// skipped _Py_atomic_load_int32_relaxed +// skipped _Py_atomic_load_int64_relaxed +// skipped _Py_atomic_load_intptr_relaxed +// skipped _Py_atomic_load_uint8_relaxed +// skipped _Py_atomic_load_uint16_relaxed +// skipped _Py_atomic_load_uint32_relaxed +// skipped _Py_atomic_load_uint64_relaxed +// skipped _Py_atomic_load_uintptr_relaxed +// skipped _Py_atomic_load_uint_relaxed + +#[inline] +pub(crate) unsafe fn _Py_atomic_load_ssize_relaxed(value: *const Py_ssize_t) -> Py_ssize_t { + AtomicIsize::from_ptr(value.cast_mut()).load(Ordering::Relaxed) +} + +// skipped _Py_atomic_load_ptr_relaxed +// skipped _Py_atomic_load_ullong_relaxed + +// skipped _Py_atomic_store_int +// skipped _Py_atomic_store_int8 +// skipped _Py_atomic_store_int16 +// skipped _Py_atomic_store_int32 +// skipped _Py_atomic_store_int64 +// skipped _Py_atomic_store_intptr +// skipped _Py_atomic_store_uint8 +// skipped _Py_atomic_store_uint16 +// skipped _Py_atomic_store_uint32 +// skipped _Py_atomic_store_uint64 +// skipped _Py_atomic_store_uintptr +// skipped _Py_atomic_store_uint +// skipped _Py_atomic_store_ptr +// skipped _Py_atomic_store_ssize + +// skipped _Py_atomic_store_int_relaxed +// skipped _Py_atomic_store_int8_relaxed +// skipped _Py_atomic_store_int16_relaxed +// skipped _Py_atomic_store_int32_relaxed +// skipped _Py_atomic_store_int64_relaxed +// skipped _Py_atomic_store_intptr_relaxed +// skipped _Py_atomic_store_uint8_relaxed +// skipped _Py_atomic_store_uint16_relaxed +// skipped _Py_atomic_store_uint32_relaxed +// skipped _Py_atomic_store_uint64_relaxed +// skipped _Py_atomic_store_uintptr_relaxed +// skipped _Py_atomic_store_uint_relaxed +// skipped _Py_atomic_store_ptr_relaxed +// skipped _Py_atomic_store_ssize_relaxed +// skipped _Py_atomic_store_ullong_relaxed +// skipped _Py_atomic_store_char_relaxed +// skipped _Py_atomic_store_uchar_relaxed +// skipped _Py_atomic_store_short_relaxed +// skipped _Py_atomic_store_ushort_relaxed +// skipped _Py_atomic_store_long_relaxed +// skipped _Py_atomic_store_float_relaxed +// skipped _Py_atomic_store_double_relaxed +// skipped _Py_atomic_store_llong_relaxed + +// skipped _Py_atomic_load_ptr_acquire +// skipped _Py_atomic_load_uintptr_acquire +// skipped _Py_atomic_store_ptr_release +// skipped _Py_atomic_store_uintptr_release +// skipped _Py_atomic_store_ssize_release +// skipped _Py_atomic_store_int8_release +// skipped _Py_atomic_store_int_release +// skipped _Py_atomic_load_int_acquire +// skipped _Py_atomic_store_uint_release +// skipped _Py_atomic_store_uint32_release +// skipped _Py_atomic_store_uint64_release +// skipped _Py_atomic_load_uint64_acquire +// skipped _Py_atomic_load_uint32_acquire +// skipped _Py_atomic_load_ssize_acquire + +// skipped _Py_atomic_fence_seq_cst +// skipped _Py_atomic_fence_acquire +// skipped _Py_atomic_fence_release + +// skipped _Py_atomic_load_ptr_consume +// skipped _Py_atomic_load_ulong +// skipped _Py_atomic_load_ulong_relaxed +// skipped _Py_atomic_store_ulong +// skipped _Py_atomic_store_ulong_relaxed diff --git a/pyo3-ffi/src/cpython/setobject.rs b/pyo3-ffi/src/cpython/setobject.rs new file mode 100644 index 00000000000..2977e431151 --- /dev/null +++ b/pyo3-ffi/src/cpython/setobject.rs @@ -0,0 +1,50 @@ +#[cfg(Py_GIL_DISABLED)] +use crate::pyatomic::_Py_atomic_load_ssize_relaxed; +#[cfg(not(any(PyPy, GraalPy)))] +use crate::{PyAnySet_Check, PyObject, Py_hash_t, Py_ssize_t}; + +pub const PySet_MINSIZE: usize = 8; + +#[cfg(not(any(PyPy, GraalPy)))] +#[repr(C)] +#[derive(Debug)] +pub struct setentry { + pub key: *mut PyObject, + pub hash: Py_hash_t, +} + +#[cfg(not(any(PyPy, GraalPy)))] +#[repr(C)] +#[derive(Debug)] +pub struct PySetObject { + pub ob_base: PyObject, + pub fill: Py_ssize_t, + pub used: Py_ssize_t, + pub mask: Py_ssize_t, + pub table: *mut setentry, + pub hash: Py_hash_t, + pub finger: Py_ssize_t, + pub smalltable: [setentry; PySet_MINSIZE], + pub weakreflist: *mut PyObject, +} + +#[inline] +#[cfg(not(any(PyPy, GraalPy)))] +pub(crate) unsafe fn _PySet_CAST(so: *mut PyObject) -> *mut PySetObject { + debug_assert_eq!(PyAnySet_Check(so), 1); + so.cast() +} + +#[inline] +#[cfg(not(any(PyPy, GraalPy)))] +pub unsafe fn PySet_GET_SIZE(so: *mut PyObject) -> Py_ssize_t { + let so = _PySet_CAST(so); + #[cfg(Py_GIL_DISABLED)] + { + _Py_atomic_load_ssize_relaxed(&raw const (*so).used) + } + #[cfg(not(Py_GIL_DISABLED))] + { + (*so).used + } +} diff --git a/pyo3-ffi/src/object.rs b/pyo3-ffi/src/object.rs index 6c2d4c65a4c..a9bc476b4b7 100644 --- a/pyo3-ffi/src/object.rs +++ b/pyo3-ffi/src/object.rs @@ -172,7 +172,10 @@ pub struct PyVarObject { pub _ob_size_graalpy: Py_ssize_t, } -// skipped private _PyVarObject_CAST +#[inline] +pub(crate) unsafe fn _PyVarObject_CAST(op: *mut PyObject) -> *mut PyVarObject { + op.cast() +} #[inline] #[cfg(not(any(GraalPy, PyPy, RustPython)))] @@ -204,7 +207,7 @@ extern_libpython! { } #[inline] -#[cfg(not(Py_3_14))] +#[cfg(not(all(Py_LIMITED_API, Py_3_14)))] pub unsafe fn Py_TYPE(ob: *mut PyObject) -> *mut PyTypeObject { #[cfg(not(GraalPy))] return (*ob).ob_type; @@ -212,7 +215,7 @@ pub unsafe fn Py_TYPE(ob: *mut PyObject) -> *mut PyTypeObject { return _Py_TYPE(ob); } -#[cfg(Py_3_14)] +#[cfg(all(Py_LIMITED_API, Py_3_14))] extern_libpython! { #[cfg_attr(PyPy, link_name = "PyPy_TYPE")] pub fn Py_TYPE(ob: *mut PyObject) -> *mut PyTypeObject; diff --git a/pyo3-ffi/src/setobject.rs b/pyo3-ffi/src/setobject.rs index 8fcb3006fcc..505b50d6bed 100644 --- a/pyo3-ffi/src/setobject.rs +++ b/pyo3-ffi/src/setobject.rs @@ -1,50 +1,7 @@ use crate::object::*; -#[cfg(not(any(Py_LIMITED_API, PyPy, GraalPy)))] -use crate::pyport::Py_hash_t; use crate::pyport::Py_ssize_t; use core::ffi::c_int; -pub const PySet_MINSIZE: usize = 8; - -#[cfg(not(any(Py_LIMITED_API, PyPy, GraalPy)))] -#[repr(C)] -#[derive(Debug)] -pub struct setentry { - pub key: *mut PyObject, - pub hash: Py_hash_t, -} - -#[cfg(not(any(Py_LIMITED_API, PyPy, GraalPy)))] -#[repr(C)] -#[derive(Debug)] -pub struct PySetObject { - pub ob_base: PyObject, - pub fill: Py_ssize_t, - pub used: Py_ssize_t, - pub mask: Py_ssize_t, - pub table: *mut setentry, - pub hash: Py_hash_t, - pub finger: Py_ssize_t, - pub smalltable: [setentry; PySet_MINSIZE], - pub weakreflist: *mut PyObject, -} - -// skipped -#[inline] -#[cfg(all(not(any(PyPy, GraalPy)), not(Py_LIMITED_API)))] -pub unsafe fn PySet_GET_SIZE(so: *mut PyObject) -> Py_ssize_t { - debug_assert_eq!(PyAnySet_Check(so), 1); - let so = so.cast::(); - (*so).used -} - -// skipped _PySet_Dummy - -extern_libpython! { - // skipped non-limited _PySet_NextEntry - // skipped non-limited _PySet_Update -} - #[cfg(not(RustPython))] extern_libpython! { #[cfg_attr(PyPy, link_name = "PyPySet_Type")] diff --git a/pyo3-introspection/Cargo.toml b/pyo3-introspection/Cargo.toml index 2771bd962ed..3d8f363b4fa 100644 --- a/pyo3-introspection/Cargo.toml +++ b/pyo3-introspection/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3-introspection" -version = "0.29.0" +version = "0.29.1" description = "Introspect dynamic libraries built with PyO3 to get metadata about the exported Python types" authors = ["PyO3 Project and Contributors "] homepage = "https://github.com/pyo3/pyo3" diff --git a/pyo3-introspection/LICENSE-APACHE b/pyo3-introspection/LICENSE-APACHE deleted file mode 120000 index 965b606f331..00000000000 --- a/pyo3-introspection/LICENSE-APACHE +++ /dev/null @@ -1 +0,0 @@ -../LICENSE-APACHE \ No newline at end of file diff --git a/pyo3-introspection/LICENSE-MIT b/pyo3-introspection/LICENSE-MIT deleted file mode 120000 index 76219eb72e8..00000000000 --- a/pyo3-introspection/LICENSE-MIT +++ /dev/null @@ -1 +0,0 @@ -../LICENSE-MIT \ No newline at end of file diff --git a/pyo3-introspection/src/stubs.rs b/pyo3-introspection/src/stubs.rs index 97ef14bcfc3..f89c250eef8 100644 --- a/pyo3-introspection/src/stubs.rs +++ b/pyo3-introspection/src/stubs.rs @@ -149,28 +149,26 @@ fn class_stubs(class: &Class, imports: &Imports) -> String { buffer.push_str(" ..."); } if let Some(docstring) = &class.docstring { - buffer.push_str("\n \"\"\""); - for line in docstring.lines() { - buffer.push_str("\n "); - buffer.push_str(line); - } - buffer.push_str("\n \"\"\""); + push_docstring(&mut buffer, " ", docstring); } for attribute in &class.attributes { // We do the indentation buffer.push_str("\n "); - buffer.push_str(&attribute_stubs(attribute, imports).replace('\n', "\n ")); + push_indented(&mut buffer, " ", &attribute_stubs(attribute, imports)); } for method in &class.methods { // We do the indentation buffer.push_str("\n "); - buffer - .push_str(&function_stubs(method, imports, Some(&class.name)).replace('\n', "\n ")); + push_indented( + &mut buffer, + " ", + &function_stubs(method, imports, Some(&class.name)), + ); } for inner_class in &class.inner_classes { // We do the indentation buffer.push_str("\n "); - buffer.push_str(&class_stubs(inner_class, imports).replace('\n', "\n ")); + push_indented(&mut buffer, " ", &class_stubs(inner_class, imports)); } buffer } @@ -232,18 +230,50 @@ fn function_stubs(function: &Function, imports: &Imports, class_name: Option<&st imports.serialize_expr(returns, &mut buffer); } if let Some(docstring) = &function.docstring { - buffer.push_str(":\n \"\"\""); - for line in docstring.lines() { - buffer.push_str("\n "); - buffer.push_str(line); - } - buffer.push_str("\n \"\"\""); + buffer.push(':'); + push_docstring(&mut buffer, " ", docstring); } else { buffer.push_str(": ..."); } buffer } +/// Appends `text` to `buffer`, prefixing every line after the first with `indent`. +/// +/// The first line is left alone because callers have already written the indentation for it; this +/// is the same contract `text.replace('\n', "\n{indent}")` had, minus one thing: a blank line stays +/// blank instead of being padded out to the indentation. Trailing whitespace on an otherwise empty +/// line is invisible in the source but still trailing whitespace, it trips `W293` in every Python +/// linter, and a generated file is exactly the kind of file nobody gets to hand-fix. +fn push_indented(buffer: &mut String, indent: &str, text: &str) { + for (index, line) in text.split('\n').enumerate() { + if index > 0 { + buffer.push('\n'); + if !line.is_empty() { + buffer.push_str(indent); + } + } + buffer.push_str(line); + } +} + +/// Appends a `"""`-quoted docstring indented by `indent`, starting on a fresh line. +fn push_docstring(buffer: &mut String, indent: &str, docstring: &str) { + buffer.push('\n'); + buffer.push_str(indent); + buffer.push_str("\"\"\""); + for line in docstring.lines() { + buffer.push('\n'); + if !line.is_empty() { + buffer.push_str(indent); + buffer.push_str(line); + } + } + buffer.push('\n'); + buffer.push_str(indent); + buffer.push_str("\"\"\""); +} + fn attribute_stubs(attribute: &Attribute, imports: &Imports) -> String { let mut buffer = attribute.name.clone(); if let Some(annotation) = &attribute.annotation { @@ -255,12 +285,7 @@ fn attribute_stubs(attribute: &Attribute, imports: &Imports) -> String { imports.serialize_expr(value, &mut buffer); } if let Some(docstring) = &attribute.docstring { - buffer.push_str("\n\"\"\""); - for line in docstring.lines() { - buffer.push('\n'); - buffer.push_str(line); - } - buffer.push_str("\n\"\"\""); + push_docstring(&mut buffer, "", docstring); } buffer } @@ -942,4 +967,58 @@ mod tests { assert_eq!(make_module_path_relative("foo", "foo.la", false), "."); assert_eq!(make_module_path_relative("foo", "bar", true), "foo"); } + + /// Docstrings are re-indented into the class or function body, and a paragraph break inside one + /// is an empty line. Padding it out to the body indentation is trailing whitespace, which + /// `W293` flags and which nobody can fix by hand in a generated file. + #[test] + fn docstring_blank_lines_are_not_padded_with_indentation() { + let module = Module { + name: "bar".into(), + modules: Vec::new(), + classes: vec![Class { + name: "Zulu".into(), + bases: Vec::new(), + methods: vec![Function { + name: "method".into(), + decorators: Vec::new(), + arguments: Arguments { + positional_only_arguments: Vec::new(), + arguments: Vec::new(), + vararg: None, + keyword_only_arguments: Vec::new(), + kwarg: None, + }, + returns: None, + is_async: false, + docstring: Some("Summary.\n\nDetail.".into()), + }], + attributes: Vec::new(), + decorators: Vec::new(), + inner_classes: Vec::new(), + docstring: Some("Class summary.\n\nClass detail.".into()), + }], + functions: Vec::new(), + attributes: vec![Attribute { + name: "CONST".into(), + value: None, + annotation: None, + docstring: Some("Const summary.\n\nConst detail.".into()), + }], + incomplete: false, + docstring: None, + }; + + let stubs = module_stubs(&module, &["foo"]); + assert!( + !stubs + .lines() + .any(|line| !line.is_empty() && line.trim().is_empty()), + "generated stubs contain a blank line padded with whitespace:\n{stubs:?}" + ); + // The indentation of the non-empty lines is unaffected. + assert!(stubs.contains("\n Class summary.\n\n Class detail.\n")); + assert!(stubs.contains("\n Summary.\n\n Detail.\n")); + assert!(stubs.contains("\nConst summary.\n\nConst detail.\n")); + } } diff --git a/pyo3-macros-backend/Cargo.toml b/pyo3-macros-backend/Cargo.toml index 31bfbbb8256..35de7109471 100644 --- a/pyo3-macros-backend/Cargo.toml +++ b/pyo3-macros-backend/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3-macros-backend" -version = "0.29.0" +version = "0.29.1" description = "Code generation for PyO3 package" authors = ["PyO3 Project and Contributors "] keywords = ["pyo3", "python", "cpython", "ffi"] diff --git a/pyo3-macros-backend/src/introspection.rs b/pyo3-macros-backend/src/introspection.rs index c6b89b192f0..c597f11ec56 100644 --- a/pyo3-macros-backend/src/introspection.rs +++ b/pyo3-macros-backend/src/introspection.rs @@ -425,20 +425,31 @@ impl IntrospectionNode<'_> { } Self::List(list) => { content.push_str("["); - for (i, AttributedIntrospectionNode { node, attributes }) in - list.into_iter().enumerate() - { - if attributes.is_empty() { + if list.iter().all(|element| element.attributes.is_empty()) { + for (i, element) in list.into_iter().enumerate() { if i > 0 { content.push_str(","); } - node.add_to_serialization(content, pyo3_crate_path); - } else { + element.node.add_to_serialization(content, pyo3_crate_path); + } + } else { + // A `,` must only be written if at least one of the elements before the one + // it precedes is compiled in, so we gate it behind an `any(..)` of their + // `cfg`s on top of the element's own. This needs no special case: `any()` is + // false, so the first element gets no separator, and `all()` is true, so an + // element without `cfg` makes every later separator unconditional. + let mut preceding = Vec::new(); + for AttributedIntrospectionNode { node, attributes } in list { + content.push_tokens( + quote! { #[cfg(any(#(#preceding),*))] #(#attributes)* ",".as_bytes() }, + ); + let cfgs = attributes + .iter() + .filter_map(|attribute| attribute.meta.require_list().ok()) + .map(|cfg| &cfg.tokens); + preceding.push(quote! { all(#(#cfgs),*) }); // We serialize the element to easily gate it behind the attributes let mut nested_builder = ConcatenationBuilder::default(); - if i > 0 { - nested_builder.push_str(","); - } node.add_to_serialization(&mut nested_builder, pyo3_crate_path); let nested_content = nested_builder.into_token_stream(pyo3_crate_path); content.push_tokens(quote! { #(#attributes)* #nested_content }); diff --git a/pyo3-macros-backend/src/module.rs b/pyo3-macros-backend/src/module.rs index 429bcf60b65..9e3f7951e1e 100644 --- a/pyo3-macros-backend/src/module.rs +++ b/pyo3-macros-backend/src/module.rs @@ -528,7 +528,7 @@ fn module_initialization( let mut result = quote! { #[doc(hidden)] - pub const __PYO3_NAME: &'static ::std::ffi::CStr = #pyo3_name; + pub static __PYO3_NAME: &'static ::std::ffi::CStr = #pyo3_name; // This structure exists for `fn` modules declared within `fn` bodies, where due to the hidden // module (used for importing) the `fn` to initialize the module cannot be seen from the #module_def @@ -544,19 +544,20 @@ fn module_initialization( #pyo3_path::impl_::trampoline::module_exec(module, #module_exec) } - // The full slots, used for the PyModExport initializaiton - static SLOTS: impl_::PyModuleSlots = impl_::PyModuleSlotsBuilder::new() + static DOC: &'static ::std::ffi::CStr = #doc; + static SLOTS: impl_::PrimaryModuleSlots = impl_::PyModuleSlotsBuilder::new() .with_mod_exec(__pyo3_module_exec) .with_abi_info() .with_gil_used(#gil_used) .with_name(__PYO3_NAME) - .with_doc(#doc) + .with_doc(DOC) .build(); + static SECONDARY_SLOTS: impl_::SecondaryModuleSlots = impl_::secondary_slots(&SLOTS); // Since the macros need to be written agnostic to the Python version // we need to explicitly pass the name and docstring for PyModuleDef - // initializaiton. - impl_::ModuleDef::new(__PYO3_NAME, #doc, &SLOTS) + // initialization. + impl_::ModuleDef::new(__PYO3_NAME, DOC, &SLOTS, &SECONDARY_SLOTS) }; }; if !is_submodule { diff --git a/pyo3-macros-backend/src/pyclass.rs b/pyo3-macros-backend/src/pyclass.rs index d0619108f9c..9ad86ecf736 100644 --- a/pyo3-macros-backend/src/pyclass.rs +++ b/pyo3-macros-backend/src/pyclass.rs @@ -1484,8 +1484,7 @@ fn impl_complex_enum_struct_variant_cls( match &*slf.into_super() { #enum_name::#variant_ident { #field_name, .. } => #pyo3_path::impl_::pyclass::ConvertField::< - { #pyo3_path::impl_::pyclass::IsIntoPyObjectRef::<#field_type>::VALUE }, - { #pyo3_path::impl_::pyclass::IsIntoPyObject::<#field_type>::VALUE }, + { #pyo3_path::impl_::pyclass::IsIntoPyObjectRef::<#field_type>::VALUE } >::convert_field::<#field_type>(#field_name, py), _ => ::core::unreachable!("Wrong complex enum variant found in variant wrapper PyClass"), } @@ -1577,8 +1576,7 @@ fn impl_complex_enum_tuple_variant_field_getters( match &*slf.into_super() { #enum_name::#variant_ident ( #(#field_access_tokens), *) => #pyo3_path::impl_::pyclass::ConvertField::< - { #pyo3_path::impl_::pyclass::IsIntoPyObjectRef::<#field_type>::VALUE }, - { #pyo3_path::impl_::pyclass::IsIntoPyObject::<#field_type>::VALUE }, + { #pyo3_path::impl_::pyclass::IsIntoPyObjectRef::<#field_type>::VALUE } >::convert_field::<#field_type>(val, py), _ => ::core::unreachable!("Wrong complex enum variant found in variant wrapper PyClass"), } @@ -1795,8 +1793,13 @@ struct FunctionIntrospectionData<'a> { #[cfg(feature = "experimental-inspect")] impl FunctionIntrospectionData<'_> { - fn generate(self, ctx: &Ctx, cls: &syn::Type) -> TokenStream { - let signature = FunctionSignature::from_arguments(self.arguments); + fn generate(self, ctx: &Ctx, cls: &syn::Type, slot: &SlotDef) -> TokenStream { + let mut signature = FunctionSignature::from_arguments(self.arguments); + if !slot.takes_args_and_kwargs() { + signature + .python_signature + .make_all_parameters_positional_only(); + } let returns = self.returns; self.names .iter() @@ -1835,7 +1838,7 @@ fn generate_protocol_slot( #[cfg_attr(not(feature = "experimental-inspect"), allow(unused_mut))] let mut def = slot.generate_type_slot(cls, &spec, name, ctx)?; #[cfg(feature = "experimental-inspect")] - def.add_introspection(introspection_data.generate(ctx, cls)); + def.add_introspection(introspection_data.generate(ctx, cls, slot)); Ok(def) } @@ -1860,7 +1863,7 @@ fn generate_default_protocol_slot( ctx, )?; #[cfg(feature = "experimental-inspect")] - def.add_introspection(introspection_data.generate(ctx, cls)); + def.add_introspection(introspection_data.generate(ctx, cls, slot)); Ok(def) } @@ -2435,8 +2438,9 @@ fn pyclass_richcmp_simple_enum( #[cfg(feature = "experimental-inspect")] let introspection = FunctionIntrospectionData { names: &["__eq__", "__ne__"], + // `value` is the parameter name CPython uses in the `tp_richcompare` slot wrappers arguments: vec![FnArg::Regular(RegularArg { - name: Cow::Owned(format_ident!("other")), + name: Cow::Owned(format_ident!("value")), ty: &any, from_py_with: None, default_value: None, @@ -2511,8 +2515,9 @@ fn pyclass_richcmp( } else { &["__eq__", "__ne__"] }, + // `value` is the parameter name CPython uses in the `tp_richcompare` slot wrappers arguments: vec![FnArg::Regular(RegularArg { - name: Cow::Owned(format_ident!("other")), + name: Cow::Owned(format_ident!("value")), ty: &parse_quote!(&#cls), from_py_with: None, default_value: None, diff --git a/pyo3-macros-backend/src/pyfunction.rs b/pyo3-macros-backend/src/pyfunction.rs index 3605005323f..cc45dcb1042 100644 --- a/pyo3-macros-backend/src/pyfunction.rs +++ b/pyo3-macros-backend/src/pyfunction.rs @@ -405,7 +405,8 @@ pub fn impl_wrap_pyfunction( let introspection = function_introspection_code( pyo3_path, Some(name), - &name.to_string(), + // `name` is the Rust identifier, which `#[pyo3(name = "...")]` overrides for Python + &spec.python_name.to_string(), &spec.signature, None, func.sig.output.clone(), diff --git a/pyo3-macros-backend/src/pyfunction/signature.rs b/pyo3-macros-backend/src/pyfunction/signature.rs index b80247859b3..f8873eebffd 100644 --- a/pyo3-macros-backend/src/pyfunction/signature.rs +++ b/pyo3-macros-backend/src/pyfunction/signature.rs @@ -296,6 +296,12 @@ impl PythonSignature { .checked_sub(self.default_positional_parameters.len()) .expect("should always have positional defaults <= positional parameters") } + + /// Makes every positional parameter positional-only, exactly as a trailing `/` in a + /// signature does. Deliberately leaves keyword-only parameters alone. + pub fn make_all_parameters_positional_only(&mut self) { + self.positional_only_parameters = self.positional_parameters.len(); + } } #[derive(Clone)] @@ -391,7 +397,7 @@ impl ParseState { ) -> syn::Result<()> { match self { ParseState::Positional => { - signature.positional_only_parameters = signature.positional_parameters.len(); + signature.make_all_parameters_positional_only(); *self = ParseState::PositionalAfterPosargs; Ok(()) } diff --git a/pyo3-macros-backend/src/pyimpl.rs b/pyo3-macros-backend/src/pyimpl.rs index bb5de553634..b0fa7820201 100644 --- a/pyo3-macros-backend/src/pyimpl.rs +++ b/pyo3-macros-backend/src/pyimpl.rs @@ -407,6 +407,10 @@ pub fn method_introspection_code( // We cant to keep the first argument type, hence this hack spec.signature.arguments.pop(); spec.signature.python_signature.positional_parameters.pop(); + // the `CompareOp` parameter is gone; keep the positional-only count in range + spec.signature + .python_signature + .make_all_parameters_positional_only(); method_introspection_code( &spec, attrs, diff --git a/pyo3-macros-backend/src/pymethod.rs b/pyo3-macros-backend/src/pymethod.rs index c529c40f4ff..099a138e6b0 100644 --- a/pyo3-macros-backend/src/pymethod.rs +++ b/pyo3-macros-backend/src/pymethod.rs @@ -189,6 +189,28 @@ enum PyMethodProtoKind { SlotFragment(&'static SlotFragmentDef), } +impl PyMethodProtoKind { + /// Whether Python hands this protocol an `args`/`kwargs` pair. + /// + /// `tp_new`, `tp_init` and `tp_call` do, so they accept keyword arguments and a + /// `#[pyo3(signature = ...)]` has something to act on. Every other protocol is reached + /// through a slot wrapper with a fixed C-level signature: its parameters are positional-only + /// and a signature attribute would have nothing to unpack. + /// + /// Should `signature` ever be accepted on protocol methods just to supply type hints, only + /// the `ensure_no_forbidden_protocol_attributes` use goes away: which parameters Python sees + /// as positional-only is dictated by CPython, not by what the user declares. + fn takes_args_and_kwargs(&self) -> bool { + match self { + PyMethodProtoKind::Slot(slot) => slot.takes_args_and_kwargs(), + PyMethodProtoKind::Call => true, + PyMethodProtoKind::SlotFragment(_) + | PyMethodProtoKind::Traverse + | PyMethodProtoKind::Clear => false, + } + } +} + impl<'a> PyMethod<'a> { pub fn parse( sig: &'a mut syn::Signature, @@ -197,11 +219,22 @@ impl<'a> PyMethod<'a> { ) -> Result { check_generic(sig)?; ensure_function_options_valid(&options)?; - let spec = FnSpec::parse(sig, meth_attrs, options)?; + let mut spec = FnSpec::parse(sig, meth_attrs, options)?; let method_name = spec.python_name.to_string(); let kind = PyMethodKind::from_name(&method_name); + // The parameters of a method backed by a type slot are positional-only, record that in + // the signature. Nothing else can: `#[pyo3(signature = ...)]` is rejected for exactly + // these methods, see `ensure_no_forbidden_protocol_attributes`. + if let PyMethodKind::Proto(proto) = &kind { + if !proto.takes_args_and_kwargs() { + spec.signature + .python_signature + .make_all_parameters_positional_only(); + } + } + Ok(Self { kind, method_name, @@ -339,14 +372,7 @@ fn ensure_no_forbidden_protocol_attributes( ) -> syn::Result<()> { if let Some(signature) = &spec.signature.attribute { // __new__, __init__ and __call__ are allowed to have a signature, but nothing else is. - if !matches!( - proto_kind, - PyMethodProtoKind::Slot(SlotDef { - calling_convention: SlotCallingConvention::TpNew | SlotCallingConvention::TpInit, - .. - }) - ) && !matches!(proto_kind, PyMethodProtoKind::Call) - { + if !proto_kind.takes_args_and_kwargs() { bail_spanned!(signature.kw.span() => format!("`signature` cannot be used with magic method `{}`", method_name)); } } @@ -515,7 +541,7 @@ fn impl_clear_slot(cls: &syn::Type, spec: &FnSpec<'_>, ctx: &Ctx) -> syn::Result pub unsafe extern "C" fn __pymethod___clear____( _slf: *mut #pyo3_path::ffi::PyObject, ) -> ::std::ffi::c_int { - #pyo3_path::impl_::pymethods::_call_clear(_slf, |py, _slf| { + #pyo3_path::impl_::pymethods::_call_clear::<#cls>(_slf, |py, _slf| { #holders let result = #fncall; let result = #pyo3_path::impl_::wrap::converter(&result).wrap(result)?; @@ -872,7 +898,6 @@ pub fn impl_py_getter_def( { ::std::mem::offset_of!(#cls, #field) }, { #pyo3_path::impl_::pyclass::IsPyT::<#ty>::VALUE }, { #pyo3_path::impl_::pyclass::IsIntoPyObjectRef::<#ty>::VALUE }, - { #pyo3_path::impl_::pyclass::IsIntoPyObject::<#ty>::VALUE }, > = unsafe { #pyo3_path::impl_::pyclass::PyClassGetterGenerator::new() }; #generator } @@ -1084,8 +1109,8 @@ pub const __LEN__: SlotDef = SlotDef::new("Py_mp_length", "lenfunc"); const __CONTAINS__: SlotDef = SlotDef::new("Py_sq_contains", "objobjproc"); const __CONCAT__: SlotDef = SlotDef::new("Py_sq_concat", "binaryfunc"); const __REPEAT__: SlotDef = SlotDef::new("Py_sq_repeat", "ssizeargfunc"); -const __INPLACE_CONCAT__: SlotDef = SlotDef::new("Py_sq_concat", "binaryfunc"); -const __INPLACE_REPEAT__: SlotDef = SlotDef::new("Py_sq_repeat", "ssizeargfunc"); +const __INPLACE_CONCAT__: SlotDef = SlotDef::new("Py_sq_inplace_concat", "binaryfunc"); +const __INPLACE_REPEAT__: SlotDef = SlotDef::new("Py_sq_inplace_repeat", "ssizeargfunc"); pub const __GETITEM__: SlotDef = SlotDef::new("Py_mp_subscript", "binaryfunc"); const __POS__: SlotDef = SlotDef::new("Py_nb_positive", "unaryfunc"); @@ -1322,6 +1347,19 @@ enum SlotCallingConvention { } impl SlotDef { + /// Whether this slot receives Python's `args`/`kwargs` pair rather than a fixed set of + /// positional arguments. + /// + /// Only `tp_new` and `tp_init` do; every other slot is exposed to Python through a slot + /// wrapper which rejects keyword arguments, e.g. `__eq__` has the signature + /// `($self, value, /)`. + pub const fn takes_args_and_kwargs(&self) -> bool { + matches!( + self.calling_convention, + SlotCallingConvention::TpNew | SlotCallingConvention::TpInit + ) + } + const fn new(slot: &'static str, func_ty: &'static str) -> Self { // The FFI function pointer type determines the arguments and return type let (calling_convention, ret_ty) = match func_ty.as_bytes() { diff --git a/pyo3-macros/Cargo.toml b/pyo3-macros/Cargo.toml index 6c9da9b1fc7..1aef98e6717 100644 --- a/pyo3-macros/Cargo.toml +++ b/pyo3-macros/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "pyo3-macros" -version = "0.29.0" +version = "0.29.1" description = "Proc macros for PyO3 package" authors = ["PyO3 Project and Contributors "] keywords = ["pyo3", "python", "cpython", "ffi"] @@ -23,7 +23,7 @@ experimental-inspect = ["pyo3-macros-backend/experimental-inspect"] proc-macro2 = { version = "1.0.60", default-features = false } quote = "1" syn = { version = "2", features = ["full", "extra-traits"] } -pyo3-macros-backend = { path = "../pyo3-macros-backend", version = "=0.29.0" } +pyo3-macros-backend = { path = "../pyo3-macros-backend", version = "=0.29.1" } [lints] workspace = true diff --git a/pyproject.toml b/pyproject.toml index 4e89ec87e46..09ec325199d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -14,7 +14,7 @@ dynamic = ["version"] [tool.towncrier] filename = "CHANGELOG.md" -version = "0.29.0" +version = "0.29.1" start_string = "\n" template = ".towncrier.template.md" title_format = "## [{version}] - {project_date}" @@ -71,5 +71,7 @@ package = false [dependency-groups] dev = [ + "ruff<0.16", "rumdl", + "typos", ] diff --git a/pytests/pyproject.toml b/pytests/pyproject.toml index 94c23f2fb3b..f36f6d94376 100644 --- a/pytests/pyproject.toml +++ b/pytests/pyproject.toml @@ -21,7 +21,8 @@ classifiers = [ [project.optional-dependencies] dev = [ "hypothesis>=3.55", - "mypy~=1.0", + # mypy doesn't build on GraalPy when installed via uv + "mypy~=1.0; platform_python_implementation != 'GraalVM'", "pyrefly~=0.57.0", "pytest-asyncio>=0.21,<2", "pytest-benchmark>=3.4", diff --git a/pytests/src/pyfunctions.rs b/pytests/src/pyfunctions.rs index e1ffb444cac..e0c8d882514 100644 --- a/pytests/src/pyfunctions.rs +++ b/pytests/src/pyfunctions.rs @@ -4,6 +4,12 @@ use pyo3::types::{PyDict, PyTuple}; #[pyfunction(signature = ())] fn none() {} +// Exposed under a different name than the Rust one, which the generated stubs have to use. +#[pyfunction(name = "renamed")] +fn rust_name_of_renamed() -> usize { + 42 +} + type Any<'py> = Bound<'py, PyAny>; type Dict<'py> = Bound<'py, PyDict>; type Tuple<'py> = Bound<'py, PyTuple>; @@ -129,12 +135,24 @@ fn many_keyword_arguments<'py>( #[pymodule] pub mod pyfunctions { + // `any()` is never true. Keeps the introspection data of a `#[pymodule]` whose first member + // is `cfg`-ed out covered by `nox -s test-introspection`; it used to serialize the member + // list as the invalid JSON `[,"..."]`. + #[cfg(any())] + #[pymodule_export] + use super::none; + #[cfg(feature = "experimental-async")] #[pymodule_export] use super::with_async; #[pymodule_export] use super::{ - args_kwargs, many_keyword_arguments, none, positional_only, simple, simple_args, - simple_args_kwargs, simple_kwargs, with_typed_args, + args_kwargs, many_keyword_arguments, none, positional_only, rust_name_of_renamed, simple, + simple_args, simple_args_kwargs, simple_kwargs, with_typed_args, }; + + // Likewise for a `cfg`-ed out last member. + #[cfg(any())] + #[pymodule_export] + use super::simple; } diff --git a/pytests/stubs/comparisons.pyi b/pytests/stubs/comparisons.pyi index 0cbfe04f94b..6f044552dca 100644 --- a/pytests/stubs/comparisons.pyi +++ b/pytests/stubs/comparisons.pyi @@ -2,58 +2,58 @@ from typing import final @final class Eq: - def __eq__(self, /, other: object) -> bool: ... - def __ne__(self, /, other: object) -> bool: ... + def __eq__(self, other: object, /) -> bool: ... + def __ne__(self, other: object, /) -> bool: ... def __new__(cls, /, value: int) -> Eq: ... @final class EqDefaultNe: - def __eq__(self, /, other: object) -> bool: ... + def __eq__(self, other: object, /) -> bool: ... def __new__(cls, /, value: int) -> EqDefaultNe: ... @final class EqDerived: - def __eq__(self, /, other: object) -> bool: ... - def __ne__(self, /, other: object) -> bool: ... + def __eq__(self, value: object, /) -> bool: ... + def __ne__(self, value: object, /) -> bool: ... def __new__(cls, /, value: int) -> EqDerived: ... @final class Ordered: - def __eq__(self, /, other: object) -> bool: ... - def __ge__(self, /, other: object) -> bool: ... - def __gt__(self, /, other: object) -> bool: ... - def __le__(self, /, other: object) -> bool: ... - def __lt__(self, /, other: object) -> bool: ... - def __ne__(self, /, other: object) -> bool: ... + def __eq__(self, other: object, /) -> bool: ... + def __ge__(self, other: object, /) -> bool: ... + def __gt__(self, other: object, /) -> bool: ... + def __le__(self, other: object, /) -> bool: ... + def __lt__(self, other: object, /) -> bool: ... + def __ne__(self, other: object, /) -> bool: ... def __new__(cls, /, value: int) -> Ordered: ... @final class OrderedDefaultNe: - def __eq__(self, /, other: object) -> bool: ... - def __ge__(self, /, other: object) -> bool: ... - def __gt__(self, /, other: object) -> bool: ... - def __le__(self, /, other: object) -> bool: ... - def __lt__(self, /, other: object) -> bool: ... + def __eq__(self, other: object, /) -> bool: ... + def __ge__(self, other: object, /) -> bool: ... + def __gt__(self, other: object, /) -> bool: ... + def __le__(self, other: object, /) -> bool: ... + def __lt__(self, other: object, /) -> bool: ... def __new__(cls, /, value: int) -> OrderedDefaultNe: ... @final class OrderedDerived: - def __eq__(self, /, other: object) -> bool: ... - def __ge__(self, /, other: object) -> bool: ... - def __gt__(self, /, other: object) -> bool: ... + def __eq__(self, value: object, /) -> bool: ... + def __ge__(self, value: object, /) -> bool: ... + def __gt__(self, value: object, /) -> bool: ... def __hash__(self, /) -> int: ... - def __le__(self, /, other: object) -> bool: ... - def __lt__(self, /, other: object) -> bool: ... - def __ne__(self, /, other: object) -> bool: ... + def __le__(self, value: object, /) -> bool: ... + def __lt__(self, value: object, /) -> bool: ... + def __ne__(self, value: object, /) -> bool: ... def __new__(cls, /, value: int) -> OrderedDerived: ... def __str__(self, /) -> str: ... @final class OrderedRichCmp: - def __eq__(self, /, other: object) -> bool: ... - def __ge__(self, /, other: object) -> bool: ... - def __gt__(self, /, other: object) -> bool: ... - def __le__(self, /, other: object) -> bool: ... - def __lt__(self, /, other: object) -> bool: ... - def __ne__(self, /, other: object) -> bool: ... + def __eq__(self, other: object, /) -> bool: ... + def __ge__(self, other: object, /) -> bool: ... + def __gt__(self, other: object, /) -> bool: ... + def __le__(self, other: object, /) -> bool: ... + def __lt__(self, other: object, /) -> bool: ... + def __ne__(self, other: object, /) -> bool: ... def __new__(cls, /, value: int) -> OrderedRichCmp: ... diff --git a/pytests/stubs/enums.pyi b/pytests/stubs/enums.pyi index 59d3d281016..cc0794f5ee2 100644 --- a/pytests/stubs/enums.pyi +++ b/pytests/stubs/enums.pyi @@ -62,7 +62,7 @@ class MixedComplexEnum: @final class Empty(MixedComplexEnum): __match_args__: Final = () - def __getitem__(self, /, key: int) -> Any: ... + def __getitem__(self, key: int, /) -> Any: ... def __len__(self, /) -> int: ... def __new__(cls, /) -> MixedComplexEnum.Empty: ... @@ -83,9 +83,9 @@ class SimpleEnum: Thursday: Final[SimpleEnum] Tuesday: Final[SimpleEnum] Wednesday: Final[SimpleEnum] - def __eq__(self, /, other: object) -> bool: ... + def __eq__(self, value: object, /) -> bool: ... def __int__(self, /) -> int: ... - def __ne__(self, /, other: object) -> bool: ... + def __ne__(self, value: object, /) -> bool: ... def __repr__(self, /) -> str: ... @final @@ -101,7 +101,7 @@ class SimpleTupleEnum: __match_args__: Final = ("_0",) @property def _0(self, /) -> int: ... - def __getitem__(self, /, key: int) -> Any: ... + def __getitem__(self, key: int, /) -> Any: ... def __len__(self, /) -> int: ... def __new__(cls, /, _0: int) -> SimpleTupleEnum.Int: ... @@ -110,7 +110,7 @@ class SimpleTupleEnum: __match_args__: Final = ("_0",) @property def _0(self, /) -> str: ... - def __getitem__(self, /, key: int) -> Any: ... + def __getitem__(self, key: int, /) -> Any: ... def __len__(self, /) -> int: ... def __new__(cls, /, _0: str) -> SimpleTupleEnum.Str: ... @@ -118,7 +118,7 @@ class TupleEnum: @final class EmptyTuple(TupleEnum): __match_args__: Final = () - def __getitem__(self, /, key: int) -> Any: ... + def __getitem__(self, key: int, /) -> Any: ... def __len__(self, /) -> int: ... def __new__(cls, /) -> TupleEnum.EmptyTuple: ... @@ -131,7 +131,7 @@ class TupleEnum: def _1(self, /) -> float: ... @property def _2(self, /) -> bool: ... - def __getitem__(self, /, key: int) -> Any: ... + def __getitem__(self, key: int, /) -> Any: ... def __len__(self, /) -> int: ... def __new__(cls, /, _0: int, _1: float, _2: bool) -> TupleEnum.Full: ... @@ -148,7 +148,7 @@ class TupleEnum: def _1(self, /) -> float: ... @property def _2(self, /) -> bool: ... - def __getitem__(self, /, key: int) -> Any: ... + def __getitem__(self, key: int, /) -> Any: ... def __len__(self, /) -> int: ... def __new__( cls, /, _0: int = 1, _1: float = 1.0, _2: bool = True diff --git a/pytests/stubs/pyclasses.pyi b/pytests/stubs/pyclasses.pyi index 9dd201c2d03..64692e0dc9c 100644 --- a/pytests/stubs/pyclasses.pyi +++ b/pytests/stubs/pyclasses.pyi @@ -56,36 +56,36 @@ class EmptyClass: @final class Number: def __abs__(self, /) -> Number: ... - def __add__(self, /, other: object) -> Number: ... - def __and__(self, /, other: object) -> Number: ... + def __add__(self, other: object, /) -> Number: ... + def __and__(self, other: object, /) -> Number: ... def __complex__(self, /) -> complex: ... - def __divmod__(self, /, other: object) -> tuple[Number, Number]: ... - def __eq__(self, /, other: object) -> bool: ... + def __divmod__(self, other: object, /) -> tuple[Number, Number]: ... + def __eq__(self, other: object, /) -> bool: ... def __float__(self, /) -> float: ... - def __floordiv__(self, /, other: object) -> Number: ... - def __ge__(self, /, other: object) -> bool: ... - def __gt__(self, /, other: object) -> bool: ... + def __floordiv__(self, other: object, /) -> Number: ... + def __ge__(self, other: object, /) -> bool: ... + def __gt__(self, other: object, /) -> bool: ... def __hash__(self, /) -> int: ... def __int__(self, /) -> int: ... def __invert__(self, /) -> Number: ... - def __le__(self, /, other: object) -> bool: ... - def __lshift__(self, /, other: object) -> Number: ... - def __lt__(self, /, other: object) -> bool: ... - def __matmul__(self, /, other: object) -> Number: ... - def __mod__(self, /, other: object) -> Number: ... - def __mul__(self, /, other: object) -> Number: ... - def __ne__(self, /, other: object) -> bool: ... + def __le__(self, other: object, /) -> bool: ... + def __lshift__(self, other: object, /) -> Number: ... + def __lt__(self, other: object, /) -> bool: ... + def __matmul__(self, other: object, /) -> Number: ... + def __mod__(self, other: object, /) -> Number: ... + def __mul__(self, other: object, /) -> Number: ... + def __ne__(self, other: object, /) -> bool: ... def __neg__(self, /) -> Number: ... def __new__(cls, /, value: int) -> Number: ... - def __or__(self, /, other: object) -> Number: ... + def __or__(self, other: object, /) -> Number: ... def __pos__(self, /) -> Number: ... - def __pow__(self, /, other: object, modulo: object) -> Number: ... + def __pow__(self, other: object, modulo: object, /) -> Number: ... def __repr__(self, /) -> str: ... - def __rshift__(self, /, other: object) -> Number: ... + def __rshift__(self, other: object, /) -> Number: ... def __str__(self, /) -> str: ... - def __sub__(self, /, other: object) -> Number: ... - def __truediv__(self, /, other: object) -> Number: ... - def __xor__(self, /, other: object) -> Number: ... + def __sub__(self, other: object, /) -> Number: ... + def __truediv__(self, other: object, /) -> Number: ... + def __xor__(self, other: object, /) -> Number: ... @final class PlainObject: diff --git a/pytests/stubs/pyfunctions.pyi b/pytests/stubs/pyfunctions.pyi index 322f2642339..3428d60375a 100644 --- a/pytests/stubs/pyfunctions.pyi +++ b/pytests/stubs/pyfunctions.pyi @@ -22,6 +22,7 @@ def many_keyword_arguments( ) -> None: ... def none() -> None: ... def positional_only(a: Any, /, b: Any) -> tuple[Any, Any]: ... +def renamed() -> int: ... def simple( a: Any, b: Any | None = None, *, c: Any | None = None ) -> tuple[Any, Any | None, Any | None]: ... diff --git a/pytests/tests/test_pyfunctions.py b/pytests/tests/test_pyfunctions.py index 8595da76142..347934d6822 100644 --- a/pytests/tests/test_pyfunctions.py +++ b/pytests/tests/test_pyfunctions.py @@ -167,3 +167,8 @@ def test_many_keyword_arguments_rs(benchmark): ) py = call_with_many_keyword_arguments(many_keyword_arguments_py) assert rust == py + + +def test_renamed(): + assert pyfunctions.renamed() == 42 + assert not hasattr(pyfunctions, "rust_name_of_renamed") diff --git a/src/conversions/std/osstr.rs b/src/conversions/std/osstr.rs index 113673a6ec9..714ac1e83d3 100644 --- a/src/conversions/std/osstr.rs +++ b/src/conversions/std/osstr.rs @@ -1,6 +1,3 @@ -// TODO https://github.com/PyO3/pyo3/issues/5487 -#![allow(clippy::undocumented_unsafe_blocks)] - use crate::conversion::IntoPyObject; #[cfg(not(target_os = "wasi"))] use crate::ffi; @@ -50,6 +47,7 @@ impl<'py> IntoPyObject<'py> for &OsStr { let bytes = self.as_bytes(); let ptr = bytes.as_ptr().cast(); let len = bytes.len() as ffi::Py_ssize_t; + // SAFETY: passing valid pointer to python API unsafe { // DecodeFSDefault automatically chooses an appropriate decoding mechanism to // parse os strings losslessly (i.e. surrogateescape most of the time) @@ -62,6 +60,7 @@ impl<'py> IntoPyObject<'py> for &OsStr { #[cfg(windows)] { let wstr: Vec = self.encode_wide().collect(); + // SAFETY: passing valid pointer to python API unsafe { // This will not panic because the data from encode_wide is well-formed Windows // string data @@ -130,6 +129,7 @@ impl FromPyObject<'_, '_> for OsString { // Get an owned allocated wide char buffer from PyString, which we have to deallocate // ourselves + // SAFETY: passing valid pointer to python API let size = unsafe { ffi::PyUnicode_AsWideChar(pystring.as_ptr(), core::ptr::null_mut(), 0) }; crate::err::error_on_minusone(ob.py(), size)?; @@ -141,6 +141,7 @@ impl FromPyObject<'_, '_> for OsString { let size = size - 1; // exclude null terminator let mut buffer = vec![0; size as usize]; + // SAFETY: passing valid pointer to python API let bytes_read = unsafe { ffi::PyUnicode_AsWideChar(pystring.as_ptr(), buffer.as_mut_ptr(), size) }; assert_eq!(bytes_read, size); diff --git a/src/conversions/time.rs b/src/conversions/time.rs index 0e9a9b10733..4996ffc51c6 100644 --- a/src/conversions/time.rs +++ b/src/conversions/time.rs @@ -50,7 +50,7 @@ //! } //! ``` -use crate::exceptions::{PyTypeError, PyValueError}; +use crate::exceptions::{PyOverflowError, PyTypeError, PyValueError}; #[cfg(feature = "experimental-inspect")] use crate::inspect::PyStaticExpr; #[cfg(Py_LIMITED_API)] @@ -178,11 +178,13 @@ impl<'py> IntoPyObject<'py> for Duration { total_seconds % SECONDS_PER_DAY, ) }; - // Create the timedelta with days, seconds, microseconds - // Safe to unwrap as we've verified the values are within bounds + let days = days + .try_into() + .map_err(|_| PyOverflowError::new_err("duration out of range for Python timedelta"))?; + PyDelta::new( py, - days.try_into().expect("days overflow"), + days, seconds.try_into().expect("seconds overflow"), micro_seconds, true, @@ -861,6 +863,11 @@ mod tests { assert!(result.is_err()); let err_type = result.unwrap_err().get_type(py).name().unwrap(); assert_eq!(err_type, "OverflowError"); + + for duration in [Duration::MIN, Duration::MAX] { + let err = duration.into_pyobject(py).unwrap_err(); + assert_eq!(err.get_type(py).name().unwrap(), "OverflowError"); + } }); } diff --git a/src/exceptions.rs b/src/exceptions.rs index 94d6c0e2c72..2cf91845789 100644 --- a/src/exceptions.rs +++ b/src/exceptions.rs @@ -1,6 +1,3 @@ -// TODO https://github.com/PyO3/pyo3/issues/5487 -#![allow(clippy::undocumented_unsafe_blocks)] - //! Exception and warning types defined by Python. //! //! The structs in this module represent Python's built-in exceptions and @@ -280,7 +277,11 @@ macro_rules! impl_native_exception ( pub struct $name($crate::PyAny); $crate::impl_exception_boilerplate!($name); - $crate::pyobject_native_type!($name, $layout, |_py| unsafe { $crate::ffi::$exc_name as *mut $crate::ffi::PyTypeObject }, "builtins", $python_name $(, #checkfunction=$checkfunction)?); + $crate::pyobject_native_type!($name, $layout, |_py| { + // SAFETY: cpython docs state that all exception types are available as global variables and are class objects + // https://docs.python.org/3/c-api/exceptions.html#exception-and-warning-types + unsafe { $crate::ffi::$exc_name as *mut $crate::ffi::PyTypeObject } + }, "builtins", $python_name $(, #checkfunction=$checkfunction)?); $crate::pyobject_subclassable_native_type!($name, $layout); ); ($name:ident, $exc_name:ident, $python_name:expr, $doc:expr) => ( @@ -730,6 +731,7 @@ impl PyUnicodeDecodeError { ) -> PyResult> { use crate::ffi_ptr_ext::FfiPtrExt; use crate::py_result_ext::PyResultExt; + // SAFETY: calling python API with correct pointers unsafe { ffi::PyUnicodeDecodeError_Create( encoding.as_ptr(), diff --git a/src/ffi_ptr_ext.rs b/src/ffi_ptr_ext.rs index 844d40753dd..e877bce524f 100644 --- a/src/ffi_ptr_ext.rs +++ b/src/ffi_ptr_ext.rs @@ -1,6 +1,3 @@ -// TODO https://github.com/PyO3/pyo3/issues/5487 -#![allow(clippy::undocumented_unsafe_blocks)] - use crate::sealed::Sealed; use crate::{ ffi, @@ -41,48 +38,80 @@ pub(crate) trait FfiPtrExt: Sealed { } impl FfiPtrExt for *mut ffi::PyObject { + /// # Safety + /// + /// see requirements for [`Bound::from_owned_ptr_or_err`] #[inline] unsafe fn assume_owned_or_err(self, py: Python<'_>) -> PyResult> { + // SAFETY: caller upholds requirements unsafe { Bound::from_owned_ptr_or_err(py, self) } } + /// # Safety + /// + /// see requirements for [`Bound::from_owned_ptr_or_opt`] #[inline] unsafe fn assume_owned_or_opt(self, py: Python<'_>) -> Option> { + // SAFETY: caller upholds requirements unsafe { Bound::from_owned_ptr_or_opt(py, self) } } + /// # Safety + /// + /// see requirements for [`Bound::from_owned_ptr`] #[inline] #[track_caller] unsafe fn assume_owned(self, py: Python<'_>) -> Bound<'_, PyAny> { + // SAFETY: caller upholds requirements unsafe { Bound::from_owned_ptr(py, self) } } + /// # Safety + /// + /// see requirements for [`Bound::from_owned_ptr_unchecked`] #[inline] unsafe fn assume_owned_unchecked(self, py: Python<'_>) -> Bound<'_, PyAny> { + // SAFETY: caller upholds requirements unsafe { Bound::from_owned_ptr_unchecked(py, self) } } + /// # Safety + /// + /// see requirements for [`Borrowed::from_ptr_or_err`] #[inline] unsafe fn assume_borrowed_or_err<'a>( self, py: Python<'_>, ) -> PyResult> { + // SAFETY: caller upholds requirements unsafe { Borrowed::from_ptr_or_err(py, self) } } + /// # Safety + /// + /// see requirements for [`Borrowed::from_ptr_or_opt`] #[inline] unsafe fn assume_borrowed_or_opt<'a>(self, py: Python<'_>) -> Option> { + // SAFETY: caller upholds requirements unsafe { Borrowed::from_ptr_or_opt(py, self) } } + /// # Safety + /// + /// see requirements for [`Borrowed::from_ptr`] #[inline] #[track_caller] unsafe fn assume_borrowed<'a>(self, py: Python<'_>) -> Borrowed<'a, '_, PyAny> { + // SAFETY: caller upholds requirements unsafe { Borrowed::from_ptr(py, self) } } + /// # Safety + /// + /// see requirements for [`Borrowed::from_ptr_unchecked`] #[inline] unsafe fn assume_borrowed_unchecked<'a>(self, py: Python<'_>) -> Borrowed<'a, '_, PyAny> { + // SAFETY: caller upholds requirements unsafe { Borrowed::from_ptr_unchecked(py, self) } } } diff --git a/src/impl_/introspection.rs b/src/impl_/introspection.rs index 8d295ad6996..90618902bd3 100644 --- a/src/impl_/introspection.rs +++ b/src/impl_/introspection.rs @@ -95,11 +95,11 @@ pub const fn escape_json_string(input: &str, output: &mut [u8]) -> usize { output_i += 1; output[output_i] = b'0' + (c / 16); output_i += 1; - let remainer = c % 16; - output[output_i] = if remainer >= 10 { - b'a' + remainer - 10 + let remainder = c % 16; + output[output_i] = if remainder >= 10 { + b'a' + remainder - 10 } else { - b'0' + remainer + b'0' + remainder }; output_i += 1; } diff --git a/src/impl_/pyclass.rs b/src/impl_/pyclass.rs index bc539a5d7d0..83584cd9724 100644 --- a/src/impl_/pyclass.rs +++ b/src/impl_/pyclass.rs @@ -62,7 +62,19 @@ pub trait PyClassDict: sealed::Sealed { const INIT: Self; /// Empties the dictionary of its key-value pairs. #[inline] - fn clear_dict(&mut self, _py: Python<'_>) {} + fn clear_dict(&self, _py: Python<'_>) {} + /// Releases the owned reference to the dictionary. + #[inline] + fn release_dict(&mut self, _py: Python<'_>) {} + /// Visits the `__dict__`, if any, on behalf of `tp_traverse`. + /// + /// # Safety + /// - Must only be called from a `tp_traverse` implementation, passing that + /// implementation's `visit` and `arg` unchanged. + #[inline] + unsafe fn traverse_dict(&self, _visit: ffi::visitproc, _arg: *mut c_void) -> c_int { + 0 + } } /// Represents the `__weakref__` field for `#[pyclass]`. @@ -98,11 +110,23 @@ pub struct PyClassDictSlot(*mut ffi::PyObject); impl PyClassDict for PyClassDictSlot { const INIT: Self = Self(core::ptr::null_mut()); #[inline] - fn clear_dict(&mut self, _py: Python<'_>) { + fn clear_dict(&self, _py: Python<'_>) { if !self.0.is_null() { unsafe { ffi::PyDict_Clear(self.0) } } } + #[inline] + fn release_dict(&mut self, _py: Python<'_>) { + unsafe { ffi::Py_CLEAR(&raw mut self.0) } + } + #[inline] + unsafe fn traverse_dict(&self, visit: ffi::visitproc, arg: *mut c_void) -> c_int { + if self.0.is_null() { + 0 + } else { + unsafe { visit(self.0, arg) } + } + } } /// Actual weakref field, which holds the pointer to `__weakref__`. @@ -986,10 +1010,6 @@ pub unsafe extern "C" fn free_with_freelist(obj: *mut c_ ffi::PyObject_Free }; free(obj.as_ptr().cast()); - - if ffi::PyType_HasFeature(ty, ffi::Py_TPFLAGS_HEAPTYPE) != 0 { - ffi::Py_DECREF(ty as *mut ffi::PyObject); - } } } } @@ -1216,7 +1236,6 @@ pub struct PyClassGetterGenerator< // at compile time const IS_PY_T: bool, const IMPLEMENTS_INTOPYOBJECT_REF: bool, - const IMPLEMENTS_INTOPYOBJECT: bool, >(PhantomData<(ClassT, FieldT)>); impl< @@ -1225,16 +1244,7 @@ impl< const OFFSET: usize, const IS_PY_T: bool, const IMPLEMENTS_INTOPYOBJECT_REF: bool, - const IMPLEMENTS_INTOPYOBJECT: bool, - > - PyClassGetterGenerator< - ClassT, - FieldT, - OFFSET, - IS_PY_T, - IMPLEMENTS_INTOPYOBJECT_REF, - IMPLEMENTS_INTOPYOBJECT, - > + > PyClassGetterGenerator { /// Safety: constructing this type requires that there exists a value of type FieldT /// at the calculated offset within the type ClassT. @@ -1248,16 +1258,7 @@ impl< U: PyTypeCheck, const OFFSET: usize, const IMPLEMENTS_INTOPYOBJECT_REF: bool, - const IMPLEMENTS_INTOPYOBJECT: bool, - > - PyClassGetterGenerator< - ClassT, - Py, - OFFSET, - true, - IMPLEMENTS_INTOPYOBJECT_REF, - IMPLEMENTS_INTOPYOBJECT, - > + > PyClassGetterGenerator, OFFSET, true, IMPLEMENTS_INTOPYOBJECT_REF> { /// `Py` fields have a potential optimization to use Python's "struct members" to read /// the field directly from the struct, rather than using a getter function. @@ -1302,8 +1303,8 @@ impl< /// Field is not `Py`; try to use `IntoPyObject` for `&T` (preferred over `ToPyObject`) to avoid /// potentially expensive clones of containers like `Vec` -impl - PyClassGetterGenerator +impl + PyClassGetterGenerator where ClassT: PyClass, for<'a, 'py> &'a FieldT: IntoPyObject<'py>, @@ -1330,8 +1331,8 @@ pub trait PyO3GetField<'py>: IntoPyObject<'py> + Clone + pyo3_get_field::Sealed impl<'py, T> PyO3GetField<'py> for T where T: IntoPyObject<'py> + Clone {} /// Base case attempts to use IntoPyObject + Clone -impl - PyClassGetterGenerator +impl + PyClassGetterGenerator { pub const fn generate(&self, name: &'static CStr, doc: Option<&'static CStr>) -> PyMethodDefType // The bound goes here rather than on the block so that this impl is always available @@ -1445,12 +1446,9 @@ where unsafe { inner::(py, NonNull::from(class_obj.contents()).cast(), OFFSET) } } -pub struct ConvertField< - const IMPLEMENTS_INTOPYOBJECT_REF: bool, - const IMPLEMENTS_INTOPYOBJECT: bool, ->; +pub struct ConvertField; -impl ConvertField { +impl ConvertField { #[inline] pub fn convert_field<'a, 'py, T>(obj: &'a T, py: Python<'py>) -> PyResult> where @@ -1460,7 +1458,7 @@ impl ConvertField ConvertField { +impl ConvertField { #[inline] pub fn convert_field<'py, T>(obj: &T, py: Python<'py>) -> PyResult> where @@ -1566,9 +1564,8 @@ mod tests { // generate for a non-py field using IntoPyObject for &i32 // SAFETY: offset is correct - let generator = unsafe { - PyClassGetterGenerator::::new() - }; + let generator = + unsafe { PyClassGetterGenerator::::new() }; let PyMethodDefType::Getter(def) = generator.generate(c"my_field", Some(c"My field doc")) else { panic!("Expected a Getter"); @@ -1589,9 +1586,8 @@ mod tests { // generate for a field via `IntoPyObject` + `Clone` // SAFETY: offset is correct - let generator = unsafe { - PyClassGetterGenerator::::new() - }; + let generator = + unsafe { PyClassGetterGenerator::::new() }; let PyMethodDefType::Getter(def) = generator.generate(c"my_field", Some(c"My field doc")) else { panic!("Expected a Getter"); @@ -1620,7 +1616,7 @@ mod tests { const FIELD_OFFSET: usize = offset_of!(MyClass, my_field); // SAFETY: offset is correct let generator = unsafe { - PyClassGetterGenerator::, FIELD_OFFSET, true, true, true>::new() + PyClassGetterGenerator::, FIELD_OFFSET, true, true>::new() }; let PyMethodDefType::StructMember(def) = generator.generate(c"my_field", Some(c"My field doc")) @@ -1655,7 +1651,7 @@ mod tests { const FIELD_OFFSET: usize = offset_of!(MyClass, my_field); // SAFETY: offset is correct let generator = unsafe { - PyClassGetterGenerator::, FIELD_OFFSET, true, true, true>::new() + PyClassGetterGenerator::, FIELD_OFFSET, true, true>::new() }; let PyMethodDefType::Getter(def) = generator.generate(c"my_field", Some(c"My field doc")) else { diff --git a/src/impl_/pyclass/probes.rs b/src/impl_/pyclass/probes.rs index 6497ccd55c0..bc74178c8fd 100644 --- a/src/impl_/pyclass/probes.rs +++ b/src/impl_/pyclass/probes.rs @@ -44,15 +44,6 @@ where pub const VALUE: bool = true; } -probe!(IsIntoPyObject); - -impl<'py, T> IsIntoPyObject -where - T: IntoPyObject<'py>, -{ - pub const VALUE: bool = true; -} - probe!(IsSend); impl IsSend { diff --git a/src/impl_/pymethods.rs b/src/impl_/pymethods.rs index 9493fb981e0..917f4863b2a 100644 --- a/src/impl_/pymethods.rs +++ b/src/impl_/pymethods.rs @@ -5,6 +5,7 @@ use crate::exceptions::PyStopAsyncIteration; use crate::impl_::callback::IntoPyCallbackOutput; use crate::impl_::panic::PanicTrap; use crate::impl_::pycell::PyClassObjectBaseLayout; +use crate::impl_::pyclass::PyClassDict as _; use crate::internal::get_slot::{get_slot, TP_BASE, TP_CLEAR, TP_TRAVERSE}; use crate::internal::pyclass_init::PyClassInit; use crate::internal::state::ForbidAttaching; @@ -362,9 +363,38 @@ where // token to the user code and forbid safe methods for attaching. // (This includes enforcing the `&self` method receiver as e.g. `PyRef` could // reconstruct a Python token via `PyRef::py`.) + // + // The traversal lives in `traverse_impl` so that it can return early: `trap` is armed until + // `disarm` below, and dropping it armed panics, which aborts out of `tp_traverse`. let trap = PanicTrap::new("uncaught panic inside __traverse__ handler"); let lock = ForbidAttaching::during_traverse(); + let retval = unsafe { traverse_impl(slf, impl_, visit, arg, current_traverse) }; + + // Drop lock before trap just in case dropping lock panics + drop(lock); + trap.disarm(); + retval +} + +/// Visits the base type, the instance `__dict__` and the pyclass's own data, stopping as soon as +/// one of them returns non-zero. +/// +/// # Safety +/// - `slf` must be a valid pointer to an instance of `T`. +/// - Must only be called from `_call_traverse`, which holds the `PanicTrap` and `ForbidAttaching` +/// lock this relies on. +unsafe fn traverse_impl( + slf: *mut ffi::PyObject, + impl_: fn(&T, PyVisit<'_>) -> Result<(), PyTraverseError>, + visit: ffi::visitproc, + arg: *mut c_void, + current_traverse: ffi::traverseproc, +) -> c_int +where + T: PyClass, +{ + // A non-zero return means a `visitproc` has asked us to stop the traversal. let super_retval = unsafe { call_super_traverse(slf, visit, arg, current_traverse) }; if super_retval != 0 { return super_retval; @@ -374,39 +404,45 @@ where // traversal is running so no mutations can occur. let class_object: &::Layout = unsafe { &*slf.cast() }; - let retval = - // `#[pyclass(unsendable)]` types can only be deallocated by their own thread, so - // do not traverse them if not on their owning thread :( - if class_object.check_threadsafe().is_ok() - // ... and we cannot traverse a type which might be being mutated by a Rust thread - && class_object.borrow_checker().try_borrow().is_ok() { - struct TraverseGuard<'a, T: PyClassImpl>(&'a T::Layout); - impl Drop for TraverseGuard<'_, T> { - fn drop(&mut self) { - self.0.borrow_checker().release_borrow() - } - } - - // `.try_borrow()` above created a borrow, we need to release it when we're done - // traversing the object. This allows us to read `instance` safely. - let _guard = TraverseGuard::(class_object); - let instance = unsafe {&*class_object.contents().value.get()}; + // The `__dict__` is not Rust data, so it is visited without the thread and borrow checks + // below: it must stay reachable to the GC even when the pyclass data cannot be traversed. + let dict_retval = unsafe { class_object.contents().dict.traverse_dict(visit, arg) }; + if dict_retval != 0 { + return dict_retval; + } - let visit = PyVisit { visit, arg, _guard: PhantomData }; + // `#[pyclass(unsendable)]` types can only be deallocated by their own thread, so do not + // traverse them if not on their owning thread :( + // ... and we cannot traverse a type which might be being mutated by a Rust thread. + if class_object.check_threadsafe().is_err() + || class_object.borrow_checker().try_borrow().is_err() + { + return 0; + } - match catch_unwind(AssertUnwindSafe(move || impl_(instance, visit))) { - Ok(Ok(())) => 0, - Ok(Err(traverse_error)) => traverse_error.into_inner(), - Err(_err) => -1, + struct TraverseGuard<'a, T: PyClassImpl>(&'a T::Layout); + impl Drop for TraverseGuard<'_, T> { + fn drop(&mut self) { + self.0.borrow_checker().release_borrow() } - } else { - 0 + } + + // `.try_borrow()` above created a borrow, we need to release it when we're done + // traversing the object. This allows us to read `instance` safely. + let _guard = TraverseGuard::(class_object); + let instance = unsafe { &*class_object.contents().value.get() }; + + let visit = PyVisit { + visit, + arg, + _guard: PhantomData, }; - // Drop lock before trap just in case dropping lock panics - drop(lock); - trap.disarm(); - retval + match catch_unwind(AssertUnwindSafe(move || impl_(instance, visit))) { + Ok(Ok(())) => 0, + Ok(Err(traverse_error)) => traverse_error.into_inner(), + Err(_err) => -1, + } } /// Call super-type traverse method, if necessary. @@ -464,11 +500,14 @@ unsafe fn call_super_traverse( } /// Calls an implementation of __clear__ for tp_clear -pub unsafe fn _call_clear( +pub unsafe fn _call_clear( slf: *mut ffi::PyObject, impl_: for<'py> unsafe fn(Python<'py>, *mut ffi::PyObject) -> PyResult<()>, current_clear: ffi::inquiry, -) -> c_int { +) -> c_int +where + T: PyClass, +{ unsafe { trampoline::trampoline(move |py| { let super_retval = call_super_clear(py, slf, current_clear); @@ -476,11 +515,50 @@ pub unsafe fn _call_clear( return Err(PyErr::fetch(py)); } impl_(py, slf)?; + + // Clear the `__dict__`, breaking any reference cycle through the instance's + // attributes. + // + // SAFETY: `slf` is a valid instance of `T`. A shared reference suffices: clearing + // the `__dict__` never touches the pyclass data, so needs no borrow check. + let class_object: &::Layout = &*slf.cast(); + class_object.contents().dict.clear_dict(py); + Ok(0) }) } } +/// `tp_traverse` for a `#[pyclass]` which defines no `__traverse__` of its own: visits the +/// base type and the instance `__dict__` (if it is a `#[pyclass(dict)]`). +pub unsafe extern "C" fn synthesized_traverse( + slf: *mut ffi::PyObject, + visit: ffi::visitproc, + arg: *mut c_void, +) -> c_int +where + T: PyClass, +{ + let super_retval = unsafe { call_super_traverse(slf, visit, arg, synthesized_traverse::) }; + if super_retval != 0 { + return super_retval; + } + + // SAFETY: `slf` is a valid pointer to an instance of `T`, and traversal is running so no + // mutations can occur. The `__dict__` is not Rust data, so needs no thread or borrow check. + let class_object: &::Layout = unsafe { &*slf.cast() }; + unsafe { class_object.contents().dict.traverse_dict(visit, arg) } +} + +/// `tp_clear` for a `#[pyclass]` which defines no `__clear__` of its own: calls the base type +/// and clears the instance `__dict__` (if it is a `#[pyclass(dict)]`). +pub unsafe extern "C" fn synthesized_clear(slf: *mut ffi::PyObject) -> c_int +where + T: PyClass, +{ + unsafe { _call_clear::(slf, |_, _| Ok(()), synthesized_clear::) } +} + /// Call super-type traverse method, if necessary. /// /// Adapted from diff --git a/src/impl_/pymodule.rs b/src/impl_/pymodule.rs index 8e52d1bf996..64f767becfc 100644 --- a/src/impl_/pymodule.rs +++ b/src/impl_/pymodule.rs @@ -33,6 +33,8 @@ use portable_atomic::AtomicI64; #[cfg(not(any(PyPy, GraalPy)))] use crate::exceptions::PyImportError; +#[cfg(any(not(all(Py_LIMITED_API, Py_GIL_DISABLED)), Py_3_15))] +use crate::internal_tricks::array_ptr_as_mut; use crate::prelude::PyTypeMethods; use crate::{ ffi, @@ -76,7 +78,8 @@ impl ModuleDef { pub const fn new( name: &'static CStr, doc: &'static CStr, - slots: &'static PyModuleSlots, + slots: &'static PrimaryModuleSlots, + secondary_slots: &'static SecondaryModuleSlots, ) -> Self { // This is only used in PyO3 for append_to_inittab on Python 3.15 and newer. // There could also be other tools that need the legacy init hook. @@ -98,12 +101,18 @@ impl ModuleDef { let ffi_def = UnsafeCell::new(ffi::PyModuleDef { m_name: name.as_ptr(), m_doc: doc.as_ptr(), - // TODO: would be slightly nicer to use `[T]::as_mut_ptr()` here, - // but that requires mut ptr deref on MSRV. - m_slots: slots.0.get() as _, + m_slots: array_ptr_as_mut({ + cfg_select! { + Py_3_15 => secondary_slots.0.get(), + _ => slots.0.get(), + } + }), ..INIT }); + #[cfg(any(not(Py_3_15), all(Py_LIMITED_API, Py_GIL_DISABLED)))] + let _ = secondary_slots; + ModuleDef { #[cfg(not(all(Py_LIMITED_API, Py_GIL_DISABLED)))] ffi_def, @@ -239,9 +248,10 @@ impl ModuleDef { .map(|py_module| py_module.clone_ref(py)) } } + #[cfg(Py_3_15)] pub fn get_slots(&'static self) -> *mut ffi::PySlot { - self.slots.0.get() as *mut ffi::PySlot + array_ptr_as_mut(self.slots.0.get()) } } @@ -306,15 +316,41 @@ const MAX_SLOTS: usize = 3 * (cfg!(Py_3_15) as usize); const MAX_SLOTS_WITH_TRAILING_NULL: usize = MAX_SLOTS + 1; -/// Builder to create `PyModuleSlots`. The size of the number of slots desired must +/// On Python 3.15+ we use `PySlot` system and `PyModule_FromSlotsAndSpec` +#[cfg(Py_3_15)] +pub type PrimaryModuleSlots = PyModuleSlots; +#[cfg(all(Py_3_15, not(all(Py_LIMITED_API, Py_GIL_DISABLED))))] +pub type SecondaryModuleSlots = PyModuleDefSlots; + +/// On Python 3.14 and older the primary system is `ffi::PyModuleDef`. +#[cfg(not(Py_3_15))] +pub type PrimaryModuleSlots = PyModuleDefSlots; +#[cfg(not(all(Py_3_15, not(all(Py_LIMITED_API, Py_GIL_DISABLED)))))] +pub type SecondaryModuleSlots = (); + +pub const fn secondary_slots(slots: &'static PrimaryModuleSlots) -> SecondaryModuleSlots { + cfg_select! { + // On Python 3.15+ we populate `PyModuleDefSlots` to point at primary slots + // (as long as not using abi3t where `PyModuleDef` is opaque and we cannot know the layout) + all(Py_3_15, not(all(Py_LIMITED_API, Py_GIL_DISABLED))) => PyModuleDefSlots(UnsafeCell::new([ + ffi::PyModuleDef_Slot { + slot: ffi::Py_slot_subslots, + value: slots.0.get().cast(), + }, + // SAFETY: terminator of C-style array + unsafe { core::mem::zeroed() }, + ])), + // Older versions have no secondary slots + _ => { let _ = slots; } + } +} + +/// Builder to create module slots. The size of the number of slots desired must /// be known up front, and N needs to be at least one greater than the number of /// actual slots pushed due to the need to have a zeroed element on the end. pub struct PyModuleSlotsBuilder { // values (initially all zeroed) - #[cfg(not(Py_3_15))] - values: [ffi::PyModuleDef_Slot; MAX_SLOTS_WITH_TRAILING_NULL], - #[cfg(Py_3_15)] - values: [ffi::PySlot; MAX_SLOTS_WITH_TRAILING_NULL], + slots: PrimaryModuleSlots, // current length len: usize, } @@ -329,7 +365,16 @@ impl PyModuleSlotsBuilder { #[allow(clippy::new_without_default)] pub const fn new() -> Self { Self { - values: [unsafe { core::mem::zeroed() }; MAX_SLOTS_WITH_TRAILING_NULL], + slots: cfg_select! { + Py_3_15 => PyModuleSlots(UnsafeCell::new( + // SAFETY: `PySlot` is legal to be zeroed (terminates C-style array) + [unsafe { core::mem::zeroed::() }; MAX_SLOTS_WITH_TRAILING_NULL], + )), + _ => PyModuleDefSlots(UnsafeCell::new( + // SAFETY: `PyModuleDef_Slot` is legal to be zeroed (terminates C-style array) + [unsafe { core::mem::zeroed::() }; MAX_SLOTS_WITH_TRAILING_NULL], + )) + }, len: 0, } } @@ -429,8 +474,8 @@ impl PyModuleSlotsBuilder { } } - pub const fn build(self) -> PyModuleSlots { - PyModuleSlots(UnsafeCell::new(self.values)) + pub const fn build(self) -> PrimaryModuleSlots { + self.slots } #[cfg(not(Py_3_15))] @@ -441,7 +486,7 @@ impl PyModuleSlotsBuilder { self.len < MAX_SLOTS, "Cannot add more than MAX_SLOTS slots to a PyModuleSlots", ); - self.values[self.len] = ffi::PyModuleDef_Slot { slot, value }; + self.slots.0.get_mut()[self.len] = ffi::PyModuleDef_Slot { slot, value }; self.len += 1; self } @@ -452,23 +497,40 @@ impl PyModuleSlotsBuilder { self.len < MAX_SLOTS, "Cannot add more than MAX_SLOTS slots to a PyModuleSlots", ); - self.values[self.len] = value; + self.slots.0.get_mut()[self.len] = value; self.len += 1; self } } /// Wrapper to safely store module slots, to be used in a `ModuleDef`. -#[cfg(not(Py_3_15))] -pub struct PyModuleSlots(UnsafeCell<[ffi::PyModuleDef_Slot; MAX_SLOTS_WITH_TRAILING_NULL]>); -#[cfg(Py_3_15)] -pub struct PyModuleSlots(UnsafeCell<[ffi::PySlot; MAX_SLOTS_WITH_TRAILING_NULL]>); +pub struct PyModuleSlots( + // necessarily empty before Python 3.15; PySlot doesn't exist + #[cfg(Py_3_15)] UnsafeCell<[ffi::PySlot; MAX_SLOTS_WITH_TRAILING_NULL]>, +); + +/// Slots to populate a `PyModuleDef` +/// Cannot create a `PyModuleDef` on abi3t due to lack of knowledge of object layout +#[cfg(not(all(Py_LIMITED_API, Py_GIL_DISABLED)))] +pub struct PyModuleDefSlots( + UnsafeCell< + [ffi::PyModuleDef_Slot; cfg_select! { + // on Python 3.15+ only one slot for pointing at the primary slots, plus trailing null + Py_3_15 => 2, + _ => MAX_SLOTS_WITH_TRAILING_NULL + }], + >, +); // It might be possible to avoid this with SyncUnsafeCell in the future // // SAFETY: the inner values are only accessed within a `ModuleDef`, -// which only uses them to build the `ffi::ModuleDef`. +// used to call `PyModule_FromSlotsAndSpec` unsafe impl Sync for PyModuleSlots {} +// SAFETY: the inner values are only accessed within a `ModuleDef`, +// which only uses them to build the `ffi::ModuleDef`. +#[cfg(not(all(Py_LIMITED_API, Py_GIL_DISABLED)))] +unsafe impl Sync for PyModuleDefSlots {} /// Trait to add an element (class, function...) to a module. /// @@ -530,17 +592,9 @@ mod tests { use alloc::borrow::Cow; use core::{ffi::c_int, ffi::CStr}; - use crate::{ - ffi, - impl_::{ - pymodule::{PyModuleSlots, PyModuleSlotsBuilder}, - trampoline, - }, - types::{any::PyAnyMethods, module::PyModuleMethods}, - Python, - }; + use crate::impl_::trampoline; - use super::{ModuleDef, MAX_SLOTS}; + use super::*; unsafe extern "C" fn module_exec(_module: *mut ffi::PyObject) -> c_int { 0 @@ -560,7 +614,7 @@ mod tests { static NAME: &CStr = c"test_module"; static DOC: &CStr = c"some doc"; - static SLOTS: PyModuleSlots = PyModuleSlotsBuilder::new() + static SLOTS: PrimaryModuleSlots = PyModuleSlotsBuilder::new() .with_mod_exec(module_exec) .with_gil_used(false) .with_abi_info() @@ -568,7 +622,9 @@ mod tests { .with_doc(DOC) .build(); - static MODULE_DEF: ModuleDef = ModuleDef::new(NAME, DOC, &SLOTS); + static SECONDARY_SLOTS: SecondaryModuleSlots = secondary_slots(&SLOTS); + + static MODULE_DEF: ModuleDef = ModuleDef::new(NAME, DOC, &SLOTS, &SECONDARY_SLOTS); Python::attach(|py| { let module = MODULE_DEF.make_module(py).unwrap().into_bound(py); @@ -606,13 +662,25 @@ mod tests { static NAME: &CStr = c"test_module"; static DOC: &CStr = c"some doc"; - static SLOTS: PyModuleSlots = PyModuleSlotsBuilder::new().build(); + static SLOTS: PrimaryModuleSlots = PyModuleSlotsBuilder::new().build(); + static SECONDARY_SLOTS: SecondaryModuleSlots = secondary_slots(&SLOTS); - let module_def: ModuleDef = ModuleDef::new(NAME, DOC, &SLOTS); + let module_def: ModuleDef = ModuleDef::new(NAME, DOC, &SLOTS, &SECONDARY_SLOTS); #[cfg(not(all(Py_LIMITED_API, Py_GIL_DISABLED)))] unsafe { - assert_eq!((*module_def.ffi_def.get()).m_slots, SLOTS.0.get().cast()); + let expected_slots = cfg_select! { + Py_3_15 => SECONDARY_SLOTS.0.get().cast(), + _ => SLOTS.0.get().cast(), + }; + assert_eq!((*module_def.ffi_def.get()).m_slots, expected_slots); + } + #[cfg(all(Py_3_15, not(all(Py_LIMITED_API, Py_GIL_DISABLED))))] + unsafe { + let secondary_slots = &*SECONDARY_SLOTS.0.get(); + assert_eq!(secondary_slots[0].slot, ffi::Py_slot_subslots); + assert_eq!(secondary_slots[0].value, SLOTS.0.get().cast()); + assert!(secondary_slots[1] == ffi::PyModuleDef_Slot::default()); } #[cfg(Py_3_15)] { @@ -624,17 +692,17 @@ mod tests { #[test] #[cfg(panic = "unwind")] fn test_build_maximal_slots() { - let builder = PyModuleSlotsBuilder::new() + let mut builder = PyModuleSlotsBuilder::new() .with_mod_exec(module_exec) .with_name(c"test_module") .with_doc(c"some doc") .with_gil_used(false) .with_abi_info(); - let second_last = builder.values[builder.len - 1]; - let last = builder.values[builder.len]; #[cfg(Py_3_15)] { + let second_last = builder.slots.0.get_mut()[builder.len - 1]; + let last = builder.slots.0.get_mut()[builder.len]; let zeroed = unsafe { core::mem::zeroed() }; fn raw_bytes(inst: &ffi::PySlot) -> &[u8] { unsafe { @@ -650,6 +718,8 @@ mod tests { } #[cfg(not(Py_3_15))] { + let second_last = builder.slots.0.get_mut()[builder.len - 1]; + let last = builder.slots.0.get_mut()[builder.len]; let zeroed = ffi::PyModuleDef_Slot::default(); assert!(last == zeroed); assert!(second_last != zeroed); diff --git a/src/impl_/trampoline.rs b/src/impl_/trampoline.rs index f1d9e040687..98c12fb4ebb 100644 --- a/src/impl_/trampoline.rs +++ b/src/impl_/trampoline.rs @@ -146,7 +146,7 @@ trampolines!( ) -> *mut ffi::PyObject; ); -/// "fastcall" method calls only avaible on abi3 in Python 3.10 and up, otherwise fall back to the older call convention. +/// "fastcall" method calls only available on abi3 in Python 3.10 and up, otherwise fall back to the older call convention. #[cfg(any(Py_3_10, not(Py_LIMITED_API)))] pub use self::fastcall_cfunction_with_keywords as maybe_fastcall_cfunction_with_keywords; @@ -230,7 +230,7 @@ pub unsafe extern "C" fn releasebufferproc $final_arm:expr $(,)? ) => { - #[cfg($cfg)] + #[cfg(all($cfg, not(any($($clauses),*))))] { $final_arm } @@ -30,30 +33,30 @@ macro_rules! cfg_select { ); }; + // Non-terminating expression arm requires trailing comma ( @parsed($($clauses:meta),*) $cfg:meta => $arm:expr, $($rest:tt)* ) => { - #[cfg($cfg)] + #[cfg(all($cfg, not(any($($clauses),*))))] { $arm } - cfg_select! { @parsed($($clauses,)* $cfg) $($rest)* } }; + // Non-terminating block doesn't require trailing comma ( @parsed($($clauses:meta),*) $cfg:meta => $arm:block $($rest:tt)* ) => { - #[cfg($cfg)] + #[cfg(all($cfg, not(any($($clauses),*))))] $arm - cfg_select! { @parsed($($clauses,)* $cfg) $($rest)* @@ -71,3 +74,17 @@ macro_rules! cfg_select { } }; } + +#[cfg(test)] +mod tests { + #[test] + #[cfg(not(cfg_select))] + fn test_cfg_select_polyfill_short_circuit() { + cfg_select! { + all() => {}, + all() => { + unreachable!("the first arm should be selected, so this arm should not be evaluated"); + } + } + } +} diff --git a/src/internal/pyclass_init.rs b/src/internal/pyclass_init.rs index 74bd169ca5b..037f31e92d7 100644 --- a/src/internal/pyclass_init.rs +++ b/src/internal/pyclass_init.rs @@ -177,7 +177,7 @@ where } } -/// Analagous to `Into>`, but just an internal equivalent +/// Analogous to `Into>`, but just an internal equivalent /// to avoid allowing user code to define custom return types from `#[new]`. trait IntoPyClassInitializer: Sized { fn into_pyclass_initializer(self) -> PyClassInitializer; diff --git a/src/internal_tricks.rs b/src/internal_tricks.rs index 0d21c69dcc7..04d6a8f56d7 100644 --- a/src/internal_tricks.rs +++ b/src/internal_tricks.rs @@ -55,3 +55,10 @@ pub(crate) fn box_into_non_null(b: Box) -> NonNull { // SAFETY: `Box::into_raw` guarantees an non-null pointer unsafe { NonNull::new_unchecked(Box::into_raw(b)) } } + +/// Replacement for the unstable `<*mut [T; N]>::as_mut_ptr` method, which avoids +/// possibility of type getting lost from using e.g. `.cast()` to change array +/// type to point to the data. +pub(crate) const fn array_ptr_as_mut(ptr: *mut [T; N]) -> *mut T { + ptr.cast() +} diff --git a/src/lib.rs b/src/lib.rs index 8dd6148b675..b8397f75ed7 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -150,7 +150,7 @@ //! //! # Minimum supported Rust and Python versions //! -//! Requires Rust 1.63 or greater. +//! Requires Rust 1.83 or greater. //! //! PyO3 supports the following Python distributions: //! - CPython 3.8 or greater diff --git a/src/marker.rs b/src/marker.rs index 99d21bd1593..436a7edf90d 100644 --- a/src/marker.rs +++ b/src/marker.rs @@ -118,7 +118,7 @@ //! //! [`SendWrapper`]: https://docs.rs/send_wrapper/latest/send_wrapper/struct.SendWrapper.html //! [`Rc`]: alloc::rc::Rc -//! [`Py`]: crate::Py +//! [`Py`]: Py use crate::conversion::IntoPyObject; use crate::err::{self, PyResult}; use crate::internal::state::{AttachGuard, SuspendAttach}; @@ -286,10 +286,11 @@ mod nightly { impl !Ungil for crate::ffi::PyThreadState {} impl !Ungil for crate::ffi::PyInterpreterState {} + #[cfg(not(any(PyPy, GraalPy)))] impl !Ungil for crate::ffi::PyWeakReference {} impl !Ungil for crate::ffi::PyFrameObject {} impl !Ungil for crate::ffi::PyCodeObject {} - #[cfg(not(Py_LIMITED_API))] + #[cfg(all(not(PyPy), not(Py_LIMITED_API)))] impl !Ungil for crate::ffi::PyDictKeysObject {} #[cfg(not(any(Py_LIMITED_API, Py_3_10)))] impl !Ungil for crate::ffi::PyArena {} diff --git a/src/platform.rs b/src/platform.rs index 8bec1fda621..a295e253a45 100644 --- a/src/platform.rs +++ b/src/platform.rs @@ -1,4 +1,4 @@ -//! This module is to support platform compatiblity with `no_std` environments. +//! This module is to support platform compatibility with `no_std` environments. #![allow(unused_imports)] #[cfg(feature = "hashbrown")] diff --git a/src/pycell/impl_.rs b/src/pycell/impl_.rs index 2475acd937f..a5b13017c37 100644 --- a/src/pycell/impl_.rs +++ b/src/pycell/impl_.rs @@ -14,7 +14,7 @@ use crate::internal::get_slot::{TP_DEALLOC, TP_FREE}; use crate::sync::PyOnceLock; use crate::type_object::{PyLayout, PySizedLayout, PyTypeInfo}; use crate::types::PyType; -use crate::{ffi, PyClass, Python}; +use crate::{ffi, Bound, PyClass, Python}; use crate::types::PyTypeMethods; @@ -263,30 +263,34 @@ unsafe fn tp_dealloc(slf: *mut ffi::PyObject, type_obj: &crate::Bound<'_, PyType // FIXME: there is potentially subtle issues here if the base is overwritten // at runtime? To be investigated. let type_ptr = type_obj.as_type_ptr(); - let actual_type = PyType::from_borrowed_type_ptr(py, ffi::Py_TYPE(slf)); + let actual_type_ptr = ffi::Py_TYPE(slf); + + // For heap types, instances must decref the type object when they + // are deallocated, so we create a bound from a borrowed pointer as + // as if it was an owned pointer. In this way, when the bound is dropped, + // it will decref the type object. + debug_assert!(ffi::PyType_HasFeature(actual_type_ptr, ffi::Py_TPFLAGS_HEAPTYPE) != 0); + let actual_type = Bound::from_owned_ptr(py, actual_type_ptr as *mut ffi::PyObject) + .cast_into_unchecked::(); // For `#[pyclass]` types which inherit from PyAny, we can just call tp_free #[cfg(not(RustPython))] - if core::ptr::eq(type_ptr, &raw const ffi::PyBaseObject_Type) { - let tp_free = actual_type - .get_slot(TP_FREE) - .expect("PyBaseObject_Type should have tp_free"); - return tp_free(slf.cast()); - } + let base_object_type_ptr = &raw const ffi::PyBaseObject_Type; #[cfg(RustPython)] - if core::ptr::eq(type_ptr, { + let base_object_type_ptr = { static TYPE: PyOnceLock> = PyOnceLock::new(); TYPE.import(py, "builtins", "object").unwrap().as_type_ptr() - }) { + }; + + if core::ptr::eq(type_ptr, base_object_type_ptr) { let tp_free = actual_type .get_slot(TP_FREE) .expect("PyBaseObject_Type should have tp_free"); - return tp_free(slf.cast()); + tp_free(slf.cast()); } - // More complex native types (e.g. `extends=PyDict`) require calling the base's dealloc. // FIXME: should this be using actual_type.tp_dealloc? - if let Some(dealloc) = type_obj.get_slot(TP_DEALLOC) { + else if let Some(dealloc) = type_obj.get_slot(TP_DEALLOC) { // Before CPython 3.11 BaseException_dealloc would use Py_GC_UNTRACK which // assumes the exception is currently GC tracked, so we have to re-track // before calling the dealloc so that it can safely call Py_GC_UNTRACK. @@ -298,6 +302,10 @@ unsafe fn tp_dealloc(slf: *mut ffi::PyObject, type_obj: &crate::Bound<'_, PyType } else { type_obj.get_slot(TP_FREE).expect("type missing tp_free")(slf.cast()); } + + // Cause the reference to the type to be decrefed for heap types, which + // is necessary to avoid a reference leak. + drop(actual_type); } } @@ -380,7 +388,7 @@ impl PyClassObjectContents { if self.thread_checker.can_drop(py) { unsafe { ManuallyDrop::drop(&mut self.value) }; } - self.dict.clear_dict(py); + self.dict.release_dict(py); unsafe { self.weakref.clear_weakrefs(py_object, py) }; } } diff --git a/src/pyclass/create_type_object.rs b/src/pyclass/create_type_object.rs index 0f13bcb8832..47a07a2ba85 100644 --- a/src/pyclass/create_type_object.rs +++ b/src/pyclass/create_type_object.rs @@ -15,7 +15,10 @@ use crate::{ assign_sequence_item_from_mapping, get_sequence_item_from_mapping, tp_dealloc, tp_dealloc_with_gc, PyClassImpl, PyClassItemsIter, PyObjectOffset, }, - pymethods::{_call_clear, Getter, PyGetterDef, PyMethodDefType, PySetterDef, Setter}, + pymethods::{ + synthesized_clear, synthesized_traverse, Getter, PyGetterDef, PyMethodDefType, + PySetterDef, Setter, + }, trampoline::trampoline, }, pycell::impl_::PyClassObjectLayout, @@ -50,6 +53,8 @@ where base: *mut ffi::PyTypeObject, dealloc: unsafe extern "C" fn(*mut ffi::PyObject), dealloc_with_gc: unsafe extern "C" fn(*mut ffi::PyObject), + synthesized_traverse: ffi::traverseproc, + synthesized_clear: ffi::inquiry, is_mapping: bool, is_sequence: bool, is_immutable_type: bool, @@ -73,6 +78,8 @@ where tp_base: base, tp_dealloc: dealloc, tp_dealloc_with_gc: dealloc_with_gc, + synthesized_traverse, + synthesized_clear, is_mapping, is_sequence, is_immutable_type, @@ -101,6 +108,8 @@ where T::BaseType::type_object_raw(py), tp_dealloc::, tp_dealloc_with_gc::, + synthesized_traverse::, + synthesized_clear::, T::IS_MAPPING, T::IS_SEQUENCE, T::IS_IMMUTABLE_TYPE, @@ -132,6 +141,10 @@ struct PyTypeBuilder { tp_base: *mut ffi::PyTypeObject, tp_dealloc: ffi::destructor, tp_dealloc_with_gc: ffi::destructor, + /// `tp_traverse` / `tp_clear` to install when the class needs the slot but defines no + /// `__traverse__` / `__clear__` of its own. + synthesized_traverse: ffi::traverseproc, + synthesized_clear: ffi::inquiry, is_mapping: bool, is_sequence: bool, is_immutable_type: bool, @@ -464,6 +477,25 @@ impl PyTypeBuilder { } } + // A reference cycle can run through the instance `__dict__` (`obj.x = obj`), so a class + // with a `__dict__` must be a GC type. `_call_traverse` / `_call_clear` service the + // `__dict__` when the user defines `__traverse__` / `__clear__`; synthesize whichever + // slot they did not. + // + // Must run before the `tp_dealloc` selection below, which keys off `has_traverse`. + if self.dict_offset.is_some() { + if !self.has_traverse { + let synthesized_traverse = self.synthesized_traverse; + // Safety: This is the correct slot type for Py_tp_traverse + unsafe { self.push_slot(ffi::Py_tp_traverse, synthesized_traverse as *mut c_void) } + } + if !self.has_clear { + let synthesized_clear = self.synthesized_clear; + // Safety: This is the correct slot type for Py_tp_clear + unsafe { self.push_slot(ffi::Py_tp_clear, synthesized_clear as *mut c_void) } + } + } + let base_is_gc = unsafe { ffi::PyType_IS_GC(self.tp_base) == 1 }; let tp_dealloc = if self.has_traverse || base_is_gc { self.tp_dealloc_with_gc @@ -489,8 +521,9 @@ impl PyTypeBuilder { assert!(self.has_traverse); // Py_TPFLAGS_HAVE_GC is set when a `__traverse__` method is found if !self.has_clear { + let synthesized_clear = self.synthesized_clear; // Safety: This is the correct slot type for Py_tp_clear - unsafe { self.push_slot(ffi::Py_tp_clear, call_super_clear as *mut c_void) } + unsafe { self.push_slot(ffi::Py_tp_clear, synthesized_clear as *mut c_void) } } } @@ -629,10 +662,6 @@ unsafe extern "C" fn no_constructor_defined( } } -unsafe extern "C" fn call_super_clear(slf: *mut ffi::PyObject) -> c_int { - unsafe { _call_clear(slf, |_, _| Ok(()), call_super_clear) } -} - #[derive(Default)] struct GetSetDefBuilder { doc: Option<&'static CStr>, diff --git a/src/pyclass/guard.rs b/src/pyclass/guard.rs index 7566e56df52..c8c6c910d05 100644 --- a/src/pyclass/guard.rs +++ b/src/pyclass/guard.rs @@ -884,11 +884,8 @@ where /// Borrows a mutable reference to `PyClassGuardMut`. /// /// See [`PyClassGuardMut::as_super`] for more. - pub fn as_super(&mut self) -> PyClassGuardMutSuper<'a, 'g, T::BaseType> { - PyClassGuardMutSuper { - // SAFETY: `PyClassGuardMut` and `PyClassGuardMut` have the same layout - guard: unsafe { NonNull::from(&mut *self.guard).cast().as_mut() }, - } + pub fn as_super(&mut self) -> PyClassGuardMutSuper<'_, 'g, T::BaseType> { + self.guard.as_super() } } diff --git a/src/pyclass_init.rs b/src/pyclass_init.rs index 8e996d9b541..6b8ac028222 100644 --- a/src/pyclass_init.rs +++ b/src/pyclass_init.rs @@ -155,9 +155,38 @@ impl PyClassInitializer { // SAFETY: `obj` is constructed using `T::Layout` but has not been initialized yet let contents = unsafe { ::Layout::contents_uninit(obj) }; + + let new_contents = PyClassObjectContents::new(self.init); + + // CPython 3.11 and 3.12 eagerly create the instance dict for types with a nonzero + // `tp_dictoffset` in `_PyObject_InitializeDict`, storing an owned reference in + // the `__dict__` slot, which lives inside `contents`. Carry that value over + // instead of clobbering it below, otherwise it leaks. Python 3.13 returned to + // creating the instance dict lazily + // + // The condition is `not(Py_3_13)` rather than `all(Py_3_11, not(Py_3_13))` + // because an abi3 build with a lower minimum version can still run on 3.11 and + // 3.12; on 3.10 and older this is a harmless no-op (the slot is always null + // there). + #[cfg(not(Py_3_13))] + let new_contents = { + let mut new_contents = new_contents; + if eagerly_created_dict_possible::(py) { + // SAFETY: `tp_alloc` zero-initializes the object, so the slot contains either + // zeroes (a valid empty slot value) or a valid owned pointer stored by the base + // `tp_new` through the type's `tp_dictoffset`. + unsafe { + let contents_ptr = (*contents).as_mut_ptr(); + let dict_ptr = &raw const (*contents_ptr).dict; + new_contents.dict = core::ptr::read(dict_ptr); + } + } + new_contents + }; + // SAFETY: `contents` is a non-null pointer to the space allocated for our // `PyClassObjectContents` (either statically in Rust or dynamically by Python) - unsafe { (*contents).write(PyClassObjectContents::new(self.init)) }; + unsafe { (*contents).write(new_contents) }; // Safety: obj is a valid pointer to an object of type `target_type`, which` is a known // subclass of `T` @@ -165,6 +194,35 @@ impl PyClassInitializer { } } +/// Whether the running interpreter may have eagerly created an instance dict for `T` +/// during `tp_new` (CPython 3.11 and 3.12 only). +/// +/// For native builds the compile-time `not(Py_3_13)` gate at the call site is exact; abi3 +/// builds with a minimum version below 3.13 must check the interpreter version at runtime +#[cfg(not(Py_3_13))] +#[inline] +fn eagerly_created_dict_possible(py: Python<'_>) -> bool { + if core::mem::size_of::() == 0 { + return false; + } + cfg_select! { + Py_LIMITED_API => + { + use crate::sync::PyOnceLock; + static IS_PYTHON_3_11_OR_3_12: PyOnceLock = PyOnceLock::new(); + *IS_PYTHON_3_11_OR_3_12.get_or_init(py, || { + let version_info = py.version_info(); + matches!((version_info.major, version_info.minor), (3, 11) | (3, 12)) + }) + } + not(Py_LIMITED_API) => + { + let _ = py; + cfg!(Py_3_11) + } + } +} + impl PyObjectInit for PyClassInitializer { unsafe fn into_new_object( self, diff --git a/src/sync.rs b/src/sync.rs index f3babca27e7..0ec45ff9c2b 100644 --- a/src/sync.rs +++ b/src/sync.rs @@ -88,11 +88,13 @@ impl Default for GILOnceCell { } } +// SAFETY: Sync is only implemented if the inner type is Sync // T: Send is needed for Sync because the thread which drops the GILOnceCell can be different // to the thread which fills it. (e.g. think scoped thread which fills the cell and then exits, // leaving the cell to be dropped by the main thread). #[allow(deprecated)] unsafe impl Sync for GILOnceCell {} +// SAFETY: send is only implemented if the inner type is send #[allow(deprecated)] unsafe impl Send for GILOnceCell {} diff --git a/src/sync/critical_section.rs b/src/sync/critical_section.rs index f07c4af769d..eb2eb99a7fd 100644 --- a/src/sync/critical_section.rs +++ b/src/sync/critical_section.rs @@ -1,3 +1,6 @@ +// TODO https://github.com/PyO3/pyo3/issues/5487 +#![allow(clippy::undocumented_unsafe_blocks)] + //! Wrappers for the Python critical section API //! //! [Critical Sections](https://docs.python.org/3/c-api/init.html#python-critical-section-api) allow @@ -14,7 +17,7 @@ //! until exiting the critical section unless the critical section is suspended. Any call into the //! CPython C API may cause the critical section to be suspended. Creating an inner critical //! section, for example by accessing an item in a Python list or dict, will cause the outer -//! critical section to be relased while the inner critical section is active. +//! critical section to be released while the inner critical section is active. //! //! As a consequence, it is only possible to lock one or two objects at a time. If you need two lock //! two objects, you should use the variants that accept two arguments. The outer critical section diff --git a/src/types/mod.rs b/src/types/mod.rs index 43da1492f33..9f7eebd0d0b 100644 --- a/src/types/mod.rs +++ b/src/types/mod.rs @@ -242,6 +242,7 @@ macro_rules! pyobject_subclassable_native_type { #[macro_export] macro_rules! pyobject_native_type_sized { ($name:ty, $layout:path $(;$generics:ident)*) => { + // SAFETY: native objects are valid unsafe impl $crate::type_object::PyLayout<$name> for $layout {} impl $crate::type_object::PySizedLayout<$name> for $layout {} }; diff --git a/src/types/module.rs b/src/types/module.rs index bea42e5d78e..fd6ce434c7f 100644 --- a/src/types/module.rs +++ b/src/types/module.rs @@ -126,9 +126,7 @@ impl PyModule { ///
///
⚠ ️
///
-    //
-    ///  Warning: This will compile and execute code. Never pass untrusted code to this function!
-    ///
+    /// Warning: This will compile and execute code. Never pass untrusted code to this function!
     /// 
/// /// # Errors diff --git a/src/types/tuple.rs b/src/types/tuple.rs index 08256d7156d..e14725aacff 100644 --- a/src/types/tuple.rs +++ b/src/types/tuple.rs @@ -39,14 +39,14 @@ fn try_new_from_iter<'py>( py: Python<'py>, mut elements: impl ExactSizeIterator>>, ) -> PyResult> { - #[cfg(not(RustPython))] - unsafe { - // PyTuple_New checks for overflow but has a bad error message, so we check ourselves - let len: Py_ssize_t = elements - .len() - .try_into() - .expect("out of range integral type conversion attempted on `elements.len()`"); + // PyTuple_New checks for overflow but has a bad error message, so we check ourselves + let len: Py_ssize_t = elements + .len() + .try_into() + .expect("out of range integral type conversion attempted on `elements.len()`"); + #[cfg(not(RustPython))] + let (tup, counter) = unsafe { let ptr = ffi::PyTuple_New(len); // - Panics if the ptr is null @@ -63,20 +63,26 @@ fn try_new_from_iter<'py>( counter += 1; } - assert!(elements.next().is_none(), "Attempted to create PyTuple but `elements` was larger than reported by its `ExactSizeIterator` implementation."); - assert_eq!(len, counter, "Attempted to create PyTuple but `elements` was smaller than reported by its `ExactSizeIterator` implementation."); - - Ok(tup) - } + (tup, counter) + }; #[cfg(RustPython)] - unsafe { - let elements = elements.collect::>>()?; + let (tup, counter) = unsafe { + let elements = (&mut elements) + .take(len as _) + .collect::>>()?; // SAFETY: list is layout compatible with *const *mut crate::PyObject - ffi::PyTuple_FromArray(elements.as_ptr().cast(), elements.len() as _) + let tup = ffi::PyTuple_FromArray(elements.as_ptr().cast(), elements.len() as _) .assume_owned_or_err(py) - .cast_into_unchecked() - } + .cast_into_unchecked()?; + + (tup, elements.len() as Py_ssize_t) + }; + + assert!(elements.next().is_none(), "Attempted to create PyTuple but `elements` was larger than reported by its `ExactSizeIterator` implementation."); + assert_eq!(len, counter, "Attempted to create PyTuple but `elements` was smaller than reported by its `ExactSizeIterator` implementation."); + + Ok(tup) } /// Represents a Python `tuple` object. diff --git a/tests/test_class_basics.rs b/tests/test_class_basics.rs index 90c9c1706fa..4b5f6a4ac34 100644 --- a/tests/test_class_basics.rs +++ b/tests/test_class_basics.rs @@ -478,6 +478,85 @@ fn access_dunder_dict() { }); } +#[cfg(any(Py_3_9, not(Py_LIMITED_API)))] +#[test] +fn dunder_dict_is_released() { + Python::attach(|py| { + let inst = Py::new( + py, + DunderDictSupport { + _pad: *b"DEADBEEFDEADBEEFDEADBEEFDEADBEEF", + }, + ) + .unwrap(); + + inst.setattr(py, "a", 1).unwrap(); + + let dict = inst.bind(py).getattr("__dict__").unwrap(); + let get_refcnt = || { + // SAFETY: `dict` holds a valid reference while its reference count is read. + unsafe { pyo3::ffi::Py_REFCNT(dict.as_ptr()) } + }; + let refcnt = get_refcnt(); + + drop(inst); + + assert_eq!(get_refcnt(), refcnt - 1); + py_assert!(py, dict, "dict == {'a': 1}"); + }); +} + +// The `__dict__` slot must hold exactly what CPython's `tp_new` left there: CPython 3.11 +// and 3.12 eagerly create the instance dict in `object.__new__` for types with a nonzero +// `tp_dictoffset`, and the pyclass contents initialization must preserve it rather than +// clobber it. All other versions create the dict lazily on first access. +#[cfg(all(not(Py_LIMITED_API), not(PyPy), not(GraalPy)))] +#[test] +fn instance_dict_slot_is_not_clobbered() { + Python::attach(|py| { + let inst = Py::new( + py, + DunderDictSupport { + _pad: *b"DEADBEEFDEADBEEFDEADBEEFDEADBEEF", + }, + ) + .unwrap(); + + // Read the `__dict__` slot directly (see `_PyObject_GetDictPtr`), *without* + // going through `__dict__`. + // SAFETY: `inst` is a valid object whose type has a positive `tp_dictoffset`. + let read_slot = || unsafe { + let offset = (*pyo3::ffi::Py_TYPE(inst.as_ptr())).tp_dictoffset; + assert!(offset > 0); + *inst + .as_ptr() + .cast::() + .offset(offset) + .cast::<*mut pyo3::ffi::PyObject>() + }; + + let slot_dict = read_slot(); + if cfg!(all(Py_3_11, not(Py_3_13))) { + // `object.__new__` created the dict; it must survive pyclass initialization. + assert!( + !slot_dict.is_null(), + "the eagerly created __dict__ was clobbered during pyclass initialization" + ); + // The slot holds the only reference to it. + // SAFETY: previous assert guarantees it's a valid PyObject + assert_eq!(unsafe { pyo3::ffi::Py_REFCNT(slot_dict) }, 1); + } else { + // No eager creation on these versions; the slot starts out empty. + assert!(slot_dict.is_null()); + } + + // Whichever way the dict comes into existence, `__dict__` must be the dict + // stored in the slot. + let dict_attr = inst.bind(py).getattr("__dict__").unwrap(); + assert_eq!(read_slot(), dict_attr.as_ptr()); + }); +} + // If the base class has dict support, child class also has dict #[cfg(any(Py_3_9, not(Py_LIMITED_API)))] #[pyclass(extends=DunderDictSupport)] diff --git a/tests/test_compile_error.rs b/tests/test_compile_error.rs index 0164347c38e..8784c28581c 100644 --- a/tests/test_compile_error.rs +++ b/tests/test_compile_error.rs @@ -152,6 +152,11 @@ fn main() { Regex::new(r"and \d+ others").unwrap().into(), b"and $$N others".to_vec(), ), + // Normalize paths into the Rust toolchain sources + ( + Regex::new(r"[^\s]*?/rustlib/src/rust").unwrap().into(), + b"$$RUST_SRC".to_vec(), + ), // Some trait implementations which are only emitted with certain // features enabled ( @@ -171,7 +176,7 @@ fn main() { .insert("with-experimental-inspect", |parser, _args, span| { parser.set_custom_once( "with-experimental-inspect", - SplitBuildOnExperimentalInpsect { + SplitBuildOnExperimentalInspect { requires_inspect: true, }, span, @@ -182,7 +187,7 @@ fn main() { .insert("without-experimental-inspect", |parser, _args, span| { parser.set_custom_once( "without-experimental-inspect", - SplitBuildOnExperimentalInpsect { + SplitBuildOnExperimentalInspect { requires_inspect: false, }, span, @@ -269,13 +274,40 @@ fn normalize_src_blocks(output: &[u8]) -> Vec { .into_owned() } +fn check_rust_src_paths(output: &[u8], errors: &mut Vec) -> bool { + use std::sync::LazyLock; + + use regex::bytes::Regex; + + static REMAPPED_RUST_SRC: LazyLock = + LazyLock::new(|| Regex::new(r"/rustc/[0-9a-f]{40}/library/").unwrap()); + + if REMAPPED_RUST_SRC.is_match(output) { + // This causes `ui_test` to emit: + // + // ``` + // error: a bug in `ui_test` occurred + // rust-src is required for UI tests; install it with `rustup component add rust-src` + // ``` + errors.push(ui_test::Error::Bug( + "rust-src is required for UI tests; install it with `rustup component add rust-src`" + .into(), + )); + false + } else { + true + } +} + fn error_on_output_conflict_normalized( path: &std::path::Path, output: &[u8], errors: &mut Vec, config: &ui_test::per_test_config::TestConfig, ) { - ui_test::error_on_output_conflict(path, &normalize_src_blocks(output), errors, config); + if check_rust_src_paths(output, errors) { + ui_test::error_on_output_conflict(path, &normalize_src_blocks(output), errors, config); + } } fn bless_output_files_normalized( @@ -284,17 +316,19 @@ fn bless_output_files_normalized( errors: &mut Vec, config: &ui_test::per_test_config::TestConfig, ) { - ui_test::bless_output_files(path, &normalize_src_blocks(output), errors, config); + if check_rust_src_paths(output, errors) { + ui_test::bless_output_files(path, &normalize_src_blocks(output), errors, config); + } } /// Some tests have different error messages when the `experimental-inspect` feature is /// enabled. #[derive(Clone, Debug)] -struct SplitBuildOnExperimentalInpsect { +struct SplitBuildOnExperimentalInspect { requires_inspect: bool, } -impl ui_test::custom_flags::Flag for SplitBuildOnExperimentalInpsect { +impl ui_test::custom_flags::Flag for SplitBuildOnExperimentalInspect { fn clone_inner(&self) -> Box { Box::new(self.clone()) } diff --git a/tests/test_gc.rs b/tests/test_gc.rs index 822168096a5..d2edf2a2a2e 100644 --- a/tests/test_gc.rs +++ b/tests/test_gc.rs @@ -10,7 +10,7 @@ use pyo3::prelude::*; use pyo3::py_run; #[cfg(not(target_arch = "wasm32"))] use std::cell::Cell; -use std::sync::atomic::{AtomicBool, Ordering}; +use std::sync::atomic::{AtomicBool, AtomicPtr, Ordering}; use std::sync::Once; use std::sync::{Arc, Mutex}; @@ -736,6 +736,27 @@ extern "C" fn visit_error( -1 } +// the fields visited below, set before driving the traversal +static BASE_FIELD: AtomicPtr = AtomicPtr::new(std::ptr::null_mut()); +static CHILD_FIELD: AtomicPtr = AtomicPtr::new(std::ptr::null_mut()); +static BASE_VISITED: AtomicBool = AtomicBool::new(false); +static CHILD_VISITED: AtomicBool = AtomicBool::new(false); + +// a visitor function which errors on `BASE_FIELD` only +extern "C" fn visit_error_on_base_field( + object: *mut pyo3::ffi::PyObject, + _arg: *mut core::ffi::c_void, +) -> std::ffi::c_int { + if object == CHILD_FIELD.load(Ordering::SeqCst) { + CHILD_VISITED.store(true, Ordering::SeqCst); + } + if object == BASE_FIELD.load(Ordering::SeqCst) { + BASE_VISITED.store(true, Ordering::SeqCst); + return -1; + } + 0 +} + #[test] #[cfg(any(not(Py_LIMITED_API), Py_3_11))] // buffer availability fn test_drop_buffer_during_traversal_without_gil() { @@ -787,3 +808,187 @@ fn test_drop_buffer_during_traversal_without_gil() { check.assert_drops_with_gc(ptr); }); } + +#[test] +fn test_super_traverse_early_return_does_not_abort() { + #[pyclass(subclass)] + struct TraverseBase { + field: Py, + } + + #[pymethods] + impl TraverseBase { + fn __traverse__(&self, visit: PyVisit<'_>) -> Result<(), PyTraverseError> { + visit.call(&self.field) + } + } + + #[pyclass(extends=TraverseBase)] + struct TraverseChild { + field: Py, + } + + #[pymethods] + impl TraverseChild { + fn __traverse__(&self, visit: PyVisit<'_>) -> Result<(), PyTraverseError> { + visit.call(&self.field) + } + } + + Python::attach(|py| { + let base_field = pyo3::types::PyList::empty(py).into_any().unbind(); + let child_field = pyo3::types::PyList::empty(py).into_any().unbind(); + + let initializer = PyClassInitializer::from(TraverseBase { + field: base_field.clone_ref(py), + }) + .add_subclass(TraverseChild { + field: child_field.clone_ref(py), + }); + let child = Bound::new(py, initializer).unwrap(); + + BASE_FIELD.store(base_field.as_ptr(), Ordering::SeqCst); + CHILD_FIELD.store(child_field.as_ptr(), Ordering::SeqCst); + + let traverse = + unsafe { get_type_traverse(py.get_type::().as_type_ptr()).unwrap() }; + + let retval = unsafe { + traverse( + child.as_ptr(), + visit_error_on_base_field, + std::ptr::null_mut(), + ) + }; + + assert!( + BASE_VISITED.load(Ordering::SeqCst), + "super-type traverse never visited its field, so the early return was not exercised" + ); + assert_eq!( + retval, -1, + "traverse did not propagate the non-zero return of the super-type traverse" + ); + assert!( + !CHILD_VISITED.load(Ordering::SeqCst), + "child __traverse__ ran despite the super-type traverse returning non-zero" + ); + }); +} + +// A `#[pyclass(dict)]` can form a reference cycle through its instance `__dict__` +// (`obj.attr = obj`). The tests below cover each valid combination of user-defined +// `__traverse__` / `__clear__`; a `__clear__` without a `__traverse__` is rejected at +// type-creation time. + +#[cfg(any(Py_3_9, not(Py_LIMITED_API)))] +#[test] +fn dict_class_is_a_gc_type() { + Python::attach(|py| { + let ty = py.get_type::(); + let flags = unsafe { ffi::PyType_GetFlags(ty.as_type_ptr()) }; + assert_ne!(flags & ffi::Py_TPFLAGS_HAVE_GC, 0); + }); +} + +/// `#[pyclass(dict)]` with neither `__traverse__` nor `__clear__`: both slots are synthesized. +#[cfg(any(Py_3_9, not(Py_LIMITED_API)))] +#[pyclass(dict)] +struct DictCycleNoTraverse { + _guard: DropGuard, +} + +#[cfg(any(Py_3_9, not(Py_LIMITED_API)))] +#[test] +fn dict_cycle_collected_without_traverse() { + let (guard, check) = drop_check(); + + let ptr = Python::attach(|py| { + let inst = Bound::new(py, DictCycleNoTraverse { _guard: guard }).unwrap(); + // Reference cycle through the instance `__dict__`: inst.__dict__["cycle"] -> inst. + inst.setattr("cycle", &inst).unwrap(); + check.assert_not_dropped(); + inst.as_ptr() + }); + + check.assert_drops_with_gc(ptr); +} + +/// `#[pyclass(dict)]` with `__traverse__` but no `__clear__`: the `__dict__` is visited by +/// `_call_traverse` and cleared by a synthesized `tp_clear`. +#[cfg(any(Py_3_9, not(Py_LIMITED_API)))] +#[pyclass(dict)] +struct DictCycleTraverseOnly { + _guard: DropGuard, +} + +#[cfg(any(Py_3_9, not(Py_LIMITED_API)))] +#[pymethods] +impl DictCycleTraverseOnly { + #[expect(clippy::unnecessary_wraps)] + fn __traverse__(&self, _visit: PyVisit<'_>) -> Result<(), PyTraverseError> { + // No Rust references to visit; the `__dict__` is visited by `_call_traverse` itself. + Ok(()) + } +} + +#[cfg(any(Py_3_9, not(Py_LIMITED_API)))] +#[test] +fn dict_cycle_collected_with_traverse_only() { + let (guard, check) = drop_check(); + + let ptr = Python::attach(|py| { + let inst = Bound::new(py, DictCycleTraverseOnly { _guard: guard }).unwrap(); + inst.setattr("cycle", &inst).unwrap(); + check.assert_not_dropped(); + inst.as_ptr() + }); + + check.assert_drops_with_gc(ptr); +} + +/// `#[pyclass(dict)]` with both `__traverse__` and `__clear__`: the `__dict__` is folded into +/// the user-defined slots by `_call_traverse` / `_call_clear`. +#[cfg(any(Py_3_9, not(Py_LIMITED_API)))] +#[pyclass(dict)] +struct DictCycleTraverseAndClear { + _guard: DropGuard, + field: Option>, +} + +#[cfg(any(Py_3_9, not(Py_LIMITED_API)))] +#[pymethods] +impl DictCycleTraverseAndClear { + fn __traverse__(&self, visit: PyVisit<'_>) -> Result<(), PyTraverseError> { + if let Some(field) = &self.field { + visit.call(field)?; + } + Ok(()) + } + + fn __clear__(&mut self) { + self.field = None; + } +} + +#[cfg(any(Py_3_9, not(Py_LIMITED_API)))] +#[test] +fn dict_cycle_collected_with_traverse_and_clear() { + let (guard, check) = drop_check(); + + let ptr = Python::attach(|py| { + let inst = Bound::new( + py, + DictCycleTraverseAndClear { + _guard: guard, + field: None, + }, + ) + .unwrap(); + inst.setattr("cycle", &inst).unwrap(); + check.assert_not_dropped(); + inst.as_ptr() + }); + + check.assert_drops_with_gc(ptr); +} diff --git a/tests/test_inheritance.rs b/tests/test_inheritance.rs index 1bdab33c96f..f8e52430f4e 100644 --- a/tests/test_inheritance.rs +++ b/tests/test_inheritance.rs @@ -6,6 +6,47 @@ use pyo3::types::IntoPyDict; mod test_utils; +/// Macro to generate refcount leak tests for types. +/// Ensures that creating and destroying instances doesn't leak references to the type. +/// Regression test for issues #1363 and #6223. +#[cfg(not(Py_GIL_DISABLED))] +macro_rules! assert_type_refcount_stable { + // Simple case: type with parameterless constructor + ($type_name:ty) => { + assert_type_refcount_stable!($type_name, stringify!($type_name), "Type()"); + }; + // With custom constructor + ($type_name:ty, $test_name:expr, $ctor:expr) => {{ + Python::attach(|py| { + #[expect(non_snake_case)] + let Type = py.get_type::<$type_name>(); + let ctor_code = $ctor; + py_run!( + py, + Type, + &format!( + r#" + import gc + import sys + + gc.collect() + count = sys.getrefcount(Type) + + for i in range(1000): + obj = {} + del obj + + gc.collect() + after = sys.getrefcount(Type) + assert after == count, f"Type ref count leaked: {{after}} vs {{count}}" + "#, + ctor_code + ) + ); + }); + }}; +} + #[pyclass(subclass)] struct BaseClass { #[pyo3(get)] @@ -188,25 +229,28 @@ mod inheriting_native_type { }; #[cfg(not(any(PyPy, GraalPy)))] - #[test] - fn inherit_set() { - use pyo3::types::PySet; + use pyo3::types::PySet; - #[pyclass(extends=PySet)] - #[derive(Debug)] - struct SetWithName { - #[pyo3(get, name = "name")] - _name: &'static str, - } + #[cfg(not(any(PyPy, GraalPy)))] + #[pyclass(extends=PySet)] + #[derive(Debug)] + pub struct SetWithName { + #[pyo3(get, name = "name")] + _name: &'static str, + } - #[pymethods] - impl SetWithName { - #[new] - fn new() -> Self { - SetWithName { _name: "Hello :)" } - } + #[cfg(not(any(PyPy, GraalPy)))] + #[pymethods] + impl SetWithName { + #[new] + fn new() -> Self { + SetWithName { _name: "Hello :)" } } + } + #[cfg(not(any(PyPy, GraalPy)))] + #[test] + fn inherit_set() { Python::attach(|py| { let set_sub = pyo3::Py::new(py, SetWithName::new()).unwrap(); py_run!( @@ -314,31 +358,33 @@ mod inheriting_native_type { } #[cfg(Py_3_12)] - #[test] - fn inherit_tzinfo() { - #[pyclass(extends=pyo3::types::PyTzInfo)] - struct TzInfoWithName { - #[pyo3(get)] - name: &'static str, - } + #[pyclass(extends=pyo3::types::PyTzInfo)] + struct TzInfoWithName { + #[pyo3(get)] + name: &'static str, + } - #[pymethods] - impl TzInfoWithName { - #[new] - fn new() -> Self { - Self { name: "Hello :)" } - } + #[cfg(Py_3_12)] + #[pymethods] + impl TzInfoWithName { + #[new] + fn new() -> Self { + Self { name: "Hello :)" } + } - #[pyo3(signature = (_dt, /))] - fn utcoffset<'py>( - &self, - _dt: Option<&Bound<'_, pyo3::types::PyDateTime>>, - py: Python<'py>, - ) -> PyResult> { - pyo3::types::PyDelta::new(py, 0, 3600, 0, true) - } + #[pyo3(signature = (_dt, /))] + fn utcoffset<'py>( + &self, + _dt: Option<&Bound<'_, pyo3::types::PyDateTime>>, + py: Python<'py>, + ) -> PyResult> { + pyo3::types::PyDelta::new(py, 0, 3600, 0, true) } + } + #[cfg(Py_3_12)] + #[test] + fn inherit_tzinfo() { Python::attach(|py| { let tz = pyo3::Py::new(py, TzInfoWithName::new()).unwrap(); py_run!( @@ -357,39 +403,43 @@ mod inheriting_native_type { }); } - #[test] #[cfg(Py_3_12)] - fn inherit_list() { - #[pyclass(extends=pyo3::types::PyList, subclass)] - struct ListWithName { - #[pyo3(get)] - name: &'static str, - } + #[pyclass(extends=pyo3::types::PyList, subclass)] + struct ListWithName { + #[pyo3(get)] + name: &'static str, + } - #[pymethods] - impl ListWithName { - #[new] - fn new() -> Self { - Self { name: "Hello :)" } - } + #[cfg(Py_3_12)] + #[pymethods] + impl ListWithName { + #[new] + fn new() -> Self { + Self { name: "Hello :)" } } + } - #[pyclass(extends=ListWithName)] - struct SubListWithName { - #[pyo3(get)] - sub_name: &'static str, - } + #[cfg(Py_3_12)] + #[pyclass(extends=ListWithName)] + struct SubListWithName { + #[pyo3(get)] + sub_name: &'static str, + } - #[pymethods] - impl SubListWithName { - #[new] - fn new() -> PyClassInitializer { - PyClassInitializer::from(ListWithName::new()).add_subclass(Self { - sub_name: "Sublist", - }) - } + #[cfg(Py_3_12)] + #[pymethods] + impl SubListWithName { + #[new] + fn new() -> PyClassInitializer { + PyClassInitializer::from(ListWithName::new()).add_subclass(Self { + sub_name: "Sublist", + }) } + } + #[cfg(Py_3_12)] + #[test] + fn inherit_list() { Python::attach(|py| { let list_with_name = pyo3::Bound::new(py, ListWithName::new()).unwrap(); let sub_list_with_name = pyo3::Bound::new(py, SubListWithName::new()).unwrap(); @@ -409,6 +459,43 @@ mod inheriting_native_type { ); }); } + + // Refcount tests for native type classes + #[cfg(not(any(PyPy, GraalPy, Py_GIL_DISABLED)))] + #[test] + fn test_setwitname_ref_counts() { + assert_type_refcount_stable!(SetWithName); + } + + #[cfg(not(any(GraalPy, Py_GIL_DISABLED)))] + #[test] + fn test_dictwithname_ref_counts() { + assert_type_refcount_stable!(DictWithName); + } + + #[cfg(not(Py_GIL_DISABLED))] + #[test] + fn test_customexception_ref_counts() { + assert_type_refcount_stable!(CustomException, "custom_exception", r#"Type('test')"#); + } + + #[cfg(all(Py_3_12, not(Py_GIL_DISABLED)))] + #[test] + fn test_tzinfowithname_ref_counts() { + assert_type_refcount_stable!(TzInfoWithName); + } + + #[cfg(all(Py_3_12, not(Py_GIL_DISABLED)))] + #[test] + fn test_listwithname_ref_counts() { + assert_type_refcount_stable!(ListWithName); + } + + #[cfg(all(Py_3_12, not(Py_GIL_DISABLED)))] + #[test] + fn test_sublistwithname_ref_counts() { + assert_type_refcount_stable!(SubListWithName); + } } #[pyclass(subclass)] @@ -422,37 +509,33 @@ impl SimpleClass { } } +// Generate refcount tests for all top-level types +#[cfg(not(Py_GIL_DISABLED))] +#[test] +fn test_baseclass_ref_counts() { + assert_type_refcount_stable!(BaseClass); +} + +#[cfg(not(Py_GIL_DISABLED))] #[test] fn test_subclass_ref_counts() { - // regression test for issue #1363 - Python::attach(|py| { - #[expect(non_snake_case)] - let SimpleClass = py.get_type::(); - py_run!( - py, - SimpleClass, - r#" - import gc - import sys - - class SubClass(SimpleClass): - pass - - gc.collect() - count = sys.getrefcount(SubClass) - - for i in range(1000): - c = SubClass() - del c - - gc.collect() - after = sys.getrefcount(SubClass) - # depending on Python's GC the count may be either identical or exactly 1000 higher, - # both are expected values that are not representative of the issue. - # - # (With issue #1363 the count will be decreased.) - assert after == count or (after == count + 1000), f"{after} vs {count}" - "# - ); - }) + assert_type_refcount_stable!(SubClass); +} + +#[cfg(not(Py_GIL_DISABLED))] +#[test] +fn test_base_class_with_result_ref_counts() { + assert_type_refcount_stable!(BaseClassWithResult, "base_class_with_result", "Type(10)"); +} + +#[cfg(not(Py_GIL_DISABLED))] +#[test] +fn test_subclass2_ref_counts() { + assert_type_refcount_stable!(SubClass2, "subclass2", "Type(10)"); +} + +#[cfg(not(Py_GIL_DISABLED))] +#[test] +fn test_simpleclass_ref_counts() { + assert_type_refcount_stable!(SimpleClass); } diff --git a/tests/test_sequence.rs b/tests/test_sequence.rs index 55d20353577..8dd20d45dee 100644 --- a/tests/test_sequence.rs +++ b/tests/test_sequence.rs @@ -253,6 +253,47 @@ fn test_inplace_repeat() { }); } +#[pyclass(sequence)] +struct SequenceOperators; + +#[pymethods] +impl SequenceOperators { + fn __concat__(&self, _other: &Self) -> &'static str { + "concat" + } + + fn __inplace_concat__(&self, _other: &Self) -> &'static str { + "inplace_concat" + } + + fn __repeat__(&self, _count: isize) -> &'static str { + "repeat" + } + + fn __inplace_repeat__(&self, _count: isize) -> &'static str { + "inplace_repeat" + } +} + +#[test] +fn sequence_operators_use_distinct_slots() { + Python::attach(|py| { + let d = [ + ("left", Bound::new(py, SequenceOperators).unwrap()), + ("right", Bound::new(py, SequenceOperators).unwrap()), + ] + .into_py_dict(py) + .unwrap(); + + py_assert!(py, *d, "left + right == 'concat'"); + py_run!(py, *d, "result = left; result += right"); + py_assert!(py, *d, "result == 'inplace_concat'"); + py_assert!(py, *d, "left * 2 == 'repeat'"); + py_run!(py, *d, "result = left; result *= 2"); + py_assert!(py, *d, "result == 'inplace_repeat'"); + }); +} + // Check that #[pyo3(get, set)] works correctly for Vec #[pyclass] diff --git a/tests/ui/base/Cargo.toml b/tests/ui/base/Cargo.toml index 4c84240d7c3..6658ab67c38 100644 --- a/tests/ui/base/Cargo.toml +++ b/tests/ui/base/Cargo.toml @@ -4,7 +4,7 @@ version = "0.1.0" edition = "2021" [dependencies] -pyo3 = { version = "0.29.0", default-features = false, path = "../../../" } +pyo3 = { version = "0.29.1", default-features = false, path = "../../../" } [features] macros = ["pyo3/macros"] diff --git a/tests/ui/invalid_property_args.default.stderr b/tests/ui/invalid_property_args.default.stderr index a57beafcd98..e525cebde63 100644 --- a/tests/ui/invalid_property_args.default.stderr +++ b/tests/ui/invalid_property_args.default.stderr @@ -65,14 +65,14 @@ error[E0277]: `PhantomData` cannot be converted to a Python object &'a (T0, T1, T2, T3, T4) and $N others = note: required for `PhantomData` to implement `for<'py> pyo3::impl_::pyclass::PyO3GetField<'py>` -note: required by a bound in `pyo3::impl_::pyclass::PyClassGetterGenerator::::generate` +note: required by a bound in `pyo3::impl_::pyclass::PyClassGetterGenerator::::generate` --> src/impl_/pyclass.rs | | pub const fn generate(&self, name: &'static CStr, doc: Option<&'static CStr>) -> PyMethodDefType | -------- required by a bound in this associated function ... | for<'py> FieldT: PyO3GetField<'py>, - | ^^^^^^^^^^^^^^^^^ required by this bound in `PyClassGetterGenerator::::generate` + | ^^^^^^^^^^^^^^^^^ required by this bound in `PyClassGetterGenerator::::generate` error: aborting due to 9 previous errors diff --git a/tests/ui/invalid_property_args.inspect.stderr b/tests/ui/invalid_property_args.inspect.stderr index 1cb449bf76e..919e39d8bbd 100644 --- a/tests/ui/invalid_property_args.inspect.stderr +++ b/tests/ui/invalid_property_args.inspect.stderr @@ -91,14 +91,14 @@ error[E0277]: `PhantomData` cannot be converted to a Python object &'a (T0, T1, T2, T3, T4) and $N others = note: required for `PhantomData` to implement `for<'py> pyo3::impl_::pyclass::PyO3GetField<'py>` -note: required by a bound in `pyo3::impl_::pyclass::PyClassGetterGenerator::::generate` +note: required by a bound in `pyo3::impl_::pyclass::PyClassGetterGenerator::::generate` --> src/impl_/pyclass.rs | | pub const fn generate(&self, name: &'static CStr, doc: Option<&'static CStr>) -> PyMethodDefType | -------- required by a bound in this associated function ... | for<'py> FieldT: PyO3GetField<'py>, - | ^^^^^^^^^^^^^^^^^ required by this bound in `PyClassGetterGenerator::::generate` + | ^^^^^^^^^^^^^^^^^ required by this bound in `PyClassGetterGenerator::::generate` error: aborting due to 10 previous errors diff --git a/tests/ui/not_send.rs b/tests/ui/not_send.rs index 6ff785d4635..38278464e54 100644 --- a/tests/ui/not_send.rs +++ b/tests/ui/not_send.rs @@ -1,4 +1,4 @@ -//@normalize-stderr-test: ".*/src/rust/(.*)" -> "../src/$1" + use pyo3::prelude::*; fn test_not_send_detach(py: Python<'_>) { diff --git a/tests/ui/not_send.stderr b/tests/ui/not_send.stderr index b274d66b193..7168494f88c 100644 --- a/tests/ui/not_send.stderr +++ b/tests/ui/not_send.stderr @@ -8,7 +8,7 @@ error[E0277]: `*mut pyo3::Python<'static>` cannot be shared between threads safe | = help: within `pyo3::Python<'_>`, the trait `Sync` is not implemented for `*mut pyo3::Python<'static>` note: required because it appears within the type `PhantomData<*mut pyo3::Python<'static>>` -../src/library/core/src/marker.rs:811:12 + --> $RUST_SRC/library/core/src/marker.rs:811:12 | 811 | pub struct PhantomData; | ^^^^^^^^^^^ @@ -18,7 +18,7 @@ note: required because it appears within the type `pyo3::marker::NotSend` | struct NotSend(PhantomData<*mut Python<'static>>); | ^^^^^^^ note: required because it appears within the type `PhantomData` -../src/library/core/src/marker.rs:811:12 + --> $RUST_SRC/library/core/src/marker.rs:811:12 | 811 | pub struct PhantomData; | ^^^^^^^^^^^ diff --git a/tests/ui/not_send2.rs b/tests/ui/not_send2.rs index 58382a7c207..738c7c4b2f1 100644 --- a/tests/ui/not_send2.rs +++ b/tests/ui/not_send2.rs @@ -1,4 +1,4 @@ -//@normalize-stderr-test: ".*/src/rust/(.*)" -> "../src/$1" + use pyo3::prelude::*; use pyo3::types::PyString; diff --git a/tests/ui/not_send2.stderr b/tests/ui/not_send2.stderr index 4ac694283ec..d533a258c82 100644 --- a/tests/ui/not_send2.stderr +++ b/tests/ui/not_send2.stderr @@ -12,7 +12,7 @@ error[E0277]: `*mut pyo3::Python<'static>` cannot be shared between threads safe | = help: within `pyo3::Bound<'_, PyString>`, the trait `Sync` is not implemented for `*mut pyo3::Python<'static>` note: required because it appears within the type `PhantomData<*mut pyo3::Python<'static>>` -../src/library/core/src/marker.rs:811:12 + --> $RUST_SRC/library/core/src/marker.rs:811:12 | 811 | pub struct PhantomData; | ^^^^^^^^^^^ @@ -22,7 +22,7 @@ note: required because it appears within the type `pyo3::marker::NotSend` | struct NotSend(PhantomData<*mut Python<'static>>); | ^^^^^^^ note: required because it appears within the type `PhantomData` -../src/library/core/src/marker.rs:811:12 + --> $RUST_SRC/library/core/src/marker.rs:811:12 | 811 | pub struct PhantomData; | ^^^^^^^^^^^ diff --git a/tests/ui/reject_generics.stderr b/tests/ui/reject_generics.stderr index f52bd3b5511..dda2cc0ef2d 100644 --- a/tests/ui/reject_generics.stderr +++ b/tests/ui/reject_generics.stderr @@ -1,10 +1,10 @@ -error: #[pyclass] cannot have generic parameters. For an explanation, see https://pyo3.rs/v0.29.0/class.html#no-generic-parameters +error: #[pyclass] cannot have generic parameters. For an explanation, see https://pyo3.rs/v0.29.1/class.html#no-generic-parameters --> tests/ui/reject_generics.rs:4:25 | 4 | struct ClassWithGenerics { | ^ -error: #[pyclass] cannot have lifetime parameters. For an explanation, see https://pyo3.rs/v0.29.0/class.html#no-lifetime-parameters +error: #[pyclass] cannot have lifetime parameters. For an explanation, see https://pyo3.rs/v0.29.1/class.html#no-lifetime-parameters --> tests/ui/reject_generics.rs:10:27 | 10 | struct ClassWithLifetimes<'a> { diff --git a/typos.toml b/typos.toml new file mode 100644 index 00000000000..4e44d94ac65 --- /dev/null +++ b/typos.toml @@ -0,0 +1,64 @@ +[files] +extend-exclude = [] + +[default.extend-words] +# error: `fo` should be `of`, `for`, `do`, `go`, `to` +# --> .\src\exceptions.rs:1202:30 +# | +# 1202 | let invalid_utf8 = b"fo\xd8o"; +# | ^^ +fo = "fo" +# error: `BA` should be `BY`, `BE` +# --> .\tests\test_arithmetics.rs:403:13 +# | +# 403 | // "BA" +# | ^^ +BA = "BA" +# error: `cpy` should be `copy`, `cpu` +# --> .\pyo3-build-config\src\impl_.rs:4153:52 +# | +# 4153 | let lib_name = default_lib_name_for_target(cpy313_abi3, &win_arm64); +# | ^^^ +cpy = "cpy" +# error: `tpe` should be `type` +# --> .\pytests\tests\test_pyclasses.py:91:18 +# | +#91 | for _ in tpe.map(closure, range(max_workers)): +# | ^^^ +tpe = "tpe" +# error: `ND` should be `AND` +# --> .\pyo3-ffi\src\cpython\object.rs:100:46 +# | +# 100 | pub const PyBUF_CONTIG_RO: c_int = PyBUF_ND; +# | ^^ +ND = "ND" +# error: `arro` should be `arrow` +# --> .\README.md:190:76 +# | +# 190 | - [arro3-io](https://github.com/kylebarron/arro3/tree/main/arro3-io) _`arro3-io`_ +# | ^^^^ +arro = "arro" +# error: `placehold` should be `placeholder` +# --> .\pyo3-build-config\src\impl_.rs:1942:46 +# | +# 1942 | if key == "Py_ENABLE_SHARED" and "_h_env_placehold" in build_time_vars.get("prefix"): +# | ^^^^^^^^^ +placehold = "placehold" +# error: `WRITEABLE` should be `WRITABLE` +# --> .\pyo3-ffi\src\pybuffer.rs:112:17 +# | +# 112 | pub const PyBUF_WRITEABLE: c_int = PyBUF_WRITABLE; +# | ^^^^^^^^^ +WRITEABLE = "WRITEABLE" +# error: `numer` should be `number` +# --> .\src\conversions\num_rational.rs:118:51 +# | +# 118 | get_fraction_cls(py)?.call1((self.numer().clone(), self.denom().clone())) +# | ^^^^^ +numer = "numer" +# error: `TBE` should be `THE` +# --> .\guide\src\python-typing-hints.md:34:66 +# | +# 34 | def with_traceback(self: _TBE, tb: TracebackType | None) -> _TBE: ... +# | ^^^ +TBE = "TBE" \ No newline at end of file diff --git a/uv.lock b/uv.lock index fe00358a59d..75f4e6a5ba3 100644 --- a/uv.lock +++ b/uv.lock @@ -2,23 +2,50 @@ version = 1 revision = 3 requires-python = ">=3.8" -[options] -exclude-newer = "0001-01-01T00:00:00Z" # This has no effect and is included for backwards compatibility when using relative exclude-newer values. -exclude-newer-span = "P7D" - [[package]] name = "pyo3" source = { virtual = "." } [package.dev-dependencies] dev = [ + { name = "ruff" }, { name = "rumdl" }, + { name = "typos" }, ] [package.metadata] [package.metadata.requires-dev] -dev = [{ name = "rumdl" }] +dev = [ + { name = "ruff", specifier = "<0.16" }, + { name = "rumdl" }, + { name = "typos" }, +] + +[[package]] +name = "ruff" +version = "0.15.22" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/3a/06/ae069393fc66e8ff33036d4b368003833bf6e88ccf182e17e7a2f1c754fd/ruff-0.15.22.tar.gz", hash = "sha256:3f15175b1fb580126f58285a5dae6b2ea89000136d980c64499211f116b54809", size = 4785063, upload-time = "2026-07-16T15:14:13.244Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/23/18/ee54b7ae1e121be7a28ea6da4b67564ebb0530e183a54415ab7e3bcd2c4e/ruff-0.15.22-py3-none-linux_armv6l.whl", hash = "sha256:44423e73493737f5e7c5b41d475483898ff37afcdae38bc3da5085e29af1c2d8", size = 10781258, upload-time = "2026-07-16T15:13:19.452Z" }, + { url = "https://files.pythonhosted.org/packages/2f/d2/2520cb14761ddbeaf57642a76942fc36adcbdbe53b4532241995f6fc485c/ruff-0.15.22-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:b82c6482946e9eda7ff2e091d25b8bad3f718684e1916d41bd56873cee05b697", size = 10999477, upload-time = "2026-07-16T15:13:23.318Z" }, + { url = "https://files.pythonhosted.org/packages/c9/10/74e53572aa758dfaa678c2a2646b5c5515d884b7ca56be4d2ce03ca4b560/ruff-0.15.22-py3-none-macosx_11_0_arm64.whl", hash = "sha256:11c1c715af53a09f714e011106bffc419751ec8232fcb5da42173284ea3fec6f", size = 10466716, upload-time = "2026-07-16T15:13:26.162Z" }, + { url = "https://files.pythonhosted.org/packages/1e/cc/44eaaf0844e028182f2d0a8f2190d0f359159aed0a9e5ab861d892f1ae2a/ruff-0.15.22-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:742a29cf29bddb7c8327895d6a10e0e6c5b38a96dd407af9b5d0857f809c0576", size = 10892644, upload-time = "2026-07-16T15:13:29.229Z" }, + { url = "https://files.pythonhosted.org/packages/9f/21/8edf559014d2b0f82beea19cfb713993ad802ccda16868769979c6090a84/ruff-0.15.22-py3-none-manylinux_2_17_armv7l.manylinux2014_armv7l.whl", hash = "sha256:72af58b951b0ae395935ae79763dc349bc0eb706319d28f7a33ad2cfb3cfc178", size = 10576719, upload-time = "2026-07-16T15:13:32.35Z" }, + { url = "https://files.pythonhosted.org/packages/bf/1e/3a13abd392a3b50b62e5938a831f9ab6e588358cacad5c18545b716d2182/ruff-0.15.22-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:62d425005c1835eb24e2ee4161cb90e8db263415f4a71c8c72c33abaa6c0c224", size = 11376494, upload-time = "2026-07-16T15:13:35.958Z" }, + { url = "https://files.pythonhosted.org/packages/bf/3e/422d3d95bcf04dd78e1aeac22184d4f9a8fb2c01865d39d44618484a0317/ruff-0.15.22-py3-none-manylinux_2_17_ppc64le.manylinux2014_ppc64le.whl", hash = "sha256:e8b9b3f8779a4f08c969defc3c8c35abffaa757e601ed5ae66d6d1db6519969a", size = 12208370, upload-time = "2026-07-16T15:13:39.185Z" }, + { url = "https://files.pythonhosted.org/packages/1e/91/5d065a0e0a02bf4813f5119ad278462eed081d2b832eb7c021ade0ec9e65/ruff-0.15.22-py3-none-manylinux_2_17_s390x.manylinux2014_s390x.whl", hash = "sha256:1e0dd1b2e4d3d585f897a0d137cbf4eaf6223bef4e8ce34d6bb12556c5f9249e", size = 11581098, upload-time = "2026-07-16T15:13:42.132Z" }, + { url = "https://files.pythonhosted.org/packages/f6/f9/a0d4871d12fae702eb1f41b686caf05f1f8b124dc6db6f784f53d74918fa/ruff-0.15.22-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:365523eb91d9224e1bcb03b022fbf0facb8f9e23792a2c53d9d4b3924bdbdebb", size = 11399422, upload-time = "2026-07-16T15:13:45.2Z" }, + { url = "https://files.pythonhosted.org/packages/18/80/c843a5176cddbceb0b7e8dd41cf9993490796c1c469348d384f5a5c13c56/ruff-0.15.22-py3-none-manylinux_2_31_riscv64.whl", hash = "sha256:fabfd168afdf29fee5be98b831efa9683c94d7c5a3b58b9ce5a2e38444589a74", size = 11381683, upload-time = "2026-07-16T15:13:48.46Z" }, + { url = "https://files.pythonhosted.org/packages/d4/00/8485de0ae92239438a36cfc51350db9b9e85c9ebdfaea91b18e422706662/ruff-0.15.22-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:225dbf095a87f1d9f90f5fd7924d2613ee452a75a4308c63a8f50f761787aa7c", size = 10850295, upload-time = "2026-07-16T15:13:51.655Z" }, + { url = "https://files.pythonhosted.org/packages/fa/91/24977ec2ec72eaf15e4394ace2959fdff2dd1e14f03e005e838023407169/ruff-0.15.22-py3-none-musllinux_1_2_armv7l.whl", hash = "sha256:1877d63b9d24ed278744f1523fd11b85540566d54641f97c566d7d9dc5ca5296", size = 10579640, upload-time = "2026-07-16T15:13:54.79Z" }, + { url = "https://files.pythonhosted.org/packages/9c/47/9b51216951974df1f263ac19da550d34252e0ed7218c25f10c5ef9ed7517/ruff-0.15.22-py3-none-musllinux_1_2_i686.whl", hash = "sha256:a1606c510bd7215680d32efab38965f7cdec3ef69f5170a3f4791404ffdd5262", size = 11105077, upload-time = "2026-07-16T15:13:57.915Z" }, + { url = "https://files.pythonhosted.org/packages/c2/47/20e9d4a3b8016778acea5fc32bb50d35d207500a17ddb529ffa6996feef8/ruff-0.15.22-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:630479b18625f5ffc373f77603a22a9f8ac0acd7ff0501178b5db28ec71e9c64", size = 11490980, upload-time = "2026-07-16T15:14:01.032Z" }, + { url = "https://files.pythonhosted.org/packages/4d/76/3f72d8fc38c1cb77b38c56a70da9d0c17700cc1cc50f9649c9d3c8f5ba71/ruff-0.15.22-py3-none-win32.whl", hash = "sha256:e5ba0e4a13fd14abbed2a77b517a3911290c6c6c59ef67784328d1668fab76cf", size = 10789165, upload-time = "2026-07-16T15:14:04.16Z" }, + { url = "https://files.pythonhosted.org/packages/cb/46/4965251734c2b6fcdca1b1b187d20bcac3af0ee5b083b89c910bb961ce3a/ruff-0.15.22-py3-none-win_amd64.whl", hash = "sha256:9be63ba1eb936acd2d1342fb8337c356353706fce233b2a15a09a97037e6acde", size = 11938297, upload-time = "2026-07-16T15:14:07.316Z" }, + { url = "https://files.pythonhosted.org/packages/57/c9/e69b1ff4c8b69093ef08b8919ab767af0569666865b39c30a8795d88d3c6/ruff-0.15.22-py3-none-win_arm64.whl", hash = "sha256:e1168075b72158510839f250027659cdd78476f40507dd517892304c41318661", size = 11298172, upload-time = "2026-07-16T15:14:10.51Z" }, +] [[package]] name = "rumdl" @@ -34,3 +61,20 @@ wheels = [ { url = "https://files.pythonhosted.org/packages/d0/45/bb5fa6563676d3fef2ba7ea8dacb3626ec60d241e247a372d54ef0b1aab9/rumdl-0.1.19-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:d502e60386e02aace44f2aa048954598aa859a6b8fed2a7b9ed7957adb186142", size = 5307665, upload-time = "2026-02-13T04:01:23.711Z" }, { url = "https://files.pythonhosted.org/packages/0d/7c/f0cc6d564638bb131b4ae0adcc369d0f0c7795006f98dc9b007bc98fad35/rumdl-0.1.19-py3-none-win_amd64.whl", hash = "sha256:a82901a651007e63f57f7985c5944f6b639fd77ce5ff809574564cfa2c247868", size = 5375583, upload-time = "2026-02-13T04:01:21.365Z" }, ] + +[[package]] +name = "typos" +version = "1.47.2" +source = { registry = "https://pypi.org/simple" } +sdist = { url = "https://files.pythonhosted.org/packages/77/9b/ae967bee92f0db3916350bd433f700cc0cd2942d51a22f3bfafbc97392f6/typos-1.47.2.tar.gz", hash = "sha256:d303e8c495ea870f750d8b37f2d3c3fe2441b00cf18ca5d7e0b52eca1938c7b7", size = 1829889, upload-time = "2026-06-04T01:03:07.169Z" } +wheels = [ + { url = "https://files.pythonhosted.org/packages/d7/9f/0612f6272666784ee60bee890afd517641044e5f1cda76ce22252e61b4e1/typos-1.47.2-py3-none-macosx_10_12_x86_64.whl", hash = "sha256:87df3040f9d34afd9b19a9437045fbb8838a0435eb00f047e4bac48d92f2fc44", size = 3468120, upload-time = "2026-06-04T01:02:51.404Z" }, + { url = "https://files.pythonhosted.org/packages/4c/cc/9ecd96a40cf5e8aca68cc07c0b7e4a8d8d3376b19c5774a95aeded8a935f/typos-1.47.2-py3-none-macosx_11_0_arm64.whl", hash = "sha256:287e2718a058c561baf5f55ec6b466d9270546bcb1951a2c120e594c574b9597", size = 3379520, upload-time = "2026-06-04T01:02:53.117Z" }, + { url = "https://files.pythonhosted.org/packages/d9/6e/77d23f926d6f774d6939e24f9545db702e2ee359b21e38006337539d0903/typos-1.47.2-py3-none-manylinux_2_17_aarch64.manylinux2014_aarch64.whl", hash = "sha256:3e4ef6632b280ce237caaec38d80dd3c2d956e28aa6925f80d4e915335b94a36", size = 8242414, upload-time = "2026-06-04T01:02:55.443Z" }, + { url = "https://files.pythonhosted.org/packages/a2/fd/e292e2bde1144a135949cf6d5615e571eab7f0ff2ba6cca55c074b302dd1/typos-1.47.2-py3-none-manylinux_2_17_i686.manylinux2014_i686.whl", hash = "sha256:cd7b310019943e26552809bd17f9f202b45eb0c9694f437f1708ab0868248ced", size = 7327674, upload-time = "2026-06-04T01:02:57.243Z" }, + { url = "https://files.pythonhosted.org/packages/a7/58/b7a3f4df5ff9ddde7c9e42d29b4c0569662af5dbad3573eedc167cd8d1e1/typos-1.47.2-py3-none-manylinux_2_17_x86_64.manylinux2014_x86_64.whl", hash = "sha256:f525edf9b67d3ede552bb70bd4171f23e5e8edec3187189dfe8d1676df630b44", size = 7747368, upload-time = "2026-06-04T01:02:59.031Z" }, + { url = "https://files.pythonhosted.org/packages/78/18/758682974e36f5b77eb592807fd748ce1c4010509b4255f867433aa3e44f/typos-1.47.2-py3-none-musllinux_1_2_aarch64.whl", hash = "sha256:6c36a97ab3dd8c8924cd9b907a32e9aac504fc779d0c3b05e19204ca93385c37", size = 7097338, upload-time = "2026-06-04T01:03:00.932Z" }, + { url = "https://files.pythonhosted.org/packages/9d/b3/e6bccecd186d8f8c2d592baa7e1b17ef4f02a1e0a5ce4d7f53e9693fd3d9/typos-1.47.2-py3-none-musllinux_1_2_x86_64.whl", hash = "sha256:4eb36a44daed1d719ce417d2a6dd7a323d814ccdc647d9bb20d17ac2bed9e38c", size = 8136631, upload-time = "2026-06-04T01:03:02.744Z" }, + { url = "https://files.pythonhosted.org/packages/50/90/cceb1be7159dd020c0c9a93371b6444b8bdad7bce55bc7fa119e49d2c264/typos-1.47.2-py3-none-win32.whl", hash = "sha256:d0c01034bc029d8883406f3e2bed46dfa9b090ce6ad4a99e580070ae51307cfa", size = 3139942, upload-time = "2026-06-04T01:03:04.163Z" }, + { url = "https://files.pythonhosted.org/packages/82/0c/7c44ffabf6020d2fc456912bdcfb0d68102e05775fbc5cf82b88979be2ea/typos-1.47.2-py3-none-win_amd64.whl", hash = "sha256:749bbba363067bfc0e54ccc6e7580750e17f5ef093c91fedf6c2eb27d32efee6", size = 3311629, upload-time = "2026-06-04T01:03:05.718Z" }, +] diff --git a/wasm/common.mk b/wasm/common.mk new file mode 100644 index 00000000000..8c0ce9848ce --- /dev/null +++ b/wasm/common.mk @@ -0,0 +1,44 @@ +# Shared logic to download Python source tarball for the wasm builds. + +CURDIR=$(abspath .) + +# These three are passed in from nox. +BUILDROOT ?= $(CURDIR)/builddir +PYTHON ?= python3 +PYMAJORMINORMICRO ?= $(shell $(PYTHON) --version 2>&1 | awk '{print $$2}') + +# Set version variables. +version_tuple := $(subst ., ,$(PYMAJORMINORMICRO:v%=%)) +PYMAJOR=$(word 1,$(version_tuple)) +PYMINOR=$(word 2,$(version_tuple)) +PYMICRO=$(word 3,$(version_tuple)) +PYVERSION=$(PYMAJORMINORMICRO) +PYMAJORMINOR=$(PYMAJOR).$(PYMINOR) + +ifneq ($(PYMAJORMINOR),3.14) +$(error PYMAJORMINOR must be 3.14, got '$(PYMAJORMINOR)') +endif + +PYTHONURL=https://www.python.org/ftp/python/$(PYMAJORMINORMICRO)/Python-$(PYVERSION).tgz +PYTHONTARBALL=$(BUILDROOT)/downloads/Python-$(PYVERSION).tgz +PYTHONBUILD=$(BUILDROOT)/build/Python-$(PYVERSION) + +.DEFAULT_GOAL := all + +$(BUILDROOT)/.exists: + mkdir -p $(BUILDROOT) + touch $@ + +$(PYTHONTARBALL): $(BUILDROOT)/.exists + mkdir -p $(BUILDROOT)/downloads + curl -sL $(PYTHONURL) -o $@ + +$(PYTHONBUILD)/.exists: $(PYTHONTARBALL) + [ -d $(PYTHONBUILD) ] || ( \ + mkdir -p $(dir $(PYTHONBUILD));\ + tar -C $(dir $(PYTHONBUILD)) -xf $(PYTHONTARBALL) \ + ) + touch $@ + +clean: + rm -rf $(BUILDROOT) diff --git a/emscripten/.gitignore b/wasm/emscripten/.gitignore similarity index 100% rename from emscripten/.gitignore rename to wasm/emscripten/.gitignore diff --git a/wasm/emscripten/Makefile b/wasm/emscripten/Makefile new file mode 100644 index 00000000000..e0c36435234 --- /dev/null +++ b/wasm/emscripten/Makefile @@ -0,0 +1,58 @@ +include ../common.mk + +NODE_VERSION=24.18.0 + +PLATFORM=wasm32_emscripten +SYSCONFIGDATA_NAME=_sysconfigdata__$(PLATFORM) + +export EMSDKDIR = $(PYTHONBUILD)/emsdk-cache + +PYTHONLIBDIR=$(BUILDROOT)/install/Python-$(PYVERSION)/lib +CROSS_PYTHON=$(PYTHONBUILD)/cross-build/wasm32-emscripten/build/python/python.sh + +# Download Node.js when building locally; in CI it is provided by actions/setup-node. +ifdef GITHUB_ACTIONS +NODE_DEP= +NODE_BIN=$(shell which node) +else +NODE_DIR=$(BUILDROOT)/node +NODE_DEP=$(NODE_DIR)/.exists +NODE_BIN=$(NODE_DIR)/bin/node +NODE_OS := $(shell uname -s | tr '[:upper:]' '[:lower:]') +NODE_ARCH := $(shell uname -m | sed -e 's/x86_64/x64/' -e 's/aarch64/arm64/') +NODE_URL=https://nodejs.org/dist/v$(NODE_VERSION)/node-v$(NODE_VERSION)-$(NODE_OS)-$(NODE_ARCH).tar.xz +endif + +# Prepend the downloaded Node.js to PATH (no-op in CI, where setup-node provides it). +export PATH := $(NODE_BIN)$(PATH) + +all: $(PYTHONLIBDIR)/libpython$(PYMAJORMINOR).a + +$(NODE_DIR)/.exists: $(BUILDROOT)/.exists + [ -d $(NODE_DIR) ] || mkdir -p $(NODE_DIR) + curl -s -S --location $(NODE_URL) | \ + tar --strip-components 1 --directory $(NODE_DIR) --extract --xz + touch $@ + +$(CROSS_PYTHON): $(PYTHONBUILD)/.exists $(NODE_DEP) + cd $(PYTHONBUILD) && \ + $(PYTHON) Platforms/emscripten install-emscripten --emsdk-cache=$(EMSDKDIR) && \ + $(PYTHON) Platforms/emscripten build --emsdk-cache=$(EMSDKDIR) --host-runner=$(NODE_BIN) + +$(PYTHONLIBDIR)/libpython$(PYMAJORMINOR).a: $(CROSS_PYTHON) + # Generate sysconfigdata + _PYTHON_SYSCONFIGDATA_NAME=$(SYSCONFIGDATA_NAME) _PYTHON_PROJECT_BASE=$(PYTHONBUILD)/cross-build/wasm32-emscripten/build/python $(CROSS_PYTHON) -m sysconfig --generate-posix-vars + cp `cat pybuilddir.txt`/$(SYSCONFIGDATA_NAME).py $(PYTHONBUILD)/Lib + + mkdir -p $(PYTHONLIBDIR) + # Make a static library for _hacl, for some reason these are missing from the build? + # source emsdk_env.sh to get emar in PATH, works best when done from the emsdk directory + EMSDK_ENV=$$(find $(PYTHONBUILD)/emsdk-cache -name 'emsdk_env.sh' | head -n 1) && \ + cd $$(dirname $$EMSDK_ENV) && \ + . $$EMSDK_ENV && \ + cd $(PYTHONBUILD)/cross-build/wasm32-emscripten/build/python && \ + emar rcs Modules/_hacl/libhacl.a Modules/_hacl/*.o + # Copy all .a libraries + find $(PYTHONBUILD)/cross-build/wasm32-emscripten/ -name '*.a' -exec cp {} $(PYTHONLIBDIR) \; + # Install Python stdlib + cp -r $(PYTHONBUILD)/Lib $(PYTHONLIBDIR)/python$(PYMAJORMINOR) diff --git a/emscripten/runner.py b/wasm/emscripten/runner.py similarity index 100% rename from emscripten/runner.py rename to wasm/emscripten/runner.py diff --git a/wasm/wasi/.gitignore b/wasm/wasi/.gitignore new file mode 100644 index 00000000000..a57078d9742 --- /dev/null +++ b/wasm/wasi/.gitignore @@ -0,0 +1 @@ +builddir diff --git a/wasm/wasi/Makefile b/wasm/wasi/Makefile new file mode 100644 index 00000000000..4d5bb241d5d --- /dev/null +++ b/wasm/wasi/Makefile @@ -0,0 +1,67 @@ +include ../common.mk + +WASI_SDK_VERSION=24 +WASMTIME_VERSION=46.0.1 + +CONFIG_SITE=$(PYTHONBUILD)/Tools/wasm/wasi/config.site-wasm32-wasi +CROSSBUILD=$(PYTHONBUILD)/cross-build/wasm32-wasip1 +LIBDIR=$(CROSSBUILD)/build/lib.wasi-wasm32-$(PYMAJORMINOR) + +# In CI the WASI SDK is installed by bytecodealliance/setup-wasi-sdk-action, which sets +# WASI_SDK_PATH; otherwise download the release asset matching this host into the build dir. +ifdef GITHUB_ACTIONS +WASI_SDK_DIR=$(WASI_SDK_PATH) +WASI_SDK_DEP= +else +WASI_SDK_DIR=$(BUILDROOT)/wasi-sdk +WASI_SDK_DEP=$(WASI_SDK_DIR)/.exists +WASI_SDK_OS := $(shell uname -s | tr '[:upper:]' '[:lower:]' | sed 's/darwin/macos/') +WASI_SDK_ARCH := $(shell uname -m | sed 's/aarch64/arm64/') +WASI_SDK_URL=https://github.com/WebAssembly/wasi-sdk/releases/download/wasi-sdk-$(WASI_SDK_VERSION)/wasi-sdk-$(WASI_SDK_VERSION).0-$(WASI_SDK_ARCH)-$(WASI_SDK_OS).tar.gz +endif + +# Similar for wasmtime, which is otherwise expected on PATH. +ifdef GITHUB_ACTIONS +WASMTIME_DEP= +WASMTIME_PATH= +else +WASMTIME_DIR=$(BUILDROOT)/wasmtime +WASMTIME_DEP=$(WASMTIME_DIR)/.exists +WASMTIME_PATH=$(WASMTIME_DIR): +WASMTIME_OS := $(shell uname -s | tr '[:upper:]' '[:lower:]' | sed 's/darwin/macos/') +# NB wasmtime names artifacts with aarch64 architecture, unlike wasi which uses arm64 +WASMTIME_ARCH := $(shell uname -m | sed 's/arm64/aarch64/') +WASMTIME_URL=https://github.com/bytecodealliance/wasmtime/releases/download/v$(WASMTIME_VERSION)/wasmtime-v$(WASMTIME_VERSION)-$(WASMTIME_ARCH)-$(WASMTIME_OS).tar.xz +endif + +all: $(LIBDIR)/libpython$(PYMAJORMINOR).a + +$(WASI_SDK_DIR)/.exists: $(BUILDROOT)/.exists + [ -d $(WASI_SDK_DIR) ] || mkdir -p $(WASI_SDK_DIR) + curl -s -S --location $(WASI_SDK_URL) | \ + tar --strip-components 1 --directory $(WASI_SDK_DIR) --extract --gunzip + touch $@ + +$(WASMTIME_DIR)/.exists: $(BUILDROOT)/.exists + [ -d $(WASMTIME_DIR) ] || mkdir -p $(WASMTIME_DIR) + curl -s -S --location $(WASMTIME_URL) | \ + tar --strip-components 1 --directory $(WASMTIME_DIR) --extract --xz + touch $@ + +$(PYTHONBUILD)/.patched: $(PYTHONBUILD)/.exists + # Force-disable POSIX dynamic loading for WASI + grep -q '^ac_cv_func_dlopen=no' $(CONFIG_SITE) || \ + printf '\nac_cv_func_dlopen=no\nac_cv_lib_dl_dlopen=no\n' >> $(CONFIG_SITE) + touch $@ + +$(LIBDIR)/libpython$(PYMAJORMINOR).a: $(PYTHONBUILD)/.patched $(WASI_SDK_DEP) $(WASMTIME_DEP) + cd $(PYTHONBUILD) && \ + WASI_SDK_PATH=$(WASI_SDK_DIR) \ + PATH=$(WASMTIME_PATH)$(PATH) \ + $(PYTHON) Tools/wasm/wasi build -- --config-cache + # Collect the static libraries the test build links against + cp $(CROSSBUILD)/libpython$(PYMAJORMINOR).a \ + $(CROSSBUILD)/Modules/_hacl/libHacl_HMAC.a \ + $(CROSSBUILD)/Modules/_decimal/libmpdec/libmpdec.a \ + $(CROSSBUILD)/Modules/expat/libexpat.a \ + $(LIBDIR)