Release Python #300
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Release Python | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| # Latest commit to include with the release. If omitted, use the latest commit on the main branch. | |
| sha: | |
| description: Commit SHA | |
| type: string | |
| # Create the sdist and build the wheels, but do not publish to PyPI / GitHub. | |
| dry-run: | |
| description: Dry run | |
| type: boolean | |
| default: false | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| PYTHON_VERSION: '3.9' | |
| PYTHON_VERSION_WIN_ARM64: '3.11' # ARM64 Windows doesn't have older versions | |
| CARGO_INCREMENTAL: 0 | |
| CARGO_NET_RETRY: 10 | |
| RUSTUP_MAX_RETRIES: 10 | |
| COMPAT_TUNE_CPU: '' | |
| COMPAT_FEATURES: '+sse3,+ssse3,+sse4.1,+sse4.2,+popcnt,+cmpxchg16b' | |
| COMPAT_CC_FEATURES: '-msse3 -mssse3 -msse4.1 -msse4.2 -mpopcnt -mcx16' | |
| NONCOMPAT_TUNE_CPU: 'skylake' | |
| NONCOMPAT_FEATURES: '+sse3,+ssse3,+sse4.1,+sse4.2,+popcnt,+cmpxchg16b,+avx,+avx2,+fma,+bmi1,+bmi2,+lzcnt,+pclmulqdq,+movbe' | |
| NONCOMPAT_CC_FEATURES: '-msse3 -mssse3 -msse4.1 -msse4.2 -mpopcnt -mcx16 -mavx -mavx2 -mfma -mbmi -mbmi2 -mlzcnt -mpclmul -mmovbe' | |
| defaults: | |
| run: | |
| shell: bash | |
| jobs: | |
| base-package: | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| steps: | |
| - name: Checkout | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ inputs.sha }} | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Install yq | |
| run: pip install yq | |
| - name: Check runtime versions | |
| env: | |
| POLARS_PYPROJECT_TOML: py-polars/pyproject.toml | |
| POLARS_CARGO_TOML: py-polars/runtime/Cargo.toml | |
| run: | | |
| set -e | |
| VERSION=$(tomlq '.project.version' "$POLARS_PYPROJECT_TOML" | tr -d '"') | |
| CARGO_VERSION=$(tomlq '.package.version' "$POLARS_CARGO_TOML" | tr -d '"') | |
| RT32_VERSION=($(tomlq '.project.dependencies[0]' "$POLARS_PYPROJECT_TOML" | tr -d '"')) | |
| RT64_VERSION=($(tomlq '.project."optional-dependencies".rt64[0]' "$POLARS_PYPROJECT_TOML" | tr -d '"')) | |
| COMPAT_VERSION=($(tomlq '.project."optional-dependencies".rtcompat[0]' "$POLARS_PYPROJECT_TOML" | tr -d '"')) | |
| [[ "$VERSION" == "$(echo $CARGO_VERSION | sed 's/-beta\./b/')" ]] || (echo "Incorrect cargo version" && exit 1) | |
| [[ "$VERSION" == "${RT32_VERSION[2]}" ]] || (echo "Incorrect rt32 version" && exit 1) | |
| [[ "$VERSION" == "${RT64_VERSION[2]}" ]] || (echo "Incorrect rt64 version" && exit 1) | |
| [[ "$VERSION" == "${COMPAT_VERSION[2]}" ]] || (echo "Incorrect rtcompat version" && exit 1) | |
| - name: Install uv | |
| run: pip install uv | |
| - name: Build sdist and wheel | |
| run: | | |
| uv build py-polars --out-dir dist | |
| - name: Upload base package | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| path: dist | |
| create-sdist: | |
| needs: [base-package] | |
| runs-on: ubuntu-latest | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| package: [polars-runtime-32, polars-runtime-64, polars-runtime-compat] | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ inputs.sha }} | |
| - name: Download base package | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: dist | |
| merge-multiple: true | |
| # Avoid potential out-of-memory errors | |
| - name: Set swap space for Linux | |
| uses: pierotofy/set-swap-space@master | |
| with: | |
| swap-size-gb: 10 | |
| - name: Set up Python | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Install yq | |
| run: | | |
| pip install --force-reinstall --no-dependencies dist/polars-*.whl | |
| rm -rf dist/* | |
| pip install yq | |
| - name: Rename target directory | |
| if: matrix.package != 'polars-runtime-64' | |
| run: | | |
| SNAKE_CASE="$(echo '${{ matrix.package }}' | sed 's/-/_/g')" | |
| mv py-polars/runtime/_polars_runtime_64 py-polars/runtime/_$SNAKE_CASE | |
| - name: Update runtime package and module names | |
| run: | | |
| cp rust-toolchain.toml py-polars/runtime | |
| SNAKE_CASE="$(echo '${{ matrix.package }}' | sed 's/-/_/g')" | |
| tomlq -i -t ".project.name = \"${{ matrix.package }}\"" py-polars/runtime/pyproject.toml | |
| tomlq -i -t ".lib.name = \"_$SNAKE_CASE\"" py-polars/runtime/Cargo.toml | |
| sed -i "s/pub fn _polars_runtime_64/pub fn _$SNAKE_CASE/" crates/polars-python/src/c_api/mod.rs | |
| - name: Set runtime repr in plr | |
| run: | | |
| sed -i "s/^pub static RUNTIME_REPR: \&str = \"unknown\";$/pub static RUNTIME_REPR: \&str = \"rt$(echo '${{ matrix.package }}' | cut -d '-' -f 3)\";/g" crates/polars-python/src/c_api/mod.rs | |
| - name: Update 64-bit runtime to bigidx | |
| if: matrix.package == 'polars-runtime-64' | |
| run: | | |
| tomlq -i -t '.dependencies.polars.features += ["bigidx"]' py-polars/runtime/Cargo.toml | |
| - name: Create source distribution | |
| uses: PyO3/maturin-action@v1 | |
| with: | |
| command: sdist | |
| args: > | |
| --manifest-path py-polars/runtime/Cargo.toml | |
| --out dist | |
| maturin-version: 1.8.3 | |
| - name: Test sdist | |
| run: | | |
| pip install --force-reinstall --verbose dist/*.tar.gz | |
| python -c 'import polars; polars.show_versions()' | |
| - name: Upload sdist | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: sdist-${{ matrix.package }} | |
| path: dist/*.tar.gz | |
| build-wheels: | |
| needs: [base-package] | |
| runs-on: ${{ matrix.os }} | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| package: [polars-runtime-32, polars-runtime-64, polars-runtime-compat] | |
| # macos-13 is x86-64 | |
| # macos-15 is aarch64 | |
| os: [ubuntu-latest, macos-13, macos-15, windows-latest, windows-11-arm] | |
| architecture: [x86-64, aarch64] | |
| exclude: | |
| - os: windows-latest | |
| architecture: aarch64 | |
| - os: windows-11-arm | |
| architecture: x86-64 | |
| - os: macos-15 | |
| architecture: x86-64 | |
| - os: macos-13 | |
| architecture: aarch64 | |
| env: | |
| SED_INPLACE: ${{ startsWith(matrix.os, 'macos') && '-i ''''' || '-i' }} | |
| steps: | |
| - name: Setup build environment (ARM64 Windows) | |
| if: matrix.os == 'windows-11-arm' | |
| shell: | |
| powershell | |
| # Notes | |
| # * We update `Expand-Archive` to avoid "" is not a supported archive file format when extracting | |
| # files that don't end in `.zip` | |
| run: | | |
| Write-Output "> Update Expand-Archive (Microsoft.PowerShell.Archive)" | |
| Install-PackageProvider -Name NuGet -Force | |
| Install-Module -Name Microsoft.PowerShell.Archive -Force | |
| Write-Output "> Setup bash.exe (git-for-windows/PortableGit)" | |
| Invoke-WebRequest "https://github.com/git-for-windows/git/releases/download/v2.47.1.windows.1/PortableGit-2.47.1-arm64.7z.exe" -OutFile /git.7z.exe | |
| /git.7z.exe -o/git -y | |
| Write-Output "> Setup Rust" | |
| Invoke-WebRequest "https://static.rust-lang.org/rustup/dist/aarch64-pc-windows-msvc/rustup-init.exe" -OutFile /rustup-init.exe | |
| /rustup-init.exe --default-host aarch64-pc-windows-msvc -y | |
| Write-Output "> Setup VS Build Tools" | |
| Invoke-WebRequest "https://aka.ms/vs/17/release/vs_BuildTools.exe" -OutFile /vs_BuildTools.exe | |
| Start-Process C:/vs_BuildTools.exe -ArgumentList " ` | |
| --add Microsoft.VisualStudio.Workload.NativeDesktop ` | |
| --add Microsoft.VisualStudio.Workload.VCTools ` | |
| --add Microsoft.VisualStudio.Component.VC.Tools.arm64 ` | |
| --add Microsoft.VisualStudio.Component.VC.Llvm.Clang ` | |
| --add Microsoft.VisualStudio.Component.VC.Llvm.ClangToolset ` | |
| --includeRecommended --quiet --norestart --wait" -Wait | |
| Write-Output "> Setup CMake" | |
| Invoke-WebRequest "https://github.com/Kitware/CMake/releases/download/v3.31.2/cmake-3.31.2-windows-arm64.zip" -OutFile /cmake.zip | |
| Expand-Archive /cmake.zip -DestinationPath / | |
| Write-Output "> Download jq.exe (github.com/jqlang) (needed for tomlq / yq)" | |
| Invoke-WebRequest https://github.com/jqlang/jq/releases/download/jq-1.7.1/jq-windows-i386.exe -OutFile /jq.exe | |
| Write-Output "> Update GITHUB_PATH" | |
| [System.IO.File]::AppendAllText($Env:GITHUB_PATH, "`n" + "C:/git/bin/") | |
| [System.IO.File]::AppendAllText($Env:GITHUB_PATH, "`n" + $Env:USERPROFILE + "/.cargo/bin/") | |
| [System.IO.File]::AppendAllText($Env:GITHUB_PATH, "`n" + "C:/Program Files (x86)/Microsoft Visual Studio/2022/BuildTools/VC/Tools/Llvm/bin/") | |
| [System.IO.File]::AppendAllText($Env:GITHUB_PATH, "`n" + "C:/cmake-3.31.2-windows-arm64/bin") | |
| [System.IO.File]::AppendAllText($Env:GITHUB_PATH, "`n" + "C:/") | |
| [System.IO.File]::AppendAllText($Env:GITHUB_PATH, "`n") | |
| Get-Content $Env:GITHUB_PATH | Out-Host | |
| - name: Check build environment (ARM64 Windows) | |
| if: matrix.os == 'windows-11-arm' | |
| run: | | |
| set -x | |
| bash --version | |
| rustup show | |
| clang -v | |
| cmake --version | |
| - uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ inputs.sha }} | |
| - name: Download base package | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: dist | |
| merge-multiple: true | |
| # Avoid potential out-of-memory errors | |
| - name: Set swap space for Linux | |
| if: matrix.os == 'ubuntu-latest' | |
| uses: pierotofy/set-swap-space@master | |
| with: | |
| swap-size-gb: 10 | |
| - name: Set up Python | |
| if: matrix.os != 'windows-11-arm' | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION }} | |
| - name: Set up Python (ARM64 Windows) | |
| if: matrix.os == 'windows-11-arm' | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: ${{ env.PYTHON_VERSION_WIN_ARM64 }} | |
| # Otherwise can't find `tomlq` after `pip install yq` | |
| - name: Add Python scripts folder to GITHUB_PATH (ARM64 Windows) | |
| if: matrix.os == 'windows-11-arm' | |
| run: | | |
| python -c "import sysconfig; print(sysconfig.get_path('scripts'))" >> $GITHUB_PATH | |
| - name: Install yq | |
| run: | | |
| pip install --force-reinstall --no-dependencies dist/polars-*.whl | |
| rm -rf dist/* | |
| pip install yq | |
| - name: Rename target directory | |
| if: matrix.package != 'polars-runtime-64' | |
| run: | | |
| SNAKE_CASE="$(echo '${{ matrix.package }}' | sed 's/-/_/g')" | |
| mv py-polars/runtime/_polars_runtime_64 py-polars/runtime/_$SNAKE_CASE | |
| - name: Update runtime package and module names | |
| run: | | |
| SNAKE_CASE="$(echo '${{ matrix.package }}' | sed 's/-/_/g')" | |
| tomlq -i -t ".project.name = \"${{ matrix.package }}\"" py-polars/runtime/pyproject.toml | |
| tomlq -i -t ".lib.name = \"_$SNAKE_CASE\"" py-polars/runtime/Cargo.toml | |
| sed $SED_INPLACE "s/pub fn _polars_runtime_64/pub fn _$SNAKE_CASE/" crates/polars-python/src/c_api/mod.rs | |
| - name: Set runtime repr in plr | |
| run: | | |
| sed $SED_INPLACE "s/^pub static RUNTIME_REPR: \&str = \"unknown\";$/pub static RUNTIME_REPR: \&str = \"rt$(echo '${{ matrix.package }}' | cut -d '-' -f 3)\";/g" crates/polars-python/src/c_api/mod.rs | |
| - name: Update 64-bit runtime to bigidx | |
| if: matrix.package == 'polars-runtime-64' | |
| run: | | |
| tomlq -i -t '.dependencies.polars.features += ["bigidx"]' py-polars/runtime/Cargo.toml | |
| - name: Determine CPU features for x86-64 | |
| id: features | |
| if: matrix.architecture == 'x86-64' | |
| env: | |
| IS_RT_COMPAT: ${{ matrix.package == 'polars-runtime-compat' }} | |
| # IMPORTANT: All features enabled here should also be included in py-polars/polars/_cpu_check.py | |
| run: | | |
| if [[ "$IS_RT_COMPAT" = true ]]; then | |
| TUNE_CPU="$COMPAT_TUNE_CPU" | |
| FEATURES="$COMPAT_FEATURES" | |
| CC_FEATURES="$COMPAT_CC_FEATURES" | |
| else | |
| TUNE_CPU="$NONCOMPAT_TUNE_CPU" | |
| FEATURES="$NONCOMPAT_FEATURES" | |
| CC_FEATURES="$NONCOMPAT_CC_FEATURES" | |
| fi | |
| echo "features=$FEATURES" >> $GITHUB_OUTPUT | |
| echo "tune_cpu=$TUNE_CPU" >> $GITHUB_OUTPUT | |
| echo "cc_features=$CC_FEATURES" >> $GITHUB_OUTPUT | |
| - name: Set RUSTFLAGS for x86-64 | |
| if: matrix.architecture == 'x86-64' | |
| env: | |
| FEATURES: ${{ steps.features.outputs.features }} | |
| TUNE_CPU: ${{ steps.features.outputs.tune_cpu }} | |
| CC_FEATURES: ${{ steps.features.outputs.cc_features }} | |
| CFG: ${{ matrix.package == 'polars-runtime-compat' && '--cfg allocator="default"' || '' }} | |
| run: | | |
| if [[ -z "$TUNE_CPU" ]]; then | |
| echo "RUSTFLAGS=-C target-feature=$FEATURES $CFG" >> $GITHUB_ENV | |
| echo "CFLAGS=$CC_FEATURES" >> $GITHUB_ENV | |
| else | |
| echo "RUSTFLAGS=-C target-feature=$FEATURES -Z tune-cpu=$TUNE_CPU $CFG" >> $GITHUB_ENV | |
| echo "CFLAGS=$CC_FEATURES -mtune=$TUNE_CPU" >> $GITHUB_ENV | |
| fi | |
| - name: Set Rust target for aarch64 | |
| if: matrix.architecture == 'aarch64' | |
| id: target | |
| run: | | |
| TARGET=$( | |
| if [[ "${{ matrix.os }}" == "macos-15" ]]; then | |
| echo "aarch64-apple-darwin"; | |
| elif [[ "${{ matrix.os }}" == "windows-11-arm" ]]; then | |
| echo "aarch64-pc-windows-msvc"; | |
| else | |
| echo "aarch64-unknown-linux-gnu"; | |
| fi | |
| ) | |
| echo "target=$TARGET" >> $GITHUB_OUTPUT | |
| - name: Set jemalloc for aarch64 Linux | |
| if: matrix.architecture == 'aarch64' && matrix.os == 'ubuntu-latest' | |
| run: | | |
| echo "JEMALLOC_SYS_WITH_LG_PAGE=16" >> $GITHUB_ENV | |
| - name: Copy toolchain to py-polars/ (ARM64 Windows) | |
| # Manual fix for: | |
| # TomlError: Unknown character "46" at row 1, col 2, pos 1: | |
| # 1> ../rust-toolchain.toml | |
| if: matrix.os == 'windows-11-arm' | |
| run: cp rust-toolchain.toml py-polars/runtime | |
| - name: Build wheel | |
| uses: PyO3/maturin-action@v1 | |
| with: | |
| command: build | |
| target: ${{ steps.target.outputs.target }} | |
| args: > | |
| --profile dist-release | |
| --manifest-path py-polars/runtime/Cargo.toml | |
| --out dist | |
| manylinux: ${{ matrix.architecture == 'aarch64' && '2_24' || 'auto' }} | |
| maturin-version: 1.8.3 | |
| - name: Test wheel | |
| # For linux; only test on x86-64 for now as this matches the runner architecture | |
| if: matrix.architecture == 'x86-64' || startsWith(matrix.os, 'macos') || startsWith(matrix.os, 'windows') | |
| run: | | |
| pip install --force-reinstall --verbose dist/*.whl | |
| python -c 'import polars; polars.show_versions()' | |
| - name: Upload wheel | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: wheel-${{ matrix.package }}-${{ matrix.os }}-${{ matrix.architecture }} | |
| path: dist/*.whl | |
| publish-to-pypi: | |
| needs: [base-package, create-sdist, build-wheels] | |
| environment: | |
| name: release-python | |
| url: https://pypi.org/project/polars | |
| runs-on: ubuntu-latest | |
| permissions: | |
| id-token: write | |
| steps: | |
| - name: Download sdists and wheels | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: dist | |
| merge-multiple: true | |
| - name: Remove Emscripten wheel | |
| run: rm -f dist/*emscripten*.whl | |
| - name: Publish to PyPI | |
| if: inputs.dry-run == false | |
| uses: pypa/gh-action-pypi-publish@release/v1 | |
| with: | |
| verbose: true | |
| publish-to-github: | |
| needs: [publish-to-pypi] | |
| runs-on: ubuntu-latest | |
| steps: | |
| - uses: actions/checkout@v4 | |
| with: | |
| ref: ${{ inputs.sha }} | |
| - name: Get version from Cargo.toml | |
| id: version | |
| run: | | |
| VERSION=$(grep -m 1 -oP 'version = "\K[^"]+' py-polars/runtime/Cargo.toml) | |
| if [[ "$VERSION" == *"-"* ]]; then | |
| IS_PRERELEASE=true | |
| else | |
| IS_PRERELEASE=false | |
| fi | |
| echo "version=$VERSION" >> $GITHUB_OUTPUT | |
| echo "is_prerelease=$IS_PRERELEASE" >> $GITHUB_OUTPUT | |
| - name: Create GitHub release | |
| id: github-release | |
| uses: release-drafter/release-drafter@v6 | |
| with: | |
| config-name: release-drafter-python.yml | |
| name: Python Polars ${{ steps.version.outputs.version }} | |
| tag: py-${{ steps.version.outputs.version }} | |
| version: ${{ steps.version.outputs.version }} | |
| prerelease: ${{ steps.version.outputs.is_prerelease }} | |
| commitish: ${{ inputs.sha || github.sha }} | |
| disable-autolabeler: true | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Publish GitHub release | |
| if: inputs.dry-run == false | |
| run: gh release edit $TAG --draft=false | |
| env: | |
| GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| TAG: ${{ steps.github-release.outputs.tag_name }} | |
| - name: Trigger other workflows related to the release | |
| if: inputs.dry-run == false | |
| uses: peter-evans/repository-dispatch@v3 | |
| with: | |
| event-type: python-release | |
| client-payload: > | |
| { | |
| "version": "${{ steps.version.outputs.version }}", | |
| "is_prerelease": "${{ steps.version.outputs.is_prerelease }}", | |
| "tag": "${{ steps.github-release.outputs.tag_name }}", | |
| "sha": "${{ inputs.sha || github.sha }}" | |
| } |