diff --git a/.github/workflows/build-and-push-docker-image.yml b/.github/workflows/build-and-push-docker-image.yml new file mode 100644 index 0000000..271de1e --- /dev/null +++ b/.github/workflows/build-and-push-docker-image.yml @@ -0,0 +1,58 @@ +--- + +name: ๐Ÿ—๏ธ + +on: # yamllint disable-line rule:truthy + pull_request: + push: + branches: ["release/*", "unstable/*"] + workflow_dispatch: + inputs: + tag: + description: Docker image tag + required: true + type: string + +jobs: + smoke-test: + uses: ./.github/workflows/reusable-smoke-test.yml + build-and-push: + if: github.event_name != 'pull_request' + runs-on: ubuntu-latest + needs: + - smoke-test + timeout-minutes: 10 + steps: + - uses: actions/checkout@v4 + - name: Build Docker image + run: | + DOCKER_TAG="${DOCKER_TAG/'/'/'-'}" + DOCKER_TAG_MAJOR=$(echo "$DOCKER_TAG" | cut -d '.' -f 1) + DOCKER_TAG_MAJOR_MINOR=$(echo "$DOCKER_TAG" | cut -d '.' -f 1-2) + IMAGE="ghcr.io/$GITHUB_REPOSITORY:${DOCKER_TAG}" + IMAGE_MAJOR="ghcr.io/$GITHUB_REPOSITORY:${DOCKER_TAG_MAJOR}" + IMAGE_MAJOR_MINOR="ghcr.io/$GITHUB_REPOSITORY:${DOCKER_TAG_MAJOR_MINOR}" + IMAGE_SHA="ghcr.io/$GITHUB_REPOSITORY:${GITHUB_SHA}" + echo "IMAGE=$IMAGE" >>"$GITHUB_ENV" + echo "IMAGE_MAJOR=$IMAGE_MAJOR" >>"$GITHUB_ENV" + echo "IMAGE_MAJOR_MINOR=$IMAGE_MAJOR_MINOR" >>"$GITHUB_ENV" + echo "IMAGE_SHA=$IMAGE_SHA" >>"$GITHUB_ENV" + docker build . \ + --build-arg BUILDKIT_INLINE_CACHE=1 \ + --cache-from $IMAGE \ + --tag $IMAGE + docker tag $IMAGE $IMAGE_MAJOR + docker tag $IMAGE $IMAGE_MAJOR_MINOR + docker tag $IMAGE $IMAGE_SHA + env: + DOCKER_TAG: ${{ inputs.tag || github.ref_name }} + - name: Log in to GHCR + run: >- + echo ${{ secrets.GITHUB_TOKEN }} | + docker login ghcr.io -u $GITHUB_ACTOR --password-stdin + - name: Push Docker image to GHCR + run: | + docker push $IMAGE + docker push $IMAGE_MAJOR + docker push $IMAGE_MAJOR_MINOR + docker push $IMAGE_SHA diff --git a/.github/workflows/self-smoke-test-action.yml b/.github/workflows/reusable-smoke-test.yml similarity index 65% rename from .github/workflows/self-smoke-test-action.yml rename to .github/workflows/reusable-smoke-test.yml index b655019..1b59efa 100644 --- a/.github/workflows/self-smoke-test-action.yml +++ b/.github/workflows/reusable-smoke-test.yml @@ -1,10 +1,9 @@ --- -name: ๐Ÿงช +name: โ™ป๏ธ ๐Ÿงช on: # yamllint disable-line rule:truthy - push: - pull_request: + workflow_call: env: devpi-password: abcd1234 @@ -27,7 +26,33 @@ env: PYTEST_THEME_MODE jobs: + fail-fast: + + strategy: + matrix: + os: [macos-latest, windows-latest] + + runs-on: ${{ matrix.os }} + + timeout-minutes: 2 + + steps: + - name: Check out the action locally + uses: actions/checkout@v4 + with: + path: test + - name: Fail-fast in unsupported environments + continue-on-error: true + id: fail-fast + uses: ./test + - name: Error if action did not fail-fast in unsupported environments + if: steps.fail-fast.outcome == 'success' + run: | + >&2 echo This action should fail-fast in unsupported environments. + exit 1 + smoke-test: + runs-on: ubuntu-latest services: @@ -71,8 +96,31 @@ jobs: readme = "README.md" - name: Build the stub package sdist and wheel distributions run: python3 -m build + - name: Create the Rust package directory + run: mkdir -pv rust-example + - name: Initialize a Rust project + run: cargo init + working-directory: rust-example + - name: Populate the Rust package `pyproject.toml` + run: echo "$CONTENTS" > pyproject.toml + env: + CONTENTS: | + [build-system] + requires = [ + "maturin ~=1.0", + ] + build-backend = "maturin" + working-directory: rust-example + - name: Build the stub package sdist and wheel distributions + run: python3 -m build -o ../dist/ + working-directory: rust-example - name: Register the stub package in devpi - run: twine register dist/*.tar.gz + run: | + for dist in dist/*.tar.gz + do + echo "Registering ${dist}..." + twine register "${dist}" + done env: TWINE_USERNAME: ${{ env.devpi-username }} TWINE_PASSWORD: ${{ env.devpi-password }} diff --git a/README.md b/README.md index c0998c5..8ddf0d7 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,10 @@ walkthrough check out the [PyPA guide]. If you have any feedback regarding specific action versions, please leave comments in the corresponding [per-release announcement discussions]. +> [!TIP] +> A limited number of usage scenarios is supported, including the +> [PyPA guide] example. See the [non-goals] for more detail. + ## ๐ŸŒ‡ `master` branch sunset โ— @@ -111,16 +115,17 @@ filter to the job: > Generating and uploading digital attestations currently requires > authentication with a [trusted publisher]. -You can generate signed [digital attestations] for all the distribution files and -upload them all together by enabling the `attestations` setting: +Generating signed [digital attestations] for all the distribution files +and uploading them all together is now on by default for all projects +using Trusted Publishing. To disable it, set `attestations` as follows: ```yml with: - attestations: true + attestations: false ``` -This will use [Sigstore] to create attestation -objects for each distribution package, signing them with the identity provided +The attestation objects are created using [Sigstore] for each +distribution package, signing them with the identity provided by the GitHub's OIDC token associated with the current workflow. This means both the trusted publishing authentication and the attestations are tied to the same identity. @@ -130,6 +135,9 @@ same identity. This GitHub Action [has nothing to do with _building package distributions_]. Users are responsible for preparing dists for upload by putting them into the `dist/` folder prior to running this Action. +They are typically expected to do this in a _separate GitHub Actions +CI/CD job_ running before the one where they call this action and having +restricted privileges. > [!IMPORTANT] > Since this GitHub Action is docker-based, it can only @@ -154,6 +162,72 @@ by putting them into the `dist/` folder prior to running this Action. > sharing the built dists across stages and jobs. Then, use the `needs` > setting to order the build, test and publish stages. +The expected environment for running `pypi-publish` is the +GitHub-provided Ubuntu VM. We are running a smoke-test against +`ubuntu-latest` in CI but any currently available numbered versions +should do. We'll consider them supported for as long as GitHub itself +supports them. + +Running the action in a job that has a `container:` set is not +supported. It might work for you but you're on your own when it breaks. +If you feel the need to use it, it's likely that you're not following +the recommendation of invoking the build automation in a separate job, +which is considered a security issue (especially, when using [Trusted +Publishing][trusted publisher] that may cause privilege escalation and +would enable the attackers to impersonate the GitHub-backed identity of +the repository through transitive build dependency poisoning). The +solution is to have one job (or multiple, in case of projects with +C-extensions) for building the distribution packages, followed by +another that publishes them. + +Self-hosted runners are best effort, provided no other unsupported +things influence them. We are unable to test this in CI and they may +break. This is often the case when using custom runtimes and not the +official GitHub-provided VMs. In general, if you follow the +recommendation of building in a separate job, you shouldn't need to run +this action within a self-hosted runner โ€” it should be possible to +build your dists in a self-hosted runner, save them as a GitHub Actions +artifact in that job, and then invoke the publishing job that would run +within GitHub-provided runners, downloading the artifact with the dists +and publishing them. Such separation is the _recommended_/**supported** +way of handling this scenario. +Our understandng is that Trusted publishing is expected to work on +self-hosted runners. It is backed by OIDC. If it doesn't work, you +should probably ask GitHub if you missed something. We wouldn't be able +to assist here. + +Trusted Publishing cannot be tested in CI at the moment, sadly. It is +supported and bugs should be reported but it may take time to sort out +as it often requires cross-project collaboration to debug (sometimes, +problems occur due to changes in PyPI and not in the action). + +The only case that is explicitly unsupported at the moment is [Trusted +Publishing][trusted publisher] in reusable workflows. This requires +support on the PyPI side and is being worked on. Please, do not report +bugs related to this case. The current recommendation is to put +everything else you want into a reusable workflow but keep the job +calling `pypi-publish` in a top-level one. + +Invoking `pypi-publish` from composite actions is unsupported. It is not +tested. GitHub Runners have limitations and bugs in this case. But more +importantly, this is usually an indication of using it insecurely. When +using [Trusted Publishing][trusted publisher], it is imperative to keep +build machinery invocation in a separate job with restrictive privileges +as [Trusted Publishing][trusted publisher] itself requires elevated +permissions to make use of OIDC. Our observation is that the users +sometimes create in-project composite actions that invoke building and +publishing in the same job. As such, we don't seek to support such a +dangerous configuration in the first place. The solution is pretty much +the same as with the previous problem โ€” use a separate job with +dedicated and scoped privileges just for publishing; and invoke that +in-project composite action from a different job. + +And finally, invoking `pypi-publish` more than once in the same job is +not considered supported. It may work in a limited number of scenarios +but please, don't do this. If you want to publish to several indexes, +build the dists in one job and add several publishing jobs, one per +upload. + ## Advanced release management @@ -278,7 +352,7 @@ are released under the [BSD 3-clause license](LICENSE.md). [๐Ÿงช GitHub Actions CI/CD workflow tests badge]: -https://github.com/pypa/gh-action-pypi-publish/actions/workflows/self-smoke-test-action.yml/badge.svg?branch=unstable%2Fv1&event=push +https://github.com/pypa/gh-action-pypi-publish/actions/workflows/build-and-push-docker-image.yml/badge.svg?branch=unstable%2Fv1&event=push [GHA workflow runs list]: https://github.com/pypa/gh-action-pypi-publish/actions/workflows/self-smoke-test-action.yml?query=branch%3Aunstable%2Fv1 @@ -293,6 +367,8 @@ https://julienrenaux.fr/2019/12/20/github-actions-security-risk/ [per-release announcement discussions]: https://github.com/pypa/gh-action-pypi-publish/discussions/categories/announcements +[non-goals]: #Non-goals + [Creating & using secrets]: https://help.github.com/en/actions/automating-your-workflow-with-github-actions/creating-and-using-encrypted-secrets [has nothing to do with _building package distributions_]: diff --git a/action.yml b/action.yml index 40fed97..3e39fd1 100644 --- a/action.yml +++ b/action.yml @@ -86,20 +86,95 @@ inputs: Enable experimental support for PEP 740 attestations. Only works with PyPI and TestPyPI via Trusted Publishing. required: false - default: 'false' + default: 'true' branding: color: yellow icon: upload-cloud runs: - using: docker - image: Dockerfile - args: - - ${{ inputs.user }} - - ${{ inputs.password }} - - ${{ inputs.repository-url }} - - ${{ inputs.packages-dir }} - - ${{ inputs.verify-metadata }} - - ${{ inputs.skip-existing }} - - ${{ inputs.verbose }} - - ${{ inputs.print-hash }} - - ${{ inputs.attestations }} + using: composite + steps: + - name: Fail-fast in unsupported environments + if: runner.os != 'Linux' + run: | + >&2 echo This action is only able to run under GNU/Linux environments + exit 1 + shell: bash -eEuo pipefail {0} + - name: Reset path if needed + run: | + # Reset path if needed + # https://github.com/pypa/gh-action-pypi-publish/issues/112 + if [[ $PATH != *"/usr/bin"* ]]; then + echo "\$PATH=$PATH. Resetting \$PATH for GitHub Actions." + PATH="/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin" + echo "PATH=$PATH" >>"$GITHUB_ENV" + echo "$PATH" >>"$GITHUB_PATH" + echo "\$PATH reset. \$PATH=$PATH" + fi + shell: bash + - name: Set repo and ref from which to run Docker container action + id: set-repo-and-ref + run: | + # Set repo and ref from which to run Docker container action + # to handle cases in which `github.action_` context is not set + # https://github.com/actions/runner/issues/2473 + REF=${{ env.ACTION_REF || env.PR_REF || github.ref_name }} + REPO=${{ env.ACTION_REPO || env.PR_REPO || github.repository }} + REPO_ID=${{ env.PR_REPO_ID || github.repository_id }} + echo "ref=$REF" >>"$GITHUB_OUTPUT" + echo "repo=$REPO" >>"$GITHUB_OUTPUT" + echo "repo-id=$REPO_ID" >>"$GITHUB_OUTPUT" + shell: bash + env: + ACTION_REF: ${{ github.action_ref }} + ACTION_REPO: ${{ github.action_repository }} + PR_REF: ${{ github.event.pull_request.head.ref }} + PR_REPO: ${{ github.event.pull_request.head.repo.full_name }} + PR_REPO_ID: ${{ github.event.pull_request.base.repo.id }} + - name: Discover pre-installed Python + id: pre-installed-python + run: | + # ๐Ÿ”Ž Discover pre-installed Python + echo "python-path=$(command -v python3 || :)" | tee -a "${GITHUB_OUTPUT}" + shell: bash + - name: Install Python 3 + if: steps.pre-installed-python.outputs.python-path == '' + id: new-python + uses: actions/setup-python@v5 + with: + python-version: 3.x + - name: Create Docker container action + run: | + # Create Docker container action + ${{ + steps.pre-installed-python.outputs.python-path == '' + && steps.new-python.outputs.python-path + || steps.pre-installed-python.outputs.python-path + }} '${{ github.action_path }}/create-docker-action.py' + env: + REF: ${{ steps.set-repo-and-ref.outputs.ref }} + REPO: ${{ steps.set-repo-and-ref.outputs.repo }} + REPO_ID: ${{ steps.set-repo-and-ref.outputs.repo-id }} + shell: bash + - name: Run Docker container + # The generated trampoline action must exist in the allowlisted + # runner-defined working directory so it can be referenced by the + # relative path starting with `./`. + # + # This mutates the end-user's workspace slightly but uses a path + # that is unlikely to clash with somebody else's use. + # + # We cannot use randomized paths because the composite action + # syntax does not allow accessing variables in `uses:`. This + # means that we end up having to hardcode this path both here and + # in `create-docker-action.py`. + uses: ./.github/.tmp/.generated-actions/run-pypi-publish-in-docker-container + with: + user: ${{ inputs.user }} + password: ${{ inputs.password }} + repository-url: ${{ inputs.repository-url || inputs.repository_url }} + packages-dir: ${{ inputs.packages-dir || inputs.packages_dir }} + verify-metadata: ${{ inputs.verify-metadata || inputs.verify_metadata }} + skip-existing: ${{ inputs.skip-existing || inputs.skip_existing }} + verbose: ${{ inputs.verbose }} + print-hash: ${{ inputs.print-hash || inputs.print_hash }} + attestations: ${{ inputs.attestations }} diff --git a/attestations.py b/attestations.py index d8bca49..8c66cff 100644 --- a/attestations.py +++ b/attestations.py @@ -54,6 +54,7 @@ def debug(msg: str): def collect_dists(packages_dir: Path) -> list[Path]: # Collect all sdists and wheels. dist_paths = [sdist.resolve() for sdist in packages_dir.glob('*.tar.gz')] + dist_paths.extend(sdist.resolve() for sdist in packages_dir.glob('*.zip')) dist_paths.extend(whl.resolve() for whl in packages_dir.glob('*.whl')) # Make sure everything that looks like a dist actually is one. diff --git a/create-docker-action.py b/create-docker-action.py new file mode 100644 index 0000000..6829a9f --- /dev/null +++ b/create-docker-action.py @@ -0,0 +1,91 @@ +import json +import os +import pathlib + +DESCRIPTION = 'description' +REQUIRED = 'required' + +REF = os.environ['REF'] +REPO = os.environ['REPO'] +REPO_ID = os.environ['REPO_ID'] +REPO_ID_GH_ACTION = '178055147' + +ACTION_SHELL_CHECKOUT_PATH = pathlib.Path(__file__).parent.resolve() + + +def set_image(ref: str, repo: str, repo_id: str) -> str: + if repo_id == REPO_ID_GH_ACTION: + return str(ACTION_SHELL_CHECKOUT_PATH / 'Dockerfile') + docker_ref = ref.replace('/', '-') + return f'docker://ghcr.io/{repo}:{docker_ref}' + + +image = set_image(REF, REPO, REPO_ID) + +action = { + 'name': '๐Ÿƒ', + DESCRIPTION: ( + 'Run Docker container to upload Python distribution packages to PyPI' + ), + 'inputs': { + 'user': {DESCRIPTION: 'PyPI user', REQUIRED: False}, + 'password': { + DESCRIPTION: 'Password for your PyPI user or an access token', + REQUIRED: False, + }, + 'repository-url': { + DESCRIPTION: 'The repository URL to use', + REQUIRED: False, + }, + 'packages-dir': { + DESCRIPTION: 'The target directory for distribution', + REQUIRED: False, + }, + 'verify-metadata': { + DESCRIPTION: 'Check metadata before uploading', + REQUIRED: False, + }, + 'skip-existing': { + DESCRIPTION: ( + 'Do not fail if a Python package distribution' + ' exists in the target package index' + ), + REQUIRED: False, + }, + 'verbose': {DESCRIPTION: 'Show verbose output.', REQUIRED: False}, + 'print-hash': { + DESCRIPTION: 'Show hash values of files to be uploaded', + REQUIRED: False, + }, + 'attestations': { + DESCRIPTION: ( + '[EXPERIMENTAL]' + ' Enable experimental support for PEP 740 attestations.' + ' Only works with PyPI and TestPyPI via Trusted Publishing.' + ), + REQUIRED: False, + }, + }, + 'runs': { + 'using': 'docker', + 'image': image, + }, +} + +# The generated trampoline action must exist in the allowlisted +# runner-defined working directory so it can be referenced by the +# relative path starting with `./`. +# +# This mutates the end-user's workspace slightly but uses a path +# that is unlikely to clash with somebody else's use. +# +# We cannot use randomized paths because the composite action +# syntax does not allow accessing variables in `uses:`. This +# means that we end up having to hardcode this path both here and +# in `action.yml`. +action_path = pathlib.Path( + '.github/.tmp/.generated-actions/' + 'run-pypi-publish-in-docker-container/action.yml', +) +action_path.parent.mkdir(parents=True, exist_ok=True) +action_path.write_text(json.dumps(action, ensure_ascii=False), encoding='utf-8') diff --git a/oidc-exchange.py b/oidc-exchange.py index 0ada77b..227b9c4 100644 --- a/oidc-exchange.py +++ b/oidc-exchange.py @@ -83,6 +83,7 @@ * `repository`: `{repository}` * `repository_owner`: `{repository_owner}` * `repository_owner_id`: `{repository_owner_id}` +* `workflow_ref`: `{workflow_ref}` * `job_workflow_ref`: `{job_workflow_ref}` * `ref`: `{ref}` @@ -175,6 +176,7 @@ def _get(name: str) -> str: # noqa: WPS430 repository=_get('repository'), repository_owner=_get('repository_owner'), repository_owner_id=_get('repository_owner_id'), + workflow_ref=_get('workflow_ref'), job_workflow_ref=_get('job_workflow_ref'), ref=_get('ref'), ) diff --git a/requirements/runtime-constraints.in b/requirements/runtime-constraints.in new file mode 100644 index 0000000..e4afedd --- /dev/null +++ b/requirements/runtime-constraints.in @@ -0,0 +1,20 @@ +############################################################################### +# # +# This file is only meant to exclude broken dependency versions, not feature # +# dependencies. # +# # +# GUIDELINES: # +# 1. Only list PyPI project versions that need to be excluded using `!=` # +# and `<`. # +# 2. It is allowed to have transitive dependency limitations in this file. # +# 3. Apply bare minimum constraints under narrow conditions, use # +# environment markers if possible. E.g. `; python_version < "3.12"`. # +# 4. Whenever there are no constraints, let the file and this header # +# remain in Git. # +# # +############################################################################### + +# NOTE: 1.12.0 and later enable support for metadata 2.4 +# NOTE: This can be dropped once twine stops using pkginfo +# Ref: https://github.com/pypa/twine/pull/1180 +pkginfo >= 1.12.0 diff --git a/requirements/runtime.in b/requirements/runtime.in index 3758e3a..5861e0b 100644 --- a/requirements/runtime.in +++ b/requirements/runtime.in @@ -1,4 +1,7 @@ -twine +-c runtime-constraints.in # limits known broken versions + +# NOTE: v6 is needed to support metadata v2.4 +twine >= 6.0 # NOTE: Used to detect an ambient OIDC credential for OIDC publishing, # NOTE: as well as PEP 740 attestations. @@ -10,8 +13,8 @@ id ~= 1.0 requests # NOTE: Used to generate attestations. -pypi-attestations ~= 0.0.12 -sigstore ~= 3.2.0 +pypi-attestations ~= 0.0.15 +sigstore ~= 3.5.1 # NOTE: Used to detect the PyPI package name from the distribution files packaging diff --git a/requirements/runtime.txt b/requirements/runtime.txt index 5ff03bb..ac8be5d 100644 --- a/requirements/runtime.txt +++ b/requirements/runtime.txt @@ -18,6 +18,7 @@ cryptography==42.0.7 # via # pyopenssl # pypi-attestations + # secretstorage # sigstore dnspython==2.6.1 # via email-validator @@ -41,14 +42,16 @@ idna==3.7 # via # email-validator # requests -importlib-metadata==7.1.0 - # via twine jaraco-classes==3.4.0 # via keyring jaraco-context==5.3.0 # via keyring jaraco-functools==4.0.1 # via keyring +jeepney==0.8.0 + # via + # keyring + # secretstorage keyring==25.2.1 # via twine markdown-it-py==3.0.0 @@ -67,12 +70,17 @@ packaging==24.1 # via # -r runtime.in # pypi-attestations -pkginfo==1.10.0 - # via twine + # twine +pkginfo==1.12.0 + # via + # -c runtime-constraints.in + # twine platformdirs==4.2.2 # via sigstore pyasn1==0.6.0 - # via sigstore + # via + # pypi-attestations + # sigstore pycparser==2.22 # via cffi pydantic==2.7.1 @@ -91,7 +99,7 @@ pyjwt==2.8.0 # via sigstore pyopenssl==24.1.0 # via sigstore -pypi-attestations==0.0.12 +pypi-attestations==0.0.15 # via -r runtime.in python-dateutil==2.9.0.post0 # via betterproto @@ -115,9 +123,11 @@ rich==13.7.1 # via # sigstore # twine +secretstorage==3.3.3 + # via keyring securesystemslib==1.0.0 # via tuf -sigstore==3.2.0 +sigstore==3.5.1 # via # -r runtime.in # pypi-attestations @@ -131,7 +141,7 @@ six==1.16.0 # via python-dateutil tuf==5.0.0 # via sigstore -twine==5.1.1 +twine==6.0.1 # via -r runtime.in typing-extensions==4.11.0 # via @@ -141,5 +151,3 @@ urllib3==2.2.1 # via # requests # twine -zipp==3.18.2 - # via importlib-metadata