#76: publish releases to PyPI, with a TestPyPI rehearsal

.github/scripts/check-index-digests.py

@@ -0,0 +1,144 @@
+#!/usr/bin/env python3
+"""Check that a package index serves exactly the files listed in a SHA256SUMS file, byte for byte.
+
+release.yml runs this after each upload (PyPI on a tag, TestPyPI on an opted-in rehearsal), with the
+SHA256SUMS the build job generated and signed.  It is what makes "the PyPI files are the signed GitHub
+Release files" a checked statement rather than an assumption (#76; plan/tickets/76/plan.md, D5).
+
+The version is read from the wheel's filename in SHA256SUMS, so tag builds and rehearsal (.devN) builds
+are handled alike.  The index's JSON API (`<index>/pypi/<project>/<version>/json`) is polled until every
+expected file is listed or the timeout expires, since a fresh upload can take a few minutes to appear
+through the CDN.  Then it fails on any expected file missing from the index, any file on the index for
+that version that SHA256SUMS does not list, and any digest mismatch, printing both sides.
+
+Files skipped by the upload (`skip-existing: true`, plan D3a) are compared like any other: a re-run
+that skips a file already uploaded passes, and a different build at a version already on the index
+fails here, loudly, rather than at upload time.
+
+Each run starts with a self-test of the comparison, so a rule that stops matching fails instead of
+passing everything.
+"""
+
+import argparse
+import json
+import sys
+import time
+import urllib.error
+import urllib.request
+from pathlib import Path
+
+
+def read_sums(path: Path) -> dict[str, str]:
+    """Parse `sha256sum` output into {filename: hex digest}."""
+    sums: dict[str, str] = {}
+    for line in path.read_text().splitlines():
+        if not line.strip():
+            continue
+        digest, name = line.split(maxsplit=1)
+        sums[name.lstrip("*")] = digest.lower()
+    if not sums:
+        raise ValueError(f"{path} lists no files")
+    return sums
+
+
+def wheel_version(sums: dict[str, str]) -> str:
+    """The version from the one wheel's filename (`<name>-<version>-<tags>.whl`)."""
+    wheels = [name for name in sums if name.endswith(".whl")]
+    if len(wheels) != 1:
+        raise ValueError(f"expected exactly one wheel in SHA256SUMS, found {wheels}")
+    return wheels[0].split("-")[1]
+
+
+def compare(expected: dict[str, str], served: dict[str, str]) -> list[str]:
+    """Every difference between the signed files and the index's files, as error messages."""
+    errors = []
+    for name in sorted(expected.keys() - served.keys()):
+        errors.append(f"missing from the index: {name} (SHA256SUMS: {expected[name]})")
+    for name in sorted(served.keys() - expected.keys()):
+        errors.append(f"on the index but not in SHA256SUMS: {name} (index: {served[name]})")
+    for name in sorted(expected.keys() & served.keys()):
+        if expected[name] != served[name]:
+            errors.append(f"digest mismatch: {name} (SHA256SUMS: {expected[name]}, index: {served[name]})")
+    return errors
+
+
+def fetch_served(url: str) -> dict[str, str] | None:
+    """{filename: sha256} for the release at `url`, or None if the index does not have it yet."""
+    request = urllib.request.Request(url, headers={"Accept": "application/json", "Cache-Control": "no-cache"})
+    try:
+        with urllib.request.urlopen(request, timeout=30) as response:
+            data = json.load(response)
+    except urllib.error.HTTPError as e:
+        if e.code == 404:
+            return None
+        raise
+    return {f["filename"]: f["digests"]["sha256"].lower() for f in data["urls"]}
+
+
+def self_test() -> list[str]:
+    a, b = "a" * 64, "b" * 64
+    failures = []
+    cases = [
+        ("identical", {"x.whl": a, "x.tar.gz": b}, {"x.whl": a, "x.tar.gz": b}, 0),
+        ("missing", {"x.whl": a, "x.tar.gz": b}, {"x.whl": a}, 1),
+        ("extra", {"x.whl": a}, {"x.whl": a, "x.tar.gz": b}, 1),
+        ("mismatch", {"x.whl": a}, {"x.whl": b}, 1),
+        ("all three", {"x.whl": a, "y.whl": a}, {"x.whl": b, "z.whl": a}, 3),
+    ]
+    for label, expected, served, count in cases:
+        got = len(compare(expected, served))
+        if got != count:
+            failures.append(f"compare() self-test '{label}': expected {count} error(s), got {got}")
+    sums = {"dp_python_lib-1.17.0.dev3-py3-none-any.whl": a, "dp_python_lib-1.17.0.dev3.tar.gz": b}
+    if wheel_version(sums) != "1.17.0.dev3":
+        failures.append("wheel_version() self-test: did not read 1.17.0.dev3")
+    return failures
+
+
+def main() -> int:
+    parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
+    parser.add_argument("sums", type=Path, help="the SHA256SUMS file the build job signed")
+    parser.add_argument("--index-url", required=True, help="e.g. https://pypi.org or https://test.pypi.org")
+    parser.add_argument("--project", required=True, help="the project name on the index")
+    parser.add_argument("--timeout", type=int, default=300, help="seconds to wait for the files to appear")
+    parser.add_argument("--interval", type=int, default=15, help="seconds between polls")
+    args = parser.parse_args()
+
+    failures = self_test()
+    if failures:
+        for failure in failures:
+            print(f"::error::self-test: {failure}")
+        return 1
+
+    expected = read_sums(args.sums)
+    version = wheel_version(expected)
+    url = f"{args.index_url.rstrip('/')}/pypi/{args.project}/{version}/json"
+    print(f"Checking {url} against {args.sums}:")
+    for name, digest in sorted(expected.items()):
+        print(f"  {digest}  {name}")
+
+    deadline = time.monotonic() + args.timeout
+    while True:
+        served = fetch_served(url)
+        if served is not None and expected.keys() <= served.keys():
+            break
+        if time.monotonic() >= deadline:
+            break
+        state = "not found" if served is None else f"lists {sorted(served)}"
+        print(f"Index {state}; retrying in {args.interval}s")
+        time.sleep(args.interval)
+
+    if served is None:
+        print(f"::error::{url} still returns 404 after {args.timeout}s.")
+        return 1
+    errors = compare(expected, served)
+    for error in errors:
+        print(f"::error::{error}")
+    if errors:
+        return 1
+    print(f"The index serves exactly the {len(expected)} files in {args.sums}, byte for byte.")
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

.github/workflows/release.yml

@@ -5,10 +5,17 @@ on:
     tags:
       - "rel-*"
   # Manual dispatch is always a rehearsal: it builds, verifies, and signs, but never
-  # publishes.  Publishing is gated on a `rel-` tag push and nothing else, so there is no
-  # path to cutting a release from an arbitrary ref.  Choose the ref to rehearse against
-  # with the branch/tag selector in the Actions UI.
+  # publishes to PyPI or a GitHub Release.  Both are gated on a `rel-` tag push and nothing
+  # else, so there is no path to cutting a release from an arbitrary ref.  Choose the ref to
+  # rehearse against with the branch/tag selector in the Actions UI.  The one thing a
+  # dispatch can publish is the rehearsal build to test.pypi.org, and only when the
+  # `testpypi` input is set (#76).
   workflow_dispatch:
+    inputs:
+      testpypi:
+        description: "Also upload the rehearsal build to test.pypi.org (a sandbox; versions there cannot be re-uploaded either)"
+        type: boolean
+        default: false
 
 # A publish must never be cancelled halfway through, so unlike CI this does not set
 # cancel-in-progress.
@@ -98,6 +105,14 @@ jobs:
 
       - name: Build wheel and sdist
         run: |
+          set -euo pipefail
+          # A rehearsal builds an untagged commit, which setuptools-scm versions with a local
+          # segment (1.16.1.dev61+g496f0e0).  PyPI and TestPyPI both reject local versions, so
+          # a dispatch drops it (1.16.1.dev61) to stay uploadable to TestPyPI.  A tag build has
+          # no local segment and is left alone.
+          if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then
+            export SETUPTOOLS_SCM_OVERRIDES_FOR_DP_PYTHON_LIB='{local_scheme="no-local-version"}'
+          fi
           python -m pip install --upgrade pip build twine
           python -m build
 
@@ -168,7 +183,11 @@ jobs:
         with:
           name: release-dist
           path: dist/
-          retention-days: 7
+          # GitHub holds a deployment awaiting approval for up to 30 days, and publish-pypi
+          # needs these exact files: a full re-run would rebuild and re-sign, and the new files
+          # would not match the SHA256SUMS and bundles already on the GitHub Release.  So the
+          # artifact lives as long as the approval can wait (#76, plan D2).
+          retention-days: 30
 
   publish-github-release:
     name: Publish GitHub Release
@@ -229,32 +248,43 @@ jobs:
           echo "Published body matches doc/release-notes/${TAG}.md."
 
   # ---------------------------------------------------------------------------------------
-  # PyPI publishing -- WIRED UP BUT INTENTIONALLY DISABLED.
+  # PyPI publishing (#76; plan/tickets/76/plan.md).
+  #
+  # Trusted Publishing (OIDC): no API token is stored anywhere.  The publisher on pypi.org
+  # names this repository, workflow release.yml, and environment `pypi`; the environment
+  # requires an approval and admits only rel-* tags.  The environment must exist BEFORE any
+  # job names it: GitHub creates a missing one on the fly, with no protection rules.
   #
-  # To enable:
-  #   1. Claim the project name on PyPI.
-  #   2. Configure a Trusted Publisher for osprey-dcs/dp-python-lib, workflow release.yml,
-  #      environment `pypi` (PyPI project settings -> Publishing).  Trusted Publishing uses
-  #      OIDC, so no API token is ever stored in repo secrets.
-  #   3. Create a GitHub environment named `pypi`, ideally with required reviewers.
-  #   4. Change the `if:` below to:
-  #        if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/rel-')
+  # publish-pypi runs last, after the GitHub Release is published and its body verified,
+  # because a PyPI file can never be replaced.  It uploads the same files the build job
+  # signed, then checks that the index serves them byte for byte (check-index-digests.py).
   #
-  # Note: PyPI rejects Sigstore bundles as uploads, hence the cleanup step -- the bundles
-  # still go to the GitHub Release.  attestations: true emits PEP 740 attestations instead.
+  # PyPI rejects Sigstore bundles as uploads, hence the cleanup step -- the bundles still go
+  # to the GitHub Release.  attestations: true emits PEP 740 attestations instead.
+  #
+  # publish-testpypi is the rehearsal of the same path, run by dispatch with testpypi=true,
+  # through its own publisher and an unprotected `testpypi` environment.  The two jobs'
+  # steps are duplicated on purpose; keep them in step.
   # ---------------------------------------------------------------------------------------
   publish-pypi:
     name: Publish to PyPI
-    needs: build
+    needs: [build, publish-github-release]
     runs-on: ubuntu-latest
-    if: false # <-- flip this to enable; see the comment block above
+    if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/rel-')
     environment:
       name: pypi
       url: https://pypi.org/p/dp-python-lib
     permissions:
+      contents: read
       id-token: write
 
     steps:
+      - name: Check out the digest checker
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          sparse-checkout: .github/scripts
+          persist-credentials: false
+
       - name: Download build outputs
         uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
         with:
@@ -263,10 +293,64 @@ jobs:
 
       - name: Remove non-distribution files
         # Only sdists and wheels may be uploaded; the checksums file, the release notes, and
-        # the Sigstore bundles would all be rejected.
-        run: rm -f dist/SHA256SUMS dist/RELEASE_NOTES.md dist/*.sigstore.json
+        # the Sigstore bundles would all be rejected.  SHA256SUMS is kept aside for the check.
+        run: |
+          mv dist/SHA256SUMS SHA256SUMS
+          rm -f dist/RELEASE_NOTES.md dist/*.sigstore.json
 
       - name: Publish to PyPI
         uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2
         with:
           attestations: true
+          # Files go up one at a time, so a wheel can land and the sdist fail; without this,
+          # every re-run would then fail on the wheel.  A re-run uploads the same artifact, and
+          # the digest check below compares skipped files too, so a real conflict still fails
+          # (plan D3a).
+          skip-existing: true
+
+      - name: Verify PyPI serves the signed files
+        run: python3 .github/scripts/check-index-digests.py SHA256SUMS --index-url https://pypi.org --project dp-python-lib
+
+  publish-testpypi:
+    name: Publish rehearsal to TestPyPI
+    needs: build
+    runs-on: ubuntu-latest
+    if: github.event_name == 'workflow_dispatch' && inputs.testpypi
+    environment:
+      name: testpypi
+      url: https://test.pypi.org/p/dp-python-lib
+    permissions:
+      contents: read
+      id-token: write
+
+    steps:
+      - name: Check out the digest checker
+        uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
+        with:
+          sparse-checkout: .github/scripts
+          persist-credentials: false
+
+      - name: Download build outputs
+        uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1
+        with:
+          name: release-dist
+          path: dist
+
+      - name: Remove non-distribution files
+        # As in publish-pypi.  A dispatch stages no RELEASE_NOTES.md; rm -f covers that.
+        run: |
+          mv dist/SHA256SUMS SHA256SUMS
+          rm -f dist/RELEASE_NOTES.md dist/*.sigstore.json
+
+      - name: Publish to TestPyPI
+        uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2
+        with:
+          repository-url: https://test.pypi.org/legacy/
+          attestations: true
+          # As in publish-pypi.  A .devN version counts commits since the last tag, so two
+          # different commits can share one; the second rehearsal's upload is then skipped,
+          # and the digest check below fails on the mismatch, as it should.
+          skip-existing: true
+
+      - name: Verify TestPyPI serves the signed files
+        run: python3 .github/scripts/check-index-digests.py SHA256SUMS --index-url https://test.pypi.org --project dp-python-lib

CLAUDE.md

@@ -95,8 +95,20 @@ GitHub Actions workflows live in `.github/workflows/`:
   signs everything with keyless Sigstore, and publishes a GitHub Release.  A
   `workflow_dispatch` trigger allows rehearsing the whole path without cutting a
   tag: publishing is gated on a `rel-` tag push, so a manual run always stops after
-  build/verify/sign.  A PyPI publish job is wired up but disabled (`if: false`); the
-  comment block above it lists the steps to enable it.
+  build/verify/sign.  On a tag, `publish-pypi` then uploads the same signed files to PyPI
+  (#76; `plan/tickets/76/plan.md`): Trusted Publishing (OIDC, no token), in the `pypi`
+  environment, which requires an approval and admits only `rel-*` tags.  It runs **after**
+  `publish-github-release`, because a PyPI file can never be replaced, sets `skip-existing`
+  so a partial upload can be finished by "Re-run failed jobs", and ends with
+  `.github/scripts/check-index-digests.py`, which fails unless the index serves exactly the
+  files in `SHA256SUMS`, byte for byte (skipped files included, so a real conflict is still
+  loud).  The `release-dist` artifact is kept 30 days, GitHub's approval window, because a
+  full re-run rebuilds and re-signs files that no longer match the release.  A dispatch with
+  the `testpypi` input set rehearses the same path against test.pypi.org (`publish-testpypi`,
+  an unprotected `testpypi` environment); a dispatch build drops setuptools-scm's local
+  version segment (`+g<sha>`), which both indexes reject, so a rehearsal is `X.Y.Z.devN`.
+  **A job that names a missing environment creates it, with no protection rules**: `pypi`
+  must exist, with its reviewer and tag rule, before any workflow change that names it merges.
 
 **Action pinning**: every `uses:` reference in both workflows is pinned to a full commit
 SHA with a trailing `# vX.Y.Z` comment naming the version — a tag is mutable, so whoever
@@ -113,7 +125,10 @@ grep -rnE 'uses: *[^ ]+@' .github/workflows/ | grep -vE '@[0-9a-f]{40} # v'   #
 ```
 
 **Cutting a release**: the version comes from the git tag alone (setuptools-scm),
-so there is no version to bump in a file.  Tag `rel-X.Y.Z` and push the tag.
+so there is no version to bump in a file.  Tag `rel-X.Y.Z` and push the tag, then approve
+the `pypi` deployment once the GitHub Release job is green, and confirm the digest check
+passed.  The approver also pushes the tag, so the environment's "prevent self-review" stays
+off: the approval is a deliberate pause before an irreversible upload, not a second reviewer.
 The tag must be exactly `rel-X.Y.Z` with no suffix — prerelease shapes like
 `rel-1.15.0-rc1` are rejected up front, because setuptools-scm would normalize them
 (`1.15.0rc1`) and fail the tag-vs-built version assertion with a confusing error.
@@ -720,7 +735,7 @@ for page in q.iter_query_samples(params):      # raises RuntimeError on a page e
 for page in q.iter_query_samples_stream(params):
     table = page.column_table
 
-# Pythonic conversions (require the optional [analysis] extra: pip install dp-python-lib[analysis])
+# Pythonic conversions (require the optional [analysis] extra: pip install "dp-python-lib[analysis]")
 df = q.query_samples(params).to_dataframe()      # one page -> pandas.DataFrame (UTC datetime index)
 arrays = q.query_samples(params).to_numpy()      # one page -> {"timestamps": ndarray, "<col>": ndarray, ...}