From ebf3812d5e9ea11c413dc8f3715dadefff4a5e8d Mon Sep 17 00:00:00 2001 From: Craig McChesney Date: Mon, 5 Oct 2026 13:48:05 -0600 Subject: [PATCH] #76: publish releases to PyPI, with a TestPyPI rehearsal - release.yml: enable publish-pypi on rel-* tags, after the GitHub Release (needs publish-github-release), with skip-existing and a post-upload digest check against SHA256SUMS. New publish-testpypi job behind a testpypi dispatch input; dispatch builds drop the local version segment. release-dist retention 7 -> 30 days to match the approval window. - .github/scripts/check-index-digests.py: the digest check (self-tested). - pyproject.toml: Documentation and Changelog URLs. - README: lead with pip install from PyPI, quoted extras, absolute links. - README.env: installing from PyPI and verifying a pip download. - Cookbook: quote the extras. NEXT.md: #76 section, Installing, and the approval step in the cut checklist. CLAUDE.md: the enabled flow. Co-Authored-By: Claude Opus 5.5 Claude-Session: https://claude.ai/code/session_01UCxWh2HqjqhsmLwSJbJAWE --- .github/scripts/check-index-digests.py | 144 +++++++++++++++++++++++++ .github/workflows/release.yml | 122 +++++++++++++++++---- CLAUDE.md | 23 +++- README.env | 57 +++++++++- README.md | 59 +++++----- doc/cookbook/README.md | 2 +- doc/cookbook/conventions.md | 2 +- doc/cookbook/query.md | 2 +- doc/release-notes/NEXT.md | 32 ++++++ plan/tickets/76/plan.md | 8 +- pyproject.toml | 2 + 11 files changed, 394 insertions(+), 59 deletions(-) create mode 100644 .github/scripts/check-index-digests.py diff --git a/.github/scripts/check-index-digests.py b/.github/scripts/check-index-digests.py new file mode 100644 index 0000000..b695ebb --- /dev/null +++ b/.github/scripts/check-index-digests.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python3 +"""Check that a package index serves exactly the files listed in a SHA256SUMS file, byte for byte. + +release.yml runs this after each upload (PyPI on a tag, TestPyPI on an opted-in rehearsal), with the +SHA256SUMS the build job generated and signed. It is what makes "the PyPI files are the signed GitHub +Release files" a checked statement rather than an assumption (#76; plan/tickets/76/plan.md, D5). + +The version is read from the wheel's filename in SHA256SUMS, so tag builds and rehearsal (.devN) builds +are handled alike. The index's JSON API (`/pypi///json`) is polled until every +expected file is listed or the timeout expires, since a fresh upload can take a few minutes to appear +through the CDN. Then it fails on any expected file missing from the index, any file on the index for +that version that SHA256SUMS does not list, and any digest mismatch, printing both sides. + +Files skipped by the upload (`skip-existing: true`, plan D3a) are compared like any other: a re-run +that skips a file already uploaded passes, and a different build at a version already on the index +fails here, loudly, rather than at upload time. + +Each run starts with a self-test of the comparison, so a rule that stops matching fails instead of +passing everything. +""" + +import argparse +import json +import sys +import time +import urllib.error +import urllib.request +from pathlib import Path + + +def read_sums(path: Path) -> dict[str, str]: + """Parse `sha256sum` output into {filename: hex digest}.""" + sums: dict[str, str] = {} + for line in path.read_text().splitlines(): + if not line.strip(): + continue + digest, name = line.split(maxsplit=1) + sums[name.lstrip("*")] = digest.lower() + if not sums: + raise ValueError(f"{path} lists no files") + return sums + + +def wheel_version(sums: dict[str, str]) -> str: + """The version from the one wheel's filename (`--.whl`).""" + wheels = [name for name in sums if name.endswith(".whl")] + if len(wheels) != 1: + raise ValueError(f"expected exactly one wheel in SHA256SUMS, found {wheels}") + return wheels[0].split("-")[1] + + +def compare(expected: dict[str, str], served: dict[str, str]) -> list[str]: + """Every difference between the signed files and the index's files, as error messages.""" + errors = [] + for name in sorted(expected.keys() - served.keys()): + errors.append(f"missing from the index: {name} (SHA256SUMS: {expected[name]})") + for name in sorted(served.keys() - expected.keys()): + errors.append(f"on the index but not in SHA256SUMS: {name} (index: {served[name]})") + for name in sorted(expected.keys() & served.keys()): + if expected[name] != served[name]: + errors.append(f"digest mismatch: {name} (SHA256SUMS: {expected[name]}, index: {served[name]})") + return errors + + +def fetch_served(url: str) -> dict[str, str] | None: + """{filename: sha256} for the release at `url`, or None if the index does not have it yet.""" + request = urllib.request.Request(url, headers={"Accept": "application/json", "Cache-Control": "no-cache"}) + try: + with urllib.request.urlopen(request, timeout=30) as response: + data = json.load(response) + except urllib.error.HTTPError as e: + if e.code == 404: + return None + raise + return {f["filename"]: f["digests"]["sha256"].lower() for f in data["urls"]} + + +def self_test() -> list[str]: + a, b = "a" * 64, "b" * 64 + failures = [] + cases = [ + ("identical", {"x.whl": a, "x.tar.gz": b}, {"x.whl": a, "x.tar.gz": b}, 0), + ("missing", {"x.whl": a, "x.tar.gz": b}, {"x.whl": a}, 1), + ("extra", {"x.whl": a}, {"x.whl": a, "x.tar.gz": b}, 1), + ("mismatch", {"x.whl": a}, {"x.whl": b}, 1), + ("all three", {"x.whl": a, "y.whl": a}, {"x.whl": b, "z.whl": a}, 3), + ] + for label, expected, served, count in cases: + got = len(compare(expected, served)) + if got != count: + failures.append(f"compare() self-test '{label}': expected {count} error(s), got {got}") + sums = {"dp_python_lib-1.17.0.dev3-py3-none-any.whl": a, "dp_python_lib-1.17.0.dev3.tar.gz": b} + if wheel_version(sums) != "1.17.0.dev3": + failures.append("wheel_version() self-test: did not read 1.17.0.dev3") + return failures + + +def main() -> int: + parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + parser.add_argument("sums", type=Path, help="the SHA256SUMS file the build job signed") + parser.add_argument("--index-url", required=True, help="e.g. https://pypi.org or https://test.pypi.org") + parser.add_argument("--project", required=True, help="the project name on the index") + parser.add_argument("--timeout", type=int, default=300, help="seconds to wait for the files to appear") + parser.add_argument("--interval", type=int, default=15, help="seconds between polls") + args = parser.parse_args() + + failures = self_test() + if failures: + for failure in failures: + print(f"::error::self-test: {failure}") + return 1 + + expected = read_sums(args.sums) + version = wheel_version(expected) + url = f"{args.index_url.rstrip('/')}/pypi/{args.project}/{version}/json" + print(f"Checking {url} against {args.sums}:") + for name, digest in sorted(expected.items()): + print(f" {digest} {name}") + + deadline = time.monotonic() + args.timeout + while True: + served = fetch_served(url) + if served is not None and expected.keys() <= served.keys(): + break + if time.monotonic() >= deadline: + break + state = "not found" if served is None else f"lists {sorted(served)}" + print(f"Index {state}; retrying in {args.interval}s") + time.sleep(args.interval) + + if served is None: + print(f"::error::{url} still returns 404 after {args.timeout}s.") + return 1 + errors = compare(expected, served) + for error in errors: + print(f"::error::{error}") + if errors: + return 1 + print(f"The index serves exactly the {len(expected)} files in {args.sums}, byte for byte.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1bb39b1..5ff38d9 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -5,10 +5,17 @@ on: tags: - "rel-*" # Manual dispatch is always a rehearsal: it builds, verifies, and signs, but never - # publishes. Publishing is gated on a `rel-` tag push and nothing else, so there is no - # path to cutting a release from an arbitrary ref. Choose the ref to rehearse against - # with the branch/tag selector in the Actions UI. + # publishes to PyPI or a GitHub Release. Both are gated on a `rel-` tag push and nothing + # else, so there is no path to cutting a release from an arbitrary ref. Choose the ref to + # rehearse against with the branch/tag selector in the Actions UI. The one thing a + # dispatch can publish is the rehearsal build to test.pypi.org, and only when the + # `testpypi` input is set (#76). workflow_dispatch: + inputs: + testpypi: + description: "Also upload the rehearsal build to test.pypi.org (a sandbox; versions there cannot be re-uploaded either)" + type: boolean + default: false # A publish must never be cancelled halfway through, so unlike CI this does not set # cancel-in-progress. @@ -98,6 +105,14 @@ jobs: - name: Build wheel and sdist run: | + set -euo pipefail + # A rehearsal builds an untagged commit, which setuptools-scm versions with a local + # segment (1.16.1.dev61+g496f0e0). PyPI and TestPyPI both reject local versions, so + # a dispatch drops it (1.16.1.dev61) to stay uploadable to TestPyPI. A tag build has + # no local segment and is left alone. + if [[ "${{ github.event_name }}" == "workflow_dispatch" ]]; then + export SETUPTOOLS_SCM_OVERRIDES_FOR_DP_PYTHON_LIB='{local_scheme="no-local-version"}' + fi python -m pip install --upgrade pip build twine python -m build @@ -168,7 +183,11 @@ jobs: with: name: release-dist path: dist/ - retention-days: 7 + # GitHub holds a deployment awaiting approval for up to 30 days, and publish-pypi + # needs these exact files: a full re-run would rebuild and re-sign, and the new files + # would not match the SHA256SUMS and bundles already on the GitHub Release. So the + # artifact lives as long as the approval can wait (#76, plan D2). + retention-days: 30 publish-github-release: name: Publish GitHub Release @@ -229,32 +248,43 @@ jobs: echo "Published body matches doc/release-notes/${TAG}.md." # --------------------------------------------------------------------------------------- - # PyPI publishing -- WIRED UP BUT INTENTIONALLY DISABLED. + # PyPI publishing (#76; plan/tickets/76/plan.md). + # + # Trusted Publishing (OIDC): no API token is stored anywhere. The publisher on pypi.org + # names this repository, workflow release.yml, and environment `pypi`; the environment + # requires an approval and admits only rel-* tags. The environment must exist BEFORE any + # job names it: GitHub creates a missing one on the fly, with no protection rules. # - # To enable: - # 1. Claim the project name on PyPI. - # 2. Configure a Trusted Publisher for osprey-dcs/dp-python-lib, workflow release.yml, - # environment `pypi` (PyPI project settings -> Publishing). Trusted Publishing uses - # OIDC, so no API token is ever stored in repo secrets. - # 3. Create a GitHub environment named `pypi`, ideally with required reviewers. - # 4. Change the `if:` below to: - # if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/rel-') + # publish-pypi runs last, after the GitHub Release is published and its body verified, + # because a PyPI file can never be replaced. It uploads the same files the build job + # signed, then checks that the index serves them byte for byte (check-index-digests.py). # - # Note: PyPI rejects Sigstore bundles as uploads, hence the cleanup step -- the bundles - # still go to the GitHub Release. attestations: true emits PEP 740 attestations instead. + # PyPI rejects Sigstore bundles as uploads, hence the cleanup step -- the bundles still go + # to the GitHub Release. attestations: true emits PEP 740 attestations instead. + # + # publish-testpypi is the rehearsal of the same path, run by dispatch with testpypi=true, + # through its own publisher and an unprotected `testpypi` environment. The two jobs' + # steps are duplicated on purpose; keep them in step. # --------------------------------------------------------------------------------------- publish-pypi: name: Publish to PyPI - needs: build + needs: [build, publish-github-release] runs-on: ubuntu-latest - if: false # <-- flip this to enable; see the comment block above + if: github.event_name == 'push' && startsWith(github.ref, 'refs/tags/rel-') environment: name: pypi url: https://pypi.org/p/dp-python-lib permissions: + contents: read id-token: write steps: + - name: Check out the digest checker + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + sparse-checkout: .github/scripts + persist-credentials: false + - name: Download build outputs uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 with: @@ -263,10 +293,64 @@ jobs: - name: Remove non-distribution files # Only sdists and wheels may be uploaded; the checksums file, the release notes, and - # the Sigstore bundles would all be rejected. - run: rm -f dist/SHA256SUMS dist/RELEASE_NOTES.md dist/*.sigstore.json + # the Sigstore bundles would all be rejected. SHA256SUMS is kept aside for the check. + run: | + mv dist/SHA256SUMS SHA256SUMS + rm -f dist/RELEASE_NOTES.md dist/*.sigstore.json - name: Publish to PyPI uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2 with: attestations: true + # Files go up one at a time, so a wheel can land and the sdist fail; without this, + # every re-run would then fail on the wheel. A re-run uploads the same artifact, and + # the digest check below compares skipped files too, so a real conflict still fails + # (plan D3a). + skip-existing: true + + - name: Verify PyPI serves the signed files + run: python3 .github/scripts/check-index-digests.py SHA256SUMS --index-url https://pypi.org --project dp-python-lib + + publish-testpypi: + name: Publish rehearsal to TestPyPI + needs: build + runs-on: ubuntu-latest + if: github.event_name == 'workflow_dispatch' && inputs.testpypi + environment: + name: testpypi + url: https://test.pypi.org/p/dp-python-lib + permissions: + contents: read + id-token: write + + steps: + - name: Check out the digest checker + uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1 + with: + sparse-checkout: .github/scripts + persist-credentials: false + + - name: Download build outputs + uses: actions/download-artifact@3e5f45b2cfb9172054b4087a40e8e0b5a5461e7c # v8.0.1 + with: + name: release-dist + path: dist + + - name: Remove non-distribution files + # As in publish-pypi. A dispatch stages no RELEASE_NOTES.md; rm -f covers that. + run: | + mv dist/SHA256SUMS SHA256SUMS + rm -f dist/RELEASE_NOTES.md dist/*.sigstore.json + + - name: Publish to TestPyPI + uses: pypa/gh-action-pypi-publish@dc37677b2e1c63e2034f94d8a5b11f265b73ba33 # v1.14.2 + with: + repository-url: https://test.pypi.org/legacy/ + attestations: true + # As in publish-pypi. A .devN version counts commits since the last tag, so two + # different commits can share one; the second rehearsal's upload is then skipped, + # and the digest check below fails on the mismatch, as it should. + skip-existing: true + + - name: Verify TestPyPI serves the signed files + run: python3 .github/scripts/check-index-digests.py SHA256SUMS --index-url https://test.pypi.org --project dp-python-lib diff --git a/CLAUDE.md b/CLAUDE.md index b64b046..1a8e848 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -95,8 +95,20 @@ GitHub Actions workflows live in `.github/workflows/`: signs everything with keyless Sigstore, and publishes a GitHub Release. A `workflow_dispatch` trigger allows rehearsing the whole path without cutting a tag: publishing is gated on a `rel-` tag push, so a manual run always stops after - build/verify/sign. A PyPI publish job is wired up but disabled (`if: false`); the - comment block above it lists the steps to enable it. + build/verify/sign. On a tag, `publish-pypi` then uploads the same signed files to PyPI + (#76; `plan/tickets/76/plan.md`): Trusted Publishing (OIDC, no token), in the `pypi` + environment, which requires an approval and admits only `rel-*` tags. It runs **after** + `publish-github-release`, because a PyPI file can never be replaced, sets `skip-existing` + so a partial upload can be finished by "Re-run failed jobs", and ends with + `.github/scripts/check-index-digests.py`, which fails unless the index serves exactly the + files in `SHA256SUMS`, byte for byte (skipped files included, so a real conflict is still + loud). The `release-dist` artifact is kept 30 days, GitHub's approval window, because a + full re-run rebuilds and re-signs files that no longer match the release. A dispatch with + the `testpypi` input set rehearses the same path against test.pypi.org (`publish-testpypi`, + an unprotected `testpypi` environment); a dispatch build drops setuptools-scm's local + version segment (`+g`), which both indexes reject, so a rehearsal is `X.Y.Z.devN`. + **A job that names a missing environment creates it, with no protection rules**: `pypi` + must exist, with its reviewer and tag rule, before any workflow change that names it merges. **Action pinning**: every `uses:` reference in both workflows is pinned to a full commit SHA with a trailing `# vX.Y.Z` comment naming the version — a tag is mutable, so whoever @@ -113,7 +125,10 @@ grep -rnE 'uses: *[^ ]+@' .github/workflows/ | grep -vE '@[0-9a-f]{40} # v' # ``` **Cutting a release**: the version comes from the git tag alone (setuptools-scm), -so there is no version to bump in a file. Tag `rel-X.Y.Z` and push the tag. +so there is no version to bump in a file. Tag `rel-X.Y.Z` and push the tag, then approve +the `pypi` deployment once the GitHub Release job is green, and confirm the digest check +passed. The approver also pushes the tag, so the environment's "prevent self-review" stays +off: the approval is a deliberate pause before an irreversible upload, not a second reviewer. The tag must be exactly `rel-X.Y.Z` with no suffix — prerelease shapes like `rel-1.15.0-rc1` are rejected up front, because setuptools-scm would normalize them (`1.15.0rc1`) and fail the tag-vs-built version assertion with a confusing error. @@ -720,7 +735,7 @@ for page in q.iter_query_samples(params): # raises RuntimeError on a page e for page in q.iter_query_samples_stream(params): table = page.column_table -# Pythonic conversions (require the optional [analysis] extra: pip install dp-python-lib[analysis]) +# Pythonic conversions (require the optional [analysis] extra: pip install "dp-python-lib[analysis]") df = q.query_samples(params).to_dataframe() # one page -> pandas.DataFrame (UTC datetime index) arrays = q.query_samples(params).to_numpy() # one page -> {"timestamps": ndarray, "": ndarray, ...} diff --git a/README.env b/README.env index 3d49ee8..7f1b722 100644 --- a/README.env +++ b/README.env @@ -2,7 +2,8 @@ dp-python-lib — Release Artifacts and Verification ================================================== This repository publishes versioned dp-python-lib Python distributions via -GitHub Releases. Each release corresponds to a Git tag of the form: +GitHub Releases and, from the first release after rel-1.16.0, PyPI. Each +release corresponds to a Git tag of the form: rel- @@ -103,6 +104,56 @@ Download and Installation pip install "dp_python_lib--py3-none-any.whl[analysis]" +------------------------------------------------------------ +Installing from PyPI +------------------------------------------------------------ + +The usual install is from PyPI: + + pip install dp-python-lib + pip install "dp-python-lib[analysis]" + +(Quote the extra: zsh, the macOS default shell, otherwise treats the brackets +as a glob.) + +The wheel and sdist on PyPI are the same files, byte for byte, as those on the +GitHub Release. The release workflow uploads the files it signed, and only +after the GitHub Release is published; it then fetches the digests PyPI +reports and fails unless they match SHA256SUMS exactly. So a file from PyPI +can be verified with the release's SHA256SUMS and Sigstore bundles: + +1. Download the wheel from PyPI, without its dependencies: + + pip download --no-deps "dp-python-lib==" + +2. Into the same directory, download SHA256SUMS and the bundles from the + GitHub Release: + + gh release download rel- -R osprey-dcs/dp-python-lib \ + -p SHA256SUMS -p '*.sigstore.json' + +3. Check the wheel's checksum. SHA256SUMS also lists the sdist, which is not + here, so --ignore-missing is needed; confirm the wheel is reported OK: + + sha256sum --ignore-missing -c SHA256SUMS + +4. Verify the signatures of the wheel and SHA256SUMS as in step 3 of + "Download and Installation", naming only those two files: + + sigstore verify identity \ + --cert-identity "https://github.com/osprey-dcs/dp-python-lib/.github/workflows/release.yml@refs/tags/rel-" \ + --cert-oidc-issuer "https://token.actions.githubusercontent.com" \ + dp_python_lib-*.whl SHA256SUMS + +Without --no-deps, pip download also fetches every dependency, none of which +SHA256SUMS lists. Without --ignore-missing, sha256sum reports the sdist +missing and exits non-zero. + +PyPI also holds PEP 740 attestations for each file, made by the same workflow +through PyPI's Trusted Publishing. They are a second, independent record of +where the files came from, shown on each file's page on pypi.org; the steps +above do not depend on them. + ------------------------------------------------------------ Release Notes ------------------------------------------------------------ @@ -119,5 +170,5 @@ Notes - dp-python-lib is a client library; it does not run as a standalone service. - It talks to the MLDP services implemented in dp-service, over the gRPC API defined in dp-grpc. -- Publishing to PyPI is not enabled yet; GitHub Releases are the distribution - channel. +- PyPI and GitHub Releases carry the same files; see "Installing from PyPI" + above for how that is checked. diff --git a/README.md b/README.md index 8002327..b03dd07 100644 --- a/README.md +++ b/README.md @@ -18,7 +18,7 @@ learning workflows — can use that platform without writing gRPC code. It prov data science workflow actually wants — retrieving a labelled dataset, exporting to common formats, feeding a training pipeline. -The generated gRPC stubs live in [src/dp_python_lib/grpc](src/dp_python_lib/grpc). They are +The generated gRPC stubs live in [src/dp_python_lib/grpc](https://github.com/osprey-dcs/dp-python-lib/tree/main/src/dp_python_lib/grpc). They are produced by an Actions workflow in the dp-grpc repo (`generate-python-stubs.yml`), which can be run manually and fires automatically on a release tag; it opens a pull request against this repo. **These files are generated and must not be edited by hand** — fix the generator instead. @@ -43,8 +43,8 @@ offers: dataset assembly and reuse, Pythonic data structures (pandas, NumPy, and confirmation — PyTorch tensors), export to common file formats, and end-to-end ingest → annotate → query → train workflows. -The [current state](#current-state) below is the part of this that exists today; the -[TODO](#todo) is what remains. +The [current state](https://github.com/osprey-dcs/dp-python-lib/blob/main/README.md#current-state) below is the part of this that exists today; the +[TODO](https://github.com/osprey-dcs/dp-python-lib/blob/main/README.md#todo) is what remains. ## Current state @@ -81,7 +81,7 @@ for their service. bucket keeps its stored column type, time axis, and column metadata, so this is how array, image, struct, and serialized columns are read back. `bucket_conversions` reads them in plain Python, trims them exactly to a range on request, and, with the `[analysis]` extra, assembles one pandas - DataFrame per PV. See [whole buckets](doc/cookbook/query.md#whole-buckets-arrays-images-and-stored-metadata). + DataFrame per PV. See [whole buckets](https://github.com/osprey-dcs/dp-python-lib/blob/main/doc/cookbook/query.md#whole-buckets-arrays-images-and-stored-metadata). - **DataSets** — `client.annotation.datasets`. Name a region of the archive (time ranges plus the PVs covered over them) so it can be found, annotated, and exported later: `save_dataset()`, `get_dataset()`, `query_datasets()`, `iter_datasets()`, `delete_dataset()`, and a @@ -101,7 +101,7 @@ for their service. `query_request_status()`. `data_frame.split_data_frame()` cuts a large frame into chunks under the server's message-size and time-span limits, and the `data_frame` builders now cover array, image, struct, and serialized columns as well as scalars. See the - [ingestion recipe](doc/cookbook/ingestion.md). + [ingestion recipe](https://github.com/osprey-dcs/dp-python-lib/blob/main/doc/cookbook/ingestion.md). **Supporting framework:** YAML + environment-variable configuration (`MLDP_*`, via pydantic-settings), TLS-capable channel creation, hierarchical logging, three-tier error handling @@ -132,29 +132,30 @@ older than your `dp_python_lib` will not implement everything listed here. The - Data science conveniences beyond the current DataFrame / NumPy conversions — PyTorch tensor support is the likely next step, additive on top of the existing NumPy path -**Project infrastructure** - -- Publishing to PyPI. The release workflow has the job wired up but disabled; everything else — - unit tests across Python 3.10-3.13, lint, format, and type checks, the cookbook snippet checker, - and signed release artifacts — runs in CI today. - ## Installation -Python 3.10 or later. +Python 3.10 or later. Releases are published to [PyPI](https://pypi.org/p/dp-python-lib): ```bash # core client -pip install -e . +pip install dp-python-lib # with pandas / NumPy / Excel conversions for query results -pip install -e .[analysis] +pip install "dp-python-lib[analysis]" +``` -# development tooling (pytest, ruff, mypy) -pip install -e .[dev] +The quotes matter in zsh (the macOS default shell), which otherwise treats the brackets as a glob. + +For development, install from a checkout instead: + +```bash +pip install -e ".[analysis,dev]" # dev adds pytest, ruff, mypy ``` -Released wheels are also attached to each [GitHub release](https://github.com/osprey-dcs/dp-python-lib/releases), -with checksums and Sigstore signatures; [`README.env`](README.env) says how to verify them. +Each [GitHub release](https://github.com/osprey-dcs/dp-python-lib/releases) carries the same wheel and +sdist, byte for byte, with checksums and Sigstore signatures; +[`README.env`](https://github.com/osprey-dcs/dp-python-lib/blob/main/README.env) says how to verify +them, including a download from PyPI. **Upgrading from 1.15.0 or earlier:** 1.16.0 raises the `grpcio` floor to 1.84.0, because the regenerated stubs require it. `pip install` picks that up, but an existing editable install will @@ -162,7 +163,7 @@ not upgrade it on its own — the stubs then fail at import with a version misma required release. `pip install -e . --upgrade` resolves it. Point the client at your MLDP services with an `mldp-config.yaml` file or `MLDP_*` environment -variables — see [Creating and connecting a client](doc/cookbook/connecting.md). +variables — see [Creating and connecting a client](https://github.com/osprey-dcs/dp-python-lib/blob/main/doc/cookbook/connecting.md). ## Hello, MLDP @@ -196,23 +197,23 @@ failure, so skipping the check turns an error into silently missing data. ## How to use it -The **[cookbook](doc/cookbook/)** is the guide. Each recipe walks through a complete task, and +The **[cookbook](https://github.com/osprey-dcs/dp-python-lib/tree/main/doc/cookbook)** is the guide. Each recipe walks through a complete task, and the recipes share one continuous worked example drawn from an accelerator facility. | Recipe | Covers | |---|---| -| [API conventions](doc/cookbook/conventions.md) | Patterns every call shares: checking results, paging, criteria AND/OR rules, full-replace saves, time handling | -| [Creating and connecting a client](doc/cookbook/connecting.md) | Building an `MldpClient`, config files and environment variables, TLS, logging | -| [Cataloguing PVs](doc/cookbook/pv-metadata.md) | Recording what a PV is, then finding PVs by property instead of by name | -| [Recording machine configuration](doc/cookbook/machine-configuration.md) | Defining configurations, recording when each was active, and answering "what was the machine doing at 18:04?" | -| [Ingesting data](doc/cookbook/ingestion.md) | Registering a provider, sending frames of samples, confirming they landed, and chunking data too big for one message | -| [Querying time-series data](doc/cookbook/query.md) | Retrieving samples by PV, metadata, or machine configuration, and converting to pandas / NumPy / Excel; reading whole stored buckets, including array, image, and struct columns | -| [Labeling samples](doc/cookbook/sample-status.md) | Recording per-sample status codes, reading them back, and querying data with flagged samples excluded | -| [DataSets and annotations](doc/cookbook/datasets-and-annotations.md) | Naming a region of the archive, attaching analysis results with column-level provenance, and exporting | +| [API conventions](https://github.com/osprey-dcs/dp-python-lib/blob/main/doc/cookbook/conventions.md) | Patterns every call shares: checking results, paging, criteria AND/OR rules, full-replace saves, time handling | +| [Creating and connecting a client](https://github.com/osprey-dcs/dp-python-lib/blob/main/doc/cookbook/connecting.md) | Building an `MldpClient`, config files and environment variables, TLS, logging | +| [Cataloguing PVs](https://github.com/osprey-dcs/dp-python-lib/blob/main/doc/cookbook/pv-metadata.md) | Recording what a PV is, then finding PVs by property instead of by name | +| [Recording machine configuration](https://github.com/osprey-dcs/dp-python-lib/blob/main/doc/cookbook/machine-configuration.md) | Defining configurations, recording when each was active, and answering "what was the machine doing at 18:04?" | +| [Ingesting data](https://github.com/osprey-dcs/dp-python-lib/blob/main/doc/cookbook/ingestion.md) | Registering a provider, sending frames of samples, confirming they landed, and chunking data too big for one message | +| [Querying time-series data](https://github.com/osprey-dcs/dp-python-lib/blob/main/doc/cookbook/query.md) | Retrieving samples by PV, metadata, or machine configuration, and converting to pandas / NumPy / Excel; reading whole stored buckets, including array, image, and struct columns | +| [Labeling samples](https://github.com/osprey-dcs/dp-python-lib/blob/main/doc/cookbook/sample-status.md) | Recording per-sample status codes, reading them back, and querying data with flagged samples excluded | +| [DataSets and annotations](https://github.com/osprey-dcs/dp-python-lib/blob/main/doc/cookbook/datasets-and-annotations.md) | Naming a region of the archive, attaching analysis results with column-level provenance, and exporting | Every Python snippet in the cookbook is mechanically syntax- and type-checked against the installed package. -For further examples, see the [integration tests](tests/integration). For the wire protocol +For further examples, see the [integration tests](https://github.com/osprey-dcs/dp-python-lib/tree/main/tests/integration). For the wire protocol beneath this library — the protobuf messages and RPC semantics, documented in Java — see the [dp-grpc cookbook](https://github.com/osprey-dcs/dp-grpc/tree/main/doc/cookbook). diff --git a/doc/cookbook/README.md b/doc/cookbook/README.md index d8d998c..7bfc940 100644 --- a/doc/cookbook/README.md +++ b/doc/cookbook/README.md @@ -70,7 +70,7 @@ type-checked against the installed package to catch wrong attribute and method n recipe is checked for importing every library name its fragments use: ```bash -pip install -e .[dev] +pip install -e ".[dev]" .venv/bin/python .dev/tools/check-cookbook-snippets.py ``` diff --git a/doc/cookbook/conventions.md b/doc/cookbook/conventions.md index 2049e1d..384130e 100644 --- a/doc/cookbook/conventions.md +++ b/doc/cookbook/conventions.md @@ -317,7 +317,7 @@ The core install is deliberately lightweight. Converting query results to panda Excel requires the `analysis` extra: ``` -pip install dp-python-lib[analysis] +pip install "dp-python-lib[analysis]" ``` Without it, `to_dataframe()` and `to_numpy()` raise `ImportError` at the point of use — the diff --git a/doc/cookbook/query.md b/doc/cookbook/query.md index ba64ef0..a99efcd 100644 --- a/doc/cookbook/query.md +++ b/doc/cookbook/query.md @@ -256,7 +256,7 @@ range and a `limit`, and prefer the streaming form below. These conversions need the optional extra: ``` -pip install dp-python-lib[analysis] +pip install "dp-python-lib[analysis]" ``` Without it, the calls below raise `ImportError` — the imports are lazy, so the rest of the diff --git a/doc/release-notes/NEXT.md b/doc/release-notes/NEXT.md index 6212b3c..9865efb 100644 --- a/doc/release-notes/NEXT.md +++ b/doc/release-notes/NEXT.md @@ -37,6 +37,7 @@ person cutting the release has any reason to re-read. - [Detecting open-ended activations (#26)](#detecting-open-ended-activations-issue-26) - [Ingesting data (#17)](#ingesting-data-issue-17) - [Querying whole buckets (#16)](#querying-whole-buckets-issue-16) +- [Install from PyPI (#76)](#install-from-pypi-issue-76) - [Cutting the release](#cutting-the-release) --- @@ -271,8 +272,35 @@ the column metadata stored with a column. See [#16](https://github.com/osprey-dcs/dp-python-lib/issues/16) and `plan/tickets/16/plan.md`. +## Install from PyPI (Issue #76) + +This is the first release published to [PyPI](https://pypi.org/p/dp-python-lib), so a download +from the release page is no longer needed: + +```bash +pip install dp-python-lib +pip install "dp-python-lib[analysis]" # with the pandas / NumPy / Excel conversions +``` + +Quote the extra in zsh, the macOS default shell, which otherwise reads the brackets as a glob. + +The files on PyPI are the files attached to this release, byte for byte. The release workflow +uploads them through PyPI's Trusted Publishing only after this page is published, then fails +unless the digests PyPI reports match `SHA256SUMS`. So a PyPI download can be checked against this +release's `SHA256SUMS` and Sigstore bundles; [`README.env`](https://github.com/osprey-dcs/dp-python-lib/blob/main/README.env) +gives the commands. PyPI also shows PEP 740 attestations for each file, a second record of the +same provenance. Earlier releases are not on PyPI and will not be added. + +See [#76](https://github.com/osprey-dcs/dp-python-lib/issues/76) and `plan/tickets/76/plan.md`. + ## Installing +```bash +pip install dp-python-lib +``` + +or, from the files attached to this release: + ```bash pip install dp_python_lib-*.whl ``` @@ -320,3 +348,7 @@ When the version is known and the release is being cut: section and empty Contents down to the "Cutting the release" entry. Keep the preamble, `## Installing`, and this checklist. 10. **Merge, then push the `rel-` tag.** The notes must be on the tagged commit. +11. **Approve the `pypi` deployment** once the "Publish GitHub Release" job is green: the run + waits on it in the Actions UI. Then confirm the "Verify PyPI serves the signed files" step + passed. If the PyPI job fails, use "Re-run failed jobs", not a full re-run, which would + rebuild and re-sign files that no longer match the release; the build artifact is kept 30 days. diff --git a/plan/tickets/76/plan.md b/plan/tickets/76/plan.md index 0e47acf..bb69304 100644 --- a/plan/tickets/76/plan.md +++ b/plan/tickets/76/plan.md @@ -1,6 +1,7 @@ # Issue #76 — Publish releases to PyPI -**Status:** triaged 2026-10-05; open questions resolved the same day. Implementation has not started. +**Status:** triaged 2026-10-05; open questions resolved the same day. Implementation in progress on +`feat/76-pypi-publish`. ## Overview @@ -125,6 +126,11 @@ sha256sum --ignore-missing -c SHA256SUMS Rejected: a post-publish `pip install` smoke test, which the build job already does against the same wheel; it would add resolver and CDN flakiness without checking anything new. +*Implementation note:* the check is a script, `.github/scripts/check-index-digests.py`, rather than +inline shell in both jobs, so it is linted, self-tested on each run, and runnable locally against any +project on PyPI. The publish jobs fetch it with a sparse checkout of `.github/scripts` at the run's +ref, which adds `contents: read` to their permissions. + **D6. Only the next release goes to PyPI.** rel-1.16.0 and earlier are not backfilled: that would be a hand upload with an API token and no PEP 740 attestations, of a release carrying the #19 config-precedence bug. The first PyPI version is whatever the next tag is. *(Decided 2026-10-05.)* diff --git a/pyproject.toml b/pyproject.toml index e6accd1..79b5989 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,6 +44,8 @@ dependencies = [ Homepage = "https://github.com/osprey-dcs/dp-python-lib" Repository = "https://github.com/osprey-dcs/dp-python-lib" Issues = "https://github.com/osprey-dcs/dp-python-lib/issues" +Documentation = "https://github.com/osprey-dcs/dp-python-lib/blob/main/doc/cookbook/README.md" +Changelog = "https://github.com/osprey-dcs/dp-python-lib/releases" [project.optional-dependencies] # Pythonic query-result conversions (query_conversions): DataFrame / NumPy / Excel. Kept out of the core