feat(bootstrap)!: source hyperd from the PyPI tableauhyperapi wheels

.github/scripts/verify-npm-hyperd-pin.py

@@ -2,8 +2,9 @@
 """Guard: keep the npm release workflow's hyperd pin in sync with the toml.
 
 `.github/workflows/npm-build-publish.yml` bundles `hyperd` into the npm
-packages using its OWN hardcoded `HYPERD_VERSION` / `HYPERD_BUILD_ID` /
-per-platform matrix `hyperd-sha256`s. Those are decoupled from
+packages from the PyPI `tableauhyperapi` wheel, using its OWN hardcoded
+`HYPERD_VERSION` plus per-platform matrix `hyperd-wheel-tag`s and
+`hyperd-sha256`s. Those are decoupled from
 `hyperdb-bootstrap/hyperd-version.toml`, which is what `make download-hyperd`
 and the crates.io path use.
 
@@ -12,10 +13,16 @@
 This script fails CI whenever the two drift, so that can't recur silently.
 
 The platform slug (`macos-arm64`, `linux-x86_64`, `windows-x86_64`) is the join
-key: it is identical between the toml's `[sha256]` table and the workflow's
-`hyperd-slug` matrix field. Only slugs the workflow actually builds are checked,
-so a commented-out matrix entry (invisible to the YAML parser) and any unused
-extra toml sha are both fine.
+key: it is identical between the toml's `[wheel_tag]` / `[sha256]` tables and
+the workflow's `hyperd-slug` matrix field. Only slugs the workflow actually
+builds are checked, so a commented-out matrix entry (invisible to the YAML
+parser) and any unused extra toml entry are both fine.
+
+The wheel tag (`macosx_13_0_arm64`, `manylinux2014_x86_64`, ...) is checked
+alongside the digest because it is a drift vector of its own: it is not
+derivable from the version, and a wrong tag resolves to a URL that does not
+exist, so it would surface as an opaque 404 mid-release rather than as a
+mismatch. The sha256s are digests of the downloaded `.whl` archive.
 """
 
 from __future__ import annotations
@@ -36,12 +43,12 @@ def main() -> int:
     workflow = yaml.safe_load(WORKFLOW.read_text())
 
     env = workflow.get("env", {})
+    toml_wheel_tag = toml_data.get("wheel_tag", {})
     toml_sha = toml_data.get("sha256", {})
 
     # (label, expected-from-toml, actual-from-workflow)
     checks: list[tuple[str, str, str | None]] = [
         ("HYPERD_VERSION", str(toml_data["version"]), env.get("HYPERD_VERSION")),
-        ("HYPERD_BUILD_ID", str(toml_data["build_id"]), env.get("HYPERD_BUILD_ID")),
     ]
 
     errors: list[str] = []
@@ -51,13 +58,22 @@ def main() -> int:
         slug = entry.get("hyperd-slug")
         if slug is None:
             continue
-        expected = toml_sha.get(slug)
-        if expected is None:
+        expected_tag = toml_wheel_tag.get(slug)
+        if expected_tag is None:
+            errors.append(
+                f'matrix slug "{slug}" has no [wheel_tag]."{slug}" entry in {TOML.name}'
+            )
+        else:
+            checks.append(
+                (f"wheel_tag[{slug}]", expected_tag, entry.get("hyperd-wheel-tag"))
+            )
+        expected_sha = toml_sha.get(slug)
+        if expected_sha is None:
             errors.append(
                 f'matrix slug "{slug}" has no [sha256]."{slug}" entry in {TOML.name}'
             )
             continue
-        checks.append((f"sha256[{slug}]", expected, entry.get("hyperd-sha256")))
+        checks.append((f"sha256[{slug}]", expected_sha, entry.get("hyperd-sha256")))
 
     for label, expected, actual in checks:
         if actual == expected:
@@ -72,8 +88,9 @@ def main() -> int:
         sys.stdout.flush()
         print(
             f"\n{WORKFLOW.name} is out of sync with {TOML.name}. "
-            "Update the workflow's env vars and matrix sha256s to match the toml "
-            "(or vice versa) so npm bundles the same hyperd as crates.io.",
+            "Update the workflow's env vars and matrix wheel tags / sha256s to "
+            "match the toml (or vice versa) so npm bundles the same hyperd as "
+            "crates.io.",
             file=sys.stderr,
         )
         return 1

.github/workflows/npm-build-publish.yml

@@ -23,8 +23,11 @@ permissions:
 
 env:
   CARGO_TERM_COLOR: always
+  # hyperd is bundled from the PyPI `tableauhyperapi` wheel. This version and
+  # the per-platform wheel tags / sha256s in the build-npm matrix below must
+  # match hyperdb-bootstrap/hyperd-version.toml — enforced by
+  # .github/scripts/verify-npm-hyperd-pin.py.
   HYPERD_VERSION: "0.0.26479"
-  HYPERD_BUILD_ID: "r96880f6a"
 
 jobs:
   verify-ci:
@@ -92,27 +95,35 @@ jobs:
       fail-fast: false
       matrix:
         include:
+          # hyperd-wheel-tag is the PyPI platform tag for this slug. It is NOT
+          # derivable from HYPERD_VERSION, so it has to be carried explicitly;
+          # a wrong tag yields a 404 rather than a version mismatch.
+          # hyperd-sha256 is the digest of the downloaded .whl archive.
           - platform: darwin-arm64
             os: macos-14
             target: aarch64-apple-darwin
             hyperd-slug: macos-arm64
-            hyperd-sha256: "65bd021b3d3470ac74728ec287866a3ee0dfd806daf2589670feb3580955ee95"
+            hyperd-wheel-tag: macosx_13_0_arm64
+            hyperd-sha256: "e80e4dac6d8437ad8c20f36add7e523b18bc06d90d4c605a256c57df8df2c118"
           # TODO: re-enable when macos-13 runners are more available
           # - platform: darwin-x64
           #   os: macos-13
           #   target: x86_64-apple-darwin
           #   hyperd-slug: macos-x86_64
-          #   hyperd-sha256: "6690669c8a6a6c7c6794c101beb31b83e1589d76cb14b0c817523069591f694c"
+          #   hyperd-wheel-tag: macosx_10_11_x86_64
+          #   hyperd-sha256: "960e276028137847a3870695d9c2d5a1392c173b1e119ff1146d24a75deca71a"
           - platform: linux-x64-gnu
             os: ubuntu-latest
             target: x86_64-unknown-linux-gnu
             hyperd-slug: linux-x86_64
-            hyperd-sha256: "c20be5b6874d319c7db01dcec763d7e65ae2483c0becc75f2914a58accc4f932"
+            hyperd-wheel-tag: manylinux2014_x86_64
+            hyperd-sha256: "9f5ff04c0dc3c17224b7a3f36f297775f2f49aae084da84614003cd6508213bc"
           - platform: win32-x64-msvc
             os: windows-latest
             target: x86_64-pc-windows-msvc
             hyperd-slug: windows-x86_64
-            hyperd-sha256: "3a400508e79c67ce9dcd8bd317ab164a61b5aed8ca880031ed4bbe6621514e1d"
+            hyperd-wheel-tag: win_amd64
+            hyperd-sha256: "7a4f96d2a22351e944fea6db5d03ab5272ad4c0577acc987bfcb3739ed639502"
     runs-on: ${{ matrix.os }}
     defaults:
       run:
@@ -154,16 +165,20 @@ jobs:
       - name: Download and verify hyperd
         env:
           SLUG: ${{ matrix.hyperd-slug }}
+          WHEEL_TAG: ${{ matrix.hyperd-wheel-tag }}
           EXPECTED_SHA256: ${{ matrix.hyperd-sha256 }}
         run: |
           set -euo pipefail
-          URL="https://downloads.tableau.com/tssoftware/tableauhyperapi-java-${SLUG}-release-main.${HYPERD_VERSION}.${HYPERD_BUILD_ID}.zip"
-          echo "Downloading: $URL"
-          curl --fail --silent --show-error --location --output hyperd-archive.zip "$URL"
+          # The engine ships inside the PyPI `tableauhyperapi` wheel, which is
+          # a zip. This URL 302-redirects to the content-addressed download.
+          URL="https://files.pythonhosted.org/packages/py3/t/tableauhyperapi/tableauhyperapi-${HYPERD_VERSION}-py3-none-${WHEEL_TAG}.whl"
+          echo "Downloading hyperd for ${SLUG}: $URL"
+          curl --fail --silent --show-error --location --output hyperd-archive.whl "$URL"
+          # Digest is over the downloaded archive, not the extracted binary.
           if command -v sha256sum &>/dev/null; then
-            ACTUAL_SHA256=$(sha256sum hyperd-archive.zip | awk '{print $1}')
+            ACTUAL_SHA256=$(sha256sum hyperd-archive.whl | awk '{print $1}')
           else
-            ACTUAL_SHA256=$(shasum -a 256 hyperd-archive.zip | awk '{print $1}')
+            ACTUAL_SHA256=$(shasum -a 256 hyperd-archive.whl | awk '{print $1}')
           fi
           if [[ "$ACTUAL_SHA256" != "$EXPECTED_SHA256" ]]; then
             echo "::error::SHA256 mismatch: expected $EXPECTED_SHA256, got $ACTUAL_SHA256"
@@ -175,8 +190,11 @@ jobs:
         run: |
           set -euo pipefail
           mkdir -p hyperd-extracted
-          unzip -q hyperd-archive.zip -d hyperd-raw
-          # Find the lib/hyper or bin/hyper directory (matches extract.rs logic)
+          unzip -q hyperd-archive.whl -d hyperd-raw
+          # Find the lib/hyper or bin/hyper directory (matches extract.rs
+          # logic). The wheel lays the engine out at tableauhyperapi/bin/hyper/
+          # on every platform, so in practice the bin/hyper branch matches;
+          # the lib/hyper branch is kept for parity with extract.rs.
           HYPER_DIR=$(find hyperd-raw -type d -name "hyper" -path "*/lib/hyper" -o -type d -name "hyper" -path "*/bin/hyper" | head -1)
           if [[ -z "$HYPER_DIR" ]]; then
             echo "::error::Could not find lib/hyper or bin/hyper in archive"
@@ -195,6 +213,10 @@ jobs:
       - name: Extract LICENSE from hyperd archive
         run: |
           set -euo pipefail
+          # The wheel's tableauhyperapi-<version>.dist-info/ carries LICENSE,
+          # NOTICES.txt and HYPER_API_OSS_disclosure.txt, so this glob matches.
+          # The heredoc below is the fallback if that ever changes, so a miss
+          # is non-fatal.
           LICENSE_FILE=$(find hyperd-raw -iname "LICENSE*" -o -iname "NOTICE*" | head -1)
           if [[ -n "$LICENSE_FILE" ]]; then
             cp "$LICENSE_FILE" LICENSE-HYPERD

.github/workflows/verify-hyperd-pin.yml

@@ -1,8 +1,9 @@
 name: verify-hyperd-pin
 
-# HEAD every platform URL for the pinned Hyper release. Catches
-# Tableau yanking / renaming an archive, and catches typos in
-# hyperd-version.toml on PRs that touch the pin.
+# HEAD every platform URL for the pinned Hyper release. The archives are the
+# PyPI `tableauhyperapi` wheels, so this catches a wheel being yanked and
+# catches typos in hyperd-version.toml (version or wheel tag — a wrong tag is
+# just a URL that 404s) on PRs that touch the pin.
 
 on:
   push:
@@ -46,7 +47,8 @@ jobs:
 
       - name: Verify npm-build-publish.yml hyperd pin matches the toml
         # The release workflow bundles hyperd into the npm packages from its
-        # OWN hardcoded version/build_id/sha256s, decoupled from the toml.
+        # OWN hardcoded version + per-platform wheel tags/sha256s, decoupled
+        # from the toml.
         # 0.7.1 shipped npm with the stale 0.0.25080 engine because only the
         # toml was bumped. This guard fails the build if they ever drift again.
         run: python3 .github/scripts/verify-npm-hyperd-pin.py

AGENTS.md

@@ -9,10 +9,14 @@ running `make download-hyperd` (or `.\build.ps1 download-hyperd`). The
 implementation lives in the [`hyperdb-bootstrap`](hyperdb-bootstrap/) crate;
 the pinned release is baked into
 [`hyperdb-bootstrap/hyperd-version.toml`](hyperdb-bootstrap/hyperd-version.toml).
-Bumping `hyperd` = edit that file (version + build_id + per-platform sha256s),
-then let the `fix(bootstrap):` commit drive the version via release-please (the
-crate uses `version.workspace = true` — don't hand-edit a crate version). The
-full repeatable procedure — verify the pin, run the suite, A/B benchmark
+Bumping `hyperd` = edit that file (`version` + the four `[wheel_tag]` entries +
+the four per-platform sha256s — there is no `build_id`), then let the
+`fix(bootstrap):` commit drive the version via release-please (the crate uses
+`version.workspace = true` — don't hand-edit a crate version). `hyperd` comes
+out of the PyPI `tableauhyperapi` wheels, so **you don't compute the digests**:
+read them straight off the JSON API with
+`curl -s https://pypi.org/pypi/tableauhyperapi/<version>/json | jq -r '.urls[] | "\(.filename)  \(.digests.sha256)"'`.
+The full repeatable procedure — verify the pin, run the suite, A/B benchmark
 against the previous pin, and log the result — is captured in the
 [`update-hyperd-release`](.claude/skills/update-hyperd-release/SKILL.md) skill.
 

DEVELOPMENT.md

@@ -384,10 +384,9 @@ in companion crates:
    This installs `hyperd` at `.hyperd/current/hyperd` (auto-discovered by
    `make`/`build.ps1` — no `HYPERD_PATH` needed). The pinned release is
    baked into [`hyperdb-bootstrap/hyperd-version.toml`](hyperdb-bootstrap/hyperd-version.toml);
-   to upgrade, edit that file (version + build_id + per-platform sha256s)
-   and bump the crate version. Pass `ARGS="--latest"` to fetch the newest
-   release via best-effort scraping, or `ARGS="--version X --build-id Y"`
-   for an ad-hoc pin.
+   to upgrade, edit that file (version + the four `[wheel_tag]` entries +
+   per-platform sha256s) and bump the crate version. Pass
+   `ARGS="--version X"` for an ad-hoc pin.
 
    If you already have a `hyperd` elsewhere, set `HYPERD_PATH` instead:
 
@@ -420,11 +419,9 @@ in companion crates:
    after the `build.ps1 download-hyperd` command):
 
    ```bash
-   # Best-effort scrape of the latest release (skips sha256 verification).
-   make download-hyperd ARGS="--latest"
-
-   # Pin to a specific release ad-hoc.
-   make download-hyperd ARGS="--version 0.0.24457 --build-id rc36858b6"
+   # Pin to a specific release ad-hoc. Inherits the builtin pin's wheel tags
+   # and carries no digests, so the download is unverified (logs a WARN).
+   make download-hyperd ARGS="--version 0.0.26359"
 
    # Install to a custom location, e.g. shared across repos.
    make download-hyperd ARGS="--dest /opt/hyperd"
@@ -435,9 +432,10 @@ in companion crates:
 
    Bumping the baked-in pin is an edit to
    [`hyperdb-bootstrap/hyperd-version.toml`](hyperdb-bootstrap/hyperd-version.toml)
-   (version + build_id + per-platform sha256s) plus a crate version bump.
-   `build.rs` validates the file on every compile, and the
-   `verify-hyperd-pin` CI workflow confirms the URLs resolve.
+   (version + the four `[wheel_tag]` entries + per-platform sha256s, all read
+   off the PyPI JSON API) plus a crate version bump. `build.rs` validates the
+   file on every compile, and the `verify-hyperd-pin` CI workflow confirms the
+   URLs resolve and that each pinned digest matches the one PyPI publishes.
 
 4. **Windows only**: Install Visual Studio Build Tools with "Desktop development with C++"
    workload (provides the MSVC linker, not for C++ compilation).

Makefile

@@ -141,13 +141,14 @@ test-api-release:
 examples:
 	./run_all_examples.sh
 
-# Download hyperd from Tableau's Hyper C++ API release into .hyperd/current/
-# Forward extra flags via ARGS, e.g. `make download-hyperd ARGS="--latest"`.
+# Download hyperd from the PyPI tableauhyperapi wheel into .hyperd/current/
+# Forward extra flags via ARGS, e.g. `make download-hyperd ARGS="--force"`.
 download-hyperd:
 	cargo run --release -p hyperdb-bootstrap --bin hyperdb-bootstrap -- download $(ARGS)
 
-# Network-only check: HEAD each supported platform URL for the pinned
-# release. Intended for CI (nightly + on PRs touching hyperd-version.toml).
+# Network-only check: probe each supported platform's wheel URL for the pinned
+# release and cross-check its digest against PyPI. Intended for CI (nightly +
+# on PRs touching hyperd-version.toml).
 verify-hyperd-pin:
 	cargo run --release -p hyperdb-bootstrap --bin hyperdb-bootstrap -- verify $(ARGS)