From 6ca385a2ba45accf2738c21c14e35e8b7ad05408 Mon Sep 17 00:00:00 2001 From: debuggingfuture Date: Sun, 25 Oct 2026 19:39:20 +0800 Subject: [PATCH] feat(release): publish each contextful profile as an amd64 + arm64 image index Both Linux release cells build their platform's image with `contextful-ci image` and stage its layer and `.image.json` descriptor. The parent pushes each profile's config and manifest verbatim, by digest, then one OCI image index under the version tag, so the index digest equals the one the build computed. The Worker verifies every blob against its digest before pushing, and opens a layer from R2 only when the registry lacks it. --- infra/Dockerfile.sandbox-release | 2 - infra/build-oci-layer.sh | 25 --- infra/build-oci-layer.test.sh | 48 ------ infra/build-oci-layout.py | 77 --------- packages/core/src/fakes/github-fake.ts | 5 +- packages/core/src/services/github.ts | 15 +- packages/runtime-cf/src/ghcr-push.test.ts | 190 ++++++++++++---------- packages/runtime-cf/src/ghcr-push.ts | 120 +++++++++----- packages/runtime-cf/src/github-live.ts | 75 ++++++--- runs/contextful-release.test.ts | 48 ++++-- runs/contextful-release.ts | 90 +++++----- 11 files changed, 339 insertions(+), 356 deletions(-) delete mode 100755 infra/build-oci-layer.sh delete mode 100755 infra/build-oci-layer.test.sh delete mode 100755 infra/build-oci-layout.py diff --git a/infra/Dockerfile.sandbox-release b/infra/Dockerfile.sandbox-release index dbe78641..94b42a34 100644 --- a/infra/Dockerfile.sandbox-release +++ b/infra/Dockerfile.sandbox-release @@ -27,8 +27,6 @@ RUN python3 -m venv /opt/zig && \ # Text-only TAPI records only the macOS system symbols referenced by # iana-time-zone. The runtime framework comes from macOS itself. COPY infra/darwin-stubs /opt/darwin-stubs -COPY infra/build-oci-layer.sh /opt/build-oci-layer.sh -COPY infra/build-oci-layout.py /opt/build-oci-layout.py ENV CARGO_TARGET_AARCH64_APPLE_DARWIN_RUSTFLAGS="-C link-arg=-F/opt/darwin-stubs" \ CARGO_TARGET_X86_64_APPLE_DARWIN_RUSTFLAGS="-C link-arg=-F/opt/darwin-stubs" \ CARGO_BUILD_JOBS=4 diff --git a/infra/build-oci-layer.sh b/infra/build-oci-layer.sh deleted file mode 100755 index a1245e17..00000000 --- a/infra/build-oci-layer.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/sh -set -eu - -binary=$1 -root=$2 -output=$3 -image=$4 -version=$5 - -file "$binary" | grep -q 'ELF 64-bit.*x86-64' -if readelf -l "$binary" | grep -q INTERP; then - echo 'OCI release binary has a dynamic interpreter' >&2 - exit 1 -fi - -mkdir -p "$root/usr/local/bin" "$root/etc/ssl/certs" "$root/data" -cp "$binary" "$root/usr/local/bin/contextful" -cp /etc/ssl/certs/ca-certificates.crt "$root/etc/ssl/certs/ca-certificates.crt" -chown -R 0:0 "$root" -chown 65532:65532 "$root/data" -tar --sort=name --mtime=@0 --numeric-owner -C "$root" -cf "${output%.gz}" . -gzip -n -c "${output%.gz}" > "$output" -python3 /opt/build-oci-layout.py "$output" "${output%.gz}" "$image" "$version" -sha256sum "${output%.gz}" "$output" -stat -c %s "$output" diff --git a/infra/build-oci-layer.test.sh b/infra/build-oci-layer.test.sh deleted file mode 100755 index ea8d7370..00000000 --- a/infra/build-oci-layer.test.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/bin/sh -set -eu - -work=$(mktemp -d) -trap 'rm -rf "$work"' EXIT -chmod 755 "$work" -cat > "$work/main.c" <<'SOURCE' -int main(void) { return 0; } -SOURCE -zig cc -target x86_64-linux-musl -static -o "$work/contextful" "$work/main.c" -/opt/build-oci-layer.sh "$work/contextful" "$work/root" "$work/layer.tar.gz" "$work/image.oci.tar" "0.5.0" > "$work/result" -[ "$(wc -l < "$work/result")" -eq 3 ] -file "$work/contextful" | grep -q 'ELF 64-bit.*x86-64' -tar --numeric-owner -tvf "$work/layer.tar" | grep -q '65532/65532.* ./data/' -mkdir "$work/extracted" -tar -C "$work/extracted" -xf "$work/layer.tar" -setpriv --reuid 65532 --regid 65532 --clear-groups -- touch "$work/extracted/data/write-smoke" -[ -f "$work/extracted/data/write-smoke" ] -tar -tf "$work/image.oci.tar" | grep -q '^index.json$' -tar -tf "$work/image.oci.tar" | grep -q '^oci-layout$' -tar -xOf "$work/image.oci.tar" index.json | grep -q '"architecture":"amd64"' -python3 - "$work/image.oci.tar" <<'PY' -import gzip -import hashlib -import json -import sys -import tarfile - -with tarfile.open(sys.argv[1]) as archive: - def read(name): - return archive.extractfile(name).read() - - index = json.loads(read("index.json")) - descriptor = index["manifests"][0] - assert descriptor["annotations"]["org.opencontainers.image.ref.name"] == "0.5.0" - manifest_bytes = read(f"blobs/sha256/{descriptor['digest'][7:]}") - assert hashlib.sha256(manifest_bytes).hexdigest() == descriptor["digest"][7:] - manifest = json.loads(manifest_bytes) - config_bytes = read(f"blobs/sha256/{manifest['config']['digest'][7:]}") - assert hashlib.sha256(config_bytes).hexdigest() == manifest["config"]["digest"][7:] - config = json.loads(config_bytes) - assert config["config"]["User"] == "65532:65532" - assert config["config"]["Entrypoint"] == ["/usr/local/bin/contextful"] - layer = manifest["layers"][0] - layer_bytes = read(f"blobs/sha256/{layer['digest'][7:]}") - assert hashlib.sha256(layer_bytes).hexdigest() == layer["digest"][7:] - assert hashlib.sha256(gzip.decompress(layer_bytes)).hexdigest() == config["rootfs"]["diff_ids"][0][7:] -PY diff --git a/infra/build-oci-layout.py b/infra/build-oci-layout.py deleted file mode 100755 index b6472fab..00000000 --- a/infra/build-oci-layout.py +++ /dev/null @@ -1,77 +0,0 @@ -#!/usr/bin/env python3 -"""Package one static Contextful layer as an OCI image layout archive.""" - -import hashlib -import json -import pathlib -import shutil -import sys -import tarfile -import tempfile - -layer = pathlib.Path(sys.argv[1]) -uncompressed = pathlib.Path(sys.argv[2]) -output = pathlib.Path(sys.argv[3]) -version = sys.argv[4] - - -def encoded(value): - return json.dumps(value, separators=(",", ":")).encode("utf-8") - - -def sha256(path): - digest = hashlib.sha256() - with path.open("rb") as stream: - for chunk in iter(lambda: stream.read(1024 * 1024), b""): - digest.update(chunk) - return digest.hexdigest() - - -layer_digest = sha256(layer) -diff_id = sha256(uncompressed) -config = encoded({ - "architecture": "amd64", "os": "linux", - "config": { - "User": "65532:65532", "WorkingDir": "/data", - "Entrypoint": ["/usr/local/bin/contextful"], "Cmd": ["--version"], - "Volumes": {"/data": {}}, - }, - "rootfs": {"type": "layers", "diff_ids": [f"sha256:{diff_id}"]}, - "history": [{"created_by": "contextful release"}], -}) -config_digest = hashlib.sha256(config).hexdigest() -manifest = encoded({ - "schemaVersion": 2, "mediaType": "application/vnd.oci.image.manifest.v1+json", - "config": {"mediaType": "application/vnd.oci.image.config.v1+json", - "digest": f"sha256:{config_digest}", "size": len(config)}, - "layers": [{"mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", - "digest": f"sha256:{layer_digest}", "size": layer.stat().st_size}], -}) -manifest_digest = hashlib.sha256(manifest).hexdigest() -index = encoded({ - "schemaVersion": 2, - "manifests": [{"mediaType": "application/vnd.oci.image.manifest.v1+json", - "digest": f"sha256:{manifest_digest}", "size": len(manifest), - "platform": {"architecture": "amd64", "os": "linux"}, - "annotations": {"org.opencontainers.image.ref.name": version}}], -}) - -with tempfile.TemporaryDirectory() as temporary: - root = pathlib.Path(temporary) - blobs = root / "blobs" / "sha256" - blobs.mkdir(parents=True) - shutil.copyfile(layer, blobs / layer_digest) - (blobs / config_digest).write_bytes(config) - (blobs / manifest_digest).write_bytes(manifest) - (root / "index.json").write_bytes(index) - (root / "oci-layout").write_bytes(encoded({"imageLayoutVersion": "1.0.0"})) - with tarfile.open(output, "w") as archive: - for path in sorted(root.rglob("*")): - info = archive.gettarinfo(str(path), arcname=str(path.relative_to(root))) - info.uid = info.gid = info.mtime = 0 - info.uname = info.gname = "" - if path.is_file(): - with path.open("rb") as stream: - archive.addfile(info, stream) - else: - archive.addfile(info) diff --git a/packages/core/src/fakes/github-fake.ts b/packages/core/src/fakes/github-fake.ts index feb584bf..e8154348 100644 --- a/packages/core/src/fakes/github-fake.ts +++ b/packages/core/src/fakes/github-fake.ts @@ -129,7 +129,7 @@ export type GithubFakeState = { /** Every `createRelease` call, in order — lets a test assert a release published. */ readonly createReleaseCalls: CreateRelease[]; readonly publishReleaseCalls: Array<{ repo: string; releaseId: number }>; - readonly publishContainerImageCalls: Array<{ repo: string; profile: string; version: string }>; + readonly publishContainerImageCalls: Array<{ repo: string; profile: string; version: string; images: string[] }>; readonly releases: Record; readonly tagCommits: Record; /** Every label ADD, in order. */ @@ -536,7 +536,8 @@ export const makeGithubFake = ( downloadUrl: `https://github.com/${req.repo}/releases/download/${req.releaseId}/${req.artifactName}`, }), publishContainerImage: (req) => Effect.sync(() => { - state.publishContainerImageCalls.push({ repo: req.repo, profile: req.profile, version: req.version }); + state.publishContainerImageCalls.push({ repo: req.repo, profile: req.profile, version: req.version, + images: req.platforms.map((platform) => platform.imageArtifactName) }); }), }; diff --git a/packages/core/src/services/github.ts b/packages/core/src/services/github.ts index f4246ff0..033c53a6 100644 --- a/packages/core/src/services/github.ts +++ b/packages/core/src/services/github.ts @@ -303,15 +303,20 @@ export type ReleaseAssetResult = { readonly downloadUrl: string; }; +/** + * A multi-platform release image: each platform's `.image.json` descriptor, + * staged by the release cell that built it, pushed as one image index under + * `version`. + */ export type PublishContainerImage = { readonly repo: string; readonly profile: string; readonly version: string; - readonly sourceExecutionId: string; - readonly layerArtifactName: string; - readonly layerSize: number; - readonly layerDigest: string; - readonly diffId: string; + /** In index order, amd64 before arm64. */ + readonly platforms: ReadonlyArray<{ + readonly sourceExecutionId: string; + readonly imageArtifactName: string; + }>; }; /** diff --git a/packages/runtime-cf/src/ghcr-push.test.ts b/packages/runtime-cf/src/ghcr-push.test.ts index 49887ceb..be236081 100644 --- a/packages/runtime-cf/src/ghcr-push.test.ts +++ b/packages/runtime-cf/src/ghcr-push.test.ts @@ -1,106 +1,122 @@ import { describe, expect, it } from "vitest"; -import { classifyGhcrFailure, pushGhcrImage } from "./ghcr-push"; +import { classifyGhcrFailure, pushGhcrImageIndex, type GhcrPlatformImage } from "./ghcr-push"; -describe("pushGhcrImage", () => { - it("publishes OCI blobs and a versioned manifest without sending credentials in image bytes", async () => { - const requests: Array<{ url: string; method: string; auth: string | null; body: Uint8Array }> = []; - const fetchImpl: typeof fetch = async (input, init) => { - const request = new Request(input, init); - const url = new URL(request.url); - const bytes = request.body === null ? new Uint8Array() : new Uint8Array(await request.arrayBuffer()); - requests.push({ url: request.url, method: request.method, auth: request.headers.get("authorization"), body: bytes }); - if (url.pathname === "/token") return Response.json({ token: "registry-bearer" }); - if (request.method === "HEAD") return new Response(null, { status: 404 }); - if (request.method === "POST") return new Response(null, { status: 202, headers: { Location: `/v2/${"fractalboxdev/contextful/contextful-edge"}/blobs/uploads/session` } }); - if (request.method === "PUT") return new Response(null, { status: 201 }); - throw new Error(`unexpected ${request.method} ${request.url}`); - }; - await pushGhcrImage({ username: "bot", token: "test-pat", name: "fractalboxdev/contextful/contextful-edge", tag: "0.5.0", - layer: new Uint8Array([1, 2, 3]), layerSize: 3, - layerDigest: "a".repeat(64), diffId: "b".repeat(64), fetchImpl }); +const hex = async (bytes: Uint8Array): Promise => + Array.from(new Uint8Array(await crypto.subtle.digest("SHA-256", bytes)), (b) => b.toString(16).padStart(2, "0")).join(""); +const MANIFEST = "application/vnd.oci.image.manifest.v1+json"; +const INDEX = "application/vnd.oci.image.index.v1+json"; + +/** One platform as `contextful-ci image` describes it: keys sorted, blobs verbatim. */ +const platformImage = async (architecture: "amd64" | "arm64", layerByte: number): Promise => { + const layer = new Uint8Array([layerByte, 2, 3]); + const layerDigest = `sha256:${await hex(layer)}`; + const config = new TextEncoder().encode(JSON.stringify({ architecture, os: "linux" })); + const configDigest = `sha256:${await hex(config)}`; + const manifest = new TextEncoder().encode(JSON.stringify({ + config: { digest: configDigest, mediaType: "application/vnd.oci.image.config.v1+json", size: config.byteLength }, + layers: [{ digest: layerDigest, mediaType: "application/vnd.oci.image.layer.v1.tar+gzip", size: layer.byteLength }], + mediaType: MANIFEST, schemaVersion: 2, + })); + const manifestDigest = `sha256:${await hex(manifest)}`; + return { layerDigest, layerSize: layer.byteLength, openLayer: async () => layer, configDigest, config, + manifestDigest, manifest, entry: { digest: manifestDigest, mediaType: MANIFEST, + platform: { architecture, os: "linux", ...(architecture === "arm64" ? { variant: "v8" } : {}) }, + size: manifest.byteLength } }; +}; + +type Recorded = { url: string; method: string; auth: string | null; type: string | null; body: Uint8Array }; +/** A registry that has nothing and accepts every upload, recording each request. */ +const emptyRegistry = (requests: Recorded[], location = "/v2/o/r/p/blobs/uploads/session"): typeof fetch => + async (input, init) => { + const request = new Request(input, init); + const url = new URL(request.url); + const body = request.body === null ? new Uint8Array() : new Uint8Array(await request.arrayBuffer()); + requests.push({ url: request.url, method: request.method, auth: request.headers.get("authorization"), + type: request.headers.get("content-type"), body }); + if (url.pathname === "/token") return Response.json({ token: "registry-bearer" }); + if (request.method === "HEAD") return new Response(null, { status: 404 }); + if (request.method === "POST") return new Response(null, { status: 202, headers: { Location: location } }); + if (request.method === "PUT") return new Response(null, { status: 201 }); + throw new Error(`unexpected ${request.method} ${request.url}`); + }; + +describe("pushGhcrImageIndex", () => { + it("pushes each platform's blobs and manifest by digest, then the index by tag", async () => { + const requests: Recorded[] = []; + const platforms = [await platformImage("amd64", 1), await platformImage("arm64", 9)]; + await pushGhcrImageIndex({ username: "bot", token: "test-pat", name: "o/r/p", tag: "0.5.0", platforms, + fetchImpl: emptyRegistry(requests) }); expect(requests[0]?.auth).toBe(`Basic ${btoa("bot:test-pat")}`); - expect(requests.filter((r) => r.method === "PUT")).toHaveLength(3); - expect(requests.at(-1)?.url).toContain("/manifests/0.5.0"); - expect(requests.at(-1)?.auth).toBe("Bearer registry-bearer"); - expect(new TextDecoder().decode(requests.at(-1)!.body)).not.toContain("test-pat"); + const manifests = requests.filter((r) => r.method === "PUT" && r.url.includes("/manifests/")); + expect(manifests.map((r) => new URL(r.url).pathname)).toEqual([ + `/v2/o/r/p/manifests/${platforms[0]!.manifestDigest}`, + `/v2/o/r/p/manifests/${platforms[1]!.manifestDigest}`, + "/v2/o/r/p/manifests/0.5.0", + ]); + expect(manifests.map((r) => r.type)).toEqual([MANIFEST, MANIFEST, INDEX]); + expect(manifests[0]!.body).toEqual(platforms[0]!.manifest); + expect(new TextDecoder().decode(manifests[2]!.body)).toBe(`{"manifests":[${ + platforms.map((p) => JSON.stringify(p.entry)).join(",")}],"mediaType":"${INDEX}","schemaVersion":2}`); + expect(requests.every((r) => !new TextDecoder().decode(r.body).includes("test-pat"))).toBe(true); }); - it("uploads every blob as application/octet-stream, which GHCR requires, and types only the manifest", async () => { + it("uploads every blob as application/octet-stream, which GHCR requires, config verbatim", async () => { // GHCR answers 404 to a blob PUT carrying an OCI media type as its Content-Type. - const puts: Array<{ path: string; type: string | null }> = []; - const fetchImpl: typeof fetch = async (input, init) => { - const request = new Request(input, init); - const url = new URL(request.url); - if (url.pathname === "/token") return Response.json({ token: "registry-bearer" }); - if (request.method === "HEAD") return new Response(null, { status: 404 }); - if (request.method === "POST") return new Response(null, { status: 202, headers: { Location: "/v2/o/r/p/blobs/upload/1.session" } }); - puts.push({ path: url.pathname, type: request.headers.get("content-type") }); - const blobUpload = url.pathname.includes("/blobs/"); - return blobUpload && request.headers.get("content-type") !== "application/octet-stream" - ? new Response(null, { status: 404 }) : new Response(null, { status: 201 }); - }; - await pushGhcrImage({ username: "bot", token: "test-pat", name: "o/r/p", tag: "0.5.0", - layer: new Uint8Array([1, 2, 3]), layerSize: 3, - layerDigest: "a".repeat(64), diffId: "b".repeat(64), fetchImpl }); - expect(puts.filter((put) => put.path.includes("/blobs/")).map((put) => put.type)) - .toEqual(["application/octet-stream", "application/octet-stream"]); - expect(puts.at(-1)).toEqual({ path: "/v2/o/r/p/manifests/0.5.0", type: "application/vnd.oci.image.manifest.v1+json" }); + const requests: Recorded[] = []; + const platform = await platformImage("amd64", 1); + await pushGhcrImageIndex({ username: "bot", token: "test-pat", name: "o/r/p", tag: "0.5.0", + platforms: [platform], fetchImpl: emptyRegistry(requests) }); + const blobs = requests.filter((r) => r.method === "PUT" && !r.url.includes("/manifests/")); + expect(blobs.map((r) => r.type)).toEqual(["application/octet-stream", "application/octet-stream"]); + expect(blobs.map((r) => new URL(r.url).searchParams.get("digest"))).toEqual([platform.layerDigest, platform.configDigest]); + expect(blobs[1]!.body).toEqual(platform.config); }); - it("keeps an existing matching image tag and refuses a conflicting tag", async () => { - let manifestDigest: string | undefined; - const requests: string[] = []; + it("keeps an existing matching index tag and refuses a conflicting one", async () => { + let indexDigest: string | undefined; + const requests: Recorded[] = []; + const registry = emptyRegistry(requests); const fetchImpl: typeof fetch = async (input, init) => { const request = new Request(input, init); - requests.push(`${request.method} ${request.url}`); - if (new URL(request.url).pathname === "/token") return Response.json({ token: "registry-bearer" }); - if (request.method === "HEAD" && request.url.endsWith("/manifests/0.5.0")) { - return manifestDigest === undefined ? new Response(null, { status: 404 }) : - new Response(null, { status: 200, headers: { "Docker-Content-Digest": manifestDigest } }); - } - if (request.method === "HEAD") return new Response(null, { status: 404 }); - if (request.method === "POST") return new Response(null, { status: 202, - headers: { Location: "/v2/fractalboxdev/contextful/contextful-edge/blobs/uploads/session" } }); - if (request.method === "PUT" && request.url.endsWith("/manifests/0.5.0")) { - const hash = new Uint8Array(await crypto.subtle.digest("SHA-256", await request.arrayBuffer())); - manifestDigest = `sha256:${Array.from(hash, (byte) => byte.toString(16).padStart(2, "0")).join("")}`; - return new Response(null, { status: 201 }); + if (request.url.endsWith("/manifests/0.5.0")) { + if (request.method === "HEAD") { + return indexDigest === undefined ? new Response(null, { status: 404 }) : + new Response(null, { status: 200, headers: { "Docker-Content-Digest": indexDigest } }); + } + indexDigest = `sha256:${await hex(new Uint8Array(await request.clone().arrayBuffer()))}`; } - if (request.method === "PUT") return new Response(null, { status: 201 }); - throw new Error(`unexpected ${request.method} ${request.url}`); + return registry(request); }; - const opts = { username: "bot", token: "test-pat", name: "fractalboxdev/contextful/contextful-edge", - tag: "0.5.0", layer: new Uint8Array([1]), layerSize: 1, - layerDigest: "a".repeat(64), diffId: "b".repeat(64), fetchImpl }; - await pushGhcrImage(opts); + const opts = { username: "bot", token: "test-pat", name: "o/r/p", tag: "0.5.0", + platforms: [await platformImage("amd64", 1), await platformImage("arm64", 9)], fetchImpl }; + await pushGhcrImageIndex(opts); const afterPublish = requests.length; - await pushGhcrImage(opts); - expect(requests.slice(afterPublish)).toHaveLength(2); - manifestDigest = "sha256:" + "c".repeat(64); - await expect(pushGhcrImage(opts)).rejects.toThrow("already names a different manifest"); + await pushGhcrImageIndex(opts); + expect(requests.slice(afterPublish)).toHaveLength(1); + indexDigest = `sha256:${"c".repeat(64)}`; + await expect(pushGhcrImageIndex(opts)).rejects.toThrow("already names a different manifest"); }); - it("does not forward the registry bearer token to a cross-host upload URL", async () => { - const uploads: Request[] = []; - const fetchImpl: typeof fetch = async (input, init) => { - const request = new Request(input, init); - const url = new URL(request.url); - if (url.pathname === "/token") return Response.json({ token: "registry-bearer" }); - if (request.method === "HEAD") return new Response(null, { status: 404 }); - if (request.method === "POST") return new Response(null, { status: 202, - headers: { Location: "https://upload.example.test/blob?signature=test" } }); - if (request.method === "PUT" && url.hostname === "upload.example.test") { - uploads.push(request); - return new Response(null, { status: 201 }); - } - if (request.method === "PUT") return new Response(null, { status: 201 }); - throw new Error(`unexpected ${request.method} ${request.url}`); - }; - await pushGhcrImage({ username: "bot", token: "test-pat", name: "fractalboxdev/contextful/contextful-edge", - tag: "0.5.0", layer: new Uint8Array([1]), layerSize: 1, - layerDigest: "a".repeat(64), diffId: "b".repeat(64), fetchImpl }); + it("refuses a platform whose blobs differ from their digests or whose entry names another manifest", async () => { + const good = await platformImage("amd64", 1); + const push = (platform: GhcrPlatformImage) => pushGhcrImageIndex({ username: "bot", token: "t", name: "o/r/p", + tag: "0.5.0", platforms: [platform], fetchImpl: emptyRegistry([]) }); + await expect(push({ ...good, config: new TextEncoder().encode("{}") })).rejects.toThrow("invalid OCI image"); + await expect(push({ ...good, layerDigest: `sha256:${"e".repeat(64)}` })).rejects.toThrow("invalid OCI image"); + await expect(push({ ...good, entry: { ...good.entry, digest: `sha256:${"e".repeat(64)}` } })).rejects.toThrow("invalid OCI image"); + }); + + it("opens a layer only when the registry lacks it, and never forwards the bearer to a cross-host upload", async () => { + const requests: Recorded[] = []; + let opened = 0; + const platform = await platformImage("amd64", 1); + await pushGhcrImageIndex({ username: "bot", token: "test-pat", name: "o/r/p", tag: "0.5.0", + platforms: [{ ...platform, openLayer: async () => { opened++; return platform.openLayer(); } }], + fetchImpl: emptyRegistry(requests, "https://upload.example.test/blob?signature=test") }); + expect(opened).toBe(1); + const uploads = requests.filter((r) => new URL(r.url).hostname === "upload.example.test"); expect(uploads).toHaveLength(2); - expect(uploads.every((request) => request.headers.get("authorization") === null)).toBe(true); + expect(uploads.every((r) => r.auth === null)).toBe(true); }); }); diff --git a/packages/runtime-cf/src/ghcr-push.ts b/packages/runtime-cf/src/ghcr-push.ts index f50b5de6..f9dbe08b 100644 --- a/packages/runtime-cf/src/ghcr-push.ts +++ b/packages/runtime-cf/src/ghcr-push.ts @@ -1,16 +1,35 @@ type BlobBody = Uint8Array | ReadableStream; -export type GhcrImage = { +const INDEX = "application/vnd.oci.image.index.v1+json"; +const MANIFEST = "application/vnd.oci.image.manifest.v1+json"; + +/** + * One platform of an image index, as `contextful-ci image` writes it: the + * config and manifest blobs verbatim, the layer they name, and the manifest's + * index entry. The push uploads each verbatim, so the index digest equals the + * one the build computed. + */ +export type GhcrPlatformImage = { + readonly layerDigest: string; + readonly layerSize: number; + /** Opened only when the registry lacks the layer. */ + readonly openLayer: () => Promise; + readonly configDigest: string; + readonly config: Uint8Array; + readonly manifestDigest: string; + readonly manifest: Uint8Array; + /** The manifest's descriptor in the index, with its `platform`. */ + readonly entry: Readonly>; +}; + +export type GhcrImageIndex = { readonly username: string; readonly token: string; /** Registry path without ghcr.io, e.g. owner/repo/profile. */ readonly name: string; readonly tag: string; - readonly layer: BlobBody; - readonly layerSize: number; - readonly layerDigest: string; - /** SHA-256 of the uncompressed tar stream. */ - readonly diffId: string; + /** In index order. */ + readonly platforms: readonly GhcrPlatformImage[]; readonly fetchImpl?: typeof fetch; }; @@ -31,14 +50,43 @@ const fixedBody = (body: BlobBody, size: number): { body: BlobBody; pump?: Promi return { body: fixed.readable, pump: body.pipeTo(fixed.writable) }; }; -/** Push a single-platform static image through the OCI Distribution API. */ -export const pushGhcrImage = async (opts: GhcrImage): Promise => { +const SHA256 = /^sha256:[a-f0-9]{64}$/; + +/** + * Refuse a platform whose blobs differ from their digests, whose manifest + * names other blobs, or whose entry describes another manifest. + */ +const verifyPlatform = async (platform: GhcrPlatformImage): Promise => { + const invalid = () => new TypeError("invalid OCI image name, tag, digest, or size"); + if (![platform.layerDigest, platform.configDigest, platform.manifestDigest].every((d) => SHA256.test(d)) || + !Number.isSafeInteger(platform.layerSize) || platform.layerSize < 0) throw invalid(); + if (`sha256:${await digest(platform.config)}` !== platform.configDigest || + `sha256:${await digest(platform.manifest)}` !== platform.manifestDigest) throw invalid(); + const manifest = JSON.parse(new TextDecoder().decode(platform.manifest)) as { + mediaType?: unknown; config?: { digest?: unknown; size?: unknown }; + layers?: ReadonlyArray<{ digest?: unknown; size?: unknown }>; + }; + if (manifest.mediaType !== MANIFEST || manifest.config?.digest !== platform.configDigest || + manifest.config.size !== platform.config.byteLength || manifest.layers?.length !== 1 || + manifest.layers[0]?.digest !== platform.layerDigest || manifest.layers[0].size !== platform.layerSize) throw invalid(); + const { entry } = platform; + const os = (entry.platform as { os?: unknown } | undefined)?.os; + if (entry.mediaType !== MANIFEST || entry.digest !== platform.manifestDigest || + entry.size !== platform.manifest.byteLength || os !== "linux") throw invalid(); +}; + +/** + * Push a multi-platform image through the OCI Distribution API: each + * platform's blobs and manifest by digest, then one image index under `tag`. + * The index is compact JSON with its keys in sorted order, the form + * `contextful-ci image` writes, so its digest is the build's. + */ +export const pushGhcrImageIndex = async (opts: GhcrImageIndex): Promise => { if (!/^[a-z0-9]+(?:[._/-][a-z0-9]+)*$/.test(opts.name) || - !/^[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}$/.test(opts.tag) || - !/^[a-f0-9]{64}$/.test(opts.layerDigest) || !/^[a-f0-9]{64}$/.test(opts.diffId) || - !Number.isSafeInteger(opts.layerSize) || opts.layerSize < 0) { + !/^[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}$/.test(opts.tag) || opts.platforms.length === 0) { throw new TypeError("invalid OCI image name, tag, digest, or size"); } + for (const platform of opts.platforms) await verifyPlatform(platform); const doFetch = opts.fetchImpl ?? fetch; const scope = `repository:${opts.name}:pull,push`; const tokenUrl = new URL("https://ghcr.io/token"); @@ -52,33 +100,21 @@ export const pushGhcrImage = async (opts: GhcrImage): Promise => { if (!token) throw new Error("GHCR returned no scoped registry token"); const auth = { Authorization: `Bearer ${token}` }; const base = `https://ghcr.io/v2/${opts.name}`; - const config = new TextEncoder().encode(JSON.stringify({ - architecture: "amd64", os: "linux", - config: { User: "65532:65532", WorkingDir: "/data", - Entrypoint: ["/usr/local/bin/contextful"], Cmd: ["--version"], Volumes: { "/data": {} } }, - rootfs: { type: "layers", diff_ids: [`sha256:${opts.diffId}`] }, - history: [{ created_by: "contextful release" }], + const index = new TextEncoder().encode(JSON.stringify({ + manifests: opts.platforms.map((platform) => platform.entry), mediaType: INDEX, schemaVersion: 2, })); - const configDigest = await digest(config); - const manifest = new TextEncoder().encode(JSON.stringify({ - schemaVersion: 2, mediaType: "application/vnd.oci.image.manifest.v1+json", - config: { mediaType: "application/vnd.oci.image.config.v1+json", digest: `sha256:${configDigest}`, size: config.byteLength }, - layers: [{ mediaType: "application/vnd.oci.image.layer.v1.tar+gzip", digest: `sha256:${opts.layerDigest}`, size: opts.layerSize }], - })); - const manifestDigest = `sha256:${await digest(manifest)}`; - const manifestUrl = `${base}/manifests/${encodeURIComponent(opts.tag)}`; - const existing = await doFetch(manifestUrl, { - method: "HEAD", headers: { ...auth, Accept: "application/vnd.oci.image.manifest.v1+json" }, - }); + const indexDigest = `sha256:${await digest(index)}`; + const tagUrl = `${base}/manifests/${encodeURIComponent(opts.tag)}`; + const existing = await doFetch(tagUrl, { method: "HEAD", headers: { ...auth, Accept: INDEX } }); if (existing.ok) { - if (existing.headers.get("docker-content-digest") === manifestDigest) return; + if (existing.headers.get("docker-content-digest") === indexDigest) return; throw new Error("GHCR image tag already names a different manifest"); } if (existing.status !== 404) throw new Error(`GHCR manifest lookup failed with HTTP ${existing.status}`); // A blob upload is `application/octet-stream` (OCI Distribution); GHCR answers 404 to // a blob PUT typed with an image media type. Media types belong to the manifest. - const uploadBlob = async (sha256: string, body: BlobBody, size: number): Promise => { - const exists = await doFetch(`${base}/blobs/sha256:${sha256}`, { method: "HEAD", headers: auth }); + const uploadBlob = async (ref: string, open: () => Promise, size: number): Promise => { + const exists = await doFetch(`${base}/blobs/${ref}`, { method: "HEAD", headers: auth }); if (exists.ok) return; if (exists.status !== 404) throw new Error(`GHCR blob lookup failed with HTTP ${exists.status}`); const started = await doFetch(`${base}/blobs/uploads/`, { method: "POST", headers: auth }); @@ -89,8 +125,8 @@ export const pushGhcrImage = async (opts: GhcrImage): Promise => { if (uploadUrl.protocol !== "https:" || uploadUrl.username || uploadUrl.password) { throw new Error("GHCR upload location is not HTTPS"); } - uploadUrl.searchParams.set("digest", `sha256:${sha256}`); - const fixed = fixedBody(body, size); + uploadUrl.searchParams.set("digest", ref); + const fixed = fixedBody(await open(), size); const request = doFetch(uploadUrl, { method: "PUT", headers: { ...(uploadUrl.hostname === "ghcr.io" ? auth : {}), @@ -101,17 +137,21 @@ export const pushGhcrImage = async (opts: GhcrImage): Promise => { const [finished] = await Promise.all([request, fixed.pump ?? Promise.resolve()]); await requireOk(finished, "GHCR blob upload"); }; + const putManifest = async (url: string, mediaType: string, body: Uint8Array): Promise => { + const published = await doFetch(url, { method: "PUT", headers: { ...auth, "Content-Type": mediaType }, body }); + await requireOk(published, "GHCR manifest publish"); + }; - await uploadBlob(configDigest, config, config.byteLength); - await uploadBlob(opts.layerDigest, opts.layer, opts.layerSize); - const published = await doFetch(manifestUrl, { - method: "PUT", headers: { ...auth, "Content-Type": "application/vnd.oci.image.manifest.v1+json" }, body: manifest, - }); - await requireOk(published, "GHCR manifest publish"); + for (const platform of opts.platforms) { + await uploadBlob(platform.layerDigest, platform.openLayer, platform.layerSize); + await uploadBlob(platform.configDigest, async () => platform.config, platform.config.byteLength); + await putManifest(`${base}/manifests/${platform.manifestDigest}`, MANIFEST, platform.manifest); + } + await putManifest(tagUrl, INDEX, index); }; /** - * A `pushGhcrImage` failure as a GitHub API outcome: the registry's HTTP status + * A `pushGhcrImageIndex` failure as a GitHub API outcome: the registry's HTTP status * from the failing operation's message, its retry class, and the message itself. * A network failure (no status) is transient; a refused input or a conflicting * tag is final. diff --git a/packages/runtime-cf/src/github-live.ts b/packages/runtime-cf/src/github-live.ts index bbcf7a6a..c49ff54d 100644 --- a/packages/runtime-cf/src/github-live.ts +++ b/packages/runtime-cf/src/github-live.ts @@ -71,7 +71,7 @@ import { type WorkflowRunRef, } from "@fractalboxdev/flare-dispatch-core"; import { artifactKey } from "./artifact-r2"; -import { classifyGhcrFailure, pushGhcrImage } from "./ghcr-push"; +import { classifyGhcrFailure, pushGhcrImageIndex, type GhcrPlatformImage } from "./ghcr-push"; /** The GitHub App credentials the live `pullReview` write needs. */ export type GithubLiveConfig = { @@ -602,32 +602,69 @@ export const makeGithubLive = ( if (ghcr === undefined || artifactsBucket === undefined || executionId === undefined || db === undefined) { return yield* Effect.fail(new GitHubApiError({ status: 0, reason: "other" })); } + const arches = ["amd64", "arm64"] as const; if (req.repo !== "fractalboxdev/contextful" || !["contextful-control", "contextful-edge", "contextful-full"].includes(req.profile) || !/^[0-9]+\.[0-9]+\.[0-9]+(?:-[0-9A-Za-z.-]+)?$/.test(req.version) || - req.layerArtifactName !== `${req.profile}-${req.version}-linux-amd64.layer.tar.gz` || - !/^[a-f0-9]{64}$/.test(req.layerDigest) || !/^[a-f0-9]{64}$/.test(req.diffId)) { + req.platforms.length !== arches.length || + req.platforms.some((platform, i) => + platform.imageArtifactName !== `${req.profile}-${req.version}-linux-${arches[i]}.image.json`)) { return yield* Effect.fail(new GitHubApiError({ status: 400, reason: "other" })); } - const child = yield* Effect.tryPromise({ - try: () => db.prepare("SELECT status FROM executions WHERE id = ? AND parent_execution_id = ? AND run = ?") - .bind(req.sourceExecutionId, executionId, "contextful-release-cell").first<{ status: string }>(), - catch: () => new GitHubApiError({ status: 0, reason: "transient" }), - }); - if (child?.status !== "success") return yield* Effect.fail(new GitHubApiError({ status: 403, reason: "other" })); - const source = yield* Effect.tryPromise({ - try: () => artifactsBucket.get(artifactKey(req.sourceExecutionId, req.layerArtifactName)), - catch: () => new GitHubApiError({ status: 0, reason: "transient" }), - }); - if (source === null || source.size !== req.layerSize) { - return yield* Effect.fail(new GitHubApiError({ status: 404, reason: "other" })); + const platforms: GhcrPlatformImage[] = []; + for (const [i, platform] of req.platforms.entries()) { + const child = yield* Effect.tryPromise({ + try: () => db.prepare("SELECT status FROM executions WHERE id = ? AND parent_execution_id = ? AND run = ?") + .bind(platform.sourceExecutionId, executionId, "contextful-release-cell").first<{ status: string }>(), + catch: () => new GitHubApiError({ status: 0, reason: "transient" }), + }); + if (child?.status !== "success") return yield* Effect.fail(new GitHubApiError({ status: 403, reason: "other" })); + const descriptorObject = yield* Effect.tryPromise({ + try: () => artifactsBucket.get(artifactKey(platform.sourceExecutionId, platform.imageArtifactName)), + catch: () => new GitHubApiError({ status: 0, reason: "transient" }), + }); + if (descriptorObject === null) return yield* Effect.fail(new GitHubApiError({ status: 404, reason: "other" })); + const descriptor = yield* Effect.tryPromise({ + try: async () => JSON.parse(await descriptorObject.text()) as { + profile?: unknown; version?: unknown; platform?: { os?: unknown; architecture?: unknown }; + layer?: { digest?: unknown; size?: unknown; path?: unknown }; + config?: { digest?: unknown; content?: unknown }; + manifest?: { digest?: unknown; content?: unknown }; + entry?: Record; + }, + catch: () => new GitHubApiError({ status: 400, reason: "other" }), + }); + const layerName = `${req.profile}-${req.version}-linux-${arches[i]}.layer.tar.gz`; + const { layer, config: imageConfig, manifest, entry } = descriptor; + if (descriptor.profile !== req.profile || descriptor.version !== req.version || + descriptor.platform?.os !== "linux" || descriptor.platform.architecture !== arches[i] || + layer?.path !== layerName || typeof layer.digest !== "string" || typeof layer.size !== "number" || + typeof imageConfig?.digest !== "string" || typeof imageConfig.content !== "string" || + typeof manifest?.digest !== "string" || typeof manifest.content !== "string" || + typeof entry !== "object" || entry === null || + (entry.platform as { architecture?: unknown } | undefined)?.architecture !== arches[i]) { + return yield* Effect.fail(new GitHubApiError({ status: 400, reason: "other" })); + } + const sourceExecutionId = platform.sourceExecutionId; + const layerSize = layer.size; + platforms.push({ + layerDigest: layer.digest, layerSize, + // The layer streams from R2 only when the registry lacks it; a size + // that differs from the descriptor's is a staging defect. + openLayer: async () => { + const source = await artifactsBucket.get(artifactKey(sourceExecutionId, layerName)); + if (source === null || source.size !== layerSize) throw new TypeError("invalid OCI image name, tag, digest, or size"); + return source.body; + }, + configDigest: imageConfig.digest, config: new TextEncoder().encode(imageConfig.content), + manifestDigest: manifest.digest, manifest: new TextEncoder().encode(manifest.content), + entry, + }); } return yield* Effect.tryPromise({ - try: () => pushGhcrImage({ + try: () => pushGhcrImageIndex({ username: ghcr.username, token: ghcr.token, - name: `${req.repo}/${req.profile}`, tag: req.version, - layer: source.body, layerSize: source.size, - layerDigest: req.layerDigest, diffId: req.diffId, + name: `${req.repo}/${req.profile}`, tag: req.version, platforms, }), catch: (cause) => new GitHubApiError(classifyGhcrFailure(cause)), }); diff --git a/runs/contextful-release.test.ts b/runs/contextful-release.test.ts index a3e614cd..f32218ac 100644 --- a/runs/contextful-release.test.ts +++ b/runs/contextful-release.test.ts @@ -8,6 +8,17 @@ import { contextfulRelease, contextfulReleaseCell, contextfulReleaseFormula, REL const sha = "a".repeat(40); const repo = "fractalboxdev/contextful"; +/** The `.image.json` `contextful-ci image` writes for a Linux cell, at the path the cell reads. */ +const imageFiles = (profile: string, target: string): Record => { + const arch = target.startsWith("x86_64") ? "amd64" : "arm64"; + const name = `${profile}-0.5.0-linux-${arch}`; + return { [`/workspace/contextful/dist/${name}.image.json`]: JSON.stringify({ + profile, version: "0.5.0", target, platform: { architecture: arch, os: "linux" }, + layer: { digest: `sha256:${"c".repeat(64)}`, diffId: `sha256:${"d".repeat(64)}`, size: 3, + path: `${name}.layer.tar.gz`, mediaType: "application/vnd.oci.image.layer.v1.tar+gzip" }, + }) }; +}; + /** * The cell verdict a failed release reports: a typed `AcceptanceFailed` whose * markdown the check-run renders, one bullet line per outcome. @@ -47,7 +58,7 @@ describe("contextful-release", () => { live!.idempotencyKey!({ payload: { ...base, ref: "refs/tags/v0.5.0" } })); }); - it.effect("builds and stages a dry-run cell without publishing", () => { + it.effect("builds and stages a dry-run Linux cell and its platform image without publishing", () => { const profile = "contextful-control"; const target = "aarch64-unknown-linux-musl"; const stem = `contextful-control-0.5.0-${target}`; @@ -56,16 +67,21 @@ describe("contextful-release", () => { const { layer, handles } = makeCFRuntimeTest({ sandboxProgram: { "sha256sum": { exitCode: 0, stdout: `${"a".repeat(64)} asset\n` } }, sandboxFiles: { [`/workspace/contextful/dist/${stem}.release.json`]: manifest, - [`/workspace/contextful/dist/${stem}.tar.gz.sha256`]: `${"a".repeat(64)} ${stem}.tar.gz\n` }, + [`/workspace/contextful/dist/${stem}.tar.gz.sha256`]: `${"a".repeat(64)} ${stem}.tar.gz\n`, + ...imageFiles(profile, target) }, }); return Effect.gen(function* () { const result = yield* contextfulReleaseCell.run({ repo, tag: "v0.5.0", sha, profile, target, releaseId: 0, dryRun: true }); expect(result.assets).toEqual([`${stem}.tar.gz`, `${stem}.tar.gz.sha256`, `${stem}.cdx.json`]); - expect(result.imageLayer).toBeUndefined(); - expect(handles.sandbox.execs.map((exec) => exec.command)).toEqual( - expect.arrayContaining([expect.stringContaining("release --builder zigbuild")])); - expect(handles.artifact.uploads).toHaveLength(3); + expect(result.imageLayer).toEqual({ name: "contextful-control-0.5.0-linux-arm64.layer.tar.gz", + imageName: "contextful-control-0.5.0-linux-arm64.image.json", arch: "arm64", size: 3, + digest: "c".repeat(64), diffId: "d".repeat(64) }); + expect(handles.sandbox.execs.map((exec) => exec.command)).toEqual(expect.arrayContaining([ + expect.stringContaining("release --builder zigbuild"), + expect.stringContaining(`image --profile ${profile} --version 0.5.0 --binary ${target}=target/release-artifacts/${target}/release/contextful`), + ])); + expect(handles.artifact.uploads).toHaveLength(5); expect(handles.executions.steps.some((entry) => entry.name.startsWith("publish-"))).toBe(false); }).pipe(Effect.provide(layer)); }); @@ -79,7 +95,8 @@ describe("contextful-release", () => { return { profile, target, name, runtime: makeCFRuntimeTest({ sandboxProgram: { "sha256sum": { exitCode: 0, stdout: `${"a".repeat(64)} asset\n` } }, sandboxFiles: { [`/workspace/contextful/dist/${name}.release.json`]: manifest, - [`/workspace/contextful/dist/${name}.tar.gz.sha256`]: `${"a".repeat(64)} ${name}.tar.gz\n` }, + [`/workspace/contextful/dist/${name}.tar.gz.sha256`]: `${"a".repeat(64)} ${name}.tar.gz\n`, + ...imageFiles(profile, target) }, }) }; }; /** A durable owner whose build runs for `polls` observations, then exits 0. */ @@ -291,11 +308,12 @@ describe("contextful-release", () => { .map((target) => ({ profile, target }))); const FORMULA_ASSETS = ["contextful-control.rb", "contextful-edge.rb", "contextful-full.rb", "contextful.rb", "SHA256SUMS", "install.sh"]; const cellOutput = (cell: Cell) => { - const imageLayer = cell.target === "x86_64-unknown-linux-musl" ? { - name: `${cell.profile}-0.5.0-linux-amd64.layer.tar.gz`, size: 3, - archiveName: `${cell.profile}-0.5.0-linux-amd64.oci.tar`, + const arch = { "x86_64-unknown-linux-musl": "amd64", "aarch64-unknown-linux-musl": "arm64" }[cell.target]; + const imageLayer = arch === undefined ? undefined : { + name: `${cell.profile}-0.5.0-linux-${arch}.layer.tar.gz`, size: 3, + imageName: `${cell.profile}-0.5.0-linux-${arch}.image.json`, arch, digest: "a".repeat(64), diffId: "b".repeat(64), - } : undefined; + }; return JSON.stringify({ ...cell, manifest: JSON.stringify({ ...cell, archive: "archive.tar.gz", sha256: "a".repeat(64), sbom: "sbom.cdx.json" }), ...(imageLayer === undefined ? {} : { imageLayer }) }); @@ -361,7 +379,7 @@ describe("contextful-release", () => { }).pipe(Effect.provide(layer)); }); - it.effect("publishes a draft only after ten cells, formula assets, and three images succeed", () => { + it.effect("publishes a draft only after ten cells, formula assets, and three two-platform images succeed", () => { const { layer, handles } = releaseFixture(() => "success"); handles.github.tagCommits[`${repo}:v0.5.0`] = sha; return Effect.gen(function* () { @@ -369,7 +387,9 @@ describe("contextful-release", () => { expect(result).toMatchObject({ dryRun: false, releaseId: 1, cells: 10, assets: 36 }); expect(handles.github.createReleaseCalls).toHaveLength(1); expect(handles.github.createReleaseCalls[0]?.draft).toBe(true); - expect(handles.github.publishContainerImageCalls).toHaveLength(3); + expect(handles.github.publishContainerImageCalls.map((call) => call.images)).toEqual( + ["contextful-control", "contextful-edge", "contextful-full"].map((profile) => + [`${profile}-0.5.0-linux-amd64.image.json`, `${profile}-0.5.0-linux-arm64.image.json`])); expect(handles.github.publishReleaseCalls).toEqual([{ repo, releaseId: 1, body: "Release v0.5.0\n\n## Consumer compatibility\n" }]); expect(handles.github.releases[`${repo}:v0.5.0`]?.draft).toBe(false); @@ -522,7 +542,7 @@ describe("contextful-release", () => { // The parent queues behind other releases, never behind lean-pool gates. expect(contextfulRelease.sandboxImage).toBe("release"); expect(contextfulReleaseCell.sandboxImage).toBe("release"); - expect(contextfulRelease.version).toBe("1.3.0"); + expect(contextfulRelease.version).toBe("1.4.0"); }); it.effect("window steps share no name with the fixed batches a release in flight checkpointed", () => { diff --git a/runs/contextful-release.ts b/runs/contextful-release.ts index e22fa093..0f00ced7 100644 --- a/runs/contextful-release.ts +++ b/runs/contextful-release.ts @@ -14,6 +14,11 @@ const SHA = /^[a-f0-9]{40}$/; const DRY_RUN_TAG_PREFIX = "refs/tags/dry-run/v"; const PROFILES = ["contextful-control", "contextful-edge", "contextful-full"] as const; const LINUX = ["x86_64-unknown-linux-musl", "aarch64-unknown-linux-musl"] as const; +/** Each Linux target's image platform, in image-index order. */ +const IMAGE_ARCH: Readonly> = { + "x86_64-unknown-linux-musl": "amd64", + "aarch64-unknown-linux-musl": "arm64", +}; const DARWIN = ["x86_64-apple-darwin", "aarch64-apple-darwin"] as const; /** The build command's own deadline; the supervisor ends the process at it. */ const RELEASE_BUILD_TIMEOUT_SEC = 14400; @@ -116,7 +121,7 @@ const ReleaseCellOutput = Schema.Struct({ manifest: Schema.String, assets: Schema.Array(Schema.String), imageLayer: Schema.optional(Schema.Struct({ - name: Schema.String, archiveName: Schema.String, + name: Schema.String, imageName: Schema.String, arch: Schema.String, size: Schema.Number, digest: Schema.String, diffId: Schema.String, })), }); @@ -150,7 +155,7 @@ const requireValidRequest = (repo: string, tag: string, sha: string): void => { } }; -type ImageLayer = { name: string; archiveName: string; size: number; digest: string; diffId: string }; +type ImageLayer = { name: string; imageName: string; arch: string; size: number; digest: string; diffId: string }; type StagedAsset = { name: string; sha256: string; contentType: string }; type Collected = | { kind: "lost" } @@ -199,29 +204,37 @@ const collect = (input: { profile: string; target: string }, version: string, ws yield* artifact.upload({ name: asset, path, container: ws.container }); assets.push({ name: asset, sha256, contentType: contentType(asset) }); } - if (input.target !== "x86_64-unknown-linux-musl") return { kind: "staged", manifest, assets } as Collected; - const layerName = `${input.profile}-${version}-linux-amd64.layer.tar.gz`; - const archiveName = `${input.profile}-${version}-linux-amd64.oci.tar`; - const base = `${ws.dir}/dist/${input.profile}-${version}-image`; - const binary = `${ws.dir}/target/release-artifacts/${input.target}/release/contextful`; + const arch = IMAGE_ARCH[input.target]; + if (arch === undefined) return { kind: "staged", manifest, assets } as Collected; + const layerName = `${input.profile}-${version}-linux-${arch}.layer.tar.gz`; + const imageName = `${input.profile}-${version}-linux-${arch}.image.json`; + const binary = `target/release-artifacts/${input.target}/release/contextful`; const footprint = yield* sandbox.exec({ container: ws.container, cwd: ws.dir, command: ["cargo", "run", "--locked", "-q", "-p", "contextful-ci", "--", "footprint", - "--profile", input.profile, binary], timeoutSec: 600 }); + "--profile", input.profile, `${ws.dir}/${binary}`], timeoutSec: 600 }); if (footprint.exitCode !== 0) return yield* Effect.die(new Error(`image footprint failed for ${input.profile}`)); + // `assurance.build.release-image`: this platform's layer, config and + // manifest, and the manifest's index entry; the parent composes the index. const built = yield* sandbox.exec({ container: ws.container, cwd: ws.dir, - command: ["/opt/build-oci-layer.sh", binary, `${base}/root`, - `${ws.dir}/dist/${layerName}`, `${ws.dir}/dist/${archiveName}`, version], + command: ["cargo", "run", "--locked", "-q", "-p", "contextful-ci", "--", "image", + "--profile", input.profile, "--version", version, "--binary", `${input.target}=${binary}`, + "--ca-bundle", "/etc/ssl/certs/ca-certificates.crt", "--out", "dist"], timeoutSec: 600 }); - const lines = built.stdout.trim().split("\n"); - const diffId = lines[0]?.match(/^([a-f0-9]{64})\s/)?.[1]; - const digest = lines[1]?.match(/^([a-f0-9]{64})\s/)?.[1]; - const size = Number(lines[2]); - if (built.exitCode !== 0 || diffId === undefined || digest === undefined || !Number.isSafeInteger(size)) { - return yield* Effect.die(new Error("OCI image layer failed")); + if (built.exitCode !== 0) return yield* Effect.die(new Error(`OCI image failed for ${input.profile}/${input.target}`)); + const image = JSON.parse(yield* readOutput(`${ws.dir}/dist/${imageName}`)) as { + target?: string; platform?: { architecture?: string }; + layer?: { digest?: string; size?: number; diffId?: string; path?: string }; + }; + const digest = image.layer?.digest?.match(/^sha256:([a-f0-9]{64})$/)?.[1]; + const diffId = image.layer?.diffId?.match(/^sha256:([a-f0-9]{64})$/)?.[1]; + const size = image.layer?.size; + if (image.target !== input.target || image.platform?.architecture !== arch || image.layer?.path !== layerName || + digest === undefined || diffId === undefined || size === undefined || !Number.isSafeInteger(size)) { + return yield* Effect.die(new Error("OCI image descriptor does not match its inputs")); } yield* artifact.upload({ name: layerName, path: `${ws.dir}/dist/${layerName}`, container: ws.container }); - yield* artifact.upload({ name: archiveName, path: `${ws.dir}/dist/${archiveName}`, container: ws.container }); - return { kind: "staged", manifest, assets, imageLayer: { name: layerName, archiveName, size, digest, diffId } } as Collected; + yield* artifact.upload({ name: imageName, path: `${ws.dir}/dist/${imageName}`, container: ws.container }); + return { kind: "staged", manifest, assets, imageLayer: { name: layerName, imageName, arch, size, digest, diffId } } as Collected; }); /** @@ -246,8 +259,9 @@ const publishOrRateLimit = ( export const contextfulReleaseCell = defineRun({ name: "contextful-release-cell", + // 1.3.0 — every Linux cell stages its platform's image descriptor and layer. // 1.2.0 — a rate-limited asset upload waits out its retry hint durably. - version: "1.2.0", + version: "1.3.0", sandboxImage: "release", inputs: ReleaseCellInput, outputs: ReleaseCellOutput, @@ -362,10 +376,11 @@ export const contextfulReleaseFormula = defineRun({ export const contextfulRelease = defineRun({ name: "contextful-release", + // 1.4.0 — each profile's image is an index of its amd64 and arm64 manifests. // 1.3.0 — a `dry-run/v` tag push starts a dry run of `v`. // 1.2.0 — publishes the formula step's release notes as the release body. // 1.1.0 — release pool, sliding window of cells. - version: "1.3.0", + version: "1.4.0", // The parent makes no sandbox call and hands its admission row off after // its first spawn. On the release pool it queues only behind other // releases, never behind pull-request gates on the lean pool. @@ -563,26 +578,27 @@ export const contextfulRelease = defineRun({ } const { assets: formulaAssets, notes } = JSON.parse(formulaChildren[0].summaryJson ?? "{}") as { assets?: string[]; notes?: string }; if (formulaAssets?.length !== 6) return yield* Effect.die(new Error("release formula output is incomplete")); - const imageChildren = children.filter((child) => { - const output = JSON.parse(child.summaryJson ?? "{}") as { target?: string }; - return output.target === "x86_64-unknown-linux-musl"; + const version = input.tag.slice(1); + const images = children.flatMap((child) => { + const output = JSON.parse(child.summaryJson ?? "{}") as { + profile?: string; target?: string; imageLayer?: { imageName?: string; arch?: string }; + }; + const arch = output.target === undefined ? undefined : IMAGE_ARCH[output.target]; + return arch === undefined ? [] : [{ executionId: child.executionId, profile: output.profile, arch, + imageName: output.imageLayer?.arch === arch ? output.imageLayer.imageName : undefined }]; }); - if (imageChildren.length !== 3 || imageChildren.some((child) => { - const output = JSON.parse(child.summaryJson ?? "{}") as { profile?: string; imageLayer?: { archiveName?: string } }; - return output.imageLayer?.archiveName !== `${output.profile}-${input.tag.slice(1)}-linux-amd64.oci.tar`; - })) return yield* Effect.die(new Error("release needs three Linux amd64 OCI image archives")); + if (images.length !== PROFILES.length * LINUX.length || images.some((image) => + image.imageName !== `${image.profile}-${version}-linux-${image.arch}.image.json`)) { + return yield* Effect.die(new Error("release needs a Linux amd64 and arm64 image descriptor for every profile")); + } if (dryRun) return { tag: input.tag, sha: tag.commitSha, dryRun: true, releaseId: 0, cells: CELLS.length, assets: CELLS.length * 3 + formulaAssets.length }; - for (const child of imageChildren) { - const output = JSON.parse(child.summaryJson!) as { - profile: string; imageLayer?: { name: string; archiveName: string; size: number; digest: string; diffId: string }; - }; - if (output.imageLayer === undefined) return yield* Effect.die(new Error("release image layer is missing")); - yield* step(`publish-image-${output.profile}`, () => github.publishContainerImage({ - repo: input.repo, profile: output.profile, version: input.tag.slice(1), - sourceExecutionId: child.executionId, - layerArtifactName: output.imageLayer!.name, layerSize: output.imageLayer!.size, - layerDigest: output.imageLayer!.digest, diffId: output.imageLayer!.diffId, + for (const profile of PROFILES) { + const platforms = LINUX.map((target) => images.find((image) => + image.profile === profile && image.arch === IMAGE_ARCH[target])!); + yield* step(`publish-image-${profile}`, () => github.publishContainerImage({ + repo: input.repo, profile, version, + platforms: platforms.map((image) => ({ sourceExecutionId: image.executionId, imageArtifactName: image.imageName! })), })); } const verified = yield* step("verify-tag-for-publish", () => github.tagTarget({ repo: input.repo, tag: input.tag }));