diff --git a/infra/Dockerfile.sandbox-release b/infra/Dockerfile.sandbox-release index dbe7864..94b42a3 100644 --- a/infra/Dockerfile.sandbox-release +++ b/infra/Dockerfile.sandbox-release @@ -27,8 +27,6 @@ RUN python3 -m venv /opt/zig && \ # Text-only TAPI records only the macOS system symbols referenced by # iana-time-zone. The runtime framework comes from macOS itself. COPY infra/darwin-stubs /opt/darwin-stubs -COPY infra/build-oci-layer.sh /opt/build-oci-layer.sh -COPY infra/build-oci-layout.py /opt/build-oci-layout.py ENV CARGO_TARGET_AARCH64_APPLE_DARWIN_RUSTFLAGS="-C link-arg=-F/opt/darwin-stubs" \ CARGO_TARGET_X86_64_APPLE_DARWIN_RUSTFLAGS="-C link-arg=-F/opt/darwin-stubs" \ CARGO_BUILD_JOBS=4 diff --git a/infra/build-oci-layer.sh b/infra/build-oci-layer.sh deleted file mode 100755 index a1245e1..0000000 --- a/infra/build-oci-layer.sh +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/sh -set -eu - -binary=$1 -root=$2 -output=$3 -image=$4 -version=$5 - -file "$binary" | grep -q 'ELF 64-bit.*x86-64' -if readelf -l "$binary" | grep -q INTERP; then - echo 'OCI release binary has a dynamic interpreter' >&2 - exit 1 -fi - -mkdir -p "$root/usr/local/bin" "$root/etc/ssl/certs" "$root/data" -cp "$binary" "$root/usr/local/bin/contextful" -cp /etc/ssl/certs/ca-certificates.crt "$root/etc/ssl/certs/ca-certificates.crt" -chown -R 0:0 "$root" -chown 65532:65532 "$root/data" -tar --sort=name --mtime=@0 --numeric-owner -C "$root" -cf "${output%.gz}" . -gzip -n -c "${output%.gz}" > "$output" -python3 /opt/build-oci-layout.py "$output" "${output%.gz}" "$image" "$version" -sha256sum "${output%.gz}" "$output" -stat -c %s "$output" diff --git a/infra/build-oci-layer.test.sh b/infra/build-oci-layer.test.sh deleted file mode 100755 index ea8d737..0000000 --- a/infra/build-oci-layer.test.sh +++ /dev/null @@ -1,48 +0,0 @@ -#!/bin/sh -set -eu - -work=$(mktemp -d) -trap 'rm -rf "$work"' EXIT -chmod 755 "$work" -cat > "$work/main.c" <<'SOURCE' -int main(void) { return 0; } -SOURCE -zig cc -target x86_64-linux-musl -static -o "$work/contextful" "$work/main.c" -/opt/build-oci-layer.sh "$work/contextful" "$work/root" "$work/layer.tar.gz" "$work/image.oci.tar" "0.5.0" > "$work/result" -[ "$(wc -l < "$work/result")" -eq 3 ] -file "$work/contextful" | grep -q 'ELF 64-bit.*x86-64' -tar --numeric-owner -tvf "$work/layer.tar" | grep -q '65532/65532.* ./data/' -mkdir "$work/extracted" -tar -C "$work/extracted" -xf "$work/layer.tar" -setpriv --reuid 65532 --regid 65532 --clear-groups -- touch "$work/extracted/data/write-smoke" -[ -f "$work/extracted/data/write-smoke" ] -tar -tf "$work/image.oci.tar" | grep -q '^index.json$' -tar -tf "$work/image.oci.tar" | grep -q '^oci-layout$' -tar -xOf "$work/image.oci.tar" index.json | grep -q '"architecture":"amd64"' -python3 - "$work/image.oci.tar" <<'PY' -import gzip -import hashlib -import json -import sys -import tarfile - -with tarfile.open(sys.argv[1]) as archive: - def read(name): - return archive.extractfile(name).read() - - index = json.loads(read("index.json")) - descriptor = index["manifests"][0] - assert descriptor["annotations"]["org.opencontainers.image.ref.name"] == "0.5.0" - manifest_bytes = read(f"blobs/sha256/{descriptor['digest'][7:]}") - assert hashlib.sha256(manifest_bytes).hexdigest() == descriptor["digest"][7:] - manifest = json.loads(manifest_bytes) - config_bytes = read(f"blobs/sha256/{manifest['config']['digest'][7:]}") - assert hashlib.sha256(config_bytes).hexdigest() == manifest["config"]["digest"][7:] - config = json.loads(config_bytes) - assert config["config"]["User"] == "65532:65532" - assert config["config"]["Entrypoint"] == ["/usr/local/bin/contextful"] - layer = manifest["layers"][0] - layer_bytes = read(f"blobs/sha256/{layer['digest'][7:]}") - assert hashlib.sha256(layer_bytes).hexdigest() == layer["digest"][7:] - assert hashlib.sha256(gzip.decompress(layer_bytes)).hexdigest() == config["rootfs"]["diff_ids"][0][7:] -PY diff --git a/infra/build-oci-layout.py b/infra/build-oci-layout.py deleted file mode 100755 index b6472fa..0000000 --- a/infra/build-oci-layout.py +++ /dev/null @@ -1,77 +0,0 @@ -#!/usr/bin/env python3 -"""Package one static Contextful layer as an OCI image layout archive.""" - -import hashlib -import json -import pathlib -import shutil -import sys -import tarfile -import tempfile - -layer = pathlib.Path(sys.argv[1]) -uncompressed = pathlib.Path(sys.argv[2]) -output = pathlib.Path(sys.argv[3]) -version = sys.argv[4] - - -def encoded(value): - return json.dumps(value, separators=(",", ":")).encode("utf-8") - - -def sha256(path): - digest = hashlib.sha256() - with path.open("rb") as stream: - for chunk in iter(lambda: stream.read(1024 * 1024), b""): - digest.update(chunk) - return digest.hexdigest() - - -layer_digest = sha256(layer) -diff_id = sha256(uncompressed) -config = encoded({ - "architecture": "amd64", "os": "linux", - "config": { - "User": "65532:65532", "WorkingDir": "/data", - "Entrypoint": ["/usr/local/bin/contextful"], "Cmd": ["--version"], - "Volumes": {"/data": {}}, - }, - "rootfs": {"type": "layers", "diff_ids": [f"sha256:{diff_id}"]}, - "history": [{"created_by": "contextful release"}], -}) -config_digest = hashlib.sha256(config).hexdigest() -manifest = encoded({ - "schemaVersion": 2, "mediaType": "application/vnd.oci.image.manifest.v1+json", - "config": {"mediaType": "application/vnd.oci.image.config.v1+json", - "digest": f"sha256:{config_digest}", "size": len(config)}, - "layers": [{"mediaType": "application/vnd.oci.image.layer.v1.tar+gzip", - "digest": f"sha256:{layer_digest}", "size": layer.stat().st_size}], -}) -manifest_digest = hashlib.sha256(manifest).hexdigest() -index = encoded({ - "schemaVersion": 2, - "manifests": [{"mediaType": "application/vnd.oci.image.manifest.v1+json", - "digest": f"sha256:{manifest_digest}", "size": len(manifest), - "platform": {"architecture": "amd64", "os": "linux"}, - "annotations": {"org.opencontainers.image.ref.name": version}}], -}) - -with tempfile.TemporaryDirectory() as temporary: - root = pathlib.Path(temporary) - blobs = root / "blobs" / "sha256" - blobs.mkdir(parents=True) - shutil.copyfile(layer, blobs / layer_digest) - (blobs / config_digest).write_bytes(config) - (blobs / manifest_digest).write_bytes(manifest) - (root / "index.json").write_bytes(index) - (root / "oci-layout").write_bytes(encoded({"imageLayoutVersion": "1.0.0"})) - with tarfile.open(output, "w") as archive: - for path in sorted(root.rglob("*")): - info = archive.gettarinfo(str(path), arcname=str(path.relative_to(root))) - info.uid = info.gid = info.mtime = 0 - info.uname = info.gname = "" - if path.is_file(): - with path.open("rb") as stream: - archive.addfile(info, stream) - else: - archive.addfile(info) diff --git a/packages/core/src/fakes/github-fake.ts b/packages/core/src/fakes/github-fake.ts index feb584b..e815434 100644 --- a/packages/core/src/fakes/github-fake.ts +++ b/packages/core/src/fakes/github-fake.ts @@ -129,7 +129,7 @@ export type GithubFakeState = { /** Every `createRelease` call, in order — lets a test assert a release published. */ readonly createReleaseCalls: CreateRelease[]; readonly publishReleaseCalls: Array<{ repo: string; releaseId: number }>; - readonly publishContainerImageCalls: Array<{ repo: string; profile: string; version: string }>; + readonly publishContainerImageCalls: Array<{ repo: string; profile: string; version: string; images: string[] }>; readonly releases: Record; readonly tagCommits: Record; /** Every label ADD, in order. */ @@ -536,7 +536,8 @@ export const makeGithubFake = ( downloadUrl: `https://github.com/${req.repo}/releases/download/${req.releaseId}/${req.artifactName}`, }), publishContainerImage: (req) => Effect.sync(() => { - state.publishContainerImageCalls.push({ repo: req.repo, profile: req.profile, version: req.version }); + state.publishContainerImageCalls.push({ repo: req.repo, profile: req.profile, version: req.version, + images: req.platforms.map((platform) => platform.imageArtifactName) }); }), }; diff --git a/packages/core/src/services/github.ts b/packages/core/src/services/github.ts index f4246ff..033c53a 100644 --- a/packages/core/src/services/github.ts +++ b/packages/core/src/services/github.ts @@ -303,15 +303,20 @@ export type ReleaseAssetResult = { readonly downloadUrl: string; }; +/** + * A multi-platform release image: each platform's `.image.json` descriptor, + * staged by the release cell that built it, pushed as one image index under + * `version`. + */ export type PublishContainerImage = { readonly repo: string; readonly profile: string; readonly version: string; - readonly sourceExecutionId: string; - readonly layerArtifactName: string; - readonly layerSize: number; - readonly layerDigest: string; - readonly diffId: string; + /** In index order, amd64 before arm64. */ + readonly platforms: ReadonlyArray<{ + readonly sourceExecutionId: string; + readonly imageArtifactName: string; + }>; }; /** diff --git a/packages/runtime-cf/src/ghcr-push.test.ts b/packages/runtime-cf/src/ghcr-push.test.ts index 49887ce..be23608 100644 --- a/packages/runtime-cf/src/ghcr-push.test.ts +++ b/packages/runtime-cf/src/ghcr-push.test.ts @@ -1,106 +1,122 @@ import { describe, expect, it } from "vitest"; -import { classifyGhcrFailure, pushGhcrImage } from "./ghcr-push"; +import { classifyGhcrFailure, pushGhcrImageIndex, type GhcrPlatformImage } from "./ghcr-push"; -describe("pushGhcrImage", () => { - it("publishes OCI blobs and a versioned manifest without sending credentials in image bytes", async () => { - const requests: Array<{ url: string; method: string; auth: string | null; body: Uint8Array }> = []; - const fetchImpl: typeof fetch = async (input, init) => { - const request = new Request(input, init); - const url = new URL(request.url); - const bytes = request.body === null ? new Uint8Array() : new Uint8Array(await request.arrayBuffer()); - requests.push({ url: request.url, method: request.method, auth: request.headers.get("authorization"), body: bytes }); - if (url.pathname === "/token") return Response.json({ token: "registry-bearer" }); - if (request.method === "HEAD") return new Response(null, { status: 404 }); - if (request.method === "POST") return new Response(null, { status: 202, headers: { Location: `/v2/${"fractalboxdev/contextful/contextful-edge"}/blobs/uploads/session` } }); - if (request.method === "PUT") return new Response(null, { status: 201 }); - throw new Error(`unexpected ${request.method} ${request.url}`); - }; - await pushGhcrImage({ username: "bot", token: "test-pat", name: "fractalboxdev/contextful/contextful-edge", tag: "0.5.0", - layer: new Uint8Array([1, 2, 3]), layerSize: 3, - layerDigest: "a".repeat(64), diffId: "b".repeat(64), fetchImpl }); +const hex = async (bytes: Uint8Array): Promise => + Array.from(new Uint8Array(await crypto.subtle.digest("SHA-256", bytes)), (b) => b.toString(16).padStart(2, "0")).join(""); +const MANIFEST = "application/vnd.oci.image.manifest.v1+json"; +const INDEX = "application/vnd.oci.image.index.v1+json"; + +/** One platform as `contextful-ci image` describes it: keys sorted, blobs verbatim. */ +const platformImage = async (architecture: "amd64" | "arm64", layerByte: number): Promise => { + const layer = new Uint8Array([layerByte, 2, 3]); + const layerDigest = `sha256:${await hex(layer)}`; + const config = new TextEncoder().encode(JSON.stringify({ architecture, os: "linux" })); + const configDigest = `sha256:${await hex(config)}`; + const manifest = new TextEncoder().encode(JSON.stringify({ + config: { digest: configDigest, mediaType: "application/vnd.oci.image.config.v1+json", size: config.byteLength }, + layers: [{ digest: layerDigest, mediaType: "application/vnd.oci.image.layer.v1.tar+gzip", size: layer.byteLength }], + mediaType: MANIFEST, schemaVersion: 2, + })); + const manifestDigest = `sha256:${await hex(manifest)}`; + return { layerDigest, layerSize: layer.byteLength, openLayer: async () => layer, configDigest, config, + manifestDigest, manifest, entry: { digest: manifestDigest, mediaType: MANIFEST, + platform: { architecture, os: "linux", ...(architecture === "arm64" ? { variant: "v8" } : {}) }, + size: manifest.byteLength } }; +}; + +type Recorded = { url: string; method: string; auth: string | null; type: string | null; body: Uint8Array }; +/** A registry that has nothing and accepts every upload, recording each request. */ +const emptyRegistry = (requests: Recorded[], location = "/v2/o/r/p/blobs/uploads/session"): typeof fetch => + async (input, init) => { + const request = new Request(input, init); + const url = new URL(request.url); + const body = request.body === null ? new Uint8Array() : new Uint8Array(await request.arrayBuffer()); + requests.push({ url: request.url, method: request.method, auth: request.headers.get("authorization"), + type: request.headers.get("content-type"), body }); + if (url.pathname === "/token") return Response.json({ token: "registry-bearer" }); + if (request.method === "HEAD") return new Response(null, { status: 404 }); + if (request.method === "POST") return new Response(null, { status: 202, headers: { Location: location } }); + if (request.method === "PUT") return new Response(null, { status: 201 }); + throw new Error(`unexpected ${request.method} ${request.url}`); + }; + +describe("pushGhcrImageIndex", () => { + it("pushes each platform's blobs and manifest by digest, then the index by tag", async () => { + const requests: Recorded[] = []; + const platforms = [await platformImage("amd64", 1), await platformImage("arm64", 9)]; + await pushGhcrImageIndex({ username: "bot", token: "test-pat", name: "o/r/p", tag: "0.5.0", platforms, + fetchImpl: emptyRegistry(requests) }); expect(requests[0]?.auth).toBe(`Basic ${btoa("bot:test-pat")}`); - expect(requests.filter((r) => r.method === "PUT")).toHaveLength(3); - expect(requests.at(-1)?.url).toContain("/manifests/0.5.0"); - expect(requests.at(-1)?.auth).toBe("Bearer registry-bearer"); - expect(new TextDecoder().decode(requests.at(-1)!.body)).not.toContain("test-pat"); + const manifests = requests.filter((r) => r.method === "PUT" && r.url.includes("/manifests/")); + expect(manifests.map((r) => new URL(r.url).pathname)).toEqual([ + `/v2/o/r/p/manifests/${platforms[0]!.manifestDigest}`, + `/v2/o/r/p/manifests/${platforms[1]!.manifestDigest}`, + "/v2/o/r/p/manifests/0.5.0", + ]); + expect(manifests.map((r) => r.type)).toEqual([MANIFEST, MANIFEST, INDEX]); + expect(manifests[0]!.body).toEqual(platforms[0]!.manifest); + expect(new TextDecoder().decode(manifests[2]!.body)).toBe(`{"manifests":[${ + platforms.map((p) => JSON.stringify(p.entry)).join(",")}],"mediaType":"${INDEX}","schemaVersion":2}`); + expect(requests.every((r) => !new TextDecoder().decode(r.body).includes("test-pat"))).toBe(true); }); - it("uploads every blob as application/octet-stream, which GHCR requires, and types only the manifest", async () => { + it("uploads every blob as application/octet-stream, which GHCR requires, config verbatim", async () => { // GHCR answers 404 to a blob PUT carrying an OCI media type as its Content-Type. - const puts: Array<{ path: string; type: string | null }> = []; - const fetchImpl: typeof fetch = async (input, init) => { - const request = new Request(input, init); - const url = new URL(request.url); - if (url.pathname === "/token") return Response.json({ token: "registry-bearer" }); - if (request.method === "HEAD") return new Response(null, { status: 404 }); - if (request.method === "POST") return new Response(null, { status: 202, headers: { Location: "/v2/o/r/p/blobs/upload/1.session" } }); - puts.push({ path: url.pathname, type: request.headers.get("content-type") }); - const blobUpload = url.pathname.includes("/blobs/"); - return blobUpload && request.headers.get("content-type") !== "application/octet-stream" - ? new Response(null, { status: 404 }) : new Response(null, { status: 201 }); - }; - await pushGhcrImage({ username: "bot", token: "test-pat", name: "o/r/p", tag: "0.5.0", - layer: new Uint8Array([1, 2, 3]), layerSize: 3, - layerDigest: "a".repeat(64), diffId: "b".repeat(64), fetchImpl }); - expect(puts.filter((put) => put.path.includes("/blobs/")).map((put) => put.type)) - .toEqual(["application/octet-stream", "application/octet-stream"]); - expect(puts.at(-1)).toEqual({ path: "/v2/o/r/p/manifests/0.5.0", type: "application/vnd.oci.image.manifest.v1+json" }); + const requests: Recorded[] = []; + const platform = await platformImage("amd64", 1); + await pushGhcrImageIndex({ username: "bot", token: "test-pat", name: "o/r/p", tag: "0.5.0", + platforms: [platform], fetchImpl: emptyRegistry(requests) }); + const blobs = requests.filter((r) => r.method === "PUT" && !r.url.includes("/manifests/")); + expect(blobs.map((r) => r.type)).toEqual(["application/octet-stream", "application/octet-stream"]); + expect(blobs.map((r) => new URL(r.url).searchParams.get("digest"))).toEqual([platform.layerDigest, platform.configDigest]); + expect(blobs[1]!.body).toEqual(platform.config); }); - it("keeps an existing matching image tag and refuses a conflicting tag", async () => { - let manifestDigest: string | undefined; - const requests: string[] = []; + it("keeps an existing matching index tag and refuses a conflicting one", async () => { + let indexDigest: string | undefined; + const requests: Recorded[] = []; + const registry = emptyRegistry(requests); const fetchImpl: typeof fetch = async (input, init) => { const request = new Request(input, init); - requests.push(`${request.method} ${request.url}`); - if (new URL(request.url).pathname === "/token") return Response.json({ token: "registry-bearer" }); - if (request.method === "HEAD" && request.url.endsWith("/manifests/0.5.0")) { - return manifestDigest === undefined ? new Response(null, { status: 404 }) : - new Response(null, { status: 200, headers: { "Docker-Content-Digest": manifestDigest } }); - } - if (request.method === "HEAD") return new Response(null, { status: 404 }); - if (request.method === "POST") return new Response(null, { status: 202, - headers: { Location: "/v2/fractalboxdev/contextful/contextful-edge/blobs/uploads/session" } }); - if (request.method === "PUT" && request.url.endsWith("/manifests/0.5.0")) { - const hash = new Uint8Array(await crypto.subtle.digest("SHA-256", await request.arrayBuffer())); - manifestDigest = `sha256:${Array.from(hash, (byte) => byte.toString(16).padStart(2, "0")).join("")}`; - return new Response(null, { status: 201 }); + if (request.url.endsWith("/manifests/0.5.0")) { + if (request.method === "HEAD") { + return indexDigest === undefined ? new Response(null, { status: 404 }) : + new Response(null, { status: 200, headers: { "Docker-Content-Digest": indexDigest } }); + } + indexDigest = `sha256:${await hex(new Uint8Array(await request.clone().arrayBuffer()))}`; } - if (request.method === "PUT") return new Response(null, { status: 201 }); - throw new Error(`unexpected ${request.method} ${request.url}`); + return registry(request); }; - const opts = { username: "bot", token: "test-pat", name: "fractalboxdev/contextful/contextful-edge", - tag: "0.5.0", layer: new Uint8Array([1]), layerSize: 1, - layerDigest: "a".repeat(64), diffId: "b".repeat(64), fetchImpl }; - await pushGhcrImage(opts); + const opts = { username: "bot", token: "test-pat", name: "o/r/p", tag: "0.5.0", + platforms: [await platformImage("amd64", 1), await platformImage("arm64", 9)], fetchImpl }; + await pushGhcrImageIndex(opts); const afterPublish = requests.length; - await pushGhcrImage(opts); - expect(requests.slice(afterPublish)).toHaveLength(2); - manifestDigest = "sha256:" + "c".repeat(64); - await expect(pushGhcrImage(opts)).rejects.toThrow("already names a different manifest"); + await pushGhcrImageIndex(opts); + expect(requests.slice(afterPublish)).toHaveLength(1); + indexDigest = `sha256:${"c".repeat(64)}`; + await expect(pushGhcrImageIndex(opts)).rejects.toThrow("already names a different manifest"); }); - it("does not forward the registry bearer token to a cross-host upload URL", async () => { - const uploads: Request[] = []; - const fetchImpl: typeof fetch = async (input, init) => { - const request = new Request(input, init); - const url = new URL(request.url); - if (url.pathname === "/token") return Response.json({ token: "registry-bearer" }); - if (request.method === "HEAD") return new Response(null, { status: 404 }); - if (request.method === "POST") return new Response(null, { status: 202, - headers: { Location: "https://upload.example.test/blob?signature=test" } }); - if (request.method === "PUT" && url.hostname === "upload.example.test") { - uploads.push(request); - return new Response(null, { status: 201 }); - } - if (request.method === "PUT") return new Response(null, { status: 201 }); - throw new Error(`unexpected ${request.method} ${request.url}`); - }; - await pushGhcrImage({ username: "bot", token: "test-pat", name: "fractalboxdev/contextful/contextful-edge", - tag: "0.5.0", layer: new Uint8Array([1]), layerSize: 1, - layerDigest: "a".repeat(64), diffId: "b".repeat(64), fetchImpl }); + it("refuses a platform whose blobs differ from their digests or whose entry names another manifest", async () => { + const good = await platformImage("amd64", 1); + const push = (platform: GhcrPlatformImage) => pushGhcrImageIndex({ username: "bot", token: "t", name: "o/r/p", + tag: "0.5.0", platforms: [platform], fetchImpl: emptyRegistry([]) }); + await expect(push({ ...good, config: new TextEncoder().encode("{}") })).rejects.toThrow("invalid OCI image"); + await expect(push({ ...good, layerDigest: `sha256:${"e".repeat(64)}` })).rejects.toThrow("invalid OCI image"); + await expect(push({ ...good, entry: { ...good.entry, digest: `sha256:${"e".repeat(64)}` } })).rejects.toThrow("invalid OCI image"); + }); + + it("opens a layer only when the registry lacks it, and never forwards the bearer to a cross-host upload", async () => { + const requests: Recorded[] = []; + let opened = 0; + const platform = await platformImage("amd64", 1); + await pushGhcrImageIndex({ username: "bot", token: "test-pat", name: "o/r/p", tag: "0.5.0", + platforms: [{ ...platform, openLayer: async () => { opened++; return platform.openLayer(); } }], + fetchImpl: emptyRegistry(requests, "https://upload.example.test/blob?signature=test") }); + expect(opened).toBe(1); + const uploads = requests.filter((r) => new URL(r.url).hostname === "upload.example.test"); expect(uploads).toHaveLength(2); - expect(uploads.every((request) => request.headers.get("authorization") === null)).toBe(true); + expect(uploads.every((r) => r.auth === null)).toBe(true); }); }); diff --git a/packages/runtime-cf/src/ghcr-push.ts b/packages/runtime-cf/src/ghcr-push.ts index f50b5de..f9dbe08 100644 --- a/packages/runtime-cf/src/ghcr-push.ts +++ b/packages/runtime-cf/src/ghcr-push.ts @@ -1,16 +1,35 @@ type BlobBody = Uint8Array | ReadableStream; -export type GhcrImage = { +const INDEX = "application/vnd.oci.image.index.v1+json"; +const MANIFEST = "application/vnd.oci.image.manifest.v1+json"; + +/** + * One platform of an image index, as `contextful-ci image` writes it: the + * config and manifest blobs verbatim, the layer they name, and the manifest's + * index entry. The push uploads each verbatim, so the index digest equals the + * one the build computed. + */ +export type GhcrPlatformImage = { + readonly layerDigest: string; + readonly layerSize: number; + /** Opened only when the registry lacks the layer. */ + readonly openLayer: () => Promise; + readonly configDigest: string; + readonly config: Uint8Array; + readonly manifestDigest: string; + readonly manifest: Uint8Array; + /** The manifest's descriptor in the index, with its `platform`. */ + readonly entry: Readonly>; +}; + +export type GhcrImageIndex = { readonly username: string; readonly token: string; /** Registry path without ghcr.io, e.g. owner/repo/profile. */ readonly name: string; readonly tag: string; - readonly layer: BlobBody; - readonly layerSize: number; - readonly layerDigest: string; - /** SHA-256 of the uncompressed tar stream. */ - readonly diffId: string; + /** In index order. */ + readonly platforms: readonly GhcrPlatformImage[]; readonly fetchImpl?: typeof fetch; }; @@ -31,14 +50,43 @@ const fixedBody = (body: BlobBody, size: number): { body: BlobBody; pump?: Promi return { body: fixed.readable, pump: body.pipeTo(fixed.writable) }; }; -/** Push a single-platform static image through the OCI Distribution API. */ -export const pushGhcrImage = async (opts: GhcrImage): Promise => { +const SHA256 = /^sha256:[a-f0-9]{64}$/; + +/** + * Refuse a platform whose blobs differ from their digests, whose manifest + * names other blobs, or whose entry describes another manifest. + */ +const verifyPlatform = async (platform: GhcrPlatformImage): Promise => { + const invalid = () => new TypeError("invalid OCI image name, tag, digest, or size"); + if (![platform.layerDigest, platform.configDigest, platform.manifestDigest].every((d) => SHA256.test(d)) || + !Number.isSafeInteger(platform.layerSize) || platform.layerSize < 0) throw invalid(); + if (`sha256:${await digest(platform.config)}` !== platform.configDigest || + `sha256:${await digest(platform.manifest)}` !== platform.manifestDigest) throw invalid(); + const manifest = JSON.parse(new TextDecoder().decode(platform.manifest)) as { + mediaType?: unknown; config?: { digest?: unknown; size?: unknown }; + layers?: ReadonlyArray<{ digest?: unknown; size?: unknown }>; + }; + if (manifest.mediaType !== MANIFEST || manifest.config?.digest !== platform.configDigest || + manifest.config.size !== platform.config.byteLength || manifest.layers?.length !== 1 || + manifest.layers[0]?.digest !== platform.layerDigest || manifest.layers[0].size !== platform.layerSize) throw invalid(); + const { entry } = platform; + const os = (entry.platform as { os?: unknown } | undefined)?.os; + if (entry.mediaType !== MANIFEST || entry.digest !== platform.manifestDigest || + entry.size !== platform.manifest.byteLength || os !== "linux") throw invalid(); +}; + +/** + * Push a multi-platform image through the OCI Distribution API: each + * platform's blobs and manifest by digest, then one image index under `tag`. + * The index is compact JSON with its keys in sorted order, the form + * `contextful-ci image` writes, so its digest is the build's. + */ +export const pushGhcrImageIndex = async (opts: GhcrImageIndex): Promise => { if (!/^[a-z0-9]+(?:[._/-][a-z0-9]+)*$/.test(opts.name) || - !/^[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}$/.test(opts.tag) || - !/^[a-f0-9]{64}$/.test(opts.layerDigest) || !/^[a-f0-9]{64}$/.test(opts.diffId) || - !Number.isSafeInteger(opts.layerSize) || opts.layerSize < 0) { + !/^[A-Za-z0-9_][A-Za-z0-9_.-]{0,127}$/.test(opts.tag) || opts.platforms.length === 0) { throw new TypeError("invalid OCI image name, tag, digest, or size"); } + for (const platform of opts.platforms) await verifyPlatform(platform); const doFetch = opts.fetchImpl ?? fetch; const scope = `repository:${opts.name}:pull,push`; const tokenUrl = new URL("https://ghcr.io/token"); @@ -52,33 +100,21 @@ export const pushGhcrImage = async (opts: GhcrImage): Promise => { if (!token) throw new Error("GHCR returned no scoped registry token"); const auth = { Authorization: `Bearer ${token}` }; const base = `https://ghcr.io/v2/${opts.name}`; - const config = new TextEncoder().encode(JSON.stringify({ - architecture: "amd64", os: "linux", - config: { User: "65532:65532", WorkingDir: "/data", - Entrypoint: ["/usr/local/bin/contextful"], Cmd: ["--version"], Volumes: { "/data": {} } }, - rootfs: { type: "layers", diff_ids: [`sha256:${opts.diffId}`] }, - history: [{ created_by: "contextful release" }], + const index = new TextEncoder().encode(JSON.stringify({ + manifests: opts.platforms.map((platform) => platform.entry), mediaType: INDEX, schemaVersion: 2, })); - const configDigest = await digest(config); - const manifest = new TextEncoder().encode(JSON.stringify({ - schemaVersion: 2, mediaType: "application/vnd.oci.image.manifest.v1+json", - config: { mediaType: "application/vnd.oci.image.config.v1+json", digest: `sha256:${configDigest}`, size: config.byteLength }, - layers: [{ mediaType: "application/vnd.oci.image.layer.v1.tar+gzip", digest: `sha256:${opts.layerDigest}`, size: opts.layerSize }], - })); - const manifestDigest = `sha256:${await digest(manifest)}`; - const manifestUrl = `${base}/manifests/${encodeURIComponent(opts.tag)}`; - const existing = await doFetch(manifestUrl, { - method: "HEAD", headers: { ...auth, Accept: "application/vnd.oci.image.manifest.v1+json" }, - }); + const indexDigest = `sha256:${await digest(index)}`; + const tagUrl = `${base}/manifests/${encodeURIComponent(opts.tag)}`; + const existing = await doFetch(tagUrl, { method: "HEAD", headers: { ...auth, Accept: INDEX } }); if (existing.ok) { - if (existing.headers.get("docker-content-digest") === manifestDigest) return; + if (existing.headers.get("docker-content-digest") === indexDigest) return; throw new Error("GHCR image tag already names a different manifest"); } if (existing.status !== 404) throw new Error(`GHCR manifest lookup failed with HTTP ${existing.status}`); // A blob upload is `application/octet-stream` (OCI Distribution); GHCR answers 404 to // a blob PUT typed with an image media type. Media types belong to the manifest. - const uploadBlob = async (sha256: string, body: BlobBody, size: number): Promise => { - const exists = await doFetch(`${base}/blobs/sha256:${sha256}`, { method: "HEAD", headers: auth }); + const uploadBlob = async (ref: string, open: () => Promise, size: number): Promise => { + const exists = await doFetch(`${base}/blobs/${ref}`, { method: "HEAD", headers: auth }); if (exists.ok) return; if (exists.status !== 404) throw new Error(`GHCR blob lookup failed with HTTP ${exists.status}`); const started = await doFetch(`${base}/blobs/uploads/`, { method: "POST", headers: auth }); @@ -89,8 +125,8 @@ export const pushGhcrImage = async (opts: GhcrImage): Promise => { if (uploadUrl.protocol !== "https:" || uploadUrl.username || uploadUrl.password) { throw new Error("GHCR upload location is not HTTPS"); } - uploadUrl.searchParams.set("digest", `sha256:${sha256}`); - const fixed = fixedBody(body, size); + uploadUrl.searchParams.set("digest", ref); + const fixed = fixedBody(await open(), size); const request = doFetch(uploadUrl, { method: "PUT", headers: { ...(uploadUrl.hostname === "ghcr.io" ? auth : {}), @@ -101,17 +137,21 @@ export const pushGhcrImage = async (opts: GhcrImage): Promise => { const [finished] = await Promise.all([request, fixed.pump ?? Promise.resolve()]); await requireOk(finished, "GHCR blob upload"); }; + const putManifest = async (url: string, mediaType: string, body: Uint8Array): Promise => { + const published = await doFetch(url, { method: "PUT", headers: { ...auth, "Content-Type": mediaType }, body }); + await requireOk(published, "GHCR manifest publish"); + }; - await uploadBlob(configDigest, config, config.byteLength); - await uploadBlob(opts.layerDigest, opts.layer, opts.layerSize); - const published = await doFetch(manifestUrl, { - method: "PUT", headers: { ...auth, "Content-Type": "application/vnd.oci.image.manifest.v1+json" }, body: manifest, - }); - await requireOk(published, "GHCR manifest publish"); + for (const platform of opts.platforms) { + await uploadBlob(platform.layerDigest, platform.openLayer, platform.layerSize); + await uploadBlob(platform.configDigest, async () => platform.config, platform.config.byteLength); + await putManifest(`${base}/manifests/${platform.manifestDigest}`, MANIFEST, platform.manifest); + } + await putManifest(tagUrl, INDEX, index); }; /** - * A `pushGhcrImage` failure as a GitHub API outcome: the registry's HTTP status + * A `pushGhcrImageIndex` failure as a GitHub API outcome: the registry's HTTP status * from the failing operation's message, its retry class, and the message itself. * A network failure (no status) is transient; a refused input or a conflicting * tag is final. diff --git a/packages/runtime-cf/src/github-live.ts b/packages/runtime-cf/src/github-live.ts index bbcf7a6..c49ff54 100644 --- a/packages/runtime-cf/src/github-live.ts +++ b/packages/runtime-cf/src/github-live.ts @@ -71,7 +71,7 @@ import { type WorkflowRunRef, } from "@fractalboxdev/flare-dispatch-core"; import { artifactKey } from "./artifact-r2"; -import { classifyGhcrFailure, pushGhcrImage } from "./ghcr-push"; +import { classifyGhcrFailure, pushGhcrImageIndex, type GhcrPlatformImage } from "./ghcr-push"; /** The GitHub App credentials the live `pullReview` write needs. */ export type GithubLiveConfig = { @@ -602,32 +602,69 @@ export const makeGithubLive = ( if (ghcr === undefined || artifactsBucket === undefined || executionId === undefined || db === undefined) { return yield* Effect.fail(new GitHubApiError({ status: 0, reason: "other" })); } + const arches = ["amd64", "arm64"] as const; if (req.repo !== "fractalboxdev/contextful" || !["contextful-control", "contextful-edge", "contextful-full"].includes(req.profile) || !/^[0-9]+\.[0-9]+\.[0-9]+(?:-[0-9A-Za-z.-]+)?$/.test(req.version) || - req.layerArtifactName !== `${req.profile}-${req.version}-linux-amd64.layer.tar.gz` || - !/^[a-f0-9]{64}$/.test(req.layerDigest) || !/^[a-f0-9]{64}$/.test(req.diffId)) { + req.platforms.length !== arches.length || + req.platforms.some((platform, i) => + platform.imageArtifactName !== `${req.profile}-${req.version}-linux-${arches[i]}.image.json`)) { return yield* Effect.fail(new GitHubApiError({ status: 400, reason: "other" })); } - const child = yield* Effect.tryPromise({ - try: () => db.prepare("SELECT status FROM executions WHERE id = ? AND parent_execution_id = ? AND run = ?") - .bind(req.sourceExecutionId, executionId, "contextful-release-cell").first<{ status: string }>(), - catch: () => new GitHubApiError({ status: 0, reason: "transient" }), - }); - if (child?.status !== "success") return yield* Effect.fail(new GitHubApiError({ status: 403, reason: "other" })); - const source = yield* Effect.tryPromise({ - try: () => artifactsBucket.get(artifactKey(req.sourceExecutionId, req.layerArtifactName)), - catch: () => new GitHubApiError({ status: 0, reason: "transient" }), - }); - if (source === null || source.size !== req.layerSize) { - return yield* Effect.fail(new GitHubApiError({ status: 404, reason: "other" })); + const platforms: GhcrPlatformImage[] = []; + for (const [i, platform] of req.platforms.entries()) { + const child = yield* Effect.tryPromise({ + try: () => db.prepare("SELECT status FROM executions WHERE id = ? AND parent_execution_id = ? AND run = ?") + .bind(platform.sourceExecutionId, executionId, "contextful-release-cell").first<{ status: string }>(), + catch: () => new GitHubApiError({ status: 0, reason: "transient" }), + }); + if (child?.status !== "success") return yield* Effect.fail(new GitHubApiError({ status: 403, reason: "other" })); + const descriptorObject = yield* Effect.tryPromise({ + try: () => artifactsBucket.get(artifactKey(platform.sourceExecutionId, platform.imageArtifactName)), + catch: () => new GitHubApiError({ status: 0, reason: "transient" }), + }); + if (descriptorObject === null) return yield* Effect.fail(new GitHubApiError({ status: 404, reason: "other" })); + const descriptor = yield* Effect.tryPromise({ + try: async () => JSON.parse(await descriptorObject.text()) as { + profile?: unknown; version?: unknown; platform?: { os?: unknown; architecture?: unknown }; + layer?: { digest?: unknown; size?: unknown; path?: unknown }; + config?: { digest?: unknown; content?: unknown }; + manifest?: { digest?: unknown; content?: unknown }; + entry?: Record; + }, + catch: () => new GitHubApiError({ status: 400, reason: "other" }), + }); + const layerName = `${req.profile}-${req.version}-linux-${arches[i]}.layer.tar.gz`; + const { layer, config: imageConfig, manifest, entry } = descriptor; + if (descriptor.profile !== req.profile || descriptor.version !== req.version || + descriptor.platform?.os !== "linux" || descriptor.platform.architecture !== arches[i] || + layer?.path !== layerName || typeof layer.digest !== "string" || typeof layer.size !== "number" || + typeof imageConfig?.digest !== "string" || typeof imageConfig.content !== "string" || + typeof manifest?.digest !== "string" || typeof manifest.content !== "string" || + typeof entry !== "object" || entry === null || + (entry.platform as { architecture?: unknown } | undefined)?.architecture !== arches[i]) { + return yield* Effect.fail(new GitHubApiError({ status: 400, reason: "other" })); + } + const sourceExecutionId = platform.sourceExecutionId; + const layerSize = layer.size; + platforms.push({ + layerDigest: layer.digest, layerSize, + // The layer streams from R2 only when the registry lacks it; a size + // that differs from the descriptor's is a staging defect. + openLayer: async () => { + const source = await artifactsBucket.get(artifactKey(sourceExecutionId, layerName)); + if (source === null || source.size !== layerSize) throw new TypeError("invalid OCI image name, tag, digest, or size"); + return source.body; + }, + configDigest: imageConfig.digest, config: new TextEncoder().encode(imageConfig.content), + manifestDigest: manifest.digest, manifest: new TextEncoder().encode(manifest.content), + entry, + }); } return yield* Effect.tryPromise({ - try: () => pushGhcrImage({ + try: () => pushGhcrImageIndex({ username: ghcr.username, token: ghcr.token, - name: `${req.repo}/${req.profile}`, tag: req.version, - layer: source.body, layerSize: source.size, - layerDigest: req.layerDigest, diffId: req.diffId, + name: `${req.repo}/${req.profile}`, tag: req.version, platforms, }), catch: (cause) => new GitHubApiError(classifyGhcrFailure(cause)), }); diff --git a/runs/contextful-release.test.ts b/runs/contextful-release.test.ts index a3e614c..f32218a 100644 --- a/runs/contextful-release.test.ts +++ b/runs/contextful-release.test.ts @@ -8,6 +8,17 @@ import { contextfulRelease, contextfulReleaseCell, contextfulReleaseFormula, REL const sha = "a".repeat(40); const repo = "fractalboxdev/contextful"; +/** The `.image.json` `contextful-ci image` writes for a Linux cell, at the path the cell reads. */ +const imageFiles = (profile: string, target: string): Record => { + const arch = target.startsWith("x86_64") ? "amd64" : "arm64"; + const name = `${profile}-0.5.0-linux-${arch}`; + return { [`/workspace/contextful/dist/${name}.image.json`]: JSON.stringify({ + profile, version: "0.5.0", target, platform: { architecture: arch, os: "linux" }, + layer: { digest: `sha256:${"c".repeat(64)}`, diffId: `sha256:${"d".repeat(64)}`, size: 3, + path: `${name}.layer.tar.gz`, mediaType: "application/vnd.oci.image.layer.v1.tar+gzip" }, + }) }; +}; + /** * The cell verdict a failed release reports: a typed `AcceptanceFailed` whose * markdown the check-run renders, one bullet line per outcome. @@ -47,7 +58,7 @@ describe("contextful-release", () => { live!.idempotencyKey!({ payload: { ...base, ref: "refs/tags/v0.5.0" } })); }); - it.effect("builds and stages a dry-run cell without publishing", () => { + it.effect("builds and stages a dry-run Linux cell and its platform image without publishing", () => { const profile = "contextful-control"; const target = "aarch64-unknown-linux-musl"; const stem = `contextful-control-0.5.0-${target}`; @@ -56,16 +67,21 @@ describe("contextful-release", () => { const { layer, handles } = makeCFRuntimeTest({ sandboxProgram: { "sha256sum": { exitCode: 0, stdout: `${"a".repeat(64)} asset\n` } }, sandboxFiles: { [`/workspace/contextful/dist/${stem}.release.json`]: manifest, - [`/workspace/contextful/dist/${stem}.tar.gz.sha256`]: `${"a".repeat(64)} ${stem}.tar.gz\n` }, + [`/workspace/contextful/dist/${stem}.tar.gz.sha256`]: `${"a".repeat(64)} ${stem}.tar.gz\n`, + ...imageFiles(profile, target) }, }); return Effect.gen(function* () { const result = yield* contextfulReleaseCell.run({ repo, tag: "v0.5.0", sha, profile, target, releaseId: 0, dryRun: true }); expect(result.assets).toEqual([`${stem}.tar.gz`, `${stem}.tar.gz.sha256`, `${stem}.cdx.json`]); - expect(result.imageLayer).toBeUndefined(); - expect(handles.sandbox.execs.map((exec) => exec.command)).toEqual( - expect.arrayContaining([expect.stringContaining("release --builder zigbuild")])); - expect(handles.artifact.uploads).toHaveLength(3); + expect(result.imageLayer).toEqual({ name: "contextful-control-0.5.0-linux-arm64.layer.tar.gz", + imageName: "contextful-control-0.5.0-linux-arm64.image.json", arch: "arm64", size: 3, + digest: "c".repeat(64), diffId: "d".repeat(64) }); + expect(handles.sandbox.execs.map((exec) => exec.command)).toEqual(expect.arrayContaining([ + expect.stringContaining("release --builder zigbuild"), + expect.stringContaining(`image --profile ${profile} --version 0.5.0 --binary ${target}=target/release-artifacts/${target}/release/contextful`), + ])); + expect(handles.artifact.uploads).toHaveLength(5); expect(handles.executions.steps.some((entry) => entry.name.startsWith("publish-"))).toBe(false); }).pipe(Effect.provide(layer)); }); @@ -79,7 +95,8 @@ describe("contextful-release", () => { return { profile, target, name, runtime: makeCFRuntimeTest({ sandboxProgram: { "sha256sum": { exitCode: 0, stdout: `${"a".repeat(64)} asset\n` } }, sandboxFiles: { [`/workspace/contextful/dist/${name}.release.json`]: manifest, - [`/workspace/contextful/dist/${name}.tar.gz.sha256`]: `${"a".repeat(64)} ${name}.tar.gz\n` }, + [`/workspace/contextful/dist/${name}.tar.gz.sha256`]: `${"a".repeat(64)} ${name}.tar.gz\n`, + ...imageFiles(profile, target) }, }) }; }; /** A durable owner whose build runs for `polls` observations, then exits 0. */ @@ -291,11 +308,12 @@ describe("contextful-release", () => { .map((target) => ({ profile, target }))); const FORMULA_ASSETS = ["contextful-control.rb", "contextful-edge.rb", "contextful-full.rb", "contextful.rb", "SHA256SUMS", "install.sh"]; const cellOutput = (cell: Cell) => { - const imageLayer = cell.target === "x86_64-unknown-linux-musl" ? { - name: `${cell.profile}-0.5.0-linux-amd64.layer.tar.gz`, size: 3, - archiveName: `${cell.profile}-0.5.0-linux-amd64.oci.tar`, + const arch = { "x86_64-unknown-linux-musl": "amd64", "aarch64-unknown-linux-musl": "arm64" }[cell.target]; + const imageLayer = arch === undefined ? undefined : { + name: `${cell.profile}-0.5.0-linux-${arch}.layer.tar.gz`, size: 3, + imageName: `${cell.profile}-0.5.0-linux-${arch}.image.json`, arch, digest: "a".repeat(64), diffId: "b".repeat(64), - } : undefined; + }; return JSON.stringify({ ...cell, manifest: JSON.stringify({ ...cell, archive: "archive.tar.gz", sha256: "a".repeat(64), sbom: "sbom.cdx.json" }), ...(imageLayer === undefined ? {} : { imageLayer }) }); @@ -361,7 +379,7 @@ describe("contextful-release", () => { }).pipe(Effect.provide(layer)); }); - it.effect("publishes a draft only after ten cells, formula assets, and three images succeed", () => { + it.effect("publishes a draft only after ten cells, formula assets, and three two-platform images succeed", () => { const { layer, handles } = releaseFixture(() => "success"); handles.github.tagCommits[`${repo}:v0.5.0`] = sha; return Effect.gen(function* () { @@ -369,7 +387,9 @@ describe("contextful-release", () => { expect(result).toMatchObject({ dryRun: false, releaseId: 1, cells: 10, assets: 36 }); expect(handles.github.createReleaseCalls).toHaveLength(1); expect(handles.github.createReleaseCalls[0]?.draft).toBe(true); - expect(handles.github.publishContainerImageCalls).toHaveLength(3); + expect(handles.github.publishContainerImageCalls.map((call) => call.images)).toEqual( + ["contextful-control", "contextful-edge", "contextful-full"].map((profile) => + [`${profile}-0.5.0-linux-amd64.image.json`, `${profile}-0.5.0-linux-arm64.image.json`])); expect(handles.github.publishReleaseCalls).toEqual([{ repo, releaseId: 1, body: "Release v0.5.0\n\n## Consumer compatibility\n" }]); expect(handles.github.releases[`${repo}:v0.5.0`]?.draft).toBe(false); @@ -522,7 +542,7 @@ describe("contextful-release", () => { // The parent queues behind other releases, never behind lean-pool gates. expect(contextfulRelease.sandboxImage).toBe("release"); expect(contextfulReleaseCell.sandboxImage).toBe("release"); - expect(contextfulRelease.version).toBe("1.3.0"); + expect(contextfulRelease.version).toBe("1.4.0"); }); it.effect("window steps share no name with the fixed batches a release in flight checkpointed", () => { diff --git a/runs/contextful-release.ts b/runs/contextful-release.ts index e22fa09..0f00ced 100644 --- a/runs/contextful-release.ts +++ b/runs/contextful-release.ts @@ -14,6 +14,11 @@ const SHA = /^[a-f0-9]{40}$/; const DRY_RUN_TAG_PREFIX = "refs/tags/dry-run/v"; const PROFILES = ["contextful-control", "contextful-edge", "contextful-full"] as const; const LINUX = ["x86_64-unknown-linux-musl", "aarch64-unknown-linux-musl"] as const; +/** Each Linux target's image platform, in image-index order. */ +const IMAGE_ARCH: Readonly> = { + "x86_64-unknown-linux-musl": "amd64", + "aarch64-unknown-linux-musl": "arm64", +}; const DARWIN = ["x86_64-apple-darwin", "aarch64-apple-darwin"] as const; /** The build command's own deadline; the supervisor ends the process at it. */ const RELEASE_BUILD_TIMEOUT_SEC = 14400; @@ -116,7 +121,7 @@ const ReleaseCellOutput = Schema.Struct({ manifest: Schema.String, assets: Schema.Array(Schema.String), imageLayer: Schema.optional(Schema.Struct({ - name: Schema.String, archiveName: Schema.String, + name: Schema.String, imageName: Schema.String, arch: Schema.String, size: Schema.Number, digest: Schema.String, diffId: Schema.String, })), }); @@ -150,7 +155,7 @@ const requireValidRequest = (repo: string, tag: string, sha: string): void => { } }; -type ImageLayer = { name: string; archiveName: string; size: number; digest: string; diffId: string }; +type ImageLayer = { name: string; imageName: string; arch: string; size: number; digest: string; diffId: string }; type StagedAsset = { name: string; sha256: string; contentType: string }; type Collected = | { kind: "lost" } @@ -199,29 +204,37 @@ const collect = (input: { profile: string; target: string }, version: string, ws yield* artifact.upload({ name: asset, path, container: ws.container }); assets.push({ name: asset, sha256, contentType: contentType(asset) }); } - if (input.target !== "x86_64-unknown-linux-musl") return { kind: "staged", manifest, assets } as Collected; - const layerName = `${input.profile}-${version}-linux-amd64.layer.tar.gz`; - const archiveName = `${input.profile}-${version}-linux-amd64.oci.tar`; - const base = `${ws.dir}/dist/${input.profile}-${version}-image`; - const binary = `${ws.dir}/target/release-artifacts/${input.target}/release/contextful`; + const arch = IMAGE_ARCH[input.target]; + if (arch === undefined) return { kind: "staged", manifest, assets } as Collected; + const layerName = `${input.profile}-${version}-linux-${arch}.layer.tar.gz`; + const imageName = `${input.profile}-${version}-linux-${arch}.image.json`; + const binary = `target/release-artifacts/${input.target}/release/contextful`; const footprint = yield* sandbox.exec({ container: ws.container, cwd: ws.dir, command: ["cargo", "run", "--locked", "-q", "-p", "contextful-ci", "--", "footprint", - "--profile", input.profile, binary], timeoutSec: 600 }); + "--profile", input.profile, `${ws.dir}/${binary}`], timeoutSec: 600 }); if (footprint.exitCode !== 0) return yield* Effect.die(new Error(`image footprint failed for ${input.profile}`)); + // `assurance.build.release-image`: this platform's layer, config and + // manifest, and the manifest's index entry; the parent composes the index. const built = yield* sandbox.exec({ container: ws.container, cwd: ws.dir, - command: ["/opt/build-oci-layer.sh", binary, `${base}/root`, - `${ws.dir}/dist/${layerName}`, `${ws.dir}/dist/${archiveName}`, version], + command: ["cargo", "run", "--locked", "-q", "-p", "contextful-ci", "--", "image", + "--profile", input.profile, "--version", version, "--binary", `${input.target}=${binary}`, + "--ca-bundle", "/etc/ssl/certs/ca-certificates.crt", "--out", "dist"], timeoutSec: 600 }); - const lines = built.stdout.trim().split("\n"); - const diffId = lines[0]?.match(/^([a-f0-9]{64})\s/)?.[1]; - const digest = lines[1]?.match(/^([a-f0-9]{64})\s/)?.[1]; - const size = Number(lines[2]); - if (built.exitCode !== 0 || diffId === undefined || digest === undefined || !Number.isSafeInteger(size)) { - return yield* Effect.die(new Error("OCI image layer failed")); + if (built.exitCode !== 0) return yield* Effect.die(new Error(`OCI image failed for ${input.profile}/${input.target}`)); + const image = JSON.parse(yield* readOutput(`${ws.dir}/dist/${imageName}`)) as { + target?: string; platform?: { architecture?: string }; + layer?: { digest?: string; size?: number; diffId?: string; path?: string }; + }; + const digest = image.layer?.digest?.match(/^sha256:([a-f0-9]{64})$/)?.[1]; + const diffId = image.layer?.diffId?.match(/^sha256:([a-f0-9]{64})$/)?.[1]; + const size = image.layer?.size; + if (image.target !== input.target || image.platform?.architecture !== arch || image.layer?.path !== layerName || + digest === undefined || diffId === undefined || size === undefined || !Number.isSafeInteger(size)) { + return yield* Effect.die(new Error("OCI image descriptor does not match its inputs")); } yield* artifact.upload({ name: layerName, path: `${ws.dir}/dist/${layerName}`, container: ws.container }); - yield* artifact.upload({ name: archiveName, path: `${ws.dir}/dist/${archiveName}`, container: ws.container }); - return { kind: "staged", manifest, assets, imageLayer: { name: layerName, archiveName, size, digest, diffId } } as Collected; + yield* artifact.upload({ name: imageName, path: `${ws.dir}/dist/${imageName}`, container: ws.container }); + return { kind: "staged", manifest, assets, imageLayer: { name: layerName, imageName, arch, size, digest, diffId } } as Collected; }); /** @@ -246,8 +259,9 @@ const publishOrRateLimit = ( export const contextfulReleaseCell = defineRun({ name: "contextful-release-cell", + // 1.3.0 — every Linux cell stages its platform's image descriptor and layer. // 1.2.0 — a rate-limited asset upload waits out its retry hint durably. - version: "1.2.0", + version: "1.3.0", sandboxImage: "release", inputs: ReleaseCellInput, outputs: ReleaseCellOutput, @@ -362,10 +376,11 @@ export const contextfulReleaseFormula = defineRun({ export const contextfulRelease = defineRun({ name: "contextful-release", + // 1.4.0 — each profile's image is an index of its amd64 and arm64 manifests. // 1.3.0 — a `dry-run/v` tag push starts a dry run of `v`. // 1.2.0 — publishes the formula step's release notes as the release body. // 1.1.0 — release pool, sliding window of cells. - version: "1.3.0", + version: "1.4.0", // The parent makes no sandbox call and hands its admission row off after // its first spawn. On the release pool it queues only behind other // releases, never behind pull-request gates on the lean pool. @@ -563,26 +578,27 @@ export const contextfulRelease = defineRun({ } const { assets: formulaAssets, notes } = JSON.parse(formulaChildren[0].summaryJson ?? "{}") as { assets?: string[]; notes?: string }; if (formulaAssets?.length !== 6) return yield* Effect.die(new Error("release formula output is incomplete")); - const imageChildren = children.filter((child) => { - const output = JSON.parse(child.summaryJson ?? "{}") as { target?: string }; - return output.target === "x86_64-unknown-linux-musl"; + const version = input.tag.slice(1); + const images = children.flatMap((child) => { + const output = JSON.parse(child.summaryJson ?? "{}") as { + profile?: string; target?: string; imageLayer?: { imageName?: string; arch?: string }; + }; + const arch = output.target === undefined ? undefined : IMAGE_ARCH[output.target]; + return arch === undefined ? [] : [{ executionId: child.executionId, profile: output.profile, arch, + imageName: output.imageLayer?.arch === arch ? output.imageLayer.imageName : undefined }]; }); - if (imageChildren.length !== 3 || imageChildren.some((child) => { - const output = JSON.parse(child.summaryJson ?? "{}") as { profile?: string; imageLayer?: { archiveName?: string } }; - return output.imageLayer?.archiveName !== `${output.profile}-${input.tag.slice(1)}-linux-amd64.oci.tar`; - })) return yield* Effect.die(new Error("release needs three Linux amd64 OCI image archives")); + if (images.length !== PROFILES.length * LINUX.length || images.some((image) => + image.imageName !== `${image.profile}-${version}-linux-${image.arch}.image.json`)) { + return yield* Effect.die(new Error("release needs a Linux amd64 and arm64 image descriptor for every profile")); + } if (dryRun) return { tag: input.tag, sha: tag.commitSha, dryRun: true, releaseId: 0, cells: CELLS.length, assets: CELLS.length * 3 + formulaAssets.length }; - for (const child of imageChildren) { - const output = JSON.parse(child.summaryJson!) as { - profile: string; imageLayer?: { name: string; archiveName: string; size: number; digest: string; diffId: string }; - }; - if (output.imageLayer === undefined) return yield* Effect.die(new Error("release image layer is missing")); - yield* step(`publish-image-${output.profile}`, () => github.publishContainerImage({ - repo: input.repo, profile: output.profile, version: input.tag.slice(1), - sourceExecutionId: child.executionId, - layerArtifactName: output.imageLayer!.name, layerSize: output.imageLayer!.size, - layerDigest: output.imageLayer!.digest, diffId: output.imageLayer!.diffId, + for (const profile of PROFILES) { + const platforms = LINUX.map((target) => images.find((image) => + image.profile === profile && image.arch === IMAGE_ARCH[target])!); + yield* step(`publish-image-${profile}`, () => github.publishContainerImage({ + repo: input.repo, profile, version, + platforms: platforms.map((image) => ({ sourceExecutionId: image.executionId, imageArtifactName: image.imageName! })), })); } const verified = yield* step("verify-tag-for-publish", () => github.tagTarget({ repo: input.repo, tag: input.tag }));