build: fail-closed PGO for release images

.dockerignore

@@ -1,6 +1,9 @@
 .git
 .github
 target
+target-pgo-gen
+target-pgo
+bench/pgo-training/trainer/target
 
 docs
 tests/fixtures/tmp

.github/workflows/docker-image.yml

@@ -33,6 +33,7 @@ on:
       - rust-toolchain.toml
       - "crates/**"
       - "schemas/**"
+      - "bench/pgo-training/**"
   workflow_dispatch:
     inputs:
       tag:
@@ -55,10 +56,45 @@ concurrency:
 jobs:
   build:
     runs-on: ubuntu-latest
-    timeout-minutes: 45
+    # PGO builds (#967) compile the workspace twice (instrumented + optimized)
+    # with a training run in between — roughly 2.5x the plain fat-LTO build
+    # that used to fit in 45 minutes.
+    timeout-minutes: 120
     steps:
       - uses: actions/checkout@v6
 
+      # PGO build mode (#967). Push builds (dev/poc/tags/dispatch) are always
+      # PGO=on: shipped artifacts are PGO'd, fail-closed, no fallback. PR
+      # builds default to PGO=off so review latency stays at one compile —
+      # EXCEPT when the PR touches the build pipeline or the training assets,
+      # in which case the full three-phase build must prove itself pre-merge.
+      - name: Decide PGO build mode
+        id: pgomode
+        env:
+          BASE_SHA: ${{ github.event.pull_request.base.sha || '' }}
+        run: |
+          set -eu
+          mode=on
+          if [ "${{ github.event_name }}" = "pull_request" ]; then
+            mode=off
+            git fetch --no-tags --depth=1 origin "$BASE_SHA"
+            # Capture the diff as a CHECKED command before matching: inside
+            # an `if` condition a git failure is exempt from `set -e` and
+            # would silently leave mode=off (fail-open). As an assignment,
+            # a diff failure kills the step instead. Captured text also
+            # avoids grep -q closing the pipe under git (SIGPIPE).
+            changed="$(git diff --name-only "$BASE_SHA" HEAD)"
+            # Root build inputs (workspace manifest with [profile.release],
+            # lockfile, toolchain pin) are PGO inputs too: a bump can break
+            # the instrumented/profile-use build or the training run while
+            # the plain PGO=off compile stays green.
+            if printf '%s\n' "$changed" \
+                | grep -qE '^(Dockerfile|\.dockerignore|Cargo\.(toml|lock)|rust-toolchain\.toml|bench/pgo-training/|\.github/workflows/docker-image\.ya?ml)'; then
+              mode=on
+            fi
+          fi
+          echo "value=$mode" >> "$GITHUB_OUTPUT"
+
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v4
 
@@ -145,6 +181,7 @@ jobs:
           build-args: |
             BUILD_SHA=${{ steps.shortsha.outputs.value }}
             BUILD_VERSION=${{ steps.relver.outputs.value }}
+            PGO=${{ steps.pgomode.outputs.value }}
           cache-from: type=gha
           cache-to: type=gha,mode=max
           provenance: false   # keep manifest format compatible with older clients
@@ -163,6 +200,35 @@ jobs:
           docker run --rm --entrypoint /usr/local/bin/aisix "$IMG" --version | tee /tmp/aisix-version
           grep -qx "aisix ${VER}" /tmp/aisix-version
 
+      # Fail-closed release gate (#967): every image built with PGO=on must
+      # carry the proof marker the Dockerfile writes only after the
+      # profile-optimized build succeeds. An absent marker, a short shape
+      # list, or an undersized merged profile fails the workflow here —
+      # before signing, and before any human treats the artifact as good.
+      # The `if` is deliberately NOT just `pgomode == 'on'`: every push build
+      # asserts unconditionally, so a future bug in the pgomode step itself
+      # (empty output, renamed id) cannot skip both the PGO build AND its
+      # guard in the same breath.
+      - name: Assert PGO proof marker (#967)
+        if: github.event_name != 'pull_request' || steps.pgomode.outputs.value == 'on'
+        env:
+          TAGS: ${{ steps.meta.outputs.tags }}
+          DIGEST: ${{ steps.build.outputs.digest }}
+        run: |
+          set -eux
+          if [ "${{ github.event_name }}" = "pull_request" ]; then
+            IMG="$(printf '%s\n' "$TAGS" | head -n1)"
+          else
+            IMG="${REGISTRY}/${IMAGE_NAME}@${DIGEST}"
+            docker pull "$IMG"
+          fi
+          docker run --rm --entrypoint cat "$IMG" /usr/local/share/aisix/pgo-verified.json \
+            | tee /tmp/pgo-verified.json
+          shapes="$(jq -r '.shapes | length' /tmp/pgo-verified.json)"
+          test "$shapes" -ge 12
+          test "$(jq -r '.profraw_count' /tmp/pgo-verified.json)" -ge "$shapes"
+          test "$(jq -r '.profdata_bytes' /tmp/pgo-verified.json)" -ge 524288
+
       # The image contract customers rely on for hostNetwork/:80
       # deployments: the default non-root user (uid 10001) must be able
       # to bind privileged ports via the CAP_NET_BIND_SERVICE file
@@ -208,11 +274,21 @@ jobs:
       # RUSTFLAGS change, or base-image swap would break flame graphs
       # silently, so pin it here — and print the size so every PR
       # records the real Linux artifact cost.
+      # Also enforced on tag builds since #967: the PGO'd binary that ships
+      # must provably keep the profiling contract, not just the PR variant.
       - name: Verify shipped binary keeps its symbol table (#847)
-        if: github.event_name == 'pull_request'
+        if: github.event_name == 'pull_request' || startsWith(github.ref, 'refs/tags/v')
+        env:
+          TAGS: ${{ steps.meta.outputs.tags }}
+          DIGEST: ${{ steps.build.outputs.digest }}
         run: |
           set -eux
-          IMAGE="$(printf '%s\n' "${{ steps.meta.outputs.tags }}" | head -n1)"
+          if [ "${{ github.event_name }}" = "pull_request" ]; then
+            IMAGE="$(printf '%s\n' "$TAGS" | head -n1)"
+          else
+            IMAGE="${REGISTRY}/${IMAGE_NAME}@${DIGEST}"
+            docker pull "$IMAGE"
+          fi
           cid="$(docker create "$IMAGE")"
           docker cp "$cid:/usr/local/bin/aisix" /tmp/aisix-shipped
           docker rm "$cid"

.gitignore

@@ -1,5 +1,9 @@
 # Rust
 /target
+# PGO phase target dirs (native runs of the three-phase release recipe)
+/target-pgo-gen
+/target-pgo
+/bench/pgo-training/trainer/target
 Cargo.lock.bak
 **/*.rs.bk
 *.profraw

Cargo.toml

@@ -20,6 +20,9 @@ members = [
     "crates/aisix-guardrails",
     "crates/aisix-server",
 ]
+# Build-time tool for the PGO release pipeline (#967): own manifest + lockfile,
+# never part of the product build graph.
+exclude = ["bench/pgo-training/trainer"]
 
 [workspace.package]
 version = "0.3.0"

Dockerfile

@@ -12,6 +12,7 @@
 #
 # Build:
 #   docker build -t aisix:dev .
+#   docker build --build-arg PGO=off -t aisix:dev .   # quick local build, skips PGO
 #
 # Run, standalone (mount your own config):
 #   docker run --rm -v $(pwd)/config.example.yaml:/etc/aisix/config.yaml \
@@ -68,18 +69,66 @@ COPY crates ./crates
 # Docker context must carry this directory or the release build fails.
 COPY schemas ./schemas
 
+# PGO training assets (#967): trainer tool + train.sh. Copied separately from
+# crates/ so editing training assets doesn't invalidate the dependency layers
+# above.
+COPY bench/pgo-training ./bench/pgo-training
+
+# Profile-guided optimization gate. Default ON: release artifacts are always
+# PGO-built, and a forgotten build-arg ships a PGO'd image — never a silently
+# un-optimized one. CI passes PGO=off only for pull-request smoke builds.
+ARG PGO=on
+
 # `--locked` forces the build to use the exact versions in Cargo.lock —
 # fails fast if the lockfile is stale rather than silently resolving
 # fresh deps in CI.
 #
+# PGO=on runs the three-phase build (#967):
+#   A. instrumented build (-Cprofile-generate) in its own target dir;
+#   B. train.sh drives the committed 12-shape matrix through the
+#      instrumented gateway against the trainer's local mock, then merges
+#      the .profraw files with the pinned toolchain's own llvm-profdata
+#      (llvm-tools-preview — exact LLVM match with rustc, no extra deps);
+#   C. optimized build (-Cprofile-use) in a third target dir, so profile
+#      builds never share cargo fingerprints with plain builds.
+# FAIL-CLOSED: any phase failing fails this RUN and nothing is shipped.
+# The proof marker (pgo-verified.json) is written only after phase C
+# succeeds; the push workflows assert it before trusting the image.
+# The merged profile is content-addressed (merged-<sha>.profdata) because
+# cargo fingerprints the -Cprofile-use PATH, not the file content — a
+# retrained profile at a fixed path would silently reuse stale artifacts
+# from the persistent target cache mount.
+#
 # If this ever builds for linux/arm64: jemalloc bakes the build host's
 # page size into the binary, and QEMU reports 4K — set
 # JEMALLOC_SYS_WITH_LG_PAGE=16 here or the image aborts at startup on
-# 64K-page kernels (see crates/aisix-server/src/main.rs).
+# 64K-page kernels (see crates/aisix-server/src/main.rs). PGO training
+# additionally requires a native arm64 builder: an instrumented binary
+# cannot self-train under QEMU emulation.
 RUN --mount=type=cache,target=/usr/local/cargo/registry \
     --mount=type=cache,target=/src/target \
-    cargo build --locked --release --bin aisix \
-    && cp target/release/aisix /usr/local/bin/aisix
+    --mount=type=cache,target=/src/target-pgo-gen \
+    --mount=type=cache,target=/src/target-pgo \
+    set -eu; \
+    mkdir -p /usr/local/share/aisix; \
+    if [ "$PGO" = "on" ]; then \
+        RUSTFLAGS="-Cprofile-generate=/tmp/pgo-data" CARGO_TARGET_DIR=/src/target-pgo-gen \
+            cargo build --locked --release --bin aisix; \
+        cargo build --locked --release \
+            --manifest-path bench/pgo-training/trainer/Cargo.toml; \
+        bash bench/pgo-training/train.sh /src/target-pgo-gen/release/aisix /tmp/pgo-data; \
+        PROFDATA="$(ls /tmp/pgo-data/merged-*.profdata)"; \
+        RUSTFLAGS="-Cprofile-use=$PROFDATA" CARGO_TARGET_DIR=/src/target-pgo \
+            cargo build --locked --release --bin aisix; \
+        cp /src/target-pgo/release/aisix /usr/local/bin/aisix; \
+        cp /tmp/pgo-data/train-manifest.json /usr/local/share/aisix/pgo-verified.json; \
+    elif [ "$PGO" = "off" ]; then \
+        cargo build --locked --release --bin aisix; \
+        cp target/release/aisix /usr/local/bin/aisix; \
+    else \
+        echo "unsupported PGO value: '$PGO' (use on|off)" >&2; \
+        exit 2; \
+    fi
 
 # --- Stage 2: runtime --------------------------------------------------------
 FROM debian:bookworm-slim AS runtime
@@ -100,11 +149,19 @@ RUN apt-get update \
 # missing from the container's bounding set — it is in the default
 # Docker/containerd cap set, but `capabilities: {drop: [ALL]}` pod
 # specs must add NET_BIND_SERVICE back.
+# The PGO proof marker (#967) ships with the image: written by the builder
+# only after a successful profile-optimized build, asserted by the push
+# workflows before an image is trusted. Absent on PGO=off (PR smoke) builds.
 RUN --mount=type=bind,from=builder,source=/usr/local/bin/aisix,target=/mnt/aisix \
+    --mount=type=bind,from=builder,source=/usr/local/share/aisix,target=/mnt/aisix-share \
     apt-get update \
     && apt-get install -y --no-install-recommends libcap2-bin \
     && install -m 0755 /mnt/aisix /usr/local/bin/aisix \
     && setcap 'cap_net_bind_service=+ep' /usr/local/bin/aisix \
+    && mkdir -p /usr/local/share/aisix \
+    && if [ -f /mnt/aisix-share/pgo-verified.json ]; then \
+         install -m 0644 /mnt/aisix-share/pgo-verified.json /usr/local/share/aisix/pgo-verified.json; \
+       fi \
     && apt-get purge -y --auto-remove libcap2-bin \
     && rm -rf /var/lib/apt/lists/*
 

RELEASING.md

@@ -27,6 +27,27 @@ Pushing the tag triggers two workflows:
   curated-notes scaffold to fill in, then GitHub's auto-generated **What's
   Changed** list as a starting skeleton.
 
+### PGO is mandatory and fail-closed
+
+Published images are profile-guided-optimized (#967): the Docker build
+compiles an instrumented gateway, drives the committed training matrix
+(`bench/pgo-training/`) against it, and rebuilds with the merged profile.
+Any phase failing — instrumented build, training, profile merge, optimized
+build — fails the image build; there is no fallback to a plain build. After
+the push, the workflow asserts the `pgo-verified.json` proof marker inside
+the image (shape count, profile size) before signing. If a release build
+fails in a PGO phase, fix the cause; never ship around it. To inspect a
+shipped image's marker:
+
+```bash
+docker run --rm --entrypoint cat ghcr.io/api7/aisix:X.Y.Z \
+  /usr/local/share/aisix/pgo-verified.json
+```
+
+Local note: each retrained profile is content-addressed, so repeated local
+PGO builds accumulate build artifacts in the persistent BuildKit cache
+mounts; reclaim with `docker builder prune`.
+
 ## 2. Polish the release notes
 
 Edit the draft before publishing. The Get-started/Download header and the