From c24d7241416cd14365eef9cfe3e4b0b2e3df6582 Mon Sep 17 00:00:00 2001 From: Yetkin Timocin Date: Thu, 10 Sep 2026 14:06:11 -0700 Subject: [PATCH] fix: restore FIPS-capable runtime base for agent images The hub-agent, member-agent and refresh-token binaries are built with CGO_ENABLED=1 GOEXPERIMENT=systemcrypto. That backend dlopen()s libcrypto from the runtime image and, on a node with /proc/sys/crypto/fips_enabled=1, calls SetFIPS(true), which requires a loadable FIPS provider in the image. Azure Linux distroless ships one (symcryptprovider.so in /usr/lib/ossl-modules). Debian-based gcr.io/distroless/base ships libcrypto.so.3 with no ossl-modules directory at all, so the call fails and the process panics before main(): panic: opensslcrypto: FIPS mode requested (system FIPS mode) but not available: OpenSSL 3.5.6 7 Apr 2026 crypto/internal/backend.init.0() /usr/local/go/src/crypto/internal/backend/openssl_linux.go:39 +0x109 The gcr.io base arrived via the #1338 backport of upstream #745 and shipped in v0.18.14, crash-looping every hub cluster scheduled onto a FIPS-enabled node (exit 2, CrashLoopBackOff) and taking Scenario_Fleet_Hubful with it. crd-installer was never switched and stayed healthy. Restore mcr.microsoft.com/azurelinux/distroless/base:3.0. It is a real multi-arch manifest list (linux/amd64 + linux/arm64), so the multi-platform buildx from #745 and the cross-compiling builder from #858 are unaffected -- only the final FROM changes. Verified: all three images build for linux/amd64,linux/arm64; the resulting hub-agent starts normally on both architectures with FIPS forced on, where the v0.18.14 image panics with exit 2. --- docker/hub-agent.Dockerfile | 10 +++++----- docker/member-agent.Dockerfile | 10 +++++----- docker/refresh-token.Dockerfile | 10 +++++----- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/docker/hub-agent.Dockerfile b/docker/hub-agent.Dockerfile index 99a813377..0d602d8cb 100644 --- a/docker/hub-agent.Dockerfile +++ b/docker/hub-agent.Dockerfile @@ -58,11 +58,11 @@ COPY pkg/ pkg/ RUN echo "Building hubagent with GOOS=${TARGETOS} GOARCH=${TARGETARCH} CC=$(readlink -f /usr/local/bin/target-gcc)" && \ CGO_ENABLED=1 CC=target-gcc GOOS=${TARGETOS} GOARCH=${TARGETARCH} GOEXPERIMENT=systemcrypto go build -o hubagent ./cmd/hubagent/ -# Use distroless as minimal base image to package the hubagent binary. -# The pinned digest must reference a multi-arch image index so BuildKit can -# resolve the matching base layer for each target architecture. -# Refer to https://github.com/GoogleContainerTools/distroless for more details -FROM gcr.io/distroless/base:nonroot@sha256:2d7d29b504e7166f6d0c7655a18ebf5def5b37b029f8c4f8667e434ba774844f +# GOEXPERIMENT=systemcrypto above dlopen()s libcrypto from THIS image and needs +# a FIPS provider on FIPS-enabled nodes. Azure Linux has one; gcr.io/distroless +# has no ossl-modules at all and panics before main(). Upstream uses gcr.io, so +# backports will try to overwrite this - the base and systemcrypto move together. +FROM mcr.microsoft.com/azurelinux/distroless/base:3.0 WORKDIR / COPY --link --from=builder /workspace/hubagent . USER 65532:65532 diff --git a/docker/member-agent.Dockerfile b/docker/member-agent.Dockerfile index f05cf953b..e8eefcad6 100644 --- a/docker/member-agent.Dockerfile +++ b/docker/member-agent.Dockerfile @@ -58,11 +58,11 @@ COPY pkg/ pkg/ RUN echo "Building memberagent with GOOS=${TARGETOS} GOARCH=${TARGETARCH} CC=$(readlink -f /usr/local/bin/target-gcc)" && \ CGO_ENABLED=1 CC=target-gcc GOOS=${TARGETOS} GOARCH=${TARGETARCH} GOEXPERIMENT=systemcrypto go build -o memberagent ./cmd/memberagent/ -# Use distroless as minimal base image to package the memberagent binary. -# The pinned digest must reference a multi-arch image index so BuildKit can -# resolve the matching base layer for each target architecture. -# Refer to https://github.com/GoogleContainerTools/distroless for more details -FROM gcr.io/distroless/base:nonroot@sha256:2d7d29b504e7166f6d0c7655a18ebf5def5b37b029f8c4f8667e434ba774844f +# GOEXPERIMENT=systemcrypto above dlopen()s libcrypto from THIS image and needs +# a FIPS provider on FIPS-enabled nodes. Azure Linux has one; gcr.io/distroless +# has no ossl-modules at all and panics before main(). Upstream uses gcr.io, so +# backports will try to overwrite this - the base and systemcrypto move together. +FROM mcr.microsoft.com/azurelinux/distroless/base:3.0 WORKDIR / COPY --link --from=builder /workspace/memberagent . USER 65532:65532 diff --git a/docker/refresh-token.Dockerfile b/docker/refresh-token.Dockerfile index 75b2a98a8..5a1ecde62 100644 --- a/docker/refresh-token.Dockerfile +++ b/docker/refresh-token.Dockerfile @@ -59,11 +59,11 @@ COPY pkg/utils/writefile pkg/utils/writefile RUN echo "Building refreshtoken with GOOS=${TARGETOS} GOARCH=${TARGETARCH} CC=$(readlink -f /usr/local/bin/target-gcc)" && \ CGO_ENABLED=1 CC=target-gcc GOOS=${TARGETOS} GOARCH=${TARGETARCH} GOEXPERIMENT=systemcrypto go build -o refreshtoken . -# Use distroless as minimal base image to package the refreshtoken binary. -# The pinned digest must reference a multi-arch image index so BuildKit can -# resolve the matching base layer for each target architecture. -# Refer to https://github.com/GoogleContainerTools/distroless for more details -FROM gcr.io/distroless/base:nonroot@sha256:2d7d29b504e7166f6d0c7655a18ebf5def5b37b029f8c4f8667e434ba774844f +# GOEXPERIMENT=systemcrypto above dlopen()s libcrypto from THIS image and needs +# a FIPS provider on FIPS-enabled nodes. Azure Linux has one; gcr.io/distroless +# has no ossl-modules at all and panics before main(). Upstream uses gcr.io, so +# backports will try to overwrite this - the base and systemcrypto move together. +FROM mcr.microsoft.com/azurelinux/distroless/base:3.0 WORKDIR / COPY --link --from=builder /workspace/refreshtoken . USER 65532:65532