From 8cefcac928b65962ed549ab1de4a1e323649bfa9 Mon Sep 17 00:00:00 2001 From: Charlie Le Date: Sat, 12 Sep 2026 12:28:12 -0700 Subject: [PATCH 1/2] Pull minio from quay.io instead of Docker Hub Every integration leg is failing at Preload Images: Error response from daemon: pull access denied for minio/minio, repository does not exist or may require 'docker login': denied: requested access to the resource is denied minio/minio is the first Docker Hub pull in the step, so no leg gets past it and all 24 fail in about 30 seconds. A CHANGELOG-only pull request reproduces it, so this is not specific to any change under test. master was last green at 4061a3dd6d. This is not a rate limit: the same pull fails right after a successful 'docker login' with the repository credentials. The docker.io/minio/minio repository is simply no longer accessible. MinIO still publishes the identical image to quay.io. quay.io/minio/minio :RELEASE.2024-05-28T17-19-04Z is public and is a manifest list with 8 children, so it covers both the amd64 and arm64 runners. Point the integration tests, the CI preload list, and the three development docker-compose stacks at quay.io. The tag is unchanged, so no behaviour changes. Signed-off-by: Charlie Le --- .github/workflows/test-build-deploy.yml | 2 +- development/tsdb-blocks-storage-s3-gossip/docker-compose.yml | 2 +- .../tsdb-blocks-storage-s3-single-binary/docker-compose.yml | 2 +- development/tsdb-blocks-storage-s3/docker-compose.yml | 2 +- integration/e2e/images/images.go | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/test-build-deploy.yml b/.github/workflows/test-build-deploy.yml index ee2696660ab..8cd17e5cb7e 100644 --- a/.github/workflows/test-build-deploy.yml +++ b/.github/workflows/test-build-deploy.yml @@ -317,7 +317,7 @@ jobs: done } - retry docker pull minio/minio:RELEASE.2024-05-28T17-19-04Z + retry docker pull quay.io/minio/minio:RELEASE.2024-05-28T17-19-04Z retry docker pull consul:1.8.4 retry docker pull quay.io/coreos/etcd:v3.5.29 if [ "$TEST_TAGS" = "integration_backward_compatibility" ]; then diff --git a/development/tsdb-blocks-storage-s3-gossip/docker-compose.yml b/development/tsdb-blocks-storage-s3-gossip/docker-compose.yml index 455a3e1f7ae..0bfbc0f6c1d 100644 --- a/development/tsdb-blocks-storage-s3-gossip/docker-compose.yml +++ b/development/tsdb-blocks-storage-s3-gossip/docker-compose.yml @@ -8,7 +8,7 @@ services: - 8500:8500 minio: - image: minio/minio + image: quay.io/minio/minio command: [ "server", "/data" ] environment: - MINIO_ACCESS_KEY=cortex diff --git a/development/tsdb-blocks-storage-s3-single-binary/docker-compose.yml b/development/tsdb-blocks-storage-s3-single-binary/docker-compose.yml index 4e6b396d194..ea247084cfa 100644 --- a/development/tsdb-blocks-storage-s3-single-binary/docker-compose.yml +++ b/development/tsdb-blocks-storage-s3-single-binary/docker-compose.yml @@ -8,7 +8,7 @@ services: - 8500:8500 minio: - image: minio/minio + image: quay.io/minio/minio command: [ "server", "/data" ] environment: - MINIO_ACCESS_KEY=cortex diff --git a/development/tsdb-blocks-storage-s3/docker-compose.yml b/development/tsdb-blocks-storage-s3/docker-compose.yml index 5bb3a9d3708..54d819d745d 100644 --- a/development/tsdb-blocks-storage-s3/docker-compose.yml +++ b/development/tsdb-blocks-storage-s3/docker-compose.yml @@ -8,7 +8,7 @@ services: - 8500:8500 minio: - image: minio/minio + image: quay.io/minio/minio command: [ "server", "/data" ] environment: - MINIO_ACCESS_KEY=cortex diff --git a/integration/e2e/images/images.go b/integration/e2e/images/images.go index aeb5858974e..c2714676b14 100644 --- a/integration/e2e/images/images.go +++ b/integration/e2e/images/images.go @@ -8,7 +8,7 @@ package images var ( Memcached = "memcached:1.6.1" Redis = "docker.io/redis:7.0.4-alpine" - Minio = "minio/minio:RELEASE.2024-05-28T17-19-04Z" + Minio = "quay.io/minio/minio:RELEASE.2024-05-28T17-19-04Z" Consul = "consul:1.8.4" ETCD = "quay.io/coreos/etcd:v3.5.29" Prometheus = "quay.io/prometheus/prometheus:v3.9.1" From 69accec4393f5f806c28bb8dfa9d785145aba5a7 Mon Sep 17 00:00:00 2001 From: Charlie Le Date: Sat, 12 Sep 2026 12:28:30 -0700 Subject: [PATCH 2/2] Authenticate the integration job's remaining Docker Hub pulls Preload Images still pulls consul, memcached, redis and postgres from Docker Hub. #7464 removed the Install Docker Client step from this job, and that script is where 'docker login' runs, so those pulls have been anonymous since and are subject to the anonymous rate limit. Log in explicitly, matching what the build job does. Pull requests from forks have no secrets, so skip the login there and leave those pulls anonymous instead of failing the step. This is hardening, not the fix for the current breakage: minio failed even when authenticated. Signed-off-by: Charlie Le --- .github/workflows/test-build-deploy.yml | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/.github/workflows/test-build-deploy.yml b/.github/workflows/test-build-deploy.yml index 8cd17e5cb7e..d8465618e55 100644 --- a/.github/workflows/test-build-deploy.yml +++ b/.github/workflows/test-build-deploy.yml @@ -294,6 +294,23 @@ jobs: name: integration-tests-${{ matrix.arch }} - name: Extract Integration Tests Archive run: tar -xzvf integration-tests-${{ matrix.arch }}.tar.gz + - name: Login to Docker Hub + # Preload Images still pulls consul, memcached, redis and postgres from Docker Hub. #7464 + # dropped the Install Docker Client step from this job, and with it the `docker login` that + # script performs, so those pulls have been anonymous ever since and are exposed to the + # anonymous rate limit. Authenticate here, as the build job already does. + # + # The secret is empty on pull requests from forks, so skip the login there and let the pulls + # stay anonymous rather than failing the step outright. + env: + DOCKER_REGISTRY_USER: ${{ secrets.DOCKER_REGISTRY_USER }} + DOCKER_REGISTRY_PASSWORD: ${{ secrets.DOCKER_REGISTRY_PASSWORD }} + run: | + if [ -z "${DOCKER_REGISTRY_PASSWORD:-}" ]; then + echo "No Docker Hub credentials available (fork pull request); pulling anonymously." + exit 0 + fi + docker login -u "$DOCKER_REGISTRY_USER" -p "$DOCKER_REGISTRY_PASSWORD" - name: Preload Images # We download docker images used by integration tests so that all images are available # locally and the download time doesn't account in the test execution time, which is subject