From b5e7f35ac97ceef9d4dc5854b1f8c5fba0b64e62 Mon Sep 17 00:00:00 2001 From: Bobbie Soedirgo Date: Fri, 4 Sep 2026 15:02:59 +0800 Subject: [PATCH 1/2] chore(revert): #2160, #2338, #2344 --- .github/workflows/docker-image-test.yml | 57 +-- .../workflows/dockerhub-release-matrix.yml | 164 +++++--- .github/workflows/manual-docker-release.yml | 220 +++++----- Dockerfile-multigres | 384 +++++++++++++----- Dockerfile-supabase | 224 ---------- README.md | 30 -- ansible/vars.yml | 46 --- .../00-multigres-fixups.sql | 6 - docker/pgctld/pgctld-layered.sh | 7 - docker/pgctld/postgresql.conf.tmpl | 6 +- docker/pgctld/pre-init/00-postgres-role.sql | 5 - nix/packages/docker-image-test.nix | 203 ++++----- nix/packages/image-size-analyzer.nix | 36 +- nix/tests/prime-multigres.sql | 23 -- 14 files changed, 609 insertions(+), 802 deletions(-) delete mode 100644 Dockerfile-supabase delete mode 100644 docker/pgctld/multigres-migrations/00-multigres-fixups.sql delete mode 100755 docker/pgctld/pgctld-layered.sh delete mode 100644 docker/pgctld/pre-init/00-postgres-role.sql diff --git a/.github/workflows/docker-image-test.yml b/.github/workflows/docker-image-test.yml index cf6ca92afd..b8eb0fc4c0 100644 --- a/.github/workflows/docker-image-test.yml +++ b/.github/workflows/docker-image-test.yml @@ -56,24 +56,11 @@ jobs: fail-fast: false matrix: include: - # CHANGED: Dockerfile-15/17 replaced by parameterised Dockerfile-supabase. - # pg_version is passed as --build-arg PG_VERSION to select the PostgreSQL version. - - dockerfile: Dockerfile-supabase - name: 15 - pg_version: "15" - - dockerfile: Dockerfile-supabase - name: 17 - pg_version: "17" - - dockerfile: Dockerfile-orioledb-17 - name: orioledb-17 - pg_version: "17" - # CHANGED: base_dockerfile causes the build step to build the supabase base image - # locally first and pass it as SUPABASE_IMAGE. The variant-orioledb-17 entry was - # removed — that target does not exist in Dockerfile-multigres. - - dockerfile: Dockerfile-multigres - name: multigres-17 - pg_version: "17" - base_dockerfile: Dockerfile-supabase + - { dockerfile: Dockerfile-15, target: "", name: 15 } + - { dockerfile: Dockerfile-17, target: "", name: 17 } + - { dockerfile: Dockerfile-orioledb-17, target: "", name: orioledb-17 } + - { dockerfile: Dockerfile-multigres, target: variant-17, name: multigres-17 } + - { dockerfile: Dockerfile-multigres, target: variant-orioledb-17, name: multigres-orioledb-17 } steps: - name: Checkout Repo uses: actions/checkout@8e8c483db84b4bee98b60c0593521ed34d9990e8 # v6.0.1 @@ -92,45 +79,32 @@ jobs: - name: Build Docker image run: | echo "Building ${{ matrix.name }}..." - # CHANGED: pass PG_VERSION for Dockerfile-supabase and Dockerfile-multigres builds. - PG_VERSION_ARG="" - if [ -n "${{ matrix.pg_version }}" ]; then - PG_VERSION_ARG="--build-arg PG_VERSION=${{ matrix.pg_version }}" + TARGET_ARG="" + if [ -n "${{ matrix.target }}" ]; then + TARGET_ARG="--target ${{ matrix.target }}" fi - # CHANGED: layered images (multigres) need their base image available locally. - # Each matrix job runs on an isolated runner, so we build the base inline here - # and pass it as SUPABASE_IMAGE rather than pulling from a registry. - BASE_IMAGE_ARG="" - if [ -n "${{ matrix.base_dockerfile }}" ]; then - docker build -f "${{ matrix.base_dockerfile }}" \ - --build-arg PG_VERSION=${{ matrix.pg_version }} \ - --target production \ - -t "pg-docker-test:base-${{ matrix.name }}" \ - . - BASE_IMAGE_ARG="--build-arg SUPABASE_IMAGE=pg-docker-test:base-${{ matrix.name }}" - fi - docker build -f "${{ matrix.dockerfile }}" --target production $PG_VERSION_ARG $BASE_IMAGE_ARG \ + docker build -f "${{ matrix.dockerfile }}" $TARGET_ARG \ -t "pg-docker-test:${{ matrix.name }}" \ -t "supabase-postgres:${{ matrix.name }}-analyze" \ . - name: Run image size analysis - if: ${{ matrix.base_dockerfile == '' }} + if: ${{ matrix.target == '' }} run: | echo "=== Image Size Analysis for ${{ matrix.name }} ===" - nix run --accept-flake-config .#image-size-analyzer -- --image ${{ matrix.dockerfile }} --pg-version ${{ matrix.pg_version }} --no-build + nix run --accept-flake-config .#image-size-analyzer -- --image Dockerfile-${{ matrix.name }} --no-build - name: Run Docker image tests - if: ${{ matrix.base_dockerfile == '' }} + if: ${{ matrix.target == '' }} run: | echo "=== Running tests for ${{ matrix.name }} ===" - nix run --accept-flake-config .#docker-image-test -- --no-build --pg-version ${{ matrix.pg_version }} ${{ matrix.dockerfile }} + nix run --accept-flake-config .#docker-image-test -- --no-build Dockerfile-${{ matrix.name }} - name: Run multigres Docker image tests - if: ${{ matrix.base_dockerfile != '' }} + if: ${{ matrix.target != '' }} run: | echo "=== Running tests for ${{ matrix.name }} ===" - nix run --accept-flake-config .#docker-image-test -- --no-build --target production ${{ matrix.dockerfile }} + nix run --accept-flake-config .#docker-image-test -- --no-build --target ${{ matrix.target }} ${{ matrix.dockerfile }} - name: Show container logs on failure if: failure() @@ -146,7 +120,6 @@ jobs: run: | docker ps -a --filter "name=pg-test-${{ matrix.name }}" -q | xargs -r docker rm -f || true docker rmi "pg-docker-test:${{ matrix.name }}" || true - docker rmi "pg-docker-test:base-${{ matrix.name }}" || true # CHANGED: remove ephemeral base image built for layered builds docker rmi "supabase-postgres:${{ matrix.name }}-analyze" || true skip-notification: diff --git a/.github/workflows/dockerhub-release-matrix.yml b/.github/workflows/dockerhub-release-matrix.yml index 90169f55ce..fccf54c27e 100644 --- a/.github/workflows/dockerhub-release-matrix.yml +++ b/.github/workflows/dockerhub-release-matrix.yml @@ -19,8 +19,6 @@ jobs: runs-on: blacksmith-4vcpu-ubuntu-2404 outputs: matrix_config: ${{ steps.set-matrix.outputs.matrix_config }} - base_matrix: ${{ steps.set-matrix.outputs.base_matrix }} - layered_matrix: ${{ steps.set-matrix.outputs.layered_matrix }} steps: - name: Checkout Repo uses: supabase/postgres/.github/actions/shared-checkout@HEAD @@ -28,20 +26,45 @@ jobs: - name: Generate build matrix id: set-matrix run: | - nix run nixpkgs#nushell -- -c ' - let releases = (open ansible/vars.yml | get postgres_release) - let base = (open ansible/vars.yml | get release_matrix_base - | each { |e| $e | insert tag ($releases | get $e.release_key) }) - let layered = (open ansible/vars.yml | get release_matrix_layered - | each { |e| - let ver = ($releases | get $e.release_key) - $e | insert tag $"($ver)($e.tag_suffix)" | insert base_tag $ver - }) - let combined = ($base | append $layered) - $"base_matrix=({include: $base} | to json -r)(char eol)" | save --append $env.GITHUB_OUTPUT - $"layered_matrix=({include: $layered} | to json -r)(char eol)" | save --append $env.GITHUB_OUTPUT - $"matrix_config=({include: $combined} | to json -r)(char eol)" | save --append $env.GITHUB_OUTPUT - ' + nix run nixpkgs#nushell -- -c 'let versions = (open ansible/vars.yml | get postgres_major) + let base_matrix = ($versions | each { |ver| + let version = ($ver | str trim) + let dockerfile = $"Dockerfile-($version)" + if ($dockerfile | path exists) { + { + version: $version, + dockerfile: $dockerfile, + target: "production" + } + } else { + null + } + } | compact) + + # Discover multigres variants by checking for matching targets in Dockerfile-multigres + let multigres_matrix = ($versions | each { |ver| + let version = ($ver | str trim) + let mg_version = $"multigres-($version)" + let mg_dockerfile = "Dockerfile-multigres" + let mg_target = $"variant-($version)" + if ($mg_dockerfile | path exists) and (open --raw $mg_dockerfile | str contains $"AS ($mg_target)") { + { + version: $mg_version, + dockerfile: $mg_dockerfile, + target: $mg_target + } + } else { + null + } + } | compact) + + let matrix = ($base_matrix | append $multigres_matrix) + + let matrix_config = { + include: $matrix + } + + $"matrix_config=($matrix_config | to json -r)" | save --append $env.GITHUB_OUTPUT' build: needs: prepare strategy: @@ -66,45 +89,11 @@ jobs: | str join "\n" | save --append $env.GITHUB_OUTPUT ' - build_base_images: + build_release_image: needs: [prepare, build] strategy: matrix: - postgres: ${{ fromJson(needs.prepare.outputs.base_matrix).include }} - arch: [amd64, arm64] - runs-on: ${{ matrix.arch == 'amd64' && 'large-linux-x86' || 'large-linux-arm' }} - timeout-minutes: 180 - steps: - - name: Checkout Repo - uses: supabase/postgres/.github/actions/shared-checkout@HEAD - - uses: ./.github/actions/nix-install-ephemeral - - run: docker context create builders - - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 - with: - endpoint: builders - - uses: docker/login-action@465a07811f14bebb1938fbed4728c6a1ff8901fc # v2.2.0 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - id: build - uses: docker/build-push-action@ca052bb54ab0790a636c9b5f226502c73d547a25 # v5.4.0 - with: - push: true - build-args: | - ${{ needs.build.outputs.build_args }} - PG_VERSION=${{ matrix.postgres.pg_version }} - target: ${{ matrix.postgres.target }} - tags: supabase/postgres:${{ matrix.postgres.tag }}_${{ matrix.arch }} - platforms: linux/${{ matrix.arch }} - cache-from: type=gha,scope=${{ github.ref_name }}-latest-${{ matrix.arch }} - cache-to: type=gha,mode=max,scope=${{ github.ref_name }}-latest-${{ matrix.arch }} - file: ${{ matrix.postgres.dockerfile }} - - build_layered_images: - needs: [prepare, build, build_base_images] - strategy: - matrix: - postgres: ${{ fromJson(needs.prepare.outputs.layered_matrix).include }} + postgres: ${{ fromJson(needs.prepare.outputs.matrix_config).include }} arch: [amd64, arm64] runs-on: ${{ matrix.arch == 'amd64' && 'large-linux-x86' || 'large-linux-arm' }} timeout-minutes: 180 @@ -120,22 +109,52 @@ jobs: with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} + - name: Get image tag + id: image + run: | + if [[ "${{ matrix.arch }}" == "arm64" ]]; then + pg_version=$(nix run nixpkgs#nushell -- -c ' + let version = "${{ matrix.postgres.version }}" + let is_multigres = ($version | str starts-with "multigres-") + let base_version = if $is_multigres { $version | str replace "multigres-" "" } else { $version } + let release_key = if ($base_version | str contains "orioledb") { + $"postgresorioledb-17" + } else { + $"postgres($base_version)" + } + let base_tag = (open ansible/vars.yml | get postgres_release | get $release_key | str trim) + if $is_multigres { $"($base_tag)-multigres" } else { $base_tag } + ') + echo "pg_version=supabase/postgres:$pg_version" >> $GITHUB_OUTPUT + else + pg_version=$(nix run nixpkgs#nushell -- -c ' + let version = "${{ matrix.postgres.version }}" + let is_multigres = ($version | str starts-with "multigres-") + let base_version = if $is_multigres { $version | str replace "multigres-" "" } else { $version } + let release_key = if ($base_version | str contains "orioledb") { + $"postgresorioledb-17" + } else { + $"postgres($base_version)" + } + let base_tag = (open ansible/vars.yml | get postgres_release | get $release_key | str trim) + if $is_multigres { $"($base_tag)-multigres" } else { $base_tag } + ') + echo "pg_version=supabase/postgres:$pg_version" >> $GITHUB_OUTPUT + fi - id: build uses: docker/build-push-action@ca052bb54ab0790a636c9b5f226502c73d547a25 # v5.4.0 with: push: true build-args: | ${{ needs.build.outputs.build_args }} - PG_VERSION=${{ matrix.postgres.pg_version }} - SUPABASE_IMAGE=supabase/postgres:${{ matrix.postgres.base_tag }}_${{ matrix.arch }} target: ${{ matrix.postgres.target }} - tags: supabase/postgres:${{ matrix.postgres.tag }}_${{ matrix.arch }} + tags: ${{ steps.image.outputs.pg_version }}_${{ matrix.arch }} platforms: linux/${{ matrix.arch }} cache-from: type=gha,scope=${{ github.ref_name }}-latest-${{ matrix.arch }} cache-to: type=gha,mode=max,scope=${{ github.ref_name }}-latest-${{ matrix.arch }} file: ${{ matrix.postgres.dockerfile }} merge_manifest: - needs: [prepare, build, build_base_images, build_layered_images] + needs: [prepare, build, build_release_image] strategy: matrix: include: ${{ fromJson(needs.prepare.outputs.matrix_config).include }} @@ -151,7 +170,20 @@ jobs: password: ${{ secrets.DOCKER_PASSWORD }} - name: Get image tag id: get_version - run: echo "pg_version=supabase/postgres:${{ matrix.tag }}" >> $GITHUB_OUTPUT + run: | + nix run nixpkgs#nushell -- -c ' + let version = "${{ matrix.version }}" + let is_multigres = ($version | str starts-with "multigres-") + let base_version = if $is_multigres { $version | str replace "multigres-" "" } else { $version } + let release_key = if ($base_version | str contains "orioledb") { + $"postgresorioledb-17" + } else { + $"postgres($base_version)" + } + let base_tag = (open ansible/vars.yml | get postgres_release | get $release_key | str trim) + let pg_version = if $is_multigres { $"($base_tag)-multigres" } else { $base_tag } + $"pg_version=supabase/postgres:($pg_version)" | save --append $env.GITHUB_OUTPUT + ' - name: Output version id: output_version run: | @@ -163,7 +195,7 @@ jobs: - name: Upload Results Artifact uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: - name: merge_results-${{ matrix.tag }} + name: merge_results-${{ matrix.version }} path: results.txt if-no-files-found: warn - name: Merge multi-arch manifests @@ -189,14 +221,14 @@ jobs: nix run nixpkgs#nushell -- -c ' # Parse the matrix configuration directly let matrix_config = (${{ toJson(needs.prepare.outputs.matrix_config) }} | from json) - - # Get tags directly from include array - let tags = ($matrix_config.include | get tag) - - echo "Tags: $tags" - - # Convert the tags to a comma-separated string - let versions_str = ($tags | str join ",") + + # Get versions directly from include array + let versions = ($matrix_config.include | get version) + + echo "Versions: $versions" + + # Convert the versions to a comma-separated string + let versions_str = ($versions | str join ",") $"versions=$versions_str" | save --append $env.GITHUB_ENV ' - name: Download Results Artifacts diff --git a/.github/workflows/manual-docker-release.yml b/.github/workflows/manual-docker-release.yml index afa9fa6808..22cad16426 100644 --- a/.github/workflows/manual-docker-release.yml +++ b/.github/workflows/manual-docker-release.yml @@ -1,4 +1,4 @@ -name: Manual Docker Artifacts Release +name: Manual Docker Artifacts Release on: workflow_dispatch: @@ -6,19 +6,7 @@ on: postgresVersion: description: 'Optional. Postgres version to publish against, i.e. 15.1.1.78' required: false - imageLabel: - description: 'Optional. Extra label appended to the image tag, e.g. a short SHA or test suffix (default: no suffix)' - required: false - default: '' - dockerfiles: - description: 'Which Dockerfiles to build. "legacy" uses Dockerfile-15/17/orioledb-17; "layered" uses Dockerfile-supabase + Dockerfile-multigres.' - required: false - default: 'legacy' - type: choice - options: - - legacy - - layered - + permissions: id-token: write contents: read @@ -28,46 +16,52 @@ jobs: runs-on: blacksmith-8vcpu-ubuntu-2404 outputs: matrix_config: ${{ steps.set-matrix.outputs.matrix_config }} - base_matrix: ${{ steps.set-matrix.outputs.base_matrix }} - layered_matrix: ${{ steps.set-matrix.outputs.layered_matrix }} - use_layered: ${{ steps.set-matrix.outputs.use_layered }} steps: - uses: ./.github/actions/nix-install-ephemeral - name: Checkout Repo uses: supabase/postgres/.github/actions/shared-checkout@HEAD - name: Generate build matrix id: set-matrix - env: - DOCKERFILES_MODE: ${{ inputs.dockerfiles || 'legacy' }} run: | - nix run nixpkgs#nushell -- -c ' - let releases = (open ansible/vars.yml | get postgres_release) - let use_layered = ($env.DOCKERFILES_MODE == "layered") + nix run nixpkgs#nushell -- -c 'let versions = (open ansible/vars.yml | get postgres_major) + let base_matrix = ($versions | each { |ver| + let version = ($ver | str trim) + let dockerfile = $"Dockerfile-($version)" + if ($dockerfile | path exists) { + { + version: $version, + dockerfile: $dockerfile, + target: "production" + } + } else { + null + } + } | compact) - let base = if $use_layered { - (open ansible/vars.yml | get release_matrix_base - | each { |e| $e | insert tag ($releases | get $e.release_key) }) - } else { - (open ansible/vars.yml | get release_matrix_legacy - | each { |e| $e | insert tag ($releases | get $e.release_key) }) - } + # Discover multigres variants by checking for matching targets in Dockerfile-multigres + let multigres_matrix = ($versions | each { |ver| + let version = ($ver | str trim) + let mg_version = $"multigres-($version)" + let mg_dockerfile = "Dockerfile-multigres" + let mg_target = $"variant-($version)" + if ($mg_dockerfile | path exists) and (open --raw $mg_dockerfile | str contains $"AS ($mg_target)") { + { + version: $mg_version, + dockerfile: $mg_dockerfile, + target: $mg_target + } + } else { + null + } + } | compact) - let layered = if $use_layered { - (open ansible/vars.yml | get release_matrix_layered - | each { |e| - let ver = ($releases | get $e.release_key) - $e | insert tag $"($ver)($e.tag_suffix)" | insert base_tag $ver - }) - } else { - [] + let matrix = ($base_matrix | append $multigres_matrix) + + let matrix_config = { + include: $matrix } - let combined = ($base | append $layered) - $"use_layered=($use_layered)" | save --append $env.GITHUB_OUTPUT - $"base_matrix=({include: $base} | to json -r)" | save --append $env.GITHUB_OUTPUT - $"layered_matrix=({include: $layered} | to json -r)" | save --append $env.GITHUB_OUTPUT - $"matrix_config=({include: $combined} | to json -r)" | save --append $env.GITHUB_OUTPUT - ' + $"matrix_config=($matrix_config | to json -r)" | save --append $env.GITHUB_OUTPUT' build: needs: prepare strategy: @@ -79,56 +73,24 @@ jobs: - name: Checkout Repo uses: supabase/postgres/.github/actions/shared-checkout@HEAD - uses: ./.github/actions/nix-install-ephemeral + - name: Set PostgreSQL version environment variable + run: echo "POSTGRES_MAJOR_VERSION=${{ matrix.version }}" >> $GITHUB_ENV + - id: args run: | nix run nixpkgs#nushell -- -c ' - open ansible/vars.yml - | items { |key value| {name: $key, item: $value} } - | where { |it| ($it.item | describe) == "string" } - | each { |it| $"($it.name)=($it.item)" } - | str join "\n" + open ansible/vars.yml + | items { |key value| {name: $key, item: $value} } + | where { |it| ($it.item | describe) == "string" } + | each { |it| $"($it.name)=($it.item)" } + | str join "\n" | save --append $env.GITHUB_OUTPUT - ' - build_base_images: + ' + build_release_image: needs: [prepare, build] strategy: matrix: - postgres: ${{ fromJson(needs.prepare.outputs.base_matrix).include }} - arch: [amd64, arm64] - runs-on: ${{ matrix.arch == 'amd64' && 'blacksmith-8vcpu-ubuntu-2404' || 'large-linux-arm' }} - timeout-minutes: 180 - steps: - - name: Checkout Repo - uses: supabase/postgres/.github/actions/shared-checkout@HEAD - - uses: ./.github/actions/nix-install-ephemeral - - run: docker context create builders - - uses: docker/setup-buildx-action@8d2750c68a42422c14e847fe6c8ac0403b4cbd6f # v3.12.0 - with: - endpoint: builders - - uses: docker/login-action@465a07811f14bebb1938fbed4728c6a1ff8901fc # v2.2.0 - with: - username: ${{ secrets.DOCKER_USERNAME }} - password: ${{ secrets.DOCKER_PASSWORD }} - - id: build - uses: docker/build-push-action@ca052bb54ab0790a636c9b5f226502c73d547a25 # v5.4.0 - with: - push: true - build-args: | - ${{ needs.build.outputs.build_args }} - PG_VERSION=${{ matrix.postgres.pg_version }} - target: ${{ matrix.postgres.target }} - tags: supabase/postgres:${{ inputs.postgresVersion || matrix.postgres.tag }}${{ inputs.imageLabel != '' && format('-{0}', inputs.imageLabel) || '' }}_${{ matrix.arch }} - platforms: linux/${{ matrix.arch }} - cache-from: type=gha,scope=${{ github.ref_name }}-latest-${{ matrix.arch }} - cache-to: type=gha,mode=max,scope=${{ github.ref_name }}-latest-${{ matrix.arch }} - file: ${{ matrix.postgres.dockerfile }} - - build_layered_images: - if: needs.prepare.outputs.use_layered == 'true' - needs: [prepare, build, build_base_images] - strategy: - matrix: - postgres: ${{ fromJson(needs.prepare.outputs.layered_matrix).include }} + postgres: ${{ fromJson(needs.prepare.outputs.matrix_config).include }} arch: [amd64, arm64] runs-on: ${{ matrix.arch == 'amd64' && 'blacksmith-8vcpu-ubuntu-2404' || 'large-linux-arm' }} timeout-minutes: 180 @@ -144,24 +106,62 @@ jobs: with: username: ${{ secrets.DOCKER_USERNAME }} password: ${{ secrets.DOCKER_PASSWORD }} + - name: Get image tag + id: image + run: | + if [[ "${{ matrix.arch }}" == "arm64" ]]; then + pg_version=$(nix run nixpkgs#nushell -- -c ' + let version = "${{ matrix.postgres.version }}" + let is_multigres = ($version | str starts-with "multigres-") + let base_version = if $is_multigres { $version | str replace "multigres-" "" } else { $version } + let release_key = if ($base_version | str contains "orioledb") { + $"postgresorioledb-17" + } else { + $"postgres($base_version)" + } + let base_tag = (open ansible/vars.yml | get postgres_release | get $release_key | str trim) + let final_version = if "${{ inputs.postgresVersion }}" != "" { + "${{ inputs.postgresVersion }}" + } else { + $base_tag + } + if $is_multigres { $"($final_version)-multigres" | str trim } else { $final_version | str trim } + ') + echo "pg_version=supabase/postgres:$pg_version" >> $GITHUB_OUTPUT + else + pg_version=$(nix run nixpkgs#nushell -- -c ' + let version = "${{ matrix.postgres.version }}" + let is_multigres = ($version | str starts-with "multigres-") + let base_version = if $is_multigres { $version | str replace "multigres-" "" } else { $version } + let release_key = if ($base_version | str contains "orioledb") { + $"postgresorioledb-17" + } else { + $"postgres($base_version)" + } + let base_tag = (open ansible/vars.yml | get postgres_release | get $release_key | str trim) + let final_version = if "${{ inputs.postgresVersion }}" != "" { + "${{ inputs.postgresVersion }}" + } else { + $base_tag + } + if $is_multigres { $"($final_version)-multigres" | str trim } else { $final_version | str trim } + ') + echo "pg_version=supabase/postgres:$pg_version" >> $GITHUB_OUTPUT + fi - id: build uses: docker/build-push-action@ca052bb54ab0790a636c9b5f226502c73d547a25 # v5.4.0 with: push: true build-args: | ${{ needs.build.outputs.build_args }} - PG_VERSION=${{ matrix.postgres.pg_version }} - SUPABASE_IMAGE=supabase/postgres:${{ inputs.postgresVersion || matrix.postgres.base_tag }}${{ inputs.imageLabel != '' && format('-{0}', inputs.imageLabel) || '' }}_${{ matrix.arch }} target: ${{ matrix.postgres.target }} - tags: supabase/postgres:${{ inputs.postgresVersion != '' && format('{0}-multigres', inputs.postgresVersion) || matrix.postgres.tag }}${{ inputs.imageLabel != '' && format('-{0}', inputs.imageLabel) || '' }}_${{ matrix.arch }} + tags: ${{ steps.image.outputs.pg_version }}_${{ matrix.arch }} platforms: linux/${{ matrix.arch }} cache-from: type=gha,scope=${{ github.ref_name }}-latest-${{ matrix.arch }} cache-to: type=gha,mode=max,scope=${{ github.ref_name }}-latest-${{ matrix.arch }} file: ${{ matrix.postgres.dockerfile }} - merge_manifest: - if: always() && !failure() && !cancelled() - needs: [prepare, build, build_base_images, build_layered_images] + needs: [prepare, build, build_release_image] strategy: matrix: include: ${{ fromJson(needs.prepare.outputs.matrix_config).include }} @@ -177,7 +177,20 @@ jobs: password: ${{ secrets.DOCKER_PASSWORD }} - name: Get image tag id: get_version - run: echo "pg_version=supabase/postgres:${{ inputs.postgresVersion != '' && (contains(matrix.tag, '-multigres') && format('{0}-multigres', inputs.postgresVersion) || inputs.postgresVersion) || matrix.tag }}${{ inputs.imageLabel != '' && format('-{0}', inputs.imageLabel) || '' }}" >> $GITHUB_OUTPUT + run: | + nix run nixpkgs#nushell -- -c ' + let version = "${{ matrix.version }}" + let is_multigres = ($version | str starts-with "multigres-") + let base_version = if $is_multigres { $version | str replace "multigres-" "" } else { $version } + let release_key = if ($base_version | str contains "orioledb") { + $"postgresorioledb-17" + } else { + $"postgres($base_version)" + } + let base_tag = (open ansible/vars.yml | get postgres_release | get $release_key | str trim) + let pg_version = if $is_multigres { $"($base_tag)-multigres" } else { $base_tag } + $"pg_version=supabase/postgres:($pg_version)" | save --append $env.GITHUB_OUTPUT + ' - name: Output version id: output_version run: | @@ -189,7 +202,7 @@ jobs: - name: Upload Results Artifact uses: actions/upload-artifact@ea165f8d65b6e75b540449e92b4886f43607fa02 # v4.6.2 with: - name: merge_results-${{ matrix.tag }} + name: merge_results-${{ matrix.version }} path: results.txt if-no-files-found: warn - name: Merge multi-arch manifests @@ -198,7 +211,6 @@ jobs: ${{ steps.get_version.outputs.pg_version }}_amd64 \ ${{ steps.get_version.outputs.pg_version }}_arm64 combine_results: - if: always() && !failure() && !cancelled() needs: [prepare, merge_manifest] runs-on: blacksmith-8vcpu-ubuntu-2404 steps: @@ -216,12 +228,12 @@ jobs: nix run nixpkgs#nushell -- -c ' # Parse the matrix configuration directly let matrix_config = (${{ toJson(needs.prepare.outputs.matrix_config) }} | from json) - + # Get versions directly from include array - let versions = ($matrix_config.include | get tag) - + let versions = ($matrix_config.include | get version) + echo "Versions: $versions" - + # Convert the versions to a comma-separated string let versions_str = ($versions | str join ",") $"versions=$versions_str" | save --append $env.GITHUB_ENV @@ -237,7 +249,7 @@ jobs: # Get all results files and process them in one go let files = (ls **/results.txt | get name) echo $"Found files: ($files)" - + let matrix = { include: ( $files @@ -245,17 +257,17 @@ jobs: | each { |content| $content | lines } # Split into lines | flatten # Flatten the nested lists | where { |line| $line != "" } # Filter empty lines - | each { |line| + | each { |line| # Extract just the version part after the last colon let version = ($line | parse "supabase/postgres:{version}" | get version.0) {version: $version} } ) } - + let json_output = ($matrix | to json -r) # -r for raw output echo $"Debug output: ($json_output)" - + $"matrix=($json_output)" | save --append $env.GITHUB_OUTPUT ' - name: Debug Combined Results @@ -273,5 +285,5 @@ jobs: matrix: ${{ fromJson(needs.combine_results.outputs.matrix) }} uses: ./.github/workflows/mirror.yml with: - version: ${{ matrix.version }} + version: ${{ inputs.postgresVersion != '' && (contains(matrix.version, '-multigres') && format('{0}-multigres', inputs.postgresVersion) || inputs.postgresVersion) || matrix.version }} secrets: inherit diff --git a/Dockerfile-multigres b/Dockerfile-multigres index 7d9ae0fbac..5184938fc4 100644 --- a/Dockerfile-multigres +++ b/Dockerfile-multigres @@ -1,28 +1,14 @@ -# syntax=docker/dockerfile:1.4 -# 1.4: minimum version for heredoc support in RUN (used for nix config below); also covers COPY --chmod (1.2+) -# Multigres PostgreSQL image — layered on top of the supabase base image. +# syntax=docker/dockerfile:1.6 +# Multigres PostgreSQL 17 variants — vanilla and OrioleDB # -# Put all ARGs up top for better discoverability. -# Use non-versioned `ARG THE_ARG` in the stage that uses it to get the ref to this one. -# -# The supabase image must be built first: -# docker build -f Dockerfile-supabase -t supabase-postgres:17 . -# docker build -f Dockerfile-multigres -t multigres:17 . -# -# Override the base image at build time: -# docker build -f Dockerfile-multigres \ -# --build-arg SUPABASE_IMAGE=registry.example.com/supabase-postgres:17 \ -# -t multigres:17 . - -ARG ALPINE_VERSION=3.23 -ARG GOLANG_VERSION=1.26 -ARG PGCTLD_REV=b713432298450dd0095498193e86d9f9a9f2c348 -ARG SUPABASE_IMAGE=supabase-postgres:17 +# Build targets: +# docker build -f Dockerfile-multigres --target variant-17 -t multigres-17 . +# docker build -f Dockerfile-multigres --target variant-orioledb-17 -t multigres-orioledb-17 . #################### -# Stage 1: pgBackRest / pg_activity nix builder +# Stage 0: Nix base — shared Alpine + Nix setup for all builders #################### -FROM alpine:${ALPINE_VERSION} AS pgbackrest-builder +FROM alpine:3.23 AS nix-base RUN apk add --no-cache \ bash \ @@ -32,106 +18,304 @@ RUN apk add --no-cache \ sudo \ xz -RUN cat >/tmp/extra-nix.conf < /tmp/extra-nix.conf extra-experimental-features = nix-command flakes extra-substituters = https://nix-postgres-artifacts.s3.amazonaws.com extra-trusted-public-keys = nix-postgres-artifacts:dGZlQOvKcNEjvT7QEAJbcV6b6uk7VF/hWMjhYleiaLI= EOF RUN curl -L https://releases.nixos.org/nix/nix-2.34.6/install | sh -s -- --daemon --no-channel-add --yes --nix-extra-conf-file /tmp/extra-nix.conf ENV PATH="${PATH}:/nix/var/nix/profiles/default/bin" +RUN nix --version WORKDIR /nixpg -COPY flake.nix flake.lock ./ -COPY nix/ ./nix/ +COPY . . + +#################### +# Stage 1a: Nix builder — PostgreSQL 17 +#################### +FROM nix-base AS nix-builder-17 + +RUN nix profile add path:.#psql_17_slim/bin path:.#pg-backrest path:.#pg-activity path:.#pgctld -RUN nix profile add path:.#pg-backrest path:.#pg-activity && \ - nix store gc && \ - readlink -f /nix/var/nix/profiles/default/bin/pgbackrest > /pgbackrest-store-path && \ - readlink -f /nix/var/nix/profiles/default/bin/pg_activity > /pg-activity-store-path +RUN nix store gc + +RUN nix profile add path:.#supabase-groonga && \ + mkdir -p /tmp/groonga-plugins && \ + cp -r /nix/var/nix/profiles/default/lib/groonga/plugins /tmp/groonga-plugins/ + +RUN nix store gc #################### -# Stage 2: pgctld builder +# Stage 1b: Nix builder — OrioleDB 17 #################### +FROM nix-base AS nix-builder-orioledb-17 -FROM golang:${GOLANG_VERSION}-alpine AS pgctld-builder +RUN nix profile add path:.#psql_orioledb-17_slim/bin path:.#pg-backrest path:.#pg-activity path:.#pgctld -ARG PGCTLD_REV +RUN nix store gc -RUN apk add --no-cache git +RUN nix profile add path:.#supabase-groonga && \ + mkdir -p /tmp/groonga-plugins && \ + cp -r /nix/var/nix/profiles/default/lib/groonga/plugins /tmp/groonga-plugins/ -RUN git clone https://github.com/multigres/multigres.git /multigres && \ - cd /multigres && \ - git checkout ${PGCTLD_REV} && \ - # Copy pico CSS assets before build (mirrors pgctld.nix preBuild step) - cp external/pico/pico.* go/common/web/templates/css/ 2>/dev/null || true && \ - CGO_ENABLED=0 go build -ldflags="-s -w" -o /usr/local/bin/pgctld ./go/cmd/pgctld +RUN nix store gc #################### -# Stage 3: Multigres image +# Stage 2: Gosu builder #################### -# SUPABASE_IMAGE is set by the release workflow via --build-arg, derived from -# PG_VERSION and the release matrix in ansible/vars.yml. -FROM ${SUPABASE_IMAGE} AS production - -# Merge pgBackRest/pg_activity nix store closure into the inherited /nix/store, then symlink the binaries. -# We copy only the store (not the profile) to avoid clobbering the supabase image's profile. -COPY --from=pgbackrest-builder /nix/store /nix/store -COPY --from=pgbackrest-builder /pgbackrest-store-path /pgbackrest-store-path -COPY --from=pgbackrest-builder /pg-activity-store-path /pg-activity-store-path -RUN ln -sf "$(cat /pgbackrest-store-path)" /usr/local/bin/pgbackrest && \ - ln -sf "$(cat /pg-activity-store-path)" /usr/local/bin/pg_activity && \ - rm /pgbackrest-store-path /pg-activity-store-path - -# Copy pgctld binary; keep it separate so the wrapper script can reference it cleanly -COPY --from=pgctld-builder /usr/local/bin/pgctld /usr/local/bin/pgctld-bin - -# pgctld config template — /etc/pgctld is a mount point in k8s so use a custom dir -COPY docker/pgctld/postgresql.conf.tmpl /etc/pgctld-custom/postgresql.conf.tmpl -# Pre-init SQL: creates the postgres superuser before supabase init scripts run. -# docker-entrypoint.sh does this automatically; pgctld init does not. -COPY docker/pgctld/pre-init/ /etc/pgctld-custom/pre-init/ - -# Multigres-specific migration fixups: equivalents of Ansible after-create hooks -# (extension ownership, not run automatically in Docker). -# Filename sorts after 00-extension.sql so pg_stat_statements exists first. -COPY docker/pgctld/multigres-migrations/ /docker-entrypoint-initdb.d/migrations/ - -# Wrapper: injects --postgres-config-template on every pgctld call so unmodified -# k8s manifests and local provisioner commands work without extra flags -COPY --chmod=0755 docker/pgctld/pgctld-layered.sh /usr/local/bin/pgctld - -# The supabase base config /etc/postgresql/postgresql.conf IS loaded in Multigres: -# the operator's --pg-initdb-extra-conf appends `include = '/etc/postgresql/postgresql.conf'` -# onto pgctld's generated config. It therefore must not carry settings that assume the -# base image's `postgres -D /etc/postgresql` layout, so we neutralize two here: -# -# - data_directory = '/var/lib/postgresql/data' would override pgctld's -D/PGDATA and -# redirect PostgreSQL off the freshly-initdb'd /pg_data to the base image's -# empty dir, which fails startup ("data directory ... has invalid permissions") and -# leaves the pooler stuck retrying the first backup forever. pgctld pins the data dir -# via -D, so comment it out (matching the pre-layered image, which shipped it disabled). -# - the wal-g include is dangling (wal-g is unused in Multigres; pgbackrest handles -# backups), so delete it and its now-orphaned files. +FROM alpine:3.23 AS gosu-builder + +ARG TARGETARCH +ARG GOSU_VERSION=1.19 +ARG GO_VERSION=1.26.1 + +RUN apk add --no-cache curl git + +# Install Go +RUN curl -fsSL "https://go.dev/dl/go${GO_VERSION}.linux-${TARGETARCH}.tar.gz" | tar -C /usr/local -xz +ENV PATH="/usr/local/go/bin:${PATH}" + +# Build gosu from source +RUN git clone --depth 1 --branch "${GOSU_VERSION}" https://github.com/tianon/gosu.git /gosu && \ + cd /gosu && \ + CGO_ENABLED=0 go build -ldflags="-s -w" -o /usr/local/bin/gosu . && \ + chmod +x /usr/local/bin/gosu + +#################### +# Stage 3: Shared base — runtime Alpine + config + migrations +# Both variants inherit from this stage. No /nix dependency here. +#################### +FROM alpine:3.23 AS base + +RUN apk add --no-cache \ + bash \ + curl \ + openssh \ + procps \ + shadow \ + su-exec \ + tzdata \ + musl-locales \ + musl-locales-lang \ + && rm -rf /var/cache/apk/* + +RUN addgroup -S postgres && \ + adduser -S -G postgres -h /var/lib/postgresql -s /bin/bash postgres && \ + addgroup -S wal-g && \ + adduser -S -G wal-g -s /bin/bash wal-g && \ + adduser postgres wal-g + +RUN mkdir -p \ + /usr/lib/postgresql/bin \ + /usr/lib/postgresql/share/postgresql \ + /usr/share/postgresql \ + /var/lib/postgresql/data \ + /var/log/postgresql \ + /var/run/postgresql \ + /etc/postgresql \ + /etc/postgresql-custom \ + && chown -R postgres:postgres \ + /usr/lib/postgresql \ + /var/lib/postgresql \ + /usr/share/postgresql \ + /var/log/postgresql \ + /var/run/postgresql \ + /etc/postgresql \ + /etc/postgresql-custom + +COPY --chown=postgres:postgres ansible/files/postgresql_config/postgresql.conf.j2 /etc/postgresql/postgresql.conf +COPY --chown=postgres:postgres ansible/files/postgresql_config/pg_hba.conf.j2 /etc/postgresql/pg_hba.conf +COPY --chown=postgres:postgres ansible/files/postgresql_config/pg_ident.conf.j2 /etc/postgresql/pg_ident.conf +COPY --chown=postgres:postgres ansible/files/postgresql_config/conf.d /etc/postgresql-custom/conf.d +COPY --chown=postgres:postgres ansible/files/postgresql_config/postgresql-jsonlog.conf /etc/postgresql/logging.conf +COPY --chown=postgres:postgres ansible/files/postgresql_config/supautils.conf.j2 /etc/postgresql-custom/supautils.conf +COPY --chown=postgres:postgres ansible/files/postgresql_extension_custom_scripts /etc/postgresql-custom/extension-custom-scripts +COPY --chown=postgres:postgres ansible/files/postgresql_config/custom_walg.conf /etc/postgresql-custom/wal-g.conf +COPY --chown=postgres:postgres ansible/files/postgresql_config/custom_read_replica.conf /etc/postgresql-custom/read-replica.conf +COPY --chown=postgres:postgres ansible/files/pgsodium_getkey_urandom.sh.j2 /usr/lib/postgresql/bin/pgsodium_getkey.sh + +# Apply settings shared by all variants: +# - enable unix socket, session preload, config includes +# - strip timescaledb and pgsodium (absent from all pg17 builds) +# - comment out db_user_namespace (not supported in pg17) RUN sed -i \ - -e "s|^data_directory = '/var/lib/postgresql/data'|#data_directory = '/var/lib/postgresql/data'|" \ - -e "/^include = '\/etc\/postgresql-custom\/wal-g.conf'/d" \ - /etc/postgresql/postgresql.conf && \ - rm -f /etc/postgresql-custom/wal-g.conf /home/postgres/wal_fetch.sh /root/wal_change_ownership.sh + -e "s|#unix_socket_directories = '/tmp'|unix_socket_directories = '/var/run/postgresql'|g" \ + -e "s|#session_preload_libraries = ''|session_preload_libraries = 'supautils'|g" \ + -e "s|#include = '/etc/postgresql-custom/supautils.conf'|include = '/etc/postgresql-custom/supautils.conf'|g" \ + # skip wal-g - unused by multigres + # -e "s|#include = '/etc/postgresql-custom/wal-g.conf'|include = '/etc/postgresql-custom/wal-g.conf'|g" \ + -e "s/ timescaledb,//g" \ + -e "s/ pgsodium,//g" \ + -e "s/db_user_namespace = off/#db_user_namespace = off/g" \ + # skip - managed by pgctld + -e "s|^data_directory |#data_directory |g" \ + /etc/postgresql/postgresql.conf && \ + echo "pgsodium.getkey_script= '/usr/lib/postgresql/bin/pgsodium_getkey.sh'" >> /etc/postgresql/postgresql.conf && \ + echo "vault.getkey_script= '/usr/lib/postgresql/bin/pgsodium_getkey.sh'" >> /etc/postgresql/postgresql.conf && \ + chown -R postgres:postgres /etc/postgresql-custom && \ + mkdir -p /usr/share/postgresql/extension/ && \ + ln -s /usr/lib/postgresql/bin/pgsodium_getkey.sh /usr/share/postgresql/extension/pgsodium_getkey && \ + chmod +x /usr/lib/postgresql/bin/pgsodium_getkey.sh + +COPY migrations/db /docker-entrypoint-initdb.d/ +COPY ansible/files/pgbouncer_config/pgbouncer_auth_schema.sql /docker-entrypoint-initdb.d/init-scripts/00-schema.sql +COPY ansible/files/stat_extension.sql /docker-entrypoint-initdb.d/migrations/00-extension.sql + +ENV PGDATA=/var/lib/postgresql/data +ENV POSTGRES_HOST=/var/run/postgresql +ENV POSTGRES_USER=supabase_admin +ENV POSTGRES_DB=postgres +ENV POSTGRES_INITDB_ARGS="--allow-group-access --locale-provider=icu --encoding=UTF-8 --icu-locale=en_US.UTF-8" +ENV LANG=en_US.UTF-8 +ENV LANGUAGE=en_US:en +ENV LC_ALL=en_US.UTF-8 +ENV GRN_PLUGINS_DIR=/usr/lib/groonga/plugins + +#################### +# Stage 4a: variant-17 — PostgreSQL 17 (vanilla) +#################### +FROM base AS variant-17 + +COPY --from=nix-builder-17 /nix /nix +COPY --from=nix-builder-17 /tmp/groonga-plugins/plugins /usr/lib/groonga/plugins +COPY --from=gosu-builder /usr/local/bin/gosu /usr/local/bin/gosu + +RUN for f in /nix/var/nix/profiles/default/bin/*; do \ + ln -sf "$f" /usr/lib/postgresql/bin/ 2>/dev/null || true; \ + ln -sf "$f" /usr/bin/ 2>/dev/null || true; \ + done && \ + ln -sf /nix/var/nix/profiles/default/share/postgresql/* /usr/lib/postgresql/share/postgresql/ 2>/dev/null || true && \ + ln -sf /nix/var/nix/profiles/default/share/postgresql/* /usr/share/postgresql/ 2>/dev/null || true && \ + ln -sf /usr/lib/postgresql/share/postgresql/timezonesets /usr/share/postgresql/timezonesets 2>/dev/null || true + +RUN chown -R postgres:postgres /usr/lib/postgresql && \ + chown -R postgres:postgres /usr/share/postgresql + +# Can't use /etc/pgctld because it's a mount point +COPY docker/pgctld/postgresql.conf.tmpl /etc/pgctld-custom/postgresql.conf.tmpl + +# Wrapper: injects --postgres-config-template on every pgctld call AND bridges +# postgres's JSON log file to container stdout via a /proc/1/fd/1 symlink so +# kubelet + Vector can ship it without a sidecar. See docker/pgctld/pgctld-wrapper. +COPY --chmod=755 docker/pgctld/pgctld-wrapper.sh /usr/local/bin/pgctld +ENV POSTGRES_CONFIG_TEMPLATE_PATH=/etc/pgctld-custom/postgresql.conf.tmpl + +# Strip extensions absent from pg17 vanilla build +RUN sed -i 's/ timescaledb,//g; s/ plv8,//g' /etc/postgresql-custom/supautils.conf + +# Generate a single SQL manifest that pgctld runs via --init-db-sql-file after initdb. +# Creates the postgres role, runs init-scripts as postgres (matching migrate.sh), +# then runs migrations as supabase_admin. +RUN set -e && \ + manifest=/docker-entrypoint-initdb.d/multigres-init.sql && \ + printf -- "-- Auto-generated: run init-scripts and migrations after initdb\n" > "$manifest" && \ + printf "DO \$\$ BEGIN\n IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = 'postgres') THEN\n CREATE ROLE postgres SUPERUSER LOGIN;\n END IF;\nEND \$\$;\n" >> "$manifest" && \ + printf "ALTER DATABASE postgres OWNER TO postgres;\n\n" >> "$manifest" && \ + printf "SET SESSION AUTHORIZATION postgres;\n" >> "$manifest" && \ + for f in $(ls /docker-entrypoint-initdb.d/init-scripts/*.sql 2>/dev/null | sort); do \ + printf '\\ir init-scripts/%s\n' "$(basename "$f")" >> "$manifest"; \ + done && \ + printf "\nRESET SESSION AUTHORIZATION;\n\n" >> "$manifest" && \ + for f in $(ls /docker-entrypoint-initdb.d/migrations/*.sql 2>/dev/null | sort); do \ + printf '\\ir migrations/%s\n' "$(basename "$f")" >> "$manifest"; \ + done && \ + chown postgres:postgres "$manifest" + +ENV POSTGRES_INITDB_SQL_FILES=/docker-entrypoint-initdb.d/multigres-init.sql +ENV PATH="/nix/var/nix/profiles/default/bin:/usr/lib/postgresql/bin:${PATH}" +ENV LOCALE_ARCHIVE=/nix/var/nix/profiles/default/lib/locale/locale-archive + +HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \ + CMD pg_isready -U postgres -h localhost || true +STOPSIGNAL SIGINT +USER postgres +EXPOSE 5432 + +ENTRYPOINT ["tail"] +CMD ["-f", "/dev/null"] + +#################### +# Stage 4b: variant-orioledb-17 — PostgreSQL 17 with OrioleDB +#################### +FROM base AS variant-orioledb-17 + +COPY --from=nix-builder-orioledb-17 /nix /nix +COPY --from=nix-builder-orioledb-17 /tmp/groonga-plugins/plugins /usr/lib/groonga/plugins +COPY --from=gosu-builder /usr/local/bin/gosu /usr/local/bin/gosu + +RUN for f in /nix/var/nix/profiles/default/bin/*; do \ + ln -sf "$f" /usr/lib/postgresql/bin/ 2>/dev/null || true; \ + ln -sf "$f" /usr/bin/ 2>/dev/null || true; \ + done && \ + ln -sf /nix/var/nix/profiles/default/share/postgresql/* /usr/lib/postgresql/share/postgresql/ 2>/dev/null || true && \ + ln -sf /nix/var/nix/profiles/default/share/postgresql/* /usr/share/postgresql/ 2>/dev/null || true && \ + ln -sf /usr/lib/postgresql/share/postgresql/timezonesets /usr/share/postgresql/timezonesets 2>/dev/null || true + +RUN chown -R postgres:postgres /usr/lib/postgresql && \ + chown -R postgres:postgres /usr/share/postgresql + +# Strip extensions absent from orioledb-17 build +RUN sed -i 's/ timescaledb,//g; s/ plv8,//g; s/ postgis,//g; s/ pgrouting,//g' \ + /etc/postgresql-custom/supautils.conf + +# Add orioledb to shared_preload_libraries and configure as default table access method +# Rewind: 1200s window, 100k transactions, 1280 x 8KB = 10MB buffer +RUN sed -i "s/\(shared_preload_libraries.*\)'/\1, orioledb'/" /etc/postgresql/postgresql.conf && \ + echo "default_table_access_method = 'orioledb'" >> /etc/postgresql/postgresql.conf && \ + echo "orioledb.enable_rewind = true" >> /etc/postgresql/postgresql.conf && \ + echo "orioledb.rewind_max_time = 1200" >> /etc/postgresql/postgresql.conf && \ + echo "orioledb.rewind_max_transactions = 100000" >> /etc/postgresql/postgresql.conf && \ + echo "orioledb.rewind_buffers = 1280" >> /etc/postgresql/postgresql.conf + +# Register orioledb before initdb migrations run +RUN echo "CREATE EXTENSION orioledb;" > /docker-entrypoint-initdb.d/init-scripts/00-pre-init.sql && \ + chown postgres:postgres /docker-entrypoint-initdb.d/init-scripts/00-pre-init.sql + +# pgctld calls initdb without locale flags, so it picks up LANG from the environment. +# OrioleDB requires C, POSIX, or ICU collations; override the base stage's en_US.UTF-8. +ENV LANG=C +ENV LC_ALL=C +ENV LANGUAGE=C + +# Can't use /etc/pgctld because it's a mount point +COPY docker/pgctld/orioledb-postgresql.conf.tmpl /etc/pgctld-custom/orioledb-postgresql.conf.tmpl + +# Wrapper: injects --postgres-config-template on every pgctld call AND bridges +# postgres's JSON log file to container stdout via a /proc/1/fd/1 symlink so +# kubelet + Vector can ship it without a sidecar. See docker/pgctld/pgctld-wrapper. +COPY --chmod=755 docker/pgctld/pgctld-wrapper.sh /usr/local/bin/pgctld +ENV POSTGRES_CONFIG_TEMPLATE_PATH=/etc/pgctld-custom/orioledb-postgresql.conf.tmpl + +# Regenerate manifest after orioledb added 00-pre-init.sql to init-scripts/ +RUN set -e && \ + manifest=/docker-entrypoint-initdb.d/multigres-init.sql && \ + printf -- "-- Auto-generated: run init-scripts and migrations after initdb\n" > "$manifest" && \ + printf "DO \$\$ BEGIN\n IF NOT EXISTS (SELECT FROM pg_roles WHERE rolname = 'postgres') THEN\n CREATE ROLE postgres SUPERUSER LOGIN;\n END IF;\nEND \$\$;\n" >> "$manifest" && \ + printf "ALTER DATABASE postgres OWNER TO postgres;\n\n" >> "$manifest" && \ + printf "SET SESSION AUTHORIZATION postgres;\n" >> "$manifest" && \ + for f in $(ls /docker-entrypoint-initdb.d/init-scripts/*.sql 2>/dev/null | sort); do \ + printf '\\ir init-scripts/%s\n' "$(basename "$f")" >> "$manifest"; \ + done && \ + printf "\nRESET SESSION AUTHORIZATION;\n\n" >> "$manifest" && \ + for f in $(ls /docker-entrypoint-initdb.d/migrations/*.sql 2>/dev/null | sort); do \ + printf '\\ir migrations/%s\n' "$(basename "$f")" >> "$manifest"; \ + done && \ + chown postgres:postgres "$manifest" -# pgctld uses pooler-dir for pgBackRest config, unix sockets, and state files. -RUN mkdir -p /var/lib/pgctld && chown postgres:postgres /var/lib/pgctld +ENV POSTGRES_INITDB_SQL_FILES=/docker-entrypoint-initdb.d/multigres-init.sql +ENV PATH="/nix/var/nix/profiles/default/bin:/usr/lib/postgresql/bin:${PATH}" +ENV LOCALE_ARCHIVE=/nix/var/nix/profiles/default/lib/locale/locale-archive -# No HEALTHCHECK defined here: inherits the probe from the supabase base image. -# Kubernetes ignores Docker HEALTHCHECK entirely — use readinessProbe in the Pod spec. -# STOPSIGNAL inherited from supabase base image (SIGINT — smart shutdown). +HEALTHCHECK --interval=30s --timeout=5s --start-period=10s --retries=3 \ + CMD pg_isready -U postgres -h localhost || true +STOPSIGNAL SIGINT USER postgres +EXPOSE 5432 -# pgctld is the cluster lifecycle manager for Multigres: it handles initdb, -# config templating, replication setup, and coordinated restarts. Running it -# as PID 1 ensures it receives stop signals directly and can shut down -# PostgreSQL cleanly before the container exits. -ENTRYPOINT ["/usr/local/bin/pgctld"] -# "server" is pgctld's daemon mode: auto-inits PGDATA if empty, starts PostgreSQL, -# and runs as a gRPC lifecycle server. The wrapper injects --postgres-config-template -# and --pg-initdb-sql-dirs; the CMD provides the subcommand and pooler-dir. -CMD ["server", "--pooler-dir", "/var/lib/pgctld"] +ENTRYPOINT ["tail"] +CMD ["-f", "/dev/null"] diff --git a/Dockerfile-supabase b/Dockerfile-supabase deleted file mode 100644 index 446345d933..0000000000 --- a/Dockerfile-supabase +++ /dev/null @@ -1,224 +0,0 @@ -# syntax=docker/dockerfile:1.4 -# 1.4: minimum version for heredoc support in RUN (used for nix config below) -# Supabase PostgreSQL image with Nix extensions — parameterised by major version. -# -# Build (defaults to PostgreSQL 17): -# docker build -f Dockerfile-supabase -t supabase-postgres:17 . -# -# Build for a different PostgreSQL version: -# docker build -f Dockerfile-supabase --build-arg PG_VERSION=15 -t supabase-postgres:15 . - -# Put all ARGs up top for better discoverability -# Use non-versioned `ARG THE_ARG` in the layer that uses it to get the ref to this one -ARG ALPINE_VERSION=3.23 -ARG GO_VERSION=1.26.1 -ARG GOSU_VERSION=1.19 -ARG PG_VERSION=17 - -#################### -# Stage 1: Nix builder -#################### -FROM alpine:${ALPINE_VERSION} AS nix-builder - -ARG PG_VERSION - -# Install dependencies for nix installer (coreutils for GNU cp, sudo for installer) -RUN apk add --no-cache \ - bash \ - coreutils \ - curl \ - shadow \ - sudo \ - xz - -# Create users (Alpine syntax) -RUN addgroup -S postgres && \ - adduser -S -h /var/lib/postgresql -s /bin/bash -G postgres postgres && \ - addgroup -S wal-g && \ - adduser -S -s /bin/bash -G wal-g wal-g - -# Create nix config -RUN cat >/tmp/extra-nix.conf </dev/null || true; \ - ln -sf "$f" /usr/bin/ 2>/dev/null || true; \ - done - -# Create symbolic links for PostgreSQL shares -RUN ln -sf /nix/var/nix/profiles/default/share/postgresql/* /usr/lib/postgresql/share/postgresql/ 2>/dev/null || true && \ - ln -sf /nix/var/nix/profiles/default/share/postgresql/* /usr/share/postgresql/ 2>/dev/null || true && \ - ln -sf /usr/lib/postgresql/share/postgresql/timezonesets /usr/share/postgresql/timezonesets 2>/dev/null || true - -# Set permissions -RUN chown -R postgres:postgres /usr/lib/postgresql && \ - chown -R postgres:postgres /usr/share/postgresql - -# Setup configs -COPY --chown=postgres:postgres ansible/files/postgresql_config/postgresql.conf.j2 /etc/postgresql/postgresql.conf -COPY --chown=postgres:postgres ansible/files/postgresql_config/pg_hba.conf.j2 /etc/postgresql/pg_hba.conf -COPY --chown=postgres:postgres ansible/files/postgresql_config/pg_ident.conf.j2 /etc/postgresql/pg_ident.conf -COPY --chown=postgres:postgres ansible/files/postgresql_config/conf.d /etc/postgresql/postgresql.conf.d -COPY --chown=postgres:postgres ansible/files/postgresql_config/postgresql-stdout-log.conf /etc/postgresql/logging.conf -COPY --chown=postgres:postgres ansible/files/postgresql_config/supautils.conf.j2 /etc/postgresql-custom/supautils.conf -COPY --chown=postgres:postgres ansible/files/postgresql_extension_custom_scripts /etc/postgresql-custom/extension-custom-scripts -COPY --chown=postgres:postgres ansible/files/pgsodium_getkey_urandom.sh.j2 /usr/lib/postgresql/bin/pgsodium_getkey.sh -COPY --chown=postgres:postgres ansible/files/postgresql_config/custom_walg.conf /etc/postgresql-custom/wal-g.conf -COPY --chown=postgres:postgres ansible/files/postgresql_config/custom_read_replica.conf /etc/postgresql-custom/read-replica.conf -COPY --chown=postgres:postgres ansible/files/walg_helper_scripts/wal_fetch.sh /home/postgres/wal_fetch.sh -COPY ansible/files/walg_helper_scripts/wal_change_ownership.sh /root/wal_change_ownership.sh - -# Configure PostgreSQL settings -RUN sed -i \ - -e "s|#unix_socket_directories = '/tmp'|unix_socket_directories = '/var/run/postgresql'|g" \ - -e "s|#session_preload_libraries = ''|session_preload_libraries = 'supautils'|g" \ - -e "s|#include = '/etc/postgresql-custom/supautils.conf'|include = '/etc/postgresql-custom/supautils.conf'|g" \ - -e "s|#include = '/etc/postgresql-custom/wal-g.conf'|include = '/etc/postgresql-custom/wal-g.conf'|g" /etc/postgresql/postgresql.conf && \ - echo "pgsodium.getkey_script= '/usr/lib/postgresql/bin/pgsodium_getkey.sh'" >> /etc/postgresql/postgresql.conf && \ - echo "vault.getkey_script= '/usr/lib/postgresql/bin/pgsodium_getkey.sh'" >> /etc/postgresql/postgresql.conf && \ - chown -R postgres:postgres /etc/postgresql-custom && \ - ln -s /etc/postgresql/postgresql.conf.d /etc/postgresql-custom/conf.d - -# pg17+ does not ship timescaledb or plv8; db_user_namespace was removed in pg17. -# Applied conditionally so the same Dockerfile works for pg15 (where these are valid). -RUN if [ "${PG_VERSION}" -ge 17 ]; then \ - sed -i 's/ timescaledb,//g; s/ plv8,//g' /etc/postgresql/postgresql.conf && \ - sed -i 's/db_user_namespace = off/#db_user_namespace = off/g' /etc/postgresql/postgresql.conf && \ - sed -i 's/ timescaledb,//g; s/ plv8,//g' /etc/postgresql-custom/supautils.conf; \ - fi - -# Include schema migrations -COPY migrations/db /docker-entrypoint-initdb.d/ -COPY ansible/files/pgbouncer_config/pgbouncer_auth_schema.sql /docker-entrypoint-initdb.d/init-scripts/00-schema.sql -COPY ansible/files/stat_extension.sql /docker-entrypoint-initdb.d/migrations/00-extension.sql - -# Add entrypoint script from the official postgres Docker library (version-matched) -ADD --chmod=0755 \ - https://raw.githubusercontent.com/docker-library/postgres/6edb0a8c4def40c371514b34aef9037ec82d9110/${PG_VERSION}/alpine3.23/docker-entrypoint.sh \ - /usr/local/bin/docker-entrypoint.sh - -# Setup pgsodium key script -RUN mkdir -p /usr/share/postgresql/extension/ && \ - ln -s /usr/lib/postgresql/bin/pgsodium_getkey.sh /usr/share/postgresql/extension/pgsodium_getkey && \ - chmod +x /usr/lib/postgresql/bin/pgsodium_getkey.sh - -# Environment variables -ENV PATH="/nix/var/nix/profiles/default/bin:/usr/lib/postgresql/bin:${PATH}" -ENV PGDATA=/var/lib/postgresql/data -ENV POSTGRES_HOST=/var/run/postgresql -ENV POSTGRES_USER=supabase_admin -ENV POSTGRES_DB=postgres -ENV POSTGRES_INITDB_ARGS="--allow-group-access --locale-provider=icu --encoding=UTF-8 --icu-locale=en_US.UTF-8" -ENV LANG=en_US.UTF-8 -ENV LANGUAGE=en_US:en -ENV LC_ALL=en_US.UTF-8 -ENV GRN_PLUGINS_DIR=/usr/lib/groonga/plugins -# Point to minimal glibc locales included in slim Nix package for initdb locale support -ENV LOCALE_ARCHIVE=/nix/var/nix/profiles/default/lib/locale/locale-archive - -# Marks the container unhealthy after 10 failed pg_isready probes, which blocks -# dependent services in Docker Compose (depends_on: condition: service_healthy). -# Kubernetes ignores Docker HEALTHCHECK entirely — use readinessProbe in the Pod spec. -HEALTHCHECK --interval=2s --timeout=2s --retries=10 CMD pg_isready -U postgres -h localhost - -# SIGINT triggers PostgreSQL smart shutdown: waits for active sessions to finish -# before stopping. This avoids interrupting in-flight transactions but can delay -# pod termination if long-running queries are active. -# Consider SIGTERM (fast shutdown) to disconnect clients immediately, which -# respects Kubernetes terminationGracePeriodSeconds more predictably. -STOPSIGNAL SIGINT -EXPOSE 5432 - -# No USER directive: the entrypoint starts as root to fix volume ownership and -# set up permissions, then drops to the postgres user via gosu before exec'ing -# the postgres process. This follows the standard official PostgreSQL image pattern. -ENTRYPOINT ["docker-entrypoint.sh"] -CMD ["postgres", "-D", "/etc/postgresql"] diff --git a/README.md b/README.md index 6add67a24e..fa7af7f823 100644 --- a/README.md +++ b/README.md @@ -69,8 +69,6 @@ Here's a comprehensive overview of the project's directory structure: | migrations/tests/extensions/ | Extension migration tests | | Dockerfile-15 | Docker image definition for PostgreSQL 15 | | Dockerfile-17 | Docker image definition for PostgreSQL 17 | -| Dockerfile-supabase | Parameterised supabase base image (supports any PostgreSQL major version via `--build-arg PG_VERSION=17`) | -| Dockerfile-multigres | Multigres image layered on top of `Dockerfile-supabase` | | **tests/** | Integration and system tests | | testinfra/ | Infrastructure tests using pytest framework | | tests/ | General integration test suites | @@ -161,34 +159,6 @@ docker build -f Dockerfile-15 -t supabase-postgres:15 . docker build -f Dockerfile-17 -t supabase-postgres:17 . ``` -#### Supabase base image (version-parameterised) - -`Dockerfile-supabase` is a single Dockerfile that can target any supported PostgreSQL major version via `--build-arg`: - -```bash -# Build for PostgreSQL 17 (default) -docker build -f Dockerfile-supabase -t supabase-postgres:17 . - -# Build for PostgreSQL 15 -docker build -f Dockerfile-supabase --build-arg PG_VERSION=15 -t supabase-postgres:15 . -``` - -#### Multigres image - -`Dockerfile-multigres` layers on top of the supabase image, adding `pgctld` and `pgbackrest`. Build the supabase image first, then point `SUPABASE_IMAGE` at it: - -```bash -# Build supabase base, then multigres on top -docker build -f Dockerfile-supabase -t supabase-postgres:17 . -docker build -f Dockerfile-multigres -t multigres:17 . - -# Target a different PostgreSQL version -docker build -f Dockerfile-supabase --build-arg PG_VERSION=15 -t supabase-postgres:15 . -docker build -f Dockerfile-multigres \ - --build-arg SUPABASE_IMAGE=supabase-postgres:15 \ - -t multigres:15 . -``` - ## Next Steps Now that you understand the basics of Supabase Postgres: diff --git a/ansible/vars.yml b/ansible/vars.yml index 904d2afdb5..c91c2307cb 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -6,56 +6,10 @@ postgres_major: - "15" - "17" - orioledb-17 -# Full version strings for each major version. -# -# This is the source of truth for Postgres versions used in the Dockerfiles, and -# is used to derive image tags and base images in the release matrix. postgres_release: postgresorioledb-17: "17.9.0.023-orioledb" postgres17: "17.6.1.170" postgres15: "15.14.1.170" -# Docker release matrix — base images built first, layered images built on top. -# tag and base_tag are derived at build time from postgres_release via release_key. -# tag_suffix is appended to the release version to form the final image tag. -release_matrix_base: - - dockerfile: Dockerfile-supabase - target: production - pg_version: "15" - release_key: postgres15 - - dockerfile: Dockerfile-supabase - target: production - pg_version: "17" - release_key: postgres17 -release_matrix_layered: - - dockerfile: Dockerfile-multigres - target: production - pg_version: "17" - release_key: postgres17 - tag_suffix: "-multigres" - # OrioleDB uses a patched version of Postgres 17 so it builds from scratch but - # logically belongs as a layered image. The layered build job passes SUPABASE_IMAGE - # as a build arg which Docker silently ignores since it is undeclared in this Dockerfile. - - dockerfile: Dockerfile-orioledb-17 - target: production - pg_version: "17" - release_key: postgresorioledb-17 - tag_suffix: "" -# Legacy (pre-Dockerfile-supabase) build matrix — flat list used when the manual -# release workflow is triggered with dockerfiles: legacy (the default). -# All images are built independently; there is no base→layered dependency. -release_matrix_legacy: - - dockerfile: Dockerfile-15 - target: production - pg_version: "15" - release_key: postgres15 - - dockerfile: Dockerfile-17 - target: production - pg_version: "17" - release_key: postgres17 - - dockerfile: Dockerfile-orioledb-17 - target: production - pg_version: "17" - release_key: postgresorioledb-17 supabase_admin_agent_splay: 30s ############################################################################################################### # The following block of yaml is for get_url and co throughout the playbook # diff --git a/docker/pgctld/multigres-migrations/00-multigres-fixups.sql b/docker/pgctld/multigres-migrations/00-multigres-fixups.sql deleted file mode 100644 index 8b17183321..0000000000 --- a/docker/pgctld/multigres-migrations/00-multigres-fixups.sql +++ /dev/null @@ -1,6 +0,0 @@ --- Multigres Docker init fixups — equivalents of Ansible after-create hooks. --- Runs as supabase_admin (the migrations role) after all standard migrations. - --- init-scripts run as postgres, so pgcrypto and uuid-ossp extowners are postgres. --- Standard supabase expected output has supabase_admin as extowner. -UPDATE pg_extension SET extowner = 'supabase_admin'::regrole WHERE extname IN ('pgcrypto', 'uuid-ossp'); diff --git a/docker/pgctld/pgctld-layered.sh b/docker/pgctld/pgctld-layered.sh deleted file mode 100755 index 8541ab2485..0000000000 --- a/docker/pgctld/pgctld-layered.sh +++ /dev/null @@ -1,7 +0,0 @@ -#!/bin/sh -exec /usr/local/bin/pgctld-bin \ - --postgres-config-template /etc/pgctld-custom/postgresql.conf.tmpl \ - --pg-initdb-sql-dirs supabase_admin:/etc/pgctld-custom/pre-init \ - --pg-initdb-sql-dirs postgres:/docker-entrypoint-initdb.d/init-scripts \ - --pg-initdb-sql-dirs supabase_admin:/docker-entrypoint-initdb.d/migrations \ - "$@" diff --git a/docker/pgctld/postgresql.conf.tmpl b/docker/pgctld/postgresql.conf.tmpl index 4067e5ab80..cb74a6d819 100644 --- a/docker/pgctld/postgresql.conf.tmpl +++ b/docker/pgctld/postgresql.conf.tmpl @@ -232,7 +232,6 @@ default_text_search_config = 'pg_catalog.english' session_preload_libraries = 'supautils' shared_preload_libraries = 'pg_stat_statements, pgaudit, plpgsql, plpgsql_check, pg_cron, pg_net, auto_explain, pg_tle, plan_filter, supabase_vault' -session_preload_libraries = 'supautils' jit_provider = 'llvmjit' # JIT library to use @@ -274,11 +273,14 @@ restart_after_crash = off # reinitialize after backend crash? # default postgresql.conf. Note that these are directives, not variable # assignments, so they can usefully be given more than once. +# Automatically generated optimizations # User-supplied custom parameters, override any automatically generated ones -include_if_exists '/etc/postgresql-custom/supautils.conf' # WAL-G specific configurations +# supautils specific configurations +include = '/etc/postgresql-custom/supautils.conf' + #------------------------------------------------------------------------------ # CUSTOMIZED OPTIONS #------------------------------------------------------------------------------ diff --git a/docker/pgctld/pre-init/00-postgres-role.sql b/docker/pgctld/pre-init/00-postgres-role.sql deleted file mode 100644 index 1f40699bdf..0000000000 --- a/docker/pgctld/pre-init/00-postgres-role.sql +++ /dev/null @@ -1,5 +0,0 @@ --- docker-entrypoint.sh creates a postgres superuser role when POSTGRES_USER != postgres. --- pgctld init does not replicate this behaviour, so we create it here before the --- supabase init scripts run (they reference the postgres role at line 39 of --- 00000000000000-initial-schema.sql). -CREATE USER postgres SUPERUSER; diff --git a/nix/packages/docker-image-test.nix b/nix/packages/docker-image-test.nix index 324d6d6e5e..6cdeec405c 100644 --- a/nix/packages/docker-image-test.nix +++ b/nix/packages/docker-image-test.nix @@ -21,8 +21,8 @@ writeShellApplication { # Usage: # nix run .#docker-image-test -- Dockerfile-17 # nix run .#docker-image-test -- --no-build Dockerfile-15 - # nix run .#docker-image-test -- --target production Dockerfile-multigres - # nix run .#docker-image-test -- --no-build --target production Dockerfile-multigres + # nix run .#docker-image-test -- --target variant-17 Dockerfile-multigres + # nix run .#docker-image-test -- --no-build --target variant-orioledb-17 Dockerfile-multigres set -euo pipefail @@ -41,7 +41,6 @@ writeShellApplication { HTTP_MOCK_PID="" KEEP_CONTAINER=false TARGET="" - PG_VERSION="" # Colors for output RED='\033[0;31m' @@ -67,19 +66,16 @@ writeShellApplication { --no-build Skip building the image (use existing) --keep Keep the container running after tests (for debugging) --target TARGET Build target (required for Dockerfile-multigres) - Values: production - --pg-version VER PostgreSQL major version (required for Dockerfile-supabase) - Values: 15, 17 + Values: variant-17, variant-orioledb-17 Examples: nix run .#docker-image-test -- Dockerfile-17 nix run .#docker-image-test -- Dockerfile-15 nix run .#docker-image-test -- Dockerfile-orioledb-17 nix run .#docker-image-test -- --no-build Dockerfile-17 - nix run .#docker-image-test -- --pg-version 17 Dockerfile-supabase - nix run .#docker-image-test -- --no-build --pg-version 15 Dockerfile-supabase - nix run .#docker-image-test -- --target production Dockerfile-multigres - nix run .#docker-image-test -- --no-build --target production Dockerfile-multigres + nix run .#docker-image-test -- --target variant-17 Dockerfile-multigres + nix run .#docker-image-test -- --target variant-orioledb-17 Dockerfile-multigres + nix run .#docker-image-test -- --no-build --target variant-17 Dockerfile-multigres EOF } @@ -89,29 +85,19 @@ writeShellApplication { Dockerfile-15) echo "15 5436" ;; Dockerfile-17) echo "17 5435" ;; Dockerfile-orioledb-17) echo "orioledb-17 5437" ;; - Dockerfile-supabase) - if [[ -z "$PG_VERSION" ]]; then - log_error "Dockerfile-supabase requires --pg-version (15 or 17)" - exit 1 - fi - case "$PG_VERSION" in - 15) echo "15 5436" ;; - 17) echo "17 5435" ;; - *) log_error "Unknown --pg-version: $PG_VERSION (expected 15 or 17)"; exit 1 ;; - esac - ;; Dockerfile-multigres) case "''${TARGET}" in - production) echo "multigres-17 5438" ;; + variant-17) echo "multigres-17 5438" ;; + variant-orioledb-17) echo "multigres-orioledb-17 5439" ;; *) - log_error "Dockerfile-multigres requires --target production" + log_error "Dockerfile-multigres requires --target (variant-17 or variant-orioledb-17)" exit 1 ;; esac ;; *) log_error "Unknown Dockerfile: $dockerfile" - log_error "Supported: Dockerfile-15, Dockerfile-17, Dockerfile-supabase, Dockerfile-orioledb-17, Dockerfile-multigres" + log_error "Supported: Dockerfile-15, Dockerfile-17, Dockerfile-orioledb-17, Dockerfile-multigres" exit 1 ;; esac @@ -306,15 +292,14 @@ writeShellApplication { return 1 } - # Verify pgctld integration and bootstrap postgres for multigres images. - # pgctld server (PID 1) is a gRPC management daemon — it does NOT auto-start - # PostgreSQL. We call pgctld init + start via docker exec (both are standalone - # commands that don't communicate with the running server). - # Must be called BEFORE wait_for_postgres. + # Verify pgctld integration for multigres images. + # Runs a short-lived pgctld cluster in /tmp (separate from the SQL-test postgres). + # Tests: container user is postgres, /usr/local/bin/pgctld works, pgctld init+start + # succeeds with NO extra flags, and (orioledb variant) orioledb loads automatically. verify_pgctld_integration() { local container="$1" local version="$2" - local pooler_dir="/var/lib/pgctld" + local pooler_dir="/tmp/pgctld-verify" log_info "=== pgctld integration checks ===" @@ -334,53 +319,20 @@ writeShellApplication { fi log_info " ✓ /usr/local/bin/pgctld exists" - # 3. /usr/local/bin/pgctld must be a wrapper script (not the raw binary) - local first_line - first_line=$(docker exec "$container" sh -c "head -1 /usr/local/bin/pgctld") - if [[ "$first_line" != "#!/bin/sh" ]]; then - log_error " /usr/local/bin/pgctld is not a wrapper script (first line: $first_line)" - exit 1 - fi - log_info " ✓ /usr/local/bin/pgctld is a wrapper script" - - # 3.5. wal-g must be fully removed: no include line left dangling in the - # static /etc/postgresql/postgresql.conf. (Currently, pgctld doesn't load - # this file today, but if it's ever loaded directly it should not error out. - # See Dockerfile-multigres.) None of the inherited wal-g files should remain. - if docker exec "$container" sh -c "grep -q 'wal-g.conf' /etc/postgresql/postgresql.conf"; then - log_error " /etc/postgresql/postgresql.conf still references wal-g.conf" - exit 1 - fi - for f in /etc/postgresql-custom/wal-g.conf /home/postgres/wal_fetch.sh /root/wal_change_ownership.sh; do - if docker exec "$container" test -e "$f"; then - log_error " $f should have been removed from the multigres image" - exit 1 - fi - done - log_info " ✓ wal-g fully removed (no dangling include, no inherited files)" - - # 4. pgctld init: initializes PGDATA and runs SQL init scripts (via wrapper's - # --pg-initdb-sql-dirs). Uses pooler-dir for socket dir and pgBackRest config. - local init_out init_rc=0 - init_out=$(docker exec \ - -e POSTGRES_PASSWORD="$POSTGRES_PASSWORD" \ - "$container" \ - sh -c "/usr/local/bin/pgctld init --pooler-dir $pooler_dir 2>&1") || init_rc=$? - if [[ $init_rc -ne 0 ]] || ! echo "$init_out" | grep -q "initialized successfully"; then - log_error " pgctld init failed (exit $init_rc): $init_out" + # 3. pgctld init must succeed with no extra flags (tests USER postgres fix) + local init_out + init_out=$(docker exec -e PGDATA="$pooler_dir/pg_data" "$container" sh -c "/usr/local/bin/pgctld init --pooler-dir $pooler_dir 2>&1") + if ! echo "$init_out" | grep -q "initialized successfully"; then + log_error " pgctld init failed: $init_out" exit 1 fi log_info " ✓ pgctld init --pooler-dir $pooler_dir" - # 5. pgctld start: starts PostgreSQL (pg_ctl start). Postgres will listen on - # port 5432 inside the container and on the pooler-dir unix socket. - local start_out start_rc=0 - start_out=$(docker exec \ - -e POSTGRES_PASSWORD="$POSTGRES_PASSWORD" \ - "$container" \ - sh -c "/usr/local/bin/pgctld start --pooler-dir $pooler_dir 2>&1") || start_rc=$? - if [[ $start_rc -ne 0 ]] || ! echo "$start_out" | grep -q "started successfully"; then - log_error " pgctld start failed (exit $start_rc): $start_out" + # 4. pgctld start must succeed + local start_out + start_out=$(docker exec -e PGDATA="$pooler_dir/pg_data" "$container" sh -c "/usr/local/bin/pgctld start --pooler-dir $pooler_dir 2>&1") + if ! echo "$start_out" | grep -q "started successfully"; then + log_error " pgctld start failed: $start_out" exit 1 fi log_info " ✓ pgctld start --pooler-dir $pooler_dir" @@ -426,43 +378,69 @@ writeShellApplication { fi log_info " ✓ /usr/local/bin/pgctld is a wrapper script" - # orioledb checks (shared_preload_libraries, default_table_access_method) are deferred - # to verify_orioledb_integration(), called after wait_for_postgres succeeds. - log_info " (orioledb checks deferred until postgres is ready)" + # 8. shared_preload_libraries must include orioledb — injected by wrapper, no flags needed + local spl + spl=$(docker exec -e PGPASSWORD="$POSTGRES_PASSWORD" "$container" sh -c " + psql -U $POSTGRES_USER -d postgres -h $pooler_dir/pg_sockets \ + -tAc \"SHOW shared_preload_libraries;\" 2>&1") || true + if ! echo "$spl" | grep -q "orioledb"; then + log_error " orioledb not in shared_preload_libraries (got: $spl)" + log_error " Check that wrapper script injects --postgres-config-template" + exit 1 + fi + log_info " ✓ shared_preload_libraries contains orioledb" + + # 9. default_table_access_method must be orioledb + local tam + tam=$(docker exec -e PGPASSWORD="$POSTGRES_PASSWORD" "$container" sh -c " + psql -U $POSTGRES_USER -d postgres -h $pooler_dir/pg_sockets \ + -tAc \"SHOW default_table_access_method;\" 2>&1") || true + if ! echo "$tam" | grep -q "orioledb"; then + log_error " default_table_access_method is not orioledb (got: $tam)" + exit 1 + fi + log_info " ✓ default_table_access_method = orioledb" fi + # Shut down the pgctld test cluster so it doesn't hold the unix socket + docker exec "$container" sh -c " + pg_ctl stop -D $pooler_dir/pg_data -m immediate 2>/dev/null || true + " log_info "=== pgctld integration checks passed ===" } - # For multigres-orioledb-17: verify orioledb is loaded after postgres is ready. - verify_orioledb_integration() { + # Bootstrap a multigres container: initdb + pg_ctl start + create supabase_admin + run migrations + # Multigres images use "tail -f /dev/null" as entrypoint so postgres must be started manually. + start_multigres_postgres() { local container="$1" - log_info "=== orioledb integration checks ===" - - # shared_preload_libraries must include orioledb — injected by wrapper via config template - local spl - spl=$(docker exec -e PGPASSWORD="$POSTGRES_PASSWORD" "$container" \ - psql -U "$POSTGRES_USER" -d postgres -h /var/run/postgresql \ - -tAc "SHOW shared_preload_libraries;" 2>&1) || true - if ! echo "$spl" | grep -q "orioledb"; then - log_error " orioledb not in shared_preload_libraries (got: $spl)" - log_error " Check that wrapper script injects --postgres-config-template" - exit 1 - fi - log_info " ✓ shared_preload_libraries contains orioledb" - - local tam - tam=$(docker exec -e PGPASSWORD="$POSTGRES_PASSWORD" "$container" \ - psql -U "$POSTGRES_USER" -d postgres -h /var/run/postgresql \ - -tAc "SHOW default_table_access_method;" 2>&1) || true - if ! echo "$tam" | grep -q "orioledb"; then - log_error " default_table_access_method is not orioledb (got: $tam)" - exit 1 - fi - log_info " ✓ default_table_access_method = orioledb" - - log_info "=== orioledb integration checks passed ===" + log_info "Multigres: initializing PostgreSQL cluster..." + docker exec -u postgres "$container" \ + bash -c "echo '$POSTGRES_PASSWORD' > /tmp/pgpwfile && \ + initdb \ + -D /var/lib/postgresql/data \ + --username=supabase_admin \ + --pwfile=/tmp/pgpwfile \ + --allow-group-access \ + --locale-provider=icu \ + --encoding=UTF-8 \ + --icu-locale=en_US.UTF-8 && \ + rm /tmp/pgpwfile" + + log_info "Multigres: starting PostgreSQL (config at /etc/postgresql)..." + docker exec -u postgres "$container" \ + pg_ctl start -D /var/lib/postgresql/data \ + -o "-c config_file=/etc/postgresql/postgresql.conf" \ + -w -t 60 + + log_info "Multigres: running schema migrations..." + docker exec \ + -e POSTGRES_PASSWORD="$POSTGRES_PASSWORD" \ + -e POSTGRES_DB="$POSTGRES_DB" \ + -e POSTGRES_HOST=/var/run/postgresql \ + -e POSTGRES_PORT=5432 \ + "$container" \ + sh /docker-entrypoint-initdb.d/migrate.sh } main() { @@ -474,8 +452,7 @@ writeShellApplication { -h|--help) print_help; exit 0 ;; --no-build) skip_build=true; shift ;; --keep) KEEP_CONTAINER=true; shift ;; - --target) if [[ -z "''${2:-}" ]]; then log_error "--target requires a value"; exit 1; fi; TARGET="$2"; shift; shift ;; - --pg-version) if [[ -z "''${2:-}" ]]; then log_error "--pg-version requires a value"; exit 1; fi; PG_VERSION="$2"; shift; shift ;; + --target) TARGET="$2"; shift; shift ;; -*) log_error "Unknown option: $1"; print_help; exit 1 ;; *) dockerfile="$1"; shift ;; esac @@ -506,12 +483,8 @@ writeShellApplication { if [[ -n "$TARGET" ]]; then target_arg="--target $TARGET" fi - local pg_version_arg="" - if [[ -n "$PG_VERSION" ]]; then - pg_version_arg="--build-arg PG_VERSION=$PG_VERSION" - fi # shellcheck disable=SC2086 - if ! docker build -f "$REPO_ROOT/$dockerfile" $target_arg $pg_version_arg -t "$IMAGE_TAG" "$REPO_ROOT"; then + if ! docker build -f "$REPO_ROOT/$dockerfile" $target_arg -t "$IMAGE_TAG" "$REPO_ROOT"; then log_error "Failed to build image" exit 1 fi @@ -551,10 +524,11 @@ writeShellApplication { -p "$PORT:5432" \ "$IMAGE_TAG" - # Multigres: pgctld server (PID 1) does not auto-start PostgreSQL. - # verify_pgctld_integration starts Postgres and ensures its listening for us. + # Multigres images use "tail -f /dev/null" as their entrypoint — postgres must be + # started manually before we can run tests against them. if [[ "$VERSION" == multigres-* ]]; then verify_pgctld_integration "$CONTAINER_NAME" "$VERSION" + start_multigres_postgres "$CONTAINER_NAME" fi if ! wait_for_postgres "localhost" "$PORT"; then @@ -563,11 +537,6 @@ writeShellApplication { exit 1 fi - # orioledb variant: verify orioledb loaded now that postgres is ready. - if [[ "$VERSION" == "multigres-orioledb-17" ]]; then - verify_orioledb_integration "$CONTAINER_NAME" - fi - log_info "Starting HTTP mock server on host..." HTTP_MOCK_PORT=8880 diff --git a/nix/packages/image-size-analyzer.nix b/nix/packages/image-size-analyzer.nix index e770d5e5cb..c12b1e0405 100644 --- a/nix/packages/image-size-analyzer.nix +++ b/nix/packages/image-size-analyzer.nix @@ -31,9 +31,8 @@ runCommand "image-size-analyzer" # Default values OUTPUT_JSON=false NO_BUILD=false - PG_VERSION="" declare -a IMAGES=() - ALL_DOCKERFILES=("Dockerfile-15" "Dockerfile-17" "Dockerfile-supabase" "Dockerfile-orioledb-17") + ALL_DOCKERFILES=("Dockerfile-15" "Dockerfile-17" "Dockerfile-orioledb-17") TIMESTAMP=$(date +%s) TEMP_DIR="/tmp/image-size-analyzer-$TIMESTAMP" @@ -46,9 +45,7 @@ runCommand "image-size-analyzer" Options: --json Output results as JSON instead of TUI --image DOCKERFILE Analyze specific Dockerfile (can be used multiple times) - Valid values: Dockerfile-15, Dockerfile-17, Dockerfile-supabase, Dockerfile-orioledb-17 - --pg-version VER PostgreSQL major version (required when --image Dockerfile-supabase) - Values: 15, 17 + Valid values: Dockerfile-15, Dockerfile-17, Dockerfile-orioledb-17 --no-build Skip building images, analyze existing ones --help Show this help message @@ -77,16 +74,8 @@ runCommand "image-size-analyzer" NO_BUILD=true shift ;; - --pg-version) - if [[ -z "''${2:-}" ]]; then - echo "Error: --pg-version requires a value" - exit 1 - fi - PG_VERSION="$2" - shift 2 - ;; --image) - if [[ -z "''${2:-}" ]]; then + if [[ -z "$2" ]]; then echo "Error: --image requires a value" exit 1 fi @@ -152,16 +141,8 @@ runCommand "image-size-analyzer" # Get tag name from Dockerfile name get_tag() { local dockerfile=$1 - if [[ "$dockerfile" == "Dockerfile-supabase" ]]; then - if [[ -z "''${PG_VERSION:-}" ]]; then - echo "Error: --pg-version required for Dockerfile-supabase" >&2 - exit 1 - fi - echo "supabase-postgres:$PG_VERSION-analyze" - else - local suffix=''${dockerfile#Dockerfile-} - echo "supabase-postgres:$suffix-analyze" - fi + local suffix=''${dockerfile#Dockerfile-} + echo "supabase-postgres:$suffix-analyze" } # Build a single image @@ -171,12 +152,7 @@ runCommand "image-size-analyzer" tag=$(get_tag "$dockerfile") echo "Building $dockerfile as $tag..." - local pg_version_arg="" - if [[ -n "$PG_VERSION" ]]; then - pg_version_arg="--build-arg PG_VERSION=$PG_VERSION" - fi - # shellcheck disable=SC2086 - if ! docker build -f "$dockerfile" $pg_version_arg -t "$tag" . ; then + if ! docker build -f "$dockerfile" -t "$tag" . ; then echo "Error: Failed to build $dockerfile" return 1 fi diff --git a/nix/tests/prime-multigres.sql b/nix/tests/prime-multigres.sql index 53b476539c..e7e65e576c 100644 --- a/nix/tests/prime-multigres.sql +++ b/nix/tests/prime-multigres.sql @@ -46,37 +46,14 @@ create extension if not exists pg_freespacemap; create extension if not exists pg_hashids; create extension if not exists pg_prewarm; create extension if not exists pgmq; --- Mirrors ansible/files/postgresql_extension_custom_scripts/pgmq/after-create.sql: --- reassign all pgmq objects to postgres (ownership hooks don't fire in Docker). -do $$ -declare r record; -begin - update pg_extension set extowner = 'postgres'::regrole where extname = 'pgmq'; - for r in select oid from pg_proc where pronamespace = 'pgmq'::regnamespace loop - execute format('alter function %s(%s) owner to postgres', - r.oid::regproc, pg_get_function_identity_arguments(r.oid)); - end loop; - for r in select relname from pg_class - where relnamespace = 'pgmq'::regnamespace - and relkind in ('r', 'S', 'v', 'm', 'p') loop - execute format('alter table pgmq.%I owner to postgres', r.relname); - end loop; -end $$; create extension if not exists pg_jsonschema; create schema if not exists partman; create extension if not exists pg_partman with schema partman; create extension if not exists pg_repack; --- Mirrors ansible/files/postgresql_extension_custom_scripts/pg_repack/after-create.sql -grant all on all tables in schema repack to postgres; -grant all on schema repack to postgres; -alter default privileges in schema repack grant all on tables to postgres; -alter default privileges in schema repack grant all on sequences to postgres; create extension if not exists pg_stat_monitor; create extension if not exists pg_stat_statements; create extension if not exists pg_surgery; create extension if not exists pg_tle; --- Mirrors ansible/files/postgresql_extension_custom_scripts/pg_tle/after-create.sql -grant pgtle_admin to postgres; create extension if not exists pg_trgm; create extension if not exists pg_visibility; create extension if not exists pg_walinspect; From 11edaf51bca9c2aa4ea4a1579ad76bcad8b81e3b Mon Sep 17 00:00:00 2001 From: Bobbie Soedirgo Date: Wed, 9 Sep 2026 02:34:13 +0800 Subject: [PATCH 2/2] chore: update snapshot Caused by https://github.com/supabase/postgres/pull/2240 --- .../z_multigres-orioledb-17_ext_interface.out | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/nix/tests/expected/z_multigres-orioledb-17_ext_interface.out b/nix/tests/expected/z_multigres-orioledb-17_ext_interface.out index c6d7b86175..ea3ac73d4e 100644 --- a/nix/tests/expected/z_multigres-orioledb-17_ext_interface.out +++ b/nix/tests/expected/z_multigres-orioledb-17_ext_interface.out @@ -163,14 +163,14 @@ order by address_standardizer | public | parse_address | text, OUT num text, OUT street text, OUT street2 text, OUT address1 text, OUT city text, OUT state text, OUT zip text, OUT zipplus text, OUT country text | record address_standardizer | public | standardize_address | lextab text, gaztab text, rultab text, address text | stdaddr address_standardizer | public | standardize_address | lextab text, gaztab text, rultab text, micro text, macro text | stdaddr - amcheck | public | bt_index_check | index regclass | void - amcheck | public | bt_index_check | index regclass, heapallindexed boolean, checkunique boolean | void - amcheck | public | bt_index_check | index regclass, heapallindexed boolean | void - amcheck | public | bt_index_parent_check | index regclass | void - amcheck | public | bt_index_parent_check | index regclass, heapallindexed boolean, rootdescend boolean, checkunique boolean | void - amcheck | public | bt_index_parent_check | index regclass, heapallindexed boolean | void - amcheck | public | bt_index_parent_check | index regclass, heapallindexed boolean, rootdescend boolean | void - amcheck | public | verify_heapam | relation regclass, on_error_stop boolean, check_toast boolean, skip text, startblock bigint, endblock bigint, OUT blkno bigint, OUT offnum integer, OUT attnum integer, OUT msg text | SETOF record + amcheck | pg_catalog | bt_index_check | index regclass | void + amcheck | pg_catalog | bt_index_check | index regclass, heapallindexed boolean, checkunique boolean | void + amcheck | pg_catalog | bt_index_check | index regclass, heapallindexed boolean | void + amcheck | pg_catalog | bt_index_parent_check | index regclass | void + amcheck | pg_catalog | bt_index_parent_check | index regclass, heapallindexed boolean, rootdescend boolean, checkunique boolean | void + amcheck | pg_catalog | bt_index_parent_check | index regclass, heapallindexed boolean | void + amcheck | pg_catalog | bt_index_parent_check | index regclass, heapallindexed boolean, rootdescend boolean | void + amcheck | pg_catalog | verify_heapam | relation regclass, on_error_stop boolean, check_toast boolean, skip text, startblock bigint, endblock bigint, OUT blkno bigint, OUT offnum integer, OUT attnum integer, OUT msg text | SETOF record autoinc | public | autoinc | | trigger bloom | public | blhandler | internal | index_am_handler btree_gin | public | gin_btree_consistent | internal, smallint, anyelement, integer, internal, internal | boolean