diff --git a/.github/workflows/ami-release-nix.yml b/.github/workflows/ami-release-nix.yml index 773aee018..919e89396 100644 --- a/.github/workflows/ami-release-nix.yml +++ b/.github/workflows/ami-release-nix.yml @@ -11,6 +11,7 @@ on: - flake.lock - flake.nix - nix/packages/build-ami.nix + - nix/** workflow_dispatch: permissions: @@ -203,6 +204,24 @@ jobs: echo "Catalog uploaded to ${CATALOG_S3}" + - name: Update site-env catalogs + run: | + GIT_SHA="${{ steps.resolve-git-sha.outputs.sha }}" + SYSTEM=$(nix eval --impure --raw --expr 'builtins.currentSystem') + + SITE_ENV_NAME="site-env-${POSTGRES_MAJOR_VERSION}" + SITE_ENV_PATH=$(nix eval --raw ".#${SITE_ENV_NAME}.outPath") + jq -n --arg sys "$SYSTEM" --arg path "$SITE_ENV_PATH" '{($sys): $path}' > /tmp/site-env-catalog.json + aws s3 cp /tmp/site-env-catalog.json \ + "s3://${{ secrets.SHARED_AWS_ARTIFACTS_BUCKET }}/nix-catalog/${GIT_SHA}-${SITE_ENV_NAME}-${SYSTEM}.json" \ + --content-type "application/json" + + UPDATE_PROFILE_PATH=$(nix eval --raw ".#update-profile.outPath") + jq -n --arg sys "$SYSTEM" --arg path "$UPDATE_PROFILE_PATH" '{($sys): $path}' > /tmp/update-profile-catalog.json + aws s3 cp /tmp/update-profile-catalog.json \ + "s3://${{ secrets.SHARED_AWS_ARTIFACTS_BUCKET }}/nix-catalog/${GIT_SHA}-update-profile-${SYSTEM}.json" \ + --content-type "application/json" + - name: Create release uses: softprops/action-gh-release@a06a81a03ee405af7f2048a818ed3f03bbf83c7b # v2.5.0 with: diff --git a/ansible/tasks/stage2-setup-postgres.yml b/ansible/tasks/stage2-setup-postgres.yml index d4823238a..4a4a83e73 100644 --- a/ansible/tasks/stage2-setup-postgres.yml +++ b/ansible/tasks/stage2-setup-postgres.yml @@ -77,6 +77,17 @@ nix-env --set {{ postgres_env_path.stdout }} " + - name: Resolve update-profile store path + ansible.builtin.shell: | + . /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && + nix build --no-link --print-out-paths github:supabase/postgres/{{ git_commit_sha }}#update-profile + register: update_profile_path + + - name: Install update-profile + ansible.builtin.shell: | + . /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && + nix-env --profile /nix/var/nix/profiles/update-profile --set {{ update_profile_path.stdout }} + - name: Install supascan for baseline validation ansible.builtin.shell: | sudo -u ubuntu bash -c ". /nix/var/nix/profiles/default/etc/profile.d/nix-daemon.sh && nix profile install github:supabase/postgres/{{ git_commit_sha }}#supascan" diff --git a/nix/ext/tests/update-profile.nix b/nix/ext/tests/update-profile.nix new file mode 100644 index 000000000..f34ddf37d --- /dev/null +++ b/nix/ext/tests/update-profile.nix @@ -0,0 +1,29 @@ +{ self, pkgs }: +let + system = pkgs.pkgsLinux.stdenv.hostPlatform.system; + update-profile = self.packages.${system}.update-profile; + site-env-17 = self.packages.${system}."site-env-17"; +in +pkgs.testers.runNixOSTest { + name = "update-profile"; + nodes.machine = + { ... }: + { + environment.systemPackages = [ + update-profile + site-env-17 + ]; + }; + testScript = '' + machine.succeed("echo '{\"${system}\": \"${site-env-17}\"}' > /tmp/catalog.json") + # sha is only needed to fetch from S3 — omitted here since UPDATE_PROFILE_CATALOG bypasses that + machine.succeed("UPDATE_PROFILE_CATALOG=/tmp/catalog.json update-profile site-env-17") + machine.succeed("[ \"$(readlink -f /nix/var/nix/profiles/site-env-17)\" = \"${site-env-17}\" ]") + + # idempotent: same catalog again is a no-op success + machine.succeed("UPDATE_PROFILE_CATALOG=/tmp/catalog.json update-profile site-env-17") + + # wrong profile for the resolved path: must refuse + machine.fail("UPDATE_PROFILE_CATALOG=/tmp/catalog.json update-profile site-env-15") + ''; +} diff --git a/nix/packages/extension-catalog.nix b/nix/packages/extension-catalog.nix index d8d559121..d6e54aab9 100644 --- a/nix/packages/extension-catalog.nix +++ b/nix/packages/extension-catalog.nix @@ -110,8 +110,8 @@ makeWrapper ${self'.packages.site-extensions-resolve}/bin/site-extensions-resolve \ "$out/bin/site-extensions-resolve" \ --set PG_EXTENSIONS_CATALOG "$out/share/pg-extensions-catalog.json" - makeWrapper ${self'.packages.site-extensions-update}/bin/site-extensions-update \ - "$out/bin/site-extensions-update" \ + makeWrapper ${self'.packages.update-site-extensions}/bin/update-site-extensions \ + "$out/bin/update-site-extensions" \ --set PG_EXTENSIONS_CATALOG "$out/share/pg-extensions-catalog.json" '' ) @@ -148,18 +148,16 @@ # Takes manifest json as argument. # Downloads paths and installs them as an env into the profile, replacing all existing ones. - site-extensions-update = pkgs.writeShellApplication { - name = "site-extensions-update"; + update-site-extensions = pkgs.writeShellApplication { + name = "update-site-extensions"; runtimeInputs = [ self'.packages.site-extensions-resolve - pkgs.nix + self'.packages.update-profile-paths ]; text = '' manifest="''${1:?Usage: $0 path-to/pg-extensions.json}" - profile="''${NIX_PROFILE:-/nix/var/nix/profiles/site-extensions}" readarray -t paths < <(site-extensions-resolve "$manifest") - nix-store -r --option stalled-download-timeout 120 "''${paths[@]}" >/dev/null - nix-env --profile "$profile" --install "''${paths[@]}" --remove-all + update-profile-paths site-extensions "''${paths[@]}" ''; }; }; diff --git a/nix/packages/site-env.nix b/nix/packages/site-env.nix index 4fa580e99..b53ef165c 100644 --- a/nix/packages/site-env.nix +++ b/nix/packages/site-env.nix @@ -1,5 +1,5 @@ # These are envs (package sets per pg major version) deployed to instances -# at /nix/var/nix/profiles/site and updated regularly. +# at /nix/var/nix/profiles/ and updated regularly. { perSystem = { @@ -28,9 +28,62 @@ lib.optionals pkgs.stdenv.isLinux [ self'.packages.gatekeeper ] ); }; + + # Given a profile name (e.g. site-env-17, postgres-env-17) and a git sha, + # fetches that name's catalog entry and flips /nix/var/nix/profiles/ + # to it. Generic across any single-package catalog entry named -.json. + # Assumes `aws` is provided by the environment (AMIs already install AWS CLI v2). + update-profile = pkgs.writeShellApplication { + name = "update-profile"; + runtimeInputs = [ + pkgs.jq + pkgs.nix + ]; + text = '' + profile_name="''${1:?Usage: $0 }" + system="$(uname -m)-linux" + profile_path="/nix/var/nix/profiles/''${profile_name}" + + catalog="''${UPDATE_PROFILE_CATALOG:-}" + if [[ -z "$catalog" ]]; then + sha="''${2:?Usage: $0 }" + catalog="/tmp/''${profile_name}-catalog-''${sha}-''${system}.json" + aws s3 cp "s3://supabase-internal-artifacts/nix-catalog/''${sha}-''${profile_name}-''${system}.json" \ + "$catalog" --region ap-southeast-1 + fi + + path="$(jq -er --arg s "$system" '.[$s]' "$catalog")" + [[ "$(basename "$path")" == *"-''${profile_name}" ]] || { + echo "error: resolved path $path is not tagged for profile $profile_name" >&2 + exit 1 + } + + [[ "$(readlink -f "$profile_path")" == "$path" ]] && exit 0 + nix-store --realise --option stalled-download-timeout 120 "$path" >/dev/null + nix-env --profile "$profile_path" --set "$path" + ''; + }; + + # Given a profile name and already-resolved store paths, installs them + # as an env into /nix/var/nix/profiles/, replacing all existing ones. + update-profile-paths = pkgs.writeShellApplication { + name = "update-profile-paths"; + runtimeInputs = [ pkgs.nix ]; + text = '' + profile_name="''${1:?Usage: $0 ...}" + shift + [ "$#" -ge 1 ] || { echo "Usage: $0 ..." >&2; exit 1; } + nix-store --realise --option stalled-download-timeout 120 "$@" >/dev/null + nix-env --profile "/nix/var/nix/profiles/''${profile_name}" --install "$@" --remove-all + ''; + }; in { - packages = siteEnvs; - legacyPackages = siteEnvs; + packages = siteEnvs // { + inherit update-profile update-profile-paths; + }; + legacyPackages = siteEnvs // { + inherit update-profile update-profile-paths; + }; }; }