From a2f12f0f01d64bf9acf7d756aac6fd3554af69d1 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 23 May 2025 14:40:05 -0400 Subject: [PATCH 01/12] feat: support a pg 14 version of postgres bundle --- flake.nix | 13 ++++++++++++- nix/ext/pgaudit.nix | 4 ++++ nix/postgresql/14.nix | 14 ++++++++++++++ nix/postgresql/default.nix | 1 + nix/tools/run-client.sh.in | 8 ++++++-- nix/tools/run-server.sh.in | 9 +++++++-- 6 files changed, 44 insertions(+), 5 deletions(-) create mode 100644 nix/postgresql/14.nix diff --git a/flake.nix b/flake.nix index d49c9e3fa..5b99834b1 100644 --- a/flake.nix +++ b/flake.nix @@ -166,7 +166,11 @@ x != ./nix/ext/timescaledb-2.9.1.nix && x != ./nix/ext/plv8.nix ) ourExtensions; - + pg14FilteredExtensions = builtins.filter + ( + x: + x != ./nix/ext/pg_stat_monitor.nix + ) ourExtensions; orioledbExtensions = orioleFilteredExtensions ++ [ ./nix/ext/orioledb.nix ]; dbExtensions17 = orioleFilteredExtensions; getPostgresqlPackage = version: @@ -213,6 +217,8 @@ then orioledbExtensions else if (builtins.elem version [ "17" ]) then dbExtensions17 + else if (builtins.elem version [ "14" ]) + then pg14FilteredExtensions else ourExtensions; in map (path: pkgs.callPackage path { inherit postgresql; }) extensionsToUse; @@ -326,6 +332,7 @@ SHELL_PATH = "${pkgs.bash}/bin/bash"; PGSQL_DEFAULT_PORT = "${pgsqlDefaultPort}"; PGSQL_SUPERUSER = "${pgsqlSuperuser}"; + PSQL14_BINDIR = "${basePackages.psql_14.bin}"; PSQL15_BINDIR = "${basePackages.psql_15.bin}"; PSQL17_BINDIR = "${basePackages.psql_17.bin}"; PSQL_CONF_FILE = "${paths.pgconfigFile}"; @@ -392,6 +399,7 @@ # Define the available PostgreSQL versions postgresVersions = { + psql_14 = makePostgres "14"; psql_15 = makePostgres "15"; psql_17 = makePostgres "17"; psql_orioledb-17 = makePostgres "orioledb-17"; @@ -408,6 +416,7 @@ pkgs.callPackage ./nix/ext/pg_regress.nix { postgresql = postgresqlPackage; }; + postgresql_14 = getPostgresqlPackage "14"; postgresql_15 = getPostgresqlPackage "15"; postgresql_17 = getPostgresqlPackage "17"; postgresql_orioledb-17 = getPostgresqlPackage "orioledb-17"; @@ -418,6 +427,7 @@ cargo-pgrx_0_12_6 = pkgs.cargo-pgrx.cargo-pgrx_0_12_6; cargo-pgrx_0_12_9 = pkgs.cargo-pgrx.cargo-pgrx_0_12_9; # PostgreSQL versions. + psql_14 = postgresVersions.psql_14; psql_15 = postgresVersions.psql_15; psql_17 = postgresVersions.psql_17; psql_orioledb-17 = postgresVersions.psql_orioledb-17; @@ -515,6 +525,7 @@ substitute ${./nix/tools/run-client.sh.in} $out/bin/start-postgres-client \ --subst-var-by 'PGSQL_DEFAULT_PORT' '${pgsqlDefaultPort}' \ --subst-var-by 'PGSQL_SUPERUSER' '${pgsqlSuperuser}' \ + --subst-var-by 'PSQL14_BINDIR' '${basePackages.psql_14.bin}' \ --subst-var-by 'PSQL15_BINDIR' '${basePackages.psql_15.bin}' \ --subst-var-by 'PSQL17_BINDIR' '${basePackages.psql_17.bin}' \ --subst-var-by 'PSQLORIOLEDB17_BINDIR' '${basePackages.psql_orioledb-17.bin}' \ diff --git a/nix/ext/pgaudit.nix b/nix/ext/pgaudit.nix index 01ab66f15..1b8bfb8de 100644 --- a/nix/ext/pgaudit.nix +++ b/nix/ext/pgaudit.nix @@ -14,6 +14,10 @@ let version = "1.7.0"; hash = "sha256-8pShPr4HJaJQPjW1iPJIpj3CutTx8Tgr+rOqoXtgCcw="; }; + "14" = { + version = "1.6.3"; + hash = "sha256-KgLidJHjUK9BTp6ffmGUj1chcwIe6IzlcadRpGCfNdM="; + }; }.${lib.versions.major postgresql.version} or (throw "Source for pgaudit is not available for ${postgresql.version}"); in stdenv.mkDerivation { diff --git a/nix/postgresql/14.nix b/nix/postgresql/14.nix new file mode 100644 index 000000000..daec57264 --- /dev/null +++ b/nix/postgresql/14.nix @@ -0,0 +1,14 @@ +import ./generic.nix { + version = "14.18"; + hash = "sha256-g6sp1r/D3Fiy7TxmQRT9++tqBFDEuNf6aa7pHjyhT44="; + # muslPatches = { + # disable-test-collate-icu-utf8 = { + # url = "https://git.alpinelinux.org/aports/plain/main/postgresql14/disable-test-collate.icu.utf8.patch?id=56999e6d0265ceff5c5239f85fdd33e146f06cb7"; + # hash = "sha256-jXe23AxnFjEl+TZQm4R7rStk2Leo08ctxMNmu1xr5zM="; + # }; + # dont-use-locale-a = { + # url = "https://git.alpinelinux.org/aports/plain/main/postgresql14/dont-use-locale-a-on-musl.patch?id=56999e6d0265ceff5c5239f85fdd33e146f06cb7"; + # hash = "sha256-fk+y/SvyA4Tt8OIvDl7rje5dLs3Zw+Ln1oddyYzerOo="; + # }; + # }; +} \ No newline at end of file diff --git a/nix/postgresql/default.nix b/nix/postgresql/default.nix index f7b3c5583..e61583ddf 100644 --- a/nix/postgresql/default.nix +++ b/nix/postgresql/default.nix @@ -1,6 +1,7 @@ self: let versions = { + postgresql_14 = ./14.nix; postgresql_15 = ./15.nix; postgresql_17 = ./17.nix; postgresql_orioledb-17 = ./orioledb-17.nix; diff --git a/nix/tools/run-client.sh.in b/nix/tools/run-client.sh.in index 6acb4d6c0..329971178 100644 --- a/nix/tools/run-client.sh.in +++ b/nix/tools/run-client.sh.in @@ -13,7 +13,7 @@ print_help() { echo "Usage: nix run .#start-client -- [options]" echo echo "Options:" - echo " -v, --version [15|16|orioledb-16] Specify the PostgreSQL version to use (required)" + echo " -v, --version [14|15|16|orioledb-16] Specify the PostgreSQL version to use (required)" echo " -u, --user USER Specify the user/role to use (default: postgres)" echo " -h, --help Show this help message" echo @@ -81,7 +81,11 @@ if [[ -z "$PSQL_VERSION" ]]; then fi # Determine PostgreSQL version -if [ "$PSQL_VERSION" == "15" ]; then +if [ "$PSQL_VERSION" == "14" ]; then + echo "Starting client for PSQL 14" + PSQL14=@PSQL14_BINDIR@ + BINDIR="$PSQL14" +elif [ "$PSQL_VERSION" == "15" ]; then echo "Starting client for PSQL 15" PSQL15=@PSQL15_BINDIR@ BINDIR="$PSQL15" diff --git a/nix/tools/run-server.sh.in b/nix/tools/run-server.sh.in index a1e289d66..b68568546 100644 --- a/nix/tools/run-server.sh.in +++ b/nix/tools/run-server.sh.in @@ -141,7 +141,12 @@ else PGSODIUM_GETKEY_SCRIPT="${PGSODIUM_GETKEY_SCRIPT:-@PGSODIUM_GETKEY@}" fi # Verify version and set binary directory -if [ "$VERSION" == "15" ]; then + +if [ "$VERSION" == "14" ]; then + echo "Starting server for PSQL 14" + PSQL14=@PSQL14_BINDIR@ + BINDIR="$PSQL14" +elif [ "$VERSION" == "15" ]; then echo "Starting server for PSQL 15" PSQL15=@PSQL15_BINDIR@ BINDIR="$PSQL15" @@ -154,7 +159,7 @@ elif [ "$VERSION" == "orioledb-17" ]; then PSQLORIOLEDB17=@PSQLORIOLEDB17_BINDIR@ BINDIR="$PSQLORIOLEDB17" else - echo "Please provide a valid Postgres version (15, 17, orioledb-17)" + echo "Please provide a valid Postgres version (14, 15, 17, orioledb-17)" exit 1 fi From 60b25e45ee37b7e1a12dc7bbbf758a8a814baac7 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Fri, 23 May 2025 14:42:21 -0400 Subject: [PATCH 02/12] chore: cleanup pkg --- nix/postgresql/14.nix | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/nix/postgresql/14.nix b/nix/postgresql/14.nix index daec57264..35d48e3fe 100644 --- a/nix/postgresql/14.nix +++ b/nix/postgresql/14.nix @@ -1,14 +1,5 @@ import ./generic.nix { version = "14.18"; hash = "sha256-g6sp1r/D3Fiy7TxmQRT9++tqBFDEuNf6aa7pHjyhT44="; - # muslPatches = { - # disable-test-collate-icu-utf8 = { - # url = "https://git.alpinelinux.org/aports/plain/main/postgresql14/disable-test-collate.icu.utf8.patch?id=56999e6d0265ceff5c5239f85fdd33e146f06cb7"; - # hash = "sha256-jXe23AxnFjEl+TZQm4R7rStk2Leo08ctxMNmu1xr5zM="; - # }; - # dont-use-locale-a = { - # url = "https://git.alpinelinux.org/aports/plain/main/postgresql14/dont-use-locale-a-on-musl.patch?id=56999e6d0265ceff5c5239f85fdd33e146f06cb7"; - # hash = "sha256-fk+y/SvyA4Tt8OIvDl7rje5dLs3Zw+Ln1oddyYzerOo="; - # }; - # }; -} \ No newline at end of file + +} From 56c8c39954295df36ed4f96fba3029787e04b4ea Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 27 May 2025 15:52:43 -0400 Subject: [PATCH 03/12] test: collect garbage --- .github/workflows/test.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index a218ef882..d7c2dbd7d 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -81,6 +81,7 @@ jobs: echo "EOF" >> $GITHUB_OUTPUT - name: verify schema.sql is committed run: | + nix-collect-garbage -d nix run github:supabase/postgres/${{ github.sha }}#dbmate-tool -- --version ${{ env.PGMAJOR }} --flake-url github:supabase/postgres/${{ github.sha }} if ! git diff --exit-code --quiet migrations/schema-${{ env.PGMAJOR }}.sql; then echo "Detected changes in schema.sql:" From 9adc9fa444ac3f05c1da1b8b0a73df5ad8c44398 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 27 May 2025 16:29:19 -0400 Subject: [PATCH 04/12] chore: build and cacne pg 14 bundle too --- docker/nix/build_nix.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docker/nix/build_nix.sh b/docker/nix/build_nix.sh index efaafbe89..c36185941 100644 --- a/docker/nix/build_nix.sh +++ b/docker/nix/build_nix.sh @@ -14,6 +14,7 @@ nix build .#checks.$SYSTEM.psql_17 -L --no-link nix build .#psql_15/bin -o psql_15 -L nix build .#psql_orioledb-17/bin -o psql_orioledb_17 -L nix build .#psql_17/bin -o psql_17 -L +nix build .#psql_14/bin -o psql_14 -L nix build .#wal-g-2 -o wal-g-2 -L nix build .#wal-g-3 -o wal-g-3 -L @@ -23,6 +24,7 @@ nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./wal-g-3 nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_15 nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_orioledb_17 nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_17 +nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_14 if [ "$SYSTEM" = "aarch64-linux" ]; then nix build .#postgresql_15_debug -o ./postgresql_15_debug From e0524a1ec1a2c6b3bd3a5bab31d4566ddd8e0a2d Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Tue, 27 May 2025 18:45:18 -0400 Subject: [PATCH 05/12] feat: workflow_dispatch release of single version of postgres --- .github/workflows/ami-release-nix-single.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/ami-release-nix-single.yml b/.github/workflows/ami-release-nix-single.yml index f2c2abf9e..04e346d78 100644 --- a/.github/workflows/ami-release-nix-single.yml +++ b/.github/workflows/ami-release-nix-single.yml @@ -136,5 +136,9 @@ jobs: - name: Cleanup resources on build cancellation if: ${{ cancelled() }} run: | +<<<<<<< HEAD aws ec2 describe-instances --filters "Name=tag:packerExecutionId,Values=${GITHUB_RUN_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --instance-ids +======= + aws ec2 describe-instances --filters "Name=tag:packerExecutionId,Values=${GITHUB_RUN_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --instance-ids +>>>>>>> 0bbd35ab (feat: workflow_dispatch release of single version of postgres) From 98a161a215943492b869116a02e366925e0ad316 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 28 May 2025 15:02:10 -0400 Subject: [PATCH 06/12] chore: include pg 14 in versions --- .github/workflows/ami-release-nix-single.yml | 4 ---- ansible/vars.yml | 10 ++++++++-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ami-release-nix-single.yml b/.github/workflows/ami-release-nix-single.yml index 04e346d78..f2c2abf9e 100644 --- a/.github/workflows/ami-release-nix-single.yml +++ b/.github/workflows/ami-release-nix-single.yml @@ -136,9 +136,5 @@ jobs: - name: Cleanup resources on build cancellation if: ${{ cancelled() }} run: | -<<<<<<< HEAD aws ec2 describe-instances --filters "Name=tag:packerExecutionId,Values=${GITHUB_RUN_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --instance-ids -======= - aws ec2 describe-instances --filters "Name=tag:packerExecutionId,Values=${GITHUB_RUN_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --instance-ids ->>>>>>> 0bbd35ab (feat: workflow_dispatch release of single version of postgres) diff --git a/ansible/vars.yml b/ansible/vars.yml index 9a07c087c..e088a25ec 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -9,9 +9,15 @@ postgres_major: # Full version strings for each major version postgres_release: - postgresorioledb-17: "17.0.1.087-orioledb" - postgres17: "17.4.1.037" + postgresorioledb-17: "17.0.1.088-orioledb" + postgres17: "17.4.1.038" postgres15: "15.8.1.094" + postgres14: "14.18.1.001" + +# Note, postgres14 is not built on every release, but is available in the nix cache. +# postgres14 is included here to support upgrading from postgres14 to newer versions. +# an image can be created with a special workflow that builds postgres14 image. +# Supabase will not create and release postgres14 images other than as a utility for internal use. # Non Postgres Extensions pgbouncer_release: "1.19.0" From 89010a0bbac6feb66fc2ba37c9ff81f88cf6be9d Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Wed, 28 May 2025 16:05:48 -0400 Subject: [PATCH 07/12] chore: get and use the git commit from submitted branch --- .github/workflows/ami-release-nix-single.yml | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ami-release-nix-single.yml b/.github/workflows/ami-release-nix-single.yml index f2c2abf9e..86a1d40fc 100644 --- a/.github/workflows/ami-release-nix-single.yml +++ b/.github/workflows/ami-release-nix-single.yml @@ -28,6 +28,11 @@ jobs: with: ref: ${{ github.event.inputs.branch }} + - name: Get current branch SHA + id: get_sha + run: | + echo "sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT + - uses: DeterminateSystems/nix-installer-action@main - name: Set PostgreSQL version environment variable @@ -46,7 +51,7 @@ jobs: POSTGRES_MAJOR_VERSION: ${{ env.POSTGRES_MAJOR_VERSION }} run: | packer init amazon-arm64-nix.pkr.hcl - GIT_SHA=${{github.sha}} + GIT_SHA=${{ steps.get_sha.outputs.sha }} packer build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=-e postgresql_major=${POSTGRES_MAJOR_VERSION}" amazon-arm64-nix.pkr.hcl - name: Build AMI stage 2 @@ -54,7 +59,7 @@ jobs: POSTGRES_MAJOR_VERSION: ${{ env.POSTGRES_MAJOR_VERSION }} run: | packer init stage2-nix-psql.pkr.hcl - GIT_SHA=${{github.sha}} + GIT_SHA=${{ steps.get_sha.outputs.sha }} POSTGRES_MAJOR_VERSION=${{ env.POSTGRES_MAJOR_VERSION }} packer build -var "git_sha=${GIT_SHA}" -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var "postgres_major_version=${POSTGRES_MAJOR_VERSION}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" stage2-nix-psql.pkr.hcl @@ -66,7 +71,7 @@ jobs: - name: Create nix flake revision tarball run: | - GIT_SHA=${{github.sha}} + GIT_SHA=${{ steps.get_sha.outputs.sha }} MAJOR_VERSION=${{ env.POSTGRES_MAJOR_VERSION }} mkdir -p "/tmp/pg_upgrade_bin/${MAJOR_VERSION}" @@ -116,7 +121,7 @@ jobs: with: name: ${{ steps.process_release_version.outputs.version }} tag_name: ${{ steps.process_release_version.outputs.version }} - target_commitish: ${{github.sha}} + target_commitish: ${{ steps.get_sha.outputs.sha }} - name: Slack Notification on Failure if: ${{ failure() }} From 710a04af15523ed19230490e3b1fc4df415911e2 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 29 May 2025 08:54:50 -0400 Subject: [PATCH 08/12] chore: version bump for test --- ansible/vars.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/vars.yml b/ansible/vars.yml index e088a25ec..a4b8c0a0d 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -12,7 +12,7 @@ postgres_release: postgresorioledb-17: "17.0.1.088-orioledb" postgres17: "17.4.1.038" postgres15: "15.8.1.094" - postgres14: "14.18.1.001" + postgres14: "14.18.1.001-test-1" # Note, postgres14 is not built on every release, but is available in the nix cache. # postgres14 is included here to support upgrading from postgres14 to newer versions. From 977b9f4bad3051982598722aa4fbb8b8650dcea1 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 29 May 2025 09:25:40 -0400 Subject: [PATCH 09/12] chore: bump version and include workflow to run from this branch --- .github/workflows/ami-release-nix-single.yml | 133 +++++++++++++++++++ ansible/vars.yml | 2 +- 2 files changed, 134 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ami-release-nix-single.yml b/.github/workflows/ami-release-nix-single.yml index 86a1d40fc..bb84c420b 100644 --- a/.github/workflows/ami-release-nix-single.yml +++ b/.github/workflows/ami-release-nix-single.yml @@ -143,3 +143,136 @@ jobs: run: | aws ec2 describe-instances --filters "Name=tag:packerExecutionId,Values=${GITHUB_RUN_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --instance-ids +name: Release Single AMI Nix + +on: + workflow_dispatch: + inputs: + postgres_version: + description: 'PostgreSQL major version to build (e.g. 15)' + required: true + type: string + branch: + description: 'Branch to run the workflow from' + required: true + type: string + default: 'main' + +permissions: + contents: write + id-token: write + +jobs: + build: + runs-on: arm-runner + timeout-minutes: 150 + + steps: + - name: Checkout Repo + uses: actions/checkout@v3 + with: + ref: ${{ github.event.inputs.branch }} + + - name: Get current branch SHA + id: get_sha + run: | + echo "sha=$(git rev-parse HEAD)" >> $GITHUB_OUTPUT + - uses: DeterminateSystems/nix-installer-action@main + + - name: Set PostgreSQL version environment variable + run: echo "POSTGRES_MAJOR_VERSION=${{ github.event.inputs.postgres_version }}" >> $GITHUB_ENV + + - name: Generate common-nix.vars.pkr.hcl + run: | + PG_VERSION=$(sudo nix run nixpkgs#yq -- '.postgres_release["postgres'${{ env.POSTGRES_MAJOR_VERSION }}'"]' ansible/vars.yml) + PG_VERSION=$(echo "$PG_VERSION" | tr -d '"') # Remove any surrounding quotes + echo 'postgres-version = "'$PG_VERSION'"' > common-nix.vars.pkr.hcl + # Ensure there's a newline at the end of the file + echo "" >> common-nix.vars.pkr.hcl + - name: Build AMI stage 1 + env: + POSTGRES_MAJOR_VERSION: ${{ env.POSTGRES_MAJOR_VERSION }} + run: | + packer init amazon-arm64-nix.pkr.hcl + GIT_SHA=${{ steps.get_sha.outputs.sha }} + packer build -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" -var "ansible_arguments=-e postgresql_major=${POSTGRES_MAJOR_VERSION}" amazon-arm64-nix.pkr.hcl + - name: Build AMI stage 2 + env: + POSTGRES_MAJOR_VERSION: ${{ env.POSTGRES_MAJOR_VERSION }} + run: | + packer init stage2-nix-psql.pkr.hcl + GIT_SHA=${{ steps.get_sha.outputs.sha }} + POSTGRES_MAJOR_VERSION=${{ env.POSTGRES_MAJOR_VERSION }} + packer build -var "git_sha=${GIT_SHA}" -var "git-head-version=${GIT_SHA}" -var "packer-execution-id=${GITHUB_RUN_ID}" -var "postgres_major_version=${POSTGRES_MAJOR_VERSION}" -var-file="development-arm.vars.pkr.hcl" -var-file="common-nix.vars.pkr.hcl" stage2-nix-psql.pkr.hcl + - name: Grab release version + id: process_release_version + run: | + VERSION=$(cat common-nix.vars.pkr.hcl | sed -e 's/postgres-version = "\(.*\)"/\1/g') + echo "version=$VERSION" >> $GITHUB_OUTPUT + - name: Create nix flake revision tarball + run: | + GIT_SHA=${{ steps.get_sha.outputs.sha }} + MAJOR_VERSION=${{ env.POSTGRES_MAJOR_VERSION }} + mkdir -p "/tmp/pg_upgrade_bin/${MAJOR_VERSION}" + echo "$GIT_SHA" >> "/tmp/pg_upgrade_bin/${MAJOR_VERSION}/nix_flake_version" + tar -czf "/tmp/pg_binaries.tar.gz" -C "/tmp/pg_upgrade_bin" . + - name: configure aws credentials - staging + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets.DEV_AWS_ROLE }} + aws-region: "us-east-1" + + - name: Upload software manifest to s3 staging + run: | + cd ansible + ansible-playbook -i localhost \ + -e "ami_release_version=${{ steps.process_release_version.outputs.version }}" \ + -e "internal_artifacts_bucket=${{ secrets.ARTIFACTS_BUCKET }}" \ + -e "postgres_major_version=${{ env.POSTGRES_MAJOR_VERSION }}" \ + manifest-playbook.yml + - name: Upload nix flake revision to s3 staging + run: | + aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/20.04.tar.gz + - name: configure aws credentials - prod + uses: aws-actions/configure-aws-credentials@v4 + with: + role-to-assume: ${{ secrets.PROD_AWS_ROLE }} + aws-region: "us-east-1" + + - name: Upload software manifest to s3 prod + run: | + cd ansible + ansible-playbook -i localhost \ + -e "ami_release_version=${{ steps.process_release_version.outputs.version }}" \ + -e "internal_artifacts_bucket=${{ secrets.PROD_ARTIFACTS_BUCKET }}" \ + -e "postgres_major_version=${{ env.POSTGRES_MAJOR_VERSION }}" \ + manifest-playbook.yml + + - name: Upload nix flake revision to s3 prod + run: | + aws s3 cp /tmp/pg_binaries.tar.gz s3://${{ secrets.PROD_ARTIFACTS_BUCKET }}/upgrades/postgres/supabase-postgres-${{ steps.process_release_version.outputs.version }}/20.04.tar.gz + - name: Create release + uses: softprops/action-gh-release@v2 + with: + name: ${{ steps.process_release_version.outputs.version }} + tag_name: ${{ steps.process_release_version.outputs.version }} + target_commitish: ${{ steps.get_sha.outputs.sha }} + + - name: Slack Notification on Failure + if: ${{ failure() }} + uses: rtCamp/action-slack-notify@v2 + env: + SLACK_WEBHOOK: ${{ secrets.SLACK_NOTIFICATIONS_WEBHOOK }} + SLACK_USERNAME: 'gha-failures-notifier' + SLACK_COLOR: 'danger' + SLACK_MESSAGE: 'Building Postgres AMI failed' + SLACK_FOOTER: '' + + - name: Cleanup resources after build + if: ${{ always() }} + run: | + aws ec2 describe-instances --filters "Name=tag:packerExecutionId,Values=${GITHUB_RUN_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --instance-ids + - name: Cleanup resources on build cancellation + if: ${{ cancelled() }} + run: | + aws ec2 describe-instances --filters "Name=tag:packerExecutionId,Values=${GITHUB_RUN_ID}" --query "Reservations[].Instances[].InstanceId" --output text | xargs -r aws ec2 terminate-instances --instance-ids diff --git a/ansible/vars.yml b/ansible/vars.yml index a4b8c0a0d..287310b45 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -12,7 +12,7 @@ postgres_release: postgresorioledb-17: "17.0.1.088-orioledb" postgres17: "17.4.1.038" postgres15: "15.8.1.094" - postgres14: "14.18.1.001-test-1" + postgres14: "14.18.1.001-test-2" # Note, postgres14 is not built on every release, but is available in the nix cache. # postgres14 is included here to support upgrading from postgres14 to newer versions. From 56c1f27ca0172a13fc7f89e3d78f5fd6db5d5922 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 29 May 2025 09:54:14 -0400 Subject: [PATCH 10/12] feat: include src and debug for pg14 --- flake.nix | 99 +++++++++++++++++++------------------------------------ 1 file changed, 34 insertions(+), 65 deletions(-) diff --git a/flake.nix b/flake.nix index 5b99834b1..5d1c7d22d 100644 --- a/flake.nix +++ b/flake.nix @@ -420,6 +420,34 @@ postgresql_15 = getPostgresqlPackage "15"; postgresql_17 = getPostgresqlPackage "17"; postgresql_orioledb-17 = getPostgresqlPackage "orioledb-17"; + + # Function to create PostgreSQL source packages + mkPostgresSrc = name: pkg: pkgs.stdenv.mkDerivation { + pname = "postgresql-${name}-src"; + version = pkg.version; + src = pkg.src; + nativeBuildInputs = [ pkgs.bzip2 ]; + + phases = [ "unpackPhase" "installPhase" ]; + + installPhase = '' + mkdir -p $out + cp -r . $out + ''; + + meta = with pkgs.lib; { + description = "PostgreSQL ${name} source files"; + homepage = "https://www.postgresql.org/"; + license = licenses.postgresql; + platforms = platforms.all; + }; + }; + + # Create source packages for each PostgreSQL version + postgresql_14_src = mkPostgresSrc "14" postgresql_14; + postgresql_15_src = mkPostgresSrc "15" postgresql_15; + postgresql_17_src = mkPostgresSrc "17" postgresql_17; + postgresql_orioledb-17_src = mkPostgresSrc "orioledb-17" postgresql_orioledb-17; in postgresVersions // { supabase-groonga = supabase-groonga; @@ -435,74 +463,15 @@ wal-g-3 = wal-g-3; sfcgal = sfcgal; pg_prove = pkgs.perlPackages.TAPParserSourceHandlerpgTAP; - inherit postgresql_15 postgresql_17 postgresql_orioledb-17; + inherit postgresql_14 postgresql_15 postgresql_17 postgresql_orioledb-17; + postgresql_14_debug = if pkgs.stdenv.isLinux then postgresql_14.debug else null; postgresql_15_debug = if pkgs.stdenv.isLinux then postgresql_15.debug else null; postgresql_17_debug = if pkgs.stdenv.isLinux then postgresql_17.debug else null; postgresql_orioledb-17_debug = if pkgs.stdenv.isLinux then postgresql_orioledb-17.debug else null; - postgresql_15_src = pkgs.stdenv.mkDerivation { - pname = "postgresql-15-src"; - version = postgresql_15.version; - - src = postgresql_15.src; - - nativeBuildInputs = [ pkgs.bzip2 ]; - - phases = [ "unpackPhase" "installPhase" ]; - - installPhase = '' - mkdir -p $out - cp -r . $out - ''; - - meta = with pkgs.lib; { - description = "PostgreSQL 15 source files"; - homepage = "https://www.postgresql.org/"; - license = licenses.postgresql; - platforms = platforms.all; - }; - }; - postgresql_17_src = pkgs.stdenv.mkDerivation { - pname = "postgresql-17-src"; - version = postgresql_17.version; - src = postgresql_17.src; - - nativeBuildInputs = [ pkgs.bzip2 ]; - - phases = [ "unpackPhase" "installPhase" ]; - - installPhase = '' - mkdir -p $out - cp -r . $out - ''; - meta = with pkgs.lib; { - description = "PostgreSQL 17 source files"; - homepage = "https://www.postgresql.org/"; - license = licenses.postgresql; - platforms = platforms.all; - }; - }; - postgresql_orioledb-17_src = pkgs.stdenv.mkDerivation { - pname = "postgresql-17-src"; - version = postgresql_orioledb-17.version; - - src = postgresql_orioledb-17.src; - - nativeBuildInputs = [ pkgs.bzip2 ]; - - phases = [ "unpackPhase" "installPhase" ]; - - installPhase = '' - mkdir -p $out - cp -r . $out - ''; - - meta = with pkgs.lib; { - description = "PostgreSQL 15 source files"; - homepage = "https://www.postgresql.org/"; - license = licenses.postgresql; - platforms = platforms.all; - }; - }; + postgresql_14_src = postgresql_14_src; + postgresql_15_src = postgresql_15_src; + postgresql_17_src = postgresql_17_src; + postgresql_orioledb-17_src = postgresql_orioledb-17_src; mecab_naist_jdic = mecab-naist-jdic; supabase_groonga = supabase-groonga; pg_regress = makePgRegress activeVersion; From 50e7fdb5e3ccb0c49d004fa47533db7c018cd322 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 29 May 2025 10:35:21 -0400 Subject: [PATCH 11/12] chore: build/cache 14 src and debug --- docker/nix/build_nix.sh | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docker/nix/build_nix.sh b/docker/nix/build_nix.sh index c36185941..27eeebeb3 100644 --- a/docker/nix/build_nix.sh +++ b/docker/nix/build_nix.sh @@ -33,6 +33,10 @@ if [ "$SYSTEM" = "aarch64-linux" ]; then nix build .#postgresql_orioledb-17_src -o ./postgresql_orioledb-17_src nix build .#postgresql_17_debug -o ./postgresql_17_debug nix build .#postgresql_17_src -o ./postgresql_17_src + nix build .#postgresql_14_debug -o ./postgresql_14_debug + nix build .#postgresql_14_src -o ./postgresql_14_src + nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./postgresql_14_debug-debug + nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./postgresql_14_src nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./postgresql_15_debug-debug nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./postgresql_15_src nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./postgresql_orioledb-17_debug-debug From 8065c4cba6b490a8fea343b36a2f20c7b3c26192 Mon Sep 17 00:00:00 2001 From: Sam Rose Date: Thu, 29 May 2025 11:37:51 -0400 Subject: [PATCH 12/12] chore: bump test version --- ansible/vars.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/vars.yml b/ansible/vars.yml index 287310b45..bf64beaa6 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -12,7 +12,7 @@ postgres_release: postgresorioledb-17: "17.0.1.088-orioledb" postgres17: "17.4.1.038" postgres15: "15.8.1.094" - postgres14: "14.18.1.001-test-2" + postgres14: "14.18.1.001-test-3" # Note, postgres14 is not built on every release, but is available in the nix cache. # postgres14 is included here to support upgrading from postgres14 to newer versions.