From 549f9eb4e5eca6465f6cb2695b644a50d4ff6955 Mon Sep 17 00:00:00 2001 From: Bobbie Soedirgo Date: Thu, 26 Sep 2024 13:53:44 +0100 Subject: [PATCH 01/44] fix: account for `public` grantee --- ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh index e978b2a54..4c776844b 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh @@ -356,6 +356,7 @@ begin end , case when rec.grantee = 'postgres'::regrole then 'supabase_admin' when rec.grantee = 'supabase_admin'::regrole then 'postgres' + when rec.grantee = 0 then 'public' else rec.grantee::regrole end )); @@ -382,7 +383,7 @@ begin when obj->>'objtype' = 'T' then 'types' when obj->>'objtype' = 'n' then 'schemas' end - , rec.grantee::regrole + , case when rec.grantee = 0 then 'public' else rec.grantee::regrole end , case when rec.is_grantable then 'with grant option' else '' end )); end if; From c83b1c6eb4d3bdf2af9e5b5477b56a924a9eae84 Mon Sep 17 00:00:00 2001 From: Bobbie Soedirgo <31685197+soedirgo@users.noreply.github.com> Date: Thu, 26 Sep 2024 14:19:38 +0100 Subject: [PATCH 02/44] fix(ci): respect postgresVersion input (#1237) --- .github/workflows/publish-nix-pgupgrade-scripts.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/publish-nix-pgupgrade-scripts.yml b/.github/workflows/publish-nix-pgupgrade-scripts.yml index 5d373ad8f..eb5f7a755 100644 --- a/.github/workflows/publish-nix-pgupgrade-scripts.yml +++ b/.github/workflows/publish-nix-pgupgrade-scripts.yml @@ -72,6 +72,9 @@ jobs: id: process_release_version run: | VERSION=$(grep 'postgres-version' common-nix.vars.pkr.hcl | sed -e 's/postgres-version = "\(.*\)"/\1/g') + if [[ "${{ inputs.postgresVersion }}" != "" ]]; then + VERSION=${{ inputs.postgresVersion }} + fi echo "version=$VERSION" >> "$GITHUB_OUTPUT" - name: Create a tarball containing pg_upgrade scripts From 5671a3996e90ce9fcf23881ac9de28f88b78c3ba Mon Sep 17 00:00:00 2001 From: Kang Ming Date: Sat, 28 Sep 2024 06:33:03 -0700 Subject: [PATCH 03/44] feat: bump gotrue version to v2.162.0 (#1241) --- ansible/vars.yml | 4 ++-- common-nix.vars.pkr.hcl | 2 +- common.vars.pkr.hcl | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/ansible/vars.yml b/ansible/vars.yml index e76e99cb0..471b073c6 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -17,8 +17,8 @@ postgrest_release: "12.2.3" postgrest_arm_release_checksum: sha1:fbfd6613d711ce1afa25c42d5df8f1b017f396f9 postgrest_x86_release_checksum: sha1:61c513f91a8931be4062587b9d4a18b42acf5c05 -gotrue_release: 2.161.0 -gotrue_release_checksum: sha1:8e45f3511fee8f99a0b1567c73673991a0a5986c +gotrue_release: 2.162.0 +gotrue_release_checksum: sha1:855b23bd002577290c7d42d7042ac0f5316984b1 aws_cli_release: "2.2.7" diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index bacdd2c63..b7089a21b 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.124" +postgres-version = "15.6.1.125" diff --git a/common.vars.pkr.hcl b/common.vars.pkr.hcl index 3c999dccd..fa1387690 100644 --- a/common.vars.pkr.hcl +++ b/common.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.1.1.93" +postgres-version = "15.1.1.94" From a2f1ff813807601ba8e22a921c5358aa2f5916a7 Mon Sep 17 00:00:00 2001 From: Bobbie Soedirgo <31685197+soedirgo@users.noreply.github.com> Date: Mon, 30 Sep 2024 16:32:13 +0100 Subject: [PATCH 04/44] fix: only grant pg_read_all_data if it exists (#1242) * fix: only grant pg_read_all_data if it exists * fix: prevent `public` from being casted into `regrole` --- .../admin_api_scripts/pg_upgrade_scripts/common.sh | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh index 4c776844b..a11f8ff32 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh @@ -357,7 +357,7 @@ begin , case when rec.grantee = 'postgres'::regrole then 'supabase_admin' when rec.grantee = 'supabase_admin'::regrole then 'postgres' when rec.grantee = 0 then 'public' - else rec.grantee::regrole + else rec.grantee::regrole::text end )); end if; @@ -383,7 +383,7 @@ begin when obj->>'objtype' = 'T' then 'types' when obj->>'objtype' = 'n' then 'schemas' end - , case when rec.grantee = 0 then 'public' else rec.grantee::regrole end + , case when rec.grantee = 0 then 'public' else rec.grantee::regrole::text end , case when rec.is_grantable then 'with grant option' else '' end )); end if; @@ -530,7 +530,14 @@ $$; alter database postgres connection limit -1; -- #incident-2024-09-12-project-upgrades-are-temporarily-disabled -grant pg_read_all_data, pg_signal_backend to postgres; +do $$ +begin + if exists (select from pg_authid where rolname = 'pg_read_all_data') then + execute('grant pg_read_all_data to postgres'); + end if; +end +$$; +grant pg_signal_backend to postgres; set session authorization supabase_admin; drop role supabase_tmp; From d71ec5f7f2465a35654a1070d02ea377026a6b5f Mon Sep 17 00:00:00 2001 From: Paul Cioanca Date: Wed, 2 Oct 2024 12:58:08 +0300 Subject: [PATCH 05/44] fix(15.6): account for pg_stat_monitor major version upgrade (#1247) --- ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh index 9443d2acb..c0a213d02 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh @@ -11,6 +11,7 @@ # them depending on regtypes referencing system OIDs or outdated library files. EXTENSIONS_TO_DISABLE=( "pg_graphql" + "pg_stat_monitor" ) PG14_EXTENSIONS_TO_DISABLE=( From 8706e3adf1ee060f77e7a1bdeb0e1d4bbfbb684b Mon Sep 17 00:00:00 2001 From: angelico Date: Wed, 2 Oct 2024 18:37:29 +0800 Subject: [PATCH 06/44] chore: release updates to run physical backup as a service to 15.6 image (#1248) * chore: updates to run physical backups as a service (#1235) * chore: init commence-backup service * chore: bump adminapi and adminmgr * chore: bump version * fix: provide correct filename * chore: bump postgres version --- ansible/files/adminapi.sudoers.conf | 2 ++ ansible/files/commence-backup.service.j2 | 12 ++++++++++++ ansible/tasks/internal/admin-api.yml | 5 +++++ ansible/vars.yml | 4 ++-- common-nix.vars.pkr.hcl | 2 +- 5 files changed, 22 insertions(+), 3 deletions(-) create mode 100644 ansible/files/commence-backup.service.j2 diff --git a/ansible/files/adminapi.sudoers.conf b/ansible/files/adminapi.sudoers.conf index eada0a94b..ae5537715 100644 --- a/ansible/files/adminapi.sudoers.conf +++ b/ansible/files/adminapi.sudoers.conf @@ -17,6 +17,8 @@ Cmnd_Alias PGBOUNCER = /bin/systemctl start pgbouncer.service, /bin/systemctl st %adminapi ALL= NOPASSWD: /usr/bin/systemctl restart postgresql.service %adminapi ALL= NOPASSWD: /usr/bin/systemctl show -p NRestarts postgresql.service %adminapi ALL= NOPASSWD: /usr/bin/systemctl restart adminapi.service +%adminapi ALL= NOPASSWD: /usr/bin/systemctl is-active commence-backup.service +%adminapi ALL= NOPASSWD: /usr/bin/systemctl start commence-backup.service %adminapi ALL= NOPASSWD: /bin/systemctl daemon-reload %adminapi ALL= NOPASSWD: /bin/systemctl restart services.slice %adminapi ALL= NOPASSWD: /usr/sbin/nft -f /etc/nftables/supabase_managed.conf diff --git a/ansible/files/commence-backup.service.j2 b/ansible/files/commence-backup.service.j2 new file mode 100644 index 000000000..9d4ad0c68 --- /dev/null +++ b/ansible/files/commence-backup.service.j2 @@ -0,0 +1,12 @@ +[Unit] +Description=Async commence physical backup + +[Service] +Type=simple +User=adminapi +ExecStart=/usr/bin/admin-mgr commence-backup --run-as-service true +Restart=no +OOMScoreAdjust=-1000 + +[Install] +WantedBy=multi-user.target diff --git a/ansible/tasks/internal/admin-api.yml b/ansible/tasks/internal/admin-api.yml index cea0109fd..da93fef62 100644 --- a/ansible/tasks/internal/admin-api.yml +++ b/ansible/tasks/internal/admin-api.yml @@ -79,6 +79,11 @@ src: files/adminapi.service.j2 dest: /etc/systemd/system/adminapi.service +- name: adminapi - create service file for commence backup process + template: + src: files/commence-backup.service.j2 + dest: /etc/systemd/system/commence-backup.service + - name: UFW - Allow connections to adminapi ports ufw: rule: allow diff --git a/ansible/vars.yml b/ansible/vars.yml index 471b073c6..3c5ef2172 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -50,8 +50,8 @@ postgres_exporter_release_checksum: arm64: sha256:29ba62d538b92d39952afe12ee2e1f4401250d678ff4b354ff2752f4321c87a0 amd64: sha256:cb89fc5bf4485fb554e0d640d9684fae143a4b2d5fa443009bd29c59f9129e84 -adminapi_release: 0.68.0 -adminmgr_release: 0.22.1 +adminapi_release: 0.71.1 +adminmgr_release: 0.24.0 # Postgres Extensions postgis_release: "3.3.2" diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index b7089a21b..3a55dd70e 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.125" +postgres-version = "15.6.1.126" From 1d1396e434000431e09cd723c9268afb64611b9b Mon Sep 17 00:00:00 2001 From: Paul Cioanca Date: Wed, 2 Oct 2024 14:57:18 +0300 Subject: [PATCH 07/44] fix(15.6/pg_upgrade): retry commands within the cleanup step; wait until PG is ready to accept connections (#1250) --- .../admin_api_scripts/pg_upgrade_scripts/initiate.sh | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh index c0a213d02..31a7cd5b3 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh @@ -120,20 +120,22 @@ cleanup() { CI_start_postgres fi + retry 8 pg_isready -h localhost -U supabase_admin + echo "Re-enabling extensions" if [ -f $POST_UPGRADE_EXTENSION_SCRIPT ]; then - run_sql -f $POST_UPGRADE_EXTENSION_SCRIPT + retry 5 run_sql -f $POST_UPGRADE_EXTENSION_SCRIPT fi echo "Removing SUPERUSER grant from postgres" - run_sql -c "ALTER USER postgres WITH NOSUPERUSER;" + retry 5 run_sql -c "ALTER USER postgres WITH NOSUPERUSER;" echo "Resetting postgres database connection limit" - run_sql -c "ALTER DATABASE postgres CONNECTION LIMIT -1;" + retry 5 run_sql -c "ALTER DATABASE postgres CONNECTION LIMIT -1;" if [ -z "$IS_CI" ] && [ -z "$IS_LOCAL_UPGRADE" ]; then echo "Unmounting data disk from ${MOUNT_POINT}" - umount $MOUNT_POINT + retry 3 umount $MOUNT_POINT fi echo "$UPGRADE_STATUS" > /tmp/pg-upgrade-status From 320305e9721b3055c91560187cd6a7f52cd56280 Mon Sep 17 00:00:00 2001 From: Paul Cioanca Date: Wed, 2 Oct 2024 20:37:59 +0300 Subject: [PATCH 08/44] fix(15.6/upgrades): collision when patching wrappers lib locations for upgrades (#1252) --- .../files/admin_api_scripts/pg_upgrade_scripts/initiate.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh index 31a7cd5b3..235b4f4c1 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh @@ -211,7 +211,7 @@ function patch_wrappers { WRAPPERS_LIB_PATH_DIR=$(dirname "$WRAPPERS_LIB_PATH") if [ "$WRAPPERS_LIB_PATH" != "$WRAPPERS_LIB_PATH_DIR/${OLD_LIB_FILE_NAME}" ]; then echo "Copying $WRAPPERS_LIB_PATH to $WRAPPERS_LIB_PATH_DIR/${OLD_LIB_FILE_NAME}" - cp "$WRAPPERS_LIB_PATH" "$WRAPPERS_LIB_PATH_DIR/${OLD_LIB_FILE_NAME}" + cp "$WRAPPERS_LIB_PATH" "$WRAPPERS_LIB_PATH_DIR/${OLD_LIB_FILE_NAME}" || true fi fi done @@ -225,7 +225,7 @@ function patch_wrappers { LIB_FILE_NAME=$(basename "$OLD_WRAPPER_LIB_PATH") if [ "$WRAPPERS_LIB_PATH" != "$PGLIBNEW/${LIB_FILE_NAME}" ]; then echo "Copying $WRAPPERS_LIB_PATH to $PGLIBNEW/${LIB_FILE_NAME}" - cp "$WRAPPERS_LIB_PATH" "$PGLIBNEW/${LIB_FILE_NAME}" + cp "$WRAPPERS_LIB_PATH" "$PGLIBNEW/${LIB_FILE_NAME}" || true fi fi fi From d264b9dae820baa123c955c7dbd7220f114f3e27 Mon Sep 17 00:00:00 2001 From: Stojan Dimitrovski Date: Thu, 3 Oct 2024 11:57:23 +0200 Subject: [PATCH 09/44] feat: bump auth 2.162.1 on 15.6 (#1256) --- ansible/vars.yml | 4 ++-- common-nix.vars.pkr.hcl | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ansible/vars.yml b/ansible/vars.yml index 3c5ef2172..ed746051d 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -17,8 +17,8 @@ postgrest_release: "12.2.3" postgrest_arm_release_checksum: sha1:fbfd6613d711ce1afa25c42d5df8f1b017f396f9 postgrest_x86_release_checksum: sha1:61c513f91a8931be4062587b9d4a18b42acf5c05 -gotrue_release: 2.162.0 -gotrue_release_checksum: sha1:855b23bd002577290c7d42d7042ac0f5316984b1 +gotrue_release: 2.162.1 +gotrue_release_checksum: sha1:a8b248521f000e027feea2b44e8d9dfb6b054b2e aws_cli_release: "2.2.7" diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index 3a55dd70e..89abf35c5 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.126" +postgres-version = "15.6.1.127" From 5177fa3822b752d19ff840154c012b3eb324d93f Mon Sep 17 00:00:00 2001 From: Paul Cioanca Date: Fri, 4 Oct 2024 20:05:59 +0300 Subject: [PATCH 10/44] fix(15.6): disable pg_stat_monitor (#1260) * fix: disable pg_stat_monitor * chore: bump version --- common-nix.vars.pkr.hcl | 2 +- nix/tests/postgresql.conf.in | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index 89abf35c5..d988536f3 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.127" +postgres-version = "15.6.1.128" diff --git a/nix/tests/postgresql.conf.in b/nix/tests/postgresql.conf.in index d2d667b9a..ef860afcb 100644 --- a/nix/tests/postgresql.conf.in +++ b/nix/tests/postgresql.conf.in @@ -718,7 +718,7 @@ default_text_search_config = 'pg_catalog.english' #local_preload_libraries = '' #session_preload_libraries = '' -shared_preload_libraries = 'pg_stat_statements, pg_stat_monitor, pgaudit, plpgsql, plpgsql_check, pg_cron, pg_net, pgsodium, timescaledb, auto_explain, pg_tle, plan_filter, pg_backtrace' # (change requires restart) +shared_preload_libraries = 'pg_stat_statements, pgaudit, plpgsql, plpgsql_check, pg_cron, pg_net, pgsodium, timescaledb, auto_explain, pg_tle, plan_filter, pg_backtrace' # (change requires restart) jit_provider = 'llvmjit' # JIT library to use From ce2eb0c55030770d56c944ee479274903bcefdfe Mon Sep 17 00:00:00 2001 From: Paul Cioanca Date: Fri, 4 Oct 2024 22:50:38 +0300 Subject: [PATCH 11/44] fix(15.6): disable pg_stat_monitor (#1262) * fix: disable pg_stat_monitor * chore: bump version --- ansible/files/postgresql_config/postgresql.conf.j2 | 2 +- common-nix.vars.pkr.hcl | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/ansible/files/postgresql_config/postgresql.conf.j2 b/ansible/files/postgresql_config/postgresql.conf.j2 index ee16715a6..1604d94f9 100644 --- a/ansible/files/postgresql_config/postgresql.conf.j2 +++ b/ansible/files/postgresql_config/postgresql.conf.j2 @@ -688,7 +688,7 @@ default_text_search_config = 'pg_catalog.english' #local_preload_libraries = '' #session_preload_libraries = '' -shared_preload_libraries = 'pg_stat_statements, pg_stat_monitor, pgaudit, plpgsql, plpgsql_check, pg_cron, pg_net, pgsodium, timescaledb, auto_explain, pg_tle, plan_filter' # (change requires restart) +shared_preload_libraries = 'pg_stat_statements, pgaudit, plpgsql, plpgsql_check, pg_cron, pg_net, pgsodium, timescaledb, auto_explain, pg_tle, plan_filter' # (change requires restart) jit_provider = 'llvmjit' # JIT library to use # - Other Defaults - diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index d988536f3..dd5d407f5 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.128" +postgres-version = "15.6.1.129" From 9062da41f618509f60f5f3eaa005e8ea9b9b6ae0 Mon Sep 17 00:00:00 2001 From: Kang Ming Date: Mon, 7 Oct 2024 00:07:08 -0700 Subject: [PATCH 12/44] feat: bump gotrue to v2.162.2 (#1264) --- ansible/vars.yml | 4 ++-- common-nix.vars.pkr.hcl | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ansible/vars.yml b/ansible/vars.yml index ed746051d..c4c9e52be 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -17,8 +17,8 @@ postgrest_release: "12.2.3" postgrest_arm_release_checksum: sha1:fbfd6613d711ce1afa25c42d5df8f1b017f396f9 postgrest_x86_release_checksum: sha1:61c513f91a8931be4062587b9d4a18b42acf5c05 -gotrue_release: 2.162.1 -gotrue_release_checksum: sha1:a8b248521f000e027feea2b44e8d9dfb6b054b2e +gotrue_release: 2.162.2 +gotrue_release_checksum: sha1:283c30b68b61332a05e6368aa59ece34cd519fe0 aws_cli_release: "2.2.7" diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index dd5d407f5..e5e2f4712 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.129" +postgres-version = "15.6.1.130" From 0c52b1f2b4c8ba671d9247fe610ca94a30348bdb Mon Sep 17 00:00:00 2001 From: Paul Cioanca Date: Tue, 15 Oct 2024 19:45:41 +0300 Subject: [PATCH 13/44] chore: add timescaledb 2.9.1; wrappers upgrade fix; wrappers & plv8 naming fix (#1259) Co-authored-by: Bobbie Soedirgo Co-authored-by: Bobbie Soedirgo <31685197+soedirgo@users.noreply.github.com> Co-authored-by: Kang Ming Co-authored-by: Stojan Dimitrovski Co-authored-by: Sam Rose fix(ci): respect postgresVersion input (#1237) fix: only grant pg_read_all_data if it exists (#1242) fix(15.6): disable pg_stat_monitor (#1260) --- .../pg_upgrade_scripts/initiate.sh | 55 ---------------- flake.nix | 1 + nix/ext/plv8.nix | 16 +++-- nix/ext/timescaledb-2.9.1.nix | 51 +++++++++++++++ nix/ext/wrappers/default.nix | 64 +++++++++++++------ 5 files changed, 108 insertions(+), 79 deletions(-) create mode 100644 nix/ext/timescaledb-2.9.1.nix diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh index 235b4f4c1..ed9a7ed9b 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh @@ -47,7 +47,6 @@ SERVER_ENCODING=$(run_sql -A -t -c "SHOW server_encoding;") POSTGRES_CONFIG_PATH="/etc/postgresql/postgresql.conf" PGBINOLD="/usr/lib/postgresql/bin" -PGLIBOLD="/usr/lib/postgresql/lib" PG_UPGRADE_BIN_DIR="/tmp/pg_upgrade_bin/$PGVERSION" NIX_INSTALLER_PATH="/tmp/persistent/nix-installer" @@ -181,58 +180,6 @@ EOF done } -function patch_wrappers { - local IS_NIX_UPGRADE=$1 - - WRAPPERS_ENABLED=$(run_sql -A -t -c "SELECT EXISTS(SELECT 1 FROM pg_extension WHERE extname = 'wrappers');") - if [ "$WRAPPERS_ENABLED" = "f" ]; then - echo "Wrappers extension not enabled. Skipping." - return - fi - - # This is a workaround for older versions of wrappers which don't have the expected - # naming scheme, containing the version in their library's file name - # e.g. wrappers-0.1.16.so, rather than wrappers.so - # pg_upgrade errors out when it doesn't find an equivalent file in the new PG version's - # library directory, so we're making sure the new version has the expected (old version's) - # file name. - # After the upgrade completes, the new version's library file is used. - # i.e. - # - old version: wrappers-0.1.16.so - # - new version: wrappers-0.1.18.so - # - workaround to make pg_upgrade happy: copy wrappers-0.1.18.so to wrappers-0.1.16.so - if [ "$IS_NIX_UPGRADE" = "true" ]; then - if [ -d "$PGLIBOLD" ]; then - OLD_WRAPPER_LIB_PATH=$(find "$PGLIBOLD" -name "wrappers*so" -print -quit) - OLD_LIB_FILE_NAME=$(basename "$OLD_WRAPPER_LIB_PATH") - - find /nix/store/ -name "wrappers*so" -print0 | while read -r -d $'\0' WRAPPERS_LIB_PATH; do - if [ -f "$WRAPPERS_LIB_PATH" ]; then - WRAPPERS_LIB_PATH_DIR=$(dirname "$WRAPPERS_LIB_PATH") - if [ "$WRAPPERS_LIB_PATH" != "$WRAPPERS_LIB_PATH_DIR/${OLD_LIB_FILE_NAME}" ]; then - echo "Copying $WRAPPERS_LIB_PATH to $WRAPPERS_LIB_PATH_DIR/${OLD_LIB_FILE_NAME}" - cp "$WRAPPERS_LIB_PATH" "$WRAPPERS_LIB_PATH_DIR/${OLD_LIB_FILE_NAME}" || true - fi - fi - done - fi - else - if [ -d "$PGLIBOLD" ]; then - WRAPPERS_LIB_PATH=$(find "$PGLIBNEW" -name "wrappers*so" -print -quit) - if [ -f "$WRAPPERS_LIB_PATH" ]; then - OLD_WRAPPER_LIB_PATH=$(find "$PGLIBOLD" -name "wrappers*so" -print -quit) - if [ -f "$OLD_WRAPPER_LIB_PATH" ]; then - LIB_FILE_NAME=$(basename "$OLD_WRAPPER_LIB_PATH") - if [ "$WRAPPERS_LIB_PATH" != "$PGLIBNEW/${LIB_FILE_NAME}" ]; then - echo "Copying $WRAPPERS_LIB_PATH to $PGLIBNEW/${LIB_FILE_NAME}" - cp "$WRAPPERS_LIB_PATH" "$PGLIBNEW/${LIB_FILE_NAME}" || true - fi - fi - fi - fi - fi -} - function initiate_upgrade { mkdir -p "$MOUNT_POINT" SHARED_PRELOAD_LIBRARIES=$(cat "$POSTGRES_CONFIG_PATH" | grep shared_preload_libraries | sed "s/shared_preload_libraries =\s\{0,1\}'\(.*\)'.*/\1/") @@ -409,8 +356,6 @@ function initiate_upgrade { export LD_LIBRARY_PATH="${PGLIBNEW}" fi - patch_wrappers "$IS_NIX_UPGRADE" - echo "9. Creating new data directory, initializing database" chown -R postgres:postgres "$MOUNT_POINT/" rm -rf "${PGDATANEW:?}/" diff --git a/flake.nix b/flake.nix index 1f9d5f64d..512d32d31 100644 --- a/flake.nix +++ b/flake.nix @@ -107,6 +107,7 @@ ourExtensions = [ ./nix/ext/rum.nix ./nix/ext/timescaledb.nix + ./nix/ext/timescaledb-2.9.1.nix ./nix/ext/pgroonga.nix ./nix/ext/index_advisor.nix ./nix/ext/wal2json.nix diff --git a/nix/ext/plv8.nix b/nix/ext/plv8.nix index 5cb4fb67e..338bba62e 100644 --- a/nix/ext/plv8.nix +++ b/nix/ext/plv8.nix @@ -103,16 +103,20 @@ stdenv.mkDerivation (finalAttrs: { # so changing them does not cause issues. mv "$out/nix/store"/*/* "$out" rmdir "$out/nix/store"/* "$out/nix/store" "$out/nix" - + mv "$out/lib/plv8-${finalAttrs.version}.so" "$out/lib/plv8.so" + ln -s "$out/lib/plv8.so" "$out/lib/plv8-${finalAttrs.version}.so" + sed -i 's|module_pathname = '"'"'$libdir/plv8-[0-9.]*'"'"'|module_pathname = '"'"'$libdir/plv8'"'"'|' "$out/share/postgresql/extension/plv8.control" + sed -i 's|module_pathname = '"'"'$libdir/plv8-[0-9.]*'"'"'|module_pathname = '"'"'$libdir/plv8'"'"'|' "$out/share/postgresql/extension/plcoffee.control" + sed -i 's|module_pathname = '"'"'$libdir/plv8-[0-9.]*'"'"'|module_pathname = '"'"'$libdir/plv8'"'"'|' "$out/share/postgresql/extension/plls.control" ${lib.optionalString stdenv.isDarwin '' - install_name_tool -add_rpath "${v8}/lib" $out/lib/plv8-${finalAttrs.version}.so - install_name_tool -add_rpath "${postgresql}/lib" $out/lib/plv8-${finalAttrs.version}.so - install_name_tool -add_rpath "${stdenv.cc.cc.lib}/lib" $out/lib/plv8-${finalAttrs.version}.so - install_name_tool -change @rpath/libv8_monolith.dylib ${v8}/lib/libv8_monolith.dylib $out/lib/plv8-${finalAttrs.version}.so + install_name_tool -add_rpath "${v8}/lib" $out/lib/plv8.so + install_name_tool -add_rpath "${postgresql}/lib" $out/lib/plv8.so + install_name_tool -add_rpath "${stdenv.cc.cc.lib}/lib" $out/lib/plv8.so + install_name_tool -change @rpath/libv8_monolith.dylib ${v8}/lib/libv8_monolith.dylib $out/lib/plv8.so ''} ${lib.optionalString (!stdenv.isDarwin) '' - ${patchelf}/bin/patchelf --set-rpath "${v8}/lib:${postgresql}/lib:${stdenv.cc.cc.lib}/lib" $out/lib/plv8-${finalAttrs.version}.so + ${patchelf}/bin/patchelf --set-rpath "${v8}/lib:${postgresql}/lib:${stdenv.cc.cc.lib}/lib" $out/lib/plv8.so ''} ''; diff --git a/nix/ext/timescaledb-2.9.1.nix b/nix/ext/timescaledb-2.9.1.nix new file mode 100644 index 000000000..ad955e86f --- /dev/null +++ b/nix/ext/timescaledb-2.9.1.nix @@ -0,0 +1,51 @@ +{ lib, stdenv, fetchFromGitHub, cmake, postgresql, openssl, libkrb5 }: + +stdenv.mkDerivation rec { + pname = "timescaledb-apache"; + version = "2.9.1"; + + nativeBuildInputs = [ cmake ]; + buildInputs = [ postgresql openssl libkrb5 ]; + + src = fetchFromGitHub { + owner = "timescale"; + repo = "timescaledb"; + rev = version; + hash = "sha256-fvVSxDiGZAewyuQ2vZDb0I6tmlDXl6trjZp8+qDBtb8="; + }; + + cmakeFlags = [ "-DSEND_TELEMETRY_DEFAULT=OFF" "-DREGRESS_CHECKS=OFF" "-DTAP_CHECKS=OFF" "-DAPACHE_ONLY=1" ] + ++ lib.optionals stdenv.isDarwin [ "-DLINTER=OFF" ]; + + # Fix the install phase which tries to install into the pgsql extension dir, + # and cannot be manually overridden. This is rather fragile but works OK. + postPatch = '' + for x in CMakeLists.txt sql/CMakeLists.txt; do + substituteInPlace "$x" \ + --replace 'DESTINATION "''${PG_SHAREDIR}/extension"' "DESTINATION \"$out/share/postgresql/extension\"" + done + + for x in src/CMakeLists.txt src/loader/CMakeLists.txt tsl/src/CMakeLists.txt; do + substituteInPlace "$x" \ + --replace 'DESTINATION ''${PG_PKGLIBDIR}' "DESTINATION \"$out/lib\"" + done + ''; + + + # timescaledb-2.9.1.so already exists in the lib directory + # we have no need for the timescaledb.so or control file + postInstall = '' + rm $out/lib/timescaledb.so + rm $out/share/postgresql/extension/timescaledb.control + ''; + + meta = with lib; { + description = "Scales PostgreSQL for time-series data via automatic partitioning across time and space"; + homepage = "https://www.timescale.com/"; + changelog = "https://github.com/timescale/timescaledb/blob/${version}/CHANGELOG.md"; + maintainers = with maintainers; [ samrose ]; + platforms = postgresql.meta.platforms; + license = licenses.asl20; + broken = versionOlder postgresql.version "13"; + }; +} diff --git a/nix/ext/wrappers/default.nix b/nix/ext/wrappers/default.nix index 98d6a7cbc..808946d3a 100644 --- a/nix/ext/wrappers/default.nix +++ b/nix/ext/wrappers/default.nix @@ -63,24 +63,52 @@ buildPgrxExtension_0_11_3 rec { echo '${builtins.concatStringsSep "," previousVersions}' | sed 's/,/\n/g' > git_tags.txt ''; - postInstall = '' - echo "Creating SQL files for previous versions..." - current_version="${version}" - sql_file="$out/share/postgresql/extension/wrappers--$current_version.sql" - - if [ -f "$sql_file" ]; then - while read -r previous_version; do - if [ "$(printf '%s\n' "$previous_version" "$current_version" | sort -V | head -n1)" = "$previous_version" ] && [ "$previous_version" != "$current_version" ]; then - new_file="$out/share/postgresql/extension/wrappers--$previous_version--$current_version.sql" - echo "Creating $new_file" - cp "$sql_file" "$new_file" - fi - done < git_tags.txt - else - echo "Warning: $sql_file not found" - fi - rm git_tags.txt - ''; + postInstall = '' + echo "Modifying main SQL file to use unversioned library name..." + current_version="${version}" + main_sql_file="$out/share/postgresql/extension/wrappers--$current_version.sql" + if [ -f "$main_sql_file" ]; then + sed -i 's|$libdir/wrappers-[0-9.]*|$libdir/wrappers|g' "$main_sql_file" + echo "Modified $main_sql_file" + else + echo "Warning: $main_sql_file not found" + fi + echo "Creating and modifying SQL files for previous versions..." + + if [ -f "$main_sql_file" ]; then + while read -r previous_version; do + if [ "$(printf '%s\n' "$previous_version" "$current_version" | sort -V | head -n1)" = "$previous_version" ] && [ "$previous_version" != "$current_version" ]; then + new_file="$out/share/postgresql/extension/wrappers--$previous_version--$current_version.sql" + echo "Creating $new_file" + cp "$main_sql_file" "$new_file" + sed -i 's|$libdir/wrappers-[0-9.]*|$libdir/wrappers|g' "$new_file" + echo "Modified $new_file" + fi + done < git_tags.txt + else + echo "Warning: $main_sql_file not found" + fi + mv $out/lib/wrappers-${version}.so $out/lib/wrappers.so + + echo "Creating wrappers.so symlinks to support pg_upgrade..." + if [ -f "$out/lib/wrappers.so" ]; then + while read -r previous_version; do + if [ "$(printf '%s\n' "$previous_version" "$current_version" | sort -V | head -n1)" = "$previous_version" ] && [ "$previous_version" != "$current_version" ]; then + new_file="$out/lib/wrappers-$previous_version.so" + echo "Creating $new_file" + ln -s "$out/lib/wrappers.so" "$new_file" + fi + done < git_tags.txt + else + echo "Warning: $out/lib/wrappers.so not found" + fi + + rm git_tags.txt + echo "Contents of updated wrappers.control:" + cat "$out/share/postgresql/extension/wrappers.control" + echo "List of generated SQL files:" + ls -l $out/share/postgresql/extension/wrappers--*.sql + ''; meta = with lib; { description = "Various Foreign Data Wrappers (FDWs) for PostreSQL"; From ba417af1242b6ce179b36b8036fb44f8414d6efd Mon Sep 17 00:00:00 2001 From: Paul Cioanca Date: Tue, 15 Oct 2024 20:13:28 +0300 Subject: [PATCH 14/44] chore(15.6): bump pg version (#1273) --- common-nix.vars.pkr.hcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index e5e2f4712..beeddf2c3 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.130" +postgres-version = "15.6.1.131" From 82152fad96d18a9b170b0752ec097bc98156c691 Mon Sep 17 00:00:00 2001 From: Stojan Dimitrovski Date: Tue, 15 Oct 2024 21:04:09 +0200 Subject: [PATCH 15/44] feat: bump auth to v2.163.0 on 15.6 (#1275) --- ansible/vars.yml | 4 ++-- common-nix.vars.pkr.hcl | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ansible/vars.yml b/ansible/vars.yml index c4c9e52be..29bef7162 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -17,8 +17,8 @@ postgrest_release: "12.2.3" postgrest_arm_release_checksum: sha1:fbfd6613d711ce1afa25c42d5df8f1b017f396f9 postgrest_x86_release_checksum: sha1:61c513f91a8931be4062587b9d4a18b42acf5c05 -gotrue_release: 2.162.2 -gotrue_release_checksum: sha1:283c30b68b61332a05e6368aa59ece34cd519fe0 +gotrue_release: 2.163.0 +gotrue_release_checksum: sha1:e4005515371c41cb04f6dee863ad1bfbcd706286 aws_cli_release: "2.2.7" diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index beeddf2c3..2b2aa48c5 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.131" +postgres-version = "15.6.1.132" From 1912ed8f2c7fad38be6973cdf1269ac100c0bebb Mon Sep 17 00:00:00 2001 From: Paul Cioanca Date: Wed, 16 Oct 2024 10:54:40 +0300 Subject: [PATCH 16/44] fix: restart PG during pre-upgrade steps to shed hanging connections (#1271) --- .../files/admin_api_scripts/pg_upgrade_scripts/common.sh | 9 +++++++++ .../admin_api_scripts/pg_upgrade_scripts/initiate.sh | 8 ++++++++ 2 files changed, 17 insertions(+) diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh index a11f8ff32..cc87fadd0 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh @@ -91,6 +91,15 @@ swap_postgres_and_supabase_admin() { alter database postgres connection limit 0; select pg_terminate_backend(pid) from pg_stat_activity where backend_type = 'client backend' and pid != pg_backend_pid(); EOSQL + + if [ -z "$IS_CI" ]; then + retry 5 systemctl restart postgresql + else + CI_start_postgres "" + fi + + retry 8 pg_isready -h localhost -U supabase_admin + run_sql <<'EOSQL' set statement_timeout = '600s'; begin; diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh index ed9a7ed9b..deb462012 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh @@ -147,6 +147,14 @@ cleanup() { } function handle_extensions { + if [ -z "$IS_CI" ]; then + retry 5 systemctl restart postgresql + else + CI_start_postgres + fi + + retry 8 pg_isready -h localhost -U supabase_admin + rm -f $POST_UPGRADE_EXTENSION_SCRIPT touch $POST_UPGRADE_EXTENSION_SCRIPT From 3cf6055e8e850db799f509b57a22eca629939ab1 Mon Sep 17 00:00:00 2001 From: Paul Cioanca Date: Thu, 17 Oct 2024 15:33:54 +0300 Subject: [PATCH 17/44] fix(upgrades): wrappers 4.2.0 -> wrappers 4.2.0 (#1278) --- nix/ext/wrappers/default.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/nix/ext/wrappers/default.nix b/nix/ext/wrappers/default.nix index 808946d3a..1a4360044 100644 --- a/nix/ext/wrappers/default.nix +++ b/nix/ext/wrappers/default.nix @@ -89,6 +89,7 @@ buildPgrxExtension_0_11_3 rec { echo "Warning: $main_sql_file not found" fi mv $out/lib/wrappers-${version}.so $out/lib/wrappers.so + ln -s $out/lib/wrappers.so $out/lib/wrappers-${version}.so echo "Creating wrappers.so symlinks to support pg_upgrade..." if [ -f "$out/lib/wrappers.so" ]; then From 054dbc16c1e705581925b907d3e2282f8727de19 Mon Sep 17 00:00:00 2001 From: Bobbie Soedirgo Date: Mon, 21 Oct 2024 22:59:37 +0800 Subject: [PATCH 18/44] fix: handle supabase_admin authenticator membership snowflake --- ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh index cc87fadd0..ea7217fcc 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh @@ -106,6 +106,9 @@ begin; create role supabase_tmp superuser; set session authorization supabase_tmp; +-- to handle snowflakes that happened in the past +revoke supabase_admin from authenticator; + do $$ begin if exists (select from pg_extension where extname = 'timescaledb') then From fb024e822a974ec3daf00ba505bb2f60af511202 Mon Sep 17 00:00:00 2001 From: Stojan Dimitrovski Date: Tue, 22 Oct 2024 12:17:40 +0200 Subject: [PATCH 19/44] feat: add auth v2.163.1 to 15.6 (#1283) --- ansible/vars.yml | 4 ++-- common-nix.vars.pkr.hcl | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ansible/vars.yml b/ansible/vars.yml index 29bef7162..a863fc27d 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -17,8 +17,8 @@ postgrest_release: "12.2.3" postgrest_arm_release_checksum: sha1:fbfd6613d711ce1afa25c42d5df8f1b017f396f9 postgrest_x86_release_checksum: sha1:61c513f91a8931be4062587b9d4a18b42acf5c05 -gotrue_release: 2.163.0 -gotrue_release_checksum: sha1:e4005515371c41cb04f6dee863ad1bfbcd706286 +gotrue_release: 2.163.1 +gotrue_release_checksum: sha1:f4f3fee930ac72cadccbbcef1ff076d72e1c31c0 aws_cli_release: "2.2.7" diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index 2b2aa48c5..b86205058 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.132" +postgres-version = "15.6.1.133" From 223952baaaabaaf4a22a2e1ffcf57852b137d7e7 Mon Sep 17 00:00:00 2001 From: Kang Ming Date: Wed, 23 Oct 2024 11:57:06 +0800 Subject: [PATCH 20/44] feat: bump gotrue to v2.163.2 (#1287) --- ansible/vars.yml | 4 ++-- common-nix.vars.pkr.hcl | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ansible/vars.yml b/ansible/vars.yml index a863fc27d..28a499a7f 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -17,8 +17,8 @@ postgrest_release: "12.2.3" postgrest_arm_release_checksum: sha1:fbfd6613d711ce1afa25c42d5df8f1b017f396f9 postgrest_x86_release_checksum: sha1:61c513f91a8931be4062587b9d4a18b42acf5c05 -gotrue_release: 2.163.1 -gotrue_release_checksum: sha1:f4f3fee930ac72cadccbbcef1ff076d72e1c31c0 +gotrue_release: 2.163.2 +gotrue_release_checksum: sha1:31889bc8c498b924c2cb3b6c4084ef6e57ed97c0 aws_cli_release: "2.2.7" diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index b86205058..a6e4461e0 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.133" +postgres-version = "15.6.1.134" From 2d658fc9fe74a386b9427fd0a614820a658e0cf0 Mon Sep 17 00:00:00 2001 From: Oliver Rice Date: Thu, 24 Oct 2024 10:24:33 -0500 Subject: [PATCH 21/44] pg_net 0.11 on 15.6 release branch (#1290) * bump pg_net to 0.11.0 * bump image to 15.6.1.135 --- common-nix.vars.pkr.hcl | 2 +- nix/ext/pg_net.nix | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index a6e4461e0..eafb1e602 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.134" +postgres-version = "15.6.1.135" diff --git a/nix/ext/pg_net.nix b/nix/ext/pg_net.nix index fe4404cdf..dc923b9a8 100644 --- a/nix/ext/pg_net.nix +++ b/nix/ext/pg_net.nix @@ -2,7 +2,7 @@ stdenv.mkDerivation rec { pname = "pg_net"; - version = "0.10.0"; + version = "0.11.0"; buildInputs = [ curl postgresql ]; @@ -10,7 +10,7 @@ stdenv.mkDerivation rec { owner = "supabase"; repo = pname; rev = "refs/tags/v${version}"; - hash = "sha256-R9Mzw5gvV7b2R59LTOzuOc0AI99+3ncFNzijI4mySUg="; + hash = "sha256-XN441jXK1q+I/LZRNwvzbSsebXHgZ8iYsslZvcPFlAs="; }; env.NIX_CFLAGS_COMPILE = "-Wno-error"; @@ -18,7 +18,7 @@ stdenv.mkDerivation rec { installPhase = '' mkdir -p $out/{lib,share/postgresql/extension} - cp *.so $out/lib + cp *${postgresql.dlSuffix} $out/lib cp sql/*.sql $out/share/postgresql/extension cp *.control $out/share/postgresql/extension ''; From a5e5919b701c6e28b272f956c022a24719874c37 Mon Sep 17 00:00:00 2001 From: Paul Cioanca Date: Fri, 25 Oct 2024 11:14:45 +0300 Subject: [PATCH 22/44] fix: don't copy custom extension scripts during pg_upgrade (#1291) --- ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh | 1 + 1 file changed, 1 insertion(+) diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh index deb462012..fd1466c07 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh @@ -426,6 +426,7 @@ EOF cp -R /etc/postgresql-custom/* "$MOUNT_POINT/conf/" # removing supautils config as to allow the latest one provided by the latest image to be used rm -f "$MOUNT_POINT/conf/supautils.conf" || true + rm -f "$MOUNT_POINT/conf/extension-custom-scripts" || true # removing wal-g config as to allow it to be explicitly enabled on the new instance rm -f "$MOUNT_POINT/conf/wal-g.conf" From 1211077fa28f668f70908eadc6b57787bad150c2 Mon Sep 17 00:00:00 2001 From: Paul Cioanca Date: Fri, 25 Oct 2024 11:20:23 +0300 Subject: [PATCH 23/44] fix: add recursive flag to custom extension script directory delete (#1292) --- ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh index fd1466c07..a301bcdbd 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh @@ -426,7 +426,7 @@ EOF cp -R /etc/postgresql-custom/* "$MOUNT_POINT/conf/" # removing supautils config as to allow the latest one provided by the latest image to be used rm -f "$MOUNT_POINT/conf/supautils.conf" || true - rm -f "$MOUNT_POINT/conf/extension-custom-scripts" || true + rm -rf "$MOUNT_POINT/conf/extension-custom-scripts" || true # removing wal-g config as to allow it to be explicitly enabled on the new instance rm -f "$MOUNT_POINT/conf/wal-g.conf" From 407d41aeab2d6d7bc3241804c8b13497173f0228 Mon Sep 17 00:00:00 2001 From: Paul Cioanca Date: Wed, 30 Oct 2024 15:35:59 +0200 Subject: [PATCH 24/44] [GEN-11027] chore: reserve a fixed amount of blocks for the data volume; remove ansible pkg + ppa (#1295) --- common-nix.vars.pkr.hcl | 2 +- ebssurrogate/scripts/surrogate-bootstrap-nix.sh | 5 +++++ scripts/90-cleanup.sh | 5 ++++- scripts/nix-provision.sh | 6 +++++- 4 files changed, 15 insertions(+), 3 deletions(-) diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index eafb1e602..adcb6c8f3 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.135" +postgres-version = "15.6.1.136" diff --git a/ebssurrogate/scripts/surrogate-bootstrap-nix.sh b/ebssurrogate/scripts/surrogate-bootstrap-nix.sh index 45cbc8ac6..3b0bb93c8 100755 --- a/ebssurrogate/scripts/surrogate-bootstrap-nix.sh +++ b/ebssurrogate/scripts/surrogate-bootstrap-nix.sh @@ -110,6 +110,11 @@ function format_and_mount_rootfs { fi mkfs.ext4 /dev/xvdh + + # Explicitly reserving 100MiB worth of blocks for the data volume + RESERVED_DATA_VOLUME_BLOCK_COUNT=$((100 * 1024 * 1024 / 4096)) + tune2fs -r $RESERVED_DATA_VOLUME_BLOCK_COUNT /dev/xvdh + mkdir -p /mnt/data mount -o defaults,discard /dev/xvdh /mnt/data } diff --git a/scripts/90-cleanup.sh b/scripts/90-cleanup.sh index ebbf49080..f2e19686d 100644 --- a/scripts/90-cleanup.sh +++ b/scripts/90-cleanup.sh @@ -36,7 +36,10 @@ elif [ -n "$(command -v apt-get)" ]; then libicu-dev \ libcgal-dev \ libgcc-9-dev \ - libgcc-8-dev + libgcc-8-dev \ + ansible + + add-apt-repository --yes --remove ppa:ansible/ansible source /etc/os-release apt-get -y remove --purge linux-headers-5.11.0-1021-aws diff --git a/scripts/nix-provision.sh b/scripts/nix-provision.sh index 223e84926..1c7acbd58 100644 --- a/scripts/nix-provision.sh +++ b/scripts/nix-provision.sh @@ -38,8 +38,12 @@ EOF $ARGS } +function cleanup_packages { + apt-get -y remove --purge ansible + add-apt-repository --yes --remove ppa:ansible/ansible +} install_packages install_nix execute_stage2_playbook - +cleanup_packages From 19ac49312771398e41256a0b83def82daa659091 Mon Sep 17 00:00:00 2001 From: Bobbie Soedirgo Date: Thu, 31 Oct 2024 13:49:30 +0700 Subject: [PATCH 25/44] fix: grant predefined roles post-upgrade --- .../admin_api_scripts/pg_upgrade_scripts/common.sh | 10 ---------- .../admin_api_scripts/pg_upgrade_scripts/complete.sh | 3 +++ 2 files changed, 3 insertions(+), 10 deletions(-) diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh index ea7217fcc..1b46d1712 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh @@ -541,16 +541,6 @@ $$; alter database postgres connection limit -1; --- #incident-2024-09-12-project-upgrades-are-temporarily-disabled -do $$ -begin - if exists (select from pg_authid where rolname = 'pg_read_all_data') then - execute('grant pg_read_all_data to postgres'); - end if; -end -$$; -grant pg_signal_backend to postgres; - set session authorization supabase_admin; drop role supabase_tmp; commit; diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/complete.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/complete.sh index d19b72422..c8542c685 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/complete.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/complete.sh @@ -78,6 +78,9 @@ EOF run_sql -c "$RECREATE_PG_CRON_QUERY" fi + + # #incident-2024-09-12-project-upgrades-are-temporarily-disabled + run_sql -c "grant pg_read_all_data, pg_signal_backend to postgres" } function complete_pg_upgrade { From 21d3823ec4641a5156842d59389bfc3e0511968e Mon Sep 17 00:00:00 2001 From: Bobbie Soedirgo <31685197+soedirgo@users.noreply.github.com> Date: Thu, 31 Oct 2024 14:15:36 +0700 Subject: [PATCH 26/44] fix: add more roles to reserved_roles & reserved_memberships (#1303) * fix: add more roles to reserved_roles & reserved_memberships * Update common-nix.vars.pkr.hcl --- ansible/files/postgresql_config/supautils.conf.j2 | 4 ++-- common-nix.vars.pkr.hcl | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ansible/files/postgresql_config/supautils.conf.j2 b/ansible/files/postgresql_config/supautils.conf.j2 index 2ab4257fd..b2a259693 100644 --- a/ansible/files/postgresql_config/supautils.conf.j2 +++ b/ansible/files/postgresql_config/supautils.conf.j2 @@ -8,5 +8,5 @@ supautils.privileged_extensions_custom_scripts_path = '/etc/postgresql-custom/ex supautils.privileged_extensions_superuser = 'supabase_admin' supautils.privileged_role = 'postgres' supautils.privileged_role_allowed_configs = 'auto_explain.log_min_duration, auto_explain.log_nested_statements, log_lock_waits, log_min_messages, pg_net.batch_size, pg_net.ttl, pgaudit.log, pgaudit.log_catalog, pgaudit.log_client, pgaudit.log_level, pgaudit.log_relation, pgaudit.log_rows, pgaudit.log_statement, pgaudit.log_statement_once, pgaudit.role, pgrst.*, plan_filter.*, safeupdate.enabled, session_replication_role, track_io_timing' -supautils.reserved_memberships = 'pg_read_server_files, pg_write_server_files, pg_execute_server_program, authenticator' -supautils.reserved_roles = 'supabase_admin, supabase_auth_admin, supabase_storage_admin, supabase_read_only_user, supabase_replication_admin, dashboard_user, pgbouncer, service_role*, authenticator*, authenticated*, anon*' +supautils.reserved_memberships = 'pg_read_server_files, pg_write_server_files, pg_execute_server_program, supabase_admin, supabase_auth_admin, supabase_storage_admin, supabase_read_only_user, supabase_realtime_admin, supabase_replication_admin, dashboard_user, pgbouncer, authenticator' +supautils.reserved_roles = 'supabase_admin, supabase_auth_admin, supabase_storage_admin, supabase_read_only_user, supabase_realtime_admin, supabase_replication_admin, dashboard_user, pgbouncer, service_role*, authenticator*, authenticated*, anon*' diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index adcb6c8f3..3b06b818f 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.136" +postgres-version = "15.6.1.137" From 1273c413e3f977df54c2b67bf3b8a49ceecca7c9 Mon Sep 17 00:00:00 2001 From: Stojan Dimitrovski Date: Fri, 1 Nov 2024 13:14:29 +0100 Subject: [PATCH 27/44] feat: update envoy lds config with auth jwks, oidc URLs, strip `sb-opk` header (#1296) --- ansible/files/envoy_config/lds.yaml | 23 ++++++++++++++++++++++- common-nix.vars.pkr.hcl | 2 +- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/ansible/files/envoy_config/lds.yaml b/ansible/files/envoy_config/lds.yaml index 60a38f65d..2fc7cae13 100644 --- a/ansible/files/envoy_config/lds.yaml +++ b/ansible/files/envoy_config/lds.yaml @@ -254,8 +254,13 @@ resources: type.googleapis.com/envoy.extensions.filters.http.rbac.v3.RBACPerRoute - match: safe_regex: + google_re2: + max_program_size: 150 regex: >- - /auth/v1/(verify|callback|authorize|sso/saml/(acs|metadata|slo)) + /auth/v1/(verify|callback|authorize|sso/saml/(acs|metadata|slo)|\.well-known/(openid-configuration|jwks\.json)) + request_headers_to_remove: + - apikey + - sb-opk route: cluster: gotrue regex_rewrite: @@ -269,6 +274,9 @@ resources: typed_per_filter_config: *ref_0 - match: prefix: /auth/v1/ + request_headers_to_remove: + - apikey + - sb-opk route: cluster: gotrue prefix_rewrite: / @@ -280,6 +288,7 @@ resources: present_match: true request_headers_to_remove: - apikey + - sb-opk route: cluster: postgrest prefix_rewrite: / @@ -293,6 +302,7 @@ resources: prefix: /rest/v1/ request_headers_to_remove: - apikey + - sb-opk route: cluster: postgrest prefix_rewrite: / @@ -309,6 +319,7 @@ resources: present_match: true request_headers_to_remove: - apikey + - sb-opk route: cluster: postgrest_admin prefix_rewrite: / @@ -321,6 +332,7 @@ resources: prefix: /rest-admin/v1/ request_headers_to_remove: - apikey + - sb-opk route: cluster: postgrest_admin prefix_rewrite: / @@ -330,18 +342,25 @@ resources: header: key: Content-Profile value: graphql_public + request_headers_to_remove: + - apikey + - sb-opk route: cluster: postgrest prefix_rewrite: /rpc/graphql timeout: 125s - match: prefix: /admin/v1/ + request_headers_to_remove: + - sb-opk route: cluster: admin_api prefix_rewrite: / timeout: 600s - match: prefix: /customer/v1/privileged/ + request_headers_to_remove: + - sb-opk route: cluster: admin_api prefix_rewrite: /privileged/ @@ -365,6 +384,8 @@ resources: treat_missing_header_as_empty: true - match: prefix: /metrics/aggregated + request_headers_to_remove: + - sb-opk route: cluster: admin_api prefix_rewrite: /supabase-internal/metrics diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index 3b06b818f..c4285caa1 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.137" +postgres-version = "15.6.1.138" From 936713c36ae34b1bf3370f04dc423ee1b8820863 Mon Sep 17 00:00:00 2001 From: angelico Date: Mon, 4 Nov 2024 17:07:32 +0800 Subject: [PATCH 28/44] chore: udpate package repo for salt (#1307) --- ansible/tasks/internal/install-salt.yml | 34 +++++++++++++++++++------ 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/ansible/tasks/internal/install-salt.yml b/ansible/tasks/internal/install-salt.yml index 281e370bb..73cd6ee85 100644 --- a/ansible/tasks/internal/install-salt.yml +++ b/ansible/tasks/internal/install-salt.yml @@ -1,29 +1,47 @@ - name: Add apt repository for Saltstack (arm) block: + - name: Ensure /etc/apt/keyrings directory exists + file: + path: /etc/apt/keyrings + state: directory + mode: '0755' + - name: salt gpg key - ansible.builtin.apt_key: - url: https://repo.saltproject.io/salt/py3/ubuntu/20.04/arm64/SALT-PROJECT-GPG-PUBKEY-2023.gpg - keyring: /etc/apt/trusted.gpg.d/salt-archive-keyring-2023-arm.gpg + ansible.builtin.get_url: + url: https://packages.broadcom.com/artifactory/api/security/keypair/SaltProjectKey/public + dest: /etc/apt/keyrings/salt-archive-keyring-2023.pgp + mode: '0644' - name: salt apt repo ansible.builtin.apt_repository: - repo: deb [signed-by=/etc/apt/trusted.gpg.d/salt-archive-keyring-2023-arm.gpg arch=arm64] https://repo.saltproject.io/salt/py3/ubuntu/20.04/arm64/{{ salt_minion_version }} focal main + repo: "deb [signed-by=/etc/apt/keyrings/salt-archive-keyring-2023.pgp arch=arm64] https://packages.broadcom.com/artifactory/saltproject-deb/ stable main" + filename: 'salt.list' state: present when: platform == "arm64" - name: Add apt repository for Saltstack (amd) block: + - name: Ensure /etc/apt/keyrings directory exists + file: + path: /etc/apt/keyrings + state: directory + mode: '0755' + - name: salt gpg key - ansible.builtin.apt_key: - url: https://repo.saltproject.io/salt/py3/ubuntu/20.04/amd64/SALT-PROJECT-GPG-PUBKEY-2023.gpg - keyring: /etc/apt/trusted.gpg.d/salt-archive-keyring-2023-amd.gpg + ansible.builtin.get_url: + url: https://packages.broadcom.com/artifactory/api/security/keypair/SaltProjectKey/public + dest: /etc/apt/keyrings/salt-archive-keyring-2023.pgp + mode: '0644' - name: salt apt repo ansible.builtin.apt_repository: - repo: deb [signed-by=/etc/apt/trusted.gpg.d/salt-archive-keyring-2023-amd.gpg arch=amd64] https://repo.saltproject.io/salt/py3/ubuntu/20.04/amd64/{{ salt_minon_version }} focal main + repo: "deb [signed-by=/etc/apt/keyrings/salt-archive-keyring-2023.pgp arch=amd64] https://packages.broadcom.com/artifactory/saltproject-deb/ stable main" + filename: 'salt.list' state: present when: platform == "amd64" - name: Salt minion install apt: name: salt-minion + state: present + update_cache: yes From 535042092c656c11376c91e58806d9b60f67b27b Mon Sep 17 00:00:00 2001 From: Bobbie Soedirgo Date: Wed, 6 Nov 2024 15:17:57 +0800 Subject: [PATCH 29/44] fix: use sudo for apt-get commands --- scripts/nix-provision.sh | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/scripts/nix-provision.sh b/scripts/nix-provision.sh index 1c7acbd58..5ed678d7a 100644 --- a/scripts/nix-provision.sh +++ b/scripts/nix-provision.sh @@ -39,8 +39,8 @@ EOF } function cleanup_packages { - apt-get -y remove --purge ansible - add-apt-repository --yes --remove ppa:ansible/ansible + sudo apt-get -y remove --purge ansible + sudo add-apt-repository --yes --remove ppa:ansible/ansible } install_packages From 9e285dd9843ca32592ed23a035489d5324f8768e Mon Sep 17 00:00:00 2001 From: Oliver Rice Date: Fri, 8 Nov 2024 06:11:52 -0600 Subject: [PATCH 30/44] Update pg net to 0.13.0 on pg 15.6 branch (#1315) * upgrade pg_net to 0.13.0 on 15.6 * bump postgres-version --- common-nix.vars.pkr.hcl | 2 +- nix/ext/pg_net.nix | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index c4285caa1..797387418 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.138" +postgres-version = "15.6.1.139" diff --git a/nix/ext/pg_net.nix b/nix/ext/pg_net.nix index dc923b9a8..3bf04ff94 100644 --- a/nix/ext/pg_net.nix +++ b/nix/ext/pg_net.nix @@ -2,7 +2,7 @@ stdenv.mkDerivation rec { pname = "pg_net"; - version = "0.11.0"; + version = "0.13.0"; buildInputs = [ curl postgresql ]; @@ -10,7 +10,7 @@ stdenv.mkDerivation rec { owner = "supabase"; repo = pname; rev = "refs/tags/v${version}"; - hash = "sha256-XN441jXK1q+I/LZRNwvzbSsebXHgZ8iYsslZvcPFlAs="; + hash = "sha256-FRaTZPCJQPYAFmsJg22hYJJ0+gH1tMdDQoCQgiqEnaA="; }; env.NIX_CFLAGS_COMPILE = "-Wno-error"; From a627e176c1e8816bc73f5e49ccb7042e5647fbca Mon Sep 17 00:00:00 2001 From: Angelico Date: Wed, 6 Nov 2024 15:18:41 +0800 Subject: [PATCH 31/44] chore: cleanup pgbouncer.get_auth ownership accordingly --- .../pg_upgrade_scripts/initiate.sh | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh index a301bcdbd..e28eda38e 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh @@ -39,6 +39,7 @@ MOUNT_POINT="/data_migration" LOG_FILE="/var/log/pg-upgrade-initiate.log" POST_UPGRADE_EXTENSION_SCRIPT="/tmp/pg_upgrade/pg_upgrade_extensions.sql" +POST_UPGRADE_PGBOUNCER_CLEANUP_SCRIPT="/tmp/pg_upgrade/pg_upgrade_pgbouncer_cleanup.sql" OLD_PGVERSION=$(run_sql -A -t -c "SHOW server_version;") SERVER_LC_COLLATE=$(run_sql -A -t -c "SHOW lc_collate;") @@ -132,6 +133,22 @@ cleanup() { echo "Resetting postgres database connection limit" retry 5 run_sql -c "ALTER DATABASE postgres CONNECTION LIMIT -1;" + echo "Making sure postgres still has access to pg_shadow" + cat << EOF >> $POST_UPGRADE_PGBOUNCER_CLEANUP_SCRIPT +DO \$\$ +BEGIN + IF EXISTS (SELECT 1 FROM pg_views WHERE viewname = 'pg_shadow' AND viewowner = 'supabase_admin') THEN + ALTER function pgbouncer.get_auth owner to supabase_admin; + GRANT EXECUTE ON FUNCTION pgbouncer.get_auth(p_usename TEXT) TO postgres; + END IF; +END; +\$\$; +EOF + + if [ -f $POST_UPGRADE_PGBOUNCER_CLEANUP_SCRIPT ]; then + retry 5 run_sql -f $POST_UPGRADE_PGBOUNCER_CLEANUP_SCRIPT + fi + if [ -z "$IS_CI" ] && [ -z "$IS_LOCAL_UPGRADE" ]; then echo "Unmounting data disk from ${MOUNT_POINT}" retry 3 umount $MOUNT_POINT From 4b0e66101ae45032803894f3ab5c90ca6093f7b9 Mon Sep 17 00:00:00 2001 From: Angelico Date: Wed, 6 Nov 2024 15:48:34 +0800 Subject: [PATCH 32/44] chore: standardize --- .../pg_upgrade_scripts/initiate.sh | 20 +++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh index e28eda38e..88518766b 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/initiate.sh @@ -39,7 +39,7 @@ MOUNT_POINT="/data_migration" LOG_FILE="/var/log/pg-upgrade-initiate.log" POST_UPGRADE_EXTENSION_SCRIPT="/tmp/pg_upgrade/pg_upgrade_extensions.sql" -POST_UPGRADE_PGBOUNCER_CLEANUP_SCRIPT="/tmp/pg_upgrade/pg_upgrade_pgbouncer_cleanup.sql" +POST_UPGRADE_POSTGRES_PERMS_SCRIPT="/tmp/pg_upgrade/pg_upgrade_postgres_perms.sql" OLD_PGVERSION=$(run_sql -A -t -c "SHOW server_version;") SERVER_LC_COLLATE=$(run_sql -A -t -c "SHOW lc_collate;") @@ -134,19 +134,19 @@ cleanup() { retry 5 run_sql -c "ALTER DATABASE postgres CONNECTION LIMIT -1;" echo "Making sure postgres still has access to pg_shadow" - cat << EOF >> $POST_UPGRADE_PGBOUNCER_CLEANUP_SCRIPT + cat << EOF >> $POST_UPGRADE_POSTGRES_PERMS_SCRIPT DO \$\$ -BEGIN - IF EXISTS (SELECT 1 FROM pg_views WHERE viewname = 'pg_shadow' AND viewowner = 'supabase_admin') THEN - ALTER function pgbouncer.get_auth owner to supabase_admin; - GRANT EXECUTE ON FUNCTION pgbouncer.get_auth(p_usename TEXT) TO postgres; - END IF; -END; +begin + if exists (select from pg_authid where rolname = 'pg_read_all_data') then + execute('grant pg_read_all_data to postgres'); + end if; +end \$\$; +grant pg_signal_backend to postgres; EOF - if [ -f $POST_UPGRADE_PGBOUNCER_CLEANUP_SCRIPT ]; then - retry 5 run_sql -f $POST_UPGRADE_PGBOUNCER_CLEANUP_SCRIPT + if [ -f $POST_UPGRADE_POSTGRES_PERMS_SCRIPT ]; then + retry 5 run_sql -f $POST_UPGRADE_POSTGRES_PERMS_SCRIPT fi if [ -z "$IS_CI" ] && [ -z "$IS_LOCAL_UPGRADE" ]; then From 7065738d65d50f327ad77df15474cba0754a9527 Mon Sep 17 00:00:00 2001 From: Pavel Borisov Date: Mon, 11 Nov 2024 17:38:14 +0400 Subject: [PATCH 33/44] Upgrade pgvector to 0.8.0 --- ansible/vars.yml | 4 ++-- nix/ext/pgvector.nix | 4 ++-- nix/tests/expected/extensions_sql_interface.out | 6 +++++- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/ansible/vars.yml b/ansible/vars.yml index 28a499a7f..c6f684a0a 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -141,8 +141,8 @@ hypopg_release_checksum: sha256:9afe6357fd389d8d33fad81703038ce520b09275ec00153c pg_repack_release: "1.5.0" pg_repack_release_checksum: sha256:9a14d6a95bfa29f856aa10538238622c1f351d38eb350b196c06720a878ccc52 -pgvector_release: "0.7.4" -pgvector_release_checksum: sha256:0341edf89b1924ae0d552f617e14fb7f8867c0194ed775bcc44fa40288642583 +pgvector_release: "0.8.0" +pgvector_release_checksum: sha256:867a2c328d4928a5a9d6f052cd3bc78c7d60228a9b914ad32aa3db88e9de27b0 pg_tle_release: "1.3.2" pg_tle_release_checksum: sha256:d04f72d88b21b954656609743560684ac42645b64a36c800d4d2f84d1f180de1 diff --git a/nix/ext/pgvector.nix b/nix/ext/pgvector.nix index 476f7606c..965be3656 100644 --- a/nix/ext/pgvector.nix +++ b/nix/ext/pgvector.nix @@ -2,7 +2,7 @@ stdenv.mkDerivation rec { pname = "pgvector"; - version = "0.7.4"; + version = "0.8.0"; buildInputs = [ postgresql ]; @@ -10,7 +10,7 @@ stdenv.mkDerivation rec { owner = "pgvector"; repo = pname; rev = "refs/tags/v${version}"; - hash = "sha256-qwPaguQUdDHV8q6GDneLq5MuhVroPizpbqt7f08gKJI="; + hash = "sha256-JsZV+I4eRMypXTjGmjCtMBXDVpqTIPHQa28ogXncE/Q="; }; installPhase = '' diff --git a/nix/tests/expected/extensions_sql_interface.out b/nix/tests/expected/extensions_sql_interface.out index ab43f54e6..96df9add5 100644 --- a/nix/tests/expected/extensions_sql_interface.out +++ b/nix/tests/expected/extensions_sql_interface.out @@ -4994,6 +4994,10 @@ order by vector | public | array_to_halfvec | integer[], integer, boolean | halfvec vector | public | array_to_halfvec | numeric[], integer, boolean | halfvec vector | public | array_to_halfvec | real[], integer, boolean | halfvec + vector | public | array_to_sparsevec | double precision[], integer, boolean | sparsevec + vector | public | array_to_sparsevec | integer[], integer, boolean | sparsevec + vector | public | array_to_sparsevec | numeric[], integer, boolean | sparsevec + vector | public | array_to_sparsevec | real[], integer, boolean | sparsevec vector | public | array_to_vector | double precision[], integer, boolean | vector vector | public | array_to_vector | integer[], integer, boolean | vector vector | public | array_to_vector | numeric[], integer, boolean | vector @@ -5156,7 +5160,7 @@ order by xml2 | public | xpath_table | text, text, text, text, text | SETOF record xml2 | public | xslt_process | text, text | text xml2 | public | xslt_process | text, text, text | text -(5005 rows) +(5009 rows) /* From f34519d9cf29afc6aa3e17b5b1788aad90659421 Mon Sep 17 00:00:00 2001 From: Pavel Borisov <63344111+pashkinelfe@users.noreply.github.com> Date: Tue, 19 Nov 2024 00:00:20 +0400 Subject: [PATCH 34/44] Update common-nix.vars.pkr.hcl Bump version to 15.6.1.140 --- common-nix.vars.pkr.hcl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index 797387418..3049b89e2 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.139" +postgres-version = "15.6.1.140" From 42c6d7ed678c608cb901ae67fe7c43b6af49159c Mon Sep 17 00:00:00 2001 From: Joel Lee Date: Thu, 21 Nov 2024 13:05:13 +0800 Subject: [PATCH 35/44] feat: upgrade to auth v2.164.0 (#1329) * fix: update auth to v2.164.0 * fix: upgrade version * fix: bump nix instead --- ansible/vars.yml | 4 ++-- common-nix.vars.pkr.hcl | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ansible/vars.yml b/ansible/vars.yml index c6f684a0a..ab8d460cb 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -17,8 +17,8 @@ postgrest_release: "12.2.3" postgrest_arm_release_checksum: sha1:fbfd6613d711ce1afa25c42d5df8f1b017f396f9 postgrest_x86_release_checksum: sha1:61c513f91a8931be4062587b9d4a18b42acf5c05 -gotrue_release: 2.163.2 -gotrue_release_checksum: sha1:31889bc8c498b924c2cb3b6c4084ef6e57ed97c0 +gotrue_release: 2.164.0 +gotrue_release_checksum: sha1:7305f82b02fb172d8f7272c0966c5d9eab41c5bd aws_cli_release: "2.2.7" diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index 3049b89e2..d79117c67 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.140" +postgres-version = "15.6.1.141" From bf452e49d1c8452c8f862068347771d148d152ce Mon Sep 17 00:00:00 2001 From: Bobbie Soedirgo <31685197+soedirgo@users.noreply.github.com> Date: Thu, 21 Nov 2024 20:50:53 +0800 Subject: [PATCH 36/44] fix: replace `alter routine` with `alter function|procedure` (#1333) --- ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh index 1b46d1712..871015ef4 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh @@ -456,7 +456,10 @@ begin foreach obj in array functions loop if obj->>'owner' = 'postgres' then - execute(format('alter routine %s(%s) owner to postgres;', (obj->>'oid')::regproc, pg_get_function_identity_arguments((obj->>'oid')::regproc))); + execute(format('alter %s %s(%s) owner to postgres;' + , case when obj->>'kind' = 'p' then 'procedure' else 'function' + , (obj->>'oid')::regproc + , pg_get_function_identity_arguments((obj->>'oid')::regproc))); end if; for rec in select grantor, grantee, privilege_type, is_grantable From 3bc17104be989d96da8f920d55b8b11ee3067184 Mon Sep 17 00:00:00 2001 From: Paul Cioanca Date: Thu, 21 Nov 2024 15:25:32 +0200 Subject: [PATCH 37/44] fix: pg_upgrade_script (#1336) --- ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh b/ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh index 871015ef4..4d465036d 100755 --- a/ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh +++ b/ansible/files/admin_api_scripts/pg_upgrade_scripts/common.sh @@ -457,7 +457,7 @@ begin loop if obj->>'owner' = 'postgres' then execute(format('alter %s %s(%s) owner to postgres;' - , case when obj->>'kind' = 'p' then 'procedure' else 'function' + , case when obj->>'kind' = 'p' then 'procedure' else 'function' end , (obj->>'oid')::regproc , pg_get_function_identity_arguments((obj->>'oid')::regproc))); end if; From d4bcb35da9c5e876d6b793caad36d9294be332cf Mon Sep 17 00:00:00 2001 From: Bobbie Soedirgo Date: Thu, 21 Nov 2024 18:05:00 +0800 Subject: [PATCH 38/44] feat: supautils v2.5.0 --- Dockerfile | 2 +- ansible/files/postgresql_config/supautils.conf.j2 | 1 + ansible/vars.yml | 8 ++++---- common-nix.vars.pkr.hcl | 2 +- docker/orioledb/Dockerfile | 2 +- nix/ext/supautils.nix | 6 +++--- 6 files changed, 11 insertions(+), 10 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9e98a73ee..8309c2585 100644 --- a/Dockerfile +++ b/Dockerfile @@ -36,7 +36,7 @@ ARG hypopg_release=1.3.1 ARG pgvector_release=0.4.0 ARG pg_tle_release=1.3.2 ARG index_advisor_release=0.2.0 -ARG supautils_release=2.2.1 +ARG supautils_release=2.5.0 ARG wal_g_release=2.0.1 #################### diff --git a/ansible/files/postgresql_config/supautils.conf.j2 b/ansible/files/postgresql_config/supautils.conf.j2 index b2a259693..697b843d7 100644 --- a/ansible/files/postgresql_config/supautils.conf.j2 +++ b/ansible/files/postgresql_config/supautils.conf.j2 @@ -1,5 +1,6 @@ supautils.extensions_parameter_overrides = '{"pg_cron":{"schema":"pg_catalog"}}' supautils.policy_grants = '{"postgres":["auth.audit_log_entries","auth.identities","auth.refresh_tokens","auth.sessions","auth.users","realtime.messages","storage.buckets","storage.migrations","storage.objects","storage.s3_multipart_uploads","storage.s3_multipart_uploads_parts"]}' +supautils.drop_trigger_grants = '{"postgres":["auth.audit_log_entries","auth.identities","auth.refresh_tokens","auth.sessions","auth.users","realtime.messages","storage.buckets","storage.migrations","storage.objects","storage.s3_multipart_uploads","storage.s3_multipart_uploads_parts"]}' # full list: address_standardizer, address_standardizer_data_us, adminpack, amcheck, autoinc, bloom, btree_gin, btree_gist, citext, cube, dblink, dict_int, dict_xsyn, earthdistance, file_fdw, fuzzystrmatch, hstore, http, hypopg, index_advisor, insert_username, intagg, intarray, isn, lo, ltree, moddatetime, old_snapshot, orioledb, pageinspect, pg_buffercache, pg_cron, pg_freespacemap, pg_graphql, pg_hashids, pg_jsonschema, pg_net, pg_prewarm, pg_repack, pg_stat_monitor, pg_stat_statements, pg_surgery, pg_tle, pg_trgm, pg_visibility, pg_walinspect, pgaudit, pgcrypto, pgjwt, pgroonga, pgroonga_database, pgrouting, pgrowlocks, pgsodium, pgstattuple, pgtap, plcoffee, pljava, plls, plpgsql, plpgsql_check, plv8, postgis, postgis_raster, postgis_sfcgal, postgis_tiger_geocoder, postgis_topology, postgres_fdw, refint, rum, seg, sslinfo, supabase_vault, supautils, tablefunc, tcn, timescaledb, tsm_system_rows, tsm_system_time, unaccent, uuid-ossp, vector, wrappers, xml2 # omitted because may be unsafe: adminpack, amcheck, file_fdw, lo, old_snapshot, pageinspect, pg_buffercache, pg_freespacemap, pg_surgery, pg_visibility # omitted because deprecated: intagg, xml2 diff --git a/ansible/vars.yml b/ansible/vars.yml index ab8d460cb..aa5ea9773 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -86,10 +86,10 @@ timescaledb_release_checksum: sha256:883638f2e79d25ec88ee58f603f3c81c999b6364cb4 wal2json_release: "2_5" wal2json_release_checksum: sha256:b516653575541cf221b99cf3f8be9b6821f6dbcfc125675c85f35090f824f00e -supautils_release: "2.2.1" -supautils_release_arm64_deb_checksum: sha256:1a2d2b8fe604d38921ed9cf3a0d56dd142a274035d0dca17ad21cdc81ddd9569 -supautils_release_amd64_deb_checksum: sha256:4674fac146976dc179f5871271830718112f5eaa853bc461df9e303c8df2e1ab -supautils_release_tar_checksum: sha256:f1f33371390322ac830645b8b0b8e249cb8ca10b19fdeae917f383014ed01b5d +supautils_release: "2.5.0" +supautils_release_arm64_deb_checksum: sha256:406e4a816f719bd6c4b2143e9bb38078fbe60d7e85018ec0aed5d76924e28000 +supautils_release_amd64_deb_checksum: sha256:71f182b478d8aaf167609dd382875cdce3fbe992e888988b3d51cdad39e08202 +supautils_release_tar_checksum: sha256:07c41244e4374248da9c2df2822152f3ae8f1e74c8a92d361300480193219b63 pljava_release: master pljava_release_checksum: sha256:e99b1c52f7b57f64c8986fe6ea4a6cc09d78e779c1643db060d0ac66c93be8b6 diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index d79117c67..8de62aece 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.141" +postgres-version = "15.6.1.142" diff --git a/docker/orioledb/Dockerfile b/docker/orioledb/Dockerfile index 72bc9ba62..7adb7d74a 100644 --- a/docker/orioledb/Dockerfile +++ b/docker/orioledb/Dockerfile @@ -36,7 +36,7 @@ ARG hypopg_release=1.3.1 ARG pgvector_release=0.4.0 ARG pg_tle_release=1.3.2 ARG index_advisor_release=0.2.0 -ARG supautils_release=2.2.1 +ARG supautils_release=2.5.0 ARG wal_g_release=2.0.1 #################### diff --git a/nix/ext/supautils.nix b/nix/ext/supautils.nix index 9a32062b9..82cf9bb20 100644 --- a/nix/ext/supautils.nix +++ b/nix/ext/supautils.nix @@ -2,7 +2,7 @@ stdenv.mkDerivation rec { pname = "supautils"; - version = "2.2.1"; + version = "2.5.0"; buildInputs = [ postgresql ]; @@ -10,13 +10,13 @@ stdenv.mkDerivation rec { owner = "supabase"; repo = pname; rev = "refs/tags/v${version}"; - hash = "sha256-wSUEG0at00TPAoHv6+NMzuUE8mfW6fnHH0MNxvBdUiE="; + hash = "sha256-NyAk+QpQEdibmFY4yceO/FzMOhRYhKXf4XUw9XJ5rOY="; }; installPhase = '' mkdir -p $out/lib - install -D supautils.so -t $out/lib + install -D supautils${postgresql.dlSuffix} -t $out/lib ''; meta = with lib; { From 63303c6528261649259254b122faae03cb9cfe9a Mon Sep 17 00:00:00 2001 From: Oliver Rice Date: Mon, 25 Nov 2024 09:50:14 -0600 Subject: [PATCH 39/44] pgmq in 15.6 image (#1337) * pick pgmq to 15.6 * update extension interface test output * bump image version --- .../files/postgresql_config/supautils.conf.j2 | 3 +- common-nix.vars.pkr.hcl | 2 +- common.vars.pkr.hcl | 2 +- flake.nix | 1 + nix/ext/pgmq.nix | 33 ++++ .../expected/extensions_sql_interface.out | 41 ++++- nix/tests/expected/pgmq.out | 141 ++++++++++++++++++ nix/tests/prime.sql | 1 + nix/tests/sql/pgmq.sql | 90 +++++++++++ 9 files changed, 307 insertions(+), 7 deletions(-) create mode 100644 nix/ext/pgmq.nix create mode 100644 nix/tests/expected/pgmq.out create mode 100644 nix/tests/sql/pgmq.sql diff --git a/ansible/files/postgresql_config/supautils.conf.j2 b/ansible/files/postgresql_config/supautils.conf.j2 index 697b843d7..5c0a1e214 100644 --- a/ansible/files/postgresql_config/supautils.conf.j2 +++ b/ansible/files/postgresql_config/supautils.conf.j2 @@ -1,9 +1,10 @@ supautils.extensions_parameter_overrides = '{"pg_cron":{"schema":"pg_catalog"}}' supautils.policy_grants = '{"postgres":["auth.audit_log_entries","auth.identities","auth.refresh_tokens","auth.sessions","auth.users","realtime.messages","storage.buckets","storage.migrations","storage.objects","storage.s3_multipart_uploads","storage.s3_multipart_uploads_parts"]}' supautils.drop_trigger_grants = '{"postgres":["auth.audit_log_entries","auth.identities","auth.refresh_tokens","auth.sessions","auth.users","realtime.messages","storage.buckets","storage.migrations","storage.objects","storage.s3_multipart_uploads","storage.s3_multipart_uploads_parts"]}' -# full list: address_standardizer, address_standardizer_data_us, adminpack, amcheck, autoinc, bloom, btree_gin, btree_gist, citext, cube, dblink, dict_int, dict_xsyn, earthdistance, file_fdw, fuzzystrmatch, hstore, http, hypopg, index_advisor, insert_username, intagg, intarray, isn, lo, ltree, moddatetime, old_snapshot, orioledb, pageinspect, pg_buffercache, pg_cron, pg_freespacemap, pg_graphql, pg_hashids, pg_jsonschema, pg_net, pg_prewarm, pg_repack, pg_stat_monitor, pg_stat_statements, pg_surgery, pg_tle, pg_trgm, pg_visibility, pg_walinspect, pgaudit, pgcrypto, pgjwt, pgroonga, pgroonga_database, pgrouting, pgrowlocks, pgsodium, pgstattuple, pgtap, plcoffee, pljava, plls, plpgsql, plpgsql_check, plv8, postgis, postgis_raster, postgis_sfcgal, postgis_tiger_geocoder, postgis_topology, postgres_fdw, refint, rum, seg, sslinfo, supabase_vault, supautils, tablefunc, tcn, timescaledb, tsm_system_rows, tsm_system_time, unaccent, uuid-ossp, vector, wrappers, xml2 +# full list: address_standardizer, address_standardizer_data_us, adminpack, amcheck, autoinc, bloom, btree_gin, btree_gist, citext, cube, dblink, dict_int, dict_xsyn, earthdistance, file_fdw, fuzzystrmatch, hstore, http, hypopg, index_advisor, insert_username, intagg, intarray, isn, lo, ltree, moddatetime, old_snapshot, orioledb, pageinspect, pg_buffercache, pg_cron, pg_freespacemap, pg_graphql, pg_hashids, pg_jsonschema, pg_net, pg_prewarm, pg_repack, pg_stat_monitor, pg_stat_statements, pg_surgery, pg_tle, pg_trgm, pg_visibility, pg_walinspect, pgaudit, pgcrypto, pgjwt, pgmq, pgroonga, pgroonga_database, pgrouting, pgrowlocks, pgsodium, pgstattuple, pgtap, plcoffee, pljava, plls, plpgsql, plpgsql_check, plv8, postgis, postgis_raster, postgis_sfcgal, postgis_tiger_geocoder, postgis_topology, postgres_fdw, refint, rum, seg, sslinfo, supabase_vault, supautils, tablefunc, tcn, timescaledb, tsm_system_rows, tsm_system_time, unaccent, uuid-ossp, vector, wrappers, xml2 # omitted because may be unsafe: adminpack, amcheck, file_fdw, lo, old_snapshot, pageinspect, pg_buffercache, pg_freespacemap, pg_surgery, pg_visibility # omitted because deprecated: intagg, xml2 +# omitted because doesn't require superuser: pgmq supautils.privileged_extensions = 'address_standardizer, address_standardizer_data_us, autoinc, bloom, btree_gin, btree_gist, citext, cube, dblink, dict_int, dict_xsyn, earthdistance, fuzzystrmatch, hstore, http, hypopg, index_advisor, insert_username, intarray, isn, ltree, moddatetime, orioledb, pg_cron, pg_graphql, pg_hashids, pg_jsonschema, pg_net, pg_repack, pg_stat_monitor, pg_stat_statements, pg_tle, pg_trgm, pg_walinspect, pgaudit, pgcrypto, pgjwt, pg_prewarm, pgroonga, pgroonga_database, pgrouting, pgrowlocks, pgstattuple, pgsodium, pgtap, plcoffee, pljava, plls, plpgsql, plpgsql_check, plv8, postgis, postgis_raster, postgis_sfcgal, postgis_tiger_geocoder, postgis_topology, postgres_fdw, refint, rum, seg, sslinfo, supabase_vault, supautils, tablefunc, tcn, timescaledb, tsm_system_rows, tsm_system_time, unaccent, uuid-ossp, vector, wrappers' supautils.privileged_extensions_custom_scripts_path = '/etc/postgresql-custom/extension-custom-scripts' supautils.privileged_extensions_superuser = 'supabase_admin' diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index 8de62aece..241978e34 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.142" +postgres-version = "15.6.1.143" diff --git a/common.vars.pkr.hcl b/common.vars.pkr.hcl index fa1387690..871647b06 100644 --- a/common.vars.pkr.hcl +++ b/common.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.1.1.94" +postgres-version = "15.1.1.95" diff --git a/flake.nix b/flake.nix index 512d32d31..37117324c 100644 --- a/flake.nix +++ b/flake.nix @@ -111,6 +111,7 @@ ./nix/ext/pgroonga.nix ./nix/ext/index_advisor.nix ./nix/ext/wal2json.nix + ./nix/ext/pgmq.nix ./nix/ext/pg_repack.nix ./nix/ext/pg-safeupdate.nix ./nix/ext/plpgsql-check.nix diff --git a/nix/ext/pgmq.nix b/nix/ext/pgmq.nix new file mode 100644 index 000000000..97a3c27e3 --- /dev/null +++ b/nix/ext/pgmq.nix @@ -0,0 +1,33 @@ +{ lib, stdenv, fetchFromGitHub, postgresql }: + +stdenv.mkDerivation rec { + pname = "pgmq"; + version = "1.4.4"; + buildInputs = [ postgresql ]; + src = fetchFromGitHub { + owner = "tembo-io"; + repo = pname; + rev = "v${version}"; + hash = "sha256-z+8/BqIlHwlMnuIzMz6eylmYbSmhtsNt7TJf/CxbdVw="; + }; + + buildPhase = '' + cd pgmq-extension + ''; + + installPhase = '' + mkdir -p $out/{lib,share/postgresql/extension} + + mv sql/pgmq.sql $out/share/postgresql/extension/pgmq--${version}.sql + cp sql/*.sql $out/share/postgresql/extension + cp *.control $out/share/postgresql/extension + ''; + + meta = with lib; { + description = "A lightweight message queue. Like AWS SQS and RSMQ but on Postgres."; + homepage = "https://github.com/tembo-io/pgmq"; + maintainers = with maintainers; [ olirice ]; + platforms = postgresql.meta.platforms; + license = licenses.postgresql; + }; +} diff --git a/nix/tests/expected/extensions_sql_interface.out b/nix/tests/expected/extensions_sql_interface.out index 96df9add5..9b8e86fd5 100644 --- a/nix/tests/expected/extensions_sql_interface.out +++ b/nix/tests/expected/extensions_sql_interface.out @@ -88,6 +88,7 @@ order by pgaudit | t pgcrypto | t pgjwt | f + pgmq | f pgroonga | f pgroonga_database | f pgrouting | t @@ -121,7 +122,7 @@ order by vector | t wrappers | f xml2 | f -(81 rows) +(82 rows) /* @@ -1306,6 +1307,34 @@ order by pgjwt | public | url_decode | data text | bytea pgjwt | public | url_encode | data bytea | text pgjwt | public | verify | token text, secret text, algorithm text | TABLE(header json, payload json, valid boolean) + pgmq | pgmq | _belongs_to_pgmq | table_name text | boolean + pgmq | pgmq | _ensure_pg_partman_installed | | void + pgmq | pgmq | _get_partition_col | partition_interval text | text + pgmq | pgmq | _get_pg_partman_major_version | | integer + pgmq | pgmq | _get_pg_partman_schema | | text + pgmq | pgmq | archive | queue_name text, msg_id bigint | boolean + pgmq | pgmq | archive | queue_name text, msg_ids bigint[] | SETOF bigint + pgmq | pgmq | convert_archive_partitioned | table_name text, partition_interval text, retention_interval text, leading_partition integer | void + pgmq | pgmq | create | queue_name text | void + pgmq | pgmq | create_non_partitioned | queue_name text | void + pgmq | pgmq | create_partitioned | queue_name text, partition_interval text, retention_interval text | void + pgmq | pgmq | create_unlogged | queue_name text | void + pgmq | pgmq | delete | queue_name text, msg_id bigint | boolean + pgmq | pgmq | delete | queue_name text, msg_ids bigint[] | SETOF bigint + pgmq | pgmq | detach_archive | queue_name text | void + pgmq | pgmq | drop_queue | queue_name text, partitioned boolean | boolean + pgmq | pgmq | format_table_name | queue_name text, prefix text | text + pgmq | pgmq | list_queues | | SETOF pgmq.queue_record + pgmq | pgmq | metrics | queue_name text | pgmq.metrics_result + pgmq | pgmq | metrics_all | | SETOF pgmq.metrics_result + pgmq | pgmq | pop | queue_name text | SETOF pgmq.message_record + pgmq | pgmq | purge_queue | queue_name text | bigint + pgmq | pgmq | read | queue_name text, vt integer, qty integer | SETOF pgmq.message_record + pgmq | pgmq | read_with_poll | queue_name text, vt integer, qty integer, max_poll_seconds integer, poll_interval_ms integer | SETOF pgmq.message_record + pgmq | pgmq | send | queue_name text, msg jsonb, delay integer | SETOF bigint + pgmq | pgmq | send_batch | queue_name text, msgs jsonb[], delay integer | SETOF bigint + pgmq | pgmq | set_vt | queue_name text, msg_id bigint, vt integer | SETOF pgmq.message_record + pgmq | pgmq | validate_queue_name | queue_name text | void pgroonga | pgroonga | command | groongacommand text | text pgroonga | pgroonga | command | groongacommand text, arguments text[] | text pgroonga | pgroonga | command_escape_value | value text | text @@ -5160,7 +5189,7 @@ order by xml2 | public | xpath_table | text, text, text, text, text | SETOF record xml2 | public | xslt_process | text, text | text xml2 | public | xslt_process | text, text, text | text -(5009 rows) +(5037 rows) /* @@ -5382,6 +5411,10 @@ order by pg_tle | pgtle | feature_info | obj_identity pg_tle | pgtle | feature_info | proname pg_tle | pgtle | feature_info | schema_name + pgmq | pgmq | meta | created_at + pgmq | pgmq | meta | is_partitioned + pgmq | pgmq | meta | is_unlogged + pgmq | pgmq | meta | queue_name pgsodium | pgsodium | decrypted_key | associated_data pgsodium | pgsodium | decrypted_key | comment pgsodium | pgsodium | decrypted_key | created @@ -6082,8 +6115,8 @@ order by timescaledb | timescaledb_information | compression_settings | orderby_asc timescaledb | timescaledb_information | compression_settings | orderby_column_index timescaledb | _timescaledb_catalog | compression_settings | orderby_desc - timescaledb | _timescaledb_catalog | compression_settings | orderby_nullsfirst timescaledb | timescaledb_information | compression_settings | orderby_nullsfirst + timescaledb | _timescaledb_catalog | compression_settings | orderby_nullsfirst timescaledb | _timescaledb_catalog | compression_settings | relid timescaledb | _timescaledb_catalog | compression_settings | segmentby timescaledb | timescaledb_information | compression_settings | segmentby_column_index @@ -6275,5 +6308,5 @@ order by wrappers | public | wrappers_fdw_stats | rows_in wrappers | public | wrappers_fdw_stats | rows_out wrappers | public | wrappers_fdw_stats | updated_at -(1082 rows) +(1086 rows) diff --git a/nix/tests/expected/pgmq.out b/nix/tests/expected/pgmq.out new file mode 100644 index 000000000..9fb18191b --- /dev/null +++ b/nix/tests/expected/pgmq.out @@ -0,0 +1,141 @@ +-- Test the standard flow +select + pgmq.create('Foo'); + create +-------- + +(1 row) + +select + * +from + pgmq.send( + queue_name:='Foo', + msg:='{"foo": "bar1"}' + ); + send +------ + 1 +(1 row) + +-- Test queue is not case sensitive +select + * +from + pgmq.send( + queue_name:='foo', -- note: lowercase useage + msg:='{"foo": "bar2"}', + delay:=5 + ); + send +------ + 2 +(1 row) + +select + msg_id, + read_ct, + message +from + pgmq.read( + queue_name:='Foo', + vt:=30, + qty:=2 + ); + msg_id | read_ct | message +--------+---------+----------------- + 1 | 1 | {"foo": "bar1"} +(1 row) + +select + msg_id, + read_ct, + message +from + pgmq.pop('Foo'); + msg_id | read_ct | message +--------+---------+--------- +(0 rows) + +-- Archive message with msg_id=2. +select + pgmq.archive( + queue_name:='Foo', + msg_id:=2 + ); + archive +--------- + t +(1 row) + +select + pgmq.create('my_queue'); + create +-------- + +(1 row) + +select + pgmq.send_batch( + queue_name:='my_queue', + msgs:=array['{"foo": "bar3"}','{"foo": "bar4"}','{"foo": "bar5"}']::jsonb[] +); + send_batch +------------ + 1 + 2 + 3 +(3 rows) + +select + pgmq.archive( + queue_name:='my_queue', + msg_ids:=array[3, 4, 5] + ); + archive +--------- + 3 +(1 row) + +select + pgmq.delete('my_queue', 6); + delete +-------- + f +(1 row) + +select + pgmq.drop_queue('my_queue'); + drop_queue +------------ + t +(1 row) + +/* +-- Disabled until pg_partman goes back into the image +select + pgmq.create_partitioned( + 'my_partitioned_queue', + '5 seconds', + '10 seconds' +); +*/ +-- Make sure SQLI enabling characters are blocked +select pgmq.create('F--oo'); +ERROR: queue name contains invalid characters: $, ;, --, or \' +CONTEXT: PL/pgSQL function pgmq.format_table_name(text,text) line 5 at RAISE +PL/pgSQL function pgmq.create_non_partitioned(text) line 3 during statement block local variable initialization +SQL statement "SELECT pgmq.create_non_partitioned(queue_name)" +PL/pgSQL function pgmq."create"(text) line 3 at PERFORM +select pgmq.create('F$oo'); +ERROR: queue name contains invalid characters: $, ;, --, or \' +CONTEXT: PL/pgSQL function pgmq.format_table_name(text,text) line 5 at RAISE +PL/pgSQL function pgmq.create_non_partitioned(text) line 3 during statement block local variable initialization +SQL statement "SELECT pgmq.create_non_partitioned(queue_name)" +PL/pgSQL function pgmq."create"(text) line 3 at PERFORM +select pgmq.create($$F'oo$$); +ERROR: queue name contains invalid characters: $, ;, --, or \' +CONTEXT: PL/pgSQL function pgmq.format_table_name(text,text) line 5 at RAISE +PL/pgSQL function pgmq.create_non_partitioned(text) line 3 during statement block local variable initialization +SQL statement "SELECT pgmq.create_non_partitioned(queue_name)" +PL/pgSQL function pgmq."create"(text) line 3 at PERFORM diff --git a/nix/tests/prime.sql b/nix/tests/prime.sql index f31558531..5406a9044 100644 --- a/nix/tests/prime.sql +++ b/nix/tests/prime.sql @@ -42,6 +42,7 @@ create extension pg_graphql; create extension pg_freespacemap; create extension pg_hashids; create extension pg_prewarm; +create extension pgmq; create extension pg_jsonschema; create extension pg_repack; create extension pg_stat_monitor; diff --git a/nix/tests/sql/pgmq.sql b/nix/tests/sql/pgmq.sql new file mode 100644 index 000000000..cd47cc34d --- /dev/null +++ b/nix/tests/sql/pgmq.sql @@ -0,0 +1,90 @@ +-- Test the standard flow +select + pgmq.create('Foo'); + +select + * +from + pgmq.send( + queue_name:='Foo', + msg:='{"foo": "bar1"}' + ); + +-- Test queue is not case sensitive +select + * +from + pgmq.send( + queue_name:='foo', -- note: lowercase useage + msg:='{"foo": "bar2"}', + delay:=5 + ); + +select + msg_id, + read_ct, + message +from + pgmq.read( + queue_name:='Foo', + vt:=30, + qty:=2 + ); + +select + msg_id, + read_ct, + message +from + pgmq.pop('Foo'); + + +-- Archive message with msg_id=2. +select + pgmq.archive( + queue_name:='Foo', + msg_id:=2 + ); + + +select + pgmq.create('my_queue'); + +select + pgmq.send_batch( + queue_name:='my_queue', + msgs:=array['{"foo": "bar3"}','{"foo": "bar4"}','{"foo": "bar5"}']::jsonb[] +); + +select + pgmq.archive( + queue_name:='my_queue', + msg_ids:=array[3, 4, 5] + ); + +select + pgmq.delete('my_queue', 6); + + +select + pgmq.drop_queue('my_queue'); + +/* +-- Disabled until pg_partman goes back into the image +select + pgmq.create_partitioned( + 'my_partitioned_queue', + '5 seconds', + '10 seconds' +); +*/ + + +-- Make sure SQLI enabling characters are blocked +select pgmq.create('F--oo'); +select pgmq.create('F$oo'); +select pgmq.create($$F'oo$$); + + + + From 6bbd27ae1d2c86f95b206ae9919e118cfad601f3 Mon Sep 17 00:00:00 2001 From: Kang Ming Date: Thu, 28 Nov 2024 17:03:53 +0800 Subject: [PATCH 40/44] fix: upload gotrue binary to internal-artifacts as a gz file (#1340) --- ansible/manifest-playbook.yml | 20 ++------------------ 1 file changed, 2 insertions(+), 18 deletions(-) diff --git a/ansible/manifest-playbook.yml b/ansible/manifest-playbook.yml index 93f0e15c5..5c1c65053 100644 --- a/ansible/manifest-playbook.yml +++ b/ansible/manifest-playbook.yml @@ -18,26 +18,10 @@ - name: gotrue - download commit archive get_url: url: "https://github.com/supabase/gotrue/releases/download/v{{ gotrue_release }}/auth-v{{ gotrue_release }}-arm64.tar.gz" - dest: /tmp/gotrue.tar.gz + dest: /tmp/auth-v{{ gotrue_release }}-arm64.tar.gz checksum: "{{ gotrue_release_checksum }}" timeout: 60 - - name: gotrue - create /tmp/gotrue - file: - path: /tmp/gotrue - state: directory - mode: 0775 - - - name: gotrue - unpack archive in /tmp/gotrue - unarchive: - remote_src: yes - src: /tmp/gotrue.tar.gz - dest: /tmp/gotrue - - - name: gotrue - pack archive - shell: | - cd /tmp && tar -cJf gotrue-v{{ gotrue_release }}-arm64.tar.xz gotrue - - name: PostgREST - download ubuntu binary archive (arm) get_url: url: "https://github.com/PostgREST/postgrest/releases/download/v{{ postgrest_release }}/postgrest-v{{ postgrest_release }}-ubuntu-aarch64.tar.xz" @@ -82,7 +66,7 @@ aws s3 cp /tmp/{{ item.file }} s3://{{ internal_artifacts_bucket }}/upgrades/{{ item.service }}/{{ item.file }} with_items: - service: gotrue - file: gotrue-v{{ gotrue_release }}-arm64.tar.xz + file: auth-v{{ gotrue_release }}-arm64.tar.gz - service: postgrest file: postgrest-{{ postgrest_release }}-arm64.tar.xz - service: supabase-admin-api From af0f41b75a601fc163b5a74d1a4e7dc7881ff430 Mon Sep 17 00:00:00 2001 From: Chris Stockton <180184+cstockton@users.noreply.github.com> Date: Thu, 5 Dec 2024 20:49:38 -0700 Subject: [PATCH 41/44] feat: upgrade to auth v2.165.0 (#1357) Upgrade Auth version to v2.165.0 https://github.com/supabase/auth/pull/1846 Co-authored-by: Chris Stockton --- ansible/vars.yml | 4 ++-- common-nix.vars.pkr.hcl | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ansible/vars.yml b/ansible/vars.yml index aa5ea9773..f2224be91 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -17,8 +17,8 @@ postgrest_release: "12.2.3" postgrest_arm_release_checksum: sha1:fbfd6613d711ce1afa25c42d5df8f1b017f396f9 postgrest_x86_release_checksum: sha1:61c513f91a8931be4062587b9d4a18b42acf5c05 -gotrue_release: 2.164.0 -gotrue_release_checksum: sha1:7305f82b02fb172d8f7272c0966c5d9eab41c5bd +gotrue_release: 2.165.0 +gotrue_release_checksum: sha1:4543b2257aad9dd258cac3791f244322b79fbc0e aws_cli_release: "2.2.7" diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index 241978e34..5c568f84c 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.143" +postgres-version = "15.6.1.144" From 57149e1c070cf464d50e7a9fca4c145742340597 Mon Sep 17 00:00:00 2001 From: Kang Ming Date: Fri, 6 Dec 2024 17:56:32 +0800 Subject: [PATCH 42/44] feat: bump gotrue to v2.165.1 (#1358) --- ansible/vars.yml | 4 ++-- common-nix.vars.pkr.hcl | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/ansible/vars.yml b/ansible/vars.yml index f2224be91..30f21c4c9 100644 --- a/ansible/vars.yml +++ b/ansible/vars.yml @@ -17,8 +17,8 @@ postgrest_release: "12.2.3" postgrest_arm_release_checksum: sha1:fbfd6613d711ce1afa25c42d5df8f1b017f396f9 postgrest_x86_release_checksum: sha1:61c513f91a8931be4062587b9d4a18b42acf5c05 -gotrue_release: 2.165.0 -gotrue_release_checksum: sha1:4543b2257aad9dd258cac3791f244322b79fbc0e +gotrue_release: 2.165.1 +gotrue_release_checksum: sha1:bbd62327d8612ac756177dde81d5368b660ca4c8 aws_cli_release: "2.2.7" diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index 5c568f84c..fa25f1104 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.144" +postgres-version = "15.6.1.145" From 34cbe366d9bdc004ee42556d41602aeb5d563f72 Mon Sep 17 00:00:00 2001 From: samrose Date: Fri, 6 Dec 2024 10:53:08 -0500 Subject: [PATCH 43/44] feat: build and cache debug and src on this branch (#1360) --- docker/nix/build_nix.sh | 7 +++++++ flake.nix | 2 +- 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/docker/nix/build_nix.sh b/docker/nix/build_nix.sh index a8a8639e2..5035be062 100644 --- a/docker/nix/build_nix.sh +++ b/docker/nix/build_nix.sh @@ -5,6 +5,13 @@ nix --version if [ -d "/workspace" ]; then cd /workspace fi +SYSTEM=$(nix-instantiate --eval -E builtins.currentSystem | tr -d '"') nix build .#psql_15/bin -o psql_15 nix flake check -L nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./psql_15 +if [ "$SYSTEM" = "aarch64-linux" ]; then + nix build .#postgresql_15_debug -o ./postgresql_15_debug + nix build .#postgresql_15_src -o ./postgresql_15_src + nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./postgresql_15_debug-debug + nix copy --to s3://nix-postgres-artifacts?secret-key=nix-secret-key ./postgresql_15_src +fi diff --git a/flake.nix b/flake.nix index 37117324c..82e76983b 100644 --- a/flake.nix +++ b/flake.nix @@ -296,7 +296,7 @@ pg_regress = pg_regress; pg_prove = pkgs.perlPackages.TAPParserSourceHandlerpgTAP; postgresql_15 = pkgs.postgresql_15; - + postgresql_15_debug = if pkgs.stdenv.isLinux then postgresql_15.debug else null; postgresql_15_src = pkgs.stdenv.mkDerivation { pname = "postgresql-15-src"; version = pkgs.postgresql_15.version; From 8b3db2e7a3084f1a9af2974ce113e11c3f976047 Mon Sep 17 00:00:00 2001 From: Bobbie Soedirgo <31685197+soedirgo@users.noreply.github.com> Date: Tue, 10 Dec 2024 05:56:24 +0800 Subject: [PATCH 44/44] fix: pgmq ownership (#1362) * fix: pgmq ownership * tmp * bump version for production release --------- Co-authored-by: Oliver Rice --- .../pgmq/after-create.sql | 19 +++++++++++++++++++ common-nix.vars.pkr.hcl | 2 +- 2 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 ansible/files/postgresql_extension_custom_scripts/pgmq/after-create.sql diff --git a/ansible/files/postgresql_extension_custom_scripts/pgmq/after-create.sql b/ansible/files/postgresql_extension_custom_scripts/pgmq/after-create.sql new file mode 100644 index 000000000..8b126d403 --- /dev/null +++ b/ansible/files/postgresql_extension_custom_scripts/pgmq/after-create.sql @@ -0,0 +1,19 @@ +do $$ +declare + extoid oid := (select oid from pg_extension where extname = 'pgmq'); + r record; +begin + set local search_path = ''; + update pg_extension set extowner = 'postgres'::regrole where extname = 'pgmq'; + for r in (select * from pg_depend where refobjid = extoid) loop + if r.classid = 'pg_type'::regclass then + execute(format('alter type %s owner to postgres;', r.objid::regtype)); + elsif r.classid = 'pg_proc'::regclass then + execute(format('alter function %s(%s) owner to postgres;', r.objid::regproc, pg_get_function_identity_arguments(r.objid))); + elsif r.classid = 'pg_class'::regclass then + execute(format('alter table %s owner to postgres;', r.objid::regclass)); + else + raise exception 'error on pgmq after-create script: unexpected object type %', r.classid; + end if; + end loop; +end $$; diff --git a/common-nix.vars.pkr.hcl b/common-nix.vars.pkr.hcl index fa25f1104..587e717cd 100644 --- a/common-nix.vars.pkr.hcl +++ b/common-nix.vars.pkr.hcl @@ -1 +1 @@ -postgres-version = "15.6.1.145" +postgres-version = "15.6.1.146"