Restore pg_repack to our extensions list (#886)

* restore pg_repack 1.4.8 for non-superuser cli testing * bump image tag for staging * bump pg_repack for pg16 support * update pg_repack in readme --------- Co-authored-by: Div Arora <[email protected]>
supabase · Feb 16, 2024 · 81baf1c · 81baf1c
1 parent b27a1fd
commit 81baf1c
Show file tree

Hide file tree

Showing 9 changed files with 107 additions and 2 deletions.
diff --git a/Dockerfile b/Dockerfile
@@ -27,6 +27,7 @@ ARG pgsodium_release=3.1.6
 ARG pg_graphql_release=1.2.2
 ARG pg_stat_monitor_release=1.1.1
 ARG pg_jsonschema_release=0.1.4
+ARG pg_repack_release=1.4.8
 ARG vault_release=0.2.8
 ARG groonga_release=12.0.8
 ARG pgroonga_release=2.4.0
@@ -736,6 +737,32 @@ RUN --mount=type=cache,target=/ccache,from=public.ecr.aws/supabase/postgres:ccac
 # Create debian package
 RUN checkinstall -D --install=no --fstrans=no --backup=no --pakdir=/tmp --nodoc
 
+####################
+ # 27-pg_repack.yml
+ ####################
+ FROM ccache as pg_repack-source
+ ARG pg_repack_release
+ ARG pg_repack_release_checksum
+ ADD --checksum=${pg_repack_release_checksum} \
+     "https://github.com/reorg/pg_repack/archive/refs/tags/ver_${pg_repack_release}.tar.gz" \
+     /tmp/pg_repack.tar.gz
+ RUN tar -xvf /tmp/pg_repack.tar.gz -C /tmp && \
+     rm -rf /tmp/pg_repack.tar.gz
+ # Install build dependencies
+ RUN apt-get update && apt-get install -y --no-install-recommends \
+     liblz4-dev \
+     libz-dev \
+     libzstd-dev \
+     libreadline-dev \
+     && rm -rf /var/lib/apt/lists/*
+ # Build from source
+ WORKDIR /tmp/pg_repack-ver_${pg_repack_release}
+ ENV USE_PGXS=1
+ RUN --mount=type=cache,target=/ccache,from=public.ecr.aws/supabase/postgres:ccache \
+     make -j$(nproc)
+ # Create debian package
+ RUN checkinstall -D --install=no --fstrans=no --backup=no --pakdir=/tmp --pkgversion=${pg_repack_release} --nodoc
+
 ####################
 # 28-pgvector.yml
 ####################
@@ -827,6 +854,7 @@ COPY --from=vault-source /tmp/*.deb /tmp/
 COPY --from=pgroonga-source /tmp/*.deb /tmp/
 COPY --from=wrappers /tmp/*.deb /tmp/
 COPY --from=hypopg-source /tmp/*.deb /tmp/
+COPY --from=pg_repack-source /tmp/*.deb /tmp/
 COPY --from=pgvector-source /tmp/*.deb /tmp/
 COPY --from=pg_tle-source /tmp/*.deb /tmp/
 COPY --from=supautils /tmp/*.deb /tmp/

diff --git a/README.md b/README.md
@@ -26,6 +26,7 @@ Unmodified Postgres with some useful plugins. Our goal with this repo is not to
 | [plv8](https://github.com/plv8/plv8) | [commit](https://github.com/plv8/plv8/commit/bcddd92f71530e117f2f98b92d206dafe824f73a) | Write in Javascript functions in Postgres. |
 | [pg_plan_filter](https://github.com/pgexperts/pg_plan_filter) | [commit](https://github.com/pgexperts/pg_plan_filter/commit/5081a7b5cb890876e67d8e7486b6a64c38c9a492) | Only allow statements that fulfill set criteria to be executed. |
 | [pg_net](https://github.com/supabase/pg_net) | [v0.6.1](https://github.com/supabase/pg_net/releases/tag/v0.6.1) | Expose the SQL interface for async networking. |
+| [pg_repack](https://github.com/reorg/pg_repack) | [ver_1.5.0](https://github.com/reorg/pg_repack/releases/tag/ver_1.5.0) | Tool to remove bloat from tables and indexes
 | [rum](https://github.com/postgrespro/rum) | [1.3.13](https://github.com/postgrespro/rum/releases/tag/1.3.13) | An alternative to the GIN index. |
 | [pg_hashids](https://github.com/iCyberon/pg_hashids) | [commit](https://github.com/iCyberon/pg_hashids/commit/83398bcbb616aac2970f5e77d93a3200f0f28e74) | Generate unique identifiers from numbers. |
 | [pgsodium](https://github.com/michelp/pgsodium) | [3.1.0](https://github.com/michelp/pgsodium/releases/tag/2.0.0) | Modern encryption API using libsodium. |

diff --git a/ansible/files/postgresql_config/supautils.conf.j2 b/ansible/files/postgresql_config/supautils.conf.j2
@@ -1,7 +1,7 @@
-# full list:                                 address_standardizer, address_standardizer_data_us, adminpack, amcheck, autoinc, bloom, btree_gin, btree_gist, citext, cube, dblink, dict_int, dict_xsyn, earthdistance, file_fdw, fuzzystrmatch, hstore, http, hypopg, insert_username, intagg, intarray, isn, lo, ltree, moddatetime, old_snapshot, orioledb, pageinspect, pg_buffercache, pg_cron, pg_freespacemap, pg_graphql, pg_hashids, pg_jsonschema, pg_net, pg_prewarm, pg_stat_monitor, pg_stat_statements, pg_surgery, pg_tle, pg_trgm, pg_visibility, pg_walinspect, pgaudit, pgcrypto, pgjwt, pgroonga, pgroonga_database, pgrouting, pgrowlocks, pgsodium, pgstattuple, pgtap, plcoffee, pljava, plls, plpgsql, plpgsql_check, plv8, postgis, postgis_raster, postgis_sfcgal, postgis_tiger_geocoder, postgis_topology, postgres_fdw, refint, rum, seg, sslinfo, supabase_vault, supautils, tablefunc, tcn, timescaledb, tsm_system_rows, tsm_system_time, unaccent, uuid-ossp, vector, wrappers, xml2
+# full list:                                 address_standardizer, address_standardizer_data_us, adminpack, amcheck, autoinc, bloom, btree_gin, btree_gist, citext, cube, dblink, dict_int, dict_xsyn, earthdistance, file_fdw, fuzzystrmatch, hstore, http, hypopg, insert_username, intagg, intarray, isn, lo, ltree, moddatetime, old_snapshot, orioledb, pageinspect, pg_buffercache, pg_cron, pg_freespacemap, pg_graphql, pg_hashids, pg_jsonschema, pg_net, pg_prewarm, pg_repack, pg_stat_monitor, pg_stat_statements, pg_surgery, pg_tle, pg_trgm, pg_visibility, pg_walinspect, pgaudit, pgcrypto, pgjwt, pgroonga, pgroonga_database, pgrouting, pgrowlocks, pgsodium, pgstattuple, pgtap, plcoffee, pljava, plls, plpgsql, plpgsql_check, plv8, postgis, postgis_raster, postgis_sfcgal, postgis_tiger_geocoder, postgis_topology, postgres_fdw, refint, rum, seg, sslinfo, supabase_vault, supautils, tablefunc, tcn, timescaledb, tsm_system_rows, tsm_system_time, unaccent, uuid-ossp, vector, wrappers, xml2
 # omitted because may be unsafe:             adminpack, amcheck, file_fdw, lo, old_snapshot, pageinspect, pg_buffercache, pg_freespacemap, pg_prewarm, pg_surgery, pg_visibility, pgstattuple
 # omitted because deprecated:                intagg, xml2
-supautils.privileged_extensions = 'address_standardizer, address_standardizer_data_us, autoinc, bloom, btree_gin, btree_gist, citext, cube, dblink, dict_int, dict_xsyn, earthdistance, fuzzystrmatch, hstore, http, hypopg, insert_username, intarray, isn, ltree, moddatetime, orioledb, pg_cron, pg_graphql, pg_hashids, pg_jsonschema, pg_net, pg_stat_monitor, pg_stat_statements, pg_tle, pg_trgm, pg_walinspect, pgaudit, pgcrypto, pgjwt, pgroonga, pgroonga_database, pgrouting, pgrowlocks, pgsodium, pgtap, plcoffee, pljava, plls, plpgsql, plpgsql_check, plv8, postgis, postgis_raster, postgis_sfcgal, postgis_tiger_geocoder, postgis_topology, postgres_fdw, refint, rum, seg, sslinfo, supabase_vault, supautils, tablefunc, tcn, timescaledb, tsm_system_rows, tsm_system_time, unaccent, uuid-ossp, vector, wrappers'
+supautils.privileged_extensions = 'address_standardizer, address_standardizer_data_us, autoinc, bloom, btree_gin, btree_gist, citext, cube, dblink, dict_int, dict_xsyn, earthdistance, fuzzystrmatch, hstore, http, hypopg, insert_username, intarray, isn, ltree, moddatetime, orioledb, pg_cron, pg_graphql, pg_hashids, pg_jsonschema, pg_net, pg_repack, pg_stat_monitor, pg_stat_statements, pg_tle, pg_trgm, pg_walinspect, pgaudit, pgcrypto, pgjwt, pgroonga, pgroonga_database, pgrouting, pgrowlocks, pgsodium, pgtap, plcoffee, pljava, plls, plpgsql, plpgsql_check, plv8, postgis, postgis_raster, postgis_sfcgal, postgis_tiger_geocoder, postgis_topology, postgres_fdw, refint, rum, seg, sslinfo, supabase_vault, supautils, tablefunc, tcn, timescaledb, tsm_system_rows, tsm_system_time, unaccent, uuid-ossp, vector, wrappers'
 supautils.privileged_extensions_custom_scripts_path = '/etc/postgresql-custom/extension-custom-scripts'
 supautils.privileged_extensions_superuser = 'supabase_admin'
 supautils.privileged_role = 'postgres'

diff --git a/ansible/tasks/postgres-extensions/27-pg_repack.yml b/ansible/tasks/postgres-extensions/27-pg_repack.yml
@@ -0,0 +1,38 @@
+# pg_repack
+ - name: pg_repack - download & install dependencies
+   apt:
+     pkg:
+       - liblz4-dev
+       - libz-dev
+       - libzstd-dev
+       - libreadline-dev
+     update_cache: yes
+     install_recommends: no
+
+ - name: pg_repack - download latest release
+   git:
+     repo: https://github.com/reorg/pg_repack.git
+     dest: /tmp/pg_repack
+     version: "ver_{{ pg_repack_release }}"
+   become: yes
+
+ - name: pg_repack - build
+   make:
+     chdir: /tmp/pg_repack
+     params:
+       USE_PGXS: 1
+   become: yes
+
+ - name: pg_repack - install
+   make:
+     chdir: /tmp/pg_repack
+     target: install
+     params:
+       USE_PGXS: 1
+   become: yes
+
+ - name: pg_repack - cleanup
+   file:
+     state: absent
+     path: /tmp/pg_repack
+
diff --git a/ansible/tasks/setup-extensions.yml b/ansible/tasks/setup-extensions.yml
@@ -79,6 +79,9 @@
 
 - name: Install hypopg
   import_tasks: tasks/postgres-extensions/26-hypopg.yml
+
+ - name: Install pg_repack
+  import_tasks: tasks/postgres-extensions/27-pg_repack.yml
 
 - name: Install pgvector
   import_tasks: tasks/postgres-extensions/28-pgvector.yml

diff --git a/ansible/vars.yml b/ansible/vars.yml
@@ -132,6 +132,9 @@ wrappers_release: "0.2.0"
 hypopg_release: "1.3.1"
 hypopg_release_checksum: sha256:e7f01ee0259dc1713f318a108f987663d60f3041948c2ada57a94b469565ca8e
 
+pg_repack_release: "1.5.0"
+pg_repack_release_checksum: sha256:9a14d6a95bfa29f856aa10538238622c1f351d38eb350b196c06720a878ccc52
+
 pgvector_release: "0.6.0"
 pgvector_release_checksum: sha256:b0cf4ba1ab016335ac8fb1cada0d2106235889a194fffeece217c5bda90b2f19
 

diff --git a/docker/orioledb/Dockerfile b/docker/orioledb/Dockerfile
@@ -27,6 +27,7 @@ ARG pgsodium_release=3.1.6
 ARG pg_graphql_release=1.4.2
 ARG pg_stat_monitor_release=1.1.1
 ARG pg_jsonschema_release=0.2.0
+ARG pg_repack_release=1.4.8
 ARG vault_release=0.2.8
 ARG groonga_release=12.0.8
 ARG pgroonga_release=2.4.0
@@ -838,6 +839,32 @@ RUN --mount=type=cache,target=/ccache,from=public.ecr.aws/supabase/postgres:ccac
 # Create debian package
 RUN checkinstall -D --install=no --fstrans=no --backup=no --pakdir=/tmp --nodoc
 
+####################
+ # 27-pg_repack.yml
+ ####################
+ FROM ccache as pg_repack-source
+ ARG pg_repack_release
+ ARG pg_repack_release_checksum
+ ADD --checksum=${pg_repack_release_checksum} \
+     "https://github.com/reorg/pg_repack/archive/refs/tags/ver_${pg_repack_release}.tar.gz" \
+     /tmp/pg_repack.tar.gz
+ RUN tar -xvf /tmp/pg_repack.tar.gz -C /tmp && \
+     rm -rf /tmp/pg_repack.tar.gz
+ # Install build dependencies
+ RUN apt-get update && apt-get install -y --no-install-recommends \
+     liblz4-dev \
+     libz-dev \
+     libzstd-dev \
+     libreadline-dev \
+     && rm -rf /var/lib/apt/lists/*
+ # Build from source
+ WORKDIR /tmp/pg_repack-ver_${pg_repack_release}
+ ENV USE_PGXS=1
+ RUN --mount=type=cache,target=/ccache,from=public.ecr.aws/supabase/postgres:ccache \
+     make -j$(nproc)
+ # Create debian package
+ RUN checkinstall -D --install=no --fstrans=no --backup=no --pakdir=/tmp --pkgversion=${pg_repack_release} --nodoc
+
 ####################
 # 28-pgvector.yml
 ####################
@@ -944,6 +971,7 @@ COPY --from=vault-source /tmp/*.deb /tmp/
 COPY --from=pgroonga-source /tmp/*.deb /tmp/
 COPY --from=wrappers-source /tmp/*.deb /tmp/
 COPY --from=hypopg-source /tmp/*.deb /tmp/
+COPY --from=pg_repack-source /tmp/*.deb /tmp/
 COPY --from=pgvector-source /tmp/*.deb /tmp/
 COPY --from=pg_tle-source /tmp/*.deb /tmp/
 COPY --from=supautils-source /tmp/*.deb /tmp/

diff --git a/migrations/tests/extensions/27-pg_repack.sql b/migrations/tests/extensions/27-pg_repack.sql
@@ -0,0 +1,3 @@
+BEGIN;
+create extension if not exists pg_repack with schema "extensions";
+ROLLBACK;
diff --git a/migrations/tests/extensions/test.sql b/migrations/tests/extensions/test.sql
@@ -25,5 +25,6 @@
 \ir 24-pgroonga.sql
 \ir 25-wrappers.sql
 \ir 26-hypopg.sql
+\ir 27-pg_repack.sql
 \ir 28-pgvector.sql
 \ir 29-pg_tle.sql