Merge pull request #257 from dutow/pg17imp3

PG 17 merge
percona · Aug 14, 2024 · 9cf87a4 · 9cf87a4
2 parents 1b184bf + 2dd56f6
commit 9cf87a4
Show file tree

Hide file tree

Showing 20 changed files with 25,975 additions and 0 deletions.
diff --git a/.github/workflows/postgresql-17-src-make.yml b/.github/workflows/postgresql-17-src-make.yml
@@ -0,0 +1,97 @@
+name: postgresql-17-src-make
+on: [pull_request, workflow_dispatch]
+
+jobs:
+  build:
+    name: pg-17-src-make-test
+    runs-on: ubuntu-22.04
+    steps:
+
+
+      - name: Remove old postgres
+        run: |
+          sudo apt purge postgresql-client-common postgresql-common \
+            postgresql postgresql*
+          sudo rm -rf /var/lib/postgresql /var/log/postgresql /etc/postgresql \
+           /usr/lib/postgresql /usr/include/postgresql /usr/share/postgresql \
+           /etc/postgresql
+          sudo rm -f /usr/bin/pg_config
+
+      - name: Install dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y libreadline6-dev systemtap-sdt-dev \
+            zlib1g-dev libssl-dev libpam0g-dev bison flex \
+            libipc-run-perl -y docbook-xsl docbook-xsl libxml2 libxml2-utils \
+            libxml2-dev libxslt-dev xsltproc libkrb5-dev libldap2-dev \
+            libsystemd-dev gettext tcl-dev libperl-dev pkg-config clang-11 \
+            llvm-11 llvm-11-dev libselinux1-dev python3-dev \
+            uuid-dev liblz4-dev libcurl4-openssl-dev libhttp-server-simple-perl
+          sudo /usr/bin/perl -MCPAN -e 'install IPC::Run'
+          sudo /usr/bin/perl -MCPAN -e 'install Text::Trim'
+          wget -O- https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg
+          echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list
+          sudo apt update && sudo apt install -y vault
+
+      - name: Clone postgres repository
+        uses: actions/checkout@v4
+        with:
+          repository: 'postgres/postgres'
+          ref: 'REL_17_STABLE'
+          path: 'src'
+
+      - name: Clone pg_tde repository
+        uses: actions/checkout@v4
+        with:
+          path: 'src/contrib/pg_tde'
+
+      - name: Create pgsql dir
+        run: mkdir -p /opt/pgsql
+
+      - name: Build postgres
+        run: |
+          ./configure --with-openssl  --enable-tap-tests=no --enable-cassert
+          make -j
+          sudo make install
+        working-directory: src
+
+      - name: Build pg_tde
+        run: |
+          ./configure
+          make -j MAJORVERSION=17
+          sudo make install
+        working-directory: src/contrib/pg_tde
+
+      - name: Start postgresql cluster with pg_tde
+        run: |
+          export PATH="/usr/local/pgsql/bin:$PATH"
+          sudo cp /usr/local/pgsql/bin/pg_config /usr/bin
+          initdb -D /opt/pgsql/data
+          echo "shared_preload_libraries = 'pg_tde'" >> \
+            /opt/pgsql/data/postgresql.conf
+          pg_ctl -D /opt/pgsql/data -l logfile start
+
+      - name: Test pg_tde
+        run: |
+          TV=$(mktemp)
+          { exec >$TV; vault server -dev; } &
+          sleep 10
+          export ROOT_TOKEN=$(cat $TV | grep "Root Token" | cut -d ":" -f 2 | xargs echo -n)
+          echo "Root token: $ROOT_TOKEN"
+          make installcheck
+        working-directory: src/contrib/pg_tde
+
+      - name: Report on test fail
+        uses: actions/upload-artifact@v4
+        if: ${{ failure() }}
+        with:
+          name: Regressions diff and postgresql log
+          path: |
+            src/contrib/pg_tde/regression.diffs
+            logfile
+          retention-days: 3
+
+      - name: Report on test fail 2
+        if: ${{ failure() }}
+        run: |
+          cat src/contrib/pg_tde/regression.diffs
diff --git a/.github/workflows/postgresql-17-src-meson.yml b/.github/workflows/postgresql-17-src-meson.yml
@@ -0,0 +1,78 @@
+name: postgresql-17-src-meson
+on: [pull_request, workflow_dispatch]
+
+jobs:
+  build:
+    name: pg-17-src-meson-test
+    runs-on: ubuntu-22.04
+    steps:
+
+
+      - name: Remove old postgres
+        run: |
+          sudo apt purge postgresql-client-common postgresql-common \
+            postgresql postgresql*
+          sudo rm -rf /var/lib/postgresql /var/log/postgresql /etc/postgresql \
+           /usr/lib/postgresql /usr/include/postgresql /usr/share/postgresql \
+           /etc/postgresql
+          sudo rm -f /usr/bin/pg_config
+
+      - name: Install dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y libreadline6-dev systemtap-sdt-dev \
+            zlib1g-dev libssl-dev libpam0g-dev bison flex \
+            libipc-run-perl -y docbook-xsl docbook-xsl libxml2 libxml2-utils \
+            libxml2-dev libxslt-dev xsltproc libkrb5-dev libldap2-dev \
+            libsystemd-dev gettext tcl-dev libperl-dev pkg-config clang-11 \
+            llvm-11 llvm-11-dev libselinux1-dev python3-dev \
+            uuid-dev liblz4-dev meson ninja-build  \
+            gpg wget libcurl4-openssl-dev libhttp-server-simple-perl
+          sudo /usr/bin/perl -MCPAN -e 'install IPC::Run'
+          sudo /usr/bin/perl -MCPAN -e 'install Text::Trim'
+          wget -O- https://apt.releases.hashicorp.com/gpg | sudo gpg --dearmor -o /usr/share/keyrings/hashicorp-archive-keyring.gpg
+          echo "deb [arch=$(dpkg --print-architecture) signed-by=/usr/share/keyrings/hashicorp-archive-keyring.gpg] https://apt.releases.hashicorp.com $(lsb_release -cs) main" | sudo tee /etc/apt/sources.list.d/hashicorp.list
+          sudo apt update && sudo apt install -y vault
+
+      - name: Clone postgres repository
+        uses: actions/checkout@v4
+        with:
+          repository: 'postgres/postgres'
+          ref: 'REL_17_STABLE'
+          path: 'src'
+
+      - name: Clone pg_tde repository
+        uses: actions/checkout@v4
+        with:
+          path: 'src/contrib/pg_tde'
+
+      - name: Include pg_tde in meson build
+        run: |
+          echo "subdir('pg_tde')" >> src/contrib/meson.build
+
+      - name: Build postgres
+        run: |
+          meson setup build --prefix `pwd`/../inst --buildtype=debug -Dcassert=true
+          cd build && ninja && ninja install
+        working-directory: src
+
+      - name: Test pg_tde
+        run: |
+          TV=$(mktemp)
+          { exec >$TV; vault server -dev; } &
+          sleep 10
+          export ROOT_TOKEN=$(cat $TV | grep "Root Token" | cut -d ":" -f 2 | xargs echo -n)
+          echo "Root token: $ROOT_TOKEN"
+
+          meson test --suite setup -v
+          meson test --suite pg_tde -v --num-processes 1
+        working-directory: src/build
+
+      - name: Report on test fail
+        uses: actions/upload-artifact@v4
+        if: ${{ failure() }}
+        with:
+          name: Regressions diff and postgresql log
+          path: |
+            src/build/testrun/pg_tde/regress/
+          retention-days: 3
diff --git a/src/access/pg_tde_slot.c b/src/access/pg_tde_slot.c
@@ -19,6 +19,7 @@
 #include "access/heaptoast.h"
 #include "access/htup_details.h"
 #include "access/tupdesc_details.h"
+#include "access/xact.h"
 #include "catalog/pg_type.h"
 #include "funcapi.h"
 #include "nodes/nodeFuncs.h"
@@ -119,6 +120,29 @@ tdeheap_tts_buffer_heap_getsysattr(TupleTableSlot *slot, int attnum, bool *isnul
 						slot->tts_tupleDescriptor, isnull);
 }
 
+static bool
+tdeheap_buffer_is_current_xact_tuple(TupleTableSlot *slot)
+{
+        BufferHeapTupleTableSlot *bslot = (BufferHeapTupleTableSlot *) slot;
+        TransactionId xmin;
+
+        Assert(!TTS_EMPTY(slot));
+
+        /*
+         * In some code paths it's possible to get here with a non-materialized
+         * slot, in which case we can't check if tuple is created by the current
+         * transaction.
+         */
+        if (!bslot->base.tuple)
+                ereport(ERROR,
+                                (errcode(ERRCODE_FEATURE_NOT_SUPPORTED),
+                                 errmsg("don't have a storage tuple in this context")));
+
+        xmin = HeapTupleHeaderGetRawXmin(bslot->base.tuple->t_data);
+
+        return TransactionIdIsCurrentTransactionId(xmin);
+}
+
 static void
 tdeheap_tts_buffer_heap_materialize(TupleTableSlot *slot)
 {
@@ -432,6 +456,9 @@ const TupleTableSlotOps TTSOpsTDEBufferHeapTuple = {
 	.getsomeattrs = tdeheap_tts_buffer_heap_getsomeattrs,
 	.getsysattr = tdeheap_tts_buffer_heap_getsysattr,
 	.materialize = tdeheap_tts_buffer_heap_materialize,
+#if PG_VERSION_NUM >= 170000
+	.is_current_xact_tuple = tdeheap_buffer_is_current_xact_tuple,
+#endif
 	.copyslot = tdeheap_tts_buffer_heap_copyslot,
 	.get_heap_tuple = tdeheap_tts_buffer_heap_get_heap_tuple,
 

diff --git a/src/catalog/tde_principal_key.c b/src/catalog/tde_principal_key.c
@@ -152,6 +152,9 @@ void initialize_objects_in_dsa_area(dsa_area *dsa, void *raw_dsa_area)
     sharedState->rawDsaArea = raw_dsa_area;
     sharedState->hashTrancheId = LWLockNewTrancheId();
     principal_key_dsh_params.tranche_id = sharedState->hashTrancheId;
+#if PG_VERSION_NUM >= 170000
+    principal_key_dsh_params.copy_function = dshash_memcpy;
+#endif
     dsh = dshash_create(dsa, &principal_key_dsh_params, 0);
     sharedState->hashHandle = dshash_get_hash_table_handle(dsh);
     dshash_detach(dsh);

diff --git a/src17/COMMIT b/src17/COMMIT
@@ -0,0 +1 @@
+84e40a3e113b2d74a655358d8791dc556579a241