Merge pull request #247 from dutow/pg16move

Heap clone refactoring / cleanup

.github/workflows/postgresql-16-pgdg-package-pgxs.yml

@@ -56,7 +56,7 @@ jobs:
         run: |
           sudo -u postgres bash -c './configure'
           sudo -u postgres bash -c 'make USE_PGXS=1'
-          sudo make USE_PGXS=1 install
+          sudo make USE_PGXS=1 MAJORVERSION=16 install
         working-directory: src/pg_tde
 
       - name: Start pg_tde tests

.github/workflows/postgresql-16-src-make-macos.yml

@@ -39,7 +39,7 @@ jobs:
       - name: Build pg_tde
         run: |
           ./configure
-          make -j
+          make -j MAJORVERSION=16
           sudo make install
         working-directory: src/contrib/pg_tde
 

.github/workflows/postgresql-16-src-make-ssl11.yml

@@ -58,7 +58,7 @@ jobs:
       - name: Build pg_tde
         run: |
           ./configure
-          make -j
+          make -j MAJORVERSION=16
           sudo make install
         working-directory: src/contrib/pg_tde
 

.github/workflows/postgresql-16-src-make.yml

@@ -58,7 +58,7 @@ jobs:
       - name: Build pg_tde
         run: |
           ./configure
-          make -j
+          make -j MAJORVERSION=16
           sudo make install
         working-directory: src/contrib/pg_tde
 

.gitignore

@@ -1,9 +1,10 @@
 *.so
 *.o
+__pycache__
 
 /config.cache
 /config.log
 /config.status
 /Makefile
 /autom4te.cache
-/configure~
+/configure~

Makefile.in

@@ -24,16 +24,16 @@ TAP_TESTS = 1
 OBJS = src/encryption/enc_tde.o \
 src/encryption/enc_aes.o \
 src/access/pg_tde_slot.o \
-src/access/pg_tde_io.o \
-src/access/pg_tdeam_visibility.o \
 src/access/pg_tde_tdemap.o \
-src/access/pg_tdeam.o \
-src/access/pg_tdetoast.o \
-src/access/pg_tde_prune.o \
-src/access/pg_tde_vacuumlazy.o \
-src/access/pg_tde_visibilitymap.o \
-src/access/pg_tde_rewrite.o \
-src/access/pg_tdeam_handler.o \
+src$(MAJORVERSION)/access/pg_tde_io.o \
+src$(MAJORVERSION)/access/pg_tdeam_visibility.o \
+src$(MAJORVERSION)/access/pg_tdeam.o \
+src$(MAJORVERSION)/access/pg_tdetoast.o \
+src$(MAJORVERSION)/access/pg_tde_prune.o \
+src$(MAJORVERSION)/access/pg_tde_vacuumlazy.o \
+src$(MAJORVERSION)/access/pg_tde_visibilitymap.o \
+src$(MAJORVERSION)/access/pg_tde_rewrite.o \
+src$(MAJORVERSION)/access/pg_tdeam_handler.o \
 src/access/pg_tde_ddl.o \
 src/access/pg_tde_xlog.o \
 src/transam/pg_tde_xact_handler.o \
@@ -57,12 +57,12 @@ override PG_CPPFLAGS += @tde_CPPFLAGS@
 ifdef USE_PGXS
 PG_CONFIG = pg_config
 PGXS := $(shell $(PG_CONFIG) --pgxs)
-override PG_CPPFLAGS += -I$(CURDIR)/src/include
+override PG_CPPFLAGS += -I$(CURDIR)/src/include -I$(CURDIR)/src$(MAJORVERSION)/include
 include $(PGXS)
 else
 subdir = contrib/pg_tde
 top_builddir = ../..
-override PG_CPPFLAGS += -I$(top_srcdir)/$(subdir)/src/include
+override PG_CPPFLAGS += -I$(top_srcdir)/$(subdir)/src/include -I$(top_srcdir)/$(subdir)/src$(MAJORVERSION)/include
 include $(top_builddir)/src/Makefile.global
 include $(top_srcdir)/contrib/contrib-global.mk
 endif

docker/Dockerfile

@@ -14,7 +14,7 @@ WORKDIR /opt/pg_tde
 COPY . .
 
 RUN ./configure && \
-    make USE_PGXS=1 && \
+    make USE_PGXS=1 MAJORVERSION=16 && \
     make USE_PGXS=1 install
 RUN cp /usr/share/postgresql/postgresql.conf.sample /etc/postgresql/postgresql.conf; \
     echo "shared_preload_libraries = 'pg_tde'" >> /etc/postgresql/postgresql.conf; \

meson.build

@@ -13,20 +13,23 @@ conf_data.set_quoted('PACKAGE_TARNAME', 'pg_tde')
 configure_file(output : 'config.h',
                configuration : conf_data)
 
+pg_version = meson.project_version().substring(0,2)
+src_version = 'src' + pg_version
+
 pg_tde_sources = files(
         'src/pg_tde.c',
         'src/transam/pg_tde_xact_handler.c',
         'src/access/pg_tde_tdemap.c',
         'src/access/pg_tde_slot.c',
-        'src/access/pg_tdeam.c',
-        'src/access/pg_tdeam_handler.c',
-        'src/access/pg_tdeam_visibility.c',
-        'src/access/pg_tdetoast.c',
-        'src/access/pg_tde_io.c',
-        'src/access/pg_tde_prune.c',
-        'src/access/pg_tde_rewrite.c',
-        'src/access/pg_tde_vacuumlazy.c',
-        'src/access/pg_tde_visibilitymap.c',
+        src_version / 'access/pg_tdeam.c',
+        src_version / 'access/pg_tdeam_handler.c',
+        src_version / 'access/pg_tdeam_visibility.c',
+        src_version / 'access/pg_tdetoast.c',
+        src_version / 'access/pg_tde_io.c',
+        src_version / 'access/pg_tde_prune.c',
+        src_version / 'access/pg_tde_rewrite.c',
+        src_version / 'access/pg_tde_vacuumlazy.c',
+        src_version / 'access/pg_tde_visibilitymap.c',
         'src/access/pg_tde_ddl.c',
         'src/access/pg_tde_xlog.c',
 
@@ -51,7 +54,7 @@ pg_tde_sources = files(
         'src/pg_tde_event_capture.c',
 )
 
-incdir = include_directories('src/include', '.')
+incdir = include_directories(src_version / 'include', 'src/include', '.')
 
 deps_update = {'dependencies': contrib_mod_args.get('dependencies') + [curldep]}
 

src16/COMMIT

@@ -0,0 +1 @@
+f199436c12819d2c01b72eaa6429de0ca5838471

src/access/pg_tde_prune.c → src16/access/pg_tde_prune.c

@@ -127,7 +127,6 @@ tdeheap_page_prune_opt(Relation relation, Buffer buffer)
 	if (RecoveryInProgress())
 		return;
 
-#if PG_VERSION_NUM < 170000
 	/*
 	 * XXX: Magic to keep old_snapshot_threshold tests appear "working". They
 	 * currently are broken, and discussion of what to do about them is
@@ -136,7 +135,7 @@ tdeheap_page_prune_opt(Relation relation, Buffer buffer)
 	 */
 	if (old_snapshot_threshold == 0)
 		SnapshotTooOldMagicForTest();
-#endif
+
 	/*
 	 * First check whether there's any chance there's something to prune,
 	 * determining the appropriate horizon is a waste if there's no prune_xid
@@ -167,14 +166,14 @@ tdeheap_page_prune_opt(Relation relation, Buffer buffer)
 
 	if (!GlobalVisTestIsRemovableXid(vistest, prune_xid))
 	{
-#if PG_VERSION_NUM < 170000
-		if ( !OldSnapshotThresholdActive())
+		if (!OldSnapshotThresholdActive())
 			return;
+
 		if (!TransactionIdLimitedForOldSnapshots(GlobalVisTestNonRemovableHorizon(vistest),
 												 relation,
 												 &limited_xmin, &limited_ts))
 			return;
-#endif
+
 		if (!TransactionIdPrecedes(prune_xid, limited_xmin))
 			return;
 	}
@@ -540,7 +539,6 @@ tdeheap_prune_satisfies_vacuum(PruneState *prstate, HeapTuple tup, Buffer buffer
 	 */
 	if (GlobalVisTestIsRemovableXid(prstate->vistest, dead_after))
 		res = HEAPTUPLE_DEAD;
-#if PG_VERSION_NUM < 170000
 	else if (OldSnapshotThresholdActive())
 	{
 		/* haven't determined limited horizon yet, requests */
@@ -568,7 +566,7 @@ tdeheap_prune_satisfies_vacuum(PruneState *prstate, HeapTuple tup, Buffer buffer
 			res = HEAPTUPLE_DEAD;
 		}
 	}
-#endif
+
 	return res;
 }
 
@@ -1228,6 +1226,7 @@ tdeheap_get_root_tuples(Page page, OffsetNumber *root_offsets)
 	}
 }
 
+// TODO: move to own file so it can be autoupdated
 // FROM src/page/bufpage.c
 
 /*

src/access/pg_tde_rewrite.c → src16/access/pg_tde_rewrite.c

@@ -676,7 +676,7 @@ raw_tdeheap_insert(RewriteState state, HeapTuple tup)
 		{
 			/*
 			 * Doesn't fit, so write out the existing page.  It always
-			 * contains a tuple.  Hence, unlike RelationGetBufferForTuple(),
+			 * contains a tuple.  Hence, unlike tdeheap_RelationGetBufferForTuple(),
 			 * enforce saveFreeSpace unconditionally.
 			 */
 
@@ -731,8 +731,6 @@ raw_tdeheap_insert(RewriteState state, HeapTuple tup)
 		newitemid = PageGetItemId(page, newoff);
 		onpage_tup = (HeapTupleHeader) PageGetItem(page, newitemid);
 
-		// TODO: decrypt/encrypt
-
 		onpage_tup->t_ctid = tup->t_self;
 	}
 

src/access/pg_tde_vacuumlazy.c → src16/access/pg_tde_vacuumlazy.c

@@ -1421,7 +1421,7 @@ lazy_scan_new_or_empty(LVRelState *vacrel, Buffer buf, BlockNumber blkno,
 		 * (which creates a number of empty pages at the tail end of the
 		 * relation), and then enters them into the FSM.
 		 *
-		 * Note we do not enter the page into the tdeheap_visibilitymap. That has the
+		 * Note we do not enter the page into the visibilitymap. That has the
 		 * downside that we repeatedly visit this page in subsequent vacuums,
 		 * but otherwise we'll never discover the space on a promoted standby.
 		 * The harm of repeated checking ought to normally not be too bad. The
@@ -2014,7 +2014,6 @@ lazy_scan_noprune(LVRelState *vacrel,
 
 		*hastup = true;			/* page prevents rel truncation */
 		tupleheader = (HeapTupleHeader) PageGetItem(page, itemid);
-		// TODO: decrypt
 		if  (tdeheap_tuple_should_freeze(tupleheader, &vacrel->cutoffs,
 									 &NoFreezePageRelfrozenXid,
 									 &NoFreezePageRelminMxid))
@@ -2822,11 +2821,8 @@ should_attempt_truncation(LVRelState *vacrel)
 {
 	BlockNumber possibly_freeable;
 
-	if (!vacrel->do_rel_truncate || VacuumFailsafeActive
-#if PG_VERSION_NUM < 170000
-	 || old_snapshot_threshold >= 0
-#endif
-		)
+	if (!vacrel->do_rel_truncate || VacuumFailsafeActive ||
+		old_snapshot_threshold >= 0)
 		return false;
 
 	possibly_freeable = vacrel->rel_pages - vacrel->nonempty_pages;