ADBDEV-6339 Implement files tracking for arenadata_toolkit

gpcontrib/arenadata_toolkit/Makefile

@@ -3,7 +3,7 @@
 MODULES = arenadata_toolkit
 
 EXTENSION = arenadata_toolkit
-EXTENSION_VERSION = 1.6
+EXTENSION_VERSION = 1.7
 DATA = \
        arenadata_toolkit--1.0.sql \
        arenadata_toolkit--1.0--1.1.sql \
@@ -12,15 +12,29 @@ DATA = \
        arenadata_toolkit--1.3--1.4.sql \
        arenadata_toolkit--1.4--1.5.sql \
        arenadata_toolkit--1.5--1.6.sql \
+       arenadata_toolkit--1.6--1.7.sql \
 
 DATA_built = $(EXTENSION)--$(EXTENSION_VERSION).sql
 
-$(DATA_built): $(DATA)
-	cat $(DATA) > $(DATA_built)
+MODULE_big = arenadata_toolkit
+OBJS = \
+	src/arenadata_toolkit_guc.o \
+	src/bloom.o \
+	src/bloom_set.o \
+	src/drops_track.o \
+	src/file_hook.o \
+	src/tf_shmem.o \
+	src/arenadata_toolkit.o \
+	src/arenadata_toolkit_worker.o \
+	src/track_files.o \
+	src/dbsize.o \
+
+PG_CFLAGS = -I$(libpq_srcdir) -I$(CURDIR)/src/include
 
 REGRESS = arenadata_toolkit_test arenadata_toolkit_skew_test adb_get_relfilenodes_test \
           adb_collect_table_stats_test adb_vacuum_strategy_test adb_relation_storage_size_test \
-          tablespace_location upgrade_test adb_hba_file_rules_view_test
+          tablespace_location upgrade_test adb_hba_file_rules_view_test \
+          arenadata_toolkit_guc arenadata_toolkit_tracking
 REGRESS_OPTS += --init-file=$(top_srcdir)/src/test/regress/init_file
 
 ifdef USE_PGXS
@@ -33,3 +47,6 @@ top_builddir = ../..
 include $(top_builddir)/src/Makefile.global
 include $(top_srcdir)/contrib/contrib-global.mk
 endif
+
+$(DATA_built): $(DATA)
+	cat $(DATA) > $(DATA_built)

gpcontrib/arenadata_toolkit/README.md

@@ -0,0 +1,59 @@
+## arenadata_toolkit - database objects tracking extension for GPDB
+
+`arenadata_toolkit` starting from version 1.7 represents a GPDB extension that efficiently tracks file system changes (extend, truncate, create, unlink operations) using space-efficient Bloom filters stored in shared memory. This extension is particularly useful for monitoring and maintaining database files sizes across a distributed environment.
+
+The main purpose of this code is achieving fast database size calculation and tracking file changes at relation
+level. The extension implements a probabilistic tracking system using Bloom filters to monitor file changes across Greenplum segments. It utilizes shared memory for state management and employs background workers to maintain consistency. 
+
+#### Configuring GPDB and extension usage
+Since extension uses shared memory, configuration on all GPDB segments must be changed by setting
+```shell script
+gpconfig -c shared_preload_libraries -v 'arenadata_toolkit'
+```
+Extension may track restricted number of databases. The maximum number of them is defined by GUC
+| arenadata_toolkit.tracking_db_track_count | Need restart |Possible values [1, 1000]; Default 5|
+|--|--|--|
+For each tracked database there allocated a Bloom filter in shared memory. The size of each filter is controlled via
+| arenadata_toolkit.tracking_bloom_size | Need restart |Possible values (bytes) [64, 128000000] Default 1048576|
+|--|--|--|
+The specific database can be bound to unoccupied filter with function
+```shell script
+psql -d my_db -c select arenadata_toolkit.tracking_register_db()
+or
+psql -c select arenadata_toolkit.tracking_register_db(12345)
+```
+After registering each relation file change within the database will be noted in Bloom filter.
+Using Bloom filter allows us to calculate the sizes of only relations whose relfilenode is present in the filter.
+The current size snapshot can be taken via view:
+```
+select * from arenadata_toolkit.tables_track;
+```
+In order to get the snapshot of all database relations you should call in the database of interest
+```
+arenadata_toolkit.tracking_trigger_initial_snapshot();
+```
+
+#### Choosing optimal Bloom size
+
+Choosing the optimal Bloom filter size is crucial for balancing memory usage and accuracy.
+First of all, when choosing the filter size, you should take into account your system resources, because bloom filters are allocated in shared memory for each segment, and too wide structures (tracking_db_track_count * tracking_bloom_size) could decrease overall performance.
+
+Next, choose the filter size satisfying your performance goals:
+- Define false positive tolerance, p. Since Bloom filter is probabilistic data structure there is a probability to calculate the size of relation, which has not been modified. And the smaller filter is, the more often this occurs.
+- Memory constraints 
+- Query patterns, if queries are mostly reading then huge sizes are unnecessary.
+
+If you will estimate number of objects in your database, you can calculate theoretical size:
+$$m = -\frac{n \ln p}{(\ln 2)^2}$$
+- n = estimated number of elements
+- p = target false positive rate
+- m = filter size in bits
+
+Quick Reference Table
+
+ Deployment Size | Files      | Target FPR | Recommended Size, bytes|
+|----------------|------------|------------|------------------------|
+| Small          | < 100K     | 1%         | 1048576                |
+| Medium         | 100K - 1M  | 1%         | 8388608                |
+| Large          | > 1M       | 1%         | 33554432               |
+| Enterprise     | > 10M      | 1%         | 134217728              |

gpcontrib/arenadata_toolkit/arenadata_toolkit--1.6--1.7.sql

@@ -0,0 +1,127 @@
+/* gpcontrib/arenadata_toolkit/arenadata_toolkit--1.6--1.7.sql */
+
+CREATE FUNCTION arenadata_toolkit.tracking_register_db(dbid OID DEFAULT 0)
+returns BOOL AS '$libdir/arenadata_toolkit',
+'tracking_register_db' LANGUAGE C EXECUTE ON MASTER;
+
+REVOKE ALL ON FUNCTION arenadata_toolkit.tracking_register_db(dbid OID) FROM public;
+
+CREATE FUNCTION arenadata_toolkit.tracking_register_db_main(reg BOOL, dbid OID DEFAULT 0)
+returns BOOL AS '$libdir/arenadata_toolkit',
+'tracking_register_db_main' LANGUAGE C;
+
+REVOKE ALL ON FUNCTION arenadata_toolkit.tracking_register_db_main(reg BOOL, dbid OID) FROM public;
+
+CREATE FUNCTION arenadata_toolkit.tracking_unregister_db(dbid OID DEFAULT 0)
+returns BOOL AS '$libdir/arenadata_toolkit',
+'tracking_unregister_db' LANGUAGE C EXECUTE ON MASTER;
+
+REVOKE ALL ON FUNCTION arenadata_toolkit.tracking_unregister_db(dbid OID) FROM public;
+
+CREATE FUNCTION arenadata_toolkit.tracking_register_schema(schemaname NAME, dbid OID DEFAULT 0)
+returns BOOL AS '$libdir/arenadata_toolkit',
+'tracking_register_schema' LANGUAGE C EXECUTE ON master;
+
+REVOKE ALL ON FUNCTION arenadata_toolkit.tracking_register_schema(schema NAME, dbid OID) FROM public;
+
+CREATE FUNCTION arenadata_toolkit.tracking_unregister_schema(schema NAME, dbid OID DEFAULT 0)
+returns BOOL AS '$libdir/arenadata_toolkit',
+'tracking_unregister_schema' LANGUAGE C EXECUTE ON master;
+
+REVOKE ALL ON FUNCTION arenadata_toolkit.tracking_unregister_schema(schema NAME, dbid OID) FROM public;
+
+CREATE FUNCTION arenadata_toolkit.tracking_set_relkinds(relkinds NAME, dbid OID DEFAULT 0)
+returns BOOL AS '$libdir/arenadata_toolkit',
+'tracking_set_relkinds' LANGUAGE C EXECUTE ON master;
+
+REVOKE ALL ON FUNCTION arenadata_toolkit.tracking_set_relkinds(relkinds NAME, dbid OID) FROM public;
+
+CREATE FUNCTION arenadata_toolkit.tracking_set_relstorages(relstorages NAME, dbid OID DEFAULT 0)
+returns BOOL AS '$libdir/arenadata_toolkit',
+'tracking_set_relstorages' LANGUAGE C EXECUTE ON master;
+
+REVOKE ALL ON FUNCTION arenadata_toolkit.tracking_set_relstorages(relstorages NAME, dbid OID) FROM public;
+
+CREATE FUNCTION arenadata_toolkit.tracking_set_snapshot_on_recovery(val BOOL, dbid OID DEFAULT 0)
+returns BOOL AS '$libdir/arenadata_toolkit',
+'tracking_set_snapshot_on_recovery' LANGUAGE C EXECUTE ON master;
+
+REVOKE ALL ON FUNCTION arenadata_toolkit.tracking_set_snapshot_on_recovery(val BOOL, dbid OID) FROM public;
+
+CREATE FUNCTION arenadata_toolkit.tracking_trigger_initial_snapshot(dbid OID DEFAULT 0)
+returns BOOL AS '$libdir/arenadata_toolkit',
+'tracking_trigger_initial_snapshot' LANGUAGE C;
+
+CREATE FUNCTION arenadata_toolkit.tracking_is_initial_snapshot_triggered(dbid OID DEFAULT 0)
+returns BOOL AS '$libdir/arenadata_toolkit',
+'tracking_is_initial_snapshot_triggered' LANGUAGE C;
+
+REVOKE ALL ON FUNCTION arenadata_toolkit.tracking_is_initial_snapshot_triggered(dbid OID) FROM public;
+
+CREATE FUNCTION arenadata_toolkit.tracking_is_initial_snapshot_triggered_master(dbid OID DEFAULT 0)
+returns BOOL AS '$libdir/arenadata_toolkit',
+'tracking_is_initial_snapshot_triggered' LANGUAGE C EXECUTE ON master;
+
+CREATE FUNCTION arenadata_toolkit.tracking_is_initial_snapshot_triggered_segments(dbid OID DEFAULT 0)
+returns BOOL AS '$libdir/arenadata_toolkit',
+'tracking_is_initial_snapshot_triggered' LANGUAGE C EXECUTE ON ALL segments;
+
+CREATE FUNCTION arenadata_toolkit.tracking_is_segment_initialized()
+returns TABLE(segindex INT, is_initialized BOOL) AS '$libdir/arenadata_toolkit',
+'tracking_is_segment_initialized' LANGUAGE C;
+
+REVOKE ALL ON FUNCTION arenadata_toolkit.tracking_is_segment_initialized() FROM public;
+
+CREATE FUNCTION arenadata_toolkit.tracking_track_version()
+returns BIGINT AS '$libdir/arenadata_toolkit',
+'tracking_track_version' LANGUAGE C STABLE EXECUTE ON MASTER;
+
+REVOKE ALL ON FUNCTION arenadata_toolkit.tracking_track_version() FROM public;
+
+CREATE FUNCTION arenadata_toolkit.tracking_get_track_master(version BIGINT)
+RETURNS TABLE(relid OID, relname NAME, relfilenode OID, size BIGINT, state "char", segid INT,
+relnamespace OID, relkind "char", relstorage "char") AS '$libdir/arenadata_toolkit',
+'tracking_get_track' LANGUAGE C EXECUTE ON MASTER;
+
+REVOKE ALL ON FUNCTION arenadata_toolkit.tracking_get_track_master(version BIGINT) FROM public;
+
+CREATE FUNCTION arenadata_toolkit.tracking_get_track_segments(version BIGINT)
+RETURNS TABLE(relid OID, relname NAME, relfilenode OID, size BIGINT, state "char", segid INT,
+relnamespace OID, relkind "char", relstorage "char") AS '$libdir/arenadata_toolkit',
+'tracking_get_track' LANGUAGE C EXECUTE ON ALL SEGMENTS;
+
+REVOKE ALL ON FUNCTION arenadata_toolkit.tracking_get_track_segments(version BIGINT) FROM public;
+
+CREATE VIEW arenadata_toolkit.tables_track AS
+SELECT t.*, coalesce(c.oid, i.indrelid, vm.relid, blk.relid, seg.relid) AS parent_relid
+FROM arenadata_toolkit.tracking_get_track_master(arenadata_toolkit.tracking_track_version()) AS t
+LEFT JOIN pg_class AS c
+    ON c.reltoastrelid = t.relid AND t.relkind = 't'
+LEFT JOIN pg_index AS i
+    ON i.indexrelid = t.relid AND t.relkind = 'i'
+LEFT JOIN pg_catalog.pg_appendonly AS vm
+    ON vm.visimaprelid = t.relid AND t.relkind = 'M'
+LEFT JOIN pg_catalog.pg_appendonly AS blk
+    ON blk.blkdirrelid = t.relid AND t.relkind = 'b'
+LEFT JOIN pg_catalog.pg_appendonly AS seg
+    ON seg.segrelid = t.relid AND t.relkind = 'o'
+UNION ALL
+SELECT t.*, coalesce(c.oid, i.indrelid, vm.relid, blk.relid, seg.relid) AS parent_relid
+FROM arenadata_toolkit.tracking_get_track_segments(arenadata_toolkit.tracking_track_version()) AS t
+LEFT JOIN pg_class AS c
+    ON c.reltoastrelid = t.relid AND t.relkind = 't'
+LEFT JOIN pg_index AS i
+    ON i.indexrelid = t.relid AND t.relkind = 'i'
+LEFT JOIN pg_catalog.pg_appendonly AS vm
+    ON vm.visimaprelid = t.relid AND t.relkind = 'M'
+LEFT JOIN pg_catalog.pg_appendonly AS blk
+    ON blk.blkdirrelid = t.relid AND t.relkind = 'b'
+LEFT JOIN pg_catalog.pg_appendonly AS seg
+    ON seg.segrelid = t.relid AND t.relkind = 'o';
+
+CREATE VIEW arenadata_toolkit.is_initial_snapshot_triggered AS
+SELECT CASE 
+	WHEN TRUE = ALL(select arenadata_toolkit.tracking_is_initial_snapshot_triggered_segments()) 
+	AND
+	arenadata_toolkit.tracking_is_initial_snapshot_triggered_master() 
+	THEN 1 ELSE NULL END AS is_triggered;

gpcontrib/arenadata_toolkit/arenadata_toolkit.control

@@ -1,5 +1,5 @@
 # arenadata_toolkit extension
 comment = 'extension is used for manipulation of objects created by adb-bundle'
-default_version = '1.6'
+default_version = '1.7'
 module_pathname = '$libdir/arenadata_toolkit'
 relocatable = false