Skip to content

Commit

Permalink
Initial solution
Browse files Browse the repository at this point in the history
  • Loading branch information
@bimboterminator1
bimboterminator1 committed Oct 15, 2024
1 parent fea5f63 commit 0e4ce97
Show file tree
Hide file tree
Showing 27 changed files with 3,950 additions and 101 deletions.
25 changes: 21 additions & 4 deletions gpcontrib/arenadata_toolkit/Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
MODULES = arenadata_toolkit

EXTENSION = arenadata_toolkit
EXTENSION_VERSION = 1.6
EXTENSION_VERSION = 1.7
DATA = \
arenadata_toolkit--1.0.sql \
arenadata_toolkit--1.0--1.1.sql \
Expand All @@ -12,15 +12,29 @@ DATA = \
arenadata_toolkit--1.3--1.4.sql \
arenadata_toolkit--1.4--1.5.sql \
arenadata_toolkit--1.5--1.6.sql \
arenadata_toolkit--1.6--1.7.sql \

DATA_built = $(EXTENSION)--$(EXTENSION_VERSION).sql

$(DATA_built): $(DATA)
cat $(DATA) > $(DATA_built)
MODULE_big = arenadata_toolkit
OBJS = \
src/arenadata_toolkit_guc.o \
src/bloom.o \
src/bloom_set.o \
src/drops_track.o \
src/file_hook.o \
src/tf_shmem.o \
src/arenadata_toolkit.o \
src/arenadata_toolkit_worker.o \
src/track_files.o \
src/dbsize.o \

PG_CFLAGS = -I$(libpq_srcdir) -I$(CURDIR)/src/include

REGRESS = arenadata_toolkit_test arenadata_toolkit_skew_test adb_get_relfilenodes_test \
adb_collect_table_stats_test adb_vacuum_strategy_test adb_relation_storage_size_test \
tablespace_location upgrade_test adb_hba_file_rules_view_test
tablespace_location upgrade_test adb_hba_file_rules_view_test \
arenadata_toolkit_guc arenadata_toolkit_tracking
REGRESS_OPTS += --init-file=$(top_srcdir)/src/test/regress/init_file

ifdef USE_PGXS
Expand All @@ -33,3 +47,6 @@ top_builddir = ../..
include $(top_builddir)/src/Makefile.global
include $(top_srcdir)/contrib/contrib-global.mk
endif

$(DATA_built): $(DATA)
cat $(DATA) > $(DATA_built)
98 changes: 98 additions & 0 deletions gpcontrib/arenadata_toolkit/arenadata_toolkit--1.6--1.7.sql
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,98 @@
/* gpcontrib/arenadata_toolkit/arenadata_toolkit--1.6--1.7.sql */

CREATE FUNCTION arenadata_toolkit.tracking_register_db(dbid OID default 0)
returns BOOL AS '$libdir/arenadata_toolkit',
'tracking_register_db' LANGUAGE C;

REVOKE ALL ON FUNCTION arenadata_toolkit.tracking_register_db(dbid OID) FROM public;

CREATE FUNCTION arenadata_toolkit.tracking_unregister_db(dbid OID default 0)
returns BOOL AS '$libdir/arenadata_toolkit',
'tracking_unregister_db' LANGUAGE C;

REVOKE ALL ON FUNCTION arenadata_toolkit.tracking_unregister_db(dbid OID) FROM public;

CREATE FUNCTION arenadata_toolkit.tracking_register_schema(schemaname NAME, dbid OID default 0)
returns BOOL AS '$libdir/arenadata_toolkit',
'tracking_register_schema' LANGUAGE C EXECUTE ON master;

REVOKE ALL ON FUNCTION arenadata_toolkit.tracking_register_schema(schema NAME, dbid OID) FROM public;

CREATE FUNCTION arenadata_toolkit.tracking_unregister_schema(schema NAME, dbid OID default 0)
returns BOOL AS '$libdir/arenadata_toolkit',
'tracking_unregister_schema' LANGUAGE C EXECUTE ON master;

REVOKE ALL ON FUNCTION arenadata_toolkit.tracking_unregister_schema(schema NAME, dbid OID) FROM public;

CREATE FUNCTION arenadata_toolkit.tracking_set_relkinds(relkinds NAME, dbid OID DEFAULT 0)
returns BOOL AS '$libdir/arenadata_toolkit',
'tracking_set_relkinds' LANGUAGE C EXECUTE ON master;

REVOKE ALL ON FUNCTION arenadata_toolkit.tracking_set_relkinds(relkinds NAME, dbid OID) FROM public;

CREATE FUNCTION arenadata_toolkit.tracking_set_relstorages(relstorages NAME, dbid OID DEFAULT 0)
returns BOOL AS '$libdir/arenadata_toolkit',
'tracking_set_relstorages' LANGUAGE C EXECUTE ON master;

REVOKE ALL ON FUNCTION arenadata_toolkit.tracking_set_relstorages(relstorages NAME, dbid OID) FROM public;

CREATE FUNCTION arenadata_toolkit.tracking_set_snapshot_on_recovery(val BOOL, dbid OID DEFAULT 0)
returns BOOL AS '$libdir/arenadata_toolkit',
'tracking_set_snapshot_on_recovery' LANGUAGE C EXECUTE ON master;

REVOKE ALL ON FUNCTION arenadata_toolkit.tracking_set_snapshot_on_recovery(val BOOL, dbid OID) FROM public;

CREATE FUNCTION arenadata_toolkit.tracking_trigger_initial_snapshot(dbid OID DEFAULT 0)
returns BOOL AS '$libdir/arenadata_toolkit',
'tracking_trigger_initial_snapshot' LANGUAGE C;

CREATE FUNCTION arenadata_toolkit.tracking_is_initial_snapshot_triggered(dbid OID DEFAULT 0)
returns BOOL AS '$libdir/arenadata_toolkit',
'tracking_is_initial_snapshot_triggered' LANGUAGE C;

REVOKE ALL ON FUNCTION arenadata_toolkit.tracking_is_initial_snapshot_triggered(dbid OID) FROM public;

CREATE FUNCTION arenadata_toolkit.tracking_is_initial_snapshot_triggered_master(dbid OID DEFAULT 0)
returns BOOL AS '$libdir/arenadata_toolkit',
'tracking_is_initial_snapshot_triggered' LANGUAGE C EXECUTE ON master;

CREATE FUNCTION arenadata_toolkit.tracking_is_initial_snapshot_triggered_segments(dbid OID DEFAULT 0)
returns BOOL AS '$libdir/arenadata_toolkit',
'tracking_is_initial_snapshot_triggered' LANGUAGE C EXECUTE ON ALL segments;

CREATE FUNCTION arenadata_toolkit.tracking_is_segment_initialized()
returns TABLE(segindex INT, is_initialized BOOL) AS '$libdir/arenadata_toolkit',
'tracking_is_segment_initialized' LANGUAGE C;

REVOKE ALL ON FUNCTION arenadata_toolkit.tracking_is_segment_initialized() FROM public;

CREATE FUNCTION arenadata_toolkit.tracking_get_track_main()
RETURNS TABLE(relid OID, relname NAME, relfilenode OID, size BIGINT, state "char", segid INT,
relnamespace OID, relkind "char", relstorage "char") AS '$libdir/arenadata_toolkit',
'tracking_get_track_main' LANGUAGE C;

CREATE FUNCTION arenadata_toolkit.tracking_get_track()
RETURNS TABLE(relid OID, relname NAME, relfilenode OID, size BIGINT, state "char", segid INT,
relnamespace OID, relkind "char", relstorage "char") AS '$libdir/arenadata_toolkit',
'tracking_get_track' LANGUAGE C EXECUTE ON master;

CREATE VIEW arenadata_toolkit.tables_track AS
SELECT t.*, coalesce(c.oid, i.indrelid, vm.relid, blk.relid, seg.relid) AS parent_relid
FROM arenadata_toolkit.tracking_get_track() AS t
LEFT JOIN pg_class AS c
ON c.reltoastrelid = t.relid AND t.relkind = 't'
LEFT JOIN pg_index AS i
ON i.indexrelid = t.relid AND t.relkind = 'i'
LEFT JOIN pg_catalog.pg_appendonly AS vm
ON vm.visimaprelid = t.relid AND t.relkind = 'M'
LEFT JOIN pg_catalog.pg_appendonly AS blk
ON blk.blkdirrelid = t.relid AND t.relkind = 'b'
LEFT JOIN pg_catalog.pg_appendonly AS seg
ON seg.segrelid = t.relid AND t.relkind = 'o';

CREATE VIEW arenadata_toolkit.is_initial_snapshot_triggered AS
SELECT CASE
WHEN TRUE = ALL(select arenadata_toolkit.tracking_is_initial_snapshot_triggered_segments())
AND
arenadata_toolkit.tracking_is_initial_snapshot_triggered_master()
THEN 1 ELSE NULL END AS is_triggered;
2 changes: 1 addition & 1 deletion gpcontrib/arenadata_toolkit/arenadata_toolkit.control
View file
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
# arenadata_toolkit extension
comment = 'extension is used for manipulation of objects created by adb-bundle'
default_version = '1.6'
default_version = '1.7'
module_pathname = '$libdir/arenadata_toolkit'
relocatable = false
233 changes: 233 additions & 0 deletions gpcontrib/arenadata_toolkit/expected/arenadata_toolkit_guc.out
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,233 @@
-- start_matchsubs
--
-- m/ERROR: \[arenadata_toolkit\] exceeded maximum number of tracked databases \(track_files\.c:\d+\)/
-- s/\d+/XXX/g
--
-- end_matchsubs
-- Test database registering GUC.
CREATE DATABASE tracking1;
\c tracking1;
CREATE EXTENSION arenadata_toolkit;
SHOW arenadata_toolkit.tracking_is_db_tracked;
arenadata_toolkit.tracking_is_db_tracked
------------------------------------------
off
(1 row)

SELECT datname, setconfig FROM pg_db_role_setting JOIN pg_database ON
setdatabase=oid WHERE datname=current_database();
datname | setconfig
---------+-----------
(0 rows)

\c -;
\c tracking1;
SELECT arenadata_toolkit.tracking_register_db();
tracking_register_db
----------------------
t
(1 row)

SHOW arenadata_toolkit.tracking_is_db_tracked;
arenadata_toolkit.tracking_is_db_tracked
------------------------------------------
off
(1 row)

SELECT datname, setconfig FROM pg_db_role_setting JOIN pg_database ON
setdatabase=oid WHERE datname=current_database();
datname | setconfig
-----------+----------------------------------------------
tracking1 | {arenadata_toolkit.tracking_is_db_tracked=t}
(1 row)

SELECT arenadata_toolkit.tracking_unregister_db();
tracking_unregister_db
------------------------
t
(1 row)

\c -;
\c tracking1;
SHOW arenadata_toolkit.tracking_is_db_tracked;
arenadata_toolkit.tracking_is_db_tracked
------------------------------------------
off
(1 row)

SELECT datname, setconfig FROM pg_db_role_setting JOIN pg_database ON
setdatabase=oid WHERE datname=current_database();
datname | setconfig
-----------+----------------------------------------------
tracking1 | {arenadata_toolkit.tracking_is_db_tracked=f}
(1 row)

-- Prohibit manual GUC setting.
SET arenadata_toolkit.tracking_is_db_tracked = true;
ERROR: cannot change tracking status outside the tracking_register_db function
ALTER DATABASE tracking1 SET arenadata_toolkit.tracking_is_db_tracked = true;
ERROR: cannot change tracking status outside the tracking_register_db function
-- Test limit of tracking databases.
SHOW arenadata_toolkit.tracking_db_track_count;
arenadata_toolkit.tracking_db_track_count
-------------------------------------------
5
(1 row)

CREATE DATABASE tracking2;
CREATE DATABASE tracking3;
CREATE DATABASE tracking4;
CREATE DATABASE tracking5;
CREATE DATABASE tracking6;
SELECT arenadata_toolkit.tracking_register_db(oid) FROM pg_database WHERE datname IN
('tracking1', 'tracking2', 'tracking3', 'tracking4', 'tracking5');
ERROR: [arenadata_toolkit] exceeded maximum number of tracked databases (track_files.c:XXX)
SELECT arenadata_toolkit.tracking_register_db(oid) FROM pg_database WHERE datname IN
('tracking6');
ERROR: [arenadata_toolkit] exceeded maximum number of tracked databases (track_files.c:XXX)
SELECT arenadata_toolkit.tracking_unregister_db(oid) FROM pg_database WHERE datname IN
('tracking1', 'tracking2', 'tracking3', 'tracking4', 'tracking5', 'tracking6');
tracking_unregister_db
------------------------
t
t
t
t
t
t
(6 rows)

DROP DATABASE IF EXISTS tracking2;
DROP DATABASE IF EXISTS tracking3;
DROP DATABASE IF EXISTS tracking4;
DROP DATABASE IF EXISTS tracking5;
DROP DATABASE IF EXISTS tracking6;
-- Test arenadata_toolkit.tracking_snapshot_on_recovery GUC
SELECT arenadata_toolkit.tracking_set_snapshot_on_recovery(true);
tracking_set_snapshot_on_recovery
-----------------------------------
t
(1 row)

SELECT datname, setconfig FROM pg_db_role_setting JOIN pg_database ON
setdatabase=oid WHERE datname=current_database();
datname | setconfig
-----------+------------------------------------------------------------------------------------------------
tracking1 | {arenadata_toolkit.tracking_is_db_tracked=f,arenadata_toolkit.tracking_snapshot_on_recovery=t}
(1 row)

-- Prohibit manual GUC setting.
SET arenadata_toolkit.tracking_set_snapshot_on_recovery = false;
ALTER DATABASE tracking1 SET arenadata_toolkit.tracking_set_snapshot_on_recovery = false;
-- Test arenadata_toolkit.tracking_relstorages GUC
SELECT arenadata_toolkit.tracking_set_relstorages('f,a,x');
tracking_set_relstorages
--------------------------
t
(1 row)

SELECT datname, setconfig FROM pg_db_role_setting JOIN pg_database ON
setdatabase=oid WHERE datname=current_database();
datname | setconfig
-----------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
tracking1 | {arenadata_toolkit.tracking_is_db_tracked=f,arenadata_toolkit.tracking_snapshot_on_recovery=t,arenadata_toolkit.tracking_set_snapshot_on_recovery=false,"arenadata_toolkit.tracking_relstorages=f,a,x"}
(1 row)

SELECT arenadata_toolkit.tracking_set_relstorages('v,v,v,,,');
tracking_set_relstorages
--------------------------
t
(1 row)

SELECT datname, setconfig FROM pg_db_role_setting JOIN pg_database ON
setdatabase=oid WHERE datname=current_database();
datname | setconfig
-----------+---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
tracking1 | {arenadata_toolkit.tracking_is_db_tracked=f,arenadata_toolkit.tracking_snapshot_on_recovery=t,arenadata_toolkit.tracking_set_snapshot_on_recovery=false,arenadata_toolkit.tracking_relstorages=v}
(1 row)

SELECT arenadata_toolkit.tracking_set_relstorages('d,b,c');
ERROR: Invalid relstorage type: d
HINT: Valid relstorages are: 'h', 'x', 'a', 'v', 'c', 'f'
-- Prohibit manual GUC setting.
SET arenadata_toolkit.tracking_relstorages = "h, a, x";
ERROR: cannot change tracking status outside the tracking_register_relstorages function
ALTER DATABASE tracking1 SET arenadata_toolkit.tracking_relstorages = "h, a, x";
ERROR: cannot change tracking status outside the tracking_register_relstorages function
-- Test arenadata_toolkit.tracking_relkinds GUC
SELECT arenadata_toolkit.tracking_set_relkinds('r,t,o,S');
tracking_set_relkinds
-----------------------
t
(1 row)

SELECT datname, setconfig FROM pg_db_role_setting JOIN pg_database ON
setdatabase=oid WHERE datname=current_database();
datname | setconfig
-----------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
tracking1 | {arenadata_toolkit.tracking_is_db_tracked=f,arenadata_toolkit.tracking_snapshot_on_recovery=t,arenadata_toolkit.tracking_set_snapshot_on_recovery=false,arenadata_toolkit.tracking_relstorages=v,"arenadata_toolkit.tracking_relkinds=r,t,o,S"}
(1 row)

SELECT arenadata_toolkit.tracking_set_relkinds('m,M,o,,,');
tracking_set_relkinds
-----------------------
t
(1 row)

SELECT datname, setconfig FROM pg_db_role_setting JOIN pg_database ON
setdatabase=oid WHERE datname=current_database();
datname | setconfig
-----------+-----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
tracking1 | {arenadata_toolkit.tracking_is_db_tracked=f,arenadata_toolkit.tracking_snapshot_on_recovery=t,arenadata_toolkit.tracking_set_snapshot_on_recovery=false,arenadata_toolkit.tracking_relstorages=v,"arenadata_toolkit.tracking_relkinds=m,M,o"}
(1 row)

SELECT arenadata_toolkit.tracking_set_relkinds('d,b,c');
ERROR: Invalid relkind: d
HINT: Valid relkinds are: 'r', 'i', 'S', 't', 'v', 'c', 'f', 'u', 'm', 'o', 'b', 'M'
-- Prohibit manual GUC setting.
SET arenadata_toolkit.tracking_relkinds = "h, a, x";
ERROR: cannot change tracking status outside the tracking_register_relkinds function
ALTER DATABASE tracking1 SET arenadata_toolkit.tracking_relkinds = "h, a, x";
ERROR: cannot change tracking status outside the tracking_register_relkinds function
-- Test arenadata_toolkit.tracking_schemas GUC
SELECT arenadata_toolkit.tracking_register_schema('arenadata_toolkit');
tracking_register_schema
--------------------------
t
(1 row)

SELECT arenadata_toolkit.tracking_register_schema('public');
tracking_register_schema
--------------------------
t
(1 row)

SELECT datname, setconfig FROM pg_db_role_setting JOIN pg_database ON
setdatabase=oid WHERE datname=current_database();
datname | setconfig
-----------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
tracking1 | {arenadata_toolkit.tracking_is_db_tracked=f,arenadata_toolkit.tracking_snapshot_on_recovery=t,arenadata_toolkit.tracking_set_snapshot_on_recovery=false,arenadata_toolkit.tracking_relstorages=v,"arenadata_toolkit.tracking_relkinds=m,M,o","arenadata_toolkit.tracking_schemas=arenadata_toolkit,public"}
(1 row)

SELECT arenadata_toolkit.tracking_unregister_schema('public');
tracking_unregister_schema
----------------------------
t
(1 row)

SELECT datname, setconfig FROM pg_db_role_setting JOIN pg_database ON
setdatabase=oid WHERE datname=current_database();
datname | setconfig
-----------+----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------
tracking1 | {arenadata_toolkit.tracking_is_db_tracked=f,arenadata_toolkit.tracking_snapshot_on_recovery=t,arenadata_toolkit.tracking_set_snapshot_on_recovery=false,arenadata_toolkit.tracking_relstorages=v,"arenadata_toolkit.tracking_relkinds=m,M,o",arenadata_toolkit.tracking_schemas=arenadata_toolkit}
(1 row)

SELECT arenadata_toolkit.tracking_register_schema('pg_pg');
ERROR: schema pg_pg does not exist
-- Prohibit manual GUC setting.
SET arenadata_toolkit.tracking_schemas = "pg_catalog, mychema";
ERROR: cannot change tracking status outside the tracking_register_schema function
ALTER DATABASE tracking1 SET arenadata_toolkit.tracking_schemas = "pg_catalog, mychema";
ERROR: cannot change tracking status outside the tracking_register_schema function
\c contrib_regression;
DROP DATABASE tracking1;
Loading

0 comments on commit 0e4ce97

Please sign in to comment.