From f7b0ca1eba2c55746a3685b5730df4ace56ee535 Mon Sep 17 00:00:00 2001 From: Viktor Kurilko Date: Mon, 27 Nov 2023 12:29:04 +0700 Subject: [PATCH 01/31] Limit diskquota hash table's size according initial request MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Diskquota does not control the size of its hash tables in shared memory and may consume shared memory that is not intended for it, which may affect the other database subsystems. Hash tables can also grow indefinitely if the background process on the coordinator has not started, which can happen for a number of reasons: gone done with error, pause, isn’t started. This patch adds a limit on the size of hash tables in shared memory by adding a wrapper over hash_search that controls the size of the table and stops adding elements when the number of elements allocated at the beginning is reached and report a warning. GUC is also added, which limits the frequency of report of the warning, since in some cases it may be report too often. --- src/diskquota.c | 4 ++ src/diskquota.h | 3 ++ src/diskquota_utility.c | 24 +++++++++++ src/gp_activetable.c | 32 ++++++++------ src/quotamodel.c | 32 ++++++++++---- src/relation_cache.c | 31 ++++++++++++-- .../expected/test_activetable_limit.out | 42 +++++++++++-------- tests/regress/sql/test_activetable_limit.sql | 29 +++++++------ 8 files changed, 141 insertions(+), 56 deletions(-) diff --git a/src/diskquota.c b/src/diskquota.c index f714a4f0..834656da 100644 --- a/src/diskquota.c +++ b/src/diskquota.c @@ -78,6 +78,7 @@ int diskquota_max_workers = 10; int diskquota_max_table_segments = 0; int diskquota_max_monitored_databases = 0; int diskquota_max_quota_probes = 0; +time_t diskquota_hashmap_overflow_report_timeout = 0; DiskQuotaLocks diskquota_locks; ExtensionDDLMessage *extension_ddl_message = NULL; @@ -414,6 +415,9 @@ define_guc_variables(void) DefineCustomIntVariable("diskquota.max_quota_probes", "Max number of quotas on the cluster.", NULL, &diskquota_max_quota_probes, 1024 * 1024, 1024 * INIT_QUOTA_MAP_ENTRIES, INT_MAX, PGC_POSTMASTER, 0, NULL, NULL, NULL); + DefineCustomIntVariable("diskquota.hashmap_overflow_report_timeout", + "Time interval in seconds between shared hash map overflow report.", NULL, + &diskquota_hashmap_overflow_report_timeout, 60, 0, INT_MAX, PGC_SUSET, 0, NULL, NULL, NULL); } /* ---- Functions for disk quota worker process ---- */ diff --git a/src/diskquota.h b/src/diskquota.h index b3d3481c..006c62ab 100644 --- a/src/diskquota.h +++ b/src/diskquota.h @@ -286,6 +286,7 @@ extern Datum diskquota_fetch_table_stat(PG_FUNCTION_ARGS); extern int diskquota_naptime; extern int diskquota_max_active_tables; extern bool diskquota_hardlimit; +extern time_t diskquota_hashmap_overflow_report_timeout; extern int SEGCOUNT; extern int worker_spi_get_extension_version(int *major, int *minor); @@ -316,4 +317,6 @@ extern HTAB *diskquota_hash_create(const char *tabname, long nelem, HASHC extern HTAB *DiskquotaShmemInitHash(const char *name, long init_size, long max_size, HASHCTL *infoP, int hash_flags, DiskquotaHashFunction hash_function); extern void refresh_monitored_dbid_cache(void); +extern void *shm_hash_enter(HTAB *hashp, void *keyPtr, bool *foundPtr, uint max_size, const char *warning_message, + time_t *last_overflow_report); #endif diff --git a/src/diskquota_utility.c b/src/diskquota_utility.c index 28b874e8..2cfc28d3 100644 --- a/src/diskquota_utility.c +++ b/src/diskquota_utility.c @@ -1683,3 +1683,27 @@ DiskquotaShmemInitHash(const char *name, /* table string name fo return ShmemInitHash(name, init_size, max_size, infoP, hash_flags | HASH_BLOBS); #endif /* GP_VERSION_NUM */ } + +// Add or find an entry in a hash table with a size limit. If the limit is reached, only the search will be performed. +// When overflowing, the warning warning_message will be report. But not more often than specified in +// diskquota_hashmap_overflow_report_timeout. The time of the last warning is passed in last_overflow_report. +void * +shm_hash_enter(HTAB *hashp, void *keyPtr, bool *foundPtr, uint max_size, const char *warning_message, + time_t *last_overflow_report) +{ + if (hash_get_num_entries(hashp) >= max_size) + { + return hash_search(hashp, keyPtr, HASH_FIND, foundPtr); + } + else + { + void *result = hash_search(hashp, keyPtr, HASH_ENTER, foundPtr); + if (hash_get_num_entries(hashp) >= max_size && + (time(NULL) - *last_overflow_report) >= diskquota_hashmap_overflow_report_timeout) + { + ereport(WARNING, (errmsg(warning_message, max_size))); + *last_overflow_report = time(NULL); + } + return result; + } +} diff --git a/src/gp_activetable.c b/src/gp_activetable.c index 6e76633b..549a1c5f 100644 --- a/src/gp_activetable.c +++ b/src/gp_activetable.c @@ -52,6 +52,14 @@ typedef struct DiskQuotaSetOFCache } DiskQuotaSetOFCache; HTAB *active_tables_map = NULL; // Set +time_t active_tables_last_overflow_report = 0; + +#define ACTIVE_TABLE_ENTER(keyPtr, foundPtr) \ + shm_hash_enter(active_tables_map, keyPtr, foundPtr, diskquota_max_active_tables, \ + "[diskquota] the number of active tables reached the limit, please increase " \ + "the GUC value for diskquota.max_active_tables. Current " \ + "diskquota.max_active_tables value: %d", \ + &active_tables_last_overflow_report) /* * monitored_dbid_cache is a allow list for diskquota @@ -61,6 +69,14 @@ HTAB *active_tables_map = NULL; // Set * dbid will be removed from it when droping diskquota extension */ HTAB *altered_reloid_cache = NULL; // Set +time_t altered_reloid_cache_last_overflow_report = 0; + +#define ALTERED_RELOID_CACHE_ENTER(keyPtr, foundPtr) \ + shm_hash_enter(altered_reloid_cache, keyPtr, foundPtr, diskquota_max_active_tables, \ + "[diskquota] the number of altered reloid cache entries reached the limit, please increase " \ + "the GUC value for diskquota.max_active_tables. Current " \ + "diskquota.max_active_tables value: %d", \ + &altered_reloid_cache_last_overflow_report) /* active table hooks which detect the disk file size change. */ static file_create_hook_type prev_file_create_hook = NULL; @@ -236,7 +252,7 @@ report_altered_reloid(Oid reloid) if (IsRoleMirror() || IS_QUERY_DISPATCHER()) return; LWLockAcquire(diskquota_locks.altered_reloid_cache_lock, LW_EXCLUSIVE); - hash_search(altered_reloid_cache, &reloid, HASH_ENTER, NULL); + ALTERED_RELOID_CACHE_ENTER(&reloid, NULL); LWLockRelease(diskquota_locks.altered_reloid_cache_lock); } @@ -318,17 +334,9 @@ report_active_table_helper(const RelFileNodeBackend *relFileNode) item.tablespaceoid = relFileNode->node.spcNode; LWLockAcquire(diskquota_locks.active_table_lock, LW_EXCLUSIVE); - entry = hash_search(active_tables_map, &item, HASH_ENTER_NULL, &found); + entry = ACTIVE_TABLE_ENTER(&item, &found); if (entry && !found) *entry = item; - if (!found && entry == NULL) - { - /* - * We may miss the file size change of this relation at current - * refresh interval. - */ - ereport(WARNING, (errmsg("Share memory is not enough for active tables."))); - } LWLockRelease(diskquota_locks.active_table_lock); } @@ -857,7 +865,7 @@ get_active_tables_oid(void) while ((active_table_file_entry = (DiskQuotaActiveTableFileEntry *)hash_seq_search(&iter)) != NULL) { /* TODO: handle possible ERROR here so that the bgworker will not go down. */ - hash_search(active_tables_map, active_table_file_entry, HASH_ENTER, NULL); + ACTIVE_TABLE_ENTER(active_table_file_entry, NULL); } /* TODO: hash_seq_term(&iter); */ LWLockRelease(diskquota_locks.active_table_lock); @@ -919,7 +927,7 @@ get_active_tables_oid(void) LWLockAcquire(diskquota_locks.active_table_lock, LW_EXCLUSIVE); while ((active_table_file_entry = (DiskQuotaActiveTableFileEntry *)hash_seq_search(&iter)) != NULL) { - entry = hash_search(active_tables_map, active_table_file_entry, HASH_ENTER_NULL, &found); + entry = ACTIVE_TABLE_ENTER(active_table_file_entry, &found); if (entry) *entry = *active_table_file_entry; } LWLockRelease(diskquota_locks.active_table_lock); diff --git a/src/quotamodel.c b/src/quotamodel.c index ccfe7859..ed3f6b16 100644 --- a/src/quotamodel.c +++ b/src/quotamodel.c @@ -190,6 +190,21 @@ static HTAB *table_size_map = NULL; static HTAB *disk_quota_reject_map = NULL; static HTAB *local_disk_quota_reject_map = NULL; +static time_t disk_quota_reject_last_overflow_report = 0; +static time_t local_disk_quota_reject_last_overflow_report = 0; + +#define REJECT_MAP_ENTER(keyPtr, foundPtr) \ + shm_hash_enter(disk_quota_reject_map, keyPtr, foundPtr, MAX_DISK_QUOTA_REJECT_ENTRIES, \ + "[diskquota] Shared disk quota reject map size limit reached." \ + "Some out-of-limit schemas or roles will be lost" \ + "in rejectmap.", \ + &disk_quota_reject_last_overflow_report) + +#define LOCAL_REJECT_MAP_ENTER(keyPtr, foundPtr) \ + shm_hash_enter(local_disk_quota_reject_map, keyPtr, foundPtr, MAX_DISK_QUOTA_REJECT_ENTRIES, \ + "[diskquota] the number of local reject map entries reached the limit (%d)", \ + &local_disk_quota_reject_last_overflow_report) + static shmem_startup_hook_type prev_shmem_startup_hook = NULL; /* functions to maintain the quota maps */ @@ -326,9 +341,12 @@ add_quota_to_rejectmap(QuotaType type, Oid targetOid, Oid tablespaceoid, bool se keyitem.tablespaceoid = tablespaceoid; keyitem.targettype = (uint32)type; ereport(DEBUG1, (errmsg("[diskquota] Put object %u to rejectmap", targetOid))); - localrejectentry = (LocalRejectMapEntry *)hash_search(local_disk_quota_reject_map, &keyitem, HASH_ENTER, NULL); - localrejectentry->isexceeded = true; - localrejectentry->segexceeded = segexceeded; + localrejectentry = (LocalRejectMapEntry *)LOCAL_REJECT_MAP_ENTER(&keyitem, NULL); + if (localrejectentry) + { + localrejectentry->isexceeded = true; + localrejectentry->segexceeded = segexceeded; + } } /* @@ -1305,13 +1323,9 @@ flush_local_reject_map(void) */ if (localrejectentry->isexceeded) { - rejectentry = (GlobalRejectMapEntry *)hash_search(disk_quota_reject_map, (void *)&localrejectentry->keyitem, - HASH_ENTER_NULL, &found); + rejectentry = (GlobalRejectMapEntry *)REJECT_MAP_ENTER(&localrejectentry->keyitem, &found); if (rejectentry == NULL) { - ereport(WARNING, (errmsg("[diskquota] Shared disk quota reject map size limit reached." - "Some out-of-limit schemas or roles will be lost" - "in rejectmap."))); continue; } /* new db objects which exceed quota limit */ @@ -2145,7 +2159,7 @@ refresh_rejectmap(PG_FUNCTION_ARGS) */ if (OidIsValid(rejectmapentry->keyitem.targetoid)) continue; - new_entry = hash_search(disk_quota_reject_map, &rejectmapentry->keyitem, HASH_ENTER_NULL, &found); + new_entry = REJECT_MAP_ENTER(&rejectmapentry->keyitem, &found); if (!found && new_entry) memcpy(new_entry, rejectmapentry, sizeof(GlobalRejectMapEntry)); } LWLockRelease(diskquota_locks.reject_map_lock); diff --git a/src/relation_cache.c b/src/relation_cache.c index 647779de..d325b32b 100644 --- a/src/relation_cache.c +++ b/src/relation_cache.c @@ -32,6 +32,22 @@ HTAB *relation_cache = NULL; HTAB *relid_cache = NULL; +extern time_t active_tables_last_overflow_report; + +#define RELATION_CACHE_ENTER(keyPtr, foundPtr) \ + shm_hash_enter(relation_cache, keyPtr, foundPtr, diskquota_max_active_tables, \ + "[diskquota] the number of relation cache entries reached the limit, please increase " \ + "the GUC value for diskquota.max_active_tables. Current " \ + "diskquota.max_active_tables value: %d", \ + &active_tables_last_overflow_report) + +#define RELID_CACHE_ENTER(keyPtr, foundPtr) \ + shm_hash_enter(relid_cache, keyPtr, foundPtr, diskquota_max_active_tables, \ + "[diskquota] the number of relid cache entries reached the limit, please increase " \ + "the GUC value for diskquota.max_active_tables. Current " \ + "diskquota.max_active_tables value: %d", \ + &active_tables_last_overflow_report) + static void update_relation_entry(Oid relid, DiskQuotaRelationCacheEntry *relation_entry, DiskQuotaRelidCacheEntry *relid_entry); @@ -177,10 +193,19 @@ update_relation_cache(Oid relid) update_relation_entry(relid, &relation_entry_data, &relid_entry_data); LWLockAcquire(diskquota_locks.relation_cache_lock, LW_EXCLUSIVE); - relation_entry = hash_search(relation_cache, &relation_entry_data.relid, HASH_ENTER, NULL); + relation_entry = RELATION_CACHE_ENTER(&relation_entry_data.relid, NULL); + if (relation_entry == NULL) + { + LWLockRelease(diskquota_locks.relation_cache_lock); + return; + } memcpy(relation_entry, &relation_entry_data, sizeof(DiskQuotaRelationCacheEntry)); - - relid_entry = hash_search(relid_cache, &relid_entry_data.relfilenode, HASH_ENTER, NULL); + relid_entry = RELID_CACHE_ENTER(&relid_entry_data.relfilenode, NULL); + if (relid_entry == NULL) + { + LWLockRelease(diskquota_locks.relation_cache_lock); + return; + } memcpy(relid_entry, &relid_entry_data, sizeof(DiskQuotaRelidCacheEntry)); LWLockRelease(diskquota_locks.relation_cache_lock); diff --git a/tests/regress/expected/test_activetable_limit.out b/tests/regress/expected/test_activetable_limit.out index c556f32b..ad87865a 100644 --- a/tests/regress/expected/test_activetable_limit.out +++ b/tests/regress/expected/test_activetable_limit.out @@ -1,5 +1,6 @@ -- table in 'diskquota not enabled database' should not be activetable -\! gpconfig -c diskquota.max_active_tables -v 2 > /dev/null +\! gpconfig -c diskquota.max_active_tables -v 5 > /dev/null +\! gpconfig -c diskquota.naptime -v 2 > /dev/null \! gpstop -arf > /dev/null \c CREATE DATABASE test_tablenum_limit_01; @@ -26,31 +27,38 @@ SELECT diskquota.wait_for_worker_new_epoch(); t (1 row) -CREATE TABLE s.t1(i int) DISTRIBUTED BY (i); -- activetable = 1 -INSERT INTO s.t1 SELECT generate_series(1, 100000); -- ok. diskquota soft limit does not check when first write +-- the other two active tables are in a different database +CREATE TABLE s.t1(i int) DISTRIBUTED BY (i); -- activetable = 3 +CREATE TABLE s.t2(i int) DISTRIBUTED BY (i); -- activetable = 4 +CREATE TABLE s.t3(i int) DISTRIBUTED BY (i); -- activetable = 5. expected warning. +WARNING: [diskquota] the number of active tables reached the limit, please increase the GUC value for diskquota.max_active_tables. Current diskquota.max_active_tables value: 5 (seg0 127.0.1.1:6002 pid=995937) +WARNING: [diskquota] the number of active tables reached the limit, please increase the GUC value for diskquota.max_active_tables. Current diskquota.max_active_tables value: 5 (seg2 127.0.1.1:6004 pid=995938) +WARNING: [diskquota] the number of active tables reached the limit, please increase the GUC value for diskquota.max_active_tables. Current diskquota.max_active_tables value: 5 (seg1 127.0.1.1:6003 pid=995939) +CREATE TABLE s.t4(i int) DISTRIBUTED BY (i); +INSERT INTO s.t4 SELECT generate_series(1, 100000); SELECT diskquota.wait_for_worker_new_epoch(); wait_for_worker_new_epoch --------------------------- t (1 row) -CREATE TABLE s.t2(i int) DISTRIBUTED BY (i); -- activetable = 2 -INSERT INTO s.t2 SELECT generate_series(1, 10); -- expect failed -ERROR: schema's disk space quota exceeded with name: s -CREATE TABLE s.t3(i int) DISTRIBUTED BY (i); -- activetable = 3 should not crash. -INSERT INTO s.t3 SELECT generate_series(1, 10); -- expect failed -ERROR: schema's disk space quota exceeded with name: s --- Q: why diskquota still works when activetable = 3? --- A: the activetable limit by shmem size, calculate by hash_estimate_size() --- the result will bigger than sizeof(DiskQuotaActiveTableEntry) * max_active_tables --- the real capacity of this data structure based on the hash conflict probability. --- so we can not predict when the data structure will be fill in fully. --- --- this test case is useless, remove this if anyone dislike it. --- but the hash capacity is smaller than 6, so the test case works for issue 51 +INSERT INTO s.t1 SELECT generate_series(1, 10); -- should be successful +select count(*) from s.t1; + count +------- + 10 +(1 row) + +-- altered reloid cache overflow check. expected warning. +vacuum full; +WARNING: [diskquota] the number of active tables reached the limit, please increase the GUC value for diskquota.max_active_tables. Current diskquota.max_active_tables value: 5 +WARNING: [diskquota] the number of altered reloid cache entries reached the limit, please increase the GUC value for diskquota.max_active_tables. Current diskquota.max_active_tables value: 5 (seg0 127.0.1.1:6002 pid=995937) +WARNING: [diskquota] the number of altered reloid cache entries reached the limit, please increase the GUC value for diskquota.max_active_tables. Current diskquota.max_active_tables value: 5 (seg1 127.0.1.1:6003 pid=995939) +WARNING: [diskquota] the number of altered reloid cache entries reached the limit, please increase the GUC value for diskquota.max_active_tables. Current diskquota.max_active_tables value: 5 (seg2 127.0.1.1:6004 pid=995938) DROP EXTENSION diskquota; \c contrib_regression DROP DATABASE test_tablenum_limit_01; DROP DATABASE test_tablenum_limit_02; \! gpconfig -r diskquota.max_active_tables > /dev/null +\! gpconfig -c diskquota.naptime -v 0 > /dev/null \! gpstop -arf > /dev/null diff --git a/tests/regress/sql/test_activetable_limit.sql b/tests/regress/sql/test_activetable_limit.sql index 9ab6666a..5b58359d 100644 --- a/tests/regress/sql/test_activetable_limit.sql +++ b/tests/regress/sql/test_activetable_limit.sql @@ -1,5 +1,6 @@ -- table in 'diskquota not enabled database' should not be activetable -\! gpconfig -c diskquota.max_active_tables -v 2 > /dev/null +\! gpconfig -c diskquota.max_active_tables -v 5 > /dev/null +\! gpconfig -c diskquota.naptime -v 2 > /dev/null \! gpstop -arf > /dev/null \c @@ -24,24 +25,21 @@ SELECT diskquota.set_schema_quota('s', '1 MB'); SELECT diskquota.wait_for_worker_new_epoch(); -CREATE TABLE s.t1(i int) DISTRIBUTED BY (i); -- activetable = 1 -INSERT INTO s.t1 SELECT generate_series(1, 100000); -- ok. diskquota soft limit does not check when first write +-- the other two active tables are in a different database +CREATE TABLE s.t1(i int) DISTRIBUTED BY (i); -- activetable = 3 +CREATE TABLE s.t2(i int) DISTRIBUTED BY (i); -- activetable = 4 +CREATE TABLE s.t3(i int) DISTRIBUTED BY (i); -- activetable = 5. expected warning. +CREATE TABLE s.t4(i int) DISTRIBUTED BY (i); + +INSERT INTO s.t4 SELECT generate_series(1, 100000); SELECT diskquota.wait_for_worker_new_epoch(); -CREATE TABLE s.t2(i int) DISTRIBUTED BY (i); -- activetable = 2 -INSERT INTO s.t2 SELECT generate_series(1, 10); -- expect failed -CREATE TABLE s.t3(i int) DISTRIBUTED BY (i); -- activetable = 3 should not crash. -INSERT INTO s.t3 SELECT generate_series(1, 10); -- expect failed +INSERT INTO s.t1 SELECT generate_series(1, 10); -- should be successful +select count(*) from s.t1; --- Q: why diskquota still works when activetable = 3? --- A: the activetable limit by shmem size, calculate by hash_estimate_size() --- the result will bigger than sizeof(DiskQuotaActiveTableEntry) * max_active_tables --- the real capacity of this data structure based on the hash conflict probability. --- so we can not predict when the data structure will be fill in fully. --- --- this test case is useless, remove this if anyone dislike it. --- but the hash capacity is smaller than 6, so the test case works for issue 51 +-- altered reloid cache overflow check. expected warning. +vacuum full; DROP EXTENSION diskquota; @@ -50,4 +48,5 @@ DROP DATABASE test_tablenum_limit_01; DROP DATABASE test_tablenum_limit_02; \! gpconfig -r diskquota.max_active_tables > /dev/null +\! gpconfig -c diskquota.naptime -v 0 > /dev/null \! gpstop -arf > /dev/null From 705d2ca0bd614a11201a2cb384caf832f718cb39 Mon Sep 17 00:00:00 2001 From: Viktor Kurilko Date: Tue, 28 Nov 2023 19:44:27 +0700 Subject: [PATCH 02/31] using functions to work with time from gpdb instead of libc. add flag HASH_FIXED_SIZE to all hash tables in shared memory. --- src/diskquota.c | 5 +++-- src/diskquota.h | 6 +++--- src/diskquota_utility.c | 28 +++++++++++----------------- src/gp_activetable.c | 4 ++-- src/quotamodel.c | 14 +++++++------- 5 files changed, 26 insertions(+), 31 deletions(-) diff --git a/src/diskquota.c b/src/diskquota.c index 834656da..2d9116b8 100644 --- a/src/diskquota.c +++ b/src/diskquota.c @@ -78,7 +78,7 @@ int diskquota_max_workers = 10; int diskquota_max_table_segments = 0; int diskquota_max_monitored_databases = 0; int diskquota_max_quota_probes = 0; -time_t diskquota_hashmap_overflow_report_timeout = 0; +int diskquota_hashmap_overflow_report_timeout = 0; DiskQuotaLocks diskquota_locks; ExtensionDDLMessage *extension_ddl_message = NULL; @@ -417,7 +417,8 @@ define_guc_variables(void) PGC_POSTMASTER, 0, NULL, NULL, NULL); DefineCustomIntVariable("diskquota.hashmap_overflow_report_timeout", "Time interval in seconds between shared hash map overflow report.", NULL, - &diskquota_hashmap_overflow_report_timeout, 60, 0, INT_MAX, PGC_SUSET, 0, NULL, NULL, NULL); + &diskquota_hashmap_overflow_report_timeout, 60, 0, INT_MAX / 1000, PGC_SUSET, 0, NULL, NULL, + NULL); } /* ---- Functions for disk quota worker process ---- */ diff --git a/src/diskquota.h b/src/diskquota.h index 006c62ab..28a6c469 100644 --- a/src/diskquota.h +++ b/src/diskquota.h @@ -286,7 +286,7 @@ extern Datum diskquota_fetch_table_stat(PG_FUNCTION_ARGS); extern int diskquota_naptime; extern int diskquota_max_active_tables; extern bool diskquota_hardlimit; -extern time_t diskquota_hashmap_overflow_report_timeout; +extern int diskquota_hashmap_overflow_report_timeout; extern int SEGCOUNT; extern int worker_spi_get_extension_version(int *major, int *minor); @@ -317,6 +317,6 @@ extern HTAB *diskquota_hash_create(const char *tabname, long nelem, HASHC extern HTAB *DiskquotaShmemInitHash(const char *name, long init_size, long max_size, HASHCTL *infoP, int hash_flags, DiskquotaHashFunction hash_function); extern void refresh_monitored_dbid_cache(void); -extern void *shm_hash_enter(HTAB *hashp, void *keyPtr, bool *foundPtr, uint max_size, const char *warning_message, - time_t *last_overflow_report); +extern void *shm_hash_enter(HTAB *hashp, const void *keyPtr, bool *foundPtr, int max_size, const char *warning_message, + TimestampTz *last_overflow_report); #endif diff --git a/src/diskquota_utility.c b/src/diskquota_utility.c index 2cfc28d3..14c192f2 100644 --- a/src/diskquota_utility.c +++ b/src/diskquota_utility.c @@ -1678,9 +1678,9 @@ DiskquotaShmemInitHash(const char *name, /* table string name fo infoP->hash = oid_hash; else infoP->hash = string_hash; - return ShmemInitHash(name, init_size, max_size, infoP, hash_flags | HASH_FUNCTION); + return ShmemInitHash(name, init_size, max_size, infoP, hash_flags | HASH_FUNCTION | HASH_FIXED_SIZE); #else - return ShmemInitHash(name, init_size, max_size, infoP, hash_flags | HASH_BLOBS); + return ShmemInitHash(name, init_size, max_size, infoP, hash_flags | HASH_BLOBS | HASH_FIXED_SIZE); #endif /* GP_VERSION_NUM */ } @@ -1688,22 +1688,16 @@ DiskquotaShmemInitHash(const char *name, /* table string name fo // When overflowing, the warning warning_message will be report. But not more often than specified in // diskquota_hashmap_overflow_report_timeout. The time of the last warning is passed in last_overflow_report. void * -shm_hash_enter(HTAB *hashp, void *keyPtr, bool *foundPtr, uint max_size, const char *warning_message, - time_t *last_overflow_report) +shm_hash_enter(HTAB *hashp, const void *keyPtr, bool *foundPtr, int max_size, const char *warning_message, + TimestampTz *last_overflow_report) { - if (hash_get_num_entries(hashp) >= max_size) + void *result = hash_search(hashp, keyPtr, HASH_ENTER_NULL, foundPtr); + if (hash_get_num_entries(hashp) >= max_size && + TimestampDifferenceExceeds(*last_overflow_report, GetCurrentTimestamp(), + diskquota_hashmap_overflow_report_timeout * 1000)) { - return hash_search(hashp, keyPtr, HASH_FIND, foundPtr); - } - else - { - void *result = hash_search(hashp, keyPtr, HASH_ENTER, foundPtr); - if (hash_get_num_entries(hashp) >= max_size && - (time(NULL) - *last_overflow_report) >= diskquota_hashmap_overflow_report_timeout) - { - ereport(WARNING, (errmsg(warning_message, max_size))); - *last_overflow_report = time(NULL); - } - return result; + ereport(WARNING, (errmsg(warning_message, max_size))); + *last_overflow_report = GetCurrentTimestamp(); } + return result; } diff --git a/src/gp_activetable.c b/src/gp_activetable.c index 549a1c5f..4f6039f9 100644 --- a/src/gp_activetable.c +++ b/src/gp_activetable.c @@ -52,7 +52,7 @@ typedef struct DiskQuotaSetOFCache } DiskQuotaSetOFCache; HTAB *active_tables_map = NULL; // Set -time_t active_tables_last_overflow_report = 0; +TimestampTz active_tables_last_overflow_report = 0; #define ACTIVE_TABLE_ENTER(keyPtr, foundPtr) \ shm_hash_enter(active_tables_map, keyPtr, foundPtr, diskquota_max_active_tables, \ @@ -69,7 +69,7 @@ time_t active_tables_last_overflow_report = 0; * dbid will be removed from it when droping diskquota extension */ HTAB *altered_reloid_cache = NULL; // Set -time_t altered_reloid_cache_last_overflow_report = 0; +TimestampTz altered_reloid_cache_last_overflow_report = 0; #define ALTERED_RELOID_CACHE_ENTER(keyPtr, foundPtr) \ shm_hash_enter(altered_reloid_cache, keyPtr, foundPtr, diskquota_max_active_tables, \ diff --git a/src/quotamodel.c b/src/quotamodel.c index ed3f6b16..cb73de8d 100644 --- a/src/quotamodel.c +++ b/src/quotamodel.c @@ -473,7 +473,7 @@ disk_quota_shmem_startup(void) hash_ctl.keysize = sizeof(RejectMapEntry); hash_ctl.entrysize = sizeof(GlobalRejectMapEntry); disk_quota_reject_map = - DiskquotaShmemInitHash("rejectmap whose quota limitation is reached", INIT_DISK_QUOTA_REJECT_ENTRIES, + DiskquotaShmemInitHash("rejectmap whose quota limitation is reached", MAX_DISK_QUOTA_REJECT_ENTRIES, MAX_DISK_QUOTA_REJECT_ENTRIES, &hash_ctl, HASH_ELEM, DISKQUOTA_TAG_HASH); init_shm_worker_active_tables(); @@ -582,8 +582,8 @@ init_disk_quota_model(uint32 id) memset(&hash_ctl, 0, sizeof(hash_ctl)); hash_ctl.keysize = sizeof(TableSizeEntryKey); hash_ctl.entrysize = sizeof(TableSizeEntry); - table_size_map = DiskquotaShmemInitHash(str.data, INIT_NUM_TABLE_SIZE_ENTRIES, MAX_NUM_TABLE_SIZE_ENTRIES, - &hash_ctl, HASH_ELEM, DISKQUOTA_TAG_HASH); + table_size_map = DiskquotaShmemInitHash(str.data, MAX_NUM_TABLE_SIZE_ENTRIES, MAX_NUM_TABLE_SIZE_ENTRIES, &hash_ctl, + HASH_ELEM, DISKQUOTA_TAG_HASH); /* for localrejectmap */ /* WARNNING: The max length of name of the map is 48 */ @@ -600,7 +600,7 @@ init_disk_quota_model(uint32 id) memset(&hash_ctl, 0, sizeof(hash_ctl)); hash_ctl.entrysize = sizeof(QuotaInfoEntry); hash_ctl.keysize = sizeof(QuotaInfoEntryKey); - quota_info_map = DiskquotaShmemInitHash(str.data, INIT_QUOTA_MAP_ENTRIES, MAX_QUOTA_MAP_ENTRIES, &hash_ctl, + quota_info_map = DiskquotaShmemInitHash(str.data, MAX_QUOTA_MAP_ENTRIES, MAX_QUOTA_MAP_ENTRIES, &hash_ctl, HASH_ELEM, DISKQUOTA_TAG_HASH); pfree(str.data); @@ -635,8 +635,8 @@ vacuum_disk_quota_model(uint32 id) memset(&hash_ctl, 0, sizeof(hash_ctl)); hash_ctl.keysize = sizeof(TableSizeEntryKey); hash_ctl.entrysize = sizeof(TableSizeEntry); - table_size_map = DiskquotaShmemInitHash(str.data, INIT_NUM_TABLE_SIZE_ENTRIES, MAX_NUM_TABLE_SIZE_ENTRIES, - &hash_ctl, HASH_ELEM, DISKQUOTA_TAG_HASH); + table_size_map = DiskquotaShmemInitHash(str.data, MAX_NUM_TABLE_SIZE_ENTRIES, MAX_NUM_TABLE_SIZE_ENTRIES, &hash_ctl, + HASH_ELEM, DISKQUOTA_TAG_HASH); hash_seq_init(&iter, table_size_map); while ((tsentry = hash_seq_search(&iter)) != NULL) { @@ -663,7 +663,7 @@ vacuum_disk_quota_model(uint32 id) memset(&hash_ctl, 0, sizeof(hash_ctl)); hash_ctl.entrysize = sizeof(QuotaInfoEntry); hash_ctl.keysize = sizeof(QuotaInfoEntryKey); - quota_info_map = DiskquotaShmemInitHash(str.data, INIT_QUOTA_MAP_ENTRIES, MAX_QUOTA_MAP_ENTRIES, &hash_ctl, + quota_info_map = DiskquotaShmemInitHash(str.data, MAX_QUOTA_MAP_ENTRIES, MAX_QUOTA_MAP_ENTRIES, &hash_ctl, HASH_ELEM, DISKQUOTA_TAG_HASH); hash_seq_init(&iter, quota_info_map); while ((qentry = hash_seq_search(&iter)) != NULL) From 2e1eed7161ee54cb476d025090e97c3ac2abc83c Mon Sep 17 00:00:00 2001 From: Viktor Kurilko Date: Wed, 29 Nov 2023 12:18:21 +0700 Subject: [PATCH 03/31] Revert "using functions to work with time from gpdb instead of libc. add flag HASH_FIXED_SIZE to all hash tables in shared memory." This reverts commit 705d2ca0bd614a11201a2cb384caf832f718cb39. --- src/diskquota.c | 5 ++--- src/diskquota.h | 6 +++--- src/diskquota_utility.c | 28 +++++++++++++++++----------- src/gp_activetable.c | 4 ++-- src/quotamodel.c | 14 +++++++------- 5 files changed, 31 insertions(+), 26 deletions(-) diff --git a/src/diskquota.c b/src/diskquota.c index 2d9116b8..834656da 100644 --- a/src/diskquota.c +++ b/src/diskquota.c @@ -78,7 +78,7 @@ int diskquota_max_workers = 10; int diskquota_max_table_segments = 0; int diskquota_max_monitored_databases = 0; int diskquota_max_quota_probes = 0; -int diskquota_hashmap_overflow_report_timeout = 0; +time_t diskquota_hashmap_overflow_report_timeout = 0; DiskQuotaLocks diskquota_locks; ExtensionDDLMessage *extension_ddl_message = NULL; @@ -417,8 +417,7 @@ define_guc_variables(void) PGC_POSTMASTER, 0, NULL, NULL, NULL); DefineCustomIntVariable("diskquota.hashmap_overflow_report_timeout", "Time interval in seconds between shared hash map overflow report.", NULL, - &diskquota_hashmap_overflow_report_timeout, 60, 0, INT_MAX / 1000, PGC_SUSET, 0, NULL, NULL, - NULL); + &diskquota_hashmap_overflow_report_timeout, 60, 0, INT_MAX, PGC_SUSET, 0, NULL, NULL, NULL); } /* ---- Functions for disk quota worker process ---- */ diff --git a/src/diskquota.h b/src/diskquota.h index 28a6c469..006c62ab 100644 --- a/src/diskquota.h +++ b/src/diskquota.h @@ -286,7 +286,7 @@ extern Datum diskquota_fetch_table_stat(PG_FUNCTION_ARGS); extern int diskquota_naptime; extern int diskquota_max_active_tables; extern bool diskquota_hardlimit; -extern int diskquota_hashmap_overflow_report_timeout; +extern time_t diskquota_hashmap_overflow_report_timeout; extern int SEGCOUNT; extern int worker_spi_get_extension_version(int *major, int *minor); @@ -317,6 +317,6 @@ extern HTAB *diskquota_hash_create(const char *tabname, long nelem, HASHC extern HTAB *DiskquotaShmemInitHash(const char *name, long init_size, long max_size, HASHCTL *infoP, int hash_flags, DiskquotaHashFunction hash_function); extern void refresh_monitored_dbid_cache(void); -extern void *shm_hash_enter(HTAB *hashp, const void *keyPtr, bool *foundPtr, int max_size, const char *warning_message, - TimestampTz *last_overflow_report); +extern void *shm_hash_enter(HTAB *hashp, void *keyPtr, bool *foundPtr, uint max_size, const char *warning_message, + time_t *last_overflow_report); #endif diff --git a/src/diskquota_utility.c b/src/diskquota_utility.c index 14c192f2..2cfc28d3 100644 --- a/src/diskquota_utility.c +++ b/src/diskquota_utility.c @@ -1678,9 +1678,9 @@ DiskquotaShmemInitHash(const char *name, /* table string name fo infoP->hash = oid_hash; else infoP->hash = string_hash; - return ShmemInitHash(name, init_size, max_size, infoP, hash_flags | HASH_FUNCTION | HASH_FIXED_SIZE); + return ShmemInitHash(name, init_size, max_size, infoP, hash_flags | HASH_FUNCTION); #else - return ShmemInitHash(name, init_size, max_size, infoP, hash_flags | HASH_BLOBS | HASH_FIXED_SIZE); + return ShmemInitHash(name, init_size, max_size, infoP, hash_flags | HASH_BLOBS); #endif /* GP_VERSION_NUM */ } @@ -1688,16 +1688,22 @@ DiskquotaShmemInitHash(const char *name, /* table string name fo // When overflowing, the warning warning_message will be report. But not more often than specified in // diskquota_hashmap_overflow_report_timeout. The time of the last warning is passed in last_overflow_report. void * -shm_hash_enter(HTAB *hashp, const void *keyPtr, bool *foundPtr, int max_size, const char *warning_message, - TimestampTz *last_overflow_report) +shm_hash_enter(HTAB *hashp, void *keyPtr, bool *foundPtr, uint max_size, const char *warning_message, + time_t *last_overflow_report) { - void *result = hash_search(hashp, keyPtr, HASH_ENTER_NULL, foundPtr); - if (hash_get_num_entries(hashp) >= max_size && - TimestampDifferenceExceeds(*last_overflow_report, GetCurrentTimestamp(), - diskquota_hashmap_overflow_report_timeout * 1000)) + if (hash_get_num_entries(hashp) >= max_size) { - ereport(WARNING, (errmsg(warning_message, max_size))); - *last_overflow_report = GetCurrentTimestamp(); + return hash_search(hashp, keyPtr, HASH_FIND, foundPtr); + } + else + { + void *result = hash_search(hashp, keyPtr, HASH_ENTER, foundPtr); + if (hash_get_num_entries(hashp) >= max_size && + (time(NULL) - *last_overflow_report) >= diskquota_hashmap_overflow_report_timeout) + { + ereport(WARNING, (errmsg(warning_message, max_size))); + *last_overflow_report = time(NULL); + } + return result; } - return result; } diff --git a/src/gp_activetable.c b/src/gp_activetable.c index 4f6039f9..549a1c5f 100644 --- a/src/gp_activetable.c +++ b/src/gp_activetable.c @@ -52,7 +52,7 @@ typedef struct DiskQuotaSetOFCache } DiskQuotaSetOFCache; HTAB *active_tables_map = NULL; // Set -TimestampTz active_tables_last_overflow_report = 0; +time_t active_tables_last_overflow_report = 0; #define ACTIVE_TABLE_ENTER(keyPtr, foundPtr) \ shm_hash_enter(active_tables_map, keyPtr, foundPtr, diskquota_max_active_tables, \ @@ -69,7 +69,7 @@ TimestampTz active_tables_last_overflow_report = 0; * dbid will be removed from it when droping diskquota extension */ HTAB *altered_reloid_cache = NULL; // Set -TimestampTz altered_reloid_cache_last_overflow_report = 0; +time_t altered_reloid_cache_last_overflow_report = 0; #define ALTERED_RELOID_CACHE_ENTER(keyPtr, foundPtr) \ shm_hash_enter(altered_reloid_cache, keyPtr, foundPtr, diskquota_max_active_tables, \ diff --git a/src/quotamodel.c b/src/quotamodel.c index cb73de8d..ed3f6b16 100644 --- a/src/quotamodel.c +++ b/src/quotamodel.c @@ -473,7 +473,7 @@ disk_quota_shmem_startup(void) hash_ctl.keysize = sizeof(RejectMapEntry); hash_ctl.entrysize = sizeof(GlobalRejectMapEntry); disk_quota_reject_map = - DiskquotaShmemInitHash("rejectmap whose quota limitation is reached", MAX_DISK_QUOTA_REJECT_ENTRIES, + DiskquotaShmemInitHash("rejectmap whose quota limitation is reached", INIT_DISK_QUOTA_REJECT_ENTRIES, MAX_DISK_QUOTA_REJECT_ENTRIES, &hash_ctl, HASH_ELEM, DISKQUOTA_TAG_HASH); init_shm_worker_active_tables(); @@ -582,8 +582,8 @@ init_disk_quota_model(uint32 id) memset(&hash_ctl, 0, sizeof(hash_ctl)); hash_ctl.keysize = sizeof(TableSizeEntryKey); hash_ctl.entrysize = sizeof(TableSizeEntry); - table_size_map = DiskquotaShmemInitHash(str.data, MAX_NUM_TABLE_SIZE_ENTRIES, MAX_NUM_TABLE_SIZE_ENTRIES, &hash_ctl, - HASH_ELEM, DISKQUOTA_TAG_HASH); + table_size_map = DiskquotaShmemInitHash(str.data, INIT_NUM_TABLE_SIZE_ENTRIES, MAX_NUM_TABLE_SIZE_ENTRIES, + &hash_ctl, HASH_ELEM, DISKQUOTA_TAG_HASH); /* for localrejectmap */ /* WARNNING: The max length of name of the map is 48 */ @@ -600,7 +600,7 @@ init_disk_quota_model(uint32 id) memset(&hash_ctl, 0, sizeof(hash_ctl)); hash_ctl.entrysize = sizeof(QuotaInfoEntry); hash_ctl.keysize = sizeof(QuotaInfoEntryKey); - quota_info_map = DiskquotaShmemInitHash(str.data, MAX_QUOTA_MAP_ENTRIES, MAX_QUOTA_MAP_ENTRIES, &hash_ctl, + quota_info_map = DiskquotaShmemInitHash(str.data, INIT_QUOTA_MAP_ENTRIES, MAX_QUOTA_MAP_ENTRIES, &hash_ctl, HASH_ELEM, DISKQUOTA_TAG_HASH); pfree(str.data); @@ -635,8 +635,8 @@ vacuum_disk_quota_model(uint32 id) memset(&hash_ctl, 0, sizeof(hash_ctl)); hash_ctl.keysize = sizeof(TableSizeEntryKey); hash_ctl.entrysize = sizeof(TableSizeEntry); - table_size_map = DiskquotaShmemInitHash(str.data, MAX_NUM_TABLE_SIZE_ENTRIES, MAX_NUM_TABLE_SIZE_ENTRIES, &hash_ctl, - HASH_ELEM, DISKQUOTA_TAG_HASH); + table_size_map = DiskquotaShmemInitHash(str.data, INIT_NUM_TABLE_SIZE_ENTRIES, MAX_NUM_TABLE_SIZE_ENTRIES, + &hash_ctl, HASH_ELEM, DISKQUOTA_TAG_HASH); hash_seq_init(&iter, table_size_map); while ((tsentry = hash_seq_search(&iter)) != NULL) { @@ -663,7 +663,7 @@ vacuum_disk_quota_model(uint32 id) memset(&hash_ctl, 0, sizeof(hash_ctl)); hash_ctl.entrysize = sizeof(QuotaInfoEntry); hash_ctl.keysize = sizeof(QuotaInfoEntryKey); - quota_info_map = DiskquotaShmemInitHash(str.data, MAX_QUOTA_MAP_ENTRIES, MAX_QUOTA_MAP_ENTRIES, &hash_ctl, + quota_info_map = DiskquotaShmemInitHash(str.data, INIT_QUOTA_MAP_ENTRIES, MAX_QUOTA_MAP_ENTRIES, &hash_ctl, HASH_ELEM, DISKQUOTA_TAG_HASH); hash_seq_init(&iter, quota_info_map); while ((qentry = hash_seq_search(&iter)) != NULL) From 645fd60b7e018b1d2540dd66eabf1d9f2636084b Mon Sep 17 00:00:00 2001 From: Viktor Kurilko Date: Wed, 29 Nov 2023 12:30:36 +0700 Subject: [PATCH 04/31] using functions to work with time from gpdb instead of libc --- src/diskquota.c | 5 +++-- src/diskquota.h | 6 +++--- src/diskquota_utility.c | 22 +++++++++++----------- 3 files changed, 17 insertions(+), 16 deletions(-) diff --git a/src/diskquota.c b/src/diskquota.c index 834656da..2d9116b8 100644 --- a/src/diskquota.c +++ b/src/diskquota.c @@ -78,7 +78,7 @@ int diskquota_max_workers = 10; int diskquota_max_table_segments = 0; int diskquota_max_monitored_databases = 0; int diskquota_max_quota_probes = 0; -time_t diskquota_hashmap_overflow_report_timeout = 0; +int diskquota_hashmap_overflow_report_timeout = 0; DiskQuotaLocks diskquota_locks; ExtensionDDLMessage *extension_ddl_message = NULL; @@ -417,7 +417,8 @@ define_guc_variables(void) PGC_POSTMASTER, 0, NULL, NULL, NULL); DefineCustomIntVariable("diskquota.hashmap_overflow_report_timeout", "Time interval in seconds between shared hash map overflow report.", NULL, - &diskquota_hashmap_overflow_report_timeout, 60, 0, INT_MAX, PGC_SUSET, 0, NULL, NULL, NULL); + &diskquota_hashmap_overflow_report_timeout, 60, 0, INT_MAX / 1000, PGC_SUSET, 0, NULL, NULL, + NULL); } /* ---- Functions for disk quota worker process ---- */ diff --git a/src/diskquota.h b/src/diskquota.h index 006c62ab..28a6c469 100644 --- a/src/diskquota.h +++ b/src/diskquota.h @@ -286,7 +286,7 @@ extern Datum diskquota_fetch_table_stat(PG_FUNCTION_ARGS); extern int diskquota_naptime; extern int diskquota_max_active_tables; extern bool diskquota_hardlimit; -extern time_t diskquota_hashmap_overflow_report_timeout; +extern int diskquota_hashmap_overflow_report_timeout; extern int SEGCOUNT; extern int worker_spi_get_extension_version(int *major, int *minor); @@ -317,6 +317,6 @@ extern HTAB *diskquota_hash_create(const char *tabname, long nelem, HASHC extern HTAB *DiskquotaShmemInitHash(const char *name, long init_size, long max_size, HASHCTL *infoP, int hash_flags, DiskquotaHashFunction hash_function); extern void refresh_monitored_dbid_cache(void); -extern void *shm_hash_enter(HTAB *hashp, void *keyPtr, bool *foundPtr, uint max_size, const char *warning_message, - time_t *last_overflow_report); +extern void *shm_hash_enter(HTAB *hashp, const void *keyPtr, bool *foundPtr, int max_size, const char *warning_message, + TimestampTz *last_overflow_report); #endif diff --git a/src/diskquota_utility.c b/src/diskquota_utility.c index 2cfc28d3..b96bac91 100644 --- a/src/diskquota_utility.c +++ b/src/diskquota_utility.c @@ -1688,22 +1688,22 @@ DiskquotaShmemInitHash(const char *name, /* table string name fo // When overflowing, the warning warning_message will be report. But not more often than specified in // diskquota_hashmap_overflow_report_timeout. The time of the last warning is passed in last_overflow_report. void * -shm_hash_enter(HTAB *hashp, void *keyPtr, bool *foundPtr, uint max_size, const char *warning_message, - time_t *last_overflow_report) +shm_hash_enter(HTAB *hashp, const void *keyPtr, bool *foundPtr, int max_size, const char *warning_message, + TimestampTz *last_overflow_report) { if (hash_get_num_entries(hashp) >= max_size) { return hash_search(hashp, keyPtr, HASH_FIND, foundPtr); } - else + void *result = hash_search(hashp, keyPtr, HASH_ENTER, foundPtr); + + TimestampTz current_time = GetCurrentTimestamp(); + if (hash_get_num_entries(hashp) >= max_size && + TimestampDifferenceExceeds(*last_overflow_report, current_time, + diskquota_hashmap_overflow_report_timeout * 1000)) { - void *result = hash_search(hashp, keyPtr, HASH_ENTER, foundPtr); - if (hash_get_num_entries(hashp) >= max_size && - (time(NULL) - *last_overflow_report) >= diskquota_hashmap_overflow_report_timeout) - { - ereport(WARNING, (errmsg(warning_message, max_size))); - *last_overflow_report = time(NULL); - } - return result; + ereport(WARNING, (errmsg(warning_message, max_size))); + *last_overflow_report = current_time; } + return result; } From 1e3f23376d999af9e0d9a84e5696b808b456ba38 Mon Sep 17 00:00:00 2001 From: Viktor Kurilko Date: Thu, 30 Nov 2023 14:49:34 +0700 Subject: [PATCH 05/31] unified logic for adding elements to hash tables in shared memory for all tables --- src/diskquota.c | 15 ----- src/diskquota.h | 2 +- src/diskquota_utility.c | 4 +- src/gp_activetable.c | 4 +- src/quotamodel.c | 125 +++++++++++++--------------------------- src/relation_cache.c | 4 +- 6 files changed, 47 insertions(+), 107 deletions(-) diff --git a/src/diskquota.c b/src/diskquota.c index 2d9116b8..b0854750 100644 --- a/src/diskquota.c +++ b/src/diskquota.c @@ -90,12 +90,6 @@ static DiskQuotaWorkerEntry *volatile MyWorkerInfo = NULL; // how many database diskquota are monitoring on static int num_db = 0; -/* how many TableSizeEntry are maintained in all the table_size_map in shared memory*/ -pg_atomic_uint32 *diskquota_table_size_entry_num; - -/* how many QuotaInfoEntry are maintained in all the quota_info_map in shared memory*/ -pg_atomic_uint32 *diskquota_quota_info_entry_num; - static DiskquotaLauncherShmemStruct *DiskquotaLauncherShmem; #define MIN_SLEEPTIME 100 /* milliseconds */ @@ -1807,15 +1801,6 @@ init_launcher_shmem() DiskquotaLauncherShmem->dbArray[i].workerId = INVALID_WORKER_ID; } } - /* init TableSizeEntry counter */ - diskquota_table_size_entry_num = - ShmemInitStruct("diskquota TableSizeEntry counter", sizeof(pg_atomic_uint32), &found); - if (!found) pg_atomic_init_u32(diskquota_table_size_entry_num, 0); - - /* init QuotaInfoEntry counter */ - diskquota_quota_info_entry_num = - ShmemInitStruct("diskquota QuotaInfoEntry counter", sizeof(pg_atomic_uint32), &found); - if (!found) pg_atomic_init_u32(diskquota_quota_info_entry_num, 0); } /* diff --git a/src/diskquota.h b/src/diskquota.h index 28a6c469..0520bb19 100644 --- a/src/diskquota.h +++ b/src/diskquota.h @@ -318,5 +318,5 @@ extern HTAB *DiskquotaShmemInitHash(const char *name, long init_size, long max_s DiskquotaHashFunction hash_function); extern void refresh_monitored_dbid_cache(void); extern void *shm_hash_enter(HTAB *hashp, const void *keyPtr, bool *foundPtr, int max_size, const char *warning_message, - TimestampTz *last_overflow_report); + TimestampTz *last_overflow_report, int guc_value); #endif diff --git a/src/diskquota_utility.c b/src/diskquota_utility.c index b96bac91..e00f8b82 100644 --- a/src/diskquota_utility.c +++ b/src/diskquota_utility.c @@ -1689,7 +1689,7 @@ DiskquotaShmemInitHash(const char *name, /* table string name fo // diskquota_hashmap_overflow_report_timeout. The time of the last warning is passed in last_overflow_report. void * shm_hash_enter(HTAB *hashp, const void *keyPtr, bool *foundPtr, int max_size, const char *warning_message, - TimestampTz *last_overflow_report) + TimestampTz *last_overflow_report, int guc_value) { if (hash_get_num_entries(hashp) >= max_size) { @@ -1702,7 +1702,7 @@ shm_hash_enter(HTAB *hashp, const void *keyPtr, bool *foundPtr, int max_size, co TimestampDifferenceExceeds(*last_overflow_report, current_time, diskquota_hashmap_overflow_report_timeout * 1000)) { - ereport(WARNING, (errmsg(warning_message, max_size))); + ereport(WARNING, (errmsg(warning_message, guc_value))); *last_overflow_report = current_time; } return result; diff --git a/src/gp_activetable.c b/src/gp_activetable.c index 549a1c5f..458fbbfe 100644 --- a/src/gp_activetable.c +++ b/src/gp_activetable.c @@ -59,7 +59,7 @@ time_t active_tables_last_overflow_report = 0; "[diskquota] the number of active tables reached the limit, please increase " \ "the GUC value for diskquota.max_active_tables. Current " \ "diskquota.max_active_tables value: %d", \ - &active_tables_last_overflow_report) + &active_tables_last_overflow_report, diskquota_max_active_tables) /* * monitored_dbid_cache is a allow list for diskquota @@ -76,7 +76,7 @@ time_t altered_reloid_cache_last_overflow_report = 0; "[diskquota] the number of altered reloid cache entries reached the limit, please increase " \ "the GUC value for diskquota.max_active_tables. Current " \ "diskquota.max_active_tables value: %d", \ - &altered_reloid_cache_last_overflow_report) + &altered_reloid_cache_last_overflow_report, diskquota_max_active_tables) /* active table hooks which detect the disk file size change. */ static file_create_hook_type prev_file_create_hook = NULL; diff --git a/src/quotamodel.c b/src/quotamodel.c index ed3f6b16..612ebfac 100644 --- a/src/quotamodel.c +++ b/src/quotamodel.c @@ -85,10 +85,8 @@ typedef struct LocalRejectMapEntry LocalRejectMapEntry; int SEGCOUNT = 0; extern int diskquota_max_table_segments; -extern pg_atomic_uint32 *diskquota_table_size_entry_num; extern int diskquota_max_monitored_databases; extern int diskquota_max_quota_probes; -extern pg_atomic_uint32 *diskquota_quota_info_entry_num; /* * local cache of table disk size and corresponding schema and owner. @@ -145,6 +143,14 @@ uint16 quota_key_num[NUM_QUOTA_TYPES] = {1, 1, 2, 2, Oid quota_key_caches[NUM_QUOTA_TYPES][MAX_NUM_KEYS_QUOTA_MAP] = { {NAMESPACEOID}, {AUTHOID}, {NAMESPACEOID, TABLESPACEOID}, {AUTHOID, TABLESPACEOID}, {TABLESPACEOID}}; HTAB *quota_info_map; +TimestampTz quota_info_map_last_overflow_report = 0; + +#define QUOTA_INFO_ENTER(keyPtr, foundPtr) \ + shm_hash_enter(quota_info_map, keyPtr, foundPtr, diskquota_max_quota_probes, \ + "[diskquota] the number of quota probe reached the limit, please " \ + "increase the GUC value for diskquota.max_quota_probes. Current " \ + "diskquota.max_quota_probes value: %d", \ + "a_info_map_last_overflow_report, diskquota_max_quota_probes) /* global rejectmap for which exceed their quota limit */ struct RejectMapEntry @@ -185,6 +191,14 @@ struct LocalRejectMapEntry /* using hash table to support incremental update the table size entry.*/ static HTAB *table_size_map = NULL; +static TimestampTz table_size_last_overflow_report = 0; + +#define TABLE_SIZE_ENTER(keyPtr, foundPtr) \ + shm_hash_enter(table_size_map, keyPtr, foundPtr, MAX_NUM_TABLE_SIZE_ENTRIES, \ + "[diskquota] the number of tables reached the limit, please increase " \ + "the GUC value for diskquota.max_table_segments. Current " \ + "diskquota.max_table_segments value: %d", \ + &table_size_last_overflow_report, diskquota_max_table_segments) /* rejectmap for database objects which exceed their quota limit */ static HTAB *disk_quota_reject_map = NULL; @@ -195,15 +209,15 @@ static time_t local_disk_quota_reject_last_overflow_report = 0; #define REJECT_MAP_ENTER(keyPtr, foundPtr) \ shm_hash_enter(disk_quota_reject_map, keyPtr, foundPtr, MAX_DISK_QUOTA_REJECT_ENTRIES, \ - "[diskquota] Shared disk quota reject map size limit reached." \ + "[diskquota] Shared disk quota reject map size limit reached (%d)." \ "Some out-of-limit schemas or roles will be lost" \ "in rejectmap.", \ - &disk_quota_reject_last_overflow_report) + &disk_quota_reject_last_overflow_report, MAX_DISK_QUOTA_REJECT_ENTRIES) #define LOCAL_REJECT_MAP_ENTER(keyPtr, foundPtr) \ shm_hash_enter(local_disk_quota_reject_map, keyPtr, foundPtr, MAX_DISK_QUOTA_REJECT_ENTRIES, \ "[diskquota] the number of local reject map entries reached the limit (%d)", \ - &local_disk_quota_reject_last_overflow_report) + &local_disk_quota_reject_last_overflow_report, MAX_DISK_QUOTA_REJECT_ENTRIES) static shmem_startup_hook_type prev_shmem_startup_hook = NULL; @@ -214,7 +228,6 @@ static void add_quota_to_rejectmap(QuotaType type, Oid targetOid, Oid tablespace static void refresh_quota_info_map(void); static void clean_all_quota_limit(void); static void transfer_table_for_quota(int64 totalsize, QuotaType type, Oid *old_keys, Oid *new_keys, int16 segid); -static QuotaInfoEntry *put_quota_map_entry(QuotaInfoEntryKey *key, bool *found); /* functions to refresh disk quota model*/ static void refresh_disk_quota_usage(bool is_init); @@ -239,44 +252,6 @@ static void set_table_size_entry_flag(TableSizeEntry *entry, TableSizeEntryFlag static void delete_from_table_size_map(char *str); -/* - * put QuotaInfoEntry into quota_info_map and return this entry. - * return NULL: no free SHM for quota_info_map - * found cannot be NULL - */ -static QuotaInfoEntry * -put_quota_map_entry(QuotaInfoEntryKey *key, bool *found) -{ - QuotaInfoEntry *entry; - uint32 counter = pg_atomic_read_u32(diskquota_quota_info_entry_num); - if (counter >= diskquota_max_quota_probes) - { - entry = hash_search(quota_info_map, key, HASH_FIND, found); - /* - * Too many quotas have been added to the quota_info_map, to avoid diskquota using - * too much shared memory, just return NULL. The diskquota won't work correctly - * anymore. - */ - if (!(*found)) return NULL; - } - else - { - entry = hash_search(quota_info_map, key, HASH_ENTER, found); - if (!(*found)) - { - counter = pg_atomic_add_fetch_u32(diskquota_quota_info_entry_num, 1); - if (counter >= diskquota_max_quota_probes) - { - ereport(WARNING, (errmsg("[diskquota] the number of quota probe exceeds the limit, please " - "increase the GUC value for diskquota.max_quota_probes. Current " - "diskquota.max_quota_probes value: %d", - diskquota_max_quota_probes))); - } - } - } - return entry; -} - /* add a new entry quota or update the old entry quota */ static void update_size_for_quota(int64 size, QuotaType type, Oid *keys, int16 segid) @@ -288,7 +263,7 @@ update_size_for_quota(int64 size, QuotaType type, Oid *keys, int16 segid) memcpy(key.keys, keys, quota_key_num[type] * sizeof(Oid)); key.type = type; key.segid = segid; - entry = put_quota_map_entry(&key, &found); + entry = QUOTA_INFO_ENTER(&key, &found); /* If the number of quota exceeds the limit, entry will be NULL */ if (entry == NULL) return; if (!found) @@ -312,7 +287,7 @@ update_limit_for_quota(int64 limit, float segratio, QuotaType type, Oid *keys) memcpy(key.keys, keys, quota_key_num[type] * sizeof(Oid)); key.type = type; key.segid = i; - entry = put_quota_map_entry(&key, &found); + entry = QUOTA_INFO_ENTER(&key, &found); /* If the number of quota exceeds the limit, entry will be NULL */ if (entry == NULL) continue; if (!found) @@ -372,7 +347,6 @@ refresh_quota_info_map(void) if (!HeapTupleIsValid(tuple)) { hash_search(quota_info_map, &entry->key, HASH_REMOVE, NULL); - pg_atomic_fetch_sub_u32(diskquota_quota_info_entry_num, 1); removed = true; break; } @@ -558,7 +532,6 @@ DiskQuotaShmemSize(void) if (IS_QUERY_DISPATCHER()) { size = add_size(size, diskquota_launcher_shmem_size()); - size = add_size(size, sizeof(pg_atomic_uint32)); size = add_size(size, diskquota_worker_shmem_size() * diskquota_max_monitored_databases); size = add_size(size, hash_estimate_size(MAX_QUOTA_MAP_ENTRIES, sizeof(QuotaInfoEntry)) * diskquota_max_monitored_databases); @@ -641,7 +614,6 @@ vacuum_disk_quota_model(uint32 id) while ((tsentry = hash_seq_search(&iter)) != NULL) { hash_search(table_size_map, &tsentry->key, HASH_REMOVE, NULL); - pg_atomic_fetch_sub_u32(diskquota_table_size_entry_num, 1); } /* localrejectmap */ @@ -669,7 +641,6 @@ vacuum_disk_quota_model(uint32 id) while ((qentry = hash_seq_search(&iter)) != NULL) { hash_search(quota_info_map, &qentry->key, HASH_REMOVE, NULL); - pg_atomic_fetch_sub_u32(diskquota_quota_info_entry_num, 1); } pfree(str.data); @@ -1020,45 +991,30 @@ calculate_table_disk_usage(bool is_init, HTAB *local_active_table_stat_map) key.reloid = relOid; key.id = TableSizeEntryId(cur_segid); - uint32 counter = pg_atomic_read_u32(diskquota_table_size_entry_num); - if (counter > MAX_NUM_TABLE_SIZE_ENTRIES) + tsentry = TABLE_SIZE_ENTER(&key, &table_size_map_found); + + if (!table_size_map_found) { - tsentry = (TableSizeEntry *)hash_search(table_size_map, &key, HASH_FIND, &table_size_map_found); - /* Too many tables have been added to the table_size_map, to avoid diskquota using - too much share memory, just quit the loop. The diskquota won't work correctly - anymore. */ - if (!table_size_map_found) + if (tsentry == NULL) { + /* Too many tables have been added to the table_size_map, to avoid diskquota using + too much share memory, just quit the loop. The diskquota won't work correctly + anymore. */ break; } - } - else - { - tsentry = (TableSizeEntry *)hash_search(table_size_map, &key, HASH_ENTER, &table_size_map_found); - if (!table_size_map_found) - { - counter = pg_atomic_add_fetch_u32(diskquota_table_size_entry_num, 1); - if (counter > MAX_NUM_TABLE_SIZE_ENTRIES) - { - ereport(WARNING, (errmsg("[diskquota] the number of tables exceeds the limit, please increase " - "the GUC value for diskquota.max_table_segments. Current " - "diskquota.max_table_segments value: %d", - diskquota_max_table_segments))); - } - tsentry->key.reloid = relOid; - tsentry->key.id = key.id; - Assert(TableSizeEntrySegidStart(tsentry) == cur_segid); - memset(tsentry->totalsize, 0, sizeof(tsentry->totalsize)); - tsentry->owneroid = InvalidOid; - tsentry->namespaceoid = InvalidOid; - tsentry->tablespaceoid = InvalidOid; - tsentry->flag = 0; - - int seg_st = TableSizeEntrySegidStart(tsentry); - int seg_ed = TableSizeEntrySegidEnd(tsentry); - for (int j = seg_st; j < seg_ed; j++) TableSizeEntrySetFlushFlag(tsentry, j); - } + tsentry->key.reloid = relOid; + tsentry->key.id = key.id; + Assert(TableSizeEntrySegidStart(tsentry) == cur_segid); + memset(tsentry->totalsize, 0, sizeof(tsentry->totalsize)); + tsentry->owneroid = InvalidOid; + tsentry->namespaceoid = InvalidOid; + tsentry->tablespaceoid = InvalidOid; + tsentry->flag = 0; + + int seg_st = TableSizeEntrySegidStart(tsentry); + int seg_ed = TableSizeEntrySegidEnd(tsentry); + for (int j = seg_st; j < seg_ed; j++) TableSizeEntrySetFlushFlag(tsentry, j); } /* mark tsentry is_exist */ @@ -1279,7 +1235,6 @@ flush_to_table_size(void) if (!get_table_size_entry_flag(tsentry, TABLE_EXIST)) { hash_search(table_size_map, &tsentry->key, HASH_REMOVE, NULL); - pg_atomic_fetch_sub_u32(diskquota_table_size_entry_num, 1); } } diff --git a/src/relation_cache.c b/src/relation_cache.c index d325b32b..f45b541d 100644 --- a/src/relation_cache.c +++ b/src/relation_cache.c @@ -39,14 +39,14 @@ extern time_t active_tables_last_overflow_report; "[diskquota] the number of relation cache entries reached the limit, please increase " \ "the GUC value for diskquota.max_active_tables. Current " \ "diskquota.max_active_tables value: %d", \ - &active_tables_last_overflow_report) + &active_tables_last_overflow_report, diskquota_max_active_tables) #define RELID_CACHE_ENTER(keyPtr, foundPtr) \ shm_hash_enter(relid_cache, keyPtr, foundPtr, diskquota_max_active_tables, \ "[diskquota] the number of relid cache entries reached the limit, please increase " \ "the GUC value for diskquota.max_active_tables. Current " \ "diskquota.max_active_tables value: %d", \ - &active_tables_last_overflow_report) + &active_tables_last_overflow_report, diskquota_max_active_tables) static void update_relation_entry(Oid relid, DiskQuotaRelationCacheEntry *relation_entry, DiskQuotaRelidCacheEntry *relid_entry); From 7158307e77a8b47b74b8323e7ba2ab4c53c7fc7d Mon Sep 17 00:00:00 2001 From: Viktor Kurilko Date: Thu, 30 Nov 2023 17:23:16 +0700 Subject: [PATCH 06/31] few fixes --- src/diskquota_utility.c | 25 ++++++++++++++----------- src/gp_activetable.c | 2 +- 2 files changed, 15 insertions(+), 12 deletions(-) diff --git a/src/diskquota_utility.c b/src/diskquota_utility.c index e00f8b82..e1ccec5a 100644 --- a/src/diskquota_utility.c +++ b/src/diskquota_utility.c @@ -1691,19 +1691,22 @@ void * shm_hash_enter(HTAB *hashp, const void *keyPtr, bool *foundPtr, int max_size, const char *warning_message, TimestampTz *last_overflow_report, int guc_value) { - if (hash_get_num_entries(hashp) >= max_size) - { - return hash_search(hashp, keyPtr, HASH_FIND, foundPtr); - } - void *result = hash_search(hashp, keyPtr, HASH_ENTER, foundPtr); + void *result; + TimestampTz current_time; + + if (hash_get_num_entries(hashp) >= max_size) return hash_search(hashp, keyPtr, HASH_FIND, foundPtr); - TimestampTz current_time = GetCurrentTimestamp(); - if (hash_get_num_entries(hashp) >= max_size && - TimestampDifferenceExceeds(*last_overflow_report, current_time, - diskquota_hashmap_overflow_report_timeout * 1000)) + result = hash_search(hashp, keyPtr, HASH_ENTER, foundPtr); + + if (hash_get_num_entries(hashp) >= max_size) { - ereport(WARNING, (errmsg(warning_message, guc_value))); - *last_overflow_report = current_time; + current_time = GetCurrentTimestamp(); + if (TimestampDifferenceExceeds(*last_overflow_report, current_time, + diskquota_hashmap_overflow_report_timeout * 1000)) + { + ereport(WARNING, (errmsg(warning_message, guc_value))); + *last_overflow_report = current_time; + } } return result; } diff --git a/src/gp_activetable.c b/src/gp_activetable.c index 458fbbfe..0a78c64c 100644 --- a/src/gp_activetable.c +++ b/src/gp_activetable.c @@ -52,7 +52,7 @@ typedef struct DiskQuotaSetOFCache } DiskQuotaSetOFCache; HTAB *active_tables_map = NULL; // Set -time_t active_tables_last_overflow_report = 0; +TimestampTz active_tables_last_overflow_report = 0; #define ACTIVE_TABLE_ENTER(keyPtr, foundPtr) \ shm_hash_enter(active_tables_map, keyPtr, foundPtr, diskquota_max_active_tables, \ From 800adc15d971e211809fbf2a5b66696f9801d377 Mon Sep 17 00:00:00 2001 From: Viktor Kurilko Date: Thu, 30 Nov 2023 18:37:14 +0700 Subject: [PATCH 07/31] remake shm_hash_enter --- src/diskquota_utility.c | 13 ++++--------- tests/regress/expected/test_activetable_limit.out | 2 +- 2 files changed, 5 insertions(+), 10 deletions(-) diff --git a/src/diskquota_utility.c b/src/diskquota_utility.c index e1ccec5a..29d3bc0e 100644 --- a/src/diskquota_utility.c +++ b/src/diskquota_utility.c @@ -1691,16 +1691,11 @@ void * shm_hash_enter(HTAB *hashp, const void *keyPtr, bool *foundPtr, int max_size, const char *warning_message, TimestampTz *last_overflow_report, int guc_value) { - void *result; - TimestampTz current_time; + if (hash_get_num_entries(hashp) < max_size) return hash_search(hashp, keyPtr, HASH_ENTER, foundPtr); - if (hash_get_num_entries(hashp) >= max_size) return hash_search(hashp, keyPtr, HASH_FIND, foundPtr); - - result = hash_search(hashp, keyPtr, HASH_ENTER, foundPtr); - - if (hash_get_num_entries(hashp) >= max_size) + if (hash_get_num_entries(hashp) == max_size) { - current_time = GetCurrentTimestamp(); + TimestampTz current_time = GetCurrentTimestamp(); if (TimestampDifferenceExceeds(*last_overflow_report, current_time, diskquota_hashmap_overflow_report_timeout * 1000)) { @@ -1708,5 +1703,5 @@ shm_hash_enter(HTAB *hashp, const void *keyPtr, bool *foundPtr, int max_size, co *last_overflow_report = current_time; } } - return result; + return hash_search(hashp, keyPtr, HASH_FIND, foundPtr); } diff --git a/tests/regress/expected/test_activetable_limit.out b/tests/regress/expected/test_activetable_limit.out index ad87865a..bb1caa25 100644 --- a/tests/regress/expected/test_activetable_limit.out +++ b/tests/regress/expected/test_activetable_limit.out @@ -31,10 +31,10 @@ SELECT diskquota.wait_for_worker_new_epoch(); CREATE TABLE s.t1(i int) DISTRIBUTED BY (i); -- activetable = 3 CREATE TABLE s.t2(i int) DISTRIBUTED BY (i); -- activetable = 4 CREATE TABLE s.t3(i int) DISTRIBUTED BY (i); -- activetable = 5. expected warning. +CREATE TABLE s.t4(i int) DISTRIBUTED BY (i); WARNING: [diskquota] the number of active tables reached the limit, please increase the GUC value for diskquota.max_active_tables. Current diskquota.max_active_tables value: 5 (seg0 127.0.1.1:6002 pid=995937) WARNING: [diskquota] the number of active tables reached the limit, please increase the GUC value for diskquota.max_active_tables. Current diskquota.max_active_tables value: 5 (seg2 127.0.1.1:6004 pid=995938) WARNING: [diskquota] the number of active tables reached the limit, please increase the GUC value for diskquota.max_active_tables. Current diskquota.max_active_tables value: 5 (seg1 127.0.1.1:6003 pid=995939) -CREATE TABLE s.t4(i int) DISTRIBUTED BY (i); INSERT INTO s.t4 SELECT generate_series(1, 100000); SELECT diskquota.wait_for_worker_new_epoch(); wait_for_worker_new_epoch From 7ff7717d8e7ac46fa78bbacf031fcf360a324fb3 Mon Sep 17 00:00:00 2001 From: Viktor Kurilko Date: Thu, 30 Nov 2023 18:59:39 +0700 Subject: [PATCH 08/31] use TimestampTz instead of time_t --- src/gp_activetable.c | 2 +- src/quotamodel.c | 4 ++-- src/relation_cache.c | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/src/gp_activetable.c b/src/gp_activetable.c index 0a78c64c..e06b613a 100644 --- a/src/gp_activetable.c +++ b/src/gp_activetable.c @@ -69,7 +69,7 @@ TimestampTz active_tables_last_overflow_report = 0; * dbid will be removed from it when droping diskquota extension */ HTAB *altered_reloid_cache = NULL; // Set -time_t altered_reloid_cache_last_overflow_report = 0; +TimestampTz altered_reloid_cache_last_overflow_report = 0; #define ALTERED_RELOID_CACHE_ENTER(keyPtr, foundPtr) \ shm_hash_enter(altered_reloid_cache, keyPtr, foundPtr, diskquota_max_active_tables, \ diff --git a/src/quotamodel.c b/src/quotamodel.c index 612ebfac..533becfb 100644 --- a/src/quotamodel.c +++ b/src/quotamodel.c @@ -204,8 +204,8 @@ static TimestampTz table_size_last_overflow_report = 0; static HTAB *disk_quota_reject_map = NULL; static HTAB *local_disk_quota_reject_map = NULL; -static time_t disk_quota_reject_last_overflow_report = 0; -static time_t local_disk_quota_reject_last_overflow_report = 0; +static TimestampTz disk_quota_reject_last_overflow_report = 0; +static TimestampTz local_disk_quota_reject_last_overflow_report = 0; #define REJECT_MAP_ENTER(keyPtr, foundPtr) \ shm_hash_enter(disk_quota_reject_map, keyPtr, foundPtr, MAX_DISK_QUOTA_REJECT_ENTRIES, \ diff --git a/src/relation_cache.c b/src/relation_cache.c index f45b541d..94633903 100644 --- a/src/relation_cache.c +++ b/src/relation_cache.c @@ -32,7 +32,7 @@ HTAB *relation_cache = NULL; HTAB *relid_cache = NULL; -extern time_t active_tables_last_overflow_report; +extern TimestampTz active_tables_last_overflow_report; #define RELATION_CACHE_ENTER(keyPtr, foundPtr) \ shm_hash_enter(relation_cache, keyPtr, foundPtr, diskquota_max_active_tables, \ From bcae9e164f6292d1cab5cf7cf896c1c4aa1e1308 Mon Sep 17 00:00:00 2001 From: Viktor Kurilko Date: Tue, 5 Dec 2023 14:21:11 +0700 Subject: [PATCH 09/31] fix formating --- src/quotamodel.c | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/src/quotamodel.c b/src/quotamodel.c index 533becfb..c738d237 100644 --- a/src/quotamodel.c +++ b/src/quotamodel.c @@ -83,10 +83,10 @@ typedef struct RejectMapEntry RejectMapEntry; typedef struct GlobalRejectMapEntry GlobalRejectMapEntry; typedef struct LocalRejectMapEntry LocalRejectMapEntry; -int SEGCOUNT = 0; -extern int diskquota_max_table_segments; -extern int diskquota_max_monitored_databases; -extern int diskquota_max_quota_probes; +int SEGCOUNT = 0; +extern int diskquota_max_table_segments; +extern int diskquota_max_monitored_databases; +extern int diskquota_max_quota_probes; /* * local cache of table disk size and corresponding schema and owner. @@ -142,7 +142,7 @@ typedef enum uint16 quota_key_num[NUM_QUOTA_TYPES] = {1, 1, 2, 2, 1}; Oid quota_key_caches[NUM_QUOTA_TYPES][MAX_NUM_KEYS_QUOTA_MAP] = { {NAMESPACEOID}, {AUTHOID}, {NAMESPACEOID, TABLESPACEOID}, {AUTHOID, TABLESPACEOID}, {TABLESPACEOID}}; -HTAB *quota_info_map; +HTAB *quota_info_map; TimestampTz quota_info_map_last_overflow_report = 0; #define QUOTA_INFO_ENTER(keyPtr, foundPtr) \ @@ -190,7 +190,7 @@ struct LocalRejectMapEntry }; /* using hash table to support incremental update the table size entry.*/ -static HTAB *table_size_map = NULL; +static HTAB *table_size_map = NULL; static TimestampTz table_size_last_overflow_report = 0; #define TABLE_SIZE_ENTER(keyPtr, foundPtr) \ From 9844e59504f99024f1f463f011b7698990ff3e03 Mon Sep 17 00:00:00 2001 From: Viktor Kurilko Date: Tue, 5 Dec 2023 14:34:03 +0700 Subject: [PATCH 10/31] fix formating --- src/diskquota.c | 16 ++++++++-------- src/gp_activetable.c | 4 ++-- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/diskquota.c b/src/diskquota.c index b0854750..e6755906 100644 --- a/src/diskquota.c +++ b/src/diskquota.c @@ -70,14 +70,14 @@ static volatile sig_atomic_t got_sigusr1 = false; static volatile sig_atomic_t got_sigusr2 = false; /* GUC variables */ -int diskquota_naptime = 0; -int diskquota_max_active_tables = 0; -int diskquota_worker_timeout = 60; /* default timeout is 60 seconds */ -bool diskquota_hardlimit = false; -int diskquota_max_workers = 10; -int diskquota_max_table_segments = 0; -int diskquota_max_monitored_databases = 0; -int diskquota_max_quota_probes = 0; +int diskquota_naptime = 0; +int diskquota_max_active_tables = 0; +int diskquota_worker_timeout = 60; /* default timeout is 60 seconds */ +bool diskquota_hardlimit = false; +int diskquota_max_workers = 10; +int diskquota_max_table_segments = 0; +int diskquota_max_monitored_databases = 0; +int diskquota_max_quota_probes = 0; int diskquota_hashmap_overflow_report_timeout = 0; DiskQuotaLocks diskquota_locks; diff --git a/src/gp_activetable.c b/src/gp_activetable.c index e06b613a..37af9116 100644 --- a/src/gp_activetable.c +++ b/src/gp_activetable.c @@ -51,7 +51,7 @@ typedef struct DiskQuotaSetOFCache HASH_SEQ_STATUS pos; } DiskQuotaSetOFCache; -HTAB *active_tables_map = NULL; // Set +HTAB *active_tables_map = NULL; // Set TimestampTz active_tables_last_overflow_report = 0; #define ACTIVE_TABLE_ENTER(keyPtr, foundPtr) \ @@ -68,7 +68,7 @@ TimestampTz active_tables_last_overflow_report = 0; * dbid will be added to it when creating diskquota extension * dbid will be removed from it when droping diskquota extension */ -HTAB *altered_reloid_cache = NULL; // Set +HTAB *altered_reloid_cache = NULL; // Set TimestampTz altered_reloid_cache_last_overflow_report = 0; #define ALTERED_RELOID_CACHE_ENTER(keyPtr, foundPtr) \ From 8af2777a1b8244eef23c8d8cbc0e6b64a655869a Mon Sep 17 00:00:00 2001 From: Viktor Kurilko Date: Tue, 5 Dec 2023 19:53:36 +0700 Subject: [PATCH 11/31] make tests more stable --- .../expected/test_activetable_limit.out | 39 ++++++++++++------- tests/regress/sql/test_activetable_limit.sql | 15 ++++--- 2 files changed, 31 insertions(+), 23 deletions(-) diff --git a/tests/regress/expected/test_activetable_limit.out b/tests/regress/expected/test_activetable_limit.out index bb1caa25..017e38c5 100644 --- a/tests/regress/expected/test_activetable_limit.out +++ b/tests/regress/expected/test_activetable_limit.out @@ -1,6 +1,6 @@ -- table in 'diskquota not enabled database' should not be activetable \! gpconfig -c diskquota.max_active_tables -v 5 > /dev/null -\! gpconfig -c diskquota.naptime -v 2 > /dev/null +\! gpconfig -c diskquota.naptime -v 1 > /dev/null \! gpstop -arf > /dev/null \c CREATE DATABASE test_tablenum_limit_01; @@ -27,22 +27,32 @@ SELECT diskquota.wait_for_worker_new_epoch(); t (1 row) --- the other two active tables are in a different database -CREATE TABLE s.t1(i int) DISTRIBUTED BY (i); -- activetable = 3 -CREATE TABLE s.t2(i int) DISTRIBUTED BY (i); -- activetable = 4 -CREATE TABLE s.t3(i int) DISTRIBUTED BY (i); -- activetable = 5. expected warning. -CREATE TABLE s.t4(i int) DISTRIBUTED BY (i); -WARNING: [diskquota] the number of active tables reached the limit, please increase the GUC value for diskquota.max_active_tables. Current diskquota.max_active_tables value: 5 (seg0 127.0.1.1:6002 pid=995937) -WARNING: [diskquota] the number of active tables reached the limit, please increase the GUC value for diskquota.max_active_tables. Current diskquota.max_active_tables value: 5 (seg2 127.0.1.1:6004 pid=995938) -WARNING: [diskquota] the number of active tables reached the limit, please increase the GUC value for diskquota.max_active_tables. Current diskquota.max_active_tables value: 5 (seg1 127.0.1.1:6003 pid=995939) -INSERT INTO s.t4 SELECT generate_series(1, 100000); +-- We create twice as many tables as the limit to ensure that the active_tables table is overflow. +CREATE TABLE s.t1 (a int, b int) DISTRIBUTED BY (a) + PARTITION BY RANGE (b) ( START (0) END (10) EVERY (1) ); +NOTICE: CREATE TABLE will create partition "t1_1_prt_1" for table "t1" +NOTICE: CREATE TABLE will create partition "t1_1_prt_2" for table "t1" +NOTICE: CREATE TABLE will create partition "t1_1_prt_3" for table "t1" +NOTICE: CREATE TABLE will create partition "t1_1_prt_4" for table "t1" +NOTICE: CREATE TABLE will create partition "t1_1_prt_5" for table "t1" +NOTICE: CREATE TABLE will create partition "t1_1_prt_6" for table "t1" +NOTICE: CREATE TABLE will create partition "t1_1_prt_7" for table "t1" +NOTICE: CREATE TABLE will create partition "t1_1_prt_8" for table "t1" +NOTICE: CREATE TABLE will create partition "t1_1_prt_9" for table "t1" +NOTICE: CREATE TABLE will create partition "t1_1_prt_10" for table "t1" +WARNING: [diskquota] the number of active tables reached the limit, please increase the GUC value for diskquota.max_active_tables. Current diskquota.max_active_tables value: 5 +WARNING: [diskquota] the number of active tables reached the limit, please increase the GUC value for diskquota.max_active_tables. Current diskquota.max_active_tables value: 5 (seg0 127.0.1.1:6002 pid=281310) +WARNING: [diskquota] the number of active tables reached the limit, please increase the GUC value for diskquota.max_active_tables. Current diskquota.max_active_tables value: 5 (seg1 127.0.1.1:6003 pid=281312) +WARNING: [diskquota] the number of active tables reached the limit, please increase the GUC value for diskquota.max_active_tables. Current diskquota.max_active_tables value: 5 (seg2 127.0.1.1:6004 pid=281311) +CREATE TABLE s.t2(i int) DISTRIBUTED BY (i); +INSERT INTO s.t2 SELECT generate_series(1, 100000); SELECT diskquota.wait_for_worker_new_epoch(); wait_for_worker_new_epoch --------------------------- t (1 row) -INSERT INTO s.t1 SELECT generate_series(1, 10); -- should be successful +INSERT INTO s.t1 SELECT a, a from generate_series(0, 9)a; -- should be successful select count(*) from s.t1; count ------- @@ -51,10 +61,9 @@ select count(*) from s.t1; -- altered reloid cache overflow check. expected warning. vacuum full; -WARNING: [diskquota] the number of active tables reached the limit, please increase the GUC value for diskquota.max_active_tables. Current diskquota.max_active_tables value: 5 -WARNING: [diskquota] the number of altered reloid cache entries reached the limit, please increase the GUC value for diskquota.max_active_tables. Current diskquota.max_active_tables value: 5 (seg0 127.0.1.1:6002 pid=995937) -WARNING: [diskquota] the number of altered reloid cache entries reached the limit, please increase the GUC value for diskquota.max_active_tables. Current diskquota.max_active_tables value: 5 (seg1 127.0.1.1:6003 pid=995939) -WARNING: [diskquota] the number of altered reloid cache entries reached the limit, please increase the GUC value for diskquota.max_active_tables. Current diskquota.max_active_tables value: 5 (seg2 127.0.1.1:6004 pid=995938) +WARNING: [diskquota] the number of altered reloid cache entries reached the limit, please increase the GUC value for diskquota.max_active_tables. Current diskquota.max_active_tables value: 5 (seg0 127.0.1.1:6002 pid=281310) +WARNING: [diskquota] the number of altered reloid cache entries reached the limit, please increase the GUC value for diskquota.max_active_tables. Current diskquota.max_active_tables value: 5 (seg1 127.0.1.1:6003 pid=281312) +WARNING: [diskquota] the number of altered reloid cache entries reached the limit, please increase the GUC value for diskquota.max_active_tables. Current diskquota.max_active_tables value: 5 (seg2 127.0.1.1:6004 pid=281311) DROP EXTENSION diskquota; \c contrib_regression DROP DATABASE test_tablenum_limit_01; diff --git a/tests/regress/sql/test_activetable_limit.sql b/tests/regress/sql/test_activetable_limit.sql index 5b58359d..f06f9df9 100644 --- a/tests/regress/sql/test_activetable_limit.sql +++ b/tests/regress/sql/test_activetable_limit.sql @@ -1,6 +1,6 @@ -- table in 'diskquota not enabled database' should not be activetable \! gpconfig -c diskquota.max_active_tables -v 5 > /dev/null -\! gpconfig -c diskquota.naptime -v 2 > /dev/null +\! gpconfig -c diskquota.naptime -v 1 > /dev/null \! gpstop -arf > /dev/null \c @@ -25,17 +25,16 @@ SELECT diskquota.set_schema_quota('s', '1 MB'); SELECT diskquota.wait_for_worker_new_epoch(); --- the other two active tables are in a different database -CREATE TABLE s.t1(i int) DISTRIBUTED BY (i); -- activetable = 3 -CREATE TABLE s.t2(i int) DISTRIBUTED BY (i); -- activetable = 4 -CREATE TABLE s.t3(i int) DISTRIBUTED BY (i); -- activetable = 5. expected warning. -CREATE TABLE s.t4(i int) DISTRIBUTED BY (i); +-- We create twice as many tables as the limit to ensure that the active_tables table is overflow. +CREATE TABLE s.t1 (a int, b int) DISTRIBUTED BY (a) + PARTITION BY RANGE (b) ( START (0) END (10) EVERY (1) ); +CREATE TABLE s.t2(i int) DISTRIBUTED BY (i); -INSERT INTO s.t4 SELECT generate_series(1, 100000); +INSERT INTO s.t2 SELECT generate_series(1, 100000); SELECT diskquota.wait_for_worker_new_epoch(); -INSERT INTO s.t1 SELECT generate_series(1, 10); -- should be successful +INSERT INTO s.t1 SELECT a, a from generate_series(0, 9)a; -- should be successful select count(*) from s.t1; -- altered reloid cache overflow check. expected warning. From edbda84829d1d8885eb5a008a26ab2be4f635055 Mon Sep 17 00:00:00 2001 From: Viktor Kurilko Date: Wed, 6 Dec 2023 11:25:13 +0700 Subject: [PATCH 12/31] change guc description. fix warning message --- src/diskquota.c | 6 +++--- src/quotamodel.c | 5 ++--- 2 files changed, 5 insertions(+), 6 deletions(-) diff --git a/src/diskquota.c b/src/diskquota.c index e6755906..725894a9 100644 --- a/src/diskquota.c +++ b/src/diskquota.c @@ -410,9 +410,9 @@ define_guc_variables(void) &diskquota_max_quota_probes, 1024 * 1024, 1024 * INIT_QUOTA_MAP_ENTRIES, INT_MAX, PGC_POSTMASTER, 0, NULL, NULL, NULL); DefineCustomIntVariable("diskquota.hashmap_overflow_report_timeout", - "Time interval in seconds between shared hash map overflow report.", NULL, - &diskquota_hashmap_overflow_report_timeout, 60, 0, INT_MAX / 1000, PGC_SUSET, 0, NULL, NULL, - NULL); + "The duration between each warning report about the shared hashmap overflow (in seconds).", + NULL, &diskquota_hashmap_overflow_report_timeout, 60, 0, INT_MAX / 1000, PGC_SUSET, 0, NULL, + NULL, NULL); } /* ---- Functions for disk quota worker process ---- */ diff --git a/src/quotamodel.c b/src/quotamodel.c index c738d237..5bd257ce 100644 --- a/src/quotamodel.c +++ b/src/quotamodel.c @@ -209,9 +209,8 @@ static TimestampTz local_disk_quota_reject_last_overflow_report = 0; #define REJECT_MAP_ENTER(keyPtr, foundPtr) \ shm_hash_enter(disk_quota_reject_map, keyPtr, foundPtr, MAX_DISK_QUOTA_REJECT_ENTRIES, \ - "[diskquota] Shared disk quota reject map size limit reached (%d)." \ - "Some out-of-limit schemas or roles will be lost" \ - "in rejectmap.", \ + "[diskquota] Shared disk quota reject map size limit reached (%d). " \ + "Some out-of-limit schemas or roles will be lost in rejectmap.", \ &disk_quota_reject_last_overflow_report, MAX_DISK_QUOTA_REJECT_ENTRIES) #define LOCAL_REJECT_MAP_ENTER(keyPtr, foundPtr) \ From a94fd9293423e0410da638175c28cbcf4d8ed577 Mon Sep 17 00:00:00 2001 From: Viktor Kurilko Date: Wed, 13 Dec 2023 19:32:11 +0700 Subject: [PATCH 13/31] rework --- src/diskquota.h | 4 +-- src/diskquota_utility.c | 16 +++++------ src/gp_activetable.c | 38 +++++++++++++++----------- src/quotamodel.c | 60 ++++++++++++++++++++++------------------- src/relation_cache.c | 34 ++++++++++++----------- 5 files changed, 84 insertions(+), 68 deletions(-) diff --git a/src/diskquota.h b/src/diskquota.h index 0520bb19..bbb62052 100644 --- a/src/diskquota.h +++ b/src/diskquota.h @@ -317,6 +317,6 @@ extern HTAB *diskquota_hash_create(const char *tabname, long nelem, HASHC extern HTAB *DiskquotaShmemInitHash(const char *name, long init_size, long max_size, HASHCTL *infoP, int hash_flags, DiskquotaHashFunction hash_function); extern void refresh_monitored_dbid_cache(void); -extern void *shm_hash_enter(HTAB *hashp, const void *keyPtr, bool *foundPtr, int max_size, const char *warning_message, - TimestampTz *last_overflow_report, int guc_value); +extern HASHACTION check_hash_fullness(HTAB *hashp, int max_size, const char *warning_message, + TimestampTz *last_overflow_report, int guc_value); #endif diff --git a/src/diskquota_utility.c b/src/diskquota_utility.c index 29d3bc0e..4827bc42 100644 --- a/src/diskquota_utility.c +++ b/src/diskquota_utility.c @@ -1684,14 +1684,14 @@ DiskquotaShmemInitHash(const char *name, /* table string name fo #endif /* GP_VERSION_NUM */ } -// Add or find an entry in a hash table with a size limit. If the limit is reached, only the search will be performed. -// When overflowing, the warning warning_message will be report. But not more often than specified in -// diskquota_hashmap_overflow_report_timeout. The time of the last warning is passed in last_overflow_report. -void * -shm_hash_enter(HTAB *hashp, const void *keyPtr, bool *foundPtr, int max_size, const char *warning_message, - TimestampTz *last_overflow_report, int guc_value) +/* + * Returns HASH_FIND if hash table is full and HASH_ENTER otherwise. + */ +HASHACTION +check_hash_fullness(HTAB *hashp, int max_size, const char *warning_message, TimestampTz *last_overflow_report, + int guc_value) { - if (hash_get_num_entries(hashp) < max_size) return hash_search(hashp, keyPtr, HASH_ENTER, foundPtr); + if (hash_get_num_entries(hashp) < max_size) return HASH_ENTER; if (hash_get_num_entries(hashp) == max_size) { @@ -1703,5 +1703,5 @@ shm_hash_enter(HTAB *hashp, const void *keyPtr, bool *foundPtr, int max_size, co *last_overflow_report = current_time; } } - return hash_search(hashp, keyPtr, HASH_FIND, foundPtr); + return HASH_FIND; } diff --git a/src/gp_activetable.c b/src/gp_activetable.c index 37af9116..0bcae887 100644 --- a/src/gp_activetable.c +++ b/src/gp_activetable.c @@ -54,12 +54,10 @@ typedef struct DiskQuotaSetOFCache HTAB *active_tables_map = NULL; // Set TimestampTz active_tables_last_overflow_report = 0; -#define ACTIVE_TABLE_ENTER(keyPtr, foundPtr) \ - shm_hash_enter(active_tables_map, keyPtr, foundPtr, diskquota_max_active_tables, \ - "[diskquota] the number of active tables reached the limit, please increase " \ - "the GUC value for diskquota.max_active_tables. Current " \ - "diskquota.max_active_tables value: %d", \ - &active_tables_last_overflow_report, diskquota_max_active_tables) +#define ACTIVE_TABLE_WARNING \ + "[diskquota] the number of active tables reached the limit, please increase " \ + "the GUC value for diskquota.max_active_tables. Current " \ + "diskquota.max_active_tables value: %d" /* * monitored_dbid_cache is a allow list for diskquota @@ -71,12 +69,10 @@ TimestampTz active_tables_last_overflow_report = 0; HTAB *altered_reloid_cache = NULL; // Set TimestampTz altered_reloid_cache_last_overflow_report = 0; -#define ALTERED_RELOID_CACHE_ENTER(keyPtr, foundPtr) \ - shm_hash_enter(altered_reloid_cache, keyPtr, foundPtr, diskquota_max_active_tables, \ - "[diskquota] the number of altered reloid cache entries reached the limit, please increase " \ - "the GUC value for diskquota.max_active_tables. Current " \ - "diskquota.max_active_tables value: %d", \ - &altered_reloid_cache_last_overflow_report, diskquota_max_active_tables) +#define ALTERED_RELOID_CACHE_WARNING \ + "[diskquota] the number of altered reloid cache entries reached the limit, please increase " \ + "the GUC value for diskquota.max_active_tables. Current " \ + "diskquota.max_active_tables value: %d" /* active table hooks which detect the disk file size change. */ static file_create_hook_type prev_file_create_hook = NULL; @@ -252,7 +248,10 @@ report_altered_reloid(Oid reloid) if (IsRoleMirror() || IS_QUERY_DISPATCHER()) return; LWLockAcquire(diskquota_locks.altered_reloid_cache_lock, LW_EXCLUSIVE); - ALTERED_RELOID_CACHE_ENTER(&reloid, NULL); + HASHACTION action = + check_hash_fullness(altered_reloid_cache, diskquota_max_active_tables, ALTERED_RELOID_CACHE_WARNING, + &altered_reloid_cache_last_overflow_report, diskquota_max_active_tables); + hash_search(altered_reloid_cache, &reloid, action, NULL); LWLockRelease(diskquota_locks.altered_reloid_cache_lock); } @@ -334,7 +333,9 @@ report_active_table_helper(const RelFileNodeBackend *relFileNode) item.tablespaceoid = relFileNode->node.spcNode; LWLockAcquire(diskquota_locks.active_table_lock, LW_EXCLUSIVE); - entry = ACTIVE_TABLE_ENTER(&item, &found); + HASHACTION action = check_hash_fullness(active_tables_map, diskquota_max_active_tables, ACTIVE_TABLE_WARNING, + &active_tables_last_overflow_report, diskquota_max_active_tables); + entry = hash_search(active_tables_map, &item, action, &found); if (entry && !found) *entry = item; LWLockRelease(diskquota_locks.active_table_lock); @@ -865,7 +866,9 @@ get_active_tables_oid(void) while ((active_table_file_entry = (DiskQuotaActiveTableFileEntry *)hash_seq_search(&iter)) != NULL) { /* TODO: handle possible ERROR here so that the bgworker will not go down. */ - ACTIVE_TABLE_ENTER(active_table_file_entry, NULL); + HASHACTION action = check_hash_fullness(active_tables_map, diskquota_max_active_tables, ACTIVE_TABLE_WARNING, + &active_tables_last_overflow_report, diskquota_max_active_tables); + hash_search(active_tables_map, active_table_file_entry, action, NULL); } /* TODO: hash_seq_term(&iter); */ LWLockRelease(diskquota_locks.active_table_lock); @@ -927,7 +930,10 @@ get_active_tables_oid(void) LWLockAcquire(diskquota_locks.active_table_lock, LW_EXCLUSIVE); while ((active_table_file_entry = (DiskQuotaActiveTableFileEntry *)hash_seq_search(&iter)) != NULL) { - entry = ACTIVE_TABLE_ENTER(active_table_file_entry, &found); + HASHACTION action = + check_hash_fullness(active_tables_map, diskquota_max_active_tables, ACTIVE_TABLE_WARNING, + &active_tables_last_overflow_report, diskquota_max_active_tables); + entry = hash_search(active_tables_map, active_table_file_entry, action, &found); if (entry) *entry = *active_table_file_entry; } LWLockRelease(diskquota_locks.active_table_lock); diff --git a/src/quotamodel.c b/src/quotamodel.c index 5bd257ce..5d74918c 100644 --- a/src/quotamodel.c +++ b/src/quotamodel.c @@ -145,12 +145,10 @@ Oid quota_key_caches[NUM_QUOTA_TYPES][MAX_NUM_KEYS_QUOTA_MAP] = { HTAB *quota_info_map; TimestampTz quota_info_map_last_overflow_report = 0; -#define QUOTA_INFO_ENTER(keyPtr, foundPtr) \ - shm_hash_enter(quota_info_map, keyPtr, foundPtr, diskquota_max_quota_probes, \ - "[diskquota] the number of quota probe reached the limit, please " \ - "increase the GUC value for diskquota.max_quota_probes. Current " \ - "diskquota.max_quota_probes value: %d", \ - "a_info_map_last_overflow_report, diskquota_max_quota_probes) +#define QUOTA_INFO_WARNING \ + "[diskquota] the number of quota probe reached the limit, please " \ + "increase the GUC value for diskquota.max_quota_probes. Current " \ + "diskquota.max_quota_probes value: %d" /* global rejectmap for which exceed their quota limit */ struct RejectMapEntry @@ -193,12 +191,10 @@ struct LocalRejectMapEntry static HTAB *table_size_map = NULL; static TimestampTz table_size_last_overflow_report = 0; -#define TABLE_SIZE_ENTER(keyPtr, foundPtr) \ - shm_hash_enter(table_size_map, keyPtr, foundPtr, MAX_NUM_TABLE_SIZE_ENTRIES, \ - "[diskquota] the number of tables reached the limit, please increase " \ - "the GUC value for diskquota.max_table_segments. Current " \ - "diskquota.max_table_segments value: %d", \ - &table_size_last_overflow_report, diskquota_max_table_segments) +#define TABLE_SIZE_WARNING \ + "[diskquota] the number of tables reached the limit, please increase " \ + "the GUC value for diskquota.max_table_segments. Current " \ + "diskquota.max_table_segments value: %d" /* rejectmap for database objects which exceed their quota limit */ static HTAB *disk_quota_reject_map = NULL; @@ -207,16 +203,11 @@ static HTAB *local_disk_quota_reject_map = NULL; static TimestampTz disk_quota_reject_last_overflow_report = 0; static TimestampTz local_disk_quota_reject_last_overflow_report = 0; -#define REJECT_MAP_ENTER(keyPtr, foundPtr) \ - shm_hash_enter(disk_quota_reject_map, keyPtr, foundPtr, MAX_DISK_QUOTA_REJECT_ENTRIES, \ - "[diskquota] Shared disk quota reject map size limit reached (%d). " \ - "Some out-of-limit schemas or roles will be lost in rejectmap.", \ - &disk_quota_reject_last_overflow_report, MAX_DISK_QUOTA_REJECT_ENTRIES) +#define REJECT_MAP_WARNING \ + "[diskquota] Shared disk quota reject map size limit reached (%d). " \ + "Some out-of-limit schemas or roles will be lost in rejectmap." -#define LOCAL_REJECT_MAP_ENTER(keyPtr, foundPtr) \ - shm_hash_enter(local_disk_quota_reject_map, keyPtr, foundPtr, MAX_DISK_QUOTA_REJECT_ENTRIES, \ - "[diskquota] the number of local reject map entries reached the limit (%d)", \ - &local_disk_quota_reject_last_overflow_report, MAX_DISK_QUOTA_REJECT_ENTRIES) +#define LOCAL_REJECT_MAP_WARNING "[diskquota] the number of local reject map entries reached the limit (%d)" static shmem_startup_hook_type prev_shmem_startup_hook = NULL; @@ -262,7 +253,9 @@ update_size_for_quota(int64 size, QuotaType type, Oid *keys, int16 segid) memcpy(key.keys, keys, quota_key_num[type] * sizeof(Oid)); key.type = type; key.segid = segid; - entry = QUOTA_INFO_ENTER(&key, &found); + HASHACTION action = check_hash_fullness(quota_info_map, diskquota_max_quota_probes, QUOTA_INFO_WARNING, + "a_info_map_last_overflow_report, diskquota_max_quota_probes); + entry = hash_search(quota_info_map, &key, action, &found); /* If the number of quota exceeds the limit, entry will be NULL */ if (entry == NULL) return; if (!found) @@ -286,7 +279,9 @@ update_limit_for_quota(int64 limit, float segratio, QuotaType type, Oid *keys) memcpy(key.keys, keys, quota_key_num[type] * sizeof(Oid)); key.type = type; key.segid = i; - entry = QUOTA_INFO_ENTER(&key, &found); + HASHACTION action = check_hash_fullness(quota_info_map, diskquota_max_quota_probes, QUOTA_INFO_WARNING, + "a_info_map_last_overflow_report, diskquota_max_quota_probes); + entry = hash_search(quota_info_map, &key, action, &found); /* If the number of quota exceeds the limit, entry will be NULL */ if (entry == NULL) continue; if (!found) @@ -315,7 +310,10 @@ add_quota_to_rejectmap(QuotaType type, Oid targetOid, Oid tablespaceoid, bool se keyitem.tablespaceoid = tablespaceoid; keyitem.targettype = (uint32)type; ereport(DEBUG1, (errmsg("[diskquota] Put object %u to rejectmap", targetOid))); - localrejectentry = (LocalRejectMapEntry *)LOCAL_REJECT_MAP_ENTER(&keyitem, NULL); + HASHACTION action = + check_hash_fullness(local_disk_quota_reject_map, MAX_DISK_QUOTA_REJECT_ENTRIES, LOCAL_REJECT_MAP_WARNING, + &local_disk_quota_reject_last_overflow_report, MAX_DISK_QUOTA_REJECT_ENTRIES); + localrejectentry = hash_search(local_disk_quota_reject_map, &keyitem, action, NULL); if (localrejectentry) { localrejectentry->isexceeded = true; @@ -990,7 +988,9 @@ calculate_table_disk_usage(bool is_init, HTAB *local_active_table_stat_map) key.reloid = relOid; key.id = TableSizeEntryId(cur_segid); - tsentry = TABLE_SIZE_ENTER(&key, &table_size_map_found); + HASHACTION action = check_hash_fullness(table_size_map, MAX_NUM_TABLE_SIZE_ENTRIES, TABLE_SIZE_WARNING, + &table_size_last_overflow_report, diskquota_max_table_segments); + tsentry = hash_search(table_size_map, &key, action, &table_size_map_found); if (!table_size_map_found) { @@ -1277,7 +1277,10 @@ flush_local_reject_map(void) */ if (localrejectentry->isexceeded) { - rejectentry = (GlobalRejectMapEntry *)REJECT_MAP_ENTER(&localrejectentry->keyitem, &found); + HASHACTION action = + check_hash_fullness(disk_quota_reject_map, MAX_DISK_QUOTA_REJECT_ENTRIES, REJECT_MAP_WARNING, + &disk_quota_reject_last_overflow_report, MAX_DISK_QUOTA_REJECT_ENTRIES); + rejectentry = hash_search(disk_quota_reject_map, &localrejectentry->keyitem, action, &found); if (rejectentry == NULL) { continue; @@ -2113,7 +2116,10 @@ refresh_rejectmap(PG_FUNCTION_ARGS) */ if (OidIsValid(rejectmapentry->keyitem.targetoid)) continue; - new_entry = REJECT_MAP_ENTER(&rejectmapentry->keyitem, &found); + HASHACTION action = + check_hash_fullness(disk_quota_reject_map, MAX_DISK_QUOTA_REJECT_ENTRIES, REJECT_MAP_WARNING, + &disk_quota_reject_last_overflow_report, MAX_DISK_QUOTA_REJECT_ENTRIES); + new_entry = hash_search(disk_quota_reject_map, &rejectmapentry->keyitem, action, &found); if (!found && new_entry) memcpy(new_entry, rejectmapentry, sizeof(GlobalRejectMapEntry)); } LWLockRelease(diskquota_locks.reject_map_lock); diff --git a/src/relation_cache.c b/src/relation_cache.c index 94633903..5d742334 100644 --- a/src/relation_cache.c +++ b/src/relation_cache.c @@ -34,19 +34,15 @@ HTAB *relid_cache = NULL; extern TimestampTz active_tables_last_overflow_report; -#define RELATION_CACHE_ENTER(keyPtr, foundPtr) \ - shm_hash_enter(relation_cache, keyPtr, foundPtr, diskquota_max_active_tables, \ - "[diskquota] the number of relation cache entries reached the limit, please increase " \ - "the GUC value for diskquota.max_active_tables. Current " \ - "diskquota.max_active_tables value: %d", \ - &active_tables_last_overflow_report, diskquota_max_active_tables) - -#define RELID_CACHE_ENTER(keyPtr, foundPtr) \ - shm_hash_enter(relid_cache, keyPtr, foundPtr, diskquota_max_active_tables, \ - "[diskquota] the number of relid cache entries reached the limit, please increase " \ - "the GUC value for diskquota.max_active_tables. Current " \ - "diskquota.max_active_tables value: %d", \ - &active_tables_last_overflow_report, diskquota_max_active_tables) +#define RELATION_CACHE_WARNING \ + "[diskquota] the number of relation cache entries reached the limit, please increase " \ + "the GUC value for diskquota.max_active_tables. Current " \ + "diskquota.max_active_tables value: %d" + +#define RELID_CACHE_WARNING \ + "[diskquota] the number of relation cache entries reached the limit, please increase " \ + "the GUC value for diskquota.max_active_tables. Current " \ + "diskquota.max_active_tables value: %d" static void update_relation_entry(Oid relid, DiskQuotaRelationCacheEntry *relation_entry, DiskQuotaRelidCacheEntry *relid_entry); @@ -189,18 +185,26 @@ update_relation_cache(Oid relid) DiskQuotaRelidCacheEntry relid_entry_data = {0}; DiskQuotaRelidCacheEntry *relid_entry; Oid prelid; + HASHACTION action; update_relation_entry(relid, &relation_entry_data, &relid_entry_data); LWLockAcquire(diskquota_locks.relation_cache_lock, LW_EXCLUSIVE); - relation_entry = RELATION_CACHE_ENTER(&relation_entry_data.relid, NULL); + + action = check_hash_fullness(relation_cache, diskquota_max_active_tables, RELATION_CACHE_WARNING, + &active_tables_last_overflow_report, diskquota_max_active_tables); + relation_entry = hash_search(relation_cache, &relation_entry_data.relid, action, NULL); + if (relation_entry == NULL) { LWLockRelease(diskquota_locks.relation_cache_lock); return; } memcpy(relation_entry, &relation_entry_data, sizeof(DiskQuotaRelationCacheEntry)); - relid_entry = RELID_CACHE_ENTER(&relid_entry_data.relfilenode, NULL); + + action = check_hash_fullness(relid_cache, diskquota_max_active_tables, RELID_CACHE_WARNING, + &active_tables_last_overflow_report, diskquota_max_active_tables); + relid_entry = hash_search(relid_cache, &relid_entry_data.relfilenode, action, NULL); if (relid_entry == NULL) { LWLockRelease(diskquota_locks.relation_cache_lock); From cdc9414bef50f7e398e5b542f67d2468a10a3276 Mon Sep 17 00:00:00 2001 From: Viktor Kurilko Date: Wed, 13 Dec 2023 19:47:21 +0700 Subject: [PATCH 14/31] fix format --- src/diskquota.h | 4 ++-- src/quotamodel.c | 14 ++++++++------ 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/src/diskquota.h b/src/diskquota.h index bbb62052..1999caa6 100644 --- a/src/diskquota.h +++ b/src/diskquota.h @@ -317,6 +317,6 @@ extern HTAB *diskquota_hash_create(const char *tabname, long nelem, HASHC extern HTAB *DiskquotaShmemInitHash(const char *name, long init_size, long max_size, HASHCTL *infoP, int hash_flags, DiskquotaHashFunction hash_function); extern void refresh_monitored_dbid_cache(void); -extern HASHACTION check_hash_fullness(HTAB *hashp, int max_size, const char *warning_message, - TimestampTz *last_overflow_report, int guc_value); +extern HASHACTION check_hash_fullness(HTAB *hashp, int max_size, const char *warning_message, + TimestampTz *last_overflow_report, int guc_value); #endif diff --git a/src/quotamodel.c b/src/quotamodel.c index 5d74918c..d29d002f 100644 --- a/src/quotamodel.c +++ b/src/quotamodel.c @@ -249,13 +249,14 @@ update_size_for_quota(int64 size, QuotaType type, Oid *keys, int16 segid) bool found; QuotaInfoEntry *entry; QuotaInfoEntryKey key = {0}; + HASHACTION action; memcpy(key.keys, keys, quota_key_num[type] * sizeof(Oid)); key.type = type; key.segid = segid; - HASHACTION action = check_hash_fullness(quota_info_map, diskquota_max_quota_probes, QUOTA_INFO_WARNING, - "a_info_map_last_overflow_report, diskquota_max_quota_probes); - entry = hash_search(quota_info_map, &key, action, &found); + action = check_hash_fullness(quota_info_map, diskquota_max_quota_probes, QUOTA_INFO_WARNING, + "a_info_map_last_overflow_report, diskquota_max_quota_probes); + entry = hash_search(quota_info_map, &key, action, &found); /* If the number of quota exceeds the limit, entry will be NULL */ if (entry == NULL) return; if (!found) @@ -275,13 +276,14 @@ update_limit_for_quota(int64 limit, float segratio, QuotaType type, Oid *keys) { QuotaInfoEntry *entry; QuotaInfoEntryKey key = {0}; + HASHACTION action; memcpy(key.keys, keys, quota_key_num[type] * sizeof(Oid)); key.type = type; key.segid = i; - HASHACTION action = check_hash_fullness(quota_info_map, diskquota_max_quota_probes, QUOTA_INFO_WARNING, - "a_info_map_last_overflow_report, diskquota_max_quota_probes); - entry = hash_search(quota_info_map, &key, action, &found); + action = check_hash_fullness(quota_info_map, diskquota_max_quota_probes, QUOTA_INFO_WARNING, + "a_info_map_last_overflow_report, diskquota_max_quota_probes); + entry = hash_search(quota_info_map, &key, action, &found); /* If the number of quota exceeds the limit, entry will be NULL */ if (entry == NULL) continue; if (!found) From db86b9b79146d5cf1e8ab5d5456b47f4933ea909 Mon Sep 17 00:00:00 2001 From: Viktor Kurilko Date: Thu, 14 Dec 2023 14:43:31 +0700 Subject: [PATCH 15/31] change warning messages --- src/diskquota_utility.c | 9 ++++++--- src/gp_activetable.c | 5 ++--- src/quotamodel.c | 12 ++++++------ src/relation_cache.c | 4 ++-- 4 files changed, 16 insertions(+), 14 deletions(-) diff --git a/src/diskquota_utility.c b/src/diskquota_utility.c index 4827bc42..253b2726 100644 --- a/src/diskquota_utility.c +++ b/src/diskquota_utility.c @@ -1686,20 +1686,23 @@ DiskquotaShmemInitHash(const char *name, /* table string name fo /* * Returns HASH_FIND if hash table is full and HASH_ENTER otherwise. + * It can be used only under lock. */ HASHACTION check_hash_fullness(HTAB *hashp, int max_size, const char *warning_message, TimestampTz *last_overflow_report, int guc_value) { - if (hash_get_num_entries(hashp) < max_size) return HASH_ENTER; + long num_entries = hash_get_num_entries(hashp); - if (hash_get_num_entries(hashp) == max_size) + if (num_entries < max_size) return HASH_ENTER; + + if (num_entries == max_size) { TimestampTz current_time = GetCurrentTimestamp(); if (TimestampDifferenceExceeds(*last_overflow_report, current_time, diskquota_hashmap_overflow_report_timeout * 1000)) { - ereport(WARNING, (errmsg(warning_message, guc_value))); + ereport(WARNING, (errmsg("%s %d", warning_message, guc_value))); *last_overflow_report = current_time; } } diff --git a/src/gp_activetable.c b/src/gp_activetable.c index 0bcae887..c55a0ab2 100644 --- a/src/gp_activetable.c +++ b/src/gp_activetable.c @@ -57,7 +57,7 @@ TimestampTz active_tables_last_overflow_report = 0; #define ACTIVE_TABLE_WARNING \ "[diskquota] the number of active tables reached the limit, please increase " \ "the GUC value for diskquota.max_active_tables. Current " \ - "diskquota.max_active_tables value: %d" + "diskquota.max_active_tables value:" /* * monitored_dbid_cache is a allow list for diskquota @@ -72,7 +72,7 @@ TimestampTz altered_reloid_cache_last_overflow_report = 0; #define ALTERED_RELOID_CACHE_WARNING \ "[diskquota] the number of altered reloid cache entries reached the limit, please increase " \ "the GUC value for diskquota.max_active_tables. Current " \ - "diskquota.max_active_tables value: %d" + "diskquota.max_active_tables value:" /* active table hooks which detect the disk file size change. */ static file_create_hook_type prev_file_create_hook = NULL; @@ -865,7 +865,6 @@ get_active_tables_oid(void) hash_seq_init(&iter, local_active_table_file_map); while ((active_table_file_entry = (DiskQuotaActiveTableFileEntry *)hash_seq_search(&iter)) != NULL) { - /* TODO: handle possible ERROR here so that the bgworker will not go down. */ HASHACTION action = check_hash_fullness(active_tables_map, diskquota_max_active_tables, ACTIVE_TABLE_WARNING, &active_tables_last_overflow_report, diskquota_max_active_tables); hash_search(active_tables_map, active_table_file_entry, action, NULL); diff --git a/src/quotamodel.c b/src/quotamodel.c index d29d002f..98bacc52 100644 --- a/src/quotamodel.c +++ b/src/quotamodel.c @@ -148,7 +148,7 @@ TimestampTz quota_info_map_last_overflow_report = 0; #define QUOTA_INFO_WARNING \ "[diskquota] the number of quota probe reached the limit, please " \ "increase the GUC value for diskquota.max_quota_probes. Current " \ - "diskquota.max_quota_probes value: %d" + "diskquota.max_quota_probes value:" /* global rejectmap for which exceed their quota limit */ struct RejectMapEntry @@ -194,7 +194,7 @@ static TimestampTz table_size_last_overflow_report = 0; #define TABLE_SIZE_WARNING \ "[diskquota] the number of tables reached the limit, please increase " \ "the GUC value for diskquota.max_table_segments. Current " \ - "diskquota.max_table_segments value: %d" + "diskquota.max_table_segments value:" /* rejectmap for database objects which exceed their quota limit */ static HTAB *disk_quota_reject_map = NULL; @@ -203,11 +203,11 @@ static HTAB *local_disk_quota_reject_map = NULL; static TimestampTz disk_quota_reject_last_overflow_report = 0; static TimestampTz local_disk_quota_reject_last_overflow_report = 0; -#define REJECT_MAP_WARNING \ - "[diskquota] Shared disk quota reject map size limit reached (%d). " \ - "Some out-of-limit schemas or roles will be lost in rejectmap." +#define REJECT_MAP_WARNING \ + "[diskquota] Shared disk quota reject map size limit reached. " \ + "Some out-of-limit schemas or roles will be lost in rejectmap. Current limit:" -#define LOCAL_REJECT_MAP_WARNING "[diskquota] the number of local reject map entries reached the limit (%d)" +#define LOCAL_REJECT_MAP_WARNING "[diskquota] the number of local reject map entries reached the limit:" static shmem_startup_hook_type prev_shmem_startup_hook = NULL; diff --git a/src/relation_cache.c b/src/relation_cache.c index 5d742334..d97397f7 100644 --- a/src/relation_cache.c +++ b/src/relation_cache.c @@ -37,12 +37,12 @@ extern TimestampTz active_tables_last_overflow_report; #define RELATION_CACHE_WARNING \ "[diskquota] the number of relation cache entries reached the limit, please increase " \ "the GUC value for diskquota.max_active_tables. Current " \ - "diskquota.max_active_tables value: %d" + "diskquota.max_active_tables value:" #define RELID_CACHE_WARNING \ "[diskquota] the number of relation cache entries reached the limit, please increase " \ "the GUC value for diskquota.max_active_tables. Current " \ - "diskquota.max_active_tables value: %d" + "diskquota.max_active_tables value:" static void update_relation_entry(Oid relid, DiskQuotaRelationCacheEntry *relation_entry, DiskQuotaRelidCacheEntry *relid_entry); From 652c111629c73b24cf26af5f4746340efafe5479 Mon Sep 17 00:00:00 2001 From: Viktor Kurilko Date: Thu, 14 Dec 2023 18:20:49 +0700 Subject: [PATCH 16/31] change macro names --- src/diskquota_utility.c | 2 ++ src/gp_activetable.c | 11 ++++++----- src/quotamodel.c | 28 ++++++++++++++-------------- 3 files changed, 22 insertions(+), 19 deletions(-) diff --git a/src/diskquota_utility.c b/src/diskquota_utility.c index 253b2726..04bb5696 100644 --- a/src/diskquota_utility.c +++ b/src/diskquota_utility.c @@ -1699,6 +1699,7 @@ check_hash_fullness(HTAB *hashp, int max_size, const char *warning_message, Time if (num_entries == max_size) { TimestampTz current_time = GetCurrentTimestamp(); + if (TimestampDifferenceExceeds(*last_overflow_report, current_time, diskquota_hashmap_overflow_report_timeout * 1000)) { @@ -1706,5 +1707,6 @@ check_hash_fullness(HTAB *hashp, int max_size, const char *warning_message, Time *last_overflow_report = current_time; } } + return HASH_FIND; } diff --git a/src/gp_activetable.c b/src/gp_activetable.c index c55a0ab2..17491048 100644 --- a/src/gp_activetable.c +++ b/src/gp_activetable.c @@ -54,7 +54,7 @@ typedef struct DiskQuotaSetOFCache HTAB *active_tables_map = NULL; // Set TimestampTz active_tables_last_overflow_report = 0; -#define ACTIVE_TABLE_WARNING \ +#define ACTIVE_TABLES_MAP_WARNING \ "[diskquota] the number of active tables reached the limit, please increase " \ "the GUC value for diskquota.max_active_tables. Current " \ "diskquota.max_active_tables value:" @@ -333,7 +333,7 @@ report_active_table_helper(const RelFileNodeBackend *relFileNode) item.tablespaceoid = relFileNode->node.spcNode; LWLockAcquire(diskquota_locks.active_table_lock, LW_EXCLUSIVE); - HASHACTION action = check_hash_fullness(active_tables_map, diskquota_max_active_tables, ACTIVE_TABLE_WARNING, + HASHACTION action = check_hash_fullness(active_tables_map, diskquota_max_active_tables, ACTIVE_TABLES_MAP_WARNING, &active_tables_last_overflow_report, diskquota_max_active_tables); entry = hash_search(active_tables_map, &item, action, &found); if (entry && !found) *entry = item; @@ -865,8 +865,9 @@ get_active_tables_oid(void) hash_seq_init(&iter, local_active_table_file_map); while ((active_table_file_entry = (DiskQuotaActiveTableFileEntry *)hash_seq_search(&iter)) != NULL) { - HASHACTION action = check_hash_fullness(active_tables_map, diskquota_max_active_tables, ACTIVE_TABLE_WARNING, - &active_tables_last_overflow_report, diskquota_max_active_tables); + HASHACTION action = + check_hash_fullness(active_tables_map, diskquota_max_active_tables, ACTIVE_TABLES_MAP_WARNING, + &active_tables_last_overflow_report, diskquota_max_active_tables); hash_search(active_tables_map, active_table_file_entry, action, NULL); } /* TODO: hash_seq_term(&iter); */ @@ -930,7 +931,7 @@ get_active_tables_oid(void) while ((active_table_file_entry = (DiskQuotaActiveTableFileEntry *)hash_seq_search(&iter)) != NULL) { HASHACTION action = - check_hash_fullness(active_tables_map, diskquota_max_active_tables, ACTIVE_TABLE_WARNING, + check_hash_fullness(active_tables_map, diskquota_max_active_tables, ACTIVE_TABLES_MAP_WARNING, &active_tables_last_overflow_report, diskquota_max_active_tables); entry = hash_search(active_tables_map, active_table_file_entry, action, &found); if (entry) *entry = *active_table_file_entry; diff --git a/src/quotamodel.c b/src/quotamodel.c index 98bacc52..a8e7c6fc 100644 --- a/src/quotamodel.c +++ b/src/quotamodel.c @@ -145,7 +145,7 @@ Oid quota_key_caches[NUM_QUOTA_TYPES][MAX_NUM_KEYS_QUOTA_MAP] = { HTAB *quota_info_map; TimestampTz quota_info_map_last_overflow_report = 0; -#define QUOTA_INFO_WARNING \ +#define QUOTA_INFO_MAP_WARNING \ "[diskquota] the number of quota probe reached the limit, please " \ "increase the GUC value for diskquota.max_quota_probes. Current " \ "diskquota.max_quota_probes value:" @@ -191,7 +191,7 @@ struct LocalRejectMapEntry static HTAB *table_size_map = NULL; static TimestampTz table_size_last_overflow_report = 0; -#define TABLE_SIZE_WARNING \ +#define TABLE_SIZE_MAP_WARNING \ "[diskquota] the number of tables reached the limit, please increase " \ "the GUC value for diskquota.max_table_segments. Current " \ "diskquota.max_table_segments value:" @@ -203,11 +203,11 @@ static HTAB *local_disk_quota_reject_map = NULL; static TimestampTz disk_quota_reject_last_overflow_report = 0; static TimestampTz local_disk_quota_reject_last_overflow_report = 0; -#define REJECT_MAP_WARNING \ +#define DISK_QUOTA_REJECT_MAP_WARNING \ "[diskquota] Shared disk quota reject map size limit reached. " \ "Some out-of-limit schemas or roles will be lost in rejectmap. Current limit:" -#define LOCAL_REJECT_MAP_WARNING "[diskquota] the number of local reject map entries reached the limit:" +#define LOCAL_DISK_QUOTA_REJECT_MAP_WARNING "[diskquota] the number of local reject map entries reached the limit:" static shmem_startup_hook_type prev_shmem_startup_hook = NULL; @@ -254,7 +254,7 @@ update_size_for_quota(int64 size, QuotaType type, Oid *keys, int16 segid) memcpy(key.keys, keys, quota_key_num[type] * sizeof(Oid)); key.type = type; key.segid = segid; - action = check_hash_fullness(quota_info_map, diskquota_max_quota_probes, QUOTA_INFO_WARNING, + action = check_hash_fullness(quota_info_map, diskquota_max_quota_probes, QUOTA_INFO_MAP_WARNING, "a_info_map_last_overflow_report, diskquota_max_quota_probes); entry = hash_search(quota_info_map, &key, action, &found); /* If the number of quota exceeds the limit, entry will be NULL */ @@ -281,7 +281,7 @@ update_limit_for_quota(int64 limit, float segratio, QuotaType type, Oid *keys) memcpy(key.keys, keys, quota_key_num[type] * sizeof(Oid)); key.type = type; key.segid = i; - action = check_hash_fullness(quota_info_map, diskquota_max_quota_probes, QUOTA_INFO_WARNING, + action = check_hash_fullness(quota_info_map, diskquota_max_quota_probes, QUOTA_INFO_MAP_WARNING, "a_info_map_last_overflow_report, diskquota_max_quota_probes); entry = hash_search(quota_info_map, &key, action, &found); /* If the number of quota exceeds the limit, entry will be NULL */ @@ -312,9 +312,9 @@ add_quota_to_rejectmap(QuotaType type, Oid targetOid, Oid tablespaceoid, bool se keyitem.tablespaceoid = tablespaceoid; keyitem.targettype = (uint32)type; ereport(DEBUG1, (errmsg("[diskquota] Put object %u to rejectmap", targetOid))); - HASHACTION action = - check_hash_fullness(local_disk_quota_reject_map, MAX_DISK_QUOTA_REJECT_ENTRIES, LOCAL_REJECT_MAP_WARNING, - &local_disk_quota_reject_last_overflow_report, MAX_DISK_QUOTA_REJECT_ENTRIES); + HASHACTION action = check_hash_fullness( + local_disk_quota_reject_map, MAX_DISK_QUOTA_REJECT_ENTRIES, LOCAL_DISK_QUOTA_REJECT_MAP_WARNING, + &local_disk_quota_reject_last_overflow_report, MAX_DISK_QUOTA_REJECT_ENTRIES); localrejectentry = hash_search(local_disk_quota_reject_map, &keyitem, action, NULL); if (localrejectentry) { @@ -990,7 +990,7 @@ calculate_table_disk_usage(bool is_init, HTAB *local_active_table_stat_map) key.reloid = relOid; key.id = TableSizeEntryId(cur_segid); - HASHACTION action = check_hash_fullness(table_size_map, MAX_NUM_TABLE_SIZE_ENTRIES, TABLE_SIZE_WARNING, + HASHACTION action = check_hash_fullness(table_size_map, MAX_NUM_TABLE_SIZE_ENTRIES, TABLE_SIZE_MAP_WARNING, &table_size_last_overflow_report, diskquota_max_table_segments); tsentry = hash_search(table_size_map, &key, action, &table_size_map_found); @@ -1279,9 +1279,9 @@ flush_local_reject_map(void) */ if (localrejectentry->isexceeded) { - HASHACTION action = - check_hash_fullness(disk_quota_reject_map, MAX_DISK_QUOTA_REJECT_ENTRIES, REJECT_MAP_WARNING, - &disk_quota_reject_last_overflow_report, MAX_DISK_QUOTA_REJECT_ENTRIES); + HASHACTION action = check_hash_fullness( + disk_quota_reject_map, MAX_DISK_QUOTA_REJECT_ENTRIES, DISK_QUOTA_REJECT_MAP_WARNING, + &disk_quota_reject_last_overflow_report, MAX_DISK_QUOTA_REJECT_ENTRIES); rejectentry = hash_search(disk_quota_reject_map, &localrejectentry->keyitem, action, &found); if (rejectentry == NULL) { @@ -2119,7 +2119,7 @@ refresh_rejectmap(PG_FUNCTION_ARGS) if (OidIsValid(rejectmapentry->keyitem.targetoid)) continue; HASHACTION action = - check_hash_fullness(disk_quota_reject_map, MAX_DISK_QUOTA_REJECT_ENTRIES, REJECT_MAP_WARNING, + check_hash_fullness(disk_quota_reject_map, MAX_DISK_QUOTA_REJECT_ENTRIES, DISK_QUOTA_REJECT_MAP_WARNING, &disk_quota_reject_last_overflow_report, MAX_DISK_QUOTA_REJECT_ENTRIES); new_entry = hash_search(disk_quota_reject_map, &rejectmapentry->keyitem, action, &found); if (!found && new_entry) memcpy(new_entry, rejectmapentry, sizeof(GlobalRejectMapEntry)); From 785fecd5f44ae1aef8a5cee30cb30599f8069298 Mon Sep 17 00:00:00 2001 From: Viktor Kurilko Date: Thu, 21 Dec 2023 12:12:44 +0700 Subject: [PATCH 17/31] use an external table to check warnings in the test. --- .../expected/test_activetable_limit.out | 31 ++++++++++++++----- tests/regress/sql/test_activetable_limit.sql | 15 +++++++-- 2 files changed, 35 insertions(+), 11 deletions(-) diff --git a/tests/regress/expected/test_activetable_limit.out b/tests/regress/expected/test_activetable_limit.out index 017e38c5..7699f2cc 100644 --- a/tests/regress/expected/test_activetable_limit.out +++ b/tests/regress/expected/test_activetable_limit.out @@ -14,6 +14,9 @@ INSERT INTO a02 values(generate_series(0, 500)); INSERT INTO a03 values(generate_series(0, 500)); \c test_tablenum_limit_02 CREATE EXTENSION diskquota; +CREATE EXTERNAL WEB TABLE segment_logs(line text) + EXECUTE 'cat $GP_SEG_DATADIR/pg_log/$(ls -Art $GP_SEG_DATADIR/pg_log | tail -n 1)' + ON ALL FORMAT 'TEXT' (DELIMITER 'OFF'); CREATE SCHEMA s; SELECT diskquota.set_schema_quota('s', '1 MB'); set_schema_quota @@ -41,9 +44,15 @@ NOTICE: CREATE TABLE will create partition "t1_1_prt_8" for table "t1" NOTICE: CREATE TABLE will create partition "t1_1_prt_9" for table "t1" NOTICE: CREATE TABLE will create partition "t1_1_prt_10" for table "t1" WARNING: [diskquota] the number of active tables reached the limit, please increase the GUC value for diskquota.max_active_tables. Current diskquota.max_active_tables value: 5 -WARNING: [diskquota] the number of active tables reached the limit, please increase the GUC value for diskquota.max_active_tables. Current diskquota.max_active_tables value: 5 (seg0 127.0.1.1:6002 pid=281310) -WARNING: [diskquota] the number of active tables reached the limit, please increase the GUC value for diskquota.max_active_tables. Current diskquota.max_active_tables value: 5 (seg1 127.0.1.1:6003 pid=281312) -WARNING: [diskquota] the number of active tables reached the limit, please increase the GUC value for diskquota.max_active_tables. Current diskquota.max_active_tables value: 5 (seg2 127.0.1.1:6004 pid=281311) +WARNING: [diskquota] the number of active tables reached the limit, please increase the GUC value for diskquota.max_active_tables. Current diskquota.max_active_tables value: 5 (seg0 127.0.1.1:6002 pid=195302) +WARNING: [diskquota] the number of active tables reached the limit, please increase the GUC value for diskquota.max_active_tables. Current diskquota.max_active_tables value: 5 (seg1 127.0.1.1:6003 pid=195303) +WARNING: [diskquota] the number of active tables reached the limit, please increase the GUC value for diskquota.max_active_tables. Current diskquota.max_active_tables value: 5 (seg2 127.0.1.1:6004 pid=195304) +SELECT count(*) FROM segment_logs WHERE line LIKE '%the number of active tables reached the limit%'; + count +------- + 3 +(1 row) + CREATE TABLE s.t2(i int) DISTRIBUTED BY (i); INSERT INTO s.t2 SELECT generate_series(1, 100000); SELECT diskquota.wait_for_worker_new_epoch(); @@ -53,17 +62,23 @@ SELECT diskquota.wait_for_worker_new_epoch(); (1 row) INSERT INTO s.t1 SELECT a, a from generate_series(0, 9)a; -- should be successful -select count(*) from s.t1; +SELECT count(*) FROM s.t1; count ------- 10 (1 row) -- altered reloid cache overflow check. expected warning. -vacuum full; -WARNING: [diskquota] the number of altered reloid cache entries reached the limit, please increase the GUC value for diskquota.max_active_tables. Current diskquota.max_active_tables value: 5 (seg0 127.0.1.1:6002 pid=281310) -WARNING: [diskquota] the number of altered reloid cache entries reached the limit, please increase the GUC value for diskquota.max_active_tables. Current diskquota.max_active_tables value: 5 (seg1 127.0.1.1:6003 pid=281312) -WARNING: [diskquota] the number of altered reloid cache entries reached the limit, please increase the GUC value for diskquota.max_active_tables. Current diskquota.max_active_tables value: 5 (seg2 127.0.1.1:6004 pid=281311) +VACUUM FULL; +WARNING: [diskquota] the number of altered reloid cache entries reached the limit, please increase the GUC value for diskquota.max_active_tables. Current diskquota.max_active_tables value: 5 (seg2 127.0.1.1:6004 pid=195304) +WARNING: [diskquota] the number of altered reloid cache entries reached the limit, please increase the GUC value for diskquota.max_active_tables. Current diskquota.max_active_tables value: 5 (seg0 127.0.1.1:6002 pid=195302) +WARNING: [diskquota] the number of altered reloid cache entries reached the limit, please increase the GUC value for diskquota.max_active_tables. Current diskquota.max_active_tables value: 5 (seg1 127.0.1.1:6003 pid=195303) +SELECT count(*) FROM segment_logs WHERE line LIKE '%the number of altered reloid cache entries reached the limit%'; + count +------- + 3 +(1 row) + DROP EXTENSION diskquota; \c contrib_regression DROP DATABASE test_tablenum_limit_01; diff --git a/tests/regress/sql/test_activetable_limit.sql b/tests/regress/sql/test_activetable_limit.sql index f06f9df9..bd94e8a0 100644 --- a/tests/regress/sql/test_activetable_limit.sql +++ b/tests/regress/sql/test_activetable_limit.sql @@ -20,6 +20,11 @@ INSERT INTO a03 values(generate_series(0, 500)); \c test_tablenum_limit_02 CREATE EXTENSION diskquota; + +CREATE EXTERNAL WEB TABLE segment_logs(line text) + EXECUTE 'cat $GP_SEG_DATADIR/pg_log/$(ls -Art $GP_SEG_DATADIR/pg_log | tail -n 1)' + ON ALL FORMAT 'TEXT' (DELIMITER 'OFF'); + CREATE SCHEMA s; SELECT diskquota.set_schema_quota('s', '1 MB'); @@ -28,17 +33,21 @@ SELECT diskquota.wait_for_worker_new_epoch(); -- We create twice as many tables as the limit to ensure that the active_tables table is overflow. CREATE TABLE s.t1 (a int, b int) DISTRIBUTED BY (a) PARTITION BY RANGE (b) ( START (0) END (10) EVERY (1) ); -CREATE TABLE s.t2(i int) DISTRIBUTED BY (i); +SELECT count(*) FROM segment_logs WHERE line LIKE '%the number of active tables reached the limit%'; + +CREATE TABLE s.t2(i int) DISTRIBUTED BY (i); INSERT INTO s.t2 SELECT generate_series(1, 100000); SELECT diskquota.wait_for_worker_new_epoch(); INSERT INTO s.t1 SELECT a, a from generate_series(0, 9)a; -- should be successful -select count(*) from s.t1; +SELECT count(*) FROM s.t1; -- altered reloid cache overflow check. expected warning. -vacuum full; +VACUUM FULL; + +SELECT count(*) FROM segment_logs WHERE line LIKE '%the number of altered reloid cache entries reached the limit%'; DROP EXTENSION diskquota; From cb4e1bf01aed18de2c39c1607d03003427995ee2 Mon Sep 17 00:00:00 2001 From: Viktor Kurilko Date: Fri, 22 Dec 2023 15:40:26 +0700 Subject: [PATCH 18/31] rename waribles. remove GUC value from warning messages. --- src/diskquota.h | 4 +- src/diskquota_utility.c | 9 ++-- src/gp_activetable.c | 31 ++++++-------- src/gp_activetable.h | 2 - src/quotamodel.c | 42 +++++++++---------- src/relation_cache.c | 12 +++--- .../expected/test_activetable_limit.out | 9 +--- 7 files changed, 46 insertions(+), 63 deletions(-) diff --git a/src/diskquota.h b/src/diskquota.h index 1999caa6..d9ba794b 100644 --- a/src/diskquota.h +++ b/src/diskquota.h @@ -317,6 +317,6 @@ extern HTAB *diskquota_hash_create(const char *tabname, long nelem, HASHC extern HTAB *DiskquotaShmemInitHash(const char *name, long init_size, long max_size, HASHCTL *infoP, int hash_flags, DiskquotaHashFunction hash_function); extern void refresh_monitored_dbid_cache(void); -extern HASHACTION check_hash_fullness(HTAB *hashp, int max_size, const char *warning_message, - TimestampTz *last_overflow_report, int guc_value); +extern HASHACTION check_hash_fullness(HTAB *hashp, int max_size, const char *warning_message, + TimestampTz *last_overflow_report); #endif diff --git a/src/diskquota_utility.c b/src/diskquota_utility.c index 04bb5696..d576f54a 100644 --- a/src/diskquota_utility.c +++ b/src/diskquota_utility.c @@ -1689,8 +1689,7 @@ DiskquotaShmemInitHash(const char *name, /* table string name fo * It can be used only under lock. */ HASHACTION -check_hash_fullness(HTAB *hashp, int max_size, const char *warning_message, TimestampTz *last_overflow_report, - int guc_value) +check_hash_fullness(HTAB *hashp, int max_size, const char *warning_message, TimestampTz *last_overflow_report) { long num_entries = hash_get_num_entries(hashp); @@ -1700,10 +1699,10 @@ check_hash_fullness(HTAB *hashp, int max_size, const char *warning_message, Time { TimestampTz current_time = GetCurrentTimestamp(); - if (TimestampDifferenceExceeds(*last_overflow_report, current_time, - diskquota_hashmap_overflow_report_timeout * 1000)) + if (*last_overflow_report == 0 || TimestampDifferenceExceeds(*last_overflow_report, current_time, + diskquota_hashmap_overflow_report_timeout * 1000)) { - ereport(WARNING, (errmsg("%s %d", warning_message, guc_value))); + ereport(WARNING, (errmsg(warning_message))); *last_overflow_report = current_time; } } diff --git a/src/gp_activetable.c b/src/gp_activetable.c index 17491048..fb1dc939 100644 --- a/src/gp_activetable.c +++ b/src/gp_activetable.c @@ -51,13 +51,12 @@ typedef struct DiskQuotaSetOFCache HASH_SEQ_STATUS pos; } DiskQuotaSetOFCache; -HTAB *active_tables_map = NULL; // Set -TimestampTz active_tables_last_overflow_report = 0; +static HTAB *active_tables_map = NULL; // Set +TimestampTz active_tables_map_last_overflow_report = 0; #define ACTIVE_TABLES_MAP_WARNING \ "[diskquota] the number of active tables reached the limit, please increase " \ - "the GUC value for diskquota.max_active_tables. Current " \ - "diskquota.max_active_tables value:" + "the GUC value for diskquota.max_active_tables." /* * monitored_dbid_cache is a allow list for diskquota @@ -66,13 +65,12 @@ TimestampTz active_tables_last_overflow_report = 0; * dbid will be added to it when creating diskquota extension * dbid will be removed from it when droping diskquota extension */ -HTAB *altered_reloid_cache = NULL; // Set -TimestampTz altered_reloid_cache_last_overflow_report = 0; +static HTAB *altered_reloid_cache = NULL; // Set +static TimestampTz altered_reloid_cache_last_overflow_report = 0; #define ALTERED_RELOID_CACHE_WARNING \ "[diskquota] the number of altered reloid cache entries reached the limit, please increase " \ - "the GUC value for diskquota.max_active_tables. Current " \ - "diskquota.max_active_tables value:" + "the GUC value for diskquota.max_active_tables." /* active table hooks which detect the disk file size change. */ static file_create_hook_type prev_file_create_hook = NULL; @@ -248,9 +246,8 @@ report_altered_reloid(Oid reloid) if (IsRoleMirror() || IS_QUERY_DISPATCHER()) return; LWLockAcquire(diskquota_locks.altered_reloid_cache_lock, LW_EXCLUSIVE); - HASHACTION action = - check_hash_fullness(altered_reloid_cache, diskquota_max_active_tables, ALTERED_RELOID_CACHE_WARNING, - &altered_reloid_cache_last_overflow_report, diskquota_max_active_tables); + HASHACTION action = check_hash_fullness(altered_reloid_cache, diskquota_max_active_tables, + ALTERED_RELOID_CACHE_WARNING, &altered_reloid_cache_last_overflow_report); hash_search(altered_reloid_cache, &reloid, action, NULL); LWLockRelease(diskquota_locks.altered_reloid_cache_lock); } @@ -334,7 +331,7 @@ report_active_table_helper(const RelFileNodeBackend *relFileNode) LWLockAcquire(diskquota_locks.active_table_lock, LW_EXCLUSIVE); HASHACTION action = check_hash_fullness(active_tables_map, diskquota_max_active_tables, ACTIVE_TABLES_MAP_WARNING, - &active_tables_last_overflow_report, diskquota_max_active_tables); + &active_tables_map_last_overflow_report); entry = hash_search(active_tables_map, &item, action, &found); if (entry && !found) *entry = item; @@ -865,9 +862,8 @@ get_active_tables_oid(void) hash_seq_init(&iter, local_active_table_file_map); while ((active_table_file_entry = (DiskQuotaActiveTableFileEntry *)hash_seq_search(&iter)) != NULL) { - HASHACTION action = - check_hash_fullness(active_tables_map, diskquota_max_active_tables, ACTIVE_TABLES_MAP_WARNING, - &active_tables_last_overflow_report, diskquota_max_active_tables); + HASHACTION action = check_hash_fullness(active_tables_map, diskquota_max_active_tables, + ACTIVE_TABLES_MAP_WARNING, &active_tables_map_last_overflow_report); hash_search(active_tables_map, active_table_file_entry, action, NULL); } /* TODO: hash_seq_term(&iter); */ @@ -930,9 +926,8 @@ get_active_tables_oid(void) LWLockAcquire(diskquota_locks.active_table_lock, LW_EXCLUSIVE); while ((active_table_file_entry = (DiskQuotaActiveTableFileEntry *)hash_seq_search(&iter)) != NULL) { - HASHACTION action = - check_hash_fullness(active_tables_map, diskquota_max_active_tables, ACTIVE_TABLES_MAP_WARNING, - &active_tables_last_overflow_report, diskquota_max_active_tables); + HASHACTION action = check_hash_fullness(active_tables_map, diskquota_max_active_tables, + ACTIVE_TABLES_MAP_WARNING, &active_tables_map_last_overflow_report); entry = hash_search(active_tables_map, active_table_file_entry, action, &found); if (entry) *entry = *active_table_file_entry; } diff --git a/src/gp_activetable.h b/src/gp_activetable.h index 6b513fe9..1ff10d60 100644 --- a/src/gp_activetable.h +++ b/src/gp_activetable.h @@ -42,9 +42,7 @@ extern void init_active_table_hook(void); extern void init_shm_worker_active_tables(void); extern void init_lock_active_tables(void); -extern HTAB *active_tables_map; extern HTAB *monitored_dbid_cache; -extern HTAB *altered_reloid_cache; #ifndef atooid #define atooid(x) ((Oid)strtoul((x), NULL, 10)) diff --git a/src/quotamodel.c b/src/quotamodel.c index a8e7c6fc..67676a0b 100644 --- a/src/quotamodel.c +++ b/src/quotamodel.c @@ -142,13 +142,12 @@ typedef enum uint16 quota_key_num[NUM_QUOTA_TYPES] = {1, 1, 2, 2, 1}; Oid quota_key_caches[NUM_QUOTA_TYPES][MAX_NUM_KEYS_QUOTA_MAP] = { {NAMESPACEOID}, {AUTHOID}, {NAMESPACEOID, TABLESPACEOID}, {AUTHOID, TABLESPACEOID}, {TABLESPACEOID}}; -HTAB *quota_info_map; -TimestampTz quota_info_map_last_overflow_report = 0; +static HTAB *quota_info_map; +static TimestampTz quota_info_map_last_overflow_report = 0; #define QUOTA_INFO_MAP_WARNING \ "[diskquota] the number of quota probe reached the limit, please " \ - "increase the GUC value for diskquota.max_quota_probes. Current " \ - "diskquota.max_quota_probes value:" + "increase the GUC value for diskquota.max_quota_probes." /* global rejectmap for which exceed their quota limit */ struct RejectMapEntry @@ -188,26 +187,25 @@ struct LocalRejectMapEntry }; /* using hash table to support incremental update the table size entry.*/ -static HTAB *table_size_map = NULL; -static TimestampTz table_size_last_overflow_report = 0; +static HTAB *table_size_map = NULL; +static TimestampTz table_size_map_last_overflow_report = 0; #define TABLE_SIZE_MAP_WARNING \ "[diskquota] the number of tables reached the limit, please increase " \ - "the GUC value for diskquota.max_table_segments. Current " \ - "diskquota.max_table_segments value:" + "the GUC value for diskquota.max_table_segments." /* rejectmap for database objects which exceed their quota limit */ static HTAB *disk_quota_reject_map = NULL; static HTAB *local_disk_quota_reject_map = NULL; -static TimestampTz disk_quota_reject_last_overflow_report = 0; -static TimestampTz local_disk_quota_reject_last_overflow_report = 0; +static TimestampTz disk_quota_reject_map_last_overflow_report = 0; +static TimestampTz local_disk_quota_reject_map_last_overflow_report = 0; #define DISK_QUOTA_REJECT_MAP_WARNING \ "[diskquota] Shared disk quota reject map size limit reached. " \ - "Some out-of-limit schemas or roles will be lost in rejectmap. Current limit:" + "Some out-of-limit schemas or roles will be lost in rejectmap." -#define LOCAL_DISK_QUOTA_REJECT_MAP_WARNING "[diskquota] the number of local reject map entries reached the limit:" +#define LOCAL_DISK_QUOTA_REJECT_MAP_WARNING "[diskquota] the number of local reject map entries reached the limit." static shmem_startup_hook_type prev_shmem_startup_hook = NULL; @@ -255,7 +253,7 @@ update_size_for_quota(int64 size, QuotaType type, Oid *keys, int16 segid) key.type = type; key.segid = segid; action = check_hash_fullness(quota_info_map, diskquota_max_quota_probes, QUOTA_INFO_MAP_WARNING, - "a_info_map_last_overflow_report, diskquota_max_quota_probes); + "a_info_map_last_overflow_report); entry = hash_search(quota_info_map, &key, action, &found); /* If the number of quota exceeds the limit, entry will be NULL */ if (entry == NULL) return; @@ -282,7 +280,7 @@ update_limit_for_quota(int64 limit, float segratio, QuotaType type, Oid *keys) key.type = type; key.segid = i; action = check_hash_fullness(quota_info_map, diskquota_max_quota_probes, QUOTA_INFO_MAP_WARNING, - "a_info_map_last_overflow_report, diskquota_max_quota_probes); + "a_info_map_last_overflow_report); entry = hash_search(quota_info_map, &key, action, &found); /* If the number of quota exceeds the limit, entry will be NULL */ if (entry == NULL) continue; @@ -312,9 +310,9 @@ add_quota_to_rejectmap(QuotaType type, Oid targetOid, Oid tablespaceoid, bool se keyitem.tablespaceoid = tablespaceoid; keyitem.targettype = (uint32)type; ereport(DEBUG1, (errmsg("[diskquota] Put object %u to rejectmap", targetOid))); - HASHACTION action = check_hash_fullness( - local_disk_quota_reject_map, MAX_DISK_QUOTA_REJECT_ENTRIES, LOCAL_DISK_QUOTA_REJECT_MAP_WARNING, - &local_disk_quota_reject_last_overflow_report, MAX_DISK_QUOTA_REJECT_ENTRIES); + HASHACTION action = + check_hash_fullness(local_disk_quota_reject_map, MAX_DISK_QUOTA_REJECT_ENTRIES, + LOCAL_DISK_QUOTA_REJECT_MAP_WARNING, &local_disk_quota_reject_map_last_overflow_report); localrejectentry = hash_search(local_disk_quota_reject_map, &keyitem, action, NULL); if (localrejectentry) { @@ -991,7 +989,7 @@ calculate_table_disk_usage(bool is_init, HTAB *local_active_table_stat_map) key.id = TableSizeEntryId(cur_segid); HASHACTION action = check_hash_fullness(table_size_map, MAX_NUM_TABLE_SIZE_ENTRIES, TABLE_SIZE_MAP_WARNING, - &table_size_last_overflow_report, diskquota_max_table_segments); + &table_size_map_last_overflow_report); tsentry = hash_search(table_size_map, &key, action, &table_size_map_found); if (!table_size_map_found) @@ -1279,9 +1277,9 @@ flush_local_reject_map(void) */ if (localrejectentry->isexceeded) { - HASHACTION action = check_hash_fullness( - disk_quota_reject_map, MAX_DISK_QUOTA_REJECT_ENTRIES, DISK_QUOTA_REJECT_MAP_WARNING, - &disk_quota_reject_last_overflow_report, MAX_DISK_QUOTA_REJECT_ENTRIES); + HASHACTION action = + check_hash_fullness(disk_quota_reject_map, MAX_DISK_QUOTA_REJECT_ENTRIES, + DISK_QUOTA_REJECT_MAP_WARNING, &disk_quota_reject_map_last_overflow_report); rejectentry = hash_search(disk_quota_reject_map, &localrejectentry->keyitem, action, &found); if (rejectentry == NULL) { @@ -2120,7 +2118,7 @@ refresh_rejectmap(PG_FUNCTION_ARGS) HASHACTION action = check_hash_fullness(disk_quota_reject_map, MAX_DISK_QUOTA_REJECT_ENTRIES, DISK_QUOTA_REJECT_MAP_WARNING, - &disk_quota_reject_last_overflow_report, MAX_DISK_QUOTA_REJECT_ENTRIES); + &disk_quota_reject_map_last_overflow_report); new_entry = hash_search(disk_quota_reject_map, &rejectmapentry->keyitem, action, &found); if (!found && new_entry) memcpy(new_entry, rejectmapentry, sizeof(GlobalRejectMapEntry)); } diff --git a/src/relation_cache.c b/src/relation_cache.c index d97397f7..efac7797 100644 --- a/src/relation_cache.c +++ b/src/relation_cache.c @@ -32,17 +32,15 @@ HTAB *relation_cache = NULL; HTAB *relid_cache = NULL; -extern TimestampTz active_tables_last_overflow_report; +extern TimestampTz active_tables_map_last_overflow_report; #define RELATION_CACHE_WARNING \ "[diskquota] the number of relation cache entries reached the limit, please increase " \ - "the GUC value for diskquota.max_active_tables. Current " \ - "diskquota.max_active_tables value:" + "the GUC value for diskquota.max_active_tables." #define RELID_CACHE_WARNING \ "[diskquota] the number of relation cache entries reached the limit, please increase " \ - "the GUC value for diskquota.max_active_tables. Current " \ - "diskquota.max_active_tables value:" + "the GUC value for diskquota.max_active_tables." static void update_relation_entry(Oid relid, DiskQuotaRelationCacheEntry *relation_entry, DiskQuotaRelidCacheEntry *relid_entry); @@ -192,7 +190,7 @@ update_relation_cache(Oid relid) LWLockAcquire(diskquota_locks.relation_cache_lock, LW_EXCLUSIVE); action = check_hash_fullness(relation_cache, diskquota_max_active_tables, RELATION_CACHE_WARNING, - &active_tables_last_overflow_report, diskquota_max_active_tables); + &active_tables_map_last_overflow_report); relation_entry = hash_search(relation_cache, &relation_entry_data.relid, action, NULL); if (relation_entry == NULL) @@ -203,7 +201,7 @@ update_relation_cache(Oid relid) memcpy(relation_entry, &relation_entry_data, sizeof(DiskQuotaRelationCacheEntry)); action = check_hash_fullness(relid_cache, diskquota_max_active_tables, RELID_CACHE_WARNING, - &active_tables_last_overflow_report, diskquota_max_active_tables); + &active_tables_map_last_overflow_report); relid_entry = hash_search(relid_cache, &relid_entry_data.relfilenode, action, NULL); if (relid_entry == NULL) { diff --git a/tests/regress/expected/test_activetable_limit.out b/tests/regress/expected/test_activetable_limit.out index 7699f2cc..0b34e601 100644 --- a/tests/regress/expected/test_activetable_limit.out +++ b/tests/regress/expected/test_activetable_limit.out @@ -43,10 +43,7 @@ NOTICE: CREATE TABLE will create partition "t1_1_prt_7" for table "t1" NOTICE: CREATE TABLE will create partition "t1_1_prt_8" for table "t1" NOTICE: CREATE TABLE will create partition "t1_1_prt_9" for table "t1" NOTICE: CREATE TABLE will create partition "t1_1_prt_10" for table "t1" -WARNING: [diskquota] the number of active tables reached the limit, please increase the GUC value for diskquota.max_active_tables. Current diskquota.max_active_tables value: 5 -WARNING: [diskquota] the number of active tables reached the limit, please increase the GUC value for diskquota.max_active_tables. Current diskquota.max_active_tables value: 5 (seg0 127.0.1.1:6002 pid=195302) -WARNING: [diskquota] the number of active tables reached the limit, please increase the GUC value for diskquota.max_active_tables. Current diskquota.max_active_tables value: 5 (seg1 127.0.1.1:6003 pid=195303) -WARNING: [diskquota] the number of active tables reached the limit, please increase the GUC value for diskquota.max_active_tables. Current diskquota.max_active_tables value: 5 (seg2 127.0.1.1:6004 pid=195304) +WARNING: [diskquota] the number of active tables reached the limit, please increase the GUC value for diskquota.max_active_tables. SELECT count(*) FROM segment_logs WHERE line LIKE '%the number of active tables reached the limit%'; count ------- @@ -70,9 +67,7 @@ SELECT count(*) FROM s.t1; -- altered reloid cache overflow check. expected warning. VACUUM FULL; -WARNING: [diskquota] the number of altered reloid cache entries reached the limit, please increase the GUC value for diskquota.max_active_tables. Current diskquota.max_active_tables value: 5 (seg2 127.0.1.1:6004 pid=195304) -WARNING: [diskquota] the number of altered reloid cache entries reached the limit, please increase the GUC value for diskquota.max_active_tables. Current diskquota.max_active_tables value: 5 (seg0 127.0.1.1:6002 pid=195302) -WARNING: [diskquota] the number of altered reloid cache entries reached the limit, please increase the GUC value for diskquota.max_active_tables. Current diskquota.max_active_tables value: 5 (seg1 127.0.1.1:6003 pid=195303) +WARNING: [diskquota] the number of altered reloid cache entries reached the limit, please increase the GUC value for diskquota.max_active_tables. SELECT count(*) FROM segment_logs WHERE line LIKE '%the number of altered reloid cache entries reached the limit%'; count ------- From b8bbfc8d0bd67bcb6b062c7504ec637c24d99073 Mon Sep 17 00:00:00 2001 From: Viktor Kurilko Date: Mon, 25 Dec 2023 16:44:58 +0700 Subject: [PATCH 19/31] Add guc diskquota.max_reject_entries that control size of local_disk_quota_reject_map. The size of disk_quota_reject_map is calculated as the maximum size of the local table multiplied by the maximum number of databases. The variables that store the time of the last report on reaching the limit are now stored in shared memory to prevent spam from dynamic workers. --- src/diskquota.c | 3 + src/diskquota_utility.c | 2 +- src/quotamodel.c | 75 ++++++++++------ tests/regress/diskquota_schedule | 1 + .../regress/expected/test_rejectmap_limit.out | 87 +++++++++++++++++++ tests/regress/sql/test_rejectmap_limit.sql | 56 ++++++++++++ 6 files changed, 197 insertions(+), 27 deletions(-) create mode 100644 tests/regress/expected/test_rejectmap_limit.out create mode 100644 tests/regress/sql/test_rejectmap_limit.sql diff --git a/src/diskquota.c b/src/diskquota.c index 725894a9..cc25b70a 100644 --- a/src/diskquota.c +++ b/src/diskquota.c @@ -78,6 +78,7 @@ int diskquota_max_workers = 10; int diskquota_max_table_segments = 0; int diskquota_max_monitored_databases = 0; int diskquota_max_quota_probes = 0; +int diskquota_max_local_reject_entries = 0; int diskquota_hashmap_overflow_report_timeout = 0; DiskQuotaLocks diskquota_locks; @@ -409,6 +410,8 @@ define_guc_variables(void) DefineCustomIntVariable("diskquota.max_quota_probes", "Max number of quotas on the cluster.", NULL, &diskquota_max_quota_probes, 1024 * 1024, 1024 * INIT_QUOTA_MAP_ENTRIES, INT_MAX, PGC_POSTMASTER, 0, NULL, NULL, NULL); + DefineCustomIntVariable("diskquota.max_reject_entries", "Max number of reject entries per database.", NULL, + &diskquota_max_local_reject_entries, 8192, 1, INT_MAX, PGC_POSTMASTER, 0, NULL, NULL, NULL); DefineCustomIntVariable("diskquota.hashmap_overflow_report_timeout", "The duration between each warning report about the shared hashmap overflow (in seconds).", NULL, &diskquota_hashmap_overflow_report_timeout, 60, 0, INT_MAX / 1000, PGC_SUSET, 0, NULL, diff --git a/src/diskquota_utility.c b/src/diskquota_utility.c index d576f54a..f8929b84 100644 --- a/src/diskquota_utility.c +++ b/src/diskquota_utility.c @@ -1702,7 +1702,7 @@ check_hash_fullness(HTAB *hashp, int max_size, const char *warning_message, Time if (*last_overflow_report == 0 || TimestampDifferenceExceeds(*last_overflow_report, current_time, diskquota_hashmap_overflow_report_timeout * 1000)) { - ereport(WARNING, (errmsg(warning_message))); + ereport(WARNING, (errmsg("%s", warning_message))); *last_overflow_report = current_time; } } diff --git a/src/quotamodel.c b/src/quotamodel.c index 67676a0b..a87c26a2 100644 --- a/src/quotamodel.c +++ b/src/quotamodel.c @@ -45,11 +45,7 @@ #include /* cluster level max size of rejectmap */ -#define MAX_DISK_QUOTA_REJECT_ENTRIES (1024 * 1024) -/* cluster level init size of rejectmap */ -#define INIT_DISK_QUOTA_REJECT_ENTRIES 8192 -/* per database level max size of rejectmap */ -#define MAX_LOCAL_DISK_QUOTA_REJECT_ENTRIES 8192 +#define MAX_DISK_QUOTA_REJECT_ENTRIES (diskquota_max_local_reject_entries * diskquota_max_monitored_databases) /* Number of attributes in quota configuration records. */ #define NUM_QUOTA_CONFIG_ATTRS 6 /* Number of entries for diskquota.table_size update SQL */ @@ -87,7 +83,7 @@ int SEGCOUNT = 0; extern int diskquota_max_table_segments; extern int diskquota_max_monitored_databases; extern int diskquota_max_quota_probes; - +extern int diskquota_max_local_reject_entries; /* * local cache of table disk size and corresponding schema and owner. * @@ -142,8 +138,8 @@ typedef enum uint16 quota_key_num[NUM_QUOTA_TYPES] = {1, 1, 2, 2, 1}; Oid quota_key_caches[NUM_QUOTA_TYPES][MAX_NUM_KEYS_QUOTA_MAP] = { {NAMESPACEOID}, {AUTHOID}, {NAMESPACEOID, TABLESPACEOID}, {AUTHOID, TABLESPACEOID}, {TABLESPACEOID}}; -static HTAB *quota_info_map; -static TimestampTz quota_info_map_last_overflow_report = 0; +static HTAB *quota_info_map; +static TimestampTz *quota_info_map_last_overflow_report = NULL; #define QUOTA_INFO_MAP_WARNING \ "[diskquota] the number of quota probe reached the limit, please " \ @@ -187,8 +183,8 @@ struct LocalRejectMapEntry }; /* using hash table to support incremental update the table size entry.*/ -static HTAB *table_size_map = NULL; -static TimestampTz table_size_map_last_overflow_report = 0; +static HTAB *table_size_map = NULL; +static TimestampTz *table_size_map_last_overflow_report = 0; #define TABLE_SIZE_MAP_WARNING \ "[diskquota] the number of tables reached the limit, please increase " \ @@ -198,14 +194,16 @@ static TimestampTz table_size_map_last_overflow_report = 0; static HTAB *disk_quota_reject_map = NULL; static HTAB *local_disk_quota_reject_map = NULL; -static TimestampTz disk_quota_reject_map_last_overflow_report = 0; -static TimestampTz local_disk_quota_reject_map_last_overflow_report = 0; +static TimestampTz disk_quota_reject_map_last_overflow_report = 0; +static TimestampTz *local_disk_quota_reject_map_last_overflow_report = NULL; -#define DISK_QUOTA_REJECT_MAP_WARNING \ - "[diskquota] Shared disk quota reject map size limit reached. " \ - "Some out-of-limit schemas or roles will be lost in rejectmap." +#define DISK_QUOTA_REJECT_MAP_WARNING \ + "[diskquota] the number of quota reject map entries reached the limit, " \ + "please increase the GUC value for diskquota.max_reject_entries." -#define LOCAL_DISK_QUOTA_REJECT_MAP_WARNING "[diskquota] the number of local reject map entries reached the limit." +#define LOCAL_DISK_QUOTA_REJECT_MAP_WARNING \ + "[diskquota] the number of local quota reject map entries reached the limit, " \ + "please increase the GUC value for diskquota.max_reject_entries." static shmem_startup_hook_type prev_shmem_startup_hook = NULL; @@ -253,7 +251,7 @@ update_size_for_quota(int64 size, QuotaType type, Oid *keys, int16 segid) key.type = type; key.segid = segid; action = check_hash_fullness(quota_info_map, diskquota_max_quota_probes, QUOTA_INFO_MAP_WARNING, - "a_info_map_last_overflow_report); + quota_info_map_last_overflow_report); entry = hash_search(quota_info_map, &key, action, &found); /* If the number of quota exceeds the limit, entry will be NULL */ if (entry == NULL) return; @@ -280,7 +278,7 @@ update_limit_for_quota(int64 limit, float segratio, QuotaType type, Oid *keys) key.type = type; key.segid = i; action = check_hash_fullness(quota_info_map, diskquota_max_quota_probes, QUOTA_INFO_MAP_WARNING, - "a_info_map_last_overflow_report); + quota_info_map_last_overflow_report); entry = hash_search(quota_info_map, &key, action, &found); /* If the number of quota exceeds the limit, entry will be NULL */ if (entry == NULL) continue; @@ -309,13 +307,13 @@ add_quota_to_rejectmap(QuotaType type, Oid targetOid, Oid tablespaceoid, bool se keyitem.databaseoid = MyDatabaseId; keyitem.tablespaceoid = tablespaceoid; keyitem.targettype = (uint32)type; - ereport(DEBUG1, (errmsg("[diskquota] Put object %u to rejectmap", targetOid))); HASHACTION action = - check_hash_fullness(local_disk_quota_reject_map, MAX_DISK_QUOTA_REJECT_ENTRIES, - LOCAL_DISK_QUOTA_REJECT_MAP_WARNING, &local_disk_quota_reject_map_last_overflow_report); + check_hash_fullness(local_disk_quota_reject_map, diskquota_max_local_reject_entries, + LOCAL_DISK_QUOTA_REJECT_MAP_WARNING, local_disk_quota_reject_map_last_overflow_report); localrejectentry = hash_search(local_disk_quota_reject_map, &keyitem, action, NULL); if (localrejectentry) { + ereport(DEBUG1, (errmsg("[diskquota] Put object %u to rejectmap", targetOid))); localrejectentry->isexceeded = true; localrejectentry->segexceeded = segexceeded; } @@ -444,7 +442,7 @@ disk_quota_shmem_startup(void) hash_ctl.keysize = sizeof(RejectMapEntry); hash_ctl.entrysize = sizeof(GlobalRejectMapEntry); disk_quota_reject_map = - DiskquotaShmemInitHash("rejectmap whose quota limitation is reached", INIT_DISK_QUOTA_REJECT_ENTRIES, + DiskquotaShmemInitHash("rejectmap whose quota limitation is reached", diskquota_max_local_reject_entries, MAX_DISK_QUOTA_REJECT_ENTRIES, &hash_ctl, HASH_ELEM, DISKQUOTA_TAG_HASH); init_shm_worker_active_tables(); @@ -505,7 +503,9 @@ diskquota_worker_shmem_size() Size size; size = hash_estimate_size(MAX_NUM_TABLE_SIZE_ENTRIES / diskquota_max_monitored_databases + 100, sizeof(TableSizeEntry)); - size = add_size(size, hash_estimate_size(MAX_LOCAL_DISK_QUOTA_REJECT_ENTRIES, sizeof(LocalRejectMapEntry))); + size = add_size(size, hash_estimate_size(diskquota_max_local_reject_entries, sizeof(LocalRejectMapEntry))); + size = add_size(size, sizeof(TimestampTz)); // table_size_map_last_overflow_report + size = add_size(size, sizeof(TimestampTz)); // local_disk_quota_reject_map_last_overflow_report return size; } @@ -532,6 +532,8 @@ DiskQuotaShmemSize(void) size = add_size(size, diskquota_worker_shmem_size() * diskquota_max_monitored_databases); size = add_size(size, hash_estimate_size(MAX_QUOTA_MAP_ENTRIES, sizeof(QuotaInfoEntry)) * diskquota_max_monitored_databases); + size = add_size(size, + sizeof(TimestampTz) * diskquota_max_monitored_databases); // quota_info_map_last_overflow_report } return size; @@ -546,6 +548,7 @@ init_disk_quota_model(uint32 id) { HASHCTL hash_ctl; StringInfoData str; + bool found; initStringInfo(&str); format_name("TableSizeEntrymap", id, &str); @@ -554,6 +557,9 @@ init_disk_quota_model(uint32 id) hash_ctl.entrysize = sizeof(TableSizeEntry); table_size_map = DiskquotaShmemInitHash(str.data, INIT_NUM_TABLE_SIZE_ENTRIES, MAX_NUM_TABLE_SIZE_ENTRIES, &hash_ctl, HASH_ELEM, DISKQUOTA_TAG_HASH); + format_name("TableSizeEntrymap_last_overflow_report", id, &str); + table_size_map_last_overflow_report = ShmemInitStruct(str.data, sizeof(TimestampTz), &found); + if (!found) *table_size_map_last_overflow_report = 0; /* for localrejectmap */ /* WARNNING: The max length of name of the map is 48 */ @@ -562,9 +568,13 @@ init_disk_quota_model(uint32 id) hash_ctl.keysize = sizeof(RejectMapEntry); hash_ctl.entrysize = sizeof(LocalRejectMapEntry); local_disk_quota_reject_map = - DiskquotaShmemInitHash(str.data, MAX_LOCAL_DISK_QUOTA_REJECT_ENTRIES, MAX_LOCAL_DISK_QUOTA_REJECT_ENTRIES, + DiskquotaShmemInitHash(str.data, diskquota_max_local_reject_entries, diskquota_max_local_reject_entries, &hash_ctl, HASH_ELEM, DISKQUOTA_TAG_HASH); + format_name("localrejectmap_last_overflow_report", id, &str); + local_disk_quota_reject_map_last_overflow_report = ShmemInitStruct(str.data, sizeof(TimestampTz), &found); + if (!found) *local_disk_quota_reject_map_last_overflow_report = 0; + /* for quota_info_map */ format_name("QuotaInfoMap", id, &str); memset(&hash_ctl, 0, sizeof(hash_ctl)); @@ -572,6 +582,9 @@ init_disk_quota_model(uint32 id) hash_ctl.keysize = sizeof(QuotaInfoEntryKey); quota_info_map = DiskquotaShmemInitHash(str.data, INIT_QUOTA_MAP_ENTRIES, MAX_QUOTA_MAP_ENTRIES, &hash_ctl, HASH_ELEM, DISKQUOTA_TAG_HASH); + format_name("QuotaInfoMap_last_overflow_report", id, &str); + quota_info_map_last_overflow_report = ShmemInitStruct(str.data, sizeof(TimestampTz), &found); + if (!found) *quota_info_map_last_overflow_report = 0; pfree(str.data); } @@ -595,6 +608,7 @@ vacuum_disk_quota_model(uint32 id) TableSizeEntry *tsentry = NULL; LocalRejectMapEntry *localrejectentry; QuotaInfoEntry *qentry; + bool found; HASHCTL hash_ctl; StringInfoData str; @@ -613,19 +627,25 @@ vacuum_disk_quota_model(uint32 id) hash_search(table_size_map, &tsentry->key, HASH_REMOVE, NULL); } + format_name("TableSizeEntrymap_last_overflow_report", id, &str); + table_size_map_last_overflow_report = ShmemInitStruct(str.data, sizeof(TimestampTz), &found); + if (!found) *table_size_map_last_overflow_report = 0; /* localrejectmap */ format_name("localrejectmap", id, &str); memset(&hash_ctl, 0, sizeof(hash_ctl)); hash_ctl.keysize = sizeof(RejectMapEntry); hash_ctl.entrysize = sizeof(LocalRejectMapEntry); local_disk_quota_reject_map = - DiskquotaShmemInitHash(str.data, MAX_LOCAL_DISK_QUOTA_REJECT_ENTRIES, MAX_LOCAL_DISK_QUOTA_REJECT_ENTRIES, + DiskquotaShmemInitHash(str.data, diskquota_max_local_reject_entries, diskquota_max_local_reject_entries, &hash_ctl, HASH_ELEM, DISKQUOTA_TAG_HASH); hash_seq_init(&iter, local_disk_quota_reject_map); while ((localrejectentry = hash_seq_search(&iter)) != NULL) { hash_search(local_disk_quota_reject_map, &localrejectentry->keyitem, HASH_REMOVE, NULL); } + format_name("localrejectmap_last_overflow_report", id, &str); + local_disk_quota_reject_map_last_overflow_report = ShmemInitStruct(str.data, sizeof(TimestampTz), &found); + if (!found) *local_disk_quota_reject_map_last_overflow_report = 0; /* quota_info_map */ format_name("QuotaInfoMap", id, &str); @@ -639,6 +659,9 @@ vacuum_disk_quota_model(uint32 id) { hash_search(quota_info_map, &qentry->key, HASH_REMOVE, NULL); } + format_name("QuotaInfoMap_last_overflow_report", id, &str); + quota_info_map_last_overflow_report = ShmemInitStruct(str.data, sizeof(TimestampTz), &found); + if (!found) *quota_info_map_last_overflow_report = 0; pfree(str.data); } @@ -989,7 +1012,7 @@ calculate_table_disk_usage(bool is_init, HTAB *local_active_table_stat_map) key.id = TableSizeEntryId(cur_segid); HASHACTION action = check_hash_fullness(table_size_map, MAX_NUM_TABLE_SIZE_ENTRIES, TABLE_SIZE_MAP_WARNING, - &table_size_map_last_overflow_report); + table_size_map_last_overflow_report); tsentry = hash_search(table_size_map, &key, action, &table_size_map_found); if (!table_size_map_found) diff --git a/tests/regress/diskquota_schedule b/tests/regress/diskquota_schedule index 82560063..05baeef3 100644 --- a/tests/regress/diskquota_schedule +++ b/tests/regress/diskquota_schedule @@ -32,6 +32,7 @@ test: test_appendonly test: test_rejectmap test: test_clean_rejectmap_after_drop test: test_rejectmap_mul_db +test: test_rejectmap_limit test: test_ctas_pause test: test_ctas_role test: test_ctas_schema diff --git a/tests/regress/expected/test_rejectmap_limit.out b/tests/regress/expected/test_rejectmap_limit.out new file mode 100644 index 00000000..05361c5d --- /dev/null +++ b/tests/regress/expected/test_rejectmap_limit.out @@ -0,0 +1,87 @@ +-- +-- This file contains tests for limiting reject map +-- +\! gpconfig -c diskquota.max_reject_entries -v 4 > /dev/null +\! gpstop -arf > /dev/null +\c +CREATE DATABASE test_reject_map_limit_01; +\c test_reject_map_limit_01 +CREATE EXTENSION diskquota; +SELECT diskquota.wait_for_worker_new_epoch(); + wait_for_worker_new_epoch +--------------------------- + t +(1 row) + +CREATE EXTERNAL WEB TABLE master_log(line text) + EXECUTE 'cat $GP_SEG_DATADIR/pg_log/$(ls -Art $GP_SEG_DATADIR/pg_log | tail -n 1)' + ON MASTER FORMAT 'TEXT' (DELIMITER 'OFF'); +CREATE SCHEMA s1; +CREATE SCHEMA s2; +CREATE SCHEMA s3; +CREATE SCHEMA s4; +CREATE SCHEMA s5; +SELECT diskquota.set_schema_quota('s1', '1 MB'); + set_schema_quota +------------------ + +(1 row) + +SELECT diskquota.set_schema_quota('s2', '1 MB'); + set_schema_quota +------------------ + +(1 row) + +SELECT diskquota.set_schema_quota('s3', '1 MB'); + set_schema_quota +------------------ + +(1 row) + +SELECT diskquota.set_schema_quota('s4', '1 MB'); + set_schema_quota +------------------ + +(1 row) + +SELECT diskquota.set_schema_quota('s5', '1 MB'); + set_schema_quota +------------------ + +(1 row) + +SELECT diskquota.wait_for_worker_new_epoch(); + wait_for_worker_new_epoch +--------------------------- + t +(1 row) + +CREATE TABLE s1.a(i int) DISTRIBUTED BY (i); +CREATE TABLE s2.a(i int) DISTRIBUTED BY (i); +CREATE TABLE s3.a(i int) DISTRIBUTED BY (i); +CREATE TABLE s4.a(i int) DISTRIBUTED BY (i); +CREATE TABLE s5.a(i int) DISTRIBUTED BY (i); +INSERT INTO s1.a SELECT generate_series(1,100000); +INSERT INTO s2.a SELECT generate_series(1,100000); +INSERT INTO s3.a SELECT generate_series(1,100000); +INSERT INTO s4.a SELECT generate_series(1,100000); +INSERT INTO s5.a SELECT generate_series(1,100000); +SELECT diskquota.wait_for_worker_new_epoch(); + wait_for_worker_new_epoch +--------------------------- + t +(1 row) + +SELECT count(*) FROM master_log WHERE line LIKE '%the number of local quota reject map entries reached the limit%' AND line NOT LIKE '%LOG%'; + count +------- + 1 +(1 row) + +INSERT INTO s5.a SELECT generate_series(1,10); -- should be successful +DROP EXTENSION diskquota; +\c contrib_regression +DROP DATABASE test_reject_map_limit_01; +\! gpconfig -r diskquota.max_reject_entries > /dev/null +\! gpstop -arf > /dev/null diff --git a/tests/regress/sql/test_rejectmap_limit.sql b/tests/regress/sql/test_rejectmap_limit.sql new file mode 100644 index 00000000..4598f572 --- /dev/null +++ b/tests/regress/sql/test_rejectmap_limit.sql @@ -0,0 +1,56 @@ +-- +-- This file contains tests for limiting reject map +-- + +\! gpconfig -c diskquota.max_reject_entries -v 4 > /dev/null +\! gpstop -arf > /dev/null + +\c + +CREATE DATABASE test_reject_map_limit_01; + +\c test_reject_map_limit_01 +CREATE EXTENSION diskquota; +SELECT diskquota.wait_for_worker_new_epoch(); +CREATE EXTERNAL WEB TABLE master_log(line text) + EXECUTE 'cat $GP_SEG_DATADIR/pg_log/$(ls -Art $GP_SEG_DATADIR/pg_log | tail -n 1)' + ON MASTER FORMAT 'TEXT' (DELIMITER 'OFF'); + +CREATE SCHEMA s1; +CREATE SCHEMA s2; +CREATE SCHEMA s3; +CREATE SCHEMA s4; +CREATE SCHEMA s5; + +SELECT diskquota.set_schema_quota('s1', '1 MB'); +SELECT diskquota.set_schema_quota('s2', '1 MB'); +SELECT diskquota.set_schema_quota('s3', '1 MB'); +SELECT diskquota.set_schema_quota('s4', '1 MB'); +SELECT diskquota.set_schema_quota('s5', '1 MB'); +SELECT diskquota.wait_for_worker_new_epoch(); + +CREATE TABLE s1.a(i int) DISTRIBUTED BY (i); +CREATE TABLE s2.a(i int) DISTRIBUTED BY (i); +CREATE TABLE s3.a(i int) DISTRIBUTED BY (i); +CREATE TABLE s4.a(i int) DISTRIBUTED BY (i); +CREATE TABLE s5.a(i int) DISTRIBUTED BY (i); + +INSERT INTO s1.a SELECT generate_series(1,100000); +INSERT INTO s2.a SELECT generate_series(1,100000); +INSERT INTO s3.a SELECT generate_series(1,100000); +INSERT INTO s4.a SELECT generate_series(1,100000); +INSERT INTO s5.a SELECT generate_series(1,100000); + +SELECT diskquota.wait_for_worker_new_epoch(); + +SELECT count(*) FROM master_log WHERE line LIKE '%the number of local quota reject map entries reached the limit%' AND line NOT LIKE '%LOG%'; + +INSERT INTO s5.a SELECT generate_series(1,10); -- should be successful + +DROP EXTENSION diskquota; + +\c contrib_regression +DROP DATABASE test_reject_map_limit_01; + +\! gpconfig -r diskquota.max_reject_entries > /dev/null +\! gpstop -arf > /dev/null \ No newline at end of file From f650b93f34dbddcc8abb2791523ac66808c34930 Mon Sep 17 00:00:00 2001 From: Viktor Kurilko Date: Mon, 25 Dec 2023 16:49:18 +0700 Subject: [PATCH 20/31] fix format --- src/diskquota.h | 4 ++-- src/gp_activetable.c | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/src/diskquota.h b/src/diskquota.h index d9ba794b..7c2bbb15 100644 --- a/src/diskquota.h +++ b/src/diskquota.h @@ -317,6 +317,6 @@ extern HTAB *diskquota_hash_create(const char *tabname, long nelem, HASHC extern HTAB *DiskquotaShmemInitHash(const char *name, long init_size, long max_size, HASHCTL *infoP, int hash_flags, DiskquotaHashFunction hash_function); extern void refresh_monitored_dbid_cache(void); -extern HASHACTION check_hash_fullness(HTAB *hashp, int max_size, const char *warning_message, - TimestampTz *last_overflow_report); +extern HASHACTION check_hash_fullness(HTAB *hashp, int max_size, const char *warning_message, + TimestampTz *last_overflow_report); #endif diff --git a/src/gp_activetable.c b/src/gp_activetable.c index fb1dc939..1a843d22 100644 --- a/src/gp_activetable.c +++ b/src/gp_activetable.c @@ -928,7 +928,7 @@ get_active_tables_oid(void) { HASHACTION action = check_hash_fullness(active_tables_map, diskquota_max_active_tables, ACTIVE_TABLES_MAP_WARNING, &active_tables_map_last_overflow_report); - entry = hash_search(active_tables_map, active_table_file_entry, action, &found); + entry = hash_search(active_tables_map, active_table_file_entry, action, &found); if (entry) *entry = *active_table_file_entry; } LWLockRelease(diskquota_locks.active_table_lock); From 77e6caa8620a4bd5cb1f25feeb1e64e9eae0eac1 Mon Sep 17 00:00:00 2001 From: Viktor Kurilko Date: Mon, 25 Dec 2023 17:01:09 +0700 Subject: [PATCH 21/31] change default value of table_size_map_last_overflow_report from 0 to NULL --- src/quotamodel.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/quotamodel.c b/src/quotamodel.c index a87c26a2..4e9224b8 100644 --- a/src/quotamodel.c +++ b/src/quotamodel.c @@ -184,7 +184,7 @@ struct LocalRejectMapEntry /* using hash table to support incremental update the table size entry.*/ static HTAB *table_size_map = NULL; -static TimestampTz *table_size_map_last_overflow_report = 0; +static TimestampTz *table_size_map_last_overflow_report = NULL; #define TABLE_SIZE_MAP_WARNING \ "[diskquota] the number of tables reached the limit, please increase " \ From a03e278ca9ffb614c2fc8808dc92fa98b4836e10 Mon Sep 17 00:00:00 2001 From: Viktor Kurilko Date: Mon, 25 Dec 2023 17:14:39 +0700 Subject: [PATCH 22/31] add comment. --- src/quotamodel.c | 3 +++ 1 file changed, 3 insertions(+) diff --git a/src/quotamodel.c b/src/quotamodel.c index 4e9224b8..b8364d7c 100644 --- a/src/quotamodel.c +++ b/src/quotamodel.c @@ -139,6 +139,7 @@ uint16 quota_key_num[NUM_QUOTA_TYPES] = {1, 1, 2, 2, Oid quota_key_caches[NUM_QUOTA_TYPES][MAX_NUM_KEYS_QUOTA_MAP] = { {NAMESPACEOID}, {AUTHOID}, {NAMESPACEOID, TABLESPACEOID}, {AUTHOID, TABLESPACEOID}, {TABLESPACEOID}}; static HTAB *quota_info_map; +/* stored in shared memory */ static TimestampTz *quota_info_map_last_overflow_report = NULL; #define QUOTA_INFO_MAP_WARNING \ @@ -184,6 +185,7 @@ struct LocalRejectMapEntry /* using hash table to support incremental update the table size entry.*/ static HTAB *table_size_map = NULL; +/* stored in shared memory */ static TimestampTz *table_size_map_last_overflow_report = NULL; #define TABLE_SIZE_MAP_WARNING \ @@ -195,6 +197,7 @@ static HTAB *disk_quota_reject_map = NULL; static HTAB *local_disk_quota_reject_map = NULL; static TimestampTz disk_quota_reject_map_last_overflow_report = 0; +/* stored in shared memory */ static TimestampTz *local_disk_quota_reject_map_last_overflow_report = NULL; #define DISK_QUOTA_REJECT_MAP_WARNING \ From f65bddd184bfe9966846423b9a94c423cca4b3a2 Mon Sep 17 00:00:00 2001 From: Viktor Kurilko Date: Mon, 25 Dec 2023 17:25:59 +0700 Subject: [PATCH 23/31] fix format --- src/quotamodel.c | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/quotamodel.c b/src/quotamodel.c index b8364d7c..ab1519f3 100644 --- a/src/quotamodel.c +++ b/src/quotamodel.c @@ -138,7 +138,7 @@ typedef enum uint16 quota_key_num[NUM_QUOTA_TYPES] = {1, 1, 2, 2, 1}; Oid quota_key_caches[NUM_QUOTA_TYPES][MAX_NUM_KEYS_QUOTA_MAP] = { {NAMESPACEOID}, {AUTHOID}, {NAMESPACEOID, TABLESPACEOID}, {AUTHOID, TABLESPACEOID}, {TABLESPACEOID}}; -static HTAB *quota_info_map; +static HTAB *quota_info_map; /* stored in shared memory */ static TimestampTz *quota_info_map_last_overflow_report = NULL; @@ -184,7 +184,7 @@ struct LocalRejectMapEntry }; /* using hash table to support incremental update the table size entry.*/ -static HTAB *table_size_map = NULL; +static HTAB *table_size_map = NULL; /* stored in shared memory */ static TimestampTz *table_size_map_last_overflow_report = NULL; @@ -196,7 +196,7 @@ static TimestampTz *table_size_map_last_overflow_report = NULL; static HTAB *disk_quota_reject_map = NULL; static HTAB *local_disk_quota_reject_map = NULL; -static TimestampTz disk_quota_reject_map_last_overflow_report = 0; +static TimestampTz disk_quota_reject_map_last_overflow_report = 0; /* stored in shared memory */ static TimestampTz *local_disk_quota_reject_map_last_overflow_report = NULL; From 58acb43d612d5dd8f2014af8ca58b57b5536abe7 Mon Sep 17 00:00:00 2001 From: Viktor Kurilko Date: Tue, 26 Dec 2023 11:39:07 +0700 Subject: [PATCH 24/31] fix test --- tests/regress/expected/test_rejectmap_limit.out | 2 +- tests/regress/sql/test_rejectmap_limit.sql | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/regress/expected/test_rejectmap_limit.out b/tests/regress/expected/test_rejectmap_limit.out index 05361c5d..e45facc3 100644 --- a/tests/regress/expected/test_rejectmap_limit.out +++ b/tests/regress/expected/test_rejectmap_limit.out @@ -73,13 +73,13 @@ SELECT diskquota.wait_for_worker_new_epoch(); t (1 row) +INSERT INTO s5.a SELECT generate_series(1,10); -- should be successful SELECT count(*) FROM master_log WHERE line LIKE '%the number of local quota reject map entries reached the limit%' AND line NOT LIKE '%LOG%'; count ------- 1 (1 row) -INSERT INTO s5.a SELECT generate_series(1,10); -- should be successful DROP EXTENSION diskquota; \c contrib_regression DROP DATABASE test_reject_map_limit_01; diff --git a/tests/regress/sql/test_rejectmap_limit.sql b/tests/regress/sql/test_rejectmap_limit.sql index 4598f572..838033ef 100644 --- a/tests/regress/sql/test_rejectmap_limit.sql +++ b/tests/regress/sql/test_rejectmap_limit.sql @@ -43,10 +43,10 @@ INSERT INTO s5.a SELECT generate_series(1,100000); SELECT diskquota.wait_for_worker_new_epoch(); -SELECT count(*) FROM master_log WHERE line LIKE '%the number of local quota reject map entries reached the limit%' AND line NOT LIKE '%LOG%'; - INSERT INTO s5.a SELECT generate_series(1,10); -- should be successful +SELECT count(*) FROM master_log WHERE line LIKE '%the number of local quota reject map entries reached the limit%' AND line NOT LIKE '%LOG%'; + DROP EXTENSION diskquota; \c contrib_regression From a26964f7852a0251e3a2207d48c575aa4cfbce5d Mon Sep 17 00:00:00 2001 From: Viktor Kurilko Date: Tue, 26 Dec 2023 11:44:26 +0700 Subject: [PATCH 25/31] move quota_info_map_last_overflow_report size adding to diskquota_worker_shmem_size --- src/quotamodel.c | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/src/quotamodel.c b/src/quotamodel.c index ab1519f3..b57de0e7 100644 --- a/src/quotamodel.c +++ b/src/quotamodel.c @@ -509,6 +509,7 @@ diskquota_worker_shmem_size() size = add_size(size, hash_estimate_size(diskquota_max_local_reject_entries, sizeof(LocalRejectMapEntry))); size = add_size(size, sizeof(TimestampTz)); // table_size_map_last_overflow_report size = add_size(size, sizeof(TimestampTz)); // local_disk_quota_reject_map_last_overflow_report + size = add_size(size, sizeof(TimestampTz)); // quota_info_map_last_overflow_report return size; } @@ -535,8 +536,6 @@ DiskQuotaShmemSize(void) size = add_size(size, diskquota_worker_shmem_size() * diskquota_max_monitored_databases); size = add_size(size, hash_estimate_size(MAX_QUOTA_MAP_ENTRIES, sizeof(QuotaInfoEntry)) * diskquota_max_monitored_databases); - size = add_size(size, - sizeof(TimestampTz) * diskquota_max_monitored_databases); // quota_info_map_last_overflow_report } return size; From a84787d303931ace36eae87e9f56fbcd23997d18 Mon Sep 17 00:00:00 2001 From: Viktor Kurilko Date: Tue, 26 Dec 2023 12:05:27 +0700 Subject: [PATCH 26/31] change warning messages. move size adding of quota_info_map to diskquota_worker_shmem_size --- src/diskquota_utility.c | 2 +- src/gp_activetable.c | 8 ++++---- src/quotamodel.c | 19 +++++++++---------- src/relation_cache.c | 8 ++++---- 4 files changed, 18 insertions(+), 19 deletions(-) diff --git a/src/diskquota_utility.c b/src/diskquota_utility.c index f8929b84..f306cef8 100644 --- a/src/diskquota_utility.c +++ b/src/diskquota_utility.c @@ -1702,7 +1702,7 @@ check_hash_fullness(HTAB *hashp, int max_size, const char *warning_message, Time if (*last_overflow_report == 0 || TimestampDifferenceExceeds(*last_overflow_report, current_time, diskquota_hashmap_overflow_report_timeout * 1000)) { - ereport(WARNING, (errmsg("%s", warning_message))); + ereport(WARNING, (errmsg("[diskquota] %s", warning_message))); *last_overflow_report = current_time; } } diff --git a/src/gp_activetable.c b/src/gp_activetable.c index 1a843d22..de4062ff 100644 --- a/src/gp_activetable.c +++ b/src/gp_activetable.c @@ -54,8 +54,8 @@ typedef struct DiskQuotaSetOFCache static HTAB *active_tables_map = NULL; // Set TimestampTz active_tables_map_last_overflow_report = 0; -#define ACTIVE_TABLES_MAP_WARNING \ - "[diskquota] the number of active tables reached the limit, please increase " \ +#define ACTIVE_TABLES_MAP_WARNING \ + "the number of active tables reached the limit, please increase " \ "the GUC value for diskquota.max_active_tables." /* @@ -68,8 +68,8 @@ TimestampTz active_tables_map_last_overflow_report = 0; static HTAB *altered_reloid_cache = NULL; // Set static TimestampTz altered_reloid_cache_last_overflow_report = 0; -#define ALTERED_RELOID_CACHE_WARNING \ - "[diskquota] the number of altered reloid cache entries reached the limit, please increase " \ +#define ALTERED_RELOID_CACHE_WARNING \ + "the number of altered reloid cache entries reached the limit, please increase " \ "the GUC value for diskquota.max_active_tables." /* active table hooks which detect the disk file size change. */ diff --git a/src/quotamodel.c b/src/quotamodel.c index b57de0e7..7f75b079 100644 --- a/src/quotamodel.c +++ b/src/quotamodel.c @@ -142,8 +142,8 @@ static HTAB *quota_info_map; /* stored in shared memory */ static TimestampTz *quota_info_map_last_overflow_report = NULL; -#define QUOTA_INFO_MAP_WARNING \ - "[diskquota] the number of quota probe reached the limit, please " \ +#define QUOTA_INFO_MAP_WARNING \ + "the number of quota probe reached the limit, please " \ "increase the GUC value for diskquota.max_quota_probes." /* global rejectmap for which exceed their quota limit */ @@ -188,8 +188,8 @@ static HTAB *table_size_map = NULL; /* stored in shared memory */ static TimestampTz *table_size_map_last_overflow_report = NULL; -#define TABLE_SIZE_MAP_WARNING \ - "[diskquota] the number of tables reached the limit, please increase " \ +#define TABLE_SIZE_MAP_WARNING \ + "the number of tables reached the limit, please increase " \ "the GUC value for diskquota.max_table_segments." /* rejectmap for database objects which exceed their quota limit */ @@ -200,12 +200,12 @@ static TimestampTz disk_quota_reject_map_last_overflow_report = 0; /* stored in shared memory */ static TimestampTz *local_disk_quota_reject_map_last_overflow_report = NULL; -#define DISK_QUOTA_REJECT_MAP_WARNING \ - "[diskquota] the number of quota reject map entries reached the limit, " \ +#define DISK_QUOTA_REJECT_MAP_WARNING \ + "the number of quota reject map entries reached the limit, " \ "please increase the GUC value for diskquota.max_reject_entries." -#define LOCAL_DISK_QUOTA_REJECT_MAP_WARNING \ - "[diskquota] the number of local quota reject map entries reached the limit, " \ +#define LOCAL_DISK_QUOTA_REJECT_MAP_WARNING \ + "the number of local quota reject map entries reached the limit, " \ "please increase the GUC value for diskquota.max_reject_entries." static shmem_startup_hook_type prev_shmem_startup_hook = NULL; @@ -507,6 +507,7 @@ diskquota_worker_shmem_size() size = hash_estimate_size(MAX_NUM_TABLE_SIZE_ENTRIES / diskquota_max_monitored_databases + 100, sizeof(TableSizeEntry)); size = add_size(size, hash_estimate_size(diskquota_max_local_reject_entries, sizeof(LocalRejectMapEntry))); + size = add_size(size, hash_estimate_size(MAX_QUOTA_MAP_ENTRIES, sizeof(QuotaInfoEntry))); size = add_size(size, sizeof(TimestampTz)); // table_size_map_last_overflow_report size = add_size(size, sizeof(TimestampTz)); // local_disk_quota_reject_map_last_overflow_report size = add_size(size, sizeof(TimestampTz)); // quota_info_map_last_overflow_report @@ -534,8 +535,6 @@ DiskQuotaShmemSize(void) { size = add_size(size, diskquota_launcher_shmem_size()); size = add_size(size, diskquota_worker_shmem_size() * diskquota_max_monitored_databases); - size = add_size(size, hash_estimate_size(MAX_QUOTA_MAP_ENTRIES, sizeof(QuotaInfoEntry)) * - diskquota_max_monitored_databases); } return size; diff --git a/src/relation_cache.c b/src/relation_cache.c index efac7797..614abb4d 100644 --- a/src/relation_cache.c +++ b/src/relation_cache.c @@ -34,12 +34,12 @@ HTAB *relid_cache = NULL; extern TimestampTz active_tables_map_last_overflow_report; -#define RELATION_CACHE_WARNING \ - "[diskquota] the number of relation cache entries reached the limit, please increase " \ +#define RELATION_CACHE_WARNING \ + "the number of relation cache entries reached the limit, please increase " \ "the GUC value for diskquota.max_active_tables." -#define RELID_CACHE_WARNING \ - "[diskquota] the number of relation cache entries reached the limit, please increase " \ +#define RELID_CACHE_WARNING \ + "the number of relid cache entries reached the limit, please increase " \ "the GUC value for diskquota.max_active_tables." static void update_relation_entry(Oid relid, DiskQuotaRelationCacheEntry *relation_entry, From 2a2ed3678bc59575bb26b718462b8f15b3f6dfd9 Mon Sep 17 00:00:00 2001 From: Viktor Kurilko Date: Tue, 26 Dec 2023 17:59:33 +0700 Subject: [PATCH 27/31] make tests more stable --- tests/regress/expected/test_rejectmap_limit.out | 1 - tests/regress/sql/test_rejectmap_limit.sql | 2 -- 2 files changed, 3 deletions(-) diff --git a/tests/regress/expected/test_rejectmap_limit.out b/tests/regress/expected/test_rejectmap_limit.out index e45facc3..f3818a5d 100644 --- a/tests/regress/expected/test_rejectmap_limit.out +++ b/tests/regress/expected/test_rejectmap_limit.out @@ -73,7 +73,6 @@ SELECT diskquota.wait_for_worker_new_epoch(); t (1 row) -INSERT INTO s5.a SELECT generate_series(1,10); -- should be successful SELECT count(*) FROM master_log WHERE line LIKE '%the number of local quota reject map entries reached the limit%' AND line NOT LIKE '%LOG%'; count ------- diff --git a/tests/regress/sql/test_rejectmap_limit.sql b/tests/regress/sql/test_rejectmap_limit.sql index 838033ef..bed80aa4 100644 --- a/tests/regress/sql/test_rejectmap_limit.sql +++ b/tests/regress/sql/test_rejectmap_limit.sql @@ -43,8 +43,6 @@ INSERT INTO s5.a SELECT generate_series(1,100000); SELECT diskquota.wait_for_worker_new_epoch(); -INSERT INTO s5.a SELECT generate_series(1,10); -- should be successful - SELECT count(*) FROM master_log WHERE line LIKE '%the number of local quota reject map entries reached the limit%' AND line NOT LIKE '%LOG%'; DROP EXTENSION diskquota; From df6ef6f94ccb3a92397a85bf6abbdfcca7fa02cd Mon Sep 17 00:00:00 2001 From: Viktor Kurilko Date: Wed, 27 Dec 2023 12:46:23 +0700 Subject: [PATCH 28/31] add comment to tests. add new line at the end of new test --- tests/regress/expected/test_activetable_limit.out | 1 + tests/regress/expected/test_rejectmap_limit.out | 1 + tests/regress/sql/test_activetable_limit.sql | 2 +- tests/regress/sql/test_rejectmap_limit.sql | 3 ++- 4 files changed, 5 insertions(+), 2 deletions(-) diff --git a/tests/regress/expected/test_activetable_limit.out b/tests/regress/expected/test_activetable_limit.out index 0b34e601..5b132120 100644 --- a/tests/regress/expected/test_activetable_limit.out +++ b/tests/regress/expected/test_activetable_limit.out @@ -14,6 +14,7 @@ INSERT INTO a02 values(generate_series(0, 500)); INSERT INTO a03 values(generate_series(0, 500)); \c test_tablenum_limit_02 CREATE EXTENSION diskquota; +-- we only read the current log file CREATE EXTERNAL WEB TABLE segment_logs(line text) EXECUTE 'cat $GP_SEG_DATADIR/pg_log/$(ls -Art $GP_SEG_DATADIR/pg_log | tail -n 1)' ON ALL FORMAT 'TEXT' (DELIMITER 'OFF'); diff --git a/tests/regress/expected/test_rejectmap_limit.out b/tests/regress/expected/test_rejectmap_limit.out index f3818a5d..8e1c0eaa 100644 --- a/tests/regress/expected/test_rejectmap_limit.out +++ b/tests/regress/expected/test_rejectmap_limit.out @@ -13,6 +13,7 @@ SELECT diskquota.wait_for_worker_new_epoch(); t (1 row) +-- we only read the current log file CREATE EXTERNAL WEB TABLE master_log(line text) EXECUTE 'cat $GP_SEG_DATADIR/pg_log/$(ls -Art $GP_SEG_DATADIR/pg_log | tail -n 1)' ON MASTER FORMAT 'TEXT' (DELIMITER 'OFF'); diff --git a/tests/regress/sql/test_activetable_limit.sql b/tests/regress/sql/test_activetable_limit.sql index bd94e8a0..601d7111 100644 --- a/tests/regress/sql/test_activetable_limit.sql +++ b/tests/regress/sql/test_activetable_limit.sql @@ -20,7 +20,7 @@ INSERT INTO a03 values(generate_series(0, 500)); \c test_tablenum_limit_02 CREATE EXTENSION diskquota; - +-- we only read the current log file CREATE EXTERNAL WEB TABLE segment_logs(line text) EXECUTE 'cat $GP_SEG_DATADIR/pg_log/$(ls -Art $GP_SEG_DATADIR/pg_log | tail -n 1)' ON ALL FORMAT 'TEXT' (DELIMITER 'OFF'); diff --git a/tests/regress/sql/test_rejectmap_limit.sql b/tests/regress/sql/test_rejectmap_limit.sql index bed80aa4..8bed5cf9 100644 --- a/tests/regress/sql/test_rejectmap_limit.sql +++ b/tests/regress/sql/test_rejectmap_limit.sql @@ -12,6 +12,7 @@ CREATE DATABASE test_reject_map_limit_01; \c test_reject_map_limit_01 CREATE EXTENSION diskquota; SELECT diskquota.wait_for_worker_new_epoch(); +-- we only read the current log file CREATE EXTERNAL WEB TABLE master_log(line text) EXECUTE 'cat $GP_SEG_DATADIR/pg_log/$(ls -Art $GP_SEG_DATADIR/pg_log | tail -n 1)' ON MASTER FORMAT 'TEXT' (DELIMITER 'OFF'); @@ -51,4 +52,4 @@ DROP EXTENSION diskquota; DROP DATABASE test_reject_map_limit_01; \! gpconfig -r diskquota.max_reject_entries > /dev/null -\! gpstop -arf > /dev/null \ No newline at end of file +\! gpstop -arf > /dev/null From 45085010854816e9afcb087a0d6bc2ab27c098d9 Mon Sep 17 00:00:00 2001 From: Viktor Kurilko Date: Fri, 29 Dec 2023 17:27:59 +0700 Subject: [PATCH 29/31] change macro to const varibles. --- src/gp_activetable.c | 20 ++++++++++---------- src/quotamodel.c | 36 ++++++++++++++++++------------------ src/relation_cache.c | 16 ++++++++-------- 3 files changed, 36 insertions(+), 36 deletions(-) diff --git a/src/gp_activetable.c b/src/gp_activetable.c index de4062ff..d22de2a9 100644 --- a/src/gp_activetable.c +++ b/src/gp_activetable.c @@ -54,9 +54,9 @@ typedef struct DiskQuotaSetOFCache static HTAB *active_tables_map = NULL; // Set TimestampTz active_tables_map_last_overflow_report = 0; -#define ACTIVE_TABLES_MAP_WARNING \ - "the number of active tables reached the limit, please increase " \ - "the GUC value for diskquota.max_active_tables." +const char *active_tables_map_warning = + "the number of active tables reached the limit, please increase " + "the GUC value for diskquota.max_active_tables."; /* * monitored_dbid_cache is a allow list for diskquota @@ -68,9 +68,9 @@ TimestampTz active_tables_map_last_overflow_report = 0; static HTAB *altered_reloid_cache = NULL; // Set static TimestampTz altered_reloid_cache_last_overflow_report = 0; -#define ALTERED_RELOID_CACHE_WARNING \ - "the number of altered reloid cache entries reached the limit, please increase " \ - "the GUC value for diskquota.max_active_tables." +const char *altered_reloid_cache_warning = + "the number of altered reloid cache entries reached the limit, please increase " + "the GUC value for diskquota.max_active_tables."; /* active table hooks which detect the disk file size change. */ static file_create_hook_type prev_file_create_hook = NULL; @@ -247,7 +247,7 @@ report_altered_reloid(Oid reloid) LWLockAcquire(diskquota_locks.altered_reloid_cache_lock, LW_EXCLUSIVE); HASHACTION action = check_hash_fullness(altered_reloid_cache, diskquota_max_active_tables, - ALTERED_RELOID_CACHE_WARNING, &altered_reloid_cache_last_overflow_report); + altered_reloid_cache_warning, &altered_reloid_cache_last_overflow_report); hash_search(altered_reloid_cache, &reloid, action, NULL); LWLockRelease(diskquota_locks.altered_reloid_cache_lock); } @@ -330,7 +330,7 @@ report_active_table_helper(const RelFileNodeBackend *relFileNode) item.tablespaceoid = relFileNode->node.spcNode; LWLockAcquire(diskquota_locks.active_table_lock, LW_EXCLUSIVE); - HASHACTION action = check_hash_fullness(active_tables_map, diskquota_max_active_tables, ACTIVE_TABLES_MAP_WARNING, + HASHACTION action = check_hash_fullness(active_tables_map, diskquota_max_active_tables, active_tables_map_warning, &active_tables_map_last_overflow_report); entry = hash_search(active_tables_map, &item, action, &found); if (entry && !found) *entry = item; @@ -863,7 +863,7 @@ get_active_tables_oid(void) while ((active_table_file_entry = (DiskQuotaActiveTableFileEntry *)hash_seq_search(&iter)) != NULL) { HASHACTION action = check_hash_fullness(active_tables_map, diskquota_max_active_tables, - ACTIVE_TABLES_MAP_WARNING, &active_tables_map_last_overflow_report); + active_tables_map_warning, &active_tables_map_last_overflow_report); hash_search(active_tables_map, active_table_file_entry, action, NULL); } /* TODO: hash_seq_term(&iter); */ @@ -927,7 +927,7 @@ get_active_tables_oid(void) while ((active_table_file_entry = (DiskQuotaActiveTableFileEntry *)hash_seq_search(&iter)) != NULL) { HASHACTION action = check_hash_fullness(active_tables_map, diskquota_max_active_tables, - ACTIVE_TABLES_MAP_WARNING, &active_tables_map_last_overflow_report); + active_tables_map_warning, &active_tables_map_last_overflow_report); entry = hash_search(active_tables_map, active_table_file_entry, action, &found); if (entry) *entry = *active_table_file_entry; } diff --git a/src/quotamodel.c b/src/quotamodel.c index 9bb1e9ae..9badcf40 100644 --- a/src/quotamodel.c +++ b/src/quotamodel.c @@ -142,9 +142,9 @@ static HTAB *quota_info_map; /* stored in shared memory */ static TimestampTz *quota_info_map_last_overflow_report = NULL; -#define QUOTA_INFO_MAP_WARNING \ - "the number of quota probe reached the limit, please " \ - "increase the GUC value for diskquota.max_quota_probes." +const char *quota_info_map_warning = + "the number of quota probe reached the limit, please " + "increase the GUC value for diskquota.max_quota_probes."; /* global rejectmap for which exceed their quota limit */ struct RejectMapEntry @@ -188,9 +188,9 @@ static HTAB *table_size_map = NULL; /* stored in shared memory */ static TimestampTz *table_size_map_last_overflow_report = NULL; -#define TABLE_SIZE_MAP_WARNING \ - "the number of tables reached the limit, please increase " \ - "the GUC value for diskquota.max_table_segments." +const char *table_size_map_warning = + "the number of tables reached the limit, please increase " + "the GUC value for diskquota.max_table_segments."; /* rejectmap for database objects which exceed their quota limit */ static HTAB *disk_quota_reject_map = NULL; @@ -200,13 +200,13 @@ static TimestampTz disk_quota_reject_map_last_overflow_report = 0; /* stored in shared memory */ static TimestampTz *local_disk_quota_reject_map_last_overflow_report = NULL; -#define DISK_QUOTA_REJECT_MAP_WARNING \ - "the number of quota reject map entries reached the limit, " \ - "please increase the GUC value for diskquota.max_reject_entries." +const char *disk_quota_reject_map_warning = + "the number of quota reject map entries reached the limit, " + "please increase the GUC value for diskquota.max_reject_entries."; -#define LOCAL_DISK_QUOTA_REJECT_MAP_WARNING \ - "the number of local quota reject map entries reached the limit, " \ - "please increase the GUC value for diskquota.max_reject_entries." +const char *local_disk_quota_reject_map_warning = + "the number of local quota reject map entries reached the limit, " + "please increase the GUC value for diskquota.max_reject_entries."; static shmem_startup_hook_type prev_shmem_startup_hook = NULL; @@ -253,7 +253,7 @@ update_size_for_quota(int64 size, QuotaType type, Oid *keys, int16 segid) memcpy(key.keys, keys, quota_key_num[type] * sizeof(Oid)); key.type = type; key.segid = segid; - action = check_hash_fullness(quota_info_map, diskquota_max_quota_probes, QUOTA_INFO_MAP_WARNING, + action = check_hash_fullness(quota_info_map, diskquota_max_quota_probes, quota_info_map_warning, quota_info_map_last_overflow_report); entry = hash_search(quota_info_map, &key, action, &found); /* If the number of quota exceeds the limit, entry will be NULL */ @@ -280,7 +280,7 @@ update_limit_for_quota(int64 limit, float segratio, QuotaType type, Oid *keys) memcpy(key.keys, keys, quota_key_num[type] * sizeof(Oid)); key.type = type; key.segid = i; - action = check_hash_fullness(quota_info_map, diskquota_max_quota_probes, QUOTA_INFO_MAP_WARNING, + action = check_hash_fullness(quota_info_map, diskquota_max_quota_probes, quota_info_map_warning, quota_info_map_last_overflow_report); entry = hash_search(quota_info_map, &key, action, &found); /* If the number of quota exceeds the limit, entry will be NULL */ @@ -312,7 +312,7 @@ add_quota_to_rejectmap(QuotaType type, Oid targetOid, Oid tablespaceoid, bool se keyitem.targettype = (uint32)type; HASHACTION action = check_hash_fullness(local_disk_quota_reject_map, diskquota_max_local_reject_entries, - LOCAL_DISK_QUOTA_REJECT_MAP_WARNING, local_disk_quota_reject_map_last_overflow_report); + local_disk_quota_reject_map_warning, local_disk_quota_reject_map_last_overflow_report); localrejectentry = hash_search(local_disk_quota_reject_map, &keyitem, action, NULL); if (localrejectentry) { @@ -1012,7 +1012,7 @@ calculate_table_disk_usage(bool is_init, HTAB *local_active_table_stat_map) key.reloid = relOid; key.id = TableSizeEntryId(cur_segid); - HASHACTION action = check_hash_fullness(table_size_map, MAX_NUM_TABLE_SIZE_ENTRIES, TABLE_SIZE_MAP_WARNING, + HASHACTION action = check_hash_fullness(table_size_map, MAX_NUM_TABLE_SIZE_ENTRIES, table_size_map_warning, table_size_map_last_overflow_report); tsentry = hash_search(table_size_map, &key, action, &table_size_map_found); @@ -1303,7 +1303,7 @@ flush_local_reject_map(void) { HASHACTION action = check_hash_fullness(disk_quota_reject_map, MAX_DISK_QUOTA_REJECT_ENTRIES, - DISK_QUOTA_REJECT_MAP_WARNING, &disk_quota_reject_map_last_overflow_report); + disk_quota_reject_map_warning, &disk_quota_reject_map_last_overflow_report); rejectentry = hash_search(disk_quota_reject_map, &localrejectentry->keyitem, action, &found); if (rejectentry == NULL) { @@ -2145,7 +2145,7 @@ refresh_rejectmap(PG_FUNCTION_ARGS) if (OidIsValid(rejectmapentry->keyitem.targetoid)) continue; HASHACTION action = - check_hash_fullness(disk_quota_reject_map, MAX_DISK_QUOTA_REJECT_ENTRIES, DISK_QUOTA_REJECT_MAP_WARNING, + check_hash_fullness(disk_quota_reject_map, MAX_DISK_QUOTA_REJECT_ENTRIES, disk_quota_reject_map_warning, &disk_quota_reject_map_last_overflow_report); new_entry = hash_search(disk_quota_reject_map, &rejectmapentry->keyitem, action, &found); if (!found && new_entry) memcpy(new_entry, rejectmapentry, sizeof(GlobalRejectMapEntry)); diff --git a/src/relation_cache.c b/src/relation_cache.c index 614abb4d..50f3434c 100644 --- a/src/relation_cache.c +++ b/src/relation_cache.c @@ -34,13 +34,13 @@ HTAB *relid_cache = NULL; extern TimestampTz active_tables_map_last_overflow_report; -#define RELATION_CACHE_WARNING \ - "the number of relation cache entries reached the limit, please increase " \ - "the GUC value for diskquota.max_active_tables." +const char *relation_cache_warning = + "the number of relation cache entries reached the limit, please increase " + "the GUC value for diskquota.max_active_tables."; -#define RELID_CACHE_WARNING \ - "the number of relid cache entries reached the limit, please increase " \ - "the GUC value for diskquota.max_active_tables." +const char *relid_cache_warning = + "the number of relid cache entries reached the limit, please increase " + "the GUC value for diskquota.max_active_tables."; static void update_relation_entry(Oid relid, DiskQuotaRelationCacheEntry *relation_entry, DiskQuotaRelidCacheEntry *relid_entry); @@ -189,7 +189,7 @@ update_relation_cache(Oid relid) LWLockAcquire(diskquota_locks.relation_cache_lock, LW_EXCLUSIVE); - action = check_hash_fullness(relation_cache, diskquota_max_active_tables, RELATION_CACHE_WARNING, + action = check_hash_fullness(relation_cache, diskquota_max_active_tables, relation_cache_warning, &active_tables_map_last_overflow_report); relation_entry = hash_search(relation_cache, &relation_entry_data.relid, action, NULL); @@ -200,7 +200,7 @@ update_relation_cache(Oid relid) } memcpy(relation_entry, &relation_entry_data, sizeof(DiskQuotaRelationCacheEntry)); - action = check_hash_fullness(relid_cache, diskquota_max_active_tables, RELID_CACHE_WARNING, + action = check_hash_fullness(relid_cache, diskquota_max_active_tables, relid_cache_warning, &active_tables_map_last_overflow_report); relid_entry = hash_search(relid_cache, &relid_entry_data.relfilenode, action, NULL); if (relid_entry == NULL) From 7f66cd5925282506af7259ef9741cd5293b423a6 Mon Sep 17 00:00:00 2001 From: Viktor Kurilko Date: Fri, 29 Dec 2023 18:00:56 +0700 Subject: [PATCH 30/31] make warning constats static --- src/gp_activetable.c | 2 +- src/quotamodel.c | 8 ++++---- src/relation_cache.c | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/gp_activetable.c b/src/gp_activetable.c index d22de2a9..c6673a59 100644 --- a/src/gp_activetable.c +++ b/src/gp_activetable.c @@ -54,7 +54,7 @@ typedef struct DiskQuotaSetOFCache static HTAB *active_tables_map = NULL; // Set TimestampTz active_tables_map_last_overflow_report = 0; -const char *active_tables_map_warning = +static const char *active_tables_map_warning = "the number of active tables reached the limit, please increase " "the GUC value for diskquota.max_active_tables."; diff --git a/src/quotamodel.c b/src/quotamodel.c index 9badcf40..f558bc66 100644 --- a/src/quotamodel.c +++ b/src/quotamodel.c @@ -142,7 +142,7 @@ static HTAB *quota_info_map; /* stored in shared memory */ static TimestampTz *quota_info_map_last_overflow_report = NULL; -const char *quota_info_map_warning = +static const char *quota_info_map_warning = "the number of quota probe reached the limit, please " "increase the GUC value for diskquota.max_quota_probes."; @@ -188,7 +188,7 @@ static HTAB *table_size_map = NULL; /* stored in shared memory */ static TimestampTz *table_size_map_last_overflow_report = NULL; -const char *table_size_map_warning = +static const char *table_size_map_warning = "the number of tables reached the limit, please increase " "the GUC value for diskquota.max_table_segments."; @@ -200,11 +200,11 @@ static TimestampTz disk_quota_reject_map_last_overflow_report = 0; /* stored in shared memory */ static TimestampTz *local_disk_quota_reject_map_last_overflow_report = NULL; -const char *disk_quota_reject_map_warning = +static const char *disk_quota_reject_map_warning = "the number of quota reject map entries reached the limit, " "please increase the GUC value for diskquota.max_reject_entries."; -const char *local_disk_quota_reject_map_warning = +static const char *local_disk_quota_reject_map_warning = "the number of local quota reject map entries reached the limit, " "please increase the GUC value for diskquota.max_reject_entries."; diff --git a/src/relation_cache.c b/src/relation_cache.c index 50f3434c..b5624c42 100644 --- a/src/relation_cache.c +++ b/src/relation_cache.c @@ -34,11 +34,11 @@ HTAB *relid_cache = NULL; extern TimestampTz active_tables_map_last_overflow_report; -const char *relation_cache_warning = +static const char *relation_cache_warning = "the number of relation cache entries reached the limit, please increase " "the GUC value for diskquota.max_active_tables."; -const char *relid_cache_warning = +static const char *relid_cache_warning = "the number of relid cache entries reached the limit, please increase " "the GUC value for diskquota.max_active_tables."; From 7139176019589de88b04453d08f931974b1aca44 Mon Sep 17 00:00:00 2001 From: Viktor Kurilko Date: Fri, 29 Dec 2023 18:10:22 +0700 Subject: [PATCH 31/31] add static to altered_reloid_cache_warning --- src/gp_activetable.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/gp_activetable.c b/src/gp_activetable.c index c6673a59..adbd69ef 100644 --- a/src/gp_activetable.c +++ b/src/gp_activetable.c @@ -68,7 +68,7 @@ static const char *active_tables_map_warning = static HTAB *altered_reloid_cache = NULL; // Set static TimestampTz altered_reloid_cache_last_overflow_report = 0; -const char *altered_reloid_cache_warning = +static const char *altered_reloid_cache_warning = "the number of altered reloid cache entries reached the limit, please increase " "the GUC value for diskquota.max_active_tables.";