WAL keyring improvements (percona#238)

- Rename database key rotation functions to make room for the global space ones.
- Now, during the first start, we would create a default temporary key provider for the global space. A user can (and should) create their own key provider afterwards. This allows use the same codepath and internal interfaces for the keyring management across databases and the global space.
- Now need to cache the principal key for the global space as we use it only at the server start to decrypt internal key. Then internal key persists in the memory cache.

Fixes https://perconadev.atlassian.net/browse/PG-835, https://perconadev.atlassian.net/browse/PG-833

Makefile.in

@@ -42,7 +42,7 @@ src/keyring/keyring_curl.o \
 src/keyring/keyring_file.o \
 src/keyring/keyring_vault.o \
 src/keyring/keyring_api.o \
-src/catalog/tde_global_catalog.o \
+src/catalog/tde_global_space.o \
 src/catalog/tde_keyring.o \
 src/catalog/tde_principal_key.o \
 src/common/pg_tde_shmem.o \

documentation/docs/functions.md

@@ -39,37 +39,37 @@ Sets the principal key for the database using the specified key provider.
 
 The principal key name is also used for constructing the name in the provider, for example on the remote Vault server.
 
-You can use this function only to a principal key. For changes in the principal key, use the [`pg_tde_rotate_key`](#pg_tde_rotate_key) function.
+You can use this function only to a principal key. For changes in the principal key, use the [`pg_tde_rotate_principal_key`](#pg_tde_rotate_principal_key) function.
 
 ```sql
 SELECT pg_tde_set_principal_key('name-of-the-principal-key', 'provider-name');
 ```
 
-## pg_tde_rotate_key
+## pg_tde_rotate_principal_key
 
 Creates a new version of the specified principal key and updates the database so that it uses the new principal key version.
 
 When used without any parameters, the function will just create a new version of the current database
 principal key, using the same provider:
 
 ```sql
-SELECT pg_tde_rotate_key();
+SELECT pg_tde_rotate_principal_key();
 ```
 
 Alternatively, you can pass two parameters to the function, specifying both a new key name and a new provider name:
 
 ```sql
-SELECT pg_tde_rotate_key('name-of-the-new-principal-key', 'name-of-the-new-provider');
+SELECT pg_tde_rotate_principal_key('name-of-the-new-principal-key', 'name-of-the-new-provider');
 ```
 
 Both parameters support the `NULL` value, which means that the parameter won't be changed:
 
 ```sql
 -- creates  new principal key on the same provider as before
-SELECT pg_tde_rotate_key('name-of-the-new-principal-key', NULL);
+SELECT pg_tde_rotate_principal_key('name-of-the-new-principal-key', NULL);
 
 -- copies the current principal key to a new provider
-SELECT pg_tde_rotate_key(NULL, 'name-of-the-new-provider');
+SELECT pg_tde_rotate_principal_key(NULL, 'name-of-the-new-provider');
 ```
 
 ## pg_tde_is_encrypted

documentation/docs/test.md

@@ -27,9 +27,9 @@ To check if the data is encrypted, do the following:
 3. Rotate the principal key when needed:
 
     ```sql
-    SELECT pg_tde_rotate_key(); -- uses automatic key versionin
+    SELECT pg_tde_rotate_principal_key(); -- uses automatic key versionin
     -- or
-    SELECT pg_tde_rotate_key('new-principal-key', NULL); -- specify new key name
+    SELECT pg_tde_rotate_principal_key('new-principal-key', NULL); -- specify new key name
     -- or
-    SELECT pg_tde_rotate_key('new-principal-key', 'new-provider'); -- change provider
+    SELECT pg_tde_rotate_principal_key('new-principal-key', 'new-provider'); -- change provider
     ```

expected/pg_tde_is_encrypted.out

@@ -2,6 +2,7 @@ CREATE EXTENSION pg_tde;
 SELECT  * FROM pg_tde_principal_key_info();
 ERROR:  Principal key does not exists for the database
 HINT:  Use set_principal_key interface to set the principal key
+CONTEXT:  SQL function "pg_tde_principal_key_info" statement 1
 SELECT pg_tde_add_key_provider_file('file-vault','/tmp/pg_tde_test_keyring.per');
  pg_tde_add_key_provider_file 
 ------------------------------

expected/test_issue_153_fix.out

@@ -3,6 +3,7 @@ SET datestyle TO 'iso, dmy';
 SELECT  * FROM pg_tde_principal_key_info();
 ERROR:  Principal key does not exists for the database
 HINT:  Use set_principal_key interface to set the principal key
+CONTEXT:  SQL function "pg_tde_principal_key_info" statement 1
 SELECT pg_tde_add_key_provider_file('file-ring','/tmp/pg_tde_test_keyring.per');
  pg_tde_add_key_provider_file 
 ------------------------------

meson.build

@@ -41,7 +41,7 @@ pg_tde_sources = files(
 
         'src/smgr/pg_tde_smgr.c',
 
-        'src/catalog/tde_global_catalog.c',
+        'src/catalog/tde_global_space.c',
         'src/catalog/tde_keyring.c',
         'src/catalog/tde_principal_key.c',
         'src/common/pg_tde_shmem.c',

pg_tde--1.0.sql

@@ -3,16 +3,18 @@
 -- complain if script is sourced in psql, rather than via CREATE EXTENSION
 \echo Use "CREATE EXTENSION pg_tde" to load this file. \quit
 
+CREATE type PG_TDE_GLOBAL AS ENUM('PG_TDE_GLOBAL');
+
 -- Key Provider Management
-CREATE FUNCTION pg_tde_add_key_provider_internal(provider_type VARCHAR(10), provider_name VARCHAR(128), options JSON)
+CREATE FUNCTION pg_tde_add_key_provider_internal(provider_type VARCHAR(10), provider_name VARCHAR(128), options JSON, is_global BOOLEAN)
 RETURNS INT
 AS 'MODULE_PATHNAME'
 LANGUAGE C;
 
 CREATE OR REPLACE FUNCTION pg_tde_add_key_provider(provider_type VARCHAR(10), provider_name VARCHAR(128), options JSON)
 RETURNS INT
 AS $$
-    SELECT pg_tde_add_key_provider_internal(provider_type, provider_name, options);
+    SELECT pg_tde_add_key_provider_internal(provider_type, provider_name, options, FALSE);
 $$
 LANGUAGE SQL;
 
@@ -65,6 +67,55 @@ RETURNS SETOF record
 AS 'MODULE_PATHNAME'
 LANGUAGE C STRICT VOLATILE;
 
+-- Global Tblespace Key Provider Management
+CREATE OR REPLACE FUNCTION pg_tde_add_key_provider(PG_TDE_GLOBAL, provider_type VARCHAR(10), provider_name VARCHAR(128), options JSON)
+RETURNS INT
+AS $$
+    SELECT pg_tde_add_key_provider_internal(provider_type, provider_name, options, TRUE);
+$$
+LANGUAGE SQL;
+
+CREATE OR REPLACE FUNCTION pg_tde_add_key_provider_file(PG_TDE_GLOBAL, provider_name VARCHAR(128), file_path TEXT)
+RETURNS INT
+AS $$
+-- JSON keys in the options must be matched to the keys in
+-- load_file_keyring_provider_options function.
+
+    SELECT pg_tde_add_key_provider('PG_TDE_GLOBAL', 'file', provider_name,
+                json_object('type' VALUE 'file', 'path' VALUE COALESCE(file_path, '')));
+$$
+LANGUAGE SQL;
+
+CREATE OR REPLACE FUNCTION pg_tde_add_key_provider_file(PG_TDE_GLOBAL, provider_name VARCHAR(128), file_path JSON)
+RETURNS INT
+AS $$
+-- JSON keys in the options must be matched to the keys in
+-- load_file_keyring_provider_options function.
+
+    SELECT pg_tde_add_key_provider('PG_TDE_GLOBAL', 'file', provider_name,
+                json_object('type' VALUE 'file', 'path' VALUE file_path));
+$$
+LANGUAGE SQL;
+
+CREATE OR REPLACE FUNCTION pg_tde_add_key_provider_vault_v2(PG_TDE_GLOBAL, 
+														provider_name VARCHAR(128),
+                                                        vault_token TEXT,
+                                                        vault_url TEXT,
+                                                        vault_mount_path TEXT,
+                                                        vault_ca_path TEXT)
+RETURNS INT
+AS $$
+-- JSON keys in the options must be matched to the keys in
+-- load_vaultV2_keyring_provider_options function.
+    SELECT pg_tde_add_key_provider('PG_TDE_GLOBAL', 'vault-v2', provider_name,
+                            json_object('type' VALUE 'vault-v2',
+                            'url' VALUE COALESCE(vault_url,''),
+                            'token' VALUE COALESCE(vault_token,''),
+                            'mountPath' VALUE COALESCE(vault_mount_path,''),
+                            'caPath' VALUE COALESCE(vault_ca_path,'')));
+$$
+LANGUAGE SQL;
+
 -- Table access method
 CREATE FUNCTION pg_tdeam_basic_handler(internal)
 RETURNS table_am_handler
@@ -82,11 +133,25 @@ SELECT EXISTS (
     )$$
 LANGUAGE SQL;
 
-CREATE FUNCTION pg_tde_rotate_key(new_principal_key_name VARCHAR(255) DEFAULT NULL, new_provider_name VARCHAR(255) DEFAULT NULL, ensure_new_key BOOLEAN DEFAULT TRUE)
+CREATE FUNCTION pg_tde_rotate_principal_key_internal(new_principal_key_name VARCHAR(255) DEFAULT NULL, new_provider_name VARCHAR(255) DEFAULT NULL, ensure_new_key BOOLEAN DEFAULT TRUE, is_global BOOLEAN DEFAULT FALSE)
 RETURNS boolean
 AS 'MODULE_PATHNAME'
 LANGUAGE C;
 
+CREATE FUNCTION pg_tde_rotate_principal_key(new_principal_key_name VARCHAR(255) DEFAULT NULL, new_provider_name VARCHAR(255) DEFAULT NULL)
+RETURNS boolean
+AS $$
+    SELECT pg_tde_rotate_principal_key_internal(new_principal_key_name, new_provider_name, TRUE, FALSE);
+$$
+LANGUAGE SQL;
+
+CREATE FUNCTION pg_tde_rotate_principal_key(PG_TDE_GLOBAL, new_principal_key_name VARCHAR(255) DEFAULT NULL, new_provider_name VARCHAR(255) DEFAULT NULL)
+RETURNS boolean
+AS $$
+    SELECT pg_tde_rotate_principal_key_internal(new_principal_key_name, new_provider_name, TRUE, TRUE);
+$$
+LANGUAGE SQL;
+
 CREATE FUNCTION pg_tde_set_principal_key(principal_key_name VARCHAR(255), provider_name VARCHAR(255), ensure_new_key BOOLEAN DEFAULT FALSE)
 RETURNS boolean
 AS 'MODULE_PATHNAME'
@@ -97,7 +162,7 @@ RETURNS VOID
 AS 'MODULE_PATHNAME'
 LANGUAGE C;
 
-CREATE FUNCTION pg_tde_principal_key_info()
+CREATE FUNCTION pg_tde_principal_key_info_internal(is_global BOOLEAN)
 RETURNS TABLE ( principal_key_name text,
                 key_provider_name text,
                 key_provider_id integer,
@@ -107,6 +172,30 @@ RETURNS TABLE ( principal_key_name text,
 AS 'MODULE_PATHNAME'
 LANGUAGE C;
 
+CREATE FUNCTION pg_tde_principal_key_info()
+RETURNS TABLE ( principal_key_name text,
+                key_provider_name text,
+                key_provider_id integer,
+                principal_key_internal_name text,
+                principal_key_version integer,
+                key_createion_time timestamp with time zone)
+AS $$
+    SELECT pg_tde_principal_key_info_internal(FALSE);
+$$
+LANGUAGE SQL;
+
+CREATE FUNCTION pg_tde_principal_key_info(PG_TDE_GLOBAL)
+RETURNS TABLE ( principal_key_name text,
+                key_provider_name text,
+                key_provider_id integer,
+                principal_key_internal_name text,
+                principal_key_version integer,
+                key_createion_time timestamp with time zone)
+AS $$
+    SELECT pg_tde_principal_key_info_internal(TRUE);
+$$
+LANGUAGE SQL;
+
 CREATE FUNCTION pg_tde_version() RETURNS TEXT AS 'MODULE_PATHNAME' LANGUAGE C;
 
 -- Access method

src/access/pg_tde_tdemap.c

@@ -92,7 +92,6 @@ static bool pg_tde_read_one_map_entry(int fd, const RelFileLocator *rlocator, in
 
 static void pg_tde_write_keydata(char *db_keydata_path, TDEPrincipalKeyInfo *principal_key_info, int32 key_index, RelKeyData *enc_rel_key_data);
 static void pg_tde_write_one_keydata(int keydata_fd, int32 key_index, RelKeyData *enc_rel_key_data);
-static RelKeyData* pg_tde_get_key_from_file(const RelFileLocator *rlocator, GenericKeyring *keyring);
 static RelKeyData* pg_tde_read_keydata(char *db_keydata_path, int32 key_index, TDEPrincipalKey *principal_key);
 static RelKeyData* pg_tde_read_one_keydata(int keydata_fd, int32 key_index, TDEPrincipalKey *principal_key);
 
@@ -111,7 +110,7 @@ pg_tde_create_key_map_entry(const RelFileLocator *newrlocator)
 	TDEPrincipalKey *principal_key;
 	XLogRelKey xlrec;
 
-	principal_key = GetPrincipalKey(newrlocator->dbOid, newrlocator->spcOid, NULL);
+	principal_key = GetPrincipalKey(newrlocator->dbOid, newrlocator->spcOid);
 	if (principal_key == NULL)
 	{
 		ereport(ERROR,
@@ -164,12 +163,6 @@ RelKey *tde_rel_key_map = NULL;
  */
 RelKeyData *
 GetRelationKey(RelFileLocator rel)
-{
-	return GetRelationKeyWithKeyring(rel, NULL);
-}
-
-RelKeyData *
-GetRelationKeyWithKeyring(RelFileLocator rel, GenericKeyring *keyring)
 {
 	RelKey		*curr;
 	RelKeyData *key;
@@ -183,7 +176,7 @@ GetRelationKeyWithKeyring(RelFileLocator rel, GenericKeyring *keyring)
 		}
 	}
 
-	key = pg_tde_get_key_from_file(&rel, keyring);
+	key = pg_tde_get_key_from_file(&rel);
 
 	if (key != NULL)
 	{
@@ -996,8 +989,8 @@ pg_tde_free_key_map_entry(const RelFileLocator *rlocator, off_t offset)
  * Reads the key of the required relation. It identifies its map entry and then simply
  * reads the key data from the keydata file.
  */
-static RelKeyData *
-pg_tde_get_key_from_file(const RelFileLocator *rlocator, GenericKeyring *keyring)
+RelKeyData *
+pg_tde_get_key_from_file(const RelFileLocator *rlocator)
 {
 	int32		key_index = 0;
 	TDEPrincipalKey	*principal_key;
@@ -1013,7 +1006,7 @@ pg_tde_get_key_from_file(const RelFileLocator *rlocator, GenericKeyring *keyring
 	LWLockAcquire(lock_files, LW_SHARED);
 
 	/* Get/generate a principal key, create the key for relation and get the encrypted key with bytes to write */
-	principal_key = GetPrincipalKey(rlocator->dbOid, rlocator->spcOid, keyring);
+	principal_key = GetPrincipalKey(rlocator->dbOid, rlocator->spcOid);
 	if (principal_key == NULL)
 	{
 		LWLockRelease(lock_files);
@@ -1179,6 +1172,7 @@ pg_tde_perform_rotate_key(TDEPrincipalKey *principal_key, TDEPrincipalKey *new_p
 	xlrec->keydata_size = keydata_size;
 
 	/* TODO: pgstat_report_wait_start / pgstat_report_wait_end */
+	/* TODO: error handling */
 	pg_pread(m_fd[NEW_PRINCIPAL_KEY], xlrec->buff, xlrec->map_size, 0);
 	pg_pread(k_fd[NEW_PRINCIPAL_KEY], &xlrec->buff[xlrec->map_size], xlrec->keydata_size, 0);
 

src/access/pg_tde_xlog.c

@@ -28,7 +28,7 @@
 #include "access/pg_tde_xlog.h"
 #include "encryption/enc_tde.h"
 #ifdef PERCONA_FORK
-#include "catalog/tde_global_catalog.h"
+#include "catalog/tde_global_space.h"
 
 static char *TDEXLogEncryptBuf = NULL;
 
@@ -240,7 +240,7 @@ TDEXLogWriteEncryptedPages(int fd, const void *buf, size_t count, off_t offset)
 	size_t	data_size = 0;
 	XLogPageHeader	curr_page_hdr = &EncryptCurrentPageHrd;
 	XLogPageHeader	enc_buf_page;
-	RelKeyData		*key = GetGlCatInternalKey(XLOG_TDE_OID);
+	RelKeyData		*key = TDEGetGlobalInternalKey(XLOG_TDE_OID);
 	off_t	enc_off;
 	size_t	page_size = XLOG_BLCKSZ - offset % XLOG_BLCKSZ;
 	uint32	iv_ctr = 0;
@@ -327,7 +327,7 @@ tdeheap_xlog_seg_read(int fd, void *buf, size_t count, off_t offset)
 	char	iv_prefix[16] = {0,};
 	size_t	data_size = 0;
 	XLogPageHeader	curr_page_hdr = &DecryptCurrentPageHrd;
-	RelKeyData		*key = GetGlCatInternalKey(XLOG_TDE_OID);
+	RelKeyData		*key = TDEGetGlobalInternalKey(XLOG_TDE_OID);
 	size_t	page_size = XLOG_BLCKSZ - offset % XLOG_BLCKSZ;
 	off_t	dec_off;
 	uint32	iv_ctr = 0;