From 44bb73baf4d231a0b31f5ef1c6eaec748d3afdda Mon Sep 17 00:00:00 2001 From: Niklas Gehlen Date: Fri, 10 Jan 2025 13:01:06 +0100 Subject: [PATCH 1/2] Create experimental ability to provision Postgres helper functions Towards NSL-5018 --- .../postgres/prepare/database/functions.sql | 49 +++++++++++++++++++ library/oss/postgres/prepare/database/main.go | 21 +++++++- library/oss/postgres/types.pb.go | 24 ++++++--- library/oss/postgres/types.proto | 7 +-- 4 files changed, 91 insertions(+), 10 deletions(-) create mode 100644 library/oss/postgres/prepare/database/functions.sql diff --git a/library/oss/postgres/prepare/database/functions.sql b/library/oss/postgres/prepare/database/functions.sql new file mode 100644 index 000000000..2eb25279a --- /dev/null +++ b/library/oss/postgres/prepare/database/functions.sql @@ -0,0 +1,49 @@ +-- Collection of optional helper functions +-- To provision these functions add +-- provision_helper_functions: true +-- to the database intent. + +-- `CREATE TABLE IF NOT EXISTS` and `ALTER TABLE … ADD COLUMN IF NOT EXISTS` +-- both require exclusive locks with Postgres, even if the table/column already exists. +-- The functions below provide ensure semantics while only acquiring exclusive locks on mutations. + +-- fn_ensure_table is a lock-friendly replacement for `CREATE TABLE IF NOT EXISTS`. +-- +-- Example usage: +-- +-- SELECT fn_ensure_table('testtable', $$ +-- UserID TEXT NOT NULL, +-- PRIMARY KEY(UserID) +-- $$); +CREATE OR REPLACE FUNCTION fn_ensure_table(tname TEXT, def TEXT) + RETURNS void + LANGUAGE plpgsql AS +$func$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM pg_tables + WHERE schemaname = 'public' AND tablename = LOWER(tname) + ) THEN + EXECUTE 'CREATE TABLE IF NOT EXISTS ' || tname || ' (' || def || ');'; + END IF; +END +$func$; + +-- fn_ensure_column is a lock-friendly replacement for `ALTER TABLE ... ADD COLUMN IF NOT EXISTS`. +-- +-- Example usage: +-- +-- SELECT fn_ensure_column('testtable', 'CreatedAt', 'TIMESTAMP DEFAULT CURRENT_TIMESTAMP'); +CREATE OR REPLACE FUNCTION fn_ensure_column(tname TEXT, cname TEXT, def TEXT) + RETURNS void + LANGUAGE plpgsql AS +$func$ +BEGIN + IF NOT EXISTS ( + SELECT 1 FROM information_schema.columns + WHERE table_name = LOWER(tname) AND column_name = LOWER(cname) + ) THEN + EXECUTE 'ALTER TABLE ' || tname || ' ADD COLUMN IF NOT EXISTS ' || cname || ' ' || def; + END IF; +END +$func$; \ No newline at end of file diff --git a/library/oss/postgres/prepare/database/main.go b/library/oss/postgres/prepare/database/main.go index 0a4398809..fb9789528 100644 --- a/library/oss/postgres/prepare/database/main.go +++ b/library/oss/postgres/prepare/database/main.go @@ -6,7 +6,9 @@ package main import ( "context" + "embed" "fmt" + "io/fs" "log" "math/rand" "os" @@ -27,7 +29,13 @@ const ( connIdleTimeout = 15 * time.Minute connTimeout = 5 * time.Minute - caCertPath = "/tmp/ca.pem" + caCertPath = "/tmp/ca.pem" + helperFunctionsPath = "functions.sql" +) + +var ( + //go:embed *.sql + data embed.FS ) func main() { @@ -86,6 +94,17 @@ func run(ctx context.Context, p *provider.Provider[*postgres.DatabaseIntent]) er } }() + if p.Intent.ProvisionHelperFunctions { + content, err := fs.ReadFile(data, helperFunctionsPath) + if err != nil { + return fmt.Errorf("failed to read %s: %w", helperFunctionsPath, err) + } + + if err := applyWithRetry(ctx, db, string(content)); err != nil { + return fmt.Errorf("unable to apply helper functions: %w", err) + } + } + for _, schema := range p.Intent.Schema { if err := applyWithRetry(ctx, db, string(schema.Contents)); err != nil { return fmt.Errorf("unable to apply schema %q: %w", schema.Path, err) diff --git a/library/oss/postgres/types.pb.go b/library/oss/postgres/types.pb.go index fed50fdb9..eaff76aa4 100644 --- a/library/oss/postgres/types.pb.go +++ b/library/oss/postgres/types.pb.go @@ -91,6 +91,7 @@ type DatabaseIntent struct { Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"` Schema []*schema.FileContents `protobuf:"bytes,2,rep,name=schema,proto3" json:"schema,omitempty"` SkipSchemaInitializationIfExists bool `protobuf:"varint,3,opt,name=skip_schema_initialization_if_exists,json=skipSchemaInitializationIfExists,proto3" json:"skip_schema_initialization_if_exists,omitempty"` + ProvisionHelperFunctions bool `protobuf:"varint,4,opt,name=provision_helper_functions,json=provisionHelperFunctions,proto3" json:"provision_helper_functions,omitempty"` } func (x *DatabaseIntent) Reset() { @@ -146,6 +147,13 @@ func (x *DatabaseIntent) GetSkipSchemaInitializationIfExists() bool { return false } +func (x *DatabaseIntent) GetProvisionHelperFunctions() bool { + if x != nil { + return x.ProvisionHelperFunctions + } + return false +} + var File_library_oss_postgres_types_proto protoreflect.FileDescriptor var file_library_oss_postgres_types_proto_rawDesc = []byte{ @@ -164,7 +172,7 @@ var file_library_oss_postgres_types_proto_rawDesc = []byte{ 0x65, 0x63, 0x72, 0x65, 0x74, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0b, 0x32, 0x1d, 0x2e, 0x66, 0x6f, 0x75, 0x6e, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2e, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x2e, 0x50, 0x61, 0x63, 0x6b, 0x61, 0x67, 0x65, 0x52, 0x65, 0x66, 0x52, 0x0e, 0x70, 0x61, 0x73, 0x73, - 0x77, 0x6f, 0x72, 0x64, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x22, 0xad, 0x01, 0x0a, 0x0e, 0x44, + 0x77, 0x6f, 0x72, 0x64, 0x53, 0x65, 0x63, 0x72, 0x65, 0x74, 0x22, 0xeb, 0x01, 0x0a, 0x0e, 0x44, 0x61, 0x74, 0x61, 0x62, 0x61, 0x73, 0x65, 0x49, 0x6e, 0x74, 0x65, 0x6e, 0x74, 0x12, 0x12, 0x0a, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x18, 0x01, 0x20, 0x01, 0x28, 0x09, 0x52, 0x04, 0x6e, 0x61, 0x6d, 0x65, 0x12, 0x37, 0x0a, 0x06, 0x73, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x18, 0x02, 0x20, 0x03, 0x28, @@ -175,11 +183,15 @@ var file_library_oss_postgres_types_proto_rawDesc = []byte{ 0x6c, 0x69, 0x7a, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x5f, 0x69, 0x66, 0x5f, 0x65, 0x78, 0x69, 0x73, 0x74, 0x73, 0x18, 0x03, 0x20, 0x01, 0x28, 0x08, 0x52, 0x20, 0x73, 0x6b, 0x69, 0x70, 0x53, 0x63, 0x68, 0x65, 0x6d, 0x61, 0x49, 0x6e, 0x69, 0x74, 0x69, 0x61, 0x6c, 0x69, 0x7a, 0x61, 0x74, 0x69, - 0x6f, 0x6e, 0x49, 0x66, 0x45, 0x78, 0x69, 0x73, 0x74, 0x73, 0x42, 0x33, 0x5a, 0x31, 0x6e, 0x61, - 0x6d, 0x65, 0x73, 0x70, 0x61, 0x63, 0x65, 0x6c, 0x61, 0x62, 0x73, 0x2e, 0x64, 0x65, 0x76, 0x2f, - 0x66, 0x6f, 0x75, 0x6e, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x6c, 0x69, 0x62, 0x72, 0x61, - 0x72, 0x79, 0x2f, 0x6f, 0x73, 0x73, 0x2f, 0x70, 0x6f, 0x73, 0x74, 0x67, 0x72, 0x65, 0x73, 0x62, - 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, + 0x6f, 0x6e, 0x49, 0x66, 0x45, 0x78, 0x69, 0x73, 0x74, 0x73, 0x12, 0x3c, 0x0a, 0x1a, 0x70, 0x72, + 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x5f, 0x68, 0x65, 0x6c, 0x70, 0x65, 0x72, 0x5f, 0x66, + 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x18, 0x04, 0x20, 0x01, 0x28, 0x08, 0x52, 0x18, + 0x70, 0x72, 0x6f, 0x76, 0x69, 0x73, 0x69, 0x6f, 0x6e, 0x48, 0x65, 0x6c, 0x70, 0x65, 0x72, 0x46, + 0x75, 0x6e, 0x63, 0x74, 0x69, 0x6f, 0x6e, 0x73, 0x42, 0x33, 0x5a, 0x31, 0x6e, 0x61, 0x6d, 0x65, + 0x73, 0x70, 0x61, 0x63, 0x65, 0x6c, 0x61, 0x62, 0x73, 0x2e, 0x64, 0x65, 0x76, 0x2f, 0x66, 0x6f, + 0x75, 0x6e, 0x64, 0x61, 0x74, 0x69, 0x6f, 0x6e, 0x2f, 0x6c, 0x69, 0x62, 0x72, 0x61, 0x72, 0x79, + 0x2f, 0x6f, 0x73, 0x73, 0x2f, 0x70, 0x6f, 0x73, 0x74, 0x67, 0x72, 0x65, 0x73, 0x62, 0x06, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x33, } var ( diff --git a/library/oss/postgres/types.proto b/library/oss/postgres/types.proto index c88a192c9..9c6a27a1a 100644 --- a/library/oss/postgres/types.proto +++ b/library/oss/postgres/types.proto @@ -20,7 +20,8 @@ message ClusterIntent { message DatabaseIntent { // The database name is applied as is (e.g. it is case-sensitive). - string name = 1; - repeated foundation.schema.FileContents schema = 2; - bool skip_schema_initialization_if_exists = 3; + string name = 1; + repeated foundation.schema.FileContents schema = 2; + bool skip_schema_initialization_if_exists = 3; + bool provision_helper_functions = 4; } From 6a7d21be110380f5d2b484db2074b99a93cb181e Mon Sep 17 00:00:00 2001 From: Niklas Gehlen Date: Fri, 10 Jan 2025 13:06:44 +0100 Subject: [PATCH 2/2] Add a warning. --- library/oss/postgres/prepare/database/functions.sql | 2 ++ 1 file changed, 2 insertions(+) diff --git a/library/oss/postgres/prepare/database/functions.sql b/library/oss/postgres/prepare/database/functions.sql index 2eb25279a..91254db2d 100644 --- a/library/oss/postgres/prepare/database/functions.sql +++ b/library/oss/postgres/prepare/database/functions.sql @@ -8,6 +8,7 @@ -- The functions below provide ensure semantics while only acquiring exclusive locks on mutations. -- fn_ensure_table is a lock-friendly replacement for `CREATE TABLE IF NOT EXISTS`. +-- WARNING: This function does not support uppercase names. -- -- Example usage: -- @@ -30,6 +31,7 @@ END $func$; -- fn_ensure_column is a lock-friendly replacement for `ALTER TABLE ... ADD COLUMN IF NOT EXISTS`. +-- WARNING: This function does not support uppercase names. -- -- Example usage: --