From d17b35efab3c57df11eeddf825347b37ba2a4729 Mon Sep 17 00:00:00 2001 From: Julius Remigio <14811322+juls858@users.noreply.github.com> Date: Tue, 3 Oct 2023 17:48:11 -0700 Subject: [PATCH 1/4] - add secrets schema --- macros/core/secrets.yaml.sql | 65 ++++++++++++++++++++++++++++++++++ models/deploy/core/secrets.sql | 4 +++ 2 files changed, 69 insertions(+) create mode 100644 macros/core/secrets.yaml.sql create mode 100644 models/deploy/core/secrets.sql diff --git a/macros/core/secrets.yaml.sql b/macros/core/secrets.yaml.sql new file mode 100644 index 00000000..1fad3753 --- /dev/null +++ b/macros/core/secrets.yaml.sql @@ -0,0 +1,65 @@ +{% macro config_core_secrets(schema="utils") %} + + +- name: {{ schema }}.udf_register_secret + signature: + - [request_id, STRING] + - [key, STRING] + func_type: SECURE + return_type: TEXT + options: | + NOT NULL + RETURNS NULL ON NULL INPUT + IMMUTABLE + sql: | + SELECT + _utils.UDF_REGISTER_SECRET(REQUEST_ID, _utils.UDF_WHOAMI(), KEY) + +- name: {{ schema }}.udf_get_secret + signature: + - [secret_name, STRING] + func_type: SECURE + return_type: VARIANT + options: | + NOT NULL + RETURNS NULL ON NULL INPUT + IMMUTABLE + sql: | + SELECT + live.udf_api(CONCAT_WS('/', {{ construct_api_route("/secret") }}, _utils.UDF_WHOAMI(), SECRET_NAME)) + +- name: {{ schema }}.udf_create_secret + signature: + - [secret_name, STRING] + - [secret, OBJECT] + func_type: SECURE + return_type: VARIANT + options: | + NOT NULL + RETURNS NULL ON NULL INPUT + IMMUTABLE + sql: | + SELECT + live.udf_api( + CONCAT_WS('/', {{ construct_api_route("/secret") }}, _utils.UDF_WHOAMI(), SECRET_NAME), + SECRET + ) + +- name: {{ schema }}.udf_delete_secret + signature: + - [secret_name, STRING] + func_type: SECURE + return_type: VARIANT + options: | + NOT NULL + RETURNS NULL ON NULL INPUT + IMMUTABLE + sql: | + SELECT + live.udf_api( + 'DELETE', + CONCAT_WS('/', {{ construct_api_route("/secret") }}, _utils.UDF_WHOAMI(), SECRET_NAME) + ) + + +{% endmacro %} \ No newline at end of file diff --git a/models/deploy/core/secrets.sql b/models/deploy/core/secrets.sql new file mode 100644 index 00000000..e59636e3 --- /dev/null +++ b/models/deploy/core/secrets.sql @@ -0,0 +1,4 @@ +-- depends_on: {{ ref('_utils') }} +-- depends_on: {{ ref('live') }} +{% set config = config_core_secrets %} +{{ ephemeral_deploy_core(config) }} From df924c910ac8c583a3026cc1659eef974500d747 Mon Sep 17 00:00:00 2001 From: Julius Remigio <14811322+juls858@users.noreply.github.com> Date: Wed, 4 Oct 2023 13:15:03 -0700 Subject: [PATCH 2/4] - clean up secrets udfs --- macros/core/secrets.yaml.sql | 48 ++++++++++++++++++++++++++---------- 1 file changed, 35 insertions(+), 13 deletions(-) diff --git a/macros/core/secrets.yaml.sql b/macros/core/secrets.yaml.sql index 1fad3753..a467a7a7 100644 --- a/macros/core/secrets.yaml.sql +++ b/macros/core/secrets.yaml.sql @@ -1,4 +1,4 @@ -{% macro config_core_secrets(schema="utils") %} +{% macro config_core_secrets(schema="secrets") %} - name: {{ schema }}.udf_register_secret @@ -11,55 +11,77 @@ NOT NULL RETURNS NULL ON NULL INPUT IMMUTABLE + COMMENT = $$ Registers a secret with the given request ID and key. $$ sql: | SELECT _utils.UDF_REGISTER_SECRET(REQUEST_ID, _utils.UDF_WHOAMI(), KEY) - name: {{ schema }}.udf_get_secret signature: - - [secret_name, STRING] + - [name, STRING] func_type: SECURE - return_type: VARIANT + return_type: OBJECT options: | NOT NULL RETURNS NULL ON NULL INPUT IMMUTABLE + COMMENT = $$ Returns the secret value for the given secret name. $$ sql: | SELECT - live.udf_api(CONCAT_WS('/', {{ construct_api_route("/secret") }}, _utils.UDF_WHOAMI(), SECRET_NAME)) + live.udf_api( + CONCAT_WS('/', {{ construct_api_route("secret") }}, _utils.UDF_WHOAMI(), NAME) + ):data::OBJECT + +- name: {{ schema }}.udf_get_secrets + signature: [] + func_type: SECURE + return_type: OBJECT + options: | + NOT NULL + RETURNS NULL ON NULL INPUT + IMMUTABLE + COMMENT = $$ Returns all secrets for the current user. $$ + sql: | + SELECT + {{ schema }}.udf_get_secret('') - name: {{ schema }}.udf_create_secret signature: - - [secret_name, STRING] + - [name, STRING] - [secret, OBJECT] func_type: SECURE - return_type: VARIANT + return_type: INTEGER options: | NOT NULL RETURNS NULL ON NULL INPUT IMMUTABLE + COMMENT = $$ Creates a new secret with the given name and value. $$ sql: | SELECT live.udf_api( - CONCAT_WS('/', {{ construct_api_route("/secret") }}, _utils.UDF_WHOAMI(), SECRET_NAME), + CONCAT_WS('/', {{ construct_api_route("secret") }}, _utils.UDF_WHOAMI(), NAME), SECRET - ) + ):status_code::INTEGER - name: {{ schema }}.udf_delete_secret signature: - - [secret_name, STRING] + - [name, STRING] func_type: SECURE - return_type: VARIANT + return_type: INTEGER options: | - NOT NULL + NULL RETURNS NULL ON NULL INPUT IMMUTABLE + COMMENT = $$ Deletes the secret with the given name. $$ sql: | SELECT live.udf_api( 'DELETE', - CONCAT_WS('/', {{ construct_api_route("/secret") }}, _utils.UDF_WHOAMI(), SECRET_NAME) - ) + CONCAT_WS('/', {{ construct_api_route("secret") }}, _utils.UDF_WHOAMI(), NAME), + {}, + {}, + '' + ):status_code::INTEGER {% endmacro %} \ No newline at end of file From 140217d592d55cfe68c05faae1d945ec7e3a85e5 Mon Sep 17 00:00:00 2001 From: Julius Remigio <14811322+juls858@users.noreply.github.com> Date: Wed, 4 Oct 2023 14:41:16 -0700 Subject: [PATCH 3/4] - set return type for udf_register_secret to VARIANT - add tests --- macros/core/_utils.yaml.sql | 3 +-- macros/core/secrets.yaml.sql | 2 +- models/deploy/core/secrets.yml | 45 ++++++++++++++++++++++++++++++++++ 3 files changed, 47 insertions(+), 3 deletions(-) create mode 100644 models/deploy/core/secrets.yml diff --git a/macros/core/_utils.yaml.sql b/macros/core/_utils.yaml.sql index 8cf1e4ce..dfb9f221 100644 --- a/macros/core/_utils.yaml.sql +++ b/macros/core/_utils.yaml.sql @@ -8,7 +8,6 @@ api_integration: '{{ var("API_INTEGRATION") }}' sql: introspect - - name: {{ schema }}.udf_user_tier signature: [] func_type: SECURE @@ -53,7 +52,7 @@ - [request_id, STRING] - [user_id, STRING] - [key, STRING] - return_type: TEXT + return_type: OBJECT func_type: SECURE EXTERNAL api_integration: '{{ var("API_INTEGRATION") }}' options: | diff --git a/macros/core/secrets.yaml.sql b/macros/core/secrets.yaml.sql index a467a7a7..3bee1120 100644 --- a/macros/core/secrets.yaml.sql +++ b/macros/core/secrets.yaml.sql @@ -6,7 +6,7 @@ - [request_id, STRING] - [key, STRING] func_type: SECURE - return_type: TEXT + return_type: VARIANT options: | NOT NULL RETURNS NULL ON NULL INPUT diff --git a/models/deploy/core/secrets.yml b/models/deploy/core/secrets.yml new file mode 100644 index 00000000..7d54d746 --- /dev/null +++ b/models/deploy/core/secrets.yml @@ -0,0 +1,45 @@ +version: 2 +models: + - name: secrets + columns: + - name: udf_register_secret + tests: + - test_udf: + name: test_secrets__udf_register_secret + args: > + 'test', 'test' + assertions: + - result = {'request_id':'test','success':false} + - name: udf_get_secret + tests: + - test_udf: + name: test_secrets__udf_get_secret + args: > + 'test' + assertions: + - > + result::OBJECT = {'key': 'value'} + - name: udf_get_secrets + tests: + - test_udf: + name: test_secrets__udf_get_secrets + args: '' + assertions: + - > + result = {'test': {'key': 'value'}} + - name: udf_create_secret + tests: + - test_udf: + name: test_secrets__udf_create_secret + args: > + 'test', {'key': 'value'} + assertions: + - result = 200 + - name: udf_delete_secret + tests: + - test_udf: + name: test_secrets__udf_delete_secret + args: > + 'test_delete' + assertions: + - result = 200 \ No newline at end of file From 459b66189310edc785e9207e4670809f8d5a7c77 Mon Sep 17 00:00:00 2001 From: Julius Remigio <14811322+juls858@users.noreply.github.com> Date: Wed, 11 Oct 2023 11:04:54 -0700 Subject: [PATCH 4/4] - add description for return types returning status_code --- dbt_project.yml | 1 + macros/core/secrets.yaml.sql | 6 +++--- models/deploy/core/secrets.sql | 6 ++++-- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/dbt_project.yml b/dbt_project.yml index 1cc61910..2cacd974 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -70,4 +70,5 @@ vars: API_INTEGRATION: AWS_LIVE_QUERY{{ "_STG" if target.name != "prod" else "" }} AWS_REGION: us-east-1 STUDIO_TEST_USER_ID: '{{ env_var("STUDIO_TEST_USER_ID", "98d15c30-9fa5-43cd-9c69-3d4c0bb269f5") }}' + ENABLE_SNOWFLAKE_SECRETS: '{{ env_var("ENABLE_SNOWFLAKE_SECRETS", "") }}' diff --git a/macros/core/secrets.yaml.sql b/macros/core/secrets.yaml.sql index 3bee1120..6c7ef035 100644 --- a/macros/core/secrets.yaml.sql +++ b/macros/core/secrets.yaml.sql @@ -6,7 +6,7 @@ - [request_id, STRING] - [key, STRING] func_type: SECURE - return_type: VARIANT + return_type: OBJECT options: | NOT NULL RETURNS NULL ON NULL INPUT @@ -50,7 +50,7 @@ - [name, STRING] - [secret, OBJECT] func_type: SECURE - return_type: INTEGER + return_type: [INTEGER, the HTTP status code - 200 indicates success] options: | NOT NULL RETURNS NULL ON NULL INPUT @@ -67,7 +67,7 @@ signature: - [name, STRING] func_type: SECURE - return_type: INTEGER + return_type: [INTEGER, the HTTP status code - 200 indicates success] options: | NULL RETURNS NULL ON NULL INPUT diff --git a/models/deploy/core/secrets.sql b/models/deploy/core/secrets.sql index e59636e3..36c33d0f 100644 --- a/models/deploy/core/secrets.sql +++ b/models/deploy/core/secrets.sql @@ -1,4 +1,6 @@ -- depends_on: {{ ref('_utils') }} -- depends_on: {{ ref('live') }} -{% set config = config_core_secrets %} -{{ ephemeral_deploy_core(config) }} +{% if var("ENABLE_SNOWFLAKE_SECRETS") %} + {% set config = config_core_secrets %} + {{ ephemeral_deploy_core(config) }} +{% endif %}