STREAM-383/STREAM-668/deploy-to-hosted-account (#54)

* - removed utils.register_secret
- add hosted profile

* - refactor project variables to use config object

* - add \n

* - uncommented code

* - update return type

* - reverted name truncation

* - change GHA workflow environments to align with dbt profiles

* - add hosted option to enviornment dropdown

* - add integration test

* Update dbt_project.yml

Add INTERNAL_DEV role to stg profile

* - fix grants
- enforce dbt target

.github/workflows/dbt.yml

@@ -56,7 +56,7 @@ jobs:
           dbt deps
       - name: Run DBT Jobs
         run: |
-          dbt ${{ inputs.command }}
+          dbt ${{ inputs.command }} --target ${{ inputs.environment }}
       - name: Store logs
         uses: actions/upload-artifact@v3
         with:

.github/workflows/dbt_docs_update.yml

@@ -25,7 +25,7 @@ jobs:
     name: docs_update
     runs-on: ubuntu-latest
     environment:
-      name: workflow_prod
+      name: prod
 
     steps:
       - uses: actions/checkout@v3

.github/workflows/dbt_integration_test.yml

@@ -0,0 +1,57 @@
+name: integration test
+run-name: ${{ github.event.inputs.branch }}
+
+on:
+  workflow_dispatch:
+    inputs:
+      environment:
+        type: choice
+        description: DBT Run Environment
+        required: true
+        options:
+          - dev
+          - prod
+          - hosted
+        default: dev
+      warehouse:
+        type: choice
+        description: Snowflake warehouse
+        required: true
+        options:
+          - DBT
+          - DBT_CLOUD
+        default: DBT
+  schedule:
+    # Runs “Daily at midnight GMT” (see https://crontab.guru)
+    - cron: '0 0 * * *'
+
+concurrency:
+  group: ${{ github.workflow }}
+
+jobs:
+  hosted:
+    uses: ./.github/workflows/dbt.yml
+    if: github.event_name == 'schedule' || github.event_name == 'push'
+    secrets: inherit
+    with:
+      warehouse: DBT_CLOUD
+      environment: hosted
+      command: test -s test___utils_udf_introspect
+
+  prod:
+    uses: ./.github/workflows/dbt.yml
+    if: github.event_name == 'schedule' || github.event_name == 'push'
+    secrets: inherit
+    with:
+      warehouse: DBT_CLOUD
+      environment: prod
+      command: test -s test___utils_udf_introspect
+
+  dispatched:
+    uses: ./.github/workflows/dbt.yml
+    if:  github.event_name == 'workflow_dispatch'
+    secrets: inherit
+    with:
+      warehouse: ${{ inputs.warehouse }}
+      environment: ${{ inputs.environment }}
+      command: test -s test___utils_udf_introspect

.github/workflows/dbt_run_adhoc.yml

@@ -11,6 +11,7 @@ on:
         options:
           - dev
           - prod
+          - hosted
         default: dev
       warehouse:
         type: choice
@@ -35,5 +36,5 @@ jobs:
     secrets: inherit
     with:
       warehouse: ${{ inputs.warehouse }}
-      environment: workflow_${{ inputs.environment }}
+      environment: ${{ inputs.environment }}
       command: ${{ inputs.dbt_command }}

.github/workflows/dbt_run_dev_refresh.yml

@@ -15,5 +15,5 @@ jobs:
     uses: ./.github/workflows/dbt.yml
     secrets: inherit
     with:
-      environment: workflow_prod
+      environment: prod
       command: run-operation run_sp_create_prod_clone

.github/workflows/dbt_test.yml → .github/workflows/dbt_udf_test.yml

@@ -34,14 +34,14 @@ jobs:
     secrets: inherit
     with:
       warehouse: DBT_CLOUD
-      environment: workflow_prod
+      environment: prod
       command: test --selector test_udfs --threads 24
 
-  dispached:
+  dispatched:
     uses: ./.github/workflows/dbt.yml
     if:  github.event_name == 'workflow_dispatch'
     secrets: inherit
     with:
       warehouse: ${{ inputs.warehouse }}
-      environment: workflow_${{ inputs.environment }}
+      environment: ${{ inputs.environment }}
       command: test --selector test_udfs --threads 24

cspell.yml

@@ -2,10 +2,14 @@ version: "0.2"
 language: en
 words:
   - aave
+  - arbitrum
+  - AVAX
   - bitquery
   - blockpour
+  - CELO
   - chainbase
   - credmark
+  - crosschain
   - dapplooker
   - doseq
   - fromyaml
@@ -15,7 +19,9 @@ words:
   - livequery
   - quicknode
   - reingest
+  - sepolia
   - solana
   - strangelove
   - subquery
   - udfs
+  - zkevm

dbt_project.yml

@@ -13,7 +13,7 @@ profile: "livequery"
 # found in the "models/" directory. You probably won't need to change these!
 model-paths: ["models"]
 analysis-paths: ["analysis"]
-# test-paths: ["tests"]
+test-paths: ["tests"]
 seed-paths: ["data"]
 macro-paths: ["macros"]
 snapshot-paths: ["snapshots"]
@@ -65,10 +65,30 @@ vars:
   UPDATE_SNOWFLAKE_TAGS: true
   STREAMLINE_INVOKE_STREAMS: False
   STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES: False
-  REST_API_PREFIX_PROD: bqco8lkjsb.execute-api.us-east-1.amazonaws.com/prod/
-  REST_API_PREFIX_DEV: u5z0tu43sc.execute-api.us-east-1.amazonaws.com/stg/
-  API_INTEGRATION: AWS_LIVE_QUERY{{ "_STG" if target.name != "prod" else "" }}
-  AWS_REGION: us-east-1
   STUDIO_TEST_USER_ID: '{{ env_var("STUDIO_TEST_USER_ID", "98d15c30-9fa5-43cd-9c69-3d4c0bb269f5") }}'
+
+  API_INTEGRATION: '{{ var("config")[target.name]["API_INTEGRATION"] }}'
+  EXTERNAL_FUNCTION_URI: '{{ var("config")[target.name]["EXTERNAL_FUNCTION_URI"] }}'
   ENABLE_SNOWFLAKE_SECRETS: '{{ env_var("ENABLE_SNOWFLAKE_SECRETS", "") }}'
+  ROLES: '{{ var("config")[target.name]["ROLES"] }}'
 
+  config:
+  # The keys correspond to dbt profiles and are case sensitive
+    dev:
+      API_INTEGRATION: AWS_LIVE_QUERY_STG
+      EXTERNAL_FUNCTION_URI: u5z0tu43sc.execute-api.us-east-1.amazonaws.com/stg/
+      ROLES:
+        - INTERNAL_DEV
+    prod:
+      API_INTEGRATION: AWS_LIVE_QUERY
+      EXTERNAL_FUNCTION_URI: bqco8lkjsb.execute-api.us-east-1.amazonaws.com/prod/
+      ROLES:
+        - VELOCITY_INTERNAL
+        - VELOCITY_ETHEREUM
+        - INTERNAL_DEV
+        - BI_ANALYTICS_READER
+    hosted:
+      API_INTEGRATION: AWS_LIVEQUERY
+      EXTERNAL_FUNCTION_URI: dlcb3tpiz8.execute-api.us-east-1.amazonaws.com/hosted/
+      ROLES:
+        - DATA_READER

macros/core/utils.yaml.sql

@@ -6,7 +6,7 @@
     - [request_id, STRING]
     - [key, STRING]
   func_type: SECURE
-  return_type: TEXT
+  return_type: OBJECT
   options: |
     NOT NULL
     RETURNS NULL ON NULL INPUT

macros/grants.sql

@@ -6,9 +6,9 @@
     This can be manually run to grant permissions to a new schema:
     `dbt run-operation apply_grants_by_schema --args '{"schema": "my_schema"}'`
  #}
-    {% if target.name == "prod" %}
+    {% if target.name in ("prod", "hosted") %}
         {%- set outer = namespace(sql="") -%}
-        {% for role in ["VELOCITY_INTERNAL", "VELOCITY_ETHEREUM", "INTERNAL_DEV", "BI_ANALYTICS_READER"] %}
+        {% for role in var("ROLES") %}
                 {% set sql -%}
                     {% if schema.startswith("_") %}
                         REVOKE USAGE ON SCHEMA {{ target.database }}.{{ schema }} FROM {{ role }};

macros/livequery/manage_udfs.sql

@@ -6,7 +6,7 @@
 {% endmacro %}
 
 {%- macro construct_api_route(route) -%}
-    'https://{{ var("REST_API_PREFIX_PROD") | lower if target.name == "prod" else var("REST_API_PREFIX_DEV") | lower }}{{ route }}'
+    'https://{{ var("EXTERNAL_FUNCTION_URI") | lower }}{{ route }}'
 {%- endmacro -%}
 
 {%- macro compile_signature(

models/deploy/core/_utils.yml

@@ -4,6 +4,7 @@ models:
       - name: udf_introspect
         tests:
           - test_udf:
+              name: test___utils_udf_introspect
               args: "'hello'"
               assertions:
                 - result = 'hello'

models/mapping_tables/_eth__decoded_logs.sql

@@ -1,11 +1,11 @@
 {{ config(
     materialized = 'view',
-    grants = {'+select': ['INTERNAL_DEV']}
+    grants = {'+select': var('ROLES')}
 ) }}
 
 SELECT *
 FROM
     {{ source(
         'ethereum_core',
         'ez_decoded_event_logs'
-    ) }}
+    ) }}

models/mapping_tables/_eth__logs.sql

@@ -1,11 +1,11 @@
 {{ config(
     materialized = 'view',
-    grants = {'+select': ['INTERNAL_DEV']}
+    grants = {'+select': var('ROLES')}
 ) }}
 
 SELECT *
 FROM
     {{ source(
         'ethereum_core',
         'fact_event_logs'
-    ) }}
+    ) }}

models/mapping_tables/_evm__abi_map.sql

@@ -1,6 +1,6 @@
 {{ config(
     materialized = 'view',
-    grants = {'+select': ['INTERNAL_DEV']}
+    grants = {'+select': var('ROLES')}
 ) }}
 
 SELECT

models/mapping_tables/_evm__contracts_map.sql

@@ -1,6 +1,6 @@
 {{ config(
     materialized = 'view',
-    grants = {'+select': ['INTERNAL_DEV']}
+    grants = {'+select': var('ROLES')}
 ) }}
 
 SELECT

models/mapping_tables/_evm__native_symbol_map.sql

@@ -1,6 +1,6 @@
 {{ config(
     materialized = 'view',
-    grants = {'+select': ['INTERNAL_DEV']}
+    grants = {'+select': var('ROLES')}
 ) }}
 
 WITH blockchain_assets AS (

profiles.yml

@@ -1,5 +1,5 @@
 livequery:
-  target: prod
+  target: dev
   outputs:
     dev:
       type: snowflake
@@ -10,10 +10,10 @@ livequery:
       region: "{{ env_var('REGION') }}"
       database: "{{ env_var('DATABASE') }}"
       warehouse: "{{ env_var('WAREHOUSE') }}"
-      schema: SILVER
-      threads: 4
+      schema: LIVE
+      threads: 24
       client_session_keep_alive: False
-      query_tag: livequery_curator
+      query_tag: dbt-livequery-dev
     prod:
       type: snowflake
       account: "{{ env_var('ACCOUNT') }}"
@@ -23,7 +23,20 @@ livequery:
       region: "{{ env_var('REGION') }}"
       database: "{{ env_var('DATABASE') }}"
       warehouse: "{{ env_var('WAREHOUSE') }}"
-      schema: SILVER
-      threads: 4
+      schema: LIVE
+      threads: 24
       client_session_keep_alive: False
-      query_tag: livequery_curator
+      query_tag: dbt-livequery-prod
+    hosted:
+      type: snowflake
+      account: "{{ env_var('ACCOUNT') }}"
+      role: "{{ env_var('ROLE') }}"
+      user: "{{ env_var('USER') }}"
+      password: "{{ env_var('PASSWORD') }}"
+      region: "{{ env_var('REGION') }}"
+      database: "{{ env_var('DATABASE') }}"
+      warehouse: "{{ env_var('WAREHOUSE') }}"
+      schema: LIVE
+      threads: 24
+      client_session_keep_alive: False
+      query_tag: dbt-livequery-hosted-prod