diff --git a/.github/workflows/dbt.yml b/.github/workflows/dbt.yml index 38f22a9d..20381905 100644 --- a/.github/workflows/dbt.yml +++ b/.github/workflows/dbt.yml @@ -56,7 +56,7 @@ jobs: dbt deps - name: Run DBT Jobs run: | - dbt ${{ inputs.command }} + dbt ${{ inputs.command }} --target ${{ inputs.environment }} - name: Store logs uses: actions/upload-artifact@v3 with: diff --git a/.github/workflows/dbt_docs_update.yml b/.github/workflows/dbt_docs_update.yml index dece5458..902fcb48 100644 --- a/.github/workflows/dbt_docs_update.yml +++ b/.github/workflows/dbt_docs_update.yml @@ -25,7 +25,7 @@ jobs: name: docs_update runs-on: ubuntu-latest environment: - name: workflow_prod + name: prod steps: - uses: actions/checkout@v3 diff --git a/.github/workflows/dbt_integration_test.yml b/.github/workflows/dbt_integration_test.yml new file mode 100644 index 00000000..a0b1360f --- /dev/null +++ b/.github/workflows/dbt_integration_test.yml @@ -0,0 +1,57 @@ +name: integration test +run-name: ${{ github.event.inputs.branch }} + +on: + workflow_dispatch: + inputs: + environment: + type: choice + description: DBT Run Environment + required: true + options: + - dev + - prod + - hosted + default: dev + warehouse: + type: choice + description: Snowflake warehouse + required: true + options: + - DBT + - DBT_CLOUD + default: DBT + schedule: + # Runs “Daily at midnight GMT” (see https://crontab.guru) + - cron: '0 0 * * *' + +concurrency: + group: ${{ github.workflow }} + +jobs: + hosted: + uses: ./.github/workflows/dbt.yml + if: github.event_name == 'schedule' || github.event_name == 'push' + secrets: inherit + with: + warehouse: DBT_CLOUD + environment: hosted + command: test -s test___utils_udf_introspect + + prod: + uses: ./.github/workflows/dbt.yml + if: github.event_name == 'schedule' || github.event_name == 'push' + secrets: inherit + with: + warehouse: DBT_CLOUD + environment: prod + command: test -s test___utils_udf_introspect + + dispatched: + uses: ./.github/workflows/dbt.yml + if: github.event_name == 'workflow_dispatch' + secrets: inherit + with: + warehouse: ${{ inputs.warehouse }} + environment: ${{ inputs.environment }} + command: test -s test___utils_udf_introspect \ No newline at end of file diff --git a/.github/workflows/dbt_run_adhoc.yml b/.github/workflows/dbt_run_adhoc.yml index 2d02088a..9970d188 100644 --- a/.github/workflows/dbt_run_adhoc.yml +++ b/.github/workflows/dbt_run_adhoc.yml @@ -11,6 +11,7 @@ on: options: - dev - prod + - hosted default: dev warehouse: type: choice @@ -35,5 +36,5 @@ jobs: secrets: inherit with: warehouse: ${{ inputs.warehouse }} - environment: workflow_${{ inputs.environment }} + environment: ${{ inputs.environment }} command: ${{ inputs.dbt_command }} \ No newline at end of file diff --git a/.github/workflows/dbt_run_dev_refresh.yml b/.github/workflows/dbt_run_dev_refresh.yml index 23f791d8..7b483599 100644 --- a/.github/workflows/dbt_run_dev_refresh.yml +++ b/.github/workflows/dbt_run_dev_refresh.yml @@ -15,5 +15,5 @@ jobs: uses: ./.github/workflows/dbt.yml secrets: inherit with: - environment: workflow_prod + environment: prod command: run-operation run_sp_create_prod_clone \ No newline at end of file diff --git a/.github/workflows/dbt_test.yml b/.github/workflows/dbt_udf_test.yml similarity index 91% rename from .github/workflows/dbt_test.yml rename to .github/workflows/dbt_udf_test.yml index d1daa575..57e08a8d 100644 --- a/.github/workflows/dbt_test.yml +++ b/.github/workflows/dbt_udf_test.yml @@ -34,14 +34,14 @@ jobs: secrets: inherit with: warehouse: DBT_CLOUD - environment: workflow_prod + environment: prod command: test --selector test_udfs --threads 24 - dispached: + dispatched: uses: ./.github/workflows/dbt.yml if: github.event_name == 'workflow_dispatch' secrets: inherit with: warehouse: ${{ inputs.warehouse }} - environment: workflow_${{ inputs.environment }} + environment: ${{ inputs.environment }} command: test --selector test_udfs --threads 24 \ No newline at end of file diff --git a/cspell.yml b/cspell.yml index 5c7e3578..87b301fa 100644 --- a/cspell.yml +++ b/cspell.yml @@ -2,10 +2,14 @@ version: "0.2" language: en words: - aave + - arbitrum + - AVAX - bitquery - blockpour + - CELO - chainbase - credmark + - crosschain - dapplooker - doseq - fromyaml @@ -15,7 +19,9 @@ words: - livequery - quicknode - reingest + - sepolia - solana - strangelove - subquery - udfs + - zkevm diff --git a/dbt_project.yml b/dbt_project.yml index 2cacd974..e260d367 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -13,7 +13,7 @@ profile: "livequery" # found in the "models/" directory. You probably won't need to change these! model-paths: ["models"] analysis-paths: ["analysis"] -# test-paths: ["tests"] +test-paths: ["tests"] seed-paths: ["data"] macro-paths: ["macros"] snapshot-paths: ["snapshots"] @@ -65,10 +65,30 @@ vars: UPDATE_SNOWFLAKE_TAGS: true STREAMLINE_INVOKE_STREAMS: False STREAMLINE_USE_DEV_FOR_EXTERNAL_TABLES: False - REST_API_PREFIX_PROD: bqco8lkjsb.execute-api.us-east-1.amazonaws.com/prod/ - REST_API_PREFIX_DEV: u5z0tu43sc.execute-api.us-east-1.amazonaws.com/stg/ - API_INTEGRATION: AWS_LIVE_QUERY{{ "_STG" if target.name != "prod" else "" }} - AWS_REGION: us-east-1 STUDIO_TEST_USER_ID: '{{ env_var("STUDIO_TEST_USER_ID", "98d15c30-9fa5-43cd-9c69-3d4c0bb269f5") }}' + + API_INTEGRATION: '{{ var("config")[target.name]["API_INTEGRATION"] }}' + EXTERNAL_FUNCTION_URI: '{{ var("config")[target.name]["EXTERNAL_FUNCTION_URI"] }}' ENABLE_SNOWFLAKE_SECRETS: '{{ env_var("ENABLE_SNOWFLAKE_SECRETS", "") }}' + ROLES: '{{ var("config")[target.name]["ROLES"] }}' + config: + # The keys correspond to dbt profiles and are case sensitive + dev: + API_INTEGRATION: AWS_LIVE_QUERY_STG + EXTERNAL_FUNCTION_URI: u5z0tu43sc.execute-api.us-east-1.amazonaws.com/stg/ + ROLES: + - INTERNAL_DEV + prod: + API_INTEGRATION: AWS_LIVE_QUERY + EXTERNAL_FUNCTION_URI: bqco8lkjsb.execute-api.us-east-1.amazonaws.com/prod/ + ROLES: + - VELOCITY_INTERNAL + - VELOCITY_ETHEREUM + - INTERNAL_DEV + - BI_ANALYTICS_READER + hosted: + API_INTEGRATION: AWS_LIVEQUERY + EXTERNAL_FUNCTION_URI: dlcb3tpiz8.execute-api.us-east-1.amazonaws.com/hosted/ + ROLES: + - DATA_READER diff --git a/macros/core/utils.yaml.sql b/macros/core/utils.yaml.sql index 2fca2105..41302226 100644 --- a/macros/core/utils.yaml.sql +++ b/macros/core/utils.yaml.sql @@ -6,7 +6,7 @@ - [request_id, STRING] - [key, STRING] func_type: SECURE - return_type: TEXT + return_type: OBJECT options: | NOT NULL RETURNS NULL ON NULL INPUT diff --git a/macros/grants.sql b/macros/grants.sql index 5324add9..06b13f56 100644 --- a/macros/grants.sql +++ b/macros/grants.sql @@ -6,9 +6,9 @@ This can be manually run to grant permissions to a new schema: `dbt run-operation apply_grants_by_schema --args '{"schema": "my_schema"}'` #} - {% if target.name == "prod" %} + {% if target.name in ("prod", "hosted") %} {%- set outer = namespace(sql="") -%} - {% for role in ["VELOCITY_INTERNAL", "VELOCITY_ETHEREUM", "INTERNAL_DEV", "BI_ANALYTICS_READER"] %} + {% for role in var("ROLES") %} {% set sql -%} {% if schema.startswith("_") %} REVOKE USAGE ON SCHEMA {{ target.database }}.{{ schema }} FROM {{ role }}; diff --git a/macros/livequery/manage_udfs.sql b/macros/livequery/manage_udfs.sql index e5308b1a..b5b656f4 100644 --- a/macros/livequery/manage_udfs.sql +++ b/macros/livequery/manage_udfs.sql @@ -6,7 +6,7 @@ {% endmacro %} {%- macro construct_api_route(route) -%} - 'https://{{ var("REST_API_PREFIX_PROD") | lower if target.name == "prod" else var("REST_API_PREFIX_DEV") | lower }}{{ route }}' + 'https://{{ var("EXTERNAL_FUNCTION_URI") | lower }}{{ route }}' {%- endmacro -%} {%- macro compile_signature( diff --git a/models/deploy/core/_utils.yml b/models/deploy/core/_utils.yml index 15875581..b69fe2b8 100644 --- a/models/deploy/core/_utils.yml +++ b/models/deploy/core/_utils.yml @@ -4,6 +4,7 @@ models: - name: udf_introspect tests: - test_udf: + name: test___utils_udf_introspect args: "'hello'" assertions: - result = 'hello' \ No newline at end of file diff --git a/models/mapping_tables/_eth__decoded_logs.sql b/models/mapping_tables/_eth__decoded_logs.sql index 56acf9c0..31eb4caf 100644 --- a/models/mapping_tables/_eth__decoded_logs.sql +++ b/models/mapping_tables/_eth__decoded_logs.sql @@ -1,6 +1,6 @@ {{ config( materialized = 'view', - grants = {'+select': ['INTERNAL_DEV']} + grants = {'+select': var('ROLES')} ) }} SELECT * @@ -8,4 +8,4 @@ FROM {{ source( 'ethereum_core', 'ez_decoded_event_logs' - ) }} \ No newline at end of file + ) }} diff --git a/models/mapping_tables/_eth__logs.sql b/models/mapping_tables/_eth__logs.sql index 1d566c59..71992f25 100644 --- a/models/mapping_tables/_eth__logs.sql +++ b/models/mapping_tables/_eth__logs.sql @@ -1,6 +1,6 @@ {{ config( materialized = 'view', - grants = {'+select': ['INTERNAL_DEV']} + grants = {'+select': var('ROLES')} ) }} SELECT * @@ -8,4 +8,4 @@ FROM {{ source( 'ethereum_core', 'fact_event_logs' - ) }} \ No newline at end of file + ) }} diff --git a/models/mapping_tables/_evm__abi_map.sql b/models/mapping_tables/_evm__abi_map.sql index 58f885d8..4f01b092 100644 --- a/models/mapping_tables/_evm__abi_map.sql +++ b/models/mapping_tables/_evm__abi_map.sql @@ -1,6 +1,6 @@ {{ config( materialized = 'view', - grants = {'+select': ['INTERNAL_DEV']} + grants = {'+select': var('ROLES')} ) }} SELECT diff --git a/models/mapping_tables/_evm__contracts_map.sql b/models/mapping_tables/_evm__contracts_map.sql index 4162f38a..5fc6d224 100644 --- a/models/mapping_tables/_evm__contracts_map.sql +++ b/models/mapping_tables/_evm__contracts_map.sql @@ -1,6 +1,6 @@ {{ config( materialized = 'view', - grants = {'+select': ['INTERNAL_DEV']} + grants = {'+select': var('ROLES')} ) }} SELECT diff --git a/models/mapping_tables/_evm__native_symbol_map.sql b/models/mapping_tables/_evm__native_symbol_map.sql index 0e4159e9..2fadec44 100644 --- a/models/mapping_tables/_evm__native_symbol_map.sql +++ b/models/mapping_tables/_evm__native_symbol_map.sql @@ -1,6 +1,6 @@ {{ config( materialized = 'view', - grants = {'+select': ['INTERNAL_DEV']} + grants = {'+select': var('ROLES')} ) }} WITH blockchain_assets AS ( diff --git a/profiles.yml b/profiles.yml index 7781bd23..5d3c4d4b 100644 --- a/profiles.yml +++ b/profiles.yml @@ -1,5 +1,5 @@ livequery: - target: prod + target: dev outputs: dev: type: snowflake @@ -10,10 +10,10 @@ livequery: region: "{{ env_var('REGION') }}" database: "{{ env_var('DATABASE') }}" warehouse: "{{ env_var('WAREHOUSE') }}" - schema: SILVER - threads: 4 + schema: LIVE + threads: 24 client_session_keep_alive: False - query_tag: livequery_curator + query_tag: dbt-livequery-dev prod: type: snowflake account: "{{ env_var('ACCOUNT') }}" @@ -23,7 +23,20 @@ livequery: region: "{{ env_var('REGION') }}" database: "{{ env_var('DATABASE') }}" warehouse: "{{ env_var('WAREHOUSE') }}" - schema: SILVER - threads: 4 + schema: LIVE + threads: 24 client_session_keep_alive: False - query_tag: livequery_curator \ No newline at end of file + query_tag: dbt-livequery-prod + hosted: + type: snowflake + account: "{{ env_var('ACCOUNT') }}" + role: "{{ env_var('ROLE') }}" + user: "{{ env_var('USER') }}" + password: "{{ env_var('PASSWORD') }}" + region: "{{ env_var('REGION') }}" + database: "{{ env_var('DATABASE') }}" + warehouse: "{{ env_var('WAREHOUSE') }}" + schema: LIVE + threads: 24 + client_session_keep_alive: False + query_tag: dbt-livequery-hosted-prod \ No newline at end of file