diff --git a/.github/workflows/el_ssa_ip_addresses.yml b/.github/workflows/el_ssa_ip_addresses.yml new file mode 100644 index 00000000..36596529 --- /dev/null +++ b/.github/workflows/el_ssa_ip_addresses.yml @@ -0,0 +1,31 @@ +name: Meltano - Spreadsheets Anywhere IP Addresses + +on: + schedule: + - cron: '0 10 * * *' + + workflow_dispatch: + +jobs: + el_ssa_ip_addresses: + runs-on: ubuntu-latest + defaults: + run: + working-directory: data/ + environment: prod + env: + MELTANO_ENVIRONMENT: prod + MELTANO_SEND_ANONYMOUS_USAGE_STATS: 'false' + SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }} + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: '3.11' + cache: 'pip' + - run: pip install -r requirements.txt + - run: echo "${{secrets.MELTANO_ENV_FILE }}" > .env + # Install Plugins + - run: meltano install --schedule=spreadsheets_anywhere_el + # Run Test + - run: meltano run spreadsheets_anywhere_el \ No newline at end of file diff --git a/.github/workflows/hub_metrics_publish.yml b/.github/workflows/hub_metrics_publish.yml new file mode 100644 index 00000000..b5cd5080 --- /dev/null +++ b/.github/workflows/hub_metrics_publish.yml @@ -0,0 +1,35 @@ +name: Meltano - Hub Metrics Publish + +on: + schedule: + - cron: '0 12 * * *' + + workflow_dispatch: + +jobs: + hub_metrics_publish: + runs-on: ubuntu-latest + defaults: + run: + working-directory: data/ + environment: prod + env: + MELTANO_ENVIRONMENT: prod + MELTANO_SEND_ANONYMOUS_USAGE_STATS: 'false' + SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }} + TAP_SNOWFLAKE_METRICS_TABLES: '["b${{ github.SHA }}_MELTANO_HUB.FACT_VARIANT_HUB_METRICS"]' + TAP_SNOWFLAKE_AUDIT_TABLES: '["b${{ github.SHA }}_MELTANO_HUB.HUB_METRICS_AUDIT"]' + AWS_ACCESS_KEY_ID: ${{ secrets.AWS_ACCESS_KEY_ID }} + AWS_SECRET_ACCESS_KEY: ${{ secrets.AWS_SECRET_ACCESS_KEY }} + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: '3.11' + cache: 'pip' + - run: pip install -r requirements.txt + - run: echo "${{secrets.MELTANO_ENV_FILE }}" > .env + # Install Plugins + - run: meltano install --schedule=hub_metrics_publish + # Run Test + - run: meltano run hub_metrics_publish \ No newline at end of file diff --git a/.github/workflows/slack_notifications.yml b/.github/workflows/slack_notifications.yml new file mode 100644 index 00000000..3b88730d --- /dev/null +++ b/.github/workflows/slack_notifications.yml @@ -0,0 +1,35 @@ +name: Meltano - Slack Publish + +on: + schedule: + - cron: '0 12 * * *' + + workflow_dispatch: + +jobs: + slack_notifications: + runs-on: ubuntu-latest + defaults: + run: + working-directory: data/ + environment: prod + env: + MELTANO_ENVIRONMENT: prod + MELTANO_SEND_ANONYMOUS_USAGE_STATS: 'false' + SNOWFLAKE_PASSWORD: ${{ secrets.SNOWFLAKE_PASSWORD }} + TAP_SNOWFLAKE_SINGER_ACTIVITY: '["MELTANO_HUB.SINGER_ACTIVITY_NOTIFICATIONS"]' + TAP_SNOWFLAKE_MELTANO_ACTIVITY: '["MELTANO_HUB.MELTANO_ACTIVITY_NOTIFICATIONS"]' + TARGET_APPRISE_SINGER_ACTIVITY_URIS: ${{ secrets.APPRISE_SINGER_ACTIVITY_WEBHOOK }} + TARGET_APPRISE_MELTANO_ACTIVITY_URIS: ${{ secrets.APPRISE_MELTANO_ACTIVITY_WEBHOOK }} + steps: + - uses: actions/checkout@v3 + - uses: actions/setup-python@v4 + with: + python-version: '3.11' + cache: 'pip' + - run: pip install -r requirements.txt + - run: echo "${{secrets.MELTANO_ENV_FILE }}" > .env + # Install Plugins + - run: meltano install --schedule=slack_notifications + # Run Test + - run: meltano run slack_notifications \ No newline at end of file diff --git a/data/orchestrate/orchestrators.meltano.yml b/data/orchestrate/orchestrators.meltano.yml index 2f0c2ac1..88701aef 100644 --- a/data/orchestrate/orchestrators.meltano.yml +++ b/data/orchestrate/orchestrators.meltano.yml @@ -97,7 +97,7 @@ jobs: - name: slack_notifications tasks: - - dbt-snowflake:run_slack_notifications + # - dbt-snowflake:run_slack_notifications - tap-snowflake-singer-activity target-apprise-singer-activity - tap-snowflake-meltano-activity target-apprise-meltano-activity @@ -117,8 +117,8 @@ jobs: - name: spreadsheets_anywhere_el tasks: - tap-spreadsheets-anywhere coalesce-gcp-ips target-snowflake - - dbt-snowflake:test_source_spreadsheets_anywhere - - dbt-snowflake:run_snapshot_spreadsheets_anywhere + # - dbt-snowflake:test_source_spreadsheets_anywhere + # - dbt-snowflake:run_snapshot_spreadsheets_anywhere - name: snowplow_el tasks: @@ -160,8 +160,8 @@ jobs: - name: hub_metrics_publish tasks: - - dbt-snowflake:run_hub_metrics - - dbt-snowflake:test_hub_metrics + # - dbt-snowflake:run_hub_metrics + # - dbt-snowflake:test_hub_metrics # - great_expectations:test_dbt_hub_metrics - tap-snowflake-metrics target-yaml-metrics awscli:s3_copy_metrics - tap-snowflake-audit target-yaml-audit awscli:s3_copy_audit diff --git a/data/transform/dbt_project.yml b/data/transform/dbt_project.yml index aafe491f..a87c0a04 100644 --- a/data/transform/dbt_project.yml +++ b/data/transform/dbt_project.yml @@ -23,14 +23,14 @@ clean-targets: - dbt_packages - logs -on-run-end: "{% if target.name in ('prod', 'cicd') %}{{ dbt_artifacts.upload_results(results) }}{% endif %}" +on-run-end: "{% if target.name in ('prod', 'cicd') %} {{ dbt_artifacts.upload_results(results) }} {% endif %}" seeds: quote_columns: false +schema: SEED snapshots: - target_database: "{{ env_var('DBT_SNOWFLAKE_DATABASE_PREP') }}" + target_database: "{{ env_var('DBT_SNOWFLAKE_DATABASE_PREP', 'PREP') }}" target_schema: snapshots transient: false @@ -41,7 +41,7 @@ models: # Staging # ------------------ staging: - +database: "{{ env_var('DBT_SNOWFLAKE_DATABASE_PREP') }}" + +database: "{{ env_var('DBT_SNOWFLAKE_DATABASE_PREP', 'PREP') }}" cloudwatch: +schema: cloudwatch dynamodb: @@ -77,7 +77,7 @@ models: # Common # ------------------ common: - +database: "{{ env_var('DBT_SNOWFLAKE_DATABASE_PREP') }}" + +database: "{{ env_var('DBT_SNOWFLAKE_DATABASE_PREP', 'PREP') }}" +schema: workspace +materialized: view # ------------------ @@ -87,21 +87,21 @@ models: +materialized: table telemetry: base: - +database: "{{ env_var('DBT_SNOWFLAKE_DATABASE_PREP') }}" + +database: "{{ env_var('DBT_SNOWFLAKE_DATABASE_PREP', 'PREP') }}" +schema: workspace +materialized: view +schema: telemetry community: base: - +database: "{{ env_var('DBT_SNOWFLAKE_DATABASE_PREP') }}" + +database: "{{ env_var('DBT_SNOWFLAKE_DATABASE_PREP', 'PREP') }}" +schema: workspace +materialized: view +schema: community dbt_artifacts: +schema: dbt_artifacts staging: - +database: "{{ env_var('DBT_SNOWFLAKE_DATABASE_PREP') }}" + +database: "{{ env_var('DBT_SNOWFLAKE_DATABASE_PREP', 'PREP') }}" +schema: dbt_artifacts sources: - +database: "{{ env_var('DBT_SNOWFLAKE_DATABASE_PREP') }}" + +database: "{{ env_var('DBT_SNOWFLAKE_DATABASE_PREP', 'PREP') }}" +schema: "{{ env_var('DBT_SNOWFLAKE_SOURCE_SCHEMA_PREFIX', '') }}SOURCE_DBT_ARTIFACTS" diff --git a/data/transform/models/staging/cloudwatch/sources.yml b/data/transform/models/staging/cloudwatch/sources.yml index 6277dd24..422d309e 100644 --- a/data/transform/models/staging/cloudwatch/sources.yml +++ b/data/transform/models/staging/cloudwatch/sources.yml @@ -2,7 +2,7 @@ config-version: 2 version: 2 sources: - name: tap_cloudwatch - database: '{{ env_var("DBT_SNOWFLAKE_DATABASE_RAW") }}' + database: '{{ env_var("DBT_SNOWFLAKE_DATABASE_RAW", "RAW") }}' schema: '{{ env_var("DBT_SNOWFLAKE_SOURCE_SCHEMA_PREFIX", "") }}TAP_CLOUDWATCH' tables: - name: log diff --git a/data/transform/models/staging/dynamodb/sources.yml b/data/transform/models/staging/dynamodb/sources.yml index c86926ea..86f1e409 100644 --- a/data/transform/models/staging/dynamodb/sources.yml +++ b/data/transform/models/staging/dynamodb/sources.yml @@ -2,7 +2,7 @@ config-version: 2 version: 2 sources: - name: tap_dynamodb - database: '{{ env_var("DBT_SNOWFLAKE_DATABASE_RAW") }}' + database: '{{ env_var("DBT_SNOWFLAKE_DATABASE_RAW", "RAW") }}' schema: '{{ env_var("DBT_SNOWFLAKE_SOURCE_SCHEMA_PREFIX", "") }}TAP_DYNAMODB' tables: - name: workload_metadata_table diff --git a/data/transform/models/staging/github_meltano/sources.yml b/data/transform/models/staging/github_meltano/sources.yml index aed8b21e..cb4ce944 100644 --- a/data/transform/models/staging/github_meltano/sources.yml +++ b/data/transform/models/staging/github_meltano/sources.yml @@ -2,7 +2,7 @@ config-version: 2 version: 2 sources: - name: tap_github_meltano - database: '{{ env_var("DBT_SNOWFLAKE_DATABASE_RAW") }}' + database: '{{ env_var("DBT_SNOWFLAKE_DATABASE_RAW", "RAW") }}' schema: '{{ env_var("DBT_SNOWFLAKE_SOURCE_SCHEMA_PREFIX", "") }}TAP_GITHUB_MELTANO' tables: - name: issues diff --git a/data/transform/models/staging/github_search/sources.yml b/data/transform/models/staging/github_search/sources.yml index a6fcc2ef..b7a97892 100644 --- a/data/transform/models/staging/github_search/sources.yml +++ b/data/transform/models/staging/github_search/sources.yml @@ -2,7 +2,7 @@ config-version: 2 version: 2 sources: - name: tap_github_search - database: '{{ env_var("DBT_SNOWFLAKE_DATABASE_RAW") }}' + database: '{{ env_var("DBT_SNOWFLAKE_DATABASE_RAW", "RAW") }}' schema: '{{ env_var("DBT_SNOWFLAKE_SOURCE_SCHEMA_PREFIX", "") }}TAP_GITHUB_SEARCH' tables: - name: issues diff --git a/data/transform/models/staging/gitlab/sources.yml b/data/transform/models/staging/gitlab/sources.yml index 4482eeef..e20dc828 100644 --- a/data/transform/models/staging/gitlab/sources.yml +++ b/data/transform/models/staging/gitlab/sources.yml @@ -2,14 +2,14 @@ config-version: 2 version: 2 sources: - name: tap_gitlab - database: '{{ env_var("DBT_SNOWFLAKE_DATABASE_RAW") }}' + database: '{{ env_var("DBT_SNOWFLAKE_DATABASE_RAW", "RAW") }}' schema: '{{ env_var("DBT_SNOWFLAKE_SOURCE_SCHEMA_PREFIX", "") }}TAP_GITLAB' tables: - name: issues - name: merge_requests - name: projects - name: tap_gitlab_hotglue - database: '{{ env_var("DBT_SNOWFLAKE_DATABASE_RAW") }}' + database: '{{ env_var("DBT_SNOWFLAKE_DATABASE_RAW", "RAW") }}' schema: '{{ env_var("DBT_SNOWFLAKE_SOURCE_SCHEMA_PREFIX", "") }}TAP_GITLAB_HOTGLUE' tables: - name: issues diff --git a/data/transform/models/staging/gitlab/stg_gitlab__issues.sql b/data/transform/models/staging/gitlab/stg_gitlab__issues.sql index 596ccf0a..4e1bf9c7 100644 --- a/data/transform/models/staging/gitlab/stg_gitlab__issues.sql +++ b/data/transform/models/staging/gitlab/stg_gitlab__issues.sql @@ -1,5 +1,3 @@ -{{ config(materialized='table') }} - WITH source_meltano AS ( SELECT diff --git a/data/transform/models/staging/gitlab/stg_gitlab__merge_requests.sql b/data/transform/models/staging/gitlab/stg_gitlab__merge_requests.sql index 44bbbb17..026d09a1 100644 --- a/data/transform/models/staging/gitlab/stg_gitlab__merge_requests.sql +++ b/data/transform/models/staging/gitlab/stg_gitlab__merge_requests.sql @@ -1,5 +1,3 @@ -{{ config(materialized='table') }} - WITH source_meltano AS ( SELECT diff --git a/data/transform/models/staging/gitlab/stg_gitlab__projects.sql b/data/transform/models/staging/gitlab/stg_gitlab__projects.sql index 32a47120..e2a25b6c 100644 --- a/data/transform/models/staging/gitlab/stg_gitlab__projects.sql +++ b/data/transform/models/staging/gitlab/stg_gitlab__projects.sql @@ -1,5 +1,3 @@ -{{ config(materialized='table') }} - WITH source AS ( SELECT diff --git a/data/transform/models/staging/google_analytics/sources.yml b/data/transform/models/staging/google_analytics/sources.yml index b6908915..0299057d 100644 --- a/data/transform/models/staging/google_analytics/sources.yml +++ b/data/transform/models/staging/google_analytics/sources.yml @@ -2,7 +2,7 @@ config-version: 2 version: 2 sources: - name: tap_google_analytics - database: '{{ env_var("DBT_SNOWFLAKE_DATABASE_RAW") }}' + database: '{{ env_var("DBT_SNOWFLAKE_DATABASE_RAW", "RAW") }}' schema: '{{ env_var("DBT_SNOWFLAKE_SOURCE_SCHEMA_PREFIX", "") }}TAP_GOOGLE_ANALYTICS' tables: - name: events \ No newline at end of file diff --git a/data/transform/models/staging/meltanohub/sources.yml b/data/transform/models/staging/meltanohub/sources.yml index 701e4437..aa035094 100644 --- a/data/transform/models/staging/meltanohub/sources.yml +++ b/data/transform/models/staging/meltanohub/sources.yml @@ -2,7 +2,7 @@ config-version: 2 version: 2 sources: - name: tap_meltanohub - database: '{{ env_var("DBT_SNOWFLAKE_DATABASE_RAW") }}' + database: '{{ env_var("DBT_SNOWFLAKE_DATABASE_RAW", "RAW") }}' schema: '{{ env_var("DBT_SNOWFLAKE_SOURCE_SCHEMA_PREFIX", "") }}TAP_MELTANOHUB' tables: - name: plugins diff --git a/data/transform/models/staging/slack/sources.yml b/data/transform/models/staging/slack/sources.yml index 9eeb8454..e1a2ee47 100644 --- a/data/transform/models/staging/slack/sources.yml +++ b/data/transform/models/staging/slack/sources.yml @@ -2,14 +2,14 @@ config-version: 2 version: 2 sources: - name: tap_slack - database: '{{ env_var("DBT_SNOWFLAKE_DATABASE_RAW") }}' + database: '{{ env_var("DBT_SNOWFLAKE_DATABASE_RAW", "RAW") }}' schema: '{{ env_var("DBT_SNOWFLAKE_SOURCE_SCHEMA_PREFIX", "") }}TAP_SLACK' tables: - name: users - name: messages - name: channels - name: tap_slack_public - database: '{{ env_var("DBT_SNOWFLAKE_DATABASE_RAW") }}' + database: '{{ env_var("DBT_SNOWFLAKE_DATABASE_RAW", "RAW") }}' schema: '{{ env_var("DBT_SNOWFLAKE_SOURCE_SCHEMA_PREFIX", "") }}TAP_SLACK_PUBLIC' tables: - name: messages diff --git a/data/transform/models/staging/snowplow/sources.yml b/data/transform/models/staging/snowplow/sources.yml index f0fdd427..49cf86ce 100644 --- a/data/transform/models/staging/snowplow/sources.yml +++ b/data/transform/models/staging/snowplow/sources.yml @@ -2,7 +2,7 @@ config-version: 2 version: 2 sources: - name: snowplow - database: '{{ env_var("DBT_SNOWFLAKE_DATABASE_RAW") }}' + database: '{{ env_var("DBT_SNOWFLAKE_DATABASE_RAW", "RAW") }}' schema: '{{ env_var("DBT_SNOWFLAKE_SOURCE_SCHEMA_PREFIX", "") }}SNOWPLOW' tables: - name: events diff --git a/data/transform/profiles/snowflake/profiles.yml b/data/transform/profiles/squared_snowflake/profiles.yml similarity index 100% rename from data/transform/profiles/snowflake/profiles.yml rename to data/transform/profiles/squared_snowflake/profiles.yml diff --git a/data/transform/snapshots/spreadsheets_anywhere/sources.yml b/data/transform/snapshots/spreadsheets_anywhere/sources.yml index 410c1f4d..eb256c95 100644 --- a/data/transform/snapshots/spreadsheets_anywhere/sources.yml +++ b/data/transform/snapshots/spreadsheets_anywhere/sources.yml @@ -2,7 +2,7 @@ config-version: 2 version: 2 sources: - name: tap_spreadsheets_anywhere - database: '{{ env_var("DBT_SNOWFLAKE_DATABASE_RAW") }}' + database: '{{ env_var("DBT_SNOWFLAKE_DATABASE_RAW", "RAW") }}' schema: '{{ env_var("DBT_SNOWFLAKE_SOURCE_SCHEMA_PREFIX", "") }}TAP_SPREADSHEETS_ANYWHERE' tables: - name: aws_ips diff --git a/data/transform/transformers.meltano.yml b/data/transform/transformers.meltano.yml index 504851d7..a7197bd9 100644 --- a/data/transform/transformers.meltano.yml +++ b/data/transform/transformers.meltano.yml @@ -45,3 +45,4 @@ plugins: account: epa06486 database_raw: RAW database_prep: PREP + profiles_dir: $MELTANO_PROJECT_ROOT/transform/profiles/squared_snowflake