dbt_run_decode_logs_orchestrator #17682
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: dbt_run_decode_logs_orchestrator | |
| run-name: dbt_run_decode_logs_orchestrator | |
| on: | |
| workflow_dispatch: | |
| branches: | |
| - "main" | |
| env: | |
| DBT_PROFILES_DIR: "${{ vars.DBT_PROFILES_DIR }}" | |
| ACCOUNT: "${{ vars.ACCOUNT }}" | |
| ROLE: "${{ vars.ROLE }}" | |
| USER: "${{ vars.USER }}" | |
| PASSWORD: "${{ secrets.PASSWORD }}" | |
| REGION: "${{ vars.REGION }}" | |
| DATABASE: "${{ vars.DATABASE }}" | |
| WAREHOUSE: "${{ vars.WAREHOUSE }}" | |
| SCHEMA: "${{ vars.SCHEMA }}" | |
| concurrency: | |
| group: ${{ github.workflow }} | |
| jobs: | |
| run_dbt_jobs: | |
| runs-on: ubuntu-latest | |
| environment: | |
| name: workflow_prod | |
| steps: | |
| - uses: actions/checkout@v3 | |
| - uses: actions/setup-python@v4 | |
| with: | |
| python-version: "${{ vars.PYTHON_VERSION }}" | |
| cache: "pip" | |
| - name: Capture Formatted Minute | |
| id: capture_minute | |
| run: | | |
| formatted_minute=$(date '+%M') | |
| echo "Formatted Minute: $formatted_minute" | |
| echo "formatted_minute=$formatted_minute" >> "$GITHUB_OUTPUT" | |
| - name: Log step | |
| run: | | |
| echo "Using Formatted Minute: ${{ steps.capture_minute.outputs.formatted_minute }}" | |
| - name: install dependencies | |
| run: | | |
| pip install -r requirements.txt | |
| dbt deps | |
| - name: Run DBT Jobs | |
| run: | | |
| dbt run -s streamline__complete_decoded_logs streamline__complete_decoded_logs_2 | |
| - name: Run Real Time Core on minutes 0 and 30 every hour | |
| run: | | |
| dbt run-operation dispatch_github_workflow --args "{'workflow_name': 'dbt_run_decode_logs'}" | |
| if: ${{ github.event_name == 'workflow_dispatch' && contains(fromJSON('["00", "30"]'), steps.capture_minute.outputs.formatted_minute) }} | |
| - name: Run Backfill on minutes 45 every hour | |
| run: | | |
| dbt run-operation dispatch_github_workflow --args "{'workflow_name': 'dbt_run_decode_logs_backfill'}" | |
| if: ${{ github.event_name == 'workflow_dispatch' && contains(fromJSON('["45"]'), steps.capture_minute.outputs.formatted_minute) }} |