gdrive fsspec #2312
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: test dbt runner | |
| on: | |
| pull_request: | |
| branches: | |
| - master | |
| - devel | |
| workflow_dispatch: | |
| env: | |
| # all credentials must be present to be passed to dbt runner | |
| DESTINATION__POSTGRES__CREDENTIALS: postgresql://[email protected]:5432/dlt_data | |
| DESTINATION__REDSHIFT__CREDENTIALS: postgresql://[email protected]:5439/dlt_ci | |
| DESTINATION__SNOWFLAKE__CREDENTIALS: snowflake://loader@kgiotue-wn98412/dlt_data?warehouse=COMPUTE_WH&role=DLT_LOADER_ROLE | |
| DESTINATION__CREDENTIALS__PASSWORD: ${{ secrets.PG_PASSWORD }} | |
| DESTINATION__CREDENTIALS__PROJECT_ID: chat-analytics-rasa-ci | |
| DESTINATION__CREDENTIALS__CLIENT_EMAIL: chat-analytics-loader@chat-analytics-rasa-ci.iam.gserviceaccount.com | |
| DESTINATION__BIGQUERY__CREDENTIALS__PRIVATE_KEY: ${{ secrets.BQ_CRED_PRIVATE_KEY }} | |
| DESTINATION__CREDENTIALS__TOKEN_URI: https://oauth2.googleapis.com/token | |
| RUNTIME__LOG_LEVEL: ERROR | |
| jobs: | |
| get_docs_changes: | |
| uses: ./.github/workflows/get_docs_changes.yml | |
| if: ${{ !github.event.pull_request.head.repo.fork }} | |
| run_dbt: | |
| name: Tests dbt runner | |
| needs: get_docs_changes | |
| if: needs.get_docs_changes.outputs.changes_outside_docs == 'true' | |
| strategy: | |
| fail-fast: false | |
| matrix: | |
| os: ["ubuntu-latest"] | |
| # os: ["ubuntu-latest", "macos-latest", "windows-latest"] | |
| defaults: | |
| run: | |
| shell: bash | |
| runs-on: ${{ matrix.os }} | |
| steps: | |
| - name: Check out | |
| uses: actions/checkout@master | |
| - name: Setup Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: "3.10.x" | |
| - name: Install Poetry without dbt | |
| uses: snok/[email protected] | |
| with: | |
| virtualenvs-create: true | |
| virtualenvs-in-project: true | |
| installer-parallel: true | |
| - name: Load cached venv | |
| id: cached-poetry-dependencies | |
| uses: actions/cache@v3 | |
| with: | |
| # path: ${{ steps.pip-cache.outputs.dir }} | |
| path: .venv | |
| key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}-dbt-runner | |
| - name: Install dependencies | |
| # install dlt with postgres support | |
| run: poetry install --no-interaction -E postgres -E dbt | |
| - run: | | |
| poetry run pytest tests/helpers/dbt_tests -k '(not venv)' | |
| if: runner.os != 'Windows' | |
| name: Run dbt runner - Linux/MAC | |
| - run: | | |
| poetry run pytest tests/helpers/dbt_tests -k "(not venv)" | |
| if: runner.os == 'Windows' | |
| name: Run dbt runner - Windows | |
| shell: cmd | |
| - name: Remove dbt-core | |
| # install dlt with postgres support | |
| run: poetry run pip uninstall dbt-core -y | |
| - run: | | |
| poetry run pytest tests/helpers/dbt_tests --ignore=tests/helpers/dbt_tests/local -k '(not local)' | |
| if: runner.os != 'Windows' | |
| name: Run dbt runner with venv - Linux/MAC | |
| - run: | | |
| poetry run pytest tests/helpers/dbt_tests --ignore=tests/helpers/dbt_tests/local -m "not forked" -k "(not local)" | |
| if: runner.os == 'Windows' | |
| name: Run dbt runner with venv - Windows | |
| shell: cmd | |
| matrix_job_required_check: | |
| name: dbt runner tests | |
| needs: run_dbt | |
| runs-on: ubuntu-latest | |
| if: always() | |
| steps: | |
| - name: Check matrix job results | |
| if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') | |
| run: | | |
| echo "One or more matrix job tests failed or were cancelled. You may need to re-run them." && exit 1 |