test_dbt_runner.yml

name: test dbt runner

on:
  pull_request:
    branches:
      - master
      - devel
  workflow_dispatch:

concurrency:
  group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
  cancel-in-progress: true

env:

  DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }}
  RUNTIME__LOG_LEVEL: ERROR

jobs:
  get_docs_changes:
    uses: ./.github/workflows/get_docs_changes.yml
    if: ${{ !github.event.pull_request.head.repo.fork }}

  run_dbt:
    name: Tests dbt runner
    needs: get_docs_changes
    if: needs.get_docs_changes.outputs.changes_outside_docs == 'true'
    strategy:
      fail-fast: false
      matrix:
        os: ["ubuntu-latest"]
        # os: ["ubuntu-latest", "macos-latest", "windows-latest"]
    defaults:
      run:
        shell: bash
    runs-on: ${{ matrix.os }}

    steps:

      - name: Check out
        uses: actions/checkout@master

      - name: Setup Python
        uses: actions/setup-python@v4
        with:
          python-version: "3.10.x"

      - name: Install Poetry without dbt
        uses: snok/install-poetry@v1.3.2
        with:
          virtualenvs-create: true
          virtualenvs-in-project: true
          installer-parallel: true

      - name: Load cached venv
        id: cached-poetry-dependencies
        uses: actions/cache@v3
        with:
          # path: ${{ steps.pip-cache.outputs.dir }}
          path: .venv
          key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}-dbt-runner

      - name: Install dependencies
        # install dlt with postgres support
        run: poetry install --no-interaction -E postgres -E dbt --with sentry-sdk

      - name: create secrets.toml
        run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml

      - run: |
          poetry run pytest tests/helpers/dbt_tests -k '(not venv)'
        if: runner.os != 'Windows'
        name: Run dbt runner - Linux/MAC
      - run: |
          poetry run pytest tests/helpers/dbt_tests -k "(not venv)"
        if: runner.os == 'Windows'
        name: Run dbt runner - Windows
        shell: cmd

      - name: Remove dbt-core
        # install dlt with postgres support
        run: poetry run pip uninstall dbt-core -y

      - run: |
          poetry run pytest tests/helpers/dbt_tests --ignore=tests/helpers/dbt_tests/local -k '(not local)'
        if: runner.os != 'Windows'
        name: Run dbt runner with venv - Linux/MAC
      - run: |
          poetry run pytest tests/helpers/dbt_tests --ignore=tests/helpers/dbt_tests/local -m "not forked" -k "(not local)"
        if: runner.os == 'Windows'
        name: Run dbt runner with venv - Windows
        shell: cmd

  matrix_job_required_check:
    name: dbt runner tests
    needs: run_dbt
    runs-on: ubuntu-latest
    if: always()
    steps:
      - name: Check matrix job results
        if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')
        run: |
          echo "One or more matrix job tests failed or were cancelled. You may need to re-run them." && exit 1