remove matrix stuff and point destination tests to essential

dlt-hub · Mar 28, 2024 · 5bda072 · 5bda072
1 parent 7e8eed7
commit 5bda072
Show file tree

Hide file tree

Showing 12 changed files with 29 additions and 270 deletions.
diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
@@ -65,12 +65,6 @@ jobs:
           export PATH=$PATH:"/c/Program Files/usr/bin" # needed for Windows
           make lint
 
-      # - name: print envs
-      #   run: |
-      #     echo "The GitHub Actor's username is: $GITHUB_ACTOR"
-      #     echo "The GitHub repo owner is: $GITHUB_REPOSITORY_OWNER"
-      #     echo "The GitHub repo is: $GITHUB_REPOSITORY"
-
   matrix_job_required_check:
     name: Lint results
     needs: run_lint

diff --git a/.github/workflows/test_dbt_cloud.yml b/.github/workflows/test_dbt_cloud.yml
@@ -29,15 +29,10 @@ jobs:
     name: Tests dbt cloud
     needs: get_docs_changes
     if: needs.get_docs_changes.outputs.changes_outside_docs == 'true'
-    strategy:
-      fail-fast: false
-      matrix:
-        os: ["ubuntu-latest"]
-        # os: ["ubuntu-latest", "macos-latest", "windows-latest"]
     defaults:
       run:
         shell: bash
-    runs-on: ${{ matrix.os }}
+    runs-on: "ubuntu-latest"
 
     steps:
 
@@ -70,21 +65,4 @@ jobs:
 
       - run: |
           poetry run pytest tests/helpers/dbt_cloud_tests -k '(not venv)'
-        if: runner.os != 'Windows'
         name: Run dbt cloud - Linux/MAC
-      - run: |
-          poetry run pytest tests/helpers/dbt_cloud_tests -k "(not venv)"
-        if: runner.os == 'Windows'
-        name: Run dbt cloud - Windows
-        shell: cmd
-
-  matrix_job_required_check:
-    name: dbt cloud tests
-    needs: run_dbt_cloud
-    runs-on: ubuntu-latest
-    if: always()
-    steps:
-      - name: Check matrix job results
-        if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')
-        run: |
-          echo "One or more matrix job tests failed or were cancelled. You may need to re-run them." && exit 1
diff --git a/.github/workflows/test_dbt_runner.yml b/.github/workflows/test_dbt_runner.yml
@@ -26,15 +26,10 @@ jobs:
     name: Tests dbt runner
     needs: get_docs_changes
     if: needs.get_docs_changes.outputs.changes_outside_docs == 'true'
-    strategy:
-      fail-fast: false
-      matrix:
-        os: ["ubuntu-latest"]
-        # os: ["ubuntu-latest", "macos-latest", "windows-latest"]
     defaults:
       run:
         shell: bash
-    runs-on: ${{ matrix.os }}
+    runs-on: "ubuntu-latest"
 
     steps:
 
@@ -84,21 +79,5 @@ jobs:
 
       - run: |
           poetry run pytest tests/helpers/dbt_tests --ignore=tests/helpers/dbt_tests/local -k '(not local)'
-        if: runner.os != 'Windows'
         name: Run dbt runner with venv - Linux/MAC
-      - run: |
-          poetry run pytest tests/helpers/dbt_tests --ignore=tests/helpers/dbt_tests/local -m "not forked" -k "(not local)"
-        if: runner.os == 'Windows'
-        name: Run dbt runner with venv - Windows
-        shell: cmd
 
-  matrix_job_required_check:
-    name: dbt runner tests
-    needs: run_dbt
-    runs-on: ubuntu-latest
-    if: always()
-    steps:
-      - name: Check matrix job results
-        if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')
-        run: |
-          echo "One or more matrix job tests failed or were cancelled. You may need to re-run them." && exit 1
diff --git a/.github/workflows/test_destination_athena.yml b/.github/workflows/test_destination_athena.yml
@@ -32,15 +32,10 @@ jobs:
     name: test destination athena
     needs: get_docs_changes
     if: needs.get_docs_changes.outputs.changes_outside_docs == 'true'
-    strategy:
-      fail-fast: false
-      matrix:
-        os: ["ubuntu-latest"]
-        # os: ["ubuntu-latest", "macos-latest", "windows-latest"]
     defaults:
       run:
         shell: bash
-    runs-on: ${{ matrix.os }}
+    runs-on: "ubuntu-latest"
 
     steps:
 
@@ -75,22 +70,5 @@ jobs:
         run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml
 
       - run: |
-          poetry run pytest tests/load
-        if: runner.os != 'Windows'
-        name: Run tests Linux/MAC
-      - run: |
-          poetry run pytest tests/load
-        if: runner.os == 'Windows'
-        name: Run tests Windows
-        shell: cmd
-
-  matrix_job_required_check:
-    name: Redshift, PostgreSQL and DuckDB tests
-    needs: run_loader
-    runs-on: ubuntu-latest
-    if: always()
-    steps:
-      - name: Check matrix job results
-        if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')
-        run: |
-          echo "One or more matrix job tests failed or were cancelled. You may need to re-run them." && exit 1
+          poetry run pytest tests/load/essential
+        name: Run tests Linux/MAC
diff --git a/.github/workflows/test_destination_athena_iceberg.yml b/.github/workflows/test_destination_athena_iceberg.yml
@@ -32,15 +32,10 @@ jobs:
     name: test destination athena iceberg
     needs: get_docs_changes
     if: needs.get_docs_changes.outputs.changes_outside_docs == 'true'
-    strategy:
-      fail-fast: false
-      matrix:
-        os: ["ubuntu-latest"]
-        # os: ["ubuntu-latest", "macos-latest", "windows-latest"]
     defaults:
       run:
         shell: bash
-    runs-on: ${{ matrix.os }}
+    runs-on: "ubuntu-latest"
 
     steps:
 
@@ -75,22 +70,5 @@ jobs:
         run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml
 
       - run: |
-          poetry run pytest tests/load
-        if: runner.os != 'Windows'
-        name: Run tests Linux/MAC
-      - run: |
-          poetry run pytest tests/load
-        if: runner.os == 'Windows'
-        name: Run tests Windows
-        shell: cmd
-
-  matrix_job_required_check:
-    name: Redshift, PostgreSQL and DuckDB tests
-    needs: run_loader
-    runs-on: ubuntu-latest
-    if: always()
-    steps:
-      - name: Check matrix job results
-        if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')
-        run: |
-          echo "One or more matrix job tests failed or were cancelled. You may need to re-run them." && exit 1
+          poetry run pytest tests/load/essential
+        name: Run tests Linux/MAC
diff --git a/.github/workflows/test_destination_bigquery.yml b/.github/workflows/test_destination_bigquery.yml
@@ -30,15 +30,10 @@ jobs:
     name: Tests BigQuery loader
     needs: get_docs_changes
     if: needs.get_docs_changes.outputs.changes_outside_docs == 'true'
-    strategy:
-      fail-fast: false
-      matrix:
-        os: ["ubuntu-latest"]
-        # os: ["ubuntu-latest", "macos-latest", "windows-latest"]
     defaults:
       run:
         shell: bash
-    runs-on: ${{ matrix.os }}
+    runs-on: "ubuntu-latest"
 
     steps:
 
@@ -57,12 +52,6 @@ jobs:
           virtualenvs-in-project: true
           installer-parallel: true
 
-      # - name: Get pip cache dir
-      #   id: pip-cache
-      #   run: |
-      #     echo "::set-output name=dir::$(poetry env info -p)"
-      #     echo "$(poetry env info -p)"
-
       - name: Load cached venv
         id: cached-poetry-dependencies
         uses: actions/cache@v3
@@ -79,22 +68,5 @@ jobs:
         run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml
 
       - run: |
-          poetry run pytest tests/helpers/providers tests/load
-        if: runner.os != 'Windows'
-        name: Run tests Linux/MAC
-      - run: |
-          poetry run pytest tests/helpers/providers tests/load
-        if: runner.os == 'Windows'
-        name: Run tests Windows
-        shell: cmd
-
-  matrix_job_required_check:
-    name: BigQuery loader tests
-    needs: run_loader
-    runs-on: ubuntu-latest
-    if: always()
-    steps:
-      - name: Check matrix job results
-        if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')
-        run: |
-          echo "One or more matrix job tests failed or were cancelled. You may need to re-run them." && exit 1
+          poetry run pytest tests/load/essential
+        name: Run tests Linux/MAC
diff --git a/.github/workflows/test_destination_databricks.yml b/.github/workflows/test_destination_databricks.yml
@@ -30,15 +30,10 @@ jobs:
     name: Tests Databricks loader
     needs: get_docs_changes
     if: needs.get_docs_changes.outputs.changes_outside_docs == 'true'
-    strategy:
-      fail-fast: false
-      matrix:
-        os: ["ubuntu-latest"]
-        # os: ["ubuntu-latest", "macos-latest", "windows-latest"]
     defaults:
       run:
         shell: bash
-    runs-on: ${{ matrix.os }}
+    runs-on: "ubuntu-latest"
 
     steps:
 
@@ -71,22 +66,5 @@ jobs:
         run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml
 
       - run: |
-          poetry run pytest tests/load
-        if: runner.os != 'Windows'
-        name: Run tests Linux/MAC
-      - run: |
-          poetry run pytest tests/load
-        if: runner.os == 'Windows'
-        name: Run tests Windows
-        shell: cmd
-
-  matrix_job_required_check:
-    name: Databricks loader tests
-    needs: run_loader
-    runs-on: ubuntu-latest
-    if: always()
-    steps:
-      - name: Check matrix job results
-        if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')
-        run: |
-          echo "One or more matrix job tests failed or were cancelled. You may need to re-run them." && exit 1
+          poetry run pytest tests/load/essential
+        name: Run tests Linux/MAC
diff --git a/.github/workflows/test_destination_mssql.yml b/.github/workflows/test_destination_mssql.yml
@@ -31,14 +31,10 @@ jobs:
     name: Tests MS SQL loader
     needs: get_docs_changes
     if: needs.get_docs_changes.outputs.changes_outside_docs == 'true'
-    strategy:
-      fail-fast: false
-      matrix:
-        os: ["ubuntu-latest"]
     defaults:
       run:
         shell: bash
-    runs-on: ${{ matrix.os }}
+    runs-on: "ubuntu-latest"
 
     steps:
 
@@ -75,22 +71,5 @@ jobs:
         run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml
 
       - run: |
-          poetry run pytest tests/load
-        if: runner.os != 'Windows'
-        name: Run tests Linux/MAC
-      - run: |
-          poetry run pytest tests/load
-        if: runner.os == 'Windows'
-        name: Run tests Windows
-        shell: cmd
-
-  matrix_job_required_check:
-    name: MS SQL loader tests
-    needs: run_loader
-    runs-on: ubuntu-latest
-    if: always()
-    steps:
-      - name: Check matrix job results
-        if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')
-        run: |
-          echo "One or more matrix job tests failed or were cancelled. You may need to re-run them." && exit 1
+          poetry run pytest tests/load/essential
+        name: Run tests Linux/MAC
diff --git a/.github/workflows/test_destination_qdrant.yml b/.github/workflows/test_destination_qdrant.yml
@@ -29,15 +29,10 @@ jobs:
     name: Tests Qdrant loader
     needs: get_docs_changes
     if: needs.get_docs_changes.outputs.changes_outside_docs == 'true'
-    strategy:
-      fail-fast: false
-      matrix:
-        os:
-          - ubuntu-latest
     defaults:
       run:
         shell: bash
-    runs-on: ${{ matrix.os }}
+    runs-on: "ubuntu-latest"
 
     steps:
       - name: Check out
@@ -67,17 +62,7 @@ jobs:
 
       - name: Install dependencies
         run: poetry install --no-interaction -E qdrant -E parquet --with sentry-sdk --with pipeline
-      - run: |
-          poetry run pytest tests/load/
-        name: Run tests Linux
 
-  matrix_job_required_check:
-    name: Qdrant loader tests
-    needs: run_loader
-    runs-on: ubuntu-latest
-    if: always()
-    steps:
-      - name: Check matrix job results
-        if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')
-        run: |
-          echo "One or more matrix job tests failed or were cancelled. You may need to re-run them." && exit 1
+      - run: |
+          poetry run pytest tests/load/essential
+        name: Run tests Linux/MAC
diff --git a/.github/workflows/test_destination_snowflake.yml b/.github/workflows/test_destination_snowflake.yml
@@ -30,15 +30,10 @@ jobs:
     name: Tests Snowflake loader
     needs: get_docs_changes
     if: needs.get_docs_changes.outputs.changes_outside_docs == 'true'
-    strategy:
-      fail-fast: false
-      matrix:
-        os: ["ubuntu-latest"]
-        # os: ["ubuntu-latest", "macos-latest", "windows-latest"]
     defaults:
       run:
         shell: bash
-    runs-on: ${{ matrix.os }}
+    runs-on: "ubuntu-latest"
 
     steps:
 
@@ -71,22 +66,5 @@ jobs:
         run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml
 
       - run: |
-          poetry run pytest tests/load
-        if: runner.os != 'Windows'
-        name: Run tests Linux/MAC
-      - run: |
-          poetry run pytest tests/load
-        if: runner.os == 'Windows'
-        name: Run tests Windows
-        shell: cmd
-
-  matrix_job_required_check:
-    name: Snowflake loader tests
-    needs: run_loader
-    runs-on: ubuntu-latest
-    if: always()
-    steps:
-      - name: Check matrix job results
-        if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')
-        run: |
-          echo "One or more matrix job tests failed or were cancelled. You may need to re-run them." && exit 1
+          poetry run pytest tests/load/essential
+        name: Run tests Linux/MAC