ci improvements (#1159)

* remove matrix stuff and point destination tests to essential * mark one test as essential and switch destination tests to run essential tests * add prefixes to workflow files * unify naming a bit more * update snippets * increase time difference value in gdrive tests * allow workflows to run on forks if so labelled * remove secrets from local destinations tests * guard secrets leaking on docs tests * update job output * test label on bigquery * force rerun * put secrets back in local destinations * add new run setup to all destinations * mark a couple of tests as essential * add nightly schedule for full destination test run * fix tests condition * fix type * add a bunch of more essential markers * run full tests on mssql for all branches * fix newlines in yaml file * register essential marker and remove some essential tests * exclude \"athena-parquet-staging-iceberg\" from regular athena tests * remove regular athena tests from iceberg tests --------- Co-authored-by: rudolfix <[email protected]>
dlt-hub · Apr 7, 2024 · e0b4731 · e0b4731
1 parent d3ecc9e
commit e0b4731
Show file tree

Hide file tree

Showing 57 changed files with 263 additions and 315 deletions.
diff --git a/.github/workflows/deploy_docs.yml b/.github/workflows/deploy_docs.yml
@@ -1,4 +1,4 @@
-name: deploy docs
+name: docs | deploy docs
 
 on:
   schedule:

diff --git a/.github/workflows/get_docs_changes.yml b/.github/workflows/get_docs_changes.yml
@@ -1,4 +1,4 @@
-name: get docs changes
+name: util | get docs changes
 
 on:
   workflow_call:
@@ -13,6 +13,7 @@ env:
 
 jobs:
   get_docs_changes:
+    name: docs changes
     runs-on: ubuntu-latest
     outputs:
       changes_outside_docs: ${{ steps.check_changes.outputs.changes_outside_docs }}

diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml
@@ -1,5 +1,5 @@
 
-name: lint
+name: lint | code & tests
 
 on:
   pull_request:
@@ -14,10 +14,11 @@ concurrency:
 
 jobs:
   get_docs_changes:
+    name: docs changes
     uses: ./.github/workflows/get_docs_changes.yml
 
   run_lint:
-    name: Lint
+    name: lint
     needs: get_docs_changes
     if: needs.get_docs_changes.outputs.changes_outside_docs == 'true'
     strategy:
@@ -65,12 +66,6 @@ jobs:
           export PATH=$PATH:"/c/Program Files/usr/bin" # needed for Windows
           make lint
 
-      # - name: print envs
-      #   run: |
-      #     echo "The GitHub Actor's username is: $GITHUB_ACTOR"
-      #     echo "The GitHub repo owner is: $GITHUB_REPOSITORY_OWNER"
-      #     echo "The GitHub repo is: $GITHUB_REPOSITORY"
-
   matrix_job_required_check:
     name: Lint results
     needs: run_lint

diff --git a/.github/workflows/test_airflow.yml b/.github/workflows/test_airflow.yml
@@ -1,4 +1,4 @@
-name: test airflow integration
+name: tools | airflow
 
 on:
   pull_request:
@@ -13,10 +13,11 @@ concurrency:
 
 jobs:
   get_docs_changes:
+    name: docs changes
     uses: ./.github/workflows/get_docs_changes.yml
 
   run_airflow:
-    name: Tests Airflow integration
+    name: test
     needs: get_docs_changes
     if: needs.get_docs_changes.outputs.changes_outside_docs == 'true'
     runs-on: ubuntu-latest

diff --git a/.github/workflows/test_build_images.yml b/.github/workflows/test_build_images.yml
@@ -1,4 +1,4 @@
-name: test build docker images
+name: tools | docker images
 
 on:
   pull_request:
@@ -13,10 +13,11 @@ concurrency:
 
 jobs:
   get_docs_changes:
+    name: docs changes
     uses: ./.github/workflows/get_docs_changes.yml
 
   run_airflow:
-    name: Build alpine and airflow images
+    name: build
     needs: get_docs_changes
     if: needs.get_docs_changes.outputs.changes_outside_docs == 'true'
     runs-on: ubuntu-latest
@@ -38,4 +39,4 @@ jobs:
           installer-parallel: true
 
       - name: Build images
-        run: make test-build-images
+        run: make test-build-images
diff --git a/.github/workflows/test_common.yml b/.github/workflows/test_common.yml
@@ -1,4 +1,4 @@
-name: test common
+name: common | common
 
 on:
   pull_request:
@@ -16,10 +16,11 @@ env:
 
 jobs:
   get_docs_changes:
+    name: docs changes
     uses: ./.github/workflows/get_docs_changes.yml
 
   run_common:
-    name: Tests common dlt code
+    name: test
     needs: get_docs_changes
     if: needs.get_docs_changes.outputs.changes_outside_docs == 'true'
     strategy:

diff --git a/.github/workflows/test_dbt_cloud.yml b/.github/workflows/test_dbt_cloud.yml
@@ -1,5 +1,5 @@
 
-name: test dbt cloud
+name: tools | dbt cloud
 
 on:
   pull_request:
@@ -22,22 +22,18 @@ env:
 
 jobs:
   get_docs_changes:
+    name: docs changes
     uses: ./.github/workflows/get_docs_changes.yml
-    if: ${{ !github.event.pull_request.head.repo.fork }}
+    if: ${{ !github.event.pull_request.head.repo.fork || contains(github.event.pull_request.labels.*.name, 'ci from fork')}}
 
   run_dbt_cloud:
-    name: Tests dbt cloud
+    name: test
     needs: get_docs_changes
     if: needs.get_docs_changes.outputs.changes_outside_docs == 'true'
-    strategy:
-      fail-fast: false
-      matrix:
-        os: ["ubuntu-latest"]
-        # os: ["ubuntu-latest", "macos-latest", "windows-latest"]
     defaults:
       run:
         shell: bash
-    runs-on: ${{ matrix.os }}
+    runs-on: "ubuntu-latest"
 
     steps:
 
@@ -70,21 +66,4 @@ jobs:
 
       - run: |
           poetry run pytest tests/helpers/dbt_cloud_tests -k '(not venv)'
-        if: runner.os != 'Windows'
         name: Run dbt cloud - Linux/MAC
-      - run: |
-          poetry run pytest tests/helpers/dbt_cloud_tests -k "(not venv)"
-        if: runner.os == 'Windows'
-        name: Run dbt cloud - Windows
-        shell: cmd
-
-  matrix_job_required_check:
-    name: dbt cloud tests
-    needs: run_dbt_cloud
-    runs-on: ubuntu-latest
-    if: always()
-    steps:
-      - name: Check matrix job results
-        if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')
-        run: |
-          echo "One or more matrix job tests failed or were cancelled. You may need to re-run them." && exit 1
diff --git a/.github/workflows/test_dbt_runner.yml b/.github/workflows/test_dbt_runner.yml
@@ -1,5 +1,5 @@
 
-name: test dbt runner
+name: tools | dbt runner
 
 on:
   pull_request:
@@ -19,22 +19,18 @@ env:
 
 jobs:
   get_docs_changes:
+    name: docs changes
     uses: ./.github/workflows/get_docs_changes.yml
-    if: ${{ !github.event.pull_request.head.repo.fork }}
+    if: ${{ !github.event.pull_request.head.repo.fork || contains(github.event.pull_request.labels.*.name, 'ci from fork')}}
 
   run_dbt:
-    name: Tests dbt runner
+    name: test
     needs: get_docs_changes
     if: needs.get_docs_changes.outputs.changes_outside_docs == 'true'
-    strategy:
-      fail-fast: false
-      matrix:
-        os: ["ubuntu-latest"]
-        # os: ["ubuntu-latest", "macos-latest", "windows-latest"]
     defaults:
       run:
         shell: bash
-    runs-on: ${{ matrix.os }}
+    runs-on: "ubuntu-latest"
 
     steps:
 
@@ -84,21 +80,4 @@ jobs:
 
       - run: |
           poetry run pytest tests/helpers/dbt_tests --ignore=tests/helpers/dbt_tests/local -k '(not local)'
-        if: runner.os != 'Windows'
         name: Run dbt runner with venv - Linux/MAC
-      - run: |
-          poetry run pytest tests/helpers/dbt_tests --ignore=tests/helpers/dbt_tests/local -m "not forked" -k "(not local)"
-        if: runner.os == 'Windows'
-        name: Run dbt runner with venv - Windows
-        shell: cmd
-
-  matrix_job_required_check:
-    name: dbt runner tests
-    needs: run_dbt
-    runs-on: ubuntu-latest
-    if: always()
-    steps:
-      - name: Check matrix job results
-        if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')
-        run: |
-          echo "One or more matrix job tests failed or were cancelled. You may need to re-run them." && exit 1
diff --git a/.github/workflows/test_destination_athena.yml b/.github/workflows/test_destination_athena.yml
@@ -1,12 +1,14 @@
 
-name: test athena
+name: dest | athena
 
 on:
   pull_request:
     branches:
       - master
       - devel
   workflow_dispatch:
+  schedule:
+    - cron:  '0 2 * * *'
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
@@ -20,27 +22,23 @@ env:
   RUNTIME__DLTHUB_TELEMETRY_SEGMENT_WRITE_KEY: TLJiyRkGVZGCi2TtjClamXpFcxAA1rSB
   ACTIVE_DESTINATIONS: "[\"athena\"]"
   ALL_FILESYSTEM_DRIVERS: "[\"memory\"]"
-  EXCLUDED_DESTINATION_CONFIGURATIONS: "[\"athena-parquet-staging-iceberg\"]"
+  EXCLUDED_DESTINATION_CONFIGURATIONS: "[\"athena-parquet-staging-iceberg\", \"athena-parquet-no-staging-iceberg\"]"
 
 jobs:
   get_docs_changes:
+    name: docs changes
     uses: ./.github/workflows/get_docs_changes.yml
     # Tests that require credentials do not run in forks
-    if: ${{ !github.event.pull_request.head.repo.fork }}
+    if: ${{ !github.event.pull_request.head.repo.fork || contains(github.event.pull_request.labels.*.name, 'ci from fork')}}
 
   run_loader:
-    name: test destination athena
+    name: test
     needs: get_docs_changes
     if: needs.get_docs_changes.outputs.changes_outside_docs == 'true'
-    strategy:
-      fail-fast: false
-      matrix:
-        os: ["ubuntu-latest"]
-        # os: ["ubuntu-latest", "macos-latest", "windows-latest"]
     defaults:
       run:
         shell: bash
-    runs-on: ${{ matrix.os }}
+    runs-on: "ubuntu-latest"
 
     steps:
 
@@ -75,22 +73,11 @@ jobs:
         run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml
 
       - run: |
-          poetry run pytest tests/load
-        if: runner.os != 'Windows'
-        name: Run tests Linux/MAC
+          poetry run pytest tests/load -m "essential"
+        name: Run essential tests Linux
+        if: ${{ ! (contains(github.event.pull_request.labels.*.name, 'ci full') || !github.event_name == 'schedule')}}
+
       - run: |
           poetry run pytest tests/load
-        if: runner.os == 'Windows'
-        name: Run tests Windows
-        shell: cmd
-
-  matrix_job_required_check:
-    name: Redshift, PostgreSQL and DuckDB tests
-    needs: run_loader
-    runs-on: ubuntu-latest
-    if: always()
-    steps:
-      - name: Check matrix job results
-        if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')
-        run: |
-          echo "One or more matrix job tests failed or were cancelled. You may need to re-run them." && exit 1
+        name: Run all tests Linux
+        if: ${{ contains(github.event.pull_request.labels.*.name, 'ci full') || github.event_name == 'schedule'}}
diff --git a/.github/workflows/test_destination_athena_iceberg.yml b/.github/workflows/test_destination_athena_iceberg.yml
@@ -1,12 +1,14 @@
 
-name: test athena iceberg
+name: dest | athena iceberg
 
 on:
   pull_request:
     branches:
       - master
       - devel
   workflow_dispatch:
+  schedule:
+    - cron:  '0 2 * * *'
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}
@@ -20,27 +22,23 @@ env:
   RUNTIME__DLTHUB_TELEMETRY_SEGMENT_WRITE_KEY: TLJiyRkGVZGCi2TtjClamXpFcxAA1rSB
   ACTIVE_DESTINATIONS: "[\"athena\"]"
   ALL_FILESYSTEM_DRIVERS: "[\"memory\"]"
-  EXCLUDED_DESTINATION_CONFIGURATIONS: "[\"athena-no-staging\"]"
+  EXCLUDED_DESTINATION_CONFIGURATIONS: "[\"athena-no-staging\", \"athena-parquet-no-staging\"]"
 
 jobs:
   get_docs_changes:
+    name: docs changes
     uses: ./.github/workflows/get_docs_changes.yml
     # Tests that require credentials do not run in forks
-    if: ${{ !github.event.pull_request.head.repo.fork }}
+    if: ${{ !github.event.pull_request.head.repo.fork || contains(github.event.pull_request.labels.*.name, 'ci from fork')}}
 
   run_loader:
-    name: test destination athena iceberg
+    name: test
     needs: get_docs_changes
     if: needs.get_docs_changes.outputs.changes_outside_docs == 'true'
-    strategy:
-      fail-fast: false
-      matrix:
-        os: ["ubuntu-latest"]
-        # os: ["ubuntu-latest", "macos-latest", "windows-latest"]
     defaults:
       run:
         shell: bash
-    runs-on: ${{ matrix.os }}
+    runs-on: "ubuntu-latest"
 
     steps:
 
@@ -75,22 +73,11 @@ jobs:
         run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml
 
       - run: |
-          poetry run pytest tests/load
-        if: runner.os != 'Windows'
-        name: Run tests Linux/MAC
+          poetry run pytest tests/load -m "essential"
+        name: Run essential tests Linux
+        if: ${{ ! (contains(github.event.pull_request.labels.*.name, 'ci full') || github.event_name == 'schedule')}}
+
       - run: |
           poetry run pytest tests/load
-        if: runner.os == 'Windows'
-        name: Run tests Windows
-        shell: cmd
-
-  matrix_job_required_check:
-    name: Redshift, PostgreSQL and DuckDB tests
-    needs: run_loader
-    runs-on: ubuntu-latest
-    if: always()
-    steps:
-      - name: Check matrix job results
-        if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled')
-        run: |
-          echo "One or more matrix job tests failed or were cancelled. You may need to re-run them." && exit 1
+        name: Run all tests Linux
+        if: ${{ contains(github.event.pull_request.labels.*.name, 'ci full') || github.event_name == 'schedule'}}