From 5bda07215ad827c5781521377956e4e4abb09399 Mon Sep 17 00:00:00 2001 From: Dave Date: Thu, 28 Mar 2024 16:40:35 +0100 Subject: [PATCH 01/24] remove matrix stuff and point destination tests to essential --- .github/workflows/lint.yml | 6 ---- .github/workflows/test_dbt_cloud.yml | 24 +------------ .github/workflows/test_dbt_runner.yml | 23 +------------ .github/workflows/test_destination_athena.yml | 28 ++------------- .../test_destination_athena_iceberg.yml | 28 ++------------- .../workflows/test_destination_bigquery.yml | 34 ++----------------- .../workflows/test_destination_databricks.yml | 28 ++------------- .github/workflows/test_destination_mssql.yml | 27 ++------------- .github/workflows/test_destination_qdrant.yml | 23 +++---------- .../workflows/test_destination_snowflake.yml | 28 ++------------- .../workflows/test_destination_synapse.yml | 25 ++------------ .github/workflows/test_destinations.yml | 25 ++------------ 12 files changed, 29 insertions(+), 270 deletions(-) diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 317124f8c8..8edf8f44ce 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -65,12 +65,6 @@ jobs: export PATH=$PATH:"/c/Program Files/usr/bin" # needed for Windows make lint - # - name: print envs - # run: | - # echo "The GitHub Actor's username is: $GITHUB_ACTOR" - # echo "The GitHub repo owner is: $GITHUB_REPOSITORY_OWNER" - # echo "The GitHub repo is: $GITHUB_REPOSITORY" - matrix_job_required_check: name: Lint results needs: run_lint diff --git a/.github/workflows/test_dbt_cloud.yml b/.github/workflows/test_dbt_cloud.yml index a123e051e8..4d8a67b681 100644 --- a/.github/workflows/test_dbt_cloud.yml +++ b/.github/workflows/test_dbt_cloud.yml @@ -29,15 +29,10 @@ jobs: name: Tests dbt cloud needs: get_docs_changes if: needs.get_docs_changes.outputs.changes_outside_docs == 'true' - strategy: - fail-fast: false - matrix: - os: ["ubuntu-latest"] - # os: ["ubuntu-latest", "macos-latest", "windows-latest"] defaults: run: shell: bash - runs-on: ${{ matrix.os }} + runs-on: "ubuntu-latest" steps: @@ -70,21 +65,4 @@ jobs: - run: | poetry run pytest tests/helpers/dbt_cloud_tests -k '(not venv)' - if: runner.os != 'Windows' name: Run dbt cloud - Linux/MAC - - run: | - poetry run pytest tests/helpers/dbt_cloud_tests -k "(not venv)" - if: runner.os == 'Windows' - name: Run dbt cloud - Windows - shell: cmd - - matrix_job_required_check: - name: dbt cloud tests - needs: run_dbt_cloud - runs-on: ubuntu-latest - if: always() - steps: - - name: Check matrix job results - if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') - run: | - echo "One or more matrix job tests failed or were cancelled. You may need to re-run them." && exit 1 diff --git a/.github/workflows/test_dbt_runner.yml b/.github/workflows/test_dbt_runner.yml index 1c425f14e9..44aaf42397 100644 --- a/.github/workflows/test_dbt_runner.yml +++ b/.github/workflows/test_dbt_runner.yml @@ -26,15 +26,10 @@ jobs: name: Tests dbt runner needs: get_docs_changes if: needs.get_docs_changes.outputs.changes_outside_docs == 'true' - strategy: - fail-fast: false - matrix: - os: ["ubuntu-latest"] - # os: ["ubuntu-latest", "macos-latest", "windows-latest"] defaults: run: shell: bash - runs-on: ${{ matrix.os }} + runs-on: "ubuntu-latest" steps: @@ -84,21 +79,5 @@ jobs: - run: | poetry run pytest tests/helpers/dbt_tests --ignore=tests/helpers/dbt_tests/local -k '(not local)' - if: runner.os != 'Windows' name: Run dbt runner with venv - Linux/MAC - - run: | - poetry run pytest tests/helpers/dbt_tests --ignore=tests/helpers/dbt_tests/local -m "not forked" -k "(not local)" - if: runner.os == 'Windows' - name: Run dbt runner with venv - Windows - shell: cmd - matrix_job_required_check: - name: dbt runner tests - needs: run_dbt - runs-on: ubuntu-latest - if: always() - steps: - - name: Check matrix job results - if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') - run: | - echo "One or more matrix job tests failed or were cancelled. You may need to re-run them." && exit 1 diff --git a/.github/workflows/test_destination_athena.yml b/.github/workflows/test_destination_athena.yml index b94bdc6ee2..53ac5b320f 100644 --- a/.github/workflows/test_destination_athena.yml +++ b/.github/workflows/test_destination_athena.yml @@ -32,15 +32,10 @@ jobs: name: test destination athena needs: get_docs_changes if: needs.get_docs_changes.outputs.changes_outside_docs == 'true' - strategy: - fail-fast: false - matrix: - os: ["ubuntu-latest"] - # os: ["ubuntu-latest", "macos-latest", "windows-latest"] defaults: run: shell: bash - runs-on: ${{ matrix.os }} + runs-on: "ubuntu-latest" steps: @@ -75,22 +70,5 @@ jobs: run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml - run: | - poetry run pytest tests/load - if: runner.os != 'Windows' - name: Run tests Linux/MAC - - run: | - poetry run pytest tests/load - if: runner.os == 'Windows' - name: Run tests Windows - shell: cmd - - matrix_job_required_check: - name: Redshift, PostgreSQL and DuckDB tests - needs: run_loader - runs-on: ubuntu-latest - if: always() - steps: - - name: Check matrix job results - if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') - run: | - echo "One or more matrix job tests failed or were cancelled. You may need to re-run them." && exit 1 + poetry run pytest tests/load/essential + name: Run tests Linux/MAC \ No newline at end of file diff --git a/.github/workflows/test_destination_athena_iceberg.yml b/.github/workflows/test_destination_athena_iceberg.yml index acb5f35dfd..920ca3eb2b 100644 --- a/.github/workflows/test_destination_athena_iceberg.yml +++ b/.github/workflows/test_destination_athena_iceberg.yml @@ -32,15 +32,10 @@ jobs: name: test destination athena iceberg needs: get_docs_changes if: needs.get_docs_changes.outputs.changes_outside_docs == 'true' - strategy: - fail-fast: false - matrix: - os: ["ubuntu-latest"] - # os: ["ubuntu-latest", "macos-latest", "windows-latest"] defaults: run: shell: bash - runs-on: ${{ matrix.os }} + runs-on: "ubuntu-latest" steps: @@ -75,22 +70,5 @@ jobs: run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml - run: | - poetry run pytest tests/load - if: runner.os != 'Windows' - name: Run tests Linux/MAC - - run: | - poetry run pytest tests/load - if: runner.os == 'Windows' - name: Run tests Windows - shell: cmd - - matrix_job_required_check: - name: Redshift, PostgreSQL and DuckDB tests - needs: run_loader - runs-on: ubuntu-latest - if: always() - steps: - - name: Check matrix job results - if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') - run: | - echo "One or more matrix job tests failed or were cancelled. You may need to re-run them." && exit 1 + poetry run pytest tests/load/essential + name: Run tests Linux/MAC \ No newline at end of file diff --git a/.github/workflows/test_destination_bigquery.yml b/.github/workflows/test_destination_bigquery.yml index d11f7155d4..b2ad6c2edf 100644 --- a/.github/workflows/test_destination_bigquery.yml +++ b/.github/workflows/test_destination_bigquery.yml @@ -30,15 +30,10 @@ jobs: name: Tests BigQuery loader needs: get_docs_changes if: needs.get_docs_changes.outputs.changes_outside_docs == 'true' - strategy: - fail-fast: false - matrix: - os: ["ubuntu-latest"] - # os: ["ubuntu-latest", "macos-latest", "windows-latest"] defaults: run: shell: bash - runs-on: ${{ matrix.os }} + runs-on: "ubuntu-latest" steps: @@ -57,12 +52,6 @@ jobs: virtualenvs-in-project: true installer-parallel: true - # - name: Get pip cache dir - # id: pip-cache - # run: | - # echo "::set-output name=dir::$(poetry env info -p)" - # echo "$(poetry env info -p)" - - name: Load cached venv id: cached-poetry-dependencies uses: actions/cache@v3 @@ -79,22 +68,5 @@ jobs: run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml - run: | - poetry run pytest tests/helpers/providers tests/load - if: runner.os != 'Windows' - name: Run tests Linux/MAC - - run: | - poetry run pytest tests/helpers/providers tests/load - if: runner.os == 'Windows' - name: Run tests Windows - shell: cmd - - matrix_job_required_check: - name: BigQuery loader tests - needs: run_loader - runs-on: ubuntu-latest - if: always() - steps: - - name: Check matrix job results - if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') - run: | - echo "One or more matrix job tests failed or were cancelled. You may need to re-run them." && exit 1 + poetry run pytest tests/load/essential + name: Run tests Linux/MAC \ No newline at end of file diff --git a/.github/workflows/test_destination_databricks.yml b/.github/workflows/test_destination_databricks.yml index 2a2fa8e10d..9ecded0234 100644 --- a/.github/workflows/test_destination_databricks.yml +++ b/.github/workflows/test_destination_databricks.yml @@ -30,15 +30,10 @@ jobs: name: Tests Databricks loader needs: get_docs_changes if: needs.get_docs_changes.outputs.changes_outside_docs == 'true' - strategy: - fail-fast: false - matrix: - os: ["ubuntu-latest"] - # os: ["ubuntu-latest", "macos-latest", "windows-latest"] defaults: run: shell: bash - runs-on: ${{ matrix.os }} + runs-on: "ubuntu-latest" steps: @@ -71,22 +66,5 @@ jobs: run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml - run: | - poetry run pytest tests/load - if: runner.os != 'Windows' - name: Run tests Linux/MAC - - run: | - poetry run pytest tests/load - if: runner.os == 'Windows' - name: Run tests Windows - shell: cmd - - matrix_job_required_check: - name: Databricks loader tests - needs: run_loader - runs-on: ubuntu-latest - if: always() - steps: - - name: Check matrix job results - if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') - run: | - echo "One or more matrix job tests failed or were cancelled. You may need to re-run them." && exit 1 + poetry run pytest tests/load/essential + name: Run tests Linux/MAC \ No newline at end of file diff --git a/.github/workflows/test_destination_mssql.yml b/.github/workflows/test_destination_mssql.yml index f96c64219d..179755e527 100644 --- a/.github/workflows/test_destination_mssql.yml +++ b/.github/workflows/test_destination_mssql.yml @@ -31,14 +31,10 @@ jobs: name: Tests MS SQL loader needs: get_docs_changes if: needs.get_docs_changes.outputs.changes_outside_docs == 'true' - strategy: - fail-fast: false - matrix: - os: ["ubuntu-latest"] defaults: run: shell: bash - runs-on: ${{ matrix.os }} + runs-on: "ubuntu-latest" steps: @@ -75,22 +71,5 @@ jobs: run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml - run: | - poetry run pytest tests/load - if: runner.os != 'Windows' - name: Run tests Linux/MAC - - run: | - poetry run pytest tests/load - if: runner.os == 'Windows' - name: Run tests Windows - shell: cmd - - matrix_job_required_check: - name: MS SQL loader tests - needs: run_loader - runs-on: ubuntu-latest - if: always() - steps: - - name: Check matrix job results - if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') - run: | - echo "One or more matrix job tests failed or were cancelled. You may need to re-run them." && exit 1 + poetry run pytest tests/load/essential + name: Run tests Linux/MAC \ No newline at end of file diff --git a/.github/workflows/test_destination_qdrant.yml b/.github/workflows/test_destination_qdrant.yml index 3237801dbf..1b24a32597 100644 --- a/.github/workflows/test_destination_qdrant.yml +++ b/.github/workflows/test_destination_qdrant.yml @@ -29,15 +29,10 @@ jobs: name: Tests Qdrant loader needs: get_docs_changes if: needs.get_docs_changes.outputs.changes_outside_docs == 'true' - strategy: - fail-fast: false - matrix: - os: - - ubuntu-latest defaults: run: shell: bash - runs-on: ${{ matrix.os }} + runs-on: "ubuntu-latest" steps: - name: Check out @@ -67,17 +62,7 @@ jobs: - name: Install dependencies run: poetry install --no-interaction -E qdrant -E parquet --with sentry-sdk --with pipeline - - run: | - poetry run pytest tests/load/ - name: Run tests Linux - matrix_job_required_check: - name: Qdrant loader tests - needs: run_loader - runs-on: ubuntu-latest - if: always() - steps: - - name: Check matrix job results - if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') - run: | - echo "One or more matrix job tests failed or were cancelled. You may need to re-run them." && exit 1 + - run: | + poetry run pytest tests/load/essential + name: Run tests Linux/MAC \ No newline at end of file diff --git a/.github/workflows/test_destination_snowflake.yml b/.github/workflows/test_destination_snowflake.yml index 1ef290682c..efe398c8ea 100644 --- a/.github/workflows/test_destination_snowflake.yml +++ b/.github/workflows/test_destination_snowflake.yml @@ -30,15 +30,10 @@ jobs: name: Tests Snowflake loader needs: get_docs_changes if: needs.get_docs_changes.outputs.changes_outside_docs == 'true' - strategy: - fail-fast: false - matrix: - os: ["ubuntu-latest"] - # os: ["ubuntu-latest", "macos-latest", "windows-latest"] defaults: run: shell: bash - runs-on: ${{ matrix.os }} + runs-on: "ubuntu-latest" steps: @@ -71,22 +66,5 @@ jobs: run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml - run: | - poetry run pytest tests/load - if: runner.os != 'Windows' - name: Run tests Linux/MAC - - run: | - poetry run pytest tests/load - if: runner.os == 'Windows' - name: Run tests Windows - shell: cmd - - matrix_job_required_check: - name: Snowflake loader tests - needs: run_loader - runs-on: ubuntu-latest - if: always() - steps: - - name: Check matrix job results - if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') - run: | - echo "One or more matrix job tests failed or were cancelled. You may need to re-run them." && exit 1 + poetry run pytest tests/load/essential + name: Run tests Linux/MAC \ No newline at end of file diff --git a/.github/workflows/test_destination_synapse.yml b/.github/workflows/test_destination_synapse.yml index 774c83314f..69dbbc4dff 100644 --- a/.github/workflows/test_destination_synapse.yml +++ b/.github/workflows/test_destination_synapse.yml @@ -29,14 +29,10 @@ jobs: name: Tests Synapse loader needs: get_docs_changes if: needs.get_docs_changes.outputs.changes_outside_docs == 'true' - strategy: - fail-fast: false - matrix: - os: ["ubuntu-latest"] defaults: run: shell: bash - runs-on: ${{ matrix.os }} + runs-on: "ubuntu-latest" steps: @@ -73,22 +69,5 @@ jobs: run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml - run: | - poetry run pytest tests/load - if: runner.os != 'Windows' + poetry run pytest tests/load/essential name: Run tests Linux/MAC - - run: | - poetry run pytest tests/load - if: runner.os == 'Windows' - name: Run tests Windows - shell: cmd - - matrix_job_required_check: - name: Synapse loader tests - needs: run_loader - runs-on: ubuntu-latest - if: always() - steps: - - name: Check matrix job results - if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') - run: | - echo "One or more matrix job tests failed or were cancelled. You may need to re-run them." && exit 1 diff --git a/.github/workflows/test_destinations.yml b/.github/workflows/test_destinations.yml index a635e2865c..0c2e954e45 100644 --- a/.github/workflows/test_destinations.yml +++ b/.github/workflows/test_destinations.yml @@ -40,13 +40,10 @@ jobs: if: needs.get_docs_changes.outputs.changes_outside_docs == 'true' strategy: fail-fast: false - matrix: - os: ["ubuntu-latest"] - # os: ["ubuntu-latest", "macos-latest", "windows-latest"] defaults: run: shell: bash - runs-on: ${{ matrix.os }} + runs-on: "ubuntu-latest" steps: @@ -81,22 +78,6 @@ jobs: run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml - run: | - poetry run pytest tests/load - if: runner.os != 'Windows' + poetry run pytest tests/load/essential name: Run tests Linux/MAC - - run: | - poetry run pytest tests/load - if: runner.os == 'Windows' - name: Run tests Windows - shell: cmd - - matrix_job_required_check: - name: Redshift, PostgreSQL and DuckDB tests - needs: run_loader - runs-on: ubuntu-latest - if: always() - steps: - - name: Check matrix job results - if: contains(needs.*.result, 'failure') || contains(needs.*.result, 'cancelled') - run: | - echo "One or more matrix job tests failed or were cancelled. You may need to re-run them." && exit 1 + From 190fd2f26fe895a1cf3201874955026d72ff45cd Mon Sep 17 00:00:00 2001 From: Dave Date: Thu, 28 Mar 2024 16:48:12 +0100 Subject: [PATCH 02/24] mark one test as essential and switch destination tests to run essential tests --- .github/workflows/test_destination_athena.yml | 2 +- .github/workflows/test_destination_athena_iceberg.yml | 2 +- .github/workflows/test_destination_bigquery.yml | 2 +- .github/workflows/test_destination_databricks.yml | 2 +- .github/workflows/test_destination_mssql.yml | 2 +- .github/workflows/test_destination_qdrant.yml | 2 +- .github/workflows/test_destination_snowflake.yml | 2 +- .github/workflows/test_destination_synapse.yml | 2 +- .github/workflows/test_destinations.yml | 2 +- tests/load/pipeline/test_restore_state.py | 1 + 10 files changed, 10 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test_destination_athena.yml b/.github/workflows/test_destination_athena.yml index 53ac5b320f..177d7f6096 100644 --- a/.github/workflows/test_destination_athena.yml +++ b/.github/workflows/test_destination_athena.yml @@ -70,5 +70,5 @@ jobs: run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml - run: | - poetry run pytest tests/load/essential + poetry run pytest tests/load -m "essential" name: Run tests Linux/MAC \ No newline at end of file diff --git a/.github/workflows/test_destination_athena_iceberg.yml b/.github/workflows/test_destination_athena_iceberg.yml index 920ca3eb2b..920f894aab 100644 --- a/.github/workflows/test_destination_athena_iceberg.yml +++ b/.github/workflows/test_destination_athena_iceberg.yml @@ -70,5 +70,5 @@ jobs: run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml - run: | - poetry run pytest tests/load/essential + poetry run pytest tests/load -m "essential" name: Run tests Linux/MAC \ No newline at end of file diff --git a/.github/workflows/test_destination_bigquery.yml b/.github/workflows/test_destination_bigquery.yml index b2ad6c2edf..4314893a6f 100644 --- a/.github/workflows/test_destination_bigquery.yml +++ b/.github/workflows/test_destination_bigquery.yml @@ -68,5 +68,5 @@ jobs: run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml - run: | - poetry run pytest tests/load/essential + poetry run pytest tests/load -m "essential" name: Run tests Linux/MAC \ No newline at end of file diff --git a/.github/workflows/test_destination_databricks.yml b/.github/workflows/test_destination_databricks.yml index 9ecded0234..cb6dc27954 100644 --- a/.github/workflows/test_destination_databricks.yml +++ b/.github/workflows/test_destination_databricks.yml @@ -66,5 +66,5 @@ jobs: run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml - run: | - poetry run pytest tests/load/essential + poetry run pytest tests/load -m "essential" name: Run tests Linux/MAC \ No newline at end of file diff --git a/.github/workflows/test_destination_mssql.yml b/.github/workflows/test_destination_mssql.yml index 179755e527..7b14dfd976 100644 --- a/.github/workflows/test_destination_mssql.yml +++ b/.github/workflows/test_destination_mssql.yml @@ -71,5 +71,5 @@ jobs: run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml - run: | - poetry run pytest tests/load/essential + poetry run pytest tests/load -m "essential" name: Run tests Linux/MAC \ No newline at end of file diff --git a/.github/workflows/test_destination_qdrant.yml b/.github/workflows/test_destination_qdrant.yml index 1b24a32597..9ef3253629 100644 --- a/.github/workflows/test_destination_qdrant.yml +++ b/.github/workflows/test_destination_qdrant.yml @@ -64,5 +64,5 @@ jobs: run: poetry install --no-interaction -E qdrant -E parquet --with sentry-sdk --with pipeline - run: | - poetry run pytest tests/load/essential + poetry run pytest tests/load -m "essential" name: Run tests Linux/MAC \ No newline at end of file diff --git a/.github/workflows/test_destination_snowflake.yml b/.github/workflows/test_destination_snowflake.yml index efe398c8ea..7535616b60 100644 --- a/.github/workflows/test_destination_snowflake.yml +++ b/.github/workflows/test_destination_snowflake.yml @@ -66,5 +66,5 @@ jobs: run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml - run: | - poetry run pytest tests/load/essential + poetry run pytest tests/load -m "essential" name: Run tests Linux/MAC \ No newline at end of file diff --git a/.github/workflows/test_destination_synapse.yml b/.github/workflows/test_destination_synapse.yml index 69dbbc4dff..c287c751ac 100644 --- a/.github/workflows/test_destination_synapse.yml +++ b/.github/workflows/test_destination_synapse.yml @@ -69,5 +69,5 @@ jobs: run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml - run: | - poetry run pytest tests/load/essential + poetry run pytest tests/load -m "essential" name: Run tests Linux/MAC diff --git a/.github/workflows/test_destinations.yml b/.github/workflows/test_destinations.yml index 0c2e954e45..6742f372c3 100644 --- a/.github/workflows/test_destinations.yml +++ b/.github/workflows/test_destinations.yml @@ -78,6 +78,6 @@ jobs: run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml - run: | - poetry run pytest tests/load/essential + poetry run pytest tests/load -m "essential" name: Run tests Linux/MAC diff --git a/tests/load/pipeline/test_restore_state.py b/tests/load/pipeline/test_restore_state.py index e50654adcc..d421819121 100644 --- a/tests/load/pipeline/test_restore_state.py +++ b/tests/load/pipeline/test_restore_state.py @@ -39,6 +39,7 @@ def duckdb_pipeline_location() -> None: del os.environ["DESTINATION__DUCKDB__CREDENTIALS"] +@pytest.mark.essential @pytest.mark.parametrize( "destination_config", destinations_configs( From 053ea3ad7abcb0ea632434860cfc089ef8d4329e Mon Sep 17 00:00:00 2001 From: Dave Date: Thu, 28 Mar 2024 16:55:33 +0100 Subject: [PATCH 03/24] add prefixes to workflow files --- .github/workflows/deploy_docs.yml | 2 +- .github/workflows/get_docs_changes.yml | 2 +- .github/workflows/lint.yml | 2 +- .github/workflows/test_airflow.yml | 2 +- .github/workflows/test_build_images.yml | 2 +- .github/workflows/test_common.yml | 2 +- .github/workflows/test_dbt_cloud.yml | 2 +- .github/workflows/test_dbt_runner.yml | 2 +- .github/workflows/test_destination_athena.yml | 2 +- .github/workflows/test_destination_athena_iceberg.yml | 2 +- .github/workflows/test_destination_bigquery.yml | 2 +- .github/workflows/test_destination_databricks.yml | 2 +- .github/workflows/test_destination_mssql.yml | 2 +- .github/workflows/test_destination_qdrant.yml | 2 +- .github/workflows/test_destination_snowflake.yml | 2 +- .github/workflows/test_destination_synapse.yml | 2 +- .github/workflows/test_destinations.yml | 2 +- .github/workflows/test_doc_snippets.yml | 2 +- .github/workflows/test_local_destinations.yml | 2 +- 19 files changed, 19 insertions(+), 19 deletions(-) diff --git a/.github/workflows/deploy_docs.yml b/.github/workflows/deploy_docs.yml index 4dbfd3fb9a..b7a2834745 100644 --- a/.github/workflows/deploy_docs.yml +++ b/.github/workflows/deploy_docs.yml @@ -1,4 +1,4 @@ -name: deploy docs +name: docs | deploy docs on: schedule: diff --git a/.github/workflows/get_docs_changes.yml b/.github/workflows/get_docs_changes.yml index d0fd936e00..8f2e630480 100644 --- a/.github/workflows/get_docs_changes.yml +++ b/.github/workflows/get_docs_changes.yml @@ -1,4 +1,4 @@ -name: get docs changes +name: util | get docs changes on: workflow_call: diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 8edf8f44ce..c300643985 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -1,5 +1,5 @@ -name: lint +name: lint | lint code and tests on: pull_request: diff --git a/.github/workflows/test_airflow.yml b/.github/workflows/test_airflow.yml index 02513618d6..43b5cfeac1 100644 --- a/.github/workflows/test_airflow.yml +++ b/.github/workflows/test_airflow.yml @@ -1,4 +1,4 @@ -name: test airflow integration +name: tools | test airflow integration on: pull_request: diff --git a/.github/workflows/test_build_images.yml b/.github/workflows/test_build_images.yml index 489d776f40..a1a73ad010 100644 --- a/.github/workflows/test_build_images.yml +++ b/.github/workflows/test_build_images.yml @@ -1,4 +1,4 @@ -name: test build docker images +name: tools | test build docker images on: pull_request: diff --git a/.github/workflows/test_common.yml b/.github/workflows/test_common.yml index 2d96d2eb95..a692ffd3de 100644 --- a/.github/workflows/test_common.yml +++ b/.github/workflows/test_common.yml @@ -1,4 +1,4 @@ -name: test common +name: common | test common on: pull_request: diff --git a/.github/workflows/test_dbt_cloud.yml b/.github/workflows/test_dbt_cloud.yml index 4d8a67b681..7247db5ce2 100644 --- a/.github/workflows/test_dbt_cloud.yml +++ b/.github/workflows/test_dbt_cloud.yml @@ -1,5 +1,5 @@ -name: test dbt cloud +name: tools | test dbt cloud on: pull_request: diff --git a/.github/workflows/test_dbt_runner.yml b/.github/workflows/test_dbt_runner.yml index 44aaf42397..e53f326baa 100644 --- a/.github/workflows/test_dbt_runner.yml +++ b/.github/workflows/test_dbt_runner.yml @@ -1,5 +1,5 @@ -name: test dbt runner +name: tools | test dbt runner on: pull_request: diff --git a/.github/workflows/test_destination_athena.yml b/.github/workflows/test_destination_athena.yml index 177d7f6096..5d5c11d9c6 100644 --- a/.github/workflows/test_destination_athena.yml +++ b/.github/workflows/test_destination_athena.yml @@ -1,5 +1,5 @@ -name: test athena +name: destinations | test athena on: pull_request: diff --git a/.github/workflows/test_destination_athena_iceberg.yml b/.github/workflows/test_destination_athena_iceberg.yml index 920f894aab..1a428f45fd 100644 --- a/.github/workflows/test_destination_athena_iceberg.yml +++ b/.github/workflows/test_destination_athena_iceberg.yml @@ -1,5 +1,5 @@ -name: test athena iceberg +name: destinations | test athena iceberg on: pull_request: diff --git a/.github/workflows/test_destination_bigquery.yml b/.github/workflows/test_destination_bigquery.yml index 4314893a6f..53f9018d38 100644 --- a/.github/workflows/test_destination_bigquery.yml +++ b/.github/workflows/test_destination_bigquery.yml @@ -1,5 +1,5 @@ -name: test bigquery +name: destinations | test bigquery on: pull_request: diff --git a/.github/workflows/test_destination_databricks.yml b/.github/workflows/test_destination_databricks.yml index cb6dc27954..6e1173e3d8 100644 --- a/.github/workflows/test_destination_databricks.yml +++ b/.github/workflows/test_destination_databricks.yml @@ -1,5 +1,5 @@ -name: test databricks +name: destinations | test databricks on: pull_request: diff --git a/.github/workflows/test_destination_mssql.yml b/.github/workflows/test_destination_mssql.yml index 7b14dfd976..8b892e158a 100644 --- a/.github/workflows/test_destination_mssql.yml +++ b/.github/workflows/test_destination_mssql.yml @@ -1,5 +1,5 @@ -name: test MS SQL +name: destinations | test MS SQL on: pull_request: diff --git a/.github/workflows/test_destination_qdrant.yml b/.github/workflows/test_destination_qdrant.yml index 9ef3253629..24fa0d9636 100644 --- a/.github/workflows/test_destination_qdrant.yml +++ b/.github/workflows/test_destination_qdrant.yml @@ -1,4 +1,4 @@ -name: test Qdrant +name: destinations | test Qdrant on: pull_request: diff --git a/.github/workflows/test_destination_snowflake.yml b/.github/workflows/test_destination_snowflake.yml index 7535616b60..53e8af468b 100644 --- a/.github/workflows/test_destination_snowflake.yml +++ b/.github/workflows/test_destination_snowflake.yml @@ -1,5 +1,5 @@ -name: test snowflake +name: destinations | test snowflake on: pull_request: diff --git a/.github/workflows/test_destination_synapse.yml b/.github/workflows/test_destination_synapse.yml index c287c751ac..be818acb39 100644 --- a/.github/workflows/test_destination_synapse.yml +++ b/.github/workflows/test_destination_synapse.yml @@ -1,4 +1,4 @@ -name: test synapse +name: destinations | test synapse on: pull_request: diff --git a/.github/workflows/test_destinations.yml b/.github/workflows/test_destinations.yml index 6742f372c3..246325ed4a 100644 --- a/.github/workflows/test_destinations.yml +++ b/.github/workflows/test_destinations.yml @@ -1,5 +1,5 @@ -name: test redshift, postgres and filesystem buckets +name: destinations | test redshift, postgres and filesystem buckets on: pull_request: diff --git a/.github/workflows/test_doc_snippets.yml b/.github/workflows/test_doc_snippets.yml index 7a862c5800..5587f7ca31 100644 --- a/.github/workflows/test_doc_snippets.yml +++ b/.github/workflows/test_doc_snippets.yml @@ -1,5 +1,5 @@ -name: docs snippet lint and test +name: docs | lint and test snippets & examples on: pull_request: diff --git a/.github/workflows/test_local_destinations.yml b/.github/workflows/test_local_destinations.yml index 11377095d0..b9f5cbc6b9 100644 --- a/.github/workflows/test_local_destinations.yml +++ b/.github/workflows/test_local_destinations.yml @@ -1,7 +1,7 @@ # Tests destinations that can run without credentials. # i.e. local postgres, duckdb, filesystem (with local fs/memory bucket) -name: test local destinations +name: destinations | test local destinations on: pull_request: From 7851ec226f74271aa115f561996089334f78d36b Mon Sep 17 00:00:00 2001 From: Dave Date: Thu, 28 Mar 2024 18:12:26 +0100 Subject: [PATCH 04/24] unify naming a bit more --- .github/workflows/get_docs_changes.yml | 3 ++- .github/workflows/lint.yml | 5 +++-- .github/workflows/test_airflow.yml | 5 +++-- .github/workflows/test_build_images.yml | 5 +++-- .github/workflows/test_common.yml | 5 +++-- .github/workflows/test_dbt_cloud.yml | 5 +++-- .github/workflows/test_dbt_runner.yml | 5 +++-- .github/workflows/test_destination_athena.yml | 5 +++-- .github/workflows/test_destination_athena_iceberg.yml | 5 +++-- .github/workflows/test_destination_bigquery.yml | 5 +++-- .github/workflows/test_destination_databricks.yml | 5 +++-- .github/workflows/test_destination_mssql.yml | 5 +++-- .github/workflows/test_destination_qdrant.yml | 5 +++-- .github/workflows/test_destination_snowflake.yml | 5 +++-- .github/workflows/test_destination_synapse.yml | 5 +++-- .github/workflows/test_destinations.yml | 5 +++-- .github/workflows/test_doc_snippets.yml | 2 +- .github/workflows/test_local_destinations.yml | 5 +++-- 18 files changed, 51 insertions(+), 34 deletions(-) diff --git a/.github/workflows/get_docs_changes.yml b/.github/workflows/get_docs_changes.yml index 8f2e630480..49c6a6aa0c 100644 --- a/.github/workflows/get_docs_changes.yml +++ b/.github/workflows/get_docs_changes.yml @@ -1,4 +1,4 @@ -name: util | get docs changes +name: util | get docs changes on: workflow_call: @@ -13,6 +13,7 @@ env: jobs: get_docs_changes: + name: docs changes runs-on: ubuntu-latest outputs: changes_outside_docs: ${{ steps.check_changes.outputs.changes_outside_docs }} diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index c300643985..0c6ddcee73 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -1,5 +1,5 @@ -name: lint | lint code and tests +name: lint | code & tests on: pull_request: @@ -14,10 +14,11 @@ concurrency: jobs: get_docs_changes: + name: docs changes uses: ./.github/workflows/get_docs_changes.yml run_lint: - name: Lint + name: lint needs: get_docs_changes if: needs.get_docs_changes.outputs.changes_outside_docs == 'true' strategy: diff --git a/.github/workflows/test_airflow.yml b/.github/workflows/test_airflow.yml index 43b5cfeac1..8e3a9cf3d8 100644 --- a/.github/workflows/test_airflow.yml +++ b/.github/workflows/test_airflow.yml @@ -1,4 +1,4 @@ -name: tools | test airflow integration +name: tools | airflow on: pull_request: @@ -13,10 +13,11 @@ concurrency: jobs: get_docs_changes: + name: docs changes uses: ./.github/workflows/get_docs_changes.yml run_airflow: - name: Tests Airflow integration + name: test needs: get_docs_changes if: needs.get_docs_changes.outputs.changes_outside_docs == 'true' runs-on: ubuntu-latest diff --git a/.github/workflows/test_build_images.yml b/.github/workflows/test_build_images.yml index a1a73ad010..a26c93d5cc 100644 --- a/.github/workflows/test_build_images.yml +++ b/.github/workflows/test_build_images.yml @@ -1,4 +1,4 @@ -name: tools | test build docker images +name: tools | docker images on: pull_request: @@ -13,10 +13,11 @@ concurrency: jobs: get_docs_changes: + name: docs changes uses: ./.github/workflows/get_docs_changes.yml run_airflow: - name: Build alpine and airflow images + name: build needs: get_docs_changes if: needs.get_docs_changes.outputs.changes_outside_docs == 'true' runs-on: ubuntu-latest diff --git a/.github/workflows/test_common.yml b/.github/workflows/test_common.yml index a692ffd3de..116d4e6b6a 100644 --- a/.github/workflows/test_common.yml +++ b/.github/workflows/test_common.yml @@ -1,4 +1,4 @@ -name: common | test common +name: common | common on: pull_request: @@ -16,10 +16,11 @@ env: jobs: get_docs_changes: + name: docs changes uses: ./.github/workflows/get_docs_changes.yml run_common: - name: Tests common dlt code + name: test needs: get_docs_changes if: needs.get_docs_changes.outputs.changes_outside_docs == 'true' strategy: diff --git a/.github/workflows/test_dbt_cloud.yml b/.github/workflows/test_dbt_cloud.yml index 7247db5ce2..3c7bbddb94 100644 --- a/.github/workflows/test_dbt_cloud.yml +++ b/.github/workflows/test_dbt_cloud.yml @@ -1,5 +1,5 @@ -name: tools | test dbt cloud +name: tools | dbt cloud on: pull_request: @@ -22,11 +22,12 @@ env: jobs: get_docs_changes: + name: docs changes uses: ./.github/workflows/get_docs_changes.yml if: ${{ !github.event.pull_request.head.repo.fork }} run_dbt_cloud: - name: Tests dbt cloud + name: test needs: get_docs_changes if: needs.get_docs_changes.outputs.changes_outside_docs == 'true' defaults: diff --git a/.github/workflows/test_dbt_runner.yml b/.github/workflows/test_dbt_runner.yml index e53f326baa..0ed90d9d3d 100644 --- a/.github/workflows/test_dbt_runner.yml +++ b/.github/workflows/test_dbt_runner.yml @@ -1,5 +1,5 @@ -name: tools | test dbt runner +name: tools | dbt runner on: pull_request: @@ -19,11 +19,12 @@ env: jobs: get_docs_changes: + name: docs changes uses: ./.github/workflows/get_docs_changes.yml if: ${{ !github.event.pull_request.head.repo.fork }} run_dbt: - name: Tests dbt runner + name: test needs: get_docs_changes if: needs.get_docs_changes.outputs.changes_outside_docs == 'true' defaults: diff --git a/.github/workflows/test_destination_athena.yml b/.github/workflows/test_destination_athena.yml index 5d5c11d9c6..d6833ce41e 100644 --- a/.github/workflows/test_destination_athena.yml +++ b/.github/workflows/test_destination_athena.yml @@ -1,5 +1,5 @@ -name: destinations | test athena +name: dest | athena on: pull_request: @@ -24,12 +24,13 @@ env: jobs: get_docs_changes: + name: docs changes uses: ./.github/workflows/get_docs_changes.yml # Tests that require credentials do not run in forks if: ${{ !github.event.pull_request.head.repo.fork }} run_loader: - name: test destination athena + name: test needs: get_docs_changes if: needs.get_docs_changes.outputs.changes_outside_docs == 'true' defaults: diff --git a/.github/workflows/test_destination_athena_iceberg.yml b/.github/workflows/test_destination_athena_iceberg.yml index 1a428f45fd..8a66e12e63 100644 --- a/.github/workflows/test_destination_athena_iceberg.yml +++ b/.github/workflows/test_destination_athena_iceberg.yml @@ -1,5 +1,5 @@ -name: destinations | test athena iceberg +name: dest | athena iceberg on: pull_request: @@ -24,12 +24,13 @@ env: jobs: get_docs_changes: + name: docs changes uses: ./.github/workflows/get_docs_changes.yml # Tests that require credentials do not run in forks if: ${{ !github.event.pull_request.head.repo.fork }} run_loader: - name: test destination athena iceberg + name: test needs: get_docs_changes if: needs.get_docs_changes.outputs.changes_outside_docs == 'true' defaults: diff --git a/.github/workflows/test_destination_bigquery.yml b/.github/workflows/test_destination_bigquery.yml index 53f9018d38..19124f16b7 100644 --- a/.github/workflows/test_destination_bigquery.yml +++ b/.github/workflows/test_destination_bigquery.yml @@ -1,5 +1,5 @@ -name: destinations | test bigquery +name: dest | bigquery on: pull_request: @@ -23,11 +23,12 @@ env: jobs: get_docs_changes: + name: docs changes uses: ./.github/workflows/get_docs_changes.yml if: ${{ !github.event.pull_request.head.repo.fork }} run_loader: - name: Tests BigQuery loader + name: test needs: get_docs_changes if: needs.get_docs_changes.outputs.changes_outside_docs == 'true' defaults: diff --git a/.github/workflows/test_destination_databricks.yml b/.github/workflows/test_destination_databricks.yml index 6e1173e3d8..279ee72b1d 100644 --- a/.github/workflows/test_destination_databricks.yml +++ b/.github/workflows/test_destination_databricks.yml @@ -1,5 +1,5 @@ -name: destinations | test databricks +name: dest | databricks on: pull_request: @@ -23,11 +23,12 @@ env: jobs: get_docs_changes: + name: docs changes uses: ./.github/workflows/get_docs_changes.yml if: ${{ !github.event.pull_request.head.repo.fork }} run_loader: - name: Tests Databricks loader + name: test needs: get_docs_changes if: needs.get_docs_changes.outputs.changes_outside_docs == 'true' defaults: diff --git a/.github/workflows/test_destination_mssql.yml b/.github/workflows/test_destination_mssql.yml index 8b892e158a..07c9767fa8 100644 --- a/.github/workflows/test_destination_mssql.yml +++ b/.github/workflows/test_destination_mssql.yml @@ -1,5 +1,5 @@ -name: destinations | test MS SQL +name: dest | mssql on: pull_request: @@ -24,11 +24,12 @@ env: jobs: get_docs_changes: + name: docs changes uses: ./.github/workflows/get_docs_changes.yml if: ${{ !github.event.pull_request.head.repo.fork }} run_loader: - name: Tests MS SQL loader + name: test needs: get_docs_changes if: needs.get_docs_changes.outputs.changes_outside_docs == 'true' defaults: diff --git a/.github/workflows/test_destination_qdrant.yml b/.github/workflows/test_destination_qdrant.yml index 24fa0d9636..ddbc1f0ac7 100644 --- a/.github/workflows/test_destination_qdrant.yml +++ b/.github/workflows/test_destination_qdrant.yml @@ -1,4 +1,4 @@ -name: destinations | test Qdrant +name: dest | qdrant on: pull_request: @@ -22,11 +22,12 @@ env: jobs: get_docs_changes: + name: docs changes uses: ./.github/workflows/get_docs_changes.yml if: ${{ !github.event.pull_request.head.repo.fork }} run_loader: - name: Tests Qdrant loader + name: test ß needs: get_docs_changes if: needs.get_docs_changes.outputs.changes_outside_docs == 'true' defaults: diff --git a/.github/workflows/test_destination_snowflake.yml b/.github/workflows/test_destination_snowflake.yml index 53e8af468b..756ce17649 100644 --- a/.github/workflows/test_destination_snowflake.yml +++ b/.github/workflows/test_destination_snowflake.yml @@ -1,5 +1,5 @@ -name: destinations | test snowflake +name: dest | snowflake on: pull_request: @@ -23,11 +23,12 @@ env: jobs: get_docs_changes: + name: docs changes uses: ./.github/workflows/get_docs_changes.yml if: ${{ !github.event.pull_request.head.repo.fork }} run_loader: - name: Tests Snowflake loader + name: test needs: get_docs_changes if: needs.get_docs_changes.outputs.changes_outside_docs == 'true' defaults: diff --git a/.github/workflows/test_destination_synapse.yml b/.github/workflows/test_destination_synapse.yml index be818acb39..58d71cb79c 100644 --- a/.github/workflows/test_destination_synapse.yml +++ b/.github/workflows/test_destination_synapse.yml @@ -1,4 +1,4 @@ -name: destinations | test synapse +name: dest | synapse on: pull_request: @@ -22,11 +22,12 @@ env: jobs: get_docs_changes: + name: docs changes uses: ./.github/workflows/get_docs_changes.yml if: ${{ !github.event.pull_request.head.repo.fork }} run_loader: - name: Tests Synapse loader + name: test needs: get_docs_changes if: needs.get_docs_changes.outputs.changes_outside_docs == 'true' defaults: diff --git a/.github/workflows/test_destinations.yml b/.github/workflows/test_destinations.yml index 246325ed4a..0a6ed1226b 100644 --- a/.github/workflows/test_destinations.yml +++ b/.github/workflows/test_destinations.yml @@ -1,5 +1,5 @@ -name: destinations | test redshift, postgres and filesystem buckets +name: dest | redshift, postgres and fs on: pull_request: @@ -30,12 +30,13 @@ env: jobs: get_docs_changes: + name: docs changes uses: ./.github/workflows/get_docs_changes.yml # Tests that require credentials do not run in forks if: ${{ !github.event.pull_request.head.repo.fork }} run_loader: - name: test destinations redshift, postgres and filesystem + name: test needs: get_docs_changes if: needs.get_docs_changes.outputs.changes_outside_docs == 'true' strategy: diff --git a/.github/workflows/test_doc_snippets.yml b/.github/workflows/test_doc_snippets.yml index 5587f7ca31..7ad07e1554 100644 --- a/.github/workflows/test_doc_snippets.yml +++ b/.github/workflows/test_doc_snippets.yml @@ -26,7 +26,7 @@ env: jobs: run_lint: - name: Runs linter and tests on docs snippets + name: lint runs-on: ubuntu-latest steps: diff --git a/.github/workflows/test_local_destinations.yml b/.github/workflows/test_local_destinations.yml index b9f5cbc6b9..47cef3b43e 100644 --- a/.github/workflows/test_local_destinations.yml +++ b/.github/workflows/test_local_destinations.yml @@ -1,7 +1,7 @@ # Tests destinations that can run without credentials. # i.e. local postgres, duckdb, filesystem (with local fs/memory bucket) -name: destinations | test local destinations +name: dest | postgres, duckdb and fs on: pull_request: @@ -25,10 +25,11 @@ env: jobs: get_docs_changes: + name: docs changes uses: ./.github/workflows/get_docs_changes.yml run_loader: - name: test destinations postgres, duckdb and filesystem + name: test needs: get_docs_changes if: needs.get_docs_changes.outputs.changes_outside_docs == 'true' strategy: From 25b5edbef065fbca397b453c0c6bdba95e0dabda Mon Sep 17 00:00:00 2001 From: Dave Date: Thu, 28 Mar 2024 18:14:30 +0100 Subject: [PATCH 05/24] update snippets --- .github/workflows/test_doc_snippets.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/test_doc_snippets.yml b/.github/workflows/test_doc_snippets.yml index 7ad07e1554..a47d3c276b 100644 --- a/.github/workflows/test_doc_snippets.yml +++ b/.github/workflows/test_doc_snippets.yml @@ -1,5 +1,5 @@ -name: docs | lint and test snippets & examples +name: docs | snippets & examples on: pull_request: @@ -26,7 +26,7 @@ env: jobs: run_lint: - name: lint + name: lint and test runs-on: ubuntu-latest steps: From ddf55ad595ed79f6ecbd5ca7b01da37961caa198 Mon Sep 17 00:00:00 2001 From: Dave Date: Thu, 28 Mar 2024 19:18:53 +0100 Subject: [PATCH 06/24] increase time difference value in gdrive tests --- tests/load/filesystem/test_filesystem_common.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/load/filesystem/test_filesystem_common.py b/tests/load/filesystem/test_filesystem_common.py index 4c94766097..919b89a826 100644 --- a/tests/load/filesystem/test_filesystem_common.py +++ b/tests/load/filesystem/test_filesystem_common.py @@ -52,7 +52,7 @@ def check_file_exists(): def check_file_changed(): details = filesystem.info(file_url) assert details["size"] == 11 - assert (MTIME_DISPATCH[config.protocol](details) - now).seconds < 120 + assert (MTIME_DISPATCH[config.protocol](details) - now).seconds < 200 bucket_url = os.environ["DESTINATION__FILESYSTEM__BUCKET_URL"] config = get_config() From b227c60016c1e3712d6bce0fe14724d305f6ca4f Mon Sep 17 00:00:00 2001 From: Dave Date: Tue, 2 Apr 2024 19:38:54 +0200 Subject: [PATCH 07/24] allow workflows to run on forks if so labelled --- .github/workflows/test_dbt_cloud.yml | 2 +- .github/workflows/test_dbt_runner.yml | 2 +- .github/workflows/test_destination_athena.yml | 2 +- .github/workflows/test_destination_athena_iceberg.yml | 2 +- .github/workflows/test_destination_bigquery.yml | 2 +- .github/workflows/test_destination_databricks.yml | 2 +- .github/workflows/test_destination_mssql.yml | 2 +- .github/workflows/test_destination_qdrant.yml | 2 +- .github/workflows/test_destination_snowflake.yml | 2 +- .github/workflows/test_destination_synapse.yml | 2 +- .github/workflows/test_destinations.yml | 2 +- 11 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/test_dbt_cloud.yml b/.github/workflows/test_dbt_cloud.yml index 3c7bbddb94..0f5c169e6e 100644 --- a/.github/workflows/test_dbt_cloud.yml +++ b/.github/workflows/test_dbt_cloud.yml @@ -24,7 +24,7 @@ jobs: get_docs_changes: name: docs changes uses: ./.github/workflows/get_docs_changes.yml - if: ${{ !github.event.pull_request.head.repo.fork }} + if: ${{ !github.event.pull_request.head.repo.fork || contains(github.event.pull_request.labels.*.name, 'ci from fork')}} run_dbt_cloud: name: test diff --git a/.github/workflows/test_dbt_runner.yml b/.github/workflows/test_dbt_runner.yml index 0ed90d9d3d..ae26a5d852 100644 --- a/.github/workflows/test_dbt_runner.yml +++ b/.github/workflows/test_dbt_runner.yml @@ -21,7 +21,7 @@ jobs: get_docs_changes: name: docs changes uses: ./.github/workflows/get_docs_changes.yml - if: ${{ !github.event.pull_request.head.repo.fork }} + if: ${{ !github.event.pull_request.head.repo.fork || contains(github.event.pull_request.labels.*.name, 'ci from fork')}} run_dbt: name: test diff --git a/.github/workflows/test_destination_athena.yml b/.github/workflows/test_destination_athena.yml index d6833ce41e..75048f1e8d 100644 --- a/.github/workflows/test_destination_athena.yml +++ b/.github/workflows/test_destination_athena.yml @@ -27,7 +27,7 @@ jobs: name: docs changes uses: ./.github/workflows/get_docs_changes.yml # Tests that require credentials do not run in forks - if: ${{ !github.event.pull_request.head.repo.fork }} + if: ${{ !github.event.pull_request.head.repo.fork || contains(github.event.pull_request.labels.*.name, 'ci from fork')}} run_loader: name: test diff --git a/.github/workflows/test_destination_athena_iceberg.yml b/.github/workflows/test_destination_athena_iceberg.yml index 8a66e12e63..c41b9290b0 100644 --- a/.github/workflows/test_destination_athena_iceberg.yml +++ b/.github/workflows/test_destination_athena_iceberg.yml @@ -27,7 +27,7 @@ jobs: name: docs changes uses: ./.github/workflows/get_docs_changes.yml # Tests that require credentials do not run in forks - if: ${{ !github.event.pull_request.head.repo.fork }} + if: ${{ !github.event.pull_request.head.repo.fork || contains(github.event.pull_request.labels.*.name, 'ci from fork')}} run_loader: name: test diff --git a/.github/workflows/test_destination_bigquery.yml b/.github/workflows/test_destination_bigquery.yml index 19124f16b7..12d460b7aa 100644 --- a/.github/workflows/test_destination_bigquery.yml +++ b/.github/workflows/test_destination_bigquery.yml @@ -25,7 +25,7 @@ jobs: get_docs_changes: name: docs changes uses: ./.github/workflows/get_docs_changes.yml - if: ${{ !github.event.pull_request.head.repo.fork }} + if: ${{ !github.event.pull_request.head.repo.fork || contains(github.event.pull_request.labels.*.name, 'ci from fork')}} run_loader: name: test diff --git a/.github/workflows/test_destination_databricks.yml b/.github/workflows/test_destination_databricks.yml index 279ee72b1d..6c788a92ba 100644 --- a/.github/workflows/test_destination_databricks.yml +++ b/.github/workflows/test_destination_databricks.yml @@ -25,7 +25,7 @@ jobs: get_docs_changes: name: docs changes uses: ./.github/workflows/get_docs_changes.yml - if: ${{ !github.event.pull_request.head.repo.fork }} + if: ${{ !github.event.pull_request.head.repo.fork || contains(github.event.pull_request.labels.*.name, 'ci from fork')}} run_loader: name: test diff --git a/.github/workflows/test_destination_mssql.yml b/.github/workflows/test_destination_mssql.yml index 07c9767fa8..5105ab9250 100644 --- a/.github/workflows/test_destination_mssql.yml +++ b/.github/workflows/test_destination_mssql.yml @@ -26,7 +26,7 @@ jobs: get_docs_changes: name: docs changes uses: ./.github/workflows/get_docs_changes.yml - if: ${{ !github.event.pull_request.head.repo.fork }} + if: ${{ !github.event.pull_request.head.repo.fork || contains(github.event.pull_request.labels.*.name, 'ci from fork')}} run_loader: name: test diff --git a/.github/workflows/test_destination_qdrant.yml b/.github/workflows/test_destination_qdrant.yml index ddbc1f0ac7..5fbd7c0c3b 100644 --- a/.github/workflows/test_destination_qdrant.yml +++ b/.github/workflows/test_destination_qdrant.yml @@ -24,7 +24,7 @@ jobs: get_docs_changes: name: docs changes uses: ./.github/workflows/get_docs_changes.yml - if: ${{ !github.event.pull_request.head.repo.fork }} + if: ${{ !github.event.pull_request.head.repo.fork || contains(github.event.pull_request.labels.*.name, 'ci from fork')}} run_loader: name: test ß diff --git a/.github/workflows/test_destination_snowflake.yml b/.github/workflows/test_destination_snowflake.yml index 756ce17649..7668a8449b 100644 --- a/.github/workflows/test_destination_snowflake.yml +++ b/.github/workflows/test_destination_snowflake.yml @@ -25,7 +25,7 @@ jobs: get_docs_changes: name: docs changes uses: ./.github/workflows/get_docs_changes.yml - if: ${{ !github.event.pull_request.head.repo.fork }} + if: ${{ !github.event.pull_request.head.repo.fork || contains(github.event.pull_request.labels.*.name, 'ci from fork')}} run_loader: name: test diff --git a/.github/workflows/test_destination_synapse.yml b/.github/workflows/test_destination_synapse.yml index 58d71cb79c..a1901e4072 100644 --- a/.github/workflows/test_destination_synapse.yml +++ b/.github/workflows/test_destination_synapse.yml @@ -24,7 +24,7 @@ jobs: get_docs_changes: name: docs changes uses: ./.github/workflows/get_docs_changes.yml - if: ${{ !github.event.pull_request.head.repo.fork }} + if: ${{ !github.event.pull_request.head.repo.fork || contains(github.event.pull_request.labels.*.name, 'ci from fork')}} run_loader: name: test diff --git a/.github/workflows/test_destinations.yml b/.github/workflows/test_destinations.yml index 0a6ed1226b..6e15e2da65 100644 --- a/.github/workflows/test_destinations.yml +++ b/.github/workflows/test_destinations.yml @@ -33,7 +33,7 @@ jobs: name: docs changes uses: ./.github/workflows/get_docs_changes.yml # Tests that require credentials do not run in forks - if: ${{ !github.event.pull_request.head.repo.fork }} + if: ${{ !github.event.pull_request.head.repo.fork || contains(github.event.pull_request.labels.*.name, 'ci from fork')}} run_loader: name: test From bd141d8e075212cb0efa40792c3e87dd8d1c9074 Mon Sep 17 00:00:00 2001 From: Dave Date: Tue, 2 Apr 2024 19:39:56 +0200 Subject: [PATCH 08/24] remove secrets from local destinations tests --- .github/workflows/test_local_destinations.yml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/.github/workflows/test_local_destinations.yml b/.github/workflows/test_local_destinations.yml index 47cef3b43e..36443a9a37 100644 --- a/.github/workflows/test_local_destinations.yml +++ b/.github/workflows/test_local_destinations.yml @@ -15,8 +15,6 @@ concurrency: cancel-in-progress: true env: - DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }} - RUNTIME__SENTRY_DSN: https://6f6f7b6f8e0f458a89be4187603b55fe@o1061158.ingest.sentry.io/4504819859914752 RUNTIME__LOG_LEVEL: ERROR RUNTIME__DLTHUB_TELEMETRY_SEGMENT_WRITE_KEY: TLJiyRkGVZGCi2TtjClamXpFcxAA1rSB @@ -88,9 +86,6 @@ jobs: - name: Install dependencies run: poetry install --no-interaction -E postgres -E duckdb -E parquet -E filesystem -E cli -E weaviate --with sentry-sdk --with pipeline - - name: create secrets.toml - run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml - - run: poetry run pytest tests/load && poetry run pytest tests/cli name: Run tests Linux env: From 4adba775c9489c5052364bf935a2957135f64604 Mon Sep 17 00:00:00 2001 From: Dave Date: Tue, 2 Apr 2024 19:47:58 +0200 Subject: [PATCH 09/24] guard secrets leaking on docs tests --- .github/workflows/test_doc_snippets.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/test_doc_snippets.yml b/.github/workflows/test_doc_snippets.yml index a47d3c276b..ebdcbd368f 100644 --- a/.github/workflows/test_doc_snippets.yml +++ b/.github/workflows/test_doc_snippets.yml @@ -23,11 +23,14 @@ env: RUNTIME__SLACK_INCOMING_HOOK: ${{ secrets.RUNTIME__SLACK_INCOMING_HOOK }} # detect if the workflow is executed in a repo fork IS_FORK: ${{ github.event.pull_request.head.repo.fork }} + jobs: run_lint: name: lint and test runs-on: ubuntu-latest + # Do not run on forks, unless allowed, secrets are used here + if: ${{ !github.event.pull_request.head.repo.fork || contains(github.event.pull_request.labels.*.name, 'ci from fork')}} steps: From a0efc88d128c87359ba41ca4cc3103bc4fc6e9b8 Mon Sep 17 00:00:00 2001 From: Dave Date: Tue, 2 Apr 2024 19:48:09 +0200 Subject: [PATCH 10/24] update job output --- .github/workflows/test_destination_athena.yml | 2 +- .github/workflows/test_destination_athena_iceberg.yml | 2 +- .github/workflows/test_destination_bigquery.yml | 2 +- .github/workflows/test_destination_databricks.yml | 2 +- .github/workflows/test_destination_mssql.yml | 2 +- .github/workflows/test_destination_qdrant.yml | 2 +- .github/workflows/test_destination_snowflake.yml | 2 +- .github/workflows/test_destination_synapse.yml | 4 +++- .github/workflows/test_destinations.yml | 2 +- .github/workflows/test_local_destinations.yml | 3 ++- 10 files changed, 13 insertions(+), 10 deletions(-) diff --git a/.github/workflows/test_destination_athena.yml b/.github/workflows/test_destination_athena.yml index 75048f1e8d..47f79a951c 100644 --- a/.github/workflows/test_destination_athena.yml +++ b/.github/workflows/test_destination_athena.yml @@ -72,4 +72,4 @@ jobs: - run: | poetry run pytest tests/load -m "essential" - name: Run tests Linux/MAC \ No newline at end of file + name: Run tests Linux \ No newline at end of file diff --git a/.github/workflows/test_destination_athena_iceberg.yml b/.github/workflows/test_destination_athena_iceberg.yml index c41b9290b0..05dd1ea4ad 100644 --- a/.github/workflows/test_destination_athena_iceberg.yml +++ b/.github/workflows/test_destination_athena_iceberg.yml @@ -72,4 +72,4 @@ jobs: - run: | poetry run pytest tests/load -m "essential" - name: Run tests Linux/MAC \ No newline at end of file + name: Run tests Linux \ No newline at end of file diff --git a/.github/workflows/test_destination_bigquery.yml b/.github/workflows/test_destination_bigquery.yml index 12d460b7aa..5af3890290 100644 --- a/.github/workflows/test_destination_bigquery.yml +++ b/.github/workflows/test_destination_bigquery.yml @@ -70,4 +70,4 @@ jobs: - run: | poetry run pytest tests/load -m "essential" - name: Run tests Linux/MAC \ No newline at end of file + name: Run tests Linux \ No newline at end of file diff --git a/.github/workflows/test_destination_databricks.yml b/.github/workflows/test_destination_databricks.yml index 6c788a92ba..ad102b7566 100644 --- a/.github/workflows/test_destination_databricks.yml +++ b/.github/workflows/test_destination_databricks.yml @@ -68,4 +68,4 @@ jobs: - run: | poetry run pytest tests/load -m "essential" - name: Run tests Linux/MAC \ No newline at end of file + name: Run tests Linux \ No newline at end of file diff --git a/.github/workflows/test_destination_mssql.yml b/.github/workflows/test_destination_mssql.yml index 5105ab9250..dd8147403d 100644 --- a/.github/workflows/test_destination_mssql.yml +++ b/.github/workflows/test_destination_mssql.yml @@ -73,4 +73,4 @@ jobs: - run: | poetry run pytest tests/load -m "essential" - name: Run tests Linux/MAC \ No newline at end of file + name: Run tests Linux \ No newline at end of file diff --git a/.github/workflows/test_destination_qdrant.yml b/.github/workflows/test_destination_qdrant.yml index 5fbd7c0c3b..c9d47a9da7 100644 --- a/.github/workflows/test_destination_qdrant.yml +++ b/.github/workflows/test_destination_qdrant.yml @@ -66,4 +66,4 @@ jobs: - run: | poetry run pytest tests/load -m "essential" - name: Run tests Linux/MAC \ No newline at end of file + name: Run tests Linux \ No newline at end of file diff --git a/.github/workflows/test_destination_snowflake.yml b/.github/workflows/test_destination_snowflake.yml index 7668a8449b..55badf7860 100644 --- a/.github/workflows/test_destination_snowflake.yml +++ b/.github/workflows/test_destination_snowflake.yml @@ -68,4 +68,4 @@ jobs: - run: | poetry run pytest tests/load -m "essential" - name: Run tests Linux/MAC \ No newline at end of file + name: Run tests Linux \ No newline at end of file diff --git a/.github/workflows/test_destination_synapse.yml b/.github/workflows/test_destination_synapse.yml index a1901e4072..6943a27c85 100644 --- a/.github/workflows/test_destination_synapse.yml +++ b/.github/workflows/test_destination_synapse.yml @@ -71,4 +71,6 @@ jobs: - run: | poetry run pytest tests/load -m "essential" - name: Run tests Linux/MAC + name: Run tests Linux + + diff --git a/.github/workflows/test_destinations.yml b/.github/workflows/test_destinations.yml index 6e15e2da65..b0a3d2efa1 100644 --- a/.github/workflows/test_destinations.yml +++ b/.github/workflows/test_destinations.yml @@ -80,5 +80,5 @@ jobs: - run: | poetry run pytest tests/load -m "essential" - name: Run tests Linux/MAC + name: Run tests Linux diff --git a/.github/workflows/test_local_destinations.yml b/.github/workflows/test_local_destinations.yml index 36443a9a37..fe907dafb9 100644 --- a/.github/workflows/test_local_destinations.yml +++ b/.github/workflows/test_local_destinations.yml @@ -85,7 +85,8 @@ jobs: - name: Install dependencies run: poetry install --no-interaction -E postgres -E duckdb -E parquet -E filesystem -E cli -E weaviate --with sentry-sdk --with pipeline - + + # always run full suite, also on branches - run: poetry run pytest tests/load && poetry run pytest tests/cli name: Run tests Linux env: From bea80295594465bc8264bf2458d865f7cb032118 Mon Sep 17 00:00:00 2001 From: Dave Date: Tue, 2 Apr 2024 19:49:53 +0200 Subject: [PATCH 11/24] test label on bigquery --- .github/workflows/test_destination_bigquery.yml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/.github/workflows/test_destination_bigquery.yml b/.github/workflows/test_destination_bigquery.yml index 5af3890290..1f5b08aa54 100644 --- a/.github/workflows/test_destination_bigquery.yml +++ b/.github/workflows/test_destination_bigquery.yml @@ -70,4 +70,10 @@ jobs: - run: | poetry run pytest tests/load -m "essential" - name: Run tests Linux \ No newline at end of file + name: Run essential tests Linux + if: ${{ !contains(github.event.pull_request.labels.*.name, 'ci full')}} + + - run: | + poetry run pytest tests/load + name: Run all tests Linux + if: ${{ contains(github.event.pull_request.labels.*.name, 'ci full')}} \ No newline at end of file From 1d6777be5c00b82364f5228093017dbafae71711 Mon Sep 17 00:00:00 2001 From: Dave Date: Tue, 2 Apr 2024 19:58:30 +0200 Subject: [PATCH 12/24] force rerun --- .github/workflows/test_destination_bigquery.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test_destination_bigquery.yml b/.github/workflows/test_destination_bigquery.yml index 1f5b08aa54..0b4178c091 100644 --- a/.github/workflows/test_destination_bigquery.yml +++ b/.github/workflows/test_destination_bigquery.yml @@ -68,6 +68,7 @@ jobs: - name: create secrets.toml run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml + - run: | poetry run pytest tests/load -m "essential" name: Run essential tests Linux From 2668e0a7b1e258a3c4a4f3e2d9c99bfefff0afee Mon Sep 17 00:00:00 2001 From: Dave Date: Tue, 2 Apr 2024 20:43:09 +0200 Subject: [PATCH 13/24] put secrets back in local destinations --- .github/workflows/test_local_destinations.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/test_local_destinations.yml b/.github/workflows/test_local_destinations.yml index fe907dafb9..653b4dbd75 100644 --- a/.github/workflows/test_local_destinations.yml +++ b/.github/workflows/test_local_destinations.yml @@ -15,6 +15,8 @@ concurrency: cancel-in-progress: true env: + DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }} + RUNTIME__SENTRY_DSN: https://6f6f7b6f8e0f458a89be4187603b55fe@o1061158.ingest.sentry.io/4504819859914752 RUNTIME__LOG_LEVEL: ERROR RUNTIME__DLTHUB_TELEMETRY_SEGMENT_WRITE_KEY: TLJiyRkGVZGCi2TtjClamXpFcxAA1rSB @@ -86,6 +88,9 @@ jobs: - name: Install dependencies run: poetry install --no-interaction -E postgres -E duckdb -E parquet -E filesystem -E cli -E weaviate --with sentry-sdk --with pipeline + - name: create secrets.toml + run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml + # always run full suite, also on branches - run: poetry run pytest tests/load && poetry run pytest tests/cli name: Run tests Linux From 48dc15acdb7dc6ba08b82f093f93bcb30495f84f Mon Sep 17 00:00:00 2001 From: Dave Date: Tue, 2 Apr 2024 20:45:41 +0200 Subject: [PATCH 14/24] add new run setup to all destinations --- .github/workflows/test_destination_athena.yml | 8 +++++++- .github/workflows/test_destination_athena_iceberg.yml | 10 +++++++++- .github/workflows/test_destination_bigquery.yml | 3 +-- .github/workflows/test_destination_databricks.yml | 8 +++++++- .github/workflows/test_destination_mssql.yml | 8 +++++++- .github/workflows/test_destination_qdrant.yml | 8 +++++++- .github/workflows/test_destination_snowflake.yml | 9 ++++++++- .github/workflows/test_destination_synapse.yml | 9 +++++++-- 8 files changed, 53 insertions(+), 10 deletions(-) diff --git a/.github/workflows/test_destination_athena.yml b/.github/workflows/test_destination_athena.yml index 47f79a951c..1876f872f7 100644 --- a/.github/workflows/test_destination_athena.yml +++ b/.github/workflows/test_destination_athena.yml @@ -72,4 +72,10 @@ jobs: - run: | poetry run pytest tests/load -m "essential" - name: Run tests Linux \ No newline at end of file + name: Run essential tests Linux + if: ${{ !contains(github.event.pull_request.labels.*.name, 'ci full')}} + + - run: | + poetry run pytest tests/load + name: Run all tests Linux + if: ${{ contains(github.event.pull_request.labels.*.name, 'ci full')}} diff --git a/.github/workflows/test_destination_athena_iceberg.yml b/.github/workflows/test_destination_athena_iceberg.yml index 05dd1ea4ad..6f3605f14c 100644 --- a/.github/workflows/test_destination_athena_iceberg.yml +++ b/.github/workflows/test_destination_athena_iceberg.yml @@ -72,4 +72,12 @@ jobs: - run: | poetry run pytest tests/load -m "essential" - name: Run tests Linux \ No newline at end of file + name: Run essential tests Linux + if: ${{ !contains(github.event.pull_request.labels.*.name, 'ci full')}} + + - run: | + poetry run pytest tests/load + name: Run all tests Linux + if: ${{ contains(github.event.pull_request.labels.*.name, 'ci full')}} + + \ No newline at end of file diff --git a/.github/workflows/test_destination_bigquery.yml b/.github/workflows/test_destination_bigquery.yml index 0b4178c091..41ba2ed8ea 100644 --- a/.github/workflows/test_destination_bigquery.yml +++ b/.github/workflows/test_destination_bigquery.yml @@ -67,8 +67,7 @@ jobs: - name: create secrets.toml run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml - - + - run: | poetry run pytest tests/load -m "essential" name: Run essential tests Linux diff --git a/.github/workflows/test_destination_databricks.yml b/.github/workflows/test_destination_databricks.yml index ad102b7566..5f893994dc 100644 --- a/.github/workflows/test_destination_databricks.yml +++ b/.github/workflows/test_destination_databricks.yml @@ -68,4 +68,10 @@ jobs: - run: | poetry run pytest tests/load -m "essential" - name: Run tests Linux \ No newline at end of file + name: Run essential tests Linux + if: ${{ !contains(github.event.pull_request.labels.*.name, 'ci full')}} + + - run: | + poetry run pytest tests/load + name: Run all tests Linux + if: ${{ contains(github.event.pull_request.labels.*.name, 'ci full')}} diff --git a/.github/workflows/test_destination_mssql.yml b/.github/workflows/test_destination_mssql.yml index dd8147403d..0da7c0ba64 100644 --- a/.github/workflows/test_destination_mssql.yml +++ b/.github/workflows/test_destination_mssql.yml @@ -73,4 +73,10 @@ jobs: - run: | poetry run pytest tests/load -m "essential" - name: Run tests Linux \ No newline at end of file + name: Run essential tests Linux + if: ${{ !contains(github.event.pull_request.labels.*.name, 'ci full')}} + + - run: | + poetry run pytest tests/load + name: Run all tests Linux + if: ${{ contains(github.event.pull_request.labels.*.name, 'ci full')}} diff --git a/.github/workflows/test_destination_qdrant.yml b/.github/workflows/test_destination_qdrant.yml index c9d47a9da7..bd06ae6cf8 100644 --- a/.github/workflows/test_destination_qdrant.yml +++ b/.github/workflows/test_destination_qdrant.yml @@ -66,4 +66,10 @@ jobs: - run: | poetry run pytest tests/load -m "essential" - name: Run tests Linux \ No newline at end of file + name: Run essential tests Linux + if: ${{ !contains(github.event.pull_request.labels.*.name, 'ci full')}} + + - run: | + poetry run pytest tests/load + name: Run all tests Linux + if: ${{ contains(github.event.pull_request.labels.*.name, 'ci full')}} diff --git a/.github/workflows/test_destination_snowflake.yml b/.github/workflows/test_destination_snowflake.yml index 55badf7860..fb31bdf76f 100644 --- a/.github/workflows/test_destination_snowflake.yml +++ b/.github/workflows/test_destination_snowflake.yml @@ -68,4 +68,11 @@ jobs: - run: | poetry run pytest tests/load -m "essential" - name: Run tests Linux \ No newline at end of file + name: Run essential tests Linux + if: ${{ !contains(github.event.pull_request.labels.*.name, 'ci full')}} + + - run: | + poetry run pytest tests/load + name: Run all tests Linux + if: ${{ contains(github.event.pull_request.labels.*.name, 'ci full')}} + diff --git a/.github/workflows/test_destination_synapse.yml b/.github/workflows/test_destination_synapse.yml index 6943a27c85..8948d4e8fc 100644 --- a/.github/workflows/test_destination_synapse.yml +++ b/.github/workflows/test_destination_synapse.yml @@ -71,6 +71,11 @@ jobs: - run: | poetry run pytest tests/load -m "essential" - name: Run tests Linux + name: Run essential tests Linux + if: ${{ !contains(github.event.pull_request.labels.*.name, 'ci full')}} + + - run: | + poetry run pytest tests/load + name: Run all tests Linux + if: ${{ contains(github.event.pull_request.labels.*.name, 'ci full')}} - From d59a77861fffd846142ba2e39d282d5febc8c3d9 Mon Sep 17 00:00:00 2001 From: Dave Date: Tue, 2 Apr 2024 20:45:51 +0200 Subject: [PATCH 15/24] mark a couple of tests as essential --- tests/load/pipeline/test_merge_disposition.py | 2 ++ tests/load/pipeline/test_replace_disposition.py | 3 ++- tests/load/test_sql_client.py | 6 ++++++ 3 files changed, 10 insertions(+), 1 deletion(-) diff --git a/tests/load/pipeline/test_merge_disposition.py b/tests/load/pipeline/test_merge_disposition.py index 19ee9a34c8..fc6b82d1f4 100644 --- a/tests/load/pipeline/test_merge_disposition.py +++ b/tests/load/pipeline/test_merge_disposition.py @@ -26,6 +26,7 @@ # ACTIVE_DESTINATIONS += ["motherduck"] +@pytest.mark.essential @pytest.mark.parametrize( "destination_config", destinations_configs(default_sql_configs=True), ids=lambda x: x.name ) @@ -153,6 +154,7 @@ def load_issues(): return load_issues +@pytest.mark.essential @pytest.mark.parametrize( "destination_config", destinations_configs(default_sql_configs=True), ids=lambda x: x.name ) diff --git a/tests/load/pipeline/test_replace_disposition.py b/tests/load/pipeline/test_replace_disposition.py index a69d4440dc..0bf55c28cf 100644 --- a/tests/load/pipeline/test_replace_disposition.py +++ b/tests/load/pipeline/test_replace_disposition.py @@ -16,6 +16,7 @@ ) +@pytest.mark.essential @pytest.mark.parametrize( "destination_config", destinations_configs( @@ -245,7 +246,7 @@ def load_items_none(): "_dlt_version": increase_loads(dlt_versions), # new table name -> new schema } - +@pytest.mark.essential @pytest.mark.parametrize( "destination_config", destinations_configs( diff --git a/tests/load/test_sql_client.py b/tests/load/test_sql_client.py index d82925a7d3..67372012ff 100644 --- a/tests/load/test_sql_client.py +++ b/tests/load/test_sql_client.py @@ -36,6 +36,7 @@ def client(request) -> Iterator[SqlJobClientBase]: yield from yield_client_with_storage(request.param.destination) +@pytest.mark.essential @pytest.mark.parametrize( "client", destinations_configs(default_sql_configs=True, exclude=["mssql", "synapse"]), @@ -139,6 +140,7 @@ def test_malformed_execute_parameters(client: SqlJobClientBase) -> None: assert client.sql_client.is_dbapi_exception(term_ex.value.dbapi_exception) +@pytest.mark.essential @pytest.mark.parametrize( "client", destinations_configs(default_sql_configs=True), indirect=True, ids=lambda x: x.name ) @@ -186,6 +188,7 @@ def test_execute_sql(client: SqlJobClientBase) -> None: assert len(rows) == 0 +@pytest.mark.essential @pytest.mark.parametrize( "client", destinations_configs(default_sql_configs=True), indirect=True, ids=lambda x: x.name ) @@ -204,6 +207,7 @@ def test_execute_ddl(client: SqlJobClientBase) -> None: assert rows[0][0] == Decimal("1.0") +@pytest.mark.essential @pytest.mark.parametrize( "client", destinations_configs(default_sql_configs=True), indirect=True, ids=lambda x: x.name ) @@ -246,6 +250,7 @@ def test_execute_query(client: SqlJobClientBase) -> None: assert len(rows) == 0 +@pytest.mark.essential @pytest.mark.parametrize( "client", destinations_configs(default_sql_configs=True), indirect=True, ids=lambda x: x.name ) @@ -297,6 +302,7 @@ def test_execute_df(client: SqlJobClientBase) -> None: assert df_3 is None +@pytest.mark.essential @pytest.mark.parametrize( "client", destinations_configs(default_sql_configs=True), indirect=True, ids=lambda x: x.name ) From 6b51a3fb586d090794d48ea7cb4a76e911d9a544 Mon Sep 17 00:00:00 2001 From: Dave Date: Tue, 2 Apr 2024 20:50:12 +0200 Subject: [PATCH 16/24] add nightly schedule for full destination test run --- .github/workflows/test_destination_athena.yml | 8 ++++++-- .../workflows/test_destination_athena_iceberg.yml | 8 ++++++-- .github/workflows/test_destination_bigquery.yml | 7 +++++-- .github/workflows/test_destination_databricks.yml | 8 ++++++-- .github/workflows/test_destination_mssql.yml | 8 ++++++-- .github/workflows/test_destination_qdrant.yml | 8 ++++++-- .github/workflows/test_destination_snowflake.yml | 8 ++++++-- .github/workflows/test_destination_synapse.yml | 6 ++++-- .github/workflows/test_destinations.yml | 12 +++++++++++- 9 files changed, 56 insertions(+), 17 deletions(-) diff --git a/.github/workflows/test_destination_athena.yml b/.github/workflows/test_destination_athena.yml index 1876f872f7..e2944bd9d0 100644 --- a/.github/workflows/test_destination_athena.yml +++ b/.github/workflows/test_destination_athena.yml @@ -7,6 +7,8 @@ on: - master - devel workflow_dispatch: + schedule: + - cron: '0 2 * * *' concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} @@ -73,9 +75,11 @@ jobs: - run: | poetry run pytest tests/load -m "essential" name: Run essential tests Linux - if: ${{ !contains(github.event.pull_request.labels.*.name, 'ci full')}} + if: ${{ !contains(github.event.pull_request.labels.*.name, 'ci full') && !github.event_name == 'schedule'}} - run: | poetry run pytest tests/load name: Run all tests Linux - if: ${{ contains(github.event.pull_request.labels.*.name, 'ci full')}} + if: ${{ contains(github.event.pull_request.labels.*.name, 'ci full') || github.event_name == 'schedule'}} + + diff --git a/.github/workflows/test_destination_athena_iceberg.yml b/.github/workflows/test_destination_athena_iceberg.yml index 6f3605f14c..4af3bc346a 100644 --- a/.github/workflows/test_destination_athena_iceberg.yml +++ b/.github/workflows/test_destination_athena_iceberg.yml @@ -7,6 +7,8 @@ on: - master - devel workflow_dispatch: + schedule: + - cron: '0 2 * * *' concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} @@ -73,11 +75,13 @@ jobs: - run: | poetry run pytest tests/load -m "essential" name: Run essential tests Linux - if: ${{ !contains(github.event.pull_request.labels.*.name, 'ci full')}} + if: ${{ !contains(github.event.pull_request.labels.*.name, 'ci full') && !github.event_name == 'schedule'}} - run: | poetry run pytest tests/load name: Run all tests Linux - if: ${{ contains(github.event.pull_request.labels.*.name, 'ci full')}} + if: ${{ contains(github.event.pull_request.labels.*.name, 'ci full') || github.event_name == 'schedule'}} + + \ No newline at end of file diff --git a/.github/workflows/test_destination_bigquery.yml b/.github/workflows/test_destination_bigquery.yml index 41ba2ed8ea..5e5523e19e 100644 --- a/.github/workflows/test_destination_bigquery.yml +++ b/.github/workflows/test_destination_bigquery.yml @@ -7,6 +7,8 @@ on: - master - devel workflow_dispatch: + schedule: + - cron: '0 2 * * *' concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} @@ -71,9 +73,10 @@ jobs: - run: | poetry run pytest tests/load -m "essential" name: Run essential tests Linux - if: ${{ !contains(github.event.pull_request.labels.*.name, 'ci full')}} + if: ${{ !contains(github.event.pull_request.labels.*.name, 'ci full') && !github.event_name == 'schedule'}} - run: | poetry run pytest tests/load name: Run all tests Linux - if: ${{ contains(github.event.pull_request.labels.*.name, 'ci full')}} \ No newline at end of file + if: ${{ contains(github.event.pull_request.labels.*.name, 'ci full') || github.event_name == 'schedule'}} + diff --git a/.github/workflows/test_destination_databricks.yml b/.github/workflows/test_destination_databricks.yml index 5f893994dc..68b262482e 100644 --- a/.github/workflows/test_destination_databricks.yml +++ b/.github/workflows/test_destination_databricks.yml @@ -7,6 +7,8 @@ on: - master - devel workflow_dispatch: + schedule: + - cron: '0 2 * * *' concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} @@ -69,9 +71,11 @@ jobs: - run: | poetry run pytest tests/load -m "essential" name: Run essential tests Linux - if: ${{ !contains(github.event.pull_request.labels.*.name, 'ci full')}} + if: ${{ !contains(github.event.pull_request.labels.*.name, 'ci full') && !github.event_name == 'schedule'}} - run: | poetry run pytest tests/load name: Run all tests Linux - if: ${{ contains(github.event.pull_request.labels.*.name, 'ci full')}} + if: ${{ contains(github.event.pull_request.labels.*.name, 'ci full') || github.event_name == 'schedule'}} + + diff --git a/.github/workflows/test_destination_mssql.yml b/.github/workflows/test_destination_mssql.yml index 0da7c0ba64..970b2f90d2 100644 --- a/.github/workflows/test_destination_mssql.yml +++ b/.github/workflows/test_destination_mssql.yml @@ -7,6 +7,8 @@ on: - master - devel workflow_dispatch: + schedule: + - cron: '0 2 * * *' concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} @@ -74,9 +76,11 @@ jobs: - run: | poetry run pytest tests/load -m "essential" name: Run essential tests Linux - if: ${{ !contains(github.event.pull_request.labels.*.name, 'ci full')}} + if: ${{ !contains(github.event.pull_request.labels.*.name, 'ci full') && !github.event_name == 'schedule'}} - run: | poetry run pytest tests/load name: Run all tests Linux - if: ${{ contains(github.event.pull_request.labels.*.name, 'ci full')}} + if: ${{ contains(github.event.pull_request.labels.*.name, 'ci full') || github.event_name == 'schedule'}} + + diff --git a/.github/workflows/test_destination_qdrant.yml b/.github/workflows/test_destination_qdrant.yml index bd06ae6cf8..2d3a022414 100644 --- a/.github/workflows/test_destination_qdrant.yml +++ b/.github/workflows/test_destination_qdrant.yml @@ -6,6 +6,8 @@ on: - master - devel workflow_dispatch: + schedule: + - cron: '0 2 * * *' concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} @@ -67,9 +69,11 @@ jobs: - run: | poetry run pytest tests/load -m "essential" name: Run essential tests Linux - if: ${{ !contains(github.event.pull_request.labels.*.name, 'ci full')}} + if: ${{ !contains(github.event.pull_request.labels.*.name, 'ci full') && !github.event_name == 'schedule'}} - run: | poetry run pytest tests/load name: Run all tests Linux - if: ${{ contains(github.event.pull_request.labels.*.name, 'ci full')}} + if: ${{ contains(github.event.pull_request.labels.*.name, 'ci full') || github.event_name == 'schedule'}} + + diff --git a/.github/workflows/test_destination_snowflake.yml b/.github/workflows/test_destination_snowflake.yml index fb31bdf76f..088ec21db8 100644 --- a/.github/workflows/test_destination_snowflake.yml +++ b/.github/workflows/test_destination_snowflake.yml @@ -7,6 +7,8 @@ on: - master - devel workflow_dispatch: + schedule: + - cron: '0 2 * * *' concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} @@ -69,10 +71,12 @@ jobs: - run: | poetry run pytest tests/load -m "essential" name: Run essential tests Linux - if: ${{ !contains(github.event.pull_request.labels.*.name, 'ci full')}} + if: ${{ !contains(github.event.pull_request.labels.*.name, 'ci full') && !github.event_name == 'schedule'}} - run: | poetry run pytest tests/load name: Run all tests Linux - if: ${{ contains(github.event.pull_request.labels.*.name, 'ci full')}} + if: ${{ contains(github.event.pull_request.labels.*.name, 'ci full') || github.event_name == 'schedule'}} + + diff --git a/.github/workflows/test_destination_synapse.yml b/.github/workflows/test_destination_synapse.yml index 8948d4e8fc..cb2bb2309e 100644 --- a/.github/workflows/test_destination_synapse.yml +++ b/.github/workflows/test_destination_synapse.yml @@ -6,6 +6,8 @@ on: - master - devel workflow_dispatch: + schedule: + - cron: '0 2 * * *' concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} @@ -72,10 +74,10 @@ jobs: - run: | poetry run pytest tests/load -m "essential" name: Run essential tests Linux - if: ${{ !contains(github.event.pull_request.labels.*.name, 'ci full')}} + if: ${{ !contains(github.event.pull_request.labels.*.name, 'ci full') && !github.event_name == 'schedule'}} - run: | poetry run pytest tests/load name: Run all tests Linux - if: ${{ contains(github.event.pull_request.labels.*.name, 'ci full')}} + if: ${{ contains(github.event.pull_request.labels.*.name, 'ci full') || github.event_name == 'schedule'}} diff --git a/.github/workflows/test_destinations.yml b/.github/workflows/test_destinations.yml index b0a3d2efa1..b38715e1d3 100644 --- a/.github/workflows/test_destinations.yml +++ b/.github/workflows/test_destinations.yml @@ -7,6 +7,8 @@ on: - master - devel workflow_dispatch: + schedule: + - cron: '0 2 * * *' concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} @@ -80,5 +82,13 @@ jobs: - run: | poetry run pytest tests/load -m "essential" - name: Run tests Linux + name: Run essential tests Linux + if: ${{ !contains(github.event.pull_request.labels.*.name, 'ci full') && !github.event_name == 'schedule'}} + + - run: | + poetry run pytest tests/load + name: Run all tests Linux + if: ${{ contains(github.event.pull_request.labels.*.name, 'ci full') || github.event_name == 'schedule'}} + + From 9d634cd8642f30fa3cfa6a79054e78ffdd1877be Mon Sep 17 00:00:00 2001 From: Dave Date: Tue, 2 Apr 2024 21:36:09 +0200 Subject: [PATCH 17/24] fix tests condition --- .github/workflows/test_destination_athena.yml | 2 +- .github/workflows/test_destination_athena_iceberg.yml | 2 +- .github/workflows/test_destination_bigquery.yml | 2 +- .github/workflows/test_destination_databricks.yml | 2 +- .github/workflows/test_destination_mssql.yml | 2 +- .github/workflows/test_destination_qdrant.yml | 2 +- .github/workflows/test_destination_snowflake.yml | 2 +- .github/workflows/test_destination_synapse.yml | 2 +- .github/workflows/test_destinations.yml | 2 +- 9 files changed, 9 insertions(+), 9 deletions(-) diff --git a/.github/workflows/test_destination_athena.yml b/.github/workflows/test_destination_athena.yml index e2944bd9d0..2867ff4fd7 100644 --- a/.github/workflows/test_destination_athena.yml +++ b/.github/workflows/test_destination_athena.yml @@ -75,7 +75,7 @@ jobs: - run: | poetry run pytest tests/load -m "essential" name: Run essential tests Linux - if: ${{ !contains(github.event.pull_request.labels.*.name, 'ci full') && !github.event_name == 'schedule'}} + if: ${{ ! (contains(github.event.pull_request.labels.*.name, 'ci full') || !github.event_name == 'schedule')}} - run: | poetry run pytest tests/load diff --git a/.github/workflows/test_destination_athena_iceberg.yml b/.github/workflows/test_destination_athena_iceberg.yml index 4af3bc346a..d3db5603be 100644 --- a/.github/workflows/test_destination_athena_iceberg.yml +++ b/.github/workflows/test_destination_athena_iceberg.yml @@ -75,7 +75,7 @@ jobs: - run: | poetry run pytest tests/load -m "essential" name: Run essential tests Linux - if: ${{ !contains(github.event.pull_request.labels.*.name, 'ci full') && !github.event_name == 'schedule'}} + if: ${{ ! (contains(github.event.pull_request.labels.*.name, 'ci full') || github.event_name == 'schedule')}} - run: | poetry run pytest tests/load diff --git a/.github/workflows/test_destination_bigquery.yml b/.github/workflows/test_destination_bigquery.yml index 5e5523e19e..4cfdd03dda 100644 --- a/.github/workflows/test_destination_bigquery.yml +++ b/.github/workflows/test_destination_bigquery.yml @@ -73,7 +73,7 @@ jobs: - run: | poetry run pytest tests/load -m "essential" name: Run essential tests Linux - if: ${{ !contains(github.event.pull_request.labels.*.name, 'ci full') && !github.event_name == 'schedule'}} + if: ${{ ! (contains(github.event.pull_request.labels.*.name, 'ci full') || github.event_name == 'schedule')}} - run: | poetry run pytest tests/load diff --git a/.github/workflows/test_destination_databricks.yml b/.github/workflows/test_destination_databricks.yml index 68b262482e..dac516b2fd 100644 --- a/.github/workflows/test_destination_databricks.yml +++ b/.github/workflows/test_destination_databricks.yml @@ -71,7 +71,7 @@ jobs: - run: | poetry run pytest tests/load -m "essential" name: Run essential tests Linux - if: ${{ !contains(github.event.pull_request.labels.*.name, 'ci full') && !github.event_name == 'schedule'}} + if: ${{ ! (contains(github.event.pull_request.labels.*.name, 'ci full') || github.event_name == 'schedule')}} - run: | poetry run pytest tests/load diff --git a/.github/workflows/test_destination_mssql.yml b/.github/workflows/test_destination_mssql.yml index 970b2f90d2..874eb64940 100644 --- a/.github/workflows/test_destination_mssql.yml +++ b/.github/workflows/test_destination_mssql.yml @@ -76,7 +76,7 @@ jobs: - run: | poetry run pytest tests/load -m "essential" name: Run essential tests Linux - if: ${{ !contains(github.event.pull_request.labels.*.name, 'ci full') && !github.event_name == 'schedule'}} + if: ${{ ! (contains(github.event.pull_request.labels.*.name, 'ci full') || github.event_name == 'schedule')}} - run: | poetry run pytest tests/load diff --git a/.github/workflows/test_destination_qdrant.yml b/.github/workflows/test_destination_qdrant.yml index 2d3a022414..ef30c94e9d 100644 --- a/.github/workflows/test_destination_qdrant.yml +++ b/.github/workflows/test_destination_qdrant.yml @@ -69,7 +69,7 @@ jobs: - run: | poetry run pytest tests/load -m "essential" name: Run essential tests Linux - if: ${{ !contains(github.event.pull_request.labels.*.name, 'ci full') && !github.event_name == 'schedule'}} + if: ${{ ! (contains(github.event.pull_request.labels.*.name, 'ci full') || github.event_name == 'schedule')}} - run: | poetry run pytest tests/load diff --git a/.github/workflows/test_destination_snowflake.yml b/.github/workflows/test_destination_snowflake.yml index 088ec21db8..cdeb890e73 100644 --- a/.github/workflows/test_destination_snowflake.yml +++ b/.github/workflows/test_destination_snowflake.yml @@ -71,7 +71,7 @@ jobs: - run: | poetry run pytest tests/load -m "essential" name: Run essential tests Linux - if: ${{ !contains(github.event.pull_request.labels.*.name, 'ci full') && !github.event_name == 'schedule'}} + if: ${{ ! (contains(github.event.pull_request.labels.*.name, 'ci full') || github.event_name == 'schedule')}} - run: | poetry run pytest tests/load diff --git a/.github/workflows/test_destination_synapse.yml b/.github/workflows/test_destination_synapse.yml index cb2bb2309e..d5f2df0a9f 100644 --- a/.github/workflows/test_destination_synapse.yml +++ b/.github/workflows/test_destination_synapse.yml @@ -74,7 +74,7 @@ jobs: - run: | poetry run pytest tests/load -m "essential" name: Run essential tests Linux - if: ${{ !contains(github.event.pull_request.labels.*.name, 'ci full') && !github.event_name == 'schedule'}} + if: ${{ ! (contains(github.event.pull_request.labels.*.name, 'ci full') || github.event_name == 'schedule')}} - run: | poetry run pytest tests/load diff --git a/.github/workflows/test_destinations.yml b/.github/workflows/test_destinations.yml index b38715e1d3..81422092f9 100644 --- a/.github/workflows/test_destinations.yml +++ b/.github/workflows/test_destinations.yml @@ -83,7 +83,7 @@ jobs: - run: | poetry run pytest tests/load -m "essential" name: Run essential tests Linux - if: ${{ !contains(github.event.pull_request.labels.*.name, 'ci full') && !github.event_name == 'schedule'}} + if: ${{ ! (contains(github.event.pull_request.labels.*.name, 'ci full') || github.event_name == 'schedule')}} - run: | poetry run pytest tests/load From cd9415b824beca43e155ce5a9a0ca0b29a8c789b Mon Sep 17 00:00:00 2001 From: Dave Date: Wed, 3 Apr 2024 10:53:38 +0200 Subject: [PATCH 18/24] fix type --- .github/workflows/test_destination_qdrant.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_destination_qdrant.yml b/.github/workflows/test_destination_qdrant.yml index ef30c94e9d..9ab32cfd57 100644 --- a/.github/workflows/test_destination_qdrant.yml +++ b/.github/workflows/test_destination_qdrant.yml @@ -29,7 +29,7 @@ jobs: if: ${{ !github.event.pull_request.head.repo.fork || contains(github.event.pull_request.labels.*.name, 'ci from fork')}} run_loader: - name: test ß + name: test needs: get_docs_changes if: needs.get_docs_changes.outputs.changes_outside_docs == 'true' defaults: From 9b08e8ca8adcd4b00d77159922193bffcdf9fac9 Mon Sep 17 00:00:00 2001 From: Dave Date: Wed, 3 Apr 2024 12:00:42 +0200 Subject: [PATCH 19/24] add a bunch of more essential markers --- tests/load/athena_iceberg/test_athena_iceberg.py | 2 ++ tests/load/bigquery/test_bigquery_client.py | 3 +++ tests/load/bigquery/test_bigquery_table_builder.py | 3 +++ tests/load/databricks/test_databricks_configuration.py | 3 +++ tests/load/duckdb/test_duckdb_client.py | 3 +++ tests/load/duckdb/test_duckdb_table_builder.py | 3 +++ tests/load/duckdb/test_motherduck_client.py | 3 +++ tests/load/filesystem/test_aws_credentials.py | 3 +++ tests/load/filesystem/test_azure_credentials.py | 3 +++ tests/load/filesystem/test_filesystem_client.py | 3 +++ tests/load/filesystem/test_filesystem_common.py | 4 ++++ tests/load/mssql/test_mssql_credentials.py | 3 +++ tests/load/mssql/test_mssql_table_builder.py | 3 +++ tests/load/pipeline/test_arrow_loading.py | 3 +++ tests/load/pipeline/test_athena.py | 3 +++ tests/load/pipeline/test_bigquery.py | 3 +++ tests/load/pipeline/test_pipelines.py | 3 +++ tests/load/pipeline/test_redshift.py | 3 +++ tests/load/pipeline/test_replace_disposition.py | 1 + tests/load/postgres/test_postgres_client.py | 3 +++ tests/load/postgres/test_postgres_table_builder.py | 3 +++ tests/load/qdrant/test_pipeline.py | 3 +++ tests/load/redshift/test_redshift_client.py | 3 +++ tests/load/redshift/test_redshift_table_builder.py | 3 +++ tests/load/snowflake/test_snowflake_configuration.py | 3 +++ tests/load/snowflake/test_snowflake_table_builder.py | 3 +++ tests/load/synapse/test_synapse_configuration.py | 3 +++ tests/load/synapse/test_synapse_table_builder.py | 3 +++ tests/load/synapse/test_synapse_table_indexing.py | 2 ++ tests/load/test_insert_job_client.py | 3 +++ tests/load/test_job_client.py | 3 +++ tests/load/weaviate/test_naming.py | 3 +++ tests/load/weaviate/test_pipeline.py | 3 +++ tests/load/weaviate/test_weaviate_client.py | 3 +++ 34 files changed, 99 insertions(+) diff --git a/tests/load/athena_iceberg/test_athena_iceberg.py b/tests/load/athena_iceberg/test_athena_iceberg.py index 6804b98427..0b8ca9c6ff 100644 --- a/tests/load/athena_iceberg/test_athena_iceberg.py +++ b/tests/load/athena_iceberg/test_athena_iceberg.py @@ -15,6 +15,8 @@ from tests.utils import skip_if_not_active from dlt.destinations.exceptions import DatabaseTerminalException +# mark all tests as essential, do not remove +pytestmark = pytest.mark.essential skip_if_not_active("athena") diff --git a/tests/load/bigquery/test_bigquery_client.py b/tests/load/bigquery/test_bigquery_client.py index a97b612ad0..b16790b07d 100644 --- a/tests/load/bigquery/test_bigquery_client.py +++ b/tests/load/bigquery/test_bigquery_client.py @@ -32,6 +32,9 @@ cm_yield_client_with_storage, ) +# mark all tests as essential, do not remove +pytestmark = pytest.mark.essential + @pytest.fixture(scope="module") def client() -> Iterator[BigQueryClient]: diff --git a/tests/load/bigquery/test_bigquery_table_builder.py b/tests/load/bigquery/test_bigquery_table_builder.py index fd58a6e033..dee7934216 100644 --- a/tests/load/bigquery/test_bigquery_table_builder.py +++ b/tests/load/bigquery/test_bigquery_table_builder.py @@ -33,6 +33,9 @@ ) from tests.load.utils import TABLE_UPDATE, sequence_generator, empty_schema +# mark all tests as essential, do not remove +pytestmark = pytest.mark.essential + def test_configuration() -> None: os.environ["MYBG__CREDENTIALS__CLIENT_EMAIL"] = "1234" diff --git a/tests/load/databricks/test_databricks_configuration.py b/tests/load/databricks/test_databricks_configuration.py index 8d30d05e42..cc353f5894 100644 --- a/tests/load/databricks/test_databricks_configuration.py +++ b/tests/load/databricks/test_databricks_configuration.py @@ -8,6 +8,9 @@ from dlt.common.configuration import resolve_configuration from tests.utils import preserve_environ +# mark all tests as essential, do not remove +pytestmark = pytest.mark.essential + def test_databricks_credentials_to_connector_params(): os.environ["CREDENTIALS__SERVER_HOSTNAME"] = "my-databricks.example.com" diff --git a/tests/load/duckdb/test_duckdb_client.py b/tests/load/duckdb/test_duckdb_client.py index 3deed7a77d..6cfb77d613 100644 --- a/tests/load/duckdb/test_duckdb_client.py +++ b/tests/load/duckdb/test_duckdb_client.py @@ -17,6 +17,9 @@ from tests.load.pipeline.utils import drop_pipeline, assert_table from tests.utils import patch_home_dir, autouse_test_storage, preserve_environ, TEST_STORAGE_ROOT +# mark all tests as essential, do not remove +pytestmark = pytest.mark.essential + @pytest.fixture(autouse=True) def delete_default_duckdb_credentials() -> Iterator[None]: diff --git a/tests/load/duckdb/test_duckdb_table_builder.py b/tests/load/duckdb/test_duckdb_table_builder.py index 542b18993c..d5e8ab59be 100644 --- a/tests/load/duckdb/test_duckdb_table_builder.py +++ b/tests/load/duckdb/test_duckdb_table_builder.py @@ -10,6 +10,9 @@ from tests.load.utils import TABLE_UPDATE, empty_schema +# mark all tests as essential, do not remove +pytestmark = pytest.mark.essential + @pytest.fixture def client(empty_schema: Schema) -> DuckDbClient: diff --git a/tests/load/duckdb/test_motherduck_client.py b/tests/load/duckdb/test_motherduck_client.py index ba60e0de6d..c1444eb87e 100644 --- a/tests/load/duckdb/test_motherduck_client.py +++ b/tests/load/duckdb/test_motherduck_client.py @@ -10,6 +10,9 @@ from tests.utils import patch_home_dir, preserve_environ, skip_if_not_active +# mark all tests as essential, do not remove +pytestmark = pytest.mark.essential + skip_if_not_active("motherduck") diff --git a/tests/load/filesystem/test_aws_credentials.py b/tests/load/filesystem/test_aws_credentials.py index 62c2e3cd85..1a41144744 100644 --- a/tests/load/filesystem/test_aws_credentials.py +++ b/tests/load/filesystem/test_aws_credentials.py @@ -10,6 +10,9 @@ from tests.load.utils import ALL_FILESYSTEM_DRIVERS from tests.utils import preserve_environ, autouse_test_storage +# mark all tests as essential, do not remove +pytestmark = pytest.mark.essential + if "s3" not in ALL_FILESYSTEM_DRIVERS: pytest.skip("s3 filesystem driver not configured", allow_module_level=True) diff --git a/tests/load/filesystem/test_azure_credentials.py b/tests/load/filesystem/test_azure_credentials.py index 093cd6dd19..467ba55a4f 100644 --- a/tests/load/filesystem/test_azure_credentials.py +++ b/tests/load/filesystem/test_azure_credentials.py @@ -11,6 +11,9 @@ from tests.common.configuration.utils import environment from tests.utils import preserve_environ, autouse_test_storage +# mark all tests as essential, do not remove +pytestmark = pytest.mark.essential + if "az" not in ALL_FILESYSTEM_DRIVERS: pytest.skip("az filesystem driver not configured", allow_module_level=True) diff --git a/tests/load/filesystem/test_filesystem_client.py b/tests/load/filesystem/test_filesystem_client.py index 9948e26882..5d6dbe33ef 100644 --- a/tests/load/filesystem/test_filesystem_client.py +++ b/tests/load/filesystem/test_filesystem_client.py @@ -15,6 +15,9 @@ from tests.utils import clean_test_storage, init_test_logging from tests.utils import preserve_environ, autouse_test_storage +# mark all tests as essential, do not remove +pytestmark = pytest.mark.essential + @pytest.fixture(autouse=True) def storage() -> FileStorage: diff --git a/tests/load/filesystem/test_filesystem_common.py b/tests/load/filesystem/test_filesystem_common.py index 919b89a826..193ff0921b 100644 --- a/tests/load/filesystem/test_filesystem_common.py +++ b/tests/load/filesystem/test_filesystem_common.py @@ -20,6 +20,10 @@ from tests.common.configuration.utils import environment +# mark all tests as essential, do not remove +pytestmark = pytest.mark.essential + + @with_config(spec=FilesystemConfiguration, sections=("destination", "filesystem")) def get_config(config: FilesystemConfiguration = None) -> FilesystemConfiguration: return config diff --git a/tests/load/mssql/test_mssql_credentials.py b/tests/load/mssql/test_mssql_credentials.py index 0e38791f22..7d49196531 100644 --- a/tests/load/mssql/test_mssql_credentials.py +++ b/tests/load/mssql/test_mssql_credentials.py @@ -6,6 +6,9 @@ from dlt.destinations.impl.mssql.configuration import MsSqlCredentials +# mark all tests as essential, do not remove +pytestmark = pytest.mark.essential + def test_mssql_credentials_defaults() -> None: creds = MsSqlCredentials() diff --git a/tests/load/mssql/test_mssql_table_builder.py b/tests/load/mssql/test_mssql_table_builder.py index 1b4a77a2ab..f7a87c14ee 100644 --- a/tests/load/mssql/test_mssql_table_builder.py +++ b/tests/load/mssql/test_mssql_table_builder.py @@ -11,6 +11,9 @@ from tests.load.utils import TABLE_UPDATE, empty_schema +# mark all tests as essential, do not remove +pytestmark = pytest.mark.essential + @pytest.fixture def client(empty_schema: Schema) -> MsSqlClient: diff --git a/tests/load/pipeline/test_arrow_loading.py b/tests/load/pipeline/test_arrow_loading.py index 59cd90c535..6ec09c2ccd 100644 --- a/tests/load/pipeline/test_arrow_loading.py +++ b/tests/load/pipeline/test_arrow_loading.py @@ -17,6 +17,9 @@ from tests.utils import preserve_environ from tests.cases import arrow_table_all_data_types, TArrowFormat +# mark all tests as essential, do not remove +pytestmark = pytest.mark.essential + @pytest.mark.parametrize( "destination_config", diff --git a/tests/load/pipeline/test_athena.py b/tests/load/pipeline/test_athena.py index 9c17be318f..845f9b8a27 100644 --- a/tests/load/pipeline/test_athena.py +++ b/tests/load/pipeline/test_athena.py @@ -11,6 +11,9 @@ from tests.load.pipeline.utils import destinations_configs, DestinationTestConfiguration +# mark all tests as essential, do not remove +pytestmark = pytest.mark.essential + @pytest.mark.parametrize( "destination_config", diff --git a/tests/load/pipeline/test_bigquery.py b/tests/load/pipeline/test_bigquery.py index 711d45fb1f..68533a5d43 100644 --- a/tests/load/pipeline/test_bigquery.py +++ b/tests/load/pipeline/test_bigquery.py @@ -6,6 +6,9 @@ from tests.load.pipeline.utils import destinations_configs, DestinationTestConfiguration from tests.load.utils import delete_dataset +# mark all tests as essential, do not remove +pytestmark = pytest.mark.essential + @pytest.mark.parametrize( "destination_config", diff --git a/tests/load/pipeline/test_pipelines.py b/tests/load/pipeline/test_pipelines.py index 017bef2c01..ea03fa91f1 100644 --- a/tests/load/pipeline/test_pipelines.py +++ b/tests/load/pipeline/test_pipelines.py @@ -43,6 +43,9 @@ ) from tests.load.pipeline.utils import destinations_configs, DestinationTestConfiguration +# mark all tests as essential, do not remove +pytestmark = pytest.mark.essential + @pytest.mark.parametrize( "destination_config", diff --git a/tests/load/pipeline/test_redshift.py b/tests/load/pipeline/test_redshift.py index 44234ec64b..40e13f1a4c 100644 --- a/tests/load/pipeline/test_redshift.py +++ b/tests/load/pipeline/test_redshift.py @@ -8,6 +8,9 @@ from tests.cases import table_update_and_row, assert_all_data_types_row from tests.pipeline.utils import assert_load_info +# mark all tests as essential, do not remove +pytestmark = pytest.mark.essential + @pytest.mark.parametrize( "destination_config", diff --git a/tests/load/pipeline/test_replace_disposition.py b/tests/load/pipeline/test_replace_disposition.py index 0bf55c28cf..23e85a13d8 100644 --- a/tests/load/pipeline/test_replace_disposition.py +++ b/tests/load/pipeline/test_replace_disposition.py @@ -246,6 +246,7 @@ def load_items_none(): "_dlt_version": increase_loads(dlt_versions), # new table name -> new schema } + @pytest.mark.essential @pytest.mark.parametrize( "destination_config", diff --git a/tests/load/postgres/test_postgres_client.py b/tests/load/postgres/test_postgres_client.py index 896e449b28..a0fbd85b5b 100644 --- a/tests/load/postgres/test_postgres_client.py +++ b/tests/load/postgres/test_postgres_client.py @@ -15,6 +15,9 @@ from tests.load.utils import expect_load_file, prepare_table, yield_client_with_storage from tests.common.configuration.utils import environment +# mark all tests as essential, do not remove +pytestmark = pytest.mark.essential + @pytest.fixture def file_storage() -> FileStorage: diff --git a/tests/load/postgres/test_postgres_table_builder.py b/tests/load/postgres/test_postgres_table_builder.py index 0ab1343a3b..9362b44e18 100644 --- a/tests/load/postgres/test_postgres_table_builder.py +++ b/tests/load/postgres/test_postgres_table_builder.py @@ -13,6 +13,9 @@ from tests.load.utils import TABLE_UPDATE, empty_schema +# mark all tests as essential, do not remove +pytestmark = pytest.mark.essential + @pytest.fixture def client(empty_schema: Schema) -> PostgresClient: diff --git a/tests/load/qdrant/test_pipeline.py b/tests/load/qdrant/test_pipeline.py index 4c1361dcca..fcc8fcbd71 100644 --- a/tests/load/qdrant/test_pipeline.py +++ b/tests/load/qdrant/test_pipeline.py @@ -11,6 +11,9 @@ from tests.load.qdrant.utils import drop_active_pipeline_data, assert_collection from tests.load.utils import sequence_generator +# mark all tests as essential, do not remove +pytestmark = pytest.mark.essential + @pytest.fixture(autouse=True) def drop_qdrant_data() -> Iterator[None]: diff --git a/tests/load/redshift/test_redshift_client.py b/tests/load/redshift/test_redshift_client.py index f5efc16a47..03bb57c3b4 100644 --- a/tests/load/redshift/test_redshift_client.py +++ b/tests/load/redshift/test_redshift_client.py @@ -19,6 +19,9 @@ from tests.utils import TEST_STORAGE_ROOT, autouse_test_storage, skipifpypy from tests.load.utils import expect_load_file, prepare_table, yield_client_with_storage +# mark all tests as essential, do not remove +pytestmark = pytest.mark.essential + @pytest.fixture def file_storage() -> FileStorage: diff --git a/tests/load/redshift/test_redshift_table_builder.py b/tests/load/redshift/test_redshift_table_builder.py index bc132c7818..2427bc7cfe 100644 --- a/tests/load/redshift/test_redshift_table_builder.py +++ b/tests/load/redshift/test_redshift_table_builder.py @@ -14,6 +14,9 @@ from tests.load.utils import TABLE_UPDATE, empty_schema +# mark all tests as essential, do not remove +pytestmark = pytest.mark.essential + @pytest.fixture def client(empty_schema: Schema) -> RedshiftClient: diff --git a/tests/load/snowflake/test_snowflake_configuration.py b/tests/load/snowflake/test_snowflake_configuration.py index d0ca4de41b..2add5c0017 100644 --- a/tests/load/snowflake/test_snowflake_configuration.py +++ b/tests/load/snowflake/test_snowflake_configuration.py @@ -16,6 +16,9 @@ from tests.common.configuration.utils import environment +# mark all tests as essential, do not remove +pytestmark = pytest.mark.essential + def test_connection_string_with_all_params() -> None: url = "snowflake://user1:pass1@host1/db1?warehouse=warehouse1&role=role1&private_key=cGs%3D&private_key_passphrase=paphr" diff --git a/tests/load/snowflake/test_snowflake_table_builder.py b/tests/load/snowflake/test_snowflake_table_builder.py index 5d7108803e..bdbe888fb5 100644 --- a/tests/load/snowflake/test_snowflake_table_builder.py +++ b/tests/load/snowflake/test_snowflake_table_builder.py @@ -14,6 +14,9 @@ from tests.load.utils import TABLE_UPDATE, empty_schema +# mark all tests as essential, do not remove +pytestmark = pytest.mark.essential + @pytest.fixture def snowflake_client(empty_schema: Schema) -> SnowflakeClient: diff --git a/tests/load/synapse/test_synapse_configuration.py b/tests/load/synapse/test_synapse_configuration.py index 4055cbab38..f366d87d09 100644 --- a/tests/load/synapse/test_synapse_configuration.py +++ b/tests/load/synapse/test_synapse_configuration.py @@ -8,6 +8,9 @@ SynapseCredentials, ) +# mark all tests as essential, do not remove +pytestmark = pytest.mark.essential + def test_synapse_configuration() -> None: # By default, unique indexes should not be created. diff --git a/tests/load/synapse/test_synapse_table_builder.py b/tests/load/synapse/test_synapse_table_builder.py index 8575835820..9ee2ebe202 100644 --- a/tests/load/synapse/test_synapse_table_builder.py +++ b/tests/load/synapse/test_synapse_table_builder.py @@ -19,6 +19,9 @@ TABLE_INDEX_TYPE_TO_SYNAPSE_ATTR, ) +# mark all tests as essential, do not remove +pytestmark = pytest.mark.essential + @pytest.fixture def client(empty_schema: Schema) -> SynapseClient: diff --git a/tests/load/synapse/test_synapse_table_indexing.py b/tests/load/synapse/test_synapse_table_indexing.py index df90933de4..71f419cbca 100644 --- a/tests/load/synapse/test_synapse_table_indexing.py +++ b/tests/load/synapse/test_synapse_table_indexing.py @@ -17,6 +17,8 @@ ) # this import ensures all test data gets removed from tests.load.synapse.utils import get_storage_table_index_type +# mark all tests as essential, do not remove +pytestmark = pytest.mark.essential TABLE_INDEX_TYPE_COLUMN_SCHEMA_PARAM_GRID = [ ("heap", None), diff --git a/tests/load/test_insert_job_client.py b/tests/load/test_insert_job_client.py index 1c79b733e5..826e154607 100644 --- a/tests/load/test_insert_job_client.py +++ b/tests/load/test_insert_job_client.py @@ -18,6 +18,9 @@ from tests.load.utils import expect_load_file, prepare_table, yield_client_with_storage from tests.load.pipeline.utils import destinations_configs, DestinationTestConfiguration +# mark all tests as essential, do not remove +pytestmark = pytest.mark.essential + DEFAULT_SUBSET = ["duckdb", "redshift", "postgres"] diff --git a/tests/load/test_job_client.py b/tests/load/test_job_client.py index 2e23086f81..0a205293ab 100644 --- a/tests/load/test_job_client.py +++ b/tests/load/test_job_client.py @@ -44,6 +44,9 @@ ) from tests.load.pipeline.utils import destinations_configs, DestinationTestConfiguration +# mark all tests as essential, do not remove +pytestmark = pytest.mark.essential + @pytest.fixture def file_storage() -> FileStorage: diff --git a/tests/load/weaviate/test_naming.py b/tests/load/weaviate/test_naming.py index 290879cb67..613582e9f8 100644 --- a/tests/load/weaviate/test_naming.py +++ b/tests/load/weaviate/test_naming.py @@ -5,6 +5,9 @@ from tests.common.utils import load_yml_case +# mark all tests as essential, do not remove +pytestmark = pytest.mark.essential + @dlt.source def small(): diff --git a/tests/load/weaviate/test_pipeline.py b/tests/load/weaviate/test_pipeline.py index dc23644940..25adc29c89 100644 --- a/tests/load/weaviate/test_pipeline.py +++ b/tests/load/weaviate/test_pipeline.py @@ -16,6 +16,9 @@ from .utils import assert_class, drop_active_pipeline_data from tests.load.utils import sequence_generator +# mark all tests as essential, do not remove +pytestmark = pytest.mark.essential + @pytest.fixture(autouse=True) def drop_weaviate_schema() -> Iterator[None]: diff --git a/tests/load/weaviate/test_weaviate_client.py b/tests/load/weaviate/test_weaviate_client.py index 3f966c2330..11d3f13db9 100644 --- a/tests/load/weaviate/test_weaviate_client.py +++ b/tests/load/weaviate/test_weaviate_client.py @@ -26,6 +26,9 @@ from .utils import drop_active_pipeline_data +# mark all tests as essential, do not remove +pytestmark = pytest.mark.essential + @pytest.fixture(autouse=True) def drop_weaviate_schema() -> Iterator[None]: From d7de408a31fff1e1e7b11b34b437150cdd87db20 Mon Sep 17 00:00:00 2001 From: Dave Date: Wed, 3 Apr 2024 12:02:46 +0200 Subject: [PATCH 20/24] run full tests on mssql for all branches --- .github/workflows/test_destination_mssql.yml | 14 +++----------- 1 file changed, 3 insertions(+), 11 deletions(-) diff --git a/.github/workflows/test_destination_mssql.yml b/.github/workflows/test_destination_mssql.yml index 874eb64940..adf7437f1b 100644 --- a/.github/workflows/test_destination_mssql.yml +++ b/.github/workflows/test_destination_mssql.yml @@ -73,14 +73,6 @@ jobs: - name: create secrets.toml run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml - - run: | - poetry run pytest tests/load -m "essential" - name: Run essential tests Linux - if: ${{ ! (contains(github.event.pull_request.labels.*.name, 'ci full') || github.event_name == 'schedule')}} - - - run: | - poetry run pytest tests/load - name: Run all tests Linux - if: ${{ contains(github.event.pull_request.labels.*.name, 'ci full') || github.event_name == 'schedule'}} - - + # always run full suite, also on branches + - run: poetry run pytest tests/load + name: Run tests Linux From f3eba6b4ec003303fa60235774caab14f9cf5ed9 Mon Sep 17 00:00:00 2001 From: Dave Date: Wed, 3 Apr 2024 12:04:03 +0200 Subject: [PATCH 21/24] fix newlines in yaml file --- .github/workflows/test_build_images.yml | 2 +- .github/workflows/test_dbt_runner.yml | 1 - .github/workflows/test_destination_athena.yml | 2 -- .github/workflows/test_destination_athena_iceberg.yml | 4 ---- .github/workflows/test_destination_bigquery.yml | 1 - .github/workflows/test_destination_databricks.yml | 2 -- .github/workflows/test_destination_qdrant.yml | 2 -- .github/workflows/test_destination_snowflake.yml | 3 --- .github/workflows/test_destination_synapse.yml | 1 - .github/workflows/test_destinations.yml | 3 --- .github/workflows/test_doc_snippets.yml | 1 - 11 files changed, 1 insertion(+), 21 deletions(-) diff --git a/.github/workflows/test_build_images.yml b/.github/workflows/test_build_images.yml index a26c93d5cc..c9a99eda2d 100644 --- a/.github/workflows/test_build_images.yml +++ b/.github/workflows/test_build_images.yml @@ -39,4 +39,4 @@ jobs: installer-parallel: true - name: Build images - run: make test-build-images \ No newline at end of file + run: make test-build-images diff --git a/.github/workflows/test_dbt_runner.yml b/.github/workflows/test_dbt_runner.yml index ae26a5d852..0ca784a1ae 100644 --- a/.github/workflows/test_dbt_runner.yml +++ b/.github/workflows/test_dbt_runner.yml @@ -81,4 +81,3 @@ jobs: - run: | poetry run pytest tests/helpers/dbt_tests --ignore=tests/helpers/dbt_tests/local -k '(not local)' name: Run dbt runner with venv - Linux/MAC - diff --git a/.github/workflows/test_destination_athena.yml b/.github/workflows/test_destination_athena.yml index 2867ff4fd7..08d4e6dd64 100644 --- a/.github/workflows/test_destination_athena.yml +++ b/.github/workflows/test_destination_athena.yml @@ -81,5 +81,3 @@ jobs: poetry run pytest tests/load name: Run all tests Linux if: ${{ contains(github.event.pull_request.labels.*.name, 'ci full') || github.event_name == 'schedule'}} - - diff --git a/.github/workflows/test_destination_athena_iceberg.yml b/.github/workflows/test_destination_athena_iceberg.yml index d3db5603be..c4e9047634 100644 --- a/.github/workflows/test_destination_athena_iceberg.yml +++ b/.github/workflows/test_destination_athena_iceberg.yml @@ -81,7 +81,3 @@ jobs: poetry run pytest tests/load name: Run all tests Linux if: ${{ contains(github.event.pull_request.labels.*.name, 'ci full') || github.event_name == 'schedule'}} - - - - \ No newline at end of file diff --git a/.github/workflows/test_destination_bigquery.yml b/.github/workflows/test_destination_bigquery.yml index 4cfdd03dda..95f7edfb4d 100644 --- a/.github/workflows/test_destination_bigquery.yml +++ b/.github/workflows/test_destination_bigquery.yml @@ -79,4 +79,3 @@ jobs: poetry run pytest tests/load name: Run all tests Linux if: ${{ contains(github.event.pull_request.labels.*.name, 'ci full') || github.event_name == 'schedule'}} - diff --git a/.github/workflows/test_destination_databricks.yml b/.github/workflows/test_destination_databricks.yml index dac516b2fd..95ce20fb90 100644 --- a/.github/workflows/test_destination_databricks.yml +++ b/.github/workflows/test_destination_databricks.yml @@ -77,5 +77,3 @@ jobs: poetry run pytest tests/load name: Run all tests Linux if: ${{ contains(github.event.pull_request.labels.*.name, 'ci full') || github.event_name == 'schedule'}} - - diff --git a/.github/workflows/test_destination_qdrant.yml b/.github/workflows/test_destination_qdrant.yml index 9ab32cfd57..9131e9c62d 100644 --- a/.github/workflows/test_destination_qdrant.yml +++ b/.github/workflows/test_destination_qdrant.yml @@ -75,5 +75,3 @@ jobs: poetry run pytest tests/load name: Run all tests Linux if: ${{ contains(github.event.pull_request.labels.*.name, 'ci full') || github.event_name == 'schedule'}} - - diff --git a/.github/workflows/test_destination_snowflake.yml b/.github/workflows/test_destination_snowflake.yml index cdeb890e73..c46ca95a6b 100644 --- a/.github/workflows/test_destination_snowflake.yml +++ b/.github/workflows/test_destination_snowflake.yml @@ -77,6 +77,3 @@ jobs: poetry run pytest tests/load name: Run all tests Linux if: ${{ contains(github.event.pull_request.labels.*.name, 'ci full') || github.event_name == 'schedule'}} - - - diff --git a/.github/workflows/test_destination_synapse.yml b/.github/workflows/test_destination_synapse.yml index d5f2df0a9f..997b0a2903 100644 --- a/.github/workflows/test_destination_synapse.yml +++ b/.github/workflows/test_destination_synapse.yml @@ -80,4 +80,3 @@ jobs: poetry run pytest tests/load name: Run all tests Linux if: ${{ contains(github.event.pull_request.labels.*.name, 'ci full') || github.event_name == 'schedule'}} - diff --git a/.github/workflows/test_destinations.yml b/.github/workflows/test_destinations.yml index 81422092f9..a23f5f3f4d 100644 --- a/.github/workflows/test_destinations.yml +++ b/.github/workflows/test_destinations.yml @@ -89,6 +89,3 @@ jobs: poetry run pytest tests/load name: Run all tests Linux if: ${{ contains(github.event.pull_request.labels.*.name, 'ci full') || github.event_name == 'schedule'}} - - - diff --git a/.github/workflows/test_doc_snippets.yml b/.github/workflows/test_doc_snippets.yml index ebdcbd368f..107ad46418 100644 --- a/.github/workflows/test_doc_snippets.yml +++ b/.github/workflows/test_doc_snippets.yml @@ -68,4 +68,3 @@ jobs: - name: Run linter and tests run: make test-and-lint-snippets - From d2c0c6ea8fc3166fd9354addacccfc21e6360b4f Mon Sep 17 00:00:00 2001 From: Dave Date: Thu, 4 Apr 2024 15:59:00 +0200 Subject: [PATCH 22/24] register essential marker and remove some essential tests --- pytest.ini | 4 +++- tests/load/pipeline/test_replace_disposition.py | 1 - tests/load/test_insert_job_client.py | 4 +--- tests/load/test_job_client.py | 4 +--- tests/load/test_sql_client.py | 1 - 5 files changed, 5 insertions(+), 9 deletions(-) diff --git a/pytest.ini b/pytest.ini index 0f9f6ab0d8..f3a85576ac 100644 --- a/pytest.ini +++ b/pytest.ini @@ -7,4 +7,6 @@ log_cli= 1 log_cli_level= INFO python_files = test_*.py *_test.py *snippets.py *snippet.pytest python_functions = *_test test_* *_snippet -filterwarnings= ignore::DeprecationWarning \ No newline at end of file +filterwarnings= ignore::DeprecationWarning +markers = + essential: marks all essential tests \ No newline at end of file diff --git a/tests/load/pipeline/test_replace_disposition.py b/tests/load/pipeline/test_replace_disposition.py index 23e85a13d8..6efde6e019 100644 --- a/tests/load/pipeline/test_replace_disposition.py +++ b/tests/load/pipeline/test_replace_disposition.py @@ -247,7 +247,6 @@ def load_items_none(): } -@pytest.mark.essential @pytest.mark.parametrize( "destination_config", destinations_configs( diff --git a/tests/load/test_insert_job_client.py b/tests/load/test_insert_job_client.py index 826e154607..57963c13d4 100644 --- a/tests/load/test_insert_job_client.py +++ b/tests/load/test_insert_job_client.py @@ -18,8 +18,6 @@ from tests.load.utils import expect_load_file, prepare_table, yield_client_with_storage from tests.load.pipeline.utils import destinations_configs, DestinationTestConfiguration -# mark all tests as essential, do not remove -pytestmark = pytest.mark.essential DEFAULT_SUBSET = ["duckdb", "redshift", "postgres"] @@ -33,7 +31,7 @@ def file_storage() -> FileStorage: def client(request) -> Iterator[InsertValuesJobClient]: yield from yield_client_with_storage(request.param.destination) # type: ignore[misc] - +@pytest.mark.essential @pytest.mark.parametrize( "client", destinations_configs(default_sql_configs=True, subset=DEFAULT_SUBSET), diff --git a/tests/load/test_job_client.py b/tests/load/test_job_client.py index 0a205293ab..a5dfb5d829 100644 --- a/tests/load/test_job_client.py +++ b/tests/load/test_job_client.py @@ -44,9 +44,6 @@ ) from tests.load.pipeline.utils import destinations_configs, DestinationTestConfiguration -# mark all tests as essential, do not remove -pytestmark = pytest.mark.essential - @pytest.fixture def file_storage() -> FileStorage: @@ -159,6 +156,7 @@ def test_get_update_basic_schema(client: SqlJobClientBase) -> None: assert this_schema == newest_schema +@pytest.mark.essential @pytest.mark.parametrize( "client", destinations_configs(default_sql_configs=True), indirect=True, ids=lambda x: x.name ) diff --git a/tests/load/test_sql_client.py b/tests/load/test_sql_client.py index 67372012ff..da162621b1 100644 --- a/tests/load/test_sql_client.py +++ b/tests/load/test_sql_client.py @@ -36,7 +36,6 @@ def client(request) -> Iterator[SqlJobClientBase]: yield from yield_client_with_storage(request.param.destination) -@pytest.mark.essential @pytest.mark.parametrize( "client", destinations_configs(default_sql_configs=True, exclude=["mssql", "synapse"]), From 65c754deaffdf74fbacc42735581258885259498 Mon Sep 17 00:00:00 2001 From: Dave Date: Thu, 4 Apr 2024 23:18:27 +0200 Subject: [PATCH 23/24] exclude \"athena-parquet-staging-iceberg\" from regular athena tests --- .github/workflows/test_destination_athena.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_destination_athena.yml b/.github/workflows/test_destination_athena.yml index 08d4e6dd64..959fffcfd4 100644 --- a/.github/workflows/test_destination_athena.yml +++ b/.github/workflows/test_destination_athena.yml @@ -22,7 +22,7 @@ env: RUNTIME__DLTHUB_TELEMETRY_SEGMENT_WRITE_KEY: TLJiyRkGVZGCi2TtjClamXpFcxAA1rSB ACTIVE_DESTINATIONS: "[\"athena\"]" ALL_FILESYSTEM_DRIVERS: "[\"memory\"]" - EXCLUDED_DESTINATION_CONFIGURATIONS: "[\"athena-parquet-staging-iceberg\"]" + EXCLUDED_DESTINATION_CONFIGURATIONS: "[\"athena-parquet-staging-iceberg\", \"athena-parquet-no-staging-iceberg\"]" jobs: get_docs_changes: From a461d9cd6acb3c82bcc7ed775e23de0ce8aefb1a Mon Sep 17 00:00:00 2001 From: Dave Date: Thu, 4 Apr 2024 23:20:44 +0200 Subject: [PATCH 24/24] remove regular athena tests from iceberg tests --- .github/workflows/test_destination_athena_iceberg.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/test_destination_athena_iceberg.yml b/.github/workflows/test_destination_athena_iceberg.yml index c4e9047634..ea3cb3c06b 100644 --- a/.github/workflows/test_destination_athena_iceberg.yml +++ b/.github/workflows/test_destination_athena_iceberg.yml @@ -22,7 +22,7 @@ env: RUNTIME__DLTHUB_TELEMETRY_SEGMENT_WRITE_KEY: TLJiyRkGVZGCi2TtjClamXpFcxAA1rSB ACTIVE_DESTINATIONS: "[\"athena\"]" ALL_FILESYSTEM_DRIVERS: "[\"memory\"]" - EXCLUDED_DESTINATION_CONFIGURATIONS: "[\"athena-no-staging\"]" + EXCLUDED_DESTINATION_CONFIGURATIONS: "[\"athena-no-staging\", \"athena-parquet-no-staging\"]" jobs: get_docs_changes: