implements secret masking for auth helpers #1267
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: test | clickhouse | |
| on: | |
| pull_request: | |
| branches: | |
| - master | |
| - devel | |
| workflow_dispatch: | |
| schedule: | |
| - cron: '0 2 * * *' | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} | |
| cancel-in-progress: true | |
| env: | |
| RUNTIME__SENTRY_DSN: https://[email protected]/4504819859914752 | |
| RUNTIME__LOG_LEVEL: ERROR | |
| DLT_SECRETS_TOML: ${{ secrets.DLT_SECRETS_TOML }} | |
| ACTIVE_DESTINATIONS: "[\"clickhouse\"]" | |
| ALL_FILESYSTEM_DRIVERS: "[\"memory\", \"file\"]" | |
| jobs: | |
| get_docs_changes: | |
| name: docs changes | |
| uses: ./.github/workflows/get_docs_changes.yml | |
| if: ${{ !github.event.pull_request.head.repo.fork || contains(github.event.pull_request.labels.*.name, 'ci from fork')}} | |
| run_loader: | |
| name: test | clickhouse tests | |
| needs: get_docs_changes | |
| if: needs.get_docs_changes.outputs.changes_outside_docs == 'true' | |
| defaults: | |
| run: | |
| shell: bash | |
| runs-on: "ubuntu-latest" | |
| steps: | |
| - name: Check out | |
| uses: actions/checkout@master | |
| - name: Setup Python | |
| uses: actions/setup-python@v4 | |
| with: | |
| python-version: "3.10.x" | |
| - name: Install Poetry | |
| uses: snok/[email protected] | |
| with: | |
| virtualenvs-create: true | |
| virtualenvs-in-project: true | |
| installer-parallel: true | |
| - name: Load cached venv | |
| id: cached-poetry-dependencies | |
| uses: actions/cache@v3 | |
| with: | |
| path: .venv | |
| key: venv-${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-${{ hashFiles('**/poetry.lock') }}-gcp | |
| - name: Install dependencies | |
| run: poetry install --no-interaction -E clickhouse --with providers -E parquet --with sentry-sdk --with pipeline | |
| - name: create secrets.toml | |
| run: pwd && echo "$DLT_SECRETS_TOML" > tests/.dlt/secrets.toml | |
| # OSS ClickHouse | |
| - run: | | |
| docker-compose -f "tests/load/clickhouse/clickhouse-compose.yml" up -d | |
| echo "Waiting for ClickHouse to be healthy..." | |
| timeout 30s bash -c 'until docker-compose -f "tests/load/clickhouse/clickhouse-compose.yml" ps | grep -q "healthy"; do sleep 1; done' | |
| echo "ClickHouse is up and running" | |
| name: Start ClickHouse OSS | |
| - run: poetry run pytest tests/load -m "essential" | |
| name: Run essential tests Linux (ClickHouse OSS) | |
| if: ${{ ! (contains(github.event.pull_request.labels.*.name, 'ci full') || github.event_name == 'schedule')}} | |
| env: | |
| DESTINATION__CLICKHOUSE__CREDENTIALS__HOST: localhost | |
| DESTINATION__CLICKHOUSE__CREDENTIALS__DATABASE: dlt_data | |
| DESTINATION__CLICKHOUSE__CREDENTIALS__USERNAME: loader | |
| DESTINATION__CLICKHOUSE__CREDENTIALS__PASSWORD: loader | |
| DESTINATION__CLICKHOUSE__CREDENTIALS__PORT: 9000 | |
| DESTINATION__CLICKHOUSE__CREDENTIALS__HTTP_PORT: 8123 | |
| DESTINATION__CLICKHOUSE__CREDENTIALS__SECURE: 0 | |
| - run: poetry run pytest tests/load | |
| name: Run all tests Linux (ClickHouse OSS) | |
| if: ${{ contains(github.event.pull_request.labels.*.name, 'ci full') || github.event_name == 'schedule'}} | |
| env: | |
| DESTINATION__CLICKHOUSE__CREDENTIALS__HOST: localhost | |
| DESTINATION__CLICKHOUSE__CREDENTIALS__DATABASE: dlt_data | |
| DESTINATION__CLICKHOUSE__CREDENTIALS__USERNAME: loader | |
| DESTINATION__CLICKHOUSE__CREDENTIALS__PASSWORD: loader | |
| DESTINATION__CLICKHOUSE__CREDENTIALS__PORT: 9000 | |
| DESTINATION__CLICKHOUSE__CREDENTIALS__HTTP_PORT: 8123 | |
| DESTINATION__CLICKHOUSE__CREDENTIALS__SECURE: 0 | |
| - name: Stop ClickHouse OSS | |
| if: always() | |
| run: docker-compose -f "tests/load/clickhouse/clickhouse-compose.yml" down -v | |
| # ClickHouse Cloud | |
| - run: | | |
| poetry run pytest tests/load -m "essential" | |
| name: Run essential tests Linux (ClickHouse Cloud) | |
| if: ${{ ! (contains(github.event.pull_request.labels.*.name, 'ci full') || github.event_name == 'schedule')}} | |
| - run: | | |
| poetry run pytest tests/load | |
| name: Run all tests Linux (ClickHouse Cloud) | |
| if: ${{ contains(github.event.pull_request.labels.*.name, 'ci full') || github.event_name == 'schedule'}} | |