Destination S3 V2: copy over legacy DATs, enable tests #66775
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Connectors Tests | |
| concurrency: | |
| # This is the name of the concurrency group. It is used to prevent concurrent runs of the same workflow. | |
| # | |
| # - github.head_ref is only defined on PR runs, it makes sure that the concurrency group is unique for pull requests | |
| # ensuring that only one run per pull request is active at a time. | |
| # | |
| # - github.run_id is defined on all runs, it makes sure that the concurrency group is unique for workflow dispatches. | |
| # This allows us to run multiple workflow dispatches in parallel. | |
| group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} | |
| cancel-in-progress: true | |
| on: | |
| workflow_dispatch: | |
| inputs: | |
| test-connectors-options: | |
| description: "Options to pass to the 'airbyte-ci connectors test' command" | |
| default: "--modified" | |
| pull_request: | |
| types: | |
| - opened | |
| - synchronize | |
| jobs: | |
| changes: | |
| runs-on: ubuntu-latest | |
| outputs: | |
| connectors: ${{ steps.changes.outputs.connectors }} | |
| permissions: | |
| statuses: write | |
| steps: | |
| - name: Checkout Airbyte | |
| if: github.event_name != 'pull_request' | |
| uses: actions/checkout@v4 | |
| - id: changes | |
| uses: dorny/paths-filter@v2 | |
| with: | |
| # Note: expressions within a filter are OR'ed | |
| filters: | | |
| connectors: | |
| - 'docs/integrations/**/*' | |
| - 'airbyte-ci/**/*' | |
| - 'airbyte-integrations/connectors/**/*' | |
| - 'airbyte-cdk/java/**/*' | |
| - 'buildSrc/**/*' | |
| # The Connector CI Tests & Regression Test Results Reviewed and Approved are status checks emitted by airbyte-ci | |
| # We make them pass once we have determined that there are no changes to the connectors | |
| - name: "Skip Connectors CI tests" | |
| if: steps.changes.outputs.connectors != 'true' && github.event_name == 'pull_request' | |
| run: | | |
| curl --request POST \ | |
| --url https://api.github.com/repos/${{ github.repository }}/statuses/${{ github.event.pull_request.head.sha }} \ | |
| --header 'authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' \ | |
| --header 'content-type: application/json' \ | |
| --data '{ | |
| "state": "success", | |
| "context": "Connectors CI tests", | |
| "target_url": "${{ github.event.workflow_run.html_url }}" | |
| }' \ | |
| - name: "Skip Regression Test Results Reviewed and Approved" | |
| if: steps.changes.outputs.connectors != 'true' && github.event_name == 'pull_request' | |
| run: | | |
| curl --request POST \ | |
| --url https://api.github.com/repos/${{ github.repository }}/statuses/${{ github.event.pull_request.head.sha }} \ | |
| --header 'authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' \ | |
| --header 'content-type: application/json' \ | |
| --data '{ | |
| "state": "success", | |
| "description": "[Skipped]", | |
| "context": "Regression Test Results Reviewed and Approved", | |
| "target_url": "${{ github.event.workflow_run.html_url }}" | |
| }' \ | |
| connectors_ci: | |
| needs: changes | |
| # We only run the Connectors CI job if there are changes to the connectors on a non-forked PR | |
| # Forked PRs are handled by the community_ci.yml workflow | |
| # If the condition is not met the job will be skipped (it will not fail) | |
| if: (github.event_name == 'pull_request' && needs.changes.outputs.connectors == 'true' && github.event.pull_request.head.repo.fork != true) || github.event_name == 'workflow_dispatch' | |
| name: Connectors CI | |
| runs-on: connector-test-large | |
| timeout-minutes: 360 # 6 hours | |
| steps: | |
| - name: Checkout Airbyte | |
| uses: actions/checkout@v4 | |
| - name: Check PAT rate limits | |
| run: | | |
| ./tools/bin/find_non_rate_limited_PAT \ | |
| ${{ secrets.GH_PAT_MAINTENANCE_OSS }} | |
| - name: Extract branch name [WORKFLOW DISPATCH] | |
| shell: bash | |
| if: github.event_name == 'workflow_dispatch' | |
| run: echo "branch=${GITHUB_REF#refs/heads/}" >> $GITHUB_OUTPUT | |
| id: extract_branch | |
| - name: Fetch last commit id from remote branch [PULL REQUESTS] | |
| if: github.event_name == 'pull_request' | |
| id: fetch_last_commit_id_pr | |
| run: echo "commit_id=$(git ls-remote --heads origin refs/heads/${{ github.head_ref }} | cut -f 1)" >> $GITHUB_OUTPUT | |
| - name: Fetch last commit id from remote branch [WORKFLOW DISPATCH] | |
| if: github.event_name == 'workflow_dispatch' | |
| id: fetch_last_commit_id_wd | |
| run: echo "commit_id=$(git rev-parse origin/${{ steps.extract_branch.outputs.branch }})" >> $GITHUB_OUTPUT | |
| - name: Test connectors [WORKFLOW DISPATCH] | |
| if: github.event_name == 'workflow_dispatch' | |
| uses: ./.github/actions/run-airbyte-ci | |
| with: | |
| context: "manual" | |
| dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN_CACHE_3 }} | |
| docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} | |
| docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} | |
| gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }} | |
| sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} | |
| git_branch: ${{ steps.extract_branch.outputs.branch }} | |
| git_revision: ${{ steps.fetch_last_commit_id_wd.outputs.commit_id }} | |
| github_token: ${{ env.PAT }} | |
| s3_build_cache_access_key_id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} | |
| s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} | |
| subcommand: "connectors ${{ github.event.inputs.test-connectors-options }} test" | |
| - name: Test connectors [PULL REQUESTS] | |
| if: github.event_name == 'pull_request' | |
| uses: ./.github/actions/run-airbyte-ci | |
| with: | |
| context: "pull_request" | |
| dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN_CACHE_3 }} | |
| docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} | |
| docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} | |
| gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }} | |
| gcp_integration_tester_credentials: ${{ secrets.GCLOUD_INTEGRATION_TESTER }} | |
| sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} | |
| git_branch: ${{ github.head_ref }} | |
| git_revision: ${{ steps.fetch_last_commit_id_pr.outputs.commit_id }} | |
| github_token: ${{ env.PAT }} | |
| s3_build_cache_access_key_id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} | |
| s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} | |
| # A connector test can't take more than 5 hours to run (5 * 60 * 60 = 18000 seconds) | |
| subcommand: "connectors --execute-timeout=18000 --modified test" |