Bulk Load CDK: Staging refactor + tests; multi-sync ITs correctly wait for ack #35733
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Community CI | |
| concurrency: | |
| # This is the name of the concurrency group. It is used to prevent concurrent runs of the same workflow. | |
| # | |
| # - github.head_ref is only defined on PR runs, it makes sure that the concurrency group is unique for pull requests | |
| # ensuring that only one run per pull request is active at a time. | |
| group: ${{ github.workflow }}-${{ github.head_ref }} | |
| cancel-in-progress: true | |
| on: | |
| # The pull request target event is triggered on PR open and synchronize events. | |
| # The executed workflow code is the one from the target branch. | |
| # It guarantees that the workflow logic is not altered by the PR. | |
| pull_request_target: | |
| types: [opened, synchronize] | |
| jobs: | |
| fail_on_protected_path_changes: | |
| name: "Check fork do not change protected paths" | |
| if: github.event.pull_request.head.repo.fork == true | |
| runs-on: ubuntu-latest | |
| permissions: | |
| pull-requests: read | |
| steps: | |
| - name: Check for changes in protected paths | |
| id: check_for_changes_in_protected_paths | |
| uses: dorny/paths-filter@v2 | |
| with: | |
| filters: | | |
| protected_paths: | |
| - '.github/**' | |
| - name: Fail if changes in protected paths | |
| if: steps.check_for_changes_in_protected_paths.outputs.protected_paths == 'true' | |
| run: | | |
| echo "The fork has changes in protected paths. This is not allowed." | |
| exit 1 | |
| format_check: | |
| # IMPORTANT: This name must match the require check name on the branch protection settings | |
| name: "Check for formatting errors" | |
| if: github.event.pull_request.head.repo.fork == true | |
| environment: community-ci-auto | |
| runs-on: community-tooling-test-small | |
| needs: fail_on_protected_path_changes | |
| timeout-minutes: 30 | |
| env: | |
| MAIN_BRANCH_NAME: "master" | |
| steps: | |
| # This checkouts a fork which can contain untrusted code | |
| # It's deemed safe as the formatter are not executing any checked out code | |
| - name: Checkout fork | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: ${{ github.event.pull_request.head.repo.full_name }} | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| fetch-depth: 1 | |
| # This will sync the .github folder of the main repo with the fork | |
| # This allows us to use up to date actions and CI logic from the main repo | |
| - name: Pull .github folder and internal packages from main repository | |
| id: pull_github_folder | |
| run: | | |
| git remote add main https://github.com/airbytehq/airbyte.git | |
| git fetch main ${MAIN_BRANCH_NAME} | |
| git checkout main/${MAIN_BRANCH_NAME} -- .github | |
| git checkout main/${MAIN_BRANCH_NAME} -- airbyte-ci | |
| - name: Run airbyte-ci format check all | |
| # This path refers to the fork .github folder. | |
| # We make sure its content is in sync with the main repo .github folder by pulling it in the previous step | |
| uses: ./.github/actions/run-airbyte-ci | |
| with: | |
| context: "pull_request" | |
| sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} | |
| subcommand: "format check all" | |
| is_fork: "true" | |
| connectors_early_ci: | |
| name: Run connectors early CI on fork | |
| if: github.event.pull_request.head.repo.fork == true | |
| needs: fail_on_protected_path_changes | |
| environment: community-ci-auto | |
| runs-on: community-tooling-test-small | |
| timeout-minutes: 10 | |
| env: | |
| MAIN_BRANCH_NAME: "master" | |
| permissions: | |
| statuses: write | |
| steps: | |
| # This checkouts a fork which can contain untrusted code | |
| # It's deemed safe as the static checks are not executing any checked out code | |
| - name: Checkout fork | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: ${{ github.event.pull_request.head.repo.full_name }} | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| fetch-depth: 1 | |
| # This will sync the .github folder of the main repo with the fork | |
| # This allows us to use up to date actions and CI logic from the main repo | |
| - name: Pull .github folder from main repository | |
| id: pull_github_folder | |
| run: | | |
| git remote add main https://github.com/airbytehq/airbyte.git | |
| git fetch main ${MAIN_BRANCH_NAME} | |
| git checkout main/${MAIN_BRANCH_NAME} -- .github | |
| git checkout main/${MAIN_BRANCH_NAME} -- airbyte-ci | |
| - name: Run airbyte-ci static checks and version increment checks on modified connectors | |
| # This path refers to the fork .github folder. | |
| # We make sure its content is in sync with the main repo .github folder by pulling it in the previous step | |
| uses: ./.github/actions/run-airbyte-ci | |
| with: | |
| context: "pull_request" | |
| sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} | |
| subcommand: "connectors --modified test --only-step=qa_checks --only-step=version_inc_check --global-status-check-context='Connectors early CI checks' --global-status-check-description='Running early CI checks on connectors'" | |
| is_fork: "true" | |
| git_repo_url: ${{ github.event.pull_request.head.repo.clone_url }} | |
| git_branch: ${{ github.head_ref }} | |
| git_revision: ${{ github.event.pull_request.head.sha }} | |
| github_token: ${{ github.token }} | |
| docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} | |
| docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} | |
| - name: Upload pipeline reports | |
| id: upload-artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: early-ci-pipeline-reports | |
| path: /home/runner/work/airbyte/airbyte/airbyte-ci/connectors/pipelines/pipeline_reports/airbyte-ci/connectors/test/pull_request/**/output.html | |
| retention-days: 7 | |
| connectors_full_ci: | |
| name: Run connectors full CI on fork | |
| if: github.event.pull_request.head.repo.fork == true | |
| # Deployment of jobs on the community-ci environment requires manual approval | |
| # This is something we set up in the GitHub environment settings: | |
| # https://github.com/airbytehq/airbyte/settings/environments/2091483613/edit | |
| # This is a safety measure to make sure the code running on our infrastructure has been reviewed before running on it | |
| needs: fail_on_protected_path_changes | |
| environment: community-ci | |
| runs-on: community-connector-test-large | |
| timeout-minutes: 180 # 3 hours | |
| permissions: | |
| statuses: write | |
| env: | |
| MAIN_BRANCH_NAME: "master" | |
| steps: | |
| # This checkouts a fork which can contain untrusted code | |
| # It's deemed safe as the community-ci environment requires manual reviewer approval to run | |
| - name: Checkout fork | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: ${{ github.event.pull_request.head.repo.full_name }} | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| fetch-depth: 1 | |
| # This will sync the .github folder of the main repo with the fork | |
| # This allows us to use up to date actions and CI logic from the main repo | |
| - name: Pull .github folder from main repository | |
| id: pull_github_folder | |
| run: | | |
| git remote add main https://github.com/airbytehq/airbyte.git | |
| git fetch main ${MAIN_BRANCH_NAME} | |
| git checkout main/${MAIN_BRANCH_NAME} -- .github | |
| git checkout main/${MAIN_BRANCH_NAME} -- airbyte-ci | |
| - name: Run airbyte-ci connectors test | |
| # This path refers to the fork .github folder. | |
| # We make sure its content is in sync with the main repo .github folder by pulling it in the previous step | |
| uses: ./.github/actions/run-airbyte-ci | |
| with: | |
| context: "pull_request" | |
| docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} | |
| docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} | |
| gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }} | |
| gcp_integration_tester_credentials: ${{ secrets.GCLOUD_INTEGRATION_TESTER }} | |
| sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} | |
| git_repo_url: ${{ github.event.pull_request.head.repo.clone_url }} | |
| git_branch: ${{ github.head_ref }} | |
| git_revision: ${{ github.event.pull_request.head.sha }} | |
| github_token: ${{ secrets.GH_PAT_MAINTENANCE_OSS }} | |
| s3_build_cache_access_key_id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} | |
| s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} | |
| subcommand: "connectors --modified test" | |
| is_fork: "true" | |
| - name: Upload pipeline reports | |
| id: upload-artifact | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: pipeline-reports | |
| path: /home/runner/work/airbyte/airbyte/airbyte-ci/connectors/pipelines/pipeline_reports/airbyte-ci/connectors/test/pull_request/**/output.html | |
| retention-days: 7 | |
| internal_poetry_packages_ci: | |
| name: Internal Poetry packages CI | |
| if: github.event.pull_request.head.repo.fork == true | |
| # Deployment of jobs on the community-ci environment requires manual approval | |
| # This is something we set up in the GitHub environment settings: | |
| # https://github.com/airbytehq/airbyte/settings/environments/2091483613/edit | |
| # This is a safety measure to make sure the code running on our infrastructure has been reviewed before running on it | |
| needs: fail_on_protected_path_changes | |
| environment: community-ci | |
| runs-on: community-tooling-test-small | |
| timeout-minutes: 180 # 3 hours | |
| permissions: | |
| statuses: write | |
| env: | |
| MAIN_BRANCH_NAME: "master" | |
| steps: | |
| # This checkouts a fork which can contain untrusted code | |
| # It's deemed safe as the community-ci environment requires manual reviewer approval to run | |
| - name: Checkout fork | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: ${{ github.event.pull_request.head.repo.full_name }} | |
| ref: ${{ github.event.pull_request.head.sha }} | |
| fetch-depth: 1 | |
| # This will sync the .github folder of the main repo with the fork | |
| # This allows us to use up to date actions and CI logic from the main repo | |
| - name: Pull .github folder from main repository | |
| id: pull_github_folder | |
| run: | | |
| git remote add main https://github.com/airbytehq/airbyte.git | |
| git fetch main ${MAIN_BRANCH_NAME} | |
| git checkout main/${MAIN_BRANCH_NAME} -- .github | |
| git checkout main/${MAIN_BRANCH_NAME} -- airbyte-ci | |
| - name: Run poe tasks for modified internal packages [PULL REQUEST] | |
| # This path refers to the fork .github folder. | |
| # We make sure its content is in sync with the main repo .github folder by pulling it in the previous step | |
| id: run-airbyte-ci-test-pr | |
| uses: ./.github/actions/run-airbyte-ci | |
| with: | |
| context: "pull_request" | |
| docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} | |
| docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} | |
| gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }} | |
| gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }} | |
| git_repo_url: ${{ github.event.pull_request.head.repo.clone_url }} | |
| git_branch: ${{ github.head_ref }} | |
| git_revision: ${{ github.event.pull_request.head.sha }} | |
| github_token: ${{ github.token }} | |
| subcommand: "test --modified" | |
| is_fork: "true" |