Connector Ops CI - Publish Connectors #10412
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Connector Ops CI - Publish Connectors | |
| on: | |
| push: | |
| branches: | |
| - master | |
| paths: | |
| - "airbyte-integrations/connectors/**/metadata.yaml" | |
| workflow_dispatch: | |
| inputs: | |
| connectors-options: | |
| description: "Options to pass to the 'airbyte-ci connectors' command group." | |
| default: "--name=source-pokeapi" | |
| publish-options: | |
| description: "Options to pass to the 'airbyte-ci connectors publish' command. Use --pre-release or --main-release depending on whether you want to publish a dev image or not. " | |
| default: "--pre-release" | |
| airbyte_ci_binary_url: | |
| description: "URL to the airbyte-ci binary to use for the action. If not provided, the action will use the latest release of airbyte-ci." | |
| default: "https://connectors.airbyte.com/airbyte-ci/releases/ubuntu/latest/airbyte-ci" | |
| jobs: | |
| publish_connectors: | |
| name: Publish connectors | |
| runs-on: connector-publish-large | |
| steps: | |
| - name: Checkout Airbyte | |
| uses: actions/checkout@v4 | |
| - name: Publish modified connectors [On merge to master] | |
| id: publish-modified-connectors | |
| if: github.event_name == 'push' | |
| uses: ./.github/actions/run-airbyte-ci | |
| with: | |
| context: "master" | |
| dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN_CACHE_4 }} | |
| docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} | |
| docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} | |
| gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }} | |
| gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }} | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| metadata_service_gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }} | |
| sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} | |
| slack_webhook_url: ${{ secrets.PUBLISH_ON_MERGE_SLACK_WEBHOOK }} | |
| spec_cache_gcs_credentials: ${{ secrets.SPEC_CACHE_SERVICE_ACCOUNT_KEY_PUBLISH }} | |
| s3_build_cache_access_key_id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} | |
| s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} | |
| subcommand: "connectors --concurrency=1 --execute-timeout=3600 --metadata-changes-only publish --main-release" | |
| python_registry_token: ${{ secrets.PYPI_TOKEN }} | |
| max_attempts: 2 | |
| retry_wait_seconds: 600 # 10 minutes | |
| - name: Publish connectors [manual] | |
| id: publish-connectors | |
| if: github.event_name == 'workflow_dispatch' | |
| uses: ./.github/actions/run-airbyte-ci | |
| with: | |
| context: "manual" | |
| dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN_CACHE_4 }} | |
| docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} | |
| docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} | |
| gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }} | |
| gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }} | |
| github_token: ${{ secrets.GITHUB_TOKEN }} | |
| metadata_service_gcs_credentials: ${{ secrets.METADATA_SERVICE_PROD_GCS_CREDENTIALS }} | |
| sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} | |
| slack_webhook_url: ${{ secrets.PUBLISH_ON_MERGE_SLACK_WEBHOOK }} | |
| spec_cache_gcs_credentials: ${{ secrets.SPEC_CACHE_SERVICE_ACCOUNT_KEY_PUBLISH }} | |
| s3_build_cache_access_key_id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} | |
| s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} | |
| subcommand: "connectors ${{ github.event.inputs.connectors-options }} publish ${{ github.event.inputs.publish-options }}" | |
| python_registry_token: ${{ secrets.PYPI_TOKEN }} | |
| airbyte_ci_binary_url: ${{ github.event.inputs.airbyte_ci_binary_url }} | |
| max_attempts: 2 | |
| notify-failure-slack-channel: | |
| name: "Notify Slack Channel on Publish Failures" | |
| runs-on: ubuntu-latest | |
| needs: | |
| - publish_connectors | |
| if: ${{ always() && contains(needs.*.result, 'failure') && github.ref == 'refs/heads/master' }} | |
| steps: | |
| - name: Checkout Airbyte | |
| uses: actions/checkout@v4 | |
| - name: Match GitHub User to Slack User | |
| id: match-github-to-slack-user | |
| uses: ./.github/actions/match-github-to-slack-user | |
| env: | |
| AIRBYTE_TEAM_BOT_SLACK_TOKEN: ${{ secrets.SLACK_AIRBYTE_TEAM_READ_USERS }} | |
| GITHUB_API_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Send publish failures to connector-publish-failures channel | |
| id: slack | |
| uses: slackapi/[email protected] | |
| with: | |
| # This data can be any valid JSON from a previous step in the GitHub Action | |
| payload: | | |
| { | |
| "channel": "#connector-publish-failures", | |
| "username": "Connectors CI/CD Bot", | |
| "text": "🚨 Publish workflow failed:\n ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} \n merged by ${{ github.actor }} (<@${{ steps.match-github-to-slack-user.outputs.slack_user_ids }}>). " | |
| } | |
| env: | |
| SLACK_WEBHOOK_URL: ${{ secrets.PUBLISH_ON_MERGE_SLACK_WEBHOOK }} | |
| notify-failure-pager-duty: | |
| name: "Notify PagerDuty on Publish Failures" | |
| runs-on: ubuntu-latest | |
| needs: | |
| - publish_connectors | |
| if: ${{ always() && contains(needs.*.result, 'failure') && github.ref == 'refs/heads/master' }} | |
| steps: | |
| - name: Checkout Airbyte | |
| uses: actions/checkout@v4 | |
| - name: Notify PagerDuty | |
| id: pager-duty | |
| uses: ./.github/actions/send-pager-duty-event | |
| with: | |
| # Integration URL: https://airbyte.pagerduty.com/services/P5GNI5T/integrations/PGKH9JV | |
| integration_key: ${{ secrets.PAGER_DUTY_PUBLISH_FAILURES_INTEGRATION_KEY }} | |
| summary: "Publish workflow failed: ${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }} merged by ${{ github.actor }}" | |
| severity: "critical" | |
| source: "${{ github.server_url }}/${{ github.repository }}/actions/runs/${{ github.run_id }}" |