update with master

.bumpversion.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.63.0
+current_version = 0.63.19
 commit = False
 tag = False
 parse = (?P<major>\d+)\.(?P<minor>\d+)\.(?P<patch>\d+)(\-[a-z]+)?

.github/CODEOWNERS

@@ -8,20 +8,27 @@
 /airbyte-cdk/python/airbyte_cdk/destinations/vector_db_based @airbytehq/ai-language-models
 
 # CI/CD
-/.github/ @airbytehq/connector-extensibility
-/airbyte-ci/ @airbytehq/connector-extensibility
+/.github/ @airbytehq/dev-tooling
+/airbyte-ci/ @airbytehq/dev-tooling
 
 # Python CDK and Connector Acceptance Tests
-/airbyte-cdk/python @airbytehq/connector-extensibility
-/airbyte-integrations/connector-templates/ @airbytehq/connector-extensibility
-/airbyte-integrations/bases/connector-acceptance-test/ @airbytehq/connector-extensibility @lazebnyi @oustynova
+/airbyte-cdk/python @airbytehq/python-team
+/airbyte-integrations/connector-templates/ @airbytehq/dev-marketplace-contributions
+/airbyte-integrations/bases/connector-acceptance-test/ @airbytehq/dev-tooling
 
 # Build customization file change
-/airbyte-integrations/connectors/**/build_customization.py @airbytehq/connector-extensibility
+/airbyte-integrations/connectors/**/build_customization.py @airbytehq/dev-tooling
 
 # Protocol related items
 /docs/understanding-airbyte/airbyte-protocol.md @airbytehq/protocol-reviewers
 
+# Bulk CDK
+/airbyte-cdk/bulk @airbytehq/dbsources @airbytehq/destinations
+/airbyte-cdk/bulk/core/extract/ @airbytehq/dbsources
+/airbyte-cdk/bulk/core/load/ @airbytehq/destinations
+/airbyte-cdk/bulk/toolkits/extract-*/ @airbytehq/dbsources
+/airbyte-cdk/bulk/toolkits/load-*/ @airbytehq/destinations
+
 # Java CDK
 /airbyte-cdk/java/airbyte-cdk @airbytehq/dbsources @airbytehq/destinations
 /airbyte-cdk/java/airbyte-cdk/*-sources/ @airbytehq/dbsources
@@ -54,8 +61,8 @@
 /airbyte-integrations/connectors/destination-redshift/ @airbytehq/destinations
 
 # Python critical connectors
-/airbyte-integrations/connectors/source-facebook-marketing/ @airbytehq/critical-connectors
-/airbyte-integrations/connectors/source-hubspot/ @airbytehq/critical-connectors
-/airbyte-integrations/connectors/source-salesforce/ @airbytehq/critical-connectors
-/airbyte-integrations/connectors/source-shopify/ @airbytehq/critical-connectors
-/airbyte-integrations/connectors/source-stripe/ @airbytehq/critical-connectors
+/airbyte-integrations/connectors/source-facebook-marketing/ @airbytehq/python-team
+/airbyte-integrations/connectors/source-hubspot/ @airbytehq/python-team
+/airbyte-integrations/connectors/source-salesforce/ @airbytehq/python-team
+/airbyte-integrations/connectors/source-shopify/ @airbytehq/python-team
+/airbyte-integrations/connectors/source-stripe/ @airbytehq/python-team

.github/actions/run-airbyte-ci/action.yml

@@ -87,6 +87,15 @@ inputs:
     description: "Whether the PR is from a fork"
     required: false
     default: "false"
+  max_attempts:
+    description: "Number of attempts at running the airbyte-ci command"
+    required: false
+    default: 1
+  retry_wait_seconds:
+    description: "Number of seconds to wait between retry attempts"
+    required: false
+    default: 60
+
 runs:
   using: "composite"
   steps:
@@ -109,7 +118,7 @@ runs:
         is_fork: ${{ inputs.is_fork }}
     - name: Run airbyte-ci
       id: run-airbyte-ci
-      shell: bash
+      uses: nick-fields/retry@v3
       env:
         CI: "True"
         CI_GIT_USER: ${{ github.repository_owner }}
@@ -140,8 +149,14 @@ runs:
         SLACK_WEBHOOK: ${{ inputs.slack_webhook_url }}
         SPEC_CACHE_BUCKET_NAME: ${{ inputs.spec_cache_bucket_name }}
         SPEC_CACHE_GCS_CREDENTIALS: ${{ inputs.spec_cache_gcs_credentials }}
-      run: |
-        airbyte-ci --disable-update-check --disable-dagger-run --is-ci --gha-workflow-run-id=${{ github.run_id }} ${{ inputs.subcommand }} ${{ inputs.options }}
+      with:
+        shell: bash
+        max_attempts: ${{ inputs.max_attempts }}
+        retry_wait_seconds: ${{ inputs.retry_wait_seconds }}
+        # 360mn > 6 hours: it's the GitHub runner max job duration
+        timeout_minutes: 360
+        command: |
+          airbyte-ci --disable-update-check --disable-dagger-run --is-ci --gha-workflow-run-id=${{ github.run_id }} ${{ inputs.subcommand }} ${{ inputs.options }}
     - name: Stop Engine
       id: stop-engine
       if: always()

.github/actions/send-pager-duty-event/action.yml

@@ -0,0 +1,36 @@
+name: "Send PagerDuty event"
+description: "Use Event API V2 to send a PagerDuty event."
+inputs:
+  integration_key:
+    description: "The integration key for the PagerDuty service."
+    required: true
+  summary:
+    description: "A brief text summary of the event."
+    required: true
+  severity:
+    description: "The severity of the event. (info, warning, error, critical)"
+    required: true
+  source:
+    description: "Specific human-readable unique identifier, such as a hostname, for the system having the problem."
+    required: true
+
+runs:
+  using: "composite"
+  steps:
+    - name: Send PagerDuty event
+      id: send-pager-duty-event
+      shell: bash
+      run: |
+        curl --request 'POST' \
+          --fail \
+          --url "https://events.pagerduty.com/v2/enqueue" \
+          --header 'Content-Type: application/json' \
+          --data '{
+            "payload": {
+              "summary": "${{ inputs.summary }}",
+              "severity": "${{ inputs.severity }}",
+              "source": "${{ inputs.source }}"
+            },
+            "event_action": "trigger",
+            "routing_key": "${{ inputs.integration_key }}"
+          }'

.github/workflows/airbyte-ci-tests.yml

@@ -35,9 +35,11 @@ jobs:
             - airbyte-ci/connectors/pipelines/**
             - airbyte-ci/connectors/base_images/**
             - airbyte-ci/connectors/common_utils/**
+            - airbyte-ci/connectors/connectors_insights/**
             - airbyte-ci/connectors/connector_ops/**
             - airbyte-ci/connectors/connectors_qa/**
             - airbyte-ci/connectors/ci_credentials/**
+            - airbyte-ci/connectors/erd/**
             - airbyte-ci/connectors/metadata_service/lib/**
             - airbyte-ci/connectors/metadata_service/orchestrator/**
             - airbyte-cdk/python/**

.github/workflows/approve-regression-tests-command.yml

@@ -0,0 +1,125 @@
+name: Approve Regression Tests
+permissions:
+  pull-requests: write
+  statuses: write
+on:
+  workflow_dispatch:
+    inputs:
+      pr:
+        description: "Pull request number. Used to pull the proper branch ref, including on forks."
+        type: number
+        required: false
+      comment-id:
+        description: "Optional. The comment-id of the slash command. Used to update the comment with the status."
+        required: false
+      connector-name:
+        description: "Optional. Name of the connector whose regression tests are approved."
+        required: false
+
+      # These must be declared, but they are unused and ignored.
+      # TODO: Infer 'repo' and 'gitref' from PR number on other workflows, so we can remove these.
+      repo:
+        description: "Repo (Ignored)"
+        required: false
+        default: "airbytehq/airbyte"
+      gitref:
+        description: "Ref (Ignored)"
+        required: false
+
+run-name: "Approve Regression Tests #${{ github.event.inputs.pr }}"
+
+jobs:
+  approve-regression-tests:
+    name: "Approve Regression Tests"
+    runs-on: ubuntu-latest
+    steps:
+      - name: Get job variables
+        id: job-vars
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        shell: bash
+        run: |
+          PR_JSON=$(gh api repos/${{ github.repository }}/pulls/${{ github.event.inputs.pr }})
+          echo "PR_JSON: $PR_JSON"
+          echo "repo=$(echo "$PR_JSON" | jq -r .head.repo.full_name)" >> $GITHUB_OUTPUT
+          BRANCH=$(echo "$PR_JSON" | jq -r .head.ref)
+          echo "branch=$BRANCH" >> $GITHUB_OUTPUT
+          echo "run-url=https://github.com/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" >> $GITHUB_OUTPUT
+          LATEST_COMMIT=$(gh api repos/${{ github.repository }}/commits/$BRANCH | jq -r .sha)
+          echo "latest_commit=$LATEST_COMMIT" >> $GITHUB_OUTPUT
+
+      - name: Append comment with job run link
+        # If comment-id is not provided, this will create a new
+        # comment with the job run link.
+        id: first-comment-action
+        uses: peter-evans/create-or-update-comment@v4
+        with:
+          comment-id: ${{ github.event.inputs.comment-id }}
+          issue-number: ${{ github.event.inputs.pr }}
+          body: |
+
+            > [Check job output.][1]
+
+            [1]: ${{ steps.job-vars.outputs.run-url }}
+
+      - name: Approve regression tests
+        id: approve-regression-tests
+        if: github.event.inputs.connector-name != null
+        run: |
+          echo "approving regression test status check for ${{ github.event.inputs.connector-name }} if it exists ...."
+          response=$(curl --write-out '%{http_code}' --silent --output /dev/null \
+            --request POST \
+            --url ${{ github.api_url }}/repos/${{ github.repository }}/statuses/${{ steps.job-vars.outputs.latest_commit }} \
+            --header 'authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' \
+            --header 'content-type: application/json' \
+            --data '{
+              "state": "success",
+              "context": "Regression Tests on ${{ github.event.inputs.connector-name }}",
+              "target_url": "https://github.com/airbytehq/airbyte/tree/master/airbyte-ci/connectors/live-tests"
+            }')
+          if [ $response -ne 201 ]; then
+            echo "Failed to approve regression tests for ${{ github.event.inputs.connector-name }}. HTTP status code: $response"
+            exit 1
+          else
+            echo "Regression tests for ${{ github.event.inputs.connector-name }} approved."
+          fi
+
+      - name: Update global regression test approval
+        id: update-global-regression-test-approval
+        if: github.event.inputs.connector-name == null
+        run: |
+          echo "updating regression test approval status check if it exists ...."
+          response=$(curl --write-out '%{http_code}' --silent --output /dev/null \
+            --request POST \
+            --url ${{ github.api_url }}/repos/${{ github.repository }}/statuses/${{ steps.job-vars.outputs.latest_commit }} \
+            --header 'authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' \
+            --header 'content-type: application/json' \
+            --data '{
+              "state": "success",
+              "context": "Regression Test Results Reviewed and Approved",
+              "target_url": "https://github.com/airbytehq/airbyte/tree/master/airbyte-ci/connectors/live-tests"
+            }')
+          if [ $response -ne 201 ]; then
+            echo "Failed to update regression test approval status check. HTTP status code: $response"
+            exit 1
+          else
+            echo "Regression test status check updated."
+          fi
+
+      - name: Append success comment
+        uses: peter-evans/create-or-update-comment@v4
+        if: success()
+        with:
+          comment-id: ${{ steps.first-comment-action.outputs.comment-id }}
+          reactions: "+1"
+          body: |
+            > ✅ Approving regression tests
+
+      - name: Append failure comment
+        uses: peter-evans/create-or-update-comment@v4
+        if: failure()
+        with:
+          comment-id: ${{ steps.first-comment-action.outputs.comment-id }}
+          reactions: confused
+          body: |
+            > ❌ Regression test approval failed

.github/workflows/auto_merge.yml

@@ -25,7 +25,7 @@ jobs:
           # We need a custom Github Token as some API endpoints
           # are not available from GHA auto generated tokens
           # like the one to list branch protection rules...
-          GITHUB_TOKEN: ${{ secrets.AUTO_MERGE_GITHUB_TOKEN }}
+          GITHUB_TOKEN: ${{ secrets.GH_PAT_MAINTENANCE_OSS }}
           AUTO_MERGE_PRODUCTION: ${{ vars.ENABLE_CONNECTOR_AUTO_MERGE }}
         run: |
           poetry install

.github/workflows/cdk_connectors_tests.yml

@@ -0,0 +1,94 @@
+name: CDK Changes - Connectors Tests
+
+concurrency:
+  # This is the name of the concurrency group. It is used to prevent concurrent runs of the same workflow.
+  #
+  # - github.head_ref is only defined on PR runs, it makes sure that the concurrency group is unique for pull requests
+  #  ensuring that only one run per pull request is active at a time.
+  #
+  # - github.run_id is defined on all runs, it makes sure that the concurrency group is unique for workflow dispatches.
+  # This allows us to run multiple workflow dispatches in parallel.
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+on:
+  workflow_dispatch:
+  pull_request:
+    types:
+      - opened
+      - synchronize
+jobs:
+  cdk_changes:
+    runs-on: ubuntu-latest
+    outputs:
+      python_cdk: ${{ steps.changes.outputs.python_cdk }}
+    permissions:
+      statuses: write
+    steps:
+      - name: Checkout Airbyte
+        if: github.event_name != 'pull_request'
+        uses: actions/checkout@v4
+      - id: changes
+        uses: dorny/paths-filter@v2
+        with:
+          filters: |
+            python_cdk:
+              - 'airbyte-cdk/python/airbyte_cdk/**'
+              - 'airbyte-cdk/python/bin/**'
+              - 'airbyte-cdk/python/poetry.lock'
+              - 'airbyte-cdk/python/pyproject.toml'
+      # The Connector CI Tests is a status check emitted by airbyte-ci
+      # We make it pass once we have determined that there are no changes to the connectors
+      - name: "Skip Connectors CI tests"
+        if: steps.changes.outputs.python_cdk != 'true' && github.event_name == 'pull_request'
+        run: |
+          curl --request POST \
+          --url https://api.github.com/repos/${{ github.repository }}/statuses/${{ github.event.pull_request.head.sha }} \
+          --header 'authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' \
+          --header 'content-type: application/json' \
+          --data '{
+            "state": "success",
+            "context": "CDK Changes - Connectors Tests",
+            "target_url": "${{ github.event.workflow_run.html_url }}"
+            }' \
+
+  connectors_ci:
+    needs: cdk_changes
+    # We only run the Connectors CI job if there are changes to the connectors on a non-forked PR
+    # Forked PRs are handled by the community_ci.yml workflow
+    # If the condition is not met the job will be skipped (it will not fail)
+    if: (github.event_name == 'pull_request' && needs.cdk_changes.outputs.python_cdk == 'true' && github.event.pull_request.head.repo.fork != true) || github.event_name == 'workflow_dispatch'
+    name: Connectors CI
+    runs-on: connector-test-large
+    timeout-minutes: 360 # 6 hours
+    steps:
+      - name: Checkout Airbyte
+        uses: actions/checkout@v4
+      - name: Check PAT rate limits
+        run: |
+          ./tools/bin/find_non_rate_limited_PAT \
+            ${{ secrets.GH_PAT_MAINTENANCE_OSS }}
+      - name: Fetch last commit id from remote branch [PULL REQUESTS]
+        if: github.event_name == 'pull_request'
+        id: fetch_last_commit_id_pr
+        run: echo "commit_id=$(git ls-remote --heads origin ${{ github.head_ref }} | cut -f 1)" >> $GITHUB_OUTPUT
+      - name: Fetch last commit id from remote branch [WORKFLOW DISPATCH]
+        if: github.event_name == 'workflow_dispatch'
+        id: fetch_last_commit_id_wd
+        run: echo "commit_id=$(git rev-parse origin/${{ steps.extract_branch.outputs.branch }})" >> $GITHUB_OUTPUT
+      - name: Test connectors
+        uses: ./.github/actions/run-airbyte-ci
+        with:
+          context: ${{ github.event_name == 'pull_request' && 'pull_request' || 'manual' }}
+          dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN_2 }}
+          docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }}
+          docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }}
+          gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }}
+          sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }}
+          git_branch: ${{ github.head_ref }}
+          git_revision: ${{ github.event_name == 'pull_request' && steps.fetch_last_commit_id_pr.outputs.commit_id || steps.fetch_last_commit_id_wd.outputs.commit_id }}
+          github_token: ${{ env.PAT }}
+          s3_build_cache_access_key_id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }}
+          s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }}
+          # A connector test can't take more than 5 hours to run (5 * 60 * 60 = 18000 seconds)
+          subcommand: "connectors --use-local-cdk --name source-shopify --name source-zendesk-support --execute-timeout=18000 test --global-status-check-context='CDK Changes - Connectors Tests'"

.github/workflows/community_ci.yml

@@ -30,7 +30,6 @@ jobs:
           filters: |
             protected_paths:
               - '.github/**'
-              - 'airbyte-ci/**'
 
       - name: Fail if changes in protected paths
         if: steps.check_for_changes_in_protected_paths.outputs.protected_paths == 'true'