forked from airbytehq/airbyte
-
Notifications
You must be signed in to change notification settings - Fork 3
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Update with master
- Loading branch information
Showing
5,514 changed files
with
651,381 additions
and
351,811 deletions.
The diff you're trying to view is too large. We only load the first 3000 changed files.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,36 @@ | ||
| name: "Send PagerDuty event" | ||
| description: "Use Event API V2 to send a PagerDuty event." | ||
| inputs: | ||
| integration_key: | ||
| description: "The integration key for the PagerDuty service." | ||
| required: true | ||
| summary: | ||
| description: "A brief text summary of the event." | ||
| required: true | ||
| severity: | ||
| description: "The severity of the event. (info, warning, error, critical)" | ||
| required: true | ||
| source: | ||
| description: "Specific human-readable unique identifier, such as a hostname, for the system having the problem." | ||
| required: true | ||
|
|
||
| runs: | ||
| using: "composite" | ||
| steps: | ||
| - name: Send PagerDuty event | ||
| id: send-pager-duty-event | ||
| shell: bash | ||
| run: | | ||
| curl --request 'POST' \ | ||
| --fail \ | ||
| --url "https://events.pagerduty.com/v2/enqueue" \ | ||
| --header 'Content-Type: application/json' \ | ||
| --data '{ | ||
| "payload": { | ||
| "summary": "${{ inputs.summary }}", | ||
| "severity": "${{ inputs.severity }}", | ||
| "source": "${{ inputs.source }}" | ||
| }, | ||
| "event_action": "trigger", | ||
| "routing_key": "${{ inputs.integration_key }}" | ||
| }' |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,125 @@ | ||
| name: Approve Regression Tests | ||
| permissions: | ||
| pull-requests: write | ||
| statuses: write | ||
| on: | ||
| workflow_dispatch: | ||
| inputs: | ||
| pr: | ||
| description: "Pull request number. Used to pull the proper branch ref, including on forks." | ||
| type: number | ||
| required: false | ||
| comment-id: | ||
| description: "Optional. The comment-id of the slash command. Used to update the comment with the status." | ||
| required: false | ||
| connector-name: | ||
| description: "Optional. Name of the connector whose regression tests are approved." | ||
| required: false | ||
|
|
||
| # These must be declared, but they are unused and ignored. | ||
| # TODO: Infer 'repo' and 'gitref' from PR number on other workflows, so we can remove these. | ||
| repo: | ||
| description: "Repo (Ignored)" | ||
| required: false | ||
| default: "airbytehq/airbyte" | ||
| gitref: | ||
| description: "Ref (Ignored)" | ||
| required: false | ||
|
|
||
| run-name: "Approve Regression Tests #${{ github.event.inputs.pr }}" | ||
|
|
||
| jobs: | ||
| approve-regression-tests: | ||
| name: "Approve Regression Tests" | ||
| runs-on: ubuntu-latest | ||
| steps: | ||
| - name: Get job variables | ||
| id: job-vars | ||
| env: | ||
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | ||
| shell: bash | ||
| run: | | ||
| PR_JSON=$(gh api repos/${{ github.repository }}/pulls/${{ github.event.inputs.pr }}) | ||
| echo "PR_JSON: $PR_JSON" | ||
| echo "repo=$(echo "$PR_JSON" | jq -r .head.repo.full_name)" >> $GITHUB_OUTPUT | ||
| BRANCH=$(echo "$PR_JSON" | jq -r .head.ref) | ||
| echo "branch=$BRANCH" >> $GITHUB_OUTPUT | ||
| echo "run-url=https://github.com/$GITHUB_REPOSITORY/actions/runs/$GITHUB_RUN_ID" >> $GITHUB_OUTPUT | ||
| LATEST_COMMIT=$(gh api repos/${{ github.repository }}/commits/$BRANCH | jq -r .sha) | ||
| echo "latest_commit=$LATEST_COMMIT" >> $GITHUB_OUTPUT | ||
| - name: Append comment with job run link | ||
| # If comment-id is not provided, this will create a new | ||
| # comment with the job run link. | ||
| id: first-comment-action | ||
| uses: peter-evans/create-or-update-comment@v4 | ||
| with: | ||
| comment-id: ${{ github.event.inputs.comment-id }} | ||
| issue-number: ${{ github.event.inputs.pr }} | ||
| body: | | ||
| > [Check job output.][1] | ||
| [1]: ${{ steps.job-vars.outputs.run-url }} | ||
| - name: Approve regression tests | ||
| id: approve-regression-tests | ||
| if: github.event.inputs.connector-name != null | ||
| run: | | ||
| echo "approving regression test status check for ${{ github.event.inputs.connector-name }} if it exists ...." | ||
| response=$(curl --write-out '%{http_code}' --silent --output /dev/null \ | ||
| --request POST \ | ||
| --url ${{ github.api_url }}/repos/${{ github.repository }}/statuses/${{ steps.job-vars.outputs.latest_commit }} \ | ||
| --header 'authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' \ | ||
| --header 'content-type: application/json' \ | ||
| --data '{ | ||
| "state": "success", | ||
| "context": "Regression Tests on ${{ github.event.inputs.connector-name }}", | ||
| "target_url": "https://github.com/airbytehq/airbyte/tree/master/airbyte-ci/connectors/live-tests" | ||
| }') | ||
| if [ $response -ne 201 ]; then | ||
| echo "Failed to approve regression tests for ${{ github.event.inputs.connector-name }}. HTTP status code: $response" | ||
| exit 1 | ||
| else | ||
| echo "Regression tests for ${{ github.event.inputs.connector-name }} approved." | ||
| fi | ||
| - name: Update global regression test approval | ||
| id: update-global-regression-test-approval | ||
| if: github.event.inputs.connector-name == null | ||
| run: | | ||
| echo "updating regression test approval status check if it exists ...." | ||
| response=$(curl --write-out '%{http_code}' --silent --output /dev/null \ | ||
| --request POST \ | ||
| --url ${{ github.api_url }}/repos/${{ github.repository }}/statuses/${{ steps.job-vars.outputs.latest_commit }} \ | ||
| --header 'authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' \ | ||
| --header 'content-type: application/json' \ | ||
| --data '{ | ||
| "state": "success", | ||
| "context": "Regression Test Results Reviewed and Approved", | ||
| "target_url": "https://github.com/airbytehq/airbyte/tree/master/airbyte-ci/connectors/live-tests" | ||
| }') | ||
| if [ $response -ne 201 ]; then | ||
| echo "Failed to update regression test approval status check. HTTP status code: $response" | ||
| exit 1 | ||
| else | ||
| echo "Regression test status check updated." | ||
| fi | ||
| - name: Append success comment | ||
| uses: peter-evans/create-or-update-comment@v4 | ||
| if: success() | ||
| with: | ||
| comment-id: ${{ steps.first-comment-action.outputs.comment-id }} | ||
| reactions: "+1" | ||
| body: | | ||
| > ✅ Approving regression tests | ||
| - name: Append failure comment | ||
| uses: peter-evans/create-or-update-comment@v4 | ||
| if: failure() | ||
| with: | ||
| comment-id: ${{ steps.first-comment-action.outputs.comment-id }} | ||
| reactions: confused | ||
| body: | | ||
| > ❌ Regression test approval failed |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,94 @@ | ||
| name: CDK Changes - Connectors Tests | ||
|
|
||
| concurrency: | ||
| # This is the name of the concurrency group. It is used to prevent concurrent runs of the same workflow. | ||
| # | ||
| # - github.head_ref is only defined on PR runs, it makes sure that the concurrency group is unique for pull requests | ||
| # ensuring that only one run per pull request is active at a time. | ||
| # | ||
| # - github.run_id is defined on all runs, it makes sure that the concurrency group is unique for workflow dispatches. | ||
| # This allows us to run multiple workflow dispatches in parallel. | ||
| group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} | ||
| cancel-in-progress: true | ||
|
|
||
| on: | ||
| workflow_dispatch: | ||
| pull_request: | ||
| types: | ||
| - opened | ||
| - synchronize | ||
| jobs: | ||
| cdk_changes: | ||
| runs-on: ubuntu-latest | ||
| outputs: | ||
| python_cdk: ${{ steps.changes.outputs.python_cdk }} | ||
| permissions: | ||
| statuses: write | ||
| steps: | ||
| - name: Checkout Airbyte | ||
| if: github.event_name != 'pull_request' | ||
| uses: actions/checkout@v4 | ||
| - id: changes | ||
| uses: dorny/paths-filter@v2 | ||
| with: | ||
| filters: | | ||
| python_cdk: | ||
| - 'airbyte-cdk/python/airbyte_cdk/**' | ||
| - 'airbyte-cdk/python/bin/**' | ||
| - 'airbyte-cdk/python/poetry.lock' | ||
| - 'airbyte-cdk/python/pyproject.toml' | ||
| # The Connector CI Tests is a status check emitted by airbyte-ci | ||
| # We make it pass once we have determined that there are no changes to the connectors | ||
| - name: "Skip Connectors CI tests" | ||
| if: steps.changes.outputs.python_cdk != 'true' && github.event_name == 'pull_request' | ||
| run: | | ||
| curl --request POST \ | ||
| --url https://api.github.com/repos/${{ github.repository }}/statuses/${{ github.event.pull_request.head.sha }} \ | ||
| --header 'authorization: Bearer ${{ secrets.GITHUB_TOKEN }}' \ | ||
| --header 'content-type: application/json' \ | ||
| --data '{ | ||
| "state": "success", | ||
| "context": "CDK Changes - Connectors Tests", | ||
| "target_url": "${{ github.event.workflow_run.html_url }}" | ||
| }' \ | ||
| connectors_ci: | ||
| needs: cdk_changes | ||
| # We only run the Connectors CI job if there are changes to the connectors on a non-forked PR | ||
| # Forked PRs are handled by the community_ci.yml workflow | ||
| # If the condition is not met the job will be skipped (it will not fail) | ||
| if: (github.event_name == 'pull_request' && needs.cdk_changes.outputs.python_cdk == 'true' && github.event.pull_request.head.repo.fork != true) || github.event_name == 'workflow_dispatch' | ||
| name: Connectors CI | ||
| runs-on: connector-test-large | ||
| timeout-minutes: 360 # 6 hours | ||
| steps: | ||
| - name: Checkout Airbyte | ||
| uses: actions/checkout@v4 | ||
| - name: Check PAT rate limits | ||
| run: | | ||
| ./tools/bin/find_non_rate_limited_PAT \ | ||
| ${{ secrets.GH_PAT_MAINTENANCE_OSS }} | ||
| - name: Fetch last commit id from remote branch [PULL REQUESTS] | ||
| if: github.event_name == 'pull_request' | ||
| id: fetch_last_commit_id_pr | ||
| run: echo "commit_id=$(git ls-remote --heads origin ${{ github.head_ref }} | cut -f 1)" >> $GITHUB_OUTPUT | ||
| - name: Fetch last commit id from remote branch [WORKFLOW DISPATCH] | ||
| if: github.event_name == 'workflow_dispatch' | ||
| id: fetch_last_commit_id_wd | ||
| run: echo "commit_id=$(git rev-parse origin/${{ steps.extract_branch.outputs.branch }})" >> $GITHUB_OUTPUT | ||
| - name: Test connectors | ||
| uses: ./.github/actions/run-airbyte-ci | ||
| with: | ||
| context: ${{ github.event_name == 'pull_request' && 'pull_request' || 'manual' }} | ||
| dagger_cloud_token: ${{ secrets.DAGGER_CLOUD_TOKEN_2 }} | ||
| docker_hub_password: ${{ secrets.DOCKER_HUB_PASSWORD }} | ||
| docker_hub_username: ${{ secrets.DOCKER_HUB_USERNAME }} | ||
| gcp_gsm_credentials: ${{ secrets.GCP_GSM_CREDENTIALS }} | ||
| sentry_dsn: ${{ secrets.SENTRY_AIRBYTE_CI_DSN }} | ||
| git_branch: ${{ github.head_ref }} | ||
| git_revision: ${{ github.event_name == 'pull_request' && steps.fetch_last_commit_id_pr.outputs.commit_id || steps.fetch_last_commit_id_wd.outputs.commit_id }} | ||
| github_token: ${{ env.PAT }} | ||
| s3_build_cache_access_key_id: ${{ secrets.SELF_RUNNER_AWS_ACCESS_KEY_ID }} | ||
| s3_build_cache_secret_key: ${{ secrets.SELF_RUNNER_AWS_SECRET_ACCESS_KEY }} | ||
| # A connector test can't take more than 5 hours to run (5 * 60 * 60 = 18000 seconds) | ||
| subcommand: "connectors --use-local-cdk --name source-shopify --name source-zendesk-support --execute-timeout=18000 test --global-status-check-context='CDK Changes - Connectors Tests'" |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.