diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md index 4e30a808db..2d4b6c4e1f 100644 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ b/.github/PULL_REQUEST_TEMPLATE.md @@ -1,7 +1,9 @@ # Description What - + Why - + How - ## Type of change @@ -15,6 +17,40 @@ Please leave one option from the following and delete the rest: - [ ] Non-breaking change (fix of existing functionality that will not change current behavior) - [ ] Documentation (added/updated documentation) +

All tests should be run against the port production environment(using a testing org).

+ +### Core testing checklist + +- [ ] Integration able to create all default resources from scratch +- [ ] Resync finishes successfully +- [ ] Resync able to create entities +- [ ] Resync able to update entities +- [ ] Resync able to detect and delete entities +- [ ] Scheduled resync able to abort existing resync and start a new one +- [ ] Tested with at least 2 integrations from scratch +- [ ] Tested with Kafka and Polling event listeners +- [ ] Tested deletion of entities that don't pass the selector + + +### Integration testing checklist + +- [ ] Integration able to create all default resources from scratch +- [ ] Resync able to create entities +- [ ] Resync able to update entities +- [ ] Resync able to detect and delete entities +- [ ] Resync finishes successfully +- [ ] If new resource kind is added or updated in the integration, add example raw data, mapping and expected result to the `examples` folder in the integration directory. +- [ ] If resource kind is updated, run the integration with the example data and check if the expected result is achieved +- [ ] If new resource kind is added or updated, validate that live-events for that resource are working as expected +- [ ] Docs PR link [here](#) + +### Preflight checklist + +- [ ] Handled rate limiting +- [ ] Handled pagination +- [ ] Implemented the code in async +- [ ] Support Multi account + ## Screenshots Include screenshots from your environment showing how the resources of the integration will look. diff --git a/.github/workflows/apply-release.yml b/.github/workflows/apply-release.yml index 830cce2300..acee3aff11 100644 --- a/.github/workflows/apply-release.yml +++ b/.github/workflows/apply-release.yml @@ -41,7 +41,7 @@ jobs: git config --local user.email "action@github.com" git config --local user.name "GitHub Action" - ./scripts/bump-all.sh ^${{ steps.version.outputs.version }} + ./scripts/bump-all.sh ${{ steps.version.outputs.version }} - name: Open pull request diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 328e5ae9ad..d4745630a1 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -47,17 +47,26 @@ jobs: contents: read needs: [prepare-matrix] strategy: + max-parallel: 10 matrix: integration: ${{fromJson(needs.prepare-matrix.outputs.matrix)}} + platform: + - linux/amd64 + - linux/arm64 steps: - name: Check out code uses: actions/checkout@v4 + - name: Set up QEMU uses: docker/setup-qemu-action@v3 with: - platforms: linux/amd64,linux/arm64 + platforms: ${{ matrix.platform }} + - name: Set up Docker Buildx uses: docker/setup-buildx-action@v3 + with: + platforms: ${{ matrix.platform }} + - name: Prepare Docker images tags id: prepare_tags run: | @@ -73,6 +82,7 @@ jobs: dockerfile_path=integrations/_infra/Dockerfile if test -e $folder/../Dockerfile; then + echo "Choosing a custom Dockerfile for ${{ matrix.integration }}" dockerfile_path=$folder/../Dockerfile fi echo "dockerfile_path=$dockerfile_path" >> $GITHUB_OUTPUT @@ -91,14 +101,31 @@ jobs: echo "is_dev_version=false" >> $GITHUB_OUTPUT fi + - name: Get used docker base image + id: get-docker-image + run: | + echo "base_image=$(cat ${{ steps.prepare_tags.outputs.dockerfile_path }} | head -n 1 | awk -F '=' '{print $2}' )" >> $GITHUB_OUTPUT + + - name: Cache Docker images + uses: ScribeMD/docker-cache@0.5.0 + with: + key: docker-${{ matrix.integration }}-${{ steps.get-docker-image.outputs.base_image }}-${{ matrix.platform }} + - name: Build uses: docker/build-push-action@v6 with: context: . file: ${{ steps.prepare_tags.outputs.dockerfile_path }} - platforms: linux/amd64,linux/arm64 + platforms: ${{ matrix.platform }} push: false + load: true + cache-from: type=gha + cache-to: type=gha,mode=max tags: ${{ steps.prepare_tags.outputs.tags }} build-args: | BUILD_CONTEXT=${{ steps.prepare_tags.outputs.context_dir }} INTEGRATION_VERSION=${{ steps.prepare_tags.outputs.version }} + + - name: Verify Built Image + run: | + docker run --platform ${{ matrix.platform }} --rm --entrypoint bash ${{ steps.prepare_tags.outputs.tags }} -c 'ocean version' diff --git a/.github/workflows/core-test.yml b/.github/workflows/core-test.yml new file mode 100644 index 0000000000..46f53bd63d --- /dev/null +++ b/.github/workflows/core-test.yml @@ -0,0 +1,88 @@ +name: 🌊 Ocean Core Tests + +on: + pull_request: + workflow_dispatch: + +jobs: + detect-changes: + uses: ./.github/workflows/detect-changes-matrix.yml + test: + name: 🌊 Ocean Core Tests + needs: detect-changes + runs-on: ubuntu-latest + if: ${{ needs.detect-changes.outputs.core == 'true' }} + steps: + - name: Checkout Repo + uses: actions/checkout@v4 + + - name: Install poetry + run: pipx install poetry + + - name: Set up Python 3.11 + uses: actions/setup-python@v5 + with: + python-version: '3.11' + cache: 'poetry' + + - name: Install dependencies + run: | + make install + + - name: Unit Test Core + env: + PYTEST_ADDOPTS: --junitxml=junit/unit-test-results-ocean/core.xml + run: | + make test + + - name: Build core for smoke test + run: | + make build + + - name: Run fake integration for core test + env: + PORT_CLIENT_ID: ${{ secrets.PORT_CLIENT_ID }} + PORT_CLIENT_SECRET: ${{ secrets.PORT_CLIENT_SECRET }} + PORT_BASE_URL: ${{ secrets.PORT_BASE_URL }} + SMOKE_TEST_SUFFIX: ${{ github.run_id }} + run: | + ./scripts/run-smoke-test.sh + + - name: Smoke Test Core + env: + PYTEST_ADDOPTS: --junitxml=junit/smoke-test-results-ocean/core.xml + PORT_CLIENT_ID: ${{ secrets.PORT_CLIENT_ID }} + PORT_CLIENT_SECRET: ${{ secrets.PORT_CLIENT_SECRET }} + PORT_BASE_URL: ${{ secrets.PORT_BASE_URL }} + SMOKE_TEST_SUFFIX: ${{ github.run_id }} + run: | + make test/smoke + + - name: Cleanup Smoke Test + env: + PYTEST_ADDOPTS: --junitxml=junit/smoke-test-results-ocean/core.xml + PORT_CLIENT_ID: ${{ secrets.PORT_CLIENT_ID }} + PORT_CLIENT_SECRET: ${{ secrets.PORT_CLIENT_SECRET }} + PORT_BASE_URL: ${{ secrets.PORT_BASE_URL }} + SMOKE_TEST_SUFFIX: ${{ github.run_id }} + run: | + make test/smoke + + - name: Install current core for all integrations + run: | + echo "Installing local core for all integrations" + SCRIPT_TO_RUN='make install/local-core' make execute/all + + - name: Test all integrations with current core + run: | + echo "Testing all integrations with local core" + SCRIPT_TO_RUN="PYTEST_ADDOPTS=--junitxml=${PWD}/junit/test-results-core-change/\`pwd | xargs basename\`.xml make test" make execute/all + + - name: Publish Test Report + uses: mikepenz/action-junit-report@v4 + if: ${{ always() }} + with: + report_paths: '**/junit/**-test-results-**/*.xml' + include_passed: true + require_tests: true + fail_on_failure: true diff --git a/.github/workflows/create-new-sonarcloud-project.yml b/.github/workflows/create-new-sonarcloud-project.yml index 8518a61b38..e26f5d59f2 100644 --- a/.github/workflows/create-new-sonarcloud-project.yml +++ b/.github/workflows/create-new-sonarcloud-project.yml @@ -20,7 +20,7 @@ jobs: fetch-depth: 0 - name: Get all changed integrations id: changed-integrations - uses: tj-actions/changed-files@v44 + uses: tj-actions/changed-files@v45 with: json: true dir_names: true @@ -42,6 +42,7 @@ jobs: runs-on: ubuntu-latest name: Create new project for integrations strategy: + max-parallel: 5 matrix: ${{ fromJson(needs.pre-run.outputs.changed_integrations) }} steps: - name: Create integration variable diff --git a/.github/workflows/detect-changes-matrix.yml b/.github/workflows/detect-changes-matrix.yml new file mode 100644 index 0000000000..91dd9c31fc --- /dev/null +++ b/.github/workflows/detect-changes-matrix.yml @@ -0,0 +1,57 @@ +name: Detect Changes +on: + workflow_call: + outputs: + matrix: + value: ${{ jobs.detect-changes.outputs.matrix }} + description: "Matrix of changed integrations / Ocean Core per git commit changes" + integrations: + description: "Matrix of changed integrations per git commit changes" + value: ${{ jobs.detect-changes.outputs.integrations }} + core: + value: ${{ jobs.detect-changes.outputs.core }} + description: "Determine if any core changes per git commit changes" + +jobs: + detect-changes: + name: Detect changes + runs-on: ubuntu-latest + outputs: + matrix: ${{ steps.set-all-matrix.outputs.matrix }} + integrations: ${{ steps.set-all-matrix.outputs.integrations }} + core: ${{ steps.set-all-matrix.outputs.core }} + steps: + - name: Checkout Repo + uses: actions/checkout@v4 + + - name: Get list of changed files + id: changed-files + uses: tj-actions/changed-files@v45 + with: + dir_names: true + json: true + dir_names_max_depth: 2 + escape_json: false + files_yaml: | + core: + - '!integrations/**' + - '!scripts/*' + - '!scripts/*' + - '!./*.md' + integrations: + - 'integrations/**' + - '!integrations/**/*.md' + - '!integrations/_infra/*' + + - name: Set integrations and all matrix + id: set-all-matrix + run: | + INTEGRATIONS=$(node -e 'integrations=${{ steps.changed-files.outputs.integrations_all_changed_files }};console.log(JSON.stringify(integrations.map(integration => integration.split("/")[1])))') + HAS_CORE=${{ steps.changed-files.outputs.core_all_changed_files != '[]' }} + echo "Core changes : ${HAS_CORE}" + MATRIX=$(node -e "integrations=${INTEGRATIONS}; hasCore=${HAS_CORE}; console.log(JSON.stringify(hasCore ? integrations.concat(['.']) : integrations))") + echo "Integration changes : ${INTEGRATIONS}" + echo "All changes : ${MATRIX}" + echo "core=${HAS_CORE}" >> $GITHUB_OUTPUT + echo "integrations=${INTEGRATIONS}" >> $GITHUB_OUTPUT + echo "matrix=${MATRIX}" >> $GITHUB_OUTPUT diff --git a/.github/workflows/docker-images-security-scan.yml b/.github/workflows/docker-images-security-scan.yml new file mode 100644 index 0000000000..13d78e5443 --- /dev/null +++ b/.github/workflows/docker-images-security-scan.yml @@ -0,0 +1,136 @@ +name: Scan docker images +on: + workflow_dispatch: + inputs: + image: + type: choice + description: Image/s to scan + # This is a bit annoying, there's no real way to display the integrations dynamically in a dropdown for the action dispatcher + options: + - all + - aws + - azure-devops + - dynatrace + - fake-integration + - gcp + - jenkins + - kafka + - launchdarkly + - newrelic + - opencost + - pagerduty + - servicenow + - sonarqube + - terraform-cloud + - argocd + - azure + - datadog + - firehydrant + - gitlab + - jira + - kubecost + - linear + - octopus + - opsgenie + - sentry + - snyk + - statuspage + - wiz + +jobs: + detect-images: + runs-on: ubuntu-latest + outputs: + images: ${{ steps.set-images.outputs.images }} + steps: + - name: Checkout Repo + uses: actions/checkout@v4 + + - name: Determine which image to scan + id: set-images + run: | + PROJECTS=$(ls --color=never ./integrations | grep -Ev '_infra') + if [[ "${{ inputs.image }}" != "all" ]]; then + PROJECTS="${{ inputs.image }}" + fi + IMAGES_WITH_VERSIONS=() + for PROJECT in ${PROJECTS}; do + if [[ ! -f ./integrations/"${PROJECT}"/pyproject.toml ]]; then + continue + fi + VERSION=$(cat ./integrations/"${PROJECT}"/pyproject.toml | grep -E '^version = "(.*)"$' | awk -F ' ' '{print $3};' | sed 's/"//g') + if [[ -n ${VERSION} ]]; then + IMAGES_WITH_VERSIONS+=( "${PROJECT}:${VERSION}" ) + fi + done + IMAGES=$(echo "${IMAGES_WITH_VERSIONS[@]}" | jq -R -s -c 'split(" ") | map(select(length > 0))') + echo "Images to scan: ${IMAGES}" + echo "images=${IMAGES}" >> $GITHUB_OUTPUT + scan-images: + needs: detect-images + runs-on: ubuntu-latest + strategy: + max-parallel: 2 + matrix: + image: ${{ fromJson(needs.detect-images.outputs.images) }} + steps: + - name: Checkout Repo + uses: actions/checkout@v4 + + - name: Set up QEMU + uses: docker/setup-qemu-action@v3 + with: + platforms: linux/amd64,linux/arm64 + + - name: Set up Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Extract version and image tag + id: enrich-version + run: | + INTEGRATION=$(echo "${{ matrix.image }}" | awk -F ':' '{print $1};') + VERSION=$(echo "${{ matrix.image }}" | awk -F ':' '{print $2};') + IDENTIFIER="${INTEGRATION}-${VERSION}-${{ github.sha }}" + IMAGE_FULL_TAG="port-ocean-security-tests-${INTEGRATION}:${VERSON}${{ github.sha }}" + echo "integration=${INTEGRATION}" >> ${GITHUB_OUTPUT} + echo "version=${VERSION}" >> ${GITHUB_OUTPUT} + echo "identifier=${IDENTIFIER}" >> ${GITHUB_OUTPUT} + echo "image_tag=${IMAGE_FULL_TAG}" >> ${GITHUB_OUTPUT} + + - name: Build + uses: docker/build-push-action@v6 + with: + context: . + file: ./integrations/_infra/Dockerfile + platforms: linux/amd64 + push: false + tags: ${{ steps.enrich-version.outputs.image_tag }} + load: true + cache-from: type=gha + cache-to: type=gha,mode=max + build-args: | + BUILD_CONTEXT=./integrations/${{ steps.enrich-version.outputs.integration }} + INTEGRATION_VERSION=${{ steps.enrich-version.outputs.version }} + + - name: Run Trivy vulnerability scanner + uses: aquasecurity/trivy-action@0.24.0 + with: + image-ref: ${{ steps.enrich-version.outputs.image_tag }} + ignore-unfixed: true + vuln-type: 'os,library' + severity: 'CRITICAL,HIGH' + output: trivy-${{ steps.enrich-version.outputs.integration }}.txt + + - name: Publish Trivy Output to Summary + run: | + if [[ -s trivy-${{ steps.enrich-version.outputs.integration }}.txt ]]; then + { + echo "### Security Output" + echo "
Click to expand" + echo "" + echo '```terraform' + cat trivy-${{ steps.enrich-version.outputs.integration }}.txt + echo '```' + echo "
" + } >> $GITHUB_STEP_SUMMARY + fi diff --git a/.github/workflows/integrations-test.yml b/.github/workflows/integrations-test.yml new file mode 100644 index 0000000000..a645eb6828 --- /dev/null +++ b/.github/workflows/integrations-test.yml @@ -0,0 +1,51 @@ +name: Integrations Test + +on: + pull_request: + workflow_dispatch: + +jobs: + detect-changes: + uses: ./.github/workflows/detect-changes-matrix.yml + test: + name: ${{ format('🚢 {0}', matrix.folder) }} + needs: detect-changes + runs-on: ubuntu-latest + if: ${{ needs.detect-changes.outputs.integrations != '[]' }} + strategy: + max-parallel: 5 + matrix: + folder: ${{ fromJson(needs.detect-changes.outputs.integrations) }} + steps: + - name: Checkout Repo + uses: actions/checkout@v4 + + - name: Install poetry + run: pipx install poetry + + - name: Set up Python 3.11 + uses: actions/setup-python@v5 + with: + python-version: '3.11' + cache: 'poetry' + + - name: Install dependencies + working-directory: ${{ format('integrations/{0}', matrix.folder) }} + run: | + make install + + - name: Test + working-directory: ${{ format('integrations/{0}', matrix.folder) }} + env: + PYTEST_ADDOPTS: --junitxml=junit/test-results-${{ format('integrations/{0}', matrix.folder) }}.xml + run: | + make test + + - name: Publish Test Report + uses: mikepenz/action-junit-report@v4 + if: ${{ always() }} + with: + report_paths: '**/junit/test-results-**/*.xml' + include_passed: true + require_tests: true + fail_on_failure: true diff --git a/.github/workflows/lint.yml b/.github/workflows/lint.yml index 711622e0c5..f73f50c903 100644 --- a/.github/workflows/lint.yml +++ b/.github/workflows/lint.yml @@ -6,54 +6,33 @@ on: jobs: detect-changes: - name: Detect changes - runs-on: ubuntu-latest - outputs: - matrix: ${{ steps.set-matrix.outputs.matrix }} - steps: - - name: Checkout Repo - uses: actions/checkout@v4 - - - name: Get list of changed files - id: changed-files - uses: tj-actions/changed-files@v44.5.2 - - name: Set matrix - id: set-matrix - run: | - folders_to_ignore="integrations/*/LICENSE.md|integrations/*/README.md|integrations/*/CONTRIBUTING.md|integrations/*/CHANGELOG.md" - changed_folders=$(echo "${{ steps.changed-files.outputs.all_changed_files }}" | tr ' ' '\n' | grep '^integrations/' | grep -v '^($folders_to_ignore)' | cut -d'/' -f2 | sort | uniq) - if [ -z "$changed_folders" ]; then - changed_folders="" - fi - - folders_to_ignore="integrations/|scripts/|assets/|docs/|LICENSE.md|README.md|CONTRIBUTING.md|CHANGELOG.md" - other_changes=$(echo "${{ steps.changed-files.outputs.all_changed_files }}" | tr ' ' '\n' | grep -v '^($folders_to_ignore)' | wc -l) - if [ "$other_changes" -ne 0 ]; then - # Add the root directory to the matrix if there are changes outside the integrations folder - changed_folders=$(echo -e "$changed_folders\n.") - fi - - matrix=$(echo "$changed_folders" | jq -R -s -c 'split("\n") | map(select(length > 0))') - echo "matrix=$matrix" >> $GITHUB_OUTPUT - + uses: ./.github/workflows/detect-changes-matrix.yml lint: name: ${{ matrix.folder == '.' && '🌊 Ocean Core' || format('🚢 {0}', matrix.folder) }} needs: detect-changes runs-on: ubuntu-latest strategy: + max-parallel: 5 matrix: folder: ${{ fromJson(needs.detect-changes.outputs.matrix) }} steps: + - name: Checkout Repo + uses: actions/checkout@v4 + + - name: Install poetry + run: pipx install poetry + - name: Set up Python 3.11 uses: actions/setup-python@v5 with: python-version: '3.11' - - name: Checkout Repo - uses: actions/checkout@v4 + cache: 'poetry' + - name: Install dependencies working-directory: ${{ matrix.folder != '.' && format('integrations/{0}', matrix.folder) || '.' }} run: | make install + - name: Lint working-directory: ${{ matrix.folder != '.' && format('integrations/{0}', matrix.folder) || '.' }} run: | diff --git a/.github/workflows/release-integrations.yml b/.github/workflows/release-integrations.yml index 0c3ed64a28..f79a312754 100644 --- a/.github/workflows/release-integrations.yml +++ b/.github/workflows/release-integrations.yml @@ -56,6 +56,8 @@ jobs: contents: read needs: [prepare-matrix] strategy: + # limit the number of parallel jobs to avoid hitting the ghcr.io rate limit + max-parallel: 5 matrix: integration: ${{fromJson(needs.prepare-matrix.outputs.matrix)}} steps: @@ -113,6 +115,8 @@ jobs: file: ${{ steps.prepare_tags.outputs.dockerfile_path }} platforms: linux/amd64,linux/arm64 push: true + cache-from: type=gha + cache-to: type=gha,mode=max tags: ${{ steps.prepare_tags.outputs.tags }} build-args: | BUILD_CONTEXT=${{ steps.prepare_tags.outputs.context_dir }} @@ -146,59 +150,64 @@ jobs: aws_s3_bucket="ocean-registry" # Fetch existing index file or create empty one of not exists - if aws s3 ls "s3://$aws_s3_bucket/$index_file" > /dev/null 2>&1; then - aws s3 cp "s3://$aws_s3_bucket/$index_file" $index_file - echo "Successfully fetched global index file from s3 bucket." + if aws s3 ls "s3://$aws_s3_bucket/$index_file" >/dev/null 2>&1; then + aws s3 cp "s3://$aws_s3_bucket/$index_file" $index_file + echo "Successfully fetched global index file from s3 bucket." else - echo "Index file does not exist in the S3 bucket, Creating new one..." - echo "[]" > "$index_file" + echo "Index file does not exist in the S3 bucket, Creating new one..." + echo "[]" >"$index_file" fi # Find all ocean-spec.yaml files under the specified directory - find integrations/*/.port -type f -name "spec.yaml" > file_list.txt + find integrations/*/.port -type f -name "spec.yaml" >file_list.txt while IFS= read -r file; do - integration_dir=$(dirname "$file") - integration_name=$(echo "$integration_dir" | awk -F'/' '{print $2}') - - # Extract the type for pyproject.toml - type=$(grep -E '^name = ".*"' "$integration_dir/../pyproject.toml" | cut -d'"' -f2) - - # Extract the version from pyproject.toml - version=$(grep -E '^version = ".*"' "$integration_dir/../pyproject.toml" | cut -d'"' -f2) - - integration_spec="$integration_name-$version.json" - integration_dest="$integration_name/$integration_spec" - integration_legacy_dest="$integration_name.json" # TODO: legacy support, remove once not in use - - # Convert YAML to JSON - yq -o json "$file" > "$temp_file" - - # Add version attribute - jq --arg type "$type" --arg version "$version" '. + {type: $type, version: $version}' "$temp_file" > "$integration_spec" - - # Upload integration's version manifest to s3 - aws s3 cp "$integration_spec" "s3://$aws_s3_bucket/$integration_dest" - aws s3 cp "$integration_spec" "s3://$aws_s3_bucket/$integration_legacy_dest" # TODO: legacy support, remove once not in use - echo "Successfully uploaded $integration_spec to s3 bucket." - - # Get the latest version of the current integration - latest_version=$(jq --arg type "$type" -r '.[] | select(.type == $type) | .version' "$index_file") - - # Add integration's spec to global index file - regexp="^[0-9.]+$" - if [[ ! "$latest_version" ]]; then - # Add new integration spec if latest tag doesn't exist - jq --argjson new_spec "[$(cat "$integration_spec")]" '. += $new_spec' "$index_file" > "$temp_file" - mv "$temp_file" "$index_file" - elif [[ ! "$latest_version" =~ $regexp ]] || [[ "$version" =~ $regexp ]]; then - # Override global index file if released non-dev version or integration doesn't have non-dev latest tag - jq --argjson updated_spec "$(cat "$integration_spec")" --arg type "$type" \ - 'map(if .type == $type then $updated_spec else . end)' "$index_file" > "$temp_file" - mv "$temp_file" "$index_file" - fi - done < file_list.txt - - # Upload global index file to s3 - aws s3 cp "$index_file" "s3://$aws_s3_bucket/$index_file" - echo "Successfully uploaded $index_file to s3 bucket." + integration_dir=$(dirname "$file") + integration_name=$(echo "$integration_dir" | awk -F'/' '{print $2}') + + # Extract the type for pyproject.toml + type=$(grep -E '^name = ".*"' "$integration_dir/../pyproject.toml" | cut -d'"' -f2) + + # Extract the version from pyproject.toml + version=$(grep -E '^version = ".*"' "$integration_dir/../pyproject.toml" | cut -d'"' -f2) + + integration_spec="$integration_name-$version.json" + integration_dest="$integration_name/$integration_spec" + integration_legacy_dest="$integration_name.json" # TODO: legacy support, remove once not in use + + # Convert YAML to JSON + yq -o json "$file" >"$temp_file" + + # Add version attribute + jq --arg type "$type" --arg version "$version" '. + {type: $type, version: $version}' "$temp_file" >"$integration_spec" + + if [[ ! ${version} =~ ^.+-dev$ ]]; then + # Upload integration's version manifest to s3 + aws s3 cp "$integration_spec" "s3://$aws_s3_bucket/$integration_dest" + aws s3 cp "$integration_spec" "s3://$aws_s3_bucket/$integration_legacy_dest" # TODO: legacy support, remove once not in use + echo "Successfully uploaded $integration_spec to s3 bucket." + + # Get the latest version of the current integration + latest_version=$(jq --arg type "$type" -r '.[] | select(.type == $type) | .version' "$index_file") + + # Add integration's spec to global index file + regexp="^[0-9.]+$" + if [[ ! "$latest_version" ]]; then + # Add new integration spec if latest tag doesn't exist + jq --argjson new_spec "[$(cat "$integration_spec")]" '. += $new_spec' "$index_file" >"$temp_file" + mv "$temp_file" "$index_file" + elif [[ ! "$latest_version" =~ $regexp ]] || [[ "$version" =~ $regexp ]]; then + # Override global index file if released non-dev version or integration doesn't have non-dev latest tag + jq --argjson updated_spec "$(cat "$integration_spec")" --arg type "$type" \ + 'map(if .type == $type then $updated_spec else . end)' "$index_file" >"$temp_file" + mv "$temp_file" "$index_file" + fi + + # Upload global index file to s3 + aws s3 cp "$index_file" "s3://$aws_s3_bucket/$index_file" + echo "Successfully uploaded $index_file to s3 bucket." + else + echo "Did not upload integration spec since the version was in development ('${version}')" + fi + + done 0 }} strategy: + max-parallel: 5 matrix: ${{ fromJson(needs.pre-run.outputs.changed_integrations) }} runs-on: ubuntu-latest steps: diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml deleted file mode 100644 index 86316706d6..0000000000 --- a/.github/workflows/test.yml +++ /dev/null @@ -1,67 +0,0 @@ -name: Test - -on: - pull_request: - workflow_dispatch: - -jobs: - detect-changes: - name: Detect changes - runs-on: ubuntu-latest - outputs: - matrix: ${{ steps.set-matrix.outputs.matrix }} - steps: - - name: Checkout Repo - uses: actions/checkout@v4 - - - name: Get list of changed files - id: changed-files - uses: tj-actions/changed-files@v44.5.2 - - name: Set matrix - id: set-matrix - run: | - folders_to_ignore="integrations/*/LICENSE.md|integrations/*/README.md|integrations/*/CONTRIBUTING.md|integrations/*/CHANGELOG.md" - changed_folders=$(echo "${{ steps.changed-files.outputs.all_changed_files }}" | tr ' ' '\n' | grep '^integrations/' | grep -v '^($folders_to_ignore)' | cut -d'/' -f2 | sort | uniq) - if [ -z "$changed_folders" ]; then - changed_folders="" - fi - - folders_to_ignore="integrations/|scripts/|assets/|docs/|LICENSE.md|README.md|CONTRIBUTING.md|CHANGELOG.md" - other_changes=$(echo "${{ steps.changed-files.outputs.all_changed_files }}" | tr ' ' '\n' | grep -v '^($folders_to_ignore)' | wc -l) - if [ "$other_changes" -ne 0 ]; then - # Add the root directory to the matrix if there are changes outside the integrations folder - changed_folders=$(echo -e "$changed_folders\n.") - fi - - matrix=$(echo "$changed_folders" | jq -R -s -c 'split("\n") | map(select(length > 0))') - echo "matrix=$matrix" >> $GITHUB_OUTPUT - - test: - name: ${{ matrix.folder == '.' && '🌊 Ocean Core' || format('🚢 {0}', matrix.folder) }} - needs: detect-changes - runs-on: ubuntu-latest - strategy: - matrix: - folder: ${{ fromJson(needs.detect-changes.outputs.matrix) }} - steps: - - name: Set up Python 3.11 - uses: actions/setup-python@v5 - with: - python-version: '3.11' - - name: Checkout Repo - uses: actions/checkout@v4 - - name: Install dependencies - working-directory: ${{ matrix.folder != '.' && format('integrations/{0}', matrix.folder) || '.' }} - run: | - make install - - name: Test - working-directory: ${{ matrix.folder != '.' && format('integrations/{0}', matrix.folder) || '.' }} - env: - PYTEST_ADDOPTS: --junitxml=junit/test-results-${{ matrix.folder != '.' && format('integrations/{0}', matrix.folder) || '.' }}.xml - run: | - make test - - name: Publish Test Report - uses: mikepenz/action-junit-report@v4 - if: ${{ always() }} - with: - report_paths: '**/junit/test-results-**/*.xml' diff --git a/.gitignore b/.gitignore index ca6e11d3da..28e67c9209 100644 --- a/.gitignore +++ b/.gitignore @@ -159,3 +159,8 @@ cython_debug/ # VSCode .vscode/* !.vscode/launch.json + +# Junit + +junit/* +**/junit/* diff --git a/.idea/watcherTasks.xml b/.idea/watcherTasks.xml index 6df1a76d86..78fc450138 100644 --- a/.idea/watcherTasks.xml +++ b/.idea/watcherTasks.xml @@ -1,7 +1,7 @@ - +