From c975947a57038ccb1703f759a68e534936368d6f Mon Sep 17 00:00:00 2001 From: omby8888 <160610297+omby8888@users.noreply.github.com> Date: Mon, 12 Aug 2024 17:04:44 +0300 Subject: [PATCH] [CI] Release spec for each version (#895) # Description What - Release spec for each version Why - Support in versioning and infra for allowing api breakage S3 structure before: ![Screenshot 2024-08-12 at 13 23 43](https://github.com/user-attachments/assets/f352812d-0c28-4712-9d60-c27a39a81bc6) S3 structure after: ![Screenshot 2024-08-12 at 13 23 16](https://github.com/user-attachments/assets/648f4017-4a88-433e-8c8e-ae2e872ace02) ![Screenshot 2024-08-12 at 13 24 05](https://github.com/user-attachments/assets/074f49a9-714e-40db-9de0-c06712f67dd0) ## Type of change Please leave one option from the following and delete the rest: - [ ] Bug fix (non-breaking change which fixes an issue) - [ ] New feature (non-breaking change which adds functionality) - [ ] New Integration (non-breaking change which adds a new integration) - [ ] Breaking change (fix or feature that would cause existing functionality to not work as expected) - [X] Non-breaking change (fix of existing functionality that will not change current behavior) - [ ] Documentation (added/updated documentation) ## Screenshots Include screenshots from your environment showing how the resources of the integration will look. ## API Documentation Provide links to the API documentation used for this integration. --- .github/workflows/release-integrations.yml | 68 +++++++++++++++------- 1 file changed, 47 insertions(+), 21 deletions(-) diff --git a/.github/workflows/release-integrations.yml b/.github/workflows/release-integrations.yml index f6f78c2b62..0c3ed64a28 100644 --- a/.github/workflows/release-integrations.yml +++ b/.github/workflows/release-integrations.yml @@ -124,7 +124,6 @@ jobs: packages: write contents: read needs: release-integration - if: needs.release-integration.outputs.is_dev_version == 'false' steps: - name: Check out code uses: actions/checkout@v4 @@ -137,42 +136,69 @@ jobs: aws-region: ${{ secrets.AWS_REGION }} - name: Upload specifications to s3 run: | - # Temporary file to store the concatenated YAML content - all_integrations_temp_file="all_integrations_temp_file.yaml" - - # Temporary file to store each integration YAML content - specific_integration_temp_file="specific_integration_temp_file.yaml" + # Temporary file + temp_file="temp.yaml" # Output file name - output_file="index.json" + index_file="index.json" # AWS S3 bucket details aws_s3_bucket="ocean-registry" + # Fetch existing index file or create empty one of not exists + if aws s3 ls "s3://$aws_s3_bucket/$index_file" > /dev/null 2>&1; then + aws s3 cp "s3://$aws_s3_bucket/$index_file" $index_file + echo "Successfully fetched global index file from s3 bucket." + else + echo "Index file does not exist in the S3 bucket, Creating new one..." + echo "[]" > "$index_file" + fi + # Find all ocean-spec.yaml files under the specified directory find integrations/*/.port -type f -name "spec.yaml" > file_list.txt while IFS= read -r file; do integration_dir=$(dirname "$file") + integration_name=$(echo "$integration_dir" | awk -F'/' '{print $2}') + # Extract the type for pyproject.toml type=$(grep -E '^name = ".*"' "$integration_dir/../pyproject.toml" | cut -d'"' -f2) # Extract the version from pyproject.toml version=$(grep -E '^version = ".*"' "$integration_dir/../pyproject.toml" | cut -d'"' -f2) - # Store the integration's yaml file content into a temporary file - integration_dest="$(echo $integration_dir | awk -F'/' '{print $2}').json" - sed 's/^/ /' "$file" > "$specific_integration_temp_file" - echo " type: $type" >> "$specific_integration_temp_file" - echo " version: $version" >> "$specific_integration_temp_file" - - yq -o=json . < "$specific_integration_temp_file" > "$integration_dest" - aws s3 cp "$integration_dest" "s3://$aws_s3_bucket/$integration_dest" - - # Concatenate the YAML files into a temporary file - echo "- " >> "$all_integrations_temp_file" - cat "$specific_integration_temp_file" >> "$all_integrations_temp_file" + integration_spec="$integration_name-$version.json" + integration_dest="$integration_name/$integration_spec" + integration_legacy_dest="$integration_name.json" # TODO: legacy support, remove once not in use + + # Convert YAML to JSON + yq -o json "$file" > "$temp_file" + + # Add version attribute + jq --arg type "$type" --arg version "$version" '. + {type: $type, version: $version}' "$temp_file" > "$integration_spec" + + # Upload integration's version manifest to s3 + aws s3 cp "$integration_spec" "s3://$aws_s3_bucket/$integration_dest" + aws s3 cp "$integration_spec" "s3://$aws_s3_bucket/$integration_legacy_dest" # TODO: legacy support, remove once not in use + echo "Successfully uploaded $integration_spec to s3 bucket." + + # Get the latest version of the current integration + latest_version=$(jq --arg type "$type" -r '.[] | select(.type == $type) | .version' "$index_file") + + # Add integration's spec to global index file + regexp="^[0-9.]+$" + if [[ ! "$latest_version" ]]; then + # Add new integration spec if latest tag doesn't exist + jq --argjson new_spec "[$(cat "$integration_spec")]" '. += $new_spec' "$index_file" > "$temp_file" + mv "$temp_file" "$index_file" + elif [[ ! "$latest_version" =~ $regexp ]] || [[ "$version" =~ $regexp ]]; then + # Override global index file if released non-dev version or integration doesn't have non-dev latest tag + jq --argjson updated_spec "$(cat "$integration_spec")" --arg type "$type" \ + 'map(if .type == $type then $updated_spec else . end)' "$index_file" > "$temp_file" + mv "$temp_file" "$index_file" + fi done < file_list.txt - yq -o=json . < "$all_integrations_temp_file" > "$output_file" - aws s3 cp "$output_file" "s3://$aws_s3_bucket/$output_file" + # Upload global index file to s3 + aws s3 cp "$index_file" "s3://$aws_s3_bucket/$index_file" + echo "Successfully uploaded $index_file to s3 bucket."