[CI] Release spec for each version (#895)

# Description

What - Release spec for each version
Why - Support in versioning and infra for allowing api breakage

S3 structure before:
![Screenshot 2024-08-12 at 13 23
43](https://github.com/user-attachments/assets/f352812d-0c28-4712-9d60-c27a39a81bc6)

S3 structure after:
![Screenshot 2024-08-12 at 13 23
16](https://github.com/user-attachments/assets/648f4017-4a88-433e-8c8e-ae2e872ace02)
![Screenshot 2024-08-12 at 13 24
05](https://github.com/user-attachments/assets/074f49a9-714e-40db-9de0-c06712f67dd0)


## Type of change

Please leave one option from the following and delete the rest:

- [ ] Bug fix (non-breaking change which fixes an issue)
- [ ] New feature (non-breaking change which adds functionality)
- [ ] New Integration (non-breaking change which adds a new integration)
- [ ] Breaking change (fix or feature that would cause existing
functionality to not work as expected)
- [X] Non-breaking change (fix of existing functionality that will not
change current behavior)
- [ ] Documentation (added/updated documentation)

## Screenshots

Include screenshots from your environment showing how the resources of
the integration will look.

## API Documentation

Provide links to the API documentation used for this integration.

.github/workflows/release-integrations.yml

@@ -124,7 +124,6 @@ jobs:
       packages: write
       contents: read
     needs: release-integration
-    if: needs.release-integration.outputs.is_dev_version == 'false'
     steps:
       - name: Check out code
         uses: actions/checkout@v4
@@ -137,42 +136,69 @@ jobs:
           aws-region: ${{ secrets.AWS_REGION }}
       - name: Upload specifications to s3
         run: |
-          # Temporary file to store the concatenated YAML content
-          all_integrations_temp_file="all_integrations_temp_file.yaml"
-
-          # Temporary file to store each integration YAML content
-          specific_integration_temp_file="specific_integration_temp_file.yaml"
+          # Temporary file
+          temp_file="temp.yaml"
 
           # Output file name
-          output_file="index.json"
+          index_file="index.json"
 
           # AWS S3 bucket details
           aws_s3_bucket="ocean-registry"
 
+          # Fetch existing index file or create empty one of not exists
+          if aws s3 ls "s3://$aws_s3_bucket/$index_file" > /dev/null 2>&1; then
+            aws s3 cp "s3://$aws_s3_bucket/$index_file" $index_file
+            echo "Successfully fetched global index file from s3 bucket."
+          else
+            echo "Index file does not exist in the S3 bucket, Creating new one..."
+            echo "[]" > "$index_file"
+          fi
+
           # Find all ocean-spec.yaml files under the specified directory
           find integrations/*/.port -type f -name "spec.yaml" > file_list.txt
 
           while IFS= read -r file; do
           integration_dir=$(dirname "$file")
+          integration_name=$(echo "$integration_dir" | awk -F'/' '{print $2}')
+
           # Extract the type for pyproject.toml
           type=$(grep -E '^name = ".*"' "$integration_dir/../pyproject.toml" | cut -d'"' -f2)
 
           # Extract the version from pyproject.toml
           version=$(grep -E '^version = ".*"' "$integration_dir/../pyproject.toml" | cut -d'"' -f2)
 
-          # Store the integration's yaml file content into a temporary file
-          integration_dest="$(echo $integration_dir | awk -F'/' '{print $2}').json"
-          sed 's/^/  /' "$file" > "$specific_integration_temp_file"
-          echo "  type: $type" >> "$specific_integration_temp_file"
-          echo "  version: $version" >> "$specific_integration_temp_file"
-
-          yq -o=json . < "$specific_integration_temp_file" > "$integration_dest"
-          aws s3 cp "$integration_dest" "s3://$aws_s3_bucket/$integration_dest"
-
-          # Concatenate the YAML files into a temporary file
-          echo "- " >> "$all_integrations_temp_file"
-          cat "$specific_integration_temp_file" >> "$all_integrations_temp_file"
+          integration_spec="$integration_name-$version.json"
+          integration_dest="$integration_name/$integration_spec"
+          integration_legacy_dest="$integration_name.json" # TODO: legacy support, remove once not in use
+
+          # Convert YAML to JSON
+          yq -o json "$file" > "$temp_file"
+
+          # Add version attribute
+          jq --arg type "$type" --arg version "$version" '. + {type: $type, version: $version}' "$temp_file" > "$integration_spec"
+
+          # Upload integration's version manifest to s3
+          aws s3 cp "$integration_spec" "s3://$aws_s3_bucket/$integration_dest"
+          aws s3 cp "$integration_spec" "s3://$aws_s3_bucket/$integration_legacy_dest" # TODO: legacy support, remove once not in use
+          echo "Successfully uploaded $integration_spec to s3 bucket."
+
+          # Get the latest version of the current integration
+          latest_version=$(jq --arg type "$type" -r '.[] | select(.type == $type) | .version' "$index_file")
+
+          # Add integration's spec to global index file
+          regexp="^[0-9.]+$"
+          if [[ ! "$latest_version" ]]; then
+            # Add new integration spec if latest tag doesn't exist
+            jq --argjson new_spec "[$(cat "$integration_spec")]" '. += $new_spec' "$index_file" > "$temp_file"
+            mv "$temp_file" "$index_file"
+          elif [[ ! "$latest_version" =~ $regexp ]] || [[ "$version" =~ $regexp ]]; then
+            # Override global index file if released non-dev version or integration doesn't have non-dev latest tag
+            jq --argjson updated_spec "$(cat "$integration_spec")" --arg type "$type" \
+              'map(if .type == $type then $updated_spec else . end)' "$index_file" > "$temp_file"
+            mv "$temp_file" "$index_file"
+          fi
           done < file_list.txt
 
-          yq -o=json . < "$all_integrations_temp_file" > "$output_file"
-          aws s3 cp "$output_file" "s3://$aws_s3_bucket/$output_file"
+          # Upload global index file to s3
+          aws s3 cp "$index_file" "s3://$aws_s3_bucket/$index_file"
+          echo "Successfully uploaded $index_file to s3 bucket."