Update Dependencies From Metadata (Retrieve, Metadata, Compile, Test, Create PR) #1103
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Update Dependencies From Metadata (Retrieve, Metadata, Compile, Test, Create PR) | |
| on: | |
| workflow_dispatch: | |
| schedule: | |
| - cron: '57 13 * * *' # daily at 13:57 UTC | |
| jobs: | |
| retrieve: | |
| name: Retrieve New Versions and Generate Metadata | |
| runs-on: ubuntu-latest | |
| outputs: | |
| metadata-filepath: ${{ steps.retrieve.outputs.metadata-filepath }} | |
| metadata-json: ${{ steps.retrieve.outputs.metadata-json }} | |
| # from-source-metadata-filepath is the path to a file containing a subset | |
| # of metadata-json entries for NON-compiled dependencies | |
| from-source-metadata-filepath: ${{ steps.retrieve.outputs.from-source-metadata-filepath }} | |
| # compilation-json is a subset of metadata-json entries which are missing | |
| # a `checksum` and `uri` | |
| compilation-json: ${{ steps.retrieve.outputs.compilation-json }} | |
| id: ${{ steps.retrieve.outputs.id }} | |
| length: ${{ steps.retrieve.outputs.length }} | |
| compilation-length: ${{ steps.retrieve.outputs.compilation-length }} | |
| steps: | |
| - name: Check out code | |
| uses: actions/checkout@v4 | |
| - name: Setup Go | |
| uses: actions/setup-go@v5 | |
| with: | |
| go-version: 'stable' | |
| - name: Run Retrieve | |
| id: retrieve | |
| working-directory: dependency | |
| run: | | |
| #!/usr/bin/env bash | |
| set -euo pipefail | |
| shopt -s inherit_errexit | |
| OUTPUT="/tmp/metadata.json" | |
| make retrieve \ | |
| buildpackTomlPath="${{ github.workspace }}/buildpack.toml" \ | |
| output="${OUTPUT}" | |
| id=$(jq -r .[0].id < "${OUTPUT}") | |
| content=$(jq -r < "${OUTPUT}") | |
| length=$(echo $content | jq -r '. | length') | |
| compilation=$(echo $content | jq -r 'map(select(.checksum == null and .uri == null))'?) | |
| complength=$(echo $compilation | jq -r '. | length') | |
| echo $content | jq -r 'map(select(.checksum != null and .uri != null))'? > "/tmp/from-source-metadata.json" | |
| echo "from-source-metadata-filepath=/tmp/from-source-metadata.json" >> "$GITHUB_OUTPUT" | |
| delimiter="$(uuidgen)" | |
| echo "metadata-filepath=${OUTPUT}" >> "$GITHUB_OUTPUT" | |
| printf "metadata-json<<%s\n%s\n%s\n" "${delimiter}" "${content}" "${delimiter}" >> "$GITHUB_OUTPUT" # see https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#multiline-strings | |
| echo "id=$id" >> "$GITHUB_OUTPUT" | |
| echo "length=$length" >> "$GITHUB_OUTPUT" | |
| printf "compilation-json<<%s\n%s\n%s\n" "${delimiter}" "${compilation}" "${delimiter}" >> "$GITHUB_OUTPUT" # see https://docs.github.com/en/actions/using-workflows/workflow-commands-for-github-actions#multiline-strings | |
| echo "compilation-length=$complength" >> "$GITHUB_OUTPUT" | |
| - name: Upload `${{ steps.retrieve.outputs.metadata-filepath }}` | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: metadata.json | |
| path: ${{ steps.retrieve.outputs.metadata-filepath }} | |
| - name: Upload `${{ steps.retrieve.outputs.from-source-metadata-filepath }}` | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: from-source-metadata.json | |
| path: ${{ steps.retrieve.outputs.from-source-metadata-filepath }} | |
| # Check if there is buildpack-provided compilation code and testing code | |
| # Optional compilation code expected at: <buildpack>/dependency/actions/compile/ | |
| # Optional testing code expected at: <buildpack>/dependency/test/ | |
| get-compile-and-test: | |
| name: Get Compilation and Testing Code | |
| outputs: | |
| should-compile: ${{ steps.compile-check.outputs.should-compile }} | |
| should-test: ${{ steps.test-check.outputs.should-test }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Check out code | |
| uses: actions/checkout@v4 | |
| - name: Has Compilation Action? | |
| id: compile-check | |
| run: | | |
| if test -d "dependency/actions/compile"; then | |
| echo "Compilation action provided" | |
| echo "should-compile=true" >> "$GITHUB_OUTPUT" | |
| fi | |
| - name: Has Testing Action? | |
| id: test-check | |
| run: | | |
| if test -d "dependency/test"; then | |
| echo "Testing file provided" | |
| echo "should-test=true" >> "$GITHUB_OUTPUT" | |
| fi | |
| test: | |
| name: Test Non-Compiled Dependency | |
| needs: | |
| - retrieve | |
| - get-compile-and-test | |
| strategy: | |
| matrix: | |
| includes: ${{ fromJSON(needs.retrieve.outputs.metadata-json) }} | |
| # Run job step if BOTH: | |
| # (1) needs.get-compile-and-test.outputs.should-test = TRUE -> if there is a dependency/test directory in the buildpack | |
| # (2) needs.get-compile-and-test.outputs.should-compile = FALSE -> if there is NOT a dependency/actions/compile directory in the buildpack | |
| # AND: | |
| # (3) there is at least one new version to test | |
| if: ${{ needs.retrieve.outputs.length > 0 && needs.get-compile-and-test.outputs.should-test == 'true' && needs.get-compile-and-test.outputs.should-compile == 'false' }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Check out code | |
| uses: actions/checkout@v4 | |
| - name: Make Temporary Artifact Directory | |
| id: make-outputdir | |
| run: echo "outputdir=$(mktemp -d)" >> "$GITHUB_OUTPUT" | |
| # Download the tarball for testing if: | |
| # (1) dependency testing code is present in the buildpack directory | |
| # (2) URI in metadata.json is available | |
| - name: Download upstream tarball (if not compiled) | |
| if: ${{ matrix.includes.uri != '' && needs.get-compile-and-test.outputs.should-test == 'true' }} | |
| run: | | |
| #!/usr/bin/env bash | |
| set -euo pipefail | |
| shopt -s inherit_errexit | |
| curl ${{ matrix.includes.uri }} \ | |
| --fail-with-body \ | |
| --show-error \ | |
| --silent \ | |
| --location \ | |
| --output ${{ steps.make-outputdir.outputs.outputdir }}/dependency.tgz | |
| # Test the dependency tarball if: | |
| # (1) dependency testing code is present in the buildpack directory | |
| - name: Test Upstream Dependency | |
| working-directory: dependency | |
| if: ${{ needs.get-compile-and-test.outputs.should-test == 'true' }} | |
| run: | | |
| make test \ | |
| version="${{ matrix.includes.version }}" \ | |
| tarballPath="${{ steps.make-outputdir.outputs.outputdir }}/*.tgz" | |
| compile: | |
| name: Compile and Test Dependency | |
| needs: | |
| - retrieve | |
| - get-compile-and-test | |
| strategy: | |
| matrix: | |
| includes: ${{ fromJSON(needs.retrieve.outputs.compilation-json) }} | |
| # Run job step if: | |
| # (1) needs.get-compile-and-test.outputs.should-compile -> if there is a dependency/actions/compile directory in the buildpack | |
| # (2) OR needs.get-compile-and-test.outputs.should-test -> if there is a dependency/test directory in the buildpack | |
| # AND: | |
| # (3) there is at least one version to compile/test | |
| if: ${{ needs.retrieve.outputs.compilation-length > 0 && (needs.get-compile-and-test.outputs.should-compile == 'true' || needs.get-compile-and-test.outputs.should-test == 'true') }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Check out code | |
| uses: actions/checkout@v4 | |
| - name: Make Temporary Artifact Directory | |
| id: make-outputdir | |
| run: | | |
| echo "outputdir=$(mktemp -d)" >> "$GITHUB_OUTPUT" | |
| # Compile if all of the following conditions are met: | |
| # (1) compilation Github Action presetn in the buildpack directory | |
| # (2) checksum in metadata.json is empty | |
| # (3) URI in metadata.json is empty | |
| - name: Compile version ${{ matrix.includes.version }} on ${{ matrix.includes.target }} Dockerfile | |
| id: compile | |
| if: ${{ needs.get-compile-and-test.outputs.should-compile && matrix.includes.checksum == '' && matrix.includes.uri == '' }} | |
| uses: ./dependency/actions/compile | |
| with: | |
| version: "${{ matrix.includes.version }}" | |
| outputdir: "${{ steps.make-outputdir.outputs.outputdir }}" | |
| target: "${{ matrix.includes.target }}" | |
| # If compiled, upload the tarball and checksum file for usage in the Update metadata job | |
| - name: Upload workflow asset | |
| uses: actions/upload-artifact@v4 | |
| if: ${{ needs.get-compile-and-test.outputs.should-compile && matrix.includes.checksum == '' && matrix.includes.uri == '' }} | |
| with: | |
| name: '${{ needs.retrieve.outputs.id }}-${{ matrix.includes.version }}-${{ matrix.includes.target }}' | |
| path: '${{ steps.make-outputdir.outputs.outputdir }}/*' | |
| # Test the dependency tarball if: | |
| # (1) dependency testing code is present in the buildpack directory | |
| - name: Test Dependency | |
| working-directory: dependency | |
| if: ${{ needs.get-compile-and-test.outputs.should-test == 'true' }} | |
| run: | | |
| #!/usr/bin/env bash | |
| set -euo pipefail | |
| shopt -s inherit_errexit | |
| make test \ | |
| version="${{ matrix.includes.version }}" \ | |
| tarballPath="${{ steps.make-outputdir.outputs.outputdir }}/*.tgz" | |
| # Add in the checksum and URI fields to the metadata if the dependency was compiled | |
| update-metadata: | |
| name: Update Metadata (if compiled) | |
| needs: | |
| - retrieve | |
| - get-compile-and-test | |
| - compile | |
| strategy: | |
| matrix: | |
| includes: ${{ fromJSON(needs.retrieve.outputs.compilation-json) }} | |
| if: ${{ needs.retrieve.outputs.compilation-length > 0 && needs.get-compile-and-test.outputs.should-compile == 'true' }} | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Check out code | |
| uses: actions/checkout@v4 | |
| - name: Download artifact files | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: '${{ needs.retrieve.outputs.id }}-${{ matrix.includes.version }}-${{ matrix.includes.target }}' | |
| - name: Get artifact file name | |
| id: get-file-names | |
| run: | | |
| #!/usr/bin/env bash | |
| set -euo pipefail | |
| shopt -s inherit_errexit | |
| echo "artifact-file=$(basename ./*.tgz)" >> "$GITHUB_OUTPUT" | |
| echo "checksum-file=$(basename ./*.tgz.checksum)" >> "$GITHUB_OUTPUT" | |
| - name: Configure AWS Credentials | |
| uses: aws-actions/configure-aws-credentials@v2 | |
| with: | |
| aws-access-key-id: ${{ secrets.AWS_S3_DEPENDENCIES_ACCESS_KEY_ID }} | |
| aws-secret-access-key: ${{ secrets.AWS_S3_DEPENDENCIES_SECRET_ACCESS_KEY }} | |
| aws-region: us-east-1 | |
| - name: Upload to S3 | |
| id: upload | |
| uses: paketo-buildpacks/github-config/actions/dependency/upload-to-s3@main | |
| with: | |
| bucket-name: "paketo-buildpacks" | |
| dependency-name: ${{ needs.retrieve.outputs.id }} | |
| artifact-path: ${{ steps.get-file-names.outputs.artifact-file }} | |
| - name: Get Checksum | |
| id: get-checksum | |
| run: echo "checksum=$(cat ${{ steps.get-file-names.outputs.checksum-file }})" >> "$GITHUB_OUTPUT" | |
| - name: Download metadata.json | |
| uses: actions/download-artifact@v4 | |
| with: | |
| name: metadata.json | |
| # Create target/version specific metadata files | |
| # Due to limitations with the upload action, we can no longer modify/upload the same metadata file | |
| - name: Write dependency-specific metadata to new file | |
| id: dependency-metadata | |
| run: | | |
| #!/usr/bin/env bash | |
| set -euo pipefail | |
| shopt -s inherit_errexit | |
| metadata_file_name="${{ matrix.includes.target }}-${{ matrix.includes.version }}-metadata-file.json" | |
| cat metadata.json | jq -r ['.[] | select( .version == "${{ matrix.includes.version }}" and .target == "${{ matrix.includes.target }}")'] > $metadata_file_name | |
| echo "file=$(echo $metadata_file_name)" >> "$GITHUB_OUTPUT" | |
| - name: Update `checksum` and `uri` in metadata for ${{ matrix.includes.target }} ${{ matrix.includes.version }} | |
| if: ${{ matrix.includes.checksum == '' && matrix.includes.uri == '' }} | |
| uses: paketo-buildpacks/github-config/actions/dependency/update-metadata-json@main | |
| with: | |
| version: ${{ matrix.includes.version }} | |
| target: ${{ matrix.includes.target }} | |
| checksum: ${{ steps.get-checksum.outputs.checksum }} | |
| uri: ${{ steps.upload.outputs.dependency-uri }} | |
| file: ${{ steps.dependency-metadata.outputs.file }} | |
| - name: Upload modified metadata | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ steps.dependency-metadata.outputs.file }} | |
| path: ${{ steps.dependency-metadata.outputs.file }} | |
| assemble: | |
| name: Update buildpack.toml | |
| needs: | |
| - retrieve | |
| - test | |
| - compile | |
| - update-metadata | |
| # Update buildpack.toml only if ALL of the following conditions are met: | |
| # (1) Retrieval step has succeeded and has found at least 1 new version | |
| # (2) Testing step has succeeded OR been skipped | |
| # (3) Compilation/Testing step has succeeded OR been skipped | |
| # (4) Update metadata step has succeeded OR been skipped | |
| if: always() && needs.retrieve.result == 'success' && needs.retrieve.outputs.length > 0 && (needs.test.result == 'success' || needs.test.result == 'skipped') && (needs.compile.result == 'success' || needs.compile.result == 'skipped') && (needs.update-metadata.result == 'success' || needs.update-metadata.result == 'skipped') | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Check out code | |
| uses: actions/checkout@v4 | |
| - name: Checkout Branch | |
| uses: paketo-buildpacks/github-config/actions/pull-request/checkout-branch@main | |
| with: | |
| branch: automation/dependencies/update-from-metadata | |
| - name: Make Temporary Artifact Directory | |
| id: make-outputdir | |
| run: echo "outputdir=$(mktemp -d)" >> "$GITHUB_OUTPUT" | |
| # Metadata file for the non-compiled dependencies, if there are any | |
| - name: Download metadata.json file | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: "${{ steps.make-outputdir.outputs.outputdir }}/metadata-files" | |
| pattern: "from-source-metadata.json" | |
| merge-multiple: true | |
| # If we compiled the dependency, and updated the metadata: | |
| # Download each metadata file, and combine them into one | |
| - name: Download individual metadata-file.json file(s) | |
| if: ${{ needs.update-metadata.result == 'success' }} | |
| uses: actions/download-artifact@v4 | |
| with: | |
| path: "${{ steps.make-outputdir.outputs.outputdir }}/metadata-files" | |
| pattern: "*metadata-file.json" | |
| merge-multiple: true | |
| - name: Display Metadata Files | |
| run: ls "${{ steps.make-outputdir.outputs.outputdir }}/metadata-files" | |
| - name: Combine Metadata Files | |
| run: | | |
| #!/usr/bin/env bash | |
| set -euo pipefail | |
| shopt -s inherit_errexit | |
| jq -s 'add' ${{ steps.make-outputdir.outputs.outputdir }}/metadata-files/* > "${{ steps.make-outputdir.outputs.outputdir }}/metadata.json" | |
| - name: Update dependencies from metadata.json | |
| id: update | |
| uses: paketo-buildpacks/github-config/actions/dependency/update-from-metadata@main | |
| with: | |
| buildpack_toml_path: "${{ github.workspace }}/buildpack.toml" | |
| metadata_file_path: "${{ steps.make-outputdir.outputs.outputdir }}/metadata.json" | |
| - name: Show git diff | |
| run: | | |
| git diff | |
| - name: Commit | |
| id: commit | |
| uses: paketo-buildpacks/github-config/actions/pull-request/create-commit@main | |
| with: | |
| message: "Updating buildpack.toml with new versions ${{ steps.update.outputs.new-versions }}" | |
| pathspec: "." | |
| keyid: ${{ secrets.PAKETO_BOT_GPG_SIGNING_KEY_ID }} | |
| key: ${{ secrets.PAKETO_BOT_GPG_SIGNING_KEY }} | |
| - name: Push Branch 'automation/dependencies/update-from-metadata' | |
| if: ${{ steps.commit.outputs.commit_sha != '' }} | |
| uses: paketo-buildpacks/github-config/actions/pull-request/push-branch@main | |
| with: | |
| branch: automation/dependencies/update-from-metadata | |
| - name: Open Pull Request | |
| if: ${{ steps.commit.outputs.commit_sha != '' }} | |
| uses: paketo-buildpacks/github-config/actions/pull-request/open@main | |
| with: | |
| token: ${{ secrets.PAKETO_BOT_GITHUB_TOKEN }} | |
| title: "Updates buildpack.toml with ${{ steps.update.outputs.new-versions }}" | |
| branch: automation/buildpack.toml/update-from-metadata | |
| failure: | |
| name: Alert on Failure | |
| runs-on: ubuntu-22.04 | |
| needs: [ retrieve, get-compile-and-test, test, compile, update-metadata, assemble ] | |
| if: ${{ always() && needs.retrieve.result == 'failure' || needs.get-compile-and-test.result == 'failure' || needs.test.result == 'failure' || needs.compile.result == 'failure' || needs.update-metadata.result == 'failure' || needs.assemble.result == 'failure' }} | |
| steps: | |
| - name: File Failure Alert Issue | |
| uses: paketo-buildpacks/github-config/actions/issue/file@main | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| repo: ${{ github.repository }} | |
| label: "failure:update-dependencies" | |
| comment_if_exists: true | |
| issue_title: "Failure: Update Dependencies workflow" | |
| issue_body: | | |
| Update Dependencies From Metadata workflow [failed](https://github.com/${{github.repository}}/actions/runs/${{github.run_id}}). | |
| comment_body: | | |
| Another failure occurred: https://github.com/${{github.repository}}/actions/runs/${{github.run_id}} |