[feat][gcp] Improve the way of collection for DiskTypes and MachineTy… #873
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Publish | |
| on: | |
| push: | |
| tags: | |
| - "*.*.*" | |
| branches: | |
| - main | |
| workflow_dispatch: | |
| concurrency: | |
| group: ${{ github.workflow }}-${{ github.run_id }} | |
| cancel-in-progress: true | |
| jobs: | |
| edge: | |
| name: Update edge docs | |
| if: github.ref == 'refs/heads/main' | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Check out repository | |
| uses: actions/checkout@v4 | |
| - name: Check out someengineering/docs.fix.security | |
| uses: actions/checkout@v4 | |
| with: | |
| repository: someengineering/docs.fix.security | |
| path: docs.fix.security | |
| token: ${{ secrets.SOME_CI_PAT }} | |
| - name: Install dependencies | |
| working-directory: ./docs.fix.security | |
| run: | | |
| yarn install --frozen-lockfile | |
| - name: Wait for AWS policies to be uploaded | |
| if: github.event_name != 'workflow_dispatch' | |
| uses: lewagon/[email protected] | |
| with: | |
| ref: ${{ github.ref }} | |
| check-name: aws | |
| repo-token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Update AWS policy JSON | |
| shell: bash | |
| working-directory: ./docs.fix.security/iam/aws | |
| run: | | |
| wget -qO FixOrgList.json https://cdn.some.engineering/fix/aws/edge/FixOrgList.json | |
| wget -qO FixCollect.json https://cdn.some.engineering/fix/aws/edge/FixCollect.json | |
| - name: Wait for GCP policies to be uploaded | |
| if: github.event_name != 'workflow_dispatch' | |
| uses: lewagon/[email protected] | |
| with: | |
| ref: ${{ github.ref }} | |
| check-name: gcp | |
| repo-token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Update GCP policy JSON | |
| shell: bash | |
| working-directory: ./docs.fix.security/iam/gcp | |
| run: | | |
| wget -qO fix_access.json https://cdn.some.engineering/fix/gcp/edge/fix_access.json | |
| - name: Clean existing Kroki images | |
| shell: bash | |
| working-directory: ./docs.fix.security/static/img/kroki | |
| run: | | |
| find . -type f -name "*.svg" -prune -exec rm {} \+ | |
| - name: Wait for Docker images to build | |
| if: github.event_name != 'workflow_dispatch' | |
| uses: lewagon/[email protected] | |
| with: | |
| ref: ${{ github.ref }} | |
| check-name: Build split Docker images | |
| repo-token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Update resource data models | |
| continue-on-error: true | |
| shell: bash | |
| run: | | |
| yq '.services.fixcore.environment += "FIXCORE_MODEL_FROM_PLUGINS=true"' docker-compose.yaml > docker-compose-model-gen.yaml | |
| PSK= FIXCORE_ANALYTICS_OPT_OUT=true docker-compose -f docker-compose-model-gen.yaml up -d | |
| cd ${{ github.workspace }}/docs.fix.security/docs/resources | |
| python3 ${{ github.workspace }}/docs.fix.security/tools/export_models.py | |
| - name: Build | |
| continue-on-error: true | |
| working-directory: ./docs.fix.security | |
| run: | | |
| yarn build | |
| - name: Optimize and format | |
| working-directory: ./docs.fix.security | |
| run: | | |
| yarn optimize | |
| yarn format | |
| - name: Create someengineering/docs.fix.security pull request | |
| uses: peter-evans/create-pull-request@v6 | |
| env: | |
| HUSKY: 0 | |
| with: | |
| path: docs.fix.security | |
| commit-message: "chore: update documentation" | |
| title: "chore: update documentation" | |
| body: | | |
| Updates documentation to reflect changes in [`${{ github.sha }}`](https://github.com/someengineering/fixinventory/commit/${{ github.sha }}). | |
| labels: | | |
| 🤖 bot | |
| branch: some-ci # stable branch name so any additional commits to main update the existing PR instead of creating a new one | |
| delete-branch: true | |
| token: ${{ secrets.SOME_CI_PAT }} | |
| committer: C.K. <[email protected]> | |
| author: C.K. <[email protected]> | |
| - name: Check out someengineering/inventory.fix.security | |
| uses: actions/checkout@v3 | |
| with: | |
| repository: someengineering/inventory.fix.security | |
| path: inventory.fix.security | |
| token: ${{ secrets.SOME_CI_PAT }} | |
| - name: Install dependencies | |
| working-directory: ./inventory.fix.security | |
| run: | | |
| yarn install --frozen-lockfile | |
| - name: Update fixcore API YAML | |
| shell: bash | |
| run: | | |
| yq '.servers[0].url = "https://{host}:{port}" | .servers[0].variables.host.default="localhost" | .servers[0].variables.port.default="8900" | del(.servers[0].description)' fixcore/fixcore/static/api-doc.yaml > inventory.fix.security/openapi/fixcore-edge.yml | |
| - name: Regenerate API docs | |
| working-directory: ./inventory.fix.security | |
| run: | | |
| yarn gen-api-docs | |
| - name: Update AWS policy JSON | |
| shell: bash | |
| working-directory: ./inventory.fix.security/iam/aws/edge | |
| run: | | |
| wget -qO FixOrgList.json https://cdn.some.engineering/fix/aws/edge/FixOrgList.json | |
| wget -qO FixCollect.json https://cdn.some.engineering/fix/aws/edge/FixCollect.json | |
| wget -qO FixMutate.json https://cdn.some.engineering/fix/aws/edge/FixMutate.json | |
| - name: Wait for GCP policies to be uploaded | |
| if: github.event_name != 'workflow_dispatch' | |
| uses: lewagon/[email protected] | |
| with: | |
| ref: ${{ github.ref }} | |
| check-name: gcp | |
| repo-token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Update GCP policy JSON | |
| shell: bash | |
| working-directory: ./inventory.fix.security/iam/gcp/edge | |
| run: | | |
| wget -qO fix_access.json https://cdn.some.engineering/fix/gcp/edge/fix_access.json | |
| wget -qO fix_mutate.json https://cdn.some.engineering/fix/gcp/edge/fix_mutate.json | |
| - name: Clean existing Kroki images | |
| if: github.event_name == 'workflow_dispatch' # only when triggered manually | |
| shell: bash | |
| working-directory: ./inventory.fix.security/static/img/kroki | |
| run: | | |
| find . -type f -name "*.svg" -prune -exec rm {} \+ | |
| - name: Update resource data models | |
| continue-on-error: true | |
| shell: bash | |
| run: | | |
| cd ${{ github.workspace }}/inventory.fix.security/docs/reference/unified-data-model | |
| python3 ${{ github.workspace }}/inventory.fix.security/tools/export_models.py | |
| - name: Build | |
| continue-on-error: true | |
| working-directory: ./inventory.fix.security | |
| run: | | |
| yarn build | |
| - name: Optimize and format | |
| working-directory: ./inventory.fix.security | |
| run: | | |
| yarn optimize | |
| yarn format | |
| - name: Create someengineering/inventory.fix.security pull request | |
| uses: peter-evans/create-pull-request@v6 | |
| env: | |
| HUSKY: 0 | |
| with: | |
| path: inventory.fix.security | |
| commit-message: "chore: update edge documentation" | |
| title: "chore: update edge documentation" | |
| body: | | |
| Updates `edge` documentation to reflect changes in [`${{ github.sha }}`](https://github.com/someengineering/fixinventory/commit/${{ github.sha }}). | |
| labels: | | |
| 🤖 bot | |
| branch: edge # stable branch name so any additional commits to main update the existing PR instead of creating a new one | |
| delete-branch: true | |
| token: ${{ secrets.SOME_CI_PAT }} | |
| committer: C.K. <[email protected]> | |
| author: C.K. <[email protected]> | |
| release: | |
| name: Update stable docs & create release | |
| if: github.ref_type == 'tag' # on tagging of releases and prereleases | |
| runs-on: ubuntu-latest | |
| steps: | |
| - name: Get release tag and type | |
| id: release | |
| shell: bash | |
| run: | | |
| GITHUB_REF="${{ github.ref }}" | |
| tag=${GITHUB_REF##*/} | |
| echo "tag=${tag}" >> $GITHUB_OUTPUT | |
| if [[ ${{ github.ref }} =~ ^refs/tags/[0-9]+\.[0-9]+\.[0-9]+$ ]]; then | |
| echo "prerelease=false" >> $GITHUB_OUTPUT | |
| echo "docsVersion=$(cut -d '.' -f 1 <<< ${GITHUB_REF##*/}).X" >> $GITHUB_OUTPUT | |
| else | |
| echo "prerelease=true" >> $GITHUB_OUTPUT | |
| fi | |
| - name: Check out repository | |
| uses: actions/checkout@v3 | |
| with: | |
| fetch-depth: 0 | |
| - name: Check out someengineering/inventory.fix.security | |
| if: steps.release.outputs.prerelease == 'false' | |
| uses: actions/checkout@v3 | |
| with: | |
| repository: someengineering/inventory.fix.security | |
| path: inventory.fix.security | |
| token: ${{ secrets.SOME_CI_PAT }} | |
| - name: Install dependencies | |
| if: steps.release.outputs.prerelease == 'false' | |
| working-directory: ./inventory.fix.security | |
| run: | | |
| yarn install --frozen-lockfile | |
| - name: Tag docs version | |
| if: steps.release.outputs.prerelease == 'false' | |
| continue-on-error: true # versioned doc may already exist | |
| working-directory: ./inventory.fix.security | |
| run: | | |
| yarn run docusaurus docs:version "${{ steps.release.outputs.docsVersion }}" | |
| - name: Generate release notes | |
| if: steps.release.outputs.prerelease == 'false' | |
| id: release_notes | |
| shell: bash | |
| env: | |
| GH_TOKEN: ${{ secrets.GITHUB_TOKEN }} | |
| run: | | |
| GITHUB_REF="${{ github.ref }}" | |
| tag=${GITHUB_REF##*/} | |
| version_major=$(cut -d '.' -f 1 <<< ${GITHUB_REF##*/}) | |
| prev_release=$(echo $(gh api repos/someengineering/fixinventory/releases) | jq -r "map(select(.tag_name | startswith(\"${version_major}.\")) | select(.prerelease == false))[0].tag_name") | |
| [ -n "$prev_release" ] && prev_release=$(echo $(gh api repos/someengineering/fixinventory/releases/latest) | jq -r '.tag_name') | |
| dir="inventory.fix.security/releases/${{ steps.release.outputs.tag }}" | |
| mkdir -p $dir | |
| file="${dir}/index.mdx" | |
| python3 tools/release_notes.py ${prev_release} ${{ steps.release.outputs.tag }} > $file | |
| link="/releases/${{ steps.release.outputs.tag }}" | |
| echo $(jq '.["${{ steps.release.outputs.docsVersion }}"]="${{ steps.release.outputs.tag }}"' inventory.fix.security/latestRelease.json) > inventory.fix.security/latestRelease.json | |
| echo "tag=${{ steps.release.outputs.tag }}" >> $GITHUB_OUTPUT | |
| echo "file=$file" >> $GITHUB_OUTPUT | |
| echo "link=$link" >> $GITHUB_OUTPUT | |
| - name: Update API YAML | |
| if: steps.release.outputs.prerelease == 'false' | |
| shell: bash | |
| run: | | |
| yq '.servers[0].url = "https://{host}:{port}" | .servers[0].variables.host.default="localhost" | .servers[0].variables.port.default="8900" | del(.servers[0].description)' fixcore/fixcore/static/api-doc.yaml > inventory.fix.security/openapi/fixcore-${{ steps.release.outputs.docsVersion }}.yml | |
| - name: Regenerate API docs | |
| if: steps.release.outputs.prerelease == 'false' | |
| working-directory: ./inventory.fix.security | |
| run: | | |
| yarn gen-api-docs | |
| - name: Wait for AWS policies to be uploaded | |
| if: steps.release.outputs.prerelease == 'false' && github.event_name != 'workflow_dispatch' | |
| uses: lewagon/[email protected] | |
| with: | |
| ref: ${{ github.ref }} | |
| check-name: aws | |
| repo-token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Update AWS policy JSON | |
| if: steps.release.outputs.prerelease == 'false' | |
| shell: bash | |
| working-directory: ./inventory.fix.security/iam/aws/${{ steps.release.outputs.docsVersion }} | |
| run: | | |
| wget -qO FixOrgList.json https://cdn.some.engineering/fix/aws/${{ steps.release.outputs.tag }}/FixOrgList.json | |
| wget -qO FixCollect.json https://cdn.some.engineering/fix/aws/${{ steps.release.outputs.tag }}/FixCollect.json | |
| wget -qO FixMutate.json https://cdn.some.engineering/fix/aws/${{ steps.release.outputs.tag }}/FixMutate.json | |
| - name: Wait for GCP policies to be uploaded | |
| if: steps.release.outputs.prerelease == 'false' && github.event_name != 'workflow_dispatch' | |
| uses: lewagon/[email protected] | |
| with: | |
| ref: ${{ github.ref }} | |
| check-name: gcp | |
| repo-token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Update GCP policy JSON | |
| if: steps.release.outputs.prerelease == 'false' | |
| shell: bash | |
| working-directory: ./inventory.fix.security/iam/gcp/${{ steps.release.outputs.docsVersion }} | |
| run: | | |
| wget -qO fix_access.json https://cdn.some.engineering/fix/gcp/${{ steps.release.outputs.tag }}/fix_access.json | |
| wget -qO fix_mutate.json https://cdn.some.engineering/fix/gcp/${{ steps.release.outputs.tag }}/fix_mutate.json | |
| - name: Modify Docker Compose YAML | |
| if: steps.release.outputs.prerelease == 'false' | |
| shell: bash | |
| run: | | |
| yq -i '.services.[].image |= sub(":edge", ":${{ steps.release.outputs.tag }}")' docker-compose.yaml | |
| - name: Clean existing Kroki images | |
| if: steps.release.outputs.prerelease == 'false' | |
| shell: bash | |
| working-directory: ./inventory.fix.security/static/img/kroki | |
| run: | | |
| find . -type f -name "*.svg" -prune -exec rm {} \+ | |
| - name: Wait for Docker images to build | |
| if: steps.release.outputs.prerelease == 'false' && github.event_name != 'workflow_dispatch' | |
| uses: lewagon/[email protected] | |
| with: | |
| ref: ${{ github.ref }} | |
| check-name: Build split Docker images | |
| repo-token: ${{ secrets.GITHUB_TOKEN }} | |
| - name: Update resource data models | |
| if: steps.release.outputs.prerelease == 'false' | |
| continue-on-error: true | |
| shell: bash | |
| run: | | |
| yq '.services.fixcore.environment += "FIXCORE_MODEL_FROM_PLUGINS=true"' docker-compose.yaml > docker-compose-model-gen.yaml | |
| PSK= FIXCORE_ANALYTICS_OPT_OUT=true docker-compose -f docker-compose-model-gen.yaml up -d | |
| cd ${{ github.workspace }}/inventory.fix.security/versioned_docs/version-${{ steps.release.outputs.docsVersion }}/reference/unified-data-model | |
| python3 ${{ github.workspace }}/inventory.fix.security/tools/export_models.py | |
| - name: Build | |
| if: steps.release.outputs.prerelease == 'false' | |
| continue-on-error: true | |
| working-directory: ./inventory.fix.security | |
| run: | | |
| yarn build | |
| - name: Optimize and format | |
| if: steps.release.outputs.prerelease == 'false' | |
| working-directory: ./inventory.fix.security | |
| run: | | |
| yarn optimize | |
| yarn format | |
| - name: Create someengineering/inventory.fix.security pull request | |
| if: steps.release.outputs.prerelease == 'false' | |
| uses: peter-evans/create-pull-request@v6 | |
| env: | |
| HUSKY: 0 | |
| with: | |
| path: inventory.fix.security | |
| commit-message: "chore: ${{ steps.release.outputs.tag }} release" | |
| title: "chore: ${{ steps.release.outputs.tag }} release" | |
| body: | | |
| Adds automatically generated release notes for [`${{ steps.release.outputs.tag }}`](https://github.com/someengineering/fixinventory/releases/tag/${{ steps.release.outputs.tag }}). | |
| Also updates `${{ steps.release.outputs.docsVersion }}` docs. | |
| labels: | | |
| 🤖 bot | |
| branch: ${{ steps.release.outputs.tag }} | |
| delete-branch: true | |
| token: ${{ secrets.SOME_CI_PAT }} | |
| committer: C.K. <[email protected]> | |
| author: C.K. <[email protected]> | |
| - name: Write release body | |
| shell: bash | |
| run: | | |
| [ ${{ steps.release.outputs.prerelease }} == 'false' ] && echo -e "### Release Notes\n\nhttps://inventory.fix.security${{ steps.release_notes.outputs.link }}\n" > release_body.txt | |
| echo -e "### Docker Images\n" >> release_body.txt | |
| echo -e "- \`somecr.io/someengineering/fixcore:${{ steps.release.outputs.tag }}\`" >> release_body.txt | |
| echo -e "- \`somecr.io/someengineering/fixworker:${{ steps.release.outputs.tag }}\`" >> release_body.txt | |
| echo -e "- \`somecr.io/someengineering/fixshell:${{ steps.release.outputs.tag }}\`" >> release_body.txt | |
| echo -e "- \`somecr.io/someengineering/fixmetrics:${{ steps.release.outputs.tag }}\`" >> release_body.txt | |
| - name: Create release | |
| uses: ncipollo/release-action@v1 | |
| with: | |
| token: ${{ secrets.GITHUB_TOKEN }} | |
| prerelease: ${{ steps.release.outputs.prerelease }} | |
| bodyFile: release_body.txt | |
| artifacts: docker-compose.yaml | |
| # - name: Check out someengineering/helm-charts | |
| # if: steps.release.outputs.prerelease == 'false' | |
| # uses: actions/checkout@v3 | |
| # with: | |
| # repository: someengineering/helm-charts | |
| # path: helm-charts | |
| # token: ${{ secrets.SOME_CI_PAT }} | |
| # - name: Get current chart version | |
| # if: steps.release.outputs.prerelease == 'false' | |
| # id: current_chart_version | |
| # shell: bash | |
| # run: | | |
| # echo "result=$(yq '.version' helm-charts/someengineering/fixinventory/Chart.yaml)" >> $GITHUB_OUTPUT | |
| # - name: Get new chart version | |
| # if: steps.release.outputs.prerelease == 'false' | |
| # id: new_chart_version | |
| # uses: WyriHaximus/github-action-next-semvers@v1 | |
| # with: | |
| # version: ${{ steps.current_chart_version.outputs.result }} | |
| # - name: Update appVersion and bump chart version | |
| # if: steps.release.outputs.prerelease == 'false' | |
| # shell: bash | |
| # run: | | |
| # yq -i '.version = "${{ steps.new_chart_version.outputs.patch }}" | .appVersion = "${{ steps.release.outputs.tag }}"' helm-charts/someengineering/fixinventory/Chart.yaml | |
| # - name: Set up Helm | |
| # if: steps.release.outputs.prerelease == 'false' | |
| # uses: azure/setup-helm@v1 | |
| # with: | |
| # version: v3.8.1 | |
| # - name: Install helm-docs | |
| # if: steps.release.outputs.prerelease == 'false' | |
| # uses: supplypike/setup-bin@v1 | |
| # with: | |
| # uri: https://github.com/norwoodj/helm-docs/releases/download/v1.11.0/helm-docs_1.11.0_Linux_x86_64.tar.gz | |
| # name: helm-docs | |
| # version: "1.11.0" | |
| # - name: Generate helm-charts README.md | |
| # if: steps.release.outputs.prerelease == 'false' | |
| # run: (cd helm-charts && helm-docs) | |
| # - name: Create someengineering/helm-charts pull request | |
| # if: steps.release.outputs.prerelease == 'false' | |
| # uses: peter-evans/create-pull-request@v6 | |
| # with: | |
| # path: helm-charts | |
| # commit-message: "chore: bump Fix Inventory appVersion to ${{ steps.release.outputs.tag }}" | |
| # title: "chore: bump Fix Inventory appVersion to ${{ steps.release.outputs.tag }}" | |
| # body: | | |
| # Bumps Fix Inventory version to [${{ steps.release.outputs.tag }}](https://github.com/someengineering/fixinventory/releases/tag/${{ steps.release.outputs.tag }}). | |
| # labels: | | |
| # 🤖 bot | |
| # branch: ${{ steps.release.outputs.tag }} | |
| # delete-branch: true | |
| # token: ${{ secrets.SOME_CI_PAT }} | |
| # committer: C.K. <[email protected]> | |
| # author: C.K. <[email protected]> |