From cd3e9effd896649b205c78441e87fcb1ac79d4cf Mon Sep 17 00:00:00 2001 From: Vladyslav Diachenko <82767850+vlad-diachenko@users.noreply.github.com> Date: Thu, 8 Aug 2024 15:52:49 +0300 Subject: [PATCH] chore: [release-3.1.x] fix(ci): add cleanup step into job `dist` (#13802) --- .github/jsonnetfile.json | 2 +- .github/jsonnetfile.lock.json | 4 +- .../loki-release/workflows/build.libsonnet | 1 + .../loki-release/workflows/common.libsonnet | 4 + .../loki-release/workflows/release.libsonnet | 2 +- .../loki-release/workflows/validate.libsonnet | 203 ++++++++++++++---- .github/workflows/minor-release-pr.yml | 2 + .github/workflows/patch-release-pr.yml | 2 + .github/workflows/release.yml | 2 +- 9 files changed, 172 insertions(+), 50 deletions(-) diff --git a/.github/jsonnetfile.json b/.github/jsonnetfile.json index 92b8f0db112b4..1038aebdcd66c 100644 --- a/.github/jsonnetfile.json +++ b/.github/jsonnetfile.json @@ -8,7 +8,7 @@ "subdir": "workflows" } }, - "version": "124c4d996f9625478a79f1884465e29ea082d224" + "version": "87cb5090c36b5332e7f21b5c59e136962d5f4f56" } ], "legacyImports": true diff --git a/.github/jsonnetfile.lock.json b/.github/jsonnetfile.lock.json index 4ec13c9337380..9eef1872519bf 100644 --- a/.github/jsonnetfile.lock.json +++ b/.github/jsonnetfile.lock.json @@ -8,8 +8,8 @@ "subdir": "workflows" } }, - "version": "124c4d996f9625478a79f1884465e29ea082d224", - "sum": "8wrJURq48ZBAtZcReO1W7AiXmvUyLqb932Q9sXyfFVo=" + "version": "87cb5090c36b5332e7f21b5c59e136962d5f4f56", + "sum": "kVlVZPpPz8d/D6UGK9Hto+NeGy7z8NvGygcB1QboxWw=" } ], "legacyImports": false diff --git a/.github/vendor/github.com/grafana/loki-release/workflows/build.libsonnet b/.github/vendor/github.com/grafana/loki-release/workflows/build.libsonnet index 1857836d66655..bd9d2e2e9b119 100644 --- a/.github/vendor/github.com/grafana/loki-release/workflows/build.libsonnet +++ b/.github/vendor/github.com/grafana/loki-release/workflows/build.libsonnet @@ -176,6 +176,7 @@ local releaseLibStep = common.releaseLibStep; dist: function(buildImage, skipArm=true, useGCR=false, makeTargets=['dist', 'packages']) job.new() + job.withSteps([ + common.cleanUpBuildCache, common.fetchReleaseRepo, common.googleAuth, common.setupGoogleCloudSdk, diff --git a/.github/vendor/github.com/grafana/loki-release/workflows/common.libsonnet b/.github/vendor/github.com/grafana/loki-release/workflows/common.libsonnet index 03f4aa112045f..d989eb40f6abe 100644 --- a/.github/vendor/github.com/grafana/loki-release/workflows/common.libsonnet +++ b/.github/vendor/github.com/grafana/loki-release/workflows/common.libsonnet @@ -75,6 +75,10 @@ checkout: $.step.new('checkout', 'actions/checkout@v4'), + cleanUpBuildCache: + $.step.new('clean up build tools cache') + + $.step.withRun('rm -rf /opt/hostedtoolcache'), + fetchReleaseRepo: $.step.new('pull code to release', 'actions/checkout@v4') + $.step.with({ diff --git a/.github/vendor/github.com/grafana/loki-release/workflows/release.libsonnet b/.github/vendor/github.com/grafana/loki-release/workflows/release.libsonnet index 62f065b40288a..b77d8fe615ec9 100644 --- a/.github/vendor/github.com/grafana/loki-release/workflows/release.libsonnet +++ b/.github/vendor/github.com/grafana/loki-release/workflows/release.libsonnet @@ -176,7 +176,7 @@ local pullRequestFooter = 'Merging this PR will release the [artifacts](https:// step.new('download images') + step.withRun(||| echo "downloading images to $(pwd)/images" - gsutil cp -r gs://loki-build-artifacts/${{ needs.createRelease.outputs.sha }}/images . + gsutil cp -r gs://${BUILD_ARTIFACTS_BUCKET}/${{ needs.createRelease.outputs.sha }}/images . |||), step.new('publish docker images', './lib/actions/push-images') + step.with({ diff --git a/.github/vendor/github.com/grafana/loki-release/workflows/validate.libsonnet b/.github/vendor/github.com/grafana/loki-release/workflows/validate.libsonnet index 28df05dd215d6..40bf097049e83 100644 --- a/.github/vendor/github.com/grafana/loki-release/workflows/validate.libsonnet +++ b/.github/vendor/github.com/grafana/loki-release/workflows/validate.libsonnet @@ -24,15 +24,6 @@ local setupValidationDeps = function(job) job { smoke_test: '${binary} --version', tar_args: 'xvf', }), - step.new('install jsonnetfmt', './lib/actions/install-binary') - + step.withIf('${{ !fromJSON(env.SKIP_VALIDATION) }}') - + step.with({ - binary: 'jsonnetfmt', - version: '0.18.0', - download_url: 'https://github.com/google/go-jsonnet/releases/download/v${version}/go-jsonnet_${version}_Linux_x86_64.tar.gz', - tarball_binary_path: '${binary}', - smoke_test: '${binary} --version', - }), ] + job.steps, }; @@ -44,45 +35,77 @@ local validationJob = _validationJob(false); + step.withIf('${{ !fromJSON(env.SKIP_VALIDATION) }}') + step.withRun(common.makeTarget(target)), - test: setupValidationDeps( - validationJob - + job.withSteps([ - validationMakeStep('test', 'test'), - ]) - ), + // Test jobs + collectPackages: job.new() + + job.withSteps([ + common.checkout, + common.fixDubiousOwnership, + step.new('gather packages') + + step.withId('gather-tests') + + step.withRun(||| + echo "packages=$(find . -path '*_test.go' -printf '%h\n' \ + | grep -e "pkg/push" -e "integration" -e "operator" -e "lambda-promtail" -e "helm" -v \ + | cut -d / -f 2,3 \ + | uniq \ + | sort \ + | jq --raw-input --slurp --compact-output 'split("\n")[:-1]')" >> ${GITHUB_OUTPUT} + |||), + ]) + + job.withOutputs({ + packages: '${{ steps.gather-tests.outputs.packages }}', + }), - integration: setupValidationDeps( - validationJob - + job.withSteps([ - validationMakeStep('integration', 'test-integration'), - ]) - ), + integration: validationJob + + job.withSteps([ + common.checkout, + common.fixDubiousOwnership, + validationMakeStep('integration', 'test-integration'), + ]), - lint: setupValidationDeps( - validationJob - + job.withSteps( - [ - validationMakeStep('lint', 'lint'), - validationMakeStep('lint jsonnet', 'lint-jsonnet'), - validationMakeStep('lint scripts', 'lint-scripts'), - step.new('check format') - + step.withIf('${{ !fromJSON(env.SKIP_VALIDATION) }}') - + step.withRun(||| - git fetch origin - make check-format - |||), - ] + [ - step.new('golangci-lint', 'golangci/golangci-lint-action@08e2f20817b15149a52b5b3ebe7de50aff2ba8c5') - + step.withIf('${{ !fromJSON(env.SKIP_VALIDATION) }}') - + step.with({ - version: '${{ inputs.golang_ci_lint_version }}', - 'only-new-issues': true, - }), - ], - ) - ), + testPackages: validationJob + + job.withNeeds(['collectPackages']) + + job.withStrategy({ + matrix: { + package: '${{fromJson(needs.collectPackages.outputs.packages)}}', + }, + }) + + job.withSteps([ + common.checkout, + common.fixDubiousOwnership, + step.new('test ${{ matrix.package }}') + + step.withIf('${{ !fromJSON(env.SKIP_VALIDATION) }}') + + step.withRun(||| + gotestsum -- -covermode=atomic -coverprofile=coverage.txt -p=4 ./${{ matrix.package }}/... + |||), + ]), - check: setupValidationDeps( + + testLambdaPromtail: validationJob + + job.withSteps([ + common.checkout, + common.fixDubiousOwnership, + step.new('test push package') + + step.withIf('${{ !fromJSON(env.SKIP_VALIDATION) }}') + + step.withWorkingDirectory('tools/lambda-promtail') + + step.withRun(||| + gotestsum -- -covermode=atomic -coverprofile=coverage.txt -p=4 ./... + |||), + ]), + + testPushPackage: validationJob + + job.withSteps([ + common.checkout, + common.fixDubiousOwnership, + step.new('test push package') + + step.withIf('${{ !fromJSON(env.SKIP_VALIDATION) }}') + + step.withWorkingDirectory('pkg/push') + + step.withRun(||| + gotestsum -- -covermode=atomic -coverprofile=coverage.txt -p=4 ./... + |||), + ]), + + // Check / lint jobs + checkFiles: setupValidationDeps( validationJob + job.withSteps([ validationMakeStep('check generated files', 'check-generated-files'), @@ -115,4 +138,94 @@ local validationJob = _validationJob(false); ], } ), + + faillint: + validationJob + + job.withSteps([ + common.checkout, + common.fixDubiousOwnership, + step.new('faillint') + + step.withIf('${{ !fromJSON(env.SKIP_VALIDATION) }}') + + step.withRun(||| + faillint -paths "sync/atomic=go.uber.org/atomic" ./... + + |||), + ]), + + golangciLint: setupValidationDeps( + validationJob + + job.withSteps( + [ + step.new('golangci-lint', 'golangci/golangci-lint-action@08e2f20817b15149a52b5b3ebe7de50aff2ba8c5') + + step.withIf('${{ !fromJSON(env.SKIP_VALIDATION) }}') + + step.with({ + version: '${{ inputs.golang_ci_lint_version }}', + 'only-new-issues': true, + }), + ], + ) + ), + + lintFiles: setupValidationDeps( + validationJob + + job.withSteps( + [ + validationMakeStep('lint scripts', 'lint-scripts'), + step.new('check format') + + step.withIf('${{ !fromJSON(env.SKIP_VALIDATION) }}') + + step.withRun(||| + git fetch origin + make check-format + |||), + ] + ) + ), + + failCheck: job.new() + + job.withNeeds([ + 'checkFiles', + 'faillint', + 'golangciLint', + 'lintFiles', + 'integration', + 'testLambdaPromtail', + 'testPackages', + 'testPushPackage', + ]) + + job.withEnv({ + SKIP_VALIDATION: '${{ inputs.skip_validation }}', + }) + + job.withIf("${{ !fromJSON(inputs.skip_validation) && (cancelled() || contains(needs.*.result, 'cancelled') || contains(needs.*.result, 'failure')) }}") + + job.withSteps([ + common.checkout, + step.new('verify checks passed') + + step.withRun(||| + echo "Some checks have failed!" + exit 1, + |||), + ]), + + check: job.new() + + job.withNeeds([ + 'checkFiles', + 'faillint', + 'golangciLint', + 'lintFiles', + 'integration', + 'testLambdaPromtail', + 'testPackages', + 'testPushPackage', + ]) + + job.withEnv({ + SKIP_VALIDATION: '${{ inputs.skip_validation }}', + }) + + job.withSteps([ + common.checkout, + step.new('checks passed') + + step.withIf('${{ !fromJSON(env.SKIP_VALIDATION) }}') + + step.withRun(||| + echo "All checks passed" + |||), + ]), + } diff --git a/.github/workflows/minor-release-pr.yml b/.github/workflows/minor-release-pr.yml index e6527ef1250d0..0d5b261e6f464 100644 --- a/.github/workflows/minor-release-pr.yml +++ b/.github/workflows/minor-release-pr.yml @@ -101,6 +101,8 @@ jobs: version: "${{ needs.version.outputs.version }}" runs-on: "ubuntu-latest" steps: + - name: "clean up build tools cache" + run: "rm -rf /opt/hostedtoolcache" - name: "pull code to release" uses: "actions/checkout@v4" with: diff --git a/.github/workflows/patch-release-pr.yml b/.github/workflows/patch-release-pr.yml index 0461ce62c3eb9..5fe2e39d6ce98 100644 --- a/.github/workflows/patch-release-pr.yml +++ b/.github/workflows/patch-release-pr.yml @@ -101,6 +101,8 @@ jobs: version: "${{ needs.version.outputs.version }}" runs-on: "ubuntu-latest" steps: + - name: "clean up build tools cache" + run: "rm -rf /opt/hostedtoolcache" - name: "pull code to release" uses: "actions/checkout@v4" with: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 0bbf7eb48465f..9a87b37917757 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -138,7 +138,7 @@ jobs: - name: "download images" run: | echo "downloading images to $(pwd)/images" - gsutil cp -r gs://loki-build-artifacts/${{ needs.createRelease.outputs.sha }}/images . + gsutil cp -r gs://${BUILD_ARTIFACTS_BUCKET}/${{ needs.createRelease.outputs.sha }}/images . - name: "publish docker images" uses: "./lib/actions/push-images" with: