From 4d71ca3bbaf54841bfbcfbd7ca9503b481ee058a Mon Sep 17 00:00:00 2001 From: Trevor Whitney Date: Tue, 17 Dec 2024 17:31:10 -0700 Subject: [PATCH] ci: fix the docker driver build (#15354) --- .github/jsonnetfile.json | 2 +- .github/jsonnetfile.lock.json | 4 +- .github/release-workflows.jsonnet | 19 ++-- .../loki-release/workflows/build.libsonnet | 104 +++--------------- .../loki-release/workflows/main.jsonnet | 5 +- .../loki-release/workflows/release.libsonnet | 68 +++++++++--- .../loki-release/workflows/workflows.jsonnet | 21 +++- .github/workflows/check.yml | 2 +- .github/workflows/images.yml | 2 +- .github/workflows/minor-release-pr.yml | 37 +++---- .github/workflows/patch-release-pr.yml | 37 +++---- .github/workflows/release.yml | 43 ++++++++ Makefile | 10 +- 13 files changed, 189 insertions(+), 165 deletions(-) diff --git a/.github/jsonnetfile.json b/.github/jsonnetfile.json index 731d3c3c82974..e75ed698ab80f 100644 --- a/.github/jsonnetfile.json +++ b/.github/jsonnetfile.json @@ -8,7 +8,7 @@ "subdir": "workflows" } }, - "version": "21f1189544e3976070cbdb6463f64c7a32dcc176" + "version": "cfa24256090828f566f1ba59292ce65d8db4a4ae" } ], "legacyImports": true diff --git a/.github/jsonnetfile.lock.json b/.github/jsonnetfile.lock.json index d11f133725822..0f978214e1e77 100644 --- a/.github/jsonnetfile.lock.json +++ b/.github/jsonnetfile.lock.json @@ -8,8 +8,8 @@ "subdir": "workflows" } }, - "version": "21f1189544e3976070cbdb6463f64c7a32dcc176", - "sum": "IPS1oGR8k7jk6J2snciTycWFgtISCwXSPhJ3A+nEGvY=" + "version": "cfa24256090828f566f1ba59292ce65d8db4a4ae", + "sum": "tml1dcFlo15kEE6JvN/nPY2xkhfeF3ERZjAyFbnguHA=" } ], "legacyImports": false diff --git a/.github/release-workflows.jsonnet b/.github/release-workflows.jsonnet index 080dcae2d39c4..9c7aeb2cd1e0d 100644 --- a/.github/release-workflows.jsonnet +++ b/.github/release-workflows.jsonnet @@ -1,9 +1,14 @@ local lokiRelease = import 'workflows/main.jsonnet'; -local build = lokiRelease.build; +local build = lokiRelease.build; local releaseLibRef = 'main'; - local checkTemplate = 'grafana/loki-release/.github/workflows/check.yml@%s' % releaseLibRef; +local buildImageVersion = std.extVar('BUILD_IMAGE_VERSION'); +local buildImage = 'grafana/loki-build-image:%s' % buildImageVersion; +local golangCiLintVersion = 'v1.60.3'; +local imageBuildTimeoutMin = 60; +local imagePrefix = 'grafana'; +local dockerPluginDir = 'clients/cmd/docker-driver'; local imageJobs = { loki: build.image('loki', 'cmd/loki'), @@ -15,7 +20,7 @@ local imageJobs = { 'loki-canary-boringcrypto': build.image('loki-canary-boringcrypto', 'cmd/loki-canary-boringcrypto'), promtail: build.image('promtail', 'clients/cmd/promtail'), querytee: build.image('loki-query-tee', 'cmd/querytee', platform=['linux/amd64']), - 'loki-docker-driver': build.dockerPlugin('grafana/loki-docker-driver', 'clients/cmd/docker-driver', platform=['linux/amd64', 'linux/arm64']), + 'loki-docker-driver': build.dockerPlugin('loki-docker-driver', dockerPluginDir, buildImage=buildImage, platform=['linux/amd64', 'linux/arm64']), }; local weeklyImageJobs = { @@ -25,13 +30,6 @@ local weeklyImageJobs = { promtail: build.weeklyImage('promtail', 'clients/cmd/promtail'), }; -local buildImageVersion = std.extVar('BUILD_IMAGE_VERSION'); -local buildImage = 'grafana/loki-build-image:%s' % buildImageVersion; -local golangCiLintVersion = 'v1.60.3'; - -local imageBuildTimeoutMin = 60; -local imagePrefix = 'grafana'; - { 'patch-release-pr.yml': std.manifestYamlDoc( lokiRelease.releasePRWorkflow( @@ -77,6 +75,7 @@ local imagePrefix = 'grafana'; getDockerCredsFromVault=true, imagePrefix='grafana', releaseLibRef=releaseLibRef, + pluginBuildDir=dockerPluginDir, releaseRepo='grafana/loki', useGitHubAppToken=true, ), false, false diff --git a/.github/vendor/github.com/grafana/loki-release/workflows/build.libsonnet b/.github/vendor/github.com/grafana/loki-release/workflows/build.libsonnet index 72a47e901e890..d174c362c691a 100644 --- a/.github/vendor/github.com/grafana/loki-release/workflows/build.libsonnet +++ b/.github/vendor/github.com/grafana/loki-release/workflows/build.libsonnet @@ -108,14 +108,14 @@ local releaseLibStep = common.releaseLibStep; dockerPlugin: function( name, path, + buildImage, dockerfile='Dockerfile', context='release', platform=[ 'linux/amd64', 'linux/arm64', - 'linux/arm', ] - ) + ) job.new() + job.withStrategy({ 'fail-fast': true, @@ -136,6 +136,7 @@ local releaseLibStep = common.releaseLibStep; + step.withId('platform') + step.withRun(||| mkdir -p images + mkdir -p plugins platform="$(echo "${{ matrix.platform}}" | sed "s/\(.*\)\/\(.*\)/\1-\2/")" echo "platform=${platform}" >> $GITHUB_OUTPUT @@ -150,108 +151,39 @@ local releaseLibStep = common.releaseLibStep; step.new('Build and export', 'docker/build-push-action@v6') + step.withTimeoutMinutes('${{ fromJSON(env.BUILD_TIMEOUT) }}') + step.withIf('${{ fromJSON(needs.version.outputs.pr_created) }}') - + step.withEnv({ - IMAGE_TAG: '${{ needs.version.outputs.version }}', - }) + step.with({ context: context, file: 'release/%s/%s' % [path, dockerfile], platforms: '${{ matrix.platform }}', push: false, tags: '${{ env.IMAGE_PREFIX }}/%s:${{ needs.version.outputs.version }}-${{ steps.platform.outputs.platform_short }}' % [name], - outputs: 'type=docker,dest=release/images/%s-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar' % name, - 'build-args': 'IMAGE_TAG=${{ needs.version.outputs.version }},GOARCH=${{ steps.platform.outputs.platform_short }}', + outputs: 'type=local,dest=release/plugins/%s-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}' % name, + 'build-args': ||| + %s + ||| % std.rstripChars(std.lines([ + 'IMAGE_TAG=${{ needs.version.outputs.version }}', + 'GOARCH=${{ steps.platform.outputs.platform_short }}', + ('BUILD_IMAGE=%s' % buildImage), + ]), '\n'), }), - releaseStep('Package as Docker plugin') + step.new('compress rootfs') + step.withIf('${{ fromJSON(needs.version.outputs.pr_created) }}') - + step.withEnv({ - IMAGE_TAG: '${{ needs.version.outputs.version }}', - BUILD_DIR: 'release/%s' % [path], - }) + step.withRun(||| - rm -rf "${{ env.BUILD_DIR }}/rootfs" || true - mkdir "${{ env.BUILD_DIR }}/rootfs" - tar -x -C "${{ env.BUILD_DIR }}/rootfs" -f "release/images/%s-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" - docker plugin create "${{ env.IMAGE_TAG }}${{ steps.platform.outputs.plugin_arch }}" "${{ env.BUILD_DIR }}" - |||), + tar -cf release/plugins/%s-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar \ + -C release/plugins/%s-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }} \ + . + ||| % [name, name]), step.new('upload artifacts', 'google-github-actions/upload-cloud-storage@v2') + step.withIf('${{ fromJSON(needs.version.outputs.pr_created) }}') + step.with({ - path: 'release/images/%s-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar' % name, - destination: '${{ env.BUILD_ARTIFACTS_BUCKET }}/${{ github.sha }}/images', //TODO: make bucket configurable + path: 'release/plugins/%s-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar' % name, + destination: '${{ env.BUILD_ARTIFACTS_BUCKET }}/${{ github.sha }}/plugins', process_gcloudignore: false, }), ]), - weeklyDockerPlugin: function( - name, - path, - dockerfile='Dockerfile', - context='release', - platform=[ - 'linux/amd64', - 'linux/arm64', - 'linux/arm', - ] - ) - job.new() - + job.withStrategy({ - matrix: { - platform: platform, - }, - }) - + job.withSteps([ - common.fetchReleaseLib, - common.fetchReleaseRepo, - common.setupNode, - - step.new('Set up QEMU', 'docker/setup-qemu-action@v3'), - step.new('set up docker buildx', 'docker/setup-buildx-action@v3'), - step.new('Login to DockerHub (from vault)', 'grafana/shared-workflows/actions/dockerhub-login@main'), - - releaseStep('Get weekly version') - + step.withId('weekly-version') - + step.withRun(||| - echo "version=$(./tools/image-tag)" >> $GITHUB_OUTPUT - - platform="$(echo "${{ matrix.platform}}" | sed "s/\(.*\)\/\(.*\)/\1-\2/")" - echo "platform=${platform}" >> $GITHUB_OUTPUT - echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT - if [[ "${platform}" == "linux/arm64" ]]; then - echo "plugin_arch=-arm64" >> $GITHUB_OUTPUT - else - echo "plugin_arch=" >> $GITHUB_OUTPUT - fi - |||), - - step.new('Build and export', 'docker/build-push-action@v6') - + step.withTimeoutMinutes('${{ fromJSON(env.BUILD_TIMEOUT) }}') - + step.with({ - context: context, - file: 'release/%s/%s' % [path, dockerfile], - platforms: '${{ matrix.platform }}', - push: false, - tags: '${{ env.IMAGE_PREFIX }}/%s:${{ steps.weekly-version.outputs.version }}' % [name], - outputs: 'type=docker,dest=release/images/%s-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar' % name, - 'build-args': 'IMAGE_TAG=${{ steps.weekly-version.outputs.version }},GOARCH=${{ steps.weekly-version.outputs.platform_short }}', - }), - - releaseStep('Package and push as Docker plugin') - + step.withEnv({ - IMAGE_TAG: '${{ steps.weekly-version.outputs.version }}', - BUILD_DIR: 'release/%s' % [path], - }) - + step.withRun(||| - rm -rf "${{ env.BUILD_DIR }}/rootfs" || true - mkdir "${{ env.BUILD_DIR }}/rootfs" - tar -x -C "${{ env.BUILD_DIR }}/rootfs" -f "release/images/%s-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" - docker plugin create "${{ env.IMAGE_TAG }}${{ steps.platform.outputs.plugin_arch }}" "${{ env.BUILD_DIR }}" - docker plugin push "${{ env.IMAGE_TAG }}${{ steps.platform.outputs.plugin_arch }}" - |||), - ]), - version: job.new() + job.withSteps([ diff --git a/.github/vendor/github.com/grafana/loki-release/workflows/main.jsonnet b/.github/vendor/github.com/grafana/loki-release/workflows/main.jsonnet index b4d7b24246cf9..a642294498558 100644 --- a/.github/vendor/github.com/grafana/loki-release/workflows/main.jsonnet +++ b/.github/vendor/github.com/grafana/loki-release/workflows/main.jsonnet @@ -83,11 +83,13 @@ dockerUsername='grafanabot', getDockerCredsFromVault=false, imagePrefix='grafana', + pluginBuildDir='release/plugin-tmp-dir', publishBucket='', publishToGCS=false, releaseLibRef='main', releaseRepo='grafana/loki-release', useGitHubAppToken=true, + dockerPluginPath='clients/cmd/docker-driver', ) { name: 'create release', on: { @@ -119,7 +121,8 @@ shouldRelease: $.release.shouldRelease, createRelease: $.release.createRelease, publishImages: $.release.publishImages(getDockerCredsFromVault, dockerUsername), - publishRelease: $.release.publishRelease, + publishDockerPlugins: $.release.publishDockerPlugins(pluginBuildDir, getDockerCredsFromVault, dockerUsername), + publishRelease: $.release.publishRelease(['createRelease', 'publishImages', 'publishDockerPlugins']), }, }, check: { diff --git a/.github/vendor/github.com/grafana/loki-release/workflows/release.libsonnet b/.github/vendor/github.com/grafana/loki-release/workflows/release.libsonnet index b77d8fe615ec9..d35c8a76661e8 100644 --- a/.github/vendor/github.com/grafana/loki-release/workflows/release.libsonnet +++ b/.github/vendor/github.com/grafana/loki-release/workflows/release.libsonnet @@ -186,19 +186,57 @@ local pullRequestFooter = 'Merging this PR will release the [artifacts](https:// ] ), - publishRelease: job.new() - + job.withNeeds(['createRelease', 'publishImages']) - + job.withSteps([ - common.fetchReleaseRepo, - common.githubAppToken, - common.setToken, - releaseStep('publish release') - + step.withIf('${{ !fromJSON(needs.createRelease.outputs.exists) || (needs.createRelease.outputs.draft && fromJSON(needs.createRelease.outputs.draft)) }}') - + step.withEnv({ - GH_TOKEN: '${{ steps.github_app_token.outputs.token }}', - }) - + step.withRun(||| - gh release edit ${{ needs.createRelease.outputs.name }} --draft=false --latest=${{ needs.createRelease.outputs.isLatest }} - |||), - ]), + publishDockerPlugins: function(path, getDockerCredsFromVault=false, dockerUsername='grafanabot') + job.new() + + job.withNeeds(['createRelease']) + + job.withSteps( + [ + common.fetchReleaseLib, + common.fetchReleaseRepo, + common.googleAuth, + common.setupGoogleCloudSdk, + step.new('Set up QEMU', 'docker/setup-qemu-action@v3'), + step.new('set up docker buildx', 'docker/setup-buildx-action@v3'), + ] + (if getDockerCredsFromVault then [ + step.new('Login to DockerHub (from vault)', 'grafana/shared-workflows/actions/dockerhub-login@main'), + ] else [ + step.new('Login to DockerHub (from secrets)', 'docker/login-action@v3') + + step.with({ + username: dockerUsername, + password: '${{ secrets.DOCKER_PASSWORD }}', + }), + ]) + + [ + step.new('download and prepare plugins') + + step.withRun(||| + echo "downloading images to $(pwd)/plugins" + gsutil cp -r gs://${BUILD_ARTIFACTS_BUCKET}/${{ needs.createRelease.outputs.sha }}/plugins . + mkdir -p "release/%s" + ||| % path), + step.new('publish docker driver', './lib/actions/push-images') + + step.with({ + imageDir: 'plugins', + imagePrefix: '${{ env.IMAGE_PREFIX }}', + isPlugin: true, + buildDir: 'release/%s' % path, + }), + ] + ), + + publishRelease: function(dependencies=['createRelease']) + job.new() + + job.withNeeds(dependencies) + + job.withSteps([ + common.fetchReleaseRepo, + common.githubAppToken, + common.setToken, + releaseStep('publish release') + + step.withIf('${{ !fromJSON(needs.createRelease.outputs.exists) || (needs.createRelease.outputs.draft && fromJSON(needs.createRelease.outputs.draft)) }}') + + step.withEnv({ + GH_TOKEN: '${{ steps.github_app_token.outputs.token }}', + }) + + step.withRun(||| + gh release edit ${{ needs.createRelease.outputs.name }} --draft=false --latest=${{ needs.createRelease.outputs.isLatest }} + |||), + ]), } diff --git a/.github/vendor/github.com/grafana/loki-release/workflows/workflows.jsonnet b/.github/vendor/github.com/grafana/loki-release/workflows/workflows.jsonnet index 1992d23f0d3b9..930aa539f25c6 100644 --- a/.github/vendor/github.com/grafana/loki-release/workflows/workflows.jsonnet +++ b/.github/vendor/github.com/grafana/loki-release/workflows/workflows.jsonnet @@ -1,15 +1,22 @@ local lokiRelease = import 'main.jsonnet'; local build = lokiRelease.build; + + +local buildImage = 'grafana/loki-build-image:0.34.3'; +local dockerPluginDir = 'clients/cmd/docker-driver'; + { '.github/workflows/release-pr.yml': std.manifestYamlDoc( lokiRelease.releasePRWorkflow( imageJobs={ loki: build.image('fake-loki', 'cmd/loki'), + 'loki-docker-driver': build.dockerPlugin('loki-docker-driver', dockerPluginDir, buildImage=buildImage, platform=['linux/amd64', 'linux/arm64']), }, + buildImage=buildImage, buildArtifactsBucket='loki-build-artifacts', branches=['release-[0-9]+.[0-9]+.x'], imagePrefix='trevorwhitney075', - releaseLibRef='release-1.12.x', + releaseLibRef='release-1.14.x', releaseRepo='grafana/loki-release', skipValidation=false, versioningStrategy='always-bump-patch', @@ -21,12 +28,14 @@ local build = lokiRelease.build; lokiRelease.releasePRWorkflow( imageJobs={ loki: build.image('fake-loki', 'cmd/loki'), + 'loki-docker-driver': build.dockerPlugin('loki-docker-driver', dockerPluginDir, buildImage=buildImage, platform=['linux/amd64', 'linux/arm64']), }, + buildImage=buildImage, buildArtifactsBucket='loki-build-artifacts', branches=['release-[0-9]+.[0-9]+.x'], dryRun=true, imagePrefix='trevorwhitney075', - releaseLibRef='release-1.12.x', + releaseLibRef='release-1.14.x', releaseRepo='grafana/loki-release', skipValidation=false, versioningStrategy='always-bump-patch', @@ -41,11 +50,13 @@ local build = lokiRelease.build; lokiRelease.releaseWorkflow( branches=['release-[0-9]+.[0-9]+.x'], buildArtifactsBucket='loki-build-artifacts', - getDockerCredsFromVault=true, + dockerUsername='trevorwhitney075', + getDockerCredsFromVault=false, imagePrefix='trevorwhitney075', - releaseLibRef='release-1.12.x', + pluginBuildDir=dockerPluginDir, + releaseLibRef='release-1.14.x', releaseRepo='grafana/loki-release', - useGitHubAppToken=false, + useGitHubAppToken=true, ) + { name: 'Create Release', on+: { diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 5e90ce2c01920..82a801ec8668a 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -2,7 +2,7 @@ "check": "uses": "grafana/loki-release/.github/workflows/check.yml@main" "with": - "build_image": "grafana/loki-build-image:0.34.1" + "build_image": "grafana/loki-build-image:0.34.3" "golang_ci_lint_version": "v1.60.3" "release_lib_ref": "main" "skip_validation": false diff --git a/.github/workflows/images.yml b/.github/workflows/images.yml index 55bbd64d6c3f9..cc56042b4420f 100644 --- a/.github/workflows/images.yml +++ b/.github/workflows/images.yml @@ -2,7 +2,7 @@ "check": "uses": "grafana/loki-release/.github/workflows/check.yml@main" "with": - "build_image": "grafana/loki-build-image:0.34.1" + "build_image": "grafana/loki-build-image:0.34.3" "golang_ci_lint_version": "v1.60.3" "release_lib_ref": "main" "skip_validation": false diff --git a/.github/workflows/minor-release-pr.yml b/.github/workflows/minor-release-pr.yml index 0db833dec9a2f..b8e46bf6816df 100644 --- a/.github/workflows/minor-release-pr.yml +++ b/.github/workflows/minor-release-pr.yml @@ -16,7 +16,7 @@ jobs: check: uses: "grafana/loki-release/.github/workflows/check.yml@main" with: - build_image: "grafana/loki-build-image:0.34.1" + build_image: "grafana/loki-build-image:0.34.3" golang_ci_lint_version: "v1.60.3" release_lib_ref: "main" skip_validation: false @@ -144,7 +144,7 @@ jobs: --env SKIP_ARM \ --volume .:/src/loki \ --workdir /src/loki \ - --entrypoint /bin/sh "grafana/loki-build-image:0.34.1" + --entrypoint /bin/sh "grafana/loki-build-image:0.34.3" git config --global --add safe.directory /src/loki echo "${NFPM_SIGNING_KEY}" > $NFPM_SIGNING_KEY_FILE make dist packages @@ -631,6 +631,7 @@ jobs: name: "parse image platform" run: | mkdir -p images + mkdir -p plugins platform="$(echo "${{ matrix.platform}}" | sed "s/\(.*\)\/\(.*\)/\1-\2/")" echo "platform=${platform}" >> $GITHUB_OUTPUT @@ -641,37 +642,33 @@ jobs: echo "plugin_arch=" >> $GITHUB_OUTPUT fi working-directory: "release" - - env: - IMAGE_TAG: "${{ needs.version.outputs.version }}" - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" name: "Build and export" timeout-minutes: "${{ fromJSON(env.BUILD_TIMEOUT) }}" uses: "docker/build-push-action@v6" with: - build-args: "IMAGE_TAG=${{ needs.version.outputs.version }},GOARCH=${{ steps.platform.outputs.platform_short }}" + build-args: | + IMAGE_TAG=${{ needs.version.outputs.version }} + GOARCH=${{ steps.platform.outputs.platform_short }} + BUILD_IMAGE=grafana/loki-build-image:0.34.3 context: "release" file: "release/clients/cmd/docker-driver/Dockerfile" - outputs: "type=docker,dest=release/images/grafana/loki-docker-driver-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + outputs: "type=local,dest=release/plugins/loki-docker-driver-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}" platforms: "${{ matrix.platform }}" push: false - tags: "${{ env.IMAGE_PREFIX }}/grafana/loki-docker-driver:${{ needs.version.outputs.version }}-${{ steps.platform.outputs.platform_short }}" - - env: - BUILD_DIR: "release/clients/cmd/docker-driver" - IMAGE_TAG: "${{ needs.version.outputs.version }}" - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" - name: "Package as Docker plugin" + tags: "${{ env.IMAGE_PREFIX }}/loki-docker-driver:${{ needs.version.outputs.version }}-${{ steps.platform.outputs.platform_short }}" + - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + name: "compress rootfs" run: | - rm -rf "${{ env.BUILD_DIR }}/rootfs" || true - mkdir "${{ env.BUILD_DIR }}/rootfs" - tar -x -C "${{ env.BUILD_DIR }}/rootfs" -f "release/images/%s-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" - docker plugin create "${{ env.IMAGE_TAG }}${{ steps.platform.outputs.plugin_arch }}" "${{ env.BUILD_DIR }}" - working-directory: "release" + tar -cf release/plugins/loki-docker-driver-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar \ + -C release/plugins/loki-docker-driver-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }} \ + . - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" name: "upload artifacts" uses: "google-github-actions/upload-cloud-storage@v2" with: - destination: "${{ env.BUILD_ARTIFACTS_BUCKET }}/${{ github.sha }}/images" - path: "release/images/grafana/loki-docker-driver-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + destination: "${{ env.BUILD_ARTIFACTS_BUCKET }}/${{ github.sha }}/plugins" + path: "release/plugins/loki-docker-driver-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" process_gcloudignore: false strategy: fail-fast: true diff --git a/.github/workflows/patch-release-pr.yml b/.github/workflows/patch-release-pr.yml index f1b904d22f407..cfd24aefb9c20 100644 --- a/.github/workflows/patch-release-pr.yml +++ b/.github/workflows/patch-release-pr.yml @@ -16,7 +16,7 @@ jobs: check: uses: "grafana/loki-release/.github/workflows/check.yml@main" with: - build_image: "grafana/loki-build-image:0.34.1" + build_image: "grafana/loki-build-image:0.34.3" golang_ci_lint_version: "v1.60.3" release_lib_ref: "main" skip_validation: false @@ -144,7 +144,7 @@ jobs: --env SKIP_ARM \ --volume .:/src/loki \ --workdir /src/loki \ - --entrypoint /bin/sh "grafana/loki-build-image:0.34.1" + --entrypoint /bin/sh "grafana/loki-build-image:0.34.3" git config --global --add safe.directory /src/loki echo "${NFPM_SIGNING_KEY}" > $NFPM_SIGNING_KEY_FILE make dist packages @@ -631,6 +631,7 @@ jobs: name: "parse image platform" run: | mkdir -p images + mkdir -p plugins platform="$(echo "${{ matrix.platform}}" | sed "s/\(.*\)\/\(.*\)/\1-\2/")" echo "platform=${platform}" >> $GITHUB_OUTPUT @@ -641,37 +642,33 @@ jobs: echo "plugin_arch=" >> $GITHUB_OUTPUT fi working-directory: "release" - - env: - IMAGE_TAG: "${{ needs.version.outputs.version }}" - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" name: "Build and export" timeout-minutes: "${{ fromJSON(env.BUILD_TIMEOUT) }}" uses: "docker/build-push-action@v6" with: - build-args: "IMAGE_TAG=${{ needs.version.outputs.version }},GOARCH=${{ steps.platform.outputs.platform_short }}" + build-args: | + IMAGE_TAG=${{ needs.version.outputs.version }} + GOARCH=${{ steps.platform.outputs.platform_short }} + BUILD_IMAGE=grafana/loki-build-image:0.34.3 context: "release" file: "release/clients/cmd/docker-driver/Dockerfile" - outputs: "type=docker,dest=release/images/grafana/loki-docker-driver-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + outputs: "type=local,dest=release/plugins/loki-docker-driver-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}" platforms: "${{ matrix.platform }}" push: false - tags: "${{ env.IMAGE_PREFIX }}/grafana/loki-docker-driver:${{ needs.version.outputs.version }}-${{ steps.platform.outputs.platform_short }}" - - env: - BUILD_DIR: "release/clients/cmd/docker-driver" - IMAGE_TAG: "${{ needs.version.outputs.version }}" - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" - name: "Package as Docker plugin" + tags: "${{ env.IMAGE_PREFIX }}/loki-docker-driver:${{ needs.version.outputs.version }}-${{ steps.platform.outputs.platform_short }}" + - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" + name: "compress rootfs" run: | - rm -rf "${{ env.BUILD_DIR }}/rootfs" || true - mkdir "${{ env.BUILD_DIR }}/rootfs" - tar -x -C "${{ env.BUILD_DIR }}/rootfs" -f "release/images/%s-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" - docker plugin create "${{ env.IMAGE_TAG }}${{ steps.platform.outputs.plugin_arch }}" "${{ env.BUILD_DIR }}" - working-directory: "release" + tar -cf release/plugins/loki-docker-driver-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar \ + -C release/plugins/loki-docker-driver-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }} \ + . - if: "${{ fromJSON(needs.version.outputs.pr_created) }}" name: "upload artifacts" uses: "google-github-actions/upload-cloud-storage@v2" with: - destination: "${{ env.BUILD_ARTIFACTS_BUCKET }}/${{ github.sha }}/images" - path: "release/images/grafana/loki-docker-driver-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" + destination: "${{ env.BUILD_ARTIFACTS_BUCKET }}/${{ github.sha }}/plugins" + path: "release/plugins/loki-docker-driver-${{ needs.version.outputs.version}}-${{ steps.platform.outputs.platform }}.tar" process_gcloudignore: false strategy: fail-fast: true diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 1628a0b57dfb5..426f523f80da7 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -110,6 +110,48 @@ jobs: parent: false path: "release/dist" process_gcloudignore: false + publishDockerPlugins: + needs: + - "createRelease" + runs-on: "ubuntu-latest" + steps: + - name: "pull release library code" + uses: "actions/checkout@v4" + with: + path: "lib" + ref: "${{ env.RELEASE_LIB_REF }}" + repository: "grafana/loki-release" + - name: "pull code to release" + uses: "actions/checkout@v4" + with: + path: "release" + repository: "${{ env.RELEASE_REPO }}" + - name: "auth gcs" + uses: "google-github-actions/auth@v2" + with: + credentials_json: "${{ secrets.GCS_SERVICE_ACCOUNT_KEY }}" + - name: "Set up Cloud SDK" + uses: "google-github-actions/setup-gcloud@v2" + with: + version: ">= 452.0.0" + - name: "Set up QEMU" + uses: "docker/setup-qemu-action@v3" + - name: "set up docker buildx" + uses: "docker/setup-buildx-action@v3" + - name: "Login to DockerHub (from vault)" + uses: "grafana/shared-workflows/actions/dockerhub-login@main" + - name: "download and prepare plugins" + run: | + echo "downloading images to $(pwd)/plugins" + gsutil cp -r gs://${BUILD_ARTIFACTS_BUCKET}/${{ needs.createRelease.outputs.sha }}/plugins . + mkdir -p "release/clients/cmd/docker-driver" + - name: "publish docker driver" + uses: "./lib/actions/push-images" + with: + buildDir: "release/clients/cmd/docker-driver" + imageDir: "plugins" + imagePrefix: "${{ env.IMAGE_PREFIX }}" + isPlugin: true publishImages: needs: - "createRelease" @@ -148,6 +190,7 @@ jobs: needs: - "createRelease" - "publishImages" + - "publishDockerPlugins" runs-on: "ubuntu-latest" steps: - name: "pull code to release" diff --git a/Makefile b/Makefile index 2b152ac6ae739..76de2fac98dfc 100644 --- a/Makefile +++ b/Makefile @@ -19,7 +19,7 @@ CI ?= false # Ensure you run `make release-workflows` after changing this GO_VERSION := 1.23.1 -BUILD_IMAGE_TAG := 0.34.1 +BUILD_IMAGE_TAG := 0.34.3 IMAGE_TAG ?= $(shell ./tools/image-tag) GIT_REVISION := $(shell git rev-parse --short HEAD) @@ -672,12 +672,12 @@ logql-analyzer-push: logql-analyzer-image # Build image build-image: ## build the build docker image $(OCI_BUILD) -t $(BUILD_IMAGE) ./loki-build-image -build-image-push: build-image +build-image-push: ifneq (,$(findstring WIP,$(IMAGE_TAG))) @echo "Cannot push a WIP image, commit changes first"; \ false; endif - $(OCI_PUSH) -t $(BUILD_IMAGE) + DOCKER_BUILDKIT=1 docker buildx build $(OCI_PLATFORMS) $(OCI_BUILD_ARGS) $(OCI_PUSH_ARGS) -t $(BUILD_IMAGE) ./loki-build-image # Loki Operator loki-operator-image: ## build the operator docker image @@ -873,8 +873,12 @@ scan-vulnerabilities: trivy snyk .PHONY: release-workflows release-workflows: +ifeq ($(BUILD_IN_CONTAINER),true) + $(run_in_container) +else pushd $(CURDIR)/.github && jb update && popd jsonnet -SJ .github/vendor -m .github/workflows -V BUILD_IMAGE_VERSION=$(BUILD_IMAGE_TAG) .github/release-workflows.jsonnet +endif .PHONY: release-workflows-check release-workflows-check: