From 273c8c97f7409d4c81150588ececf9b7d31382a8 Mon Sep 17 00:00:00 2001 From: George Robinson Date: Wed, 29 May 2024 12:12:51 +0100 Subject: [PATCH 1/4] feat: Add ingester_chunks_flush_failures_total (#12925) Signed-off-by: George Robinson --- pkg/ingester/flush.go | 1 + pkg/ingester/metrics.go | 6 ++++++ 2 files changed, 7 insertions(+) diff --git a/pkg/ingester/flush.go b/pkg/ingester/flush.go index 53c1ca04af644..be4899325ed45 100644 --- a/pkg/ingester/flush.go +++ b/pkg/ingester/flush.go @@ -363,6 +363,7 @@ func (i *Ingester) encodeChunk(ctx context.Context, ch *chunk.Chunk, desc *chunk // chunk to have another opportunity to be flushed. func (i *Ingester) flushChunk(ctx context.Context, ch *chunk.Chunk) error { if err := i.store.Put(ctx, []chunk.Chunk{*ch}); err != nil { + i.metrics.chunksFlushFailures.Inc() return fmt.Errorf("store put chunk: %w", err) } i.metrics.flushedChunksStats.Inc(1) diff --git a/pkg/ingester/metrics.go b/pkg/ingester/metrics.go index 96b2af61f379d..4d0edd9090267 100644 --- a/pkg/ingester/metrics.go +++ b/pkg/ingester/metrics.go @@ -46,6 +46,7 @@ type ingesterMetrics struct { chunkSizePerTenant *prometheus.CounterVec chunkAge prometheus.Histogram chunkEncodeTime prometheus.Histogram + chunksFlushFailures prometheus.Counter chunksFlushedPerReason *prometheus.CounterVec chunkLifespan prometheus.Histogram flushedChunksStats *analytics.Counter @@ -229,6 +230,11 @@ func newIngesterMetrics(r prometheus.Registerer) *ingesterMetrics { // 10ms to 10s. Buckets: prometheus.ExponentialBuckets(0.01, 4, 6), }), + chunksFlushFailures: promauto.With(r).NewCounter(prometheus.CounterOpts{ + Namespace: constants.Loki, + Name: "ingester_chunks_flush_failures_total", + Help: "Total number of flush failures.", + }), chunksFlushedPerReason: promauto.With(r).NewCounterVec(prometheus.CounterOpts{ Namespace: "loki", Name: "ingester_chunks_flushed_total", From 9b7f8fd721bef36fe47efb2f4ad23493a8aeaebf Mon Sep 17 00:00:00 2001 From: George Robinson Date: Thu, 20 Jun 2024 16:19:51 +0100 Subject: [PATCH 2/4] Fix missing constant in 2.9.x --- pkg/ingester/metrics.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/ingester/metrics.go b/pkg/ingester/metrics.go index 4d0edd9090267..cb376ce18b837 100644 --- a/pkg/ingester/metrics.go +++ b/pkg/ingester/metrics.go @@ -231,7 +231,7 @@ func newIngesterMetrics(r prometheus.Registerer) *ingesterMetrics { Buckets: prometheus.ExponentialBuckets(0.01, 4, 6), }), chunksFlushFailures: promauto.With(r).NewCounter(prometheus.CounterOpts{ - Namespace: constants.Loki, + Namespace: "loki", Name: "ingester_chunks_flush_failures_total", Help: "Total number of flush failures.", }), From 06e2325ea0859b38961ca6a4ff61f9927bd01709 Mon Sep 17 00:00:00 2001 From: Trevor Whitney Date: Mon, 1 Jul 2024 12:08:45 -0600 Subject: [PATCH 3/4] ci: backport failCheck --- .github/release-workflows.jsonnet | 6 +--- .github/workflows/check.yml | 8 ++--- .github/workflows/minor-release-pr.yml | 42 +++++++++++++------------- .github/workflows/patch-release-pr.yml | 42 +++++++++++++------------- .github/workflows/release.yml | 2 +- 5 files changed, 48 insertions(+), 52 deletions(-) diff --git a/.github/release-workflows.jsonnet b/.github/release-workflows.jsonnet index cef76ba0563d3..1d3f2aa6dcf92 100644 --- a/.github/release-workflows.jsonnet +++ b/.github/release-workflows.jsonnet @@ -2,11 +2,7 @@ local lokiRelease = import 'workflows/main.jsonnet'; local build = lokiRelease.build; local job = lokiRelease.job; -local releaseLibRef = std.filter( - function(dep) dep.source.git.remote == 'https://github.com/grafana/loki-release.git', - (import 'jsonnetfile.json').dependencies -)[0].version; - +local releaseLibRef = "release-1.12.x"; local checkTemplate = 'grafana/loki-release/.github/workflows/check.yml@%s' % releaseLibRef; local imageJobs = { diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index 084c8638f5c17..b9a0adc1aeddd 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -1,10 +1,10 @@ "jobs": "check": - "uses": "grafana/loki-release/.github/workflows/check.yml@loki-2.9.x" + "uses": "grafana/loki-release/.github/workflows/check.yml@release-1.12.x" "with": - "build_image": "grafana/loki-build-image:0.33.1" - "golang_ci_lint_version": "v1.55.1" - "release_lib_ref": "loki-2.9.x" + "build_image": "grafana/loki-build-image:0.30.1" + "golang_ci_lint_version": "v1.51.2" + "release_lib_ref": "release-1.12.x" "skip_validation": false "use_github_app_token": true "name": "check" diff --git a/.github/workflows/minor-release-pr.yml b/.github/workflows/minor-release-pr.yml index 3a05f63e1ada8..c3f21b35d5792 100644 --- a/.github/workflows/minor-release-pr.yml +++ b/.github/workflows/minor-release-pr.yml @@ -5,18 +5,18 @@ env: CHANGELOG_PATH: "CHANGELOG.md" DOCKER_USERNAME: "grafana" IMAGE_PREFIX: "grafana" - RELEASE_LIB_REF: "loki-2.9.x" + RELEASE_LIB_REF: "release-1.12.x" RELEASE_REPO: "grafana/loki" SKIP_VALIDATION: false USE_GITHUB_APP_TOKEN: true VERSIONING_STRATEGY: "always-bump-minor" jobs: check: - uses: "grafana/loki-release/.github/workflows/check.yml@loki-2.9.x" + uses: "grafana/loki-release/.github/workflows/check.yml@release-1.12.x" with: - build_image: "grafana/loki-build-image:0.33.1" - golang_ci_lint_version: "v1.55.1" - release_lib_ref: "loki-2.9.x" + build_image: "grafana/loki-build-image:0.30.1" + golang_ci_lint_version: "v1.51.2" + release_lib_ref: "release-1.12.x" skip_validation: false use_github_app_token: true create-release-pr: @@ -89,7 +89,7 @@ jobs: --target-branch "${{ steps.extract_branch.outputs.branch }}" \ --token "${{ steps.github_app_token.outputs.token }}" \ --versioning-strategy "${{ env.VERSIONING_STRATEGY }}" - + working-directory: "lib" dist: needs: @@ -136,7 +136,7 @@ jobs: --env SKIP_ARM \ --volume .:/src/loki \ --workdir /src/loki \ - --entrypoint /bin/sh "grafana/loki-build-image:0.33.1" + --entrypoint /bin/sh "grafana/loki-build-image:0.30.1" git config --global --add safe.directory /src/loki echo "${NFPM_SIGNING_KEY}" > $NFPM_SIGNING_KEY_FILE make dist packages @@ -181,7 +181,7 @@ jobs: name: "parse image platform" run: | mkdir -p images - + platform="$(echo "${{ matrix.platform}}" | sed "s/\(.*\)\/\(.*\)/\1-\2/")" echo "platform=${platform}" >> $GITHUB_OUTPUT echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT @@ -243,7 +243,7 @@ jobs: name: "parse image platform" run: | mkdir -p images - + platform="$(echo "${{ matrix.platform}}" | sed "s/\(.*\)\/\(.*\)/\1-\2/")" echo "platform=${platform}" >> $GITHUB_OUTPUT echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT @@ -305,7 +305,7 @@ jobs: name: "parse image platform" run: | mkdir -p images - + platform="$(echo "${{ matrix.platform}}" | sed "s/\(.*\)\/\(.*\)/\1-\2/")" echo "platform=${platform}" >> $GITHUB_OUTPUT echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT @@ -369,7 +369,7 @@ jobs: name: "parse image platform" run: | mkdir -p images - + platform="$(echo "${{ matrix.platform}}" | sed "s/\(.*\)\/\(.*\)/\1-\2/")" echo "platform=${platform}" >> $GITHUB_OUTPUT echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT @@ -431,7 +431,7 @@ jobs: name: "parse image platform" run: | mkdir -p images - + platform="$(echo "${{ matrix.platform}}" | sed "s/\(.*\)\/\(.*\)/\1-\2/")" echo "platform=${platform}" >> $GITHUB_OUTPUT echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT @@ -495,7 +495,7 @@ jobs: name: "parse image platform" run: | mkdir -p images - + platform="$(echo "${{ matrix.platform}}" | sed "s/\(.*\)\/\(.*\)/\1-\2/")" echo "platform=${platform}" >> $GITHUB_OUTPUT echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT @@ -559,7 +559,7 @@ jobs: name: "parse image platform" run: | mkdir -p images - + platform="$(echo "${{ matrix.platform}}" | sed "s/\(.*\)\/\(.*\)/\1-\2/")" echo "platform=${platform}" >> $GITHUB_OUTPUT echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT @@ -623,7 +623,7 @@ jobs: name: "parse image platform" run: | mkdir -p images - + platform="$(echo "${{ matrix.platform}}" | sed "s/\(.*\)\/\(.*\)/\1-\2/")" echo "platform=${platform}" >> $GITHUB_OUTPUT echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT @@ -687,7 +687,7 @@ jobs: name: "parse image platform" run: | mkdir -p images - + platform="$(echo "${{ matrix.platform}}" | sed "s/\(.*\)\/\(.*\)/\1-\2/")" echo "platform=${platform}" >> $GITHUB_OUTPUT echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT @@ -778,16 +778,16 @@ jobs: --target-branch "${{ steps.extract_branch.outputs.branch }}" \ --token "${{ steps.github_app_token.outputs.token }}" \ --versioning-strategy "${{ env.VERSIONING_STRATEGY }}" - + cat release.json - - if [[ `jq length release.json` -gt 1 ]]; then + + if [[ `jq length release.json` -gt 1 ]]; then echo 'release-please would create more than 1 PR, so cannot determine correct version' echo "pr_created=false" >> $GITHUB_OUTPUT exit 1 fi - - if [[ `jq length release.json` -eq 0 ]]; then + + if [[ `jq length release.json` -eq 0 ]]; then echo "pr_created=false" >> $GITHUB_OUTPUT else version="$(npm run --silent get-version)" diff --git a/.github/workflows/patch-release-pr.yml b/.github/workflows/patch-release-pr.yml index 3d67b37398c98..8f79316865bad 100644 --- a/.github/workflows/patch-release-pr.yml +++ b/.github/workflows/patch-release-pr.yml @@ -5,18 +5,18 @@ env: CHANGELOG_PATH: "CHANGELOG.md" DOCKER_USERNAME: "grafana" IMAGE_PREFIX: "grafana" - RELEASE_LIB_REF: "loki-2.9.x" + RELEASE_LIB_REF: "release-1.12.x" RELEASE_REPO: "grafana/loki" SKIP_VALIDATION: false USE_GITHUB_APP_TOKEN: true VERSIONING_STRATEGY: "always-bump-patch" jobs: check: - uses: "grafana/loki-release/.github/workflows/check.yml@loki-2.9.x" + uses: "grafana/loki-release/.github/workflows/check.yml@release-1.12.x" with: - build_image: "grafana/loki-build-image:0.33.1" - golang_ci_lint_version: "v1.55.1" - release_lib_ref: "loki-2.9.x" + build_image: "grafana/loki-build-image:0.30.1" + golang_ci_lint_version: "v1.51.2" + release_lib_ref: "release-1.12.x" skip_validation: false use_github_app_token: true create-release-pr: @@ -89,7 +89,7 @@ jobs: --target-branch "${{ steps.extract_branch.outputs.branch }}" \ --token "${{ steps.github_app_token.outputs.token }}" \ --versioning-strategy "${{ env.VERSIONING_STRATEGY }}" - + working-directory: "lib" dist: needs: @@ -136,7 +136,7 @@ jobs: --env SKIP_ARM \ --volume .:/src/loki \ --workdir /src/loki \ - --entrypoint /bin/sh "grafana/loki-build-image:0.33.1" + --entrypoint /bin/sh "grafana/loki-build-image:0.30.1" git config --global --add safe.directory /src/loki echo "${NFPM_SIGNING_KEY}" > $NFPM_SIGNING_KEY_FILE make dist packages @@ -181,7 +181,7 @@ jobs: name: "parse image platform" run: | mkdir -p images - + platform="$(echo "${{ matrix.platform}}" | sed "s/\(.*\)\/\(.*\)/\1-\2/")" echo "platform=${platform}" >> $GITHUB_OUTPUT echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT @@ -243,7 +243,7 @@ jobs: name: "parse image platform" run: | mkdir -p images - + platform="$(echo "${{ matrix.platform}}" | sed "s/\(.*\)\/\(.*\)/\1-\2/")" echo "platform=${platform}" >> $GITHUB_OUTPUT echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT @@ -305,7 +305,7 @@ jobs: name: "parse image platform" run: | mkdir -p images - + platform="$(echo "${{ matrix.platform}}" | sed "s/\(.*\)\/\(.*\)/\1-\2/")" echo "platform=${platform}" >> $GITHUB_OUTPUT echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT @@ -369,7 +369,7 @@ jobs: name: "parse image platform" run: | mkdir -p images - + platform="$(echo "${{ matrix.platform}}" | sed "s/\(.*\)\/\(.*\)/\1-\2/")" echo "platform=${platform}" >> $GITHUB_OUTPUT echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT @@ -431,7 +431,7 @@ jobs: name: "parse image platform" run: | mkdir -p images - + platform="$(echo "${{ matrix.platform}}" | sed "s/\(.*\)\/\(.*\)/\1-\2/")" echo "platform=${platform}" >> $GITHUB_OUTPUT echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT @@ -495,7 +495,7 @@ jobs: name: "parse image platform" run: | mkdir -p images - + platform="$(echo "${{ matrix.platform}}" | sed "s/\(.*\)\/\(.*\)/\1-\2/")" echo "platform=${platform}" >> $GITHUB_OUTPUT echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT @@ -559,7 +559,7 @@ jobs: name: "parse image platform" run: | mkdir -p images - + platform="$(echo "${{ matrix.platform}}" | sed "s/\(.*\)\/\(.*\)/\1-\2/")" echo "platform=${platform}" >> $GITHUB_OUTPUT echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT @@ -623,7 +623,7 @@ jobs: name: "parse image platform" run: | mkdir -p images - + platform="$(echo "${{ matrix.platform}}" | sed "s/\(.*\)\/\(.*\)/\1-\2/")" echo "platform=${platform}" >> $GITHUB_OUTPUT echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT @@ -687,7 +687,7 @@ jobs: name: "parse image platform" run: | mkdir -p images - + platform="$(echo "${{ matrix.platform}}" | sed "s/\(.*\)\/\(.*\)/\1-\2/")" echo "platform=${platform}" >> $GITHUB_OUTPUT echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT @@ -778,16 +778,16 @@ jobs: --target-branch "${{ steps.extract_branch.outputs.branch }}" \ --token "${{ steps.github_app_token.outputs.token }}" \ --versioning-strategy "${{ env.VERSIONING_STRATEGY }}" - + cat release.json - - if [[ `jq length release.json` -gt 1 ]]; then + + if [[ `jq length release.json` -gt 1 ]]; then echo 'release-please would create more than 1 PR, so cannot determine correct version' echo "pr_created=false" >> $GITHUB_OUTPUT exit 1 fi - - if [[ `jq length release.json` -eq 0 ]]; then + + if [[ `jq length release.json` -eq 0 ]]; then echo "pr_created=false" >> $GITHUB_OUTPUT else version="$(npm run --silent get-version)" diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index 9303c31f6ddcb..b00dd63ec566a 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -3,7 +3,7 @@ concurrency: env: IMAGE_PREFIX: "grafana" PUBLISH_TO_GCS: false - RELEASE_LIB_REF: "loki-2.9.x" + RELEASE_LIB_REF: "release-1.12.x" RELEASE_REPO: "grafana/loki" USE_GITHUB_APP_TOKEN: false jobs: From 31470b26e4107f1fb6cb88175491d6a27748cab7 Mon Sep 17 00:00:00 2001 From: Trevor Whitney Date: Mon, 1 Jul 2024 12:19:30 -0600 Subject: [PATCH 4/4] ci: back to using tag --- .github/release-workflows.jsonnet | 2 +- .github/workflows/check.yml | 4 ++-- .github/workflows/minor-release-pr.yml | 6 +++--- .github/workflows/patch-release-pr.yml | 6 +++--- .github/workflows/release.yml | 2 +- 5 files changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/release-workflows.jsonnet b/.github/release-workflows.jsonnet index 1d3f2aa6dcf92..ca7f22085e251 100644 --- a/.github/release-workflows.jsonnet +++ b/.github/release-workflows.jsonnet @@ -2,7 +2,7 @@ local lokiRelease = import 'workflows/main.jsonnet'; local build = lokiRelease.build; local job = lokiRelease.job; -local releaseLibRef = "release-1.12.x"; +local releaseLibRef = 'loki-2.9.x'; local checkTemplate = 'grafana/loki-release/.github/workflows/check.yml@%s' % releaseLibRef; local imageJobs = { diff --git a/.github/workflows/check.yml b/.github/workflows/check.yml index b9a0adc1aeddd..50aff9f6ddec4 100644 --- a/.github/workflows/check.yml +++ b/.github/workflows/check.yml @@ -1,10 +1,10 @@ "jobs": "check": - "uses": "grafana/loki-release/.github/workflows/check.yml@release-1.12.x" + "uses": "grafana/loki-release/.github/workflows/check.yml@loki-2.9.x" "with": "build_image": "grafana/loki-build-image:0.30.1" "golang_ci_lint_version": "v1.51.2" - "release_lib_ref": "release-1.12.x" + "release_lib_ref": "loki-2.9.x" "skip_validation": false "use_github_app_token": true "name": "check" diff --git a/.github/workflows/minor-release-pr.yml b/.github/workflows/minor-release-pr.yml index c3f21b35d5792..68a65b48b0ba9 100644 --- a/.github/workflows/minor-release-pr.yml +++ b/.github/workflows/minor-release-pr.yml @@ -5,18 +5,18 @@ env: CHANGELOG_PATH: "CHANGELOG.md" DOCKER_USERNAME: "grafana" IMAGE_PREFIX: "grafana" - RELEASE_LIB_REF: "release-1.12.x" + RELEASE_LIB_REF: "loki-2.9.x" RELEASE_REPO: "grafana/loki" SKIP_VALIDATION: false USE_GITHUB_APP_TOKEN: true VERSIONING_STRATEGY: "always-bump-minor" jobs: check: - uses: "grafana/loki-release/.github/workflows/check.yml@release-1.12.x" + uses: "grafana/loki-release/.github/workflows/check.yml@loki-2.9.x" with: build_image: "grafana/loki-build-image:0.30.1" golang_ci_lint_version: "v1.51.2" - release_lib_ref: "release-1.12.x" + release_lib_ref: "loki-2.9.x" skip_validation: false use_github_app_token: true create-release-pr: diff --git a/.github/workflows/patch-release-pr.yml b/.github/workflows/patch-release-pr.yml index 8f79316865bad..6c25955897441 100644 --- a/.github/workflows/patch-release-pr.yml +++ b/.github/workflows/patch-release-pr.yml @@ -5,18 +5,18 @@ env: CHANGELOG_PATH: "CHANGELOG.md" DOCKER_USERNAME: "grafana" IMAGE_PREFIX: "grafana" - RELEASE_LIB_REF: "release-1.12.x" + RELEASE_LIB_REF: "loki-2.9.x" RELEASE_REPO: "grafana/loki" SKIP_VALIDATION: false USE_GITHUB_APP_TOKEN: true VERSIONING_STRATEGY: "always-bump-patch" jobs: check: - uses: "grafana/loki-release/.github/workflows/check.yml@release-1.12.x" + uses: "grafana/loki-release/.github/workflows/check.yml@loki-2.9.x" with: build_image: "grafana/loki-build-image:0.30.1" golang_ci_lint_version: "v1.51.2" - release_lib_ref: "release-1.12.x" + release_lib_ref: "loki-2.9.x" skip_validation: false use_github_app_token: true create-release-pr: diff --git a/.github/workflows/release.yml b/.github/workflows/release.yml index b00dd63ec566a..9303c31f6ddcb 100644 --- a/.github/workflows/release.yml +++ b/.github/workflows/release.yml @@ -3,7 +3,7 @@ concurrency: env: IMAGE_PREFIX: "grafana" PUBLISH_TO_GCS: false - RELEASE_LIB_REF: "release-1.12.x" + RELEASE_LIB_REF: "loki-2.9.x" RELEASE_REPO: "grafana/loki" USE_GITHUB_APP_TOKEN: false jobs: