From d0f1905cf45e183c1c7fc0b3421e4a36ab5ddf04 Mon Sep 17 00:00:00 2001 From: Huang Chen-Yi Date: Wed, 25 Sep 2024 23:27:44 +0800 Subject: [PATCH 01/11] docs: Fix link bracket for query-acceleration-blooms (#14262) --- docs/sources/operations/query-acceleration-blooms.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/sources/operations/query-acceleration-blooms.md b/docs/sources/operations/query-acceleration-blooms.md index 50999419030f5..2fec5f2922705 100644 --- a/docs/sources/operations/query-acceleration-blooms.md +++ b/docs/sources/operations/query-acceleration-blooms.md @@ -152,7 +152,7 @@ Example calculation for storage requirements of blooms for a single tenant. Since reading blooms depends heavily on disk IOPS, Bloom Gateways should make use of multiple, locally attached SSD disks (NVMe) to increase i/o throughput. -Multiple directories on different disk mounts can be specified using the `-bloom.shipper.working-directory` [setting]storage-config-cfg] +Multiple directories on different disk mounts can be specified using the `-bloom.shipper.working-directory` [setting][storage-config-cfg] when using a comma separated list of mount points, for example: ``` -bloom.shipper.working-directory="/mnt/data0,/mnt/data1,/mnt/data2,/mnt/data3" From c001a1d93af5438fef521460dcba650b44629a93 Mon Sep 17 00:00:00 2001 From: Trevor Whitney Date: Wed, 25 Sep 2024 09:37:16 -0600 Subject: [PATCH 02/11] fix: allow any level for aggregated metrics (#14255) --- pkg/pattern/instance.go | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/pkg/pattern/instance.go b/pkg/pattern/instance.go index 24f2814e467f5..6e3a3de998be5 100644 --- a/pkg/pattern/instance.go +++ b/pkg/pattern/instance.go @@ -9,7 +9,6 @@ import ( "sync" "github.com/go-kit/log" - "github.com/go-kit/log/level" "github.com/grafana/dskit/httpgrpc" "github.com/grafana/dskit/multierror" "github.com/grafana/dskit/ring" @@ -264,20 +263,11 @@ func (i *instance) Observe(stream string, entries []logproto.Entry) { streamMetrics, ok := i.aggMetricsByStreamAndLevel[stream] if !ok { - streamMetrics = make(map[string]*aggregatedMetrics, len(constants.LogLevels)) - for _, l := range constants.LogLevels { - streamMetrics[l] = &aggregatedMetrics{} - } + streamMetrics = map[string]*aggregatedMetrics{} } if _, ok := streamMetrics[lvl]; !ok { - level.Warn(i.logger).Log( - "msg", "unknown log level while observing stream", - "level", lvl, - "stream", stream, - ) - - lvl = constants.LogLevelUnknown + streamMetrics[lvl] = &aggregatedMetrics{} } streamMetrics[lvl].bytes += uint64(len(entry.Line)) From 73d69a11fe875eae39e4915a01ae125dfbbf40b1 Mon Sep 17 00:00:00 2001 From: Ashwanth Date: Wed, 25 Sep 2024 21:37:24 +0530 Subject: [PATCH 03/11] chore: fix push pkg vendoring (#14261) --- go.mod | 2 +- vendor/modules.txt | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index d17bd8e1ace81..6b76cbd817ad8 100644 --- a/go.mod +++ b/go.mod @@ -124,7 +124,7 @@ require ( github.com/fsnotify/fsnotify v1.7.0 github.com/gogo/googleapis v1.4.1 github.com/grafana/jsonparser v0.0.0-20240425183733-ea80629e1a32 - github.com/grafana/loki/pkg/push v0.0.0-20240923094301-8ebb2b5300f0 + github.com/grafana/loki/pkg/push v0.0.0-20240924133635-758364c7775f github.com/hashicorp/golang-lru/v2 v2.0.7 github.com/hashicorp/raft v1.7.1 github.com/hashicorp/raft-wal v0.4.1 diff --git a/vendor/modules.txt b/vendor/modules.txt index 1c75c23800a42..b2b8f38c4b5f6 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1034,7 +1034,7 @@ github.com/grafana/gomemcache/memcache # github.com/grafana/jsonparser v0.0.0-20240425183733-ea80629e1a32 ## explicit; go 1.13 github.com/grafana/jsonparser -# github.com/grafana/loki/pkg/push v0.0.0-20240923094301-8ebb2b5300f0 => ./pkg/push +# github.com/grafana/loki/pkg/push v0.0.0-20240924133635-758364c7775f => ./pkg/push ## explicit; go 1.19 github.com/grafana/loki/pkg/push # github.com/grafana/pyroscope-go/godeltaprof v0.1.8 From ee6e1cf78864ad3ed915056f695e1f556cc4a22e Mon Sep 17 00:00:00 2001 From: Andre Ziviani <7469258+AndreZiviani@users.noreply.github.com> Date: Wed, 25 Sep 2024 13:40:55 -0300 Subject: [PATCH 04/11] fix(helm): Fix persistence configuration for Memcached (#14049) * Memcached must create a file smaller than disk size * `ext_wbuf_size` must be bigger than max item size and be divisible by `wbuf_size` which is 64 by default * Add pod permissions to access local volume --- docs/sources/setup/install/helm/reference.md | 11 ++++++++--- production/helm/loki/CHANGELOG.md | 4 ++++ production/helm/loki/Chart.yaml | 2 +- production/helm/loki/README.md | 2 +- .../templates/memcached/_memcached-statefulset.tpl | 4 +++- production/helm/loki/values.yaml | 10 +++++++--- 6 files changed, 24 insertions(+), 9 deletions(-) diff --git a/docs/sources/setup/install/helm/reference.md b/docs/sources/setup/install/helm/reference.md index a2f8abbbc945b..21c1b00ac58a0 100644 --- a/docs/sources/setup/install/helm/reference.md +++ b/docs/sources/setup/install/helm/reference.md @@ -2039,7 +2039,7 @@ null chunksCache.persistence.storageSize string - Size of persistent disk + Size of persistent disk, must be in G or Gi 
 "10G"
@@ -6545,7 +6545,12 @@ false object The SecurityContext override for memcached pods 
-{}
+{
+  "fsGroup": 11211,
+  "runAsGroup": 11211,
+  "runAsNonRoot": true,
+  "runAsUser": 11211
+}
@@ -9584,7 +9589,7 @@ null resultsCache.persistence.storageSize string - Size of persistent disk + Size of persistent disk, must be in G or Gi 
 "10G"
diff --git a/production/helm/loki/CHANGELOG.md b/production/helm/loki/CHANGELOG.md index 3525bc396547b..f41649010447e 100644 --- a/production/helm/loki/CHANGELOG.md +++ b/production/helm/loki/CHANGELOG.md @@ -13,6 +13,10 @@ Entries should include a reference to the pull request that introduced the chang [//]: # ( : do not remove this line. This locator is used by the CI pipeline to automatically create a changelog entry for each new Loki release. Add other chart versions and respective changelog entries bellow this line.) +## 6.14.1 + +- [BUGFIX] Fixed Memcached persistence options. + ## 6.14.0 - [FEATURE] Add additional service annotations for components in distributed mode diff --git a/production/helm/loki/Chart.yaml b/production/helm/loki/Chart.yaml index b05f8ff0fdd04..24e94eb8bc508 100644 --- a/production/helm/loki/Chart.yaml +++ b/production/helm/loki/Chart.yaml @@ -3,7 +3,7 @@ name: loki description: Helm chart for Grafana Loki and Grafana Enterprise Logs supporting both simple, scalable and distributed modes. type: application appVersion: 3.1.1 -version: 6.14.0 +version: 6.14.1 home: https://grafana.github.io/helm-charts sources: - https://github.com/grafana/loki diff --git a/production/helm/loki/README.md b/production/helm/loki/README.md index f55b4186debf4..dea0ec488b023 100644 --- a/production/helm/loki/README.md +++ b/production/helm/loki/README.md @@ -1,6 +1,6 @@ # loki -![Version: 6.14.0](https://img.shields.io/badge/Version-6.14.0-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 3.1.1](https://img.shields.io/badge/AppVersion-3.1.1-informational?style=flat-square) +![Version: 6.14.1](https://img.shields.io/badge/Version-6.14.1-informational?style=flat-square) ![Type: application](https://img.shields.io/badge/Type-application-informational?style=flat-square) ![AppVersion: 3.1.1](https://img.shields.io/badge/AppVersion-3.1.1-informational?style=flat-square) Helm chart for Grafana Loki and Grafana Enterprise Logs supporting both simple, scalable and distributed modes. diff --git a/production/helm/loki/templates/memcached/_memcached-statefulset.tpl b/production/helm/loki/templates/memcached/_memcached-statefulset.tpl index ce490ee6cd713..0664ba43c6a2b 100644 --- a/production/helm/loki/templates/memcached/_memcached-statefulset.tpl +++ b/production/helm/loki/templates/memcached/_memcached-statefulset.tpl @@ -102,9 +102,11 @@ spec: ports: - containerPort: {{ .port }} name: client + {{- /* Calculate storage size as round(.persistence.storageSize * 0.9). But with integer built-in operators. */}} + {{- $persistenceSize := (div (mul (trimSuffix "Gi" .persistence.storageSize | trimSuffix "G") 9) 10 ) }} args: - -m {{ .allocatedMemory }} - - --extended=modern,track_sizes{{ if .persistence.enabled }},ext_path={{ .persistence.mountPath }}/file:{{ .persistence.storageSize }}{{ end }}{{ with .extraExtendedOptions }},{{ . }}{{ end }} + - --extended=modern,track_sizes{{ if .persistence.enabled }},ext_path={{ .persistence.mountPath }}/file:{{ $persistenceSize }}G,ext_wbuf_size=16{{ end }}{{ with .extraExtendedOptions }},{{ . }}{{ end }} - -I {{ .maxItemMemory }}m - -c {{ .connectionLimit }} - -v diff --git a/production/helm/loki/values.yaml b/production/helm/loki/values.yaml index 6c72cc6b38914..fe7b9273ee80a 100644 --- a/production/helm/loki/values.yaml +++ b/production/helm/loki/values.yaml @@ -2962,7 +2962,11 @@ memcached: # -- Memcached Docker image pull policy pullPolicy: IfNotPresent # -- The SecurityContext override for memcached pods - podSecurityContext: {} + podSecurityContext: + runAsNonRoot: true + runAsUser: 11211 + runAsGroup: 11211 + fsGroup: 11211 # -- The name of the PriorityClass for memcached pods priorityClassName: null # -- The SecurityContext for memcached containers @@ -3085,7 +3089,7 @@ resultsCache: persistence: # -- Enable creating PVCs for the results-cache enabled: false - # -- Size of persistent disk + # -- Size of persistent disk, must be in G or Gi storageSize: 10G # -- Storage class to be used. # If defined, storageClassName: . @@ -3187,7 +3191,7 @@ chunksCache: persistence: # -- Enable creating PVCs for the chunks-cache enabled: false - # -- Size of persistent disk + # -- Size of persistent disk, must be in G or Gi storageSize: 10G # -- Storage class to be used. # If defined, storageClassName: . From ac422b3bc3e822b4525401496a8b73e91d566128 Mon Sep 17 00:00:00 2001 From: Dylan Guedes Date: Wed, 25 Sep 2024 14:29:12 -0300 Subject: [PATCH 05/11] feat: Introduce new `ObjectExistsWithSize` API to (#14268) **What this PR does / why we need it**: Introduce a new `ObjectExistsWithSize` API to our object storage interface. This is the same as `ObjectExists` but with the object size as part of the return value. This is useful to compare the object size present. --- pkg/ingester-rf1/objstore/storage.go | 8 ++++++ .../chunk/client/alibaba/oss_object_client.go | 19 +++++++++++--- .../chunk/client/aws/s3_storage_client.go | 26 ++++++++++++++----- .../client/aws/s3_storage_client_test.go | 18 +++++++++++++ .../chunk/client/azure/blob_storage_client.go | 23 +++++++++++++--- .../client/baidubce/bos_storage_client.go | 21 +++++++++++---- .../chunk/client/congestion/controller.go | 7 +++++ .../client/congestion/controller_test.go | 4 +++ .../chunk/client/gcp/gcs_object_client.go | 14 +++++++--- .../client/ibmcloud/cos_object_client.go | 21 +++++++++++---- .../chunk/client/local/fs_object_client.go | 13 +++++++--- .../client/local/fs_object_client_test.go | 5 ++++ pkg/storage/chunk/client/object_client.go | 1 + .../client/openstack/swift_object_client.go | 11 +++++--- .../chunk/client/prefixed_object_client.go | 4 +++ .../testutils/inmemory_storage_client.go | 15 +++++++---- pkg/tool/audit/audit_test.go | 11 +++++--- 17 files changed, 178 insertions(+), 43 deletions(-) diff --git a/pkg/ingester-rf1/objstore/storage.go b/pkg/ingester-rf1/objstore/storage.go index ec0d734b316b7..5a8c61fd5117b 100644 --- a/pkg/ingester-rf1/objstore/storage.go +++ b/pkg/ingester-rf1/objstore/storage.go @@ -70,6 +70,14 @@ func (m *Multi) GetStoreFor(ts model.Time) (client.ObjectClient, error) { return nil, fmt.Errorf("no store found for timestamp %s", ts) } +func (m *Multi) ObjectExistsWithSize(ctx context.Context, objectKey string) (bool, int64, error) { + s, err := m.GetStoreFor(model.Now()) + if err != nil { + return false, 0, err + } + return s.ObjectExistsWithSize(ctx, objectKey) +} + func (m *Multi) ObjectExists(ctx context.Context, objectKey string) (bool, error) { s, err := m.GetStoreFor(model.Now()) if err != nil { diff --git a/pkg/storage/chunk/client/alibaba/oss_object_client.go b/pkg/storage/chunk/client/alibaba/oss_object_client.go index 423a7348086e4..9d7f4cc48ce1c 100644 --- a/pkg/storage/chunk/client/alibaba/oss_object_client.go +++ b/pkg/storage/chunk/client/alibaba/oss_object_client.go @@ -73,16 +73,27 @@ func (s *OssObjectClient) Stop() { } func (s *OssObjectClient) ObjectExists(ctx context.Context, objectKey string) (bool, error) { + exists, _, err := s.ObjectExistsWithSize(ctx, objectKey) + return exists, err +} + +func (s *OssObjectClient) ObjectExistsWithSize(ctx context.Context, objectKey string) (bool, int64, error) { var options []oss.Option + var objectSize int64 err := instrument.CollectedRequest(ctx, "OSS.ObjectExists", ossRequestDuration, instrument.ErrorCode, func(_ context.Context) error { - _, requestErr := s.defaultBucket.GetObjectMeta(objectKey, options...) - return requestErr + headers, requestErr := s.defaultBucket.GetObjectMeta(objectKey, options...) + if requestErr != nil { + return requestErr + } + + objectSize, _ = strconv.ParseInt(headers.Get(oss.HTTPHeaderContentLength), 10, 64) + return nil }) if err != nil { - return false, err + return false, 0, err } - return true, nil + return true, objectSize, nil } // GetObject returns a reader and the size for the specified object key from the configured OSS bucket. diff --git a/pkg/storage/chunk/client/aws/s3_storage_client.go b/pkg/storage/chunk/client/aws/s3_storage_client.go index a7489bf847706..26c2807e120e7 100644 --- a/pkg/storage/chunk/client/aws/s3_storage_client.go +++ b/pkg/storage/chunk/client/aws/s3_storage_client.go @@ -310,37 +310,49 @@ func buckets(cfg S3Config) ([]string, error) { func (a *S3ObjectClient) Stop() {} func (a *S3ObjectClient) ObjectExists(ctx context.Context, objectKey string) (bool, error) { + exists, _, err := a.ObjectExistsWithSize(ctx, objectKey) + return exists, err +} + +func (a *S3ObjectClient) ObjectExistsWithSize(ctx context.Context, objectKey string) (bool, int64, error) { var lastErr error + var objectSize int64 retries := backoff.New(ctx, a.cfg.BackoffConfig) for retries.Ongoing() { if ctx.Err() != nil { - return false, errors.Wrap(ctx.Err(), "ctx related error during s3 objectExists") + return false, 0, errors.Wrap(ctx.Err(), "ctx related error during s3 objectExists") } lastErr = instrument.CollectedRequest(ctx, "S3.ObjectExists", s3RequestDuration, instrument.ErrorCode, func(_ context.Context) error { headObjectInput := &s3.HeadObjectInput{ Bucket: aws.String(a.bucketFromKey(objectKey)), Key: aws.String(objectKey), } - _, requestErr := a.S3.HeadObject(headObjectInput) - return requestErr + headOutput, requestErr := a.S3.HeadObject(headObjectInput) + if requestErr != nil { + return requestErr + } + if headOutput != nil && headOutput.ContentLength != nil { + objectSize = *headOutput.ContentLength + } + return nil }) if lastErr == nil { - return true, nil + return true, 0, nil } if a.IsObjectNotFoundErr(lastErr) { - return false, lastErr + return false, 0, lastErr } retries.Wait() } if lastErr != nil { - return false, lastErr + return false, 0, lastErr } - return true, nil + return true, objectSize, nil } // DeleteObject deletes the specified objectKey from the appropriate S3 bucket diff --git a/pkg/storage/chunk/client/aws/s3_storage_client_test.go b/pkg/storage/chunk/client/aws/s3_storage_client_test.go index ba2939ff46884..e18160a3fa003 100644 --- a/pkg/storage/chunk/client/aws/s3_storage_client_test.go +++ b/pkg/storage/chunk/client/aws/s3_storage_client_test.go @@ -334,6 +334,15 @@ func Test_RetryLogic(t *testing.T) { return err }, }, + { + "object exists with size with retries", + 3, + true, + func(c *S3ObjectClient) error { + _, _, err := c.ObjectExistsWithSize(context.Background(), "foo") + return err + }, + }, { "object doesn't exist with retries", 3, @@ -343,6 +352,15 @@ func Test_RetryLogic(t *testing.T) { return err }, }, + { + "object doesn't exist (with size) with retries", + 3, + false, + func(c *S3ObjectClient) error { + _, _, err := c.ObjectExistsWithSize(context.Background(), "foo") + return err + }, + }, } { t.Run(tc.name, func(t *testing.T) { callCount := atomic.NewInt32(0) diff --git a/pkg/storage/chunk/client/azure/blob_storage_client.go b/pkg/storage/chunk/client/azure/blob_storage_client.go index 0a9d6300b1634..2a7014c29898e 100644 --- a/pkg/storage/chunk/client/azure/blob_storage_client.go +++ b/pkg/storage/chunk/client/azure/blob_storage_client.go @@ -220,20 +220,35 @@ func NewBlobStorage(cfg *BlobStorageConfig, metrics BlobStorageMetrics, hedgingC func (b *BlobStorage) Stop() {} func (b *BlobStorage) ObjectExists(ctx context.Context, objectKey string) (bool, error) { + exists, _, err := b.ObjectExistsWithSize(ctx, objectKey) + return exists, err +} + +func (b *BlobStorage) ObjectExistsWithSize(ctx context.Context, objectKey string) (bool, int64, error) { + var objectSize int64 err := loki_instrument.TimeRequest(ctx, "azure.ObjectExists", instrument.NewHistogramCollector(b.metrics.requestDuration), instrument.ErrorCode, func(ctx context.Context) error { blockBlobURL, err := b.getBlobURL(objectKey, false) if err != nil { return err } - _, err = blockBlobURL.GetProperties(ctx, azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{}) - return err + response, err := blockBlobURL.GetProperties(ctx, azblob.BlobAccessConditions{}, azblob.ClientProvidedKeyOptions{}) + if err != nil { + return err + } + if response != nil { + rawResponse := response.Response() + if rawResponse != nil { + objectSize = rawResponse.ContentLength + } + } + return nil }) if err != nil { - return false, err + return false, 0, err } - return true, nil + return true, objectSize, nil } // GetObject returns a reader and the size for the specified object key. diff --git a/pkg/storage/chunk/client/baidubce/bos_storage_client.go b/pkg/storage/chunk/client/baidubce/bos_storage_client.go index b76db38e47c60..cc76b21624294 100644 --- a/pkg/storage/chunk/client/baidubce/bos_storage_client.go +++ b/pkg/storage/chunk/client/baidubce/bos_storage_client.go @@ -91,16 +91,27 @@ func (b *BOSObjectStorage) PutObject(ctx context.Context, objectKey string, obje } func (b *BOSObjectStorage) ObjectExists(ctx context.Context, objectKey string) (bool, error) { + exists, _, err := b.ObjectExistsWithSize(ctx, objectKey) + return exists, err +} + +func (b *BOSObjectStorage) ObjectExistsWithSize(ctx context.Context, objectKey string) (bool, int64, error) { + var objectSize int64 err := instrument.CollectedRequest(ctx, "BOS.ObjectExists", bosRequestDuration, instrument.ErrorCode, func(_ context.Context) error { - var requestErr error - _, requestErr = b.client.GetObjectMeta(b.cfg.BucketName, objectKey) - return requestErr + metaResult, requestErr := b.client.GetObjectMeta(b.cfg.BucketName, objectKey) + if requestErr != nil { + return requestErr + } + if metaResult != nil { + objectSize = metaResult.ContentLength + } + return nil }) if err != nil { - return false, err + return false, 0, err } - return true, nil + return true, objectSize, nil } func (b *BOSObjectStorage) GetObject(ctx context.Context, objectKey string) (io.ReadCloser, int64, error) { diff --git a/pkg/storage/chunk/client/congestion/controller.go b/pkg/storage/chunk/client/congestion/controller.go index 1e3e2ee0dcb3b..1c69ef16139f7 100644 --- a/pkg/storage/chunk/client/congestion/controller.go +++ b/pkg/storage/chunk/client/congestion/controller.go @@ -145,6 +145,10 @@ func (a *AIMDController) ObjectExists(ctx context.Context, objectKey string) (bo return a.inner.ObjectExists(ctx, objectKey) } +func (a *AIMDController) ObjectExistsWithSize(ctx context.Context, objectKey string) (bool, int64, error) { + return a.inner.ObjectExistsWithSize(ctx, objectKey) +} + func (a *AIMDController) DeleteObject(ctx context.Context, objectKey string) error { return a.inner.DeleteObject(ctx, objectKey) } @@ -212,6 +216,9 @@ func NewNoopController(Config) *NoopController { return &NoopController{} } +func (n *NoopController) ObjectExistsWithSize(context.Context, string) (bool, int64, error) { + return true, 0, nil +} func (n *NoopController) ObjectExists(context.Context, string) (bool, error) { return true, nil } func (n *NoopController) PutObject(context.Context, string, io.Reader) error { return nil } func (n *NoopController) GetObject(context.Context, string) (io.ReadCloser, int64, error) { diff --git a/pkg/storage/chunk/client/congestion/controller_test.go b/pkg/storage/chunk/client/congestion/controller_test.go index a46466ebfc546..6d17573248f50 100644 --- a/pkg/storage/chunk/client/congestion/controller_test.go +++ b/pkg/storage/chunk/client/congestion/controller_test.go @@ -267,6 +267,10 @@ func (m *mockObjectClient) ObjectExists(context.Context, string) (bool, error) { panic("not implemented") } +func (m *mockObjectClient) ObjectExistsWithSize(context.Context, string) (bool, int64, error) { + panic("not implemented") +} + func (m *mockObjectClient) List(context.Context, string, string) ([]client.StorageObject, []client.StorageCommonPrefix, error) { panic("not implemented") } diff --git a/pkg/storage/chunk/client/gcp/gcs_object_client.go b/pkg/storage/chunk/client/gcp/gcs_object_client.go index e8f5b4a64d850..c161705ecf7f5 100644 --- a/pkg/storage/chunk/client/gcp/gcs_object_client.go +++ b/pkg/storage/chunk/client/gcp/gcs_object_client.go @@ -127,12 +127,20 @@ func (s *GCSObjectClient) Stop() { } func (s *GCSObjectClient) ObjectExists(ctx context.Context, objectKey string) (bool, error) { - _, err := s.getsBuckets.Object(objectKey).Attrs(ctx) + exists, _, err := s.ObjectExistsWithSize(ctx, objectKey) + return exists, err +} + +func (s *GCSObjectClient) ObjectExistsWithSize(ctx context.Context, objectKey string) (bool, int64, error) { + attrs, err := s.getsBuckets.Object(objectKey).Attrs(ctx) if err != nil { - return false, err + return false, 0, err } - return true, nil + if attrs != nil { + return true, attrs.Size, nil + } + return true, 0, nil } // GetObject returns a reader and the size for the specified object key from the configured GCS bucket. diff --git a/pkg/storage/chunk/client/ibmcloud/cos_object_client.go b/pkg/storage/chunk/client/ibmcloud/cos_object_client.go index d432071293054..51a9c9fd93086 100644 --- a/pkg/storage/chunk/client/ibmcloud/cos_object_client.go +++ b/pkg/storage/chunk/client/ibmcloud/cos_object_client.go @@ -320,20 +320,31 @@ func (c *COSObjectClient) DeleteObject(ctx context.Context, objectKey string) er } func (c *COSObjectClient) ObjectExists(ctx context.Context, objectKey string) (bool, error) { + exists, _, err := c.ObjectExistsWithSize(ctx, objectKey) + return exists, err +} + +func (c *COSObjectClient) ObjectExistsWithSize(ctx context.Context, objectKey string) (bool, int64, error) { bucket := c.bucketFromKey(objectKey) + var objectSize int64 err := instrument.CollectedRequest(ctx, "COS.GetObject", cosRequestDuration, instrument.ErrorCode, func(_ context.Context) error { - var requestErr error - _, requestErr = c.hedgedCOS.HeadObject(&cos.HeadObjectInput{ + headOutput, requestErr := c.hedgedCOS.HeadObject(&cos.HeadObjectInput{ Bucket: ibm.String(bucket), Key: ibm.String(objectKey), }) - return requestErr + if requestErr != nil { + return requestErr + } + if headOutput != nil && headOutput.ContentLength != nil { + objectSize = *headOutput.ContentLength + } + return nil }) if err != nil { - return false, err + return false, 0, err } - return true, nil + return true, objectSize, nil } // GetObject returns a reader and the size for the specified object key from the configured S3 bucket. diff --git a/pkg/storage/chunk/client/local/fs_object_client.go b/pkg/storage/chunk/client/local/fs_object_client.go index 0eb027e9fd3cf..671b5df285870 100644 --- a/pkg/storage/chunk/client/local/fs_object_client.go +++ b/pkg/storage/chunk/client/local/fs_object_client.go @@ -67,14 +67,19 @@ func NewFSObjectClient(cfg FSConfig) (*FSObjectClient, error) { // Stop implements ObjectClient func (FSObjectClient) Stop() {} -func (f *FSObjectClient) ObjectExists(_ context.Context, objectKey string) (bool, error) { +func (f *FSObjectClient) ObjectExists(ctx context.Context, objectKey string) (bool, error) { + exists, _, err := f.ObjectExistsWithSize(ctx, objectKey) + return exists, err +} + +func (f *FSObjectClient) ObjectExistsWithSize(_ context.Context, objectKey string) (bool, int64, error) { fullPath := filepath.Join(f.cfg.Directory, filepath.FromSlash(objectKey)) - _, err := os.Lstat(fullPath) + fi, err := os.Lstat(fullPath) if err != nil { - return false, err + return false, 0, err } - return true, nil + return true, fi.Size(), nil } // GetObject from the store diff --git a/pkg/storage/chunk/client/local/fs_object_client_test.go b/pkg/storage/chunk/client/local/fs_object_client_test.go index 2dc059b3f5f1a..15ad96425dc9c 100644 --- a/pkg/storage/chunk/client/local/fs_object_client_test.go +++ b/pkg/storage/chunk/client/local/fs_object_client_test.go @@ -156,6 +156,11 @@ func TestFSObjectClient_List_and_ObjectExists(t *testing.T) { ok, err := bucketClient.ObjectExists(context.Background(), "outer-file2") require.NoError(t, err) require.True(t, ok) + + ok, objectSize, err := bucketClient.ObjectExistsWithSize(context.Background(), "outer-file2") + require.NoError(t, err) + require.True(t, ok) + require.EqualValues(t, len("outer-file2"), objectSize) } func TestFSObjectClient_DeleteObject(t *testing.T) { diff --git a/pkg/storage/chunk/client/object_client.go b/pkg/storage/chunk/client/object_client.go index 225f5025b1d51..95672c286ad09 100644 --- a/pkg/storage/chunk/client/object_client.go +++ b/pkg/storage/chunk/client/object_client.go @@ -19,6 +19,7 @@ import ( // ObjectClient is used to store arbitrary data in Object Store (S3/GCS/Azure/...) type ObjectClient interface { ObjectExists(ctx context.Context, objectKey string) (bool, error) + ObjectExistsWithSize(ctx context.Context, objectKey string) (bool, int64, error) PutObject(ctx context.Context, objectKey string, object io.Reader) error // NOTE: The consumer of GetObject should always call the Close method when it is done reading which otherwise could cause a resource leak. diff --git a/pkg/storage/chunk/client/openstack/swift_object_client.go b/pkg/storage/chunk/client/openstack/swift_object_client.go index 951a4d652a5a1..d3d978cd5ef72 100644 --- a/pkg/storage/chunk/client/openstack/swift_object_client.go +++ b/pkg/storage/chunk/client/openstack/swift_object_client.go @@ -125,12 +125,17 @@ func (s *SwiftObjectClient) Stop() { } func (s *SwiftObjectClient) ObjectExists(ctx context.Context, objectKey string) (bool, error) { - _, _, err := s.hedgingConn.Object(ctx, s.cfg.Config.ContainerName, objectKey) + exists, _, err := s.ObjectExistsWithSize(ctx, objectKey) + return exists, err +} + +func (s *SwiftObjectClient) ObjectExistsWithSize(ctx context.Context, objectKey string) (bool, int64, error) { + info, _, err := s.hedgingConn.Object(ctx, s.cfg.Config.ContainerName, objectKey) if err != nil { - return false, err + return false, 0, err } - return true, nil + return true, info.Bytes, nil } // GetObject returns a reader and the size for the specified object key from the configured swift container. diff --git a/pkg/storage/chunk/client/prefixed_object_client.go b/pkg/storage/chunk/client/prefixed_object_client.go index 899dcd2b21123..5a5bda7627708 100644 --- a/pkg/storage/chunk/client/prefixed_object_client.go +++ b/pkg/storage/chunk/client/prefixed_object_client.go @@ -23,6 +23,10 @@ func (p PrefixedObjectClient) ObjectExists(ctx context.Context, objectKey string return p.downstreamClient.ObjectExists(ctx, p.prefix+objectKey) } +func (p PrefixedObjectClient) ObjectExistsWithSize(ctx context.Context, objectKey string) (bool, int64, error) { + return p.downstreamClient.ObjectExistsWithSize(ctx, p.prefix+objectKey) +} + func (p PrefixedObjectClient) GetObject(ctx context.Context, objectKey string) (io.ReadCloser, int64, error) { return p.downstreamClient.GetObject(ctx, p.prefix+objectKey) } diff --git a/pkg/storage/chunk/client/testutils/inmemory_storage_client.go b/pkg/storage/chunk/client/testutils/inmemory_storage_client.go index da65a35d53173..d937cd5fdfd4d 100644 --- a/pkg/storage/chunk/client/testutils/inmemory_storage_client.go +++ b/pkg/storage/chunk/client/testutils/inmemory_storage_client.go @@ -392,20 +392,25 @@ func (m *MockStorage) query(ctx context.Context, query index.Query, callback fun } // ObjectExists implments client.ObjectClient -func (m *InMemoryObjectClient) ObjectExists(_ context.Context, objectKey string) (bool, error) { +func (m *InMemoryObjectClient) ObjectExists(ctx context.Context, objectKey string) (bool, error) { + exists, _, err := m.ObjectExistsWithSize(ctx, objectKey) + return exists, err +} + +func (m *InMemoryObjectClient) ObjectExistsWithSize(_ context.Context, objectKey string) (bool, int64, error) { m.mtx.RLock() defer m.mtx.RUnlock() if m.mode == MockStorageModeWriteOnly { - return false, errPermissionDenied + return false, 0, errPermissionDenied } _, ok := m.objects[objectKey] if !ok { - return false, nil + return false, 0, nil } - - return true, nil + objectSize := len(m.objects[objectKey]) + return true, int64(objectSize), nil } // GetObject implements client.ObjectClient. diff --git a/pkg/tool/audit/audit_test.go b/pkg/tool/audit/audit_test.go index c9afa7b34f357..d591fca7e1ddb 100644 --- a/pkg/tool/audit/audit_test.go +++ b/pkg/tool/audit/audit_test.go @@ -17,11 +17,16 @@ type testObjClient struct { client.ObjectClient } -func (t testObjClient) ObjectExists(_ context.Context, object string) (bool, error) { +func (t testObjClient) ObjectExistsWithSize(_ context.Context, object string) (bool, int64, error) { if strings.Contains(object, "missing") { - return false, nil + return false, 0, nil } - return true, nil + return true, 0, nil +} + +func (t testObjClient) ObjectExists(ctx context.Context, object string) (bool, error) { + exists, _, err := t.ObjectExistsWithSize(ctx, object) + return exists, err } type testCompactedIdx struct { From 684baf7dbacef4b85a08db8de9934458745124d8 Mon Sep 17 00:00:00 2001 From: "Oleksandr K." Date: Wed, 25 Sep 2024 19:51:03 +0200 Subject: [PATCH 06/11] feat: support ruler sidecar in singleBinary mode (#13572) Co-authored-by: Trevor Whitney --- .../loki/templates/backend/clusterrole.yaml | 3 +- .../templates/backend/clusterrolebinding.yaml | 2 +- .../templates/single-binary/statefulset.yaml | 82 +++++++++++++++++++ 3 files changed, 84 insertions(+), 3 deletions(-) diff --git a/production/helm/loki/templates/backend/clusterrole.yaml b/production/helm/loki/templates/backend/clusterrole.yaml index e8631c35a501b..36c8a0fe0e805 100644 --- a/production/helm/loki/templates/backend/clusterrole.yaml +++ b/production/helm/loki/templates/backend/clusterrole.yaml @@ -1,5 +1,4 @@ -{{- $isSimpleScalable := eq (include "loki.deployment.isScalable" .) "true" -}} -{{- if and $isSimpleScalable (not .Values.rbac.namespaced) (not .Values.rbac.useExistingRole) }} +{{- if and (not .Values.rbac.namespaced) (not .Values.rbac.useExistingRole) }} kind: ClusterRole apiVersion: rbac.authorization.k8s.io/v1 metadata: diff --git a/production/helm/loki/templates/backend/clusterrolebinding.yaml b/production/helm/loki/templates/backend/clusterrolebinding.yaml index 619b70260cd4f..92f86a47d4f13 100644 --- a/production/helm/loki/templates/backend/clusterrolebinding.yaml +++ b/production/helm/loki/templates/backend/clusterrolebinding.yaml @@ -1,5 +1,5 @@ {{- $isSimpleScalable := eq (include "loki.deployment.isScalable" .) "true" -}} -{{- if and $isSimpleScalable (not .Values.rbac.namespaced) }} +{{- if (not .Values.rbac.namespaced) }} kind: ClusterRoleBinding apiVersion: rbac.authorization.k8s.io/v1 metadata: diff --git a/production/helm/loki/templates/single-binary/statefulset.yaml b/production/helm/loki/templates/single-binary/statefulset.yaml index 7bd2b9813f609..5e28902e5677f 100644 --- a/production/helm/loki/templates/single-binary/statefulset.yaml +++ b/production/helm/loki/templates/single-binary/statefulset.yaml @@ -79,6 +79,75 @@ spec: {{- end }} {{- end }} containers: + {{- if .Values.sidecar.rules.enabled }} + - name: loki-sc-rules + {{- if .Values.sidecar.image.sha }} + image: "{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}@sha256:{{ .Values.sidecar.image.sha }}" + {{- else }} + image: "{{ .Values.sidecar.image.repository }}:{{ .Values.sidecar.image.tag }}" + {{- end }} + imagePullPolicy: {{ .Values.sidecar.image.pullPolicy }} + env: + - name: METHOD + value: {{ .Values.sidecar.rules.watchMethod }} + - name: LABEL + value: "{{ .Values.sidecar.rules.label }}" + {{- if .Values.sidecar.rules.labelValue }} + - name: LABEL_VALUE + value: {{ quote .Values.sidecar.rules.labelValue }} + {{- end }} + - name: FOLDER + value: "{{ .Values.sidecar.rules.folder }}" + - name: RESOURCE + value: {{ quote .Values.sidecar.rules.resource }} + {{- if .Values.sidecar.enableUniqueFilenames }} + - name: UNIQUE_FILENAMES + value: "{{ .Values.sidecar.enableUniqueFilenames }}" + {{- end }} + {{- if .Values.sidecar.rules.searchNamespace }} + - name: NAMESPACE + value: "{{ .Values.sidecar.rules.searchNamespace | join "," }}" + {{- end }} + {{- if .Values.sidecar.skipTlsVerify }} + - name: SKIP_TLS_VERIFY + value: "{{ .Values.sidecar.skipTlsVerify }}" + {{- end }} + {{- if .Values.sidecar.rules.script }} + - name: SCRIPT + value: "{{ .Values.sidecar.rules.script }}" + {{- end }} + {{- if .Values.sidecar.rules.watchServerTimeout }} + - name: WATCH_SERVER_TIMEOUT + value: "{{ .Values.sidecar.rules.watchServerTimeout }}" + {{- end }} + {{- if .Values.sidecar.rules.watchClientTimeout }} + - name: WATCH_CLIENT_TIMEOUT + value: "{{ .Values.sidecar.rules.watchClientTimeout }}" + {{- end }} + {{- if .Values.sidecar.rules.logLevel }} + - name: LOG_LEVEL + value: "{{ .Values.sidecar.rules.logLevel }}" + {{- end }} + {{- if .Values.sidecar.livenessProbe }} + livenessProbe: + {{- toYaml .Values.sidecar.livenessProbe | nindent 12 }} + {{- end }} + {{- if .Values.sidecar.readinessProbe }} + readinessProbe: + {{- toYaml .Values.sidecar.readinessProbe | nindent 12 }} + {{- end }} + {{- if .Values.sidecar.resources }} + resources: + {{- toYaml .Values.sidecar.resources | nindent 12 }} + {{- end }} + {{- if .Values.sidecar.securityContext }} + securityContext: + {{- toYaml .Values.sidecar.securityContext | nindent 12 }} + {{- end }} + volumeMounts: + - name: sc-rules-volume + mountPath: {{ .Values.sidecar.rules.folder | quote }} + {{- end}} - name: loki image: {{ include "loki.image" . }} imagePullPolicy: {{ .Values.loki.image.pullPolicy }} @@ -125,6 +194,10 @@ spec: - name: license mountPath: /etc/loki/license {{- end }} + {{- if .Values.sidecar.rules.enabled }} + - name: sc-rules-volume + mountPath: {{ .Values.sidecar.rules.folder | quote }} + {{- end}} {{- with .Values.singleBinary.extraVolumeMounts }} {{- toYaml . | nindent 12 }} {{- end }} @@ -166,6 +239,15 @@ spec: secretName: enterprise-logs-license {{- end }} {{- end }} + {{- if .Values.sidecar.rules.enabled }} + - name: sc-rules-volume + {{- if .Values.sidecar.rules.sizeLimit }} + emptyDir: + sizeLimit: {{ .Values.sidecar.rules.sizeLimit }} + {{- else }} + emptyDir: {} + {{- end -}} + {{- end -}} {{- with .Values.singleBinary.extraVolumes }} {{- toYaml . | nindent 8 }} {{- end }} From cf1d4a31af5c376e82756eaaab267369f862265d Mon Sep 17 00:00:00 2001 From: Jonathan Date: Wed, 25 Sep 2024 10:55:02 -0700 Subject: [PATCH 07/11] fix: Rename mispelled filename (#14237) Co-authored-by: Trevor Whitney --- production/helm/loki/CHANGELOG.md | 1 + ...ery-frontent.yaml => poddisruptionbudget-query-frontend.yaml} | 0 2 files changed, 1 insertion(+) rename production/helm/loki/templates/query-frontend/{poddisruptionbudget-query-frontent.yaml => poddisruptionbudget-query-frontend.yaml} (100%) diff --git a/production/helm/loki/CHANGELOG.md b/production/helm/loki/CHANGELOG.md index f41649010447e..b39023e85effc 100644 --- a/production/helm/loki/CHANGELOG.md +++ b/production/helm/loki/CHANGELOG.md @@ -20,6 +20,7 @@ Entries should include a reference to the pull request that introduced the chang ## 6.14.0 - [FEATURE] Add additional service annotations for components in distributed mode +- [FIX] Rename loki/templates/query-frontend/poddisruptionbudget-query-frontend.yaml to fix spelling mistake. ## 6.13.0 diff --git a/production/helm/loki/templates/query-frontend/poddisruptionbudget-query-frontent.yaml b/production/helm/loki/templates/query-frontend/poddisruptionbudget-query-frontend.yaml similarity index 100% rename from production/helm/loki/templates/query-frontend/poddisruptionbudget-query-frontent.yaml rename to production/helm/loki/templates/query-frontend/poddisruptionbudget-query-frontend.yaml From f74b44c0e3e8cc9e09dcad96e8cc9ab3bf019b46 Mon Sep 17 00:00:00 2001 From: Trevor Whitney Date: Wed, 25 Sep 2024 17:01:37 -0600 Subject: [PATCH 08/11] ci: remove drone, add GH action to build images (#14273) --- .drone/docker-manifest-build-image.tmpl | 17 - .drone/docker-manifest-ecr.tmpl | 21 - .drone/docker-manifest-operator.tmpl | 26 - .drone/docker-manifest.tmpl | 26 - .drone/drone.jsonnet | 669 --------- .drone/drone.yml | 1313 ----------------- .github/jsonnetfile.json | 2 +- .github/jsonnetfile.lock.json | 4 +- .github/release-workflows.jsonnet | 52 +- .../loki-release/workflows/build.libsonnet | 4 +- .../loki-release/workflows/validate.libsonnet | 1 - .github/workflows/images.yml | 433 ++++++ .github/workflows/minor-release-pr.yml | 20 +- .github/workflows/patch-release-pr.yml | 20 +- Makefile | 25 +- 15 files changed, 511 insertions(+), 2122 deletions(-) delete mode 100644 .drone/docker-manifest-build-image.tmpl delete mode 100644 .drone/docker-manifest-ecr.tmpl delete mode 100644 .drone/docker-manifest-operator.tmpl delete mode 100644 .drone/docker-manifest.tmpl delete mode 100644 .drone/drone.jsonnet delete mode 100644 .drone/drone.yml create mode 100644 .github/workflows/images.yml diff --git a/.drone/docker-manifest-build-image.tmpl b/.drone/docker-manifest-build-image.tmpl deleted file mode 100644 index b870dd5c6165f..0000000000000 --- a/.drone/docker-manifest-build-image.tmpl +++ /dev/null @@ -1,17 +0,0 @@ -image: grafana/{{config.target}} -tags: -{{#if build.tags}} -{{#each build.tags}} - - {{this}} -{{/each}} -{{/if}} -manifests: - - image: grafana/{{config.target}}-amd64 - platform: - architecture: amd64 - os: linux - - image: grafana/{{config.target}}-arm64 - platform: - architecture: arm64 - os: linux - variant: v8 diff --git a/.drone/docker-manifest-ecr.tmpl b/.drone/docker-manifest-ecr.tmpl deleted file mode 100644 index 1c8c98417ac05..0000000000000 --- a/.drone/docker-manifest-ecr.tmpl +++ /dev/null @@ -1,21 +0,0 @@ -image: public.ecr.aws/grafana/{{config.target}}:{{#if build.tag}}{{trimPrefix "v" build.tag}}{{else}}{{build.branch}}-{{substr 0 7 build.commit}}{{/if}} -tags: - - main -{{#if build.tag}} - - latest -{{/if}} -{{#if build.tags}} -{{#each build.tags}} - - {{this}} -{{/each}} -{{/if}} -manifests: - - image: public.ecr.aws/grafana/{{config.target}}:{{#if build.tag}}{{trimPrefix "v" build.tag}}{{else}}{{build.branch}}-{{substr 0 7 build.commit}}{{/if}}-amd64 - platform: - architecture: amd64 - os: linux - - image: public.ecr.aws/grafana/{{config.target}}:{{#if build.tag}}{{trimPrefix "v" build.tag}}{{else}}{{build.branch}}-{{substr 0 7 build.commit}}{{/if}}-arm64 - platform: - architecture: arm64 - os: linux - variant: v8 diff --git a/.drone/docker-manifest-operator.tmpl b/.drone/docker-manifest-operator.tmpl deleted file mode 100644 index f5aaa400867df..0000000000000 --- a/.drone/docker-manifest-operator.tmpl +++ /dev/null @@ -1,26 +0,0 @@ -image: grafana/{{config.target}}:{{#if build.tag}}{{trimPrefix "operator/v" build.tag}}{{else}}{{build.branch}}-{{substr 0 7 build.commit}}{{/if}} -tags: - - main -{{#if build.tag}} - - latest -{{/if}} -{{#if build.tags}} -{{#each build.tags}} - - {{this}} -{{/each}} -{{/if}} -manifests: - - image: grafana/{{config.target}}:{{#if build.tag}}{{trimPrefix "operator/v" build.tag}}{{else}}{{build.branch}}-{{substr 0 7 build.commit}}{{/if}}-amd64 - platform: - architecture: amd64 - os: linux - - image: grafana/{{config.target}}:{{#if build.tag}}{{trimPrefix "operator/v" build.tag}}{{else}}{{build.branch}}-{{substr 0 7 build.commit}}{{/if}}-arm64 - platform: - architecture: arm64 - os: linux - variant: v8 - - image: grafana/{{config.target}}:{{#if build.tag}}{{trimPrefix "operator/v" build.tag}}{{else}}{{build.branch}}-{{substr 0 7 build.commit}}{{/if}}-arm - platform: - architecture: arm - os: linux - variant: v7 diff --git a/.drone/docker-manifest.tmpl b/.drone/docker-manifest.tmpl deleted file mode 100644 index 7191e602b4ac0..0000000000000 --- a/.drone/docker-manifest.tmpl +++ /dev/null @@ -1,26 +0,0 @@ -image: grafana/{{config.target}}:{{#if build.tag}}{{trimPrefix "v" build.tag}}{{else}}{{build.branch}}-{{substr 0 7 build.commit}}{{/if}} -tags: - - main -{{#if build.tag}} - - latest -{{/if}} -{{#if build.tags}} -{{#each build.tags}} - - {{this}} -{{/each}} -{{/if}} -manifests: - - image: grafana/{{config.target}}:{{#if build.tag}}{{trimPrefix "v" build.tag}}{{else}}{{build.branch}}-{{substr 0 7 build.commit}}{{/if}}-amd64 - platform: - architecture: amd64 - os: linux - - image: grafana/{{config.target}}:{{#if build.tag}}{{trimPrefix "v" build.tag}}{{else}}{{build.branch}}-{{substr 0 7 build.commit}}{{/if}}-arm64 - platform: - architecture: arm64 - os: linux - variant: v8 - - image: grafana/{{config.target}}:{{#if build.tag}}{{trimPrefix "v" build.tag}}{{else}}{{build.branch}}-{{substr 0 7 build.commit}}{{/if}}-arm - platform: - architecture: arm - os: linux - variant: v7 diff --git a/.drone/drone.jsonnet b/.drone/drone.jsonnet deleted file mode 100644 index 718f0a84aa886..0000000000000 --- a/.drone/drone.jsonnet +++ /dev/null @@ -1,669 +0,0 @@ -local apps = ['loki', 'loki-canary', 'loki-canary-boringcrypto', 'logcli']; -local archs = ['amd64', 'arm64', 'arm']; - -local build_image_version = std.extVar('__build-image-version'); - -local drone_updater_plugin_image = 'us.gcr.io/kubernetes-dev/drone/plugins/updater@sha256:cbcb09c74f96a34c528f52bf9b4815a036b11fed65f685be216e0c8b8e84285b'; - -local onPRs = { - event: ['pull_request'], -}; - -local onTagOrMain = { - event: ['push', 'tag'], -}; - -local onTag = { - event: ['tag'], -}; - -local onPath(path) = { - paths+: [path], -}; - -local pipeline(name) = { - kind: 'pipeline', - name: name, - steps: [], - trigger: { - // Only trigger pipelines for PRs, tags (v*), or pushes to "main". Excluding runs on grafana/loki (non fork) branches - ref: ['refs/heads/main', 'refs/heads/k???', 'refs/tags/v*', 'refs/pull/*/head'], - }, -}; - -local secret(name, vault_path, vault_key) = { - kind: 'secret', - name: name, - get: { - path: vault_path, - name: vault_key, - }, -}; -local docker_username_secret = secret('docker_username', 'infra/data/ci/docker_hub', 'username'); -local docker_password_secret = secret('docker_password', 'infra/data/ci/docker_hub', 'password'); -local ecr_key = secret('ecr_key', 'infra/data/ci/loki/aws-credentials', 'access_key_id'); -local ecr_secret_key = secret('ecr_secret_key', 'infra/data/ci/loki/aws-credentials', 'secret_access_key'); -local pull_secret = secret('dockerconfigjson', 'secret/data/common/gcr', '.dockerconfigjson'); -local github_secret = secret('github_token', 'infra/data/ci/github/grafanabot', 'pat'); -local gpg_passphrase = secret('gpg_passphrase', 'infra/data/ci/packages-publish/gpg', 'passphrase'); -local gpg_private_key = secret('gpg_private_key', 'infra/data/ci/packages-publish/gpg', 'private-key'); - -// Injected in a secret because this is a public repository and having the config here would leak our environment names -local updater_config_template = secret('updater_config_template', 'secret/data/common/loki_ci_autodeploy', 'updater-config-template.json'); -local helm_chart_auto_update_config_template = secret('helm-chart-update-config-template', 'secret/data/common/loki-helm-chart-auto-update', 'on-loki-release-config.json'); - - -local run(name, commands, env={}, image='grafana/loki-build-image:%s' % build_image_version) = { - name: name, - image: image, - commands: commands, - environment: env, -}; - -local make(target, container=true, args=[]) = run(target, [ - std.join(' ', [ - 'make', - 'BUILD_IN_CONTAINER=' + container, - target, - ] + args), -]); - -// The only indication we have that we're running in a fork is the presence of a secret. -// If a secret is blank, it means we're running in a fork. -local skipMissingSecretPipelineStep(secretName) = run( - 'skip pipeline if missing secret', - [ - 'if [ "$${#TEST_SECRET}" -eq 0 ]; then', - ' echo "Missing a secret to run this pipeline. This branch needs to be re-pushed as a branch in main grafana/loki repository in order to run." && exit 78', - 'fi', - ], - image='alpine', - env={ - TEST_SECRET: { from_secret: secretName }, - }, -); - -local docker(arch, app) = { - name: '%s-image' % if $.settings.dry_run then 'build-' + app else 'publish-' + app, - image: if arch == 'arm' then 'plugins/docker:linux-arm' else 'plugins/docker', - settings: { - repo: 'grafana/%s' % app, - dockerfile: 'cmd/%s/Dockerfile' % app, - username: { from_secret: docker_username_secret.name }, - password: { from_secret: docker_password_secret.name }, - dry_run: false, - }, -}; - -local clients_docker(arch, app) = { - name: '%s-image' % if $.settings.dry_run then 'build-' + app else 'publish-' + app, - image: if arch == 'arm' then 'plugins/docker:linux-arm' else 'plugins/docker', - settings: { - repo: 'grafana/%s' % app, - dockerfile: 'clients/cmd/%s/Dockerfile' % app, - username: { from_secret: docker_username_secret.name }, - password: { from_secret: docker_password_secret.name }, - dry_run: false, - }, -}; - -local docker_operator(arch, operator) = { - name: '%s-image' % if $.settings.dry_run then 'build-' + operator else 'publish-' + operator, - image: if arch == 'arm' then 'plugins/docker:linux-arm' else 'plugins/docker', - settings: { - repo: 'grafana/%s' % operator, - context: 'operator', - dockerfile: 'operator/Dockerfile', - username: { from_secret: docker_username_secret.name }, - password: { from_secret: docker_password_secret.name }, - dry_run: false, - }, -}; - -local lambda_promtail_ecr(app) = { - name: '%s-image' % if $.settings.dry_run then 'build-' + app else 'publish-' + app, - image: 'cstyan/ecr', - privileged: true, - settings: { - repo: 'public.ecr.aws/grafana/lambda-promtail', - registry: 'public.ecr.aws/grafana', - dockerfile: 'tools/%s/Dockerfile' % app, - access_key: { from_secret: ecr_key.name }, - secret_key: { from_secret: ecr_secret_key.name }, - dry_run: false, - region: 'us-east-1', - }, -}; - -local arch_image(arch, tags='') = { - platform: { - os: 'linux', - arch: arch, - }, - steps: [{ - name: 'image-tag', - image: 'alpine', - commands: [ - 'apk add --no-cache bash git', - 'git fetch origin --tags', - 'echo $(./tools/image-tag)-%s > .tags' % arch, - ] + if tags != '' then ['echo ",%s" >> .tags' % tags] else [], - }], -}; - -local querytee() = pipeline('querytee-amd64') + arch_image('amd64', 'main') { - steps+: [ - // publish for tag or main - docker('amd64', 'querytee') { - depends_on: ['image-tag'], - when: onTagOrMain, - settings+: { - repo: 'grafana/loki-query-tee', - }, - }, - ], -}; - -local fluentbit(arch) = pipeline('fluent-bit-' + arch) + arch_image(arch) { - steps+: [ - // publish for tag or main - clients_docker(arch, 'fluent-bit') { - depends_on: ['image-tag'], - when: onTagOrMain, - settings+: { - repo: 'grafana/fluent-bit-plugin-loki', - }, - }, - ], -}; - -local fluentd() = pipeline('fluentd-amd64') + arch_image('amd64', 'main') { - steps+: [ - // publish for tag or main - clients_docker('amd64', 'fluentd') { - depends_on: ['image-tag'], - when: onTagOrMain, - settings+: { - repo: 'grafana/fluent-plugin-loki', - }, - }, - ], -}; - -local logstash() = pipeline('logstash-amd64') + arch_image('amd64', 'main') { - steps+: [ - // publish for tag or main - clients_docker('amd64', 'logstash') { - depends_on: ['image-tag'], - when: onTagOrMain, - settings+: { - repo: 'grafana/logstash-output-loki', - }, - }, - ], -}; - -local promtail(arch) = pipeline('promtail-' + arch) + arch_image(arch) { - steps+: [ - // publish for tag or main - clients_docker(arch, 'promtail') { - depends_on: ['image-tag'], - when: onTagOrMain, - settings+: {}, - }, - ], -}; - -local lambda_promtail(arch) = pipeline('lambda-promtail-' + arch) + arch_image(arch) { - local skipStep = skipMissingSecretPipelineStep(ecr_key.name), // Needs ECR secrets to run - - steps+: [ - skipStep, - // publish for tag or main - lambda_promtail_ecr('lambda-promtail') { - depends_on: ['image-tag'], - when: onTagOrMain, - settings+: {}, - }, - ], -}; - -local lokioperator(arch) = pipeline('lokioperator-' + arch) + arch_image(arch) { - steps+: [ - // publish for tag or main - docker_operator(arch, 'loki-operator') { - depends_on: ['image-tag'], - when: onTagOrMain { - ref: ['refs/heads/main', 'refs/tags/operator/v*'], - }, - settings+: {}, - }, - ], -}; - -local logql_analyzer() = pipeline('logql-analyzer') + arch_image('amd64') { - steps+: [ - // publish for tag or main - docker('amd64', 'logql-analyzer') { - depends_on: ['image-tag'], - when: onTagOrMain, - settings+: { - repo: 'grafana/logql-analyzer', - }, - }, - ], -}; - -local multiarch_image(arch) = pipeline('docker-' + arch) + arch_image(arch) { - steps+: [ - // publish for tag or main - docker(arch, app) { - depends_on: ['image-tag'], - when: onTagOrMain, - settings+: {}, - } - for app in apps - ], -}; - -local manifest(apps) = pipeline('manifest') { - steps: std.foldl( - function(acc, app) acc + [{ - name: 'manifest-' + app, - image: 'plugins/manifest:1.4.0', - settings: { - // the target parameter is abused for the app's name, - // as it is unused in spec mode. See docker-manifest.tmpl - target: app, - spec: '.drone/docker-manifest.tmpl', - ignore_missing: false, - username: { from_secret: docker_username_secret.name }, - password: { from_secret: docker_password_secret.name }, - }, - depends_on: ['clone'] + ( - // Depend on the previous app, if any. - if std.length(acc) > 0 - then [acc[std.length(acc) - 1].name] - else [] - ), - }], - apps, - [], - ), - depends_on: [ - 'docker-%s' % arch - for arch in archs - ] + [ - 'promtail-%s' % arch - for arch in archs - ] + [ - 'fluent-bit-%s' % arch - for arch in archs - ], -}; - -local manifest_operator(app) = pipeline('manifest-operator') { - steps: [{ - name: 'manifest-' + app, - image: 'plugins/manifest:1.4.0', - settings: { - // the target parameter is abused for the app's name, - // as it is unused in spec mode. See docker-manifest-operator.tmpl - target: app, - spec: '.drone/docker-manifest-operator.tmpl', - ignore_missing: false, - username: { from_secret: docker_username_secret.name }, - password: { from_secret: docker_password_secret.name }, - }, - depends_on: ['clone'], - }], - depends_on: [ - 'lokioperator-%s' % arch - for arch in archs - ], -}; - - -local manifest_ecr(apps, archs) = pipeline('manifest-ecr') { - steps: std.foldl( - function(acc, app) acc + [{ - name: 'manifest-' + app, - image: 'plugins/manifest:1.4.0', - volumes: [{ - name: 'dockerconf', - path: '/.docker', - }], - settings: { - // the target parameter is abused for the app's name, - // as it is unused in spec mode. See docker-manifest-ecr.tmpl - target: app, - spec: '.drone/docker-manifest-ecr.tmpl', - ignore_missing: true, - }, - depends_on: ['clone'] + ( - // Depend on the previous app, if any. - if std.length(acc) > 0 - then [acc[std.length(acc) - 1].name] - else [] - ), - }], - apps, - [{ - name: 'ecr-login', - image: 'docker:dind', - volumes: [{ - name: 'dockerconf', - path: '/root/.docker', - }], - environment: { - AWS_ACCESS_KEY_ID: { from_secret: ecr_key.name }, - AWS_SECRET_ACCESS_KEY: { from_secret: ecr_secret_key.name }, - }, - commands: [ - 'apk add --no-cache aws-cli', - 'docker login --username AWS --password $(aws ecr-public get-login-password --region us-east-1) public.ecr.aws', - ], - depends_on: ['clone'], - }], - ), - volumes: [{ - name: 'dockerconf', - temp: {}, - }], - depends_on: [ - 'lambda-promtail-%s' % arch - for arch in archs - ], -}; - -local build_image_tag = '0.33.2'; -[ - pipeline('loki-build-image-' + arch) { - workspace: { - base: '/src', - path: 'loki', - }, - platform: { - os: 'linux', - arch: arch, - }, - steps: [ - { - name: 'push', - image: 'plugins/docker', - when: onTagOrMain + onPath('loki-build-image/**'), - environment: { - DOCKER_BUILDKIT: 1, - }, - settings: { - repo: 'grafana/loki-build-image', - context: 'loki-build-image', - dockerfile: 'loki-build-image/Dockerfile', - username: { from_secret: docker_username_secret.name }, - password: { from_secret: docker_password_secret.name }, - tags: [build_image_tag + '-' + arch], - dry_run: false, - }, - }, - ], - } - for arch in ['amd64', 'arm64'] -] + [ - pipeline('loki-build-image-publish') { - steps: [ - { - name: 'manifest', - image: 'plugins/manifest:1.4.0', - when: onTagOrMain + onPath('loki-build-image/**'), - settings: { - // the target parameter is abused for the app's name, as it is unused in spec mode. - target: 'loki-build-image:' + build_image_tag, - spec: '.drone/docker-manifest-build-image.tmpl', - ignore_missing: false, - username: { from_secret: docker_username_secret.name }, - password: { from_secret: docker_password_secret.name }, - }, - }, - ], - depends_on: [ - 'loki-build-image-%s' % arch - for arch in ['amd64', 'arm64'] - ], - }, - pipeline('helm-test-image') { - workspace: { - base: '/src', - path: 'loki', - }, - steps: [ - { - name: 'push-image', - image: 'plugins/docker', - when: onTagOrMain + onPath('production/helm/loki/src/helm-test/**'), - settings: { - repo: 'grafana/loki-helm-test', - dockerfile: 'production/helm/loki/src/helm-test/Dockerfile', - username: { from_secret: docker_username_secret.name }, - password: { from_secret: docker_password_secret.name }, - dry_run: false, - }, - }, - ], - }, - pipeline('documentation-checks') { - workspace: { - base: '/src', - path: 'loki', - }, - steps: [ - make('documentation-helm-reference-check', container=false) { - depends_on: ['clone'], - }, - ], - }, -] + [ - multiarch_image(arch) - for arch in archs -] + [ - promtail(arch) + ( - // When we're building Promtail for ARM, we want to use Dockerfile.arm32 to fix - // a problem with the published Drone image. See Dockerfile.arm32 for more - // information. - // - // This is really really hacky and a better more permanent solution will be to use - // buildkit. - if arch == 'arm' - then { - steps: [ - step + ( - if std.objectHas(step, 'settings') && step.settings.dockerfile == 'clients/cmd/promtail/Dockerfile' - then { - settings+: { - dockerfile: 'clients/cmd/promtail/Dockerfile.arm32', - }, - } - else {} - ) - for step in super.steps - ], - } - else {} - ) - for arch in archs -] + [ - lokioperator(arch) { - trigger+: { - ref: [ - 'refs/heads/main', - 'refs/tags/operator/v*', - 'refs/pull/*/head', - ], - }, - } - for arch in archs -] + [ - fluentbit(arch) - for arch in archs -] + [ - fluentd(), - logstash(), - querytee(), - manifest(['promtail', 'loki', 'loki-canary', 'loki-canary-boringcrypto', 'fluent-bit-plugin-loki']) { - trigger+: onTagOrMain, - }, - manifest_operator('loki-operator') { - trigger+: onTagOrMain { - ref: [ - 'refs/heads/main', - 'refs/tags/operator/v*', - ], - }, - }, - pipeline('deploy') { - local configFileName = 'updater-config.json', - trigger: onTagOrMain { - ref: ['refs/heads/main', 'refs/tags/v*'], - }, - depends_on: ['manifest'], - image_pull_secrets: [pull_secret.name], - steps: [ - { - name: 'prepare-updater-config', - image: 'alpine', - environment: { - MAJOR_MINOR_VERSION_REGEXP: '([0-9]+\\.[0-9]+)', - RELEASE_TAG_REGEXP: '^([0-9]+\\.[0-9]+\\.[0-9]+)$', - }, - commands: [ - 'apk add --no-cache bash git', - 'git fetch origin --tags', - 'echo $(./tools/image-tag) > .tag', - 'export RELEASE_TAG=$(cat .tag)', - // if the tag matches the pattern `D.D.D` then RELEASE_NAME="D-D-x", otherwise RELEASE_NAME="next" - 'export RELEASE_NAME=$([[ $RELEASE_TAG =~ $RELEASE_TAG_REGEXP ]] && echo $RELEASE_TAG | grep -oE $MAJOR_MINOR_VERSION_REGEXP | sed "s/\\./-/g" | sed "s/$/-x/" || echo "next")', - 'echo $RELEASE_NAME', - 'echo $PLUGIN_CONFIG_TEMPLATE > %s' % configFileName, - // replace placeholders with RELEASE_NAME and RELEASE TAG - 'sed -i "s/\\"{{release}}\\"/\\"$RELEASE_NAME\\"/g" %s' % configFileName, - 'sed -i "s/{{version}}/$RELEASE_TAG/g" %s' % configFileName, - ], - settings: { - config_template: { from_secret: updater_config_template.name }, - }, - depends_on: ['clone'], - }, - { - name: 'trigger', - image: drone_updater_plugin_image, - settings: { - github_token: { from_secret: github_secret.name }, - config_file: configFileName, - }, - depends_on: ['prepare-updater-config'], - }, - ], - }, - pipeline('update-loki-helm-chart-on-loki-release') { - local configFileName = 'updater-config.json', - depends_on: ['manifest'], - image_pull_secrets: [pull_secret.name], - trigger: { - // we need to run it only on Loki tags that starts with `v`. - ref: ['refs/tags/v*'], - }, - steps: [ - { - name: 'check-version-is-latest', - image: 'alpine', - when: onTag, - commands: [ - 'apk add --no-cache bash git', - 'git fetch --tags', - "latest_version=$(git tag -l 'v[0-9]*.[0-9]*.[0-9]*' | sort -V | tail -n 1 | sed 's/v//g')", - 'RELEASE_TAG=$(./tools/image-tag)', - 'if [ "$RELEASE_TAG" != "$latest_version" ]; then echo "Current version $RELEASE_TAG is not the latest version of Loki. The latest version is $latest_version" && exit 78; fi', - ], - }, - { - name: 'prepare-helm-chart-update-config', - image: 'alpine', - depends_on: ['check-version-is-latest'], - commands: [ - 'apk add --no-cache bash git', - 'git fetch origin --tags', - 'RELEASE_TAG=$(./tools/image-tag)', - 'echo $PLUGIN_CONFIG_TEMPLATE > %s' % configFileName, - // replace placeholders with RELEASE TAG - 'sed -i -E "s/\\{\\{release\\}\\}/$RELEASE_TAG/g" %s' % configFileName, - ], - settings: { - config_template: { from_secret: helm_chart_auto_update_config_template.name }, - }, - }, - { - name: 'trigger-helm-chart-update', - image: drone_updater_plugin_image, - settings: { - github_token: { - from_secret: github_secret.name, - }, - config_file: configFileName, - }, - depends_on: ['prepare-helm-chart-update-config'], - }, - ], - }, - logql_analyzer(), - pipeline('docker-driver') { - trigger+: onTagOrMain, - steps: [ - { - name: 'build and push', - image: 'grafana/loki-build-image:%s' % build_image_version, - depends_on: ['clone'], - environment: { - DOCKER_USERNAME: { from_secret: docker_username_secret.name }, - DOCKER_PASSWORD: { from_secret: docker_password_secret.name }, - }, - commands: [ - 'git fetch origin --tags', - 'make docker-driver-push', - ], - volumes: [ - { - name: 'docker', - path: '/var/run/docker.sock', - }, - ], - privileged: true, - }, - ], - volumes: [ - { - name: 'docker', - host: { - path: '/var/run/docker.sock', - }, - }, - ], - }, -] -+ [ - lambda_promtail(arch) - for arch in ['amd64', 'arm64'] -] + [ - manifest_ecr(['lambda-promtail'], ['amd64', 'arm64']) { - trigger+: { event: ['push'] }, - }, -] + [ - github_secret, - pull_secret, - docker_username_secret, - docker_password_secret, - ecr_key, - ecr_secret_key, - updater_config_template, - helm_chart_auto_update_config_template, - gpg_passphrase, - gpg_private_key, -] diff --git a/.drone/drone.yml b/.drone/drone.yml deleted file mode 100644 index 3710a4b9d3dda..0000000000000 --- a/.drone/drone.yml +++ /dev/null @@ -1,1313 +0,0 @@ ---- -kind: pipeline -name: loki-build-image-amd64 -platform: - arch: amd64 - os: linux -steps: -- environment: - DOCKER_BUILDKIT: 1 - image: plugins/docker - name: push - settings: - context: loki-build-image - dockerfile: loki-build-image/Dockerfile - dry_run: false - password: - from_secret: docker_password - repo: grafana/loki-build-image - tags: - - 0.33.2-amd64 - username: - from_secret: docker_username - when: - event: - - push - - tag - paths: - - loki-build-image/** -trigger: - ref: - - refs/heads/main - - refs/heads/k??? - - refs/tags/v* - - refs/pull/*/head -workspace: - base: /src - path: loki ---- -kind: pipeline -name: loki-build-image-arm64 -platform: - arch: arm64 - os: linux -steps: -- environment: - DOCKER_BUILDKIT: 1 - image: plugins/docker - name: push - settings: - context: loki-build-image - dockerfile: loki-build-image/Dockerfile - dry_run: false - password: - from_secret: docker_password - repo: grafana/loki-build-image - tags: - - 0.33.2-arm64 - username: - from_secret: docker_username - when: - event: - - push - - tag - paths: - - loki-build-image/** -trigger: - ref: - - refs/heads/main - - refs/heads/k??? - - refs/tags/v* - - refs/pull/*/head -workspace: - base: /src - path: loki ---- -depends_on: -- loki-build-image-amd64 -- loki-build-image-arm64 -kind: pipeline -name: loki-build-image-publish -steps: -- image: plugins/manifest:1.4.0 - name: manifest - settings: - ignore_missing: false - password: - from_secret: docker_password - spec: .drone/docker-manifest-build-image.tmpl - target: loki-build-image:0.33.2 - username: - from_secret: docker_username - when: - event: - - push - - tag - paths: - - loki-build-image/** -trigger: - ref: - - refs/heads/main - - refs/heads/k??? - - refs/tags/v* - - refs/pull/*/head ---- -kind: pipeline -name: helm-test-image -steps: -- image: plugins/docker - name: push-image - settings: - dockerfile: production/helm/loki/src/helm-test/Dockerfile - dry_run: false - password: - from_secret: docker_password - repo: grafana/loki-helm-test - username: - from_secret: docker_username - when: - event: - - push - - tag - paths: - - production/helm/loki/src/helm-test/** -trigger: - ref: - - refs/heads/main - - refs/heads/k??? - - refs/tags/v* - - refs/pull/*/head -workspace: - base: /src - path: loki ---- -kind: pipeline -name: documentation-checks -steps: -- commands: - - make BUILD_IN_CONTAINER=false documentation-helm-reference-check - depends_on: - - clone - environment: {} - image: grafana/loki-build-image:0.33.6 - name: documentation-helm-reference-check -trigger: - ref: - - refs/heads/main - - refs/heads/k??? - - refs/tags/v* - - refs/pull/*/head -workspace: - base: /src - path: loki ---- -kind: pipeline -name: docker-amd64 -platform: - arch: amd64 - os: linux -steps: -- commands: - - apk add --no-cache bash git - - git fetch origin --tags - - echo $(./tools/image-tag)-amd64 > .tags - image: alpine - name: image-tag -- depends_on: - - image-tag - image: plugins/docker - name: publish-loki-image - settings: - dockerfile: cmd/loki/Dockerfile - dry_run: false - password: - from_secret: docker_password - repo: grafana/loki - username: - from_secret: docker_username - when: - event: - - push - - tag -- depends_on: - - image-tag - image: plugins/docker - name: publish-loki-canary-image - settings: - dockerfile: cmd/loki-canary/Dockerfile - dry_run: false - password: - from_secret: docker_password - repo: grafana/loki-canary - username: - from_secret: docker_username - when: - event: - - push - - tag -- depends_on: - - image-tag - image: plugins/docker - name: publish-loki-canary-boringcrypto-image - settings: - dockerfile: cmd/loki-canary-boringcrypto/Dockerfile - dry_run: false - password: - from_secret: docker_password - repo: grafana/loki-canary-boringcrypto - username: - from_secret: docker_username - when: - event: - - push - - tag -- depends_on: - - image-tag - image: plugins/docker - name: publish-logcli-image - settings: - dockerfile: cmd/logcli/Dockerfile - dry_run: false - password: - from_secret: docker_password - repo: grafana/logcli - username: - from_secret: docker_username - when: - event: - - push - - tag -trigger: - ref: - - refs/heads/main - - refs/heads/k??? - - refs/tags/v* - - refs/pull/*/head ---- -kind: pipeline -name: docker-arm64 -platform: - arch: arm64 - os: linux -steps: -- commands: - - apk add --no-cache bash git - - git fetch origin --tags - - echo $(./tools/image-tag)-arm64 > .tags - image: alpine - name: image-tag -- depends_on: - - image-tag - image: plugins/docker - name: publish-loki-image - settings: - dockerfile: cmd/loki/Dockerfile - dry_run: false - password: - from_secret: docker_password - repo: grafana/loki - username: - from_secret: docker_username - when: - event: - - push - - tag -- depends_on: - - image-tag - image: plugins/docker - name: publish-loki-canary-image - settings: - dockerfile: cmd/loki-canary/Dockerfile - dry_run: false - password: - from_secret: docker_password - repo: grafana/loki-canary - username: - from_secret: docker_username - when: - event: - - push - - tag -- depends_on: - - image-tag - image: plugins/docker - name: publish-loki-canary-boringcrypto-image - settings: - dockerfile: cmd/loki-canary-boringcrypto/Dockerfile - dry_run: false - password: - from_secret: docker_password - repo: grafana/loki-canary-boringcrypto - username: - from_secret: docker_username - when: - event: - - push - - tag -- depends_on: - - image-tag - image: plugins/docker - name: publish-logcli-image - settings: - dockerfile: cmd/logcli/Dockerfile - dry_run: false - password: - from_secret: docker_password - repo: grafana/logcli - username: - from_secret: docker_username - when: - event: - - push - - tag -trigger: - ref: - - refs/heads/main - - refs/heads/k??? - - refs/tags/v* - - refs/pull/*/head ---- -kind: pipeline -name: docker-arm -platform: - arch: arm - os: linux -steps: -- commands: - - apk add --no-cache bash git - - git fetch origin --tags - - echo $(./tools/image-tag)-arm > .tags - image: alpine - name: image-tag -- depends_on: - - image-tag - image: plugins/docker:linux-arm - name: publish-loki-image - settings: - dockerfile: cmd/loki/Dockerfile - dry_run: false - password: - from_secret: docker_password - repo: grafana/loki - username: - from_secret: docker_username - when: - event: - - push - - tag -- depends_on: - - image-tag - image: plugins/docker:linux-arm - name: publish-loki-canary-image - settings: - dockerfile: cmd/loki-canary/Dockerfile - dry_run: false - password: - from_secret: docker_password - repo: grafana/loki-canary - username: - from_secret: docker_username - when: - event: - - push - - tag -- depends_on: - - image-tag - image: plugins/docker:linux-arm - name: publish-loki-canary-boringcrypto-image - settings: - dockerfile: cmd/loki-canary-boringcrypto/Dockerfile - dry_run: false - password: - from_secret: docker_password - repo: grafana/loki-canary-boringcrypto - username: - from_secret: docker_username - when: - event: - - push - - tag -- depends_on: - - image-tag - image: plugins/docker:linux-arm - name: publish-logcli-image - settings: - dockerfile: cmd/logcli/Dockerfile - dry_run: false - password: - from_secret: docker_password - repo: grafana/logcli - username: - from_secret: docker_username - when: - event: - - push - - tag -trigger: - ref: - - refs/heads/main - - refs/heads/k??? - - refs/tags/v* - - refs/pull/*/head ---- -kind: pipeline -name: promtail-amd64 -platform: - arch: amd64 - os: linux -steps: -- commands: - - apk add --no-cache bash git - - git fetch origin --tags - - echo $(./tools/image-tag)-amd64 > .tags - image: alpine - name: image-tag -- depends_on: - - image-tag - image: plugins/docker - name: publish-promtail-image - settings: - dockerfile: clients/cmd/promtail/Dockerfile - dry_run: false - password: - from_secret: docker_password - repo: grafana/promtail - username: - from_secret: docker_username - when: - event: - - push - - tag -trigger: - ref: - - refs/heads/main - - refs/heads/k??? - - refs/tags/v* - - refs/pull/*/head ---- -kind: pipeline -name: promtail-arm64 -platform: - arch: arm64 - os: linux -steps: -- commands: - - apk add --no-cache bash git - - git fetch origin --tags - - echo $(./tools/image-tag)-arm64 > .tags - image: alpine - name: image-tag -- depends_on: - - image-tag - image: plugins/docker - name: publish-promtail-image - settings: - dockerfile: clients/cmd/promtail/Dockerfile - dry_run: false - password: - from_secret: docker_password - repo: grafana/promtail - username: - from_secret: docker_username - when: - event: - - push - - tag -trigger: - ref: - - refs/heads/main - - refs/heads/k??? - - refs/tags/v* - - refs/pull/*/head ---- -kind: pipeline -name: promtail-arm -platform: - arch: arm - os: linux -steps: -- commands: - - apk add --no-cache bash git - - git fetch origin --tags - - echo $(./tools/image-tag)-arm > .tags - image: alpine - name: image-tag -- depends_on: - - image-tag - image: plugins/docker:linux-arm - name: publish-promtail-image - settings: - dockerfile: clients/cmd/promtail/Dockerfile.arm32 - dry_run: false - password: - from_secret: docker_password - repo: grafana/promtail - username: - from_secret: docker_username - when: - event: - - push - - tag -trigger: - ref: - - refs/heads/main - - refs/heads/k??? - - refs/tags/v* - - refs/pull/*/head ---- -kind: pipeline -name: lokioperator-amd64 -platform: - arch: amd64 - os: linux -steps: -- commands: - - apk add --no-cache bash git - - git fetch origin --tags - - echo $(./tools/image-tag)-amd64 > .tags - image: alpine - name: image-tag -- depends_on: - - image-tag - image: plugins/docker - name: publish-loki-operator-image - settings: - context: operator - dockerfile: operator/Dockerfile - dry_run: false - password: - from_secret: docker_password - repo: grafana/loki-operator - username: - from_secret: docker_username - when: - event: - - push - - tag - ref: - - refs/heads/main - - refs/tags/operator/v* -trigger: - ref: - - refs/heads/main - - refs/tags/operator/v* - - refs/pull/*/head ---- -kind: pipeline -name: lokioperator-arm64 -platform: - arch: arm64 - os: linux -steps: -- commands: - - apk add --no-cache bash git - - git fetch origin --tags - - echo $(./tools/image-tag)-arm64 > .tags - image: alpine - name: image-tag -- depends_on: - - image-tag - image: plugins/docker - name: publish-loki-operator-image - settings: - context: operator - dockerfile: operator/Dockerfile - dry_run: false - password: - from_secret: docker_password - repo: grafana/loki-operator - username: - from_secret: docker_username - when: - event: - - push - - tag - ref: - - refs/heads/main - - refs/tags/operator/v* -trigger: - ref: - - refs/heads/main - - refs/tags/operator/v* - - refs/pull/*/head ---- -kind: pipeline -name: lokioperator-arm -platform: - arch: arm - os: linux -steps: -- commands: - - apk add --no-cache bash git - - git fetch origin --tags - - echo $(./tools/image-tag)-arm > .tags - image: alpine - name: image-tag -- depends_on: - - image-tag - image: plugins/docker:linux-arm - name: publish-loki-operator-image - settings: - context: operator - dockerfile: operator/Dockerfile - dry_run: false - password: - from_secret: docker_password - repo: grafana/loki-operator - username: - from_secret: docker_username - when: - event: - - push - - tag - ref: - - refs/heads/main - - refs/tags/operator/v* -trigger: - ref: - - refs/heads/main - - refs/tags/operator/v* - - refs/pull/*/head ---- -kind: pipeline -name: fluent-bit-amd64 -platform: - arch: amd64 - os: linux -steps: -- commands: - - apk add --no-cache bash git - - git fetch origin --tags - - echo $(./tools/image-tag)-amd64 > .tags - image: alpine - name: image-tag -- depends_on: - - image-tag - image: plugins/docker - name: publish-fluent-bit-image - settings: - dockerfile: clients/cmd/fluent-bit/Dockerfile - dry_run: false - password: - from_secret: docker_password - repo: grafana/fluent-bit-plugin-loki - username: - from_secret: docker_username - when: - event: - - push - - tag -trigger: - ref: - - refs/heads/main - - refs/heads/k??? - - refs/tags/v* - - refs/pull/*/head ---- -kind: pipeline -name: fluent-bit-arm64 -platform: - arch: arm64 - os: linux -steps: -- commands: - - apk add --no-cache bash git - - git fetch origin --tags - - echo $(./tools/image-tag)-arm64 > .tags - image: alpine - name: image-tag -- depends_on: - - image-tag - image: plugins/docker - name: publish-fluent-bit-image - settings: - dockerfile: clients/cmd/fluent-bit/Dockerfile - dry_run: false - password: - from_secret: docker_password - repo: grafana/fluent-bit-plugin-loki - username: - from_secret: docker_username - when: - event: - - push - - tag -trigger: - ref: - - refs/heads/main - - refs/heads/k??? - - refs/tags/v* - - refs/pull/*/head ---- -kind: pipeline -name: fluent-bit-arm -platform: - arch: arm - os: linux -steps: -- commands: - - apk add --no-cache bash git - - git fetch origin --tags - - echo $(./tools/image-tag)-arm > .tags - image: alpine - name: image-tag -- depends_on: - - image-tag - image: plugins/docker:linux-arm - name: publish-fluent-bit-image - settings: - dockerfile: clients/cmd/fluent-bit/Dockerfile - dry_run: false - password: - from_secret: docker_password - repo: grafana/fluent-bit-plugin-loki - username: - from_secret: docker_username - when: - event: - - push - - tag -trigger: - ref: - - refs/heads/main - - refs/heads/k??? - - refs/tags/v* - - refs/pull/*/head ---- -kind: pipeline -name: fluentd-amd64 -platform: - arch: amd64 - os: linux -steps: -- commands: - - apk add --no-cache bash git - - git fetch origin --tags - - echo $(./tools/image-tag)-amd64 > .tags - - echo ",main" >> .tags - image: alpine - name: image-tag -- depends_on: - - image-tag - image: plugins/docker - name: publish-fluentd-image - settings: - dockerfile: clients/cmd/fluentd/Dockerfile - dry_run: false - password: - from_secret: docker_password - repo: grafana/fluent-plugin-loki - username: - from_secret: docker_username - when: - event: - - push - - tag -trigger: - ref: - - refs/heads/main - - refs/heads/k??? - - refs/tags/v* - - refs/pull/*/head ---- -kind: pipeline -name: logstash-amd64 -platform: - arch: amd64 - os: linux -steps: -- commands: - - apk add --no-cache bash git - - git fetch origin --tags - - echo $(./tools/image-tag)-amd64 > .tags - - echo ",main" >> .tags - image: alpine - name: image-tag -- depends_on: - - image-tag - image: plugins/docker - name: publish-logstash-image - settings: - dockerfile: clients/cmd/logstash/Dockerfile - dry_run: false - password: - from_secret: docker_password - repo: grafana/logstash-output-loki - username: - from_secret: docker_username - when: - event: - - push - - tag -trigger: - ref: - - refs/heads/main - - refs/heads/k??? - - refs/tags/v* - - refs/pull/*/head ---- -kind: pipeline -name: querytee-amd64 -platform: - arch: amd64 - os: linux -steps: -- commands: - - apk add --no-cache bash git - - git fetch origin --tags - - echo $(./tools/image-tag)-amd64 > .tags - - echo ",main" >> .tags - image: alpine - name: image-tag -- depends_on: - - image-tag - image: plugins/docker - name: publish-querytee-image - settings: - dockerfile: cmd/querytee/Dockerfile - dry_run: false - password: - from_secret: docker_password - repo: grafana/loki-query-tee - username: - from_secret: docker_username - when: - event: - - push - - tag -trigger: - ref: - - refs/heads/main - - refs/heads/k??? - - refs/tags/v* - - refs/pull/*/head ---- -depends_on: -- docker-amd64 -- docker-arm64 -- docker-arm -- promtail-amd64 -- promtail-arm64 -- promtail-arm -- fluent-bit-amd64 -- fluent-bit-arm64 -- fluent-bit-arm -kind: pipeline -name: manifest -steps: -- depends_on: - - clone - image: plugins/manifest:1.4.0 - name: manifest-promtail - settings: - ignore_missing: false - password: - from_secret: docker_password - spec: .drone/docker-manifest.tmpl - target: promtail - username: - from_secret: docker_username -- depends_on: - - clone - - manifest-promtail - image: plugins/manifest:1.4.0 - name: manifest-loki - settings: - ignore_missing: false - password: - from_secret: docker_password - spec: .drone/docker-manifest.tmpl - target: loki - username: - from_secret: docker_username -- depends_on: - - clone - - manifest-loki - image: plugins/manifest:1.4.0 - name: manifest-loki-canary - settings: - ignore_missing: false - password: - from_secret: docker_password - spec: .drone/docker-manifest.tmpl - target: loki-canary - username: - from_secret: docker_username -- depends_on: - - clone - - manifest-loki-canary - image: plugins/manifest:1.4.0 - name: manifest-loki-canary-boringcrypto - settings: - ignore_missing: false - password: - from_secret: docker_password - spec: .drone/docker-manifest.tmpl - target: loki-canary-boringcrypto - username: - from_secret: docker_username -- depends_on: - - clone - - manifest-loki-canary-boringcrypto - image: plugins/manifest:1.4.0 - name: manifest-fluent-bit-plugin-loki - settings: - ignore_missing: false - password: - from_secret: docker_password - spec: .drone/docker-manifest.tmpl - target: fluent-bit-plugin-loki - username: - from_secret: docker_username -trigger: - event: - - push - - tag - ref: - - refs/heads/main - - refs/heads/k??? - - refs/tags/v* - - refs/pull/*/head ---- -depends_on: -- lokioperator-amd64 -- lokioperator-arm64 -- lokioperator-arm -kind: pipeline -name: manifest-operator -steps: -- depends_on: - - clone - image: plugins/manifest:1.4.0 - name: manifest-loki-operator - settings: - ignore_missing: false - password: - from_secret: docker_password - spec: .drone/docker-manifest-operator.tmpl - target: loki-operator - username: - from_secret: docker_username -trigger: - event: - - push - - tag - ref: - - refs/heads/main - - refs/tags/operator/v* ---- -depends_on: -- manifest -image_pull_secrets: -- dockerconfigjson -kind: pipeline -name: deploy -steps: -- commands: - - apk add --no-cache bash git - - git fetch origin --tags - - echo $(./tools/image-tag) > .tag - - export RELEASE_TAG=$(cat .tag) - - export RELEASE_NAME=$([[ $RELEASE_TAG =~ $RELEASE_TAG_REGEXP ]] && echo $RELEASE_TAG - | grep -oE $MAJOR_MINOR_VERSION_REGEXP | sed "s/\./-/g" | sed "s/$/-x/" || echo - "next") - - echo $RELEASE_NAME - - echo $PLUGIN_CONFIG_TEMPLATE > updater-config.json - - sed -i "s/\"{{release}}\"/\"$RELEASE_NAME\"/g" updater-config.json - - sed -i "s/{{version}}/$RELEASE_TAG/g" updater-config.json - depends_on: - - clone - environment: - MAJOR_MINOR_VERSION_REGEXP: ([0-9]+\.[0-9]+) - RELEASE_TAG_REGEXP: ^([0-9]+\.[0-9]+\.[0-9]+)$ - image: alpine - name: prepare-updater-config - settings: - config_template: - from_secret: updater_config_template -- depends_on: - - prepare-updater-config - image: us.gcr.io/kubernetes-dev/drone/plugins/updater@sha256:cbcb09c74f96a34c528f52bf9b4815a036b11fed65f685be216e0c8b8e84285b - name: trigger - settings: - config_file: updater-config.json - github_token: - from_secret: github_token -trigger: - event: - - push - - tag - ref: - - refs/heads/main - - refs/tags/v* ---- -depends_on: -- manifest -image_pull_secrets: -- dockerconfigjson -kind: pipeline -name: update-loki-helm-chart-on-loki-release -steps: -- commands: - - apk add --no-cache bash git - - git fetch --tags - - latest_version=$(git tag -l 'v[0-9]*.[0-9]*.[0-9]*' | sort -V | tail -n 1 | sed - 's/v//g') - - RELEASE_TAG=$(./tools/image-tag) - - if [ "$RELEASE_TAG" != "$latest_version" ]; then echo "Current version $RELEASE_TAG - is not the latest version of Loki. The latest version is $latest_version" && exit - 78; fi - image: alpine - name: check-version-is-latest - when: - event: - - tag -- commands: - - apk add --no-cache bash git - - git fetch origin --tags - - RELEASE_TAG=$(./tools/image-tag) - - echo $PLUGIN_CONFIG_TEMPLATE > updater-config.json - - sed -i -E "s/\{\{release\}\}/$RELEASE_TAG/g" updater-config.json - depends_on: - - check-version-is-latest - image: alpine - name: prepare-helm-chart-update-config - settings: - config_template: - from_secret: helm-chart-update-config-template -- depends_on: - - prepare-helm-chart-update-config - image: us.gcr.io/kubernetes-dev/drone/plugins/updater@sha256:cbcb09c74f96a34c528f52bf9b4815a036b11fed65f685be216e0c8b8e84285b - name: trigger-helm-chart-update - settings: - config_file: updater-config.json - github_token: - from_secret: github_token -trigger: - ref: - - refs/tags/v* ---- -kind: pipeline -name: logql-analyzer -platform: - arch: amd64 - os: linux -steps: -- commands: - - apk add --no-cache bash git - - git fetch origin --tags - - echo $(./tools/image-tag)-amd64 > .tags - image: alpine - name: image-tag -- depends_on: - - image-tag - image: plugins/docker - name: publish-logql-analyzer-image - settings: - dockerfile: cmd/logql-analyzer/Dockerfile - dry_run: false - password: - from_secret: docker_password - repo: grafana/logql-analyzer - username: - from_secret: docker_username - when: - event: - - push - - tag -trigger: - ref: - - refs/heads/main - - refs/heads/k??? - - refs/tags/v* - - refs/pull/*/head ---- -kind: pipeline -name: docker-driver -steps: -- commands: - - git fetch origin --tags - - make docker-driver-push - depends_on: - - clone - environment: - DOCKER_PASSWORD: - from_secret: docker_password - DOCKER_USERNAME: - from_secret: docker_username - image: grafana/loki-build-image:0.33.6 - name: build and push - privileged: true - volumes: - - name: docker - path: /var/run/docker.sock -trigger: - event: - - push - - tag - ref: - - refs/heads/main - - refs/heads/k??? - - refs/tags/v* - - refs/pull/*/head -volumes: -- host: - path: /var/run/docker.sock - name: docker ---- -kind: pipeline -name: lambda-promtail-amd64 -platform: - arch: amd64 - os: linux -steps: -- commands: - - apk add --no-cache bash git - - git fetch origin --tags - - echo $(./tools/image-tag)-amd64 > .tags - image: alpine - name: image-tag -- commands: - - if [ "$${#TEST_SECRET}" -eq 0 ]; then - - ' echo "Missing a secret to run this pipeline. This branch needs to be re-pushed - as a branch in main grafana/loki repository in order to run." && exit 78' - - fi - environment: - TEST_SECRET: - from_secret: ecr_key - image: alpine - name: skip pipeline if missing secret -- depends_on: - - image-tag - image: cstyan/ecr - name: publish-lambda-promtail-image - privileged: true - settings: - access_key: - from_secret: ecr_key - dockerfile: tools/lambda-promtail/Dockerfile - dry_run: false - region: us-east-1 - registry: public.ecr.aws/grafana - repo: public.ecr.aws/grafana/lambda-promtail - secret_key: - from_secret: ecr_secret_key - when: - event: - - push - - tag -trigger: - ref: - - refs/heads/main - - refs/heads/k??? - - refs/tags/v* - - refs/pull/*/head ---- -kind: pipeline -name: lambda-promtail-arm64 -platform: - arch: arm64 - os: linux -steps: -- commands: - - apk add --no-cache bash git - - git fetch origin --tags - - echo $(./tools/image-tag)-arm64 > .tags - image: alpine - name: image-tag -- commands: - - if [ "$${#TEST_SECRET}" -eq 0 ]; then - - ' echo "Missing a secret to run this pipeline. This branch needs to be re-pushed - as a branch in main grafana/loki repository in order to run." && exit 78' - - fi - environment: - TEST_SECRET: - from_secret: ecr_key - image: alpine - name: skip pipeline if missing secret -- depends_on: - - image-tag - image: cstyan/ecr - name: publish-lambda-promtail-image - privileged: true - settings: - access_key: - from_secret: ecr_key - dockerfile: tools/lambda-promtail/Dockerfile - dry_run: false - region: us-east-1 - registry: public.ecr.aws/grafana - repo: public.ecr.aws/grafana/lambda-promtail - secret_key: - from_secret: ecr_secret_key - when: - event: - - push - - tag -trigger: - ref: - - refs/heads/main - - refs/heads/k??? - - refs/tags/v* - - refs/pull/*/head ---- -depends_on: -- lambda-promtail-amd64 -- lambda-promtail-arm64 -kind: pipeline -name: manifest-ecr -steps: -- commands: - - apk add --no-cache aws-cli - - docker login --username AWS --password $(aws ecr-public get-login-password --region - us-east-1) public.ecr.aws - depends_on: - - clone - environment: - AWS_ACCESS_KEY_ID: - from_secret: ecr_key - AWS_SECRET_ACCESS_KEY: - from_secret: ecr_secret_key - image: docker:dind - name: ecr-login - volumes: - - name: dockerconf - path: /root/.docker -- depends_on: - - clone - - ecr-login - image: plugins/manifest:1.4.0 - name: manifest-lambda-promtail - settings: - ignore_missing: true - spec: .drone/docker-manifest-ecr.tmpl - target: lambda-promtail - volumes: - - name: dockerconf - path: /.docker -trigger: - event: - - push - ref: - - refs/heads/main - - refs/heads/k??? - - refs/tags/v* - - refs/pull/*/head -volumes: -- name: dockerconf - temp: {} ---- -get: - name: pat - path: infra/data/ci/github/grafanabot -kind: secret -name: github_token ---- -get: - name: .dockerconfigjson - path: secret/data/common/gcr -kind: secret -name: dockerconfigjson ---- -get: - name: username - path: infra/data/ci/docker_hub -kind: secret -name: docker_username ---- -get: - name: password - path: infra/data/ci/docker_hub -kind: secret -name: docker_password ---- -get: - name: access_key_id - path: infra/data/ci/loki/aws-credentials -kind: secret -name: ecr_key ---- -get: - name: secret_access_key - path: infra/data/ci/loki/aws-credentials -kind: secret -name: ecr_secret_key ---- -get: - name: updater-config-template.json - path: secret/data/common/loki_ci_autodeploy -kind: secret -name: updater_config_template ---- -get: - name: on-loki-release-config.json - path: secret/data/common/loki-helm-chart-auto-update -kind: secret -name: helm-chart-update-config-template ---- -get: - name: passphrase - path: infra/data/ci/packages-publish/gpg -kind: secret -name: gpg_passphrase ---- -get: - name: private-key - path: infra/data/ci/packages-publish/gpg -kind: secret -name: gpg_private_key ---- -kind: signature -hmac: 3b3b039769ab8c44318749efec569ffe50c4cfb173f577422ec9d514054f0a9e - -... diff --git a/.github/jsonnetfile.json b/.github/jsonnetfile.json index 1038aebdcd66c..130eaeb4e984e 100644 --- a/.github/jsonnetfile.json +++ b/.github/jsonnetfile.json @@ -8,7 +8,7 @@ "subdir": "workflows" } }, - "version": "87cb5090c36b5332e7f21b5c59e136962d5f4f56" + "version": "98ce96e408db867d64fb95b59a99c24440ddf441" } ], "legacyImports": true diff --git a/.github/jsonnetfile.lock.json b/.github/jsonnetfile.lock.json index 9eef1872519bf..172082408f8c7 100644 --- a/.github/jsonnetfile.lock.json +++ b/.github/jsonnetfile.lock.json @@ -8,8 +8,8 @@ "subdir": "workflows" } }, - "version": "87cb5090c36b5332e7f21b5c59e136962d5f4f56", - "sum": "kVlVZPpPz8d/D6UGK9Hto+NeGy7z8NvGygcB1QboxWw=" + "version": "98ce96e408db867d64fb95b59a99c24440ddf441", + "sum": "pqEiutdl50ghtCY0wReq+Xa3AymHEyMa1OJQvRQXINI=" } ], "legacyImports": false diff --git a/.github/release-workflows.jsonnet b/.github/release-workflows.jsonnet index 72fdbfd5dc17e..9cf613dfc042d 100644 --- a/.github/release-workflows.jsonnet +++ b/.github/release-workflows.jsonnet @@ -17,11 +17,23 @@ local imageJobs = { querytee: build.image('loki-query-tee', 'cmd/querytee', platform=['linux/amd64']), }; +local weeklyImageJobs = { + loki: build.weeklyImage('loki', 'cmd/loki'), + fluentd: build.weeklyImage('fluent-plugin-loki', 'clients/cmd/fluentd', platform=['linux/amd64']), + 'fluent-bit': build.weeklyImage('fluent-bit-plugin-loki', 'clients/cmd/fluent-bit', platform=['linux/amd64']), + logstash: build.weeklyImage('logstash-output-loki', 'clients/cmd/logstash', platform=['linux/amd64']), + logcli: build.weeklyImage('logcli', 'cmd/logcli'), + 'loki-canary': build.weeklyImage('loki-canary', 'cmd/loki-canary'), + 'loki-canary-boringcrypto': build.weeklyImage('loki-canary-boringcrypto', 'cmd/loki-canary-boringcrypto'), + promtail: build.weeklyImage('promtail', 'clients/cmd/promtail'), + querytee: build.weeklyImage('loki-query-tee', 'cmd/querytee', platform=['linux/amd64']), +}; + local buildImageVersion = std.extVar('BUILD_IMAGE_VERSION'); local buildImage = 'grafana/loki-build-image:%s' % buildImageVersion; local golangCiLintVersion = 'v1.55.1'; -local imageBuildTimeoutMin = 40; +local imageBuildTimeoutMin = 60; local imagePrefix = 'grafana'; { @@ -94,4 +106,42 @@ local imagePrefix = 'grafana'; }, }, }), + 'images.yml': std.manifestYamlDoc({ + name: 'publish images', + on: { + push: { + branches: [ + 'k[0-9]+*', // This is a weird glob pattern, not a regexp, do not use ".*", see https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#filter-pattern-cheat-sheet + 'main', + ], + }, + }, + permissions: { + 'id-token': 'write', + contents: 'write', + 'pull-requests': 'write', + }, + jobs: { + check: { + uses: checkTemplate, + with: { + build_image: buildImage, + golang_ci_lint_version: golangCiLintVersion, + release_lib_ref: releaseLibRef, + skip_validation: false, + use_github_app_token: true, + }, + }, + } + std.mapWithKey(function(name, job) + job + + lokiRelease.job.withNeeds(['check']) + + { + env: { + BUILD_TIMEOUT: imageBuildTimeoutMin, + RELEASE_REPO: 'grafana/loki', + RELEASE_LIB_REF: releaseLibRef, + IMAGE_PREFIX: imagePrefix, + }, + }, weeklyImageJobs), + }), } diff --git a/.github/vendor/github.com/grafana/loki-release/workflows/build.libsonnet b/.github/vendor/github.com/grafana/loki-release/workflows/build.libsonnet index bd9d2e2e9b119..7343c7d72963d 100644 --- a/.github/vendor/github.com/grafana/loki-release/workflows/build.libsonnet +++ b/.github/vendor/github.com/grafana/loki-release/workflows/build.libsonnet @@ -42,7 +42,7 @@ local releaseLibStep = common.releaseLibStep; echo "platform_short=$(echo ${{ matrix.platform }} | cut -d / -f 2)" >> $GITHUB_OUTPUT |||), - step.new('Build and export', 'docker/build-push-action@v5') + step.new('Build and export', 'docker/build-push-action@v6') + step.withTimeoutMinutes('${{ fromJSON(env.BUILD_TIMEOUT) }}') + step.withIf('${{ fromJSON(needs.version.outputs.pr_created) }}') + step.withEnv({ @@ -93,7 +93,7 @@ local releaseLibStep = common.releaseLibStep; echo "version=$(./tools/image-tag)" >> $GITHUB_OUTPUT |||), - step.new('Build and push', 'docker/build-push-action@v5') + step.new('Build and push', 'docker/build-push-action@v6') + step.withTimeoutMinutes('${{ fromJSON(env.BUILD_TIMEOUT) }}') + step.with({ context: context, diff --git a/.github/vendor/github.com/grafana/loki-release/workflows/validate.libsonnet b/.github/vendor/github.com/grafana/loki-release/workflows/validate.libsonnet index 40bf097049e83..44f4984e4b785 100644 --- a/.github/vendor/github.com/grafana/loki-release/workflows/validate.libsonnet +++ b/.github/vendor/github.com/grafana/loki-release/workflows/validate.libsonnet @@ -115,7 +115,6 @@ local validationJob = _validationJob(false); validationMakeStep('validate dev cluster config', 'validate-dev-cluster-config'), validationMakeStep('check example config docs', 'check-example-config-doc'), validationMakeStep('check helm reference doc', 'documentation-helm-reference-check'), - validationMakeStep('check drone drift', 'check-drone-drift'), ]) + { steps+: [ step.new('build docs website') diff --git a/.github/workflows/images.yml b/.github/workflows/images.yml new file mode 100644 index 0000000000000..f6d8ca6e08aab --- /dev/null +++ b/.github/workflows/images.yml @@ -0,0 +1,433 @@ +"jobs": + "check": + "uses": "grafana/loki-release/.github/workflows/check.yml@main" + "with": + "build_image": "grafana/loki-build-image:0.33.6" + "golang_ci_lint_version": "v1.55.1" + "release_lib_ref": "main" + "skip_validation": false + "use_github_app_token": true + "fluent-bit": + "env": + "BUILD_TIMEOUT": 60 + "IMAGE_PREFIX": "grafana" + "RELEASE_LIB_REF": "main" + "RELEASE_REPO": "grafana/loki" + "needs": + - "check" + "runs-on": "ubuntu-latest" + "steps": + - "name": "pull release library code" + "uses": "actions/checkout@v4" + "with": + "path": "lib" + "ref": "${{ env.RELEASE_LIB_REF }}" + "repository": "grafana/loki-release" + - "name": "pull code to release" + "uses": "actions/checkout@v4" + "with": + "path": "release" + "repository": "${{ env.RELEASE_REPO }}" + - "name": "setup node" + "uses": "actions/setup-node@v4" + "with": + "node-version": 20 + - "name": "Set up QEMU" + "uses": "docker/setup-qemu-action@v3" + - "name": "set up docker buildx" + "uses": "docker/setup-buildx-action@v3" + - "name": "Login to DockerHub (from vault)" + "uses": "grafana/shared-workflows/actions/dockerhub-login@main" + - "id": "weekly-version" + "name": "Get weekly version" + "run": | + echo "version=$(./tools/image-tag)" >> $GITHUB_OUTPUT + "working-directory": "release" + - "name": "Build and push" + "timeout-minutes": "${{ fromJSON(env.BUILD_TIMEOUT) }}" + "uses": "docker/build-push-action@v6" + "with": + "build-args": "IMAGE_TAG=${{ steps.weekly-version.outputs.version }}" + "context": "release" + "file": "release/clients/cmd/fluent-bit/Dockerfile" + "platforms": "linux/amd64" + "push": true + "tags": "${{ env.IMAGE_PREFIX }}/fluent-bit-plugin-loki:${{ steps.weekly-version.outputs.version }}" + "fluentd": + "env": + "BUILD_TIMEOUT": 60 + "IMAGE_PREFIX": "grafana" + "RELEASE_LIB_REF": "main" + "RELEASE_REPO": "grafana/loki" + "needs": + - "check" + "runs-on": "ubuntu-latest" + "steps": + - "name": "pull release library code" + "uses": "actions/checkout@v4" + "with": + "path": "lib" + "ref": "${{ env.RELEASE_LIB_REF }}" + "repository": "grafana/loki-release" + - "name": "pull code to release" + "uses": "actions/checkout@v4" + "with": + "path": "release" + "repository": "${{ env.RELEASE_REPO }}" + - "name": "setup node" + "uses": "actions/setup-node@v4" + "with": + "node-version": 20 + - "name": "Set up QEMU" + "uses": "docker/setup-qemu-action@v3" + - "name": "set up docker buildx" + "uses": "docker/setup-buildx-action@v3" + - "name": "Login to DockerHub (from vault)" + "uses": "grafana/shared-workflows/actions/dockerhub-login@main" + - "id": "weekly-version" + "name": "Get weekly version" + "run": | + echo "version=$(./tools/image-tag)" >> $GITHUB_OUTPUT + "working-directory": "release" + - "name": "Build and push" + "timeout-minutes": "${{ fromJSON(env.BUILD_TIMEOUT) }}" + "uses": "docker/build-push-action@v6" + "with": + "build-args": "IMAGE_TAG=${{ steps.weekly-version.outputs.version }}" + "context": "release" + "file": "release/clients/cmd/fluentd/Dockerfile" + "platforms": "linux/amd64" + "push": true + "tags": "${{ env.IMAGE_PREFIX }}/fluent-plugin-loki:${{ steps.weekly-version.outputs.version }}" + "logcli": + "env": + "BUILD_TIMEOUT": 60 + "IMAGE_PREFIX": "grafana" + "RELEASE_LIB_REF": "main" + "RELEASE_REPO": "grafana/loki" + "needs": + - "check" + "runs-on": "ubuntu-latest" + "steps": + - "name": "pull release library code" + "uses": "actions/checkout@v4" + "with": + "path": "lib" + "ref": "${{ env.RELEASE_LIB_REF }}" + "repository": "grafana/loki-release" + - "name": "pull code to release" + "uses": "actions/checkout@v4" + "with": + "path": "release" + "repository": "${{ env.RELEASE_REPO }}" + - "name": "setup node" + "uses": "actions/setup-node@v4" + "with": + "node-version": 20 + - "name": "Set up QEMU" + "uses": "docker/setup-qemu-action@v3" + - "name": "set up docker buildx" + "uses": "docker/setup-buildx-action@v3" + - "name": "Login to DockerHub (from vault)" + "uses": "grafana/shared-workflows/actions/dockerhub-login@main" + - "id": "weekly-version" + "name": "Get weekly version" + "run": | + echo "version=$(./tools/image-tag)" >> $GITHUB_OUTPUT + "working-directory": "release" + - "name": "Build and push" + "timeout-minutes": "${{ fromJSON(env.BUILD_TIMEOUT) }}" + "uses": "docker/build-push-action@v6" + "with": + "build-args": "IMAGE_TAG=${{ steps.weekly-version.outputs.version }}" + "context": "release" + "file": "release/cmd/logcli/Dockerfile" + "platforms": "linux/amd64,linux/arm64,linux/arm" + "push": true + "tags": "${{ env.IMAGE_PREFIX }}/logcli:${{ steps.weekly-version.outputs.version }}" + "logstash": + "env": + "BUILD_TIMEOUT": 60 + "IMAGE_PREFIX": "grafana" + "RELEASE_LIB_REF": "main" + "RELEASE_REPO": "grafana/loki" + "needs": + - "check" + "runs-on": "ubuntu-latest" + "steps": + - "name": "pull release library code" + "uses": "actions/checkout@v4" + "with": + "path": "lib" + "ref": "${{ env.RELEASE_LIB_REF }}" + "repository": "grafana/loki-release" + - "name": "pull code to release" + "uses": "actions/checkout@v4" + "with": + "path": "release" + "repository": "${{ env.RELEASE_REPO }}" + - "name": "setup node" + "uses": "actions/setup-node@v4" + "with": + "node-version": 20 + - "name": "Set up QEMU" + "uses": "docker/setup-qemu-action@v3" + - "name": "set up docker buildx" + "uses": "docker/setup-buildx-action@v3" + - "name": "Login to DockerHub (from vault)" + "uses": "grafana/shared-workflows/actions/dockerhub-login@main" + - "id": "weekly-version" + "name": "Get weekly version" + "run": | + echo "version=$(./tools/image-tag)" >> $GITHUB_OUTPUT + "working-directory": "release" + - "name": "Build and push" + "timeout-minutes": "${{ fromJSON(env.BUILD_TIMEOUT) }}" + "uses": "docker/build-push-action@v6" + "with": + "build-args": "IMAGE_TAG=${{ steps.weekly-version.outputs.version }}" + "context": "release" + "file": "release/clients/cmd/logstash/Dockerfile" + "platforms": "linux/amd64" + "push": true + "tags": "${{ env.IMAGE_PREFIX }}/logstash-output-loki:${{ steps.weekly-version.outputs.version }}" + "loki": + "env": + "BUILD_TIMEOUT": 60 + "IMAGE_PREFIX": "grafana" + "RELEASE_LIB_REF": "main" + "RELEASE_REPO": "grafana/loki" + "needs": + - "check" + "runs-on": "ubuntu-latest" + "steps": + - "name": "pull release library code" + "uses": "actions/checkout@v4" + "with": + "path": "lib" + "ref": "${{ env.RELEASE_LIB_REF }}" + "repository": "grafana/loki-release" + - "name": "pull code to release" + "uses": "actions/checkout@v4" + "with": + "path": "release" + "repository": "${{ env.RELEASE_REPO }}" + - "name": "setup node" + "uses": "actions/setup-node@v4" + "with": + "node-version": 20 + - "name": "Set up QEMU" + "uses": "docker/setup-qemu-action@v3" + - "name": "set up docker buildx" + "uses": "docker/setup-buildx-action@v3" + - "name": "Login to DockerHub (from vault)" + "uses": "grafana/shared-workflows/actions/dockerhub-login@main" + - "id": "weekly-version" + "name": "Get weekly version" + "run": | + echo "version=$(./tools/image-tag)" >> $GITHUB_OUTPUT + "working-directory": "release" + - "name": "Build and push" + "timeout-minutes": "${{ fromJSON(env.BUILD_TIMEOUT) }}" + "uses": "docker/build-push-action@v6" + "with": + "build-args": "IMAGE_TAG=${{ steps.weekly-version.outputs.version }}" + "context": "release" + "file": "release/cmd/loki/Dockerfile" + "platforms": "linux/amd64,linux/arm64,linux/arm" + "push": true + "tags": "${{ env.IMAGE_PREFIX }}/loki:${{ steps.weekly-version.outputs.version }}" + "loki-canary": + "env": + "BUILD_TIMEOUT": 60 + "IMAGE_PREFIX": "grafana" + "RELEASE_LIB_REF": "main" + "RELEASE_REPO": "grafana/loki" + "needs": + - "check" + "runs-on": "ubuntu-latest" + "steps": + - "name": "pull release library code" + "uses": "actions/checkout@v4" + "with": + "path": "lib" + "ref": "${{ env.RELEASE_LIB_REF }}" + "repository": "grafana/loki-release" + - "name": "pull code to release" + "uses": "actions/checkout@v4" + "with": + "path": "release" + "repository": "${{ env.RELEASE_REPO }}" + - "name": "setup node" + "uses": "actions/setup-node@v4" + "with": + "node-version": 20 + - "name": "Set up QEMU" + "uses": "docker/setup-qemu-action@v3" + - "name": "set up docker buildx" + "uses": "docker/setup-buildx-action@v3" + - "name": "Login to DockerHub (from vault)" + "uses": "grafana/shared-workflows/actions/dockerhub-login@main" + - "id": "weekly-version" + "name": "Get weekly version" + "run": | + echo "version=$(./tools/image-tag)" >> $GITHUB_OUTPUT + "working-directory": "release" + - "name": "Build and push" + "timeout-minutes": "${{ fromJSON(env.BUILD_TIMEOUT) }}" + "uses": "docker/build-push-action@v6" + "with": + "build-args": "IMAGE_TAG=${{ steps.weekly-version.outputs.version }}" + "context": "release" + "file": "release/cmd/loki-canary/Dockerfile" + "platforms": "linux/amd64,linux/arm64,linux/arm" + "push": true + "tags": "${{ env.IMAGE_PREFIX }}/loki-canary:${{ steps.weekly-version.outputs.version }}" + "loki-canary-boringcrypto": + "env": + "BUILD_TIMEOUT": 60 + "IMAGE_PREFIX": "grafana" + "RELEASE_LIB_REF": "main" + "RELEASE_REPO": "grafana/loki" + "needs": + - "check" + "runs-on": "ubuntu-latest" + "steps": + - "name": "pull release library code" + "uses": "actions/checkout@v4" + "with": + "path": "lib" + "ref": "${{ env.RELEASE_LIB_REF }}" + "repository": "grafana/loki-release" + - "name": "pull code to release" + "uses": "actions/checkout@v4" + "with": + "path": "release" + "repository": "${{ env.RELEASE_REPO }}" + - "name": "setup node" + "uses": "actions/setup-node@v4" + "with": + "node-version": 20 + - "name": "Set up QEMU" + "uses": "docker/setup-qemu-action@v3" + - "name": "set up docker buildx" + "uses": "docker/setup-buildx-action@v3" + - "name": "Login to DockerHub (from vault)" + "uses": "grafana/shared-workflows/actions/dockerhub-login@main" + - "id": "weekly-version" + "name": "Get weekly version" + "run": | + echo "version=$(./tools/image-tag)" >> $GITHUB_OUTPUT + "working-directory": "release" + - "name": "Build and push" + "timeout-minutes": "${{ fromJSON(env.BUILD_TIMEOUT) }}" + "uses": "docker/build-push-action@v6" + "with": + "build-args": "IMAGE_TAG=${{ steps.weekly-version.outputs.version }}" + "context": "release" + "file": "release/cmd/loki-canary-boringcrypto/Dockerfile" + "platforms": "linux/amd64,linux/arm64,linux/arm" + "push": true + "tags": "${{ env.IMAGE_PREFIX }}/loki-canary-boringcrypto:${{ steps.weekly-version.outputs.version }}" + "promtail": + "env": + "BUILD_TIMEOUT": 60 + "IMAGE_PREFIX": "grafana" + "RELEASE_LIB_REF": "main" + "RELEASE_REPO": "grafana/loki" + "needs": + - "check" + "runs-on": "ubuntu-latest" + "steps": + - "name": "pull release library code" + "uses": "actions/checkout@v4" + "with": + "path": "lib" + "ref": "${{ env.RELEASE_LIB_REF }}" + "repository": "grafana/loki-release" + - "name": "pull code to release" + "uses": "actions/checkout@v4" + "with": + "path": "release" + "repository": "${{ env.RELEASE_REPO }}" + - "name": "setup node" + "uses": "actions/setup-node@v4" + "with": + "node-version": 20 + - "name": "Set up QEMU" + "uses": "docker/setup-qemu-action@v3" + - "name": "set up docker buildx" + "uses": "docker/setup-buildx-action@v3" + - "name": "Login to DockerHub (from vault)" + "uses": "grafana/shared-workflows/actions/dockerhub-login@main" + - "id": "weekly-version" + "name": "Get weekly version" + "run": | + echo "version=$(./tools/image-tag)" >> $GITHUB_OUTPUT + "working-directory": "release" + - "name": "Build and push" + "timeout-minutes": "${{ fromJSON(env.BUILD_TIMEOUT) }}" + "uses": "docker/build-push-action@v6" + "with": + "build-args": "IMAGE_TAG=${{ steps.weekly-version.outputs.version }}" + "context": "release" + "file": "release/clients/cmd/promtail/Dockerfile" + "platforms": "linux/amd64,linux/arm64,linux/arm" + "push": true + "tags": "${{ env.IMAGE_PREFIX }}/promtail:${{ steps.weekly-version.outputs.version }}" + "querytee": + "env": + "BUILD_TIMEOUT": 60 + "IMAGE_PREFIX": "grafana" + "RELEASE_LIB_REF": "main" + "RELEASE_REPO": "grafana/loki" + "needs": + - "check" + "runs-on": "ubuntu-latest" + "steps": + - "name": "pull release library code" + "uses": "actions/checkout@v4" + "with": + "path": "lib" + "ref": "${{ env.RELEASE_LIB_REF }}" + "repository": "grafana/loki-release" + - "name": "pull code to release" + "uses": "actions/checkout@v4" + "with": + "path": "release" + "repository": "${{ env.RELEASE_REPO }}" + - "name": "setup node" + "uses": "actions/setup-node@v4" + "with": + "node-version": 20 + - "name": "Set up QEMU" + "uses": "docker/setup-qemu-action@v3" + - "name": "set up docker buildx" + "uses": "docker/setup-buildx-action@v3" + - "name": "Login to DockerHub (from vault)" + "uses": "grafana/shared-workflows/actions/dockerhub-login@main" + - "id": "weekly-version" + "name": "Get weekly version" + "run": | + echo "version=$(./tools/image-tag)" >> $GITHUB_OUTPUT + "working-directory": "release" + - "name": "Build and push" + "timeout-minutes": "${{ fromJSON(env.BUILD_TIMEOUT) }}" + "uses": "docker/build-push-action@v6" + "with": + "build-args": "IMAGE_TAG=${{ steps.weekly-version.outputs.version }}" + "context": "release" + "file": "release/cmd/querytee/Dockerfile" + "platforms": "linux/amd64" + "push": true + "tags": "${{ env.IMAGE_PREFIX }}/loki-query-tee:${{ steps.weekly-version.outputs.version }}" +"name": "publish images" +"on": + "push": + "branches": + - "k[0-9]+*" + - "main" +"permissions": + "contents": "write" + "id-token": "write" + "pull-requests": "write" \ No newline at end of file diff --git a/.github/workflows/minor-release-pr.yml b/.github/workflows/minor-release-pr.yml index 74c00fdf23c44..765744e35eb52 100644 --- a/.github/workflows/minor-release-pr.yml +++ b/.github/workflows/minor-release-pr.yml @@ -2,7 +2,7 @@ concurrency: group: "create-release-pr-${{ github.sha }}" env: BUILD_ARTIFACTS_BUCKET: "loki-build-artifacts" - BUILD_TIMEOUT: 40 + BUILD_TIMEOUT: 60 CHANGELOG_PATH: "CHANGELOG.md" DOCKER_USERNAME: "grafana" DRY_RUN: false @@ -198,7 +198,7 @@ jobs: if: "${{ fromJSON(needs.version.outputs.pr_created) }}" name: "Build and export" timeout-minutes: "${{ fromJSON(env.BUILD_TIMEOUT) }}" - uses: "docker/build-push-action@v5" + uses: "docker/build-push-action@v6" with: build-args: "IMAGE_TAG=${{ needs.version.outputs.version }}" context: "release" @@ -260,7 +260,7 @@ jobs: if: "${{ fromJSON(needs.version.outputs.pr_created) }}" name: "Build and export" timeout-minutes: "${{ fromJSON(env.BUILD_TIMEOUT) }}" - uses: "docker/build-push-action@v5" + uses: "docker/build-push-action@v6" with: build-args: "IMAGE_TAG=${{ needs.version.outputs.version }}" context: "release" @@ -322,7 +322,7 @@ jobs: if: "${{ fromJSON(needs.version.outputs.pr_created) }}" name: "Build and export" timeout-minutes: "${{ fromJSON(env.BUILD_TIMEOUT) }}" - uses: "docker/build-push-action@v5" + uses: "docker/build-push-action@v6" with: build-args: "IMAGE_TAG=${{ needs.version.outputs.version }}" context: "release" @@ -386,7 +386,7 @@ jobs: if: "${{ fromJSON(needs.version.outputs.pr_created) }}" name: "Build and export" timeout-minutes: "${{ fromJSON(env.BUILD_TIMEOUT) }}" - uses: "docker/build-push-action@v5" + uses: "docker/build-push-action@v6" with: build-args: "IMAGE_TAG=${{ needs.version.outputs.version }}" context: "release" @@ -448,7 +448,7 @@ jobs: if: "${{ fromJSON(needs.version.outputs.pr_created) }}" name: "Build and export" timeout-minutes: "${{ fromJSON(env.BUILD_TIMEOUT) }}" - uses: "docker/build-push-action@v5" + uses: "docker/build-push-action@v6" with: build-args: "IMAGE_TAG=${{ needs.version.outputs.version }}" context: "release" @@ -512,7 +512,7 @@ jobs: if: "${{ fromJSON(needs.version.outputs.pr_created) }}" name: "Build and export" timeout-minutes: "${{ fromJSON(env.BUILD_TIMEOUT) }}" - uses: "docker/build-push-action@v5" + uses: "docker/build-push-action@v6" with: build-args: "IMAGE_TAG=${{ needs.version.outputs.version }}" context: "release" @@ -576,7 +576,7 @@ jobs: if: "${{ fromJSON(needs.version.outputs.pr_created) }}" name: "Build and export" timeout-minutes: "${{ fromJSON(env.BUILD_TIMEOUT) }}" - uses: "docker/build-push-action@v5" + uses: "docker/build-push-action@v6" with: build-args: "IMAGE_TAG=${{ needs.version.outputs.version }}" context: "release" @@ -640,7 +640,7 @@ jobs: if: "${{ fromJSON(needs.version.outputs.pr_created) }}" name: "Build and export" timeout-minutes: "${{ fromJSON(env.BUILD_TIMEOUT) }}" - uses: "docker/build-push-action@v5" + uses: "docker/build-push-action@v6" with: build-args: "IMAGE_TAG=${{ needs.version.outputs.version }}" context: "release" @@ -704,7 +704,7 @@ jobs: if: "${{ fromJSON(needs.version.outputs.pr_created) }}" name: "Build and export" timeout-minutes: "${{ fromJSON(env.BUILD_TIMEOUT) }}" - uses: "docker/build-push-action@v5" + uses: "docker/build-push-action@v6" with: build-args: "IMAGE_TAG=${{ needs.version.outputs.version }}" context: "release" diff --git a/.github/workflows/patch-release-pr.yml b/.github/workflows/patch-release-pr.yml index 0a62fa1c9228e..96f6d86e27750 100644 --- a/.github/workflows/patch-release-pr.yml +++ b/.github/workflows/patch-release-pr.yml @@ -2,7 +2,7 @@ concurrency: group: "create-release-pr-${{ github.sha }}" env: BUILD_ARTIFACTS_BUCKET: "loki-build-artifacts" - BUILD_TIMEOUT: 40 + BUILD_TIMEOUT: 60 CHANGELOG_PATH: "CHANGELOG.md" DOCKER_USERNAME: "grafana" DRY_RUN: false @@ -198,7 +198,7 @@ jobs: if: "${{ fromJSON(needs.version.outputs.pr_created) }}" name: "Build and export" timeout-minutes: "${{ fromJSON(env.BUILD_TIMEOUT) }}" - uses: "docker/build-push-action@v5" + uses: "docker/build-push-action@v6" with: build-args: "IMAGE_TAG=${{ needs.version.outputs.version }}" context: "release" @@ -260,7 +260,7 @@ jobs: if: "${{ fromJSON(needs.version.outputs.pr_created) }}" name: "Build and export" timeout-minutes: "${{ fromJSON(env.BUILD_TIMEOUT) }}" - uses: "docker/build-push-action@v5" + uses: "docker/build-push-action@v6" with: build-args: "IMAGE_TAG=${{ needs.version.outputs.version }}" context: "release" @@ -322,7 +322,7 @@ jobs: if: "${{ fromJSON(needs.version.outputs.pr_created) }}" name: "Build and export" timeout-minutes: "${{ fromJSON(env.BUILD_TIMEOUT) }}" - uses: "docker/build-push-action@v5" + uses: "docker/build-push-action@v6" with: build-args: "IMAGE_TAG=${{ needs.version.outputs.version }}" context: "release" @@ -386,7 +386,7 @@ jobs: if: "${{ fromJSON(needs.version.outputs.pr_created) }}" name: "Build and export" timeout-minutes: "${{ fromJSON(env.BUILD_TIMEOUT) }}" - uses: "docker/build-push-action@v5" + uses: "docker/build-push-action@v6" with: build-args: "IMAGE_TAG=${{ needs.version.outputs.version }}" context: "release" @@ -448,7 +448,7 @@ jobs: if: "${{ fromJSON(needs.version.outputs.pr_created) }}" name: "Build and export" timeout-minutes: "${{ fromJSON(env.BUILD_TIMEOUT) }}" - uses: "docker/build-push-action@v5" + uses: "docker/build-push-action@v6" with: build-args: "IMAGE_TAG=${{ needs.version.outputs.version }}" context: "release" @@ -512,7 +512,7 @@ jobs: if: "${{ fromJSON(needs.version.outputs.pr_created) }}" name: "Build and export" timeout-minutes: "${{ fromJSON(env.BUILD_TIMEOUT) }}" - uses: "docker/build-push-action@v5" + uses: "docker/build-push-action@v6" with: build-args: "IMAGE_TAG=${{ needs.version.outputs.version }}" context: "release" @@ -576,7 +576,7 @@ jobs: if: "${{ fromJSON(needs.version.outputs.pr_created) }}" name: "Build and export" timeout-minutes: "${{ fromJSON(env.BUILD_TIMEOUT) }}" - uses: "docker/build-push-action@v5" + uses: "docker/build-push-action@v6" with: build-args: "IMAGE_TAG=${{ needs.version.outputs.version }}" context: "release" @@ -640,7 +640,7 @@ jobs: if: "${{ fromJSON(needs.version.outputs.pr_created) }}" name: "Build and export" timeout-minutes: "${{ fromJSON(env.BUILD_TIMEOUT) }}" - uses: "docker/build-push-action@v5" + uses: "docker/build-push-action@v6" with: build-args: "IMAGE_TAG=${{ needs.version.outputs.version }}" context: "release" @@ -704,7 +704,7 @@ jobs: if: "${{ fromJSON(needs.version.outputs.pr_created) }}" name: "Build and export" timeout-minutes: "${{ fromJSON(env.BUILD_TIMEOUT) }}" - uses: "docker/build-push-action@v5" + uses: "docker/build-push-action@v6" with: build-args: "IMAGE_TAG=${{ needs.version.outputs.version }}" context: "release" diff --git a/Makefile b/Makefile index bacc4bb9a429a..476b08b29c1f8 100644 --- a/Makefile +++ b/Makefile @@ -12,7 +12,7 @@ help: .PHONY: fluentd-image, fluentd-push, fluentd-test .PHONY: push-images push-latest save-images load-images promtail-image loki-image build-image build-image-push .PHONY: bigtable-backup, push-bigtable-backup -.PHONY: benchmark-store, drone, check-drone-drift, check-mod +.PHONY: benchmark-store, check-mod .PHONY: migrate migrate-image lint-markdown ragel .PHONY: doc check-doc .PHONY: validate-example-configs generate-example-config-doc check-example-config-doc @@ -36,7 +36,7 @@ DOCKER_IMAGE_DIRS := $(patsubst %/Dockerfile,%,$(DOCKERFILES)) # or you can override this with an environment variable BUILD_IN_CONTAINER ?= true -# ensure you run `make drone` and `make release-workflows` after changing this +# ensure you run `make release-workflows` after changing this BUILD_IMAGE_VERSION ?= 0.33.6 GO_VERSION := 1.22.6 @@ -699,27 +699,6 @@ benchmark-store: go run ./pkg/storage/hack/main.go $(GOTEST) ./pkg/storage/ -bench=. -benchmem -memprofile memprofile.out -cpuprofile cpuprofile.out -trace trace.out -# regenerate drone yaml -drone: -ifeq ($(BUILD_IN_CONTAINER),true) - @mkdir -p $(shell pwd)/.pkg - @mkdir -p $(shell pwd)/.cache - $(SUDO) docker run $(RM) $(TTY) -i \ - -e DRONE_SERVER -e DRONE_TOKEN \ - -v $(shell pwd)/.cache:/go/cache$(MOUNT_FLAGS) \ - -v $(shell pwd)/.pkg:/go/pkg$(MOUNT_FLAGS) \ - -v $(shell pwd):/src/loki$(MOUNT_FLAGS) \ - $(IMAGE_PREFIX)/loki-build-image:$(BUILD_IMAGE_VERSION) $@; -else - drone jsonnet --stream --format -V __build-image-version=$(BUILD_IMAGE_VERSION) --source .drone/drone.jsonnet --target .drone/drone.yml - drone lint .drone/drone.yml --trusted - drone sign --save grafana/loki .drone/drone.yml || echo "You must set DRONE_SERVER and DRONE_TOKEN. These values can be found on your [drone account](http://drone.grafana.net/account) page." -endif - -check-drone-drift: - ./tools/check-drone-drift.sh $(BUILD_IMAGE_VERSION) - - # support go modules check-mod: ifeq ($(BUILD_IN_CONTAINER),true) From 6143a548f129131a82233da9a84bbfdf1157fcb9 Mon Sep 17 00:00:00 2001 From: Christian Haudum Date: Thu, 26 Sep 2024 08:35:36 +0200 Subject: [PATCH 09/11] chore: Disable bloom filtering for bounded sharding strategy (#14272) This commit temporarily disables bloom filtering chunk refs in the `GetShards()` call on the index gateway when using the `bounded` sharding strategy. This will result in chunks only being filtered once by the bloom gateway, reducing the amount of checks that need to be done on the bloom filters for a single request.Signed-off-by: Christian Haudum --- pkg/indexgateway/gateway.go | 28 +++++++++++++++------------- 1 file changed, 15 insertions(+), 13 deletions(-) diff --git a/pkg/indexgateway/gateway.go b/pkg/indexgateway/gateway.go index 745b114c08ac0..92d476d496673 100644 --- a/pkg/indexgateway/gateway.go +++ b/pkg/indexgateway/gateway.go @@ -413,7 +413,7 @@ func (g *Gateway) GetShards(request *logproto.ShardsRequest, server logproto.Ind return g.boundedShards(ctx, request, server, instanceID, p, forSeries) } -// boundedShards handles bounded shard requests, optionally using blooms and/or returning precomputed chunks. +// boundedShards handles bounded shard requests, optionally returning precomputed chunks. func (g *Gateway) boundedShards( ctx context.Context, req *logproto.ShardsRequest, @@ -466,18 +466,20 @@ func (g *Gateway) boundedShards( // 2) filter via blooms if enabled filters := v1.ExtractTestableLabelMatchers(p.Plan().AST) - if g.bloomQuerier != nil && len(filters) > 0 { - xs, err := g.bloomQuerier.FilterChunkRefs(ctx, instanceID, req.From, req.Through, refs, p.Plan()) - if err != nil { - level.Error(logger).Log("msg", "failed to filter chunk refs", "err", err) - } else { - filtered = xs - } - sp.LogKV( - "stage", "queried bloom gateway", - "err", err, - ) - } + // NOTE(chaudum): Temporarily disable bloom filtering of chunk refs, + // as this doubles the load on bloom gateways. + // if g.bloomQuerier != nil && len(filters) > 0 { + // xs, err := g.bloomQuerier.FilterChunkRefs(ctx, instanceID, req.From, req.Through, refs, p.Plan()) + // if err != nil { + // level.Error(logger).Log("msg", "failed to filter chunk refs", "err", err) + // } else { + // filtered = xs + // } + // sp.LogKV( + // "stage", "queried bloom gateway", + // "err", err, + // ) + // } g.metrics.preFilterChunks.WithLabelValues(routeShards).Observe(float64(ct)) g.metrics.postFilterChunks.WithLabelValues(routeShards).Observe(float64(len(filtered))) From fac3177814b8d2914eb3af618d571104eba18934 Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Thu, 26 Sep 2024 09:51:59 +0200 Subject: [PATCH 10/11] feat(jsonnet): Allow to name prefix zoned ingesters (#14260) --- production/ksonnet/loki/multi-zone.libsonnet | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/production/ksonnet/loki/multi-zone.libsonnet b/production/ksonnet/loki/multi-zone.libsonnet index 606f70099d0f8..a3d48f21a96d8 100644 --- a/production/ksonnet/loki/multi-zone.libsonnet +++ b/production/ksonnet/loki/multi-zone.libsonnet @@ -100,8 +100,8 @@ local rolloutOperator = import 'rollout-operator.libsonnet'; }, )), - newIngesterZoneStatefulSet(zone, container):: - local name = '%(prefix)s-%(zone)s' % { prefix: $._config.multi_zone_ingester_name_prefix, zone: zone }; + newIngesterZoneStatefulSet(zone, container, name_prefix=''):: + local name = '%(prefix)s-%(zone)s' % { prefix: if name_prefix == '' then $._config.multi_zone_ingester_name_prefix else name_prefix, zone: zone }; self.newIngesterStatefulSet(name, container, with_anti_affinity=false) + statefulSet.mixin.metadata.withLabels({ 'rollout-group': 'ingester' }) + From f80d68a1edbd85a605be882eb0104b169343cf00 Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Thu, 26 Sep 2024 10:14:32 +0200 Subject: [PATCH 11/11] feat(distributors): Use a pool of worker to push to ingesters. (#14245) --- docs/sources/shared/configuration.md | 4 ++ pkg/distributor/distributor.go | 70 +++++++++++++++++++++++----- 2 files changed, 62 insertions(+), 12 deletions(-) diff --git a/docs/sources/shared/configuration.md b/docs/sources/shared/configuration.md index 08acf80bfcc22..15426e54d088f 100644 --- a/docs/sources/shared/configuration.md +++ b/docs/sources/shared/configuration.md @@ -2248,6 +2248,10 @@ ring: # CLI flag: -distributor.ring.instance-interface-names [instance_interface_names: | default = []] +# Number of workers to push batches to ingesters. +# CLI flag: -distributor.push-worker-count +[push_worker_count: | default = 256] + rate_store: # The max number of concurrent requests to make to ingester stream apis # CLI flag: -distributor.rate-store.max-request-parallelism diff --git a/pkg/distributor/distributor.go b/pkg/distributor/distributor.go index 3ad586f3e596f..476bad507ea0b 100644 --- a/pkg/distributor/distributor.go +++ b/pkg/distributor/distributor.go @@ -10,6 +10,7 @@ import ( "sort" "strconv" "strings" + "sync" "time" "unicode" "unsafe" @@ -79,6 +80,7 @@ var allowedLabelsForLevel = map[string]struct{}{ type Config struct { // Distributors ring DistributorRing RingConfig `yaml:"ring,omitempty"` + PushWorkerCount int `yaml:"push_worker_count"` // For testing. factory ring_client.PoolFactory `yaml:"-"` @@ -102,7 +104,7 @@ func (cfg *Config) RegisterFlags(fs *flag.FlagSet) { cfg.DistributorRing.RegisterFlags(fs) cfg.RateStore.RegisterFlagsWithPrefix("distributor.rate-store", fs) cfg.WriteFailuresLogging.RegisterFlagsWithPrefix("distributor.write-failures-logging", fs) - + fs.IntVar(&cfg.PushWorkerCount, "distributor.push-worker-count", 256, "Number of workers to push batches to ingesters.") fs.BoolVar(&cfg.KafkaEnabled, "distributor.kafka-writes-enabled", false, "Enable writes to Kafka during Push requests.") fs.BoolVar(&cfg.IngesterEnabled, "distributor.ingester-writes-enabled", true, "Enable writes to Ingesters during Push requests. Defaults to true.") } @@ -166,7 +168,9 @@ type Distributor struct { replicationFactor prometheus.Gauge streamShardCount prometheus.Counter - usageTracker push.UsageTracker + usageTracker push.UsageTracker + ingesterTasks chan pushIngesterTask + ingesterTaskWg sync.WaitGroup // kafka kafkaWriter KafkaProducer @@ -253,6 +257,7 @@ func New( rateLimitStrat: rateLimitStrat, tee: tee, usageTracker: usageTracker, + ingesterTasks: make(chan pushIngesterTask), ingesterAppends: promauto.With(registerer).NewCounterVec(prometheus.CounterOpts{ Namespace: constants.Loki, Name: "distributor_ingester_appends_total", @@ -354,6 +359,15 @@ func (d *Distributor) starting(ctx context.Context) error { } func (d *Distributor) running(ctx context.Context) error { + ctx, cancel := context.WithCancel(ctx) + defer func() { + cancel() + d.ingesterTaskWg.Wait() + }() + d.ingesterTaskWg.Add(d.cfg.PushWorkerCount) + for i := 0; i < d.cfg.PushWorkerCount; i++ { + go d.pushIngesterWorker(ctx) + } select { case <-ctx.Done(): return nil @@ -630,15 +644,26 @@ func (d *Distributor) Push(ctx context.Context, req *logproto.PushRequest) (*log } for ingester, streams := range streamsByIngester { - go func(ingester ring.InstanceDesc, samples []*streamTracker) { + func(ingester ring.InstanceDesc, samples []*streamTracker) { // Use a background context to make sure all ingesters get samples even if we return early localCtx, cancel := context.WithTimeout(context.Background(), d.clientCfg.RemoteTimeout) - defer cancel() localCtx = user.InjectOrgID(localCtx, tenantID) if sp := opentracing.SpanFromContext(ctx); sp != nil { localCtx = opentracing.ContextWithSpan(localCtx, sp) } - d.sendStreams(localCtx, ingester, samples, &tracker) + select { + case <-ctx.Done(): + cancel() + return + case d.ingesterTasks <- pushIngesterTask{ + ingester: ingester, + streamTracker: samples, + pushTracker: &tracker, + ctx: localCtx, + cancel: cancel, + }: + return + } }(ingesterDescs[ingester], streams) } } @@ -830,9 +855,30 @@ func (d *Distributor) truncateLines(vContext validationContext, stream *logproto validation.MutatedBytes.WithLabelValues(validation.LineTooLong, vContext.userID).Add(float64(truncatedBytes)) } +type pushIngesterTask struct { + streamTracker []*streamTracker + pushTracker *pushTracker + ingester ring.InstanceDesc + ctx context.Context + cancel context.CancelFunc +} + +func (d *Distributor) pushIngesterWorker(ctx context.Context) { + defer d.ingesterTaskWg.Done() + for { + select { + case <-ctx.Done(): + return + case task := <-d.ingesterTasks: + d.sendStreams(task) + } + } +} + // TODO taken from Cortex, see if we can refactor out an usable interface. -func (d *Distributor) sendStreams(ctx context.Context, ingester ring.InstanceDesc, streamTrackers []*streamTracker, pushTracker *pushTracker) { - err := d.sendStreamsErr(ctx, ingester, streamTrackers) +func (d *Distributor) sendStreams(task pushIngesterTask) { + defer task.cancel() + err := d.sendStreamsErr(task.ctx, task.ingester, task.streamTracker) // If we succeed, decrement each stream's pending count by one. // If we reach the required number of successful puts on this stream, then @@ -843,17 +889,17 @@ func (d *Distributor) sendStreams(ctx context.Context, ingester ring.InstanceDes // // The use of atomic increments here guarantees only a single sendStreams // goroutine will write to either channel. - for i := range streamTrackers { + for i := range task.streamTracker { if err != nil { - if streamTrackers[i].failed.Inc() <= int32(streamTrackers[i].maxFailures) { + if task.streamTracker[i].failed.Inc() <= int32(task.streamTracker[i].maxFailures) { continue } - pushTracker.doneWithResult(err) + task.pushTracker.doneWithResult(err) } else { - if streamTrackers[i].succeeded.Inc() != int32(streamTrackers[i].minSuccess) { + if task.streamTracker[i].succeeded.Inc() != int32(task.streamTracker[i].minSuccess) { continue } - pushTracker.doneWithResult(nil) + task.pushTracker.doneWithResult(nil) } } }