From f917cf37a2718734996be02b9bb5816c9a31aea5 Mon Sep 17 00:00:00 2001 From: Periklis Tsirakidis Date: Wed, 2 Oct 2024 10:30:10 +0200 Subject: [PATCH 01/10] feat!(operator): Update Loki operand to v3.2.0 (#14331) --- .../loki-operator.clusterserviceversion.yaml | 6 ++--- .../loki-operator.clusterserviceversion.yaml | 6 ++--- .../loki-operator.clusterserviceversion.yaml | 6 ++--- .../manager_related_image_patch.yaml | 2 +- .../manager_related_image_patch.yaml | 2 +- .../manager_related_image_patch.yaml | 2 +- .../manager_related_image_patch.yaml | 2 +- operator/docs/operator/compatibility.md | 1 + operator/hack/addons_dev.yaml | 4 +-- operator/hack/addons_ocp.yaml | 4 +-- .../grafana-dashboard-lokistack-reads.json | 26 +++++++++---------- .../grafana-dashboard-lokistack-writes.json | 10 +++---- operator/internal/manifests/var.go | 2 +- operator/jsonnet/jsonnetfile.json | 2 +- operator/jsonnet/jsonnetfile.lock.json | 4 +-- 15 files changed, 40 insertions(+), 39 deletions(-) diff --git a/operator/bundle/community-openshift/manifests/loki-operator.clusterserviceversion.yaml b/operator/bundle/community-openshift/manifests/loki-operator.clusterserviceversion.yaml index f517b672cf914..e088428b959ce 100644 --- a/operator/bundle/community-openshift/manifests/loki-operator.clusterserviceversion.yaml +++ b/operator/bundle/community-openshift/manifests/loki-operator.clusterserviceversion.yaml @@ -150,7 +150,7 @@ metadata: categories: OpenShift Optional, Logging & Tracing certified: "false" containerImage: docker.io/grafana/loki-operator:0.6.2 - createdAt: "2024-09-30T09:33:40Z" + createdAt: "2024-10-01T11:39:45Z" description: The Community Loki Operator provides Kubernetes native deployment and management of Loki and related logging components. features.operators.openshift.io/disconnected: "true" @@ -1846,7 +1846,7 @@ spec: - /manager env: - name: RELATED_IMAGE_LOKI - value: docker.io/grafana/loki:3.1.0 + value: docker.io/grafana/loki:3.2.0 - name: RELATED_IMAGE_GATEWAY value: quay.io/observatorium/api:latest - name: RELATED_IMAGE_OPA @@ -1970,7 +1970,7 @@ spec: provider: name: Grafana Loki SIG Operator relatedImages: - - image: docker.io/grafana/loki:3.1.0 + - image: docker.io/grafana/loki:3.2.0 name: loki - image: quay.io/observatorium/api:latest name: gateway diff --git a/operator/bundle/community/manifests/loki-operator.clusterserviceversion.yaml b/operator/bundle/community/manifests/loki-operator.clusterserviceversion.yaml index 6ef7166fd1c6f..f8137b4c9f3d4 100644 --- a/operator/bundle/community/manifests/loki-operator.clusterserviceversion.yaml +++ b/operator/bundle/community/manifests/loki-operator.clusterserviceversion.yaml @@ -150,7 +150,7 @@ metadata: categories: OpenShift Optional, Logging & Tracing certified: "false" containerImage: docker.io/grafana/loki-operator:0.6.2 - createdAt: "2024-09-30T09:33:38Z" + createdAt: "2024-10-01T11:39:42Z" description: The Community Loki Operator provides Kubernetes native deployment and management of Loki and related logging components. operators.operatorframework.io/builder: operator-sdk-unknown @@ -1826,7 +1826,7 @@ spec: - /manager env: - name: RELATED_IMAGE_LOKI - value: docker.io/grafana/loki:3.1.1 + value: docker.io/grafana/loki:3.2.0 - name: RELATED_IMAGE_GATEWAY value: quay.io/observatorium/api:latest - name: RELATED_IMAGE_OPA @@ -1938,7 +1938,7 @@ spec: provider: name: Grafana Loki SIG Operator relatedImages: - - image: docker.io/grafana/loki:3.1.1 + - image: docker.io/grafana/loki:3.2.0 name: loki - image: quay.io/observatorium/api:latest name: gateway diff --git a/operator/bundle/openshift/manifests/loki-operator.clusterserviceversion.yaml b/operator/bundle/openshift/manifests/loki-operator.clusterserviceversion.yaml index b4108e243e95a..d57ae2d5af902 100644 --- a/operator/bundle/openshift/manifests/loki-operator.clusterserviceversion.yaml +++ b/operator/bundle/openshift/manifests/loki-operator.clusterserviceversion.yaml @@ -150,7 +150,7 @@ metadata: categories: OpenShift Optional, Logging & Tracing certified: "false" containerImage: quay.io/openshift-logging/loki-operator:0.1.0 - createdAt: "2024-09-30T09:33:41Z" + createdAt: "2024-10-01T11:39:47Z" description: | The Loki Operator for OCP provides a means for configuring and managing a Loki stack for cluster logging. ## Prerequisites and Requirements @@ -1831,7 +1831,7 @@ spec: - /manager env: - name: RELATED_IMAGE_LOKI - value: quay.io/openshift-logging/loki:v3.1.1 + value: quay.io/openshift-logging/loki:v3.2.0 - name: RELATED_IMAGE_GATEWAY value: quay.io/observatorium/api:latest - name: RELATED_IMAGE_OPA @@ -1955,7 +1955,7 @@ spec: provider: name: Red Hat relatedImages: - - image: quay.io/openshift-logging/loki:v3.1.1 + - image: quay.io/openshift-logging/loki:v3.2.0 name: loki - image: quay.io/observatorium/api:latest name: gateway diff --git a/operator/config/overlays/community-openshift/manager_related_image_patch.yaml b/operator/config/overlays/community-openshift/manager_related_image_patch.yaml index 5e454e400cc87..538e6658383b3 100644 --- a/operator/config/overlays/community-openshift/manager_related_image_patch.yaml +++ b/operator/config/overlays/community-openshift/manager_related_image_patch.yaml @@ -9,7 +9,7 @@ spec: - name: manager env: - name: RELATED_IMAGE_LOKI - value: docker.io/grafana/loki:3.1.0 + value: docker.io/grafana/loki:3.2.0 - name: RELATED_IMAGE_GATEWAY value: quay.io/observatorium/api:latest - name: RELATED_IMAGE_OPA diff --git a/operator/config/overlays/community/manager_related_image_patch.yaml b/operator/config/overlays/community/manager_related_image_patch.yaml index 52681b95a4b1c..538e6658383b3 100644 --- a/operator/config/overlays/community/manager_related_image_patch.yaml +++ b/operator/config/overlays/community/manager_related_image_patch.yaml @@ -9,7 +9,7 @@ spec: - name: manager env: - name: RELATED_IMAGE_LOKI - value: docker.io/grafana/loki:3.1.1 + value: docker.io/grafana/loki:3.2.0 - name: RELATED_IMAGE_GATEWAY value: quay.io/observatorium/api:latest - name: RELATED_IMAGE_OPA diff --git a/operator/config/overlays/development/manager_related_image_patch.yaml b/operator/config/overlays/development/manager_related_image_patch.yaml index 21aa79f99a7d7..e4af25f50e449 100644 --- a/operator/config/overlays/development/manager_related_image_patch.yaml +++ b/operator/config/overlays/development/manager_related_image_patch.yaml @@ -9,6 +9,6 @@ spec: - name: manager env: - name: RELATED_IMAGE_LOKI - value: docker.io/grafana/loki:3.1.1 + value: docker.io/grafana/loki:3.2.0 - name: RELATED_IMAGE_GATEWAY value: quay.io/observatorium/api:latest diff --git a/operator/config/overlays/openshift/manager_related_image_patch.yaml b/operator/config/overlays/openshift/manager_related_image_patch.yaml index 3bb2b65cf4617..f7ae10c2aaf3c 100644 --- a/operator/config/overlays/openshift/manager_related_image_patch.yaml +++ b/operator/config/overlays/openshift/manager_related_image_patch.yaml @@ -9,7 +9,7 @@ spec: - name: manager env: - name: RELATED_IMAGE_LOKI - value: quay.io/openshift-logging/loki:v3.1.1 + value: quay.io/openshift-logging/loki:v3.2.0 - name: RELATED_IMAGE_GATEWAY value: quay.io/observatorium/api:latest - name: RELATED_IMAGE_OPA diff --git a/operator/docs/operator/compatibility.md b/operator/docs/operator/compatibility.md index 382e7e6a7673f..5f573dc89906e 100644 --- a/operator/docs/operator/compatibility.md +++ b/operator/docs/operator/compatibility.md @@ -29,3 +29,4 @@ The versions of Loki compatible to be run with the Loki Operator are: * v3.1.0 * v3.1.1 +* v3.2.0 diff --git a/operator/hack/addons_dev.yaml b/operator/hack/addons_dev.yaml index 320743bc37325..9c67cfc65b6f1 100644 --- a/operator/hack/addons_dev.yaml +++ b/operator/hack/addons_dev.yaml @@ -29,7 +29,7 @@ spec: spec: containers: - name: logcli - image: docker.io/grafana/logcli:3.1.1-amd64 + image: docker.io/grafana/logcli:3.2.0-amd64 imagePullPolicy: IfNotPresent command: - /bin/sh @@ -73,7 +73,7 @@ spec: spec: containers: - name: promtail - image: docker.io/grafana/promtail:3.1.1 + image: docker.io/grafana/promtail:3.2.0 args: - -config.file=/etc/promtail/promtail.yaml - -log.level=info diff --git a/operator/hack/addons_ocp.yaml b/operator/hack/addons_ocp.yaml index 6e49cbd847c8a..0a00d83044624 100644 --- a/operator/hack/addons_ocp.yaml +++ b/operator/hack/addons_ocp.yaml @@ -29,7 +29,7 @@ spec: spec: containers: - name: logcli - image: docker.io/grafana/logcli:3.1.1-amd64 + image: docker.io/grafana/logcli:3.2.0-amd64 imagePullPolicy: IfNotPresent command: - /bin/sh @@ -70,7 +70,7 @@ spec: spec: containers: - name: promtail - image: docker.io/grafana/promtail:3.1.1 + image: docker.io/grafana/promtail:3.2.0 args: - -config.file=/etc/promtail/promtail.yaml - -log.level=info diff --git a/operator/internal/manifests/openshift/internal/dashboards/static/grafana-dashboard-lokistack-reads.json b/operator/internal/manifests/openshift/internal/dashboards/static/grafana-dashboard-lokistack-reads.json index ca4cca0f11058..ec1656e27bd7d 100644 --- a/operator/internal/manifests/openshift/internal/dashboards/static/grafana-dashboard-lokistack-reads.json +++ b/operator/internal/manifests/openshift/internal/dashboards/static/grafana-dashboard-lokistack-reads.json @@ -213,7 +213,7 @@ "stack": true, "targets": [ { - "expr": "sum by (status) (\n label_replace(label_replace(rate(loki_request_duration_seconds_count{namespace=\"$namespace\",job=~\".+-query-frontend-http\", route=~\"(api_prom_rules|api_prom_rules_namespace_groupname|api_v1_rules|loki_api_v1_delete|loki_api_v1_detected_labels|loki_api_v1_index_stats|loki_api_v1_index_volume|loki_api_v1_index_volume_range|loki_api_v1_label_name_values|loki_api_v1_label_values|loki_api_v1_labels|loki_api_v1_patterns|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_series|otlp_v1_logs|prometheus_api_v1_rules)\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(loki_request_duration_seconds_count{namespace=\"$namespace\",job=~\".+-query-frontend-http\", route=~\"(api_prom_rules|api_prom_rules_namespace_groupname|api_v1_rules|loki_api_v1_delete|loki_api_v1_detected_labels|loki_api_v1_index_stats|loki_api_v1_index_volume|loki_api_v1_index_volume_range|loki_api_v1_label_name_values|loki_api_v1_label_values|loki_api_v1_labels|loki_api_v1_patterns|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_series|prometheus_api_v1_rules)\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{status}}", @@ -263,7 +263,7 @@ "span": 4, "targets": [ { - "expr": "histogram_quantile(0.99, sum by (le,route) (namespace_job_route:loki_request_duration_seconds_bucket:sum_rate{namespace=\"$namespace\", job=~\".+-query-frontend-http\", route=~\"(api_prom_rules|api_prom_rules_namespace_groupname|api_v1_rules|loki_api_v1_delete|loki_api_v1_detected_labels|loki_api_v1_index_stats|loki_api_v1_index_volume|loki_api_v1_index_volume_range|loki_api_v1_label_name_values|loki_api_v1_label_values|loki_api_v1_labels|loki_api_v1_patterns|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_series|otlp_v1_logs|prometheus_api_v1_rules)\"})) * 1e3", + "expr": "histogram_quantile(0.99, sum by (le,route) (namespace_job_route:loki_request_duration_seconds_bucket:sum_rate{namespace=\"$namespace\", job=~\".+-query-frontend-http\", route=~\"(api_prom_rules|api_prom_rules_namespace_groupname|api_v1_rules|loki_api_v1_delete|loki_api_v1_detected_labels|loki_api_v1_index_stats|loki_api_v1_index_volume|loki_api_v1_index_volume_range|loki_api_v1_label_name_values|loki_api_v1_label_values|loki_api_v1_labels|loki_api_v1_patterns|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_series|prometheus_api_v1_rules)\"})) * 1e3", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{ route }} 99th Percentile", @@ -271,7 +271,7 @@ "step": 10 }, { - "expr": "histogram_quantile(0.50, sum by (le,route) (namespace_job_route:loki_request_duration_seconds_bucket:sum_rate{namespace=\"$namespace\", job=~\".+-query-frontend-http\", route=~\"(api_prom_rules|api_prom_rules_namespace_groupname|api_v1_rules|loki_api_v1_delete|loki_api_v1_detected_labels|loki_api_v1_index_stats|loki_api_v1_index_volume|loki_api_v1_index_volume_range|loki_api_v1_label_name_values|loki_api_v1_label_values|loki_api_v1_labels|loki_api_v1_patterns|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_series|otlp_v1_logs|prometheus_api_v1_rules)\"})) * 1e3", + "expr": "histogram_quantile(0.50, sum by (le,route) (namespace_job_route:loki_request_duration_seconds_bucket:sum_rate{namespace=\"$namespace\", job=~\".+-query-frontend-http\", route=~\"(api_prom_rules|api_prom_rules_namespace_groupname|api_v1_rules|loki_api_v1_delete|loki_api_v1_detected_labels|loki_api_v1_index_stats|loki_api_v1_index_volume|loki_api_v1_index_volume_range|loki_api_v1_label_name_values|loki_api_v1_label_values|loki_api_v1_labels|loki_api_v1_patterns|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_series|prometheus_api_v1_rules)\"})) * 1e3", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{ route }} 50th Percentile", @@ -279,7 +279,7 @@ "step": 10 }, { - "expr": "1e3 * sum(namespace_job_route:loki_request_duration_seconds_sum:sum_rate{namespace=\"$namespace\", job=~\".+-query-frontend-http\", route=~\"(api_prom_rules|api_prom_rules_namespace_groupname|api_v1_rules|loki_api_v1_delete|loki_api_v1_detected_labels|loki_api_v1_index_stats|loki_api_v1_index_volume|loki_api_v1_index_volume_range|loki_api_v1_label_name_values|loki_api_v1_label_values|loki_api_v1_labels|loki_api_v1_patterns|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_series|otlp_v1_logs|prometheus_api_v1_rules)\"}) by (route) / sum(namespace_job_route:loki_request_duration_seconds_count:sum_rate{namespace=\"$namespace\", job=~\".+-query-frontend-http\", route=~\"(api_prom_rules|api_prom_rules_namespace_groupname|api_v1_rules|loki_api_v1_delete|loki_api_v1_detected_labels|loki_api_v1_index_stats|loki_api_v1_index_volume|loki_api_v1_index_volume_range|loki_api_v1_label_name_values|loki_api_v1_label_values|loki_api_v1_labels|loki_api_v1_patterns|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_series|otlp_v1_logs|prometheus_api_v1_rules)\"}) by (route) ", + "expr": "1e3 * sum(namespace_job_route:loki_request_duration_seconds_sum:sum_rate{namespace=\"$namespace\", job=~\".+-query-frontend-http\", route=~\"(api_prom_rules|api_prom_rules_namespace_groupname|api_v1_rules|loki_api_v1_delete|loki_api_v1_detected_labels|loki_api_v1_index_stats|loki_api_v1_index_volume|loki_api_v1_index_volume_range|loki_api_v1_label_name_values|loki_api_v1_label_values|loki_api_v1_labels|loki_api_v1_patterns|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_series|prometheus_api_v1_rules)\"}) by (route) / sum(namespace_job_route:loki_request_duration_seconds_count:sum_rate{namespace=\"$namespace\", job=~\".+-query-frontend-http\", route=~\"(api_prom_rules|api_prom_rules_namespace_groupname|api_v1_rules|loki_api_v1_delete|loki_api_v1_detected_labels|loki_api_v1_index_stats|loki_api_v1_index_volume|loki_api_v1_index_volume_range|loki_api_v1_label_name_values|loki_api_v1_label_values|loki_api_v1_labels|loki_api_v1_patterns|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_series|prometheus_api_v1_rules)\"}) by (route) ", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{ route }} Average", @@ -347,7 +347,7 @@ "span": 4, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(loki_request_duration_seconds_bucket{namespace=\"$namespace\", job=~\".+-query-frontend-http\", route=~\"(api_prom_rules|api_prom_rules_namespace_groupname|api_v1_rules|loki_api_v1_delete|loki_api_v1_detected_labels|loki_api_v1_index_stats|loki_api_v1_index_volume|loki_api_v1_index_volume_range|loki_api_v1_label_name_values|loki_api_v1_label_values|loki_api_v1_labels|loki_api_v1_patterns|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_series|otlp_v1_logs|prometheus_api_v1_rules)\"}[$__rate_interval])) by (le,pod)) * 1e3", + "expr": "histogram_quantile(0.99, sum(rate(loki_request_duration_seconds_bucket{namespace=\"$namespace\", job=~\".+-query-frontend-http\", route=~\"(api_prom_rules|api_prom_rules_namespace_groupname|api_v1_rules|loki_api_v1_delete|loki_api_v1_detected_labels|loki_api_v1_index_stats|loki_api_v1_index_volume|loki_api_v1_index_volume_range|loki_api_v1_label_name_values|loki_api_v1_label_values|loki_api_v1_labels|loki_api_v1_patterns|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_series|prometheus_api_v1_rules)\"}[$__rate_interval])) by (le,pod)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -558,7 +558,7 @@ "stack": true, "targets": [ { - "expr": "sum by (status) (\n label_replace(label_replace(rate(loki_request_duration_seconds_count{namespace=\"$namespace\",job=~\".+-querier-http\", route=~\"(api_prom_rules|api_prom_rules_namespace_groupname|api_v1_rules|loki_api_v1_delete|loki_api_v1_detected_labels|loki_api_v1_index_stats|loki_api_v1_index_volume|loki_api_v1_index_volume_range|loki_api_v1_label_name_values|loki_api_v1_label_values|loki_api_v1_labels|loki_api_v1_patterns|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_series|otlp_v1_logs|prometheus_api_v1_rules)\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(loki_request_duration_seconds_count{namespace=\"$namespace\",job=~\".+-querier-http\", route=~\"(api_prom_rules|api_prom_rules_namespace_groupname|api_v1_rules|loki_api_v1_delete|loki_api_v1_detected_labels|loki_api_v1_index_stats|loki_api_v1_index_volume|loki_api_v1_index_volume_range|loki_api_v1_label_name_values|loki_api_v1_label_values|loki_api_v1_labels|loki_api_v1_patterns|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_series|prometheus_api_v1_rules)\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{status}}", @@ -608,7 +608,7 @@ "span": 4, "targets": [ { - "expr": "histogram_quantile(0.99, sum by (le,route) (namespace_job_route:loki_request_duration_seconds_bucket:sum_rate{namespace=\"$namespace\", job=~\".+-querier-http\", route=~\"(api_prom_rules|api_prom_rules_namespace_groupname|api_v1_rules|loki_api_v1_delete|loki_api_v1_detected_labels|loki_api_v1_index_stats|loki_api_v1_index_volume|loki_api_v1_index_volume_range|loki_api_v1_label_name_values|loki_api_v1_label_values|loki_api_v1_labels|loki_api_v1_patterns|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_series|otlp_v1_logs|prometheus_api_v1_rules)\"})) * 1e3", + "expr": "histogram_quantile(0.99, sum by (le,route) (namespace_job_route:loki_request_duration_seconds_bucket:sum_rate{namespace=\"$namespace\", job=~\".+-querier-http\", route=~\"(api_prom_rules|api_prom_rules_namespace_groupname|api_v1_rules|loki_api_v1_delete|loki_api_v1_detected_labels|loki_api_v1_index_stats|loki_api_v1_index_volume|loki_api_v1_index_volume_range|loki_api_v1_label_name_values|loki_api_v1_label_values|loki_api_v1_labels|loki_api_v1_patterns|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_series|prometheus_api_v1_rules)\"})) * 1e3", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{ route }} 99th Percentile", @@ -616,7 +616,7 @@ "step": 10 }, { - "expr": "histogram_quantile(0.50, sum by (le,route) (namespace_job_route:loki_request_duration_seconds_bucket:sum_rate{namespace=\"$namespace\", job=~\".+-querier-http\", route=~\"(api_prom_rules|api_prom_rules_namespace_groupname|api_v1_rules|loki_api_v1_delete|loki_api_v1_detected_labels|loki_api_v1_index_stats|loki_api_v1_index_volume|loki_api_v1_index_volume_range|loki_api_v1_label_name_values|loki_api_v1_label_values|loki_api_v1_labels|loki_api_v1_patterns|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_series|otlp_v1_logs|prometheus_api_v1_rules)\"})) * 1e3", + "expr": "histogram_quantile(0.50, sum by (le,route) (namespace_job_route:loki_request_duration_seconds_bucket:sum_rate{namespace=\"$namespace\", job=~\".+-querier-http\", route=~\"(api_prom_rules|api_prom_rules_namespace_groupname|api_v1_rules|loki_api_v1_delete|loki_api_v1_detected_labels|loki_api_v1_index_stats|loki_api_v1_index_volume|loki_api_v1_index_volume_range|loki_api_v1_label_name_values|loki_api_v1_label_values|loki_api_v1_labels|loki_api_v1_patterns|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_series|prometheus_api_v1_rules)\"})) * 1e3", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{ route }} 50th Percentile", @@ -624,7 +624,7 @@ "step": 10 }, { - "expr": "1e3 * sum(namespace_job_route:loki_request_duration_seconds_sum:sum_rate{namespace=\"$namespace\", job=~\".+-querier-http\", route=~\"(api_prom_rules|api_prom_rules_namespace_groupname|api_v1_rules|loki_api_v1_delete|loki_api_v1_detected_labels|loki_api_v1_index_stats|loki_api_v1_index_volume|loki_api_v1_index_volume_range|loki_api_v1_label_name_values|loki_api_v1_label_values|loki_api_v1_labels|loki_api_v1_patterns|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_series|otlp_v1_logs|prometheus_api_v1_rules)\"}) by (route) / sum(namespace_job_route:loki_request_duration_seconds_count:sum_rate{namespace=\"$namespace\", job=~\".+-querier-http\", route=~\"(api_prom_rules|api_prom_rules_namespace_groupname|api_v1_rules|loki_api_v1_delete|loki_api_v1_detected_labels|loki_api_v1_index_stats|loki_api_v1_index_volume|loki_api_v1_index_volume_range|loki_api_v1_label_name_values|loki_api_v1_label_values|loki_api_v1_labels|loki_api_v1_patterns|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_series|otlp_v1_logs|prometheus_api_v1_rules)\"}) by (route) ", + "expr": "1e3 * sum(namespace_job_route:loki_request_duration_seconds_sum:sum_rate{namespace=\"$namespace\", job=~\".+-querier-http\", route=~\"(api_prom_rules|api_prom_rules_namespace_groupname|api_v1_rules|loki_api_v1_delete|loki_api_v1_detected_labels|loki_api_v1_index_stats|loki_api_v1_index_volume|loki_api_v1_index_volume_range|loki_api_v1_label_name_values|loki_api_v1_label_values|loki_api_v1_labels|loki_api_v1_patterns|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_series|prometheus_api_v1_rules)\"}) by (route) / sum(namespace_job_route:loki_request_duration_seconds_count:sum_rate{namespace=\"$namespace\", job=~\".+-querier-http\", route=~\"(api_prom_rules|api_prom_rules_namespace_groupname|api_v1_rules|loki_api_v1_delete|loki_api_v1_detected_labels|loki_api_v1_index_stats|loki_api_v1_index_volume|loki_api_v1_index_volume_range|loki_api_v1_label_name_values|loki_api_v1_label_values|loki_api_v1_labels|loki_api_v1_patterns|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_series|prometheus_api_v1_rules)\"}) by (route) ", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{ route }} Average", @@ -692,7 +692,7 @@ "span": 4, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(loki_request_duration_seconds_bucket{namespace=\"$namespace\", job=~\".+-querier-http\", route=~\"(api_prom_rules|api_prom_rules_namespace_groupname|api_v1_rules|loki_api_v1_delete|loki_api_v1_detected_labels|loki_api_v1_index_stats|loki_api_v1_index_volume|loki_api_v1_index_volume_range|loki_api_v1_label_name_values|loki_api_v1_label_values|loki_api_v1_labels|loki_api_v1_patterns|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_series|otlp_v1_logs|prometheus_api_v1_rules)\"}[$__rate_interval])) by (le,pod)) * 1e3", + "expr": "histogram_quantile(0.99, sum(rate(loki_request_duration_seconds_bucket{namespace=\"$namespace\", job=~\".+-querier-http\", route=~\"(api_prom_rules|api_prom_rules_namespace_groupname|api_v1_rules|loki_api_v1_delete|loki_api_v1_detected_labels|loki_api_v1_index_stats|loki_api_v1_index_volume|loki_api_v1_index_volume_range|loki_api_v1_label_name_values|loki_api_v1_label_values|loki_api_v1_labels|loki_api_v1_patterns|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_series|prometheus_api_v1_rules)\"}[$__rate_interval])) by (le,pod)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -1298,7 +1298,7 @@ "span": 4, "targets": [ { - "expr": "histogram_quantile(0.99, sum by (le,route) (route:loki_request_duration_seconds_bucket:sum_rate{route=~\"(/base.Ruler/Rules|/indexgatewaypb.IndexGateway/GetChunkRef|/indexgatewaypb.IndexGateway/GetSeries|/indexgatewaypb.IndexGateway/GetShards|/indexgatewaypb.IndexGateway/GetStats|/indexgatewaypb.IndexGateway/GetVolume|/indexgatewaypb.IndexGateway/LabelNamesForMetricName|/indexgatewaypb.IndexGateway/LabelValuesForMetricName|/indexgatewaypb.IndexGateway/QueryIndex|/logproto.BloomGateway/FilterChunkRefs|/logproto.Pattern/Query|/logproto.Querier/GetChunkIDs|/logproto.Querier/GetDetectedLabels|/logproto.Querier/GetStats|/logproto.Querier/GetVolume|/logproto.Querier/Label|/logproto.Querier/Query|/logproto.Querier/QuerySample|/logproto.Querier/Series|/logproto.StreamData/GetStreamRates)\"})) * 1e3", + "expr": "histogram_quantile(0.99, sum by (le,route) (namespace_job_route:loki_request_duration_seconds_bucket:sum_rate{namespace=\"$namespace\", job=~\".+-index-gateway-http\", route=~\"(/base.Ruler/Rules|/indexgatewaypb.IndexGateway/GetChunkRef|/indexgatewaypb.IndexGateway/GetSeries|/indexgatewaypb.IndexGateway/GetShards|/indexgatewaypb.IndexGateway/GetStats|/indexgatewaypb.IndexGateway/GetVolume|/indexgatewaypb.IndexGateway/LabelNamesForMetricName|/indexgatewaypb.IndexGateway/LabelValuesForMetricName|/indexgatewaypb.IndexGateway/QueryIndex|/logproto.BloomGateway/FilterChunkRefs|/logproto.Pattern/Query|/logproto.Querier/GetChunkIDs|/logproto.Querier/GetDetectedLabels|/logproto.Querier/GetStats|/logproto.Querier/GetVolume|/logproto.Querier/Label|/logproto.Querier/Query|/logproto.Querier/QuerySample|/logproto.Querier/Series|/logproto.StreamData/GetStreamRates)\"})) * 1e3", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{ route }} 99th Percentile", @@ -1306,7 +1306,7 @@ "step": 10 }, { - "expr": "histogram_quantile(0.50, sum by (le,route) (route:loki_request_duration_seconds_bucket:sum_rate{route=~\"(/base.Ruler/Rules|/indexgatewaypb.IndexGateway/GetChunkRef|/indexgatewaypb.IndexGateway/GetSeries|/indexgatewaypb.IndexGateway/GetShards|/indexgatewaypb.IndexGateway/GetStats|/indexgatewaypb.IndexGateway/GetVolume|/indexgatewaypb.IndexGateway/LabelNamesForMetricName|/indexgatewaypb.IndexGateway/LabelValuesForMetricName|/indexgatewaypb.IndexGateway/QueryIndex|/logproto.BloomGateway/FilterChunkRefs|/logproto.Pattern/Query|/logproto.Querier/GetChunkIDs|/logproto.Querier/GetDetectedLabels|/logproto.Querier/GetStats|/logproto.Querier/GetVolume|/logproto.Querier/Label|/logproto.Querier/Query|/logproto.Querier/QuerySample|/logproto.Querier/Series|/logproto.StreamData/GetStreamRates)\"})) * 1e3", + "expr": "histogram_quantile(0.50, sum by (le,route) (namespace_job_route:loki_request_duration_seconds_bucket:sum_rate{namespace=\"$namespace\", job=~\".+-index-gateway-http\", route=~\"(/base.Ruler/Rules|/indexgatewaypb.IndexGateway/GetChunkRef|/indexgatewaypb.IndexGateway/GetSeries|/indexgatewaypb.IndexGateway/GetShards|/indexgatewaypb.IndexGateway/GetStats|/indexgatewaypb.IndexGateway/GetVolume|/indexgatewaypb.IndexGateway/LabelNamesForMetricName|/indexgatewaypb.IndexGateway/LabelValuesForMetricName|/indexgatewaypb.IndexGateway/QueryIndex|/logproto.BloomGateway/FilterChunkRefs|/logproto.Pattern/Query|/logproto.Querier/GetChunkIDs|/logproto.Querier/GetDetectedLabels|/logproto.Querier/GetStats|/logproto.Querier/GetVolume|/logproto.Querier/Label|/logproto.Querier/Query|/logproto.Querier/QuerySample|/logproto.Querier/Series|/logproto.StreamData/GetStreamRates)\"})) * 1e3", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{ route }} 50th Percentile", @@ -1314,7 +1314,7 @@ "step": 10 }, { - "expr": "1e3 * sum(route:loki_request_duration_seconds_sum:sum_rate{route=~\"(/base.Ruler/Rules|/indexgatewaypb.IndexGateway/GetChunkRef|/indexgatewaypb.IndexGateway/GetSeries|/indexgatewaypb.IndexGateway/GetShards|/indexgatewaypb.IndexGateway/GetStats|/indexgatewaypb.IndexGateway/GetVolume|/indexgatewaypb.IndexGateway/LabelNamesForMetricName|/indexgatewaypb.IndexGateway/LabelValuesForMetricName|/indexgatewaypb.IndexGateway/QueryIndex|/logproto.BloomGateway/FilterChunkRefs|/logproto.Pattern/Query|/logproto.Querier/GetChunkIDs|/logproto.Querier/GetDetectedLabels|/logproto.Querier/GetStats|/logproto.Querier/GetVolume|/logproto.Querier/Label|/logproto.Querier/Query|/logproto.Querier/QuerySample|/logproto.Querier/Series|/logproto.StreamData/GetStreamRates)\"}) by (route) / sum(route:loki_request_duration_seconds_count:sum_rate{route=~\"(/base.Ruler/Rules|/indexgatewaypb.IndexGateway/GetChunkRef|/indexgatewaypb.IndexGateway/GetSeries|/indexgatewaypb.IndexGateway/GetShards|/indexgatewaypb.IndexGateway/GetStats|/indexgatewaypb.IndexGateway/GetVolume|/indexgatewaypb.IndexGateway/LabelNamesForMetricName|/indexgatewaypb.IndexGateway/LabelValuesForMetricName|/indexgatewaypb.IndexGateway/QueryIndex|/logproto.BloomGateway/FilterChunkRefs|/logproto.Pattern/Query|/logproto.Querier/GetChunkIDs|/logproto.Querier/GetDetectedLabels|/logproto.Querier/GetStats|/logproto.Querier/GetVolume|/logproto.Querier/Label|/logproto.Querier/Query|/logproto.Querier/QuerySample|/logproto.Querier/Series|/logproto.StreamData/GetStreamRates)\"}) by (route) ", + "expr": "1e3 * sum(namespace_job_route:loki_request_duration_seconds_sum:sum_rate{namespace=\"$namespace\", job=~\".+-index-gateway-http\", route=~\"(/base.Ruler/Rules|/indexgatewaypb.IndexGateway/GetChunkRef|/indexgatewaypb.IndexGateway/GetSeries|/indexgatewaypb.IndexGateway/GetShards|/indexgatewaypb.IndexGateway/GetStats|/indexgatewaypb.IndexGateway/GetVolume|/indexgatewaypb.IndexGateway/LabelNamesForMetricName|/indexgatewaypb.IndexGateway/LabelValuesForMetricName|/indexgatewaypb.IndexGateway/QueryIndex|/logproto.BloomGateway/FilterChunkRefs|/logproto.Pattern/Query|/logproto.Querier/GetChunkIDs|/logproto.Querier/GetDetectedLabels|/logproto.Querier/GetStats|/logproto.Querier/GetVolume|/logproto.Querier/Label|/logproto.Querier/Query|/logproto.Querier/QuerySample|/logproto.Querier/Series|/logproto.StreamData/GetStreamRates)\"}) by (route) / sum(namespace_job_route:loki_request_duration_seconds_count:sum_rate{namespace=\"$namespace\", job=~\".+-index-gateway-http\", route=~\"(/base.Ruler/Rules|/indexgatewaypb.IndexGateway/GetChunkRef|/indexgatewaypb.IndexGateway/GetSeries|/indexgatewaypb.IndexGateway/GetShards|/indexgatewaypb.IndexGateway/GetStats|/indexgatewaypb.IndexGateway/GetVolume|/indexgatewaypb.IndexGateway/LabelNamesForMetricName|/indexgatewaypb.IndexGateway/LabelValuesForMetricName|/indexgatewaypb.IndexGateway/QueryIndex|/logproto.BloomGateway/FilterChunkRefs|/logproto.Pattern/Query|/logproto.Querier/GetChunkIDs|/logproto.Querier/GetDetectedLabels|/logproto.Querier/GetStats|/logproto.Querier/GetVolume|/logproto.Querier/Label|/logproto.Querier/Query|/logproto.Querier/QuerySample|/logproto.Querier/Series|/logproto.StreamData/GetStreamRates)\"}) by (route) ", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{ route }} Average", diff --git a/operator/internal/manifests/openshift/internal/dashboards/static/grafana-dashboard-lokistack-writes.json b/operator/internal/manifests/openshift/internal/dashboards/static/grafana-dashboard-lokistack-writes.json index 334045ddfc11c..50f5f492e8896 100644 --- a/operator/internal/manifests/openshift/internal/dashboards/static/grafana-dashboard-lokistack-writes.json +++ b/operator/internal/manifests/openshift/internal/dashboards/static/grafana-dashboard-lokistack-writes.json @@ -213,7 +213,7 @@ "stack": true, "targets": [ { - "expr": "sum by (status) (\n label_replace(label_replace(rate(loki_request_duration_seconds_count{namespace=\"$namespace\",job=~\".+-distributor-http\", route=~\"api_prom_push|loki_api_v1_push|/httpgrpc.HTTP/Handle\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(loki_request_duration_seconds_count{namespace=\"$namespace\",job=~\".+-distributor-http\", route=~\"api_prom_push|loki_api_v1_push|otlp_v1_logs|/httpgrpc.HTTP/Handle\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", "format": "time_series", "intervalFactor": 2, "legendFormat": "{{status}}", @@ -263,7 +263,7 @@ "span": 4, "targets": [ { - "expr": "histogram_quantile(0.99, sum by (le) (namespace_job_route:loki_request_duration_seconds_bucket:sum_rate{namespace=\"$namespace\", job=~\".+-distributor-http\", route=~\"api_prom_push|loki_api_v1_push|/httpgrpc.HTTP/Handle\"})) * 1e3", + "expr": "histogram_quantile(0.99, sum by (le) (namespace_job_route:loki_request_duration_seconds_bucket:sum_rate{namespace=\"$namespace\", job=~\".+-distributor-http\", route=~\"api_prom_push|loki_api_v1_push|otlp_v1_logs|/httpgrpc.HTTP/Handle\"})) * 1e3", "format": "time_series", "intervalFactor": 2, "legendFormat": "99th Percentile", @@ -271,7 +271,7 @@ "step": 10 }, { - "expr": "histogram_quantile(0.50, sum by (le) (namespace_job_route:loki_request_duration_seconds_bucket:sum_rate{namespace=\"$namespace\", job=~\".+-distributor-http\", route=~\"api_prom_push|loki_api_v1_push|/httpgrpc.HTTP/Handle\"})) * 1e3", + "expr": "histogram_quantile(0.50, sum by (le) (namespace_job_route:loki_request_duration_seconds_bucket:sum_rate{namespace=\"$namespace\", job=~\".+-distributor-http\", route=~\"api_prom_push|loki_api_v1_push|otlp_v1_logs|/httpgrpc.HTTP/Handle\"})) * 1e3", "format": "time_series", "intervalFactor": 2, "legendFormat": "50th Percentile", @@ -279,7 +279,7 @@ "step": 10 }, { - "expr": "1e3 * sum(namespace_job_route:loki_request_duration_seconds_sum:sum_rate{namespace=\"$namespace\", job=~\".+-distributor-http\", route=~\"api_prom_push|loki_api_v1_push|/httpgrpc.HTTP/Handle\"}) / sum(namespace_job_route:loki_request_duration_seconds_count:sum_rate{namespace=\"$namespace\", job=~\".+-distributor-http\", route=~\"api_prom_push|loki_api_v1_push|/httpgrpc.HTTP/Handle\"})", + "expr": "1e3 * sum(namespace_job_route:loki_request_duration_seconds_sum:sum_rate{namespace=\"$namespace\", job=~\".+-distributor-http\", route=~\"api_prom_push|loki_api_v1_push|otlp_v1_logs|/httpgrpc.HTTP/Handle\"}) / sum(namespace_job_route:loki_request_duration_seconds_count:sum_rate{namespace=\"$namespace\", job=~\".+-distributor-http\", route=~\"api_prom_push|loki_api_v1_push|otlp_v1_logs|/httpgrpc.HTTP/Handle\"})", "format": "time_series", "intervalFactor": 2, "legendFormat": "Average", @@ -347,7 +347,7 @@ "span": 4, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(loki_request_duration_seconds_bucket{namespace=\"$namespace\", job=~\".+-distributor-http\", route=~\"api_prom_push|loki_api_v1_push|/httpgrpc.HTTP/Handle\"}[$__rate_interval])) by (le,pod)) * 1e3", + "expr": "histogram_quantile(0.99, sum(rate(loki_request_duration_seconds_bucket{namespace=\"$namespace\", job=~\".+-distributor-http\", route=~\"api_prom_push|loki_api_v1_push|otlp_v1_logs|/httpgrpc.HTTP/Handle\"}[$__rate_interval])) by (le,pod)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, diff --git a/operator/internal/manifests/var.go b/operator/internal/manifests/var.go index 1fed3e1422982..46ba72d450850 100644 --- a/operator/internal/manifests/var.go +++ b/operator/internal/manifests/var.go @@ -59,7 +59,7 @@ const ( EnvRelatedImageGateway = "RELATED_IMAGE_GATEWAY" // DefaultContainerImage declares the default fallback for loki image. - DefaultContainerImage = "docker.io/grafana/loki:3.1.1" + DefaultContainerImage = "docker.io/grafana/loki:3.2.0" // DefaultLokiStackGatewayImage declares the default image for lokiStack-gateway. DefaultLokiStackGatewayImage = "quay.io/observatorium/api:latest" diff --git a/operator/jsonnet/jsonnetfile.json b/operator/jsonnet/jsonnetfile.json index 7d1b707308153..a2ac102b07587 100644 --- a/operator/jsonnet/jsonnetfile.json +++ b/operator/jsonnet/jsonnetfile.json @@ -8,7 +8,7 @@ "subdir": "production/loki-mixin" } }, - "version": "v3.1.0" + "version": "v3.2.0" } ], "legacyImports": true diff --git a/operator/jsonnet/jsonnetfile.lock.json b/operator/jsonnet/jsonnetfile.lock.json index acc2cd702fe38..40bf4f23a2544 100644 --- a/operator/jsonnet/jsonnetfile.lock.json +++ b/operator/jsonnet/jsonnetfile.lock.json @@ -38,8 +38,8 @@ "subdir": "production/loki-mixin" } }, - "version": "935aee77ed389c825d36b8d6a85c0d83895a24d1", - "sum": "FsHTEwIRvbbC2qPwDuzOQx8ilfa1+gR/8r52bFsNQMU=" + "version": "659f5421dde1dc4b27d6a0afd1a568673f50dfcc", + "sum": "mVKuwcuL1Wm+JMgt2MiwOAtKkmuClzE75+liMe0AGek=" }, { "source": { From da04f5007edd85f35d1af5ba8c2c5a4eb96d2149 Mon Sep 17 00:00:00 2001 From: Quentin Bisson Date: Wed, 2 Oct 2024 11:20:07 +0200 Subject: [PATCH 02/10] feat: mixin / add loki compaction not successfull alert (#14239) Co-authored-by: Ashwanth --- .../loki-mixin-compiled-ssd/alerts.yaml | 33 +++++++++++++ production/loki-mixin-compiled/alerts.yaml | 33 +++++++++++++ production/loki-mixin/alerts.libsonnet | 47 +++++++++++++++++++ 3 files changed, 113 insertions(+) diff --git a/production/loki-mixin-compiled-ssd/alerts.yaml b/production/loki-mixin-compiled-ssd/alerts.yaml index 09b9b6f543412..e33c010ff0560 100644 --- a/production/loki-mixin-compiled-ssd/alerts.yaml +++ b/production/loki-mixin-compiled-ssd/alerts.yaml @@ -43,3 +43,36 @@ groups: for: 5m labels: severity: warning + - alert: LokiCompactorHasNotSuccessfullyRunCompaction + annotations: + description: | + {{ $labels.cluster }} {{ $labels.namespace }} has not run compaction in the last 3 hours since the last compaction. This may indicate a problem with the compactor. + summary: Loki compaction has not run in the last 3 hours since the last compaction. + expr: | + # The "last successful run" metric is updated even if the compactor owns no tenants, + # so this alert correctly doesn't fire if compactor has nothing to do. + min ( + time() - (loki_boltdb_shipper_compact_tables_operation_last_successful_run_timestamp_seconds{} > 0) + ) + by (cluster, namespace) + > 60 * 60 * 3 + for: 1h + labels: + severity: critical + - alert: LokiCompactorHasNotSuccessfullyRunCompaction + annotations: + description: | + {{ $labels.cluster }} {{ $labels.namespace }} has not run compaction in the last 3h since startup. This may indicate a problem with the compactor. + summary: Loki compaction has not run in the last 3h since startup. + expr: | + # The "last successful run" metric is updated even if the compactor owns no tenants, + # so this alert correctly doesn't fire if compactor has nothing to do. + max( + max_over_time( + loki_boltdb_shipper_compact_tables_operation_last_successful_run_timestamp_seconds{}[3h] + ) + ) by (cluster, namespace) + == 0 + for: 1h + labels: + severity: critical diff --git a/production/loki-mixin-compiled/alerts.yaml b/production/loki-mixin-compiled/alerts.yaml index 09b9b6f543412..e33c010ff0560 100644 --- a/production/loki-mixin-compiled/alerts.yaml +++ b/production/loki-mixin-compiled/alerts.yaml @@ -43,3 +43,36 @@ groups: for: 5m labels: severity: warning + - alert: LokiCompactorHasNotSuccessfullyRunCompaction + annotations: + description: | + {{ $labels.cluster }} {{ $labels.namespace }} has not run compaction in the last 3 hours since the last compaction. This may indicate a problem with the compactor. + summary: Loki compaction has not run in the last 3 hours since the last compaction. + expr: | + # The "last successful run" metric is updated even if the compactor owns no tenants, + # so this alert correctly doesn't fire if compactor has nothing to do. + min ( + time() - (loki_boltdb_shipper_compact_tables_operation_last_successful_run_timestamp_seconds{} > 0) + ) + by (cluster, namespace) + > 60 * 60 * 3 + for: 1h + labels: + severity: critical + - alert: LokiCompactorHasNotSuccessfullyRunCompaction + annotations: + description: | + {{ $labels.cluster }} {{ $labels.namespace }} has not run compaction in the last 3h since startup. This may indicate a problem with the compactor. + summary: Loki compaction has not run in the last 3h since startup. + expr: | + # The "last successful run" metric is updated even if the compactor owns no tenants, + # so this alert correctly doesn't fire if compactor has nothing to do. + max( + max_over_time( + loki_boltdb_shipper_compact_tables_operation_last_successful_run_timestamp_seconds{}[3h] + ) + ) by (cluster, namespace) + == 0 + for: 1h + labels: + severity: critical diff --git a/production/loki-mixin/alerts.libsonnet b/production/loki-mixin/alerts.libsonnet index 9261dbccecf99..02fb2a0ee5662 100644 --- a/production/loki-mixin/alerts.libsonnet +++ b/production/loki-mixin/alerts.libsonnet @@ -70,6 +70,53 @@ |||, 'cluster', $._config.per_cluster_label), }, }, + { + // Alert if the compactor has not successfully run compaction in the last 3h since the last compaction. + alert: 'LokiCompactorHasNotSuccessfullyRunCompaction', + expr: ||| + # The "last successful run" metric is updated even if the compactor owns no tenants, + # so this alert correctly doesn't fire if compactor has nothing to do. + min ( + time() - (loki_boltdb_shipper_compact_tables_operation_last_successful_run_timestamp_seconds{} > 0) + ) + by (%s, namespace) + > 60 * 60 * 3 + ||| % $._config.per_cluster_label, + 'for': '1h', + labels: { + severity: 'critical', + }, + annotations: { + summary: 'Loki compaction has not run in the last 3 hours since the last compaction.', + description: std.strReplace(||| + {{ $labels.cluster }} {{ $labels.namespace }} has not run compaction in the last 3 hours since the last compaction. This may indicate a problem with the compactor. + |||, 'cluster', $._config.per_cluster_label), + }, + }, + { + // Alert if the compactor has not successfully run compaction in the last 3h since startup. + alert: 'LokiCompactorHasNotSuccessfullyRunCompaction', + expr: ||| + # The "last successful run" metric is updated even if the compactor owns no tenants, + # so this alert correctly doesn't fire if compactor has nothing to do. + max( + max_over_time( + loki_boltdb_shipper_compact_tables_operation_last_successful_run_timestamp_seconds{}[3h] + ) + ) by (%s, namespace) + == 0 + ||| % $._config.per_cluster_label, + 'for': '1h', + labels: { + severity: 'critical', + }, + annotations: { + summary: 'Loki compaction has not run in the last 3h since startup.', + description: std.strReplace(||| + {{ $labels.cluster }} {{ $labels.namespace }} has not run compaction in the last 3h since startup. This may indicate a problem with the compactor. + |||, 'cluster', $._config.per_cluster_label), + }, + }, ], }, ], From 30528c9f6695d7e55aa904b83df1308becc93d21 Mon Sep 17 00:00:00 2001 From: Joao Marcal Date: Wed, 2 Oct 2024 11:20:15 +0200 Subject: [PATCH 03/10] chore(operator): add JoaoBraveCoding to loki-operator owners (#14332) --- CODEOWNERS | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/CODEOWNERS b/CODEOWNERS index 32ae698aa9e0a..a118c5a40c0b1 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -5,8 +5,8 @@ /docs/ @grafana/docs-logs @grafana/loki-team # Loki operator -/operator/ @grafana/loki-team @periklis @xperimental -/.github/workflows/operator* @grafana/loki-team @periklis @xperimental +/operator/ @grafana/loki-team @periklis @xperimental @JoaoBraveCoding +/.github/workflows/operator* @grafana/loki-team @periklis @xperimental @JoaoBraveCoding # Logql grammar # The observability logs team is listed as co-codeowner for grammar file. This is to receive notifications about updates, so these can be implemented in https://github.com/grafana/lezer-logql From 1a4436c41721e3e6aca82c26abaec8fe6f775d9f Mon Sep 17 00:00:00 2001 From: Cyril Tovena Date: Wed, 2 Oct 2024 15:02:48 +0200 Subject: [PATCH 04/10] feat(ingester): implement partition shuffle sharding for ingester (#14304) --- docs/sources/shared/configuration.md | 6 ++++++ pkg/distributor/distributor.go | 14 +++++++++----- pkg/distributor/limits.go | 2 ++ pkg/ingester/ingester.go | 2 +- pkg/ingester/limiter.go | 1 + pkg/ingester/recalculate_owned_streams.go | 17 +++++++++++------ pkg/ingester/recalculate_owned_streams_test.go | 9 ++++----- pkg/validation/limits.go | 8 ++++++++ 8 files changed, 42 insertions(+), 17 deletions(-) diff --git a/docs/sources/shared/configuration.md b/docs/sources/shared/configuration.md index 15426e54d088f..ce2c8d359d752 100644 --- a/docs/sources/shared/configuration.md +++ b/docs/sources/shared/configuration.md @@ -3813,6 +3813,12 @@ otlp_config: # status code (260) is returned to the client along with an error message. # CLI flag: -limits.block-ingestion-status-code [block_ingestion_status_code: | default = 260] + +# The number of partitions a tenant's data should be sharded to when using kafka +# ingestion. Tenants are sharded across partitions using shuffle-sharding. 0 +# disables shuffle sharding and tenant is sharded across all partitions. +# CLI flag: -limits.ingestion-partition-tenant-shard-size +[ingestion_partitions_tenant_shard_size: | default = 0] ``` ### local_storage_config diff --git a/pkg/distributor/distributor.go b/pkg/distributor/distributor.go index 476bad507ea0b..01dae3ee6e0cf 100644 --- a/pkg/distributor/distributor.go +++ b/pkg/distributor/distributor.go @@ -604,8 +604,12 @@ func (d *Distributor) Push(ctx context.Context, req *logproto.PushRequest) (*log tracker.streamsPending.Store(int32(streamsToWrite)) if d.cfg.KafkaEnabled { + subring, err := d.partitionRing.PartitionRing().ShuffleShard(tenantID, d.validator.IngestionPartitionsTenantShardSize(tenantID)) + if err != nil { + return nil, err + } // We don't need to create a new context like the ingester writes, because we don't return unless all writes have succeeded. - d.sendStreamsToKafka(ctx, streams, tenantID, &tracker) + d.sendStreamsToKafka(ctx, streams, tenantID, &tracker, subring) } if d.cfg.IngesterEnabled { @@ -931,10 +935,10 @@ func (d *Distributor) sendStreamsErr(ctx context.Context, ingester ring.Instance return err } -func (d *Distributor) sendStreamsToKafka(ctx context.Context, streams []KeyedStream, tenant string, tracker *pushTracker) { +func (d *Distributor) sendStreamsToKafka(ctx context.Context, streams []KeyedStream, tenant string, tracker *pushTracker, subring *ring.PartitionRing) { for _, s := range streams { go func(s KeyedStream) { - err := d.sendStreamToKafka(ctx, s, tenant) + err := d.sendStreamToKafka(ctx, s, tenant, subring) if err != nil { err = fmt.Errorf("failed to write stream to kafka: %w", err) } @@ -943,11 +947,11 @@ func (d *Distributor) sendStreamsToKafka(ctx context.Context, streams []KeyedStr } } -func (d *Distributor) sendStreamToKafka(ctx context.Context, stream KeyedStream, tenant string) error { +func (d *Distributor) sendStreamToKafka(ctx context.Context, stream KeyedStream, tenant string, subring *ring.PartitionRing) error { if len(stream.Stream.Entries) == 0 { return nil } - partitionID, err := d.partitionRing.PartitionRing().ActivePartitionForKey(stream.HashKey) + partitionID, err := subring.ActivePartitionForKey(stream.HashKey) if err != nil { d.kafkaAppends.WithLabelValues("kafka", "fail").Inc() return fmt.Errorf("failed to find active partition for stream: %w", err) diff --git a/pkg/distributor/limits.go b/pkg/distributor/limits.go index 7e7e6050d612b..c72eb1939a3bc 100644 --- a/pkg/distributor/limits.go +++ b/pkg/distributor/limits.go @@ -36,4 +36,6 @@ type Limits interface { BlockIngestionUntil(userID string) time.Time BlockIngestionStatusCode(userID string) int + + IngestionPartitionsTenantShardSize(userID string) int } diff --git a/pkg/ingester/ingester.go b/pkg/ingester/ingester.go index 529336a58561c..8672d5f4cad76 100644 --- a/pkg/ingester/ingester.go +++ b/pkg/ingester/ingester.go @@ -410,7 +410,7 @@ func New(cfg Config, clientConfig client.Config, store Store, limits Limits, con var ownedStreamsStrategy ownershipStrategy if i.cfg.KafkaIngestion.Enabled { - ownedStreamsStrategy = newOwnedStreamsPartitionStrategy(i.ingestPartitionID, partitionRingWatcher, util_log.Logger) + ownedStreamsStrategy = newOwnedStreamsPartitionStrategy(i.ingestPartitionID, partitionRingWatcher, limits.IngestionPartitionsTenantShardSize, util_log.Logger) } else { ownedStreamsStrategy = newOwnedStreamsIngesterStrategy(i.lifecycler.ID, i.readRing, util_log.Logger) } diff --git a/pkg/ingester/limiter.go b/pkg/ingester/limiter.go index 1ed3a3ea27163..a9ddd2ba3ba3c 100644 --- a/pkg/ingester/limiter.go +++ b/pkg/ingester/limiter.go @@ -31,6 +31,7 @@ type Limits interface { MaxGlobalStreamsPerUser(userID string) int PerStreamRateLimit(userID string) validation.RateLimit ShardStreams(userID string) shardstreams.Config + IngestionPartitionsTenantShardSize(userID string) int } // Limiter implements primitives to get the maximum number of streams diff --git a/pkg/ingester/recalculate_owned_streams.go b/pkg/ingester/recalculate_owned_streams.go index b1f6bd62ebfc7..c521e55e2d43d 100644 --- a/pkg/ingester/recalculate_owned_streams.go +++ b/pkg/ingester/recalculate_owned_streams.go @@ -160,11 +160,12 @@ type ownedStreamsPartitionStrategy struct { getPartitionShardSize func(user string) int } -func newOwnedStreamsPartitionStrategy(partitionID int32, ring ring.PartitionRingReader, logger log.Logger) *ownedStreamsPartitionStrategy { +func newOwnedStreamsPartitionStrategy(partitionID int32, ring ring.PartitionRingReader, getPartitionShardSize func(user string) int, logger log.Logger) *ownedStreamsPartitionStrategy { return &ownedStreamsPartitionStrategy{ - partitionID: partitionID, - partitionRingWatcher: ring, - logger: logger, + partitionID: partitionID, + partitionRingWatcher: ring, + logger: logger, + getPartitionShardSize: getPartitionShardSize, } } @@ -174,7 +175,7 @@ func (s *ownedStreamsPartitionStrategy) checkRingForChanges() (bool, error) { if r.PartitionsCount() == 0 { return false, ring.ErrEmptyRing } - + // todo(ctovena): We might need to consider partition shard size changes as well. activePartitions := r.ActivePartitionIDs() ringChanged := !slices.Equal(s.previousActivePartitions, activePartitions) s.previousActivePartitions = activePartitions @@ -182,7 +183,11 @@ func (s *ownedStreamsPartitionStrategy) checkRingForChanges() (bool, error) { } func (s *ownedStreamsPartitionStrategy) isOwnedStream(str *stream) (bool, error) { - partitionForStream, err := s.partitionRingWatcher.PartitionRing().ActivePartitionForKey(lokiring.TokenFor(str.tenant, str.labelsString)) + subring, err := s.partitionRingWatcher.PartitionRing().ShuffleShard(str.tenant, s.getPartitionShardSize(str.tenant)) + if err != nil { + return false, fmt.Errorf("failed to get shuffle shard for stream: %w", err) + } + partitionForStream, err := subring.ActivePartitionForKey(lokiring.TokenFor(str.tenant, str.labelsString)) if err != nil { return false, fmt.Errorf("failed to find active partition for stream: %w", err) } diff --git a/pkg/ingester/recalculate_owned_streams_test.go b/pkg/ingester/recalculate_owned_streams_test.go index 82a733e593d61..d5dce8599287b 100644 --- a/pkg/ingester/recalculate_owned_streams_test.go +++ b/pkg/ingester/recalculate_owned_streams_test.go @@ -108,7 +108,7 @@ func Test_recalculateOwnedStreams_recalculateWithIngesterStrategy(t *testing.T) strategy := newOwnedStreamsIngesterStrategy(currentIngesterName, mockRing, log.NewNopLogger()) service := newRecalculateOwnedStreamsSvc(mockTenantsSupplier.get, strategy, 50*time.Millisecond, log.NewNopLogger()) - //change the limit to assert that fixed limit is updated after the recalculation + // change the limit to assert that fixed limit is updated after the recalculation limits.DefaultLimits().MaxGlobalStreamsPerUser = 50 service.recalculate() @@ -120,7 +120,6 @@ func Test_recalculateOwnedStreams_recalculateWithIngesterStrategy(t *testing.T) require.Len(t, tenant.ownedStreamsSvc.notOwnedStreams, testData.expectedNotOwnedStreamCount) }) } - } type mockStreamsOwnershipRing struct { @@ -203,7 +202,7 @@ func Test_ownedStreamsPartitionStrategy_checkRingForChanges(t *testing.T) { ringReader := &mockPartitionRingReader{ ring: newMockPartitionRingWithActivePartitions(1), } - service := newOwnedStreamsPartitionStrategy(1, ringReader, log.NewNopLogger()) + service := newOwnedStreamsPartitionStrategy(1, ringReader, func(string) int { return 1 }, log.NewNopLogger()) ringChanged, err := service.checkRingForChanges() require.NoError(t, err) @@ -226,12 +225,12 @@ func Test_ownedStreamsPartitionStrategy_isOwnedStream(t *testing.T) { } stream := &stream{tenant: "test1", labelsString: "mock=1"} // has a hashkey mapping to partition 1 - service1 := newOwnedStreamsPartitionStrategy(1, ringReader, log.NewNopLogger()) + service1 := newOwnedStreamsPartitionStrategy(1, ringReader, func(string) int { return 1 }, log.NewNopLogger()) owned, err := service1.isOwnedStream(stream) require.NoError(t, err) require.True(t, owned) - service2 := newOwnedStreamsPartitionStrategy(2, ringReader, log.NewNopLogger()) + service2 := newOwnedStreamsPartitionStrategy(2, ringReader, func(string) int { return 1 }, log.NewNopLogger()) owned, err = service2.isOwnedStream(stream) require.NoError(t, err) require.False(t, owned) diff --git a/pkg/validation/limits.go b/pkg/validation/limits.go index 9b362f20704f4..5da6bc9cfc61d 100644 --- a/pkg/validation/limits.go +++ b/pkg/validation/limits.go @@ -222,6 +222,8 @@ type Limits struct { BlockIngestionUntil dskit_flagext.Time `yaml:"block_ingestion_until" json:"block_ingestion_until"` BlockIngestionStatusCode int `yaml:"block_ingestion_status_code" json:"block_ingestion_status_code"` + + IngestionPartitionsTenantShardSize int `yaml:"ingestion_partitions_tenant_shard_size" json:"ingestion_partitions_tenant_shard_size" category:"experimental"` } type StreamRetention struct { @@ -412,6 +414,8 @@ func (l *Limits) RegisterFlags(f *flag.FlagSet) { f.Var(&l.BlockIngestionUntil, "limits.block-ingestion-until", "Block ingestion until the configured date. The time should be in RFC3339 format.") f.IntVar(&l.BlockIngestionStatusCode, "limits.block-ingestion-status-code", defaultBlockedIngestionStatusCode, "HTTP status code to return when ingestion is blocked. If 200, the ingestion will be blocked without returning an error to the client. By Default, a custom status code (260) is returned to the client along with an error message.") + + f.IntVar(&l.IngestionPartitionsTenantShardSize, "limits.ingestion-partition-tenant-shard-size", 0, "The number of partitions a tenant's data should be sharded to when using kafka ingestion. Tenants are sharded across partitions using shuffle-sharding. 0 disables shuffle sharding and tenant is sharded across all partitions.") } // SetGlobalOTLPConfig set GlobalOTLPConfig which is used while unmarshaling per-tenant otlp config to use the default list of resource attributes picked as index labels. @@ -778,6 +782,10 @@ func (o *Overrides) RulerTenantShardSize(userID string) int { return o.getOverridesForUser(userID).RulerTenantShardSize } +func (o *Overrides) IngestionPartitionsTenantShardSize(userID string) int { + return o.getOverridesForUser(userID).IngestionPartitionsTenantShardSize +} + // RulerMaxRulesPerRuleGroup returns the maximum number of rules per rule group for a given user. func (o *Overrides) RulerMaxRulesPerRuleGroup(userID string) int { return o.getOverridesForUser(userID).RulerMaxRulesPerRuleGroup From 41fafd87933224d5d43592e91e339322fc90a466 Mon Sep 17 00:00:00 2001 From: Ravishankar Date: Wed, 2 Oct 2024 19:08:41 +0530 Subject: [PATCH 05/10] feat: Apply patterns line length limit to json message key (#14296) --- pkg/pattern/drain/drain.go | 9 ++---- pkg/pattern/drain/line_tokenizer.go | 39 ++++++++++++++++++------ pkg/pattern/drain/line_tokenizer_test.go | 35 ++++++++++++++++----- 3 files changed, 59 insertions(+), 24 deletions(-) diff --git a/pkg/pattern/drain/drain.go b/pkg/pattern/drain/drain.go index e6706e0517bb0..9e6062432cc6a 100644 --- a/pkg/pattern/drain/drain.go +++ b/pkg/pattern/drain/drain.go @@ -153,11 +153,11 @@ func New(config *Config, format string, metrics *Metrics) *Drain { var tokenizer LineTokenizer switch format { case FormatJSON: - tokenizer = newJSONTokenizer(config.ParamString) + tokenizer = newJSONTokenizer(config.ParamString, config.MaxAllowedLineLength) case FormatLogfmt: - tokenizer = newLogfmtTokenizer(config.ParamString) + tokenizer = newLogfmtTokenizer(config.ParamString, config.MaxAllowedLineLength) default: - tokenizer = newPunctuationTokenizer() + tokenizer = newPunctuationTokenizer(config.MaxAllowedLineLength) } d.idToCluster = createLogClusterCache(config.MaxClusters, func(int, *LogCluster) { @@ -206,9 +206,6 @@ func (d *Drain) Train(content string, ts int64) *LogCluster { if !d.limiter.Allow() { return nil } - if len(content) > d.config.MaxAllowedLineLength { - return nil - } d.tokens, d.state = d.tokenizer.Tokenize(content, d.tokens, d.state) return d.train(d.tokens, d.state, ts) } diff --git a/pkg/pattern/drain/line_tokenizer.go b/pkg/pattern/drain/line_tokenizer.go index e2acd8228bcfa..87b98afaea6de 100644 --- a/pkg/pattern/drain/line_tokenizer.go +++ b/pkg/pattern/drain/line_tokenizer.go @@ -37,9 +37,10 @@ func (spacesTokenizer) Clone(tokens []string, _ interface{}) ([]string, interfac type punctuationTokenizer struct { includeDelimiters [128]rune excludeDelimiters [128]rune + maxLineLength int } -func newPunctuationTokenizer() *punctuationTokenizer { +func newPunctuationTokenizer(maxLineLength int) *punctuationTokenizer { var included [128]rune var excluded [128]rune included['='] = 1 @@ -51,10 +52,15 @@ func newPunctuationTokenizer() *punctuationTokenizer { return &punctuationTokenizer{ includeDelimiters: included, excludeDelimiters: excluded, + maxLineLength: maxLineLength, } } func (p *punctuationTokenizer) Tokenize(line string, tokens []string, state interface{}) ([]string, interface{}) { + if len(line) > p.maxLineLength { + return nil, nil + } + if cap(tokens) == 0 { tokens = make([]string, 0, 128) } @@ -190,18 +196,24 @@ func (splittingTokenizer) Clone(tokens []string, state interface{}) ([]string, i } type logfmtTokenizer struct { - dec *logfmt.Decoder - varReplace string + dec *logfmt.Decoder + varReplace string + maxLineLength int } -func newLogfmtTokenizer(varReplace string) *logfmtTokenizer { +func newLogfmtTokenizer(varReplace string, maxLineLength int) *logfmtTokenizer { return &logfmtTokenizer{ - dec: logfmt.NewDecoder(nil), - varReplace: varReplace, + dec: logfmt.NewDecoder(nil), + varReplace: varReplace, + maxLineLength: maxLineLength, } } func (t *logfmtTokenizer) Tokenize(line string, tokens []string, _ interface{}) ([]string, interface{}) { + if len(line) > t.maxLineLength { + return nil, nil + } + if cap(tokens) == 0 { tokens = make([]string, 0, 64) } @@ -251,11 +263,12 @@ func (t *logfmtTokenizer) Clone(tokens []string, _ interface{}) ([]string, inter type jsonTokenizer struct { *punctuationTokenizer - varReplace string + varReplace string + maxLineLength int } -func newJSONTokenizer(varReplace string) *jsonTokenizer { - return &jsonTokenizer{newPunctuationTokenizer(), varReplace} +func newJSONTokenizer(varReplace string, maxLineLength int) *jsonTokenizer { + return &jsonTokenizer{newPunctuationTokenizer(maxLineLength), varReplace, maxLineLength} } func (t *jsonTokenizer) Tokenize(line string, tokens []string, state interface{}) ([]string, interface{}) { @@ -272,7 +285,13 @@ func (t *jsonTokenizer) Tokenize(line string, tokens []string, state interface{} return nil, nil } - return t.punctuationTokenizer.Tokenize(unsafeString(found), tokens, state) + foundLine := unsafeString(found) + + if len(foundLine) > t.maxLineLength { + return nil, nil + } + + return t.punctuationTokenizer.Tokenize(foundLine, tokens, state) } func (t *jsonTokenizer) Join(tokens []string, state interface{}) string { diff --git a/pkg/pattern/drain/line_tokenizer_test.go b/pkg/pattern/drain/line_tokenizer_test.go index 200d1a8f510e6..f825a8d86bbc6 100644 --- a/pkg/pattern/drain/line_tokenizer_test.go +++ b/pkg/pattern/drain/line_tokenizer_test.go @@ -115,6 +115,14 @@ var testCases = []TestCase{ typeSplitting: {`!@£$%^&*()`}, }, }, + { + name: "line length greater than max allowed length", + line: `09:17:38.033366 ▶ INFO route ops sending to dest https://graphite-cortex-ops-blocks-us-east4.grafana.net/graphite/metrics: service_is_carbon-relay-ng.instance_is_carbon-relay-ng-c665b7b-j2trk.mtype_is_counter.dest_is_https_graphite-cortex-ops-blocks-us-east4_grafana_netgraphitemetrics.unit_is_Metric.action_is_drop.reason_is_queue_full 0 1717060658 userid invalid`, + want: map[string][]string{ + typePunctuation: []string(nil), + typeSplitting: {`09:`, `17:`, `38.033366`, `▶`, `INFO`, ``, `route`, `ops`, `sending`, `to`, `dest`, `https:`, `//graphite-cortex-ops-blocks-us-east4.grafana.net/graphite/metrics:`, ``, `service_is_carbon-relay-ng.instance_is_carbon-relay-ng-c665b7b-j2trk.mtype_is_counter.dest_is_https_graphite-cortex-ops-blocks-us-east4_grafana_netgraphitemetrics.unit_is_Metric.action_is_drop.reason_is_queue_full`, `0`, `1717060658`, `userid`, `invalid`}, + }, + }, } func TestTokenizer_Tokenize(t *testing.T) { @@ -124,7 +132,7 @@ func TestTokenizer_Tokenize(t *testing.T) { }{ { name: typePunctuation, - tokenizer: newPunctuationTokenizer(), + tokenizer: newPunctuationTokenizer(360), }, { name: typeSplitting, @@ -149,7 +157,7 @@ func TestTokenizer_TokenizeAndJoin(t *testing.T) { }{ { name: typePunctuation, - tokenizer: newPunctuationTokenizer(), + tokenizer: newPunctuationTokenizer(DefaultConfig().MaxAllowedLineLength), }, { name: typeSplitting, @@ -168,7 +176,7 @@ func TestTokenizer_TokenizeAndJoin(t *testing.T) { } func BenchmarkSplittingTokenizer(b *testing.B) { - tokenizer := newPunctuationTokenizer() + tokenizer := newPunctuationTokenizer(DefaultConfig().MaxAllowedLineLength) for _, tt := range testCases { tc := tt @@ -213,9 +221,13 @@ func TestLogFmtTokenizer(t *testing.T) { line: `logger=sqlstore.metrics traceID=c933fefbe893411d3be8e1648d6bcf37 t=2024-07-10T16:00:15.564896897Z level=debug msg="query finished" status=success elapsedtime=1.324305ms error=null`, want: []string{"logger", "sqlstore.metrics", "traceID", "<_>", "t", "<_>", "level", "debug", "msg", "query finished", "status", "success", "elapsedtime", "1.324305ms", "", "", "error", "null"}, }, + { + line: `ts=2024-05-30T12:50:36.648377186Z caller=scheduler_processor.go:143 level=warn msg="error contacting scheduler" err="rpc error: code = Unavailable desc = connection error: desc = \"error reading server preface: EOF\"" addr=10.0.151.101:9095 ip=127.0.0.1 userid=1234456`, + want: []string(nil), + }, } - tokenizer := newLogfmtTokenizer(param) + tokenizer := newLogfmtTokenizer(param, 250) for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -268,7 +280,7 @@ func TestLogFmtTokenizerJoin(t *testing.T) { }, } - tokenizer := newLogfmtTokenizer("") + tokenizer := newLogfmtTokenizer("", DefaultConfig().MaxAllowedLineLength) for _, tt := range tests { t.Run("", func(t *testing.T) { @@ -306,16 +318,23 @@ func TestJsonTokenizer(t *testing.T) { want: []string{"successfully", "discovered", "15", "agent", "IP", "addresses"}, pattern: "<_>successfully discovered 15 agent IP addresses<_>", }, + { + line: `{"msg":{"actor":{"alternateId":"foo@grafana.com","displayName":"Foo bar","id":"dq23","type":"User"},"authenticationContext":{"authenticationStep":0,"externalSessionId":"123d"},"client":{"device":"Computer","geographicalContext":{"city":"Berlin","country":"DE","state":"Land Berlin"},"ipAddress":"0.0.0.0","userAgent":{"browser":"CHROME","os":"Mac OS X","rawUserAgent":"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36"},"zone":"null"},"debugContext":{"debugData":{"authMethodFirstEnrollment":"123","authMethodFirstType":"foo","authMethodFirstVerificationTime":"2024-07-02T11:28:03.219Z","authMethodSecondEnrollment":"var","authMethodSecondType":"ddd","authMethodSecondVerificationTime":"2024-07-03T06:59:09.151Z","authnRequestId":"1","dtHash":"1","logOnlySecurityData":"{\"risk\":{\"level\":\"LOW\"},\"behaviors\":{\"New Geo-Location\":\"NEGATIVE\",\"New Device\":\"NEGATIVE\",\"New IP\":\"NEGATIVE\",\"New State\":\"NEGATIVE\",\"New Country\":\"NEGATIVE\",\"Velocity\":\"NEGATIVE\",\"New City\":\"NEGATIVE\"}}","requestId":"1","threatSuspected":"false","url":"/foo?"}},"displayMessage":"Evaluation of sign-on policy","eventType":"policy.evaluate_sign_on","legacyEventType":"app.oauth2.token.grant.refresh_token_success","outcome":{"reason":"Sign-on policy evaluation resulted in AUTHENTICATED","result":"ALLOW"},"published":"2024-07-03T09:19:59.973Z","request":{"ipChain":[{"geographicalContext":{"city":"Berlin","country":"Germany","geolocation":{"lat":52.5363,"lon":13.4169},"postalCode":"10435","state":"Land Berlin"},"ip":"95.90.234.241","version":"V4"}]},"securityContext":{"asNumber":3209,"asOrg":"kabel deutschland breitband customer 19","domain":"kabel-deutschland.de","isProxy":false,"isp":"vodafone gmbh"},"severity":"INFO","target":[{"alternateId":"Salesforce.com","detailEntry":{"signOnModeEvaluationResult":"AUTHENTICATED","signOnModeType":"SAML_2_0"},"displayName":"Salesforce.com","id":"0oa5sfmj3hz0mTgoW357","type":"AppInstance"},{"alternateId":"unknown","detailEntry":{"policyRuleFactorMode":"2FA"},"displayName":"Catch-all Rule","id":"1","type":"Rule"}],"transaction":{"detail":{},"id":"1","type":"WEB"},"context":[{"repo":{"id":27826205,"name":"hermanwahyudi/selenium","url":"https://api.github.com/repos/hermanwahyudi/selenium"},"payload":{"push_id":536863976,"size":1,"distinct_size":0,"ref":"refs/heads/master","head":"1b58dd4c4e14ea9cf5212b981774bd448a266c3c","before":"20b10e3a605bd177efff62f1130943774ac07bf3","commits":[{"sha":"1b58dd4c4e14ea9cf5212b981774bd448a266c3c","author":{"email":"2bb20d8a71fb7adbc1d6239cc9ff4130f26819dc@gmail.com","name":"Herman"},"message":"Update README.md","distinct":false,"url":"https://api.github.com/repos/hermanwahyudi/selenium/commits/1b58dd4c4e14ea9cf5212b981774bd448a266c3c"}]}},{"repo":{"id":27826205,"name":"hermanwahyudi/selenium","url":"https://api.github.com/repos/hermanwahyudi/selenium"},"payload":{"push_id":536863976,"size":1,"distinct_size":0,"ref":"refs/heads/master","head":"1b58dd4c4e14ea9cf5212b981774bd448a266c3c","before":"20b10e3a605bd177efff62f1130943774ac07bf3","commits":[{"sha":"1b58dd4c4e14ea9cf5212b981774bd448a266c3c","author":{"email":"2bb20d8a71fb7adbc1d6239cc9ff4130f26819dc@gmail.com","name":"Herman"},"message":"Update README.md","distinct":false,"url":"https://api.github.com/repos/hermanwahyudi/selenium/commits/1b58dd4c4e14ea9cf5212b981774bd448a266c3c"}]}}],"uuid":"1","version":"0"},"level":"info","type":"received event","time":"2024-07-03T09:19:59Z"}`, + want: []string(nil), + pattern: "", + }, } - tokenizer := newJSONTokenizer(param) + tokenizer := newJSONTokenizer(param, DefaultConfig().MaxAllowedLineLength) for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { got, state := tokenizer.Tokenize(tt.line, nil, nil) require.Equal(t, tt.want, got) - pattern := tokenizer.Join(got, state) - require.Equal(t, tt.pattern, pattern) + if len(got) == len(tt.want) && len(tt.want) != 0 { + pattern := tokenizer.Join(got, state) + require.Equal(t, tt.pattern, pattern) + } }) } } From d7ff42664681794b9ef5026ac3758cdd9569ac1a Mon Sep 17 00:00:00 2001 From: Trevor Whitney Date: Wed, 2 Oct 2024 09:17:38 -0600 Subject: [PATCH 06/10] feat: ability to log stream selectors before service name detection (#14154) --- .../promtail/targets/lokipush/pushtarget.go | 2 +- pkg/distributor/http.go | 5 +-- pkg/distributor/http_test.go | 11 ++++++- pkg/loghttp/push/otlp.go | 33 +++++++++++++++++-- pkg/loghttp/push/otlp_test.go | 14 +++++++- pkg/loghttp/push/push.go | 23 ++++++++++--- pkg/loghttp/push/push_test.go | 19 ++++++++--- 7 files changed, 89 insertions(+), 18 deletions(-) diff --git a/clients/pkg/promtail/targets/lokipush/pushtarget.go b/clients/pkg/promtail/targets/lokipush/pushtarget.go index 1ec021c0b28a8..f6e33eb8f72d9 100644 --- a/clients/pkg/promtail/targets/lokipush/pushtarget.go +++ b/clients/pkg/promtail/targets/lokipush/pushtarget.go @@ -111,7 +111,7 @@ func (t *PushTarget) run() error { func (t *PushTarget) handleLoki(w http.ResponseWriter, r *http.Request) { logger := util_log.WithContext(r.Context(), util_log.Logger) userID, _ := tenant.TenantID(r.Context()) - req, err := push.ParseRequest(logger, userID, r, nil, push.EmptyLimits{}, push.ParseLokiRequest, nil) + req, err := push.ParseRequest(logger, userID, r, nil, push.EmptyLimits{}, push.ParseLokiRequest, nil, false) if err != nil { level.Warn(t.logger).Log("msg", "failed to parse incoming push request", "err", err.Error()) http.Error(w, err.Error(), http.StatusBadRequest) diff --git a/pkg/distributor/http.go b/pkg/distributor/http.go index ec0660b91bc01..636a16bb507b1 100644 --- a/pkg/distributor/http.go +++ b/pkg/distributor/http.go @@ -58,7 +58,8 @@ func (d *Distributor) pushHandler(w http.ResponseWriter, r *http.Request, pushRe pushRequestParser = d.RequestParserWrapper(pushRequestParser) } - req, err := push.ParseRequest(logger, tenantID, r, d.tenantsRetention, d.validator.Limits, pushRequestParser, d.usageTracker) + logPushRequestStreams := d.tenantConfigs.LogPushRequestStreams(tenantID) + req, err := push.ParseRequest(logger, tenantID, r, d.tenantsRetention, d.validator.Limits, pushRequestParser, d.usageTracker, logPushRequestStreams) if err != nil { if d.tenantConfigs.LogPushRequest(tenantID) { level.Debug(logger).Log( @@ -73,7 +74,7 @@ func (d *Distributor) pushHandler(w http.ResponseWriter, r *http.Request, pushRe return } - if d.tenantConfigs.LogPushRequestStreams(tenantID) { + if logPushRequestStreams { var sb strings.Builder for _, s := range req.Streams { sb.WriteString(s.Labels) diff --git a/pkg/distributor/http_test.go b/pkg/distributor/http_test.go index b6281b81bf3d7..c6b8f3d017af6 100644 --- a/pkg/distributor/http_test.go +++ b/pkg/distributor/http_test.go @@ -7,6 +7,7 @@ import ( "net/http/httptest" "testing" + "github.com/go-kit/log" "github.com/grafana/dskit/user" "github.com/grafana/loki/v3/pkg/loghttp/push" @@ -114,6 +115,14 @@ func Test_OtelErrorHeaderInterceptor(t *testing.T) { } } -func stubParser(_ string, _ *http.Request, _ push.TenantsRetention, _ push.Limits, _ push.UsageTracker) (*logproto.PushRequest, *push.Stats, error) { +func stubParser( + _ string, + _ *http.Request, + _ push.TenantsRetention, + _ push.Limits, + _ push.UsageTracker, + _ bool, + _ log.Logger, +) (*logproto.PushRequest, *push.Stats, error) { return &logproto.PushRequest{}, &push.Stats{}, nil } diff --git a/pkg/loghttp/push/otlp.go b/pkg/loghttp/push/otlp.go index 13aea9ee59caa..3e654b9c21ef2 100644 --- a/pkg/loghttp/push/otlp.go +++ b/pkg/loghttp/push/otlp.go @@ -8,8 +8,11 @@ import ( "io" "net/http" "sort" + "strings" "time" + "github.com/go-kit/log" + "github.com/go-kit/log/level" "github.com/pkg/errors" "github.com/prometheus/common/model" "github.com/prometheus/prometheus/model/labels" @@ -40,14 +43,14 @@ func newPushStats() *Stats { } } -func ParseOTLPRequest(userID string, r *http.Request, tenantsRetention TenantsRetention, limits Limits, tracker UsageTracker) (*logproto.PushRequest, *Stats, error) { +func ParseOTLPRequest(userID string, r *http.Request, tenantsRetention TenantsRetention, limits Limits, tracker UsageTracker, logPushRequestStreams bool, logger log.Logger) (*logproto.PushRequest, *Stats, error) { stats := newPushStats() otlpLogs, err := extractLogs(r, stats) if err != nil { return nil, nil, err } - req := otlpToLokiPushRequest(r.Context(), otlpLogs, userID, tenantsRetention, limits.OTLPConfig(userID), limits.DiscoverServiceName(userID), tracker, stats) + req := otlpToLokiPushRequest(r.Context(), otlpLogs, userID, tenantsRetention, limits.OTLPConfig(userID), limits.DiscoverServiceName(userID), tracker, stats, logPushRequestStreams, logger) return req, stats, nil } @@ -98,7 +101,7 @@ func extractLogs(r *http.Request, pushStats *Stats) (plog.Logs, error) { return req.Logs(), nil } -func otlpToLokiPushRequest(ctx context.Context, ld plog.Logs, userID string, tenantsRetention TenantsRetention, otlpConfig OTLPConfig, discoverServiceName []string, tracker UsageTracker, stats *Stats) *logproto.PushRequest { +func otlpToLokiPushRequest(ctx context.Context, ld plog.Logs, userID string, tenantsRetention TenantsRetention, otlpConfig OTLPConfig, discoverServiceName []string, tracker UsageTracker, stats *Stats, logPushRequestStreams bool, logger log.Logger) *logproto.PushRequest { if ld.LogRecordCount() == 0 { return &logproto.PushRequest{} } @@ -113,6 +116,10 @@ func otlpToLokiPushRequest(ctx context.Context, ld plog.Logs, userID string, ten resourceAttributesAsStructuredMetadata := make(push.LabelsAdapter, 0, resAttrs.Len()) streamLabels := make(model.LabelSet, 30) // we have a default labels limit of 30 so just initialize the map of same size + var pushedLabels model.LabelSet + if logPushRequestStreams { + pushedLabels = make(model.LabelSet, 30) + } shouldDiscoverServiceName := len(discoverServiceName) > 0 && !stats.IsAggregatedMetric hasServiceName := false @@ -129,6 +136,9 @@ func otlpToLokiPushRequest(ctx context.Context, ld plog.Logs, userID string, ten if action == IndexLabel { for _, lbl := range attributeAsLabels { streamLabels[model.LabelName(lbl.Name)] = model.LabelValue(lbl.Value) + if logPushRequestStreams && pushedLabels != nil { + pushedLabels[model.LabelName(lbl.Name)] = model.LabelValue(lbl.Value) + } if !hasServiceName && shouldDiscoverServiceName { for _, labelName := range discoverServiceName { @@ -151,6 +161,23 @@ func otlpToLokiPushRequest(ctx context.Context, ld plog.Logs, userID string, ten streamLabels[model.LabelName(LabelServiceName)] = model.LabelValue(ServiceUnknown) } + if logPushRequestStreams { + var sb strings.Builder + sb.WriteString("{") + labels := make([]string, 0, len(pushedLabels)) + for name, value := range pushedLabels { + labels = append(labels, fmt.Sprintf(`%s="%s"`, name, value)) + } + sb.WriteString(strings.Join(labels, ", ")) + sb.WriteString("}") + + level.Debug(logger).Log( + "msg", "OTLP push request stream before service name discovery", + "stream", sb.String(), + "service_name", streamLabels[model.LabelName(LabelServiceName)], + ) + } + if err := streamLabels.Validate(); err != nil { stats.Errs = append(stats.Errs, fmt.Errorf("invalid labels: %w", err)) continue diff --git a/pkg/loghttp/push/otlp_test.go b/pkg/loghttp/push/otlp_test.go index e2ca137f274c0..5e5632eec0082 100644 --- a/pkg/loghttp/push/otlp_test.go +++ b/pkg/loghttp/push/otlp_test.go @@ -7,6 +7,7 @@ import ( "testing" "time" + "github.com/go-kit/log" "github.com/prometheus/prometheus/model/labels" "github.com/prometheus/prometheus/model/relabel" "github.com/stretchr/testify/require" @@ -508,7 +509,18 @@ func TestOTLPToLokiPushRequest(t *testing.T) { t.Run(tc.name, func(t *testing.T) { stats := newPushStats() tracker := NewMockTracker() - pushReq := otlpToLokiPushRequest(context.Background(), tc.generateLogs(), "foo", fakeRetention{}, tc.otlpConfig, defaultServiceDetection, tracker, stats) + pushReq := otlpToLokiPushRequest( + context.Background(), + tc.generateLogs(), + "foo", + fakeRetention{}, + tc.otlpConfig, + defaultServiceDetection, + tracker, + stats, + false, + log.NewNopLogger(), + ) require.Equal(t, tc.expectedPushRequest, *pushReq) require.Equal(t, tc.expectedStats, *stats) diff --git a/pkg/loghttp/push/push.go b/pkg/loghttp/push/push.go index e048546fb4083..be1d8b34b9f31 100644 --- a/pkg/loghttp/push/push.go +++ b/pkg/loghttp/push/push.go @@ -84,7 +84,7 @@ func (EmptyLimits) DiscoverServiceName(string) []string { } type ( - RequestParser func(userID string, r *http.Request, tenantsRetention TenantsRetention, limits Limits, tracker UsageTracker) (*logproto.PushRequest, *Stats, error) + RequestParser func(userID string, r *http.Request, tenantsRetention TenantsRetention, limits Limits, tracker UsageTracker, logPushRequestStreams bool, logger log.Logger) (*logproto.PushRequest, *Stats, error) RequestParserWrapper func(inner RequestParser) RequestParser ) @@ -106,8 +106,8 @@ type Stats struct { IsAggregatedMetric bool } -func ParseRequest(logger log.Logger, userID string, r *http.Request, tenantsRetention TenantsRetention, limits Limits, pushRequestParser RequestParser, tracker UsageTracker) (*logproto.PushRequest, error) { - req, pushStats, err := pushRequestParser(userID, r, tenantsRetention, limits, tracker) +func ParseRequest(logger log.Logger, userID string, r *http.Request, tenantsRetention TenantsRetention, limits Limits, pushRequestParser RequestParser, tracker UsageTracker, logPushRequestStreams bool) (*logproto.PushRequest, error) { + req, pushStats, err := pushRequestParser(userID, r, tenantsRetention, limits, tracker, logPushRequestStreams, logger) if err != nil { return nil, err } @@ -164,7 +164,7 @@ func ParseRequest(logger log.Logger, userID string, r *http.Request, tenantsRete return req, nil } -func ParseLokiRequest(userID string, r *http.Request, tenantsRetention TenantsRetention, limits Limits, tracker UsageTracker) (*logproto.PushRequest, *Stats, error) { +func ParseLokiRequest(userID string, r *http.Request, tenantsRetention TenantsRetention, limits Limits, tracker UsageTracker, logPushRequestStreams bool, logger log.Logger) (*logproto.PushRequest, *Stats, error) { // Body var body io.Reader // bodySize should always reflect the compressed size of the request body @@ -247,8 +247,13 @@ func ParseLokiRequest(userID string, r *http.Request, tenantsRetention TenantsRe pushStats.IsAggregatedMetric = true } + var beforeServiceName string + if logPushRequestStreams { + beforeServiceName = lbs.String() + } + + serviceName := ServiceUnknown if !lbs.Has(LabelServiceName) && len(discoverServiceName) > 0 && !pushStats.IsAggregatedMetric { - serviceName := ServiceUnknown for _, labelName := range discoverServiceName { if labelVal := lbs.Get(labelName); labelVal != "" { serviceName = labelVal @@ -264,6 +269,14 @@ func ParseLokiRequest(userID string, r *http.Request, tenantsRetention TenantsRe lbs = lb.Del(LabelServiceName).Labels() } + if logPushRequestStreams { + level.Debug(logger).Log( + "msg", "push request stream before service name discovery", + "labels", beforeServiceName, + "service_name", serviceName, + ) + } + var retentionPeriod time.Duration if tenantsRetention != nil { retentionPeriod = tenantsRetention.RetentionPeriodFor(userID, lbs) diff --git a/pkg/loghttp/push/push_test.go b/pkg/loghttp/push/push_test.go index e63b2c873c8de..b5609dec57a69 100644 --- a/pkg/loghttp/push/push_test.go +++ b/pkg/loghttp/push/push_test.go @@ -262,7 +262,16 @@ func TestParseRequest(t *testing.T) { } tracker := NewMockTracker() - data, err := ParseRequest(util_log.Logger, "fake", request, nil, &fakeLimits{enabled: test.enableServiceDiscovery}, ParseLokiRequest, tracker) + data, err := ParseRequest( + util_log.Logger, + "fake", + request, + nil, + &fakeLimits{enabled: test.enableServiceDiscovery}, + ParseLokiRequest, + tracker, + false, + ) structuredMetadataBytesReceived := int(structuredMetadataBytesReceivedStats.Value()["total"].(int64)) - previousStructuredMetadataBytesReceived previousStructuredMetadataBytesReceived += structuredMetadataBytesReceived @@ -355,7 +364,7 @@ func Test_ServiceDetection(t *testing.T) { request := createRequest("/loki/api/v1/push", strings.NewReader(body)) limits := &fakeLimits{enabled: true, labels: []string{"foo"}} - data, err := ParseRequest(util_log.Logger, "fake", request, nil, limits, ParseLokiRequest, tracker) + data, err := ParseRequest(util_log.Logger, "fake", request, nil, limits, ParseLokiRequest, tracker, false) require.NoError(t, err) require.Equal(t, labels.FromStrings("foo", "bar", LabelServiceName, "bar").String(), data.Streams[0].Labels) @@ -366,7 +375,7 @@ func Test_ServiceDetection(t *testing.T) { request := createRequest("/otlp/v1/push", bytes.NewReader(body)) limits := &fakeLimits{enabled: true} - data, err := ParseRequest(util_log.Logger, "fake", request, limits, limits, ParseOTLPRequest, tracker) + data, err := ParseRequest(util_log.Logger, "fake", request, limits, limits, ParseOTLPRequest, tracker, false) require.NoError(t, err) require.Equal(t, labels.FromStrings("k8s_job_name", "bar", LabelServiceName, "bar").String(), data.Streams[0].Labels) }) @@ -380,7 +389,7 @@ func Test_ServiceDetection(t *testing.T) { labels: []string{"special"}, indexAttributes: []string{"special"}, } - data, err := ParseRequest(util_log.Logger, "fake", request, limits, limits, ParseOTLPRequest, tracker) + data, err := ParseRequest(util_log.Logger, "fake", request, limits, limits, ParseOTLPRequest, tracker, false) require.NoError(t, err) require.Equal(t, labels.FromStrings("special", "sauce", LabelServiceName, "sauce").String(), data.Streams[0].Labels) }) @@ -394,7 +403,7 @@ func Test_ServiceDetection(t *testing.T) { labels: []string{"special"}, indexAttributes: []string{}, } - data, err := ParseRequest(util_log.Logger, "fake", request, limits, limits, ParseOTLPRequest, tracker) + data, err := ParseRequest(util_log.Logger, "fake", request, limits, limits, ParseOTLPRequest, tracker, false) require.NoError(t, err) require.Equal(t, labels.FromStrings(LabelServiceName, ServiceUnknown).String(), data.Streams[0].Labels) }) From d0e3ef709a222821fd764f6af72308c302faefb3 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 2 Oct 2024 11:33:25 -0400 Subject: [PATCH 07/10] fix(deps): update module github.com/minio/minio-go/v7 to v7.0.77 (#14353) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 +- .../minio-go/v7/api-put-object-streaming.go | 49 ++-- .../minio/minio-go/v7/api-put-object.go | 28 +- .../minio-go/v7/api-putobject-snowball.go | 2 +- vendor/github.com/minio/minio-go/v7/api.go | 4 +- .../minio/minio-go/v7/functional_tests.go | 253 +++++++++++++++--- .../minio/minio-go/v7/post-policy.go | 19 ++ vendor/github.com/minio/minio-go/v7/retry.go | 5 +- vendor/modules.txt | 2 +- 10 files changed, 305 insertions(+), 63 deletions(-) diff --git a/go.mod b/go.mod index 62887962ff38b..a35443eded513 100644 --- a/go.mod +++ b/go.mod @@ -67,7 +67,7 @@ require ( github.com/klauspost/pgzip v1.2.6 github.com/leodido/go-syslog/v4 v4.1.0 github.com/mattn/go-ieproxy v0.0.12 - github.com/minio/minio-go/v7 v7.0.76 + github.com/minio/minio-go/v7 v7.0.77 github.com/mitchellh/go-wordwrap v1.0.1 github.com/mitchellh/mapstructure v1.5.0 github.com/modern-go/reflect2 v1.0.2 diff --git a/go.sum b/go.sum index 7fbf5705ba189..2d2062153ef44 100644 --- a/go.sum +++ b/go.sum @@ -1414,8 +1414,8 @@ github.com/miekg/dns v1.1.61/go.mod h1:mnAarhS3nWaW+NVP2wTkYVIZyHNJ098SJZUki3eyk github.com/mikioh/ipaddr v0.0.0-20190404000644-d465c8ab6721/go.mod h1:Ickgr2WtCLZ2MDGd4Gr0geeCH5HybhRJbonOgQpvSxc= github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= -github.com/minio/minio-go/v7 v7.0.76 h1:9nxHH2XDai61cT/EFhyIw/wW4vJfpPNvl7lSFpRt+Ng= -github.com/minio/minio-go/v7 v7.0.76/go.mod h1:AVM3IUN6WwKzmwBxVdjzhH8xq+f57JSbbvzqvUzR6eg= +github.com/minio/minio-go/v7 v7.0.77 h1:GaGghJRg9nwDVlNbwYjSDJT1rqltQkBFDsypWX1v3Bw= +github.com/minio/minio-go/v7 v7.0.77/go.mod h1:AVM3IUN6WwKzmwBxVdjzhH8xq+f57JSbbvzqvUzR6eg= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/cli v1.1.0/go.mod h1:xcISNoH86gajksDmfB23e/pu+B+GeFRMYmoHXxx3xhI= github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db h1:62I3jR2EmQ4l5rM/4FEfDWcRD+abF5XlKShorW5LRoQ= diff --git a/vendor/github.com/minio/minio-go/v7/api-put-object-streaming.go b/vendor/github.com/minio/minio-go/v7/api-put-object-streaming.go index 7f316564b3ab5..eef976c8c40c6 100644 --- a/vendor/github.com/minio/minio-go/v7/api-put-object-streaming.go +++ b/vendor/github.com/minio/minio-go/v7/api-put-object-streaming.go @@ -108,7 +108,9 @@ func (c *Client) putObjectMultipartStreamFromReadAt(ctx context.Context, bucketN if err != nil { return UploadInfo{}, err } - + if opts.Checksum.IsSet() { + opts.AutoChecksum = opts.Checksum + } withChecksum := c.trailingHeaderSupport if withChecksum { if opts.UserMetadata == nil { @@ -304,6 +306,11 @@ func (c *Client) putObjectMultipartStreamOptionalChecksum(ctx context.Context, b return UploadInfo{}, err } + if opts.Checksum.IsSet() { + opts.AutoChecksum = opts.Checksum + opts.SendContentMd5 = false + } + if !opts.SendContentMd5 { if opts.UserMetadata == nil { opts.UserMetadata = make(map[string]string, 1) @@ -463,7 +470,10 @@ func (c *Client) putObjectMultipartStreamParallel(ctx context.Context, bucketNam if err = s3utils.CheckValidObjectName(objectName); err != nil { return UploadInfo{}, err } - + if opts.Checksum.IsSet() { + opts.SendContentMd5 = false + opts.AutoChecksum = opts.Checksum + } if !opts.SendContentMd5 { if opts.UserMetadata == nil { opts.UserMetadata = make(map[string]string, 1) @@ -555,7 +565,7 @@ func (c *Client) putObjectMultipartStreamParallel(ctx context.Context, bucketNam // Calculate md5sum. customHeader := make(http.Header) if !opts.SendContentMd5 { - // Add CRC32C instead. + // Add Checksum instead. crc.Reset() crc.Write(buf[:length]) cSum := crc.Sum(nil) @@ -677,6 +687,9 @@ func (c *Client) putObject(ctx context.Context, bucketName, objectName string, r if opts.SendContentMd5 && s3utils.IsGoogleEndpoint(*c.endpointURL) && size < 0 { return UploadInfo{}, errInvalidArgument("MD5Sum cannot be calculated with size '-1'") } + if opts.Checksum.IsSet() { + opts.SendContentMd5 = false + } var readSeeker io.Seeker if size > 0 { @@ -746,17 +759,6 @@ func (c *Client) putObjectDo(ctx context.Context, bucketName, objectName string, // Set headers. customHeader := opts.Header() - // Add CRC when client supports it, MD5 is not set, not Google and we don't add SHA256 to chunks. - addCrc := c.trailingHeaderSupport && md5Base64 == "" && !s3utils.IsGoogleEndpoint(*c.endpointURL) && (opts.DisableContentSha256 || c.secure) - - if addCrc { - // If user has added checksums, don't add them ourselves. - for k := range opts.UserMetadata { - if strings.HasPrefix(strings.ToLower(k), "x-amz-checksum-") { - addCrc = false - } - } - } // Populate request metadata. reqMetadata := requestMetadata{ bucketName: bucketName, @@ -768,10 +770,23 @@ func (c *Client) putObjectDo(ctx context.Context, bucketName, objectName string, contentSHA256Hex: sha256Hex, streamSha256: !opts.DisableContentSha256, } - if addCrc { - opts.AutoChecksum.SetDefault(ChecksumCRC32C) - reqMetadata.addCrc = &opts.AutoChecksum + // Add CRC when client supports it, MD5 is not set, not Google and we don't add SHA256 to chunks. + addCrc := c.trailingHeaderSupport && md5Base64 == "" && !s3utils.IsGoogleEndpoint(*c.endpointURL) && (opts.DisableContentSha256 || c.secure) + if opts.Checksum.IsSet() { + reqMetadata.addCrc = &opts.Checksum + } else if addCrc { + // If user has added checksums, don't add them ourselves. + for k := range opts.UserMetadata { + if strings.HasPrefix(strings.ToLower(k), "x-amz-checksum-") { + addCrc = false + } + } + if addCrc { + opts.AutoChecksum.SetDefault(ChecksumCRC32C) + reqMetadata.addCrc = &opts.AutoChecksum + } } + if opts.Internal.SourceVersionID != "" { if opts.Internal.SourceVersionID != nullVersionID { if _, err := uuid.Parse(opts.Internal.SourceVersionID); err != nil { diff --git a/vendor/github.com/minio/minio-go/v7/api-put-object.go b/vendor/github.com/minio/minio-go/v7/api-put-object.go index a792cfe390ce3..d769648a7ef34 100644 --- a/vendor/github.com/minio/minio-go/v7/api-put-object.go +++ b/vendor/github.com/minio/minio-go/v7/api-put-object.go @@ -94,6 +94,13 @@ type PutObjectOptions struct { // If none is specified CRC32C is used, since it is generally the fastest. AutoChecksum ChecksumType + // Checksum will force a checksum of the specific type. + // This requires that the client was created with "TrailingHeaders:true" option, + // and that the destination server supports it. + // Unavailable with V2 signatures & Google endpoints. + // This will disable content MD5 checksums if set. + Checksum ChecksumType + // ConcurrentStreamParts will create NumThreads buffers of PartSize bytes, // fill them serially and upload them in parallel. // This can be used for faster uploads on non-seekable or slow-to-seek input. @@ -240,7 +247,7 @@ func (opts PutObjectOptions) Header() (header http.Header) { } // validate() checks if the UserMetadata map has standard headers or and raises an error if so. -func (opts PutObjectOptions) validate() (err error) { +func (opts PutObjectOptions) validate(c *Client) (err error) { for k, v := range opts.UserMetadata { if !httpguts.ValidHeaderFieldName(k) || isStandardHeader(k) || isSSEHeader(k) || isStorageClassHeader(k) || isMinioHeader(k) { return errInvalidArgument(k + " unsupported user defined metadata name") @@ -255,6 +262,17 @@ func (opts PutObjectOptions) validate() (err error) { if opts.LegalHold != "" && !opts.LegalHold.IsValid() { return errInvalidArgument(opts.LegalHold.String() + " unsupported legal-hold status") } + if opts.Checksum.IsSet() { + switch { + case !c.trailingHeaderSupport: + return errInvalidArgument("Checksum requires Client with TrailingHeaders enabled") + case c.overrideSignerType.IsV2(): + return errInvalidArgument("Checksum cannot be used with v2 signatures") + case s3utils.IsGoogleEndpoint(*c.endpointURL): + return errInvalidArgument("Checksum cannot be used with GCS endpoints") + } + } + return nil } @@ -291,7 +309,7 @@ func (c *Client) PutObject(ctx context.Context, bucketName, objectName string, r return UploadInfo{}, errors.New("object size must be provided with disable multipart upload") } - err = opts.validate() + err = opts.validate(c) if err != nil { return UploadInfo{}, err } @@ -333,7 +351,7 @@ func (c *Client) putObjectCommon(ctx context.Context, bucketName, objectName str return c.putObjectMultipartStreamNoLength(ctx, bucketName, objectName, reader, opts) } - if size < int64(partSize) || opts.DisableMultipart { + if size <= int64(partSize) || opts.DisableMultipart { return c.putObject(ctx, bucketName, objectName, reader, size, opts) } @@ -362,6 +380,10 @@ func (c *Client) putObjectMultipartStreamNoLength(ctx context.Context, bucketNam return UploadInfo{}, err } + if opts.Checksum.IsSet() { + opts.SendContentMd5 = false + opts.AutoChecksum = opts.Checksum + } if !opts.SendContentMd5 { if opts.UserMetadata == nil { opts.UserMetadata = make(map[string]string, 1) diff --git a/vendor/github.com/minio/minio-go/v7/api-putobject-snowball.go b/vendor/github.com/minio/minio-go/v7/api-putobject-snowball.go index eb4da41471b83..6b6559bf76d03 100644 --- a/vendor/github.com/minio/minio-go/v7/api-putobject-snowball.go +++ b/vendor/github.com/minio/minio-go/v7/api-putobject-snowball.go @@ -107,7 +107,7 @@ type readSeekCloser interface { // Total size should be < 5TB. // This function blocks until 'objs' is closed and the content has been uploaded. func (c Client) PutObjectsSnowball(ctx context.Context, bucketName string, opts SnowballOptions, objs <-chan SnowballObject) (err error) { - err = opts.Opts.validate() + err = opts.Opts.validate(&c) if err != nil { return err } diff --git a/vendor/github.com/minio/minio-go/v7/api.go b/vendor/github.com/minio/minio-go/v7/api.go index be28e3fdfc41c..1d6b6650250a3 100644 --- a/vendor/github.com/minio/minio-go/v7/api.go +++ b/vendor/github.com/minio/minio-go/v7/api.go @@ -128,7 +128,7 @@ type Options struct { // Global constants. const ( libraryName = "minio-go" - libraryVersion = "v7.0.76" + libraryVersion = "v7.0.77" ) // User Agent should always following the below style. @@ -661,7 +661,7 @@ func (c *Client) executeMethod(ctx context.Context, method string, metadata requ // Initiate the request. res, err = c.do(req) if err != nil { - if isRequestErrorRetryable(err) { + if isRequestErrorRetryable(ctx, err) { // Retry the request continue } diff --git a/vendor/github.com/minio/minio-go/v7/functional_tests.go b/vendor/github.com/minio/minio-go/v7/functional_tests.go index 8a908e3fd8fa1..780dc899795c8 100644 --- a/vendor/github.com/minio/minio-go/v7/functional_tests.go +++ b/vendor/github.com/minio/minio-go/v7/functional_tests.go @@ -83,7 +83,7 @@ func createHTTPTransport() (transport *http.Transport) { return nil } - if mustParseBool(os.Getenv(skipCERTValidation)) { + if mustParseBool(os.Getenv(enableHTTPS)) && mustParseBool(os.Getenv(skipCERTValidation)) { transport.TLSClientConfig.InsecureSkipVerify = true } @@ -2334,7 +2334,7 @@ func testPutObjectWithChecksums() { } // Test PutObject with custom checksums. -func testPutMultipartObjectWithChecksums() { +func testPutObjectWithTrailingChecksums() { // initialize logging params startTime := time.Now() testName := getFuncName() @@ -2342,7 +2342,7 @@ func testPutMultipartObjectWithChecksums() { args := map[string]interface{}{ "bucketName": "", "objectName": "", - "opts": "minio.PutObjectOptions{UserMetadata: metadata, Progress: progress}", + "opts": "minio.PutObjectOptions{UserMetadata: metadata, Progress: progress, TrailChecksum: xxx}", } if !isFullMode() { @@ -2356,9 +2356,201 @@ func testPutMultipartObjectWithChecksums() { // Instantiate new minio client object. c, err := minio.New(os.Getenv(serverEndpoint), &minio.Options{ - Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), - Transport: createHTTPTransport(), - Secure: mustParseBool(os.Getenv(enableHTTPS)), + Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Transport: createHTTPTransport(), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + TrailingHeaders: true, + }) + if err != nil { + logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) + return + } + + // Enable tracing, write to stderr. + // c.TraceOn(os.Stderr) + + // Set user agent. + c.SetAppInfo("MinIO-go-FunctionalTest", appVersion) + + // Generate a new random bucket name. + bucketName := randString(60, rand.NewSource(time.Now().UnixNano()), "minio-go-test-") + args["bucketName"] = bucketName + + // Make a new bucket. + err = c.MakeBucket(context.Background(), bucketName, minio.MakeBucketOptions{Region: "us-east-1"}) + if err != nil { + logError(testName, function, args, startTime, "", "Make bucket failed", err) + return + } + + defer cleanupBucket(bucketName, c) + tests := []struct { + cs minio.ChecksumType + }{ + {cs: minio.ChecksumCRC32C}, + {cs: minio.ChecksumCRC32}, + {cs: minio.ChecksumSHA1}, + {cs: minio.ChecksumSHA256}, + } + + for _, test := range tests { + function := "PutObject(bucketName, objectName, reader,size, opts)" + bufSize := dataFileMap["datafile-10-kB"] + + // Save the data + objectName := randString(60, rand.NewSource(time.Now().UnixNano()), "") + args["objectName"] = objectName + + cmpChecksum := func(got, want string) { + if want != got { + logError(testName, function, args, startTime, "", "checksum mismatch", fmt.Errorf("want %s, got %s", want, got)) + return + } + } + + meta := map[string]string{} + reader := getDataReader("datafile-10-kB") + b, err := io.ReadAll(reader) + if err != nil { + logError(testName, function, args, startTime, "", "Read failed", err) + return + } + h := test.cs.Hasher() + h.Reset() + + // Test with Wrong CRC. + args["metadata"] = meta + args["range"] = "false" + args["checksum"] = test.cs.String() + + resp, err := c.PutObject(context.Background(), bucketName, objectName, bytes.NewReader(b), int64(bufSize), minio.PutObjectOptions{ + DisableMultipart: true, + DisableContentSha256: true, + UserMetadata: meta, + Checksum: test.cs, + }) + if err != nil { + logError(testName, function, args, startTime, "", "PutObject failed", err) + return + } + + h.Write(b) + meta[test.cs.Key()] = base64.StdEncoding.EncodeToString(h.Sum(nil)) + + cmpChecksum(resp.ChecksumSHA256, meta["x-amz-checksum-sha256"]) + cmpChecksum(resp.ChecksumSHA1, meta["x-amz-checksum-sha1"]) + cmpChecksum(resp.ChecksumCRC32, meta["x-amz-checksum-crc32"]) + cmpChecksum(resp.ChecksumCRC32C, meta["x-amz-checksum-crc32c"]) + + // Read the data back + gopts := minio.GetObjectOptions{Checksum: true} + + function = "GetObject(...)" + r, err := c.GetObject(context.Background(), bucketName, objectName, gopts) + if err != nil { + logError(testName, function, args, startTime, "", "GetObject failed", err) + return + } + + st, err := r.Stat() + if err != nil { + logError(testName, function, args, startTime, "", "Stat failed", err) + return + } + cmpChecksum(st.ChecksumSHA256, meta["x-amz-checksum-sha256"]) + cmpChecksum(st.ChecksumSHA1, meta["x-amz-checksum-sha1"]) + cmpChecksum(st.ChecksumCRC32, meta["x-amz-checksum-crc32"]) + cmpChecksum(st.ChecksumCRC32C, meta["x-amz-checksum-crc32c"]) + + if st.Size != int64(bufSize) { + logError(testName, function, args, startTime, "", "Number of bytes returned by PutObject does not match GetObject, expected "+string(bufSize)+" got "+string(st.Size), err) + return + } + + if err := r.Close(); err != nil { + logError(testName, function, args, startTime, "", "Object Close failed", err) + return + } + if err := r.Close(); err == nil { + logError(testName, function, args, startTime, "", "Object already closed, should respond with error", err) + return + } + + function = "GetObject( Range...)" + args["range"] = "true" + err = gopts.SetRange(100, 1000) + if err != nil { + logError(testName, function, args, startTime, "", "SetRange failed", err) + return + } + r, err = c.GetObject(context.Background(), bucketName, objectName, gopts) + if err != nil { + logError(testName, function, args, startTime, "", "GetObject failed", err) + return + } + + b, err = io.ReadAll(r) + if err != nil { + logError(testName, function, args, startTime, "", "Read failed", err) + return + } + st, err = r.Stat() + if err != nil { + logError(testName, function, args, startTime, "", "Stat failed", err) + return + } + + // Range requests should return empty checksums... + cmpChecksum(st.ChecksumSHA256, "") + cmpChecksum(st.ChecksumSHA1, "") + cmpChecksum(st.ChecksumCRC32, "") + cmpChecksum(st.ChecksumCRC32C, "") + + function = "GetObjectAttributes(...)" + s, err := c.GetObjectAttributes(context.Background(), bucketName, objectName, minio.ObjectAttributesOptions{}) + if err != nil { + logError(testName, function, args, startTime, "", "GetObjectAttributes failed", err) + return + } + cmpChecksum(s.Checksum.ChecksumSHA256, meta["x-amz-checksum-sha256"]) + cmpChecksum(s.Checksum.ChecksumSHA1, meta["x-amz-checksum-sha1"]) + cmpChecksum(s.Checksum.ChecksumCRC32, meta["x-amz-checksum-crc32"]) + cmpChecksum(s.Checksum.ChecksumCRC32C, meta["x-amz-checksum-crc32c"]) + + delete(args, "range") + delete(args, "metadata") + } + + logSuccess(testName, function, args, startTime) +} + +// Test PutObject with custom checksums. +func testPutMultipartObjectWithChecksums(trailing bool) { + // initialize logging params + startTime := time.Now() + testName := getFuncName() + function := "PutObject(bucketName, objectName, reader,size, opts)" + args := map[string]interface{}{ + "bucketName": "", + "objectName": "", + "opts": fmt.Sprintf("minio.PutObjectOptions{UserMetadata: metadata, Progress: progress Checksum: %v}", trailing), + } + + if !isFullMode() { + logIgnored(testName, function, args, startTime, "Skipping functional tests for short/quick runs") + return + } + + // Seed random based on current time. + rand.Seed(time.Now().Unix()) + + // Instantiate new minio client object. + c, err := minio.New(os.Getenv(serverEndpoint), + &minio.Options{ + Creds: credentials.NewStaticV4(os.Getenv(accessKey), os.Getenv(secretKey), ""), + Transport: createHTTPTransport(), + Secure: mustParseBool(os.Getenv(enableHTTPS)), + TrailingHeaders: trailing, }) if err != nil { logError(testName, function, args, startTime, "", "MinIO client object creation failed", err) @@ -2445,14 +2637,20 @@ func testPutMultipartObjectWithChecksums() { h.Reset() want := hashMultiPart(b, partSize, test.cs.Hasher()) + var cs minio.ChecksumType + rd := io.Reader(io.NopCloser(bytes.NewReader(b))) + if trailing { + cs = test.cs + rd = bytes.NewReader(b) + } // Set correct CRC. - - resp, err := c.PutObject(context.Background(), bucketName, objectName, io.NopCloser(bytes.NewReader(b)), int64(bufSize), minio.PutObjectOptions{ + resp, err := c.PutObject(context.Background(), bucketName, objectName, rd, int64(bufSize), minio.PutObjectOptions{ DisableContentSha256: true, DisableMultipart: false, UserMetadata: nil, PartSize: partSize, AutoChecksum: test.cs, + Checksum: cs, }) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) @@ -2982,6 +3180,7 @@ func testGetObjectAttributes() { testFiles[i].UploadInfo, err = c.PutObject(context.Background(), v.Bucket, v.Object, reader, int64(bufSize), minio.PutObjectOptions{ ContentType: v.ContentType, SendContentMd5: v.SendContentMd5, + Checksum: minio.ChecksumCRC32C, }) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) @@ -3063,7 +3262,7 @@ func testGetObjectAttributes() { test: objectAttributesTestOptions{ TestFileName: "file1", StorageClass: "STANDARD", - HasFullChecksum: false, + HasFullChecksum: true, }, } @@ -3152,9 +3351,10 @@ func testGetObjectAttributesSSECEncryption() { info, err := c.PutObject(context.Background(), bucketName, objectName, reader, int64(bufSize), minio.PutObjectOptions{ ContentType: "content/custom", - SendContentMd5: true, + SendContentMd5: false, ServerSideEncryption: sse, PartSize: uint64(bufSize) / 2, + Checksum: minio.ChecksumCRC32C, }) if err != nil { logError(testName, function, args, startTime, "", "PutObject failed", err) @@ -3174,9 +3374,9 @@ func testGetObjectAttributesSSECEncryption() { ETag: info.ETag, NumberOfParts: 2, ObjectSize: int(info.Size), - HasFullChecksum: false, + HasFullChecksum: true, HasParts: true, - HasPartChecksums: false, + HasPartChecksums: true, }) if err != nil { logError(testName, function, args, startTime, "", "Validating GetObjectsAttributes response failed", err) @@ -5594,18 +5794,12 @@ func testPresignedPostPolicy() { } writer.Close() - transport, err := minio.DefaultTransport(mustParseBool(os.Getenv(enableHTTPS))) - if err != nil { - logError(testName, function, args, startTime, "", "DefaultTransport failed", err) - return - } - httpClient := &http.Client{ // Setting a sensible time out of 30secs to wait for response // headers. Request is pro-actively canceled after 30secs // with no response. Timeout: 30 * time.Second, - Transport: transport, + Transport: createHTTPTransport(), } args["url"] = presignedPostPolicyURL.String() @@ -7519,7 +7713,7 @@ func testFunctional() { return } - transport, err := minio.DefaultTransport(mustParseBool(os.Getenv(enableHTTPS))) + transport := createHTTPTransport() if err != nil { logError(testName, function, args, startTime, "", "DefaultTransport failed", err) return @@ -12450,18 +12644,12 @@ func testFunctionalV2() { return } - transport, err := minio.DefaultTransport(mustParseBool(os.Getenv(enableHTTPS))) - if err != nil { - logError(testName, function, args, startTime, "", "DefaultTransport failed", err) - return - } - httpClient := &http.Client{ // Setting a sensible time out of 30secs to wait for response // headers. Request is pro-actively canceled after 30secs // with no response. Timeout: 30 * time.Second, - Transport: transport, + Transport: createHTTPTransport(), } req, err := http.NewRequest(http.MethodHead, presignedHeadURL.String(), nil) @@ -13556,14 +13744,9 @@ func testCors() { bucketURL := c.EndpointURL().String() + "/" + bucketName + "/" objectURL := bucketURL + objectName - transport, err := minio.DefaultTransport(mustParseBool(os.Getenv(enableHTTPS))) - if err != nil { - logError(testName, function, args, startTime, "", "DefaultTransport failed", err) - return - } httpClient := &http.Client{ Timeout: 30 * time.Second, - Transport: transport, + Transport: createHTTPTransport(), } errStrAccessForbidden := `AccessForbiddenCORSResponse: This CORS request is not allowed. This is usually because the evalution of Origin, request method / Access-Control-Request-Method or Access-Control-Request-Headers are not whitelisted` @@ -14757,7 +14940,9 @@ func main() { testCompose10KSourcesV2() testUserMetadataCopyingV2() testPutObjectWithChecksums() - testPutMultipartObjectWithChecksums() + testPutObjectWithTrailingChecksums() + testPutMultipartObjectWithChecksums(false) + testPutMultipartObjectWithChecksums(true) testPutObject0ByteV2() testPutObjectNoLengthV2() testPutObjectsUnknownV2() diff --git a/vendor/github.com/minio/minio-go/v7/post-policy.go b/vendor/github.com/minio/minio-go/v7/post-policy.go index 3f023704a46e2..19687e027d017 100644 --- a/vendor/github.com/minio/minio-go/v7/post-policy.go +++ b/vendor/github.com/minio/minio-go/v7/post-policy.go @@ -301,6 +301,25 @@ func (p *PostPolicy) SetUserMetadata(key, value string) error { return nil } +// SetUserMetadataStartsWith - Set how an user metadata should starts with. +// Can be retrieved through a HEAD request or an event. +func (p *PostPolicy) SetUserMetadataStartsWith(key, value string) error { + if strings.TrimSpace(key) == "" || key == "" { + return errInvalidArgument("Key is empty") + } + headerName := fmt.Sprintf("x-amz-meta-%s", key) + policyCond := policyCondition{ + matchType: "starts-with", + condition: fmt.Sprintf("$%s", headerName), + value: value, + } + if err := p.addNewPolicy(policyCond); err != nil { + return err + } + p.formData[headerName] = value + return nil +} + // SetChecksum sets the checksum of the request. func (p *PostPolicy) SetChecksum(c Checksum) { if c.IsSet() { diff --git a/vendor/github.com/minio/minio-go/v7/retry.go b/vendor/github.com/minio/minio-go/v7/retry.go index 5ddcad8976984..d15eb59013e38 100644 --- a/vendor/github.com/minio/minio-go/v7/retry.go +++ b/vendor/github.com/minio/minio-go/v7/retry.go @@ -129,9 +129,10 @@ func isHTTPStatusRetryable(httpStatusCode int) (ok bool) { } // For now, all http Do() requests are retriable except some well defined errors -func isRequestErrorRetryable(err error) bool { +func isRequestErrorRetryable(ctx context.Context, err error) bool { if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) { - return false + // Retry if internal timeout in the HTTP call. + return ctx.Err() == nil } if ue, ok := err.(*url.Error); ok { e := ue.Unwrap() diff --git a/vendor/modules.txt b/vendor/modules.txt index 5bd662067c5c5..2805571bfebf3 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1267,7 +1267,7 @@ github.com/miekg/dns # github.com/minio/md5-simd v1.1.2 ## explicit; go 1.14 github.com/minio/md5-simd -# github.com/minio/minio-go/v7 v7.0.76 +# github.com/minio/minio-go/v7 v7.0.77 ## explicit; go 1.21 github.com/minio/minio-go/v7 github.com/minio/minio-go/v7/pkg/cors From e23c5ed9fa97010ef4c985afea25af3922ca215b Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 2 Oct 2024 11:34:20 -0400 Subject: [PATCH 08/10] fix(deps): update module github.com/klauspost/compress to v1.17.10 (#14352) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 +- .../github.com/klauspost/compress/README.md | 22 +++++- .../klauspost/compress/flate/deflate.go | 2 +- .../klauspost/compress/flate/inflate.go | 74 ++++++++++++++----- .../klauspost/compress/fse/decompress.go | 2 +- .../klauspost/compress/huff0/decompress.go | 4 +- .../klauspost/compress/s2/writer.go | 31 +++++++- .../klauspost/compress/zstd/blockdec.go | 4 +- .../klauspost/compress/zstd/enc_better.go | 32 ++++---- .../klauspost/compress/zstd/enc_dfast.go | 16 ++-- .../klauspost/compress/zstd/encoder.go | 19 +++-- .../klauspost/compress/zstd/framedec.go | 4 +- .../klauspost/compress/zstd/seqdec_amd64.go | 4 +- .../klauspost/compress/zstd/seqdec_amd64.s | 8 +- vendor/modules.txt | 4 +- 16 files changed, 157 insertions(+), 75 deletions(-) diff --git a/go.mod b/go.mod index a35443eded513..2a11d7accb99e 100644 --- a/go.mod +++ b/go.mod @@ -63,7 +63,7 @@ require ( github.com/jmespath/go-jmespath v0.4.0 github.com/joncrlsn/dque v0.0.0-20211108142734-c2ef48c5192a github.com/json-iterator/go v1.1.12 - github.com/klauspost/compress v1.17.9 + github.com/klauspost/compress v1.17.10 github.com/klauspost/pgzip v1.2.6 github.com/leodido/go-syslog/v4 v4.1.0 github.com/mattn/go-ieproxy v0.0.12 diff --git a/go.sum b/go.sum index 2d2062153ef44..08cf0d839fa61 100644 --- a/go.sum +++ b/go.sum @@ -1304,8 +1304,8 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/klauspost/compress v1.9.5/go.mod h1:RyIbtBH6LamlWaDj8nUwkbUhJ87Yi3uG0guNDohfE1A= github.com/klauspost/compress v1.11.0/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYsUV+/s2qKfXs= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= -github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= +github.com/klauspost/compress v1.17.10 h1:oXAz+Vh0PMUvJczoi+flxpnBEPxoER1IaAnU/NMPtT0= +github.com/klauspost/compress v1.17.10/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0= github.com/klauspost/cpuid v1.2.0/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.2.8 h1:+StwCXwm9PdpiEkPyzBXIy+M9KUb4ODm0Zarf1kS5BM= diff --git a/vendor/github.com/klauspost/compress/README.md b/vendor/github.com/klauspost/compress/README.md index 05c7359e481fe..684a30853ab64 100644 --- a/vendor/github.com/klauspost/compress/README.md +++ b/vendor/github.com/klauspost/compress/README.md @@ -16,6 +16,20 @@ This package provides various compression algorithms. # changelog +* Jun 12th, 2024 - [1.17.9](https://github.com/klauspost/compress/releases/tag/v1.17.9) + * s2: Reduce ReadFrom temporary allocations https://github.com/klauspost/compress/pull/949 + * flate, zstd: Shave some bytes off amd64 matchLen by @greatroar in https://github.com/klauspost/compress/pull/963 + * Upgrade zip/zlib to 1.22.4 upstream https://github.com/klauspost/compress/pull/970 https://github.com/klauspost/compress/pull/971 + * zstd: BuildDict fails with RLE table https://github.com/klauspost/compress/pull/951 + +* Apr 9th, 2024 - [1.17.8](https://github.com/klauspost/compress/releases/tag/v1.17.8) + * zstd: Reject blocks where reserved values are not 0 https://github.com/klauspost/compress/pull/885 + * zstd: Add RLE detection+encoding https://github.com/klauspost/compress/pull/938 + +* Feb 21st, 2024 - [1.17.7](https://github.com/klauspost/compress/releases/tag/v1.17.7) + * s2: Add AsyncFlush method: Complete the block without flushing by @Jille in https://github.com/klauspost/compress/pull/927 + * s2: Fix literal+repeat exceeds dst crash https://github.com/klauspost/compress/pull/930 + * Feb 5th, 2024 - [1.17.6](https://github.com/klauspost/compress/releases/tag/v1.17.6) * zstd: Fix incorrect repeat coding in best mode https://github.com/klauspost/compress/pull/923 * s2: Fix DecodeConcurrent deadlock on errors https://github.com/klauspost/compress/pull/925 @@ -81,7 +95,7 @@ https://github.com/klauspost/compress/pull/919 https://github.com/klauspost/comp * zstd: Various minor improvements by @greatroar in https://github.com/klauspost/compress/pull/788 https://github.com/klauspost/compress/pull/794 https://github.com/klauspost/compress/pull/795 * s2: Fix huge block overflow https://github.com/klauspost/compress/pull/779 * s2: Allow CustomEncoder fallback https://github.com/klauspost/compress/pull/780 - * gzhttp: Suppport ResponseWriter Unwrap() in gzhttp handler by @jgimenez in https://github.com/klauspost/compress/pull/799 + * gzhttp: Support ResponseWriter Unwrap() in gzhttp handler by @jgimenez in https://github.com/klauspost/compress/pull/799 * Mar 13, 2023 - [v1.16.1](https://github.com/klauspost/compress/releases/tag/v1.16.1) * zstd: Speed up + improve best encoder by @greatroar in https://github.com/klauspost/compress/pull/776 @@ -136,7 +150,7 @@ https://github.com/klauspost/compress/pull/919 https://github.com/klauspost/comp * zstd: Add [WithDecodeAllCapLimit](https://pkg.go.dev/github.com/klauspost/compress@v1.15.10/zstd#WithDecodeAllCapLimit) https://github.com/klauspost/compress/pull/649 * Add Go 1.19 - deprecate Go 1.16 https://github.com/klauspost/compress/pull/651 * flate: Improve level 5+6 compression https://github.com/klauspost/compress/pull/656 - * zstd: Improve "better" compresssion https://github.com/klauspost/compress/pull/657 + * zstd: Improve "better" compression https://github.com/klauspost/compress/pull/657 * s2: Improve "best" compression https://github.com/klauspost/compress/pull/658 * s2: Improve "better" compression. https://github.com/klauspost/compress/pull/635 * s2: Slightly faster non-assembly decompression https://github.com/klauspost/compress/pull/646 @@ -339,7 +353,7 @@ While the release has been extensively tested, it is recommended to testing when * s2: Fix binaries. * Feb 25, 2021 (v1.11.8) - * s2: Fixed occational out-of-bounds write on amd64. Upgrade recommended. + * s2: Fixed occasional out-of-bounds write on amd64. Upgrade recommended. * s2: Add AMD64 assembly for better mode. 25-50% faster. [#315](https://github.com/klauspost/compress/pull/315) * s2: Less upfront decoder allocation. [#322](https://github.com/klauspost/compress/pull/322) * zstd: Faster "compression" of incompressible data. [#314](https://github.com/klauspost/compress/pull/314) @@ -518,7 +532,7 @@ While the release has been extensively tested, it is recommended to testing when * Feb 19, 2016: Faster bit writer, level -2 is 15% faster, level 1 is 4% faster. * Feb 19, 2016: Handle small payloads faster in level 1-3. * Feb 19, 2016: Added faster level 2 + 3 compression modes. -* Feb 19, 2016: [Rebalanced compression levels](https://blog.klauspost.com/rebalancing-deflate-compression-levels/), so there is a more even progresssion in terms of compression. New default level is 5. +* Feb 19, 2016: [Rebalanced compression levels](https://blog.klauspost.com/rebalancing-deflate-compression-levels/), so there is a more even progression in terms of compression. New default level is 5. * Feb 14, 2016: Snappy: Merge upstream changes. * Feb 14, 2016: Snappy: Fix aggressive skipping. * Feb 14, 2016: Snappy: Update benchmark. diff --git a/vendor/github.com/klauspost/compress/flate/deflate.go b/vendor/github.com/klauspost/compress/flate/deflate.go index 66d1657d2c6a7..af53fb860cc80 100644 --- a/vendor/github.com/klauspost/compress/flate/deflate.go +++ b/vendor/github.com/klauspost/compress/flate/deflate.go @@ -861,7 +861,7 @@ func (d *compressor) reset(w io.Writer) { } switch d.compressionLevel.chain { case 0: - // level was NoCompression or ConstantCompresssion. + // level was NoCompression or ConstantCompression. d.windowEnd = 0 default: s := d.state diff --git a/vendor/github.com/klauspost/compress/flate/inflate.go b/vendor/github.com/klauspost/compress/flate/inflate.go index 2f410d64f5aff..0d7b437f1c6f9 100644 --- a/vendor/github.com/klauspost/compress/flate/inflate.go +++ b/vendor/github.com/klauspost/compress/flate/inflate.go @@ -298,6 +298,14 @@ const ( huffmanGenericReader ) +// flushMode tells decompressor when to return data +type flushMode uint8 + +const ( + syncFlush flushMode = iota // return data after sync flush block + partialFlush // return data after each block +) + // Decompress state. type decompressor struct { // Input source. @@ -332,6 +340,8 @@ type decompressor struct { nb uint final bool + + flushMode flushMode } func (f *decompressor) nextBlock() { @@ -618,7 +628,10 @@ func (f *decompressor) dataBlock() { } if n == 0 { - f.toRead = f.dict.readFlush() + if f.flushMode == syncFlush { + f.toRead = f.dict.readFlush() + } + f.finishBlock() return } @@ -657,8 +670,12 @@ func (f *decompressor) finishBlock() { if f.dict.availRead() > 0 { f.toRead = f.dict.readFlush() } + f.err = io.EOF + } else if f.flushMode == partialFlush && f.dict.availRead() > 0 { + f.toRead = f.dict.readFlush() } + f.step = nextBlock } @@ -789,15 +806,25 @@ func (f *decompressor) Reset(r io.Reader, dict []byte) error { return nil } -// NewReader returns a new ReadCloser that can be used -// to read the uncompressed version of r. -// If r does not also implement io.ByteReader, -// the decompressor may read more data than necessary from r. -// It is the caller's responsibility to call Close on the ReadCloser -// when finished reading. -// -// The ReadCloser returned by NewReader also implements Resetter. -func NewReader(r io.Reader) io.ReadCloser { +type ReaderOpt func(*decompressor) + +// WithPartialBlock tells decompressor to return after each block, +// so it can read data written with partial flush +func WithPartialBlock() ReaderOpt { + return func(f *decompressor) { + f.flushMode = partialFlush + } +} + +// WithDict initializes the reader with a preset dictionary +func WithDict(dict []byte) ReaderOpt { + return func(f *decompressor) { + f.dict.init(maxMatchOffset, dict) + } +} + +// NewReaderOpts returns new reader with provided options +func NewReaderOpts(r io.Reader, opts ...ReaderOpt) io.ReadCloser { fixedHuffmanDecoderInit() var f decompressor @@ -806,9 +833,26 @@ func NewReader(r io.Reader) io.ReadCloser { f.codebits = new([numCodes]int) f.step = nextBlock f.dict.init(maxMatchOffset, nil) + + for _, opt := range opts { + opt(&f) + } + return &f } +// NewReader returns a new ReadCloser that can be used +// to read the uncompressed version of r. +// If r does not also implement io.ByteReader, +// the decompressor may read more data than necessary from r. +// It is the caller's responsibility to call Close on the ReadCloser +// when finished reading. +// +// The ReadCloser returned by NewReader also implements Resetter. +func NewReader(r io.Reader) io.ReadCloser { + return NewReaderOpts(r) +} + // NewReaderDict is like NewReader but initializes the reader // with a preset dictionary. The returned Reader behaves as if // the uncompressed data stream started with the given dictionary, @@ -817,13 +861,5 @@ func NewReader(r io.Reader) io.ReadCloser { // // The ReadCloser returned by NewReader also implements Resetter. func NewReaderDict(r io.Reader, dict []byte) io.ReadCloser { - fixedHuffmanDecoderInit() - - var f decompressor - f.r = makeReader(r) - f.bits = new([maxNumLit + maxNumDist]int) - f.codebits = new([numCodes]int) - f.step = nextBlock - f.dict.init(maxMatchOffset, dict) - return &f + return NewReaderOpts(r, WithDict(dict)) } diff --git a/vendor/github.com/klauspost/compress/fse/decompress.go b/vendor/github.com/klauspost/compress/fse/decompress.go index cc05d0f7ea908..0c7dd4ffef9fb 100644 --- a/vendor/github.com/klauspost/compress/fse/decompress.go +++ b/vendor/github.com/klauspost/compress/fse/decompress.go @@ -15,7 +15,7 @@ const ( // It is possible, but by no way guaranteed that corrupt data will // return an error. // It is up to the caller to verify integrity of the returned data. -// Use a predefined Scrach to set maximum acceptable output size. +// Use a predefined Scratch to set maximum acceptable output size. func Decompress(b []byte, s *Scratch) ([]byte, error) { s, err := s.prepare(b) if err != nil { diff --git a/vendor/github.com/klauspost/compress/huff0/decompress.go b/vendor/github.com/klauspost/compress/huff0/decompress.go index 54bd08b25c0ae..0f56b02d7474a 100644 --- a/vendor/github.com/klauspost/compress/huff0/decompress.go +++ b/vendor/github.com/klauspost/compress/huff0/decompress.go @@ -1136,7 +1136,7 @@ func (s *Scratch) matches(ct cTable, w io.Writer) { errs++ } if errs > 0 { - fmt.Fprintf(w, "%d errros in base, stopping\n", errs) + fmt.Fprintf(w, "%d errors in base, stopping\n", errs) continue } // Ensure that all combinations are covered. @@ -1152,7 +1152,7 @@ func (s *Scratch) matches(ct cTable, w io.Writer) { errs++ } if errs > 20 { - fmt.Fprintf(w, "%d errros, stopping\n", errs) + fmt.Fprintf(w, "%d errors, stopping\n", errs) break } } diff --git a/vendor/github.com/klauspost/compress/s2/writer.go b/vendor/github.com/klauspost/compress/s2/writer.go index 0a46f2b984fc9..fd15078f7df0f 100644 --- a/vendor/github.com/klauspost/compress/s2/writer.go +++ b/vendor/github.com/klauspost/compress/s2/writer.go @@ -83,11 +83,14 @@ type Writer struct { snappy bool flushOnWrite bool appendIndex bool + bufferCB func([]byte) level uint8 } type result struct { b []byte + // return when writing + ret []byte // Uncompressed start offset startOffset int64 } @@ -146,6 +149,10 @@ func (w *Writer) Reset(writer io.Writer) { for write := range toWrite { // Wait for the data to be available. input := <-write + if input.ret != nil && w.bufferCB != nil { + w.bufferCB(input.ret) + input.ret = nil + } in := input.b if len(in) > 0 { if w.err(nil) == nil { @@ -341,7 +348,8 @@ func (w *Writer) AddSkippableBlock(id uint8, data []byte) (err error) { // but the input buffer cannot be written to by the caller // until Flush or Close has been called when concurrency != 1. // -// If you cannot control that, use the regular Write function. +// Use the WriterBufferDone to receive a callback when the buffer is done +// Processing. // // Note that input is not buffered. // This means that each write will result in discrete blocks being created. @@ -364,6 +372,9 @@ func (w *Writer) EncodeBuffer(buf []byte) (err error) { } if w.concurrency == 1 { _, err := w.writeSync(buf) + if w.bufferCB != nil { + w.bufferCB(buf) + } return err } @@ -378,7 +389,7 @@ func (w *Writer) EncodeBuffer(buf []byte) (err error) { hWriter <- result{startOffset: w.uncompWritten, b: magicChunkBytes} } } - + orgBuf := buf for len(buf) > 0 { // Cut input. uncompressed := buf @@ -397,6 +408,9 @@ func (w *Writer) EncodeBuffer(buf []byte) (err error) { startOffset: w.uncompWritten, } w.uncompWritten += int64(len(uncompressed)) + if len(buf) == 0 && w.bufferCB != nil { + res.ret = orgBuf + } go func() { race.ReadSlice(uncompressed) @@ -922,7 +936,7 @@ func WriterBetterCompression() WriterOption { } // WriterBestCompression will enable better compression. -// EncodeBetter compresses better than Encode but typically with a +// EncodeBest compresses better than Encode but typically with a // big speed decrease on compression. func WriterBestCompression() WriterOption { return func(w *Writer) error { @@ -941,6 +955,17 @@ func WriterUncompressed() WriterOption { } } +// WriterBufferDone will perform a callback when EncodeBuffer has finished +// writing a buffer to the output and the buffer can safely be reused. +// If the buffer was split into several blocks, it will be sent after the last block. +// Callbacks will not be done concurrently. +func WriterBufferDone(fn func(b []byte)) WriterOption { + return func(w *Writer) error { + w.bufferCB = fn + return nil + } +} + // WriterBlockSize allows to override the default block size. // Blocks will be this size or smaller. // Minimum size is 4KB and maximum size is 4MB. diff --git a/vendor/github.com/klauspost/compress/zstd/blockdec.go b/vendor/github.com/klauspost/compress/zstd/blockdec.go index 03744fbc76534..9c28840c3bd77 100644 --- a/vendor/github.com/klauspost/compress/zstd/blockdec.go +++ b/vendor/github.com/klauspost/compress/zstd/blockdec.go @@ -598,7 +598,9 @@ func (b *blockDec) prepareSequences(in []byte, hist *history) (err error) { printf("RLE set to 0x%x, code: %v", symb, v) } case compModeFSE: - println("Reading table for", tableIndex(i)) + if debugDecoder { + println("Reading table for", tableIndex(i)) + } if seq.fse == nil || seq.fse.preDefined { seq.fse = fseDecoderPool.Get().(*fseDecoder) } diff --git a/vendor/github.com/klauspost/compress/zstd/enc_better.go b/vendor/github.com/klauspost/compress/zstd/enc_better.go index a4f5bf91fc63b..84a79fde76775 100644 --- a/vendor/github.com/klauspost/compress/zstd/enc_better.go +++ b/vendor/github.com/klauspost/compress/zstd/enc_better.go @@ -179,9 +179,9 @@ encodeLoop: if repIndex >= 0 && load3232(src, repIndex) == uint32(cv>>(repOff*8)) { // Consider history as well. var seq seq - lenght := 4 + e.matchlen(s+4+repOff, repIndex+4, src) + length := 4 + e.matchlen(s+4+repOff, repIndex+4, src) - seq.matchLen = uint32(lenght - zstdMinMatch) + seq.matchLen = uint32(length - zstdMinMatch) // We might be able to match backwards. // Extend as long as we can. @@ -210,12 +210,12 @@ encodeLoop: // Index match start+1 (long) -> s - 1 index0 := s + repOff - s += lenght + repOff + s += length + repOff nextEmit = s if s >= sLimit { if debugEncoder { - println("repeat ended", s, lenght) + println("repeat ended", s, length) } break encodeLoop @@ -241,9 +241,9 @@ encodeLoop: if false && repIndex >= 0 && load6432(src, repIndex) == load6432(src, s+repOff) { // Consider history as well. var seq seq - lenght := 8 + e.matchlen(s+8+repOff2, repIndex+8, src) + length := 8 + e.matchlen(s+8+repOff2, repIndex+8, src) - seq.matchLen = uint32(lenght - zstdMinMatch) + seq.matchLen = uint32(length - zstdMinMatch) // We might be able to match backwards. // Extend as long as we can. @@ -270,11 +270,11 @@ encodeLoop: } blk.sequences = append(blk.sequences, seq) - s += lenght + repOff2 + s += length + repOff2 nextEmit = s if s >= sLimit { if debugEncoder { - println("repeat ended", s, lenght) + println("repeat ended", s, length) } break encodeLoop @@ -708,9 +708,9 @@ encodeLoop: if repIndex >= 0 && load3232(src, repIndex) == uint32(cv>>(repOff*8)) { // Consider history as well. var seq seq - lenght := 4 + e.matchlen(s+4+repOff, repIndex+4, src) + length := 4 + e.matchlen(s+4+repOff, repIndex+4, src) - seq.matchLen = uint32(lenght - zstdMinMatch) + seq.matchLen = uint32(length - zstdMinMatch) // We might be able to match backwards. // Extend as long as we can. @@ -738,12 +738,12 @@ encodeLoop: blk.sequences = append(blk.sequences, seq) // Index match start+1 (long) -> s - 1 - s += lenght + repOff + s += length + repOff nextEmit = s if s >= sLimit { if debugEncoder { - println("repeat ended", s, lenght) + println("repeat ended", s, length) } break encodeLoop @@ -772,9 +772,9 @@ encodeLoop: if false && repIndex >= 0 && load6432(src, repIndex) == load6432(src, s+repOff) { // Consider history as well. var seq seq - lenght := 8 + e.matchlen(s+8+repOff2, repIndex+8, src) + length := 8 + e.matchlen(s+8+repOff2, repIndex+8, src) - seq.matchLen = uint32(lenght - zstdMinMatch) + seq.matchLen = uint32(length - zstdMinMatch) // We might be able to match backwards. // Extend as long as we can. @@ -801,11 +801,11 @@ encodeLoop: } blk.sequences = append(blk.sequences, seq) - s += lenght + repOff2 + s += length + repOff2 nextEmit = s if s >= sLimit { if debugEncoder { - println("repeat ended", s, lenght) + println("repeat ended", s, length) } break encodeLoop diff --git a/vendor/github.com/klauspost/compress/zstd/enc_dfast.go b/vendor/github.com/klauspost/compress/zstd/enc_dfast.go index a154c18f74161..d36be7bd8c248 100644 --- a/vendor/github.com/klauspost/compress/zstd/enc_dfast.go +++ b/vendor/github.com/klauspost/compress/zstd/enc_dfast.go @@ -138,9 +138,9 @@ encodeLoop: if repIndex >= 0 && load3232(src, repIndex) == uint32(cv>>(repOff*8)) { // Consider history as well. var seq seq - lenght := 4 + e.matchlen(s+4+repOff, repIndex+4, src) + length := 4 + e.matchlen(s+4+repOff, repIndex+4, src) - seq.matchLen = uint32(lenght - zstdMinMatch) + seq.matchLen = uint32(length - zstdMinMatch) // We might be able to match backwards. // Extend as long as we can. @@ -166,11 +166,11 @@ encodeLoop: println("repeat sequence", seq, "next s:", s) } blk.sequences = append(blk.sequences, seq) - s += lenght + repOff + s += length + repOff nextEmit = s if s >= sLimit { if debugEncoder { - println("repeat ended", s, lenght) + println("repeat ended", s, length) } break encodeLoop @@ -798,9 +798,9 @@ encodeLoop: if repIndex >= 0 && load3232(src, repIndex) == uint32(cv>>(repOff*8)) { // Consider history as well. var seq seq - lenght := 4 + e.matchlen(s+4+repOff, repIndex+4, src) + length := 4 + e.matchlen(s+4+repOff, repIndex+4, src) - seq.matchLen = uint32(lenght - zstdMinMatch) + seq.matchLen = uint32(length - zstdMinMatch) // We might be able to match backwards. // Extend as long as we can. @@ -826,11 +826,11 @@ encodeLoop: println("repeat sequence", seq, "next s:", s) } blk.sequences = append(blk.sequences, seq) - s += lenght + repOff + s += length + repOff nextEmit = s if s >= sLimit { if debugEncoder { - println("repeat ended", s, lenght) + println("repeat ended", s, length) } break encodeLoop diff --git a/vendor/github.com/klauspost/compress/zstd/encoder.go b/vendor/github.com/klauspost/compress/zstd/encoder.go index 72af7ef0fe001..a79c4a527c62c 100644 --- a/vendor/github.com/klauspost/compress/zstd/encoder.go +++ b/vendor/github.com/klauspost/compress/zstd/encoder.go @@ -202,7 +202,7 @@ func (e *Encoder) nextBlock(final bool) error { return nil } if final && len(s.filling) > 0 { - s.current = e.EncodeAll(s.filling, s.current[:0]) + s.current = e.encodeAll(s.encoder, s.filling, s.current[:0]) var n2 int n2, s.err = s.w.Write(s.current) if s.err != nil { @@ -469,6 +469,15 @@ func (e *Encoder) Close() error { // Data compressed with EncodeAll can be decoded with the Decoder, // using either a stream or DecodeAll. func (e *Encoder) EncodeAll(src, dst []byte) []byte { + e.init.Do(e.initialize) + enc := <-e.encoders + defer func() { + e.encoders <- enc + }() + return e.encodeAll(enc, src, dst) +} + +func (e *Encoder) encodeAll(enc encoder, src, dst []byte) []byte { if len(src) == 0 { if e.o.fullZero { // Add frame header. @@ -491,13 +500,7 @@ func (e *Encoder) EncodeAll(src, dst []byte) []byte { } return dst } - e.init.Do(e.initialize) - enc := <-e.encoders - defer func() { - // Release encoder reference to last block. - // If a non-single block is needed the encoder will reset again. - e.encoders <- enc - }() + // Use single segments when above minimum window and below window size. single := len(src) <= e.o.windowSize && len(src) > MinWindowSize if e.o.single != nil { diff --git a/vendor/github.com/klauspost/compress/zstd/framedec.go b/vendor/github.com/klauspost/compress/zstd/framedec.go index 53e160f7e5a2f..e47af66e7c903 100644 --- a/vendor/github.com/klauspost/compress/zstd/framedec.go +++ b/vendor/github.com/klauspost/compress/zstd/framedec.go @@ -146,7 +146,9 @@ func (d *frameDec) reset(br byteBuffer) error { } return err } - printf("raw: %x, mantissa: %d, exponent: %d\n", wd, wd&7, wd>>3) + if debugDecoder { + printf("raw: %x, mantissa: %d, exponent: %d\n", wd, wd&7, wd>>3) + } windowLog := 10 + (wd >> 3) windowBase := uint64(1) << windowLog windowAdd := (windowBase / 8) * uint64(wd&0x7) diff --git a/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.go b/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.go index 8adabd8287776..c59f17e07ad1d 100644 --- a/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.go +++ b/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.go @@ -146,7 +146,7 @@ func (s *sequenceDecs) decodeSyncSimple(hist []byte) (bool, error) { return true, fmt.Errorf("output bigger than max block size (%d)", maxBlockSize) default: - return true, fmt.Errorf("sequenceDecs_decode returned erronous code %d", errCode) + return true, fmt.Errorf("sequenceDecs_decode returned erroneous code %d", errCode) } s.seqSize += ctx.litRemain @@ -292,7 +292,7 @@ func (s *sequenceDecs) decode(seqs []seqVals) error { return io.ErrUnexpectedEOF } - return fmt.Errorf("sequenceDecs_decode_amd64 returned erronous code %d", errCode) + return fmt.Errorf("sequenceDecs_decode_amd64 returned erroneous code %d", errCode) } if ctx.litRemain < 0 { diff --git a/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.s b/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.s index 5b06174b89819..f5591fa1e8669 100644 --- a/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.s +++ b/vendor/github.com/klauspost/compress/zstd/seqdec_amd64.s @@ -1814,7 +1814,7 @@ TEXT ·sequenceDecs_decodeSync_amd64(SB), $64-32 MOVQ 40(SP), AX ADDQ AX, 48(SP) - // Calculate poiter to s.out[cap(s.out)] (a past-end pointer) + // Calculate pointer to s.out[cap(s.out)] (a past-end pointer) ADDQ R10, 32(SP) // outBase += outPosition @@ -2376,7 +2376,7 @@ TEXT ·sequenceDecs_decodeSync_bmi2(SB), $64-32 MOVQ 40(SP), CX ADDQ CX, 48(SP) - // Calculate poiter to s.out[cap(s.out)] (a past-end pointer) + // Calculate pointer to s.out[cap(s.out)] (a past-end pointer) ADDQ R9, 32(SP) // outBase += outPosition @@ -2896,7 +2896,7 @@ TEXT ·sequenceDecs_decodeSync_safe_amd64(SB), $64-32 MOVQ 40(SP), AX ADDQ AX, 48(SP) - // Calculate poiter to s.out[cap(s.out)] (a past-end pointer) + // Calculate pointer to s.out[cap(s.out)] (a past-end pointer) ADDQ R10, 32(SP) // outBase += outPosition @@ -3560,7 +3560,7 @@ TEXT ·sequenceDecs_decodeSync_safe_bmi2(SB), $64-32 MOVQ 40(SP), CX ADDQ CX, 48(SP) - // Calculate poiter to s.out[cap(s.out)] (a past-end pointer) + // Calculate pointer to s.out[cap(s.out)] (a past-end pointer) ADDQ R9, 32(SP) // outBase += outPosition diff --git a/vendor/modules.txt b/vendor/modules.txt index 2805571bfebf3..d59232ee194da 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1204,8 +1204,8 @@ github.com/json-iterator/go # github.com/julienschmidt/httprouter v1.3.0 ## explicit; go 1.7 github.com/julienschmidt/httprouter -# github.com/klauspost/compress v1.17.9 -## explicit; go 1.20 +# github.com/klauspost/compress v1.17.10 +## explicit; go 1.21 github.com/klauspost/compress github.com/klauspost/compress/flate github.com/klauspost/compress/fse From c8432887d3d4459ad4bc40deba3a3a3726a2f5eb Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Wed, 2 Oct 2024 11:45:12 -0400 Subject: [PATCH 09/10] fix(deps): update module github.com/ncw/swift/v2 to v2.0.3 (#14356) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- vendor/github.com/ncw/swift/v2/README.md | 1 + vendor/github.com/ncw/swift/v2/RELEASE.md | 25 ++++++++++++++++++++++ vendor/github.com/ncw/swift/v2/slo.go | 4 ++-- vendor/github.com/ncw/swift/v2/swift.go | 26 ++++++++++++++++++----- vendor/modules.txt | 4 ++-- 7 files changed, 54 insertions(+), 12 deletions(-) create mode 100644 vendor/github.com/ncw/swift/v2/RELEASE.md diff --git a/go.mod b/go.mod index 2a11d7accb99e..0c57aaded1b51 100644 --- a/go.mod +++ b/go.mod @@ -131,7 +131,7 @@ require ( github.com/heroku/x v0.0.61 github.com/influxdata/tdigest v0.0.2-0.20210216194612-fc98d27c9e8b github.com/mitchellh/colorstring v0.0.0-20190213212951-d06e56a500db - github.com/ncw/swift/v2 v2.0.2 + github.com/ncw/swift/v2 v2.0.3 github.com/prometheus/alertmanager v0.27.0 github.com/prometheus/common/sigv4 v0.1.0 github.com/richardartoul/molecule v1.0.0 diff --git a/go.sum b/go.sum index 08cf0d839fa61..eae4cb57fffb1 100644 --- a/go.sum +++ b/go.sum @@ -1490,8 +1490,8 @@ github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OS github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms= github.com/ncw/swift v1.0.53 h1:luHjjTNtekIEvHg5KdAFIBaH7bWfNkefwFnpDffSIks= github.com/ncw/swift v1.0.53/go.mod h1:23YIA4yWVnGwv2dQlN4bB7egfYX6YLn0Yo/S6zZO/ZM= -github.com/ncw/swift/v2 v2.0.2 h1:jx282pcAKFhmoZBSdMcCRFn9VWkoBIRsCpe+yZq7vEk= -github.com/ncw/swift/v2 v2.0.2/go.mod h1:z0A9RVdYPjNjXVo2pDOPxZ4eu3oarO1P91fTItcb+Kg= +github.com/ncw/swift/v2 v2.0.3 h1:8R9dmgFIWs+RiVlisCEfiQiik1hjuR0JnOkLxaP9ihg= +github.com/ncw/swift/v2 v2.0.3/go.mod h1:cbAO76/ZwcFrFlHdXPjaqWZ9R7Hdar7HpjRXBfbjigk= github.com/newrelic/newrelic-telemetry-sdk-go v0.2.0/go.mod h1:G9MqE/cHGv3Hx3qpYhfuyFUsGx2DpVcGi1iJIqTg+JQ= github.com/nicolai86/scaleway-sdk v1.10.2-0.20180628010248-798f60e20bb2/go.mod h1:TLb2Sg7HQcgGdloNxkrmtgDNR9uVYF3lfdFIN4Ro6Sk= github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e/go.mod h1:zD1mROLANZcx1PVRCS0qkT7pwLkGfwJo4zjcN/Tysno= diff --git a/vendor/github.com/ncw/swift/v2/README.md b/vendor/github.com/ncw/swift/v2/README.md index eae1952e9877d..72da43430941f 100644 --- a/vendor/github.com/ncw/swift/v2/README.md +++ b/vendor/github.com/ncw/swift/v2/README.md @@ -170,3 +170,4 @@ Contributors - Jos Houtman - Paul Collins - Joe Cai +- fsantagostinobietti <6057026+fsantagostinobietti@users.noreply.github.com> diff --git a/vendor/github.com/ncw/swift/v2/RELEASE.md b/vendor/github.com/ncw/swift/v2/RELEASE.md new file mode 100644 index 0000000000000..27e1211543782 --- /dev/null +++ b/vendor/github.com/ncw/swift/v2/RELEASE.md @@ -0,0 +1,25 @@ +# How to make a release + +Check master is building properly + +Check on master + + git checkout master + +Tag + + git tag -s v2.0.x -m "Release v2.0.x" + +Push the signed tag + + git push --follow-tags + +Go to https://github.com/ncw/swift/tags and check the tag is there. + +From there click create a release. + +Use the generate release notes button and publish. + +Possibly use this instead? + + gh release create v2.0.x --generate-notes diff --git a/vendor/github.com/ncw/swift/v2/slo.go b/vendor/github.com/ncw/swift/v2/slo.go index 45be15efe3f89..00990d1bbbdfa 100644 --- a/vendor/github.com/ncw/swift/v2/slo.go +++ b/vendor/github.com/ncw/swift/v2/slo.go @@ -6,7 +6,7 @@ import ( "encoding/json" "errors" "fmt" - "io/ioutil" + "io" "net/url" "os" ) @@ -160,7 +160,7 @@ func (c *Connection) getAllSLOSegments(ctx context.Context, container, path stri return "", nil, err } - content, err := ioutil.ReadAll(file) + content, err := io.ReadAll(file) if err != nil { return "", nil, err } diff --git a/vendor/github.com/ncw/swift/v2/swift.go b/vendor/github.com/ncw/swift/v2/swift.go index cb2b8b1e8a778..a11986eb3a37a 100644 --- a/vendor/github.com/ncw/swift/v2/swift.go +++ b/vendor/github.com/ncw/swift/v2/swift.go @@ -12,7 +12,6 @@ import ( "fmt" "hash" "io" - "io/ioutil" "mime" "net/http" "net/url" @@ -364,7 +363,7 @@ func drainAndClose(rd io.ReadCloser, err *error) { return } - _, _ = io.Copy(ioutil.Discard, rd) + _, _ = io.Copy(io.Discard, rd) cerr := rd.Close() if err != nil && *err == nil { *err = cerr @@ -787,6 +786,18 @@ func (c *Connection) Call(ctx context.Context, targetUrl string, p RequestOpts) drainAndClose(resp.Body, nil) c.UnAuthenticate() retries-- + err = AuthorizationFailed + + // Attempt to rewind the body + if p.Body != nil { + if do, ok := p.Body.(io.Seeker); ok { + if _, seekErr := do.Seek(0, io.SeekStart); seekErr != nil { + return + } + } else { + return + } + } } else { break } @@ -920,9 +931,11 @@ func (c *Connection) ContainerNames(ctx context.Context, opts *ContainersOpts) ( // Container contains information about a container type Container struct { - Name string // Name of the container - Count int64 // Number of objects in the container - Bytes int64 // Total number of bytes used in the container + Name string // Name of the container + Count int64 // Number of objects in the container + Bytes int64 // Total number of bytes used in the container + QuotaCount int64 // Maximum object count of the container. 0 if not available + QuotaBytes int64 // Maximum size of the container, in bytes. 0 if not available } // Containers returns a slice of structures with full information as @@ -1350,6 +1363,9 @@ func (c *Connection) Container(ctx context.Context, container string) (info Cont if info.Count, err = getInt64FromHeader(resp, "X-Container-Object-Count"); err != nil { return } + // optional headers + info.QuotaBytes, _ = getInt64FromHeader(resp, "X-Container-Meta-Quota-Bytes") + info.QuotaCount, _ = getInt64FromHeader(resp, "X-Container-Meta-Quota-Count") return } diff --git a/vendor/modules.txt b/vendor/modules.txt index d59232ee194da..ff9365f2c4c99 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1327,8 +1327,8 @@ github.com/mwitkow/go-conntrack # github.com/ncw/swift v1.0.53 ## explicit github.com/ncw/swift -# github.com/ncw/swift/v2 v2.0.2 -## explicit; go 1.15 +# github.com/ncw/swift/v2 v2.0.3 +## explicit; go 1.16 github.com/ncw/swift/v2 # github.com/oklog/run v1.1.0 ## explicit; go 1.13 From 2f56f50cc6591ca482358933c719d005446d0c01 Mon Sep 17 00:00:00 2001 From: Dylan Guedes Date: Wed, 2 Oct 2024 13:51:20 -0300 Subject: [PATCH 10/10] fix: Transform `ObjectExistsWithSize` into `GetAttributes` (#14329) - Transform our `ObjectExistsWithSize` API into `GetAttributes` to conform with the existing Thanos API. - Fix we not returning the size when we should on our s3 implementation of `ObjectSize` - Fix not returning `false, nil` to `ObjectExists` when the only error seen is a `NotFound`. If someone calls `ObjectExists` it should return `false, nil` to a `NotExist` error, instead of `false, err` --- pkg/ingester-rf1/objstore/storage.go | 6 +-- .../chunk/client/alibaba/oss_object_client.go | 22 +++++++--- .../chunk/client/aws/s3_storage_client.go | 29 +++++++------ .../client/aws/s3_storage_client_test.go | 41 +++++++++++++++++-- .../chunk/client/azure/blob_storage_client.go | 21 +++++++--- .../client/baidubce/bos_storage_client.go | 21 +++++++--- .../chunk/client/congestion/controller.go | 8 ++-- .../client/congestion/controller_test.go | 2 +- .../chunk/client/gcp/gcs_object_client.go | 17 +++++--- .../client/ibmcloud/cos_object_client.go | 21 +++++++--- .../chunk/client/local/fs_object_client.go | 15 ++++--- .../client/local/fs_object_client_test.go | 4 +- pkg/storage/chunk/client/object_client.go | 6 ++- .../client/openstack/swift_object_client.go | 15 ++++--- .../chunk/client/prefixed_object_client.go | 4 +- .../testutils/inmemory_storage_client.go | 17 +++++--- pkg/tool/audit/audit_test.go | 26 ++++++++---- 17 files changed, 194 insertions(+), 81 deletions(-) diff --git a/pkg/ingester-rf1/objstore/storage.go b/pkg/ingester-rf1/objstore/storage.go index 5a8c61fd5117b..9937ee20ee818 100644 --- a/pkg/ingester-rf1/objstore/storage.go +++ b/pkg/ingester-rf1/objstore/storage.go @@ -70,12 +70,12 @@ func (m *Multi) GetStoreFor(ts model.Time) (client.ObjectClient, error) { return nil, fmt.Errorf("no store found for timestamp %s", ts) } -func (m *Multi) ObjectExistsWithSize(ctx context.Context, objectKey string) (bool, int64, error) { +func (m *Multi) GetAttributes(ctx context.Context, objectKey string) (client.ObjectAttributes, error) { s, err := m.GetStoreFor(model.Now()) if err != nil { - return false, 0, err + return client.ObjectAttributes{}, err } - return s.ObjectExistsWithSize(ctx, objectKey) + return s.GetAttributes(ctx, objectKey) } func (m *Multi) ObjectExists(ctx context.Context, objectKey string) (bool, error) { diff --git a/pkg/storage/chunk/client/alibaba/oss_object_client.go b/pkg/storage/chunk/client/alibaba/oss_object_client.go index 9d7f4cc48ce1c..5cf200c202cc5 100644 --- a/pkg/storage/chunk/client/alibaba/oss_object_client.go +++ b/pkg/storage/chunk/client/alibaba/oss_object_client.go @@ -73,14 +73,24 @@ func (s *OssObjectClient) Stop() { } func (s *OssObjectClient) ObjectExists(ctx context.Context, objectKey string) (bool, error) { - exists, _, err := s.ObjectExistsWithSize(ctx, objectKey) - return exists, err + if _, err := s.objectAttributes(ctx, objectKey, "OSS.ObjectExists"); err != nil { + if s.IsObjectNotFoundErr(err) { + return false, nil + } + return false, err + } + + return true, nil +} + +func (s *OssObjectClient) GetAttributes(ctx context.Context, objectKey string) (client.ObjectAttributes, error) { + return s.objectAttributes(ctx, objectKey, "OSS.GetAttributes") } -func (s *OssObjectClient) ObjectExistsWithSize(ctx context.Context, objectKey string) (bool, int64, error) { +func (s *OssObjectClient) objectAttributes(ctx context.Context, objectKey, operation string) (client.ObjectAttributes, error) { var options []oss.Option var objectSize int64 - err := instrument.CollectedRequest(ctx, "OSS.ObjectExists", ossRequestDuration, instrument.ErrorCode, func(_ context.Context) error { + err := instrument.CollectedRequest(ctx, operation, ossRequestDuration, instrument.ErrorCode, func(_ context.Context) error { headers, requestErr := s.defaultBucket.GetObjectMeta(objectKey, options...) if requestErr != nil { return requestErr @@ -90,10 +100,10 @@ func (s *OssObjectClient) ObjectExistsWithSize(ctx context.Context, objectKey st return nil }) if err != nil { - return false, 0, err + return client.ObjectAttributes{}, err } - return true, objectSize, nil + return client.ObjectAttributes{Size: objectSize}, nil } // GetObject returns a reader and the size for the specified object key from the configured OSS bucket. diff --git a/pkg/storage/chunk/client/aws/s3_storage_client.go b/pkg/storage/chunk/client/aws/s3_storage_client.go index 26c2807e120e7..7747f27618008 100644 --- a/pkg/storage/chunk/client/aws/s3_storage_client.go +++ b/pkg/storage/chunk/client/aws/s3_storage_client.go @@ -310,20 +310,29 @@ func buckets(cfg S3Config) ([]string, error) { func (a *S3ObjectClient) Stop() {} func (a *S3ObjectClient) ObjectExists(ctx context.Context, objectKey string) (bool, error) { - exists, _, err := a.ObjectExistsWithSize(ctx, objectKey) - return exists, err + if _, err := a.objectAttributes(ctx, objectKey, "S3.ObjectExists"); err != nil { + if a.IsObjectNotFoundErr(err) { + return false, nil + } + return false, err + } + return true, nil +} + +func (a *S3ObjectClient) GetAttributes(ctx context.Context, objectKey string) (client.ObjectAttributes, error) { + return a.objectAttributes(ctx, objectKey, "S3.GetAttributes") } -func (a *S3ObjectClient) ObjectExistsWithSize(ctx context.Context, objectKey string) (bool, int64, error) { +func (a *S3ObjectClient) objectAttributes(ctx context.Context, objectKey, method string) (client.ObjectAttributes, error) { var lastErr error var objectSize int64 retries := backoff.New(ctx, a.cfg.BackoffConfig) for retries.Ongoing() { if ctx.Err() != nil { - return false, 0, errors.Wrap(ctx.Err(), "ctx related error during s3 objectExists") + return client.ObjectAttributes{}, errors.Wrap(ctx.Err(), "ctx related error during s3 objectExists") } - lastErr = instrument.CollectedRequest(ctx, "S3.ObjectExists", s3RequestDuration, instrument.ErrorCode, func(_ context.Context) error { + lastErr = instrument.CollectedRequest(ctx, method, s3RequestDuration, instrument.ErrorCode, func(_ context.Context) error { headObjectInput := &s3.HeadObjectInput{ Bucket: aws.String(a.bucketFromKey(objectKey)), Key: aws.String(objectKey), @@ -338,21 +347,17 @@ func (a *S3ObjectClient) ObjectExistsWithSize(ctx context.Context, objectKey str return nil }) if lastErr == nil { - return true, 0, nil + return client.ObjectAttributes{Size: objectSize}, nil } if a.IsObjectNotFoundErr(lastErr) { - return false, 0, lastErr + return client.ObjectAttributes{}, lastErr } retries.Wait() } - if lastErr != nil { - return false, 0, lastErr - } - - return true, objectSize, nil + return client.ObjectAttributes{}, lastErr } // DeleteObject deletes the specified objectKey from the appropriate S3 bucket diff --git a/pkg/storage/chunk/client/aws/s3_storage_client_test.go b/pkg/storage/chunk/client/aws/s3_storage_client_test.go index e18160a3fa003..5647934e1afa7 100644 --- a/pkg/storage/chunk/client/aws/s3_storage_client_test.go +++ b/pkg/storage/chunk/client/aws/s3_storage_client_test.go @@ -309,6 +309,36 @@ func (m *MockS3Client) HeadObject(input *s3.HeadObjectInput) (*s3.HeadObjectOutp return m.HeadObjectFunc(input) } +func Test_GetAttributes(t *testing.T) { + mockS3 := &MockS3Client{ + HeadObjectFunc: func(_ *s3.HeadObjectInput) (*s3.HeadObjectOutput, error) { + var size int64 = 128 + return &s3.HeadObjectOutput{ContentLength: &size}, nil + }, + } + + c, err := NewS3ObjectClient(S3Config{ + AccessKeyID: "foo", + SecretAccessKey: flagext.SecretWithValue("bar"), + BackoffConfig: backoff.Config{MaxRetries: 3}, + BucketNames: "foo", + Inject: func(_ http.RoundTripper) http.RoundTripper { + return RoundTripperFunc(func(_ *http.Request) (*http.Response, error) { + return &http.Response{ + StatusCode: http.StatusOK, + Body: io.NopCloser(bytes.NewReader([]byte("object content"))), + }, nil + }) + }, + }, hedging.Config{}) + require.NoError(t, err) + c.S3 = mockS3 + + attrs, err := c.GetAttributes(context.Background(), "abc") + require.NoError(t, err) + require.EqualValues(t, 128, attrs.Size) +} + func Test_RetryLogic(t *testing.T) { for _, tc := range []struct { name string @@ -339,7 +369,7 @@ func Test_RetryLogic(t *testing.T) { 3, true, func(c *S3ObjectClient) error { - _, _, err := c.ObjectExistsWithSize(context.Background(), "foo") + _, err := c.ObjectExists(context.Background(), "foo") return err }, }, @@ -348,7 +378,12 @@ func Test_RetryLogic(t *testing.T) { 3, false, func(c *S3ObjectClient) error { - _, err := c.ObjectExists(context.Background(), "foo") + exists, err := c.ObjectExists(context.Background(), "foo") + if err == nil && !exists { + return awserr.NewRequestFailure( + awserr.New("NotFound", "Not Found", nil), 404, "abc", + ) + } return err }, }, @@ -357,7 +392,7 @@ func Test_RetryLogic(t *testing.T) { 3, false, func(c *S3ObjectClient) error { - _, _, err := c.ObjectExistsWithSize(context.Background(), "foo") + _, err := c.GetAttributes(context.Background(), "foo") return err }, }, diff --git a/pkg/storage/chunk/client/azure/blob_storage_client.go b/pkg/storage/chunk/client/azure/blob_storage_client.go index 2a7014c29898e..86b59bc6f420b 100644 --- a/pkg/storage/chunk/client/azure/blob_storage_client.go +++ b/pkg/storage/chunk/client/azure/blob_storage_client.go @@ -220,13 +220,22 @@ func NewBlobStorage(cfg *BlobStorageConfig, metrics BlobStorageMetrics, hedgingC func (b *BlobStorage) Stop() {} func (b *BlobStorage) ObjectExists(ctx context.Context, objectKey string) (bool, error) { - exists, _, err := b.ObjectExistsWithSize(ctx, objectKey) - return exists, err + if _, err := b.objectAttributes(ctx, objectKey, "azure.ObjectExists"); err != nil { + if b.IsObjectNotFoundErr(err) { + return false, nil + } + return false, err + } + return true, nil +} + +func (b *BlobStorage) GetAttributes(ctx context.Context, objectKey string) (client.ObjectAttributes, error) { + return b.objectAttributes(ctx, objectKey, "azure.GetAttributes") } -func (b *BlobStorage) ObjectExistsWithSize(ctx context.Context, objectKey string) (bool, int64, error) { +func (b *BlobStorage) objectAttributes(ctx context.Context, objectKey, source string) (client.ObjectAttributes, error) { var objectSize int64 - err := loki_instrument.TimeRequest(ctx, "azure.ObjectExists", instrument.NewHistogramCollector(b.metrics.requestDuration), instrument.ErrorCode, func(ctx context.Context) error { + err := loki_instrument.TimeRequest(ctx, source, instrument.NewHistogramCollector(b.metrics.requestDuration), instrument.ErrorCode, func(ctx context.Context) error { blockBlobURL, err := b.getBlobURL(objectKey, false) if err != nil { return err @@ -245,10 +254,10 @@ func (b *BlobStorage) ObjectExistsWithSize(ctx context.Context, objectKey string return nil }) if err != nil { - return false, 0, err + return client.ObjectAttributes{}, err } - return true, objectSize, nil + return client.ObjectAttributes{Size: objectSize}, nil } // GetObject returns a reader and the size for the specified object key. diff --git a/pkg/storage/chunk/client/baidubce/bos_storage_client.go b/pkg/storage/chunk/client/baidubce/bos_storage_client.go index cc76b21624294..ccdc7e1b324a5 100644 --- a/pkg/storage/chunk/client/baidubce/bos_storage_client.go +++ b/pkg/storage/chunk/client/baidubce/bos_storage_client.go @@ -91,13 +91,22 @@ func (b *BOSObjectStorage) PutObject(ctx context.Context, objectKey string, obje } func (b *BOSObjectStorage) ObjectExists(ctx context.Context, objectKey string) (bool, error) { - exists, _, err := b.ObjectExistsWithSize(ctx, objectKey) - return exists, err + if _, err := b.objectAttributes(ctx, objectKey, "BOS.ObjectExists"); err != nil { + if b.IsObjectNotFoundErr(err) { + return false, nil + } + return false, err + } + return true, nil +} + +func (b *BOSObjectStorage) GetAttributes(ctx context.Context, objectKey string) (client.ObjectAttributes, error) { + return b.objectAttributes(ctx, objectKey, "BOS.GetAttributes") } -func (b *BOSObjectStorage) ObjectExistsWithSize(ctx context.Context, objectKey string) (bool, int64, error) { +func (b *BOSObjectStorage) objectAttributes(ctx context.Context, objectKey, source string) (client.ObjectAttributes, error) { var objectSize int64 - err := instrument.CollectedRequest(ctx, "BOS.ObjectExists", bosRequestDuration, instrument.ErrorCode, func(_ context.Context) error { + err := instrument.CollectedRequest(ctx, source, bosRequestDuration, instrument.ErrorCode, func(_ context.Context) error { metaResult, requestErr := b.client.GetObjectMeta(b.cfg.BucketName, objectKey) if requestErr != nil { return requestErr @@ -108,10 +117,10 @@ func (b *BOSObjectStorage) ObjectExistsWithSize(ctx context.Context, objectKey s return nil }) if err != nil { - return false, 0, err + return client.ObjectAttributes{}, err } - return true, objectSize, nil + return client.ObjectAttributes{Size: objectSize}, nil } func (b *BOSObjectStorage) GetObject(ctx context.Context, objectKey string) (io.ReadCloser, int64, error) { diff --git a/pkg/storage/chunk/client/congestion/controller.go b/pkg/storage/chunk/client/congestion/controller.go index 1c69ef16139f7..fd8429d0ba7e5 100644 --- a/pkg/storage/chunk/client/congestion/controller.go +++ b/pkg/storage/chunk/client/congestion/controller.go @@ -145,8 +145,8 @@ func (a *AIMDController) ObjectExists(ctx context.Context, objectKey string) (bo return a.inner.ObjectExists(ctx, objectKey) } -func (a *AIMDController) ObjectExistsWithSize(ctx context.Context, objectKey string) (bool, int64, error) { - return a.inner.ObjectExistsWithSize(ctx, objectKey) +func (a *AIMDController) GetAttributes(ctx context.Context, objectKey string) (client.ObjectAttributes, error) { + return a.inner.GetAttributes(ctx, objectKey) } func (a *AIMDController) DeleteObject(ctx context.Context, objectKey string) error { @@ -216,8 +216,8 @@ func NewNoopController(Config) *NoopController { return &NoopController{} } -func (n *NoopController) ObjectExistsWithSize(context.Context, string) (bool, int64, error) { - return true, 0, nil +func (n *NoopController) GetAttributes(context.Context, string) (client.ObjectAttributes, error) { + return client.ObjectAttributes{}, nil } func (n *NoopController) ObjectExists(context.Context, string) (bool, error) { return true, nil } func (n *NoopController) PutObject(context.Context, string, io.Reader) error { return nil } diff --git a/pkg/storage/chunk/client/congestion/controller_test.go b/pkg/storage/chunk/client/congestion/controller_test.go index 6d17573248f50..6be3779f5c36a 100644 --- a/pkg/storage/chunk/client/congestion/controller_test.go +++ b/pkg/storage/chunk/client/congestion/controller_test.go @@ -267,7 +267,7 @@ func (m *mockObjectClient) ObjectExists(context.Context, string) (bool, error) { panic("not implemented") } -func (m *mockObjectClient) ObjectExistsWithSize(context.Context, string) (bool, int64, error) { +func (m *mockObjectClient) GetAttributes(context.Context, string) (client.ObjectAttributes, error) { panic("not implemented") } diff --git a/pkg/storage/chunk/client/gcp/gcs_object_client.go b/pkg/storage/chunk/client/gcp/gcs_object_client.go index c161705ecf7f5..9b05b57404c49 100644 --- a/pkg/storage/chunk/client/gcp/gcs_object_client.go +++ b/pkg/storage/chunk/client/gcp/gcs_object_client.go @@ -127,20 +127,25 @@ func (s *GCSObjectClient) Stop() { } func (s *GCSObjectClient) ObjectExists(ctx context.Context, objectKey string) (bool, error) { - exists, _, err := s.ObjectExistsWithSize(ctx, objectKey) - return exists, err + if _, err := s.GetAttributes(ctx, objectKey); err != nil { + if s.IsObjectNotFoundErr(err) { + return false, nil + } + return false, err + } + return true, nil } -func (s *GCSObjectClient) ObjectExistsWithSize(ctx context.Context, objectKey string) (bool, int64, error) { +func (s *GCSObjectClient) GetAttributes(ctx context.Context, objectKey string) (client.ObjectAttributes, error) { attrs, err := s.getsBuckets.Object(objectKey).Attrs(ctx) if err != nil { - return false, 0, err + return client.ObjectAttributes{}, err } if attrs != nil { - return true, attrs.Size, nil + return client.ObjectAttributes{Size: attrs.Size}, nil } - return true, 0, nil + return client.ObjectAttributes{}, nil } // GetObject returns a reader and the size for the specified object key from the configured GCS bucket. diff --git a/pkg/storage/chunk/client/ibmcloud/cos_object_client.go b/pkg/storage/chunk/client/ibmcloud/cos_object_client.go index 51a9c9fd93086..da8436ccc03a3 100644 --- a/pkg/storage/chunk/client/ibmcloud/cos_object_client.go +++ b/pkg/storage/chunk/client/ibmcloud/cos_object_client.go @@ -320,14 +320,23 @@ func (c *COSObjectClient) DeleteObject(ctx context.Context, objectKey string) er } func (c *COSObjectClient) ObjectExists(ctx context.Context, objectKey string) (bool, error) { - exists, _, err := c.ObjectExistsWithSize(ctx, objectKey) - return exists, err + if _, err := c.objectAttributes(ctx, objectKey, "COS.ObjectExists"); err != nil { + if c.IsObjectNotFoundErr(err) { + return false, nil + } + return false, err + } + return true, nil +} + +func (c *COSObjectClient) GetAttributes(ctx context.Context, objectKey string) (client.ObjectAttributes, error) { + return c.objectAttributes(ctx, objectKey, "COS.GetAttributes") } -func (c *COSObjectClient) ObjectExistsWithSize(ctx context.Context, objectKey string) (bool, int64, error) { +func (c *COSObjectClient) objectAttributes(ctx context.Context, objectKey, source string) (client.ObjectAttributes, error) { bucket := c.bucketFromKey(objectKey) var objectSize int64 - err := instrument.CollectedRequest(ctx, "COS.GetObject", cosRequestDuration, instrument.ErrorCode, func(_ context.Context) error { + err := instrument.CollectedRequest(ctx, source, cosRequestDuration, instrument.ErrorCode, func(_ context.Context) error { headOutput, requestErr := c.hedgedCOS.HeadObject(&cos.HeadObjectInput{ Bucket: ibm.String(bucket), Key: ibm.String(objectKey), @@ -341,10 +350,10 @@ func (c *COSObjectClient) ObjectExistsWithSize(ctx context.Context, objectKey st return nil }) if err != nil { - return false, 0, err + return client.ObjectAttributes{}, err } - return true, objectSize, nil + return client.ObjectAttributes{Size: objectSize}, nil } // GetObject returns a reader and the size for the specified object key from the configured S3 bucket. diff --git a/pkg/storage/chunk/client/local/fs_object_client.go b/pkg/storage/chunk/client/local/fs_object_client.go index 671b5df285870..0db9f7a7c1767 100644 --- a/pkg/storage/chunk/client/local/fs_object_client.go +++ b/pkg/storage/chunk/client/local/fs_object_client.go @@ -68,18 +68,23 @@ func NewFSObjectClient(cfg FSConfig) (*FSObjectClient, error) { func (FSObjectClient) Stop() {} func (f *FSObjectClient) ObjectExists(ctx context.Context, objectKey string) (bool, error) { - exists, _, err := f.ObjectExistsWithSize(ctx, objectKey) - return exists, err + if _, err := f.GetAttributes(ctx, objectKey); err != nil { + if f.IsObjectNotFoundErr(err) { + return false, nil + } + return false, err + } + return true, nil } -func (f *FSObjectClient) ObjectExistsWithSize(_ context.Context, objectKey string) (bool, int64, error) { +func (f *FSObjectClient) GetAttributes(_ context.Context, objectKey string) (client.ObjectAttributes, error) { fullPath := filepath.Join(f.cfg.Directory, filepath.FromSlash(objectKey)) fi, err := os.Lstat(fullPath) if err != nil { - return false, 0, err + return client.ObjectAttributes{}, err } - return true, fi.Size(), nil + return client.ObjectAttributes{Size: fi.Size()}, nil } // GetObject from the store diff --git a/pkg/storage/chunk/client/local/fs_object_client_test.go b/pkg/storage/chunk/client/local/fs_object_client_test.go index 15ad96425dc9c..4da1d58eb2e5d 100644 --- a/pkg/storage/chunk/client/local/fs_object_client_test.go +++ b/pkg/storage/chunk/client/local/fs_object_client_test.go @@ -157,10 +157,10 @@ func TestFSObjectClient_List_and_ObjectExists(t *testing.T) { require.NoError(t, err) require.True(t, ok) - ok, objectSize, err := bucketClient.ObjectExistsWithSize(context.Background(), "outer-file2") + attrs, err := bucketClient.GetAttributes(context.Background(), "outer-file2") require.NoError(t, err) require.True(t, ok) - require.EqualValues(t, len("outer-file2"), objectSize) + require.EqualValues(t, len("outer-file2"), attrs.Size) } func TestFSObjectClient_DeleteObject(t *testing.T) { diff --git a/pkg/storage/chunk/client/object_client.go b/pkg/storage/chunk/client/object_client.go index 95672c286ad09..41026486c55e9 100644 --- a/pkg/storage/chunk/client/object_client.go +++ b/pkg/storage/chunk/client/object_client.go @@ -16,10 +16,14 @@ import ( "github.com/grafana/loki/v3/pkg/storage/config" ) +type ObjectAttributes struct { + Size int64 +} + // ObjectClient is used to store arbitrary data in Object Store (S3/GCS/Azure/...) type ObjectClient interface { ObjectExists(ctx context.Context, objectKey string) (bool, error) - ObjectExistsWithSize(ctx context.Context, objectKey string) (bool, int64, error) + GetAttributes(ctx context.Context, objectKey string) (ObjectAttributes, error) PutObject(ctx context.Context, objectKey string, object io.Reader) error // NOTE: The consumer of GetObject should always call the Close method when it is done reading which otherwise could cause a resource leak. diff --git a/pkg/storage/chunk/client/openstack/swift_object_client.go b/pkg/storage/chunk/client/openstack/swift_object_client.go index d3d978cd5ef72..03721d3e16407 100644 --- a/pkg/storage/chunk/client/openstack/swift_object_client.go +++ b/pkg/storage/chunk/client/openstack/swift_object_client.go @@ -125,17 +125,22 @@ func (s *SwiftObjectClient) Stop() { } func (s *SwiftObjectClient) ObjectExists(ctx context.Context, objectKey string) (bool, error) { - exists, _, err := s.ObjectExistsWithSize(ctx, objectKey) - return exists, err + if _, err := s.GetAttributes(ctx, objectKey); err != nil { + if s.IsObjectNotFoundErr(err) { + return false, nil + } + return false, err + } + return true, nil } -func (s *SwiftObjectClient) ObjectExistsWithSize(ctx context.Context, objectKey string) (bool, int64, error) { +func (s *SwiftObjectClient) GetAttributes(ctx context.Context, objectKey string) (client.ObjectAttributes, error) { info, _, err := s.hedgingConn.Object(ctx, s.cfg.Config.ContainerName, objectKey) if err != nil { - return false, 0, err + return client.ObjectAttributes{}, nil } - return true, info.Bytes, nil + return client.ObjectAttributes{Size: info.Bytes}, nil } // GetObject returns a reader and the size for the specified object key from the configured swift container. diff --git a/pkg/storage/chunk/client/prefixed_object_client.go b/pkg/storage/chunk/client/prefixed_object_client.go index 5a5bda7627708..9d0f862b55813 100644 --- a/pkg/storage/chunk/client/prefixed_object_client.go +++ b/pkg/storage/chunk/client/prefixed_object_client.go @@ -23,8 +23,8 @@ func (p PrefixedObjectClient) ObjectExists(ctx context.Context, objectKey string return p.downstreamClient.ObjectExists(ctx, p.prefix+objectKey) } -func (p PrefixedObjectClient) ObjectExistsWithSize(ctx context.Context, objectKey string) (bool, int64, error) { - return p.downstreamClient.ObjectExistsWithSize(ctx, p.prefix+objectKey) +func (p PrefixedObjectClient) GetAttributes(ctx context.Context, objectKey string) (ObjectAttributes, error) { + return p.downstreamClient.GetAttributes(ctx, p.prefix+objectKey) } func (p PrefixedObjectClient) GetObject(ctx context.Context, objectKey string) (io.ReadCloser, int64, error) { diff --git a/pkg/storage/chunk/client/testutils/inmemory_storage_client.go b/pkg/storage/chunk/client/testutils/inmemory_storage_client.go index d937cd5fdfd4d..b9e8db8ed86a5 100644 --- a/pkg/storage/chunk/client/testutils/inmemory_storage_client.go +++ b/pkg/storage/chunk/client/testutils/inmemory_storage_client.go @@ -393,24 +393,29 @@ func (m *MockStorage) query(ctx context.Context, query index.Query, callback fun // ObjectExists implments client.ObjectClient func (m *InMemoryObjectClient) ObjectExists(ctx context.Context, objectKey string) (bool, error) { - exists, _, err := m.ObjectExistsWithSize(ctx, objectKey) - return exists, err + if _, err := m.GetAttributes(ctx, objectKey); err != nil { + if m.IsObjectNotFoundErr(err) { + return false, nil + } + return false, err + } + return true, nil } -func (m *InMemoryObjectClient) ObjectExistsWithSize(_ context.Context, objectKey string) (bool, int64, error) { +func (m *InMemoryObjectClient) GetAttributes(_ context.Context, objectKey string) (client.ObjectAttributes, error) { m.mtx.RLock() defer m.mtx.RUnlock() if m.mode == MockStorageModeWriteOnly { - return false, 0, errPermissionDenied + return client.ObjectAttributes{}, errPermissionDenied } _, ok := m.objects[objectKey] if !ok { - return false, 0, nil + return client.ObjectAttributes{}, errStorageObjectNotFound } objectSize := len(m.objects[objectKey]) - return true, int64(objectSize), nil + return client.ObjectAttributes{Size: int64(objectSize)}, nil } // GetObject implements client.ObjectClient. diff --git a/pkg/tool/audit/audit_test.go b/pkg/tool/audit/audit_test.go index d591fca7e1ddb..4e20b075be857 100644 --- a/pkg/tool/audit/audit_test.go +++ b/pkg/tool/audit/audit_test.go @@ -2,6 +2,7 @@ package audit import ( "context" + "errors" "strings" "testing" @@ -13,20 +14,31 @@ import ( "github.com/grafana/loki/v3/pkg/storage/chunk/client" ) +var errObjectNotFound = errors.New("object not found") + type testObjClient struct { client.ObjectClient } -func (t testObjClient) ObjectExistsWithSize(_ context.Context, object string) (bool, int64, error) { - if strings.Contains(object, "missing") { - return false, 0, nil +func (t testObjClient) ObjectExists(ctx context.Context, object string) (bool, error) { + if _, err := t.GetAttributes(ctx, object); err != nil { + if t.IsObjectNotFoundErr(err) { + return false, nil + } + return false, err } - return true, 0, nil + return true, nil } -func (t testObjClient) ObjectExists(ctx context.Context, object string) (bool, error) { - exists, _, err := t.ObjectExistsWithSize(ctx, object) - return exists, err +func (t testObjClient) IsObjectNotFoundErr(err error) bool { + return errors.Is(err, errObjectNotFound) +} + +func (t testObjClient) GetAttributes(_ context.Context, object string) (client.ObjectAttributes, error) { + if strings.Contains(object, "missing") { + return client.ObjectAttributes{}, errObjectNotFound + } + return client.ObjectAttributes{}, nil } type testCompactedIdx struct {