diff --git a/.chloggen/red_metrics_namespace.yaml b/.chloggen/red_metrics_namespace.yaml new file mode 100644 index 000000000..4249a22d6 --- /dev/null +++ b/.chloggen/red_metrics_namespace.yaml @@ -0,0 +1,15 @@ +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: breaking +# The name of the component, or a single word describing the area of concern, (e.g. tempostack, tempomonolithic, github action) +component: tempostack +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Use default Jaeger RED metrics namespace if field is unset +# One or more tracking issues related to the change +issues: [1096] +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: | + Use the default Jaeger RED metrics namespace if the field is unset. + Before Jaeger 1.62 the default namespace was empty, since [Jaeger 1.62](https://github.com/jaegertracing/jaeger/releases/tag/v1.62.0) (shipped in Tempo Operator v0.14.0) the default namespace is "traces_span_metrics". + Before OpenTelemetry Collector v0.109.0 the default namespace of the spanmetrics connector was empty, since [OpenTelemetry Collector v0.109.0](https://github.com/open-telemetry/opentelemetry-collector-contrib/releases/tag/v0.109.0) the default namespace is "traces_span_metrics". diff --git a/apis/tempo/v1alpha1/tempostack_types.go b/apis/tempo/v1alpha1/tempostack_types.go index 8a9103e12..c4ea725c0 100644 --- a/apis/tempo/v1alpha1/tempostack_types.go +++ b/apis/tempo/v1alpha1/tempostack_types.go @@ -690,12 +690,10 @@ type JaegerQueryMonitor struct { // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Prometheus endpoint" PrometheusEndpoint string `json:"prometheusEndpoint"` // REDMetricsNamespace defines the a prefix used retrieve span rate, error, and duration (RED) metrics. - // By default it is set to `traces.span.metrics` following the default namespace of the OpenTelemetry Collector since Version 0.109.0. // +optional // +kubebuilder:validation:Optional - // +kubebuilder:default:=traces.span.metrics // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="RED Metric Namespace" - REDMetricsNamespace string `json:"redMetricsNamespace"` + REDMetricsNamespace *string `json:"redMetricsNamespace,omitempty"` } // IngressSpec defines Jaeger Query Ingress options. diff --git a/apis/tempo/v1alpha1/zz_generated.deepcopy.go b/apis/tempo/v1alpha1/zz_generated.deepcopy.go index f5a9d2534..d9a80f782 100644 --- a/apis/tempo/v1alpha1/zz_generated.deepcopy.go +++ b/apis/tempo/v1alpha1/zz_generated.deepcopy.go @@ -304,6 +304,11 @@ func (in *JaegerQueryAuthenticationSpec) DeepCopy() *JaegerQueryAuthenticationSp // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *JaegerQueryMonitor) DeepCopyInto(out *JaegerQueryMonitor) { *out = *in + if in.REDMetricsNamespace != nil { + in, out := &in.REDMetricsNamespace, &out.REDMetricsNamespace + *out = new(string) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new JaegerQueryMonitor. @@ -320,7 +325,7 @@ func (in *JaegerQueryMonitor) DeepCopy() *JaegerQueryMonitor { func (in *JaegerQuerySpec) DeepCopyInto(out *JaegerQuerySpec) { *out = *in in.Ingress.DeepCopyInto(&out.Ingress) - out.MonitorTab = in.MonitorTab + in.MonitorTab.DeepCopyInto(&out.MonitorTab) if in.Resources != nil { in, out := &in.Resources, &out.Resources *out = new(v1.ResourceRequirements) diff --git a/bundle/community/manifests/tempo-operator.clusterserviceversion.yaml b/bundle/community/manifests/tempo-operator.clusterserviceversion.yaml index 2c9f15a27..ee19050bf 100644 --- a/bundle/community/manifests/tempo-operator.clusterserviceversion.yaml +++ b/bundle/community/manifests/tempo-operator.clusterserviceversion.yaml @@ -74,7 +74,7 @@ metadata: capabilities: Deep Insights categories: Logging & Tracing,Monitoring containerImage: ghcr.io/grafana/tempo-operator/tempo-operator:v0.14.1 - createdAt: "2024-11-06T14:25:14Z" + createdAt: "2024-12-02T11:40:40Z" description: Create and manage deployments of Tempo, a high-scale distributed tracing backend. operatorframework.io/cluster-monitoring: "true" @@ -1071,9 +1071,7 @@ spec: displayName: Prometheus endpoint path: template.queryFrontend.jaegerQuery.monitorTab.prometheusEndpoint - description: REDMetricsNamespace defines the a prefix used retrieve span rate, - error, and duration (RED) metrics. By default it is set to `traces.span.metrics` - following the default namespace of the OpenTelemetry Collector since Version - 0.109.0. + error, and duration (RED) metrics. displayName: RED Metric Namespace path: template.queryFrontend.jaegerQuery.monitorTab.redMetricsNamespace - description: Resources defines resources for this component, this will override diff --git a/bundle/community/manifests/tempo.grafana.com_tempostacks.yaml b/bundle/community/manifests/tempo.grafana.com_tempostacks.yaml index 68eb5046f..6c93b0307 100644 --- a/bundle/community/manifests/tempo.grafana.com_tempostacks.yaml +++ b/bundle/community/manifests/tempo.grafana.com_tempostacks.yaml @@ -2471,10 +2471,9 @@ spec: For instance on OpenShift this is set to https://thanos-querier.openshift-monitoring.svc.cluster.local:9091 type: string redMetricsNamespace: - default: traces.span.metrics - description: |- - REDMetricsNamespace defines the a prefix used retrieve span rate, error, and duration (RED) metrics. - By default it is set to `traces.span.metrics` following the default namespace of the OpenTelemetry Collector since Version 0.109.0. + description: REDMetricsNamespace defines the a prefix + used retrieve span rate, error, and duration (RED) + metrics. type: string type: object resources: diff --git a/bundle/openshift/manifests/tempo-operator.clusterserviceversion.yaml b/bundle/openshift/manifests/tempo-operator.clusterserviceversion.yaml index b62e83a47..16140c563 100644 --- a/bundle/openshift/manifests/tempo-operator.clusterserviceversion.yaml +++ b/bundle/openshift/manifests/tempo-operator.clusterserviceversion.yaml @@ -74,7 +74,7 @@ metadata: capabilities: Deep Insights categories: Logging & Tracing,Monitoring containerImage: ghcr.io/grafana/tempo-operator/tempo-operator:v0.14.1 - createdAt: "2024-11-06T14:25:12Z" + createdAt: "2024-12-02T11:40:38Z" description: Create and manage deployments of Tempo, a high-scale distributed tracing backend. operatorframework.io/cluster-monitoring: "true" @@ -1071,9 +1071,7 @@ spec: displayName: Prometheus endpoint path: template.queryFrontend.jaegerQuery.monitorTab.prometheusEndpoint - description: REDMetricsNamespace defines the a prefix used retrieve span rate, - error, and duration (RED) metrics. By default it is set to `traces.span.metrics` - following the default namespace of the OpenTelemetry Collector since Version - 0.109.0. + error, and duration (RED) metrics. displayName: RED Metric Namespace path: template.queryFrontend.jaegerQuery.monitorTab.redMetricsNamespace - description: Resources defines resources for this component, this will override diff --git a/bundle/openshift/manifests/tempo.grafana.com_tempostacks.yaml b/bundle/openshift/manifests/tempo.grafana.com_tempostacks.yaml index 68eb5046f..6c93b0307 100644 --- a/bundle/openshift/manifests/tempo.grafana.com_tempostacks.yaml +++ b/bundle/openshift/manifests/tempo.grafana.com_tempostacks.yaml @@ -2471,10 +2471,9 @@ spec: For instance on OpenShift this is set to https://thanos-querier.openshift-monitoring.svc.cluster.local:9091 type: string redMetricsNamespace: - default: traces.span.metrics - description: |- - REDMetricsNamespace defines the a prefix used retrieve span rate, error, and duration (RED) metrics. - By default it is set to `traces.span.metrics` following the default namespace of the OpenTelemetry Collector since Version 0.109.0. + description: REDMetricsNamespace defines the a prefix + used retrieve span rate, error, and duration (RED) + metrics. type: string type: object resources: diff --git a/config/crd/bases/tempo.grafana.com_tempostacks.yaml b/config/crd/bases/tempo.grafana.com_tempostacks.yaml index 61e06807a..f753af9c2 100644 --- a/config/crd/bases/tempo.grafana.com_tempostacks.yaml +++ b/config/crd/bases/tempo.grafana.com_tempostacks.yaml @@ -2467,10 +2467,9 @@ spec: For instance on OpenShift this is set to https://thanos-querier.openshift-monitoring.svc.cluster.local:9091 type: string redMetricsNamespace: - default: traces.span.metrics - description: |- - REDMetricsNamespace defines the a prefix used retrieve span rate, error, and duration (RED) metrics. - By default it is set to `traces.span.metrics` following the default namespace of the OpenTelemetry Collector since Version 0.109.0. + description: REDMetricsNamespace defines the a prefix + used retrieve span rate, error, and duration (RED) + metrics. type: string type: object resources: diff --git a/config/manifests/community/bases/tempo-operator.clusterserviceversion.yaml b/config/manifests/community/bases/tempo-operator.clusterserviceversion.yaml index c717d3c71..19bd8ece3 100644 --- a/config/manifests/community/bases/tempo-operator.clusterserviceversion.yaml +++ b/config/manifests/community/bases/tempo-operator.clusterserviceversion.yaml @@ -1000,9 +1000,7 @@ spec: displayName: Prometheus endpoint path: template.queryFrontend.jaegerQuery.monitorTab.prometheusEndpoint - description: REDMetricsNamespace defines the a prefix used retrieve span rate, - error, and duration (RED) metrics. By default it is set to `traces.span.metrics` - following the default namespace of the OpenTelemetry Collector since Version - 0.109.0. + error, and duration (RED) metrics. displayName: RED Metric Namespace path: template.queryFrontend.jaegerQuery.monitorTab.redMetricsNamespace - description: Resources defines resources for this component, this will override diff --git a/config/manifests/openshift/bases/tempo-operator.clusterserviceversion.yaml b/config/manifests/openshift/bases/tempo-operator.clusterserviceversion.yaml index 244cef7a2..7d786bb90 100644 --- a/config/manifests/openshift/bases/tempo-operator.clusterserviceversion.yaml +++ b/config/manifests/openshift/bases/tempo-operator.clusterserviceversion.yaml @@ -1000,9 +1000,7 @@ spec: displayName: Prometheus endpoint path: template.queryFrontend.jaegerQuery.monitorTab.prometheusEndpoint - description: REDMetricsNamespace defines the a prefix used retrieve span rate, - error, and duration (RED) metrics. By default it is set to `traces.span.metrics` - following the default namespace of the OpenTelemetry Collector since Version - 0.109.0. + error, and duration (RED) metrics. displayName: RED Metric Namespace path: template.queryFrontend.jaegerQuery.monitorTab.redMetricsNamespace - description: Resources defines resources for this component, this will override diff --git a/docs/spec/tempo.grafana.com_tempostacks.yaml b/docs/spec/tempo.grafana.com_tempostacks.yaml index b946cb156..dee7ff186 100644 --- a/docs/spec/tempo.grafana.com_tempostacks.yaml +++ b/docs/spec/tempo.grafana.com_tempostacks.yaml @@ -342,7 +342,7 @@ spec: # TempoStackSpec defines the desired st monitorTab: # MonitorTab defines the monitor tab configuration. enabled: false # Enabled enables the monitor tab in the Jaeger console. The PrometheusEndpoint must be configured to enable this feature. prometheusEndpoint: "" # PrometheusEndpoint defines the endpoint to the Prometheus instance that contains the span rate, error, and duration (RED) metrics. For instance on OpenShift this is set to https://thanos-querier.openshift-monitoring.svc.cluster.local:9091 - redMetricsNamespace: "traces.span.metrics" # REDMetricsNamespace defines the a prefix used retrieve span rate, error, and duration (RED) metrics. By default it is set to `traces.span.metrics` following the default namespace of the OpenTelemetry Collector since Version 0.109.0. + redMetricsNamespace: "" # REDMetricsNamespace defines the a prefix used retrieve span rate, error, and duration (RED) metrics. servicesQueryDuration: "" # ServicesQueryDuration defines how long the services will be available in the services list tempoQuery: # TempoQuery defines options specific to the Tempoo Query component. resources: # Resources defines resources for this component, this will override the calculated resources derived from total diff --git a/internal/manifests/queryfrontend/query_frontend.go b/internal/manifests/queryfrontend/query_frontend.go index 55d1afe70..ffb17158b 100644 --- a/internal/manifests/queryfrontend/query_frontend.go +++ b/internal/manifests/queryfrontend/query_frontend.go @@ -398,11 +398,6 @@ func enableMonitoringTab(tempo v1alpha1.TempoStack, jaegerQueryContainer corev1. // However, we do not intend to support them. // --prometheus.query.normalize-calls // --prometheus.query.normalize-duration - // - // NOTE: Jaeger 1.62 default namespace changed to "traces_span_metrics". - // We fallback to no namespace. - // See https://github.com/jaegertracing/jaeger/pull/6007. - fmt.Sprintf("--prometheus.query.namespace=%s", tempo.Spec.Template.QueryFrontend.JaegerQuery.MonitorTab.REDMetricsNamespace), }, } // If the endpoint matches Prometheus on OpenShift, configure TLS and token based auth @@ -417,6 +412,15 @@ func enableMonitoringTab(tempo v1alpha1.TempoStack, jaegerQueryContainer corev1. "--prometheus.tls.ca=/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt") } + if tempo.Spec.Template.QueryFrontend.JaegerQuery.MonitorTab.REDMetricsNamespace != nil { + // NOTE: Jaeger 1.62 default namespace changed to "traces_span_metrics". + // Set .spec.template.queryFrontend.jaegerQuery.monitorTab.redMetricsNamespace explicitly to "" to disable the namespace. + // See https://github.com/jaegertracing/jaeger/pull/6007. + container.Args = append(container.Args, + fmt.Sprintf("--prometheus.query.namespace=%s", *tempo.Spec.Template.QueryFrontend.JaegerQuery.MonitorTab.REDMetricsNamespace), + ) + } + err := mergo.Merge(&jaegerQueryContainer, container, mergo.WithAppendSlice) if err != nil { return corev1.Container{}, err diff --git a/internal/manifests/queryfrontend/query_frontend_test.go b/internal/manifests/queryfrontend/query_frontend_test.go index 9ffb26b6a..fc726fc85 100644 --- a/internal/manifests/queryfrontend/query_frontend_test.go +++ b/internal/manifests/queryfrontend/query_frontend_test.go @@ -614,6 +614,26 @@ func TestBuildQueryFrontendWithJaegerMonitorTab(t *testing.T) { }, { name: "custom prometheus", + tempo: v1alpha1.TempoStack{ + Spec: v1alpha1.TempoStackSpec{ + Template: v1alpha1.TempoTemplateSpec{ + QueryFrontend: v1alpha1.TempoQueryFrontendSpec{ + JaegerQuery: v1alpha1.JaegerQuerySpec{ + Enabled: true, + MonitorTab: v1alpha1.JaegerQueryMonitor{ + Enabled: true, + PrometheusEndpoint: "http://prometheus:9091", + }, + }, + }, + }, + }, + }, + args: []string{"--query.base-path=/", "--span-storage.type=grpc", "--grpc-storage.server=localhost:7777", "--query.bearer-token-propagation=true"}, + env: []corev1.EnvVar{{Name: "METRICS_STORAGE_TYPE", Value: "prometheus"}, {Name: "PROMETHEUS_SERVER_URL", Value: "http://prometheus:9091"}}, + }, + { + name: "custom RED metrics namespace", tempo: v1alpha1.TempoStack{ Spec: v1alpha1.TempoStackSpec{ Template: v1alpha1.TempoTemplateSpec{ @@ -623,7 +643,7 @@ func TestBuildQueryFrontendWithJaegerMonitorTab(t *testing.T) { MonitorTab: v1alpha1.JaegerQueryMonitor{ Enabled: true, PrometheusEndpoint: "http://prometheus:9091", - REDMetricsNamespace: "test", + REDMetricsNamespace: ptr.To("test"), }, }, }, @@ -633,6 +653,27 @@ func TestBuildQueryFrontendWithJaegerMonitorTab(t *testing.T) { args: []string{"--query.base-path=/", "--span-storage.type=grpc", "--grpc-storage.server=localhost:7777", "--query.bearer-token-propagation=true", "--prometheus.query.namespace=test"}, env: []corev1.EnvVar{{Name: "METRICS_STORAGE_TYPE", Value: "prometheus"}, {Name: "PROMETHEUS_SERVER_URL", Value: "http://prometheus:9091"}}, }, + { + name: "disable default RED metrics namespace", + tempo: v1alpha1.TempoStack{ + Spec: v1alpha1.TempoStackSpec{ + Template: v1alpha1.TempoTemplateSpec{ + QueryFrontend: v1alpha1.TempoQueryFrontendSpec{ + JaegerQuery: v1alpha1.JaegerQuerySpec{ + Enabled: true, + MonitorTab: v1alpha1.JaegerQueryMonitor{ + Enabled: true, + PrometheusEndpoint: "http://prometheus:9091", + REDMetricsNamespace: ptr.To(""), + }, + }, + }, + }, + }, + }, + args: []string{"--query.base-path=/", "--span-storage.type=grpc", "--grpc-storage.server=localhost:7777", "--query.bearer-token-propagation=true", "--prometheus.query.namespace="}, + env: []corev1.EnvVar{{Name: "METRICS_STORAGE_TYPE", Value: "prometheus"}, {Name: "PROMETHEUS_SERVER_URL", Value: "http://prometheus:9091"}}, + }, { name: "OpenShift user-workload monitoring", tempo: v1alpha1.TempoStack{ @@ -658,7 +699,6 @@ func TestBuildQueryFrontendWithJaegerMonitorTab(t *testing.T) { "--span-storage.type=grpc", "--grpc-storage.server=localhost:7777", "--query.bearer-token-propagation=true", - "--prometheus.query.namespace=", "--prometheus.tls.enabled=true", "--prometheus.token-file=/var/run/secrets/kubernetes.io/serviceaccount/token", "--prometheus.token-override-from-context=false", diff --git a/tests/e2e-openshift/red-metrics/03-install-tempo.yaml b/tests/e2e-openshift/red-metrics/03-install-tempo.yaml index 9f045de85..dd8121223 100644 --- a/tests/e2e-openshift/red-metrics/03-install-tempo.yaml +++ b/tests/e2e-openshift/red-metrics/03-install-tempo.yaml @@ -28,6 +28,5 @@ spec: monitorTab: enabled: true prometheusEndpoint: https://thanos-querier.openshift-monitoring.svc.cluster.local:9091 - redMetricsNamespace: traces_span_metrics ingress: type: route