diff --git a/.chloggen/tempo-query-find-traces-jobs.yaml b/.chloggen/tempo-query-find-traces-jobs.yaml new file mode 100755 index 000000000..294deef9f --- /dev/null +++ b/.chloggen/tempo-query-find-traces-jobs.yaml @@ -0,0 +1,21 @@ +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: enhancement + +# The name of the component, or a single word describing the area of concern, (e.g. tempostack, tempomonolithic, github action) +component: tempostack, tempomonolithic + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Add tempo-query CRD option to speed up trace search. + +# One or more tracking issues related to the change +issues: [1048] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: | + Following CRD options were added to speed up trace search in Jaeger UI/API. The trace search first + searches for traceids and then it gets a full trace. With this configuration option the requests + to get the full trace can be run in parallel: + For `TempoStack` - `spec.template.queryFrontend.jaegerQuery.findTracesConcurrentRequests` + For `TempoMonolithic` - `spec.jaegerui.findTracesConcurrentRequests` diff --git a/Makefile b/Makefile index 8a459b1cd..8b5c89e13 100644 --- a/Makefile +++ b/Makefile @@ -1,8 +1,8 @@ # Current Operator version OPERATOR_VERSION ?= 0.13.0 TEMPO_VERSION ?= 2.5.0 -TEMPO_QUERY_VERSION ?= main-2999520 JAEGER_QUERY_VERSION ?= 1.62.0 +TEMPO_QUERY_VERSION ?= main-1de25ca TEMPO_GATEWAY_VERSION ?= main-2024-08-05-11d0d94 TEMPO_GATEWAY_OPA_VERSION ?= main-2024-04-29-914c13f OAUTH_PROXY_VERSION=4.14 diff --git a/apis/tempo/v1alpha1/tempomonolithic_defaults.go b/apis/tempo/v1alpha1/tempomonolithic_defaults.go index 39fc395f9..148422d60 100644 --- a/apis/tempo/v1alpha1/tempomonolithic_defaults.go +++ b/apis/tempo/v1alpha1/tempomonolithic_defaults.go @@ -13,10 +13,11 @@ import ( ) var ( - twoGBQuantity = resource.MustParse("2Gi") - tenGBQuantity = resource.MustParse("10Gi") - defaultServicesDuration = metav1.Duration{Duration: time.Hour * 24 * 3} - defaultTimeout = metav1.Duration{Duration: time.Second * 30} + twoGBQuantity = resource.MustParse("2Gi") + tenGBQuantity = resource.MustParse("10Gi") + defaultServicesDuration = metav1.Duration{Duration: time.Hour * 24 * 3} + defaultTimeout = metav1.Duration{Duration: time.Second * 30} + defaultFindTracesConcurrentRequests = 2 ) // Default sets all default values in a central place, instead of setting it at every place where the value is accessed. @@ -88,6 +89,9 @@ func (r *TempoMonolithic) Default(ctrlConfig configv1alpha1.ProjectConfig) { if r.Spec.JaegerUI.ServicesQueryDuration == nil { r.Spec.JaegerUI.ServicesQueryDuration = &defaultServicesDuration } + if r.Spec.JaegerUI.FindTracesConcurrentRequests == 0 { + r.Spec.JaegerUI.FindTracesConcurrentRequests = defaultFindTracesConcurrentRequests + } } if r.Spec.Timeout.Duration == 0 { diff --git a/apis/tempo/v1alpha1/tempomonolithic_defaults_test.go b/apis/tempo/v1alpha1/tempomonolithic_defaults_test.go index be28ff060..0768c3688 100644 --- a/apis/tempo/v1alpha1/tempomonolithic_defaults_test.go +++ b/apis/tempo/v1alpha1/tempomonolithic_defaults_test.go @@ -196,7 +196,8 @@ func TestMonolithicDefault(t *testing.T) { Enabled: true, SAR: "{\"namespace\": \"testns\", \"resource\": \"pods\", \"verb\": \"get\"}", }, - ServicesQueryDuration: &defaultServicesDuration, + ServicesQueryDuration: &defaultServicesDuration, + FindTracesConcurrentRequests: 2, }, Management: "Managed", Timeout: metav1.Duration{Duration: time.Second * 30}, @@ -269,7 +270,8 @@ func TestMonolithicDefault(t *testing.T) { Enabled: false, SAR: "{\"namespace\": \"testns\", \"resource\": \"pods\", \"verb\": \"get\"}", }, - ServicesQueryDuration: &defaultServicesDuration, + ServicesQueryDuration: &defaultServicesDuration, + FindTracesConcurrentRequests: 2, }, Management: "Managed", Timeout: metav1.Duration{Duration: time.Second * 30}, @@ -334,7 +336,8 @@ func TestMonolithicDefault(t *testing.T) { Enabled: true, SAR: "{\"namespace\": \"testns\", \"resource\": \"pods\", \"verb\": \"get\"}", }, - ServicesQueryDuration: &defaultServicesDuration, + ServicesQueryDuration: &defaultServicesDuration, + FindTracesConcurrentRequests: 2, }, Management: "Managed", Timeout: metav1.Duration{Duration: time.Second * 30}, @@ -398,7 +401,8 @@ func TestMonolithicDefault(t *testing.T) { Enabled: false, SAR: "{\"namespace\": \"testns\", \"resource\": \"pods\", \"verb\": \"get\"}", }, - ServicesQueryDuration: &defaultServicesDuration, + ServicesQueryDuration: &defaultServicesDuration, + FindTracesConcurrentRequests: 2, }, Management: "Managed", Timeout: metav1.Duration{Duration: time.Second * 30}, @@ -406,7 +410,7 @@ func TestMonolithicDefault(t *testing.T) { }, }, { - name: "define custom duration for services list and timeout", + name: "define custom duration for services list, timeout and find traces", input: &TempoMonolithic{ ObjectMeta: v1.ObjectMeta{ Name: "test", @@ -424,7 +428,8 @@ func TestMonolithicDefault(t *testing.T) { Route: &MonolithicJaegerUIRouteSpec{ Enabled: true, }, - ServicesQueryDuration: &v1.Duration{Duration: time.Duration(100 * 100)}, + ServicesQueryDuration: &v1.Duration{Duration: time.Duration(100 * 100)}, + FindTracesConcurrentRequests: 40, }, Timeout: metav1.Duration{Duration: time.Hour}, }, @@ -461,7 +466,8 @@ func TestMonolithicDefault(t *testing.T) { Enabled: false, SAR: "{\"namespace\": \"testns\", \"resource\": \"pods\", \"verb\": \"get\"}", }, - ServicesQueryDuration: &v1.Duration{Duration: time.Duration(100 * 100)}, + ServicesQueryDuration: &v1.Duration{Duration: time.Duration(100 * 100)}, + FindTracesConcurrentRequests: 40, }, Management: "Managed", Timeout: metav1.Duration{Duration: time.Hour}, diff --git a/apis/tempo/v1alpha1/tempomonolithic_types.go b/apis/tempo/v1alpha1/tempomonolithic_types.go index fef946876..b6db8fe0a 100644 --- a/apis/tempo/v1alpha1/tempomonolithic_types.go +++ b/apis/tempo/v1alpha1/tempomonolithic_types.go @@ -261,6 +261,19 @@ type MonolithicJaegerUISpec struct { // +optional // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="ServicesQueryDuration",xDescriptors="urn:alm:descriptor:com.tectonic.ui:advanced" ServicesQueryDuration *metav1.Duration `json:"servicesQueryDuration,omitempty"` + + // FindTracesConcurrentRequests defines how many concurrent request a single trace search can submit (defaults 2). + // The search for traces in Jaeger submits limit+1 requests. First requests finds trace IDs and then it fetches + // entire traces by ID. This property allows Jaeger to fetch traces in parallel. + // Note that by default a single Tempo querier can process 20 concurrent search jobs. + // Increasing this property might require scaling up querier instances, especially on error "job queue full" + // See also Tempo's extraConfig: + // querier.max_concurrent_queries (20 default) + // query_frontend.max_outstanding_per_tenant: (2000 default). Increase if the query-frontend returns 429 + // + // +optional + // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="FindTracesConcurrentRequests",xDescriptors="urn:alm:descriptor:com.tectonic.ui:advanced" + FindTracesConcurrentRequests int `json:"findTracesConcurrentRequests,omitempty"` } // MonolithicJaegerUIIngressSpec defines the settings for the Jaeger UI ingress. diff --git a/apis/tempo/v1alpha1/tempostack_types.go b/apis/tempo/v1alpha1/tempostack_types.go index cc0186ab0..e868b207d 100644 --- a/apis/tempo/v1alpha1/tempostack_types.go +++ b/apis/tempo/v1alpha1/tempostack_types.go @@ -618,6 +618,19 @@ type JaegerQuerySpec struct { // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="ServicesQueryDuration" ServicesQueryDuration *metav1.Duration `json:"servicesQueryDuration,omitempty"` + // FindTracesConcurrentRequests defines how many concurrent request a single trace search can submit (defaults querier.replicas*2). + // The search for traces in Jaeger submits limit+1 requests. First requests finds trace IDs and then it fetches + // entire traces by ID. This property allows Jaeger to fetch traces in parallel. + // Note that by default a single Tempo querier can process 20 concurrent search jobs. + // Increasing this property might require scaling up querier instances, especially on error "job queue full" + // See also Tempo's extraConfig: + // querier.max_concurrent_queries (20 default) + // query_frontend.max_outstanding_per_tenant: (2000 default). Increase if the query-frontend returns 429 + // + // +optional + // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="FindTracesConcurrentRequests",xDescriptors="urn:alm:descriptor:com.tectonic.ui:advanced" + FindTracesConcurrentRequests int `json:"findTracesConcurrentRequests,omitempty"` + // Authentication defines the options for the oauth proxy used to protect jaeger UI // // +optional diff --git a/bundle/community/manifests/tempo-operator.clusterserviceversion.yaml b/bundle/community/manifests/tempo-operator.clusterserviceversion.yaml index aa33b1897..e597be8a4 100644 --- a/bundle/community/manifests/tempo-operator.clusterserviceversion.yaml +++ b/bundle/community/manifests/tempo-operator.clusterserviceversion.yaml @@ -74,7 +74,7 @@ metadata: capabilities: Deep Insights categories: Logging & Tracing,Monitoring containerImage: ghcr.io/grafana/tempo-operator/tempo-operator:v0.13.0 - createdAt: "2024-10-07T07:11:28Z" + createdAt: "2024-10-10T15:57:46Z" description: Create and manage deployments of Tempo, a high-scale distributed tracing backend. operatorframework.io/cluster-monitoring: "true" @@ -307,6 +307,19 @@ spec: "{"namespace": "", "resource": "pods", "verb": "get"}' displayName: SAR path: jaegerui.authentication.sar + - description: 'FindTracesConcurrentRequests defines how many concurrent request + a single trace search can submit (defaults 2). The search for traces in + Jaeger submits limit+1 requests. First requests finds trace IDs and then + it fetches entire traces by ID. This property allows Jaeger to fetch traces + in parallel. Note that by default a single Tempo querier can process 20 + concurrent search jobs. Increasing this property might require scaling up + querier instances, especially on error "job queue full" See also Tempo''s + extraConfig: querier.max_concurrent_queries (20 default) query_frontend.max_outstanding_per_tenant: + (2000 default). Increase if the query-frontend returns 429' + displayName: FindTracesConcurrentRequests + path: jaegerui.findTracesConcurrentRequests + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:advanced - description: Annotations defines the annotations of the Ingress object. displayName: Annotations path: jaegerui.ingress.annotations @@ -1002,6 +1015,19 @@ spec: path: template.queryFrontend.jaegerQuery.enabled x-descriptors: - urn:alm:descriptor:com.tectonic.ui:booleanSwitch + - description: 'FindTracesConcurrentRequests defines how many concurrent request + a single trace search can submit (defaults querier.replicas*2). The search + for traces in Jaeger submits limit+1 requests. First requests finds trace + IDs and then it fetches entire traces by ID. This property allows Jaeger + to fetch traces in parallel. Note that by default a single Tempo querier + can process 20 concurrent search jobs. Increasing this property might require + scaling up querier instances, especially on error "job queue full" See also + Tempo''s extraConfig: querier.max_concurrent_queries (20 default) query_frontend.max_outstanding_per_tenant: + (2000 default). Increase if the query-frontend returns 429' + displayName: FindTracesConcurrentRequests + path: template.queryFrontend.jaegerQuery.findTracesConcurrentRequests + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:advanced - description: Ingress defines the options for the Jaeger Query ingress. displayName: Jaeger Query UI Ingress Settings path: template.queryFrontend.jaegerQuery.ingress @@ -1427,7 +1453,7 @@ spec: - name: RELATED_IMAGE_JAEGER_QUERY value: docker.io/jaegertracing/jaeger-query:1.62.0 - name: RELATED_IMAGE_TEMPO_QUERY - value: docker.io/grafana/tempo-query:main-2999520 + value: docker.io/grafana/tempo-query:main-1de25ca - name: RELATED_IMAGE_TEMPO_GATEWAY value: quay.io/observatorium/api:main-2024-08-05-11d0d94 - name: RELATED_IMAGE_TEMPO_GATEWAY_OPA @@ -1576,7 +1602,7 @@ spec: name: tempo - image: docker.io/jaegertracing/jaeger-query:1.62.0 name: jaeger-query - - image: docker.io/grafana/tempo-query:main-2999520 + - image: docker.io/grafana/tempo-query:main-1de25ca name: tempo-query - image: quay.io/observatorium/api:main-2024-08-05-11d0d94 name: tempo-gateway diff --git a/bundle/community/manifests/tempo.grafana.com_tempomonolithics.yaml b/bundle/community/manifests/tempo.grafana.com_tempomonolithics.yaml index 115daff46..6d285729c 100644 --- a/bundle/community/manifests/tempo.grafana.com_tempomonolithics.yaml +++ b/bundle/community/manifests/tempo.grafana.com_tempomonolithics.yaml @@ -1109,6 +1109,17 @@ spec: description: Enabled defines if the Jaeger UI component should be created. type: boolean + findTracesConcurrentRequests: + description: |- + FindTracesConcurrentRequests defines how many concurrent request a single trace search can submit (defaults 2). + The search for traces in Jaeger submits limit+1 requests. First requests finds trace IDs and then it fetches + entire traces by ID. This property allows Jaeger to fetch traces in parallel. + Note that by default a single Tempo querier can process 20 concurrent search jobs. + Increasing this property might require scaling up querier instances, especially on error "job queue full" + See also Tempo's extraConfig: + querier.max_concurrent_queries (20 default) + query_frontend.max_outstanding_per_tenant: (2000 default). Increase if the query-frontend returns 429 + type: integer ingress: description: Ingress defines the Ingress configuration for the Jaeger UI. diff --git a/bundle/community/manifests/tempo.grafana.com_tempostacks.yaml b/bundle/community/manifests/tempo.grafana.com_tempostacks.yaml index eaede14e7..a34011a10 100644 --- a/bundle/community/manifests/tempo.grafana.com_tempostacks.yaml +++ b/bundle/community/manifests/tempo.grafana.com_tempostacks.yaml @@ -2391,6 +2391,17 @@ spec: description: Enabled defines if the Jaeger Query component should be created. type: boolean + findTracesConcurrentRequests: + description: |- + FindTracesConcurrentRequests defines how many concurrent request a single trace search can submit (defaults querier.replicas*2). + The search for traces in Jaeger submits limit+1 requests. First requests finds trace IDs and then it fetches + entire traces by ID. This property allows Jaeger to fetch traces in parallel. + Note that by default a single Tempo querier can process 20 concurrent search jobs. + Increasing this property might require scaling up querier instances, especially on error "job queue full" + See also Tempo's extraConfig: + querier.max_concurrent_queries (20 default) + query_frontend.max_outstanding_per_tenant: (2000 default). Increase if the query-frontend returns 429 + type: integer ingress: description: Ingress defines the options for the Jaeger Query ingress. diff --git a/bundle/openshift/manifests/tempo-operator.clusterserviceversion.yaml b/bundle/openshift/manifests/tempo-operator.clusterserviceversion.yaml index 5f8dba924..172fe9a94 100644 --- a/bundle/openshift/manifests/tempo-operator.clusterserviceversion.yaml +++ b/bundle/openshift/manifests/tempo-operator.clusterserviceversion.yaml @@ -74,7 +74,7 @@ metadata: capabilities: Deep Insights categories: Logging & Tracing,Monitoring containerImage: ghcr.io/grafana/tempo-operator/tempo-operator:v0.13.0 - createdAt: "2024-10-07T07:11:27Z" + createdAt: "2024-10-10T15:57:44Z" description: Create and manage deployments of Tempo, a high-scale distributed tracing backend. operatorframework.io/cluster-monitoring: "true" @@ -307,6 +307,19 @@ spec: "{"namespace": "", "resource": "pods", "verb": "get"}' displayName: SAR path: jaegerui.authentication.sar + - description: 'FindTracesConcurrentRequests defines how many concurrent request + a single trace search can submit (defaults 2). The search for traces in + Jaeger submits limit+1 requests. First requests finds trace IDs and then + it fetches entire traces by ID. This property allows Jaeger to fetch traces + in parallel. Note that by default a single Tempo querier can process 20 + concurrent search jobs. Increasing this property might require scaling up + querier instances, especially on error "job queue full" See also Tempo''s + extraConfig: querier.max_concurrent_queries (20 default) query_frontend.max_outstanding_per_tenant: + (2000 default). Increase if the query-frontend returns 429' + displayName: FindTracesConcurrentRequests + path: jaegerui.findTracesConcurrentRequests + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:advanced - description: Annotations defines the annotations of the Ingress object. displayName: Annotations path: jaegerui.ingress.annotations @@ -1002,6 +1015,19 @@ spec: path: template.queryFrontend.jaegerQuery.enabled x-descriptors: - urn:alm:descriptor:com.tectonic.ui:booleanSwitch + - description: 'FindTracesConcurrentRequests defines how many concurrent request + a single trace search can submit (defaults querier.replicas*2). The search + for traces in Jaeger submits limit+1 requests. First requests finds trace + IDs and then it fetches entire traces by ID. This property allows Jaeger + to fetch traces in parallel. Note that by default a single Tempo querier + can process 20 concurrent search jobs. Increasing this property might require + scaling up querier instances, especially on error "job queue full" See also + Tempo''s extraConfig: querier.max_concurrent_queries (20 default) query_frontend.max_outstanding_per_tenant: + (2000 default). Increase if the query-frontend returns 429' + displayName: FindTracesConcurrentRequests + path: template.queryFrontend.jaegerQuery.findTracesConcurrentRequests + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:advanced - description: Ingress defines the options for the Jaeger Query ingress. displayName: Jaeger Query UI Ingress Settings path: template.queryFrontend.jaegerQuery.ingress @@ -1437,7 +1463,7 @@ spec: - name: RELATED_IMAGE_JAEGER_QUERY value: docker.io/jaegertracing/jaeger-query:1.62.0 - name: RELATED_IMAGE_TEMPO_QUERY - value: docker.io/grafana/tempo-query:main-2999520 + value: docker.io/grafana/tempo-query:main-1de25ca - name: RELATED_IMAGE_TEMPO_GATEWAY value: quay.io/observatorium/api:main-2024-08-05-11d0d94 - name: RELATED_IMAGE_TEMPO_GATEWAY_OPA @@ -1597,7 +1623,7 @@ spec: name: tempo - image: docker.io/jaegertracing/jaeger-query:1.62.0 name: jaeger-query - - image: docker.io/grafana/tempo-query:main-2999520 + - image: docker.io/grafana/tempo-query:main-1de25ca name: tempo-query - image: quay.io/observatorium/api:main-2024-08-05-11d0d94 name: tempo-gateway diff --git a/bundle/openshift/manifests/tempo.grafana.com_tempomonolithics.yaml b/bundle/openshift/manifests/tempo.grafana.com_tempomonolithics.yaml index 115daff46..6d285729c 100644 --- a/bundle/openshift/manifests/tempo.grafana.com_tempomonolithics.yaml +++ b/bundle/openshift/manifests/tempo.grafana.com_tempomonolithics.yaml @@ -1109,6 +1109,17 @@ spec: description: Enabled defines if the Jaeger UI component should be created. type: boolean + findTracesConcurrentRequests: + description: |- + FindTracesConcurrentRequests defines how many concurrent request a single trace search can submit (defaults 2). + The search for traces in Jaeger submits limit+1 requests. First requests finds trace IDs and then it fetches + entire traces by ID. This property allows Jaeger to fetch traces in parallel. + Note that by default a single Tempo querier can process 20 concurrent search jobs. + Increasing this property might require scaling up querier instances, especially on error "job queue full" + See also Tempo's extraConfig: + querier.max_concurrent_queries (20 default) + query_frontend.max_outstanding_per_tenant: (2000 default). Increase if the query-frontend returns 429 + type: integer ingress: description: Ingress defines the Ingress configuration for the Jaeger UI. diff --git a/bundle/openshift/manifests/tempo.grafana.com_tempostacks.yaml b/bundle/openshift/manifests/tempo.grafana.com_tempostacks.yaml index eaede14e7..a34011a10 100644 --- a/bundle/openshift/manifests/tempo.grafana.com_tempostacks.yaml +++ b/bundle/openshift/manifests/tempo.grafana.com_tempostacks.yaml @@ -2391,6 +2391,17 @@ spec: description: Enabled defines if the Jaeger Query component should be created. type: boolean + findTracesConcurrentRequests: + description: |- + FindTracesConcurrentRequests defines how many concurrent request a single trace search can submit (defaults querier.replicas*2). + The search for traces in Jaeger submits limit+1 requests. First requests finds trace IDs and then it fetches + entire traces by ID. This property allows Jaeger to fetch traces in parallel. + Note that by default a single Tempo querier can process 20 concurrent search jobs. + Increasing this property might require scaling up querier instances, especially on error "job queue full" + See also Tempo's extraConfig: + querier.max_concurrent_queries (20 default) + query_frontend.max_outstanding_per_tenant: (2000 default). Increase if the query-frontend returns 429 + type: integer ingress: description: Ingress defines the options for the Jaeger Query ingress. diff --git a/config/crd/bases/tempo.grafana.com_tempomonolithics.yaml b/config/crd/bases/tempo.grafana.com_tempomonolithics.yaml index 92a7a6753..918f93e42 100644 --- a/config/crd/bases/tempo.grafana.com_tempomonolithics.yaml +++ b/config/crd/bases/tempo.grafana.com_tempomonolithics.yaml @@ -1105,6 +1105,17 @@ spec: description: Enabled defines if the Jaeger UI component should be created. type: boolean + findTracesConcurrentRequests: + description: |- + FindTracesConcurrentRequests defines how many concurrent request a single trace search can submit (defaults 2). + The search for traces in Jaeger submits limit+1 requests. First requests finds trace IDs and then it fetches + entire traces by ID. This property allows Jaeger to fetch traces in parallel. + Note that by default a single Tempo querier can process 20 concurrent search jobs. + Increasing this property might require scaling up querier instances, especially on error "job queue full" + See also Tempo's extraConfig: + querier.max_concurrent_queries (20 default) + query_frontend.max_outstanding_per_tenant: (2000 default). Increase if the query-frontend returns 429 + type: integer ingress: description: Ingress defines the Ingress configuration for the Jaeger UI. diff --git a/config/crd/bases/tempo.grafana.com_tempostacks.yaml b/config/crd/bases/tempo.grafana.com_tempostacks.yaml index f1374fbd5..6212f3001 100644 --- a/config/crd/bases/tempo.grafana.com_tempostacks.yaml +++ b/config/crd/bases/tempo.grafana.com_tempostacks.yaml @@ -2387,6 +2387,17 @@ spec: description: Enabled defines if the Jaeger Query component should be created. type: boolean + findTracesConcurrentRequests: + description: |- + FindTracesConcurrentRequests defines how many concurrent request a single trace search can submit (defaults querier.replicas*2). + The search for traces in Jaeger submits limit+1 requests. First requests finds trace IDs and then it fetches + entire traces by ID. This property allows Jaeger to fetch traces in parallel. + Note that by default a single Tempo querier can process 20 concurrent search jobs. + Increasing this property might require scaling up querier instances, especially on error "job queue full" + See also Tempo's extraConfig: + querier.max_concurrent_queries (20 default) + query_frontend.max_outstanding_per_tenant: (2000 default). Increase if the query-frontend returns 429 + type: integer ingress: description: Ingress defines the options for the Jaeger Query ingress. diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index 4db4cd802..a1a7f323f 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -44,7 +44,7 @@ spec: - name: RELATED_IMAGE_JAEGER_QUERY value: docker.io/jaegertracing/jaeger-query:1.62.0 - name: RELATED_IMAGE_TEMPO_QUERY - value: docker.io/grafana/tempo-query:main-2999520 + value: docker.io/grafana/tempo-query:main-1de25ca - name: RELATED_IMAGE_TEMPO_GATEWAY value: quay.io/observatorium/api:main-2024-08-05-11d0d94 - name: RELATED_IMAGE_TEMPO_GATEWAY_OPA diff --git a/config/manifests/community/bases/tempo-operator.clusterserviceversion.yaml b/config/manifests/community/bases/tempo-operator.clusterserviceversion.yaml index 419a32f01..183b5d776 100644 --- a/config/manifests/community/bases/tempo-operator.clusterserviceversion.yaml +++ b/config/manifests/community/bases/tempo-operator.clusterserviceversion.yaml @@ -236,6 +236,19 @@ spec: "{"namespace": "", "resource": "pods", "verb": "get"}' displayName: SAR path: jaegerui.authentication.sar + - description: 'FindTracesConcurrentRequests defines how many concurrent request + a single trace search can submit (defaults 2). The search for traces in + Jaeger submits limit+1 requests. First requests finds trace IDs and then + it fetches entire traces by ID. This property allows Jaeger to fetch traces + in parallel. Note that by default a single Tempo querier can process 20 + concurrent search jobs. Increasing this property might require scaling up + querier instances, especially on error "job queue full" See also Tempo''s + extraConfig: querier.max_concurrent_queries (20 default) query_frontend.max_outstanding_per_tenant: + (2000 default). Increase if the query-frontend returns 429' + displayName: FindTracesConcurrentRequests + path: jaegerui.findTracesConcurrentRequests + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:advanced - description: Annotations defines the annotations of the Ingress object. displayName: Annotations path: jaegerui.ingress.annotations @@ -931,6 +944,19 @@ spec: path: template.queryFrontend.jaegerQuery.enabled x-descriptors: - urn:alm:descriptor:com.tectonic.ui:booleanSwitch + - description: 'FindTracesConcurrentRequests defines how many concurrent request + a single trace search can submit (defaults querier.replicas*2). The search + for traces in Jaeger submits limit+1 requests. First requests finds trace + IDs and then it fetches entire traces by ID. This property allows Jaeger + to fetch traces in parallel. Note that by default a single Tempo querier + can process 20 concurrent search jobs. Increasing this property might require + scaling up querier instances, especially on error "job queue full" See also + Tempo''s extraConfig: querier.max_concurrent_queries (20 default) query_frontend.max_outstanding_per_tenant: + (2000 default). Increase if the query-frontend returns 429' + displayName: FindTracesConcurrentRequests + path: template.queryFrontend.jaegerQuery.findTracesConcurrentRequests + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:advanced - description: Ingress defines the options for the Jaeger Query ingress. displayName: Jaeger Query UI Ingress Settings path: template.queryFrontend.jaegerQuery.ingress diff --git a/config/manifests/openshift/bases/tempo-operator.clusterserviceversion.yaml b/config/manifests/openshift/bases/tempo-operator.clusterserviceversion.yaml index 1b04fa519..55a0cd54a 100644 --- a/config/manifests/openshift/bases/tempo-operator.clusterserviceversion.yaml +++ b/config/manifests/openshift/bases/tempo-operator.clusterserviceversion.yaml @@ -236,6 +236,19 @@ spec: "{"namespace": "", "resource": "pods", "verb": "get"}' displayName: SAR path: jaegerui.authentication.sar + - description: 'FindTracesConcurrentRequests defines how many concurrent request + a single trace search can submit (defaults 2). The search for traces in + Jaeger submits limit+1 requests. First requests finds trace IDs and then + it fetches entire traces by ID. This property allows Jaeger to fetch traces + in parallel. Note that by default a single Tempo querier can process 20 + concurrent search jobs. Increasing this property might require scaling up + querier instances, especially on error "job queue full" See also Tempo''s + extraConfig: querier.max_concurrent_queries (20 default) query_frontend.max_outstanding_per_tenant: + (2000 default). Increase if the query-frontend returns 429' + displayName: FindTracesConcurrentRequests + path: jaegerui.findTracesConcurrentRequests + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:advanced - description: Annotations defines the annotations of the Ingress object. displayName: Annotations path: jaegerui.ingress.annotations @@ -931,6 +944,19 @@ spec: path: template.queryFrontend.jaegerQuery.enabled x-descriptors: - urn:alm:descriptor:com.tectonic.ui:booleanSwitch + - description: 'FindTracesConcurrentRequests defines how many concurrent request + a single trace search can submit (defaults querier.replicas*2). The search + for traces in Jaeger submits limit+1 requests. First requests finds trace + IDs and then it fetches entire traces by ID. This property allows Jaeger + to fetch traces in parallel. Note that by default a single Tempo querier + can process 20 concurrent search jobs. Increasing this property might require + scaling up querier instances, especially on error "job queue full" See also + Tempo''s extraConfig: querier.max_concurrent_queries (20 default) query_frontend.max_outstanding_per_tenant: + (2000 default). Increase if the query-frontend returns 429' + displayName: FindTracesConcurrentRequests + path: template.queryFrontend.jaegerQuery.findTracesConcurrentRequests + x-descriptors: + - urn:alm:descriptor:com.tectonic.ui:advanced - description: Ingress defines the options for the Jaeger Query ingress. displayName: Jaeger Query UI Ingress Settings path: template.queryFrontend.jaegerQuery.ingress diff --git a/docs/spec/tempo.grafana.com_tempomonolithics.yaml b/docs/spec/tempo.grafana.com_tempomonolithics.yaml index 487a6cfd7..f4ab9605b 100644 --- a/docs/spec/tempo.grafana.com_tempomonolithics.yaml +++ b/docs/spec/tempo.grafana.com_tempomonolithics.yaml @@ -35,6 +35,7 @@ spec: # TempoMonolithicSpec defines the desir requests: # Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. Requests cannot exceed Limits. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ cpu: "500m" memory: "1Gi" + findTracesConcurrentRequests: 0 # FindTracesConcurrentRequests defines how many concurrent request a single trace search can submit (defaults 2). The search for traces in Jaeger submits limit+1 requests. First requests finds trace IDs and then it fetches entire traces by ID. This property allows Jaeger to fetch traces in parallel. Note that by default a single Tempo querier can process 20 concurrent search jobs. Increasing this property might require scaling up querier instances, especially on error "job queue full" See also Tempo's extraConfig: querier.max_concurrent_queries (20 default) query_frontend.max_outstanding_per_tenant: (2000 default). Increase if the query-frontend returns 429 ingress: # Ingress defines the Ingress configuration for the Jaeger UI. enabled: false # Enabled defines if an Ingress object should be created for Jaeger UI. annotations: {} # Annotations defines the annotations of the Ingress object. diff --git a/docs/spec/tempo.grafana.com_tempostacks.yaml b/docs/spec/tempo.grafana.com_tempostacks.yaml index 2b65cc6a7..ac32bdb41 100644 --- a/docs/spec/tempo.grafana.com_tempostacks.yaml +++ b/docs/spec/tempo.grafana.com_tempostacks.yaml @@ -330,6 +330,7 @@ spec: # TempoStackSpec defines the desired st requests: # Requests describes the minimum amount of compute resources required. If Requests is omitted for a container, it defaults to Limits if that is explicitly specified, otherwise to an implementation-defined value. Requests cannot exceed Limits. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/ cpu: "500m" memory: "1Gi" + findTracesConcurrentRequests: 0 # FindTracesConcurrentRequests defines how many concurrent request a single trace search can submit (defaults querier.replicas*2). The search for traces in Jaeger submits limit+1 requests. First requests finds trace IDs and then it fetches entire traces by ID. This property allows Jaeger to fetch traces in parallel. Note that by default a single Tempo querier can process 20 concurrent search jobs. Increasing this property might require scaling up querier instances, especially on error "job queue full" See also Tempo's extraConfig: querier.max_concurrent_queries (20 default) query_frontend.max_outstanding_per_tenant: (2000 default). Increase if the query-frontend returns 429 ingress: # Ingress defines the options for the Jaeger Query ingress. annotations: {} # Annotations defines the annotations of the Ingress object. host: "" # Host defines the hostname of the Ingress object. diff --git a/internal/manifests/config/build.go b/internal/manifests/config/build.go index a20a60d78..9b17bc705 100644 --- a/internal/manifests/config/build.go +++ b/internal/manifests/config/build.go @@ -169,6 +169,14 @@ func buildTempoQueryConfig(params manifestutils.Params) ([]byte, error) { return []byte{}, err } + findTracesConcurrentRequests := params.Tempo.Spec.Template.QueryFrontend.JaegerQuery.FindTracesConcurrentRequests + if findTracesConcurrentRequests == 0 { + querierReplicas := int32(1) + if params.Tempo.Spec.Template.Querier.Replicas != nil { + querierReplicas = *params.Tempo.Spec.Template.Querier.Replicas + } + findTracesConcurrentRequests = int(querierReplicas) * 2 + } return renderTempoQueryTemplate(tempoQueryOptions{ TLS: tlsopts, HTTPPort: manifestutils.PortHTTPServer, @@ -176,9 +184,10 @@ func buildTempoQueryConfig(params manifestutils.Params) ([]byte, error) { GRPCEncryption: params.CtrlConfig.Gates.GRPCEncryption, HTTPEncryption: params.CtrlConfig.Gates.HTTPEncryption, }, - TenantHeader: manifestutils.TenantHeader, - Gateway: params.Tempo.Spec.Template.Gateway.Enabled, - ServicesQueryDuration: params.Tempo.Spec.Template.QueryFrontend.JaegerQuery.ServicesQueryDuration.Duration.String(), + TenantHeader: manifestutils.TenantHeader, + Gateway: params.Tempo.Spec.Template.Gateway.Enabled, + ServicesQueryDuration: params.Tempo.Spec.Template.QueryFrontend.JaegerQuery.ServicesQueryDuration.Duration.String(), + FindTracesConcurrentRequests: findTracesConcurrentRequests, }) } diff --git a/internal/manifests/config/options.go b/internal/manifests/config/options.go index 02ce67481..d34519764 100644 --- a/internal/manifests/config/options.go +++ b/internal/manifests/config/options.go @@ -27,12 +27,13 @@ type options struct { } type tempoQueryOptions struct { - Gates featureGates - TLS tlsOptions - HTTPPort int - TenantHeader string - Gateway bool - ServicesQueryDuration string + Gates featureGates + TLS tlsOptions + HTTPPort int + TenantHeader string + Gateway bool + ServicesQueryDuration string + FindTracesConcurrentRequests int } type featureGates struct { diff --git a/internal/manifests/config/tempo-query.yaml b/internal/manifests/config/tempo-query.yaml index 4096bc107..782c95d3d 100644 --- a/internal/manifests/config/tempo-query.yaml +++ b/internal/manifests/config/tempo-query.yaml @@ -10,3 +10,4 @@ tls_insecure_skip_verify: false tls_server_name: {{ .TLS.ServerNames.QueryFrontend }} {{- end }} services_query_duration: {{ .ServicesQueryDuration }} +find_traces_concurrent_requests: {{ .FindTracesConcurrentRequests }} diff --git a/internal/manifests/monolithic/configmap.go b/internal/manifests/monolithic/configmap.go index 8c1a1ea94..c9ae7ba2f 100644 --- a/internal/manifests/monolithic/configmap.go +++ b/internal/manifests/monolithic/configmap.go @@ -96,10 +96,11 @@ type tempoConfig struct { } type tempoQueryConfig struct { - Address string `yaml:"address"` - Backend string `yaml:"backend"` - TenantHeaderKey string `yaml:"tenant_header_key"` - ServicesQueryDuration time.Duration `yaml:"services_query_duration"` + Address string `yaml:"address"` + Backend string `yaml:"backend"` + TenantHeaderKey string `yaml:"tenant_header_key"` + ServicesQueryDuration time.Duration `yaml:"services_query_duration"` + FindTracesConcurrentRequests int `yaml:"find_traces_concurrent_requests"` } // BuildConfigMap creates the Tempo ConfigMap for a monolithic deployment. @@ -294,5 +295,6 @@ func buildTempoQueryConfig(jaegerUISpec *v1alpha1.MonolithicJaegerUISpec) ([]byt config.Backend = fmt.Sprintf("127.0.0.1:%d", manifestutils.PortHTTPServer) config.TenantHeaderKey = manifestutils.TenantHeader config.ServicesQueryDuration = jaegerUISpec.ServicesQueryDuration.Duration + config.FindTracesConcurrentRequests = jaegerUISpec.FindTracesConcurrentRequests return yaml.Marshal(&config) } diff --git a/internal/manifests/monolithic/configmap_test.go b/internal/manifests/monolithic/configmap_test.go index 7eab5b3f3..79ed15f45 100644 --- a/internal/manifests/monolithic/configmap_test.go +++ b/internal/manifests/monolithic/configmap_test.go @@ -42,8 +42,9 @@ func TestBuildConfigMap(t *testing.T) { }, }, JaegerUI: &v1alpha1.MonolithicJaegerUISpec{ - Enabled: true, - ServicesQueryDuration: &metav1.Duration{Duration: time.Duration(3 * 24 * time.Hour)}, + Enabled: true, + ServicesQueryDuration: &metav1.Duration{Duration: time.Duration(3 * 24 * time.Hour)}, + FindTracesConcurrentRequests: 22, }, }, }, @@ -63,6 +64,7 @@ address: 127.0.0.1:7777 backend: 127.0.0.1:3200 tenant_header_key: x-scope-orgid services_query_duration: 72h0m0s +find_traces_concurrent_requests: 22 ` require.YAMLEq(t, tempoQueryCfg, cm.Data["tempo-query.yaml"]) } diff --git a/tests/e2e/monolithic-extraconfig/install-tempo-assert.yaml b/tests/e2e/monolithic-extraconfig/install-tempo-assert.yaml index b2244e1df..883e687d7 100644 --- a/tests/e2e/monolithic-extraconfig/install-tempo-assert.yaml +++ b/tests/e2e/monolithic-extraconfig/install-tempo-assert.yaml @@ -81,6 +81,7 @@ data: backend: 127.0.0.1:3200 tenant_header_key: x-scope-orgid services_query_duration: 72h0m0s + find_traces_concurrent_requests: 2 tempo.yaml: | distributor: receivers: diff --git a/tests/e2e/tempostack-extraconfig/install-tempostack-assert.yaml b/tests/e2e/tempostack-extraconfig/install-tempostack-assert.yaml index 4cad151d3..6b7d3b59c 100644 --- a/tests/e2e/tempostack-extraconfig/install-tempostack-assert.yaml +++ b/tests/e2e/tempostack-extraconfig/install-tempostack-assert.yaml @@ -480,6 +480,7 @@ data: backend: 127.0.0.1:3200 tenant_header_key: x-scope-orgid services_query_duration: 72h0m0s + find_traces_concurrent_requests: 2 tempo.yaml: | compactor: compaction: