Skip to content

Commit

Permalink
Tempo query improve search performance
Browse files Browse the repository at this point in the history 
Signed-off-by: Pavol Loffay <[email protected]>
  • Loading branch information
@pavolloffay
pavolloffay committed Oct 7, 2024
1 parent c7eb01b commit a63eb5f
Show file tree
Hide file tree
Showing 13 changed files with 83 additions and 32 deletions.
12 changes: 8 additions & 4 deletions apis/tempo/v1alpha1/tempomonolithic_defaults.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -13,10 +13,11 @@ import (
)

var (
twoGBQuantity = resource.MustParse("2Gi")
tenGBQuantity = resource.MustParse("10Gi")
defaultServicesDuration = metav1.Duration{Duration: time.Hour * 24 * 3}
defaultTimeout = metav1.Duration{Duration: time.Second * 30}
twoGBQuantity = resource.MustParse("2Gi")
tenGBQuantity = resource.MustParse("10Gi")
defaultServicesDuration = metav1.Duration{Duration: time.Hour * 24 * 3}
defaultTimeout = metav1.Duration{Duration: time.Second * 30}
defaultFindTracesConcurrentRequests = 1
)

// Default sets all default values in a central place, instead of setting it at every place where the value is accessed.
Expand Down Expand Up @@ -88,6 +89,9 @@ func (r *TempoMonolithic) Default(ctrlConfig configv1alpha1.ProjectConfig) {
if r.Spec.JaegerUI.ServicesQueryDuration == nil {
r.Spec.JaegerUI.ServicesQueryDuration = &defaultServicesDuration
}
if r.Spec.JaegerUI.FindTracesConcurrentRequests == 0 {
r.Spec.JaegerUI.FindTracesConcurrentRequests = defaultFindTracesConcurrentRequests
}
}

if r.Spec.Timeout.Duration == 0 {
Expand Down
20 changes: 13 additions & 7 deletions apis/tempo/v1alpha1/tempomonolithic_defaults_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -196,7 +196,8 @@ func TestMonolithicDefault(t *testing.T) {
Enabled: true,
SAR: "{\"namespace\": \"testns\", \"resource\": \"pods\", \"verb\": \"get\"}",
},
ServicesQueryDuration: &defaultServicesDuration,
ServicesQueryDuration: &defaultServicesDuration,
FindTracesConcurrentRequests: 1,
},
Management: "Managed",
Timeout: metav1.Duration{Duration: time.Second * 30},
Expand Down Expand Up @@ -269,7 +270,8 @@ func TestMonolithicDefault(t *testing.T) {
Enabled: false,
SAR: "{\"namespace\": \"testns\", \"resource\": \"pods\", \"verb\": \"get\"}",
},
ServicesQueryDuration: &defaultServicesDuration,
ServicesQueryDuration: &defaultServicesDuration,
FindTracesConcurrentRequests: 1,
},
Management: "Managed",
Timeout: metav1.Duration{Duration: time.Second * 30},
Expand Down Expand Up @@ -334,7 +336,8 @@ func TestMonolithicDefault(t *testing.T) {
Enabled: true,
SAR: "{\"namespace\": \"testns\", \"resource\": \"pods\", \"verb\": \"get\"}",
},
ServicesQueryDuration: &defaultServicesDuration,
ServicesQueryDuration: &defaultServicesDuration,
FindTracesConcurrentRequests: 1,
},
Management: "Managed",
Timeout: metav1.Duration{Duration: time.Second * 30},
Expand Down Expand Up @@ -398,15 +401,16 @@ func TestMonolithicDefault(t *testing.T) {
Enabled: false,
SAR: "{\"namespace\": \"testns\", \"resource\": \"pods\", \"verb\": \"get\"}",
},
ServicesQueryDuration: &defaultServicesDuration,
ServicesQueryDuration: &defaultServicesDuration,
FindTracesConcurrentRequests: 1,
},
Management: "Managed",
Timeout: metav1.Duration{Duration: time.Second * 30},
},
},
},
{
name: "define custom duration for services list and timeout",
name: "define custom duration for services list, timeout and find traces",
input: &TempoMonolithic{
ObjectMeta: v1.ObjectMeta{
Name: "test",
Expand All @@ -424,7 +428,8 @@ func TestMonolithicDefault(t *testing.T) {
Route: &MonolithicJaegerUIRouteSpec{
Enabled: true,
},
ServicesQueryDuration: &v1.Duration{Duration: time.Duration(100 * 100)},
ServicesQueryDuration: &v1.Duration{Duration: time.Duration(100 * 100)},
FindTracesConcurrentRequests: 40,
},
Timeout: metav1.Duration{Duration: time.Hour},
},
Expand Down Expand Up @@ -461,7 +466,8 @@ func TestMonolithicDefault(t *testing.T) {
Enabled: false,
SAR: "{\"namespace\": \"testns\", \"resource\": \"pods\", \"verb\": \"get\"}",
},
ServicesQueryDuration: &v1.Duration{Duration: time.Duration(100 * 100)},
ServicesQueryDuration: &v1.Duration{Duration: time.Duration(100 * 100)},
FindTracesConcurrentRequests: 40,
},
Management: "Managed",
Timeout: metav1.Duration{Duration: time.Hour},
Expand Down
7 changes: 7 additions & 0 deletions apis/tempo/v1alpha1/tempomonolithic_types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -261,6 +261,13 @@ type MonolithicJaegerUISpec struct {
// +optional
// +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="ServicesQueryDuration",xDescriptors="urn:alm:descriptor:com.tectonic.ui:advanced"
ServicesQueryDuration *metav1.Duration `json:"servicesQueryDuration,omitempty"`

// FindTracesConcurrentRequests defines how many concurrent request a single trace search can submit.
// By default a querier can process 20 concurrent search jobs.
//
// +optional
// +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="FindTracesConcurrentRequests",xDescriptors="urn:alm:descriptor:com.tectonic.ui:advanced"
FindTracesConcurrentRequests int `json:"findTracesConcurrentRequests,omitempty"`
}

// MonolithicJaegerUIIngressSpec defines the settings for the Jaeger UI ingress.
Expand Down
7 changes: 7 additions & 0 deletions apis/tempo/v1alpha1/tempostack_types.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -618,6 +618,13 @@ type JaegerQuerySpec struct {
// +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="ServicesQueryDuration"
ServicesQueryDuration *metav1.Duration `json:"servicesQueryDuration,omitempty"`

// FindTracesConcurrentRequests defines how many concurrent request a single trace search can submit.
// By default a querier can process 20 concurrent search jobs.
//
// +optional
// +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="FindTracesConcurrentRequests",xDescriptors="urn:alm:descriptor:com.tectonic.ui:advanced"
FindTracesConcurrentRequests int `json:"findTracesConcurrentRequests,omitempty"`

// Authentication defines the options for the oauth proxy used to protect jaeger UI
//
// +optional
Expand Down
5 changes: 5 additions & 0 deletions config/crd/bases/tempo.grafana.com_tempomonolithics.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1105,6 +1105,11 @@ spec:
description: Enabled defines if the Jaeger UI component should
be created.
type: boolean
findTracesConcurrentRequests:
description: |-
FindTracesConcurrentRequests defines how many concurrent request a single trace search can submit.
By default a querier can process 20 concurrent search jobs.
type: integer
ingress:
description: Ingress defines the Ingress configuration for the
Jaeger UI.
Expand Down
5 changes: 5 additions & 0 deletions config/crd/bases/tempo.grafana.com_tempostacks.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2387,6 +2387,11 @@ spec:
description: Enabled defines if the Jaeger Query component
should be created.
type: boolean
findTracesConcurrentRequests:
description: |-
FindTracesConcurrentRequests defines how many concurrent request a single trace search can submit.
By default a querier can process 20 concurrent search jobs.
type: integer
ingress:
description: Ingress defines the options for the Jaeger
Query ingress.
Expand Down
7 changes: 4 additions & 3 deletions internal/manifests/config/build.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -176,9 +176,10 @@ func buildTempoQueryConfig(params manifestutils.Params) ([]byte, error) {
GRPCEncryption: params.CtrlConfig.Gates.GRPCEncryption,
HTTPEncryption: params.CtrlConfig.Gates.HTTPEncryption,
},
TenantHeader: manifestutils.TenantHeader,
Gateway: params.Tempo.Spec.Template.Gateway.Enabled,
ServicesQueryDuration: params.Tempo.Spec.Template.QueryFrontend.JaegerQuery.ServicesQueryDuration.Duration.String(),
TenantHeader: manifestutils.TenantHeader,
Gateway: params.Tempo.Spec.Template.Gateway.Enabled,
ServicesQueryDuration: params.Tempo.Spec.Template.QueryFrontend.JaegerQuery.ServicesQueryDuration.Duration.String(),
FindTracesConcurrentRequests: params.Tempo.Spec.Template.QueryFrontend.JaegerQuery.FindTracesConcurrentRequests,
})
}

Expand Down
13 changes: 7 additions & 6 deletions internal/manifests/config/options.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -27,12 +27,13 @@ type options struct {
}

type tempoQueryOptions struct {
Gates featureGates
TLS tlsOptions
HTTPPort int
TenantHeader string
Gateway bool
ServicesQueryDuration string
Gates featureGates
TLS tlsOptions
HTTPPort int
TenantHeader string
Gateway bool
ServicesQueryDuration string
FindTracesConcurrentRequests int
}

type featureGates struct {
Expand Down
1 change: 1 addition & 0 deletions internal/manifests/config/tempo-query.yaml
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,3 +10,4 @@ tls_insecure_skip_verify: false
tls_server_name: {{ .TLS.ServerNames.QueryFrontend }}
{{- end }}
services_query_duration: {{ .ServicesQueryDuration }}
find_traces_concurrent_requests: {{ .FindTracesConcurrentRequests }}
10 changes: 6 additions & 4 deletions internal/manifests/monolithic/configmap.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -96,10 +96,11 @@ type tempoConfig struct {
}

type tempoQueryConfig struct {
Address string `yaml:"address"`
Backend string `yaml:"backend"`
TenantHeaderKey string `yaml:"tenant_header_key"`
ServicesQueryDuration time.Duration `yaml:"services_query_duration"`
Address string `yaml:"address"`
Backend string `yaml:"backend"`
TenantHeaderKey string `yaml:"tenant_header_key"`
ServicesQueryDuration time.Duration `yaml:"services_query_duration"`
FindTracesConcurrentRequests int `yaml:"find_traces_concurrent_requests"`
}

// BuildConfigMap creates the Tempo ConfigMap for a monolithic deployment.
Expand Down Expand Up @@ -294,5 +295,6 @@ func buildTempoQueryConfig(jaegerUISpec *v1alpha1.MonolithicJaegerUISpec) ([]byt
config.Backend = fmt.Sprintf("127.0.0.1:%d", manifestutils.PortHTTPServer)
config.TenantHeaderKey = manifestutils.TenantHeader
config.ServicesQueryDuration = jaegerUISpec.ServicesQueryDuration.Duration
config.FindTracesConcurrentRequests = jaegerUISpec.FindTracesConcurrentRequests
return yaml.Marshal(&config)
}
6 changes: 4 additions & 2 deletions internal/manifests/monolithic/configmap_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,8 +42,9 @@ func TestBuildConfigMap(t *testing.T) {
},
},
JaegerUI: &v1alpha1.MonolithicJaegerUISpec{
Enabled: true,
ServicesQueryDuration: &metav1.Duration{Duration: time.Duration(3 * 24 * time.Hour)},
Enabled: true,
ServicesQueryDuration: &metav1.Duration{Duration: time.Duration(3 * 24 * time.Hour)},
FindTracesConcurrentRequests: 22,
},
},
},
Expand All @@ -63,6 +64,7 @@ address: 127.0.0.1:7777
backend: 127.0.0.1:3200
tenant_header_key: x-scope-orgid
services_query_duration: 72h0m0s
find_traces_concurrent_requests: 22
`
require.YAMLEq(t, tempoQueryCfg, cm.Data["tempo-query.yaml"])
}
Expand Down
4 changes: 4 additions & 0 deletions internal/webhooks/tempostack_webhook.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -42,6 +42,7 @@ type TempoStackWebhook struct {

const defaultRouteGatewayTLSTermination = v1alpha1.TLSRouteTerminationTypeReencrypt
const defaultUITLSTermination = v1alpha1.TLSRouteTerminationTypeEdge
const defaultFindTracesConcurrentRequests = 1

// SetupWebhookWithManager initializes the webhook.
func (w *TempoStackWebhook) SetupWebhookWithManager(mgr ctrl.Manager, ctrlConfig configv1alpha1.ProjectConfig) error {
Expand Down Expand Up @@ -155,6 +156,9 @@ func (d *Defaulter) Default(ctx context.Context, obj runtime.Object) error {
if r.Spec.Template.QueryFrontend.JaegerQuery.ServicesQueryDuration == nil {
r.Spec.Template.QueryFrontend.JaegerQuery.ServicesQueryDuration = &defaultServicesDuration
}
if r.Spec.Template.QueryFrontend.JaegerQuery.FindTracesConcurrentRequests == 0 {
r.Spec.Template.QueryFrontend.JaegerQuery.FindTracesConcurrentRequests = defaultFindTracesConcurrentRequests
}

if d.ctrlConfig.Gates.OpenShift.OauthProxy.DefaultEnabled {
if r.Spec.Template.QueryFrontend.JaegerQuery.Authentication == nil && r.Spec.Template.QueryFrontend.JaegerQuery.Ingress.Type == v1alpha1.IngressTypeRoute {
Expand Down
18 changes: 12 additions & 6 deletions internal/webhooks/tempostack_webhook_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -138,7 +138,8 @@ func TestDefault(t *testing.T) {
Replicas: ptr.To(int32(1)),
},
JaegerQuery: v1alpha1.JaegerQuerySpec{
ServicesQueryDuration: &defaultServicesDuration,
ServicesQueryDuration: &defaultServicesDuration,
FindTracesConcurrentRequests: 1,
},
},
},
Expand Down Expand Up @@ -204,7 +205,8 @@ func TestDefault(t *testing.T) {
Replicas: ptr.To(int32(1)),
},
JaegerQuery: v1alpha1.JaegerQuerySpec{
ServicesQueryDuration: &defaultServicesDuration,
ServicesQueryDuration: &defaultServicesDuration,
FindTracesConcurrentRequests: 1,
},
},
Gateway: v1alpha1.TempoGatewaySpec{
Expand Down Expand Up @@ -287,7 +289,8 @@ func TestDefault(t *testing.T) {
Termination: "edge",
},
},
ServicesQueryDuration: &defaultServicesDuration,
ServicesQueryDuration: &defaultServicesDuration,
FindTracesConcurrentRequests: 1,
},
},
Gateway: v1alpha1.TempoGatewaySpec{
Expand Down Expand Up @@ -375,7 +378,8 @@ func TestDefault(t *testing.T) {
Termination: "edge",
},
},
ServicesQueryDuration: &defaultServicesDuration,
ServicesQueryDuration: &defaultServicesDuration,
FindTracesConcurrentRequests: 1,
Authentication: &v1alpha1.JaegerQueryAuthenticationSpec{
Enabled: true,
SAR: "{\"namespace\": \"\", \"resource\": \"pods\", \"verb\": \"get\"}",
Expand Down Expand Up @@ -470,7 +474,8 @@ func TestDefault(t *testing.T) {
Termination: "edge",
},
},
ServicesQueryDuration: &defaultServicesDuration,
ServicesQueryDuration: &defaultServicesDuration,
FindTracesConcurrentRequests: 1,
Authentication: &v1alpha1.JaegerQueryAuthenticationSpec{
Enabled: false,
},
Expand Down Expand Up @@ -590,7 +595,8 @@ func TestDefault(t *testing.T) {
Replicas: ptr.To(int32(1)),
},
JaegerQuery: v1alpha1.JaegerQuerySpec{
ServicesQueryDuration: &defaultServicesDuration,
FindTracesConcurrentRequests: 1,
ServicesQueryDuration: &defaultServicesDuration,
},
},
},
Expand Down

0 comments on commit a63eb5f

Please sign in to comment.