From c7eb01b6d3e002323c9a034a2f0d8cf98994c2fb Mon Sep 17 00:00:00 2001 From: Pavol Loffay Date: Mon, 7 Oct 2024 09:24:37 +0200 Subject: [PATCH] Expose a single timeout setting in CRDs (#1045) Signed-off-by: Pavol Loffay --- .chloggen/timeout.yaml | 18 +++ Makefile | 2 +- .../v1alpha1/tempomonolithic_defaults.go | 5 + .../v1alpha1/tempomonolithic_defaults_test.go | 13 ++- apis/tempo/v1alpha1/tempomonolithic_types.go | 5 + apis/tempo/v1alpha1/tempostack_types.go | 5 + apis/tempo/v1alpha1/zz_generated.deepcopy.go | 2 + .../tempo-operator.clusterserviceversion.yaml | 6 +- .../tempo.grafana.com_tempomonolithics.yaml | 6 + .../tempo.grafana.com_tempostacks.yaml | 6 + .../tempo-operator.clusterserviceversion.yaml | 6 +- .../tempo.grafana.com_tempomonolithics.yaml | 6 + .../tempo.grafana.com_tempostacks.yaml | 6 + .../tempo.grafana.com_tempomonolithics.yaml | 6 + .../bases/tempo.grafana.com_tempostacks.yaml | 6 + config/manager/manager.yaml | 2 +- .../tempo.grafana.com_tempomonolithics.yaml | 1 + docs/spec/tempo.grafana.com_tempostacks.yaml | 1 + internal/manifests/config/build.go | 1 + internal/manifests/config/build_test.go | 98 +++++++++------- internal/manifests/config/options.go | 7 +- internal/manifests/config/tempo-config.yaml | 4 +- internal/manifests/gateway/gateway.go | 1 + internal/manifests/gateway/gateway_test.go | 6 +- internal/manifests/gateway/openshift.go | 12 +- internal/manifests/manifests_test.go | 2 + internal/manifests/monolithic/build.go | 1 + internal/manifests/monolithic/configmap.go | 10 +- .../manifests/monolithic/configmap_test.go | 18 ++- .../manifests/monolithic/jaegerui_ingress.go | 12 +- .../monolithic/jaegerui_ingress_test.go | 6 + internal/manifests/monolithic/statefulset.go | 1 + .../manifests/monolithic/statefulset_test.go | 3 + internal/manifests/oauthproxy/oauth_proxy.go | 24 ++-- .../manifests/oauthproxy/oauth_proxy_test.go | 6 + .../manifests/queryfrontend/query_frontend.go | 19 +++- .../queryfrontend/query_frontend_test.go | 6 + internal/webhooks/tempostack_webhook.go | 5 + internal/webhooks/tempostack_webhook_test.go | 107 ++++++++++++++++++ .../install-tempo-assert.yaml | 1 + .../scale-tempo-assert.yaml | 1 + .../install-tempo-assert.yaml | 22 ++++ .../monolithic-route/install-tempo.yaml | 1 + .../install-tempo-assert.yaml | 1 + .../e2e-openshift/multitenancy/01-assert.yaml | 1 + .../route/install-tempo-assert.yaml | 1 + tests/e2e-openshift/route/install-tempo.yaml | 1 + .../01-assert.yaml | 1 + .../install-tempo-assert.yaml | 2 + .../install-tempostack-assert.yaml | 8 +- .../install-tempostack.yaml | 4 + 51 files changed, 417 insertions(+), 78 deletions(-) create mode 100755 .chloggen/timeout.yaml diff --git a/.chloggen/timeout.yaml b/.chloggen/timeout.yaml new file mode 100755 index 000000000..e8de1bf44 --- /dev/null +++ b/.chloggen/timeout.yaml @@ -0,0 +1,18 @@ +# One of 'breaking', 'deprecation', 'new_component', 'enhancement', 'bug_fix' +change_type: breaking + +# The name of the component, or a single word describing the area of concern, (e.g. tempostack, tempomonolithic, github action) +component: tempostack, tempomonolithic + +# A brief description of the change. Surround your text with quotes ("") if it needs to start with a backtick (`). +note: Add unified timeout configuration. It changes the default to 30s. + +# One or more tracking issues related to the change +issues: [1045] + +# (Optional) One or more lines of additional information to render under the primary note. +# These lines will be padded with 2 spaces and then inserted directly into the document. +# Use pipe (|) for multiline entries. +subtext: | + Adding `spec.timeout` CRD option to configure timeout on all components and default it to 30s. + Before Tempo server was defaulting to 3m, gateway to 2m, OpenShift route to 30s (for query), oauth-proxy to 30s (for query). diff --git a/Makefile b/Makefile index fe152e9c0..f0a5bd809 100644 --- a/Makefile +++ b/Makefile @@ -6,7 +6,7 @@ TEMPO_QUERY_VERSION ?= main-2999520 JAEGER_QUERY_VERSION ?= d6631f5f2370cfc3a49efce312491031fb387600 TEMPO_GATEWAY_VERSION ?= main-2024-08-05-11d0d94 TEMPO_GATEWAY_OPA_VERSION ?= main-2024-04-29-914c13f -OAUTH_PROXY_VERSION=4.12 +OAUTH_PROXY_VERSION=4.14 MIN_KUBERNETES_VERSION ?= 1.25.0 MIN_OPENSHIFT_VERSION ?= 4.12 diff --git a/apis/tempo/v1alpha1/tempomonolithic_defaults.go b/apis/tempo/v1alpha1/tempomonolithic_defaults.go index 7dfbd738e..39fc395f9 100644 --- a/apis/tempo/v1alpha1/tempomonolithic_defaults.go +++ b/apis/tempo/v1alpha1/tempomonolithic_defaults.go @@ -16,6 +16,7 @@ var ( twoGBQuantity = resource.MustParse("2Gi") tenGBQuantity = resource.MustParse("10Gi") defaultServicesDuration = metav1.Duration{Duration: time.Hour * 24 * 3} + defaultTimeout = metav1.Duration{Duration: time.Second * 30} ) // Default sets all default values in a central place, instead of setting it at every place where the value is accessed. @@ -88,4 +89,8 @@ func (r *TempoMonolithic) Default(ctrlConfig configv1alpha1.ProjectConfig) { r.Spec.JaegerUI.ServicesQueryDuration = &defaultServicesDuration } } + + if r.Spec.Timeout.Duration == 0 { + r.Spec.Timeout = defaultTimeout + } } diff --git a/apis/tempo/v1alpha1/tempomonolithic_defaults_test.go b/apis/tempo/v1alpha1/tempomonolithic_defaults_test.go index c41828ef6..be28ff060 100644 --- a/apis/tempo/v1alpha1/tempomonolithic_defaults_test.go +++ b/apis/tempo/v1alpha1/tempomonolithic_defaults_test.go @@ -6,6 +6,7 @@ import ( "github.com/stretchr/testify/assert" "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" configv1alpha1 "github.com/grafana/tempo-operator/apis/config/v1alpha1" @@ -46,6 +47,7 @@ func TestMonolithicDefault(t *testing.T) { }, }, Management: "Managed", + Timeout: metav1.Duration{Duration: time.Second * 30}, }, }, }, @@ -79,6 +81,7 @@ func TestMonolithicDefault(t *testing.T) { }, }, Management: "Managed", + Timeout: metav1.Duration{Duration: time.Second * 30}, }, }, }, @@ -104,6 +107,7 @@ func TestMonolithicDefault(t *testing.T) { }, }, Management: "Unmanaged", + Timeout: metav1.Duration{Duration: time.Second * 30}, }, }, expected: &TempoMonolithic{ @@ -125,6 +129,7 @@ func TestMonolithicDefault(t *testing.T) { }, }, Management: "Unmanaged", + Timeout: metav1.Duration{Duration: time.Second * 30}, }, }, }, @@ -194,6 +199,7 @@ func TestMonolithicDefault(t *testing.T) { ServicesQueryDuration: &defaultServicesDuration, }, Management: "Managed", + Timeout: metav1.Duration{Duration: time.Second * 30}, }, }, }, @@ -266,6 +272,7 @@ func TestMonolithicDefault(t *testing.T) { ServicesQueryDuration: &defaultServicesDuration, }, Management: "Managed", + Timeout: metav1.Duration{Duration: time.Second * 30}, }, }, }, @@ -330,6 +337,7 @@ func TestMonolithicDefault(t *testing.T) { ServicesQueryDuration: &defaultServicesDuration, }, Management: "Managed", + Timeout: metav1.Duration{Duration: time.Second * 30}, }, }, }, @@ -393,11 +401,12 @@ func TestMonolithicDefault(t *testing.T) { ServicesQueryDuration: &defaultServicesDuration, }, Management: "Managed", + Timeout: metav1.Duration{Duration: time.Second * 30}, }, }, }, { - name: "define custom duration for services list", + name: "define custom duration for services list and timeout", input: &TempoMonolithic{ ObjectMeta: v1.ObjectMeta{ Name: "test", @@ -417,6 +426,7 @@ func TestMonolithicDefault(t *testing.T) { }, ServicesQueryDuration: &v1.Duration{Duration: time.Duration(100 * 100)}, }, + Timeout: metav1.Duration{Duration: time.Hour}, }, }, expected: &TempoMonolithic{ @@ -454,6 +464,7 @@ func TestMonolithicDefault(t *testing.T) { ServicesQueryDuration: &v1.Duration{Duration: time.Duration(100 * 100)}, }, Management: "Managed", + Timeout: metav1.Duration{Duration: time.Hour}, }, }, }, diff --git a/apis/tempo/v1alpha1/tempomonolithic_types.go b/apis/tempo/v1alpha1/tempomonolithic_types.go index 1d56c4400..fef946876 100644 --- a/apis/tempo/v1alpha1/tempomonolithic_types.go +++ b/apis/tempo/v1alpha1/tempomonolithic_types.go @@ -44,6 +44,11 @@ type TempoMonolithicSpec struct { // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Resources",order=5,xDescriptors="urn:alm:descriptor:com.tectonic.ui:resourceRequirements" Resources *corev1.ResourceRequirements `json:"resources,omitempty"` + // Timeout configures the same timeout on all components starting at ingress down to the ingestor/querier. + // Timeout configuration on a specific component has a higher precedence. + // Default is 30 seconds. + Timeout metav1.Duration `json:"timeout,omitempty"` + // ServiceAccount defines the Service Account to use for all Tempo components. // // +kubebuilder:validation:Optional diff --git a/apis/tempo/v1alpha1/tempostack_types.go b/apis/tempo/v1alpha1/tempostack_types.go index 447b93fee..cc0186ab0 100644 --- a/apis/tempo/v1alpha1/tempostack_types.go +++ b/apis/tempo/v1alpha1/tempostack_types.go @@ -40,6 +40,11 @@ type TempoStackSpec struct { // +operator-sdk:csv:customresourcedefinitions:type=spec,displayName="Ingestion and Querying Ratelimiting" LimitSpec LimitSpec `json:"limits,omitempty"` + // Timeout configures the same timeout on all components starting at ingress down to the ingestor/querier. + // Timeout configuration on a specific component has a higher precedence. + // Defaults to 30 seconds. + Timeout metav1.Duration `json:"timeout,omitempty"` + // StorageClassName for PVCs used by ingester. Defaults to nil (default storage class in the cluster). // // +optional diff --git a/apis/tempo/v1alpha1/zz_generated.deepcopy.go b/apis/tempo/v1alpha1/zz_generated.deepcopy.go index d9bbe161f..f5a9d2534 100644 --- a/apis/tempo/v1alpha1/zz_generated.deepcopy.go +++ b/apis/tempo/v1alpha1/zz_generated.deepcopy.go @@ -1367,6 +1367,7 @@ func (in *TempoMonolithicSpec) DeepCopyInto(out *TempoMonolithicSpec) { *out = new(v1.ResourceRequirements) (*in).DeepCopyInto(*out) } + out.Timeout = in.Timeout if in.ExtraConfig != nil { in, out := &in.ExtraConfig, &out.ExtraConfig *out = new(ExtraConfigSpec) @@ -1508,6 +1509,7 @@ func (in *TempoStackList) DeepCopyObject() runtime.Object { func (in *TempoStackSpec) DeepCopyInto(out *TempoStackSpec) { *out = *in in.LimitSpec.DeepCopyInto(&out.LimitSpec) + out.Timeout = in.Timeout if in.StorageClassName != nil { in, out := &in.StorageClassName, &out.StorageClassName *out = new(string) diff --git a/bundle/community/manifests/tempo-operator.clusterserviceversion.yaml b/bundle/community/manifests/tempo-operator.clusterserviceversion.yaml index b623f6b16..85bb37bf8 100644 --- a/bundle/community/manifests/tempo-operator.clusterserviceversion.yaml +++ b/bundle/community/manifests/tempo-operator.clusterserviceversion.yaml @@ -74,7 +74,7 @@ metadata: capabilities: Deep Insights categories: Logging & Tracing,Monitoring containerImage: ghcr.io/grafana/tempo-operator/tempo-operator:v0.13.0 - createdAt: "2024-10-04T12:51:28Z" + createdAt: "2024-10-07T07:11:28Z" description: Create and manage deployments of Tempo, a high-scale distributed tracing backend. operatorframework.io/cluster-monitoring: "true" @@ -1433,7 +1433,7 @@ spec: - name: RELATED_IMAGE_TEMPO_GATEWAY_OPA value: quay.io/observatorium/opa-openshift:main-2024-04-29-914c13f - name: RELATED_IMAGE_OAUTH_PROXY - value: quay.io/openshift/origin-oauth-proxy:4.12 + value: quay.io/openshift/origin-oauth-proxy:4.14 image: ghcr.io/grafana/tempo-operator/tempo-operator:v0.13.0 livenessProbe: httpGet: @@ -1582,7 +1582,7 @@ spec: name: tempo-gateway - image: quay.io/observatorium/opa-openshift:main-2024-04-29-914c13f name: tempo-gateway-opa - - image: quay.io/openshift/origin-oauth-proxy:4.12 + - image: quay.io/openshift/origin-oauth-proxy:4.14 name: oauth-proxy version: 0.13.0 webhookdefinitions: diff --git a/bundle/community/manifests/tempo.grafana.com_tempomonolithics.yaml b/bundle/community/manifests/tempo.grafana.com_tempomonolithics.yaml index ae585d652..115daff46 100644 --- a/bundle/community/manifests/tempo.grafana.com_tempomonolithics.yaml +++ b/bundle/community/manifests/tempo.grafana.com_tempomonolithics.yaml @@ -1683,6 +1683,12 @@ spec: required: - traces type: object + timeout: + description: |- + Timeout configures the same timeout on all components starting at ingress down to the ingestor/querier. + Timeout configuration on a specific component has a higher precedence. + Default is 30 seconds. + type: string tolerations: description: Tolerations defines the tolerations of a node to schedule the pod onto it. diff --git a/bundle/community/manifests/tempo.grafana.com_tempostacks.yaml b/bundle/community/manifests/tempo.grafana.com_tempostacks.yaml index 27f123dca..eaede14e7 100644 --- a/bundle/community/manifests/tempo.grafana.com_tempostacks.yaml +++ b/bundle/community/manifests/tempo.grafana.com_tempostacks.yaml @@ -2715,6 +2715,12 @@ spec: required: - mode type: object + timeout: + description: |- + Timeout configures the same timeout on all components starting at ingress down to the ingestor/querier. + Timeout configuration on a specific component has a higher precedence. + Defaults to 30 seconds. + type: string required: - storage type: object diff --git a/bundle/openshift/manifests/tempo-operator.clusterserviceversion.yaml b/bundle/openshift/manifests/tempo-operator.clusterserviceversion.yaml index 1d9f1007e..a4868857e 100644 --- a/bundle/openshift/manifests/tempo-operator.clusterserviceversion.yaml +++ b/bundle/openshift/manifests/tempo-operator.clusterserviceversion.yaml @@ -74,7 +74,7 @@ metadata: capabilities: Deep Insights categories: Logging & Tracing,Monitoring containerImage: ghcr.io/grafana/tempo-operator/tempo-operator:v0.13.0 - createdAt: "2024-10-04T12:51:27Z" + createdAt: "2024-10-07T07:11:27Z" description: Create and manage deployments of Tempo, a high-scale distributed tracing backend. operatorframework.io/cluster-monitoring: "true" @@ -1443,7 +1443,7 @@ spec: - name: RELATED_IMAGE_TEMPO_GATEWAY_OPA value: quay.io/observatorium/opa-openshift:main-2024-04-29-914c13f - name: RELATED_IMAGE_OAUTH_PROXY - value: quay.io/openshift/origin-oauth-proxy:4.12 + value: quay.io/openshift/origin-oauth-proxy:4.14 image: ghcr.io/grafana/tempo-operator/tempo-operator:v0.13.0 livenessProbe: httpGet: @@ -1603,7 +1603,7 @@ spec: name: tempo-gateway - image: quay.io/observatorium/opa-openshift:main-2024-04-29-914c13f name: tempo-gateway-opa - - image: quay.io/openshift/origin-oauth-proxy:4.12 + - image: quay.io/openshift/origin-oauth-proxy:4.14 name: oauth-proxy version: 0.13.0 webhookdefinitions: diff --git a/bundle/openshift/manifests/tempo.grafana.com_tempomonolithics.yaml b/bundle/openshift/manifests/tempo.grafana.com_tempomonolithics.yaml index ae585d652..115daff46 100644 --- a/bundle/openshift/manifests/tempo.grafana.com_tempomonolithics.yaml +++ b/bundle/openshift/manifests/tempo.grafana.com_tempomonolithics.yaml @@ -1683,6 +1683,12 @@ spec: required: - traces type: object + timeout: + description: |- + Timeout configures the same timeout on all components starting at ingress down to the ingestor/querier. + Timeout configuration on a specific component has a higher precedence. + Default is 30 seconds. + type: string tolerations: description: Tolerations defines the tolerations of a node to schedule the pod onto it. diff --git a/bundle/openshift/manifests/tempo.grafana.com_tempostacks.yaml b/bundle/openshift/manifests/tempo.grafana.com_tempostacks.yaml index 27f123dca..eaede14e7 100644 --- a/bundle/openshift/manifests/tempo.grafana.com_tempostacks.yaml +++ b/bundle/openshift/manifests/tempo.grafana.com_tempostacks.yaml @@ -2715,6 +2715,12 @@ spec: required: - mode type: object + timeout: + description: |- + Timeout configures the same timeout on all components starting at ingress down to the ingestor/querier. + Timeout configuration on a specific component has a higher precedence. + Defaults to 30 seconds. + type: string required: - storage type: object diff --git a/config/crd/bases/tempo.grafana.com_tempomonolithics.yaml b/config/crd/bases/tempo.grafana.com_tempomonolithics.yaml index 4d28c00ea..92a7a6753 100644 --- a/config/crd/bases/tempo.grafana.com_tempomonolithics.yaml +++ b/config/crd/bases/tempo.grafana.com_tempomonolithics.yaml @@ -1679,6 +1679,12 @@ spec: required: - traces type: object + timeout: + description: |- + Timeout configures the same timeout on all components starting at ingress down to the ingestor/querier. + Timeout configuration on a specific component has a higher precedence. + Default is 30 seconds. + type: string tolerations: description: Tolerations defines the tolerations of a node to schedule the pod onto it. diff --git a/config/crd/bases/tempo.grafana.com_tempostacks.yaml b/config/crd/bases/tempo.grafana.com_tempostacks.yaml index 2cdfc2c3b..f1374fbd5 100644 --- a/config/crd/bases/tempo.grafana.com_tempostacks.yaml +++ b/config/crd/bases/tempo.grafana.com_tempostacks.yaml @@ -2711,6 +2711,12 @@ spec: required: - mode type: object + timeout: + description: |- + Timeout configures the same timeout on all components starting at ingress down to the ingestor/querier. + Timeout configuration on a specific component has a higher precedence. + Defaults to 30 seconds. + type: string required: - storage type: object diff --git a/config/manager/manager.yaml b/config/manager/manager.yaml index ac0263dd1..accf9ad74 100644 --- a/config/manager/manager.yaml +++ b/config/manager/manager.yaml @@ -50,7 +50,7 @@ spec: - name: RELATED_IMAGE_TEMPO_GATEWAY_OPA value: quay.io/observatorium/opa-openshift:main-2024-04-29-914c13f - name: RELATED_IMAGE_OAUTH_PROXY - value: quay.io/openshift/origin-oauth-proxy:4.12 + value: quay.io/openshift/origin-oauth-proxy:4.14 securityContext: allowPrivilegeEscalation: false capabilities: diff --git a/docs/spec/tempo.grafana.com_tempomonolithics.yaml b/docs/spec/tempo.grafana.com_tempomonolithics.yaml index 3ab1b69a4..487a6cfd7 100644 --- a/docs/spec/tempo.grafana.com_tempomonolithics.yaml +++ b/docs/spec/tempo.grafana.com_tempomonolithics.yaml @@ -126,6 +126,7 @@ spec: # TempoMonolithicSpec defines the desir certName: "" # Cert is the name of a Secret containing a certificate (tls.crt) and private key (tls.key). It needs to be in the same namespace as the Tempo custom resource. minVersion: "" # MinVersion defines the minimum acceptable TLS version. size: 0Gi # Size defines the size of the volume where traces are stored. For in-memory storage, this defines the size of the tmpfs volume. For persistent volume storage, this defines the size of the persistent volume. For object storage, this defines the size of the persistent volume containing the Write-Ahead Log (WAL) of Tempo. Default: 2Gi for memory, 10Gi for all other backends. + timeout: "" # Timeout configures the same timeout on all components starting at ingress down to the ingestor/querier. Timeout configuration on a specific component has a higher precedence. Default is 30 seconds. affinity: # Affinity defines the Affinity rules for scheduling pods. nodeAffinity: {} # Describes node affinity scheduling rules for the pod. podAffinity: {} # Describes pod affinity scheduling rules (e.g. co-locate this pod in the same node, zone, etc. as some other pod(s)). diff --git a/docs/spec/tempo.grafana.com_tempostacks.yaml b/docs/spec/tempo.grafana.com_tempostacks.yaml index 4ac207289..2b65cc6a7 100644 --- a/docs/spec/tempo.grafana.com_tempostacks.yaml +++ b/docs/spec/tempo.grafana.com_tempostacks.yaml @@ -388,6 +388,7 @@ spec: # TempoStackSpec defines the desired st resources: - "" mode: "static" # Mode defines the multitenancy mode. + timeout: "" # Timeout configures the same timeout on all components starting at ingress down to the ingestor/querier. Timeout configuration on a specific component has a higher precedence. Defaults to 30 seconds. resources: # Resources defines resources configuration. total: # The total amount of resources for Tempo instance. The operator autonomously splits resources between deployed Tempo components. Only limits are supported, the operator calculates requests automatically. See http://github.com/grafana/tempo/issues/1540. claims: # Claims lists the names of resources, defined in spec.resourceClaims, that are used by this container. This is an alpha field and requires enabling the DynamicResourceAllocation feature gate. This field is immutable. It can only be set for containers. diff --git a/internal/manifests/config/build.go b/internal/manifests/config/build.go index 489a2c9fe..a20a60d78 100644 --- a/internal/manifests/config/build.go +++ b/internal/manifests/config/build.go @@ -89,6 +89,7 @@ func buildConfiguration(params manifestutils.Params) ([]byte, error) { TLS: tlsopts, ReceiverTLS: buildReceiverTLSConfig(tempo), S3StorageTLS: buildS3StorageTLSConfig(params), + Timeout: params.Tempo.Spec.Timeout.Duration, } if isTenantOverridesConfigRequired(tempo.Spec.LimitSpec) { diff --git a/internal/manifests/config/build_test.go b/internal/manifests/config/build_test.go index 62c50571e..23623d48d 100644 --- a/internal/manifests/config/build_test.go +++ b/internal/manifests/config/build_test.go @@ -74,8 +74,8 @@ server: grpc_server_max_recv_msg_size: 4194304 grpc_server_max_send_msg_size: 4194304 http_listen_port: 3200 - http_server_read_timeout: 3m - http_server_write_timeout: 3m + http_server_read_timeout: 30s + http_server_write_timeout: 30s log_format: logfmt storage: trace: @@ -103,6 +103,7 @@ query_frontend: Name: "test", }, Spec: v1alpha1.TempoStackSpec{ + Timeout: metav1.Duration{Duration: time.Second * 30}, Storage: v1alpha1.ObjectStorageSpec{ Secret: v1alpha1.ObjectStorageSecretSpec{ Type: v1alpha1.ObjectStorageSecretS3, @@ -197,8 +198,8 @@ server: grpc_server_max_recv_msg_size: 4194304 grpc_server_max_send_msg_size: 4194304 http_listen_port: 3200 - http_server_read_timeout: 3m - http_server_write_timeout: 3m + http_server_read_timeout: 3m0s + http_server_write_timeout: 3m0s log_format: logfmt storage: trace: @@ -286,8 +287,8 @@ server: grpc_server_max_recv_msg_size: 4194304 grpc_server_max_send_msg_size: 4194304 http_listen_port: 3200 - http_server_read_timeout: 3m - http_server_write_timeout: 3m + http_server_read_timeout: 3m0s + http_server_write_timeout: 3m0s log_format: logfmt storage: trace: @@ -374,8 +375,8 @@ server: grpc_server_max_recv_msg_size: 4194304 grpc_server_max_send_msg_size: 4194304 http_listen_port: 3200 - http_server_read_timeout: 3m - http_server_write_timeout: 3m + http_server_read_timeout: 3m0s + http_server_write_timeout: 3m0s log_format: logfmt storage: trace: @@ -463,8 +464,8 @@ server: grpc_server_max_recv_msg_size: 4194304 grpc_server_max_send_msg_size: 4194304 http_listen_port: 3200 - http_server_read_timeout: 3m - http_server_write_timeout: 3m + http_server_read_timeout: 3m0s + http_server_write_timeout: 3m0s log_format: logfmt storage: trace: @@ -552,8 +553,8 @@ server: grpc_server_max_recv_msg_size: 4194304 grpc_server_max_send_msg_size: 4194304 http_listen_port: 3200 - http_server_read_timeout: 3m - http_server_write_timeout: 3m + http_server_read_timeout: 3m0s + http_server_write_timeout: 3m0s log_format: logfmt storage: trace: @@ -641,8 +642,8 @@ server: grpc_server_max_recv_msg_size: 4194304 grpc_server_max_send_msg_size: 4194304 http_listen_port: 3200 - http_server_read_timeout: 3m - http_server_write_timeout: 3m + http_server_read_timeout: 3m0s + http_server_write_timeout: 3m0s log_format: logfmt storage: trace: @@ -730,8 +731,8 @@ server: grpc_server_max_recv_msg_size: 4194304 grpc_server_max_send_msg_size: 4194304 http_listen_port: 3200 - http_server_read_timeout: 3m - http_server_write_timeout: 3m + http_server_read_timeout: 3m0s + http_server_write_timeout: 3m0s log_format: logfmt storage: trace: @@ -831,8 +832,8 @@ server: grpc_server_max_recv_msg_size: 4194304 grpc_server_max_send_msg_size: 4194304 http_listen_port: 3200 - http_server_read_timeout: 3m - http_server_write_timeout: 3m + http_server_read_timeout: 3m0s + http_server_write_timeout: 3m0s log_format: logfmt storage: trace: @@ -930,8 +931,8 @@ server: grpc_server_max_recv_msg_size: 4194304 grpc_server_max_send_msg_size: 4194304 http_listen_port: 3200 - http_server_read_timeout: 3m - http_server_write_timeout: 3m + http_server_read_timeout: 3m0s + http_server_write_timeout: 3m0s log_format: logfmt storage: trace: @@ -964,6 +965,7 @@ query_frontend: Name: "test", }, Spec: v1alpha1.TempoStackSpec{ + Timeout: metav1.Duration{Duration: time.Minute * 3}, Storage: v1alpha1.ObjectStorageSpec{ Secret: v1alpha1.ObjectStorageSecretSpec{ Type: v1alpha1.ObjectStorageSecretS3, @@ -1091,8 +1093,8 @@ server: grpc_server_max_recv_msg_size: 4194304 grpc_server_max_send_msg_size: 4194304 http_listen_port: 3200 - http_server_read_timeout: 3m - http_server_write_timeout: 3m + http_server_read_timeout: 3m0s + http_server_write_timeout: 3m0s log_format: logfmt storage: trace: @@ -1124,6 +1126,7 @@ query_frontend: Name: "test", }, Spec: v1alpha1.TempoStackSpec{ + Timeout: metav1.Duration{Duration: time.Minute * 3}, Storage: v1alpha1.ObjectStorageSpec{ Secret: v1alpha1.ObjectStorageSecretSpec{ Type: v1alpha1.ObjectStorageSecretGCS, @@ -1205,8 +1208,8 @@ server: grpc_server_max_recv_msg_size: 4194304 grpc_server_max_send_msg_size: 4194304 http_listen_port: 3200 - http_server_read_timeout: 3m - http_server_write_timeout: 3m + http_server_read_timeout: 3m0s + http_server_write_timeout: 3m0s log_format: logfmt storage: trace: @@ -1233,6 +1236,7 @@ query_frontend: Name: "test", }, Spec: v1alpha1.TempoStackSpec{ + Timeout: metav1.Duration{Duration: time.Minute * 3}, Storage: v1alpha1.ObjectStorageSpec{ Secret: v1alpha1.ObjectStorageSecretSpec{ Type: v1alpha1.ObjectStorageSecretAzure, @@ -1309,8 +1313,8 @@ server: grpc_server_max_recv_msg_size: 4194304 grpc_server_max_send_msg_size: 4194304 http_listen_port: 3200 - http_server_read_timeout: 3m - http_server_write_timeout: 3m + http_server_read_timeout: 3m0s + http_server_write_timeout: 3m0s log_format: logfmt storage: trace: @@ -1338,6 +1342,7 @@ query_frontend: Name: "test", }, Spec: v1alpha1.TempoStackSpec{ + Timeout: metav1.Duration{Duration: time.Minute * 3}, Storage: v1alpha1.ObjectStorageSpec{ Secret: v1alpha1.ObjectStorageSecretSpec{ Type: v1alpha1.ObjectStorageSecretS3, @@ -1455,8 +1460,8 @@ server: grpc_server_max_recv_msg_size: 4194304 grpc_server_max_send_msg_size: 4194304 http_listen_port: 3200 - http_server_read_timeout: 3m - http_server_write_timeout: 3m + http_server_read_timeout: 3m0s + http_server_write_timeout: 3m0s log_format: logfmt tls_cipher_suites: TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 tls_min_version: VersionTLS12 @@ -1574,8 +1579,8 @@ server: grpc_server_max_recv_msg_size: 4194304 grpc_server_max_send_msg_size: 4194304 http_listen_port: 3200 - http_server_read_timeout: 3m - http_server_write_timeout: 3m + http_server_read_timeout: 3m0s + http_server_write_timeout: 3m0s log_format: logfmt tls_min_version: VersionTLS13 grpc_tls_config: @@ -1630,6 +1635,7 @@ ingester_client: Namespace: "nstest", }, Spec: v1alpha1.TempoStackSpec{ + Timeout: metav1.Duration{Duration: time.Minute * 3}, Storage: v1alpha1.ObjectStorageSpec{ Secret: v1alpha1.ObjectStorageSecretSpec{ Type: v1alpha1.ObjectStorageSecretS3, @@ -1737,8 +1743,8 @@ server: grpc_server_max_recv_msg_size: 4194304 grpc_server_max_send_msg_size: 4194304 http_listen_port: 3200 - http_server_read_timeout: 3m - http_server_write_timeout: 3m + http_server_read_timeout: 3m0s + http_server_write_timeout: 3m0s log_format: logfmt tls_cipher_suites: TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384,TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256,TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 tls_min_version: VersionTLS12 @@ -1789,6 +1795,7 @@ ingester_client: Namespace: "nstest", }, Spec: v1alpha1.TempoStackSpec{ + Timeout: metav1.Duration{Duration: 3 * time.Minute}, Template: v1alpha1.TempoTemplateSpec{ Gateway: v1alpha1.TempoGatewaySpec{ Enabled: true, @@ -1902,8 +1909,8 @@ server: grpc_server_max_recv_msg_size: 4194304 grpc_server_max_send_msg_size: 4194304 http_listen_port: 3200 - http_server_read_timeout: 3m - http_server_write_timeout: 3m + http_server_read_timeout: 3m0s + http_server_write_timeout: 3m0s log_format: logfmt storage: trace: @@ -2010,8 +2017,8 @@ server: grpc_server_max_recv_msg_size: 4194304 grpc_server_max_send_msg_size: 4194304 http_listen_port: 3200 - http_server_read_timeout: 3m - http_server_write_timeout: 3m + http_server_read_timeout: 3m0s + http_server_write_timeout: 3m0s log_format: logfmt storage: trace: @@ -2124,8 +2131,8 @@ server: grpc_server_max_recv_msg_size: 4194304 grpc_server_max_send_msg_size: 4194304 http_listen_port: 3200 - http_server_read_timeout: 3m - http_server_write_timeout: 3m + http_server_read_timeout: 3m0s + http_server_write_timeout: 3m0s log_format: logfmt storage: trace: @@ -2158,6 +2165,7 @@ query_frontend: Name: "test", }, Spec: v1alpha1.TempoStackSpec{ + Timeout: metav1.Duration{Duration: time.Minute * 3}, Storage: v1alpha1.ObjectStorageSpec{ Secret: v1alpha1.ObjectStorageSecretSpec{ Type: v1alpha1.ObjectStorageSecretS3, @@ -2257,8 +2265,8 @@ server: grpc_server_max_recv_msg_size: 4194304 grpc_server_max_send_msg_size: 4194304 http_listen_port: 3200 - http_server_read_timeout: 3m - http_server_write_timeout: 3m + http_server_read_timeout: 3m0s + http_server_write_timeout: 3m0s log_format: logfmt storage: trace: @@ -2340,8 +2348,8 @@ server: grpc_server_max_recv_msg_size: 4194304 grpc_server_max_send_msg_size: 4194304 http_listen_port: 3200 - http_server_read_timeout: 3m - http_server_write_timeout: 3m + http_server_read_timeout: 3m0s + http_server_write_timeout: 3m0s log_format: logfmt storage: trace: @@ -2374,6 +2382,7 @@ query_frontend: Name: "test", }, Spec: v1alpha1.TempoStackSpec{ + Timeout: metav1.Duration{Duration: 3 * time.Minute}, Storage: v1alpha1.ObjectStorageSpec{ Secret: v1alpha1.ObjectStorageSecretSpec{ Type: v1alpha1.ObjectStorageSecretS3, @@ -2464,8 +2473,8 @@ server: grpc_server_max_recv_msg_size: 4194304 grpc_server_max_send_msg_size: 4194304 http_listen_port: 3200 - http_server_read_timeout: 3m - http_server_write_timeout: 3m + http_server_read_timeout: 3m0s + http_server_write_timeout: 3m0s log_format: logfmt storage: trace: @@ -2492,6 +2501,7 @@ query_frontend: Name: "test", }, Spec: v1alpha1.TempoStackSpec{ + Timeout: metav1.Duration{Duration: time.Minute * 3}, Storage: v1alpha1.ObjectStorageSpec{ Secret: v1alpha1.ObjectStorageSecretSpec{ Type: v1alpha1.ObjectStorageSecretS3, diff --git a/internal/manifests/config/options.go b/internal/manifests/config/options.go index c3a220b3b..02ce67481 100644 --- a/internal/manifests/config/options.go +++ b/internal/manifests/config/options.go @@ -1,6 +1,10 @@ package config -import "github.com/grafana/tempo-operator/internal/manifests/manifestutils" +import ( + "time" + + "github.com/grafana/tempo-operator/internal/manifests/manifestutils" +) // options holds the configuration template options. type options struct { @@ -19,6 +23,7 @@ type options struct { Gates featureGates ReceiverTLS receiverTLSOptions S3StorageTLS storageTLSOptions + Timeout time.Duration } type tempoQueryOptions struct { diff --git a/internal/manifests/config/tempo-config.yaml b/internal/manifests/config/tempo-config.yaml index f43141207..cee5abb49 100644 --- a/internal/manifests/config/tempo-config.yaml +++ b/internal/manifests/config/tempo-config.yaml @@ -182,8 +182,8 @@ server: grpc_server_max_recv_msg_size: 4194304 grpc_server_max_send_msg_size: 4194304 http_listen_port: 3200 - http_server_read_timeout: 3m - http_server_write_timeout: 3m + http_server_read_timeout: {{ .Timeout }} + http_server_write_timeout: {{ .Timeout }} log_format: logfmt {{- if or .Gates.GRPCEncryption .Gates.HTTPEncryption }} {{- if .TLS.Profile.Ciphers }} diff --git a/internal/manifests/gateway/gateway.go b/internal/manifests/gateway/gateway.go index 5b778475d..aff673e8a 100644 --- a/internal/manifests/gateway/gateway.go +++ b/internal/manifests/gateway/gateway.go @@ -248,6 +248,7 @@ func deployment(params manifestutils.Params, rbacCfgHash string, tenantsCfgHash fmt.Sprintf("--web.internal.listen=0.0.0.0:%d", manifestutils.GatewayPortInternalHTTPServer), // serves health checks fmt.Sprintf("--traces.write.otlpgrpc.endpoint=%s:%d", naming.ServiceFqdn(tempo.Namespace, tempo.Name, manifestutils.DistributorComponentName), manifestutils.PortOtlpGrpcServer), // Tempo Distributor gRPC upstream fmt.Sprintf("--traces.write.otlphttp.endpoint=%s://%s:%d", httpScheme(params.CtrlConfig.Gates.HTTPEncryption), naming.ServiceFqdn(tempo.Namespace, tempo.Name, manifestutils.DistributorComponentName), manifestutils.PortOtlpHttp), // Tempo Distributor HTTP upstream + fmt.Sprintf("--traces.write-timeout=%s", params.Tempo.Spec.Timeout.Duration.String()), fmt.Sprintf("--traces.tempo.endpoint=%s://%s:%d", httpScheme(params.CtrlConfig.Gates.HTTPEncryption), naming.ServiceFqdn(tempo.Namespace, tempo.Name, manifestutils.QueryFrontendComponentName), manifestutils.PortHTTPServer), // Tempo API upstream fmt.Sprintf("--grpc.listen=0.0.0.0:%d", manifestutils.GatewayPortGRPCServer), // proxies Tempo Distributor gRPC diff --git a/internal/manifests/gateway/gateway_test.go b/internal/manifests/gateway/gateway_test.go index 5439d07a7..3feca416b 100644 --- a/internal/manifests/gateway/gateway_test.go +++ b/internal/manifests/gateway/gateway_test.go @@ -2,7 +2,6 @@ package gateway import ( "fmt" - "net" "reflect" "testing" @@ -249,7 +248,7 @@ func TestBuildGateway_openshift(t *testing.T) { require.True(t, ok) require.Equal(t, "Service", route.Spec.To.Kind) require.Equal(t, "tempo-simplest-gateway", route.Spec.To.Name) - require.Equal(t, map[string]string{"timeout": "30s"}, route.ObjectMeta.Annotations) + require.Equal(t, map[string]string{"timeout": "30s", "haproxy.router.openshift.io/timeout": "0s"}, route.ObjectMeta.Annotations) obj = getObjectByTypeAndName(objects, "tempo-simplest-gateway-cabundle", reflect.TypeOf(&corev1.ConfigMap{})) require.NotNil(t, obj) @@ -775,6 +774,9 @@ func TestRoute(t *testing.T) { Name: naming.Name(manifestutils.GatewayComponentName, "test"), Namespace: "project1", Labels: manifestutils.ComponentLabels("gateway", "test"), + Annotations: map[string]string{ + "haproxy.router.openshift.io/timeout": "0s", + }, }, Spec: routev1.RouteSpec{ To: routev1.RouteTargetReference{ diff --git a/internal/manifests/gateway/openshift.go b/internal/manifests/gateway/openshift.go index 858564042..7878db200 100644 --- a/internal/manifests/gateway/openshift.go +++ b/internal/manifests/gateway/openshift.go @@ -22,6 +22,8 @@ import ( const ( gatewayOPAHTTPPort = 8082 gatewayOPAInternalPort = 8083 + + timeoutRouteAnnotation = "haproxy.router.openshift.io/timeout" ) // BuildServiceAccountAnnotations returns the annotations to use a ServiceAccount as an OAuth client. @@ -111,12 +113,20 @@ func route(tempo v1alpha1.TempoStack) (*routev1.Route, error) { return nil, fmt.Errorf("unsupported tls termination specified for route") } + annotations := tempo.Spec.Template.Gateway.Ingress.Annotations + if annotations == nil { + annotations = map[string]string{} + } + if annotations[timeoutRouteAnnotation] == "" { + annotations[timeoutRouteAnnotation] = fmt.Sprintf("%ds", int(tempo.Spec.Timeout.Duration.Seconds())) + } + return &routev1.Route{ ObjectMeta: metav1.ObjectMeta{ Name: naming.Name(manifestutils.GatewayComponentName, tempo.Name), Namespace: tempo.Namespace, Labels: labels, - Annotations: tempo.Spec.Template.Gateway.Ingress.Annotations, + Annotations: annotations, }, Spec: routev1.RouteSpec{ Host: tempo.Spec.Template.Gateway.Ingress.Host, diff --git a/internal/manifests/manifests_test.go b/internal/manifests/manifests_test.go index a76404b6f..dfeeddeff 100644 --- a/internal/manifests/manifests_test.go +++ b/internal/manifests/manifests_test.go @@ -2,6 +2,7 @@ package manifests import ( "testing" + "time" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -33,6 +34,7 @@ func TestBuildAll(t *testing.T) { Namespace: "project1", }, Spec: v1alpha1.TempoStackSpec{ + Timeout: metav1.Duration{Duration: time.Second * 5}, Template: v1alpha1.TempoTemplateSpec{ Gateway: v1alpha1.TempoGatewaySpec{ Enabled: true, diff --git a/internal/manifests/monolithic/build.go b/internal/manifests/monolithic/build.go index b084eeed4..3c16670ce 100644 --- a/internal/manifests/monolithic/build.go +++ b/internal/manifests/monolithic/build.go @@ -96,6 +96,7 @@ func BuildAll(opts Options) ([]client.Object, error) { oauthproxy.PatchStatefulSetForOauthProxy( tempo.ObjectMeta, tempo.Spec.JaegerUI.Authentication, + tempo.Spec.Timeout.Duration, opts.CtrlConfig, statefulSet) oauthproxy.PatchQueryFrontEndService(getJaegerUIService(services, tempo), tempo.Name) diff --git a/internal/manifests/monolithic/configmap.go b/internal/manifests/monolithic/configmap.go index 967414f61..8c1a1ea94 100644 --- a/internal/manifests/monolithic/configmap.go +++ b/internal/manifests/monolithic/configmap.go @@ -54,9 +54,11 @@ type tempoConfig struct { MultitenancyEnabled bool `yaml:"multitenancy_enabled,omitempty"` Server struct { - HTTPListenAddress string `yaml:"http_listen_address,omitempty"` - HttpListenPort int `yaml:"http_listen_port,omitempty"` - GRPCListenAddress string `yaml:"grpc_listen_address,omitempty"` + HTTPListenAddress string `yaml:"http_listen_address,omitempty"` + HttpListenPort int `yaml:"http_listen_port,omitempty"` + GRPCListenAddress string `yaml:"grpc_listen_address,omitempty"` + HttpServerReadTimeout time.Duration `yaml:"http_server_read_timeout,omitempty"` + HttpServerWriteTimeout time.Duration `yaml:"http_server_write_timeout,omitempty"` } `yaml:"server"` InternalServer struct { @@ -170,6 +172,8 @@ func buildTempoConfig(opts Options) ([]byte, error) { config := tempoConfig{} config.MultitenancyEnabled = tempo.Spec.Multitenancy != nil && tempo.Spec.Multitenancy.Enabled config.Server.HttpListenPort = manifestutils.PortHTTPServer + config.Server.HttpServerReadTimeout = opts.Tempo.Spec.Timeout.Duration + config.Server.HttpServerWriteTimeout = opts.Tempo.Spec.Timeout.Duration if tempo.Spec.Multitenancy.IsGatewayEnabled() { // all connections to tempo must go via gateway config.Server.HTTPListenAddress = "localhost" diff --git a/internal/manifests/monolithic/configmap_test.go b/internal/manifests/monolithic/configmap_test.go index f7f31270f..7eab5b3f3 100644 --- a/internal/manifests/monolithic/configmap_test.go +++ b/internal/manifests/monolithic/configmap_test.go @@ -80,6 +80,8 @@ func TestBuildConfig(t *testing.T) { expected: ` server: http_listen_port: 3200 + http_server_read_timeout: 30s + http_server_write_timeout: 30s internal_server: enable: true http_listen_address: 0.0.0.0 @@ -112,6 +114,8 @@ usage_report: expected: ` server: http_listen_port: 3200 + http_server_read_timeout: 30s + http_server_write_timeout: 30s internal_server: enable: true http_listen_address: 0.0.0.0 @@ -155,6 +159,8 @@ usage_report: expected: ` server: http_listen_port: 3200 + http_server_read_timeout: 30s + http_server_write_timeout: 30s internal_server: enable: true http_listen_address: 0.0.0.0 @@ -202,6 +208,8 @@ usage_report: expected: ` server: http_listen_port: 3200 + http_server_read_timeout: 30s + http_server_write_timeout: 30s internal_server: enable: true http_listen_address: 0.0.0.0 @@ -255,6 +263,8 @@ usage_report: expected: ` server: http_listen_port: 3200 + http_server_read_timeout: 30s + http_server_write_timeout: 30s internal_server: enable: true http_listen_address: 0.0.0.0 @@ -313,6 +323,8 @@ usage_report: expected: ` server: http_listen_port: 3200 + http_server_read_timeout: 30s + http_server_write_timeout: 30s internal_server: enable: true http_listen_address: 0.0.0.0 @@ -369,6 +381,8 @@ usage_report: expected: ` server: http_listen_port: 3200 + http_server_read_timeout: 30s + http_server_write_timeout: 30s internal_server: enable: true http_listen_address: 0.0.0.0 @@ -397,12 +411,14 @@ usage_report: name: "extra config", spec: v1alpha1.TempoMonolithicSpec{ ExtraConfig: &v1alpha1.ExtraConfigSpec{ - Tempo: apiextensionsv1.JSON{Raw: []byte(`{"storage": {"trace": {"wal": {"overlay_setting": "abc"}}}}`)}, + Tempo: apiextensionsv1.JSON{Raw: []byte(`{"storage": {"trace": {"wal": {"overlay_setting": "abc"}}}, "server": {"http_server_read_timeout": "1m", "http_server_write_timeout": "1m"}}`)}, }, }, expected: ` server: http_listen_port: 3200 + http_server_read_timeout: 1m + http_server_write_timeout: 1m internal_server: enable: true http_listen_address: 0.0.0.0 diff --git a/internal/manifests/monolithic/jaegerui_ingress.go b/internal/manifests/monolithic/jaegerui_ingress.go index 67d1c79a1..2ccb66404 100644 --- a/internal/manifests/monolithic/jaegerui_ingress.go +++ b/internal/manifests/monolithic/jaegerui_ingress.go @@ -69,6 +69,8 @@ func BuildJaegerUIIngress(opts Options) *networkingv1.Ingress { return ingress } +const timeoutRouteAnnotation = "haproxy.router.openshift.io/timeout" + // BuildJaegerUIRoute creates a Route object for Jaeger UI. func BuildJaegerUIRoute(opts Options) (*routev1.Route, error) { tempo := opts.Tempo @@ -89,6 +91,14 @@ func BuildJaegerUIRoute(opts Options) (*routev1.Route, error) { return nil, fmt.Errorf("unsupported tls termination '%s' specified for route", tempo.Spec.JaegerUI.Route.Termination) } + annotations := opts.Tempo.Spec.JaegerUI.Route.Annotations + if annotations == nil { + annotations = map[string]string{} + } + if annotations[timeoutRouteAnnotation] == "" { + annotations[timeoutRouteAnnotation] = fmt.Sprintf("%ds", int(tempo.Spec.Timeout.Duration.Seconds())) + } + return &routev1.Route{ TypeMeta: metav1.TypeMeta{ APIVersion: networkingv1.SchemeGroupVersion.String(), @@ -98,7 +108,7 @@ func BuildJaegerUIRoute(opts Options) (*routev1.Route, error) { Name: naming.Name(manifestutils.JaegerUIComponentName, tempo.Name), Namespace: tempo.Namespace, Labels: labels, - Annotations: tempo.Spec.JaegerUI.Route.Annotations, + Annotations: annotations, }, Spec: routev1.RouteSpec{ Host: tempo.Spec.JaegerUI.Route.Host, diff --git a/internal/manifests/monolithic/jaegerui_ingress_test.go b/internal/manifests/monolithic/jaegerui_ingress_test.go index 96824a432..e5cac15ad 100644 --- a/internal/manifests/monolithic/jaegerui_ingress_test.go +++ b/internal/manifests/monolithic/jaegerui_ingress_test.go @@ -163,6 +163,9 @@ func TestBuildJaegerUIRoute(t *testing.T) { Name: "tempo-sample-jaegerui", Namespace: "default", Labels: labels, + Annotations: map[string]string{ + "haproxy.router.openshift.io/timeout": "30s", + }, }, Spec: routev1.RouteSpec{ Host: "", @@ -221,6 +224,9 @@ func TestBuildJaegerUIRoute(t *testing.T) { Name: "tempo-sample-jaegerui", Namespace: "default", Labels: labels, + Annotations: map[string]string{ + "haproxy.router.openshift.io/timeout": "30s", + }, }, Spec: routev1.RouteSpec{ Host: "", diff --git a/internal/manifests/monolithic/statefulset.go b/internal/manifests/monolithic/statefulset.go index 0ba9ae8b6..b62a466b1 100644 --- a/internal/manifests/monolithic/statefulset.go +++ b/internal/manifests/monolithic/statefulset.go @@ -398,6 +398,7 @@ func configureGateway(opts Options, sts *appsv1.StatefulSet) error { fmt.Sprintf("--web.internal.listen=0.0.0.0:%d", manifestutils.GatewayPortInternalHTTPServer), // serves health checks fmt.Sprintf("--traces.tenant-header=%s", manifestutils.TenantHeader), fmt.Sprintf("--traces.tempo.endpoint=http://localhost:%d", manifestutils.PortHTTPServer), // Tempo API upstream + fmt.Sprintf("--traces.write-timeout=%s", opts.Tempo.Spec.Timeout.Duration.String()), fmt.Sprintf("--rbac.config=%s", path.Join(gatewayMountDir, "rbac", manifestutils.GatewayRBACFileName)), fmt.Sprintf("--tenants.config=%s", path.Join(gatewayMountDir, "tenants", manifestutils.GatewayTenantFileName)), "--log.level=info", diff --git a/internal/manifests/monolithic/statefulset_test.go b/internal/manifests/monolithic/statefulset_test.go index b2651704e..aa0239afc 100644 --- a/internal/manifests/monolithic/statefulset_test.go +++ b/internal/manifests/monolithic/statefulset_test.go @@ -2,6 +2,7 @@ package monolithic import ( "testing" + "time" "github.com/operator-framework/operator-lib/proxy" "github.com/stretchr/testify/require" @@ -732,6 +733,7 @@ func TestStatefulsetGateway(t *testing.T) { Namespace: "default", }, Spec: v1alpha1.TempoMonolithicSpec{ + Timeout: metav1.Duration{Duration: time.Second * 5}, Storage: &v1alpha1.MonolithicStorageSpec{ Traces: v1alpha1.MonolithicTracesStorageSpec{ Backend: "memory", @@ -788,6 +790,7 @@ func TestStatefulsetGateway(t *testing.T) { "--web.internal.listen=0.0.0.0:8081", "--traces.tenant-header=x-scope-orgid", "--traces.tempo.endpoint=http://localhost:3200", + "--traces.write-timeout=5s", "--rbac.config=/etc/tempo-gateway/rbac/rbac.yaml", "--tenants.config=/etc/tempo-gateway/tenants/tenants.yaml", "--log.level=info", diff --git a/internal/manifests/oauthproxy/oauth_proxy.go b/internal/manifests/oauthproxy/oauth_proxy.go index 2e90f2f12..f440e9c5c 100644 --- a/internal/manifests/oauthproxy/oauth_proxy.go +++ b/internal/manifests/oauthproxy/oauth_proxy.go @@ -3,6 +3,7 @@ package oauthproxy import ( "fmt" "strings" + "time" routev1 "github.com/openshift/api/route/v1" "github.com/operator-framework/operator-lib/proxy" @@ -69,9 +70,12 @@ func PatchRouteForOauthProxy(route *routev1.Route) { // point route to the oauth } // PatchStatefulSetForOauthProxy returns a modified StatefulSet with the oauth sidecar container and the right service account. -func PatchStatefulSetForOauthProxy(tempo metav1.ObjectMeta, +func PatchStatefulSetForOauthProxy( + tempo metav1.ObjectMeta, authSpec *v1alpha1.JaegerQueryAuthenticationSpec, - config configv1alpha1.ProjectConfig, statefulSet *v1.StatefulSet) { + timeout time.Duration, + config configv1alpha1.ProjectConfig, + statefulSet *v1.StatefulSet) { statefulSet.Spec.Template.Spec.Volumes = append(statefulSet.Spec.Template.Spec.Volumes, corev1.Volume{ Name: getTLSSecretNameForFrontendService(tempo.Name), VolumeSource: corev1.VolumeSource{ @@ -82,7 +86,7 @@ func PatchStatefulSetForOauthProxy(tempo metav1.ObjectMeta, }) statefulSet.Spec.Template.Spec.Containers = append(statefulSet.Spec.Template.Spec.Containers, - oAuthProxyContainer(tempo.Name, statefulSet.Spec.Template.Spec.ServiceAccountName, authSpec, config.DefaultImages.OauthProxy)) + oAuthProxyContainer(tempo.Name, statefulSet.Spec.Template.Spec.ServiceAccountName, authSpec, timeout, config.DefaultImages.OauthProxy)) } // PatchDeploymentForOauthProxy returns a modified deployment with the oauth sidecar container and the right service account. @@ -90,6 +94,7 @@ func PatchDeploymentForOauthProxy( tempo metav1.ObjectMeta, config configv1alpha1.ProjectConfig, authSpec *v1alpha1.JaegerQueryAuthenticationSpec, + timeout time.Duration, imageSpec configv1alpha1.ImagesSpec, dep *v1.Deployment) { dep.Spec.Template.Spec.Volumes = append(dep.Spec.Template.Spec.Volumes, corev1.Volume{ @@ -109,15 +114,18 @@ func PatchDeploymentForOauthProxy( } dep.Spec.Template.Spec.Containers = append(dep.Spec.Template.Spec.Containers, - oAuthProxyContainer(tempo.Name, naming.Name(manifestutils.QueryFrontendComponentName, tempo.Name), - authSpec, oauthProxyImage)) + oAuthProxyContainer(tempo.Name, + naming.Name(manifestutils.QueryFrontendComponentName, tempo.Name), + authSpec, + timeout, + oauthProxyImage)) } func getTLSSecretNameForFrontendService(tempoName string) string { return fmt.Sprintf("%s-ui-oauth-proxy-tls", tempoName) } -func proxyInitArguments(serviceAccountName string) []string { +func proxyInitArguments(serviceAccountName string, timeout time.Duration) []string { return []string{ // The SA Token is injected by admission controller by adding a volume via pod mutation // In Kubernetes 1.24 the SA token is short-lived (default 1h) @@ -136,6 +144,7 @@ func proxyInitArguments(serviceAccountName string) []string { fmt.Sprintf("--tls-cert=%s/tls.crt", tlsProxyPath), fmt.Sprintf("--tls-key=%s/tls.key", tlsProxyPath), fmt.Sprintf("--upstream=http://localhost:%d", manifestutils.PortJaegerUI), + fmt.Sprintf("--upstream-timeout=%s", timeout.String()), } } @@ -143,9 +152,10 @@ func oAuthProxyContainer( tempo string, serviceAccountName string, authSpec *v1alpha1.JaegerQueryAuthenticationSpec, + timeout time.Duration, oauthProxyImage string, ) corev1.Container { - args := proxyInitArguments(serviceAccountName) + args := proxyInitArguments(serviceAccountName, timeout) if len(strings.TrimSpace(authSpec.SAR)) > 0 { args = append(args, fmt.Sprintf("--openshift-sar=%s", authSpec.SAR)) diff --git a/internal/manifests/oauthproxy/oauth_proxy_test.go b/internal/manifests/oauthproxy/oauth_proxy_test.go index bd09a96fe..bde7ff220 100644 --- a/internal/manifests/oauthproxy/oauth_proxy_test.go +++ b/internal/manifests/oauthproxy/oauth_proxy_test.go @@ -2,6 +2,7 @@ package oauthproxy import ( "fmt" + "time" "testing" @@ -45,6 +46,7 @@ func TestOauthProxyContainer(t *testing.T) { fmt.Sprintf("--tls-cert=%s/tls.crt", tlsProxyPath), fmt.Sprintf("--tls-key=%s/tls.key", tlsProxyPath), fmt.Sprintf("--upstream=http://localhost:%d", manifestutils.PortJaegerUI), + "--upstream-timeout=5s", }, tempo: v1alpha1.TempoStack{ ObjectMeta: metav1.ObjectMeta{ @@ -75,6 +77,7 @@ func TestOauthProxyContainer(t *testing.T) { fmt.Sprintf("--tls-cert=%s/tls.crt", tlsProxyPath), fmt.Sprintf("--tls-key=%s/tls.key", tlsProxyPath), fmt.Sprintf("--upstream=http://localhost:%d", manifestutils.PortJaegerUI), + "--upstream-timeout=5s", "--openshift-sar={\"namespace\":\"app-dev\",\"resource\":\"services\",\"resourceName\":\"proxy\",\"verb\":\"get\"}", }, tempo: v1alpha1.TempoStack{ @@ -112,6 +115,7 @@ func TestOauthProxyContainer(t *testing.T) { container := oAuthProxyContainer(params.Tempo.Name, naming.Name(manifestutils.QueryFrontendComponentName, params.Tempo.Name), params.Tempo.Spec.Template.QueryFrontend.JaegerQuery.Authentication, + time.Second*5, customImage, ) expected := corev1.Container{ @@ -340,6 +344,7 @@ func TestPatchDeploymentForOauthProxy(t *testing.T) { params.Tempo.ObjectMeta, params.CtrlConfig, params.Tempo.Spec.Template.QueryFrontend.JaegerQuery.Authentication, + time.Second*5, params.Tempo.Spec.Images, dep) @@ -478,6 +483,7 @@ func TestPatchStatefulSetForOauthProxy(t *testing.T) { PatchStatefulSetForOauthProxy( params.Tempo.ObjectMeta, params.Tempo.Spec.Template.QueryFrontend.JaegerQuery.Authentication, + time.Second*5, params.CtrlConfig, statefulSet) diff --git a/internal/manifests/queryfrontend/query_frontend.go b/internal/manifests/queryfrontend/query_frontend.go index f457135df..bb232db0e 100644 --- a/internal/manifests/queryfrontend/query_frontend.go +++ b/internal/manifests/queryfrontend/query_frontend.go @@ -34,6 +34,8 @@ const ( containerNameTempo = "tempo" containerNameJaegerQuery = "jaeger-query" containerNameTempoQuery = "tempo-query" + + timeoutRouteAnnotation = "haproxy.router.openshift.io/timeout" ) // BuildQueryFrontend creates the query-frontend objects. @@ -85,9 +87,12 @@ func BuildQueryFrontend(params manifestutils.Params) ([]client.Object, error) { if jaegerUIAuthentication != nil && jaegerUIAuthentication.Enabled { oauthproxy.PatchDeploymentForOauthProxy( - tempo.ObjectMeta, params.CtrlConfig, + tempo.ObjectMeta, + params.CtrlConfig, tempo.Spec.Template.QueryFrontend.JaegerQuery.Authentication, - tempo.Spec.Images, d) + tempo.Spec.Timeout.Duration, + tempo.Spec.Images, + d) oauthproxy.PatchQueryFrontEndService(getQueryFrontendService(tempo, svcs), tempo.Name) manifests = append(manifests, oauthproxy.OAuthServiceAccount(params)) @@ -582,12 +587,20 @@ func route(tempo v1alpha1.TempoStack) (*routev1.Route, error) { serviceName := naming.Name(manifestutils.QueryFrontendComponentName, tempo.Name) + annotations := tempo.Spec.Template.QueryFrontend.JaegerQuery.Ingress.Annotations + if annotations == nil { + annotations = map[string]string{} + } + if annotations[timeoutRouteAnnotation] == "" { + annotations[timeoutRouteAnnotation] = fmt.Sprintf("%ds", int(tempo.Spec.Timeout.Duration.Seconds())) + } + return &routev1.Route{ ObjectMeta: metav1.ObjectMeta{ Name: queryFrontendName, Namespace: tempo.Namespace, Labels: labels, - Annotations: tempo.Spec.Template.QueryFrontend.JaegerQuery.Ingress.Annotations, + Annotations: annotations, }, Spec: routev1.RouteSpec{ Host: tempo.Spec.Template.QueryFrontend.JaegerQuery.Ingress.Host, diff --git a/internal/manifests/queryfrontend/query_frontend_test.go b/internal/manifests/queryfrontend/query_frontend_test.go index c11c7f352..e9a5650f9 100644 --- a/internal/manifests/queryfrontend/query_frontend_test.go +++ b/internal/manifests/queryfrontend/query_frontend_test.go @@ -511,6 +511,9 @@ func TestQueryFrontendJaegerRoute(t *testing.T) { Name: naming.Name(manifestutils.QueryFrontendComponentName, "test"), Namespace: "project1", Labels: manifestutils.ComponentLabels("query-frontend", "test"), + Annotations: map[string]string{ + "haproxy.router.openshift.io/timeout": "0s", + }, }, Spec: routev1.RouteSpec{ To: routev1.RouteTargetReference{ @@ -810,6 +813,9 @@ func TestQueryFrontendJaegerRouteSecured(t *testing.T) { Name: naming.Name(manifestutils.QueryFrontendComponentName, "test"), Namespace: "project1", Labels: manifestutils.ComponentLabels("query-frontend", "test"), + Annotations: map[string]string{ + "haproxy.router.openshift.io/timeout": "0s", + }, }, Spec: routev1.RouteSpec{ To: routev1.RouteTargetReference{ diff --git a/internal/webhooks/tempostack_webhook.go b/internal/webhooks/tempostack_webhook.go index 7f6a1f3e1..ea0f944fa 100644 --- a/internal/webhooks/tempostack_webhook.go +++ b/internal/webhooks/tempostack_webhook.go @@ -33,6 +33,7 @@ var ( zeroQuantity = resource.MustParse("0Gi") tenGBQuantity = resource.MustParse("10Gi") defaultServicesDuration = metav1.Duration{Duration: time.Hour * 24 * 3} + defaultTimeout = metav1.Duration{Duration: time.Second * 30} ) // TempoStackWebhook provides webhooks for TempoStack CR. @@ -171,6 +172,10 @@ func (d *Defaulter) Default(ctx context.Context, obj runtime.Object) error { } } + if r.Spec.Timeout.Duration == 0 { + r.Spec.Timeout = defaultTimeout + } + return nil } diff --git a/internal/webhooks/tempostack_webhook_test.go b/internal/webhooks/tempostack_webhook_test.go index b051ab134..b24f02553 100644 --- a/internal/webhooks/tempostack_webhook_test.go +++ b/internal/webhooks/tempostack_webhook_test.go @@ -86,6 +86,7 @@ func TestDefault(t *testing.T) { }, Spec: v1alpha1.TempoStackSpec{ ReplicationFactor: 2, + Timeout: metav1.Duration{Duration: time.Second * 30}, Images: configv1alpha1.ImagesSpec{ Tempo: "docker.io/grafana/tempo:1.2.3", TempoQuery: "docker.io/grafana/tempo-query:1.2.3", @@ -162,6 +163,7 @@ func TestDefault(t *testing.T) { }, Spec: v1alpha1.TempoStackSpec{ ReplicationFactor: 1, + Timeout: metav1.Duration{Duration: time.Second * 30}, Images: configv1alpha1.ImagesSpec{}, ServiceAccount: "tempo-test", Retention: v1alpha1.RetentionSpec{ @@ -244,6 +246,7 @@ func TestDefault(t *testing.T) { }, Spec: v1alpha1.TempoStackSpec{ ReplicationFactor: 1, + Timeout: metav1.Duration{Duration: time.Second * 30}, Images: configv1alpha1.ImagesSpec{}, ServiceAccount: "tempo-test", Retention: v1alpha1.RetentionSpec{ @@ -326,6 +329,7 @@ func TestDefault(t *testing.T) { }, Spec: v1alpha1.TempoStackSpec{ ReplicationFactor: 1, + Timeout: metav1.Duration{Duration: time.Second * 30}, Images: configv1alpha1.ImagesSpec{}, ServiceAccount: "tempo-test", Retention: v1alpha1.RetentionSpec{ @@ -425,6 +429,7 @@ func TestDefault(t *testing.T) { }, Spec: v1alpha1.TempoStackSpec{ ReplicationFactor: 1, + Timeout: metav1.Duration{Duration: time.Second * 30}, Images: configv1alpha1.ImagesSpec{}, ServiceAccount: "tempo-test", Retention: v1alpha1.RetentionSpec{ @@ -491,6 +496,108 @@ func TestDefault(t *testing.T) { Distribution: "upstream", }, }, + { + name: "timeout is set", + input: &v1alpha1.TempoStack{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + }, + Spec: v1alpha1.TempoStackSpec{ + ReplicationFactor: 2, + Images: configv1alpha1.ImagesSpec{ + Tempo: "docker.io/grafana/tempo:1.2.3", + TempoQuery: "docker.io/grafana/tempo-query:1.2.3", + TempoGateway: "docker.io/observatorium/gateway:1.2.3", + TempoGatewayOpa: "docker.io/observatorium/opa-openshift:1.2.4", + OauthProxy: "docker.io/observatorium/oauth-proxy:1.2.3", + }, + ServiceAccount: "tempo-test", + Retention: v1alpha1.RetentionSpec{ + Global: v1alpha1.RetentionConfig{ + Traces: metav1.Duration{Duration: time.Hour}, + }, + }, + Timeout: metav1.Duration{Duration: time.Hour}, + StorageSize: resource.MustParse("1Gi"), + LimitSpec: v1alpha1.LimitSpec{ + Global: v1alpha1.RateLimitSpec{ + Query: v1alpha1.QueryLimit{ + MaxSearchDuration: metav1.Duration{Duration: 1 * time.Hour}, + }, + }, + }, + }, + }, + expected: &v1alpha1.TempoStack{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test", + Labels: map[string]string{ + "app.kubernetes.io/managed-by": "tempo-operator", + "tempo.grafana.com/distribution": "upstream", + }, + }, + Spec: v1alpha1.TempoStackSpec{ + ReplicationFactor: 2, + Timeout: metav1.Duration{Duration: time.Hour}, + Images: configv1alpha1.ImagesSpec{ + Tempo: "docker.io/grafana/tempo:1.2.3", + TempoQuery: "docker.io/grafana/tempo-query:1.2.3", + TempoGateway: "docker.io/observatorium/gateway:1.2.3", + TempoGatewayOpa: "docker.io/observatorium/opa-openshift:1.2.4", + OauthProxy: "docker.io/observatorium/oauth-proxy:1.2.3", + }, + ServiceAccount: "tempo-test", + Retention: v1alpha1.RetentionSpec{ + Global: v1alpha1.RetentionConfig{ + Traces: metav1.Duration{Duration: time.Hour}, + }, + }, + StorageSize: resource.MustParse("1Gi"), + LimitSpec: v1alpha1.LimitSpec{ + Global: v1alpha1.RateLimitSpec{ + Query: v1alpha1.QueryLimit{ + MaxSearchDuration: metav1.Duration{Duration: 1 * time.Hour}, + }, + }, + }, + SearchSpec: v1alpha1.SearchSpec{ + MaxDuration: metav1.Duration{Duration: 0}, + DefaultResultLimit: &defaultDefaultResultLimit, + }, + Template: v1alpha1.TempoTemplateSpec{ + Compactor: v1alpha1.TempoComponentSpec{ + Replicas: ptr.To(int32(1)), + }, + Distributor: v1alpha1.TempoDistributorSpec{ + TempoComponentSpec: v1alpha1.TempoComponentSpec{ + Replicas: ptr.To(int32(1)), + }, + TLS: v1alpha1.TLSSpec{}, + }, + Ingester: v1alpha1.TempoComponentSpec{ + Replicas: ptr.To(int32(1)), + }, + Querier: v1alpha1.TempoComponentSpec{ + Replicas: ptr.To(int32(1)), + }, + Gateway: v1alpha1.TempoGatewaySpec{ + TempoComponentSpec: v1alpha1.TempoComponentSpec{ + Replicas: ptr.To(int32(1)), + }, + }, + QueryFrontend: v1alpha1.TempoQueryFrontendSpec{ + TempoComponentSpec: v1alpha1.TempoComponentSpec{ + Replicas: ptr.To(int32(1)), + }, + JaegerQuery: v1alpha1.JaegerQuerySpec{ + ServicesQueryDuration: &defaultServicesDuration, + }, + }, + }, + }, + }, + ctrlConfig: defaultCfgConfig, + }, } for _, test := range tests { diff --git a/tests/e2e-openshift/component-replicas/install-tempo-assert.yaml b/tests/e2e-openshift/component-replicas/install-tempo-assert.yaml index 450abdec7..d306e7429 100644 --- a/tests/e2e-openshift/component-replicas/install-tempo-assert.yaml +++ b/tests/e2e-openshift/component-replicas/install-tempo-assert.yaml @@ -156,6 +156,7 @@ spec: - --web.internal.listen=0.0.0.0:8081 - --traces.write.otlpgrpc.endpoint=tempo-cmpreps-distributor.chainsaw-replicas.svc.cluster.local:4317 - --traces.write.otlphttp.endpoint=https://tempo-cmpreps-distributor.chainsaw-replicas.svc.cluster.local:4318 + - --traces.write-timeout=30s - --traces.tempo.endpoint=https://tempo-cmpreps-query-frontend.chainsaw-replicas.svc.cluster.local:3200 - --grpc.listen=0.0.0.0:8090 - --rbac.config=/etc/tempo-gateway/cm/rbac.yaml diff --git a/tests/e2e-openshift/component-replicas/scale-tempo-assert.yaml b/tests/e2e-openshift/component-replicas/scale-tempo-assert.yaml index c23c4bb52..2d07ffb5d 100644 --- a/tests/e2e-openshift/component-replicas/scale-tempo-assert.yaml +++ b/tests/e2e-openshift/component-replicas/scale-tempo-assert.yaml @@ -157,6 +157,7 @@ spec: - --traces.write.otlpgrpc.endpoint=tempo-cmpreps-distributor.chainsaw-replicas.svc.cluster.local:4317 - --traces.write.otlphttp.endpoint=https://tempo-cmpreps-distributor.chainsaw-replicas.svc.cluster.local:4318 - --traces.tempo.endpoint=https://tempo-cmpreps-query-frontend.chainsaw-replicas.svc.cluster.local:3200 + - --traces.write-timeout=30s - --grpc.listen=0.0.0.0:8090 - --rbac.config=/etc/tempo-gateway/cm/rbac.yaml - --tenants.config=/etc/tempo-gateway/secret/tenants.yaml diff --git a/tests/e2e-openshift/monolithic-route/install-tempo-assert.yaml b/tests/e2e-openshift/monolithic-route/install-tempo-assert.yaml index 241609101..e27f27322 100644 --- a/tests/e2e-openshift/monolithic-route/install-tempo-assert.yaml +++ b/tests/e2e-openshift/monolithic-route/install-tempo-assert.yaml @@ -130,6 +130,7 @@ spec: - --tls-cert=/etc/tls/private/tls.crt - --tls-key=/etc/tls/private/tls.key - --upstream=http://localhost:16686 + - --upstream-timeout=70s - '--openshift-sar={"namespace": "chainsaw-mono-route", "resource": "pods", "verb": "get"}' name: oauth-proxy @@ -248,3 +249,24 @@ spec: app.kubernetes.io/instance: mono-route app.kubernetes.io/managed-by: tempo-operator app.kubernetes.io/name: tempo-monolithic +--- +apiVersion: route.openshift.io/v1 +kind: Route +metadata: + labels: + app.kubernetes.io/component: jaegerui + app.kubernetes.io/instance: mono-route + app.kubernetes.io/managed-by: tempo-operator + app.kubernetes.io/name: tempo-monolithic + name: tempo-mono-jaegerui + namespace: chainsaw-mono-route + annotations: + haproxy.router.openshift.io/timeout: 70s +spec: + port: + targetPort: oauth-proxy + tls: + termination: reencrypt + to: + kind: Service + name: tempo-mono-jaegerui diff --git a/tests/e2e-openshift/monolithic-route/install-tempo.yaml b/tests/e2e-openshift/monolithic-route/install-tempo.yaml index ca7c389c4..c44f9bdeb 100644 --- a/tests/e2e-openshift/monolithic-route/install-tempo.yaml +++ b/tests/e2e-openshift/monolithic-route/install-tempo.yaml @@ -4,6 +4,7 @@ metadata: name: mono-route namespace: chainsaw-mono-route spec: + timeout: 70s jaegerui: enabled: true route: diff --git a/tests/e2e-openshift/monolithic-single-tenant-auth/install-tempo-assert.yaml b/tests/e2e-openshift/monolithic-single-tenant-auth/install-tempo-assert.yaml index 2ddab9151..d32052933 100644 --- a/tests/e2e-openshift/monolithic-single-tenant-auth/install-tempo-assert.yaml +++ b/tests/e2e-openshift/monolithic-single-tenant-auth/install-tempo-assert.yaml @@ -130,6 +130,7 @@ spec: - --tls-cert=/etc/tls/private/tls.crt - --tls-key=/etc/tls/private/tls.key - --upstream=http://localhost:16686 + - --upstream-timeout=30s - '--openshift-sar={"namespace": "chainsaw-mst", "resource": "pods", "verb": "get"}' name: oauth-proxy diff --git a/tests/e2e-openshift/multitenancy/01-assert.yaml b/tests/e2e-openshift/multitenancy/01-assert.yaml index cb1017286..22581acd0 100644 --- a/tests/e2e-openshift/multitenancy/01-assert.yaml +++ b/tests/e2e-openshift/multitenancy/01-assert.yaml @@ -151,6 +151,7 @@ spec: - --web.internal.listen=0.0.0.0:8081 - --traces.write.otlpgrpc.endpoint=tempo-simplest-distributor.chainsaw-multitenancy.svc.cluster.local:4317 - --traces.write.otlphttp.endpoint=https://tempo-simplest-distributor.chainsaw-multitenancy.svc.cluster.local:4318 + - --traces.write-timeout=30s - --traces.tempo.endpoint=https://tempo-simplest-query-frontend.chainsaw-multitenancy.svc.cluster.local:3200 - --grpc.listen=0.0.0.0:8090 - --rbac.config=/etc/tempo-gateway/cm/rbac.yaml diff --git a/tests/e2e-openshift/route/install-tempo-assert.yaml b/tests/e2e-openshift/route/install-tempo-assert.yaml index 78e16793f..57b4ad8e6 100644 --- a/tests/e2e-openshift/route/install-tempo-assert.yaml +++ b/tests/e2e-openshift/route/install-tempo-assert.yaml @@ -3,6 +3,7 @@ kind: Route metadata: annotations: example_annotation: example_value + haproxy.router.openshift.io/timeout: 30s labels: app.kubernetes.io/component: query-frontend app.kubernetes.io/instance: simplest diff --git a/tests/e2e-openshift/route/install-tempo.yaml b/tests/e2e-openshift/route/install-tempo.yaml index 0feb8af11..c51ea0aeb 100644 --- a/tests/e2e-openshift/route/install-tempo.yaml +++ b/tests/e2e-openshift/route/install-tempo.yaml @@ -16,6 +16,7 @@ kind: TempoStack metadata: name: simplest spec: + timeout: 70s storage: secret: name: minio-test diff --git a/tests/e2e-openshift/tls-monolithic-singletenant/01-assert.yaml b/tests/e2e-openshift/tls-monolithic-singletenant/01-assert.yaml index 0b1f9dd38..5be1ff02b 100644 --- a/tests/e2e-openshift/tls-monolithic-singletenant/01-assert.yaml +++ b/tests/e2e-openshift/tls-monolithic-singletenant/01-assert.yaml @@ -136,6 +136,7 @@ spec: - --tls-cert=/etc/tls/private/tls.crt - --tls-key=/etc/tls/private/tls.key - --upstream=http://localhost:16686 + - --upstream-timeout=30s - '--openshift-sar={"namespace": "chainsaw-tls-mono-st", "resource": "pods", "verb": "get"}' name: oauth-proxy diff --git a/tests/e2e/monolithic-extraconfig/install-tempo-assert.yaml b/tests/e2e/monolithic-extraconfig/install-tempo-assert.yaml index d9897a059..b2244e1df 100644 --- a/tests/e2e/monolithic-extraconfig/install-tempo-assert.yaml +++ b/tests/e2e/monolithic-extraconfig/install-tempo-assert.yaml @@ -98,6 +98,8 @@ data: max_retries: 3 server: http_listen_port: 3200 + http_server_read_timeout: 30s + http_server_write_timeout: 30s storage: trace: backend: local diff --git a/tests/e2e/tempostack-extraconfig/install-tempostack-assert.yaml b/tests/e2e/tempostack-extraconfig/install-tempostack-assert.yaml index aa23663bf..4cad151d3 100644 --- a/tests/e2e/tempostack-extraconfig/install-tempostack-assert.yaml +++ b/tests/e2e/tempostack-extraconfig/install-tempostack-assert.yaml @@ -456,8 +456,8 @@ data: client_ca_file: /var/run/ca/service-ca.crt key_file: /var/run/tls/server/tls.key http_listen_port: 3200 - http_server_read_timeout: 3m - http_server_write_timeout: 3m + http_server_read_timeout: 10m + http_server_write_timeout: 10m log_format: logfmt tls_min_version: VersionTLS13 storage: @@ -568,8 +568,8 @@ data: client_ca_file: /var/run/ca/service-ca.crt key_file: /var/run/tls/server/tls.key http_listen_port: 3200 - http_server_read_timeout: 3m - http_server_write_timeout: 3m + http_server_read_timeout: 10m + http_server_write_timeout: 10m http_tls_config: cert_file: /var/run/tls/server/tls.crt client_auth_type: RequireAndVerifyClientCert diff --git a/tests/e2e/tempostack-extraconfig/install-tempostack.yaml b/tests/e2e/tempostack-extraconfig/install-tempostack.yaml index 812ff0bc3..2de1a58a6 100644 --- a/tests/e2e/tempostack-extraconfig/install-tempostack.yaml +++ b/tests/e2e/tempostack-extraconfig/install-tempostack.yaml @@ -3,8 +3,12 @@ kind: TempoStack metadata: name: simplest spec: + timeout: 70s extraConfig: tempo: + server: + http_server_write_timeout: 10m + http_server_read_timeout: 10m querier: search: query_timeout: 180s