Merge branch 'main' into fix-test-coverage

kubewharf · Oct 16, 2023 · c1d9d37 · c1d9d37
2 parents b9cb118 + 64f1cd0
commit c1d9d37
Show file tree

Hide file tree

Showing 48 changed files with 2,684 additions and 1,065 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -136,7 +136,7 @@ jobs:
           TRACE_ID=$(grep "Location: /trace/" curl-output.http | cut -d/ -f3 | tr -d '\r')
 
           mkdir -p output/api/traces
-          for mode in ff{0,1,2,3}{0,1}00000{0,1}; do
+          for mode in ff{0,1,2,3}{0,1,5,6}000000; do
             mode_trace=${mode}${TRACE_ID:10}
             curl -o output/api/traces/$mode_trace http://localhost:16686/api/traces/$mode_trace
           done
@@ -172,12 +172,12 @@ jobs:
               <html>
                 <head>
                   <title>Redirecting to ${basename}</title>
-                  <meta http-equiv="refresh" content="0;URL='/kelemetry/trace/ff20000000$(cat $trace_dir/trace_id)'" />
+                  <meta http-equiv="refresh" content="0;URL='/kelemetry/trace/ff26000000$(cat $trace_dir/trace_id)'" />
                 </head>
               </html>
           EOF
-            mkdir -p output/trace/ff20000000$(cat $trace_dir/trace_id)
-            cp spa.html output/trace/ff20000000$(cat $trace_dir/trace_id)/index.html
+            mkdir -p output/trace/ff26000000$(cat $trace_dir/trace_id)
+            cp spa.html output/trace/ff26000000$(cat $trace_dir/trace_id)/index.html
             cp $trace_dir/api/traces/* output/api/traces/
           done
           mv output/dot-usage/* output/
@@ -261,7 +261,7 @@ jobs:
                 promises.push((async () => {
                   const traceName = await fs.readFile(`pages/${trace}/trace_name`, {encoding: "utf8"})
                   const partialTraceId = await fs.readFile(`pages/${trace}/trace_id`, {encoding: "utf8"})
-                  const traceId = "ff20000000" + partialTraceId
+                  const traceId = "ff26000000" + partialTraceId
 
                   console.log(`Loading trace ${traceName}`)
 

diff --git a/Makefile b/Makefile
@@ -34,6 +34,8 @@ else
 	LOG_FILE_ARG ?=
 endif
 
+LINKER_WORKER_COUNT ?= 1
+
 CONTROLLERS ?= audit-consumer,audit-producer,audit-webhook,event-informer,annotation-linker,owner-linker,resource-object-tag,resource-event-tag,diff-decorator,diff-controller,diff-api,pprof,jaeger-storage-plugin,jaeger-redirect-server,kelemetrix
 ifeq ($(CONTROLLERS),)
 	ENABLE_ARGS ?=
@@ -69,7 +71,7 @@ endif
 .PHONY: run dump-rotate test usage dot kind stack pre-commit
 run: output/kelemetry $(DUMP_ROTATE_DEP)
 	GIN_MODE=debug \
-		./output/kelemetry \
+		$(RUN_PREFIX) ./output/kelemetry $(RUN_SUFFIX) \
 		--mq=local \
 		--audit-consumer-partition=$(PARTITIONS) \
 		--http-address=0.0.0.0 \
@@ -85,6 +87,7 @@ run: output/kelemetry $(DUMP_ROTATE_DEP)
 		--log-file=$(LOG_FILE) \
 		--aggregator-pseudo-span-global-tags=runId=$(RUN_ID) \
 		--aggregator-event-span-global-tags=run=$(RUN_ID) \
+		--linker-worker-count=$(LINKER_WORKER_COUNT) \
 		--pprof-addr=:6030 \
 		--diff-cache=$(ETCD_OR_LOCAL) \
 		--diff-cache-etcd-endpoints=127.0.0.1:2379 \
@@ -95,6 +98,7 @@ run: output/kelemetry $(DUMP_ROTATE_DEP)
 		--span-cache-etcd-endpoints=127.0.0.1:2379 \
 		--tracer-otel-endpoint=$(OTEL_EXPORTER_OTLP_ENDPOINT) \
 		--tracer-otel-insecure \
+		--object-cache-size=16777216 \
 		--jaeger-cluster-names=$(CLUSTER_NAME) \
 		--jaeger-storage-plugin-address=0.0.0.0:17271 \
 		--jaeger-backend=jaeger-storage \
@@ -117,15 +121,15 @@ test:
 	go test -v -race -coverpkg=./pkg/... -coverprofile=coverage.out $(INTEGRATION_ARG) $(BUILD_ARGS) ./pkg/...
 
 usage: output/kelemetry
-	./output/kelemetry --usage=USAGE.txt
+	$(RUN_PREFIX) ./output/kelemetry $(RUN_SUFFIX) --usage=USAGE.txt
 
 dot: output/kelemetry
-	./output/kelemetry --dot=depgraph.dot
+	$(RUN_PREFIX) ./output/kelemetry $(RUN_SUFFIX) --dot=depgraph.dot
 	dot -Tpng depgraph.dot >depgraph.png
 	dot -Tsvg depgraph.dot >depgraph.svg
 
 output/kelemetry: go.mod go.sum $(shell find -type f -name "*.go")
-	go build -v $(RACE_ARG) -ldflags=$(LDFLAGS) -o $@ $(BUILD_ARGS) .
+	go build -v $(RACE_ARG) -gcflags=$(GCFLAGS) -ldflags=$(LDFLAGS) -o $@ $(BUILD_ARGS) .
 
 kind:
 	kind delete cluster --name tracetest

diff --git a/charts/kelemetry/templates/_helpers.yaml b/charts/kelemetry/templates/_helpers.yaml
@@ -111,6 +111,7 @@ span-cache-etcd-prefix: {{ .Values.aggregator.spanCache.etcd.prefix | toJson }}
 {{- end }}
 
 {{/* LINKERS */}}
+linker-worker-count: {{ .Values.linkers.workerCount }}
 annotation-linker-enable: {{ .Values.linkers.annotation }}
 owner-linker-enable: {{ .Values.linkers.ownerReference }}
 

diff --git a/charts/kelemetry/values.yaml b/charts/kelemetry/values.yaml
@@ -323,6 +323,9 @@ aggregator:
 
 # Linkers associated objects together.
 linkers:
+  # Maximum number of concurrent link jobs.
+  # Each link job runs each of the linkers for a single object in series.
+  workerCount: 8
   # Enable the owner linker, which links objects based on native owner references.
   ownerReference: true
   # Enable the annotation linker, which links objects based on the `kelemetry.kubewharf.io/parent-link` annotation.

diff --git a/dev.docker-compose.yaml b/dev.docker-compose.yaml
@@ -54,6 +54,7 @@ services:
       - 127.0.0.1:17272:17271
     volumes:
       - badger:/mnt/badger
+    restart: always
 
   # Web frontend for raw trace database view.
   jaeger-query-raw:

diff --git a/docs/USER_GUIDE.md b/docs/USER_GUIDE.md
@@ -20,8 +20,12 @@ The "Service" field selects one of the display modes:
   Additional information is available in event tags.
 - `timeline`: All events are displayed as children of the root object.
 
-By default, the whole trace is displayed, including parent and sibling objects of the searched object.
-Enabling the `exclusive` option limits the output to the subtree under the object matched in the search.
+By default, only the trace for a single object is displayed.
+More traces are available by configuration:
+
+- `full tree`: view the full tree from the deepest ancestor
+- `ancestors`: include transitive owners
+- `children`: include child objects
 
 ### Cluster
 

diff --git a/go.mod b/go.mod
@@ -7,23 +7,23 @@ require (
 	github.com/dlclark/regexp2 v1.10.0
 	github.com/gin-gonic/gin v1.9.1
 	github.com/go-logr/logr v1.2.4
-	github.com/jaegertracing/jaeger v1.48.0
-	github.com/pelletier/go-toml/v2 v2.0.9
+	github.com/jaegertracing/jaeger v1.49.0
+	github.com/pelletier/go-toml/v2 v2.1.0
 	github.com/prometheus/client_golang v1.16.0
 	github.com/sirupsen/logrus v1.9.3
 	github.com/spf13/pflag v1.0.5
 	github.com/spf13/viper v1.16.0
 	github.com/stretchr/testify v1.8.4
 	go.etcd.io/etcd/client/v3 v3.5.9
-	go.opentelemetry.io/otel v1.16.0
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.16.0
-	go.opentelemetry.io/otel/sdk v1.16.0
-	go.opentelemetry.io/otel/trace v1.16.0
-	go.uber.org/zap v1.25.0
-	google.golang.org/grpc v1.57.0
+	go.opentelemetry.io/otel v1.18.0
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.18.0
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.18.0
+	go.opentelemetry.io/otel/sdk v1.18.0
+	go.opentelemetry.io/otel/trace v1.18.0
+	go.uber.org/zap v1.26.0
+	google.golang.org/grpc v1.58.1
 	k8s.io/api v0.28.1
-	k8s.io/apimachinery v0.28.1
+	k8s.io/apimachinery v0.28.2
 	k8s.io/apiserver v0.28.1
 	k8s.io/client-go v0.28.1
 	k8s.io/klog/v2 v2.100.1
@@ -32,7 +32,7 @@ require (
 
 require (
 	github.com/Shopify/sarama v1.38.1 // indirect
-	github.com/apache/thrift v0.18.1 // indirect
+	github.com/apache/thrift v0.19.0 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/bytedance/sonic v1.9.1 // indirect
 	github.com/cenkalti/backoff/v4 v4.2.1 // indirect
@@ -74,12 +74,12 @@ require (
 	github.com/google/go-cmp v0.5.9 // indirect
 	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/google/uuid v1.3.0 // indirect
-	github.com/grpc-ecosystem/grpc-gateway/v2 v2.15.2 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect
 	github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-hclog v1.5.0 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
-	github.com/hashicorp/go-plugin v1.4.10 // indirect
+	github.com/hashicorp/go-plugin v1.5.0 // indirect
 	github.com/hashicorp/go-uuid v1.0.3 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/hashicorp/yamux v0.1.1 // indirect
@@ -106,7 +106,6 @@ require (
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/oklog/run v1.1.0 // indirect
 	github.com/olivere/elastic v6.2.37+incompatible // indirect
-	github.com/opentracing/opentracing-go v1.2.0 // indirect
 	github.com/pierrec/lz4/v4 v4.1.17 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
@@ -119,33 +118,30 @@ require (
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/subosito/gotenv v1.4.2 // indirect
 	github.com/twitchyliquid64/golang-asm v0.15.1 // indirect
-	github.com/uber/jaeger-client-go v2.30.0+incompatible // indirect
-	github.com/uber/jaeger-lib v2.4.1+incompatible // indirect
 	github.com/ugorji/go/codec v1.2.11 // indirect
 	github.com/xdg-go/pbkdf2 v1.0.0 // indirect
 	github.com/xdg-go/scram v1.1.2 // indirect
 	github.com/xdg-go/stringprep v1.0.4 // indirect
 	go.etcd.io/etcd/api/v3 v3.5.9 // indirect
 	go.etcd.io/etcd/client/pkg/v3 v3.5.9 // indirect
 	go.opencensus.io v0.24.0 // indirect
-	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.42.1-0.20230612162650-64be7e574a17 // indirect
-	go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0 // indirect
-	go.opentelemetry.io/otel/metric v1.16.0 // indirect
-	go.opentelemetry.io/proto/otlp v0.19.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.43.0 // indirect
+	go.opentelemetry.io/otel/metric v1.18.0 // indirect
+	go.opentelemetry.io/proto/otlp v1.0.0 // indirect
 	go.uber.org/atomic v1.11.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
 	golang.org/x/arch v0.3.0 // indirect
-	golang.org/x/crypto v0.12.0 // indirect
-	golang.org/x/net v0.14.0 // indirect
-	golang.org/x/oauth2 v0.8.0 // indirect
-	golang.org/x/sys v0.11.0 // indirect
-	golang.org/x/term v0.11.0 // indirect
-	golang.org/x/text v0.12.0 // indirect
+	golang.org/x/crypto v0.13.0 // indirect
+	golang.org/x/net v0.15.0 // indirect
+	golang.org/x/oauth2 v0.10.0 // indirect
+	golang.org/x/sys v0.12.0 // indirect
+	golang.org/x/term v0.12.0 // indirect
+	golang.org/x/text v0.13.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20230526161137-0005af68ea54 // indirect
-	google.golang.org/genproto/googleapis/api v0.0.0-20230525234035-dd9d682886f9 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc // indirect
+	google.golang.org/genproto v0.0.0-20230711160842-782d3b101e98 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 // indirect
 	google.golang.org/protobuf v1.31.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect