feat: support K8S application logs

CONTRIBUTING.md

@@ -30,6 +30,8 @@ Start `minikube`:
 minikube start --insecure-registry "host.docker.internal:5000"
 ```
 
+* Note that if the minikube machine already exists, you will need to delete it first with `minikube delete` - otherwise the `--insecure-registry` parameter will be ignored (more details [here](https://stackoverflow.com/a/53937716))
+
 Start a local Docker registry:
 
 ```sh
@@ -46,15 +48,15 @@ $ curl localhost:5000/v2/_catalog -v
 > Host: localhost:5000
 > User-Agent: curl/7.77.0
 > Accept: */*
-> 
+>
 * Mark bundle as not supporting multiuse
 < HTTP/1.1 200 OK
 < Content-Type: application/json; charset=utf-8
 < Docker-Distribution-Api-Version: registry/2.0
 < X-Content-Type-Options: nosniff
 < Date: Fri, 20 Jan 2023 08:10:32 GMT
 < Content-Length: 20
-< 
+<
 {"repositories":[]}
 * Connection #0 to host localhost left intact
 ```
@@ -100,7 +102,7 @@ Changing the target Lumigo backend can be done with a [`patchStrategicMerge`](ht
 echo -n "apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: lumigo-controller-manager
+  name: lumigo-lumigo-controller-manager
 spec:
   template:
     spec:
@@ -109,6 +111,8 @@ spec:
         env:
         - name: LUMIGO_ENDPOINT
           value: \"https://my.lumigo.endpoint\" # Replace this!
+        - name: LUMIGO_LOGS_ENDPOINT
+          value: \"https://my.lumigo.endpoint\" # Replace this!
 " > lumigo-endpoint.patch.yaml
 kubectl patch --patch-file lumigo-endpoint.patch.yaml --type strategic -n lumigo-system --filename=lumigo-endpoint.patch.yaml
 ```
@@ -123,8 +127,8 @@ If you see the following, it's likely because and Mac OS has [squatted over port
 docker push host.docker.internal:5000/controller
 Using default tag: latest
 The push refers to repository [host.docker.internal:5000/controller]
-377b701db379: Preparing 
-fba4381f2bb7: Preparing 
+377b701db379: Preparing
+fba4381f2bb7: Preparing
 error parsing HTTP 403 response body: unexpected end of JSON input: ""
 ```
 

Makefile

@@ -73,7 +73,7 @@ vet: ## Run go vet against code.
 
 .PHONY: test
 test: manifests generate fmt vet envtest ## Run tests.
-	(cd ./controller/src && KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" $(GOCMD) test . -coverprofile cover.out )
+	(cd ./controller/src && KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" $(GOCMD) test ./... -failfast -coverprofile cover.out )
 
 ##@ Build
 
@@ -182,8 +182,7 @@ controller-gen: $(CONTROLLER_GEN) ## Download controller-gen locally if necessar
 $(CONTROLLER_GEN): $(LOCALBIN)
 	test -s $(LOCALBIN)/controller-gen || GOBIN=$(LOCALBIN) $(GOCMD) install sigs.k8s.io/controller-tools/cmd/controller-gen@$(CONTROLLER_TOOLS_VERSION)
 
-## Pin the version of setup-envtest until https://github.com/kubernetes-sigs/controller-runtime/issues/2720 is resolved.
 .PHONY: envtest
 envtest: $(ENVTEST) ## Download envtest-setup locally if necessary.
 $(ENVTEST): $(LOCALBIN)
-	test -s $(LOCALBIN)/setup-envtest || GOBIN=$(LOCALBIN) $(GOCMD) install sigs.k8s.io/controller-runtime/tools/setup-envtest@c7e1dc9b
+	test -s $(LOCALBIN)/setup-envtest || GOBIN=$(LOCALBIN) $(GOCMD) install sigs.k8s.io/controller-runtime/tools/setup-envtest

charts/lumigo-operator/templates/controller-deployment-and-webhooks.yaml

@@ -231,6 +231,8 @@ spec:
           value: "{{ .Values.debug.enabled | default false }}"
         - name: LUMIGO_ENDPOINT
           value: "{{ .Values.endpoint.otlp.url }}"
+        - name: LUMIGO_LOGS_ENDPOINT
+          value: "{{ .Values.endpoint.otlp.logs_url }}"
         - name: LUMIGO_OPERATOR_VERSION
           value: "{{ $lumigoOperatorVersion }}"
         - name: LUMIGO_OPERATOR_DEPLOYMENT_METHOD

charts/lumigo-operator/values.yaml

@@ -61,4 +61,5 @@ metricsService:
   type: ClusterIP
 endpoint:
   otlp:
-    url: https://ga-otlp.lumigo-tracer-edge.golumigo.com
+    url: https://ga-otlp.lumigo-tracer-edge.golumigo.com
+    logs_url: https://ga-otlp.lumigo-tracer-edge.golumigo.com

config/manager/manager.yaml

@@ -131,6 +131,8 @@ spec:
               value: Kustomize
             - name: LUMIGO_ENDPOINT
               value: https://ga-otlp.lumigo-tracer-edge.golumigo.com
+            - name: LUMIGO_LOGS_ENDPOINT
+              value: https://ga-otlp.lumigo-tracer-edge.golumigo.com
           ports:
             - containerPort: 4318
               name: otlphttp

controller/src/api/v1alpha1/lumigo_webhook_suite_test.go

@@ -61,8 +61,8 @@ var _ = BeforeSuite(func() {
 
 	By("bootstrapping test environment")
 	testEnv = &envtest.Environment{
-		CRDDirectoryPaths:     []string{filepath.Join("..", "..", "config", "crd", "bases")},
-		ErrorIfCRDPathMissing: false,
+		CRDDirectoryPaths:     []string{filepath.Join("..", "..", "..", "..", "config", "crd", "bases")},
+		ErrorIfCRDPathMissing: true,
 		WebhookInstallOptions: envtest.WebhookInstallOptions{
 			Paths: []string{filepath.Join("..", "..", "config", "webhook")},
 		},

controller/src/controllers/lumigo_controller.go

@@ -77,6 +77,7 @@ type LumigoReconciler struct {
 	LumigoOperatorVersion                     string
 	LumigoInjectorImage                       string
 	TelemetryProxyOtlpServiceUrl              string
+	TelemetryProxyOtlpLogsServiceUrl          string
 	TelemetryProxyNamespaceConfigurationsPath string
 }
 
@@ -489,7 +490,7 @@ func (r *LumigoReconciler) updateStatusIfNeeded(ctx context.Context, logger logr
 }
 
 func (r *LumigoReconciler) injectLumigoIntoResources(ctx context.Context, lumigo *operatorv1alpha1.Lumigo, log *logr.Logger) error {
-	mutator, err := mutation.NewMutator(log, &lumigo.Spec.LumigoToken, r.LumigoOperatorVersion, r.LumigoInjectorImage, r.TelemetryProxyOtlpServiceUrl)
+	mutator, err := mutation.NewMutator(log, &lumigo.Spec.LumigoToken, r.LumigoOperatorVersion, r.LumigoInjectorImage, r.TelemetryProxyOtlpServiceUrl, r.TelemetryProxyOtlpLogsServiceUrl)
 	if err != nil {
 		return fmt.Errorf("cannot instantiate mutator: %w", err)
 	}
@@ -688,7 +689,7 @@ func (r *LumigoReconciler) injectLumigoIntoResources(ctx context.Context, lumigo
 
 func (r *LumigoReconciler) removeLumigoFromResources(ctx context.Context, lumigo *operatorv1alpha1.Lumigo, log *logr.Logger) error {
 	namespace := lumigo.Namespace
-	mutator, err := mutation.NewMutator(log, nil, r.LumigoOperatorVersion, r.LumigoInjectorImage, r.TelemetryProxyOtlpServiceUrl)
+	mutator, err := mutation.NewMutator(log, nil, r.LumigoOperatorVersion, r.LumigoInjectorImage, r.TelemetryProxyOtlpServiceUrl, r.TelemetryProxyOtlpLogsServiceUrl)
 	if err != nil {
 		return fmt.Errorf("cannot instantiate mutator: %w", err)
 	}

controller/src/controllers/lumigo_controller_suite_test.go

@@ -81,7 +81,7 @@ var _ = BeforeSuite(func() {
 
 	By("bootstrapping test environment")
 	testEnv = &envtest.Environment{
-		CRDDirectoryPaths:     []string{filepath.Join("..", "config", "crd", "bases")},
+		CRDDirectoryPaths:     []string{filepath.Join("..", "..", "..", "config", "crd", "bases")},
 		ErrorIfCRDPathMissing: true,
 	}
 

controller/src/main.go

@@ -129,6 +129,7 @@ func startManager(metricsAddr string, probeAddr string, enableLeaderElection boo
 	}
 
 	telemetryProxyOtlpService := lumigoEndpoint + "/v1/traces" // TODO: Fix it when the distros use the Lumigo endpoint as root
+	telemetryProxyOtlpLogsService := lumigoEndpoint + "/v1/logs"
 
 	namespaceConfigurationsPath, isSet := os.LookupEnv("LUMIGO_NAMESPACE_CONFIGURATIONS")
 	if !isSet {
@@ -151,26 +152,28 @@ func startManager(metricsAddr string, probeAddr string, enableLeaderElection boo
 	}
 
 	if err = (&controllers.LumigoReconciler{
-		Client:                       mgr.GetClient(),
-		Clientset:                    clientset,
-		DynamicClient:                dynamicClient,
-		EventRecorder:                mgr.GetEventRecorderFor(fmt.Sprintf("lumigo-operator.v%s/controller", lumigoOperatorVersion)),
-		Scheme:                       mgr.GetScheme(),
-		LumigoOperatorVersion:        lumigoOperatorVersion,
-		LumigoInjectorImage:          lumigoInjectorImage,
-		TelemetryProxyOtlpServiceUrl: telemetryProxyOtlpService,
+		Client:                           mgr.GetClient(),
+		Clientset:                        clientset,
+		DynamicClient:                    dynamicClient,
+		EventRecorder:                    mgr.GetEventRecorderFor(fmt.Sprintf("lumigo-operator.v%s/controller", lumigoOperatorVersion)),
+		Scheme:                           mgr.GetScheme(),
+		LumigoOperatorVersion:            lumigoOperatorVersion,
+		LumigoInjectorImage:              lumigoInjectorImage,
+		TelemetryProxyOtlpServiceUrl:     telemetryProxyOtlpService,
+		TelemetryProxyOtlpLogsServiceUrl: telemetryProxyOtlpLogsService,
 		TelemetryProxyNamespaceConfigurationsPath: namespaceConfigurationsPath,
 		Log: logger,
 	}).SetupWithManager(mgr); err != nil {
 		return fmt.Errorf("unable to create controller: %w", err)
 	}
 
 	if err = (&injector.LumigoInjectorWebhookHandler{
-		EventRecorder:                mgr.GetEventRecorderFor(fmt.Sprintf("lumigo-operator.v%s/injector-webhook", lumigoOperatorVersion)),
-		LumigoOperatorVersion:        lumigoOperatorVersion,
-		LumigoInjectorImage:          lumigoInjectorImage,
-		TelemetryProxyOtlpServiceUrl: telemetryProxyOtlpService,
-		Log:                          logger,
+		EventRecorder:                    mgr.GetEventRecorderFor(fmt.Sprintf("lumigo-operator.v%s/injector-webhook", lumigoOperatorVersion)),
+		LumigoOperatorVersion:            lumigoOperatorVersion,
+		LumigoInjectorImage:              lumigoInjectorImage,
+		TelemetryProxyOtlpServiceUrl:     telemetryProxyOtlpService,
+		TelemetryProxyOtlpLogsServiceUrl: telemetryProxyOtlpLogsService,
+		Log:                              logger,
 	}).SetupWebhookWithManager(mgr); err != nil {
 		return fmt.Errorf("unable to create injector webhook: %w", err)
 	}

controller/src/mutation/mutate.go

@@ -46,6 +46,7 @@ const LumigoInjectorVolumeName = "lumigo-injector"
 const LumigoInjectorVolumeMountPoint = "/opt/lumigo"
 const LumigoTracerTokenEnvVarName = "LUMIGO_TRACER_TOKEN"
 const LumigoEndpointEnvVarName = "LUMIGO_ENDPOINT"
+const LumigoLogsEndpointEnvVarName = "LUMIGO_LOGS_ENDPOINT"
 const LumigoContainerNameEnvVarName = "LUMIGO_CONTAINER_NAME"
 const LdPreloadEnvVarName = "LD_PRELOAD"
 const LdPreloadEnvVarValue = LumigoInjectorVolumeMountPoint + "/injector/lumigo_injector.so"
@@ -78,6 +79,7 @@ type mutatorImpl struct {
 	log                       *logr.Logger
 	lumigoAutotraceLabelValue string
 	lumigoEndpoint            string
+	lumigoLogsEndpoint        string
 	lumigoToken               *operatorv1alpha1.Credentials
 	lumigoInjectorImage       string
 }
@@ -86,7 +88,7 @@ func (m *mutatorImpl) GetAutotraceLabelValue() string {
 	return m.lumigoAutotraceLabelValue
 }
 
-func NewMutator(Log *logr.Logger, LumigoToken *operatorv1alpha1.Credentials, LumigoOperatorVersion string, LumigoInjectorImage string, TelemetryProxyOtlpServiceUrl string) (Mutator, error) {
+func NewMutator(Log *logr.Logger, LumigoToken *operatorv1alpha1.Credentials, LumigoOperatorVersion string, LumigoInjectorImage string, TelemetryProxyOtlpServiceUrl string, TelemetryProxyOtlpLogsServiceUrl string) (Mutator, error) {
 	version := LumigoOperatorVersion
 
 	if len(version) > 8 {
@@ -97,6 +99,7 @@ func NewMutator(Log *logr.Logger, LumigoToken *operatorv1alpha1.Credentials, Lum
 		log:                       Log,
 		lumigoAutotraceLabelValue: LumigoAutoTraceLabelVersionPrefixValue + version,
 		lumigoEndpoint:            TelemetryProxyOtlpServiceUrl,
+		lumigoLogsEndpoint:        TelemetryProxyOtlpLogsServiceUrl,
 		lumigoToken:               LumigoToken,
 		lumigoInjectorImage:       LumigoInjectorImage,
 	}, nil
@@ -403,6 +406,18 @@ func (m *mutatorImpl) injectLumigoIntoPodSpec(podSpec *corev1.PodSpec) error {
 		} else {
 			envVars[lumigoEndpointEnvVarIndex] = *lumigoEndpointEnvVar
 		}
+
+		lumigoLogsEndpointEnvVar := &corev1.EnvVar{
+			Name:  LumigoLogsEndpointEnvVarName,
+			Value: m.lumigoLogsEndpoint,
+		}
+		lumigoLogsEndpointEnvVarIndex := slices.IndexFunc(envVars, func(c corev1.EnvVar) bool { return c.Name == LumigoLogsEndpointEnvVarName })
+		if lumigoLogsEndpointEnvVarIndex < 0 {
+			envVars = append(envVars, *lumigoLogsEndpointEnvVar)
+		} else {
+			envVars[lumigoLogsEndpointEnvVarIndex] = *lumigoLogsEndpointEnvVar
+		}
+
 		lumigoContainerNameEnvVar := &corev1.EnvVar{
 			Name:  LumigoContainerNameEnvVarName,
 			Value: container.Name,

controller/src/webhooks/defaulter/defaulter_webhook_suite_test.go

@@ -71,10 +71,10 @@ var _ = BeforeSuite(func() {
 
 	By("bootstrapping test environment")
 	testEnv = &envtest.Environment{
-		CRDDirectoryPaths:     []string{filepath.Join("..", "..", "config", "crd", "bases")},
-		ErrorIfCRDPathMissing: false,
+		CRDDirectoryPaths:     []string{filepath.Join("..", "..", "..", "..", "config", "crd", "bases")},
+		ErrorIfCRDPathMissing: true,
 		WebhookInstallOptions: envtest.WebhookInstallOptions{
-			Paths: []string{filepath.Join("..", "..", "config", "webhooks")},
+			Paths: []string{filepath.Join("..", "..", "..", "..", "config", "webhooks")},
 		},
 	}
 

controller/src/webhooks/injector/injector_webhook.go

@@ -50,10 +50,11 @@ type LumigoInjectorWebhookHandler struct {
 	client.Client
 	record.EventRecorder
 	*admission.Decoder
-	LumigoOperatorVersion        string
-	LumigoInjectorImage          string
-	TelemetryProxyOtlpServiceUrl string
-	Log                          logr.Logger
+	LumigoOperatorVersion            string
+	LumigoInjectorImage              string
+	TelemetryProxyOtlpServiceUrl     string
+	TelemetryProxyOtlpLogsServiceUrl string
+	Log                              logr.Logger
 }
 
 func (h *LumigoInjectorWebhookHandler) SetupWebhookWithManager(mgr ctrl.Manager) error {
@@ -132,7 +133,7 @@ func (h *LumigoInjectorWebhookHandler) Handle(ctx context.Context, request admis
 		return admission.Allowed(fmt.Sprintf("The Lumigo object in the '%s' namespace is not active; resource will not be mutated", namespace))
 	}
 
-	mutator, err := mutation.NewMutator(&log, &lumigo.Spec.LumigoToken, h.LumigoOperatorVersion, h.LumigoInjectorImage, h.TelemetryProxyOtlpServiceUrl)
+	mutator, err := mutation.NewMutator(&log, &lumigo.Spec.LumigoToken, h.LumigoOperatorVersion, h.LumigoInjectorImage, h.TelemetryProxyOtlpServiceUrl, h.TelemetryProxyOtlpLogsServiceUrl)
 	if err != nil {
 		return admission.Allowed(fmt.Errorf("cannot instantiate mutator: %w", err).Error())
 	}

controller/src/webhooks/injector/injector_webhook_suite_test.go

@@ -64,6 +64,7 @@ var lumigoApiVersion = fmt.Sprintf("%s/%s", operatorv1alpha1.GroupVersion.Group,
 var lumigoOperatorVersion = "2b1e6b60ca871edee1d8f543c400f0b24663349144b78c79cfa006efaad6176a" // Unrealistically long, but we need to ensure we don't set label values too long
 var lumigoInjectorImage = "localhost:5000/lumigo-autotrace:test"
 var telemetryProxyOtlpServiceUrl = "lumigo-telemetry-proxy.lumigo-system.svc.cluster.local"
+var telemetryProxyOtlpLogsServiceUrl = telemetryProxyOtlpServiceUrl
 
 var statusActive = operatorv1alpha1.LumigoStatus{
 	Conditions: []operatorv1alpha1.LumigoCondition{
@@ -117,10 +118,10 @@ var _ = BeforeSuite(func() {
 
 	By("bootstrapping test environment")
 	testEnv = &envtest.Environment{
-		CRDDirectoryPaths:     []string{filepath.Join("..", "..", "config", "crd", "bases")},
-		ErrorIfCRDPathMissing: false,
+		CRDDirectoryPaths:     []string{filepath.Join("..", "..", "..", "..", "config", "crd", "bases")},
+		ErrorIfCRDPathMissing: true,
 		WebhookInstallOptions: envtest.WebhookInstallOptions{
-			Paths: []string{filepath.Join("..", "..", "config", "webhooks")},
+			Paths: []string{filepath.Join("..", "..", "..", "..", "config", "webhooks")},
 		},
 	}
 
@@ -174,11 +175,12 @@ var _ = BeforeSuite(func() {
 	Expect(err).NotTo(HaveOccurred())
 
 	err = (&LumigoInjectorWebhookHandler{
-		EventRecorder:                mgr.GetEventRecorderFor(fmt.Sprintf("lumigo-operator.v%s", lumigoOperatorVersion)),
-		LumigoOperatorVersion:        lumigoOperatorVersion,
-		LumigoInjectorImage:          lumigoInjectorImage,
-		TelemetryProxyOtlpServiceUrl: telemetryProxyOtlpServiceUrl,
-		Log:                          ctrl.Log.WithName("injector-webhook").WithName("Lumigo"),
+		EventRecorder:                    mgr.GetEventRecorderFor(fmt.Sprintf("lumigo-operator.v%s", lumigoOperatorVersion)),
+		LumigoOperatorVersion:            lumigoOperatorVersion,
+		LumigoInjectorImage:              lumigoInjectorImage,
+		TelemetryProxyOtlpServiceUrl:     telemetryProxyOtlpServiceUrl,
+		TelemetryProxyOtlpLogsServiceUrl: telemetryProxyOtlpLogsServiceUrl,
+		Log:                              ctrl.Log.WithName("injector-webhook").WithName("Lumigo"),
 	}).SetupWebhookWithManager(mgr)
 	Expect(err).NotTo(HaveOccurred())
 

helm.yaml

@@ -683,6 +683,8 @@ spec:
             readOnlyRootFilesystem: true
             runAsNonRoot: true
         - env:
+            - name: LUMIGO_LOGS_ENDPOINT
+              value: https://ga-otlp.lumigo-tracer-edge.golumigo.com
             - name: LUMIGO_ENDPOINT
               value: https://ga-otlp.lumigo-tracer-edge.golumigo.com
             - mountPath: /lumigo/etc/namespaces/

telemetryproxy/docker/etc/config.yaml.tpl

@@ -93,6 +93,10 @@ exporters:
     endpoint: {{ env.Getenv "LUMIGO_ENDPOINT" "https://ga-otlp.lumigo-tracer-edge.golumigo.com" }}
     auth:
       authenticator: headers_setter/lumigo
+  otlphttp/lumigo_logs:
+    endpoint: {{ env.Getenv "LUMIGO_LOGS_ENDPOINT" "https://ga-otlp.lumigo-tracer-edge.golumigo.com" }}
+    auth:
+      authenticator: headers_setter/lumigo
 {{- if $debug }}
   logging:
     verbosity: detailed
@@ -233,6 +237,21 @@ service:
       - logging
 {{- end }}
       - otlphttp/lumigo_ns_{{ $namespace.name }}
+    logs/application_logs_ns_{{ $namespace.name }}:
+      receivers:
+      - otlp
+      processors:
+      - k8sdataenricherprocessor
+      - transform/add_ns_attributes_ns_{{ $namespace.name }}
+{{- if $clusterName }}
+      - transform/add_cluster_name
+{{- end }}
+      - transform/inject_operator_details_into_resource
+      exporters:
+{{- if $config.debug }}
+      - logging
+{{- end }}
+      - otlphttp/lumigo_logs
     logs/k8s_objects_ns_{{ $namespace.name }}:
       receivers:
       - k8sobjects/objects_ns_{{ $namespace.name }}