[charts/bigdata-operator] add metrics collection to fluentbit

spotinst · Mar 26, 2024 · fff26c7 · fff26c7
1 parent ba1ba0c
commit fff26c7
Show file tree

Hide file tree

Showing 3 changed files with 38 additions and 2 deletions.
diff --git a/charts/bigdata-operator/templates/common_telemetry_configmap.yaml b/charts/bigdata-operator/templates/common_telemetry_configmap.yaml
@@ -14,6 +14,7 @@ data:
         Regex ^(?<time>\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}.\d{3}Z)\t(?<level>[A-Z]+)\t(?<logger>[^\s]+)\t(?<message>.*)$
         Time_Key    time
         Time_Format %Y-%m-%dT%H:%M:%S.%L%z
+        storage.max_chunks_up     10
 
   fluent-bit.conf: |
     [SERVICE]
@@ -40,6 +41,7 @@ data:
         K8S-Logging.Exclude On
 
     @INCLUDE custom-filters.conf
+    @INCLUDE metrics-collection.conf
 
     [FILTER]
         Name modify

diff --git a/charts/bigdata-operator/templates/custom_telemetry_configmap.yaml b/charts/bigdata-operator/templates/custom_telemetry_configmap.yaml
@@ -11,4 +11,35 @@ data:
         Name    grep
         Match   kube.*
         Regex   $kubernetes['labels']['bigdata.spot.io/component'] {{ include "bigdata-operator.fullname" . }}
+
+  metrics-collection.conf: |
+    [INPUT]
+      name                    prometheus_scrape
+      host                    0.0.0.0
+      port                    8080
+      tag                     {{ include "bigdata-operator.fullname" . }}
+      metrics_path            /metrics
+      scrape_interval         5s
+
+    # [OUTPUT]
+    #   Name                 http
+    #   Match                *
+    #   Host                 bigdata-telemetry-thanos-receiver-svc.spot-system.svc.cluster.local
+    #   Port                 19291
+    #   URI                  /api/v1/receive
+    #   Format               json
+    #   tls                  off
+    #   tls.verify           off
+
+
+    [OUTPUT]
+      Name                 prometheus_remote_write
+      Match                {{ include "bigdata-operator.fullname" . }}
+      Host                 bigdata-telemetry-thanos-receiver-svc.spot-system.svc.cluster.local
+      Port                 19291
+      uri                  /api/v1/receive
+      tls                  off
+      tls.verify           off
+      add_label            ClusterId ${CLUSTER_ID}
+      add_label            AccountID ${ACCOUNT_ID}
 {{- end }}
diff --git a/charts/bigdata-operator/templates/deployment.yaml b/charts/bigdata-operator/templates/deployment.yaml
@@ -121,9 +121,12 @@ spec:
             - name: config
               mountPath: /fluent-bit/etc/fluent-bit.conf
               subPath: fluent-bit.conf
-            - name: custom-filters
+            - name: custom-config
               mountPath: /fluent-bit/etc/custom-filters.conf
               subPath: custom-filters.conf
+            - name: custom-config
+              mountPath: /fluent-bit/etc/metrics-collection.conf
+              subPath: metrics-collection.conf
             - name: varlog
               readOnly: true
               mountPath: /var/log/
@@ -136,7 +139,7 @@ spec:
         - name: config
           configMap:
             name: bigdata-telemetry-fluentbit-cm
-        - name: custom-filters
+        - name: custom-config
           configMap:
             name: {{ include "bigdata-operator.fullname" . }}-custom-fluentbit-cm
         - name: varlog