filebeat-daemonset-customformat.yaml

# Example of FB daemonset intended to retrieve logs of some specific pods.
# Autodiscover with config based templates used.
# Pipeline associated:
# PUT _ingest/pipeline/custom-text-example
# {
#   "processors": [
#     {
#       "dissect": {
#         "tag": "dissect-processed",
#         "field": "message",
#         "pattern": "%{@timestamp}|%{customfield1}|%{customfield2}"
#       }
#     }
#   ]
# }
apiVersion: beat.k8s.elastic.co/v1beta1
kind: Beat
metadata:
  name: logs-customformat
  namespace: monitoring
spec:
  type: filebeat
  version: 7.16.2
  elasticsearchRef:
    name: logging-and-metrics
  config:
    http.enabled: true
    http.port: 5066
    http.host: 0.0.0.0
    monitoring.enabled: false
    filebeat:
      autodiscover:
        providers:
        - type: kubernetes
          host: ${NODE_NAME}
          # used when hostNetwork is true
          #host: ${HOSTNAME}
          scope: node
          # hints disabled (this example is intented to fetch logs only from specific pods with conditional autodiscover templates)
          hints:
            enabled: false
            default_config:
              enabled: false
          # Option2: With static templates we can configure filebeat inputs based on pods metadata
          templates:
            - condition:
                equals:
                  kubernetes.labels.purpose: "demonstrate-command-json"
              config:
                - type: container
                  paths:
                    - /var/log/containers/*${data.kubernetes.container.id}.log
                  # This won't work
                  #json.keys_under_root: false
                  #json.add_error_key: true
                  #json.message_key: message
                  # This works
                  tags: ["customformat"]
                  processors:
                    - decode_json_fields:
                        fields: ["message"]
                        process_array: false
                        max_depth: 1
                        target: ""
                        overwrite_keys: true
                        add_error_key: true
            - condition:
                equals:
                  kubernetes.labels.purpose: "demonstrate-command-text"
              config:
                - type: container
                  paths:
                    - /var/log/containers/*${data.kubernetes.container.id}.log
                  tags: ["customformat"]
                  pipeline: custom-text-example
    processors:
    - add_cloud_metadata: {}
    - add_host_metadata: {}
    logging.json: true
  daemonSet:
    podTemplate:
      metadata:
        labels:
          # allow metricbeat based monitoring of this beat
          mb_collection_enabled: "true"
      spec:
        serviceAccountName: filebeat
        automountServiceAccountToken: true
        terminationGracePeriodSeconds: 30
        # dnsPolicy: ClusterFirstWithHostNet
        # hostNetwork: true # Allows to provide richer host metadata
        securityContext:
          runAsUser: 0
          # If using Red Hat OpenShift uncomment this:
          #privileged: true
        containers:
        - name: filebeat
          env:
          # Needed when hostNetwork is false
          - name: NODE_NAME
            valueFrom:
              fieldRef:
                fieldPath: spec.nodeName
          ports:
          - containerPort: 5066
            name: monitoring
            protocol: TCP
          volumeMounts:
          - name: varlibdockercontainers
            mountPath: /var/lib/docker/containers
            readOnly: true
          - name: varlog
            mountPath: /var/log
            readOnly: true
        volumes:
        - name: varlibdockercontainers
          hostPath:
            path: /var/lib/docker/containers
        - name: varlog
          hostPath:
            path: /var/log