Populate logging.googleapis.com/instrumentation_source for app inte…

…grations (#627) * Use modify fields to set severity and instrumentation_source label * Add map_values_exclusive property to modify_fields Co-authored-by: Francisco Valente <[email protected]> Co-authored-by: Ling Huang <[email protected]>
GoogleCloudPlatform · Jun 9, 2022 · 7d8d128 · 7d8d128
1 parent dd16b52
commit 7d8d128
Show file tree

Hide file tree

Showing 110 changed files with 3,331 additions and 3,533 deletions.
diff --git a/apps/active_directory_ds.go b/apps/active_directory_ds.go
@@ -63,7 +63,14 @@ func (r LoggingReceiverActiveDirectoryDS) Components(tag string) []fluentbit.Com
 		Channels: []string{"Directory Service", "Active Directory Web Services"},
 	}
 
-	return l.Components(tag)
+	c := append(l.Components(tag),
+		confgenerator.LoggingProcessorModifyFields{
+			Fields: map[string]*confgenerator.ModifyField{
+				InstrumentationSourceLabel: instrumentationSourceValue(r.Type()),
+			},
+		}.Components(tag, "active_directory_ds")...,
+	)
+	return c
 }
 
 func init() {

diff --git a/apps/apache.go b/apps/apache.go
@@ -93,26 +93,33 @@ func (p LoggingProcessorApacheError) Components(tag string, uid string) []fluent
 
 	// Log levels documented: https://httpd.apache.org/docs/2.4/mod/core.html#loglevel
 	c = append(c,
-		fluentbit.TranslationComponents(tag, "level", "logging.googleapis.com/severity", false,
-			[]struct{ SrcVal, DestVal string }{
-				{"emerg", "EMERGENCY"},
-				{"alert", "ALERT"},
-				{"crit", "CRITICAL"},
-				{"error", "ERROR"},
-				{"warn", "WARNING"},
-				{"notice", "NOTICE"},
-				{"info", "INFO"},
-				{"debug", "DEBUG"},
-				{"trace1", "DEBUG"},
-				{"trace2", "DEBUG"},
-				{"trace3", "DEBUG"},
-				{"trace4", "DEBUG"},
-				{"trace5", "DEBUG"},
-				{"trace6", "DEBUG"},
-				{"trace7", "DEBUG"},
-				{"trace8", "DEBUG"},
+		confgenerator.LoggingProcessorModifyFields{
+			Fields: map[string]*confgenerator.ModifyField{
+				"severity": {
+					CopyFrom: "jsonPayload.level",
+					MapValues: map[string]string{
+						"emerg":  "EMERGENCY",
+						"alert":  "ALERT",
+						"crit":   "CRITICAL",
+						"error":  "ERROR",
+						"warn":   "WARNING",
+						"notice": "NOTICE",
+						"info":   "INFO",
+						"debug":  "DEBUG",
+						"trace1": "DEBUG",
+						"trace2": "DEBUG",
+						"trace3": "DEBUG",
+						"trace4": "DEBUG",
+						"trace5": "DEBUG",
+						"trace6": "DEBUG",
+						"trace7": "DEBUG",
+						"trace8": "DEBUG",
+					},
+					MapValuesExclusive: true,
+				},
+				InstrumentationSourceLabel: instrumentationSourceValue(p.Type()),
 			},
-		)...,
+		}.Components(tag, uid)...,
 	)
 
 	return c
@@ -123,7 +130,7 @@ type LoggingProcessorApacheAccess struct {
 }
 
 func (p LoggingProcessorApacheAccess) Components(tag string, uid string) []fluentbit.Component {
-	return genericAccessLogParser(tag, uid)
+	return genericAccessLogParser(p.Type(), tag, uid)
 }
 
 func (LoggingProcessorApacheAccess) Type() string {

diff --git a/apps/cassandra.go b/apps/cassandra.go
@@ -63,7 +63,7 @@ func (LoggingProcessorCassandraSystem) Type() string {
 }
 
 func (p LoggingProcessorCassandraSystem) Components(tag string, uid string) []fluentbit.Component {
-	return javaLogParsingComponents(tag, uid)
+	return javaLogParsingComponents(p.Type(), tag, uid)
 }
 
 type LoggingProcessorCassandraDebug struct {
@@ -75,10 +75,10 @@ func (LoggingProcessorCassandraDebug) Type() string {
 }
 
 func (p LoggingProcessorCassandraDebug) Components(tag string, uid string) []fluentbit.Component {
-	return javaLogParsingComponents(tag, uid)
+	return javaLogParsingComponents(p.Type(), tag, uid)
 }
 
-func javaLogParsingComponents(tag string, uid string) []fluentbit.Component {
+func javaLogParsingComponents(processorType, tag, uid string) []fluentbit.Component {
 	c := confgenerator.LoggingProcessorParseMultilineRegex{
 		LoggingProcessorParseRegexComplex: confgenerator.LoggingProcessorParseRegexComplex{
 			Parsers: []confgenerator.RegexParser{
@@ -119,15 +119,22 @@ func javaLogParsingComponents(tag string, uid string) []fluentbit.Component {
 	// Best documentation found for log levels:
 	// https://docs.datastax.com/en/cassandra-oss/3.0/cassandra/configuration/configLoggingLevels.html#Loglevels
 	c = append(c,
-		fluentbit.TranslationComponents(tag, "level", "logging.googleapis.com/severity", false,
-			[]struct{ SrcVal, DestVal string }{
-				{"TRACE", "TRACE"},
-				{"DEBUG", "DEBUG"},
-				{"INFO", "INFO"},
-				{"ERROR", "ERROR"},
-				{"WARN", "WARNING"},
+		confgenerator.LoggingProcessorModifyFields{
+			Fields: map[string]*confgenerator.ModifyField{
+				"severity": {
+					CopyFrom: "jsonPayload.level",
+					MapValues: map[string]string{
+						"TRACE": "TRACE",
+						"DEBUG": "DEBUG",
+						"INFO":  "INFO",
+						"ERROR": "ERROR",
+						"WARN":  "WARNING",
+					},
+					MapValuesExclusive: true,
+				},
+				InstrumentationSourceLabel: instrumentationSourceValue(processorType),
 			},
-		)...,
+		}.Components(tag, uid)...,
 	)
 
 	return c
@@ -178,6 +185,12 @@ func (p LoggingProcessorCassandraGC) Components(tag string, uid string) []fluent
 		},
 	}.Components(tag, uid)
 
+	c = append(c, confgenerator.LoggingProcessorModifyFields{
+		Fields: map[string]*confgenerator.ModifyField{
+			InstrumentationSourceLabel: instrumentationSourceValue(p.Type()),
+		},
+	}.Components(tag, uid)...)
+
 	return c
 }
 

diff --git a/apps/common_logging_processors.go b/apps/common_logging_processors.go
@@ -21,7 +21,16 @@ import (
 	"github.com/GoogleCloudPlatform/ops-agent/confgenerator/fluentbit"
 )
 
-func genericAccessLogParser(tag string, uid string) []fluentbit.Component {
+const InstrumentationSourceLabel = confgenerator.InstrumentationSourceLabel
+
+func instrumentationSourceValue(processorType string) *confgenerator.ModifyField {
+	val := fmt.Sprintf("agent.googleapis.com/%s", processorType)
+	return &confgenerator.ModifyField{
+		StaticValue: &val,
+	}
+}
+
+func genericAccessLogParser(processorType, tag, uid string) []fluentbit.Component {
 	c := confgenerator.LoggingProcessorParseRegex{
 		// Documentation:
 		// https://httpd.apache.org/docs/current/logs.html#accesslog
@@ -39,35 +48,41 @@ func genericAccessLogParser(tag string, uid string) []fluentbit.Component {
 			},
 		},
 	}.Components(tag, uid)
+	mf := confgenerator.LoggingProcessorModifyFields{
+		Fields: map[string]*confgenerator.ModifyField{
+			InstrumentationSourceLabel: instrumentationSourceValue(processorType),
+		},
+	}
 	// apache/nginx/varnish logs "-" when a field does not have a value. Remove the field entirely when this happens.
 	for _, field := range []string{
-		"http_request_remoteIp",
-		"host",
-		"user",
-		"http_request_responseSize",
-		"http_request_referer",
+		"jsonPayload.host",
+		"jsonPayload.user",
 	} {
-		c = append(c, fluentbit.Component{
-			Kind: "FILTER",
-			Config: map[string]string{
-				"Name":      "modify",
-				"Match":     tag,
-				"Condition": fmt.Sprintf("Key_Value_Equals %s -", field),
-				"Remove":    field,
-			},
-		})
+		mf.Fields[field] = &confgenerator.ModifyField{
+			OmitIf: fmt.Sprintf(`%s = "-"`, field),
+		}
 	}
 	// Generate the httpRequest structure.
-	c = append(c, fluentbit.Component{
-		Kind: "FILTER",
-		Config: map[string]string{
-			"Name":          "nest",
-			"Match":         tag,
-			"Operation":     "nest",
-			"Wildcard":      "http_request_*",
-			"Nest_under":    confgenerator.HttpRequestKey,
-			"Remove_prefix": "http_request_",
-		},
-	})
+	for _, field := range []string{
+		"remoteIp",
+		"requestMethod",
+		"requestUrl",
+		"protocol",
+		"status",
+		"responseSize",
+		"referer",
+		"userAgent",
+	} {
+		dest := fmt.Sprintf("httpRequest.%s", field)
+		src := fmt.Sprintf("jsonPayload.http_request_%s", field)
+		mf.Fields[dest] = &confgenerator.ModifyField{
+			MoveFrom: src,
+		}
+		if field == "referer" {
+			mf.Fields[dest].OmitIf = fmt.Sprintf(`%s = "-"`, src)
+		}
+	}
+
+	c = append(c, mf.Components(tag, uid)...)
 	return c
 }
diff --git a/apps/couchdb.go b/apps/couchdb.go
@@ -15,6 +15,8 @@
 package apps
 
 import (
+	"fmt"
+
 	"github.com/GoogleCloudPlatform/ops-agent/confgenerator"
 	"github.com/GoogleCloudPlatform/ops-agent/confgenerator/fluentbit"
 	"github.com/GoogleCloudPlatform/ops-agent/confgenerator/otel"
@@ -104,37 +106,46 @@ func (p LoggingProcessorCouchdb) Components(tag string, uid string) []fluentbit.
 		},
 	}.Components(tag, uid)
 
-	// Generate the httpRequest structure.
-	c = append(c, fluentbit.Component{
-		Kind: "FILTER",
-		Config: map[string]string{
-			"Name":          "nest",
-			"Match":         tag,
-			"Operation":     "nest",
-			"Wildcard":      "http_request_*",
-			"Nest_under":    confgenerator.HttpRequestKey,
-			"Remove_prefix": "http_request_",
+	fields := map[string]*confgenerator.ModifyField{
+		"severity": {
+			CopyFrom: "jsonPayload.level",
+			MapValues: map[string]string{
+				"emerg":     "EMERGENCY",
+				"emergency": "EMERGENCY",
+				"alert":     "ALERT",
+				"crit":      "CRITICAL",
+				"critical":  "CRITICAL",
+				"error":     "ERROR",
+				"err":       "ERROR",
+				"warn":      "WARNING",
+				"warning":   "WARNING",
+				"notice":    "NOTICE",
+				"info":      "INFO",
+				"debug":     "DEBUG",
+			},
+			MapValuesExclusive: true,
 		},
-	})
+		InstrumentationSourceLabel: instrumentationSourceValue(p.Type()),
+	}
+
+	// Generate the httpRequest structure.
+	for _, field := range []string{
+		"serverIp",
+		"remoteIp",
+		"requestMethod",
+		"status",
+		"responseSize",
+	} {
+		fields[fmt.Sprintf("httpRequest.%s", field)] = &confgenerator.ModifyField{
+			MoveFrom: fmt.Sprintf("jsonPayload.http_request_%s", field),
+		}
+	}
 
 	// Log levels documented: https://docs.couchdb.org/en/stable/config/logging.html#log/level
 	c = append(c,
-		fluentbit.TranslationComponents(tag, "level", "logging.googleapis.com/severity", true,
-			[]struct{ SrcVal, DestVal string }{
-				{"emerg", "EMERGENCY"},
-				{"emergency", "EMERGENCY"},
-				{"alert", "ALERT"},
-				{"crit", "CRITICAL"},
-				{"critical", "CRITICAL"},
-				{"error", "ERROR"},
-				{"err", "ERROR"},
-				{"warn", "WARNING"},
-				{"warning", "WARNING"},
-				{"notice", "NOTICE"},
-				{"info", "INFO"},
-				{"debug", "DEBUG"},
-			},
-		)...,
+		confgenerator.LoggingProcessorModifyFields{
+			Fields: fields,
+		}.Components(tag, uid)...,
 	)
 	return c
 }

diff --git a/apps/elasticsearch.go b/apps/elasticsearch.go
@@ -125,20 +125,25 @@ func (p LoggingProcessorElasticsearchJson) Components(tag, uid string) []fluentb
 }
 
 func (p LoggingProcessorElasticsearchJson) severityParser(tag, uid string) []fluentbit.Component {
-	severityKey := "logging.googleapis.com/severity"
-	return fluentbit.TranslationComponents(tag, "level", severityKey, true, []struct {
-		SrcVal  string
-		DestVal string
-	}{
-		{"TRACE", "DEBUG"},
-		{"DEBUG", "DEBUG"},
-		{"INFO", "INFO"},
-		{"WARN", "WARNING"},
-		{"DEPRECATION", "WARNING"},
-		{"ERROR", "ERROR"},
-		{"CRITICAL", "ERROR"},
-		{"FATAL", "FATAL"},
-	})
+	return confgenerator.LoggingProcessorModifyFields{
+		Fields: map[string]*confgenerator.ModifyField{
+			"severity": {
+				CopyFrom: "jsonPayload.level",
+				MapValues: map[string]string{
+					"TRACE":       "DEBUG",
+					"DEBUG":       "DEBUG",
+					"INFO":        "INFO",
+					"WARN":        "WARNING",
+					"DEPRECATION": "WARNING",
+					"ERROR":       "ERROR",
+					"CRITICAL":    "ERROR",
+					"FATAL":       "FATAL",
+				},
+				MapValuesExclusive: true,
+			},
+			InstrumentationSourceLabel: instrumentationSourceValue(p.Type()),
+		},
+	}.Components(tag, uid)
 }
 
 func (p LoggingProcessorElasticsearchJson) nestingProcessors(tag, uid string) []fluentbit.Component {
@@ -200,7 +205,13 @@ func (p LoggingProcessorElasticsearchGC) Components(tag, uid string) []fluentbit
 	}
 
 	c = append(c, regexParser.Components(tag, uid)...)
-
+	c = append(c,
+		confgenerator.LoggingProcessorModifyFields{
+			Fields: map[string]*confgenerator.ModifyField{
+				InstrumentationSourceLabel: instrumentationSourceValue(p.Type()),
+			},
+		}.Components(tag, uid)...,
+	)
 	return c
 }