fix: tests, allow lines skipped counter to be nil

grafana · Nov 18, 2024 · 54356c4 · 54356c4
1 parent 8de23a8
commit 54356c4
Show file tree

Hide file tree

Showing 3 changed files with 24 additions and 9 deletions.
diff --git a/pkg/pattern/drain/drain.go b/pkg/pattern/drain/drain.go
@@ -260,7 +260,7 @@ func (d *Drain) train(tokens []string, state interface{}, ts int64) *LogCluster
 }
 
 func (d *Drain) TrainPattern(content string, samples []*logproto.PatternSample) *LogCluster {
-	tokens, state := d.tokenizer.Tokenize(content, d.tokens, d.state)
+	tokens, state := d.tokenizer.Tokenize(content, d.tokens, d.state, d.metrics.LinesSkipped)
 	matchCluster := d.treeSearch(d.rootNode, tokens, d.config.SimTh, true)
 	// Match no existing log cluster
 	if matchCluster == nil {

diff --git a/pkg/pattern/drain/line_tokenizer.go b/pkg/pattern/drain/line_tokenizer.go
@@ -57,8 +57,16 @@ func newPunctuationTokenizer(maxLineLength int) *punctuationTokenizer {
 	}
 }
 
-func (p *punctuationTokenizer) Tokenize(line string, tokens []string, state interface{}, linesDropped *prometheus.CounterVec) ([]string, interface{}) {
+func (p *punctuationTokenizer) Tokenize(
+	line string,
+	tokens []string,
+	state interface{},
+	linesDropped *prometheus.CounterVec,
+) ([]string, interface{}) {
 	if len(line) > p.maxLineLength {
+		if linesDropped != nil {
+			linesDropped.WithLabelValues(LineTooLong).Inc()
+		}
 		return nil, nil
 	}
 
@@ -132,7 +140,12 @@ func (p *punctuationTokenizer) Clone(tokens []string, state interface{}) ([]stri
 
 type splittingTokenizer struct{}
 
-func (splittingTokenizer) Tokenize(line string, tokens []string, state interface{}) ([]string, interface{}) {
+func (splittingTokenizer) Tokenize(
+	line string,
+	tokens []string,
+	state interface{},
+	_ *prometheus.CounterVec,
+) ([]string, interface{}) {
 	numEquals := strings.Count(line, "=")
 	numColons := strings.Count(line, ":")
 	numSpaces := strings.Count(line, " ")
@@ -217,7 +230,9 @@ func (t *logfmtTokenizer) Tokenize(
 	linesDropped *prometheus.CounterVec,
 ) ([]string, interface{}) {
 	if len(line) > t.maxLineLength {
-		linesDropped.WithLabelValues(LineTooLong).Inc()
+		if linesDropped != nil {
+			linesDropped.WithLabelValues(LineTooLong).Inc()
+		}
 		return nil, nil
 	}
 

diff --git a/pkg/pattern/drain/line_tokenizer_test.go b/pkg/pattern/drain/line_tokenizer_test.go
@@ -143,7 +143,7 @@ func TestTokenizer_Tokenize(t *testing.T) {
 	for _, tt := range tests {
 		for _, tc := range testCases {
 			t.Run(tt.name+":"+tc.name, func(t *testing.T) {
-				got, _ := tt.tokenizer.Tokenize(tc.line, nil, nil)
+				got, _ := tt.tokenizer.Tokenize(tc.line, nil, nil, nil)
 				require.Equal(t, tc.want[tt.name], got)
 			})
 		}
@@ -168,7 +168,7 @@ func TestTokenizer_TokenizeAndJoin(t *testing.T) {
 	for _, tt := range tests {
 		for _, tc := range testCases {
 			t.Run(tt.name+":"+tc.name, func(t *testing.T) {
-				got := tt.tokenizer.Join(tt.tokenizer.Tokenize(tc.line, nil, nil))
+				got := tt.tokenizer.Join(tt.tokenizer.Tokenize(tc.line, nil, nil, nil))
 				require.Equal(t, tc.line, got)
 			})
 		}
@@ -184,7 +184,7 @@ func BenchmarkSplittingTokenizer(b *testing.B) {
 			b.ResetTimer()
 			b.ReportAllocs()
 			for i := 0; i < b.N; i++ {
-				tokenizer.Tokenize(tc.line, nil, nil)
+				tokenizer.Tokenize(tc.line, nil, nil, nil)
 			}
 		})
 	}
@@ -231,7 +231,7 @@ func TestLogFmtTokenizer(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			got, _ := tokenizer.Tokenize(tt.line, nil, nil)
+			got, _ := tokenizer.Tokenize(tt.line, nil, nil, nil)
 			require.Equal(t, tt.want, got)
 		})
 	}
@@ -330,7 +330,7 @@ func TestJsonTokenizer(t *testing.T) {
 
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
-			got, state := tokenizer.Tokenize(tt.line, nil, nil)
+			got, state := tokenizer.Tokenize(tt.line, nil, nil, nil)
 			require.Equal(t, tt.want, got)
 			if len(got) == len(tt.want) && len(tt.want) != 0 {
 				pattern := tokenizer.Join(got, state)