From acf416e40a378fd036ea65d760f04167699fb7f3 Mon Sep 17 00:00:00 2001 From: Martin Chodur Date: Tue, 24 Aug 2021 22:28:55 +0200 Subject: [PATCH] refactor: abstract structs event.Raw and event.Slo into interfaces Signed-off-by: Martin Chodur --- CHANGELOG.md | 11 ++ docs/modules/envoy_access_log_server.md | 2 + docs/modules/kafka_ingester.md | 2 + docs/modules/tailer.md | 2 + go.mod | 1 + go.sum | 2 + pkg/dynamic_classifier/dynamic_classifier.go | 32 ++-- .../dynamic_classifier_test.go | 75 ++++---- pkg/dynamic_classifier/matcher.go | 4 +- pkg/dynamic_classifier/matcher_test.go | 12 +- .../memory_exact_matcher.go | 10 +- pkg/dynamic_classifier/regexp_matcher.go | 14 +- .../access_log_server.go | 14 +- pkg/envoy_access_log_server/service_v3.go | 15 +- pkg/event/interfaces.go | 51 ++++++ pkg/event/raw.go | 99 +++++++---- pkg/event/slo.go | 85 ++++++--- pkg/event/slo_classification.go | 4 + pkg/event/test_utils.go | 15 ++ .../event_key_generator.go | 18 +- pkg/event_metadata_renamer/renamer.go | 25 +-- pkg/event_metadata_renamer/renamer_test.go | 20 +-- pkg/kafka_ingester/kafka_ingester.go | 29 +--- pkg/kafka_ingester/kafka_ingester_test.go | 53 +++--- pkg/metadata_classifier/metadata_cassifier.go | 33 ++-- .../metadata_cassifier_test.go | 34 ++-- pkg/pipeline/manager_test.go | 4 +- pkg/pipeline/module.go | 8 +- pkg/pipeline/module_test.go | 8 +- .../prometheus_exporter.go | 36 ++-- .../prometheus_exporter_test.go | 18 +- .../prometheus_ingester.go | 10 +- .../prometheus_ingester_test.go | 164 +++++++----------- pkg/prometheus_ingester/query_executor.go | 26 +-- pkg/relabel/relabel.go | 18 +- pkg/relabel/relabel_test.go | 26 +-- pkg/slo_event_producer/config_test.go | 4 +- pkg/slo_event_producer/event_evaluator.go | 4 +- .../event_evaluator_test.go | 24 +-- pkg/slo_event_producer/operator.go | 36 ++-- pkg/slo_event_producer/operator_test.go | 62 +++---- pkg/slo_event_producer/rule.go | 46 ++--- pkg/slo_event_producer/rule_test.go | 40 ++--- pkg/slo_event_producer/slo_event_producer.go | 16 +- .../statistical_classifier.go | 24 +-- pkg/stringmap/stringmap.go | 16 +- pkg/stringmap/stringmap_test.go | 38 ++++ pkg/tailer/tailer.go | 18 +- pkg/tailer/tailer_test.go | 10 +- 49 files changed, 720 insertions(+), 598 deletions(-) create mode 100644 pkg/event/interfaces.go create mode 100644 pkg/event/test_utils.go diff --git a/CHANGELOG.md b/CHANGELOG.md index a850b99..eb7caec 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,6 +4,17 @@ All notable changes to this project will be documented in this file. The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/), and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html). +## [Unreleased] + +### Added +- [#xxx]() Every event has a unique id set, so it can be filtered in logs. + The `kafkaIngester`, `tailer` and `envoyAccessLogServer` modules + has new config option `eventIdMetadataKey` to use value of this metadata key as the unique identifier. + (Mostly should be used with a trace id for example). + +### Changed +- _Internal:_ Event structs `Raw` and `Slo` has been abstracted into interfaces. + ## [v6.9.0] 2021-07-14 ### Added - [#60](https://github.com/seznam/slo-exporter/pull/60) New module eventMetadataRenamer diff --git a/docs/modules/envoy_access_log_server.md b/docs/modules/envoy_access_log_server.md index 2fb1659..98168d0 100644 --- a/docs/modules/envoy_access_log_server.md +++ b/docs/modules/envoy_access_log_server.md @@ -131,4 +131,6 @@ Please note that some of the keys may not be present | address: ":18090" # gracefulShutdownTimeout for the GRPC server. Please note also the existence of 'maximumGracefulShutdownDuration' global config option which is effectively an upper boundary of here-specified timeout value. gracefulShutdownTimeout: "5s" +# eventIdMetadataKey it's value will be used as a unique id for the generated event if present. +eventIdMetadataKey: ``` diff --git a/docs/modules/kafka_ingester.md b/docs/modules/kafka_ingester.md index b08e4c1..f876198 100644 --- a/docs/modules/kafka_ingester.md +++ b/docs/modules/kafka_ingester.md @@ -28,6 +28,8 @@ retentionTime: # fallbackStartOffset determines from whence the consumer group should begin consuming when it finds a partition without a committed offset. # Default: FirstOffset fallbackStartOffset: +# eventIdMetadataKey it's value will be used as a unique id for the generated event if present. +eventIdMetadataKey: ``` diff --git a/docs/modules/tailer.md b/docs/modules/tailer.md index facf6e0..b3b498e 100644 --- a/docs/modules/tailer.md +++ b/docs/modules/tailer.md @@ -35,5 +35,7 @@ positionPersistenceInterval: "2s" loglineParseRegexp: '^(?P[A-Fa-f0-9.:]{4,50}) \S+ \S+ \[(?P