Refactor Logstash configuration for pilot_log collector: Remove unnec…

…essary fields and fix JSON parsing error
ATLAS-Analytics · Oct 4, 2024 · c73b9e7 · c73b9e7
1 parent a7a2d6e
commit c73b9e7
Showing 1 changed file with 7 additions and 5 deletions.
diff --git a/configs/pilot_log.conf b/configs/pilot_log.conf
@@ -7,7 +7,7 @@ input {
 filter {
 
   mutate {
-    remove_field => [ "headers", "type" ]
+    remove_field => [ "headers", "type", "program", "host" ]
     gsub => ["message", "'", '"']
   }
 
@@ -21,11 +21,13 @@ filter {
   # ruby { path => "/usr/share/logstash/filters/reformat.rb" }
 
   mutate {
-    remove_field => [ "version" ]
-    remove_field => [ "@version" ]
-    remove_field => ["extra"][ "logger_name" ]
+    remove_field => [ "version", "@version" ]
+    # remove_field => ["extra"][ "logger_name" ]
+    # remove_field => ["extra"][ "logstash_async_version" ]
+    # remove_field => ["extra"][ "process_name" ]
+    # remove_field => ["extra"][ "thread_name" ]
   }
-
+# extra.path has this random thing that should be shortened /scratch/atlas_MTFrHlIS/pilot3/pilot/util/realtimelogger.py
 }
 
 output {