From a7a2d6e733f4391be8dd0be57f1bf5aa769c1a94 Mon Sep 17 00:00:00 2001
From: Ilija Vukotic <ivukotic@uchicago.edu>
Date: Fri, 4 Oct 2024 13:12:12 -0500
Subject: [PATCH] Refactor Logstash configuration for pilot_log collector:
 Remove unnecessary fields and fix JSON parsing error

---
 configs/pilot_log.conf | 16 +---------------
 1 file changed, 1 insertion(+), 15 deletions(-)

diff --git a/configs/pilot_log.conf b/configs/pilot_log.conf
index 9d30079..1e52880 100644
--- a/configs/pilot_log.conf
+++ b/configs/pilot_log.conf
@@ -23,25 +23,11 @@ filter {
   mutate {
     remove_field => [ "version" ]
     remove_field => [ "@version" ]
-    # remove_field => [ "host" ]
+    remove_field => ["extra"][ "logger_name" ]
   }
 
 }
 
-[2024-10-04T17:59:31,409][WARN ][logstash.filters.json    ][pilot_log]
-[9fc6fe456ad9f7522f2fb0236af35eb23e44fbbb9e13207e7d0707303e3ded3a] 
-Error parsing json {:source=>"message", 
-:raw=>"{
-'TaskID': '41547794', 
-'PandaJobID': '6356703513', 
-'Harvester_WorkerID': '628885473', 
-'Harvester_ID': 'CERN_central_A', 
-'RequestID': 245061, 
-'PilotTimeStamp': 1728064763.2951524, 
-'message': 'AnalysisBase_VHbb::exe... INFO    Event number = 78400'
-}", :exception=>
-#<LogStash::Json::ParserError: Unexpected character (''' (code 39)): was expecting double-quote to start field name
-
 output {
 
   #  stdout {