try

configs/pilot_log.conf

@@ -38,7 +38,7 @@ filter {
   }
 
   mutate {
-    remove_field => [ "version", "@version", "[extra][logger_name]", "[extra][logstash_async_version]", "[extra][process_name]", "[extra][thread_name]"]
+    remove_field => [ "message_kv", "version", "@version", "[extra][logger_name]", "[extra][logstash_async_version]", "[extra][process_name]", "[extra][thread_name]"]
   }
 # extra.path has this random thing that should be shortened /scratch/atlas_MTFrHlIS/pilot3/pilot/util/realtimelogger.py
 }
@@ -53,18 +53,17 @@ output {
     path => "/var/log/logstash/all.txt"
   }
 
-  if "_jsonparsefailure" in [tags] {
-    file {
-      path => "/var/log/logstash/jsonparsefailure.txt"
-    }
-  } else {
-    elasticsearch {
-      hosts => "atlas-kibana.mwt2.org"
-      ssl => true
-      index => 'pilot_log'
-      user => "uc_logstash_indexer"
-      password => "${LOGSTASH_PWD}"
-    }
+
+  elasticsearch {
+    hosts => "atlas-kibana.mwt2.org"
+    ssl => true
+    action => "create"
+    # data_stream => "true"
+    # data_stream_type => "logs"
+    index => 'pilot_log'
+    user => "uc_logstash_indexer"
+    password => "${LOGSTASH_PWD}"
   }
 
+
 }