From 5486c39a6f335839948aa718ceeb90ef5fea6147 Mon Sep 17 00:00:00 2001
From: Ilija Vukotic <ivukotic@uchicago.edu>
Date: Fri, 2 Aug 2024 14:43:36 -0500
Subject: [PATCH] move to ds

---
 Dockerfile.ps_collector    |  1 +
 configs/es-output-ds.conf  | 44 ++++++++++++++++++++++++++++++++++++++
 configs/ps-latencybg.conf  |  8 +++++--
 pipelines/ps-collector.yml |  3 +++
 4 files changed, 54 insertions(+), 2 deletions(-)
 create mode 100644 configs/es-output-ds.conf

diff --git a/Dockerfile.ps_collector b/Dockerfile.ps_collector
index 095558e..9f59a4c 100644
--- a/Dockerfile.ps_collector
+++ b/Dockerfile.ps_collector
@@ -26,6 +26,7 @@ COPY configs/ps-throughput.conf /usr/share/logstash/configs/ps-throughput.conf
 COPY configs/ps-trace.conf /usr/share/logstash/configs/ps-trace.conf
 COPY configs/ps-intake.conf /usr/share/logstash/configs/ps-intake.conf
 COPY configs/es-output.conf /usr/share/logstash/configs/es-output.conf
+COPY configs/es-output-ds.conf /usr/share/logstash/configs/es-output-ds.conf
 COPY configs/nebraska.conf /usr/share/logstash/configs/nebraska.conf
 COPY configs/ps-debug.conf /usr/share/logstash/configs/ps-debug.conf
 
diff --git a/configs/es-output-ds.conf b/configs/es-output-ds.conf
new file mode 100644
index 0000000..3049d87
--- /dev/null
+++ b/configs/es-output-ds.conf
@@ -0,0 +1,44 @@
+input { pipeline { address => "es-output" } }
+
+output {
+
+  # stdout {
+  #   codec => json
+  # }
+  
+  if "_jsonparsefailure" in [tags] {
+    file {
+      path => "/var/log/logstash/json_parse_failure.txt"
+    }
+  }
+  else if "_rubyexception" in [tags] {
+    file {
+      path => "/var/log/logstash/ruby_exception.json"
+    }
+  }  
+  else if "_extra" in [tags] {
+    file {
+      path => "/var/log/logstash/extra.json"
+    }
+  }  
+  else {
+
+  #   file{
+  #     path => "/var/log/logstash/correct_%{[@metadata][type]}.json"
+  #   }
+
+    elasticsearch {
+      hosts => "atlas-kibana.mwt2.org"
+      data_stream => true
+      action => "create"
+      ssl => true
+      index => "%{[@metadata][type]}"
+      user => "uc_logstash_indexer"
+      password => "${LOGSTASH_PWD}"
+      document_id => "%{[@metadata][id]}"
+    }
+
+
+  }
+
+}
diff --git a/configs/ps-latencybg.conf b/configs/ps-latencybg.conf
index aee8e1e..a4053e0 100644
--- a/configs/ps-latencybg.conf
+++ b/configs/ps-latencybg.conf
@@ -49,6 +49,10 @@ filter {
 }
 
 output {
-  pipeline { send_to => ["es-output"] }
-  pipeline { send_to => ["nebraska"] }
+  if [@metadata][type] == "ps_packetloss_write" {
+    pipeline { send_to => ["es-output-ds"] }
+  } else {
+    pipeline { send_to => ["es-output"] }
+    pipeline { send_to => ["nebraska"] }
+  }
 }
diff --git a/pipelines/ps-collector.yml b/pipelines/ps-collector.yml
index c0c1f3b..36be2b1 100644
--- a/pipelines/ps-collector.yml
+++ b/pipelines/ps-collector.yml
@@ -26,6 +26,9 @@
 - pipeline.id: es-output
   path.config: "/usr/share/logstash/configs/es-output.conf"
   pipeline.workers: 1
+- pipeline.id: es-output-ds
+  path.config: "/usr/share/logstash/configs/es-output-ds.conf"
+  pipeline.workers: 1
 - pipeline.id: nebraska
   path.config: "/usr/share/logstash/configs/nebraska.conf"
   pipeline.workers: 1