adding it in a separate pipeline

ATLAS-Analytics · Jan 23, 2024 · e49f24b · e49f24b
1 parent da5a600
commit e49f24b
Show file tree

Hide file tree

Showing 4 changed files with 27 additions and 10 deletions.
diff --git a/Dockerfile.ps_collector b/Dockerfile.ps_collector
@@ -26,6 +26,7 @@ COPY configs/ps-throughput.conf /usr/share/logstash/configs/ps-throughput.conf
 COPY configs/ps-trace.conf /usr/share/logstash/configs/ps-trace.conf
 COPY configs/ps-intake.conf /usr/share/logstash/configs/ps-intake.conf
 COPY configs/es-output.conf /usr/share/logstash/configs/es-output.conf
+COPY configs/nebraska.conf /usr/share/logstash/configs/nebraska.conf
 COPY configs/ps-debug.conf /usr/share/logstash/configs/ps-debug.conf
 
 COPY pipelines/ps-collector.yml /usr/share/logstash/config/pipelines.yml

diff --git a/configs/es-output.conf b/configs/es-output.conf
@@ -37,16 +37,7 @@ output {
       document_id => "%{[@metadata][id]}"
     }
 
-    rabbitmq {
-      host => "clever-turkey.rmq.cloudamqp.com"
-      user => "uc-nma"
-      password => "WT5hc7NLVp3mPC4it"
-      vhost => "osg-nma"
-      exchange => "uc-osg-nma-push"
-      exchange_type => "fanout"
-    }
-
-
+    pipeline { send_to => ["nebraska"] }
 
   }
 

diff --git a/configs/nebraska.conf b/configs/nebraska.conf
@@ -0,0 +1,22 @@
+input { 
+  pipeline { address => "nebraska" } 
+}
+
+filter {
+  mutate{ 
+    add_field => { "_index" => "[@metadata][type]" } 
+  }
+}
+
+output {
+
+    rabbitmq {
+      host => "clever-turkey.rmq.cloudamqp.com"
+      user => "uc-nma"
+      password => "WT5hc7NLVp3mPC4it"
+      vhost => "osg-nma"
+      exchange => "uc-osg-nma-push"
+      exchange_type => "fanout"
+    }
+
+}
diff --git a/pipelines/ps-collector.yml b/pipelines/ps-collector.yml
@@ -26,6 +26,9 @@
 - pipeline.id: es-output
   path.config: "/usr/share/logstash/configs/es-output.conf"
   pipeline.workers: 1
+- pipeline.id: nebraska
+  path.config: "/usr/share/logstash/configs/nebraska.conf"
+  pipeline.workers: 1
 # - pipeline.id: ps-debug
 #   path.config: "/usr/share/logstash/configs/ps-debug.conf"
 #   pipeline.workers: 1