adding catchall

Dockerfile.ps_collector

@@ -18,6 +18,7 @@ COPY logstash.yml /usr/share/logstash/config/logstash.yml
 
 # COPY configs/ps-collector.conf /usr/share/logstash/configs/ps-collector.conf
 COPY configs/ps-status.conf /usr/share/logstash/configs/ps-status.conf
+COPY configs/ps-extra.conf /usr/share/logstash/configs/ps-extra.conf
 COPY configs/ps-meta.conf /usr/share/logstash/configs/ps-meta.conf
 COPY configs/ps-latencybg.conf /usr/share/logstash/configs/ps-latencybg.conf
 COPY configs/ps-throughput.conf /usr/share/logstash/configs/ps-throughput.conf

ToDo.md

@@ -4,6 +4,5 @@
 
 * make logstash parser for "meta", "status".
 
-* add more sites for stress testing.
 * add scripted variables.
 * add trace hash calculation.

configs/es-output.conf

@@ -16,25 +16,27 @@ output {
       path => "/var/log/logstash/ruby_exception.json"
     }
   }  
-  # else if "_bad_document" in [tags] {
-  #   file {
-  #     path => "/var/log/logstash/bad_document.json"
-  #   }
-  # }
-  # else {
+  else if "_extra" in [tags] {
+    file {
+      path => "/var/log/logstash/extra.json"
+    }
+  }  
+  else {
+
   #   file{
   #     path => "/var/log/logstash/correct_%{[@metadata][type]}.json"
   #   }
-  # }
 
-  elasticsearch {
-    hosts => "atlas-kibana.mwt2.org"
-    data_stream => false
-    ssl => true
-    index => "%{[@metadata][type]}"
-    user => "uc_logstash_indexer"
-    password => "${LOGSTASH_PWD}"
-    document_id => "%{[@metadata][id]}"
+    elasticsearch {
+      hosts => "atlas-kibana.mwt2.org"
+      data_stream => false
+      ssl => true
+      index => "%{[@metadata][type]}"
+      user => "uc_logstash_indexer"
+      password => "${LOGSTASH_PWD}"
+      document_id => "%{[@metadata][id]}"
+    }
+
   }
 
 }

configs/ps-extra.conf

@@ -0,0 +1,13 @@
+input { pipeline { address => "ps-extra" } }
+
+filter {
+
+  mutate{
+    add_field => { "tags" => "_extra" } 
+  }
+
+}
+
+output {
+  pipeline { send_to => ["es-output"] }
+}

configs/ps-intake.conf

@@ -100,5 +100,8 @@ output {
   }
   else if [type]=="meta" {
     pipeline { send_to => ["ps-meta"] } 
+  } 
+  else {
+    pipeline { send_to => ["ps-extra"] }
   }
 }

pipelines/ps-collector.yml

@@ -1,3 +1,6 @@
+- pipeline.id: ps-extra
+  path.config: "/usr/share/logstash/configs/ps-extra.conf"
+  pipeline.workers: 1
 - pipeline.id: ps-trace
   path.config: "/usr/share/logstash/configs/ps-trace.conf"
   pipeline.workers: 1