diff --git a/Dockerfile.temp b/Dockerfile.temp index 27cf3cb..cd0af13 100644 --- a/Dockerfile.temp +++ b/Dockerfile.temp @@ -11,4 +11,5 @@ RUN mkdir /usr/share/logstash/configs COPY logstash.yml /usr/share/logstash/config/logstash.yml COPY configs/temps.conf /usr/share/logstash/configs/temps.conf -COPY pipelines/temps.yml /usr/share/logstash/config/pipelines.yml \ No newline at end of file +COPY configs/sidewalk.conf /usr/share/logstash/configs/sidewalk.conf +COPY pipelines/pips.yml /usr/share/logstash/config/pipelines.yml \ No newline at end of file diff --git a/configs/servicex.conf b/configs/servicex.conf deleted file mode 100644 index 1be2dc0..0000000 --- a/configs/servicex.conf +++ /dev/null @@ -1,36 +0,0 @@ -input { - http { - # host => "0.0.0.0" - # port => "80" - # codec => json - } -} - -filter { - json { - source => "message" - remove_field => [ "message", "logger_name", "port", "type" ] - } -} - -output { - - # stdout { - # codec => rubydebug - # } - - if "_jsonparsefailure" in [tags] { - file { - path => "/var/log/logstash/json_parse_failure.txt" - } - } - - elasticsearch { - hosts => "atlas-kibana.mwt2.org" - ssl => true - index => 'servicex-%{+YYYY-MM}' - user => "uc_logstash_indexer" - password => "${LOGSTASH_PWD}" - } - -} \ No newline at end of file diff --git a/configs/sidewalk.conf b/configs/sidewalk.conf new file mode 100644 index 0000000..e1f0118 --- /dev/null +++ b/configs/sidewalk.conf @@ -0,0 +1,48 @@ +input { + http { + # host => "0.0.0.0" + # port => "80" + } +} + +filter { + json { + source => "message" + } + + if ("tPmAT5Ab3j7F9" not in [api_key]) { + drop { } + } + + split { + field => "data" + } + + mutate { + remove_field => [ "headers", "api_key", "host", "port", "message" ] + } +} + +output { + + stdout { + codec => rubydebug + } + + if "_jsonparsefailure" in [tags] { + + file { + path => "/var/log/logstash/_jsonparsefailure.txt" + } + + } else { + elasticsearch { + hosts => "atlas-kibana.mwt2.org" + ssl => true + index => 'sidewalk' + user => "uc_logstash_indexer" + password => "${LOGSTASH_PWD}" + } + } + +} diff --git a/configs/temps.conf b/configs/temps.conf index e416e6e..f458943 100644 --- a/configs/temps.conf +++ b/configs/temps.conf @@ -14,12 +14,6 @@ filter { drop { } } - if ([data]) { - split { - field => "data" - } - } - mutate { remove_field => [ "headers", "api_key", "host", "port", "message" ] } diff --git a/pipelines/temps.yml b/pipelines/pips.yml similarity index 100% rename from pipelines/temps.yml rename to pipelines/pips.yml