From 5e1fbead37245bf3d1145e4f13519a03d530fdbc Mon Sep 17 00:00:00 2001
From: "hanna.yurkevich" <hanna.yurkevich@altoros.com>
Date: Mon, 28 Nov 2016 11:12:28 +0300
Subject: [PATCH] Allow disabling default elasticsearch output for parser job

Fix for https://github.com/cloudfoundry-community/logsearch-boshrelease/issues/8.
Property 'logstash_parser.outputs' is set with list of logstash output plugins, by default it is set to 'elasticsearch' plugin.
Rename 'logstash_parser' elasticsearch properties to follow the same naming convention - 'logstash_parser.elasticsearch.*'.
Add logstash-templates tests to test input_and_output.conf.erb template with different configurations.
Minor reqork in test scripts - all tests are run with default rake task.
Use FileUtils.* methods in Rakefile 'cleanup' tasks.
---
 jobs/parser/spec                              | 36 +++++++++-------
 jobs/parser/templates/bin/parser_ctl          |  2 +-
 .../config/input_and_output.conf.erb          | 41 +++++++++++--------
 src/logsearch-config/Rakefile                 | 10 ++---
 src/logsearch-config/bin/test                 |  6 +--
 .../input_and_output/compile.yml              | 28 ++++++++++++-
 .../input_and_output/spec.rb                  | 28 ++++++++++++-
 .../input_and_output/test_default-config.yml  | 22 ++++++----
 .../test_default-expected.conf                |  3 +-
 .../test_enabled_debug-config.yml             | 19 +++++----
 .../test_enabled_debug-expected.conf          |  2 +-
 .../input_and_output/test_inputs-config.yml   | 18 ++++++++
 .../test_inputs-expected.conf                 | 29 +++++++++++++
 .../test_inputs_no_redis-config.yml           | 18 ++++++++
 .../test_inputs_no_redis-expected.conf        | 20 +++++++++
 .../input_and_output/test_outputs-config.yml  | 18 ++++++++
 .../test_outputs-expected.conf                | 36 ++++++++++++++++
 ...ts_elasticsearch_all_properties-config.yml | 20 +++++++++
 ...elasticsearch_all_properties-expected.conf | 25 +++++++++++
 ...asticsearch_required_properties-config.yml | 17 ++++++++
 ...icsearch_required_properties-expected.conf | 22 ++++++++++
 .../test_outputs_no_elasticsearch-config.yml  | 18 ++++++++
 ...est_outputs_no_elasticsearch-expected.conf | 24 +++++++++++
 23 files changed, 399 insertions(+), 63 deletions(-)
 create mode 100644 src/logsearch-config/test/logstash-templates/input_and_output/test_inputs-config.yml
 create mode 100644 src/logsearch-config/test/logstash-templates/input_and_output/test_inputs-expected.conf
 create mode 100644 src/logsearch-config/test/logstash-templates/input_and_output/test_inputs_no_redis-config.yml
 create mode 100644 src/logsearch-config/test/logstash-templates/input_and_output/test_inputs_no_redis-expected.conf
 create mode 100644 src/logsearch-config/test/logstash-templates/input_and_output/test_outputs-config.yml
 create mode 100644 src/logsearch-config/test/logstash-templates/input_and_output/test_outputs-expected.conf
 create mode 100644 src/logsearch-config/test/logstash-templates/input_and_output/test_outputs_elasticsearch_all_properties-config.yml
 create mode 100644 src/logsearch-config/test/logstash-templates/input_and_output/test_outputs_elasticsearch_all_properties-expected.conf
 create mode 100644 src/logsearch-config/test/logstash-templates/input_and_output/test_outputs_elasticsearch_required_properties-config.yml
 create mode 100644 src/logsearch-config/test/logstash-templates/input_and_output/test_outputs_elasticsearch_required_properties-expected.conf
 create mode 100644 src/logsearch-config/test/logstash-templates/input_and_output/test_outputs_no_elasticsearch-config.yml
 create mode 100644 src/logsearch-config/test/logstash-templates/input_and_output/test_outputs_no_elasticsearch-expected.conf

diff --git a/jobs/parser/spec b/jobs/parser/spec
index 2c6dd50a..9ed20b82 100644
--- a/jobs/parser/spec
+++ b/jobs/parser/spec
@@ -44,23 +44,38 @@ properties:
             password: c1oudbunny
     default: [ { plugin: "redis", options : {} } ]
   logstash_parser.outputs:
-    description: "The configuration to embed into the logstash outputs section"
+    description: |
+      A list of output plugins, with a hash of options for each of them. Please refer to example below.
+    example:
+      inputs:
+        - plugin: mongodb
+          options:
+            uri: 192.168.1.1
+            database: logsearch
+            collection: logs                      
+    default: [ { plugin: "elasticsearch", options: {} } ]
   logstash_parser.workers:
     description: "The number of worker threads that logstash should use (default: auto = one per CPU)"
     default: auto
-  logstash_parser.idle_flush_time:
-    description: "How frequently to flush events if the output queue is not full."
-  logstash_parser.elasticsearch_document_id:
+  logstash_parser.elasticsearch.idle_flush_time:
+    description: "How frequently to flush events if the output queue is not full." 
+  logstash_parser.elasticsearch.document_id:
     description: "Use a specific, dynamic ID rather than an auto-generated identifier."
     default: ~
-  logstash_parser.elasticsearch_index:
+  logstash_parser.elasticsearch.index:
     description: "The specific, dynamic index name to write events to."
     default: "logstash-%{+YYYY.MM.dd}"
-  logstash_parser.elasticsearch_index_type:
+  logstash_parser.elasticsearch.index_type:
     description: "The specific, dynamic index type name to write events to."
     default: "%{@type}"
-  logstash_parser.elasticsearch_routing:
+  logstash_parser.elasticsearch.routing:
     description: "The routing to be used when indexing a document."
+  logstash.elasticsearch.data_hosts:
+    description: The list of elasticsearch data node IPs
+    default: [127.0.0.1]
+  logstash.elasticsearch.flush_size:
+    description: Controls how many logs will be buffered and sent to Elasticsearch for bulk indexing
+    default: 500  
   logstash_parser.timecop.reject_greater_than_hours:
     description: "Logs with timestamps greater than this many hours in the future won't be parsed and will get tagged with fail/timecop"
     default: 1
@@ -74,13 +89,6 @@ properties:
     description: "A list of index templates that need to be present in ElasticSearch before the process starts"
     default: ["index_template"]
 
-  logstash.output.elasticsearch.data_hosts:
-    description: The list of elasticsearch data node IPs
-    default: [127.0.0.1]
-  logstash.output.elasticsearch.flush_size:
-    description: Controls how many logs will be buffered and sent to Elasticsearch for bulk indexing
-    default: 500
-
   redis.host:
     description: Redis host of queue
   redis.port:
diff --git a/jobs/parser/templates/bin/parser_ctl b/jobs/parser/templates/bin/parser_ctl
index 08c5214f..d0d008c7 100755
--- a/jobs/parser/templates/bin/parser_ctl
+++ b/jobs/parser/templates/bin/parser_ctl
@@ -8,7 +8,7 @@ source /var/vcap/jobs/parser/helpers/ctl_setup.sh 'parser'
 
 function wait_for_template {
   local template_name="$1"
-  local MASTER_URL="<%= p("logstash.output.elasticsearch.data_hosts").first %>:9200"
+  local MASTER_URL="<%= p("logstash_parser.elasticsearch.data_hosts").first %>:9200"
 
   set +e
   while true;  do
diff --git a/jobs/parser/templates/config/input_and_output.conf.erb b/jobs/parser/templates/config/input_and_output.conf.erb
index 45fc2000..3f252d07 100644
--- a/jobs/parser/templates/config/input_and_output.conf.erb
+++ b/jobs/parser/templates/config/input_and_output.conf.erb
@@ -27,22 +27,31 @@ output {
         }
     <% end %>
 
-    elasticsearch {
-        hosts => [<%= p("logstash.output.elasticsearch.data_hosts").map { |ip| "\"#{ip}:9200\"" }.join(',') %>]
-        flush_size => <%= p("logstash.output.elasticsearch.flush_size") %>
-        <% if p('logstash_parser.idle_flush_time', nil) %>
-            idle_flush_time => <%= p('logstash_parser.idle_flush_time') %>
-        <% end %>
-        <% if p('logstash_parser.elasticsearch_document_id', nil) %>
-            document_id => "<%= p('logstash_parser.elasticsearch_document_id') %>"
+
+    <% p('logstash_parser.outputs').each do | output | %>
+        <%= output['plugin'] %> {
+        <% if 'elasticsearch' == output['plugin'] %>
+            <% output['options'] = {
+                "hosts" => [ p('logstash_parser.elasticsearch.data_hosts').map { |ip| "#{ip}:9200" }.join(',') ],
+                "flush_size" => p('logstash_parser.elasticsearch.flush_size'),               
+                "index" => p('logstash_parser.elasticsearch.index'),
+                "document_type" => p('logstash_parser.elasticsearch.index_type'),
+                "manage_template" => false              
+            } 
+            if p('logstash_parser.elasticsearch.idle_flush_time', nil)
+                output['options']['idle_flush_time'] = p('logstash_parser.elasticsearch.idle_flush_time')
+            end
+            if p('logstash_parser.elasticsearch.document_id', nil)
+                output['options']['document_id'] = p('logstash_parser.elasticsearch.document_id')
+            end
+            if p('logstash_parser.elasticsearch.routing', nil)
+                output['options']['routing'] = p('logstash_parser.elasticsearch.routing')
+            end
+            %>
         <% end %>
-        index => "<%= p('logstash_parser.elasticsearch_index') %>"
-        document_type => "<%= p('logstash_parser.elasticsearch_index_type') %>"
-        manage_template => false
-        <% if p('logstash_parser.elasticsearch_routing', nil) %>
-            routing =>  "<%= p('logstash_parser.elasticsearch_routing') %>"
+        <% output['options'].each do | k, v | %>
+          <%= k %> => <%= v.inspect %>
         <% end %>
-    }
-
-    <%= p('logstash_parser.outputs', '')  %>
+        }
+    <% end %>
 }
diff --git a/src/logsearch-config/Rakefile b/src/logsearch-config/Rakefile
index 1fa0c879..35fc0abe 100644
--- a/src/logsearch-config/Rakefile
+++ b/src/logsearch-config/Rakefile
@@ -3,8 +3,8 @@ require 'yaml'
 require 'json'
 
 task :clean do
-  mkdir_p "target"
-  rm_rf "target/*"
+  FileUtils.mkdir_p('target')
+  FileUtils.rm_rf(Dir.glob('target/*'))
 end
 
 desc "Builds filters & dashboards"
@@ -18,7 +18,7 @@ task :build => :clean do
 end
 
 desc "Runs all tests"
-task test: [
+task default: [
   :test_filters,
   :test_templates
 ]
@@ -48,8 +48,8 @@ task :test_templates, [:rspec_files] => :compile_templates do |t, args|
 end
 
 task :clean_templates do
-  mkdir_p 'test/logstash-templates/target'
-  rm_rf 'test/logstash-templates/target/*'
+  FileUtils.mkdir_p('test/logstash-templates/target')
+  FileUtils.rm_rf(Dir.glob('test/logstash-templates/target/*'))
 end
 
 desc "Compile bosh templates for tests"
diff --git a/src/logsearch-config/bin/test b/src/logsearch-config/bin/test
index c2291438..831d2924 100755
--- a/src/logsearch-config/bin/test
+++ b/src/logsearch-config/bin/test
@@ -6,9 +6,9 @@ BASE_DIR=$(cd $SCRIPT_DIR/.. ; pwd)
 cd $BASE_DIR
 
 if [ $# -eq 0 ]
-  then rake test
+  then rake
 elif [ $# -eq 1 ]
-  then rake test $1
+  then rake $1
 elif [ $# -eq 2 ]
-  then rake test $1["$2"]
+  then rake $1["$2"]
 fi
diff --git a/src/logsearch-config/test/logstash-templates/input_and_output/compile.yml b/src/logsearch-config/test/logstash-templates/input_and_output/compile.yml
index 78af72ed..c9889b2a 100644
--- a/src/logsearch-config/test/logstash-templates/input_and_output/compile.yml
+++ b/src/logsearch-config/test/logstash-templates/input_and_output/compile.yml
@@ -3,8 +3,32 @@ template:
   testcase:
     - name: "default"
       config: "input_and_output/test_default-config.yml"
-      destination: "target/input_and_output-default.conf"
+      destination: "target/input_and_output-test_default.conf"
 
     - name: "enabled debug"
       config: "input_and_output/test_enabled_debug-config.yml"
-      destination: "target/input_and_output-enabled_debug.conf"
+      destination: "target/input_and_output-test_enabled_debug.conf"
+
+    - name: "inputs"
+      config: "input_and_output/test_inputs-config.yml"
+      destination: "target/input_and_output-test_inputs.conf"
+
+    - name: "inputs - no redis"
+      config: "input_and_output/test_inputs_no_redis-config.yml"
+      destination: "target/input_and_output-test_inputs_no_redis.conf"
+    
+    - name: "outputs"
+      config: "input_and_output/test_outputs-config.yml"
+      destination: "target/input_and_output-test_outputs.conf"
+
+    - name: "outputs - no elasticsearch"
+      config: "input_and_output/test_outputs_no_elasticsearch-config.yml"
+      destination: "target/input_and_output-test_outputs_no_elasticsearch.conf"
+
+    - name: "outputs - elasticsearch properties"
+      config: "input_and_output/test_outputs_elasticsearch_all_properties-config.yml"
+      destination: "target/input_and_output-test_outputs_elasticsearch_all_properties.conf"
+
+    - name: "outputs - elasticsearch properties"
+      config: "input_and_output/test_outputs_elasticsearch_required_properties-config.yml"
+      destination: "target/input_and_output-test_outputs_elasticsearch_required_properties.conf"
diff --git a/src/logsearch-config/test/logstash-templates/input_and_output/spec.rb b/src/logsearch-config/test/logstash-templates/input_and_output/spec.rb
index 5ed5c0ef..abf37249 100644
--- a/src/logsearch-config/test/logstash-templates/input_and_output/spec.rb
+++ b/src/logsearch-config/test/logstash-templates/input_and_output/spec.rb
@@ -12,11 +12,35 @@ def verify_input_and_output(actual, expected)
 describe 'Parser input_and_output.conf' do
 
   context "default" do
-    verify_input_and_output('input_and_output-default.conf', 'test_default-expected.conf')
+    verify_input_and_output('input_and_output-test_default.conf', 'test_default-expected.conf')
   end
 
   context "enabled debug" do
-    verify_input_and_output('input_and_output-enabled_debug.conf', 'test_enabled_debug-expected.conf')
+    verify_input_and_output('input_and_output-test_enabled_debug.conf', 'test_enabled_debug-expected.conf')
+  end
+
+  context "inputs" do
+    verify_input_and_output('input_and_output-test_inputs.conf', 'test_inputs-expected.conf')
+  end
+
+  context "inputs - no redis" do
+    verify_input_and_output('input_and_output-test_inputs_no_redis.conf', 'test_inputs_no_redis-expected.conf')
+  end
+
+  context "outputs" do
+    verify_input_and_output('input_and_output-test_outputs.conf', 'test_outputs-expected.conf')
+  end
+
+  context "outputs - no elasticsearch" do
+    verify_input_and_output('input_and_output-test_outputs_no_elasticsearch.conf', 'test_outputs_no_elasticsearch-expected.conf')
+  end
+
+  context "outputs - elasticsearch all properties" do
+    verify_input_and_output('input_and_output-test_outputs_elasticsearch_all_properties.conf', 'test_outputs_elasticsearch_all_properties-expected.conf')
+  end
+
+ context "outputs - elasticsearch required properties" do
+    verify_input_and_output('input_and_output-test_outputs_elasticsearch_required_properties.conf', 'test_outputs_elasticsearch_required_properties-expected.conf')
   end
 
 end
diff --git a/src/logsearch-config/test/logstash-templates/input_and_output/test_default-config.yml b/src/logsearch-config/test/logstash-templates/input_and_output/test_default-config.yml
index 8e825964..d39f8f97 100644
--- a/src/logsearch-config/test/logstash-templates/input_and_output/test_default-config.yml
+++ b/src/logsearch-config/test/logstash-templates/input_and_output/test_default-config.yml
@@ -1,17 +1,21 @@
----
+# --
+# Required properties are set + default values (see "parser" job spec for default values).
+# --
 properties:
   logstash_parser:
     debug: false
     inputs: [ { plugin: "redis", options : {} } ]
-    elasticsearch_document_id: ~
-    elasticsearch_index: "logs-%{[@metadata][index]}-%{+YYYY.MM.dd}"
-    elasticsearch_index_type: "%{@type}"
-  logstash:
-    output:
-      elasticsearch:
-        data_hosts: [127.0.0.1]
-        flush_size: 500
+    outputs: [ { plugin: "elasticsearch", options: {} } ]
+    elasticsearch:
+      document_id: ~
+      index: "logs-%{[@metadata][index]}-%{+YYYY.MM.dd}"
+      index_type: "%{@type}"
+      idle_flush_time: 100
+      data_hosts: [127.0.0.1]
+      flush_size: 500
   redis:
+    # required, non-default
     host: 127.0.0.1
+
     port: 6379
     key: logstash
diff --git a/src/logsearch-config/test/logstash-templates/input_and_output/test_default-expected.conf b/src/logsearch-config/test/logstash-templates/input_and_output/test_default-expected.conf
index 9f0a0d79..714f3ff2 100644
--- a/src/logsearch-config/test/logstash-templates/input_and_output/test_default-expected.conf
+++ b/src/logsearch-config/test/logstash-templates/input_and_output/test_default-expected.conf
@@ -1,6 +1,6 @@
 input {
     redis {     
-        host => "127.0.0.1"     
+        host => "127.0.0.1"
         port => 6379    
         type => "redis-input"     
         data_type => "list"     
@@ -16,5 +16,6 @@ output {
         index => "logs-%{[@metadata][index]}-%{+YYYY.MM.dd}"
         document_type => "%{@type}"
         manage_template => false
+        idle_flush_time => 100
     }   
 }
diff --git a/src/logsearch-config/test/logstash-templates/input_and_output/test_enabled_debug-config.yml b/src/logsearch-config/test/logstash-templates/input_and_output/test_enabled_debug-config.yml
index c77167cc..428ee16a 100644
--- a/src/logsearch-config/test/logstash-templates/input_and_output/test_enabled_debug-config.yml
+++ b/src/logsearch-config/test/logstash-templates/input_and_output/test_enabled_debug-config.yml
@@ -1,16 +1,17 @@
----
+# --
+# Default values + enabled logstash debug
+# --
 properties:
   logstash_parser:
     debug: true
     inputs: [ { plugin: "redis", options : {} } ]
-    elasticsearch_document_id: ~
-    elasticsearch_index: "logs-%{[@metadata][index]}-%{+YYYY.MM.dd}"
-    elasticsearch_index_type: "%{@type}"
-  logstash:
-    output:
-      elasticsearch:
-        data_hosts: [127.0.0.1]
-        flush_size: 500
+    outputs: [ { plugin: "elasticsearch", options: {} } ]
+    elasticsearch:
+      document_id: ~
+      index: "logs-%{[@metadata][index]}-%{+YYYY.MM.dd}"
+      index_type: "%{@type}"
+      data_hosts: [127.0.0.1]
+      flush_size: 500
   redis:
     host: 127.0.0.1
     port: 6379
diff --git a/src/logsearch-config/test/logstash-templates/input_and_output/test_enabled_debug-expected.conf b/src/logsearch-config/test/logstash-templates/input_and_output/test_enabled_debug-expected.conf
index 9dcee45b..9f06ec86 100644
--- a/src/logsearch-config/test/logstash-templates/input_and_output/test_enabled_debug-expected.conf
+++ b/src/logsearch-config/test/logstash-templates/input_and_output/test_enabled_debug-expected.conf
@@ -1,6 +1,6 @@
 input {
     redis {     
-        host => "127.0.0.1"     
+        host => "127.0.0.1"
         port => 6379    
         type => "redis-input"     
         data_type => "list"     
diff --git a/src/logsearch-config/test/logstash-templates/input_and_output/test_inputs-config.yml b/src/logsearch-config/test/logstash-templates/input_and_output/test_inputs-config.yml
new file mode 100644
index 00000000..44a6ed38
--- /dev/null
+++ b/src/logsearch-config/test/logstash-templates/input_and_output/test_inputs-config.yml
@@ -0,0 +1,18 @@
+# --
+# Default values + additional inputs.
+# --
+properties:
+  logstash_parser:
+    debug: false
+    inputs: [ { plugin: "redis", options : {} }, { plugin: "file", options: { path: "my/path/to/file" } }, { plugin: "syslog", options: { host: "127.0.0.1", port: 123} } ]
+    outputs: [ { plugin: "elasticsearch", options: {} } ]
+    elasticsearch:
+      document_id: ~
+      index: "logs-%{[@metadata][index]}-%{+YYYY.MM.dd}"
+      index_type: "%{@type}"
+      data_hosts: [127.0.0.1]
+      flush_size: 500
+  redis:
+    host: 127.0.0.1
+    port: 6379
+    key: logstash
diff --git a/src/logsearch-config/test/logstash-templates/input_and_output/test_inputs-expected.conf b/src/logsearch-config/test/logstash-templates/input_and_output/test_inputs-expected.conf
new file mode 100644
index 00000000..697a001b
--- /dev/null
+++ b/src/logsearch-config/test/logstash-templates/input_and_output/test_inputs-expected.conf
@@ -0,0 +1,29 @@
+input {
+    redis {
+        host => "127.0.0.1"
+        port => 6379      
+        type => "redis-input"     
+        data_type => "list"      
+        key => "logstash"      
+        threads => 8      
+    }
+
+    file {     
+        path => "my/path/to/file"     
+    }
+
+    syslog {     
+        host => "127.0.0.1"     
+        port => 123     
+    }
+}
+
+output {
+    elasticsearch {
+        hosts => ["127.0.0.1:9200"]
+        flush_size => 500  
+        index => "logs-%{[@metadata][index]}-%{+YYYY.MM.dd}"
+        document_type => "%{@type}"
+        manage_template => false
+    }
+}
diff --git a/src/logsearch-config/test/logstash-templates/input_and_output/test_inputs_no_redis-config.yml b/src/logsearch-config/test/logstash-templates/input_and_output/test_inputs_no_redis-config.yml
new file mode 100644
index 00000000..97a1b69b
--- /dev/null
+++ b/src/logsearch-config/test/logstash-templates/input_and_output/test_inputs_no_redis-config.yml
@@ -0,0 +1,18 @@
+# --
+# Default values + additional inputs.
+# --
+properties:
+  logstash_parser:
+    debug: false
+    inputs: [ { plugin: "file", options: { path: "my/path/to/file" } }, { plugin: "syslog", options: { host: "127.0.0.1", port: 123} } ]
+    outputs: [ { plugin: "elasticsearch", options: {} } ]
+    elasticsearch:
+      document_id: ~
+      index: "logs-%{[@metadata][index]}-%{+YYYY.MM.dd}"
+      index_type: "%{@type}"
+      data_hosts: [127.0.0.1]
+      flush_size: 500
+  redis:
+    host: 127.0.0.1
+    port: 6379
+    key: logstash
diff --git a/src/logsearch-config/test/logstash-templates/input_and_output/test_inputs_no_redis-expected.conf b/src/logsearch-config/test/logstash-templates/input_and_output/test_inputs_no_redis-expected.conf
new file mode 100644
index 00000000..856940b3
--- /dev/null
+++ b/src/logsearch-config/test/logstash-templates/input_and_output/test_inputs_no_redis-expected.conf
@@ -0,0 +1,20 @@
+input {
+    file {     
+        path => "my/path/to/file"     
+    }
+
+    syslog {     
+        host => "127.0.0.1"     
+        port => 123     
+    }
+}
+
+output {
+    elasticsearch {
+        hosts => ["127.0.0.1:9200"]
+        flush_size => 500  
+        index => "logs-%{[@metadata][index]}-%{+YYYY.MM.dd}"
+        document_type => "%{@type}"
+        manage_template => false
+    }
+}
diff --git a/src/logsearch-config/test/logstash-templates/input_and_output/test_outputs-config.yml b/src/logsearch-config/test/logstash-templates/input_and_output/test_outputs-config.yml
new file mode 100644
index 00000000..8793b509
--- /dev/null
+++ b/src/logsearch-config/test/logstash-templates/input_and_output/test_outputs-config.yml
@@ -0,0 +1,18 @@
+# --
+# Default values + additional outputs.
+# --
+properties:
+  logstash_parser:
+    debug: true
+    inputs: [ { plugin: "redis", options : {} } ]
+    outputs: [ { plugin: "elasticsearch", options: {} }, { plugin: "mongodb", options: { uri: "192.168.1.1", database: "logsearch", collection: "logs" } }, { plugin: "syslog", options: { host: "127.0.0.1", port: 123} } ]
+    elasticsearch:
+      document_id: ~
+      index: "logs-%{[@metadata][index]}-%{+YYYY.MM.dd}"
+      index_type: "%{@type}"
+      data_hosts: [127.0.0.1]
+      flush_size: 500
+  redis:
+    host: 127.0.0.1
+    port: 6379
+    key: logstash
diff --git a/src/logsearch-config/test/logstash-templates/input_and_output/test_outputs-expected.conf b/src/logsearch-config/test/logstash-templates/input_and_output/test_outputs-expected.conf
new file mode 100644
index 00000000..0e732753
--- /dev/null
+++ b/src/logsearch-config/test/logstash-templates/input_and_output/test_outputs-expected.conf
@@ -0,0 +1,36 @@
+input {
+    redis {
+        host => "127.0.0.1"
+        port => 6379      
+        type => "redis-input"     
+        data_type => "list"      
+        key => "logstash"      
+        threads => 8      
+    }
+}
+
+output {
+
+    stdout {
+        codec => "json"
+    }
+
+    elasticsearch {
+        hosts => ["127.0.0.1:9200"]
+        flush_size => 500  
+        index => "logs-%{[@metadata][index]}-%{+YYYY.MM.dd}"
+        document_type => "%{@type}"
+        manage_template => false 
+    }
+
+    mongodb {     
+        uri => "192.168.1.1"
+        database => "logsearch"
+        collection => "logs"
+    }
+
+    syslog {     
+        host => "127.0.0.1"     
+        port => 123     
+    }
+}
diff --git a/src/logsearch-config/test/logstash-templates/input_and_output/test_outputs_elasticsearch_all_properties-config.yml b/src/logsearch-config/test/logstash-templates/input_and_output/test_outputs_elasticsearch_all_properties-config.yml
new file mode 100644
index 00000000..391fd314
--- /dev/null
+++ b/src/logsearch-config/test/logstash-templates/input_and_output/test_outputs_elasticsearch_all_properties-config.yml
@@ -0,0 +1,20 @@
+# --
+# All "elasticsearch" properties are set with non-null values.
+# --
+properties:
+  logstash_parser:
+    debug: false
+    inputs: [ { plugin: "redis", options : {} } ]
+    outputs: [ { plugin: "elasticsearch", options: {} } ]
+    elasticsearch:
+      document_id: "abc"
+      index: "logs-%{[@metadata][index]}-%{+YYYY.MM.dd}"
+      index_type: "%{@type}"    
+      routing: "%{some_field}"
+      idle_flush_time: 100
+      data_hosts: [127.0.0.1]
+      flush_size: 500
+  redis:
+    host: 127.0.0.1
+    port: 6379
+    key: logstash
diff --git a/src/logsearch-config/test/logstash-templates/input_and_output/test_outputs_elasticsearch_all_properties-expected.conf b/src/logsearch-config/test/logstash-templates/input_and_output/test_outputs_elasticsearch_all_properties-expected.conf
new file mode 100644
index 00000000..4a463977
--- /dev/null
+++ b/src/logsearch-config/test/logstash-templates/input_and_output/test_outputs_elasticsearch_all_properties-expected.conf
@@ -0,0 +1,25 @@
+input {
+    redis {
+        host => "127.0.0.1"
+        port => 6379      
+        type => "redis-input"     
+        data_type => "list"      
+        key => "logstash"      
+        threads => 8      
+    }
+}
+
+output {
+
+    elasticsearch {
+        hosts => ["127.0.0.1:9200"]
+        flush_size => 500
+        index => "logs-%{[@metadata][index]}-%{+YYYY.MM.dd}"
+        document_type => "%{@type}"
+        manage_template => false
+        document_id => "abc"
+        idle_flush_time => 100
+        routing => "%{some_field}"     
+    }
+
+}
diff --git a/src/logsearch-config/test/logstash-templates/input_and_output/test_outputs_elasticsearch_required_properties-config.yml b/src/logsearch-config/test/logstash-templates/input_and_output/test_outputs_elasticsearch_required_properties-config.yml
new file mode 100644
index 00000000..668136b1
--- /dev/null
+++ b/src/logsearch-config/test/logstash-templates/input_and_output/test_outputs_elasticsearch_required_properties-config.yml
@@ -0,0 +1,17 @@
+# --
+# Only required "elasticsearch" properties are set.
+# --
+properties:
+  logstash_parser:
+    debug: false
+    inputs: [ { plugin: "redis", options : {} } ]
+    outputs: [ { plugin: "elasticsearch", options: {} } ]
+    elasticsearch:
+      index: "logs-%{[@metadata][index]}-%{+YYYY.MM.dd}"
+      index_type: "%{@type}"
+      data_hosts: [127.0.0.1]
+      flush_size: 500
+  redis:
+    host: 127.0.0.1
+    port: 6379
+    key: logstash
diff --git a/src/logsearch-config/test/logstash-templates/input_and_output/test_outputs_elasticsearch_required_properties-expected.conf b/src/logsearch-config/test/logstash-templates/input_and_output/test_outputs_elasticsearch_required_properties-expected.conf
new file mode 100644
index 00000000..00c788bb
--- /dev/null
+++ b/src/logsearch-config/test/logstash-templates/input_and_output/test_outputs_elasticsearch_required_properties-expected.conf
@@ -0,0 +1,22 @@
+input {
+    redis {
+        host => "127.0.0.1"
+        port => 6379      
+        type => "redis-input"     
+        data_type => "list"      
+        key => "logstash"      
+        threads => 8      
+    }
+}
+
+output {
+
+    elasticsearch {
+        hosts => ["127.0.0.1:9200"]
+        flush_size => 500  
+        index => "logs-%{[@metadata][index]}-%{+YYYY.MM.dd}"
+        document_type => "%{@type}"
+        manage_template => false     
+    }
+
+}
diff --git a/src/logsearch-config/test/logstash-templates/input_and_output/test_outputs_no_elasticsearch-config.yml b/src/logsearch-config/test/logstash-templates/input_and_output/test_outputs_no_elasticsearch-config.yml
new file mode 100644
index 00000000..12fb6ad1
--- /dev/null
+++ b/src/logsearch-config/test/logstash-templates/input_and_output/test_outputs_no_elasticsearch-config.yml
@@ -0,0 +1,18 @@
+# --
+# Default values + additional outputs.
+# --
+properties:
+  logstash_parser:
+    debug: false
+    inputs: [ { plugin: "redis", options : {} } ]
+    outputs: [ { plugin: "mongodb", options: { uri: "192.168.1.1", database: "logsearch", collection: "logs" } }, { plugin: "syslog", options: { host: "127.0.0.1", port: 123} } ]
+    elasticsearch:
+      document_id: ~
+      index: "logs-%{[@metadata][index]}-%{+YYYY.MM.dd}"
+      index_type: "%{@type}"
+      data_hosts: [127.0.0.1]
+      flush_size: 500
+  redis:
+    host: 127.0.0.1
+    port: 6379
+    key: logstash
diff --git a/src/logsearch-config/test/logstash-templates/input_and_output/test_outputs_no_elasticsearch-expected.conf b/src/logsearch-config/test/logstash-templates/input_and_output/test_outputs_no_elasticsearch-expected.conf
new file mode 100644
index 00000000..896b0837
--- /dev/null
+++ b/src/logsearch-config/test/logstash-templates/input_and_output/test_outputs_no_elasticsearch-expected.conf
@@ -0,0 +1,24 @@
+input {
+    redis {
+        host => "127.0.0.1"
+        port => 6379      
+        type => "redis-input"     
+        data_type => "list"      
+        key => "logstash"      
+        threads => 8      
+    }
+}
+
+output {
+
+    mongodb {     
+        uri => "192.168.1.1"
+        database => "logsearch"
+        collection => "logs"
+    }
+
+    syslog {     
+        host => "127.0.0.1"     
+        port => 123     
+    }
+}