From 73de2d68d41947df9f7f5e93185eee87c8b2e118 Mon Sep 17 00:00:00 2001
From: Joshua Carp <jm.carp@gmail.com>
Date: Mon, 24 Jul 2017 02:19:34 -0400
Subject: [PATCH 01/19] Drop default cerebro secret. (#85)

* Drop default cerebro secret.
* Add cerebro secret to manifest templates.
---
 deployment/logsearch-deployment.yml                | 5 +++++
 jobs/cerebro/spec                                  | 1 -
 jobs/cerebro/templates/config/application.conf.erb | 1 -
 templates/logsearch-jobs.yml                       | 2 ++
 4 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/deployment/logsearch-deployment.yml b/deployment/logsearch-deployment.yml
index 04547b65..7865c686 100644
--- a/deployment/logsearch-deployment.yml
+++ b/deployment/logsearch-deployment.yml
@@ -96,6 +96,9 @@ instance_groups:
     release: logsearch
     consumes:
       elasticsearch: {from: elasticsearch_master}
+    properties:
+      cerebro:
+        secret: ((cerebro_secret_key))
   - name: syslog_forwarder
     release: logsearch
     consumes:
@@ -401,6 +404,8 @@ variables:
   type: password
 - name: firehose_client_secret
   type: password
+- name: cerebro_secret_key
+  type: password
 
 releases:
 - name: logsearch
diff --git a/jobs/cerebro/spec b/jobs/cerebro/spec
index 6a4eb673..3c4e205d 100644
--- a/jobs/cerebro/spec
+++ b/jobs/cerebro/spec
@@ -27,4 +27,3 @@ properties:
     default: []
   cerebro.secret:
     description: Secret will be used to sign Cerebro session cookies and CSRF tokens.
-    default: "ABCDEFGHIJKLMNOPQRSTUVWXYZ"
diff --git a/jobs/cerebro/templates/config/application.conf.erb b/jobs/cerebro/templates/config/application.conf.erb
index 470e3611..9f3e95bf 100644
--- a/jobs/cerebro/templates/config/application.conf.erb
+++ b/jobs/cerebro/templates/config/application.conf.erb
@@ -1,5 +1,4 @@
 # Secret will be used to sign session cookies, CSRF tokens and for other encryption utilities.
-# It is highly recommended to change this value before running cerebro in production.
 secret = "<%= p("cerebro.secret") %>"
 
 # Application base path
diff --git a/templates/logsearch-jobs.yml b/templates/logsearch-jobs.yml
index 586bd62d..708035d4 100644
--- a/templates/logsearch-jobs.yml
+++ b/templates/logsearch-jobs.yml
@@ -22,6 +22,8 @@ jobs:
       node:
         allow_master: true
         allow_data: false
+    cerebro:
+      secret: (( param "specify cerebro secret key" ))
     syslog_forwarder:
       config:
       - {service: elasticsearch, file: /var/vcap/sys/log/elasticsearch/elasticsearch.stdout.log}

From 681a8f2429831e8647651f41d336e678906efe63 Mon Sep 17 00:00:00 2001
From: Andrei Krasnitski <me@infra-red.xyz>
Date: Mon, 24 Jul 2017 21:36:46 +0300
Subject: [PATCH 02/19] Provide syslog.transport property via link

---
 jobs/ingestor_syslog/spec | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/jobs/ingestor_syslog/spec b/jobs/ingestor_syslog/spec
index 284c2ff4..ad65b943 100644
--- a/jobs/ingestor_syslog/spec
+++ b/jobs/ingestor_syslog/spec
@@ -25,6 +25,7 @@ provides:
   type: ingestor
   properties:
   - logstash_ingestor.syslog.port
+  - logstash_ingestor.syslog.transport
   - logstash_ingestor.syslog_tls.port
   - logstash_ingestor.relp.port
 - name: syslog_forwarder
@@ -88,6 +89,9 @@ properties:
   logstash_ingestor.syslog.port:
     description: Port to listen for syslog messages
     default: 5514
+  logstash_ingestor.syslog.transport:
+    description: Transport protocol to use
+    default: "tcp"
 
   logstash_ingestor.syslog_tls.port:
     description: Port to listen for syslog-TLS messages (omit to disable)

From 1bc4245610162d07999314b88b49e2a4f3bc2889 Mon Sep 17 00:00:00 2001
From: Andrei Krasnitski <me@infra-red.xyz>
Date: Tue, 25 Jul 2017 14:53:53 +0300
Subject: [PATCH 03/19] Allow to set environment variables in syslog_ctl script

---
 jobs/ingestor_syslog/spec                              | 3 +++
 jobs/ingestor_syslog/templates/bin/ingestor_syslog_ctl | 3 +++
 2 files changed, 6 insertions(+)

diff --git a/jobs/ingestor_syslog/spec b/jobs/ingestor_syslog/spec
index ad65b943..8b2f115e 100644
--- a/jobs/ingestor_syslog/spec
+++ b/jobs/ingestor_syslog/spec
@@ -47,6 +47,9 @@ properties:
     example:
     - {name: logstash-output-cloudwatchlogs, version: 2.0.0}
     default: []
+  logstash.env:
+    description: "a list of arbitrary key-value pairs to be passed on as process environment variables. eg: FOO: 123"
+    default: []
 
   logstash.queue.type:
     description: Internal queuing model, "memory" for legacy in-memory based queuing and "persisted" for disk-based acked queueing.
diff --git a/jobs/ingestor_syslog/templates/bin/ingestor_syslog_ctl b/jobs/ingestor_syslog/templates/bin/ingestor_syslog_ctl
index f05ee4fb..97c6eeb4 100644
--- a/jobs/ingestor_syslog/templates/bin/ingestor_syslog_ctl
+++ b/jobs/ingestor_syslog/templates/bin/ingestor_syslog_ctl
@@ -35,6 +35,9 @@ export HEAP_SIZE=$((( $( cat /proc/meminfo | grep MemTotal | awk '{ print $2 }'
 <% if_p('logstash.heap_size') do |heap_size| %>
 HEAP_SIZE=<%= heap_size %>
 <% end %>
+<% p("logstash.env").each do |env| %>
+export <%= env.keys[0] %>="<%= env.values[0] %>"
+<% end %>
 
 case $1 in
 

From 98604c308d63e183f2ae57867e058c1a8ad20478 Mon Sep 17 00:00:00 2001
From: Andrei Krasnitski <me@infra-red.xyz>
Date: Tue, 25 Jul 2017 21:07:52 +0300
Subject: [PATCH 04/19] Remove queue job/package

Remove message broker since logstash persistent queues enabled by
default
---
 config/blobs.yml                              |   4 -
 jobs/queue/monit                              |  15 -
 jobs/queue/spec                               |  30 -
 jobs/queue/templates/bin/monit_debugger       |  13 -
 jobs/queue/templates/bin/redis_ctl            |  32 -
 jobs/queue/templates/config/redis.conf.erb    | 748 ------------------
 jobs/queue/templates/data/properties.sh.erb   |  10 -
 jobs/queue/templates/helpers/ctl_setup.sh     |  81 --
 jobs/queue/templates/helpers/ctl_utils.sh     | 182 -----
 .../metric-collector/redis/collector          |  65 --
 packages/redis/packaging                      |  15 -
 packages/redis/spec                           |   5 -
 12 files changed, 1200 deletions(-)
 delete mode 100644 jobs/queue/monit
 delete mode 100644 jobs/queue/spec
 delete mode 100644 jobs/queue/templates/bin/monit_debugger
 delete mode 100755 jobs/queue/templates/bin/redis_ctl
 delete mode 100644 jobs/queue/templates/config/redis.conf.erb
 delete mode 100644 jobs/queue/templates/data/properties.sh.erb
 delete mode 100644 jobs/queue/templates/helpers/ctl_setup.sh
 delete mode 100644 jobs/queue/templates/helpers/ctl_utils.sh
 delete mode 100644 jobs/queue/templates/logsearch/metric-collector/redis/collector
 delete mode 100644 packages/redis/packaging
 delete mode 100644 packages/redis/spec

diff --git a/config/blobs.yml b/config/blobs.yml
index 6cf2e6fa..96a5b365 100644
--- a/config/blobs.yml
+++ b/config/blobs.yml
@@ -94,10 +94,6 @@ python/Python-3.6.1.tgz:
   size: 22540566
   object_id: db2b3522-9d58-4998-81ec-237d900ad740
   sha: 6e91434cf22414af8240dfa1bf8ab2d043b04998
-redis/redis-3.2.9.tar.gz:
-  size: 1547695
-  object_id: 11340b96-739f-4532-8a38-8e00661ee18a
-  sha: 8fad759f28bcb14b94254124d824f1f3ed7b6aa6
 ruby2.3/bundler-1.11.2.gem:
   size: 263168
   object_id: ce2c212f-c2ff-440e-9f4b-c2c214a010f9
diff --git a/jobs/queue/monit b/jobs/queue/monit
deleted file mode 100644
index 30451dd4..00000000
--- a/jobs/queue/monit
+++ /dev/null
@@ -1,15 +0,0 @@
-check process queue_redis
-  with pidfile /var/vcap/sys/run/queue/queue.pid
-  start program "/var/vcap/jobs/queue/bin/monit_debugger queue_ctl '/var/vcap/jobs/queue/bin/redis_ctl start'"
-  stop program "/var/vcap/jobs/queue/bin/monit_debugger queue_ctl '/var/vcap/jobs/queue/bin/redis_ctl stop'"
-  group vcap
-
-check device queue-ephemeral_disk with path /var/vcap/data
-  if SPACE usage > 80% then alert
-
-check device queue-persistent_disk with path /var/vcap/store
-  if SPACE usage > 80% then alert
-
-check file queue-backlog with path /var/vcap/store/queue/redis-appendonly.aof
-  if size > 2 GB then alert
-
diff --git a/jobs/queue/spec b/jobs/queue/spec
deleted file mode 100644
index be1809a2..00000000
--- a/jobs/queue/spec
+++ /dev/null
@@ -1,30 +0,0 @@
----
-name: queue
-packages:
-- redis
-templates:
-  bin/redis_ctl: bin/redis_ctl
-  bin/monit_debugger: bin/monit_debugger
-  data/properties.sh.erb: data/properties.sh
-  helpers/ctl_setup.sh: helpers/ctl_setup.sh
-  helpers/ctl_utils.sh: helpers/ctl_utils.sh
-  config/redis.conf.erb: config/redis.conf
-  logsearch/metric-collector/redis/collector: logsearch/metric-collector/redis/collector
-properties:
-  redis.port:
-    description: Redis port of queue
-    default: 6379
-  redis.key:
-    description: Name of queue to pull messages from
-    default: logstash
-  redis.maxmemory:
-    description: Maximum amount of memory to be used by Redis in % of system RAM.
-    example: 50
-    default: 90
-  redis.maxmemory-policy:
-    description: How Redis will select what to remove when maxmemory is reached. Possible values are volatile-lru, allkeys-lru, volatile-random, allkeys-random, volatile-ttl, and noeviction.
-    default: volatile-lru
-  redis.appendonly:
-    description: Enable Redis Persistence through AOF file
-    default: "yes"
-
diff --git a/jobs/queue/templates/bin/monit_debugger b/jobs/queue/templates/bin/monit_debugger
deleted file mode 100644
index 55353d2a..00000000
--- a/jobs/queue/templates/bin/monit_debugger
+++ /dev/null
@@ -1,13 +0,0 @@
-#!/bin/sh
-# USAGE monit_debugger <label> command to run
-mkdir -p /var/vcap/sys/log/monit
-{
-  echo "MONIT-DEBUG date"
-  date
-  echo "MONIT-DEBUG env"
-  env
-  echo "MONIT-DEBUG $@"
-  $2 $3 $4 $5 $6 $7
-  R=$?
-  echo "MONIT-DEBUG exit code $R"
-} >/var/vcap/sys/log/monit/monit_debugger.$1.log 2>&1
diff --git a/jobs/queue/templates/bin/redis_ctl b/jobs/queue/templates/bin/redis_ctl
deleted file mode 100755
index 773f181b..00000000
--- a/jobs/queue/templates/bin/redis_ctl
+++ /dev/null
@@ -1,32 +0,0 @@
-#!/bin/bash
-
-set -e # exit immediately if a simple command exits with a non-zero status
-set -u # report the usage of uninitialized variables
-
-# Setup env vars and folders for the webapp_ctl script
-source /var/vcap/jobs/queue/helpers/ctl_setup.sh 'queue'
-
-export LANG=en_US.UTF-8
-
-case $1 in
-
-  start)
-    pid_guard $PIDFILE $JOB_NAME
-
-    MEMORY_LIMIT=$(($(grep MemTotal /proc/meminfo | awk '{print $2}') * <%= p('redis.maxmemory') %>/100 * 1024))
-    sed -i "s/^maxmemory REPLACE_ME/maxmemory $MEMORY_LIMIT/" $JOB_DIR/config/redis.conf 
-    chpst -u vcap /var/vcap/packages/redis/bin/redis-server $JOB_DIR/config/redis.conf
-
-    ;;
-
-  stop)
-    kill_and_wait $PIDFILE
-
-    ;;
-  *)
-    echo "Usage: queue_ctl {start|stop}"
-
-    ;;
-
-esac
-exit 0
diff --git a/jobs/queue/templates/config/redis.conf.erb b/jobs/queue/templates/config/redis.conf.erb
deleted file mode 100644
index 98d0ff8d..00000000
--- a/jobs/queue/templates/config/redis.conf.erb
+++ /dev/null
@@ -1,748 +0,0 @@
-# Redis configuration file example
-
-# Note on units: when memory size is needed, it is possible to specify
-# it in the usual form of 1k 5GB 4M and so forth:
-#
-# 1k => 1000 bytes
-# 1kb => 1024 bytes
-# 1m => 1000000 bytes
-# 1mb => 1024*1024 bytes
-# 1g => 1000000000 bytes
-# 1gb => 1024*1024*1024 bytes
-#
-# units are case insensitive so 1GB 1Gb 1gB are all the same.
-
-################################## INCLUDES ###################################
-
-# Include one or more other config files here.  This is useful if you
-# have a standard template that goes to all Redis server but also need
-# to customize a few per-server settings.  Include files can include
-# other files, so use this wisely.
-#
-# Notice option "include" won't be rewritten by command "CONFIG REWRITE"
-# from admin or Redis Sentinel. Since Redis always uses the last processed
-# line as value of a configuration directive, you'd better put includes
-# at the beginning of this file to avoid overwriting config change at runtime.
-#
-# If instead you are interested in using includes to override configuration
-# options, it is better to use include as the last line.
-#
-# include /path/to/local.conf
-# include /path/to/other.conf
-
-################################ GENERAL  #####################################
-
-# By default Redis does not run as a daemon. Use 'yes' if you need it.
-# Note that Redis will write a pid file in /var/run/redis.pid when daemonized.
-daemonize yes
-
-# When running daemonized, Redis writes a pid file in /var/run/redis.pid by
-# default. You can specify a custom pid file location here.
-pidfile /var/vcap/sys/run/queue/queue.pid
-
-# Accept connections on the specified port, default is 6379.
-# If port 0 is specified Redis will not listen on a TCP socket.
-port <%= p("redis.port") %>
-
-# TCP listen() backlog.
-#
-# In high requests-per-second environments you need an high backlog in order
-# to avoid slow clients connections issues. Note that the Linux kernel
-# will silently truncate it to the value of /proc/sys/net/core/somaxconn so
-# make sure to raise both the value of somaxconn and tcp_max_syn_backlog
-# in order to get the desired effect.
-#tcp-backlog 511
-
-# By default Redis listens for connections from all the network interfaces
-# available on the server. It is possible to listen to just one or multiple
-# interfaces using the "bind" configuration directive, followed by one or
-# more IP addresses.
-#
-# Examples:
-#
-# bind 192.168.1.100 10.0.0.1
-# bind 127.0.0.1
-
-# Protected mode is a layer of security protection, in order to avoid that
-# Redis instances left open on the internet are accessed and exploited.
-#
-# When protected mode is on and if:
-#
-# 1) The server is not binding explicitly to a set of addresses using the
-#    "bind" directive.
-# 2) No password is configured.
-#
-# The server only accepts connections from clients connecting from the
-# IPv4 and IPv6 loopback addresses 127.0.0.1 and ::1, and from Unix domain
-# sockets.
-#
-# By default protected mode is enabled. You should disable it only if
-# you are sure you want clients from other hosts to connect to Redis
-# even if no authentication is configured, nor a specific set of interfaces
-# are explicitly listed using the "bind" directive.
-protected-mode no
-
-# Specify the path for the Unix socket that will be used to listen for
-# incoming connections. There is no default, so Redis will not listen
-# on a unix socket when not specified.
-#
-# unixsocket /tmp/redis.sock
-# unixsocketperm 755
-
-# Close the connection after a client is idle for N seconds (0 to disable)
-timeout 0
-
-# TCP keepalive.
-#
-# If non-zero, use SO_KEEPALIVE to send TCP ACKs to clients in absence
-# of communication. This is useful for two reasons:
-#
-# 1) Detect dead peers.
-# 2) Take the connection alive from the point of view of network
-#    equipment in the middle.
-#
-# On Linux, the specified value (in seconds) is the period used to send ACKs.
-# Note that to close the connection the double of the time is needed.
-# On other kernels the period depends on the kernel configuration.
-#
-# A reasonable value for this option is 60 seconds.
-tcp-keepalive 0
-
-# Specify the server verbosity level.
-# This can be one of:
-# debug (a lot of information, useful for development/testing)
-# verbose (many rarely useful info, but not a mess like the debug level)
-# notice (moderately verbose, what you want in production probably)
-# warning (only very important / critical messages are logged)
-loglevel notice
-
-# Specify the log file name. Also the empty string can be used to force
-# Redis to log on the standard output. Note that if you use standard
-# output for logging but daemonize, logs will be sent to /dev/null
-logfile "/var/vcap/sys/log/queue/queue.log"
-
-# To enable logging to the system logger, just set 'syslog-enabled' to yes,
-# and optionally update the other syslog parameters to suit your needs.
-# syslog-enabled no
-
-# Specify the syslog identity.
-# syslog-ident redis
-
-# Specify the syslog facility. Must be USER or between LOCAL0-LOCAL7.
-# syslog-facility local0
-
-# Set the number of databases. The default database is DB 0, you can select
-# a different one on a per-connection basis using SELECT <dbid> where
-# dbid is a number between 0 and 'databases'-1
-databases 16
-
-################################ SNAPSHOTTING  ################################
-#
-# Save the DB on disk:
-#
-#   save <seconds> <changes>
-#
-#   Will save the DB if both the given number of seconds and the given
-#   number of write operations against the DB occurred.
-#
-#   In the example below the behaviour will be to save:
-#   after 900 sec (15 min) if at least 1 key changed
-#   after 300 sec (5 min) if at least 10 keys changed
-#   after 60 sec if at least 10000 keys changed
-#
-#   Note: you can disable saving at all commenting all the "save" lines.
-#
-#   It is also possible to remove all the previously configured save
-#   points by adding a save directive with a single empty string argument
-#   like in the following example:
-#
-#   save ""
-
-# save 900 1
-# save 300 10
-# save 60 10000
-
-# By default Redis will stop accepting writes if RDB snapshots are enabled
-# (at least one save point) and the latest background save failed.
-# This will make the user aware (in a hard way) that data is not persisting
-# on disk properly, otherwise chances are that no one will notice and some
-# disaster will happen.
-#
-# If the background saving process will start working again Redis will
-# automatically allow writes again.
-#
-# However if you have setup your proper monitoring of the Redis server
-# and persistence, you may want to disable this feature so that Redis will
-# continue to work as usual even if there are problems with disk,
-# permissions, and so forth.
-stop-writes-on-bgsave-error yes
-
-# Compress string objects using LZF when dump .rdb databases?
-# For default that's set to 'yes' as it's almost always a win.
-# If you want to save some CPU in the saving child set it to 'no' but
-# the dataset will likely be bigger if you have compressible values or keys.
-rdbcompression yes
-
-# Since version 5 of RDB a CRC64 checksum is placed at the end of the file.
-# This makes the format more resistant to corruption but there is a performance
-# hit to pay (around 10%) when saving and loading RDB files, so you can disable it
-# for maximum performances.
-#
-# RDB files created with checksum disabled have a checksum of zero that will
-# tell the loading code to skip the check.
-rdbchecksum yes
-
-# The filename where to dump the DB
-dbfilename dump.rdb
-
-# The working directory.
-#
-# The DB will be written inside this directory, with the filename specified
-# above using the 'dbfilename' configuration directive.
-#
-# The Append Only File will also be created inside this directory.
-#
-# Note that you must specify a directory here, not a file name.
-dir /var/vcap/store/queue
-
-################################# REPLICATION #################################
-
-# Master-Slave replication. Use slaveof to make a Redis instance a copy of
-# another Redis server. Note that the configuration is local to the slave
-# so for example it is possible to configure the slave to save the DB with a
-# different interval, or to listen to another port, and so on.
-#
-# slaveof <masterip> <masterport>
-
-# If the master is password protected (using the "requirepass" configuration
-# directive below) it is possible to tell the slave to authenticate before
-# starting the replication synchronization process, otherwise the master will
-# refuse the slave request.
-#
-# masterauth <master-password>
-
-# When a slave loses its connection with the master, or when the replication
-# is still in progress, the slave can act in two different ways:
-#
-# 1) if slave-serve-stale-data is set to 'yes' (the default) the slave will
-#    still reply to client requests, possibly with out of date data, or the
-#    data set may just be empty if this is the first synchronization.
-#
-# 2) if slave-serve-stale-data is set to 'no' the slave will reply with
-#    an error "SYNC with master in progress" to all the kind of commands
-#    but to INFO and SLAVEOF.
-#
-slave-serve-stale-data yes
-
-# You can configure a slave instance to accept writes or not. Writing against
-# a slave instance may be useful to store some ephemeral data (because data
-# written on a slave will be easily deleted after resync with the master) but
-# may also cause problems if clients are writing to it because of a
-# misconfiguration.
-#
-# Since Redis 2.6 by default slaves are read-only.
-#
-# Note: read only slaves are not designed to be exposed to untrusted clients
-# on the internet. It's just a protection layer against misuse of the instance.
-# Still a read only slave exports by default all the administrative commands
-# such as CONFIG, DEBUG, and so forth. To a limited extent you can improve
-# security of read only slaves using 'rename-command' to shadow all the
-# administrative / dangerous commands.
-slave-read-only yes
-
-# Slaves send PINGs to server in a predefined interval. It's possible to change
-# this interval with the repl_ping_slave_period option. The default value is 10
-# seconds.
-#
-# repl-ping-slave-period 10
-
-# The following option sets the replication timeout for:
-#
-# 1) Bulk transfer I/O during SYNC, from the point of view of slave.
-# 2) Master timeout from the point of view of slaves (data, pings).
-# 3) Slave timeout from the point of view of masters (REPLCONF ACK pings).
-#
-# It is important to make sure that this value is greater than the value
-# specified for repl-ping-slave-period otherwise a timeout will be detected
-# every time there is low traffic between the master and the slave.
-#
-# repl-timeout 60
-
-# Disable TCP_NODELAY on the slave socket after SYNC?
-#
-# If you select "yes" Redis will use a smaller number of TCP packets and
-# less bandwidth to send data to slaves. But this can add a delay for
-# the data to appear on the slave side, up to 40 milliseconds with
-# Linux kernels using a default configuration.
-#
-# If you select "no" the delay for data to appear on the slave side will
-# be reduced but more bandwidth will be used for replication.
-#
-# By default we optimize for low latency, but in very high traffic conditions
-# or when the master and slaves are many hops away, turning this to "yes" may
-# be a good idea.
-repl-disable-tcp-nodelay no
-
-# Set the replication backlog size. The backlog is a buffer that accumulates
-# slave data when slaves are disconnected for some time, so that when a slave
-# wants to reconnect again, often a full resync is not needed, but a partial
-# resync is enough, just passing the portion of data the slave missed while
-# disconnected.
-#
-# The biggest the replication backlog, the longer the time the slave can be
-# disconnected and later be able to perform a partial resynchronization.
-#
-# The backlog is only allocated once there is at least a slave connected.
-#
-# repl-backlog-size 1mb
-
-# After a master has no longer connected slaves for some time, the backlog
-# will be freed. The following option configures the amount of seconds that
-# need to elapse, starting from the time the last slave disconnected, for
-# the backlog buffer to be freed.
-#
-# A value of 0 means to never release the backlog.
-#
-# repl-backlog-ttl 3600
-
-# The slave priority is an integer number published by Redis in the INFO output.
-# It is used by Redis Sentinel in order to select a slave to promote into a
-# master if the master is no longer working correctly.
-#
-# A slave with a low priority number is considered better for promotion, so
-# for instance if there are three slaves with priority 10, 100, 25 Sentinel will
-# pick the one with priority 10, that is the lowest.
-#
-# However a special priority of 0 marks the slave as not able to perform the
-# role of master, so a slave with priority of 0 will never be selected by
-# Redis Sentinel for promotion.
-#
-# By default the priority is 100.
-slave-priority 100
-
-# It is possible for a master to stop accepting writes if there are less than
-# N slaves connected, having a lag less or equal than M seconds.
-#
-# The N slaves need to be in "online" state.
-#
-# The lag in seconds, that must be <= the specified value, is calculated from
-# the last ping received from the slave, that is usually sent every second.
-#
-# This option does not GUARANTEES that N replicas will accept the write, but
-# will limit the window of exposure for lost writes in case not enough slaves
-# are available, to the specified number of seconds.
-#
-# For example to require at least 3 slaves with a lag <= 10 seconds use:
-#
-# min-slaves-to-write 3
-# min-slaves-max-lag 10
-#
-# Setting one or the other to 0 disables the feature.
-#
-# By default min-slaves-to-write is set to 0 (feature disabled) and
-# min-slaves-max-lag is set to 10.
-
-################################## SECURITY ###################################
-
-# Require clients to issue AUTH <PASSWORD> before processing any other
-# commands.  This might be useful in environments in which you do not trust
-# others with access to the host running redis-server.
-#
-# This should stay commented out for backward compatibility and because most
-# people do not need auth (e.g. they run their own servers).
-#
-# Warning: since Redis is pretty fast an outside user can try up to
-# 150k passwords per second against a good box. This means that you should
-# use a very strong password otherwise it will be very easy to break.
-#
-# requirepass foobared
-
-# Command renaming.
-#
-# It is possible to change the name of dangerous commands in a shared
-# environment. For instance the CONFIG command may be renamed into something
-# hard to guess so that it will still be available for internal-use tools
-# but not available for general clients.
-#
-# Example:
-#
-# rename-command CONFIG b840fc02d524045429941cc15f59e41cb7be6c52
-#
-# It is also possible to completely kill a command by renaming it into
-# an empty string:
-#
-# rename-command CONFIG ""
-#
-# Please note that changing the name of commands that are logged into the
-# AOF file or transmitted to slaves may cause problems.
-
-################################### LIMITS ####################################
-
-# Set the max number of connected clients at the same time. By default
-# this limit is set to 10000 clients, however if the Redis server is not
-# able to configure the process file limit to allow for the specified limit
-# the max number of allowed clients is set to the current file limit
-# minus 32 (as Redis reserves a few file descriptors for internal uses).
-#
-# Once the limit is reached Redis will close all the new connections sending
-# an error 'max number of clients reached'.
-#
-# maxclients 10000
-
-# Don't use more memory than the specified amount of bytes.
-# When the memory limit is reached Redis will try to remove keys
-# according to the eviction policy selected (see maxmemory-policy).
-#
-# If Redis can't remove keys according to the policy, or if the policy is
-# set to 'noeviction', Redis will start to reply with errors to commands
-# that would use more memory, like SET, LPUSH, and so on, and will continue
-# to reply to read-only commands like GET.
-#
-# This option is usually useful when using Redis as an LRU cache, or to set
-# a hard memory limit for an instance (using the 'noeviction' policy).
-#
-# WARNING: If you have slaves attached to an instance with maxmemory on,
-# the size of the output buffers needed to feed the slaves are subtracted
-# from the used memory count, so that network problems / resyncs will
-# not trigger a loop where keys are evicted, and in turn the output
-# buffer of slaves is full with DELs of keys evicted triggering the deletion
-# of more keys, and so forth until the database is completely emptied.
-#
-# In short... if you have slaves attached it is suggested that you set a lower
-# limit for maxmemory so that there is some free RAM on the system for slave
-# output buffers (but this is not needed if the policy is 'noeviction').
-#
-maxmemory REPLACE_ME
-
-# MAXMEMORY POLICY: how Redis will select what to remove when maxmemory
-# is reached. You can select among five behaviors:
-#
-# volatile-lru -> remove the key with an expire set using an LRU algorithm
-# allkeys-lru -> remove any key accordingly to the LRU algorithm
-# volatile-random -> remove a random key with an expire set
-# allkeys-random -> remove a random key, any key
-# volatile-ttl -> remove the key with the nearest expire time (minor TTL)
-# noeviction -> don't expire at all, just return an error on write operations
-#
-# Note: with any of the above policies, Redis will return an error on write
-#       operations, when there are not suitable keys for eviction.
-#
-#       At the date of writing this commands are: set setnx setex append
-#       incr decr rpush lpush rpushx lpushx linsert lset rpoplpush sadd
-#       sinter sinterstore sunion sunionstore sdiff sdiffstore zadd zincrby
-#       zunionstore zinterstore hset hsetnx hmset hincrby incrby decrby
-#       getset mset msetnx exec sort
-#
-# The default is:
-#
-maxmemory-policy <%= p("redis.maxmemory-policy") %>
-
-# LRU and minimal TTL algorithms are not precise algorithms but approximated
-# algorithms (in order to save memory), so you can select as well the sample
-# size to check. For instance for default Redis will check three keys and
-# pick the one that was used less recently, you can change the sample size
-# using the following configuration directive.
-#
-# maxmemory-samples 3
-
-############################## APPEND ONLY MODE ###############################
-
-# By default Redis asynchronously dumps the dataset on disk. This mode is
-# good enough in many applications, but an issue with the Redis process or
-# a power outage may result into a few minutes of writes lost (depending on
-# the configured save points).
-#
-# The Append Only File is an alternative persistence mode that provides
-# much better durability. For instance using the default data fsync policy
-# (see later in the config file) Redis can lose just one second of writes in a
-# dramatic event like a server power outage, or a single write if something
-# wrong with the Redis process itself happens, but the operating system is
-# still running correctly.
-#
-# AOF and RDB persistence can be enabled at the same time without problems.
-# If the AOF is enabled on startup Redis will load the AOF, that is the file
-# with the better durability guarantees.
-#
-# Please check http://redis.io/topics/persistence for more information.
-
-appendonly <%= p("redis.appendonly") %>
-
-# The name of the append only file (default: "appendonly.aof")
-appendfilename "redis-appendonly.aof"
-
-# The fsync() call tells the Operating System to actually write data on disk
-# instead to wait for more data in the output buffer. Some OS will really flush
-# data on disk, some other OS will just try to do it ASAP.
-#
-# Redis supports three different modes:
-#
-# no: don't fsync, just let the OS flush the data when it wants. Faster.
-# always: fsync after every write to the append only log . Slow, Safest.
-# everysec: fsync only one time every second. Compromise.
-#
-# The default is "everysec", as that's usually the right compromise between
-# speed and data safety. It's up to you to understand if you can relax this to
-# "no" that will let the operating system flush the output buffer when
-# it wants, for better performances (but if you can live with the idea of
-# some data loss consider the default persistence mode that's snapshotting),
-# or on the contrary, use "always" that's very slow but a bit safer than
-# everysec.
-#
-# More details please check the following article:
-# http://antirez.com/post/redis-persistence-demystified.html
-#
-# If unsure, use "everysec".
-
-# appendfsync always
-appendfsync everysec
-# appendfsync no
-
-# When the AOF fsync policy is set to always or everysec, and a background
-# saving process (a background save or AOF log background rewriting) is
-# performing a lot of I/O against the disk, in some Linux configurations
-# Redis may block too long on the fsync() call. Note that there is no fix for
-# this currently, as even performing fsync in a different thread will block
-# our synchronous write(2) call.
-#
-# In order to mitigate this problem it's possible to use the following option
-# that will prevent fsync() from being called in the main process while a
-# BGSAVE or BGREWRITEAOF is in progress.
-#
-# This means that while another child is saving, the durability of Redis is
-# the same as "appendfsync none". In practical terms, this means that it is
-# possible to lose up to 30 seconds of log in the worst scenario (with the
-# default Linux settings).
-#
-# If you have latency problems turn this to "yes". Otherwise leave it as
-# "no" that is the safest pick from the point of view of durability.
-
-no-appendfsync-on-rewrite no
-
-# Automatic rewrite of the append only file.
-# Redis is able to automatically rewrite the log file implicitly calling
-# BGREWRITEAOF when the AOF log size grows by the specified percentage.
-#
-# This is how it works: Redis remembers the size of the AOF file after the
-# latest rewrite (if no rewrite has happened since the restart, the size of
-# the AOF at startup is used).
-#
-# This base size is compared to the current size. If the current size is
-# bigger than the specified percentage, the rewrite is triggered. Also
-# you need to specify a minimal size for the AOF file to be rewritten, this
-# is useful to avoid rewriting the AOF file even if the percentage increase
-# is reached but it is still pretty small.
-#
-# Specify a percentage of zero in order to disable the automatic AOF
-# rewrite feature.
-
-auto-aof-rewrite-percentage 100
-auto-aof-rewrite-min-size 64mb
-
-################################ LUA SCRIPTING  ###############################
-
-# Max execution time of a Lua script in milliseconds.
-#
-# If the maximum execution time is reached Redis will log that a script is
-# still in execution after the maximum allowed time and will start to
-# reply to queries with an error.
-#
-# When a long running script exceed the maximum execution time only the
-# SCRIPT KILL and SHUTDOWN NOSAVE commands are available. The first can be
-# used to stop a script that did not yet called write commands. The second
-# is the only way to shut down the server in the case a write commands was
-# already issue by the script but the user don't want to wait for the natural
-# termination of the script.
-#
-# Set it to 0 or a negative value for unlimited execution without warnings.
-lua-time-limit 5000
-
-################################## SLOW LOG ###################################
-
-# The Redis Slow Log is a system to log queries that exceeded a specified
-# execution time. The execution time does not include the I/O operations
-# like talking with the client, sending the reply and so forth,
-# but just the time needed to actually execute the command (this is the only
-# stage of command execution where the thread is blocked and can not serve
-# other requests in the meantime).
-#
-# You can configure the slow log with two parameters: one tells Redis
-# what is the execution time, in microseconds, to exceed in order for the
-# command to get logged, and the other parameter is the length of the
-# slow log. When a new command is logged the oldest one is removed from the
-# queue of logged commands.
-
-# The following time is expressed in microseconds, so 1000000 is equivalent
-# to one second. Note that a negative number disables the slow log, while
-# a value of zero forces the logging of every command.
-slowlog-log-slower-than 10000
-
-# There is no limit to this length. Just be aware that it will consume memory.
-# You can reclaim memory used by the slow log with SLOWLOG RESET.
-slowlog-max-len 128
-
-############################# Event notification ##############################
-
-# Redis can notify Pub/Sub clients about events happening in the key space.
-# This feature is documented at http://redis.io/topics/keyspace-events
-#
-# For instance if keyspace events notification is enabled, and a client
-# performs a DEL operation on key "foo" stored in the Database 0, two
-# messages will be published via Pub/Sub:
-#
-# PUBLISH __keyspace@0__:foo del
-# PUBLISH __keyevent@0__:del foo
-#
-# It is possible to select the events that Redis will notify among a set
-# of classes. Every class is identified by a single character:
-#
-#  K     Keyspace events, published with __keyspace@<db>__ prefix.
-#  E     Keyevent events, published with __keyevent@<db>__ prefix.
-#  g     Generic commands (non-type specific) like DEL, EXPIRE, RENAME, ...
-#  $     String commands
-#  l     List commands
-#  s     Set commands
-#  h     Hash commands
-#  z     Sorted set commands
-#  x     Expired events (events generated every time a key expires)
-#  e     Evicted events (events generated when a key is evicted for maxmemory)
-#  A     Alias for g$lshzxe, so that the "AKE" string means all the events.
-#
-#  The "notify-keyspace-events" takes as argument a string that is composed
-#  by zero or multiple characters. The empty string means that notifications
-#  are disabled at all.
-#
-#  Example: to enable list and generic events, from the point of view of the
-#           event name, use:
-#
-#  notify-keyspace-events Elg
-#
-#  Example 2: to get the stream of the expired keys subscribing to channel
-#             name __keyevent@0__:expired use:
-#
-#  notify-keyspace-events Ex
-#
-#  By default all notifications are disabled because most users don't need
-#  this feature and the feature has some overhead. Note that if you don't
-#  specify at least one of K or E, no events will be delivered.
-notify-keyspace-events ""
-
-############################### ADVANCED CONFIG ###############################
-
-# Hashes are encoded using a memory efficient data structure when they have a
-# small number of entries, and the biggest entry does not exceed a given
-# threshold. These thresholds can be configured using the following directives.
-hash-max-ziplist-entries 512
-hash-max-ziplist-value 64
-
-# Similarly to hashes, small lists are also encoded in a special way in order
-# to save a lot of space. The special representation is only used when
-# you are under the following limits:
-list-max-ziplist-entries 512
-list-max-ziplist-value 64
-
-# Sets have a special encoding in just one case: when a set is composed
-# of just strings that happens to be integers in radix 10 in the range
-# of 64 bit signed integers.
-# The following configuration setting sets the limit in the size of the
-# set in order to use this special memory saving encoding.
-set-max-intset-entries 512
-
-# Similarly to hashes and lists, sorted sets are also specially encoded in
-# order to save a lot of space. This encoding is only used when the length and
-# elements of a sorted set are below the following limits:
-zset-max-ziplist-entries 128
-zset-max-ziplist-value 64
-
-# HyperLogLog sparse representation bytes limit. The limit includes the
-# 16 bytes header. When an HyperLogLog using the sparse representation crosses
-# this limit, it is converted into the dense representation.
-#
-# A value greater than 16000 is totally useless, since at that point the
-# dense representation is more memory efficient.
-#
-# The suggested value is ~ 3000 in order to have the benefits of
-# the space efficient encoding without slowing down too much PFADD,
-# which is O(N) with the sparse encoding. The value can be raised to
-# ~ 10000 when CPU is not a concern, but space is, and the data set is
-# composed of many HyperLogLogs with cardinality in the 0 - 15000 range.
-#hll-sparse-max-bytes 3000
-
-# Active rehashing uses 1 millisecond every 100 milliseconds of CPU time in
-# order to help rehashing the main Redis hash table (the one mapping top-level
-# keys to values). The hash table implementation Redis uses (see dict.c)
-# performs a lazy rehashing: the more operation you run into a hash table
-# that is rehashing, the more rehashing "steps" are performed, so if the
-# server is idle the rehashing is never complete and some more memory is used
-# by the hash table.
-#
-# The default is to use this millisecond 10 times every second in order to
-# active rehashing the main dictionaries, freeing memory when possible.
-#
-# If unsure:
-# use "activerehashing no" if you have hard latency requirements and it is
-# not a good thing in your environment that Redis can reply form time to time
-# to queries with 2 milliseconds delay.
-#
-# use "activerehashing yes" if you don't have such hard requirements but
-# want to free memory asap when possible.
-activerehashing yes
-
-# The client output buffer limits can be used to force disconnection of clients
-# that are not reading data from the server fast enough for some reason (a
-# common reason is that a Pub/Sub client can't consume messages as fast as the
-# publisher can produce them).
-#
-# The limit can be set differently for the three different classes of clients:
-#
-# normal -> normal clients
-# slave  -> slave clients and MONITOR clients
-# pubsub -> clients subscribed to at least one pubsub channel or pattern
-#
-# The syntax of every client-output-buffer-limit directive is the following:
-#
-# client-output-buffer-limit <class> <hard limit> <soft limit> <soft seconds>
-#
-# A client is immediately disconnected once the hard limit is reached, or if
-# the soft limit is reached and remains reached for the specified number of
-# seconds (continuously).
-# So for instance if the hard limit is 32 megabytes and the soft limit is
-# 16 megabytes / 10 seconds, the client will get disconnected immediately
-# if the size of the output buffers reach 32 megabytes, but will also get
-# disconnected if the client reaches 16 megabytes and continuously overcomes
-# the limit for 10 seconds.
-#
-# By default normal clients are not limited because they don't receive data
-# without asking (in a push way), but just after a request, so only
-# asynchronous clients may create a scenario where data is requested faster
-# than it can read.
-#
-# Instead there is a default limit for pubsub and slave clients, since
-# subscribers and slaves receive data in a push fashion.
-#
-# Both the hard or the soft limit can be disabled by setting them to zero.
-client-output-buffer-limit normal 0 0 0
-client-output-buffer-limit slave 256mb 64mb 60
-client-output-buffer-limit pubsub 32mb 8mb 60
-
-# Redis calls an internal function to perform many background tasks, like
-# closing connections of clients in timeout, purging expired keys that are
-# never requested, and so forth.
-#
-# Not all tasks are performed with the same frequency, but Redis checks for
-# tasks to perform accordingly to the specified "hz" value.
-#
-# By default "hz" is set to 10. Raising the value will use more CPU when
-# Redis is idle, but at the same time will make Redis more responsive when
-# there are many keys expiring at the same time, and timeouts may be
-# handled with more precision.
-#
-# The range is between 1 and 500, however a value over 100 is usually not
-# a good idea. Most users should use the default of 10 and raise this up to
-# 100 only in environments where very low latency is required.
-hz 10
-
-# When a child rewrites the AOF file, if the following option is enabled
-# the file will be fsync-ed every 32 MB of data generated. This is useful
-# in order to commit the file to the disk more incrementally and avoid
-# big latency spikes.
-aof-rewrite-incremental-fsync yes
diff --git a/jobs/queue/templates/data/properties.sh.erb b/jobs/queue/templates/data/properties.sh.erb
deleted file mode 100644
index 51396068..00000000
--- a/jobs/queue/templates/data/properties.sh.erb
+++ /dev/null
@@ -1,10 +0,0 @@
-#!/usr/bin/env bash
-
-# job template binding variables
-
-# job name & index of this VM within cluster
-# e.g. JOB_NAME=queue, JOB_INDEX=0
-export NAME='<%= name %>'
-export JOB_INDEX=<%= index %>
-# full job name, like queue/0 or webapp/3
-export JOB_FULL="$NAME/$JOB_INDEX"
diff --git a/jobs/queue/templates/helpers/ctl_setup.sh b/jobs/queue/templates/helpers/ctl_setup.sh
deleted file mode 100644
index c93ad280..00000000
--- a/jobs/queue/templates/helpers/ctl_setup.sh
+++ /dev/null
@@ -1,81 +0,0 @@
-#!/usr/bin/env bash
-
-# Setup env vars and folders for the ctl script
-# This helps keep the ctl script as readable
-# as possible
-
-# Usage options:
-# source /var/vcap/jobs/foobar/helpers/ctl_setup.sh JOB_NAME OUTPUT_LABEL
-# source /var/vcap/jobs/foobar/helpers/ctl_setup.sh foobar
-# source /var/vcap/jobs/foobar/helpers/ctl_setup.sh foobar foobar
-# source /var/vcap/jobs/foobar/helpers/ctl_setup.sh foobar nginx
-
-set -e # exit immediately if a simple command exits with a non-zero status
-set -u # report the usage of uninitialized variables
-
-JOB_NAME=$1
-output_label=${1:-JOB_NAME}
-
-export JOB_DIR=/var/vcap/jobs/$JOB_NAME
-chmod 755 $JOB_DIR # to access file via symlink
-
-# Load some bosh deployment properties into env vars
-# Try to put all ERb into data/properties.sh.erb
-# incl $NAME, $JOB_INDEX, $WEBAPP_DIR
-source $JOB_DIR/data/properties.sh
-
-source $JOB_DIR/helpers/ctl_utils.sh
-redirect_output ${output_label}
-
-export HOME=${HOME:-/home/vcap}
-
-# Add all packages' /bin & /sbin into $PATH
-for package_bin_dir in $(ls -d /var/vcap/packages/*/*bin)
-do
-  export PATH=${package_bin_dir}:$PATH
-done
-
-export LD_LIBRARY_PATH=${LD_LIBRARY_PATH:-''} # default to empty
-for package_bin_dir in $(ls -d /var/vcap/packages/*/lib)
-do
-  export LD_LIBRARY_PATH=${package_bin_dir}:$LD_LIBRARY_PATH
-done
-
-# Setup log, run and tmp folders
-
-export RUN_DIR=/var/vcap/sys/run/$JOB_NAME
-export LOG_DIR=/var/vcap/sys/log/$JOB_NAME
-export TMP_DIR=/var/vcap/sys/tmp/$JOB_NAME
-export STORE_DIR=/var/vcap/store/$JOB_NAME
-for dir in $RUN_DIR $LOG_DIR $TMP_DIR $STORE_DIR
-do
-  mkdir -p ${dir}
-  chown vcap:vcap ${dir}
-  chmod 775 ${dir}
-done
-export TMPDIR=$TMP_DIR
-
-export C_INCLUDE_PATH=/var/vcap/packages/mysqlclient/include/mysql:/var/vcap/packages/sqlite/include:/var/vcap/packages/libpq/include
-export LIBRARY_PATH=/var/vcap/packages/mysqlclient/lib/mysql:/var/vcap/packages/sqlite/lib:/var/vcap/packages/libpq/lib
-
-# consistent place for vendoring python libraries within package
-if [[ -d ${WEBAPP_DIR:-/xxxx} ]]
-then
-  export PYTHONPATH=$WEBAPP_DIR/vendor/lib/python
-fi
-
-if [[ -d /var/vcap/packages/java7 ]]
-then
-  export JAVA_HOME="/var/vcap/packages/java7"
-fi
-
-# setup CLASSPATH for all jars/ folders within packages
-export CLASSPATH=${CLASSPATH:-''} # default to empty
-for java_jar in $(ls -d /var/vcap/packages/*/*/*.jar)
-do
-  export CLASSPATH=${java_jar}:$CLASSPATH
-done
-
-PIDFILE=$RUN_DIR/$JOB_NAME.pid
-
-echo '$PATH' $PATH
diff --git a/jobs/queue/templates/helpers/ctl_utils.sh b/jobs/queue/templates/helpers/ctl_utils.sh
deleted file mode 100644
index a39b485a..00000000
--- a/jobs/queue/templates/helpers/ctl_utils.sh
+++ /dev/null
@@ -1,182 +0,0 @@
-# Helper functions used by ctl scripts
-
-# links a job file (probably a config file) into a package
-# Example usage:
-# link_job_file_to_package config/redis.yml [config/redis.yml]
-# link_job_file_to_package config/wp-config.php wp-config.php
-link_job_file_to_package() {
-  source_job_file=$1
-  target_package_file=${2:-$source_job_file}
-  full_package_file=$WEBAPP_DIR/${target_package_file}
-
-  link_job_file ${source_job_file} ${full_package_file}
-}
-
-# links a job file (probably a config file) somewhere
-# Example usage:
-# link_job_file config/bashrc /home/vcap/.bashrc
-link_job_file() {
-  source_job_file=$1
-  target_file=$2
-  full_job_file=$JOB_DIR/${source_job_file}
-
-  echo link_job_file ${full_job_file} ${target_file}
-  if [[ ! -f ${full_job_file} ]]
-  then
-    echo "file to link ${full_job_file} does not exist"
-  else
-    # Create/recreate the symlink to current job file
-    # If another process is using the file, it won't be
-    # deleted, so don't attempt to create the symlink
-    mkdir -p $(dirname ${target_file})
-    ln -nfs ${full_job_file} ${target_file}
-  fi
-}
-
-# If loaded within monit ctl scripts then pipe output
-# If loaded from 'source ../utils.sh' then normal STDOUT
-redirect_output() {
-  SCRIPT=$1
-  mkdir -p /var/vcap/sys/log/monit
-  exec 1>> /var/vcap/sys/log/monit/$SCRIPT.log
-  exec 2>> /var/vcap/sys/log/monit/$SCRIPT.err.log
-}
-
-function pid_is_running() {
-  declare pid="$1"
-  ps -p "${pid}" >/dev/null 2>&1
-}
-
-# pid_guard
-#
-# @param pidfile
-# @param name [String] an arbitrary name that might show up in STDOUT on errors
-#
-# Run this before attempting to start new processes that may use the same :pidfile:.
-# If an old process is running on the pid found in the :pidfile:, exit 1. Otherwise,
-# remove the stale :pidfile: if it exists.
-#
-function pid_guard() {
-  declare pidfile="$1" name="$2"
-
-  echo "------------ STARTING $(basename "$0") at $(date) --------------" | tee /dev/stderr
-
-  if [ ! -f "${pidfile}" ]; then
-    return 0
-  fi
-
-  local pid
-  pid=$(head -1 "${pidfile}")
-
-  if pid_is_running "${pid}"; then
-    echo "${name} is already running, please stop it first"
-    exit 1
-  fi
-
-  echo "Removing stale pidfile"
-  rm "${pidfile}"
-}
-
-# wait_pid_death
-#
-# @param pid
-# @param timeout
-#
-# Watch a :pid: for :timeout: seconds, waiting for it to die.
-# If it dies before :timeout:, exit 0. If not, exit 1.
-#
-# Note that this should be run in a subshell, so that the current
-# shell does not exit.
-#
-function wait_pid_death() {
-  declare pid="$1" timeout="$2"
-
-  local countdown
-  countdown=$(( timeout * 10 ))
-
-  while true; do
-    if ! pid_is_running "${pid}"; then
-      return 0
-    fi
-
-    if [ ${countdown} -le 0 ]; then
-      return 1
-    fi
-
-    countdown=$(( countdown - 1 ))
-    sleep 0.1
-  done
-}
-
-# kill_and_wait
-#
-# @param pidfile
-# @param timeout [default 25s]
-#
-# For a pid found in :pidfile:, send a `kill -15` TERM, then wait for :timeout: seconds to
-# see if it dies on its own. If not, send it a `kill -9`. If the process does die,
-# exit 0 and remove the :pidfile:. If after all of this, the process does not actually
-# die, exit 1.
-#
-# Note:
-# Monit default timeout for start/stop is 30s
-# Append 'with timeout {n} seconds' to monit start/stop program configs
-#
-function kill_and_wait() {
-  declare pidfile="$1" timeout="${2:-25}" sigkill_on_timeout="${3:-1}"
-
-  if [ ! -f "${pidfile}" ]; then
-    echo "Pidfile ${pidfile} doesn't exist"
-    exit 0
-  fi
-
-  local pid
-  pid=$(head -1 "${pidfile}")
-
-  if [ -z "${pid}" ]; then
-    echo "Unable to get pid from ${pidfile}"
-    exit 1
-  fi
-
-  if ! pid_is_running "${pid}"; then
-    echo "Process ${pid} is not running"
-    rm -f "${pidfile}"
-    exit 0
-  fi
-
-  echo "Killing ${pidfile}: ${pid} "
-  kill "${pid}"
-
-  if ! wait_pid_death "${pid}" "${timeout}"; then
-    if [ "${sigkill_on_timeout}" = "1" ]; then
-      echo "Kill timed out, using kill -9 on ${pid}"
-      kill -9 "${pid}"
-      sleep 0.5
-    fi
-  fi
-
-  if pid_is_running "${pid}"; then
-    echo "Timed Out"
-    exit 1
-  else
-    echo "Stopped"
-    rm -f "${pidfile}"
-  fi
-}
-
-check_nfs_mount() {
-  opts=$1
-  exports=$2
-  mount_point=$3
-
-  if grep -qs $mount_point /proc/mounts; then
-    echo "Found NFS mount $mount_point"
-  else
-    echo "Mounting NFS..."
-    mount $opts $exports $mount_point
-    if [ $? != 0 ]; then
-      echo "Cannot mount NFS from $exports to $mount_point, exiting..."
-      exit 1
-    fi
-  fi
-}
diff --git a/jobs/queue/templates/logsearch/metric-collector/redis/collector b/jobs/queue/templates/logsearch/metric-collector/redis/collector
deleted file mode 100644
index 81ed0e8f..00000000
--- a/jobs/queue/templates/logsearch/metric-collector/redis/collector
+++ /dev/null
@@ -1,65 +0,0 @@
-#!/var/vcap/bosh/bin/ruby
-
-require 'socket'
-
-STDOUT.sync = true
-
-freq = ENV.has_key?('METRIC_FREQUENCY') ? ENV['METRIC_FREQUENCY'].to_i : nil
-
-useful_info = [
-  'uptime_in_seconds',
-  'connected_clients',
-  'connected_slaves',
-  'blocked_clients',
-  'evicted_keys',
-  'used_memory',
-  'changes_since_last_save',
-  'total_connections_received',
-  'total_commands_processed',
-]
-
-while true
-  ts = Time.now.utc.to_i
-
-  sh = TCPSocket.new('127.0.0.1', 6379)
-
-
-  sh.write("llen <%= p('redis.key') %>\r\n")
-
-  # :{size}
-  puts "logstash.queue_size #{sh.gets()[1..-1].chop} #{ts}"
-
-
-  sh.write("llen <%= p('redis.key') %>_archiver\r\n")
-
-  # :{size}
-  puts "logstash.archiver_queue_size #{sh.gets()[1..-1].chop} #{ts}"
-
-
-  sh.write("info\r\n")
-
-  # ${length}
-  len = sh.gets()[1..-1].chop.to_i
-  result = ""
-
-  while result.length < len
-    result << sh.gets
-  end
-
-  result.scan /^[a-zA-Z0-9_]+:.*$/ do | match |
-    k, v = match.chop.split(':')
-
-    puts "redis.#{k} #{v} #{ts}" if useful_info.include? k
-  end
-
-  sh.close
-
-
-  exit if freq.nil?
-
-  begin
-    sleep(freq)
-  rescue Interrupt => e
-    exit
-  end
-end
diff --git a/packages/redis/packaging b/packages/redis/packaging
deleted file mode 100644
index 1c369647..00000000
--- a/packages/redis/packaging
+++ /dev/null
@@ -1,15 +0,0 @@
-set -e # exit immediately if a simple command exits with a non-zero status
-set -u # report the usage of uninitialized variables
-
-# Detect # of CPUs so make jobs can be parallelized
-CPUS=`grep -c ^processor /proc/cpuinfo`
- # Available variables
-# $BOSH_COMPILE_TARGET - where this package & spec'd source files are available
-# $BOSH_INSTALL_TARGET - where you copy/install files to be included in package
-
-REDIS_VERSION=3.2.9
-
-tar xzf redis/redis-${REDIS_VERSION}.tar.gz
-cd redis-${REDIS_VERSION}
-make -j${CPUS} PREFIX=${BOSH_INSTALL_TARGET} install
-
diff --git a/packages/redis/spec b/packages/redis/spec
deleted file mode 100644
index 692fdf59..00000000
--- a/packages/redis/spec
+++ /dev/null
@@ -1,5 +0,0 @@
----
-name: redis
-dependencies: []
-files:
-  - redis/redis-3.2.9.tar.gz

From 394a08154e53c79b2a97cf688b23d279fcbfe9d3 Mon Sep 17 00:00:00 2001
From: Andrei Krasnitski <me@infra-red.xyz>
Date: Wed, 26 Jul 2017 09:42:03 +0300
Subject: [PATCH 05/19] Bump Elastic stack to 5.5.1

---
 config/blobs.yml                  | 48 +++++++++++++++----------------
 packages/elasticsearch/easyupdate | 24 ----------------
 packages/elasticsearch/spec       |  2 +-
 packages/kibana/packaging         |  2 +-
 packages/kibana/spec              |  2 +-
 packages/logstash/README.md       | 34 ----------------------
 packages/logstash/packaging       | 18 ++++--------
 packages/logstash/spec            | 10 ++++---
 8 files changed, 39 insertions(+), 101 deletions(-)
 delete mode 100755 packages/elasticsearch/easyupdate
 delete mode 100644 packages/logstash/README.md

diff --git a/config/blobs.yml b/config/blobs.yml
index 96a5b365..a44bafb5 100644
--- a/config/blobs.yml
+++ b/config/blobs.yml
@@ -50,10 +50,10 @@ curator/vendor/voluptuous-0.9.3.tar.gz:
   size: 34097
   object_id: 2a246c17-66fb-4c0c-8095-0c5bc6e0029f
   sha: 315dd79c372380799d0d2e21ae711aa4097efb81
-elasticsearch/elasticsearch-5.5.0.tar.gz:
-  size: 33480586
-  object_id: 9267a5ae-fa79-405b-53e2-38746af2785c
-  sha: d79a3ade8b8589d13aeb99ceec1c54683596e88b
+elasticsearch/elasticsearch-5.5.1.tar.gz:
+  size: 33476827
+  object_id: 7dc3b18b-ddc5-4985-7801-9479bfd2da3d
+  sha: cb5648a6723527bdccee98caef7a9d08885a5b0f
 haproxy/haproxy-1.7.5.tar.gz:
   size: 1743979
   object_id: 27004bb4-29f6-493f-80ec-6a4170a955d5
@@ -66,26 +66,26 @@ java8/openjdk-1.8.0_141.tar.gz:
   size: 45927906
   object_id: e9caffab-2961-4a26-7428-e2c03b87c3ec
   sha: fef57709d2ca0b375a3879fe03df303afe24307f
-kibana/kibana-5.5.0-linux-x86_64.tar.gz:
-  size: 51068789
-  object_id: 6db7656b-47b5-4e40-6dd8-83e0929f092b
-  sha: 935e925713cb84eb1879a59ac68708fccf3361d4
-logstash/logstash-5.5.0.tar.gz:
-  size: 94906235
-  object_id: be68fa2d-2f93-41be-60f3-4676bfc4cf8b
-  sha: d380e8cdefb4c0d5291d0243b8ab5a9f20811826
-logstash/logstash-filter-alter-3.0.0.zip:
-  size: 6642
-  object_id: 8b0c5aa7-4675-4659-8b0a-78ebb6cd6b0c
-  sha: f7169d5bc42e99c3a89afadd5a838c4e4e0105ab
-logstash/logstash-filter-translate-3.0.1.zip:
-  size: 9437
-  object_id: 33680669-f431-43ac-9b83-c1b995617f47
-  sha: e01978407ca5a4f4a9f48d1fd04f8ad3e721e679
-logstash/logstash-input-relp-3.0.0.zip:
-  size: 11685
-  object_id: f96081d6-00d5-4689-99cc-c752cf3806c5
-  sha: 5805f45e5d3848cb45ed2163f7809e53364bbfb6
+kibana/kibana-5.5.1-linux-x86_64.tar.gz:
+  size: 50952073
+  object_id: cc9c5bb8-fa83-48fe-4863-7770776223be
+  sha: 6dba24c876841fdf116a413c843f09d3e98b4002
+logstash/logstash-5.5.1.tar.gz:
+  size: 93242602
+  object_id: 89e3ac08-9b05-46c1-573b-53d606d93343
+  sha: 2cd1cdc1c01f7e7af3919221668e9ec2547070be
+logstash/logstash-filter-alter-3.0.1.zip:
+  size: 7405
+  object_id: ddfd43a9-0585-4c0e-7109-37091ea3758f
+  sha: bd51998f9e5690d5e9fadf51ed0961ec468c15ad
+logstash/logstash-filter-translate-3.0.2.zip:
+  size: 10656
+  object_id: 45f289ea-42e1-48c0-5ab8-777cc1559556
+  sha: 2ab763179bc0fdae291f9a8facd4274fcad0fbd3
+logstash/logstash-input-relp-3.0.1.zip:
+  size: 12587
+  object_id: 29c87e17-136f-4841-5ff2-7c7ba67a8836
+  sha: c4bc33af867abf6502cfeb1e87547282b99dd47e
 nats_to_syslog/nats_to_syslog_linux_amd64.tar.gz:
   size: 2094679
   object_id: 5d5d32dc-946c-4a27-9786-df4689706cdb
diff --git a/packages/elasticsearch/easyupdate b/packages/elasticsearch/easyupdate
deleted file mode 100755
index 488d23a4..00000000
--- a/packages/elasticsearch/easyupdate
+++ /dev/null
@@ -1,24 +0,0 @@
-#!/bin/bash
-
-set -e
-set -u
-
-# find the latest version
-VERSION=$( wget -qO- https://api.github.com/repos/elastic/elasticsearch/tags?per_page=32 | jq -r '.[].name' | grep -E '^v\d+\.\d+\.\d+$' | sed -e 's/^v//' | head -n1 )
-
-if [ ! -e "blobs/elasticsearch/elasticsearch-${VERSION}.tar.gz" ] ; then
-  # if we aren't already using this version
-
-  # download
-  wget -O "blobs/elasticsearch/elasticsearch-${VERSION}.tar.gz" "https://download.elastic.co/elasticsearch/elasticsearch/elasticsearch-${VERSION}.tar.gz"
-
-  # update references
-  find packages/elasticsearch -type f -exec sed -E -i '' -e "s/elasticsearch-[0-9]\.[0-9]\.[0-9]/elasticsearch-${VERSION}/" {} \;
-
-  # upload the new blob
-  bosh -n upload blobs
-
-  # commit
-  git add packages/elasticsearch/packaging packages/elasticsearch/spec config/blobs.yml
-  git commit -m "Upgrade elasticsearch to ${VERSION}"
-fi
diff --git a/packages/elasticsearch/spec b/packages/elasticsearch/spec
index fcdbc412..6648fb02 100644
--- a/packages/elasticsearch/spec
+++ b/packages/elasticsearch/spec
@@ -2,4 +2,4 @@
 name: elasticsearch
 
 files:
-  - elasticsearch/elasticsearch-5.5.0.tar.gz
+  - elasticsearch/elasticsearch-5.5.1.tar.gz
diff --git a/packages/kibana/packaging b/packages/kibana/packaging
index 9057a991..ead637e0 100644
--- a/packages/kibana/packaging
+++ b/packages/kibana/packaging
@@ -1,3 +1,3 @@
 set -e
 
-tar -xzf kibana/kibana-*-linux-x86_64.tar.gz -C $BOSH_INSTALL_TARGET --strip-components 1
+tar xzf kibana/kibana-*-linux-x86_64.tar.gz -C $BOSH_INSTALL_TARGET --strip-components 1
diff --git a/packages/kibana/spec b/packages/kibana/spec
index 785c63e4..b49dfe50 100644
--- a/packages/kibana/spec
+++ b/packages/kibana/spec
@@ -2,4 +2,4 @@
 name: kibana
 
 files:
-  - kibana/kibana-5.5.0-linux-x86_64.tar.gz
+  - kibana/kibana-5.5.1-linux-x86_64.tar.gz
diff --git a/packages/logstash/README.md b/packages/logstash/README.md
deleted file mode 100644
index 8f79b8a1..00000000
--- a/packages/logstash/README.md
+++ /dev/null
@@ -1,34 +0,0 @@
-## How to remove gems from logstash
-
-### 1. Download logstash plugins with plugins bundle
-
-```
-wget https://download.elastic.co/logstash/logstash/logstash-all-plugins-2.3.3.tar.gz
-tar -xzf logstash-all-plugins-2.3.3.tar.gz
-cd logstash-2.3.3
-```
-
-### 2. Delete the following gems from `Gemfile`
-```
-gem "logstash-input-wmi" # licensing issues with dependency
-gem "logstash-output-zookeeper" # licensing issues with dependency
-```
-
-### 3. Delete the code we don't want in the tarball
-```
-rm -rf vendor/bundle/jruby/1.9/gems/jruby-win32ole*
-rm -rf vendor/bundle/jruby/1.9/gems/slyphon-zookeeper_jar*
-```
-
-### 4. Execute logstash so bundle can update the lock file
-
-```
-bin/logstash
-```
-
-### 5. Create the new tarball
-
-```
-cd ..
-tar -czf logstash-2.3.3-patched.tar.gz logstash-2.3.3/
-```
diff --git a/packages/logstash/packaging b/packages/logstash/packaging
index 558a7338..e5a018ce 100644
--- a/packages/logstash/packaging
+++ b/packages/logstash/packaging
@@ -1,16 +1,10 @@
-set -e # exit immediately if a simple command exits with a non-zero status
-set -u # report the usage of uninitialized variables
-
-# Available variables
-# $BOSH_COMPILE_TARGET - where this package & spec'd source files are available
-# $BOSH_INSTALL_TARGET - where you copy/install files to be included in package
+set -e
 
 export JAVA_HOME=/var/vcap/packages/java8
 
-mkdir $BOSH_INSTALL_TARGET/logstash
-tar -xzf logstash/logstash-*.tar.gz -C $BOSH_INSTALL_TARGET --strip-components 1
+tar xzf logstash/logstash-*.tar.gz -C $BOSH_INSTALL_TARGET --strip-components 1
 
-# Installs missing filters
-$BOSH_INSTALL_TARGET/bin/logstash-plugin install file://$PWD/logstash/logstash-filter-alter-3.0.0.zip
-$BOSH_INSTALL_TARGET/bin/logstash-plugin install file://$PWD/logstash/logstash-input-relp-3.0.0.zip
-$BOSH_INSTALL_TARGET/bin/logstash-plugin install file://$PWD/logstash/logstash-filter-translate-3.0.1.zip
+# Installs missing plugins
+$BOSH_INSTALL_TARGET/bin/logstash-plugin install file://$PWD/logstash/logstash-filter-alter-3.0.1.zip
+$BOSH_INSTALL_TARGET/bin/logstash-plugin install file://$PWD/logstash/logstash-input-relp-3.0.1.zip
+$BOSH_INSTALL_TARGET/bin/logstash-plugin install file://$PWD/logstash/logstash-filter-translate-3.0.2.zip
diff --git a/packages/logstash/spec b/packages/logstash/spec
index 980b3226..fa9e7c01 100644
--- a/packages/logstash/spec
+++ b/packages/logstash/spec
@@ -1,9 +1,11 @@
 ---
 name: logstash
+
 dependencies:
   - java8
+
 files:
-  - logstash/logstash-5.5.0.tar.gz
-  - logstash/logstash-filter-alter-3.0.0.zip
-  - logstash/logstash-filter-translate-3.0.1.zip
-  - logstash/logstash-input-relp-3.0.0.zip
+  - logstash/logstash-5.5.1.tar.gz
+  - logstash/logstash-filter-alter-3.0.1.zip
+  - logstash/logstash-filter-translate-3.0.2.zip
+  - logstash/logstash-input-relp-3.0.1.zip

From 8b5134e4b0586bea0b41d7be7189159b241d6391 Mon Sep 17 00:00:00 2001
From: Andrei Krasnitski <me@infra-red.xyz>
Date: Fri, 28 Jul 2017 11:41:17 +0300
Subject: [PATCH 06/19] Bump Cerebro to v0.6.6.

---
 config/blobs.yml           |  8 ++++----
 packages/cerebro/packaging | 13 ++-----------
 packages/cerebro/spec      |  4 ++--
 3 files changed, 8 insertions(+), 17 deletions(-)

diff --git a/config/blobs.yml b/config/blobs.yml
index a44bafb5..3f03f654 100644
--- a/config/blobs.yml
+++ b/config/blobs.yml
@@ -1,7 +1,7 @@
-cerebro/cerebro-0.6.5.tgz:
-  size: 51211480
-  object_id: 7a7e401f-ba46-4043-6226-eda015b49a49
-  sha: d575cb6816db88653981ad2871bce39709718569
+cerebro/cerebro-0.6.6.tgz:
+  size: 51199362
+  object_id: fda10e6d-aeba-47b6-6f0d-fa313cffc109
+  sha: 69bc2aafb2c5966163c44b19f2768eb9f497aac3
 curator/elasticsearch-curator-5.0.1.tar.gz:
   size: 182112
   object_id: a66f8365-c0b1-41ec-bfdb-d7a6037e531d
diff --git a/packages/cerebro/packaging b/packages/cerebro/packaging
index b2ddcc14..cf7dc4f8 100644
--- a/packages/cerebro/packaging
+++ b/packages/cerebro/packaging
@@ -1,12 +1,3 @@
-set -e # exit immediately if a simple command exits with a non-zero status
-set -u # report the usage of uninitialized variables
+set -e
 
-# Detect # of CPUs so make jobs can be parallelized
-CPUS=$(grep -c ^processor /proc/cpuinfo)
- # Available variables
-# $BOSH_COMPILE_TARGET - where this package & spec'd source files are available
-# $BOSH_INSTALL_TARGET - where you copy/install files to be included in package
-
-CEREBRO_VERSION=0.6.5
-
-tar xzf cerebro/cerebro-${CEREBRO_VERSION}.tgz -C ${BOSH_INSTALL_TARGET} --strip-components 1
+tar xzf cerebro/cerebro-0.6.6.tgz -C ${BOSH_INSTALL_TARGET} --strip-components 1
diff --git a/packages/cerebro/spec b/packages/cerebro/spec
index 9dc1ce97..90840acd 100644
--- a/packages/cerebro/spec
+++ b/packages/cerebro/spec
@@ -1,5 +1,5 @@
 ---
 name: cerebro
-dependencies: []
+
 files:
-  - cerebro/cerebro-0.6.5.tgz
+  - cerebro/cerebro-0.6.6.tgz

From 5f20cc39dd0f7a20f2110d0662e4460ec674db1e Mon Sep 17 00:00:00 2001
From: Andrei Krasnitski <me@infra-red.xyz>
Date: Fri, 28 Jul 2017 11:41:57 +0300
Subject: [PATCH 07/19] Simplify elasticsearch link check in Curator run script

---
 jobs/curator/templates/bin/run | 9 +++++----
 1 file changed, 5 insertions(+), 4 deletions(-)

diff --git a/jobs/curator/templates/bin/run b/jobs/curator/templates/bin/run
index 3a3cd80a..eb146e9d 100644
--- a/jobs/curator/templates/bin/run
+++ b/jobs/curator/templates/bin/run
@@ -4,13 +4,14 @@ set -e
 
 <%
   elasticsearch_host = nil
-  if_link("elasticsearch") { |elasticsearch_link| elasticsearch_host = elasticsearch_link.instances.first.address }
+  elasticsearch_port = nil
+  if_link("elasticsearch") { |elasticsearch_link|
+    elasticsearch_host = elasticsearch_link.instances.first.address
+    elasticsearch_port = elasticsearch_link.p("elasticsearch.port")
+  }
   unless elasticsearch_host
     elasticsearch_host = p("curator.elasticsearch.host")
   end
-
-  elasticsearch_port = nil
-  if_link("elasticsearch") { |elasticsearch_link| elasticsearch_port = elasticsearch_link.p("elasticsearch.port") }
   unless elasticsearch_port
     elasticsearch_port = p("curator.elasticsearch.port")
   end

From 7f6c8e52d42f272b0e518b41529474104bbc61c9 Mon Sep 17 00:00:00 2001
From: Andrei Krasnitski <me@infra-red.xyz>
Date: Fri, 28 Jul 2017 15:34:34 +0300
Subject: [PATCH 08/19] Bump elasticsearch-curator to 5.1.1.

---
 config/blobs.yml                       | 64 ++++++++++----------------
 jobs/curator/templates/bin/curator_ctl |  4 +-
 packages/curator/packaging             |  2 +-
 packages/curator/spec                  | 23 ++++-----
 packages/python3/packaging             |  9 ++--
 packages/python3/spec                  |  4 +-
 6 files changed, 41 insertions(+), 65 deletions(-)

diff --git a/config/blobs.yml b/config/blobs.yml
index 3f03f654..0d421cc4 100644
--- a/config/blobs.yml
+++ b/config/blobs.yml
@@ -2,54 +2,34 @@ cerebro/cerebro-0.6.6.tgz:
   size: 51199362
   object_id: fda10e6d-aeba-47b6-6f0d-fa313cffc109
   sha: 69bc2aafb2c5966163c44b19f2768eb9f497aac3
-curator/elasticsearch-curator-5.0.1.tar.gz:
-  size: 182112
-  object_id: a66f8365-c0b1-41ec-bfdb-d7a6037e531d
-  sha: 183924cfc48e578f9c8f94862db3ca222bd976ba
+curator/elasticsearch-curator-5.1.1.tar.gz:
+  size: 191710
+  object_id: 306cc6fb-4693-42b9-52d5-1f65e6ed1f1f
+  sha: cd584da85f3bf346dc8eceef91ae72b7eeeaf2a9
 curator/vendor/PyYAML-3.12.tar.gz:
   size: 253011
   object_id: c437dff0-e2db-4d48-9c12-626160760901
   sha: cb7fd3e58c129494ee86e41baedfec69eb7dafbe
-curator/vendor/appdirs-1.4.3-py2.py3-none-any.whl:
-  size: 12154
-  object_id: dc6228ab-aab1-457b-bb61-ea6bbab11b8c
-  sha: 092e43d970e08a67c4f8074bf7777d865f92f5fc
-curator/vendor/certifi-2017.1.23-py2.py3-none-any.whl:
-  size: 382984
-  object_id: bcd789e3-5a1c-478e-aaae-74be9b3e84f1
-  sha: fa81df1779db594a994b21bfd478982529ca5369
+curator/vendor/certifi-2017.7.27.1-py2.py3-none-any.whl:
+  size: 349554
+  object_id: b3ad6e0e-1667-448e-57a8-f098cc499ac8
+  sha: aeb9761eb1231d8309a0b593747d0c95303467b5
 curator/vendor/click-6.7-py2.py3-none-any.whl:
   size: 71175
   object_id: 06f97a3a-8cc3-4d83-8c0f-6e85d15167f9
   sha: 9490775172e63ba94233d4b201f0e8d78b0cd939
-curator/vendor/elasticsearch-5.3.0-py2.py3-none-any.whl:
-  size: 66169
-  object_id: 9b191721-a07d-4bc6-bf57-391995451264
-  sha: 9ce33ca6508c263db6e3833273089fde431e82e1
-curator/vendor/packaging-16.8-py2.py3-none-any.whl:
-  size: 23069
-  object_id: 5ae6809e-5e93-4531-aa1d-989e81dda9d6
-  sha: 6c45e475be475c385ef46f583a0214dea6a35783
-curator/vendor/pyparsing-2.2.0-py2.py3-none-any.whl:
-  size: 56385
-  object_id: a7dd692a-9de5-44ab-93e6-9095bc5c734e
-  sha: b698b51e3680761ba520149ed7b0d3a3d6a0c0ca
-curator/vendor/setuptools-34.4.1-py2.py3-none-any.whl:
-  size: 390260
-  object_id: 54d3b5da-3c8e-4e78-bf23-93b54e23e008
-  sha: 9fec5bf7062298bc901b79ab9db88fc05ddb2a3d
-curator/vendor/six-1.10.0-py2.py3-none-any.whl:
-  size: 10341
-  object_id: efb0fe0f-863b-4f27-81ad-15c46bdb8f19
-  sha: 19df5f56abce3db94d5e0e079e54bf430fddbb02
-curator/vendor/urllib3-1.20-py2.py3-none-any.whl:
-  size: 111946
-  object_id: e1067d34-40d6-41b2-8690-a550ac847c25
-  sha: f04819411c2cbbb7a2d4ba7785e344b68bfd88a6
-curator/vendor/voluptuous-0.9.3.tar.gz:
-  size: 34097
-  object_id: 2a246c17-66fb-4c0c-8095-0c5bc6e0029f
-  sha: 315dd79c372380799d0d2e21ae711aa4097efb81
+curator/vendor/elasticsearch-5.4.0-py2.py3-none-any.whl:
+  size: 58679
+  object_id: 101d0ced-4af3-4f4f-5916-8ab699f6ad0c
+  sha: 0215d94d3e6b2c3c2ffcba54a84186a8aacb86df
+curator/vendor/urllib3-1.22-py2.py3-none-any.whl:
+  size: 132332
+  object_id: 0b6e809e-090d-4545-7808-b7968f1834a0
+  sha: ae6715ae61c34b72d5e0c3241abfb20c2c4d1313
+curator/vendor/voluptuous-0.10.5.tar.gz:
+  size: 41436
+  object_id: b67881a7-7adb-4456-7d0b-f5b00e936152
+  sha: 31373cc4ba935eda76b55e64e8bd218ce4882e2b
 elasticsearch/elasticsearch-5.5.1.tar.gz:
   size: 33476827
   object_id: 7dc3b18b-ddc5-4985-7801-9479bfd2da3d
@@ -94,6 +74,10 @@ python/Python-3.6.1.tgz:
   size: 22540566
   object_id: db2b3522-9d58-4998-81ec-237d900ad740
   sha: 6e91434cf22414af8240dfa1bf8ab2d043b04998
+python/Python-3.6.2.tgz:
+  size: 22580749
+  object_id: 3f40fd31-5d63-4fe7-4b28-dcdb2cbe1f16
+  sha: b157ed490a453387874b354b743d1ae656680403
 ruby2.3/bundler-1.11.2.gem:
   size: 263168
   object_id: ce2c212f-c2ff-440e-9f4b-c2c214a010f9
diff --git a/jobs/curator/templates/bin/curator_ctl b/jobs/curator/templates/bin/curator_ctl
index e95f97f4..997e3636 100644
--- a/jobs/curator/templates/bin/curator_ctl
+++ b/jobs/curator/templates/bin/curator_ctl
@@ -14,7 +14,7 @@ case $1 in
       /usr/sbin/cron start
     fi
 
-    (crontab -l | sed /run/d; cat /var/vcap/jobs/curator/config/purge_logs.cron) | sed /^$/d | crontab 
+    (crontab -l | sed /run/d; cat /var/vcap/jobs/curator/config/purge_logs.cron) | sed /^$/d | crontab
 
     pgrep cron > /var/vcap/sys/run/curator/curator.pid
     ;;
@@ -23,7 +23,7 @@ case $1 in
     if [ -z "$(crontab -l | grep -v 'run')" ]; then
       crontab -r
     else
-      (crontab -l | sed /run/d) | sed /^$/d | crontab 
+      (crontab -l | sed /run/d) | sed /^$/d | crontab
     fi
 
 
diff --git a/packages/curator/packaging b/packages/curator/packaging
index dc600a23..644404ae 100644
--- a/packages/curator/packaging
+++ b/packages/curator/packaging
@@ -5,4 +5,4 @@ export PATH=/var/vcap/packages/python3/bin:$PATH LD_LIBRARY_PATH=/var/vcap/packa
 # --no-index prevents contacting pypi to download packages
 # --find-links tells pip where to look for the dependancies
 # --prefix installation prefix where lib, bin and other top-level folders are placed
-pip3 install --no-index --find-links ./curator/vendor/ --prefix=${BOSH_INSTALL_TARGET} curator/elasticsearch-curator-5.0.1.tar.gz
+pip3 install --no-index --find-links ./curator/vendor/ --prefix=${BOSH_INSTALL_TARGET} curator/elasticsearch-curator-5.1.1.tar.gz
diff --git a/packages/curator/spec b/packages/curator/spec
index a498a40e..e8197f8f 100644
--- a/packages/curator/spec
+++ b/packages/curator/spec
@@ -1,17 +1,14 @@
 ---
 name: curator
+
 dependencies:
-- python3
+  - python3
+
 files:
-- curator/elasticsearch-curator-5.0.1.tar.gz
-- curator/vendor/appdirs-1.4.3-py2.py3-none-any.whl
-- curator/vendor/certifi-2017.1.23-py2.py3-none-any.whl
-- curator/vendor/click-6.7-py2.py3-none-any.whl
-- curator/vendor/elasticsearch-5.3.0-py2.py3-none-any.whl
-- curator/vendor/packaging-16.8-py2.py3-none-any.whl
-- curator/vendor/pyparsing-2.2.0-py2.py3-none-any.whl
-- curator/vendor/PyYAML-3.12.tar.gz
-- curator/vendor/setuptools-34.4.1-py2.py3-none-any.whl
-- curator/vendor/six-1.10.0-py2.py3-none-any.whl
-- curator/vendor/urllib3-1.20-py2.py3-none-any.whl
-- curator/vendor/voluptuous-0.9.3.tar.gz
+  - curator/elasticsearch-curator-5.1.1.tar.gz
+  - curator/vendor/PyYAML-3.12.tar.gz
+  - curator/vendor/click-6.7-py2.py3-none-any.whl
+  - curator/vendor/urllib3-1.22-py2.py3-none-any.whl
+  - curator/vendor/certifi-2017.7.27.1-py2.py3-none-any.whl
+  - curator/vendor/elasticsearch-5.4.0-py2.py3-none-any.whl
+  - curator/vendor/voluptuous-0.10.5.tar.gz
diff --git a/packages/python3/packaging b/packages/python3/packaging
index 821dca65..a0e4c966 100644
--- a/packages/python3/packaging
+++ b/packages/python3/packaging
@@ -1,13 +1,10 @@
-set -e -x
+set -e
 
-# Detect # of CPUs so make jobs can be parallelized
 CPUS=`grep -c ^processor /proc/cpuinfo`
 
-echo "Extracting python..."
-tar xzf python/Python-3.6.1.tgz
+tar xzf python/Python-3.6.2.tgz
 
-echo "Building python..."
-pushd Python-3.6.1
+pushd Python-3.6.2
   ./configure \
     --prefix=${BOSH_INSTALL_TARGET} \
     --with-ensurepip \
diff --git a/packages/python3/spec b/packages/python3/spec
index fc53df71..c2308479 100644
--- a/packages/python3/spec
+++ b/packages/python3/spec
@@ -1,7 +1,5 @@
 ---
 name: python3
 
-dependencies: []
-
 files:
-- python/Python-3.6.1.tgz
+  - python/Python-3.6.2.tgz

From 7ee422721dbaab8ddd094ffdc21220e8f8d312aa Mon Sep 17 00:00:00 2001
From: Andrei Krasnitski <me@infra-red.xyz>
Date: Fri, 4 Aug 2017 14:25:36 +0300
Subject: [PATCH 09/19] Configure additional Curator actions in manifest

---
 jobs/curator/spec                             |  3 +++
 jobs/curator/templates/config/actions.yml.erb | 23 +++++++++++++++++++
 jobs/curator/templates/config/config.yml.erb  |  0
 3 files changed, 26 insertions(+)
 create mode 100644 jobs/curator/templates/config/actions.yml.erb
 create mode 100644 jobs/curator/templates/config/config.yml.erb

diff --git a/jobs/curator/spec b/jobs/curator/spec
index c5b4883f..8954b77c 100644
--- a/jobs/curator/spec
+++ b/jobs/curator/spec
@@ -8,6 +8,7 @@ packages:
 templates:
   bin/run: bin/run
   bin/curator_ctl: bin/curator_ctl
+  config/actions.yml.erb: config/actions.yml
   config/purge_logs.cron: config/purge_logs.cron
   helpers/ctl_utils.sh: helpers/ctl_utils.sh
   helpers/ctl_setup.sh: helpers/ctl_setup.sh
@@ -31,3 +32,5 @@ properties:
   curator.purge_logs.retention_period:
     description: The value of this setting will be used as a multiplier for unit
     default: 30
+  curator.actions:
+    description: List of Elasticsearch Curator actions in YAML format
diff --git a/jobs/curator/templates/config/actions.yml.erb b/jobs/curator/templates/config/actions.yml.erb
new file mode 100644
index 00000000..a627daf9
--- /dev/null
+++ b/jobs/curator/templates/config/actions.yml.erb
@@ -0,0 +1,23 @@
+---
+actions:
+  1:
+    action: delete_indices
+    description: >-
+      Delete indices older than <%= p('curator.purge_logs.retention_period') %> <%= p('curator.purge_logs.unit') %> (based on index name), for logs-
+      prefixed indices. Ignore the error if the filter does not result in an
+      actionable list of indices (ignore_empty_list) and exit cleanly.
+    options:
+      ignore_empty_list: True
+      disable_action: False
+    filters:
+    - filtertype: pattern
+      kind: regex
+      value: logs*
+    - filtertype: age
+      source: creation_date
+      direction: older
+      unit: <%= p('curator.purge_logs.unit') %>
+      unit_count: <%= p('curator.purge_logs.retention_period') %>
+<% p("curator.actions", []).each_with_index do | action, index | %>
+  <%= index + 2 %>:<%= action.to_yaml.gsub("---", "").gsub("\n", "\n    ") %>
+<% end %>
diff --git a/jobs/curator/templates/config/config.yml.erb b/jobs/curator/templates/config/config.yml.erb
new file mode 100644
index 00000000..e69de29b

From 59812a70ec353d833d81cc6041a4453c77621e91 Mon Sep 17 00:00:00 2001
From: Andrei Krasnitski <me@infra-red.xyz>
Date: Fri, 4 Aug 2017 21:22:15 +0300
Subject: [PATCH 10/19] Add Elasticsearch Curator config

---
 jobs/curator/spec                            |  7 +++++
 jobs/curator/templates/config/config.yml.erb | 33 ++++++++++++++++++++
 templates/logsearch-jobs.yml                 |  2 +-
 3 files changed, 41 insertions(+), 1 deletion(-)

diff --git a/jobs/curator/spec b/jobs/curator/spec
index 8954b77c..f63f8ff5 100644
--- a/jobs/curator/spec
+++ b/jobs/curator/spec
@@ -9,6 +9,7 @@ templates:
   bin/run: bin/run
   bin/curator_ctl: bin/curator_ctl
   config/actions.yml.erb: config/actions.yml
+  config/config.yml.erb: config/config.yml
   config/purge_logs.cron: config/purge_logs.cron
   helpers/ctl_utils.sh: helpers/ctl_utils.sh
   helpers/ctl_setup.sh: helpers/ctl_setup.sh
@@ -32,5 +33,11 @@ properties:
   curator.purge_logs.retention_period:
     description: The value of this setting will be used as a multiplier for unit
     default: 30
+  curator.loglevel:
+    description: Set the minimum acceptable log severity to display.
+    default: INFO
+  curator.logformat:
+    description: This should default, json, logstash.
+    default: default
   curator.actions:
     description: List of Elasticsearch Curator actions in YAML format
diff --git a/jobs/curator/templates/config/config.yml.erb b/jobs/curator/templates/config/config.yml.erb
index e69de29b..d35ebaec 100644
--- a/jobs/curator/templates/config/config.yml.erb
+++ b/jobs/curator/templates/config/config.yml.erb
@@ -0,0 +1,33 @@
+<%
+  elasticsearch_hosts = nil
+  elasticsearch_port = nil
+  if_link("elasticsearch") { |elasticsearch_link|
+    elasticsearch_hosts = elasticsearch_link.instances.map { |instance| instance.address }
+    elasticsearch_port = elasticsearch_link.p("elasticsearch.port")
+  }
+  unless elasticsearch_hosts
+    elasticsearch_hosts = p("curator.elasticsearch.hosts")
+  end
+  unless elasticsearch_port
+    elasticsearch_port = p("curator.elasticsearch.port")
+  end
+%>
+---
+client:
+  hosts: <%= elasticsearch_hosts %>
+  port: <%= elasticsearch_port %>
+  url_prefix:
+  use_ssl: False
+  certificate:
+  client_cert:
+  client_key:
+  ssl_no_validate: False
+  http_auth:
+  timeout: 30
+  master_only: False
+
+logging:
+  loglevel: <%= p('curator.loglevel') %>
+  logfile:
+  logformat: <%= p('curator.logformat') %>
+  blacklist: ['elasticsearch', 'urllib3']
diff --git a/templates/logsearch-jobs.yml b/templates/logsearch-jobs.yml
index 708035d4..f72fd870 100644
--- a/templates/logsearch-jobs.yml
+++ b/templates/logsearch-jobs.yml
@@ -236,7 +236,7 @@ properties:
       unit: days
       retention_period: 30
     elasticsearch:
-      host: (( grab jobs.elasticsearch_master.networks.default.static_ips.[0] ))
+      hosts: (( grab jobs.elasticsearch_master.networks.default.static_ips ))
       port: 9200
   logstash_parser:
     debug: false

From bb651a39d0359e357af0eb1c395dd863cef9fbbf Mon Sep 17 00:00:00 2001
From: Andrei Krasnitski <me@infra-red.xyz>
Date: Fri, 4 Aug 2017 21:32:50 +0300
Subject: [PATCH 11/19] Use Command Line Interface in Curator run script

---
 jobs/curator/templates/bin/run | 26 ++------------------------
 1 file changed, 2 insertions(+), 24 deletions(-)

diff --git a/jobs/curator/templates/bin/run b/jobs/curator/templates/bin/run
index eb146e9d..e8e3b4ad 100644
--- a/jobs/curator/templates/bin/run
+++ b/jobs/curator/templates/bin/run
@@ -2,34 +2,12 @@
 
 set -e
 
-<%
-  elasticsearch_host = nil
-  elasticsearch_port = nil
-  if_link("elasticsearch") { |elasticsearch_link|
-    elasticsearch_host = elasticsearch_link.instances.first.address
-    elasticsearch_port = elasticsearch_link.p("elasticsearch.port")
-  }
-  unless elasticsearch_host
-    elasticsearch_host = p("curator.elasticsearch.host")
-  end
-  unless elasticsearch_port
-    elasticsearch_port = p("curator.elasticsearch.port")
-  end
-%>
-
-ELASTICSEARCH_HOST=<%= elasticsearch_host %>
-ELASTICSEARCH_PORT=<%= elasticsearch_port %>
-
 export LC_ALL=en_US.UTF-8
 export LANG=en_US.UTF-8
 
 export PATH=/var/vcap/packages/curator/bin:$PATH
+export CONFIG_DIR=/var/vcap/jobs/curator/config
 export PYTHONPATH=/var/vcap/packages/curator/lib/python3.6/site-packages/
 export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/var/vcap/packages/python3/lib
 
-curator_cli \
-  --host $ELASTICSEARCH_HOST \
-  --port $ELASTICSEARCH_PORT  \
-  delete_indices \
-  --filter_list '[{"filtertype":"age","source":"creation_date","direction":"older","unit":"<%= p('curator.purge_logs.unit') %>","unit_count":<%= p('curator.purge_logs.retention_period') %>},{"filtertype":"pattern","kind":"regex","value":"logs*"}]' \
-  --ignore_empty_list
+curator --config $CONFIG_DIR/config.yml $CONFIG_DIR/actions.yml

From 19b4206e68de881d2830dc5994872f6443dd0558 Mon Sep 17 00:00:00 2001
From: Andrei Krasnitski <me@infra-red.xyz>
Date: Fri, 4 Aug 2017 21:40:16 +0300
Subject: [PATCH 12/19] Add example Elasticsearch Curator action in job spec

---
 jobs/curator/spec | 21 ++++++++++++++++++++-
 1 file changed, 20 insertions(+), 1 deletion(-)

diff --git a/jobs/curator/spec b/jobs/curator/spec
index f63f8ff5..c6fc556d 100644
--- a/jobs/curator/spec
+++ b/jobs/curator/spec
@@ -40,4 +40,23 @@ properties:
     description: This should default, json, logstash.
     default: default
   curator.actions:
-    description: List of Elasticsearch Curator actions in YAML format
+    description: List of Elasticsearch Curator actions in YAML format.
+    example: |
+      - action: delete_indices
+        description: >-
+          Delete indices older than 45 days (based on index name), for logstash-
+          prefixed indices. Ignore the error if the filter does not result in an
+          actionable list of indices (ignore_empty_list) and exit cleanly.
+        options:
+          ignore_empty_list: True
+          disable_action: True
+        filters:
+        - filtertype: pattern
+          kind: prefix
+          value: logstash-
+        - filtertype: age
+          source: name
+          direction: older
+          timestring: '%Y.%m.%d'
+          unit: days
+          unit_count: 45

From 9880d5e88ed1e81af5747e9e33a592ce749c2d95 Mon Sep 17 00:00:00 2001
From: Joshua Carp <jm.carp@gmail.com>
Date: Fri, 11 Aug 2017 02:13:21 -0400
Subject: [PATCH 13/19] Drop unused ingestor properties. (#88)

---
 jobs/ingestor_syslog/spec | 22 +++-------------------
 1 file changed, 3 insertions(+), 19 deletions(-)

diff --git a/jobs/ingestor_syslog/spec b/jobs/ingestor_syslog/spec
index 8b2f115e..b350ad2b 100644
--- a/jobs/ingestor_syslog/spec
+++ b/jobs/ingestor_syslog/spec
@@ -2,9 +2,9 @@
 name: ingestor_syslog
 
 packages:
-  - logstash
-  - logsearch-config
-  - java8
+- logstash
+- logsearch-config
+- java8
 
 templates:
   bin/ingestor_syslog_ctl: bin/ingestor_syslog_ctl
@@ -76,18 +76,6 @@ properties:
   logstash_ingestor.filters:
     description: Filters to execute on the ingestors
     default: ""
-  logstash_ingestor.outputs:
-    description: |
-      A list of output plugins, with a hash of options for each of them. Please refer to example below.
-    example:
-      outputs:
-      - plugin: rabbitmq
-        options:
-          host: 192.168.1.1
-          user: logsearch
-          password: c1oudbunny
-          exchange_type: "direct"
-    default: [ { plugin: "redis", options : {} } ]
 
   logstash_ingestor.syslog.port:
     description: Port to listen for syslog messages
@@ -110,10 +98,6 @@ properties:
     description: Port to listen for RELP messages
     default: 2514
 
-  archiver.enabled:
-    default: false
-    description: Additional queue for archive (Redis only)
-
   logstash_parser.debug:
     description: Debug level logging
     default: false

From b0fd977800072688833ede7fcabe2ea7a14d2e12 Mon Sep 17 00:00:00 2001
From: Joshua Carp <jm.carp@gmail.com>
Date: Fri, 11 Aug 2017 13:13:34 -0400
Subject: [PATCH 14/19] Add archiver syslog job.

---
 jobs/archiver_syslog/monit                    |   5 +
 jobs/archiver_syslog/spec                     | 103 ++++++++++
 .../templates/bin/archiver_syslog_ctl         |  66 ++++++
 .../templates/bin/monit_debugger              |  13 ++
 .../config/input_and_output.conf.erb          |  59 ++++++
 .../templates/config/logstash.yml.erb         |   9 +
 .../templates/config/syslog_tls.crt.erb       |   1 +
 .../templates/config/syslog_tls.key.erb       |   1 +
 .../templates/data/properties.sh.erb          |  10 +
 .../templates/helpers/ctl_setup.sh            |  75 +++++++
 .../templates/helpers/ctl_utils.sh            | 189 ++++++++++++++++++
 11 files changed, 531 insertions(+)
 create mode 100644 jobs/archiver_syslog/monit
 create mode 100644 jobs/archiver_syslog/spec
 create mode 100644 jobs/archiver_syslog/templates/bin/archiver_syslog_ctl
 create mode 100644 jobs/archiver_syslog/templates/bin/monit_debugger
 create mode 100644 jobs/archiver_syslog/templates/config/input_and_output.conf.erb
 create mode 100644 jobs/archiver_syslog/templates/config/logstash.yml.erb
 create mode 100644 jobs/archiver_syslog/templates/config/syslog_tls.crt.erb
 create mode 100644 jobs/archiver_syslog/templates/config/syslog_tls.key.erb
 create mode 100644 jobs/archiver_syslog/templates/data/properties.sh.erb
 create mode 100644 jobs/archiver_syslog/templates/helpers/ctl_setup.sh
 create mode 100644 jobs/archiver_syslog/templates/helpers/ctl_utils.sh

diff --git a/jobs/archiver_syslog/monit b/jobs/archiver_syslog/monit
new file mode 100644
index 00000000..45db0e11
--- /dev/null
+++ b/jobs/archiver_syslog/monit
@@ -0,0 +1,5 @@
+check process archiver_syslog
+  with pidfile /var/vcap/sys/run/archiver_syslog/archiver_syslog.pid
+  start program "/var/vcap/jobs/archiver_syslog/bin/monit_debugger archiver_syslog_ctl '/var/vcap/jobs/archiver_syslog/bin/archiver_syslog_ctl start'"  with timeout 120 seconds
+  stop program "/var/vcap/jobs/archiver_syslog/bin/monit_debugger archiver_syslog_ctl '/var/vcap/jobs/archiver_syslog/bin/archiver_syslog_ctl stop'"
+  group vcap
diff --git a/jobs/archiver_syslog/spec b/jobs/archiver_syslog/spec
new file mode 100644
index 00000000..dafa007a
--- /dev/null
+++ b/jobs/archiver_syslog/spec
@@ -0,0 +1,103 @@
+---
+name: archiver_syslog
+
+packages:
+- logstash
+- logsearch-config
+- java8
+
+templates:
+  bin/archiver_syslog_ctl: bin/archiver_syslog_ctl
+  bin/monit_debugger: bin/monit_debugger
+  config/input_and_output.conf.erb: config/input_and_output.conf
+  config/syslog_tls.crt.erb: config/syslog_tls.crt
+  config/syslog_tls.key.erb: config/syslog_tls.key
+  config/logstash.yml.erb: config/logstash.yml
+  data/properties.sh.erb: data/properties.sh
+  helpers/ctl_setup.sh: helpers/ctl_setup.sh
+  helpers/ctl_utils.sh: helpers/ctl_utils.sh
+
+provides:
+- name: archiver
+  type: archiver
+  properties:
+  - logstash_ingestor.syslog.port
+  - logstash_ingestor.syslog.transport
+  - logstash_ingestor.syslog_tls.port
+  - logstash_ingestor.relp.port
+- name: syslog_forwarder
+  type: syslog_forwarder
+  properties:
+  - logstash_ingestor.syslog.port
+
+properties:
+  logstash.heap_size:
+    description: sets jvm heap sized
+  logstash.metadata_level:
+    description: "Whether to include additional metadata throughout the event lifecycle. NONE = disabled, DEBUG = fully enabled"
+    default: "NONE"
+  logstash.log_level:
+    description: The default logging level (e.g. WARN, DEBUG, INFO)
+    default: info
+  logstash.plugins:
+    description: "Array of hashes describing logstash plugins to install"
+    example:
+    - {name: logstash-output-cloudwatchlogs, version: 2.0.0}
+    default: []
+  logstash.env:
+    description: "a list of arbitrary key-value pairs to be passed on as process environment variables. eg: FOO: 123"
+    default: []
+
+  logstash_ingestor.debug:
+    description: Debug level logging
+    default: false
+  logstash_ingestor.workers:
+    description: "The number of worker threads that logstash should use (default: auto = one per CPU)"
+    default: auto
+
+  logstash.queue.type:
+    description: Internal queuing model, "memory" for legacy in-memory based queuing and "persisted" for disk-based acked queueing.
+    default: persisted
+  logstash.queue.page_capacity:
+    description: The page data files size. The queue data consists of append-only data files separated into pages.
+    default: 250mb
+  logstash.queue.max_events:
+    description: The maximum number of unread events in the queue.
+    default: 0
+  logstash.queue.max_bytes:
+    description: The total capacity of the queue in number of bytes.
+    default: 1024mb
+  logstash.queue.checkpoint.acks:
+    description: The maximum number of acked events before forcing a checkpoint.
+    default: 1024
+  logstash.queue.checkpoint.writes:
+    description: The maximum number of written events before forcing a checkpoint.
+    default: 1024
+  logstash.queue.checkpoint.interval:
+    description: The interval in milliseconds when a checkpoint is forced on the head page.
+    default: 1000
+
+  logstash_ingestor.syslog.port:
+    description: Port to listen for syslog messages
+    default: 5514
+  logstash_ingestor.syslog.transport:
+    description: Transport protocol to use
+    default: "tcp"
+
+  logstash_ingestor.syslog_tls.port:
+    description: Port to listen for syslog-TLS messages (omit to disable)
+  logstash_ingestor.syslog_tls.ssl_cert:
+    description: Syslog-TLS SSL certificate (file contents, not a path) - required if logstash_ingestor.syslog_tls.port set
+  logstash_ingestor.syslog_tls.ssl_key:
+    description: Syslog-TLS SSL key (file contents, not a path) - required if logstash_ingestor.syslog_tls.port set
+  logstash_ingestor.syslog_tls.skip_ssl_validation:
+    description: Verify the identity of the other end of the SSL connection against the CA.
+    default: false
+
+  logstash_ingestor.relp.port:
+    description: Port to listen for RELP messages
+    default: 2514
+
+  logstash_ingestor.outputs:
+    description: A list of output plugins, with a hash of options for each of them.
+    default: []
diff --git a/jobs/archiver_syslog/templates/bin/archiver_syslog_ctl b/jobs/archiver_syslog/templates/bin/archiver_syslog_ctl
new file mode 100644
index 00000000..5ce8414b
--- /dev/null
+++ b/jobs/archiver_syslog/templates/bin/archiver_syslog_ctl
@@ -0,0 +1,66 @@
+#!/bin/bash
+
+set -e # exit immediately if a simple command exits with a non-zero status
+set -u # report the usage of uninitialized variables
+
+# Setup env vars and folders for the webapp_ctl script
+source /var/vcap/jobs/archiver_syslog/helpers/ctl_setup.sh 'archiver_syslog'
+
+export PORT=${PORT:-5000}
+export LANG=en_US.UTF-8
+<% if 'auto' == p('logstash_ingestor.workers') %>
+# 1 logstash worker / CPU core
+export LOGSTASH_WORKERS=`grep -c ^processor /proc/cpuinfo`
+<% else %>
+export LOGSTASH_WORKERS=<%= p('logstash_ingestor.workers') %>
+<% end %>
+export HEAP_SIZE=$((( $( cat /proc/meminfo | grep MemTotal | awk '{ print $2 }' ) * 46 ) / 100 ))K
+<% if_p('logstash.heap_size') do |heap_size| %>
+HEAP_SIZE=<%= heap_size %>
+<% end %>
+<% p("logstash.env").each do |env| %>
+export <%= env.keys[0] %>="<%= env.values[0] %>"
+<% end %>
+
+case $1 in
+
+  start)
+    pid_guard $PIDFILE $JOB_NAME
+
+    # store this processes pid in $PIDFILE, since the exec below doesn't daemonize
+    echo $$ > $PIDFILE
+
+    export LS_JAVA_OPTS="-Xms$HEAP_SIZE -Xmx$HEAP_SIZE -DPID=$$"
+
+    <% p("logstash.plugins").each do |plugin| %>
+    /var/vcap/packages/logstash/bin/logstash-plugin install \
+      <%= plugin.except("name").map { |key, value| "--#{key}=#{value}" }.join(" ") %> \
+      <%= plugin["name"] %>
+    <% end %>
+
+    # construct a complete config file from all the fragments
+    cat ${JOB_DIR}/config/input_and_output.conf > ${JOB_DIR}/config/logstash.conf
+
+    exec chpst -u vcap:vcap /var/vcap/packages/logstash/bin/logstash \
+        --path.data ${STORE_DIR} \
+        --path.config ${JOB_DIR}/config/logstash.conf \
+        --path.settings ${JOB_DIR}/config \
+        --pipeline.workers ${LOGSTASH_WORKERS} \
+        --log.format=json --log.level=<%= p("logstash.log_level") %> \
+         >>$LOG_DIR/$JOB_NAME.stdout.log \
+         2>>$LOG_DIR/$JOB_NAME.stderr.log
+
+    ;;
+
+  stop)
+    kill_and_wait $PIDFILE
+    ensure_no_more_logstash
+
+    ;;
+  *)
+    echo "Usage: archiver_syslog_ctl {start|stop}"
+
+    ;;
+
+esac
+exit 0
diff --git a/jobs/archiver_syslog/templates/bin/monit_debugger b/jobs/archiver_syslog/templates/bin/monit_debugger
new file mode 100644
index 00000000..55353d2a
--- /dev/null
+++ b/jobs/archiver_syslog/templates/bin/monit_debugger
@@ -0,0 +1,13 @@
+#!/bin/sh
+# USAGE monit_debugger <label> command to run
+mkdir -p /var/vcap/sys/log/monit
+{
+  echo "MONIT-DEBUG date"
+  date
+  echo "MONIT-DEBUG env"
+  env
+  echo "MONIT-DEBUG $@"
+  $2 $3 $4 $5 $6 $7
+  R=$?
+  echo "MONIT-DEBUG exit code $R"
+} >/var/vcap/sys/log/monit/monit_debugger.$1.log 2>&1
diff --git a/jobs/archiver_syslog/templates/config/input_and_output.conf.erb b/jobs/archiver_syslog/templates/config/input_and_output.conf.erb
new file mode 100644
index 00000000..7db65125
--- /dev/null
+++ b/jobs/archiver_syslog/templates/config/input_and_output.conf.erb
@@ -0,0 +1,59 @@
+input {
+  <% if p("logstash_ingestor.syslog.port") %>
+  tcp {
+    add_field => [ "type", "syslog" ]
+    host => "0.0.0.0"
+    port => "<%= p("logstash_ingestor.syslog.port") %>"
+  }
+  udp {
+    add_field => [ "type", "syslog" ]
+    host => "0.0.0.0"
+    port => "<%= p("logstash_ingestor.syslog.port") %>"
+  }
+  <% end %>
+
+  <% if_p("logstash_ingestor.syslog_tls.port") do | port | %>
+  tcp {
+    add_field => [ "type", "syslog" ]
+    host => "0.0.0.0"
+    port => "<%= port %>"
+    ssl_enable => true
+    <% if p("logstash_ingestor.syslog_tls.skip_ssl_validation") %>
+    ssl_verify => false
+    <% end %>
+    ssl_cert => "/var/vcap/jobs/ingestor_syslog/config/syslog_tls.crt"
+    ssl_key => "/var/vcap/jobs/ingestor_syslog/config/syslog_tls.key"
+  }
+  <% end %>
+
+  <% if_p("logstash_ingestor.relp.port") do | port | %>
+  relp {
+    add_field => { "_logstash_input" => "relp" }
+    host => "0.0.0.0"
+    port => "<%= port %>"
+  }
+  <% end %>
+}
+
+filter {
+  # drop empty logs (eg: monit connection healthcheck)
+  ruby {
+    code => 'event.cancel if event.get("message") == "\u0000"'
+  }
+}
+
+output {
+    <% if p("logstash_ingestor.debug") %>
+        stdout {
+            codec => "json"
+        }
+    <% end %>
+
+    <% p('logstash_ingestor.outputs').each do | output | %>
+        <%= output['plugin'] %> {
+        <% output['options'].each do | k, v | %>
+          <%= k %> => <%= v.inspect %>
+        <% end %>
+        }
+    <% end %>
+}
diff --git a/jobs/archiver_syslog/templates/config/logstash.yml.erb b/jobs/archiver_syslog/templates/config/logstash.yml.erb
new file mode 100644
index 00000000..d4df7031
--- /dev/null
+++ b/jobs/archiver_syslog/templates/config/logstash.yml.erb
@@ -0,0 +1,9 @@
+# ------------ Queuing Settings --------------
+
+queue.type: <%= p("logstash.queue.type") %>
+queue.page_capacity: <%= p("logstash.queue.page_capacity") %>
+queue.max_events: <%= p("logstash.queue.max_events") %>
+queue.max_bytes: <%= p("logstash.queue.max_bytes") %>
+queue.checkpoint.acks: <%= p("logstash.queue.checkpoint.acks") %>
+queue.checkpoint.writes: <%= p("logstash.queue.checkpoint.writes") %>
+queue.checkpoint.interval: <%= p("logstash.queue.checkpoint.interval") %>
diff --git a/jobs/archiver_syslog/templates/config/syslog_tls.crt.erb b/jobs/archiver_syslog/templates/config/syslog_tls.crt.erb
new file mode 100644
index 00000000..38ff81cf
--- /dev/null
+++ b/jobs/archiver_syslog/templates/config/syslog_tls.crt.erb
@@ -0,0 +1 @@
+<% if_p("logstash_ingestor.syslog_tls.ssl_cert") do | ssl | %><%= ssl %><% end %>
\ No newline at end of file
diff --git a/jobs/archiver_syslog/templates/config/syslog_tls.key.erb b/jobs/archiver_syslog/templates/config/syslog_tls.key.erb
new file mode 100644
index 00000000..1db62a33
--- /dev/null
+++ b/jobs/archiver_syslog/templates/config/syslog_tls.key.erb
@@ -0,0 +1 @@
+<% if_p("logstash_ingestor.syslog_tls.ssl_key") do | ssl | %><%= ssl %><% end %>
\ No newline at end of file
diff --git a/jobs/archiver_syslog/templates/data/properties.sh.erb b/jobs/archiver_syslog/templates/data/properties.sh.erb
new file mode 100644
index 00000000..616aecc7
--- /dev/null
+++ b/jobs/archiver_syslog/templates/data/properties.sh.erb
@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+# job template binding variables
+
+# job name & index of this VM within cluster
+# e.g. JOB_NAME=redis, JOB_INDEX=0
+export NAME='<%= name %>'
+export JOB_INDEX=<%= index %>
+# full job name, like redis/0 or webapp/3
+export JOB_FULL="$NAME/$JOB_INDEX"
diff --git a/jobs/archiver_syslog/templates/helpers/ctl_setup.sh b/jobs/archiver_syslog/templates/helpers/ctl_setup.sh
new file mode 100644
index 00000000..dca5b528
--- /dev/null
+++ b/jobs/archiver_syslog/templates/helpers/ctl_setup.sh
@@ -0,0 +1,75 @@
+#!/usr/bin/env bash
+
+# Setup env vars and folders for the ctl script
+# This helps keep the ctl script as readable
+# as possible
+
+# Usage options:
+# source /var/vcap/jobs/foobar/helpers/ctl_setup.sh JOB_NAME OUTPUT_LABEL
+# source /var/vcap/jobs/foobar/helpers/ctl_setup.sh foobar
+# source /var/vcap/jobs/foobar/helpers/ctl_setup.sh foobar foobar
+# source /var/vcap/jobs/foobar/helpers/ctl_setup.sh foobar nginx
+
+set -e # exit immediately if a simple command exits with a non-zero status
+set -u # report the usage of uninitialized variables
+
+JOB_NAME=$1
+output_label=${1:-JOB_NAME}
+
+export JOB_DIR=/var/vcap/jobs/$JOB_NAME
+chmod 755 $JOB_DIR # to access file via symlink
+
+# Load some bosh deployment properties into env vars
+# Try to put all ERb into data/properties.sh.erb
+# incl $NAME, $JOB_INDEX, $WEBAPP_DIR
+source $JOB_DIR/data/properties.sh
+
+source $JOB_DIR/helpers/ctl_utils.sh
+redirect_output ${output_label}
+
+export HOME=${HOME:-/home/vcap}
+
+# Add all packages' /bin & /sbin into $PATH
+for package_bin_dir in $(ls -d /var/vcap/packages/*/*bin)
+do
+  export PATH=${package_bin_dir}:$PATH
+done
+
+export LD_LIBRARY_PATH=${LD_LIBRARY_PATH:-''} # default to empty
+for package_bin_dir in $(ls -d /var/vcap/packages/*/lib)
+do
+  export LD_LIBRARY_PATH=${package_bin_dir}:$LD_LIBRARY_PATH
+done
+
+# Setup log, run and tmp folders
+
+export RUN_DIR=/var/vcap/sys/run/$JOB_NAME
+export LOG_DIR=/var/vcap/sys/log/$JOB_NAME
+export TMP_DIR=/var/vcap/sys/tmp/$JOB_NAME
+export STORE_DIR=/var/vcap/store/$JOB_NAME
+export DATA_DIR=/var/vcap/data/$JOB_NAME
+for dir in $RUN_DIR $LOG_DIR $TMP_DIR $STORE_DIR $DATA_DIR
+do
+  mkdir -p ${dir}
+  chown vcap:vcap ${dir}
+  chmod 775 ${dir}
+done
+export TMPDIR=$TMP_DIR
+
+export C_INCLUDE_PATH=/var/vcap/packages/mysqlclient/include/mysql:/var/vcap/packages/sqlite/include:/var/vcap/packages/libpq/include
+export LIBRARY_PATH=/var/vcap/packages/mysqlclient/lib/mysql:/var/vcap/packages/sqlite/lib:/var/vcap/packages/libpq/lib
+
+# consistent place for vendoring python libraries within package
+if [[ -d ${WEBAPP_DIR:-/xxxx} ]]
+then
+  export PYTHONPATH=$WEBAPP_DIR/vendor/lib/python
+fi
+
+if [[ -d /var/vcap/packages/java8 ]]
+then
+  export JAVA_HOME="/var/vcap/packages/java8"
+fi
+
+PIDFILE=$RUN_DIR/$JOB_NAME.pid
+
+echo '$PATH' $PATH
diff --git a/jobs/archiver_syslog/templates/helpers/ctl_utils.sh b/jobs/archiver_syslog/templates/helpers/ctl_utils.sh
new file mode 100644
index 00000000..1000c4dc
--- /dev/null
+++ b/jobs/archiver_syslog/templates/helpers/ctl_utils.sh
@@ -0,0 +1,189 @@
+# Helper functions used by ctl scripts
+
+# links a job file (probably a config file) into a package
+# Example usage:
+# link_job_file_to_package config/redis.yml [config/redis.yml]
+# link_job_file_to_package config/wp-config.php wp-config.php
+link_job_file_to_package() {
+  source_job_file=$1
+  target_package_file=${2:-$source_job_file}
+  full_package_file=$WEBAPP_DIR/${target_package_file}
+
+  link_job_file ${source_job_file} ${full_package_file}
+}
+
+# links a job file (probably a config file) somewhere
+# Example usage:
+# link_job_file config/bashrc /home/vcap/.bashrc
+link_job_file() {
+  source_job_file=$1
+  target_file=$2
+  full_job_file=$JOB_DIR/${source_job_file}
+
+  echo link_job_file ${full_job_file} ${target_file}
+  if [[ ! -f ${full_job_file} ]]
+  then
+    echo "file to link ${full_job_file} does not exist"
+  else
+    # Create/recreate the symlink to current job file
+    # If another process is using the file, it won't be
+    # deleted, so don't attempt to create the symlink
+    mkdir -p $(dirname ${target_file})
+    ln -nfs ${full_job_file} ${target_file}
+  fi
+}
+
+# If loaded within monit ctl scripts then pipe output
+# If loaded from 'source ../utils.sh' then normal STDOUT
+redirect_output() {
+  SCRIPT=$1
+  mkdir -p /var/vcap/sys/log/monit
+  exec 1>> /var/vcap/sys/log/monit/$SCRIPT.log
+  exec 2>> /var/vcap/sys/log/monit/$SCRIPT.err.log
+}
+
+function pid_is_running() {
+  declare pid="$1"
+  ps -p "${pid}" >/dev/null 2>&1
+}
+
+# pid_guard
+#
+# @param pidfile
+# @param name [String] an arbitrary name that might show up in STDOUT on errors
+#
+# Run this before attempting to start new processes that may use the same :pidfile:.
+# If an old process is running on the pid found in the :pidfile:, exit 1. Otherwise,
+# remove the stale :pidfile: if it exists.
+#
+function pid_guard() {
+  declare pidfile="$1" name="$2"
+
+  echo "------------ STARTING $(basename "$0") at $(date) --------------" | tee /dev/stderr
+
+  if [ ! -f "${pidfile}" ]; then
+    return 0
+  fi
+
+  local pid
+  pid=$(head -1 "${pidfile}")
+
+  if pid_is_running "${pid}"; then
+    echo "${name} is already running, please stop it first"
+    exit 1
+  fi
+
+  echo "Removing stale pidfile"
+  rm "${pidfile}"
+}
+
+# wait_pid_death
+#
+# @param pid
+# @param timeout
+#
+# Watch a :pid: for :timeout: seconds, waiting for it to die.
+# If it dies before :timeout:, exit 0. If not, exit 1.
+#
+# Note that this should be run in a subshell, so that the current
+# shell does not exit.
+#
+function wait_pid_death() {
+  declare pid="$1" timeout="$2"
+
+  local countdown
+  countdown=$(( timeout * 10 ))
+
+  while true; do
+    if ! pid_is_running "${pid}"; then
+      return 0
+    fi
+
+    if [ ${countdown} -le 0 ]; then
+      return 1
+    fi
+
+    countdown=$(( countdown - 1 ))
+    sleep 0.1
+  done
+}
+
+# kill_and_wait
+#
+# @param pidfile
+# @param timeout [default 25s]
+#
+# For a pid found in :pidfile:, send a `kill -15` TERM, then wait for :timeout: seconds to
+# see if it dies on its own. If not, send it a `kill -9`. If the process does die,
+# exit 0 and remove the :pidfile:. If after all of this, the process does not actually
+# die, exit 1.
+#
+# Note:
+# Monit default timeout for start/stop is 30s
+# Append 'with timeout {n} seconds' to monit start/stop program configs
+#
+function kill_and_wait() {
+  declare pidfile="$1" timeout="${2:-25}" sigkill_on_timeout="${3:-1}"
+
+  if [ ! -f "${pidfile}" ]; then
+    echo "Pidfile ${pidfile} doesn't exist"
+    exit 0
+  fi
+
+  local pid
+  pid=$(head -1 "${pidfile}")
+
+  if [ -z "${pid}" ]; then
+    echo "Unable to get pid from ${pidfile}"
+    exit 1
+  fi
+
+  if ! pid_is_running "${pid}"; then
+    echo "Process ${pid} is not running"
+    rm -f "${pidfile}"
+    exit 0
+  fi
+
+  echo "Killing ${pidfile}: ${pid} "
+  kill "${pid}"
+
+  if ! wait_pid_death "${pid}" "${timeout}"; then
+    if [ "${sigkill_on_timeout}" = "1" ]; then
+      echo "Kill timed out, using kill -9 on ${pid}"
+      kill -9 "${pid}"
+      sleep 0.5
+    fi
+  fi
+
+  if pid_is_running "${pid}"; then
+    echo "Timed Out"
+    exit 1
+  else
+    echo "Stopped"
+    rm -f "${pidfile}"
+  fi
+}
+
+check_nfs_mount() {
+  opts=$1
+  exports=$2
+  mount_point=$3
+
+  if grep -qs $mount_point /proc/mounts; then
+    echo "Found NFS mount $mount_point"
+  else
+    echo "Mounting NFS..."
+    mount $opts $exports $mount_point
+    if [ $? != 0 ]; then
+      echo "Cannot mount NFS from $exports to $mount_point, exiting..."
+      exit 1
+    fi
+  fi
+}
+
+ensure_no_more_logstash() {
+  for java_pid in $(ps -ef | grep java | grep logstash/runner.rb |grep -Po '\-DPID=\K[^ ]+'); do
+    echo "Found leftover Java process - killing($java_pid) if running: `ps -fp $java_pid`"
+    kill -9 $java_pid
+  done
+}

From ab448e746b393f6558200b9ab304c7f2677be9fa Mon Sep 17 00:00:00 2001
From: Joshua Carp <jm.carp@gmail.com>
Date: Tue, 15 Aug 2017 14:06:14 -0400
Subject: [PATCH 15/19] Temporarily drop syslog forwarder job.

---
 jobs/syslog_forwarder/monit                   |  0
 jobs/syslog_forwarder/spec                    | 31 -----------
 jobs/syslog_forwarder/templates/bin/pre-start | 11 ----
 .../config/rsyslog_file_forwarder.conf        | 54 -------------------
 4 files changed, 96 deletions(-)
 delete mode 100644 jobs/syslog_forwarder/monit
 delete mode 100644 jobs/syslog_forwarder/spec
 delete mode 100644 jobs/syslog_forwarder/templates/bin/pre-start
 delete mode 100644 jobs/syslog_forwarder/templates/config/rsyslog_file_forwarder.conf

diff --git a/jobs/syslog_forwarder/monit b/jobs/syslog_forwarder/monit
deleted file mode 100644
index e69de29b..00000000
diff --git a/jobs/syslog_forwarder/spec b/jobs/syslog_forwarder/spec
deleted file mode 100644
index cef55166..00000000
--- a/jobs/syslog_forwarder/spec
+++ /dev/null
@@ -1,31 +0,0 @@
----
-name: syslog_forwarder
-
-templates:
-  config/rsyslog_file_forwarder.conf: config/rsyslog_file_forwarder.conf
-  bin/pre-start: bin/pre-start
-
-packages: []
-
-consumes:
-- name: syslog_forwarder
-  type: syslog_forwarder
-  optional: true
-
-properties:
-  syslog_forwarder.config:
-    description: |
-      List of service name and logfile definitions.
-      Service is the syslog_program in the final log message.
-      File is the path to the log file rsyslogd should stream to the remote syslog server
-    default: []
-    example: |
-      - { service: elasticsearch, file: /var/vcap/sys/log/elasticsearch.stdout.log }
-      - { service: elasticsearch, file: /var/vcap/sys/log/elasticsearch.stderr.log }
-      - { service: kibana, file: /var/vcap/sys/log/kibana/kibana.stdout.log }
-  syslog_forwarder.host:
-    description: Hostname of the remote syslog server
-    default: ""
-  syslog_forwarder.port:
-    description: Port of the remote syslog server
-    default: 5514
diff --git a/jobs/syslog_forwarder/templates/bin/pre-start b/jobs/syslog_forwarder/templates/bin/pre-start
deleted file mode 100644
index 861e53ae..00000000
--- a/jobs/syslog_forwarder/templates/bin/pre-start
+++ /dev/null
@@ -1,11 +0,0 @@
-#!/bin/bash -e
-
-mkdir -p /var/vcap/sys/run/rsyslogd
-chown syslog /var/vcap/sys/run/rsyslogd/
-
-<% p("syslog_forwarder.config").each do |config| %>
-mkdir -p $(dirname "<%= config["file"] %>")
-<% end %>
-
-cp /var/vcap/jobs/syslog_forwarder/config/rsyslog_file_forwarder.conf /etc/rsyslog.d/
-service rsyslog restart
diff --git a/jobs/syslog_forwarder/templates/config/rsyslog_file_forwarder.conf b/jobs/syslog_forwarder/templates/config/rsyslog_file_forwarder.conf
deleted file mode 100644
index 157889c9..00000000
--- a/jobs/syslog_forwarder/templates/config/rsyslog_file_forwarder.conf
+++ /dev/null
@@ -1,54 +0,0 @@
-<% if p("syslog_forwarder.config").any? %>
-
-<%
-  networks = spec.networks.marshal_dump.values
-  network = networks.find do |network_spec|
-    network_spec.default
-  end
-
-  network ||= networks.first
-  job_ip = network.ip
-%>
-
-module(load="imfile")
-$WorkDirectory /var/vcap/sys/run/rsyslogd
-
-<% p("syslog_forwarder.config").each do |config| %>
-input(type="imfile"
-  File="<%= config["file"] %>"
-  Tag="<%= config["service"] %>"
-  Ruleset="ToMonitor")
-
-<% end %>
-
-template(name="fileForwarding" type="list") {
-  constant(value="<")
-  property(name="pri")
-  constant(value=">")
-  constant(value="1 ")
-  property(name="timestamp" dateFormat="rfc3339")
-  constant(value=" <%= job_ip %> ")
-  property(name="programname")
-  constant(value=" - - [- job=<%= name %> index=<%= index %>] ")
-  property(name="msg")
-}
-
-<%
-  syslog_forwarder_host = nil
-  if_link("syslog_forwarder") { |syslog_forwarder_link| syslog_forwarder_host = syslog_forwarder_link.instances.first.address }
-  unless syslog_forwarder_host
-    syslog_forwarder_host = p("syslog_forwarder.host")
-  end
-
-  syslog_forwarder_port = nil
-  if_link("syslog_forwarder") { |syslog_forwarder_link| syslog_forwarder_port = syslog_forwarder_link.p("logstash_ingestor.syslog.port") }
-  unless syslog_forwarder_port
-    syslog_forwarder_port = p("syslog_forwarder.port")
-  end
-%>
-
-ruleset(name="ToMonitor") {
-  action(type="omfwd" Target="<%= syslog_forwarder_host %>" Port="<%= syslog_forwarder_port %>" Protocol="tcp" Template="fileForwarding")
-}
-
-<% end %>

From b69cf6bfef975913eebb0bb563cc9d6d766228d0 Mon Sep 17 00:00:00 2001
From: Joshua Carp <jm.carp@gmail.com>
Date: Tue, 15 Aug 2017 15:05:42 -0400
Subject: [PATCH 16/19] Temporarily drop syslog forwarder job.

---
 jobs/syslog_forwarder/monit                   |  0
 jobs/syslog_forwarder/spec                    | 31 -----------
 jobs/syslog_forwarder/templates/bin/pre-start | 11 ----
 .../config/rsyslog_file_forwarder.conf        | 54 -------------------
 4 files changed, 96 deletions(-)
 delete mode 100644 jobs/syslog_forwarder/monit
 delete mode 100644 jobs/syslog_forwarder/spec
 delete mode 100644 jobs/syslog_forwarder/templates/bin/pre-start
 delete mode 100644 jobs/syslog_forwarder/templates/config/rsyslog_file_forwarder.conf

diff --git a/jobs/syslog_forwarder/monit b/jobs/syslog_forwarder/monit
deleted file mode 100644
index e69de29b..00000000
diff --git a/jobs/syslog_forwarder/spec b/jobs/syslog_forwarder/spec
deleted file mode 100644
index cef55166..00000000
--- a/jobs/syslog_forwarder/spec
+++ /dev/null
@@ -1,31 +0,0 @@
----
-name: syslog_forwarder
-
-templates:
-  config/rsyslog_file_forwarder.conf: config/rsyslog_file_forwarder.conf
-  bin/pre-start: bin/pre-start
-
-packages: []
-
-consumes:
-- name: syslog_forwarder
-  type: syslog_forwarder
-  optional: true
-
-properties:
-  syslog_forwarder.config:
-    description: |
-      List of service name and logfile definitions.
-      Service is the syslog_program in the final log message.
-      File is the path to the log file rsyslogd should stream to the remote syslog server
-    default: []
-    example: |
-      - { service: elasticsearch, file: /var/vcap/sys/log/elasticsearch.stdout.log }
-      - { service: elasticsearch, file: /var/vcap/sys/log/elasticsearch.stderr.log }
-      - { service: kibana, file: /var/vcap/sys/log/kibana/kibana.stdout.log }
-  syslog_forwarder.host:
-    description: Hostname of the remote syslog server
-    default: ""
-  syslog_forwarder.port:
-    description: Port of the remote syslog server
-    default: 5514
diff --git a/jobs/syslog_forwarder/templates/bin/pre-start b/jobs/syslog_forwarder/templates/bin/pre-start
deleted file mode 100644
index 861e53ae..00000000
--- a/jobs/syslog_forwarder/templates/bin/pre-start
+++ /dev/null
@@ -1,11 +0,0 @@
-#!/bin/bash -e
-
-mkdir -p /var/vcap/sys/run/rsyslogd
-chown syslog /var/vcap/sys/run/rsyslogd/
-
-<% p("syslog_forwarder.config").each do |config| %>
-mkdir -p $(dirname "<%= config["file"] %>")
-<% end %>
-
-cp /var/vcap/jobs/syslog_forwarder/config/rsyslog_file_forwarder.conf /etc/rsyslog.d/
-service rsyslog restart
diff --git a/jobs/syslog_forwarder/templates/config/rsyslog_file_forwarder.conf b/jobs/syslog_forwarder/templates/config/rsyslog_file_forwarder.conf
deleted file mode 100644
index 157889c9..00000000
--- a/jobs/syslog_forwarder/templates/config/rsyslog_file_forwarder.conf
+++ /dev/null
@@ -1,54 +0,0 @@
-<% if p("syslog_forwarder.config").any? %>
-
-<%
-  networks = spec.networks.marshal_dump.values
-  network = networks.find do |network_spec|
-    network_spec.default
-  end
-
-  network ||= networks.first
-  job_ip = network.ip
-%>
-
-module(load="imfile")
-$WorkDirectory /var/vcap/sys/run/rsyslogd
-
-<% p("syslog_forwarder.config").each do |config| %>
-input(type="imfile"
-  File="<%= config["file"] %>"
-  Tag="<%= config["service"] %>"
-  Ruleset="ToMonitor")
-
-<% end %>
-
-template(name="fileForwarding" type="list") {
-  constant(value="<")
-  property(name="pri")
-  constant(value=">")
-  constant(value="1 ")
-  property(name="timestamp" dateFormat="rfc3339")
-  constant(value=" <%= job_ip %> ")
-  property(name="programname")
-  constant(value=" - - [- job=<%= name %> index=<%= index %>] ")
-  property(name="msg")
-}
-
-<%
-  syslog_forwarder_host = nil
-  if_link("syslog_forwarder") { |syslog_forwarder_link| syslog_forwarder_host = syslog_forwarder_link.instances.first.address }
-  unless syslog_forwarder_host
-    syslog_forwarder_host = p("syslog_forwarder.host")
-  end
-
-  syslog_forwarder_port = nil
-  if_link("syslog_forwarder") { |syslog_forwarder_link| syslog_forwarder_port = syslog_forwarder_link.p("logstash_ingestor.syslog.port") }
-  unless syslog_forwarder_port
-    syslog_forwarder_port = p("syslog_forwarder.port")
-  end
-%>
-
-ruleset(name="ToMonitor") {
-  action(type="omfwd" Target="<%= syslog_forwarder_host %>" Port="<%= syslog_forwarder_port %>" Protocol="tcp" Template="fileForwarding")
-}
-
-<% end %>

From 31015fc94d87575eee8d0991753d83f6781143ed Mon Sep 17 00:00:00 2001
From: Joshua Carp <jm.carp@gmail.com>
Date: Tue, 15 Aug 2017 15:13:55 -0400
Subject: [PATCH 17/19] Temporarily drop syslog forwarder links.

---
 jobs/archiver_syslog/spec | 4 ----
 jobs/ingestor_syslog/spec | 4 ----
 2 files changed, 8 deletions(-)

diff --git a/jobs/archiver_syslog/spec b/jobs/archiver_syslog/spec
index dafa007a..38f05e26 100644
--- a/jobs/archiver_syslog/spec
+++ b/jobs/archiver_syslog/spec
@@ -25,10 +25,6 @@ provides:
   - logstash_ingestor.syslog.transport
   - logstash_ingestor.syslog_tls.port
   - logstash_ingestor.relp.port
-- name: syslog_forwarder
-  type: syslog_forwarder
-  properties:
-  - logstash_ingestor.syslog.port
 
 properties:
   logstash.heap_size:
diff --git a/jobs/ingestor_syslog/spec b/jobs/ingestor_syslog/spec
index b350ad2b..76cf6e5b 100644
--- a/jobs/ingestor_syslog/spec
+++ b/jobs/ingestor_syslog/spec
@@ -28,10 +28,6 @@ provides:
   - logstash_ingestor.syslog.transport
   - logstash_ingestor.syslog_tls.port
   - logstash_ingestor.relp.port
-- name: syslog_forwarder
-  type: syslog_forwarder
-  properties:
-  - logstash_ingestor.syslog.port
 
 properties:
   logstash.heap_size:

From 6fb93b4ddc00f2abf05776959c67341fa9558529 Mon Sep 17 00:00:00 2001
From: Joshua Carp <jm.carp@gmail.com>
Date: Thu, 17 Aug 2017 13:07:21 -0400
Subject: [PATCH 18/19] Allow multiple non-file parser filter.

---
 .../templates/bin/ingestor_syslog_ctl              | 14 ++++++++------
 .../templates/config/filters_override.conf.erb     | 10 +++++++++-
 2 files changed, 17 insertions(+), 7 deletions(-)

diff --git a/jobs/ingestor_syslog/templates/bin/ingestor_syslog_ctl b/jobs/ingestor_syslog/templates/bin/ingestor_syslog_ctl
index 97c6eeb4..c380464f 100644
--- a/jobs/ingestor_syslog/templates/bin/ingestor_syslog_ctl
+++ b/jobs/ingestor_syslog/templates/bin/ingestor_syslog_ctl
@@ -73,14 +73,16 @@ case $1 in
     cat /var/vcap/packages/logsearch-config/logstash-filters-default.conf >> ${JOB_DIR}/config/logstash.conf
 
     <% if p('logstash_parser.filters').is_a? Array %>
-      # logstash_parser.filters is a list of filter rules
-      <% p('logstash_parser.filters').each do |filter| name, path = filter.first %>
-        cat "<%= path %>" >> ${JOB_DIR}/config/logstash.conf
+      <% p('logstash_parser.filters').each do |filter| %>
+        <% if filter.key? 'path' %>
+          cat "<%= filter['path'] %>" >> ${JOB_DIR}/config/logstash.conf
+        <% elsif !filter.key? 'content' %>
+          <% _, path = filter.first %>
+          cat "<%= path %>" >> ${JOB_DIR}/config/logstash.conf
+        <% end %>
       <% end %>
-    <% elsif p('logstash_parser.filters') != '' %>
-      # logstash_parser.filters is a block of text, which we treat as logstash filter config
-      cat ${JOB_DIR}/config/filters_override.conf >> ${JOB_DIR}/config/logstash.conf
     <% end %>
+    cat ${JOB_DIR}/config/filters_override.conf >> ${JOB_DIR}/config/logstash.conf
 
     cat ${JOB_DIR}/config/filters_post.conf >> ${JOB_DIR}/config/logstash.conf
     <% if p('logstash_parser.enable_json_filter') %>
diff --git a/jobs/ingestor_syslog/templates/config/filters_override.conf.erb b/jobs/ingestor_syslog/templates/config/filters_override.conf.erb
index ee2577ee..9a6c8c28 100644
--- a/jobs/ingestor_syslog/templates/config/filters_override.conf.erb
+++ b/jobs/ingestor_syslog/templates/config/filters_override.conf.erb
@@ -1 +1,9 @@
-<%= p('logstash_parser.filters').empty? ? ' ' :  p('logstash_parser.filters')  %>
+<% if p('logstash_parser.filters').is_a? Array %>
+  <% p('logstash_parser.filters').each do |filter| %>
+    <% if filter.key? 'content' %>
+      <%= filter['content'] %>
+    <% end %>
+  <% end %>
+<% elsif p('logstash_parser.filters') != '' %>
+  <%= p('logstash_parser.filters') %>
+<% end %>

From 22959ed0c17e51a1aedd3cd9a92ca18f6d93085e Mon Sep 17 00:00:00 2001
From: Will Woodson <william.woodson@gsa.gov>
Date: Wed, 23 Aug 2017 13:13:24 -0500
Subject: [PATCH 19/19] Add post-start check for logstash ingestors using
 syslog.

---
 jobs/archiver_syslog/spec                     | 11 +++++++++
 .../templates/bin/post-start.erb              | 23 +++++++++++++++++++
 jobs/ingestor_syslog/spec                     | 11 +++++++++
 .../templates/bin/post-start.erb              | 23 +++++++++++++++++++
 4 files changed, 68 insertions(+)
 create mode 100644 jobs/archiver_syslog/templates/bin/post-start.erb
 create mode 100644 jobs/ingestor_syslog/templates/bin/post-start.erb

diff --git a/jobs/archiver_syslog/spec b/jobs/archiver_syslog/spec
index 38f05e26..e86491b0 100644
--- a/jobs/archiver_syslog/spec
+++ b/jobs/archiver_syslog/spec
@@ -9,6 +9,7 @@ packages:
 templates:
   bin/archiver_syslog_ctl: bin/archiver_syslog_ctl
   bin/monit_debugger: bin/monit_debugger
+  bin/post-start.erb: bin/post-start
   config/input_and_output.conf.erb: config/input_and_output.conf
   config/syslog_tls.crt.erb: config/syslog_tls.crt
   config/syslog_tls.key.erb: config/syslog_tls.key
@@ -80,6 +81,16 @@ properties:
     description: Transport protocol to use
     default: "tcp"
 
+  logstash_ingestor.health.disable_post_start:
+    description: Allow node to run post-start script? (true / false)
+    default: false
+  logstash_ingestor.health.interval:
+    description: Logstash syslog health check interval (seconds)
+    default: 5
+  logstash_ingestor.health.timeout:
+    description: Logstash syslog health check number of attempts (seconds)
+    default: 300
+
   logstash_ingestor.syslog_tls.port:
     description: Port to listen for syslog-TLS messages (omit to disable)
   logstash_ingestor.syslog_tls.ssl_cert:
diff --git a/jobs/archiver_syslog/templates/bin/post-start.erb b/jobs/archiver_syslog/templates/bin/post-start.erb
new file mode 100644
index 00000000..4a001cb9
--- /dev/null
+++ b/jobs/archiver_syslog/templates/bin/post-start.erb
@@ -0,0 +1,23 @@
+#!/bin/bash
+
+set -e
+
+<% if !p("logstash_ingestor.health.disable_post_start") %>
+
+echo "Waiting <%= p("logstash_ingestor.health.timeout") %>s for syslog to accept connections..."
+elapsed=0
+until [ $elapsed -ge <%= p("logstash_ingestor.health.timeout") %> ]
+do
+  nc -vz localhost <%= p("logstash_ingestor.syslog.port") %> 2>&1 && break
+  elapsed=$[$elapsed+<%= p("logstash_ingestor.health.interval") %>]
+  sleep <%= p("logstash_ingestor.health.interval") %>
+done
+
+if [ "$elapsed" -ge "<%= p("logstash_ingestor.health.timeout") %>" ]; then
+  echo "ERROR: Cannot connect to syslog. Exiting..."
+  exit 1
+fi
+
+<% end %>
+
+exit 0
diff --git a/jobs/ingestor_syslog/spec b/jobs/ingestor_syslog/spec
index 76cf6e5b..36f03219 100644
--- a/jobs/ingestor_syslog/spec
+++ b/jobs/ingestor_syslog/spec
@@ -9,6 +9,7 @@ packages:
 templates:
   bin/ingestor_syslog_ctl: bin/ingestor_syslog_ctl
   bin/monit_debugger: bin/monit_debugger
+  bin/post-start.erb: bin/post-start
   config/input_and_output.conf.erb: config/input_and_output.conf
   config/filters_pre.conf.erb: config/filters_pre.conf
   config/filters_post.conf.erb: config/filters_post.conf
@@ -80,6 +81,16 @@ properties:
     description: Transport protocol to use
     default: "tcp"
 
+  logstash_ingestor.health.disable_post_start:
+    description: Allow node to run post-start script? (true / false)
+    default: false
+  logstash_ingestor.health.interval:
+    description: Logstash syslog health check interval (seconds)
+    default: 5
+  logstash_ingestor.health.timeout:
+    description: Logstash syslog health check number of attempts (seconds)
+    default: 300
+
   logstash_ingestor.syslog_tls.port:
     description: Port to listen for syslog-TLS messages (omit to disable)
   logstash_ingestor.syslog_tls.ssl_cert:
diff --git a/jobs/ingestor_syslog/templates/bin/post-start.erb b/jobs/ingestor_syslog/templates/bin/post-start.erb
new file mode 100644
index 00000000..4a001cb9
--- /dev/null
+++ b/jobs/ingestor_syslog/templates/bin/post-start.erb
@@ -0,0 +1,23 @@
+#!/bin/bash
+
+set -e
+
+<% if !p("logstash_ingestor.health.disable_post_start") %>
+
+echo "Waiting <%= p("logstash_ingestor.health.timeout") %>s for syslog to accept connections..."
+elapsed=0
+until [ $elapsed -ge <%= p("logstash_ingestor.health.timeout") %> ]
+do
+  nc -vz localhost <%= p("logstash_ingestor.syslog.port") %> 2>&1 && break
+  elapsed=$[$elapsed+<%= p("logstash_ingestor.health.interval") %>]
+  sleep <%= p("logstash_ingestor.health.interval") %>
+done
+
+if [ "$elapsed" -ge "<%= p("logstash_ingestor.health.timeout") %>" ]; then
+  echo "ERROR: Cannot connect to syslog. Exiting..."
+  exit 1
+fi
+
+<% end %>
+
+exit 0