forked from cloudfoundry-community/logsearch-boshrelease
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request cloudfoundry-community#1 from 18F/logstash-post-start
Merge feature branches to 18F develop
- Loading branch information
Showing
50 changed files
with
534 additions
and
1,261 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,5 @@ | ||
| check process archiver_syslog | ||
| with pidfile /var/vcap/sys/run/archiver_syslog/archiver_syslog.pid | ||
| start program "/var/vcap/jobs/archiver_syslog/bin/monit_debugger archiver_syslog_ctl '/var/vcap/jobs/archiver_syslog/bin/archiver_syslog_ctl start'" with timeout 120 seconds | ||
| stop program "/var/vcap/jobs/archiver_syslog/bin/monit_debugger archiver_syslog_ctl '/var/vcap/jobs/archiver_syslog/bin/archiver_syslog_ctl stop'" | ||
| group vcap |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,110 @@ | ||
| --- | ||
| name: archiver_syslog | ||
|
|
||
| packages: | ||
| - logstash | ||
| - logsearch-config | ||
| - java8 | ||
|
|
||
| templates: | ||
| bin/archiver_syslog_ctl: bin/archiver_syslog_ctl | ||
| bin/monit_debugger: bin/monit_debugger | ||
| bin/post-start.erb: bin/post-start | ||
| config/input_and_output.conf.erb: config/input_and_output.conf | ||
| config/syslog_tls.crt.erb: config/syslog_tls.crt | ||
| config/syslog_tls.key.erb: config/syslog_tls.key | ||
| config/logstash.yml.erb: config/logstash.yml | ||
| data/properties.sh.erb: data/properties.sh | ||
| helpers/ctl_setup.sh: helpers/ctl_setup.sh | ||
| helpers/ctl_utils.sh: helpers/ctl_utils.sh | ||
|
|
||
| provides: | ||
| - name: archiver | ||
| type: archiver | ||
| properties: | ||
| - logstash_ingestor.syslog.port | ||
| - logstash_ingestor.syslog.transport | ||
| - logstash_ingestor.syslog_tls.port | ||
| - logstash_ingestor.relp.port | ||
|
|
||
| properties: | ||
| logstash.heap_size: | ||
| description: sets jvm heap sized | ||
| logstash.metadata_level: | ||
| description: "Whether to include additional metadata throughout the event lifecycle. NONE = disabled, DEBUG = fully enabled" | ||
| default: "NONE" | ||
| logstash.log_level: | ||
| description: The default logging level (e.g. WARN, DEBUG, INFO) | ||
| default: info | ||
| logstash.plugins: | ||
| description: "Array of hashes describing logstash plugins to install" | ||
| example: | ||
| - {name: logstash-output-cloudwatchlogs, version: 2.0.0} | ||
| default: [] | ||
| logstash.env: | ||
| description: "a list of arbitrary key-value pairs to be passed on as process environment variables. eg: FOO: 123" | ||
| default: [] | ||
|
|
||
| logstash_ingestor.debug: | ||
| description: Debug level logging | ||
| default: false | ||
| logstash_ingestor.workers: | ||
| description: "The number of worker threads that logstash should use (default: auto = one per CPU)" | ||
| default: auto | ||
|
|
||
| logstash.queue.type: | ||
| description: Internal queuing model, "memory" for legacy in-memory based queuing and "persisted" for disk-based acked queueing. | ||
| default: persisted | ||
| logstash.queue.page_capacity: | ||
| description: The page data files size. The queue data consists of append-only data files separated into pages. | ||
| default: 250mb | ||
| logstash.queue.max_events: | ||
| description: The maximum number of unread events in the queue. | ||
| default: 0 | ||
| logstash.queue.max_bytes: | ||
| description: The total capacity of the queue in number of bytes. | ||
| default: 1024mb | ||
| logstash.queue.checkpoint.acks: | ||
| description: The maximum number of acked events before forcing a checkpoint. | ||
| default: 1024 | ||
| logstash.queue.checkpoint.writes: | ||
| description: The maximum number of written events before forcing a checkpoint. | ||
| default: 1024 | ||
| logstash.queue.checkpoint.interval: | ||
| description: The interval in milliseconds when a checkpoint is forced on the head page. | ||
| default: 1000 | ||
|
|
||
| logstash_ingestor.syslog.port: | ||
| description: Port to listen for syslog messages | ||
| default: 5514 | ||
| logstash_ingestor.syslog.transport: | ||
| description: Transport protocol to use | ||
| default: "tcp" | ||
|
|
||
| logstash_ingestor.health.disable_post_start: | ||
| description: Allow node to run post-start script? (true / false) | ||
| default: false | ||
| logstash_ingestor.health.interval: | ||
| description: Logstash syslog health check interval (seconds) | ||
| default: 5 | ||
| logstash_ingestor.health.timeout: | ||
| description: Logstash syslog health check number of attempts (seconds) | ||
| default: 300 | ||
|
|
||
| logstash_ingestor.syslog_tls.port: | ||
| description: Port to listen for syslog-TLS messages (omit to disable) | ||
| logstash_ingestor.syslog_tls.ssl_cert: | ||
| description: Syslog-TLS SSL certificate (file contents, not a path) - required if logstash_ingestor.syslog_tls.port set | ||
| logstash_ingestor.syslog_tls.ssl_key: | ||
| description: Syslog-TLS SSL key (file contents, not a path) - required if logstash_ingestor.syslog_tls.port set | ||
| logstash_ingestor.syslog_tls.skip_ssl_validation: | ||
| description: Verify the identity of the other end of the SSL connection against the CA. | ||
| default: false | ||
|
|
||
| logstash_ingestor.relp.port: | ||
| description: Port to listen for RELP messages | ||
| default: 2514 | ||
|
|
||
| logstash_ingestor.outputs: | ||
| description: A list of output plugins, with a hash of options for each of them. | ||
| default: [] |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,66 @@ | ||
| #!/bin/bash | ||
|
|
||
| set -e # exit immediately if a simple command exits with a non-zero status | ||
| set -u # report the usage of uninitialized variables | ||
|
|
||
| # Setup env vars and folders for the webapp_ctl script | ||
| source /var/vcap/jobs/archiver_syslog/helpers/ctl_setup.sh 'archiver_syslog' | ||
|
|
||
| export PORT=${PORT:-5000} | ||
| export LANG=en_US.UTF-8 | ||
| <% if 'auto' == p('logstash_ingestor.workers') %> | ||
| # 1 logstash worker / CPU core | ||
| export LOGSTASH_WORKERS=`grep -c ^processor /proc/cpuinfo` | ||
| <% else %> | ||
| export LOGSTASH_WORKERS=<%= p('logstash_ingestor.workers') %> | ||
| <% end %> | ||
| export HEAP_SIZE=$((( $( cat /proc/meminfo | grep MemTotal | awk '{ print $2 }' ) * 46 ) / 100 ))K | ||
| <% if_p('logstash.heap_size') do |heap_size| %> | ||
| HEAP_SIZE=<%= heap_size %> | ||
| <% end %> | ||
| <% p("logstash.env").each do |env| %> | ||
| export <%= env.keys[0] %>="<%= env.values[0] %>" | ||
| <% end %> | ||
|
|
||
| case $1 in | ||
|
|
||
| start) | ||
| pid_guard $PIDFILE $JOB_NAME | ||
|
|
||
| # store this processes pid in $PIDFILE, since the exec below doesn't daemonize | ||
| echo $$ > $PIDFILE | ||
|
|
||
| export LS_JAVA_OPTS="-Xms$HEAP_SIZE -Xmx$HEAP_SIZE -DPID=$$" | ||
|
|
||
| <% p("logstash.plugins").each do |plugin| %> | ||
| /var/vcap/packages/logstash/bin/logstash-plugin install \ | ||
| <%= plugin.except("name").map { |key, value| "--#{key}=#{value}" }.join(" ") %> \ | ||
| <%= plugin["name"] %> | ||
| <% end %> | ||
|
|
||
| # construct a complete config file from all the fragments | ||
| cat ${JOB_DIR}/config/input_and_output.conf > ${JOB_DIR}/config/logstash.conf | ||
|
|
||
| exec chpst -u vcap:vcap /var/vcap/packages/logstash/bin/logstash \ | ||
| --path.data ${STORE_DIR} \ | ||
| --path.config ${JOB_DIR}/config/logstash.conf \ | ||
| --path.settings ${JOB_DIR}/config \ | ||
| --pipeline.workers ${LOGSTASH_WORKERS} \ | ||
| --log.format=json --log.level=<%= p("logstash.log_level") %> \ | ||
| >>$LOG_DIR/$JOB_NAME.stdout.log \ | ||
| 2>>$LOG_DIR/$JOB_NAME.stderr.log | ||
|
|
||
| ;; | ||
|
|
||
| stop) | ||
| kill_and_wait $PIDFILE | ||
| ensure_no_more_logstash | ||
|
|
||
| ;; | ||
| *) | ||
| echo "Usage: archiver_syslog_ctl {start|stop}" | ||
|
|
||
| ;; | ||
|
|
||
| esac | ||
| exit 0 |
File renamed without changes.
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,23 @@ | ||
| #!/bin/bash | ||
|
|
||
| set -e | ||
|
|
||
| <% if !p("logstash_ingestor.health.disable_post_start") %> | ||
|
|
||
| echo "Waiting <%= p("logstash_ingestor.health.timeout") %>s for syslog to accept connections..." | ||
| elapsed=0 | ||
| until [ $elapsed -ge <%= p("logstash_ingestor.health.timeout") %> ] | ||
| do | ||
| nc -vz localhost <%= p("logstash_ingestor.syslog.port") %> 2>&1 && break | ||
| elapsed=$[$elapsed+<%= p("logstash_ingestor.health.interval") %>] | ||
| sleep <%= p("logstash_ingestor.health.interval") %> | ||
| done | ||
|
|
||
| if [ "$elapsed" -ge "<%= p("logstash_ingestor.health.timeout") %>" ]; then | ||
| echo "ERROR: Cannot connect to syslog. Exiting..." | ||
| exit 1 | ||
| fi | ||
|
|
||
| <% end %> | ||
|
|
||
| exit 0 |
Oops, something went wrong.