From 607bcc23e4af5fca5e5a0a0e938195dabc7ab59f Mon Sep 17 00:00:00 2001 From: Chris Minton Date: Tue, 2 Jul 2019 14:41:20 +0100 Subject: [PATCH 1/3] Update the config and prospector resources to conform to the filebeat 6.0+ and 6.3+ configuration formats --- resources/config.rb | 11 +++++++++-- resources/prospector.rb | 7 ++++++- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/resources/config.rb b/resources/config.rb index 4751934..7e3efe2 100755 --- a/resources/config.rb +++ b/resources/config.rb @@ -3,7 +3,7 @@ # Resource:: filebeat_config # -default_config = { 'filebeat.prospectors' => [], 'filebeat.modules' => [], 'prospectors' => [] } +default_config = { 'filebeat.inputs' => [], 'filebeat.prospectors' => [], 'filebeat.modules' => [], 'prospectors' => [] } resource_name :filebeat_config @@ -29,8 +29,15 @@ logging_files_path = node['platform'] == 'windows' ? "#{filebeat_install_resource.conf_dir}/logs" : filebeat_install_resource.log_dir config['filebeat.registry_file'] = node['platform'] == 'windows' ? "#{filebeat_install_resource.conf_dir}/registry" : '/var/lib/filebeat/registry' - config['filebeat.config_dir'] = filebeat_install_resource.prospectors_dir config['logging.files']['path'] ||= logging_files_path + if filebeat_install_resource.version.to_f >= 6.0 + config['filebeat.config.inputs'] ||= { + 'enabled' => true, + 'path' => "#{filebeat_install_resource.prospectors_dir}/*.yml", + } + else + config['filebeat.config_dir'] = filebeat_install_resource.prospectors_dir + end # Filebeat and psych v1.x don't get along. if Psych::VERSION.start_with?('1') diff --git a/resources/prospector.rb b/resources/prospector.rb index d739772..913b58c 100755 --- a/resources/prospector.rb +++ b/resources/prospector.rb @@ -33,7 +33,12 @@ end # file_content = { 'filebeat' => { 'prospectors' => config } }.to_yaml - file_content = JSON.parse({ 'filebeat' => { 'prospectors' => config } }.to_json).to_yaml.lines.to_a[1..-1].join + file_content = + if filebeat_install_resource.version.to_f >= 6.3 + JSON.parse(config.to_json).to_yaml.lines.to_a[1..-1].join + else + JSON.parse({ 'filebeat' => { 'prospectors' => config } }.to_json).to_yaml.lines.to_a[1..-1].join + end # ...and put this back the way we found them. YAML::ENGINE.yamler = defaultengine if Psych::VERSION.start_with?('1') From b3c6b53f5d69d3e635b21c0436538c870a6feb0a Mon Sep 17 00:00:00 2001 From: Chris Minton Date: Tue, 2 Jul 2019 14:41:43 +0100 Subject: [PATCH 2/3] Update the integration tests for configs --- .kitchen.dokken.yml | 2 +- test/cookbooks/filebeat_test/README.md | 2 -- test/smoke/v6/default.rb | 9 +++++++++ test/smoke/v6runit/default.rb | 4 ++++ 4 files changed, 14 insertions(+), 3 deletions(-) diff --git a/.kitchen.dokken.yml b/.kitchen.dokken.yml index 4d1a257..b4d4e3a 100644 --- a/.kitchen.dokken.yml +++ b/.kitchen.dokken.yml @@ -35,7 +35,7 @@ platforms: - name: debian8 driver: - image: debian:8.9 + image: dokken/debian-8 pid_one_command: /sbin/init intermediate_instructions: - RUN /usr/bin/apt-get update -qq diff --git a/test/cookbooks/filebeat_test/README.md b/test/cookbooks/filebeat_test/README.md index 2841976..809da08 100644 --- a/test/cookbooks/filebeat_test/README.md +++ b/test/cookbooks/filebeat_test/README.md @@ -1,4 +1,2 @@ filebeat_test ===== - - diff --git a/test/smoke/v6/default.rb b/test/smoke/v6/default.rb index 0294d9c..026e364 100644 --- a/test/smoke/v6/default.rb +++ b/test/smoke/v6/default.rb @@ -15,6 +15,15 @@ end end +describe file('/etc/filebeat/filebeat.yml') do + its('content') { should match 'filebeat.config.inputs' } +end + +describe command('filebeat test config') do + its('exit_status') { should eq 0 } + its('stdout') { should match 'Config OK' } +end + describe package('filebeat') do it { should be_installed } its('version') { should match '6.4.2' } diff --git a/test/smoke/v6runit/default.rb b/test/smoke/v6runit/default.rb index e69de29..a14ed66 100644 --- a/test/smoke/v6runit/default.rb +++ b/test/smoke/v6runit/default.rb @@ -0,0 +1,4 @@ +describe command('filebeat test config') do + its('exit_status') { should eq 0 } + its('stdout') { should match 'Config OK' } +end From e207bb924033eae1e6c9864dbbe6e91b7a10d3cc Mon Sep 17 00:00:00 2001 From: Chris Minton Date: Tue, 2 Jul 2019 14:42:10 +0100 Subject: [PATCH 3/3] update the README to give the correct configuration for different versions --- README.md | 94 +++++++++++++++++++++++++++++++++++++++++-------------- 1 file changed, 71 insertions(+), 23 deletions(-) diff --git a/README.md b/README.md index 67f8cdf..f817321 100644 --- a/README.md +++ b/README.md @@ -5,31 +5,26 @@ filebeat Cookbook This is a [Chef] cookbook to manage [Filebeat]. - >> For Production environment, always prefer the [most recent release](https://supermarket.chef.io/cookbooks/filebeat). - ## Most Recent Release ```ruby cookbook 'filebeat', '~> 2.2.0' ``` - ## From Git ```ruby cookbook 'filebeat', github: 'vkhatri/chef-filebeat', tag: 'v2.2.0' ``` - ## Repository -``` +```bash https://github.com/vkhatri/chef-filebeat ``` - ## Supported OS - Windows @@ -42,20 +37,17 @@ https://github.com/vkhatri/chef-filebeat Also works on Solaris zones given a physical Solaris 11.2 server. For that, use the .kitchen.zone.yml file. Check usage at (https://github.com/criticalmass/kitchen-zone). You will need an url to a filebeat package that works on Solaris 11.2. Checkout Building-Filebeat-On-Solaris11.md for instructions to build a filebeat package. - ## Supported Chef - Chef 12 (last tested on 12.21.4) - Chef 13 (last tested on 13.3.42) - ## Supported Filebeat - 5.x - 6.x - ## Cookbook Dependency - `homebrew` @@ -64,12 +56,10 @@ Also works on Solaris zones given a physical Solaris 11.2 server. For that, use - `runit` - `windows` - ## Recipes - lwrp_test - LWRP examples recipe - ## LWRP Resources - `filebeat_config` - filebeat configuration resource @@ -82,14 +72,12 @@ Also works on Solaris zones given a physical Solaris 11.2 server. For that, use - `filebeat_runit_service` - filebeat service resource using runit -- `filebeat_prospector` - filebeat prospector resource - +- `filebeat_prospector` - filebeat prospector resource (renamed filebeat inputs in version 6.3+) ## Limitations The Mac OSX setup only allows for package installs and depends on brew, this means that version selection and preview build installs are not supported. - ## LWRP filebeat_install LWRP `filebeat_install` installs filebeat, creates log/prospectors directories, and also enable filebeat service. @@ -153,7 +141,6 @@ end - *service_retries* (optional, Integer) - default `2`, filebeat service resource attribute - *service_retry_delay* (optional, Integer) - default `0`, filebeat service resource attribute - ## LWRP filebeat_config LWRP `filebeat_config` creates filebeat configuration yaml file `/etc/filebeat/filebeat.yml`. @@ -164,8 +151,7 @@ Below filebeat configuration parameters gets overwritten by the LWRP. - `filebeat.config_dir` - `logging.files` - -**LWRP example** +### Filebeat version < 6.0 ```ruby conf = { @@ -199,6 +185,42 @@ filebeat.registry_file: "/var/lib/filebeat/registry" filebeat.config_dir: "/etc/filebeat/conf.d" ``` +### Filebeat version 6.0+ + +```ruby +conf = { + 'filebeat.modules' => [], + 'filebeat.inputs' => [], + 'logging.level' => 'info', + 'logging.to_files' => true, + 'logging.files' => { 'name' => 'filebeat' }, + 'output.elasticsearch' => { 'hosts' => ['127.0.0.1:9200'] } +} + +filebeat_config 'default' do + config conf + action :create +end +``` + +Above LWRP Resource will create a file `/etc/filebeat/filebeat.yml` with content: + +```yaml +filebeat.modules: [] +prospectors: [] +logging.level: info +logging.to_files: true +logging.files: + path: "/var/log/filebeat" +output.elasticsearch: + hosts: + - 127.0.0.1:9200 +filebeat.registry_file: "/var/lib/filebeat/registry" +filebeat.config.inputs: + enabled: true + path: "/etc/filebeat/conf.d/*.yml" +``` + **LWRP Options** - *action* (optional) - default `:create`, options: :create, :delete @@ -212,8 +234,9 @@ filebeat.config_dir: "/etc/filebeat/conf.d" - *prefix* (optional, String) - default `lwrp-prospector-`, filebeat prospecteor filename prefix, set to '' if no prefix is desired -## LWRP filebeat_prospector +## LWRP filebeat_prospector (inputs) +### Filebeat version up to 6.3 LWRP `filebeat_prospector` creates a filebeat prospector configuration yaml file under prospectors directory with file name `lwrp-prospector-#{resource_name}.yml`. `lwrp-prospector-` prefix can be changed with `prefix` property (see above). @@ -247,6 +270,36 @@ filebeat: type: messages_log ``` +### Filebeat version 6.3+ + +With `filebeat.config.inputs` set, Filebeat 6.3+ will read in the files as previous versions and the same config will work as expected + +```ruby +conf = { + 'type' => 'log', + 'enabled' => true, + 'paths' => ['/var/log/messages'], + 'type' => 'log', + 'fields' => {'type' => 'messages_log'} +} + +filebeat_prospector 'messages_log' do + config conf + action :create +end +``` + +the file is created with content: + +```yaml +- enabled: true + paths: + - "/var/log/messages" + type: log + fields: + type: messages_log +``` + **LWRP Options** - *action* (optional) - default `:create`, options: :create, :delete @@ -257,7 +310,6 @@ filebeat: - *disable_service* (optional, Boolean) - default `false`, set to `true`, to disable filebeat service - *notify_restart* (optional, Boolean) - default `true`, set to `false`, to ignore filebeat service restart notify - ## LWRP filebeat_runit_service LWRP `filebeat_runit_service` configures `filebeat` service using `runit`. @@ -286,12 +338,10 @@ end Check out filebeat test cookbook [filebeat_test](test/cookbooks/filebeat_test). - ## TODO - Add other platforms support to install_preview resource - ## Contributing 1. Fork the repository on Github @@ -303,7 +353,6 @@ Check out filebeat test cookbook [filebeat_test](test/cookbooks/filebeat_test). 7. Write description about changes to PR 8. Submit a Pull Request using Github - ## Copyright & License Authors:: Virender Khatri and [Contributors] @@ -322,7 +371,6 @@ See the License for the specific language governing permissions and limitations under the License. - [Chef]: https://www.chef.io/ [Filebeat]: https://www.elastic.co/products/beats/filebeat [Contributors]: https://github.com/vkhatri/chef-filebeat/graphs/contributors