From d6993876deb6b70ade7109c992b0d8811706f64b Mon Sep 17 00:00:00 2001 From: commiterate <111539270+commiterate@users.noreply.github.com> Date: Sun, 15 Dec 2024 23:35:08 -0500 Subject: [PATCH] fluent-bit: add NixOS module --- ci/OWNERS | 5 +- .../manual/release-notes/rl-2505.section.md | 2 + nixos/modules/module-list.nix | 1 + .../services/monitoring/fluent-bit.nix | 105 ++++++++++++++++++ nixos/tests/all-tests.nix | 1 + nixos/tests/fluent-bit.nix | 40 +++++++ 6 files changed, 153 insertions(+), 1 deletion(-) create mode 100644 nixos/modules/services/monitoring/fluent-bit.nix create mode 100644 nixos/tests/fluent-bit.nix diff --git a/ci/OWNERS b/ci/OWNERS index d67e6b019b8cc..1ac600b389c87 100644 --- a/ci/OWNERS +++ b/ci/OWNERS @@ -143,10 +143,13 @@ nixos/modules/installer/tools/nix-fallback-paths.nix @NixOS/nix-team @raitobeza /nixos/tests/amazon-ssm-agent.nix @arianvp /nixos/modules/system/boot/grow-partition.nix @arianvp +# Monitoring +/nixos/modules/services/monitoring/fluent-bit.nix @samrose @fpletz +/nixos/tests/fluent-bit.nix @samrose @fpletz + # nixos-rebuild-ng /pkgs/by-name/ni/nixos-rebuild-ng @thiagokokada - # Updaters ## update.nix /maintainers/scripts/update.nix @jtojnar diff --git a/nixos/doc/manual/release-notes/rl-2505.section.md b/nixos/doc/manual/release-notes/rl-2505.section.md index d55b5bfd42a7d..87ee1f3701a41 100644 --- a/nixos/doc/manual/release-notes/rl-2505.section.md +++ b/nixos/doc/manual/release-notes/rl-2505.section.md @@ -16,6 +16,8 @@ - [Amazon CloudWatch Agent](https://github.com/aws/amazon-cloudwatch-agent), the official telemetry collector for AWS CloudWatch and AWS X-Ray. Available as [services.amazon-cloudwatch-agent](#opt-services.amazon-cloudwatch-agent.enable). +- [Fluent Bit](https://github.com/fluent/fluent-bit), a fast Log, Metrics and Traces Processor and Forwarder. Available as [services.fluent-bit](#opt-services.fluent-bit.enable) + ## Backward Incompatibilities {#sec-release-25.05-incompatibilities} diff --git a/nixos/modules/module-list.nix b/nixos/modules/module-list.nix index 91ecee0ef265c..c20e471f9ceef 100644 --- a/nixos/modules/module-list.nix +++ b/nixos/modules/module-list.nix @@ -893,6 +893,7 @@ ./services/monitoring/das_watchdog.nix ./services/monitoring/datadog-agent.nix ./services/monitoring/do-agent.nix + ./services/monitoring/fluent-bit.nix ./services/monitoring/fusion-inventory.nix ./services/monitoring/gatus.nix ./services/monitoring/glances.nix diff --git a/nixos/modules/services/monitoring/fluent-bit.nix b/nixos/modules/services/monitoring/fluent-bit.nix new file mode 100644 index 0000000000000..14ea0cbc6ed00 --- /dev/null +++ b/nixos/modules/services/monitoring/fluent-bit.nix @@ -0,0 +1,105 @@ +{ + config, + lib, + pkgs, + ... +}: +let + cfg = config.services.fluent-bit; + + yamlFormat = pkgs.formats.yaml { }; + + configurationFile = + if (cfg.configurationFile == null) then + (yamlFormat.generate "fluent-bit.yaml" cfg.configuration) + else + cfg.configurationFile; +in +{ + options.services.fluent-bit = { + enable = lib.mkEnableOption "Fluent Bit"; + package = lib.mkPackageOption pkgs "fluent-bit" { }; + configurationFile = lib.mkOption { + type = lib.types.nullOr lib.types.path; + default = null; + description = '' + Fluent Bit configuration. See + + for supported values. + + {option}`configurationFile` takes precedence over {option}`configuration`. + + Note: Restricted evaluation blocks access to paths outside the Nix store. + This means detecting content changes for mutable paths (i.e. not input or content-addressed) can't be done. + As a result, `nixos-rebuild` won't reload/restart the systemd unit when mutable path contents change. + `systemctl restart fluent-bit.service` must be used instead. + ''; + example = /etc/fluent-bit/fluent-bit.yaml; + }; + configuration = lib.mkOption { + type = yamlFormat.type; + default = { }; + description = '' + See {option}`configurationFile`. + + {option}`configurationFile` takes precedence over {option}`configuration`. + ''; + example = { + service = { + grace = 30; + }; + pipeline = { + inputs = [ + { + name = "systemd"; + systemd_filter = "_SYSTEMD_UNIT=fluent-bit.service"; + } + ]; + outputs = [ + { + name = "stdout"; + } + ]; + }; + }; + }; + user = lib.mkOption { + type = lib.types.str; + default = "root"; + description = '' + The user that runs Fluent Bit. + ''; + example = "fluent-bit"; + }; + # See https://docs.fluentbit.io/manual/administration/configuring-fluent-bit/yaml/service-section. + grace = lib.mkOption { + type = lib.types.ints.positive; + default = 5; + description = '' + The grace time in `seconds` as an integer value. The engine loop uses a grace timeout to define the wait time before exiting. + + The `service.grace` option in the Fluent Bit configuration should be ≤ this option. + ''; + example = 30; + }; + }; + + config = lib.mkIf cfg.enable { + # See https://github.com/fluent/fluent-bit/blob/v3.2.2/init/systemd.in. + systemd.services.fluent-bit = { + description = "Fluent Bit"; + after = [ "network.target" ]; + requires = [ "network.target" ]; + wantedBy = [ "multi-user.target" ]; + serviceConfig = { + User = cfg.user; + ExecStart = builtins.concatStringsSep " " [ + "${cfg.package}/bin/fluent-bit" + "--config ${configurationFile}" + ]; + TimeoutStopSec = cfg.grace; + Restart = "always"; + }; + }; + }; +} diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix index 47c1ce41671bb..6981b9a23336f 100644 --- a/nixos/tests/all-tests.nix +++ b/nixos/tests/all-tests.nix @@ -350,6 +350,7 @@ in { flaresolverr = handleTest ./flaresolverr.nix {}; flood = handleTest ./flood.nix {}; floorp = handleTest ./firefox.nix { firefoxPackage = pkgs.floorp; }; + fluent-bit = handleTest ./fluent-bit.nix {}; fluentd = handleTest ./fluentd.nix {}; fluidd = handleTest ./fluidd.nix {}; fontconfig-default-fonts = handleTest ./fontconfig-default-fonts.nix {}; diff --git a/nixos/tests/fluent-bit.nix b/nixos/tests/fluent-bit.nix new file mode 100644 index 0000000000000..2fa6cd34c06fa --- /dev/null +++ b/nixos/tests/fluent-bit.nix @@ -0,0 +1,40 @@ +import ./make-test-python.nix ( + { lib, pkgs, ... }: + { + name = "fluent-bit"; + + nodes.machine = + { config, pkgs, ... }: + { + services.fluent-bit = { + enable = true; + configuration = { + pipeline = { + inputs = [ + { + name = "systemd"; + systemd_filter = "_SYSTEMD_UNIT=fluent-bit.service"; + } + ]; + outputs = [ + { + name = "file"; + path = "/var/log/fluent-bit"; + file = "fluent-bit.out"; + } + ]; + }; + }; + }; + + systemd.services.fluent-bit.serviceConfig.LogsDirectory = "fluent-bit"; + }; + + testScript = '' + start_all() + + machine.wait_for_unit("fluent-bit.service") + machine.wait_for_file("/var/log/fluent-bit/fluent-bit.out") + ''; + } +)