From 1a806f0155adb83ee6cb6464b1cd1a07eceb835f Mon Sep 17 00:00:00 2001 From: commiterate <111539270+commiterate@users.noreply.github.com> Date: Sun, 15 Dec 2024 23:35:08 -0500 Subject: [PATCH] fluent-bit: add NixOS module --- ci/OWNERS | 5 +- .../manual/release-notes/rl-2505.section.md | 2 + nixos/modules/module-list.nix | 1 + .../services/monitoring/fluent-bit.nix | 107 ++++++++++++++++++ nixos/tests/all-tests.nix | 1 + nixos/tests/fluent-bit.nix | 40 +++++++ pkgs/by-name/fl/fluent-bit/package.nix | 50 +++++--- 7 files changed, 190 insertions(+), 16 deletions(-) create mode 100644 nixos/modules/services/monitoring/fluent-bit.nix create mode 100644 nixos/tests/fluent-bit.nix diff --git a/ci/OWNERS b/ci/OWNERS index d67e6b019b8cc..1ac600b389c87 100644 --- a/ci/OWNERS +++ b/ci/OWNERS @@ -143,10 +143,13 @@ nixos/modules/installer/tools/nix-fallback-paths.nix @NixOS/nix-team @raitobeza /nixos/tests/amazon-ssm-agent.nix @arianvp /nixos/modules/system/boot/grow-partition.nix @arianvp +# Monitoring +/nixos/modules/services/monitoring/fluent-bit.nix @samrose @fpletz +/nixos/tests/fluent-bit.nix @samrose @fpletz + # nixos-rebuild-ng /pkgs/by-name/ni/nixos-rebuild-ng @thiagokokada - # Updaters ## update.nix /maintainers/scripts/update.nix @jtojnar diff --git a/nixos/doc/manual/release-notes/rl-2505.section.md b/nixos/doc/manual/release-notes/rl-2505.section.md index d55b5bfd42a7d..2d0247fc7ac86 100644 --- a/nixos/doc/manual/release-notes/rl-2505.section.md +++ b/nixos/doc/manual/release-notes/rl-2505.section.md @@ -16,6 +16,8 @@ - [Amazon CloudWatch Agent](https://github.com/aws/amazon-cloudwatch-agent), the official telemetry collector for AWS CloudWatch and AWS X-Ray. Available as [services.amazon-cloudwatch-agent](#opt-services.amazon-cloudwatch-agent.enable). +- [Fluent Bit](https://github.com/fluent/fluent-bit), a fast Log, Metrics and Traces Processor and Forwarder. Available as [services.fluent-bit](#opt-services.fluent-bit.enable). + ## Backward Incompatibilities {#sec-release-25.05-incompatibilities} diff --git a/nixos/modules/module-list.nix b/nixos/modules/module-list.nix index 91ecee0ef265c..c20e471f9ceef 100644 --- a/nixos/modules/module-list.nix +++ b/nixos/modules/module-list.nix @@ -893,6 +893,7 @@ ./services/monitoring/das_watchdog.nix ./services/monitoring/datadog-agent.nix ./services/monitoring/do-agent.nix + ./services/monitoring/fluent-bit.nix ./services/monitoring/fusion-inventory.nix ./services/monitoring/gatus.nix ./services/monitoring/glances.nix diff --git a/nixos/modules/services/monitoring/fluent-bit.nix b/nixos/modules/services/monitoring/fluent-bit.nix new file mode 100644 index 0000000000000..a022c144986f1 --- /dev/null +++ b/nixos/modules/services/monitoring/fluent-bit.nix @@ -0,0 +1,107 @@ +{ + config, + lib, + pkgs, + ... +}: +let + cfg = config.services.fluent-bit; + + yamlFormat = pkgs.formats.yaml { }; + + configurationFile = + if (cfg.configurationFile == null) then + (yamlFormat.generate "fluent-bit.yaml" cfg.configuration) + else + cfg.configurationFile; +in +{ + options.services.fluent-bit = { + enable = lib.mkEnableOption "Fluent Bit"; + package = lib.mkPackageOption pkgs "fluent-bit" { }; + configurationFile = lib.mkOption { + type = lib.types.nullOr lib.types.path; + default = null; + description = '' + Fluent Bit configuration. See + + for supported values. + + {option}`configurationFile` takes precedence over {option}`configuration`. + + Note: Restricted evaluation blocks access to paths outside the Nix store. + This means detecting content changes for mutable paths (i.e. not input or content-addressed) can't be done. + As a result, `nixos-rebuild` won't reload/restart the systemd unit when mutable path contents change. + `systemctl restart fluent-bit.service` must be used instead. + ''; + example = /etc/fluent-bit/fluent-bit.yaml; + }; + configuration = lib.mkOption { + type = yamlFormat.type; + default = { }; + description = '' + See {option}`configurationFile`. + + {option}`configurationFile` takes precedence over {option}`configuration`. + ''; + example = { + service = { + grace = 30; + }; + pipeline = { + inputs = [ + { + name = "systemd"; + systemd_filter = "_SYSTEMD_UNIT=fluent-bit.service"; + } + ]; + outputs = [ + { + name = "file"; + path = "/var/log/fluent-bit"; + file = "fluent-bit.out"; + } + ]; + }; + }; + }; + user = lib.mkOption { + type = lib.types.str; + default = "root"; + description = '' + The user that runs Fluent Bit. + ''; + example = "fluent-bit"; + }; + # See https://docs.fluentbit.io/manual/administration/configuring-fluent-bit/yaml/service-section. + grace = lib.mkOption { + type = lib.types.ints.positive; + default = 5; + description = '' + The grace time in `seconds` as an integer value. The engine loop uses a grace timeout to define the wait time before exiting. + + The `service.grace` option in the Fluent Bit configuration should be ≤ this option. + ''; + example = 30; + }; + }; + + config = lib.mkIf cfg.enable { + # See https://github.com/fluent/fluent-bit/blob/v3.2.2/init/systemd.in. + systemd.services.fluent-bit = { + description = "Fluent Bit"; + after = [ "network.target" ]; + requires = [ "network.target" ]; + wantedBy = [ "multi-user.target" ]; + serviceConfig = { + User = cfg.user; + ExecStart = builtins.concatStringsSep " " [ + "${cfg.package}/bin/fluent-bit" + "--config ${configurationFile}" + ]; + TimeoutStopSec = cfg.grace; + Restart = "always"; + }; + }; + }; +} diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix index 47c1ce41671bb..6981b9a23336f 100644 --- a/nixos/tests/all-tests.nix +++ b/nixos/tests/all-tests.nix @@ -350,6 +350,7 @@ in { flaresolverr = handleTest ./flaresolverr.nix {}; flood = handleTest ./flood.nix {}; floorp = handleTest ./firefox.nix { firefoxPackage = pkgs.floorp; }; + fluent-bit = handleTest ./fluent-bit.nix {}; fluentd = handleTest ./fluentd.nix {}; fluidd = handleTest ./fluidd.nix {}; fontconfig-default-fonts = handleTest ./fontconfig-default-fonts.nix {}; diff --git a/nixos/tests/fluent-bit.nix b/nixos/tests/fluent-bit.nix new file mode 100644 index 0000000000000..2fa6cd34c06fa --- /dev/null +++ b/nixos/tests/fluent-bit.nix @@ -0,0 +1,40 @@ +import ./make-test-python.nix ( + { lib, pkgs, ... }: + { + name = "fluent-bit"; + + nodes.machine = + { config, pkgs, ... }: + { + services.fluent-bit = { + enable = true; + configuration = { + pipeline = { + inputs = [ + { + name = "systemd"; + systemd_filter = "_SYSTEMD_UNIT=fluent-bit.service"; + } + ]; + outputs = [ + { + name = "file"; + path = "/var/log/fluent-bit"; + file = "fluent-bit.out"; + } + ]; + }; + }; + }; + + systemd.services.fluent-bit.serviceConfig.LogsDirectory = "fluent-bit"; + }; + + testScript = '' + start_all() + + machine.wait_for_unit("fluent-bit.service") + machine.wait_for_file("/var/log/fluent-bit/fluent-bit.out") + ''; + } +) diff --git a/pkgs/by-name/fl/fluent-bit/package.nix b/pkgs/by-name/fl/fluent-bit/package.nix index 8c6c762e66c34..d1568aeabccc3 100644 --- a/pkgs/by-name/fl/fluent-bit/package.nix +++ b/pkgs/by-name/fl/fluent-bit/package.nix @@ -1,15 +1,18 @@ { lib, - stdenv, - fetchFromGitHub, + bison, cmake, + darwin, + fetchFromGitHub, flex, - bison, - systemd, - postgresql, - openssl, libyaml, - darwin, + nix-update-script, + nixosTests, + openssl, + postgresql, + stdenv, + systemd, + versionCheckHook, }: stdenv.mkDerivation (finalAttrs: { @@ -23,9 +26,14 @@ stdenv.mkDerivation (finalAttrs: { hash = "sha256-/YKXWYaDqkJ8fWgcYktXWJPcjDJTupgRTSQVp0cWruY="; }; - # optional only to avoid linux rebuild + # Optional only to avoid Linux rebuild. patches = lib.optionals stdenv.hostPlatform.isDarwin [ ./macos-11-sdk-compat.patch ]; + postPatch = '' + substituteInPlace src/CMakeLists.txt \ + --replace /lib/systemd $out/lib/systemd + ''; + nativeBuildInputs = [ cmake flex @@ -52,9 +60,9 @@ stdenv.mkDerivation (finalAttrs: { ] ++ lib.optionals stdenv.hostPlatform.isDarwin [ "-DCMAKE_OSX_DEPLOYMENT_TARGET=10.13" ]; env.NIX_CFLAGS_COMPILE = toString ( - # Used by the embedded luajit, but is not predefined on older mac SDKs. + # Used by the embedded luajit, but is not predefined on older Apple SDKs. lib.optionals stdenv.hostPlatform.isDarwin [ "-DTARGET_OS_IPHONE=0" ] - # Assumes GNU version of strerror_r, and the posix version has an + # Assumes GNU version of strerror_r, and the POSIX version has an # incompatible return type. ++ lib.optionals (!stdenv.hostPlatform.isGnu) [ "-Wno-int-conversion" ] ); @@ -64,16 +72,28 @@ stdenv.mkDerivation (finalAttrs: { "dev" ]; - postPatch = '' - substituteInPlace src/CMakeLists.txt \ - --replace /lib/systemd $out/lib/systemd - ''; + doInstallCheck = true; + + nativeInstallCheckInputs = [ versionCheckHook ]; + + versionCheckProgram = "${builtins.placeholder "out"}/bin/${finalAttrs.meta.mainProgram}"; + + versionCheckProgramArg = "--version"; + + passthru = { + tests = lib.optionalAttrs stdenv.isLinux { + inherit (nixosTests) fluent-bit; + }; + + updateScript = nix-update-script { }; + }; meta = { changelog = "https://github.com/fluent/fluent-bit/releases/tag/v${finalAttrs.version}"; - description = "Log forwarder and processor, part of Fluentd ecosystem"; + description = "Fast and lightweight logs and metrics processor for Linux, BSD, OSX and Windows"; homepage = "https://fluentbit.io"; license = lib.licenses.asl20; + mainProgram = "fluent-bit"; maintainers = with lib.maintainers; [ samrose fpletz