diff --git a/ci/OWNERS b/ci/OWNERS index 35edeaf6cf775..995ea3248657b 100644 --- a/ci/OWNERS +++ b/ci/OWNERS @@ -143,10 +143,13 @@ nixos/modules/installer/tools/nix-fallback-paths.nix @NixOS/nix-team @raitobeza /nixos/tests/amazon-ssm-agent.nix @arianvp /nixos/modules/system/boot/grow-partition.nix @arianvp +# Monitoring +/nixos/modules/services/monitoring/fluent-bit.nix @samrose +/nixos/tests/fluent-bit.nix @samrose + # nixos-rebuild-ng /pkgs/by-name/ni/nixos-rebuild-ng @thiagokokada - # Updaters ## update.nix /maintainers/scripts/update.nix @jtojnar diff --git a/nixos/doc/manual/release-notes/rl-2505.section.md b/nixos/doc/manual/release-notes/rl-2505.section.md index a6ac39a37c791..5638ce2e4f554 100644 --- a/nixos/doc/manual/release-notes/rl-2505.section.md +++ b/nixos/doc/manual/release-notes/rl-2505.section.md @@ -56,6 +56,8 @@ - [git-worktree-switcher](https://github.com/mateusauler/git-worktree-switcher), switch between git worktrees with speed. Available as [programs.git-worktree-switcher](#opt-programs.git-worktree-switcher.enable) +- [Fluent Bit](https://github.com/fluent/fluent-bit), a fast Log, Metrics and Traces Processor and Forwarder. Available as [services.fluent-bit](#opt-services.fluent-bit.enable). + ## Backward Incompatibilities {#sec-release-25.05-incompatibilities} diff --git a/nixos/modules/module-list.nix b/nixos/modules/module-list.nix index 783f20546af44..29b39d7f94c6d 100644 --- a/nixos/modules/module-list.nix +++ b/nixos/modules/module-list.nix @@ -904,6 +904,7 @@ ./services/monitoring/das_watchdog.nix ./services/monitoring/datadog-agent.nix ./services/monitoring/do-agent.nix + ./services/monitoring/fluent-bit.nix ./services/monitoring/fusion-inventory.nix ./services/monitoring/gatus.nix ./services/monitoring/gitwatch.nix diff --git a/nixos/modules/services/monitoring/fluent-bit.nix b/nixos/modules/services/monitoring/fluent-bit.nix new file mode 100644 index 0000000000000..0957579d46182 --- /dev/null +++ b/nixos/modules/services/monitoring/fluent-bit.nix @@ -0,0 +1,115 @@ +{ + config, + lib, + pkgs, + ... +}: +let + cfg = config.services.fluent-bit; + + yamlFormat = pkgs.formats.yaml { }; + + configurationFile = + if (cfg.configurationFile == null) then + (yamlFormat.generate "fluent-bit.yaml" cfg.configuration) + else + cfg.configurationFile; +in +{ + options.services.fluent-bit = { + enable = lib.mkEnableOption "Fluent Bit"; + package = lib.mkPackageOption pkgs "fluent-bit" { }; + configurationFile = lib.mkOption { + type = lib.types.nullOr lib.types.path; + default = null; + description = '' + Fluent Bit configuration. See + + for supported values. + + {option}`configurationFile` takes precedence over {option}`configuration`. + + Note: Restricted evaluation blocks access to paths outside the Nix store. + This means detecting content changes for mutable paths (i.e. not input or content-addressed) can't be done. + As a result, `nixos-rebuild` won't reload/restart the systemd unit when mutable path contents change. + `systemctl restart fluent-bit.service` must be used instead. + ''; + example = "/etc/fluent-bit/fluent-bit.yaml"; + }; + configuration = lib.mkOption { + type = yamlFormat.type; + default = { }; + description = '' + See {option}`configurationFile`. + + {option}`configurationFile` takes precedence over {option}`configuration`. + ''; + example = { + service = { + grace = 30; + }; + pipeline = { + inputs = [ + { + name = "systemd"; + systemd_filter = "_SYSTEMD_UNIT=fluent-bit.service"; + } + ]; + outputs = [ + { + name = "file"; + path = "/var/log/fluent-bit"; + file = "fluent-bit.out"; + } + ]; + }; + }; + }; + user = lib.mkOption { + type = lib.types.str; + default = "root"; + description = '' + The user that runs Fluent Bit. + ''; + example = "fluent-bit"; + }; + # See https://docs.fluentbit.io/manual/administration/configuring-fluent-bit/yaml/service-section. + graceLimit = lib.mkOption { + type = lib.types.nullOr ( + lib.types.oneOf [ + lib.types.ints.positive + lib.types.str + ] + ); + default = null; + description = '' + The grace time limit. Sets the systemd unit's `TimeoutStopSec`. + + The `service.grace` option in the Fluent Bit configuration should be ≤ this option. + ''; + example = 30; + }; + }; + + config = lib.mkIf cfg.enable { + # See https://github.com/fluent/fluent-bit/blob/v3.2.2/init/systemd.in. + systemd.services.fluent-bit = { + description = "Fluent Bit"; + after = [ "network.target" ]; + requires = [ "network.target" ]; + wantedBy = [ "multi-user.target" ]; + serviceConfig = + { + User = cfg.user; + ExecStart = builtins.concatStringsSep " " [ + "${cfg.package}/bin/fluent-bit" + "--config ${configurationFile}" + ]; + Restart = "always"; + } + // (lib.optionalAttrs (cfg.graceLimit != null) { + TimeoutStopSec = cfg.graceLimit; + }); + }; + }; +} diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix index 7775d2f1fda78..cd8e89df8ad3f 100644 --- a/nixos/tests/all-tests.nix +++ b/nixos/tests/all-tests.nix @@ -351,6 +351,7 @@ in { flaresolverr = handleTest ./flaresolverr.nix {}; flood = handleTest ./flood.nix {}; floorp = handleTest ./firefox.nix { firefoxPackage = pkgs.floorp; }; + fluent-bit = handleTest ./fluent-bit.nix {}; fluentd = handleTest ./fluentd.nix {}; fluidd = handleTest ./fluidd.nix {}; fontconfig-default-fonts = handleTest ./fontconfig-default-fonts.nix {}; diff --git a/nixos/tests/fluent-bit.nix b/nixos/tests/fluent-bit.nix new file mode 100644 index 0000000000000..2fa6cd34c06fa --- /dev/null +++ b/nixos/tests/fluent-bit.nix @@ -0,0 +1,40 @@ +import ./make-test-python.nix ( + { lib, pkgs, ... }: + { + name = "fluent-bit"; + + nodes.machine = + { config, pkgs, ... }: + { + services.fluent-bit = { + enable = true; + configuration = { + pipeline = { + inputs = [ + { + name = "systemd"; + systemd_filter = "_SYSTEMD_UNIT=fluent-bit.service"; + } + ]; + outputs = [ + { + name = "file"; + path = "/var/log/fluent-bit"; + file = "fluent-bit.out"; + } + ]; + }; + }; + }; + + systemd.services.fluent-bit.serviceConfig.LogsDirectory = "fluent-bit"; + }; + + testScript = '' + start_all() + + machine.wait_for_unit("fluent-bit.service") + machine.wait_for_file("/var/log/fluent-bit/fluent-bit.out") + ''; + } +) diff --git a/pkgs/by-name/fl/fluent-bit/package.nix b/pkgs/by-name/fl/fluent-bit/package.nix index be8b5085c4ebf..9d840bebe75cb 100644 --- a/pkgs/by-name/fl/fluent-bit/package.nix +++ b/pkgs/by-name/fl/fluent-bit/package.nix @@ -1,15 +1,18 @@ { lib, - stdenv, - fetchFromGitHub, + bison, cmake, + darwin, + fetchFromGitHub, flex, - bison, - systemd, - postgresql, - openssl, libyaml, - darwin, + nix-update-script, + nixosTests, + openssl, + postgresql, + stdenv, + systemd, + versionCheckHook, }: stdenv.mkDerivation (finalAttrs: { @@ -19,13 +22,18 @@ stdenv.mkDerivation (finalAttrs: { src = fetchFromGitHub { owner = "fluent"; repo = "fluent-bit"; - rev = "v${finalAttrs.version}"; + rev = "refs/tags/v${finalAttrs.version}"; hash = "sha256-BnrULjcWVBAOHfxlmd1RTQ8gfwlfZcwrUyLU27/9Z3M="; }; - # optional only to avoid linux rebuild + # Optional only to avoid Linux rebuild. patches = lib.optionals stdenv.hostPlatform.isDarwin [ ./macos-11-sdk-compat.patch ]; + postPatch = '' + substituteInPlace src/CMakeLists.txt \ + --replace /lib/systemd $out/lib/systemd + ''; + nativeBuildInputs = [ cmake flex @@ -52,10 +60,9 @@ stdenv.mkDerivation (finalAttrs: { ] ++ lib.optionals stdenv.hostPlatform.isDarwin [ "-DCMAKE_OSX_DEPLOYMENT_TARGET=10.13" ]; env.NIX_CFLAGS_COMPILE = toString ( - # Used by the embedded luajit, but is not predefined on older mac SDKs. + # Used by the embedded luajit, but is not predefined on older Apple SDKs. lib.optionals stdenv.hostPlatform.isDarwin [ "-DTARGET_OS_IPHONE=0" ] - # Assumes GNU version of strerror_r, and the posix version has an - # incompatible return type. + # Assumes GNU version of strerror_r, and the POSIX version has an incompatible return type. ++ lib.optionals (!stdenv.hostPlatform.isGnu) [ "-Wno-int-conversion" ] ); @@ -64,20 +71,29 @@ stdenv.mkDerivation (finalAttrs: { "dev" ]; - postPatch = '' - substituteInPlace src/CMakeLists.txt \ - --replace /lib/systemd $out/lib/systemd - ''; + doInstallCheck = true; + + nativeInstallCheckInputs = [ versionCheckHook ]; + + versionCheckProgram = "${builtins.placeholder "out"}/bin/${finalAttrs.meta.mainProgram}"; + + versionCheckProgramArg = "--version"; + + passthru = { + tests = lib.optionalAttrs stdenv.isLinux { + inherit (nixosTests) fluent-bit; + }; + + updateScript = nix-update-script { }; + }; meta = { changelog = "https://github.com/fluent/fluent-bit/releases/tag/v${finalAttrs.version}"; - description = "Log forwarder and processor, part of Fluentd ecosystem"; + description = "Fast and lightweight logs and metrics processor for Linux, BSD, OSX and Windows"; homepage = "https://fluentbit.io"; license = lib.licenses.asl20; - maintainers = with lib.maintainers; [ - samrose - fpletz - ]; + mainProgram = "fluent-bit"; + maintainers = with lib.maintainers; [ samrose ]; platforms = lib.platforms.unix; }; })