diff --git a/ci/OWNERS b/ci/OWNERS index 35edeaf6cf7755..995ea3248657be 100644 --- a/ci/OWNERS +++ b/ci/OWNERS @@ -143,10 +143,13 @@ nixos/modules/installer/tools/nix-fallback-paths.nix @NixOS/nix-team @raitobeza /nixos/tests/amazon-ssm-agent.nix @arianvp /nixos/modules/system/boot/grow-partition.nix @arianvp +# Monitoring +/nixos/modules/services/monitoring/fluent-bit.nix @samrose +/nixos/tests/fluent-bit.nix @samrose + # nixos-rebuild-ng /pkgs/by-name/ni/nixos-rebuild-ng @thiagokokada - # Updaters ## update.nix /maintainers/scripts/update.nix @jtojnar diff --git a/nixos/doc/manual/release-notes/rl-2505.section.md b/nixos/doc/manual/release-notes/rl-2505.section.md index a6ac39a37c791a..5638ce2e4f554e 100644 --- a/nixos/doc/manual/release-notes/rl-2505.section.md +++ b/nixos/doc/manual/release-notes/rl-2505.section.md @@ -56,6 +56,8 @@ - [git-worktree-switcher](https://github.com/mateusauler/git-worktree-switcher), switch between git worktrees with speed. Available as [programs.git-worktree-switcher](#opt-programs.git-worktree-switcher.enable) +- [Fluent Bit](https://github.com/fluent/fluent-bit), a fast Log, Metrics and Traces Processor and Forwarder. Available as [services.fluent-bit](#opt-services.fluent-bit.enable). + ## Backward Incompatibilities {#sec-release-25.05-incompatibilities} diff --git a/nixos/modules/module-list.nix b/nixos/modules/module-list.nix index 783f20546af446..29b39d7f94c6d4 100644 --- a/nixos/modules/module-list.nix +++ b/nixos/modules/module-list.nix @@ -904,6 +904,7 @@ ./services/monitoring/das_watchdog.nix ./services/monitoring/datadog-agent.nix ./services/monitoring/do-agent.nix + ./services/monitoring/fluent-bit.nix ./services/monitoring/fusion-inventory.nix ./services/monitoring/gatus.nix ./services/monitoring/gitwatch.nix diff --git a/nixos/modules/services/monitoring/fluent-bit.nix b/nixos/modules/services/monitoring/fluent-bit.nix new file mode 100644 index 00000000000000..0957579d46182d --- /dev/null +++ b/nixos/modules/services/monitoring/fluent-bit.nix @@ -0,0 +1,115 @@ +{ + config, + lib, + pkgs, + ... +}: +let + cfg = config.services.fluent-bit; + + yamlFormat = pkgs.formats.yaml { }; + + configurationFile = + if (cfg.configurationFile == null) then + (yamlFormat.generate "fluent-bit.yaml" cfg.configuration) + else + cfg.configurationFile; +in +{ + options.services.fluent-bit = { + enable = lib.mkEnableOption "Fluent Bit"; + package = lib.mkPackageOption pkgs "fluent-bit" { }; + configurationFile = lib.mkOption { + type = lib.types.nullOr lib.types.path; + default = null; + description = '' + Fluent Bit configuration. See + + for supported values. + + {option}`configurationFile` takes precedence over {option}`configuration`. + + Note: Restricted evaluation blocks access to paths outside the Nix store. + This means detecting content changes for mutable paths (i.e. not input or content-addressed) can't be done. + As a result, `nixos-rebuild` won't reload/restart the systemd unit when mutable path contents change. + `systemctl restart fluent-bit.service` must be used instead. + ''; + example = "/etc/fluent-bit/fluent-bit.yaml"; + }; + configuration = lib.mkOption { + type = yamlFormat.type; + default = { }; + description = '' + See {option}`configurationFile`. + + {option}`configurationFile` takes precedence over {option}`configuration`. + ''; + example = { + service = { + grace = 30; + }; + pipeline = { + inputs = [ + { + name = "systemd"; + systemd_filter = "_SYSTEMD_UNIT=fluent-bit.service"; + } + ]; + outputs = [ + { + name = "file"; + path = "/var/log/fluent-bit"; + file = "fluent-bit.out"; + } + ]; + }; + }; + }; + user = lib.mkOption { + type = lib.types.str; + default = "root"; + description = '' + The user that runs Fluent Bit. + ''; + example = "fluent-bit"; + }; + # See https://docs.fluentbit.io/manual/administration/configuring-fluent-bit/yaml/service-section. + graceLimit = lib.mkOption { + type = lib.types.nullOr ( + lib.types.oneOf [ + lib.types.ints.positive + lib.types.str + ] + ); + default = null; + description = '' + The grace time limit. Sets the systemd unit's `TimeoutStopSec`. + + The `service.grace` option in the Fluent Bit configuration should be ≤ this option. + ''; + example = 30; + }; + }; + + config = lib.mkIf cfg.enable { + # See https://github.com/fluent/fluent-bit/blob/v3.2.2/init/systemd.in. + systemd.services.fluent-bit = { + description = "Fluent Bit"; + after = [ "network.target" ]; + requires = [ "network.target" ]; + wantedBy = [ "multi-user.target" ]; + serviceConfig = + { + User = cfg.user; + ExecStart = builtins.concatStringsSep " " [ + "${cfg.package}/bin/fluent-bit" + "--config ${configurationFile}" + ]; + Restart = "always"; + } + // (lib.optionalAttrs (cfg.graceLimit != null) { + TimeoutStopSec = cfg.graceLimit; + }); + }; + }; +} diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix index 7775d2f1fda78d..cd8e89df8ad3f4 100644 --- a/nixos/tests/all-tests.nix +++ b/nixos/tests/all-tests.nix @@ -351,6 +351,7 @@ in { flaresolverr = handleTest ./flaresolverr.nix {}; flood = handleTest ./flood.nix {}; floorp = handleTest ./firefox.nix { firefoxPackage = pkgs.floorp; }; + fluent-bit = handleTest ./fluent-bit.nix {}; fluentd = handleTest ./fluentd.nix {}; fluidd = handleTest ./fluidd.nix {}; fontconfig-default-fonts = handleTest ./fontconfig-default-fonts.nix {}; diff --git a/nixos/tests/fluent-bit.nix b/nixos/tests/fluent-bit.nix new file mode 100644 index 00000000000000..2fa6cd34c06fab --- /dev/null +++ b/nixos/tests/fluent-bit.nix @@ -0,0 +1,40 @@ +import ./make-test-python.nix ( + { lib, pkgs, ... }: + { + name = "fluent-bit"; + + nodes.machine = + { config, pkgs, ... }: + { + services.fluent-bit = { + enable = true; + configuration = { + pipeline = { + inputs = [ + { + name = "systemd"; + systemd_filter = "_SYSTEMD_UNIT=fluent-bit.service"; + } + ]; + outputs = [ + { + name = "file"; + path = "/var/log/fluent-bit"; + file = "fluent-bit.out"; + } + ]; + }; + }; + }; + + systemd.services.fluent-bit.serviceConfig.LogsDirectory = "fluent-bit"; + }; + + testScript = '' + start_all() + + machine.wait_for_unit("fluent-bit.service") + machine.wait_for_file("/var/log/fluent-bit/fluent-bit.out") + ''; + } +) diff --git a/pkgs/by-name/fl/fluent-bit/macos-11-sdk-compat.patch b/pkgs/by-name/fl/fluent-bit/macos-11-sdk-compat.patch deleted file mode 100644 index 5063e028e6564f..00000000000000 --- a/pkgs/by-name/fl/fluent-bit/macos-11-sdk-compat.patch +++ /dev/null @@ -1,17 +0,0 @@ -diff --git i/src/flb_utils.c w/src/flb_utils.c -index 87637f1331d7..7a0036566438 100644 ---- i/src/flb_utils.c -+++ w/src/flb_utils.c -@@ -1424,11 +1424,11 @@ int flb_utils_get_machine_id(char **out_id, size_t *out_size) - return 0; - } - #elif defined (FLB_SYSTEM_MACOS) - bool bret; - CFStringRef serialNumber; -- io_service_t platformExpert = IOServiceGetMatchingService(kIOMainPortDefault, -+ io_service_t platformExpert = IOServiceGetMatchingService(kIOMasterPortDefault, - IOServiceMatching("IOPlatformExpertDevice")); - - if (platformExpert) { - CFTypeRef serialNumberAsCFString = - IORegistryEntryCreateCFProperty(platformExpert, diff --git a/pkgs/by-name/fl/fluent-bit/package.nix b/pkgs/by-name/fl/fluent-bit/package.nix index be8b5085c4ebf4..9535bf0a6ed9b5 100644 --- a/pkgs/by-name/fl/fluent-bit/package.nix +++ b/pkgs/by-name/fl/fluent-bit/package.nix @@ -1,83 +1,145 @@ { lib, - stdenv, - fetchFromGitHub, + bison, + c-ares, cmake, + curl, + fetchFromGitHub, flex, - bison, - systemd, - postgresql, - openssl, + jemalloc, + libbacktrace, + libbpf, + libnghttp2, libyaml, - darwin, + luajit, + nix-update-script, + nixosTests, + openssl, + pkg-config, + postgresql, + rdkafka, + stdenv, + systemd, + versionCheckHook, + zlib, + zstd, }: -stdenv.mkDerivation (finalAttrs: { +stdenv.mkDerivation rec { pname = "fluent-bit"; version = "3.2.2"; src = fetchFromGitHub { owner = "fluent"; repo = "fluent-bit"; - rev = "v${finalAttrs.version}"; + rev = "refs/tags/v${version}"; hash = "sha256-BnrULjcWVBAOHfxlmd1RTQ8gfwlfZcwrUyLU27/9Z3M="; }; - # optional only to avoid linux rebuild - patches = lib.optionals stdenv.hostPlatform.isDarwin [ ./macos-11-sdk-compat.patch ]; + # `src/CMakeLists.txt` installs fluent-bit's systemd unit files at the path in the `SYSTEMD_UNITDIR` CMake variable. + # + # The initial value of `SYSTEMD_UNITDIR` is set in `cmake/FindJournald` which uses pkg-config to find the systemd + # unit directory. `src/CMakeLists.txt` only sets `SYSTEMD_UNITDIR` to `/lib/systemd/system` if it's unset. + # + # Unfortunately, this resolves to systemd's Nix store path which is immutable. Consequently, CMake fails when trying + # to install fluent-bit's systemd unit files to the systemd Nix store path. + # + # We fix this by replacing `${SYSTEMD_UNITDIR}` instances in `src/CMakeLists.txt`. + postPatch = '' + substituteInPlace src/CMakeLists.txt \ + --replace-fail \''${SYSTEMD_UNITDIR} $out/lib/systemd + ''; + + # The source build documentation covers some dependencies and CMake options. + # + # - Linux: https://docs.fluentbit.io/manual/installation/sources/build-and-install + # - Darwin: https://docs.fluentbit.io/manual/installation/macos#compile-from-source + # + # Unfortunately, fluent-bit vends many dependencies (e.g. luajit) as source files and tries to compile them by + # default, with none of their dependencies and CMake options documented. + # + # Fortunately, there's the undocumented `FLB_PREFER_SYSTEM_LIBS` CMake option to link against system libraries for + # some dependencies. + # + # See https://github.com/fluent/fluent-bit/blob/v3.2.3/CMakeLists.txt#L211-L218. + # + # Like `FLB_PREFER_SYSTEM_LIBS`, several CMake options aren't documented. + # + # See https://github.com/fluent/fluent-bit/blob/v3.2.3/CMakeLists.txt#L111-L157. + # + # The CMake options may differ across target platforms. We'll stick to the minimum. + # + # See https://github.com/fluent/fluent-bit/tree/v3.2.3/packaging/distros. + + strictDeps = true; nativeBuildInputs = [ + bison cmake flex - bison + pkg-config ]; buildInputs = [ - openssl + c-ares + # Needed by rdkafka. + curl + jemalloc + libbacktrace + libnghttp2 libyaml + luajit + openssl postgresql + rdkafka + # Needed by rdkafka. + zlib + # Needed by rdkafka. + zstd ] - ++ lib.optionals stdenv.hostPlatform.isLinux [ systemd ] - ++ lib.optionals stdenv.hostPlatform.isDarwin [ - darwin.apple_sdk_11_0.frameworks.IOKit - darwin.apple_sdk_11_0.frameworks.Foundation + ++ lib.optionals stdenv.hostPlatform.isLinux [ + # libbpf doesn't build for Darwin yet. + libbpf + systemd ]; - cmakeFlags = [ - "-DFLB_RELEASE=ON" - "-DFLB_METRICS=ON" - "-DFLB_HTTP_SERVER=ON" - "-DFLB_OUT_PGSQL=ON" - ] ++ lib.optionals stdenv.hostPlatform.isDarwin [ "-DCMAKE_OSX_DEPLOYMENT_TARGET=10.13" ]; - - env.NIX_CFLAGS_COMPILE = toString ( - # Used by the embedded luajit, but is not predefined on older mac SDKs. - lib.optionals stdenv.hostPlatform.isDarwin [ "-DTARGET_OS_IPHONE=0" ] - # Assumes GNU version of strerror_r, and the posix version has an - # incompatible return type. - ++ lib.optionals (!stdenv.hostPlatform.isGnu) [ "-Wno-int-conversion" ] - ); + cmakeFlags = + [ + "-DFLB_RELEASE=Yes" + "-DFLB_PREFER_SYSTEM_LIBS=Yes" + ] + ++ lib.optionals stdenv.cc.isClang [ + # FLB_SECURITY causes bad linker options for Clang to be set. + "-DFLB_SECURITY=Off" + ]; outputs = [ "out" "dev" ]; - postPatch = '' - substituteInPlace src/CMakeLists.txt \ - --replace /lib/systemd $out/lib/systemd - ''; + doInstallCheck = true; + + nativeInstallCheckInputs = [ versionCheckHook ]; + + versionCheckProgram = "${builtins.placeholder "out"}/bin/${meta.mainProgram}"; + + versionCheckProgramArg = "--version"; + + passthru = { + tests = lib.optionalAttrs stdenv.isLinux { + inherit (nixosTests) fluent-bit; + }; + + updateScript = nix-update-script { }; + }; meta = { - changelog = "https://github.com/fluent/fluent-bit/releases/tag/v${finalAttrs.version}"; - description = "Log forwarder and processor, part of Fluentd ecosystem"; + description = "Fast and lightweight logs and metrics processor for Linux, BSD, OSX and Windows"; homepage = "https://fluentbit.io"; license = lib.licenses.asl20; - maintainers = with lib.maintainers; [ - samrose - fpletz - ]; - platforms = lib.platforms.unix; + mainProgram = "fluent-bit"; + maintainers = with lib.maintainers; [ samrose ]; }; -}) +}