Skip to content

Commit

Permalink
fluent-bit: add NixOS module
Browse files Browse the repository at this point in the history
  • Loading branch information
commiterate committed Dec 18, 2024
1 parent 5bc0af1 commit 1a806f0
Show file tree
Hide file tree
Showing 7 changed files with 190 additions and 16 deletions.
5 changes: 4 additions & 1 deletion ci/OWNERS
Original file line number Diff line number Diff line change
Expand Up @@ -143,10 +143,13 @@ nixos/modules/installer/tools/nix-fallback-paths.nix @NixOS/nix-team @raitobeza
/nixos/tests/amazon-ssm-agent.nix @arianvp
/nixos/modules/system/boot/grow-partition.nix @arianvp

# Monitoring
/nixos/modules/services/monitoring/fluent-bit.nix @samrose @fpletz
/nixos/tests/fluent-bit.nix @samrose @fpletz

# nixos-rebuild-ng
/pkgs/by-name/ni/nixos-rebuild-ng @thiagokokada


# Updaters
## update.nix
/maintainers/scripts/update.nix @jtojnar
Expand Down
2 changes: 2 additions & 0 deletions nixos/doc/manual/release-notes/rl-2505.section.md
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@

- [Amazon CloudWatch Agent](https://github.com/aws/amazon-cloudwatch-agent), the official telemetry collector for AWS CloudWatch and AWS X-Ray. Available as [services.amazon-cloudwatch-agent](#opt-services.amazon-cloudwatch-agent.enable).

- [Fluent Bit](https://github.com/fluent/fluent-bit), a fast Log, Metrics and Traces Processor and Forwarder. Available as [services.fluent-bit](#opt-services.fluent-bit.enable).

<!-- To avoid merge conflicts, consider adding your item at an arbitrary place in the list instead. -->

## Backward Incompatibilities {#sec-release-25.05-incompatibilities}
Expand Down
1 change: 1 addition & 0 deletions nixos/modules/module-list.nix
Original file line number Diff line number Diff line change
Expand Up @@ -893,6 +893,7 @@
./services/monitoring/das_watchdog.nix
./services/monitoring/datadog-agent.nix
./services/monitoring/do-agent.nix
./services/monitoring/fluent-bit.nix
./services/monitoring/fusion-inventory.nix
./services/monitoring/gatus.nix
./services/monitoring/glances.nix
Expand Down
107 changes: 107 additions & 0 deletions nixos/modules/services/monitoring/fluent-bit.nix
Original file line number Diff line number Diff line change
@@ -0,0 +1,107 @@
{
config,
lib,
pkgs,
...
}:
let
cfg = config.services.fluent-bit;

yamlFormat = pkgs.formats.yaml { };

configurationFile =
if (cfg.configurationFile == null) then
(yamlFormat.generate "fluent-bit.yaml" cfg.configuration)
else
cfg.configurationFile;
in
{
options.services.fluent-bit = {
enable = lib.mkEnableOption "Fluent Bit";
package = lib.mkPackageOption pkgs "fluent-bit" { };
configurationFile = lib.mkOption {
type = lib.types.nullOr lib.types.path;
default = null;
description = ''
Fluent Bit configuration. See
<https://docs.fluentbit.io/manual/administration/configuring-fluent-bit/yaml>
for supported values.
{option}`configurationFile` takes precedence over {option}`configuration`.
Note: Restricted evaluation blocks access to paths outside the Nix store.
This means detecting content changes for mutable paths (i.e. not input or content-addressed) can't be done.
As a result, `nixos-rebuild` won't reload/restart the systemd unit when mutable path contents change.
`systemctl restart fluent-bit.service` must be used instead.
'';
example = /etc/fluent-bit/fluent-bit.yaml;
};
configuration = lib.mkOption {
type = yamlFormat.type;
default = { };
description = ''
See {option}`configurationFile`.
{option}`configurationFile` takes precedence over {option}`configuration`.
'';
example = {
service = {
grace = 30;
};
pipeline = {
inputs = [
{
name = "systemd";
systemd_filter = "_SYSTEMD_UNIT=fluent-bit.service";
}
];
outputs = [
{
name = "file";
path = "/var/log/fluent-bit";
file = "fluent-bit.out";
}
];
};
};
};
user = lib.mkOption {
type = lib.types.str;
default = "root";
description = ''
The user that runs Fluent Bit.
'';
example = "fluent-bit";
};
# See https://docs.fluentbit.io/manual/administration/configuring-fluent-bit/yaml/service-section.
grace = lib.mkOption {
type = lib.types.ints.positive;
default = 5;
description = ''
The grace time in `seconds` as an integer value. The engine loop uses a grace timeout to define the wait time before exiting.
The `service.grace` option in the Fluent Bit configuration should be ≤ this option.
'';
example = 30;
};
};

config = lib.mkIf cfg.enable {
# See https://github.com/fluent/fluent-bit/blob/v3.2.2/init/systemd.in.
systemd.services.fluent-bit = {
description = "Fluent Bit";
after = [ "network.target" ];
requires = [ "network.target" ];
wantedBy = [ "multi-user.target" ];
serviceConfig = {
User = cfg.user;
ExecStart = builtins.concatStringsSep " " [
"${cfg.package}/bin/fluent-bit"
"--config ${configurationFile}"
];
TimeoutStopSec = cfg.grace;
Restart = "always";
};
};
};
}
1 change: 1 addition & 0 deletions nixos/tests/all-tests.nix
Original file line number Diff line number Diff line change
Expand Up @@ -350,6 +350,7 @@ in {
flaresolverr = handleTest ./flaresolverr.nix {};
flood = handleTest ./flood.nix {};
floorp = handleTest ./firefox.nix { firefoxPackage = pkgs.floorp; };
fluent-bit = handleTest ./fluent-bit.nix {};
fluentd = handleTest ./fluentd.nix {};
fluidd = handleTest ./fluidd.nix {};
fontconfig-default-fonts = handleTest ./fontconfig-default-fonts.nix {};
Expand Down
40 changes: 40 additions & 0 deletions nixos/tests/fluent-bit.nix
Original file line number Diff line number Diff line change
@@ -0,0 +1,40 @@
import ./make-test-python.nix (
{ lib, pkgs, ... }:
{
name = "fluent-bit";

nodes.machine =
{ config, pkgs, ... }:
{
services.fluent-bit = {
enable = true;
configuration = {
pipeline = {
inputs = [
{
name = "systemd";
systemd_filter = "_SYSTEMD_UNIT=fluent-bit.service";
}
];
outputs = [
{
name = "file";
path = "/var/log/fluent-bit";
file = "fluent-bit.out";
}
];
};
};
};

systemd.services.fluent-bit.serviceConfig.LogsDirectory = "fluent-bit";
};

testScript = ''
start_all()
machine.wait_for_unit("fluent-bit.service")
machine.wait_for_file("/var/log/fluent-bit/fluent-bit.out")
'';
}
)
50 changes: 35 additions & 15 deletions pkgs/by-name/fl/fluent-bit/package.nix
Original file line number Diff line number Diff line change
@@ -1,15 +1,18 @@
{
lib,
stdenv,
fetchFromGitHub,
bison,
cmake,
darwin,
fetchFromGitHub,
flex,
bison,
systemd,
postgresql,
openssl,
libyaml,
darwin,
nix-update-script,
nixosTests,
openssl,
postgresql,
stdenv,
systemd,
versionCheckHook,
}:

stdenv.mkDerivation (finalAttrs: {
Expand All @@ -23,9 +26,14 @@ stdenv.mkDerivation (finalAttrs: {
hash = "sha256-/YKXWYaDqkJ8fWgcYktXWJPcjDJTupgRTSQVp0cWruY=";
};

# optional only to avoid linux rebuild
# Optional only to avoid Linux rebuild.
patches = lib.optionals stdenv.hostPlatform.isDarwin [ ./macos-11-sdk-compat.patch ];

postPatch = ''
substituteInPlace src/CMakeLists.txt \
--replace /lib/systemd $out/lib/systemd
'';

nativeBuildInputs = [
cmake
flex
Expand All @@ -52,9 +60,9 @@ stdenv.mkDerivation (finalAttrs: {
] ++ lib.optionals stdenv.hostPlatform.isDarwin [ "-DCMAKE_OSX_DEPLOYMENT_TARGET=10.13" ];

env.NIX_CFLAGS_COMPILE = toString (
# Used by the embedded luajit, but is not predefined on older mac SDKs.
# Used by the embedded luajit, but is not predefined on older Apple SDKs.
lib.optionals stdenv.hostPlatform.isDarwin [ "-DTARGET_OS_IPHONE=0" ]
# Assumes GNU version of strerror_r, and the posix version has an
# Assumes GNU version of strerror_r, and the POSIX version has an
# incompatible return type.
++ lib.optionals (!stdenv.hostPlatform.isGnu) [ "-Wno-int-conversion" ]
);
Expand All @@ -64,16 +72,28 @@ stdenv.mkDerivation (finalAttrs: {
"dev"
];

postPatch = ''
substituteInPlace src/CMakeLists.txt \
--replace /lib/systemd $out/lib/systemd
'';
doInstallCheck = true;

nativeInstallCheckInputs = [ versionCheckHook ];

versionCheckProgram = "${builtins.placeholder "out"}/bin/${finalAttrs.meta.mainProgram}";

versionCheckProgramArg = "--version";

passthru = {
tests = lib.optionalAttrs stdenv.isLinux {
inherit (nixosTests) fluent-bit;
};

updateScript = nix-update-script { };
};

meta = {
changelog = "https://github.com/fluent/fluent-bit/releases/tag/v${finalAttrs.version}";
description = "Log forwarder and processor, part of Fluentd ecosystem";
description = "Fast and lightweight logs and metrics processor for Linux, BSD, OSX and Windows";
homepage = "https://fluentbit.io";
license = lib.licenses.asl20;
mainProgram = "fluent-bit";
maintainers = with lib.maintainers; [
samrose
fpletz
Expand Down

0 comments on commit 1a806f0

Please sign in to comment.