fluent-bit: link against Nix dependencies, fix Darwin builds, and add NixOS module

ci/OWNERS

@@ -149,10 +149,13 @@ nixos/modules/installer/tools/nix-fallback-paths.nix  @NixOS/nix-team @raitobeza
 /nixos/modules/services/monitoring/amazon-cloudwatch-agent.nix @philipmw
 /nixos/tests/amazon-cloudwatch-agent.nix                       @philipmw
 
+# Monitoring
+/nixos/modules/services/monitoring/fluent-bit.nix @arianvp
+/nixos/tests/fluent-bit.nix                       @arianvp
+
 # nixos-rebuild-ng
 /pkgs/by-name/ni/nixos-rebuild-ng                 @thiagokokada
 
-
 # Updaters
 ## update.nix
 /maintainers/scripts/update.nix   @jtojnar

nixos/doc/manual/release-notes/rl-2505.section.md

@@ -116,6 +116,8 @@
 
 - [Amazon CloudWatch Agent](https://github.com/aws/amazon-cloudwatch-agent), the official telemetry collector for AWS CloudWatch and AWS X-Ray. Available as [services.amazon-cloudwatch-agent](options.html#opt-services.amazon-cloudwatch-agent.enable).
 
+- [Fluent Bit](https://github.com/fluent/fluent-bit), a fast Log, Metrics and Traces Processor and Forwarder. Available as [services.fluent-bit](#opt-services.fluent-bit.enable).
+
 - [Bat](https://github.com/sharkdp/bat), a {manpage}`cat(1)` clone with wings. Available as [programs.bat](options.html#opt-programs.bat).
 
 - [Autotier](https://github.com/45Drives/autotier), a passthrough FUSE filesystem. Available as [services.autotierfs](options.html#opt-services.autotierfs.enable).

nixos/modules/module-list.nix

@@ -934,6 +934,7 @@
   ./services/monitoring/das_watchdog.nix
   ./services/monitoring/datadog-agent.nix
   ./services/monitoring/do-agent.nix
+  ./services/monitoring/fluent-bit.nix
   ./services/monitoring/fusion-inventory.nix
   ./services/monitoring/gatus.nix
   ./services/monitoring/gitwatch.nix

nixos/modules/services/monitoring/fluent-bit.nix

@@ -0,0 +1,103 @@
+{
+  config,
+  lib,
+  pkgs,
+  utils,
+  ...
+}:
+let
+  cfg = config.services.fluent-bit;
+
+  yamlFormat = pkgs.formats.yaml { };
+in
+{
+  options.services.fluent-bit = {
+    enable = lib.mkEnableOption "Fluent Bit";
+    package = lib.mkPackageOption pkgs "fluent-bit" { };
+    configurationFile = lib.mkOption {
+      type = lib.types.path;
+      default = yamlFormat.generate "fluent-bit.yaml" cfg.settings;
+      defaultText = lib.literalExpression ''yamlFormat.generate "fluent-bit.yaml" cfg.settings'';
+      description = ''
+        Fluent Bit configuration. See
+        <https://docs.fluentbit.io/manual/administration/configuring-fluent-bit/yaml>
+        for supported values.
+
+        {option}`configurationFile` takes precedence over {option}`settings`.
+
+        Note: Restricted evaluation blocks access to paths outside the Nix store.
+        This means detecting content changes for mutable paths (i.e. not input or content-addressed) can't be done.
+        As a result, `nixos-rebuild` won't reload/restart the systemd unit when mutable path contents change.
+        `systemctl restart fluent-bit.service` must be used instead.
+      '';
+      example = "/etc/fluent-bit/fluent-bit.yaml";
+    };
+    settings = lib.mkOption {
+      type = yamlFormat.type;
+      default = { };
+      description = ''
+        See {option}`configurationFile`.
+
+        {option}`configurationFile` takes precedence over {option}`settings`.
+      '';
+      example = {
+        service = {
+          grace = 30;
+        };
+        pipeline = {
+          inputs = [
+            {
+              name = "systemd";
+              systemd_filter = "_SYSTEMD_UNIT=fluent-bit.service";
+            }
+          ];
+          outputs = [
+            {
+              name = "file";
+              path = "/var/log/fluent-bit";
+              file = "fluent-bit.out";
+            }
+          ];
+        };
+      };
+    };
+    # See https://docs.fluentbit.io/manual/administration/configuring-fluent-bit/yaml/service-section.
+    graceLimit = lib.mkOption {
+      type = lib.types.nullOr (
+        lib.types.oneOf [
+          lib.types.ints.positive
+          lib.types.str
+        ]
+      );
+      default = null;
+      description = ''
+        The grace time limit. Sets the systemd unit's `TimeoutStopSec`.
+
+        The `service.grace` option in the Fluent Bit configuration should be ≤ this option.
+      '';
+      example = 30;
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+    # See https://github.com/fluent/fluent-bit/blob/v3.2.6/init/systemd.in.
+    systemd.services.fluent-bit = {
+      description = "Fluent Bit";
+      after = [ "network.target" ];
+      requires = [ "network.target" ];
+      wantedBy = [ "multi-user.target" ];
+      serviceConfig = {
+        DynamicUser = true;
+        # See https://nixos.org/manual/nixos/stable#sec-logging.
+        SupplementaryGroups = "systemd-journal";
+        ExecStart = utils.escapeSystemdExecArgs [
+          (lib.getExe cfg.package)
+          "--config"
+          cfg.configurationFile
+        ];
+        Restart = "always";
+        TimeoutStopSec = lib.mkIf (cfg.graceLimit != null) cfg.graceLimit;
+      };
+    };
+  };
+}

nixos/tests/all-tests.nix

@@ -433,6 +433,7 @@ in {
     imports = [ ./firefox.nix ] ;
     _module.args.firefoxPackage = pkgs.floorp;
   };
+  fluent-bit = handleTest ./fluent-bit.nix {};
   fluentd = handleTest ./fluentd.nix {};
   fluidd = handleTest ./fluidd.nix {};
   fontconfig-default-fonts = handleTest ./fontconfig-default-fonts.nix {};

nixos/tests/fluent-bit.nix

@@ -0,0 +1,40 @@
+import ./make-test-python.nix (
+  { lib, pkgs, ... }:
+  {
+    name = "fluent-bit";
+
+    nodes.machine =
+      { config, pkgs, ... }:
+      {
+        services.fluent-bit = {
+          enable = true;
+          settings = {
+            pipeline = {
+              inputs = [
+                {
+                  name = "systemd";
+                  systemd_filter = "_SYSTEMD_UNIT=fluent-bit.service";
+                }
+              ];
+              outputs = [
+                {
+                  name = "file";
+                  path = "/var/log/fluent-bit";
+                  file = "fluent-bit.out";
+                }
+              ];
+            };
+          };
+        };
+
+        systemd.services.fluent-bit.serviceConfig.LogsDirectory = "fluent-bit";
+      };
+
+    testScript = ''
+      start_all()
+
+      machine.wait_for_unit("fluent-bit.service")
+      machine.wait_for_file("/var/log/fluent-bit/fluent-bit.out")
+    '';
+  }
+)

pkgs/by-name/fl/fluent-bit/macos-11-sdk-compat.patch

@@ -1,17 +0,0 @@
-diff --git i/src/flb_utils.c w/src/flb_utils.c
-index 87637f1331d7..7a0036566438 100644
---- i/src/flb_utils.c
-+++ w/src/flb_utils.c
-@@ -1424,11 +1424,11 @@ int flb_utils_get_machine_id(char **out_id, size_t *out_size)
-         return 0;
-     }
- #elif defined (FLB_SYSTEM_MACOS)
-     bool bret;
-     CFStringRef serialNumber;
--    io_service_t platformExpert = IOServiceGetMatchingService(kIOMainPortDefault,
-+    io_service_t platformExpert = IOServiceGetMatchingService(kIOMasterPortDefault,
-         IOServiceMatching("IOPlatformExpertDevice"));
-
-     if (platformExpert) {
-         CFTypeRef serialNumberAsCFString =
-             IORegistryEntryCreateCFProperty(platformExpert,

pkgs/by-name/fl/fluent-bit/package.nix

@@ -1,15 +1,29 @@
 {
   lib,
   stdenv,
-  fetchFromGitHub,
+  arrow-glib,
-  cmake,
-  flex,
   bison,
-  systemd,
+  c-ares,
+  cmake,
+  curl,
+  fetchFromGitHub,
+  flex,
+  jemalloc,
+  libbacktrace,
+  libbpf,
+  libnghttp2,
   libpq,
-  openssl,
   libyaml,
-  darwin,
+  luajit,
+  nix-update-script,
+  nixosTests,
+  openssl,
+  pkg-config,
+  rdkafka,
+  systemd,
+  versionCheckHook,
+  zlib,
+  zstd,
 }:
 
 stdenv.mkDerivation (finalAttrs: {
@@ -19,63 +33,113 @@ stdenv.mkDerivation (finalAttrs: {
   src = fetchFromGitHub {
     owner = "fluent";
     repo = "fluent-bit";
-    rev = "v${finalAttrs.version}";
+    tag = "v${finalAttrs.version}";
     hash = "sha256-E+y8lZ5fgJORFkig6aSVMYGk0US1b4xwjO9qnGu4R/Y=";
   };
 
-  # optional only to avoid linux rebuild
+  # The source build documentation covers some dependencies and CMake options.
-  patches = lib.optionals stdenv.hostPlatform.isDarwin [ ./macos-11-sdk-compat.patch ];
+  #
+  # - Linux: https://docs.fluentbit.io/manual/installation/sources/build-and-install
+  # - Darwin: https://docs.fluentbit.io/manual/installation/macos#compile-from-source
+  #
+  # Unfortunately, fluent-bit vends many dependencies (e.g. luajit) as source files and tries to compile them by
+  # default, with none of their dependencies and CMake options documented.
+  #
+  # Fortunately, there's the undocumented `FLB_PREFER_SYSTEM_LIBS` CMake option to link against system libraries for
+  # some dependencies.
+  #
+  # See https://github.com/fluent/fluent-bit/blob/v3.2.6/CMakeLists.txt#L211-L218.
+  #
+  # Like `FLB_PREFER_SYSTEM_LIBS`, several CMake options aren't documented.
+  #
+  # See https://github.com/fluent/fluent-bit/blob/v3.2.6/CMakeLists.txt#L111-L157.
+  #
+  # The CMake options may differ across target platforms. We'll stick to the minimum.
+  #
+  # See https://github.com/fluent/fluent-bit/tree/v3.2.6/packaging/distros.
+
+  strictDeps = true;
 
   nativeBuildInputs = [
+    bison
     cmake
     flex
-    bison
+    pkg-config
   ];
 
   buildInputs =
     [
-      openssl
+      arrow-glib
-      libyaml
+      c-ares
+      # Needed by rdkafka.
+      curl
+      jemalloc
+      libbacktrace
+      libnghttp2
       libpq
+      libyaml
+      luajit
+      openssl
+      rdkafka
+      # Needed by rdkafka.
+      zlib
+      # Needed by rdkafka.
+      zstd
     ]
-    ++ lib.optionals stdenv.hostPlatform.isLinux [ systemd ]
+    ++ lib.optionals stdenv.hostPlatform.isLinux [
-    ++ lib.optionals stdenv.hostPlatform.isDarwin [
+      # libbpf doesn't build for Darwin yet.
-      darwin.apple_sdk_11_0.frameworks.IOKit
+      libbpf
-      darwin.apple_sdk_11_0.frameworks.Foundation
+      systemd
     ];
 
-  cmakeFlags = [
+  cmakeFlags =
-    "-DFLB_RELEASE=ON"
+    [
-    "-DFLB_METRICS=ON"
+      (lib.cmakeBool "FLB_RELEASE" true)
-    "-DFLB_HTTP_SERVER=ON"
+      (lib.cmakeBool "FLB_PREFER_SYSTEM_LIBS" true)
-    "-DFLB_OUT_PGSQL=ON"
+    ]
-  ];
+    ++ lib.optionals stdenv.cc.isClang [
+      # `FLB_SECURITY` causes bad linker options for Clang to be set.
+      (lib.cmakeBool "FLB_SECURITY" false)
+    ];
 
-  env.NIX_CFLAGS_COMPILE = toString (
+  # `src/CMakeLists.txt` installs fluent-bit's systemd unit files at the path in the `SYSTEMD_UNITDIR` CMake variable.
-    # Assumes GNU version of strerror_r, and the posix version has an
+  #
-    # incompatible return type.
+  # The initial value of `SYSTEMD_UNITDIR` is set in `cmake/FindJournald` which uses pkg-config to find the systemd
-    lib.optionals (!stdenv.hostPlatform.isGnu) [ "-Wno-int-conversion" ]
+  # unit directory in the systemd package's `systemdsystemunitdir` pkg-config variable. `src/CMakeLists.txt` only
-  );
+  # sets `SYSTEMD_UNITDIR` to `/lib/systemd/system` if it's unset.
+  #
+  # By default, this resolves to systemd's Nix store path which is immutable. Consequently, CMake fails when trying
+  # to install fluent-bit's systemd unit files to the systemd Nix store path.
+  #
+  # We fix this by setting the systemd package's `systemdsystemunitdir` pkg-config variable.
+  #
+  # https://man.openbsd.org/pkg-config.1#PKG_CONFIG_$PACKAGE_$VARIABLE
+  PKG_CONFIG_SYSTEMD_SYSTEMDSYSTEMUNITDIR = "${builtins.placeholder "out"}/lib/systemd/system";
 
   outputs = [
     "out"
     "dev"
   ];
 
-  postPatch = ''
+  doInstallCheck = true;
-    substituteInPlace src/CMakeLists.txt \
+
-      --replace /lib/systemd $out/lib/systemd
+  nativeInstallCheckInputs = [ versionCheckHook ];
-  '';
+
+  versionCheckProgramArg = "--version";
+
+  passthru = {
+    tests = lib.optionalAttrs stdenv.isLinux {
+      inherit (nixosTests) fluent-bit;
+    };
+
+    updateScript = nix-update-script { };
+  };
 
   meta = {
-    changelog = "https://github.com/fluent/fluent-bit/releases/tag/v${finalAttrs.version}";
+    description = "Fast and lightweight logs and metrics processor for Linux, BSD, OSX and Windows";
-    description = "Log forwarder and processor, part of Fluentd ecosystem";
     homepage = "https://fluentbit.io";
     license = lib.licenses.asl20;
-    maintainers = with lib.maintainers; [
+    mainProgram = "fluent-bit";
-      samrose
+    maintainers = with lib.maintainers; [ arianvp ];
-      fpletz
-    ];
-    platforms = lib.platforms.unix;
   };
 })