fluent-bit: link against Nix dependencies, fix Darwin builds, and add NixOS module
This commit is contained in:
parent
310c01ef2f
commit
3366b27e59
@ -149,10 +149,13 @@ nixos/modules/installer/tools/nix-fallback-paths.nix @NixOS/nix-team @raitobeza
|
||||
/nixos/modules/services/monitoring/amazon-cloudwatch-agent.nix @philipmw
|
||||
/nixos/tests/amazon-cloudwatch-agent.nix @philipmw
|
||||
|
||||
# Monitoring
|
||||
/nixos/modules/services/monitoring/fluent-bit.nix @arianvp
|
||||
/nixos/tests/fluent-bit.nix @arianvp
|
||||
|
||||
# nixos-rebuild-ng
|
||||
/pkgs/by-name/ni/nixos-rebuild-ng @thiagokokada
|
||||
|
||||
|
||||
# Updaters
|
||||
## update.nix
|
||||
/maintainers/scripts/update.nix @jtojnar
|
||||
|
@ -116,6 +116,8 @@
|
||||
|
||||
- [Amazon CloudWatch Agent](https://github.com/aws/amazon-cloudwatch-agent), the official telemetry collector for AWS CloudWatch and AWS X-Ray. Available as [services.amazon-cloudwatch-agent](options.html#opt-services.amazon-cloudwatch-agent.enable).
|
||||
|
||||
- [Fluent Bit](https://github.com/fluent/fluent-bit), a fast Log, Metrics and Traces Processor and Forwarder. Available as [services.fluent-bit](#opt-services.fluent-bit.enable).
|
||||
|
||||
- [Bat](https://github.com/sharkdp/bat), a {manpage}`cat(1)` clone with wings. Available as [programs.bat](options.html#opt-programs.bat).
|
||||
|
||||
- [Autotier](https://github.com/45Drives/autotier), a passthrough FUSE filesystem. Available as [services.autotierfs](options.html#opt-services.autotierfs.enable).
|
||||
|
@ -934,6 +934,7 @@
|
||||
./services/monitoring/das_watchdog.nix
|
||||
./services/monitoring/datadog-agent.nix
|
||||
./services/monitoring/do-agent.nix
|
||||
./services/monitoring/fluent-bit.nix
|
||||
./services/monitoring/fusion-inventory.nix
|
||||
./services/monitoring/gatus.nix
|
||||
./services/monitoring/gitwatch.nix
|
||||
|
103
nixos/modules/services/monitoring/fluent-bit.nix
Normal file
103
nixos/modules/services/monitoring/fluent-bit.nix
Normal file
@ -0,0 +1,103 @@
|
||||
{
|
||||
config,
|
||||
lib,
|
||||
pkgs,
|
||||
utils,
|
||||
...
|
||||
}:
|
||||
let
|
||||
cfg = config.services.fluent-bit;
|
||||
|
||||
yamlFormat = pkgs.formats.yaml { };
|
||||
in
|
||||
{
|
||||
options.services.fluent-bit = {
|
||||
enable = lib.mkEnableOption "Fluent Bit";
|
||||
package = lib.mkPackageOption pkgs "fluent-bit" { };
|
||||
configurationFile = lib.mkOption {
|
||||
type = lib.types.path;
|
||||
default = yamlFormat.generate "fluent-bit.yaml" cfg.settings;
|
||||
defaultText = lib.literalExpression ''yamlFormat.generate "fluent-bit.yaml" cfg.settings'';
|
||||
description = ''
|
||||
Fluent Bit configuration. See
|
||||
<https://docs.fluentbit.io/manual/administration/configuring-fluent-bit/yaml>
|
||||
for supported values.
|
||||
|
||||
{option}`configurationFile` takes precedence over {option}`settings`.
|
||||
|
||||
Note: Restricted evaluation blocks access to paths outside the Nix store.
|
||||
This means detecting content changes for mutable paths (i.e. not input or content-addressed) can't be done.
|
||||
As a result, `nixos-rebuild` won't reload/restart the systemd unit when mutable path contents change.
|
||||
`systemctl restart fluent-bit.service` must be used instead.
|
||||
'';
|
||||
example = "/etc/fluent-bit/fluent-bit.yaml";
|
||||
};
|
||||
settings = lib.mkOption {
|
||||
type = yamlFormat.type;
|
||||
default = { };
|
||||
description = ''
|
||||
See {option}`configurationFile`.
|
||||
|
||||
{option}`configurationFile` takes precedence over {option}`settings`.
|
||||
'';
|
||||
example = {
|
||||
service = {
|
||||
grace = 30;
|
||||
};
|
||||
pipeline = {
|
||||
inputs = [
|
||||
{
|
||||
name = "systemd";
|
||||
systemd_filter = "_SYSTEMD_UNIT=fluent-bit.service";
|
||||
}
|
||||
];
|
||||
outputs = [
|
||||
{
|
||||
name = "file";
|
||||
path = "/var/log/fluent-bit";
|
||||
file = "fluent-bit.out";
|
||||
}
|
||||
];
|
||||
};
|
||||
};
|
||||
};
|
||||
# See https://docs.fluentbit.io/manual/administration/configuring-fluent-bit/yaml/service-section.
|
||||
graceLimit = lib.mkOption {
|
||||
type = lib.types.nullOr (
|
||||
lib.types.oneOf [
|
||||
lib.types.ints.positive
|
||||
lib.types.str
|
||||
]
|
||||
);
|
||||
default = null;
|
||||
description = ''
|
||||
The grace time limit. Sets the systemd unit's `TimeoutStopSec`.
|
||||
|
||||
The `service.grace` option in the Fluent Bit configuration should be ≤ this option.
|
||||
'';
|
||||
example = 30;
|
||||
};
|
||||
};
|
||||
|
||||
config = lib.mkIf cfg.enable {
|
||||
# See https://github.com/fluent/fluent-bit/blob/v3.2.6/init/systemd.in.
|
||||
systemd.services.fluent-bit = {
|
||||
description = "Fluent Bit";
|
||||
after = [ "network.target" ];
|
||||
requires = [ "network.target" ];
|
||||
wantedBy = [ "multi-user.target" ];
|
||||
serviceConfig = {
|
||||
DynamicUser = true;
|
||||
# See https://nixos.org/manual/nixos/stable#sec-logging.
|
||||
SupplementaryGroups = "systemd-journal";
|
||||
ExecStart = utils.escapeSystemdExecArgs [
|
||||
(lib.getExe cfg.package)
|
||||
"--config"
|
||||
cfg.configurationFile
|
||||
];
|
||||
Restart = "always";
|
||||
TimeoutStopSec = lib.mkIf (cfg.graceLimit != null) cfg.graceLimit;
|
||||
};
|
||||
};
|
||||
};
|
||||
}
|
@ -433,6 +433,7 @@ in {
|
||||
imports = [ ./firefox.nix ] ;
|
||||
_module.args.firefoxPackage = pkgs.floorp;
|
||||
};
|
||||
fluent-bit = handleTest ./fluent-bit.nix {};
|
||||
fluentd = handleTest ./fluentd.nix {};
|
||||
fluidd = handleTest ./fluidd.nix {};
|
||||
fontconfig-default-fonts = handleTest ./fontconfig-default-fonts.nix {};
|
||||
|
40
nixos/tests/fluent-bit.nix
Normal file
40
nixos/tests/fluent-bit.nix
Normal file
@ -0,0 +1,40 @@
|
||||
import ./make-test-python.nix (
|
||||
{ lib, pkgs, ... }:
|
||||
{
|
||||
name = "fluent-bit";
|
||||
|
||||
nodes.machine =
|
||||
{ config, pkgs, ... }:
|
||||
{
|
||||
services.fluent-bit = {
|
||||
enable = true;
|
||||
settings = {
|
||||
pipeline = {
|
||||
inputs = [
|
||||
{
|
||||
name = "systemd";
|
||||
systemd_filter = "_SYSTEMD_UNIT=fluent-bit.service";
|
||||
}
|
||||
];
|
||||
outputs = [
|
||||
{
|
||||
name = "file";
|
||||
path = "/var/log/fluent-bit";
|
||||
file = "fluent-bit.out";
|
||||
}
|
||||
];
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
systemd.services.fluent-bit.serviceConfig.LogsDirectory = "fluent-bit";
|
||||
};
|
||||
|
||||
testScript = ''
|
||||
start_all()
|
||||
|
||||
machine.wait_for_unit("fluent-bit.service")
|
||||
machine.wait_for_file("/var/log/fluent-bit/fluent-bit.out")
|
||||
'';
|
||||
}
|
||||
)
|
@ -1,17 +0,0 @@
|
||||
diff --git i/src/flb_utils.c w/src/flb_utils.c
|
||||
index 87637f1331d7..7a0036566438 100644
|
||||
--- i/src/flb_utils.c
|
||||
+++ w/src/flb_utils.c
|
||||
@@ -1424,11 +1424,11 @@ int flb_utils_get_machine_id(char **out_id, size_t *out_size)
|
||||
return 0;
|
||||
}
|
||||
#elif defined (FLB_SYSTEM_MACOS)
|
||||
bool bret;
|
||||
CFStringRef serialNumber;
|
||||
- io_service_t platformExpert = IOServiceGetMatchingService(kIOMainPortDefault,
|
||||
+ io_service_t platformExpert = IOServiceGetMatchingService(kIOMasterPortDefault,
|
||||
IOServiceMatching("IOPlatformExpertDevice"));
|
||||
|
||||
if (platformExpert) {
|
||||
CFTypeRef serialNumberAsCFString =
|
||||
IORegistryEntryCreateCFProperty(platformExpert,
|
@ -1,15 +1,29 @@
|
||||
{
|
||||
lib,
|
||||
stdenv,
|
||||
fetchFromGitHub,
|
||||
cmake,
|
||||
flex,
|
||||
arrow-glib,
|
||||
bison,
|
||||
systemd,
|
||||
c-ares,
|
||||
cmake,
|
||||
curl,
|
||||
fetchFromGitHub,
|
||||
flex,
|
||||
jemalloc,
|
||||
libbacktrace,
|
||||
libbpf,
|
||||
libnghttp2,
|
||||
libpq,
|
||||
openssl,
|
||||
libyaml,
|
||||
darwin,
|
||||
luajit,
|
||||
nix-update-script,
|
||||
nixosTests,
|
||||
openssl,
|
||||
pkg-config,
|
||||
rdkafka,
|
||||
systemd,
|
||||
versionCheckHook,
|
||||
zlib,
|
||||
zstd,
|
||||
}:
|
||||
|
||||
stdenv.mkDerivation (finalAttrs: {
|
||||
@ -19,63 +33,113 @@ stdenv.mkDerivation (finalAttrs: {
|
||||
src = fetchFromGitHub {
|
||||
owner = "fluent";
|
||||
repo = "fluent-bit";
|
||||
rev = "v${finalAttrs.version}";
|
||||
tag = "v${finalAttrs.version}";
|
||||
hash = "sha256-E+y8lZ5fgJORFkig6aSVMYGk0US1b4xwjO9qnGu4R/Y=";
|
||||
};
|
||||
|
||||
# optional only to avoid linux rebuild
|
||||
patches = lib.optionals stdenv.hostPlatform.isDarwin [ ./macos-11-sdk-compat.patch ];
|
||||
# The source build documentation covers some dependencies and CMake options.
|
||||
#
|
||||
# - Linux: https://docs.fluentbit.io/manual/installation/sources/build-and-install
|
||||
# - Darwin: https://docs.fluentbit.io/manual/installation/macos#compile-from-source
|
||||
#
|
||||
# Unfortunately, fluent-bit vends many dependencies (e.g. luajit) as source files and tries to compile them by
|
||||
# default, with none of their dependencies and CMake options documented.
|
||||
#
|
||||
# Fortunately, there's the undocumented `FLB_PREFER_SYSTEM_LIBS` CMake option to link against system libraries for
|
||||
# some dependencies.
|
||||
#
|
||||
# See https://github.com/fluent/fluent-bit/blob/v3.2.6/CMakeLists.txt#L211-L218.
|
||||
#
|
||||
# Like `FLB_PREFER_SYSTEM_LIBS`, several CMake options aren't documented.
|
||||
#
|
||||
# See https://github.com/fluent/fluent-bit/blob/v3.2.6/CMakeLists.txt#L111-L157.
|
||||
#
|
||||
# The CMake options may differ across target platforms. We'll stick to the minimum.
|
||||
#
|
||||
# See https://github.com/fluent/fluent-bit/tree/v3.2.6/packaging/distros.
|
||||
|
||||
strictDeps = true;
|
||||
|
||||
nativeBuildInputs = [
|
||||
bison
|
||||
cmake
|
||||
flex
|
||||
bison
|
||||
pkg-config
|
||||
];
|
||||
|
||||
buildInputs =
|
||||
[
|
||||
openssl
|
||||
libyaml
|
||||
arrow-glib
|
||||
c-ares
|
||||
# Needed by rdkafka.
|
||||
curl
|
||||
jemalloc
|
||||
libbacktrace
|
||||
libnghttp2
|
||||
libpq
|
||||
libyaml
|
||||
luajit
|
||||
openssl
|
||||
rdkafka
|
||||
# Needed by rdkafka.
|
||||
zlib
|
||||
# Needed by rdkafka.
|
||||
zstd
|
||||
]
|
||||
++ lib.optionals stdenv.hostPlatform.isLinux [ systemd ]
|
||||
++ lib.optionals stdenv.hostPlatform.isDarwin [
|
||||
darwin.apple_sdk_11_0.frameworks.IOKit
|
||||
darwin.apple_sdk_11_0.frameworks.Foundation
|
||||
++ lib.optionals stdenv.hostPlatform.isLinux [
|
||||
# libbpf doesn't build for Darwin yet.
|
||||
libbpf
|
||||
systemd
|
||||
];
|
||||
|
||||
cmakeFlags = [
|
||||
"-DFLB_RELEASE=ON"
|
||||
"-DFLB_METRICS=ON"
|
||||
"-DFLB_HTTP_SERVER=ON"
|
||||
"-DFLB_OUT_PGSQL=ON"
|
||||
];
|
||||
cmakeFlags =
|
||||
[
|
||||
(lib.cmakeBool "FLB_RELEASE" true)
|
||||
(lib.cmakeBool "FLB_PREFER_SYSTEM_LIBS" true)
|
||||
]
|
||||
++ lib.optionals stdenv.cc.isClang [
|
||||
# `FLB_SECURITY` causes bad linker options for Clang to be set.
|
||||
(lib.cmakeBool "FLB_SECURITY" false)
|
||||
];
|
||||
|
||||
env.NIX_CFLAGS_COMPILE = toString (
|
||||
# Assumes GNU version of strerror_r, and the posix version has an
|
||||
# incompatible return type.
|
||||
lib.optionals (!stdenv.hostPlatform.isGnu) [ "-Wno-int-conversion" ]
|
||||
);
|
||||
# `src/CMakeLists.txt` installs fluent-bit's systemd unit files at the path in the `SYSTEMD_UNITDIR` CMake variable.
|
||||
#
|
||||
# The initial value of `SYSTEMD_UNITDIR` is set in `cmake/FindJournald` which uses pkg-config to find the systemd
|
||||
# unit directory in the systemd package's `systemdsystemunitdir` pkg-config variable. `src/CMakeLists.txt` only
|
||||
# sets `SYSTEMD_UNITDIR` to `/lib/systemd/system` if it's unset.
|
||||
#
|
||||
# By default, this resolves to systemd's Nix store path which is immutable. Consequently, CMake fails when trying
|
||||
# to install fluent-bit's systemd unit files to the systemd Nix store path.
|
||||
#
|
||||
# We fix this by setting the systemd package's `systemdsystemunitdir` pkg-config variable.
|
||||
#
|
||||
# https://man.openbsd.org/pkg-config.1#PKG_CONFIG_$PACKAGE_$VARIABLE
|
||||
PKG_CONFIG_SYSTEMD_SYSTEMDSYSTEMUNITDIR = "${builtins.placeholder "out"}/lib/systemd/system";
|
||||
|
||||
outputs = [
|
||||
"out"
|
||||
"dev"
|
||||
];
|
||||
|
||||
postPatch = ''
|
||||
substituteInPlace src/CMakeLists.txt \
|
||||
--replace /lib/systemd $out/lib/systemd
|
||||
'';
|
||||
doInstallCheck = true;
|
||||
|
||||
nativeInstallCheckInputs = [ versionCheckHook ];
|
||||
|
||||
versionCheckProgramArg = "--version";
|
||||
|
||||
passthru = {
|
||||
tests = lib.optionalAttrs stdenv.isLinux {
|
||||
inherit (nixosTests) fluent-bit;
|
||||
};
|
||||
|
||||
updateScript = nix-update-script { };
|
||||
};
|
||||
|
||||
meta = {
|
||||
changelog = "https://github.com/fluent/fluent-bit/releases/tag/v${finalAttrs.version}";
|
||||
description = "Log forwarder and processor, part of Fluentd ecosystem";
|
||||
description = "Fast and lightweight logs and metrics processor for Linux, BSD, OSX and Windows";
|
||||
homepage = "https://fluentbit.io";
|
||||
license = lib.licenses.asl20;
|
||||
maintainers = with lib.maintainers; [
|
||||
samrose
|
||||
fpletz
|
||||
];
|
||||
platforms = lib.platforms.unix;
|
||||
mainProgram = "fluent-bit";
|
||||
maintainers = with lib.maintainers; [ arianvp ];
|
||||
};
|
||||
})
|
||||
|
Loading…
x
Reference in New Issue
Block a user