vlagent: init at 1.25.0 (#424934)

nixos/modules/module-list.nix

@@ -1036,6 +1036,7 @@
   ./services/monitoring/ups.nix
   ./services/monitoring/uptime-kuma.nix
   ./services/monitoring/uptime.nix
+  ./services/monitoring/vlagent.nix
   ./services/monitoring/vmagent.nix
   ./services/monitoring/vmalert.nix
   ./services/monitoring/vnstat.nix

nixos/modules/services/databases/victorialogs.nix

@@ -2,6 +2,7 @@
   config,
   pkgs,
   lib,
+  utils,
   ...
 }:
 let
@@ -24,7 +25,12 @@ let
     "-storageDataPath=/var/lib/${cfg.stateDir}"
     "-httpListenAddr=${cfg.listenAddress}"
   ]
-  ++ cfg.extraOptions;
+  ++ lib.optionals (cfg.basicAuthUsername != null) [
+    "-httpAuth.username=${cfg.basicAuthUsername}"
+  ]
+  ++ lib.optionals (cfg.basicAuthPasswordFile != null) [
+    "-httpAuth.password=file://%d/basic_auth_password"
+  ];
 in
 {
   options.services.victorialogs = {
@@ -45,13 +51,26 @@ in
         This directory will be created automatically using systemd's StateDirectory mechanism.
       '';
     };
+    basicAuthUsername = lib.mkOption {
+      default = null;
+      type = lib.types.nullOr lib.types.str;
+      description = ''
+        Basic Auth username used to protect VictoriaLogs instance by authorization
+      '';
+    };
+
+    basicAuthPasswordFile = lib.mkOption {
+      default = null;
+      type = lib.types.nullOr lib.types.str;
+      description = ''
+        File that contains the Basic Auth password used to protect VictoriaLogs instance by authorization
+      '';
+    };
     extraOptions = mkOption {
       type = types.listOf types.str;
       default = [ ];
       example = literalExpression ''
         [
-          "-httpAuth.username=username"
-          "-httpAuth.password=file:///abs/path/to/file"
           "-loggerLevel=WARN"
         ]
       '';
@@ -62,6 +81,16 @@ in
     };
   };
   config = mkIf cfg.enable {
+
+    assertions = [
+      {
+        assertion =
+          (cfg.basicAuthUsername == null && cfg.basicAuthPasswordFile == null)
+          || (cfg.basicAuthUsername != null && cfg.basicAuthPasswordFile != null);
+        message = "Both basicAuthUsername and basicAuthPasswordFile must be set together to enable basicAuth functionality, or neither should be set.";
+      }
+    ];
+
     systemd.services.victorialogs = {
       description = "VictoriaLogs logs database";
       wantedBy = [ "multi-user.target" ];
@@ -69,8 +98,14 @@ in
       startLimitBurst = 5;
 
       serviceConfig = {
-        ExecStart = escapeShellArgs startCLIList;
+        ExecStart = lib.concatStringsSep " " [
+          (escapeShellArgs startCLIList)
+          (utils.escapeSystemdExecArgs cfg.extraOptions)
+        ];
         DynamicUser = true;
+        LoadCredential = lib.optional (
+          cfg.basicAuthPasswordFile != null
+        ) "basic_auth_password:${cfg.basicAuthPasswordFile}";
         RestartSec = 1;
         Restart = "on-failure";
         RuntimeDirectory = "victorialogs";

nixos/modules/services/monitoring/vlagent.nix

@@ -0,0 +1,132 @@
+{
+  config,
+  pkgs,
+  lib,
+  utils,
+  ...
+}:
+
+let
+  cfg = config.services.vlagent;
+
+  startCLIList = [
+    (lib.getExe cfg.package)
+  ]
+  ++ lib.optionals (cfg.remoteWrite.url != null) [
+    "-remoteWrite.url=${cfg.remoteWrite.url}"
+    "-remoteWrite.tmpDataPath=%C/vlagent/remote_write_tmp"
+  ]
+  ++ lib.optionals (cfg.remoteWrite.basicAuthPasswordFile != null) [
+    "-remoteWrite.basicAuth.passwordFile=%d/remote_write_basic_auth_password"
+  ]
+  ++ lib.optionals (cfg.remoteWrite.basicAuthUsername != null) [
+    "-remoteWrite.basicAuth.username=${cfg.remoteWrite.basicAuthUsername}"
+  ]
+  ++ lib.optionals (cfg.remoteWrite.maxDiskUsagePerUrl != null) [
+    "-remoteWrite.maxDiskUsagePerUrl=${cfg.remoteWrite.maxDiskUsagePerUrl}"
+  ];
+in
+{
+  meta = {
+    maintainers = [ lib.maintainers.shawn8901 ];
+  };
+
+  options.services.vlagent = {
+    enable = lib.mkOption {
+      type = lib.types.bool;
+      default = false;
+      description = ''
+        Whether to enable VictoriaMetrics's `vlagent`.
+
+        `vlagent` is a tiny agent which helps you collect logs from various sources and store them in VictoriaLogs .
+      '';
+    };
+
+    package = lib.mkPackageOption pkgs "vlagent" { };
+
+    remoteWrite = {
+      url = lib.mkOption {
+        default = null;
+        type = lib.types.nullOr lib.types.str;
+        description = ''
+          Endpoint for the victorialogs instance
+        '';
+      };
+      maxDiskUsagePerUrl = lib.mkOption {
+        default = null;
+        type = lib.types.nullOr lib.types.str;
+        description = ''
+          The maximum file-based buffer size in bytes. Supports the following optional suffixes for size values: KB, MB, GB, TB, KiB, MiB, GiB, TiB.
+          See docs for more infomations: <https://docs.victoriametrics.com/vlagent.html#advanced-usage>
+        '';
+      };
+      basicAuthUsername = lib.mkOption {
+        default = null;
+        type = lib.types.nullOr lib.types.str;
+        description = ''
+          Basic Auth username used to connect to remote_write endpoint
+        '';
+      };
+      basicAuthPasswordFile = lib.mkOption {
+        default = null;
+        type = lib.types.nullOr lib.types.str;
+        description = ''
+          File that contains the Basic Auth password used to connect to remote_write endpoint
+        '';
+      };
+    };
+
+    openFirewall = lib.mkOption {
+      type = lib.types.bool;
+      default = false;
+      description = ''
+        Whether to open the firewall for the default ports.
+      '';
+    };
+
+    extraArgs = lib.mkOption {
+      type = lib.types.listOf lib.types.str;
+      default = [ ];
+      description = ''
+        Extra args to pass to `vlagent`. See the docs:
+        <https://docs.victoriametrics.com/vlagent.html#advanced-usage>
+        or {command}`vlagent -help` for more information.
+      '';
+    };
+  };
+
+  config = lib.mkIf cfg.enable {
+
+    assertions = [
+      {
+        assertion =
+          (cfg.remoteWrite.basicAuthUsername == null && cfg.remoteWrite.basicAuthPasswordFile == null)
+          || (cfg.remoteWrite.basicAuthUsername != null && cfg.remoteWrite.basicAuthPasswordFile != null);
+        message = "Both basicAuthUsername and basicAuthPasswordFile must be set together to enable basicAuth functionality, or neither should be set.";
+      }
+    ];
+
+    networking.firewall.allowedTCPPorts = lib.mkIf cfg.openFirewall [ 9429 ];
+
+    systemd.services.vlagent = {
+      wantedBy = [ "multi-user.target" ];
+      after = [ "network.target" ];
+      description = "vlagent system service";
+      serviceConfig = {
+        DynamicUser = true;
+        User = "vlagent";
+        Group = "vlagent";
+        Type = "simple";
+        Restart = "on-failure";
+        CacheDirectory = "vlagent";
+        ExecStart = lib.concatStringsSep " " [
+          (lib.escapeShellArgs startCLIList)
+          (utils.escapeSystemdExecArgs cfg.extraArgs)
+        ];
+        LoadCredential = lib.optional (
+          cfg.remoteWrite.basicAuthPasswordFile != null
+        ) "remote_write_basic_auth_password:${cfg.remoteWrite.basicAuthPasswordFile}";
+      };
+    };
+  };
+}

nixos/tests/all-tests.nix

@@ -1574,7 +1574,7 @@ in
   vector = import ./vector { inherit runTest; };
   velocity = runTest ./velocity.nix;
   vengi-tools = runTest ./vengi-tools.nix;
-  victorialogs = runTest ./victorialogs.nix;
+  victorialogs = import ./victorialogs { inherit runTest; };
   victoriametrics = import ./victoriametrics { inherit runTest; };
   vikunja = runTest ./vikunja.nix;
   virtualbox = handleTestOn [ "x86_64-linux" ] ./virtualbox.nix { };

nixos/tests/victorialogs/default.nix

@@ -0,0 +1,5 @@
+{ runTest }:
+{
+  local-write = runTest ./local-write.nix;
+  remote-write-with-vlagent = runTest ./remote-write-with-vlagent.nix;
+}

nixos/tests/victorialogs/local-write.nix

@@ -1,6 +1,6 @@
 { lib, ... }:
 {
-  name = "victorialogs";
+  name = "victorialogs-local-write";
   meta.maintainers = with lib.maintainers; [ marie ];
 
   nodes.machine =

nixos/tests/victorialogs/remote-write-with-vlagent.nix

@@ -0,0 +1,58 @@
+{ lib, pkgs, ... }:
+let
+
+  username = "vltest";
+  password = "rUceu1W41U"; # random string
+  passwordFile = pkgs.writeText "password-file" password;
+in
+{
+  name = "victorialogs-remote-write-with-vlagent";
+  meta.maintainers = [ lib.maintainers.shawn8901 ];
+
+  nodes.server =
+    { pkgs, ... }:
+    {
+      networking.firewall.allowedTCPPorts = [ 9428 ];
+      services.victorialogs = {
+        enable = true;
+        basicAuthUsername = username;
+        basicAuthPasswordFile = toString passwordFile;
+      };
+    };
+
+  nodes.client =
+    { pkgs, ... }:
+    {
+      services.vlagent = {
+        enable = true;
+        remoteWrite = {
+          url = "http://server:9428/internal/insert";
+          basicAuthUsername = username;
+          basicAuthPasswordFile = toString passwordFile;
+        };
+      };
+
+      services.journald.upload = {
+        enable = true;
+        settings = {
+          Upload.URL = "http://localhost:9429/insert/journald";
+        };
+      };
+      environment.systemPackages = [ pkgs.curl ];
+
+    };
+
+  testScript = ''
+    server.wait_for_unit("victorialogs.service")
+    server.wait_for_open_port(9428)
+
+    client.wait_for_unit("vlagent")
+    client.wait_for_open_port(9429)
+
+    client.wait_for_unit("systemd-journal-upload")
+
+    client.succeed("echo 'meow' | systemd-cat -p info")
+
+    server.wait_until_succeeds("curl -u ${username}:${password} --fail http://localhost:9428/select/logsql/query -d 'query=\"meow\"' | grep meow")
+  '';
+}

pkgs/by-name/vi/victorialogs/package.nix

@@ -4,6 +4,8 @@
   fetchFromGitHub,
   nix-update-script,
   nixosTests,
+  withServer ? true,
+  withVlAgent ? false,
 }:
 
 buildGoModule (finalAttrs: {
@@ -19,14 +21,16 @@ buildGoModule (finalAttrs: {
 
   vendorHash = null;
 
-  subPackages = [
+  subPackages =
-    "app/victoria-logs"
+    lib.optionals withServer [
-    "app/vlinsert"
+      "app/victoria-logs"
-    "app/vlselect"
+      "app/vlinsert"
-    "app/vlstorage"
+      "app/vlselect"
-    "app/vlogsgenerator"
+      "app/vlstorage"
-    "app/vlogscli"
+      "app/vlogsgenerator"
-  ];
+      "app/vlogscli"
+    ]
+    ++ lib.optionals withVlAgent [ "app/vlagent" ];
 
   ldflags = [
     "-s"
@@ -49,7 +53,10 @@ buildGoModule (finalAttrs: {
     homepage = "https://docs.victoriametrics.com/victorialogs/";
     description = "User friendly log database from VictoriaMetrics";
     license = lib.licenses.asl20;
-    maintainers = with lib.maintainers; [ marie ];
+    maintainers = with lib.maintainers; [
+      marie
+      shawn8901
+    ];
     changelog = "https://github.com/VictoriaMetrics/VictoriaLogs/releases/tag/${finalAttrs.src.tag}";
     mainProgram = "victoria-logs";
   };

pkgs/by-name/vl/vlagent/package.nix

@@ -0,0 +1,11 @@
+{ lib, victorialogs }:
+
+# This package is build out of the victorialogs package.
+# so no separate update prs are needed for vlagent
+# nixpkgs-update: no auto update
+lib.addMetaAttrs { mainProgram = "vlagent"; } (
+  victorialogs.override {
+    withServer = false;
+    withVlAgent = true;
+  }
+)