From ddf9a3eba5d0ce359f283f0b0b32a35b7aa6ba28 Mon Sep 17 00:00:00 2001 From: Jonathan Davies Date: Wed, 28 May 2025 20:10:59 +0000 Subject: [PATCH] nixos/prometheus: Extend prometheus-pair test to verify first compaction succeeds --- nixos/tests/prometheus/prometheus-pair.nix | 38 ++++++++++++++++++++++ 1 file changed, 38 insertions(+) diff --git a/nixos/tests/prometheus/prometheus-pair.nix b/nixos/tests/prometheus/prometheus-pair.nix index 6d8d5900480f..48300f926858 100644 --- a/nixos/tests/prometheus/prometheus-pair.nix +++ b/nixos/tests/prometheus/prometheus-pair.nix @@ -14,6 +14,9 @@ services.prometheus = { enable = true; globalConfig.scrape_interval = "2s"; + extraFlags = [ + "--storage.tsdb.min-block-duration=15s" + ]; scrapeConfigs = [ { job_name = "prometheus"; @@ -40,6 +43,9 @@ services.prometheus = { enable = true; globalConfig.scrape_interval = "2s"; + extraFlags = [ + "--storage.tsdb.min-block-duration=15s" + ]; scrapeConfigs = [ { job_name = "prometheus"; @@ -86,6 +92,38 @@ + "jq '.data.result[0].value[1]' | grep '\"2\"'" ) + machine.wait_until_succeeds( + "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=prometheus_tsdb_head_series_created_total\{instance=\"prometheus1:9090\"\}' | " + + "jq '.data.result[0].value[1]' | grep -v '\"0\"'" + ) + + with subtest("Compaction verification"): + for machine in prometheus1, prometheus2: + machine.wait_until_succeeds("journalctl -o cat -u prometheus.service | grep -E '(log=ERROR|write block)'") + + machine.wait_until_succeeds("journalctl -o cat -u prometheus.service | grep 'Head GC completed'") + + machine.wait_until_succeeds("journalctl -o cat -u prometheus.service | grep 'Creating checkpoint'") + + machine.wait_until_succeeds("journalctl -o cat -u prometheus.service | grep 'WAL checkpoint complete'") + + machine.wait_until_succeeds("journalctl -o cat -u prometheus.service | grep 'compact blocks'") + + machine.wait_until_succeeds("journalctl -o cat -u prometheus.service | grep 'Deleting obsolete block'") + + machine.wait_until_succeeds( + "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=prometheus_tsdb_compactions_total\{instance=\"prometheus1:9090\"\}' | " + + "jq '.data.result[0].value[1]' | grep -v '\"0\"'" + ) + + machine.wait_until_succeeds( + "curl -sf 'http://127.0.0.1:9090/api/v1/query?query=prometheus_tsdb_compactions_failed_total\{instance=\"prometheus1:9090\"\}' | " + + "jq '.data.result[0].value[1]' | grep '\"0\"'" + ) + + for machine in prometheus1, prometheus2: + machine.fail("journalctl -o cat -u prometheus.service | grep 'level=ERROR'") + prometheus1.log(prometheus1.succeed("systemd-analyze security prometheus.service | grep -v '✓'")) ''; }