diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix
index 6b928e4f3b50..c4f60d1532cc 100644
--- a/nixos/tests/all-tests.nix
+++ b/nixos/tests/all-tests.nix
@@ -306,7 +306,7 @@ in
cinnamon-wayland = runTest ./cinnamon-wayland.nix;
cjdns = runTest ./cjdns.nix;
clatd = runTest ./clatd.nix;
- clickhouse = handleTest ./clickhouse { };
+ clickhouse = import ./clickhouse { inherit runTest; };
cloud-init = handleTest ./cloud-init.nix { };
cloud-init-hostname = handleTest ./cloud-init-hostname.nix { };
cloudlog = runTest ./cloudlog.nix;
diff --git a/nixos/tests/clickhouse/base.nix b/nixos/tests/clickhouse/base.nix
index fef710b2e5ab..cbeb5b64699a 100644
--- a/nixos/tests/clickhouse/base.nix
+++ b/nixos/tests/clickhouse/base.nix
@@ -1,35 +1,33 @@
-import ../make-test-python.nix (
- { pkgs, ... }:
- {
- name = "clickhouse";
- meta.maintainers = with pkgs.lib.maintainers; [ jpds ];
+{ pkgs, ... }:
+{
+ name = "clickhouse";
+ meta.maintainers = with pkgs.lib.maintainers; [ jpds ];
- nodes.machine = {
- services.clickhouse.enable = true;
- virtualisation.memorySize = 4096;
- };
+ nodes.machine = {
+ services.clickhouse.enable = true;
+ virtualisation.memorySize = 4096;
+ };
- testScript =
- let
- # work around quote/substitution complexity by Nix, Perl, bash and SQL.
- tableDDL = pkgs.writeText "ddl.sql" "CREATE TABLE `demo` (`value` FixedString(10)) engine = MergeTree PARTITION BY value ORDER BY tuple();";
- insertQuery = pkgs.writeText "insert.sql" "INSERT INTO `demo` (`value`) VALUES ('foo');";
- selectQuery = pkgs.writeText "select.sql" "SELECT * from `demo`";
- in
- ''
- machine.start()
- machine.wait_for_unit("clickhouse.service")
- machine.wait_for_open_port(9000)
+ testScript =
+ let
+ # work around quote/substitution complexity by Nix, Perl, bash and SQL.
+ tableDDL = pkgs.writeText "ddl.sql" "CREATE TABLE `demo` (`value` FixedString(10)) engine = MergeTree PARTITION BY value ORDER BY tuple();";
+ insertQuery = pkgs.writeText "insert.sql" "INSERT INTO `demo` (`value`) VALUES ('foo');";
+ selectQuery = pkgs.writeText "select.sql" "SELECT * from `demo`";
+ in
+ ''
+ machine.start()
+ machine.wait_for_unit("clickhouse.service")
+ machine.wait_for_open_port(9000)
- machine.succeed(
- "cat ${tableDDL} | clickhouse-client"
- )
- machine.succeed(
- "cat ${insertQuery} | clickhouse-client"
- )
- machine.succeed(
- "cat ${selectQuery} | clickhouse-client | grep foo"
- )
- '';
- }
-)
+ machine.succeed(
+ "cat ${tableDDL} | clickhouse-client"
+ )
+ machine.succeed(
+ "cat ${insertQuery} | clickhouse-client"
+ )
+ machine.succeed(
+ "cat ${selectQuery} | clickhouse-client | grep foo"
+ )
+ '';
+}
diff --git a/nixos/tests/clickhouse/default.nix b/nixos/tests/clickhouse/default.nix
index 0efa24146ee3..e6568010eb66 100644
--- a/nixos/tests/clickhouse/default.nix
+++ b/nixos/tests/clickhouse/default.nix
@@ -1,12 +1,8 @@
-{
- system ? builtins.currentSystem,
- config ? { },
- pkgs ? import ../../.. { inherit system config; },
-}:
+{ runTest }:
{
- base = import ./base.nix { inherit system pkgs; };
- kafka = import ./kafka.nix { inherit system pkgs; };
- keeper = import ./keeper.nix { inherit system pkgs; };
- s3 = import ./s3.nix { inherit system pkgs; };
+ base = runTest ./base.nix;
+ kafka = runTest ./kafka.nix;
+ keeper = runTest ./keeper.nix;
+ s3 = runTest ./s3.nix;
}
diff --git a/nixos/tests/clickhouse/kafka.nix b/nixos/tests/clickhouse/kafka.nix
index 2dd6f1946e35..29e4f839d07f 100644
--- a/nixos/tests/clickhouse/kafka.nix
+++ b/nixos/tests/clickhouse/kafka.nix
@@ -1,174 +1,172 @@
-import ../make-test-python.nix (
- { pkgs, ... }:
+{ pkgs, ... }:
- let
- kafkaNamedCollectionConfig = ''
-
-
-
-
- kafka:9092
- test_topic
- clickhouse
- JSONEachRow
- 0
- 1
- 1
+let
+ kafkaNamedCollectionConfig = ''
+
+
+
+
+ kafka:9092
+ test_topic
+ clickhouse
+ JSONEachRow
+ 0
+ 1
+ 1
-
-
- all
- earliest
-
-
-
-
- '';
+
+
+ all
+ earliest
+
+
+
+
+ '';
- kafkaNamedCollection = pkgs.writeText "kafka.xml" kafkaNamedCollectionConfig;
- in
- {
- name = "clickhouse-kafka";
- meta.maintainers = with pkgs.lib.maintainers; [ jpds ];
+ kafkaNamedCollection = pkgs.writeText "kafka.xml" kafkaNamedCollectionConfig;
+in
+{
+ name = "clickhouse-kafka";
+ meta.maintainers = with pkgs.lib.maintainers; [ jpds ];
- nodes = {
- clickhouse = {
- environment.etc = {
- "clickhouse-server/config.d/kafka.xml" = {
- source = "${kafkaNamedCollection}";
- };
+ nodes = {
+ clickhouse = {
+ environment.etc = {
+ "clickhouse-server/config.d/kafka.xml" = {
+ source = "${kafkaNamedCollection}";
};
-
- services.clickhouse.enable = true;
- virtualisation.memorySize = 4096;
};
- kafka = {
- networking.firewall.allowedTCPPorts = [
- 9092
- 9093
- ];
-
- environment.systemPackages = [
- pkgs.apacheKafka
- pkgs.jq
- ];
-
- services.apache-kafka = {
- enable = true;
-
- # Randomly generated uuid. You can get one by running:
- # kafka-storage.sh random-uuid
- clusterId = "b81s-MuGSwyt_B9_h37wtQ";
-
- formatLogDirs = true;
-
- settings = {
- listeners = [
- "PLAINTEXT://:9092"
- "CONTROLLER://:9093"
- ];
- "listener.security.protocol.map" = [
- "PLAINTEXT:PLAINTEXT"
- "CONTROLLER:PLAINTEXT"
- ];
- "controller.quorum.voters" = [
- "1@kafka:9093"
- ];
- "controller.listener.names" = [ "CONTROLLER" ];
-
- "node.id" = 1;
- "broker.rack" = 1;
-
- "process.roles" = [
- "broker"
- "controller"
- ];
-
- "log.dirs" = [ "/var/lib/apache-kafka" ];
- "num.partitions" = 1;
- "offsets.topic.replication.factor" = 1;
- "transaction.state.log.replication.factor" = 1;
- "transaction.state.log.min.isr" = 1;
- };
- };
-
- systemd.services.apache-kafka.serviceConfig.StateDirectory = "apache-kafka";
- };
+ services.clickhouse.enable = true;
+ virtualisation.memorySize = 4096;
};
- testScript =
- let
- jsonTestMessage = pkgs.writeText "kafka-test-data.json" ''
- { "id": 1, "first_name": "Fred", "age": 32 }
- { "id": 2, "first_name": "Barbara", "age": 30 }
- { "id": 3, "first_name": "Nicola", "age": 12 }
- '';
- # work around quote/substitution complexity by Nix, Perl, bash and SQL.
- tableKafkaDDL = pkgs.writeText "ddl-kafka.sql" ''
- CREATE TABLE `test_kafka_topic` (
- `id` UInt32,
- `first_name` String,
- `age` UInt32
- ) ENGINE = Kafka(cluster_1);
- '';
+ kafka = {
+ networking.firewall.allowedTCPPorts = [
+ 9092
+ 9093
+ ];
- tableDDL = pkgs.writeText "ddl.sql" ''
- CREATE TABLE `test_topic` (
- `id` UInt32,
- `first_name` String,
- `age` UInt32
- ) ENGINE = MergeTree ORDER BY id;
- '';
+ environment.systemPackages = [
+ pkgs.apacheKafka
+ pkgs.jq
+ ];
- viewDDL = pkgs.writeText "view.sql" ''
- CREATE MATERIALIZED VIEW kafka_view TO test_topic AS
- SELECT
- id,
- first_name,
- age,
- FROM test_kafka_topic;
- '';
- selectQuery = pkgs.writeText "select.sql" "SELECT sum(age) from `test_topic`";
- in
- ''
- kafka.start()
- kafka.wait_for_unit("apache-kafka")
- kafka.wait_for_open_port(9092)
+ services.apache-kafka = {
+ enable = true;
- clickhouse.start()
- clickhouse.wait_for_unit("clickhouse")
- clickhouse.wait_for_open_port(9000)
+ # Randomly generated uuid. You can get one by running:
+ # kafka-storage.sh random-uuid
+ clusterId = "b81s-MuGSwyt_B9_h37wtQ";
- clickhouse.wait_until_succeeds(
- """
- journalctl -o cat -u clickhouse.service | grep "Merging configuration file '/etc/clickhouse-server/config.d/kafka.xml'"
- """
- )
+ formatLogDirs = true;
- clickhouse.succeed(
- "cat ${tableKafkaDDL} | clickhouse-client"
- )
+ settings = {
+ listeners = [
+ "PLAINTEXT://:9092"
+ "CONTROLLER://:9093"
+ ];
+ "listener.security.protocol.map" = [
+ "PLAINTEXT:PLAINTEXT"
+ "CONTROLLER:PLAINTEXT"
+ ];
+ "controller.quorum.voters" = [
+ "1@kafka:9093"
+ ];
+ "controller.listener.names" = [ "CONTROLLER" ];
- clickhouse.succeed(
- "cat ${tableDDL} | clickhouse-client"
- )
+ "node.id" = 1;
+ "broker.rack" = 1;
- clickhouse.succeed(
- "cat ${viewDDL} | clickhouse-client"
- )
+ "process.roles" = [
+ "broker"
+ "controller"
+ ];
- kafka.succeed(
- "jq -rc . ${jsonTestMessage} | kafka-console-producer.sh --topic test_topic --bootstrap-server kafka:9092"
- )
+ "log.dirs" = [ "/var/lib/apache-kafka" ];
+ "num.partitions" = 1;
+ "offsets.topic.replication.factor" = 1;
+ "transaction.state.log.replication.factor" = 1;
+ "transaction.state.log.min.isr" = 1;
+ };
+ };
- kafka.wait_until_succeeds(
- "journalctl -o cat -u apache-kafka.service | grep 'Created a new member id ClickHouse-clickhouse-default-test_kafka_topic'"
- )
+ systemd.services.apache-kafka.serviceConfig.StateDirectory = "apache-kafka";
+ };
+ };
- clickhouse.wait_until_succeeds(
- "cat ${selectQuery} | clickhouse-client | grep 74"
- )
+ testScript =
+ let
+ jsonTestMessage = pkgs.writeText "kafka-test-data.json" ''
+ { "id": 1, "first_name": "Fred", "age": 32 }
+ { "id": 2, "first_name": "Barbara", "age": 30 }
+ { "id": 3, "first_name": "Nicola", "age": 12 }
'';
- }
-)
+ # work around quote/substitution complexity by Nix, Perl, bash and SQL.
+ tableKafkaDDL = pkgs.writeText "ddl-kafka.sql" ''
+ CREATE TABLE `test_kafka_topic` (
+ `id` UInt32,
+ `first_name` String,
+ `age` UInt32
+ ) ENGINE = Kafka(cluster_1);
+ '';
+
+ tableDDL = pkgs.writeText "ddl.sql" ''
+ CREATE TABLE `test_topic` (
+ `id` UInt32,
+ `first_name` String,
+ `age` UInt32
+ ) ENGINE = MergeTree ORDER BY id;
+ '';
+
+ viewDDL = pkgs.writeText "view.sql" ''
+ CREATE MATERIALIZED VIEW kafka_view TO test_topic AS
+ SELECT
+ id,
+ first_name,
+ age,
+ FROM test_kafka_topic;
+ '';
+ selectQuery = pkgs.writeText "select.sql" "SELECT sum(age) from `test_topic`";
+ in
+ ''
+ kafka.start()
+ kafka.wait_for_unit("apache-kafka")
+ kafka.wait_for_open_port(9092)
+
+ clickhouse.start()
+ clickhouse.wait_for_unit("clickhouse")
+ clickhouse.wait_for_open_port(9000)
+
+ clickhouse.wait_until_succeeds(
+ """
+ journalctl -o cat -u clickhouse.service | grep "Merging configuration file '/etc/clickhouse-server/config.d/kafka.xml'"
+ """
+ )
+
+ clickhouse.succeed(
+ "cat ${tableKafkaDDL} | clickhouse-client"
+ )
+
+ clickhouse.succeed(
+ "cat ${tableDDL} | clickhouse-client"
+ )
+
+ clickhouse.succeed(
+ "cat ${viewDDL} | clickhouse-client"
+ )
+
+ kafka.succeed(
+ "jq -rc . ${jsonTestMessage} | kafka-console-producer.sh --topic test_topic --bootstrap-server kafka:9092"
+ )
+
+ kafka.wait_until_succeeds(
+ "journalctl -o cat -u apache-kafka.service | grep 'Created a new member id ClickHouse-clickhouse-default-test_kafka_topic'"
+ )
+
+ clickhouse.wait_until_succeeds(
+ "cat ${selectQuery} | clickhouse-client | grep 74"
+ )
+ '';
+}
diff --git a/nixos/tests/clickhouse/keeper.nix b/nixos/tests/clickhouse/keeper.nix
index 27e839e288a7..40be4c19f2cf 100644
--- a/nixos/tests/clickhouse/keeper.nix
+++ b/nixos/tests/clickhouse/keeper.nix
@@ -1,185 +1,183 @@
-import ../make-test-python.nix (
- { lib, pkgs, ... }:
- rec {
- name = "clickhouse-keeper";
- meta.maintainers = with pkgs.lib.maintainers; [ jpds ];
+{ lib, pkgs, ... }:
+rec {
+ name = "clickhouse-keeper";
+ meta.maintainers = with pkgs.lib.maintainers; [ jpds ];
- nodes =
- let
- node = i: {
+ nodes =
+ let
+ node = i: {
- environment.etc = {
- "clickhouse-server/config.d/cluster.xml".text = ''
-
-
-
- ${lib.concatStrings (
- lib.imap0 (j: name: ''
-
-
- ${name}
- 9000
-
-
- '') (builtins.attrNames nodes)
- )}
-
-
-
- '';
-
- "clickhouse-server/config.d/keeper.xml".text = ''
-
-
- ${toString i}
- 9181
- /var/lib/clickhouse/coordination/log
- /var/lib/clickhouse/coordination/snapshots
-
-
- 10000
- 30000
- trace
- 10000
-
-
-
- ${lib.concatStrings (
- lib.imap1 (j: name: ''
-
- ${toString j}
- ${name}
- 9444
-
- '') (builtins.attrNames nodes)
- )}
-
-
-
-
+ environment.etc = {
+ "clickhouse-server/config.d/cluster.xml".text = ''
+
+
+
${lib.concatStrings (
lib.imap0 (j: name: ''
-
- ${name}
- 9181
-
+
+
+ ${name}
+ 9000
+
+
'') (builtins.attrNames nodes)
)}
-
+
+
+
+ '';
-
- /clickhouse/testcluster/task_queue/ddl
-
-
- '';
+ "clickhouse-server/config.d/keeper.xml".text = ''
+
+
+ ${toString i}
+ 9181
+ /var/lib/clickhouse/coordination/log
+ /var/lib/clickhouse/coordination/snapshots
- "clickhouse-server/config.d/listen.xml".text = ''
-
- ::
-
- '';
+
+ 10000
+ 30000
+ trace
+ 10000
+
- "clickhouse-server/config.d/macros.xml".text = ''
-
-
- ${toString i}
- perftest_2shards_1replicas
-
-
- '';
- };
+
+ ${lib.concatStrings (
+ lib.imap1 (j: name: ''
+
+ ${toString j}
+ ${name}
+ 9444
+
+ '') (builtins.attrNames nodes)
+ )}
+
+
- networking.firewall.allowedTCPPorts = [
- 9009
- 9181
- 9444
- ];
+
+ ${lib.concatStrings (
+ lib.imap0 (j: name: ''
+
+ ${name}
+ 9181
+
+ '') (builtins.attrNames nodes)
+ )}
+
- services.clickhouse.enable = true;
+
+ /clickhouse/testcluster/task_queue/ddl
+
+
+ '';
- systemd.services.clickhouse = {
- after = [ "network-online.target" ];
- requires = [ "network-online.target" ];
- };
+ "clickhouse-server/config.d/listen.xml".text = ''
+
+ ::
+
+ '';
- virtualisation.memorySize = 1024 * 4;
- virtualisation.diskSize = 1024 * 10;
+ "clickhouse-server/config.d/macros.xml".text = ''
+
+
+ ${toString i}
+ perftest_2shards_1replicas
+
+
+ '';
};
- in
- {
- clickhouse1 = node 1;
- clickhouse2 = node 2;
+
+ networking.firewall.allowedTCPPorts = [
+ 9009
+ 9181
+ 9444
+ ];
+
+ services.clickhouse.enable = true;
+
+ systemd.services.clickhouse = {
+ after = [ "network-online.target" ];
+ requires = [ "network-online.target" ];
+ };
+
+ virtualisation.memorySize = 1024 * 4;
+ virtualisation.diskSize = 1024 * 10;
};
+ in
+ {
+ clickhouse1 = node 1;
+ clickhouse2 = node 2;
+ };
- testScript =
- let
- # work around quote/substitution complexity by Nix, Perl, bash and SQL.
- clustersQuery = pkgs.writeText "clusters.sql" "SHOW clusters";
- keeperQuery = pkgs.writeText "keeper.sql" "SELECT * FROM system.zookeeper WHERE path IN ('/', '/clickhouse') FORMAT VERTICAL";
- systemClustersQuery = pkgs.writeText "system-clusters.sql" "SELECT host_name, host_address, replica_num FROM system.clusters WHERE cluster = 'perftest_2shards_1replicas'";
+ testScript =
+ let
+ # work around quote/substitution complexity by Nix, Perl, bash and SQL.
+ clustersQuery = pkgs.writeText "clusters.sql" "SHOW clusters";
+ keeperQuery = pkgs.writeText "keeper.sql" "SELECT * FROM system.zookeeper WHERE path IN ('/', '/clickhouse') FORMAT VERTICAL";
+ systemClustersQuery = pkgs.writeText "system-clusters.sql" "SELECT host_name, host_address, replica_num FROM system.clusters WHERE cluster = 'perftest_2shards_1replicas'";
- tableDDL = pkgs.writeText "table.sql" ''
- CREATE TABLE test ON cluster 'perftest_2shards_1replicas' ( A Int64, S String)
- Engine = ReplicatedMergeTree('/clickhouse/{cluster}/tables/{database}/{table}', '{replica}')
- ORDER BY A;
- '';
+ tableDDL = pkgs.writeText "table.sql" ''
+ CREATE TABLE test ON cluster 'perftest_2shards_1replicas' ( A Int64, S String)
+ Engine = ReplicatedMergeTree('/clickhouse/{cluster}/tables/{database}/{table}', '{replica}')
+ ORDER BY A;
+ '';
- insertDDL = pkgs.writeText "insert.sql" "
+ insertDDL = pkgs.writeText "insert.sql" "
INSERT INTO test SELECT number, '' FROM numbers(100000000);
";
- selectCountQuery = pkgs.writeText "select-count.sql" "
+ selectCountQuery = pkgs.writeText "select-count.sql" "
select count() from test;
";
- in
- ''
- clickhouse1.start()
- clickhouse2.start()
+ in
+ ''
+ clickhouse1.start()
+ clickhouse2.start()
- for machine in clickhouse1, clickhouse2:
- machine.wait_for_unit("clickhouse.service")
- machine.wait_for_open_port(9000)
- machine.wait_for_open_port(9009)
- machine.wait_for_open_port(9181)
- machine.wait_for_open_port(9444)
+ for machine in clickhouse1, clickhouse2:
+ machine.wait_for_unit("clickhouse.service")
+ machine.wait_for_open_port(9000)
+ machine.wait_for_open_port(9009)
+ machine.wait_for_open_port(9181)
+ machine.wait_for_open_port(9444)
- machine.wait_until_succeeds(
- """
- journalctl -o cat -u clickhouse.service | grep "Merging configuration file '/etc/clickhouse-server/config.d/keeper.xml'"
- """
- )
-
- machine.log(machine.succeed(
- "cat ${clustersQuery} | clickhouse-client | grep perftest_2shards_1replicas"
- ))
-
- machine.log(machine.succeed(
- "cat ${keeperQuery} | clickhouse-client"
- ))
-
- machine.succeed(
- "cat ${systemClustersQuery} | clickhouse-client | grep clickhouse1"
- )
- machine.succeed(
- "cat ${systemClustersQuery} | clickhouse-client | grep clickhouse2"
- )
-
- machine.succeed(
- "ls /var/lib/clickhouse/coordination/log | grep changelog"
- )
-
- clickhouse2.succeed(
- "cat ${tableDDL} | clickhouse-client"
+ machine.wait_until_succeeds(
+ """
+ journalctl -o cat -u clickhouse.service | grep "Merging configuration file '/etc/clickhouse-server/config.d/keeper.xml'"
+ """
)
- clickhouse2.succeed(
- "cat ${insertDDL} | clickhouse-client"
+ machine.log(machine.succeed(
+ "cat ${clustersQuery} | clickhouse-client | grep perftest_2shards_1replicas"
+ ))
+
+ machine.log(machine.succeed(
+ "cat ${keeperQuery} | clickhouse-client"
+ ))
+
+ machine.succeed(
+ "cat ${systemClustersQuery} | clickhouse-client | grep clickhouse1"
+ )
+ machine.succeed(
+ "cat ${systemClustersQuery} | clickhouse-client | grep clickhouse2"
)
- for machine in clickhouse1, clickhouse2:
- machine.wait_until_succeeds(
- "cat ${selectCountQuery} | clickhouse-client | grep 100000000"
- )
- '';
- }
-)
+ machine.succeed(
+ "ls /var/lib/clickhouse/coordination/log | grep changelog"
+ )
+
+ clickhouse2.succeed(
+ "cat ${tableDDL} | clickhouse-client"
+ )
+
+ clickhouse2.succeed(
+ "cat ${insertDDL} | clickhouse-client"
+ )
+
+ for machine in clickhouse1, clickhouse2:
+ machine.wait_until_succeeds(
+ "cat ${selectCountQuery} | clickhouse-client | grep 100000000"
+ )
+ '';
+}
diff --git a/nixos/tests/clickhouse/s3.nix b/nixos/tests/clickhouse/s3.nix
index b3ba5b360cc0..2268b6128fe6 100644
--- a/nixos/tests/clickhouse/s3.nix
+++ b/nixos/tests/clickhouse/s3.nix
@@ -1,123 +1,121 @@
-import ../make-test-python.nix (
- { pkgs, ... }:
+{ pkgs, ... }:
- let
- s3 = {
- bucket = "clickhouse-bucket";
- accessKey = "BKIKJAA5BMMU2RHO6IBB";
- secretKey = "V7f1CwQqAcwo80UEIJEjc5gVQUSSx5ohQ9GSrr12";
- };
+let
+ s3 = {
+ bucket = "clickhouse-bucket";
+ accessKey = "BKIKJAA5BMMU2RHO6IBB";
+ secretKey = "V7f1CwQqAcwo80UEIJEjc5gVQUSSx5ohQ9GSrr12";
+ };
- clickhouseS3StorageConfig = ''
-
-
-
-
- s3
- http://minio:9000/${s3.bucket}/
- ${s3.accessKey}
- ${s3.secretKey}
- /var/lib/clickhouse/disks/s3_disk/
-
-
- cache
- s3_disk
- /var/lib/clickhouse/disks/s3_cache/
- 10Gi
-
-
-
-
-
-
- s3_disk
-
-
-
-
-
-
- '';
- in
- {
- name = "clickhouse-s3";
- meta.maintainers = with pkgs.lib.maintainers; [ jpds ];
+ clickhouseS3StorageConfig = ''
+
+
+
+
+ s3
+ http://minio:9000/${s3.bucket}/
+ ${s3.accessKey}
+ ${s3.secretKey}
+ /var/lib/clickhouse/disks/s3_disk/
+
+
+ cache
+ s3_disk
+ /var/lib/clickhouse/disks/s3_cache/
+ 10Gi
+
+
+
+
+
+
+ s3_disk
+
+
+
+
+
+
+ '';
+in
+{
+ name = "clickhouse-s3";
+ meta.maintainers = with pkgs.lib.maintainers; [ jpds ];
- nodes = {
- clickhouse = {
- environment.etc = {
- "clickhouse-server/config.d/s3.xml" = {
- text = "${clickhouseS3StorageConfig}";
- };
+ nodes = {
+ clickhouse = {
+ environment.etc = {
+ "clickhouse-server/config.d/s3.xml" = {
+ text = "${clickhouseS3StorageConfig}";
};
-
- services.clickhouse.enable = true;
- virtualisation.diskSize = 15 * 1024;
- virtualisation.memorySize = 4 * 1024;
};
- minio =
- { pkgs, ... }:
- {
- virtualisation.diskSize = 2 * 1024;
- networking.firewall.allowedTCPPorts = [ 9000 ];
-
- services.minio = {
- enable = true;
- inherit (s3) accessKey secretKey;
- };
-
- environment.systemPackages = [ pkgs.minio-client ];
- };
+ services.clickhouse.enable = true;
+ virtualisation.diskSize = 15 * 1024;
+ virtualisation.memorySize = 4 * 1024;
};
- testScript =
- let
- # work around quote/substitution complexity by Nix, Perl, bash and SQL.
- tableDDL = pkgs.writeText "ddl.sql" ''
- CREATE TABLE `demo` (
- `value` String
- )
- ENGINE = MergeTree
- ORDER BY value
- SETTINGS storage_policy = 's3_main';
- '';
- insertQuery = pkgs.writeText "insert.sql" "INSERT INTO `demo` (`value`) VALUES ('foo');";
- selectQuery = pkgs.writeText "select.sql" "SELECT * from `demo`";
- in
- ''
- minio.wait_for_unit("minio")
- minio.wait_for_open_port(9000)
- minio.succeed(
- "mc alias set minio "
- + "http://localhost:9000 "
- + "${s3.accessKey} ${s3.secretKey} --api s3v4",
- "mc mb minio/${s3.bucket}",
- )
+ minio =
+ { pkgs, ... }:
+ {
+ virtualisation.diskSize = 2 * 1024;
+ networking.firewall.allowedTCPPorts = [ 9000 ];
- clickhouse.start()
- clickhouse.wait_for_unit("clickhouse.service")
- clickhouse.wait_for_open_port(9000)
+ services.minio = {
+ enable = true;
+ inherit (s3) accessKey secretKey;
+ };
- clickhouse.wait_until_succeeds(
- """
- journalctl -o cat -u clickhouse.service | grep "Merging configuration file '/etc/clickhouse-server/config.d/s3.xml'"
- """
- )
+ environment.systemPackages = [ pkgs.minio-client ];
+ };
+ };
- clickhouse.succeed(
- "cat ${tableDDL} | clickhouse-client"
+ testScript =
+ let
+ # work around quote/substitution complexity by Nix, Perl, bash and SQL.
+ tableDDL = pkgs.writeText "ddl.sql" ''
+ CREATE TABLE `demo` (
+ `value` String
)
- clickhouse.succeed(
- "cat ${insertQuery} | clickhouse-client"
- )
- clickhouse.succeed(
- "cat ${selectQuery} | clickhouse-client | grep foo"
- )
-
- minio.log(minio.succeed(
- "mc ls minio/${s3.bucket}",
- ))
+ ENGINE = MergeTree
+ ORDER BY value
+ SETTINGS storage_policy = 's3_main';
'';
- }
-)
+ insertQuery = pkgs.writeText "insert.sql" "INSERT INTO `demo` (`value`) VALUES ('foo');";
+ selectQuery = pkgs.writeText "select.sql" "SELECT * from `demo`";
+ in
+ ''
+ minio.wait_for_unit("minio")
+ minio.wait_for_open_port(9000)
+ minio.succeed(
+ "mc alias set minio "
+ + "http://localhost:9000 "
+ + "${s3.accessKey} ${s3.secretKey} --api s3v4",
+ "mc mb minio/${s3.bucket}",
+ )
+
+ clickhouse.start()
+ clickhouse.wait_for_unit("clickhouse.service")
+ clickhouse.wait_for_open_port(9000)
+
+ clickhouse.wait_until_succeeds(
+ """
+ journalctl -o cat -u clickhouse.service | grep "Merging configuration file '/etc/clickhouse-server/config.d/s3.xml'"
+ """
+ )
+
+ clickhouse.succeed(
+ "cat ${tableDDL} | clickhouse-client"
+ )
+ clickhouse.succeed(
+ "cat ${insertQuery} | clickhouse-client"
+ )
+ clickhouse.succeed(
+ "cat ${selectQuery} | clickhouse-client | grep foo"
+ )
+
+ minio.log(minio.succeed(
+ "mc ls minio/${s3.bucket}",
+ ))
+ '';
+}