From c44331dd5d5d54db20b941bb8c04dcf8f143648b Mon Sep 17 00:00:00 2001 From: Jacek Galowicz Date: Tue, 7 Jan 2020 20:52:24 +0100 Subject: [PATCH 1/5] nixosTests.initdb: port to python --- nixos/tests/initdb.nix | 41 +++++++++++++++++------------------------ 1 file changed, 17 insertions(+), 24 deletions(-) diff --git a/nixos/tests/initdb.nix b/nixos/tests/initdb.nix index 749d7857a134..e7c83dc4002c 100644 --- a/nixos/tests/initdb.nix +++ b/nixos/tests/initdb.nix @@ -1,26 +1,19 @@ -let - pkgs = import { }; -in -with import { inherit pkgs; system = builtins.currentSystem; }; -with pkgs.lib; +import ./make-test-python.nix ({ pkgs, latestKernel ? false, ... }: { + name = "pg-initdb"; -makeTest { - name = "pg-initdb"; + machine = {...}: + { + documentation.enable = false; + services.postgresql.enable = true; + services.postgresql.package = pkgs.postgresql_9_6; + environment.pathsToLink = [ + "/share/postgresql" + ]; + }; - machine = {...}: - { - documentation.enable = false; - services.postgresql.enable = true; - services.postgresql.package = pkgs.postgresql_9_6; - environment.pathsToLink = [ - "/share/postgresql" - ]; - }; - - testScript = '' - $machine->start; - $machine->succeed("sudo -u postgres initdb -D /tmp/testpostgres2"); - $machine->shutdown; - ''; - - } \ No newline at end of file + testScript = '' + machine.start() + machine.succeed("sudo -u postgres initdb -D /tmp/testpostgres2") + machine.shutdown() + ''; +}) From 8ecd07f9e3b0ad23a4f8430945e4cb9db4dfa485 Mon Sep 17 00:00:00 2001 From: Jacek Galowicz Date: Wed, 8 Jan 2020 10:02:26 +0100 Subject: [PATCH 2/5] nixosTests.elk: port to python --- nixos/tests/elk.nix | 99 ++++++++++++++++++++++++--------------------- 1 file changed, 52 insertions(+), 47 deletions(-) diff --git a/nixos/tests/elk.nix b/nixos/tests/elk.nix index b33d98b85d69..80db0967d400 100644 --- a/nixos/tests/elk.nix +++ b/nixos/tests/elk.nix @@ -6,20 +6,12 @@ # NIXPKGS_ALLOW_UNFREE=1 nix-build nixos/tests/elk.nix -A ELK-6 --arg enableUnfree true }: -with import ../lib/testing.nix { inherit system pkgs; }; -with pkgs.lib; - let esUrl = "http://localhost:9200"; - totalHits = message : - "curl --silent --show-error '${esUrl}/_search' -H 'Content-Type: application/json' " + - ''-d '{\"query\" : { \"match\" : { \"message\" : \"${message}\"}}}' '' + - "| jq .hits.total"; - mkElkTest = name : elk : let elasticsearchGe7 = builtins.compareVersions elk.elasticsearch.version "7" >= 0; - in makeTest { + in import ./make-test-python.nix ({ inherit name; meta = with pkgs.stdenv.lib.maintainers; { maintainers = [ eelco offline basvandijk ]; @@ -50,15 +42,15 @@ let elk.journalbeat.version "6" < 0; in { enable = true; package = elk.journalbeat; - extraConfig = mkOptionDefault ('' + extraConfig = pkgs.lib.mkOptionDefault ('' logging: to_syslog: true level: warning metrics.enabled: false output.elasticsearch: hosts: [ "127.0.0.1:9200" ] - ${optionalString lt6 "template.enabled: false"} - '' + optionalString (!lt6) '' + ${pkgs.lib.optionalString lt6 "template.enabled: false"} + '' + pkgs.lib.optionalString (!lt6) '' journalbeat.inputs: - paths: [] seek: cursor @@ -130,11 +122,23 @@ let }; testScript = '' - startAll; + import json - # Wait until elasticsearch is listening for connections. - $one->waitForUnit("elasticsearch.service"); - $one->waitForOpenPort(9200); + + def total_hits(message): + dictionary = {"query": {"match": {"message": message}}} + return ( + "curl --silent --show-error '${esUrl}/_search' " + + "-H 'Content-Type: application/json' " + + "-d '{}' ".format(json.dumps(dictionary)) + + "| jq .hits.total" + ) + + + start_all() + + one.wait_for_unit("elasticsearch.service") + one.wait_for_open_port(9200) # Continue as long as the status is not "red". The status is probably # "yellow" instead of "green" because we are using a single elasticsearch @@ -142,42 +146,43 @@ let # # TODO: extend this test with multiple elasticsearch nodes # and see if the status turns "green". - $one->waitUntilSucceeds( - "curl --silent --show-error '${esUrl}/_cluster/health' " . - "| jq .status | grep -v red"); + one.wait_until_succeeds( + "curl --silent --show-error '${esUrl}/_cluster/health' | jq .status | grep -v red" + ) - # Perform some simple logstash tests. - $one->waitForUnit("logstash.service"); - $one->waitUntilSucceeds("cat /tmp/logstash.out | grep flowers"); - $one->waitUntilSucceeds("cat /tmp/logstash.out | grep -v dragons"); + with subtest("Perform some simple logstash tests"): + one.wait_for_unit("logstash.service") + one.wait_until_succeeds("cat /tmp/logstash.out | grep flowers") + one.wait_until_succeeds("cat /tmp/logstash.out | grep -v dragons") - # See if kibana is healthy. - $one->waitForUnit("kibana.service"); - $one->waitUntilSucceeds( - "curl --silent --show-error 'http://localhost:5601/api/status' " . - "| jq .status.overall.state | grep green"); + with subtest("Kibana is healthy"): + one.wait_for_unit("kibana.service") + one.wait_until_succeeds( + "curl --silent --show-error 'http://localhost:5601/api/status' | jq .status.overall.state | grep green" + ) - # See if logstash messages arive in elasticsearch. - $one->waitUntilSucceeds("${totalHits "flowers"} | grep -v 0"); - $one->waitUntilSucceeds("${totalHits "dragons"} | grep 0"); + with subtest("Logstash messages arive in elasticsearch"): + one.wait_until_succeeds(total_hits("flowers") + " | grep -v 0") + one.wait_until_succeeds(total_hits("dragons") + " | grep 0") - # Test if a message logged to the journal - # is ingested by elasticsearch via journalbeat. - $one->waitForUnit("journalbeat.service"); - $one->execute("echo 'Supercalifragilisticexpialidocious' | systemd-cat"); - $one->waitUntilSucceeds( - "${totalHits "Supercalifragilisticexpialidocious"} | grep -v 0"); - - '' + optionalString (!elasticsearchGe7) '' - # Test elasticsearch-curator. - $one->systemctl("stop logstash"); - $one->systemctl("start elasticsearch-curator"); - $one->waitUntilSucceeds( - "! curl --silent --show-error '${esUrl}/_cat/indices' " . - "| grep logstash | grep -q ^$1"); + with subtest( + "A message logged to the journal is ingested by elasticsearch via journalbeat" + ): + one.wait_for_unit("journalbeat.service") + one.execute("echo 'Supercalifragilisticexpialidocious' | systemd-cat") + one.wait_until_succeeds( + total_hits("Supercalifragilisticexpialidocious") + " | grep -v 0" + ) + '' + pkgs.lib.optionalString (!elasticsearchGe7) '' + with subtest("Elasticsearch-curator works"): + one.systemctl("stop logstash") + one.systemctl("start elasticsearch-curator") + one.wait_until_succeeds( + '! curl --silent --show-error "${esUrl}/_cat/indices" | grep logstash | grep -q ^' + ) ''; - }; -in mapAttrs mkElkTest { + }) {}; +in pkgs.lib.mapAttrs mkElkTest { ELK-6 = if enableUnfree then { From f7b274d9571dfa81f2cf6589488c43d6e77e0d71 Mon Sep 17 00:00:00 2001 From: Jacek Galowicz Date: Wed, 8 Jan 2020 10:39:29 +0100 Subject: [PATCH 3/5] nixosTests.kafka: port to python --- nixos/tests/kafka.nix | 43 +++++++++++++++++++++++++++++-------------- 1 file changed, 29 insertions(+), 14 deletions(-) diff --git a/nixos/tests/kafka.nix b/nixos/tests/kafka.nix index 48ca98da8fa1..30e5cc0417c9 100644 --- a/nixos/tests/kafka.nix +++ b/nixos/tests/kafka.nix @@ -3,11 +3,10 @@ pkgs ? import ../.. { inherit system config; } }: -with import ../lib/testing.nix { inherit system pkgs; }; with pkgs.lib; let - makeKafkaTest = name: kafkaPackage: (makeTest { + makeKafkaTest = name: kafkaPackage: (import ./make-test-python.nix ({ inherit name; meta = with pkgs.stdenv.lib.maintainers; { maintainers = [ nequissimus ]; @@ -45,24 +44,40 @@ let }; testScript = '' - startAll; + start_all() - $zookeeper1->waitForUnit("default.target"); - $zookeeper1->waitForUnit("zookeeper.service"); - $zookeeper1->waitForOpenPort(2181); + zookeeper1.wait_for_unit("default.target") + zookeeper1.wait_for_unit("zookeeper.service") + zookeeper1.wait_for_open_port(2181) - $kafka->waitForUnit("default.target"); - $kafka->waitForUnit("apache-kafka.service"); - $kafka->waitForOpenPort(9092); + kafka.wait_for_unit("default.target") + kafka.wait_for_unit("apache-kafka.service") + kafka.wait_for_open_port(9092) - $kafka->waitUntilSucceeds("${kafkaPackage}/bin/kafka-topics.sh --create --zookeeper zookeeper1:2181 --partitions 1 --replication-factor 1 --topic testtopic"); - $kafka->mustSucceed("echo 'test 1' | ${kafkaPackage}/bin/kafka-console-producer.sh --broker-list localhost:9092 --topic testtopic"); + kafka.wait_until_succeeds( + "${kafkaPackage}/bin/kafka-topics.sh --create " + + "--zookeeper zookeeper1:2181 --partitions 1 " + + "--replication-factor 1 --topic testtopic" + ) + kafka.succeed( + "echo 'test 1' | " + + "${kafkaPackage}/bin/kafka-console-producer.sh " + + "--broker-list localhost:9092 --topic testtopic" + ) '' + (if name == "kafka_0_9" then '' - $kafka->mustSucceed("${kafkaPackage}/bin/kafka-console-consumer.sh --zookeeper zookeeper1:2181 --topic testtopic --from-beginning --max-messages 1 | grep 'test 1'"); + assert "test 1" in kafka.succeed( + "${kafkaPackage}/bin/kafka-console-consumer.sh " + + "--zookeeper zookeeper1:2181 --topic testtopic " + + "--from-beginning --max-messages 1" + ) '' else '' - $kafka->mustSucceed("${kafkaPackage}/bin/kafka-console-consumer.sh --bootstrap-server localhost:9092 --topic testtopic --from-beginning --max-messages 1 | grep 'test 1'"); + assert "test 1" in kafka.succeed( + "${kafkaPackage}/bin/kafka-console-consumer.sh " + + "--bootstrap-server localhost:9092 --topic testtopic " + + "--from-beginning --max-messages 1" + ) ''); - }); + }) {}); in with pkgs; { kafka_0_9 = makeKafkaTest "kafka_0_9" apacheKafka_0_9; From fff1c2867f848c7479873beefd29d7ce4b31a455 Mon Sep 17 00:00:00 2001 From: Jacek Galowicz Date: Thu, 9 Jan 2020 23:21:08 +0100 Subject: [PATCH 4/5] nixosTests.postgres: Refactor code --- nixos/tests/postgresql.nix | 42 +++++++++++++++++++++++--------------- 1 file changed, 26 insertions(+), 16 deletions(-) diff --git a/nixos/tests/postgresql.nix b/nixos/tests/postgresql.nix index e71c38882881..5ba2d38773bb 100644 --- a/nixos/tests/postgresql.nix +++ b/nixos/tests/postgresql.nix @@ -29,11 +29,15 @@ let machine = {...}: { - services.postgresql.enable = true; - services.postgresql.package = postgresql-package; + services.postgresql = { + enable = true; + package = postgresql-package; + }; - services.postgresqlBackup.enable = true; - services.postgresqlBackup.databases = optional (!backup-all) "postgres"; + services.postgresqlBackup = { + enable = true; + databases = optional (!backup-all) "postgres"; + }; }; testScript = let @@ -49,23 +53,29 @@ let machine.start() machine.wait_for_unit("postgresql") - # postgresql should be available just after unit start - machine.succeed( - "cat ${test-sql} | sudo -u postgres psql" - ) - machine.shutdown() # make sure that postgresql survive restart (bug #1735) - time.sleep(2) - machine.start() - machine.wait_for_unit("postgresql") + with subtest("Postgresql is available just after unit start"): + machine.succeed( + "cat ${test-sql} | sudo -u postgres psql" + ) + + with subtest("Postgresql survives restart (bug #1735)"): + machine.shutdown() + time.sleep(2) + machine.start() + machine.wait_for_unit("postgresql") + machine.fail(check_count("SELECT * FROM sth;", 3)) machine.succeed(check_count("SELECT * FROM sth;", 5)) machine.fail(check_count("SELECT * FROM sth;", 4)) machine.succeed(check_count("SELECT xpath('/test/text()', doc) FROM xmltest;", 1)) - # Check backup service - machine.succeed("systemctl start ${backupService}.service") - machine.succeed("zcat /var/backup/postgresql/${backupName}.sql.gz | grep 'ok'") - machine.succeed("stat -c '%a' /var/backup/postgresql/${backupName}.sql.gz | grep 600") + with subtest("Backup service works"): + machine.succeed( + "systemctl start ${backupService}.service", + "zcat /var/backup/postgresql/${backupName}.sql.gz | grep 'ok'", + "stat -c '%a' /var/backup/postgresql/${backupName}.sql.gz | grep 600", + ) + machine.shutdown() ''; From abebf8cdb3cd01a4b8eca9f5943c145ad77431ee Mon Sep 17 00:00:00 2001 From: Jacek Galowicz Date: Thu, 9 Jan 2020 23:21:51 +0100 Subject: [PATCH 5/5] nixosTests.initdb: Move code to existing postgres test --- nixos/tests/initdb.nix | 19 ------------------- nixos/tests/postgresql.nix | 3 +++ 2 files changed, 3 insertions(+), 19 deletions(-) delete mode 100644 nixos/tests/initdb.nix diff --git a/nixos/tests/initdb.nix b/nixos/tests/initdb.nix deleted file mode 100644 index e7c83dc4002c..000000000000 --- a/nixos/tests/initdb.nix +++ /dev/null @@ -1,19 +0,0 @@ -import ./make-test-python.nix ({ pkgs, latestKernel ? false, ... }: { - name = "pg-initdb"; - - machine = {...}: - { - documentation.enable = false; - services.postgresql.enable = true; - services.postgresql.package = pkgs.postgresql_9_6; - environment.pathsToLink = [ - "/share/postgresql" - ]; - }; - - testScript = '' - machine.start() - machine.succeed("sudo -u postgres initdb -D /tmp/testpostgres2") - machine.shutdown() - ''; -}) diff --git a/nixos/tests/postgresql.nix b/nixos/tests/postgresql.nix index 5ba2d38773bb..3201e22555ea 100644 --- a/nixos/tests/postgresql.nix +++ b/nixos/tests/postgresql.nix @@ -76,6 +76,9 @@ let "stat -c '%a' /var/backup/postgresql/${backupName}.sql.gz | grep 600", ) + with subtest("Initdb works"): + machine.succeed("sudo -u postgres initdb -D /tmp/testpostgres2") + machine.shutdown() '';