From 592d1af31404261ef8c12a1393e41ba2eb82d968 Mon Sep 17 00:00:00 2001 From: Jonathan Davies Date: Tue, 1 Jul 2025 11:08:04 +0000 Subject: [PATCH 1/3] =?UTF-8?q?vector:=200.47.0=20=E2=86=92=200.48.0?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- pkgs/by-name/ve/vector/package.nix | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkgs/by-name/ve/vector/package.nix b/pkgs/by-name/ve/vector/package.nix index 1d0c100d6100..d5009adb3ae2 100644 --- a/pkgs/by-name/ve/vector/package.nix +++ b/pkgs/by-name/ve/vector/package.nix @@ -25,7 +25,7 @@ let pname = "vector"; - version = "0.47.0"; + version = "0.48.0"; in rustPlatform.buildRustPackage { inherit pname version; @@ -34,10 +34,10 @@ rustPlatform.buildRustPackage { owner = "vectordotdev"; repo = "vector"; rev = "v${version}"; - hash = "sha256-09CjhSckptXbbTzBneo5aQ76YwLPSacRlsMpexsw54c="; + hash = "sha256-qgf3aMZc1cgPlsAzgtaXLUx99KwN5no1amdkwFVyl4Y="; }; - cargoHash = "sha256-9cCqdi65C4JCMP743nhrNmBlJsIFiNPGguyVEEJpGww="; + cargoHash = "sha256-t8mfZpLrzrxj1WUpJPqZWyfBf9XobcqZY/hAeVGzhcM="; nativeBuildInputs = [ From 680cd820a57afb71788e5eecaf7da52e3ff6e370 Mon Sep 17 00:00:00 2001 From: Jonathan Davies Date: Tue, 1 Jul 2025 11:08:20 +0000 Subject: [PATCH 2/3] nixos/vector: journald-clickhouse test: * Add boot_id to ORDER BY key * Add ACL configuration to ClickHouse --- nixos/tests/vector/journald-clickhouse.nix | 60 ++++++++++++++++++++-- 1 file changed, 57 insertions(+), 3 deletions(-) diff --git a/nixos/tests/vector/journald-clickhouse.nix b/nixos/tests/vector/journald-clickhouse.nix index b0ca516306ea..2cef987d5676 100644 --- a/nixos/tests/vector/journald-clickhouse.nix +++ b/nixos/tests/vector/journald-clickhouse.nix @@ -10,6 +10,7 @@ let m = {} m.app = .SYSLOG_IDENTIFIER m.host = .host + m.boot_id = ._BOOT_ID m.severity = to_int(.PRIORITY) ?? 0 m.level = to_syslog_level(m.severity) ?? "" m.message = strip_ansi_escape_codes!(.message) @@ -60,6 +61,11 @@ in "vector_source" ]; endpoint = "http://localhost:8123"; + auth = { + strategy = "basic"; + user = "vector"; + password = "helloclickhouseworld"; + }; database = "journald"; table = "logs"; date_time_best_effort = true; @@ -72,6 +78,48 @@ in services.clickhouse = { enable = true; }; + + # ACL configuration for Vector + environment = { + etc."clickhouse-server/users.d/vector.xml".text = '' + + + + helloclickhouseworld + + 0 + + default + journald + + + GRANT INSERT ON journald.logs + + + + + ''; + + # ACL configuration for read-only client + etc."clickhouse-server/users.d/grafana.xml".text = '' + + + + helloclickhouseworld2 + + 0 + + default + journald + + + GRANT SELECT ON journald.logs + + + + + ''; + }; }; vector = @@ -108,11 +156,13 @@ in databaseDDL = pkgs.writeText "database.sql" "CREATE DATABASE IF NOT EXISTS journald"; # https://clickhouse.com/blog/storing-log-data-in-clickhouse-fluent-bit-vector-open-telemetry + # ORDER BY advice: https://kb.altinity.com/engines/mergetree-table-engine-family/pick-keys/ tableDDL = pkgs.writeText "table.sql" '' CREATE TABLE IF NOT EXISTS journald.logs ( timestamp DateTime64(6), - app LowCardinality(String), host LowCardinality(String), + boot_id LowCardinality(String), + app LowCardinality(String), level LowCardinality(String), severity UInt8, message String, @@ -120,7 +170,7 @@ in pid UInt32, ) ENGINE = MergeTree() - ORDER BY (host, app, timestamp) + ORDER BY (host, boot_id, toStartOfHour(timestamp), app, timestamp) PARTITION BY toYYYYMM(timestamp) ''; @@ -148,8 +198,12 @@ in "journalctl -o cat -u vector.service | grep 'Vector has started'" ) + clickhouse.fail( + "cat ${selectQuery} | clickhouse-client --user vector --password helloclickhouseworld | grep 2" + ) + clickhouse.wait_until_succeeds( - "cat ${selectQuery} | clickhouse-client | grep 2" + "cat ${selectQuery} | clickhouse-client --user grafana --password helloclickhouseworld2 | grep 2" ) ''; } From c012aca83d6c0833a5b544bec3784494f6ab3e70 Mon Sep 17 00:00:00 2001 From: Jonathan Davies Date: Tue, 1 Jul 2025 11:30:18 +0000 Subject: [PATCH 3/3] nixos/vector: dnstap test: * Integrate Knot DNS * Improve ORDER BY key --- nixos/tests/vector/dnstap.nix | 168 +++++++++++++++++++++++++++++++--- 1 file changed, 154 insertions(+), 14 deletions(-) diff --git a/nixos/tests/vector/dnstap.nix b/nixos/tests/vector/dnstap.nix index 02d40907f8af..894de8ebf29b 100644 --- a/nixos/tests/vector/dnstap.nix +++ b/nixos/tests/vector/dnstap.nix @@ -42,8 +42,119 @@ in services.clickhouse.enable = true; }; + knot = + { + config, + nodes, + pkgs, + ... + }: + let + exampleZone = pkgs.writeTextDir "example.com.zone" '' + @ SOA ns.example.com. noc.example.com. 2019031301 86400 7200 3600000 172800 + @ NS ns1 + @ NS ns2 + ns1 A 192.168.0.1 + ns1 AAAA fd00::1 + ns2 A 192.168.0.2 + ns2 AAAA fd00::2 + www A 192.0.2.1 + www AAAA 2001:DB8::1 + sub NS ns.example.com. + ''; + + knotZonesEnv = pkgs.buildEnv { + name = "knot-zones"; + paths = [ + exampleZone + ]; + }; + in + { + networking.firewall.allowedUDPPorts = [ 53 ]; + + services.vector = { + enable = true; + + settings = { + sources = { + dnstap = { + type = "dnstap"; + multithreaded = true; + mode = "unix"; + lowercase_hostnames = true; + socket_file_mode = 504; + socket_path = "${dnstapSocket}"; + }; + }; + + sinks = { + vector_dnstap_sink = { + type = "vector"; + inputs = [ "dnstap" ]; + address = "clickhouse:6000"; + }; + }; + }; + }; + + systemd.services.vector.serviceConfig = { + RuntimeDirectory = "vector"; + RuntimeDirectoryMode = "0770"; + }; + + services.knot = { + enable = true; + settings = { + server = { + listen = [ + "0.0.0.0@53" + "::@53" + ]; + automatic-acl = true; + }; + template.default = { + storage = knotZonesEnv; + dnssec-signing = false; + # Input-only zone files + # https://www.knot-dns.cz/docs/2.8/html/operation.html#example-3 + # prevents modification of the zonefiles, since the zonefiles are immutable + zonefile-sync = -1; + zonefile-load = "difference"; + journal-content = "changes"; + global-module = "mod-dnstap/capture_all"; + }; + zone = { + "example.com".file = "example.com.zone"; + }; + + mod-dnstap = [ + { + id = "capture_all"; + sink = "unix:${dnstapSocket}"; + } + ]; + }; + }; + + systemd.services.knot = { + after = [ "vector.service" ]; + wants = [ "vector.service" ]; + serviceConfig = { + # DNSTAP access + ReadWritePaths = [ "/var/run/vector" ]; + SupplementaryGroups = [ "vector" ]; + }; + }; + }; + unbound = - { config, pkgs, ... }: + { + config, + nodes, + pkgs, + ... + }: { networking.firewall.allowedUDPPorts = [ 53 ]; @@ -110,6 +221,16 @@ in ]; }; + forward-zone = [ + { + name = "example.com."; + forward-addr = [ + nodes.knot.networking.primaryIPv6Address + nodes.knot.networking.primaryIPAddress + ]; + } + ]; + dnstap = { dnstap-enable = "yes"; dnstap-socket-path = "${dnstapSocket}"; @@ -175,7 +296,7 @@ in ) ENGINE = MergeTree() PARTITION BY toYYYYMM(timestamp) - ORDER BY (serverId, timestamp) + ORDER BY (serverId, toStartOfHour(timestamp), domain, timestamp) POPULATE AS SELECT timestamp, @@ -186,13 +307,20 @@ in WHERE messageTypeId = 5 # ClientQuery ''; - selectQuery = pkgs.writeText "select.sql" '' + selectDomainCountQuery = pkgs.writeText "select-domain-count.sql" '' SELECT domain, count(domain) FROM dnstap.domains_view GROUP BY domain ''; + + selectAuthResponseQuery = pkgs.writeText "select-auth-response.sql" '' + SELECT + * + FROM dnstap.records + WHERE messageType = 'AuthResponse' + ''; in '' clickhouse.wait_for_unit("clickhouse") @@ -205,23 +333,27 @@ in "cat ${tableView} | clickhouse-client", ) + knot.wait_for_unit("knot") unbound.wait_for_unit("unbound") - unbound.wait_for_unit("vector") - unbound.wait_until_succeeds( - "journalctl -o cat -u vector.service | grep 'Socket permissions updated to 0o770'" - ) - unbound.wait_until_succeeds( - "journalctl -o cat -u vector.service | grep 'component_type=dnstap' | grep 'Listening... path=\"${dnstapSocket}\"'" - ) + for machine in knot, unbound: + machine.wait_for_unit("vector") - unbound.wait_for_file("${dnstapSocket}") - unbound.succeed("test 770 -eq $(stat -c '%a' ${dnstapSocket})") + machine.wait_until_succeeds( + "journalctl -o cat -u vector.service | grep 'Socket permissions updated to 0o770'" + ) + machine.wait_until_succeeds( + "journalctl -o cat -u vector.service | grep 'component_type=dnstap' | grep 'Listening... path=\"${dnstapSocket}\"'" + ) + + machine.wait_for_file("${dnstapSocket}") + machine.succeed("test 770 -eq $(stat -c '%a' ${dnstapSocket})") dnsclient.systemctl("start network-online.target") dnsclient.wait_for_unit("network-online.target") dnsclient.succeed( - "dig @unbound test.local" + "dig @unbound test.local", + "dig @unbound www.example.com" ) unbound.wait_for_file("/var/lib/vector/logs.log") @@ -234,7 +366,15 @@ in ) clickhouse.log(clickhouse.wait_until_succeeds( - "cat ${selectQuery} | clickhouse-client | grep 'test.local.'" + "cat ${selectDomainCountQuery} | clickhouse-client | grep 'test.local.'" + )) + + clickhouse.log(clickhouse.wait_until_succeeds( + "cat ${selectDomainCountQuery} | clickhouse-client | grep 'www.example.com.'" + )) + + clickhouse.log(clickhouse.wait_until_succeeds( + "cat ${selectAuthResponseQuery} | clickhouse-client | grep 'Knot DNS ${pkgs.knot-dns.version}'" )) ''; }