vector: 0.47.0 → 0.48.0 (#421502)
This commit is contained in:
Yt
+154
-14
@@ -42,8 +42,119 @@ in
|
||||
services.clickhouse.enable = true;
|
||||
};
|
||||
|
||||
knot =
|
||||
{
|
||||
config,
|
||||
nodes,
|
||||
pkgs,
|
||||
...
|
||||
}:
|
||||
let
|
||||
exampleZone = pkgs.writeTextDir "example.com.zone" ''
|
||||
@ SOA ns.example.com. noc.example.com. 2019031301 86400 7200 3600000 172800
|
||||
@ NS ns1
|
||||
@ NS ns2
|
||||
ns1 A 192.168.0.1
|
||||
ns1 AAAA fd00::1
|
||||
ns2 A 192.168.0.2
|
||||
ns2 AAAA fd00::2
|
||||
www A 192.0.2.1
|
||||
www AAAA 2001:DB8::1
|
||||
sub NS ns.example.com.
|
||||
'';
|
||||
|
||||
knotZonesEnv = pkgs.buildEnv {
|
||||
name = "knot-zones";
|
||||
paths = [
|
||||
exampleZone
|
||||
];
|
||||
};
|
||||
in
|
||||
{
|
||||
networking.firewall.allowedUDPPorts = [ 53 ];
|
||||
|
||||
services.vector = {
|
||||
enable = true;
|
||||
|
||||
settings = {
|
||||
sources = {
|
||||
dnstap = {
|
||||
type = "dnstap";
|
||||
multithreaded = true;
|
||||
mode = "unix";
|
||||
lowercase_hostnames = true;
|
||||
socket_file_mode = 504;
|
||||
socket_path = "${dnstapSocket}";
|
||||
};
|
||||
};
|
||||
|
||||
sinks = {
|
||||
vector_dnstap_sink = {
|
||||
type = "vector";
|
||||
inputs = [ "dnstap" ];
|
||||
address = "clickhouse:6000";
|
||||
};
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
systemd.services.vector.serviceConfig = {
|
||||
RuntimeDirectory = "vector";
|
||||
RuntimeDirectoryMode = "0770";
|
||||
};
|
||||
|
||||
services.knot = {
|
||||
enable = true;
|
||||
settings = {
|
||||
server = {
|
||||
listen = [
|
||||
"0.0.0.0@53"
|
||||
"::@53"
|
||||
];
|
||||
automatic-acl = true;
|
||||
};
|
||||
template.default = {
|
||||
storage = knotZonesEnv;
|
||||
dnssec-signing = false;
|
||||
# Input-only zone files
|
||||
# https://www.knot-dns.cz/docs/2.8/html/operation.html#example-3
|
||||
# prevents modification of the zonefiles, since the zonefiles are immutable
|
||||
zonefile-sync = -1;
|
||||
zonefile-load = "difference";
|
||||
journal-content = "changes";
|
||||
global-module = "mod-dnstap/capture_all";
|
||||
};
|
||||
zone = {
|
||||
"example.com".file = "example.com.zone";
|
||||
};
|
||||
|
||||
mod-dnstap = [
|
||||
{
|
||||
id = "capture_all";
|
||||
sink = "unix:${dnstapSocket}";
|
||||
}
|
||||
];
|
||||
};
|
||||
};
|
||||
|
||||
systemd.services.knot = {
|
||||
after = [ "vector.service" ];
|
||||
wants = [ "vector.service" ];
|
||||
serviceConfig = {
|
||||
# DNSTAP access
|
||||
ReadWritePaths = [ "/var/run/vector" ];
|
||||
SupplementaryGroups = [ "vector" ];
|
||||
};
|
||||
};
|
||||
};
|
||||
|
||||
unbound =
|
||||
{ config, pkgs, ... }:
|
||||
{
|
||||
config,
|
||||
nodes,
|
||||
pkgs,
|
||||
...
|
||||
}:
|
||||
{
|
||||
networking.firewall.allowedUDPPorts = [ 53 ];
|
||||
|
||||
@@ -110,6 +221,16 @@ in
|
||||
];
|
||||
};
|
||||
|
||||
forward-zone = [
|
||||
{
|
||||
name = "example.com.";
|
||||
forward-addr = [
|
||||
nodes.knot.networking.primaryIPv6Address
|
||||
nodes.knot.networking.primaryIPAddress
|
||||
];
|
||||
}
|
||||
];
|
||||
|
||||
dnstap = {
|
||||
dnstap-enable = "yes";
|
||||
dnstap-socket-path = "${dnstapSocket}";
|
||||
@@ -175,7 +296,7 @@ in
|
||||
)
|
||||
ENGINE = MergeTree()
|
||||
PARTITION BY toYYYYMM(timestamp)
|
||||
ORDER BY (serverId, timestamp)
|
||||
ORDER BY (serverId, toStartOfHour(timestamp), domain, timestamp)
|
||||
POPULATE AS
|
||||
SELECT
|
||||
timestamp,
|
||||
@@ -186,13 +307,20 @@ in
|
||||
WHERE messageTypeId = 5 # ClientQuery
|
||||
'';
|
||||
|
||||
selectQuery = pkgs.writeText "select.sql" ''
|
||||
selectDomainCountQuery = pkgs.writeText "select-domain-count.sql" ''
|
||||
SELECT
|
||||
domain,
|
||||
count(domain)
|
||||
FROM dnstap.domains_view
|
||||
GROUP BY domain
|
||||
'';
|
||||
|
||||
selectAuthResponseQuery = pkgs.writeText "select-auth-response.sql" ''
|
||||
SELECT
|
||||
*
|
||||
FROM dnstap.records
|
||||
WHERE messageType = 'AuthResponse'
|
||||
'';
|
||||
in
|
||||
''
|
||||
clickhouse.wait_for_unit("clickhouse")
|
||||
@@ -205,23 +333,27 @@ in
|
||||
"cat ${tableView} | clickhouse-client",
|
||||
)
|
||||
|
||||
knot.wait_for_unit("knot")
|
||||
unbound.wait_for_unit("unbound")
|
||||
unbound.wait_for_unit("vector")
|
||||
|
||||
unbound.wait_until_succeeds(
|
||||
"journalctl -o cat -u vector.service | grep 'Socket permissions updated to 0o770'"
|
||||
)
|
||||
unbound.wait_until_succeeds(
|
||||
"journalctl -o cat -u vector.service | grep 'component_type=dnstap' | grep 'Listening... path=\"${dnstapSocket}\"'"
|
||||
)
|
||||
for machine in knot, unbound:
|
||||
machine.wait_for_unit("vector")
|
||||
|
||||
unbound.wait_for_file("${dnstapSocket}")
|
||||
unbound.succeed("test 770 -eq $(stat -c '%a' ${dnstapSocket})")
|
||||
machine.wait_until_succeeds(
|
||||
"journalctl -o cat -u vector.service | grep 'Socket permissions updated to 0o770'"
|
||||
)
|
||||
machine.wait_until_succeeds(
|
||||
"journalctl -o cat -u vector.service | grep 'component_type=dnstap' | grep 'Listening... path=\"${dnstapSocket}\"'"
|
||||
)
|
||||
|
||||
machine.wait_for_file("${dnstapSocket}")
|
||||
machine.succeed("test 770 -eq $(stat -c '%a' ${dnstapSocket})")
|
||||
|
||||
dnsclient.systemctl("start network-online.target")
|
||||
dnsclient.wait_for_unit("network-online.target")
|
||||
dnsclient.succeed(
|
||||
"dig @unbound test.local"
|
||||
"dig @unbound test.local",
|
||||
"dig @unbound www.example.com"
|
||||
)
|
||||
|
||||
unbound.wait_for_file("/var/lib/vector/logs.log")
|
||||
@@ -234,7 +366,15 @@ in
|
||||
)
|
||||
|
||||
clickhouse.log(clickhouse.wait_until_succeeds(
|
||||
"cat ${selectQuery} | clickhouse-client | grep 'test.local.'"
|
||||
"cat ${selectDomainCountQuery} | clickhouse-client | grep 'test.local.'"
|
||||
))
|
||||
|
||||
clickhouse.log(clickhouse.wait_until_succeeds(
|
||||
"cat ${selectDomainCountQuery} | clickhouse-client | grep 'www.example.com.'"
|
||||
))
|
||||
|
||||
clickhouse.log(clickhouse.wait_until_succeeds(
|
||||
"cat ${selectAuthResponseQuery} | clickhouse-client | grep 'Knot DNS ${pkgs.knot-dns.version}'"
|
||||
))
|
||||
'';
|
||||
}
|
||||
|
||||
@@ -10,6 +10,7 @@ let
|
||||
m = {}
|
||||
m.app = .SYSLOG_IDENTIFIER
|
||||
m.host = .host
|
||||
m.boot_id = ._BOOT_ID
|
||||
m.severity = to_int(.PRIORITY) ?? 0
|
||||
m.level = to_syslog_level(m.severity) ?? ""
|
||||
m.message = strip_ansi_escape_codes!(.message)
|
||||
@@ -60,6 +61,11 @@ in
|
||||
"vector_source"
|
||||
];
|
||||
endpoint = "http://localhost:8123";
|
||||
auth = {
|
||||
strategy = "basic";
|
||||
user = "vector";
|
||||
password = "helloclickhouseworld";
|
||||
};
|
||||
database = "journald";
|
||||
table = "logs";
|
||||
date_time_best_effort = true;
|
||||
@@ -72,6 +78,48 @@ in
|
||||
services.clickhouse = {
|
||||
enable = true;
|
||||
};
|
||||
|
||||
# ACL configuration for Vector
|
||||
environment = {
|
||||
etc."clickhouse-server/users.d/vector.xml".text = ''
|
||||
<clickhouse>
|
||||
<users>
|
||||
<vector>
|
||||
<password>helloclickhouseworld</password>
|
||||
|
||||
<access_management>0</access_management>
|
||||
|
||||
<quota>default</quota>
|
||||
<default_database>journald</default_database>
|
||||
|
||||
<grants>
|
||||
<query>GRANT INSERT ON journald.logs</query>
|
||||
</grants>
|
||||
</vector>
|
||||
</users>
|
||||
</clickhouse>
|
||||
'';
|
||||
|
||||
# ACL configuration for read-only client
|
||||
etc."clickhouse-server/users.d/grafana.xml".text = ''
|
||||
<clickhouse>
|
||||
<users>
|
||||
<grafana>
|
||||
<password>helloclickhouseworld2</password>
|
||||
|
||||
<access_management>0</access_management>
|
||||
|
||||
<quota>default</quota>
|
||||
<default_database>journald</default_database>
|
||||
|
||||
<grants>
|
||||
<query>GRANT SELECT ON journald.logs</query>
|
||||
</grants>
|
||||
</grafana>
|
||||
</users>
|
||||
</clickhouse>
|
||||
'';
|
||||
};
|
||||
};
|
||||
|
||||
vector =
|
||||
@@ -108,11 +156,13 @@ in
|
||||
databaseDDL = pkgs.writeText "database.sql" "CREATE DATABASE IF NOT EXISTS journald";
|
||||
|
||||
# https://clickhouse.com/blog/storing-log-data-in-clickhouse-fluent-bit-vector-open-telemetry
|
||||
# ORDER BY advice: https://kb.altinity.com/engines/mergetree-table-engine-family/pick-keys/
|
||||
tableDDL = pkgs.writeText "table.sql" ''
|
||||
CREATE TABLE IF NOT EXISTS journald.logs (
|
||||
timestamp DateTime64(6),
|
||||
app LowCardinality(String),
|
||||
host LowCardinality(String),
|
||||
boot_id LowCardinality(String),
|
||||
app LowCardinality(String),
|
||||
level LowCardinality(String),
|
||||
severity UInt8,
|
||||
message String,
|
||||
@@ -120,7 +170,7 @@ in
|
||||
pid UInt32,
|
||||
)
|
||||
ENGINE = MergeTree()
|
||||
ORDER BY (host, app, timestamp)
|
||||
ORDER BY (host, boot_id, toStartOfHour(timestamp), app, timestamp)
|
||||
PARTITION BY toYYYYMM(timestamp)
|
||||
'';
|
||||
|
||||
@@ -148,8 +198,12 @@ in
|
||||
"journalctl -o cat -u vector.service | grep 'Vector has started'"
|
||||
)
|
||||
|
||||
clickhouse.fail(
|
||||
"cat ${selectQuery} | clickhouse-client --user vector --password helloclickhouseworld | grep 2"
|
||||
)
|
||||
|
||||
clickhouse.wait_until_succeeds(
|
||||
"cat ${selectQuery} | clickhouse-client | grep 2"
|
||||
"cat ${selectQuery} | clickhouse-client --user grafana --password helloclickhouseworld2 | grep 2"
|
||||
)
|
||||
'';
|
||||
}
|
||||
|
||||
@@ -25,7 +25,7 @@
|
||||
|
||||
let
|
||||
pname = "vector";
|
||||
version = "0.47.0";
|
||||
version = "0.48.0";
|
||||
in
|
||||
rustPlatform.buildRustPackage {
|
||||
inherit pname version;
|
||||
@@ -34,10 +34,10 @@ rustPlatform.buildRustPackage {
|
||||
owner = "vectordotdev";
|
||||
repo = "vector";
|
||||
rev = "v${version}";
|
||||
hash = "sha256-09CjhSckptXbbTzBneo5aQ76YwLPSacRlsMpexsw54c=";
|
||||
hash = "sha256-qgf3aMZc1cgPlsAzgtaXLUx99KwN5no1amdkwFVyl4Y=";
|
||||
};
|
||||
|
||||
cargoHash = "sha256-9cCqdi65C4JCMP743nhrNmBlJsIFiNPGguyVEEJpGww=";
|
||||
cargoHash = "sha256-t8mfZpLrzrxj1WUpJPqZWyfBf9XobcqZY/hAeVGzhcM=";
|
||||
|
||||
nativeBuildInputs =
|
||||
[
|
||||
|
||||
Reference in New Issue
Block a user