diff --git a/maintainers/maintainer-list.nix b/maintainers/maintainer-list.nix index 459af64996ca..2e29fd89dc3d 100644 --- a/maintainers/maintainer-list.nix +++ b/maintainers/maintainer-list.nix @@ -10188,6 +10188,14 @@ githubId = 1064477; name = "Hannes Hornwall"; }; + hougo = { + name = "Hugo Renard"; + email = "hugo.renard@proton.me"; + matrix = "@hougo:liiib.re"; + github = "hrenard"; + githubId = 7594435; + keys = [ { fingerprint = "3AE9 67F9 2C9F 55E9 03C8 283F 3A28 5FD4 7020 9C59"; } ]; + }; hoverbear = { email = "operator+nix@hoverbear.org"; matrix = "@hoverbear:matrix.org"; diff --git a/nixos/doc/manual/release-notes/rl-2511.section.md b/nixos/doc/manual/release-notes/rl-2511.section.md index 03f9fc71c4a3..0e54f9304d19 100644 --- a/nixos/doc/manual/release-notes/rl-2511.section.md +++ b/nixos/doc/manual/release-notes/rl-2511.section.md @@ -24,6 +24,8 @@ - [LACT](https://github.com/ilya-zlobintsev/LACT), a GPU monitoring and configuration tool, can now be enabled through [services.lact.enable](#opt-services.lact.enable). Note that for LACT to work properly on AMD GPU systems, you need to enable [hardware.amdgpu.overdrive.enable](#opt-hardware.amdgpu.overdrive.enable). +- [tlsrpt-reporter], an application suite to generate and deliver TLSRPT reports. Available as [services.tlsrpt](#opt-services.tlsrpt.enable). + - [Broadcast Box](https://github.com/Glimesh/broadcast-box), a WebRTC broadcast server. Available as [services.broadcast-box](options.html#opt-services.broadcast-box.enable). - Docker now defaults to 28.x, because version 27.x stopped receiving security updates and bug fixes after [May 2, 2025](https://github.com/moby/moby/pull/49910). diff --git a/nixos/modules/module-list.nix b/nixos/modules/module-list.nix index 8b9c38d4ee71..45f09d3d6a4b 100644 --- a/nixos/modules/module-list.nix +++ b/nixos/modules/module-list.nix @@ -754,6 +754,7 @@ ./services/mail/spamassassin.nix ./services/mail/stalwart-mail.nix ./services/mail/sympa.nix + ./services/mail/tlsrpt.nix ./services/mail/zeyple.nix ./services/matrix/appservice-discord.nix ./services/matrix/appservice-irc.nix diff --git a/nixos/modules/services/mail/tlsrpt.nix b/nixos/modules/services/mail/tlsrpt.nix new file mode 100644 index 000000000000..732679c9b05a --- /dev/null +++ b/nixos/modules/services/mail/tlsrpt.nix @@ -0,0 +1,324 @@ +{ + config, + lib, + pkgs, + ... +}: + +let + inherit (lib) + mkEnableOption + mkIf + mkOption + mkPackageOption + types + ; + + cfg = config.services.tlsrpt; + + format = pkgs.formats.ini { }; + dropNullValues = lib.filterAttrsRecursive (_: value: value != null); + + commonServiceSettings = { + DynamicUser = true; + User = "tlsrpt"; + Restart = "always"; + StateDirectory = "tlsrpt"; + StateDirectoryMode = "0700"; + + # Hardening + CapabilityBoundingSet = [ "" ]; + LockPersonality = true; + MemoryDenyWriteExecute = true; + PrivateDevices = true; + PrivateUsers = false; + ProcSubset = "pid"; + ProtectControlGroups = true; + ProtectClock = true; + ProtectHome = true; + ProtectHostname = true; + ProtectKernelLogs = true; + ProtectKernelModules = true; + ProtectKernelTunables = true; + ProtectProc = "noaccess"; + RestrictNamespaces = true; + RestrictRealtime = true; + SystemCallArchitectures = "native"; + SystemCallFilter = [ + "@system-service" + "~@privileged @resources" + ]; + }; + +in + +{ + options.services.tlsrpt = { + enable = mkEnableOption "the TLSRPT services"; + + package = mkPackageOption pkgs "tlsrpt-reporter" { }; + + collectd = { + settings = mkOption { + type = types.submodule { + freeformType = format.type; + options = { + storage = mkOption { + type = types.str; + default = "sqlite:///var/lib/tlsrpt/collectd.sqlite"; + description = '' + Storage backend definition. + ''; + }; + + socketname = mkOption { + type = types.path; + default = "/run/tlsrpt/collectd.sock"; + description = '' + Path at which the UNIX socket will be created. + ''; + }; + + socketmode = mkOption { + type = types.str; + default = "0220"; + description = '' + Permissions on the UNIX socket. + ''; + }; + + log_level = mkOption { + type = types.enum [ + "debug" + "info" + "warning" + "error" + "critical" + ]; + default = "info"; + description = '' + Level of log messages to emit. + ''; + }; + }; + }; + default = { }; + description = '' + Flags from {manpage}`tlsrpt-collectd(1)` as key-value pairs. + ''; + }; + + extraFlags = mkOption { + type = with types; listOf str; + default = [ ]; + description = '' + List of extra flags to pass to the tlsrpt-reportd executable. + + See {manpage}`tlsrpt-collectd(1)` for possible flags. + ''; + }; + + configurePostfix = mkOption { + type = types.bool; + default = true; + description = '' + Whether to modify the local Postfix service to grant access to the collectd socket. + ''; + }; + }; + + fetcher = { + settings = mkOption { + type = types.submodule { + freeformType = format.type; + options = { + storage = mkOption { + type = types.str; + default = config.services.tlsrpt.collectd.settings.storage; + defaultText = lib.literalExpression '' + config.services.tlsrpt.collectd.settings.storage + ''; + description = '' + Path to the collectd sqlite database. + ''; + }; + + log_level = mkOption { + type = types.enum [ + "debug" + "info" + "warning" + "error" + "critical" + ]; + default = "info"; + description = '' + Level of log messages to emit. + ''; + }; + }; + }; + default = { }; + description = '' + Flags from {manpage}`tlsrpt-fetcher(1)` as key-value pairs. + ''; + }; + }; + + reportd = { + settings = mkOption { + type = types.submodule { + freeformType = format.type; + options = { + dbname = mkOption { + type = types.str; + default = "/var/lib/tlsrpt/reportd.sqlite"; + description = '' + Path to the sqlite database. + ''; + }; + + fetchers = mkOption { + type = types.str; + default = lib.getExe' cfg.package "tlsrpt-fetcher"; + defaultText = lib.literalExpression '' + lib.getExe' cfg.package "tlsrpt-fetcher" + ''; + description = '' + Comma-separated list of fetcher programs that retrieve collectd data. + ''; + }; + + log_level = mkOption { + type = types.enum [ + "debug" + "info" + "warning" + "error" + "critical" + ]; + default = "info"; + description = '' + Level of log messages to emit. + ''; + }; + + organization_name = mkOption { + type = types.str; + example = "ACME Corp."; + description = '' + Name of the organization sending out the reports. + ''; + }; + + contact_info = mkOption { + type = types.str; + example = "smtp-tls-reporting@example.com"; + description = '' + Contact information embedded into the reports. + ''; + }; + + sender_address = mkOption { + type = types.str; + example = "noreply@example.com"; + description = '' + Sender address used for reports. + ''; + }; + + sendmail_script = mkOption { + type = with types; nullOr str; + default = if config.services.postfix.enable then "sendmail" else null; + defaultText = lib.literalExpression '' + if any [ config.services.postfix.enable ] then "sendmail" else null + ''; + description = '' + Path to a sendmail-compatible executable for delivery reports. + ''; + }; + }; + }; + default = { }; + description = '' + Flags from {manpage}`tlsrpt-reportd(1)` as key-value pairs. + ''; + }; + + extraFlags = mkOption { + type = with types; listOf str; + default = [ ]; + description = '' + List of extra flags to pass to the tlsrpt-reportd executable. + + See {manpage}`tlsrpt-report(1)` for possible flags. + ''; + }; + }; + }; + + config = mkIf cfg.enable { + environment.etc = { + "tlsrpt/collectd.cfg".source = format.generate "tlsrpt-collectd.cfg" { + tlsrpt_collectd = dropNullValues cfg.collectd.settings; + }; + "tlsrpt/fetcher.cfg".source = format.generate "tlsrpt-fetcher.cfg" { + tlsrpt_fetcher = dropNullValues cfg.fetcher.settings; + }; + "tlsrpt/reportd.cfg".source = format.generate "tlsrpt-reportd.cfg" { + tlsrpt_reportd = dropNullValues cfg.reportd.settings; + }; + }; + + systemd.services.postfix.serviceConfig.SupplementaryGroups = mkIf ( + config.services.postfix.enable && cfg.collectd.configurePostfix + ) [ "tlsrpt" ]; + + systemd.services.tlsrpt-collectd = { + description = "TLSRPT datagram collector"; + documentation = [ "man:tlsrpt-collectd(1)" ]; + + wantedBy = [ "multi-user.target" ]; + + restartTriggers = [ "/etc/tlsrpt/collectd.cfg" ]; + + serviceConfig = commonServiceSettings // { + ExecStart = toString ( + [ + (lib.getExe' cfg.package "tlsrpt-collectd") + ] + ++ cfg.collectd.extraFlags + ); + IPAddressDeny = "any"; + PrivateNetwork = true; + RestrictAddressFamilies = [ "AF_UNIX" ]; + RuntimeDirectory = "tlsrpt"; + RuntimeDirectoryMode = "0750"; + UMask = "0157"; + }; + }; + + systemd.services.tlsrpt-reportd = { + description = "TLSRPT report generator"; + documentation = [ "man:tlsrpt-reportd(1)" ]; + + wantedBy = [ "multi-user.target" ]; + + restartTriggers = [ "/etc/tlsrpt/reportd.cfg" ]; + + serviceConfig = commonServiceSettings // { + ExecStart = toString ( + [ + (lib.getExe' cfg.package "tlsrpt-reportd") + ] + ++ cfg.reportd.extraFlags + ); + RestrictAddressFamilies = [ + "AF_INET" + "AF_INET6" + ]; + UMask = "0077"; + }; + }; + }; +} diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix index 552fb324db76..14c3e0e54517 100644 --- a/nixos/tests/all-tests.nix +++ b/nixos/tests/all-tests.nix @@ -1398,6 +1398,7 @@ in tinydns = runTest ./tinydns.nix; tinyproxy = runTest ./tinyproxy.nix; tinywl = runTest ./tinywl.nix; + tlsrpt = runTest ./tlsrpt.nix; tmate-ssh-server = runTest ./tmate-ssh-server.nix; tomcat = runTest ./tomcat.nix; tor = runTest ./tor.nix; @@ -1464,7 +1465,7 @@ in vault-postgresql = runTest ./vault-postgresql.nix; vaultwarden = discoverTests (import ./vaultwarden.nix); vdirsyncer = runTest ./vdirsyncer.nix; - vector = handleTest ./vector { }; + vector = import ./vector { inherit runTest; }; velocity = runTest ./velocity.nix; vengi-tools = runTest ./vengi-tools.nix; victoriametrics = handleTest ./victoriametrics { }; diff --git a/nixos/tests/tlsrpt.nix b/nixos/tests/tlsrpt.nix new file mode 100644 index 000000000000..ccbb5c25e0fc --- /dev/null +++ b/nixos/tests/tlsrpt.nix @@ -0,0 +1,41 @@ +{ + pkgs, + ... +}: + +{ + name = "tlsrpt"; + + meta = { + inherit (pkgs.tlsrpt-reporter.meta) maintainers; + }; + + nodes.machine = { + services.tlsrpt = { + enable = true; + reportd.settings = { + organization_name = "NixOS Testers United"; + contact_info = "smtp-tls-report@localhost"; + sender_address = "noreply@localhost"; + }; + }; + + # To test the postfix integration + services.postfix.enable = true; + }; + + testScript = '' + machine.wait_for_unit("tlsrpt-collectd.service") + machine.wait_for_unit("tlsrpt-reportd.service") + + machine.wait_for_file("/run/tlsrpt/collectd.sock") + machine.wait_until_succeeds("journalctl -o cat -u tlsrpt-collectd | grep -Pq 'Database .* setup finished'") + machine.wait_until_succeeds("journalctl -o cat -u tlsrpt-reportd | grep -Pq 'Database .* setup finished'") + + # Enabling postfix should put sendmail as the sendmail setting + machine.succeed("grep -q sendmail_script=sendmail /etc/tlsrpt/reportd.cfg") + machine.succeed("systemctl show --property SupplementaryGroups postfix.service | grep tlsrpt") + + machine.log(machine.succeed("systemd-analyze security tlsrpt-collectd.service tlsrpt-reportd.service | grep -v ✓")) + ''; +} diff --git a/nixos/tests/vector/api.nix b/nixos/tests/vector/api.nix index e67097ee9428..84a140fda9e3 100644 --- a/nixos/tests/vector/api.nix +++ b/nixos/tests/vector/api.nix @@ -1,45 +1,43 @@ -import ../make-test-python.nix ( - { lib, pkgs, ... }: +{ lib, pkgs, ... }: - { - name = "vector-api"; - meta.maintainers = [ pkgs.lib.maintainers.happysalada ]; +{ + name = "vector-api"; + meta.maintainers = [ pkgs.lib.maintainers.happysalada ]; - nodes.machineapi = - { config, pkgs, ... }: - { - services.vector = { - enable = true; - journaldAccess = false; - settings = { - api.enabled = true; + nodes.machineapi = + { config, pkgs, ... }: + { + services.vector = { + enable = true; + journaldAccess = false; + settings = { + api.enabled = true; - sources = { - demo_logs = { - type = "demo_logs"; - format = "json"; - }; + sources = { + demo_logs = { + type = "demo_logs"; + format = "json"; }; + }; - sinks = { - file = { - type = "file"; - inputs = [ "demo_logs" ]; - path = "/var/lib/vector/logs.log"; - encoding = { - codec = "json"; - }; + sinks = { + file = { + type = "file"; + inputs = [ "demo_logs" ]; + path = "/var/lib/vector/logs.log"; + encoding = { + codec = "json"; }; }; }; }; }; + }; - testScript = '' - machineapi.wait_for_unit("vector") - machineapi.wait_for_open_port(8686) - machineapi.succeed("journalctl -o cat -u vector.service | grep 'API server running'") - machineapi.wait_until_succeeds("curl -sSf http://localhost:8686/health") - ''; - } -) + testScript = '' + machineapi.wait_for_unit("vector") + machineapi.wait_for_open_port(8686) + machineapi.succeed("journalctl -o cat -u vector.service | grep 'API server running'") + machineapi.wait_until_succeeds("curl -sSf http://localhost:8686/health") + ''; +} diff --git a/nixos/tests/vector/caddy-clickhouse.nix b/nixos/tests/vector/caddy-clickhouse.nix new file mode 100644 index 000000000000..c1b478f3648c --- /dev/null +++ b/nixos/tests/vector/caddy-clickhouse.nix @@ -0,0 +1,206 @@ +{ lib, pkgs, ... }: + +{ + name = "vector-caddy-clickhouse"; + meta.maintainers = [ pkgs.lib.maintainers.happysalada ]; + + nodes = { + caddy = + { config, pkgs, ... }: + { + networking.firewall.allowedTCPPorts = [ 80 ]; + + services.caddy = { + enable = true; + virtualHosts = { + "http://caddy" = { + extraConfig = '' + encode gzip + + file_server + root /srv + ''; + logFormat = " + output file ${config.services.caddy.logDir}/access-caddy.log { + mode 0640 + } + "; + }; + }; + }; + + systemd.services.vector.serviceConfig = { + SupplementaryGroups = [ "caddy" ]; + }; + + services.vector = { + enable = true; + + settings = { + sources = { + caddy-log = { + type = "file"; + include = [ "/var/log/caddy/*.log" ]; + }; + }; + + transforms = { + caddy_logs_timestamp = { + type = "remap"; + inputs = [ "caddy-log" ]; + source = '' + .tmp_timestamp, err = parse_json!(.message).ts * 1000000 + + if err != null { + log("Unable to parse ts value: " + err, level: "error") + } else { + .timestamp = from_unix_timestamp!(to_int!(.tmp_timestamp), unit: "microseconds") + } + + del(.tmp_timestamp) + ''; + }; + }; + + sinks = { + vector_sink = { + type = "vector"; + inputs = [ "caddy_logs_timestamp" ]; + address = "clickhouse:6000"; + }; + }; + }; + }; + }; + + client = + { config, pkgs, ... }: + { + environment.systemPackages = [ pkgs.curl ]; + }; + + clickhouse = + { config, pkgs, ... }: + { + virtualisation.memorySize = 4096; + + networking.firewall.allowedTCPPorts = [ 6000 ]; + + services.vector = { + enable = true; + + settings = { + sources = { + vector_source = { + type = "vector"; + address = "[::]:6000"; + }; + }; + + sinks = { + clickhouse = { + type = "clickhouse"; + inputs = [ + "vector_source" + ]; + endpoint = "http://localhost:8123"; + database = "caddy"; + table = "access_logs"; + date_time_best_effort = true; + skip_unknown_fields = true; + }; + }; + }; + + }; + + services.clickhouse = { + enable = true; + }; + }; + }; + + testScript = + let + # work around quote/substitution complexity by Nix, Perl, bash and SQL. + databaseDDL = pkgs.writeText "database.sql" "CREATE DATABASE IF NOT EXISTS caddy"; + + tableDDL = pkgs.writeText "table.sql" '' + CREATE TABLE IF NOT EXISTS caddy.access_logs ( + timestamp DateTime64(6), + host LowCardinality(String), + message String, + ) + ENGINE = MergeTree() + ORDER BY timestamp + PARTITION BY toYYYYMM(timestamp) + ''; + + tableViewBase = pkgs.writeText "table-view-base.sql" '' + CREATE TABLE IF NOT EXISTS caddy.access_logs_view_base ( + timestamp DateTime64(6), + host LowCardinality(String), + request JSON, + status UInt16, + ) + ENGINE = MergeTree() + ORDER BY timestamp + PARTITION BY toYYYYMM(timestamp) + ''; + + tableView = pkgs.writeText "table-view.sql" '' + CREATE MATERIALIZED VIEW IF NOT EXISTS caddy.access_logs_view TO caddy.access_logs_view_base + AS SELECT + timestamp, + host, + simpleJSONExtractRaw(message, 'request') AS request, + simpleJSONExtractRaw(message, 'status') AS status + FROM caddy.access_logs; + ''; + + selectQuery = pkgs.writeText "select.sql" '' + SELECT + timestamp, + request.host, + request.remote_ip, + request.proto, + request.method, + request.uri, + status + FROM caddy.access_logs_view_base + WHERE request.uri LIKE '%test-uri%' + FORMAT Pretty + ''; + in + '' + clickhouse.wait_for_unit("clickhouse") + clickhouse.wait_for_unit("vector") + clickhouse.wait_for_open_port(6000) + clickhouse.wait_for_open_port(8123) + + clickhouse.succeed( + "cat ${databaseDDL} | clickhouse-client", + "cat ${tableDDL} | clickhouse-client", + "cat ${tableViewBase} | clickhouse-client", + "cat ${tableView} | clickhouse-client", + ) + + caddy.wait_for_unit("caddy") + caddy.wait_for_open_port(80) + caddy.wait_for_unit("vector") + caddy.wait_until_succeeds( + "journalctl -o cat -u vector.service | grep 'Vector has started'" + ) + + client.systemctl("start network-online.target") + client.wait_until_succeeds("curl http://caddy/test-uri") + + caddy.wait_until_succeeds( + "journalctl -o cat -u vector.service | grep 'Found new file to watch. file=/var/log/caddy/access-caddy.log'" + ) + + clickhouse.wait_until_succeeds( + "cat ${selectQuery} | clickhouse-client | grep test-uri" + ) + ''; +} diff --git a/nixos/tests/vector/default.nix b/nixos/tests/vector/default.nix index 764689d5cb03..360fbd9a716f 100644 --- a/nixos/tests/vector/default.nix +++ b/nixos/tests/vector/default.nix @@ -1,14 +1,11 @@ -{ - system ? builtins.currentSystem, - config ? { }, - pkgs ? import ../../.. { inherit system config; }, -}: +{ runTest }: { - file-sink = import ./file-sink.nix { inherit system pkgs; }; - api = import ./api.nix { inherit system pkgs; }; - dnstap = import ./dnstap.nix { inherit system pkgs; }; - journald-clickhouse = import ./journald-clickhouse.nix { inherit system pkgs; }; - nginx-clickhouse = import ./nginx-clickhouse.nix { inherit system pkgs; }; - syslog-quickwit = import ./syslog-quickwit.nix { inherit system pkgs; }; + file-sink = runTest ./file-sink.nix; + api = runTest ./api.nix; + caddy-clickhouse = runTest ./caddy-clickhouse.nix; + dnstap = runTest ./dnstap.nix; + journald-clickhouse = runTest ./journald-clickhouse.nix; + nginx-clickhouse = runTest ./nginx-clickhouse.nix; + syslog-quickwit = runTest ./syslog-quickwit.nix; } diff --git a/nixos/tests/vector/dnstap.nix b/nixos/tests/vector/dnstap.nix index ec20ce1f6a1d..02d40907f8af 100644 --- a/nixos/tests/vector/dnstap.nix +++ b/nixos/tests/vector/dnstap.nix @@ -1,106 +1,210 @@ -import ../make-test-python.nix ( - { lib, pkgs, ... }: +{ lib, pkgs, ... }: - let - dnstapSocket = "/var/run/vector/dnstap.sock"; - in - { - name = "vector-dnstap"; - meta.maintainers = [ pkgs.lib.maintainers.happysalada ]; +let + dnstapSocket = "/var/run/vector/dnstap.sock"; +in +{ + name = "vector-dnstap"; + meta.maintainers = [ pkgs.lib.maintainers.happysalada ]; - nodes = { - unbound = - { config, pkgs, ... }: - { - networking.firewall.allowedUDPPorts = [ 53 ]; + nodes = { + clickhouse = + { config, pkgs, ... }: + { + networking.firewall.allowedTCPPorts = [ 6000 ]; - services.vector = { - enable = true; + services.vector = { + enable = true; - settings = { - sources = { - dnstap = { - type = "dnstap"; - multithreaded = true; - mode = "unix"; - lowercase_hostnames = true; - socket_file_mode = 504; - socket_path = "${dnstapSocket}"; - }; + settings = { + sources = { + vector_dnstap_source = { + type = "vector"; + address = "[::]:6000"; }; + }; - sinks = { - file = { - type = "file"; - inputs = [ "dnstap" ]; - path = "/var/lib/vector/logs.log"; - encoding = { - codec = "json"; - }; - }; + sinks = { + clickhouse = { + type = "clickhouse"; + inputs = [ + "vector_dnstap_source" + ]; + endpoint = "http://localhost:8123"; + database = "dnstap"; + table = "records"; + date_time_best_effort = true; }; }; }; + }; - systemd.services.vector.serviceConfig = { - RuntimeDirectory = "vector"; - RuntimeDirectoryMode = "0770"; - }; + services.clickhouse.enable = true; + }; - services.unbound = { - enable = true; - enableRootTrustAnchor = false; - package = pkgs.unbound-full; - settings = { - server = { - interface = [ - "0.0.0.0" - "::" - ]; - access-control = [ - "192.168.0.0/24 allow" - "::/0 allow" - ]; + unbound = + { config, pkgs, ... }: + { + networking.firewall.allowedUDPPorts = [ 53 ]; - domain-insecure = "local"; - private-domain = "local"; - - local-zone = "local. static"; - local-data = [ - ''"test.local. 10800 IN A 192.168.123.5"'' - ]; - }; + services.vector = { + enable = true; + settings = { + sources = { dnstap = { - dnstap-enable = "yes"; - dnstap-socket-path = "${dnstapSocket}"; - dnstap-send-identity = "yes"; - dnstap-send-version = "yes"; - dnstap-log-client-query-messages = "yes"; - dnstap-log-client-response-messages = "yes"; + type = "dnstap"; + multithreaded = true; + mode = "unix"; + lowercase_hostnames = true; + socket_file_mode = 504; + socket_path = "${dnstapSocket}"; + }; + }; + + sinks = { + file = { + type = "file"; + inputs = [ "dnstap" ]; + path = "/var/lib/vector/logs.log"; + encoding = { + codec = "json"; + }; + }; + + vector_dnstap_sink = { + type = "vector"; + inputs = [ "dnstap" ]; + address = "clickhouse:6000"; }; }; }; + }; - systemd.services.unbound = { - after = [ "vector.service" ]; - wants = [ "vector.service" ]; - serviceConfig = { - # DNSTAP access - ReadWritePaths = [ "/var/run/vector" ]; - SupplementaryGroups = [ "vector" ]; + systemd.services.vector.serviceConfig = { + RuntimeDirectory = "vector"; + RuntimeDirectoryMode = "0770"; + }; + + services.unbound = { + enable = true; + enableRootTrustAnchor = false; + package = pkgs.unbound-full; + settings = { + server = { + interface = [ + "0.0.0.0" + "::" + ]; + access-control = [ + "192.168.0.0/24 allow" + "::/0 allow" + ]; + + domain-insecure = "local"; + private-domain = "local"; + + local-zone = "local. static"; + local-data = [ + ''"test.local. 10800 IN A 192.168.123.5"'' + ]; + }; + + dnstap = { + dnstap-enable = "yes"; + dnstap-socket-path = "${dnstapSocket}"; + dnstap-send-identity = "yes"; + dnstap-send-version = "yes"; + dnstap-log-client-query-messages = "yes"; + dnstap-log-client-response-messages = "yes"; }; }; }; - dnsclient = - { config, pkgs, ... }: - { - environment.systemPackages = [ pkgs.dig ]; + systemd.services.unbound = { + after = [ "vector.service" ]; + wants = [ "vector.service" ]; + serviceConfig = { + # DNSTAP access + ReadWritePaths = [ "/var/run/vector" ]; + SupplementaryGroups = [ "vector" ]; + }; }; - }; + }; + + dnsclient = + { config, pkgs, ... }: + { + environment.systemPackages = [ pkgs.dig ]; + }; + }; + + testScript = + let + # work around quote/substitution complexity by Nix, Perl, bash and SQL. + databaseDDL = pkgs.writeText "database.sql" "CREATE DATABASE IF NOT EXISTS dnstap"; + + tableDDL = pkgs.writeText "table.sql" '' + CREATE TABLE IF NOT EXISTS dnstap.records ( + timestamp DateTime64(6), + dataType LowCardinality(String), + dataTypeId UInt8, + messageType LowCardinality(String), + messageTypeId UInt8, + requestData Nullable(JSON), + responseData Nullable(JSON), + responsePort UInt16, + serverId LowCardinality(String), + serverVersion LowCardinality(String), + socketFamily LowCardinality(String), + socketProtocol LowCardinality(String), + sourceAddress String, + sourcePort UInt16, + ) + ENGINE = MergeTree() + ORDER BY (serverId, timestamp) + PARTITION BY toYYYYMM(timestamp) + ''; + + tableView = pkgs.writeText "view.sql" '' + CREATE MATERIALIZED VIEW dnstap.domains_view ( + timestamp DateTime64(6), + serverId LowCardinality(String), + domain String, + record_type LowCardinality(String) + ) + ENGINE = MergeTree() + PARTITION BY toYYYYMM(timestamp) + ORDER BY (serverId, timestamp) + POPULATE AS + SELECT + timestamp, + serverId, + JSONExtractString(requestData.question[1]::String, 'domainName') as domain, + JSONExtractString(requestData.question[1]::String, 'questionType') as record_type + FROM dnstap.records + WHERE messageTypeId = 5 # ClientQuery + ''; + + selectQuery = pkgs.writeText "select.sql" '' + SELECT + domain, + count(domain) + FROM dnstap.domains_view + GROUP BY domain + ''; + in + '' + clickhouse.wait_for_unit("clickhouse") + clickhouse.wait_for_open_port(6000) + clickhouse.wait_for_open_port(8123) + + clickhouse.succeed( + "cat ${databaseDDL} | clickhouse-client", + "cat ${tableDDL} | clickhouse-client", + "cat ${tableView} | clickhouse-client", + ) - testScript = '' unbound.wait_for_unit("unbound") unbound.wait_for_unit("vector") @@ -128,6 +232,9 @@ import ../make-test-python.nix ( unbound.wait_until_succeeds( "grep ClientResponse /var/lib/vector/logs.log | grep '\"domainName\":\"test.local.\"' | grep '\"rData\":\"192.168.123.5\"'" ) + + clickhouse.log(clickhouse.wait_until_succeeds( + "cat ${selectQuery} | clickhouse-client | grep 'test.local.'" + )) ''; - } -) +} diff --git a/nixos/tests/vector/file-sink.nix b/nixos/tests/vector/file-sink.nix index 999e8d58443c..13389717dcfc 100644 --- a/nixos/tests/vector/file-sink.nix +++ b/nixos/tests/vector/file-sink.nix @@ -1,58 +1,56 @@ -import ../make-test-python.nix ( - { lib, pkgs, ... }: +{ lib, pkgs, ... }: - { - name = "vector-test1"; - meta.maintainers = [ pkgs.lib.maintainers.happysalada ]; +{ + name = "vector-test1"; + meta.maintainers = [ pkgs.lib.maintainers.happysalada ]; - nodes.machine = - { config, pkgs, ... }: - { - services.vector = { - enable = true; - journaldAccess = true; - settings = { - sources = { - journald.type = "journald"; + nodes.machine = + { config, pkgs, ... }: + { + services.vector = { + enable = true; + journaldAccess = true; + settings = { + sources = { + journald.type = "journald"; - vector_metrics.type = "internal_metrics"; + vector_metrics.type = "internal_metrics"; - vector_logs.type = "internal_logs"; + vector_logs.type = "internal_logs"; + }; + + sinks = { + file = { + type = "file"; + inputs = [ + "journald" + "vector_logs" + ]; + path = "/var/lib/vector/logs.log"; + encoding = { + codec = "json"; + }; }; - sinks = { - file = { - type = "file"; - inputs = [ - "journald" - "vector_logs" - ]; - path = "/var/lib/vector/logs.log"; - encoding = { - codec = "json"; - }; - }; - - prometheus_exporter = { - type = "prometheus_exporter"; - inputs = [ "vector_metrics" ]; - address = "[::]:9598"; - }; + prometheus_exporter = { + type = "prometheus_exporter"; + inputs = [ "vector_metrics" ]; + address = "[::]:9598"; }; }; }; }; + }; - # ensure vector is forwarding the messages appropriately - testScript = '' - machine.wait_for_unit("vector.service") - machine.wait_for_open_port(9598) - machine.wait_until_succeeds("journalctl -o cat -u vector.service | grep 'version=\"${pkgs.vector.version}\"'") - machine.wait_until_succeeds("journalctl -o cat -u vector.service | grep 'API is disabled'") - machine.wait_until_succeeds("curl -sSf http://localhost:9598/metrics | grep vector_build_info") - machine.wait_until_succeeds("curl -sSf http://localhost:9598/metrics | grep vector_component_received_bytes_total | grep journald") - machine.wait_until_succeeds("curl -sSf http://localhost:9598/metrics | grep vector_utilization | grep prometheus_exporter") - machine.wait_for_file("/var/lib/vector/logs.log") - ''; - } -) + # ensure vector is forwarding the messages appropriately + testScript = '' + machine.wait_for_unit("vector.service") + machine.wait_for_open_port(9598) + machine.wait_until_succeeds("journalctl -o cat -u vector.service | grep 'version=\"${pkgs.vector.version}\"'") + machine.wait_until_succeeds("journalctl -o cat -u vector.service | grep 'API is disabled'") + machine.wait_until_succeeds("curl -sSf http://localhost:9598/metrics | grep vector_build_info") + machine.wait_until_succeeds("curl -sSf http://localhost:9598/metrics | grep vector_component_received_bytes_total | grep journald") + machine.wait_until_succeeds("curl -sSf http://localhost:9598/metrics | grep vector_utilization | grep prometheus_exporter") + machine.wait_for_file("/var/lib/vector/logs.log") + ''; +} diff --git a/nixos/tests/vector/journald-clickhouse.nix b/nixos/tests/vector/journald-clickhouse.nix index 6979ed5a9c87..b0ca516306ea 100644 --- a/nixos/tests/vector/journald-clickhouse.nix +++ b/nixos/tests/vector/journald-clickhouse.nix @@ -1,157 +1,155 @@ -import ../make-test-python.nix ( - { lib, pkgs, ... }: - let - # Take the original journald message and create a new payload which only - # contains the relevant fields - these must match the database columns. - journalVrlRemapTransform = { - journald_remap = { - inputs = [ "journald" ]; - type = "remap"; - source = '' - m = {} - m.app = .SYSLOG_IDENTIFIER - m.host = .host - m.severity = to_int(.PRIORITY) ?? 0 - m.level = to_syslog_level(m.severity) ?? "" - m.message = strip_ansi_escape_codes!(.message) - m.timestamp = .timestamp - m.uid = to_int(._UID) ?? 0 - m.pid = to_int(._PID) ?? 0 - . = [m] - ''; - }; - }; - in - { - name = "vector-journald-clickhouse"; - meta.maintainers = [ pkgs.lib.maintainers.happysalada ]; - - nodes = { - clickhouse = - { config, pkgs, ... }: - { - virtualisation.diskSize = 5 * 1024; - virtualisation.memorySize = 4096; - - networking.firewall.allowedTCPPorts = [ 6000 ]; - - services.vector = { - enable = true; - journaldAccess = true; - - settings = { - sources = { - journald = { - type = "journald"; - }; - - vector_source = { - type = "vector"; - address = "[::]:6000"; - }; - }; - - transforms = journalVrlRemapTransform; - - sinks = { - clickhouse = { - type = "clickhouse"; - inputs = [ - "journald_remap" - "vector_source" - ]; - endpoint = "http://localhost:8123"; - database = "journald"; - table = "logs"; - date_time_best_effort = true; - }; - }; - }; - - }; - - services.clickhouse = { - enable = true; - }; - }; - - vector = - { config, pkgs, ... }: - { - services.vector = { - enable = true; - journaldAccess = true; - - settings = { - sources = { - journald = { - type = "journald"; - }; - }; - - transforms = journalVrlRemapTransform; - - sinks = { - vector_sink = { - type = "vector"; - inputs = [ "journald_remap" ]; - address = "clickhouse:6000"; - }; - }; - }; - }; - }; - }; - - testScript = - let - # work around quote/substitution complexity by Nix, Perl, bash and SQL. - databaseDDL = pkgs.writeText "database.sql" "CREATE DATABASE IF NOT EXISTS journald"; - - # https://clickhouse.com/blog/storing-log-data-in-clickhouse-fluent-bit-vector-open-telemetry - tableDDL = pkgs.writeText "table.sql" '' - CREATE TABLE IF NOT EXISTS journald.logs ( - timestamp DateTime64(6), - app LowCardinality(String), - host LowCardinality(String), - level LowCardinality(String), - severity UInt8, - message String, - uid UInt16, - pid UInt32, - ) - ENGINE = MergeTree() - ORDER BY (host, app, timestamp) - PARTITION BY toYYYYMM(timestamp) - ''; - - selectQuery = pkgs.writeText "select.sql" '' - SELECT COUNT(host) FROM journald.logs - WHERE message LIKE '%Vector has started%' - ''; - in - '' - clickhouse.wait_for_unit("clickhouse") - clickhouse.wait_for_open_port(6000) - clickhouse.wait_for_open_port(8123) - - clickhouse.succeed( - "cat ${databaseDDL} | clickhouse-client" - ) - - clickhouse.succeed( - "cat ${tableDDL} | clickhouse-client" - ) - - for machine in clickhouse, vector: - machine.wait_for_unit("vector") - machine.wait_until_succeeds( - "journalctl -o cat -u vector.service | grep 'Vector has started'" - ) - - clickhouse.wait_until_succeeds( - "cat ${selectQuery} | clickhouse-client | grep 2" - ) +{ lib, pkgs, ... }: +let + # Take the original journald message and create a new payload which only + # contains the relevant fields - these must match the database columns. + journalVrlRemapTransform = { + journald_remap = { + inputs = [ "journald" ]; + type = "remap"; + source = '' + m = {} + m.app = .SYSLOG_IDENTIFIER + m.host = .host + m.severity = to_int(.PRIORITY) ?? 0 + m.level = to_syslog_level(m.severity) ?? "" + m.message = strip_ansi_escape_codes!(.message) + m.timestamp = .timestamp + m.uid = to_int(._UID) ?? 0 + m.pid = to_int(._PID) ?? 0 + . = [m] ''; - } -) + }; + }; +in +{ + name = "vector-journald-clickhouse"; + meta.maintainers = [ pkgs.lib.maintainers.happysalada ]; + + nodes = { + clickhouse = + { config, pkgs, ... }: + { + virtualisation.diskSize = 5 * 1024; + virtualisation.memorySize = 4096; + + networking.firewall.allowedTCPPorts = [ 6000 ]; + + services.vector = { + enable = true; + journaldAccess = true; + + settings = { + sources = { + journald = { + type = "journald"; + }; + + vector_source = { + type = "vector"; + address = "[::]:6000"; + }; + }; + + transforms = journalVrlRemapTransform; + + sinks = { + clickhouse = { + type = "clickhouse"; + inputs = [ + "journald_remap" + "vector_source" + ]; + endpoint = "http://localhost:8123"; + database = "journald"; + table = "logs"; + date_time_best_effort = true; + }; + }; + }; + + }; + + services.clickhouse = { + enable = true; + }; + }; + + vector = + { config, pkgs, ... }: + { + services.vector = { + enable = true; + journaldAccess = true; + + settings = { + sources = { + journald = { + type = "journald"; + }; + }; + + transforms = journalVrlRemapTransform; + + sinks = { + vector_sink = { + type = "vector"; + inputs = [ "journald_remap" ]; + address = "clickhouse:6000"; + }; + }; + }; + }; + }; + }; + + testScript = + let + # work around quote/substitution complexity by Nix, Perl, bash and SQL. + databaseDDL = pkgs.writeText "database.sql" "CREATE DATABASE IF NOT EXISTS journald"; + + # https://clickhouse.com/blog/storing-log-data-in-clickhouse-fluent-bit-vector-open-telemetry + tableDDL = pkgs.writeText "table.sql" '' + CREATE TABLE IF NOT EXISTS journald.logs ( + timestamp DateTime64(6), + app LowCardinality(String), + host LowCardinality(String), + level LowCardinality(String), + severity UInt8, + message String, + uid UInt16, + pid UInt32, + ) + ENGINE = MergeTree() + ORDER BY (host, app, timestamp) + PARTITION BY toYYYYMM(timestamp) + ''; + + selectQuery = pkgs.writeText "select.sql" '' + SELECT COUNT(host) FROM journald.logs + WHERE message LIKE '%Vector has started%' + ''; + in + '' + clickhouse.wait_for_unit("clickhouse") + clickhouse.wait_for_open_port(6000) + clickhouse.wait_for_open_port(8123) + + clickhouse.succeed( + "cat ${databaseDDL} | clickhouse-client" + ) + + clickhouse.succeed( + "cat ${tableDDL} | clickhouse-client" + ) + + for machine in clickhouse, vector: + machine.wait_for_unit("vector") + machine.wait_until_succeeds( + "journalctl -o cat -u vector.service | grep 'Vector has started'" + ) + + clickhouse.wait_until_succeeds( + "cat ${selectQuery} | clickhouse-client | grep 2" + ) + ''; +} diff --git a/nixos/tests/vector/nginx-clickhouse.nix b/nixos/tests/vector/nginx-clickhouse.nix index 9470d78d968b..38dc0c7991c9 100644 --- a/nixos/tests/vector/nginx-clickhouse.nix +++ b/nixos/tests/vector/nginx-clickhouse.nix @@ -1,174 +1,172 @@ -import ../make-test-python.nix ( - { lib, pkgs, ... }: +{ lib, pkgs, ... }: - { - name = "vector-nginx-clickhouse"; - meta.maintainers = [ pkgs.lib.maintainers.happysalada ]; +{ + name = "vector-nginx-clickhouse"; + meta.maintainers = [ pkgs.lib.maintainers.happysalada ]; - nodes = { - clickhouse = - { config, pkgs, ... }: - { - virtualisation.memorySize = 4096; + nodes = { + clickhouse = + { config, pkgs, ... }: + { + virtualisation.memorySize = 4096; - # Clickhouse module can't listen on a non-loopback IP. - networking.firewall.allowedTCPPorts = [ 6000 ]; - services.clickhouse.enable = true; + # Clickhouse module can't listen on a non-loopback IP. + networking.firewall.allowedTCPPorts = [ 6000 ]; + services.clickhouse.enable = true; - # Exercise Vector sink->source for now. - services.vector = { - enable = true; + # Exercise Vector sink->source for now. + services.vector = { + enable = true; - settings = { - sources = { - vector_source = { - type = "vector"; - address = "[::]:6000"; - }; + settings = { + sources = { + vector_source = { + type = "vector"; + address = "[::]:6000"; }; + }; - sinks = { - clickhouse = { - type = "clickhouse"; - inputs = [ "vector_source" ]; - endpoint = "http://localhost:8123"; - database = "nginxdb"; - table = "access_logs"; - skip_unknown_fields = true; - }; + sinks = { + clickhouse = { + type = "clickhouse"; + inputs = [ "vector_source" ]; + endpoint = "http://localhost:8123"; + database = "nginxdb"; + table = "access_logs"; + skip_unknown_fields = true; + }; + }; + }; + }; + }; + + nginx = + { config, pkgs, ... }: + { + services.nginx = { + enable = true; + virtualHosts.localhost = { }; + }; + + services.vector = { + enable = true; + + settings = { + sources = { + nginx_logs = { + type = "file"; + include = [ "/var/log/nginx/access.log" ]; + read_from = "end"; + }; + }; + + sinks = { + vector_sink = { + type = "vector"; + inputs = [ "nginx_logs" ]; + address = "clickhouse:6000"; }; }; }; }; - nginx = - { config, pkgs, ... }: - { - services.nginx = { - enable = true; - virtualHosts.localhost = { }; - }; - - services.vector = { - enable = true; - - settings = { - sources = { - nginx_logs = { - type = "file"; - include = [ "/var/log/nginx/access.log" ]; - read_from = "end"; - }; - }; - - sinks = { - vector_sink = { - type = "vector"; - inputs = [ "nginx_logs" ]; - address = "clickhouse:6000"; - }; - }; - }; - }; - - systemd.services.vector.serviceConfig = { - SupplementaryGroups = [ "nginx" ]; - }; + systemd.services.vector.serviceConfig = { + SupplementaryGroups = [ "nginx" ]; }; - }; + }; + }; - testScript = - let - # work around quote/substitution complexity by Nix, Perl, bash and SQL. - databaseDDL = pkgs.writeText "database.sql" "CREATE DATABASE IF NOT EXISTS nginxdb"; + testScript = + let + # work around quote/substitution complexity by Nix, Perl, bash and SQL. + databaseDDL = pkgs.writeText "database.sql" "CREATE DATABASE IF NOT EXISTS nginxdb"; - tableDDL = pkgs.writeText "table.sql" '' - CREATE TABLE IF NOT EXISTS nginxdb.access_logs ( - message String - ) - ENGINE = MergeTree() - ORDER BY tuple() - ''; - - # Graciously taken from https://clickhouse.com/docs/en/integrations/vector - tableView = pkgs.writeText "table-view.sql" '' - CREATE MATERIALIZED VIEW nginxdb.access_logs_view - ( - RemoteAddr String, - Client String, - RemoteUser String, - TimeLocal DateTime, - RequestMethod String, - Request String, - HttpVersion String, - Status Int32, - BytesSent Int64, - UserAgent String - ) - ENGINE = MergeTree() - ORDER BY RemoteAddr - POPULATE AS - WITH - splitByWhitespace(message) as split, - splitByRegexp('\S \d+ "([^"]*)"', message) as referer - SELECT - split[1] AS RemoteAddr, - split[2] AS Client, - split[3] AS RemoteUser, - parseDateTimeBestEffort(replaceOne(trim(LEADING '[' FROM split[4]), ':', ' ')) AS TimeLocal, - trim(LEADING '"' FROM split[6]) AS RequestMethod, - split[7] AS Request, - trim(TRAILING '"' FROM split[8]) AS HttpVersion, - split[9] AS Status, - split[10] AS BytesSent, - trim(BOTH '"' from referer[2]) AS UserAgent - FROM - (SELECT message FROM nginxdb.access_logs) - ''; - - selectQuery = pkgs.writeText "select.sql" "SELECT * from nginxdb.access_logs_view"; - in - '' - clickhouse.wait_for_unit("clickhouse") - clickhouse.wait_for_open_port(8123) - - clickhouse.wait_until_succeeds( - "journalctl -o cat -u clickhouse.service | grep 'Started ClickHouse server'" - ) - - clickhouse.wait_for_unit("vector") - clickhouse.wait_for_open_port(6000) - - clickhouse.succeed( - "cat ${databaseDDL} | clickhouse-client" - ) - - clickhouse.succeed( - "cat ${tableDDL} | clickhouse-client" - ) - - clickhouse.succeed( - "cat ${tableView} | clickhouse-client" - ) - - nginx.wait_for_unit("nginx") - nginx.wait_for_open_port(80) - nginx.wait_for_unit("vector") - nginx.wait_until_succeeds( - "journalctl -o cat -u vector.service | grep 'Starting file server'" - ) - - nginx.succeed("curl http://localhost/") - nginx.succeed("curl http://localhost/") - - nginx.wait_for_file("/var/log/nginx/access.log") - nginx.wait_until_succeeds( - "journalctl -o cat -u vector.service | grep 'Found new file to watch. file=/var/log/nginx/access.log'" - ) - - clickhouse.wait_until_succeeds( - "cat ${selectQuery} | clickhouse-client | grep 'curl'" + tableDDL = pkgs.writeText "table.sql" '' + CREATE TABLE IF NOT EXISTS nginxdb.access_logs ( + message String ) + ENGINE = MergeTree() + ORDER BY tuple() ''; - } -) + + # Graciously taken from https://clickhouse.com/docs/en/integrations/vector + tableView = pkgs.writeText "table-view.sql" '' + CREATE MATERIALIZED VIEW nginxdb.access_logs_view + ( + RemoteAddr String, + Client String, + RemoteUser String, + TimeLocal DateTime, + RequestMethod String, + Request String, + HttpVersion String, + Status Int32, + BytesSent Int64, + UserAgent String + ) + ENGINE = MergeTree() + ORDER BY RemoteAddr + POPULATE AS + WITH + splitByWhitespace(message) as split, + splitByRegexp('\S \d+ "([^"]*)"', message) as referer + SELECT + split[1] AS RemoteAddr, + split[2] AS Client, + split[3] AS RemoteUser, + parseDateTimeBestEffort(replaceOne(trim(LEADING '[' FROM split[4]), ':', ' ')) AS TimeLocal, + trim(LEADING '"' FROM split[6]) AS RequestMethod, + split[7] AS Request, + trim(TRAILING '"' FROM split[8]) AS HttpVersion, + split[9] AS Status, + split[10] AS BytesSent, + trim(BOTH '"' from referer[2]) AS UserAgent + FROM + (SELECT message FROM nginxdb.access_logs) + ''; + + selectQuery = pkgs.writeText "select.sql" "SELECT * from nginxdb.access_logs_view"; + in + '' + clickhouse.wait_for_unit("clickhouse") + clickhouse.wait_for_open_port(8123) + + clickhouse.wait_until_succeeds( + "journalctl -o cat -u clickhouse.service | grep 'Started ClickHouse server'" + ) + + clickhouse.wait_for_unit("vector") + clickhouse.wait_for_open_port(6000) + + clickhouse.succeed( + "cat ${databaseDDL} | clickhouse-client" + ) + + clickhouse.succeed( + "cat ${tableDDL} | clickhouse-client" + ) + + clickhouse.succeed( + "cat ${tableView} | clickhouse-client" + ) + + nginx.wait_for_unit("nginx") + nginx.wait_for_open_port(80) + nginx.wait_for_unit("vector") + nginx.wait_until_succeeds( + "journalctl -o cat -u vector.service | grep 'Starting file server'" + ) + + nginx.succeed("curl http://localhost/") + nginx.succeed("curl http://localhost/") + + nginx.wait_for_file("/var/log/nginx/access.log") + nginx.wait_until_succeeds( + "journalctl -o cat -u vector.service | grep 'Found new file to watch. file=/var/log/nginx/access.log'" + ) + + clickhouse.wait_until_succeeds( + "cat ${selectQuery} | clickhouse-client | grep 'curl'" + ) + ''; +} diff --git a/nixos/tests/vector/syslog-quickwit.nix b/nixos/tests/vector/syslog-quickwit.nix index 68783a1642f1..affae5f86b3d 100644 --- a/nixos/tests/vector/syslog-quickwit.nix +++ b/nixos/tests/vector/syslog-quickwit.nix @@ -1,162 +1,160 @@ -import ../make-test-python.nix ( - { lib, pkgs, ... }: +{ lib, pkgs, ... }: - # Based on https://quickwit.io/docs/log-management/send-logs/using-vector +# Based on https://quickwit.io/docs/log-management/send-logs/using-vector - { - name = "vector-syslog-quickwit"; - meta.maintainers = [ pkgs.lib.maintainers.happysalada ]; +{ + name = "vector-syslog-quickwit"; + meta.maintainers = [ pkgs.lib.maintainers.happysalada ]; - nodes = { - quickwit = - { config, pkgs, ... }: - { - environment.systemPackages = [ pkgs.jq ]; + nodes = { + quickwit = + { config, pkgs, ... }: + { + environment.systemPackages = [ pkgs.jq ]; - networking.firewall.allowedTCPPorts = [ 7280 ]; + networking.firewall.allowedTCPPorts = [ 7280 ]; - services.quickwit = { - enable = true; - settings = { - listen_address = "::"; - }; + services.quickwit = { + enable = true; + settings = { + listen_address = "::"; }; }; + }; - syslog = - { config, pkgs, ... }: - { - services.vector = { - enable = true; + syslog = + { config, pkgs, ... }: + { + services.vector = { + enable = true; - settings = { - sources = { - generate_syslog = { - type = "demo_logs"; - format = "syslog"; - interval = 0.5; - }; + settings = { + sources = { + generate_syslog = { + type = "demo_logs"; + format = "syslog"; + interval = 0.5; }; + }; - transforms = { - remap_syslog = { - inputs = [ "generate_syslog" ]; - type = "remap"; - source = '' - structured = parse_syslog!(.message) - .timestamp_nanos = to_unix_timestamp!(structured.timestamp, unit: "nanoseconds") - .body = structured - .service_name = structured.appname - .resource_attributes.source_type = .source_type - .resource_attributes.host.hostname = structured.hostname - .resource_attributes.service.name = structured.appname - .attributes.syslog.procid = structured.procid - .attributes.syslog.facility = structured.facility - .attributes.syslog.version = structured.version - .severity_text = if includes(["emerg", "err", "crit", "alert"], structured.severity) { - "ERROR" - } else if structured.severity == "warning" { - "WARN" - } else if structured.severity == "debug" { - "DEBUG" - } else if includes(["info", "notice"], structured.severity) { - "INFO" - } else { - structured.severity - } - .scope_name = structured.msgid - del(.message) - del(.host) - del(.timestamp) - del(.service) - del(.source_type) - ''; - }; + transforms = { + remap_syslog = { + inputs = [ "generate_syslog" ]; + type = "remap"; + source = '' + structured = parse_syslog!(.message) + .timestamp_nanos = to_unix_timestamp!(structured.timestamp, unit: "nanoseconds") + .body = structured + .service_name = structured.appname + .resource_attributes.source_type = .source_type + .resource_attributes.host.hostname = structured.hostname + .resource_attributes.service.name = structured.appname + .attributes.syslog.procid = structured.procid + .attributes.syslog.facility = structured.facility + .attributes.syslog.version = structured.version + .severity_text = if includes(["emerg", "err", "crit", "alert"], structured.severity) { + "ERROR" + } else if structured.severity == "warning" { + "WARN" + } else if structured.severity == "debug" { + "DEBUG" + } else if includes(["info", "notice"], structured.severity) { + "INFO" + } else { + structured.severity + } + .scope_name = structured.msgid + del(.message) + del(.host) + del(.timestamp) + del(.service) + del(.source_type) + ''; }; + }; - sinks = { - #emit_syslog = { - # inputs = ["remap_syslog"]; - # type = "console"; - # encoding.codec = "json"; - #}; - quickwit_logs = { - type = "http"; - method = "post"; - inputs = [ "remap_syslog" ]; - encoding.codec = "json"; - framing.method = "newline_delimited"; - uri = "http://quickwit:7280/api/v1/otel-logs-v0_7/ingest"; - }; + sinks = { + #emit_syslog = { + # inputs = ["remap_syslog"]; + # type = "console"; + # encoding.codec = "json"; + #}; + quickwit_logs = { + type = "http"; + method = "post"; + inputs = [ "remap_syslog" ]; + encoding.codec = "json"; + framing.method = "newline_delimited"; + uri = "http://quickwit:7280/api/v1/otel-logs-v0_7/ingest"; }; }; }; }; - }; + }; + }; - testScript = - let - aggregationQuery = pkgs.writeText "aggregation-query.json" '' - { - "query": "*", - "max_hits": 0, - "aggs": { - "count_per_minute": { - "histogram": { - "field": "timestamp_nanos", - "interval": 60000000 - }, - "aggs": { - "severity_text_count": { - "terms": { - "field": "severity_text" - } + testScript = + let + aggregationQuery = pkgs.writeText "aggregation-query.json" '' + { + "query": "*", + "max_hits": 0, + "aggs": { + "count_per_minute": { + "histogram": { + "field": "timestamp_nanos", + "interval": 60000000 + }, + "aggs": { + "severity_text_count": { + "terms": { + "field": "severity_text" } } } } } - ''; - in - '' - quickwit.wait_for_unit("quickwit") - quickwit.wait_for_open_port(7280) - quickwit.wait_for_open_port(7281) - - quickwit.wait_until_succeeds( - "journalctl -o cat -u quickwit.service | grep 'transitioned to ready state'" - ) - - syslog.wait_for_unit("vector") - syslog.wait_until_succeeds( - "journalctl -o cat -u vector.service | grep 'Vector has started'" - ) - - quickwit.wait_until_succeeds( - "journalctl -o cat -u quickwit.service | grep 'publish-new-splits'" - ) - - # Wait for logs to be generated - # Test below aggregates by the minute - syslog.sleep(60 * 2) - - quickwit.wait_until_succeeds( - "curl -sSf -XGET http://127.0.0.1:7280/api/v1/otel-logs-v0_7/search?query=severity_text:ERROR |" - + " jq '.num_hits' | grep -v '0'" - ) - - quickwit.wait_until_succeeds( - "journalctl -o cat -u quickwit.service | grep 'SearchRequest'" - ) - - quickwit.wait_until_succeeds( - "curl -sSf -XPOST -H 'Content-Type: application/json' http://127.0.0.1:7280/api/v1/otel-logs-v0_7/search --data @${aggregationQuery} |" - + " jq '.num_hits' | grep -v '0'" - ) - - quickwit.wait_until_succeeds( - "journalctl -o cat -u quickwit.service | grep 'count_per_minute'" - ) + } ''; - } -) + in + '' + quickwit.wait_for_unit("quickwit") + quickwit.wait_for_open_port(7280) + quickwit.wait_for_open_port(7281) + + quickwit.wait_until_succeeds( + "journalctl -o cat -u quickwit.service | grep 'transitioned to ready state'" + ) + + syslog.wait_for_unit("vector") + syslog.wait_until_succeeds( + "journalctl -o cat -u vector.service | grep 'Vector has started'" + ) + + quickwit.wait_until_succeeds( + "journalctl -o cat -u quickwit.service | grep 'publish-new-splits'" + ) + + # Wait for logs to be generated + # Test below aggregates by the minute + syslog.sleep(60 * 2) + + quickwit.wait_until_succeeds( + "curl -sSf -XGET http://127.0.0.1:7280/api/v1/otel-logs-v0_7/search?query=severity_text:ERROR |" + + " jq '.num_hits' | grep -v '0'" + ) + + quickwit.wait_until_succeeds( + "journalctl -o cat -u quickwit.service | grep 'SearchRequest'" + ) + + quickwit.wait_until_succeeds( + "curl -sSf -XPOST -H 'Content-Type: application/json' http://127.0.0.1:7280/api/v1/otel-logs-v0_7/search --data @${aggregationQuery} |" + + " jq '.num_hits' | grep -v '0'" + ) + + quickwit.wait_until_succeeds( + "journalctl -o cat -u quickwit.service | grep 'count_per_minute'" + ) + ''; +} diff --git a/pkgs/applications/editors/vscode/extensions/charliermarsh.ruff/default.nix b/pkgs/applications/editors/vscode/extensions/charliermarsh.ruff/default.nix index c92f38180f5f..c781c5145415 100644 --- a/pkgs/applications/editors/vscode/extensions/charliermarsh.ruff/default.nix +++ b/pkgs/applications/editors/vscode/extensions/charliermarsh.ruff/default.nix @@ -12,26 +12,26 @@ vscode-utils.buildVscodeMarketplaceExtension { sources = { "x86_64-linux" = { arch = "linux-x64"; - hash = "sha256-lGV/Zc4pibm7sTVtN4UYzuroxNgUltaUT9oJPaa5S8Q="; + hash = "sha256-+EiBEYZpJYjUMvVcNgs5pdXr1g8FB1ha2bKy29OPcSM="; }; "x86_64-darwin" = { arch = "darwin-x64"; - hash = "sha256-h1cvTJ9VUHOL27F9twdbLTSzLb+NUhqrbaScoKF5jZ4="; + hash = "sha256-ijy/ZVhVU1/ZrS1Fu3vuiThcjLuKSqf3lrgl8is54Co="; }; "aarch64-linux" = { arch = "linux-arm64"; - hash = "sha256-Ca9DGjQDT5BbJUL7FtU3dS6Zb7C2Blxr69l5HpZR4ZQ="; + hash = "sha256-mpUV/xN98Xi3B7ujotXK9T6xEfZWsQuWtvuPyufxfoY="; }; "aarch64-darwin" = { arch = "darwin-arm64"; - hash = "sha256-8Qay/ynixASQ8FFyAYjBeGcjBKQGXucGlOndOYa1Fn8="; + hash = "sha256-YaNMN7887v3tFccoPBz7hVhpGbGtbys7e5D5GCBIe20="; }; }; in { name = "ruff"; publisher = "charliermarsh"; - version = "2025.22.0"; + version = "2025.24.0"; } // sources.${stdenvNoCC.hostPlatform.system} or (throw "Unsupported system ${stdenvNoCC.hostPlatform.system}"); diff --git a/pkgs/applications/networking/browsers/chromium/info.json b/pkgs/applications/networking/browsers/chromium/info.json index f8f9d7253c9f..5e25fbd81c59 100644 --- a/pkgs/applications/networking/browsers/chromium/info.json +++ b/pkgs/applications/networking/browsers/chromium/info.json @@ -1,10 +1,10 @@ { "chromium": { - "version": "137.0.7151.103", + "version": "137.0.7151.119", "chromedriver": { - "version": "137.0.7151.104", - "hash_darwin": "sha256-K7kixWvPmTX35LB6whyHetvtaGxCBYoyr30LozPjQxI=", - "hash_darwin_aarch64": "sha256-57cLYzeZi1jKTBQcsSP0JD2RJDIl9RVQSGdY6cz3J68=" + "version": "137.0.7151.120", + "hash_darwin": "sha256-3NECoMlK57ZlCUPra20rJrZcx9FnMWvTXlcdksn8FUc=", + "hash_darwin_aarch64": "sha256-P1trGStKjTD/h+avjAXE5N6nqvAra9RDsSvrR/pTRUA=" }, "deps": { "depot_tools": { @@ -20,8 +20,8 @@ "DEPS": { "src": { "url": "https://chromium.googlesource.com/chromium/src.git", - "rev": "3dcc738117a3439068c9773ccd31f9858923fc4a", - "hash": "sha256-MIEjHLpfKIBiTFh+bO+NUf6iDpizTP9yfXQqbHfiDwo=", + "rev": "e0ac9d12dff5f2d33c935958b06bf1ded7f1c08c", + "hash": "sha256-+3C2n/7bbIOpXGvBrFnSMNlgLVRMoPtOF14CDROVClI=", "recompress": true }, "src/third_party/clang-format/script": { @@ -241,8 +241,8 @@ }, "src/third_party/devtools-frontend/src": { "url": "https://chromium.googlesource.com/devtools/devtools-frontend", - "rev": "e423961606946be24c8c1ec0d1ec91511efbabc5", - "hash": "sha256-MhooXuF6aw+ixPzvVCBl+6T+79cTReCYx86qqXAZ6bg=" + "rev": "afc8e923a37090445d6d97ca23fea49d9eb7b9cf", + "hash": "sha256-io0J6tt0RXumjjSklZyJpALV5IikPbROd40xcrX4iBs=" }, "src/third_party/dom_distiller_js/dist": { "url": "https://chromium.googlesource.com/chromium/dom-distiller/dist.git", @@ -791,8 +791,8 @@ }, "src/v8": { "url": "https://chromium.googlesource.com/v8/v8.git", - "rev": "41f53aba7095888c959932bd8f2ee8b4e16af223", - "hash": "sha256-ICrdvHA6fe2CUphRgPdlofazr0L+NFypWDNOI5e5QIM=" + "rev": "075234cf3d7622d9d588a6f748fc4501aa23080c", + "hash": "sha256-wrLxRuJ3rq1yC0PIUGPsuDB/YNee1x3J/i6ZSLk70HM=" } } }, diff --git a/pkgs/by-name/au/autokuma/no-doctest.patch b/pkgs/by-name/au/autokuma/no-doctest.patch new file mode 100644 index 000000000000..b958bfccbef4 --- /dev/null +++ b/pkgs/by-name/au/autokuma/no-doctest.patch @@ -0,0 +1,11 @@ +diff --git a/kuma-client/Cargo.toml b/kuma-client/Cargo.toml +index 144be59..017e1de 100644 +--- a/kuma-client/Cargo.toml ++++ b/kuma-client/Cargo.toml +@@ -1,3 +1,6 @@ ++[lib] ++doctest = false ++ + [package] + name = "kuma-client" + description = "Rust wrapper for the Uptime Kuma Socket.IO API" diff --git a/pkgs/by-name/au/autokuma/package.nix b/pkgs/by-name/au/autokuma/package.nix new file mode 100644 index 000000000000..ec80cb3f2db3 --- /dev/null +++ b/pkgs/by-name/au/autokuma/package.nix @@ -0,0 +1,39 @@ +{ + lib, + fetchFromGitHub, + rustPlatform, + pkg-config, + openssl, +}: + +rustPlatform.buildRustPackage (finalAttrs: { + pname = "autokuma"; + version = "1.0.0"; + + src = fetchFromGitHub { + owner = "BigBoot"; + repo = "AutoKuma"; + tag = "v${finalAttrs.version}"; + hash = "sha256-o1W0ssR4cjzx9VWg3qS2RhJEe4y4Ez/Y+4yRgXs6q0Y="; + }; + + cargoHash = "sha256-nu37qOv34nZ4pkxX7mu4zoLJFZWw3QCPQDS7SMKhqVw="; + + patches = [ ./no-doctest.patch ]; + + nativeBuildInputs = [ pkg-config ]; + buildInputs = [ openssl ]; + + postInstall = '' + mv $out/bin/crdgen $out/bin/autokuma-crdgen + ''; + + meta = { + description = "Utility that automates the creation of Uptime Kuma monitors"; + homepage = "https://github.com/BigBoot/AutoKuma"; + mainProgram = "autokuma"; + license = lib.licenses.mit; + platforms = lib.platforms.linux; + maintainers = with lib.maintainers; [ hougo ]; + }; +}) diff --git a/pkgs/by-name/ho/hot-resize/package.nix b/pkgs/by-name/ho/hot-resize/package.nix index 595b90c8d7f6..aab1ef132931 100644 --- a/pkgs/by-name/ho/hot-resize/package.nix +++ b/pkgs/by-name/ho/hot-resize/package.nix @@ -19,17 +19,17 @@ rustPlatform.buildRustPackage (finalAttrs: { pname = "hot-resize"; - version = "0.1.3"; + version = "0.1.4"; src = fetchFromGitHub { owner = "liberodark"; repo = "hot-resize"; tag = "v${finalAttrs.version}"; - hash = "sha256-5mh09ZYNpuWVJ2g9p8C6Ad4k132UWjudBhTb3HfoFRc="; + hash = "sha256-JB1U7mL3rkrsekmKt+0J1nnbtnlk/typIIfz3E+1moc="; }; useFetchCargoVendor = true; - cargoHash = "sha256-kUWyL36BC1+4FjujVxeguB0VvBtIN32QpuNYV6wjC5s="; + cargoHash = "sha256-+POAqB0msStix5KNqVwy8ckLEQ/rUsD52BtyetuKt2I="; nativeBuildInputs = [ pkg-config diff --git a/pkgs/by-name/jj/jjui/package.nix b/pkgs/by-name/jj/jjui/package.nix index ff797e6f26da..bf047485534d 100644 --- a/pkgs/by-name/jj/jjui/package.nix +++ b/pkgs/by-name/jj/jjui/package.nix @@ -6,16 +6,16 @@ }: buildGoModule (finalAttrs: { pname = "jjui"; - version = "0.8.10"; + version = "0.8.11"; src = fetchFromGitHub { owner = "idursun"; repo = "jjui"; tag = "v${finalAttrs.version}"; - hash = "sha256-pDK2ZjnhSlLepOdr7QEnj2+0vMvL2LPaWw1miA1oMSA="; + hash = "sha256-MBW0hjwyR0jguCWNnXiqZL0xa+vV9f2Ojfb2/61o9KY="; }; - vendorHash = "sha256-YlOK+NvyH/3uvvFcCZixv2+Y2m26TP8+ohUSdl3ppro="; + vendorHash = "sha256-2nUU5rrVWBk+9ljC+OiAVLcRnWghPPfpvq5yoNSRdVk="; passthru.updateScript = nix-update-script { }; diff --git a/pkgs/by-name/ne/newcomputermodern/package.nix b/pkgs/by-name/ne/newcomputermodern/package.nix index 175529c1962d..d7a73622f0c5 100644 --- a/pkgs/by-name/ne/newcomputermodern/package.nix +++ b/pkgs/by-name/ne/newcomputermodern/package.nix @@ -8,12 +8,12 @@ stdenvNoCC.mkDerivation (finalAttrs: { pname = "newcomputermodern"; - version = "7.0.2"; + version = "7.0.3"; src = fetchgit { url = "https://git.gnu.org.ua/newcm.git"; rev = finalAttrs.version; - hash = "sha256-J2k3gbQgmb+hsIYQi+kCccejrNccHryuC140rNwNPTQ="; + hash = "sha256-sMjzM0nRcMxgJax3ecJ/a5YB3mH7+7RWbNkdhU+V7dU="; }; nativeBuildInputs = [ fontforge ]; diff --git a/pkgs/by-name/no/nom/package.nix b/pkgs/by-name/no/nom/package.nix index 4ee8739d1800..3bf6887a388d 100644 --- a/pkgs/by-name/no/nom/package.nix +++ b/pkgs/by-name/no/nom/package.nix @@ -5,13 +5,13 @@ }: buildGoModule rec { pname = "nom"; - version = "2.8.2"; + version = "2.10.0"; src = fetchFromGitHub { owner = "guyfedwards"; repo = "nom"; tag = "v${version}"; - hash = "sha256-SkmY3eFEAC4EJtFpe6FwRmECIZJa/Oyb1yov75ySSH0="; + hash = "sha256-F1lKBfDufotQjVNJ1yMosRl1UlGMBlYCTHXdCzeVflg="; }; vendorHash = "sha256-d5KTDZKfuzv84oMgmsjJoXGO5XYLVKxOB5XehqgRvYw="; diff --git a/pkgs/by-name/ox/oxide-rs/0001-use-crates-io-over-git-dependencies.patch b/pkgs/by-name/ox/oxide-rs/0001-use-crates-io-over-git-dependencies.patch deleted file mode 100644 index 816e8741fc8e..000000000000 --- a/pkgs/by-name/ox/oxide-rs/0001-use-crates-io-over-git-dependencies.patch +++ /dev/null @@ -1,150 +0,0 @@ -diff --git a/Cargo.lock b/Cargo.lock -index c0d18e4..45686a0 100644 ---- a/Cargo.lock -+++ b/Cargo.lock -@@ -1514,7 +1514,7 @@ dependencies = [ - "httpdate", - "itoa", - "pin-project-lite", -- "socket2 0.5.7", -+ "socket2 0.4.10", - "tokio", - "tower-service", - "tracing", -@@ -2240,7 +2240,7 @@ version = "5.0.0-rc.1" - source = "registry+https://github.com/rust-lang/crates.io-index" - checksum = "23d385da3c602d29036d2f70beed71c36604df7570be17fed4c5b839616785bf" - dependencies = [ -- "base64 0.22.1", -+ "base64 0.21.7", - "chrono", - "getrandom", - "http 1.1.0", -@@ -2354,7 +2354,7 @@ dependencies = [ - "clap", - "dirs", - "futures", -- "progenitor-client 0.8.0 (registry+https://github.com/rust-lang/crates.io-index)", -+ "progenitor-client", - "rand", - "regress", - "reqwest", -@@ -2684,9 +2684,10 @@ dependencies = [ - [[package]] - name = "progenitor" - version = "0.8.0" --source = "git+https://github.com/oxidecomputer/progenitor#04da1197662209339ae8dd3768a0157c65ff5d67" -+source = "registry+https://github.com/rust-lang/crates.io-index" -+checksum = "293df5b79211fbf0c1ebad6513ba451d267e9c15f5f19ee5d3da775e2dd27331" - dependencies = [ -- "progenitor-client 0.8.0 (git+https://github.com/oxidecomputer/progenitor)", -+ "progenitor-client", - "progenitor-impl", - "progenitor-macro", - ] -@@ -2706,24 +2707,11 @@ dependencies = [ - "serde_urlencoded", - ] - --[[package]] --name = "progenitor-client" --version = "0.8.0" --source = "git+https://github.com/oxidecomputer/progenitor#04da1197662209339ae8dd3768a0157c65ff5d67" --dependencies = [ -- "bytes", -- "futures-core", -- "percent-encoding", -- "reqwest", -- "serde", -- "serde_json", -- "serde_urlencoded", --] -- - [[package]] - name = "progenitor-impl" - version = "0.8.0" --source = "git+https://github.com/oxidecomputer/progenitor#04da1197662209339ae8dd3768a0157c65ff5d67" -+source = "registry+https://github.com/rust-lang/crates.io-index" -+checksum = "d85934a440963a69f9f04f48507ff6e7aa2952a5b2d8f96cc37fa3dd5c270f66" - dependencies = [ - "heck", - "http 1.1.0", -@@ -2736,7 +2724,7 @@ dependencies = [ - "serde", - "serde_json", - "syn", -- "thiserror 2.0.6", -+ "thiserror 1.0.69", - "typify", - "unicode-ident", - ] -@@ -2744,7 +2732,8 @@ dependencies = [ - [[package]] - name = "progenitor-macro" - version = "0.8.0" --source = "git+https://github.com/oxidecomputer/progenitor#04da1197662209339ae8dd3768a0157c65ff5d67" -+source = "registry+https://github.com/rust-lang/crates.io-index" -+checksum = "d99a5a259e2d65a4933054aa51717c70b6aba0522695731ac354a522124efc9b" - dependencies = [ - "openapiv3", - "proc-macro2", -@@ -4069,7 +4058,8 @@ checksum = "42ff0bf0c66b8238c6f3b578df37d0b7848e55df8577b3f74f92a69acceeb825" - [[package]] - name = "typify" - version = "0.2.0" --source = "git+https://github.com/oxidecomputer/typify#f3e0cc9d6a5cee617a636136b99db650817bcbde" -+source = "registry+https://github.com/rust-lang/crates.io-index" -+checksum = "b4c644dda9862f0fef3a570d8ddb3c2cfb1d5ac824a1f2ddfa7bc8f071a5ad8a" - dependencies = [ - "typify-impl", - "typify-macro", -@@ -4078,7 +4068,8 @@ dependencies = [ - [[package]] - name = "typify-impl" - version = "0.2.0" --source = "git+https://github.com/oxidecomputer/typify#f3e0cc9d6a5cee617a636136b99db650817bcbde" -+source = "registry+https://github.com/rust-lang/crates.io-index" -+checksum = "d59ab345b6c0d8ae9500b9ff334a4c7c0d316c1c628dc55726b95887eb8dbd11" - dependencies = [ - "heck", - "log", -@@ -4090,14 +4081,15 @@ dependencies = [ - "serde", - "serde_json", - "syn", -- "thiserror 2.0.6", -+ "thiserror 1.0.69", - "unicode-ident", - ] - - [[package]] - name = "typify-macro" - version = "0.2.0" --source = "git+https://github.com/oxidecomputer/typify#f3e0cc9d6a5cee617a636136b99db650817bcbde" -+source = "registry+https://github.com/rust-lang/crates.io-index" -+checksum = "785e2cdcef0df8160fdd762ed548a637aaec1e83704fdbc14da0df66013ee8d0" - dependencies = [ - "proc-macro2", - "quote", -@@ -4413,7 +4405,7 @@ version = "0.1.9" - source = "registry+https://github.com/rust-lang/crates.io-index" - checksum = "cf221c93e13a30d793f7645a0e7762c55d169dbb0a49671918a2319d289b10bb" - dependencies = [ -- "windows-sys 0.59.0", -+ "windows-sys 0.48.0", - ] - - [[package]] -diff --git a/Cargo.toml b/Cargo.toml -index 8ef28ff..15739d7 100644 ---- a/Cargo.toml -+++ b/Cargo.toml -@@ -40,7 +40,7 @@ oxide-httpmock = { path = "sdk-httpmock", version = "0.9.0" } - oxnet = { git = "https://github.com/oxidecomputer/oxnet" } - predicates = "3.1.2" - pretty_assertions = "1.4.1" --progenitor = { git = "https://github.com/oxidecomputer/progenitor" } -+progenitor = "0.8.0" - progenitor-client = "0.8.0" - rand = "0.8.5" - ratatui = "0.26.3" diff --git a/pkgs/by-name/ox/oxide-rs/package.nix b/pkgs/by-name/ox/oxide-rs/package.nix index ceeb53b7d3cf..4701b32edc63 100644 --- a/pkgs/by-name/ox/oxide-rs/package.nix +++ b/pkgs/by-name/ox/oxide-rs/package.nix @@ -11,29 +11,33 @@ rustPlatform.buildRustPackage rec { pname = "oxide-rs"; - version = "0.9.0+20241204.0.0"; + version = "0.12.0+20250604.0.0"; src = fetchFromGitHub { owner = "oxidecomputer"; repo = "oxide.rs"; rev = "v${version}"; - hash = "sha256-NtTXpXDYazcXilQNW455UDkqMCFzFPvTUkbEBQsWIDo="; - # leaveDotGit is necessary because `build.rs` expects git information which - # is used to write a `built.rs` file which is read by the CLI application - # to display version information. - leaveDotGit = true; + hash = "sha256-XtN/ZRaVrw4pB82cCmWijjTMZzte7VlUzx5BaCq2mCc="; }; - useFetchCargoVendor = true; - cargoHash = "sha256-We5yNF8gtHWAUAead0uc99FIoMcicDWdGbTzPgpiFyY="; - - cargoPatches = [ - ./0001-use-crates-io-over-git-dependencies.patch + patches = [ + # original patch: https://git.iliana.fyi/nixos-configs/tree/packages/oxide-git-version.patch?id=0e4dc0d21def9084e2c6c1e20f3da08c31590945 + ./rm-built-ref-head-lookup.patch + ./rm-commit-hash-in-version-output.patch ]; + checkFlags = [ + # skip since output check includes git commit hash + "--skip=cmd_version::version_success" + # skip due to failure with loopback on debug + "--skip=test_cmd_auth_debug_logging" + ]; + + useFetchCargoVendor = true; + cargoHash = "sha256-b3RYPjkKgmcE70wSYl5Lu2uMS2gALxRSbLoKzXisUx4="; + cargoBuildFlags = [ "--package=oxide-cli" - "--package=xtask" ]; cargoTestFlags = [ diff --git a/pkgs/by-name/ox/oxide-rs/rm-built-ref-head-lookup.patch b/pkgs/by-name/ox/oxide-rs/rm-built-ref-head-lookup.patch new file mode 100644 index 000000000000..7fda66b3d7fb --- /dev/null +++ b/pkgs/by-name/ox/oxide-rs/rm-built-ref-head-lookup.patch @@ -0,0 +1,17 @@ +diff --git a/cli/build.rs b/cli/build.rs +index adba6cf..a7a2a53 100644 +--- a/cli/build.rs ++++ b/cli/build.rs +@@ -5,12 +5,5 @@ + // Copyright 2023 Oxide Computer Company + + fn main() { +- let src = std::env::var("CARGO_MANIFEST_DIR").unwrap(); +- match built::util::get_repo_head(src.as_ref()) { +- Ok(Some((_branch, _commit, _commit_short))) => (), +- Ok(None) => panic!("Error: Build script could not find git commit information"), +- Err(e) => panic!("Build script error: {}", e), +- }; +- + built::write_built_file().expect("Failed to acquire build-time information"); + } diff --git a/pkgs/by-name/ox/oxide-rs/rm-commit-hash-in-version-output.patch b/pkgs/by-name/ox/oxide-rs/rm-commit-hash-in-version-output.patch new file mode 100644 index 000000000000..4e06e63369d1 --- /dev/null +++ b/pkgs/by-name/ox/oxide-rs/rm-commit-hash-in-version-output.patch @@ -0,0 +1,21 @@ +diff --git a/cli/src/cmd_version.rs b/cli/src/cmd_version.rs +index 72153fb..1add398 100644 +--- a/cli/src/cmd_version.rs ++++ b/cli/src/cmd_version.rs +@@ -30,16 +30,6 @@ impl RunnableCmd for CmdVersion { + + println_nopipe!("Oxide CLI {}", cli_version); + +- println_nopipe!( +- "Built from commit: {} {}", +- built_info::GIT_COMMIT_HASH.unwrap(), +- if matches!(built_info::GIT_DIRTY, Some(true)) { +- "(dirty)" +- } else { +- "" +- } +- ); +- + println_nopipe!("Oxide API: {}", api_version); + + Ok(()) diff --git a/pkgs/by-name/pa/particle-cli/package.nix b/pkgs/by-name/pa/particle-cli/package.nix index 66864caab6c3..e5329dae7b5c 100644 --- a/pkgs/by-name/pa/particle-cli/package.nix +++ b/pkgs/by-name/pa/particle-cli/package.nix @@ -8,16 +8,16 @@ buildNpmPackage (finalAttrs: { pname = "particle-cli"; - version = "3.36.1"; + version = "3.36.2"; src = fetchFromGitHub { owner = "particle-iot"; repo = "particle-cli"; tag = "v${finalAttrs.version}"; - hash = "sha256-7u0RXoUBu/aJSBVSdmheIPvQ6b6Vji2KZ2t3sNhh3kY="; + hash = "sha256-KLcQmbIuhp71dpJttKA0tWAn2Qf+zl6njBypFkaLmzE="; }; - npmDepsHash = "sha256-0yLu3iyHQwWId+EAXu4dlCNHvuFZeEts2r5Y+FpHPQI="; + npmDepsHash = "sha256-oQch+7hH+URMI15YOA3iz4FVPwckJ3K/DOC1PfrA2dU="; buildInputs = [ udev diff --git a/pkgs/by-name/pl/plasma-panel-colorizer/package.nix b/pkgs/by-name/pl/plasma-panel-colorizer/package.nix index af81ccfd45ef..4cc80035645c 100644 --- a/pkgs/by-name/pl/plasma-panel-colorizer/package.nix +++ b/pkgs/by-name/pl/plasma-panel-colorizer/package.nix @@ -9,13 +9,13 @@ stdenv.mkDerivation (finalAttrs: { pname = "plasma-panel-colorizer"; - version = "4.3.0"; + version = "4.3.1"; src = fetchFromGitHub { owner = "luisbocanegra"; repo = "plasma-panel-colorizer"; tag = "v${finalAttrs.version}"; - hash = "sha256-B0aP49udYTV/zfEdZS4uvkGG4wZUScqTVn9+d5SYCEQ="; + hash = "sha256-1vDFFQKuEwfOnYCEDvGBRCVS4m36vuAd/bpimkI4suM="; }; nativeBuildInputs = [ diff --git a/pkgs/by-name/py/pyroscope/package.nix b/pkgs/by-name/py/pyroscope/package.nix new file mode 100644 index 000000000000..aa3269d674b1 --- /dev/null +++ b/pkgs/by-name/py/pyroscope/package.nix @@ -0,0 +1,63 @@ +{ + stdenv, + buildGoModule, + lib, + fetchFromGitHub, + versionCheckHook, + installShellFiles, + nix-update-script, + ... +}: + +buildGoModule (finalAttrs: { + pname = "pyroscope"; + version = "1.13.4"; + + src = fetchFromGitHub { + owner = "grafana"; + repo = "pyroscope"; + rev = "v1.13.4"; + hash = "sha256-nyb91BO4zzJl3AG/ojBO+q7WiicZYmOtztW6FTlQHMM="; + }; + + vendorHash = "sha256-GZMoXsoE3pL0T3tkWY7i1f9sGy5uVDqeurCvBteqV9A="; + proxyVendor = true; + + subPackages = [ + "cmd/pyroscope" + "cmd/profilecli" + ]; + + ldflags = [ + "-X=github.com/grafana/pyroscope/pkg/util/build.Branch=${finalAttrs.src.rev}" + "-X=github.com/grafana/pyroscope/pkg/util/build.Version=${finalAttrs.version}" + "-X=github.com/grafana/pyroscope/pkg/util/build.Revision=${finalAttrs.src.rev}" + "-X=github.com/grafana/pyroscope/pkg/util/build.BuildDate=1970-01-01T00:00:00Z" + ]; + + # We're overriding the version in 'ldFlags', so we should check that the + # derivation 'version' string is found in 'pyroscope --version'. + nativeInstallCheckInputs = [ versionCheckHook ]; + doInstallCheck = true; + versionCheckProgram = "${placeholder "out"}/bin/${finalAttrs.meta.mainProgram}"; + versionCheckProgramArg = "--version"; + + nativeBuildInputs = [ installShellFiles ]; + postInstall = lib.optionalString (stdenv.buildPlatform.canExecute stdenv.hostPlatform) '' + installShellCompletion --cmd pyroscope \ + --bash <($out/bin/pyroscope completion bash) \ + --fish <($out/bin/pyroscope completion fish) \ + --zsh <($out/bin/pyroscope completion zsh) + ''; + + passthru.updateScript = nix-update-script { }; + + meta = { + description = "Continuous profiling platform; debug performance issues down to a single line of code"; + homepage = "https://github.com/grafana/pyroscope"; + changelog = "https://github.com/grafana/pyroscope/blob/${finalAttrs.src.rev}/CHANGELOG.md"; + license = lib.licenses.agpl3Only; + maintainers = [ lib.teams.mercury ]; + mainProgram = "pyroscope"; + }; +}) diff --git a/pkgs/by-name/si/simdutf/package.nix b/pkgs/by-name/si/simdutf/package.nix index 93cdf3dee761..14d3aa88d762 100644 --- a/pkgs/by-name/si/simdutf/package.nix +++ b/pkgs/by-name/si/simdutf/package.nix @@ -8,13 +8,13 @@ stdenv.mkDerivation (finalAttrs: { pname = "simdutf"; - version = "7.3.0"; + version = "7.3.1"; src = fetchFromGitHub { owner = "simdutf"; repo = "simdutf"; rev = "v${finalAttrs.version}"; - hash = "sha256-P1Ryi0LdibqrHfBjq2lLBMQfB0WjLA69K3SwK/apFZM="; + hash = "sha256-U53FYlojKc3Q/GIC5TtfYmJMrB+NlPhSwIjNlp/5ZvI="; }; # Fix build on darwin diff --git a/pkgs/by-name/ta/taproot-assets/package.nix b/pkgs/by-name/ta/taproot-assets/package.nix index 2ab0bf31e5e5..4f230fdfe359 100644 --- a/pkgs/by-name/ta/taproot-assets/package.nix +++ b/pkgs/by-name/ta/taproot-assets/package.nix @@ -6,16 +6,16 @@ buildGoModule rec { pname = "taproot-assets"; - version = "0.5.1"; + version = "0.6.0"; src = fetchFromGitHub { owner = "lightninglabs"; repo = "taproot-assets"; rev = "v${version}"; - hash = "sha256-R6x8M69HM7mC0XG5cAH5SwTzeoSicNwZx0ExAKwcI80="; + hash = "sha256-ZLuV52W5WTNp45tnF1mmf+Snjd14604cKpnOjhabuoc="; }; - vendorHash = "sha256-aak2TNwAXpQLsMgOkeAyQM9f6logR5U+LS10g2Jtq1U="; + vendorHash = "sha256-9Du4WHLltGqmJXDOs2t5dwK5dbFGxWn0EiEE47czW2M="; subPackages = [ "cmd/tapcli" diff --git a/pkgs/by-name/tl/tlsrpt-reporter/package.nix b/pkgs/by-name/tl/tlsrpt-reporter/package.nix new file mode 100644 index 000000000000..713870ae063e --- /dev/null +++ b/pkgs/by-name/tl/tlsrpt-reporter/package.nix @@ -0,0 +1,65 @@ +{ + lib, + asciidoctor, + automake, + installShellFiles, + python3, + fetchFromGitHub, + nixosTests, +}: + +python3.pkgs.buildPythonApplication rec { + pname = "tlsrpt-reporter"; + version = "0.5.0"; + pyproject = true; + + outputs = [ + "out" + "man" + ]; + + src = fetchFromGitHub { + owner = "sys4"; + repo = "tlsrpt-reporter"; + tag = "v${version}"; + hash = "sha256-IH8hJX9l+YonqOuszcMome4mjdIaedgGNIptxTyH1ng="; + }; + + nativeBuildInputs = [ + asciidoctor + automake + installShellFiles + ]; + + build-system = [ + python3.pkgs.hatchling + ]; + + postBuild = '' + make -C doc + ''; + + postInstall = '' + installManPage doc/*.1 + ''; + + nativeCheckInputs = [ + python3.pkgs.pytestCheckHook + ]; + + pythonImportsCheck = [ + "tlsrpt_reporter" + ]; + + passthru.tests = { + inherit (nixosTests) tlsrpt; + }; + + meta = { + description = "Application suite to receive TLSRPT datagrams and to generate and deliver TLSRPT reports"; + homepage = "https://github.com/sys4/tlsrpt-reporter"; + changelog = "https://github.com/sys4/tlsrpt-reporter/blob/${src.tag}/CHANGELOG.md"; + license = lib.licenses.gpl3Only; + maintainers = with lib.maintainers; [ hexa ]; + }; +} diff --git a/pkgs/by-name/tt/tty-solitaire/package.nix b/pkgs/by-name/tt/tty-solitaire/package.nix index b4018375295d..fd6aa41e53e6 100644 --- a/pkgs/by-name/tt/tty-solitaire/package.nix +++ b/pkgs/by-name/tt/tty-solitaire/package.nix @@ -2,31 +2,20 @@ lib, stdenv, fetchFromGitHub, - fetchpatch, ncurses, }: -stdenv.mkDerivation rec { +stdenv.mkDerivation (finalAttrs: { pname = "tty-solitaire"; - version = "1.3.1"; + version = "1.4.1"; src = fetchFromGitHub { owner = "mpereira"; repo = "tty-solitaire"; - rev = "v${version}"; - sha256 = "sha256-zMLNWJieHxHALFQoSkdAxGbUBGuZnznLX86lI3P21F0="; + tag = "v${finalAttrs.version}"; + hash = "sha256-8lEF1P2aKh0D4moCu54Z4Tv9xLFkZJzFuXJLo7oF9MQ="; }; - patches = [ - # Patch pending upstream inclusion to support ncurses-6.3: - # https://github.com/mpereira/tty-solitaire/pull/61 - (fetchpatch { - name = "ncurses-6.3.patch"; - url = "https://github.com/mpereira/tty-solitaire/commit/4d066c564d086ce272b78cb8f80717a7fb83c261.patch"; - sha256 = "sha256-E1XVG0be6JH3K1y7UPap93s8xk8Nk0dKLdKHcJ7mA8E="; - }) - ]; - postPatch = '' sed -i -e '/^CFLAGS *?= *-g *$/d' Makefile ''; @@ -38,12 +27,12 @@ stdenv.mkDerivation rec { "PREFIX=${placeholder "out"}" ]; - meta = with lib; { + meta = { description = "Klondike Solitaire in your ncurses terminal"; - license = licenses.mit; + license = lib.licenses.mit; homepage = "https://github.com/mpereira/tty-solitaire"; platforms = ncurses.meta.platforms; maintainers = [ ]; mainProgram = "ttysolitaire"; }; -} +}) diff --git a/pkgs/by-name/ty/ty/package.nix b/pkgs/by-name/ty/ty/package.nix index 8afad1b12e57..9abe7d2c715a 100644 --- a/pkgs/by-name/ty/ty/package.nix +++ b/pkgs/by-name/ty/ty/package.nix @@ -14,14 +14,14 @@ rustPlatform.buildRustPackage (finalAttrs: { pname = "ty"; - version = "0.0.1-alpha.10"; + version = "0.0.1-alpha.11"; src = fetchFromGitHub { owner = "astral-sh"; repo = "ty"; tag = finalAttrs.version; fetchSubmodules = true; - hash = "sha256-0aBvCO3ohINxwX2qa07OY/WDZj6gq+z9my+B/yD03JQ="; + hash = "sha256-ns78SUbCRGWl7fr+acN8yjrx3/odIYFqO/sOL/5ayLw="; }; # For Darwin platforms, remove the integration test for file notifications, @@ -35,7 +35,7 @@ rustPlatform.buildRustPackage (finalAttrs: { cargoBuildFlags = [ "--package=ty" ]; - cargoHash = "sha256-MLdB1vGLVnylvYj8/asbXq5fy8yw8dbZoi4fytknfR4="; + cargoHash = "sha256-GNBQ522FX7Yly963/msRfiYKybpk+XDmn1rujfbO22A="; nativeBuildInputs = [ installShellFiles ]; diff --git a/pkgs/by-name/un/unison-ucm/package.nix b/pkgs/by-name/un/unison-ucm/package.nix index 7067ec63756f..2470ff294b87 100644 --- a/pkgs/by-name/un/unison-ucm/package.nix +++ b/pkgs/by-name/un/unison-ucm/package.nix @@ -14,21 +14,21 @@ stdenv.mkDerivation (finalAttrs: { pname = "unison-code-manager"; - version = "0.5.40"; + version = "0.5.41"; src = { aarch64-darwin = fetchurl { url = "https://github.com/unisonweb/unison/releases/download/release/${finalAttrs.version}/ucm-macos-arm64.tar.gz"; - hash = "sha256-KsypPKHyscOiPXy4ZeCZcUFGIV97lsnLREJp5KAGFcM="; + hash = "sha256-0Zz8lc1s46y2JC6DAbJjahap+hsz1QuLRl4nGryhSxA="; }; x86_64-darwin = fetchurl { url = "https://github.com/unisonweb/unison/releases/download/release/${finalAttrs.version}/ucm-macos-x64.tar.gz"; - hash = "sha256-TpD2W+j7F83E+YPQRNe1K7fnNfpJEwt25ldB+nqQw7I="; + hash = "sha256-O9H62uhWnOPQp7s4yUhnUXFyk0vNS4BAddaCru4n1GU="; }; x86_64-linux = fetchurl { url = "https://github.com/unisonweb/unison/releases/download/release/${finalAttrs.version}/ucm-linux-x64.tar.gz"; - hash = "sha256-o1Zx9Vmovl0b/QMVT9XGaRM6FphsIsZQZamYlJ6b6y0="; + hash = "sha256-ul5PCDqjfpsMiZZaZaH04Mrv29U9uS/ik8KwFNmXbgg="; }; } .${stdenv.hostPlatform.system} or (throw "Unsupported platform ${stdenv.hostPlatform.system}"); diff --git a/pkgs/by-name/ve/vectorcode/package.nix b/pkgs/by-name/ve/vectorcode/package.nix index a8c776fc62af..449391e5807c 100644 --- a/pkgs/by-name/ve/vectorcode/package.nix +++ b/pkgs/by-name/ve/vectorcode/package.nix @@ -1,14 +1,56 @@ { lib, - python3Packages, + cargo, fetchFromGitHub, installShellFiles, + pkg-config, + protobuf, + python3, + rustc, + rustPlatform, versionCheckHook, lspSupport ? true, }: -python3Packages.buildPythonApplication rec { +let + python = python3.override { + packageOverrides = self: super: { + # https://github.com/Davidyz/VectorCode/pull/36 + chromadb = super.chromadb.overridePythonAttrs (old: rec { + version = "0.6.3"; + src = fetchFromGitHub { + owner = "chroma-core"; + repo = "chroma"; + tag = version; + hash = "sha256-yvAX8buETsdPvMQmRK5+WFz4fVaGIdNlfhSadtHwU5U="; + }; + cargoDeps = rustPlatform.fetchCargoVendor { + pname = "chromadb"; + inherit version src; + hash = "sha256-lHRBXJa/OFNf4x7afEJw9XcuDveTBIy3XpQ3+19JXn4="; + }; + postPatch = null; + build-system = with self; [ + setuptools + setuptools-scm + ]; + nativeBuildInputs = [ + cargo + pkg-config + protobuf + rustc + rustPlatform.cargoSetupHook + ]; + dependencies = old.dependencies ++ [ + self.chroma-hnswlib + ]; + doCheck = false; + }); + }; + }; +in +python.pkgs.buildPythonApplication rec { pname = "vectorcode"; version = "0.6.10"; pyproject = true; @@ -20,12 +62,12 @@ python3Packages.buildPythonApplication rec { hash = "sha256-k9YpsVFV1HkIIIFPB7Iz7Jar+lY5vK6gpzNIlX55ZDY="; }; - build-system = with python3Packages; [ + build-system = with python.pkgs; [ pdm-backend ]; dependencies = - with python3Packages; + with python.pkgs; [ chromadb colorlog @@ -44,7 +86,7 @@ python3Packages.buildPythonApplication rec { ] ++ lib.optionals lspSupport optional-dependencies.lsp; - optional-dependencies = with python3Packages; { + optional-dependencies = with python.pkgs; { intel = [ openvino optimum @@ -77,7 +119,7 @@ python3Packages.buildPythonApplication rec { installShellFiles versionCheckHook ] - ++ (with python3Packages; [ + ++ (with python.pkgs; [ mcp pygls pytestCheckHook diff --git a/pkgs/by-name/vh/vhs/package.nix b/pkgs/by-name/vh/vhs/package.nix index 0bda86473b72..5e8497453ed8 100644 --- a/pkgs/by-name/vh/vhs/package.nix +++ b/pkgs/by-name/vh/vhs/package.nix @@ -12,16 +12,16 @@ buildGoModule rec { pname = "vhs"; - version = "0.9.0"; + version = "0.10.0"; src = fetchFromGitHub { owner = "charmbracelet"; repo = "vhs"; rev = "v${version}"; - hash = "sha256-ceY4zLd+4EwXpwunKiWnaAB25qutSK1b1SyIriAbAI0="; + hash = "sha256-ZnE5G8kfj7qScsT+bZg90ze4scpUxeC6xF8dAhdUUCo="; }; - vendorHash = "sha256-2vRAI+Mm8Pzk3u4rndtwYnUlrAtjffe0kpoA1EHprQk="; + vendorHash = "sha256-jmabOEFHduHzOBAymnxQrvYzXzxKnS1RqZZ0re3w63Y="; nativeBuildInputs = [ installShellFiles diff --git a/pkgs/by-name/yt/ytree/0001-use-prefix-and-gzip-n.diff b/pkgs/by-name/yt/ytree/0001-use-prefix-and-gzip-n.diff index 7a682ce7e4c6..977eaa538302 100644 --- a/pkgs/by-name/yt/ytree/0001-use-prefix-and-gzip-n.diff +++ b/pkgs/by-name/yt/ytree/0001-use-prefix-and-gzip-n.diff @@ -4,36 +4,26 @@ diff -Naur ytree-2.06-old/Makefile ytree-2.06-new/Makefile @@ -11,13 +11,13 @@ # ADD_CFLAGS: Add -DVI_KEYS if you want vi-cursor-keys # - + -DESTDIR = /usr +PREFIX = /usr - - ADD_CFLAGS = -O # -DVI_KEYS - + + ADD_CFLAGS = # -DVI_KEYS + -BINDIR = $(DESTDIR)/bin -MANDIR = $(DESTDIR)/share/man/man1 -MANESDIR = $(DESTDIR)/share/man/es/man1 +BINDIR = $(DESTDIR)$(PREFIX)/bin +MANDIR = $(DESTDIR)$(PREFIX)/share/man/man1 +MANESDIR = $(DESTDIR)$(PREFIX)/share/man/es/man1 - - + + # Uncomment the lines for your system (default is linux) -@@ -224,14 +224,14 @@ - +@@ -221,7 +221,7 @@ install: $(MAIN) + if [ ! -e $(BINDIR) ]; then mkdir -p $(BINDIR); fi install $(MAIN) $(BINDIR) - gzip -9c ytree.1 > ytree.1.gz + gzip -n -9c ytree.1 > ytree.1.gz - if [ -d $(MANDIR) ]; then install -m 0644 ytree.1.gz $(MANDIR)/; fi -- gzip -9c ytree.1.es > ytree.1.es.gz -+ gzip -n -9c ytree.1.es > ytree.1.es.gz - if [ -d $(MANESDIR) ]; then install -m 0644 ytree.1.es.gz $(MANESDIR)/; fi - - clean: - rm -f core *.o *~ *.orig *.bak -- -+ - clobber: clean - rm -f $(MAIN) ytree.1.es.gz ytree.1.gz - + if [ ! -e $(MANDIR) ]; then mkdir -p $(MANDIR); fi + install -m 0644 ytree.1.gz $(MANDIR)/ diff --git a/pkgs/by-name/yt/ytree/package.nix b/pkgs/by-name/yt/ytree/package.nix index 491ac76e1bae..4981cef6f9c5 100644 --- a/pkgs/by-name/yt/ytree/package.nix +++ b/pkgs/by-name/yt/ytree/package.nix @@ -8,11 +8,11 @@ stdenv.mkDerivation (finalAttrs: { pname = "ytree"; - version = "2.06"; + version = "2.10"; src = fetchurl { url = "https://han.de/~werner/ytree-${finalAttrs.version}.tar.gz"; - hash = "sha256-QRqI779ZnnytVUC7A7Zt0zyWexRwBnp+CVQcNvnvWeY="; + hash = "sha256-O7u9MvVoza4+A/xzWxeD2MumBaLKYFbRuXEUPX3dUX0="; }; patches = [ diff --git a/pkgs/development/python-modules/google-genai/default.nix b/pkgs/development/python-modules/google-genai/default.nix index 08216b07e608..76c5c3b78d66 100644 --- a/pkgs/development/python-modules/google-genai/default.nix +++ b/pkgs/development/python-modules/google-genai/default.nix @@ -1,10 +1,13 @@ { + aiohttp, anyio, buildPythonPackage, fetchFromGitHub, google-auth, httpx, lib, + packaging, + pkginfo, pydantic, pytestCheckHook, requests, @@ -16,17 +19,19 @@ buildPythonPackage rec { pname = "google-genai"; - version = "1.19.0"; + version = "1.20.0"; pyproject = true; src = fetchFromGitHub { owner = "googleapis"; repo = "python-genai"; tag = "v${version}"; - hash = "sha256-p9W34v1ToLwketM+wOfrouLLl9pFBljL5doykuZRINo="; + hash = "sha256-7DwLIK3/VCVSt9lq0Q0IRbhfLXOWw1TbPpDgI4jr9cg="; }; build-system = [ + packaging + pkginfo setuptools twine ]; @@ -43,6 +48,10 @@ buildPythonPackage rec { websockets ]; + optional-dependencies = { + aiohttp = [ aiohttp ]; + }; + pythonImportsCheck = [ "google.genai" ]; nativeCheckInputs = [ diff --git a/pkgs/development/python-modules/openrazer/common.nix b/pkgs/development/python-modules/openrazer/common.nix index df6f5ef4d8a9..d9dd55c2c528 100644 --- a/pkgs/development/python-modules/openrazer/common.nix +++ b/pkgs/development/python-modules/openrazer/common.nix @@ -1,13 +1,13 @@ { lib, fetchFromGitHub }: rec { - version = "3.10.1"; + version = "3.10.3"; pyproject = true; src = fetchFromGitHub { owner = "openrazer"; repo = "openrazer"; tag = "v${version}"; - hash = "sha256-igrGx7Y6ENtZatJCTAW43/0q6ZjljJ9/kU3QFli4yIU="; + hash = "sha256-M5g3Rn9WuyudhWQfDooopjexEgGVB0rzfJsPg+dqwn4="; }; meta = { diff --git a/pkgs/development/python-modules/py-libnuma/default.nix b/pkgs/development/python-modules/py-libnuma/default.nix new file mode 100644 index 000000000000..21e2d05547e0 --- /dev/null +++ b/pkgs/development/python-modules/py-libnuma/default.nix @@ -0,0 +1,47 @@ +{ + lib, + stdenv, + buildPythonPackage, + fetchFromGitHub, + pythonOlder, + setuptools, + numactl, +}: + +buildPythonPackage rec { + pname = "py-libnuma"; + version = "1.2"; + pyproject = true; + + src = fetchFromGitHub { + owner = "eedalong"; + repo = "pynuma"; + rev = "66cab0e008b850a04cfec5c4fb3f50bf28e3d488"; + hash = "sha256-ALYCcdN5eXrVWsTRwkHCwo4xsLMs/du3mUl1xSlo5iU="; + }; + + postPatch = '' + substituteInPlace numa/__init__.py \ + --replace-fail \ + 'LIBNUMA = CDLL(find_library("numa"))' \ + 'LIBNUMA = CDLL("${numactl}/lib/libnuma${stdenv.hostPlatform.extensions.sharedLibrary}")' + ''; + + build-system = [ setuptools ]; + + dependencies = [ + numactl + ]; + + # Tests write NUMA configuration, which may be persistent until reboot. + doCheck = false; + + pythonImportsCheck = [ "numa" ]; + + meta = { + description = "Python3 Interface to numa Linux library"; + homepage = "https://github.com/eedalong/pynuma"; + platforms = lib.platforms.linux; + license = lib.licenses.mit; + }; +} diff --git a/pkgs/development/python-modules/txtai/default.nix b/pkgs/development/python-modules/txtai/default.nix index 4e3d4788a6c4..c2e51878a79a 100644 --- a/pkgs/development/python-modules/txtai/default.nix +++ b/pkgs/development/python-modules/txtai/default.nix @@ -93,7 +93,7 @@ pytestCheckHook, }: let - version = "8.5.0"; + version = "8.6.0"; agent = [ mcpadapt smolagents @@ -240,7 +240,7 @@ let owner = "neuml"; repo = "txtai"; tag = "v${version}"; - hash = "sha256-kYjlA7pJ+xCC+tu0aaxziKaPo3hph5Ld8P/lVrip/eM="; + hash = "sha256-xFGVX0Ustime6ttysY3dcOCWc+jB75xqpSDBuRetIJc="; }; in buildPythonPackage { diff --git a/pkgs/development/python-modules/vllm/0005-drop-intel-reqs.patch b/pkgs/development/python-modules/vllm/0005-drop-intel-reqs.patch index d324661f149a..9bf7d3cdf4d8 100644 --- a/pkgs/development/python-modules/vllm/0005-drop-intel-reqs.patch +++ b/pkgs/development/python-modules/vllm/0005-drop-intel-reqs.patch @@ -1,12 +1,25 @@ +From 7511784ceb9252091a9d63ac6b54dcc67dd2b262 Mon Sep 17 00:00:00 2001 +From: Conroy Cheers +Date: Fri, 13 Jun 2025 17:42:10 +1000 +Subject: [PATCH] drop intel reqs + +--- + requirements/cpu.txt | 3 --- + 1 file changed, 3 deletions(-) + diff --git a/requirements/cpu.txt b/requirements/cpu.txt -index 121330158..d41918883 100644 +index d7b0fc6d8..be2df751b 100644 --- a/requirements/cpu.txt +++ b/requirements/cpu.txt -@@ -20,7 +20,3 @@ datasets # for benchmark scripts - +@@ -24,8 +24,5 @@ datasets # for benchmark scripts # cpu cannot use triton 3.3.0 triton==3.2.0; platform_machine == "x86_64" -- + -# Intel Extension for PyTorch, only for x86_64 CPUs -intel-openmp==2024.2.1; platform_machine == "x86_64" -intel_extension_for_pytorch==2.7.0; platform_machine == "x86_64" + py-libnuma; platform_system != "Darwin" + psutil; platform_system != "Darwin" +-- +2.49.0 + diff --git a/pkgs/development/python-modules/vllm/default.nix b/pkgs/development/python-modules/vllm/default.nix index d17725026613..33f3ef4369c7 100644 --- a/pkgs/development/python-modules/vllm/default.nix +++ b/pkgs/development/python-modules/vllm/default.nix @@ -3,6 +3,7 @@ stdenv, python, buildPythonPackage, + pythonAtLeast, fetchFromGitHub, fetchpatch, symlinkJoin, @@ -67,6 +68,7 @@ opentelemetry-exporter-otlp, bitsandbytes, flashinfer, + py-libnuma, # internal dependency - for overriding in overlays vllm-flash-attn ? null, @@ -246,16 +248,19 @@ in buildPythonPackage rec { pname = "vllm"; - version = "0.9.0.1"; + version = "0.9.1"; pyproject = true; + # https://github.com/vllm-project/vllm/issues/12083 + disabled = pythonAtLeast "3.13"; + stdenv = torch.stdenv; src = fetchFromGitHub { owner = "vllm-project"; repo = "vllm"; tag = "v${version}"; - hash = "sha256-gNe/kdsDQno8Fd6mo29feWmbyC0c2+kljlVxY4v7R9U="; + hash = "sha256-sp7rDpewTPXTVRBJHJMj+8pJDS6wAu0/OTJZwbPPqKc="; }; patches = [ @@ -264,34 +269,33 @@ buildPythonPackage rec { url = "https://github.com/vllm-project/vllm/commit/6a5d7e45f52c3a13de43b8b4fa9033e3b342ebd2.patch"; hash = "sha256-KYthqu+6XwsYYd80PtfrMMjuRV9+ionccr7EbjE4jJE="; }) + (fetchpatch { + name = "fall-back-to-gloo-when-nccl-unavailable.patch"; + url = "https://github.com/vllm-project/vllm/commit/aa131a94410683b0a02e74fed2ce95e6c2b6b030.patch"; + hash = "sha256-jNlQZQ8xiW85JWyBjsPZ6FoRQsiG1J8bwzmQjnaWFBg="; + }) ./0002-setup.py-nix-support-respect-cmakeFlags.patch ./0003-propagate-pythonpath.patch ./0004-drop-lsmod.patch ./0005-drop-intel-reqs.patch ]; - postPatch = - '' - # pythonRelaxDeps does not cover build-system - substituteInPlace pyproject.toml \ - --replace-fail "torch ==" "torch >=" + postPatch = '' + # pythonRelaxDeps does not cover build-system + substituteInPlace pyproject.toml \ + --replace-fail "torch ==" "torch >=" - # Ignore the python version check because it hard-codes minor versions and - # lags behind `ray`'s python interpreter support - substituteInPlace CMakeLists.txt \ - --replace-fail \ - 'set(PYTHON_SUPPORTED_VERSIONS' \ - 'set(PYTHON_SUPPORTED_VERSIONS "${lib.versions.majorMinor python.version}"' + # Ignore the python version check because it hard-codes minor versions and + # lags behind `ray`'s python interpreter support + substituteInPlace CMakeLists.txt \ + --replace-fail \ + 'set(PYTHON_SUPPORTED_VERSIONS' \ + 'set(PYTHON_SUPPORTED_VERSIONS "${lib.versions.majorMinor python.version}"' - # Pass build environment PYTHONPATH to vLLM's Python configuration scripts - substituteInPlace CMakeLists.txt \ - --replace-fail '$PYTHONPATH' '$ENV{PYTHONPATH}' - '' - + lib.optionalString (nccl == null) '' - # On platforms where NCCL is not supported (e.g. Jetson), substitute Gloo (provided by Torch) - substituteInPlace vllm/distributed/parallel_state.py \ - --replace-fail '"nccl"' '"gloo"' - ''; + # Pass build environment PYTHONPATH to vLLM's Python configuration scripts + substituteInPlace CMakeLists.txt \ + --replace-fail '$PYTHONPATH' '$ENV{PYTHONPATH}' + ''; nativeBuildInputs = [ @@ -362,7 +366,6 @@ buildPythonPackage rec { outlines pandas prometheus-fastapi-instrumentator - psutil py-cpuinfo pyarrow pydantic @@ -392,9 +395,15 @@ buildPythonPackage rec { opentelemetry-api opentelemetry-exporter-otlp bitsandbytes + # vLLM needs Torch's compiler to be present in order to use torch.compile + torch.stdenv.cc ] ++ uvicorn.optional-dependencies.standard ++ aioprometheus.optional-dependencies.starlette + ++ lib.optionals stdenv.targetPlatform.isLinux [ + py-libnuma + psutil + ] ++ lib.optionals cudaSupport [ cupy pynvml @@ -404,11 +413,11 @@ buildPythonPackage rec { dontUseCmakeConfigure = true; cmakeFlags = [ + ] + ++ lib.optionals cudaSupport [ (lib.cmakeFeature "FETCHCONTENT_SOURCE_DIR_CUTLASS" "${lib.getDev cutlass}") (lib.cmakeFeature "FLASH_MLA_SRC_DIR" "${lib.getDev flashmla}") (lib.cmakeFeature "VLLM_FLASH_ATTN_SRC_DIR" "${lib.getDev vllm-flash-attn'}") - ] - ++ lib.optionals cudaSupport [ (lib.cmakeFeature "TORCH_CUDA_ARCH_LIST" "${gpuTargetString}") (lib.cmakeFeature "CUTLASS_NVCC_ARCHS_ENABLED" "${cudaPackages.flags.cmakeCudaArchitecturesString}") (lib.cmakeFeature "CUDA_TOOLKIT_ROOT_DIR" "${symlinkJoin { diff --git a/pkgs/development/python-modules/whenever/default.nix b/pkgs/development/python-modules/whenever/default.nix index 4640e150b0b4..b5ea39990fcb 100644 --- a/pkgs/development/python-modules/whenever/default.nix +++ b/pkgs/development/python-modules/whenever/default.nix @@ -20,7 +20,7 @@ buildPythonPackage rec { pname = "whenever"; - version = "0.8.0"; + version = "0.8.5"; pyproject = true; disabled = pythonOlder "3.9"; @@ -29,12 +29,12 @@ buildPythonPackage rec { owner = "ariebovenberg"; repo = "whenever"; tag = version; - hash = "sha256-HeEuzOHT0EbmkbIH/yejKu54943ItUy8oY2ZlnEwgBA="; + hash = "sha256-AXAvjCtSnm1B/2NlZzzYdlI7BPHIuwKAeF+AxDp0PYQ="; }; cargoDeps = rustPlatform.fetchCargoVendor { inherit src; - hash = "sha256-szNRzaswILPjMJ+QFUWSJPfB6mF+o78Qg6AWkkancuU="; + hash = "sha256-qIIi1yKHaVz7NegOunzzdoQbeAavbdXPM4MBupLebDs="; }; build-system = [ diff --git a/pkgs/development/python-modules/xgrammar/0001-fix-find-nanobind-from-python-module.patch b/pkgs/development/python-modules/xgrammar/0001-fix-find-nanobind-from-python-module.patch new file mode 100644 index 000000000000..bedcb2211665 --- /dev/null +++ b/pkgs/development/python-modules/xgrammar/0001-fix-find-nanobind-from-python-module.patch @@ -0,0 +1,41 @@ +From c01e056ee845ae973ec36cc50125492ef8c02c12 Mon Sep 17 00:00:00 2001 +From: Conroy Cheers +Date: Thu, 12 Jun 2025 17:45:27 +1000 +Subject: [PATCH] [Fix] find nanobind from Python module + +--- + cpp/nanobind/CMakeLists.txt | 4 ++++ + pyproject.toml | 2 +- + 2 files changed, 5 insertions(+), 1 deletion(-) + +diff --git a/cpp/nanobind/CMakeLists.txt b/cpp/nanobind/CMakeLists.txt +index 8ea5622..02500ac 100644 +--- a/cpp/nanobind/CMakeLists.txt ++++ b/cpp/nanobind/CMakeLists.txt +@@ -3,6 +3,10 @@ find_package( + COMPONENTS Interpreter Development.Module + REQUIRED + ) ++ ++execute_process( ++ COMMAND "${Python_EXECUTABLE}" -m nanobind --cmake_dir ++ OUTPUT_STRIP_TRAILING_WHITESPACE OUTPUT_VARIABLE nanobind_DIR) + find_package(nanobind CONFIG REQUIRED) + + # Compile this source file seperately. Nanobind suggests to optimize bindings code for size, but +diff --git a/pyproject.toml b/pyproject.toml +index 11fae7d..d2078b1 100644 +--- a/pyproject.toml ++++ b/pyproject.toml +@@ -44,7 +44,7 @@ provider = "scikit_build_core.metadata.regex" + input = "python/xgrammar/version.py" + + [build-system] +-requires = ["scikit-build-core>=0.10.0", "nanobind==2.5.0"] ++requires = ["scikit-build-core>=0.10.0", "nanobind>=2.5.0"] + build-backend = "scikit_build_core.build" + + [tool.scikit-build] +-- +2.49.0 + diff --git a/pkgs/development/python-modules/xgrammar/default.nix b/pkgs/development/python-modules/xgrammar/default.nix index 90caf1208c0f..78cf62927f6d 100644 --- a/pkgs/development/python-modules/xgrammar/default.nix +++ b/pkgs/development/python-modules/xgrammar/default.nix @@ -7,7 +7,7 @@ # build-system cmake, ninja, - pybind11, + nanobind, scikit-build-core, # dependencies @@ -25,7 +25,7 @@ buildPythonPackage rec { pname = "xgrammar"; - version = "0.1.14"; + version = "0.1.19"; pyproject = true; src = fetchFromGitHub { @@ -33,13 +33,17 @@ buildPythonPackage rec { repo = "xgrammar"; tag = "v${version}"; fetchSubmodules = true; - hash = "sha256-ohsoc3g5XUp9vSXxyOGj20wXzCXZC02ktHYVQjDqNeM="; + hash = "sha256-0b2tJx1D/2X/uosbthHfevUpTCBtuSKNlxOKyidTotA="; }; + patches = [ + ./0001-fix-find-nanobind-from-python-module.patch + ]; + build-system = [ cmake ninja - pybind11 + nanobind scikit-build-core ]; dontUseCmakeConfigure = true; @@ -61,6 +65,11 @@ buildPythonPackage rec { writableTmpDirAsHomeHook ]; + NIX_CFLAGS_COMPILE = toString [ + # xgrammar hardcodes -flto=auto while using static linking, which can cause linker errors without this additional flag. + "-ffat-lto-objects" + ]; + disabledTests = [ # You are trying to access a gated repo. "test_grammar_compiler" diff --git a/pkgs/os-specific/linux/nvidia-x11/default.nix b/pkgs/os-specific/linux/nvidia-x11/default.nix index fa32f98da64c..0d5db1423241 100644 --- a/pkgs/os-specific/linux/nvidia-x11/default.nix +++ b/pkgs/os-specific/linux/nvidia-x11/default.nix @@ -101,14 +101,16 @@ rec { # Vulkan developer beta driver # See here for more information: https://developer.nvidia.com/vulkan-driver vulkan_beta = generic rec { - version = "570.123.14"; + version = "570.123.18"; persistencedVersion = "550.142"; settingsVersion = "550.142"; - sha256_64bit = "sha256-Tkh/zjv2G4v5TV0VkR2urQiCNPYruVdNm0qXFQ7yAqk="; - openSha256 = "sha256-1The9ceUuj0VuUshQw/gRRHzKbt+PrIlmWth2qkNIkg="; + sha256_64bit = "sha256-GoBNatVpits13a3xsJSUr9BFG+5xrUDROfHmvss2cSY="; + openSha256 = "sha256-AYl8En0ZAZXWlJ8J8LKbPvAEKX+y65L1aq4Hm+dJScs="; settingsSha256 = "sha256-Wk6IlVvs23cB4s0aMeZzSvbOQqB1RnxGMv3HkKBoIgY="; persistencedSha256 = "sha256-yQFrVk4i2dwReN0XoplkJ++iA1WFhnIkP7ns4ORmkFA="; url = "https://developer.nvidia.com/downloads/vulkan-beta-${lib.concatStrings (lib.splitVersion version)}-linux"; + + broken = kernel.kernelAtLeast "6.15"; }; # data center driver compatible with current default cudaPackages diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index 6dc205a1333d..95f3fe42791a 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -12017,6 +12017,8 @@ self: super: with self; { py-improv-ble-client = callPackage ../development/python-modules/py-improv-ble-client { }; + py-libnuma = callPackage ../development/python-modules/py-libnuma { }; + py-libzfs = callPackage ../development/python-modules/py-libzfs { }; py-lru-cache = callPackage ../development/python-modules/py-lru-cache { };