From ebafd551d74a6bf3d315876c401635395c5d84e9 Mon Sep 17 00:00:00 2001 From: Ilan Joselevich Date: Mon, 22 May 2023 15:05:16 +0300 Subject: [PATCH 1/5] nixos/hercules-ci-agent: sync module with upstream --- .../hercules-ci-agent/common.nix | 203 +++--------------- .../hercules-ci-agent/default.nix | 8 +- .../hercules-ci-agent/settings.nix | 153 +++++++++++++ 3 files changed, 188 insertions(+), 176 deletions(-) create mode 100644 nixos/modules/services/continuous-integration/hercules-ci-agent/settings.nix diff --git a/nixos/modules/services/continuous-integration/hercules-ci-agent/common.nix b/nixos/modules/services/continuous-integration/hercules-ci-agent/common.nix index 663f3df775c3..ea9b5ffbf43c 100644 --- a/nixos/modules/services/continuous-integration/hercules-ci-agent/common.nix +++ b/nixos/modules/services/continuous-integration/hercules-ci-agent/common.nix @@ -10,171 +10,18 @@ let inherit (lib) filterAttrs - literalMD literalExpression mkIf mkOption mkRemovedOptionModule mkRenamedOptionModule types + ; - cfg = - config.services.hercules-ci-agent; + cfg = config.services.hercules-ci-agent; - format = pkgs.formats.toml { }; - - settingsModule = { config, ... }: { - freeformType = format.type; - options = { - apiBaseUrl = mkOption { - description = lib.mdDoc '' - API base URL that the agent will connect to. - - When using Hercules CI Enterprise, set this to the URL where your - Hercules CI server is reachable. - ''; - type = types.str; - default = "https://hercules-ci.com"; - }; - baseDirectory = mkOption { - type = types.path; - default = "/var/lib/hercules-ci-agent"; - description = lib.mdDoc '' - State directory (secrets, work directory, etc) for agent - ''; - }; - concurrentTasks = mkOption { - description = lib.mdDoc '' - Number of tasks to perform simultaneously. - - A task is a single derivation build, an evaluation or an effect run. - At minimum, you need 2 concurrent tasks for `x86_64-linux` - in your cluster, to allow for import from derivation. - - `concurrentTasks` can be around the CPU core count or lower if memory is - the bottleneck. - - The optimal value depends on the resource consumption characteristics of your workload, - including memory usage and in-task parallelism. This is typically determined empirically. - - When scaling, it is generally better to have a double-size machine than two machines, - because each split of resources causes inefficiencies; particularly with regards - to build latency because of extra downloads. - ''; - type = types.either types.ints.positive (types.enum [ "auto" ]); - default = "auto"; - }; - labels = mkOption { - description = lib.mdDoc '' - A key-value map of user data. - - This data will be available to organization members in the dashboard and API. - - The values can be of any TOML type that corresponds to a JSON type, but arrays - can not contain tables/objects due to limitations of the TOML library. Values - involving arrays of non-primitive types may not be representable currently. - ''; - type = format.type; - defaultText = literalExpression '' - { - agent.source = "..."; # One of "nixpkgs", "flake", "override" - lib.version = "..."; - pkgs.version = "..."; - } - ''; - }; - workDirectory = mkOption { - description = lib.mdDoc '' - The directory in which temporary subdirectories are created for task state. This includes sources for Nix evaluation. - ''; - type = types.path; - default = config.baseDirectory + "/work"; - defaultText = literalExpression ''baseDirectory + "/work"''; - }; - staticSecretsDirectory = mkOption { - description = lib.mdDoc '' - This is the default directory to look for statically configured secrets like `cluster-join-token.key`. - - See also `clusterJoinTokenPath` and `binaryCachesPath` for fine-grained configuration. - ''; - type = types.path; - default = config.baseDirectory + "/secrets"; - defaultText = literalExpression ''baseDirectory + "/secrets"''; - }; - clusterJoinTokenPath = mkOption { - description = lib.mdDoc '' - Location of the cluster-join-token.key file. - - You can retrieve the contents of the file when creating a new agent via - . - - As this value is confidential, it should not be in the store, but - installed using other means, such as agenix, NixOps - `deployment.keys`, or manual installation. - - The contents of the file are used for authentication between the agent and the API. - ''; - type = types.path; - default = config.staticSecretsDirectory + "/cluster-join-token.key"; - defaultText = literalExpression ''staticSecretsDirectory + "/cluster-join-token.key"''; - }; - binaryCachesPath = mkOption { - description = lib.mdDoc '' - Path to a JSON file containing binary cache secret keys. - - As these values are confidential, they should not be in the store, but - copied over using other means, such as agenix, NixOps - `deployment.keys`, or manual installation. - - The format is described on . - ''; - type = types.path; - default = config.staticSecretsDirectory + "/binary-caches.json"; - defaultText = literalExpression ''staticSecretsDirectory + "/binary-caches.json"''; - }; - secretsJsonPath = mkOption { - description = lib.mdDoc '' - Path to a JSON file containing secrets for effects. - - As these values are confidential, they should not be in the store, but - copied over using other means, such as agenix, NixOps - `deployment.keys`, or manual installation. - - The format is described on . - ''; - type = types.path; - default = config.staticSecretsDirectory + "/secrets.json"; - defaultText = literalExpression ''staticSecretsDirectory + "/secrets.json"''; - }; - }; - }; - - # TODO (roberth, >=2022) remove - checkNix = - if !cfg.checkNix - then "" - else if lib.versionAtLeast config.nix.package.version "2.3.10" - then "" - else - pkgs.stdenv.mkDerivation { - name = "hercules-ci-check-system-nix-src"; - inherit (config.nix.package) src patches; - dontConfigure = true; - buildPhase = '' - echo "Checking in-memory pathInfoCache expiry" - if ! grep 'PathInfoCacheValue' src/libstore/store-api.hh >/dev/null; then - cat 1>&2 < + builtins.match ".*(^|\n)[ \t]*trusted-users[ \t]*=.*" config.nix.extraOptions == null; + message = '' + hercules-ci-agent: Please do not set `trusted-users` in `nix.extraOptions`. + + The hercules-ci-agent module by default relies on `nix.settings.trusted-users` + to be effectful, but a line like `trusted-users = ...` in `nix.extraOptions` + will override the value set in `nix.settings.trusted-users`. + + Instead of setting `trusted-users` in the `nix.extraOptions` string, you should + set an option with additive semantics, such as + - the NixOS option `nix.settings.trusted-users`, or + - the Nix option in the `extraOptions` string, `extra-trusted-users` + ''; + } + ]; + nix.extraOptions = '' # A store path that was missing at first may well have finished building, # even shortly after the previous lookup. This *also* applies to the daemon. narinfo-cache-negative-ttl = 0 @@ -251,14 +109,9 @@ in services.hercules-ci-agent = { tomlFile = format.generate "hercules-ci-agent.toml" cfg.settings; - - settings.labels = { - agent.source = - if options.services.hercules-ci-agent.package.highestPrio == (lib.modules.mkOptionDefault { }).priority - then "nixpkgs" - else lib.mkOptionDefault "override"; - pkgs.version = pkgs.lib.version; - lib.version = lib.version; + settings.config._module.args = { + packageOption = options.services.hercules-ci-agent.package; + inherit pkgs; }; }; }; diff --git a/nixos/modules/services/continuous-integration/hercules-ci-agent/default.nix b/nixos/modules/services/continuous-integration/hercules-ci-agent/default.nix index e28493ce7767..ad26b5316dde 100644 --- a/nixos/modules/services/continuous-integration/hercules-ci-agent/default.nix +++ b/nixos/modules/services/continuous-integration/hercules-ci-agent/default.nix @@ -36,8 +36,14 @@ in Restart = "on-failure"; RestartSec = 120; - LimitSTACK = 256 * 1024 * 1024; + # If a worker goes OOM, don't kill the main process. It needs to + # report the failure and it's unlikely to be part of the problem. OOMPolicy = "continue"; + + # Work around excessive stack use by libstdc++ regex + # https://gcc.gnu.org/bugzilla/show_bug.cgi?id=86164 + # A 256 MiB stack allows between 400 KiB and 1.5 MiB file to be matched by ".*". + LimitSTACK = 256 * 1024 * 1024; }; }; diff --git a/nixos/modules/services/continuous-integration/hercules-ci-agent/settings.nix b/nixos/modules/services/continuous-integration/hercules-ci-agent/settings.nix new file mode 100644 index 000000000000..8eb902313ee8 --- /dev/null +++ b/nixos/modules/services/continuous-integration/hercules-ci-agent/settings.nix @@ -0,0 +1,153 @@ +# Not a module +{ pkgs, lib }: +let + inherit (lib) + types + literalExpression + mkOption + ; + + format = pkgs.formats.toml { }; + + settingsModule = { config, packageOption, pkgs, ... }: { + freeformType = format.type; + options = { + apiBaseUrl = mkOption { + description = lib.mdDoc '' + API base URL that the agent will connect to. + + When using Hercules CI Enterprise, set this to the URL where your + Hercules CI server is reachable. + ''; + type = types.str; + default = "https://hercules-ci.com"; + }; + baseDirectory = mkOption { + type = types.path; + default = "/var/lib/hercules-ci-agent"; + description = lib.mdDoc '' + State directory (secrets, work directory, etc) for agent + ''; + }; + concurrentTasks = mkOption { + description = lib.mdDoc '' + Number of tasks to perform simultaneously. + + A task is a single derivation build, an evaluation or an effect run. + At minimum, you need 2 concurrent tasks for `x86_64-linux` + in your cluster, to allow for import from derivation. + + `concurrentTasks` can be around the CPU core count or lower if memory is + the bottleneck. + + The optimal value depends on the resource consumption characteristics of your workload, + including memory usage and in-task parallelism. This is typically determined empirically. + + When scaling, it is generally better to have a double-size machine than two machines, + because each split of resources causes inefficiencies; particularly with regards + to build latency because of extra downloads. + ''; + type = types.either types.ints.positive (types.enum [ "auto" ]); + default = "auto"; + defaultText = lib.literalMD '' + `"auto"`, meaning equal to the number of CPU cores. + ''; + }; + labels = mkOption { + description = lib.mdDoc '' + A key-value map of user data. + + This data will be available to organization members in the dashboard and API. + + The values can be of any TOML type that corresponds to a JSON type, but arrays + can not contain tables/objects due to limitations of the TOML library. Values + involving arrays of non-primitive types may not be representable currently. + ''; + type = format.type; + defaultText = literalExpression '' + { + agent.source = "..."; # One of "nixpkgs", "flake", "override" + lib.version = "..."; + pkgs.version = "..."; + } + ''; + }; + workDirectory = mkOption { + description = lib.mdDoc '' + The directory in which temporary subdirectories are created for task state. This includes sources for Nix evaluation. + ''; + type = types.path; + default = config.baseDirectory + "/work"; + defaultText = literalExpression ''baseDirectory + "/work"''; + }; + staticSecretsDirectory = mkOption { + description = lib.mdDoc '' + This is the default directory to look for statically configured secrets like `cluster-join-token.key`. + + See also `clusterJoinTokenPath` and `binaryCachesPath` for fine-grained configuration. + ''; + type = types.path; + default = config.baseDirectory + "/secrets"; + defaultText = literalExpression ''baseDirectory + "/secrets"''; + }; + clusterJoinTokenPath = mkOption { + description = lib.mdDoc '' + Location of the cluster-join-token.key file. + + You can retrieve the contents of the file when creating a new agent via + . + + As this value is confidential, it should not be in the store, but + installed using other means, such as agenix, NixOps + `deployment.keys`, or manual installation. + + The contents of the file are used for authentication between the agent and the API. + ''; + type = types.path; + default = config.staticSecretsDirectory + "/cluster-join-token.key"; + defaultText = literalExpression ''staticSecretsDirectory + "/cluster-join-token.key"''; + }; + binaryCachesPath = mkOption { + description = lib.mdDoc '' + Path to a JSON file containing binary cache secret keys. + + As these values are confidential, they should not be in the store, but + copied over using other means, such as agenix, NixOps + `deployment.keys`, or manual installation. + + The format is described on . + ''; + type = types.path; + default = config.staticSecretsDirectory + "/binary-caches.json"; + defaultText = literalExpression ''staticSecretsDirectory + "/binary-caches.json"''; + }; + secretsJsonPath = mkOption { + description = lib.mdDoc '' + Path to a JSON file containing secrets for effects. + + As these values are confidential, they should not be in the store, but + copied over using other means, such as agenix, NixOps + `deployment.keys`, or manual installation. + + The format is described on . + ''; + type = types.path; + default = config.staticSecretsDirectory + "/secrets.json"; + defaultText = literalExpression ''staticSecretsDirectory + "/secrets.json"''; + }; + }; + config = { + labels = { + agent.source = + if packageOption.highestPrio == (lib.modules.mkOptionDefault { }).priority + then "nixpkgs" + else lib.mkOptionDefault "override"; + pkgs.version = pkgs.lib.version; + lib.version = lib.version; + }; + }; + }; +in +{ + inherit format settingsModule; +} From 0d405840d3ccd2eb2d1d5adec46bb9b72f1674b4 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 23 May 2023 08:49:56 +0200 Subject: [PATCH 2/5] hercules-ci-agent: Improve passthru tests --- .../hercules-ci-agent/default.nix | 25 +++++++++++++++++-- 1 file changed, 23 insertions(+), 2 deletions(-) diff --git a/pkgs/development/tools/continuous-integration/hercules-ci-agent/default.nix b/pkgs/development/tools/continuous-integration/hercules-ci-agent/default.nix index b4a783765df9..a1ed3d6ebe72 100644 --- a/pkgs/development/tools/continuous-integration/hercules-ci-agent/default.nix +++ b/pkgs/development/tools/continuous-integration/hercules-ci-agent/default.nix @@ -22,10 +22,31 @@ in pkg.overrideAttrs (o: { }; passthru = o.passthru // { # Does not test the package, but evaluation of the related NixOS module. - tests.nixos-minimal-config = nixos { + tests.nixos-simple-config = (nixos { boot.loader.grub.enable = false; fileSystems."/".device = "bogus"; services.hercules-ci-agent.enable = true; - }; + # Dummy value for testing only. + system.stateVersion = lib.trivial.release; # TEST ONLY + }).config.system.build.toplevel; + + tests.nixos-many-options-config = (nixos ({ pkgs, ... }: { + boot.loader.grub.enable = false; + fileSystems."/".device = "bogus"; + services.hercules-ci-agent = { + enable = true; + package = pkgs.hercules-ci-agent; + settings = { + workDirectory = "/var/tmp/hci"; + binaryCachesPath = "/var/keys/binary-caches.json"; + labels.foo.bar.baz = "qux"; + labels.qux = ["q" "u"]; + apiBaseUrl = "https://hci.dev.biz.example.com"; + concurrentTasks = 42; + }; + }; + # Dummy value for testing only. + system.stateVersion = lib.trivial.release; # TEST ONLY + })).config.system.build.toplevel; }; }) From 38fd1bad36c80e19d4cd5a843afda7bfc27620fe Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 23 May 2023 09:05:45 +0200 Subject: [PATCH 3/5] hercules-ci-agent: Add ssh and use makeBinaryWrapper ... like upstream. --- .../hercules-ci-agent/default.nix | 28 ++++++++++++++++--- 1 file changed, 24 insertions(+), 4 deletions(-) diff --git a/pkgs/development/tools/continuous-integration/hercules-ci-agent/default.nix b/pkgs/development/tools/continuous-integration/hercules-ci-agent/default.nix index a1ed3d6ebe72..205d1bf1dde5 100644 --- a/pkgs/development/tools/continuous-integration/hercules-ci-agent/default.nix +++ b/pkgs/development/tools/continuous-integration/hercules-ci-agent/default.nix @@ -1,8 +1,21 @@ -{ gnutar, gzip, git, haskell, haskellPackages, lib, makeWrapper, nixos, runc, stdenv }: +{ + git, + gnutar, + gzip, + haskell, + haskellPackages, + lib, + makeBinaryWrapper, + nixos, + openssh, + runc, + runCommand, + stdenv, +}: let inherit (haskell.lib.compose) overrideCabal addBuildTools justStaticExecutables; inherit (lib) makeBinPath; - bundledBins = [ gnutar gzip git ] ++ lib.optional stdenv.isLinux runc; + bundledBins = [ gnutar gzip git openssh ] ++ lib.optional stdenv.isLinux runc; pkg = # justStaticExecutables is needed due to https://github.com/NixOS/nix/issues/2990 @@ -15,12 +28,19 @@ let makeWrapper $out/libexec/hercules-ci-agent $out/bin/hercules-ci-agent --prefix PATH : ${lib.escapeShellArg (makeBinPath bundledBins)} ''; }) - (addBuildTools [ makeWrapper ] (justStaticExecutables haskellPackages.hercules-ci-agent)); -in pkg.overrideAttrs (o: { + (addBuildTools [ makeBinaryWrapper ] (justStaticExecutables haskellPackages.hercules-ci-agent)); +in pkg.overrideAttrs (finalAttrs: o: { meta = o.meta // { position = toString ./default.nix + ":1"; }; passthru = o.passthru // { + + tests.help = runCommand "test-hercules-ci-agent-help" { } '' + (${finalAttrs.finalPackage}/bin/hercules-ci-agent --help 2>&1 || true) | grep -F -- '--config' + (${lib.getExe finalAttrs.finalPackage} --help 2>&1 || true) | grep -F -- '--config' + touch $out + ''; + # Does not test the package, but evaluation of the related NixOS module. tests.nixos-simple-config = (nixos { boot.loader.grub.enable = false; From 3746d88d79a31136293ae32d224f574258a056cd Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Tue, 23 May 2023 10:17:12 +0200 Subject: [PATCH 4/5] hercules-ci-agent: tests: Only build NixOS config for Linux --- .../hercules-ci-agent/default.nix | 64 ++++++++++--------- 1 file changed, 33 insertions(+), 31 deletions(-) diff --git a/pkgs/development/tools/continuous-integration/hercules-ci-agent/default.nix b/pkgs/development/tools/continuous-integration/hercules-ci-agent/default.nix index 205d1bf1dde5..d655941e7a27 100644 --- a/pkgs/development/tools/continuous-integration/hercules-ci-agent/default.nix +++ b/pkgs/development/tools/continuous-integration/hercules-ci-agent/default.nix @@ -35,38 +35,40 @@ in pkg.overrideAttrs (finalAttrs: o: { }; passthru = o.passthru // { - tests.help = runCommand "test-hercules-ci-agent-help" { } '' - (${finalAttrs.finalPackage}/bin/hercules-ci-agent --help 2>&1 || true) | grep -F -- '--config' - (${lib.getExe finalAttrs.finalPackage} --help 2>&1 || true) | grep -F -- '--config' - touch $out - ''; + tests = { + help = runCommand "test-hercules-ci-agent-help" { } '' + (${finalAttrs.finalPackage}/bin/hercules-ci-agent --help 2>&1 || true) | grep -F -- '--config' + (${lib.getExe finalAttrs.finalPackage} --help 2>&1 || true) | grep -F -- '--config' + touch $out + ''; + } // lib.optionalAttrs (stdenv.isLinux) { + # Does not test the package, but evaluation of the related NixOS module. + nixos-simple-config = (nixos { + boot.loader.grub.enable = false; + fileSystems."/".device = "bogus"; + services.hercules-ci-agent.enable = true; + # Dummy value for testing only. + system.stateVersion = lib.trivial.release; # TEST ONLY + }).config.system.build.toplevel; - # Does not test the package, but evaluation of the related NixOS module. - tests.nixos-simple-config = (nixos { - boot.loader.grub.enable = false; - fileSystems."/".device = "bogus"; - services.hercules-ci-agent.enable = true; - # Dummy value for testing only. - system.stateVersion = lib.trivial.release; # TEST ONLY - }).config.system.build.toplevel; - - tests.nixos-many-options-config = (nixos ({ pkgs, ... }: { - boot.loader.grub.enable = false; - fileSystems."/".device = "bogus"; - services.hercules-ci-agent = { - enable = true; - package = pkgs.hercules-ci-agent; - settings = { - workDirectory = "/var/tmp/hci"; - binaryCachesPath = "/var/keys/binary-caches.json"; - labels.foo.bar.baz = "qux"; - labels.qux = ["q" "u"]; - apiBaseUrl = "https://hci.dev.biz.example.com"; - concurrentTasks = 42; + nixos-many-options-config = (nixos ({ pkgs, ... }: { + boot.loader.grub.enable = false; + fileSystems."/".device = "bogus"; + services.hercules-ci-agent = { + enable = true; + package = pkgs.hercules-ci-agent; + settings = { + workDirectory = "/var/tmp/hci"; + binaryCachesPath = "/var/keys/binary-caches.json"; + labels.foo.bar.baz = "qux"; + labels.qux = ["q" "u"]; + apiBaseUrl = "https://hci.dev.biz.example.com"; + concurrentTasks = 42; + }; }; - }; - # Dummy value for testing only. - system.stateVersion = lib.trivial.release; # TEST ONLY - })).config.system.build.toplevel; + # Dummy value for testing only. + system.stateVersion = lib.trivial.release; # TEST ONLY + })).config.system.build.toplevel; + }; }; }) From b419a39f1f3283264ee63845cdcd526db10facae Mon Sep 17 00:00:00 2001 From: Ilan Joselevich Date: Tue, 23 May 2023 13:39:42 +0300 Subject: [PATCH 5/5] hercules-ci-agent: replace help test with a better version test --- .../hercules-ci-agent/default.nix | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/pkgs/development/tools/continuous-integration/hercules-ci-agent/default.nix b/pkgs/development/tools/continuous-integration/hercules-ci-agent/default.nix index d655941e7a27..c5c52d29d3aa 100644 --- a/pkgs/development/tools/continuous-integration/hercules-ci-agent/default.nix +++ b/pkgs/development/tools/continuous-integration/hercules-ci-agent/default.nix @@ -9,8 +9,8 @@ nixos, openssh, runc, - runCommand, stdenv, + testers, }: let inherit (haskell.lib.compose) overrideCabal addBuildTools justStaticExecutables; @@ -34,13 +34,11 @@ in pkg.overrideAttrs (finalAttrs: o: { position = toString ./default.nix + ":1"; }; passthru = o.passthru // { - tests = { - help = runCommand "test-hercules-ci-agent-help" { } '' - (${finalAttrs.finalPackage}/bin/hercules-ci-agent --help 2>&1 || true) | grep -F -- '--config' - (${lib.getExe finalAttrs.finalPackage} --help 2>&1 || true) | grep -F -- '--config' - touch $out - ''; + version = testers.testVersion { + package = finalAttrs.finalPackage; + command = "hercules-ci-agent --help"; + }; } // lib.optionalAttrs (stdenv.isLinux) { # Does not test the package, but evaluation of the related NixOS module. nixos-simple-config = (nixos {