rnhmjoj
1e4eb2df3f
This service is essentially a random sleep() call that old software that is unable to properly handle modern hardware initialisation being asynchronous can use to *mostly* get by. Over the years is has caused so many issues in NixOS, like introducing long delays in the boot process[1], complete deadlocks[2] and even failures when reloading services or activating a new configuration[3]. systemd has been discouraging its use since 2011 (15 years ago!), then it officially deprecated it in 2018 and since 2020 it started to show on every boot a huge wall of text calling out all the offending services that still use it. Around 2021 we managed to fix around 15 NixOS modules[4] that were relying on systemd-udev-settle and practically ZFS[5] was the only remaining one. However, since then, people have actually started to bring it back with new services[6][7][8]. This is not acceptable. So, to prevent any more (lazy) uses of systemd-udev-settle, we stop providing the systemd-udev-settle.service unit entirely. For existing modules that unfortunately still need it, we replace it with the command `udevadm settle --timeout=180`, which is all that the service does. Hopefully this will also increase the awareness that it's bad and something to be fixed. Note: I tested this change using - `nixosTests.zfs`, - `nixosTests.ifstate`, - `nixosTests.misc`, - `nixosTests.openvswitch`, - `nixosTests.predictable-interface-names` - `nixosTests.nvidia-container-toolkit` and making sure that none of the 165 packages that provide upstream units (via the `systemd.packages` option) had a dependency on systemd-udev-settle.service. [1]: https://github.com/NixOS/nixpkgs/pull/25311 [2]: https://github.com/NixOS/nixpkgs/issues/107341 [3]: https://github.com/NixOS/nixpkgs/pull/113804 [4]: https://github.com/NixOS/nixpkgs/issues/73095 [5]: https://github.com/openzfs/zfs/issues/10891 [6]: https://github.com/NixOS/nixpkgs/pull/257525#discussion_r1442702970 [7]: https://github.com/NixOS/nixpkgs/pull/460075 [8]: https://github.com/NixOS/nixpkgs/pull/284507
360 lines
13 KiB
Nix
360 lines
13 KiB
Nix
{
|
|
config,
|
|
lib,
|
|
pkgs,
|
|
...
|
|
}:
|
|
|
|
{
|
|
imports = [
|
|
(lib.mkRenamedOptionModule
|
|
[ "virtualisation" "containers" "cdi" "dynamic" "nvidia" "enable" ]
|
|
[ "hardware" "nvidia-container-toolkit" "enable" ]
|
|
)
|
|
];
|
|
|
|
options =
|
|
let
|
|
mountType = {
|
|
options = {
|
|
hostPath = lib.mkOption {
|
|
type = lib.types.str;
|
|
description = "Host path.";
|
|
};
|
|
containerPath = lib.mkOption {
|
|
type = lib.types.str;
|
|
description = "Container path.";
|
|
};
|
|
mountOptions = lib.mkOption {
|
|
default = [
|
|
"ro"
|
|
"nosuid"
|
|
"nodev"
|
|
"bind"
|
|
];
|
|
type = lib.types.listOf lib.types.str;
|
|
description = "Mount options.";
|
|
};
|
|
};
|
|
};
|
|
in
|
|
{
|
|
hardware.nvidia-container-toolkit = {
|
|
enable = lib.mkOption {
|
|
default = false;
|
|
type = lib.types.bool;
|
|
description = ''
|
|
Enable dynamic CDI configuration for Nvidia devices by running
|
|
nvidia-container-toolkit on boot.
|
|
'';
|
|
};
|
|
|
|
device-name-strategy = lib.mkOption {
|
|
default = "index";
|
|
type = lib.types.enum [
|
|
"index"
|
|
"uuid"
|
|
"type-index"
|
|
];
|
|
description = ''
|
|
Specify the strategy for generating device names,
|
|
passed to `nvidia-ctk cdi generate`. This will affect how
|
|
you reference the device using `nvidia.com/gpu=` in
|
|
the container runtime.
|
|
'';
|
|
};
|
|
|
|
discovery-mode = lib.mkOption {
|
|
default = "auto";
|
|
type = lib.types.enum [
|
|
"auto"
|
|
"csv"
|
|
"nvml"
|
|
"wsl"
|
|
];
|
|
description = ''
|
|
The mode to use when discovering the available entities.
|
|
'';
|
|
};
|
|
|
|
csv-files = lib.mkOption {
|
|
default = [ ];
|
|
type = lib.types.listOf lib.types.path;
|
|
description = ''
|
|
The path to the list of CSV files to use when generating the CDI specification in CSV mode.
|
|
'';
|
|
};
|
|
|
|
mounts = lib.mkOption {
|
|
type = lib.types.listOf (lib.types.submodule mountType);
|
|
default = [ ];
|
|
description = "Mounts to be added to every container under the Nvidia CDI profile.";
|
|
};
|
|
|
|
mount-nvidia-executables = lib.mkOption {
|
|
default = true;
|
|
type = lib.types.bool;
|
|
description = ''
|
|
Mount executables nvidia-smi, nvidia-cuda-mps-control, nvidia-cuda-mps-server,
|
|
nvidia-debugdump, nvidia-powerd and nvidia-ctk on containers.
|
|
'';
|
|
};
|
|
|
|
mount-nvidia-docker-1-directories = lib.mkOption {
|
|
default = true;
|
|
type = lib.types.bool;
|
|
description = ''
|
|
Mount nvidia-docker-1 directories on containers: /usr/local/nvidia/lib and
|
|
/usr/local/nvidia/lib64.
|
|
'';
|
|
};
|
|
|
|
suppressNvidiaDriverAssertion = lib.mkOption {
|
|
default = false;
|
|
type = lib.types.bool;
|
|
description = ''
|
|
Suppress the assertion for installing Nvidia driver.
|
|
Useful in WSL where drivers are mounted from Windows, not provided by NixOS.
|
|
'';
|
|
};
|
|
|
|
package = lib.mkPackageOption pkgs "nvidia-container-toolkit" { };
|
|
|
|
disable-hooks = lib.mkOption {
|
|
type = lib.types.listOf lib.types.nonEmptyStr;
|
|
default = [ "create-symlinks" ];
|
|
description = ''
|
|
List of hooks to disable when generating the CDI specification.
|
|
Each hook name will be passed as `--disable-hook <hook-name>` to nvidia-ctk.
|
|
Set to an empty list to disable no hooks.
|
|
'';
|
|
};
|
|
|
|
enable-hooks = lib.mkOption {
|
|
type = lib.types.listOf lib.types.nonEmptyStr;
|
|
default = [ ];
|
|
description = ''
|
|
List of hooks to enable when generating the CDI specification.
|
|
Each hook name will be passed as `--enable-hook <hook-name>` to nvidia-ctk.
|
|
Set to an empty list to enable no hooks.
|
|
'';
|
|
};
|
|
|
|
extraArgs = lib.mkOption {
|
|
type = lib.types.listOf lib.types.str;
|
|
default = [ ];
|
|
description = ''
|
|
Extra arguments to be passed to nvidia-ctk.
|
|
'';
|
|
};
|
|
};
|
|
};
|
|
|
|
config = lib.mkMerge [
|
|
(lib.mkIf (config.virtualisation.docker.enableNvidia || config.virtualisation.podman.enableNvidia) {
|
|
hardware.nvidia-container-toolkit.enable = lib.mkDefault true;
|
|
|
|
warnings = lib.mkIf (!config.hardware.nvidia-container-toolkit.enable) [
|
|
''
|
|
`virtualisation.docker.enableNvidia` or `virtualisation.podman.enableNvidia` is enabled,
|
|
but `hardware.nvidia-container-toolkit.enable` is disabled. The nvidia-container-toolkit
|
|
module is required for GPU support in containers.
|
|
''
|
|
];
|
|
})
|
|
(lib.mkIf config.virtualisation.docker.enableNvidia {
|
|
environment.etc."nvidia-container-runtime/config.toml".text = ''
|
|
disable-require = true
|
|
supported-driver-capabilities = "compat32,compute,display,graphics,ngx,utility,video"
|
|
[nvidia-container-cli]
|
|
environment = []
|
|
ldconfig = "@${lib.getExe' pkgs.glibc "ldconfig"}"
|
|
load-kmods = true
|
|
no-cgroups = false
|
|
path = "${lib.getExe' pkgs.libnvidia-container "nvidia-container-cli"}"
|
|
[nvidia-container-runtime]
|
|
mode = "cdi"
|
|
runtimes = ["docker-runc", "runc", "crun"]
|
|
[nvidia-container-runtime-hook]
|
|
path = "${lib.getOutput "tools" config.hardware.nvidia-container-toolkit.package}/bin/nvidia-container-runtime-hook"
|
|
skip-mode-detection = false
|
|
[nvidia-ctk]
|
|
path = "${lib.getExe' config.hardware.nvidia-container-toolkit.package "nvidia-ctk"}"
|
|
'';
|
|
|
|
virtualisation.docker = {
|
|
daemon.settings = {
|
|
default-runtime = "nvidia";
|
|
runtimes.nvidia = {
|
|
path = "${lib.getOutput "tools" config.hardware.nvidia-container-toolkit.package}/bin/nvidia-container-runtime";
|
|
args = [ ];
|
|
};
|
|
};
|
|
|
|
extraPackages = [
|
|
(lib.getOutput "tools" config.hardware.nvidia-container-toolkit.package)
|
|
];
|
|
};
|
|
})
|
|
(lib.mkIf config.hardware.nvidia-container-toolkit.enable {
|
|
assertions = [
|
|
{
|
|
assertion =
|
|
config.hardware.nvidia.datacenter.enable
|
|
|| lib.elem "nvidia" config.services.xserver.videoDrivers
|
|
|| config.hardware.nvidia-container-toolkit.suppressNvidiaDriverAssertion;
|
|
message = ''`nvidia-container-toolkit` requires nvidia drivers: set `hardware.nvidia.datacenter.enable`, add "nvidia" to `services.xserver.videoDrivers`, or set `hardware.nvidia-container-toolkit.suppressNvidiaDriverAssertion` if the driver is provided by another NixOS module (e.g. from NixOS-WSL)'';
|
|
}
|
|
{
|
|
assertion =
|
|
((builtins.length config.hardware.nvidia-container-toolkit.csv-files) > 0)
|
|
-> config.hardware.nvidia-container-toolkit.discovery-mode == "csv";
|
|
message = "When CSV files are provided, `config.hardware.nvidia-container-toolkit.discovery-mode` has to be set to `csv`.";
|
|
}
|
|
];
|
|
|
|
warnings = lib.mkMerge [
|
|
(lib.mkIf config.virtualisation.podman.enableNvidia [
|
|
"Setting virtualisation.podman.enableNvidia has no effect and will be removed soon."
|
|
])
|
|
];
|
|
|
|
services.udev.extraRules = ''
|
|
KERNEL=="nvidia", RUN+="${lib.getExe' config.systemd.package "systemctl"} --no-block restart nvidia-container-toolkit-cdi-generator.service'"
|
|
'';
|
|
|
|
virtualisation = {
|
|
containers.containersConf.settings = {
|
|
engine = {
|
|
cdi_spec_dirs = [
|
|
"/etc/cdi"
|
|
"/var/run/cdi"
|
|
];
|
|
};
|
|
};
|
|
docker =
|
|
let
|
|
dockerVersion = config.virtualisation.docker.package.version;
|
|
in
|
|
{
|
|
daemon.settings = lib.mkIf (lib.versionAtLeast dockerVersion "25") {
|
|
features.cdi = true;
|
|
};
|
|
|
|
rootless = {
|
|
daemon.settings = lib.mkIf (lib.versionAtLeast dockerVersion "25") {
|
|
features.cdi = true;
|
|
};
|
|
|
|
extraPackages = [
|
|
(lib.getOutput "tools" config.hardware.nvidia-container-toolkit.package)
|
|
];
|
|
};
|
|
};
|
|
};
|
|
|
|
hardware = {
|
|
graphics.enable = lib.mkIf (!config.hardware.nvidia.datacenter.enable) true;
|
|
|
|
nvidia-container-toolkit.mounts =
|
|
let
|
|
nvidia-driver = config.hardware.nvidia.package;
|
|
in
|
|
(lib.mkMerge [
|
|
[
|
|
{
|
|
hostPath = pkgs.addDriverRunpath.driverLink;
|
|
containerPath = pkgs.addDriverRunpath.driverLink;
|
|
}
|
|
{
|
|
hostPath = "${lib.getLib nvidia-driver}";
|
|
containerPath = "${lib.getLib nvidia-driver}";
|
|
}
|
|
{
|
|
hostPath = "${lib.getLib pkgs.glibc}/lib";
|
|
containerPath = "${lib.getLib pkgs.glibc}/lib";
|
|
}
|
|
{
|
|
hostPath = "${lib.getLib pkgs.glibc}/lib64";
|
|
containerPath = "${lib.getLib pkgs.glibc}/lib64";
|
|
}
|
|
]
|
|
(lib.mkIf config.hardware.nvidia-container-toolkit.mount-nvidia-executables [
|
|
{
|
|
hostPath = lib.getExe' nvidia-driver "nvidia-cuda-mps-control";
|
|
containerPath = "/usr/bin/nvidia-cuda-mps-control";
|
|
}
|
|
{
|
|
hostPath = lib.getExe' nvidia-driver "nvidia-cuda-mps-server";
|
|
containerPath = "/usr/bin/nvidia-cuda-mps-server";
|
|
}
|
|
{
|
|
hostPath = lib.getExe' nvidia-driver "nvidia-debugdump";
|
|
containerPath = "/usr/bin/nvidia-debugdump";
|
|
}
|
|
{
|
|
hostPath = lib.getExe' nvidia-driver "nvidia-powerd";
|
|
containerPath = "/usr/bin/nvidia-powerd";
|
|
}
|
|
{
|
|
hostPath = lib.getExe' nvidia-driver "nvidia-smi";
|
|
containerPath = "/usr/bin/nvidia-smi";
|
|
}
|
|
])
|
|
# nvidia-docker 1.0 uses /usr/local/nvidia/lib{,64}
|
|
# e.g.
|
|
# - https://gitlab.com/nvidia/container-images/cuda/-/blob/e3ff10eab3a1424fe394899df0e0f8ca5a410f0f/dist/12.3.1/ubi9/base/Dockerfile#L44
|
|
# - https://github.com/NVIDIA/nvidia-docker/blob/01d2c9436620d7dde4672e414698afe6da4a282f/src/nvidia/volumes.go#L104-L173
|
|
(lib.mkIf config.hardware.nvidia-container-toolkit.mount-nvidia-docker-1-directories [
|
|
{
|
|
hostPath = "${lib.getLib nvidia-driver}/lib";
|
|
containerPath = "/usr/local/nvidia/lib";
|
|
}
|
|
{
|
|
hostPath = "${lib.getLib nvidia-driver}/lib";
|
|
containerPath = "/usr/local/nvidia/lib64";
|
|
}
|
|
])
|
|
]);
|
|
};
|
|
|
|
systemd.services.nvidia-container-toolkit-cdi-generator = {
|
|
description = "Container Device Interface (CDI) for Nvidia generator";
|
|
requiredBy = lib.mkMerge [
|
|
(lib.mkIf config.virtualisation.docker.enable [ "docker.service" ])
|
|
(lib.mkIf config.virtualisation.podman.enable [ "podman.service" ])
|
|
];
|
|
wantedBy = [ "multi-user.target" ];
|
|
serviceConfig = {
|
|
RuntimeDirectory = "cdi";
|
|
RemainAfterExit = true;
|
|
# We wait for the udev events queue to empty in the *hope* that the
|
|
# devices needed here become available. This is terribly broken and
|
|
# essentially no better than a random sleep(). See PR #452645 for
|
|
# an attempt to fix this issue.
|
|
ExecStartPre = "-${lib.getExe' pkgs.systemd "udevadm"} settle --timeout=180";
|
|
ExecStart =
|
|
let
|
|
script = pkgs.callPackage ./cdi-generate.nix {
|
|
inherit (config.hardware.nvidia-container-toolkit)
|
|
csv-files
|
|
device-name-strategy
|
|
discovery-mode
|
|
mounts
|
|
disable-hooks
|
|
enable-hooks
|
|
extraArgs
|
|
;
|
|
nvidia-container-toolkit = config.hardware.nvidia-container-toolkit.package;
|
|
nvidia-driver = config.hardware.nvidia.package;
|
|
};
|
|
in
|
|
lib.getExe script;
|
|
Type = "oneshot";
|
|
};
|
|
};
|
|
})
|
|
];
|
|
|
|
}
|