From 2b3eaf55afc0e2a77677e3b1522879d7a5361640 Mon Sep 17 00:00:00 2001 From: Someone Serge Date: Wed, 3 Jan 2024 23:53:51 +0000 Subject: [PATCH 1/5] nvidia-docker: unbreak the runc symlink (cherry picked from commit 1e1eb8ba6ee7e3428599a27b1b81e2e1be41b4fa) --- .../virtualization/nvidia-container-toolkit/default.nix | 8 ++++++++ pkgs/top-level/all-packages.nix | 2 +- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/pkgs/applications/virtualization/nvidia-container-toolkit/default.nix b/pkgs/applications/virtualization/nvidia-container-toolkit/default.nix index a174c3031227..71fc18f0b327 100644 --- a/pkgs/applications/virtualization/nvidia-container-toolkit/default.nix +++ b/pkgs/applications/virtualization/nvidia-container-toolkit/default.nix @@ -47,6 +47,14 @@ buildGoModule rec { nativeBuildInputs = [ makeWrapper ]; + preConfigure = '' + # Ensure the runc symlink isn't broken: + if ! readlink --quiet --canonicalize-existing "${isolatedContainerRuntimePath}/runc" ; then + echo "${isolatedContainerRuntimePath}/runc: broken symlink" >&2 + exit 1 + fi + ''; + checkFlags = let skippedTests = [ diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index ace5c084e77f..c3cd76e3afd9 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -24197,7 +24197,7 @@ with pkgs; nvidia-docker = mkNvidiaContainerPkg { name = "nvidia-docker"; - containerRuntimePath = "${docker}/libexec/docker/runc"; + containerRuntimePath = "${docker}/libexec/docker/docker"; configTemplate = ../applications/virtualization/nvidia-docker/config.toml; additionalPaths = [ (callPackage ../applications/virtualization/nvidia-docker { }) ]; }; From 88f438fc5f5d1194479b6bf52dd48c8ec1810464 Mon Sep 17 00:00:00 2001 From: Someone Serge Date: Fri, 5 Jan 2024 23:31:22 +0000 Subject: [PATCH 2/5] libnvidia-container: set mainProgram (cherry picked from commit 42ed2f841a08e8b35492a3ffd7b38e2863e5c957) --- pkgs/applications/virtualization/libnvidia-container/default.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/pkgs/applications/virtualization/libnvidia-container/default.nix b/pkgs/applications/virtualization/libnvidia-container/default.nix index 622ba090e0ee..b462b24711de 100644 --- a/pkgs/applications/virtualization/libnvidia-container/default.nix +++ b/pkgs/applications/virtualization/libnvidia-container/default.nix @@ -116,6 +116,7 @@ stdenv.mkDerivation rec { description = "NVIDIA container runtime library"; license = licenses.asl20; platforms = platforms.linux; + mainProgram = "nvidia-container-cli"; maintainers = with maintainers; [ cpcloud ]; }; } From 336e2218e04f86e5593cf91ed614fad6cd63698b Mon Sep 17 00:00:00 2001 From: Someone Serge Date: Wed, 3 Jan 2024 22:55:43 +0000 Subject: [PATCH 3/5] nvidiaCtkPackages: init ...this way we expose and allow overriding the symlinkJoin constituent components (cherry picked from commit 1142433e70881b546e2ca5db87792f08829f02ce) --- .../nvidia-container-toolkit/packages.nix | 40 +++++++++++++++++++ pkgs/top-level/all-packages.nix | 32 ++++----------- 2 files changed, 47 insertions(+), 25 deletions(-) create mode 100644 pkgs/applications/virtualization/nvidia-container-toolkit/packages.nix diff --git a/pkgs/applications/virtualization/nvidia-container-toolkit/packages.nix b/pkgs/applications/virtualization/nvidia-container-toolkit/packages.nix new file mode 100644 index 000000000000..ac7da5bd33ea --- /dev/null +++ b/pkgs/applications/virtualization/nvidia-container-toolkit/packages.nix @@ -0,0 +1,40 @@ +{ + lib, + newScope, + docker, + libnvidia-container, + runc, + symlinkJoin, +}: + +lib.makeScope newScope ( + self: { + + nvidia-container-toolkit-docker = self.callPackage ./. { + containerRuntimePath = "${docker}/libexec/docker/docker"; + configTemplate = ../nvidia-docker/config.toml; + }; + nvidia-container-toolkit-podman = self.nvidia-container-toolkit-docker.override { + containerRuntimePath = lib.getExe runc; + configTemplate = ../nvidia-podman/config.toml; + }; + + nvidia-docker = symlinkJoin { + name = "nvidia-docker"; + paths = [ + libnvidia-container + self.nvidia-docker-unwrapped + self.nvidia-container-toolkit-docker + ]; + }; + nvidia-docker-unwrapped = self.callPackage ../nvidia-docker { }; + + nvidia-podman = symlinkJoin { + name = "nvidia-podman"; + paths = [ + libnvidia-container + self.nvidia-container-toolkit-podman + ]; + }; + } +) diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index c3cd76e3afd9..db8eac8a101c 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -24182,31 +24182,13 @@ with pkgs; nv-codec-headers-11 = callPackage ../development/libraries/nv-codec-headers/11_x.nix { }; nv-codec-headers-12 = callPackage ../development/libraries/nv-codec-headers/12_x.nix { }; - mkNvidiaContainerPkg = { name, containerRuntimePath, configTemplate, additionalPaths ? [] }: - let - nvidia-container-toolkit = callPackage ../applications/virtualization/nvidia-container-toolkit { - inherit containerRuntimePath configTemplate; - }; - in symlinkJoin { - inherit name; - paths = [ - libnvidia-container - nvidia-container-toolkit - ] ++ additionalPaths; - }; - - nvidia-docker = mkNvidiaContainerPkg { - name = "nvidia-docker"; - containerRuntimePath = "${docker}/libexec/docker/docker"; - configTemplate = ../applications/virtualization/nvidia-docker/config.toml; - additionalPaths = [ (callPackage ../applications/virtualization/nvidia-docker { }) ]; - }; - - nvidia-podman = mkNvidiaContainerPkg { - name = "nvidia-podman"; - containerRuntimePath = "${runc}/bin/runc"; - configTemplate = ../applications/virtualization/nvidia-podman/config.toml; - }; + nvidiaCtkPackages = + callPackage ../applications/virtualization/nvidia-container-toolkit/packages.nix + { }; + inherit (nvidiaCtkPackages) + nvidia-docker + nvidia-podman + ; nvidia-vaapi-driver = lib.hiPrio (callPackage ../development/libraries/nvidia-vaapi-driver { }); From 5e7c297ba25bdaaf2806a6dfa5f436f2a3afb7d7 Mon Sep 17 00:00:00 2001 From: Someone Serge Date: Wed, 10 Jan 2024 18:43:07 +0000 Subject: [PATCH 4/5] nvidia-docker: support config.toml as an attrset argument --- .../nvidia-container-toolkit/default.nix | 10 ++++- .../nvidia-container-toolkit/packages.nix | 41 ++++++++++++++++++- .../virtualization/nvidia-docker/config.toml | 13 ------ .../virtualization/nvidia-podman/config.toml | 13 ------ 4 files changed, 48 insertions(+), 29 deletions(-) delete mode 100644 pkgs/applications/virtualization/nvidia-docker/config.toml delete mode 100644 pkgs/applications/virtualization/nvidia-podman/config.toml diff --git a/pkgs/applications/virtualization/nvidia-container-toolkit/default.nix b/pkgs/applications/virtualization/nvidia-container-toolkit/default.nix index 71fc18f0b327..7d0ecfab53e7 100644 --- a/pkgs/applications/virtualization/nvidia-container-toolkit/default.nix +++ b/pkgs/applications/virtualization/nvidia-container-toolkit/default.nix @@ -5,10 +5,16 @@ , buildGoModule , linkFarm , writeShellScript +, formats , containerRuntimePath , configTemplate +, configTemplatePath ? null , libnvidia-container }: + +assert configTemplate != null -> (lib.isAttrs configTemplate && configTemplatePath == null); +assert configTemplatePath != null -> (lib.isStringLike configTemplatePath && configTemplate == null); + let isolatedContainerRuntimePath = linkFarm "isolated_container_runtime_path" [ { @@ -23,6 +29,8 @@ let echo >&2 "$(tput setaf 3)warning: \$XDG_CONFIG_HOME=$XDG_CONFIG_HOME$(tput sgr 0)" fi ''; + + configToml = if configTemplatePath != null then configTemplatePath else (formats.toml { }).generate "config.toml" configTemplate; in buildGoModule rec { pname = "container-toolkit/container-toolkit"; @@ -82,7 +90,7 @@ buildGoModule rec { --prefix PATH : ${isolatedContainerRuntimePath}:${libnvidia-container}/bin \ --set-default XDG_CONFIG_HOME $out/etc - cp ${configTemplate} $out/etc/nvidia-container-runtime/config.toml + cp ${configToml} $out/etc/nvidia-container-runtime/config.toml substituteInPlace $out/etc/nvidia-container-runtime/config.toml \ --subst-var-by glibcbin ${lib.getBin glibc} diff --git a/pkgs/applications/virtualization/nvidia-container-toolkit/packages.nix b/pkgs/applications/virtualization/nvidia-container-toolkit/packages.nix index ac7da5bd33ea..2d1058a00211 100644 --- a/pkgs/applications/virtualization/nvidia-container-toolkit/packages.nix +++ b/pkgs/applications/virtualization/nvidia-container-toolkit/packages.nix @@ -7,16 +7,53 @@ symlinkJoin, }: +# Note this scope isn't recursed into, at the time of writing. lib.makeScope newScope ( self: { + # The config is only exposed as an attrset so that the user may reach the + # deafult values, for inspectability purposes. + dockerConfig = { + disable-require = false; + #swarm-resource = "DOCKER_RESOURCE_GPU" + + nvidia-container-cli = { + #root = "/run/nvidia/driver"; + #path = "/usr/bin/nvidia-container-cli"; + environment = [ ]; + #debug = "/var/log/nvidia-container-runtime-hook.log"; + ldcache = "/tmp/ld.so.cache"; + load-kmods = true; + #no-cgroups = false; + #user = "root:video"; + ldconfig = "@@glibcbin@/bin/ldconfig"; + }; + }; nvidia-container-toolkit-docker = self.callPackage ./. { containerRuntimePath = "${docker}/libexec/docker/docker"; - configTemplate = ../nvidia-docker/config.toml; + configTemplate = self.dockerConfig; + }; + + podmanConfig = { + disable-require = true; + #swarm-resource = "DOCKER_RESOURCE_GPU"; + + nvidia-container-cli = { + #root = "/run/nvidia/driver"; + #path = "/usr/bin/nvidia-container-cli"; + environment = [ ]; + #debug = "/var/log/nvidia-container-runtime-hook.log"; + ldcache = "/tmp/ld.so.cache"; + load-kmods = true; + no-cgroups = true; + #user = "root:video"; + ldconfig = "@@glibcbin@/bin/ldconfig"; + }; }; nvidia-container-toolkit-podman = self.nvidia-container-toolkit-docker.override { containerRuntimePath = lib.getExe runc; - configTemplate = ../nvidia-podman/config.toml; + + configTemplate = self.podmanConfig; }; nvidia-docker = symlinkJoin { diff --git a/pkgs/applications/virtualization/nvidia-docker/config.toml b/pkgs/applications/virtualization/nvidia-docker/config.toml deleted file mode 100644 index bbd166995f36..000000000000 --- a/pkgs/applications/virtualization/nvidia-docker/config.toml +++ /dev/null @@ -1,13 +0,0 @@ -disable-require = false -#swarm-resource = "DOCKER_RESOURCE_GPU" - -[nvidia-container-cli] -#root = "/run/nvidia/driver" -#path = "/usr/bin/nvidia-container-cli" -environment = [] -#debug = "/var/log/nvidia-container-runtime-hook.log" -ldcache = "/tmp/ld.so.cache" -load-kmods = true -#no-cgroups = false -#user = "root:video" -ldconfig = "@@glibcbin@/bin/ldconfig" diff --git a/pkgs/applications/virtualization/nvidia-podman/config.toml b/pkgs/applications/virtualization/nvidia-podman/config.toml deleted file mode 100644 index eb39699b96b3..000000000000 --- a/pkgs/applications/virtualization/nvidia-podman/config.toml +++ /dev/null @@ -1,13 +0,0 @@ -disable-require = true -#swarm-resource = "DOCKER_RESOURCE_GPU" - -[nvidia-container-cli] -#root = "/run/nvidia/driver" -#path = "/usr/bin/nvidia-container-cli" -environment = [] -#debug = "/var/log/nvidia-container-runtime-hook.log" -ldcache = "/tmp/ld.so.cache" -load-kmods = true -no-cgroups = true -#user = "root:video" -ldconfig = "@@glibcbin@/bin/ldconfig" From 4160504219078cd25ab4c6a7785e0a10a57b8500 Mon Sep 17 00:00:00 2001 From: Someone Serge Date: Wed, 10 Jan 2024 18:43:16 +0000 Subject: [PATCH 5/5] nvidia-docker: add missing meta --- .../virtualization/nvidia-container-toolkit/packages.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkgs/applications/virtualization/nvidia-container-toolkit/packages.nix b/pkgs/applications/virtualization/nvidia-container-toolkit/packages.nix index 2d1058a00211..0ce76d5aed31 100644 --- a/pkgs/applications/virtualization/nvidia-container-toolkit/packages.nix +++ b/pkgs/applications/virtualization/nvidia-container-toolkit/packages.nix @@ -63,6 +63,7 @@ lib.makeScope newScope ( self.nvidia-docker-unwrapped self.nvidia-container-toolkit-docker ]; + inherit (self.nvidia-docker-unwrapped) meta; }; nvidia-docker-unwrapped = self.callPackage ../nvidia-docker { }; @@ -72,6 +73,7 @@ lib.makeScope newScope ( libnvidia-container self.nvidia-container-toolkit-podman ]; + inherit (self.nvidia-container-toolkit-podman) meta; }; } )