diff --git a/doc/build-helpers/dev-shell-tools.chapter.md b/doc/build-helpers/dev-shell-tools.chapter.md index 21636df8017b..0168ea39f7aa 100644 --- a/doc/build-helpers/dev-shell-tools.chapter.md +++ b/doc/build-helpers/dev-shell-tools.chapter.md @@ -27,3 +27,49 @@ devShellTools.valueToString (builtins.toFile "foo" "bar") devShellTools.valueToString false => "" ``` + +::: + +## `devShellTools.unstructuredDerivationInputEnv` {#sec-devShellTools-unstructuredDerivationInputEnv} + +Convert a set of derivation attributes (as would be passed to [`derivation`]) to a set of environment variables that can be used in a shell script. +This function does not support `__structuredAttrs`, but does support `passAsFile`. + +:::{.example} +## `unstructuredDerivationInputEnv` usage example + +```nix +devShellTools.unstructuredDerivationInputEnv { + drvAttrs = { + name = "foo"; + buildInputs = [ hello figlet ]; + builder = bash; + args = [ "-c" "${./builder.sh}" ]; + }; +} +=> { + name = "foo"; + buildInputs = "/nix/store/...-hello /nix/store/...-figlet"; + builder = "/nix/store/...-bash"; +} +``` + +Note that `args` is not included, because Nix does not added it to the builder process environment. + +::: + +## `devShellTools.derivationOutputEnv` {#sec-devShellTools-derivationOutputEnv} + +Takes the relevant parts of a derivation and returns a set of environment variables, that would be present in the derivation. + +:::{.example} +## `derivationOutputEnv` usage example + +```nix +let + pkg = hello; +in +devShellTools.derivationOutputEnv { outputList = pkg.outputs; outputMap = pkg; } +``` + +::: diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix index 96cc6215118c..b69b22c15b85 100644 --- a/nixos/tests/all-tests.nix +++ b/nixos/tests/all-tests.nix @@ -264,6 +264,7 @@ in { docker-rootless = handleTestOn ["aarch64-linux" "x86_64-linux"] ./docker-rootless.nix {}; docker-registry = handleTest ./docker-registry.nix {}; docker-tools = handleTestOn ["x86_64-linux"] ./docker-tools.nix {}; + docker-tools-nix-shell = runTest ./docker-tools-nix-shell.nix; docker-tools-cross = handleTestOn ["x86_64-linux" "aarch64-linux"] ./docker-tools-cross.nix {}; docker-tools-overlay = handleTestOn ["x86_64-linux"] ./docker-tools-overlay.nix {}; documize = handleTest ./documize.nix {}; diff --git a/nixos/tests/docker-tools-nix-shell.nix b/nixos/tests/docker-tools-nix-shell.nix new file mode 100644 index 000000000000..c2ae2124e0a1 --- /dev/null +++ b/nixos/tests/docker-tools-nix-shell.nix @@ -0,0 +1,95 @@ +# nix-build -A nixosTests.docker-tools-nix-shell +{ config, lib, ... }: +let + inherit (config.node.pkgs.dockerTools) examples; +in +{ + name = "docker-tools-nix-shell"; + meta = with lib.maintainers; { + maintainers = [ + infinisil + roberth + ]; + }; + + nodes = { + docker = + { ... }: + { + virtualisation = { + diskSize = 3072; + docker.enable = true; + }; + }; + }; + + testScript = '' + docker.wait_for_unit("sockets.target") + + with subtest("buildImageWithNixDB: Has a nix database"): + docker.succeed( + "docker load --input='${examples.nix}'", + "docker run --rm ${examples.nix.imageName} nix-store -q --references /bin/bash" + ) + + with subtest("buildNixShellImage: Can build a basic derivation"): + docker.succeed( + "${examples.nix-shell-basic} | docker load", + "docker run --rm nix-shell-basic bash -c 'buildDerivation && $out/bin/hello' | grep '^Hello, world!$'" + ) + + with subtest("buildNixShellImage: Runs the shell hook"): + docker.succeed( + "${examples.nix-shell-hook} | docker load", + "docker run --rm -it nix-shell-hook | grep 'This is the shell hook!'" + ) + + with subtest("buildNixShellImage: Sources stdenv, making build inputs available"): + docker.succeed( + "${examples.nix-shell-inputs} | docker load", + "docker run --rm -it nix-shell-inputs | grep 'Hello, world!'" + ) + + with subtest("buildNixShellImage: passAsFile works"): + docker.succeed( + "${examples.nix-shell-pass-as-file} | docker load", + "docker run --rm -it nix-shell-pass-as-file | grep 'this is a string'" + ) + + with subtest("buildNixShellImage: run argument works"): + docker.succeed( + "${examples.nix-shell-run} | docker load", + "docker run --rm -it nix-shell-run | grep 'This shell is not interactive'" + ) + + with subtest("buildNixShellImage: command argument works"): + docker.succeed( + "${examples.nix-shell-command} | docker load", + "docker run --rm -it nix-shell-command | grep 'This shell is interactive'" + ) + + with subtest("buildNixShellImage: home directory is writable by default"): + docker.succeed( + "${examples.nix-shell-writable-home} | docker load", + "docker run --rm -it nix-shell-writable-home" + ) + + with subtest("buildNixShellImage: home directory can be made non-existent"): + docker.succeed( + "${examples.nix-shell-nonexistent-home} | docker load", + "docker run --rm -it nix-shell-nonexistent-home" + ) + + with subtest("buildNixShellImage: can build derivations"): + docker.succeed( + "${examples.nix-shell-build-derivation} | docker load", + "docker run --rm -it nix-shell-build-derivation" + ) + + with subtest("streamLayeredImage: with nix db"): + docker.succeed( + "${examples.nix-layered} | docker load", + "docker run --rm ${examples.nix-layered.imageName} nix-store -q --references /bin/bash" + ) + ''; +} diff --git a/nixos/tests/docker-tools.nix b/nixos/tests/docker-tools.nix index 8c315ee731ea..41bd4a621545 100644 --- a/nixos/tests/docker-tools.nix +++ b/nixos/tests/docker-tools.nix @@ -60,7 +60,7 @@ let }; nonRootTestImage = - pkgs.dockerTools.streamLayeredImage rec { + pkgs.dockerTools.streamLayeredImage { name = "non-root-test"; tag = "latest"; uid = 1000; @@ -567,66 +567,6 @@ in { docker.succeed("docker run --rm image-with-certs:latest test -r /etc/pki/tls/certs/ca-bundle.crt") docker.succeed("docker image rm image-with-certs:latest") - with subtest("buildImageWithNixDB: Has a nix database"): - docker.succeed( - "docker load --input='${examples.nix}'", - "docker run --rm ${examples.nix.imageName} nix-store -q --references /bin/bash" - ) - - with subtest("buildNixShellImage: Can build a basic derivation"): - docker.succeed( - "${examples.nix-shell-basic} | docker load", - "docker run --rm nix-shell-basic bash -c 'buildDerivation && $out/bin/hello' | grep '^Hello, world!$'" - ) - - with subtest("buildNixShellImage: Runs the shell hook"): - docker.succeed( - "${examples.nix-shell-hook} | docker load", - "docker run --rm -it nix-shell-hook | grep 'This is the shell hook!'" - ) - - with subtest("buildNixShellImage: Sources stdenv, making build inputs available"): - docker.succeed( - "${examples.nix-shell-inputs} | docker load", - "docker run --rm -it nix-shell-inputs | grep 'Hello, world!'" - ) - - with subtest("buildNixShellImage: passAsFile works"): - docker.succeed( - "${examples.nix-shell-pass-as-file} | docker load", - "docker run --rm -it nix-shell-pass-as-file | grep 'this is a string'" - ) - - with subtest("buildNixShellImage: run argument works"): - docker.succeed( - "${examples.nix-shell-run} | docker load", - "docker run --rm -it nix-shell-run | grep 'This shell is not interactive'" - ) - - with subtest("buildNixShellImage: command argument works"): - docker.succeed( - "${examples.nix-shell-command} | docker load", - "docker run --rm -it nix-shell-command | grep 'This shell is interactive'" - ) - - with subtest("buildNixShellImage: home directory is writable by default"): - docker.succeed( - "${examples.nix-shell-writable-home} | docker load", - "docker run --rm -it nix-shell-writable-home" - ) - - with subtest("buildNixShellImage: home directory can be made non-existent"): - docker.succeed( - "${examples.nix-shell-nonexistent-home} | docker load", - "docker run --rm -it nix-shell-nonexistent-home" - ) - - with subtest("buildNixShellImage: can build derivations"): - docker.succeed( - "${examples.nix-shell-build-derivation} | docker load", - "docker run --rm -it nix-shell-build-derivation" - ) - with subtest("streamLayeredImage: chown is persistent in fakeRootCommands"): docker.succeed( "${chownTestImage} | docker load", @@ -638,11 +578,5 @@ in { "${nonRootTestImage} | docker load", "docker run --rm ${chownTestImage.imageName} | diff /dev/stdin <(echo 12345:12345)" ) - - with subtest("streamLayeredImage: with nix db"): - docker.succeed( - "${examples.nix-layered} | docker load", - "docker run --rm ${examples.nix-layered.imageName} nix-store -q --references /bin/bash" - ) ''; }) diff --git a/pkgs/build-support/dev-shell-tools/default.nix b/pkgs/build-support/dev-shell-tools/default.nix index cd5fa5f5937e..487be834727f 100644 --- a/pkgs/build-support/dev-shell-tools/default.nix +++ b/pkgs/build-support/dev-shell-tools/default.nix @@ -1,8 +1,13 @@ -{ lib }: +{ + lib, + writeTextFile, +}: let inherit (builtins) typeOf; in rec { + # Docs: doc/build-helpers/dev-shell-tools.chapter.md + # Tests: ./tests/default.nix # This function closely mirrors what this Nix code does: # https://github.com/NixOS/nix/blob/2.8.0/src/libexpr/primops.cc#L1102 # https://github.com/NixOS/nix/blob/2.8.0/src/libexpr/eval.cc#L1981-L2036 @@ -13,4 +18,48 @@ rec { if typeOf value == "path" then "${value}" else if typeOf value == "list" then toString (map valueToString value) else toString value; + + + # Docs: doc/build-helpers/dev-shell-tools.chapter.md + # Tests: ./tests/default.nix + # https://github.com/NixOS/nix/blob/2.8.0/src/libstore/build/local-derivation-goal.cc#L992-L1004 + unstructuredDerivationInputEnv = { drvAttrs }: + # FIXME: this should be `normalAttrs // passAsFileAttrs` + lib.mapAttrs' + (name: value: + let str = valueToString value; + in if lib.elem name (drvAttrs.passAsFile or []) + then + let + nameHash = + if builtins?convertHash + then builtins.convertHash { + hash = "sha256:" + builtins.hashString "sha256" name; + toHashFormat = "nix32"; + } + else + builtins.hashString "sha256" name; + basename = ".attr-${nameHash}"; + in + lib.nameValuePair "${name}Path" "${ + writeTextFile { + name = "shell-passAsFile-${name}"; + text = str; + destination = "/${basename}"; + } + }/${basename}" + else lib.nameValuePair name str + ) + (removeAttrs drvAttrs [ + # TODO: there may be more of these + "args" + ]); + + # Docs: doc/build-helpers/dev-shell-tools.chapter.md + # Tests: ./tests/default.nix + derivationOutputEnv = { outputList, outputMap }: + # A mapping from output name to the nix store path where they should end up + # https://github.com/NixOS/nix/blob/2.8.0/src/libexpr/primops.cc#L1253 + lib.genAttrs outputList (output: builtins.unsafeDiscardStringContext outputMap.${output}.outPath); + } diff --git a/pkgs/build-support/dev-shell-tools/tests/default.nix b/pkgs/build-support/dev-shell-tools/tests/default.nix index bfedc04409a9..06ef7e393e32 100644 --- a/pkgs/build-support/dev-shell-tools/tests/default.nix +++ b/pkgs/build-support/dev-shell-tools/tests/default.nix @@ -4,11 +4,23 @@ lib, stdenv, hello, + writeText, + zlib, + nixosTests, }: let - inherit (lib) escapeShellArg; + inherit (lib) + concatLines + escapeShellArg + isString + mapAttrsToList + ; in -{ +lib.recurseIntoAttrs { + + # nix-build -A tests.devShellTools.nixos + nixos = nixosTests.docker-tools-nix-shell; + # nix-build -A tests.devShellTools.valueToString valueToString = let inherit (devShellTools) valueToString; in @@ -42,4 +54,112 @@ in ) >log 2>&1 || { cat log; exit 1; } ''; }; + + # nix-build -A tests.devShellTools.valueToString + unstructuredDerivationInputEnv = + let + inherit (devShellTools) unstructuredDerivationInputEnv; + + drvAttrs = { + one = 1; + boolTrue = true; + boolFalse = false; + foo = "foo"; + list = [ 1 2 3 ]; + pathDefaultNix = ./default.nix; + stringWithDep = "Exe: ${hello}/bin/hello"; + aPackageAttrSet = hello; + anOutPath = hello.outPath; + anAnAlternateOutput = zlib.dev; + args = [ "args must not be added to the environment" "Nix doesn't do it." ]; + + passAsFile = [ "bar" ]; + bar = '' + bar + ${writeText "qux" "yea"} + ''; + }; + result = unstructuredDerivationInputEnv { inherit drvAttrs; }; + in + assert result // { barPath = ""; } == { + one = "1"; + boolTrue = "1"; + boolFalse = ""; + foo = "foo"; + list = "1 2 3"; + pathDefaultNix = "${./default.nix}"; + stringWithDep = "Exe: ${hello}/bin/hello"; + aPackageAttrSet = "${hello}"; + anOutPath = "${hello.outPath}"; + anAnAlternateOutput = "${zlib.dev}"; + + passAsFile = "bar"; + barPath = ""; + }; + + # Not runCommand, because it alters `passAsFile` + stdenv.mkDerivation ({ + name = "devShellTools-unstructuredDerivationInputEnv-built-tests"; + + exampleBarPathString = + assert isString result.barPath; + result.barPath; + + dontUnpack = true; + dontBuild = true; + dontFixup = true; + doCheck = true; + + installPhase = "touch $out"; + + checkPhase = '' + fail() { + echo "$@" >&2 + exit 1 + } + checkAttr() { + echo checking attribute $1... + if [[ "$2" != "$3" ]]; then + echo "expected: $3" + echo "actual: $2" + exit 1 + fi + } + ${ + concatLines + (mapAttrsToList + (name: value: + "checkAttr ${name} \"\$${name}\" ${escapeShellArg value}" + ) + (removeAttrs + result + [ + "args" + + # Nix puts it in workdir, which is not a concept for + # unstructuredDerivationInputEnv, so we have to put it in the + # store instead. This means the full path won't match. + "barPath" + ]) + ) + } + ( + set -x + + diff $exampleBarPathString $barPath + + ${lib.optionalString (builtins?convertHash) '' + [[ "$(basename $exampleBarPathString)" = "$(basename $barPath)" ]] + ''} + ) + + ''${args:+fail "args should not be set by Nix. We don't expect it to and unstructuredDerivationInputEnv removes it."} + if [[ "''${builder:-x}" == x ]]; then + fail "builder should be set by Nix. We don't remove it in unstructuredDerivationInputEnv." + fi + ''; + } // removeAttrs drvAttrs [ + # This would break the derivation. Instead, we have a check in the derivation to make sure Nix doesn't set it. + "args" + ]); } diff --git a/pkgs/build-support/docker/default.nix b/pkgs/build-support/docker/default.nix index 3c580fafe974..b06ed6149a18 100644 --- a/pkgs/build-support/docker/default.nix +++ b/pkgs/build-support/docker/default.nix @@ -1129,26 +1129,18 @@ rec { ); # This function streams a docker image that behaves like a nix-shell for a derivation + # Docs: doc/build-helpers/images/dockertools.section.md + # Tests: nixos/tests/docker-tools-nix-shell.nix streamNixShellImage = - { # The derivation whose environment this docker image should be based on - drv - , # Image Name - name ? drv.name + "-env" - , # Image tag, the Nix's output hash will be used if null - tag ? null - , # User id to run the container as. Defaults to 1000, because many - # binaries don't like to be run as root - uid ? 1000 - , # Group id to run the container as, see also uid - gid ? 1000 - , # The home directory of the user - homeDirectory ? "/build" - , # The path to the bash binary to use as the shell. See `NIX_BUILD_SHELL` in `man nix-shell` - shell ? bashInteractive + "/bin/bash" - , # Run this command in the environment of the derivation, in an interactive shell. See `--command` in `man nix-shell` - command ? null - , # Same as `command`, but runs the command in a non-interactive shell instead. See `--run` in `man nix-shell` - run ? null + { drv + , name ? drv.name + "-env" + , tag ? null + , uid ? 1000 + , gid ? 1000 + , homeDirectory ? "/build" + , shell ? bashInteractive + "/bin/bash" + , command ? null + , run ? null }: assert lib.assertMsg (! (drv.drvAttrs.__structuredAttrs or false)) "streamNixShellImage: Does not work with the derivation ${drv.name} because it uses __structuredAttrs"; @@ -1190,16 +1182,9 @@ rec { # https://github.com/NixOS/nix/blob/2.8.0/src/libstore/globals.hh#L464-L465 sandboxBuildDir = "/build"; - # https://github.com/NixOS/nix/blob/2.8.0/src/libstore/build/local-derivation-goal.cc#L992-L1004 - drvEnv = lib.mapAttrs' (name: value: - let str = valueToString value; - in if lib.elem name (drv.drvAttrs.passAsFile or []) - then lib.nameValuePair "${name}Path" (writeText "pass-as-text-${name}" str) - else lib.nameValuePair name str - ) drv.drvAttrs // - # A mapping from output name to the nix store path where they should end up - # https://github.com/NixOS/nix/blob/2.8.0/src/libexpr/primops.cc#L1253 - lib.genAttrs drv.outputs (output: builtins.unsafeDiscardStringContext drv.${output}.outPath); + drvEnv = + devShellTools.unstructuredDerivationInputEnv { inherit (drv) drvAttrs; } + // devShellTools.derivationOutputEnv { outputList = drv.outputs; outputMap = drv; }; # Environment variables set in the image envVars = { @@ -1291,6 +1276,8 @@ rec { }; # Wrapper around streamNixShellImage to build an image from the result + # Docs: doc/build-helpers/images/dockertools.section.md + # Tests: nixos/tests/docker-tools-nix-shell.nix buildNixShellImage = { drv, compressor ? "gz", ... }@args: let stream = streamNixShellImage (builtins.removeAttrs args ["compressor"]);