lib.debug.throwTestFailures: init

`lib.debug.runTests` provides a unit test evaluator for Nix, but its
results are returned in a raw and difficult-to-read form.

Currently, different callers output the results in various ways:
`builtins.throw (builtins.toJSON failures)` and `builtins.throw ("Tests
failed: " + lib.generators.toPretty { } failures)` are both used.

This change adds a new `lib.debug.throwTestFailures` function which
displays the results nicely before throwing an exception (or returns
`null` if no failures are given), unifying these disparate call-sites.

First, each failing test is pretty-printed in a `trace` message:

```
trace: FAIL testDerivation:
  Expected: <derivation a>
    Result: <derivation b>
```

Then, an exception is thrown containing the number of tests that failed
(and their names), followed by the raw JSON of the results (for parity
with previous usage, and because `lib.generators.toPretty` sometimes
omits information that `builins.toJSON` includes):

```
error:
       … while evaluating the file '...':

       … caused by explicit throw
         at /nix/store/.../lib/debug.nix:528:7:
          527|       in
          528|       throw (
             |       ^
          529|         builtins.seq traceFailures (

       error: 1 tests failed:
       - testDerivation

       [{"expected":"/nix/store/xh7kyqp69mxkwspmi81a94m9xx74r8dr-a","name":"testDerivation","result":"/nix/store/503l84nir4zw57d1shfhai25bxxn16c6-b"}]
```

lib/debug.nix

@@ -16,6 +16,7 @@
 { lib }:
 let
   inherit (lib)
+    concatMapStringsSep
     isList
     isAttrs
     substring
@@ -23,6 +24,7 @@ let
     concatLists
     const
     elem
+    foldl'
     generators
     id
     mapAttrs
@@ -454,6 +456,129 @@ rec {
       )
     );
 
+  /**
+    Pretty-print a list of test failures.
+
+    This takes an attribute set containing `failures` (a list of test failures
+    produced by `runTests`) and pretty-prints each failing test, before
+    throwing an error containing the raw test data as JSON.
+
+    If the input list is empty, `null` is returned.
+
+    # Inputs
+
+    `failures`
+
+    : A list of test failures (produced `runTests`), each containing `name`,
+      `expected`, and `result` attributes.
+
+    # Type
+
+    ```
+    throwTestFailures :: {
+      failures = [
+        {
+          name :: String;
+          expected :: a;
+          result :: a;
+        }
+      ];
+    }
+    ->
+    null
+    ```
+
+    # Examples
+    :::{.example}
+
+    ## `lib.debug.throwTestFailures` usage example
+
+    ```nix
+    throwTestFailures {
+      failures = [
+        {
+          name = "testDerivation";
+          expected = derivation {
+            name = "a";
+            builder = "bash";
+            system = "x86_64-linux";
+          };
+          result = derivation {
+            name = "b";
+            builder = "bash";
+            system = "x86_64-linux";
+          };
+        }
+      ];
+    }
+    ->
+    trace: FAIL testDerivation:
+      Expected: <derivation a>
+        Result: <derivation b>
+
+    error:
+           … while evaluating the file '...':
+
+           … caused by explicit throw
+             at /nix/store/.../lib/debug.nix:528:7:
+              527|       in
+              528|       throw (
+                 |       ^
+              529|         builtins.seq traceFailures (
+
+           error: 1 tests failed:
+           - testDerivation
+
+           [{"expected":"/nix/store/xh7kyqp69mxkwspmi81a94m9xx74r8dr-a","name":"testDerivation","result":"/nix/store/503l84nir4zw57d1shfhai25bxxn16c6-b"}]
+    null
+    ```
+
+    :::
+  */
+  throwTestFailures =
+    {
+      failures,
+      description ? "tests",
+      ...
+    }:
+    if failures == [ ] then
+      null
+    else
+      let
+        toPretty =
+          value:
+          # Thanks to @Ma27 for this:
+          #
+          # > The `unsafeDiscardStringContext` is useful when the `toPretty`
+          # > stumbles upon a derivation that would be realized without it (I
+          # > ran into the problem when writing a test for a flake helper where
+          # > I creating a bunch of "mock" derivations for different systems
+          # > and Nix then tried to build those when the error-string got
+          # > forced).
+          #
+          # See: https://github.com/NixOS/nixpkgs/pull/416207#discussion_r2145942389
+          builtins.unsafeDiscardStringContext (generators.toPretty { allowPrettyValues = true; } value);
+
+        failureToPretty = failure: ''
+          FAIL ${toPretty failure.name}:
+          Expected:
+          ${toPretty failure.expected}
+
+          Result:
+          ${toPretty failure.result}
+        '';
+
+        traceFailures = foldl' (_accumulator: failure: traceVal (failureToPretty failure)) null failures;
+      in
+      throw (
+        builtins.seq traceFailures (
+          "${builtins.toString (builtins.length failures)} ${description} failed:\n- "
+          + (concatMapStringsSep "\n- " (failure: failure.name) failures)
+          + "\n\n"
+          + builtins.toJSON failures
+        )
+      );
+
   /**
     Create a test assuming that list elements are `true`.
 

lib/path/tests/unit.nix

@@ -15,7 +15,7 @@ let
   # This is not allowed generally, but we're in the tests here, so we'll allow ourselves.
   storeDirPath = /. + builtins.storeDir;
 
-  cases = lib.runTests {
+  failures = lib.runTests {
     # Test examples from the lib.path.append documentation
     testAppendExample1 = {
       expr = append /foo "bar/baz";
@@ -326,7 +326,6 @@ let
     };
   };
 in
-if cases == [ ] then
+lib.debug.throwTestFailures {
-  "Unit tests successful"
+  inherit failures;
-else
+}
-  throw "Path unit tests failed: ${lib.generators.toPretty { } cases}"

lib/tests/misc.nix

@@ -4741,8 +4741,6 @@ runTests {
     expected = "/non-existent/this/does/not/exist/for/real/please-dont-mess-with-your-local-fs/default.nix";
   };
 
-  # Tests for cross index utilities
-
   testRenameCrossIndexFrom = {
     expr = lib.renameCrossIndexFrom "pkgs" {
       pkgsBuildBuild = "dummy-build-build";
@@ -4819,4 +4817,31 @@ runTests {
     };
   };
 
+  testThrowTestFailuresEmpty = {
+    expr = lib.debug.throwTestFailures {
+      failures = [ ];
+    };
+
+    expected = null;
+  };
+
+  testThrowTestFailures = testingThrow (
+    lib.debug.throwTestFailures {
+      failures = [
+        {
+          name = "testDerivation";
+          expected = builtins.derivation {
+            name = "a";
+            builder = "bash";
+            system = "x86_64-linux";
+          };
+          result = builtins.derivation {
+            name = "b";
+            builder = "bash";
+            system = "x86_64-linux";
+          };
+        }
+      ];
+    }
+  );
 }

pkgs/test/kernel.nix

@@ -73,6 +73,7 @@ let
   };
 in
 
-lib.optional (failures != [ ]) (
+lib.debug.throwTestFailures {
-  throw "The following kernel unit tests failed: ${lib.generators.toPretty { } failures}"
+  inherit failures;
-)
+  description = "kernel unit tests";
+}

pkgs/test/systemd/nixos/default.nix

@@ -52,6 +52,7 @@ let
   };
 in
 
-lib.optional (failures != [ ]) (
+lib.debug.throwTestFailures {
-  throw "The following systemd unit tests failed: ${lib.generators.toPretty { } failures}"
+  inherit failures;
-)
+  description = "systemd unit tests";
+}