testers.testBuildFailure': init (#383511)

doc/build-helpers/testers.chapter.md

@@ -255,6 +255,70 @@ runCommand "example" {
 
 :::
 
+## `testBuildFailure'` {#tester-testBuildFailurePrime}
+
+This tester wraps the functionality provided by [`testers.testBuildFailure`](#tester-testBuildFailure) to make writing checks easier by simplifying checking the exit code of the builder and asserting the existence of entries in the builder's log.
+Additionally, users may specify a script containing additional checks, accessing the result of applying `testers.testBuildFailure` through the variable `failed`.
+
+NOTE: This tester will produce an empty output and exit with success if none of the checks fail; there is no need to `touch "$out"` in `script`.
+
+:::{.example #ex-testBuildFailurePrime-doc-example}
+
+# Check that a build fails, and verify the changes made during build
+
+Re-using the example from [`testers.testBuildFailure`](#ex-testBuildFailure-showingenvironmentchanges), we can see how common checks are made easier and remove the need for `runCommand`:
+
+```nix
+testers.testBuildFailure' {
+  drv = runCommand "doc-example" { } ''
+    echo ok-ish >"$out"
+    echo failing though
+    exit 3
+  '';
+  expectedBuilderExitCode = 3;
+  expectedBuilderLogEntries = [ "failing though" ];
+  script = ''
+    grep --silent -F 'ok-ish' "$failed/result"
+  '';
+}
+```
+
+:::
+
+### Inputs {#tester-testBuildFailurePrime-inputs}
+
+`drv` (derivation)
+
+: The failing derivation to wrap with `testBuildFailure`.
+
+`name` (string, optional)
+
+: The name of the test.
+  When not provided, this value defaults to `testBuildFailure-${(testers.testBuildFailure drv).name}`.
+
+`expectedBuilderExitCode` (integer, optional)
+
+: The expected exit code of the builder of `drv`.
+  When not provided, this value defaults to `1`.
+
+`expectedBuilderLogEntries` (array of string-like values, optional)
+
+: A list of string-like values which must be found in the builder's log by exact match.
+  When not provided, this value defaults to `[ ]`.
+
+  NOTE: Patterns and regular expressions are not supported.
+
+`script` (string, optional)
+
+: A string containing additional checks to run.
+  When not provided, this value defaults to `""`.
+  The result of `testers.testBuildFailure drv` is available through the variable `failed`.
+  As an example, the builder's log is at `"$failed/testBuildFailure.log"`.
+
+### Return value {#tester-testBuildFailurePrime-return}
+
+The tester produces an empty output and only succeeds when the checks using `expectedBuilderExitCode`, `expectedBuilderLogEntries`, and `script` succeed.
+
 ## `testEqualContents` {#tester-testEqualContents}
 
 Check that two paths have the same contents.

doc/redirects.json

@@ -8,6 +8,9 @@
   "ex-build-helpers-extendMkDerivation": [
     "index.html#ex-build-helpers-extendMkDerivation"
   ],
+  "ex-testBuildFailurePrime-doc-example": [
+    "index.html#ex-testBuildFailurePrime-doc-example"
+  ],
   "neovim": [
     "index.html#neovim"
   ],
@@ -332,6 +335,15 @@
   "footnote-stdenv-find-inputs-location.__back.0": [
     "index.html#footnote-stdenv-find-inputs-location.__back.0"
   ],
+  "tester-testBuildFailurePrime": [
+    "index.html#tester-testBuildFailurePrime"
+  ],
+  "tester-testBuildFailurePrime-inputs": [
+    "index.html#tester-testBuildFailurePrime-inputs"
+  ],
+  "tester-testBuildFailurePrime-return": [
+    "index.html#tester-testBuildFailurePrime-return"
+  ],
   "variables-specifying-dependencies": [
     "index.html#variables-specifying-dependencies"
   ],

pkgs/build-support/testers/default.nix

@@ -34,6 +34,10 @@
     ] ++ orig.args or ["-e" ../../stdenv/generic/source-stdenv.sh (orig.builder or ../../stdenv/generic/default-builder.sh)];
   });
 
+  # See https://nixos.org/manual/nixpkgs/unstable/#tester-testBuildFailurePrime
+  # or doc/build-helpers/testers.chapter.md
+  testBuildFailure' = callPackage ./testBuildFailurePrime { };
+
   # See https://nixos.org/manual/nixpkgs/unstable/#tester-testEqualDerivation
   # or doc/build-helpers/testers.chapter.md
   testEqualDerivation = callPackage ./test-equal-derivation.nix { };

pkgs/build-support/testers/test/default.nix

@@ -220,6 +220,10 @@ lib.recurseIntoAttrs {
     sideEffectStructuredAttrs = overrideStructuredAttrs true sideEffects;
   };
 
+  testBuildFailure' = lib.recurseIntoAttrs (
+    pkgs.callPackages ../testBuildFailurePrime/tests.nix { inherit overrideStructuredAttrs; }
+  );
+
   testEqualContents = lib.recurseIntoAttrs {
     equalDir = testers.testEqualContents {
       assertion = "The same directory contents at different paths are recognized as equal";

pkgs/build-support/testers/testBuildFailurePrime/build-command.sh

@@ -0,0 +1,47 @@
+# shellcheck shell=bash
+
+set -eu
+
+declare -ag preScriptHooks=(testBuilderExitCode)
+# shellcheck disable=SC2154
+((${#expectedBuilderLogEntries[@]})) && preScriptHooks+=(testBuilderLogEntries)
+
+testBuilderExitCode() {
+  nixLog "checking original builder exit code"
+  local -ir builderExitCode=$(<"${failed:?}/testBuildFailure.exit")
+  # shellcheck disable=SC2154
+  if ((expectedBuilderExitCode == builderExitCode)); then
+    nixLog "original builder exit code matches expected value of $expectedBuilderExitCode"
+    return 0
+  else
+    nixErrorLog "original builder produced exit code $builderExitCode but was expected to produce $expectedBuilderExitCode"
+    return 1
+  fi
+}
+
+testBuilderLogEntries() {
+  nixLog "checking original builder log"
+  local -r builderLogEntries="$(<"${failed:?}/testBuildFailure.log")"
+  local -i shouldExit=0
+  local expectedBuilderLogEntry
+  for expectedBuilderLogEntry in "${expectedBuilderLogEntries[@]}"; do
+    if [[ ${builderLogEntries} == *"$expectedBuilderLogEntry"* ]]; then
+      nixLog "original builder log contains ${expectedBuilderLogEntry@Q}"
+    else
+      nixErrorLog "original builder log does not contain ${expectedBuilderLogEntry@Q}"
+      shouldExit=1
+    fi
+  done
+  return $shouldExit
+}
+
+scriptPhase() {
+  runHook preScript
+
+  runHook script
+
+  runHook postScript
+}
+
+runHook scriptPhase
+touch "${out:?}"

pkgs/build-support/testers/testBuildFailurePrime/default.nix

@@ -0,0 +1,37 @@
+{
+  lib,
+  stdenvNoCC,
+  testers,
+}:
+# See https://nixos.org/manual/nixpkgs/unstable/#tester-testBuildFailurePrime
+# or doc/build-helpers/testers.chapter.md
+lib.makeOverridable (
+  {
+    drv,
+    name ? "testBuildFailure-${drv.name}",
+    expectedBuilderExitCode ? 1,
+    expectedBuilderLogEntries ? [ ],
+    script ? "",
+  }:
+  stdenvNoCC.mkDerivation (finalAttrs: {
+    __structuredAttrs = true;
+    strictDeps = true;
+
+    inherit name;
+
+    nativeBuildInputs = [ finalAttrs.failed ];
+
+    failed = testers.testBuildFailure drv;
+
+    inherit expectedBuilderExitCode expectedBuilderLogEntries;
+
+    inherit script;
+
+    buildCommandPath = ./build-command.sh;
+
+    meta = {
+      description = "A wrapper around testers.testBuildFailure to simplify common use cases";
+      maintainers = [ lib.maintainers.connorbaker ];
+    };
+  })
+)

pkgs/build-support/testers/testBuildFailurePrime/tests.nix

@@ -0,0 +1,149 @@
+{
+  emptyDirectory,
+  hello,
+  lib,
+  overrideStructuredAttrs,
+  runCommand,
+  stdenvNoCC,
+  testers,
+}:
+let
+  inherit (lib.attrsets) recurseIntoAttrs;
+  final = {
+    # NOTE: This example is used in the docs.
+    # See https://nixos.org/manual/nixpkgs/unstable/#tester-testBuildFailurePrime
+    # or doc/build-helpers/testers.chapter.md
+    doc-example = testers.testBuildFailure' {
+      drv = runCommand "doc-example" { } ''
+        echo ok-ish >"$out"
+        echo failing though
+        exit 3
+      '';
+      expectedBuilderExitCode = 3;
+      expectedBuilderLogEntries = [ "failing though" ];
+      script = ''
+        grep --silent -F 'ok-ish' "$failed/result"
+      '';
+    };
+
+    happy = testers.testBuildFailure' {
+      drv = runCommand "happy" { } ''
+        echo ok-ish >$out
+
+        echo failing though
+        echo also stderr 1>&2
+        echo 'line\nwith-\bbackslashes'
+        printf "incomplete line - no newline"
+
+        exit 3
+      '';
+      expectedBuilderExitCode = 3;
+      expectedBuilderLogEntries = [
+        "failing though"
+        "also stderr"
+        ''line\nwith-\bbackslashes''
+        "incomplete line - no newline"
+      ];
+      script = ''
+        grep --silent -F 'ok-ish' "$failed/result"
+      '';
+    };
+
+    happyStructuredAttrs = overrideStructuredAttrs true final.happy;
+
+    helloDoesNotFail = testers.testBuildFailure' {
+      drv = testers.testBuildFailure hello;
+      expectedBuilderLogEntries = [
+        "testBuildFailure: The builder did not fail, but a failure was expected"
+      ];
+    };
+
+    multiOutput = testers.testBuildFailure' {
+      drv =
+        runCommand "multiOutput"
+          {
+            # dev will be the default output
+            outputs = [
+              "dev"
+              "doc"
+              "out"
+            ];
+          }
+          ''
+            echo i am failing
+            exit 1
+          '';
+      expectedBuilderLogEntries = [
+        "i am failing"
+      ];
+      script = ''
+        # Checking our note that dev is the default output
+        echo $failed/_ | grep -- '-dev/_' >/dev/null
+        echo 'All good.'
+      '';
+    };
+
+    multiOutputStructuredAttrs = overrideStructuredAttrs true final.multiOutput;
+
+    sideEffects = testers.testBuildFailure' {
+      drv = stdenvNoCC.mkDerivation {
+        name = "fail-with-side-effects";
+        src = emptyDirectory;
+
+        postHook = ''
+          echo touching side-effect...
+          # Assert that the side-effect doesn't exist yet...
+          # We're checking that this hook isn't run by expect-failure.sh
+          if [[ -e side-effect ]]; then
+            echo "side-effect already exists"
+            exit 1
+          fi
+          touch side-effect
+        '';
+
+        buildPhase = ''
+          echo i am failing
+          exit 1
+        '';
+      };
+      expectedBuilderLogEntries = [
+        "touching side-effect..."
+        "i am failing"
+      ];
+      script = ''
+        [[ ! -e side-effect ]]
+      '';
+    };
+
+    sideEffectsStructuredAttrs = overrideStructuredAttrs true final.sideEffects;
+
+    exitCodeNegativeTest = testers.testBuildFailure' {
+      drv = testers.testBuildFailure' {
+        drv = runCommand "exit-code" { } "exit 3";
+        # Default expected exit code is 1
+      };
+      expectedBuilderLogEntries = [
+        "ERROR: testBuilderExitCode: original builder produced exit code 3 but was expected to produce 1"
+      ];
+    };
+
+    exitCodeNegativeTestStructuredAttrs = overrideStructuredAttrs true final.exitCodeNegativeTest;
+
+    logNegativeTest = testers.testBuildFailure' {
+      drv = testers.testBuildFailure' {
+        drv = runCommand "exit-code" { } ''
+          nixLog "apples"
+          exit 3
+        '';
+        expectedBuilderExitCode = 3;
+        expectedBuilderLogEntries = [ "bees" ];
+      };
+      expectedBuilderLogEntries = [
+        "ERROR: testBuilderLogEntries: original builder log does not contain 'bees'"
+      ];
+    };
+
+    logNegativeTestStructuredAttrs = overrideStructuredAttrs true final.logNegativeTest;
+  };
+in
+recurseIntoAttrs final