From 44d0f3783387cdd7dfe05a32ff9c623eeebd0b57 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sun, 16 Oct 2022 09:57:50 +0200 Subject: [PATCH 1/3] testers.testBuildFailure: init --- doc/builders/testers.chapter.md | 40 +++++++++++++ pkgs/build-support/testers/default.nix | 12 +++- pkgs/build-support/testers/expect-failure.sh | 62 ++++++++++++++++++++ pkgs/build-support/testers/test/default.nix | 51 +++++++++++++++- 4 files changed, 163 insertions(+), 2 deletions(-) create mode 100644 pkgs/build-support/testers/expect-failure.sh diff --git a/doc/builders/testers.chapter.md b/doc/builders/testers.chapter.md index ad1e1036d508..57f66a428482 100644 --- a/doc/builders/testers.chapter.md +++ b/doc/builders/testers.chapter.md @@ -35,6 +35,46 @@ passthru.tests.version = testers.testVersion { }; ``` +## `testBuildFailure` {#tester-testBuildFailure} + +Make sure that a build does not succeed. This is useful for testing testers. + +This returns a derivation with an override on the builder, with the following effects: + + - Fail the build when the original builder succeeds + - Move `$out` to `$out/result`, if it exists (assuming `out` is the default output) + - Save the build log to `$out/testBuildFailure.log` (same) + +Example: + +```nix +runCommand "example" { + failed = testers.testBuildFailure (runCommand "fail" {} '' + echo ok-ish >$out + echo failing though + exit 3 + ''); +} '' + grep -F 'ok-ish' $failed/result + grep -F 'failing though' $failed/testBuildFailure.log + [[ 3 = $(cat $failed/testBuildFailure.exit) ]] + touch $out +''; +``` + +While `testBuildFailure` is designed to keep changes to the original builder's +environment to a minimum, some small changes are inevitable. + + - The file `$TMPDIR/testBuildFailure.log` is present. It should not be deleted. + - `stdout` and `stderr` are a pipe instead of a tty. This could be improved. + - One or two extra processes are present in the sandbox during the original + builder's execution. + - The derivation and output hashes are different, but not unusual. + - The derivation includes a dependency on `buildPackages.bash` and + `expect-failure.sh`, which is built to include a transitive dependency on + `buildPackages.coreutils` and possibly more. These are not added to `PATH` + or any other environment variable, so they should be hard to observe. + ## `testEqualDerivation` {#tester-testEqualDerivation} Checks that two packages produce the exact same build instructions. diff --git a/pkgs/build-support/testers/default.nix b/pkgs/build-support/testers/default.nix index 7244d3d38575..fd08e9c6c47f 100644 --- a/pkgs/build-support/testers/default.nix +++ b/pkgs/build-support/testers/default.nix @@ -1,6 +1,16 @@ -{ pkgs, lib, callPackage, runCommand, stdenv }: +{ pkgs, buildPackages, lib, callPackage, runCommand, stdenv, substituteAll, }: # Documentation is in doc/builders/testers.chapter.md { + # See https://nixos.org/manual/nixpkgs/unstable/#tester-testBuildFailure + # or doc/builders/testers.chapter.md + testBuildFailure = drv: drv.overrideAttrs (orig: { + builder = buildPackages.bash; + args = [ + (substituteAll { coreutils = buildPackages.coreutils; src = ./expect-failure.sh; }) + orig.realBuilder or stdenv.shell + ] ++ orig.args or ["-e" (orig.builder or ../../stdenv/generic/default-builder.sh)]; + }); + testEqualDerivation = callPackage ./test-equal-derivation.nix { }; testVersion = diff --git a/pkgs/build-support/testers/expect-failure.sh b/pkgs/build-support/testers/expect-failure.sh new file mode 100644 index 000000000000..23e8698cf479 --- /dev/null +++ b/pkgs/build-support/testers/expect-failure.sh @@ -0,0 +1,62 @@ +# Run a builder, flip exit code, save log and fix outputs +# +# Sub-goals: +# - Delegate to another original builder passed via args +# - Save the build log to output for further checks +# - Make the derivation succeed if the original builder fails +# - Make the derivation fail if the original builder returns exit code 0 +# +# Requirements: +# This runs before, without and after stdenv. Do not modify the environment; +# especially not before invoking the original builder. For example, use +# "@" substitutions instead of PATH. +# Do not export any variables. + +# Stricter bash +set -eu + +# ------------------------ +# Run the original builder + +echo "testBuildFailure: Expecting non-zero exit from builder and args: ${*@Q}" + +("$@" 2>&1) | @coreutils@/bin/tee $TMPDIR/testBuildFailure.log \ + | while read ln; do + echo "original builder: $ln" + done + +r=${PIPESTATUS[0]} +if [[ $r = 0 ]]; then + echo "testBuildFailure: The builder did not fail, but a failure was expected!" + exit 1 +fi +echo "testBuildFailure: Original builder produced exit code: $r" + +# ----------------------------------------- +# Write the build log to the default output + +outs=( $outputs ) +defOut=${outs[0]} +defOutPath=${!defOut} + +if [[ ! -d $defOutPath ]]; then + if [[ -e $defOutPath ]]; then + @coreutils@/bin/mv $defOutPath $TMPDIR/out-node + @coreutils@/bin/mkdir $defOutPath + @coreutils@/bin/mv $TMPDIR/out-node $defOutPath/result + fi +fi + +@coreutils@/bin/mkdir -p $defOutPath +@coreutils@/bin/mv $TMPDIR/testBuildFailure.log $defOutPath/testBuildFailure.log +echo $r >$defOutPath/testBuildFailure.exit + +# ------------------------------------------------------ +# Put empty directories in place for any missing outputs + +for outputName in ${outputs:-out}; do + outputPath="${!outputName}" + if [[ ! -e "${outputPath}" ]]; then + @coreutils@/bin/mkdir "${outputPath}"; + fi +done diff --git a/pkgs/build-support/testers/test/default.nix b/pkgs/build-support/testers/test/default.nix index 30e778cf652e..22869baae159 100644 --- a/pkgs/build-support/testers/test/default.nix +++ b/pkgs/build-support/testers/test/default.nix @@ -1,4 +1,4 @@ -{ testers, lib, pkgs, ... }: +{ testers, lib, pkgs, hello, runCommand, ... }: let pkgs-with-overlay = pkgs.extend(final: prev: { proof-of-overlay-hello = prev.hello; @@ -24,4 +24,53 @@ lib.recurseIntoAttrs { machine.succeed("hello | figlet >/dev/console") ''; }); + + testBuildFailure = lib.recurseIntoAttrs { + happy = runCommand "testBuildFailure-happy" { + failed = testers.testBuildFailure (runCommand "fail" {} '' + echo ok-ish >$out + echo failing though + echo also stderr 1>&2 + exit 3 + ''); + } '' + grep -F 'failing though' $failed/testBuildFailure.log + grep -F 'also stderr' $failed/testBuildFailure.log + grep -F 'ok-ish' $failed/result + [[ 3 = $(cat $failed/testBuildFailure.exit) ]] + touch $out + ''; + + helloDoesNotFail = runCommand "testBuildFailure-helloDoesNotFail" { + failed = testers.testBuildFailure (testers.testBuildFailure hello); + + # Add hello itself as a prerequisite, so we don't try to run this test if + # there's an actual failure in hello. + inherit hello; + } '' + echo "Checking $failed/testBuildFailure.log" + grep -F 'testBuildFailure: The builder did not fail, but a failure was expected' $failed/testBuildFailure.log + [[ 1 = $(cat $failed/testBuildFailure.exit) ]] + touch $out + ''; + + multiOutput = runCommand "testBuildFailure-multiOutput" { + failed = testers.testBuildFailure (runCommand "fail" { + # dev will be the default output + outputs = ["dev" "doc" "out"]; + } '' + echo i am failing + exit 1 + ''); + } '' + grep -F 'i am failing' $failed/testBuildFailure.log >/dev/null + [[ 1 = $(cat $failed/testBuildFailure.exit) ]] + + # Checking our note that dev is the default output + echo $failed/_ | grep -- '-dev/_' >/dev/null + + touch $out + ''; + }; + } From e20a362908fa6d4393efb05390e7dd38a64237da Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Sat, 15 Oct 2022 17:24:09 +0200 Subject: [PATCH 2/3] testers.testEqualContents: init --- doc/builders/testers.chapter.md | 24 ++++ pkgs/build-support/testers/default.nix | 38 +++++++ pkgs/build-support/testers/test/default.nix | 116 +++++++++++++++++++- 3 files changed, 177 insertions(+), 1 deletion(-) diff --git a/doc/builders/testers.chapter.md b/doc/builders/testers.chapter.md index 57f66a428482..58bb06f23137 100644 --- a/doc/builders/testers.chapter.md +++ b/doc/builders/testers.chapter.md @@ -75,6 +75,30 @@ environment to a minimum, some small changes are inevitable. `buildPackages.coreutils` and possibly more. These are not added to `PATH` or any other environment variable, so they should be hard to observe. +## `testEqualContents` {#tester-equalContents} + +Check that two paths have the same contents. + +Example: + +```nix +testers.testEqualContents { + assertion = "sed -e performs replacement"; + expected = writeText "expected" '' + foo baz baz + ''; + actual = runCommand "actual" { + # not really necessary for a package that's in stdenv + nativeBuildInputs = [ gnused ]; + base = writeText "base" '' + foo bar baz + ''; + } '' + sed -e 's/bar/baz/g' $base >$out + ''; +} +``` + ## `testEqualDerivation` {#tester-testEqualDerivation} Checks that two packages produce the exact same build instructions. diff --git a/pkgs/build-support/testers/default.nix b/pkgs/build-support/testers/default.nix index fd08e9c6c47f..c565b6e72535 100644 --- a/pkgs/build-support/testers/default.nix +++ b/pkgs/build-support/testers/default.nix @@ -13,6 +13,44 @@ testEqualDerivation = callPackage ./test-equal-derivation.nix { }; + # See https://nixos.org/manual/nixpkgs/unstable/#tester-testEqualContents + # or doc/builders/testers.chapter.md + testEqualContents = { + assertion, + actual, + expected, + }: runCommand "equal-contents-${lib.strings.toLower assertion}" { + inherit assertion actual expected; + } '' + echo "Checking:" + echo "$assertion" + if ! diff -U5 -r "$actual" "$expected" --color=always + then + echo + echo 'Contents must be equal, but were not!' + echo + echo "+: expected, at $expected" + echo "-: unexpected, at $actual" + exit 1 + else + find "$expected" -type f -executable > expected-executables | sort + find "$actual" -type f -executable > actual-executables | sort + if ! diff -U0 actual-executables expected-executables --color=always + then + echo + echo "Contents must be equal, but some files' executable bits don't match" + echo + echo "+: make this file executable in the actual contents" + echo "-: make this file non-executable in the actual contents" + exit 1 + else + echo "expected $expected and actual $actual match." + echo 'OK' + touch $out + fi + fi + ''; + testVersion = { package, command ? "${package.meta.mainProgram or package.pname or package.name} --version", diff --git a/pkgs/build-support/testers/test/default.nix b/pkgs/build-support/testers/test/default.nix index 22869baae159..d6dfbe34fd21 100644 --- a/pkgs/build-support/testers/test/default.nix +++ b/pkgs/build-support/testers/test/default.nix @@ -68,9 +68,123 @@ lib.recurseIntoAttrs { # Checking our note that dev is the default output echo $failed/_ | grep -- '-dev/_' >/dev/null - + echo 'All good.' touch $out ''; }; + testEqualContents = lib.recurseIntoAttrs { + happy = testers.testEqualContents { + assertion = "The same directory contents at different paths are recognized as equal"; + expected = runCommand "expected" {} '' + mkdir -p $out/c + echo a >$out/a + echo b >$out/b + echo d >$out/c/d + ''; + actual = runCommand "actual" {} '' + mkdir -p $out/c + echo a >$out/a + echo b >$out/b + echo d >$out/c/d + ''; + }; + + unequalExe = + runCommand "testEqualContents-unequalExe" { + log = testers.testBuildFailure (testers.testEqualContents { + assertion = "The same directory contents at different paths are recognized as equal"; + expected = runCommand "expected" {} '' + mkdir -p $out/c + echo a >$out/a + chmod a+x $out/a + echo b >$out/b + echo d >$out/c/d + ''; + actual = runCommand "actual" {} '' + mkdir -p $out/c + echo a >$out/a + echo b >$out/b + chmod a+x $out/b + echo d >$out/c/d + ''; + }); + } '' + ( + set -x + grep -F -- "executable bits don't match" $log/testBuildFailure.log + grep -E -- '+.*-actual/a' $log/testBuildFailure.log + grep -E -- '-.*-actual/b' $log/testBuildFailure.log + grep -F -- "--- actual-executables" $log/testBuildFailure.log + grep -F -- "+++ expected-executables" $log/testBuildFailure.log + ) || { + echo "Test failed: could not find pattern in build log $log" + exit 1 + } + echo 'All good.' + touch $out + ''; + + fileDiff = + runCommand "testEqualContents-fileDiff" { + log = testers.testBuildFailure (testers.testEqualContents { + assertion = "The same directory contents at different paths are recognized as equal"; + expected = runCommand "expected" {} '' + mkdir -p $out/c + echo a >$out/a + echo b >$out/b + echo d >$out/c/d + ''; + actual = runCommand "actual" {} '' + mkdir -p $out/c + echo a >$out/a + echo B >$out/b + echo d >$out/c/d + ''; + }); + } '' + ( + set -x + grep -F -- "Contents must be equal but were not" $log/testBuildFailure.log + grep -E -- '+++ .*-actual/b' $log/testBuildFailure.log + grep -E -- '--- .*-actual/b' $log/testBuildFailure.log + grep -F -- "-B" $log/testBuildFailure.log + grep -F -- "+b" $log/testBuildFailure.log + ) || { + echo "Test failed: could not find pattern in build log $log" + exit 1 + } + echo 'All good.' + touch $out + ''; + + fileMissing = + runCommand "testEqualContents-fileMissing" { + log = testers.testBuildFailure (testers.testEqualContents { + assertion = "The same directory contents at different paths are recognized as equal"; + expected = runCommand "expected" {} '' + mkdir -p $out/c + echo a >$out/a + echo b >$out/b + echo d >$out/c/d + ''; + actual = runCommand "actual" {} '' + mkdir -p $out/c + echo a >$out/a + echo d >$out/c/d + ''; + }); + } '' + ( + set -x + grep -F -- "Contents must be equal but were not" $log/testBuildFailure.log + grep -E -- 'Only in .*-expected: b' $log/testBuildFailure.log + ) || { + echo "Test failed: could not find pattern in build log $log" + exit 1 + } + echo 'All good.' + touch $out + ''; + }; } From 3cf3fef37260eb9c9cbf00096c48064f11dc83a9 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Thu, 27 Oct 2022 14:05:19 +0200 Subject: [PATCH 3/3] testers: Add missing doc link comments --- pkgs/build-support/testers/default.nix | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/pkgs/build-support/testers/default.nix b/pkgs/build-support/testers/default.nix index c565b6e72535..6ab0ee843cb0 100644 --- a/pkgs/build-support/testers/default.nix +++ b/pkgs/build-support/testers/default.nix @@ -11,6 +11,8 @@ ] ++ orig.args or ["-e" (orig.builder or ../../stdenv/generic/default-builder.sh)]; }); + # See https://nixos.org/manual/nixpkgs/unstable/#tester-testEqualDerivation + # or doc/builders/testers.chapter.md testEqualDerivation = callPackage ./test-equal-derivation.nix { }; # See https://nixos.org/manual/nixpkgs/unstable/#tester-testEqualContents @@ -51,6 +53,8 @@ fi ''; + # See https://nixos.org/manual/nixpkgs/unstable/#tester-testVersion + # or doc/builders/testers.chapter.md testVersion = { package, command ? "${package.meta.mainProgram or package.pname or package.name} --version",