From 8a6c85d88678f53143e63542649e7836d699e9ac Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 12 Aug 2024 12:52:26 +0200 Subject: [PATCH 1/4] lib/tests/modules.sh: Report failure source location --- lib/tests/modules.sh | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/lib/tests/modules.sh b/lib/tests/modules.sh index 280d0b47d574..06713ec3ef67 100755 --- a/lib/tests/modules.sh +++ b/lib/tests/modules.sh @@ -20,6 +20,15 @@ cd "$DIR"/modules pass=0 fail=0 +# prints the location of the call of to the function that calls it +loc() { + local caller depth + depth=1 + # ( lineno fnname file ) of the caller + caller=( $(caller $depth) ) + echo "${caller[2]}:${caller[0]}" +} + evalConfig() { local attr=$1 shift @@ -42,6 +51,7 @@ checkConfigOutput() { if evalConfig "$@" 2>/dev/null | grep -E --silent "$outputContains" ; then ((++pass)) else + echo "test failure at $(loc):" echo 2>&1 "error: Expected result matching '$outputContains', while evaluating" reportFailure "$@" fi @@ -52,12 +62,14 @@ checkConfigError() { local err="" shift if err="$(evalConfig "$@" 2>&1 >/dev/null)"; then + echo "test failure at $(loc):" echo 2>&1 "error: Expected error code, got exit code 0, while evaluating" reportFailure "$@" else if echo "$err" | grep -zP --silent "$errorContains" ; then ((++pass)) else + echo "test failure at $(loc):" echo 2>&1 "error: Expected error matching '$errorContains', while evaluating" reportFailure "$@" fi From c516c03bf4296809ee0f6db30cd02b08ba71ebe8 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 12 Aug 2024 13:15:50 +0200 Subject: [PATCH 2/4] lib/tests/modules.sh: Do not redirect diagnostics to stdout It still prints its own diagnostics to stdout, but it's always done that. --- lib/tests/modules.sh | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/tests/modules.sh b/lib/tests/modules.sh index 06713ec3ef67..7dcfa5231cd1 100755 --- a/lib/tests/modules.sh +++ b/lib/tests/modules.sh @@ -13,7 +13,7 @@ set -o errexit -o noclobber -o nounset -o pipefail shopt -s failglob inherit_errexit # https://stackoverflow.com/a/246128/6605742 -DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" +DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null && pwd )" cd "$DIR"/modules @@ -40,7 +40,7 @@ reportFailure() { local attr=$1 shift local script="import ./default.nix { modules = [ $* ];}" - echo 2>&1 "$ nix-instantiate -E '$script' -A '$attr' --eval-only --json" + echo "$ nix-instantiate -E '$script' -A '$attr' --eval-only --json" evalConfig "$attr" "$@" || true ((++fail)) } @@ -52,7 +52,7 @@ checkConfigOutput() { ((++pass)) else echo "test failure at $(loc):" - echo 2>&1 "error: Expected result matching '$outputContains', while evaluating" + echo "error: Expected result matching '$outputContains', while evaluating" reportFailure "$@" fi } @@ -63,14 +63,14 @@ checkConfigError() { shift if err="$(evalConfig "$@" 2>&1 >/dev/null)"; then echo "test failure at $(loc):" - echo 2>&1 "error: Expected error code, got exit code 0, while evaluating" + echo "error: Expected error code, got exit code 0, while evaluating" reportFailure "$@" else if echo "$err" | grep -zP --silent "$errorContains" ; then ((++pass)) else echo "test failure at $(loc):" - echo 2>&1 "error: Expected error matching '$errorContains', while evaluating" + echo "error: Expected error matching '$errorContains', while evaluating" reportFailure "$@" fi fi From 6fa24da815b755e7a521cd7e6808956480af3613 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 12 Aug 2024 13:25:30 +0200 Subject: [PATCH 3/4] lib/tests/modules.sh: Add loc optional parameter --- lib/tests/modules.sh | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/tests/modules.sh b/lib/tests/modules.sh index 7dcfa5231cd1..e7c7f881338b 100755 --- a/lib/tests/modules.sh +++ b/lib/tests/modules.sh @@ -20,10 +20,16 @@ cd "$DIR"/modules pass=0 fail=0 -# prints the location of the call of to the function that calls it +# loc +# prints the location of the call of to the function that calls it +# loc n +# prints the location n levels up the call stack loc() { local caller depth depth=1 + if [[ $# -gt 0 ]]; then + depth=$1 + fi # ( lineno fnname file ) of the caller caller=( $(caller $depth) ) echo "${caller[2]}:${caller[0]}" From 7f838d4c54891ad1d9c2eb7ed8065c840fb21924 Mon Sep 17 00:00:00 2001 From: Robert Hensing Date: Mon, 12 Aug 2024 13:36:13 +0200 Subject: [PATCH 4/4] lib/tests/modules.sh: Improve failure log format - Clear separation between failures - Move error regex close to error message, which is at the bottom of a fairly long trace - Move most relevant and consistent info to bottom of terminal: the location of the failure. Some editors including vscode heuristically resolve file paths on Ctrl+click. - Less wordy - easy to glance - Capitalized prefixes to distinguish from Nix's own logging --- lib/tests/modules.sh | 37 +++++++++++++++++++++++++++++++------ 1 file changed, 31 insertions(+), 6 deletions(-) diff --git a/lib/tests/modules.sh b/lib/tests/modules.sh index e7c7f881338b..3a23766a17f5 100755 --- a/lib/tests/modules.sh +++ b/lib/tests/modules.sh @@ -35,6 +35,22 @@ loc() { echo "${caller[2]}:${caller[0]}" } +line() { + echo "----------------------------------------" +} +logStartFailure() { + line +} +logEndFailure() { + line + echo +} + +logFailure() { + # bold red + printf '\033[1;31mTEST FAILED\033[0m at %s\n' "$(loc 2)" +} + evalConfig() { local attr=$1 shift @@ -57,9 +73,12 @@ checkConfigOutput() { if evalConfig "$@" 2>/dev/null | grep -E --silent "$outputContains" ; then ((++pass)) else - echo "test failure at $(loc):" - echo "error: Expected result matching '$outputContains', while evaluating" + logStartFailure + echo "ACTUAL:" reportFailure "$@" + echo "EXPECTED: result matching '$outputContains'" + logFailure + logEndFailure fi } @@ -68,16 +87,22 @@ checkConfigError() { local err="" shift if err="$(evalConfig "$@" 2>&1 >/dev/null)"; then - echo "test failure at $(loc):" - echo "error: Expected error code, got exit code 0, while evaluating" + logStartFailure + echo "ACTUAL: exit code 0, output:" reportFailure "$@" + echo "EXPECTED: non-zero exit code" + logFailure + logEndFailure else if echo "$err" | grep -zP --silent "$errorContains" ; then ((++pass)) else - echo "test failure at $(loc):" - echo "error: Expected error matching '$errorContains', while evaluating" + logStartFailure + echo "ACTUAL:" reportFailure "$@" + echo "EXPECTED: error matching '$errorContains'" + logFailure + logEndFailure fi fi }