From 45b4efbacf84af2ebff0bc5da22f1194a3575e82 Mon Sep 17 00:00:00 2001 From: DavHau Date: Thu, 2 Jan 2025 15:59:14 +0700 Subject: [PATCH 1/3] breakpointHook: rename to breakPointHookCntr to free this name for an improved implementation while keeping the old one. --- .../br/{breakpointHook => breakpointHookCntr}/breakpoint-hook.sh | 0 .../by-name/br/{breakpointHook => breakpointHookCntr}/package.nix | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename pkgs/by-name/br/{breakpointHook => breakpointHookCntr}/breakpoint-hook.sh (100%) rename pkgs/by-name/br/{breakpointHook => breakpointHookCntr}/package.nix (100%) diff --git a/pkgs/by-name/br/breakpointHook/breakpoint-hook.sh b/pkgs/by-name/br/breakpointHookCntr/breakpoint-hook.sh similarity index 100% rename from pkgs/by-name/br/breakpointHook/breakpoint-hook.sh rename to pkgs/by-name/br/breakpointHookCntr/breakpoint-hook.sh diff --git a/pkgs/by-name/br/breakpointHook/package.nix b/pkgs/by-name/br/breakpointHookCntr/package.nix similarity index 100% rename from pkgs/by-name/br/breakpointHook/package.nix rename to pkgs/by-name/br/breakpointHookCntr/package.nix From d7d59e137cdd8307caa4fb7d0a687fc3f88191b6 Mon Sep 17 00:00:00 2001 From: DavHau Date: Thu, 2 Jan 2025 17:50:44 +0700 Subject: [PATCH 2/3] breakpointHook: re-implement using nsenter breakpointHook is amazing and has been a daily driver for me. Though it has been buggy several times in the past and recently it stopped working on my system, as the approach using cntr seems to be a bit sensitive to the environment and which shell is used etc. After speaking with @mic92, it turned out there is an alternative approach using `nsenter`. After playing with that for a while, I think we should replace the current implementation of breakpointHook. The new output when a build fails will look like this: ``` build failed in configurePhase with exit code 1 To attach, run the following command: sudo /nix/store/2kzihv08w9vsyi6zhs0pi2i9117v4q1v-attach/bin/attach 28715971284 ``` Benefits of the new implementation over the old one: - no need to install cntr tool on the local machine first, the copy pasted command will just work - no need to execute `cntr exec ...` manually to enter the namespace - interactive shell by default - dropped into correct working directory by default - $out available by default - possibility to override the shell with zfs or fish, etc. by defining `debugShell` in the derivation to debug - probably less prone to bugs as it only uses nsenter cc @mic92 --- pkgs/by-name/br/breakpointHook/attach.sh | 46 +++++++++++++++++++ .../br/breakpointHook/breakpoint-hook.sh | 20 ++++++++ pkgs/by-name/br/breakpointHook/package.nix | 38 +++++++++++++++ 3 files changed, 104 insertions(+) create mode 100755 pkgs/by-name/br/breakpointHook/attach.sh create mode 100644 pkgs/by-name/br/breakpointHook/breakpoint-hook.sh create mode 100644 pkgs/by-name/br/breakpointHook/package.nix diff --git a/pkgs/by-name/br/breakpointHook/attach.sh b/pkgs/by-name/br/breakpointHook/attach.sh new file mode 100755 index 000000000000..06ccdc3eee06 --- /dev/null +++ b/pkgs/by-name/br/breakpointHook/attach.sh @@ -0,0 +1,46 @@ +#! /usr/bin/env bash + +set -eu -o pipefail + +id="$1" +pids="$(pgrep -f "sleep $id" || :)" +if [ -z "$pids" ]; then + echo "Error: No process found for 'sleep $id'. The build must still be running in order to attach. Also make sure it's not on a remote builder." >&2 + exit 1 +elif [ "$(echo "$pids" | wc -l)" -ne 1 ]; then + echo "Error: Multiple processes found matching 'sleep $id'" >&2 + exit 1 +fi +pid="$(echo "$pids" | head -n1)" + +# helper to extract variables from the build env +getVar(){ + while IFS= read -r -d $'\0' line; do + case "$line" in + *"$1="* ) + echo "$line" + ;; + esac + done < /proc/$pid/environ \ + | cut -d "=" -f 2 +} + +# bash is needed to load the env vars, as we do not know the syntax of the debug shell. +# bashInteractive is used instead of bash, as we depend on it anyways, due to it being +# the default debug shell +bashInteractive="$(getVar bashInteractive)" +# the debug shell will be started as interactive shell after loading the env vars +debugShell="$(getVar debugShell)" +# to drop the user into the working directory at the point of failure +pwd="$(readlink /proc/$pid/cwd)" + +# enter the namespace of the failed build +exec nsenter --mount --net --target "$pid" "$bashInteractive" -c " + set -eu -o pipefail + while IFS= read -r -d \$'\0' line; do + export \"\$line\" + done <&3 + exec 3>&- + cd \"$pwd\" + exec \"$debugShell\" +" 3< /proc/$pid/environ diff --git a/pkgs/by-name/br/breakpointHook/breakpoint-hook.sh b/pkgs/by-name/br/breakpointHook/breakpoint-hook.sh new file mode 100644 index 000000000000..6c79f58604cd --- /dev/null +++ b/pkgs/by-name/br/breakpointHook/breakpoint-hook.sh @@ -0,0 +1,20 @@ +breakpointHook() { + local red='\033[0;31m' + local cyan='\033[0;36m' + local green='\033[0;32m' + local no_color='\033[0m' + + # provide the user with an interactive shell for better experience + export bashInteractive="@bashInteractive@" + if [ -z "$debugShell" ]; then + export debugShell="@bashInteractive@" + fi + + local id + id="$(shuf -i 999999-9999999 -n1)" + echo -e "${red}build for ${cyan}${name:-unknown}${red} failed in ${curPhase:-unknown} with exit code ${exitCode:-unknown}${no_color}" + echo -e "${green}To attach, run the following command:${no_color}" + echo -e "${green} sudo @attach@ $id${no_color}" + sleep "$id" +} +failureHooks+=(breakpointHook) diff --git a/pkgs/by-name/br/breakpointHook/package.nix b/pkgs/by-name/br/breakpointHook/package.nix new file mode 100644 index 000000000000..a94ef881a430 --- /dev/null +++ b/pkgs/by-name/br/breakpointHook/package.nix @@ -0,0 +1,38 @@ +{ + lib, + stdenv, + + bash, + bashInteractive, + coreutils, + makeSetupHook, + procps, + util-linux, + writeShellScriptBin, +}: + +let + attach = writeShellScriptBin "attach" '' + export PATH="${ + lib.makeBinPath [ + bash + coreutils + procps # needed for pgrep + util-linux # needed for nsenter + ] + }" + exec bash ${./attach.sh} "$@" + ''; +in + +makeSetupHook { + name = "breakpoint-hook"; + meta.broken = !stdenv.buildPlatform.isLinux; + substitutions = { + attach = "${attach}/bin/attach"; + # The default interactive shell in case $debugShell is not set in the derivation. + # Can be overridden to zsh or fish, etc. + # This shell is also used to load the env variables before the $debugShell is started. + bashInteractive = lib.getExe bashInteractive; + }; +} ./breakpoint-hook.sh From 3e81636c90a956cf9a0c7d590502f9d77105242d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rg=20Thalheim?= Date: Sun, 5 Jan 2025 14:30:43 +0100 Subject: [PATCH 3/3] docs/breakpointHook: simplify instructions --- doc/hooks/breakpoint.section.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/doc/hooks/breakpoint.section.md b/doc/hooks/breakpoint.section.md index b7f1979586de..cbc93b0fb988 100644 --- a/doc/hooks/breakpoint.section.md +++ b/doc/hooks/breakpoint.section.md @@ -1,6 +1,6 @@ # breakpointHook {#breakpointhook} -This hook will make a build pause instead of stopping when a failure happens. It prevents nix from cleaning up the build environment immediately and allows the user to attach to a build environment using the `cntr` command. Upon build error it will print instructions on how to use `cntr`, which can be used to enter the environment for debugging. Installing cntr and running the command will provide shell access to the build sandbox of failed build. At `/var/lib/cntr` the sandboxed filesystem is mounted. All commands and files of the system are still accessible within the shell. To execute commands from the sandbox use the cntr exec subcommand. `cntr` is only supported on Linux-based platforms. To use it first add `cntr` to your `environment.systemPackages` on NixOS or alternatively to the root user on non-NixOS systems. Then in the package that is supposed to be inspected, add `breakpointHook` to `nativeBuildInputs`. +This hook makes a build pause instead of stopping when a failure occurs. It prevents Nix from cleaning up the build environment immediately and allows the user to attach to the build environment. Upon a build error, it will print instructions that can be used to enter the environment for debugging. breakpointHook is only available on Linux. To use it, add `breakpointHook` to `nativeBuildInputs` in the package to be inspected. ```nix { @@ -8,10 +8,10 @@ This hook will make a build pause instead of stopping when a failure happens. It } ``` -When a build failure happens there will be an instruction printed that shows how to attach with `cntr` to the build sandbox. +When a build failure occurs, an instruction will be printed showing how to attach to the build sandbox. ::: {.note} Caution with remote builds -This won’t work with remote builds as the build environment is on a different machine and can’t be accessed by `cntr`. Remote builds can be turned off by setting `--option builders ''` for `nix-build` or `--builders ''` for `nix build`. +For remote builds, the printed instructions need to be run on the remote machine, as the build sandbox is only accessible on the machine running the builds. Remote builds can be turned off by setting `--option builders ''` for `nix-build` or `--builders ''` for `nix build`. ::: :::