A debuginfod support must be able to map a build-id to
- debug symbols
- the original elf file for which the debug symbols where separated
- the corresponding source files
Currently, hydra provides an index from build-id to the nar of the debug
output containing the debug symbols.
Add symlinks in these outputs so that we can recover the store path of
the source and original elf file. We can then fetch them by the normal
binary cache protocol.
About source files: to minimize storage demands, in the ideal case,
software would be built from the source store path $src and the
debuginfod server would just have to serve source files from this store
path. In practice, source files are sometimes patched as part of the
build. This commit stores the modified files in the debug output is a so
called source overlay so that the debuginfod serve can serve the patched
content of the file.
The checksum was chosen as follows (where big is 4GB of zeros):
$ hyperfine -L s sysv,bsd,crc,sha1,sha224,sha256,sha384,sha512,blake2b,sm3 'cksum -a {s} big'
Benchmark 1: cksum -a sysv big
Time (mean ± σ): 854.5 ms ± 270.5 ms [User: 245.3 ms, System: 601.8 ms]
Range (min … max): 760.5 ms … 1623.8 ms 10 runs
Warning: The first benchmarking run for this command was significantly slower than the rest (1.624 s). This could be caused by (filesystem) caches that were not filled until after the first run. You should consider using the '--warmup' option to fill those caches before the actual benchmark. Alternatively, use the '--prepare' option to clear the caches before each timing run.
Benchmark 2: cksum -a bsd big
Time (mean ± σ): 5.838 s ± 0.045 s [User: 5.118 s, System: 0.693 s]
Range (min … max): 5.767 s … 5.897 s 10 runs
Benchmark 3: cksum -a crc big
Time (mean ± σ): 829.9 ms ± 28.6 ms [User: 274.5 ms, System: 551.0 ms]
Range (min … max): 803.2 ms … 904.8 ms 10 runs
Benchmark 4: cksum -a sha1 big
Time (mean ± σ): 2.553 s ± 0.010 s [User: 1.912 s, System: 0.631 s]
Range (min … max): 2.543 s … 2.575 s 10 runs
Benchmark 5: cksum -a sha224 big
Time (mean ± σ): 2.716 s ± 0.018 s [User: 2.054 s, System: 0.645 s]
Range (min … max): 2.695 s … 2.743 s 10 runs
Benchmark 6: cksum -a sha256 big
Time (mean ± σ): 2.751 s ± 0.029 s [User: 2.057 s, System: 0.674 s]
Range (min … max): 2.712 s … 2.812 s 10 runs
Benchmark 7: cksum -a sha384 big
Time (mean ± σ): 5.600 s ± 0.049 s [User: 4.820 s, System: 0.753 s]
Range (min … max): 5.515 s … 5.683 s 10 runs
Benchmark 8: cksum -a sha512 big
Time (mean ± σ): 5.543 s ± 0.021 s [User: 4.751 s, System: 0.768 s]
Range (min … max): 5.523 s … 5.579 s 10 runs
Benchmark 9: cksum -a blake2b big
Time (mean ± σ): 5.091 s ± 0.025 s [User: 4.306 s, System: 0.764 s]
Range (min … max): 5.048 s … 5.125 s 10 runs
Benchmark 10: cksum -a sm3 big
Time (mean ± σ): 14.220 s ± 0.120 s [User: 13.376 s, System: 0.783 s]
Range (min … max): 14.077 s … 14.497 s 10 runs
Summary
cksum -a crc big ran
1.03 ± 0.33 times faster than cksum -a sysv big
3.08 ± 0.11 times faster than cksum -a sha1 big
3.27 ± 0.11 times faster than cksum -a sha224 big
3.31 ± 0.12 times faster than cksum -a sha256 big
6.13 ± 0.21 times faster than cksum -a blake2b big
6.68 ± 0.23 times faster than cksum -a sha512 big
6.75 ± 0.24 times faster than cksum -a sha384 big
7.03 ± 0.25 times faster than cksum -a bsd big
17.13 ± 0.61 times faster than cksum -a sm3 big
unfortunately, crc (and sysv) are not supported by --check, so they are
disqualified. sha1 sha224 and sha256 are sensibly as fast as one
another, so let's use a non broken one, even though cryptographic
qualities are not needed here.
121 lines
5.0 KiB
Bash
121 lines
5.0 KiB
Bash
export NIX_SET_BUILD_ID=1
|
|
export NIX_LDFLAGS+=" --compress-debug-sections=zlib"
|
|
export NIX_CFLAGS_COMPILE+=" -ggdb -Wa,--compress-debug-sections"
|
|
export NIX_RUSTFLAGS+=" -g -C strip=none"
|
|
|
|
cksumAlgo=sha256
|
|
|
|
fixupOutputHooks+=(_separateDebugInfo)
|
|
postUnpackHooks+=(_recordPristineSourceHashes)
|
|
|
|
_recordPristineSourceHashes() {
|
|
# shellcheck disable=2154
|
|
[ -e "$sourceRoot" ] || return 0
|
|
|
|
local checksumFileName=__nix_source_checksums
|
|
echo "separate-debug-info: recording checksum of source files for debug support..."
|
|
find "$sourceRoot" -type f -exec cksum -a "$cksumAlgo" '{}' \+ > "$checksumFileName"
|
|
recordedSourceChecksumsFileName="$(readlink -f "$checksumFileName")"
|
|
}
|
|
|
|
_separateDebugInfo() {
|
|
# shellcheck disable=2154
|
|
[ -e "$prefix" ] || return 0
|
|
|
|
local debugOutput="${debug:-$out}"
|
|
if [ "$prefix" = "$debugOutput" ]; then return 0; fi
|
|
|
|
# in case there is nothing to strip, don't fail the build
|
|
mkdir -p "$debugOutput"
|
|
|
|
local dst="$debugOutput/lib/debug/.build-id"
|
|
|
|
local source
|
|
local sourceOverlay
|
|
# shellcheck disable=2154
|
|
if [ -e "$src" ]; then
|
|
source="$src"
|
|
if [ -n "${recordedSourceChecksumsFileName:-}" ]; then
|
|
sourceOverlay="$debugOutput/src/overlay"
|
|
else
|
|
sourceOverlay=""
|
|
fi
|
|
else
|
|
source=""
|
|
sourceOverlay=""
|
|
fi
|
|
|
|
# Find executables and dynamic libraries.
|
|
local i
|
|
while IFS= read -r -d $'\0' i; do
|
|
if ! isELF "$i"; then continue; fi
|
|
|
|
[ -z "${READELF:-}" ] && echo "_separateDebugInfo: '\$READELF' variable is empty, skipping." 1>&2 && break
|
|
[ -z "${OBJCOPY:-}" ] && echo "_separateDebugInfo: '\$OBJCOPY' variable is empty, skipping." 1>&2 && break
|
|
|
|
# Extract the Build ID. FIXME: there's probably a cleaner way.
|
|
local id
|
|
id="$($READELF -n "$i" | sed 's/.*Build ID: \([0-9a-f]*\).*/\1/; t; d')"
|
|
if [ "${#id}" != 40 ]; then
|
|
echo "could not find build ID of $i, skipping" >&2
|
|
continue
|
|
fi
|
|
|
|
|
|
# Extract the debug info.
|
|
echo "separating debug info from $i (build ID $id)"
|
|
|
|
local debuginfoDir="$dst/${id:0:2}"
|
|
local buildIdPrefix="$debuginfoDir/${id:2}"
|
|
local debuginfoFile="$buildIdPrefix.debug"
|
|
local executableSymlink="$buildIdPrefix.executable"
|
|
local sourceSymlink="$buildIdPrefix.source"
|
|
local sourceOverlaySymlink="$buildIdPrefix.sourceoverlay"
|
|
|
|
mkdir -p "$debuginfoDir"
|
|
|
|
if [ -f "$debuginfoFile" ]; then
|
|
echo "separate-debug-info: warning: multiple files with build id $id found, overwriting"
|
|
fi
|
|
|
|
# This may fail, e.g. if the binary is for a different
|
|
# architecture than we're building for. (This happens with
|
|
# firmware blobs in QEMU.)
|
|
if $OBJCOPY --only-keep-debug "$i" "$debuginfoFile"; then
|
|
# If we succeeded, also a create a symlink <original-name>.debug.
|
|
ln -sfn "$debuginfoFile" "$dst/../$(basename "$i")"
|
|
# also create a symlink mapping the build-id to the original elf file and the source
|
|
# debuginfod protocol relies on it
|
|
ln -sfn "$i" "$executableSymlink"
|
|
if [ -n "$source" ]; then
|
|
ln -sfn "$source" "$sourceSymlink"
|
|
fi
|
|
if [ -n "$sourceOverlay" ]; then
|
|
# create it lazily
|
|
if [ ! -d "$sourceOverlay" ]; then
|
|
echo "separate-debug-info: copying patched source files to $sourceOverlay..."
|
|
mkdir -p "$sourceOverlay"
|
|
pushd "$(dirname "$recordedSourceChecksumsFileName")" || { echo "separate-debug-info: failed to cd parent directory of $recordedSourceChecksumsFileName"; return 1; }
|
|
while IFS= read -r -d $'\0' modifiedSourceFile; do
|
|
if [ -z "$modifiedSourceFile" ]; then
|
|
continue
|
|
fi
|
|
# this can happen with files with '\n' in their name
|
|
if [ ! -f "$modifiedSourceFile" ]; then
|
|
echo "separate-debug-info: cannot save modified source file $modifiedSourceFile: does not exist. ignoring"
|
|
continue
|
|
fi
|
|
mkdir -p "$sourceOverlay/$(dirname "$modifiedSourceFile")"
|
|
cp -v "$modifiedSourceFile" "$sourceOverlay/$modifiedSourceFile"
|
|
done < <(LANG=C cksum -a "$cksumAlgo" --check --ignore-missing --quiet "$recordedSourceChecksumsFileName" 2>&1 | sed -n -e 's/: FAILED$/\x00/p' | sed -z -e 's/^\n//')
|
|
popd || { echo "separate-debug-info: failed to popd" ; return 1; }
|
|
fi
|
|
ln -sfn "$sourceOverlay" "$sourceOverlaySymlink"
|
|
fi
|
|
else
|
|
# If we failed, try to clean up unnecessary directories
|
|
rmdir -p "$dst/${id:0:2}" --ignore-fail-on-non-empty
|
|
fi
|
|
done < <(find "$prefix" -type f -print0 | sort -z)
|
|
}
|