From b60c9fd2fe52118f460df4ff917caaa513f4f38f Mon Sep 17 00:00:00 2001 From: David McFarland Date: Sat, 24 Sep 2022 12:59:00 -0300 Subject: [PATCH] nuget-to-nix: find sources deterministically The source used to download a particular package still isn't deterministic in nuget. Even worse, the hash of the package can vary between sources. This makes nuget use the first enabled source containing the package. The order of the dependencies may be slightly different because it now uses glob order of the lower-case package names and versions, instead of sorting the output. If the package actually downloaded was the first source that contains the package, then it will be hashed from disk to avoid downloading it again. --- .../dotnet/build-dotnet-module/default.nix | 4 +- .../dotnet/nuget-to-nix/default.nix | 10 ++- .../dotnet/nuget-to-nix/nuget-to-nix.sh | 88 +++++++++++++------ .../github-runner/default.nix | 6 +- 4 files changed, 74 insertions(+), 34 deletions(-) diff --git a/pkgs/build-support/dotnet/build-dotnet-module/default.nix b/pkgs/build-support/dotnet/build-dotnet-module/default.nix index a8754218dece..5a465b48c1d7 100644 --- a/pkgs/build-support/dotnet/build-dotnet-module/default.nix +++ b/pkgs/build-support/dotnet/build-dotnet-module/default.nix @@ -172,7 +172,7 @@ stdenvNoCC.mkDerivation (args // { writeShellScript "fetch-${pname}-deps" '' set -euo pipefail - export PATH="${lib.makeBinPath [ coreutils dotnet-sdk nuget-to-nix ]}" + export PATH="${lib.makeBinPath [ coreutils dotnet-sdk (nuget-to-nix.override { inherit dotnet-sdk; }) ]}" for arg in "$@"; do case "$arg" in @@ -218,6 +218,8 @@ stdenvNoCC.mkDerivation (args // { -p:Deterministic=true \ --packages "$tmp/nuget_pkgs" \ --runtime "$rid" \ + --no-cache \ + --force \ ${lib.optionalString (!enableParallelBuilding) "--disable-parallel"} \ ${lib.optionalString (flags != []) (toString flags)} } diff --git a/pkgs/build-support/dotnet/nuget-to-nix/default.nix b/pkgs/build-support/dotnet/nuget-to-nix/default.nix index e3a3e45ac20c..3fdda4ac68d3 100644 --- a/pkgs/build-support/dotnet/nuget-to-nix/default.nix +++ b/pkgs/build-support/dotnet/nuget-to-nix/default.nix @@ -4,11 +4,12 @@ , substituteAll , nix , coreutils -, findutils -, gnused , jq +, yq , curl , gnugrep +, gawk +, dotnet-sdk }: runCommandLocal "nuget-to-nix" { @@ -19,11 +20,12 @@ runCommandLocal "nuget-to-nix" { binPath = lib.makeBinPath [ nix coreutils - findutils - gnused jq + yq curl gnugrep + gawk + dotnet-sdk ]; }; diff --git a/pkgs/build-support/dotnet/nuget-to-nix/nuget-to-nix.sh b/pkgs/build-support/dotnet/nuget-to-nix/nuget-to-nix.sh index d8c928f5179e..760350ab0517 100755 --- a/pkgs/build-support/dotnet/nuget-to-nix/nuget-to-nix.sh +++ b/pkgs/build-support/dotnet/nuget-to-nix/nuget-to-nix.sh @@ -3,6 +3,8 @@ set -euo pipefail export PATH="@binPath@" +# used for glob ordering of package names +export LC_ALL=C if [ $# -eq 0 ]; then >&2 echo "Usage: $0 [path to excluded package source] > deps.nix" @@ -10,37 +12,73 @@ if [ $# -eq 0 ]; then fi pkgs=$1 -tmpfile=$(mktemp /tmp/nuget-to-nix.XXXXXX) -trap "rm -f ${tmpfile}" EXIT +tmp=$(realpath "$(mktemp -td nuget-to-nix.XXXXXX)") +trap 'rm -r "$tmp"' EXIT excluded_source=$(realpath "${2:-$tmp/empty}") -declare -A nuget_sources_cache +export DOTNET_NOLOGO=1 +export DOTNET_CLI_TELEMETRY_OPTOUT=1 + +mapfile -t sources < <(dotnet nuget list source --format short | awk '/^E / { print $2 }') + +declare -A base_addresses + +for index in "${sources[@]}"; do + base_addresses[$index]=$( + curl --compressed --netrc -fsL "$index" | \ + jq -r '.resources[] | select(."@type" == "PackageBaseAddress/3.0.0")."@id"') +done echo "{ fetchNuGet }: [" -while read pkg_spec; do - { read pkg_name; read pkg_version; } < <( - # Build version part should be ignored: `3.0.0-beta2.20059.3+77df2220` -> `3.0.0-beta2.20059.3` - sed -nE 's/.*([^<]*).*/\1/p; s/.*([^<+]*).*/\1/p' "$pkg_spec") +cd "$pkgs" +for package in *; do + cd "$package" + for version in *; do + id=$(xq -r .package.metadata.id "$version/$package".nuspec) - if [[ -e "$excluded_source/${pkg_name}.$pkg_version".nupkg ]]; then - continue - fi - - pkg_sha256="$(nix-hash --type sha256 --flat --base32 "$(dirname "$pkg_spec")"/*.nupkg)" - - pkg_src="$(jq --raw-output '.source' "$(dirname "$pkg_spec")/.nupkg.metadata")" - if [[ -d $pkg_src ]]; then + if [[ -e "$excluded_source/$id.$version".nupkg ]]; then continue - elif [[ $pkg_src != https://api.nuget.org/* ]]; then - pkg_source_url="${nuget_sources_cache[$pkg_src]:=$(curl -n --fail "$pkg_src" | jq --raw-output '.resources[] | select(."@type" == "PackageBaseAddress/3.0.0")."@id"')}" - pkg_url="$pkg_source_url${pkg_name,,}/${pkg_version,,}/${pkg_name,,}.${pkg_version,,}.nupkg" - echo " (fetchNuGet { pname = \"$pkg_name\"; version = \"$pkg_version\"; sha256 = \"$pkg_sha256\"; url = \"$pkg_url\"; })" >> ${tmpfile} - else - echo " (fetchNuGet { pname = \"$pkg_name\"; version = \"$pkg_version\"; sha256 = \"$pkg_sha256\"; })" >> ${tmpfile} - fi -done < <(find $1 -name '*.nuspec') + fi -LC_ALL=C sort --ignore-case ${tmpfile} + used_source="$(jq -r '.source' "$version"/.nupkg.metadata)" + for source in "${sources[@]}"; do + url="${base_addresses[$source]}$package/$version/$package.$version.nupkg" + if [[ "$source" == "$used_source" ]]; then + sha256="$(nix-hash --type sha256 --flat --base32 "$version/$package.$version".nupkg)" + found=true + break + else + if sha256=$(nix-prefetch-url "$url" 2>"$tmp"/error); then + # If multiple remote sources are enabled, nuget will try them all + # concurrently and use the one that responds first. We always use the + # first source that has the package. + echo "$package $version is available on $url, but was downloaded from ${base_addresses[$used_source]}$package/$version/$package.$version.nupkg" 1>&2 + found=true + break + else + if ! grep -q 'HTTP error 404' "$tmp/error"; then + cat "$tmp/error" 1>&2 + exit 1 + fi + fi + fi + done -echo "]" + if ! ${found-false}; then + echo "couldn't find $package $version" >&2 + exit 1 + fi + + if [[ "$source" != https://api.nuget.org/v3/index.json ]]; then + echo " (fetchNuGet { pname = \"$id\"; version = \"$version\"; sha256 = \"$sha256\"; url = \"$url\"; })" + else + echo " (fetchNuGet { pname = \"$id\"; version = \"$version\"; sha256 = \"$sha256\"; })" + fi + done + cd .. +done + +cat << EOL +] +EOL diff --git a/pkgs/development/tools/continuous-integration/github-runner/default.nix b/pkgs/development/tools/continuous-integration/github-runner/default.nix index 5dcc556660fa..a9737a0e12d4 100644 --- a/pkgs/development/tools/continuous-integration/github-runner/default.nix +++ b/pkgs/development/tools/continuous-integration/github-runner/default.nix @@ -302,12 +302,12 @@ stdenv.mkDerivation rec { # Inspired by passthru.fetch-deps in pkgs/build-support/build-dotnet-module/default.nix passthru.createDepsFile = writeShellApplication { name = "create-deps-file"; - runtimeInputs = [ dotnetSdk nuget-to-nix ]; + runtimeInputs = [ dotnetSdk (nuget-to-nix.override { dotnet-sdk = dotnetSdk; }) ]; text = '' # Disable telemetry data export DOTNET_CLI_TELEMETRY_OPTOUT=1 - rundir=$(pwd) + deps_file="$(realpath "''${1:-$(mktemp -t "${pname}-deps-XXXXXX.nix")}")" printf "\n* Setup workdir\n" workdir="$(mktemp -d /tmp/${pname}.XXX)" @@ -324,8 +324,6 @@ stdenv.mkDerivation rec { dotnet restore src/ActionsRunner.sln --packages nuget_pkgs --no-cache --force --runtime "${rid}" '') (lib.attrValues runtimeIds)} - cd "$rundir" - deps_file=''${1-"/tmp/${pname}-deps.nix"} printf "\n* Make %s file\n" "$(basename "$deps_file")" nuget-to-nix "$workdir/nuget_pkgs" > "$deps_file" printf "\n* Dependency file writen to %s" "$deps_file"