diff --git a/pkgs/build-support/dotnet/build-dotnet-module/default.nix b/pkgs/build-support/dotnet/build-dotnet-module/default.nix index a8754218dece..5a465b48c1d7 100644 --- a/pkgs/build-support/dotnet/build-dotnet-module/default.nix +++ b/pkgs/build-support/dotnet/build-dotnet-module/default.nix @@ -172,7 +172,7 @@ stdenvNoCC.mkDerivation (args // { writeShellScript "fetch-${pname}-deps" '' set -euo pipefail - export PATH="${lib.makeBinPath [ coreutils dotnet-sdk nuget-to-nix ]}" + export PATH="${lib.makeBinPath [ coreutils dotnet-sdk (nuget-to-nix.override { inherit dotnet-sdk; }) ]}" for arg in "$@"; do case "$arg" in @@ -218,6 +218,8 @@ stdenvNoCC.mkDerivation (args // { -p:Deterministic=true \ --packages "$tmp/nuget_pkgs" \ --runtime "$rid" \ + --no-cache \ + --force \ ${lib.optionalString (!enableParallelBuilding) "--disable-parallel"} \ ${lib.optionalString (flags != []) (toString flags)} } diff --git a/pkgs/build-support/dotnet/nuget-to-nix/default.nix b/pkgs/build-support/dotnet/nuget-to-nix/default.nix index e3a3e45ac20c..3fdda4ac68d3 100644 --- a/pkgs/build-support/dotnet/nuget-to-nix/default.nix +++ b/pkgs/build-support/dotnet/nuget-to-nix/default.nix @@ -4,11 +4,12 @@ , substituteAll , nix , coreutils -, findutils -, gnused , jq +, yq , curl , gnugrep +, gawk +, dotnet-sdk }: runCommandLocal "nuget-to-nix" { @@ -19,11 +20,12 @@ runCommandLocal "nuget-to-nix" { binPath = lib.makeBinPath [ nix coreutils - findutils - gnused jq + yq curl gnugrep + gawk + dotnet-sdk ]; }; diff --git a/pkgs/build-support/dotnet/nuget-to-nix/nuget-to-nix.sh b/pkgs/build-support/dotnet/nuget-to-nix/nuget-to-nix.sh index d8c928f5179e..760350ab0517 100755 --- a/pkgs/build-support/dotnet/nuget-to-nix/nuget-to-nix.sh +++ b/pkgs/build-support/dotnet/nuget-to-nix/nuget-to-nix.sh @@ -3,6 +3,8 @@ set -euo pipefail export PATH="@binPath@" +# used for glob ordering of package names +export LC_ALL=C if [ $# -eq 0 ]; then >&2 echo "Usage: $0 [path to excluded package source] > deps.nix" @@ -10,37 +12,73 @@ if [ $# -eq 0 ]; then fi pkgs=$1 -tmpfile=$(mktemp /tmp/nuget-to-nix.XXXXXX) -trap "rm -f ${tmpfile}" EXIT +tmp=$(realpath "$(mktemp -td nuget-to-nix.XXXXXX)") +trap 'rm -r "$tmp"' EXIT excluded_source=$(realpath "${2:-$tmp/empty}") -declare -A nuget_sources_cache +export DOTNET_NOLOGO=1 +export DOTNET_CLI_TELEMETRY_OPTOUT=1 + +mapfile -t sources < <(dotnet nuget list source --format short | awk '/^E / { print $2 }') + +declare -A base_addresses + +for index in "${sources[@]}"; do + base_addresses[$index]=$( + curl --compressed --netrc -fsL "$index" | \ + jq -r '.resources[] | select(."@type" == "PackageBaseAddress/3.0.0")."@id"') +done echo "{ fetchNuGet }: [" -while read pkg_spec; do - { read pkg_name; read pkg_version; } < <( - # Build version part should be ignored: `3.0.0-beta2.20059.3+77df2220` -> `3.0.0-beta2.20059.3` - sed -nE 's/.*([^<]*).*/\1/p; s/.*([^<+]*).*/\1/p' "$pkg_spec") +cd "$pkgs" +for package in *; do + cd "$package" + for version in *; do + id=$(xq -r .package.metadata.id "$version/$package".nuspec) - if [[ -e "$excluded_source/${pkg_name}.$pkg_version".nupkg ]]; then - continue - fi - - pkg_sha256="$(nix-hash --type sha256 --flat --base32 "$(dirname "$pkg_spec")"/*.nupkg)" - - pkg_src="$(jq --raw-output '.source' "$(dirname "$pkg_spec")/.nupkg.metadata")" - if [[ -d $pkg_src ]]; then + if [[ -e "$excluded_source/$id.$version".nupkg ]]; then continue - elif [[ $pkg_src != https://api.nuget.org/* ]]; then - pkg_source_url="${nuget_sources_cache[$pkg_src]:=$(curl -n --fail "$pkg_src" | jq --raw-output '.resources[] | select(."@type" == "PackageBaseAddress/3.0.0")."@id"')}" - pkg_url="$pkg_source_url${pkg_name,,}/${pkg_version,,}/${pkg_name,,}.${pkg_version,,}.nupkg" - echo " (fetchNuGet { pname = \"$pkg_name\"; version = \"$pkg_version\"; sha256 = \"$pkg_sha256\"; url = \"$pkg_url\"; })" >> ${tmpfile} - else - echo " (fetchNuGet { pname = \"$pkg_name\"; version = \"$pkg_version\"; sha256 = \"$pkg_sha256\"; })" >> ${tmpfile} - fi -done < <(find $1 -name '*.nuspec') + fi -LC_ALL=C sort --ignore-case ${tmpfile} + used_source="$(jq -r '.source' "$version"/.nupkg.metadata)" + for source in "${sources[@]}"; do + url="${base_addresses[$source]}$package/$version/$package.$version.nupkg" + if [[ "$source" == "$used_source" ]]; then + sha256="$(nix-hash --type sha256 --flat --base32 "$version/$package.$version".nupkg)" + found=true + break + else + if sha256=$(nix-prefetch-url "$url" 2>"$tmp"/error); then + # If multiple remote sources are enabled, nuget will try them all + # concurrently and use the one that responds first. We always use the + # first source that has the package. + echo "$package $version is available on $url, but was downloaded from ${base_addresses[$used_source]}$package/$version/$package.$version.nupkg" 1>&2 + found=true + break + else + if ! grep -q 'HTTP error 404' "$tmp/error"; then + cat "$tmp/error" 1>&2 + exit 1 + fi + fi + fi + done -echo "]" + if ! ${found-false}; then + echo "couldn't find $package $version" >&2 + exit 1 + fi + + if [[ "$source" != https://api.nuget.org/v3/index.json ]]; then + echo " (fetchNuGet { pname = \"$id\"; version = \"$version\"; sha256 = \"$sha256\"; url = \"$url\"; })" + else + echo " (fetchNuGet { pname = \"$id\"; version = \"$version\"; sha256 = \"$sha256\"; })" + fi + done + cd .. +done + +cat << EOL +] +EOL diff --git a/pkgs/development/tools/continuous-integration/github-runner/default.nix b/pkgs/development/tools/continuous-integration/github-runner/default.nix index 5dcc556660fa..a9737a0e12d4 100644 --- a/pkgs/development/tools/continuous-integration/github-runner/default.nix +++ b/pkgs/development/tools/continuous-integration/github-runner/default.nix @@ -302,12 +302,12 @@ stdenv.mkDerivation rec { # Inspired by passthru.fetch-deps in pkgs/build-support/build-dotnet-module/default.nix passthru.createDepsFile = writeShellApplication { name = "create-deps-file"; - runtimeInputs = [ dotnetSdk nuget-to-nix ]; + runtimeInputs = [ dotnetSdk (nuget-to-nix.override { dotnet-sdk = dotnetSdk; }) ]; text = '' # Disable telemetry data export DOTNET_CLI_TELEMETRY_OPTOUT=1 - rundir=$(pwd) + deps_file="$(realpath "''${1:-$(mktemp -t "${pname}-deps-XXXXXX.nix")}")" printf "\n* Setup workdir\n" workdir="$(mktemp -d /tmp/${pname}.XXX)" @@ -324,8 +324,6 @@ stdenv.mkDerivation rec { dotnet restore src/ActionsRunner.sln --packages nuget_pkgs --no-cache --force --runtime "${rid}" '') (lib.attrValues runtimeIds)} - cd "$rundir" - deps_file=''${1-"/tmp/${pname}-deps.nix"} printf "\n* Make %s file\n" "$(basename "$deps_file")" nuget-to-nix "$workdir/nuget_pkgs" > "$deps_file" printf "\n* Dependency file writen to %s" "$deps_file"