diff --git a/pkgs/development/compilers/dotnet/binary/build-dotnet.nix b/pkgs/development/compilers/dotnet/binary/build-dotnet.nix index c027e206b63b..3f023ff70757 100644 --- a/pkgs/development/compilers/dotnet/binary/build-dotnet.nix +++ b/pkgs/development/compilers/dotnet/binary/build-dotnet.nix @@ -205,15 +205,6 @@ mkWrapper type ( ); targetPackages = lib.mapAttrs (_: map forceSDKEval) targetPackages; inherit runtime aspnetcore; - - updateScript = - let - majorVersion = lib.concatStringsSep "." (lib.take 2 (lib.splitVersion version)); - in - [ - ./update.sh - majorVersion - ]; } ); diff --git a/pkgs/development/compilers/dotnet/binary/update.sh b/pkgs/development/compilers/dotnet/binary/update.sh index aff85a32ecad..116fed9c9aad 100755 --- a/pkgs/development/compilers/dotnet/binary/update.sh +++ b/pkgs/development/compilers/dotnet/binary/update.sh @@ -266,7 +266,7 @@ update() { return 1 fi - : ${output:="$(realpath "$(dirname "${BASH_SOURCE[0]}")")"/$sem_version/releases.nix} + : ${output:="$(realpath "$(dirname "${BASH_SOURCE[0]}")")"/../$sem_version/releases.nix} echo "Generating $output" # Make sure the x.y version is properly passed to .NET release metadata url. @@ -413,6 +413,7 @@ in rec { )> "$result" nixfmt "$result" + mkdir -p "$(dirname -- "$output")" cp "$result" "$output" echo "Generated $output" } diff --git a/pkgs/development/compilers/dotnet/default.nix b/pkgs/development/compilers/dotnet/default.nix index 60b43dd7aef3..840196c9c895 100644 --- a/pkgs/development/compilers/dotnet/default.nix +++ b/pkgs/development/compilers/dotnet/default.nix @@ -84,30 +84,29 @@ let // ( let dotnet_6 = callPackage ./dotnet.nix { - dir = ./6.0; + channel = "6.0"; withVMR = false; }; dotnet_7 = callPackage ./dotnet.nix { - dir = ./7.0; + channel = "7.0"; withVMR = false; }; dotnet_8 = callPackage ./dotnet.nix { - dir = ./8.0; + channel = "8.0"; }; dotnet_9 = callPackage ./dotnet.nix { - dir = ./9.0; + channel = "9.0"; }; dotnet_10 = callPackage ./dotnet.nix { - dir = ./10.0; + channel = "10.0"; }; dotnet_11 = callPackage ./dotnet.nix { - dir = ./11.0; - allowPrerelease = true; + channel = "11.0"; }; in lib.optionalAttrs config.allowAliases { diff --git a/pkgs/development/compilers/dotnet/dotnet.nix b/pkgs/development/compilers/dotnet/dotnet.nix index e223a926bf4d..b484ef3e3ef4 100644 --- a/pkgs/development/compilers/dotnet/dotnet.nix +++ b/pkgs/development/compilers/dotnet/dotnet.nix @@ -2,15 +2,15 @@ config, callPackage, lib, - dir ? null, - releasesFile ? dir + "/releases.nix", + channel, + dir ? ./. + ("/" + channel), buildDotnetSdk, withVMR ? true, ... }@attrs: let - binary = buildDotnetSdk releasesFile; + binary = buildDotnetSdk (dir + "/releases.nix"); sourcePackages = lib.optionalAttrs withVMR (callPackage ./source (attrs // { inherit binary; })); @@ -18,5 +18,18 @@ let lib.optionalAttrs config.allowAliases binary // lib.mapAttrs' (k: v: lib.nameValuePair "${k}-bin" v) binary // sourcePackages; + + suffix = lib.replaceStrings [ "." ] [ "_" ] channel; + sdkAttr = "sdk_${suffix}" + lib.optionalString (!withVMR) "-bin"; in pkgs +// { + ${sdkAttr} = pkgs.${sdkAttr}.overrideAttrs (prev: { + passthru = prev.passthru or { } // { + updateScript = [ + ./update.sh + channel + ]; + }; + }); +} diff --git a/pkgs/development/compilers/dotnet/combine-deps.nix b/pkgs/development/compilers/dotnet/source/combine-deps.nix similarity index 96% rename from pkgs/development/compilers/dotnet/combine-deps.nix rename to pkgs/development/compilers/dotnet/source/combine-deps.nix index 0f60566aa43f..8e8886a953d9 100644 --- a/pkgs/development/compilers/dotnet/combine-deps.nix +++ b/pkgs/development/compilers/dotnet/source/combine-deps.nix @@ -2,7 +2,7 @@ list, baseRid, otherRids, - pkgs ? import ../../../.. { }, + pkgs ? import ../../../../.. { }, }: let inherit (pkgs) writeText; diff --git a/pkgs/development/compilers/dotnet/source/default.nix b/pkgs/development/compilers/dotnet/source/default.nix index 7a9672da606d..88f33c851ac0 100644 --- a/pkgs/development/compilers/dotnet/source/default.nix +++ b/pkgs/development/compilers/dotnet/source/default.nix @@ -4,12 +4,13 @@ callPackage, lib, fetchurl, - dir ? null, + channel, + featureBand ? "1xx", + dir ? ../. + ("/" + channel), releaseManifestFile ? dir + "/release.json", releaseInfoFile ? dir + "/release-info.json", bootstrapSdkFile ? dir + "/bootstrap-sdk.nix", bootstrapSdk ? null, - allowPrerelease ? false, depsFile ? dir + "/deps.json", pkgsBuildHost, buildDotnetSdk, @@ -22,22 +23,16 @@ assert bootstrapSdk != null || bootstrapSdkFile != null; let - inherit (lib.importJSON releaseManifestFile) - sdkVersion - ; - suffix = let - parts = lib.take 3 (lib.splitVersion sdkVersion); - patch = lib.elemAt parts 2; - band = lib.substring 0 (lib.stringLength patch - 2) patch; + parts = lib.splitVersion channel; major = lib.elemAt parts 0; - channel = lib.concatStringsSep "_" (lib.take 2 parts); - sdk = lib.concatStringsSep "_" (lib.replaceElemAt parts 2 "${band}xx"); + sdk = lib.concatStringsSep "_" (parts ++ [ featureBand ]); in { - source = if band == "1" then major else sdk; - inherit major channel sdk; + source = if featureBand == "1xx" then major else sdk; + channel = lib.concatStringsSep "_" (lib.take 2 parts); + inherit major sdk; }; releaseInfo = (lib.importJSON releaseInfoFile); @@ -96,23 +91,7 @@ let inherit vmr fallbackTargetPackages; }; in - lib.recurseIntoAttrs ( - pkgs - // { - vmr = pkgs.vmr.overrideAttrs (old: { - passthru = old.passthru // { - updateScript = pkgsBuildHost.callPackage ./update.nix { - inherit - releaseManifestFile - releaseInfoFile - bootstrapSdkFile - allowPrerelease - ; - }; - }; - }); - } - ); + lib.recurseIntoAttrs pkgs; source = vmrPackages @@ -222,6 +201,7 @@ in callPackage ./default.nix ( attrs // { + inherit featureBand; dir = dir + "/${featureBand}"; withBinary = false; bootstrapSdk = vmrPackages.sdk; diff --git a/pkgs/development/compilers/dotnet/source/packages.nix b/pkgs/development/compilers/dotnet/source/packages.nix index d804fa7032bc..0b5f653c51ba 100644 --- a/pkgs/development/compilers/dotnet/source/packages.nix +++ b/pkgs/development/compilers/dotnet/source/packages.nix @@ -178,7 +178,11 @@ let ''; passthru = { - inherit (vmr) icu targetRid hasILCompiler; + inherit (vmr) + icu + targetRid + hasILCompiler + ; inherit packages diff --git a/pkgs/development/compilers/dotnet/source/update-dotnet-vmr.sh b/pkgs/development/compilers/dotnet/source/update-dotnet-vmr.sh deleted file mode 100755 index 7746fbb5a525..000000000000 --- a/pkgs/development/compilers/dotnet/source/update-dotnet-vmr.sh +++ /dev/null @@ -1,139 +0,0 @@ -#! @nix@/bin/nix-shell -#! nix-shell -i @runtimeShell@ --pure @drv@ --keep UPDATE_NIX_ATTR_PATH -set -euo pipefail - -tag=${1-} - -if [[ -n $tag ]]; then - query=$(cat <&2 echo "release is already $tagName" - exit - fi - - tarballUrl=https://github.com/dotnet/dotnet/archive/refs/tags/$tagName.tar.gz - - mapfile -t prefetch < <(nix-prefetch-url --print-path "$tarballUrl") - tarballHash=$(nix-hash --to-sri --type sha256 "${prefetch[0]}") - tarball=${prefetch[1]} - - # recent dotnet 10 releases don't have a signature for the github tarball - if [[ ! $sigUrl = */dotnet-source-* ]]; then - curl -fssL "$sigUrl" -o release.sig - - ( - export GNUPGHOME=$PWD/.gnupg - mkdir -m 700 -p $GNUPGHOME - trap 'gpgconf --kill all' EXIT - gpg --no-autostart --batch --import @releaseKey@ - gpg --no-autostart --batch --verify release.sig "$tarball" - ) - fi - - tar --strip-components=1 --no-wildcards-match-slash --wildcards -xzf "$tarball" \*/eng/Versions.props \*/global.json \*/prep\*.sh - artifactsVersion=$(xq -r '.Project.PropertyGroup | - map(select(.PrivateSourceBuiltArtifactsVersion)) - | .[] | .PrivateSourceBuiltArtifactsVersion' eng/Versions.props) - - if [[ "$artifactsVersion" != "" ]]; then - artifactVar=$(grep ^defaultArtifactsRid= prep-source-build.sh) - eval "$artifactVar" - - artifactsFile=Private.SourceBuilt.Artifacts.$artifactsVersion.$defaultArtifactsRid.tar.gz - artifactsUrl=@dotnetBuildUrl@/$artifactsFile - - curl -fsSL "$artifactsUrl" --head || { - [[ $? == 22 ]] - artifactsUrl=https://ci.dot.net/public/source-build/$artifactsFile - } - else - artifactsUrl=$(xq -r '.Project.PropertyGroup | - map(select(.PrivateSourceBuiltArtifactsUrl)) - | .[] | .PrivateSourceBuiltArtifactsUrl' eng/Versions.props) - artifactsUrl="${artifactsUrl/dotnetcli.azureedge.net/builds.dotnet.microsoft.com}" - fi - - artifactsHash=$(nix-prefetch-url "$artifactsUrl") - artifactsHash=$(nix-hash --to-sri --type sha256 "$artifactsHash") - - sdkVersion=$(jq -er .tools.dotnet global.json) - - # below needs to be run in nixpkgs because toOutputPath uses relative paths - cd - - - cp "$tmp"/release.json "@releaseManifestFile@" - - jq --null-input \ - --arg _0 "$tarballHash" \ - --arg _1 "$artifactsUrl" \ - --arg _2 "$artifactsHash" \ - '{ - "tarballHash": $_0, - "artifactsUrl": $_1, - "artifactsHash": $_2, - }' > "@releaseInfoFile@" - - if [[ -n @bootstrapSdkFile@ ]]; then - updateSDK() { - @updateScript@ \ - -o @bootstrapSdkFile@ --sdk "$1" >&2 - } - - updateSDK "$sdkVersion" || if [[ $? == 2 ]]; then - >&2 echo "WARNING: bootstrap sdk missing, attempting to bootstrap with self" - updateSDK "$(jq -er .sdkVersion "$tmp"/release.json)" - else - exit 1 - fi - fi - - $(nix-build -A $UPDATE_NIX_ATTR_PATH.fetch-deps --no-out-link) >&2 -) diff --git a/pkgs/development/compilers/dotnet/source/update.nix b/pkgs/development/compilers/dotnet/source/update.nix deleted file mode 100644 index d40c18bbcf08..000000000000 --- a/pkgs/development/compilers/dotnet/source/update.nix +++ /dev/null @@ -1,86 +0,0 @@ -{ - stdenvNoCC, - lib, - fetchurl, - writeScript, - nix, - runtimeShell, - curl, - cacert, - jq, - yq, - gnupg, - replaceVarsWith, - - releaseManifestFile, - releaseInfoFile, - bootstrapSdkFile, - allowPrerelease, -}: - -let - inherit (lib.importJSON releaseManifestFile) channel tag sdkVersion; - - # version including up to the sdk feature band - sdkVersionPrefix = - let - parts = lib.take 3 (lib.splitVersion sdkVersion); - patch = lib.elemAt parts 2; - band = lib.substring 0 (lib.stringLength patch - 2) patch; - in - lib.concatStringsSep "." (lib.replaceElemAt parts 2 band); - - pkg = stdenvNoCC.mkDerivation { - name = "update-dotnet-vmr-env"; - - nativeBuildInputs = [ - nix - curl - cacert - jq - yq - gnupg - ]; - }; - - releaseKey = fetchurl { - url = "https://dotnet.microsoft.com/download/dotnet/release-key-2023.asc"; - hash = "sha256-F668QB55md0GQvoG0jeA66Fb2RbrsRhFTzTbXIX3GUo="; - }; - - drv = builtins.unsafeDiscardOutputDependency pkg.drvPath; - - toOutputPath = - path: - let - root = ../../../..; - in - lib.path.removePrefix root path; - -in -replaceVarsWith { - src = ./update-dotnet-vmr.sh; - - replacements = { - inherit - nix - runtimeShell - drv - sdkVersionPrefix - tag - releaseKey - ; - prereleaseExpr = lib.optionalString (!allowPrerelease) ".prerelease == false and"; - dotnetBuildUrl = - "https://builds.dotnet.microsoft.com/" - + ( - if lib.versionAtLeast channel "10" then "dotnet/source-build" else "source-built-artifacts/assets" - ); - releaseManifestFile = toOutputPath releaseManifestFile; - releaseInfoFile = toOutputPath releaseInfoFile; - updateScript = lib.escapeShellArg (toOutputPath ./update.sh); - bootstrapSdkFile = lib.escapeShellArg (lib.mapNullable toOutputPath bootstrapSdkFile); - }; - - isExecutable = true; -} diff --git a/pkgs/development/compilers/dotnet/source/update.sh b/pkgs/development/compilers/dotnet/source/update.sh new file mode 100755 index 000000000000..fc78d31aab08 --- /dev/null +++ b/pkgs/development/compilers/dotnet/source/update.sh @@ -0,0 +1,190 @@ +#!/usr/bin/env nix-shell +#!nix-shell -I nixpkgs=./. -i bash -p curl jq nix yq gnupg +set -euo pipefail + +updateSdk() { + tag=${1-} + + if [[ -n $tag ]]; then + query=$(cat <&2 echo "release is already $tagName" + exit + } + + # TMPDIR might be really long, which breaks gpg + tmp="$(TMPDIR=/tmp mktemp -d)" + trap 'rm -rf "$tmp"' EXIT + + cd "$tmp" + + curl -fsSL "$releaseUrl" -o release.json + + tarballUrl=https://github.com/dotnet/dotnet/archive/refs/tags/$tagName.tar.gz + + mapfile -t prefetch < <(nix-prefetch-url --print-path "$tarballUrl") + tarballHash=$(nix-hash --to-sri --type sha256 "${prefetch[0]}") + tarball=${prefetch[1]} + + # recent dotnet 10 releases don't have a signature for the github tarball + if [[ ! $sigUrl = */dotnet-source-* ]]; then + curl -fssL "$sigUrl" -o release.sig + curl -fssLO https://dotnet.microsoft.com/download/dotnet/release-key-2023.asc + echo '17aebc401e7999dd0642fa06d23780eba15bd916ebb118454f34db5c85f7194a release-key-2023.asc' | + sha256sum -c + + ( + export GNUPGHOME=$PWD/.gnupg + mkdir -m 700 -p $GNUPGHOME + trap 'gpgconf --kill all' EXIT + gpg --no-autostart --batch --import release-key-2023.asc + gpg --no-autostart --batch --verify release.sig "$tarball" + ) + fi + + tar --strip-components=1 --no-wildcards-match-slash --wildcards -xzf "$tarball" \*/eng/Versions.props \*/global.json \*/prep\*.sh + artifactsVersion=$(xq -r '.Project.PropertyGroup | + map(select(.PrivateSourceBuiltArtifactsVersion)) + | .[] | .PrivateSourceBuiltArtifactsVersion' eng/Versions.props) + + if [[ "$artifactsVersion" != "" ]]; then + artifactVar=$(grep ^defaultArtifactsRid= prep-source-build.sh) + eval "$artifactVar" + + artifactsFile=Private.SourceBuilt.Artifacts.$artifactsVersion.$defaultArtifactsRid.tar.gz + + dotnetBuildUrl=https://builds.dotnet.microsoft.com/ + + if [[ $major -ge 10 ]]; then + dotnetBuildUrl+=dotnet/source-build + else + dotnetBuildUrl+=source-built-artifacts/assets + fi + + artifactsUrl=$dotnetBuildUrl/$artifactsFile + + curl -fsSL "$artifactsUrl" --head || { + [[ $? == 22 ]] + artifactsUrl=https://ci.dot.net/public/source-build/$artifactsFile + } + else + artifactsUrl=$(xq -r '.Project.PropertyGroup | + map(select(.PrivateSourceBuiltArtifactsUrl)) + | .[] | .PrivateSourceBuiltArtifactsUrl' eng/Versions.props) + artifactsUrl="${artifactsUrl/dotnetcli.azureedge.net/builds.dotnet.microsoft.com}" + fi + + artifactsHash=$(nix-prefetch-url "$artifactsUrl") + artifactsHash=$(nix-hash --to-sri --type sha256 "$artifactsHash") + + sdkVersion=$(jq -er .tools.dotnet global.json) + + # below needs to be run in nixpkgs because toOutputPath uses relative paths + cd - + + cp "$tmp"/release.json "$output" + + jq --null-input \ + --arg _0 "$tarballHash" \ + --arg _1 "$artifactsUrl" \ + --arg _2 "$artifactsHash" \ + '{ + "tarballHash": $_0, + "artifactsUrl": $_1, + "artifactsHash": $_2, + }' > "$output"/release-info.json + + if [[ $band == 1xx ]]; then + getBootstrap() { + pkgs/development/compilers/dotnet/binary/update.sh \ + -o "$output"/bootstrap-sdk.nix --sdk "$1" >&2 + } + + getBootstrap "$sdkVersion" || if [[ $? == 2 ]]; then + >&2 echo "WARNING: bootstrap sdk missing, attempting to bootstrap with self" + getBootstrap "$(jq -er .sdkVersion "$tmp"/release.json)" + else + exit 1 + fi + + $(nix-build -A dotnetCorePackages.dotnet_$major.vmr.fetch-deps --no-out-link) >&2 + fi + ) +} + +while [ $# -gt 0 ]; do + channel="$1" + shift + + major="${channel%.*}" + + if [[ $major -lt 8 ]]; then + >&2 echo "dotnet $major has no vmr" + continue + fi + + curl -fsSL https://dotnetcli.blob.core.windows.net/dotnet/release-metadata/"$channel"/releases.json | + jq -r '.releases[0].sdks.[] | .version' | { + while read sdk; do + if [[ $major -lt 10 && $sdk != *.*.1* ]]; then + >&2 echo "sdk $sdk has no vmr" + continue + fi + + updateSdk v"$sdk" + done + } +done diff --git a/pkgs/development/compilers/dotnet/update.sh b/pkgs/development/compilers/dotnet/update.sh new file mode 100755 index 000000000000..408899d2da7f --- /dev/null +++ b/pkgs/development/compilers/dotnet/update.sh @@ -0,0 +1,7 @@ +#!/bin/sh +# shellcheck shell=bash + +set -Eeuo pipefail + +pkgs/development/compilers/dotnet/binary/update.sh "$@" +pkgs/development/compilers/dotnet/source/update.sh "$@"