yarn-berry: init fetcher infrastructure

pkgs/by-name/ya/yarn-berry/fetcher/berry-3-offline.patch

@@ -0,0 +1,90 @@
+diff --git a/packages/plugin-essentials/sources/commands/install.ts b/packages/plugin-essentials/sources/commands/install.ts
+index 9dcd02d12..cf1765a20 100644
+--- a/packages/plugin-essentials/sources/commands/install.ts
++++ b/packages/plugin-essentials/sources/commands/install.ts
+@@ -254,6 +254,7 @@ export default class YarnCommand extends BaseCommand {
+       // If migrating from a v1 install, we automatically enable the node-modules linker,
+       // since that's likely what the author intended to do.
+       if (content?.includes(`yarn lockfile v1`)) {
++        throw new Error("Tried to use yarn-berry_3.yarnConfigHook (nixpkgs), but found a yarn v1 lockfile");
+         const nmReport = await StreamReport.start({
+           configuration,
+           json: this.json,
+diff --git a/packages/plugin-git/sources/GitFetcher.ts b/packages/plugin-git/sources/GitFetcher.ts
+index fe2a4fce8..bfa82728e 100644
+--- a/packages/plugin-git/sources/GitFetcher.ts
++++ b/packages/plugin-git/sources/GitFetcher.ts
+@@ -50,9 +50,14 @@ export class GitFetcher implements Fetcher {
+   }
+ 
+   async cloneFromRemote(locator: Locator, opts: FetchOptions) {
+-    const cloneTarget = await gitUtils.clone(locator.reference, opts.project.configuration);
+-
+     const repoUrlParts = gitUtils.splitRepoUrl(locator.reference);
++
++    if (repoUrlParts.treeish.protocol !== "commit") {
++        throw new Error(`Missing source for git dependency ${locator.reference}`);
++    };
++
++    const cloneTarget = opts.cache.checkoutPath(repoUrlParts.treeish.request);
++
+     const packagePath = ppath.join(cloneTarget, `package.tgz` as PortablePath);
+ 
+     await scriptUtils.prepareExternalProject(cloneTarget, packagePath, {
+diff --git a/packages/plugin-npm/sources/NpmSemverFetcher.ts b/packages/plugin-npm/sources/NpmSemverFetcher.ts
+index 0f69423c7..5b21462a5 100644
+--- a/packages/plugin-npm/sources/NpmSemverFetcher.ts
++++ b/packages/plugin-npm/sources/NpmSemverFetcher.ts
+@@ -47,6 +47,7 @@ export class NpmSemverFetcher implements Fetcher {
+   }
+ 
+   private async fetchFromNetwork(locator: Locator, opts: FetchOptions) {
++    throw new Error(`Missing sources for ${structUtils.prettyLocator(opts.project.configuration, locator)}`);
+     let sourceBuffer;
+     try {
+       sourceBuffer = await npmHttpUtils.get(NpmSemverFetcher.getLocatorUrl(locator), {
+diff --git a/packages/yarnpkg-core/sources/Cache.ts b/packages/yarnpkg-core/sources/Cache.ts
+index d5e686420..374b5d67f 100644
+--- a/packages/yarnpkg-core/sources/Cache.ts
++++ b/packages/yarnpkg-core/sources/Cache.ts
+@@ -158,6 +158,10 @@ export class Cache {
+     }
+   }
+ 
++  checkoutPath(commit: string): string {
++    return ppath.join(ppath.join(this.cwd, "../checkouts"), commit);
++  }
++
+   async fetchPackageFromCache(locator: Locator, expectedChecksum: string | null, {onHit, onMiss, loader, ...opts}: {onHit?: () => void, onMiss?: () => void, loader?: () => Promise<ZipFS> } & CacheOptions): Promise<[FakeFS<PortablePath>, () => void, string | null]> {
+     const mirrorPath = this.getLocatorMirrorPath(locator);
+ 
+diff --git a/packages/yarnpkg-core/sources/scriptUtils.ts b/packages/yarnpkg-core/sources/scriptUtils.ts
+index b3c2c5903..15cae13fd 100644
+--- a/packages/yarnpkg-core/sources/scriptUtils.ts
++++ b/packages/yarnpkg-core/sources/scriptUtils.ts
+@@ -287,9 +287,9 @@ export async function prepareExternalProject(cwd: PortablePath, outputPath: Port
+             // Run an install; we can't avoid it unless we inspect the
+             // package.json, which I don't want to do to keep the codebase
+             // clean (even if it has a slight perf cost when cloning v1 repos)
+-            const install = await execUtils.pipevp(`yarn`, [`install`], {cwd, env, stdin, stdout, stderr, end: execUtils.EndStrategy.ErrorCode});
+-            if (install.code !== 0)
+-              return install.code;
++            //const install = await execUtils.pipevp(`yarn`, [`install`], {cwd, env, stdin, stdout, stderr, end: execUtils.EndStrategy.ErrorCode});
++            //if (install.code !== 0)
++            //  return install.code;
+ 
+             stdout.write(`\n`);
+ 
+@@ -375,9 +375,9 @@ export async function prepareExternalProject(cwd: PortablePath, outputPath: Port
+             // We can't use `npm ci` because some projects don't have npm
+             // lockfiles that are up-to-date. Hopefully npm won't decide
+             // to change the versions randomly.
+-            const install = await execUtils.pipevp(`npm`, [`install`], {cwd, env, stdin, stdout, stderr, end: execUtils.EndStrategy.ErrorCode});
+-            if (install.code !== 0)
+-              return install.code;
++            //const install = await execUtils.pipevp(`npm`, [`install`], {cwd, env, stdin, stdout, stderr, end: execUtils.EndStrategy.ErrorCode});
++            //if (install.code !== 0)
++            //  return install.code;
+ 
+             const packStream = new PassThrough();
+             const packPromise = miscUtils.bufferStream(packStream);

pkgs/by-name/ya/yarn-berry/fetcher/berry-4-offline.patch

@@ -0,0 +1,86 @@
+diff --git a/packages/plugin-essentials/sources/commands/install.ts b/packages/plugin-essentials/sources/commands/install.ts
+index 90ba55349..ef5368c1b 100644
+--- a/packages/plugin-essentials/sources/commands/install.ts
++++ b/packages/plugin-essentials/sources/commands/install.ts
+@@ -302,6 +302,7 @@ export default class YarnCommand extends BaseCommand {
+ 
+         for (const rule of LOCKFILE_MIGRATION_RULES) {
+           if (rule.selector(lockfileLastVersion) && typeof configuration.sources.get(rule.name) === `undefined`) {
++            throw new Error(`Tried to use yarn-berry_4.yarnConfigHook (nixpkgs) which expects lockfile version 8, but found lockfile version ${lockfileLastVersion}`);
+             configuration.use(`<compat>`, {[rule.name]: rule.value}, project.cwd, {overwrite: true});
+             newSettings[rule.name] = rule.value;
+           }
+diff --git a/packages/plugin-git/sources/GitFetcher.ts b/packages/plugin-git/sources/GitFetcher.ts
+index d9f8d85c7..4db9f9008 100644
+--- a/packages/plugin-git/sources/GitFetcher.ts
++++ b/packages/plugin-git/sources/GitFetcher.ts
+@@ -50,7 +50,11 @@ export class GitFetcher implements Fetcher {
+   async cloneFromRemote(locator: Locator, opts: FetchOptions) {
+     const repoUrlParts = gitUtils.splitRepoUrl(locator.reference);
+ 
+-    const cloneTarget = await gitUtils.clone(locator.reference, opts.project.configuration);
++    if (repoUrlParts.treeish.protocol !== "commit") {
++        throw new Error(`Missing source for git dependency ${locator.reference}`);
++    };
++
++    const cloneTarget = opts.cache.checkoutPath(repoUrlParts.treeish.request);
+     const projectPath = ppath.resolve(cloneTarget, repoUrlParts.extra.cwd ?? PortablePath.dot);
+ 
+     const packagePath = ppath.join(projectPath, `package.tgz`);
+diff --git a/packages/plugin-npm/sources/NpmSemverFetcher.ts b/packages/plugin-npm/sources/NpmSemverFetcher.ts
+index 7347859aa..ea5767f88 100644
+--- a/packages/plugin-npm/sources/NpmSemverFetcher.ts
++++ b/packages/plugin-npm/sources/NpmSemverFetcher.ts
+@@ -45,6 +45,7 @@ export class NpmSemverFetcher implements Fetcher {
+   }
+ 
+   private async fetchFromNetwork(locator: Locator, opts: FetchOptions) {
++    throw new Error(`Missing sources for ${structUtils.prettyLocator(opts.project.configuration, locator)}`);
+     let sourceBuffer;
+     try {
+       sourceBuffer = await npmHttpUtils.get(NpmSemverFetcher.getLocatorUrl(locator), {
+diff --git a/packages/yarnpkg-core/sources/Cache.ts b/packages/yarnpkg-core/sources/Cache.ts
+index b712ecf11..c7effbc61 100644
+--- a/packages/yarnpkg-core/sources/Cache.ts
++++ b/packages/yarnpkg-core/sources/Cache.ts
+@@ -225,6 +225,10 @@ export class Cache {
+     }
+   }
+ 
++  checkoutPath(commit: string): string {
++    return ppath.join(ppath.join(this.cwd, "../checkouts"), commit);
++  }
++
+   async fetchPackageFromCache(locator: Locator, expectedChecksum: string | null, {onHit, onMiss, loader, ...opts}: {onHit?: () => void, onMiss?: () => void, loader?: () => Promise<ZipFS>} & CacheOptions): Promise<[FakeFS<PortablePath>, () => void, string | null]> {
+     const mirrorPath = this.getLocatorMirrorPath(locator);
+ 
+diff --git a/packages/yarnpkg-core/sources/scriptUtils.ts b/packages/yarnpkg-core/sources/scriptUtils.ts
+index 2dcd7e59e..3079ad635 100644
+--- a/packages/yarnpkg-core/sources/scriptUtils.ts
++++ b/packages/yarnpkg-core/sources/scriptUtils.ts
+@@ -287,9 +287,9 @@ export async function prepareExternalProject(cwd: PortablePath, outputPath: Port
+             // Run an install; we can't avoid it unless we inspect the
+             // package.json, which I don't want to do to keep the codebase
+             // clean (even if it has a slight perf cost when cloning v1 repos)
+-            const install = await execUtils.pipevp(`yarn`, [`install`], {cwd, env, stdin, stdout, stderr, end: execUtils.EndStrategy.ErrorCode});
+-            if (install.code !== 0)
+-              return install.code;
++            //const install = await execUtils.pipevp(`yarn`, [`install`], {cwd, env, stdin, stdout, stderr, end: execUtils.EndStrategy.ErrorCode});
++            //if (install.code !== 0)
++            //  return install.code;
+ 
+             stdout.write(`\n`);
+ 
+@@ -375,9 +375,9 @@ export async function prepareExternalProject(cwd: PortablePath, outputPath: Port
+             // We can't use `npm ci` because some projects don't have npm
+             // lockfiles that are up-to-date. Hopefully npm won't decide
+             // to change the versions randomly.
+-            const install = await execUtils.pipevp(`npm`, [`install`, `--legacy-peer-deps`], {cwd, env, stdin, stdout, stderr, end: execUtils.EndStrategy.ErrorCode});
+-            if (install.code !== 0)
+-              return install.code;
++            //const install = await execUtils.pipevp(`npm`, [`install`, , `--legacy-peer-deps`], {cwd, env, stdin, stdout, stderr, end: execUtils.EndStrategy.ErrorCode});
++            //if (install.code !== 0)
++            //  return install.code;
+ 
+             const packStream = new PassThrough();
+             const packPromise = miscUtils.bufferStream(packStream);

pkgs/by-name/ya/yarn-berry/fetcher/default.nix

@@ -0,0 +1,77 @@
+{
+  lib,
+  newScope,
+  yarn-berry,
+  libzip,
+  zlib,
+  zlib-ng,
+}:
+
+let
+  variantOverlays = {
+    "3" = final: {
+      berryCacheVersion = "8";
+
+      berryOfflinePatches = [ ./berry-3-offline.patch ];
+
+      # Known good version: 1.11.3
+      libzip =
+        (libzip.override {
+          # Known good version: 1.3.1
+          zlib = zlib;
+        }).overrideAttrs
+          (oA: {
+            patches = (oA.patches or [ ]) ++ [
+              (final.yarn-berry-fetcher.src + "/libzip-revert-to-old-versionneeded-behavior.patch")
+            ];
+          });
+    };
+    "4" = final: {
+      berryCacheVersion = "10";
+
+      berryOfflinePatches = [ ./berry-4-offline.patch ];
+
+      # Known good version: 1.11.3
+      libzip =
+        (libzip.override {
+          # Known good version: 2.2.4
+          zlib = zlib-ng.override {
+            withZlibCompat = true;
+          };
+        }).overrideAttrs
+          (oA: {
+            patches = (oA.patches or [ ]) ++ [
+              (final.yarn-berry-fetcher.src + "/libzip-revert-to-old-versionneeded-behavior.patch")
+            ];
+          });
+    };
+  };
+in
+
+lib.makeScope newScope (
+  final:
+  let
+    berryVersion = lib.versions.major yarn-berry.version;
+
+    err = throw ''
+      Berry version ${toString berryVersion} not supported by yarn-berry-fetcher.
+      Supported versions: ${lib.concatStringsSep ", " (lib.attrNames variantOverlays)}
+    '';
+    variantOverlay = (variantOverlays.${berryVersion} or err) final;
+  in
+  (
+    {
+      inherit yarn-berry berryVersion;
+
+      yarn-berry-offline = final.yarn-berry.overrideAttrs (old: {
+        pname = old.pname + "-offline";
+        patches = (old.patches or [ ]) ++ final.berryOfflinePatches;
+      });
+
+      yarn-berry-fetcher = final.callPackage ./yarn-berry-fetcher.nix { };
+      fetchYarnBerryDeps = final.callPackage ./fetch-yarn-berry-deps.nix { };
+      yarnBerryConfigHook = final.callPackage ./yarn-berry-config-hook.nix { };
+    }
+    // variantOverlay
+  )
+)

pkgs/by-name/ya/yarn-berry/fetcher/fetch-yarn-berry-deps.nix

@@ -0,0 +1,76 @@
+{
+  lib,
+  stdenv,
+  yarn-berry-fetcher,
+  nix-prefetch-git,
+  cacert,
+  berryVersion,
+}:
+
+{
+  src ? null,
+  hash ? "",
+  sha256 ? "",
+  ...
+}@args:
+
+let
+  hash_ =
+    if hash != "" then
+      {
+        outputHashAlgo = null;
+        outputHash = hash;
+      }
+    else if sha256 != "" then
+      {
+        outputHashAlgo = "sha256";
+        outputHash = sha256;
+      }
+    else
+      {
+        outputHashAlgo = "sha256";
+        outputHash = lib.fakeSha256;
+      };
+in
+
+stdenv.mkDerivation (
+  {
+    # The name is fixed as to not produce multiple store paths with the same content
+    name = "offline";
+
+    dontUnpack = src == null;
+    dontInstall = true;
+
+    nativeBuildInputs = [
+      yarn-berry-fetcher
+      nix-prefetch-git
+      cacert
+    ];
+
+    buildPhase = ''
+      runHook preBuild
+
+      yarnLock=''${yarnLock:=$PWD/yarn.lock}
+      yarn-berry-fetcher fetch $yarnLock $missingHashes
+
+      runHook postBuild
+    '';
+
+    outputHashMode = "recursive";
+
+    passthru = {
+      inherit berryVersion;
+    };
+  }
+  // hash_
+  // (removeAttrs args (
+    [
+      "name"
+      "pname"
+      "version"
+      "hash"
+      "sha256"
+    ]
+    ++ (lib.optional (src == null) "src")
+  ))
+)

pkgs/by-name/ya/yarn-berry/fetcher/yarn-berry-config-hook.nix

@@ -0,0 +1,24 @@
+{
+  makeSetupHook,
+  yarn-berry-offline,
+  srcOnly,
+  nodejs,
+  diffutils,
+}:
+
+makeSetupHook {
+  name = "yarn-berry-config-hook";
+  substitutions = {
+    # Specify `diff` by abspath to ensure that the user's build
+    # inputs do not cause us to find the wrong binaries.
+    diff = "${diffutils}/bin/diff";
+
+    yarn_offline = "${yarn-berry-offline}/bin/yarn";
+
+    nodeSrc = srcOnly nodejs;
+    nodeGyp = "${nodejs}/lib/node_modules/npm/node_modules/node-gyp/bin/node-gyp.js";
+  };
+  meta = {
+    description = "Install nodejs dependencies from an offline yarn cache produced by fetchYarnDeps";
+  };
+} ./yarn-berry-config-hook.sh

pkgs/by-name/ya/yarn-berry/fetcher/yarn-berry-config-hook.sh

@@ -0,0 +1,85 @@
+yarnBerryConfigHook() {
+    echo "Executing yarnBerryConfigHook"
+
+    # Use a constant HOME directory
+    export HOME=$(mktemp -d)
+    if [[ -n "$yarnOfflineCache" ]]; then
+        offlineCache="$yarnOfflineCache"
+    fi
+    if [[ -z "$offlineCache" ]]; then
+        echo yarnBerryConfigHook: No yarnOfflineCache or offlineCache were defined\! >&2
+        exit 2
+    fi
+
+    local -r cacheLockfile="$offlineCache/yarn.lock"
+    local -r srcLockfile="$PWD/yarn.lock"
+
+    echo "Validating consistency between $srcLockfile and $cacheLockfile"
+
+    if ! @diff@ "$srcLockfile" "$cacheLockfile"; then
+      # If the diff failed, first double-check that the file exists, so we can
+      # give a friendlier error msg.
+      if ! [ -e "$srcLockfile" ]; then
+        echo
+        echo "ERROR: Missing yarn.lock from src. Expected to find it at: $srcLockfile"
+        echo "Hint: You can copy a vendored yarn.lock file via postPatch."
+        echo
+
+        exit 1
+      fi
+
+      if ! [ -e "$cacheLockfile" ]; then
+        echo
+        echo "ERROR: Missing lockfile from cache. Expected to find it at: $cacheLockfile"
+        echo
+
+        exit 1
+      fi
+
+      echo
+      echo "ERROR: fetchYarnDeps hash is out of date"
+      echo
+      echo "The yarn.lock in src is not the same as the in $offlineCache."
+      echo
+      echo "To fix the issue:"
+      echo '1. Use `lib.fakeHash` as the fetchYarnBerryDeps hash value'
+      echo "2. Build the derivation and wait for it to fail with a hash mismatch"
+      echo "3. Copy the 'got: sha256-' value back into the fetchYarnBerryDeps hash field"
+      echo
+
+      exit 1
+    fi
+
+    if [[ -n "$missingHashes" ]] || [[ -f "$offlineCache/missing-hashes.json" ]]; then
+      echo "Validating consistency of missing-hashes.json"
+      if [[ -z "$missingHashes" ]]; then
+        echo "You must specify missingHashes in your derivation"
+        exit 1
+      fi
+      if ! @diff@ "$missingHashes" "$offlineCache/missing-hashes.json"; then
+        exit 1
+      fi
+    fi
+
+    YARN_IGNORE_PATH=1 @yarn_offline@ config set enableTelemetry false
+    YARN_IGNORE_PATH=1 @yarn_offline@ config set enableGlobalCache false
+
+    # The cache needs to be writable in case yarn needs to re-pack any patch: or git dependencies
+    rm -rf ./.yarn/cache
+    mkdir -p ./.yarn
+    cp -r --reflink=auto $offlineCache/cache ./.yarn/cache
+    chmod u+w -R ./.yarn/cache
+    [ -d $offlineCache/checkouts ] && cp -r --reflink=auto $offlineCache/checkouts ./.yarn/checkouts
+    [ -d $offlineCache/checkouts ] && chmod u+w -R ./.yarn/checkouts
+
+    export npm_config_nodedir="@nodeSrc@"
+    export npm_config_node_gyp="@nodeGyp@"
+
+    YARN_IGNORE_PATH=1 @yarn_offline@ install --inline-builds
+
+    echo "finished yarnBerryConfigHook"
+}
+
+if [[ -z "${dontYarnBerryInstallDeps-}" ]]; then
+    postConfigureHooks+=(yarnBerryConfigHook)
+fi

pkgs/by-name/ya/yarn-berry/fetcher/yarn-berry-fetcher.nix

@@ -0,0 +1,52 @@
+{
+  lib,
+  rustPlatform,
+  fetchFromGitLab,
+  pkg-config,
+  libzip,
+  zlib,
+  zlib-ng,
+  openssl,
+
+  berryVersion,
+  berryCacheVersion,
+}:
+
+rustPlatform.buildRustPackage (finalAttrs: {
+  pname = "yarn-berry-${toString berryVersion}-fetcher";
+  version = "1.0.0";
+
+  src = fetchFromGitLab {
+    domain = "cyberchaos.dev";
+    owner = "yuka";
+    repo = "yarn-berry-fetcher";
+    tag = "1.0.0";
+    hash = "sha256-iMU/SadzrNv8pZSgp2fBwWVgrgZsnyPRsvs0ugvwyks=";
+  };
+
+  useFetchCargoVendor = true;
+  cargoHash = "sha256-ETFaCu+6Ar7tEeRCbTbesEqx9BdztSvPXB7Dc5KGIx0=";
+
+  YARN_ZIP_SUPPORTED_CACHE_VERSION = berryCacheVersion;
+
+  LIBZIP_SYS_USE_PKG_CONFIG = 1;
+
+  nativeBuildInputs = [
+    rustPlatform.bindgenHook
+    pkg-config
+  ];
+  buildInputs = [
+    libzip
+    openssl
+  ];
+
+  meta = with lib; {
+    homepage = "https://cyberchaos.dev/yuka/yarn-berry-fetcher";
+    license = licenses.mit;
+    mainProgram = "yarn-berry-fetcher";
+    maintainers = [
+      maintainers.yuka
+      maintainers.flokli
+    ];
+  };
+})

pkgs/by-name/ya/yarn-berry/package.nix

@@ -5,6 +5,7 @@
   stdenv,
   testers,
   yarn,
+  callPackage,
   berryVersion ? 4,
 }:
 
@@ -48,13 +49,15 @@ stdenv.mkDerivation (finalAttrs: {
     runHook postInstall
   '';
 
-  passthru.updateScript = ./update.sh;
+  passthru = {
+    updateScript = ./update.sh;
 
-  passthru.tests = {
+    tests = {
       version = testers.testVersion {
         package = finalAttrs.finalPackage;
       };
     };
+  } // (callPackage ./fetcher { yarn-berry = finalAttrs; });
 
   meta = with lib; {
     homepage = "https://yarnpkg.com/";