diff --git a/pkgs/by-name/ya/yarn-berry/fetcher/berry-3-offline.patch b/pkgs/by-name/ya/yarn-berry/fetcher/berry-3-offline.patch new file mode 100644 index 000000000000..749022a5f530 --- /dev/null +++ b/pkgs/by-name/ya/yarn-berry/fetcher/berry-3-offline.patch @@ -0,0 +1,90 @@ +diff --git a/packages/plugin-essentials/sources/commands/install.ts b/packages/plugin-essentials/sources/commands/install.ts +index 9dcd02d12..cf1765a20 100644 +--- a/packages/plugin-essentials/sources/commands/install.ts ++++ b/packages/plugin-essentials/sources/commands/install.ts +@@ -254,6 +254,7 @@ export default class YarnCommand extends BaseCommand { + // If migrating from a v1 install, we automatically enable the node-modules linker, + // since that's likely what the author intended to do. + if (content?.includes(`yarn lockfile v1`)) { ++ throw new Error("Tried to use yarn-berry_3.yarnConfigHook (nixpkgs), but found a yarn v1 lockfile"); + const nmReport = await StreamReport.start({ + configuration, + json: this.json, +diff --git a/packages/plugin-git/sources/GitFetcher.ts b/packages/plugin-git/sources/GitFetcher.ts +index fe2a4fce8..bfa82728e 100644 +--- a/packages/plugin-git/sources/GitFetcher.ts ++++ b/packages/plugin-git/sources/GitFetcher.ts +@@ -50,9 +50,14 @@ export class GitFetcher implements Fetcher { + } + + async cloneFromRemote(locator: Locator, opts: FetchOptions) { +- const cloneTarget = await gitUtils.clone(locator.reference, opts.project.configuration); +- + const repoUrlParts = gitUtils.splitRepoUrl(locator.reference); ++ ++ if (repoUrlParts.treeish.protocol !== "commit") { ++ throw new Error(`Missing source for git dependency ${locator.reference}`); ++ }; ++ ++ const cloneTarget = opts.cache.checkoutPath(repoUrlParts.treeish.request); ++ + const packagePath = ppath.join(cloneTarget, `package.tgz` as PortablePath); + + await scriptUtils.prepareExternalProject(cloneTarget, packagePath, { +diff --git a/packages/plugin-npm/sources/NpmSemverFetcher.ts b/packages/plugin-npm/sources/NpmSemverFetcher.ts +index 0f69423c7..5b21462a5 100644 +--- a/packages/plugin-npm/sources/NpmSemverFetcher.ts ++++ b/packages/plugin-npm/sources/NpmSemverFetcher.ts +@@ -47,6 +47,7 @@ export class NpmSemverFetcher implements Fetcher { + } + + private async fetchFromNetwork(locator: Locator, opts: FetchOptions) { ++ throw new Error(`Missing sources for ${structUtils.prettyLocator(opts.project.configuration, locator)}`); + let sourceBuffer; + try { + sourceBuffer = await npmHttpUtils.get(NpmSemverFetcher.getLocatorUrl(locator), { +diff --git a/packages/yarnpkg-core/sources/Cache.ts b/packages/yarnpkg-core/sources/Cache.ts +index d5e686420..374b5d67f 100644 +--- a/packages/yarnpkg-core/sources/Cache.ts ++++ b/packages/yarnpkg-core/sources/Cache.ts +@@ -158,6 +158,10 @@ export class Cache { + } + } + ++ checkoutPath(commit: string): string { ++ return ppath.join(ppath.join(this.cwd, "../checkouts"), commit); ++ } ++ + async fetchPackageFromCache(locator: Locator, expectedChecksum: string | null, {onHit, onMiss, loader, ...opts}: {onHit?: () => void, onMiss?: () => void, loader?: () => Promise } & CacheOptions): Promise<[FakeFS, () => void, string | null]> { + const mirrorPath = this.getLocatorMirrorPath(locator); + +diff --git a/packages/yarnpkg-core/sources/scriptUtils.ts b/packages/yarnpkg-core/sources/scriptUtils.ts +index b3c2c5903..15cae13fd 100644 +--- a/packages/yarnpkg-core/sources/scriptUtils.ts ++++ b/packages/yarnpkg-core/sources/scriptUtils.ts +@@ -287,9 +287,9 @@ export async function prepareExternalProject(cwd: PortablePath, outputPath: Port + // Run an install; we can't avoid it unless we inspect the + // package.json, which I don't want to do to keep the codebase + // clean (even if it has a slight perf cost when cloning v1 repos) +- const install = await execUtils.pipevp(`yarn`, [`install`], {cwd, env, stdin, stdout, stderr, end: execUtils.EndStrategy.ErrorCode}); +- if (install.code !== 0) +- return install.code; ++ //const install = await execUtils.pipevp(`yarn`, [`install`], {cwd, env, stdin, stdout, stderr, end: execUtils.EndStrategy.ErrorCode}); ++ //if (install.code !== 0) ++ // return install.code; + + stdout.write(`\n`); + +@@ -375,9 +375,9 @@ export async function prepareExternalProject(cwd: PortablePath, outputPath: Port + // We can't use `npm ci` because some projects don't have npm + // lockfiles that are up-to-date. Hopefully npm won't decide + // to change the versions randomly. +- const install = await execUtils.pipevp(`npm`, [`install`], {cwd, env, stdin, stdout, stderr, end: execUtils.EndStrategy.ErrorCode}); +- if (install.code !== 0) +- return install.code; ++ //const install = await execUtils.pipevp(`npm`, [`install`], {cwd, env, stdin, stdout, stderr, end: execUtils.EndStrategy.ErrorCode}); ++ //if (install.code !== 0) ++ // return install.code; + + const packStream = new PassThrough(); + const packPromise = miscUtils.bufferStream(packStream); diff --git a/pkgs/by-name/ya/yarn-berry/fetcher/berry-4-offline.patch b/pkgs/by-name/ya/yarn-berry/fetcher/berry-4-offline.patch new file mode 100644 index 000000000000..355cf2b058c2 --- /dev/null +++ b/pkgs/by-name/ya/yarn-berry/fetcher/berry-4-offline.patch @@ -0,0 +1,86 @@ +diff --git a/packages/plugin-essentials/sources/commands/install.ts b/packages/plugin-essentials/sources/commands/install.ts +index 90ba55349..ef5368c1b 100644 +--- a/packages/plugin-essentials/sources/commands/install.ts ++++ b/packages/plugin-essentials/sources/commands/install.ts +@@ -302,6 +302,7 @@ export default class YarnCommand extends BaseCommand { + + for (const rule of LOCKFILE_MIGRATION_RULES) { + if (rule.selector(lockfileLastVersion) && typeof configuration.sources.get(rule.name) === `undefined`) { ++ throw new Error(`Tried to use yarn-berry_4.yarnConfigHook (nixpkgs) which expects lockfile version 8, but found lockfile version ${lockfileLastVersion}`); + configuration.use(``, {[rule.name]: rule.value}, project.cwd, {overwrite: true}); + newSettings[rule.name] = rule.value; + } +diff --git a/packages/plugin-git/sources/GitFetcher.ts b/packages/plugin-git/sources/GitFetcher.ts +index d9f8d85c7..4db9f9008 100644 +--- a/packages/plugin-git/sources/GitFetcher.ts ++++ b/packages/plugin-git/sources/GitFetcher.ts +@@ -50,7 +50,11 @@ export class GitFetcher implements Fetcher { + async cloneFromRemote(locator: Locator, opts: FetchOptions) { + const repoUrlParts = gitUtils.splitRepoUrl(locator.reference); + +- const cloneTarget = await gitUtils.clone(locator.reference, opts.project.configuration); ++ if (repoUrlParts.treeish.protocol !== "commit") { ++ throw new Error(`Missing source for git dependency ${locator.reference}`); ++ }; ++ ++ const cloneTarget = opts.cache.checkoutPath(repoUrlParts.treeish.request); + const projectPath = ppath.resolve(cloneTarget, repoUrlParts.extra.cwd ?? PortablePath.dot); + + const packagePath = ppath.join(projectPath, `package.tgz`); +diff --git a/packages/plugin-npm/sources/NpmSemverFetcher.ts b/packages/plugin-npm/sources/NpmSemverFetcher.ts +index 7347859aa..ea5767f88 100644 +--- a/packages/plugin-npm/sources/NpmSemverFetcher.ts ++++ b/packages/plugin-npm/sources/NpmSemverFetcher.ts +@@ -45,6 +45,7 @@ export class NpmSemverFetcher implements Fetcher { + } + + private async fetchFromNetwork(locator: Locator, opts: FetchOptions) { ++ throw new Error(`Missing sources for ${structUtils.prettyLocator(opts.project.configuration, locator)}`); + let sourceBuffer; + try { + sourceBuffer = await npmHttpUtils.get(NpmSemverFetcher.getLocatorUrl(locator), { +diff --git a/packages/yarnpkg-core/sources/Cache.ts b/packages/yarnpkg-core/sources/Cache.ts +index b712ecf11..c7effbc61 100644 +--- a/packages/yarnpkg-core/sources/Cache.ts ++++ b/packages/yarnpkg-core/sources/Cache.ts +@@ -225,6 +225,10 @@ export class Cache { + } + } + ++ checkoutPath(commit: string): string { ++ return ppath.join(ppath.join(this.cwd, "../checkouts"), commit); ++ } ++ + async fetchPackageFromCache(locator: Locator, expectedChecksum: string | null, {onHit, onMiss, loader, ...opts}: {onHit?: () => void, onMiss?: () => void, loader?: () => Promise} & CacheOptions): Promise<[FakeFS, () => void, string | null]> { + const mirrorPath = this.getLocatorMirrorPath(locator); + +diff --git a/packages/yarnpkg-core/sources/scriptUtils.ts b/packages/yarnpkg-core/sources/scriptUtils.ts +index 2dcd7e59e..3079ad635 100644 +--- a/packages/yarnpkg-core/sources/scriptUtils.ts ++++ b/packages/yarnpkg-core/sources/scriptUtils.ts +@@ -287,9 +287,9 @@ export async function prepareExternalProject(cwd: PortablePath, outputPath: Port + // Run an install; we can't avoid it unless we inspect the + // package.json, which I don't want to do to keep the codebase + // clean (even if it has a slight perf cost when cloning v1 repos) +- const install = await execUtils.pipevp(`yarn`, [`install`], {cwd, env, stdin, stdout, stderr, end: execUtils.EndStrategy.ErrorCode}); +- if (install.code !== 0) +- return install.code; ++ //const install = await execUtils.pipevp(`yarn`, [`install`], {cwd, env, stdin, stdout, stderr, end: execUtils.EndStrategy.ErrorCode}); ++ //if (install.code !== 0) ++ // return install.code; + + stdout.write(`\n`); + +@@ -375,9 +375,9 @@ export async function prepareExternalProject(cwd: PortablePath, outputPath: Port + // We can't use `npm ci` because some projects don't have npm + // lockfiles that are up-to-date. Hopefully npm won't decide + // to change the versions randomly. +- const install = await execUtils.pipevp(`npm`, [`install`, `--legacy-peer-deps`], {cwd, env, stdin, stdout, stderr, end: execUtils.EndStrategy.ErrorCode}); +- if (install.code !== 0) +- return install.code; ++ //const install = await execUtils.pipevp(`npm`, [`install`, , `--legacy-peer-deps`], {cwd, env, stdin, stdout, stderr, end: execUtils.EndStrategy.ErrorCode}); ++ //if (install.code !== 0) ++ // return install.code; + + const packStream = new PassThrough(); + const packPromise = miscUtils.bufferStream(packStream); diff --git a/pkgs/by-name/ya/yarn-berry/fetcher/default.nix b/pkgs/by-name/ya/yarn-berry/fetcher/default.nix new file mode 100644 index 000000000000..f5389311896d --- /dev/null +++ b/pkgs/by-name/ya/yarn-berry/fetcher/default.nix @@ -0,0 +1,77 @@ +{ + lib, + newScope, + yarn-berry, + libzip, + zlib, + zlib-ng, +}: + +let + variantOverlays = { + "3" = final: { + berryCacheVersion = "8"; + + berryOfflinePatches = [ ./berry-3-offline.patch ]; + + # Known good version: 1.11.3 + libzip = + (libzip.override { + # Known good version: 1.3.1 + zlib = zlib; + }).overrideAttrs + (oA: { + patches = (oA.patches or [ ]) ++ [ + (final.yarn-berry-fetcher.src + "/libzip-revert-to-old-versionneeded-behavior.patch") + ]; + }); + }; + "4" = final: { + berryCacheVersion = "10"; + + berryOfflinePatches = [ ./berry-4-offline.patch ]; + + # Known good version: 1.11.3 + libzip = + (libzip.override { + # Known good version: 2.2.4 + zlib = zlib-ng.override { + withZlibCompat = true; + }; + }).overrideAttrs + (oA: { + patches = (oA.patches or [ ]) ++ [ + (final.yarn-berry-fetcher.src + "/libzip-revert-to-old-versionneeded-behavior.patch") + ]; + }); + }; + }; +in + +lib.makeScope newScope ( + final: + let + berryVersion = lib.versions.major yarn-berry.version; + + err = throw '' + Berry version ${toString berryVersion} not supported by yarn-berry-fetcher. + Supported versions: ${lib.concatStringsSep ", " (lib.attrNames variantOverlays)} + ''; + variantOverlay = (variantOverlays.${berryVersion} or err) final; + in + ( + { + inherit yarn-berry berryVersion; + + yarn-berry-offline = final.yarn-berry.overrideAttrs (old: { + pname = old.pname + "-offline"; + patches = (old.patches or [ ]) ++ final.berryOfflinePatches; + }); + + yarn-berry-fetcher = final.callPackage ./yarn-berry-fetcher.nix { }; + fetchYarnBerryDeps = final.callPackage ./fetch-yarn-berry-deps.nix { }; + yarnBerryConfigHook = final.callPackage ./yarn-berry-config-hook.nix { }; + } + // variantOverlay + ) +) diff --git a/pkgs/by-name/ya/yarn-berry/fetcher/fetch-yarn-berry-deps.nix b/pkgs/by-name/ya/yarn-berry/fetcher/fetch-yarn-berry-deps.nix new file mode 100644 index 000000000000..3fef9b87644a --- /dev/null +++ b/pkgs/by-name/ya/yarn-berry/fetcher/fetch-yarn-berry-deps.nix @@ -0,0 +1,76 @@ +{ + lib, + stdenv, + yarn-berry-fetcher, + nix-prefetch-git, + cacert, + berryVersion, +}: + +{ + src ? null, + hash ? "", + sha256 ? "", + ... +}@args: + +let + hash_ = + if hash != "" then + { + outputHashAlgo = null; + outputHash = hash; + } + else if sha256 != "" then + { + outputHashAlgo = "sha256"; + outputHash = sha256; + } + else + { + outputHashAlgo = "sha256"; + outputHash = lib.fakeSha256; + }; +in + +stdenv.mkDerivation ( + { + # The name is fixed as to not produce multiple store paths with the same content + name = "offline"; + + dontUnpack = src == null; + dontInstall = true; + + nativeBuildInputs = [ + yarn-berry-fetcher + nix-prefetch-git + cacert + ]; + + buildPhase = '' + runHook preBuild + + yarnLock=''${yarnLock:=$PWD/yarn.lock} + yarn-berry-fetcher fetch $yarnLock $missingHashes + + runHook postBuild + ''; + + outputHashMode = "recursive"; + + passthru = { + inherit berryVersion; + }; + } + // hash_ + // (removeAttrs args ( + [ + "name" + "pname" + "version" + "hash" + "sha256" + ] + ++ (lib.optional (src == null) "src") + )) +) diff --git a/pkgs/by-name/ya/yarn-berry/fetcher/yarn-berry-config-hook.nix b/pkgs/by-name/ya/yarn-berry/fetcher/yarn-berry-config-hook.nix new file mode 100644 index 000000000000..f7999975258c --- /dev/null +++ b/pkgs/by-name/ya/yarn-berry/fetcher/yarn-berry-config-hook.nix @@ -0,0 +1,24 @@ +{ + makeSetupHook, + yarn-berry-offline, + srcOnly, + nodejs, + diffutils, +}: + +makeSetupHook { + name = "yarn-berry-config-hook"; + substitutions = { + # Specify `diff` by abspath to ensure that the user's build + # inputs do not cause us to find the wrong binaries. + diff = "${diffutils}/bin/diff"; + + yarn_offline = "${yarn-berry-offline}/bin/yarn"; + + nodeSrc = srcOnly nodejs; + nodeGyp = "${nodejs}/lib/node_modules/npm/node_modules/node-gyp/bin/node-gyp.js"; + }; + meta = { + description = "Install nodejs dependencies from an offline yarn cache produced by fetchYarnDeps"; + }; +} ./yarn-berry-config-hook.sh diff --git a/pkgs/by-name/ya/yarn-berry/fetcher/yarn-berry-config-hook.sh b/pkgs/by-name/ya/yarn-berry/fetcher/yarn-berry-config-hook.sh new file mode 100644 index 000000000000..65c4e3e689ad --- /dev/null +++ b/pkgs/by-name/ya/yarn-berry/fetcher/yarn-berry-config-hook.sh @@ -0,0 +1,85 @@ +yarnBerryConfigHook() { + echo "Executing yarnBerryConfigHook" + + # Use a constant HOME directory + export HOME=$(mktemp -d) + if [[ -n "$yarnOfflineCache" ]]; then + offlineCache="$yarnOfflineCache" + fi + if [[ -z "$offlineCache" ]]; then + echo yarnBerryConfigHook: No yarnOfflineCache or offlineCache were defined\! >&2 + exit 2 + fi + + local -r cacheLockfile="$offlineCache/yarn.lock" + local -r srcLockfile="$PWD/yarn.lock" + + echo "Validating consistency between $srcLockfile and $cacheLockfile" + + if ! @diff@ "$srcLockfile" "$cacheLockfile"; then + # If the diff failed, first double-check that the file exists, so we can + # give a friendlier error msg. + if ! [ -e "$srcLockfile" ]; then + echo + echo "ERROR: Missing yarn.lock from src. Expected to find it at: $srcLockfile" + echo "Hint: You can copy a vendored yarn.lock file via postPatch." + echo + + exit 1 + fi + + if ! [ -e "$cacheLockfile" ]; then + echo + echo "ERROR: Missing lockfile from cache. Expected to find it at: $cacheLockfile" + echo + + exit 1 + fi + + echo + echo "ERROR: fetchYarnDeps hash is out of date" + echo + echo "The yarn.lock in src is not the same as the in $offlineCache." + echo + echo "To fix the issue:" + echo '1. Use `lib.fakeHash` as the fetchYarnBerryDeps hash value' + echo "2. Build the derivation and wait for it to fail with a hash mismatch" + echo "3. Copy the 'got: sha256-' value back into the fetchYarnBerryDeps hash field" + echo + + exit 1 + fi + + if [[ -n "$missingHashes" ]] || [[ -f "$offlineCache/missing-hashes.json" ]]; then + echo "Validating consistency of missing-hashes.json" + if [[ -z "$missingHashes" ]]; then + echo "You must specify missingHashes in your derivation" + exit 1 + fi + if ! @diff@ "$missingHashes" "$offlineCache/missing-hashes.json"; then + exit 1 + fi + fi + + YARN_IGNORE_PATH=1 @yarn_offline@ config set enableTelemetry false + YARN_IGNORE_PATH=1 @yarn_offline@ config set enableGlobalCache false + + # The cache needs to be writable in case yarn needs to re-pack any patch: or git dependencies + rm -rf ./.yarn/cache + mkdir -p ./.yarn + cp -r --reflink=auto $offlineCache/cache ./.yarn/cache + chmod u+w -R ./.yarn/cache + [ -d $offlineCache/checkouts ] && cp -r --reflink=auto $offlineCache/checkouts ./.yarn/checkouts + [ -d $offlineCache/checkouts ] && chmod u+w -R ./.yarn/checkouts + + export npm_config_nodedir="@nodeSrc@" + export npm_config_node_gyp="@nodeGyp@" + + YARN_IGNORE_PATH=1 @yarn_offline@ install --inline-builds + + echo "finished yarnBerryConfigHook" +} + +if [[ -z "${dontYarnBerryInstallDeps-}" ]]; then + postConfigureHooks+=(yarnBerryConfigHook) +fi diff --git a/pkgs/by-name/ya/yarn-berry/fetcher/yarn-berry-fetcher.nix b/pkgs/by-name/ya/yarn-berry/fetcher/yarn-berry-fetcher.nix new file mode 100644 index 000000000000..5bb5fb1b3ebc --- /dev/null +++ b/pkgs/by-name/ya/yarn-berry/fetcher/yarn-berry-fetcher.nix @@ -0,0 +1,52 @@ +{ + lib, + rustPlatform, + fetchFromGitLab, + pkg-config, + libzip, + zlib, + zlib-ng, + openssl, + + berryVersion, + berryCacheVersion, +}: + +rustPlatform.buildRustPackage (finalAttrs: { + pname = "yarn-berry-${toString berryVersion}-fetcher"; + version = "1.0.0"; + + src = fetchFromGitLab { + domain = "cyberchaos.dev"; + owner = "yuka"; + repo = "yarn-berry-fetcher"; + tag = "1.0.0"; + hash = "sha256-iMU/SadzrNv8pZSgp2fBwWVgrgZsnyPRsvs0ugvwyks="; + }; + + useFetchCargoVendor = true; + cargoHash = "sha256-ETFaCu+6Ar7tEeRCbTbesEqx9BdztSvPXB7Dc5KGIx0="; + + YARN_ZIP_SUPPORTED_CACHE_VERSION = berryCacheVersion; + + LIBZIP_SYS_USE_PKG_CONFIG = 1; + + nativeBuildInputs = [ + rustPlatform.bindgenHook + pkg-config + ]; + buildInputs = [ + libzip + openssl + ]; + + meta = with lib; { + homepage = "https://cyberchaos.dev/yuka/yarn-berry-fetcher"; + license = licenses.mit; + mainProgram = "yarn-berry-fetcher"; + maintainers = [ + maintainers.yuka + maintainers.flokli + ]; + }; +}) diff --git a/pkgs/by-name/ya/yarn-berry/package.nix b/pkgs/by-name/ya/yarn-berry/package.nix index 9e648d2279a1..3baa9bdeb52c 100644 --- a/pkgs/by-name/ya/yarn-berry/package.nix +++ b/pkgs/by-name/ya/yarn-berry/package.nix @@ -5,6 +5,7 @@ stdenv, testers, yarn, + callPackage, berryVersion ? 4, }: @@ -48,13 +49,15 @@ stdenv.mkDerivation (finalAttrs: { runHook postInstall ''; - passthru.updateScript = ./update.sh; + passthru = { + updateScript = ./update.sh; - passthru.tests = { - version = testers.testVersion { - package = finalAttrs.finalPackage; + tests = { + version = testers.testVersion { + package = finalAttrs.finalPackage; + }; }; - }; + } // (callPackage ./fetcher { yarn-berry = finalAttrs; }); meta = with lib; { homepage = "https://yarnpkg.com/";