From baed2fcac22b00830a0c97909d5c69513d23634b Mon Sep 17 00:00:00 2001 From: Yureka Date: Sun, 27 Apr 2025 12:44:41 +0200 Subject: [PATCH 1/3] yarn-berry.yarn-berry-offline: improved support for yarn v1 git dependencies When berry detects a Yarn v1 git dependency, it will attempt to run yarn install in it. We already patched that out, but it still requires an internet connection for the `yarn set version classic` command, so patch that out too and instead point to our packaged version of yarn v1. --- .../yarn-berry/fetcher/berry-3-offline.patch | 65 ++++++++++++++----- .../yarn-berry/fetcher/berry-4-offline.patch | 65 ++++++++++++++----- .../by-name/ya/yarn-berry/fetcher/default.nix | 14 +++- 3 files changed, 110 insertions(+), 34 deletions(-) diff --git a/pkgs/by-name/ya/yarn-berry/fetcher/berry-3-offline.patch b/pkgs/by-name/ya/yarn-berry/fetcher/berry-3-offline.patch index 0470e213e663..52e99f2fb408 100644 --- a/pkgs/by-name/ya/yarn-berry/fetcher/berry-3-offline.patch +++ b/pkgs/by-name/ya/yarn-berry/fetcher/berry-3-offline.patch @@ -1,5 +1,5 @@ diff --git a/packages/plugin-essentials/sources/commands/install.ts b/packages/plugin-essentials/sources/commands/install.ts -index 9dcd02d..cf1765a 100644 +index 9dcd02d12..cf1765a20 100644 --- a/packages/plugin-essentials/sources/commands/install.ts +++ b/packages/plugin-essentials/sources/commands/install.ts @@ -254,6 +254,7 @@ export default class YarnCommand extends BaseCommand { @@ -11,7 +11,7 @@ index 9dcd02d..cf1765a 100644 configuration, json: this.json, diff --git a/packages/plugin-git/sources/GitFetcher.ts b/packages/plugin-git/sources/GitFetcher.ts -index fe2a4fc..bfa8272 100644 +index fe2a4fce8..bfa82728e 100644 --- a/packages/plugin-git/sources/GitFetcher.ts +++ b/packages/plugin-git/sources/GitFetcher.ts @@ -50,9 +50,14 @@ export class GitFetcher implements Fetcher { @@ -32,7 +32,7 @@ index fe2a4fc..bfa8272 100644 await scriptUtils.prepareExternalProject(cloneTarget, packagePath, { diff --git a/packages/plugin-npm/sources/NpmSemverFetcher.ts b/packages/plugin-npm/sources/NpmSemverFetcher.ts -index 0f69423..5b21462 100644 +index 0f69423c7..5b21462a5 100644 --- a/packages/plugin-npm/sources/NpmSemverFetcher.ts +++ b/packages/plugin-npm/sources/NpmSemverFetcher.ts @@ -47,6 +47,7 @@ export class NpmSemverFetcher implements Fetcher { @@ -44,7 +44,7 @@ index 0f69423..5b21462 100644 try { sourceBuffer = await npmHttpUtils.get(NpmSemverFetcher.getLocatorUrl(locator), { diff --git a/packages/yarnpkg-core/sources/Cache.ts b/packages/yarnpkg-core/sources/Cache.ts -index d5e6864..374b5d6 100644 +index d5e686420..374b5d67f 100644 --- a/packages/yarnpkg-core/sources/Cache.ts +++ b/packages/yarnpkg-core/sources/Cache.ts @@ -158,6 +158,10 @@ export class Cache { @@ -59,27 +59,60 @@ index d5e6864..374b5d6 100644 const mirrorPath = this.getLocatorMirrorPath(locator); diff --git a/packages/yarnpkg-core/sources/scriptUtils.ts b/packages/yarnpkg-core/sources/scriptUtils.ts -index b3c2c59..6b9eb2f 100644 +index b3c2c5903..641687745 100644 --- a/packages/yarnpkg-core/sources/scriptUtils.ts +++ b/packages/yarnpkg-core/sources/scriptUtils.ts -@@ -287,10 +287,6 @@ export async function prepareExternalProject(cwd: PortablePath, outputPath: Port - // Run an install; we can't avoid it unless we inspect the - // package.json, which I don't want to do to keep the codebase - // clean (even if it has a slight perf cost when cloning v1 repos) +@@ -262,20 +262,6 @@ export async function prepareExternalProject(cwd: PortablePath, outputPath: Port + ? [`workspace`, workspace] + : []; + +- // `set version` will update the Manifest to contain a `packageManager` field with the latest +- // Yarn version which causes the results to change depending on when this command was run, +- // therefore we revert any change made to it. +- const manifestPath = ppath.join(cwd, Filename.manifest); +- const manifestBuffer = await xfs.readFilePromise(manifestPath); +- +- // Makes sure that we'll be using Yarn 1.x +- const version = await execUtils.pipevp(process.execPath, [process.argv[1], `set`, `version`, `classic`, `--only-if-needed`], {cwd, env, stdin, stdout, stderr, end: execUtils.EndStrategy.ErrorCode}); +- if (version.code !== 0) +- return version.code; +- +- // Revert any changes made to the Manifest by `set version`. +- await xfs.writeFilePromise(manifestPath, manifestBuffer); +- + // Otherwise Yarn 1 will pack the .yarn directory :( + await xfs.appendFilePromise(ppath.join(cwd, `.npmignore` as PortablePath), `/.yarn\n`); + +@@ -284,16 +270,8 @@ export async function prepareExternalProject(cwd: PortablePath, outputPath: Port + // Remove environment variables that limit the install to just production dependencies + delete env.NODE_ENV; + +- // Run an install; we can't avoid it unless we inspect the +- // package.json, which I don't want to do to keep the codebase +- // clean (even if it has a slight perf cost when cloning v1 repos) - const install = await execUtils.pipevp(`yarn`, [`install`], {cwd, env, stdin, stdout, stderr, end: execUtils.EndStrategy.ErrorCode}); - if (install.code !== 0) - return install.code; - - stdout.write(`\n`); +- stdout.write(`\n`); +- +- const pack = await execUtils.pipevp(`yarn`, [...workspaceCli, `pack`, `--filename`, npath.fromPortablePath(outputPath)], {cwd, env, stdin, stdout, stderr}); ++ env["SKIP_YARN_COREPACK_CHECK"] = "1"; ++ const pack = await execUtils.pipevp(`@yarnv1@`, [...workspaceCli, `--offline`, `pack`, `--filename`, npath.fromPortablePath(outputPath)], {cwd, env, stdin, stdout, stderr}); + if (pack.code !== 0) + return pack.code; - const pack = await execUtils.pipevp(`yarn`, [...workspaceCli, `pack`, `--filename`, npath.fromPortablePath(outputPath)], {cwd, env, stdin, stdout, stderr}); -@@ -375,9 +371,6 @@ export async function prepareExternalProject(cwd: PortablePath, outputPath: Port - // We can't use `npm ci` because some projects don't have npm - // lockfiles that are up-to-date. Hopefully npm won't decide - // to change the versions randomly. +@@ -372,13 +350,6 @@ export async function prepareExternalProject(cwd: PortablePath, outputPath: Port + delete env.NPM_CONFIG_PRODUCTION; + delete env.NODE_ENV; + +- // We can't use `npm ci` because some projects don't have npm +- // lockfiles that are up-to-date. Hopefully npm won't decide +- // to change the versions randomly. - const install = await execUtils.pipevp(`npm`, [`install`], {cwd, env, stdin, stdout, stderr, end: execUtils.EndStrategy.ErrorCode}); - if (install.code !== 0) - return install.code; - +- const packStream = new PassThrough(); const packPromise = miscUtils.bufferStream(packStream); + diff --git a/pkgs/by-name/ya/yarn-berry/fetcher/berry-4-offline.patch b/pkgs/by-name/ya/yarn-berry/fetcher/berry-4-offline.patch index c4f87a57ab19..7ebf48b64233 100644 --- a/pkgs/by-name/ya/yarn-berry/fetcher/berry-4-offline.patch +++ b/pkgs/by-name/ya/yarn-berry/fetcher/berry-4-offline.patch @@ -1,5 +1,5 @@ diff --git a/packages/plugin-essentials/sources/commands/install.ts b/packages/plugin-essentials/sources/commands/install.ts -index 90ba553..ef5368c 100644 +index 90ba55349..ef5368c1b 100644 --- a/packages/plugin-essentials/sources/commands/install.ts +++ b/packages/plugin-essentials/sources/commands/install.ts @@ -302,6 +302,7 @@ export default class YarnCommand extends BaseCommand { @@ -11,7 +11,7 @@ index 90ba553..ef5368c 100644 newSettings[rule.name] = rule.value; } diff --git a/packages/plugin-git/sources/GitFetcher.ts b/packages/plugin-git/sources/GitFetcher.ts -index d9f8d85..4db9f90 100644 +index d9f8d85c7..4db9f9008 100644 --- a/packages/plugin-git/sources/GitFetcher.ts +++ b/packages/plugin-git/sources/GitFetcher.ts @@ -50,7 +50,11 @@ export class GitFetcher implements Fetcher { @@ -28,7 +28,7 @@ index d9f8d85..4db9f90 100644 const packagePath = ppath.join(projectPath, `package.tgz`); diff --git a/packages/plugin-npm/sources/NpmSemverFetcher.ts b/packages/plugin-npm/sources/NpmSemverFetcher.ts -index 7347859..ea5767f 100644 +index 7347859aa..ea5767f88 100644 --- a/packages/plugin-npm/sources/NpmSemverFetcher.ts +++ b/packages/plugin-npm/sources/NpmSemverFetcher.ts @@ -45,6 +45,7 @@ export class NpmSemverFetcher implements Fetcher { @@ -40,7 +40,7 @@ index 7347859..ea5767f 100644 try { sourceBuffer = await npmHttpUtils.get(NpmSemverFetcher.getLocatorUrl(locator), { diff --git a/packages/yarnpkg-core/sources/Cache.ts b/packages/yarnpkg-core/sources/Cache.ts -index b712ecf..c7effbc 100644 +index b712ecf11..c7effbc61 100644 --- a/packages/yarnpkg-core/sources/Cache.ts +++ b/packages/yarnpkg-core/sources/Cache.ts @@ -225,6 +225,10 @@ export class Cache { @@ -55,27 +55,60 @@ index b712ecf..c7effbc 100644 const mirrorPath = this.getLocatorMirrorPath(locator); diff --git a/packages/yarnpkg-core/sources/scriptUtils.ts b/packages/yarnpkg-core/sources/scriptUtils.ts -index 2dcd7e5..fe40fbb 100644 +index 2dcd7e59e..a2a7f9791 100644 --- a/packages/yarnpkg-core/sources/scriptUtils.ts +++ b/packages/yarnpkg-core/sources/scriptUtils.ts -@@ -287,10 +287,6 @@ export async function prepareExternalProject(cwd: PortablePath, outputPath: Port - // Run an install; we can't avoid it unless we inspect the - // package.json, which I don't want to do to keep the codebase - // clean (even if it has a slight perf cost when cloning v1 repos) +@@ -262,20 +262,6 @@ export async function prepareExternalProject(cwd: PortablePath, outputPath: Port + ? [`workspace`, workspace] + : []; + +- // `set version` will update the Manifest to contain a `packageManager` field with the latest +- // Yarn version which causes the results to change depending on when this command was run, +- // therefore we revert any change made to it. +- const manifestPath = ppath.join(cwd, Filename.manifest); +- const manifestBuffer = await xfs.readFilePromise(manifestPath); +- +- // Makes sure that we'll be using Yarn 1.x +- const version = await execUtils.pipevp(process.execPath, [process.argv[1], `set`, `version`, `classic`, `--only-if-needed`, `--yarn-path`], {cwd, env, stdin, stdout, stderr, end: execUtils.EndStrategy.ErrorCode}); +- if (version.code !== 0) +- return version.code; +- +- // Revert any changes made to the Manifest by `set version`. +- await xfs.writeFilePromise(manifestPath, manifestBuffer); +- + // Otherwise Yarn 1 will pack the .yarn directory :( + await xfs.appendFilePromise(ppath.join(cwd, `.npmignore`), `/.yarn\n`); + +@@ -284,16 +270,8 @@ export async function prepareExternalProject(cwd: PortablePath, outputPath: Port + // Remove environment variables that limit the install to just production dependencies + delete env.NODE_ENV; + +- // Run an install; we can't avoid it unless we inspect the +- // package.json, which I don't want to do to keep the codebase +- // clean (even if it has a slight perf cost when cloning v1 repos) - const install = await execUtils.pipevp(`yarn`, [`install`], {cwd, env, stdin, stdout, stderr, end: execUtils.EndStrategy.ErrorCode}); - if (install.code !== 0) - return install.code; - - stdout.write(`\n`); +- stdout.write(`\n`); +- +- const pack = await execUtils.pipevp(`yarn`, [...workspaceCli, `pack`, `--filename`, npath.fromPortablePath(outputPath)], {cwd, env, stdin, stdout, stderr}); ++ env["SKIP_YARN_COREPACK_CHECK"] = "1"; ++ const pack = await execUtils.pipevp(`@yarnv1@`, [...workspaceCli, `--offline`, `pack`, `--filename`, npath.fromPortablePath(outputPath)], {cwd, env, stdin, stdout, stderr}); + if (pack.code !== 0) + return pack.code; - const pack = await execUtils.pipevp(`yarn`, [...workspaceCli, `pack`, `--filename`, npath.fromPortablePath(outputPath)], {cwd, env, stdin, stdout, stderr}); -@@ -375,9 +371,6 @@ export async function prepareExternalProject(cwd: PortablePath, outputPath: Port - // We can't use `npm ci` because some projects don't have npm - // lockfiles that are up-to-date. Hopefully npm won't decide - // to change the versions randomly. +@@ -372,13 +350,6 @@ export async function prepareExternalProject(cwd: PortablePath, outputPath: Port + delete env.NPM_CONFIG_PRODUCTION; + delete env.NODE_ENV; + +- // We can't use `npm ci` because some projects don't have npm +- // lockfiles that are up-to-date. Hopefully npm won't decide +- // to change the versions randomly. - const install = await execUtils.pipevp(`npm`, [`install`, `--legacy-peer-deps`], {cwd, env, stdin, stdout, stderr, end: execUtils.EndStrategy.ErrorCode}); - if (install.code !== 0) - return install.code; - +- const packStream = new PassThrough(); const packPromise = miscUtils.bufferStream(packStream); + diff --git a/pkgs/by-name/ya/yarn-berry/fetcher/default.nix b/pkgs/by-name/ya/yarn-berry/fetcher/default.nix index f5389311896d..00925e28b195 100644 --- a/pkgs/by-name/ya/yarn-berry/fetcher/default.nix +++ b/pkgs/by-name/ya/yarn-berry/fetcher/default.nix @@ -2,6 +2,8 @@ lib, newScope, yarn-berry, + yarn, + replaceVars, libzip, zlib, zlib-ng, @@ -12,7 +14,11 @@ let "3" = final: { berryCacheVersion = "8"; - berryOfflinePatches = [ ./berry-3-offline.patch ]; + berryOfflinePatches = [ + (replaceVars ./berry-3-offline.patch { + yarnv1 = lib.getExe yarn; + }) + ]; # Known good version: 1.11.3 libzip = @@ -29,7 +35,11 @@ let "4" = final: { berryCacheVersion = "10"; - berryOfflinePatches = [ ./berry-4-offline.patch ]; + berryOfflinePatches = [ + (replaceVars ./berry-4-offline.patch { + yarnv1 = lib.getExe yarn; + }) + ]; # Known good version: 1.11.3 libzip = From 2232ace41941d77e6cc88169a3f8ab832b7af102 Mon Sep 17 00:00:00 2001 From: Yureka Date: Sun, 27 Apr 2025 12:45:32 +0200 Subject: [PATCH 2/3] yarn-berry.yarnBerryConfigHook: Add --inline-builds for git prepack too --- pkgs/by-name/ya/yarn-berry/fetcher/yarn-berry-config-hook.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/by-name/ya/yarn-berry/fetcher/yarn-berry-config-hook.sh b/pkgs/by-name/ya/yarn-berry/fetcher/yarn-berry-config-hook.sh index 0fda1c6e1c1a..6c6cde8d0bd0 100644 --- a/pkgs/by-name/ya/yarn-berry/fetcher/yarn-berry-config-hook.sh +++ b/pkgs/by-name/ya/yarn-berry/fetcher/yarn-berry-config-hook.sh @@ -75,7 +75,7 @@ yarnBerryConfigHook() { export npm_config_nodedir="@nodeSrc@" export npm_config_node_gyp="@nodeGyp@" - YARN_IGNORE_PATH=1 @yarn_offline@ install --mode=skip-build + YARN_IGNORE_PATH=1 @yarn_offline@ install --mode=skip-build --inline-builds if [[ -z "${dontYarnBerryPatchShebangs-}" ]]; then echo "Running patchShebangs in between the Link and the Build step..." patchShebangs node_modules From 781b892e314211fcfaef9328d4310453a90e8f38 Mon Sep 17 00:00:00 2001 From: Yureka Date: Sun, 27 Apr 2025 12:49:34 +0200 Subject: [PATCH 3/3] yarn-berry.yarn-berry-fetcher: 1.2.1 -> 1.2.2 Diff: https://cyberchaos.dev/yuka/yarn-berry-fetcher/-/compare/1.2.1...1.2.2 - Implement handling of XGlobalHeader tar entries found in npm packages busboy & streamsearch --- pkgs/by-name/ya/yarn-berry/fetcher/yarn-berry-fetcher.nix | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkgs/by-name/ya/yarn-berry/fetcher/yarn-berry-fetcher.nix b/pkgs/by-name/ya/yarn-berry/fetcher/yarn-berry-fetcher.nix index df15e691d95f..a48ae9db9b27 100644 --- a/pkgs/by-name/ya/yarn-berry/fetcher/yarn-berry-fetcher.nix +++ b/pkgs/by-name/ya/yarn-berry/fetcher/yarn-berry-fetcher.nix @@ -13,18 +13,18 @@ rustPlatform.buildRustPackage (finalAttrs: { pname = "yarn-berry-${toString berryVersion}-fetcher"; - version = "1.2.1"; + version = "1.2.2"; src = fetchFromGitLab { domain = "cyberchaos.dev"; owner = "yuka"; repo = "yarn-berry-fetcher"; - tag = "1.2.1"; - hash = "sha256-gBre1LGyDVhikigWoWWW5qZyDCHp4lDONPqg1CRtaFM="; + tag = finalAttrs.version; + hash = "sha256-4lPnjLvS9MBtFyIqFKY91w/6MAesziGMJLrE60djEsI="; }; useFetchCargoVendor = true; - cargoHash = "sha256-KNVfwv+qaJEu3TqhCKpiTfuRvFIFcHstcpjre/QXDso="; + cargoHash = "sha256-i0AJ3z+7oSqBW45Vs6dojcipQPoaIa6bejhqCfkdZNA="; env.YARN_ZIP_SUPPORTED_CACHE_VERSION = berryCacheVersion; env.LIBZIP_SYS_USE_PKG_CONFIG = 1;