diff --git a/doc/release-notes/rl-2605.section.md b/doc/release-notes/rl-2605.section.md index 46a4447d65e1..067b9f227d30 100644 --- a/doc/release-notes/rl-2605.section.md +++ b/doc/release-notes/rl-2605.section.md @@ -144,6 +144,8 @@ - `fetchPnpmDeps` and `pnpmConfigHook` were added as top-level attributes, replacing the now deprecated `pnpm.fetchDeps` and `pnpm.configHook` attributes. +- `buildNpmPackage` now supports `npmDepsCacheVersion`. Set to `2` to enable packument caching, which fixes builds for projects using npm workspaces. + - Added `dell-bios-fan-control` package and service. - `openrgb` was updated to 1.0rc2, which now uses Plugin API version 4. diff --git a/pkgs/build-support/node/build-npm-package/default.nix b/pkgs/build-support/node/build-npm-package/default.nix index 747043e3d12a..1e316d1be9c6 100644 --- a/pkgs/build-support/node/build-npm-package/default.nix +++ b/pkgs/build-support/node/build-npm-package/default.nix @@ -26,6 +26,9 @@ lib.extendMkDerivation { # The output hash of the dependencies for this project. # Can be calculated in advance with prefetch-npm-deps. npmDepsHash ? "", + # Fetcher format version for npmDeps. Set to 2 to enable packument caching + # for workspace support. Changing this will invalidate npmDepsHash. + npmDepsFetcherVersion ? 1, # Whether to force the usage of Git dependencies that have install scripts, but not a lockfile. # Use with care. forceGitDeps ? false, @@ -66,6 +69,7 @@ lib.extendMkDerivation { ; name = "${name}-npm-deps"; hash = npmDepsHash; + fetcherVersion = npmDepsFetcherVersion; }, # Custom npmConfigHook npmConfigHook ? null, @@ -85,6 +89,10 @@ lib.extendMkDerivation { { inherit npmDeps npmBuildScript; + env = (args.env or { }) // { + NIX_NPM_FETCHER_VERSION = npmDepsFetcherVersion; + }; + nativeBuildInputs = nativeBuildInputs ++ [ diff --git a/pkgs/build-support/node/build-npm-package/hooks/npm-config-hook.sh b/pkgs/build-support/node/build-npm-package/hooks/npm-config-hook.sh index d954bb010df2..3e7fcbd69133 100644 --- a/pkgs/build-support/node/build-npm-package/hooks/npm-config-hook.sh +++ b/pkgs/build-support/node/build-npm-package/hooks/npm-config-hook.sh @@ -28,6 +28,29 @@ npmConfigHook() { exit 1 fi + if [[ -e "$npmDeps/.fetcher-version" ]]; then + local -r fetcherVersion=$(cat "$npmDeps/.fetcher-version") + else + local -r fetcherVersion="1" + fi + + # Only run this in buildNpmPackage, this is just for a nicer error message; we trust that + # people using the setup hook directly also know how FODs work. ;) + if [[ -n ${NIX_NPM_FETCHER_VERSION+x} ]] && [[ $NIX_NPM_FETCHER_VERSION != $fetcherVersion ]]; then + echo + echo "ERROR: npmDepsHash is out of date" + echo + echo "The fetcher version in the arguments to buildNpmPackage ($NIX_NPM_FETCHER_VERSION) is not the same as the one in $npmDeps ($fetcherVersion)." + echo + echo "To fix the issue:" + echo '1. Use `lib.fakeHash` as the npmDepsHash value' + echo "2. Build the derivation and wait for it to fail with a hash mismatch" + echo "3. Copy the 'got: sha256-' value back into the npmDepsHash field" + echo + + exit 1 + fi + local -r cacheLockfile="$npmDeps/package-lock.json" if [[ -f npm-shrinkwrap.json ]]; then local -r srcLockfile="$PWD/npm-shrinkwrap.json" @@ -78,7 +101,11 @@ npmConfigHook() { local cachePath - if [ -z "${makeCacheWritable-}" ]; then + # When a given cache key has multiple entries (which is the case with + # fetcher version 2), npm always needs to write to the cache. + # + # TODO(winter): report upstream? + if [ -z "${makeCacheWritable-}" ] && (( fetcherVersion == 1 )); then cachePath="$npmDeps" else echo "Making cache writable" @@ -100,9 +127,10 @@ npmConfigHook() { echo "ERROR: npm failed to install dependencies" echo echo "Here are a few things you can try, depending on the error:" - echo '1. Set `makeCacheWritable = true`' + echo '1. Set `npmDepsFetcherVersion = 2` (and update `npmDepsHash`)' + echo '2. Set `makeCacheWritable = true`' echo " Note that this won't help if npm is complaining about not being able to write to the logs directory -- look above that for the actual error." - echo '2. Set `npmFlags = [ "--legacy-peer-deps" ]`' + echo '3. Set `npmFlags = [ "--legacy-peer-deps" ]`' echo exit 1 diff --git a/pkgs/build-support/node/prefetch-npm-deps/default.nix b/pkgs/build-support/node/prefetch-npm-deps/default.nix index d5c67dbca65e..bfa8fb863e4b 100644 --- a/pkgs/build-support/node/prefetch-npm-deps/default.nix +++ b/pkgs/build-support/node/prefetch-npm-deps/default.nix @@ -215,6 +215,10 @@ # A string with a JSON attrset specifying registry mirrors, for example # {"registry.example.org": "my-mirror.local/registry.example.org"} npmRegistryOverridesString ? config.npmRegistryOverridesString, + # Fetcher format version. Bump this to invalidate all existing hashes. + # Version 1: original format (tarballs only) + # Version 2: includes packuments for workspace support + fetcherVersion ? 1, ... }@args: let @@ -272,6 +276,10 @@ NIX_NPM_REGISTRY_OVERRIDES = npmRegistryOverridesString; + # Fetcher version controls which features are enabled in prefetch-npm-deps + # Version 2+ enables packument fetching for workspace support + NPM_FETCHER_VERSION = toString fetcherVersion; + SSL_CERT_FILE = if ( diff --git a/pkgs/build-support/node/prefetch-npm-deps/src/cacache.rs b/pkgs/build-support/node/prefetch-npm-deps/src/cacache.rs index 403c909dee11..648a9ae1a828 100644 --- a/pkgs/build-support/node/prefetch-npm-deps/src/cacache.rs +++ b/pkgs/build-support/node/prefetch-npm-deps/src/cacache.rs @@ -24,9 +24,16 @@ pub(super) struct Key { #[derive(Serialize, Deserialize)] pub(super) struct Metadata { pub(super) url: Url, + #[serde(rename = "reqHeaders", skip_serializing_if = "Option::is_none")] + pub(super) req_headers: Option, pub(super) options: Options, } +#[derive(Serialize, Deserialize)] +pub struct ReqHeaders { + pub accept: String, +} + #[derive(Serialize, Deserialize)] pub(super) struct Options { pub(super) compress: bool, @@ -58,6 +65,7 @@ impl Cache { url: Url, data: &[u8], integrity: Option, + req_headers: Option, ) -> anyhow::Result<()> { let (algo, hash, integrity) = if let Some(integrity) = integrity { let (algo, hash) = integrity @@ -115,13 +123,27 @@ impl Cache { size: data.len(), metadata: Metadata { url, + req_headers, options: Options { compress: true }, }, })?; - let mut file = File::options().append(true).create(true).open(index_path)?; + let mut file = File::options() + .append(true) + .create(true) + .open(&index_path)?; - write!(file, "{:x}\t{data}", Sha1::new().chain(&data).finalize())?; + // cacache format uses newline as entry separator (see cacache entry-index.js) + // Only add newline prefix if file already has content, to maintain backwards compatibility with previous versions of prefetch-npm-deps, which handled this case incorrectly. + let needs_newline = fs::metadata(&index_path) + .map(|m| m.len() > 0) + .unwrap_or(false); + let prefix = if needs_newline { "\n" } else { "" }; + write!( + file, + "{prefix}{:x}\t{data}", + Sha1::new().chain(&data).finalize() + )?; Ok(()) } diff --git a/pkgs/build-support/node/prefetch-npm-deps/src/main.rs b/pkgs/build-support/node/prefetch-npm-deps/src/main.rs index cdac7ef84c0b..53cc075bfdc0 100644 --- a/pkgs/build-support/node/prefetch-npm-deps/src/main.rs +++ b/pkgs/build-support/node/prefetch-npm-deps/src/main.rs @@ -1,11 +1,12 @@ #![warn(clippy::pedantic)] -use crate::cacache::{Cache, Key}; +use crate::cacache::{Cache, Key, ReqHeaders}; use anyhow::{anyhow, bail}; +use log::info; use rayon::prelude::*; use serde_json::{Map, Value}; use std::{ - collections::HashMap, + collections::{HashMap, HashSet}, env, fs, path::{Path, PathBuf}, process, @@ -22,6 +23,160 @@ fn cache_map_path() -> Option { env::var_os("CACHE_MAP_PATH").map(PathBuf::from) } +/// Get the packument URL for a package name +fn get_packument_url(registry: &str, package_name: &str) -> anyhow::Result { + // URL-encode the package name for scoped packages + let encoded_name = package_name.replace('/', "%2f"); + Url::parse(&format!("{registry}/{encoded_name}")) + .map_err(|e| anyhow!("failed to construct packument URL: {e}")) +} + +/// Normalize packument data to ensure determinism. +/// +/// Filters to whitelisted fields and requested versions only. +/// Allowed top-level fields in normalized packuments. +/// +/// For lockfile-based installs, versions are exact (e.g., "4.17.21") so npm-pick-manifest +/// just does a direct `versions[ver]` lookup. Tarballs are fetched via the resolved URL. +const ALLOWED_TOP_LEVEL_FIELDS: &[&str] = &["name", "versions"]; + +/// Allowed fields in version objects. +/// +/// Based on analysis of pacote, npm-pick-manifest, npm-install-checks, and arborist. +/// Only fields actually read during `npm install` are included. +const ALLOWED_VERSION_FIELDS: &[&str] = &[ + "name", + "version", + // Dependencies + "dependencies", + "devDependencies", + "peerDependencies", + "peerDependenciesMeta", + "optionalDependencies", + "bundleDependencies", + "bundledDependencies", + // Distribution (tarball URL and integrity) + "dist", + // Executables + "bin", + // Platform constraints (npm-install-checks) + "engines", + "os", + "cpu", + // Lifecycle scripts + "scripts", + // Version selection hint (npm-pick-manifest) + "deprecated", +]; + +fn normalize_packument( + package_name: &str, + data: &[u8], + requested_versions: &HashSet, +) -> anyhow::Result> { + let mut json: Value = serde_json::from_slice(data) + .map_err(|e| anyhow!("failed to parse packument JSON for {package_name}: {e}"))?; + + let obj = json + .as_object_mut() + .ok_or_else(|| anyhow!("packument for {package_name} is not a JSON object"))?; + + // Keep only whitelisted top-level fields to ensure determinism + obj.retain(|key, _| ALLOWED_TOP_LEVEL_FIELDS.contains(&key.as_str())); + + // Filter and normalize versions + if let Some(Value::Object(versions)) = obj.get_mut("versions") { + // Only keep versions that are in the lockfile + versions.retain(|version, _| requested_versions.contains(version)); + + // Normalize each version object to only include necessary fields + for version_val in versions.values_mut() { + if let Some(version_obj) = version_val.as_object_mut() { + version_obj.retain(|key, _| ALLOWED_VERSION_FIELDS.contains(&key.as_str())); + } + } + } + + serde_json::to_vec(&json) + .map_err(|e| anyhow!("failed to re-serialize packument for {package_name}: {e}")) +} + +/// Fetch and cache packuments (package metadata) for all packages. +/// +/// This is needed because npm may query package metadata for optional peer dependencies +/// and for workspace packages. +/// +/// npm's cache policy checks that the Accept header matches between the cached +/// request and the new request. npm can request packuments with two different headers: +/// 1. "corgiDoc" (abbreviated metadata) - used initially +/// 2. "fullDoc" (full metadata) - used when npm needs full package info (e.g., workspaces) +/// +/// We cache both versions to ensure cache hits regardless of which header npm uses. +/// See: pacote/lib/registry.js and @npmcli/arborist/lib/arborist/build-ideal-tree.js +fn fetch_packuments( + cache: &Cache, + package_versions: HashMap>, +) -> anyhow::Result<()> { + const CORGI_DOC: &str = + "application/vnd.npm.install-v1+json; q=1.0, application/json; q=0.8, */*"; + const FULL_DOC: &str = "application/json"; + + info!("Fetching {} packuments", package_versions.len()); + + package_versions + .into_par_iter() + .try_for_each(|(package_name, requested_versions)| { + let packument_url = get_packument_url("https://registry.npmjs.org", &package_name)?; + + match util::get_url_body_with_retry(&packument_url) { + Ok(packument_data) => { + let normalized_data = + normalize_packument(&package_name, &packument_data, &requested_versions)?; + + // npm's make-fetch-happen uses the URL-encoded form for cache keys + // e.g., "https://registry.npmjs.org/@types%2freact-dom" not "@types/react-dom" + // We must use the encoded form in both the cache key string AND the metadata URL + + // Cache with corgiDoc header (for initial requests) + cache + .put( + format!("make-fetch-happen:request-cache:{packument_url}"), + packument_url.clone(), + &normalized_data, + None, // Packuments don't have integrity hashes + Some(ReqHeaders { + accept: String::from(CORGI_DOC), + }), + ) + .map_err(|e| { + anyhow!("couldn't insert packument cache entry (corgi) for {package_name}: {e:?}") + })?; + + // Cache with fullDoc header (for workspace/full metadata requests) + cache + .put( + format!("make-fetch-happen:request-cache:{packument_url}"), + packument_url.clone(), + &normalized_data, + None, + Some(ReqHeaders { + accept: String::from(FULL_DOC), + }), + ) + .map_err(|e| { + anyhow!("couldn't insert packument cache entry (full) for {package_name}: {e:?}") + })?; + } + Err(e) => { + // Log but don't fail - some packages might not need packuments + info!("Warning: couldn't fetch packument for {package_name}: {e}"); + } + } + + Ok::<_, anyhow::Error>(()) + }) +} + /// `fixup_lockfile` rewrites `integrity` hashes to match cache and removes the `integrity` field from Git dependencies. /// /// Sometimes npm has multiple instances of a given `resolved` URL that have different types of `integrity` hashes (e.g. SHA-1 @@ -157,9 +312,21 @@ fn map_cache() -> anyhow::Result> { if entry.file_type().is_file() { let content = fs::read_to_string(entry.path())?; - let key: Key = serde_json::from_str(content.split_ascii_whitespace().nth(1).unwrap())?; + // cacache index format: each line is \t + // Multiple entries can exist in the same file (e.g., same URL with different headers) + for line in content.lines() { + if line.is_empty() { + continue; + } + // Split on tab, not whitespace, because JSON values may contain spaces + let json_part = line + .split_once('\t') + .map(|(_, json)| json) + .ok_or_else(|| anyhow!("invalid cache index entry: missing tab separator"))?; + let key: Key = serde_json::from_str(json_part)?; - hashes.insert(key.metadata.url, key.integrity); + hashes.insert(key.metadata.url, key.integrity); + } } } @@ -241,6 +408,26 @@ fn main() -> anyhow::Result<()> { let cache = Cache::new(out.join("_cacache")); cache.init()?; + // Collect package names and their versions from the lockfile for packument filtering. + // For non-aliased packages: extract from lockfile keys like "node_modules/@scope/name" -> "@scope/name" + // For aliased packages: the name is already the real package name (e.g., "string-width" not "string-width-cjs") + // We only care about packages that have a version string (registry packages). + let mut package_versions: HashMap> = HashMap::new(); + for p in &packages { + if let Some(version) = &p.version { + let pkg_name = p + .name + .rsplit_once("node_modules/") + .map(|(_, name)| name.to_string()) + .unwrap_or_else(|| p.name.clone()); + package_versions + .entry(pkg_name) + .or_default() + .insert(version.to_string()); + } + } + + // Fetch and cache tarballs packages.into_par_iter().try_for_each(|package| { let tarball = package .tarball() @@ -253,12 +440,24 @@ fn main() -> anyhow::Result<()> { package.url, &tarball, integrity, + None, // tarballs don't need special request headers ) .map_err(|e| anyhow!("couldn't insert cache entry for {}: {e:?}", package.name))?; Ok::<_, anyhow::Error>(()) })?; + // Fetch and cache packuments (package metadata) - only for fetcher version 2+ + let fetcher_version: u32 = env::var("NPM_FETCHER_VERSION") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(1); + + if fetcher_version >= 2 { + fetch_packuments(&cache, package_versions)?; + fs::write(out.join(".fetcher-version"), format!("{fetcher_version}"))?; + } + fs::write(out.join("package-lock.json"), lock_content)?; if print_hash { @@ -422,4 +621,5 @@ mod tests { Ok(()) } + } diff --git a/pkgs/build-support/node/prefetch-npm-deps/src/parse/lock.rs b/pkgs/build-support/node/prefetch-npm-deps/src/parse/lock.rs index 17c6c633ee8f..c420fb1e3ede 100644 --- a/pkgs/build-support/node/prefetch-npm-deps/src/parse/lock.rs +++ b/pkgs/build-support/node/prefetch-npm-deps/src/parse/lock.rs @@ -25,7 +25,12 @@ pub(super) fn packages(content: &str) -> anyhow::Result> { .unwrap_or_default() .into_iter() .filter(|(n, p)| !n.is_empty() && matches!(p.resolved, Some(UrlOrString::Url(_)))) - .map(|(n, p)| Package { name: Some(n), ..p }) + .map(|(n, p)| Package { + // Use the package's own name if present (for aliases like string-width-cjs -> string-width), + // otherwise extract from the lockfile key + name: Some(p.name.unwrap_or(n)), + ..p + }) .collect(), _ => bail!( "We don't support lockfile version {}, please file an issue.", @@ -73,6 +78,7 @@ struct OldPackage { pub(super) struct Package { #[serde(default)] pub(super) name: Option, + pub(super) version: Option, pub(super) resolved: Option, pub(super) integrity: Option, } @@ -249,6 +255,7 @@ fn to_new_packages( new.push(Package { name: Some(name), + version: None, // v1 lockfiles don't include version in the same structure resolved: if matches!(package.version, UrlOrString::Url(_)) { Some(package.version) } else { @@ -310,6 +317,7 @@ mod tests { assert_eq!(new.len(), 1, "new packages map should contain 1 value"); assert_eq!(new[0], Package { name: Some(String::from("sqlite3")), + version: None, resolved: Some(UrlOrString::Url(Url::parse("git+ssh://git@github.com/mapbox/node-sqlite3.git#593c9d498be2510d286349134537e3bf89401c4a").unwrap())), integrity: None }); diff --git a/pkgs/build-support/node/prefetch-npm-deps/src/parse/mod.rs b/pkgs/build-support/node/prefetch-npm-deps/src/parse/mod.rs index 6c54d404f3cb..3e6f5019ea33 100644 --- a/pkgs/build-support/node/prefetch-npm-deps/src/parse/mod.rs +++ b/pkgs/build-support/node/prefetch-npm-deps/src/parse/mod.rs @@ -112,6 +112,7 @@ pub fn lockfile( #[derive(Debug)] pub struct Package { pub name: String, + pub version: Option, pub url: Url, specifics: Specifics, } @@ -175,6 +176,7 @@ impl Package { Ok(Package { name: pkg.name.unwrap(), + version: pkg.version, url: resolved, specifics, })