diff --git a/pkgs/build-support/node/build-npm-package/default.nix b/pkgs/build-support/node/build-npm-package/default.nix index 747043e3d12a..14275ca5cbd6 100644 --- a/pkgs/build-support/node/build-npm-package/default.nix +++ b/pkgs/build-support/node/build-npm-package/default.nix @@ -26,6 +26,9 @@ lib.extendMkDerivation { # The output hash of the dependencies for this project. # Can be calculated in advance with prefetch-npm-deps. npmDepsHash ? "", + # Cache format version for npmDeps. Set to 2 to enable packument caching + # for workspace support. Changing this will invalidate npmDepsHash. + npmDepsCacheVersion ? 1, # Whether to force the usage of Git dependencies that have install scripts, but not a lockfile. # Use with care. forceGitDeps ? false, @@ -66,6 +69,7 @@ lib.extendMkDerivation { ; name = "${name}-npm-deps"; hash = npmDepsHash; + cacheVersion = npmDepsCacheVersion; }, # Custom npmConfigHook npmConfigHook ? null, diff --git a/pkgs/build-support/node/prefetch-npm-deps/default.nix b/pkgs/build-support/node/prefetch-npm-deps/default.nix index d5c67dbca65e..ecf4e36a982f 100644 --- a/pkgs/build-support/node/prefetch-npm-deps/default.nix +++ b/pkgs/build-support/node/prefetch-npm-deps/default.nix @@ -215,6 +215,10 @@ # A string with a JSON attrset specifying registry mirrors, for example # {"registry.example.org": "my-mirror.local/registry.example.org"} npmRegistryOverridesString ? config.npmRegistryOverridesString, + # Cache format version. Bump this to invalidate all existing hashes. + # Version 1: original format (tarballs only) + # Version 2: includes packuments for workspace support + cacheVersion ? 1, ... }@args: let @@ -272,6 +276,10 @@ NIX_NPM_REGISTRY_OVERRIDES = npmRegistryOverridesString; + # Cache version controls which features are enabled in prefetch-npm-deps + # Version 2+ enables packument fetching for workspace support + NPM_CACHE_VERSION = toString cacheVersion; + SSL_CERT_FILE = if ( diff --git a/pkgs/build-support/node/prefetch-npm-deps/src/cacache.rs b/pkgs/build-support/node/prefetch-npm-deps/src/cacache.rs index 403c909dee11..19f330768f44 100644 --- a/pkgs/build-support/node/prefetch-npm-deps/src/cacache.rs +++ b/pkgs/build-support/node/prefetch-npm-deps/src/cacache.rs @@ -24,9 +24,16 @@ pub(super) struct Key { #[derive(Serialize, Deserialize)] pub(super) struct Metadata { pub(super) url: Url, + #[serde(rename = "reqHeaders", skip_serializing_if = "Option::is_none")] + pub(super) req_headers: Option, pub(super) options: Options, } +#[derive(Serialize, Deserialize)] +pub struct ReqHeaders { + pub accept: String, +} + #[derive(Serialize, Deserialize)] pub(super) struct Options { pub(super) compress: bool, @@ -58,6 +65,7 @@ impl Cache { url: Url, data: &[u8], integrity: Option, + req_headers: Option, ) -> anyhow::Result<()> { let (algo, hash, integrity) = if let Some(integrity) = integrity { let (algo, hash) = integrity @@ -115,13 +123,27 @@ impl Cache { size: data.len(), metadata: Metadata { url, + req_headers, options: Options { compress: true }, }, })?; - let mut file = File::options().append(true).create(true).open(index_path)?; + let mut file = File::options() + .append(true) + .create(true) + .open(&index_path)?; - write!(file, "{:x}\t{data}", Sha1::new().chain(&data).finalize())?; + // cacache format uses newline as entry separator (see cacache entry-index.js) + // Only add newline prefix if file already has content, to maintain backwards compatibility + let needs_newline = fs::metadata(&index_path) + .map(|m| m.len() > 0) + .unwrap_or(false); + let prefix = if needs_newline { "\n" } else { "" }; + write!( + file, + "{prefix}{:x}\t{data}", + Sha1::new().chain(&data).finalize() + )?; Ok(()) } diff --git a/pkgs/build-support/node/prefetch-npm-deps/src/main.rs b/pkgs/build-support/node/prefetch-npm-deps/src/main.rs index cdac7ef84c0b..03db14740aa5 100644 --- a/pkgs/build-support/node/prefetch-npm-deps/src/main.rs +++ b/pkgs/build-support/node/prefetch-npm-deps/src/main.rs @@ -1,11 +1,12 @@ #![warn(clippy::pedantic)] -use crate::cacache::{Cache, Key}; +use crate::cacache::{Cache, Key, ReqHeaders}; use anyhow::{anyhow, bail}; +use log::info; use rayon::prelude::*; use serde_json::{Map, Value}; use std::{ - collections::HashMap, + collections::{HashMap, HashSet}, env, fs, path::{Path, PathBuf}, process, @@ -22,6 +23,104 @@ fn cache_map_path() -> Option { env::var_os("CACHE_MAP_PATH").map(PathBuf::from) } +/// Extract the package name from an npm registry tarball URL. +/// e.g., "https://registry.npmjs.org/@types/react-dom/-/react-dom-19.1.6.tgz" +/// -> Some("@types/react-dom") +fn extract_package_name_from_url(url: &Url) -> Option { + // Only handle npm registry URLs + let host = url.host_str()?; + if !host.contains("npmjs.org") && !host.contains("npm.") { + return None; + } + + let path = url.path(); + // npm tarball URLs look like: + // /@scope/name/-/name-version.tgz + // /name/-/name-version.tgz + + // Find the "/-/" separator which precedes the tarball filename + let separator_idx = path.find("/-/")?; + let package_path = &path[1..separator_idx]; // Skip leading / + + Some(package_path.to_string()) +} + +/// Get the packument URL for a package name +fn get_packument_url(registry: &str, package_name: &str) -> anyhow::Result { + // URL-encode the package name for scoped packages + let encoded_name = package_name.replace('/', "%2f"); + Url::parse(&format!("{registry}/{encoded_name}")) + .map_err(|e| anyhow!("failed to construct packument URL: {e}")) +} + +/// Fetch and cache packuments (package metadata) for all packages. +/// +/// This is needed because npm may query package metadata for optional peer dependencies +/// and for workspace packages. +/// +/// npm's cache policy checks that the Accept header matches between the cached +/// request and the new request. npm can request packuments with two different headers: +/// 1. "corgiDoc" (abbreviated metadata) - used initially +/// 2. "fullDoc" (full metadata) - used when npm needs full package info (e.g., workspaces) +/// +/// We cache both versions to ensure cache hits regardless of which header npm uses. +/// See: pacote/lib/registry.js and @npmcli/arborist/lib/arborist/build-ideal-tree.js +fn fetch_packuments(cache: &Cache, package_names: HashSet) -> anyhow::Result<()> { + const CORGI_DOC: &str = + "application/vnd.npm.install-v1+json; q=1.0, application/json; q=0.8, */*"; + const FULL_DOC: &str = "application/json"; + + info!("Fetching {} packuments", package_names.len()); + + package_names.into_par_iter().try_for_each(|package_name| { + let packument_url = get_packument_url("https://registry.npmjs.org", &package_name)?; + + match util::get_url_body_with_retry(&packument_url) { + Ok(packument_data) => { + // npm's make-fetch-happen uses the URL-encoded form for cache keys + // e.g., "https://registry.npmjs.org/@types%2freact-dom" not "@types/react-dom" + // We must use the encoded form in both the cache key string AND the metadata URL + + // Cache with corgiDoc header (for initial requests) + cache + .put( + format!("make-fetch-happen:request-cache:{packument_url}"), + packument_url.clone(), + &packument_data, + None, // Packuments don't have integrity hashes + Some(ReqHeaders { + accept: String::from(CORGI_DOC), + }), + ) + .map_err(|e| { + anyhow!("couldn't insert packument cache entry (corgi) for {package_name}: {e:?}") + })?; + + // Cache with fullDoc header (for workspace/full metadata requests) + cache + .put( + format!("make-fetch-happen:request-cache:{packument_url}"), + packument_url.clone(), + &packument_data, + None, + Some(ReqHeaders { + accept: String::from(FULL_DOC), + }), + ) + .map_err(|e| { + anyhow!("couldn't insert packument cache entry (full) for {package_name}: {e:?}") + })?; + } + Err(e) => { + // Log but don't fail - some packages might not need packuments + info!("Warning: couldn't fetch packument for {package_name}: {e}"); + } + } + + Ok::<_, anyhow::Error>(()) + }) +} + /// `fixup_lockfile` rewrites `integrity` hashes to match cache and removes the `integrity` field from Git dependencies. /// /// Sometimes npm has multiple instances of a given `resolved` URL that have different types of `integrity` hashes (e.g. SHA-1 @@ -157,9 +256,21 @@ fn map_cache() -> anyhow::Result> { if entry.file_type().is_file() { let content = fs::read_to_string(entry.path())?; - let key: Key = serde_json::from_str(content.split_ascii_whitespace().nth(1).unwrap())?; + // cacache index format: each line is \t + // Multiple entries can exist in the same file (e.g., same URL with different headers) + for line in content.lines() { + if line.is_empty() { + continue; + } + // Split on tab, not whitespace, because JSON values may contain spaces + let json_part = line + .split_once('\t') + .map(|(_, json)| json) + .ok_or_else(|| anyhow!("invalid cache index entry: missing tab separator"))?; + let key: Key = serde_json::from_str(json_part)?; - hashes.insert(key.metadata.url, key.integrity); + hashes.insert(key.metadata.url, key.integrity); + } } } @@ -241,6 +352,13 @@ fn main() -> anyhow::Result<()> { let cache = Cache::new(out.join("_cacache")); cache.init()?; + // Collect unique package names for packument fetching + let package_names: HashSet = packages + .iter() + .filter_map(|p| extract_package_name_from_url(&p.url)) + .collect(); + + // Fetch and cache tarballs packages.into_par_iter().try_for_each(|package| { let tarball = package .tarball() @@ -253,12 +371,23 @@ fn main() -> anyhow::Result<()> { package.url, &tarball, integrity, + None, // tarballs don't need special request headers ) .map_err(|e| anyhow!("couldn't insert cache entry for {}: {e:?}", package.name))?; Ok::<_, anyhow::Error>(()) })?; + // Fetch and cache packuments (package metadata) - only for cache version 2+ + let cache_version: u32 = env::var("NPM_CACHE_VERSION") + .ok() + .and_then(|v| v.parse().ok()) + .unwrap_or(1); + + if cache_version >= 2 { + fetch_packuments(&cache, package_names)?; + } + fs::write(out.join("package-lock.json"), lock_content)?; if print_hash { @@ -422,4 +551,35 @@ mod tests { Ok(()) } + + #[test] + fn test_extract_package_name_from_url() { + use super::extract_package_name_from_url; + use url::Url; + + // Regular package + assert_eq!( + extract_package_name_from_url( + &Url::parse("https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz").unwrap() + ), + Some("lodash".to_string()) + ); + + // Scoped package + assert_eq!( + extract_package_name_from_url( + &Url::parse("https://registry.npmjs.org/@types/react-dom/-/react-dom-19.1.6.tgz") + .unwrap() + ), + Some("@types/react-dom".to_string()) + ); + + // Non-npm URL should return None + assert_eq!( + extract_package_name_from_url( + &Url::parse("https://github.com/foo/bar/archive/main.tar.gz").unwrap() + ), + None + ); + } }