gador/nixpkgs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

prefetch-npm-deps: add cacheVersion for packument support (#470517)

Browse Source
This commit is contained in:
Philip Taron
2026-01-19 15:38:43 +00:00
committed by GitHub
parent a3d32740e4 08e805c550
commit 963873d175
8 changed files with 288 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
doc/release-notes/rl-2605.section.md
+2
View File
@@ -144,6 +144,8 @@
- `fetchPnpmDeps` and `pnpmConfigHook` were added as top-level attributes, replacing the now deprecated `pnpm.fetchDeps` and `pnpm.configHook` attributes.
- `buildNpmPackage` now supports `npmDepsCacheVersion`. Set to `2` to enable packument caching, which fixes builds for projects using npm workspaces.
- Added `dell-bios-fan-control` package and service.
- `openrgb` was updated to 1.0rc2, which now uses Plugin API version 4.
pkgs/build-support/node/build-npm-package/default.nix
+8
View File
@@ -26,6 +26,9 @@ lib.extendMkDerivation {
# The output hash of the dependencies for this project.
# Can be calculated in advance with prefetch-npm-deps.
npmDepsHash ? "",
# Fetcher format version for npmDeps. Set to 2 to enable packument caching
# for workspace support. Changing this will invalidate npmDepsHash.
npmDepsFetcherVersion ? 1,
# Whether to force the usage of Git dependencies that have install scripts, but not a lockfile.
# Use with care.
forceGitDeps ? false,
@@ -66,6 +69,7 @@ lib.extendMkDerivation {
;
name = "${name}-npm-deps";
hash = npmDepsHash;
fetcherVersion = npmDepsFetcherVersion;
},
# Custom npmConfigHook
npmConfigHook ? null,
@@ -85,6 +89,10 @@ lib.extendMkDerivation {
{
inherit npmDeps npmBuildScript;
env = (args.env or { }) // {
NIX_NPM_FETCHER_VERSION = npmDepsFetcherVersion;
};
nativeBuildInputs =
nativeBuildInputs
++ [
pkgs/build-support/node/build-npm-package/hooks/npm-config-hook.sh
+31 -3
View File
@@ -28,6 +28,29 @@ npmConfigHook() {
exit 1
fi
if [[ -e "$npmDeps/.fetcher-version" ]]; then
local -r fetcherVersion=$(cat "$npmDeps/.fetcher-version")
else
local -r fetcherVersion="1"
fi
# Only run this in buildNpmPackage, this is just for a nicer error message; we trust that
# people using the setup hook directly also know how FODs work. ;)
if [[ -n ${NIX_NPM_FETCHER_VERSION+x} ]] && [[ $NIX_NPM_FETCHER_VERSION != $fetcherVersion ]]; then
echo
echo "ERROR: npmDepsHash is out of date"
echo
echo "The fetcher version in the arguments to buildNpmPackage ($NIX_NPM_FETCHER_VERSION) is not the same as the one in $npmDeps ($fetcherVersion)."
echo
echo "To fix the issue:"
echo '1. Use `lib.fakeHash` as the npmDepsHash value'
echo "2. Build the derivation and wait for it to fail with a hash mismatch"
echo "3. Copy the 'got: sha256-' value back into the npmDepsHash field"
echo
exit 1
fi
local -r cacheLockfile="$npmDeps/package-lock.json"
if [[ -f npm-shrinkwrap.json ]]; then
local -r srcLockfile="$PWD/npm-shrinkwrap.json"
@@ -78,7 +101,11 @@ npmConfigHook() {
local cachePath
if [ -z "${makeCacheWritable-}" ]; then
# When a given cache key has multiple entries (which is the case with
# fetcher version 2), npm always needs to write to the cache.
#
# TODO(winter): report upstream?
if [ -z "${makeCacheWritable-}" ] && (( fetcherVersion == 1 )); then
cachePath="$npmDeps"
else
echo "Making cache writable"
@@ -100,9 +127,10 @@ npmConfigHook() {
echo "ERROR: npm failed to install dependencies"
echo
echo "Here are a few things you can try, depending on the error:"
echo '1. Set `makeCacheWritable = true`'
echo '1. Set `npmDepsFetcherVersion = 2` (and update `npmDepsHash`)'
echo '2. Set `makeCacheWritable = true`'
echo " Note that this won't help if npm is complaining about not being able to write to the logs directory -- look above that for the actual error."
echo '2. Set `npmFlags = [ "--legacy-peer-deps" ]`'
echo '3. Set `npmFlags = [ "--legacy-peer-deps" ]`'
echo
exit 1
pkgs/build-support/node/prefetch-npm-deps/default.nix
+8
View File
@@ -215,6 +215,10 @@
# A string with a JSON attrset specifying registry mirrors, for example
# {"registry.example.org": "my-mirror.local/registry.example.org"}
npmRegistryOverridesString ? config.npmRegistryOverridesString,
# Fetcher format version. Bump this to invalidate all existing hashes.
# Version 1: original format (tarballs only)
# Version 2: includes packuments for workspace support
fetcherVersion ? 1,
...
}@args:
let
@@ -272,6 +276,10 @@
NIX_NPM_REGISTRY_OVERRIDES = npmRegistryOverridesString;
# Fetcher version controls which features are enabled in prefetch-npm-deps
# Version 2+ enables packument fetching for workspace support
NPM_FETCHER_VERSION = toString fetcherVersion;
SSL_CERT_FILE =
if
(
pkgs/build-support/node/prefetch-npm-deps/src/cacache.rs
+24 -2
View File
@@ -24,9 +24,16 @@ pub(super) struct Key {
#[derive(Serialize, Deserialize)]
pub(super) struct Metadata {
pub(super) url: Url,
#[serde(rename = "reqHeaders", skip_serializing_if = "Option::is_none")]
pub(super) req_headers: Option<ReqHeaders>,
pub(super) options: Options,
}
#[derive(Serialize, Deserialize)]
pub struct ReqHeaders {
pub accept: String,
}
#[derive(Serialize, Deserialize)]
pub(super) struct Options {
pub(super) compress: bool,
@@ -58,6 +65,7 @@ impl Cache {
url: Url,
data: &[u8],
integrity: Option<String>,
req_headers: Option<ReqHeaders>,
) -> anyhow::Result<()> {
let (algo, hash, integrity) = if let Some(integrity) = integrity {
let (algo, hash) = integrity
@@ -115,13 +123,27 @@ impl Cache {
size: data.len(),
metadata: Metadata {
url,
req_headers,
options: Options { compress: true },
},
})?;
let mut file = File::options().append(true).create(true).open(index_path)?;
let mut file = File::options()
.append(true)
.create(true)
.open(&index_path)?;
write!(file, "{:x}\t{data}", Sha1::new().chain(&data).finalize())?;
// cacache format uses newline as entry separator (see cacache entry-index.js)
// Only add newline prefix if file already has content, to maintain backwards compatibility with previous versions of prefetch-npm-deps, which handled this case incorrectly.
let needs_newline = fs::metadata(&index_path)
.map(|m| m.len() > 0)
.unwrap_or(false);
let prefix = if needs_newline { "\n" } else { "" };
write!(
file,
"{prefix}{:x}\t{data}",
Sha1::new().chain(&data).finalize()
)?;
Ok(())
}
pkgs/build-support/node/prefetch-npm-deps/src/main.rs
+204 -4
View File
@@ -1,11 +1,12 @@
#![warn(clippy::pedantic)]
use crate::cacache::{Cache, Key};
use crate::cacache::{Cache, Key, ReqHeaders};
use anyhow::{anyhow, bail};
use log::info;
use rayon::prelude::*;
use serde_json::{Map, Value};
use std::{
collections::HashMap,
collections::{HashMap, HashSet},
env, fs,
path::{Path, PathBuf},
process,
@@ -22,6 +23,160 @@ fn cache_map_path() -> Option<PathBuf> {
env::var_os("CACHE_MAP_PATH").map(PathBuf::from)
}
/// Get the packument URL for a package name
fn get_packument_url(registry: &str, package_name: &str) -> anyhow::Result<Url> {
// URL-encode the package name for scoped packages
let encoded_name = package_name.replace('/', "%2f");
Url::parse(&format!("{registry}/{encoded_name}"))
.map_err(|e| anyhow!("failed to construct packument URL: {e}"))
}
/// Normalize packument data to ensure determinism.
///
/// Filters to whitelisted fields and requested versions only.
/// Allowed top-level fields in normalized packuments.
///
/// For lockfile-based installs, versions are exact (e.g., "4.17.21") so npm-pick-manifest
/// just does a direct `versions[ver]` lookup. Tarballs are fetched via the resolved URL.
const ALLOWED_TOP_LEVEL_FIELDS: &[&str] = &["name", "versions"];
/// Allowed fields in version objects.
///
/// Based on analysis of pacote, npm-pick-manifest, npm-install-checks, and arborist.
/// Only fields actually read during `npm install` are included.
const ALLOWED_VERSION_FIELDS: &[&str] = &[
"name",
"version",
// Dependencies
"dependencies",
"devDependencies",
"peerDependencies",
"peerDependenciesMeta",
"optionalDependencies",
"bundleDependencies",
"bundledDependencies",
// Distribution (tarball URL and integrity)
"dist",
// Executables
"bin",
// Platform constraints (npm-install-checks)
"engines",
"os",
"cpu",
// Lifecycle scripts
"scripts",
// Version selection hint (npm-pick-manifest)
"deprecated",
];
fn normalize_packument(
package_name: &str,
data: &[u8],
requested_versions: &HashSet<String>,
) -> anyhow::Result<Vec<u8>> {
let mut json: Value = serde_json::from_slice(data)
.map_err(|e| anyhow!("failed to parse packument JSON for {package_name}: {e}"))?;
let obj = json
.as_object_mut()
.ok_or_else(|| anyhow!("packument for {package_name} is not a JSON object"))?;
// Keep only whitelisted top-level fields to ensure determinism
obj.retain(|key, _| ALLOWED_TOP_LEVEL_FIELDS.contains(&key.as_str()));
// Filter and normalize versions
if let Some(Value::Object(versions)) = obj.get_mut("versions") {
// Only keep versions that are in the lockfile
versions.retain(|version, _| requested_versions.contains(version));
// Normalize each version object to only include necessary fields
for version_val in versions.values_mut() {
if let Some(version_obj) = version_val.as_object_mut() {
version_obj.retain(|key, _| ALLOWED_VERSION_FIELDS.contains(&key.as_str()));
}
}
}
serde_json::to_vec(&json)
.map_err(|e| anyhow!("failed to re-serialize packument for {package_name}: {e}"))
}
/// Fetch and cache packuments (package metadata) for all packages.
///
/// This is needed because npm may query package metadata for optional peer dependencies
/// and for workspace packages.
///
/// npm's cache policy checks that the Accept header matches between the cached
/// request and the new request. npm can request packuments with two different headers:
/// 1. "corgiDoc" (abbreviated metadata) - used initially
/// 2. "fullDoc" (full metadata) - used when npm needs full package info (e.g., workspaces)
///
/// We cache both versions to ensure cache hits regardless of which header npm uses.
/// See: pacote/lib/registry.js and @npmcli/arborist/lib/arborist/build-ideal-tree.js
fn fetch_packuments(
cache: &Cache,
package_versions: HashMap<String, HashSet<String>>,
) -> anyhow::Result<()> {
const CORGI_DOC: &str =
"application/vnd.npm.install-v1+json; q=1.0, application/json; q=0.8, */*";
const FULL_DOC: &str = "application/json";
info!("Fetching {} packuments", package_versions.len());
package_versions
.into_par_iter()
.try_for_each(|(package_name, requested_versions)| {
let packument_url = get_packument_url("https://registry.npmjs.org", &package_name)?;
match util::get_url_body_with_retry(&packument_url) {
Ok(packument_data) => {
let normalized_data =
normalize_packument(&package_name, &packument_data, &requested_versions)?;
// npm's make-fetch-happen uses the URL-encoded form for cache keys
// e.g., "https://registry.npmjs.org/@types%2freact-dom" not "@types/react-dom"
// We must use the encoded form in both the cache key string AND the metadata URL
// Cache with corgiDoc header (for initial requests)
cache
.put(
format!("make-fetch-happen:request-cache:{packument_url}"),
packument_url.clone(),
&normalized_data,
None, // Packuments don't have integrity hashes
Some(ReqHeaders {
accept: String::from(CORGI_DOC),
}),
)
.map_err(|e| {
anyhow!("couldn't insert packument cache entry (corgi) for {package_name}: {e:?}")
})?;
// Cache with fullDoc header (for workspace/full metadata requests)
cache
.put(
format!("make-fetch-happen:request-cache:{packument_url}"),
packument_url.clone(),
&normalized_data,
None,
Some(ReqHeaders {
accept: String::from(FULL_DOC),
}),
)
.map_err(|e| {
anyhow!("couldn't insert packument cache entry (full) for {package_name}: {e:?}")
})?;
}
Err(e) => {
// Log but don't fail - some packages might not need packuments
info!("Warning: couldn't fetch packument for {package_name}: {e}");
}
}
Ok::<_, anyhow::Error>(())
})
}
/// `fixup_lockfile` rewrites `integrity` hashes to match cache and removes the `integrity` field from Git dependencies.
///
/// Sometimes npm has multiple instances of a given `resolved` URL that have different types of `integrity` hashes (e.g. SHA-1
@@ -157,9 +312,21 @@ fn map_cache() -> anyhow::Result<HashMap<Url, String>> {
if entry.file_type().is_file() {
let content = fs::read_to_string(entry.path())?;
let key: Key = serde_json::from_str(content.split_ascii_whitespace().nth(1).unwrap())?;
// cacache index format: each line is <sha1_hash>\t<json>
// Multiple entries can exist in the same file (e.g., same URL with different headers)
for line in content.lines() {
if line.is_empty() {
continue;
}
// Split on tab, not whitespace, because JSON values may contain spaces
let json_part = line
.split_once('\t')
.map(|(_, json)| json)
.ok_or_else(|| anyhow!("invalid cache index entry: missing tab separator"))?;
let key: Key = serde_json::from_str(json_part)?;
hashes.insert(key.metadata.url, key.integrity);
hashes.insert(key.metadata.url, key.integrity);
}
}
}
@@ -241,6 +408,26 @@ fn main() -> anyhow::Result<()> {
let cache = Cache::new(out.join("_cacache"));
cache.init()?;
// Collect package names and their versions from the lockfile for packument filtering.
// For non-aliased packages: extract from lockfile keys like "node_modules/@scope/name" -> "@scope/name"
// For aliased packages: the name is already the real package name (e.g., "string-width" not "string-width-cjs")
// We only care about packages that have a version string (registry packages).
let mut package_versions: HashMap<String, HashSet<String>> = HashMap::new();
for p in &packages {
if let Some(version) = &p.version {
let pkg_name = p
.name
.rsplit_once("node_modules/")
.map(|(_, name)| name.to_string())
.unwrap_or_else(|| p.name.clone());
package_versions
.entry(pkg_name)
.or_default()
.insert(version.to_string());
}
}
// Fetch and cache tarballs
packages.into_par_iter().try_for_each(|package| {
let tarball = package
.tarball()
@@ -253,12 +440,24 @@ fn main() -> anyhow::Result<()> {
package.url,
&tarball,
integrity,
None, // tarballs don't need special request headers
)
.map_err(|e| anyhow!("couldn't insert cache entry for {}: {e:?}", package.name))?;
Ok::<_, anyhow::Error>(())
})?;
// Fetch and cache packuments (package metadata) - only for fetcher version 2+
let fetcher_version: u32 = env::var("NPM_FETCHER_VERSION")
.ok()
.and_then(|v| v.parse().ok())
.unwrap_or(1);
if fetcher_version >= 2 {
fetch_packuments(&cache, package_versions)?;
fs::write(out.join(".fetcher-version"), format!("{fetcher_version}"))?;
}
fs::write(out.join("package-lock.json"), lock_content)?;
if print_hash {
@@ -422,4 +621,5 @@ mod tests {
Ok(())
}
}
pkgs/build-support/node/prefetch-npm-deps/src/parse/lock.rs
+9 -1
View File
@@ -25,7 +25,12 @@ pub(super) fn packages(content: &str) -> anyhow::Result<Vec<Package>> {
.unwrap_or_default()
.into_iter()
.filter(|(n, p)| !n.is_empty() && matches!(p.resolved, Some(UrlOrString::Url(_))))
.map(|(n, p)| Package { name: Some(n), ..p })
.map(|(n, p)| Package {
// Use the package's own name if present (for aliases like string-width-cjs -> string-width),
// otherwise extract from the lockfile key
name: Some(p.name.unwrap_or(n)),
..p
})
.collect(),
_ => bail!(
"We don't support lockfile version {}, please file an issue.",
@@ -73,6 +78,7 @@ struct OldPackage {
pub(super) struct Package {
#[serde(default)]
pub(super) name: Option<String>,
pub(super) version: Option<String>,
pub(super) resolved: Option<UrlOrString>,
pub(super) integrity: Option<HashCollection>,
}
@@ -249,6 +255,7 @@ fn to_new_packages(
new.push(Package {
name: Some(name),
version: None, // v1 lockfiles don't include version in the same structure
resolved: if matches!(package.version, UrlOrString::Url(_)) {
Some(package.version)
} else {
@@ -310,6 +317,7 @@ mod tests {
assert_eq!(new.len(), 1, "new packages map should contain 1 value");
assert_eq!(new[0], Package {
name: Some(String::from("sqlite3")),
version: None,
resolved: Some(UrlOrString::Url(Url::parse("git+ssh://git@github.com/mapbox/node-sqlite3.git#593c9d498be2510d286349134537e3bf89401c4a").unwrap())),
integrity: None
});
pkgs/build-support/node/prefetch-npm-deps/src/parse/mod.rs
+2
View File
@@ -112,6 +112,7 @@ pub fn lockfile(
#[derive(Debug)]
pub struct Package {
pub name: String,
pub version: Option<String>,
pub url: Url,
specifics: Specifics,
}
@@ -175,6 +176,7 @@ impl Package {
Ok(Package {
name: pkg.name.unwrap(),
version: pkg.version,
url: resolved,
specifics,
})