gador/nixpkgs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

prefetch-npm-deps: add cacheVersion for packument support

Browse Source 
Add a cacheVersion parameter to fetchNpmDeps and npmDepsCacheVersion to
buildNpmPackage. When set to 2, prefetch-npm-deps will also fetch and
cache packuments (package metadata) in addition to tarballs.

npm can request packuments with two different Accept headers:
- corgiDoc: abbreviated metadata (default)
- fullDoc: full metadata (used for workspaces)

npm's cache policy requires headers to match, so we cache both versions.

This is opt-in via cacheVersion to avoid breaking existing hashes.
Set npmDepsCacheVersion = 2 for projects using npm workspaces.

Also fix cacache index format to properly separate multiple entries
with newlines, and update map_cache() to parse multi-line index files.
This commit is contained in:
Jörg Thalheim
2025-12-31 00:37:52 -05:00
committed by Winter
parent 8ebf0e3108
commit 5c99ffcb8e
4 changed files with 200 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkgs/build-support/node/build-npm-package/default.nix
+4
View File
@@ -26,6 +26,9 @@ lib.extendMkDerivation {
# The output hash of the dependencies for this project.
# Can be calculated in advance with prefetch-npm-deps.
npmDepsHash ? "",
# Cache format version for npmDeps. Set to 2 to enable packument caching
# for workspace support. Changing this will invalidate npmDepsHash.
npmDepsCacheVersion ? 1,
# Whether to force the usage of Git dependencies that have install scripts, but not a lockfile.
# Use with care.
forceGitDeps ? false,
@@ -66,6 +69,7 @@ lib.extendMkDerivation {
;
name = "${name}-npm-deps";
hash = npmDepsHash;
cacheVersion = npmDepsCacheVersion;
},
# Custom npmConfigHook
npmConfigHook ? null,
pkgs/build-support/node/prefetch-npm-deps/default.nix
+8
View File
@@ -215,6 +215,10 @@
# A string with a JSON attrset specifying registry mirrors, for example
# {"registry.example.org": "my-mirror.local/registry.example.org"}
npmRegistryOverridesString ? config.npmRegistryOverridesString,
# Cache format version. Bump this to invalidate all existing hashes.
# Version 1: original format (tarballs only)
# Version 2: includes packuments for workspace support
cacheVersion ? 1,
...
}@args:
let
@@ -272,6 +276,10 @@
NIX_NPM_REGISTRY_OVERRIDES = npmRegistryOverridesString;
# Cache version controls which features are enabled in prefetch-npm-deps
# Version 2+ enables packument fetching for workspace support
NPM_CACHE_VERSION = toString cacheVersion;
SSL_CERT_FILE =
if
(
pkgs/build-support/node/prefetch-npm-deps/src/cacache.rs
+24 -2
View File
@@ -24,9 +24,16 @@ pub(super) struct Key {
#[derive(Serialize, Deserialize)]
pub(super) struct Metadata {
pub(super) url: Url,
#[serde(rename = "reqHeaders", skip_serializing_if = "Option::is_none")]
pub(super) req_headers: Option<ReqHeaders>,
pub(super) options: Options,
}
#[derive(Serialize, Deserialize)]
pub struct ReqHeaders {
pub accept: String,
}
#[derive(Serialize, Deserialize)]
pub(super) struct Options {
pub(super) compress: bool,
@@ -58,6 +65,7 @@ impl Cache {
url: Url,
data: &[u8],
integrity: Option<String>,
req_headers: Option<ReqHeaders>,
) -> anyhow::Result<()> {
let (algo, hash, integrity) = if let Some(integrity) = integrity {
let (algo, hash) = integrity
@@ -115,13 +123,27 @@ impl Cache {
size: data.len(),
metadata: Metadata {
url,
req_headers,
options: Options { compress: true },
},
})?;
let mut file = File::options().append(true).create(true).open(index_path)?;
let mut file = File::options()
.append(true)
.create(true)
.open(&index_path)?;
write!(file, "{:x}\t{data}", Sha1::new().chain(&data).finalize())?;
// cacache format uses newline as entry separator (see cacache entry-index.js)
// Only add newline prefix if file already has content, to maintain backwards compatibility
let needs_newline = fs::metadata(&index_path)
.map(|m| m.len() > 0)
.unwrap_or(false);
let prefix = if needs_newline { "\n" } else { "" };
write!(
file,
"{prefix}{:x}\t{data}",
Sha1::new().chain(&data).finalize()
)?;
Ok(())
}
pkgs/build-support/node/prefetch-npm-deps/src/main.rs
+164 -4
View File
@@ -1,11 +1,12 @@
#![warn(clippy::pedantic)]
use crate::cacache::{Cache, Key};
use crate::cacache::{Cache, Key, ReqHeaders};
use anyhow::{anyhow, bail};
use log::info;
use rayon::prelude::*;
use serde_json::{Map, Value};
use std::{
collections::HashMap,
collections::{HashMap, HashSet},
env, fs,
path::{Path, PathBuf},
process,
@@ -22,6 +23,104 @@ fn cache_map_path() -> Option<PathBuf> {
env::var_os("CACHE_MAP_PATH").map(PathBuf::from)
}
/// Extract the package name from an npm registry tarball URL.
/// e.g., "https://registry.npmjs.org/@types/react-dom/-/react-dom-19.1.6.tgz"
/// -> Some("@types/react-dom")
fn extract_package_name_from_url(url: &Url) -> Option<String> {
// Only handle npm registry URLs
let host = url.host_str()?;
if !host.contains("npmjs.org") && !host.contains("npm.") {
return None;
}
let path = url.path();
// npm tarball URLs look like:
// /@scope/name/-/name-version.tgz
// /name/-/name-version.tgz
// Find the "/-/" separator which precedes the tarball filename
let separator_idx = path.find("/-/")?;
let package_path = &path[1..separator_idx]; // Skip leading /
Some(package_path.to_string())
}
/// Get the packument URL for a package name
fn get_packument_url(registry: &str, package_name: &str) -> anyhow::Result<Url> {
// URL-encode the package name for scoped packages
let encoded_name = package_name.replace('/', "%2f");
Url::parse(&format!("{registry}/{encoded_name}"))
.map_err(|e| anyhow!("failed to construct packument URL: {e}"))
}
/// Fetch and cache packuments (package metadata) for all packages.
///
/// This is needed because npm may query package metadata for optional peer dependencies
/// and for workspace packages.
///
/// npm's cache policy checks that the Accept header matches between the cached
/// request and the new request. npm can request packuments with two different headers:
/// 1. "corgiDoc" (abbreviated metadata) - used initially
/// 2. "fullDoc" (full metadata) - used when npm needs full package info (e.g., workspaces)
///
/// We cache both versions to ensure cache hits regardless of which header npm uses.
/// See: pacote/lib/registry.js and @npmcli/arborist/lib/arborist/build-ideal-tree.js
fn fetch_packuments(cache: &Cache, package_names: HashSet<String>) -> anyhow::Result<()> {
const CORGI_DOC: &str =
"application/vnd.npm.install-v1+json; q=1.0, application/json; q=0.8, */*";
const FULL_DOC: &str = "application/json";
info!("Fetching {} packuments", package_names.len());
package_names.into_par_iter().try_for_each(|package_name| {
let packument_url = get_packument_url("https://registry.npmjs.org", &package_name)?;
match util::get_url_body_with_retry(&packument_url) {
Ok(packument_data) => {
// npm's make-fetch-happen uses the URL-encoded form for cache keys
// e.g., "https://registry.npmjs.org/@types%2freact-dom" not "@types/react-dom"
// We must use the encoded form in both the cache key string AND the metadata URL
// Cache with corgiDoc header (for initial requests)
cache
.put(
format!("make-fetch-happen:request-cache:{packument_url}"),
packument_url.clone(),
&packument_data,
None, // Packuments don't have integrity hashes
Some(ReqHeaders {
accept: String::from(CORGI_DOC),
}),
)
.map_err(|e| {
anyhow!("couldn't insert packument cache entry (corgi) for {package_name}: {e:?}")
})?;
// Cache with fullDoc header (for workspace/full metadata requests)
cache
.put(
format!("make-fetch-happen:request-cache:{packument_url}"),
packument_url.clone(),
&packument_data,
None,
Some(ReqHeaders {
accept: String::from(FULL_DOC),
}),
)
.map_err(|e| {
anyhow!("couldn't insert packument cache entry (full) for {package_name}: {e:?}")
})?;
}
Err(e) => {
// Log but don't fail - some packages might not need packuments
info!("Warning: couldn't fetch packument for {package_name}: {e}");
}
}
Ok::<_, anyhow::Error>(())
})
}
/// `fixup_lockfile` rewrites `integrity` hashes to match cache and removes the `integrity` field from Git dependencies.
///
/// Sometimes npm has multiple instances of a given `resolved` URL that have different types of `integrity` hashes (e.g. SHA-1
@@ -157,9 +256,21 @@ fn map_cache() -> anyhow::Result<HashMap<Url, String>> {
if entry.file_type().is_file() {
let content = fs::read_to_string(entry.path())?;
let key: Key = serde_json::from_str(content.split_ascii_whitespace().nth(1).unwrap())?;
// cacache index format: each line is <sha1_hash>\t<json>
// Multiple entries can exist in the same file (e.g., same URL with different headers)
for line in content.lines() {
if line.is_empty() {
continue;
}
// Split on tab, not whitespace, because JSON values may contain spaces
let json_part = line
.split_once('\t')
.map(|(_, json)| json)
.ok_or_else(|| anyhow!("invalid cache index entry: missing tab separator"))?;
let key: Key = serde_json::from_str(json_part)?;
hashes.insert(key.metadata.url, key.integrity);
hashes.insert(key.metadata.url, key.integrity);
}
}
}
@@ -241,6 +352,13 @@ fn main() -> anyhow::Result<()> {
let cache = Cache::new(out.join("_cacache"));
cache.init()?;
// Collect unique package names for packument fetching
let package_names: HashSet<String> = packages
.iter()
.filter_map(|p| extract_package_name_from_url(&p.url))
.collect();
// Fetch and cache tarballs
packages.into_par_iter().try_for_each(|package| {
let tarball = package
.tarball()
@@ -253,12 +371,23 @@ fn main() -> anyhow::Result<()> {
package.url,
&tarball,
integrity,
None, // tarballs don't need special request headers
)
.map_err(|e| anyhow!("couldn't insert cache entry for {}: {e:?}", package.name))?;
Ok::<_, anyhow::Error>(())
})?;
// Fetch and cache packuments (package metadata) - only for cache version 2+
let cache_version: u32 = env::var("NPM_CACHE_VERSION")
.ok()
.and_then(|v| v.parse().ok())
.unwrap_or(1);
if cache_version >= 2 {
fetch_packuments(&cache, package_names)?;
}
fs::write(out.join("package-lock.json"), lock_content)?;
if print_hash {
@@ -422,4 +551,35 @@ mod tests {
Ok(())
}
#[test]
fn test_extract_package_name_from_url() {
use super::extract_package_name_from_url;
use url::Url;
// Regular package
assert_eq!(
extract_package_name_from_url(
&Url::parse("https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz").unwrap()
),
Some("lodash".to_string())
);
// Scoped package
assert_eq!(
extract_package_name_from_url(
&Url::parse("https://registry.npmjs.org/@types/react-dom/-/react-dom-19.1.6.tgz")
.unwrap()
),
Some("@types/react-dom".to_string())
);
// Non-npm URL should return None
assert_eq!(
extract_package_name_from_url(
&Url::parse("https://github.com/foo/bar/archive/main.tar.gz").unwrap()
),
None
);
}
}