gador/nixpkgs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

prefetch-npm-deps: Bump edition to 2024 and apply cargo clippy and rustfmt fixes

Browse Source
This commit is contained in:
Jonathan Davies
2025-12-02 14:31:14 +00:00
parent 1cfd281c2c
commit bb37dfe300
5 changed files with 108 additions and 98 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkgs/build-support/node/prefetch-npm-deps/Cargo.toml
+1 -1
View File
@@ -1,7 +1,7 @@
[package]
name = "prefetch-npm-deps"
version = "0.1.0"
edition = "2021"
edition = "2024"
# See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
pkgs/build-support/node/prefetch-npm-deps/src/main.rs
+57 -59
View File
@@ -38,7 +38,7 @@ fn cache_map_path() -> Option<PathBuf> {
/// `dependencies` key in v2 lockfiles designed for backwards compatibility with v1 parsers is removed because of inconsistent data.
fn fixup_lockfile(
mut lock: Map<String, Value>,
cache: &Option<HashMap<String, String>>,
cache: Option<&HashMap<String, String>>,
) -> anyhow::Result<Option<Map<String, Value>>> {
let mut fixed = false;
@@ -64,29 +64,29 @@ fn fixup_lockfile(
.ok_or_else(|| anyhow!("packages isn't a map"))?
.values_mut()
{
if let Some(Value::String(resolved)) = package.get("resolved") {
if let Some(Value::String(integrity)) = package.get("integrity") {
if resolved.starts_with("git+") {
if let Some(Value::String(resolved)) = package.get("resolved")
&& let Some(Value::String(integrity)) = package.get("integrity")
{
if resolved.starts_with("git+") {
fixed = true;
package
.as_object_mut()
.ok_or_else(|| anyhow!("package isn't a map"))?
.remove("integrity");
} else if let Some(cache_hashes) = cache {
let cache_hash = cache_hashes
.get(resolved)
.expect("dependency should have a hash");
if integrity != cache_hash {
fixed = true;
package
*package
.as_object_mut()
.ok_or_else(|| anyhow!("package isn't a map"))?
.remove("integrity");
} else if let Some(cache_hashes) = cache {
let cache_hash = cache_hashes
.get(resolved)
.expect("dependency should have a hash");
if integrity != cache_hash {
fixed = true;
*package
.as_object_mut()
.ok_or_else(|| anyhow!("package isn't a map"))?
.get_mut("integrity")
.unwrap() = Value::String(cache_hash.clone());
}
.get_mut("integrity")
.unwrap() = Value::String(cache_hash.clone());
}
}
}
@@ -99,17 +99,13 @@ fn fixup_lockfile(
v => bail!("unsupported lockfile version {v}"),
}
if fixed {
Ok(Some(lock))
} else {
Ok(None)
}
if fixed { Ok(Some(lock)) } else { Ok(None) }
}
// Recursive helper to fixup v1 lockfile deps
fn fixup_v1_deps(
dependencies: &mut Map<String, Value>,
cache: &Option<HashMap<String, String>>,
cache: Option<&HashMap<String, String>>,
fixed: &mut bool,
) {
for dep in dependencies.values_mut() {
@@ -117,31 +113,29 @@ fn fixup_v1_deps(
.as_object()
.expect("v1 dep must be object")
.get("resolved")
{
if let Some(Value::String(integrity)) = dep
&& let Some(Value::String(integrity)) = dep
.as_object()
.expect("v1 dep must be object")
.get("integrity")
{
if resolved.starts_with("git+ssh://") {
{
if resolved.starts_with("git+ssh://") {
*fixed = true;
dep.as_object_mut()
.expect("v1 dep must be object")
.remove("integrity");
} else if let Some(cache_hashes) = cache {
let cache_hash = cache_hashes
.get(resolved)
.expect("dependency should have a hash");
if integrity != cache_hash {
*fixed = true;
dep.as_object_mut()
*dep.as_object_mut()
.expect("v1 dep must be object")
.remove("integrity");
} else if let Some(cache_hashes) = cache {
let cache_hash = cache_hashes
.get(resolved)
.expect("dependency should have a hash");
if integrity != cache_hash {
*fixed = true;
*dep.as_object_mut()
.expect("v1 dep must be object")
.get_mut("integrity")
.unwrap() = Value::String(cache_hash.clone());
}
.get_mut("integrity")
.unwrap() = Value::String(cache_hash.clone());
}
}
}
@@ -185,26 +179,30 @@ fn main() -> anyhow::Result<()> {
process::exit(1);
}
if let Ok(jobs) = env::var("NIX_BUILD_CORES") {
if !jobs.is_empty() {
rayon::ThreadPoolBuilder::new()
.num_threads(
jobs.parse()
.expect("NIX_BUILD_CORES must be a whole number"),
)
.build_global()
.unwrap();
}
if let Ok(jobs) = env::var("NIX_BUILD_CORES")
&& !jobs.is_empty()
{
rayon::ThreadPoolBuilder::new()
.num_threads(
jobs.parse()
.expect("NIX_BUILD_CORES must be a whole number"),
)
.build_global()
.unwrap();
}
if args[1] == "--fixup-lockfile" {
let lock = serde_json::from_str(&fs::read_to_string(&args[2])?)?;
let cache = cache_map_path()
.map(|map_path| Ok::<_, anyhow::Error>(serde_json::from_slice(&fs::read(map_path)?)?))
.map(|map_path| {
Ok::<_, anyhow::Error>(serde_json::from_slice::<HashMap<String, String>>(
&fs::read(map_path)?,
)?)
})
.transpose()?;
if let Some(fixed) = fixup_lockfile(lock, &cache)? {
if let Some(fixed) = fixup_lockfile(lock, cache.as_ref())? {
println!("Fixing lockfile");
fs::write(&args[2], serde_json::to_string(&fixed)?)?;
@@ -345,7 +343,7 @@ mod tests {
hashes.insert(String::from("foo"), String::from("sha512-foo"));
assert_eq!(
fixup_lockfile(input.as_object().unwrap().clone(), &Some(hashes))?,
fixup_lockfile(input.as_object().unwrap().clone(), Some(hashes).as_ref())?,
Some(expected.as_object().unwrap().clone())
);
@@ -418,7 +416,7 @@ mod tests {
hashes.insert(String::from("foo"), String::from("sha512-foo"));
assert_eq!(
fixup_lockfile(input.as_object().unwrap().clone(), &Some(hashes))?,
fixup_lockfile(input.as_object().unwrap().clone(), Some(hashes).as_ref())?,
Some(expected.as_object().unwrap().clone())
);
pkgs/build-support/node/prefetch-npm-deps/src/parse/lock.rs
+7 -7
View File
@@ -1,8 +1,8 @@
use anyhow::{anyhow, bail, Context};
use anyhow::{Context, anyhow, bail};
use rayon::slice::ParallelSliceMut;
use serde::{
de::{self, Visitor},
Deserialize, Deserializer,
de::{self, Visitor},
};
use std::{
cmp::Ordering,
@@ -132,7 +132,7 @@ impl<'de> Deserialize<'de> for HashCollection {
struct HashCollectionVisitor;
impl<'de> Visitor<'de> for HashCollectionVisitor {
impl Visitor<'_> for HashCollectionVisitor {
type Value = HashCollection;
fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result {
@@ -161,7 +161,7 @@ impl Hash {
.ok_or_else(|| anyhow!("expected SRI hash, got {:?}", s.as_ref()))?
.0;
if ALGOS.iter().any(|&a| algo == a) {
if ALGOS.contains(&algo) {
Ok(Hash(s.as_ref().to_string()))
} else {
Err(anyhow!("unknown hash algorithm {algo:?}"))
@@ -215,7 +215,7 @@ fn to_new_packages(
if let UrlOrString::Url(v) = &package.version {
if v.scheme() == "npm" {
if let Some(UrlOrString::Url(ref url)) = &package.resolved {
if let Some(UrlOrString::Url(url)) = &package.resolved {
package.version = UrlOrString::Url(url.clone());
}
} else {
@@ -272,8 +272,8 @@ fn get_initial_url() -> anyhow::Result<Url> {
#[cfg(test)]
mod tests {
use super::{
get_initial_url, packages, to_new_packages, Hash, HashCollection, OldPackage, Package,
UrlOrString,
Hash, HashCollection, OldPackage, Package, UrlOrString, get_initial_url, packages,
to_new_packages,
};
use std::{
cmp::Ordering,
pkgs/build-support/node/prefetch-npm-deps/src/parse/mod.rs
+22 -9
View File
@@ -1,4 +1,4 @@
use anyhow::{anyhow, bail, Context};
use anyhow::{Context, anyhow, bail};
use lock::UrlOrString;
use log::{debug, info};
use rayon::prelude::*;
@@ -8,7 +8,7 @@ use std::{
io::Write,
process::{Command, Stdio},
};
use tempfile::{tempdir, TempDir};
use tempfile::{TempDir, tempdir};
use url::Url;
use crate::util;
@@ -33,7 +33,9 @@ pub fn lockfile(
.collect::<anyhow::Result<Vec<_>>>()?;
if packages.is_empty() && !force_empty_cache {
bail!("No cacheable dependencies were found. Please inspect the upstream `package-lock.json` file and ensure that remote dependencies have `resolved` URLs and `integrity` hashes. If the lockfile is missing this data, attempt to get upstream to fix it via a tool like <https://github.com/jeslie0/npm-lockfile-fix>. If generating an empty cache is intentional and you would like to do it anyways, set `forceEmptyCache = true`.");
bail!(
"No cacheable dependencies were found. Please inspect the upstream `package-lock.json` file and ensure that remote dependencies have `resolved` URLs and `integrity` hashes. If the lockfile is missing this data, attempt to get upstream to fix it via a tool like <https://github.com/jeslie0/npm-lockfile-fix>. If generating an empty cache is intentional and you would like to do it anyways, set `forceEmptyCache = true`."
);
}
let mut new = Vec::new();
@@ -49,7 +51,11 @@ pub fn lockfile(
let path = dir.path().join("package");
info!("recursively parsing lockfile for {} at {path:?}", pkg.name);
info!(
"recursively parsing lockfile for {} at {}",
pkg.name,
path.display()
);
let lockfile_contents = fs::read_to_string(path.join("package-lock.json"));
@@ -71,7 +77,10 @@ pub fn lockfile(
"prepare",
] {
if scripts.contains_key(typ) && lockfile_contents.is_err() && !force_git_deps {
bail!("Git dependency {} contains install scripts, but has no lockfile, which is something that will probably break. Open an issue if you can't feasibly patch this dependency out, and we'll come up with a workaround.\nIf you'd like to attempt to try to use this dependency anyways, set `forceGitDeps = true`.", pkg.name);
bail!(
"Git dependency {} contains install scripts, but has no lockfile, which is something that will probably break. Open an issue if you can't feasibly patch this dependency out, and we'll come up with a workaround.\nIf you'd like to attempt to try to use this dependency anyways, set `forceGitDeps = true`.",
pkg.name
);
}
}
}
@@ -310,7 +319,9 @@ fn get_hosted_git_url(url: &Url) -> anyhow::Result<Option<Url>> {
match get_url() {
Some(u) => Ok(Some(u)),
None => Err(anyhow!("This lockfile either contains a Git dependency with an unsupported host, or a malformed URL in the lockfile: {url}"))
None => Err(anyhow!(
"This lockfile either contains a Git dependency with an unsupported host, or a malformed URL in the lockfile: {url}"
)),
}
} else {
Ok(None)
@@ -327,15 +338,17 @@ mod tests {
for (input, expected) in [
(
"git+ssh://git@github.com/castlabs/electron-releases.git#fc5f78d046e8d7cdeb66345a2633c383ab41f525",
Some("https://codeload.github.com/castlabs/electron-releases/tar.gz/fc5f78d046e8d7cdeb66345a2633c383ab41f525"),
Some(
"https://codeload.github.com/castlabs/electron-releases/tar.gz/fc5f78d046e8d7cdeb66345a2633c383ab41f525",
),
),
(
"git+ssh://bitbucket.org/foo/bar#branch",
Some("https://bitbucket.org/foo/bar/get/branch.tar.gz")
Some("https://bitbucket.org/foo/bar/get/branch.tar.gz"),
),
(
"git+ssh://git.sr.ht/~foo/bar#branch",
Some("https://git.sr.ht/~foo/bar/archive/branch.tar.gz")
Some("https://git.sr.ht/~foo/bar/archive/branch.tar.gz"),
),
] {
assert_eq!(
pkgs/build-support/node/prefetch-npm-deps/src/util.rs
+21 -22
View File
@@ -1,10 +1,10 @@
use anyhow::bail;
use backoff::{retry, ExponentialBackoff};
use backoff::{ExponentialBackoff, retry};
use data_encoding::BASE64;
use digest::Digest;
use isahc::{
config::{CaCertificate, Configurable, RedirectPolicy, SslOption},
Body, Request, RequestExt,
config::{CaCertificate, Configurable, RedirectPolicy, SslOption},
};
use log::info;
use nix_nar::{Encoder, NarError};
@@ -22,18 +22,19 @@ pub fn get_url(url: &Url) -> Result<Body, anyhow::Error> {
let mut url = url.clone();
// Respect NIX_NPM_REGISTRY_OVERRIDES environment variable, which should be a JSON mapping in the shape of:
// `{ "registry.example.com": "my-registry.local", ... }`
if let Some(host) = url.host_str() {
if let Ok(npm_mirrors) = env::var("NIX_NPM_REGISTRY_OVERRIDES") {
if let Ok(mirrors) = serde_json::from_str::<Map<String, Value>>(&npm_mirrors) {
if let Some(mirror) = mirrors.get(host).and_then(serde_json::Value::as_str) {
let mirror_url = Url::parse(mirror)?;
url.set_path(&(mirror_url.path().to_owned() + url.path()));
url.set_host(Some(mirror_url.host_str().expect(format!("Mirror URL without host part: {mirror_url}").as_str())))?;
eprintln!("Replaced URL {url_} with {url}");
}
}
}
if let Some(host) = url.host_str()
&& let Ok(npm_mirrors) = env::var("NIX_NPM_REGISTRY_OVERRIDES")
&& let Ok(mirrors) = serde_json::from_str::<Map<String, Value>>(&npm_mirrors)
&& let Some(mirror) = mirrors.get(host).and_then(serde_json::Value::as_str)
{
let mirror_url = Url::parse(mirror)?;
url.set_path(&(mirror_url.path().to_owned() + url.path()));
url.set_host(Some(mirror_url.host_str().unwrap_or_else(|| {
panic!("Mirror URL without host part: {mirror_url}")
})))?;
eprintln!("Replaced URL {url_} with {url}");
}
let mut request = Request::get(url.as_str()).redirect_policy(RedirectPolicy::Limit(10));
// Respect SSL_CERT_FILE if environment variable exists
@@ -51,15 +52,13 @@ pub fn get_url(url: &Url) -> Result<Body, anyhow::Error> {
// Respect NIX_NPM_TOKENS environment variable, which should be a JSON mapping in the shape of:
// `{ "registry.example.com": "example-registry-bearer-token", ... }`
if let Some(host) = url.host_str() {
if let Ok(npm_tokens) = env::var("NIX_NPM_TOKENS") {
if let Ok(tokens) = serde_json::from_str::<Map<String, Value>>(&npm_tokens) {
if let Some(token) = tokens.get(host).and_then(serde_json::Value::as_str) {
info!("Found NPM token for {}. Adding authorization header to request.", host);
request = request.header("Authorization", format!("Bearer {token}"));
}
}
}
if let Some(host) = url.host_str()
&& let Ok(npm_tokens) = env::var("NIX_NPM_TOKENS")
&& let Ok(tokens) = serde_json::from_str::<Map<String, Value>>(&npm_tokens)
&& let Some(token) = tokens.get(host).and_then(serde_json::Value::as_str)
{
info!("Found NPM token for {host}. Adding authorization header to request.");
request = request.header("Authorization", format!("Bearer {token}"));
}
let res = request.body(())?.send()?;