diff --git a/pkgs/build-support/node/prefetch-npm-deps/Cargo.toml b/pkgs/build-support/node/prefetch-npm-deps/Cargo.toml index 976d7c68c26e..39ee48ce9f6e 100644 --- a/pkgs/build-support/node/prefetch-npm-deps/Cargo.toml +++ b/pkgs/build-support/node/prefetch-npm-deps/Cargo.toml @@ -1,7 +1,7 @@ [package] name = "prefetch-npm-deps" version = "0.1.0" -edition = "2021" +edition = "2024" # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html diff --git a/pkgs/build-support/node/prefetch-npm-deps/src/main.rs b/pkgs/build-support/node/prefetch-npm-deps/src/main.rs index 8cc8708211dc..cdac7ef84c0b 100644 --- a/pkgs/build-support/node/prefetch-npm-deps/src/main.rs +++ b/pkgs/build-support/node/prefetch-npm-deps/src/main.rs @@ -38,7 +38,7 @@ fn cache_map_path() -> Option { /// `dependencies` key in v2 lockfiles designed for backwards compatibility with v1 parsers is removed because of inconsistent data. fn fixup_lockfile( mut lock: Map, - cache: &Option>, + cache: Option<&HashMap>, ) -> anyhow::Result>> { let mut fixed = false; @@ -64,29 +64,29 @@ fn fixup_lockfile( .ok_or_else(|| anyhow!("packages isn't a map"))? .values_mut() { - if let Some(Value::String(resolved)) = package.get("resolved") { - if let Some(Value::String(integrity)) = package.get("integrity") { - if resolved.starts_with("git+") { + if let Some(Value::String(resolved)) = package.get("resolved") + && let Some(Value::String(integrity)) = package.get("integrity") + { + if resolved.starts_with("git+") { + fixed = true; + + package + .as_object_mut() + .ok_or_else(|| anyhow!("package isn't a map"))? + .remove("integrity"); + } else if let Some(cache_hashes) = cache { + let cache_hash = cache_hashes + .get(resolved) + .expect("dependency should have a hash"); + + if integrity != cache_hash { fixed = true; - package + *package .as_object_mut() .ok_or_else(|| anyhow!("package isn't a map"))? - .remove("integrity"); - } else if let Some(cache_hashes) = cache { - let cache_hash = cache_hashes - .get(resolved) - .expect("dependency should have a hash"); - - if integrity != cache_hash { - fixed = true; - - *package - .as_object_mut() - .ok_or_else(|| anyhow!("package isn't a map"))? - .get_mut("integrity") - .unwrap() = Value::String(cache_hash.clone()); - } + .get_mut("integrity") + .unwrap() = Value::String(cache_hash.clone()); } } } @@ -99,17 +99,13 @@ fn fixup_lockfile( v => bail!("unsupported lockfile version {v}"), } - if fixed { - Ok(Some(lock)) - } else { - Ok(None) - } + if fixed { Ok(Some(lock)) } else { Ok(None) } } // Recursive helper to fixup v1 lockfile deps fn fixup_v1_deps( dependencies: &mut Map, - cache: &Option>, + cache: Option<&HashMap>, fixed: &mut bool, ) { for dep in dependencies.values_mut() { @@ -117,31 +113,29 @@ fn fixup_v1_deps( .as_object() .expect("v1 dep must be object") .get("resolved") - { - if let Some(Value::String(integrity)) = dep + && let Some(Value::String(integrity)) = dep .as_object() .expect("v1 dep must be object") .get("integrity") - { - if resolved.starts_with("git+ssh://") { + { + if resolved.starts_with("git+ssh://") { + *fixed = true; + + dep.as_object_mut() + .expect("v1 dep must be object") + .remove("integrity"); + } else if let Some(cache_hashes) = cache { + let cache_hash = cache_hashes + .get(resolved) + .expect("dependency should have a hash"); + + if integrity != cache_hash { *fixed = true; - dep.as_object_mut() + *dep.as_object_mut() .expect("v1 dep must be object") - .remove("integrity"); - } else if let Some(cache_hashes) = cache { - let cache_hash = cache_hashes - .get(resolved) - .expect("dependency should have a hash"); - - if integrity != cache_hash { - *fixed = true; - - *dep.as_object_mut() - .expect("v1 dep must be object") - .get_mut("integrity") - .unwrap() = Value::String(cache_hash.clone()); - } + .get_mut("integrity") + .unwrap() = Value::String(cache_hash.clone()); } } } @@ -185,26 +179,30 @@ fn main() -> anyhow::Result<()> { process::exit(1); } - if let Ok(jobs) = env::var("NIX_BUILD_CORES") { - if !jobs.is_empty() { - rayon::ThreadPoolBuilder::new() - .num_threads( - jobs.parse() - .expect("NIX_BUILD_CORES must be a whole number"), - ) - .build_global() - .unwrap(); - } + if let Ok(jobs) = env::var("NIX_BUILD_CORES") + && !jobs.is_empty() + { + rayon::ThreadPoolBuilder::new() + .num_threads( + jobs.parse() + .expect("NIX_BUILD_CORES must be a whole number"), + ) + .build_global() + .unwrap(); } if args[1] == "--fixup-lockfile" { let lock = serde_json::from_str(&fs::read_to_string(&args[2])?)?; let cache = cache_map_path() - .map(|map_path| Ok::<_, anyhow::Error>(serde_json::from_slice(&fs::read(map_path)?)?)) + .map(|map_path| { + Ok::<_, anyhow::Error>(serde_json::from_slice::>( + &fs::read(map_path)?, + )?) + }) .transpose()?; - if let Some(fixed) = fixup_lockfile(lock, &cache)? { + if let Some(fixed) = fixup_lockfile(lock, cache.as_ref())? { println!("Fixing lockfile"); fs::write(&args[2], serde_json::to_string(&fixed)?)?; @@ -345,7 +343,7 @@ mod tests { hashes.insert(String::from("foo"), String::from("sha512-foo")); assert_eq!( - fixup_lockfile(input.as_object().unwrap().clone(), &Some(hashes))?, + fixup_lockfile(input.as_object().unwrap().clone(), Some(hashes).as_ref())?, Some(expected.as_object().unwrap().clone()) ); @@ -418,7 +416,7 @@ mod tests { hashes.insert(String::from("foo"), String::from("sha512-foo")); assert_eq!( - fixup_lockfile(input.as_object().unwrap().clone(), &Some(hashes))?, + fixup_lockfile(input.as_object().unwrap().clone(), Some(hashes).as_ref())?, Some(expected.as_object().unwrap().clone()) ); diff --git a/pkgs/build-support/node/prefetch-npm-deps/src/parse/lock.rs b/pkgs/build-support/node/prefetch-npm-deps/src/parse/lock.rs index 49bba8780c97..17c6c633ee8f 100644 --- a/pkgs/build-support/node/prefetch-npm-deps/src/parse/lock.rs +++ b/pkgs/build-support/node/prefetch-npm-deps/src/parse/lock.rs @@ -1,8 +1,8 @@ -use anyhow::{anyhow, bail, Context}; +use anyhow::{Context, anyhow, bail}; use rayon::slice::ParallelSliceMut; use serde::{ - de::{self, Visitor}, Deserialize, Deserializer, + de::{self, Visitor}, }; use std::{ cmp::Ordering, @@ -132,7 +132,7 @@ impl<'de> Deserialize<'de> for HashCollection { struct HashCollectionVisitor; -impl<'de> Visitor<'de> for HashCollectionVisitor { +impl Visitor<'_> for HashCollectionVisitor { type Value = HashCollection; fn expecting(&self, formatter: &mut fmt::Formatter) -> fmt::Result { @@ -161,7 +161,7 @@ impl Hash { .ok_or_else(|| anyhow!("expected SRI hash, got {:?}", s.as_ref()))? .0; - if ALGOS.iter().any(|&a| algo == a) { + if ALGOS.contains(&algo) { Ok(Hash(s.as_ref().to_string())) } else { Err(anyhow!("unknown hash algorithm {algo:?}")) @@ -215,7 +215,7 @@ fn to_new_packages( if let UrlOrString::Url(v) = &package.version { if v.scheme() == "npm" { - if let Some(UrlOrString::Url(ref url)) = &package.resolved { + if let Some(UrlOrString::Url(url)) = &package.resolved { package.version = UrlOrString::Url(url.clone()); } } else { @@ -272,8 +272,8 @@ fn get_initial_url() -> anyhow::Result { #[cfg(test)] mod tests { use super::{ - get_initial_url, packages, to_new_packages, Hash, HashCollection, OldPackage, Package, - UrlOrString, + Hash, HashCollection, OldPackage, Package, UrlOrString, get_initial_url, packages, + to_new_packages, }; use std::{ cmp::Ordering, diff --git a/pkgs/build-support/node/prefetch-npm-deps/src/parse/mod.rs b/pkgs/build-support/node/prefetch-npm-deps/src/parse/mod.rs index 0bca33f03915..6c54d404f3cb 100644 --- a/pkgs/build-support/node/prefetch-npm-deps/src/parse/mod.rs +++ b/pkgs/build-support/node/prefetch-npm-deps/src/parse/mod.rs @@ -1,4 +1,4 @@ -use anyhow::{anyhow, bail, Context}; +use anyhow::{Context, anyhow, bail}; use lock::UrlOrString; use log::{debug, info}; use rayon::prelude::*; @@ -8,7 +8,7 @@ use std::{ io::Write, process::{Command, Stdio}, }; -use tempfile::{tempdir, TempDir}; +use tempfile::{TempDir, tempdir}; use url::Url; use crate::util; @@ -33,7 +33,9 @@ pub fn lockfile( .collect::>>()?; if packages.is_empty() && !force_empty_cache { - bail!("No cacheable dependencies were found. Please inspect the upstream `package-lock.json` file and ensure that remote dependencies have `resolved` URLs and `integrity` hashes. If the lockfile is missing this data, attempt to get upstream to fix it via a tool like . If generating an empty cache is intentional and you would like to do it anyways, set `forceEmptyCache = true`."); + bail!( + "No cacheable dependencies were found. Please inspect the upstream `package-lock.json` file and ensure that remote dependencies have `resolved` URLs and `integrity` hashes. If the lockfile is missing this data, attempt to get upstream to fix it via a tool like . If generating an empty cache is intentional and you would like to do it anyways, set `forceEmptyCache = true`." + ); } let mut new = Vec::new(); @@ -49,7 +51,11 @@ pub fn lockfile( let path = dir.path().join("package"); - info!("recursively parsing lockfile for {} at {path:?}", pkg.name); + info!( + "recursively parsing lockfile for {} at {}", + pkg.name, + path.display() + ); let lockfile_contents = fs::read_to_string(path.join("package-lock.json")); @@ -71,7 +77,10 @@ pub fn lockfile( "prepare", ] { if scripts.contains_key(typ) && lockfile_contents.is_err() && !force_git_deps { - bail!("Git dependency {} contains install scripts, but has no lockfile, which is something that will probably break. Open an issue if you can't feasibly patch this dependency out, and we'll come up with a workaround.\nIf you'd like to attempt to try to use this dependency anyways, set `forceGitDeps = true`.", pkg.name); + bail!( + "Git dependency {} contains install scripts, but has no lockfile, which is something that will probably break. Open an issue if you can't feasibly patch this dependency out, and we'll come up with a workaround.\nIf you'd like to attempt to try to use this dependency anyways, set `forceGitDeps = true`.", + pkg.name + ); } } } @@ -310,7 +319,9 @@ fn get_hosted_git_url(url: &Url) -> anyhow::Result> { match get_url() { Some(u) => Ok(Some(u)), - None => Err(anyhow!("This lockfile either contains a Git dependency with an unsupported host, or a malformed URL in the lockfile: {url}")) + None => Err(anyhow!( + "This lockfile either contains a Git dependency with an unsupported host, or a malformed URL in the lockfile: {url}" + )), } } else { Ok(None) @@ -327,15 +338,17 @@ mod tests { for (input, expected) in [ ( "git+ssh://git@github.com/castlabs/electron-releases.git#fc5f78d046e8d7cdeb66345a2633c383ab41f525", - Some("https://codeload.github.com/castlabs/electron-releases/tar.gz/fc5f78d046e8d7cdeb66345a2633c383ab41f525"), + Some( + "https://codeload.github.com/castlabs/electron-releases/tar.gz/fc5f78d046e8d7cdeb66345a2633c383ab41f525", + ), ), ( "git+ssh://bitbucket.org/foo/bar#branch", - Some("https://bitbucket.org/foo/bar/get/branch.tar.gz") + Some("https://bitbucket.org/foo/bar/get/branch.tar.gz"), ), ( "git+ssh://git.sr.ht/~foo/bar#branch", - Some("https://git.sr.ht/~foo/bar/archive/branch.tar.gz") + Some("https://git.sr.ht/~foo/bar/archive/branch.tar.gz"), ), ] { assert_eq!( diff --git a/pkgs/build-support/node/prefetch-npm-deps/src/util.rs b/pkgs/build-support/node/prefetch-npm-deps/src/util.rs index 835f77a2f52c..aca0affad824 100644 --- a/pkgs/build-support/node/prefetch-npm-deps/src/util.rs +++ b/pkgs/build-support/node/prefetch-npm-deps/src/util.rs @@ -1,10 +1,10 @@ use anyhow::bail; -use backoff::{retry, ExponentialBackoff}; +use backoff::{ExponentialBackoff, retry}; use data_encoding::BASE64; use digest::Digest; use isahc::{ - config::{CaCertificate, Configurable, RedirectPolicy, SslOption}, Body, Request, RequestExt, + config::{CaCertificate, Configurable, RedirectPolicy, SslOption}, }; use log::info; use nix_nar::{Encoder, NarError}; @@ -22,18 +22,19 @@ pub fn get_url(url: &Url) -> Result { let mut url = url.clone(); // Respect NIX_NPM_REGISTRY_OVERRIDES environment variable, which should be a JSON mapping in the shape of: // `{ "registry.example.com": "my-registry.local", ... }` - if let Some(host) = url.host_str() { - if let Ok(npm_mirrors) = env::var("NIX_NPM_REGISTRY_OVERRIDES") { - if let Ok(mirrors) = serde_json::from_str::>(&npm_mirrors) { - if let Some(mirror) = mirrors.get(host).and_then(serde_json::Value::as_str) { - let mirror_url = Url::parse(mirror)?; - url.set_path(&(mirror_url.path().to_owned() + url.path())); - url.set_host(Some(mirror_url.host_str().expect(format!("Mirror URL without host part: {mirror_url}").as_str())))?; - eprintln!("Replaced URL {url_} with {url}"); - } - } - } + if let Some(host) = url.host_str() + && let Ok(npm_mirrors) = env::var("NIX_NPM_REGISTRY_OVERRIDES") + && let Ok(mirrors) = serde_json::from_str::>(&npm_mirrors) + && let Some(mirror) = mirrors.get(host).and_then(serde_json::Value::as_str) + { + let mirror_url = Url::parse(mirror)?; + url.set_path(&(mirror_url.path().to_owned() + url.path())); + url.set_host(Some(mirror_url.host_str().unwrap_or_else(|| { + panic!("Mirror URL without host part: {mirror_url}") + })))?; + eprintln!("Replaced URL {url_} with {url}"); } + let mut request = Request::get(url.as_str()).redirect_policy(RedirectPolicy::Limit(10)); // Respect SSL_CERT_FILE if environment variable exists @@ -51,15 +52,13 @@ pub fn get_url(url: &Url) -> Result { // Respect NIX_NPM_TOKENS environment variable, which should be a JSON mapping in the shape of: // `{ "registry.example.com": "example-registry-bearer-token", ... }` - if let Some(host) = url.host_str() { - if let Ok(npm_tokens) = env::var("NIX_NPM_TOKENS") { - if let Ok(tokens) = serde_json::from_str::>(&npm_tokens) { - if let Some(token) = tokens.get(host).and_then(serde_json::Value::as_str) { - info!("Found NPM token for {}. Adding authorization header to request.", host); - request = request.header("Authorization", format!("Bearer {token}")); - } - } - } + if let Some(host) = url.host_str() + && let Ok(npm_tokens) = env::var("NIX_NPM_TOKENS") + && let Ok(tokens) = serde_json::from_str::>(&npm_tokens) + && let Some(token) = tokens.get(host).and_then(serde_json::Value::as_str) + { + info!("Found NPM token for {host}. Adding authorization header to request."); + request = request.header("Authorization", format!("Bearer {token}")); } let res = request.body(())?.send()?;