From 72fdf499ff134faa40a405bbb196f8be262877c7 Mon Sep 17 00:00:00 2001 From: TomaSajt <62384384+TomaSajt@users.noreply.github.com> Date: Wed, 5 Mar 2025 17:17:17 +0100 Subject: [PATCH] rustPlatform.fetchCargoVendor: allow duplicated dependencies This is achieved by having subdirs in the vendor dir --- .../rust/fetch-cargo-vendor-util.py | 119 +++++++++++++----- .../build-support/rust/fetch-cargo-vendor.nix | 1 + 2 files changed, 89 insertions(+), 31 deletions(-) diff --git a/pkgs/build-support/rust/fetch-cargo-vendor-util.py b/pkgs/build-support/rust/fetch-cargo-vendor-util.py index eed3675f3186..3bee7e150c8e 100644 --- a/pkgs/build-support/rust/fetch-cargo-vendor-util.py +++ b/pkgs/build-support/rust/fetch-cargo-vendor-util.py @@ -13,6 +13,7 @@ from typing import Any, TypedDict, cast from urllib.parse import unquote import requests +import tomli_w from requests.adapters import HTTPAdapter, Retry eprint = functools.partial(print, file=sys.stderr) @@ -264,6 +265,21 @@ def extract_crate_tarball_contents(tarball_path: Path, crate_out_dir: Path) -> N subprocess.check_output(cmd) +def make_git_source_selector(source_info: GitSourceInfo) -> dict[str, str]: + selector = {} + selector["git"] = source_info["url"] + if source_info["type"] is not None: + selector[source_info["type"]] = source_info["value"] + return selector + + +def make_registry_source_selector(source: str) -> dict[str, str]: + registry = source[9:] if source.startswith("registry+") else source + selector = {} + selector["registry"] = registry + return selector + + def create_vendor(vendor_staging_dir: Path, out_dir: Path) -> None: lockfile_path = vendor_staging_dir / "Cargo.lock" out_dir.mkdir(exist_ok=True) @@ -272,14 +288,67 @@ def create_vendor(vendor_staging_dir: Path, out_dir: Path) -> None: cargo_lock_toml = load_toml(lockfile_path) lockfile_version = get_lockfile_version(cargo_lock_toml) - config_lines = [ - '[source.vendored-sources]', - 'directory = "@vendor@"', - '[source.crates-io]', - 'replace-with = "vendored-sources"', - ] + source_to_ind: dict[str, str] = {} + source_config = {} + next_registry_ind = 0 + next_git_ind = 0 + + def add_source_replacement( + orig_key: str, + orig_selector: dict[str, str], + vendored_key: str, + vendored_dir: str + ) -> None: + source_config[vendored_key] = {} + source_config[vendored_key]["directory"] = vendored_dir + source_config[orig_key] = orig_selector + source_config[orig_key]["replace-with"] = vendored_key + + # we reserve registry index 0 for crates-io + source_to_ind["registry+https://github.com/rust-lang/crates.io-index"] = "registry-0" + source_to_ind["sparse+https://index.crates.io/"] = "registry-0" + add_source_replacement( + orig_key="crates-io", + orig_selector={}, # there is an internal selector defined for the `crates-io` source + vendored_key="vendored-source-registry-0", + vendored_dir="@vendor@/source-registry-0" + ) + next_registry_ind += 1 + + for pkg in cargo_lock_toml["package"]: + # ignore local dependencies + if "source" not in pkg.keys(): + continue + source: str = pkg["source"] + if source in source_to_ind: + continue + + if source.startswith("git+"): + ind = f"git-{next_git_ind}" + next_git_ind += 1 + source_info = parse_git_source(source, lockfile_version) + selector = make_git_source_selector(source_info) + elif source.startswith("registry+") or source.startswith("sparse+"): + ind = f"registry-{next_registry_ind}" + next_registry_ind += 1 + selector = make_registry_source_selector(source) + else: + raise Exception(f"Can't process source: {source}.") + + source_to_ind[source] = ind + add_source_replacement( + orig_key=f"original-source-{ind}", + orig_selector=selector, + vendored_key=f"vendored-source-{ind}", + vendored_dir=f"@vendor@/source-{ind}" + ) + + config_path = out_dir / ".cargo" / "config.toml" + config_path.parent.mkdir() + + with open(config_path, "wb") as config_file: + tomli_w.dump({"source": source_config}, config_file) - seen_source_keys = set() for pkg in cargo_lock_toml["package"]: # ignore local dependenices @@ -287,13 +356,15 @@ def create_vendor(vendor_staging_dir: Path, out_dir: Path) -> None: continue source: str = pkg["source"] - - dir_name = f"{pkg["name"]}-{pkg["version"]}" - crate_out_dir = out_dir / dir_name + source_ind = source_to_ind[source] + crate_dir_name = f"{pkg["name"]}-{pkg["version"]}" + source_dir_name = f"source-{source_ind}" + crate_out_dir = out_dir / source_dir_name / crate_dir_name + crate_out_dir.parent.mkdir(exist_ok=True) if source.startswith("git+"): - source_info = parse_git_source(pkg["source"], lockfile_version) + source_info = parse_git_source(source, lockfile_version) git_sha_rev = source_info["git_sha_rev"] git_tree = vendor_staging_dir / "git" / git_sha_rev @@ -304,23 +375,13 @@ def create_vendor(vendor_staging_dir: Path, out_dir: Path) -> None: with open(crate_out_dir / ".cargo-checksum.json", "w") as f: json.dump({"files": {}}, f) - source_key = source[0:source.find("#")] - - if source_key in seen_source_keys: - continue - - seen_source_keys.add(source_key) - - config_lines.append(f'[source."{source_key}"]') - config_lines.append(f'git = "{source_info["url"]}"') - if source_info["type"] is not None: - config_lines.append(f'{source_info["type"]} = "{source_info["value"]}"') - config_lines.append('replace-with = "vendored-sources"') - - elif source.startswith("registry+"): - + elif source.startswith("registry+") or source.startswith("sparse+"): filename = f"{pkg["name"]}-{pkg["version"]}.tar.gz" - tarball_path = vendor_staging_dir / "tarballs" / filename + + # TODO: change this when non-crates-io registries are supported + dir_name = "tarballs" + + tarball_path = vendor_staging_dir / dir_name / filename extract_crate_tarball_contents(tarball_path, crate_out_dir) @@ -331,10 +392,6 @@ def create_vendor(vendor_staging_dir: Path, out_dir: Path) -> None: else: raise Exception(f"Can't process source: {source}.") - (out_dir / ".cargo").mkdir() - with open(out_dir / ".cargo" / "config.toml", "w") as config_file: - config_file.writelines(line + "\n" for line in config_lines) - def main() -> None: subcommand = sys.argv[1] diff --git a/pkgs/build-support/rust/fetch-cargo-vendor.nix b/pkgs/build-support/rust/fetch-cargo-vendor.nix index 15b6214a4ae3..43a56c8e30cb 100644 --- a/pkgs/build-support/rust/fetch-cargo-vendor.nix +++ b/pkgs/build-support/rust/fetch-cargo-vendor.nix @@ -26,6 +26,7 @@ let with python3Packages; [ requests + tomli-w ] ++ requests.optional-dependencies.socks; # to support socks proxy envs like ALL_PROXY in requests flakeIgnore = [