From 1db27e9394165ef0b15b35cbfadedb35a3f1cda8 Mon Sep 17 00:00:00 2001 From: TomaSajt <62384384+TomaSajt@users.noreply.github.com> Date: Tue, 7 Jan 2025 14:01:12 +0100 Subject: [PATCH 1/2] rustPlatform.fetchCargoVendor: support lockfile v4 escaping --- .../rust/fetch-cargo-vendor-util.py | 37 +++++++++++++++---- 1 file changed, 29 insertions(+), 8 deletions(-) diff --git a/pkgs/build-support/rust/fetch-cargo-vendor-util.py b/pkgs/build-support/rust/fetch-cargo-vendor-util.py index 697387f292ba..bb70a559fe96 100644 --- a/pkgs/build-support/rust/fetch-cargo-vendor-util.py +++ b/pkgs/build-support/rust/fetch-cargo-vendor-util.py @@ -9,6 +9,7 @@ import sys import tomllib from pathlib import Path from typing import Any, TypedDict, cast +from urllib.parse import unquote import requests from requests.adapters import HTTPAdapter, Retry @@ -21,6 +22,15 @@ def load_toml(path: Path) -> dict[str, Any]: return tomllib.load(f) +def get_lockfile_version(cargo_lock_toml: dict[str, Any]) -> int: + # lockfile v1 and v2 don't have the `version` key, so assume v2 + version = cargo_lock_toml.get("version", 2) + + # TODO: add logic for differentiating between v1 and v2 + + return version + + def download_file_with_checksum(url: str, destination_path: Path) -> str: retries = Retry( total=5, @@ -93,20 +103,29 @@ class GitSourceInfo(TypedDict): git_sha_rev: str -def parse_git_source(source: str) -> GitSourceInfo: +def parse_git_source(source: str, lockfile_version: int) -> GitSourceInfo: match = GIT_SOURCE_REGEX.match(source) if match is None: raise Exception(f"Unable to process git source: {source}.") - return cast(GitSourceInfo, match.groupdict(default=None)) + + source_info = cast(GitSourceInfo, match.groupdict(default=None)) + + # the source URL is URL-encoded in lockfile_version >=4 + # since we just used regex to parse it we have to manually decode the escaped branch/tag name + if lockfile_version >= 4 and source_info["value"] is not None: + source_info["value"] = unquote(source_info["value"]) + + return source_info def create_vendor_staging(lockfile_path: Path, out_dir: Path) -> None: - cargo_toml = load_toml(lockfile_path) + cargo_lock_toml = load_toml(lockfile_path) + lockfile_version = get_lockfile_version(cargo_lock_toml) git_packages: list[dict[str, Any]] = [] registry_packages: list[dict[str, Any]] = [] - for pkg in cargo_toml["package"]: + for pkg in cargo_lock_toml["package"]: # ignore local dependenices if "source" not in pkg.keys(): eprint(f"Skipping local dependency: {pkg["name"]}") @@ -122,7 +141,7 @@ def create_vendor_staging(lockfile_path: Path, out_dir: Path) -> None: git_sha_rev_to_url: dict[str, str] = {} for pkg in git_packages: - source_info = parse_git_source(pkg["source"]) + source_info = parse_git_source(pkg["source"], lockfile_version) git_sha_rev_to_url[source_info["git_sha_rev"]] = source_info["url"] out_dir.mkdir(exist_ok=True) @@ -207,7 +226,8 @@ def create_vendor(vendor_staging_dir: Path, out_dir: Path) -> None: out_dir.mkdir(exist_ok=True) shutil.copy(lockfile_path, out_dir / "Cargo.lock") - cargo_toml = load_toml(lockfile_path) + cargo_lock_toml = load_toml(lockfile_path) + lockfile_version = get_lockfile_version(cargo_lock_toml) config_lines = [ '[source.vendored-sources]', @@ -217,7 +237,7 @@ def create_vendor(vendor_staging_dir: Path, out_dir: Path) -> None: ] seen_source_keys = set() - for pkg in cargo_toml["package"]: + for pkg in cargo_lock_toml["package"]: # ignore local dependenices if "source" not in pkg.keys(): @@ -230,7 +250,8 @@ def create_vendor(vendor_staging_dir: Path, out_dir: Path) -> None: if source.startswith("git+"): - source_info = parse_git_source(pkg["source"]) + source_info = parse_git_source(pkg["source"], lockfile_version) + git_sha_rev = source_info["git_sha_rev"] git_tree = vendor_staging_dir / "git" / git_sha_rev From 0bfdb03af347712c4e110174f64f92313e3d84e0 Mon Sep 17 00:00:00 2001 From: TomaSajt <62384384+TomaSajt@users.noreply.github.com> Date: Tue, 7 Jan 2025 21:03:55 +0100 Subject: [PATCH 2/2] rustPlatform.importCargoLock: support lockfile v4 escaping --- pkgs/build-support/rust/import-cargo-lock.nix | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/pkgs/build-support/rust/import-cargo-lock.nix b/pkgs/build-support/rust/import-cargo-lock.nix index e88931d0f383..b39b56f4ccb5 100644 --- a/pkgs/build-support/rust/import-cargo-lock.nix +++ b/pkgs/build-support/rust/import-cargo-lock.nix @@ -49,6 +49,10 @@ let parsedLockFile = builtins.fromTOML lockFileContents; + # lockfile v1 and v2 don't have the `version` key, so assume v2 + # we can implement more fine-grained detection later, if needed + lockFileVersion = parsedLockFile.version or 2; + packages = parsedLockFile.package; # There is no source attribute for the source package itself. But @@ -202,11 +206,20 @@ let # Cargo is happy with empty metadata. printf '{"files":{},"package":null}' > "$out/.cargo-checksum.json" + ${lib.optionalString (gitParts ? type) '' + gitPartsValue=${lib.escapeShellArg gitParts.value} + # starting with lockfile version v4 the git source url contains encoded query parameters + # our regex parser does not know how to unescape them to get the actual value, so we do it here + ${lib.optionalString (lockFileVersion >= 4) '' + gitPartsValue=$(${lib.getExe python3Packages.python} -c "import sys, urllib.parse; print(urllib.parse.unquote(sys.argv[1]))" "$gitPartsValue") + ''} + ''} + # Set up configuration for the vendor directory. cat > $out/.cargo-config <