rustPlatform.{fetchCargoVendor,importCargoLock}: support lockfile v4 escaping (#371795)

pkgs/build-support/rust/fetch-cargo-vendor-util.py

@@ -9,6 +9,7 @@ import sys
 import tomllib
 from pathlib import Path
 from typing import Any, TypedDict, cast
+from urllib.parse import unquote
 
 import requests
 from requests.adapters import HTTPAdapter, Retry
@@ -21,6 +22,15 @@ def load_toml(path: Path) -> dict[str, Any]:
         return tomllib.load(f)
 
 
+def get_lockfile_version(cargo_lock_toml: dict[str, Any]) -> int:
+    # lockfile v1 and v2 don't have the `version` key, so assume v2
+    version = cargo_lock_toml.get("version", 2)
+
+    # TODO: add logic for differentiating between v1 and v2
+
+    return version
+
+
 def download_file_with_checksum(url: str, destination_path: Path) -> str:
     retries = Retry(
         total=5,
@@ -93,20 +103,29 @@ class GitSourceInfo(TypedDict):
     git_sha_rev: str
 
 
-def parse_git_source(source: str) -> GitSourceInfo:
+def parse_git_source(source: str, lockfile_version: int) -> GitSourceInfo:
     match = GIT_SOURCE_REGEX.match(source)
     if match is None:
         raise Exception(f"Unable to process git source: {source}.")
-    return cast(GitSourceInfo, match.groupdict(default=None))
+
+    source_info = cast(GitSourceInfo, match.groupdict(default=None))
+
+    # the source URL is URL-encoded in lockfile_version >=4
+    # since we just used regex to parse it we have to manually decode the escaped branch/tag name
+    if lockfile_version >= 4 and source_info["value"] is not None:
+        source_info["value"] = unquote(source_info["value"])
+
+    return source_info
 
 
 def create_vendor_staging(lockfile_path: Path, out_dir: Path) -> None:
-    cargo_toml = load_toml(lockfile_path)
+    cargo_lock_toml = load_toml(lockfile_path)
+    lockfile_version = get_lockfile_version(cargo_lock_toml)
 
     git_packages: list[dict[str, Any]] = []
     registry_packages: list[dict[str, Any]] = []
 
-    for pkg in cargo_toml["package"]:
+    for pkg in cargo_lock_toml["package"]:
         # ignore local dependenices
         if "source" not in pkg.keys():
             eprint(f"Skipping local dependency: {pkg["name"]}")
@@ -122,7 +141,7 @@ def create_vendor_staging(lockfile_path: Path, out_dir: Path) -> None:
 
     git_sha_rev_to_url: dict[str, str] = {}
     for pkg in git_packages:
-        source_info = parse_git_source(pkg["source"])
+        source_info = parse_git_source(pkg["source"], lockfile_version)
         git_sha_rev_to_url[source_info["git_sha_rev"]] = source_info["url"]
 
     out_dir.mkdir(exist_ok=True)
@@ -207,7 +226,8 @@ def create_vendor(vendor_staging_dir: Path, out_dir: Path) -> None:
     out_dir.mkdir(exist_ok=True)
     shutil.copy(lockfile_path, out_dir / "Cargo.lock")
 
-    cargo_toml = load_toml(lockfile_path)
+    cargo_lock_toml = load_toml(lockfile_path)
+    lockfile_version = get_lockfile_version(cargo_lock_toml)
 
     config_lines = [
         '[source.vendored-sources]',
@@ -217,7 +237,7 @@ def create_vendor(vendor_staging_dir: Path, out_dir: Path) -> None:
     ]
 
     seen_source_keys = set()
-    for pkg in cargo_toml["package"]:
+    for pkg in cargo_lock_toml["package"]:
 
         # ignore local dependenices
         if "source" not in pkg.keys():
@@ -230,7 +250,8 @@ def create_vendor(vendor_staging_dir: Path, out_dir: Path) -> None:
 
         if source.startswith("git+"):
 
-            source_info = parse_git_source(pkg["source"])
+            source_info = parse_git_source(pkg["source"], lockfile_version)
+
             git_sha_rev = source_info["git_sha_rev"]
             git_tree = vendor_staging_dir / "git" / git_sha_rev
 

pkgs/build-support/rust/import-cargo-lock.nix

@@ -49,6 +49,10 @@ let
 
   parsedLockFile = builtins.fromTOML lockFileContents;
 
+  # lockfile v1 and v2 don't have the `version` key, so assume v2
+  # we can implement more fine-grained detection later, if needed
+  lockFileVersion = parsedLockFile.version or 2;
+
   packages = parsedLockFile.package;
 
   # There is no source attribute for the source package itself. But
@@ -202,11 +206,20 @@ let
         # Cargo is happy with empty metadata.
         printf '{"files":{},"package":null}' > "$out/.cargo-checksum.json"
 
+        ${lib.optionalString (gitParts ? type) ''
+          gitPartsValue=${lib.escapeShellArg gitParts.value}
+          # starting with lockfile version v4 the git source url contains encoded query parameters
+          # our regex parser does not know how to unescape them to get the actual value, so we do it here
+          ${lib.optionalString (lockFileVersion >= 4) ''
+            gitPartsValue=$(${lib.getExe python3Packages.python} -c "import sys, urllib.parse; print(urllib.parse.unquote(sys.argv[1]))" "$gitPartsValue")
+          ''}
+        ''}
+
         # Set up configuration for the vendor directory.
         cat > $out/.cargo-config <<EOF
-        [source."${gitParts.url}${lib.optionalString (gitParts ? type) "?${gitParts.type}=${gitParts.value}"}"]
+        [source."${pkg.source}"]
         git = "${gitParts.url}"
-        ${lib.optionalString (gitParts ? type) "${gitParts.type} = \"${gitParts.value}\""}
+        ${lib.optionalString (gitParts ? type) "${gitParts.type} = \"$gitPartsValue\""}
         replace-with = "vendored-sources"
         EOF
       ''