julia.withPackages: improve weak dependency handling

pkgs/development/julia-modules/python/extract_artifacts.py

@@ -47,14 +47,17 @@ def get_archive_derivation(uuid, artifact_name, url, sha256, closure_dependencie
 
             ''"""
   else:
+    # We provide gcc.cc.lib by default in order to get some common libraries
+    # like libquadmath.so. A number of packages expect this to be available and
+    # will give linker errors if it isn't.
     fixup = f"""fixupPhase = let
             libs = lib.concatMap (lib.mapAttrsToList (k: v: v.path))
                                [{" ".join(["uuid-" + x for x in depends_on])}];
             in ''
               find $out -type f -executable -exec \
-                patchelf --set-rpath \$ORIGIN:\$ORIGIN/../lib:${{lib.makeLibraryPath (["$out" glibc] ++ libs ++ (with pkgs; [{" ".join(other_libs)}]))}} {{}} \;
+                patchelf --set-rpath \\$ORIGIN:\\$ORIGIN/../lib:${{lib.makeLibraryPath (["$out" glibc gcc.cc.lib] ++ libs ++ (with pkgs; [{" ".join(other_libs)}]))}} {{}} \\;
               find $out -type f -executable -exec \
-                patchelf --set-interpreter ${{glibc}}/lib/ld-linux-x86-64.so.2 {{}} \;
+                patchelf --set-interpreter ${{glibc}}/lib/ld-linux-x86-64.so.2 {{}} \\;
             ''"""
 
   return f"""stdenv.mkDerivation {{
@@ -145,7 +148,7 @@ def main():
     if is_darwin:
       f.write("{ lib, fetchurl, pkgs, stdenv }:\n\n")
     else:
-      f.write("{ lib, fetchurl, glibc, pkgs, stdenv }:\n\n")
+      f.write("{ lib, fetchurl, gcc, glibc, pkgs, stdenv }:\n\n")
 
     f.write("rec {\n")
 

pkgs/development/julia-modules/python/minimal_registry.py

@@ -24,14 +24,15 @@ with open(desired_packages_path, "r") as f:
 
 uuid_to_versions = defaultdict(list)
 for pkg in desired_packages:
-    uuid_to_versions[pkg["uuid"]].append(pkg["version"])
+  uuid_to_versions[pkg["uuid"]].append(pkg["version"])
 
 with open(dependencies_path, "r") as f:
   uuid_to_store_path = yaml.safe_load(f)
 
 os.makedirs(out_path)
 
-registry = toml.load(registry_path / "Registry.toml")
+full_registry = toml.load(registry_path / "Registry.toml")
+registry = full_registry.copy()
 registry["packages"] = {k: v for k, v in registry["packages"].items() if k in uuid_to_versions}
 
 for (uuid, versions) in uuid_to_versions.items():
@@ -80,20 +81,48 @@ for (uuid, versions) in uuid_to_versions.items():
       if (registry_path / path / f).exists():
         shutil.copy2(registry_path / path / f, out_path / path)
 
-    # Copy the Versions.toml file, trimming down to the versions we care about
+    # Copy the Versions.toml file, trimming down to the versions we care about.
+    # In the case where versions=None, this is a weak dep, and we keep all versions.
     all_versions = toml.load(registry_path / path / "Versions.toml")
-    versions_to_keep = {k: v for k, v in all_versions.items() if k in versions}
+    versions_to_keep = {k: v for k, v in all_versions.items() if k in versions} if versions != None else all_versions
     for k, v in versions_to_keep.items():
       del v["nix-sha256"]
     with open(out_path / path / "Versions.toml", "w") as f:
       toml.dump(versions_to_keep, f)
 
-    # Fill in the local store path for the repo
-    if not uuid in uuid_to_store_path: continue
-    package_toml = toml.load(registry_path / path / "Package.toml")
-    package_toml["repo"] = "file://" + uuid_to_store_path[uuid]
-    with open(out_path / path / "Package.toml", "w") as f:
-      toml.dump(package_toml, f)
+    if versions is None:
+      # This is a weak dep; just grab the whole Package.toml
+      shutil.copy2(registry_path / path / "Package.toml", out_path / path / "Package.toml")
+    elif uuid in uuid_to_store_path:
+      # Fill in the local store path for the repo
+      package_toml = toml.load(registry_path / path / "Package.toml")
+      package_toml["repo"] = "file://" + uuid_to_store_path[uuid]
+      with open(out_path / path / "Package.toml", "w") as f:
+        toml.dump(package_toml, f)
 
+# Look for missing weak deps and include them. This can happen when our initial
+# resolve step finds dependencies, but we fail to resolve them at the project.py
+# stage. Usually this happens because the package that depends on them does so
+# as a weak dep, but doesn't have a Package.toml in its repo making this clear.
+for pkg in desired_packages:
+  for dep in (pkg.get("deps", []) or []):
+    uuid = dep["uuid"]
+    if not uuid in uuid_to_versions:
+      entry = full_registry["packages"].get(uuid)
+      if not entry:
+        print(f"""WARNING: found missing UUID but couldn't resolve it: {uuid}""")
+        continue
+
+      # Add this entry back to the minimal Registry.toml
+      registry["packages"][uuid] = entry
+
+      # Bring over the Package.toml
+      path = Path(entry["path"])
+      if (out_path / path / "Package.toml").exists():
+        continue
+      Path(out_path / path).mkdir(parents=True, exist_ok=True)
+      shutil.copy2(registry_path / path / "Package.toml", out_path / path / "Package.toml")
+
+# Finally, dump the Registry.toml
 with open(out_path / "Registry.toml", "w") as f:
     toml.dump(registry, f)

pkgs/development/julia-modules/python/project.py

@@ -0,0 +1,104 @@
+
+from collections import defaultdict
+import json
+import os
+from pathlib import Path
+import sys
+import toml
+import yaml
+
+
+desired_packages_path = Path(sys.argv[1])
+stdlib_infos_path = Path(sys.argv[2])
+package_overrides = json.loads(sys.argv[3])
+dependencies_path = Path(sys.argv[4])
+out_path = Path(sys.argv[5])
+
+with open(desired_packages_path, "r") as f:
+  desired_packages = yaml.safe_load(f) or []
+
+with open(stdlib_infos_path, "r") as f:
+  stdlib_infos = yaml.safe_load(f) or []
+
+with open(dependencies_path, "r") as f:
+  uuid_to_store_path = yaml.safe_load(f)
+
+result = {
+  "deps": defaultdict(list)
+}
+
+for pkg in desired_packages:
+  if pkg["uuid"] in package_overrides:
+    info = package_overrides[pkg["uuid"]]
+    result["deps"][info["name"]].append({
+      "uuid": pkg["uuid"],
+      "path": info["src"],
+    })
+    continue
+
+  path = uuid_to_store_path.get(pkg["uuid"], None)
+  isStdLib = False
+  if pkg["uuid"] in stdlib_infos["stdlibs"]:
+    path = stdlib_infos["stdlib_root"] + "/" + stdlib_infos["stdlibs"][pkg["uuid"]]["name"]
+    isStdLib = True
+
+  if path:
+    if (Path(path) / "Project.toml").exists():
+      project_toml = toml.load(Path(path) / "Project.toml")
+
+      deps = []
+      weak_deps = project_toml.get("weakdeps", {})
+      extensions = project_toml.get("extensions", {})
+
+      if "deps" in project_toml:
+        # Build up deps for the manifest, excluding weak deps
+        weak_deps_uuids = weak_deps.values()
+        for (dep_name, dep_uuid) in project_toml["deps"].items():
+          if not (dep_uuid in weak_deps_uuids):
+            deps.append(dep_name)
+    else:
+      # Not all projects have a Project.toml. In this case, use the deps we
+      # calculated from the package resolve step. This isn't perfect since it
+      # will fail to properly split out weak deps, but it's better than nothing.
+      print(f"""WARNING: package {pkg["name"]} didn't have a Project.toml in {path}""")
+      deps = [x["name"] for x in pkg.get("deps", [])]
+      weak_deps = {}
+      extensions = {}
+
+    tree_hash = pkg.get("tree_hash", "")
+
+    result["deps"][pkg["name"]].append({
+      "version": pkg["version"],
+      "uuid": pkg["uuid"],
+      "git-tree-sha1": (tree_hash if tree_hash != "nothing" else None) or None,
+      "deps": deps or None,
+      "weakdeps": weak_deps or None,
+      "extensions": extensions or None,
+
+      # We *don't* set "path" here, because then Julia will try to use the
+      # read-only Nix store path instead of cloning to the depot. This will
+      # cause packages like Conda.jl to fail during the Pkg.build() step.
+      #
+      # "path": None if isStdLib else path ,
+    })
+  else:
+    print("WARNING: adding a package that we didn't have a path for, and it doesn't seem to be a stdlib", pkg)
+    result["deps"][pkg["name"]].append({
+      "version": pkg["version"],
+      "uuid": pkg["uuid"],
+      "deps": [x["name"] for x in pkg["deps"]]
+    })
+
+os.makedirs(out_path)
+
+with open(out_path / "Manifest.toml", "w") as f:
+  f.write(f'julia_version = "{stdlib_infos["julia_version"]}"\n')
+  f.write('manifest_format = "2.0"\n\n')
+  toml.dump(result, f)
+
+with open(out_path / "Project.toml", "w") as f:
+  f.write('[deps]\n')
+
+  for pkg in desired_packages:
+    if pkg.get("is_input", False):
+      f.write(f'''{pkg["name"]} = "{pkg["uuid"]}"\n''')

pkgs/development/julia-modules/python/sources_nix.py

@@ -24,7 +24,7 @@ def ensure_version_valid(version):
   Ensure a version string is a valid Julia-parsable version.
   It doesn't really matter what it looks like as it's just used for overrides.
   """
-  return re.sub('[^0-9\.]','', version)
+  return re.sub('[^0-9.]','', version)
 
 with open(out_path, "w") as f:
   f.write("{fetchgit}:\n")
@@ -41,6 +41,9 @@ with open(out_path, "w") as f:
     treehash = "{treehash}";
   }};\n""")
     elif uuid in registry["packages"]:
+      # The treehash is missing for stdlib packages. Don't bother downloading these.
+      if (not ("tree_hash" in pkg)) or pkg["tree_hash"] == "nothing": continue
+
       registry_info = registry["packages"][uuid]
       path = registry_info["path"]
       packageToml = toml.load(registry_path / path / "Package.toml")
@@ -65,7 +68,8 @@ with open(out_path, "w") as f:
     treehash = "{version_to_use["git-tree-sha1"]}";
   }};\n""")
     else:
-      # print("Warning: couldn't figure out what to do with pkg in sources_nix.py", pkg)
+      # This is probably a stdlib
+      # print("WARNING: couldn't figure out what to do with pkg in sources_nix.py", pkg)
       pass
 
   f.write("}")