julia.withPackages: init on supported Julias (1.6, 1.8, 1.9)

Be able to build arbitrary Julia environments in Nixpkgs, in the same style as python.withPackages.

pkgs/development/julia-modules/python/dag.py

@@ -0,0 +1,59 @@
+
+# This file based on a ChatGPT reponse for the following prompt:
+# "can you write code in python to build up a DAG representing
+# a dependency tree, and then a function that can return all the
+# dependencies of a given node?"
+
+class Node:
+    def __init__(self, name):
+        self.name = name
+        self.dependencies = set()
+
+
+class DAG:
+    def __init__(self):
+        self.nodes = {}
+
+    def add_node(self, node_name, dependencies=None):
+        if node_name in self.nodes:
+            raise ValueError(f"Node '{node_name}' already exists in the graph.")
+
+        node = Node(node_name)
+        if dependencies:
+            node.dependencies.update(dependencies)
+
+        self.nodes[node_name] = node
+
+    def add_dependency(self, node_name, dependency_name):
+        if node_name not in self.nodes:
+            raise ValueError(f"Node '{node_name}' does not exist in the graph.")
+
+        if dependency_name not in self.nodes:
+            raise ValueError(f"Dependency '{dependency_name}' does not exist in the graph.")
+
+        self.nodes[node_name].dependencies.add(dependency_name)
+
+    def get_dependencies(self, node_name):
+        if node_name not in self.nodes:
+            raise ValueError(f"Node '{node_name}' does not exist in the graph.")
+
+        node = self.nodes[node_name]
+        dependencies = set()
+
+        def traverse_dependencies(current_node):
+            for dependency in current_node.dependencies:
+                dependencies.add(dependency)
+                if dependency in self.nodes:
+                  traverse_dependencies(self.nodes[dependency])
+
+        traverse_dependencies(node)
+        return dependencies
+
+    def has_node(self, node_name):
+        return node_name in self.nodes
+
+    def __str__(self):
+        graph_str = ""
+        for node_name, node in self.nodes.items():
+            graph_str += f"{node_name} -> {', '.join(node.dependencies)}\n"
+        return graph_str

pkgs/development/julia-modules/python/dedup_overrides.py

@@ -0,0 +1,14 @@
+
+import json
+from pathlib import Path
+import sys
+import toml
+
+overrides_path = Path(sys.argv[1])
+out_path = Path(sys.argv[2])
+
+with open(overrides_path, "r") as f:
+  overrides = json.loads(f.read())
+
+with open(out_path, "w") as f:
+  toml.dump(overrides, f)

pkgs/development/julia-modules/python/extract_artifacts.py

@@ -0,0 +1,99 @@
+
+import json
+from pathlib import Path
+import multiprocessing
+import subprocess
+import sys
+import toml
+import yaml
+
+import dag
+
+dependencies_path = Path(sys.argv[1])
+closure_yaml_path = Path(sys.argv[2])
+julia_path = Path(sys.argv[3])
+extract_artifacts_script = Path(sys.argv[4])
+extra_libs = json.loads(sys.argv[5])
+out_path = Path(sys.argv[6])
+
+with open(dependencies_path, "r") as f:
+  dependencies = yaml.safe_load(f)
+  dependency_uuids = dependencies.keys()
+
+with open(closure_yaml_path, "r") as f:
+  # Build up a map of UUID -> closure information
+  closure_yaml_list = yaml.safe_load(f) or []
+  closure_yaml = {}
+  for item in closure_yaml_list:
+    closure_yaml[item["uuid"]] = item
+
+  # Build up a dependency graph of UUIDs
+  closure_dependencies_dag = dag.DAG()
+  for uuid, contents in closure_yaml.items():
+    if contents.get("depends_on"):
+      closure_dependencies_dag.add_node(uuid, dependencies=contents["depends_on"].values())
+
+with open(out_path, "w") as f:
+  f.write("{ lib, fetchurl, glibc, pkgs, stdenv }:\n\n")
+  f.write("rec {\n")
+
+  def process_item(item):
+    uuid, src = item
+    lines = []
+    artifacts = toml.loads(subprocess.check_output([julia_path, extract_artifacts_script, uuid, src]).decode())
+    if not artifacts: return f'  uuid-{uuid} = {{}};\n'
+
+    lines.append(f'  uuid-{uuid} = {{')
+
+    for artifact_name, details in artifacts.items():
+      if len(details["download"]) == 0: continue
+      download = details["download"][0]
+      url = download["url"]
+      sha256 = download["sha256"]
+
+      git_tree_sha1 = details["git-tree-sha1"]
+
+      depends_on = set()
+      if closure_dependencies_dag.has_node(uuid):
+        depends_on = set(closure_dependencies_dag.get_dependencies(uuid)).intersection(dependency_uuids)
+
+      other_libs = extra_libs.get(uuid, [])
+
+      fixup = f"""fixupPhase = let
+          libs = lib.concatMap (lib.mapAttrsToList (k: v: v.path))
+                               [{" ".join(["uuid-" + x for x in depends_on])}];
+          in ''
+            find $out -type f -executable -exec \
+              patchelf --set-rpath \$ORIGIN:\$ORIGIN/../lib:${{lib.makeLibraryPath (["$out" glibc] ++ libs ++ (with pkgs; [{" ".join(other_libs)}]))}} {{}} \;
+            find $out -type f -executable -exec \
+              patchelf --set-interpreter ${{glibc}}/lib/ld-linux-x86-64.so.2 {{}} \;
+          ''"""
+
+      derivation = f"""{{
+        name = "{artifact_name}";
+        src = fetchurl {{
+          url = "{url}";
+          sha256 = "{sha256}";
+        }};
+        sourceRoot = ".";
+        dontConfigure = true;
+        dontBuild = true;
+        installPhase = "cp -r . $out";
+        {fixup};
+      }}"""
+
+      lines.append(f"""    "{artifact_name}" = {{
+      sha1 = "{git_tree_sha1}";
+      path = stdenv.mkDerivation {derivation};
+    }};\n""")
+
+    lines.append('  };\n')
+
+    return "\n".join(lines)
+
+  with multiprocessing.Pool(10) as pool:
+    for s in pool.map(process_item, dependencies.items()):
+      f.write(s)
+
+  f.write(f"""
+}}\n""")

pkgs/development/julia-modules/python/find_package_implications.py

@@ -0,0 +1,24 @@
+
+import json
+import os
+from pathlib import Path
+import subprocess
+import sys
+import yaml
+
+dependencies_path = Path(sys.argv[1])
+package_implications_json = sys.argv[2]
+out_path = Path(sys.argv[3])
+
+package_implications = json.loads(package_implications_json)
+with open(dependencies_path) as f:
+  desired_packages = yaml.safe_load(f) or []
+
+extra_package_names = []
+for pkg in desired_packages:
+  if pkg["name"] in package_implications:
+    extra_package_names.extend(package_implications[pkg["name"]])
+
+if len(extra_package_names) > 0:
+  with open(out_path, "w") as f:
+    f.write("\n".join(extra_package_names))

pkgs/development/julia-modules/python/format_overrides.py

@@ -0,0 +1,22 @@
+
+import json
+from pathlib import Path
+import sys
+import toml
+
+overrides_path = Path(sys.argv[1])
+out_path = Path(sys.argv[2])
+
+with open(overrides_path, "r") as f:
+  overrides = json.loads(f.read())
+
+result = {}
+
+for (uuid, artifacts) in overrides.items():
+  if len(artifacts) == 0: continue
+
+  for (name, info) in artifacts.items():
+    result[info["sha1"]] = info["path"]
+
+with open(out_path, "w") as f:
+  toml.dump(result, f)

pkgs/development/julia-modules/python/minimal_registry.py

@@ -0,0 +1,98 @@
+
+from collections import defaultdict
+import copy
+import json
+import os
+from pathlib import Path
+import shutil
+import subprocess
+import sys
+import tempfile
+import toml
+import util
+import yaml
+
+
+registry_path = Path(sys.argv[1])
+desired_packages_path = Path(sys.argv[2])
+package_overrides = json.loads(sys.argv[3])
+dependencies_path = Path(sys.argv[4])
+out_path = Path(sys.argv[5])
+
+with open(desired_packages_path, "r") as f:
+  desired_packages = yaml.safe_load(f) or []
+
+uuid_to_versions = defaultdict(list)
+for pkg in desired_packages:
+    uuid_to_versions[pkg["uuid"]].append(pkg["version"])
+
+with open(dependencies_path, "r") as f:
+  uuid_to_store_path = yaml.safe_load(f)
+
+os.makedirs(out_path)
+
+registry = toml.load(registry_path / "Registry.toml")
+registry["packages"] = {k: v for k, v in registry["packages"].items() if k in uuid_to_versions}
+
+for (uuid, versions) in uuid_to_versions.items():
+  if uuid in package_overrides:
+    info = package_overrides[uuid]
+
+    # Make a registry entry based on the info from the package override
+    path = Path(info["name"][0].upper()) / Path(info["name"])
+    registry["packages"][uuid] = {
+      "name": info["name"],
+      "path": str(path),
+    }
+
+    os.makedirs(out_path / path)
+
+    # Read the Project.yaml from the src
+    project = toml.load(Path(info["src"]) / "Project.toml")
+
+    # Generate all the registry files
+    with open(out_path / path / Path("Compat.toml"), "w") as f:
+      f.write('["%s"]\n' % info["version"])
+      # Write nothing in Compat.toml, because we've already resolved everything
+    with open(out_path / path / Path("Deps.toml"), "w") as f:
+      f.write('["%s"]\n' % info["version"])
+      toml.dump(project["deps"], f)
+    with open(out_path / path / Path("Versions.toml"), "w") as f:
+      f.write('["%s"]\n' % info["version"])
+      f.write('git-tree-sha1 = "%s"\n' % info["treehash"])
+    with open(out_path / path / Path("Package.toml"), "w") as f:
+      toml.dump({
+        "name": info["name"],
+        "uuid": uuid,
+        "repo": "file://" + info["src"],
+      }, f)
+
+  elif uuid in registry["packages"]:
+    registry_info = registry["packages"][uuid]
+    name = registry_info["name"]
+    path = registry_info["path"]
+
+    os.makedirs(out_path / path)
+
+    # Copy some files to the minimal repo unchanged
+    for f in ["Compat.toml", "Deps.toml"]:
+      if (registry_path / path / f).exists():
+        shutil.copy2(registry_path / path / f, out_path / path)
+
+    # Copy the Versions.toml file, trimming down to the versions we care about
+    all_versions = toml.load(registry_path / path / "Versions.toml")
+    versions_to_keep = {k: v for k, v in all_versions.items() if k in versions}
+    for k, v in versions_to_keep.items():
+      del v["nix-sha256"]
+    with open(out_path / path / "Versions.toml", "w") as f:
+      toml.dump(versions_to_keep, f)
+
+    # Fill in the local store path for the repo
+    if not uuid in uuid_to_store_path: continue
+    package_toml = toml.load(registry_path / path / "Package.toml")
+    package_toml["repo"] = "file://" + uuid_to_store_path[uuid]
+    with open(out_path / path / "Package.toml", "w") as f:
+      toml.dump(package_toml, f)
+
+with open(out_path / "Registry.toml", "w") as f:
+    toml.dump(registry, f)

pkgs/development/julia-modules/python/sources_nix.py

@@ -0,0 +1,67 @@
+
+import json
+from pathlib import Path
+import re
+import shutil
+import sys
+import toml
+import util
+import yaml
+
+
+registry_path = Path(sys.argv[1])
+package_overrides = json.loads(sys.argv[2])
+desired_packages_path = Path(sys.argv[3])
+out_path = Path(sys.argv[4])
+
+with open(desired_packages_path, "r") as f:
+  desired_packages = yaml.safe_load(f) or []
+
+registry = toml.load(registry_path / "Registry.toml")
+
+def ensure_version_valid(version):
+  """
+  Ensure a version string is a valid Julia-parsable version.
+  It doesn't really matter what it looks like as it's just used for overrides.
+  """
+  return re.sub('[^0-9\.]','', version)
+
+with open(out_path, "w") as f:
+  f.write("{fetchgit}:\n")
+  f.write("{\n")
+  for pkg in desired_packages:
+    uuid = pkg["uuid"]
+
+    if pkg["name"] in package_overrides:
+      treehash = util.get_commit_info(package_overrides[pkg["name"]])["tree"]
+      f.write(f"""  "{uuid}" = {{
+    src = null; # Overridden: will fill in later
+    name = "{pkg["name"]}";
+    version = "{ensure_version_valid(pkg["version"])}";
+    treehash = "{treehash}";
+  }};\n""")
+    elif uuid in registry["packages"]:
+      registry_info = registry["packages"][uuid]
+      path = registry_info["path"]
+      packageToml = toml.load(registry_path / path / "Package.toml")
+
+      all_versions = toml.load(registry_path / path / "Versions.toml")
+      if not pkg["version"] in all_versions: continue
+      version_to_use = all_versions[pkg["version"]]
+
+      repo = packageToml["repo"]
+      f.write(f"""  "{uuid}" = {{
+    src = fetchgit {{
+      url = "{repo}";
+      rev = "{version_to_use["git-tree-sha1"]}";
+      sha256 = "{version_to_use["nix-sha256"]}";
+    }};
+    name = "{pkg["name"]}";
+    version = "{pkg["version"]}";
+    treehash = "{version_to_use["git-tree-sha1"]}";
+  }};\n""")
+    else:
+      # print("Warning: couldn't figure out what to do with pkg in sources_nix.py", pkg)
+      pass
+
+  f.write("}")

pkgs/development/julia-modules/python/util.py

@@ -0,0 +1,12 @@
+
+import os
+import subprocess
+import tempfile
+
+def get_commit_info(repo):
+  with tempfile.TemporaryDirectory() as home_dir:
+    env_with_home = os.environ.copy()
+    env_with_home["HOME"] = home_dir
+    subprocess.check_output(["git", "config", "--global", "--add", "safe.directory", repo], env=env_with_home)
+    lines = subprocess.check_output(["git", "log", "--pretty=raw"], cwd=repo, env=env_with_home).decode().split("\n")
+    return dict([x.split() for x in lines if len(x.split()) == 2])