From 1c1dfd06c28f720ad4b4191b33c229c0c99c655b Mon Sep 17 00:00:00 2001 From: Tom McLaughlin Date: Wed, 27 Nov 2024 04:39:58 -0800 Subject: [PATCH 1/3] julia.withPackages: fix multiprocessing usage to work on macOS --- .../julia-modules/python/extract_artifacts.py | 117 ++++++++++-------- 1 file changed, 65 insertions(+), 52 deletions(-) diff --git a/pkgs/development/julia-modules/python/extract_artifacts.py b/pkgs/development/julia-modules/python/extract_artifacts.py index 4ac450d88a10..0ebe167aa7e3 100755 --- a/pkgs/development/julia-modules/python/extract_artifacts.py +++ b/pkgs/development/julia-modules/python/extract_artifacts.py @@ -32,31 +32,7 @@ archive_extensions = [ ".zip" ] -dependencies_path = Path(sys.argv[1]) -closure_yaml_path = Path(sys.argv[2]) -julia_path = Path(sys.argv[3]) -extract_artifacts_script = Path(sys.argv[4]) -extra_libs = json.loads(sys.argv[5]) -out_path = Path(sys.argv[6]) - -with open(dependencies_path, "r") as f: - dependencies = yaml.safe_load(f) - dependency_uuids = dependencies.keys() - -with open(closure_yaml_path, "r") as f: - # Build up a map of UUID -> closure information - closure_yaml_list = yaml.safe_load(f) or [] - closure_yaml = {} - for item in closure_yaml_list: - closure_yaml[item["uuid"]] = item - - # Build up a dependency graph of UUIDs - closure_dependencies_dag = dag.DAG() - for uuid, contents in closure_yaml.items(): - if contents.get("depends_on"): - closure_dependencies_dag.add_node(uuid, dependencies=contents["depends_on"].values()) - -def get_archive_derivation(uuid, artifact_name, url, sha256): +def get_archive_derivation(uuid, artifact_name, url, sha256, closure_dependencies_dag, dependency_uuids, extra_libs): depends_on = set() if closure_dependencies_dag.has_node(uuid): depends_on = set(closure_dependencies_dag.get_dependencies(uuid)).intersection(dependency_uuids) @@ -96,44 +72,81 @@ def get_plain_derivation(url, sha256): sha256 = "{sha256}"; }}""" -with open(out_path, "w") as f: - f.write("{ lib, fetchurl, glibc, pkgs, stdenv }:\n\n") - f.write("rec {\n") +def process_item(args): + item, julia_path, extract_artifacts_script, closure_dependencies_dag, dependency_uuids, extra_libs = args + uuid, src = item + lines = [] - def process_item(item): - uuid, src = item - lines = [] - artifacts = toml.loads(subprocess.check_output([julia_path, extract_artifacts_script, uuid, src]).decode()) - if not artifacts: return f' uuid-{uuid} = {{}};\n' + artifacts = toml.loads(subprocess.check_output([julia_path, extract_artifacts_script, uuid, src]).decode()) + if not artifacts: + return f' uuid-{uuid} = {{}};\n' - lines.append(f' uuid-{uuid} = {{') + lines.append(f' uuid-{uuid} = {{') - for artifact_name, details in artifacts.items(): - if len(details["download"]) == 0: continue - download = details["download"][0] - url = download["url"] - sha256 = download["sha256"] + for artifact_name, details in artifacts.items(): + if len(details["download"]) == 0: + continue + download = details["download"][0] + url = download["url"] + sha256 = download["sha256"] - git_tree_sha1 = details["git-tree-sha1"] + git_tree_sha1 = details["git-tree-sha1"] - parsed_url = urlparse(url) - if any(parsed_url.path.endswith(x) for x in archive_extensions): - derivation = get_archive_derivation(uuid, artifact_name, url, sha256) - else: - derivation = get_plain_derivation(url, sha256) + parsed_url = urlparse(url) + if any(parsed_url.path.endswith(x) for x in archive_extensions): + derivation = get_archive_derivation(uuid, artifact_name, url, sha256, closure_dependencies_dag, dependency_uuids, extra_libs) + else: + derivation = get_plain_derivation(url, sha256) - lines.append(f""" "{artifact_name}" = {{ + lines.append(f""" "{artifact_name}" = {{ sha1 = "{git_tree_sha1}"; path = {derivation}; }};\n""") - lines.append(' };\n') + lines.append(' };\n') - return "\n".join(lines) + return "\n".join(lines) - with multiprocessing.Pool(10) as pool: - for s in pool.map(process_item, dependencies.items()): - f.write(s) +def main(): + dependencies_path = Path(sys.argv[1]) + closure_yaml_path = Path(sys.argv[2]) + julia_path = Path(sys.argv[3]) + extract_artifacts_script = Path(sys.argv[4]) + extra_libs = json.loads(sys.argv[5]) + out_path = Path(sys.argv[6]) - f.write(f""" + with open(dependencies_path, "r") as f: + dependencies = yaml.safe_load(f) + dependency_uuids = list(dependencies.keys()) # Convert dict_keys to list + + with open(closure_yaml_path, "r") as f: + # Build up a map of UUID -> closure information + closure_yaml_list = yaml.safe_load(f) or [] + closure_yaml = {} + for item in closure_yaml_list: + closure_yaml[item["uuid"]] = item + + # Build up a dependency graph of UUIDs + closure_dependencies_dag = dag.DAG() + for uuid, contents in closure_yaml.items(): + if contents.get("depends_on"): + closure_dependencies_dag.add_node(uuid, dependencies=contents["depends_on"].values()) + + with open(out_path, "w") as f: + f.write("{ lib, fetchurl, glibc, pkgs, stdenv }:\n\n") + f.write("rec {\n") + + with multiprocessing.Pool(10) as pool: + # Create args tuples for each item + process_args = [ + (item, julia_path, extract_artifacts_script, closure_dependencies_dag, dependency_uuids, extra_libs) + for item in dependencies.items() + ] + for s in pool.map(process_item, process_args): + f.write(s) + + f.write(f""" }}\n""") + +if __name__ == "__main__": + main() From c31efe91504527031a5deb261cd606ccb0a38324 Mon Sep 17 00:00:00 2001 From: Tom McLaughlin Date: Thu, 28 Nov 2024 11:32:37 -0800 Subject: [PATCH 2/3] julia.withPackages: fix artifact fixupPhase on darwin --- pkgs/development/julia-modules/default.nix | 7 +++- .../julia-modules/python/extract_artifacts.py | 41 ++++++++++++------- 2 files changed, 33 insertions(+), 15 deletions(-) diff --git a/pkgs/development/julia-modules/default.nix b/pkgs/development/julia-modules/default.nix index 737cb9345a4b..07f17dc206f3 100644 --- a/pkgs/development/julia-modules/default.nix +++ b/pkgs/development/julia-modules/default.nix @@ -134,11 +134,16 @@ let "${juliaWrapped}/bin/julia" \ "${if lib.versionAtLeast julia.version "1.7" then ./extract_artifacts.jl else ./extract_artifacts_16.jl}" \ '${lib.generators.toJSON {} (import ./extra-libs.nix)}' \ + '${lib.generators.toJSON {} (stdenv.hostPlatform.isDarwin)}' \ "$out" ''; # Import the artifacts Nix to build Overrides.toml (IFD) - artifacts = import artifactsNix { inherit lib fetchurl pkgs glibc stdenv; }; + artifacts = import artifactsNix ({ + inherit lib fetchurl pkgs stdenv; + } // lib.optionalAttrs (!stdenv.targetPlatform.isDarwin) { + inherit glibc; + }); overridesJson = writeTextFile { name = "Overrides.json"; text = lib.generators.toJSON {} artifacts; diff --git a/pkgs/development/julia-modules/python/extract_artifacts.py b/pkgs/development/julia-modules/python/extract_artifacts.py index 0ebe167aa7e3..134294321f26 100755 --- a/pkgs/development/julia-modules/python/extract_artifacts.py +++ b/pkgs/development/julia-modules/python/extract_artifacts.py @@ -32,22 +32,30 @@ archive_extensions = [ ".zip" ] -def get_archive_derivation(uuid, artifact_name, url, sha256, closure_dependencies_dag, dependency_uuids, extra_libs): +def get_archive_derivation(uuid, artifact_name, url, sha256, closure_dependencies_dag, dependency_uuids, extra_libs, is_darwin): depends_on = set() if closure_dependencies_dag.has_node(uuid): depends_on = set(closure_dependencies_dag.get_dependencies(uuid)).intersection(dependency_uuids) other_libs = extra_libs.get(uuid, []) - fixup = f"""fixupPhase = let - libs = lib.concatMap (lib.mapAttrsToList (k: v: v.path)) + if is_darwin: + fixup = f"""fixupPhase = let + libs = lib.concatMap (lib.mapAttrsToList (k: v: v.path)) [{" ".join(["uuid-" + x for x in depends_on])}]; - in '' - find $out -type f -executable -exec \ - patchelf --set-rpath \$ORIGIN:\$ORIGIN/../lib:${{lib.makeLibraryPath (["$out" glibc] ++ libs ++ (with pkgs; [{" ".join(other_libs)}]))}} {{}} \; - find $out -type f -executable -exec \ - patchelf --set-interpreter ${{glibc}}/lib/ld-linux-x86-64.so.2 {{}} \; - ''""" + in '' + + ''""" + else: + fixup = f"""fixupPhase = let + libs = lib.concatMap (lib.mapAttrsToList (k: v: v.path)) + [{" ".join(["uuid-" + x for x in depends_on])}]; + in '' + find $out -type f -executable -exec \ + patchelf --set-rpath \$ORIGIN:\$ORIGIN/../lib:${{lib.makeLibraryPath (["$out" glibc] ++ libs ++ (with pkgs; [{" ".join(other_libs)}]))}} {{}} \; + find $out -type f -executable -exec \ + patchelf --set-interpreter ${{glibc}}/lib/ld-linux-x86-64.so.2 {{}} \; + ''""" return f"""stdenv.mkDerivation {{ name = "{artifact_name}"; @@ -73,7 +81,7 @@ def get_plain_derivation(url, sha256): }}""" def process_item(args): - item, julia_path, extract_artifacts_script, closure_dependencies_dag, dependency_uuids, extra_libs = args + item, julia_path, extract_artifacts_script, closure_dependencies_dag, dependency_uuids, extra_libs, is_darwin = args uuid, src = item lines = [] @@ -94,7 +102,7 @@ def process_item(args): parsed_url = urlparse(url) if any(parsed_url.path.endswith(x) for x in archive_extensions): - derivation = get_archive_derivation(uuid, artifact_name, url, sha256, closure_dependencies_dag, dependency_uuids, extra_libs) + derivation = get_archive_derivation(uuid, artifact_name, url, sha256, closure_dependencies_dag, dependency_uuids, extra_libs, is_darwin) else: derivation = get_plain_derivation(url, sha256) @@ -113,7 +121,8 @@ def main(): julia_path = Path(sys.argv[3]) extract_artifacts_script = Path(sys.argv[4]) extra_libs = json.loads(sys.argv[5]) - out_path = Path(sys.argv[6]) + is_darwin = json.loads(sys.argv[6]) + out_path = Path(sys.argv[7]) with open(dependencies_path, "r") as f: dependencies = yaml.safe_load(f) @@ -133,13 +142,17 @@ def main(): closure_dependencies_dag.add_node(uuid, dependencies=contents["depends_on"].values()) with open(out_path, "w") as f: - f.write("{ lib, fetchurl, glibc, pkgs, stdenv }:\n\n") + if is_darwin: + f.write("{ lib, fetchurl, pkgs, stdenv }:\n\n") + else: + f.write("{ lib, fetchurl, glibc, pkgs, stdenv }:\n\n") + f.write("rec {\n") with multiprocessing.Pool(10) as pool: # Create args tuples for each item process_args = [ - (item, julia_path, extract_artifacts_script, closure_dependencies_dag, dependency_uuids, extra_libs) + (item, julia_path, extract_artifacts_script, closure_dependencies_dag, dependency_uuids, extra_libs, is_darwin) for item in dependencies.items() ] for s in pool.map(process_item, process_args): From 23038522f11831d6a8c0232c3ba98bf322ef4102 Mon Sep 17 00:00:00 2001 From: Tom McLaughlin Date: Thu, 28 Nov 2024 12:02:28 -0800 Subject: [PATCH 3/3] julia.withPackages: set empty JULIA_SSL_CA_ROOTS_PATH on darwin --- pkgs/development/julia-modules/depot.nix | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/pkgs/development/julia-modules/depot.nix b/pkgs/development/julia-modules/depot.nix index 017bc19acd50..6e10f46cc72c 100644 --- a/pkgs/development/julia-modules/depot.nix +++ b/pkgs/development/julia-modules/depot.nix @@ -6,6 +6,7 @@ , git , julia , python3 +, stdenv , closureYaml , extraLibs @@ -16,6 +17,20 @@ , registry }: +let + # On darwin, we don't want to specify JULIA_SSL_CA_ROOTS_PATH. If we do (using a -bin julia derivation, which is the + # only kind darwin currently supports), you get an error like this: + # + # GitError(Code:ERROR, Class:SSL, Your Julia is built with a SSL/TLS engine that libgit2 doesn't know how to configure + # to use a file or directory of certificate authority roots, but your environment specifies one via the SSL_CERT_FILE + # variable. If you believe your system's root certificates are safe to use, you can `export JULIA_SSL_CA_ROOTS_PATH=""` + # in your environment to use those instead.) + setJuliaSslCaRootsPath = if stdenv.targetPlatform.isDarwin + then ''export JULIA_SSL_CA_ROOTS_PATH=""'' + else ''export JULIA_SSL_CA_ROOTS_PATH="${cacert}/etc/ssl/certs/ca-bundle.crt"''; + +in + runCommand "julia-depot" { nativeBuildInputs = [curl git julia (python3.withPackages (ps: with ps; [pyyaml]))] ++ extraLibs; inherit precompile registry; @@ -38,7 +53,7 @@ runCommand "julia-depot" { # export JULIA_DEBUG=Pkg # export JULIA_DEBUG=loading - export JULIA_SSL_CA_ROOTS_PATH="${cacert}/etc/ssl/certs/ca-bundle.crt" + ${setJuliaSslCaRootsPath} # Only precompile if configured to below export JULIA_PKG_PRECOMPILE_AUTO=0