gador/nixpkgs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

julia.withPackages: Darwin support (#360353)

Browse Source
This commit is contained in:
Nick Cao
2024-12-02 14:13:31 -05:00
committed by GitHub
parent b696e63615 23038522f1
commit 31c8248362
3 changed files with 108 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkgs/development/julia-modules/default.nix
+6 -1
View File
@@ -134,11 +134,16 @@ let
"${juliaWrapped}/bin/julia" \
"${if lib.versionAtLeast julia.version "1.7" then ./extract_artifacts.jl else ./extract_artifacts_16.jl}" \
'${lib.generators.toJSON {} (import ./extra-libs.nix)}' \
'${lib.generators.toJSON {} (stdenv.hostPlatform.isDarwin)}' \
"$out"
'';
# Import the artifacts Nix to build Overrides.toml (IFD)
artifacts = import artifactsNix { inherit lib fetchurl pkgs glibc stdenv; };
artifacts = import artifactsNix ({
inherit lib fetchurl pkgs stdenv;
} // lib.optionalAttrs (!stdenv.targetPlatform.isDarwin) {
inherit glibc;
});
overridesJson = writeTextFile {
name = "Overrides.json";
text = lib.generators.toJSON {} artifacts;
pkgs/development/julia-modules/depot.nix
+16 -1
View File
@@ -6,6 +6,7 @@
, git
, julia
, python3
, stdenv
, closureYaml
, extraLibs
@@ -16,6 +17,20 @@
, registry
}:
let
# On darwin, we don't want to specify JULIA_SSL_CA_ROOTS_PATH. If we do (using a -bin julia derivation, which is the
# only kind darwin currently supports), you get an error like this:
#
# GitError(Code:ERROR, Class:SSL, Your Julia is built with a SSL/TLS engine that libgit2 doesn't know how to configure
# to use a file or directory of certificate authority roots, but your environment specifies one via the SSL_CERT_FILE
# variable. If you believe your system's root certificates are safe to use, you can `export JULIA_SSL_CA_ROOTS_PATH=""`
# in your environment to use those instead.)
setJuliaSslCaRootsPath = if stdenv.targetPlatform.isDarwin
then ''export JULIA_SSL_CA_ROOTS_PATH=""''
else ''export JULIA_SSL_CA_ROOTS_PATH="${cacert}/etc/ssl/certs/ca-bundle.crt"'';
in
runCommand "julia-depot" {
nativeBuildInputs = [curl git julia (python3.withPackages (ps: with ps; [pyyaml]))] ++ extraLibs;
inherit precompile registry;
@@ -38,7 +53,7 @@ runCommand "julia-depot" {
# export JULIA_DEBUG=Pkg
# export JULIA_DEBUG=loading
export JULIA_SSL_CA_ROOTS_PATH="${cacert}/etc/ssl/certs/ca-bundle.crt"
${setJuliaSslCaRootsPath}
# Only precompile if configured to below
export JULIA_PKG_PRECOMPILE_AUTO=0
pkgs/development/julia-modules/python/extract_artifacts.py
+86 -60
View File
@@ -32,46 +32,30 @@ archive_extensions = [
".zip"
]
dependencies_path = Path(sys.argv[1])
closure_yaml_path = Path(sys.argv[2])
julia_path = Path(sys.argv[3])
extract_artifacts_script = Path(sys.argv[4])
extra_libs = json.loads(sys.argv[5])
out_path = Path(sys.argv[6])
with open(dependencies_path, "r") as f:
dependencies = yaml.safe_load(f)
dependency_uuids = dependencies.keys()
with open(closure_yaml_path, "r") as f:
# Build up a map of UUID -> closure information
closure_yaml_list = yaml.safe_load(f) or []
closure_yaml = {}
for item in closure_yaml_list:
closure_yaml[item["uuid"]] = item
# Build up a dependency graph of UUIDs
closure_dependencies_dag = dag.DAG()
for uuid, contents in closure_yaml.items():
if contents.get("depends_on"):
closure_dependencies_dag.add_node(uuid, dependencies=contents["depends_on"].values())
def get_archive_derivation(uuid, artifact_name, url, sha256):
def get_archive_derivation(uuid, artifact_name, url, sha256, closure_dependencies_dag, dependency_uuids, extra_libs, is_darwin):
depends_on = set()
if closure_dependencies_dag.has_node(uuid):
depends_on = set(closure_dependencies_dag.get_dependencies(uuid)).intersection(dependency_uuids)
other_libs = extra_libs.get(uuid, [])
fixup = f"""fixupPhase = let
libs = lib.concatMap (lib.mapAttrsToList (k: v: v.path))
if is_darwin:
fixup = f"""fixupPhase = let
libs = lib.concatMap (lib.mapAttrsToList (k: v: v.path))
[{" ".join(["uuid-" + x for x in depends_on])}];
in ''
find $out -type f -executable -exec \
patchelf --set-rpath \$ORIGIN:\$ORIGIN/../lib:${{lib.makeLibraryPath (["$out" glibc] ++ libs ++ (with pkgs; [{" ".join(other_libs)}]))}} {{}} \;
find $out -type f -executable -exec \
patchelf --set-interpreter ${{glibc}}/lib/ld-linux-x86-64.so.2 {{}} \;
''"""
in ''
''"""
else:
fixup = f"""fixupPhase = let
libs = lib.concatMap (lib.mapAttrsToList (k: v: v.path))
[{" ".join(["uuid-" + x for x in depends_on])}];
in ''
find $out -type f -executable -exec \
patchelf --set-rpath \$ORIGIN:\$ORIGIN/../lib:${{lib.makeLibraryPath (["$out" glibc] ++ libs ++ (with pkgs; [{" ".join(other_libs)}]))}} {{}} \;
find $out -type f -executable -exec \
patchelf --set-interpreter ${{glibc}}/lib/ld-linux-x86-64.so.2 {{}} \;
''"""
return f"""stdenv.mkDerivation {{
name = "{artifact_name}";
@@ -96,44 +80,86 @@ def get_plain_derivation(url, sha256):
sha256 = "{sha256}";
}}"""
with open(out_path, "w") as f:
f.write("{ lib, fetchurl, glibc, pkgs, stdenv }:\n\n")
f.write("rec {\n")
def process_item(args):
item, julia_path, extract_artifacts_script, closure_dependencies_dag, dependency_uuids, extra_libs, is_darwin = args
uuid, src = item
lines = []
def process_item(item):
uuid, src = item
lines = []
artifacts = toml.loads(subprocess.check_output([julia_path, extract_artifacts_script, uuid, src]).decode())
if not artifacts: return f' uuid-{uuid} = {{}};\n'
artifacts = toml.loads(subprocess.check_output([julia_path, extract_artifacts_script, uuid, src]).decode())
if not artifacts:
return f' uuid-{uuid} = {{}};\n'
lines.append(f' uuid-{uuid} = {{')
lines.append(f' uuid-{uuid} = {{')
for artifact_name, details in artifacts.items():
if len(details["download"]) == 0: continue
download = details["download"][0]
url = download["url"]
sha256 = download["sha256"]
for artifact_name, details in artifacts.items():
if len(details["download"]) == 0:
continue
download = details["download"][0]
url = download["url"]
sha256 = download["sha256"]
git_tree_sha1 = details["git-tree-sha1"]
git_tree_sha1 = details["git-tree-sha1"]
parsed_url = urlparse(url)
if any(parsed_url.path.endswith(x) for x in archive_extensions):
derivation = get_archive_derivation(uuid, artifact_name, url, sha256)
else:
derivation = get_plain_derivation(url, sha256)
parsed_url = urlparse(url)
if any(parsed_url.path.endswith(x) for x in archive_extensions):
derivation = get_archive_derivation(uuid, artifact_name, url, sha256, closure_dependencies_dag, dependency_uuids, extra_libs, is_darwin)
else:
derivation = get_plain_derivation(url, sha256)
lines.append(f""" "{artifact_name}" = {{
lines.append(f""" "{artifact_name}" = {{
sha1 = "{git_tree_sha1}";
path = {derivation};
}};\n""")
lines.append(' };\n')
lines.append(' };\n')
return "\n".join(lines)
return "\n".join(lines)
with multiprocessing.Pool(10) as pool:
for s in pool.map(process_item, dependencies.items()):
f.write(s)
def main():
dependencies_path = Path(sys.argv[1])
closure_yaml_path = Path(sys.argv[2])
julia_path = Path(sys.argv[3])
extract_artifacts_script = Path(sys.argv[4])
extra_libs = json.loads(sys.argv[5])
is_darwin = json.loads(sys.argv[6])
out_path = Path(sys.argv[7])
f.write(f"""
with open(dependencies_path, "r") as f:
dependencies = yaml.safe_load(f)
dependency_uuids = list(dependencies.keys()) # Convert dict_keys to list
with open(closure_yaml_path, "r") as f:
# Build up a map of UUID -> closure information
closure_yaml_list = yaml.safe_load(f) or []
closure_yaml = {}
for item in closure_yaml_list:
closure_yaml[item["uuid"]] = item
# Build up a dependency graph of UUIDs
closure_dependencies_dag = dag.DAG()
for uuid, contents in closure_yaml.items():
if contents.get("depends_on"):
closure_dependencies_dag.add_node(uuid, dependencies=contents["depends_on"].values())
with open(out_path, "w") as f:
if is_darwin:
f.write("{ lib, fetchurl, pkgs, stdenv }:\n\n")
else:
f.write("{ lib, fetchurl, glibc, pkgs, stdenv }:\n\n")
f.write("rec {\n")
with multiprocessing.Pool(10) as pool:
# Create args tuples for each item
process_args = [
(item, julia_path, extract_artifacts_script, closure_dependencies_dag, dependency_uuids, extra_libs, is_darwin)
for item in dependencies.items()
]
for s in pool.map(process_item, process_args):
f.write(s)
f.write(f"""
}}\n""")
if __name__ == "__main__":
main()