diff --git a/pkgs/development/compilers/julia/1.6-bin.nix b/pkgs/development/compilers/julia/1.6-bin.nix deleted file mode 100644 index dea8a140ee5f..000000000000 --- a/pkgs/development/compilers/julia/1.6-bin.nix +++ /dev/null @@ -1,72 +0,0 @@ -{ autoPatchelfHook, fetchurl, lib, stdenv }: - -stdenv.mkDerivation rec { - pname = "julia-bin"; - version = "1.6.7"; - - src = { - x86_64-linux = fetchurl { - url = "https://julialang-s3.julialang.org/bin/linux/x64/${lib.versions.majorMinor version}/julia-${version}-linux-x86_64.tar.gz"; - sha256 = "sha256-bEUi1ZXky80AFXrEWKcviuwBdXBT0gc/mdqjnkQrKjY="; - }; - }.${stdenv.hostPlatform.system} or (throw "Unsupported system: ${stdenv.hostPlatform.system}"); - - # Julia’s source files are in different locations for source and binary - # releases. Thus we temporarily create a symlink to allow us to share patches - # with source releases. - prePatch = '' - ln -s share/julia/test - ''; - patches = [ - # Source release Nix patch(es) relevant for binary releases as well. - ./patches/1.6-bin/0005-nix-Enable-parallel-unit-tests-for-sandbox.patch - ]; - postPatch = '' - # Revert symlink hack. - rm test - - # Julia fails to pick up our Certification Authority root certificates, but - # it provides its own so we can simply disable the test. Patching in the - # dynamic path to ours require us to rebuild the Julia system image. - substituteInPlace share/julia/stdlib/v${lib.versions.majorMinor version}/NetworkOptions/test/runtests.jl \ - --replace '@test ca_roots_path() != bundled_ca_roots()' \ - '@test_skip ca_roots_path() != bundled_ca_roots()' - ''; - - nativeBuildInputs = [ autoPatchelfHook ]; - - installPhase = '' - runHook preInstall - cp -r . $out - runHook postInstall - ''; - - # Breaks backtraces, etc. - dontStrip = true; - - doInstallCheck = true; - preInstallCheck = '' - # Some tests require read/write access to $HOME. - export HOME="$TMPDIR" - ''; - installCheckPhase = '' - runHook preInstallCheck - # Command lifted from `test/Makefile`. - $out/bin/julia \ - --check-bounds=yes \ - --startup-file=no \ - --depwarn=error \ - $out/share/julia/test/runtests.jl - runHook postInstallCheck - ''; - - meta = { - description = "High-level, high-performance, dynamic language for technical computing"; - homepage = "https://julialang.org"; - # Bundled and linked with various GPL code, although Julia itself is MIT. - license = lib.licenses.gpl2Plus; - maintainers = with lib.maintainers; [ raskin thomasjm ]; - platforms = [ "x86_64-linux" ]; - mainProgram = "julia"; - }; -} diff --git a/pkgs/development/compilers/julia/default.nix b/pkgs/development/compilers/julia/default.nix index becbd64ea55b..6a805b5ccb83 100644 --- a/pkgs/development/compilers/julia/default.nix +++ b/pkgs/development/compilers/julia/default.nix @@ -12,7 +12,6 @@ let in { - julia_16-bin = wrapJulia (callPackage ./1.6-bin.nix { }); julia_19-bin = wrapJulia (callPackage (import ./generic-bin.nix { version = "1.9.4"; @@ -39,6 +38,17 @@ in }; }) { }); + julia_111-bin = wrapJulia (callPackage + (import ./generic-bin.nix { + version = "1.11.1"; + sha256 = { + x86_64-linux = "cca8d13dc4507e4f62a129322293313ee574f300d4df9e7db30b7b41c5f8a8f3"; + aarch64-linux = "bd623ef3801c5a56103464d349c7901d5cc034405ad289332c67f1e8ecc05840"; + x86_64-darwin = "59885de9310788c1ed12f41e7d2c2f05eabd314888cd105d299837b76a4a7240"; + aarch64-darwin = "e09d13e1c6c98452e91e698220688dd784ec8e5367e9e6443099c5f9aa2add78"; + }; + }) + { }); julia_19 = wrapJulia (callPackage (import ./generic.nix { version = "1.9.4"; @@ -57,4 +67,13 @@ in ]; }) { }); + julia_111 = wrapJulia (callPackage + (import ./generic.nix { + version = "1.11.1"; + hash = "sha256-pJuATeboagP+Jsc/WIUeruH/JD1yBPK1rk28XB3CdY0="; + patches = [ + ./patches/1.11/0002-skip-failing-and-flaky-tests.patch + ]; + }) + { }); } diff --git a/pkgs/development/compilers/julia/generic-bin.nix b/pkgs/development/compilers/julia/generic-bin.nix index fd78e6113850..a7d7f127d5e4 100644 --- a/pkgs/development/compilers/julia/generic-bin.nix +++ b/pkgs/development/compilers/julia/generic-bin.nix @@ -24,6 +24,11 @@ let "REPL" # Test flaky "ccall" + ] ++ lib.optionals (lib.versionAtLeast version "1.11") [ + # Test flaky + # https://github.com/JuliaLang/julia/issues/54280 + "loading" + "cmdlineargs" ] ++ lib.optionals stdenv.hostPlatform.isDarwin [ # Test flaky on ofborg "FileWatching" diff --git a/pkgs/development/compilers/julia/generic.nix b/pkgs/development/compilers/julia/generic.nix index 53a5067e783a..16f7bbce0759 100644 --- a/pkgs/development/compilers/julia/generic.nix +++ b/pkgs/development/compilers/julia/generic.nix @@ -9,12 +9,14 @@ , which , python3 , gfortran +, cacert , cmake , perl , gnum4 , openssl , libxml2 , zlib +, buildPackages }: stdenv.mkDerivation rec { @@ -42,12 +44,18 @@ stdenv.mkDerivation rec { buildInputs = [ libxml2 zlib + ] ++ lib.optionals (lib.versionAtLeast version "1.11") [ + cacert ]; dontUseCmakeConfigure = true; postPatch = '' patchShebangs . + '' + lib.optionalString (lib.versionAtLeast version "1.11") '' + substituteInPlace deps/curl.mk \ + --replace-fail 'cd $(dir $<) && $(TAR) jxf $(notdir $<)' \ + 'cd $(dir $<) && $(TAR) jxf $(notdir $<) && sed -i "s|/usr/bin/env perl|${lib.getExe buildPackages.perl}|" curl-$(CURL_VER)/scripts/cd2nroff' ''; makeFlags = [ diff --git a/pkgs/development/compilers/julia/patches/1.11/0002-skip-failing-and-flaky-tests.patch b/pkgs/development/compilers/julia/patches/1.11/0002-skip-failing-and-flaky-tests.patch new file mode 100644 index 000000000000..3577e6e71e15 --- /dev/null +++ b/pkgs/development/compilers/julia/patches/1.11/0002-skip-failing-and-flaky-tests.patch @@ -0,0 +1,25 @@ +From 9da2f2596db9f4f1a61825d82d9b8c3f3b2e99aa Mon Sep 17 00:00:00 2001 +From: Nick Cao +Date: Wed, 10 Jan 2024 20:58:20 -0500 +Subject: [PATCH 2/2] skip failing and flaky tests + +--- + test/Makefile | 2 +- + 1 file changed, 1 insertion(+), 1 deletion(-) + +diff --git a/test/Makefile b/test/Makefile +index 88dbe5b2b4..a2a7a55e20 100644 +--- a/test/Makefile ++++ b/test/Makefile +@@ -28,7 +28,7 @@ default: + + $(TESTS): + @cd $(SRCDIR) && \ +- $(call PRINT_JULIA, $(call spawn,$(JULIA_EXECUTABLE)) --check-bounds=yes --startup-file=no --depwarn=error ./runtests.jl $@) ++ $(call PRINT_JULIA, $(call spawn,$(JULIA_EXECUTABLE)) --check-bounds=yes --startup-file=no --depwarn=error ./runtests.jl --skip NetworkOptions REPL channels FileWatching ccall loading cmdlineargs $@) + + $(addprefix revise-, $(TESTS)): revise-% : + @cd $(SRCDIR) && \ +-- +2.43.0 + diff --git a/pkgs/development/compilers/julia/patches/1.6-bin/0005-nix-Enable-parallel-unit-tests-for-sandbox.patch b/pkgs/development/compilers/julia/patches/1.6-bin/0005-nix-Enable-parallel-unit-tests-for-sandbox.patch deleted file mode 100644 index 243a9cfd76ae..000000000000 --- a/pkgs/development/compilers/julia/patches/1.6-bin/0005-nix-Enable-parallel-unit-tests-for-sandbox.patch +++ /dev/null @@ -1,30 +0,0 @@ -From 44c2c979c4f2222567ce65f506cf47fb87482348 Mon Sep 17 00:00:00 2001 -From: Pontus Stenetorp -Date: Thu, 8 Apr 2021 04:37:44 +0000 -Subject: [PATCH 5/6] nix: Enable parallel unit tests for sandbox - -Disabled by default due to lack of networking in the Nix sandbox. This -greatly speeds up the build process on a multi-core system. ---- - test/runtests.jl | 5 +++-- - 1 file changed, 3 insertions(+), 2 deletions(-) - -diff --git a/test/runtests.jl b/test/runtests.jl -index 2f9cd058bb..2f8c19fa32 100644 ---- a/test/runtests.jl -+++ b/test/runtests.jl -@@ -83,8 +83,9 @@ prepend!(tests, linalg_tests) - import LinearAlgebra - cd(@__DIR__) do - n = 1 -- if net_on -- n = min(Sys.CPU_THREADS, length(tests)) -+ if net_on || haskey(ENV, "NIX_BUILD_CORES") -+ x = haskey(ENV, "NIX_BUILD_CORES") ? parse(Int, ENV["NIX_BUILD_CORES"]) : Sys.CPU_THREADS -+ n = min(x, Sys.CPU_THREADS, length(tests)) - n > 1 && addprocs_with_testenv(n) - LinearAlgebra.BLAS.set_num_threads(1) - end --- -2.29.3 - diff --git a/pkgs/development/julia-modules/package-closure.nix b/pkgs/development/julia-modules/package-closure.nix index bb0b08b33e41..8bc9e2774d81 100644 --- a/pkgs/development/julia-modules/package-closure.nix +++ b/pkgs/development/julia-modules/package-closure.nix @@ -10,131 +10,6 @@ }: let - # The specific package resolution code depends on the Julia version - # These are pretty similar and could be combined to reduce duplication - resolveCode = if lib.versionOlder julia.version "1.7" then resolveCode1_6 else resolveCode1_8; - - resolveCode1_6 = '' - import Pkg.API: check_package_name - import Pkg.Types: Context!, PRESERVE_NONE, manifest_info, project_deps_resolve!, registry_resolve!, stdlib_resolve!, ensure_resolved - import Pkg.Operations: _resolve, assert_can_add, is_dep, update_package_add - - foreach(pkg -> check_package_name(pkg.name, :add), pkgs) - pkgs = deepcopy(pkgs) # deepcopy for avoid mutating PackageSpec members - Context!(ctx) - - project_deps_resolve!(ctx, pkgs) - registry_resolve!(ctx, pkgs) - stdlib_resolve!(pkgs) - ensure_resolved(ctx, pkgs, registry=true) - - assert_can_add(ctx, pkgs) - - for (i, pkg) in pairs(pkgs) - entry = manifest_info(ctx, pkg.uuid) - pkgs[i] = update_package_add(ctx, pkg, entry, is_dep(ctx, pkg)) - end - - foreach(pkg -> ctx.env.project.deps[pkg.name] = pkg.uuid, pkgs) - - pkgs, deps_map = _resolve(ctx, pkgs, PRESERVE_NONE) -''; - - resolveCode1_8 = '' - import Pkg.API: handle_package_input! - import Pkg.Types: PRESERVE_NONE, UUID, VersionSpec, project_deps_resolve!, registry_resolve!, stdlib_resolve!, ensure_resolved - import Pkg.Operations: _resolve, assert_can_add, update_package_add - import TOML - - foreach(handle_package_input!, pkgs) - - # The handle_package_input! call above clears pkg.path, so we have to apply package overrides after - overrides = Dict{String, String}(${builtins.concatStringsSep ", " (lib.mapAttrsToList (name: path: ''"${name}" => "${path}"'') packageOverrides)}) - println("Package overrides: ") - println(overrides) - for pkg in pkgs - if pkg.name in keys(overrides) - pkg.path = overrides[pkg.name] - - # Try to read the UUID from $(pkg.path)/Project.toml. If successful, put the package into ctx.env.project.deps. - # This is necessary for the ensure_resolved call below to succeed, and will allow us to use an override even - # if it does not appear in the registry. - # See https://github.com/NixOS/nixpkgs/issues/279853 - project_toml = joinpath(pkg.path, "Project.toml") - if isfile(project_toml) - toml_data = TOML.parsefile(project_toml) - if haskey(toml_data, "uuid") - ctx.env.project.deps[pkg.name] = UUID(toml_data["uuid"]) - end - end - end - end - - project_deps_resolve!(ctx.env, pkgs) - registry_resolve!(ctx.registries, pkgs) - stdlib_resolve!(pkgs) - ensure_resolved(ctx, ctx.env.manifest, pkgs, registry=true) - - assert_can_add(ctx, pkgs) - - for (i, pkg) in pairs(pkgs) - entry = Pkg.Types.manifest_info(ctx.env.manifest, pkg.uuid) - is_dep = any(uuid -> uuid == pkg.uuid, [uuid for (name, uuid) in ctx.env.project.deps]) - pkgs[i] = update_package_add(ctx, pkg, entry, is_dep) - end - - foreach(pkg -> ctx.env.project.deps[pkg.name] = pkg.uuid, pkgs) - - # Save the original pkgs for later. We might need to augment it with the weak dependencies - orig_pkgs = pkgs - - pkgs, deps_map = _resolve(ctx.io, ctx.env, ctx.registries, pkgs, PRESERVE_NONE, ctx.julia_version) - - if VERSION >= VersionNumber("1.9") - while true - # Check for weak dependencies, which appear on the RHS of the deps_map but not in pkgs. - # Build up weak_name_to_uuid - uuid_to_name = Dict() - for pkg in pkgs - uuid_to_name[pkg.uuid] = pkg.name - end - weak_name_to_uuid = Dict() - for (uuid, deps) in pairs(deps_map) - for (dep_name, dep_uuid) in pairs(deps) - if !haskey(uuid_to_name, dep_uuid) - weak_name_to_uuid[dep_name] = dep_uuid - end - end - end - - if isempty(weak_name_to_uuid) - break - end - - # We have nontrivial weak dependencies, so add each one to the initial pkgs and then re-run _resolve - println("Found weak dependencies: $(keys(weak_name_to_uuid))") - - orig_uuids = Set([pkg.uuid for pkg in orig_pkgs]) - - for (name, uuid) in pairs(weak_name_to_uuid) - if uuid in orig_uuids - continue - end - - pkg = PackageSpec(name, uuid) - - push!(orig_uuids, uuid) - push!(orig_pkgs, pkg) - ctx.env.project.deps[name] = uuid - entry = Pkg.Types.manifest_info(ctx.env.manifest, uuid) - orig_pkgs[length(orig_pkgs)] = update_package_add(ctx, pkg, entry, false) - end - - global pkgs, deps_map = _resolve(ctx.io, ctx.env, ctx.registries, orig_pkgs, PRESERVE_NONE, ctx.julia_version) - end - end - ''; - juliaExpression = packageNames: '' import Pkg Pkg.Registry.add(Pkg.RegistrySpec(path="${augmentedRegistry}")) @@ -144,7 +19,7 @@ let input = ${lib.generators.toJSON {} packageNames} if isfile("extra_package_names.txt") - append!(input, readlines("extra_package_names.txt")) + append!(input, readlines("extra_package_names.txt")) end input = unique(input) @@ -155,20 +30,21 @@ let ctx = Context() - ${resolveCode} + overrides = Dict{String, String}(${builtins.concatStringsSep ", " (lib.mapAttrsToList (name: path: ''"${name}" => "${path}"'') packageOverrides)}) + ${builtins.readFile ./resolve_packages.jl} open(ENV["out"], "w") do io - for spec in pkgs - println(io, "- name: " * spec.name) - println(io, " uuid: " * string(spec.uuid)) - println(io, " version: " * string(spec.version)) - if endswith(spec.name, "_jll") && haskey(deps_map, spec.uuid) - println(io, " depends_on: ") - for (dep_name, dep_uuid) in pairs(deps_map[spec.uuid]) - println(io, " \"$(dep_name)\": \"$(dep_uuid)\"") - end + for spec in pkgs + println(io, "- name: " * spec.name) + println(io, " uuid: " * string(spec.uuid)) + println(io, " version: " * string(spec.version)) + if endswith(spec.name, "_jll") && haskey(deps_map, spec.uuid) + println(io, " depends_on: ") + for (dep_name, dep_uuid) in pairs(deps_map[spec.uuid]) + println(io, " \"$(dep_name)\": \"$(dep_uuid)\"") + end + end end - end end ''; in diff --git a/pkgs/development/julia-modules/resolve_packages.jl b/pkgs/development/julia-modules/resolve_packages.jl new file mode 100644 index 000000000000..fce60035d5f5 --- /dev/null +++ b/pkgs/development/julia-modules/resolve_packages.jl @@ -0,0 +1,99 @@ +import Pkg.API: handle_package_input! +import Pkg.Types: PRESERVE_NONE, UUID, VersionSpec, project_deps_resolve!, registry_resolve!, stdlib_resolve!, ensure_resolved +import Pkg.Operations: _resolve, assert_can_add, update_package_add +import TOML + +foreach(handle_package_input!, pkgs) + +# The handle_package_input! call above clears pkg.path, so we have to apply package overrides after +println("Package overrides: ") +println(overrides) +for pkg in pkgs + if pkg.name in keys(overrides) + pkg.path = overrides[pkg.name] + + # Try to read the UUID from $(pkg.path)/Project.toml. If successful, put the package into ctx.env.project.deps. + # This is necessary for the ensure_resolved call below to succeed, and will allow us to use an override even + # if it does not appear in the registry. + # See https://github.com/NixOS/nixpkgs/issues/279853 + project_toml = joinpath(pkg.path, "Project.toml") + if isfile(project_toml) + toml_data = TOML.parsefile(project_toml) + if haskey(toml_data, "uuid") + ctx.env.project.deps[pkg.name] = UUID(toml_data["uuid"]) + end + end + end +end + +project_deps_resolve!(ctx.env, pkgs) +registry_resolve!(ctx.registries, pkgs) +stdlib_resolve!(pkgs) +ensure_resolved(ctx, ctx.env.manifest, pkgs, registry=true) + +assert_can_add(ctx, pkgs) + +for (i, pkg) in pairs(pkgs) + entry = Pkg.Types.manifest_info(ctx.env.manifest, pkg.uuid) + is_dep = any(uuid -> uuid == pkg.uuid, [uuid for (name, uuid) in ctx.env.project.deps]) + if VERSION >= VersionNumber("1.11") + pkgs[i] = update_package_add(ctx, pkg, entry, nothing, nothing, is_dep) + else + pkgs[i] = update_package_add(ctx, pkg, entry, is_dep) + end +end + +foreach(pkg -> ctx.env.project.deps[pkg.name] = pkg.uuid, pkgs) + +# Save the original pkgs for later. We might need to augment it with the weak dependencies +orig_pkgs = pkgs + +pkgs, deps_map = _resolve(ctx.io, ctx.env, ctx.registries, pkgs, PRESERVE_NONE, ctx.julia_version) + +if VERSION >= VersionNumber("1.9") + while true + # Check for weak dependencies, which appear on the RHS of the deps_map but not in pkgs. + # Build up weak_name_to_uuid + uuid_to_name = Dict() + for pkg in pkgs + uuid_to_name[pkg.uuid] = pkg.name + end + weak_name_to_uuid = Dict() + for (uuid, deps) in pairs(deps_map) + for (dep_name, dep_uuid) in pairs(deps) + if !haskey(uuid_to_name, dep_uuid) + weak_name_to_uuid[dep_name] = dep_uuid + end + end + end + + if isempty(weak_name_to_uuid) + break + end + + # We have nontrivial weak dependencies, so add each one to the initial pkgs and then re-run _resolve + println("Found weak dependencies: $(keys(weak_name_to_uuid))") + + orig_uuids = Set([pkg.uuid for pkg in orig_pkgs]) + + for (name, uuid) in pairs(weak_name_to_uuid) + if uuid in orig_uuids + continue + end + + pkg = PackageSpec(name, uuid) + + push!(orig_uuids, uuid) + push!(orig_pkgs, pkg) + ctx.env.project.deps[name] = uuid + entry = Pkg.Types.manifest_info(ctx.env.manifest, uuid) + if VERSION >= VersionNumber("1.11") + orig_pkgs[length(orig_pkgs)] = update_package_add(ctx, pkg, entry, nothing, nothing, false) + else + orig_pkgs[length(orig_pkgs)] = update_package_add(ctx, pkg, entry, false) + end + end + + global pkgs, deps_map = _resolve(ctx.io, ctx.env, ctx.registries, orig_pkgs, PRESERVE_NONE, ctx.julia_version) + end +end diff --git a/pkgs/top-level/aliases.nix b/pkgs/top-level/aliases.nix index 16bc4f059475..87cec31134e5 100644 --- a/pkgs/top-level/aliases.nix +++ b/pkgs/top-level/aliases.nix @@ -779,6 +779,7 @@ mapAliases { jsawk = throw "'jsawk' has been removed because it is unmaintained upstream"; # Added 2028-08-07 # Julia + julia_16-bin = throw "'julia_16-bin' has been removed from nixpkgs as it has reached end of life"; # Added 2024-10-08 julia_18 = throw "'julia_18' has been removed from nixpkgs as it has reached end of life"; # Added 2024-01-11 julia_18-bin = throw "'julia_18-bin' has been removed from nixpkgs as it has reached end of life"; # Added 2024-01-11 diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index e5e6adaedd9d..9476379b9689 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -15142,18 +15142,19 @@ with pkgs; juniper = callPackage ../development/compilers/juniper { }; inherit (callPackage ../development/compilers/julia { }) - julia_16-bin julia_19-bin julia_110-bin + julia_111-bin julia_19 - julia_110; + julia_110 + julia_111; - julia-lts = julia_16-bin; - julia-stable = julia_110; + julia-lts = julia_110-bin; + julia-stable = julia_111; julia = julia-stable; - julia-lts-bin = julia_16-bin; - julia-stable-bin = julia_110-bin; + julia-lts-bin = julia_110-bin; + julia-stable-bin = julia_111-bin; julia-bin = julia-stable-bin; kind2 = darwin.apple_sdk_11_0.callPackage ../development/compilers/kind2 { };