From 61dd3fdae2528e2344f4c3a675883af0b5e547f7 Mon Sep 17 00:00:00 2001 From: illustris Date: Fri, 13 Dec 2024 21:19:06 +0530 Subject: [PATCH 1/5] nixos/hadoop: fix lib function calls --- nixos/modules/services/cluster/hadoop/yarn.nix | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/nixos/modules/services/cluster/hadoop/yarn.nix b/nixos/modules/services/cluster/hadoop/yarn.nix index 7c0bc8e972d9..cae6d9c1ad4f 100644 --- a/nixos/modules/services/cluster/hadoop/yarn.nix +++ b/nixos/modules/services/cluster/hadoop/yarn.nix @@ -188,15 +188,15 @@ in services.hadoop.yarnSiteInternal = with cfg.yarn.nodemanager; - mkMerge [ + lib.mkMerge [ ({ - "yarn.nodemanager.local-dirs" = mkIf (localDir != null) (concatStringsSep "," localDir); + "yarn.nodemanager.local-dirs" = lib.mkIf (localDir != null) (concatStringsSep "," localDir); "yarn.scheduler.maximum-allocation-vcores" = resource.maximumAllocationVCores; "yarn.scheduler.maximum-allocation-mb" = resource.maximumAllocationMB; "yarn.nodemanager.resource.cpu-vcores" = resource.cpuVCores; "yarn.nodemanager.resource.memory-mb" = resource.memoryMB; }) - (mkIf useCGroups { + (lib.mkIf useCGroups { "yarn.nodemanager.linux-container-executor.cgroups.hierarchy" = "/hadoop-yarn"; "yarn.nodemanager.linux-container-executor.resources-handler.class" = "org.apache.hadoop.yarn.server.nodemanager.util.CgroupsLCEResourcesHandler"; From bdd10c641ebe9183c87c8d571ab0cfd50765be7f Mon Sep 17 00:00:00 2001 From: illustris Date: Fri, 13 Dec 2024 23:18:14 +0530 Subject: [PATCH 2/5] nixos/hadoop: fix failing yarn tests --- nixos/modules/services/cluster/hadoop/yarn.nix | 8 ++++++-- nixos/tests/hadoop/hadoop.nix | 1 + nixos/tests/hadoop/yarn.nix | 1 + 3 files changed, 8 insertions(+), 2 deletions(-) diff --git a/nixos/modules/services/cluster/hadoop/yarn.nix b/nixos/modules/services/cluster/hadoop/yarn.nix index cae6d9c1ad4f..2072171262bb 100644 --- a/nixos/modules/services/cluster/hadoop/yarn.nix +++ b/nixos/modules/services/cluster/hadoop/yarn.nix @@ -196,14 +196,18 @@ in "yarn.nodemanager.resource.cpu-vcores" = resource.cpuVCores; "yarn.nodemanager.resource.memory-mb" = resource.memoryMB; }) - (lib.mkIf useCGroups { + (lib.mkIf useCGroups ( lib.warnIf (lib.versionOlder cfg.package.version "3.5.0") '' + hadoop < 3.5.0 does not support cgroup v2 + setting `services.hadoop.yarn.nodemanager.useCGroups = false` is recommended + see: https://issues.apache.org/jira/browse/YARN-11669 + '' { "yarn.nodemanager.linux-container-executor.cgroups.hierarchy" = "/hadoop-yarn"; "yarn.nodemanager.linux-container-executor.resources-handler.class" = "org.apache.hadoop.yarn.server.nodemanager.util.CgroupsLCEResourcesHandler"; "yarn.nodemanager.linux-container-executor.cgroups.mount" = "true"; "yarn.nodemanager.linux-container-executor.cgroups.mount-path" = "/run/wrappers/yarn-nodemanager/cgroup"; - }) + })) ]; networking.firewall.allowedTCPPortRanges = [ diff --git a/nixos/tests/hadoop/hadoop.nix b/nixos/tests/hadoop/hadoop.nix index e4db1953d369..3f2b41bbd2c8 100644 --- a/nixos/tests/hadoop/hadoop.nix +++ b/nixos/tests/hadoop/hadoop.nix @@ -173,6 +173,7 @@ import ../make-test-python.nix ( yarn.nodemanager = { enable = true; openFirewall = true; + useCGroups = false; }; }; }; diff --git a/nixos/tests/hadoop/yarn.nix b/nixos/tests/hadoop/yarn.nix index c1a03e7ec7bd..3c3db2d52a95 100644 --- a/nixos/tests/hadoop/yarn.nix +++ b/nixos/tests/hadoop/yarn.nix @@ -24,6 +24,7 @@ import ../make-test-python.nix ( yarn.nodemanager = { enable = true; openFirewall = true; + useCGroups = false; }; yarnSite = { "yarn.resourcemanager.hostname" = "resourcemanager"; From b1e4a232f4c59daed94234f41b847bb91afcb94a Mon Sep 17 00:00:00 2001 From: illustris Date: Fri, 13 Dec 2024 23:20:27 +0530 Subject: [PATCH 3/5] nixos/hadoop: fix failing hdfs test --- nixos/tests/hadoop/hdfs.nix | 1 + 1 file changed, 1 insertion(+) diff --git a/nixos/tests/hadoop/hdfs.nix b/nixos/tests/hadoop/hdfs.nix index 45206eefa6c2..31b02964b4c6 100644 --- a/nixos/tests/hadoop/hdfs.nix +++ b/nixos/tests/hadoop/hdfs.nix @@ -36,6 +36,7 @@ import ../make-test-python.nix ( datanode = { pkgs, ... }: { + virtualisation.diskSize = 4096; services.hadoop = { inherit package; hdfs.datanode = { From 999e8c7ff06219593f337dff7e0f94a5fe4b1324 Mon Sep 17 00:00:00 2001 From: illustris Date: Sat, 14 Dec 2024 00:15:05 +0530 Subject: [PATCH 4/5] hadoop: 3.4.0 -> 3.4.1 --- .../networking/cluster/hadoop/default.nix | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) diff --git a/pkgs/applications/networking/cluster/hadoop/default.nix b/pkgs/applications/networking/cluster/hadoop/default.nix index 1a9aa3667e45..98275c25798d 100644 --- a/pkgs/applications/networking/cluster/hadoop/default.nix +++ b/pkgs/applications/networking/cluster/hadoop/default.nix @@ -101,7 +101,9 @@ let # hadoop 3.3+ depends on protobuf 3.18, 3.2 depends on 3.8 find $out/lib/native -name 'libhdfspp.so*' | \ xargs -r -n1 patchelf --replace-needed libprotobuf.so.${ - if (lib.versionAtLeast finalAttrs.version "3.3") then "18" else "8" + if (lib.versionAtLeast finalAttrs.version "3.4.1") then "32" + else if (lib.versionAtLeast finalAttrs.version "3.3") then "18" + else "8" } libprotobuf.so patchelf --replace-needed libcrypto.so.1.1 libcrypto.so \ @@ -163,13 +165,15 @@ in pname = "hadoop"; platformAttrs = rec { x86_64-linux = { - version = "3.4.0"; - hash = "sha256-4xGnhIBBQDD57GNUml1oXmnibyBxA9mr8hpIud0DyGw="; - srcHash = "sha256-viDF3LdRCZHqFycOYfN7nUQBPHiMCIjmu7jgIAaaK9E="; + version = "3.4.1"; + hash = "sha256-mtVIeDOZbf5VFOdW9DkQKckFKf0i6NAC/T3QwUwEukY="; + srcHash = "sha256-lE9uSohy6GWXprFEYbEin2ITqTms2h6EWXe4nEd3U4Y="; }; x86_64-darwin = x86_64-linux; aarch64-linux = x86_64-linux // { + version = "3.4.0"; hash = "sha256-QWxzKtNyw/AzcHMv0v7kj91pw1HO7VAN9MHO84caFk8="; + srcHash = "sha256-viDF3LdRCZHqFycOYfN7nUQBPHiMCIjmu7jgIAaaK9E="; }; aarch64-darwin = aarch64-linux; }; From c62ec80d1fc0cc4a665f3dca74a4d1292a8820e5 Mon Sep 17 00:00:00 2001 From: illustris Date: Sat, 14 Dec 2024 00:28:13 +0530 Subject: [PATCH 5/5] hadoop, nixos/hadoop, nixosTests/hadoop: nixfmt --- .../modules/services/cluster/hadoop/yarn.nix | 28 +++++++++++-------- nixos/tests/hadoop/default.nix | 16 ++++++++--- .../networking/cluster/hadoop/default.nix | 13 ++++++--- 3 files changed, 37 insertions(+), 20 deletions(-) diff --git a/nixos/modules/services/cluster/hadoop/yarn.nix b/nixos/modules/services/cluster/hadoop/yarn.nix index 2072171262bb..b21a4ca5c277 100644 --- a/nixos/modules/services/cluster/hadoop/yarn.nix +++ b/nixos/modules/services/cluster/hadoop/yarn.nix @@ -196,18 +196,22 @@ in "yarn.nodemanager.resource.cpu-vcores" = resource.cpuVCores; "yarn.nodemanager.resource.memory-mb" = resource.memoryMB; }) - (lib.mkIf useCGroups ( lib.warnIf (lib.versionOlder cfg.package.version "3.5.0") '' - hadoop < 3.5.0 does not support cgroup v2 - setting `services.hadoop.yarn.nodemanager.useCGroups = false` is recommended - see: https://issues.apache.org/jira/browse/YARN-11669 - '' { - "yarn.nodemanager.linux-container-executor.cgroups.hierarchy" = "/hadoop-yarn"; - "yarn.nodemanager.linux-container-executor.resources-handler.class" = - "org.apache.hadoop.yarn.server.nodemanager.util.CgroupsLCEResourcesHandler"; - "yarn.nodemanager.linux-container-executor.cgroups.mount" = "true"; - "yarn.nodemanager.linux-container-executor.cgroups.mount-path" = - "/run/wrappers/yarn-nodemanager/cgroup"; - })) + (lib.mkIf useCGroups ( + lib.warnIf (lib.versionOlder cfg.package.version "3.5.0") + '' + hadoop < 3.5.0 does not support cgroup v2 + setting `services.hadoop.yarn.nodemanager.useCGroups = false` is recommended + see: https://issues.apache.org/jira/browse/YARN-11669 + '' + { + "yarn.nodemanager.linux-container-executor.cgroups.hierarchy" = "/hadoop-yarn"; + "yarn.nodemanager.linux-container-executor.resources-handler.class" = + "org.apache.hadoop.yarn.server.nodemanager.util.CgroupsLCEResourcesHandler"; + "yarn.nodemanager.linux-container-executor.cgroups.mount" = "true"; + "yarn.nodemanager.linux-container-executor.cgroups.mount-path" = + "/run/wrappers/yarn-nodemanager/cgroup"; + } + )) ]; networking.firewall.allowedTCPPortRanges = [ diff --git a/nixos/tests/hadoop/default.nix b/nixos/tests/hadoop/default.nix index 479690adc064..aef5bd79e359 100644 --- a/nixos/tests/hadoop/default.nix +++ b/nixos/tests/hadoop/default.nix @@ -1,8 +1,16 @@ { handleTestOn, package, ... }: { - all = handleTestOn [ "x86_64-linux" "aarch64-linux" ] ./hadoop.nix { inherit package; }; - hdfs = handleTestOn [ "x86_64-linux" "aarch64-linux" ] ./hdfs.nix { inherit package; }; - yarn = handleTestOn [ "x86_64-linux" "aarch64-linux" ] ./yarn.nix { inherit package; }; - hbase = handleTestOn [ "x86_64-linux" "aarch64-linux" ] ./hbase.nix { inherit package; }; + all = handleTestOn [ "x86_64-linux" "aarch64-linux" ] ./hadoop.nix { + inherit package; + }; + hdfs = handleTestOn [ "x86_64-linux" "aarch64-linux" ] ./hdfs.nix { + inherit package; + }; + yarn = handleTestOn [ "x86_64-linux" "aarch64-linux" ] ./yarn.nix { + inherit package; + }; + hbase = handleTestOn [ "x86_64-linux" "aarch64-linux" ] ./hbase.nix { + inherit package; + }; } diff --git a/pkgs/applications/networking/cluster/hadoop/default.nix b/pkgs/applications/networking/cluster/hadoop/default.nix index 98275c25798d..47cd3764a77a 100644 --- a/pkgs/applications/networking/cluster/hadoop/default.nix +++ b/pkgs/applications/networking/cluster/hadoop/default.nix @@ -46,7 +46,9 @@ let "mirror://apache/hadoop/common/hadoop-${finalAttrs.version}/hadoop-${finalAttrs.version}" + lib.optionalString stdenv.hostPlatform.isAarch64 "-aarch64" + ".tar.gz"; - inherit (platformAttrs.${stdenv.system} or (throw "Unsupported system: ${stdenv.system}")) hash; + inherit (platformAttrs.${stdenv.system} or (throw "Unsupported system: ${stdenv.system}")) + hash + ; }; doCheck = true; @@ -101,9 +103,12 @@ let # hadoop 3.3+ depends on protobuf 3.18, 3.2 depends on 3.8 find $out/lib/native -name 'libhdfspp.so*' | \ xargs -r -n1 patchelf --replace-needed libprotobuf.so.${ - if (lib.versionAtLeast finalAttrs.version "3.4.1") then "32" - else if (lib.versionAtLeast finalAttrs.version "3.3") then "18" - else "8" + if (lib.versionAtLeast finalAttrs.version "3.4.1") then + "32" + else if (lib.versionAtLeast finalAttrs.version "3.3") then + "18" + else + "8" } libprotobuf.so patchelf --replace-needed libcrypto.so.1.1 libcrypto.so \