gador/nixpkgs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

nixos/services.hadoop.yarn: remove with lib;

Browse Source
This commit is contained in:
Felix Buehler
2024-12-08 13:18:23 +01:00
parent e7e4c15a19
commit c1109e87b0
1 changed files with 35 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files

71
nixos/modules/services/cluster/hadoop/yarn.nix
View File

@@ -1,10 +1,9 @@
{ config, lib, pkgs, ...}: { config, lib, pkgs, ...}:
with lib;
let let
cfg = config.services.hadoop; cfg = config.services.hadoop;
hadoopConf = "${import ./conf.nix { inherit cfg pkgs lib; }}/"; hadoopConf = "${import ./conf.nix { inherit cfg pkgs lib; }}/";
restartIfChanged = mkOption { restartIfChanged = lib.mkOption {
type = types.bool; type = lib.types.bool;
description = '' description = ''
Automatically restart the service on config change. Automatically restart the service on config change.
This can be set to false to defer restarts on clusters running critical applications. This can be set to false to defer restarts on clusters running critical applications.
@@ -13,8 +12,8 @@ let
''; '';
default = false; default = false;
}; };
extraFlags = mkOption{ extraFlags = lib.mkOption{
type = with types; listOf str; type = with lib.types; listOf str;
default = []; default = [];
description = "Extra command line flags to pass to the service"; description = "Extra command line flags to pass to the service";
example = [ example = [
@@ -22,8 +21,8 @@ let
"-Dcom.sun.management.jmxremote.port=8010" "-Dcom.sun.management.jmxremote.port=8010"
]; ];
}; };
extraEnv = mkOption{ extraEnv = lib.mkOption{
type = with types; attrsOf str; type = with lib.types; attrsOf str;
default = {}; default = {};
description = "Extra environment variables"; description = "Extra environment variables";
}; };
@@ -31,11 +30,11 @@ in
{ {
options.services.hadoop.yarn = { options.services.hadoop.yarn = {
resourcemanager = { resourcemanager = {
enable = mkEnableOption "Hadoop YARN ResourceManager"; enable = lib.mkEnableOption "Hadoop YARN ResourceManager";
inherit restartIfChanged extraFlags extraEnv; inherit restartIfChanged extraFlags extraEnv;
openFirewall = mkOption { openFirewall = lib.mkOption {
type = types.bool; type = lib.types.bool;
default = false; default = false;
description = '' description = ''
Open firewall ports for resourcemanager Open firewall ports for resourcemanager
@@ -43,56 +42,56 @@ in
}; };
}; };
nodemanager = { nodemanager = {
enable = mkEnableOption "Hadoop YARN NodeManager"; enable = lib.mkEnableOption "Hadoop YARN NodeManager";
inherit restartIfChanged extraFlags extraEnv; inherit restartIfChanged extraFlags extraEnv;
resource = { resource = {
cpuVCores = mkOption { cpuVCores = lib.mkOption {
description = "Number of vcores that can be allocated for containers."; description = "Number of vcores that can be allocated for containers.";
type = with types; nullOr ints.positive; type = with lib.types; nullOr ints.positive;
default = null; default = null;
}; };
maximumAllocationVCores = mkOption { maximumAllocationVCores = lib.mkOption {
description = "The maximum virtual CPU cores any container can be allocated."; description = "The maximum virtual CPU cores any container can be allocated.";
type = with types; nullOr ints.positive; type = with lib.types; nullOr ints.positive;
default = null; default = null;
}; };
memoryMB = mkOption { memoryMB = lib.mkOption {
description = "Amount of physical memory, in MB, that can be allocated for containers."; description = "Amount of physical memory, in MB, that can be allocated for containers.";
type = with types; nullOr ints.positive; type = with lib.types; nullOr ints.positive;
default = null; default = null;
}; };
maximumAllocationMB = mkOption { maximumAllocationMB = lib.mkOption {
description = "The maximum physical memory any container can be allocated."; description = "The maximum physical memory any container can be allocated.";
type = with types; nullOr ints.positive; type = with lib.types; nullOr ints.positive;
default = null; default = null;
}; };
}; };
useCGroups = mkOption { useCGroups = lib.mkOption {
type = types.bool; type = lib.types.bool;
default = true; default = true;
description = '' description = ''
Use cgroups to enforce resource limits on containers Use cgroups to enforce resource limits on containers
''; '';
}; };
localDir = mkOption { localDir = lib.mkOption {
description = "List of directories to store localized files in."; description = "List of directories to store localized files in.";
type = with types; nullOr (listOf path); type = with lib.types; nullOr (listOf path);
example = [ "/var/lib/hadoop/yarn/nm" ]; example = [ "/var/lib/hadoop/yarn/nm" ];
default = null; default = null;
}; };
addBinBash = mkOption { addBinBash = lib.mkOption {
type = types.bool; type = lib.types.bool;
default = true; default = true;
description = '' description = ''
Add /bin/bash. This is needed by the linux container executor's launch script. Add /bin/bash. This is needed by the linux container executor's launch script.
''; '';
}; };
openFirewall = mkOption { openFirewall = lib.mkOption {
type = types.bool; type = lib.types.bool;
default = false; default = false;
description = '' description = ''
Open firewall ports for nodemanager. Open firewall ports for nodemanager.
@@ -102,8 +101,8 @@ in
}; };
}; };
config = mkMerge [ config = lib.mkMerge [
(mkIf cfg.gatewayRole.enable { (lib.mkIf cfg.gatewayRole.enable {
users.users.yarn = { users.users.yarn = {
description = "Hadoop YARN user"; description = "Hadoop YARN user";
group = "hadoop"; group = "hadoop";
@@ -111,7 +110,7 @@ in
}; };
}) })
(mkIf cfg.yarn.resourcemanager.enable { (lib.mkIf cfg.yarn.resourcemanager.enable {
systemd.services.yarn-resourcemanager = { systemd.services.yarn-resourcemanager = {
description = "Hadoop YARN ResourceManager"; description = "Hadoop YARN ResourceManager";
wantedBy = [ "multi-user.target" ]; wantedBy = [ "multi-user.target" ];
@@ -122,14 +121,14 @@ in
User = "yarn"; User = "yarn";
SyslogIdentifier = "yarn-resourcemanager"; SyslogIdentifier = "yarn-resourcemanager";
ExecStart = "${cfg.package}/bin/yarn --config ${hadoopConf} " + ExecStart = "${cfg.package}/bin/yarn --config ${hadoopConf} " +
" resourcemanager ${escapeShellArgs cfg.yarn.resourcemanager.extraFlags}"; " resourcemanager ${lib.escapeShellArgs cfg.yarn.resourcemanager.extraFlags}";
Restart = "always"; Restart = "always";
}; };
}; };
services.hadoop.gatewayRole.enable = true; services.hadoop.gatewayRole.enable = true;
networking.firewall.allowedTCPPorts = (mkIf cfg.yarn.resourcemanager.openFirewall [ networking.firewall.allowedTCPPorts = (lib.mkIf cfg.yarn.resourcemanager.openFirewall [
8088 # resourcemanager.webapp.address 8088 # resourcemanager.webapp.address
8030 # resourcemanager.scheduler.address 8030 # resourcemanager.scheduler.address
8031 # resourcemanager.resource-tracker.address 8031 # resourcemanager.resource-tracker.address
@@ -138,11 +137,11 @@ in
]); ]);
}) })
(mkIf cfg.yarn.nodemanager.enable { (lib.mkIf cfg.yarn.nodemanager.enable {
# Needed because yarn hardcodes /bin/bash in container start scripts # Needed because yarn hardcodes /bin/bash in container start scripts
# These scripts can't be patched, they are generated at runtime # These scripts can't be patched, they are generated at runtime
systemd.tmpfiles.rules = [ systemd.tmpfiles.rules = [
(mkIf cfg.yarn.nodemanager.addBinBash "L /bin/bash - - - - /run/current-system/sw/bin/bash") (lib.mkIf cfg.yarn.nodemanager.addBinBash "L /bin/bash - - - - /run/current-system/sw/bin/bash")
]; ];
systemd.services.yarn-nodemanager = { systemd.services.yarn-nodemanager = {
@@ -171,7 +170,7 @@ in
SyslogIdentifier = "yarn-nodemanager"; SyslogIdentifier = "yarn-nodemanager";
PermissionsStartOnly = true; PermissionsStartOnly = true;
ExecStart = "${cfg.package}/bin/yarn --config ${hadoopConf} " + ExecStart = "${cfg.package}/bin/yarn --config ${hadoopConf} " +
" nodemanager ${escapeShellArgs cfg.yarn.nodemanager.extraFlags}"; " nodemanager ${lib.escapeShellArgs cfg.yarn.nodemanager.extraFlags}";
Restart = "always"; Restart = "always";
}; };
}; };
@@ -192,7 +191,7 @@ in
})]; })];
networking.firewall.allowedTCPPortRanges = [ networking.firewall.allowedTCPPortRanges = [
(mkIf (cfg.yarn.nodemanager.openFirewall) {from = 1024; to = 65535;}) (lib.mkIf (cfg.yarn.nodemanager.openFirewall) {from = 1024; to = 65535;})
]; ];
}) })