gador/nixpkgs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

checkPhaseThreadLimitHook: rename from openmpCheckPhaseHook, set more thread env vars (#539853)

Browse Source
This commit is contained in:
Martin Weinelt
2026-07-10 12:21:00 +00:00
committed by GitHub
parent 54eb644221 b96ed4d602
commit eb0dbe6935
22 changed files with 152 additions and 49 deletions
Download Patch File Download Diff File Expand all files Collapse all files
doc/hooks/check-phase-thread-limit-hook.section.md
+24
View File
@@ -0,0 +1,24 @@
# checkPhaseThreadLimitHook {#setup-hook-check-phase-thread-limit}
This hook defaults a variety of environment variables known
to control thread counts to 1. Many of these otherwise default
to `$(nproc)`, which causes massive overloads on build machines
if nix build jobs and build cores are already tuned to fully utilize
compute capacity of a builder without additional parallelism.
Currently sets the following environment variables:
- [`OMP_NUM_THREADS`](https://www.openmp.org/spec-html/5.0/openmpse50.html)
- [`OPENBLAS_NUM_THREADS`](https://github.com/OpenMathLib/OpenBLAS/blob/e7b45174355edec1f04de1cabcf5ca6a98ea7fbc/USAGE.md#how-can-i-use-openblas-in-multi-threaded-applications)
- [`MKL_NUM_THREADS`](https://www.intel.com/content/www/us/en/docs/onemkl/developer-guide-linux/2023-0/mkl-domain-num-threads.html)
- [`BLIS_NUM_THREADS`](https://github.com/flame/blis/blob/b8b75b4e19459f5d618b57aa814ca38b1d82eb82/docs/Multithreading.md#specifying-multithreading)
- `VECLIB_MAXIMUM_THREADS`: Only affects darwin, see [`man 7 Accelerate`](https://manp.gs/mac/7/Accelerate)
- [`NUMBA_NUM_THREADS`](https://numba.readthedocs.io/en/stable/reference/envvars.html#threading-control)
- [`NUMEXPR_NUM_THREADS`](https://numexpr.readthedocs.io/en/latest/user_guide.html#threadpool-configuration)
The `NIX_CHECK_PHASE_DEFAULT_NUM_THREADS` environment variable
can be used to override the default thread count limit.
`dontLimitCheckPhaseThreads = true;` can be used to disable
thread limiting on an individual package.
This hook will not attempt to override already existing
definitions for thread count environment variables.
doc/hooks/index.md
+1 -1
View File
@@ -13,6 +13,7 @@ aws-c-common.section.md
bmake.section.md
breakpoint.section.md
cernlib.section.md
check-phase-thread-limit-hook.section.md
cmake.section.md
desktop-file-utils.section.md
gdk-pixbuf.section.md
@@ -34,7 +35,6 @@ nodejs-install-manuals.section.md
npm-build-hook.section.md
npm-config-hook.section.md
npm-install-hook.section.md
openmp-check-hook.section.md
patch-rc-path-hooks.section.md
perl.section.md
pkg-config.section.md
doc/hooks/openmp-check-hook.section.md
-10
View File
@@ -1,10 +0,0 @@
# openmpCheckPhaseHook {#setup-hook-omp-check}
This hook can be used to setup a check phase that
requires running a OpenMP application. It mostly
serves to limit `OMP_NUM_THREADS` to avoid overloading
build machines.
This hook will not attempt to override an already existing
definition of `OMP_NUM_THREADS` in the environment.
doc/redirects.json
+2 -1
View File
@@ -2920,7 +2920,8 @@
"setup-hook-mpi-check": [
"index.html#setup-hook-mpi-check"
],
"setup-hook-omp-check": [
"setup-hook-check-phase-thread-limit": [
"index.html#setup-hook-check-phase-thread-limit",
"index.html#setup-hook-omp-check"
],
"ninja": [
pkgs/by-name/bl/blas/package.nix
+2 -2
View File
@@ -3,7 +3,7 @@
stdenv,
lapack-reference,
openblas,
openmpCheckPhaseHook,
checkPhaseThreadLimitHook,
isILP64 ? false,
blasProvider ? openblas,
}:
@@ -188,7 +188,7 @@ stdenv.mkDerivation {
];
propagatedNativeBuildInputs = [
openmpCheckPhaseHook
checkPhaseThreadLimitHook
];
meta = (blasProvider'.meta or { }) // {
pkgs/by-name/bl/blis/package.nix
+10
View File
@@ -5,6 +5,12 @@
perl,
python3,
# sets BLIS_NUM_THREADS and OMP_NUM_THREADS for packages
# invoking blis during checkPhase/installCheckPhase to
# avoid overloading builders with excessive parallelism
# See also: https://github.com/flame/blis/blob/b8b75b4e19459f5d618b57aa814ca38b1d82eb82/docs/Multithreading.md#specifying-multithreading
checkPhaseThreadLimitHook,
# Enable BLAS interface with 64-bit integer width.
blas64 ? false,
@@ -36,6 +42,10 @@ stdenv.mkDerivation (finalAttrs: {
python3
];
propagatedNativeBuildInputs = [
checkPhaseThreadLimitHook
];
doCheck = true;
enableParallelBuilding = true;
pkgs/by-name/ch/checkPhaseThreadLimitHook/hook.sh
+32
View File
@@ -0,0 +1,32 @@
setupThreadLimit() {
# Limit number of threads used during checks. Default is "all cores".
# Using all cores causes high load on builders if checks are executed with NIX_BUILD_CORE parallelism.
# This gets even worse if multiple builds are scheduled on the same machine, potentially growing O(n^3) without explicit core limits.
# global value to override all of these at once
NIX_CHECK_PHASE_DEFAULT_NUM_THREADS="${NIX_CHECK_PHASE_DEFAULT_NUM_THREADS:-1}"
thread_vars=(
OMP_NUM_THREADS
OPENBLAS_NUM_THREADS
MKL_NUM_THREADS
BLIS_NUM_THREADS
VECLIB_MAXIMUM_THREADS
NUMBA_NUM_THREADS
NUMEXPR_NUM_THREADS
)
echo "setting known thread variables to default: $NIX_CHECK_PHASE_DEFAULT_NUM_THREADS"
for var in "${thread_vars[@]}"; do
export "$var=${!var:-$NIX_CHECK_PHASE_DEFAULT_NUM_THREADS}"
printf " %s=%s" "$var" "${!var}"
done
printf "\n"
}
if [[ -z "${dontLimitCheckPhaseThreads-}" ]]; then
echo "Using checkPhaseThreadLimitHook"
preCheckHooks+=('setupThreadLimit')
preInstallCheckHooks+=('setupThreadLimit')
fi
pkgs/by-name/ch/checkPhaseThreadLimitHook/package.nix
+15
View File
@@ -0,0 +1,15 @@
{
lib,
makeSetupHook,
}:
makeSetupHook {
name = "check-phase-thread-limit-hook";
__structuredAttrs = true;
meta = {
license = lib.licenses.mit;
maintainers = with lib.maintainers; [ grimmauld ];
};
} ./hook.sh
pkgs/by-name/mc/mctc-lib/package.nix
+2 -2
View File
@@ -10,7 +10,7 @@
pkg-config,
python3,
jonquil,
openmpCheckPhaseHook,
checkPhaseThreadLimitHook,
}:
assert (
@@ -44,7 +44,7 @@ stdenv.mkDerivation (finalAttrs: {
gfortran
pkg-config
python3
openmpCheckPhaseHook
checkPhaseThreadLimitHook
]
++ lib.optionals (buildType == "meson") [
meson
pkgs/by-name/mk/mkl/package.nix
+9
View File
@@ -8,6 +8,11 @@
_7zz,
cctools,
validatePkgConfig,
# sets MKL_NUM_THREADS for packages
# invoking mkl during checkPhase/installCheckPhase to
# avoid overloading builders with excessive parallelism
# See also: https://www.intel.com/content/www/us/en/docs/onemkl/developer-guide-linux/2023-0/mkl-domain-num-threads.html
checkPhaseThreadLimitHook,
enableStatic ? stdenv.hostPlatform.isStatic,
}:
@@ -86,6 +91,10 @@ stdenvNoCC.mkDerivation (
[ rpmextract ]
);
propagatedNativeBuildInputs = [
checkPhaseThreadLimitHook
];
buildPhase =
if stdenvNoCC.hostPlatform.isDarwin then
''
pkgs/by-name/mp/mpiCheckPhaseHook/package.nix
+2 -2
View File
@@ -2,7 +2,7 @@
lib,
makeSetupHook,
stdenv,
openmpCheckPhaseHook,
checkPhaseThreadLimitHook,
}:
makeSetupHook {
@@ -14,7 +14,7 @@ makeSetupHook {
};
propagatedNativeBuildInputs = [
openmpCheckPhaseHook
checkPhaseThreadLimitHook
];
meta.license = lib.licenses.mit;
pkgs/by-name/op/openmpCheckPhaseHook/omp-check-hook.sh
-11
View File
@@ -1,11 +0,0 @@
preCheckHooks+=('setupOmpCheck')
preInstallCheckHooks+=('setupOmpCheck')
setupOmpCheck() {
# Limit number of OpenMP threads. Default is "all cores".
# Using all cores causes high load on builders if checks are executed with NIX_BUILD_CORE parallelism.
# This gets even worse if multiple builds are scheduled on the same machine, potentially growing O(n^3) without explicit core limits.
export OMP_NUM_THREADS="${OMP_NUM_THREADS:-1}"
}
pkgs/by-name/op/openmpCheckPhaseHook/package.nix
-12
View File
@@ -1,12 +0,0 @@
{
lib,
makeSetupHook,
}:
makeSetupHook {
name = "omp-checkPhase-hook";
__structuredAttrs = true;
meta.license = lib.licenses.mit;
} ./omp-check-hook.sh
pkgs/by-name/pi/pixman/package.nix
+2 -2
View File
@@ -6,7 +6,7 @@
ninja,
pkg-config,
libpng,
openmpCheckPhaseHook,
checkPhaseThreadLimitHook,
glib, # just passthru
# for passthru.tests
@@ -47,7 +47,7 @@ stdenv.mkDerivation (finalAttrs: {
meson
ninja
pkg-config
openmpCheckPhaseHook
checkPhaseThreadLimitHook
__flattenIncludeHackHook
];
pkgs/development/compilers/llvm/common/openmp/default.nix
+2 -2
View File
@@ -19,7 +19,7 @@
ompdSupport ? true,
ompdGdbSupport ? ompdSupport,
getVersionFile,
openmpCheckPhaseHook,
checkPhaseThreadLimitHook,
}:
assert lib.assertMsg (ompdGdbSupport -> ompdSupport) "OMPD GDB support requires OMPD support!";
@@ -61,7 +61,7 @@ stdenv.mkDerivation (finalAttrs: {
];
propagatedNativeBuildInputs = [
openmpCheckPhaseHook
checkPhaseThreadLimitHook
];
buildInputs = [
pkgs/development/libraries/science/math/openblas/default.nix
+9
View File
@@ -4,6 +4,11 @@
fetchFromGitHub,
fetchpatch,
cmake,
# sets OPENBLAS_NUM_THREADS and OMP_NUM_THREADS for packages
# invoking openblas during checkPhase/installCheckPhase to
# avoid overloading builders with excessive parallelism
# See also: https://github.com/OpenMathLib/OpenBLAS/blob/e7b45174355edec1f04de1cabcf5ca6a98ea7fbc/USAGE.md#how-can-i-use-openblas-in-multi-threaded-applications
checkPhaseThreadLimitHook,
# Most packages depending on openblas expect integer width to match
# pointer width, but some expect to use 32-bit integers always
# (for compatibility with reference BLAS).
@@ -232,6 +237,10 @@ stdenv.mkDerivation (finalAttrs: {
cmake
];
propagatedNativeBuildInputs = [
checkPhaseThreadLimitHook
];
buildInputs = lib.optional (stdenv.cc.isClang && config.USE_OPENMP) openmp;
depsBuildBuild = [
pkgs/development/python-modules/joblib/default.nix
+16
View File
@@ -5,6 +5,14 @@
pythonAtLeast,
stdenv,
# sets various thread limit env vars for packages
# invoking joblib during checkPhase/installCheckPhase to
# avoid overloading builders with excessive parallelism
# See also:
# https://github.com/joblib/joblib/blob/b030e4e1ed6a227c0e587c31b266c03b3b692372/joblib/_parallel_backends.py#L59-L67
# https://github.com/joblib/joblib/blob/b030e4e1ed6a227c0e587c31b266c03b3b692372/joblib/_parallel_backends.py#L221-L248
checkPhaseThreadLimitHook,
# build-system
setuptools,
@@ -40,6 +48,14 @@ buildPythonPackage rec {
threadpoolctl
];
propagatedNativeBuildInputs = [
checkPhaseThreadLimitHook
];
# joblib expects to set thread limits itself while checking for propagation of thread limit environment variables.
# Setting these via checkPhaseThreadLimitHook on joblib itself causes tests to fail, but we do want the hook to propagate.
dontLimitCheckPhaseThreads = true;
enabledTestPaths = [ "joblib/test" ];
disabledTests = [
pkgs/development/python-modules/numba/default.nix
+10
View File
@@ -8,6 +8,12 @@
# nativeBuildInputs
setuptools,
# sets NUMBA_NUM_THREADS and OMP_NUM_THREADS for packages
# invoking numba during checkPhase/installCheckPhase to
# avoid overloading builders with excessive parallelism
# See also: https://numba.readthedocs.io/en/stable/reference/envvars.html#threading-control
checkPhaseThreadLimitHook,
# dependencies
llvmlite,
numpy,
@@ -104,6 +110,10 @@ buildPythonPackage (finalAttrs: {
writableTmpDirAsHomeHook
];
propagatedNativeBuildInputs = [
checkPhaseThreadLimitHook
];
# https://github.com/NixOS/nixpkgs/issues/255262
preCheck = ''
cd $out
pkgs/development/python-modules/numexpr/default.nix
+9
View File
@@ -5,6 +5,11 @@
numpy,
pytestCheckHook,
setuptools,
# sets NUMEXPR_NUM_THREADS and OMP_NUM_THREADS for packages
# invoking numexpr during checkPhase/installCheckPhase to
# avoid overloading builders with excessive parallelism
# See also: https://numexpr.readthedocs.io/en/latest/user_guide.html#threadpool-configuration
checkPhaseThreadLimitHook,
}:
buildPythonPackage rec {
@@ -31,6 +36,10 @@ buildPythonPackage rec {
nativeCheckInputs = [ pytestCheckHook ];
propagatedNativeBuildInputs = [
checkPhaseThreadLimitHook
];
preCheck = ''
pushd $out
'';
pkgs/development/python-modules/numpy/1.nix
+2 -2
View File
@@ -18,7 +18,7 @@
blas,
lapack,
openmpCheckPhaseHook,
checkPhaseThreadLimitHook,
# Reverse dependency
sage,
@@ -112,7 +112,7 @@ buildPythonPackage (finalAttrs: {
];
propagatedNativeBuildInputs = [
openmpCheckPhaseHook
checkPhaseThreadLimitHook
];
preCheck = ''
pkgs/development/python-modules/numpy/2.nix
+2 -2
View File
@@ -19,7 +19,7 @@
coreutils,
lapack,
openmpCheckPhaseHook,
checkPhaseThreadLimitHook,
# Reverse dependency
astropy,
@@ -118,7 +118,7 @@ buildPythonPackage (finalAttrs: {
'';
propagatedNativeBuildInputs = [
openmpCheckPhaseHook
checkPhaseThreadLimitHook
];
preCheck = ''
pkgs/top-level/aliases.nix
+1
View File
@@ -1736,6 +1736,7 @@ mapAliases {
openjfx23 = throw "OpenJFX 23 was removed as it has reached its end of life"; # Added 2025-11-04
openjfx24 = throw "OpenJFX 24 was removed as it has reached its end of life"; # Added 2025-10-04
openmodelica = throw "'openmodelica' has been removed as it was unmaintained in nixpkgs and depends on insecure&unmtaintained qtwebkit"; # Added 2026-04-26
openmpCheckPhaseHook = warnAlias "'openmpCheckPhaseHook' has been renamed to 'checkPhaseThreadLimitHook' to reflect its handling of all known thread-limiting mechanisms during check phase" checkPhaseThreadLimitHook; # Added 2026-07-09
openmw-tes3mp = throw "'openmw-tes3mp' has been removed due to lack of maintenance upstream"; # Added 2025-08-30
openssl_3_0 = throw "'openssl_3_0' has been renamed to/replaced by 'openssl_3'"; # Converted to throw 2025-10-27
opensycl = throw "'opensycl' has been renamed to/replaced by 'adaptivecpp'"; # Converted to throw 2025-10-27