diff --git a/pkgs/by-name/bl/blis/package.nix b/pkgs/by-name/bl/blis/package.nix index 1edd86ae85c7..43b1e17c59ed 100644 --- a/pkgs/by-name/bl/blis/package.nix +++ b/pkgs/by-name/bl/blis/package.nix @@ -5,6 +5,12 @@ perl, python3, + # sets BLIS_NUM_THREADS and OMP_NUM_THREADS for packages + # invoking blis during checkPhase/installCheckPhase to + # avoid overloading builders with excessive parallelism + # See also: https://github.com/flame/blis/blob/b8b75b4e19459f5d618b57aa814ca38b1d82eb82/docs/Multithreading.md#specifying-multithreading + checkPhaseThreadLimitHook, + # Enable BLAS interface with 64-bit integer width. blas64 ? false, @@ -36,6 +42,10 @@ stdenv.mkDerivation (finalAttrs: { python3 ]; + propagatedNativeBuildInputs = [ + checkPhaseThreadLimitHook + ]; + doCheck = true; enableParallelBuilding = true; diff --git a/pkgs/by-name/mk/mkl/package.nix b/pkgs/by-name/mk/mkl/package.nix index f9e1dda4e7ab..31b724759e8a 100644 --- a/pkgs/by-name/mk/mkl/package.nix +++ b/pkgs/by-name/mk/mkl/package.nix @@ -8,6 +8,11 @@ _7zz, cctools, validatePkgConfig, + # sets MKL_NUM_THREADS for packages + # invoking mkl during checkPhase/installCheckPhase to + # avoid overloading builders with excessive parallelism + # See also: https://www.intel.com/content/www/us/en/docs/onemkl/developer-guide-linux/2023-0/mkl-domain-num-threads.html + checkPhaseThreadLimitHook, enableStatic ? stdenv.hostPlatform.isStatic, }: @@ -86,6 +91,10 @@ stdenvNoCC.mkDerivation ( [ rpmextract ] ); + propagatedNativeBuildInputs = [ + checkPhaseThreadLimitHook + ]; + buildPhase = if stdenvNoCC.hostPlatform.isDarwin then '' diff --git a/pkgs/development/libraries/science/math/openblas/default.nix b/pkgs/development/libraries/science/math/openblas/default.nix index 97b7596b937c..ee5e6f12caa8 100644 --- a/pkgs/development/libraries/science/math/openblas/default.nix +++ b/pkgs/development/libraries/science/math/openblas/default.nix @@ -4,6 +4,11 @@ fetchFromGitHub, fetchpatch, cmake, + # sets OPENBLAS_NUM_THREADS and OMP_NUM_THREADS for packages + # invoking openblas during checkPhase/installCheckPhase to + # avoid overloading builders with excessive parallelism + # See also: https://github.com/OpenMathLib/OpenBLAS/blob/e7b45174355edec1f04de1cabcf5ca6a98ea7fbc/USAGE.md#how-can-i-use-openblas-in-multi-threaded-applications + checkPhaseThreadLimitHook, # Most packages depending on openblas expect integer width to match # pointer width, but some expect to use 32-bit integers always # (for compatibility with reference BLAS). @@ -232,6 +237,10 @@ stdenv.mkDerivation (finalAttrs: { cmake ]; + propagatedNativeBuildInputs = [ + checkPhaseThreadLimitHook + ]; + buildInputs = lib.optional (stdenv.cc.isClang && config.USE_OPENMP) openmp; depsBuildBuild = [ diff --git a/pkgs/development/python-modules/joblib/default.nix b/pkgs/development/python-modules/joblib/default.nix index 2474439fc910..b4b9c7bf2217 100644 --- a/pkgs/development/python-modules/joblib/default.nix +++ b/pkgs/development/python-modules/joblib/default.nix @@ -5,6 +5,14 @@ pythonAtLeast, stdenv, + # sets various thread limit env vars for packages + # invoking joblib during checkPhase/installCheckPhase to + # avoid overloading builders with excessive parallelism + # See also: + # https://github.com/joblib/joblib/blob/b030e4e1ed6a227c0e587c31b266c03b3b692372/joblib/_parallel_backends.py#L59-L67 + # https://github.com/joblib/joblib/blob/b030e4e1ed6a227c0e587c31b266c03b3b692372/joblib/_parallel_backends.py#L221-L248 + checkPhaseThreadLimitHook, + # build-system setuptools, @@ -40,6 +48,14 @@ buildPythonPackage rec { threadpoolctl ]; + propagatedNativeBuildInputs = [ + checkPhaseThreadLimitHook + ]; + + # joblib expects to set thread limits itself while checking for propagation of thread limit environment variables. + # Setting these via checkPhaseThreadLimitHook on joblib itself causes tests to fail, but we do want the hook to propagate. + dontLimitCheckPhaseThreads = true; + enabledTestPaths = [ "joblib/test" ]; disabledTests = [ diff --git a/pkgs/development/python-modules/numba/default.nix b/pkgs/development/python-modules/numba/default.nix index 11c25c98168d..7c215cdb4aaf 100644 --- a/pkgs/development/python-modules/numba/default.nix +++ b/pkgs/development/python-modules/numba/default.nix @@ -8,6 +8,12 @@ # nativeBuildInputs setuptools, + # sets NUMBA_NUM_THREADS and OMP_NUM_THREADS for packages + # invoking numba during checkPhase/installCheckPhase to + # avoid overloading builders with excessive parallelism + # See also: https://numba.readthedocs.io/en/stable/reference/envvars.html#threading-control + checkPhaseThreadLimitHook, + # dependencies llvmlite, numpy, @@ -104,6 +110,10 @@ buildPythonPackage (finalAttrs: { writableTmpDirAsHomeHook ]; + propagatedNativeBuildInputs = [ + checkPhaseThreadLimitHook + ]; + # https://github.com/NixOS/nixpkgs/issues/255262 preCheck = '' cd $out diff --git a/pkgs/development/python-modules/numexpr/default.nix b/pkgs/development/python-modules/numexpr/default.nix index f4b552c108ec..426b53e7814d 100644 --- a/pkgs/development/python-modules/numexpr/default.nix +++ b/pkgs/development/python-modules/numexpr/default.nix @@ -5,6 +5,11 @@ numpy, pytestCheckHook, setuptools, + # sets NUMEXPR_NUM_THREADS and OMP_NUM_THREADS for packages + # invoking numexpr during checkPhase/installCheckPhase to + # avoid overloading builders with excessive parallelism + # See also: https://numexpr.readthedocs.io/en/latest/user_guide.html#threadpool-configuration + checkPhaseThreadLimitHook, }: buildPythonPackage rec { @@ -31,6 +36,10 @@ buildPythonPackage rec { nativeCheckInputs = [ pytestCheckHook ]; + propagatedNativeBuildInputs = [ + checkPhaseThreadLimitHook + ]; + preCheck = '' pushd $out '';