From e4900a80298e6d3c6b5713b73bba2c92d1a063f2 Mon Sep 17 00:00:00 2001
From: Doron Behar <doron.behar@gmail.com>
Date: Mon, 15 Jul 2024 15:38:27 +0300
Subject: [PATCH 01/20] pmix: use substituteInPlace in cross-compilation fix

This ensures that this doesn't fail if the sed pattern all of a sudden
is incorrect
---
 pkgs/development/libraries/pmix/default.nix | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/pkgs/development/libraries/pmix/default.nix b/pkgs/development/libraries/pmix/default.nix
index 10cc3cac59b7..31325bd4ba0a 100644
--- a/pkgs/development/libraries/pmix/default.nix
+++ b/pkgs/development/libraries/pmix/default.nix
@@ -71,8 +71,9 @@ stdenv.mkDerivation rec {
 
     # Pin the compiler to the current version in a cross compiler friendly way.
     # Same pattern as for openmpi (see https://github.com/NixOS/nixpkgs/pull/58964#discussion_r275059427).
-    sed -i 's:compiler=.*:compiler=${targetPackages.stdenv.cc}/bin/${targetPackages.stdenv.cc.targetPrefix}cc:' \
-      $dev/share/pmix/pmixcc-wrapper-data.txt
+    substituteInPlace $dev/share/pmix/pmixcc-wrapper-data.txt \
+      --replace-fail compiler=gcc \
+        compiler=${targetPackages.stdenv.cc}/bin/${targetPackages.stdenv.cc.targetPrefix}cc
   '';
 
   enableParallelBuilding = true;

From b0d8554f3c9cfa69a50d6c80d4c7cdf1fc78c2f5 Mon Sep 17 00:00:00 2001
From: Markus Kowalewski <markus.kowalewski@fysik.su.se>
Date: Tue, 2 May 2023 12:33:40 +0200
Subject: [PATCH 02/20] prrte: init at 3.0.3

Co-authored-by: Markus Kowalewski <markus.kowalewski@gmail.com>
---
 pkgs/by-name/pr/prrte/package.nix | 80 +++++++++++++++++++++++++++++++
 1 file changed, 80 insertions(+)
 create mode 100644 pkgs/by-name/pr/prrte/package.nix

diff --git a/pkgs/by-name/pr/prrte/package.nix b/pkgs/by-name/pr/prrte/package.nix
new file mode 100644
index 000000000000..c8ac31fe626a
--- /dev/null
+++ b/pkgs/by-name/pr/prrte/package.nix
@@ -0,0 +1,80 @@
+{
+  lib,
+  stdenv,
+  removeReferencesTo,
+  fetchFromGitHub,
+  autoconf,
+  automake,
+  libtool,
+  gitMinimal,
+  perl,
+  python3,
+  flex,
+  hwloc,
+  libevent,
+  zlib,
+  pmix,
+}:
+
+stdenv.mkDerivation rec {
+  pname = "prrte";
+  version = "3.0.5";
+
+  src = fetchFromGitHub {
+    owner = "openpmix";
+    repo = "prrte";
+    rev = "v${version}";
+    sha256 = "sha256-RDxd4veLGbN+T7xCDnNp2lbOM7mwKKD+SKdPmExr1C8=";
+    fetchSubmodules = true;
+  };
+
+  outputs = [
+    "out"
+    "dev"
+  ];
+
+  postPatch = ''
+    patchShebangs ./autogen.pl ./config
+  '';
+
+  preConfigure = ''
+    ./autogen.pl
+  '';
+
+  postInstall = ''
+    moveToOutput "bin/prte_info" "''${!outputDev}"
+    # Fix a broken symlink, created due to FHS assumptions
+    rm "$out/bin/pcc"
+    ln -s ${lib.getDev pmix}/bin/pmixcc "''${!outputDev}"/bin/pcc
+
+    remove-references-to -t "''${!outputDev}" $(readlink -f $out/lib/libprrte${stdenv.hostPlatform.extensions.library})
+  '';
+
+  nativeBuildInputs = [
+    removeReferencesTo
+    perl
+    python3
+    autoconf
+    automake
+    libtool
+    flex
+    gitMinimal
+  ];
+
+  buildInputs = [
+    libevent
+    hwloc
+    zlib
+    pmix
+  ];
+
+  enableParallelBuilding = true;
+
+  meta = {
+    description = "PMIx Reference Runtime Environment";
+    homepage = "https://docs.prrte.org/";
+    license = lib.licenses.bsd3;
+    maintainers = with lib.maintainers; [ markuskowa ];
+    platforms = lib.platforms.linux;
+  };
+}

From f25c42c4da19101df833eac49327db134ac3834b Mon Sep 17 00:00:00 2001
From: Doron Behar <doron.behar@gmail.com>
Date: Tue, 16 Jul 2024 12:38:04 +0300
Subject: [PATCH 03/20] openmpi: nixfmt-rfc-style

---
 .../development/libraries/openmpi/default.nix | 177 +++++++++++-------
 1 file changed, 112 insertions(+), 65 deletions(-)

diff --git a/pkgs/development/libraries/openmpi/default.nix b/pkgs/development/libraries/openmpi/default.nix
index 55fc330024d1..dedd49cd346f 100644
--- a/pkgs/development/libraries/openmpi/default.nix
+++ b/pkgs/development/libraries/openmpi/default.nix
@@ -1,31 +1,47 @@
-{ lib, stdenv, fetchurl, removeReferencesTo, gfortran, perl, libnl
-, rdma-core, zlib, numactl, libevent, hwloc, targetPackages
-, libpsm2, libfabric, pmix, ucx, ucc, makeWrapper
-, config
-# Enable CUDA support
-, cudaSupport ? config.cudaSupport, cudaPackages
-
-# Enable the Sun Grid Engine bindings
-, enableSGE ? false
-
-# Pass PATH/LD_LIBRARY_PATH to point to current mpirun by default
-, enablePrefix ? false
-
-# Enable libfabric support (necessary for Omnipath networks) on x86_64 linux
-, fabricSupport ? stdenv.isLinux && stdenv.isx86_64
-
-# Enable Fortran support
-, fortranSupport ? true
+{
+  lib,
+  stdenv,
+  fetchurl,
+  removeReferencesTo,
+  gfortran,
+  perl,
+  libnl,
+  rdma-core,
+  zlib,
+  numactl,
+  libevent,
+  hwloc,
+  targetPackages,
+  libpsm2,
+  libfabric,
+  pmix,
+  ucx,
+  ucc,
+  makeWrapper,
+  config,
+  # Enable CUDA support
+  cudaSupport ? config.cudaSupport,
+  cudaPackages,
+  # Enable the Sun Grid Engine bindings
+  enableSGE ? false,
+  # Pass PATH/LD_LIBRARY_PATH to point to current mpirun by default
+  enablePrefix ? false,
+  # Enable libfabric support (necessary for Omnipath networks) on x86_64 linux
+  fabricSupport ? stdenv.isLinux && stdenv.isx86_64,
+  # Enable Fortran support
+  fortranSupport ? true,
 }:
 
 stdenv.mkDerivation rec {
   pname = "openmpi";
   version = "4.1.6";
 
-  src = with lib.versions; fetchurl {
-    url = "https://www.open-mpi.org/software/ompi/v${major version}.${minor version}/downloads/${pname}-${version}.tar.bz2";
-    sha256 = "sha256-90CZRIVRbetjtTEa8SLCZRefUyig2FelZ7hdsAsR5BU=";
-  };
+  src =
+    with lib.versions;
+    fetchurl {
+      url = "https://www.open-mpi.org/software/ompi/v${major version}.${minor version}/downloads/${pname}-${version}.tar.bz2";
+      sha256 = "sha256-90CZRIVRbetjtTEa8SLCZRefUyig2FelZ7hdsAsR5BU=";
+    };
 
   postPatch = ''
     patchShebangs ./
@@ -38,34 +54,63 @@ stdenv.mkDerivation rec {
     find -name "Makefile.in" -exec sed -i "s/\`date\`/$ts/" \{} \;
   '';
 
-  outputs = [ "out" "man" "dev" ];
+  outputs = [
+    "out"
+    "man"
+    "dev"
+  ];
 
-  buildInputs = [ zlib ]
-    ++ lib.optionals stdenv.isLinux [ libnl numactl pmix ucx ucc ]
+  buildInputs =
+    [ zlib ]
+    ++ lib.optionals stdenv.isLinux [
+      libnl
+      numactl
+      pmix
+      ucx
+      ucc
+    ]
     ++ lib.optionals cudaSupport [ cudaPackages.cuda_cudart ]
-    ++ [ libevent hwloc ]
+    ++ [
+      libevent
+      hwloc
+    ]
     ++ lib.optional (stdenv.isLinux || stdenv.isFreeBSD) rdma-core
-    ++ lib.optionals fabricSupport [ libpsm2 libfabric ];
+    ++ lib.optionals fabricSupport [
+      libpsm2
+      libfabric
+    ];
 
-  nativeBuildInputs = [ perl removeReferencesTo makeWrapper ]
+  nativeBuildInputs =
+    [
+      perl
+      removeReferencesTo
+      makeWrapper
+    ]
     ++ lib.optionals cudaSupport [ cudaPackages.cuda_nvcc ]
     ++ lib.optionals fortranSupport [ gfortran ];
 
-  configureFlags = lib.optional (!cudaSupport) "--disable-mca-dso"
+  configureFlags =
+    lib.optional (!cudaSupport) "--disable-mca-dso"
     ++ lib.optional (!fortranSupport) "--disable-mpi-fortran"
-    ++ lib.optionals stdenv.isLinux  [
+    ++ lib.optionals stdenv.isLinux [
       "--with-libnl=${lib.getDev libnl}"
       "--with-pmix=${lib.getDev pmix}"
       "--with-pmix-libdir=${pmix}/lib"
       "--enable-mpi-cxx"
-    ] ++ lib.optional enableSGE "--with-sge"
+    ]
+    ++ lib.optional enableSGE "--with-sge"
     ++ lib.optional enablePrefix "--enable-mpirun-prefix-by-default"
     # TODO: add UCX support, which is recommended to use with cuda for the most robust OpenMPI build
     # https://github.com/openucx/ucx
     # https://www.open-mpi.org/faq/?category=buildcuda
-    ++ lib.optionals cudaSupport [ "--with-cuda=${lib.getDev cudaPackages.cuda_cudart}" "--enable-dlopen" ]
-    ++ lib.optionals fabricSupport [ "--with-psm2=${lib.getDev libpsm2}" "--with-libfabric=${lib.getDev libfabric}" ]
-    ;
+    ++ lib.optionals cudaSupport [
+      "--with-cuda=${lib.getDev cudaPackages.cuda_cudart}"
+      "--enable-dlopen"
+    ]
+    ++ lib.optionals fabricSupport [
+      "--with-psm2=${lib.getDev libpsm2}"
+      "--with-libfabric=${lib.getDev libfabric}"
+    ];
 
   enableParallelBuilding = true;
 
@@ -85,39 +130,41 @@ stdenv.mkDerivation rec {
     done
 
     moveToOutput "share/openmpi/ortecc-wrapper-data.txt" "''${!outputDev}"
-   '';
-
-  postFixup = ''
-    remove-references-to -t $dev $(readlink -f $out/lib/libopen-pal${stdenv.hostPlatform.extensions.sharedLibrary})
-    remove-references-to -t $man $(readlink -f $out/lib/libopen-pal${stdenv.hostPlatform.extensions.sharedLibrary})
-
-    # The path to the wrapper is hard coded in libopen-pal.so, which we just cleared.
-    wrapProgram $dev/bin/opal_wrapper \
-      --set OPAL_INCLUDEDIR $dev/include \
-      --set OPAL_PKGDATADIR $dev/share/openmpi
-
-    # default compilers should be indentical to the
-    # compilers at build time
-
-    echo "$dev/share/openmpi/mpicc-wrapper-data.txt"
-    sed -i 's:compiler=.*:compiler=${targetPackages.stdenv.cc}/bin/${targetPackages.stdenv.cc.targetPrefix}cc:' \
-      $dev/share/openmpi/mpicc-wrapper-data.txt
-
-    echo "$dev/share/openmpi/ortecc-wrapper-data.txt"
-    sed -i 's:compiler=.*:compiler=${targetPackages.stdenv.cc}/bin/${targetPackages.stdenv.cc.targetPrefix}cc:' \
-       $dev/share/openmpi/ortecc-wrapper-data.txt
-
-    echo "$dev/share/openmpi/mpic++-wrapper-data.txt"
-    sed -i 's:compiler=.*:compiler=${targetPackages.stdenv.cc}/bin/${targetPackages.stdenv.cc.targetPrefix}c++:' \
-       $dev/share/openmpi/mpic++-wrapper-data.txt
-  '' + lib.optionalString fortranSupport ''
-
-    echo "$dev/share/openmpi/mpifort-wrapper-data.txt"
-    sed -i 's:compiler=.*:compiler=${gfortran}/bin/${gfortran.targetPrefix}gfortran:'  \
-       $dev/share/openmpi/mpifort-wrapper-data.txt
-
   '';
 
+  postFixup =
+    ''
+      remove-references-to -t $dev $(readlink -f $out/lib/libopen-pal${stdenv.hostPlatform.extensions.sharedLibrary})
+      remove-references-to -t $man $(readlink -f $out/lib/libopen-pal${stdenv.hostPlatform.extensions.sharedLibrary})
+
+      # The path to the wrapper is hard coded in libopen-pal.so, which we just cleared.
+      wrapProgram $dev/bin/opal_wrapper \
+        --set OPAL_INCLUDEDIR $dev/include \
+        --set OPAL_PKGDATADIR $dev/share/openmpi
+
+      # default compilers should be indentical to the
+      # compilers at build time
+
+      echo "$dev/share/openmpi/mpicc-wrapper-data.txt"
+      sed -i 's:compiler=.*:compiler=${targetPackages.stdenv.cc}/bin/${targetPackages.stdenv.cc.targetPrefix}cc:' \
+        $dev/share/openmpi/mpicc-wrapper-data.txt
+
+      echo "$dev/share/openmpi/ortecc-wrapper-data.txt"
+      sed -i 's:compiler=.*:compiler=${targetPackages.stdenv.cc}/bin/${targetPackages.stdenv.cc.targetPrefix}cc:' \
+         $dev/share/openmpi/ortecc-wrapper-data.txt
+
+      echo "$dev/share/openmpi/mpic++-wrapper-data.txt"
+      sed -i 's:compiler=.*:compiler=${targetPackages.stdenv.cc}/bin/${targetPackages.stdenv.cc.targetPrefix}c++:' \
+         $dev/share/openmpi/mpic++-wrapper-data.txt
+    ''
+    + lib.optionalString fortranSupport ''
+
+      echo "$dev/share/openmpi/mpifort-wrapper-data.txt"
+      sed -i 's:compiler=.*:compiler=${gfortran}/bin/${gfortran.targetPrefix}gfortran:'  \
+         $dev/share/openmpi/mpifort-wrapper-data.txt
+
+    '';
+
   doCheck = true;
 
   passthru = {

From 5ee3a6e510b25537017460bc9a7c55152180e189 Mon Sep 17 00:00:00 2001
From: Doron Behar <doron.behar@gmail.com>
Date: Tue, 16 Jul 2024 12:38:37 +0300
Subject: [PATCH 04/20] openmpi: don't overuse 'with lib;' in meta

---
 pkgs/development/libraries/openmpi/default.nix | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/pkgs/development/libraries/openmpi/default.nix b/pkgs/development/libraries/openmpi/default.nix
index dedd49cd346f..53b9753b6df8 100644
--- a/pkgs/development/libraries/openmpi/default.nix
+++ b/pkgs/development/libraries/openmpi/default.nix
@@ -172,12 +172,12 @@ stdenv.mkDerivation rec {
     cudatoolkit = cudaPackages.cudatoolkit; # For backward compatibility only
   };
 
-  meta = with lib; {
+  meta = {
     homepage = "https://www.open-mpi.org/";
     description = "Open source MPI-3 implementation";
     longDescription = "The Open MPI Project is an open source MPI-3 implementation that is developed and maintained by a consortium of academic, research, and industry partners. Open MPI is therefore able to combine the expertise, technologies, and resources from all across the High Performance Computing community in order to build the best MPI library available. Open MPI offers advantages for system and software vendors, application developers and computer science researchers.";
-    maintainers = with maintainers; [ markuskowa ];
-    license = licenses.bsd3;
-    platforms = platforms.unix;
+    maintainers = with lib.maintainers; [ markuskowa ];
+    license = lib.licenses.bsd3;
+    platforms = lib.platforms.unix;
   };
 }

From 2613455670f6d1fd78be08a0396e818535ee6062 Mon Sep 17 00:00:00 2001
From: Doron Behar <doron.behar@gmail.com>
Date: Tue, 16 Jul 2024 12:39:43 +0300
Subject: [PATCH 05/20] openmpi: use a finalAttrs function to mkDerivation

---
 pkgs/development/libraries/openmpi/default.nix | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/pkgs/development/libraries/openmpi/default.nix b/pkgs/development/libraries/openmpi/default.nix
index 53b9753b6df8..b0433f71bfd4 100644
--- a/pkgs/development/libraries/openmpi/default.nix
+++ b/pkgs/development/libraries/openmpi/default.nix
@@ -32,14 +32,14 @@
   fortranSupport ? true,
 }:
 
-stdenv.mkDerivation rec {
+stdenv.mkDerivation (finalAttrs: {
   pname = "openmpi";
   version = "4.1.6";
 
   src =
     with lib.versions;
     fetchurl {
-      url = "https://www.open-mpi.org/software/ompi/v${major version}.${minor version}/downloads/${pname}-${version}.tar.bz2";
+      url = "https://www.open-mpi.org/software/ompi/v${major finalAttrs.version}.${minor finalAttrs.version}/downloads/${finalAttrs.pname}-${finalAttrs.version}.tar.bz2";
       sha256 = "sha256-90CZRIVRbetjtTEa8SLCZRefUyig2FelZ7hdsAsR5BU=";
     };
 
@@ -180,4 +180,4 @@ stdenv.mkDerivation rec {
     license = lib.licenses.bsd3;
     platforms = lib.platforms.unix;
   };
-}
+})

From 51f5d82d757904fc62a91b87dc7432b58f9e80a7 Mon Sep 17 00:00:00 2001
From: Doron Behar <doron.behar@gmail.com>
Date: Tue, 16 Jul 2024 12:41:07 +0300
Subject: [PATCH 06/20] openmpi: use more elegent lib.versions.majorMinor in
 src

---
 pkgs/development/libraries/openmpi/default.nix | 10 ++++------
 1 file changed, 4 insertions(+), 6 deletions(-)

diff --git a/pkgs/development/libraries/openmpi/default.nix b/pkgs/development/libraries/openmpi/default.nix
index b0433f71bfd4..809fb4746c61 100644
--- a/pkgs/development/libraries/openmpi/default.nix
+++ b/pkgs/development/libraries/openmpi/default.nix
@@ -36,12 +36,10 @@ stdenv.mkDerivation (finalAttrs: {
   pname = "openmpi";
   version = "4.1.6";
 
-  src =
-    with lib.versions;
-    fetchurl {
-      url = "https://www.open-mpi.org/software/ompi/v${major finalAttrs.version}.${minor finalAttrs.version}/downloads/${finalAttrs.pname}-${finalAttrs.version}.tar.bz2";
-      sha256 = "sha256-90CZRIVRbetjtTEa8SLCZRefUyig2FelZ7hdsAsR5BU=";
-    };
+  src = fetchurl {
+    url = "https://www.open-mpi.org/software/ompi/v${lib.versions.majorMinor finalAttrs.version}/downloads/${finalAttrs.pname}-${finalAttrs.version}.tar.bz2";
+    sha256 = "sha256-90CZRIVRbetjtTEa8SLCZRefUyig2FelZ7hdsAsR5BU=";
+  };
 
   postPatch = ''
     patchShebangs ./

From 494efeb70a8ffac4e9a01a2e3975551d1836a859 Mon Sep 17 00:00:00 2001
From: Doron Behar <doron.behar@gmail.com>
Date: Tue, 16 Jul 2024 12:45:20 +0300
Subject: [PATCH 07/20] openmpi: put all common buildInputs in the same place

---
 pkgs/development/libraries/openmpi/default.nix | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/pkgs/development/libraries/openmpi/default.nix b/pkgs/development/libraries/openmpi/default.nix
index 809fb4746c61..fa238737d4c2 100644
--- a/pkgs/development/libraries/openmpi/default.nix
+++ b/pkgs/development/libraries/openmpi/default.nix
@@ -59,7 +59,11 @@ stdenv.mkDerivation (finalAttrs: {
   ];
 
   buildInputs =
-    [ zlib ]
+    [
+      zlib
+      libevent
+      hwloc
+    ]
     ++ lib.optionals stdenv.isLinux [
       libnl
       numactl
@@ -68,10 +72,6 @@ stdenv.mkDerivation (finalAttrs: {
       ucc
     ]
     ++ lib.optionals cudaSupport [ cudaPackages.cuda_cudart ]
-    ++ [
-      libevent
-      hwloc
-    ]
     ++ lib.optional (stdenv.isLinux || stdenv.isFreeBSD) rdma-core
     ++ lib.optionals fabricSupport [
       libpsm2

From 6f348b83efdf573f69938d5c36b7401d2ea31e01 Mon Sep 17 00:00:00 2001
From: Doron Behar <doron.behar@gmail.com>
Date: Tue, 16 Jul 2024 12:46:45 +0300
Subject: [PATCH 08/20] openmpi: always use lib.optionals and not lib.optional

Less confusing when to use a list and a single argument.
---
 pkgs/development/libraries/openmpi/default.nix | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/pkgs/development/libraries/openmpi/default.nix b/pkgs/development/libraries/openmpi/default.nix
index fa238737d4c2..000f63b37c79 100644
--- a/pkgs/development/libraries/openmpi/default.nix
+++ b/pkgs/development/libraries/openmpi/default.nix
@@ -72,7 +72,7 @@ stdenv.mkDerivation (finalAttrs: {
       ucc
     ]
     ++ lib.optionals cudaSupport [ cudaPackages.cuda_cudart ]
-    ++ lib.optional (stdenv.isLinux || stdenv.isFreeBSD) rdma-core
+    ++ lib.optionals (stdenv.isLinux || stdenv.isFreeBSD) [ rdma-core ]
     ++ lib.optionals fabricSupport [
       libpsm2
       libfabric
@@ -88,8 +88,8 @@ stdenv.mkDerivation (finalAttrs: {
     ++ lib.optionals fortranSupport [ gfortran ];
 
   configureFlags =
-    lib.optional (!cudaSupport) "--disable-mca-dso"
-    ++ lib.optional (!fortranSupport) "--disable-mpi-fortran"
+    lib.optionals (!cudaSupport) [ "--disable-mca-dso" ]
+    ++ lib.optionals (!fortranSupport) [ "--disable-mpi-fortran"]
     ++ lib.optionals stdenv.isLinux [
       "--with-libnl=${lib.getDev libnl}"
       "--with-pmix=${lib.getDev pmix}"

From 0d81346e5f244268ad1e3d1b2116895b2fa5545e Mon Sep 17 00:00:00 2001
From: Doron Behar <doron.behar@gmail.com>
Date: Tue, 16 Jul 2024 12:48:22 +0300
Subject: [PATCH 09/20] openmpi: use lib.*Feature functions to create
 configureFlags

---
 .../development/libraries/openmpi/default.nix | 33 ++++++++-----------
 1 file changed, 14 insertions(+), 19 deletions(-)

diff --git a/pkgs/development/libraries/openmpi/default.nix b/pkgs/development/libraries/openmpi/default.nix
index 000f63b37c79..2469e36df478 100644
--- a/pkgs/development/libraries/openmpi/default.nix
+++ b/pkgs/development/libraries/openmpi/default.nix
@@ -87,28 +87,23 @@ stdenv.mkDerivation (finalAttrs: {
     ++ lib.optionals cudaSupport [ cudaPackages.cuda_nvcc ]
     ++ lib.optionals fortranSupport [ gfortran ];
 
-  configureFlags =
-    lib.optionals (!cudaSupport) [ "--disable-mca-dso" ]
-    ++ lib.optionals (!fortranSupport) [ "--disable-mpi-fortran"]
-    ++ lib.optionals stdenv.isLinux [
-      "--with-libnl=${lib.getDev libnl}"
-      "--with-pmix=${lib.getDev pmix}"
-      "--with-pmix-libdir=${pmix}/lib"
-      "--enable-mpi-cxx"
-    ]
-    ++ lib.optional enableSGE "--with-sge"
-    ++ lib.optional enablePrefix "--enable-mpirun-prefix-by-default"
+  configureFlags = [
+    (lib.enableFeature cudaSupport "mca-dso")
+    (lib.enableFeature fortranSupport "mpi-fortran")
+    (lib.withFeatureAs stdenv.isLinux "libnl" (lib.getDev libnl))
+    (lib.withFeatureAs stdenv.isLinux "pmix" (lib.getDev pmix))
+    (lib.withFeatureAs stdenv.isLinux "pmix-libdir" "${lib.getLib pmix}/lib")
+    (lib.enableFeature stdenv.isLinux "mpi-cxx")
+    (lib.withFeature enableSGE "sge")
+    (lib.enableFeature enablePrefix "mpirun-prefix-by-default")
     # TODO: add UCX support, which is recommended to use with cuda for the most robust OpenMPI build
     # https://github.com/openucx/ucx
     # https://www.open-mpi.org/faq/?category=buildcuda
-    ++ lib.optionals cudaSupport [
-      "--with-cuda=${lib.getDev cudaPackages.cuda_cudart}"
-      "--enable-dlopen"
-    ]
-    ++ lib.optionals fabricSupport [
-      "--with-psm2=${lib.getDev libpsm2}"
-      "--with-libfabric=${lib.getDev libfabric}"
-    ];
+    (lib.withFeatureAs cudaSupport "cuda" (lib.getDev cudaPackages.cuda_cudart))
+    (lib.enableFeature cudaSupport "dlopen")
+    (lib.withFeatureAs fabricSupport "psm2" (lib.getDev libpsm2))
+    (lib.withFeatureAs fabricSupport "libfabric" (lib.getDev libfabric))
+  ];
 
   enableParallelBuilding = true;
 

From 2a4636b06ceea03f0d7449e9b173433a08a7557e Mon Sep 17 00:00:00 2001
From: Doron Behar <doron.behar@gmail.com>
Date: Tue, 16 Jul 2024 12:55:30 +0300
Subject: [PATCH 10/20] openmpi: always use ${!outputDev} to allow easily
 disabling multiple outputs

---
 .../development/libraries/openmpi/default.nix | 26 +++++++++----------
 1 file changed, 13 insertions(+), 13 deletions(-)

diff --git a/pkgs/development/libraries/openmpi/default.nix b/pkgs/development/libraries/openmpi/default.nix
index 2469e36df478..2f1b25dfe510 100644
--- a/pkgs/development/libraries/openmpi/default.nix
+++ b/pkgs/development/libraries/openmpi/default.nix
@@ -127,34 +127,34 @@ stdenv.mkDerivation (finalAttrs: {
 
   postFixup =
     ''
-      remove-references-to -t $dev $(readlink -f $out/lib/libopen-pal${stdenv.hostPlatform.extensions.sharedLibrary})
-      remove-references-to -t $man $(readlink -f $out/lib/libopen-pal${stdenv.hostPlatform.extensions.sharedLibrary})
+      remove-references-to -t "''${!outputDev}" $(readlink -f $out/lib/libopen-pal${stdenv.hostPlatform.extensions.sharedLibrary})
+      remove-references-to -t "''${!outputMan}" $(readlink -f $out/lib/libopen-pal${stdenv.hostPlatform.extensions.sharedLibrary})
 
       # The path to the wrapper is hard coded in libopen-pal.so, which we just cleared.
-      wrapProgram $dev/bin/opal_wrapper \
-        --set OPAL_INCLUDEDIR $dev/include \
-        --set OPAL_PKGDATADIR $dev/share/openmpi
+      wrapProgram "''${!outputDev}/bin/opal_wrapper" \
+        --set OPAL_INCLUDEDIR "''${!outputDev}/include" \
+        --set OPAL_PKGDATADIR "''${!outputDev}/share/openmpi"
 
       # default compilers should be indentical to the
       # compilers at build time
 
-      echo "$dev/share/openmpi/mpicc-wrapper-data.txt"
+      echo "''${!outputDev}/share/openmpi/mpicc-wrapper-data.txt"
       sed -i 's:compiler=.*:compiler=${targetPackages.stdenv.cc}/bin/${targetPackages.stdenv.cc.targetPrefix}cc:' \
-        $dev/share/openmpi/mpicc-wrapper-data.txt
+        ''${!outputDev}/share/openmpi/mpicc-wrapper-data.txt
 
-      echo "$dev/share/openmpi/ortecc-wrapper-data.txt"
+      echo "''${!outputDev}/share/openmpi/ortecc-wrapper-data.txt"
       sed -i 's:compiler=.*:compiler=${targetPackages.stdenv.cc}/bin/${targetPackages.stdenv.cc.targetPrefix}cc:' \
-         $dev/share/openmpi/ortecc-wrapper-data.txt
+         ''${!outputDev}/share/openmpi/ortecc-wrapper-data.txt
 
-      echo "$dev/share/openmpi/mpic++-wrapper-data.txt"
+      echo "''${!outputDev}/share/openmpi/mpic++-wrapper-data.txt"
       sed -i 's:compiler=.*:compiler=${targetPackages.stdenv.cc}/bin/${targetPackages.stdenv.cc.targetPrefix}c++:' \
-         $dev/share/openmpi/mpic++-wrapper-data.txt
+         ''${!outputDev}/share/openmpi/mpic++-wrapper-data.txt
     ''
     + lib.optionalString fortranSupport ''
 
-      echo "$dev/share/openmpi/mpifort-wrapper-data.txt"
+      echo "''${!outputDev}/share/openmpi/mpifort-wrapper-data.txt"
       sed -i 's:compiler=.*:compiler=${gfortran}/bin/${gfortran.targetPrefix}gfortran:'  \
-         $dev/share/openmpi/mpifort-wrapper-data.txt
+         ''${!outputDev}/share/openmpi/mpifort-wrapper-data.txt
 
     '';
 

From d07b88749a32a2db91784ca93a96f88b26c4f6da Mon Sep 17 00:00:00 2001
From: Doron Behar <doron.behar@gmail.com>
Date: Tue, 16 Jul 2024 13:00:07 +0300
Subject: [PATCH 11/20] openmpi: separate opal_wrapper moveToOutput a bit

---
 pkgs/development/libraries/openmpi/default.nix | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/pkgs/development/libraries/openmpi/default.nix b/pkgs/development/libraries/openmpi/default.nix
index 2f1b25dfe510..47a7bab79e16 100644
--- a/pkgs/development/libraries/openmpi/default.nix
+++ b/pkgs/development/libraries/openmpi/default.nix
@@ -110,6 +110,9 @@ stdenv.mkDerivation (finalAttrs: {
   postInstall = ''
     find $out/lib/ -name "*.la" -exec rm -f \{} \;
 
+    # The main wrapper that all the rest of the commonly used binaries are
+    # symlinked to
+    moveToOutput "bin/opal_wrapper" "''${!outputDev}"
     for f in mpi shmem osh; do
       for i in f77 f90 CC c++ cxx cc fort; do
         moveToOutput "bin/$f$i" "''${!outputDev}"
@@ -118,7 +121,7 @@ stdenv.mkDerivation (finalAttrs: {
       done
     done
 
-    for i in ortecc orte-info ompi_info oshmem_info opal_wrapper; do
+    for i in ortecc orte-info ompi_info oshmem_info; do
       moveToOutput "bin/$i" "''${!outputDev}"
     done
 

From eaf636742e0694d371484acec9217d1d8ddcee7c Mon Sep 17 00:00:00 2001
From: Doron Behar <doron.behar@gmail.com>
Date: Tue, 16 Jul 2024 13:02:21 +0300
Subject: [PATCH 12/20] openmpi: separate ortecc related moveToOutput commands

---
 pkgs/development/libraries/openmpi/default.nix | 9 ++++++---
 1 file changed, 6 insertions(+), 3 deletions(-)

diff --git a/pkgs/development/libraries/openmpi/default.nix b/pkgs/development/libraries/openmpi/default.nix
index 47a7bab79e16..9feb2b7999fa 100644
--- a/pkgs/development/libraries/openmpi/default.nix
+++ b/pkgs/development/libraries/openmpi/default.nix
@@ -121,11 +121,14 @@ stdenv.mkDerivation (finalAttrs: {
       done
     done
 
-    for i in ortecc orte-info ompi_info oshmem_info; do
+    # ortecc's files don't have c++ and fort companions so it is handled
+    # outside the for loops above
+    moveToOutput "bin/ortecc" "''${!outputDev}"
+    moveToOutput "share/openmpi/ortecc-wrapper-data.txt" "''${!outputDev}"
+
+    for i in orte-info ompi_info oshmem_info; do
       moveToOutput "bin/$i" "''${!outputDev}"
     done
-
-    moveToOutput "share/openmpi/ortecc-wrapper-data.txt" "''${!outputDev}"
   '';
 
   postFixup =

From 5809830ee6ab5b40a1eaf73f8613747b0ca726f3 Mon Sep 17 00:00:00 2001
From: Doron Behar <doron.behar@gmail.com>
Date: Tue, 16 Jul 2024 13:08:33 +0300
Subject: [PATCH 13/20] openmpi: replace sed cross compilation fixes with
 substituteInPlace

Using substituteInPlace (with `--replace-fail`) is much safer.
---
 .../development/libraries/openmpi/default.nix | 24 ++++++++++++-------
 1 file changed, 16 insertions(+), 8 deletions(-)

diff --git a/pkgs/development/libraries/openmpi/default.nix b/pkgs/development/libraries/openmpi/default.nix
index 9feb2b7999fa..121d212b959d 100644
--- a/pkgs/development/libraries/openmpi/default.nix
+++ b/pkgs/development/libraries/openmpi/default.nix
@@ -145,22 +145,30 @@ stdenv.mkDerivation (finalAttrs: {
       # compilers at build time
 
       echo "''${!outputDev}/share/openmpi/mpicc-wrapper-data.txt"
-      sed -i 's:compiler=.*:compiler=${targetPackages.stdenv.cc}/bin/${targetPackages.stdenv.cc.targetPrefix}cc:' \
-        ''${!outputDev}/share/openmpi/mpicc-wrapper-data.txt
+      substituteInPlace ''${!outputDev}/share/openmpi/mpicc-wrapper-data.txt \
+        --replace-fail \
+          compiler=gcc \
+          compiler=${targetPackages.stdenv.cc}/bin/${targetPackages.stdenv.cc.targetPrefix}cc
 
       echo "''${!outputDev}/share/openmpi/ortecc-wrapper-data.txt"
-      sed -i 's:compiler=.*:compiler=${targetPackages.stdenv.cc}/bin/${targetPackages.stdenv.cc.targetPrefix}cc:' \
-         ''${!outputDev}/share/openmpi/ortecc-wrapper-data.txt
+      substituteInPlace ''${!outputDev}/share/openmpi/ortecc-wrapper-data.txt \
+        --replace-fail \
+          compiler=gcc \
+          compiler=${targetPackages.stdenv.cc}/bin/${targetPackages.stdenv.cc.targetPrefix}cc
 
       echo "''${!outputDev}/share/openmpi/mpic++-wrapper-data.txt"
-      sed -i 's:compiler=.*:compiler=${targetPackages.stdenv.cc}/bin/${targetPackages.stdenv.cc.targetPrefix}c++:' \
-         ''${!outputDev}/share/openmpi/mpic++-wrapper-data.txt
+      substituteInPlace ''${!outputDev}/share/openmpi/mpic++-wrapper-data.txt \
+        --replace-fail \
+          compiler=g++ \
+          compiler=${targetPackages.stdenv.cc}/bin/${targetPackages.stdenv.cc.targetPrefix}c++
     ''
     + lib.optionalString fortranSupport ''
 
       echo "''${!outputDev}/share/openmpi/mpifort-wrapper-data.txt"
-      sed -i 's:compiler=.*:compiler=${gfortran}/bin/${gfortran.targetPrefix}gfortran:'  \
-         ''${!outputDev}/share/openmpi/mpifort-wrapper-data.txt
+      substituteInPlace ''${!outputDev}/share/openmpi/mpifort-wrapper-data.txt \
+        --replace-fail \
+          compiler=gfortran \
+          compiler=${gfortran}/bin/${gfortran.targetPrefix}gfortran
 
     '';
 

From 13e4a573de392bc9466edb7d6ed50dd5557f3292 Mon Sep 17 00:00:00 2001
From: Doron Behar <doron.behar@gmail.com>
Date: Tue, 16 Jul 2024 13:09:59 +0300
Subject: [PATCH 14/20] openmpi: fix gfortran targetPackages Nix attribute

---
 pkgs/development/libraries/openmpi/default.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkgs/development/libraries/openmpi/default.nix b/pkgs/development/libraries/openmpi/default.nix
index 121d212b959d..02e782d6ff36 100644
--- a/pkgs/development/libraries/openmpi/default.nix
+++ b/pkgs/development/libraries/openmpi/default.nix
@@ -168,7 +168,7 @@ stdenv.mkDerivation (finalAttrs: {
       substituteInPlace ''${!outputDev}/share/openmpi/mpifort-wrapper-data.txt \
         --replace-fail \
           compiler=gfortran \
-          compiler=${gfortran}/bin/${gfortran.targetPrefix}gfortran
+          compiler=${targetPackages.gfortran}/bin/${targetPackages.gfortran.targetPrefix}gfortran
 
     '';
 

From 36494fe30f07f33c3d456e0e0c546ca13a6c6306 Mon Sep 17 00:00:00 2001
From: Doron Behar <doron.behar@gmail.com>
Date: Tue, 16 Jul 2024 13:13:18 +0300
Subject: [PATCH 15/20] openmpi: Handle all compiler wrappers consistently &
 declaratively

---
 .../development/libraries/openmpi/default.nix | 124 ++++++++++++------
 1 file changed, 84 insertions(+), 40 deletions(-)

diff --git a/pkgs/development/libraries/openmpi/default.nix b/pkgs/development/libraries/openmpi/default.nix
index 02e782d6ff36..9d3eed1a8dbf 100644
--- a/pkgs/development/libraries/openmpi/default.nix
+++ b/pkgs/development/libraries/openmpi/default.nix
@@ -107,24 +107,98 @@ stdenv.mkDerivation (finalAttrs: {
 
   enableParallelBuilding = true;
 
-  postInstall = ''
+  postInstall = let
+    # The file names we need to iterate are a combination of ${p}${s}, and there
+    # are 7x3 such options. We use lib.mapCartesianProduct to iterate them all.
+    fileNamesToIterate = {
+      p = [
+        "mpi"
+        "shmem"
+        "osh"
+      ];
+      s =
+        [
+          "CC"
+          "c++"
+          "cxx"
+          "cc"
+        ]
+        ++ lib.optionals fortranSupport [
+          "f77"
+          "f90"
+          "fort"
+        ];
+    };
+    wrapperDataSubstitutions =
+      {
+        # The attr key is the filename prefix. The list's 1st value is the
+        # compiler=_ line that should be replaced by a compiler=#2 string, where
+        # #2 is the 2nd value in the list.
+        "cc" = [
+          "gcc"
+          "${targetPackages.stdenv.cc}/bin/${targetPackages.stdenv.cc.targetPrefix}cc"
+        ];
+        "c++" = [
+          "g++"
+          "${targetPackages.stdenv.cc}/bin/${targetPackages.stdenv.cc.targetPrefix}c++"
+        ];
+      }
+      // lib.optionalAttrs fortranSupport {
+        "fort" = [
+          "gfortran"
+          "${targetPackages.gfortran}/bin/${targetPackages.gfortran.targetPrefix}gfortran"
+        ];
+      };
+    # The -wrapper-data.txt files that are not symlinks, need to be iterated as
+    # well, here they start withw ${part1}${part2}, and we use
+    # lib.mapCartesianProduct as well.
+    wrapperDataFileNames = {
+      part1 = [
+        "mpi"
+        "shmem"
+      ];
+      part2 = builtins.attrNames wrapperDataSubstitutions;
+    };
+  in ''
     find $out/lib/ -name "*.la" -exec rm -f \{} \;
 
     # The main wrapper that all the rest of the commonly used binaries are
     # symlinked to
     moveToOutput "bin/opal_wrapper" "''${!outputDev}"
-    for f in mpi shmem osh; do
-      for i in f77 f90 CC c++ cxx cc fort; do
-        moveToOutput "bin/$f$i" "''${!outputDev}"
-        echo "move $fi$i"
-        moveToOutput "share/openmpi/$f$i-wrapper-data.txt" "''${!outputDev}"
-      done
-    done
-
+    # All of the following files are symlinks to opal_wrapper
+    ${lib.pipe fileNamesToIterate [
+      (lib.mapCartesianProduct (
+        { p, s }:
+        ''
+          echo "handling ${p}${s}"
+          moveToOutput "bin/${p}${s}" "''${!outputDev}"
+          moveToOutput "share/openmpi/${p}${s}-wrapper-data.txt" "''${!outputDev}"
+        ''
+      ))
+      (lib.concatStringsSep "\n")
+    ]}
+    # default compilers should be indentical to the
+    # compilers at build time
+    ${lib.pipe wrapperDataFileNames [
+      (lib.mapCartesianProduct (
+        { part1, part2 }:
+        ''
+          substituteInPlace "''${!outputDev}/share/openmpi/${part1}${part2}-wrapper-data.txt" \
+            --replace-fail \
+              compiler=${lib.elemAt wrapperDataSubstitutions.${part2} 0} \
+              compiler=${lib.elemAt wrapperDataSubstitutions.${part2} 1}
+        ''
+      ))
+      (lib.concatStringsSep "\n")
+    ]}
     # ortecc's files don't have c++ and fort companions so it is handled
-    # outside the for loops above
+    # outside the Nix concatenations above.
     moveToOutput "bin/ortecc" "''${!outputDev}"
     moveToOutput "share/openmpi/ortecc-wrapper-data.txt" "''${!outputDev}"
+    substituteInPlace "''${!outputDev}/share/openmpi/ortecc-wrapper-data.txt" \
+      --replace-fail \
+        compiler=${lib.elemAt wrapperDataSubstitutions.cc 0} \
+        compiler=${lib.elemAt wrapperDataSubstitutions.cc 1}
 
     for i in orte-info ompi_info oshmem_info; do
       moveToOutput "bin/$i" "''${!outputDev}"
@@ -140,36 +214,6 @@ stdenv.mkDerivation (finalAttrs: {
       wrapProgram "''${!outputDev}/bin/opal_wrapper" \
         --set OPAL_INCLUDEDIR "''${!outputDev}/include" \
         --set OPAL_PKGDATADIR "''${!outputDev}/share/openmpi"
-
-      # default compilers should be indentical to the
-      # compilers at build time
-
-      echo "''${!outputDev}/share/openmpi/mpicc-wrapper-data.txt"
-      substituteInPlace ''${!outputDev}/share/openmpi/mpicc-wrapper-data.txt \
-        --replace-fail \
-          compiler=gcc \
-          compiler=${targetPackages.stdenv.cc}/bin/${targetPackages.stdenv.cc.targetPrefix}cc
-
-      echo "''${!outputDev}/share/openmpi/ortecc-wrapper-data.txt"
-      substituteInPlace ''${!outputDev}/share/openmpi/ortecc-wrapper-data.txt \
-        --replace-fail \
-          compiler=gcc \
-          compiler=${targetPackages.stdenv.cc}/bin/${targetPackages.stdenv.cc.targetPrefix}cc
-
-      echo "''${!outputDev}/share/openmpi/mpic++-wrapper-data.txt"
-      substituteInPlace ''${!outputDev}/share/openmpi/mpic++-wrapper-data.txt \
-        --replace-fail \
-          compiler=g++ \
-          compiler=${targetPackages.stdenv.cc}/bin/${targetPackages.stdenv.cc.targetPrefix}c++
-    ''
-    + lib.optionalString fortranSupport ''
-
-      echo "''${!outputDev}/share/openmpi/mpifort-wrapper-data.txt"
-      substituteInPlace ''${!outputDev}/share/openmpi/mpifort-wrapper-data.txt \
-        --replace-fail \
-          compiler=gfortran \
-          compiler=${targetPackages.gfortran}/bin/${targetPackages.gfortran.targetPrefix}gfortran
-
     '';
 
   doCheck = true;

From 75897919c7575d17563bc77235994c7c8245b572 Mon Sep 17 00:00:00 2001
From: Doron Behar <doron.behar@gmail.com>
Date: Tue, 16 Jul 2024 16:19:51 +0300
Subject: [PATCH 16/20] openmpi: another nixfmt-rfc-style

---
 .../development/libraries/openmpi/default.nix | 207 +++++++++---------
 1 file changed, 104 insertions(+), 103 deletions(-)

diff --git a/pkgs/development/libraries/openmpi/default.nix b/pkgs/development/libraries/openmpi/default.nix
index 9d3eed1a8dbf..690d4ddf3e44 100644
--- a/pkgs/development/libraries/openmpi/default.nix
+++ b/pkgs/development/libraries/openmpi/default.nix
@@ -107,115 +107,116 @@ stdenv.mkDerivation (finalAttrs: {
 
   enableParallelBuilding = true;
 
-  postInstall = let
-    # The file names we need to iterate are a combination of ${p}${s}, and there
-    # are 7x3 such options. We use lib.mapCartesianProduct to iterate them all.
-    fileNamesToIterate = {
-      p = [
-        "mpi"
-        "shmem"
-        "osh"
-      ];
-      s =
-        [
-          "CC"
-          "c++"
-          "cxx"
-          "cc"
-        ]
-        ++ lib.optionals fortranSupport [
-          "f77"
-          "f90"
-          "fort"
-        ];
-    };
-    wrapperDataSubstitutions =
-      {
-        # The attr key is the filename prefix. The list's 1st value is the
-        # compiler=_ line that should be replaced by a compiler=#2 string, where
-        # #2 is the 2nd value in the list.
-        "cc" = [
-          "gcc"
-          "${targetPackages.stdenv.cc}/bin/${targetPackages.stdenv.cc.targetPrefix}cc"
-        ];
-        "c++" = [
-          "g++"
-          "${targetPackages.stdenv.cc}/bin/${targetPackages.stdenv.cc.targetPrefix}c++"
-        ];
-      }
-      // lib.optionalAttrs fortranSupport {
-        "fort" = [
-          "gfortran"
-          "${targetPackages.gfortran}/bin/${targetPackages.gfortran.targetPrefix}gfortran"
+  postInstall =
+    let
+      # The file names we need to iterate are a combination of ${p}${s}, and there
+      # are 7x3 such options. We use lib.mapCartesianProduct to iterate them all.
+      fileNamesToIterate = {
+        p = [
+          "mpi"
+          "shmem"
+          "osh"
         ];
+        s =
+          [
+            "CC"
+            "c++"
+            "cxx"
+            "cc"
+          ]
+          ++ lib.optionals fortranSupport [
+            "f77"
+            "f90"
+            "fort"
+          ];
       };
-    # The -wrapper-data.txt files that are not symlinks, need to be iterated as
-    # well, here they start withw ${part1}${part2}, and we use
-    # lib.mapCartesianProduct as well.
-    wrapperDataFileNames = {
-      part1 = [
-        "mpi"
-        "shmem"
-      ];
-      part2 = builtins.attrNames wrapperDataSubstitutions;
-    };
-  in ''
-    find $out/lib/ -name "*.la" -exec rm -f \{} \;
-
-    # The main wrapper that all the rest of the commonly used binaries are
-    # symlinked to
-    moveToOutput "bin/opal_wrapper" "''${!outputDev}"
-    # All of the following files are symlinks to opal_wrapper
-    ${lib.pipe fileNamesToIterate [
-      (lib.mapCartesianProduct (
-        { p, s }:
-        ''
-          echo "handling ${p}${s}"
-          moveToOutput "bin/${p}${s}" "''${!outputDev}"
-          moveToOutput "share/openmpi/${p}${s}-wrapper-data.txt" "''${!outputDev}"
-        ''
-      ))
-      (lib.concatStringsSep "\n")
-    ]}
-    # default compilers should be indentical to the
-    # compilers at build time
-    ${lib.pipe wrapperDataFileNames [
-      (lib.mapCartesianProduct (
-        { part1, part2 }:
-        ''
-          substituteInPlace "''${!outputDev}/share/openmpi/${part1}${part2}-wrapper-data.txt" \
-            --replace-fail \
-              compiler=${lib.elemAt wrapperDataSubstitutions.${part2} 0} \
-              compiler=${lib.elemAt wrapperDataSubstitutions.${part2} 1}
-        ''
-      ))
-      (lib.concatStringsSep "\n")
-    ]}
-    # ortecc's files don't have c++ and fort companions so it is handled
-    # outside the Nix concatenations above.
-    moveToOutput "bin/ortecc" "''${!outputDev}"
-    moveToOutput "share/openmpi/ortecc-wrapper-data.txt" "''${!outputDev}"
-    substituteInPlace "''${!outputDev}/share/openmpi/ortecc-wrapper-data.txt" \
-      --replace-fail \
-        compiler=${lib.elemAt wrapperDataSubstitutions.cc 0} \
-        compiler=${lib.elemAt wrapperDataSubstitutions.cc 1}
-
-    for i in orte-info ompi_info oshmem_info; do
-      moveToOutput "bin/$i" "''${!outputDev}"
-    done
-  '';
-
-  postFixup =
+      wrapperDataSubstitutions =
+        {
+          # The attr key is the filename prefix. The list's 1st value is the
+          # compiler=_ line that should be replaced by a compiler=#2 string, where
+          # #2 is the 2nd value in the list.
+          "cc" = [
+            "gcc"
+            "${targetPackages.stdenv.cc}/bin/${targetPackages.stdenv.cc.targetPrefix}cc"
+          ];
+          "c++" = [
+            "g++"
+            "${targetPackages.stdenv.cc}/bin/${targetPackages.stdenv.cc.targetPrefix}c++"
+          ];
+        }
+        // lib.optionalAttrs fortranSupport {
+          "fort" = [
+            "gfortran"
+            "${targetPackages.gfortran}/bin/${targetPackages.gfortran.targetPrefix}gfortran"
+          ];
+        };
+      # The -wrapper-data.txt files that are not symlinks, need to be iterated as
+      # well, here they start withw ${part1}${part2}, and we use
+      # lib.mapCartesianProduct as well.
+      wrapperDataFileNames = {
+        part1 = [
+          "mpi"
+          "shmem"
+        ];
+        part2 = builtins.attrNames wrapperDataSubstitutions;
+      };
+    in
     ''
-      remove-references-to -t "''${!outputDev}" $(readlink -f $out/lib/libopen-pal${stdenv.hostPlatform.extensions.sharedLibrary})
-      remove-references-to -t "''${!outputMan}" $(readlink -f $out/lib/libopen-pal${stdenv.hostPlatform.extensions.sharedLibrary})
+      find $out/lib/ -name "*.la" -exec rm -f \{} \;
 
-      # The path to the wrapper is hard coded in libopen-pal.so, which we just cleared.
-      wrapProgram "''${!outputDev}/bin/opal_wrapper" \
-        --set OPAL_INCLUDEDIR "''${!outputDev}/include" \
-        --set OPAL_PKGDATADIR "''${!outputDev}/share/openmpi"
+      # The main wrapper that all the rest of the commonly used binaries are
+      # symlinked to
+      moveToOutput "bin/opal_wrapper" "''${!outputDev}"
+      # All of the following files are symlinks to opal_wrapper
+      ${lib.pipe fileNamesToIterate [
+        (lib.mapCartesianProduct (
+          { p, s }:
+          ''
+            echo "handling ${p}${s}"
+            moveToOutput "bin/${p}${s}" "''${!outputDev}"
+            moveToOutput "share/openmpi/${p}${s}-wrapper-data.txt" "''${!outputDev}"
+          ''
+        ))
+        (lib.concatStringsSep "\n")
+      ]}
+      # default compilers should be indentical to the
+      # compilers at build time
+      ${lib.pipe wrapperDataFileNames [
+        (lib.mapCartesianProduct (
+          { part1, part2 }:
+          ''
+            substituteInPlace "''${!outputDev}/share/openmpi/${part1}${part2}-wrapper-data.txt" \
+              --replace-fail \
+                compiler=${lib.elemAt wrapperDataSubstitutions.${part2} 0} \
+                compiler=${lib.elemAt wrapperDataSubstitutions.${part2} 1}
+          ''
+        ))
+        (lib.concatStringsSep "\n")
+      ]}
+      # ortecc's files don't have c++ and fort companions so it is handled
+      # outside the Nix concatenations above.
+      moveToOutput "bin/ortecc" "''${!outputDev}"
+      moveToOutput "share/openmpi/ortecc-wrapper-data.txt" "''${!outputDev}"
+      substituteInPlace "''${!outputDev}/share/openmpi/ortecc-wrapper-data.txt" \
+        --replace-fail \
+          compiler=${lib.elemAt wrapperDataSubstitutions.cc 0} \
+          compiler=${lib.elemAt wrapperDataSubstitutions.cc 1}
+
+      for i in orte-info ompi_info oshmem_info; do
+        moveToOutput "bin/$i" "''${!outputDev}"
+      done
     '';
 
+  postFixup = ''
+    remove-references-to -t "''${!outputDev}" $(readlink -f $out/lib/libopen-pal${stdenv.hostPlatform.extensions.sharedLibrary})
+    remove-references-to -t "''${!outputMan}" $(readlink -f $out/lib/libopen-pal${stdenv.hostPlatform.extensions.sharedLibrary})
+
+    # The path to the wrapper is hard coded in libopen-pal.so, which we just cleared.
+    wrapProgram "''${!outputDev}/bin/opal_wrapper" \
+      --set OPAL_INCLUDEDIR "''${!outputDev}/include" \
+      --set OPAL_PKGDATADIR "''${!outputDev}/share/openmpi"
+  '';
+
   doCheck = true;
 
   passthru = {

From bf35ff5b8c1d5c999437b96de4ad4c317d293bb9 Mon Sep 17 00:00:00 2001
From: Doron Behar <doron.behar@gmail.com>
Date: Mon, 15 Jul 2024 13:31:17 +0300
Subject: [PATCH 17/20] openmpi: 4.1.6 -> 5.0.3

https://docs.open-mpi.org/en/v5.0.x/release-notes/changelog/v5.0.x.html

Make build reproducible in a different, nicer manner

Co-authored-by: Markus Kowalewski <markus.kowalewski@gmail.com>
---
 .../development/libraries/openmpi/default.nix | 42 ++++++++++---------
 1 file changed, 22 insertions(+), 20 deletions(-)

diff --git a/pkgs/development/libraries/openmpi/default.nix b/pkgs/development/libraries/openmpi/default.nix
index 690d4ddf3e44..0460b90cea51 100644
--- a/pkgs/development/libraries/openmpi/default.nix
+++ b/pkgs/development/libraries/openmpi/default.nix
@@ -17,6 +17,7 @@
   pmix,
   ucx,
   ucc,
+  prrte,
   makeWrapper,
   config,
   # Enable CUDA support
@@ -34,24 +35,25 @@
 
 stdenv.mkDerivation (finalAttrs: {
   pname = "openmpi";
-  version = "4.1.6";
+  version = "5.0.3";
 
   src = fetchurl {
     url = "https://www.open-mpi.org/software/ompi/v${lib.versions.majorMinor finalAttrs.version}/downloads/${finalAttrs.pname}-${finalAttrs.version}.tar.bz2";
-    sha256 = "sha256-90CZRIVRbetjtTEa8SLCZRefUyig2FelZ7hdsAsR5BU=";
+    sha256 = "sha256-mQWC8gazqzLpOKoxu/B8Y5No5EBdyhlvq+fw927tqQs=";
   };
 
   postPatch = ''
     patchShebangs ./
-
-    # Ensure build is reproducible
-    ts=`date -d @$SOURCE_DATE_EPOCH`
-    sed -i 's/OPAL_CONFIGURE_USER=.*/OPAL_CONFIGURE_USER="nixbld"/' configure
-    sed -i 's/OPAL_CONFIGURE_HOST=.*/OPAL_CONFIGURE_HOST="localhost"/' configure
-    sed -i "s/OPAL_CONFIGURE_DATE=.*/OPAL_CONFIGURE_DATE=\"$ts\"/" configure
-    find -name "Makefile.in" -exec sed -i "s/\`date\`/$ts/" \{} \;
   '';
 
+  # Ensure build is reproducible according to manual
+  # https://docs.open-mpi.org/en/v5.0.x/release-notes/general.html#general-notes
+  env = {
+    USER = "nixbld";
+    HOSTNAME = "localhost";
+    SOURCE_DATE_EPOCH = "0";
+  };
+
   outputs = [
     "out"
     "man"
@@ -70,6 +72,7 @@ stdenv.mkDerivation (finalAttrs: {
       pmix
       ucx
       ucc
+      prrte
     ]
     ++ lib.optionals cudaSupport [ cudaPackages.cuda_cudart ]
     ++ lib.optionals (stdenv.isLinux || stdenv.isFreeBSD) [ rdma-core ]
@@ -93,7 +96,8 @@ stdenv.mkDerivation (finalAttrs: {
     (lib.withFeatureAs stdenv.isLinux "libnl" (lib.getDev libnl))
     (lib.withFeatureAs stdenv.isLinux "pmix" (lib.getDev pmix))
     (lib.withFeatureAs stdenv.isLinux "pmix-libdir" "${lib.getLib pmix}/lib")
-    (lib.enableFeature stdenv.isLinux "mpi-cxx")
+    # Puts a "default OMPI_PRTERUN" value to mpirun / mpiexec executables
+    (lib.withFeatureAs stdenv.isLinux "prrte" (lib.getBin prrte))
     (lib.withFeature enableSGE "sge")
     (lib.enableFeature enablePrefix "mpirun-prefix-by-default")
     # TODO: add UCX support, which is recommended to use with cuda for the most robust OpenMPI build
@@ -102,7 +106,8 @@ stdenv.mkDerivation (finalAttrs: {
     (lib.withFeatureAs cudaSupport "cuda" (lib.getDev cudaPackages.cuda_cudart))
     (lib.enableFeature cudaSupport "dlopen")
     (lib.withFeatureAs fabricSupport "psm2" (lib.getDev libpsm2))
-    (lib.withFeatureAs fabricSupport "libfabric" (lib.getDev libfabric))
+    (lib.withFeatureAs fabricSupport "ofi" (lib.getDev libfabric))
+    (lib.withFeatureAs fabricSupport "ofi-libdir" "${lib.getLib libfabric}/lib")
   ];
 
   enableParallelBuilding = true;
@@ -193,21 +198,18 @@ stdenv.mkDerivation (finalAttrs: {
         ))
         (lib.concatStringsSep "\n")
       ]}
-      # ortecc's files don't have c++ and fort companions so it is handled
-      # outside the Nix concatenations above.
-      moveToOutput "bin/ortecc" "''${!outputDev}"
-      moveToOutput "share/openmpi/ortecc-wrapper-data.txt" "''${!outputDev}"
-      substituteInPlace "''${!outputDev}/share/openmpi/ortecc-wrapper-data.txt" \
-        --replace-fail \
-          compiler=${lib.elemAt wrapperDataSubstitutions.cc 0} \
-          compiler=${lib.elemAt wrapperDataSubstitutions.cc 1}
+      # A symlink to ${lib.getDev pmix}/bin/pmixcc upstreeam puts here as well
+      # from some reason.
+      moveToOutput "bin/pcc" "''${!outputDev}"
 
-      for i in orte-info ompi_info oshmem_info; do
+      # Handle informative binaries about the compilation
+      for i in {prte,ompi,oshmem}_info; do
         moveToOutput "bin/$i" "''${!outputDev}"
       done
     '';
 
   postFixup = ''
+    remove-references-to -t "''${!outputDev}" $out/bin/mpirun
     remove-references-to -t "''${!outputDev}" $(readlink -f $out/lib/libopen-pal${stdenv.hostPlatform.extensions.sharedLibrary})
     remove-references-to -t "''${!outputMan}" $(readlink -f $out/lib/libopen-pal${stdenv.hostPlatform.extensions.sharedLibrary})
 

From bae91146e07c75eab1063fa0e77f3445a01ef2f0 Mon Sep 17 00:00:00 2001
From: Doron Behar <doron.behar@gmail.com>
Date: Tue, 16 Jul 2024 01:07:58 +0300
Subject: [PATCH 18/20] openmpi: support setting avxOptions

Co-authored-by: Markus Kowalewski <markus.kowalewski@gmail.com>
---
 .../development/libraries/openmpi/default.nix | 25 +++++++++++++++++++
 1 file changed, 25 insertions(+)

diff --git a/pkgs/development/libraries/openmpi/default.nix b/pkgs/development/libraries/openmpi/default.nix
index 0460b90cea51..c0d6beb26c2a 100644
--- a/pkgs/development/libraries/openmpi/default.nix
+++ b/pkgs/development/libraries/openmpi/default.nix
@@ -31,6 +31,10 @@
   fabricSupport ? stdenv.isLinux && stdenv.isx86_64,
   # Enable Fortran support
   fortranSupport ? true,
+  # AVX/SSE options. See passthru.defaultAvxOptions for the available options.
+  # note that opempi fails to build with AVX disabled, meaning that everything
+  # up to AVX is enabled by default.
+  avxOptions ? { },
 }:
 
 stdenv.mkDerivation (finalAttrs: {
@@ -44,6 +48,20 @@ stdenv.mkDerivation (finalAttrs: {
 
   postPatch = ''
     patchShebangs ./
+
+    # This is dynamically detected. Configure does not provide fine grained options
+    # We just disable the check in the configure script for now
+    ${lib.pipe (finalAttrs.passthru.defaultAvxOptions // avxOptions) [
+      (lib.mapAttrsToList (
+        option: val: ''
+          substituteInPlace configure \
+            --replace-fail \
+              ompi_cv_op_avx_check_${option}=yes \
+              ompi_cv_op_avx_check_${option}=${if val then "yes" else "no"}
+        ''
+      ))
+      (lib.concatStringsSep "\n")
+    ]}
   '';
 
   # Ensure build is reproducible according to manual
@@ -222,6 +240,13 @@ stdenv.mkDerivation (finalAttrs: {
   doCheck = true;
 
   passthru = {
+    defaultAvxOptions = {
+      sse3 = true;
+      sse41 = true;
+      avx = true;
+      avx2 = stdenv.hostPlatform.avx2Support;
+      avx512 = stdenv.hostPlatform.avx512Support;
+    };
     inherit cudaSupport;
     cudatoolkit = cudaPackages.cudatoolkit; # For backward compatibility only
   };

From 3d70c20fb51518d0a5b4f69056ffb3018bd0fe35 Mon Sep 17 00:00:00 2001
From: Doron Behar <doron.behar@gmail.com>
Date: Tue, 16 Jul 2024 01:31:15 +0300
Subject: [PATCH 19/20] openmpi: fix build on non x86_64-linux platforms

Co-authored-by: Markus Kowalewski <markus.kowalewski@gmail.com>
---
 .../development/libraries/openmpi/default.nix | 42 +++++++++++--------
 1 file changed, 25 insertions(+), 17 deletions(-)

diff --git a/pkgs/development/libraries/openmpi/default.nix b/pkgs/development/libraries/openmpi/default.nix
index c0d6beb26c2a..ef8b215dbc6f 100644
--- a/pkgs/development/libraries/openmpi/default.nix
+++ b/pkgs/development/libraries/openmpi/default.nix
@@ -19,6 +19,7 @@
   ucc,
   prrte,
   makeWrapper,
+  python3,
   config,
   # Enable CUDA support
   cudaSupport ? config.cudaSupport,
@@ -72,11 +73,12 @@ stdenv.mkDerivation (finalAttrs: {
     SOURCE_DATE_EPOCH = "0";
   };
 
-  outputs = [
-    "out"
-    "man"
-    "dev"
-  ];
+  outputs =
+    [ "out" ]
+    ++ lib.optionals stdenv.isLinux [
+      "man"
+      "dev"
+    ];
 
   buildInputs =
     [
@@ -94,6 +96,8 @@ stdenv.mkDerivation (finalAttrs: {
     ]
     ++ lib.optionals cudaSupport [ cudaPackages.cuda_cudart ]
     ++ lib.optionals (stdenv.isLinux || stdenv.isFreeBSD) [ rdma-core ]
+    # needed for internal pmix
+    ++ lib.optionals (!stdenv.isLinux) [ python3 ]
     ++ lib.optionals fabricSupport [
       libpsm2
       libfabric
@@ -112,7 +116,7 @@ stdenv.mkDerivation (finalAttrs: {
     (lib.enableFeature cudaSupport "mca-dso")
     (lib.enableFeature fortranSupport "mpi-fortran")
     (lib.withFeatureAs stdenv.isLinux "libnl" (lib.getDev libnl))
-    (lib.withFeatureAs stdenv.isLinux "pmix" (lib.getDev pmix))
+    "--with-pmix=${if stdenv.isLinux then (lib.getDev pmix) else "internal"}"
     (lib.withFeatureAs stdenv.isLinux "pmix-libdir" "${lib.getLib pmix}/lib")
     # Puts a "default OMPI_PRTERUN" value to mpirun / mpiexec executables
     (lib.withFeatureAs stdenv.isLinux "prrte" (lib.getBin prrte))
@@ -125,8 +129,9 @@ stdenv.mkDerivation (finalAttrs: {
     (lib.enableFeature cudaSupport "dlopen")
     (lib.withFeatureAs fabricSupport "psm2" (lib.getDev libpsm2))
     (lib.withFeatureAs fabricSupport "ofi" (lib.getDev libfabric))
-    (lib.withFeatureAs fabricSupport "ofi-libdir" "${lib.getLib libfabric}/lib")
-  ];
+    # The flag --without-ofi-libdir is not supported from some reason, so we
+    # don't use lib.withFeatureAs
+  ] ++ lib.optionals fabricSupport [ "--with-ofi-libdir=${lib.getLib libfabric}/lib" ];
 
   enableParallelBuilding = true;
 
@@ -226,16 +231,19 @@ stdenv.mkDerivation (finalAttrs: {
       done
     '';
 
-  postFixup = ''
-    remove-references-to -t "''${!outputDev}" $out/bin/mpirun
-    remove-references-to -t "''${!outputDev}" $(readlink -f $out/lib/libopen-pal${stdenv.hostPlatform.extensions.sharedLibrary})
-    remove-references-to -t "''${!outputMan}" $(readlink -f $out/lib/libopen-pal${stdenv.hostPlatform.extensions.sharedLibrary})
+  postFixup =
+    lib.optionalString (lib.elem "man" finalAttrs.outputs) ''
+      remove-references-to -t "''${!outputMan}" $(readlink -f $out/lib/libopen-pal${stdenv.hostPlatform.extensions.sharedLibrary})
+    ''
+    + lib.optionalString (lib.elem "dev" finalAttrs.outputs) ''
+      remove-references-to -t "''${!outputDev}" $out/bin/mpirun
+      remove-references-to -t "''${!outputDev}" $(readlink -f $out/lib/libopen-pal${stdenv.hostPlatform.extensions.sharedLibrary})
 
-    # The path to the wrapper is hard coded in libopen-pal.so, which we just cleared.
-    wrapProgram "''${!outputDev}/bin/opal_wrapper" \
-      --set OPAL_INCLUDEDIR "''${!outputDev}/include" \
-      --set OPAL_PKGDATADIR "''${!outputDev}/share/openmpi"
-  '';
+      # The path to the wrapper is hard coded in libopen-pal.so, which we just cleared.
+      wrapProgram "''${!outputDev}/bin/opal_wrapper" \
+        --set OPAL_INCLUDEDIR "''${!outputDev}/include" \
+        --set OPAL_PKGDATADIR "''${!outputDev}/share/openmpi"
+    '';
 
   doCheck = true;
 

From da423c7975b44fe9476b0e1c98d0621fb9306d61 Mon Sep 17 00:00:00 2001
From: Doron Behar <doron.behar@gmail.com>
Date: Tue, 16 Jul 2024 23:19:16 +0300
Subject: [PATCH 20/20] openmpi: don't use finalAttrs.pname in src.url

---
 pkgs/development/libraries/openmpi/default.nix | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/pkgs/development/libraries/openmpi/default.nix b/pkgs/development/libraries/openmpi/default.nix
index ef8b215dbc6f..c695571ff1f8 100644
--- a/pkgs/development/libraries/openmpi/default.nix
+++ b/pkgs/development/libraries/openmpi/default.nix
@@ -43,7 +43,7 @@ stdenv.mkDerivation (finalAttrs: {
   version = "5.0.3";
 
   src = fetchurl {
-    url = "https://www.open-mpi.org/software/ompi/v${lib.versions.majorMinor finalAttrs.version}/downloads/${finalAttrs.pname}-${finalAttrs.version}.tar.bz2";
+    url = "https://www.open-mpi.org/software/ompi/v${lib.versions.majorMinor finalAttrs.version}/downloads/openmpi-${finalAttrs.version}.tar.bz2";
     sha256 = "sha256-mQWC8gazqzLpOKoxu/B8Y5No5EBdyhlvq+fw927tqQs=";
   };