Merge pull request #145161 from smancill/shogun-fix-build

shogun: refactor derivation and fix build

pkgs/applications/science/machine-learning/shogun/default.nix

@@ -1,13 +1,38 @@
-{ stdenv, lib, fetchFromGitHub, fetchpatch, fetchurl, cmake, ctags, swig
-# data, compression
-, bzip2, curl, hdf5, json_c, xz, lzo, protobuf, snappy
-# maths
-, blas, lapack, eigen, nlopt, lp_solve, colpack, glpk
-# libraries
-, libarchive, libxml2
-# extra support
-, pythonSupport ? true, pythonPackages ? null
-, opencvSupport ? false, opencv ? null
+{ lib
+, stdenv
+, fetchFromGitHub
+, fetchpatch
+, fetchurl
+  # build
+, cmake
+, ctags
+, pythonPackages
+, swig
+  # math
+, eigen
+, blas
+, lapack
+, glpk
+  # data
+, protobuf
+, json_c
+, libxml2
+, hdf5
+, curl
+  # compression
+, libarchive
+, bzip2
+, xz
+, snappy
+, lzo
+  # more math
+, nlopt
+, lp_solve
+, colpack
+  # extra support
+, pythonSupport ? true
+, opencvSupport ? false
+, opencv ? null
 , withSvmLight ? false
 }:
 
@@ -19,8 +44,10 @@ assert (!blas.isILP64) && (!lapack.isILP64);
 let
   pname = "shogun";
   version = "6.1.4";
+
   rxcppVersion = "4.0.0";
   gtestVersion = "1.8.0";
+
   srcs = {
     toolbox = fetchFromGitHub {
       owner = pname + "-toolbox";
@@ -29,7 +56,8 @@ let
       sha256 = "05s9dclmk7x5d7wnnj4qr6r6c827m72a44gizcv09lxr28pr9inz";
       fetchSubmodules = true;
     };
-    # we need the packed archive
+
+    # The CMake external projects expect the packed archives
     rxcpp = fetchurl {
       url = "https://github.com/Reactive-Extensions/RxCpp/archive/v${rxcppVersion}.tar.gz";
       sha256 = "0y2isr8dy2n1yjr9c5570kpc9lvdlch6jv0jvw000amwn5d3krsh";
@@ -42,54 +70,90 @@ let
 in
 
 stdenv.mkDerivation rec {
-
   inherit pname version;
 
+  outputs = [ "out" "dev" "doc" ];
+
   src = srcs.toolbox;
 
-  postUnpack = ''
-    mkdir -p $sourceRoot/third_party/{rxcpp,gtest}
-    ln -s ${srcs.rxcpp} $sourceRoot/third_party/rxcpp/v${rxcppVersion}.tar.gz
-    ln -s ${srcs.gtest} $sourceRoot/third_party/gtest/release-${gtestVersion}.tar.gz
-  '';
-
-  # broken
-  doCheck = false;
-
   patches = [
+    # Fix compile errors with json-c
+    # https://github.com/shogun-toolbox/shogun/pull/4104
     (fetchpatch {
-      url = "https://github.com/awild82/shogun/commit/365ce4c4c700736d2eec8ba6c975327a5ac2cd9b.patch";
+      url = "https://github.com/shogun-toolbox/shogun/commit/365ce4c4c700736d2eec8ba6c975327a5ac2cd9b.patch";
       sha256 = "158hqv4xzw648pmjbwrhxjp7qcppqa7kvriif87gn3zdn711c49s";
     })
+
+    # Fix compile errors with GCC 9+
+    # https://github.com/shogun-toolbox/shogun/pull/4811
+    (fetchpatch {
+      url = "https://github.com/shogun-toolbox/shogun/commit/c8b670be4790e0f06804b048a6f3d77c17c3ee95.patch";
+      sha256 = "sha256-MxsR3Y2noFQevfqWK3nmX5iK4OVWeKBl5tfeDNgjcXk=";
+    })
+    (fetchpatch {
+      url = "https://github.com/shogun-toolbox/shogun/commit/5aceefd9fb0e2132c354b9a0c0ceb9160cc9b2f7.patch";
+      sha256 = "sha256-AgJJKQA8vc5oKaTQDqMdwBR4hT4sn9+uW0jLe7GteJw=";
+    })
+
+    # Fix compile errors with Eigen 3.4
+    ./eigen-3.4.patch
+
   ] ++ lib.optional (!withSvmLight) ./svmlight-scrubber.patch;
 
-  CCACHE_DISABLE="1";
-  CCACHE_DIR=".ccache";
+  nativeBuildInputs = [ cmake swig ctags ]
+    ++ (with pythonPackages; [ python jinja2 ply ]);
 
-  nativeBuildInputs = [ cmake ];
-  buildInputs = with lib; [
-      blas lapack bzip2 colpack curl ctags eigen hdf5 json_c lp_solve xz lzo
-      protobuf nlopt snappy swig (libarchive.dev) libxml2 lapack glpk
-    ]
-    ++ optionals (pythonSupport) (with pythonPackages; [ python ply numpy ])
-    ++ optional  (opencvSupport) opencv;
+  buildInputs = [
+    eigen
+    blas
+    lapack
+    glpk
+    protobuf
+    json_c
+    libxml2
+    hdf5
+    curl
+    libarchive
+    bzip2
+    xz
+    snappy
+    lzo
+    nlopt
+    lp_solve
+    colpack
+  ] ++ lib.optionals pythonSupport (with pythonPackages; [ python numpy ])
+    ++ lib.optional opencvSupport opencv;
 
-  NIX_CFLAGS_COMPILE="-faligned-new";
-
-  cmakeFlags =
-  let
-      onOff = b: if b then "ON" else "OFF";
-      flag = n: b: "-D"+n+"="+onOff b;
-  in
-  with lib; [
-    (flag "ENABLE_TESTING" doCheck)
-    (flag "BUILD_META_EXAMPLES" doCheck)
-    (flag "CMAKE_VERBOSE_MAKEFILE:BOOL" doCheck)
-    (flag "PythonModular" pythonSupport)
-    (flag "OpenCV" opencvSupport)
-    (flag "USE_SVMLIGHT" withSvmLight)
+  cmakeFlags = let
+    enableIf = cond: if cond then "ON" else "OFF";
+  in [
+    "-DBUILD_META_EXAMPLES=ON"
+    "-DCMAKE_DISABLE_FIND_PACKAGE_ARPACK=ON"
+    "-DCMAKE_DISABLE_FIND_PACKAGE_ARPREC=ON"
+    "-DCMAKE_DISABLE_FIND_PACKAGE_CPLEX=ON"
+    "-DCMAKE_DISABLE_FIND_PACKAGE_Mosek=ON"
+    "-DCMAKE_DISABLE_FIND_PACKAGE_TFLogger=ON"
+    "-DCMAKE_DISABLE_FIND_PACKAGE_ViennaCL=ON"
+    "-DCMAKE_SKIP_BUILD_RPATH=OFF"
+    "-DCMAKE_CTEST_ARGUMENTS='--exclude-regex;TrainedModelSerialization'"  # Sporadic segfault
+    "-DENABLE_TESTING=${enableIf doCheck}"
+    "-DDISABLE_META_INTEGRATION_TESTS=ON"
+    "-DTRAVIS_DISABLE_META_CPP=ON"
+    "-DPythonModular=${enableIf pythonSupport}"
+    "-DOpenCV=${enableIf opencvSupport}"
+    "-DUSE_SVMLIGHT=${enableIf withSvmLight}"
   ];
 
+  CXXFLAGS = "-faligned-new";
+
+  doCheck = true;
+
+  postUnpack = ''
+    mkdir -p $sourceRoot/third_party/{rxcpp,GoogleMock}
+    ln -s ${srcs.rxcpp} $sourceRoot/third_party/rxcpp/v${rxcppVersion}.tar.gz
+    ln -s ${srcs.gtest} $sourceRoot/third_party/GoogleMock/release-${gtestVersion}.tar.gz
+  '';
+
   postPatch = ''
     # Fix preprocessing SVMlight code
     sed -i \
@@ -106,10 +170,17 @@ stdenv.mkDerivation rec {
     ./scripts/light-scrubber.sh
   '';
 
+  postInstall = ''
+    mkdir -p $doc/share/doc/shogun/examples
+    mv $out/share/shogun/examples/cpp $doc/share/doc/shogun/examples
+    cp ../examples/undocumented/libshogun/*.cpp $doc/share/doc/shogun/examples/cpp
+    rm -r $out/share
+  '';
+
   meta = with lib; {
     description = "A toolbox which offers a wide range of efficient and unified machine learning methods";
     homepage = "http://shogun-toolbox.org/";
     license = if withSvmLight then licenses.unfree else licenses.gpl3Plus;
-    maintainers = with maintainers; [ edwtjo ];
+    maintainers = with maintainers; [ edwtjo smancill ];
   };
 }

pkgs/applications/science/machine-learning/shogun/eigen-3.4.patch

@@ -0,0 +1,74 @@
+From: Sebastián Mancilla <smancill@smancill.dev>
+Subject: [PATCH] Fix compile errors when using Eigen 3.4
+
+---
+ .../machine/gp/MultiLaplaceInferenceMethod.cpp | 18 +++++++++---------
+ 1 file changed, 9 insertions(+), 9 deletions(-)
+
+diff --git a/src/shogun/machine/gp/MultiLaplaceInferenceMethod.cpp b/src/shogun/machine/gp/MultiLaplaceInferenceMethod.cpp
+index 2e27678d2..60050afea 100644
+--- a/src/shogun/machine/gp/MultiLaplaceInferenceMethod.cpp
++++ b/src/shogun/machine/gp/MultiLaplaceInferenceMethod.cpp
+@@ -84,9 +84,9 @@ class CMultiPsiLine : public func_base
+ 		float64_t result=0;
+ 		for(index_t bl=0; bl<C; bl++)
+ 		{
+-			eigen_f.block(bl*n,0,n,1)=K*alpha->block(bl*n,0,n,1)*CMath::exp(log_scale*2.0);
+-			result+=alpha->block(bl*n,0,n,1).dot(eigen_f.block(bl*n,0,n,1))/2.0;
+-			eigen_f.block(bl*n,0,n,1)+=eigen_m;
++			eigen_f.segment(bl*n,n)=K*alpha->segment(bl*n,n)*CMath::exp(log_scale*2.0);
++			result+=alpha->segment(bl*n,n).dot(eigen_f.segment(bl*n,n))/2.0;
++			eigen_f.segment(bl*n,n)+=eigen_m;
+ 		}
+ 
+ 		// get first and second derivatives of log likelihood
+@@ -272,7 +272,7 @@ void CMultiLaplaceInferenceMethod::update_alpha()
+ 	{
+ 		Map<VectorXd> alpha(m_alpha.vector, m_alpha.vlen);
+ 		for(index_t bl=0; bl<C; bl++)
+-			eigen_mu.block(bl*n,0,n,1)=eigen_ktrtr*CMath::exp(m_log_scale*2.0)*alpha.block(bl*n,0,n,1);
++			eigen_mu.segment(bl*n,n)=eigen_ktrtr*CMath::exp(m_log_scale*2.0)*alpha.segment(bl*n,n);
+ 
+ 		//alpha'*(f-m)/2.0
+ 		Psi_New=alpha.dot(eigen_mu)/2.0;
+@@ -316,7 +316,7 @@ void CMultiLaplaceInferenceMethod::update_alpha()
+ 
+ 		for(index_t bl=0; bl<C; bl++)
+ 		{
+-			VectorXd eigen_sD=eigen_dpi.block(bl*n,0,n,1).cwiseSqrt();
++			VectorXd eigen_sD=eigen_dpi.segment(bl*n,n).cwiseSqrt();
+ 			LLT<MatrixXd> chol_tmp((eigen_sD*eigen_sD.transpose()).cwiseProduct(eigen_ktrtr*CMath::exp(m_log_scale*2.0))+
+ 				MatrixXd::Identity(m_ktrtr.num_rows, m_ktrtr.num_cols));
+ 			MatrixXd eigen_L_tmp=chol_tmp.matrixU();
+@@ -341,11 +341,11 @@ void CMultiLaplaceInferenceMethod::update_alpha()
+ 		VectorXd tmp2=m_tmp.array().rowwise().sum();
+ 
+ 		for(index_t bl=0; bl<C; bl++)
+-			eigen_b.block(bl*n,0,n,1)+=eigen_dpi.block(bl*n,0,n,1).cwiseProduct(eigen_mu.block(bl*n,0,n,1)-eigen_mean_bl-tmp2);
++			eigen_b.segment(bl*n,n)+=eigen_dpi.segment(bl*n,n).cwiseProduct(eigen_mu.segment(bl*n,n)-eigen_mean_bl-tmp2);
+ 
+ 		Map<VectorXd> &eigen_c=eigen_W;
+ 		for(index_t bl=0; bl<C; bl++)
+-			eigen_c.block(bl*n,0,n,1)=eigen_E.block(0,bl*n,n,n)*(eigen_ktrtr*CMath::exp(m_log_scale*2.0)*eigen_b.block(bl*n,0,n,1));
++			eigen_c.segment(bl*n,n)=eigen_E.block(0,bl*n,n,n)*(eigen_ktrtr*CMath::exp(m_log_scale*2.0)*eigen_b.segment(bl*n,n));
+ 
+ 		Map<MatrixXd> c_tmp(eigen_c.data(),n,C);
+ 
+@@ -409,7 +409,7 @@ float64_t CMultiLaplaceInferenceMethod::get_derivative_helper(SGMatrix<float64_t
+ 	{
+ 		result+=((eigen_E.block(0,bl*n,n,n)-eigen_U.block(0,bl*n,n,n).transpose()*eigen_U.block(0,bl*n,n,n)).array()
+ 			*eigen_dK.array()).sum();
+-		result-=(eigen_dK*eigen_alpha.block(bl*n,0,n,1)).dot(eigen_alpha.block(bl*n,0,n,1));
++		result-=(eigen_dK*eigen_alpha.segment(bl*n,n)).dot(eigen_alpha.segment(bl*n,n));
+ 	}
+ 
+ 	return result/2.0;
+@@ -489,7 +489,7 @@ SGVector<float64_t> CMultiLaplaceInferenceMethod::get_derivative_wrt_mean(
+ 		result[i]=0;
+ 		//currently only compute the explicit term
+ 		for(index_t bl=0; bl<C; bl++)
+-			result[i]-=eigen_alpha.block(bl*n,0,n,1).dot(eigen_dmu);
++			result[i]-=eigen_alpha.segment(bl*n,n).dot(eigen_dmu);
+ 	}
+ 
+ 	return result;

pkgs/top-level/all-packages.nix

@@ -26935,16 +26935,7 @@ with pkgs;
   shotcut = libsForQt5.callPackage ../applications/video/shotcut { };
 
   shogun = callPackage ../applications/science/machine-learning/shogun {
-    stdenv = gcc8Stdenv;
-
-    # Workaround for the glibc abi version mismatch.
-    # Please note that opencv builds are by default disabled.
-    opencv = opencv3.override {
-      stdenv = gcc8Stdenv;
-      openexr = openexr.override {
-        stdenv = gcc8Stdenv;
-      };
-    };
+    opencv = opencv3;
   };
 
   smplayer = libsForQt5.callPackage ../applications/video/smplayer { };