From 2dd5d704d585aa432d4edf0861d12e8e5f19bc9d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Mancilla?= Date: Sat, 25 Sep 2021 17:11:24 -0300 Subject: [PATCH 1/4] shogun: format derivation --- .../machine-learning/shogun/default.nix | 122 ++++++++++++------ 1 file changed, 81 insertions(+), 41 deletions(-) diff --git a/pkgs/applications/science/machine-learning/shogun/default.nix b/pkgs/applications/science/machine-learning/shogun/default.nix index a7010966c26d..74f661d8eb9d 100644 --- a/pkgs/applications/science/machine-learning/shogun/default.nix +++ b/pkgs/applications/science/machine-learning/shogun/default.nix @@ -1,13 +1,38 @@ -{ stdenv, lib, fetchFromGitHub, fetchpatch, fetchurl, cmake, ctags, swig -# data, compression -, bzip2, curl, hdf5, json_c, xz, lzo, protobuf, snappy -# maths -, blas, lapack, eigen, nlopt, lp_solve, colpack, glpk -# libraries -, libarchive, libxml2 -# extra support -, pythonSupport ? true, pythonPackages ? null -, opencvSupport ? false, opencv ? null +{ lib +, stdenv +, fetchFromGitHub +, fetchpatch +, fetchurl + # build +, cmake +, ctags +, swig + # math +, eigen +, blas +, lapack +, glpk + # data +, protobuf +, json_c +, libxml2 +, hdf5 +, curl + # compression +, libarchive +, bzip2 +, xz +, snappy +, lzo + # more math +, nlopt +, lp_solve +, colpack + # extra support +, pythonSupport ? true +, pythonPackages ? null +, opencvSupport ? false +, opencv ? null , withSvmLight ? false }: @@ -19,8 +44,10 @@ assert (!blas.isILP64) && (!lapack.isILP64); let pname = "shogun"; version = "6.1.4"; + rxcppVersion = "4.0.0"; gtestVersion = "1.8.0"; + srcs = { toolbox = fetchFromGitHub { owner = pname + "-toolbox"; @@ -42,20 +69,10 @@ let in stdenv.mkDerivation rec { - inherit pname version; src = srcs.toolbox; - postUnpack = '' - mkdir -p $sourceRoot/third_party/{rxcpp,gtest} - ln -s ${srcs.rxcpp} $sourceRoot/third_party/rxcpp/v${rxcppVersion}.tar.gz - ln -s ${srcs.gtest} $sourceRoot/third_party/gtest/release-${gtestVersion}.tar.gz - ''; - - # broken - doCheck = false; - patches = [ (fetchpatch { url = "https://github.com/awild82/shogun/commit/365ce4c4c700736d2eec8ba6c975327a5ac2cd9b.patch"; @@ -63,32 +80,55 @@ stdenv.mkDerivation rec { }) ] ++ lib.optional (!withSvmLight) ./svmlight-scrubber.patch; + nativeBuildInputs = [ cmake ]; + + buildInputs = [ + eigen + blas + lapack + glpk + protobuf + json_c + libxml2 + hdf5 + curl + libarchive + bzip2 + xz + snappy + lzo + nlopt + lp_solve + colpack + ctags + swig + ] ++ lib.optionals pythonSupport (with pythonPackages; [ python ply numpy ]) + ++ lib.optional opencvSupport opencv; + + cmakeFlags = let + enableIf = cond: if cond then "ON" else "OFF"; + in [ + "-DBUILD_META_EXAMPLES=${enableIf doCheck}" + "-DCMAKE_VERBOSE_MAKEFILE=${enableIf doCheck}" + "-DENABLE_TESTING=${enableIf doCheck}" + "-DPythonModular=${enableIf pythonSupport}" + "-DOpenCV=${enableIf opencvSupport}" + "-DUSE_SVMLIGHT=${enableIf withSvmLight}" + ]; + CCACHE_DISABLE="1"; CCACHE_DIR=".ccache"; - nativeBuildInputs = [ cmake ]; - buildInputs = with lib; [ - blas lapack bzip2 colpack curl ctags eigen hdf5 json_c lp_solve xz lzo - protobuf nlopt snappy swig (libarchive.dev) libxml2 lapack glpk - ] - ++ optionals (pythonSupport) (with pythonPackages; [ python ply numpy ]) - ++ optional (opencvSupport) opencv; - NIX_CFLAGS_COMPILE="-faligned-new"; - cmakeFlags = - let - onOff = b: if b then "ON" else "OFF"; - flag = n: b: "-D"+n+"="+onOff b; - in - with lib; [ - (flag "ENABLE_TESTING" doCheck) - (flag "BUILD_META_EXAMPLES" doCheck) - (flag "CMAKE_VERBOSE_MAKEFILE:BOOL" doCheck) - (flag "PythonModular" pythonSupport) - (flag "OpenCV" opencvSupport) - (flag "USE_SVMLIGHT" withSvmLight) - ]; + # broken + doCheck = false; + + postUnpack = '' + mkdir -p $sourceRoot/third_party/{rxcpp,gtest} + ln -s ${srcs.rxcpp} $sourceRoot/third_party/rxcpp/v${rxcppVersion}.tar.gz + ln -s ${srcs.gtest} $sourceRoot/third_party/gtest/release-${gtestVersion}.tar.gz + ''; postPatch = '' # Fix preprocessing SVMlight code From 1e25b1831d1f5e9f7cf467287ef8825178ad4c34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Mancilla?= Date: Sat, 25 Sep 2021 23:49:23 -0300 Subject: [PATCH 2/4] shogun: add smancill to maintainers --- pkgs/applications/science/machine-learning/shogun/default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/applications/science/machine-learning/shogun/default.nix b/pkgs/applications/science/machine-learning/shogun/default.nix index 74f661d8eb9d..55bf4378dde2 100644 --- a/pkgs/applications/science/machine-learning/shogun/default.nix +++ b/pkgs/applications/science/machine-learning/shogun/default.nix @@ -150,6 +150,6 @@ stdenv.mkDerivation rec { description = "A toolbox which offers a wide range of efficient and unified machine learning methods"; homepage = "http://shogun-toolbox.org/"; license = if withSvmLight then licenses.unfree else licenses.gpl3Plus; - maintainers = with maintainers; [ edwtjo ]; + maintainers = with maintainers; [ edwtjo smancill ]; }; } From 82568e67187dcfbdd9536e4fb82cebac90c01ea3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Mancilla?= Date: Sat, 25 Sep 2021 17:41:16 -0300 Subject: [PATCH 3/4] shogun: refactor build Main changes are: - Build with default stdenv instead of forcing GCC 8, by applying upstream patches that fix building with newer compilers. Together with #134390 and #134393 (and the blas library fixes on #135493 and #136535), this enables building the derivation on darwin. - Use multiple outputs. - Move build-only inputs to nativeBuildInputs. - Generate the meta examples but do not compile them (they are compiled as part of the tests), to not make the build take more time than already is. Install the example source files into $doc (but do not install the example binaries, only the sources). - Enable testing by setting the CMake variable CMAKE_SKIP_BUILD_RPATH to false (it should not be needed if #108496 is merged). This allows the tests to locate the shogun library and run without problems. Disable integration and meta tests to speed up the build. With a few other minor changes: - Use upstream URL for the json-c patch. - Be explicit about which optional, commercial dependencies are not used, to stop CMake searching for them. - Do not set NIX_CFLAGS_COMPILE with '-faligned-new' because it causes a warning when compiling C files. Set CXXFLAGS instead. - Remove unnecessary ccache variables from the environment. - Fix wrong Google Mock download location, which was forcing CMake to need network access to download the tarball to the right location (which fails within the sandbox). --- .../machine-learning/shogun/default.nix | 62 ++++++++++++++----- pkgs/top-level/all-packages.nix | 11 +--- 2 files changed, 46 insertions(+), 27 deletions(-) diff --git a/pkgs/applications/science/machine-learning/shogun/default.nix b/pkgs/applications/science/machine-learning/shogun/default.nix index 55bf4378dde2..06a799b5182e 100644 --- a/pkgs/applications/science/machine-learning/shogun/default.nix +++ b/pkgs/applications/science/machine-learning/shogun/default.nix @@ -6,6 +6,7 @@ # build , cmake , ctags +, pythonPackages , swig # math , eigen @@ -30,7 +31,6 @@ , colpack # extra support , pythonSupport ? true -, pythonPackages ? null , opencvSupport ? false , opencv ? null , withSvmLight ? false @@ -56,7 +56,8 @@ let sha256 = "05s9dclmk7x5d7wnnj4qr6r6c827m72a44gizcv09lxr28pr9inz"; fetchSubmodules = true; }; - # we need the packed archive + + # The CMake external projects expect the packed archives rxcpp = fetchurl { url = "https://github.com/Reactive-Extensions/RxCpp/archive/v${rxcppVersion}.tar.gz"; sha256 = "0y2isr8dy2n1yjr9c5570kpc9lvdlch6jv0jvw000amwn5d3krsh"; @@ -71,16 +72,33 @@ in stdenv.mkDerivation rec { inherit pname version; + outputs = [ "out" "dev" "doc" ]; + src = srcs.toolbox; patches = [ + # Fix compile errors with json-c + # https://github.com/shogun-toolbox/shogun/pull/4104 (fetchpatch { - url = "https://github.com/awild82/shogun/commit/365ce4c4c700736d2eec8ba6c975327a5ac2cd9b.patch"; + url = "https://github.com/shogun-toolbox/shogun/commit/365ce4c4c700736d2eec8ba6c975327a5ac2cd9b.patch"; sha256 = "158hqv4xzw648pmjbwrhxjp7qcppqa7kvriif87gn3zdn711c49s"; }) + + # Fix compile errors with GCC 9+ + # https://github.com/shogun-toolbox/shogun/pull/4811 + (fetchpatch { + url = "https://github.com/shogun-toolbox/shogun/commit/c8b670be4790e0f06804b048a6f3d77c17c3ee95.patch"; + sha256 = "sha256-MxsR3Y2noFQevfqWK3nmX5iK4OVWeKBl5tfeDNgjcXk="; + }) + (fetchpatch { + url = "https://github.com/shogun-toolbox/shogun/commit/5aceefd9fb0e2132c354b9a0c0ceb9160cc9b2f7.patch"; + sha256 = "sha256-AgJJKQA8vc5oKaTQDqMdwBR4hT4sn9+uW0jLe7GteJw="; + }) + ] ++ lib.optional (!withSvmLight) ./svmlight-scrubber.patch; - nativeBuildInputs = [ cmake ]; + nativeBuildInputs = [ cmake swig ctags ] + ++ (with pythonPackages; [ python jinja2 ply ]); buildInputs = [ eigen @@ -100,34 +118,37 @@ stdenv.mkDerivation rec { nlopt lp_solve colpack - ctags - swig - ] ++ lib.optionals pythonSupport (with pythonPackages; [ python ply numpy ]) + ] ++ lib.optionals pythonSupport (with pythonPackages; [ python numpy ]) ++ lib.optional opencvSupport opencv; cmakeFlags = let enableIf = cond: if cond then "ON" else "OFF"; in [ - "-DBUILD_META_EXAMPLES=${enableIf doCheck}" - "-DCMAKE_VERBOSE_MAKEFILE=${enableIf doCheck}" + "-DBUILD_META_EXAMPLES=ON" + "-DCMAKE_DISABLE_FIND_PACKAGE_ARPACK=ON" + "-DCMAKE_DISABLE_FIND_PACKAGE_ARPREC=ON" + "-DCMAKE_DISABLE_FIND_PACKAGE_CPLEX=ON" + "-DCMAKE_DISABLE_FIND_PACKAGE_Mosek=ON" + "-DCMAKE_DISABLE_FIND_PACKAGE_TFLogger=ON" + "-DCMAKE_DISABLE_FIND_PACKAGE_ViennaCL=ON" + "-DCMAKE_SKIP_BUILD_RPATH=OFF" + "-DCMAKE_CTEST_ARGUMENTS='--exclude-regex;TrainedModelSerialization'" # Sporadic segfault "-DENABLE_TESTING=${enableIf doCheck}" + "-DDISABLE_META_INTEGRATION_TESTS=ON" + "-DTRAVIS_DISABLE_META_CPP=ON" "-DPythonModular=${enableIf pythonSupport}" "-DOpenCV=${enableIf opencvSupport}" "-DUSE_SVMLIGHT=${enableIf withSvmLight}" ]; - CCACHE_DISABLE="1"; - CCACHE_DIR=".ccache"; + CXXFLAGS = "-faligned-new"; - NIX_CFLAGS_COMPILE="-faligned-new"; - - # broken - doCheck = false; + doCheck = true; postUnpack = '' - mkdir -p $sourceRoot/third_party/{rxcpp,gtest} + mkdir -p $sourceRoot/third_party/{rxcpp,GoogleMock} ln -s ${srcs.rxcpp} $sourceRoot/third_party/rxcpp/v${rxcppVersion}.tar.gz - ln -s ${srcs.gtest} $sourceRoot/third_party/gtest/release-${gtestVersion}.tar.gz + ln -s ${srcs.gtest} $sourceRoot/third_party/GoogleMock/release-${gtestVersion}.tar.gz ''; postPatch = '' @@ -146,6 +167,13 @@ stdenv.mkDerivation rec { ./scripts/light-scrubber.sh ''; + postInstall = '' + mkdir -p $doc/share/doc/shogun/examples + mv $out/share/shogun/examples/cpp $doc/share/doc/shogun/examples + cp ../examples/undocumented/libshogun/*.cpp $doc/share/doc/shogun/examples/cpp + rm -r $out/share + ''; + meta = with lib; { description = "A toolbox which offers a wide range of efficient and unified machine learning methods"; homepage = "http://shogun-toolbox.org/"; diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 56c15d52b3d5..397532262356 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -26918,16 +26918,7 @@ with pkgs; shotcut = libsForQt5.callPackage ../applications/video/shotcut { }; shogun = callPackage ../applications/science/machine-learning/shogun { - stdenv = gcc8Stdenv; - - # Workaround for the glibc abi version mismatch. - # Please note that opencv builds are by default disabled. - opencv = opencv3.override { - stdenv = gcc8Stdenv; - openexr = openexr.override { - stdenv = gcc8Stdenv; - }; - }; + opencv = opencv3; }; smplayer = libsForQt5.callPackage ../applications/video/smplayer { }; From 203db44ed420cb4032b1a8b57a70f5177255cc9e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Sebasti=C3=A1n=20Mancilla?= Date: Mon, 8 Nov 2021 22:58:00 -0300 Subject: [PATCH 4/4] shogun: fix compile errors with Eigen 3.4 Fixes #142811. --- .../machine-learning/shogun/default.nix | 3 + .../machine-learning/shogun/eigen-3.4.patch | 74 +++++++++++++++++++ 2 files changed, 77 insertions(+) create mode 100644 pkgs/applications/science/machine-learning/shogun/eigen-3.4.patch diff --git a/pkgs/applications/science/machine-learning/shogun/default.nix b/pkgs/applications/science/machine-learning/shogun/default.nix index 06a799b5182e..911d0043ed26 100644 --- a/pkgs/applications/science/machine-learning/shogun/default.nix +++ b/pkgs/applications/science/machine-learning/shogun/default.nix @@ -95,6 +95,9 @@ stdenv.mkDerivation rec { sha256 = "sha256-AgJJKQA8vc5oKaTQDqMdwBR4hT4sn9+uW0jLe7GteJw="; }) + # Fix compile errors with Eigen 3.4 + ./eigen-3.4.patch + ] ++ lib.optional (!withSvmLight) ./svmlight-scrubber.patch; nativeBuildInputs = [ cmake swig ctags ] diff --git a/pkgs/applications/science/machine-learning/shogun/eigen-3.4.patch b/pkgs/applications/science/machine-learning/shogun/eigen-3.4.patch new file mode 100644 index 000000000000..863bd75918b0 --- /dev/null +++ b/pkgs/applications/science/machine-learning/shogun/eigen-3.4.patch @@ -0,0 +1,74 @@ +From: Sebastián Mancilla +Subject: [PATCH] Fix compile errors when using Eigen 3.4 + +--- + .../machine/gp/MultiLaplaceInferenceMethod.cpp | 18 +++++++++--------- + 1 file changed, 9 insertions(+), 9 deletions(-) + +diff --git a/src/shogun/machine/gp/MultiLaplaceInferenceMethod.cpp b/src/shogun/machine/gp/MultiLaplaceInferenceMethod.cpp +index 2e27678d2..60050afea 100644 +--- a/src/shogun/machine/gp/MultiLaplaceInferenceMethod.cpp ++++ b/src/shogun/machine/gp/MultiLaplaceInferenceMethod.cpp +@@ -84,9 +84,9 @@ class CMultiPsiLine : public func_base + float64_t result=0; + for(index_t bl=0; blblock(bl*n,0,n,1)*CMath::exp(log_scale*2.0); +- result+=alpha->block(bl*n,0,n,1).dot(eigen_f.block(bl*n,0,n,1))/2.0; +- eigen_f.block(bl*n,0,n,1)+=eigen_m; ++ eigen_f.segment(bl*n,n)=K*alpha->segment(bl*n,n)*CMath::exp(log_scale*2.0); ++ result+=alpha->segment(bl*n,n).dot(eigen_f.segment(bl*n,n))/2.0; ++ eigen_f.segment(bl*n,n)+=eigen_m; + } + + // get first and second derivatives of log likelihood +@@ -272,7 +272,7 @@ void CMultiLaplaceInferenceMethod::update_alpha() + { + Map alpha(m_alpha.vector, m_alpha.vlen); + for(index_t bl=0; bl chol_tmp((eigen_sD*eigen_sD.transpose()).cwiseProduct(eigen_ktrtr*CMath::exp(m_log_scale*2.0))+ + MatrixXd::Identity(m_ktrtr.num_rows, m_ktrtr.num_cols)); + MatrixXd eigen_L_tmp=chol_tmp.matrixU(); +@@ -341,11 +341,11 @@ void CMultiLaplaceInferenceMethod::update_alpha() + VectorXd tmp2=m_tmp.array().rowwise().sum(); + + for(index_t bl=0; bl &eigen_c=eigen_W; + for(index_t bl=0; bl c_tmp(eigen_c.data(),n,C); + +@@ -409,7 +409,7 @@ float64_t CMultiLaplaceInferenceMethod::get_derivative_helper(SGMatrix CMultiLaplaceInferenceMethod::get_derivative_wrt_mean( + result[i]=0; + //currently only compute the explicit term + for(index_t bl=0; bl