From ad8148a98bab69cc561b0b47e88365afdfa9718a Mon Sep 17 00:00:00 2001 From: Someone Serge Date: Sat, 9 Apr 2022 23:59:08 +0300 Subject: [PATCH 1/4] faiss: init at 1.7.2 --- .../libraries/science/math/faiss/default.nix | 123 ++++++++++++++++++ .../science/math/faiss/pytest-suite.nix | 30 +++++ pkgs/top-level/all-packages.nix | 5 + pkgs/top-level/python-packages.nix | 5 + 4 files changed, 163 insertions(+) create mode 100644 pkgs/development/libraries/science/math/faiss/default.nix create mode 100644 pkgs/development/libraries/science/math/faiss/pytest-suite.nix diff --git a/pkgs/development/libraries/science/math/faiss/default.nix b/pkgs/development/libraries/science/math/faiss/default.nix new file mode 100644 index 000000000000..5175b9b60961 --- /dev/null +++ b/pkgs/development/libraries/science/math/faiss/default.nix @@ -0,0 +1,123 @@ +{ lib +, config +, fetchFromGitHub +, stdenv +, cmake +, cudaPackages +, cudaSupport ? config.cudaSupport or false +, cudaCapabilities ? [ "60" "70" "80" "86" ] +, pythonSupport ? true +, pythonPackages +, blas +, swig +, addOpenGLRunpath +, optLevel ? let + optLevels = + lib.optional stdenv.hostPlatform.avx2Support "avx2" + ++ lib.optional stdenv.hostPlatform.sse4_1Support "sse4" + ++ [ "generic" ]; + in + # Choose the maximum available optimization level + builtins.head optLevels +, faiss # To run demos in the tests +, runCommand +}: + +let + pname = "faiss"; + version = "1.7.2"; + inherit (cudaPackages) cudatoolkit; +in +stdenv.mkDerivation { + inherit pname version; + + outputs = [ "out" "demos" ]; + + src = fetchFromGitHub { + owner = "facebookresearch"; + repo = pname; + rev = "v${version}"; + hash = "sha256-Tklf5AaqJbOs9qtBZVcxXPLAp+K54EViZLSOvEhmswg="; + }; + + buildInputs = [ + blas + swig + ] ++ lib.optionals pythonSupport [ + pythonPackages.setuptools + pythonPackages.pip + pythonPackages.wheel + ]; + + propagatedBuildInputs = lib.optionals pythonSupport [ + pythonPackages.numpy + ]; + + nativeBuildInputs = [ cmake ] ++ lib.optionals cudaSupport [ + cudatoolkit + addOpenGLRunpath + ] ++ lib.optional pythonSupport [ + pythonPackages.python + ]; + + passthru.extra-requires.all = [ + pythonPackages.numpy + ]; + + cmakeFlags = [ + "-DFAISS_ENABLE_GPU=${if cudaSupport then "ON" else "OFF"}" + "-DFAISS_ENABLE_PYTHON=${if pythonSupport then "ON" else "OFF"}" + "-DFAISS_OPT_LEVEL=${optLevel}" + ] ++ lib.optionals cudaSupport [ + "-DCMAKE_CUDA_ARCHITECTURES=${lib.concatStringsSep ";" cudaCapabilities}" + ]; + + + # pip wheel->pip install commands copied over from opencv4 + + buildPhase = '' + make -j faiss + make demo_ivfpq_indexing + '' + lib.optionalString pythonSupport '' + make -j swigfaiss + (cd faiss/python && + python -m pip wheel --verbose --no-index --no-deps --no-clean --no-build-isolation --wheel-dir dist .) + ''; + + installPhase = '' + make install + mkdir -p $demos/bin + cp ./demos/demo_ivfpq_indexing $demos/bin/ + '' + lib.optionalString pythonSupport '' + mkdir -p $out/${pythonPackages.python.sitePackages} + (cd faiss/python && python -m pip install dist/*.whl --no-index --no-warn-script-location --prefix="$out" --no-cache) + ''; + + fixupPhase = lib.optionalString (pythonSupport && cudaSupport) '' + addOpenGLRunpath $out/${pythonPackages.python.sitePackages}/faiss/*.so + addOpenGLRunpath $demos/bin/* + ''; + + passthru = { + inherit cudaSupport cudaPackages pythonSupport; + + tests = { + runDemos = runCommand "${pname}-run-demos" + { buildInputs = [ faiss.demos ]; } + # There are more demos, we run just the one that documentation mentions + '' + demo_ivfpq_indexing && touch $out + ''; + } // lib.optionalAttrs pythonSupport { + pytest = pythonPackages.callPackage ./pytest-suite.nix { }; + }; + }; + + meta = with lib; { + description = "A library for efficient similarity search and clustering of dense vectors by Facebook Research"; + homepage = "https://github.com/facebookresearch/faiss"; + license = licenses.mit; + platforms = platforms.unix; # Never tested windows + maintainers = with maintainers; [ SomeoneSerge ]; + }; +} diff --git a/pkgs/development/libraries/science/math/faiss/pytest-suite.nix b/pkgs/development/libraries/science/math/faiss/pytest-suite.nix new file mode 100644 index 000000000000..fcf57a8bc9fd --- /dev/null +++ b/pkgs/development/libraries/science/math/faiss/pytest-suite.nix @@ -0,0 +1,30 @@ +{ lib +, buildPythonPackage +, faiss +, scipy +, pytestCheckHook +}: + +assert faiss.pythonSupport; + +buildPythonPackage { + pname = "faiss-pytest-suite"; + inherit (faiss) version; + + src = "${faiss.src}/tests"; + + dontBuild = true; + dontInstall = true; + + # Tests that need GPUs and would fail in the sandbox + disabledTestPaths = lib.optionals faiss.cudaSupport [ + "test_contrib.py" + ]; + + checkInputs = [ + faiss + pytestCheckHook + scipy + ] ++ + faiss.extra-requires.all; +} diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index dd8dbfa2838e..202243efcc47 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -34477,6 +34477,11 @@ with pkgs; jre = openjdk11; }; + faiss = callPackage ../development/libraries/science/math/faiss { + pythonPackages = python3Packages; + swig = swig4; + }; + fityk = callPackage ../applications/science/misc/fityk { }; galario = callPackage ../development/libraries/galario { }; diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index ccdf1ba1b857..d4c6c2731d17 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -2988,6 +2988,11 @@ in { factory_boy = callPackage ../development/python-modules/factory_boy { }; + faiss = toPythonModule (pkgs.faiss.override { + pythonSupport = true; + pythonPackages = self; + }); + fake-useragent = callPackage ../development/python-modules/fake-useragent { }; faker = callPackage ../development/python-modules/faker { }; From 13aaa027b5fa5d4fdbdaac7ae9eef2fe9ef97005 Mon Sep 17 00:00:00 2001 From: Someone Serge Date: Mon, 11 Apr 2022 03:46:17 +0300 Subject: [PATCH 2/4] faiss: comment on why swig4 --- pkgs/top-level/all-packages.nix | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkgs/top-level/all-packages.nix b/pkgs/top-level/all-packages.nix index 202243efcc47..7b35a721202b 100644 --- a/pkgs/top-level/all-packages.nix +++ b/pkgs/top-level/all-packages.nix @@ -34479,6 +34479,8 @@ with pkgs; faiss = callPackage ../development/libraries/science/math/faiss { pythonPackages = python3Packages; + # faiss wants the "-doxygen" option + # available only since swig4 swig = swig4; }; From ad5e59c123e10f09d7fb9ae6895454dea6902eac Mon Sep 17 00:00:00 2001 From: Someone Serge Date: Tue, 2 Aug 2022 16:22:32 +0300 Subject: [PATCH 3/4] python3Packages.faiss: rm comment about windows --- pkgs/development/libraries/science/math/faiss/default.nix | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/development/libraries/science/math/faiss/default.nix b/pkgs/development/libraries/science/math/faiss/default.nix index 5175b9b60961..d62be5cce41c 100644 --- a/pkgs/development/libraries/science/math/faiss/default.nix +++ b/pkgs/development/libraries/science/math/faiss/default.nix @@ -117,7 +117,7 @@ stdenv.mkDerivation { description = "A library for efficient similarity search and clustering of dense vectors by Facebook Research"; homepage = "https://github.com/facebookresearch/faiss"; license = licenses.mit; - platforms = platforms.unix; # Never tested windows + platforms = platforms.unix; maintainers = with maintainers; [ SomeoneSerge ]; }; } From cd43db0c948f1c2e6650ad6eae2ef88a8cca6e12 Mon Sep 17 00:00:00 2001 From: Someone Serge Date: Tue, 2 Aug 2022 16:34:08 +0300 Subject: [PATCH 4/4] python3Packages.faiss: conventional "tests.nix" name --- pkgs/development/libraries/science/math/faiss/default.nix | 2 +- .../science/math/faiss/{pytest-suite.nix => tests.nix} | 0 2 files changed, 1 insertion(+), 1 deletion(-) rename pkgs/development/libraries/science/math/faiss/{pytest-suite.nix => tests.nix} (100%) diff --git a/pkgs/development/libraries/science/math/faiss/default.nix b/pkgs/development/libraries/science/math/faiss/default.nix index d62be5cce41c..f2d5e29f6a4a 100644 --- a/pkgs/development/libraries/science/math/faiss/default.nix +++ b/pkgs/development/libraries/science/math/faiss/default.nix @@ -109,7 +109,7 @@ stdenv.mkDerivation { demo_ivfpq_indexing && touch $out ''; } // lib.optionalAttrs pythonSupport { - pytest = pythonPackages.callPackage ./pytest-suite.nix { }; + pytest = pythonPackages.callPackage ./tests.nix { }; }; }; diff --git a/pkgs/development/libraries/science/math/faiss/pytest-suite.nix b/pkgs/development/libraries/science/math/faiss/tests.nix similarity index 100% rename from pkgs/development/libraries/science/math/faiss/pytest-suite.nix rename to pkgs/development/libraries/science/math/faiss/tests.nix