diff --git a/pkgs/by-name/ma/manticoresearch/cctz-cmake-policy.patch b/pkgs/by-name/ma/manticoresearch/cctz-cmake-policy.patch new file mode 100644 index 000000000000..4fa323984372 --- /dev/null +++ b/pkgs/by-name/ma/manticoresearch/cctz-cmake-policy.patch @@ -0,0 +1,9 @@ +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -1,1 +1,2 @@ +-cmake_minimum_required(VERSION 2.8.12) ++# Fix the build with CMake 4 ++cmake_minimum_required(VERSION 3.5) + + if (POLICY CMP0025) + cmake_policy(SET CMP0025 NEW) diff --git a/pkgs/by-name/ma/manticoresearch/columnar.nix b/pkgs/by-name/ma/manticoresearch/columnar.nix new file mode 100644 index 000000000000..296d50176bcb --- /dev/null +++ b/pkgs/by-name/ma/manticoresearch/columnar.nix @@ -0,0 +1,172 @@ +{ + cmake, + fetchFromGitHub, + fetchzip, + lib, + pkg-config, + replaceVars, + simde, + stdenv, +}: + +let + # Manticore fetches columnar from a tag that indicates the versions of libraries that columnar includes. + # (See NEED_COLUMNAR_API/NEED_SECONDARY_API/NEED_KNN_API in Manticore's cmake/GetColumnar.cmake) + # So this tag indicates columnar v27, secondary v18, and knn v9. + libraryVersion = "c27-s18-k9"; + # If you inspect the revision that tag points to, there's also hopefully a more normal version tag too + numericVersion = "9.0.0"; + columnarSource = fetchFromGitHub { + owner = "manticoresoftware"; + repo = "columnar"; + tag = libraryVersion; + hash = "sha256-26zSdLNJkC7zYHT8sDq/2mZPRfdc9db1vIIrkaiPqFM="; + }; + hnswlib = stdenv.mkDerivation (finalAttrs: { + pname = "hnswlib"; + version = "d7bb3bbd59220f62203012b2f649aa537cc97cdc"; # see HNSW_GITHUB in columnar's cmake/GetHNSW.cmake + src = fetchFromGitHub { + owner = "manticoresoftware"; + repo = "hnswlib"; + rev = "d7bb3bbd59220f62203012b2f649aa537cc97cdc"; + hash = "sha256-SDYf8J4auzqGBHqFMDFR6TVyNthCA4q6J4glcIssAJU="; + }; + nativeBuildInputs = [ + cmake + pkg-config + ]; + meta = { + description = "Header-only C++/python library for fast approximate nearest neighbors"; + homepage = "https://github.com/nmslib/hnswlib"; + license = lib.licenses.asl20; + platforms = lib.platforms.all; + }; + }); + pgm-index = stdenv.mkDerivation (finalAttrs: { + pname = "pgm-index"; + version = "pgm_2022_08_02"; # see PGM_GITHUB in columnar's cmake/GetPGM.cmake + src = fetchFromGitHub { + owner = "manticoresoftware"; + repo = "PGM-index"; + rev = "5a5a763c7c07e56f56fd33137dcee9f1b3c3b640"; + hash = "sha256-p8CjY9r1TEi91HBac9rZ39Yae2XuNh2yQWbo65n90jU="; + }; + nativeBuildInputs = [ + cmake + pkg-config + ]; + # Use the custom CMakeLists.txt from columnar + postPatch = '' + cp ${columnarSource}/pgm/CMakeLists.txt CMakeLists.txt + ''; + meta = { + description = "Piecewise Geometric Model index - a fast and space-efficient data structure"; + homepage = "https://github.com/gvinciguerra/PGM-index"; + license = lib.licenses.asl20; + platforms = lib.platforms.all; + }; + }); + streamvbyte = stdenv.mkDerivation (finalAttrs: rec { + pname = "streamvbyte"; + version = "efdd9dace81a4a8f844267631879b500c6d913cf"; # see SVB_GITHUB in columnar's cmake/GetStreamvbyte.cmake + src = fetchFromGitHub { + owner = "manticoresoftware"; + repo = "streamvbyte"; + rev = version; + hash = "sha256-a9E1aWBY/P7wI+kgHqhEiD3THctFfeFcy658RcNpHfQ="; + }; + nativeBuildInputs = [ + cmake + pkg-config + ]; + # Enable SSE4.1 on x86_64 to avoid GCC 14 target attribute issues + env.NIX_CFLAGS_COMPILE = lib.optionalString stdenv.hostPlatform.isx86_64 "-msse4.1"; + # Use the custom CMakeLists.txt from columnar + postPatch = '' + cp ${columnarSource}/streamvbyte/CMakeLists.txt CMakeLists.txt + ''; + meta = { + description = "Fast integer compression library using StreamVByte encoding"; + homepage = "https://github.com/manticoresoftware/streamvbyte/"; + license = lib.licenses.asl20; + platforms = lib.platforms.all; + }; + }); + fastpfor = stdenv.mkDerivation (finalAttrs: { + pname = "fastpfor"; + version = "simde"; # see FP_GITHUB in columnar's cmake/GetFastPFor.cmake + src = fetchFromGitHub { + owner = "manticoresoftware"; + repo = "FastPFor"; + rev = "5f6b7df8f0dba402bb616a81ca510bfeb04c42ce"; + hash = "sha256-O9nqgsZKT+q9dHdh2RahF1U8KXCmDp0ld7hNvVREM8k="; + }; + nativeBuildInputs = [ + cmake + pkg-config + ]; + buildInputs = [ simde ]; + # Enable SSE3/SSSE3/SSE4.1 on x86_64 to avoid GCC 14 target attribute issues + # (Columnar's custom libfastpfor/CMakeLists.txt comments these out. I think get away with it because they just build with clang?) + env.NIX_CFLAGS_COMPILE = lib.optionalString stdenv.hostPlatform.isx86_64 "-msse3 -mssse3 -msse4.1"; + + # Columnar uses its own version of CMakeLists.txt to compile libfastpfor + prePatch = '' + cp ${columnarSource}/libfastpfor/CMakeLists.txt CMakeLists.txt + ''; + patches = [ ./fastpfor-cmake-policy.patch ]; + postPatch = '' + # Use nixpkgs simde rather than FetchContent-ing from github + sed -i '/Add simde for arm/,/FetchContent_GetProperties ( simde )/c\ + message(STATUS "Using nixpkgs simde for arm")\n\ + set(simde_SOURCE_DIR "${simde.src}")\n\ + set(simde_POPULATED 1)\n\ + ' CMakeLists.txt + ''; + meta = { + description = "Fast integer compression"; + homepage = "https://github.com/manticoresoftware/FastPFor/"; + license = lib.licenses.asl20; + platforms = lib.platforms.all; + }; + }); +in +stdenv.mkDerivation (finalAttrs: { + pname = "manticore-columnar"; + version = numericVersion; + + outputs = [ + "out" + "dev" + ]; + + src = columnarSource; + nativeBuildInputs = [ + cmake + ]; + buildInputs = [ + fastpfor + hnswlib + pgm-index + streamvbyte + ]; + + patches = [ + (replaceVars ./columnar.patch { + version = numericVersion; + }) + ]; + cmakeFlags = [ + "-DSKIP_KNN=ON" # Skip KNN & embeddings for now + ]; + postPatch = '' + # I've failed to get avx compiled, skip it for now. + sed -i '/set (ADD_AVX_BUILDS 1)/d' CMakeLists.txt + ''; + + meta = with lib; { + description = "Manticore Columnar Library - columnar storage and secondary indexes library for Manticore Search"; + homepage = "https://github.com/manticoresoftware/columnar"; + license = licenses.asl20; + }; +}) diff --git a/pkgs/by-name/ma/manticoresearch/columnar.patch b/pkgs/by-name/ma/manticoresearch/columnar.patch new file mode 100644 index 000000000000..568310ebe983 --- /dev/null +++ b/pkgs/by-name/ma/manticoresearch/columnar.patch @@ -0,0 +1,33 @@ +diff --git a/CMakeLists.txt b/CMakeLists.txt +index 3bfbc5ef9043..2a25188f8f92 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -26,7 +26,7 @@ set ( CMAKE_INTERPROCEDURAL_OPTIMIZATION $ENV{CMAKE_INTERPROCEDURAL_OPTIMIZATION + set ( _CMAKE_BUILD_TYPE ${CMAKE_BUILD_TYPE} ) + + # The below version is used for versioning of the package and is replaced by the version from the pack workflow +-project ( columnar VERSION 0.0.0 ) ++project ( columnar VERSION @version@ ) + + # sometimes CMAKE_BUILD_TYPE became set after PROJECT statement, undo it. + if (NOT _CMAKE_BUILD_TYPE AND CMAKE_BUILD_TYPE) +@@ -108,7 +108,7 @@ set (_includes include/manticore-columnar-api) + target_include_directories ( columnar_api INTERFACE $$ ) + + # install columnar and secondary API (only headers, no packaging, don't build anything at all) +-if (API_ONLY) ++# if (API_ONLY) + install ( TARGETS columnar_api EXPORT apiexport ) + install ( TARGETS secondary_api EXPORT apiexport ) + install ( TARGETS knn_api EXPORT apiexport ) +@@ -147,8 +147,8 @@ include(\"\${CMAKE_CURRENT_LIST_DIR}/columnar-targets.cmake\")" ) + install ( FILES "${CMAKE_CURRENT_BINARY_DIR}/columnar-config-version.cmake" DESTINATION "${API_CMAKE_DIR}" ) + + # finish configuration, as it is API_ONLY +- return() +-endif () ++ # return() ++# endif () + + # here we came in case of full build (i.e. columnar and/or secondary and/or knn) + diff --git a/pkgs/by-name/ma/manticoresearch/fastpfor-cmake-policy.patch b/pkgs/by-name/ma/manticoresearch/fastpfor-cmake-policy.patch new file mode 100644 index 000000000000..c1820e7a5c70 --- /dev/null +++ b/pkgs/by-name/ma/manticoresearch/fastpfor-cmake-policy.patch @@ -0,0 +1,14 @@ +diff --git a/CMakeLists.txt b/CMakeLists.txt +index efedf74..effa1ec 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -4,7 +4,8 @@ + # Copyright (c) 2012 Louis Dionne + # Copyright (c) 2021-2022 Manticore Software LTD + # +-cmake_minimum_required ( VERSION 3.17 ) ++# Fix build with CMake 4 ++cmake_minimum_required ( VERSION 3.5 ) + + foreach (policy CMP0091 CMP0022) + if (POLICY ${policy}) diff --git a/pkgs/by-name/ma/manticoresearch/package.nix b/pkgs/by-name/ma/manticoresearch/package.nix index c0b190e29e60..ae6c0c857af5 100644 --- a/pkgs/by-name/ma/manticoresearch/package.nix +++ b/pkgs/by-name/ma/manticoresearch/package.nix @@ -1,40 +1,25 @@ { - lib, - stdenv, - fetchFromGitHub, bison, - cmake, - flex, - pkg-config, boost, + callPackage, + cmake, + croaring, + fetchFromGitHub, + flex, icu, + lib, libstemmer, - mariadb-connector-c, - re2, - nlohmann_json, - testers, manticoresearch, + mariadb-connector-c, + nlohmann_json, + pkg-config, + re2, + stdenv, + testers, }: let - columnar = stdenv.mkDerivation (finalAttrs: { - pname = "columnar"; - version = "c21-s10"; # see NEED_COLUMNAR_API/NEED_SECONDARY_API in Manticore's cmake/GetColumnar.cmake - src = fetchFromGitHub { - owner = "manticoresoftware"; - repo = "columnar"; - rev = finalAttrs.version; - hash = "sha256-TGFGFfoyHnPSr2U/9dpqFLUN3Dt2jDQrTF/xxDY4pdE="; - }; - nativeBuildInputs = [ cmake ]; - cmakeFlags = [ "-DAPI_ONLY=ON" ]; - meta = { - description = "Column-oriented storage and secondary indexing library"; - homepage = "https://github.com/manticoresoftware/columnar/"; - license = lib.licenses.asl20; - platforms = lib.platforms.all; - }; - }); + columnar = callPackage ./columnar.nix { }; uni-algo = stdenv.mkDerivation (finalAttrs: { pname = "uni-algo"; version = "0.7.2"; @@ -52,16 +37,60 @@ let platforms = lib.platforms.all; }; }); + cctz = stdenv.mkDerivation { + pname = "manticore-cctz"; + version = "0-unstable-2024-05-08"; + + src = fetchFromGitHub { + owner = "manticoresoftware"; + repo = "cctz"; + rev = "cf11f758e2532161c9e21c3ec2461b0fafb15853"; + hash = "sha256-oXn6YowOg+9jaXXSX1fggkgE9o9xZ4hlmrpdpEHot68="; + }; + + patches = [ ./cctz-cmake-policy.patch ]; + + nativeBuildInputs = [ cmake ]; + cmakeBuildDir = "build_dir"; # Avoid conflicts with the pre-existing `BUILD` file on case-insensitive FS + + meta = { + description = "Library for translating between absolute and civil times using the rules of a time zone"; + homepage = "https://github.com/manticoresoftware/cctz"; + license = lib.licenses.asl20; + platforms = lib.platforms.all; + }; + }; + xxhash = stdenv.mkDerivation { + pname = "manticore-xxhash"; + version = "0.8.2-unstable-2024-07-24"; + + src = fetchFromGitHub { + owner = "manticoresoftware"; + repo = "xxHash"; + rev = "72997b0070a031c063f86a308aec77ae742706d3"; + hash = "sha256-QAIxZeMiohm/BYyO0f70En6GOv7t3yLH2pJfkUek7Js="; + }; + nativeBuildInputs = [ cmake ]; + meta = { + description = "Extremely fast non-cryptographic hash algorithm"; + homepage = "https://github.com/manticoresoftware/xxhash"; + license = with lib.licenses; [ + bsd2 + gpl2 + ]; + platforms = lib.platforms.all; + }; + }; in stdenv.mkDerivation (finalAttrs: { pname = "manticoresearch"; - version = "6.2.12"; + version = "15.1.0"; src = fetchFromGitHub { owner = "manticoresoftware"; repo = "manticoresearch"; tag = finalAttrs.version; - hash = "sha256-UD/r7rlJ5mR3wg4doKT/nTwTWzlulngUjOPNEjmykB8="; + hash = "sha256-ESiM2D11o1QnctDzL7WQ+usad7nvs0YSPOpZzSfYv4Y="; }; nativeBuildInputs = [ @@ -73,41 +102,54 @@ stdenv.mkDerivation (finalAttrs: { buildInputs = [ boost - columnar + cctz + columnar.dev + croaring icu.dev libstemmer mariadb-connector-c nlohmann_json uni-algo re2 + xxhash ]; postPatch = '' sed -i 's/set ( Boost_USE_STATIC_LIBS ON )/set ( Boost_USE_STATIC_LIBS OFF )/' src/CMakeLists.txt - # supply our own packages rather than letting manticore download dependencies during build - sed -i 's/^with_get/with_menu/' CMakeLists.txt - sed -i 's/get_dep \( nlohmann_json .* \)/find_package(nlohmann_json)/' CMakeLists.txt - sed -i 's/get_dep \( uni-algo .* \)/find_package(uni-algo)/' CMakeLists.txt + # Skip jieba, it requires a bunch of additional dependencies + sed -i '/with_get ( jieba /d' CMakeLists.txt + + # Fill in a version number for the VERNUMBERS macro + sed -i 's/0\.0\.0/${finalAttrs.version}/' src/sphinxversion.h.in ''; cmakeFlags = [ "-DWITH_GALERA=0" + "-DWITH_ICU=1" + # Supply our own packages rather than letting manticore download dependencies during build + "-DWITH_ICU_FORCE_STATIC=OFF" + "-DWITH_RE2_FORCE_STATIC=OFF" + "-DWITH_STEMMER_FORCE_STATIC=OFF" "-DWITH_MYSQL=1" "-DMYSQL_INCLUDE_DIR=${mariadb-connector-c.dev}/include/mariadb" "-DMYSQL_LIB=${mariadb-connector-c.out}/lib/mariadb/libmysqlclient.a" "-DCONFDIR=${placeholder "out"}/etc" "-DLOGDIR=/var/lib/manticoresearch/log" "-DRUNDIR=/var/run/manticoresearch" + "-DDISTR=nixpkgs" ]; postFixup = '' mkdir -p $out/lib/systemd/system cp ${finalAttrs.src}/dist/deb/manticore.service.in $out/lib/systemd/system/manticore.service substituteInPlace $out/lib/systemd/system/manticore.service \ - --replace "@CMAKE_INSTALL_FULL_RUNSTATEDIR@" "/var/lib/manticore" \ - --replace "@CMAKE_INSTALL_FULL_BINDIR@" "$out/bin" \ - --replace "@CMAKE_INSTALL_FULL_SYSCONFDIR@" "$out/etc" + --replace-fail "@CMAKE_INSTALL_FULL_RUNSTATEDIR@" "/var/lib/manticore" \ + --replace-fail "@CMAKE_INSTALL_FULL_BINDIR@" "$out/bin" \ + --replace-fail "@CMAKE_INSTALL_FULL_SYSCONFDIR@" "$out/etc" + + mkdir $out/share/manticore/modules + cp ${columnar}/share/manticore/modules/* $out/share/manticore/modules ''; passthru.tests.version = testers.testVersion { @@ -120,7 +162,7 @@ stdenv.mkDerivation (finalAttrs: { description = "Easy to use open source fast database for search"; homepage = "https://manticoresearch.com"; changelog = "https://github.com/manticoresoftware/manticoresearch/releases/tag/${finalAttrs.version}"; - license = lib.licenses.gpl2; + license = lib.licenses.gpl3Plus; mainProgram = "searchd"; maintainers = [ lib.maintainers.jdelStrother ]; platforms = lib.platforms.all;