diff --git a/pkgs/development/python-modules/txtai/default.nix b/pkgs/development/python-modules/txtai/default.nix new file mode 100644 index 000000000000..acf1db58d2a6 --- /dev/null +++ b/pkgs/development/python-modules/txtai/default.nix @@ -0,0 +1,156 @@ +{ + lib +, buildPythonPackage +, pythonOlder +, fetchFromGitHub +, pythonRelaxDepsHook +# propagated build input +, faiss +, torch +, transformers +, huggingface-hub +, numpy +, pyyaml +, regex +# optional-dependencies +, aiohttp +, fastapi +, uvicorn +# TODO add apache-libcloud +# , apache-libcloud +, rich +, duckdb +, pillow +, networkx +, python-louvain +, onnx +, onnxruntime +, soundfile +, scipy +, ttstokenizer +, beautifulsoup4 +, nltk +, pandas +, tika +, imagehash +, timm +, fasttext +, sentencepiece +, accelerate +, onnxmltools +, annoy +, hnswlib +# TODO add pymagnitude-lite +#, pymagnitude-lite +, scikit-learn +, sentence-transformers +, croniter +, openpyxl +, requests +, xmltodict +# native check inputs +, unittestCheckHook +}: +let + version = "5.5.1"; + api = [ aiohttp fastapi uvicorn ]; + # cloud = [ apache-libcloud ]; + console = [ rich ]; + + database = [ duckdb pillow ]; + + graph = [ networkx python-louvain ]; + + model = [ onnx onnxruntime ]; + + pipeline-audio = [ onnx onnxruntime soundfile scipy ttstokenizer ]; + pipeline-data = [ beautifulsoup4 nltk pandas tika ]; + pipeline-image = [ imagehash pillow timm ]; + pipeline-text = [ fasttext sentencepiece ]; + pipeline-train = [ accelerate onnx onnxmltools onnxruntime ]; + pipeline = pipeline-audio ++ pipeline-data ++ pipeline-image ++ pipeline-text ++ pipeline-train; + + similarity = [ + annoy + fasttext + hnswlib + # pymagnitude-lite + scikit-learn + sentence-transformers + ]; + workflow = [ + # apache-libcloud + croniter + openpyxl + pandas + pillow + requests + xmltodict + ]; + all = api ++ console ++ database ++ graph ++ model ++ pipeline ++ similarity ++ workflow; + + optional-dependencies = { + inherit api console database graph model pipeline-audio pipeline-image + pipeline-text pipeline-train pipeline similarity workflow all; + }; +in +buildPythonPackage { + pname = "txtai"; + inherit version; + format = "setuptools"; + + disable = pythonOlder "3.8"; + + src = fetchFromGitHub { + owner = "neuml"; + repo = "txtai"; + rev = "v${version}"; + hash = "sha256-h6TwWzLYfFg5x2QMIstAZ5pkxfHobBU+b4gb0HiayzY="; + }; + + nativeBuildInputs = [ + pythonRelaxDepsHook + ]; + + pythonRemoveDeps = [ + # We call it faiss, not faiss-cpu. + "faiss-cpu" + ]; + + propagatedBuildInputs = [ + faiss + torch + transformers + huggingface-hub + numpy + pyyaml + regex + ]; + + passthru.optional-dependencies = optional-dependencies; + + pythonImportsCheck = [ "txtai" ]; + + # some tests hang forever + doCheck = false; + + preCheck = '' + export TRANSFORMERS_CACHE=$(mktemp -d) + ''; + + nativeCheckInputs = [ + unittestCheckHook + ] ++ optional-dependencies.api ++ optional-dependencies.similarity; + + unittestFlagsArray = [ + "-s" "test/python" "-v" + ]; + + meta = with lib; { + description = "Semantic search and workflows powered by language models"; + changelog = "https://github.com/neuml/txtai/releases/tag/v${version}"; + homepage = "https://github.com/neuml/txtai"; + license = licenses.asl20; + maintainers = with maintainers; [ happysalada ]; + }; +} diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index 1324c28692a0..0c8b32b1e8b7 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -12936,6 +12936,8 @@ self: super: with self; { txrequests = callPackage ../development/python-modules/txrequests { }; + txtai = callPackage ../development/python-modules/txtai { }; + txtorcon = callPackage ../development/python-modules/txtorcon { }; txzmq = callPackage ../development/python-modules/txzmq { };