From e556bb0b675a849371645b6b79eccd4130744967 Mon Sep 17 00:00:00 2001 From: happysalada Date: Tue, 7 Nov 2023 21:09:03 +0100 Subject: [PATCH] python311Packages.farm-haystack: init at 1.21.2 --- .../python-modules/farm-haystack/default.nix | 290 ++++++++++++++++++ pkgs/top-level/python-packages.nix | 2 + 2 files changed, 292 insertions(+) create mode 100644 pkgs/development/python-modules/farm-haystack/default.nix diff --git a/pkgs/development/python-modules/farm-haystack/default.nix b/pkgs/development/python-modules/farm-haystack/default.nix new file mode 100644 index 000000000000..de87016f36fc --- /dev/null +++ b/pkgs/development/python-modules/farm-haystack/default.nix @@ -0,0 +1,290 @@ +{ lib +, buildPythonPackage +, fetchFromGitHub +, pythonRelaxDepsHook +, hatchling +, boilerpy3 +, events +, httpx +, jsonschema +, lazy-imports +, more-itertools +, networkx +, pandas +, pillow +, platformdirs +, posthog +, prompthub-py +, pydantic +, quantulum3 +, rank-bm25 +, requests +, requests-cache +, scikit-learn +, sseclient-py +, tenacity +, tiktoken +, tqdm +, transformers +, openai-whisper +, boto3 +, botocore +# , beir +, selenium +, coverage +, dulwich +# , jupytercontrib +, mkdocs +, mypy +, pre-commit +, psutil +# , pydoc-markdown +, pylint +, pytest +, pytest-asyncio +, pytest-cov +# , pytest-custom-exit-code +, python-multipart +, reno +, responses +, toml +, tox +, watchdog +, elastic-transport +, elasticsearch +# , azure-ai-formrecognizer +, beautifulsoup4 +, markdown +, python-docx +, python-frontmatter +, python-magic +, tika +, black +, huggingface-hub +, sentence-transformers +, mlflow +, rapidfuzz +, scipy +, seqeval +, pdf2image +, pytesseract +, faiss +# , faiss-gpu +, pinecone-client +, onnxruntime +, onnxruntime-tools +# , onnxruntime-gpu +, opensearch-py +, pymupdf +, langdetect +, nltk +, canals +, jinja2 +, openai +, aiorwlock +, ray +, psycopg2 +, sqlalchemy +, sqlalchemy-utils +, weaviate-client +}: + +buildPythonPackage rec { + pname = "farm-haystack"; + version = "1.21.2"; + pyproject = true; + + src = fetchFromGitHub { + owner = "deepset-ai"; + repo = "haystack"; + rev = "v${version}"; + hash = "sha256-h+8F1e6fFoxT4Cu0DxtXTyDd+OFXpX1+uk21/k9qcxw="; + }; + + nativeBuildInputs = [ + hatchling + pythonRelaxDepsHook + ]; + + pythonRemoveDeps = [ + # We call it faiss, not faiss-cpu. + "faiss-cpu" + ]; + + propagatedBuildInputs = [ + boilerpy3 + events + httpx + jsonschema + lazy-imports + more-itertools + networkx + pandas + pillow + platformdirs + posthog + prompthub-py + pydantic + quantulum3 + rank-bm25 + requests + requests-cache + scikit-learn + sseclient-py + tenacity + tiktoken + tqdm + transformers + ]; + + env.HOME = "$(mktemp -d)"; + + passthru.optional-dependencies = { + # all = [ + # farm-haystack + # ]; + # all-gpu = [ + # farm-haystack + # ]; + audio = [ + openai-whisper + ]; + aws = [ + boto3 + botocore + ]; + # beir = [ + # beir + # ]; + colab = [ + pillow + ]; + crawler = [ + selenium + ]; + dev = [ + coverage + dulwich + # jupytercontrib + mkdocs + mypy + pre-commit + psutil + # pydoc-markdown + pylint + pytest + pytest-asyncio + pytest-cov + # pytest-custom-exit-code + python-multipart + reno + responses + toml + tox + watchdog + ]; + elasticsearch7 = [ + elastic-transport + elasticsearch + ]; + elasticsearch8 = [ + elastic-transport + elasticsearch + ]; + file-conversion = [ + # azure-ai-formrecognizer + beautifulsoup4 + markdown + python-docx + python-frontmatter + python-magic + # python-magic-bin + tika + ]; + formatting = [ + black + ]; + inference = [ + huggingface-hub + sentence-transformers + transformers + ]; + metrics = [ + mlflow + rapidfuzz + scipy + seqeval + ]; + ocr = [ + pdf2image + pytesseract + ]; + only-faiss = [ + faiss + ]; + # only-faiss-gpu = [ + # faiss-gpu + # ]; + only-pinecone = [ + pinecone-client + ]; + onnx = [ + onnxruntime + onnxruntime-tools + ]; + # onnx-gpu = [ + # onnxruntime-gpu + # onnxruntime-tools + # ]; + opensearch = [ + opensearch-py + ]; + pdf = [ + pymupdf + ]; + preprocessing = [ + langdetect + nltk + ]; + preview = [ + canals + jinja2 + lazy-imports + openai + pandas + rank-bm25 + requests + tenacity + tqdm + ]; + ray = [ + aiorwlock + ray + ]; + sql = [ + psycopg2 + sqlalchemy + sqlalchemy-utils + ]; + weaviate = [ + weaviate-client + ]; + }; + + # the setup for test is intensive, hopefully can be done at some point + doCheck = false; + + + pythonImportsCheck = [ "haystack" ]; + + meta = with lib; { + description = "LLM orchestration framework to build customizable, production-ready LLM applications"; + longDescription = '' + LLM orchestration framework to build customizable, production-ready LLM applications. Connect components (models, vector DBs, file converters) to pipelines or agents that can interact with your data. With advanced retrieval methods, it's best suited for building RAG, question answering, semantic search or conversational agent chatbots + ''; + changelog = "https://github.com/deepset-ai/haystack/releases/tag/${src.rev}"; + homepage = "https://github.com/deepset-ai/haystack"; + license = licenses.asl20; + maintainers = with maintainers; [ happysalada ]; + }; +} diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index 2870a4f54364..ad4748773e0a 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -3787,6 +3787,8 @@ self: super: with self; { farama-notifications = callPackage ../development/python-modules/farama-notifications { }; + farm-haystack = callPackage ../development/python-modules/farm-haystack { }; + fastai = callPackage ../development/python-modules/fastai { }; fastapi = callPackage ../development/python-modules/fastapi { };