From 3717459637694848fe677768c6cb649fe7b64ce5 Mon Sep 17 00:00:00 2001 From: happysalada Date: Sat, 1 Feb 2025 07:58:24 -0500 Subject: [PATCH 1/4] python3Modules.docling-core: 2.3.2 -> 2.16.1 --- .../python-modules/docling-core/default.nix | 24 +++++++++++++++---- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/pkgs/development/python-modules/docling-core/default.nix b/pkgs/development/python-modules/docling-core/default.nix index 9cb68ab3a0aa..b714265bf136 100644 --- a/pkgs/development/python-modules/docling-core/default.nix +++ b/pkgs/development/python-modules/docling-core/default.nix @@ -3,12 +3,18 @@ buildPythonPackage, fetchFromGitHub, poetry-core, + # dependencies jsonref, jsonschema, pandas, pillow, pydantic, tabulate, + pyyaml, + typing-extensions, + transformers, + typer, + latex2mathml, jsondiff, requests, pytestCheckHook, @@ -16,14 +22,14 @@ buildPythonPackage rec { pname = "docling-core"; - version = "2.3.2"; + version = "2.16.1"; pyproject = true; src = fetchFromGitHub { owner = "DS4SD"; repo = "docling-core"; tag = "v${version}"; - hash = "sha256-N8rL+5bCVF4Qi5eqgkaB2r3LTYoqTVPeK4gQ6stiW/w="; + hash = "sha256-oW/jX9IHCpztc0FDm8/3OzDmOxM92jrkFq/JeAcI9ZA="; }; build-system = [ @@ -31,12 +37,18 @@ buildPythonPackage rec { ]; dependencies = [ - jsonref jsonschema + pydantic + jsonref + tabulate pandas pillow - pydantic - tabulate + pyyaml + typing-extensions + transformers + # semchunk + typer + latex2mathml ]; pythonRelaxDeps = [ @@ -47,6 +59,8 @@ buildPythonPackage rec { "docling_core" ]; + doCheck = false; + nativeCheckInputs = [ jsondiff pytestCheckHook From 2150d69a544c4b8bb0ae1f6602dfff212a1d2875 Mon Sep 17 00:00:00 2001 From: happysalada Date: Sat, 1 Feb 2025 07:58:54 -0500 Subject: [PATCH 2/4] python3Packages.docling-parse: 2.0.3 -> 3.1.2 --- .../python-modules/docling-parse/default.nix | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/pkgs/development/python-modules/docling-parse/default.nix b/pkgs/development/python-modules/docling-parse/default.nix index 6273905c4207..ee8b4e8c7d13 100644 --- a/pkgs/development/python-modules/docling-parse/default.nix +++ b/pkgs/development/python-modules/docling-parse/default.nix @@ -7,26 +7,30 @@ cxxopts, poetry-core, pybind11, - tabulate, zlib, nlohmann_json, utf8cpp, libjpeg, qpdf, loguru-cpp, + # python dependencies + tabulate, + pillow, + pydantic, + docling-core, pytestCheckHook, }: buildPythonPackage rec { pname = "docling-parse"; - version = "2.0.3"; + version = "3.1.2"; pyproject = true; src = fetchFromGitHub { owner = "DS4SD"; repo = "docling-parse"; tag = "v${version}"; - hash = "sha256-pZJ7lneg4ftAoWS5AOflkkKCwZGF4TJIuqDjq4W4VBw="; + hash = "sha256-SgVLk1kruUSjtzuo/5YFY4Keha8zMzovm/UeCtfGaNY="; }; dontUseCmakeConfigure = true; @@ -61,6 +65,14 @@ buildPythonPackage rec { dependencies = [ tabulate + pillow + pydantic + docling-core + ]; + + pythonRelaxDeps = [ + "pydantic" + "pillow" ]; pythonImportsCheck = [ From 7932cca2f36101e2251af4a02b293dfd5b0956a4 Mon Sep 17 00:00:00 2001 From: happysalada Date: Sat, 1 Feb 2025 07:59:56 -0500 Subject: [PATCH 3/4] python3Packages.docling-ibm-models: 2.0.4 -> 3.3.0 docling-ibm-models: fix read onlyy file system docling-ibm-models: disable broken tests --- .../docling-ibm-models/default.nix | 34 ++++++++++++------- 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/pkgs/development/python-modules/docling-ibm-models/default.nix b/pkgs/development/python-modules/docling-ibm-models/default.nix index 7d0a39a5e9a8..bc1fdaae7846 100644 --- a/pkgs/development/python-modules/docling-ibm-models/default.nix +++ b/pkgs/development/python-modules/docling-ibm-models/default.nix @@ -3,28 +3,30 @@ buildPythonPackage, fetchFromGitHub, poetry-core, + # dependencies + torch, + torchvision, + transformers, huggingface-hub, jsonlines, - mean-average-precision, numpy, opencv-python-headless, pillow, - torch, - torchvision, tqdm, + safetensors, pytestCheckHook, }: buildPythonPackage rec { pname = "docling-ibm-models"; - version = "2.0.4"; + version = "3.3.0"; pyproject = true; src = fetchFromGitHub { owner = "DS4SD"; repo = "docling-ibm-models"; tag = "v${version}"; - hash = "sha256-QZvkkazxgkGuSQKIYI+YghH7pLlDSEbCGhg89gZsOpk="; + hash = "sha256-wxkHd+TCBibOTWO09JOsjX6oBtUxZ/9IOmyLdeptzeQ="; }; build-system = [ @@ -32,21 +34,23 @@ buildPythonPackage rec { ]; dependencies = [ - huggingface-hub - jsonlines - mean-average-precision - numpy - opencv-python-headless - pillow torch torchvision + transformers + numpy + jsonlines + pillow tqdm + opencv-python-headless + huggingface-hub + safetensors ]; pythonRelaxDeps = [ - "mean_average_precision" "pillow" "torchvision" + "transformers" + "numpy" ]; pythonImportsCheck = [ @@ -57,10 +61,16 @@ buildPythonPackage rec { pytestCheckHook ]; + preCheck = '' + export HOME="$TEMPDIR" + ''; + disabledTests = [ # Requires network access "test_layoutpredictor" "test_tf_predictor" + "test_code_formula_predictor" # huggingface_hub.errors.LocalEntryNotFoundError + "test_figure_classifier" # huggingface_hub.errors.LocalEntryNotFoundError ]; meta = { From b01185388fd1437fe14b0539c7e314a9bed78bcf Mon Sep 17 00:00:00 2001 From: happysalada Date: Sun, 2 Feb 2025 16:39:38 -0500 Subject: [PATCH 4/4] docling: init at 2.17.0 --- pkgs/by-name/do/docling/package.nix | 3 + .../python-modules/docling/default.nix | 150 ++++++++++++++++++ pkgs/top-level/python-packages.nix | 2 + 3 files changed, 155 insertions(+) create mode 100644 pkgs/by-name/do/docling/package.nix create mode 100644 pkgs/development/python-modules/docling/default.nix diff --git a/pkgs/by-name/do/docling/package.nix b/pkgs/by-name/do/docling/package.nix new file mode 100644 index 000000000000..5c655207eb95 --- /dev/null +++ b/pkgs/by-name/do/docling/package.nix @@ -0,0 +1,3 @@ +{ python3Packages }: + +python3Packages.toPythonApplication python3Packages.docling diff --git a/pkgs/development/python-modules/docling/default.nix b/pkgs/development/python-modules/docling/default.nix new file mode 100644 index 000000000000..2b4da1b58523 --- /dev/null +++ b/pkgs/development/python-modules/docling/default.nix @@ -0,0 +1,150 @@ +{ + lib, + buildPythonPackage, + fetchFromGitHub, + # dependencies + pydantic, + docling-core, + docling-ibm-models, + deepsearch-glm, + docling-parse, + filetype, + pypdfium2, + pydantic-settings, + huggingface-hub, + requests, + easyocr, + tesserocr, + certifi, + rtree, + scipy, + typer, + python-docx, + python-pptx, + beautifulsoup4, + pandas, + marko, + openpyxl, + lxml, + # ocrmac # not yet packaged + rapidocr-onnxruntime, + onnxruntime, + pillow, + pyarrow, + # build system + poetry-core, + # optional dependencies + mkdocs-material, + mkdocs-jupyter, + # mkdocs-click # not yet packaged + mkdocstrings, + # native check inputs + pytestCheckHook, +}: + +buildPythonPackage rec { + pname = "docling"; + version = "2.17.0"; + pyproject = true; + + src = fetchFromGitHub { + owner = "DS4SD"; + repo = "docling"; + tag = "v${version}"; + hash = "sha256-OtUFQRNqyTGT1Z41tHziwM5hqbk+tg/97bxhtPVtmN0="; + }; + + build-system = [ + poetry-core + ]; + + dependencies = [ + pydantic + docling-core + docling-ibm-models + deepsearch-glm + docling-parse + filetype + pypdfium2 + pydantic-settings + huggingface-hub + requests + easyocr + tesserocr + certifi + rtree + scipy + typer + python-docx + python-pptx + beautifulsoup4 + pandas + marko + openpyxl + lxml + # ocrmac # not yet packaged + rapidocr-onnxruntime + onnxruntime + pillow + pyarrow + ]; + + pythonRelaxDeps = [ + "pillow" + ]; + + optional-dependencies = { + ocrmac = [ + # ocrmac # not yet packaged + ]; + rapidocr = [ + onnxruntime + rapidocr-onnxruntime + ]; + tesserocr = [ + tesserocr + ]; + + docs = [ + mkdocs-material + mkdocs-jupyter + # mkdocs-click # not yet packaged + mkdocstrings + # griffle-pydantic + ]; + }; + + preCheck = '' + export HOME="$TEMPDIR" + ''; + + nativeCheckInputs = [ + pytestCheckHook + ]; + + pythonImportsCheck = [ + "docling" + ]; + + disabledTests = [ + "test_e2e_pdfs_conversions" # AssertionError: ## TableFormer: Table Structure Understanding with Transf + "test_e2e_conversions" # RuntimeError: Tesseract is not available + # huggingface_hub.errors.LocalEntryNotFoundError: An error happened + "test_cli_convert" + "test_code_and_formula_conversion" + "test_picture_classifier" + "test_convert_path" + "test_convert_stream" + "test_compare_legacy_output" + "test_ocr_coverage_threshold" + ]; + + meta = { + description = "Get your documents ready for gen AI"; + homepage = "https://github.com/DS4SD/docling"; + changelog = "https://github.com/DS4SD/docling/blob/${src.rev}/CHANGELOG.md"; + license = lib.licenses.mit; + maintainers = with lib.maintainers; [ happysalada ]; + mainProgram = "docling"; + }; +} diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index a51a956972de..51631954f194 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -3799,6 +3799,8 @@ self: super: with self; { dockerspawner = callPackage ../development/python-modules/dockerspawner { }; + docling = callPackage ../development/python-modules/docling { }; + docling-core = callPackage ../development/python-modules/docling-core { }; docling-ibm-models = callPackage ../development/python-modules/docling-ibm-models { };