From 648c8933d33ac726ba65e744ee69deb4e6c0eabc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20de=20Kok?= Date: Sun, 17 Nov 2024 13:53:06 +0100 Subject: [PATCH 1/5] python3Packages.curated-tokenizers: init at 0.0.8 --- .../curated-tokenizers/default.nix | 56 +++++++++++++++++++ pkgs/top-level/python-packages.nix | 2 + 2 files changed, 58 insertions(+) create mode 100644 pkgs/development/python-modules/curated-tokenizers/default.nix diff --git a/pkgs/development/python-modules/curated-tokenizers/default.nix b/pkgs/development/python-modules/curated-tokenizers/default.nix new file mode 100644 index 000000000000..77f4a77f906b --- /dev/null +++ b/pkgs/development/python-modules/curated-tokenizers/default.nix @@ -0,0 +1,56 @@ +{ + lib, + buildPythonPackage, + fetchFromGitHub, + cython, + setuptools, + regex, + pytestCheckHook, +}: + +buildPythonPackage rec { + pname = "curated-tokenizers"; + version = "0.0.9"; + pyproject = true; + + src = fetchFromGitHub { + owner = "explosion"; + repo = "curated-tokenizers"; + tag = "v${version}"; + hash = "sha256-P8kpPnaU3el7sc/vUn4waQN+JV7F9b49i6BtC4BFfIg="; + fetchSubmodules = true; + }; + + build-system = [ + cython + setuptools + ]; + + dependencies = [ + regex + ]; + + nativeCheckInputs = [ + pytestCheckHook + ]; + + # Explicitly set the path to avoid running vendored + # sentencepiece tests. + pytestFlagsArray = [ "tests" ]; + + preCheck = '' + # avoid local paths, relative imports wont resolve correctly + mv curated_tokenizers/tests tests + rm -r curated_tokenizers + ''; + + pythonImportsCheck = [ "curated_tokenizers" ]; + + meta = with lib; { + description = "Lightweight piece tokenization library"; + homepage = "https://github.com/explosion/curated-tokenizers"; + changelog = "https://github.com/explosion/curated-tokenizers/releases/tag/v${version}"; + license = licenses.mit; + maintainers = with maintainers; [ danieldk ]; + }; +} diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index bfc7dbd61102..585511d44bf1 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -2791,6 +2791,8 @@ self: super: with self; { cu2qu = callPackage ../development/python-modules/cu2qu { }; + curated-tokenizers = callPackage ../development/python-modules/curated-tokenizers { }; + customtkinter = callPackage ../development/python-modules/customtkinter { }; cucumber-tag-expressions = callPackage ../development/python-modules/cucumber-tag-expressions { }; From cc51d5006fa31d20f3b5a02cfd59ffd21be75692 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20de=20Kok?= Date: Sun, 17 Nov 2024 13:54:32 +0100 Subject: [PATCH 2/5] python3Packages.curated-transformers: init at 0.1.1 --- .../curated-transformers/default.nix | 36 +++++++++++++++++++ pkgs/top-level/python-packages.nix | 2 ++ 2 files changed, 38 insertions(+) create mode 100644 pkgs/development/python-modules/curated-transformers/default.nix diff --git a/pkgs/development/python-modules/curated-transformers/default.nix b/pkgs/development/python-modules/curated-transformers/default.nix new file mode 100644 index 000000000000..32a52a5fd06a --- /dev/null +++ b/pkgs/development/python-modules/curated-transformers/default.nix @@ -0,0 +1,36 @@ +{ + lib, + buildPythonPackage, + fetchFromGitHub, + setuptools, + torch, +}: + +buildPythonPackage rec { + pname = "curated-transformers"; + version = "0.1.1"; + pyproject = true; + + src = fetchFromGitHub { + owner = "explosion"; + repo = "curated-transformers"; + tag = "v${version}"; + hash = "sha256-QhJZnQIa9TilwdQCUlxnQCEc6Suj669cht6WHUAr/Gw="; + }; + + build-system = [ setuptools ]; + + dependencies = [ torch ]; + + # Unit tests are hard to use, since most tests rely on downloading + # models from Hugging Face Hub. + pythonImportCheck = [ "curated_transformers" ]; + + meta = with lib; { + description = "PyTorch library of curated Transformer models and their composable components"; + homepage = "https://github.com/explosion/curated-transformers"; + changelog = "https://github.com/explosion/curated-transformers/releases/tag/v${version}"; + license = licenses.mit; + maintainers = with maintainers; [ danieldk ]; + }; +} diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index 585511d44bf1..6af7b60b7a02 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -2793,6 +2793,8 @@ self: super: with self; { curated-tokenizers = callPackage ../development/python-modules/curated-tokenizers { }; + curated-transformers = callPackage ../development/python-modules/curated-transformers { }; + customtkinter = callPackage ../development/python-modules/customtkinter { }; cucumber-tag-expressions = callPackage ../development/python-modules/cucumber-tag-expressions { }; From a1accfb8f5040bd28cd0a36d05ee61e219f10175 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20de=20Kok?= Date: Sun, 17 Nov 2024 13:55:19 +0100 Subject: [PATCH 3/5] python3Packages.spacy-curated-transformers: init at 0.3.0 --- .../spacy-curated-transformers/default.nix | 44 +++++++++++++++++++ pkgs/top-level/python-packages.nix | 2 + 2 files changed, 46 insertions(+) create mode 100644 pkgs/development/python-modules/spacy-curated-transformers/default.nix diff --git a/pkgs/development/python-modules/spacy-curated-transformers/default.nix b/pkgs/development/python-modules/spacy-curated-transformers/default.nix new file mode 100644 index 000000000000..1a2cea820e5c --- /dev/null +++ b/pkgs/development/python-modules/spacy-curated-transformers/default.nix @@ -0,0 +1,44 @@ +{ + lib, + buildPythonPackage, + fetchFromGitHub, + setuptools, + curated-tokenizers, + curated-transformers, + spacy, + torch, +}: + +buildPythonPackage rec { + pname = "spacy-curated-transformers"; + version = "0.3.0"; + pyproject = true; + + src = fetchFromGitHub { + owner = "explosion"; + repo = "spacy-curated-transformers"; + tag = "release-v${version}"; + hash = "sha256-3LL0ofVsyacMzLJtttg0Tl9SlkPex7TwWL/HVF4WkfI="; + }; + + build-system = [ setuptools ]; + + dependencies = [ + curated-tokenizers + curated-transformers + spacy + torch + ]; + + # Unit tests are hard to use, since most tests rely on downloading + # models from Hugging Face Hub. + pythonImportCheck = [ "spacy_curated_transformers" ]; + + meta = with lib; { + description = "spaCy entry points for Curated Transformers"; + homepage = "https://github.com/explosion/spacy-curated-transformers"; + changelog = "https://github.com/explosion/spacy-curated-transformers/releases/tag/v${version}"; + license = licenses.mit; + maintainers = with maintainers; [ danieldk ]; + }; +} diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index 6af7b60b7a02..c91cf17d463b 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -15055,6 +15055,8 @@ self: super: with self; { spacy-alignments = callPackage ../development/python-modules/spacy-alignments { }; + spacy-curated-transformers = callPackage ../development/python-modules/spacy-curated-transformers { }; + spacy-legacy = callPackage ../development/python-modules/spacy/legacy.nix { }; spacy-loggers = callPackage ../development/python-modules/spacy-loggers { }; From 95b84e949e50f484d5c1a569909ea09a29c13f41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Dani=C3=ABl=20de=20Kok?= Date: Sun, 17 Nov 2024 13:56:27 +0100 Subject: [PATCH 4/5] python3Packages.spacy-models: fix model builds Fix various issues with building spaCy models: - Transformer models now use `spacy-curated-transformers`. - `sl_core_news_trf` requires `protobuf` and `sentencepiece`. - `ja` models require `sudachipy` and `sudachidict-core` - Loosen the `pkuseg` requirement for `zh` models (0.3 -> 1.0 does not have API changes). --- .../python-modules/spacy/models.nix | 36 ++++++++++++++----- 1 file changed, 28 insertions(+), 8 deletions(-) diff --git a/pkgs/development/python-modules/spacy/models.nix b/pkgs/development/python-modules/spacy/models.nix index a8e99669b448..9a78392fb5eb 100644 --- a/pkgs/development/python-modules/spacy/models.nix +++ b/pkgs/development/python-modules/spacy/models.nix @@ -9,7 +9,10 @@ setuptools, spacy, spacy-pkuseg, - spacy-transformers, + spacy-curated-transformers, + sudachipy, + sudachidict-core, + transformers, writeScript, stdenv, jq, @@ -27,7 +30,10 @@ let let lang = builtins.substring 0 2 pname; - requires-protobuf = pname == "fr_dep_news_trf" || pname == "uk_core_news_trf"; + requires-protobuf = + pname == "fr_dep_news_trf" || pname == "sl_core_news_trf" || pname == "uk_core_news_trf"; + requires-sentencepiece = pname == "fr_dep_news_trf" || pname == "sl_core_news_trf"; + requires-transformers = pname == "uk_core_news_trf"; in buildPythonPackage { inherit pname version; @@ -40,19 +46,33 @@ let propagatedBuildInputs = [ spacy ] - ++ lib.optionals (lib.hasSuffix "_trf" pname) [ spacy-transformers ] + ++ lib.optionals (lib.hasSuffix "_trf" pname) [ spacy-curated-transformers ] + ++ lib.optionals requires-transformers [ transformers ] + ++ lib.optionals (lang == "ja") [ + sudachidict-core + sudachipy + ] ++ lib.optionals (lang == "ru") [ pymorphy3 ] ++ lib.optionals (lang == "uk") [ pymorphy3 pymorphy3-dicts-uk ] ++ lib.optionals (lang == "zh") [ spacy-pkuseg ] - ++ lib.optionals (pname == "fr_dep_news_trf") [ sentencepiece ]; + ++ lib.optionals requires-sentencepiece [ sentencepiece ]; - postPatch = lib.optionalString requires-protobuf '' - substituteInPlace meta.json \ - --replace "protobuf<3.21.0" "protobuf" - ''; + postPatch = + lib.optionalString requires-protobuf '' + substituteInPlace meta.json \ + --replace-fail "protobuf<3.21.0" "protobuf" + '' + + lib.optionalString (lang == "zh") '' + # Uses numpy 2.x, while the rest of the dependencies still uses + # numpy 1.x. Remove once all spaCy packages are updated for + # numpy 2.x. + cat meta.json + substituteInPlace meta.json \ + --replace-fail "spacy-pkuseg>=1.0.0,<2.0.0" "spacy-pkuseg" + ''; nativeBuildInputs = [ setuptools ] ++ lib.optionals requires-protobuf [ protobuf ]; From c019ccc81c3c9a601f2122987de938477fd69845 Mon Sep 17 00:00:00 2001 From: Sandro Date: Sun, 16 Feb 2025 14:27:21 +0100 Subject: [PATCH 5/5] python3Packages.spacy: remove debug code --- pkgs/development/python-modules/spacy/models.nix | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pkgs/development/python-modules/spacy/models.nix b/pkgs/development/python-modules/spacy/models.nix index 9a78392fb5eb..b1d5ec36a10c 100644 --- a/pkgs/development/python-modules/spacy/models.nix +++ b/pkgs/development/python-modules/spacy/models.nix @@ -69,9 +69,8 @@ let # Uses numpy 2.x, while the rest of the dependencies still uses # numpy 1.x. Remove once all spaCy packages are updated for # numpy 2.x. - cat meta.json substituteInPlace meta.json \ - --replace-fail "spacy-pkuseg>=1.0.0,<2.0.0" "spacy-pkuseg" + --replace-fail "spacy-pkuseg>=1.0.0,<2.0.0" "spacy-pkuseg" ''; nativeBuildInputs = [ setuptools ] ++ lib.optionals requires-protobuf [ protobuf ];