python3Packages.spacy-models: fix model builds (#356705)

pkgs/development/python-modules/curated-tokenizers/default.nix

@@ -0,0 +1,56 @@
+{
+  lib,
+  buildPythonPackage,
+  fetchFromGitHub,
+  cython,
+  setuptools,
+  regex,
+  pytestCheckHook,
+}:
+
+buildPythonPackage rec {
+  pname = "curated-tokenizers";
+  version = "0.0.9";
+  pyproject = true;
+
+  src = fetchFromGitHub {
+    owner = "explosion";
+    repo = "curated-tokenizers";
+    tag = "v${version}";
+    hash = "sha256-P8kpPnaU3el7sc/vUn4waQN+JV7F9b49i6BtC4BFfIg=";
+    fetchSubmodules = true;
+  };
+
+  build-system = [
+    cython
+    setuptools
+  ];
+
+  dependencies = [
+    regex
+  ];
+
+  nativeCheckInputs = [
+    pytestCheckHook
+  ];
+
+  # Explicitly set the path to avoid running vendored
+  # sentencepiece tests.
+  pytestFlagsArray = [ "tests" ];
+
+  preCheck = ''
+    # avoid local paths, relative imports wont resolve correctly
+    mv curated_tokenizers/tests tests
+    rm -r curated_tokenizers
+  '';
+
+  pythonImportsCheck = [ "curated_tokenizers" ];
+
+  meta = with lib; {
+    description = "Lightweight piece tokenization library";
+    homepage = "https://github.com/explosion/curated-tokenizers";
+    changelog = "https://github.com/explosion/curated-tokenizers/releases/tag/v${version}";
+    license = licenses.mit;
+    maintainers = with maintainers; [ danieldk ];
+  };
+}

pkgs/development/python-modules/curated-transformers/default.nix

@@ -0,0 +1,36 @@
+{
+  lib,
+  buildPythonPackage,
+  fetchFromGitHub,
+  setuptools,
+  torch,
+}:
+
+buildPythonPackage rec {
+  pname = "curated-transformers";
+  version = "0.1.1";
+  pyproject = true;
+
+  src = fetchFromGitHub {
+    owner = "explosion";
+    repo = "curated-transformers";
+    tag = "v${version}";
+    hash = "sha256-QhJZnQIa9TilwdQCUlxnQCEc6Suj669cht6WHUAr/Gw=";
+  };
+
+  build-system = [ setuptools ];
+
+  dependencies = [ torch ];
+
+  # Unit tests are hard to use, since most tests rely on downloading
+  # models from Hugging Face Hub.
+  pythonImportCheck = [ "curated_transformers" ];
+
+  meta = with lib; {
+    description = "PyTorch library of curated Transformer models and their composable components";
+    homepage = "https://github.com/explosion/curated-transformers";
+    changelog = "https://github.com/explosion/curated-transformers/releases/tag/v${version}";
+    license = licenses.mit;
+    maintainers = with maintainers; [ danieldk ];
+  };
+}

pkgs/development/python-modules/spacy-curated-transformers/default.nix

@@ -0,0 +1,44 @@
+{
+  lib,
+  buildPythonPackage,
+  fetchFromGitHub,
+  setuptools,
+  curated-tokenizers,
+  curated-transformers,
+  spacy,
+  torch,
+}:
+
+buildPythonPackage rec {
+  pname = "spacy-curated-transformers";
+  version = "0.3.0";
+  pyproject = true;
+
+  src = fetchFromGitHub {
+    owner = "explosion";
+    repo = "spacy-curated-transformers";
+    tag = "release-v${version}";
+    hash = "sha256-3LL0ofVsyacMzLJtttg0Tl9SlkPex7TwWL/HVF4WkfI=";
+  };
+
+  build-system = [ setuptools ];
+
+  dependencies = [
+    curated-tokenizers
+    curated-transformers
+    spacy
+    torch
+  ];
+
+  # Unit tests are hard to use, since most tests rely on downloading
+  # models from Hugging Face Hub.
+  pythonImportCheck = [ "spacy_curated_transformers" ];
+
+  meta = with lib; {
+    description = "spaCy entry points for Curated Transformers";
+    homepage = "https://github.com/explosion/spacy-curated-transformers";
+    changelog = "https://github.com/explosion/spacy-curated-transformers/releases/tag/v${version}";
+    license = licenses.mit;
+    maintainers = with maintainers; [ danieldk ];
+  };
+}

pkgs/development/python-modules/spacy/models.nix

@@ -9,7 +9,10 @@
   setuptools,
   spacy,
   spacy-pkuseg,
-  spacy-transformers,
+  spacy-curated-transformers,
+  sudachipy,
+  sudachidict-core,
+  transformers,
   writeScript,
   stdenv,
   jq,
@@ -27,7 +30,10 @@ let
 
     let
       lang = builtins.substring 0 2 pname;
-      requires-protobuf = pname == "fr_dep_news_trf" || pname == "uk_core_news_trf";
+      requires-protobuf =
+        pname == "fr_dep_news_trf" || pname == "sl_core_news_trf" || pname == "uk_core_news_trf";
+      requires-sentencepiece = pname == "fr_dep_news_trf" || pname == "sl_core_news_trf";
+      requires-transformers = pname == "uk_core_news_trf";
     in
     buildPythonPackage {
       inherit pname version;
@@ -40,19 +46,32 @@ let
 
       propagatedBuildInputs =
         [ spacy ]
-        ++ lib.optionals (lib.hasSuffix "_trf" pname) [ spacy-transformers ]
+        ++ lib.optionals (lib.hasSuffix "_trf" pname) [ spacy-curated-transformers ]
+        ++ lib.optionals requires-transformers [ transformers ]
+        ++ lib.optionals (lang == "ja") [
+          sudachidict-core
+          sudachipy
+        ]
         ++ lib.optionals (lang == "ru") [ pymorphy3 ]
         ++ lib.optionals (lang == "uk") [
           pymorphy3
           pymorphy3-dicts-uk
         ]
         ++ lib.optionals (lang == "zh") [ spacy-pkuseg ]
-        ++ lib.optionals (pname == "fr_dep_news_trf") [ sentencepiece ];
+        ++ lib.optionals requires-sentencepiece [ sentencepiece ];
 
-      postPatch = lib.optionalString requires-protobuf ''
-        substituteInPlace meta.json \
-          --replace "protobuf<3.21.0" "protobuf"
-      '';
+      postPatch =
+        lib.optionalString requires-protobuf ''
+          substituteInPlace meta.json \
+            --replace-fail "protobuf<3.21.0" "protobuf"
+        ''
+        + lib.optionalString (lang == "zh") ''
+          # Uses numpy 2.x, while the rest of the dependencies still uses
+          # numpy 1.x. Remove once all spaCy packages are updated for
+          # numpy 2.x.
+          substituteInPlace meta.json \
+            --replace-fail "spacy-pkuseg>=1.0.0,<2.0.0" "spacy-pkuseg"
+        '';
 
       nativeBuildInputs = [ setuptools ] ++ lib.optionals requires-protobuf [ protobuf ];
 

pkgs/top-level/python-packages.nix

@@ -2831,6 +2831,10 @@ self: super: with self; {
 
   cu2qu = callPackage ../development/python-modules/cu2qu { };
 
+  curated-tokenizers = callPackage ../development/python-modules/curated-tokenizers { };
+
+  curated-transformers = callPackage ../development/python-modules/curated-transformers { };
+
   customtkinter = callPackage ../development/python-modules/customtkinter { };
 
   cucumber-tag-expressions = callPackage ../development/python-modules/cucumber-tag-expressions { };
@@ -15327,6 +15331,8 @@ self: super: with self; {
 
   spacy-alignments = callPackage ../development/python-modules/spacy-alignments { };
 
+  spacy-curated-transformers = callPackage ../development/python-modules/spacy-curated-transformers { };
+
   spacy-legacy = callPackage ../development/python-modules/spacy/legacy.nix { };
 
   spacy-loggers = callPackage ../development/python-modules/spacy-loggers { };