python3Packages.pytorch-tokenizers: 1.2.0 -> 1.3.0 (#526467)

pkgs/development/python-modules/executorch/default.nix

@@ -102,6 +102,12 @@ buildPythonPackage.override { inherit (torch) stdenv; } (finalAttrs: {
 
       sed -i "1i #include <cstdint>" backends/apple/coreml/runtime/inmemoryfs/memory_buffer.hpp
       sed -i "1i #include <cstdint>" extension/llm/tokenizers/third-party/sentencepiece/src/sentencepiece_processor.h
+    ''
+    + ''
+      substituteInPlace extension/llm/tokenizers/test/test_python_bindings.py \
+        --replace-fail \
+          'self.assertEqual(pytorch_tokenizers.__version__, "0.1.0")' \
+          'self.assertEqual(pytorch_tokenizers.__version__, "${pytorch-tokenizers.version}")'
     '';
 
   env = {

pkgs/development/python-modules/pytorch-tokenizers/default.nix

@@ -30,15 +30,16 @@ let
 in
 buildPythonPackage (finalAttrs: {
   pname = "pytorch-tokenizers";
-  version = "1.2.0";
+  version = "1.3.0";
   pyproject = true;
+  __structuredAttrs = true;
 
   src = fetchFromGitHub {
     owner = "meta-pytorch";
     repo = "tokenizers";
     tag = "v${finalAttrs.version}";
     fetchSubmodules = true;
-    hash = "sha256-4VWOKCdRx1VpYoJq7LYfpdcAAQeHnLD5mxI65XrrEHs=";
+    hash = "sha256-1G6mDUSwy4KXKgdtEimj9rrQDonGHdo8R8DvPQppvwE=";
   };
 
   patches = [
@@ -62,6 +63,17 @@ buildPythonPackage (finalAttrs: {
   ];
   dontUseCmakeConfigure = true;
 
+  # pkgs/by-name/cm/cmake/setup-hook.sh
+  preBuild = ''
+    if ! [[ -v enableParallelBuilding ]]; then
+        enableParallelBuilding=1
+        echo "cmake: enabled parallel building"
+    fi
+    if [[ "$enableParallelBuilding" -ne 0 ]]; then
+        export CMAKE_BUILD_PARALLEL_LEVEL=$NIX_BUILD_CORES
+    fi
+  '';
+
   dependencies = [
     sentencepiece
     tiktoken
@@ -90,6 +102,7 @@ buildPythonPackage (finalAttrs: {
   meta = {
     description = "C++ implementations for various tokenizers (sentencepiece, tiktoken, etc.)";
     homepage = "https://github.com/meta-pytorch/tokenizers";
+    changelog = "https://github.com/meta-pytorch/tokenizers/releases/tag/${finalAttrs.src.tag}";
     license = lib.licenses.bsd3;
     maintainers = with lib.maintainers; [ GaetanLepage ];
   };