From e3cdb1ab755147f723384af3e847e89c8367e76c Mon Sep 17 00:00:00 2001
From: Gaetan Lepage <gaetan@glepage.com>
Date: Thu, 13 Mar 2025 13:11:27 +0100
Subject: [PATCH] python312Packages.tokenizers: 0.21.0 -> 0.21.1

Diff: https://github.com/huggingface/tokenizers/compare/refs/tags/v0.21.0...v0.21.1

Changelog: https://github.com/huggingface/tokenizers/releases/tag/v0.21.1
---
 .../python-modules/tokenizers/default.nix     | 29 ++++++++++++-------
 1 file changed, 18 insertions(+), 11 deletions(-)

diff --git a/pkgs/development/python-modules/tokenizers/default.nix b/pkgs/development/python-modules/tokenizers/default.nix
index f2df5f3e37be..dcc687cb5ca2 100644
--- a/pkgs/development/python-modules/tokenizers/default.nix
+++ b/pkgs/development/python-modules/tokenizers/default.nix
@@ -24,6 +24,7 @@
   pytestCheckHook,
   requests,
   tiktoken,
+  writableTmpDirAsHomeHook,
 }:
 
 let
@@ -70,16 +71,24 @@ let
 in
 buildPythonPackage rec {
   pname = "tokenizers";
-  version = "0.21.0";
+  version = "0.21.1";
   pyproject = true;
 
   src = fetchFromGitHub {
     owner = "huggingface";
     repo = "tokenizers";
     tag = "v${version}";
-    hash = "sha256-G65XiVlvJXOC9zqcVr9vWamUnpC0aa4kyYkE2v1K2iY=";
+    hash = "sha256-3S7ZCaZnnwyNjoZ4Y/q3ngQE2MIm2iyCCjYAkdMVG2A=";
   };
 
+  # TestUnigram.test_continuing_prefix_trainer_mismatch fails with:
+  # Exception: No such file or directory (os error 2)
+  # Fix submitted upstream: https://github.com/huggingface/tokenizers/pull/1747
+  postPatch = ''
+    substituteInPlace tests/bindings/test_trainers.py \
+      --replace-fail '"data/' '"tests/data/'
+  '';
+
   cargoDeps = rustPlatform.fetchCargoVendor {
     inherit
       pname
@@ -87,7 +96,7 @@ buildPythonPackage rec {
       src
       sourceRoot
       ;
-    hash = "sha256-jj5nuwxlfJm1ugYd5zW+wjyczOZHWCmRGYpmiMDqFlk=";
+    hash = "sha256-I7LlBmeVY2rWI0ta6x311iAurQKuutsClrbUgkt9xWk=";
   };
 
   sourceRoot = "${src.name}/bindings/python";
@@ -115,17 +124,15 @@ buildPythonPackage rec {
     pytestCheckHook
     requests
     tiktoken
+    writableTmpDirAsHomeHook
   ];
 
-  postUnpack = ''
+  postUnpack =
     # Add data files for tests, otherwise tests attempt network access
-    mkdir $sourceRoot/tests/data
-    ln -s ${test-data}/* $sourceRoot/tests/data/
-  '';
-
-  preCheck = ''
-    export HOME=$(mktemp -d);
-  '';
+    ''
+      mkdir $sourceRoot/tests/data
+      ln -s ${test-data}/* $sourceRoot/tests/data/
+    '';
 
   pythonImportsCheck = [ "tokenizers" ];