diff --git a/pkgs/development/python-modules/tokenizers/default.nix b/pkgs/development/python-modules/tokenizers/default.nix index 724fcc3df3af..d307f6edae15 100644 --- a/pkgs/development/python-modules/tokenizers/default.nix +++ b/pkgs/development/python-modules/tokenizers/default.nix @@ -21,6 +21,7 @@ # tests datasets, numpy, + pytest-asyncio, pytestCheckHook, requests, tiktoken, @@ -75,14 +76,14 @@ let in buildPythonPackage rec { pname = "tokenizers"; - version = "0.21.4"; + version = "0.22.0"; pyproject = true; src = fetchFromGitHub { owner = "huggingface"; repo = "tokenizers"; tag = "v${version}"; - hash = "sha256-HJUycrNDpy2FOYi6aZ76orLewZCuLC1MoJ57peYJqvI="; + hash = "sha256-HTJQ5nPkOsVqYzcsm0GLflX+teqDsrpIb5nf5pa7Gpc="; }; cargoDeps = rustPlatform.fetchCargoVendor { @@ -92,7 +93,7 @@ buildPythonPackage rec { src sourceRoot ; - hash = "sha256-0olujhOOO/BAH4JvnmXd1kE7T/sp5Vr3Z3P2X2jhZKs="; + hash = "sha256-X9tsn4gPg7Ih/8NNiCBllgcZgUR/tok+mwCJE53Z/8g="; }; sourceRoot = "${src.name}/bindings/python"; @@ -117,6 +118,7 @@ buildPythonPackage rec { nativeCheckInputs = [ datasets numpy + pytest-asyncio pytestCheckHook requests tiktoken @@ -138,6 +140,23 @@ buildPythonPackage rec { "test_splitting" "TestTrainFromIterators" + # Require downloading from huggingface + # huggingface_hub.errors.LocalEntryNotFoundError + "test_async_methods_existence" + "test_basic_encoding" + "test_concurrency" + "test_decode" + "test_decode_skip_special_tokens" + "test_decode_stream_fallback" + "test_encode" + "test_error_handling" + "test_large_batch" + "test_numpy_inputs" + "test_performance_comparison" + "test_various_input_formats" + "test_with_special_tokens" + "test_with_truncation_padding" + # Those tests require more data "test_from_pretrained" "test_from_pretrained_revision"