diff --git a/pkgs/development/python-modules/executorch/default.nix b/pkgs/development/python-modules/executorch/default.nix index 16c5f2255d10..c727a336c17e 100644 --- a/pkgs/development/python-modules/executorch/default.nix +++ b/pkgs/development/python-modules/executorch/default.nix @@ -102,6 +102,12 @@ buildPythonPackage.override { inherit (torch) stdenv; } (finalAttrs: { sed -i "1i #include " backends/apple/coreml/runtime/inmemoryfs/memory_buffer.hpp sed -i "1i #include " extension/llm/tokenizers/third-party/sentencepiece/src/sentencepiece_processor.h + '' + + '' + substituteInPlace extension/llm/tokenizers/test/test_python_bindings.py \ + --replace-fail \ + 'self.assertEqual(pytorch_tokenizers.__version__, "0.1.0")' \ + 'self.assertEqual(pytorch_tokenizers.__version__, "${pytorch-tokenizers.version}")' ''; env = { diff --git a/pkgs/development/python-modules/pytorch-tokenizers/default.nix b/pkgs/development/python-modules/pytorch-tokenizers/default.nix index 842cef4a1697..55e789733d5c 100644 --- a/pkgs/development/python-modules/pytorch-tokenizers/default.nix +++ b/pkgs/development/python-modules/pytorch-tokenizers/default.nix @@ -30,15 +30,16 @@ let in buildPythonPackage (finalAttrs: { pname = "pytorch-tokenizers"; - version = "1.2.0"; + version = "1.3.0"; pyproject = true; + __structuredAttrs = true; src = fetchFromGitHub { owner = "meta-pytorch"; repo = "tokenizers"; tag = "v${finalAttrs.version}"; fetchSubmodules = true; - hash = "sha256-4VWOKCdRx1VpYoJq7LYfpdcAAQeHnLD5mxI65XrrEHs="; + hash = "sha256-1G6mDUSwy4KXKgdtEimj9rrQDonGHdo8R8DvPQppvwE="; }; patches = [ @@ -62,6 +63,17 @@ buildPythonPackage (finalAttrs: { ]; dontUseCmakeConfigure = true; + # pkgs/by-name/cm/cmake/setup-hook.sh + preBuild = '' + if ! [[ -v enableParallelBuilding ]]; then + enableParallelBuilding=1 + echo "cmake: enabled parallel building" + fi + if [[ "$enableParallelBuilding" -ne 0 ]]; then + export CMAKE_BUILD_PARALLEL_LEVEL=$NIX_BUILD_CORES + fi + ''; + dependencies = [ sentencepiece tiktoken @@ -90,6 +102,7 @@ buildPythonPackage (finalAttrs: { meta = { description = "C++ implementations for various tokenizers (sentencepiece, tiktoken, etc.)"; homepage = "https://github.com/meta-pytorch/tokenizers"; + changelog = "https://github.com/meta-pytorch/tokenizers/releases/tag/${finalAttrs.src.tag}"; license = lib.licenses.bsd3; maintainers = with lib.maintainers; [ GaetanLepage ]; };