diff --git a/pkgs/development/python-modules/tree-sitter-language-pack/default.nix b/pkgs/development/python-modules/tree-sitter-language-pack/default.nix index f35fbeb99301..090b6215651d 100644 --- a/pkgs/development/python-modules/tree-sitter-language-pack/default.nix +++ b/pkgs/development/python-modules/tree-sitter-language-pack/default.nix @@ -1,85 +1,159 @@ { lib, + buildPackages, buildPythonPackage, - fetchPypi, + fetchFromGitHub, + fetchurl, + python, pytestCheckHook, - nix-update-script, - - # build-system - cython, - setuptools, - typing-extensions, - - # dependencies + rustPlatform, + stdenv, tree-sitter, - tree-sitter-c-sharp, - tree-sitter-embedded-template, - tree-sitter-yaml, }: -buildPythonPackage rec { - pname = "tree-sitter-language-pack"; - version = "0.13.0"; - pyproject = true; +let + parserReleaseUrl = + version: "https://github.com/kreuzberg-dev/tree-sitter-language-pack/releases/download/v${version}"; - # Using the GitHub sources necessitates fetching the treesitter grammar parsers by using a vendored script. - # The pypi archive has the benefit of already vendoring those dependencies which makes packaging easier on our side - # See: https://github.com/Goldziher/tree-sitter-language-pack/blob/main/scripts/clone_vendors.py - src = fetchPypi { - pname = "tree_sitter_language_pack"; - inherit version; - hash = "sha256-AyA0xeJ7H24AcwuefC28ggO0cA0MaB/QGdbe/PYRg+w="; + parserBundleSpecs = { + aarch64-darwin = { + suffix = "macos-arm64"; + hash = "sha256-pYrgwhb3BkOEqot5JBi26aXBciGt7/zP/1+HcQT2vsw="; + }; + aarch64-linux = { + suffix = "linux-aarch64"; + hash = "sha256-t1rWm19iExYAZXluMQqlt9bOkEC2UumcxDov8YmYEEQ="; + }; + x86_64-linux = { + suffix = "linux-x86_64"; + hash = "sha256-o4IpLZDitTsHfF2KMnyB3Wry7Hig7Byxd0JLcZPybJ0="; + }; + }; +in +buildPythonPackage (finalAttrs: { + pname = "tree-sitter-language-pack"; + pyproject = true; + version = "1.4.1"; + + src = fetchFromGitHub { + owner = "kreuzberg-dev"; + repo = "tree-sitter-language-pack"; + tag = "v${finalAttrs.version}"; + hash = "sha256-kN2htitEOo+JF6DCrC4RHmHkZXnUA0fUo2jSbMELQHI="; }; - # Upstream bumped dependencies aggressively, but we can still use older - # versions since the newer ones aren’t packaged in nixpkgs. We can't use - # pythonRelaxDepsHook here because it runs in postBuild, while the dependency - # check occurs during the build phase. - postPatch = '' - substituteInPlace pyproject.toml \ - --replace-fail "typing-extensions>=4.15.0" "typing-extensions>=4.14.1" - ''; + cargoDeps = rustPlatform.fetchCargoVendor { + inherit (finalAttrs) + pname + version + src + ; + hash = "sha256-ii3rvAfs4xMSyEEDjUrjL2SAONd0ARCVhwQNCJLwuCk="; + }; - nativeCheckInputs = [ - pytestCheckHook + buildAndTestSubdir = "crates/ts-pack-python"; + + # Pin the release metadata and per-platform parser archive so runtime use stays offline. + parserManifest = fetchurl { + url = "${parserReleaseUrl finalAttrs.version}/parsers.json"; + hash = "sha256-8utASonvrLzOjxZcmRuzuFSGtYe5sEoMU+xz++bfmkk="; + }; + + parserBundle = + let + spec = + parserBundleSpecs.${stdenv.hostPlatform.system} + or (throw "tree-sitter-language-pack parser bundle is unavailable for ${stdenv.hostPlatform.system}"); + in + fetchurl { + url = "${parserReleaseUrl finalAttrs.version}/parsers-${spec.suffix}.tar.zst"; + inherit (spec) hash; + }; + + nativeBuildInputs = [ + buildPackages.zstd + rustPlatform.cargoSetupHook + rustPlatform.maturinBuildHook ]; - build-system = [ - cython - setuptools - typing-extensions + nativeCheckInputs = [ pytestCheckHook ]; + + dependencies = [ tree-sitter ]; + + disabledTests = [ + # tree-sitter-language-pack 1.4.1 upstream smoke tests expect these aliases + # to resolve directly in the offline cache, but the packaged bundle still + # exposes the underlying parser library names. + "test_get_language_returns_non_none" + "test_get_parser_for_previously_broken_languages" + "test_has_language_for_previously_broken" ]; - dependencies = [ - tree-sitter - tree-sitter-c-sharp - tree-sitter-embedded-template - tree-sitter-yaml - ]; - - pythonRelaxDeps = [ - "tree-sitter" - "tree-sitter-embedded-template" - "tree-sitter-yaml" - ]; - - pythonImportsCheck = [ - "tree_sitter_language_pack" - "tree_sitter_language_pack.bindings" - ]; - - # make sure import the built version, not the source one preCheck = '' - rm -r tree_sitter_language_pack + # Mirror the upstream cache layout: libs live in cache_dir, while the manifest + # is expected at cache_dir/../manifest.json. + cacheRoot=$PWD/.tree-sitter-language-pack-cache + cacheDir="$cacheRoot/libs" + mkdir -p "$cacheDir" + cp ${finalAttrs.parserManifest} "$cacheRoot/manifest.json" + ${lib.getExe buildPackages.zstd} -d -c ${finalAttrs.parserBundle} | tar -xvf - -C "$cacheDir" >/dev/null + + # Upstream smoke tests call download APIs even when the parsers are already + # available locally, so point them at the pre-fetched cache and short-circuit + # redundant network downloads during pytest. + cat > conftest.py </dev/null + + # Make the installed package default to the pre-fetched cache in $out. + substituteInPlace $out/${python.sitePackages}/tree_sitter_language_pack/__init__.py \ + --replace-fail 'SupportedLanguage: TypeAlias = str' $'configure(cache_dir="'$cacheDir$'")\n\nSupportedLanguage: TypeAlias = str' + ''; + + pythonImportsCheck = [ "tree_sitter_language_pack" ]; + + passthru.updateScript = ./update.sh; meta = { - description = "Comprehensive collection of tree-sitter languages"; - homepage = "https://github.com/Goldziher/tree-sitter-language-pack"; - changelog = "https://github.com/Goldziher/tree-sitter-language-pack/releases/tag/v${version}"; + description = "Comprehensive collection of tree-sitter language parsers with polyglot bindings"; + homepage = "https://github.com/kreuzberg-dev/tree-sitter-language-pack"; + changelog = "https://github.com/kreuzberg-dev/tree-sitter-language-pack/releases/tag/v${finalAttrs.version}"; license = lib.licenses.mit; maintainers = with lib.maintainers; [ yzx9 ]; + platforms = builtins.attrNames parserBundleSpecs; }; -} +})