From d0c3e2387f9ac57989875b8613293ea6f76d5ee3 Mon Sep 17 00:00:00 2001 From: Zexin Yuan Date: Thu, 26 Mar 2026 12:17:07 +0800 Subject: [PATCH 1/4] python3Packages.tree-sitter-language-pack: 0.13.0 -> 1.4.1 --- .../tree-sitter-language-pack/default.nix | 198 ++++++++++++------ 1 file changed, 136 insertions(+), 62 deletions(-) diff --git a/pkgs/development/python-modules/tree-sitter-language-pack/default.nix b/pkgs/development/python-modules/tree-sitter-language-pack/default.nix index f35fbeb99301..090b6215651d 100644 --- a/pkgs/development/python-modules/tree-sitter-language-pack/default.nix +++ b/pkgs/development/python-modules/tree-sitter-language-pack/default.nix @@ -1,85 +1,159 @@ { lib, + buildPackages, buildPythonPackage, - fetchPypi, + fetchFromGitHub, + fetchurl, + python, pytestCheckHook, - nix-update-script, - - # build-system - cython, - setuptools, - typing-extensions, - - # dependencies + rustPlatform, + stdenv, tree-sitter, - tree-sitter-c-sharp, - tree-sitter-embedded-template, - tree-sitter-yaml, }: -buildPythonPackage rec { - pname = "tree-sitter-language-pack"; - version = "0.13.0"; - pyproject = true; +let + parserReleaseUrl = + version: "https://github.com/kreuzberg-dev/tree-sitter-language-pack/releases/download/v${version}"; - # Using the GitHub sources necessitates fetching the treesitter grammar parsers by using a vendored script. - # The pypi archive has the benefit of already vendoring those dependencies which makes packaging easier on our side - # See: https://github.com/Goldziher/tree-sitter-language-pack/blob/main/scripts/clone_vendors.py - src = fetchPypi { - pname = "tree_sitter_language_pack"; - inherit version; - hash = "sha256-AyA0xeJ7H24AcwuefC28ggO0cA0MaB/QGdbe/PYRg+w="; + parserBundleSpecs = { + aarch64-darwin = { + suffix = "macos-arm64"; + hash = "sha256-pYrgwhb3BkOEqot5JBi26aXBciGt7/zP/1+HcQT2vsw="; + }; + aarch64-linux = { + suffix = "linux-aarch64"; + hash = "sha256-t1rWm19iExYAZXluMQqlt9bOkEC2UumcxDov8YmYEEQ="; + }; + x86_64-linux = { + suffix = "linux-x86_64"; + hash = "sha256-o4IpLZDitTsHfF2KMnyB3Wry7Hig7Byxd0JLcZPybJ0="; + }; + }; +in +buildPythonPackage (finalAttrs: { + pname = "tree-sitter-language-pack"; + pyproject = true; + version = "1.4.1"; + + src = fetchFromGitHub { + owner = "kreuzberg-dev"; + repo = "tree-sitter-language-pack"; + tag = "v${finalAttrs.version}"; + hash = "sha256-kN2htitEOo+JF6DCrC4RHmHkZXnUA0fUo2jSbMELQHI="; }; - # Upstream bumped dependencies aggressively, but we can still use older - # versions since the newer ones aren’t packaged in nixpkgs. We can't use - # pythonRelaxDepsHook here because it runs in postBuild, while the dependency - # check occurs during the build phase. - postPatch = '' - substituteInPlace pyproject.toml \ - --replace-fail "typing-extensions>=4.15.0" "typing-extensions>=4.14.1" - ''; + cargoDeps = rustPlatform.fetchCargoVendor { + inherit (finalAttrs) + pname + version + src + ; + hash = "sha256-ii3rvAfs4xMSyEEDjUrjL2SAONd0ARCVhwQNCJLwuCk="; + }; - nativeCheckInputs = [ - pytestCheckHook + buildAndTestSubdir = "crates/ts-pack-python"; + + # Pin the release metadata and per-platform parser archive so runtime use stays offline. + parserManifest = fetchurl { + url = "${parserReleaseUrl finalAttrs.version}/parsers.json"; + hash = "sha256-8utASonvrLzOjxZcmRuzuFSGtYe5sEoMU+xz++bfmkk="; + }; + + parserBundle = + let + spec = + parserBundleSpecs.${stdenv.hostPlatform.system} + or (throw "tree-sitter-language-pack parser bundle is unavailable for ${stdenv.hostPlatform.system}"); + in + fetchurl { + url = "${parserReleaseUrl finalAttrs.version}/parsers-${spec.suffix}.tar.zst"; + inherit (spec) hash; + }; + + nativeBuildInputs = [ + buildPackages.zstd + rustPlatform.cargoSetupHook + rustPlatform.maturinBuildHook ]; - build-system = [ - cython - setuptools - typing-extensions + nativeCheckInputs = [ pytestCheckHook ]; + + dependencies = [ tree-sitter ]; + + disabledTests = [ + # tree-sitter-language-pack 1.4.1 upstream smoke tests expect these aliases + # to resolve directly in the offline cache, but the packaged bundle still + # exposes the underlying parser library names. + "test_get_language_returns_non_none" + "test_get_parser_for_previously_broken_languages" + "test_has_language_for_previously_broken" ]; - dependencies = [ - tree-sitter - tree-sitter-c-sharp - tree-sitter-embedded-template - tree-sitter-yaml - ]; - - pythonRelaxDeps = [ - "tree-sitter" - "tree-sitter-embedded-template" - "tree-sitter-yaml" - ]; - - pythonImportsCheck = [ - "tree_sitter_language_pack" - "tree_sitter_language_pack.bindings" - ]; - - # make sure import the built version, not the source one preCheck = '' - rm -r tree_sitter_language_pack + # Mirror the upstream cache layout: libs live in cache_dir, while the manifest + # is expected at cache_dir/../manifest.json. + cacheRoot=$PWD/.tree-sitter-language-pack-cache + cacheDir="$cacheRoot/libs" + mkdir -p "$cacheDir" + cp ${finalAttrs.parserManifest} "$cacheRoot/manifest.json" + ${lib.getExe buildPackages.zstd} -d -c ${finalAttrs.parserBundle} | tar -xvf - -C "$cacheDir" >/dev/null + + # Upstream smoke tests call download APIs even when the parsers are already + # available locally, so point them at the pre-fetched cache and short-circuit + # redundant network downloads during pytest. + cat > conftest.py </dev/null + + # Make the installed package default to the pre-fetched cache in $out. + substituteInPlace $out/${python.sitePackages}/tree_sitter_language_pack/__init__.py \ + --replace-fail 'SupportedLanguage: TypeAlias = str' $'configure(cache_dir="'$cacheDir$'")\n\nSupportedLanguage: TypeAlias = str' + ''; + + pythonImportsCheck = [ "tree_sitter_language_pack" ]; + + passthru.updateScript = ./update.sh; meta = { - description = "Comprehensive collection of tree-sitter languages"; - homepage = "https://github.com/Goldziher/tree-sitter-language-pack"; - changelog = "https://github.com/Goldziher/tree-sitter-language-pack/releases/tag/v${version}"; + description = "Comprehensive collection of tree-sitter language parsers with polyglot bindings"; + homepage = "https://github.com/kreuzberg-dev/tree-sitter-language-pack"; + changelog = "https://github.com/kreuzberg-dev/tree-sitter-language-pack/releases/tag/v${finalAttrs.version}"; license = lib.licenses.mit; maintainers = with lib.maintainers; [ yzx9 ]; + platforms = builtins.attrNames parserBundleSpecs; }; -} +}) From f19e087f0ab5faace9f9854170f8f397891bc437 Mon Sep 17 00:00:00 2001 From: Zexin Yuan Date: Fri, 27 Mar 2026 09:37:53 +0800 Subject: [PATCH 2/4] python3Packages.tree-sitter-language-pack: add update script --- .../tree-sitter-language-pack/update.sh | 86 +++++++++++++++++++ 1 file changed, 86 insertions(+) create mode 100755 pkgs/development/python-modules/tree-sitter-language-pack/update.sh diff --git a/pkgs/development/python-modules/tree-sitter-language-pack/update.sh b/pkgs/development/python-modules/tree-sitter-language-pack/update.sh new file mode 100755 index 000000000000..9949108e7c92 --- /dev/null +++ b/pkgs/development/python-modules/tree-sitter-language-pack/update.sh @@ -0,0 +1,86 @@ +#!/usr/bin/env nix-shell +#!nix-shell -I nixpkgs=./. -i bash -p bash curl jq gnused coreutils nix nix-update + +set -euo pipefail + +PACKAGE_DIR="$(cd -- "$(dirname "$0")" >/dev/null 2>&1 && pwd -P)" +NIXPKGS_ROOT="$(cd -- "$PACKAGE_DIR/../../../.." >/dev/null 2>&1 && pwd -P)" +PACKAGE_FILE="$PACKAGE_DIR/default.nix" +ATTR_PATH="${UPDATE_NIX_ATTR_PATH:-python3Packages.tree-sitter-language-pack}" + +latest_version() { + curl -sL ${GITHUB_TOKEN:+ -H "Authorization: Bearer $GITHUB_TOKEN"} \ + "https://api.github.com/repos/kreuzberg-dev/tree-sitter-language-pack/releases/latest" \ + | jq -r '.tag_name | sub("^v"; "")' +} + +replace_value() { + local pattern="$1" + local replacement="$2" + sed -i "s|$pattern|$replacement|g" "$PACKAGE_FILE" +} + +replace_perl() { + local pattern="$1" + local replacement="$2" + perl -0pi -e "s|$pattern|$replacement|s" "$PACKAGE_FILE" +} + +prefetch_sri() { + local url="$1" + local hash + hash="$(nix-prefetch-url --type sha256 "$url")" + nix hash convert --hash-algo sha256 --to sri "$hash" +} + +version="${1:-$(latest_version)}" + +if [[ "$version" == "${UPDATE_NIX_OLD_VERSION:-}" ]]; then + echo "$ATTR_PATH is already at $version" + exit 0 +fi + +nix-update "$ATTR_PATH" --version "$version" + +release_url="https://github.com/kreuzberg-dev/tree-sitter-language-pack/releases/download/v$version" + +manifest_hash="$(prefetch_sri "$release_url/parsers.json")" +aarch64_darwin_hash="$(prefetch_sri "$release_url/parsers-macos-arm64.tar.zst")" +aarch64_linux_hash="$(prefetch_sri "$release_url/parsers-linux-aarch64.tar.zst")" +x86_64_linux_hash="$(prefetch_sri "$release_url/parsers-linux-x86_64.tar.zst")" + +replace_perl '(parserManifest = fetchurl \{\n url = "\$\{parserReleaseUrl finalAttrs\.version\}/parsers\.json";\n hash = ")[^"]*(";)' "\${1}$manifest_hash\${2}" +replace_perl '(aarch64-darwin = \{\n suffix = "macos-arm64";\n hash = ")[^"]*(";)' "\${1}$aarch64_darwin_hash\${2}" +replace_perl '(aarch64-linux = \{\n suffix = "linux-aarch64";\n hash = ")[^"]*(";)' "\${1}$aarch64_linux_hash\${2}" +replace_perl '(x86_64-linux = \{\n suffix = "linux-x86_64";\n hash = ")[^"]*(";)' "\${1}$x86_64_linux_hash\${2}" + +old_cargo_hash="$( + perl -0ne 'print "$1\n" if /cargoDeps = rustPlatform\.fetchCargoVendor \{\n(?:.*\n)*? hash = "([^"]+)";/s' "$PACKAGE_FILE" +)" +fake_hash='sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=' +replace_value "$old_cargo_hash" "$fake_hash" + +set +e +build_output="$( + cd "$NIXPKGS_ROOT" && + nix build ".#$ATTR_PATH" 2>&1 >/dev/null +)" +build_status=$? +set -e + +if [[ $build_status -eq 0 ]]; then + echo "expected cargo hash mismatch build to fail, but it succeeded" >&2 + exit 1 +fi + +new_cargo_hash="$(printf '%s\n' "$build_output" | sed -n 's/.*got:[[:space:]]*\(sha256-[A-Za-z0-9+/=]*\).*/\1/p' | head -n1)" + +if [[ -z "$new_cargo_hash" ]]; then + printf '%s\n' "$build_output" >&2 + echo "failed to extract cargo hash from build output" >&2 + exit 1 +fi + +replace_value "$fake_hash" "$new_cargo_hash" + +echo "updated $ATTR_PATH to $version" From 4b6828438676be74156ff13c764f268bbf964f45 Mon Sep 17 00:00:00 2001 From: Zexin Yuan Date: Thu, 26 Mar 2026 21:48:53 +0800 Subject: [PATCH 3/4] vectorcode: unbroken --- pkgs/by-name/ve/vectorcode/package.nix | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkgs/by-name/ve/vectorcode/package.nix b/pkgs/by-name/ve/vectorcode/package.nix index 130c7cead400..29f6325183ff 100644 --- a/pkgs/by-name/ve/vectorcode/package.nix +++ b/pkgs/by-name/ve/vectorcode/package.nix @@ -206,6 +206,9 @@ python.pkgs.buildPythonApplication rec { "test_get_reranker" "test_query_tool_success" "test_supported_rerankers_initialization" + # tree-sitter-language-pack 1.x.x raises LanguageNotFoundError for unknown + # languages here, while this test still expects LookupError. + "test_treesitter_chunker_parser_from_config_no_parser_found_error" ]; passthru = { From e0611dbc8740200b3a73292cf0ee261fd5e3ca1f Mon Sep 17 00:00:00 2001 From: Zexin Yuan Date: Thu, 26 Mar 2026 21:49:01 +0800 Subject: [PATCH 4/4] aider-chat: unbroken --- pkgs/by-name/ai/aider-chat/package.nix | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pkgs/by-name/ai/aider-chat/package.nix b/pkgs/by-name/ai/aider-chat/package.nix index 2d314a108d31..3cb0384b1e22 100644 --- a/pkgs/by-name/ai/aider-chat/package.nix +++ b/pkgs/by-name/ai/aider-chat/package.nix @@ -37,6 +37,9 @@ let pythonRemoveDeps = [ "importlib-resources" + "tree-sitter-c-sharp" + "tree-sitter-embedded-template" + "tree-sitter-yaml" ]; build-system = with python3Packages; [ setuptools-scm ]; @@ -195,6 +198,10 @@ let "test_main_exit_calls_version_check" # AssertionError: assert 2 == 1 "test_simple_send_non_retryable_error" + # Upstream tests incompatible with current litellm version + "test_max_context_tokens" + "test_cmd_tokens_output" + "test_cmd_read_only_with_image_file" ] ++ lib.optionals stdenv.hostPlatform.isDarwin [ # Tests fails on darwin