tree-sitter: make tree-sitter-grammars an overridable scope (#537058)

pkgs/applications/editors/emacs/elisp-packages/manual-packages/treesit-grammars/package.nix

@@ -15,9 +15,9 @@ let
 
   # Usage:
   # treesit-grammars.with-grammars (p: [ p.tree-sitter-bash p.tree-sitter-c ... ])
-  with-grammars = fn: grammarPackage (fn pkgs.tree-sitter.builtGrammars);
+  with-grammars = fn: grammarPackage (fn pkgs.tree-sitter-grammars.derivations);
 
-  with-all-grammars = grammarPackage pkgs.tree-sitter.allGrammars;
+  with-all-grammars = grammarPackage pkgs.tree-sitter-grammars.allGrammars;
 in
 {
   inherit with-grammars with-all-grammars;

pkgs/applications/editors/vim/plugins/nvim-treesitter/overrides.nix

@@ -4,6 +4,7 @@
   symlinkJoin,
   vimUtils,
   tree-sitter,
+  tree-sitter-grammars,
   neovim,
   neovimUtils,
   runCommand,
@@ -127,7 +128,7 @@ let
   withPlugins =
     f:
     let
-      selectedGrammars = f (tree-sitter.builtGrammars // builtGrammars);
+      selectedGrammars = f (tree-sitter-grammars.derivations // builtGrammars);
 
       grammarPlugins = map grammarToPlugin selectedGrammars;
 

pkgs/by-name/di/diffsitter/package.nix

@@ -4,32 +4,19 @@
   linkFarm,
   makeWrapper,
   rustPlatform,
-  tree-sitter,
+  tree-sitter-grammars,
   gitUpdater,
   versionCheckHook,
 }:
 
 let
   # based on https://github.com/NixOS/nixpkgs/blob/aa07b78b9606daf1145a37f6299c6066939df075/pkgs/development/tools/parsing/tree-sitter/default.nix#L85-L104
-  withPlugins =
-    grammarFn:
-    let
-      grammars = grammarFn tree-sitter.builtGrammars;
-    in
-    linkFarm "grammars" (
-      map (
-        drv:
-        let
-          name = lib.strings.getName drv;
-        in
-        {
-          name = "lib" + (lib.strings.removeSuffix "-grammar" name) + ".so";
-          path = "${drv}/parser";
-        }
-      ) grammars
-    );
+  grammarToAttrSet = drv: {
+    name = "lib" + (lib.strings.removeSuffix "-grammar" (lib.strings.getName drv)) + ".so";
+    path = "${drv}/parser";
+  };
 
-  libPath = withPlugins (_: tree-sitter.allGrammars);
+  libPath = linkFarm "grammars" (map grammarToAttrSet tree-sitter-grammars.allGrammars);
 in
 rustPlatform.buildRustPackage rec {
   pname = "diffsitter";

pkgs/by-name/he/helix/package.nix

@@ -6,7 +6,7 @@
   helix-unwrapped,
   removeReferencesTo,
   pkgs,
-  tree-sitter,
+  tree-sitter-grammars,
   lockedGrammars ? lib.importJSON ./grammars.json,
   grammarsOverlay ? (
     final: prev: {
@@ -66,13 +66,9 @@ let
         }
     ) prev;
 
-  tree-sitter-grammars =
+  helixTreeSitterGrammars =
     lib.filterAttrs (drvName: _: lib.hasAttr (lib.removePrefix "tree-sitter-" drvName) lockedGrammars)
-      (
-        tree-sitter.grammarsScope.overrideScope (
-          lib.composeExtensions lockedVersionsOverlay grammarsOverlay
-        )
-      );
+      (tree-sitter-grammars.overrideScope (lib.composeExtensions lockedVersionsOverlay grammarsOverlay));
 
   # Dynamic libraries for the grammars always use the `.so` extension, also on Darwin (should use `.dylib`)
   # See here: https://github.com/helix-editor/helix/pull/14982
@@ -82,7 +78,7 @@ let
     lib.concatMapAttrsStringSep "\n" (_: grammar: ''
       install -D ${grammar}/parser $out/${grammar.language}.so
       ${lib.getExe removeReferencesTo} -t ${grammar} $out/${grammar.language}.so
-    '') (lib.filterAttrs (_: lib.isDerivation) tree-sitter-grammars)
+    '') helixTreeSitterGrammars
   );
 
   lockedGrammarsCount = lib.length (lib.attrNames lockedGrammars);
@@ -113,7 +109,7 @@ symlinkJoin {
   passthru = {
     updateScript = ./update.sh;
     runtime = runtimeDir;
-    inherit tree-sitter-grammars;
+    tree-sitter-grammars = helixTreeSitterGrammars;
   };
 
   meta = {

pkgs/by-name/tr/tree-sitter/grammars/README.md

@@ -109,6 +109,59 @@ This includes build-related flags and metadata.
 }
 ```
 
+## Overriding the Grammar Set
+
+Use `pkgs.tree-sitter-grammars.overrideScope` when adding a grammar or replacing a grammar that another package should consume.
+`pkgs.tree-sitter-grammars` is the scoped package set used for grammar overrides and scoped helpers such as `derivations`, `allGrammars`, and `withPlugins`.
+
+```nix
+let
+  grammars = pkgs.tree-sitter-grammars.overrideScope (
+    final: prev: {
+      tree-sitter-foolang = pkgs.tree-sitter.buildGrammar {
+        language = "foolang";
+        version = "0.42.0";
+        src = pkgs.fetchFromGitHub {
+          owner = "example";
+          repo = "tree-sitter-foolang";
+          rev = "v0.42.0";
+          hash = "sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=";
+        };
+      };
+
+      tree-sitter-rust = prev.tree-sitter-rust.overrideAttrs (_: {
+        version = "custom";
+        src = pkgs.fetchFromGitHub {
+          owner = "example";
+          repo = "tree-sitter-rust";
+          rev = "custom";
+          hash = "sha256-AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=";
+        };
+      });
+    }
+  );
+in
+grammars.withPlugins (p: [
+  p.tree-sitter-foolang
+  p.tree-sitter-rust
+])
+```
+
+The scoped `withPlugins` helper receives derivations from the same overridden scope, so added or replaced grammars are visible.
+
+The set also carries package-set helpers (`callPackage`, `newScope`, `overrideScope`, …) alongside the grammars, so do not iterate it directly.
+Use one of its grammar-only views instead; each reflects any `overrideScope`:
+
+- `pkgs.tree-sitter-grammars.derivations` — attrset of every grammar derivation, including grammars marked broken.
+- `pkgs.tree-sitter-grammars.allGrammars` — list of the non-broken grammar derivations.
+- `pkgs.tree-sitter-grammars.withPlugins` — build a grammar link farm.
+
+```nix
+builtins.attrValues pkgs.tree-sitter-grammars.derivations
+```
+
+`pkgs.tree-sitter.builtGrammars` remains the plain attribute set generated directly from [grammar-sources.nix](grammar-sources.nix); use it when you specifically want the stock grammars without any scope overrides.
+
 ## Building WebAssembly Parsers
 
 `buildGrammar` builds a native `$out/parser`.

pkgs/by-name/tr/tree-sitter/package.nix

@@ -75,26 +75,12 @@ let
   */
   builtGrammars = lib.mapAttrs (_: lib.makeOverridable buildGrammar) grammars;
 
-  /**
-    # Extensible package set for tree-sitter grammars.
-    # Provides .override and .extend for customization.
-    # Note: Use builtGrammars (not this) when iterating over grammars,
-    # as this includes package set functions alongside derivations
-  */
-  grammarsScope = lib.makeScope newScope (self: builtGrammars);
+  grammarDerivationsFrom = lib.filterAttrs (
+    name: value: lib.hasPrefix "tree-sitter-" name && lib.isDerivation value
+  );
 
-  # Usage:
-  # pkgs.tree-sitter.withPlugins (p: [ p.tree-sitter-c p.tree-sitter-java ... ])
-  #
-  # or for all grammars:
-  # pkgs.tree-sitter.withPlugins (_: pkgs.tree-sitter.allGrammars)
-  # which is equivalent to
-  # pkgs.tree-sitter.withPlugins (p: builtins.attrValues p)
-  withPlugins =
-    grammarFn:
-    let
-      grammars = grammarFn builtGrammars;
-    in
+  mkGrammarLinkFarm =
+    grammars:
     linkFarm "grammars" (
       map (
         drv:
@@ -112,7 +98,32 @@ let
       ) grammars
     );
 
-  allGrammars = lib.filter (p: !(p.meta.broken or false)) (lib.attrValues builtGrammars);
+  /**
+    Extensible package set of compiled tree-sitter grammars.
+
+    Exposed as `pkgs.tree-sitter-grammars` and `pkgs.tree-sitter.grammarsScope`.
+    Customize with `.overrideScope`; overrides propagate to every consumer that
+    reads the scope, including the grammar-only views below (which the
+    `pkgs.tree-sitter` passthru re-exports so there is a single source of truth):
+
+      `.derivations`  attrset of every grammar derivation
+      `.allGrammars`  list of non-broken grammar derivations
+      `.withPlugins`  build a grammar link farm
+
+    The scope also carries package-set helpers (`callPackage`, `overrideScope`,
+    …) alongside the grammars, so prefer one of the views above when iterating.
+  */
+  grammarsScope = lib.makeScope newScope (
+    self:
+    builtGrammars
+    // {
+      derivations = grammarDerivationsFrom self;
+      allGrammars = lib.filter (p: !(p.meta.broken or false)) (
+        lib.attrValues (grammarDerivationsFrom self)
+      );
+      withPlugins = grammarFn: mkGrammarLinkFarm (grammarFn (grammarDerivationsFrom self));
+    }
+  );
 
   isWasi = stdenv.hostPlatform.isWasi;
 
@@ -237,14 +248,15 @@ rustPlatform.buildRustPackage (finalAttrs: {
 
   passthru = {
     inherit
-      grammars
       buildGrammar
       builtGrammars
+      grammars
       grammarsScope
-      withPlugins
-      allGrammars
       ;
 
+    # Keep legacy `pkgs.tree-sitter` views wired to the overridable scope.
+    inherit (grammarsScope) allGrammars withPlugins;
+
     updateScript = nix-update-script { };
 
     tests = {

pkgs/top-level/all-packages.nix

@@ -5419,7 +5419,7 @@ with pkgs;
 
   tflint-plugins = recurseIntoAttrs (callPackage ../development/tools/analysis/tflint-plugins { });
 
-  tree-sitter-grammars = recurseIntoAttrs tree-sitter.builtGrammars;
+  tree-sitter-grammars = recurseIntoAttrs tree-sitter.grammarsScope;
 
   uhdMinimal = uhd.override {
     enableUtils = false;

pkgs/top-level/python-packages.nix

@@ -20330,7 +20330,7 @@ self: super: with self; {
             "tree-sitter-sshclientconfig"
             "tree-sitter-templ"
           ])
-        ) pkgs.tree-sitter.builtGrammars
+        ) pkgs.tree-sitter-grammars.derivations
       )
   );