nvim-treesitter/update.py: source from nurr

Upstream no longer updating the sources in the lockfile.json, but they
still update in their main branch's parsers.lua. We don't need to parse
that file since nurr is already doing that for us. We can just grab
latest parser info from their json.

pkgs/applications/editors/vim/plugins/utils/nvim-treesitter/update-shell.nix

@@ -5,19 +5,16 @@
 with pkgs;
 
 let
-  inherit (vimPlugins) nvim-treesitter;
-
-  neovim = pkgs.neovim.override {
-    configure.packages.all.start = [ nvim-treesitter ];
-  };
+  pythonWithPackages = python3.withPackages (
+    ps: with ps; [
+      requests
+    ]
+  );
 in
 
 mkShell {
   packages = [
-    neovim
     nurl
-    python3
+    pythonWithPackages
   ];
-
-  NVIM_TREESITTER = nvim-treesitter;
 }

pkgs/applications/editors/vim/plugins/utils/nvim-treesitter/update.py

@@ -5,104 +5,128 @@ import json
 import logging
 import os
 import subprocess
-import sys
 from concurrent.futures import ThreadPoolExecutor
 
+import requests
+
 log = logging.getLogger("vim-updater")
 
+NURR_JSON_URL = "https://raw.githubusercontent.com/nvim-neorocks/nurr/main/tree-sitter-parsers.json"
 
-def generate_grammar(lang, rev, cfg):
-    """Generate grammar for a language"""
-    info = cfg["install_info"]
-    url = info["url"]
+def generate_grammar(lang, parser_info):
+    """Generate grammar for a language based on the parser info"""
+    try:
+        if "install_info" not in parser_info:
+            log.warning(f"Parser {lang} does not have install_info, skipping")
+            return ""
 
-    generated = f"""  {lang} = buildGrammar {{
+        install_info = parser_info["install_info"]
+
+        url = install_info["url"]
+        rev = install_info["revision"]
+
+        generated = f"""  {lang} = buildGrammar {{
     language = "{lang}";
     version = "0.0.0+rev={rev[:7]}";
     src = """
 
-    generated += subprocess.check_output(["nurl", url, rev, "--indent=4"], text=True)
-    generated += ";"
+        generated += subprocess.check_output(["nurl", url, rev, "--indent=4"], text=True)
+        generated += ";"
 
-    location = info.get("location")
-    if location:
-        generated += f"""
+        location = install_info.get("location", "")
+        if location:
+            generated += f"""
     location = "{location}";"""
 
-    if info.get("requires_generate_from_grammar"):
-        generated += """
+        if install_info.get("generate", False):
+            generated += """
     generate = true;"""
 
-    generated += f"""
+        generated += f"""
     meta.homepage = "{url}";
   }};
 """
 
-    return generated
+        return generated
+    except Exception as e:
+        log.error(f"Error generating grammar for {lang}: {e}")
+        return ""
 
 
-def update_grammars(nvim_treesitter_dir: str):
-    """
-    The lockfile contains just revisions so we start neovim to dump the
-    grammar information in a better format
-    """
-    # the lockfile
-    cmd = [
-        "nvim",
-        "--headless",
-        "-u",
-        "NONE",
-        "--cmd",
-        f"set rtp^={nvim_treesitter_dir}",
-        "+lua io.write(vim.json.encode(require('nvim-treesitter.parsers').get_parser_configs()))",
-        "+quit!",
-    ]
-    log.debug("Running command: %s", ' '.join(cmd))
-    configs = json.loads(subprocess.check_output(cmd))
+def fetch_nurr_parsers():
+    """Fetch the parser information from nurr repository"""
+    log.info("Fetching parser data from %s", NURR_JSON_URL)
+
+    headers = {}
+    github_token = os.environ.get("GITHUB_TOKEN")
+    if github_token:
+        log.info("Using GITHUB_TOKEN for authentication")
+        headers["Authorization"] = f"token {github_token}"
+    else:
+        log.warning("No GITHUB_TOKEN found. GitHub API requests may be rate-limited.")
+
+    response = requests.get(NURR_JSON_URL, headers=headers, timeout=30)
+    response.raise_for_status()
+    data = response.json()
+
+    try:
+        parsers = data["parsers"]
+    except KeyError:
+        raise ValueError(
+            "Unexpected response from NURR:\n" + json.dumps(data, indent=2)
+        )
+    log.info(f"Successfully fetched {len(parsers)} parsers")
+    return parsers
+
+
+def process_parser_info(parser_info):
+    """Process a single parser info entry and generate grammar for it"""
+    try:
+        lang = parser_info["lang"]
+        return generate_grammar(lang, parser_info)
+    except Exception as e:
+        log.error(f"Error processing parser: {e}")
+        return ""
+
+
+def update_grammars():
+    """Update grammar definitions using nurr's parser information"""
+    parsers_info = fetch_nurr_parsers()
 
     generated_file = """# generated by pkgs/applications/editors/vim/plugins/utils/nvim-treesitter/update.py
+# Using parser data from https://github.com/nvim-neorocks/nurr/blob/main/tree-sitter-parsers.json
 
 {
   buildGrammar,
   """
 
-    # Get the output and format it properly
     nurl_output = subprocess.check_output(["nurl", "-Ls", ","], text=True).strip()
-    # Add proper indentation (2 spaces) to the comma-separated list
     indented_output = nurl_output.replace(",", ",\n  ")
     generated_file += indented_output
-
     generated_file += """,
 }:
 
 {
 """
 
-    lockfile_path = os.path.join(nvim_treesitter_dir, "lockfile.json")
-    log.debug("Opening %s", lockfile_path)
-    with open(lockfile_path) as lockfile_fd:
-        lockfile = json.load(lockfile_fd)
-
-        def _generate_grammar(item):
-            lang, lock = item
-            cfg = configs.get(lang)
-            if not cfg:
-                return ""
-            return generate_grammar(lang, lock["revision"], cfg)
-
-        for generated in ThreadPoolExecutor(max_workers=5).map(
-            _generate_grammar, lockfile.items()
-        ):
+    # Process parsers in parallel for better performance
+    with ThreadPoolExecutor(max_workers=5) as executor:
+        for generated in executor.map(process_parser_info, parsers_info):
             generated_file += generated
 
-        generated_file += "}\n"
+    generated_file += "}\n"
     return generated_file
 
 
 if __name__ == "__main__":
-    generated = update_grammars(sys.argv[1])
+    logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s')
+
+    generated = update_grammars()
     output_path = os.path.join(
         os.path.dirname(__file__),
         "../../nvim-treesitter/generated.nix"
     )
-    open(output_path, "w").write(generated)
+    log.info("Writing output to %s", output_path)
+    with open(output_path, "w") as f:
+        f.write(generated)
+    log.info("Successfully updated grammar definitions")

pkgs/applications/editors/vim/plugins/utils/update.py

@@ -19,10 +19,8 @@
 #
 
 import inspect
-import json
 import logging
 import os
-import subprocess
 import textwrap
 from pathlib import Path
 from typing import List, Tuple
@@ -136,20 +134,7 @@ class VimEditor(pluginupdate.Editor):
         # TODO this should probably be skipped when running outside a nixpkgs checkout
         if self.nvim_treesitter_updated:
             print("updating nvim-treesitter grammars")
-            cmd = [
-                "nix",
-                "build",
-                "vimPlugins.nvim-treesitter.src",
-                "-f",
-                self.nixpkgs,
-                "--print-out-paths",
-            ]
-            log.debug("Running command: %s", " ".join(cmd))
-            nvim_treesitter_dir = subprocess.check_output(
-                cmd, text=True, timeout=90
-            ).strip()
-
-            generated = treesitter.update_grammars(nvim_treesitter_dir)
+            generated = treesitter.update_grammars()
             treesitter_generated_nix_path = os.path.join(
                 NIXPKGS_NVIMTREESITTER_FOLDER, "generated.nix"
             )

pkgs/applications/editors/vim/plugins/utils/updater.nix

@@ -25,6 +25,7 @@ buildPythonApplication {
 
   pythonPath = [
     python3Packages.gitpython
+    python3Packages.requests
   ];
 
   dontUnpack = true;