From 34970867e2e6513186841b99daecbc34a4c9c3f1 Mon Sep 17 00:00:00 2001 From: Austin Horstman Date: Sat, 17 May 2025 23:56:42 -0500 Subject: [PATCH] nvim-treesitter/update.py: source from nurr Upstream no longer updating the sources in the lockfile.json, but they still update in their main branch's parsers.lua. We don't need to parse that file since nurr is already doing that for us. We can just grab latest parser info from their json. --- .../utils/nvim-treesitter/update-shell.nix | 15 +- .../plugins/utils/nvim-treesitter/update.py | 132 +++++++++++------- .../editors/vim/plugins/utils/update.py | 17 +-- .../editors/vim/plugins/utils/updater.nix | 1 + 4 files changed, 86 insertions(+), 79 deletions(-) diff --git a/pkgs/applications/editors/vim/plugins/utils/nvim-treesitter/update-shell.nix b/pkgs/applications/editors/vim/plugins/utils/nvim-treesitter/update-shell.nix index 573c98993f3a..d811a1b35c35 100644 --- a/pkgs/applications/editors/vim/plugins/utils/nvim-treesitter/update-shell.nix +++ b/pkgs/applications/editors/vim/plugins/utils/nvim-treesitter/update-shell.nix @@ -5,19 +5,16 @@ with pkgs; let - inherit (vimPlugins) nvim-treesitter; - - neovim = pkgs.neovim.override { - configure.packages.all.start = [ nvim-treesitter ]; - }; + pythonWithPackages = python3.withPackages ( + ps: with ps; [ + requests + ] + ); in mkShell { packages = [ - neovim nurl - python3 + pythonWithPackages ]; - - NVIM_TREESITTER = nvim-treesitter; } diff --git a/pkgs/applications/editors/vim/plugins/utils/nvim-treesitter/update.py b/pkgs/applications/editors/vim/plugins/utils/nvim-treesitter/update.py index 55363c9db84f..6b19eb24ebf6 100755 --- a/pkgs/applications/editors/vim/plugins/utils/nvim-treesitter/update.py +++ b/pkgs/applications/editors/vim/plugins/utils/nvim-treesitter/update.py @@ -5,104 +5,128 @@ import json import logging import os import subprocess -import sys from concurrent.futures import ThreadPoolExecutor +import requests + log = logging.getLogger("vim-updater") +NURR_JSON_URL = "https://raw.githubusercontent.com/nvim-neorocks/nurr/main/tree-sitter-parsers.json" -def generate_grammar(lang, rev, cfg): - """Generate grammar for a language""" - info = cfg["install_info"] - url = info["url"] +def generate_grammar(lang, parser_info): + """Generate grammar for a language based on the parser info""" + try: + if "install_info" not in parser_info: + log.warning(f"Parser {lang} does not have install_info, skipping") + return "" - generated = f""" {lang} = buildGrammar {{ + install_info = parser_info["install_info"] + + url = install_info["url"] + rev = install_info["revision"] + + generated = f""" {lang} = buildGrammar {{ language = "{lang}"; version = "0.0.0+rev={rev[:7]}"; src = """ - generated += subprocess.check_output(["nurl", url, rev, "--indent=4"], text=True) - generated += ";" + generated += subprocess.check_output(["nurl", url, rev, "--indent=4"], text=True) + generated += ";" - location = info.get("location") - if location: - generated += f""" + location = install_info.get("location", "") + if location: + generated += f""" location = "{location}";""" - if info.get("requires_generate_from_grammar"): - generated += """ + if install_info.get("generate", False): + generated += """ generate = true;""" - generated += f""" + generated += f""" meta.homepage = "{url}"; }}; """ - return generated + return generated + except Exception as e: + log.error(f"Error generating grammar for {lang}: {e}") + return "" -def update_grammars(nvim_treesitter_dir: str): - """ - The lockfile contains just revisions so we start neovim to dump the - grammar information in a better format - """ - # the lockfile - cmd = [ - "nvim", - "--headless", - "-u", - "NONE", - "--cmd", - f"set rtp^={nvim_treesitter_dir}", - "+lua io.write(vim.json.encode(require('nvim-treesitter.parsers').get_parser_configs()))", - "+quit!", - ] - log.debug("Running command: %s", ' '.join(cmd)) - configs = json.loads(subprocess.check_output(cmd)) +def fetch_nurr_parsers(): + """Fetch the parser information from nurr repository""" + log.info("Fetching parser data from %s", NURR_JSON_URL) + + headers = {} + github_token = os.environ.get("GITHUB_TOKEN") + if github_token: + log.info("Using GITHUB_TOKEN for authentication") + headers["Authorization"] = f"token {github_token}" + else: + log.warning("No GITHUB_TOKEN found. GitHub API requests may be rate-limited.") + + response = requests.get(NURR_JSON_URL, headers=headers, timeout=30) + response.raise_for_status() + data = response.json() + + try: + parsers = data["parsers"] + except KeyError: + raise ValueError( + "Unexpected response from NURR:\n" + json.dumps(data, indent=2) + ) + log.info(f"Successfully fetched {len(parsers)} parsers") + return parsers + + +def process_parser_info(parser_info): + """Process a single parser info entry and generate grammar for it""" + try: + lang = parser_info["lang"] + return generate_grammar(lang, parser_info) + except Exception as e: + log.error(f"Error processing parser: {e}") + return "" + + +def update_grammars(): + """Update grammar definitions using nurr's parser information""" + parsers_info = fetch_nurr_parsers() generated_file = """# generated by pkgs/applications/editors/vim/plugins/utils/nvim-treesitter/update.py +# Using parser data from https://github.com/nvim-neorocks/nurr/blob/main/tree-sitter-parsers.json { buildGrammar, """ - # Get the output and format it properly nurl_output = subprocess.check_output(["nurl", "-Ls", ","], text=True).strip() - # Add proper indentation (2 spaces) to the comma-separated list indented_output = nurl_output.replace(",", ",\n ") generated_file += indented_output - generated_file += """, }: { """ - lockfile_path = os.path.join(nvim_treesitter_dir, "lockfile.json") - log.debug("Opening %s", lockfile_path) - with open(lockfile_path) as lockfile_fd: - lockfile = json.load(lockfile_fd) - - def _generate_grammar(item): - lang, lock = item - cfg = configs.get(lang) - if not cfg: - return "" - return generate_grammar(lang, lock["revision"], cfg) - - for generated in ThreadPoolExecutor(max_workers=5).map( - _generate_grammar, lockfile.items() - ): + # Process parsers in parallel for better performance + with ThreadPoolExecutor(max_workers=5) as executor: + for generated in executor.map(process_parser_info, parsers_info): generated_file += generated - generated_file += "}\n" + generated_file += "}\n" return generated_file if __name__ == "__main__": - generated = update_grammars(sys.argv[1]) + logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s') + + generated = update_grammars() output_path = os.path.join( os.path.dirname(__file__), "../../nvim-treesitter/generated.nix" ) - open(output_path, "w").write(generated) + log.info("Writing output to %s", output_path) + with open(output_path, "w") as f: + f.write(generated) + log.info("Successfully updated grammar definitions") diff --git a/pkgs/applications/editors/vim/plugins/utils/update.py b/pkgs/applications/editors/vim/plugins/utils/update.py index 9ddf64f8c307..9458b8f074d7 100755 --- a/pkgs/applications/editors/vim/plugins/utils/update.py +++ b/pkgs/applications/editors/vim/plugins/utils/update.py @@ -19,10 +19,8 @@ # import inspect -import json import logging import os -import subprocess import textwrap from pathlib import Path from typing import List, Tuple @@ -136,20 +134,7 @@ class VimEditor(pluginupdate.Editor): # TODO this should probably be skipped when running outside a nixpkgs checkout if self.nvim_treesitter_updated: print("updating nvim-treesitter grammars") - cmd = [ - "nix", - "build", - "vimPlugins.nvim-treesitter.src", - "-f", - self.nixpkgs, - "--print-out-paths", - ] - log.debug("Running command: %s", " ".join(cmd)) - nvim_treesitter_dir = subprocess.check_output( - cmd, text=True, timeout=90 - ).strip() - - generated = treesitter.update_grammars(nvim_treesitter_dir) + generated = treesitter.update_grammars() treesitter_generated_nix_path = os.path.join( NIXPKGS_NVIMTREESITTER_FOLDER, "generated.nix" ) diff --git a/pkgs/applications/editors/vim/plugins/utils/updater.nix b/pkgs/applications/editors/vim/plugins/utils/updater.nix index b2dbdf9fca41..9560708fbb0e 100644 --- a/pkgs/applications/editors/vim/plugins/utils/updater.nix +++ b/pkgs/applications/editors/vim/plugins/utils/updater.nix @@ -25,6 +25,7 @@ buildPythonApplication { pythonPath = [ python3Packages.gitpython + python3Packages.requests ]; dontUnpack = true;