diff --git a/pkgs/applications/editors/vim/plugins/utils/nvim-treesitter/update-shell.nix b/pkgs/applications/editors/vim/plugins/utils/nvim-treesitter/update-shell.nix index 573c98993f3a..d811a1b35c35 100644 --- a/pkgs/applications/editors/vim/plugins/utils/nvim-treesitter/update-shell.nix +++ b/pkgs/applications/editors/vim/plugins/utils/nvim-treesitter/update-shell.nix @@ -5,19 +5,16 @@ with pkgs; let - inherit (vimPlugins) nvim-treesitter; - - neovim = pkgs.neovim.override { - configure.packages.all.start = [ nvim-treesitter ]; - }; + pythonWithPackages = python3.withPackages ( + ps: with ps; [ + requests + ] + ); in mkShell { packages = [ - neovim nurl - python3 + pythonWithPackages ]; - - NVIM_TREESITTER = nvim-treesitter; } diff --git a/pkgs/applications/editors/vim/plugins/utils/nvim-treesitter/update.py b/pkgs/applications/editors/vim/plugins/utils/nvim-treesitter/update.py index 55363c9db84f..6b19eb24ebf6 100755 --- a/pkgs/applications/editors/vim/plugins/utils/nvim-treesitter/update.py +++ b/pkgs/applications/editors/vim/plugins/utils/nvim-treesitter/update.py @@ -5,104 +5,128 @@ import json import logging import os import subprocess -import sys from concurrent.futures import ThreadPoolExecutor +import requests + log = logging.getLogger("vim-updater") +NURR_JSON_URL = "https://raw.githubusercontent.com/nvim-neorocks/nurr/main/tree-sitter-parsers.json" -def generate_grammar(lang, rev, cfg): - """Generate grammar for a language""" - info = cfg["install_info"] - url = info["url"] +def generate_grammar(lang, parser_info): + """Generate grammar for a language based on the parser info""" + try: + if "install_info" not in parser_info: + log.warning(f"Parser {lang} does not have install_info, skipping") + return "" - generated = f""" {lang} = buildGrammar {{ + install_info = parser_info["install_info"] + + url = install_info["url"] + rev = install_info["revision"] + + generated = f""" {lang} = buildGrammar {{ language = "{lang}"; version = "0.0.0+rev={rev[:7]}"; src = """ - generated += subprocess.check_output(["nurl", url, rev, "--indent=4"], text=True) - generated += ";" + generated += subprocess.check_output(["nurl", url, rev, "--indent=4"], text=True) + generated += ";" - location = info.get("location") - if location: - generated += f""" + location = install_info.get("location", "") + if location: + generated += f""" location = "{location}";""" - if info.get("requires_generate_from_grammar"): - generated += """ + if install_info.get("generate", False): + generated += """ generate = true;""" - generated += f""" + generated += f""" meta.homepage = "{url}"; }}; """ - return generated + return generated + except Exception as e: + log.error(f"Error generating grammar for {lang}: {e}") + return "" -def update_grammars(nvim_treesitter_dir: str): - """ - The lockfile contains just revisions so we start neovim to dump the - grammar information in a better format - """ - # the lockfile - cmd = [ - "nvim", - "--headless", - "-u", - "NONE", - "--cmd", - f"set rtp^={nvim_treesitter_dir}", - "+lua io.write(vim.json.encode(require('nvim-treesitter.parsers').get_parser_configs()))", - "+quit!", - ] - log.debug("Running command: %s", ' '.join(cmd)) - configs = json.loads(subprocess.check_output(cmd)) +def fetch_nurr_parsers(): + """Fetch the parser information from nurr repository""" + log.info("Fetching parser data from %s", NURR_JSON_URL) + + headers = {} + github_token = os.environ.get("GITHUB_TOKEN") + if github_token: + log.info("Using GITHUB_TOKEN for authentication") + headers["Authorization"] = f"token {github_token}" + else: + log.warning("No GITHUB_TOKEN found. GitHub API requests may be rate-limited.") + + response = requests.get(NURR_JSON_URL, headers=headers, timeout=30) + response.raise_for_status() + data = response.json() + + try: + parsers = data["parsers"] + except KeyError: + raise ValueError( + "Unexpected response from NURR:\n" + json.dumps(data, indent=2) + ) + log.info(f"Successfully fetched {len(parsers)} parsers") + return parsers + + +def process_parser_info(parser_info): + """Process a single parser info entry and generate grammar for it""" + try: + lang = parser_info["lang"] + return generate_grammar(lang, parser_info) + except Exception as e: + log.error(f"Error processing parser: {e}") + return "" + + +def update_grammars(): + """Update grammar definitions using nurr's parser information""" + parsers_info = fetch_nurr_parsers() generated_file = """# generated by pkgs/applications/editors/vim/plugins/utils/nvim-treesitter/update.py +# Using parser data from https://github.com/nvim-neorocks/nurr/blob/main/tree-sitter-parsers.json { buildGrammar, """ - # Get the output and format it properly nurl_output = subprocess.check_output(["nurl", "-Ls", ","], text=True).strip() - # Add proper indentation (2 spaces) to the comma-separated list indented_output = nurl_output.replace(",", ",\n ") generated_file += indented_output - generated_file += """, }: { """ - lockfile_path = os.path.join(nvim_treesitter_dir, "lockfile.json") - log.debug("Opening %s", lockfile_path) - with open(lockfile_path) as lockfile_fd: - lockfile = json.load(lockfile_fd) - - def _generate_grammar(item): - lang, lock = item - cfg = configs.get(lang) - if not cfg: - return "" - return generate_grammar(lang, lock["revision"], cfg) - - for generated in ThreadPoolExecutor(max_workers=5).map( - _generate_grammar, lockfile.items() - ): + # Process parsers in parallel for better performance + with ThreadPoolExecutor(max_workers=5) as executor: + for generated in executor.map(process_parser_info, parsers_info): generated_file += generated - generated_file += "}\n" + generated_file += "}\n" return generated_file if __name__ == "__main__": - generated = update_grammars(sys.argv[1]) + logging.basicConfig(level=logging.INFO, format='%(levelname)s: %(message)s') + + generated = update_grammars() output_path = os.path.join( os.path.dirname(__file__), "../../nvim-treesitter/generated.nix" ) - open(output_path, "w").write(generated) + log.info("Writing output to %s", output_path) + with open(output_path, "w") as f: + f.write(generated) + log.info("Successfully updated grammar definitions") diff --git a/pkgs/applications/editors/vim/plugins/utils/update.py b/pkgs/applications/editors/vim/plugins/utils/update.py index 9ddf64f8c307..9458b8f074d7 100755 --- a/pkgs/applications/editors/vim/plugins/utils/update.py +++ b/pkgs/applications/editors/vim/plugins/utils/update.py @@ -19,10 +19,8 @@ # import inspect -import json import logging import os -import subprocess import textwrap from pathlib import Path from typing import List, Tuple @@ -136,20 +134,7 @@ class VimEditor(pluginupdate.Editor): # TODO this should probably be skipped when running outside a nixpkgs checkout if self.nvim_treesitter_updated: print("updating nvim-treesitter grammars") - cmd = [ - "nix", - "build", - "vimPlugins.nvim-treesitter.src", - "-f", - self.nixpkgs, - "--print-out-paths", - ] - log.debug("Running command: %s", " ".join(cmd)) - nvim_treesitter_dir = subprocess.check_output( - cmd, text=True, timeout=90 - ).strip() - - generated = treesitter.update_grammars(nvim_treesitter_dir) + generated = treesitter.update_grammars() treesitter_generated_nix_path = os.path.join( NIXPKGS_NVIMTREESITTER_FOLDER, "generated.nix" ) diff --git a/pkgs/applications/editors/vim/plugins/utils/updater.nix b/pkgs/applications/editors/vim/plugins/utils/updater.nix index b2dbdf9fca41..9560708fbb0e 100644 --- a/pkgs/applications/editors/vim/plugins/utils/updater.nix +++ b/pkgs/applications/editors/vim/plugins/utils/updater.nix @@ -25,6 +25,7 @@ buildPythonApplication { pythonPath = [ python3Packages.gitpython + python3Packages.requests ]; dontUnpack = true;