diff --git a/pkgs/by-name/lu/luarocks-packages-updater/updater.py b/pkgs/by-name/lu/luarocks-packages-updater/updater.py index 555cde43ec55..8b3d1bcd9f49 100755 --- a/pkgs/by-name/lu/luarocks-packages-updater/updater.py +++ b/pkgs/by-name/lu/luarocks-packages-updater/updater.py @@ -1,15 +1,16 @@ #!/usr/bin/env python # format: -# $ nix run nixpkgs#python3Packages.ruff -- update.py +# $ nix run nixpkgs#python3Packages.ruff -- updater.py # type-check: -# $ nix run nixpkgs#python3Packages.mypy -- update.py +# $ nix run nixpkgs#python3Packages.mypy -- updater.py # linted: -# $ nix run nixpkgs#python3Packages.flake8 -- --ignore E501,E265,E402 update.py +# $ nix run nixpkgs#python3Packages.flake8 -- --ignore E501,E265,E402 updater.py import csv import inspect import logging import os +import re import shutil import subprocess import tempfile @@ -18,8 +19,8 @@ from dataclasses import dataclass from multiprocessing.dummy import Pool from pathlib import Path -import nixpkgs_plugin_update -from nixpkgs_plugin_update import FetchConfig, update_plugins +import nixpkgs_plugin_update # type: ignore +from nixpkgs_plugin_update import FetchConfig, Redirects, update_plugins class ColoredFormatter(logging.Formatter): @@ -57,10 +58,13 @@ HEADER = """/* */ """.format(GENERATED_NIXFILE=GENERATED_NIXFILE) -FOOTER = """ +FOOTER = ( + textwrap.dedent(""" } # GENERATED - do not edit this file -""" +""").strip() + + "\n" +) @dataclass @@ -87,6 +91,20 @@ class LuaPlugin: return self.name.replace(".", "-") +def extract_version(nix_expr: str) -> str | None: + match = re.search(r'version\s*=\s*"([^"]+)"', nix_expr) + if match: + return match.group(1) + return None + + +def extract_rev(nix_expr: str) -> str | None: + match = re.search(r'rev\s*=\s*"([^"]+)"', nix_expr) + if match: + return match.group(1) + return None + + # rename Editor to LangUpdate/ EcosystemUpdater class LuaEditor(nixpkgs_plugin_update.Editor): def create_parser(self): @@ -94,10 +112,10 @@ class LuaEditor(nixpkgs_plugin_update.Editor): parser.set_defaults(proc=1) return parser - def get_current_plugins(self): + def get_current_plugins(self, _config: FetchConfig, _nixpkgs: str): return [] - def load_plugin_spec(self, input_file) -> list[LuaPlugin]: + def load_plugin_spec(self, _config: FetchConfig, input_file) -> list[LuaPlugin]: luaPackages = [] csvfilename = input_file log.info("Loading package descriptions from %s", csvfilename) @@ -119,7 +137,7 @@ class LuaEditor(nixpkgs_plugin_update.Editor): with tempfile.NamedTemporaryFile("w+") as f: f.write(HEADER) header2 = textwrap.dedent( - """ + """ { stdenv, lib, @@ -133,7 +151,7 @@ class LuaEditor(nixpkgs_plugin_update.Editor): ) f.write(header2) for plugin, nix_expr in results: - f.write(f"{plugin.normalized_name} = {nix_expr}") + f.write(f" {plugin.normalized_name} = {nix_expr}") f.write(FOOTER) f.flush() @@ -150,6 +168,51 @@ class LuaEditor(nixpkgs_plugin_update.Editor): def attr_path(self): return "luaPackages" + def parse_generated_nix(self, input_file: str) -> dict[str, str]: + plugins: dict[str, str] = {} + if not os.path.exists(input_file): + return plugins + + with open(input_file, "r") as f: + content = f.read() + + start_marker = "final: prev: {" + start_idx = content.find(start_marker) + if start_idx == -1: + log.warning("Could not find start marker in generated file") + return plugins + start_idx += len(start_marker) + + # We assume the file ends with the footer or at least a closing brace + lines = content[start_idx:].splitlines() + + current_name = None + current_lines = [] + + # Regex to match start of a plugin definition: name = callPackage ( + start_pattern = re.compile(r"^\s+([\w\.\-]+)\s+=\s+callPackage\s+\(") + # Regex to match end of a plugin definition: ) { }; + end_pattern = re.compile(r"^\s+\)\s+\{\s+\};$") + + for line in lines: + if current_name is None: + match = start_pattern.match(line) + if match: + current_name = match.group(1) + # We need to keep the RHS of the assignment + # line is " name = callPackage (" + # We want "callPackage (" + rhs = line.split("=", 1)[1].strip() + current_lines = [rhs] + else: + current_lines.append(line) + if end_pattern.match(line): + plugins[current_name] = "\n".join(current_lines) + "\n\n" + current_name = None + current_lines = [] + + return plugins + def get_update( self, input_file: str, @@ -162,18 +225,51 @@ class LuaEditor(nixpkgs_plugin_update.Editor): raise NotImplementedError("For now, lua updater doesn't support updating individual packages.") _prefetch = generate_pkg_nix - def update() -> dict: - plugin_specs = self.load_plugin_spec(input_file) + def update() -> tuple[Redirects, list[tuple[str, str, str]]]: + plugin_specs = self.load_plugin_spec(config, input_file) sorted_plugin_specs = sorted(plugin_specs, key=lambda v: v.name.lower()) + # Load existing plugins to preserve them if update fails + existing_plugins = self.parse_generated_nix(output_file) + + old_versions = {} + for name, expr in existing_plugins.items(): + v = extract_version(expr) + if v: + r = extract_rev(expr) + old_versions[name] = f"{v}-{r}" if r else v + try: pool = Pool(processes=config.proc) results = pool.map(_prefetch, sorted_plugin_specs) finally: pass - successful_results = [(plug, nix_expr) for plug, nix_expr, error in results if nix_expr is not None] - errors = [(plug, error) for plug, nix_expr, error in results if error is not None] + results_map = {} + for plug, nix_expr, error in results: + results_map[plug.normalized_name] = (nix_expr, error) + + successful_results = [] + errors = [] + updated_plugins: list[tuple[str, str, str]] = [] + + for plug in sorted_plugin_specs: + nix_expr, error = results_map.get(plug.normalized_name, (None, "Unknown error")) + + final_expr = None + if nix_expr: + final_expr = nix_expr + successful_results.append((plug, nix_expr)) + else: + # Failed + log.error(f"Update failed for {plug.name}. Error: {error}") + errors.append((plug, error)) + + if final_expr: + new_ver = extract_version(final_expr) + old_ver = old_versions.get(plug.normalized_name) + if new_ver and old_ver and new_ver != old_ver: + updated_plugins.append((plug.normalized_name, old_ver, new_ver)) self.generate_nix(successful_results, output_file) @@ -182,12 +278,12 @@ class LuaEditor(nixpkgs_plugin_update.Editor): for plug, error in errors: log.error("%s: %s", plug.name, error) - redirects = {} - return redirects + redirects: Redirects = {} + return redirects, updated_plugins return update - def rewrite_input(self, input_file: str, *args, **kwargs): + def rewrite_input(self, _config: FetchConfig, _input_file: str, *args, **kwargs): # vim plugin reads the file before update but that shouldn't be our case # not implemented yet # fieldnames = ['name', 'server', 'version', 'luaversion', 'maintainers'] @@ -235,8 +331,8 @@ def generate_pkg_nix(plug: LuaPlugin): if plug.luaversion: cmd.append(f"--lua-version={plug.luaversion}") luaver = plug.luaversion.replace(".", "") - if luaver := os.getenv(f"LUA_{luaver}"): - cmd.append(f"--lua-dir={luaver}") + if lua_dir := os.getenv(f"LUA_{luaver}"): + cmd.append(f"--lua-dir={lua_dir}") log.debug("running %s", " ".join(cmd)) @@ -254,8 +350,8 @@ def main(): "lua", ROOT, "", - default_in=PKG_LIST, - default_out=GENERATED_NIXFILE, + default_in=Path(PKG_LIST), + default_out=Path(GENERATED_NIXFILE), ) editor.run()