luarocks-packages-update: init (#262156)
* luarocks-packages-updater: init Goal is to make it possible to maintain out-of-tree luarocks packages without needing to clone nixpkgs. maintainers/scripts/update-luarocks-packages gets renamed to pkgs/development/lua-modules/updater/updater.py Once merged you can run for instance nix run nixpkgs#luarocks-packages-updater -- -i contrib/luarocks-packages.csv -o contrib/generated-packages.nix I also set the parallelism (--proc) to 1 by default else luarocks fails because of https://github.com/luarocks/luarocks/issues/1540 * Update maintainers/scripts/pluginupdate.py Co-authored-by: Marc Jakobi <mrcjkb89@outlook.com> --------- Co-authored-by: Marc Jakobi <mrcjkb89@outlook.com>
This commit is contained in:
Matthieu Coudron
co-authored by
Marc Jakobi
Marc Jakobi
parent
24df047637
commit
f15e58cbeb
@@ -0,0 +1,49 @@
|
||||
{ buildPythonApplication
|
||||
, nix
|
||||
, makeWrapper
|
||||
, python3Packages
|
||||
, lib
|
||||
# , nix-prefetch-git
|
||||
, nix-prefetch-scripts
|
||||
, luarocks-nix
|
||||
}:
|
||||
let
|
||||
|
||||
path = lib.makeBinPath [ nix nix-prefetch-scripts luarocks-nix ];
|
||||
in
|
||||
buildPythonApplication {
|
||||
pname = "luarocks-packages-updater";
|
||||
version = "0.1";
|
||||
|
||||
format = "other";
|
||||
|
||||
nativeBuildInputs = [
|
||||
makeWrapper
|
||||
python3Packages.wrapPython
|
||||
];
|
||||
propagatedBuildInputs = [
|
||||
python3Packages.gitpython
|
||||
];
|
||||
|
||||
dontUnpack = true;
|
||||
|
||||
installPhase =
|
||||
''
|
||||
mkdir -p $out/bin $out/lib
|
||||
cp ${./updater.py} $out/bin/luarocks-packages-updater
|
||||
cp ${../../../../maintainers/scripts/pluginupdate.py} $out/lib/pluginupdate.py
|
||||
|
||||
# wrap python scripts
|
||||
makeWrapperArgs+=( --prefix PATH : "${path}" --prefix PYTHONPATH : "$out/lib" )
|
||||
wrapPythonProgramsIn "$out"
|
||||
'';
|
||||
|
||||
shellHook = ''
|
||||
export PYTHONPATH="maintainers/scripts:$PYTHONPATH"
|
||||
export PATH="${path}:$PATH"
|
||||
'';
|
||||
|
||||
meta.mainProgram = "luarocks-packages-updater";
|
||||
}
|
||||
|
||||
|
||||
+217
@@ -0,0 +1,217 @@
|
||||
#!/usr/bin/env python
|
||||
# format:
|
||||
# $ nix run nixpkgs#python3Packages.black -- update.py
|
||||
# type-check:
|
||||
# $ nix run nixpkgs#python3Packages.mypy -- update.py
|
||||
# linted:
|
||||
# $ nix run nixpkgs#python3Packages.flake8 -- --ignore E501,E265,E402 update.py
|
||||
|
||||
import inspect
|
||||
import os
|
||||
import tempfile
|
||||
import shutil
|
||||
from dataclasses import dataclass
|
||||
import subprocess
|
||||
import csv
|
||||
import logging
|
||||
import textwrap
|
||||
from multiprocessing.dummy import Pool
|
||||
|
||||
from typing import List, Tuple, Optional
|
||||
from pathlib import Path
|
||||
|
||||
log = logging.getLogger()
|
||||
log.addHandler(logging.StreamHandler())
|
||||
|
||||
ROOT = Path(os.path.dirname(os.path.abspath(inspect.getfile(inspect.currentframe())))).parent.parent # type: ignore
|
||||
import pluginupdate
|
||||
from pluginupdate import update_plugins, FetchConfig, CleanEnvironment
|
||||
|
||||
PKG_LIST = "maintainers/scripts/luarocks-packages.csv"
|
||||
TMP_FILE = "$(mktemp)"
|
||||
GENERATED_NIXFILE = "pkgs/development/lua-modules/generated-packages.nix"
|
||||
|
||||
HEADER = """/* {GENERATED_NIXFILE} is an auto-generated file -- DO NOT EDIT!
|
||||
Regenerate it with: nix run nixpkgs#update-luarocks-packages
|
||||
You can customize the generated packages in pkgs/development/lua-modules/overrides.nix
|
||||
*/
|
||||
""".format(
|
||||
GENERATED_NIXFILE=GENERATED_NIXFILE
|
||||
)
|
||||
|
||||
FOOTER = """
|
||||
}
|
||||
/* GENERATED - do not edit this file */
|
||||
"""
|
||||
|
||||
|
||||
@dataclass
|
||||
class LuaPlugin:
|
||||
name: str
|
||||
"""Name of the plugin, as seen on luarocks.org"""
|
||||
src: str
|
||||
"""address to the git repository"""
|
||||
ref: Optional[str]
|
||||
"""git reference (branch name/tag)"""
|
||||
version: Optional[str]
|
||||
"""Set it to pin a package """
|
||||
server: Optional[str]
|
||||
"""luarocks.org registers packages under different manifests.
|
||||
Its value can be 'http://luarocks.org/dev'
|
||||
"""
|
||||
luaversion: Optional[str]
|
||||
"""Attribue of the lua interpreter if a package is available only for a specific lua version"""
|
||||
maintainers: Optional[str]
|
||||
""" Optional string listing maintainers separated by spaces"""
|
||||
|
||||
@property
|
||||
def normalized_name(self) -> str:
|
||||
return self.name.replace(".", "-")
|
||||
|
||||
|
||||
# rename Editor to LangUpdate/ EcosystemUpdater
|
||||
class LuaEditor(pluginupdate.Editor):
|
||||
|
||||
def create_parser(self):
|
||||
parser = super().create_parser()
|
||||
parser.set_defaults(proc=1)
|
||||
return parser
|
||||
|
||||
def get_current_plugins(self):
|
||||
return []
|
||||
|
||||
def load_plugin_spec(self, input_file) -> List[LuaPlugin]:
|
||||
luaPackages = []
|
||||
csvfilename = input_file
|
||||
log.info("Loading package descriptions from %s", csvfilename)
|
||||
|
||||
with open(csvfilename, newline="") as csvfile:
|
||||
reader = csv.DictReader(
|
||||
csvfile,
|
||||
)
|
||||
for row in reader:
|
||||
# name,server,version,luaversion,maintainers
|
||||
plugin = LuaPlugin(**row)
|
||||
luaPackages.append(plugin)
|
||||
return luaPackages
|
||||
|
||||
def update(self, args):
|
||||
update_plugins(self, args)
|
||||
|
||||
def generate_nix(self, results: List[Tuple[LuaPlugin, str]], outfilename: str):
|
||||
with tempfile.NamedTemporaryFile("w+") as f:
|
||||
f.write(HEADER)
|
||||
header2 = textwrap.dedent(
|
||||
"""
|
||||
{ stdenv, lib, fetchurl, fetchgit, callPackage, ... } @ args:
|
||||
final: prev:
|
||||
{
|
||||
"""
|
||||
)
|
||||
f.write(header2)
|
||||
for plugin, nix_expr in results:
|
||||
f.write(f"{plugin.normalized_name} = {nix_expr}")
|
||||
f.write(FOOTER)
|
||||
f.flush()
|
||||
|
||||
# if everything went fine, move the generated file to its destination
|
||||
# using copy since move doesn't work across disks
|
||||
shutil.copy(f.name, outfilename)
|
||||
|
||||
print(f"updated {outfilename}")
|
||||
|
||||
@property
|
||||
def attr_path(self):
|
||||
return "luaPackages"
|
||||
|
||||
def get_update(self, input_file: str, outfile: str, config: FetchConfig):
|
||||
_prefetch = generate_pkg_nix
|
||||
|
||||
def update() -> dict:
|
||||
plugin_specs = self.load_plugin_spec(input_file)
|
||||
sorted_plugin_specs = sorted(plugin_specs, key=lambda v: v.name.lower())
|
||||
|
||||
try:
|
||||
pool = Pool(processes=config.proc)
|
||||
results = pool.map(_prefetch, sorted_plugin_specs)
|
||||
finally:
|
||||
pass
|
||||
|
||||
self.generate_nix(results, outfile)
|
||||
|
||||
redirects = {}
|
||||
return redirects
|
||||
|
||||
return update
|
||||
|
||||
def rewrite_input(self, input_file: str, *args, **kwargs):
|
||||
# vim plugin reads the file before update but that shouldn't be our case
|
||||
# not implemented yet
|
||||
# fieldnames = ['name', 'server', 'version', 'luaversion', 'maintainers']
|
||||
# input_file = "toto.csv"
|
||||
# with open(input_file, newline='') as csvfile:
|
||||
# writer = csv.DictWriter(csvfile, fieldnames=fieldnames)
|
||||
# writer.writeheader()
|
||||
# for row in reader:
|
||||
# # name,server,version,luaversion,maintainers
|
||||
# plugin = LuaPlugin(**row)
|
||||
# luaPackages.append(plugin)
|
||||
pass
|
||||
|
||||
|
||||
def generate_pkg_nix(plug: LuaPlugin):
|
||||
"""
|
||||
Generate nix expression for a luarocks package
|
||||
Our cache key associates "p.name-p.version" to its rockspec
|
||||
"""
|
||||
log.debug("Generating nix expression for %s", plug.name)
|
||||
|
||||
cmd = ["luarocks", "nix"]
|
||||
|
||||
if plug.maintainers:
|
||||
cmd.append(f"--maintainers={plug.maintainers}")
|
||||
|
||||
# if plug.server == "src":
|
||||
if plug.src != "":
|
||||
if plug.src is None:
|
||||
msg = (
|
||||
"src must be set when 'version' is set to \"src\" for package %s"
|
||||
% plug.name
|
||||
)
|
||||
log.error(msg)
|
||||
raise RuntimeError(msg)
|
||||
log.debug("Updating from source %s", plug.src)
|
||||
cmd.append(plug.src)
|
||||
# update the plugin from luarocks
|
||||
else:
|
||||
cmd.append(plug.name)
|
||||
if plug.version and plug.version != "src":
|
||||
cmd.append(plug.version)
|
||||
|
||||
if plug.server != "src" and plug.server:
|
||||
cmd.append(f"--only-server={plug.server}")
|
||||
|
||||
if plug.luaversion:
|
||||
cmd.append(f"--lua-version={plug.luaversion}")
|
||||
|
||||
log.debug("running %s", " ".join(cmd))
|
||||
|
||||
output = subprocess.check_output(cmd, text=True)
|
||||
output = "callPackage(" + output.strip() + ") {};\n\n"
|
||||
return (plug, output)
|
||||
|
||||
|
||||
def main():
|
||||
editor = LuaEditor(
|
||||
"lua",
|
||||
ROOT,
|
||||
"",
|
||||
default_in=PKG_LIST,
|
||||
default_out=GENERATED_NIXFILE,
|
||||
)
|
||||
|
||||
editor.run()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user