nixpkgs/pkgs/development/python-modules/maubot/plugins/update.py

#!/usr/bin/env nix-shell
#!nix-shell -i python3 -p git nurl "(python3.withPackages (ps: with ps; [ toml gitpython requests ruamel-yaml ]))"

import git
import json
import os
import subprocess
import ruamel.yaml
import sys
import toml
import zipfile

HOSTNAMES = {
    "git.skeg1.se": "gitlab",
    "edugit.org": "gitlab",
    "codeberg.org": "gitea",
}
PLUGINS: dict[str, dict] = {}
# https://github.com/maubot/plugins.maubot.xyz/pull/45
SKIP = {"characterai"}
DIRS = {"matrix-to-discourse": "plugin"}

yaml = ruamel.yaml.YAML(typ="safe")

TMP = os.environ.get("TEMPDIR", "/tmp")


def process_repo(path: str, official: bool):
    global PLUGINS
    with open(path, "rt") as f:
        data = yaml.load(f)
    name, repourl, license, desc = (
        data["name"],
        data["repo"],
        data["license"],
        data["description"],
    )
    if name in SKIP:
        return
    origurl = repourl
    if "/" in name or " " in name:
        name = os.path.split(path)[-1].removesuffix(".yaml")
    name = name.replace("_", "-").lower()
    if name in PLUGINS.keys():
        raise ValueError(f"Duplicate plugin {name}, refusing to continue")
    repodir = os.path.join(TMP, "maubot-plugins", name)
    plugindir = repodir
    if "/tree/" in repourl:
        repourl, rev_path = repourl.split("/tree/")
        rev, subdir = rev_path.strip("/").split("/")
        plugindir = os.path.join(plugindir, subdir)
    elif name in DIRS.keys():
        subdir = DIRS[name]
        plugindir = os.path.join(plugindir, subdir)
    else:
        rev = None
        subdir = None

    if repourl.startswith("http:"):
        repourl = "https" + repourl[4:]
    repourl = repourl.rstrip("/")
    if not os.path.exists(repodir):
        print("Fetching", name)
        repo = git.Repo.clone_from(repourl + ".git", repodir)
    else:
        repo = git.Repo(repodir)
    tags = sorted(repo.tags, key=lambda t: t.commit.committed_datetime)
    tags = list(filter(lambda x: "rc" not in str(x), tags))
    if tags:
        repo.git.checkout(tags[-1])
        rev = str(tags[-1])
    else:
        rev = str(repo.commit("HEAD"))
    ret: dict = {"attrs": {}}
    if subdir:
        ret["attrs"]["postPatch"] = f"cd {subdir}"
    domain, query = repourl.removeprefix("https://").split("/", 1)
    hash = subprocess.run(
        ["nurl", "--hash", f"file://{repodir}", rev], capture_output=True, check=True
    ).stdout.decode("utf-8")
    ret["attrs"]["meta"] = {
        "description": desc,
        "homepage": origurl,
    }
    if domain == "github.com":
        owner, repo = query.split("/")
        ret["github"] = {
            "owner": owner,
            "repo": repo,
            "rev": rev,
            "hash": hash,
        }
        ret["attrs"]["meta"]["downloadPage"] = f"{repourl}/releases"
        ret["attrs"]["meta"]["changelog"] = f"{repourl}/releases"
        repobase = f"{repourl}/blob/{rev}"
    elif (
        HOSTNAMES.get(
            domain, "gitea" if "gitea." in domain or "forgejo." in domain else None
        )
        == "gitea"
    ):
        owner, repo = query.split("/")
        ret["gitea"] = {
            "domain": domain,
            "owner": owner,
            "repo": repo,
            "rev": rev,
            "hash": hash,
        }
        repobase = f"{repourl}/src/commit/{rev}"
        ret["attrs"]["meta"]["downloadPage"] = f"{repourl}/releases"
        ret["attrs"]["meta"]["changelog"] = f"{repourl}/releases"
    elif HOSTNAMES.get(domain, "gitlab" if "gitlab." in domain else None) == "gitlab":
        owner, repo = query.split("/")
        ret["gitlab"] = {
            "owner": owner,
            "repo": repo,
            "rev": rev,
            "hash": hash,
        }
        if domain != "gitlab.com":
            ret["gitlab"]["domain"] = domain
        repobase = f"{repourl}/-/blob/{rev}"
    else:
        raise ValueError(
            f"Is {domain} Gitea or Gitlab, or something else? Please specify in the Python script!"
        )
    if os.path.exists(os.path.join(plugindir, "CHANGELOG.md")):
        ret["attrs"]["meta"]["changelog"] = f"{repobase}/CHANGELOG.md"
    if os.path.exists(os.path.join(plugindir, "maubot.yaml")):
        with open(os.path.join(plugindir, "maubot.yaml"), "rt") as f:
            ret["manifest"] = yaml.load(f)
    elif os.path.exists(os.path.join(plugindir, "pyproject.toml")):
        ret["isPoetry"] = True
        with open(os.path.join(plugindir, "pyproject.toml"), "rt") as f:
            data = toml.load(f)
        deps = []
        for key, val in data["tool"]["poetry"].get("dependencies", {}).items():
            if key in ["maubot", "mautrix", "python"]:
                continue
            reqs = []
            for req in val.split(","):
                reqs.extend(poetry_to_pep(req))
            deps.append(key + ", ".join(reqs))
        ret["manifest"] = data["tool"]["maubot"]
        ret["manifest"]["id"] = data["tool"]["poetry"]["name"]
        ret["manifest"]["version"] = data["tool"]["poetry"]["version"]
        ret["manifest"]["license"] = data["tool"]["poetry"]["license"]
        if deps:
            ret["manifest"]["dependencies"] = deps
    else:
        raise ValueError(f"No maubot.yaml or pyproject.toml found in {repodir}")
    # normalize non-spdx-conformant licenses this way
    # (and fill out missing license info)
    if "license" not in ret["manifest"] or ret["manifest"]["license"] in [
        "GPLv3",
        "AGPL 3.0",
    ]:
        ret["attrs"]["meta"]["license"] = license
    elif ret["manifest"]["license"] != license:
        print(
            f"Warning: licenses for {repourl} don't match! {ret['manifest']['license']} != {license}"
        )
    if official:
        ret["isOfficial"] = official
    PLUGINS[name] = ret


def next_incomp(ver_s: str) -> str:
    ver = ver_s.split(".")
    zero = False
    for i in range(len(ver)):
        try:
            seg = int(ver[i])
        except ValueError:
            if zero:
                ver = ver[:i]
                break
            continue
        if zero:
            ver[i] = "0"
        elif seg:
            ver[i] = str(seg + 1)
            zero = True
    return ".".join(ver)


def poetry_to_pep(ver_req: str) -> list[str]:
    if "*" in ver_req:
        raise NotImplementedError("Wildcard poetry versions not implemented!")
    if ver_req.startswith("^"):
        return [">=" + ver_req[1:], "<" + next_incomp(ver_req[1:])]
    if ver_req.startswith("~"):
        return ["~=" + ver_req[1:]]
    return [ver_req]


def main():
    cache_path = os.path.join(TMP, "maubot-plugins")
    if not os.path.exists(cache_path):
        os.makedirs(cache_path)
        git.Repo.clone_from(
            "https://github.com/maubot/plugins.maubot.xyz",
            os.path.join(cache_path, "_repo"),
        )
    else:
        pass

    repodir = os.path.join(cache_path, "_repo")

    for suffix, official in (("official", True), ("thirdparty", False)):
        directory = os.path.join(repodir, "data", "plugins", suffix)
        for plugin_name in os.listdir(directory):
            process_repo(os.path.join(directory, plugin_name), official)

    if os.path.isdir("pkgs/development/python-modules/maubot/plugins"):
        generated = "pkgs/development/python-modules/maubot/plugins/generated.json"
    else:
        script_dir = os.path.dirname(os.path.realpath(__file__))
        generated = os.path.join(script_dir, "generated.json")

    with open(generated, "wt") as file:
        json.dump(PLUGINS, file, indent="  ", separators=(",", ": "), sort_keys=True)
        file.write("\n")


if __name__ == "__main__":
    main()