vimPluginsUpdater: generate plugin licenses (#513747)

doc/languages-frameworks/neovim.section.md

@@ -115,6 +115,25 @@ patch those plugins but expose the necessary configuration under
 `PLUGIN.passthru.initLua` for neovim plugins. For instance, the `unicode-vim` plugin
 needs the path towards a unicode database so we expose the following snippet `vim.g.Unicode_data_directory="${self.unicode-vim}/autoload/unicode"` under `vimPlugins.unicode-vim.passthru.initLua`.
 
+### Plugin license overrides {#neovim-plugin-license-overrides}
+
+Generated Vim and Neovim plugins get their `meta.license` from GitHub license metadata when possible.
+Some upstream repositories do not expose a license file that GitHub can detect, or only mention the license in a README.
+In those cases, add a manual `meta.license` override in [overrides.nix](https://github.com/NixOS/nixpkgs/blob/master/pkgs/applications/editors/vim/plugins/overrides.nix).
+
+For example, if upstream documents that a plugin uses the Vim license but GitHub does not detect it:
+
+```nix
+{
+  foo-nvim = super.foo-nvim.overrideAttrs (old: {
+    meta = old.meta // {
+      # README says this plugin is distributed under the Vim license.
+      license = lib.licenses.vim;
+    };
+  });
+}
+```
+
 ## LuaRocks based plugins {#neovim-luarocks-based-plugins}
 
 In order to automatically handle plugin dependencies, several Neovim plugins

doc/redirects.json

@@ -4483,6 +4483,9 @@
     "index.html#neovim-plugin-required-snippet",
     "index.html#vim-plugin-required-snippet"
   ],
+  "neovim-plugin-license-overrides": [
+    "index.html#neovim-plugin-license-overrides"
+  ],
   "updating-plugins-in-nixpkgs": [
     "index.html#updating-plugins-in-nixpkgs"
   ],

pkgs/applications/editors/vim/plugins/utils/get-plugins.nix

@@ -17,6 +17,7 @@ let
   parse = _name: value: {
     inherit (value) pname version;
     homePage = value.meta.homepage;
+    license = value.meta.license.spdxId or null;
     checksum =
       if hasChecksum value then
         {

pkgs/applications/editors/vim/plugins/utils/update.py

@@ -19,6 +19,7 @@
 #
 
 import inspect
+import json
 import logging
 import os
 import textwrap
@@ -36,18 +37,27 @@ from nixpkgs_plugin_update import PluginDesc, run_nix_expr
 treesitter = importlib.import_module("nvim-treesitter.update")
 
 
-HEADER = (
-    "# GENERATED by ./pkgs/applications/editors/vim/plugins/utils/update.py. Do not edit!"
-)
+HEADER = "# GENERATED by ./pkgs/applications/editors/vim/plugins/utils/update.py. Do not edit!"
 
 NIXPKGS_NVIMTREESITTER_FOLDER = "pkgs/applications/editors/vim/plugins/nvim-treesitter"
 
+SPDX_LICENSE_NORMALIZATION = {
+    "AGPL-3.0": "AGPL-3.0-only",
+    "GPL-2.0": "GPL-2.0-only",
+    "GPL-3.0": "GPL-3.0-only",
+    "LGPL-2.0": "LGPL-2.0-only",
+    "LGPL-2.1": "LGPL-2.1-only",
+    "LGPL-3.0": "LGPL-3.0-only",
+}
+
 
 class VimEditor(nixpkgs_plugin_update.Editor):
     nvim_treesitter_updated = False
 
     def generate_nix(
-        self, plugins: List[Tuple[PluginDesc, nixpkgs_plugin_update.Plugin]], outfile: str
+        self,
+        plugins: List[Tuple[PluginDesc, nixpkgs_plugin_update.Plugin]],
+        outfile: str,
     ):
         log.info("Generating nix code")
         log.debug("Loading nvim-treesitter source reference from nix...")
@@ -93,7 +103,10 @@ class VimEditor(nixpkgs_plugin_update.Editor):
             for pdesc, plugin in plugins:
                 content = self.plugin2nix(pdesc, plugin, _isNeovimPlugin(plugin))
                 f.write(content)
-                if plugin.name == "nvim-treesitter" and (plugin.tag or plugin.commit) != nvim_treesitter_ref:
+                if (
+                    plugin.name == "nvim-treesitter"
+                    and (plugin.tag or plugin.commit) != nvim_treesitter_ref
+                ):
                     self.nvim_treesitter_updated = True
             f.write("}\n")
         print(f"updated {outfile}")
@@ -102,16 +115,29 @@ class VimEditor(nixpkgs_plugin_update.Editor):
         self, pdesc: PluginDesc, plugin: nixpkgs_plugin_update.Plugin, isNeovim: bool
     ) -> str:
         if isNeovim:
-            raise RuntimeError(f"Plugin {plugin.name} is already packaged in `luaPackages`, please use that")
+            raise RuntimeError(
+                f"Plugin {plugin.name} is already packaged in `luaPackages`, please use that"
+            )
         repo = pdesc.repo
 
         content = f"  {plugin.normalized_name} = "
         src_nix = repo.as_nix(plugin)
+        license_spdx_id = plugin.license
+        if license_spdx_id is not None:
+            license_spdx_id = SPDX_LICENSE_NORMALIZATION.get(
+                license_spdx_id, license_spdx_id
+            )
+        license_nix = (
+            f"    meta.license = lib.meta.getLicenseFromSpdxId {json.dumps(license_spdx_id)};\n"
+            if license_spdx_id
+            else "    meta.license = lib.licenses.unfree;\n"
+        )
         content += """{buildFn} {{
     pname = "{plugin.name}";
     version = "{plugin.version}";
     src = {src_nix};
     meta.homepage = "{repo.uri}";
+{license_nix}\
     meta.hydraPlatforms = [ ];
   }};
 
@@ -120,6 +146,7 @@ class VimEditor(nixpkgs_plugin_update.Editor):
             plugin=plugin,
             src_nix=src_nix,
             repo=repo,
+            license_nix=license_nix,
         )
         log.debug(content)
         return content

pkgs/development/python-modules/nixpkgs-plugin-update/nixpkgs-plugin-update/src/nixpkgs_plugin_update/__init__.py

@@ -89,7 +89,7 @@ class FetchConfig:
 
 def make_request(url: str, token=None) -> urllib.request.Request:
     headers = {}
-    if token is not None:
+    if token:
         headers["Authorization"] = f"token {token}"
     return urllib.request.Request(url, headers=headers)
 
@@ -140,7 +140,7 @@ class Repo:
         self._branch = branch
         # Redirect is the new Repo to use
         self.redirect: "Repo | None" = None
-        self.token: str | None = "dummy_token"
+        self.token: str | None = None
 
     @property
     def name(self):
@@ -234,7 +234,9 @@ class Repo:
 
     def as_nix(self, plugin: "Plugin") -> str:
         ref_attr = (
-            f'tag = "{plugin.tag}";' if plugin.tag is not None else f'rev = "{plugin.commit}";'
+            f'tag = "{plugin.tag}";'
+            if plugin.tag is not None
+            else f'rev = "{plugin.commit}";'
         )
         return f"""fetchgit {{
       url = "{self.uri}";
@@ -242,6 +244,9 @@ class Repo:
       hash = "{plugin.to_sri_hash()}";
     }}"""
 
+    def get_license_spdx_id(self, fallback_license: str | None = None) -> str | None:
+        return fallback_license
+
 
 class RepoGitHub(Repo):
     def __init__(self, owner: str, repo: str, branch: str) -> None:
@@ -439,9 +444,7 @@ class RepoGitHub(Repo):
     def _check_for_redirect(self, url: str, req: http.client.HTTPResponse):
         response_url = req.geturl()
         if url != response_url:
-            new_owner, new_name = (
-                urlsplit(response_url).path.strip("/").split("/")[:2]
-            )
+            new_owner, new_name = urlsplit(response_url).path.strip("/").split("/")[:2]
 
             new_repo = RepoGitHub(owner=new_owner, repo=new_name, branch=self._branch)
             self.redirect = new_repo
@@ -460,6 +463,41 @@ class RepoGitHub(Repo):
         loaded = json.loads(data)
         return loaded["hash"]
 
+    def get_license_spdx_id(self, fallback_license: str | None = None) -> str | None:
+        license_url = f"https://api.github.com/repos/{self.owner}/{self.repo}/license"
+        log.debug("Fetching license metadata from %s", license_url)
+
+        def log_fetch_failure(reason: str) -> str | None:
+            log.warning(
+                "Failed to fetch license metadata for %s/%s: %s; reusing %s",
+                self.owner,
+                self.repo,
+                reason,
+                fallback_license or "no cached license",
+            )
+            return fallback_license
+
+        try:
+            req = make_request(license_url, self.token)
+            with urllib.request.urlopen(req, timeout=10) as response:
+                data = json.load(response)
+        except urllib.error.HTTPError as e:
+            if e.code == 404:
+                return None
+            return log_fetch_failure(f"HTTP {e.code}")
+        except urllib.error.URLError as e:
+            return log_fetch_failure(str(e))
+
+        license_info = data.get("license")
+        if not isinstance(license_info, dict):
+            return None
+
+        spdx_id = license_info.get("spdx_id")
+        if not spdx_id or spdx_id == "NOASSERTION":
+            return None
+
+        return spdx_id
+
     def as_nix(self, plugin: "Plugin") -> str:
         if plugin.has_submodules:
             submodule_attr = "\n      fetchSubmodules = true;"
@@ -467,7 +505,9 @@ class RepoGitHub(Repo):
             submodule_attr = ""
 
         ref_attr = (
-            f'tag = "{plugin.tag}";' if plugin.tag is not None else f'rev = "{plugin.commit}";'
+            f'tag = "{plugin.tag}";'
+            if plugin.tag is not None
+            else f'rev = "{plugin.commit}";'
         )
 
         return f"""fetchFromGitHub {{
@@ -526,6 +566,7 @@ class Plugin:
     date: datetime | None = None
     last_tag: str | None = None
     tag: str | None = None
+    license: str | None = None
 
     @property
     def normalized_name(self) -> str:
@@ -781,6 +822,7 @@ class Editor:
                 date,
                 last_tag=last_tag,
                 tag=source_tag,
+                license=attr.get("license"),
             )
 
             plugins.append((pdesc, p))
@@ -1105,9 +1147,16 @@ def prefetch_plugin(
         latest_tag,
     )
 
-    cached_plugin = cache[target_cache_key(p.repo.uri, commit, source_tag)] if cache else None
+    cached_plugin = (
+        cache[target_cache_key(p.repo.uri, commit, source_tag)] if cache else None
+    )
     if cached_plugin is not None:
         log.debug(f"Cache hit for {p.name}!")
+        license_spdx_id = (
+            cached_plugin.license
+            or (current_plugin.license if current_plugin else None)
+            or p.repo.get_license_spdx_id()
+        )
         return (
             replace(
                 cached_plugin,
@@ -1117,6 +1166,7 @@ def prefetch_plugin(
                 date=date,
                 last_tag=latest_tag,
                 tag=source_tag,
+                license=license_spdx_id,
             ),
             p.repo.redirect,
         )
@@ -1124,7 +1174,14 @@ def prefetch_plugin(
     has_submodules = p.repo.has_submodules()
     log.debug(f"prefetch {p.name}")
     sha256 = (
-        p.repo.prefetch(f"{GIT_TAGS_PREFIX}{source_tag}") if source_tag else p.repo.prefetch(commit)
+        p.repo.prefetch(f"{GIT_TAGS_PREFIX}{source_tag}")
+        if source_tag
+        else p.repo.prefetch(commit)
+    )
+    license_spdx_id = (
+        current_plugin.license
+        if current_plugin and current_plugin.license
+        else p.repo.get_license_spdx_id()
     )
 
     return (
@@ -1137,6 +1194,7 @@ def prefetch_plugin(
             date=date,
             last_tag=latest_tag,
             tag=source_tag,
+            license=license_spdx_id,
         ),
         p.repo.redirect,
     )
@@ -1243,6 +1301,7 @@ class Cache:
                     attr.get("version", ""),
                     last_tag=attr.get("last_tag"),
                     tag=attr.get("tag"),
+                    license=attr.get("license"),
                 )
                 downloads[cache_key] = p
         return downloads