diff --git a/pkgs/development/python-modules/nixpkgs-plugin-update/default.nix b/pkgs/development/python-modules/nixpkgs-plugin-update/default.nix index 5e3a30b167bd..fbef058c9d01 100644 --- a/pkgs/development/python-modules/nixpkgs-plugin-update/default.nix +++ b/pkgs/development/python-modules/nixpkgs-plugin-update/default.nix @@ -3,6 +3,7 @@ buildPythonPackage, uv-build, gitpython, + packaging, ruff, mypy, }: @@ -18,6 +19,7 @@ buildPythonPackage { dependencies = [ gitpython + packaging ]; nativeCheckInputs = [ diff --git a/pkgs/development/python-modules/nixpkgs-plugin-update/nixpkgs-plugin-update/src/nixpkgs_plugin_update/__init__.py b/pkgs/development/python-modules/nixpkgs-plugin-update/nixpkgs-plugin-update/src/nixpkgs_plugin_update/__init__.py index 299d5a056a99..fb5870a1424a 100644 --- a/pkgs/development/python-modules/nixpkgs-plugin-update/nixpkgs-plugin-update/src/nixpkgs_plugin_update/__init__.py +++ b/pkgs/development/python-modules/nixpkgs-plugin-update/nixpkgs-plugin-update/src/nixpkgs_plugin_update/__init__.py @@ -29,11 +29,17 @@ from typing import Any, Callable from urllib.parse import urljoin, urlparse import git +from packaging.version import InvalidVersion, parse as parse_version ATOM_ENTRY = "{http://www.w3.org/2005/Atom}entry" # " vim gets confused here ATOM_LINK = "{http://www.w3.org/2005/Atom}link" # " ATOM_UPDATED = "{http://www.w3.org/2005/Atom}updated" # " +GIT_TAGS_PREFIX = "refs/tags/" + +VERSION_DATE_PATTERN = re.compile(r"(\d{4}-\d{2}-\d{2})$") +VERSION_TAG_PATTERN = re.compile(r"^(.+?)-unstable-") + LOG_LEVELS = { logging.getLevelName(level): level for level in [logging.DEBUG, logging.INFO, logging.WARN, logging.ERROR] @@ -125,6 +131,58 @@ class Repo: return loaded["rev"], updated + @retry(urllib.error.URLError, tries=4, delay=3, backoff=2) + def get_latest_tag(self) -> str | None: + try: + # FIXME: This fetches all tags. We need to find a way to check if a tag exists in + # an ancestor of the default branch. + cmd = ["git", "ls-remote", "--tags", "--refs", self.uri] + log.debug("Fetching tags with: %s", cmd) + output = subprocess.check_output(cmd, stderr=subprocess.STDOUT, timeout=10) + lines = output.decode("utf-8").strip().split("\n") + + if not lines or lines[0] == "": + log.debug("No tags found for %s", self.uri) + return None + + tags = [] + for line in lines: + if "\t" in line: + tag_ref = line.split("\t")[1] + if tag_ref.startswith(GIT_TAGS_PREFIX): + tag_name = tag_ref[len(GIT_TAGS_PREFIX) :] + tags.append(tag_name) + + if not tags: + return None + + valid_versions = [] + invalid_tags = [] + + for tag in tags: + try: + version = parse_version(tag) + valid_versions.append((tag, version)) + except InvalidVersion: + invalid_tags.append(tag) + + if valid_versions: + latest_tag = max(valid_versions, key=lambda x: x[1])[0] + elif invalid_tags: + latest_tag = max(invalid_tags) + else: + log.debug("No tags found for %s", self.uri) + return None + + log.debug("Found latest tag: %s", latest_tag) + return latest_tag + except subprocess.CalledProcessError as e: + log.debug("Failed to fetch tags for %s: %s", self.uri, e) + return None + except Exception as e: + log.warning("Unexpected error fetching tags for %s: %s", self.uri, e) + return None + def _prefetch(self, ref: str | None): cmd = ["nix-prefetch-git", "--quiet", "--fetch-submodules", self.uri] if ref is not None: @@ -204,6 +262,139 @@ class RepoGitHub(Repo): updated = datetime.strptime(updated_tag.text, "%Y-%m-%dT%H:%M:%SZ") return Path(str(url.path)).name, updated + def _execute_graphql(self, query: str, variables: dict) -> dict: + graphql_url = "https://api.github.com/graphql" + + payload = json.dumps({"query": query, "variables": variables}).encode("utf-8") + + req = make_request(graphql_url, self.token) + req.add_header("Content-Type", "application/json") + req.data = payload + + with urllib.request.urlopen(req, timeout=10) as response: + return json.load(response) + + def _extract_commit_date(self, target: dict) -> datetime | None: + commit_date_str = None + if "committedDate" in target: + commit_date_str = target["committedDate"] + elif "target" in target and "committedDate" in target["target"]: + commit_date_str = target["target"]["committedDate"] + + if commit_date_str: + return datetime.fromisoformat(commit_date_str.replace("Z", "+00:00")) + return None + + @retry(urllib.error.URLError, tries=4, delay=3, backoff=2) + def get_latest_tag(self) -> str | None: + try: + # FIXME: This fetches all tags. We need to find a way to check if a tag exists in + # an ancestor of the default branch. + query = """ + query GetLatestVersionInfo($owner: String!, $name: String!) { + repository(owner: $owner, name: $name) { + refs(refPrefix: "refs/tags/", first: 5, orderBy: {field: TAG_COMMIT_DATE, direction: DESC}) { + nodes { + name + target { + ... on Commit { + committedDate + } + ... on Tag { + target { + ... on Commit { + committedDate + } + } + } + } + } + } + } + } + """ + + data = self._execute_graphql( + query, {"owner": self.owner, "name": self.repo} + ) + + if "errors" in data: + log.warning( + "GraphQL errors for %s/%s: %s", + self.owner, + self.repo, + data["errors"], + ) + return None + + if "data" not in data or not data["data"]: + log.warning( + "No data in GraphQL response for %s/%s", self.owner, self.repo + ) + return None + + repo = data["data"]["repository"] + if not repo: + log.debug( + "Repository %s/%s not found or inaccessible", self.owner, self.repo + ) + return None + + valid_versions = [] + invalid_tags = [] + for ref_node in repo["refs"]["nodes"]: + tag_name = ref_node["name"] + commit_date = self._extract_commit_date(ref_node["target"]) + if not commit_date: + continue + + try: + version = parse_version(tag_name) + valid_versions.append((tag_name, version, commit_date)) + except InvalidVersion: + invalid_tags.append((tag_name, None, commit_date)) + + def get_version(tag_tuple): + _, version, _ = tag_tuple + return version + + def get_date(tag_tuple): + _, _, date = tag_tuple + return date or datetime.min + + def get_max_versions(versions, sort_key): + return max(versions, key=sort_key, default=(None, None, None)) + + max_valid_tag, _, max_valid_date = get_max_versions( + valid_versions, get_version + ) + max_invalid_tag, _, max_invalid_date = get_max_versions( + invalid_tags, get_date + ) + if max_valid_tag and max_invalid_tag: + return ( + max_invalid_tag + if (max_invalid_date or datetime.min) + > (max_valid_date or datetime.min) + else max_valid_tag + ) + elif max_valid_tag: + return max_valid_tag + elif max_invalid_tag: + return max_invalid_tag + else: + return None + + except Exception as e: + log.warning( + "Error fetching version info for %s/%s: %s", + self.owner, + self.repo, + e, + exc_info=True, + ) + return None + def _check_for_redirect(self, url: str, req: http.client.HTTPResponse): response_url = req.geturl() if url != response_url: @@ -286,6 +477,7 @@ class Plugin: has_submodules: bool sha256: str date: datetime | None = None + last_tag: str | None = None @property def normalized_name(self) -> str: @@ -294,7 +486,33 @@ class Plugin: @property def version(self) -> str: assert self.date is not None - return self.date.strftime("%Y-%m-%d") + date_str = self.date.strftime("%Y-%m-%d") + + tag_part = "0" + if self.last_tag: + tag = ( + self.last_tag[1:] + if self.last_tag.startswith(("v", "V")) + else self.last_tag + ) + if tag and tag[0].isdigit(): + tag_part = tag + + return f"{tag_part}-unstable-{date_str}" + + @staticmethod + def parse_version_string(version_str: str) -> tuple[datetime, str | None]: + date_match = VERSION_DATE_PATTERN.search(version_str) + if not date_match: + raise ValueError(f"Cannot parse date from version: {version_str}") + date = datetime.fromisoformat(date_match.group(1)) + + tag_match = VERSION_TAG_PATTERN.search(version_str) + last_tag = ( + tag_match.group(1) if tag_match and tag_match.group(1) != "0" else None + ) + + return date, last_tag def as_json(self) -> dict[str, str]: copy = self.__dict__.copy() @@ -419,13 +637,9 @@ class Editor: plugins = [] for name, attr in data.items(): checksum = attr["checksum"] + version_str = attr["version"] - # https://github.com/NixOS/nixpkgs/blob/8a335419/pkgs/applications/editors/neovim/build-neovim-plugin.nix#L36 - # https://github.com/NixOS/nixpkgs/pull/344478#discussion_r1786646055 - version = re.search(r"\d\d\d\d-\d\d?-\d\d?", attr["version"]) - if version is None: - raise ValueError(f"Cannot parse version: {attr['version']}") - date = datetime.strptime(version.group(), "%Y-%m-%d") + date, last_tag = Plugin.parse_version_string(version_str) pdesc = PluginDesc.load_from_string(config, f'{attr["homePage"]} as {name}') p = Plugin( @@ -434,6 +648,7 @@ class Editor: checksum["submodules"], checksum["sha256"], date, + last_tag=last_tag, ) plugins.append((pdesc, p)) @@ -722,11 +937,18 @@ def prefetch_plugin( log.info(f"Fetching last commit for plugin {p.name} from {p.repo.uri}@{p.branch}") commit, date = p.repo.latest_commit() + latest_tag = p.repo.get_latest_tag() + if latest_tag: + log.debug("Latest tag for %s: %s", p.name, latest_tag) + else: + log.debug("No tags found for %s, will use '0' prefix", p.name) + cached_plugin = cache[commit] if cache else None if cached_plugin is not None: log.debug(f"Cache hit for {p.name}!") cached_plugin.name = p.name cached_plugin.date = date + cached_plugin.last_tag = latest_tag return cached_plugin, p.repo.redirect has_submodules = p.repo.has_submodules() @@ -734,7 +956,7 @@ def prefetch_plugin( sha256 = p.repo.prefetch(commit) return ( - Plugin(p.name, commit, has_submodules, sha256, date=date), + Plugin(p.name, commit, has_submodules, sha256, date=date, last_tag=latest_tag), p.repo.redirect, ) @@ -818,7 +1040,11 @@ class Cache: data = json.load(f) for attr in data.values(): p = Plugin( - attr["name"], attr["commit"], attr["has_submodules"], attr["sha256"] + attr["name"], + attr["commit"], + attr["has_submodules"], + attr["sha256"], + last_tag=attr.get("last_tag"), ) downloads[attr["commit"]] = p return downloads