diff --git a/pkgs/development/python-modules/nixpkgs-plugin-update/nixpkgs-plugin-update/src/nixpkgs_plugin_update/__init__.py b/pkgs/development/python-modules/nixpkgs-plugin-update/nixpkgs-plugin-update/src/nixpkgs_plugin_update/__init__.py index 4e798498c132..1960572334bb 100644 --- a/pkgs/development/python-modules/nixpkgs-plugin-update/nixpkgs-plugin-update/src/nixpkgs_plugin_update/__init__.py +++ b/pkgs/development/python-modules/nixpkgs-plugin-update/nixpkgs-plugin-update/src/nixpkgs_plugin_update/__init__.py @@ -35,6 +35,7 @@ ATOM_LINK = "{http://www.w3.org/2005/Atom}link" # " ATOM_UPDATED = "{http://www.w3.org/2005/Atom}updated" # " GIT_TAGS_PREFIX = "refs/tags/" +AUTO_BRANCH = "" VERSION_DATE_PATTERN = re.compile(r"(\d{4}-\d{2}-\d{2})$") VERSION_TAG_PATTERN = re.compile(r"^(.+?)-unstable-") @@ -97,6 +98,33 @@ def make_request(url: str, token=None) -> urllib.request.Request: Redirects = dict["PluginDesc", "Repo"] +def select_latest_tag( + tags: list[str], + normalize: Callable[[str], str | None] | None = None, +) -> str | None: + best_version: tuple[str, Any] | None = None + best_text: tuple[str, str] | None = None + + for tag in tags: + candidate = tag if normalize is None else normalize(tag) + if candidate is None: + continue + + try: + version = parse_version(candidate) + if best_version is None or version > best_version[1]: + best_version = (tag, version) + except InvalidVersion: + if best_text is None or candidate > best_text[1]: + best_text = (tag, candidate) + + if best_version is not None: + return best_version[0] + if best_text is not None: + return best_text[0] + return None + + class Repo: def __init__(self, uri: str, branch: str) -> None: self.uri = uri @@ -127,11 +155,17 @@ class Repo: @retry(urllib.error.URLError, tries=4, delay=3, backoff=2) def latest_commit(self) -> tuple[str, datetime]: log.debug("Latest commit") - loaded = self._prefetch(None) + loaded = self._prefetch(None, fetch_submodules=False) updated = datetime.strptime(loaded["date"], "%Y-%m-%dT%H:%M:%S%z") return loaded["rev"], updated + @retry(urllib.error.URLError, tries=4, delay=3, backoff=2) + def resolve_ref(self, ref: str) -> tuple[str, datetime]: + loaded = self._prefetch(ref, fetch_submodules=False) + updated = datetime.strptime(loaded["date"], "%Y-%m-%dT%H:%M:%S%z") + return loaded["rev"], updated + @retry(urllib.error.URLError, tries=4, delay=3, backoff=2) def get_latest_tag(self) -> str | None: try: @@ -157,21 +191,10 @@ class Repo: if not tags: return None - valid_versions = [] - invalid_tags = [] - - for tag in tags: - try: - version = parse_version(tag) - valid_versions.append((tag, version)) - except InvalidVersion: - invalid_tags.append(tag) - - if valid_versions: - latest_tag = max(valid_versions, key=lambda x: x[1])[0] - elif invalid_tags: - latest_tag = max(invalid_tags) - else: + latest_tag = select_latest_tag(tags, normalize_release_version) + if latest_tag is None: + latest_tag = select_latest_tag(tags) + if latest_tag is None: log.debug("No tags found for %s", self.uri) return None @@ -184,8 +207,11 @@ class Repo: log.warning("Unexpected error fetching tags for %s: %s", self.uri, e) return None - def _prefetch(self, ref: str | None): - cmd = ["nix-prefetch-git", "--quiet", "--fetch-submodules", self.uri] + def _prefetch(self, ref: str | None, fetch_submodules: bool = True): + cmd = ["nix-prefetch-git", "--quiet"] + if fetch_submodules: + cmd.append("--fetch-submodules") + cmd.append(self.uri) if ref is not None: cmd.append(ref) log.debug(cmd) @@ -266,6 +292,20 @@ class RepoGitHub(Repo): updated = datetime.strptime(updated_tag.text, "%Y-%m-%dT%H:%M:%SZ") return Path(str(url.path)).name, updated + @retry(urllib.error.URLError, tries=4, delay=3, backoff=2) + def resolve_ref(self, ref: str) -> tuple[str, datetime]: + if ref == self.branch: + return self.latest_commit() + self._check_ref_redirect(ref) + return super().resolve_ref(ref) + + @retry(urllib.error.URLError, tries=4, delay=3, backoff=2) + def _check_ref_redirect(self, ref: str) -> None: + ref_url = self.url(f"tree/{urllib.parse.quote(ref, safe='')}") + ref_req = make_request(ref_url, self.token) + with urllib.request.urlopen(ref_req, timeout=10) as req: + self._check_for_redirect(ref_url, req) + def _execute_graphql(self, query: str, variables: dict) -> dict: graphql_url = "https://api.github.com/graphql" @@ -412,7 +452,7 @@ class RepoGitHub(Repo): urlsplit(response_url).path.strip("/").split("/")[:2] ) - new_repo = RepoGitHub(owner=new_owner, repo=new_name, branch=self.branch) + new_repo = RepoGitHub(owner=new_owner, repo=new_name, branch=self._branch) self.redirect = new_repo def prefetch(self, commit: str) -> str: @@ -472,7 +512,7 @@ class PluginDesc: @staticmethod def load_from_string(config: FetchConfig, line: str) -> "PluginDesc": - branch = "HEAD" + branch = AUTO_BRANCH alias = None uri = line if " as " in uri: @@ -562,6 +602,50 @@ def make_unstable_version(date: datetime, last_tag: str | None) -> str: return f"{tag_part}-unstable-{date_str}" +def get_commit_target( + repo: Repo, + branch: str, + latest_tag: str | None, +) -> tuple[str, datetime, str, str | None]: + if branch == "HEAD": + commit, date = repo.latest_commit() + else: + commit, date = repo.resolve_ref(branch) + + return commit, date, make_unstable_version(date, latest_tag), None + + +def select_plugin_target( + plugin_desc: PluginDesc, + current_plugin: Plugin | None, + latest_tag: str | None, +) -> tuple[str, datetime, str, str | None]: + branch = plugin_desc.branch + + if branch != AUTO_BRANCH: + return get_commit_target(plugin_desc.repo, branch, latest_tag) + + release_version = ( + normalize_release_version(latest_tag) if latest_tag is not None else None + ) + if release_version is None: + return get_commit_target(plugin_desc.repo, "HEAD", latest_tag) + + release_commit, release_date = plugin_desc.repo.resolve_ref( + f"{GIT_TAGS_PREFIX}{latest_tag}" + ) + + if current_plugin is not None: + if ( + current_plugin.date is not None + and current_plugin.tag is None + and current_plugin.date.date() > release_date.date() + ): + return get_commit_target(plugin_desc.repo, "HEAD", latest_tag) + + return release_commit, release_date, release_version, latest_tag + + def load_plugins_from_csv( config: FetchConfig, input_file: Path, @@ -768,7 +852,14 @@ class Editor: cache: Cache = Cache( [plugin for _description, plugin in current_plugins], self.cache_file ) - _prefetch = functools.partial(prefetch, cache=cache) + current_plugin_map = { + plugin.normalized_name: plugin for _description, plugin in current_plugins + } + _prefetch = functools.partial( + prefetch, + cache=cache, + current_plugins=current_plugin_map, + ) to_update_for_filter = [x.replace(".", "-") for x in to_update] plugins_to_update = ( @@ -812,7 +903,6 @@ class Editor: # Track version changes for commit message generation updated_plugins = [] - current_plugin_map = {p.normalized_name: p for _, p in current_plugins} for _, new_plugin in plugins: old_plugin = current_plugin_map.get(new_plugin.normalized_name) if old_plugin and old_plugin.version != new_plugin.version: @@ -1010,9 +1100,9 @@ class CleanEnvironment(object): def prefetch_plugin( p: PluginDesc, cache: "Cache | None" = None, + current_plugin: Plugin | None = None, ) -> tuple[Plugin, Repo | None]: log.info(f"Fetching source for plugin {p.name} from {p.repo.uri}@{p.branch}") - commit, date = p.repo.latest_commit() latest_tag = p.repo.get_latest_tag() if latest_tag: @@ -1020,8 +1110,11 @@ def prefetch_plugin( else: log.debug("No tags found for %s, will use '0' prefix", p.name) - version = make_unstable_version(date, latest_tag) - source_tag = None + commit, date, version, source_tag = select_plugin_target( + p, + current_plugin, + latest_tag, + ) cached_plugin = cache[commit] if cache else None if cached_plugin is not None: @@ -1167,9 +1260,13 @@ class Cache: def prefetch( pluginDesc: PluginDesc, cache: Cache, + current_plugins: dict[str, Plugin] | None = None, ) -> tuple[PluginDesc, Exception | Plugin, Repo | None]: try: - plugin, redirect = prefetch_plugin(pluginDesc, cache) + current_plugin = None + if current_plugins is not None: + current_plugin = current_plugins.get(pluginDesc.name.replace(".", "-")) + plugin, redirect = prefetch_plugin(pluginDesc, cache, current_plugin) cache[plugin.commit] = plugin return (pluginDesc, plugin, redirect) except Exception as e: