From 1bcf7a8b53edf3f6ccd508d166ad16e2d0698f6a Mon Sep 17 00:00:00 2001 From: John Ericson Date: Mon, 1 Apr 2024 10:24:15 -0400 Subject: [PATCH] freebsd: Use's @rhelmot's version infra This is in preparation for multiple version support, and PR #298849. --- pkgs/os-specific/bsd/freebsd/default.nix | 13 +- .../bsd/freebsd/evdev-proto/default.nix | 2 +- .../bsd/freebsd/pkgs/mkDerivation.nix | 4 +- pkgs/os-specific/bsd/freebsd/pkgs/sed.nix | 4 +- pkgs/os-specific/bsd/freebsd/pkgs/source.nix | 11 + pkgs/os-specific/bsd/freebsd/update.py | 182 +++++++++++++++ pkgs/os-specific/bsd/freebsd/versions.json | 210 ++++++++++++++++++ 7 files changed, 412 insertions(+), 14 deletions(-) create mode 100644 pkgs/os-specific/bsd/freebsd/pkgs/source.nix create mode 100755 pkgs/os-specific/bsd/freebsd/update.py create mode 100644 pkgs/os-specific/bsd/freebsd/versions.json diff --git a/pkgs/os-specific/bsd/freebsd/default.nix b/pkgs/os-specific/bsd/freebsd/default.nix index 965185cc14aa..1f321f032516 100644 --- a/pkgs/os-specific/bsd/freebsd/default.nix +++ b/pkgs/os-specific/bsd/freebsd/default.nix @@ -7,15 +7,10 @@ let inherit (buildPackages.buildPackages) rsync; - version = "13.1.0"; + versions = builtins.fromJSON (builtins.readFile ./versions.json); - # `BuildPackages.fetchgit` avoids some probably splicing-caused infinite - # recursion. - freebsdSrc = buildPackages.fetchgit { - url = "https://git.FreeBSD.org/src.git"; - rev = "release/${version}"; - sha256 = "14nhk0kls83xfb64d5xy14vpi6k8laswjycjg80indq9pkcr2rlv"; - }; + version = "13.1.0"; + branch = "release/${version}"; in makeScopeWithSplicing' { otherSplices = generateSplicesForMkScope "freebsd"; @@ -23,7 +18,7 @@ in makeScopeWithSplicing' { callPackage = self.callPackage; directory = ./pkgs; } // { - inherit freebsdSrc; + sourceData = versions.${branch}; ports = fetchzip { url = "https://cgit.freebsd.org/ports/snapshot/ports-dde3b2b456c3a4bdd217d0bf3684231cc3724a0a.tar.gz"; diff --git a/pkgs/os-specific/bsd/freebsd/evdev-proto/default.nix b/pkgs/os-specific/bsd/freebsd/evdev-proto/default.nix index b6dab0d8bdfc..d62c01f5eb88 100644 --- a/pkgs/os-specific/bsd/freebsd/evdev-proto/default.nix +++ b/pkgs/os-specific/bsd/freebsd/evdev-proto/default.nix @@ -33,7 +33,7 @@ stdenv.mkDerivation { INSTALL_AS_USER = true; NO_CHECKSUM = true; NO_MTREE = true; - SRC_BASE = freebsd.freebsdSrc; + SRC_BASE = freebsd.source; preUnpack = '' export MAKE_JOBS_NUMBER="$NIX_BUILD_CORES" diff --git a/pkgs/os-specific/bsd/freebsd/pkgs/mkDerivation.nix b/pkgs/os-specific/bsd/freebsd/pkgs/mkDerivation.nix index 14c9f0405534..79428626b8ec 100644 --- a/pkgs/os-specific/bsd/freebsd/pkgs/mkDerivation.nix +++ b/pkgs/os-specific/bsd/freebsd/pkgs/mkDerivation.nix @@ -2,7 +2,7 @@ , compatIfNeeded , runCommand, rsync , freebsd-lib -, freebsdSrc +, source , bsdSetupHook, freebsdSetupHook , makeMinimal , install, tsort, lorder, mandoc, groff @@ -20,7 +20,7 @@ in stdenv'.mkDerivation (rec { set -x path="$out/$p" mkdir -p "$(dirname "$path")" - src_path="${freebsdSrc}/$p" + src_path="${source}/$p" if [[ -d "$src_path" ]]; then src_path+=/; fi rsync --chmod="+w" -r "$src_path" "$path" set +x diff --git a/pkgs/os-specific/bsd/freebsd/pkgs/sed.nix b/pkgs/os-specific/bsd/freebsd/pkgs/sed.nix index 6532506b3fc2..07b55afe133f 100644 --- a/pkgs/os-specific/bsd/freebsd/pkgs/sed.nix +++ b/pkgs/os-specific/bsd/freebsd/pkgs/sed.nix @@ -1,7 +1,7 @@ -{ mkDerivation, freebsdSrc }: +{ mkDerivation, source }: mkDerivation { path = "usr.bin/sed"; - TESTSRC = "${freebsdSrc}/contrib/netbsd-tests"; + TESTSRC = "${source}/contrib/netbsd-tests"; MK_TESTS = "no"; } diff --git a/pkgs/os-specific/bsd/freebsd/pkgs/source.nix b/pkgs/os-specific/bsd/freebsd/pkgs/source.nix new file mode 100644 index 000000000000..5e31f900e821 --- /dev/null +++ b/pkgs/os-specific/bsd/freebsd/pkgs/source.nix @@ -0,0 +1,11 @@ +{ fetchFromGitHub, sourceData }: + +# Using fetchFromGitHub from their mirror because it's a lot faster than their git server +# If you want you could fetchgit from "https://git.FreeBSD.org/src.git" instead. +# The update script still pulls directly from git.freebsd.org +fetchFromGitHub { + name = "src"; # Want to rename this next rebuild + owner = "freebsd"; + repo = "freebsd-src"; + inherit (sourceData) rev hash; +} diff --git a/pkgs/os-specific/bsd/freebsd/update.py b/pkgs/os-specific/bsd/freebsd/update.py new file mode 100755 index 000000000000..cd20f67148fa --- /dev/null +++ b/pkgs/os-specific/bsd/freebsd/update.py @@ -0,0 +1,182 @@ +#!/usr/bin/env nix-shell +#!nix-shell -i python3 -p git "python3.withPackages (ps: with ps; [ gitpython packaging beautifulsoup4 pandas lxml ])" + +import bs4 +import git +import io +import json +import os +import packaging.version +import pandas +import re +import subprocess +import sys +import tempfile +import typing +import urllib.request + +_QUERY_VERSION_PATTERN = re.compile('^([A-Z]+)="(.+)"$') +BASE_DIR = os.path.dirname(os.path.abspath(__file__)) +MIN_VERSION = packaging.version.Version("13.0.0") +MAIN_BRANCH = "main" +TAG_PATTERN = re.compile( + f"^release/({packaging.version.VERSION_PATTERN})$", re.IGNORECASE | re.VERBOSE +) +REMOTE = "origin" +BRANCH_PATTERN = re.compile( + f"^{REMOTE}/((stable|releng)/({packaging.version.VERSION_PATTERN}))$", + re.IGNORECASE | re.VERBOSE, +) + + +def request_supported_refs() -> list[str]: + # Looks pretty shady but I think this should work with every version of the page in the last 20 years + r = re.compile("^h\d$", re.IGNORECASE) + soup = bs4.BeautifulSoup( + urllib.request.urlopen("https://www.freebsd.org/security"), features="lxml" + ) + header = soup.find( + lambda tag: r.match(tag.name) is not None + and tag.text.lower() == "supported freebsd releases" + ) + table = header.find_next("table") + df = pandas.read_html(io.StringIO(table.prettify()))[0] + return list(df["Branch"]) + + +def query_version(repo: git.Repo) -> dict[str, typing.Any]: + # This only works on FreeBSD 13 and later + text = ( + subprocess.check_output( + ["bash", os.path.join(repo.working_dir, "sys", "conf", "newvers.sh"), "-v"] + ) + .decode("utf-8") + .strip() + ) + fields = dict() + for line in text.splitlines(): + m = _QUERY_VERSION_PATTERN.match(line) + if m is None: + continue + fields[m[1].lower()] = m[2] + + fields["major"] = packaging.version.parse(fields["revision"]).major + return fields + + +def handle_commit( + repo: git.Repo, + rev: git.objects.commit.Commit, + ref_name: str, + ref_type: str, + supported_refs: list[str], + old_versions: dict[str, typing.Any], +) -> dict[str, typing.Any]: + if old_versions.get(ref_name, {}).get("rev", None) == rev.hexsha: + print(f"{ref_name}: revision still {rev.hexsha}, skipping") + return old_versions[ref_name] + + repo.git.checkout(rev) + print(f"{ref_name}: checked out {rev.hexsha}") + + full_hash = ( + subprocess.check_output(["nix", "hash", "path", "--sri", repo.working_dir]) + .decode("utf-8") + .strip() + ) + print(f"{ref_name}: hash is {full_hash}") + + version = query_version(repo) + print(f"{ref_name}: version is {version['version']}") + + return { + "rev": rev.hexsha, + "hash": full_hash, + "ref": ref_name, + "refType": ref_type, + "supported": ref_name in supported_refs, + "version": query_version(repo), + } + + +def main() -> None: + # Normally uses /run/user/*, which is on a tmpfs and too small + temp_dir = tempfile.TemporaryDirectory(dir="/tmp") + print(f"Selected temporary directory {temp_dir.name}") + + if len(sys.argv) >= 2: + orig_repo = git.Repo(sys.argv[1]) + print(f"Fetching updates on {orig_repo.git_dir}") + orig_repo.remote("origin").fetch() + else: + print("Cloning source repo") + orig_repo = git.Repo.clone_from( + "https://git.FreeBSD.org/src.git", to_path=os.path.join(temp_dir.name, "orig") + ) + + supported_refs = request_supported_refs() + print(f"Supported refs are: {' '.join(supported_refs)}") + + print("Doing git crimes, do not run `git worktree prune` until after script finishes!") + workdir = os.path.join(temp_dir.name, "work") + git.cmd.Git(orig_repo.git_dir).worktree("add", "--orphan", workdir) + + # Have to create object before removing .git otherwise it will complain + repo = git.Repo(workdir) + repo.git.set_persistent_git_options(git_dir=repo.git_dir) + # Remove so that nix hash doesn't see the file + os.remove(os.path.join(workdir, ".git")) + + print(f"Working in directory {repo.working_dir} with git directory {repo.git_dir}") + + + try: + with open(os.path.join(BASE_DIR, "versions.json"), "r") as f: + old_versions = json.load(f) + except FileNotFoundError: + old_versions = dict() + + versions = dict() + for tag in repo.tags: + m = TAG_PATTERN.match(tag.name) + if m is None: + continue + version = packaging.version.parse(m[1]) + if version < MIN_VERSION: + print(f"Skipping old tag {tag.name} ({version})") + continue + + print(f"Trying tag {tag.name} ({version})") + + result = handle_commit( + repo, tag.commit, tag.name, "tag", supported_refs, old_versions + ) + versions[tag.name] = result + + for branch in repo.remote("origin").refs: + m = BRANCH_PATTERN.match(branch.name) + if m is not None: + fullname = m[1] + version = packaging.version.parse(m[3]) + if version < MIN_VERSION: + print(f"Skipping old branch {fullname} ({version})") + continue + print(f"Trying branch {fullname} ({version})") + elif branch.name == f"{REMOTE}/{MAIN_BRANCH}": + fullname = MAIN_BRANCH + print(f"Trying development branch {fullname}") + else: + continue + + result = handle_commit( + repo, branch.commit, fullname, "branch", supported_refs, old_versions + ) + versions[fullname] = result + + + with open(os.path.join(BASE_DIR, "versions.json"), "w") as out: + json.dump(versions, out, sort_keys=True, indent=2) + out.write("\n") + +if __name__ == '__main__': + main() diff --git a/pkgs/os-specific/bsd/freebsd/versions.json b/pkgs/os-specific/bsd/freebsd/versions.json new file mode 100644 index 000000000000..736c4c3a3e3c --- /dev/null +++ b/pkgs/os-specific/bsd/freebsd/versions.json @@ -0,0 +1,210 @@ +{ + "main": { + "hash": "sha256-C5ucT9BK/eK8a9HNSDDi8S1uhpPmiqV22XEooxAqbPw=", + "ref": "main", + "refType": "branch", + "rev": "125c4560bc70971b950d035cfcd2255b89984011", + "supported": false, + "version": { + "branch": "CURRENT", + "major": 15, + "reldate": "1500017", + "release": "15.0-CURRENT", + "revision": "15.0", + "type": "FreeBSD", + "version": "FreeBSD 15.0-CURRENT" + } + }, + "release/13.0.0": { + "hash": "sha256-2WYk/taxWc74uh2KJf9TzWDxUPrtkvt2nhU/qUZMu+Q=", + "ref": "release/13.0.0", + "refType": "tag", + "rev": "ea31abc261ffc01b6ff5671bffb15cf910a07f4b", + "supported": false, + "version": { + "branch": "RELEASE", + "major": 13, + "reldate": "1300139", + "release": "13.0-RELEASE", + "revision": "13.0", + "type": "FreeBSD", + "version": "FreeBSD 13.0-RELEASE" + } + }, + "release/13.1.0": { + "hash": "sha256-m2aR2bwJNxsBepJ5ybWiaJp4Nwm+l0bMcn0gTSeY0JI=", + "ref": "release/13.1.0", + "refType": "tag", + "rev": "fc952ac2212b121aa6eefc273f5960ec3e0a466d", + "supported": false, + "version": { + "branch": "RELEASE", + "major": 13, + "reldate": "1301000", + "release": "13.1-RELEASE", + "revision": "13.1", + "type": "FreeBSD", + "version": "FreeBSD 13.1-RELEASE" + } + }, + "release/13.2.0": { + "hash": "sha256-VuktVknlKYkklST0I5CUiH7OsDn3DVTE1W9O/IhaCkE=", + "ref": "release/13.2.0", + "refType": "tag", + "rev": "525ecfdad597980ea4cd59238e24c8530dbcd31d", + "supported": false, + "version": { + "branch": "RELEASE", + "major": 13, + "reldate": "1302001", + "release": "13.2-RELEASE", + "revision": "13.2", + "type": "FreeBSD", + "version": "FreeBSD 13.2-RELEASE" + } + }, + "release/13.3.0": { + "hash": "sha256-djqHlPnGlJCi9DGtX1kTULB2EEj8YUsjGTIUDQoHzAQ=", + "ref": "release/13.3.0", + "refType": "tag", + "rev": "80d2b634ddf0b459910b54a04bc09f5cbc7185a7", + "supported": false, + "version": { + "branch": "RELEASE", + "major": 13, + "reldate": "1303001", + "release": "13.3-RELEASE", + "revision": "13.3", + "type": "FreeBSD", + "version": "FreeBSD 13.3-RELEASE" + } + }, + "release/14.0.0": { + "hash": "sha256-eBKwCYcOG9Lg7gBA2gZqxQFO/3uMMrcQGtgqi8se6zA=", + "ref": "release/14.0.0", + "refType": "tag", + "rev": "f9716eee8ab45ad906d9b5c5233ca20c10226ca7", + "supported": false, + "version": { + "branch": "RELEASE", + "major": 14, + "reldate": "1400097", + "release": "14.0-RELEASE", + "revision": "14.0", + "type": "FreeBSD", + "version": "FreeBSD 14.0-RELEASE" + } + }, + "releng/13.0": { + "hash": "sha256-7PrqTb2o21IQgQ2N+zjavlzX/ju60Rw+MXjMRICmQi0=", + "ref": "releng/13.0", + "refType": "branch", + "rev": "5fe9c9de03ef3191d216964bc4d8e427d5ed5720", + "supported": false, + "version": { + "branch": "RELEASE-p13", + "major": 13, + "reldate": "1300139", + "release": "13.0-RELEASE-p13", + "revision": "13.0", + "type": "FreeBSD", + "version": "FreeBSD 13.0-RELEASE-p13" + } + }, + "releng/13.1": { + "hash": "sha256-9fou2HVWlpNRKkc8XToe8/aSxwbNsIZIAKpteeSjLnc=", + "ref": "releng/13.1", + "refType": "branch", + "rev": "39b281c2996526288c0f2ae94abe6b164bcd5954", + "supported": false, + "version": { + "branch": "RELEASE-p9", + "major": 13, + "reldate": "1301000", + "release": "13.1-RELEASE-p9", + "revision": "13.1", + "type": "FreeBSD", + "version": "FreeBSD 13.1-RELEASE-p9" + } + }, + "releng/13.2": { + "hash": "sha256-KN508aIe02Ue4TjlonO6TmAQ7DmiOOSOYrZfg5HP9AM=", + "ref": "releng/13.2", + "refType": "branch", + "rev": "f5ac4e174fdd3497749e351c27aafb34171c5730", + "supported": true, + "version": { + "branch": "RELEASE-p11", + "major": 13, + "reldate": "1302001", + "release": "13.2-RELEASE-p11", + "revision": "13.2", + "type": "FreeBSD", + "version": "FreeBSD 13.2-RELEASE-p11" + } + }, + "releng/13.3": { + "hash": "sha256-huzUiMZHfyK/mgLD3hW+DaSGgAaTUIuM51xDp+IE3qE=", + "ref": "releng/13.3", + "refType": "branch", + "rev": "7a0d63c9093222938f26cd63ff742e555168de77", + "supported": true, + "version": { + "branch": "RELEASE-p1", + "major": 13, + "reldate": "1303001", + "release": "13.3-RELEASE-p1", + "revision": "13.3", + "type": "FreeBSD", + "version": "FreeBSD 13.3-RELEASE-p1" + } + }, + "releng/14.0": { + "hash": "sha256-15B9Nglshniokju88dEKj3BIffZ6L28L+ZuhAC3UqOI=", + "ref": "releng/14.0", + "refType": "branch", + "rev": "d338712beb16ad7740bbd00bd93299a131a68045", + "supported": true, + "version": { + "branch": "RELEASE-p6", + "major": 14, + "reldate": "1400097", + "release": "14.0-RELEASE-p6", + "revision": "14.0", + "type": "FreeBSD", + "version": "FreeBSD 14.0-RELEASE-p6" + } + }, + "stable/13": { + "hash": "sha256-XateLKKs2A/HCP9Lx/nBm1cybB3otrbeXQvyCL40S0M=", + "ref": "stable/13", + "refType": "branch", + "rev": "e0a58ef24a3baf5ed4cc09a798b9fe2d85408052", + "supported": true, + "version": { + "branch": "STABLE", + "major": 13, + "reldate": "1303502", + "release": "13.3-STABLE", + "revision": "13.3", + "type": "FreeBSD", + "version": "FreeBSD 13.3-STABLE" + } + }, + "stable/14": { + "hash": "sha256-tIKnK/SYBDk9UnE5AfhjeDpqHnzspYbor0678ye/mrs=", + "ref": "stable/14", + "refType": "branch", + "rev": "ab872ab0bf195e872ed8d955aab3b2a537a230cd", + "supported": true, + "version": { + "branch": "STABLE", + "major": 14, + "reldate": "1400510", + "release": "14.0-STABLE", + "revision": "14.0", + "type": "FreeBSD", + "version": "FreeBSD 14.0-STABLE" + } + } +}