From c08efe1438362d2cdcb64849bd7e28dd688b55f6 Mon Sep 17 00:00:00 2001 From: K900 Date: Fri, 22 Sep 2023 14:28:05 +0300 Subject: [PATCH 1/2] linux: more update-script cleanups/fixes - special case linux-testing fetching - use hash instead of sha256 everywhere - respect COMMIT envvar This causes rebuilds, so should go in with the next bump probably. --- .../os-specific/linux/kernel/kernels-org.json | 20 ++++++------ pkgs/os-specific/linux/kernel/mainline.nix | 25 ++++++++++----- .../linux/kernel/update-mainline.py | 32 ++++++++++++++++--- 3 files changed, 54 insertions(+), 23 deletions(-) diff --git a/pkgs/os-specific/linux/kernel/kernels-org.json b/pkgs/os-specific/linux/kernel/kernels-org.json index b2d4b1fa83b8..7b212cbdda35 100644 --- a/pkgs/os-specific/linux/kernel/kernels-org.json +++ b/pkgs/os-specific/linux/kernel/kernels-org.json @@ -1,38 +1,38 @@ { "testing": { - "version": "6.6-rc1", - "hash": "02zh3dnikyhhlas9xccia963d4yqmzq0m4b8s10x8mjng3na45hd" + "version": "6.6-rc2", + "hash": "sha256:1hbva5vsfi48h82ll4kmhzm5hxp7340bj2smwgvjikam26icaj54" }, "6.5": { "version": "6.5.4", - "hash": "0s8nzd8yaq06bq8byk7aakbk95gh0rhlif26h1biw94v48anrxxx" + "hash": "sha256:0s8nzd8yaq06bq8byk7aakbk95gh0rhlif26h1biw94v48anrxxx" }, "6.4": { "version": "6.4.16", - "hash": "0zgj1z97jyx7wf12zrnlcp0mj4cl43ais9qsy6dh1jwylf2fq9ln" + "hash": "sha256:0zgj1z97jyx7wf12zrnlcp0mj4cl43ais9qsy6dh1jwylf2fq9ln" }, "6.1": { "version": "6.1.54", - "hash": "09sfrq2l8f777mx2n9mhb6bgz1064bl04921byqnmk87si31w653" + "hash": "sha256:09sfrq2l8f777mx2n9mhb6bgz1064bl04921byqnmk87si31w653" }, "5.15": { "version": "5.15.132", - "hash": "1b0qjsaqjw2rk86shmmrj2aasblkn27acjmc761vnjg7sv2baxs1" + "hash": "sha256:1b0qjsaqjw2rk86shmmrj2aasblkn27acjmc761vnjg7sv2baxs1" }, "5.10": { "version": "5.10.195", - "hash": "0n4vg2i9sq89wnz85arlyvwysh9s83cgzs5bk2wh98bivi5fwfs1" + "hash": "sha256:0n4vg2i9sq89wnz85arlyvwysh9s83cgzs5bk2wh98bivi5fwfs1" }, "5.4": { "version": "5.4.256", - "hash": "0fim5q9xakwnjfg48bpsic9r2r8dvrjlalqqkm9vh1rml9mhi967" + "hash": "sha256:0fim5q9xakwnjfg48bpsic9r2r8dvrjlalqqkm9vh1rml9mhi967" }, "4.19": { "version": "4.19.294", - "hash": "03x0xsb8a369zdr81hg6xdl5n5v48k6iwnhj6r29725777lvvbfc" + "hash": "sha256:03x0xsb8a369zdr81hg6xdl5n5v48k6iwnhj6r29725777lvvbfc" }, "4.14": { "version": "4.14.325", - "hash": "117p1mdha57f6d3kdwac9jrbmib7g77q4xhir8ghl6fmrs1f2sav" + "hash": "sha256:117p1mdha57f6d3kdwac9jrbmib7g77q4xhir8ghl6fmrs1f2sav" } } diff --git a/pkgs/os-specific/linux/kernel/mainline.nix b/pkgs/os-specific/linux/kernel/mainline.nix index 50053e620e46..4e1d5b8a9e87 100644 --- a/pkgs/os-specific/linux/kernel/mainline.nix +++ b/pkgs/os-specific/linux/kernel/mainline.nix @@ -1,18 +1,27 @@ -{ branch, lib, fetchurl, buildLinux, ... } @ args: +{ branch, lib, fetchurl, fetchzip, buildLinux, ... } @ args: let allKernels = builtins.fromJSON (builtins.readFile ./kernels-org.json); thisKernel = allKernels.${branch}; + inherit (thisKernel) version; + + src = + # testing kernels are a special case because they don't have tarballs on the CDN + if branch == "testing" + then fetchzip { + url = "https://git.kernel.org/torvalds/t/linux-${version}.tar.gz"; + inherit (thisKernel) hash; + } + else fetchurl { + url = "mirror://kernel/linux/kernel/v${lib.versions.major version}.x/linux-${version}.tar.xz"; + inherit (thisKernel) hash; + }; + + args' = (builtins.removeAttrs args ["branch"]) // { + inherit src version; - args' = (builtins.removeAttrs args ["branch"]) // rec { - inherit (thisKernel) version; modDirVersion = lib.versions.pad 3 version; extraMeta.branch = branch; - - src = fetchurl { - url = "mirror://kernel/linux/kernel/v${lib.versions.major version}.x/linux-${version}.tar.xz"; - sha256 = thisKernel.hash; - }; } // (args.argsOverride or {}); in buildLinux args' diff --git a/pkgs/os-specific/linux/kernel/update-mainline.py b/pkgs/os-specific/linux/kernel/update-mainline.py index e7c37e9ab999..8fac82e6633e 100755 --- a/pkgs/os-specific/linux/kernel/update-mainline.py +++ b/pkgs/os-specific/linux/kernel/update-mainline.py @@ -9,6 +9,7 @@ import re import subprocess import urllib.request import sys +import os HERE = pathlib.Path(__file__).parent @@ -24,6 +25,7 @@ class KernelNature(Enum): class KernelRelease: nature: KernelNature version: str + branch: str date: str link: str eol: bool = False @@ -43,7 +45,14 @@ def parse_release(release: Tag) -> KernelRelease | None: assert link is not None, f'link for kernel {version} is non-existent' eol = bool(release.find(class_='eolkernel')) - return KernelRelease(nature=nature, version=version, date=date, link=link, eol=eol) + return KernelRelease( + nature=nature, + branch=get_branch(version), + version=version, + date=date, + link=link, + eol=eol, + ) def get_branch(version: str): # This is a testing kernel. @@ -53,9 +62,18 @@ def get_branch(version: str): major, minor, *_ = version.split(".") return f"{major}.{minor}" +def get_hash(kernel: KernelRelease): + if kernel.branch == "testing": + args = ["--unpack"] + else: + args = [] -def get_hash(url: str): - return subprocess.check_output(["nix-prefetch-url", url]).decode().strip() + hash = ( + subprocess.check_output(["nix-prefetch-url", kernel.link] + args) + .decode() + .strip() + ) + return f"sha256:{hash}" def commit(message): @@ -91,13 +109,17 @@ def main(): print(message) - all_kernels[branch] = {"version": kernel.version, "hash": get_hash(kernel.link)} + all_kernels[branch] = { + "version": kernel.version, + "hash": get_hash(kernel), + } with VERSIONS_FILE.open("w") as fd: json.dump(all_kernels, fd, indent=4) fd.write("\n") # makes editorconfig happy - commit(message) + if os.environ.get("COMMIT") == "1": + commit(message) if __name__ == "__main__": From eadfc7b13133c0f23dd4717f25f65ea981b40344 Mon Sep 17 00:00:00 2001 From: K900 Date: Fri, 22 Sep 2023 16:10:21 +0300 Subject: [PATCH 2/2] linux/update-script: format with black + isort --- .../linux/kernel/update-mainline.py | 46 ++++++++++--------- 1 file changed, 25 insertions(+), 21 deletions(-) diff --git a/pkgs/os-specific/linux/kernel/update-mainline.py b/pkgs/os-specific/linux/kernel/update-mainline.py index 8fac82e6633e..df8257fa0ef0 100755 --- a/pkgs/os-specific/linux/kernel/update-mainline.py +++ b/pkgs/os-specific/linux/kernel/update-mainline.py @@ -1,26 +1,27 @@ #!/usr/bin/env nix-shell #!nix-shell -i python3 -p "python3.withPackages (ps: [ ps.beautifulsoup4 ps.lxml ])" -from enum import Enum -from bs4 import BeautifulSoup, NavigableString, Tag -from dataclasses import dataclass import json -import pathlib -import re -import subprocess -import urllib.request -import sys import os +import pathlib +import subprocess +import sys +import urllib.request +from dataclasses import dataclass +from enum import Enum +from bs4 import BeautifulSoup, NavigableString, Tag HERE = pathlib.Path(__file__).parent ROOT = HERE.parent.parent.parent.parent VERSIONS_FILE = HERE / "kernels-org.json" + class KernelNature(Enum): MAINLINE = 1 STABLE = 2 LONGTERM = 3 + @dataclass class KernelRelease: nature: KernelNature @@ -30,20 +31,21 @@ class KernelRelease: link: str eol: bool = False + def parse_release(release: Tag) -> KernelRelease | None: - columns: list[Tag] = list(release.find_all('td')) + columns: list[Tag] = list(release.find_all("td")) try: - nature = KernelNature[columns[0].get_text().rstrip(':').upper()] + nature = KernelNature[columns[0].get_text().rstrip(":").upper()] except KeyError: return None - version = columns[1].get_text().rstrip(' [EOL]') + version = columns[1].get_text().rstrip(" [EOL]") date = columns[2].get_text() - link = columns[3].find('a') + link = columns[3].find("a") if link is not None and isinstance(link, Tag): - link = link.attrs.get('href') - assert link is not None, f'link for kernel {version} is non-existent' - eol = bool(release.find(class_='eolkernel')) + link = link.attrs.get("href") + assert link is not None, f"link for kernel {version} is non-existent" + eol = bool(release.find(class_="eolkernel")) return KernelRelease( nature=nature, @@ -54,14 +56,16 @@ def parse_release(release: Tag) -> KernelRelease | None: eol=eol, ) + def get_branch(version: str): # This is a testing kernel. - if 'rc' in version: - return 'testing' + if "rc" in version: + return "testing" else: major, minor, *_ = version.split(".") return f"{major}.{minor}" + def get_hash(kernel: KernelRelease): if kernel.branch == "testing": args = ["--unpack"] @@ -83,19 +87,19 @@ def commit(message): def main(): kernel_org = urllib.request.urlopen("https://kernel.org/") soup = BeautifulSoup(kernel_org.read().decode(), "lxml") - release_table = soup.find(id='releases') + release_table = soup.find(id="releases") if not release_table or isinstance(release_table, NavigableString): print(release_table) - print('Failed to find the release table on https://kernel.org') + print("Failed to find the release table on https://kernel.org") sys.exit(1) - releases = release_table.find_all('tr') + releases = release_table.find_all("tr") parsed_releases = filter(None, [parse_release(release) for release in releases]) all_kernels = json.load(VERSIONS_FILE.open()) for kernel in parsed_releases: branch = get_branch(kernel.version) - nixpkgs_branch = branch.replace('.', '_') + nixpkgs_branch = branch.replace(".", "_") old_version = all_kernels.get(branch, {}).get("version") if old_version == kernel.version: