From ce63b69e482449eeb7238b329546a83346c2522e Mon Sep 17 00:00:00 2001 From: Thiago Kenji Okada Date: Sun, 5 Jan 2025 13:24:35 +0000 Subject: [PATCH 1/4] nixos-rebuild-ng: implement build-image --- .../src/nixos_rebuild/__init__.py | 49 +++++++++- .../src/nixos_rebuild/models.py | 3 + .../nixos-rebuild-ng/src/nixos_rebuild/nix.py | 54 +++++++++++ .../nixos-rebuild-ng/src/tests/test_main.py | 69 +++++++++++++ .../ni/nixos-rebuild-ng/src/tests/test_nix.py | 97 +++++++++++++++++++ 5 files changed, 268 insertions(+), 4 deletions(-) diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py index 73788bb8ccfe..793a06dee607 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py @@ -9,7 +9,7 @@ from typing import assert_never from . import nix, tmpdir from .constants import EXECUTABLE, WITH_NIX_2_18, WITH_REEXEC, WITH_SHELL_FILES -from .models import Action, BuildAttr, Flake, NRError, Profile +from .models import Action, BuildAttr, Flake, ImageVariants, NRError, Profile from .process import Remote, cleanup_ssh from .utils import Args, LogFormatter, tabulate @@ -176,6 +176,11 @@ def get_parser() -> tuple[argparse.ArgumentParser, dict[str, argparse.ArgumentPa "--target-host", help="Specifies host to activate the configuration" ) main_parser.add_argument("--no-build-nix", action="store_true", help="Deprecated") + main_parser.add_argument( + "--image-variant", + help="Selects an image variant to build from the " + + "config.system.build.images attribute of the given configuration", + ) main_parser.add_argument("action", choices=Action.values(), nargs="?") return main_parser, sub_parsers @@ -285,8 +290,7 @@ def reexec( ) except CalledProcessError: logger.warning( - "could not build a newer version of nixos-rebuild, " - + "using current version" + "could not build a newer version of nixos-rebuild, using current version" ) if drv: @@ -372,6 +376,7 @@ def execute(argv: list[str]) -> None: | Action.BUILD | Action.DRY_BUILD | Action.DRY_ACTIVATE + | Action.BUILD_IMAGE | Action.BUILD_VM | Action.BUILD_VM_WITH_BOOTLOADER ): @@ -383,7 +388,29 @@ def execute(argv: list[str]) -> None: flake_build_flags |= {"no_link": no_link, "dry_run": dry_run} rollback = bool(args.rollback) + def validate_image_variant(variants: ImageVariants) -> None: + if args.image_variant not in variants: + raise NRError( + "please specify one of the following " + + "supported image variants via --image-variant:\n" + + "\n".join(f"- {v}" for v in variants.keys()) + ) + match action: + case Action.BUILD_IMAGE if flake: + variants = nix.get_build_image_variants_flake( + flake, + eval_flags=flake_common_flags, + ) + validate_image_variant(variants) + attr = f"config.system.build.images.{args.image_variant}" + case Action.BUILD_IMAGE: + variants = nix.get_build_image_variants( + build_attr, + instantiate_flags=common_flags, + ) + validate_image_variant(variants) + attr = f"config.system.build.images.{args.image_variant}" case Action.BUILD_VM: attr = "config.system.build.vm" case Action.BUILD_VM_WITH_BOOTLOADER: @@ -473,8 +500,22 @@ def execute(argv: list[str]) -> None: # If you get `not-found`, please open an issue vm_path = next(path_to_config.glob("bin/run-*-vm"), "not-found") print( - f"Done. The virtual machine can be started by running '{vm_path}'" + "Done. The virtual machine can be started by running", + end=" ", + file=sys.stderr, + flush=True, ) + print(vm_path, flush=True) + elif action == Action.BUILD_IMAGE: + image_name = variants[args.image_variant] + disk_path = path_to_config / image_name + print( + "Done. The disk image can be found in", + end=" ", + file=sys.stderr, + flush=True, + ) + print(disk_path, flush=True) case Action.EDIT: nix.edit(flake, flake_build_flags) case Action.DRY_RUN: diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/models.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/models.py index 1910e2681bfd..267ac3357450 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/models.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/models.py @@ -8,6 +8,8 @@ from typing import Any, Callable, ClassVar, Self, TypedDict, override from .process import Remote, run_wrapper +type ImageVariants = dict[str, str] + class NRError(Exception): "nixos-rebuild general error." @@ -30,6 +32,7 @@ class Action(Enum): DRY_BUILD = "dry-build" DRY_RUN = "dry-run" DRY_ACTIVATE = "dry-activate" + BUILD_IMAGE = "build-image" BUILD_VM = "build-vm" BUILD_VM_WITH_BOOTLOADER = "build-vm-with-bootloader" LIST_GENERATIONS = "list-generations" diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/nix.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/nix.py index 876a9a06c9c7..66f561ac4fef 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/nix.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/nix.py @@ -1,5 +1,7 @@ +import json import logging import os +import textwrap from concurrent.futures import ThreadPoolExecutor from datetime import datetime from importlib.resources import files @@ -17,6 +19,7 @@ from .models import ( Flake, Generation, GenerationJson, + ImageVariants, NRError, Profile, Remote, @@ -263,6 +266,57 @@ def find_file(file: str, nix_flags: Args | None = None) -> Path | None: return Path(r.stdout.strip()) +def get_build_image_variants( + build_attr: BuildAttr, + instantiate_flags: Args | None = None, +) -> ImageVariants: + path = ( + f'"{build_attr.path.resolve()}"' + if isinstance(build_attr.path, Path) + else build_attr.path + ) + r = run_wrapper( + [ + "nix-instantiate", + "--eval", + "--strict", + "--json", + "--expr", + textwrap.dedent(f""" + let + value = import {path}; + set = if builtins.isFunction value then value {{}} else value; + in + builtins.mapAttrs (n: v: v.passthru.filePath) set.{build_attr.to_attr("config.system.build.images")} + """), + *dict_to_flags(instantiate_flags), + ], + stdout=PIPE, + ) + j: ImageVariants = json.loads(r.stdout.strip()) + return j + + +def get_build_image_variants_flake( + flake: Flake, + eval_flags: Args | None = None, +) -> ImageVariants: + r = run_wrapper( + [ + "nix", + "eval", + "--json", + flake.to_attr("config.system.build.images"), + "--apply", + "builtins.mapAttrs (n: v: v.passthru.filePath)", + *dict_to_flags(eval_flags), + ], + stdout=PIPE, + ) + j: ImageVariants = json.loads(r.stdout.strip()) + return j + + def get_nixpkgs_rev(nixpkgs_path: Path | None) -> str | None: """Get Nixpkgs path as a Git revision. diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_main.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_main.py index 63ae24147929..de376135678e 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_main.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_main.py @@ -150,6 +150,75 @@ def test_execute_nix_boot(mock_run: Any, tmp_path: Path) -> None: ) +@patch.dict(nr.process.os.environ, {}, clear=True) +@patch(get_qualified_name(nr.process.subprocess.run), autospec=True) +def test_execute_nix_build_image_flake(mock_run: Any, tmp_path: Path) -> None: + config_path = tmp_path / "test" + config_path.touch() + + def run_side_effect(args: list[str], **kwargs: Any) -> CompletedProcess[str]: + if args[0] == "nix" and "eval" in args: + return CompletedProcess( + [], + 0, + """ + { + "azure": "nixos-image-azure-25.05.20250102.6df2492-x86_64-linux.vhd", + "vmware": "nixos-image-vmware-25.05.20250102.6df2492-x86_64-linux.vmdk" + } + """, + ) + elif args[0] == "nix": + return CompletedProcess([], 0, str(config_path)) + else: + return CompletedProcess([], 0) + + mock_run.side_effect = run_side_effect + + nr.execute( + [ + "nixos-rebuild", + "build-image", + "--image-variant", + "azure", + "--flake", + "/path/to/config#hostname", + ] + ) + + assert mock_run.call_count == 2 + mock_run.assert_has_calls( + [ + call( + [ + "nix", + "eval", + "--json", + "/path/to/config#nixosConfigurations.hostname.config.system.build.images", + "--apply", + "builtins.mapAttrs (n: v: v.passthru.filePath)", + ], + check=True, + stdout=PIPE, + **DEFAULT_RUN_KWARGS, + ), + call( + [ + "nix", + "--extra-experimental-features", + "nix-command flakes", + "build", + "--print-out-paths", + "/path/to/config#nixosConfigurations.hostname.config.system.build.images.azure", + ], + check=True, + stdout=PIPE, + **DEFAULT_RUN_KWARGS, + ), + ] + ) + + @patch.dict(nr.process.os.environ, {}, clear=True) @patch(get_qualified_name(nr.process.subprocess.run), autospec=True) def test_execute_nix_switch_flake(mock_run: Any, tmp_path: Path) -> None: diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_nix.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_nix.py index a040d60e6368..8b548eb14fa5 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_nix.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/tests/test_nix.py @@ -314,6 +314,103 @@ def test_edit(mock_run: Any, monkeypatch: Any, tmpdir: Any) -> None: mock_run.assert_called_with(["editor", default_nix], check=False) +@patch( + get_qualified_name(n.run_wrapper, n), + autospec=True, + return_value=CompletedProcess( + [], + 0, + """ + { + "azure": "nixos-image-azure-25.05.20250102.6df2492-x86_64-linux.vhd", + "vmware": "nixos-image-vmware-25.05.20250102.6df2492-x86_64-linux.vmdk" + } + """, + ), +) +def test_get_build_image_variants(mock_run: Any) -> None: + build_attr = m.BuildAttr("", None) + assert n.get_build_image_variants(build_attr) == { + "azure": "nixos-image-azure-25.05.20250102.6df2492-x86_64-linux.vhd", + "vmware": "nixos-image-vmware-25.05.20250102.6df2492-x86_64-linux.vmdk", + } + mock_run.assert_called_with( + [ + "nix-instantiate", + "--eval", + "--strict", + "--json", + "--expr", + textwrap.dedent(""" + let + value = import ; + set = if builtins.isFunction value then value {} else value; + in + builtins.mapAttrs (n: v: v.passthru.filePath) set.config.system.build.images + """), + ], + stdout=PIPE, + ) + + build_attr = m.BuildAttr(Path("/tmp"), "preAttr") + assert n.get_build_image_variants(build_attr, {"inst_flag": True}) == { + "azure": "nixos-image-azure-25.05.20250102.6df2492-x86_64-linux.vhd", + "vmware": "nixos-image-vmware-25.05.20250102.6df2492-x86_64-linux.vmdk", + } + mock_run.assert_called_with( + [ + "nix-instantiate", + "--eval", + "--strict", + "--json", + "--expr", + textwrap.dedent(""" + let + value = import "/tmp"; + set = if builtins.isFunction value then value {} else value; + in + builtins.mapAttrs (n: v: v.passthru.filePath) set.preAttr.config.system.build.images + """), + "--inst-flag", + ], + stdout=PIPE, + ) + + +@patch( + get_qualified_name(n.run_wrapper, n), + autospec=True, + return_value=CompletedProcess( + [], + 0, + """ + { + "azure": "nixos-image-azure-25.05.20250102.6df2492-x86_64-linux.vhd", + "vmware": "nixos-image-vmware-25.05.20250102.6df2492-x86_64-linux.vmdk" + } + """, + ), +) +def test_get_build_image_variants_flake(mock_run: Any) -> None: + flake = m.Flake(Path("flake.nix"), "myAttr") + assert n.get_build_image_variants_flake(flake, {"eval_flag": True}) == { + "azure": "nixos-image-azure-25.05.20250102.6df2492-x86_64-linux.vhd", + "vmware": "nixos-image-vmware-25.05.20250102.6df2492-x86_64-linux.vmdk", + } + mock_run.assert_called_with( + [ + "nix", + "eval", + "--json", + "flake.nix#myAttr.config.system.build.images", + "--apply", + "builtins.mapAttrs (n: v: v.passthru.filePath)", + "--eval-flag", + ], + stdout=PIPE, + ) + + def test_get_nixpkgs_rev() -> None: assert n.get_nixpkgs_rev(None) is None From fbb00d3034c1bb3bd96ba050baeadc159280c6e1 Mon Sep 17 00:00:00 2001 From: Thiago Kenji Okada Date: Sun, 5 Jan 2025 14:23:13 +0000 Subject: [PATCH 2/4] nixos-rebuild-ng: document build-image in manpage --- .../ni/nixos-rebuild-ng/nixos-rebuild.8.scd | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/nixos-rebuild.8.scd b/pkgs/by-name/ni/nixos-rebuild-ng/nixos-rebuild.8.scd index 25581f91c0f3..cc4fc3a19ca8 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/nixos-rebuild.8.scd +++ b/pkgs/by-name/ni/nixos-rebuild-ng/nixos-rebuild.8.scd @@ -22,8 +22,9 @@ _nixos-rebuild_ \[--verbose] [--max-jobs MAX_JOBS] [--cores CORES] [--log-format \[--no-update-lock-file] [--no-write-lock-file] [--no-registries] [--commit-lock-file] [--update-input UPDATE_INPUT] [--override-input OVERRIDE_INPUT OVERRIDE_INPUT]++ \[--no-build-output] [--use-substitutes] [--help] [--file FILE] [--attr ATTR] [--flake [FLAKE]] [--no-flake] [--install-bootloader] [--profile-name PROFILE_NAME]++ \[--specialisation SPECIALISATION] [--rollback] [--upgrade] [--upgrade-all] [--json] [--ask-sudo-password] [--sudo] [--fast]++ + \[--image-variant VARIANT]++ \[--build-host BUILD_HOST] [--target-host TARGET_HOST]++ - \[{switch,boot,test,build,edit,repl,dry-build,dry-run,dry-activate,build-vm,build-vm-with-bootloader,list-generations}] + \[{switch,boot,test,build,edit,repl,dry-build,dry-run,dry-activate,build-image,build-vm,build-vm-with-bootloader,list-generations}] # DESCRIPTION @@ -106,6 +107,13 @@ It must be one of the following: *repl* Opens the configuration in *nix repl*. +*build-image* + Build a disk-image variant, pre-configured for the given + platform/provider. Select a variant with the *--image-variant* option + or run without any options to get a list of available variants. + + $ nixos-rebuild build-image --image-variant proxmox + *build-vm* Build a script that starts a NixOS virtual machine with the desired configuration. It leaves a symlink _result_ in the current directory that @@ -206,6 +214,11 @@ It must be one of the following: Activates given specialisation; when not specified, switching and testing will activate the base, unspecialised system. +*--image-variant* _variant_ + Selects an image variant to build from the _config.system.build.images_ + attribute of the given configuration. A list of variants is printed if + this option remains unset. + *--build-host* _host_ Instead of building the new configuration locally, use the specified host to perform the build. The host needs to be accessible with ssh, and must From b747c677eccb89c31a65a05e26e8d622b3ef2e1c Mon Sep 17 00:00:00 2001 From: Thiago Kenji Okada Date: Sun, 5 Jan 2025 16:32:06 +0000 Subject: [PATCH 3/4] nixos-rebuild-ng: refactor using match --- .../src/nixos_rebuild/__init__.py | 60 +++++++++---------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py index 793a06dee607..95517c0ac920 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/nixos_rebuild/__init__.py @@ -487,39 +487,37 @@ def execute(argv: list[str]) -> None: sudo=args.sudo, ) - if action in (Action.SWITCH, Action.BOOT, Action.TEST, Action.DRY_ACTIVATE): - nix.switch_to_configuration( - path_to_config, - action, - target_host=target_host, - sudo=args.sudo, - specialisation=args.specialisation, - install_bootloader=args.install_bootloader, - ) - elif action in (Action.BUILD_VM, Action.BUILD_VM_WITH_BOOTLOADER): - # If you get `not-found`, please open an issue - vm_path = next(path_to_config.glob("bin/run-*-vm"), "not-found") - print( - "Done. The virtual machine can be started by running", - end=" ", - file=sys.stderr, - flush=True, - ) - print(vm_path, flush=True) - elif action == Action.BUILD_IMAGE: - image_name = variants[args.image_variant] - disk_path = path_to_config / image_name - print( - "Done. The disk image can be found in", - end=" ", - file=sys.stderr, - flush=True, - ) - print(disk_path, flush=True) + # Print only the result to stdout to make it easier to script + def print_result(msg: str, result: str | Path) -> None: + print(msg, end=" ", file=sys.stderr, flush=True) + print(result, flush=True) + + match action: + case Action.SWITCH | Action.BOOT | Action.TEST | Action.DRY_ACTIVATE: + nix.switch_to_configuration( + path_to_config, + action, + target_host=target_host, + sudo=args.sudo, + specialisation=args.specialisation, + install_bootloader=args.install_bootloader, + ) + case Action.BUILD_VM | Action.BUILD_VM_WITH_BOOTLOADER: + # If you get `not-found`, please open an issue + vm_path = next(path_to_config.glob("bin/run-*-vm"), "not-found") + print_result( + "Done. The virtual machine can be started by running", vm_path + ) + case Action.BUILD_IMAGE: + disk_path = path_to_config / variants[args.image_variant] + print_result("Done. The disk image can be found in", disk_path) + case Action.EDIT: nix.edit(flake, flake_build_flags) + case Action.DRY_RUN: - assert False, "DRY_RUN should be a DRY_BUILD alias" + raise AssertionError("DRY_RUN should be a DRY_BUILD alias") + case Action.LIST_GENERATIONS: generations = nix.list_generations(profile) if args.json: @@ -535,11 +533,13 @@ def execute(argv: list[str]) -> None: "current": "Current", } print(tabulate(generations, headers=headers)) + case Action.REPL: if flake: nix.repl_flake("toplevel", flake, flake_build_flags) else: nix.repl("system", build_attr, build_flags) + case _: assert_never(action) From eccb8552c0b27b6c70b71e1584246808b14940f9 Mon Sep 17 00:00:00 2001 From: Thiago Kenji Okada Date: Mon, 6 Jan 2025 08:29:24 +0000 Subject: [PATCH 4/4] nixos-rebuild-ng: fix addopts in pytest --- pkgs/by-name/ni/nixos-rebuild-ng/src/pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkgs/by-name/ni/nixos-rebuild-ng/src/pyproject.toml b/pkgs/by-name/ni/nixos-rebuild-ng/src/pyproject.toml index 9d3f92f5471e..018d156fd381 100644 --- a/pkgs/by-name/ni/nixos-rebuild-ng/src/pyproject.toml +++ b/pkgs/by-name/ni/nixos-rebuild-ng/src/pyproject.toml @@ -52,4 +52,4 @@ extend-select = [ [tool.pytest.ini_options] pythonpath = ["."] -addopts = ["--import-mode=importlib"] +addopts = "--import-mode=importlib"