diff --git a/nixos/tests/all-tests.nix b/nixos/tests/all-tests.nix index db9dd1fb2561..ef321d3af0bf 100644 --- a/nixos/tests/all-tests.nix +++ b/nixos/tests/all-tests.nix @@ -364,6 +364,7 @@ in }; cloud-init = runTest ./cloud-init.nix; cloud-init-hostname = runTest ./cloud-init-hostname.nix; + cloudcompare = import ./cloudcompare.nix { inherit pkgs runTest; }; cloudlog = runTest ./cloudlog.nix; cntr = import ./cntr.nix { inherit (pkgs) lib; @@ -495,7 +496,7 @@ in drupal = runTest ./drupal.nix; dublin-traceroute = runTest ./dublin-traceroute.nix; dwl = runTestOn [ "x86_64-linux" "aarch64-linux" ] ./dwl.nix; - e57inspector = runTest ./e57inspector.nix; + e57inspector = import ./e57inspector.nix { inherit pkgs runTest; }; early-mount-options = runTest ./early-mount-options.nix; earlyoom = runTestOn [ "x86_64-linux" ] ./earlyoom.nix; easytier = runTest ./easytier.nix; diff --git a/nixos/tests/cloudcompare.nix b/nixos/tests/cloudcompare.nix new file mode 100644 index 000000000000..0e9b72da3a41 --- /dev/null +++ b/nixos/tests/cloudcompare.nix @@ -0,0 +1,55 @@ +{ pkgs, runTest }: +let + testFile = pkgs.fetchurl { + url = "https://raw.githubusercontent.com/asmaloney/libE57Format-test-data/bbcacec05d60f923869545c5eab33d94c390d50e/self/ColouredCubeFloat.e57"; + hash = "sha256-bb95crNYvX3Qhkx4k6Sqe2GjOf1u4nxxswMfdjyXfTM="; + }; + + vmTest = runTest { + name = "cloudcompare-vm"; + meta.maintainers = with pkgs.lib.maintainers; [ + nh2 + ]; + + enableOCR = true; + + nodes.machine = + { ... }: + { + imports = [ + ./common/x11.nix + ]; + + services.xserver.enable = true; + environment.systemPackages = [ + pkgs.cloudcompare + ]; + }; + + testScript = '' + start_all() + machine.wait_for_x() + + machine.execute("CloudCompare ${testFile} >&2 &") + machine.wait_for_window("CloudCompare") + + # Wait for the file to be loaded; CloudCompare shows "loaded successfully" in its log panel at the bottom. + machine.wait_for_text("loaded successfully") + + machine.screenshot("screen.png") + ''; + }; +in +{ + vm = vmTest; + + screenshot-analysis = pkgs.callPackage ./vlm-screenshot-question.nix { + name = "cloudcompare-screenshot-analysis"; + screenshot = "${vmTest}/screen.png"; + question = '' + Look at this screenshot of a desktop application. + Answer: Does the application show a 3D point cloud viewer (CloudCompare) that has successfully loaded and is displaying a coloured point cloud? + Evidence of success: a 3D viewport with coloured points is visible AND there are no error dialogs or error messages. + ''; + }; +} diff --git a/nixos/tests/e57inspector.nix b/nixos/tests/e57inspector.nix index d421b69f2101..91c4bb90e996 100644 --- a/nixos/tests/e57inspector.nix +++ b/nixos/tests/e57inspector.nix @@ -1,38 +1,53 @@ -{ pkgs, ... }: -{ - name = "e57inspector"; - meta.maintainers = with pkgs.lib.maintainers; [ - nh2 - chpatrick - ]; +{ pkgs, runTest }: +let + testFile = pkgs.fetchurl { + url = "https://raw.githubusercontent.com/asmaloney/libE57Format-test-data/bbcacec05d60f923869545c5eab33d94c390d50e/self/ColouredCubeFloat.e57"; + hash = "sha256-bb95crNYvX3Qhkx4k6Sqe2GjOf1u4nxxswMfdjyXfTM="; + }; - nodes.machine = - { ... }: - { - imports = [ - ./common/x11.nix - ]; + vmTest = runTest { + name = "e57inspector-vm"; + meta.maintainers = with pkgs.lib.maintainers; [ + nh2 + chpatrick + ]; - services.xserver.enable = true; - environment.systemPackages = [ - pkgs.e57inspector - pkgs.xdotool - ]; - }; + nodes.machine = + { ... }: + { + imports = [ + ./common/x11.nix + ]; - testScript = - let - testFile = pkgs.fetchurl { - url = "https://raw.githubusercontent.com/asmaloney/libE57Format-test-data/bbcacec05d60f923869545c5eab33d94c390d50e/self/ColouredCubeFloat.e57"; - hash = "sha256-bb95crNYvX3Qhkx4k6Sqe2GjOf1u4nxxswMfdjyXfTM="; + services.xserver.enable = true; + environment.systemPackages = [ + pkgs.e57inspector + ]; }; - in - '' + enableOCR = true; + + testScript = '' start_all() machine.wait_for_x() machine.execute("e57inspector ${testFile} >&2 &") - machine.wait_until_succeeds("xdotool search --pid $(pidof .e57inspector-wrapped)") - machine.screenshot("screen") + machine.wait_for_text("File") # menu visible + + machine.screenshot("screen.png") ''; + }; +in +{ + vm = vmTest; + + screenshot-analysis = pkgs.callPackage ./vlm-screenshot-question.nix { + name = "e57inspector-screenshot-analysis"; + screenshot = "${vmTest}/screen.png"; + question = '' + Look at this screenshot of a desktop application. + Answer: Does the application show that a file was loaded into it successfully? + For this, only scans matter, as there are no images in the file. + The inspector on the left should show child elements below 'Data 3D'. + ''; + }; } diff --git a/nixos/tests/vlm-screenshot-question.nix b/nixos/tests/vlm-screenshot-question.nix new file mode 100644 index 000000000000..9009f4a21d84 --- /dev/null +++ b/nixos/tests/vlm-screenshot-question.nix @@ -0,0 +1,118 @@ +# Reusable VLM screenshot analysis derivation. +# +# Similar to `wait_for_text()` in NixOS VM tests. +# +# Runs a VLM (Vision Language Model) on a screenshot and asserts that the +# model's answer to a yes/no question ends with "YES". +# +# This is useful to automatically test software that is otherwise +# hard to test, e.g. "does this 3D program render the bunny correctly?". +# It is especially useful to judge screenshots made in NixOS VM tests. +{ + lib, + writers, + fetchurl, + llama-cpp, + runCommand, + # VLM defaults, chosen to pick a model smart enough to be useful + # for screenshot analysis, but small enough to not consume too much RAM + # or be too slow for CI. + model ? ( + fetchurl { + url = "https://huggingface.co/unsloth/gemma-4-E2B-it-GGUF/resolve/90f9618340396838ee7ff5b0ba2da27da62953d3/gemma-4-E2B-it-Q4_0.gguf"; + hash = "sha256-nEwdSKRi9/iDsomErE9C02bJxXNDTqtoVT4POL9+tQw="; + } + ), + mmproj ? ( + fetchurl { + url = "https://huggingface.co/unsloth/gemma-4-E2B-it-GGUF/resolve/90f9618340396838ee7ff5b0ba2da27da62953d3/mmproj-F16.gguf"; + hash = "sha256-FAvo14SXQfiMUHV9UpuENz7o4nBSzCI2hVtTf0qCFfo="; + } + ), + # User-provided arguments: + name, + screenshot, + question, +}: +let + + analysisScript = + writers.writePython3 "${name}-script" { flakeIgnore = [ "E501" ]; } # allow long lines + '' + import os + import re + import subprocess + import time + + out = os.environ["out"] + screenshot = "${screenshot}" + # Using JSON here even permits preserving multi-line ASCII art questions and so on. + question = ${builtins.toJSON question} + + # Build the full prompt with output markers for reliable extraction. + prompt = ( + "Start your output with [output-start]." + f" {question}" + " Explain what you see, and your judgment." + " Then answer that question with exactly YES or NO, followed by [output-end]." + ) + + vlm_start = time.time() + result = subprocess.run( + [ + "${lib.getExe llama-cpp}", + "--single-turn", "--no-display-prompt", "--log-verbosity", "0", "--jinja", + "--simple-io", # disables the spinner whose backspace chars would corrupt captured output + "--reasoning", "off", "--temp", "0", + "--threads", "1", # for determinism + "--n-gpu-layers", "0", # force CPU-only (results on GPUs might be different and nondeterministic, see https://github.com/ggml-org/llama.cpp/pull/16016#issuecomment-3293505238) + "--model", "${model}", + "--mmproj", "${mmproj}", + "--image", screenshot, + "-p", prompt, + ], + capture_output=True, + text=True, + # `OMP_NUM_THREADS=1` prevents OpenMP from spawning extra threads in the BLAS backend + # (OpenBLAS), which causes nondeterminism with `--image`; without `--image`, `--threads 1` + # alone is already deterministic (BLAS is not used for short text prompts). + # Relevant code: https://github.com/ggml-org/llama.cpp/blob/80afa33aadcc4f71212b17e5e52904491c76b63e/ggml/src/ggml-blas/ggml-blas.cpp#L30-L148 + # PR to fix it in OpenBLAS: https://github.com/OpenMathLib/OpenBLAS/pull/5808 + env={**os.environ, "OMP_NUM_THREADS": "1"}, + ) + vlm_elapsed = time.time() - vlm_start + output = result.stdout + print(f"VLM inference took {vlm_elapsed:.1f}s") + print(f"VLM raw output: {repr(output)}") + if result.returncode != 0: + print(f"VLM stderr: {result.stderr}") + assert result.returncode == 0, f"llama-cli failed with exit code {result.returncode}" + + print() + + # Post-process: extract the answer between `[output-start]` and `[output-end]` markers. + # This is needed because llama-cli prints UI noise (banner, + # spinner, stats) to stdout alongside the model's response. + # TODO: Replace with `--quiet` once https://github.com/ggml-org/llama.cpp/pull/22848 is merged; + # then also remove the markers from the prompt and the extraction below. + matches = re.findall(r"\[output-start\](.*?)\[output-end\]", output, re.DOTALL) + assert matches, ( + f"VLM output did not contain [output-start]...[output-end] markers." + f" Raw output: {repr(output)}" + ) + answer = matches[-1].strip() # use last match (first may be prompt echo) + print("VLM answer:") + print(answer) + assert answer.upper().endswith("YES"), ( + f"VLM did not confirm expected answer. Answer: {answer}" + ) + + os.makedirs(out, exist_ok=True) + with open(os.path.join(out, "vlm-answer.txt"), "w") as f: + f.write(answer + "\n") + os.symlink(screenshot, os.path.join(out, "screen.png")) + ''; +in +runCommand name { } '' + ${analysisScript} +'' diff --git a/pkgs/by-name/cl/cloudcompare/package.nix b/pkgs/by-name/cl/cloudcompare/package.nix index a736269cf28a..28b69ee1ecee 100644 --- a/pkgs/by-name/cl/cloudcompare/package.nix +++ b/pkgs/by-name/cl/cloudcompare/package.nix @@ -16,6 +16,7 @@ mpfr, pcl, libsForQt5, + nixosTests, onetbb, xercesc, wrapGAppsHook3, @@ -162,6 +163,8 @@ stdenv.mkDerivation (finalAttrs: { }) ]; + passthru.tests = nixosTests.cloudcompare; + meta = { description = "3D point cloud and mesh processing software"; homepage = "https://cloudcompare.org"; diff --git a/pkgs/by-name/e5/e57inspector/package.nix b/pkgs/by-name/e5/e57inspector/package.nix index 895ced7a183c..3b9305bc3af6 100644 --- a/pkgs/by-name/e5/e57inspector/package.nix +++ b/pkgs/by-name/e5/e57inspector/package.nix @@ -62,9 +62,7 @@ stdenv.mkDerivation (finalAttrs: { runHook postInstall ''; - passthru.tests = { - e57inspector = nixosTests.e57inspector; - }; + passthru.tests = nixosTests.e57inspector; meta = { description = "Cross-platform E57 file viewer to list and view stored point clouds, images and metadata";