From d33227928db61fd8513d523830966b2796e16ae1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Niklas=20Hamb=C3=BCchen?= Date: Fri, 8 May 2026 23:50:52 +0200 Subject: [PATCH] llama-cpp: Use runtime instruction dispatch for massive speedups Before I added this, I made below benchmark to figure if it's worth it: llama-cpp with AVX-512 for faster inference and --quiet patch. This is MUCH faster than nixpkgs's default x86_64 build, e.g. reducing the time for a 1024x768 screenshot (on an AMD Ryzen 7 7700X 8-Core, `llama-cpp` version 8983, with `gemma-4-E2B-it-Q4_0.gguf` + `mmproj-gemma-4-E2B-it-F16.gguf`): default AVX2 AVX-512 speedup (AVX2 / AVX-512) 1 thread: CPU 360s 30s 27s 12x / 13x wall 293s 22s 19s 13x / 15x 8 threads: CPU 405s 31s 28s 13x / 14x wall 51s 6s 6s 8x / 9x where "AVX2" refers to: -DGGML_AVX=ON -DGGML_AVX2=ON -DGGML_FMA=ON -DGGML_F16C=ON -DGGML_AVX512=ON and "AVX512" refers to additionally: -DGGML_AVX512_VBMI=ON -DGGML_AVX512_VNNI=ON -DGGML_AVX512_BF16=ON AVX2 is available for CPUs from 2013 (Intel) / 2015 (AMD), while AVX512 is available for CPUs from 2017 (Intel) / 2022 (AMD). --- pkgs/by-name/ll/llama-cpp/package.nix | 17 +++++++++++++++-- 1 file changed, 15 insertions(+), 2 deletions(-) diff --git a/pkgs/by-name/ll/llama-cpp/package.nix b/pkgs/by-name/ll/llama-cpp/package.nix index 57dc752361c1..a1e57c030fca 100644 --- a/pkgs/by-name/ll/llama-cpp/package.nix +++ b/pkgs/by-name/ll/llama-cpp/package.nix @@ -15,6 +15,8 @@ rocmPackages ? { }, rocmGpuTargets ? rocmPackages.clr.localGpuTargets or rocmPackages.clr.gpuTargets, + cpuArchDynamicDispatch ? true, + openclSupport ? false, clblast, @@ -139,8 +141,7 @@ effectiveStdenv.mkDerivation (finalAttrs: { ''; cmakeFlags = [ - # -march=native is non-deterministic; override with platform-specific flags if needed - (cmakeBool "GGML_NATIVE" false) + (cmakeBool "GGML_NATIVE" false) # -march=native would make builds non-deterministic (cmakeBool "LLAMA_BUILD_EXAMPLES" false) (cmakeBool "LLAMA_BUILD_SERVER" true) (cmakeBool "LLAMA_BUILD_TESTS" (finalAttrs.finalPackage.doCheck or false)) @@ -155,6 +156,18 @@ effectiveStdenv.mkDerivation (finalAttrs: { (cmakeBool "GGML_VULKAN" vulkanSupport) (cmakeFeature "LLAMA_BUILD_NUMBER" finalAttrs.version) ] + ++ optionals cpuArchDynamicDispatch [ + # Build all CPU backend variants for runtime dynamic dispatch. + # This avoids illegal instructions on older CPUs and gives optimal performance + # on newer ones without needing separate builds. + # Enabling AVX2 can make CPU inference 13x faster compared to NixOS's x86_64 defaults. + # Note it is not a bug that the CPU variant .so files are placed in `bin/` + # (as opposed to `lib/`) alongside the executables by upstream's `CMakeLists.txt` design: + # * https://github.com/ggml-org/llama.cpp/blob/b46812de78f8fbcb6cf0154947e8633ebc78d9ac/ggml/src/CMakeLists.txt#L249-L252 + # * https://github.com/ggml-org/llama.cpp/blob/b46812de78f8fbcb6cf0154947e8633ebc78d9ac/ggml/src/ggml-backend-reg.cpp#L480-L486 + (cmakeBool "GGML_CPU_ALL_VARIANTS" true) + (cmakeBool "GGML_BACKEND_DL" true) + ] ++ optionals cudaSupport [ (cmakeFeature "CMAKE_CUDA_ARCHITECTURES" cudaPackages.flags.cmakeCudaArchitecturesString) ]