From b3a96e6a9fa0c8af951dc172bc1f816ade6b6452 Mon Sep 17 00:00:00 2001 From: abysssol Date: Tue, 17 Sep 2024 19:12:22 -0400 Subject: [PATCH 1/3] ollama: move `goBuild` environment variables to `env` --- pkgs/by-name/ol/ollama/package.nix | 209 +++++++++++++++-------------- 1 file changed, 105 insertions(+), 104 deletions(-) diff --git a/pkgs/by-name/ol/ollama/package.nix b/pkgs/by-name/ol/ollama/package.nix index c1451d42faae..694202d7f78e 100644 --- a/pkgs/by-name/ol/ollama/package.nix +++ b/pkgs/by-name/ol/ollama/package.nix @@ -114,117 +114,118 @@ let if enableCuda then buildGoModule.override { stdenv = overrideCC stdenv gcc12; } else buildGoModule; inherit (lib) licenses platforms maintainers; in -goBuild ( - (lib.optionalAttrs enableRocm { - ROCM_PATH = rocmPath; - CLBlast_DIR = "${clblast}/lib/cmake/CLBlast"; - }) - // (lib.optionalAttrs enableCuda { CUDA_LIB_DIR = "${cudaToolkit}/lib"; }) - // { - inherit - pname - version - src - vendorHash - ; +goBuild { + inherit + pname + version + src + vendorHash + ; - nativeBuildInputs = - [ cmake ] - ++ lib.optionals enableRocm [ rocmPackages.llvm.bintools ] - ++ lib.optionals enableCuda [ cudaPackages.cuda_nvcc ] - ++ lib.optionals (enableRocm || enableCuda) [ - makeWrapper - autoAddDriverRunpath - ] - ++ lib.optionals stdenv.isDarwin metalFrameworks; + env = + lib.optionalAttrs enableRocm { + ROCM_PATH = rocmPath; + CLBlast_DIR = "${clblast}/lib/cmake/CLBlast"; + } + // lib.optionalAttrs enableCuda { CUDA_LIB_DIR = "${cudaToolkit}/lib"; }; - buildInputs = - lib.optionals enableRocm (rocmLibs ++ [ libdrm ]) - ++ lib.optionals enableCuda [ - cudaToolkit - cudaPackages.cuda_cudart - cudaPackages.cuda_cccl - cudaPackages.libcublas - ] - ++ lib.optionals stdenv.isDarwin metalFrameworks; + nativeBuildInputs = + [ cmake ] + ++ lib.optionals enableRocm [ rocmPackages.llvm.bintools ] + ++ lib.optionals enableCuda [ cudaPackages.cuda_nvcc ] + ++ lib.optionals (enableRocm || enableCuda) [ + makeWrapper + autoAddDriverRunpath + ] + ++ lib.optionals stdenv.isDarwin metalFrameworks; - patches = [ - # disable uses of `git` in the `go generate` script - # ollama's build script assumes the source is a git repo, but nix removes the git directory - # this also disables necessary patches contained in `ollama/llm/patches/` - # those patches are applied in `postPatch` - ./disable-git.patch - ]; + buildInputs = + lib.optionals enableRocm (rocmLibs ++ [ libdrm ]) + ++ lib.optionals enableCuda [ + cudaToolkit + cudaPackages.cuda_cudart + cudaPackages.cuda_cccl + cudaPackages.libcublas + ] + ++ lib.optionals stdenv.isDarwin metalFrameworks; - postPatch = '' - # replace inaccurate version number with actual release version - substituteInPlace version/version.go --replace-fail 0.0.0 '${version}' + patches = [ + # disable uses of `git` in the `go generate` script + # ollama's build script assumes the source is a git repo, but nix removes the git directory + # this also disables necessary patches contained in `ollama/llm/patches/` + # those patches are applied in `postPatch` + ./disable-git.patch + ]; - # apply llama.cpp patches - for cur in llm/patches/*; do patch -p1 -d llm/llama.cpp < $cur; done + postPatch = '' + # replace inaccurate version number with actual release version + substituteInPlace version/version.go --replace-fail 0.0.0 '${version}' + + # apply llama.cpp patches + for cur in llm/patches/*; do patch -p1 -d llm/llama.cpp < $cur; done + ''; + + overrideModAttrs = ( + finalAttrs: prevAttrs: { + # don't run llama.cpp build in the module fetch phase + preBuild = ""; + } + ); + + preBuild = '' + # disable uses of `git`, since nix removes the git directory + export OLLAMA_SKIP_PATCHING=true + # build llama.cpp libraries for ollama + go generate ./... + ''; + + postFixup = + '' + # the app doesn't appear functional at the moment, so hide it + mv "$out/bin/app" "$out/bin/.ollama-app" + '' + + lib.optionalString (enableRocm || enableCuda) '' + # expose runtime libraries necessary to use the gpu + wrapProgram "$out/bin/ollama" ${wrapperArgs} ''; - overrideModAttrs = ( - finalAttrs: prevAttrs: { - # don't run llama.cpp build in the module fetch phase - preBuild = ""; - } - ); + ldflags = [ + "-s" + "-w" + "-X=github.com/ollama/ollama/version.Version=${version}" + "-X=github.com/ollama/ollama/server.mode=release" + ]; - preBuild = '' - # disable uses of `git`, since nix removes the git directory - export OLLAMA_SKIP_PATCHING=true - # build llama.cpp libraries for ollama - go generate ./... - ''; - postFixup = - '' - # the app doesn't appear functional at the moment, so hide it - mv "$out/bin/app" "$out/bin/.ollama-app" - '' - + lib.optionalString (enableRocm || enableCuda) '' - # expose runtime libraries necessary to use the gpu - wrapProgram "$out/bin/ollama" ${wrapperArgs} - ''; - - ldflags = [ - "-s" - "-w" - "-X=github.com/ollama/ollama/version.Version=${version}" - "-X=github.com/ollama/ollama/server.mode=release" - ]; - - passthru.tests = - { - inherit ollama; - version = testers.testVersion { - inherit version; - package = ollama; - }; - } - // lib.optionalAttrs stdenv.isLinux { - inherit ollama-rocm ollama-cuda; - service = nixosTests.ollama; - service-cuda = nixosTests.ollama-cuda; - service-rocm = nixosTests.ollama-rocm; + passthru.tests = + { + inherit ollama; + version = testers.testVersion { + inherit version; + package = ollama; }; - - meta = { - description = - "Get up and running with large language models locally" - + lib.optionalString rocmRequested ", using ROCm for AMD GPU acceleration" - + lib.optionalString cudaRequested ", using CUDA for NVIDIA GPU acceleration"; - homepage = "https://github.com/ollama/ollama"; - changelog = "https://github.com/ollama/ollama/releases/tag/v${version}"; - license = licenses.mit; - platforms = if (rocmRequested || cudaRequested) then platforms.linux else platforms.unix; - mainProgram = "ollama"; - maintainers = with maintainers; [ - abysssol - dit7ya - elohmeier - roydubnium - ]; + } + // lib.optionalAttrs stdenv.isLinux { + inherit ollama-rocm ollama-cuda; + service = nixosTests.ollama; + service-cuda = nixosTests.ollama-cuda; + service-rocm = nixosTests.ollama-rocm; }; - } -) + + meta = { + description = + "Get up and running with large language models locally" + + lib.optionalString rocmRequested ", using ROCm for AMD GPU acceleration" + + lib.optionalString cudaRequested ", using CUDA for NVIDIA GPU acceleration"; + homepage = "https://github.com/ollama/ollama"; + changelog = "https://github.com/ollama/ollama/releases/tag/v${version}"; + license = licenses.mit; + platforms = if (rocmRequested || cudaRequested) then platforms.linux else platforms.unix; + mainProgram = "ollama"; + maintainers = with maintainers; [ + abysssol + dit7ya + elohmeier + roydubnium + ]; + }; +} From d43a6262139b6342ebf2cb6fe2819037613bc329 Mon Sep 17 00:00:00 2001 From: abysssol Date: Tue, 17 Sep 2024 19:23:05 -0400 Subject: [PATCH 2/3] ollama: reformat bash where patches are applied --- pkgs/by-name/ol/ollama/package.nix | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pkgs/by-name/ol/ollama/package.nix b/pkgs/by-name/ol/ollama/package.nix index 694202d7f78e..0bd1defc3b72 100644 --- a/pkgs/by-name/ol/ollama/package.nix +++ b/pkgs/by-name/ol/ollama/package.nix @@ -161,8 +161,10 @@ goBuild { # replace inaccurate version number with actual release version substituteInPlace version/version.go --replace-fail 0.0.0 '${version}' - # apply llama.cpp patches - for cur in llm/patches/*; do patch -p1 -d llm/llama.cpp < $cur; done + # apply ollama's patches to `llama.cpp` submodule + for diff in llm/patches/*; do + patch -p1 -d llm/llama.cpp < $diff + done ''; overrideModAttrs = ( From d6f49d6068228a353f2a846c7fdd0dc249891c20 Mon Sep 17 00:00:00 2001 From: abysssol Date: Tue, 17 Sep 2024 17:47:43 -0400 Subject: [PATCH 3/3] ollama: add cuda libs to wrapper's `LD_LIBRARY_PATH` --- pkgs/by-name/ol/ollama/package.nix | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/pkgs/by-name/ol/ollama/package.nix b/pkgs/by-name/ol/ollama/package.nix index 0bd1defc3b72..d11e06833efc 100644 --- a/pkgs/by-name/ol/ollama/package.nix +++ b/pkgs/by-name/ol/ollama/package.nix @@ -79,13 +79,16 @@ let paths = rocmLibs ++ [ rocmClang ]; }; + cudaLibs = [ + cudaPackages.cuda_cudart + cudaPackages.libcublas + cudaPackages.cuda_cccl + ]; cudaToolkit = buildEnv { name = "cuda-merged"; - paths = [ - (lib.getBin (cudaPackages.cuda_nvcc.__spliced.buildHost or cudaPackages.cuda_nvcc)) - (lib.getLib cudaPackages.cuda_cudart) + paths = map lib.getLib cudaLibs ++ [ (lib.getOutput "static" cudaPackages.cuda_cudart) - (lib.getLib cudaPackages.libcublas) + (lib.getBin (cudaPackages.cuda_nvcc.__spliced.buildHost or cudaPackages.cuda_nvcc)) ]; }; @@ -107,6 +110,9 @@ let ++ lib.optionals enableRocm [ "--suffix LD_LIBRARY_PATH : '${rocmPath}/lib'" "--set-default HIP_PATH '${rocmPath}'" + ] + ++ lib.optionals enableCuda [ + "--suffix LD_LIBRARY_PATH : '${lib.makeLibraryPath (map lib.getLib cudaLibs)}'" ]; wrapperArgs = builtins.concatStringsSep " " wrapperOptions; @@ -141,12 +147,7 @@ goBuild { buildInputs = lib.optionals enableRocm (rocmLibs ++ [ libdrm ]) - ++ lib.optionals enableCuda [ - cudaToolkit - cudaPackages.cuda_cudart - cudaPackages.cuda_cccl - cudaPackages.libcublas - ] + ++ lib.optionals enableCuda cudaLibs ++ lib.optionals stdenv.isDarwin metalFrameworks; patches = [