From ff77d3c409924c873cfa8dd25d8d99f7acb5467a Mon Sep 17 00:00:00 2001 From: Enno Richter Date: Thu, 12 Oct 2023 08:57:30 +0200 Subject: [PATCH 1/2] llama-cpp: init at 1469 --- pkgs/by-name/ll/llama-cpp/package.nix | 130 ++++++++++++++++++++++++++ 1 file changed, 130 insertions(+) create mode 100644 pkgs/by-name/ll/llama-cpp/package.nix diff --git a/pkgs/by-name/ll/llama-cpp/package.nix b/pkgs/by-name/ll/llama-cpp/package.nix new file mode 100644 index 000000000000..1c96f5d83a60 --- /dev/null +++ b/pkgs/by-name/ll/llama-cpp/package.nix @@ -0,0 +1,130 @@ +{ lib +, cmake +, darwin +, fetchFromGitHub +, nix-update-script +, stdenv +, symlinkJoin + +, config +, cudaSupport ? config.cudaSupport +, cudaPackages ? { } + +, rocmSupport ? false +, rocmPackages ? { } + +, openclSupport ? false +, clblast + +, openblasSupport ? true +, openblas +, pkg-config +}: + +let + cudatoolkit_joined = symlinkJoin { + name = "${cudaPackages.cudatoolkit.name}-merged"; + paths = [ + cudaPackages.cudatoolkit.lib + cudaPackages.cudatoolkit.out + ] ++ lib.optionals (lib.versionOlder cudaPackages.cudatoolkit.version "11") [ + # for some reason some of the required libs are in the targets/x86_64-linux + # directory; not sure why but this works around it + "${cudaPackages.cudatoolkit}/targets/${stdenv.system}" + ]; + }; + metalSupport = stdenv.isDarwin && stdenv.isAarch64; +in +stdenv.mkDerivation (finalAttrs: { + pname = "llama-cpp"; + version = "1469"; + + src = fetchFromGitHub { + owner = "ggerganov"; + repo = "llama.cpp"; + rev = "refs/tags/b${finalAttrs.version}"; + hash = "sha256-budBvpX2SnKekGTWHomvhW+4grB8EPd9OJbufNynHsc="; + }; + + postPatch = '' + substituteInPlace ./ggml-metal.m \ + --replace '[bundle pathForResource:@"ggml-metal" ofType:@"metal"];' "@\"$out/bin/ggml-metal.metal\";" + ''; + + nativeBuildInputs = [ cmake ] ++ lib.optionals openblasSupport [ pkg-config ]; + + buildInputs = lib.optionals metalSupport + (with darwin.apple_sdk.frameworks; [ + Accelerate + CoreGraphics + CoreVideo + Foundation + MetalKit + ]) + ++ lib.optionals cudaSupport [ + cudatoolkit_joined + ] ++ lib.optionals rocmSupport [ + rocmPackages.clr + rocmPackages.hipblas + rocmPackages.rocblas + ] ++ lib.optionals openclSupport [ + clblast + ] ++ lib.optionals openblasSupport [ + openblas + ]; + + cmakeFlags = [ + "-DLLAMA_NATIVE=OFF" + "-DLLAMA_BUILD_SERVER=ON" + ] + ++ lib.optionals metalSupport [ + "-DCMAKE_C_FLAGS=-D__ARM_FEATURE_DOTPROD=1" + "-DLLAMA_METAL=ON" + ] + ++ lib.optionals cudaSupport [ + "-DLLAMA_CUBLAS=ON" + ] + ++ lib.optionals rocmSupport [ + "-DLLAMA_HIPBLAS=1" + "-DCMAKE_C_COMPILER=hipcc" + "-DCMAKE_CXX_COMPILER=hipcc" + "-DCMAKE_POSITION_INDEPENDENT_CODE=ON" + ] + ++ lib.optionals openclSupport [ + "-DLLAMA_CLBLAST=ON" + ] + ++ lib.optionals openblasSupport [ + "-DLLAMA_BLAS=ON" + "-DLLAMA_BLAS_VENDOR=OpenBLAS" + ]; + + installPhase = '' + runHook preInstall + + mkdir -p $out/bin + + for f in bin/*; do + test -x "$f" || continue + cp "$f" $out/bin/llama-cpp-"$(basename "$f")" + done + + ${lib.optionalString metalSupport "cp ./bin/ggml-metal.metal $out/bin/ggml-metal.metal"} + + runHook postInstall + ''; + + passthru.updateScript = nix-update-script { + attrPath = "llama-cpp"; + extraArgs = [ "--version-regex" "b(.*)" ]; + }; + + meta = with lib; { + description = "Port of Facebook's LLaMA model in C/C++"; + homepage = "https://github.com/ggerganov/llama.cpp/"; + license = licenses.mit; + mainProgram = "llama-cpp-main"; + maintainers = with maintainers; [ dit7ya elohmeier ]; + broken = stdenv.isDarwin && stdenv.isx86_64; + platforms = platforms.unix; + }; +}) From ea3a7c56fd076111f0f1afe5b36a9f79c5660c8c Mon Sep 17 00:00:00 2001 From: Enno Richter Date: Thu, 26 Oct 2023 09:41:05 +0200 Subject: [PATCH 2/2] ollama: 0.0.17 -> 0.1.7, use llama-cpp --- pkgs/tools/misc/ollama/default.nix | 37 +++++++++++++------ pkgs/tools/misc/ollama/disable-gqa.patch | 15 ++++++++ .../tools/misc/ollama/set-llamacpp-path.patch | 23 ++++++++++++ 3 files changed, 64 insertions(+), 11 deletions(-) create mode 100644 pkgs/tools/misc/ollama/disable-gqa.patch create mode 100644 pkgs/tools/misc/ollama/set-llamacpp-path.patch diff --git a/pkgs/tools/misc/ollama/default.nix b/pkgs/tools/misc/ollama/default.nix index be1864024888..30be00d72a15 100644 --- a/pkgs/tools/misc/ollama/default.nix +++ b/pkgs/tools/misc/ollama/default.nix @@ -1,35 +1,50 @@ { lib , buildGoModule , fetchFromGitHub +, llama-cpp , stdenv -, darwin }: buildGoModule rec { pname = "ollama"; - version = "0.0.17"; + version = "0.1.7"; src = fetchFromGitHub { owner = "jmorganca"; repo = "ollama"; rev = "v${version}"; - hash = "sha256-idsFcjsRD1zPmG742gnYQJcgSWDA2DLMHksCFNe2GiY="; + hash = "sha256-rzcuRU2qcYTMo/GxiSHwJYnvA9samfWlztMEhOGzbRg="; }; - buildInputs = lib.optionals stdenv.isDarwin (with darwin.apple_sdk_11_0.frameworks; [ - Accelerate - MetalPerformanceShaders - MetalKit - ]); + patches = [ + # disable passing the deprecated gqa flag to llama-cpp-server + # see https://github.com/ggerganov/llama.cpp/issues/2975 + ./disable-gqa.patch - vendorHash = "sha256-IgEf/WOc1eNGCif1fViIFxbgZAd6mHBqfxcaqH/WvGg="; + # replace the call to the bundled llama-cpp-server with the one in the llama-cpp package + ./set-llamacpp-path.patch + ]; - ldflags = [ "-s" "-w" ]; + postPatch = '' + substituteInPlace llm/llama.go \ + --subst-var-by llamaCppServer "${llama-cpp}/bin/llama-cpp-server" + ''; + + vendorHash = "sha256-Qt5QVqRkwK61BJPVhFWtox6b9E8BpAIseNB0yhh+/90="; + + ldflags = [ + "-s" + "-w" + "-X=github.com/jmorganca/ollama/version.Version=${version}" + "-X=github.com/jmorganca/ollama/server.mode=release" + ]; meta = with lib; { description = "Get up and running with large language models locally"; homepage = "https://github.com/jmorganca/ollama"; license = licenses.mit; - maintainers = with maintainers; [ dit7ya ]; + mainProgram = "ollama"; + maintainers = with maintainers; [ dit7ya elohmeier ]; + platforms = platforms.unix; }; } diff --git a/pkgs/tools/misc/ollama/disable-gqa.patch b/pkgs/tools/misc/ollama/disable-gqa.patch new file mode 100644 index 000000000000..b54440cd3d53 --- /dev/null +++ b/pkgs/tools/misc/ollama/disable-gqa.patch @@ -0,0 +1,15 @@ +diff --git a/llm/llama.go b/llm/llama.go +index 0b460e9..b79e04a 100644 +--- a/llm/llama.go ++++ b/llm/llama.go +@@ -299,10 +299,6 @@ func newLlama(model string, adapters []string, runners []ModelRunner, numLayers + params = append(params, "--n-gpu-layers", fmt.Sprintf("%d", numGPU)) + } + +- if opts.NumGQA > 0 { +- params = append(params, "--gqa", fmt.Sprintf("%d", opts.NumGQA)) +- } +- + if len(adapters) > 0 { + // TODO: applying multiple adapters is not supported by the llama.cpp server yet + params = append(params, "--lora", adapters[0]) diff --git a/pkgs/tools/misc/ollama/set-llamacpp-path.patch b/pkgs/tools/misc/ollama/set-llamacpp-path.patch new file mode 100644 index 000000000000..e90e552bab45 --- /dev/null +++ b/pkgs/tools/misc/ollama/set-llamacpp-path.patch @@ -0,0 +1,23 @@ +diff --git a/llm/llama.go b/llm/llama.go +index f23d5d8..6563550 100644 +--- a/llm/llama.go ++++ b/llm/llama.go +@@ -25,7 +25,6 @@ import ( + "github.com/jmorganca/ollama/api" + ) + +-//go:embed llama.cpp/*/build/*/bin/* + var llamaCppEmbed embed.FS + + type ModelRunner struct { +@@ -33,6 +32,10 @@ type ModelRunner struct { + } + + func chooseRunners(workDir, runnerType string) []ModelRunner { ++ return []ModelRunner{ ++ {Path: "@llamaCppServer@"}, ++ } ++ + buildPath := path.Join("llama.cpp", runnerType, "build") + var runners []string +