Merge pull request #257760 from elohmeier/ollama

ollama: 0.0.17 -> 0.1.7

pkgs/by-name/ll/llama-cpp/package.nix

@@ -0,0 +1,130 @@
+{ lib
+, cmake
+, darwin
+, fetchFromGitHub
+, nix-update-script
+, stdenv
+, symlinkJoin
+
+, config
+, cudaSupport ? config.cudaSupport
+, cudaPackages ? { }
+
+, rocmSupport ? false
+, rocmPackages ? { }
+
+, openclSupport ? false
+, clblast
+
+, openblasSupport ? true
+, openblas
+, pkg-config
+}:
+
+let
+  cudatoolkit_joined = symlinkJoin {
+    name = "${cudaPackages.cudatoolkit.name}-merged";
+    paths = [
+      cudaPackages.cudatoolkit.lib
+      cudaPackages.cudatoolkit.out
+    ] ++ lib.optionals (lib.versionOlder cudaPackages.cudatoolkit.version "11") [
+      # for some reason some of the required libs are in the targets/x86_64-linux
+      # directory; not sure why but this works around it
+      "${cudaPackages.cudatoolkit}/targets/${stdenv.system}"
+    ];
+  };
+  metalSupport = stdenv.isDarwin && stdenv.isAarch64;
+in
+stdenv.mkDerivation (finalAttrs: {
+  pname = "llama-cpp";
+  version = "1469";
+
+  src = fetchFromGitHub {
+    owner = "ggerganov";
+    repo = "llama.cpp";
+    rev = "refs/tags/b${finalAttrs.version}";
+    hash = "sha256-budBvpX2SnKekGTWHomvhW+4grB8EPd9OJbufNynHsc=";
+  };
+
+  postPatch = ''
+    substituteInPlace ./ggml-metal.m \
+      --replace '[bundle pathForResource:@"ggml-metal" ofType:@"metal"];' "@\"$out/bin/ggml-metal.metal\";"
+  '';
+
+  nativeBuildInputs = [ cmake ] ++ lib.optionals openblasSupport [ pkg-config ];
+
+  buildInputs = lib.optionals metalSupport
+    (with darwin.apple_sdk.frameworks; [
+      Accelerate
+      CoreGraphics
+      CoreVideo
+      Foundation
+      MetalKit
+    ])
+  ++ lib.optionals cudaSupport [
+    cudatoolkit_joined
+  ] ++ lib.optionals rocmSupport [
+    rocmPackages.clr
+    rocmPackages.hipblas
+    rocmPackages.rocblas
+  ] ++ lib.optionals openclSupport [
+    clblast
+  ] ++ lib.optionals openblasSupport [
+    openblas
+  ];
+
+  cmakeFlags = [
+    "-DLLAMA_NATIVE=OFF"
+    "-DLLAMA_BUILD_SERVER=ON"
+  ]
+  ++ lib.optionals metalSupport [
+    "-DCMAKE_C_FLAGS=-D__ARM_FEATURE_DOTPROD=1"
+    "-DLLAMA_METAL=ON"
+  ]
+  ++ lib.optionals cudaSupport [
+    "-DLLAMA_CUBLAS=ON"
+  ]
+  ++ lib.optionals rocmSupport [
+    "-DLLAMA_HIPBLAS=1"
+    "-DCMAKE_C_COMPILER=hipcc"
+    "-DCMAKE_CXX_COMPILER=hipcc"
+    "-DCMAKE_POSITION_INDEPENDENT_CODE=ON"
+  ]
+  ++ lib.optionals openclSupport [
+    "-DLLAMA_CLBLAST=ON"
+  ]
+  ++ lib.optionals openblasSupport [
+    "-DLLAMA_BLAS=ON"
+    "-DLLAMA_BLAS_VENDOR=OpenBLAS"
+  ];
+
+  installPhase = ''
+    runHook preInstall
+
+    mkdir -p $out/bin
+
+    for f in bin/*; do
+      test -x "$f" || continue
+      cp "$f" $out/bin/llama-cpp-"$(basename "$f")"
+    done
+
+    ${lib.optionalString metalSupport "cp ./bin/ggml-metal.metal $out/bin/ggml-metal.metal"}
+
+    runHook postInstall
+  '';
+
+  passthru.updateScript = nix-update-script {
+    attrPath = "llama-cpp";
+    extraArgs = [ "--version-regex" "b(.*)" ];
+  };
+
+  meta = with lib; {
+    description = "Port of Facebook's LLaMA model in C/C++";
+    homepage = "https://github.com/ggerganov/llama.cpp/";
+    license = licenses.mit;
+    mainProgram = "llama-cpp-main";
+    maintainers = with maintainers; [ dit7ya elohmeier ];
+    broken = stdenv.isDarwin && stdenv.isx86_64;
+    platforms = platforms.unix;
+  };
+})

pkgs/tools/misc/ollama/default.nix

@@ -1,35 +1,50 @@
 { lib
 , buildGoModule
 , fetchFromGitHub
+, llama-cpp
 , stdenv
-, darwin
 }:
 
 buildGoModule rec {
   pname = "ollama";
-  version = "0.0.17";
+  version = "0.1.7";
 
   src = fetchFromGitHub {
     owner = "jmorganca";
     repo = "ollama";
     rev = "v${version}";
-    hash = "sha256-idsFcjsRD1zPmG742gnYQJcgSWDA2DLMHksCFNe2GiY=";
+    hash = "sha256-rzcuRU2qcYTMo/GxiSHwJYnvA9samfWlztMEhOGzbRg=";
   };
 
-  buildInputs = lib.optionals stdenv.isDarwin (with darwin.apple_sdk_11_0.frameworks; [
-    Accelerate
-    MetalPerformanceShaders
-    MetalKit
-  ]);
+  patches = [
+    # disable passing the deprecated gqa flag to llama-cpp-server
+    # see https://github.com/ggerganov/llama.cpp/issues/2975
+    ./disable-gqa.patch
 
-  vendorHash = "sha256-IgEf/WOc1eNGCif1fViIFxbgZAd6mHBqfxcaqH/WvGg=";
+    # replace the call to the bundled llama-cpp-server with the one in the llama-cpp package
+    ./set-llamacpp-path.patch
+  ];
 
-  ldflags = [ "-s" "-w" ];
+  postPatch = ''
+    substituteInPlace llm/llama.go \
+      --subst-var-by llamaCppServer "${llama-cpp}/bin/llama-cpp-server"
+  '';
+
+  vendorHash = "sha256-Qt5QVqRkwK61BJPVhFWtox6b9E8BpAIseNB0yhh+/90=";
+
+  ldflags = [
+    "-s"
+    "-w"
+    "-X=github.com/jmorganca/ollama/version.Version=${version}"
+    "-X=github.com/jmorganca/ollama/server.mode=release"
+  ];
 
   meta = with lib; {
     description = "Get up and running with large language models locally";
     homepage = "https://github.com/jmorganca/ollama";
     license = licenses.mit;
-    maintainers = with maintainers; [ dit7ya ];
+    mainProgram = "ollama";
+    maintainers = with maintainers; [ dit7ya elohmeier ];
+    platforms = platforms.unix;
   };
 }

pkgs/tools/misc/ollama/disable-gqa.patch

@@ -0,0 +1,15 @@
+diff --git a/llm/llama.go b/llm/llama.go
+index 0b460e9..b79e04a 100644
+--- a/llm/llama.go
++++ b/llm/llama.go
+@@ -299,10 +299,6 @@ func newLlama(model string, adapters []string, runners []ModelRunner, numLayers
+ 		params = append(params, "--n-gpu-layers", fmt.Sprintf("%d", numGPU))
+ 	}
+ 
+-	if opts.NumGQA > 0 {
+-		params = append(params, "--gqa", fmt.Sprintf("%d", opts.NumGQA))
+-	}
+-
+ 	if len(adapters) > 0 {
+ 		// TODO: applying multiple adapters is not supported by the llama.cpp server yet
+ 		params = append(params, "--lora", adapters[0])

pkgs/tools/misc/ollama/set-llamacpp-path.patch

@@ -0,0 +1,23 @@
+diff --git a/llm/llama.go b/llm/llama.go
+index f23d5d8..6563550 100644
+--- a/llm/llama.go
++++ b/llm/llama.go
+@@ -25,7 +25,6 @@ import (
+ 	"github.com/jmorganca/ollama/api"
+ )
+ 
+-//go:embed llama.cpp/*/build/*/bin/*
+ var llamaCppEmbed embed.FS
+ 
+ type ModelRunner struct {
+@@ -33,6 +32,10 @@ type ModelRunner struct {
+ }
+ 
+ func chooseRunners(workDir, runnerType string) []ModelRunner {
++	return []ModelRunner{
++		{Path: "@llamaCppServer@"},
++	}
++
+ 	buildPath := path.Join("llama.cpp", runnerType, "build")
+ 	var runners []string
+