From d8bb8a6417b988e416074b0f338a9804deb273ae Mon Sep 17 00:00:00 2001
From: Alex Martens <alex@thinglab.org>
Date: Mon, 1 Jan 2024 19:31:23 -0800
Subject: [PATCH] nixos/llama-cpp: init

---
 nixos/modules/services/misc/llama-cpp.nix | 111 ++++++++++++++++++++++
 1 file changed, 111 insertions(+)
 create mode 100644 nixos/modules/services/misc/llama-cpp.nix

diff --git a/nixos/modules/services/misc/llama-cpp.nix b/nixos/modules/services/misc/llama-cpp.nix
new file mode 100644
index 000000000000..4d76456fb2fd
--- /dev/null
+++ b/nixos/modules/services/misc/llama-cpp.nix
@@ -0,0 +1,111 @@
+{ config, lib, pkgs, utils, ... }:
+
+let
+  cfg = config.services.llama-cpp;
+in {
+
+  options = {
+
+    services.llama-cpp = {
+      enable = lib.mkEnableOption "LLaMA C++ server";
+
+      package = lib.mkPackageOption pkgs "llama-cpp" { };
+
+      model = lib.mkOption {
+        type = lib.types.path;
+        example = "/models/mistral-instruct-7b/ggml-model-q4_0.gguf";
+        description = "Model path.";
+      };
+
+      extraFlags = lib.mkOption {
+        type = lib.types.listOf lib.types.str;
+        description = "Extra flags passed to llama-cpp-server.";
+        example = ["-c" "4096" "-ngl" "32" "--numa"];
+        default = [];
+      };
+
+      host = lib.mkOption {
+        type = lib.types.str;
+        default = "127.0.0.1";
+        example = "0.0.0.0";
+        description = "IP address the LLaMA C++ server listens on.";
+      };
+
+      port = lib.mkOption {
+        type = lib.types.port;
+        default = 8080;
+        description = "Listen port for LLaMA C++ server.";
+      };
+
+      openFirewall = lib.mkOption {
+        type = lib.types.bool;
+        default = false;
+        description = "Open ports in the firewall for LLaMA C++ server.";
+      };
+    };
+
+  };
+
+  config = lib.mkIf cfg.enable {
+
+    systemd.services.llama-cpp = {
+      description = "LLaMA C++ server";
+      after = ["network.target"];
+      wantedBy = ["multi-user.target"];
+
+      serviceConfig = {
+        Type = "idle";
+        KillSignal = "SIGINT";
+        ExecStart = "${cfg.package}/bin/llama-cpp-server --log-disable --host ${cfg.host} --port ${builtins.toString cfg.port} -m ${cfg.model} ${utils.escapeSystemdExecArgs cfg.extraFlags}";
+        Restart = "on-failure";
+        RestartSec = 300;
+
+        # for GPU acceleration
+        PrivateDevices = false;
+
+        # hardening
+        DynamicUser = true;
+        CapabilityBoundingSet = "";
+        RestrictAddressFamilies = [
+          "AF_INET"
+          "AF_INET6"
+          "AF_UNIX"
+        ];
+        NoNewPrivileges = true;
+        PrivateMounts = true;
+        PrivateTmp = true;
+        PrivateUsers = true;
+        ProtectClock = true;
+        ProtectControlGroups = true;
+        ProtectHome = true;
+        ProtectKernelLogs = true;
+        ProtectKernelModules = true;
+        ProtectKernelTunables = true;
+        ProtectSystem = "strict";
+        MemoryDenyWriteExecute = true;
+        LockPersonality = true;
+        RemoveIPC = true;
+        RestrictNamespaces = true;
+        RestrictRealtime = true;
+        RestrictSUIDSGID = true;
+        SystemCallArchitectures = "native";
+        SystemCallFilter = [
+          "@system-service"
+          "~@privileged"
+          "~@resources"
+        ];
+        SystemCallErrorNumber = "EPERM";
+        ProtectProc = "invisible";
+        ProtectHostname = true;
+        ProcSubset = "pid";
+      };
+    };
+
+    networking.firewall = lib.mkIf cfg.openFirewall {
+      allowedTCPPorts = [ cfg.port ];
+    };
+
+  };
+
+  meta.maintainers = with lib.maintainers; [ newam ];
+}