From 7bb012784d73fd381f02a202d248acf3bf340993 Mon Sep 17 00:00:00 2001 From: BatteredBunny Date: Mon, 9 Mar 2026 18:14:13 +0200 Subject: [PATCH] nixos/tabbyapi: init module --- .../manual/release-notes/rl-2605.section.md | 2 + nixos/modules/module-list.nix | 1 + nixos/modules/services/web-apps/tabbyapi.nix | 199 ++++++++++++++++++ pkgs/by-name/ta/tabbyapi/package.nix | 5 +- 4 files changed, 206 insertions(+), 1 deletion(-) create mode 100644 nixos/modules/services/web-apps/tabbyapi.nix diff --git a/nixos/doc/manual/release-notes/rl-2605.section.md b/nixos/doc/manual/release-notes/rl-2605.section.md index 7e7274883e06..f267d7e1235b 100644 --- a/nixos/doc/manual/release-notes/rl-2605.section.md +++ b/nixos/doc/manual/release-notes/rl-2605.section.md @@ -78,6 +78,8 @@ - [Drasl](https://github.com/unmojang/drasl), an alternative authentication server for Minecraft. Available as [services.drasl](#opt-services.drasl.enable). +- [tabbyAPI](https://github.com/theroyallab/tabbyAPI), the official OpenAI compatible API server for Exllama. Available as [services.tabbyapi](#opt-services.tabbyapi.enable). + ## Backward Incompatibilities {#sec-release-26.05-incompatibilities} diff --git a/nixos/modules/module-list.nix b/nixos/modules/module-list.nix index 9a474422d923..5f265720e12c 100644 --- a/nixos/modules/module-list.nix +++ b/nixos/modules/module-list.nix @@ -1769,6 +1769,7 @@ ./services/web-apps/strfry.nix ./services/web-apps/suwayomi-server.nix ./services/web-apps/szurubooru.nix + ./services/web-apps/tabbyapi.nix ./services/web-apps/trilium.nix ./services/web-apps/tt-rss.nix ./services/web-apps/tuliprox.nix diff --git a/nixos/modules/services/web-apps/tabbyapi.nix b/nixos/modules/services/web-apps/tabbyapi.nix new file mode 100644 index 000000000000..0340e69775a7 --- /dev/null +++ b/nixos/modules/services/web-apps/tabbyapi.nix @@ -0,0 +1,199 @@ +{ + pkgs, + config, + lib, + ... +}: + +let + cfg = config.services.tabbyapi; + yamlFormat = pkgs.formats.yaml { }; + configFile = yamlFormat.generate "config.yml" cfg.settings; +in +{ + options.services.tabbyapi = { + enable = lib.mkEnableOption "tabbyapi"; + + package = lib.mkPackageOption pkgs "tabbyapi" { }; + + openFirewall = lib.mkOption { + type = lib.types.bool; + default = false; + description = "Whether to open the firewall for the TabbyAPI port."; + }; + + settings = lib.mkOption { + description = '' + Configuration for TabbyAPI. https://github.com/theroyallab/tabbyAPI/wiki/02.-Server-options + ''; + type = lib.types.submodule { + freeformType = yamlFormat.type; + + options = { + network = { + host = lib.mkOption { + type = lib.types.str; + default = "127.0.0.1"; + description = "The IP to host on. Use 0.0.0.0 to expose on all adapters."; + }; + + port = lib.mkOption { + type = lib.types.port; + default = 5000; + description = "The port to host on."; + }; + + disable_auth = lib.mkOption { + type = lib.types.bool; + default = false; + description = "Disable HTTP token authentication. WARNING: Vulnerable if exposed."; + }; + + api_servers = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = [ "OAI" ]; + description = "Select API servers to enable. Options: OAI, Kobold."; + }; + }; + + logging = { + log_prompt = lib.mkOption { + type = lib.types.bool; + default = false; + description = "Enable prompt logging."; + }; + + log_requests = lib.mkOption { + type = lib.types.bool; + default = false; + description = "Enable request logging. Only use for debug."; + }; + }; + + model = { + model_dir = lib.mkOption { + type = lib.types.str; + default = "models"; + description = "Directory to look for models. Relative to the state directory."; + example = lib.literalExpression '' + (pkgs.linkFarm "models" { + qwen-8b = pkgs.fetchgit { + url = "https://huggingface.co/turboderp/Qwen3-VL-8B-Instruct-exl3"; + rev = "652ab6be95b3e2880e78d87269013d98ca9c392d"; # 4bpw + fetchLFS = true; + hash = "sha256-n+9Mt7EZ3XHM0w8oGUZr4EBz91EFyp1VBpvl9Php/QM="; + }; + + # Example for patching Qwen 3.5's template to work with OpenWebUI's thinking feature + Qwen3_5-9B = pkgs.applyPatches { + src = pkgs.fetchgit { + url = "https://huggingface.co/turboderp/Qwen3.5-9B-exl3"; + rev = "6f8763307a3130ae989269fbc79a8c8e9db5ee42"; # 5.0bpw + fetchLFS = true; + hash = "sha256-Y7Uw/MChXU0Iu9hb3dv+cTtNBwhPbd/I/gYDUjM1j8g="; + }; + patches = [ ./qwen-thinking.patch ]; + }; + # diff --git a/chat_template.jinja b/chat_template.jinja + # index a585dec..68f1b6f 100644 + # --- a/chat_template.jinja + # +++ b/chat_template.jinja + # @@ -148,7 +148,5 @@ + # {{- '<|im_start|>assistant\n' }} + # {%- if enable_thinking is defined and enable_thinking is false %} + # {{- '\n\n\n\n' }} + # - {%- else %} + # - {{- '\n' }} + # {%- endif %} + # {%- endif %} + # \ No newline at end of file + }).outPath; + ''; + }; + + model_name = lib.mkOption { + type = lib.types.nullOr lib.types.str; + default = null; + description = "The initial model to load on startup. Must exist in model_dir."; + example = "Qwen3_5-9B"; + }; + + max_seq_len = lib.mkOption { + type = lib.types.nullOr lib.types.int; + default = null; + description = "Max sequence length. Set null to use model defaults."; + }; + + cache_mode = lib.mkOption { + type = lib.types.str; + default = "FP16"; + description = "Cache mode for VRAM savings. ExLlamaV2: FP16, Q8, Q6, Q4. ExLlamaV3: specific pair string (e.g., '8,8')."; + }; + + gpu_split_auto = lib.mkOption { + type = lib.types.bool; + default = true; + description = "Automatically allocate resources to GPUs."; + }; + + dummy_model_names = lib.mkOption { + type = lib.types.listOf lib.types.str; + default = [ "gpt-3.5-turbo" ]; + description = "List of fake model names sent via the /v1/models endpoint."; + }; + }; + }; + }; + }; + }; + + config = lib.mkIf cfg.enable { + assertions = [ + { + assertion = cfg.package.passthru.cudaSupport; + message = '' + TabbyAPI requires CUDA support to function. The configured package does not have CUDA enabled. + Consider setting: + services.tabbyapi.package = pkgs.pkgsCuda.tabbyapi; + ''; + } + ]; + networking.firewall.allowedTCPPorts = lib.mkIf cfg.openFirewall [ + cfg.settings.network.port + ]; + + systemd.services.tabbyapi = { + enable = true; + wantedBy = [ "multi-user.target" ]; + description = "TabbyAPI - OAI compatible server for Exllama"; + + serviceConfig = { + ExecStart = "${lib.getExe cfg.package} --config=${configFile}"; + Restart = "on-failure"; + StateDirectory = "tabbyapi"; + WorkingDirectory = "/var/lib/tabbyapi"; + User = "tabbyapi"; + Group = "tabbyapi"; + DynamicUser = true; + + # Hardening + ProtectSystem = "strict"; + ProtectHome = "yes"; + LockPersonality = true; + ProtectClock = true; + ProtectControlGroups = true; + ProtectHostname = true; + ProtectKernelLogs = true; + ProtectKernelModules = true; + ProtectKernelTunables = true; + ProtectProc = "invisible"; + RestrictNamespaces = true; + RestrictRealtime = true; + RestrictSUIDSGID = true; + SystemCallArchitectures = "native"; + }; + }; + }; + + meta.maintainers = with lib.maintainers; [ BatteredBunny ]; +} diff --git a/pkgs/by-name/ta/tabbyapi/package.nix b/pkgs/by-name/ta/tabbyapi/package.nix index 8c3a41d32181..50b0a10ed69d 100644 --- a/pkgs/by-name/ta/tabbyapi/package.nix +++ b/pkgs/by-name/ta/tabbyapi/package.nix @@ -99,7 +99,10 @@ python3Packages.buildPythonApplication { --add-flags "$out/${python3Packages.python.sitePackages}/main.py" ''; - passthru.updateScript = nix-update-script { }; + passthru = { + cudaSupport = python3Packages.torch.cudaSupport; + updateScript = nix-update-script { }; + }; meta = { description = "Official API server for Exllama";