diff --git a/nixos/modules/services/web-apps/tabbyapi.nix b/nixos/modules/services/web-apps/tabbyapi.nix index bd937b0d5685..a955afe463b5 100644 --- a/nixos/modules/services/web-apps/tabbyapi.nix +++ b/nixos/modules/services/web-apps/tabbyapi.nix @@ -34,7 +34,7 @@ in host = lib.mkOption { type = lib.types.str; default = "127.0.0.1"; - description = "The IP to host on. Use 0.0.0.0 to expose on all adapters."; + description = "The IP to host on. Use 0.0.0.0 to expose on all network adapters."; }; port = lib.mkOption { @@ -46,13 +46,22 @@ in disable_auth = lib.mkOption { type = lib.types.bool; default = false; - description = "Disable HTTP token authentication. WARNING: Vulnerable if exposed."; + description = '' + Disable HTTP token authentication with requests. + WARNING: This will make your instance vulnerable! Only turn this on if you are ONLY connecting from localhost. + ''; + }; + + disable_fetch_requests = lib.mkOption { + type = lib.types.bool; + default = false; + description = "Disable fetching external content in response to requests, such as images from URLs."; }; api_servers = lib.mkOption { type = lib.types.listOf lib.types.str; default = [ "OAI" ]; - description = "Select API servers to enable. Options: OAI, Kobold."; + description = "Select API servers to enable. Possible values: OAI, Kobold."; }; }; @@ -66,7 +75,18 @@ in log_requests = lib.mkOption { type = lib.types.bool; default = false; - description = "Enable request logging. Only use for debug."; + description = "Enable request logging. NOTE: Only use this for debugging!"; + }; + + log_chat_completion_requests = lib.mkOption { + type = lib.types.bool; + default = false; + description = '' + Write every /v1/chat/completions request to logs/debug/ as JSON. + PRIVACY WARNING: Enabling this creates a comprehensive request log, including the + full message history and generation parameters. API keys are redacted, but prompts + and user-provided content are preserved for bug-report reproduction. + ''; }; }; @@ -111,35 +131,94 @@ in ''; }; + inline_model_loading = lib.mkOption { + type = lib.types.bool; + default = false; + description = '' + Allow direct loading of models from a completion or chat completion request. + This method of loading is strict by default; enable dummy models to add + exceptions for invalid model names. + ''; + }; + model_name = lib.mkOption { type = lib.types.nullOr lib.types.str; default = null; - description = "The initial model to load on startup. Must exist in model_dir."; + description = '' + An initial model to load. Make sure the model is located in the model directory! + REQUIRED: This must be filled out to load a model on startup. + ''; example = "Qwen3_5-9B"; }; max_seq_len = lib.mkOption { type = lib.types.nullOr lib.types.int; default = null; - description = "Max sequence length. Set null to use model defaults."; + description = '' + Max sequence length (default: min(max_position_embeddings, cache_size)). + Set to -1 to fetch from the model's config.json. + ''; }; cache_mode = lib.mkOption { type = lib.types.str; default = "FP16"; - description = "Cache mode for VRAM savings. ExLlamaV2: FP16, Q8, Q6, Q4. ExLlamaV3: specific pair string (e.g., '8,8')."; + description = '' + Enable different cache modes for VRAM savings. + Specify the pair k_bits,v_bits where k_bits and v_bits are integers from 2-8 (e.g. '8,8'). + The legacy values 'FP16', 'Q8', 'Q6', 'Q4' are also accepted. + ''; + }; + + tensor_parallel = lib.mkOption { + type = lib.types.bool; + default = false; + description = '' + Load model with tensor parallelism. + Falls back to autosplit if GPU split isn't provided. This ignores the gpu_split_auto value. + ''; }; gpu_split_auto = lib.mkOption { type = lib.types.bool; default = true; - description = "Automatically allocate resources to GPUs."; + description = "Automatically allocate resources to GPUs. Not parsed for single GPU users."; }; dummy_model_names = lib.mkOption { type = lib.types.listOf lib.types.str; default = [ "gpt-3.5-turbo" ]; - description = "List of fake model names sent via the /v1/models endpoint."; + description = '' + A list of fake model names that are sent via the /v1/models endpoint. + Also used as bypasses for strict mode if inline_model_loading is true. + ''; + }; + + vision = lib.mkOption { + type = lib.types.bool; + default = false; + description = "Enables vision support if the model supports it."; + }; + + reasoning = lib.mkOption { + type = lib.types.bool; + default = false; + description = '' + Enable reasoning parser. + Do NOT enable this if the model is not a reasoning model (e.g. deepseek-r1 series). + ''; + }; + + reasoning_start_token = lib.mkOption { + type = lib.types.str; + default = ""; + description = "The start token for reasoning content."; + }; + + reasoning_end_token = lib.mkOption { + type = lib.types.str; + default = ""; + description = "The end token for reasoning content."; }; }; }; diff --git a/pkgs/by-name/ta/tabbyapi/package.nix b/pkgs/by-name/ta/tabbyapi/package.nix index cac9ebfe2c59..2ecf9cb6b700 100644 --- a/pkgs/by-name/ta/tabbyapi/package.nix +++ b/pkgs/by-name/ta/tabbyapi/package.nix @@ -7,14 +7,14 @@ }: python3Packages.buildPythonApplication { pname = "tabbyapi"; - version = "0-unstable-2026-06-27"; + version = "0-unstable-2026-07-18"; pyproject = true; src = fetchFromGitHub { owner = "theroyallab"; repo = "tabbyAPI"; - rev = "3cf468c28362c28be1c8fc731ce1ccaf7b2206d0"; - hash = "sha256-s97YFyij2/oYlClmV2laDrCkkoK4uVZgRsn5WwftLag="; + rev = "0158fb48d76546a6475d1d63f6cd5b90932d1d11"; + hash = "sha256-Bkpx3MyZg7Np5zXAsq8mgxdAFsHUZhy2NZ93XSLbJgk="; }; build-system = with python3Packages; [ @@ -57,7 +57,6 @@ python3Packages.buildPythonApplication { numpy setuptools - exllamav2 exllamav3 ] ++ lib.optionals stdenv.hostPlatform.isLinux [ diff --git a/pkgs/development/python-modules/exllamav2/default.nix b/pkgs/development/python-modules/exllamav2/default.nix deleted file mode 100644 index fd8304708569..000000000000 --- a/pkgs/development/python-modules/exllamav2/default.nix +++ /dev/null @@ -1,108 +0,0 @@ -{ - lib, - buildPythonPackage, - fetchFromGitHub, - - # build-system - setuptools, - torch, - - cudaPackages, - - # nativeBuildInputs - pybind11, - - # dependencies - fastparquet, - flash-attn, - ninja, - numpy, - pandas, - pillow, - pygments, - regex, - rich, - safetensors, - tokenizers, - websockets, -}: -buildPythonPackage.override { inherit (torch) stdenv; } (finalAttrs: { - pname = "exllamav2"; - version = "0.3.2"; - pyproject = true; - __structuredAttrs = true; - - src = fetchFromGitHub { - owner = "turboderp-org"; - repo = "exllamav2"; - tag = "v${finalAttrs.version}"; - hash = "sha256-WbpbANenOuy6F0qAKVKAmolHjgRKfPxSVud8FZG1TXw="; - }; - - build-system = [ - setuptools - torch - ]; - - nativeBuildInputs = [ - ninja - ]; - - preConfigure = '' - export MAX_JOBS="$NIX_BUILD_CORES" - export NVCC_THREADS=2 - ''; - - buildInputs = [ - pybind11 - ] - ++ lib.optionals torch.cudaSupport [ - cudaPackages.cuda_cudart # cuda_runtime.h - cudaPackages.libcublas # cublas_v2.h - cudaPackages.libcurand # curand_kernel.h - cudaPackages.libcusolver # cusolverDn.h - cudaPackages.libcusparse # cusparse.h - ]; - - env = lib.optionalAttrs torch.cudaSupport { - CUDA_HOME = lib.getDev cudaPackages.cuda_nvcc; - TORCH_CUDA_ARCH_LIST = lib.concatStringsSep ";" torch.cudaCapabilities; - }; - - dependencies = [ - fastparquet - flash-attn - ninja - numpy - pandas - pillow - pygments - regex - rich - safetensors - tokenizers - torch - websockets - ]; - - pythonImportsCheck = [ "exllamav2" ]; - - # Tests require GPU hardware and external model files - doCheck = false; - - meta = { - homepage = "https://github.com/turboderp-org/exllamav2"; - description = "Inference library for running LLMs locally on modern consumer-class GPUs"; - changelog = "https://github.com/turboderp-org/exllamav2/releases/tag/${finalAttrs.src.tag}"; - license = lib.licenses.mit; - platforms = [ - "x86_64-windows" - "x86_64-linux" - ]; - - # Package requires CUDA or ROCm for functionality - # ROCm support is partially implemented but untested - broken = !torch.cudaSupport; - maintainers = with lib.maintainers; [ BatteredBunny ]; - }; -}) diff --git a/pkgs/development/python-modules/exllamav3/default.nix b/pkgs/development/python-modules/exllamav3/default.nix index ab7e43ef490f..64f380cd44fe 100644 --- a/pkgs/development/python-modules/exllamav3/default.nix +++ b/pkgs/development/python-modules/exllamav3/default.nix @@ -7,7 +7,6 @@ setuptools, - flash-attn, flash-linear-attention, formatron, kbnf, @@ -22,21 +21,20 @@ tokenizers, torch, typing-extensions, - xformers, }: let newerThanTuring = lib.filter (version: lib.versionOlder "7.9" version) torch.cudaCapabilities; in buildPythonPackage.override { inherit (torch) stdenv; } (finalAttrs: { pname = "exllamav3"; - version = "0.0.43"; + version = "1.1.0"; pyproject = true; src = fetchFromGitHub { owner = "turboderp-org"; repo = "exllamav3"; tag = "v${finalAttrs.version}"; - hash = "sha256-68v8ptvtOzRTnnRXrgU0emqmbCO0pECidgJ36bwm8/s="; + hash = "sha256-JlZt1UuTMmjaQWhiQZxzbHK3WgYYoBjP9PMEizEfsLY="; }; pythonRelaxDeps = [ @@ -60,7 +58,6 @@ buildPythonPackage.override { inherit (torch) stdenv; } (finalAttrs: { ]; dependencies = [ - flash-attn flash-linear-attention formatron kbnf @@ -74,7 +71,6 @@ buildPythonPackage.override { inherit (torch) stdenv; } (finalAttrs: { tokenizers torch typing-extensions - xformers ]; env = lib.optionalAttrs torch.cudaSupport { diff --git a/pkgs/top-level/python-aliases.nix b/pkgs/top-level/python-aliases.nix index 062af50d1470..fd1601785079 100644 --- a/pkgs/top-level/python-aliases.nix +++ b/pkgs/top-level/python-aliases.nix @@ -219,6 +219,7 @@ mapAliases { esig = throw "'esig' has been removed as it was broken and unmaintained upstream"; # Added 2026-05-27 et_xmlfile = throw "'et_xmlfile' has been renamed to/replaced by 'et-xmlfile'"; # Converted to throw 2025-10-29 ev3dev2 = throw "'ev3dev2' has been renamed to/replaced by 'python-ev3dev2'"; # Converted to throw 2025-10-29 + exllamav2 = throw "'exllamav2' was removed because it is archived upstream and support was dropped in tabbyapi"; # Added 2026-07-19 eyeD3 = throw "'eyeD3' has been renamed to/replaced by 'eyed3'"; # Converted to throw 2025-10-29 f3d_egl = lib.warnOnInstantiate "'f3d' now build with egl support by default, so `f3d_egl` is deprecated, consider using 'f3d' instead." f3d; # added 2025-07-18 Fabric = throw "'Fabric' has been renamed to/replaced by 'fabric'"; # Converted to throw 2025-10-29 diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index 5ccdc7774bb4..f5f981ffcbe7 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -5662,8 +5662,6 @@ self: super: with self; { exiv2 = callPackage ../development/python-modules/exiv2 { inherit (pkgs) exiv2; }; - exllamav2 = callPackage ../development/python-modules/exllamav2 { }; - exllamav3 = callPackage ../development/python-modules/exllamav3 { }; expandvars = callPackage ../development/python-modules/expandvars { };