From 7d41b4d86fea29605448f20421e57bb807909fb6 Mon Sep 17 00:00:00 2001 From: Conroy Cheers Date: Fri, 6 Jun 2025 10:52:23 +1000 Subject: [PATCH 1/2] python312Packages.bitsandbytes: 0.45.1 -> 0.46.0 --- .../python-modules/bitsandbytes/default.nix | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pkgs/development/python-modules/bitsandbytes/default.nix b/pkgs/development/python-modules/bitsandbytes/default.nix index 49bc8150d6e4..14299f664c4b 100644 --- a/pkgs/development/python-modules/bitsandbytes/default.nix +++ b/pkgs/development/python-modules/bitsandbytes/default.nix @@ -11,7 +11,7 @@ let pname = "bitsandbytes"; - version = "0.45.1"; + version = "0.46.0"; inherit (torch) cudaPackages cudaSupport; inherit (cudaPackages) cudaMajorMinorVersion; @@ -54,10 +54,10 @@ buildPythonPackage { pyproject = true; src = fetchFromGitHub { - owner = "TimDettmers"; + owner = "bitsandbytes-foundation"; repo = "bitsandbytes"; tag = version; - hash = "sha256-MZ+3mUXaAhRb+rBtE+eQqT3XdtFxlWJc/CmTEwQkKSA="; + hash = "sha256-q1ltNYO5Ex6F2bfCcsekdsWjzXoal7g4n/LIHVGuj+k="; }; # By default, which library is loaded depends on the result of `torch.cuda.is_available()`. @@ -112,8 +112,8 @@ buildPythonPackage { meta = { description = "8-bit CUDA functions for PyTorch"; - homepage = "https://github.com/TimDettmers/bitsandbytes"; - changelog = "https://github.com/TimDettmers/bitsandbytes/releases/tag/${version}"; + homepage = "https://github.com/bitsandbytes-foundation/bitsandbytes"; + changelog = "https://github.com/bitsandbytes-foundation/bitsandbytes/releases/tag/${version}"; license = lib.licenses.mit; maintainers = with lib.maintainers; [ bcdarwin ]; }; From 1cf4b5ef9fdb85bfa54be27ed158025a47d0996d Mon Sep 17 00:00:00 2001 From: Conroy Cheers Date: Sun, 8 Jun 2025 22:45:13 +1000 Subject: [PATCH 2/2] python312Packages.vllm: 0.8.3 -> 0.9.0.1 Co-authored-by: Gaetan Lepage --- .../vllm/0005-drop-intel-reqs.patch | 12 +++ .../python-modules/vllm/default.nix | 82 +++++++++++++------ 2 files changed, 71 insertions(+), 23 deletions(-) create mode 100644 pkgs/development/python-modules/vllm/0005-drop-intel-reqs.patch diff --git a/pkgs/development/python-modules/vllm/0005-drop-intel-reqs.patch b/pkgs/development/python-modules/vllm/0005-drop-intel-reqs.patch new file mode 100644 index 000000000000..d324661f149a --- /dev/null +++ b/pkgs/development/python-modules/vllm/0005-drop-intel-reqs.patch @@ -0,0 +1,12 @@ +diff --git a/requirements/cpu.txt b/requirements/cpu.txt +index 121330158..d41918883 100644 +--- a/requirements/cpu.txt ++++ b/requirements/cpu.txt +@@ -20,7 +20,3 @@ datasets # for benchmark scripts + + # cpu cannot use triton 3.3.0 + triton==3.2.0; platform_machine == "x86_64" +- +-# Intel Extension for PyTorch, only for x86_64 CPUs +-intel-openmp==2024.2.1; platform_machine == "x86_64" +-intel_extension_for_pytorch==2.7.0; platform_machine == "x86_64" diff --git a/pkgs/development/python-modules/vllm/default.nix b/pkgs/development/python-modules/vllm/default.nix index 7cbfb50fce01..b29cfbeeab8e 100644 --- a/pkgs/development/python-modules/vllm/default.nix +++ b/pkgs/development/python-modules/vllm/default.nix @@ -3,8 +3,8 @@ stdenv, python, buildPythonPackage, - pythonRelaxDepsHook, fetchFromGitHub, + fetchpatch, symlinkJoin, autoAddDriverRunpath, @@ -15,7 +15,6 @@ packaging, setuptools, setuptools-scm, - wheel, # dependencies which, @@ -63,6 +62,14 @@ python-json-logger, python-multipart, llvmPackages, + opentelemetry-sdk, + opentelemetry-api, + opentelemetry-exporter-otlp, + bitsandbytes, + flashinfer, + + # internal dependency - for overriding in overlays + vllm-flash-attn ? null, cudaSupport ? torch.cudaSupport, cudaPackages ? { }, @@ -88,8 +95,8 @@ let cutlass = fetchFromGitHub { owner = "NVIDIA"; repo = "cutlass"; - tag = "v3.8.0"; - hash = "sha256-oIzlbKRdOh6gp6nRZ8udLSqleBFoFtgM7liCBlHZLOk="; + tag = "v3.9.2"; + hash = "sha256-teziPNA9csYvhkG5t2ht8W8x5+1YGGbHm8VKx4JoxgI="; }; flashmla = stdenv.mkDerivation { @@ -119,36 +126,41 @@ let ''; }; - vllm-flash-attn = stdenv.mkDerivation { + vllm-flash-attn' = lib.defaultTo (stdenv.mkDerivation { pname = "vllm-flash-attn"; # https://github.com/vllm-project/flash-attention/blob/${src.rev}/vllm_flash_attn/__init__.py - version = "2.7.2.post1"; + version = "2.7.4.post1"; # grep for GIT_TAG in the following file # https://github.com/vllm-project/vllm/blob/v${version}/cmake/external_projects/vllm_flash_attn.cmake src = fetchFromGitHub { owner = "vllm-project"; repo = "flash-attention"; - rev = "dc9d410b3e2d6534a4c70724c2515f4def670a22"; - hash = "sha256-ZQ0bOBIb+8IMmya8dmimKQ17KTBplX81IirdnBJpX5M="; + rev = "8798f27777fb57f447070301bf33a9f9c607f491"; + hash = "sha256-UTUvATGN1NU/Bc8qo078q6bEgILLmlrjL7Yk2iAJhg4="; }; dontConfigure = true; - # vllm-flash-attn normally relies on `git submodule update` to fetch cutlass - buildPhase = '' - rm -rf csrc/cutlass - ln -sf ${cutlass} csrc/cutlass - ''; + # vllm-flash-attn normally relies on `git submodule update` to fetch cutlass and composable_kernel + buildPhase = + '' + rm -rf csrc/cutlass + ln -sf ${cutlass} csrc/cutlass + '' + + lib.optionalString (rocmSupport) '' + rm -rf csrc/composable_kernel; + ln -sf ${rocmPackages.composable_kernel} csrc/composable_kernel + ''; installPhase = '' cp -rva . $out ''; - }; + }) vllm-flash-attn; cpuSupport = !cudaSupport && !rocmSupport; - # https://github.com/pytorch/pytorch/blob/v2.6.0/torch/utils/cpp_extension.py#L2046-L2048 + # https://github.com/pytorch/pytorch/blob/v2.7.0/torch/utils/cpp_extension.py#L2343-L2345 supportedTorchCudaCapabilities = let real = [ @@ -170,6 +182,11 @@ let "9.0" "9.0a" "10.0" + "10.0a" + "10.1" + "10.1a" + "12.0" + "12.0a" ]; ptx = lists.map (x: "${x}+PTX") real; in @@ -229,7 +246,7 @@ in buildPythonPackage rec { pname = "vllm"; - version = "0.8.3"; + version = "0.9.0.1"; pyproject = true; stdenv = torch.stdenv; @@ -238,13 +255,19 @@ buildPythonPackage rec { owner = "vllm-project"; repo = "vllm"; tag = "v${version}"; - hash = "sha256-LiEBkVwJTT4WoCTk9pI0ykTjmv1pDMzksmFwVktoxMY="; + hash = "sha256-gNe/kdsDQno8Fd6mo29feWmbyC0c2+kljlVxY4v7R9U="; }; patches = [ + (fetchpatch { + name = "remove-unused-opentelemetry-semantic-conventions-ai-dep.patch"; + url = "https://github.com/vllm-project/vllm/commit/6a5d7e45f52c3a13de43b8b4fa9033e3b342ebd2.patch"; + hash = "sha256-KYthqu+6XwsYYd80PtfrMMjuRV9+ionccr7EbjE4jJE="; + }) ./0002-setup.py-nix-support-respect-cmakeFlags.patch ./0003-propagate-pythonpath.patch ./0004-drop-lsmod.patch + ./0005-drop-intel-reqs.patch ]; postPatch = @@ -259,6 +282,10 @@ buildPythonPackage rec { --replace-fail \ 'set(PYTHON_SUPPORTED_VERSIONS' \ 'set(PYTHON_SUPPORTED_VERSIONS "${lib.versions.majorMinor python.version}"' + + # Pass build environment PYTHONPATH to vLLM's Python configuration scripts + substituteInPlace CMakeLists.txt \ + --replace-fail '$PYTHONPATH' '$ENV{PYTHONPATH}' '' + lib.optionalString (nccl == null) '' # On platforms where NCCL is not supported (e.g. Jetson), substitute Gloo (provided by Torch) @@ -361,12 +388,17 @@ buildPythonPackage rec { xformers xgrammar numba + opentelemetry-sdk + opentelemetry-api + opentelemetry-exporter-otlp + bitsandbytes ] ++ uvicorn.optional-dependencies.standard ++ aioprometheus.optional-dependencies.starlette ++ lib.optionals cudaSupport [ cupy pynvml + flashinfer ]; dontUseCmakeConfigure = true; @@ -374,7 +406,7 @@ buildPythonPackage rec { [ (lib.cmakeFeature "FETCHCONTENT_SOURCE_DIR_CUTLASS" "${lib.getDev cutlass}") (lib.cmakeFeature "FLASH_MLA_SRC_DIR" "${lib.getDev flashmla}") - (lib.cmakeFeature "VLLM_FLASH_ATTN_SRC_DIR" "${lib.getDev vllm-flash-attn}") + (lib.cmakeFeature "VLLM_FLASH_ATTN_SRC_DIR" "${lib.getDev vllm-flash-attn'}") ] ++ lib.optionals cudaSupport [ (lib.cmakeFeature "TORCH_CUDA_ARCH_LIST" "${gpuTargetString}") @@ -416,15 +448,19 @@ buildPythonPackage rec { pythonImportsCheck = [ "vllm" ]; - # updates the cutlass fetcher instead - passthru.skipBulkUpdate = true; + passthru = { + # make internal dependency available to overlays + vllm-flash-attn = vllm-flash-attn'; + # updates the cutlass fetcher instead + skipBulkUpdate = true; + }; - meta = with lib; { + meta = { description = "High-throughput and memory-efficient inference and serving engine for LLMs"; changelog = "https://github.com/vllm-project/vllm/releases/tag/v${version}"; homepage = "https://github.com/vllm-project/vllm"; - license = licenses.asl20; - maintainers = with maintainers; [ + license = lib.licenses.asl20; + maintainers = with lib.maintainers; [ happysalada lach ];