From d13410f25e0d28bc4163f5ea2f9ceafa373ffb74 Mon Sep 17 00:00:00 2001
From: Pavol Rusnak <pavol@rusnak.io>
Date: Tue, 4 Feb 2025 21:25:59 +0100
Subject: [PATCH 1/2] python3Packages.depyf: enable tests

---
 pkgs/development/python-modules/depyf/default.nix | 5 ++++-
 1 file changed, 4 insertions(+), 1 deletion(-)

diff --git a/pkgs/development/python-modules/depyf/default.nix b/pkgs/development/python-modules/depyf/default.nix
index 3792e9997a28..a47cc7fd78a9 100644
--- a/pkgs/development/python-modules/depyf/default.nix
+++ b/pkgs/development/python-modules/depyf/default.nix
@@ -5,6 +5,7 @@
   astor,
   dill,
   filelock,
+  pytestCheckHook,
 }:
 
 buildPythonPackage rec {
@@ -28,7 +29,9 @@ buildPythonPackage rec {
     filelock
   ];
 
-  pythonImportCheck = [ "depyf" ];
+  nativeCheckInputs = [ pytestCheckHook ];
+
+  pythonImportsCheck = [ "depyf" ];
 
   meta = with lib; {
     description = "Decompile python functions, from bytecode to source code";

From d710570afb98c015c4ab665956df89fe37f563a5 Mon Sep 17 00:00:00 2001
From: Pavol Rusnak <pavol@rusnak.io>
Date: Tue, 4 Feb 2025 21:27:05 +0100
Subject: [PATCH 2/2] python3Packages.xgrammar: build from source

---
 .../python-modules/xgrammar/default.nix       | 99 +++++++++++--------
 1 file changed, 56 insertions(+), 43 deletions(-)

diff --git a/pkgs/development/python-modules/xgrammar/default.nix b/pkgs/development/python-modules/xgrammar/default.nix
index a06f11da928c..b422e74bc023 100644
--- a/pkgs/development/python-modules/xgrammar/default.nix
+++ b/pkgs/development/python-modules/xgrammar/default.nix
@@ -2,10 +2,13 @@
   lib,
   stdenv,
   buildPythonPackage,
-  fetchPypi,
-  python,
-  pythonOlder,
-  pythonAtLeast,
+  fetchFromGitHub,
+  cmake,
+  ninja,
+  pip,
+  pytestCheckHook,
+  python3,
+  pybind11,
   pydantic,
   sentencepiece,
   tiktoken,
@@ -14,53 +17,63 @@
   triton,
 }:
 
-let
-  pyShortVersion = "cp" + builtins.replaceStrings [ "." ] [ "" ] python.pythonVersion;
-  platforms = rec {
-    aarch64-darwin = "macosx_13_0_arm64";
-    x86_64-darwin = "macosx_10_15_x86_64";
-    x86_64-linux = "manylinux_2_27_x86_64.manylinux_2_28_x86_64";
-  };
-  platform = platforms.${stdenv.system} or (throw "Unsupported system: ${stdenv.system}");
-  # hashes retrieved via the following command
-  # curl https://pypi.org/pypi/xgrammar/${version}/json | jq -r '.urls[] | "\(.digests.sha256)  \(.filename)"'
-  hashes = rec {
-    cp39-aarch64-darwin = "12dd579a7073c14981e01aeee566d20e60001bf90af23024e0e6692a770ff535";
-    cp39-x86_64-darwin = "035ec93306543b99bf2141dcc7f1a6dd0c255753fc8b5a2b5f3289a59fed8e37";
-    cp39-x86_64-linux = "3b3975dcf4b3ed7b16bbe3c068738b09847f841793e1c5e1b4a07dff36bbdc37";
-    cp310-aarch64-darwin = "93bb6c10cbdf1a2bda3b458d97b47436657d780f98dccf3d266e17e13568c0a9";
-    cp310-x86_64-darwin = "5ed31db2669dc499d9d29bb16f30b3395332ff9d0fb80b759697190a5ef5258b";
-    cp310-x86_64-linux = "9c6f571121e4af45e3b5dc55f3dadd751cffff1f85f1c6fc5c4276db2bbed222";
-    cp311-aarch64-darwin = "b293443725eddad31cf7b407bb24d5f3156c4b12a2c8041743cb7068a69fadcb";
-    cp311-x86_64-darwin = "b2106bceb2ce313628af915f2c2b1c9865612026dd3c9feddbfcc69e4ee6c971";
-    cp311-x86_64-linux = "7934c968371d55759cac35be3b218cdf4b13f323f535ea0faa233240bab803b9";
-    cp312-aarch64-darwin = "561f8d4307db8cf5d3c3b3ff46eda6d95379f6e801278dbf9153a9d5e8b6126c";
-    cp312-x86_64-darwin = "6ac3cbb0a82a3a9d07f0739f63b2e26cbef7855149d236057dcc7fee74b37970";
-    cp312-x86_64-linux = "1854d0fe6b908a3d2d42251a62e627224dbf6035a4322b844b1b5a277e3d0461";
-  };
-  hash =
-    hashes."${pyShortVersion}-${stdenv.system}"
-      or (throw "Unsupported Python version: ${python.pythonVersion}");
-in
 buildPythonPackage rec {
   pname = "xgrammar";
   version = "0.1.11";
-  format = "wheel";
+  format = "other";
 
-  disabled = pythonOlder "3.9" || pythonAtLeast "3.13";
-
-  src = fetchPypi {
-    inherit pname version format;
-    dist = pyShortVersion;
-    python = pyShortVersion;
-    abi = pyShortVersion;
-    platform = platform;
-    sha256 = hash;
+  src = fetchFromGitHub {
+    owner = "mlc-ai";
+    repo = pname;
+    tag = "v${version}";
+    hash = "sha256-q5d8/9S9p9M8HlCIernT9IwPEDnbC1R9nGsLuS15RXY=";
+    fetchSubmodules = true;
   };
 
-  pythonImportCheck = [ "xgrammar" ];
+  nativeBuildInputs = [
+    cmake
+    ninja
+  ];
+
+  buildInputs = [ pip ];
+
+  dontUseNinjaInstall = true;
+
+  installPhase = ''
+    runHook preInstall
+
+    cd ../python
+    ${python3.interpreter} -m pip install --prefix=$out .
+    cd ..
+
+    runHook postInstall
+  '';
+
+  nativeCheckInputs = [ pytestCheckHook ];
+
+  preCheck = ''
+    cd tests/python/
+  '';
+
+  disabledTests = [
+    # You are trying to access a gated repo.
+    "test_grammar_compiler"
+    "test_grammar_matcher"
+    "test_grammar_matcher_ebnf"
+    "test_grammar_matcher_json"
+    "test_grammar_matcher_json_schema"
+    "test_grammar_matcher_tag_dispatch"
+    "test_regex_converter"
+    # Torch not compiled with CUDA enabled
+    "test_token_bitmask_operations"
+    # You are trying to access a gated repo.
+    "test_tokenizer_info"
+  ];
+
+  pythonImportsCheck = [ "xgrammar" ];
 
   dependencies = [
+    pybind11
     pydantic
     sentencepiece
     tiktoken