python3Packages.tree-sitter-grammars: init at 0.22.5

Co-authored-by: Robert James Hernandez <rob@sarcasticadmin.com>
Co-authored-by: Yifei Sun <ysun@hey.com>
Co-authored-by: Ali Jamadi <jamadi1377@gmail.com>
Co-authored-by: yakampe <yanis.kampe.cv@gmail.com>
Co-authored-by: GetPsyched <priyanshu@getpsyched.dev>
Co-authored-by: Adrien Faure <adrien.faure@protonmail.com>
Co-authored-by: Shahar "Dawn" Or <mightyiampresence@gmail.com>

python3Packages.tree-sitter-grammars: Filtering uncompatible grammars

py-tree-sitter is still not up to date to latest tree-sitter (0.25).
py-tree-sitter gets tree-sitter from a submodule which is at version 0.24.

Therefore all grammar with a langage version greater that 14 are not
compatible (language version is most of the time commited into the
source of the grammar, so we don't have a lot of control over it).

For now it concerns two grammars:
- sql grammar gets generated during the build and therefore ends up with
the latest language version (15) not compatible with py-tree-sitter
- templ grammar is already at version 15 in the upstream source.

pkgs/development/python-modules/tree-sitter-grammars/default.nix

@@ -0,0 +1,158 @@
+{
+  lib,
+  buildPythonPackage,
+  pytestCheckHook,
+  tree-sitter,
+  symlinkJoin,
+  writeTextDir,
+  pythonOlder,
+  # `name`: grammar derivation pname in the format of `tree-sitter-<lang>`
+  name,
+  grammarDrv,
+}:
+let
+  inherit (grammarDrv) version;
+
+  snakeCaseName = lib.replaceStrings [ "-" ] [ "_" ] name;
+  drvPrefix = "python-${name}";
+  # If the name of the grammar attribute differs from the grammar's symbol name,
+  # it could cause a symbol mismatch at load time. This manually curated collection
+  # of overrides ensures the binding can find the correct symbol
+  langIdentOverrides = {
+    tree_sitter_org_nvim = "tree_sitter_org";
+  };
+  langIdent = langIdentOverrides.${snakeCaseName} or snakeCaseName;
+in
+buildPythonPackage {
+  inherit version;
+  pname = drvPrefix;
+
+  src = symlinkJoin {
+    name = "${drvPrefix}-source";
+    paths = [
+      (writeTextDir "${snakeCaseName}/__init__.py" ''
+        from ._binding import language
+
+        __all__ = ["language"]
+      '')
+      (writeTextDir "${snakeCaseName}/binding.c" ''
+        #include <Python.h>
+
+        typedef struct TSLanguage TSLanguage;
+
+        TSLanguage *${langIdent}(void);
+
+        static PyObject* _binding_language(PyObject *self, PyObject *args) {
+            return PyLong_FromVoidPtr(${langIdent}());
+        }
+
+        static PyMethodDef methods[] = {
+            {"language", _binding_language, METH_NOARGS,
+            "Get the tree-sitter language for this grammar."},
+            {NULL, NULL, 0, NULL}
+        };
+
+        static struct PyModuleDef module = {
+            .m_base = PyModuleDef_HEAD_INIT,
+            .m_name = "_binding",
+            .m_doc = NULL,
+            .m_size = -1,
+            .m_methods = methods
+        };
+
+        PyMODINIT_FUNC PyInit__binding(void) {
+            return PyModule_Create(&module);
+        }
+      '')
+      (writeTextDir "setup.py" ''
+        from platform import system
+        from setuptools import Extension, setup
+
+
+        setup(
+          packages=["${snakeCaseName}"],
+          ext_package="${snakeCaseName}",
+          ext_modules=[
+            Extension(
+              name="_binding",
+              sources=["${snakeCaseName}/binding.c"],
+              extra_objects = ["${grammarDrv}/parser"],
+              extra_compile_args=(
+                ["-std=c11"] if system() != 'Windows' else []
+              ),
+            )
+          ],
+        )
+      '')
+      (writeTextDir "pyproject.toml" ''
+        [build-system]
+        requires = ["setuptools", "wheel"]
+        build-backend = "setuptools.build_meta"
+
+        [project]
+        name="${snakeCaseName}"
+        description = "${langIdent} grammar for tree-sitter"
+        version = "${version}"
+        keywords = ["parsing", "incremental", "python"]
+        classifiers = [
+          "Development Status :: 4 - Beta",
+          "Intended Audience :: Developers",
+          "License :: OSI Approved :: MIT License",
+          "Topic :: Software Development :: Compilers",
+          "Topic :: Text Processing :: Linguistic",
+        ]
+
+        requires-python = ">=3.8"
+        license.text = "MIT"
+        readme = "README.md"
+
+        [project.optional-dependencies]
+        core = ["tree-sitter~=0.21"]
+
+        [tool.cibuildwheel]
+        build = "cp38-*"
+        build-frontend = "build"
+      '')
+      (writeTextDir "tests/test_language.py" ''
+        from ${snakeCaseName} import language
+        from tree_sitter import Language, Parser
+
+        # This test only checks that the binding can load the grammar from the compiled shared object.
+        # It does not verify the grammar itself; that is tested in
+        # `pkgs/development/tools/parsing/tree-sitter/grammar.nix`.
+
+        def test_language():
+          lang = Language(language())
+          assert lang is not None
+          parser = Parser()
+          parser.language = lang
+          tree = parser.parse(bytes("", "utf-8"))
+          assert tree is not None
+      '')
+    ];
+  };
+
+  preCheck = ''
+    # https://github.com/NixOS/nixpkgs/issues/255262
+    rm -r ${snakeCaseName}
+  '';
+
+  disabled = pythonOlder "3.8";
+
+  nativeCheckInputs = [
+    tree-sitter
+    pytestCheckHook
+  ];
+  pythonImportsCheck = [ snakeCaseName ];
+
+  meta = {
+    description = "Python bindings for ${name}";
+    license = lib.licenses.mit;
+    maintainers = with lib.maintainers; [
+      a-jay98
+      adfaure
+      mightyiam
+      stepbrobd
+    ];
+  };
+}

pkgs/top-level/python-packages.nix

@@ -17396,6 +17396,24 @@ self: super: with self; {
     callPackage ../development/python-modules/tree-sitter-embedded-template
       { };
 
+  tree-sitter-grammars = lib.recurseIntoAttrs (
+    lib.mapAttrs
+      (
+        name: grammarDrv:
+        callPackage ../development/python-modules/tree-sitter-grammars { inherit name grammarDrv; }
+      )
+      (
+        # Filtering grammars not compatible with current py-tree-sitter version
+        lib.filterAttrs (
+          name: value:
+          !(builtins.elem name [
+            "tree-sitter-sql"
+            "tree-sitter-templ"
+          ])
+        ) pkgs.tree-sitter.builtGrammars
+      )
+  );
+
   tree-sitter-html = callPackage ../development/python-modules/tree-sitter-html { };
 
   tree-sitter-javascript = callPackage ../development/python-modules/tree-sitter-javascript { };