diff --git a/pkgs/development/python-modules/tree-sitter-grammars/default.nix b/pkgs/development/python-modules/tree-sitter-grammars/default.nix new file mode 100644 index 000000000000..a8ea305dc2ae --- /dev/null +++ b/pkgs/development/python-modules/tree-sitter-grammars/default.nix @@ -0,0 +1,158 @@ +{ + lib, + buildPythonPackage, + pytestCheckHook, + tree-sitter, + symlinkJoin, + writeTextDir, + pythonOlder, + # `name`: grammar derivation pname in the format of `tree-sitter-` + name, + grammarDrv, +}: +let + inherit (grammarDrv) version; + + snakeCaseName = lib.replaceStrings [ "-" ] [ "_" ] name; + drvPrefix = "python-${name}"; + # If the name of the grammar attribute differs from the grammar's symbol name, + # it could cause a symbol mismatch at load time. This manually curated collection + # of overrides ensures the binding can find the correct symbol + langIdentOverrides = { + tree_sitter_org_nvim = "tree_sitter_org"; + }; + langIdent = langIdentOverrides.${snakeCaseName} or snakeCaseName; +in +buildPythonPackage { + inherit version; + pname = drvPrefix; + + src = symlinkJoin { + name = "${drvPrefix}-source"; + paths = [ + (writeTextDir "${snakeCaseName}/__init__.py" '' + from ._binding import language + + __all__ = ["language"] + '') + (writeTextDir "${snakeCaseName}/binding.c" '' + #include + + typedef struct TSLanguage TSLanguage; + + TSLanguage *${langIdent}(void); + + static PyObject* _binding_language(PyObject *self, PyObject *args) { + return PyLong_FromVoidPtr(${langIdent}()); + } + + static PyMethodDef methods[] = { + {"language", _binding_language, METH_NOARGS, + "Get the tree-sitter language for this grammar."}, + {NULL, NULL, 0, NULL} + }; + + static struct PyModuleDef module = { + .m_base = PyModuleDef_HEAD_INIT, + .m_name = "_binding", + .m_doc = NULL, + .m_size = -1, + .m_methods = methods + }; + + PyMODINIT_FUNC PyInit__binding(void) { + return PyModule_Create(&module); + } + '') + (writeTextDir "setup.py" '' + from platform import system + from setuptools import Extension, setup + + + setup( + packages=["${snakeCaseName}"], + ext_package="${snakeCaseName}", + ext_modules=[ + Extension( + name="_binding", + sources=["${snakeCaseName}/binding.c"], + extra_objects = ["${grammarDrv}/parser"], + extra_compile_args=( + ["-std=c11"] if system() != 'Windows' else [] + ), + ) + ], + ) + '') + (writeTextDir "pyproject.toml" '' + [build-system] + requires = ["setuptools", "wheel"] + build-backend = "setuptools.build_meta" + + [project] + name="${snakeCaseName}" + description = "${langIdent} grammar for tree-sitter" + version = "${version}" + keywords = ["parsing", "incremental", "python"] + classifiers = [ + "Development Status :: 4 - Beta", + "Intended Audience :: Developers", + "License :: OSI Approved :: MIT License", + "Topic :: Software Development :: Compilers", + "Topic :: Text Processing :: Linguistic", + ] + + requires-python = ">=3.8" + license.text = "MIT" + readme = "README.md" + + [project.optional-dependencies] + core = ["tree-sitter~=0.21"] + + [tool.cibuildwheel] + build = "cp38-*" + build-frontend = "build" + '') + (writeTextDir "tests/test_language.py" '' + from ${snakeCaseName} import language + from tree_sitter import Language, Parser + + # This test only checks that the binding can load the grammar from the compiled shared object. + # It does not verify the grammar itself; that is tested in + # `pkgs/development/tools/parsing/tree-sitter/grammar.nix`. + + def test_language(): + lang = Language(language()) + assert lang is not None + parser = Parser() + parser.language = lang + tree = parser.parse(bytes("", "utf-8")) + assert tree is not None + '') + ]; + }; + + preCheck = '' + # https://github.com/NixOS/nixpkgs/issues/255262 + rm -r ${snakeCaseName} + ''; + + disabled = pythonOlder "3.8"; + + nativeCheckInputs = [ + tree-sitter + pytestCheckHook + ]; + pythonImportsCheck = [ snakeCaseName ]; + + meta = { + description = "Python bindings for ${name}"; + license = lib.licenses.mit; + maintainers = with lib.maintainers; [ + a-jay98 + adfaure + mightyiam + stepbrobd + ]; + }; +} diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index c30bcb888305..f316b303f4ce 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -17396,6 +17396,24 @@ self: super: with self; { callPackage ../development/python-modules/tree-sitter-embedded-template { }; + tree-sitter-grammars = lib.recurseIntoAttrs ( + lib.mapAttrs + ( + name: grammarDrv: + callPackage ../development/python-modules/tree-sitter-grammars { inherit name grammarDrv; } + ) + ( + # Filtering grammars not compatible with current py-tree-sitter version + lib.filterAttrs ( + name: value: + !(builtins.elem name [ + "tree-sitter-sql" + "tree-sitter-templ" + ]) + ) pkgs.tree-sitter.builtGrammars + ) + ); + tree-sitter-html = callPackage ../development/python-modules/tree-sitter-html { }; tree-sitter-javascript = callPackage ../development/python-modules/tree-sitter-javascript { };