From afe82997e608c19f386d39a83498819b9e7a28b2 Mon Sep 17 00:00:00 2001 From: Jeremy Fleischman Date: Wed, 10 Sep 2025 17:59:03 +0200 Subject: [PATCH] tree-sitter-grammars: expose `tree-sitter.json` Tree sitter grammars recently started adding a [`tree-sitter.json`](https://tree-sitter.github.io/tree-sitter/cli/init.html?highlight=tree-sitter.json#structure-of-tree-sitterjson) file to the root of their repos. It has some useful information for folks writing code, so I believe it makes sense to expose. Here I've made 3 changes: 1. `tree-sitter-grammars.*` now include the `tree-sitter.json`: ```console $ ls $(nix-build -A tree-sitter-grammars.tree-sitter-nix) parser queries tree-sitter.json ``` 2. `python3.pkgs.tree-sitter-grammars.*` Python packages have a new `config` method you can call to get a nicely typed datastructure: ```console $ nix-shell -p 'let pkgs = import ./. {}; in pkgs.python3.withPackages (ps: [ps.tree-sitter-grammars.tree-sitter-nix])' --run "python -c 'import tree_sitter_nix; print(tree_sitter_nix.config())'" field_schema='https://tree-sitter.github.io/tree-sitter/assets/schemas/config.schema.json' grammars=[Grammar(name='nix', camelcase='Nix', title='Nix', scope='source.nix', path='/nix/store/29vbs3fbzzjgq6jgrwlklwwbwwsvgx2b-tree-sitter-nix-grammar-0.25.6', external_files=None, file_types=['nix'], highlights='queries/highlights.scm', injections='queries/injections.scm', locals='queries/locals.scm', tags='queries/tags.scm', injection_regex='^nix$', first_line_regex=None, content_regex=None, class_name='TreeSitterNix')] metadata=Metadata(version='0.3.0', license='MIT', description='Nix grammar for tree-sitter', links=Links(repository=AnyUrl('https://github.com/nix-community/tree-sitter-nix'), funding=None, homepage=None), authors=[Author(name='Charles Strahan', email='charles@cstrahan.com', url=AnyUrl('https://www.cstrahan.com/'))], namespace='io.github.tree-sitter') bindings=Bindings(c=True, go=True, java=False, kotlin=False, node=True, python=True, rust=True, swift=True, zig=False) ``` This required introducing a new `python-modules/tree-sitter-config` for the generated `TreeSitterConfig` datatype. I'm sort of hoping this (or something like it) would be acceptable for including in upstream `tree-sitter/py-tree-sitter`. 3. I've submitted patches to all grammers whose `tree-sitter.json` didn't obey upstream's schema definition. Until those patches land, we can opt the language out of the `test_config` test by setting `passthru.checkTreeSitterJson = false`. --- .../tree-sitter-config/default.nix | 60 +++++++++++++++++++ .../tree-sitter-grammars/default.nix | 58 ++++++++++++------ .../parsing/tree-sitter/build-grammar.nix | 12 ++++ .../tools/parsing/tree-sitter/default.nix | 10 +++- .../parsing/tree-sitter/grammars/default.nix | 3 +- .../tree-sitter/grammars/grammar-sources.nix | 30 +++++++++- pkgs/top-level/python-packages.nix | 2 + 7 files changed, 153 insertions(+), 22 deletions(-) create mode 100644 pkgs/development/python-modules/tree-sitter-config/default.nix diff --git a/pkgs/development/python-modules/tree-sitter-config/default.nix b/pkgs/development/python-modules/tree-sitter-config/default.nix new file mode 100644 index 000000000000..63b70e2f7b2f --- /dev/null +++ b/pkgs/development/python-modules/tree-sitter-config/default.nix @@ -0,0 +1,60 @@ +# This package defines a `TreeSitterConfig` class autogenerated from the +# tree-sitter `config.schema.json` using `datamodel-codegen`. +{ + pkgs, # Need to access `pkgs.tree-sitter`, which is different than `pkgs.python3Packages.tree-sitter`. + buildPythonPackage, + datamodel-code-generator, + lib, + pydantic, + runCommand, + hatchling, + symlinkJoin, + writeTextDir, +}: +buildPythonPackage (finalAttrs: { + pname = "tree-sitter-config"; + version = "1.0.0"; + pyproject = true; + build-system = [ hatchling ]; + + src = symlinkJoin { + name = "tree-sitter-config-source"; + paths = [ + (writeTextDir "pyproject.toml" /* toml */ '' + [build-system] + requires = ["hatchling >= 1.26"] + build-backend = "hatchling.build" + + [project] + name = "tree_sitter_config" + version = "${finalAttrs.version}" + '') + (runCommand "tree_sitter_config/__init__.py" + { + nativeBuildInputs = [ + datamodel-code-generator + ]; + } + '' + mkdir -p $out/tree_sitter_config + datamodel-codegen \ + --input ${pkgs.tree-sitter}/config.schema.json \ + --input-file-type jsonschema \ + --output-model-type pydantic_v2.BaseModel \ + --class-name TreeSitterConfig \ + > $out/tree_sitter_config/__init__.py + '' + ) + ]; + }; + + pythonImportsCheck = [ "tree_sitter_config" ]; + + dependencies = [ pydantic ] ++ pydantic.optional-dependencies.email; + + meta = { + description = "Python types for tree-sitter.json"; + license = lib.licenses.mit; + maintainers = with lib.maintainers; [ jfly ]; + }; +}) diff --git a/pkgs/development/python-modules/tree-sitter-grammars/default.nix b/pkgs/development/python-modules/tree-sitter-grammars/default.nix index 9e5ad4a3b11d..e84085a6ab25 100644 --- a/pkgs/development/python-modules/tree-sitter-grammars/default.nix +++ b/pkgs/development/python-modules/tree-sitter-grammars/default.nix @@ -4,6 +4,7 @@ setuptools, pytestCheckHook, tree-sitter, + tree-sitter-config, symlinkJoin, writeTextDir, # `name`: grammar derivation pname in the format of `tree-sitter-` @@ -44,26 +45,40 @@ buildPythonPackage { src = symlinkJoin { name = "${drvPrefix}-source"; paths = [ - (writeTextDir "${snakeCaseName}/__init__.py" '' + (writeTextDir "${snakeCaseName}/__init__.py" /* python */ '' # AUTO-GENERATED DO NOT EDIT - # preload the parser object before importing c binding - # this way we can avoid dynamic linker kicking in when - # downstream code imports this python module + from pathlib import Path + from tree_sitter_config import TreeSitterConfig + + # Preload the parser object before importing the C binding. + # This way we can avoid dynamic linker kicking in when + # downstream code imports this python module. import ctypes - import sys - import os parser = "${grammarDrv}/parser" try: ctypes.CDLL(parser, mode=ctypes.RTLD_GLOBAL) # cached except OSError as e: raise ImportError(f"cannot load tree-sitter parser object from {parser}: {e}") - # expose binding + def config() -> TreeSitterConfig | None: + grammar_path = Path("${grammarDrv}") + json_path = grammar_path / "tree-sitter.json" + if not json_path.exists(): + return None + + # Massage each grammar's path into an absolute path so later code + # can find queries, etc. + conf = TreeSitterConfig.model_validate_json(json_path.read_text()) + for grammar in conf.grammars: + grammar.path = str(grammar_path / grammar.path) + + return conf + from ._binding import language - __all__ = ["language"] + __all__ = ["language", "config"] '') - (writeTextDir "${snakeCaseName}/binding.c" '' + (writeTextDir "${snakeCaseName}/binding.c" /* c */ '' // AUTO-GENERATED DO NOT EDIT #include @@ -94,7 +109,7 @@ buildPythonPackage { return PyModule_Create(&module); } '') - (writeTextDir "setup.py" '' + (writeTextDir "setup.py" /* python */ '' # AUTO-GENERATED DO NOT EDIT from platform import system @@ -116,7 +131,7 @@ buildPythonPackage { ], ) '') - (writeTextDir "pyproject.toml" '' + (writeTextDir "pyproject.toml" /* toml */ '' # AUTO-GENERATED DO NOT EDIT [build-system] @@ -146,26 +161,35 @@ buildPythonPackage { build = "cp38-*" build-frontend = "build" '') - (writeTextDir "tests/test_language.py" '' + (writeTextDir "tests/test_language.py" /* python */ '' # AUTO-GENERATED DO NOT EDIT - from ${snakeCaseName} import language + from ${snakeCaseName} import config, language from tree_sitter import Language, Parser - # This test only checks that the binding can load the grammar from the compiled shared object. - # It does not verify the grammar itself; that is tested in - # `pkgs/development/tools/parsing/tree-sitter/grammar.nix`. - def test_language(): + # This test only checks that the binding can load the grammar from the compiled shared object. + # It does not verify the grammar itself; that is tested in + # `pkgs/development/tools/parsing/tree-sitter/grammar.nix`. + lang = Language(language()) assert lang is not None parser = Parser(lang) tree = parser.parse(bytes("", "utf-8")) assert tree is not None + + def test_config(): + # This test only checks if we can parse the tree-sitter.json, if one exists. + # (Not all grammars have one yet). + config() '') ]; }; + dependencies = [ + tree-sitter-config + ]; + preCheck = '' # https://github.com/NixOS/nixpkgs/issues/255262 rm -r ${snakeCaseName} diff --git a/pkgs/development/tools/parsing/tree-sitter/build-grammar.nix b/pkgs/development/tools/parsing/tree-sitter/build-grammar.nix index 86a8e19f6434..4694916fbc6f 100644 --- a/pkgs/development/tools/parsing/tree-sitter/build-grammar.nix +++ b/pkgs/development/tools/parsing/tree-sitter/build-grammar.nix @@ -12,6 +12,7 @@ src, meta ? { }, generate ? false, + excludeBrokenTreeSitterJson ? false, ... }@args: @@ -40,6 +41,13 @@ stdenv.mkDerivation ( stripDebugList = [ "parser" ]; + # Not all tree-sitter.json files follow the schema. If they're invalid, + # remove them. Note: these tree-sitter.json files are not validated here, + # but are validated in python3Packages.tree-sitter-grammars. + postPatch = lib.optionalString excludeBrokenTreeSitterJson '' + rm tree-sitter.json + ''; + # Tree-sitter grammar packages contain a `tree-sitter.json` file at their # root. This provides package metadata that can be used to infer build # details. @@ -101,6 +109,9 @@ stdenv.mkDerivation ( runHook preInstall mkdir $out mv parser $out/ + if [[ -f tree-sitter.json ]]; then + cp tree-sitter.json $out/ + fi if [[ -d queries ]]; then cp -r queries $out fi @@ -124,6 +135,7 @@ stdenv.mkDerivation ( }) // removeAttrs args [ "generate" + "excludeBrokenTreeSitterJson" "meta" ] ) diff --git a/pkgs/development/tools/parsing/tree-sitter/default.nix b/pkgs/development/tools/parsing/tree-sitter/default.nix index 55b712640b80..907c315b78cf 100644 --- a/pkgs/development/tools/parsing/tree-sitter/default.nix +++ b/pkgs/development/tools/parsing/tree-sitter/default.nix @@ -4,8 +4,8 @@ fetchFromGitHub, fetchFromGitLab, fetchFromSourcehut, + fetchpatch, nix-update-script, - runCommand, which, rustPlatform, emscripten, @@ -56,6 +56,7 @@ let fetchFromGitHub fetchFromGitLab fetchFromSourcehut + fetchpatch ; }; @@ -133,6 +134,11 @@ rustPlatform.buildRustPackage (finalAttrs: { (substitute { src = ./remove-web-interface.patch; }) + (fetchpatch { + name = "feat: allow `-` in grammar names"; + url = "https://github.com/tree-sitter/tree-sitter/commit/7d3c32125379c1dc02f47277bcd4eceaac299bdb.diff"; + hash = "sha256-ZNjdNateHVHDy0/txlAW8TUdz+DVxLKXpw8ojZbIQS8="; + }) ]; postPatch = @@ -162,6 +168,8 @@ rustPlatform.buildRustPackage (finalAttrs: { PREFIX=$out make install ${lib.optionalString (!enableShared) "rm -f $out/lib/*.so{,.*}"} ${lib.optionalString (!enableStatic) "rm -f $out/lib/*.a"} + + mv docs/src/assets/schemas/config.schema.json $out/ '' + lib.optionalString (stdenv.buildPlatform.canExecute stdenv.hostPlatform) '' installShellCompletion --cmd tree-sitter \ diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/default.nix b/pkgs/development/tools/parsing/tree-sitter/grammars/default.nix index 144799432473..9fe5b982073b 100644 --- a/pkgs/development/tools/parsing/tree-sitter/grammars/default.nix +++ b/pkgs/development/tools/parsing/tree-sitter/grammars/default.nix @@ -3,6 +3,7 @@ fetchFromGitHub, fetchFromGitLab, fetchFromSourcehut, + fetchpatch, nix-update-script, }: @@ -10,7 +11,7 @@ let /** Set of grammar sources. See ./grammar-sources.nix to define a new grammar. */ - grammar-sources = import ./grammar-sources.nix { inherit lib; }; + grammar-sources = import ./grammar-sources.nix { inherit lib fetchpatch; }; /** Parse a flakeref style string to { type, owner, repo, ref } diff --git a/pkgs/development/tools/parsing/tree-sitter/grammars/grammar-sources.nix b/pkgs/development/tools/parsing/tree-sitter/grammars/grammar-sources.nix index 2c1fbf47e988..be966984ff9f 100644 --- a/pkgs/development/tools/parsing/tree-sitter/grammars/grammar-sources.nix +++ b/pkgs/development/tools/parsing/tree-sitter/grammars/grammar-sources.nix @@ -1,4 +1,4 @@ -{ lib }: +{ fetchpatch, lib }: { bash = { @@ -17,6 +17,9 @@ meta = { license = lib.licenses.mit; }; + # The tree-sitter.json in this repo is invalid, see + # . + excludeBrokenTreeSitterJson = true; }; bibtex = { @@ -152,6 +155,13 @@ meta = { license = lib.licenses.mit; }; + patches = [ + (fetchpatch { + name = "Fix invalid `tree-sitter.json`"; + url = "https://github.com/UserNobody14/tree-sitter-dart/commit/81638dbbdb76a0e88ea8c31b95ec76b9625ddb84.diff"; + hash = "sha256-oaxuKQPN/gprO4OFWYItkj5dqd2xlq3SV6qr4YkSFjM="; + }) + ]; }; devicetree = { @@ -314,6 +324,9 @@ meta = { license = lib.licenses.mit; }; + # The tree-sitter.json in this repo is invalid, see + # + excludeBrokenTreeSitterJson = true; }; glsl = { @@ -450,6 +463,9 @@ meta = { license = lib.licenses.cc0; }; + # The tree-sitter.json in this repo is invalid, see + # + excludeBrokenTreeSitterJson = true; }; java = { @@ -625,6 +641,9 @@ meta = { license = lib.licenses.mit; }; + # The tree-sitter.json in this repo is invalid, see + # + excludeBrokenTreeSitterJson = true; }; nickel = { @@ -1044,9 +1063,14 @@ }; tlaplus = rec { - # FIXME: remove language override after release is available that includes + # FIXME: remove patch after release is available that includes # https://github.com/tlaplus-community/tree-sitter-tlaplus/pull/138 - language = "@tlaplus/tlaplus"; + patches = [ + (fetchpatch { + url = "https://github.com/tlaplus-community/tree-sitter-tlaplus/commit/2d831940c782850f64dabf5b7b17e9e51f7f0ebb.diff"; + hash = "sha256-ski2aYo25kHXz3T+Z2Coitdywot3tUiEbDY7gH7mTHE="; + }) + ]; version = "1.5.0"; url = "github:tlaplus-community/tree-sitter-tlaplus?ref=${version}"; hash = "sha256-k34gkAd0ueXEAww/Hc1mtBfn0Kp1pIBQtjDZ9GQeB4Q="; diff --git a/pkgs/top-level/python-packages.nix b/pkgs/top-level/python-packages.nix index 1ba61a337860..7020d1daa779 100644 --- a/pkgs/top-level/python-packages.nix +++ b/pkgs/top-level/python-packages.nix @@ -19315,6 +19315,8 @@ self: super: with self; { tree-sitter-c-sharp = callPackage ../development/python-modules/tree-sitter-c-sharp { }; + tree-sitter-config = callPackage ../development/python-modules/tree-sitter-config { }; + tree-sitter-embedded-template = callPackage ../development/python-modules/tree-sitter-embedded-template { };