gador/nixpkgs
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

tree-sitter-grammars: expose tree-sitter.json (#442281)

Browse Source
This commit is contained in:
Matthieu Coudron
2026-02-17 18:09:32 +00:00
committed by GitHub
parent aa9d369a92 afe82997e6
commit d99da9009f
7 changed files with 153 additions and 22 deletions
Download Patch File Download Diff File Expand all files Collapse all files
pkgs/development/python-modules/tree-sitter-config/default.nix
+60
View File
@@ -0,0 +1,60 @@
# This package defines a `TreeSitterConfig` class autogenerated from the
# tree-sitter `config.schema.json` using `datamodel-codegen`.
{
pkgs, # Need to access `pkgs.tree-sitter`, which is different than `pkgs.python3Packages.tree-sitter`.
buildPythonPackage,
datamodel-code-generator,
lib,
pydantic,
runCommand,
hatchling,
symlinkJoin,
writeTextDir,
}:
buildPythonPackage (finalAttrs: {
pname = "tree-sitter-config";
version = "1.0.0";
pyproject = true;
build-system = [ hatchling ];
src = symlinkJoin {
name = "tree-sitter-config-source";
paths = [
(writeTextDir "pyproject.toml" /* toml */ ''
[build-system]
requires = ["hatchling >= 1.26"]
build-backend = "hatchling.build"
[project]
name = "tree_sitter_config"
version = "${finalAttrs.version}"
'')
(runCommand "tree_sitter_config/__init__.py"
{
nativeBuildInputs = [
datamodel-code-generator
];
}
''
mkdir -p $out/tree_sitter_config
datamodel-codegen \
--input ${pkgs.tree-sitter}/config.schema.json \
--input-file-type jsonschema \
--output-model-type pydantic_v2.BaseModel \
--class-name TreeSitterConfig \
> $out/tree_sitter_config/__init__.py
''
)
];
};
pythonImportsCheck = [ "tree_sitter_config" ];
dependencies = [ pydantic ] ++ pydantic.optional-dependencies.email;
meta = {
description = "Python types for tree-sitter.json";
license = lib.licenses.mit;
maintainers = with lib.maintainers; [ jfly ];
};
})
pkgs/development/python-modules/tree-sitter-grammars/default.nix
+41 -17
View File
@@ -4,6 +4,7 @@
setuptools,
pytestCheckHook,
tree-sitter,
tree-sitter-config,
symlinkJoin,
writeTextDir,
# `name`: grammar derivation pname in the format of `tree-sitter-<lang>`
@@ -44,26 +45,40 @@ buildPythonPackage {
src = symlinkJoin {
name = "${drvPrefix}-source";
paths = [
(writeTextDir "${snakeCaseName}/__init__.py" ''
(writeTextDir "${snakeCaseName}/__init__.py" /* python */ ''
# AUTO-GENERATED DO NOT EDIT
# preload the parser object before importing c binding
# this way we can avoid dynamic linker kicking in when
# downstream code imports this python module
from pathlib import Path
from tree_sitter_config import TreeSitterConfig
# Preload the parser object before importing the C binding.
# This way we can avoid dynamic linker kicking in when
# downstream code imports this python module.
import ctypes
import sys
import os
parser = "${grammarDrv}/parser"
try:
ctypes.CDLL(parser, mode=ctypes.RTLD_GLOBAL) # cached
except OSError as e:
raise ImportError(f"cannot load tree-sitter parser object from {parser}: {e}")
# expose binding
def config() -> TreeSitterConfig | None:
grammar_path = Path("${grammarDrv}")
json_path = grammar_path / "tree-sitter.json"
if not json_path.exists():
return None
# Massage each grammar's path into an absolute path so later code
# can find queries, etc.
conf = TreeSitterConfig.model_validate_json(json_path.read_text())
for grammar in conf.grammars:
grammar.path = str(grammar_path / grammar.path)
return conf
from ._binding import language
__all__ = ["language"]
__all__ = ["language", "config"]
'')
(writeTextDir "${snakeCaseName}/binding.c" ''
(writeTextDir "${snakeCaseName}/binding.c" /* c */ ''
// AUTO-GENERATED DO NOT EDIT
#include <Python.h>
@@ -94,7 +109,7 @@ buildPythonPackage {
return PyModule_Create(&module);
}
'')
(writeTextDir "setup.py" ''
(writeTextDir "setup.py" /* python */ ''
# AUTO-GENERATED DO NOT EDIT
from platform import system
@@ -116,7 +131,7 @@ buildPythonPackage {
],
)
'')
(writeTextDir "pyproject.toml" ''
(writeTextDir "pyproject.toml" /* toml */ ''
# AUTO-GENERATED DO NOT EDIT
[build-system]
@@ -146,26 +161,35 @@ buildPythonPackage {
build = "cp38-*"
build-frontend = "build"
'')
(writeTextDir "tests/test_language.py" ''
(writeTextDir "tests/test_language.py" /* python */ ''
# AUTO-GENERATED DO NOT EDIT
from ${snakeCaseName} import language
from ${snakeCaseName} import config, language
from tree_sitter import Language, Parser
# This test only checks that the binding can load the grammar from the compiled shared object.
# It does not verify the grammar itself; that is tested in
# `pkgs/development/tools/parsing/tree-sitter/grammar.nix`.
def test_language():
# This test only checks that the binding can load the grammar from the compiled shared object.
# It does not verify the grammar itself; that is tested in
# `pkgs/development/tools/parsing/tree-sitter/grammar.nix`.
lang = Language(language())
assert lang is not None
parser = Parser(lang)
tree = parser.parse(bytes("", "utf-8"))
assert tree is not None
def test_config():
# This test only checks if we can parse the tree-sitter.json, if one exists.
# (Not all grammars have one yet).
config()
'')
];
};
dependencies = [
tree-sitter-config
];
preCheck = ''
# https://github.com/NixOS/nixpkgs/issues/255262
rm -r ${snakeCaseName}
pkgs/development/tools/parsing/tree-sitter/build-grammar.nix
+12
View File
@@ -12,6 +12,7 @@
src,
meta ? { },
generate ? false,
excludeBrokenTreeSitterJson ? false,
...
}@args:
@@ -40,6 +41,13 @@ stdenv.mkDerivation (
stripDebugList = [ "parser" ];
# Not all tree-sitter.json files follow the schema. If they're invalid,
# remove them. Note: these tree-sitter.json files are not validated here,
# but are validated in python3Packages.tree-sitter-grammars.
postPatch = lib.optionalString excludeBrokenTreeSitterJson ''
rm tree-sitter.json
'';
# Tree-sitter grammar packages contain a `tree-sitter.json` file at their
# root. This provides package metadata that can be used to infer build
# details.
@@ -101,6 +109,9 @@ stdenv.mkDerivation (
runHook preInstall
mkdir $out
mv parser $out/
if [[ -f tree-sitter.json ]]; then
cp tree-sitter.json $out/
fi
if [[ -d queries ]]; then
cp -r queries $out
fi
@@ -124,6 +135,7 @@ stdenv.mkDerivation (
})
// removeAttrs args [
"generate"
"excludeBrokenTreeSitterJson"
"meta"
]
)
pkgs/development/tools/parsing/tree-sitter/default.nix
+9 -1
View File
@@ -4,8 +4,8 @@
fetchFromGitHub,
fetchFromGitLab,
fetchFromSourcehut,
fetchpatch,
nix-update-script,
runCommand,
which,
rustPlatform,
emscripten,
@@ -56,6 +56,7 @@ let
fetchFromGitHub
fetchFromGitLab
fetchFromSourcehut
fetchpatch
;
};
@@ -134,6 +135,11 @@ rustPlatform.buildRustPackage (finalAttrs: {
(substitute {
src = ./remove-web-interface.patch;
})
(fetchpatch {
name = "feat: allow `-` in grammar names";
url = "https://github.com/tree-sitter/tree-sitter/commit/7d3c32125379c1dc02f47277bcd4eceaac299bdb.diff";
hash = "sha256-ZNjdNateHVHDy0/txlAW8TUdz+DVxLKXpw8ojZbIQS8=";
})
];
postPatch =
@@ -163,6 +169,8 @@ rustPlatform.buildRustPackage (finalAttrs: {
PREFIX=$out make install
${lib.optionalString (!enableShared) "rm -f $out/lib/*.so{,.*}"}
${lib.optionalString (!enableStatic) "rm -f $out/lib/*.a"}
mv docs/src/assets/schemas/config.schema.json $out/
''
+ lib.optionalString (stdenv.buildPlatform.canExecute stdenv.hostPlatform) ''
installShellCompletion --cmd tree-sitter \
pkgs/development/tools/parsing/tree-sitter/grammars/default.nix
+2 -1
View File
@@ -3,6 +3,7 @@
fetchFromGitHub,
fetchFromGitLab,
fetchFromSourcehut,
fetchpatch,
nix-update-script,
}:
@@ -10,7 +11,7 @@ let
/**
Set of grammar sources. See ./grammar-sources.nix to define a new grammar.
*/
grammar-sources = import ./grammar-sources.nix { inherit lib; };
grammar-sources = import ./grammar-sources.nix { inherit lib fetchpatch; };
/**
Parse a flakeref style string to { type, owner, repo, ref }
pkgs/development/tools/parsing/tree-sitter/grammars/grammar-sources.nix
+27 -3
View File
@@ -1,4 +1,4 @@
{ lib }:
{ fetchpatch, lib }:
{
bash = {
@@ -17,6 +17,9 @@
meta = {
license = lib.licenses.mit;
};
# The tree-sitter.json in this repo is invalid, see
# <https://github.com/polarmutex/tree-sitter-beancount/pull/137>.
excludeBrokenTreeSitterJson = true;
};
bibtex = {
@@ -152,6 +155,13 @@
meta = {
license = lib.licenses.mit;
};
patches = [
(fetchpatch {
name = "Fix invalid `tree-sitter.json`";
url = "https://github.com/UserNobody14/tree-sitter-dart/commit/81638dbbdb76a0e88ea8c31b95ec76b9625ddb84.diff";
hash = "sha256-oaxuKQPN/gprO4OFWYItkj5dqd2xlq3SV6qr4YkSFjM=";
})
];
};
devicetree = {
@@ -314,6 +324,9 @@
meta = {
license = lib.licenses.mit;
};
# The tree-sitter.json in this repo is invalid, see
# <https://github.com/ember-tooling/tree-sitter-glimmer/pull/189>
excludeBrokenTreeSitterJson = true;
};
glsl = {
@@ -450,6 +463,9 @@
meta = {
license = lib.licenses.cc0;
};
# The tree-sitter.json in this repo is invalid, see
# <https://github.com/sogaiu/tree-sitter-janet-simple/pull/7>
excludeBrokenTreeSitterJson = true;
};
java = {
@@ -625,6 +641,9 @@
meta = {
license = lib.licenses.mit;
};
# The tree-sitter.json in this repo is invalid, see
# <https://github.com/Norgate-AV/tree-sitter-netlinx/pull/82>
excludeBrokenTreeSitterJson = true;
};
nickel = {
@@ -1044,9 +1063,14 @@
};
tlaplus = rec {
# FIXME: remove language override after release is available that includes
# FIXME: remove patch after release is available that includes
# https://github.com/tlaplus-community/tree-sitter-tlaplus/pull/138
language = "@tlaplus/tlaplus";
patches = [
(fetchpatch {
url = "https://github.com/tlaplus-community/tree-sitter-tlaplus/commit/2d831940c782850f64dabf5b7b17e9e51f7f0ebb.diff";
hash = "sha256-ski2aYo25kHXz3T+Z2Coitdywot3tUiEbDY7gH7mTHE=";
})
];
version = "1.5.0";
url = "github:tlaplus-community/tree-sitter-tlaplus?ref=${version}";
hash = "sha256-k34gkAd0ueXEAww/Hc1mtBfn0Kp1pIBQtjDZ9GQeB4Q=";
pkgs/top-level/python-packages.nix
+2
View File
@@ -19372,6 +19372,8 @@ self: super: with self; {
tree-sitter-c-sharp = callPackage ../development/python-modules/tree-sitter-c-sharp { };
tree-sitter-config = callPackage ../development/python-modules/tree-sitter-config { };
tree-sitter-embedded-template =
callPackage ../development/python-modules/tree-sitter-embedded-template
{ };