helix: build tree-sitter grammars from source (#452275)
This commit is contained in:
Matthieu Coudron
@@ -25,15 +25,11 @@ rustPlatform.buildRustPackage (finalAttrs: {
|
||||
env = {
|
||||
# disable fetching and building of tree-sitter grammars in the helix-term build.rs
|
||||
HELIX_DISABLE_AUTO_GRAMMAR_BUILD = "1";
|
||||
HELIX_DEFAULT_RUNTIME = "${placeholder "out"}/lib/runtime";
|
||||
HELIX_DEFAULT_RUNTIME = helix.runtime;
|
||||
};
|
||||
|
||||
postInstall = ''
|
||||
mkdir -p $out/lib
|
||||
cp -r runtime $out/lib
|
||||
# copy tree-sitter grammars from helix package
|
||||
# TODO: build it from source instead
|
||||
cp -r ${helix}/lib/runtime/grammars/* $out/lib/runtime/grammars/
|
||||
installShellCompletion contrib/completion/hx.{bash,fish,zsh}
|
||||
mkdir -p $out/share/{applications,icons/hicolor/256x256/apps}
|
||||
cp contrib/Helix.desktop $out/share/applications
|
||||
|
||||
Executable
+155
@@ -0,0 +1,155 @@
|
||||
#!/usr/bin/env nix-shell
|
||||
#! nix-shell -i python3 -p python3 nurl
|
||||
"""
|
||||
Generate grammar information for Helix editor by parsing languages.toml
|
||||
and fetching source information using nurl in parallel.
|
||||
"""
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import tomllib
|
||||
from dataclasses import dataclass
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
|
||||
@dataclass
|
||||
class Grammar:
|
||||
name: str
|
||||
git_url: str
|
||||
rev: str
|
||||
subpath: str | None = None
|
||||
|
||||
|
||||
async def run_nurl(url: str, rev: str, semaphore: asyncio.Semaphore) -> dict[str, Any]:
|
||||
"""Run nurl command for a single grammar and return parsed JSON output."""
|
||||
async with semaphore:
|
||||
proc = await asyncio.create_subprocess_exec(
|
||||
"nurl",
|
||||
url,
|
||||
rev,
|
||||
"--json",
|
||||
stdout=asyncio.subprocess.PIPE,
|
||||
stderr=asyncio.subprocess.PIPE,
|
||||
)
|
||||
stdout, stderr = await proc.communicate()
|
||||
|
||||
if proc.returncode != 0:
|
||||
raise RuntimeError(f"nurl failed for {url}@{rev}: {stderr.decode()}")
|
||||
|
||||
return json.loads(stdout.decode())
|
||||
|
||||
|
||||
def parse_languages_toml(toml_path: Path) -> list[Grammar]:
|
||||
"""Parse languages.toml and extract grammar information."""
|
||||
with open(toml_path, "rb") as f:
|
||||
config = tomllib.load(f)
|
||||
|
||||
grammars = []
|
||||
for grammar in config.get("grammar", []):
|
||||
if "source" not in grammar:
|
||||
continue
|
||||
|
||||
source = grammar["source"]
|
||||
if "git" not in source or "rev" not in source:
|
||||
continue
|
||||
|
||||
grammars.append(
|
||||
Grammar(
|
||||
name=grammar["name"].replace("_", "-"),
|
||||
git_url=source["git"],
|
||||
rev=source["rev"],
|
||||
subpath=source.get("subpath"),
|
||||
)
|
||||
)
|
||||
|
||||
return grammars
|
||||
|
||||
|
||||
async def fetch_all_grammars(
|
||||
grammars: list[Grammar], max_parallel: int
|
||||
) -> dict[str, Any]:
|
||||
"""Fetch nurl information for all grammars in parallel."""
|
||||
semaphore = asyncio.Semaphore(max_parallel)
|
||||
results = {}
|
||||
total = len(grammars)
|
||||
completed = 0
|
||||
|
||||
tasks = []
|
||||
for grammar in grammars:
|
||||
task = run_nurl(grammar.git_url, grammar.rev, semaphore)
|
||||
tasks.append((grammar, task))
|
||||
|
||||
for grammar, task in tasks:
|
||||
try:
|
||||
result = await task
|
||||
results[grammar.name] = {
|
||||
"nurl": result,
|
||||
"subpath": grammar.subpath,
|
||||
}
|
||||
completed += 1
|
||||
print(f"[{completed}/{total}] ✓ {grammar.name}", file=sys.stderr)
|
||||
except Exception as e:
|
||||
completed += 1
|
||||
print(f"[{completed}/{total}] ✗ {grammar.name}: {e}", file=sys.stderr)
|
||||
results[grammar.name] = {"error": str(e)}
|
||||
|
||||
return results
|
||||
|
||||
|
||||
async def main():
|
||||
parser = argparse.ArgumentParser(
|
||||
description="Generate grammar information for Helix editor"
|
||||
)
|
||||
parser.add_argument(
|
||||
"languages_toml",
|
||||
type=Path,
|
||||
help="path to languages.toml from Helix repository",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-o",
|
||||
"--output",
|
||||
type=Path,
|
||||
default=Path("grammars.json"),
|
||||
help="output JSON file (default: grammars.json)",
|
||||
)
|
||||
parser.add_argument(
|
||||
"-j",
|
||||
"--jobs",
|
||||
type=int,
|
||||
default=os.cpu_count(),
|
||||
help=f"number of parallel nurl instances (default: {os.cpu_count()})",
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if not args.languages_toml.exists():
|
||||
print(f"Error: {args.languages_toml} not found", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
print(f"Parsing {args.languages_toml}...", file=sys.stderr)
|
||||
grammars = parse_languages_toml(args.languages_toml)
|
||||
print(f"Found {len(grammars)} grammars", file=sys.stderr)
|
||||
|
||||
print(f"Fetching grammar information ({args.jobs} parallel jobs)...", file=sys.stderr)
|
||||
results = await fetch_all_grammars(grammars, args.jobs)
|
||||
|
||||
errors = [name for name, data in results.items() if "error" in data]
|
||||
if errors:
|
||||
print(f"\nFailed grammars ({len(errors)}):", file=sys.stderr)
|
||||
for name in errors:
|
||||
print(f" - {name}: {results[name]['error']}", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
|
||||
with open(args.output, "w") as f:
|
||||
json.dump(results, f, indent=2)
|
||||
f.write('\n')
|
||||
|
||||
print(f"\nResults written to {args.output}", file=sys.stderr)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
File diff suppressed because it is too large
Load Diff
@@ -1,88 +1,154 @@
|
||||
{
|
||||
fetchzip,
|
||||
fetchpatch,
|
||||
fetchFromGitHub,
|
||||
lib,
|
||||
rustPlatform,
|
||||
mdbook,
|
||||
git,
|
||||
gitMinimal,
|
||||
installShellFiles,
|
||||
versionCheckHook,
|
||||
nix-update-script,
|
||||
runCommand,
|
||||
removeReferencesTo,
|
||||
pkgs,
|
||||
tree-sitter,
|
||||
lockedGrammars ? lib.importJSON ./grammars.json,
|
||||
grammarsOverlay ? (
|
||||
final: prev: {
|
||||
tree-sitter-sql = prev.tree-sitter-sql.override {
|
||||
generate = false;
|
||||
};
|
||||
tree-sitter-qmljs = prev.tree-sitter-qmljs.overrideAttrs {
|
||||
dontCheckForBrokenSymlinks = true;
|
||||
};
|
||||
}
|
||||
),
|
||||
}:
|
||||
|
||||
rustPlatform.buildRustPackage (finalAttrs: {
|
||||
pname = "helix";
|
||||
version = "25.07.1";
|
||||
outputs = [
|
||||
"out"
|
||||
"doc"
|
||||
];
|
||||
rustPlatform.buildRustPackage (
|
||||
finalAttrs:
|
||||
let
|
||||
lockedVersionsOverlay =
|
||||
final: prev:
|
||||
lib.mapAttrs (
|
||||
drvName: grammar:
|
||||
let
|
||||
lockedGrammar = lockedGrammars.${lib.removePrefix "tree-sitter-" drvName};
|
||||
in
|
||||
(prev.${drvName}.override {
|
||||
location = lockedGrammar.subpath;
|
||||
}).overrideAttrs
|
||||
{
|
||||
version = lib.sources.shortRev lockedGrammar.nurl.args.rev;
|
||||
src = (pkgs.${lockedGrammar.nurl.fetcher} lockedGrammar.nurl.args);
|
||||
}
|
||||
) prev;
|
||||
|
||||
# This release tarball includes source code for the tree-sitter grammars,
|
||||
# which is not ordinarily part of the repository.
|
||||
src = fetchzip {
|
||||
url = "https://github.com/helix-editor/helix/releases/download/${finalAttrs.version}/helix-${finalAttrs.version}-source.tar.xz";
|
||||
hash = "sha256-Pj/lfcQXRWqBOTTWt6+Gk61F9F1UmeCYr+26hGdG974=";
|
||||
stripRoot = false;
|
||||
};
|
||||
tree-sitter-grammars =
|
||||
lib.filterAttrs (drvName: _: lib.hasAttr (lib.removePrefix "tree-sitter-" drvName) lockedGrammars)
|
||||
(
|
||||
tree-sitter.grammarsScope.overrideScope (
|
||||
lib.composeExtensions lockedVersionsOverlay grammarsOverlay
|
||||
)
|
||||
);
|
||||
|
||||
patches = [
|
||||
# Support mdbook 0.5.x: escape HTML tags in command descriptions
|
||||
./mdbook-0.5-support.patch
|
||||
];
|
||||
# Dynamic libraries for the grammars always use the `.so` extension, also on Darwin (should use `.dylib`)
|
||||
# See here: https://github.com/helix-editor/helix/pull/14982
|
||||
# Switch to `stdenv.hostPlatform.extensions.sharedLibrary` once the fix above reaches the next release
|
||||
|
||||
postPatch = ''
|
||||
# mdbook 0.5 uses asset hashing for CSS/JS files
|
||||
# Remove custom theme to use default mdbook theme with correct asset references
|
||||
rm -f book/theme/index.hbs
|
||||
'';
|
||||
grammarsFarm = runCommand "helix-grammars" { } (
|
||||
lib.concatMapAttrsStringSep "\n" (_: grammar: ''
|
||||
install -D ${grammar}/parser $out/${grammar.language}.so
|
||||
${lib.getExe removeReferencesTo} -t ${grammar} $out/${grammar.language}.so
|
||||
'') (lib.filterAttrs (_: lib.isDerivation) tree-sitter-grammars)
|
||||
);
|
||||
|
||||
cargoHash = "sha256-Mf0nrgMk1MlZkSyUN6mlM5lmTcrOHn3xBNzmVGtApEU=";
|
||||
lockedGrammarsCount = lib.length (lib.attrNames lockedGrammars);
|
||||
|
||||
nativeBuildInputs = [
|
||||
git
|
||||
installShellFiles
|
||||
mdbook
|
||||
];
|
||||
|
||||
env.HELIX_DEFAULT_RUNTIME = "${placeholder "out"}/lib/runtime";
|
||||
|
||||
postBuild = ''
|
||||
mdbook build book -d ../book-html
|
||||
'';
|
||||
|
||||
postInstall = ''
|
||||
# not needed at runtime
|
||||
rm -r runtime/grammars/sources
|
||||
|
||||
mkdir -p $out/lib $doc/share/doc
|
||||
cp -r runtime $out/lib
|
||||
installShellCompletion contrib/completion/hx.{bash,fish,zsh}
|
||||
mkdir -p $out/share/{applications,icons/hicolor/256x256/apps}
|
||||
cp contrib/Helix.desktop $out/share/applications
|
||||
cp contrib/helix.png $out/share/icons/hicolor/256x256/apps
|
||||
cp -r ../book-html $doc/share/doc/$name
|
||||
'';
|
||||
|
||||
nativeInstallCheckInputs = [
|
||||
versionCheckHook
|
||||
];
|
||||
versionCheckProgram = "${placeholder "out"}/bin/hx";
|
||||
doInstallCheck = true;
|
||||
|
||||
passthru = {
|
||||
updateScript = nix-update-script { };
|
||||
};
|
||||
|
||||
meta = {
|
||||
description = "Post-modern modal text editor";
|
||||
homepage = "https://helix-editor.com";
|
||||
changelog = "https://github.com/helix-editor/helix/blob/${finalAttrs.version}/CHANGELOG.md";
|
||||
license = lib.licenses.mpl20;
|
||||
mainProgram = "hx";
|
||||
maintainers = with lib.maintainers; [
|
||||
danth
|
||||
yusdacra
|
||||
runtimeDir = runCommand "helix-runtime" { } ''
|
||||
cp -r --no-preserve=mode ${finalAttrs.src}/runtime $out
|
||||
rm -r $out/grammars
|
||||
ln -s ${grammarsFarm} $out/grammars
|
||||
count=$(ls -1 "$out/grammars/" | wc -l)
|
||||
if [ "$count" -ne ${toString lockedGrammarsCount} ]; then
|
||||
echo "Expected ${toString lockedGrammarsCount} grammars, found $count"
|
||||
exit 1
|
||||
fi
|
||||
'';
|
||||
in
|
||||
{
|
||||
pname = "helix";
|
||||
version = "25.07.1";
|
||||
outputs = [
|
||||
"out"
|
||||
"doc"
|
||||
];
|
||||
};
|
||||
})
|
||||
|
||||
src = fetchFromGitHub {
|
||||
owner = "helix-editor";
|
||||
repo = "helix";
|
||||
tag = "${finalAttrs.version}";
|
||||
hash = "sha256-RFSzGAcB0mMg/02ykYfTWXzQjLFu2CJ4BkS5HZ/6pBo=";
|
||||
};
|
||||
|
||||
patches = [
|
||||
# Support mdbook 0.5.x: escape HTML tags in command descriptions
|
||||
./mdbook-0.5-support.patch
|
||||
];
|
||||
|
||||
postPatch = ''
|
||||
# mdbook 0.5 uses asset hashing for CSS/JS files
|
||||
# Remove custom theme to use default mdbook theme with correct asset references
|
||||
rm -f book/theme/index.hbs
|
||||
'';
|
||||
|
||||
cargoHash = "sha256-Mf0nrgMk1MlZkSyUN6mlM5lmTcrOHn3xBNzmVGtApEU=";
|
||||
|
||||
nativeBuildInputs = [
|
||||
gitMinimal
|
||||
installShellFiles
|
||||
mdbook
|
||||
];
|
||||
|
||||
env = {
|
||||
HELIX_DEFAULT_RUNTIME = runtimeDir;
|
||||
HELIX_DISABLE_AUTO_GRAMMAR_BUILD = "1";
|
||||
};
|
||||
|
||||
postBuild = ''
|
||||
mdbook build book -d ../book-html
|
||||
'';
|
||||
|
||||
postInstall = ''
|
||||
mkdir -p $out/lib $doc/share/doc
|
||||
installShellCompletion contrib/completion/hx.{bash,fish,zsh}
|
||||
mkdir -p $out/share/{applications,icons/hicolor/256x256/apps}
|
||||
cp contrib/Helix.desktop $out/share/applications/Helix.desktop
|
||||
cp contrib/helix.png $out/share/icons/hicolor/256x256/apps/helix.png
|
||||
cp -r ../book-html $doc/share/doc/$name
|
||||
'';
|
||||
|
||||
nativeInstallCheckInputs = [
|
||||
versionCheckHook
|
||||
];
|
||||
versionCheckProgram = "${placeholder "out"}/bin/hx";
|
||||
doInstallCheck = true;
|
||||
|
||||
passthru = {
|
||||
updateScript = ./update.sh;
|
||||
runtime = runtimeDir;
|
||||
inherit tree-sitter-grammars;
|
||||
};
|
||||
|
||||
meta = {
|
||||
description = "Post-modern modal text editor";
|
||||
homepage = "https://helix-editor.com";
|
||||
changelog = "https://github.com/helix-editor/helix/blob/${finalAttrs.version}/CHANGELOG.md";
|
||||
license = lib.licenses.mpl20;
|
||||
mainProgram = "hx";
|
||||
maintainers = with lib.maintainers; [
|
||||
aciceri
|
||||
danth
|
||||
yusdacra
|
||||
];
|
||||
};
|
||||
}
|
||||
)
|
||||
|
||||
Executable
+24
@@ -0,0 +1,24 @@
|
||||
#!/usr/bin/env nix-shell
|
||||
#!nix-shell -i bash -p nurl nix-update python3
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
|
||||
echo "Updating helix source to the latest stable..."
|
||||
nix-update helix
|
||||
|
||||
echo "Fetching updated helixSource..."
|
||||
HELIX_SRC=$(nix-instantiate --eval -A "helix.src.outPath" --raw)
|
||||
|
||||
echo "Generating grammars.json..."
|
||||
"$SCRIPT_DIR/generate_grammars.py" \
|
||||
"$HELIX_SRC/languages.toml" \
|
||||
-o "$SCRIPT_DIR/grammars.json"
|
||||
|
||||
if [ $? -ne 0 ]; then
|
||||
echo "Error: Failed to generate grammars.json" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
echo "Done! Updated grammars.json"
|
||||
@@ -1,11 +1,12 @@
|
||||
{
|
||||
lib,
|
||||
stdenv,
|
||||
newScope,
|
||||
fetchFromGitHub,
|
||||
fetchFromGitLab,
|
||||
fetchFromSourcehut,
|
||||
fetchFromCodeberg,
|
||||
nix-update-script,
|
||||
runCommand,
|
||||
which,
|
||||
rustPlatform,
|
||||
emscripten,
|
||||
@@ -56,6 +57,7 @@ let
|
||||
fetchFromGitHub
|
||||
fetchFromGitLab
|
||||
fetchFromSourcehut
|
||||
fetchFromCodeberg
|
||||
;
|
||||
};
|
||||
|
||||
@@ -69,6 +71,14 @@ let
|
||||
*/
|
||||
builtGrammars = lib.mapAttrs (_: lib.makeOverridable buildGrammar) grammars;
|
||||
|
||||
/**
|
||||
# Extensible package set for tree-sitter grammars.
|
||||
# Provides .override and .extend for customization.
|
||||
# Note: Use builtGrammars (not this) when iterating over grammars,
|
||||
# as this includes package set functions alongside derivations
|
||||
*/
|
||||
grammarsScope = lib.makeScope newScope (self: builtGrammars);
|
||||
|
||||
# Usage:
|
||||
# pkgs.tree-sitter.withPlugins (p: [ p.tree-sitter-c p.tree-sitter-java ... ])
|
||||
#
|
||||
@@ -184,6 +194,7 @@ rustPlatform.buildRustPackage (finalAttrs: {
|
||||
grammars
|
||||
buildGrammar
|
||||
builtGrammars
|
||||
grammarsScope
|
||||
withPlugins
|
||||
allGrammars
|
||||
;
|
||||
|
||||
@@ -3,6 +3,7 @@
|
||||
fetchFromGitHub,
|
||||
fetchFromGitLab,
|
||||
fetchFromSourcehut,
|
||||
fetchFromCodeberg,
|
||||
nix-update-script,
|
||||
}:
|
||||
|
||||
@@ -74,6 +75,7 @@ lib.mapAttrs' (
|
||||
github = fetchFromGitHub;
|
||||
gitlab = fetchFromGitLab;
|
||||
sourcehut = fetchFromSourcehut;
|
||||
codeberg = fetchFromCodeberg;
|
||||
# NOTE: include other types here as required
|
||||
};
|
||||
in
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
@@ -19371,7 +19371,9 @@ self: super: with self; {
|
||||
name: value:
|
||||
!(builtins.elem name [
|
||||
"tree-sitter-go-template"
|
||||
"tree-sitter-php-only"
|
||||
"tree-sitter-sql"
|
||||
"tree-sitter-sshclientconfig"
|
||||
"tree-sitter-templ"
|
||||
])
|
||||
) pkgs.tree-sitter.builtGrammars
|
||||
|
||||
Reference in New Issue
Block a user