From d2ff8a9dbcfa221dda5c525508133b32e305dc86 Mon Sep 17 00:00:00 2001 From: Austin Horstman Date: Mon, 22 Jun 2026 12:36:57 -0500 Subject: [PATCH] tree-sitter: support building grammars to WebAssembly Build grammars for a WASI host platform through the existing grammar builder so pkgsCross.wasi32.tree-sitter.builtGrammars produces parser.wasm. This keeps native grammar derivations unchanged and links parser.wasm directly with the WASI C toolchain while rejecting C++ external scanners. --- .../by-name/tr/tree-sitter/grammars/README.md | 12 ++++ .../tr/tree-sitter/grammars/build-grammar.nix | 65 ++++++++++++++----- pkgs/by-name/tr/tree-sitter/package.nix | 16 +++++ 3 files changed, 76 insertions(+), 17 deletions(-) diff --git a/pkgs/by-name/tr/tree-sitter/grammars/README.md b/pkgs/by-name/tr/tree-sitter/grammars/README.md index 8d4ba768536e..9445f68494cb 100644 --- a/pkgs/by-name/tr/tree-sitter/grammars/README.md +++ b/pkgs/by-name/tr/tree-sitter/grammars/README.md @@ -109,6 +109,18 @@ This includes build-related flags and metadata. } ``` +## Building WebAssembly Parsers + +`buildGrammar` builds a native `$out/parser`. +To build grammars as WebAssembly instead, use the `wasi32` cross package set, which installs `$out/parser.wasm`: + +```nix +pkgsCross.wasi32.tree-sitter.builtGrammars.tree-sitter-nix +``` + +The Wasm build compiles `parser.c` and a C `scanner.c`. +Grammars with C++ external scanners are rejected; use the native `buildGrammar` for those. + ## Updating All grammar sources have a default update script defined. diff --git a/pkgs/by-name/tr/tree-sitter/grammars/build-grammar.nix b/pkgs/by-name/tr/tree-sitter/grammars/build-grammar.nix index 4694916fbc6f..a0795c617d6d 100644 --- a/pkgs/by-name/tr/tree-sitter/grammars/build-grammar.nix +++ b/pkgs/by-name/tr/tree-sitter/grammars/build-grammar.nix @@ -16,6 +16,11 @@ ... }@args: +let + isWasi = stdenv.hostPlatform.isWasi; + parserOutput = if isWasi then "parser.wasm" else "parser"; + exportSymbol = "tree_sitter_${lib.replaceStrings [ "-" ] [ "_" ] language}"; +in stdenv.mkDerivation ( { pname = "tree-sitter-${language}"; @@ -32,11 +37,18 @@ stdenv.mkDerivation ( CFLAGS = [ "-Isrc" - "-O2" + # Match upstream `tree-sitter build --wasm` + (if isWasi then "-Os" else "-O2") + ] + ++ lib.optionals isWasi [ + "-fvisibility=hidden" ]; CXXFLAGS = [ "-Isrc" - "-O2" + (if isWasi then "-Os" else "-O2") + ] + ++ lib.optionals isWasi [ + "-fvisibility=hidden" ]; stripDebugList = [ "parser" ]; @@ -90,25 +102,44 @@ stdenv.mkDerivation ( tree-sitter generate ''; - # When both scanner.{c,cc} exist, we should not link both since they may be the same but in - # different languages. Just randomly prefer C++ if that happens. - buildPhase = '' - runHook preBuild - if [[ -e src/scanner.cc ]]; then - $CXX -fPIC -c src/scanner.cc -o scanner.o $CXXFLAGS - elif [[ -e src/scanner.c ]]; then - $CC -fPIC -c src/scanner.c -o scanner.o $CFLAGS - fi - $CC -fPIC -c src/parser.c -o parser.o $CFLAGS - rm -rf parser - $CXX -shared -o parser *.o - runHook postBuild - ''; + buildPhase = + if isWasi then + '' + runHook preBuild + if [[ -e src/scanner.cc || -e src/scanner.cpp ]]; then + nixErrorLog "tree-sitter wasm grammars only support C external scanners" + exit 1 + fi + if [[ -e src/scanner.c ]]; then + $CC -fPIC -c src/scanner.c -o scanner.o $CFLAGS + fi + $CC -fPIC -c src/parser.c -o parser.o $CFLAGS + rm -rf parser.wasm + $CC -shared -o parser.wasm *.o \ + -Wl,--export=${exportSymbol} \ + -Wl,--allow-undefined \ + -Wl,--no-entry \ + -nostdlib + runHook postBuild + '' + else + '' + runHook preBuild + if [[ -e src/scanner.cc ]]; then + $CXX -fPIC -c src/scanner.cc -o scanner.o $CXXFLAGS + elif [[ -e src/scanner.c ]]; then + $CC -fPIC -c src/scanner.c -o scanner.o $CFLAGS + fi + $CC -fPIC -c src/parser.c -o parser.o $CFLAGS + rm -rf parser + $CXX -shared -o parser *.o + runHook postBuild + ''; installPhase = '' runHook preInstall mkdir $out - mv parser $out/ + mv ${parserOutput} $out/ if [[ -f tree-sitter.json ]]; then cp tree-sitter.json $out/ fi diff --git a/pkgs/by-name/tr/tree-sitter/package.nix b/pkgs/by-name/tr/tree-sitter/package.nix index a456828b16b2..72e6bb2c6784 100644 --- a/pkgs/by-name/tr/tree-sitter/package.nix +++ b/pkgs/by-name/tr/tree-sitter/package.nix @@ -10,6 +10,7 @@ nix-update-script, which, rustPlatform, + runCommand, emscripten, openssl, pkg-config, @@ -112,6 +113,8 @@ let allGrammars = lib.filter (p: !(p.meta.broken or false)) (lib.attrValues builtGrammars); + isWasi = stdenv.hostPlatform.isWasi; + in rustPlatform.buildRustPackage (finalAttrs: { pname = "tree-sitter"; @@ -246,6 +249,19 @@ rustPlatform.buildRustPackage (finalAttrs: { tests = { # make sure all grammars build builtGrammars = lib.recurseIntoAttrs builtGrammars; + } + // lib.optionalAttrs isWasi { + wasmGrammar = + let + grammar = builtGrammars.tree-sitter-nix; + in + runCommand "tree-sitter-wasm-grammar-test" { } '' + test -f ${grammar}/parser.wasm + # WebAssembly binaries start with "\0asm". + test "$(od -An -tx1 -N4 ${grammar}/parser.wasm | tr -d ' \n')" = "0061736d" + test ! -e ${grammar}/parser + touch $out + ''; }; };