From d2ff8a9dbcfa221dda5c525508133b32e305dc86 Mon Sep 17 00:00:00 2001 From: Austin Horstman Date: Mon, 22 Jun 2026 12:36:57 -0500 Subject: [PATCH 1/3] tree-sitter: support building grammars to WebAssembly Build grammars for a WASI host platform through the existing grammar builder so pkgsCross.wasi32.tree-sitter.builtGrammars produces parser.wasm. This keeps native grammar derivations unchanged and links parser.wasm directly with the WASI C toolchain while rejecting C++ external scanners. --- .../by-name/tr/tree-sitter/grammars/README.md | 12 ++++ .../tr/tree-sitter/grammars/build-grammar.nix | 65 ++++++++++++++----- pkgs/by-name/tr/tree-sitter/package.nix | 16 +++++ 3 files changed, 76 insertions(+), 17 deletions(-) diff --git a/pkgs/by-name/tr/tree-sitter/grammars/README.md b/pkgs/by-name/tr/tree-sitter/grammars/README.md index 8d4ba768536e..9445f68494cb 100644 --- a/pkgs/by-name/tr/tree-sitter/grammars/README.md +++ b/pkgs/by-name/tr/tree-sitter/grammars/README.md @@ -109,6 +109,18 @@ This includes build-related flags and metadata. } ``` +## Building WebAssembly Parsers + +`buildGrammar` builds a native `$out/parser`. +To build grammars as WebAssembly instead, use the `wasi32` cross package set, which installs `$out/parser.wasm`: + +```nix +pkgsCross.wasi32.tree-sitter.builtGrammars.tree-sitter-nix +``` + +The Wasm build compiles `parser.c` and a C `scanner.c`. +Grammars with C++ external scanners are rejected; use the native `buildGrammar` for those. + ## Updating All grammar sources have a default update script defined. diff --git a/pkgs/by-name/tr/tree-sitter/grammars/build-grammar.nix b/pkgs/by-name/tr/tree-sitter/grammars/build-grammar.nix index 4694916fbc6f..a0795c617d6d 100644 --- a/pkgs/by-name/tr/tree-sitter/grammars/build-grammar.nix +++ b/pkgs/by-name/tr/tree-sitter/grammars/build-grammar.nix @@ -16,6 +16,11 @@ ... }@args: +let + isWasi = stdenv.hostPlatform.isWasi; + parserOutput = if isWasi then "parser.wasm" else "parser"; + exportSymbol = "tree_sitter_${lib.replaceStrings [ "-" ] [ "_" ] language}"; +in stdenv.mkDerivation ( { pname = "tree-sitter-${language}"; @@ -32,11 +37,18 @@ stdenv.mkDerivation ( CFLAGS = [ "-Isrc" - "-O2" + # Match upstream `tree-sitter build --wasm` + (if isWasi then "-Os" else "-O2") + ] + ++ lib.optionals isWasi [ + "-fvisibility=hidden" ]; CXXFLAGS = [ "-Isrc" - "-O2" + (if isWasi then "-Os" else "-O2") + ] + ++ lib.optionals isWasi [ + "-fvisibility=hidden" ]; stripDebugList = [ "parser" ]; @@ -90,25 +102,44 @@ stdenv.mkDerivation ( tree-sitter generate ''; - # When both scanner.{c,cc} exist, we should not link both since they may be the same but in - # different languages. Just randomly prefer C++ if that happens. - buildPhase = '' - runHook preBuild - if [[ -e src/scanner.cc ]]; then - $CXX -fPIC -c src/scanner.cc -o scanner.o $CXXFLAGS - elif [[ -e src/scanner.c ]]; then - $CC -fPIC -c src/scanner.c -o scanner.o $CFLAGS - fi - $CC -fPIC -c src/parser.c -o parser.o $CFLAGS - rm -rf parser - $CXX -shared -o parser *.o - runHook postBuild - ''; + buildPhase = + if isWasi then + '' + runHook preBuild + if [[ -e src/scanner.cc || -e src/scanner.cpp ]]; then + nixErrorLog "tree-sitter wasm grammars only support C external scanners" + exit 1 + fi + if [[ -e src/scanner.c ]]; then + $CC -fPIC -c src/scanner.c -o scanner.o $CFLAGS + fi + $CC -fPIC -c src/parser.c -o parser.o $CFLAGS + rm -rf parser.wasm + $CC -shared -o parser.wasm *.o \ + -Wl,--export=${exportSymbol} \ + -Wl,--allow-undefined \ + -Wl,--no-entry \ + -nostdlib + runHook postBuild + '' + else + '' + runHook preBuild + if [[ -e src/scanner.cc ]]; then + $CXX -fPIC -c src/scanner.cc -o scanner.o $CXXFLAGS + elif [[ -e src/scanner.c ]]; then + $CC -fPIC -c src/scanner.c -o scanner.o $CFLAGS + fi + $CC -fPIC -c src/parser.c -o parser.o $CFLAGS + rm -rf parser + $CXX -shared -o parser *.o + runHook postBuild + ''; installPhase = '' runHook preInstall mkdir $out - mv parser $out/ + mv ${parserOutput} $out/ if [[ -f tree-sitter.json ]]; then cp tree-sitter.json $out/ fi diff --git a/pkgs/by-name/tr/tree-sitter/package.nix b/pkgs/by-name/tr/tree-sitter/package.nix index a456828b16b2..72e6bb2c6784 100644 --- a/pkgs/by-name/tr/tree-sitter/package.nix +++ b/pkgs/by-name/tr/tree-sitter/package.nix @@ -10,6 +10,7 @@ nix-update-script, which, rustPlatform, + runCommand, emscripten, openssl, pkg-config, @@ -112,6 +113,8 @@ let allGrammars = lib.filter (p: !(p.meta.broken or false)) (lib.attrValues builtGrammars); + isWasi = stdenv.hostPlatform.isWasi; + in rustPlatform.buildRustPackage (finalAttrs: { pname = "tree-sitter"; @@ -246,6 +249,19 @@ rustPlatform.buildRustPackage (finalAttrs: { tests = { # make sure all grammars build builtGrammars = lib.recurseIntoAttrs builtGrammars; + } + // lib.optionalAttrs isWasi { + wasmGrammar = + let + grammar = builtGrammars.tree-sitter-nix; + in + runCommand "tree-sitter-wasm-grammar-test" { } '' + test -f ${grammar}/parser.wasm + # WebAssembly binaries start with "\0asm". + test "$(od -An -tx1 -N4 ${grammar}/parser.wasm | tr -d ' \n')" = "0061736d" + test ! -e ${grammar}/parser + touch $out + ''; }; }; From dbd7c929f29759b02c004736009aa0ca3fc41c4e Mon Sep 17 00:00:00 2001 From: Austin Horstman Date: Tue, 23 Jun 2026 15:38:46 -0500 Subject: [PATCH 2/3] tree-sitter-grammars.tree-sitter-norg: mark wasi broken norg ships a C++ external scanner, which the WASM grammar build does not support (only C scanners are linked). Mark it broken on WASI so the wasi32 cross set skips it cleanly at eval instead of failing mid-build. --- pkgs/by-name/tr/tree-sitter/grammars/default.nix | 3 ++- pkgs/by-name/tr/tree-sitter/grammars/grammar-sources.nix | 8 +++++++- pkgs/by-name/tr/tree-sitter/package.nix | 1 + 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/pkgs/by-name/tr/tree-sitter/grammars/default.nix b/pkgs/by-name/tr/tree-sitter/grammars/default.nix index da1c049d0d7e..237e0fd99c6a 100644 --- a/pkgs/by-name/tr/tree-sitter/grammars/default.nix +++ b/pkgs/by-name/tr/tree-sitter/grammars/default.nix @@ -6,13 +6,14 @@ fetchpatch, fetchFromCodeberg, nix-update-script, + stdenv, }: let /** Set of grammar sources. See ./grammar-sources.nix to define a new grammar. */ - grammar-sources = import ./grammar-sources.nix { inherit lib fetchpatch; }; + grammar-sources = import ./grammar-sources.nix { inherit lib fetchpatch stdenv; }; /** Parse a flakeref style string to { type, owner, repo, ref } diff --git a/pkgs/by-name/tr/tree-sitter/grammars/grammar-sources.nix b/pkgs/by-name/tr/tree-sitter/grammars/grammar-sources.nix index d2a78b5b174b..81bda82b084e 100644 --- a/pkgs/by-name/tr/tree-sitter/grammars/grammar-sources.nix +++ b/pkgs/by-name/tr/tree-sitter/grammars/grammar-sources.nix @@ -1,4 +1,8 @@ -{ fetchpatch, lib }: +{ + fetchpatch, + lib, + stdenv, +}: { @@ -1758,6 +1762,8 @@ url = "github:nvim-neorg/tree-sitter-norg"; hash = "sha256-z3h5qMuNKnpQgV62xZ02F5vWEq4VEnm5lxwEnIFu+Rw="; meta = { + # Uses a C++ external scanner, unsupported by the WASM grammar build. + broken = stdenv.hostPlatform.isWasi; license = lib.licenses.mit; }; }; diff --git a/pkgs/by-name/tr/tree-sitter/package.nix b/pkgs/by-name/tr/tree-sitter/package.nix index 72e6bb2c6784..cc4522b35f21 100644 --- a/pkgs/by-name/tr/tree-sitter/package.nix +++ b/pkgs/by-name/tr/tree-sitter/package.nix @@ -61,6 +61,7 @@ let fetchFromSourcehut fetchFromCodeberg fetchpatch + stdenv ; }; From 4d894682215ca90d3f076779bfaa268b2e7a4c91 Mon Sep 17 00:00:00 2001 From: Austin Horstman Date: Tue, 23 Jun 2026 16:29:05 -0500 Subject: [PATCH 3/3] tree-sitter-grammars: build wasm grammars on Hydra Add the WASI grammar builds to release-cross so Hydra covers every grammar that compiles to WebAssembly. --- pkgs/top-level/release-cross.nix | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pkgs/top-level/release-cross.nix b/pkgs/top-level/release-cross.nix index cc0ab49a8c40..13fda84028dd 100644 --- a/pkgs/top-level/release-cross.nix +++ b/pkgs/top-level/release-cross.nix @@ -121,6 +121,9 @@ let gmp = nativePlatforms; boehmgc = nativePlatforms; hello = nativePlatforms; + tree-sitter.builtGrammars = + mapAttrs (_: _: nativePlatforms) + (pkgsForCross systems.examples.wasi32 (builtins.head supportedSystems)).tree-sitter.builtGrammars; zlib = nativePlatforms; };