tree-sitter: support building grammars to WebAssembly (#534687)

pkgs/by-name/tr/tree-sitter/grammars/README.md

@@ -109,6 +109,18 @@ This includes build-related flags and metadata.
 }
 ```
 
+## Building WebAssembly Parsers
+
+`buildGrammar` builds a native `$out/parser`.
+To build grammars as WebAssembly instead, use the `wasi32` cross package set, which installs `$out/parser.wasm`:
+
+```nix
+pkgsCross.wasi32.tree-sitter.builtGrammars.tree-sitter-nix
+```
+
+The Wasm build compiles `parser.c` and a C `scanner.c`.
+Grammars with C++ external scanners are rejected; use the native `buildGrammar` for those.
+
 ## Updating
 
 All grammar sources have a default update script defined.

pkgs/by-name/tr/tree-sitter/grammars/build-grammar.nix

@@ -16,6 +16,11 @@
   ...
 }@args:
 
+let
+  isWasi = stdenv.hostPlatform.isWasi;
+  parserOutput = if isWasi then "parser.wasm" else "parser";
+  exportSymbol = "tree_sitter_${lib.replaceStrings [ "-" ] [ "_" ] language}";
+in
 stdenv.mkDerivation (
   {
     pname = "tree-sitter-${language}";
@@ -32,11 +37,18 @@ stdenv.mkDerivation (
 
     CFLAGS = [
       "-Isrc"
-      "-O2"
+      # Match upstream `tree-sitter build --wasm`
+      (if isWasi then "-Os" else "-O2")
+    ]
+    ++ lib.optionals isWasi [
+      "-fvisibility=hidden"
     ];
     CXXFLAGS = [
       "-Isrc"
-      "-O2"
+      (if isWasi then "-Os" else "-O2")
+    ]
+    ++ lib.optionals isWasi [
+      "-fvisibility=hidden"
     ];
 
     stripDebugList = [ "parser" ];
@@ -90,25 +102,44 @@ stdenv.mkDerivation (
       tree-sitter generate
     '';
 
-    # When both scanner.{c,cc} exist, we should not link both since they may be the same but in
-    # different languages. Just randomly prefer C++ if that happens.
-    buildPhase = ''
-      runHook preBuild
-      if [[ -e src/scanner.cc ]]; then
-        $CXX -fPIC -c src/scanner.cc -o scanner.o $CXXFLAGS
-      elif [[ -e src/scanner.c ]]; then
-        $CC -fPIC -c src/scanner.c -o scanner.o $CFLAGS
-      fi
-      $CC -fPIC -c src/parser.c -o parser.o $CFLAGS
-      rm -rf parser
-      $CXX -shared -o parser *.o
-      runHook postBuild
-    '';
+    buildPhase =
+      if isWasi then
+        ''
+          runHook preBuild
+          if [[ -e src/scanner.cc || -e src/scanner.cpp ]]; then
+            nixErrorLog "tree-sitter wasm grammars only support C external scanners"
+            exit 1
+          fi
+          if [[ -e src/scanner.c ]]; then
+            $CC -fPIC -c src/scanner.c -o scanner.o $CFLAGS
+          fi
+          $CC -fPIC -c src/parser.c -o parser.o $CFLAGS
+          rm -rf parser.wasm
+          $CC -shared -o parser.wasm *.o \
+            -Wl,--export=${exportSymbol} \
+            -Wl,--allow-undefined \
+            -Wl,--no-entry \
+            -nostdlib
+          runHook postBuild
+        ''
+      else
+        ''
+          runHook preBuild
+          if [[ -e src/scanner.cc ]]; then
+            $CXX -fPIC -c src/scanner.cc -o scanner.o $CXXFLAGS
+          elif [[ -e src/scanner.c ]]; then
+            $CC -fPIC -c src/scanner.c -o scanner.o $CFLAGS
+          fi
+          $CC -fPIC -c src/parser.c -o parser.o $CFLAGS
+          rm -rf parser
+          $CXX -shared -o parser *.o
+          runHook postBuild
+        '';
 
     installPhase = ''
       runHook preInstall
       mkdir $out
-      mv parser $out/
+      mv ${parserOutput} $out/
       if [[ -f tree-sitter.json ]]; then
         cp tree-sitter.json $out/
       fi

pkgs/by-name/tr/tree-sitter/grammars/default.nix

@@ -6,13 +6,14 @@
   fetchpatch,
   fetchFromCodeberg,
   nix-update-script,
+  stdenv,
 }:
 
 let
   /**
     Set of grammar sources. See ./grammar-sources.nix to define a new grammar.
   */
-  grammar-sources = import ./grammar-sources.nix { inherit lib fetchpatch; };
+  grammar-sources = import ./grammar-sources.nix { inherit lib fetchpatch stdenv; };
 
   /**
     Parse a flakeref style string to { type, owner, repo, ref }

pkgs/by-name/tr/tree-sitter/grammars/grammar-sources.nix

@@ -1,4 +1,8 @@
-{ fetchpatch, lib }:
+{
+  fetchpatch,
+  lib,
+  stdenv,
+}:
 
 {
 
@@ -1757,6 +1761,8 @@
     url = "github:nvim-neorg/tree-sitter-norg";
     hash = "sha256-z3h5qMuNKnpQgV62xZ02F5vWEq4VEnm5lxwEnIFu+Rw=";
     meta = {
+      # Uses a C++ external scanner, unsupported by the WASM grammar build.
+      broken = stdenv.hostPlatform.isWasi;
       license = lib.licenses.mit;
     };
   };

pkgs/by-name/tr/tree-sitter/package.nix

@@ -10,6 +10,7 @@
   nix-update-script,
   which,
   rustPlatform,
+  runCommand,
   emscripten,
   openssl,
   pkg-config,
@@ -60,6 +61,7 @@ let
       fetchFromSourcehut
       fetchFromCodeberg
       fetchpatch
+      stdenv
       ;
   };
 
@@ -112,6 +114,8 @@ let
 
   allGrammars = lib.filter (p: !(p.meta.broken or false)) (lib.attrValues builtGrammars);
 
+  isWasi = stdenv.hostPlatform.isWasi;
+
 in
 rustPlatform.buildRustPackage (finalAttrs: {
   pname = "tree-sitter";
@@ -246,6 +250,19 @@ rustPlatform.buildRustPackage (finalAttrs: {
     tests = {
       # make sure all grammars build
       builtGrammars = lib.recurseIntoAttrs builtGrammars;
+    }
+    // lib.optionalAttrs isWasi {
+      wasmGrammar =
+        let
+          grammar = builtGrammars.tree-sitter-nix;
+        in
+        runCommand "tree-sitter-wasm-grammar-test" { } ''
+          test -f ${grammar}/parser.wasm
+          # WebAssembly binaries start with "\0asm".
+          test "$(od -An -tx1 -N4 ${grammar}/parser.wasm | tr -d ' \n')" = "0061736d"
+          test ! -e ${grammar}/parser
+          touch $out
+        '';
     };
   };
 

pkgs/top-level/release-cross.nix

@@ -121,6 +121,9 @@ let
     gmp = nativePlatforms;
     boehmgc = nativePlatforms;
     hello = nativePlatforms;
+    tree-sitter.builtGrammars =
+      mapAttrs (_: _: nativePlatforms)
+        (pkgsForCross systems.examples.wasi32 (builtins.head supportedSystems)).tree-sitter.builtGrammars;
     zlib = nativePlatforms;
   };