diff --git a/pkgs/by-name/tr/tree-sitter/grammars/README.md b/pkgs/by-name/tr/tree-sitter/grammars/README.md index 8d4ba768536e..9445f68494cb 100644 --- a/pkgs/by-name/tr/tree-sitter/grammars/README.md +++ b/pkgs/by-name/tr/tree-sitter/grammars/README.md @@ -109,6 +109,18 @@ This includes build-related flags and metadata. } ``` +## Building WebAssembly Parsers + +`buildGrammar` builds a native `$out/parser`. +To build grammars as WebAssembly instead, use the `wasi32` cross package set, which installs `$out/parser.wasm`: + +```nix +pkgsCross.wasi32.tree-sitter.builtGrammars.tree-sitter-nix +``` + +The Wasm build compiles `parser.c` and a C `scanner.c`. +Grammars with C++ external scanners are rejected; use the native `buildGrammar` for those. + ## Updating All grammar sources have a default update script defined. diff --git a/pkgs/by-name/tr/tree-sitter/grammars/build-grammar.nix b/pkgs/by-name/tr/tree-sitter/grammars/build-grammar.nix index 4694916fbc6f..a0795c617d6d 100644 --- a/pkgs/by-name/tr/tree-sitter/grammars/build-grammar.nix +++ b/pkgs/by-name/tr/tree-sitter/grammars/build-grammar.nix @@ -16,6 +16,11 @@ ... }@args: +let + isWasi = stdenv.hostPlatform.isWasi; + parserOutput = if isWasi then "parser.wasm" else "parser"; + exportSymbol = "tree_sitter_${lib.replaceStrings [ "-" ] [ "_" ] language}"; +in stdenv.mkDerivation ( { pname = "tree-sitter-${language}"; @@ -32,11 +37,18 @@ stdenv.mkDerivation ( CFLAGS = [ "-Isrc" - "-O2" + # Match upstream `tree-sitter build --wasm` + (if isWasi then "-Os" else "-O2") + ] + ++ lib.optionals isWasi [ + "-fvisibility=hidden" ]; CXXFLAGS = [ "-Isrc" - "-O2" + (if isWasi then "-Os" else "-O2") + ] + ++ lib.optionals isWasi [ + "-fvisibility=hidden" ]; stripDebugList = [ "parser" ]; @@ -90,25 +102,44 @@ stdenv.mkDerivation ( tree-sitter generate ''; - # When both scanner.{c,cc} exist, we should not link both since they may be the same but in - # different languages. Just randomly prefer C++ if that happens. - buildPhase = '' - runHook preBuild - if [[ -e src/scanner.cc ]]; then - $CXX -fPIC -c src/scanner.cc -o scanner.o $CXXFLAGS - elif [[ -e src/scanner.c ]]; then - $CC -fPIC -c src/scanner.c -o scanner.o $CFLAGS - fi - $CC -fPIC -c src/parser.c -o parser.o $CFLAGS - rm -rf parser - $CXX -shared -o parser *.o - runHook postBuild - ''; + buildPhase = + if isWasi then + '' + runHook preBuild + if [[ -e src/scanner.cc || -e src/scanner.cpp ]]; then + nixErrorLog "tree-sitter wasm grammars only support C external scanners" + exit 1 + fi + if [[ -e src/scanner.c ]]; then + $CC -fPIC -c src/scanner.c -o scanner.o $CFLAGS + fi + $CC -fPIC -c src/parser.c -o parser.o $CFLAGS + rm -rf parser.wasm + $CC -shared -o parser.wasm *.o \ + -Wl,--export=${exportSymbol} \ + -Wl,--allow-undefined \ + -Wl,--no-entry \ + -nostdlib + runHook postBuild + '' + else + '' + runHook preBuild + if [[ -e src/scanner.cc ]]; then + $CXX -fPIC -c src/scanner.cc -o scanner.o $CXXFLAGS + elif [[ -e src/scanner.c ]]; then + $CC -fPIC -c src/scanner.c -o scanner.o $CFLAGS + fi + $CC -fPIC -c src/parser.c -o parser.o $CFLAGS + rm -rf parser + $CXX -shared -o parser *.o + runHook postBuild + ''; installPhase = '' runHook preInstall mkdir $out - mv parser $out/ + mv ${parserOutput} $out/ if [[ -f tree-sitter.json ]]; then cp tree-sitter.json $out/ fi diff --git a/pkgs/by-name/tr/tree-sitter/grammars/default.nix b/pkgs/by-name/tr/tree-sitter/grammars/default.nix index da1c049d0d7e..237e0fd99c6a 100644 --- a/pkgs/by-name/tr/tree-sitter/grammars/default.nix +++ b/pkgs/by-name/tr/tree-sitter/grammars/default.nix @@ -6,13 +6,14 @@ fetchpatch, fetchFromCodeberg, nix-update-script, + stdenv, }: let /** Set of grammar sources. See ./grammar-sources.nix to define a new grammar. */ - grammar-sources = import ./grammar-sources.nix { inherit lib fetchpatch; }; + grammar-sources = import ./grammar-sources.nix { inherit lib fetchpatch stdenv; }; /** Parse a flakeref style string to { type, owner, repo, ref } diff --git a/pkgs/by-name/tr/tree-sitter/grammars/grammar-sources.nix b/pkgs/by-name/tr/tree-sitter/grammars/grammar-sources.nix index 52cda04c8c80..3b3bc22c085e 100644 --- a/pkgs/by-name/tr/tree-sitter/grammars/grammar-sources.nix +++ b/pkgs/by-name/tr/tree-sitter/grammars/grammar-sources.nix @@ -1,4 +1,8 @@ -{ fetchpatch, lib }: +{ + fetchpatch, + lib, + stdenv, +}: { @@ -1757,6 +1761,8 @@ url = "github:nvim-neorg/tree-sitter-norg"; hash = "sha256-z3h5qMuNKnpQgV62xZ02F5vWEq4VEnm5lxwEnIFu+Rw="; meta = { + # Uses a C++ external scanner, unsupported by the WASM grammar build. + broken = stdenv.hostPlatform.isWasi; license = lib.licenses.mit; }; }; diff --git a/pkgs/by-name/tr/tree-sitter/package.nix b/pkgs/by-name/tr/tree-sitter/package.nix index a456828b16b2..cc4522b35f21 100644 --- a/pkgs/by-name/tr/tree-sitter/package.nix +++ b/pkgs/by-name/tr/tree-sitter/package.nix @@ -10,6 +10,7 @@ nix-update-script, which, rustPlatform, + runCommand, emscripten, openssl, pkg-config, @@ -60,6 +61,7 @@ let fetchFromSourcehut fetchFromCodeberg fetchpatch + stdenv ; }; @@ -112,6 +114,8 @@ let allGrammars = lib.filter (p: !(p.meta.broken or false)) (lib.attrValues builtGrammars); + isWasi = stdenv.hostPlatform.isWasi; + in rustPlatform.buildRustPackage (finalAttrs: { pname = "tree-sitter"; @@ -246,6 +250,19 @@ rustPlatform.buildRustPackage (finalAttrs: { tests = { # make sure all grammars build builtGrammars = lib.recurseIntoAttrs builtGrammars; + } + // lib.optionalAttrs isWasi { + wasmGrammar = + let + grammar = builtGrammars.tree-sitter-nix; + in + runCommand "tree-sitter-wasm-grammar-test" { } '' + test -f ${grammar}/parser.wasm + # WebAssembly binaries start with "\0asm". + test "$(od -An -tx1 -N4 ${grammar}/parser.wasm | tr -d ' \n')" = "0061736d" + test ! -e ${grammar}/parser + touch $out + ''; }; }; diff --git a/pkgs/top-level/release-cross.nix b/pkgs/top-level/release-cross.nix index cc0ab49a8c40..13fda84028dd 100644 --- a/pkgs/top-level/release-cross.nix +++ b/pkgs/top-level/release-cross.nix @@ -121,6 +121,9 @@ let gmp = nativePlatforms; boehmgc = nativePlatforms; hello = nativePlatforms; + tree-sitter.builtGrammars = + mapAttrs (_: _: nativePlatforms) + (pkgsForCross systems.examples.wasi32 (builtins.head supportedSystems)).tree-sitter.builtGrammars; zlib = nativePlatforms; };