diff --git a/pkgs/by-name/wi/windmill/download.py.config.proto.patch b/pkgs/by-name/wi/windmill/download.py.config.proto.patch new file mode 100644 index 000000000000..435d7313d10d --- /dev/null +++ b/pkgs/by-name/wi/windmill/download.py.config.proto.patch @@ -0,0 +1,37 @@ +diff --git a/windmill-worker/nsjail/download.py.config.proto b/windmill-worker/nsjail/download.py.config.proto +index 5c2989417..7896bde7a 100644 +--- a/windmill-worker/nsjail/download.py.config.proto ++++ b/windmill-worker/nsjail/download.py.config.proto +@@ -22,17 +22,24 @@ keep_caps: true + keep_env: true + mount_proc: true + ++mount { ++ src: "/nix/store" ++ dst: "/nix/store" ++ is_bind: true ++} + + mount { + src: "/bin" + dst: "/bin" + is_bind: true ++ mandatory: false + } + + mount { + src: "/lib" + dst: "/lib" + is_bind: true ++ mandatory: false + } + + mount { +@@ -46,6 +53,7 @@ mount { + src: "/usr" + dst: "/usr" + is_bind: true ++ mandatory: false + } + + mount { diff --git a/pkgs/by-name/wi/windmill/package.nix b/pkgs/by-name/wi/windmill/package.nix index 9452a1797f37..b480d0500a86 100644 --- a/pkgs/by-name/wi/windmill/package.nix +++ b/pkgs/by-name/wi/windmill/package.nix @@ -22,6 +22,7 @@ rustfmt, stdenv, swagger-cli, + perl, _experimental-update-script-combinators, nix-update-script, writeScript, @@ -30,6 +31,19 @@ inherit (callPackage ./fetchers.nix { }) fetchLibrustyV8; } ), + libxml2, + xmlsec, + libxslt, + flock, + powershell, + uv, + bun, + dotnet-sdk_9, + php, + procps, + cargo, + coreutils, + withEnterpriseFeatures ? false, }: let @@ -42,8 +56,6 @@ let rev = "v${version}"; hash = "sha256-JhgqBXiX0ClEQZkWl7YBsBlQHk2Jp4jIdHy5CDvdoAM="; }; - - pythonEnv = python3.withPackages (ps: [ ps.pip-tools ]); in rustPlatform.buildRustPackage (finalAttrs: { inherit pname version src; @@ -65,22 +77,62 @@ rustPlatform.buildRustPackage (finalAttrs: { }; }; + buildFeatures = + [ + "embedding" + "parquet" + "prometheus" + "openidconnect" + "cloud" + "jemalloc" + "deno_core" + "license" + "http_trigger" + "zip" + "oauth2" + "kafka" + "otel" + "dind" + "php" + "mysql" + "mssql" + "bigquery" + "websocket" + "python" + "smtp" + "csharp" + "static_frontend" + # "rust" # compiler environment is incomplete + ] + ++ (lib.optionals withEnterpriseFeatures [ + "enterprise" + "enterprise_saml" + "tantivy" + "stripe" + ]); + patches = [ - ./swagger-cli.patch - ./run.go.config.proto.patch - ./run.python3.config.proto.patch + ./download.py.config.proto.patch + ./python_executor.patch + ./run.ansible.config.proto.patch ./run.bash.config.proto.patch + ./run.bun.config.proto.patch + ./run.csharp.config.proto.patch + ./run.go.config.proto.patch + ./run.php.config.proto.patch + ./run.powershell.config.proto.patch + ./run.python3.config.proto.patch + ./run.rust.config.proto.patch + ./rust_executor.patch + ./swagger-cli.patch ]; postPatch = '' - substituteInPlace windmill-worker/src/bash_executor.rs \ - --replace '"/bin/bash"' '"${bash}/bin/bash"' + substituteInPlace windmill-common/src/utils.rs \ + --replace-fail 'unknown-version' 'v${version}' - substituteInPlace windmill-api/src/lib.rs \ - --replace 'unknown-version' 'v${version}' - - substituteInPlace src/main.rs \ - --replace 'unknown-version' 'v${version}' + substituteInPlace windmill-worker/src/python_executor.rs \ + --replace-fail 'unknown_system_python_version' '${python3.version}' ''; buildInputs = [ @@ -88,35 +140,46 @@ rustPlatform.buildRustPackage (finalAttrs: { rustfmt lld (lib.getLib stdenv.cc.cc) + libxml2 + xmlsec + libxslt ]; nativeBuildInputs = [ pkg-config makeWrapper cmake # for libz-ng-sys crate + perl ]; # needs a postgres database running doCheck = false; + # TODO; Check if the rpath is still required + # patchelf --set-rpath ${lib.makeLibraryPath [ openssl ]} $out/bin/windmill postFixup = '' - patchelf --set-rpath ${lib.makeLibraryPath [ openssl ]} $out/bin/windmill - wrapProgram "$out/bin/windmill" \ + --prefix LD_LIBRARY_PATH : ${lib.makeLibraryPath [ stdenv.cc.cc ]} \ --prefix PATH : ${ lib.makeBinPath [ - go - pythonEnv - deno - nsjail - bash + python3 # uv searches PATH for system python + procps # bash_executor + coreutils # bash_executor ] } \ - --prefix LD_LIBRARY_PATH : ${lib.makeLibraryPath [ stdenv.cc.cc ]} \ - --set PYTHON_PATH "${pythonEnv}/bin/python3" \ + --set PYTHON_PATH "${python3}/bin/python3" \ --set GO_PATH "${go}/bin/go" \ --set DENO_PATH "${deno}/bin/deno" \ - --set NSJAIL_PATH "${nsjail}/bin/nsjail" + --set NSJAIL_PATH "${nsjail}/bin/nsjail" \ + --set FLOCK_PATH "${flock}/bin/flock" \ + --set BASH_PATH "${bash}/bin/bash" \ + --set POWERSHELL_PATH "${powershell}/bin/pwsh" \ + --set BUN_PATH "${bun}/bin/bun" \ + --set UV_PATH "${uv}/bin/uv" \ + --set DOTNET_PATH "${dotnet-sdk_9}/bin/dotnet" \ + --set DOTNET_ROOT "${dotnet-sdk_9}/share/dotnet" \ + --set PHP_PATH "${php}/bin/php" \ + --set CARGO_PATH "${cargo}/bin/cargo" ''; passthru.web-ui = buildNpmPackage { diff --git a/pkgs/by-name/wi/windmill/python_executor.patch b/pkgs/by-name/wi/windmill/python_executor.patch new file mode 100644 index 000000000000..14a1f502e725 --- /dev/null +++ b/pkgs/by-name/wi/windmill/python_executor.patch @@ -0,0 +1,100 @@ +diff --git a/windmill-worker/src/python_executor.rs b/windmill-worker/src/python_executor.rs +index 1180652d3..7b4ccf6cf 100644 +--- a/windmill-worker/src/python_executor.rs ++++ b/windmill-worker/src/python_executor.rs +@@ -290,70 +290,8 @@ impl PyVersion { + occupancy_metrics: &mut Option<&mut OccupancyMetrics>, + ) -> error::Result<()> { + let v = self.to_string_with_dot(); +- append_logs(job_id, w_id, format!("\nINSTALLING PYTHON ({})", v), db).await; +- // Create dirs for newly installed python +- // If we dont do this, NSJAIL will not be able to mount cache +- // For the default version directory created during startup (main.rs) +- DirBuilder::new() +- .recursive(true) +- .create(self.to_cache_dir()) +- .await +- .expect("could not create initial worker dir"); +- +- let logs = String::new(); +- +- #[cfg(windows)] +- let uv_cmd = "uv"; +- +- #[cfg(unix)] +- let uv_cmd = UV_PATH.as_str(); +- +- let mut child_cmd = Command::new(uv_cmd); +- child_cmd +- .env_clear() +- .env("HOME", HOME_ENV.to_string()) +- .env("PATH", PATH_ENV.to_string()) +- .envs(PROXY_ENVS.clone()) +- .args(["python", "install", v, "--python-preference=only-managed"]) +- // TODO: Do we need these? +- .envs([("UV_PYTHON_INSTALL_DIR", PY_INSTALL_DIR)]) +- .stdout(Stdio::piped()) +- .stderr(Stdio::piped()); +- +- #[cfg(windows)] +- { +- child_cmd +- .env("SystemRoot", SYSTEM_ROOT.as_str()) +- .env("USERPROFILE", crate::USERPROFILE_ENV.as_str()) +- .env( +- "TMP", +- std::env::var("TMP").unwrap_or_else(|_| String::from("/tmp")), +- ) +- .env( +- "LOCALAPPDATA", +- std::env::var("LOCALAPPDATA") +- .unwrap_or_else(|_| format!("{}\\AppData\\Local", HOME_ENV.as_str())), +- ); +- } +- +- let child_process = start_child_process(child_cmd, "uv").await?; +- +- append_logs(&job_id, &w_id, logs, db).await; +- handle_child( +- job_id, +- db, +- mem_peak, +- &mut None, +- child_process, +- false, +- worker_name, +- &w_id, +- "uv", +- None, +- false, +- occupancy_metrics, +- ) +- .await ++ append_logs(job_id, w_id, format!("\nREQUESTED PYTHON INSTALL IGNORED ({})", v), db).await; ++ Err(error::Error::BadConfig(format!("Python is managed through the NixOS system configuration. Change the Windmill instance setting to version 'unknown_system_python_version'"))) + } + async fn find_python(self) -> error::Result> { + #[cfg(windows)] +@@ -391,11 +329,11 @@ impl PyVersion { + "find", + self.to_string_with_dot(), + "--system", +- "--python-preference=only-managed", ++ "--python-preference=only-system", + ]) + .envs([ + ("UV_PYTHON_INSTALL_DIR", PY_INSTALL_DIR), +- ("UV_PYTHON_PREFERENCE", "only-managed"), ++ ("UV_PYTHON_PREFERENCE", "only-system"), + ]) + // .stdout(Stdio::piped()) + .stderr(Stdio::piped()) +@@ -561,7 +499,7 @@ pub async fn uv_pip_compile( + "-p", + &py_version.to_string_with_dot(), + "--python-preference", +- "only-managed", ++ "only-system", + ]); + + if no_cache { diff --git a/pkgs/by-name/wi/windmill/run.ansible.config.proto.patch b/pkgs/by-name/wi/windmill/run.ansible.config.proto.patch new file mode 100644 index 000000000000..39800486d8f8 --- /dev/null +++ b/pkgs/by-name/wi/windmill/run.ansible.config.proto.patch @@ -0,0 +1,44 @@ +diff --git a/windmill-worker/nsjail/run.ansible.config.proto b/windmill-worker/nsjail/run.ansible.config.proto +index 65a8ea700..d4c8c2afc 100644 +--- a/windmill-worker/nsjail/run.ansible.config.proto ++++ b/windmill-worker/nsjail/run.ansible.config.proto +@@ -18,16 +18,24 @@ keep_caps: false + keep_env: true + mount_proc: true + ++mount { ++ src: "/nix/store" ++ dst: "/nix/store" ++ is_bind: true ++} ++ + mount { + src: "/bin" + dst: "/bin" + is_bind: true ++ mandatory: false + } + + mount { + src: "/lib" + dst: "/lib" + is_bind: true ++ mandatory: false + } + + +@@ -42,12 +50,14 @@ mount { + src: "/root/.local/share/uv/tools/ansible" + dst: "/root/.local/share/uv/tools/ansible" + is_bind: true ++ mandatory: false + } + + mount { + src: "/usr" + dst: "/usr" + is_bind: true ++ mandatory: false + } + + mount { diff --git a/pkgs/by-name/wi/windmill/run.bash.config.proto.patch b/pkgs/by-name/wi/windmill/run.bash.config.proto.patch index 874fe0607ee7..5b893c407305 100644 --- a/pkgs/by-name/wi/windmill/run.bash.config.proto.patch +++ b/pkgs/by-name/wi/windmill/run.bash.config.proto.patch @@ -1,41 +1,46 @@ -diff --git a/windmill-worker/nsjail/run.bash.config.proto b/backend/windmill-worker/nsjail/run.bash.config.proto -index e93e6b45..bbedb165 100644 +diff --git a/windmill-worker/nsjail/run.bash.config.proto b/windmill-worker/nsjail/run.bash.config.proto +index 63018f765..d8bb0c0a7 100644 --- a/windmill-worker/nsjail/run.bash.config.proto +++ b/windmill-worker/nsjail/run.bash.config.proto -@@ -18,6 +18,12 @@ clone_newuser: {CLONE_NEWUSER} - keep_caps: false +@@ -15,10 +15,17 @@ keep_caps: false keep_env: true + mount_proc: true +mount { -+ src: "/nix/store" -+ dst: "/nix/store" ++ src: "/nix/store" ++ dst: "/nix/store" + is_bind: true +} + mount { src: "/bin" dst: "/bin" -@@ -25,6 +31,7 @@ mount { + is_bind: true ++ mandatory: false } mount { +@@ -32,6 +39,7 @@ mount { + src: "/bin" + dst: "/bin" + is_bind: true + mandatory: false + } + + mount { +@@ -45,6 +53,7 @@ mount { src: "/lib" dst: "/lib" is_bind: true -@@ -32,6 +39,7 @@ mount { - - - mount { + mandatory: false - src: "/lib64" - dst: "/lib64" - is_bind: true -@@ -39,6 +47,7 @@ mount { + } - mount { -+ mandatory: false +@@ -60,6 +69,7 @@ mount { src: "/usr" dst: "/usr" is_bind: true ++ mandatory: false + } + + mount { diff --git a/pkgs/by-name/wi/windmill/run.bun.config.proto.patch b/pkgs/by-name/wi/windmill/run.bun.config.proto.patch new file mode 100644 index 000000000000..20367ab458dc --- /dev/null +++ b/pkgs/by-name/wi/windmill/run.bun.config.proto.patch @@ -0,0 +1,37 @@ +diff --git a/windmill-worker/nsjail/run.bun.config.proto b/windmill-worker/nsjail/run.bun.config.proto +index 527d49cf7..b5e64beec 100644 +--- a/windmill-worker/nsjail/run.bun.config.proto ++++ b/windmill-worker/nsjail/run.bun.config.proto +@@ -17,16 +17,24 @@ clone_newcgroup: false + keep_caps: false + keep_env: true + ++mount { ++ src: "/nix/store" ++ dst: "/nix/store" ++ is_bind: true ++} ++ + mount { + src: "/bin" + dst: "/bin" + is_bind: true ++ mandatory: false + } + + mount { + src: "/lib" + dst: "/lib" + is_bind: true ++ mandatory: false + } + + mount { +@@ -47,6 +55,7 @@ mount { + src: "/usr" + dst: "/usr" + is_bind: true ++ mandatory: false + } + + mount { diff --git a/pkgs/by-name/wi/windmill/run.csharp.config.proto.patch b/pkgs/by-name/wi/windmill/run.csharp.config.proto.patch new file mode 100644 index 000000000000..954a8bb017b4 --- /dev/null +++ b/pkgs/by-name/wi/windmill/run.csharp.config.proto.patch @@ -0,0 +1,37 @@ +diff --git a/windmill-worker/nsjail/run.csharp.config.proto b/windmill-worker/nsjail/run.csharp.config.proto +index 389448eff..ee51c1bfb 100644 +--- a/windmill-worker/nsjail/run.csharp.config.proto ++++ b/windmill-worker/nsjail/run.csharp.config.proto +@@ -15,16 +15,24 @@ keep_caps: false + keep_env: true + mount_proc: true + ++mount { ++ src: "/nix/store" ++ dst: "/nix/store" ++ is_bind: true ++} ++ + mount { + src: "/bin" + dst: "/bin" + is_bind: true ++ mandatory: false + } + + mount { + src: "/lib" + dst: "/lib" + is_bind: true ++ mandatory: false + } + + +@@ -40,6 +48,7 @@ mount { + src: "/usr" + dst: "/usr" + is_bind: true ++ mandatory: false + } + + mount { diff --git a/pkgs/by-name/wi/windmill/run.go.config.proto.patch b/pkgs/by-name/wi/windmill/run.go.config.proto.patch index a5e3b68b07d7..99973f7ec2f3 100644 --- a/pkgs/by-name/wi/windmill/run.go.config.proto.patch +++ b/pkgs/by-name/wi/windmill/run.go.config.proto.patch @@ -1,34 +1,37 @@ diff --git a/windmill-worker/nsjail/run.go.config.proto b/windmill-worker/nsjail/run.go.config.proto -index 3af548d1..39ff4da7 100644 +index 1ba6f52d5..21614fabb 100644 --- a/windmill-worker/nsjail/run.go.config.proto +++ b/windmill-worker/nsjail/run.go.config.proto -@@ -25,6 +25,13 @@ mount { - } +@@ -14,16 +14,24 @@ clone_newuser: {CLONE_NEWUSER} + keep_caps: false + keep_env: true - mount { -+ src: "/nix/store" -+ dst: "/nix/store" ++mount { ++ src: "/nix/store" ++ dst: "/nix/store" + is_bind: true +} + -+mount { + mount { + src: "/bin" + dst: "/bin" + is_bind: true + mandatory: false + } + + mount { src: "/lib" dst: "/lib" is_bind: true -@@ -32,6 +39,7 @@ mount { - - - mount { + mandatory: false - src: "/lib64" - dst: "/lib64" - is_bind: true + } + + @@ -39,6 +47,7 @@ mount { - - - mount { -+ mandatory: false src: "/usr" dst: "/usr" is_bind: true ++ mandatory: false + } + + mount { diff --git a/pkgs/by-name/wi/windmill/run.php.config.proto.patch b/pkgs/by-name/wi/windmill/run.php.config.proto.patch new file mode 100644 index 000000000000..bd867b8b7871 --- /dev/null +++ b/pkgs/by-name/wi/windmill/run.php.config.proto.patch @@ -0,0 +1,37 @@ +diff --git a/windmill-worker/nsjail/run.php.config.proto b/windmill-worker/nsjail/run.php.config.proto +index d3752d33b..9e4b19793 100644 +--- a/windmill-worker/nsjail/run.php.config.proto ++++ b/windmill-worker/nsjail/run.php.config.proto +@@ -15,16 +15,24 @@ keep_caps: false + keep_env: true + mount_proc: true + ++mount { ++ src: "/nix/store" ++ dst: "/nix/store" ++ is_bind: true ++} ++ + mount { + src: "/bin" + dst: "/bin" + is_bind: true ++ mandatory: false + } + + mount { + src: "/lib" + dst: "/lib" + is_bind: true ++ mandatory: false + } + + +@@ -40,6 +48,7 @@ mount { + src: "/usr" + dst: "/usr" + is_bind: true ++ mandatory: false + } + + mount { diff --git a/pkgs/by-name/wi/windmill/run.powershell.config.proto.patch b/pkgs/by-name/wi/windmill/run.powershell.config.proto.patch new file mode 100644 index 000000000000..68fc124b6544 --- /dev/null +++ b/pkgs/by-name/wi/windmill/run.powershell.config.proto.patch @@ -0,0 +1,38 @@ +diff --git a/windmill-worker/nsjail/run.powershell.config.proto b/windmill-worker/nsjail/run.powershell.config.proto +index 93a48d4fe..97dc236e5 100644 +--- a/windmill-worker/nsjail/run.powershell.config.proto ++++ b/windmill-worker/nsjail/run.powershell.config.proto +@@ -15,10 +15,17 @@ keep_caps: false + keep_env: true + mount_proc: true + ++mount { ++ src: "/nix/store" ++ dst: "/nix/store" ++ is_bind: true ++} ++ + mount { + src: "/bin" + dst: "/bin" + is_bind: true ++ mandatory: false + } + + mount { +@@ -38,6 +45,7 @@ mount { + src: "/lib" + dst: "/lib" + is_bind: true ++ mandatory: false + } + + +@@ -53,6 +61,7 @@ mount { + src: "/usr" + dst: "/usr" + is_bind: true ++ mandatory: false + } + + mount { diff --git a/pkgs/by-name/wi/windmill/run.python3.config.proto.patch b/pkgs/by-name/wi/windmill/run.python3.config.proto.patch index 85e87d7b3433..454250875230 100644 --- a/pkgs/by-name/wi/windmill/run.python3.config.proto.patch +++ b/pkgs/by-name/wi/windmill/run.python3.config.proto.patch @@ -1,34 +1,37 @@ diff --git a/windmill-worker/nsjail/run.python3.config.proto b/windmill-worker/nsjail/run.python3.config.proto -index 9f106c23..9da2d2a8 100644 +index b49b9cfbf..35241bbd0 100644 --- a/windmill-worker/nsjail/run.python3.config.proto +++ b/windmill-worker/nsjail/run.python3.config.proto -@@ -27,6 +27,13 @@ mount { - } +@@ -18,16 +18,24 @@ keep_caps: false + keep_env: true + mount_proc: true - mount { -+ src: "/nix/store" -+ dst: "/nix/store" ++mount { ++ src: "/nix/store" ++ dst: "/nix/store" + is_bind: true +} + -+mount { + mount { + src: "/bin" + dst: "/bin" + is_bind: true + mandatory: false + } + + mount { src: "/lib" dst: "/lib" is_bind: true -@@ -34,6 +35,7 @@ mount { - - - mount { + mandatory: false - src: "/lib64" - dst: "/lib64" - is_bind: true -@@ -41,6 +43,7 @@ mount { + } - mount { -+ mandatory: false +@@ -43,6 +51,7 @@ mount { src: "/usr" dst: "/usr" is_bind: true ++ mandatory: false + } + + mount { diff --git a/pkgs/by-name/wi/windmill/run.rust.config.proto.patch b/pkgs/by-name/wi/windmill/run.rust.config.proto.patch new file mode 100644 index 000000000000..b5d4924ef083 --- /dev/null +++ b/pkgs/by-name/wi/windmill/run.rust.config.proto.patch @@ -0,0 +1,37 @@ +diff --git a/windmill-worker/nsjail/run.rust.config.proto b/windmill-worker/nsjail/run.rust.config.proto +index 3357cd88a..c0a1e9534 100644 +--- a/windmill-worker/nsjail/run.rust.config.proto ++++ b/windmill-worker/nsjail/run.rust.config.proto +@@ -14,16 +14,24 @@ clone_newuser: {CLONE_NEWUSER} + keep_caps: false + keep_env: true + ++mount { ++ src: "/nix/store" ++ dst: "/nix/store" ++ is_bind: true ++} ++ + mount { + src: "/bin" + dst: "/bin" + is_bind: true ++ mandatory: false + } + + mount { + src: "/lib" + dst: "/lib" + is_bind: true ++ mandatory: false + } + + +@@ -39,6 +47,7 @@ mount { + src: "/usr" + dst: "/usr" + is_bind: true ++ mandatory: false + } + + mount { diff --git a/pkgs/by-name/wi/windmill/rust_executor.patch b/pkgs/by-name/wi/windmill/rust_executor.patch new file mode 100644 index 000000000000..7fdd2fe8f1d0 --- /dev/null +++ b/pkgs/by-name/wi/windmill/rust_executor.patch @@ -0,0 +1,13 @@ +diff --git a/windmill-worker/src/rust_executor.rs b/windmill-worker/src/rust_executor.rs +index 95fd82210..ed7b298e1 100644 +--- a/windmill-worker/src/rust_executor.rs ++++ b/windmill-worker/src/rust_executor.rs +@@ -29,7 +29,7 @@ use crate::SYSTEM_ROOT; + const NSJAIL_CONFIG_RUN_RUST_CONTENT: &str = include_str!("../nsjail/run.rust.config.proto"); + + lazy_static::lazy_static! { +- static ref HOME_DIR: String = std::env::var("HOME").expect("Could not find the HOME environment variable"); ++ static ref HOME_DIR: String = std::env::var("HOME").unwrap_or_else(|_| format!("{}/cargo", windmill_common::worker::ROOT_CACHE_DIR)); + static ref CARGO_HOME: String = std::env::var("CARGO_HOME").unwrap_or_else(|_| { CARGO_HOME_DEFAULT.clone() }); + static ref RUSTUP_HOME: String = std::env::var("RUSTUP_HOME").unwrap_or_else(|_| { RUSTUP_HOME_DEFAULT.clone() }); + static ref CARGO_PATH: String = std::env::var("CARGO_PATH").unwrap_or_else(|_| format!("{}/bin/cargo", CARGO_HOME.as_str()));