diff --git a/pkgs/by-name/wi/windmill/package.nix b/pkgs/by-name/wi/windmill/package.nix index ebcc658b9e4a..42b63ea89e1e 100644 --- a/pkgs/by-name/wi/windmill/package.nix +++ b/pkgs/by-name/wi/windmill/package.nix @@ -16,7 +16,7 @@ pango, pixman, pkg-config, - python3, + python312, rustfmt, stdenv, perl, @@ -27,6 +27,7 @@ inherit (callPackage ./fetchers.nix { }) fetchLibrustyV8; } ), + ui_builder ? (callPackage ./ui_builder.nix { }), libxml2, xmlsec, libxslt, @@ -40,17 +41,18 @@ cargo, coreutils, withEnterpriseFeatures ? false, + withClosedSourceFeatures ? false, }: let pname = "windmill"; - version = "1.474.0"; + version = "1.542.1"; src = fetchFromGitHub { owner = "windmill-labs"; repo = "windmill"; rev = "v${version}"; - hash = "sha256-9BtItmqyW4NbG4istssAYn4CWlfYAv33CE1enL+5LtE="; + hash = "sha256-SVE1P7NQ6mz26bS3oURAc1ImvbCJRkT8mMTuCmHeZLU="; }; in rustPlatform.buildRustPackage (finalAttrs: { @@ -63,44 +65,64 @@ rustPlatform.buildRustPackage (finalAttrs: { RUSTY_V8_ARCHIVE = librusty_v8; }; - cargoHash = "sha256-6htM6p09mPUQmS+QVBDO7Y/tuwweHgA+W/E3XTNunB8="; + cargoHash = "sha256-0OETpBXO/c2YJPta1LLAa/Jf7PusH+GAwgfAmRGh9m0="; buildFeatures = [ - "embedding" - "parquet" - "prometheus" - "openidconnect" + "agent_worker_server" + # "benchmark" # DO NOT ACTIVATE, this is for benchmark testing + #"bigquery" "cloud" - "jemalloc" - "deno_core" - "license" - "http_trigger" - "zip" - "oauth2" - "kafka" - "otel" - "dind" - "php" - "mysql" - "mssql" - "bigquery" - "websocket" - "python" - "smtp" "csharp" + "default" + "deno_core" + "dind" + #"duckdb" + "embedding" + "flow_testing" + "gcp_trigger" + "http_trigger" + #"java" + "jemalloc" + "kafka" + "license" + "loki" + "mcp" + "mqtt_trigger" + #"mssql" + #"mysql" + "nats" + #"nu" + "oauth2" + "openidconnect" + #"oracledb" + "parquet" + "php" + "postgres_trigger" + "python" + #"ruby" + #"rust" # compiler environment is incomplete + "scoped_cache" + "smtp" + "sqlx" + "sqs_trigger" "static_frontend" - # "rust" # compiler environment is incomplete + "websocket" + "zip" ] ++ (lib.optionals withEnterpriseFeatures [ - "enterprise" "enterprise_saml" - "tantivy" + "enterprise" + "otel" + "prometheus" "stripe" - ]); + "tantivy" + ]) + ++ (lib.optionals withClosedSourceFeatures [ "private" ]); patches = [ ./download.py.config.proto.patch ./python_executor.patch + ./python_versions.patch ./run.ansible.config.proto.patch ./run.bash.config.proto.patch ./run.bun.config.proto.patch @@ -111,15 +133,11 @@ rustPlatform.buildRustPackage (finalAttrs: { ./run.python3.config.proto.patch ./run.rust.config.proto.patch ./rust_executor.patch - ./swagger-cli.patch ]; postPatch = '' substituteInPlace windmill-common/src/utils.rs \ --replace-fail 'unknown-version' 'v${version}' - - substituteInPlace windmill-worker/src/python_executor.rs \ - --replace-fail 'unknown_system_python_version' '${python3.version}' ''; buildInputs = [ @@ -142,19 +160,19 @@ rustPlatform.buildRustPackage (finalAttrs: { # needs a postgres database running doCheck = false; - # TODO; Check if the rpath is still required - # patchelf --set-rpath ${lib.makeLibraryPath [ openssl ]} $out/bin/windmill postFixup = '' wrapProgram "$out/bin/windmill" \ --prefix LD_LIBRARY_PATH : ${lib.makeLibraryPath [ stdenv.cc.cc ]} \ --prefix PATH : ${ lib.makeBinPath [ - python3 # uv searches PATH for system python + # uv searches for python on path as well! + python312 + procps # bash_executor coreutils # bash_executor ] } \ - --set PYTHON_PATH "${python3}/bin/python3" \ + --set PYTHON_PATH "${python312}/bin/python3" \ --set GO_PATH "${go}/bin/go" \ --set DENO_PATH "${deno}/bin/deno" \ --set NSJAIL_PATH "${nsjail}/bin/nsjail" \ @@ -176,7 +194,7 @@ rustPlatform.buildRustPackage (finalAttrs: { sourceRoot = "${src.name}/frontend"; - npmDepsHash = "sha256-liWoAgAIgU8+J1x2mR7bGl9MOpCuGIf0Qa1nEouFnBU="; + npmDepsHash = "sha256-8t5ww8zuP1ZWASKpj6F0bFGWnUDROf7iLBTLsof5jtM="; # without these you get a # FATAL ERROR: Ineffective mark-compacts near heap limit Allocation failed - JavaScript heap out of memory @@ -186,6 +204,12 @@ rustPlatform.buildRustPackage (finalAttrs: { cp ${src}/openflow.openapi.yaml . ''; + # WORKS + npmFlags = [ + # Skip "postinstall" script that attempts to download and unpack ui-builder (patching out the url with nix-store path doesn't work) + "--ignore-scripts" + ]; + preBuild = '' npm run generate-backend-client ''; @@ -202,6 +226,9 @@ rustPlatform.buildRustPackage (finalAttrs: { installPhase = '' mkdir -p $out/share mv build $out/share/windmill-frontend + + mkdir -p $out/share/windmill-frontend/static + ln -s ${ui_builder} $out/share/windmill-frontend/static/ui_builder ''; }; @@ -213,6 +240,7 @@ rustPlatform.buildRustPackage (finalAttrs: { ]; }) (./update-librusty.sh) + (./update-ui_builder.sh) ]; meta = { diff --git a/pkgs/by-name/wi/windmill/python_executor.patch b/pkgs/by-name/wi/windmill/python_executor.patch index 14a1f502e725..8e45cefe0699 100644 --- a/pkgs/by-name/wi/windmill/python_executor.patch +++ b/pkgs/by-name/wi/windmill/python_executor.patch @@ -1,100 +1,31 @@ diff --git a/windmill-worker/src/python_executor.rs b/windmill-worker/src/python_executor.rs -index 1180652d3..7b4ccf6cf 100644 +index eb6a6e7f30..eb29758c0a 100644 --- a/windmill-worker/src/python_executor.rs +++ b/windmill-worker/src/python_executor.rs -@@ -290,70 +290,8 @@ impl PyVersion { - occupancy_metrics: &mut Option<&mut OccupancyMetrics>, - ) -> error::Result<()> { - let v = self.to_string_with_dot(); -- append_logs(job_id, w_id, format!("\nINSTALLING PYTHON ({})", v), db).await; -- // Create dirs for newly installed python -- // If we dont do this, NSJAIL will not be able to mount cache -- // For the default version directory created during startup (main.rs) -- DirBuilder::new() -- .recursive(true) -- .create(self.to_cache_dir()) -- .await -- .expect("could not create initial worker dir"); -- -- let logs = String::new(); -- -- #[cfg(windows)] -- let uv_cmd = "uv"; -- -- #[cfg(unix)] -- let uv_cmd = UV_PATH.as_str(); -- -- let mut child_cmd = Command::new(uv_cmd); -- child_cmd -- .env_clear() -- .env("HOME", HOME_ENV.to_string()) -- .env("PATH", PATH_ENV.to_string()) -- .envs(PROXY_ENVS.clone()) -- .args(["python", "install", v, "--python-preference=only-managed"]) -- // TODO: Do we need these? -- .envs([("UV_PYTHON_INSTALL_DIR", PY_INSTALL_DIR)]) -- .stdout(Stdio::piped()) -- .stderr(Stdio::piped()); -- -- #[cfg(windows)] -- { -- child_cmd -- .env("SystemRoot", SYSTEM_ROOT.as_str()) -- .env("USERPROFILE", crate::USERPROFILE_ENV.as_str()) -- .env( -- "TMP", -- std::env::var("TMP").unwrap_or_else(|_| String::from("/tmp")), -- ) -- .env( -- "LOCALAPPDATA", -- std::env::var("LOCALAPPDATA") -- .unwrap_or_else(|_| format!("{}\\AppData\\Local", HOME_ENV.as_str())), -- ); -- } -- -- let child_process = start_child_process(child_cmd, "uv").await?; -- -- append_logs(&job_id, &w_id, logs, db).await; -- handle_child( -- job_id, -- db, -- mem_peak, -- &mut None, -- child_process, -- false, -- worker_name, -- &w_id, -- "uv", -- None, -- false, -- occupancy_metrics, -- ) -- .await -+ append_logs(job_id, w_id, format!("\nREQUESTED PYTHON INSTALL IGNORED ({})", v), db).await; -+ Err(error::Error::BadConfig(format!("Python is managed through the NixOS system configuration. Change the Windmill instance setting to version 'unknown_system_python_version'"))) - } - async fn find_python(self) -> error::Result> { - #[cfg(windows)] -@@ -391,11 +329,11 @@ impl PyVersion { - "find", - self.to_string_with_dot(), - "--system", -- "--python-preference=only-managed", -+ "--python-preference=only-system", - ]) - .envs([ - ("UV_PYTHON_INSTALL_DIR", PY_INSTALL_DIR), -- ("UV_PYTHON_PREFERENCE", "only-managed"), -+ ("UV_PYTHON_PREFERENCE", "only-system"), - ]) - // .stdout(Stdio::piped()) - .stderr(Stdio::piped()) -@@ -561,7 +499,7 @@ pub async fn uv_pip_compile( - "-p", - &py_version.to_string_with_dot(), - "--python-preference", -- "only-managed", -+ "only-system", - ]); +@@ -273,7 +273,7 @@ pub async fn uv_pip_compile( + UV_CACHE_DIR, + ]; + +- args.extend(["-p", &py_version_str, "--python-preference", "only-managed"]); ++ args.extend(["-p", &py_version_str, "--python-preference", "only-system"]); if no_cache { + args.extend(["--no-cache"]); +@@ -1332,7 +1332,7 @@ async fn spawn_uv_install( + let _owner; + if let Some(py_path) = py_path.as_ref() { + _owner = format!( +- "-p {} --python-preference only-managed", ++ "-p {} --python-preference only-system", + py_path.as_str() // + ); + vars.push(("PY_PATH", &_owner)); +@@ -1399,7 +1399,7 @@ async fn spawn_uv_install( + "-p", + py_path.as_str(), + "--python-preference", +- "only-managed", // ++ "only-system", // + ]); + } else { + command_args.extend([ diff --git a/pkgs/by-name/wi/windmill/python_versions.patch b/pkgs/by-name/wi/windmill/python_versions.patch new file mode 100644 index 000000000000..aad77e5c1681 --- /dev/null +++ b/pkgs/by-name/wi/windmill/python_versions.patch @@ -0,0 +1,37 @@ +diff --git a/windmill-worker/src/python_versions.rs b/windmill-worker/src/python_versions.rs +index 3d71e9f9ec..c9166b52c6 100644 +--- a/windmill-worker/src/python_versions.rs ++++ b/windmill-worker/src/python_versions.rs +@@ -326,6 +326,7 @@ impl PyV { + } + + pub async fn list_available_python_versions() -> Vec { ++ return vec![PyVAlias::Py312.into()]; + match Self::list_available_python_versions_inner().await { + Ok(pyvs) => pyvs, + Err(e) => { +@@ -591,6 +592,10 @@ impl PyV { + w_id: &str, + occupancy_metrics: &mut Option<&mut OccupancyMetrics>, + ) -> error::Result<()> { ++ let v = self.to_string(); ++ append_logs(job_id, w_id, format!("\nREQUESTED PYTHON INSTALL IGNORED ({})", v), conn).await; ++ return Err(error::Error::BadConfig(format!("Python is managed through the NixOS system configuration. Change the Windmill instance setting to version '3.12'"))); ++ + let v = self.to_string(); + append_logs(job_id, w_id, format!("\nINSTALLING PYTHON ({})", v), conn).await; + // Create dirs for newly installed python +@@ -695,11 +700,11 @@ impl PyV { + "find", + &self.to_string(), + "--system", +- "--python-preference=only-managed", ++ "--python-preference=only-system", + ]) + .envs([ + ("UV_PYTHON_INSTALL_DIR", PY_INSTALL_DIR), +- ("UV_PYTHON_PREFERENCE", "only-managed"), ++ ("UV_PYTHON_PREFERENCE", "only-system"), + ]) + // .stdout(Stdio::piped()) + .stderr(Stdio::piped()) diff --git a/pkgs/by-name/wi/windmill/ui_builder.nix b/pkgs/by-name/wi/windmill/ui_builder.nix new file mode 100644 index 000000000000..26cce2480333 --- /dev/null +++ b/pkgs/by-name/wi/windmill/ui_builder.nix @@ -0,0 +1,8 @@ +# auto-generated file -- DO NOT EDIT! +{ fetchzip }: + +fetchzip { + url = "https://pub-06154ed168a24e73a86ab84db6bf15d8.r2.dev/ui_builder-d44b577.tar.gz"; + sha256 = "0wx9jq9h092c7y4x7dycvj10wna2q22062nvvzzb6646s6q2iycg"; + stripRoot = false; +} diff --git a/pkgs/by-name/wi/windmill/update-librusty.sh b/pkgs/by-name/wi/windmill/update-librusty.sh index 7ee601b4c573..0470804bccbf 100755 --- a/pkgs/by-name/wi/windmill/update-librusty.sh +++ b/pkgs/by-name/wi/windmill/update-librusty.sh @@ -1,14 +1,19 @@ #!/usr/bin/env nix-shell -#!nix-shell -i bash -p gnugrep gnused nix +#!nix-shell -i bash -p gnugrep gnused nix jq # shellcheck shell=bash set -eu -o pipefail -# ASSUMES; The Cargo.lock file inside this directory holds the correct librusty version +echo "librusty_v8: UPDATING" + +# ASSUMES; The Cargo.lock file is located at /backend/Cargo.toml + +WINDMILL_LATEST_VERSION=$(curl ${GITHUB_TOKEN:+-u ":$GITHUB_TOKEN"} --silent --fail --location "https://api.github.com/repos/windmill-labs/windmill/releases/latest" | jq --raw-output .tag_name) +CARGO_LOCK=$(curl ${GITHUB_TOKEN:+-u ":$GITHUB_TOKEN"} --silent --fail --location "https://github.com/windmill-labs/windmill/raw/$WINDMILL_LATEST_VERSION/backend/Cargo.lock") PACKAGE_DIR=$(dirname "$(readlink --canonicalize-existing "${BASH_SOURCE[0]}")") OUTPUT_FILE="$PACKAGE_DIR/librusty_v8.nix" -NEW_VERSION="$(grep --after-context 5 'name = "v8"' "$PACKAGE_DIR/Cargo.lock" | grep 'version =' | sed -E 's/version = "//;s/"//')" +NEW_VERSION=$(echo "$CARGO_LOCK" | grep --after-context 5 'name = "v8"' | grep 'version =' | sed -E 's/version = "//;s/"//') CURRENT_VERSION="" if [ -f "$OUTPUT_FILE" ]; then @@ -36,3 +41,5 @@ fetchLibrustyV8 { EOF mv "$TEMP_FILE" "$OUTPUT_FILE" + +echo "librusty_v8: UPDATE DONE" diff --git a/pkgs/by-name/wi/windmill/update-ui_builder.sh b/pkgs/by-name/wi/windmill/update-ui_builder.sh new file mode 100755 index 000000000000..5a023d4ca7b5 --- /dev/null +++ b/pkgs/by-name/wi/windmill/update-ui_builder.sh @@ -0,0 +1,33 @@ +#!/usr/bin/env nix-shell +#!nix-shell -i bash -p gnugrep gnused nix jq +# shellcheck shell=bash + +set -eu -o pipefail + +echo "Windmill UI builder: UPDATING" + +# ASSUMES; The UI builder package is referenced in post-install script "untar_ui_builder.js" +# REF; https://github.com/windmill-labs/windmill/blob/205618af0a60b6bf5fcdd80f2b23c3b490ddbc00/frontend/scripts/untar_ui_builder.js#L23 + +WINDMILL_LATEST_VERSION=$(curl ${GITHUB_TOKEN:+-u ":$GITHUB_TOKEN"} --silent --fail --location "https://api.github.com/repos/windmill-labs/windmill/releases/latest" | jq --raw-output .tag_name) +UI_BUILDER_SCRIPT=$(curl ${GITHUB_TOKEN:+-u ":$GITHUB_TOKEN"} --silent --fail --location "https://github.com/windmill-labs/windmill/raw/$WINDMILL_LATEST_VERSION/frontend/scripts/untar_ui_builder.js") + +PACKAGE_DIR=$(dirname "$(readlink --canonicalize-existing "${BASH_SOURCE[0]}")") +OUTPUT_FILE="$PACKAGE_DIR/ui_builder.nix" +NEW_URL=$(echo "$UI_BUILDER_SCRIPT" | sed --quiet --regexp-extended "s/.*tarUrl\s*=\s*'([^']+)['].*/\1/p") + +TEMP_FILE="$OUTPUT_FILE.tmp" +cat > "$TEMP_FILE" <