diff --git a/.github/DockerfileBackendTests b/.github/DockerfileBackendTests index 03f51de3327aa..c3990c2b4d8ce 100644 --- a/.github/DockerfileBackendTests +++ b/.github/DockerfileBackendTests @@ -43,7 +43,8 @@ RUN wget https://golang.org/dl/go1.21.5.linux-amd64.tar.gz && tar -C /usr/local ENV PATH="${PATH}:/usr/local/go/bin" ENV GO_PATH=/usr/local/go/bin/go -RUN curl -LsSf https://astral.sh/uv/install.sh | sh +# Install UV +RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.4.18/uv-installer.sh | sh ENV TZ=Etc/UTC diff --git a/Dockerfile b/Dockerfile index 3bc5088aedf27..d5d4b8cc92676 100644 --- a/Dockerfile +++ b/Dockerfile @@ -158,6 +158,9 @@ RUN set -eux; \ ENV PATH="${PATH}:/usr/local/go/bin" ENV GO_PATH=/usr/local/go/bin/go +# Install UV +RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.4.18/uv-installer.sh | sh + RUN curl -sL https://deb.nodesource.com/setup_20.x | bash - RUN apt-get -y update && apt-get install -y curl nodejs awscli && apt-get clean \ && rm -rf /var/lib/apt/lists/* diff --git a/backend/src/main.rs b/backend/src/main.rs index d7b2d776d685d..c99e551ddb144 100644 --- a/backend/src/main.rs +++ b/backend/src/main.rs @@ -67,7 +67,7 @@ use windmill_worker::{ get_hub_script_content_and_requirements, BUN_BUNDLE_CACHE_DIR, BUN_CACHE_DIR, BUN_DEPSTAR_CACHE_DIR, DENO_CACHE_DIR, DENO_CACHE_DIR_DEPS, DENO_CACHE_DIR_NPM, GO_BIN_CACHE_DIR, GO_CACHE_DIR, LOCK_CACHE_DIR, PIP_CACHE_DIR, POWERSHELL_CACHE_DIR, - RUST_CACHE_DIR, TAR_PIP_CACHE_DIR, TMP_LOGS_DIR, + RUST_CACHE_DIR, TAR_PIP_CACHE_DIR, TMP_LOGS_DIR, UV_CACHE_DIR, }; use crate::monitor::{ @@ -873,6 +873,7 @@ pub async fn run_workers( }; let lang = if &ns.language == &ScriptLang::Bun || &ns.language == &ScriptLang::Bunnative { - let anns = get_annotation(&ns.content); + let anns = get_annotation_ts(&ns.content); if anns.native_mode { ScriptLang::Bunnative } else { diff --git a/backend/windmill-common/src/worker.rs b/backend/windmill-common/src/worker.rs index 385fac616313c..89f2c148e0969 100644 --- a/backend/windmill-common/src/worker.rs +++ b/backend/windmill-common/src/worker.rs @@ -303,14 +303,14 @@ fn parse_file(path: &str) -> Option { .flatten() } -pub struct Annotations { +pub struct TypeScriptAnnotations { pub npm_mode: bool, pub nodejs_mode: bool, pub native_mode: bool, pub nobundling: bool, } -pub fn get_annotation(inner_content: &str) -> Annotations { +pub fn get_annotation_ts(inner_content: &str) -> TypeScriptAnnotations { let annotations = inner_content .lines() .take_while(|x| x.starts_with("//")) @@ -324,7 +324,25 @@ pub fn get_annotation(inner_content: &str) -> Annotations { let nobundling: bool = annotations.contains(&"nobundling".to_string()) || nodejs_mode || *DISABLE_BUNDLING; - Annotations { npm_mode, nodejs_mode, native_mode, nobundling } + TypeScriptAnnotations { npm_mode, nodejs_mode, native_mode, nobundling } +} + +pub struct PythonAnnotations { + pub no_uv: bool, + pub no_cache: bool, +} + +pub fn get_annotation_python(inner_content: &str) -> PythonAnnotations { + let annotations = inner_content + .lines() + .take_while(|x| x.starts_with("#")) + .map(|x| x.to_string().replace("#", "").trim().to_string()) + .collect_vec(); + + let no_uv: bool = annotations.contains(&"no_uv".to_string()); + let no_cache: bool = annotations.contains(&"no_cache".to_string()); + + PythonAnnotations { no_uv, no_cache } } pub struct SqlAnnotations { diff --git a/backend/windmill-worker/src/ansible_executor.rs b/backend/windmill-worker/src/ansible_executor.rs index 7df394a77c59b..9c2474d95f257 100644 --- a/backend/windmill-worker/src/ansible_executor.rs +++ b/backend/windmill-worker/src/ansible_executor.rs @@ -33,7 +33,7 @@ use crate::{ OccupancyMetrics, }, handle_child::handle_child, - python_executor::{create_dependencies_dir, handle_python_reqs, pip_compile}, + python_executor::{create_dependencies_dir, handle_python_reqs, uv_pip_compile}, AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV, TZ_ENV, }; @@ -80,7 +80,7 @@ async fn handle_ansible_python_deps( if requirements.is_empty() { "".to_string() } else { - pip_compile( + uv_pip_compile( job_id, &requirements, mem_peak, @@ -90,6 +90,8 @@ async fn handle_ansible_python_deps( worker_name, w_id, &mut Some(occupancy_metrics), + false, + false, ) .await .map_err(|e| { diff --git a/backend/windmill-worker/src/bun_executor.rs b/backend/windmill-worker/src/bun_executor.rs index 7fc786277d005..c45d4c285365d 100644 --- a/backend/windmill-worker/src/bun_executor.rs +++ b/backend/windmill-worker/src/bun_executor.rs @@ -47,7 +47,7 @@ use windmill_common::{ get_latest_hash_for_path, jobs::{QueuedJob, PREPROCESSOR_FAKE_ENTRYPOINT}, scripts::ScriptLang, - worker::{exists_in_cache, get_annotation, save_cache, write_file}, + worker::{exists_in_cache, get_annotation_ts, save_cache, write_file}, DB, }; @@ -663,7 +663,7 @@ pub async fn prebundle_bun_script( if exists_in_cache(&local_path, &remote_path).await { return Ok(()); } - let annotation = get_annotation(inner_content); + let annotation = get_annotation_ts(inner_content); if annotation.nobundling { return Ok(()); } @@ -800,7 +800,7 @@ pub async fn handle_bun_job( new_args: &mut Option>>, occupancy_metrics: &mut OccupancyMetrics, ) -> error::Result> { - let mut annotation = windmill_common::worker::get_annotation(inner_content); + let mut annotation = windmill_common::worker::get_annotation_ts(inner_content); let (mut has_bundle_cache, cache_logs, local_path, remote_path) = if requirements_o.is_some() && !annotation.nobundling && codebase.is_none() { @@ -1525,7 +1525,7 @@ pub async fn start_worker( let common_bun_proc_envs: HashMap = get_common_bun_proc_envs(Some(&base_internal_url)).await; - let mut annotation = windmill_common::worker::get_annotation(inner_content); + let mut annotation = windmill_common::worker::get_annotation_ts(inner_content); //TODO: remove this when bun dedicated workers work without issues annotation.nodejs_mode = true; diff --git a/backend/windmill-worker/src/python_executor.rs b/backend/windmill-worker/src/python_executor.rs index a5717ed12bb08..baefdfd9bc6ff 100644 --- a/backend/windmill-worker/src/python_executor.rs +++ b/backend/windmill-worker/src/python_executor.rs @@ -36,6 +36,9 @@ lazy_static::lazy_static! { static ref PIP_TRUSTED_HOST: Option = std::env::var("PIP_TRUSTED_HOST").ok(); static ref PIP_INDEX_CERT: Option = std::env::var("PIP_INDEX_CERT").ok(); + static ref USE_PIP_COMPILE: bool = std::env::var("USE_PIP_COMPILE") + .ok().map(|flag| flag == "true").unwrap_or(false); + static ref RELATIVE_IMPORT_REGEX: Regex = Regex::new(r#"(import|from)\s(((u|f)\.)|\.)"#).unwrap(); @@ -61,7 +64,7 @@ use crate::{ handle_child::handle_child, AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, HTTPS_PROXY, HTTP_PROXY, LOCK_CACHE_DIR, NO_PROXY, NSJAIL_PATH, PATH_ENV, PIP_CACHE_DIR, PIP_EXTRA_INDEX_URL, - PIP_INDEX_URL, TZ_ENV, + PIP_INDEX_URL, TZ_ENV, UV_CACHE_DIR, }; #[cfg(windows)] @@ -96,7 +99,7 @@ pub fn handle_ephemeral_token(x: String) -> String { x } -pub async fn pip_compile( +pub async fn uv_pip_compile( job_id: &Uuid, requirements: &str, mem_peak: &mut i32, @@ -106,6 +109,10 @@ pub async fn pip_compile( worker_name: &str, w_id: &str, occupancy_metrics: &mut Option<&mut OccupancyMetrics>, + // Fallback to pip-compile. Will be removed in future + mut no_uv: bool, + // Debug-only flag + no_cache: bool, ) -> error::Result { let mut logs = String::new(); logs.push_str(&format!("\nresolving dependencies...")); @@ -142,83 +149,178 @@ pub async fn pip_compile( #[cfg(feature = "enterprise")] let requirements = replace_pip_secret(db, w_id, &requirements, worker_name, job_id).await?; - let req_hash = format!("py-{}", calculate_hash(&requirements)); - if let Some(cached) = sqlx::query_scalar!( - "SELECT lockfile FROM pip_resolution_cache WHERE hash = $1", - req_hash - ) - .fetch_optional(db) - .await? - { - logs.push_str(&format!("\nfound cached resolution: {req_hash}")); - return Ok(cached); + let mut req_hash = format!("py-{}", calculate_hash(&requirements)); + + if no_uv || *USE_PIP_COMPILE { + logs.push_str(&format!("\nFallback to pip-compile (Deprecated!)")); + // Set no_uv if not setted + no_uv = true; + // Make sure that if we put #no_uv (switch to pip-compile) to python code or used `USE_PIP_COMPILE=true` variable. + // Windmill will recalculate lockfile using pip-compile and dont take potentially broken lockfile (generated by uv) from cache (our db). + // It will recalculate lockfile even if inputs have not been changed. + req_hash.push_str("-no_uv"); + // Will be in format: + // py-000..000-no_uv + } + if !no_cache { + if let Some(cached) = sqlx::query_scalar!( + "SELECT lockfile FROM pip_resolution_cache WHERE hash = $1", + req_hash + ) + .fetch_optional(db) + .await? + { + logs.push_str(&format!("\nfound cached resolution: {req_hash}")); + return Ok(cached); + } } let file = "requirements.in"; write_file(job_dir, file, &requirements)?; - let mut args = vec![ - "-q", - "--no-header", - file, - "--resolver=backtracking", - "--strip-extras", - ]; - let mut pip_args = vec![]; - let pip_extra_index_url = PIP_EXTRA_INDEX_URL - .read() + // Fallback pip-compile. Will be removed in future + if no_uv { + tracing::debug!("Fallback to pip-compile"); + + let mut args = vec![ + "-q", + "--no-header", + file, + "--resolver=backtracking", + "--strip-extras", + ]; + let mut pip_args = vec![]; + let pip_extra_index_url = PIP_EXTRA_INDEX_URL + .read() + .await + .clone() + .map(handle_ephemeral_token); + if let Some(url) = pip_extra_index_url.as_ref() { + args.extend(["--extra-index-url", url, "--no-emit-index-url"]); + pip_args.push(format!("--extra-index-url {}", url)); + } + let pip_index_url = PIP_INDEX_URL + .read() + .await + .clone() + .map(handle_ephemeral_token); + if let Some(url) = pip_index_url.as_ref() { + args.extend(["--index-url", url, "--no-emit-index-url"]); + pip_args.push(format!("--index-url {}", url)); + } + if let Some(host) = PIP_TRUSTED_HOST.as_ref() { + args.extend(["--trusted-host", host]); + } + if let Some(cert_path) = PIP_INDEX_CERT.as_ref() { + args.extend(["--cert", cert_path]); + } + let pip_args_str = pip_args.join(" "); + if pip_args.len() > 0 { + args.extend(["--pip-args", &pip_args_str]); + } + tracing::debug!("pip-compile args: {:?}", args); + + let mut child_cmd = Command::new("pip-compile"); + child_cmd + .current_dir(job_dir) + .args(args) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + let child_process = start_child_process(child_cmd, "pip-compile").await?; + append_logs(&job_id, &w_id, logs, db).await; + handle_child( + job_id, + db, + mem_peak, + canceled_by, + child_process, + false, + worker_name, + &w_id, + "pip-compile", + None, + false, + occupancy_metrics, + ) .await - .clone() - .map(handle_ephemeral_token); - if let Some(url) = pip_extra_index_url.as_ref() { - args.extend(["--extra-index-url", url, "--no-emit-index-url"]); - pip_args.push(format!("--extra-index-url {}", url)); - } - let pip_index_url = PIP_INDEX_URL - .read() + .map_err(|e| Error::ExecutionErr(format!("Lock file generation failed: {e:?}")))?; + } else { + let mut args = vec![ + "pip", + "compile", + "-q", + "--no-header", + file, + "--strip-extras", + "-o", + "requirements.txt", + // Prefer main index over extra + // https://docs.astral.sh/uv/pip/compatibility/#packages-that-exist-on-multiple-indexes + // TODO: Use env variable that can be toggled from UI + "--index-strategy", + "unsafe-best-match", + // Target to /tmp/windmill/cache/uv + "--cache-dir", + UV_CACHE_DIR, + // We dont want UV to manage python installations + "--python-preference", + "only-system", + "--no-python-downloads", + ]; + if no_cache { + args.extend(["--no-cache"]); + } + let pip_extra_index_url = PIP_EXTRA_INDEX_URL + .read() + .await + .clone() + .map(handle_ephemeral_token); + if let Some(url) = pip_extra_index_url.as_ref() { + args.extend(["--extra-index-url", url]); + } + let pip_index_url = PIP_INDEX_URL + .read() + .await + .clone() + .map(handle_ephemeral_token); + if let Some(url) = pip_index_url.as_ref() { + args.extend(["--index-url", url]); + } + if let Some(host) = PIP_TRUSTED_HOST.as_ref() { + args.extend(["--trusted-host", host]); + } + if let Some(cert_path) = PIP_INDEX_CERT.as_ref() { + args.extend(["--cert", cert_path]); + } + tracing::debug!("uv args: {:?}", args); + + let mut child_cmd = Command::new("uv"); + child_cmd + .current_dir(job_dir) + .args(args) + .stdout(Stdio::piped()) + .stderr(Stdio::piped()); + let child_process = start_child_process(child_cmd, "uv").await?; + append_logs(&job_id, &w_id, logs, db).await; + handle_child( + job_id, + db, + mem_peak, + canceled_by, + child_process, + false, + worker_name, + &w_id, + // TODO: Rename to uv-pip-compile? + "uv", + None, + false, + occupancy_metrics, + ) .await - .clone() - .map(handle_ephemeral_token); - if let Some(url) = pip_index_url.as_ref() { - args.extend(["--index-url", url, "--no-emit-index-url"]); - pip_args.push(format!("--index-url {}", url)); - } - if let Some(host) = PIP_TRUSTED_HOST.as_ref() { - args.extend(["--trusted-host", host]); - } - if let Some(cert_path) = PIP_INDEX_CERT.as_ref() { - args.extend(["--cert", cert_path]); - } - let pip_args_str = pip_args.join(" "); - if pip_args.len() > 0 { - args.extend(["--pip-args", &pip_args_str]); + .map_err(|e| Error::ExecutionErr(format!("Lock file generation failed: {e:?}")))?; } - tracing::debug!("pip-compile args: {:?}", args); - - let mut child_cmd = Command::new("pip-compile"); - child_cmd - .current_dir(job_dir) - .args(args) - .stdout(Stdio::piped()) - .stderr(Stdio::piped()); - let child_process = start_child_process(child_cmd, "pip-compile").await?; - append_logs(&job_id, &w_id, logs, db).await; - handle_child( - job_id, - db, - mem_peak, - canceled_by, - child_process, - false, - worker_name, - &w_id, - "pip-compile", - None, - false, - occupancy_metrics, - ) - .await - .map_err(|e| Error::ExecutionErr(format!("Lock file generation failed: {e:?}")))?; + let path_lock = format!("{job_dir}/requirements.txt"); let mut file = File::open(path_lock).await?; let mut req_content = "".to_string(); @@ -784,6 +886,7 @@ async fn handle_python_deps( let requirements = match requirements_o { Some(r) => r, None => { + let annotation = windmill_common::worker::get_annotation_python(inner_content); let mut already_visited = vec![]; let requirements = windmill_parser_py_imports::parse_python_imports( @@ -798,7 +901,7 @@ async fn handle_python_deps( if requirements.is_empty() { "".to_string() } else { - pip_compile( + uv_pip_compile( job_id, &requirements, mem_peak, @@ -808,6 +911,8 @@ async fn handle_python_deps( worker_name, w_id, occupancy_metrics, + annotation.no_uv, + annotation.no_cache, ) .await .map_err(|e| { diff --git a/backend/windmill-worker/src/worker.rs b/backend/windmill-worker/src/worker.rs index 918c563ea31d0..83e8adff25a0d 100644 --- a/backend/windmill-worker/src/worker.rs +++ b/backend/windmill-worker/src/worker.rs @@ -237,6 +237,7 @@ pub const ROOT_CACHE_NOMOUNT_DIR: &str = concatcp!(TMP_DIR, "/cache_nomount/"); pub const LOCK_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "lock"); pub const PIP_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "pip"); +pub const UV_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "uv"); pub const TAR_PIP_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "tar/pip"); pub const DENO_CACHE_DIR: &str = concatcp!(ROOT_CACHE_DIR, "deno"); pub const DENO_CACHE_DIR_DEPS: &str = concatcp!(ROOT_CACHE_DIR, "deno/deps"); diff --git a/backend/windmill-worker/src/worker_lockfiles.rs b/backend/windmill-worker/src/worker_lockfiles.rs index 6f25ec606bedb..ed89c535e8eb2 100644 --- a/backend/windmill-worker/src/worker_lockfiles.rs +++ b/backend/windmill-worker/src/worker_lockfiles.rs @@ -12,7 +12,7 @@ use windmill_common::flows::{FlowModule, FlowModuleValue}; use windmill_common::get_latest_deployed_hash_for_path; use windmill_common::jobs::JobPayload; use windmill_common::scripts::ScriptHash; -use windmill_common::worker::{get_annotation, to_raw_value, to_raw_value_owned, write_file}; +use windmill_common::worker::{get_annotation_ts, to_raw_value, to_raw_value_owned, write_file}; use windmill_common::{ error::{self, to_anyhow}, flows::FlowValue, @@ -26,7 +26,7 @@ use windmill_parser_ts::parse_expr_for_imports; use windmill_queue::{append_logs, CanceledBy, PushIsolationLevel}; use crate::common::OccupancyMetrics; -use crate::python_executor::{create_dependencies_dir, handle_python_reqs, pip_compile}; +use crate::python_executor::{create_dependencies_dir, handle_python_reqs, uv_pip_compile}; use crate::rust_executor::{build_rust_crate, compute_rust_hash, generate_cargo_lockfile}; use crate::{ bun_executor::gen_bun_lockfile, @@ -953,7 +953,7 @@ async fn lock_modules<'c>( } if language == ScriptLang::Bun || language == ScriptLang::Bunnative { - let anns = get_annotation(&content); + let anns = get_annotation_ts(&content); if anns.native_mode && language == ScriptLang::Bun { language = ScriptLang::Bunnative; } else if !anns.native_mode && language == ScriptLang::Bunnative { @@ -1003,7 +1003,7 @@ async fn lock_modules<'c>( fn skip_creating_new_lock(language: &ScriptLang, content: &str) -> bool { if language == &ScriptLang::Bun || language == &ScriptLang::Bunnative { - let anns = get_annotation(&content); + let anns = get_annotation_ts(&content); if anns.native_mode && language == &ScriptLang::Bun { return false; } else if !anns.native_mode && language == &ScriptLang::Bunnative { @@ -1077,7 +1077,7 @@ async fn lock_modules_app( match new_lock { Ok(new_lock) => { append_logs(&job.id, &job.workspace_id, logs, db).await; - let anns = get_annotation(&content); + let anns = get_annotation_ts(&content); let nlang = if anns.native_mode && language == ScriptLang::Bun { Some(ScriptLang::Bunnative) } else if !anns.native_mode && language == ScriptLang::Bunnative @@ -1280,7 +1280,7 @@ async fn python_dep( occupancy_metrics: &mut Option<&mut OccupancyMetrics>, ) -> std::result::Result { create_dependencies_dir(job_dir).await; - let req: std::result::Result = pip_compile( + let req: std::result::Result = uv_pip_compile( job_id, &reqs, mem_peak, @@ -1290,6 +1290,8 @@ async fn python_dep( worker_name, w_id, occupancy_metrics, + false, + false, ) .await; // install the dependencies to pre-fill the cache @@ -1434,8 +1436,9 @@ async fn capture_dependency_job( .await } ScriptLang::Bun | ScriptLang::Bunnative => { - let npm_mode = npm_mode - .unwrap_or_else(|| windmill_common::worker::get_annotation(job_raw_code).npm_mode); + let npm_mode = npm_mode.unwrap_or_else(|| { + windmill_common::worker::get_annotation_ts(job_raw_code).npm_mode + }); if !raw_deps { let _ = write_file(job_dir, "main.ts", job_raw_code)?; } diff --git a/docker/DockerfileSlim b/docker/DockerfileSlim index b94bdec85e5f4..1b1c1af88799e 100644 --- a/docker/DockerfileSlim +++ b/docker/DockerfileSlim @@ -17,6 +17,9 @@ ENV TZ=Etc/UTC RUN /usr/local/bin/python3 -m pip install pip-tools +# Install UV +RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.4.18/uv-installer.sh | sh + COPY --from=oven/bun:1.1.27 /usr/local/bin/bun /usr/bin/bun # add the docker client to call docker from a worker if enabled diff --git a/docker/DockerfileSlimEe b/docker/DockerfileSlimEe index b94bdec85e5f4..64364adb377a7 100644 --- a/docker/DockerfileSlimEe +++ b/docker/DockerfileSlimEe @@ -16,6 +16,8 @@ RUN apt-get -y update && apt-get install -y curl nodejs awscli ENV TZ=Etc/UTC RUN /usr/local/bin/python3 -m pip install pip-tools +# Install UV +RUN curl --proto '=https' --tlsv1.2 -LsSf https://github.com/astral-sh/uv/releases/download/0.4.18/uv-installer.sh | sh COPY --from=oven/bun:1.1.27 /usr/local/bin/bun /usr/bin/bun diff --git a/shell.nix b/shell.nix index 58085ee67519f..c3009eb1830cb 100644 --- a/shell.nix +++ b/shell.nix @@ -8,8 +8,8 @@ let "https://github.com/oxalica/rust-overlay/archive/master.tar.gz"); pkgs = import { overlays = [ rust_overlay ]; }; # TODO: Pin version? - rustVersion = "latest"; - # rustVersion = "1.83.0"; + # rustVersion = "latest"; + rustVersion = "2024-09-30"; rust = pkgs.rust-bin.nightly.${rustVersion}.default.override { extensions = [ "rust-src" # for rust-analyzer @@ -27,6 +27,7 @@ in pkgs.mkShell { postgresql watchexec # used in client's dev.nu poetry # for python client + uv python312Packages.pip-tools # pip-compile ]; @@ -54,6 +55,7 @@ in pkgs.mkShell { "-C link-arg=-fuse-ld=${pkgs.mold}/bin/mold -Zshare-generics=y -Z threads=4"; RUSTC_WRAPPER = "${pkgs.sccache}/bin/sccache"; - # Use mold as a linker (for faster compilation) + # Useful for development + RUST_LOG = "debug"; }