Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: Replace pip-compile with uv #4460

Merged
merged 25 commits into from
Oct 3, 2024
Merged
Show file tree
Hide file tree
Changes from 16 commits
Commits
Show all changes
25 commits
Select commit Hold shift + click to select a range
b652c37
Update `shell.nix`
pyranota Sep 30, 2024
cc1b891
Replace `pip-compile` with `uv` (dirty + untested)
pyranota Sep 30, 2024
a6cfbdb
Fix arguments passed to uv
pyranota Sep 30, 2024
25e1ea4
Remove extra `dbg!`
pyranota Sep 30, 2024
4429338
Replace 'pip-compile' with 'uv' in Dockerfile
pyranota Sep 30, 2024
8d5ac55
Add fallback option to `pip-compile` (Disabled)
pyranota Oct 1, 2024
844db2e
Add `uv` to `docker/DockerfileSlim*`
pyranota Oct 1, 2024
b3d1328
Add `get_annotation_python` and rename `get_annotation` to `get_annot…
pyranota Oct 1, 2024
e440fb7
Add option to fallback to pip-compile
pyranota Oct 1, 2024
e1aceac
Put back `pip-tools` into shell.nix
pyranota Oct 1, 2024
6bfa8dd
Make sure lockfile resolves again if `#no_uv` used
pyranota Oct 2, 2024
0cae22c
Put `pip install pip-tools` in original spot
pyranota Oct 2, 2024
80fbdd2
Merge branch 'main' into python-uv
pyranota Oct 2, 2024
4f40e46
Fix compilation error
pyranota Oct 2, 2024
ba11eed
Merge branch 'python-uv' of github.com:windmill-labs/windmill into py…
pyranota Oct 2, 2024
175528e
Fix EE compilation error
pyranota Oct 2, 2024
3065288
Add `no_cache` annotation
pyranota Oct 2, 2024
4e7e060
Target uv cache to /tmp/windmill/cache
pyranota Oct 2, 2024
4f06bfe
Prohibit uv from managing python
pyranota Oct 2, 2024
abee3d7
Add uv to DockerfileBackendTests
pyranota Oct 2, 2024
49821d3
Pin uv version to 0.4.18 in Dockerfiles
pyranota Oct 3, 2024
04aac60
Dont put `#no_uv` in requirements.in
pyranota Oct 3, 2024
5424a4b
Merge branch 'main' into python-uv
rubenfiszel Oct 3, 2024
621e5a2
Push Warning to logs if fallbacked to pip-compile
pyranota Oct 3, 2024
8e48066
Merge branch 'main' into python-uv
rubenfiszel Oct 3, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -158,6 +158,9 @@ RUN set -eux; \
ENV PATH="${PATH}:/usr/local/go/bin"
ENV GO_PATH=/usr/local/go/bin/go

# Install UV
RUN curl -LsSf https://astral.sh/uv/install.sh | sh

RUN curl -sL https://deb.nodesource.com/setup_20.x | bash -
RUN apt-get -y update && apt-get install -y curl nodejs awscli && apt-get clean \
&& rm -rf /var/lib/apt/lists/*
Expand Down
2 changes: 1 addition & 1 deletion backend/windmill-api/src/jobs.rs
Original file line number Diff line number Diff line change
Expand Up @@ -4130,7 +4130,7 @@ async fn run_dependencies_job(
JsonRawValue::from_string("true".to_string()).unwrap(),
);
if language == ScriptLang::Bun {
let annotation = windmill_common::worker::get_annotation(&raw_code);
let annotation = windmill_common::worker::get_annotation_ts(&raw_code);
hm.insert(
"npm_mode".to_string(),
JsonRawValue::from_string(annotation.npm_mode.to_string()).unwrap(),
Expand Down
4 changes: 2 additions & 2 deletions backend/windmill-api/src/scripts.rs
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,7 @@ use windmill_common::{
utils::{
not_found_if_none, paginate, query_elems_from_hub, require_admin, Pagination, StripPath,
},
worker::{get_annotation, to_raw_value},
worker::{get_annotation_ts, to_raw_value},
HUB_BASE_URL,
};
use windmill_git_sync::{handle_deployment_metadata, DeployedObject};
Expand Down Expand Up @@ -601,7 +601,7 @@ async fn create_script_internal<'c>(
};

let lang = if &ns.language == &ScriptLang::Bun || &ns.language == &ScriptLang::Bunnative {
let anns = get_annotation(&ns.content);
let anns = get_annotation_ts(&ns.content);
if anns.native_mode {
ScriptLang::Bunnative
} else {
Expand Down
22 changes: 19 additions & 3 deletions backend/windmill-common/src/worker.rs
Original file line number Diff line number Diff line change
Expand Up @@ -303,14 +303,14 @@ fn parse_file<T: FromStr>(path: &str) -> Option<T> {
.flatten()
}

pub struct Annotations {
pub struct TypeScriptAnnotations {
pub npm_mode: bool,
pub nodejs_mode: bool,
pub native_mode: bool,
pub nobundling: bool,
}

pub fn get_annotation(inner_content: &str) -> Annotations {
pub fn get_annotation_ts(inner_content: &str) -> TypeScriptAnnotations {
let annotations = inner_content
.lines()
.take_while(|x| x.starts_with("//"))
Expand All @@ -324,7 +324,23 @@ pub fn get_annotation(inner_content: &str) -> Annotations {
let nobundling: bool =
annotations.contains(&"nobundling".to_string()) || nodejs_mode || *DISABLE_BUNDLING;

Annotations { npm_mode, nodejs_mode, native_mode, nobundling }
TypeScriptAnnotations { npm_mode, nodejs_mode, native_mode, nobundling }
}

pub struct PythonAnnotations {
pub no_uv: bool,
}

pub fn get_annotation_python(inner_content: &str) -> PythonAnnotations {
let annotations = inner_content
.lines()
.take_while(|x| x.starts_with("#"))
.map(|x| x.to_string().replace("#", "").trim().to_string())
.collect_vec();

let no_uv: bool = annotations.contains(&"no_uv".to_string());

PythonAnnotations { no_uv }
}

pub struct SqlAnnotations {
Expand Down
5 changes: 3 additions & 2 deletions backend/windmill-worker/src/ansible_executor.rs
Original file line number Diff line number Diff line change
Expand Up @@ -25,7 +25,7 @@ use crate::{
OccupancyMetrics,
},
handle_child::handle_child,
python_executor::{create_dependencies_dir, handle_python_reqs, pip_compile},
python_executor::{create_dependencies_dir, handle_python_reqs, uv_pip_compile},
AuthedClientBackgroundTask, DISABLE_NSJAIL, DISABLE_NUSER, HOME_ENV, NSJAIL_PATH, PATH_ENV,
TZ_ENV,
};
Expand Down Expand Up @@ -72,7 +72,7 @@ async fn handle_ansible_python_deps(
if requirements.is_empty() {
"".to_string()
} else {
pip_compile(
uv_pip_compile(
job_id,
&requirements,
mem_peak,
Expand All @@ -82,6 +82,7 @@ async fn handle_ansible_python_deps(
worker_name,
w_id,
&mut Some(occupancy_metrics),
false,
)
.await
.map_err(|e| {
Expand Down
8 changes: 4 additions & 4 deletions backend/windmill-worker/src/bun_executor.rs
Original file line number Diff line number Diff line change
Expand Up @@ -38,7 +38,7 @@ use windmill_common::{
get_latest_hash_for_path,
jobs::{QueuedJob, PREPROCESSOR_FAKE_ENTRYPOINT},
scripts::ScriptLang,
worker::{exists_in_cache, get_annotation, save_cache, write_file},
worker::{exists_in_cache, get_annotation_ts, save_cache, write_file},
DB,
};

Expand Down Expand Up @@ -632,7 +632,7 @@ pub async fn prebundle_bun_script(
if exists_in_cache(&local_path, &remote_path).await {
return Ok(());
}
let annotation = get_annotation(inner_content);
let annotation = get_annotation_ts(inner_content);
if annotation.nobundling {
return Ok(());
}
Expand Down Expand Up @@ -765,7 +765,7 @@ pub async fn handle_bun_job(
new_args: &mut Option<HashMap<String, Box<RawValue>>>,
occupancy_metrics: &mut OccupancyMetrics,
) -> error::Result<Box<RawValue>> {
let mut annotation = windmill_common::worker::get_annotation(inner_content);
let mut annotation = windmill_common::worker::get_annotation_ts(inner_content);

let (mut has_bundle_cache, cache_logs, local_path, remote_path) =
if requirements_o.is_some() && !annotation.nobundling && codebase.is_none() {
Expand Down Expand Up @@ -1461,7 +1461,7 @@ pub async fn start_worker(
let common_bun_proc_envs: HashMap<String, String> =
get_common_bun_proc_envs(Some(&base_internal_url)).await;

let mut annotation = windmill_common::worker::get_annotation(inner_content);
let mut annotation = windmill_common::worker::get_annotation_ts(inner_content);

//TODO: remove this when bun dedicated workers work without issues
annotation.nodejs_mode = true;
Expand Down
Loading
Loading