From c44caad01c2801891aa57d4500d1caa3c3f3c2b4 Mon Sep 17 00:00:00 2001 From: sundy-li <543950155@qq.com> Date: Tue, 7 Jan 2025 09:28:29 +0800 Subject: [PATCH 1/4] feat: support display spill file stats --- bindings/nodejs/src/lib.rs | 5 +++++ bindings/python/src/types.rs | 4 ++++ cli/src/display.rs | 18 ++++++++++++++---- core/src/response.rs | 1 + driver/src/conn.rs | 2 ++ sql/src/rows.rs | 4 ++++ 6 files changed, 30 insertions(+), 4 deletions(-) diff --git a/bindings/nodejs/src/lib.rs b/bindings/nodejs/src/lib.rs index 251c115e0..79aee0a6d 100644 --- a/bindings/nodejs/src/lib.rs +++ b/bindings/nodejs/src/lib.rs @@ -547,6 +547,11 @@ impl ServerStats { self.0.write_bytes } + #[napi(getter)] + pub fn spill_file_count(&self) -> usize { + self.0.spill_file_count + } + #[napi(getter)] pub fn running_time_ms(&self) -> f64 { self.0.running_time_ms diff --git a/bindings/python/src/types.rs b/bindings/python/src/types.rs index 95f5e95b5..070a90f4b 100644 --- a/bindings/python/src/types.rs +++ b/bindings/python/src/types.rs @@ -361,6 +361,10 @@ impl ServerStats { self.0.write_bytes } #[getter] + pub fn spill_file_count(&self) -> f64 { + self.0.spill_file_count + } + #[getter] pub fn running_time_ms(&self) -> f64 { self.0.running_time_ms } diff --git a/cli/src/display.rs b/cli/src/display.rs index c63b988aa..4b055c3fe 100644 --- a/cli/src/display.rs +++ b/cli/src/display.rs @@ -410,23 +410,33 @@ impl ChunkDisplay for FormatDisplay<'_> { fn format_read_progress(ss: &ServerStats, elapsed: f64) -> String { format!( - "Processing {}/{} ({} rows/s), {}/{} ({}/s)", + "Processing {}/{} ({} rows/s), {}/{} ({}/s){}", humanize_count(ss.read_rows as f64), humanize_count(ss.total_rows as f64), humanize_count(ss.read_rows as f64 / elapsed), HumanBytes(ss.read_bytes as u64), HumanBytes(ss.total_bytes as u64), - HumanBytes((ss.read_bytes as f64 / elapsed) as u64) + HumanBytes((ss.read_bytes as f64 / elapsed) as u64), + if ss.spill_file_count > 0 { + format!(", spilled {} files", ss.spill_file_count) + } else { + "".to_string() + } ) } pub fn format_write_progress(ss: &ServerStats, elapsed: f64) -> String { format!( - "Written {} ({} rows/s), {} ({}/s)", + "Written {} ({} rows/s), {} ({}/s){}", humanize_count(ss.write_rows as f64), humanize_count(ss.write_rows as f64 / elapsed), HumanBytes(ss.write_bytes as u64), - HumanBytes((ss.write_bytes as f64 / elapsed) as u64) + HumanBytes((ss.write_bytes as f64 / elapsed) as u64), + if ss.spill_file_count > 0 { + format!(", spilled {} files", ss.spill_file_count) + } else { + "".to_string() + } ) } diff --git a/core/src/response.rs b/core/src/response.rs index e7268ea86..ba2198cc6 100644 --- a/core/src/response.rs +++ b/core/src/response.rs @@ -30,6 +30,7 @@ pub struct Progresses { pub result_progress: ProgressValues, // make it optional for backward compatibility pub total_scan: Option, + pub spill_file_count: usize, } impl Progresses { diff --git a/driver/src/conn.rs b/driver/src/conn.rs index fef14b73e..b90c8ccad 100644 --- a/driver/src/conn.rs +++ b/driver/src/conn.rs @@ -205,6 +205,7 @@ pub trait Connection: Send + Sync { read_bytes: 0, write_rows: total_count, write_bytes: total_size, + spill_file_count: 0, running_time_ms: 0.0, }; results.push(Ok(RowWithStats::Stats(ss))); @@ -259,6 +260,7 @@ pub trait Connection: Send + Sync { total_bytes: 0, read_rows: total_count, read_bytes: total_size, + spill_file_count: 0, write_rows: 0, write_bytes: 0, running_time_ms: 0.0, diff --git a/sql/src/rows.rs b/sql/src/rows.rs index 9b6fa3eb5..ee2abf81a 100644 --- a/sql/src/rows.rs +++ b/sql/src/rows.rs @@ -51,6 +51,9 @@ pub struct ServerStats { #[serde(default)] pub running_time_ms: f64, + + #[serde(default)] + pub spill_file_count: usize, } impl ServerStats { @@ -73,6 +76,7 @@ impl From for ServerStats { read_bytes: stats.progresses.scan_progress.bytes, write_rows: stats.progresses.write_progress.rows, write_bytes: stats.progresses.write_progress.bytes, + spill_file_count: stats.progresses.spill_file_count, running_time_ms: stats.running_time_ms, }; if let Some(total) = stats.progresses.total_scan { From 24faee17ffa579678b6721d1bd734d8306f0e486 Mon Sep 17 00:00:00 2001 From: sundy-li <543950155@qq.com> Date: Tue, 7 Jan 2025 10:16:14 +0800 Subject: [PATCH 2/4] feat: support display spill file stats --- bindings/nodejs/src/lib.rs | 4 ++-- bindings/python/src/types.rs | 4 ++-- cli/src/display.rs | 16 ++++++++++++---- core/src/response.rs | 8 +++++++- driver/src/conn.rs | 15 +++------------ sql/src/rows.rs | 8 ++++++-- 6 files changed, 32 insertions(+), 23 deletions(-) diff --git a/bindings/nodejs/src/lib.rs b/bindings/nodejs/src/lib.rs index 79aee0a6d..caf202df7 100644 --- a/bindings/nodejs/src/lib.rs +++ b/bindings/nodejs/src/lib.rs @@ -548,8 +548,8 @@ impl ServerStats { } #[napi(getter)] - pub fn spill_file_count(&self) -> usize { - self.0.spill_file_count + pub fn spill_file_nums(&self) -> usize { + self.0.spill_file_nums } #[napi(getter)] diff --git a/bindings/python/src/types.rs b/bindings/python/src/types.rs index 070a90f4b..ba7627b04 100644 --- a/bindings/python/src/types.rs +++ b/bindings/python/src/types.rs @@ -361,8 +361,8 @@ impl ServerStats { self.0.write_bytes } #[getter] - pub fn spill_file_count(&self) -> f64 { - self.0.spill_file_count + pub fn spill_file_nums(&self) -> f64 { + self.0.spill_file_nums } #[getter] pub fn running_time_ms(&self) -> f64 { diff --git a/cli/src/display.rs b/cli/src/display.rs index 4b055c3fe..cd7a13fc4 100644 --- a/cli/src/display.rs +++ b/cli/src/display.rs @@ -417,8 +417,12 @@ fn format_read_progress(ss: &ServerStats, elapsed: f64) -> String { HumanBytes(ss.read_bytes as u64), HumanBytes(ss.total_bytes as u64), HumanBytes((ss.read_bytes as f64 / elapsed) as u64), - if ss.spill_file_count > 0 { - format!(", spilled {} files", ss.spill_file_count) + if ss.spill_file_nums > 0 { + format!( + ", spilled {} files, {}", + ss.spill_file_nums, + HumanBytes(ss.spill_bytes as u64) + ) } else { "".to_string() } @@ -432,8 +436,12 @@ pub fn format_write_progress(ss: &ServerStats, elapsed: f64) -> String { humanize_count(ss.write_rows as f64 / elapsed), HumanBytes(ss.write_bytes as u64), HumanBytes((ss.write_bytes as f64 / elapsed) as u64), - if ss.spill_file_count > 0 { - format!(", spilled {} files", ss.spill_file_count) + if ss.spill_file_nums > 0 { + format!( + ", spilled {} files, {}", + ss.spill_file_nums, + HumanBytes(ss.spill_bytes as u64) + ) } else { "".to_string() } diff --git a/core/src/response.rs b/core/src/response.rs index ba2198cc6..005f7385b 100644 --- a/core/src/response.rs +++ b/core/src/response.rs @@ -30,7 +30,7 @@ pub struct Progresses { pub result_progress: ProgressValues, // make it optional for backward compatibility pub total_scan: Option, - pub spill_file_count: usize, + pub spill_progress: SpillProgress, } impl Progresses { @@ -54,6 +54,12 @@ pub struct ProgressValues { pub bytes: usize, } +#[derive(Debug, Clone, Deserialize, Serialize, Default)] +pub struct SpillProgress { + pub file_nums: usize, + pub bytes: usize, +} + #[derive(Serialize, Deserialize, Debug, Clone)] pub struct SchemaField { pub name: String, diff --git a/driver/src/conn.rs b/driver/src/conn.rs index b90c8ccad..60ab46840 100644 --- a/driver/src/conn.rs +++ b/driver/src/conn.rs @@ -199,14 +199,10 @@ pub trait Connection: Send + Sync { Err(e) => (entry.to_string_lossy().to_string(), e.to_string()), }; let ss = ServerStats { - total_rows: 0, - total_bytes: 0, - read_rows: 0, - read_bytes: 0, write_rows: total_count, write_bytes: total_size, - spill_file_count: 0, - running_time_ms: 0.0, + + ..Default::default() }; results.push(Ok(RowWithStats::Stats(ss))); results.push(Ok(RowWithStats::Row(Row::from_vec( @@ -256,14 +252,9 @@ pub trait Connection: Send + Sync { Err(e) => (e.to_string(), 0), }; let ss = ServerStats { - total_rows: 0, - total_bytes: 0, read_rows: total_count, read_bytes: total_size, - spill_file_count: 0, - write_rows: 0, - write_bytes: 0, - running_time_ms: 0.0, + ..Default::default() }; results.push(Ok(RowWithStats::Stats(ss))); results.push(Ok(RowWithStats::Row(Row::from_vec( diff --git a/sql/src/rows.rs b/sql/src/rows.rs index ee2abf81a..0a21bcb86 100644 --- a/sql/src/rows.rs +++ b/sql/src/rows.rs @@ -53,7 +53,10 @@ pub struct ServerStats { pub running_time_ms: f64, #[serde(default)] - pub spill_file_count: usize, + pub spill_file_nums: usize, + + #[serde(default)] + pub spill_bytes: usize, } impl ServerStats { @@ -76,7 +79,8 @@ impl From for ServerStats { read_bytes: stats.progresses.scan_progress.bytes, write_rows: stats.progresses.write_progress.rows, write_bytes: stats.progresses.write_progress.bytes, - spill_file_count: stats.progresses.spill_file_count, + spill_file_nums: stats.progresses.spill_progress.file_nums, + spill_bytes: stats.progresses.spill_progress.bytes, running_time_ms: stats.running_time_ms, }; if let Some(total) = stats.progresses.total_scan { From a755fa139dafc9ce6c5794975647a3657028a795 Mon Sep 17 00:00:00 2001 From: sundy-li <543950155@qq.com> Date: Tue, 7 Jan 2025 10:23:00 +0800 Subject: [PATCH 3/4] feat: support display spill file stats --- bindings/python/src/types.rs | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bindings/python/src/types.rs b/bindings/python/src/types.rs index ba7627b04..1afde17f2 100644 --- a/bindings/python/src/types.rs +++ b/bindings/python/src/types.rs @@ -361,7 +361,7 @@ impl ServerStats { self.0.write_bytes } #[getter] - pub fn spill_file_nums(&self) -> f64 { + pub fn spill_file_nums(&self) -> usize { self.0.spill_file_nums } #[getter] From ecf18f376a7c678d3e6681e0bbfd0f56d1ff2f57 Mon Sep 17 00:00:00 2001 From: sundy-li <543950155@qq.com> Date: Tue, 7 Jan 2025 10:33:30 +0800 Subject: [PATCH 4/4] feat(query): support send spill file stats to client --- core/src/response.rs | 1 + 1 file changed, 1 insertion(+) diff --git a/core/src/response.rs b/core/src/response.rs index 005f7385b..c5eec64bc 100644 --- a/core/src/response.rs +++ b/core/src/response.rs @@ -30,6 +30,7 @@ pub struct Progresses { pub result_progress: ProgressValues, // make it optional for backward compatibility pub total_scan: Option, + #[serde(default)] pub spill_progress: SpillProgress, }