feat: support display spill file stats (#565)

* feat: support display spill file stats * feat: support display spill file stats * feat: support display spill file stats * feat(query): support send spill file stats to client
databendlabs · Jan 7, 2025 · e3100b2 · e3100b2
1 parent f1fa5d4
commit e3100b2
Show file tree

Hide file tree

Showing 6 changed files with 50 additions and 14 deletions.
diff --git a/bindings/nodejs/src/lib.rs b/bindings/nodejs/src/lib.rs
@@ -547,6 +547,11 @@ impl ServerStats {
         self.0.write_bytes
     }
 
+    #[napi(getter)]
+    pub fn spill_file_nums(&self) -> usize {
+        self.0.spill_file_nums
+    }
+
     #[napi(getter)]
     pub fn running_time_ms(&self) -> f64 {
         self.0.running_time_ms

diff --git a/bindings/python/src/types.rs b/bindings/python/src/types.rs
@@ -361,6 +361,10 @@ impl ServerStats {
         self.0.write_bytes
     }
     #[getter]
+    pub fn spill_file_nums(&self) -> usize {
+        self.0.spill_file_nums
+    }
+    #[getter]
     pub fn running_time_ms(&self) -> f64 {
         self.0.running_time_ms
     }

diff --git a/cli/src/display.rs b/cli/src/display.rs
@@ -410,23 +410,41 @@ impl ChunkDisplay for FormatDisplay<'_> {
 
 fn format_read_progress(ss: &ServerStats, elapsed: f64) -> String {
     format!(
-        "Processing {}/{} ({} rows/s), {}/{} ({}/s)",
+        "Processing {}/{} ({} rows/s), {}/{} ({}/s){}",
         humanize_count(ss.read_rows as f64),
         humanize_count(ss.total_rows as f64),
         humanize_count(ss.read_rows as f64 / elapsed),
         HumanBytes(ss.read_bytes as u64),
         HumanBytes(ss.total_bytes as u64),
-        HumanBytes((ss.read_bytes as f64 / elapsed) as u64)
+        HumanBytes((ss.read_bytes as f64 / elapsed) as u64),
+        if ss.spill_file_nums > 0 {
+            format!(
+                ", spilled {} files, {}",
+                ss.spill_file_nums,
+                HumanBytes(ss.spill_bytes as u64)
+            )
+        } else {
+            "".to_string()
+        }
     )
 }
 
 pub fn format_write_progress(ss: &ServerStats, elapsed: f64) -> String {
     format!(
-        "Written {} ({} rows/s), {} ({}/s)",
+        "Written {} ({} rows/s), {} ({}/s){}",
         humanize_count(ss.write_rows as f64),
         humanize_count(ss.write_rows as f64 / elapsed),
         HumanBytes(ss.write_bytes as u64),
-        HumanBytes((ss.write_bytes as f64 / elapsed) as u64)
+        HumanBytes((ss.write_bytes as f64 / elapsed) as u64),
+        if ss.spill_file_nums > 0 {
+            format!(
+                ", spilled {} files, {}",
+                ss.spill_file_nums,
+                HumanBytes(ss.spill_bytes as u64)
+            )
+        } else {
+            "".to_string()
+        }
     )
 }
 

diff --git a/core/src/response.rs b/core/src/response.rs
@@ -30,6 +30,8 @@ pub struct Progresses {
     pub result_progress: ProgressValues,
     // make it optional for backward compatibility
     pub total_scan: Option<ProgressValues>,
+    #[serde(default)]
+    pub spill_progress: SpillProgress,
 }
 
 impl Progresses {
@@ -53,6 +55,12 @@ pub struct ProgressValues {
     pub bytes: usize,
 }
 
+#[derive(Debug, Clone, Deserialize, Serialize, Default)]
+pub struct SpillProgress {
+    pub file_nums: usize,
+    pub bytes: usize,
+}
+
 #[derive(Serialize, Deserialize, Debug, Clone)]
 pub struct SchemaField {
     pub name: String,

diff --git a/driver/src/conn.rs b/driver/src/conn.rs
@@ -199,13 +199,10 @@ pub trait Connection: Send + Sync {
                 Err(e) => (entry.to_string_lossy().to_string(), e.to_string()),
             };
             let ss = ServerStats {
-                total_rows: 0,
-                total_bytes: 0,
-                read_rows: 0,
-                read_bytes: 0,
                 write_rows: total_count,
                 write_bytes: total_size,
-                running_time_ms: 0.0,
+
+                ..Default::default()
             };
             results.push(Ok(RowWithStats::Stats(ss)));
             results.push(Ok(RowWithStats::Row(Row::from_vec(
@@ -255,13 +252,9 @@ pub trait Connection: Send + Sync {
                 Err(e) => (e.to_string(), 0),
             };
             let ss = ServerStats {
-                total_rows: 0,
-                total_bytes: 0,
                 read_rows: total_count,
                 read_bytes: total_size,
-                write_rows: 0,
-                write_bytes: 0,
-                running_time_ms: 0.0,
+                ..Default::default()
             };
             results.push(Ok(RowWithStats::Stats(ss)));
             results.push(Ok(RowWithStats::Row(Row::from_vec(

diff --git a/sql/src/rows.rs b/sql/src/rows.rs
@@ -51,6 +51,12 @@ pub struct ServerStats {
 
     #[serde(default)]
     pub running_time_ms: f64,
+
+    #[serde(default)]
+    pub spill_file_nums: usize,
+
+    #[serde(default)]
+    pub spill_bytes: usize,
 }
 
 impl ServerStats {
@@ -73,6 +79,8 @@ impl From<databend_client::QueryStats> for ServerStats {
             read_bytes: stats.progresses.scan_progress.bytes,
             write_rows: stats.progresses.write_progress.rows,
             write_bytes: stats.progresses.write_progress.bytes,
+            spill_file_nums: stats.progresses.spill_progress.file_nums,
+            spill_bytes: stats.progresses.spill_progress.bytes,
             running_time_ms: stats.running_time_ms,
         };
         if let Some(total) = stats.progresses.total_scan {