fix: unexpected error

wheretrue · Aug 28, 2024 · d4a920a · d4a920a
1 parent dd72ced
commit d4a920a
Show file tree

Hide file tree

Showing 2 changed files with 13 additions and 0 deletions.
diff --git a/exon/exon-core/src/datasources/exon_file_scan_config.rs b/exon/exon-core/src/datasources/exon_file_scan_config.rs
@@ -99,5 +99,10 @@ fn regroup_files_by_size(
         new_file_groups[target_partition].push(file.clone());
     }
 
+    // Filter any empty file groups
+    new_file_groups.retain(|fg| !fg.is_empty());
+
+    eprintln!("Repartitioned to {} partitions", new_file_groups.len());
+
     new_file_groups
 }
diff --git a/exon/exon-core/src/datasources/fastq/scanner.rs b/exon/exon-core/src/datasources/fastq/scanner.rs
@@ -90,11 +90,19 @@ impl ExecutionPlan for FASTQScan {
         target_partitions: usize,
         _config: &datafusion::config::ConfigOptions,
     ) -> datafusion::error::Result<Option<Arc<dyn ExecutionPlan>>> {
+        eprintln!("Repartitioning to {} partitions", target_partitions);
+        eprintln!("Have {} file groups", self.base_config.file_groups.len());
+
         if target_partitions == 1 || self.base_config.file_groups.is_empty() {
             return Ok(None);
         }
 
         let file_groups = self.base_config.regroup_files_by_size(target_partitions);
+        eprintln!("Regrouped into {} file groups", file_groups.len());
+
+        if file_groups.len() == 1 || file_groups.iter().all(|fg| fg.is_empty()) {
+            return Ok(None);
+        }
 
         let mut new_plan = self.clone();
         new_plan.base_config.file_groups = file_groups;