feat: start prefetching all files immediately (pola-rs#12201)

osawyerr · Nov 2, 2023 · 1354f5d · 1354f5d
1 parent bd89f49
commit 1354f5d
Showing 1 changed file with 3 additions and 1 deletion.
diff --git a/crates/polars-pipe/src/executors/sources/parquet.rs b/crates/polars-pipe/src/executors/sources/parquet.rs
@@ -178,7 +178,9 @@ impl Source for ParquetSource {
         if self.batched_readers.len() < self.prefetch_size && self.file_options.n_rows.is_none()
             || self.batched_readers.is_empty()
         {
-            self.init_next_reader()?
+            for _ in 0..self.prefetch_size - self.batched_readers.len() {
+                self.init_next_reader()?
+            }
         }
 
         let Some(mut reader) = self.batched_readers.pop_front() else {