From 1354f5d295ee0db9c7808b9554df13416af46094 Mon Sep 17 00:00:00 2001 From: Ritchie Vink Date: Thu, 2 Nov 2023 11:07:28 -0400 Subject: [PATCH] feat: start prefetching all files immediately (#12201) --- crates/polars-pipe/src/executors/sources/parquet.rs | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/crates/polars-pipe/src/executors/sources/parquet.rs b/crates/polars-pipe/src/executors/sources/parquet.rs index ad7e077f88ec..ed156f56884f 100644 --- a/crates/polars-pipe/src/executors/sources/parquet.rs +++ b/crates/polars-pipe/src/executors/sources/parquet.rs @@ -178,7 +178,9 @@ impl Source for ParquetSource { if self.batched_readers.len() < self.prefetch_size && self.file_options.n_rows.is_none() || self.batched_readers.is_empty() { - self.init_next_reader()? + for _ in 0..self.prefetch_size - self.batched_readers.len() { + self.init_next_reader()? + } } let Some(mut reader) = self.batched_readers.pop_front() else {