diff --git a/pageserver/src/tenant.rs b/pageserver/src/tenant.rs index a6c4aa45229d..a40ae14183d3 100644 --- a/pageserver/src/tenant.rs +++ b/pageserver/src/tenant.rs @@ -3020,7 +3020,7 @@ impl Tenant { let jobs = timeline .gc_compaction_split_jobs(GcCompactJob::from_compact_options( next_scheduled_compaction_task.options, - )) + ), None) .await .map_err(CompactionError::Other)?; if jobs.is_empty() { diff --git a/pageserver/src/tenant/timeline/compaction.rs b/pageserver/src/tenant/timeline/compaction.rs index a2addb0b5996..43b0592297c6 100644 --- a/pageserver/src/tenant/timeline/compaction.rs +++ b/pageserver/src/tenant/timeline/compaction.rs @@ -1805,12 +1805,19 @@ impl Timeline { pub(crate) async fn gc_compaction_split_jobs( self: &Arc, job: GcCompactJob, + compact_target_max_size_mb: Option, ) -> anyhow::Result> { let compact_below_lsn = if job.compact_lsn_range.end != Lsn::MAX { job.compact_lsn_range.end } else { *self.get_latest_gc_cutoff_lsn() // use the real gc cutoff }; + + // Split compaction job to about 4GB each + const GC_COMPACT_MAX_SIZE_MB: u64 = 4 * 1024; + let compact_target_max_size_mb = + compact_target_max_size_mb.unwrap_or(GC_COMPACT_MAX_SIZE_MB); + let mut compact_jobs = Vec::new(); // For now, we simply use the key partitioning information; we should do a more fine-grained partitioning // by estimating the amount of files read for a compaction job. We should also partition on LSN. @@ -1857,8 +1864,6 @@ impl Timeline { let guard = self.layers.read().await; let layer_map = guard.layer_map()?; let mut current_start = None; - // Split compaction job to about 2GB each - const GC_COMPACT_MAX_SIZE_MB: u64 = 4 * 1024; // 4GB, TODO: should be configuration in the future let ranges_num = split_key_ranges.len(); for (idx, (start, end)) in split_key_ranges.into_iter().enumerate() { if current_start.is_none() { @@ -1871,7 +1876,7 @@ impl Timeline { } let res = layer_map.range_search(start..end, compact_below_lsn); let total_size = res.found.keys().map(|x| x.layer.file_size()).sum::(); - if total_size > GC_COMPACT_MAX_SIZE_MB * 1024 * 1024 || ranges_num == idx + 1 { + if total_size > compact_target_max_size_mb * 1024 * 1024 || ranges_num == idx + 1 { // Try to extend the compaction range so that we include at least one full layer file. let extended_end = res .found @@ -1927,7 +1932,7 @@ impl Timeline { let job = GcCompactJob::from_compact_options(options); if sub_compaction { info!("running enhanced gc bottom-most compaction with sub-compaction, splitting compaction jobs"); - let jobs = self.gc_compaction_split_jobs(job).await?; + let jobs = self.gc_compaction_split_jobs(job, None).await?; let jobs_len = jobs.len(); for (idx, job) in jobs.into_iter().enumerate() { info!( diff --git a/test_runner/regress/test_compaction.py b/test_runner/regress/test_compaction.py index 1eceb6276bce..be41f33062ff 100644 --- a/test_runner/regress/test_compaction.py +++ b/test_runner/regress/test_compaction.py @@ -153,6 +153,7 @@ def test_pageserver_gc_compaction_smoke(neon_env_builder: NeonEnvBuilder): if i % 10 == 0: log.info(f"Running churn round {i}/{churn_rounds} ...") + if (i - 1) % 10 == 0: # Run gc-compaction every 10 rounds to ensure the test doesn't take too long time. ps_http.timeline_compact( tenant_id,