From d65406c2cc84be4c21b705cbfea66d57d0d77a1f Mon Sep 17 00:00:00 2001 From: Alex Chi Z Date: Mon, 11 Nov 2024 16:59:12 -0500 Subject: [PATCH 1/3] test(pageserver): add gc-compaction tests with delta will_init Signed-off-by: Alex Chi Z --- libs/pageserver_api/src/record.rs | 8 +- pageserver/src/tenant.rs | 114 ++++++++++++++---- .../tenant/storage_layer/merge_iterator.rs | 4 +- pageserver/src/walredo/apply_neon.rs | 4 + 4 files changed, 103 insertions(+), 27 deletions(-) diff --git a/libs/pageserver_api/src/record.rs b/libs/pageserver_api/src/record.rs index b80ed2f20353..5c3f3deb826e 100644 --- a/libs/pageserver_api/src/record.rs +++ b/libs/pageserver_api/src/record.rs @@ -80,18 +80,18 @@ impl NeonWalRecord { } #[cfg(feature = "testing")] - pub fn wal_clear() -> Self { + pub fn wal_clear(s: impl AsRef) -> Self { Self::Test { - append: "".to_string(), + append: s.as_ref().to_string(), clear: true, will_init: false, } } #[cfg(feature = "testing")] - pub fn wal_init() -> Self { + pub fn wal_init(s: impl AsRef) -> Self { Self::Test { - append: "".to_string(), + append: s.as_ref().to_string(), clear: true, will_init: true, } diff --git a/pageserver/src/tenant.rs b/pageserver/src/tenant.rs index e7c258d82992..c92d56e1adbe 100644 --- a/pageserver/src/tenant.rs +++ b/pageserver/src/tenant.rs @@ -7757,13 +7757,13 @@ mod tests { ( get_key(3), Lsn(0x20), - Value::WalRecord(NeonWalRecord::wal_clear()), + Value::WalRecord(NeonWalRecord::wal_clear("c")), ), (get_key(4), Lsn(0x10), Value::Image("0x10".into())), ( get_key(4), Lsn(0x20), - Value::WalRecord(NeonWalRecord::wal_init()), + Value::WalRecord(NeonWalRecord::wal_init("i")), ), ]; let image1 = vec![(get_key(1), "0x10".into())]; @@ -7912,8 +7912,30 @@ mod tests { #[cfg(feature = "testing")] #[tokio::test] - async fn test_simple_bottom_most_compaction_deltas() -> anyhow::Result<()> { - let harness = TenantHarness::create("test_simple_bottom_most_compaction_deltas").await?; + async fn test_simple_bottom_most_compaction_deltas_1() -> anyhow::Result<()> { + test_simple_bottom_most_compaction_deltas_helper( + "test_simple_bottom_most_compaction_deltas_1", + false, + ) + .await + } + + #[cfg(feature = "testing")] + #[tokio::test] + async fn test_simple_bottom_most_compaction_deltas_2() -> anyhow::Result<()> { + test_simple_bottom_most_compaction_deltas_helper( + "test_simple_bottom_most_compaction_deltas_2", + true, + ) + .await + } + + #[cfg(feature = "testing")] + async fn test_simple_bottom_most_compaction_deltas_helper( + test_name: &'static str, + use_delta_bottom_layer: bool, + ) -> anyhow::Result<()> { + let harness = TenantHarness::create(test_name).await?; let (tenant, ctx) = harness.load().await; fn get_key(id: u32) -> Key { @@ -7944,6 +7966,16 @@ mod tests { let img_layer = (0..10) .map(|id| (get_key(id), Bytes::from(format!("value {id}@0x10")))) .collect_vec(); + // or, delta layer at 0x10 if `use_delta_bottom_layer` is true + let delta4 = (0..10) + .map(|id| { + ( + get_key(id), + Lsn(0x08), + Value::WalRecord(NeonWalRecord::wal_init(format!("value {id}@0x10"))), + ) + }) + .collect_vec(); let delta1 = vec![ ( @@ -7997,21 +8029,61 @@ mod tests { ), ]; - let tline = tenant - .create_test_timeline_with_layers( - TIMELINE_ID, - Lsn(0x10), - DEFAULT_PG_VERSION, - &ctx, - vec![ - DeltaLayerTestDesc::new_with_inferred_key_range(Lsn(0x10)..Lsn(0x48), delta1), - DeltaLayerTestDesc::new_with_inferred_key_range(Lsn(0x10)..Lsn(0x48), delta2), - DeltaLayerTestDesc::new_with_inferred_key_range(Lsn(0x48)..Lsn(0x50), delta3), - ], // delta layers - vec![(Lsn(0x10), img_layer)], // image layers - Lsn(0x50), - ) - .await?; + let tline = if use_delta_bottom_layer { + tenant + .create_test_timeline_with_layers( + TIMELINE_ID, + Lsn(0x08), + DEFAULT_PG_VERSION, + &ctx, + vec![ + DeltaLayerTestDesc::new_with_inferred_key_range( + Lsn(0x20)..Lsn(0x48), + delta1, + ), + DeltaLayerTestDesc::new_with_inferred_key_range( + Lsn(0x20)..Lsn(0x48), + delta2, + ), + DeltaLayerTestDesc::new_with_inferred_key_range( + Lsn(0x48)..Lsn(0x50), + delta3, + ), + DeltaLayerTestDesc::new_with_inferred_key_range( + Lsn(0x08)..Lsn(0x10), + delta4, + ), + ], // delta layers + vec![], // image layers + Lsn(0x50), + ) + .await? + } else { + tenant + .create_test_timeline_with_layers( + TIMELINE_ID, + Lsn(0x10), + DEFAULT_PG_VERSION, + &ctx, + vec![ + DeltaLayerTestDesc::new_with_inferred_key_range( + Lsn(0x10)..Lsn(0x48), + delta1, + ), + DeltaLayerTestDesc::new_with_inferred_key_range( + Lsn(0x10)..Lsn(0x48), + delta2, + ), + DeltaLayerTestDesc::new_with_inferred_key_range( + Lsn(0x48)..Lsn(0x50), + delta3, + ), + ], // delta layers + vec![(Lsn(0x10), img_layer)], // image layers + Lsn(0x50), + ) + .await? + }; { // Update GC info let mut guard = tline.gc_info.write().unwrap(); @@ -8183,7 +8255,7 @@ mod tests { Lsn(0x20), KeyLogAtLsn(vec![( Lsn(0x20), - Value::Image(Bytes::copy_from_slice(b"0x10;0x20")), + Value::WalRecord(NeonWalRecord::wal_init(";0x20")), )]), ), ( @@ -9165,7 +9237,7 @@ mod tests { let will_init = will_init_keys.contains(&i); if will_init { - delta_layer_spec.push((key, lsn, Value::WalRecord(NeonWalRecord::wal_init()))); + delta_layer_spec.push((key, lsn, Value::WalRecord(NeonWalRecord::wal_init("")))); expected_key_values.insert(key, "".to_string()); } else { diff --git a/pageserver/src/tenant/storage_layer/merge_iterator.rs b/pageserver/src/tenant/storage_layer/merge_iterator.rs index 2667d130f585..19cfcb0867ea 100644 --- a/pageserver/src/tenant/storage_layer/merge_iterator.rs +++ b/pageserver/src/tenant/storage_layer/merge_iterator.rs @@ -562,7 +562,7 @@ mod tests { ( get_key(0), Lsn(0x10), - Value::WalRecord(NeonWalRecord::wal_init()), + Value::WalRecord(NeonWalRecord::wal_init("")), ), ( get_key(0), @@ -572,7 +572,7 @@ mod tests { ( get_key(5), Lsn(0x10), - Value::WalRecord(NeonWalRecord::wal_init()), + Value::WalRecord(NeonWalRecord::wal_init("")), ), ( get_key(5), diff --git a/pageserver/src/walredo/apply_neon.rs b/pageserver/src/walredo/apply_neon.rs index d712d8bf5efc..78601d87af04 100644 --- a/pageserver/src/walredo/apply_neon.rs +++ b/pageserver/src/walredo/apply_neon.rs @@ -253,6 +253,10 @@ pub(crate) fn apply_in_neon( use bytes::BufMut; if *will_init { assert!(*clear, "init record must be clear to ensure correctness"); + assert!( + page.is_empty(), + "init record must be the first entry to ensure correctness" + ); } if *clear { page.clear(); From 32156466b85c4731d8739d6d75b58c2299e42b95 Mon Sep 17 00:00:00 2001 From: Alex Chi Z Date: Mon, 11 Nov 2024 17:42:34 -0500 Subject: [PATCH 2/3] fix Signed-off-by: Alex Chi Z --- pageserver/src/tenant.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pageserver/src/tenant.rs b/pageserver/src/tenant.rs index c92d56e1adbe..55f1b5ea4605 100644 --- a/pageserver/src/tenant.rs +++ b/pageserver/src/tenant.rs @@ -8033,7 +8033,7 @@ mod tests { tenant .create_test_timeline_with_layers( TIMELINE_ID, - Lsn(0x08), + Lsn(0x08c), DEFAULT_PG_VERSION, &ctx, vec![ @@ -8193,7 +8193,7 @@ mod tests { ( key, Lsn(0x10), - Value::Image(Bytes::copy_from_slice(b"0x10")), + Value::WalRecord(NeonWalRecord::wal_init("0x10")), ), ( key, @@ -8255,7 +8255,7 @@ mod tests { Lsn(0x20), KeyLogAtLsn(vec![( Lsn(0x20), - Value::WalRecord(NeonWalRecord::wal_init(";0x20")), + Value::Image(Bytes::from_static(b"0x10;0x20")), )]), ), ( From 4a5d89e314c16fa30d93f24f9e0bfb8e124be3b7 Mon Sep 17 00:00:00 2001 From: Alex Chi Z Date: Mon, 11 Nov 2024 17:44:03 -0500 Subject: [PATCH 3/3] fix fmt Signed-off-by: Alex Chi Z --- pageserver/src/tenant.rs | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/pageserver/src/tenant.rs b/pageserver/src/tenant.rs index 55f1b5ea4605..d0a96e78a65b 100644 --- a/pageserver/src/tenant.rs +++ b/pageserver/src/tenant.rs @@ -8033,10 +8033,14 @@ mod tests { tenant .create_test_timeline_with_layers( TIMELINE_ID, - Lsn(0x08c), + Lsn(0x08), DEFAULT_PG_VERSION, &ctx, vec![ + DeltaLayerTestDesc::new_with_inferred_key_range( + Lsn(0x08)..Lsn(0x10), + delta4, + ), DeltaLayerTestDesc::new_with_inferred_key_range( Lsn(0x20)..Lsn(0x48), delta1, @@ -8049,10 +8053,6 @@ mod tests { Lsn(0x48)..Lsn(0x50), delta3, ), - DeltaLayerTestDesc::new_with_inferred_key_range( - Lsn(0x08)..Lsn(0x10), - delta4, - ), ], // delta layers vec![], // image layers Lsn(0x50),