From 2ee9021b213b9ab31bf08708e0c24fbcbacd11fa Mon Sep 17 00:00:00 2001 From: Alex Chi Z Date: Thu, 9 May 2024 14:43:36 -0400 Subject: [PATCH] rebase and add auto-upgrade if aux flag is set Signed-off-by: Alex Chi Z --- pageserver/src/tenant/metadata.rs | 177 ++++++++++++++++++++++++------ 1 file changed, 142 insertions(+), 35 deletions(-) diff --git a/pageserver/src/tenant/metadata.rs b/pageserver/src/tenant/metadata.rs index 12e619b630205..956c9cf0814e5 100644 --- a/pageserver/src/tenant/metadata.rs +++ b/pageserver/src/tenant/metadata.rs @@ -95,6 +95,21 @@ struct TimelineMetadataBodyV2 { pg_version: u32, } +impl TimelineMetadataBodyV2 { + pub fn from_v3(body: &TimelineMetadataBodyV3) -> Self { + assert!(body.last_aux_file_policy.is_none(), "cannot downgrade"); + Self { + disk_consistent_lsn: body.disk_consistent_lsn, + prev_record_lsn: body.prev_record_lsn, + ancestor_lsn: body.ancestor_lsn, + ancestor_timeline: body.ancestor_timeline, + latest_gc_cutoff_lsn: body.latest_gc_cutoff_lsn, + initdb_lsn: body.initdb_lsn, + pg_version: body.pg_version, + } + } +} + #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] struct TimelineMetadataBodyV1 { disk_consistent_lsn: Lsn, @@ -224,37 +239,39 @@ impl TimelineMetadata { } pub fn to_bytes(&self) -> Result, SerializeError> { - match self.hdr.format_version { - METADATA_OLD_FORMAT_VERSION_V2 => { - let body_bytes = self.body.ser()?; - let metadata_size = METADATA_HDR_SIZE + body_bytes.len(); - let hdr = TimelineMetadataHeader { - size: metadata_size as u16, - format_version: METADATA_OLD_FORMAT_VERSION_V2, - checksum: crc32c::crc32c(&body_bytes), - }; - let hdr_bytes = hdr.ser()?; - let mut metadata_bytes = vec![0u8; METADATA_MAX_SIZE]; - metadata_bytes[0..METADATA_HDR_SIZE].copy_from_slice(&hdr_bytes); - metadata_bytes[METADATA_HDR_SIZE..metadata_size].copy_from_slice(&body_bytes); - Ok(metadata_bytes) - } - METADATA_FORMAT_VERSION => { - let body_bytes = serde_json::to_vec(&self.body) - .map_err(|e| SerializeError::BadInput(e.into()))?; - let metadata_size = METADATA_HDR_SIZE + body_bytes.len(); - let hdr = TimelineMetadataHeader { - size: metadata_size as u16, - format_version: METADATA_FORMAT_VERSION, - checksum: crc32c::crc32c(&body_bytes), - }; - let hdr_bytes = hdr.ser()?; - let mut metadata_bytes = Vec::new(); - metadata_bytes.extend(hdr_bytes); - metadata_bytes.extend(body_bytes); - Ok(metadata_bytes) - } - _ => unreachable!(), + if self.hdr.format_version == METADATA_OLD_FORMAT_VERSION_V2 + && self.body.last_aux_file_policy.is_none() + { + let body_bytes = TimelineMetadataBodyV2::from_v3(&self.body).ser()?; + let metadata_size = METADATA_HDR_SIZE + body_bytes.len(); + let hdr = TimelineMetadataHeader { + size: metadata_size as u16, + format_version: METADATA_OLD_FORMAT_VERSION_V2, + checksum: crc32c::crc32c(&body_bytes), + }; + let hdr_bytes = hdr.ser()?; + let mut metadata_bytes = vec![0u8; METADATA_MAX_SIZE]; + metadata_bytes[0..METADATA_HDR_SIZE].copy_from_slice(&hdr_bytes); + metadata_bytes[METADATA_HDR_SIZE..metadata_size].copy_from_slice(&body_bytes); + Ok(metadata_bytes) + } else if self.hdr.format_version == METADATA_FORMAT_VERSION + || self.hdr.format_version == METADATA_OLD_FORMAT_VERSION_V2 + { + let body_bytes = + serde_json::to_vec(&self.body).map_err(|e| SerializeError::BadInput(e.into()))?; + let metadata_size = METADATA_HDR_SIZE + body_bytes.len(); + let hdr = TimelineMetadataHeader { + size: metadata_size as u16, + format_version: METADATA_FORMAT_VERSION, + checksum: crc32c::crc32c(&body_bytes), + }; + let hdr_bytes = hdr.ser()?; + let mut metadata_bytes = Vec::new(); + metadata_bytes.extend(hdr_bytes); + metadata_bytes.extend(body_bytes); + Ok(metadata_bytes) + } else { + unreachable!() } } @@ -558,6 +575,90 @@ mod tests { #[test] fn test_roundtrip_metadata_v2() { + let metadata_v2 = TimelineMetadata { + hdr: TimelineMetadataHeader { + checksum: 0, + size: 0, + format_version: METADATA_OLD_FORMAT_VERSION_V2, + }, + body: TimelineMetadataBodyV3 { + disk_consistent_lsn: Lsn(0x200), + prev_record_lsn: Some(Lsn(0x100)), + ancestor_timeline: Some(TIMELINE_ID), + ancestor_lsn: Lsn(0), + latest_gc_cutoff_lsn: Lsn(0), + initdb_lsn: Lsn(0), + pg_version: 16, + last_aux_file_policy: None, + }, + }; + + let metadata_bytes = metadata_v2 + .to_bytes() + .expect("Should serialize correct metadata to bytes"); + + // This should deserialize to the latest version format + let deserialized_metadata = TimelineMetadata::from_bytes(&metadata_bytes) + .expect("Should deserialize its own bytes"); + + let expected_metadata = TimelineMetadata::new( + Lsn(0x200), + Some(Lsn(0x100)), + Some(TIMELINE_ID), + Lsn(0), + Lsn(0), + Lsn(0), + 16, + None, + ); + + assert_eq!(deserialized_metadata.body, expected_metadata.body); + } + + #[test] + fn test_roundtrip_metadata_v3() { + let metadata_v3 = TimelineMetadata { + hdr: TimelineMetadataHeader { + checksum: 0, + size: 0, + format_version: METADATA_FORMAT_VERSION, + }, + body: TimelineMetadataBodyV3 { + disk_consistent_lsn: Lsn(0x200), + prev_record_lsn: Some(Lsn(0x100)), + ancestor_timeline: Some(TIMELINE_ID), + ancestor_lsn: Lsn(0), + latest_gc_cutoff_lsn: Lsn(0), + initdb_lsn: Lsn(0), + pg_version: 16, + last_aux_file_policy: Some(AuxFilePolicy::V2), + }, + }; + + let metadata_bytes = metadata_v3 + .to_bytes() + .expect("Should serialize correct metadata to bytes"); + + // This should deserialize to the latest version format + let deserialized_metadata = TimelineMetadata::from_bytes(&metadata_bytes) + .expect("Should deserialize its own bytes"); + + let expected_metadata = TimelineMetadata::new( + Lsn(0x200), + Some(Lsn(0x100)), + Some(TIMELINE_ID), + Lsn(0), + Lsn(0), + Lsn(0), + 16, + Some(AuxFilePolicy::V2), + ); + + assert_eq!(deserialized_metadata.body, expected_metadata.body); + } + + #[test] + fn test_roundtrip_metadata_aux_auto_upgrade() { let metadata_v2 = TimelineMetadata { hdr: TimelineMetadataHeader { checksum: 0, @@ -595,6 +696,10 @@ mod tests { Some(AuxFilePolicy::V2), ); + assert_eq!( + deserialized_metadata.hdr.format_version, + METADATA_FORMAT_VERSION + ); assert_eq!(deserialized_metadata.body, expected_metadata.body); } @@ -609,6 +714,7 @@ mod tests { Lsn(0), // Any version will do here, so use the default crate::DEFAULT_PG_VERSION, + None, ); original_metadata.hdr.format_version = METADATA_OLD_FORMAT_VERSION_V2; @@ -649,8 +755,7 @@ mod tests { let expected_metadata = { let mut temp_metadata = original_metadata; - let body_bytes = temp_metadata - .body + let body_bytes = TimelineMetadataBodyV2::from_v3(&temp_metadata.body) .ser() .expect("Cannot serialize the metadata body"); let metadata_size = METADATA_HDR_SIZE + body_bytes.len(); @@ -682,6 +787,7 @@ mod tests { latest_gc_cutoff_lsn: Lsn(0), initdb_lsn: Lsn(0), pg_version: 16, + last_aux_file_policy: Some(AuxFilePolicy::V2), }, }; @@ -691,7 +797,7 @@ mod tests { assert_eq!( &metadata_bytes[..METADATA_HDR_SIZE], - &[202, 106, 183, 219, 0, 205, 0, 5] + &[194, 55, 211, 146, 0, 233, 0, 5] ); let json_value: serde_json::Value = serde_json::from_slice(&metadata_bytes[METADATA_HDR_SIZE..]).unwrap(); @@ -704,7 +810,8 @@ mod tests { "initdb_lsn": "0/0", "latest_gc_cutoff_lsn": "0/0", "pg_version": 16, - "prev_record_lsn": "0/100" + "prev_record_lsn": "0/100", + "last_aux_file_policy": "V2" }) ); }