From e44f4f5c3a36e16245831dcc372baec968cc026b Mon Sep 17 00:00:00 2001 From: Anuchaithra Date: Tue, 4 Jun 2024 15:26:46 +0530 Subject: [PATCH] [Tier-3]Bucket Granular sync policy with storage class Signed-off-by: Anuchaithra --- ...cket_granularsync_storage_class_direc.yaml | 34 +++++++++++++++ ...ucket_granularsync_storage_class_symm.yaml | 29 +++++++++++++ ...t_multisite_bucket_granular_sync_policy.py | 41 +++++++++++++++++-- 3 files changed, 101 insertions(+), 3 deletions(-) create mode 100644 rgw/v2/tests/s3_swift/multisite_configs/test_bucket_granularsync_storage_class_direc.yaml create mode 100644 rgw/v2/tests/s3_swift/multisite_configs/test_bucket_granularsync_storage_class_symm.yaml diff --git a/rgw/v2/tests/s3_swift/multisite_configs/test_bucket_granularsync_storage_class_direc.yaml b/rgw/v2/tests/s3_swift/multisite_configs/test_bucket_granularsync_storage_class_direc.yaml new file mode 100644 index 000000000..e5d6e407d --- /dev/null +++ b/rgw/v2/tests/s3_swift/multisite_configs/test_bucket_granularsync_storage_class_direc.yaml @@ -0,0 +1,34 @@ +# Polarian TC : CEPH-83575143 +# script: test_s3cmd_bucket_granular_sync_policy.py +config: + user_count: 1 + bucket_count: 1 + objects_count: 100 + pool_name: data.cold + storage_class: cold + objects_size_range: + min: 5K + max: 2M + test_ops: + dest_param_storage_class: true + zonegroup_group: true + zonegroup_status: allowed + zonegroup_flow: true + zonegroup_flow_type: directional + zonegroup_source_zone: primary + zonegroup_dest_zone: secondary + zonegroup_source_zones: primary + zonegroup_dest_zones: secondary + zonegroup_pipe: true + bucket_group: true + bucket_status: enabled + bucket_flow: false + bucket_pipe: true + bucket_source_zones: primary + bucket_dest_zones: secondary + bucket_policy_details: --storage-class cold + create_object: true + create_bucket: true + should_sync: true + write_io_verify_another_site: true + zonegroup_group_remove: true diff --git a/rgw/v2/tests/s3_swift/multisite_configs/test_bucket_granularsync_storage_class_symm.yaml b/rgw/v2/tests/s3_swift/multisite_configs/test_bucket_granularsync_storage_class_symm.yaml new file mode 100644 index 000000000..3bf7160d8 --- /dev/null +++ b/rgw/v2/tests/s3_swift/multisite_configs/test_bucket_granularsync_storage_class_symm.yaml @@ -0,0 +1,29 @@ +# Polarian TC : CEPH-83575143 +# script: test_s3cmd_bucket_granular_sync_policy.py +config: + user_count: 1 + bucket_count: 1 + objects_count: 100 + pool_name: data.glacier + storage_class: glacier + objects_size_range: + min: 5K + max: 2M + test_ops: + dest_param_storage_class: true + zonegroup_group: true + zonegroup_status: allowed + zonegroup_flow: true + zonegroup_flow_type: symmetrical + zonegroup_pipe: true + bucket_group: true + bucket_status: enabled + bucket_flow: false + bucket_pipe: true + bucket_policy_details: --storage-class glacier + create_object: true + create_bucket: true + should_sync: true + write_io_verify_another_site: true + write_io_verify_should_sync: true + zonegroup_group_remove: true diff --git a/rgw/v2/tests/s3_swift/test_multisite_bucket_granular_sync_policy.py b/rgw/v2/tests/s3_swift/test_multisite_bucket_granular_sync_policy.py index 74e0fd842..c7a47afd5 100644 --- a/rgw/v2/tests/s3_swift/test_multisite_bucket_granular_sync_policy.py +++ b/rgw/v2/tests/s3_swift/test_multisite_bucket_granular_sync_policy.py @@ -13,7 +13,8 @@ multisite_configs/test_multisite_granular_bucketsync_sync_to_diff_bucket.yaml multisite_configs/test_multisite_granular_bucketsync_sync_from_diff_bucket.yaml multisite_configs/test_multisite_granular_bucketsync_archive_symmetrical.yaml - multisite_configs/test_multisite_granular_bucketsync_archive_directional.yaml + multisite_configs/test_bucket_granularsync_storage_class_symm.yaml + multisite_configs/test_bucket_granularsync_storage_class_direc.yaml Operation: Creates delete sync policy group bucket , zonegroupl level @@ -110,6 +111,9 @@ def test_exec(config, ssh_con): reusable.verify_bucket_sync_on_other_site(rgw_ssh_con, bucket) buckets.append(bucket) + if config.test_ops.get("dest_param_storage_class", False): + reusable.create_storage_class_in_all_zones(zone['name'], rgw_ssh_con, config) + if utils.is_cluster_multisite(): if config.test_ops.get("zonegroup_group", False): group_status = config.test_ops["zonegroup_status"] @@ -202,11 +206,14 @@ def test_exec(config, ssh_con): bucket_details = config.test_ops.get( "bucket_policy_details", None ) + if bucket_details is not None: + bucket_details = " " + bucket_details + pipe_id = None if config.test_ops.get( "sync_to_diff_bucket", False ): - bucket_details = " " + bucket_details.replace( + bucket_details = bucket_details.replace( "", f"{bkt.name}-new-{pipec}", ) @@ -215,7 +222,7 @@ def test_exec(config, ssh_con): if config.test_ops.get( "sync_from_diff_bucket", False ): - bucket_details = " " + bucket_details.replace( + bucket_details = bucket_details.replace( "", old_bucket ) @@ -289,6 +296,20 @@ def test_exec(config, ssh_con): reusable.verify_object_sync_on_other_site( rgw_ssh_con, bkt, config ) + + if config.test_ops.get("dest_param_storage_class", False): + log.info("Start the validation of object sync in destination with staorage class") + _, out, _ = rgw_ssh_con.exec_command( + f"radosgw-admin bucket list --bucket {bkt.name}" + ) + bkt_list = json.loads(out.read().decode()) + for obj in bkt_list: + if obj["name"].startswith(f"key_{bkt.name}_"): + if obj["meta"]["storage_class"] != config.storage_class: + raise TestExecError( + f"object synced to non-master for bucket {bkt.name}, does not belong to storage class {config.storage_class}" + ) + else: time.sleep(1200) _, stdout, _ = rgw_ssh_con.exec_command( @@ -478,6 +499,20 @@ def test_exec(config, ssh_con): f"Object synced for bucket {bkt.name}, on another site as expected" ) + if config.test_ops.get("dest_param_storage_class", False): + log.info("Start the validation of object sync in destination with staorage class") + bkt_list = json.loads( + utils.exec_shell_cmd( + f"radosgw-admin bucket list --bucket {bkt.name}" + ) + ) + for obj in bkt_list: + if obj["name"].startswith(f"new-key_{bkt.name}_"): + if obj["meta"]["storage_class"] != config.storage_class: + raise TestExecError( + f"object synced to master for bucket {bkt.name}, does not belong to storage class {config.storage_class}" + ) + elif config.test_ops.get( "sync_to_diff_bucket", False ) or config.test_ops.get("sync_from_diff_bucket", False):