From 1749d9841f50fa7d2ad5ac4357bad110e2692ce2 Mon Sep 17 00:00:00 2001 From: Anuchaithra Date: Fri, 22 Mar 2024 07:43:56 +0530 Subject: [PATCH] [Tier-3]Test by adding almost 1K buckets to the resharding queue Signed-off-by: Anuchaithra --- .../test_dynamic_bucket_resharding.py | 22 ++- ...le_dynamic_resharding_with_10k_bucket.yaml | 8 ++ ...ble_dynamic_resharding_with_1k_bucket.yaml | 8 ++ rgw/v2/tests/s3cmd/test_s3cmd.py | 129 +++++++++++++++++- 4 files changed, 163 insertions(+), 4 deletions(-) create mode 100644 rgw/v2/tests/s3cmd/configs/test_disable_and_enable_dynamic_resharding_with_10k_bucket.yaml create mode 100644 rgw/v2/tests/s3cmd/configs/test_disable_and_enable_dynamic_resharding_with_1k_bucket.yaml diff --git a/rgw/v2/tests/s3_swift/test_dynamic_bucket_resharding.py b/rgw/v2/tests/s3_swift/test_dynamic_bucket_resharding.py index 1efcb5225..fb74fb87f 100644 --- a/rgw/v2/tests/s3_swift/test_dynamic_bucket_resharding.py +++ b/rgw/v2/tests/s3_swift/test_dynamic_bucket_resharding.py @@ -4,9 +4,25 @@ Usage: test_dynamic_bucket_resharding.py -c - Note: any one of these yamls can be used - test_manual_resharding.yaml - test_dynamic_resharding.yaml + Note: any one of these yamls can be used + conf exist in both configs/ and multisite_configs/ : + test_manual_resharding.yaml + test_dynamic_resharding.yaml + test_dynamic_resharding_without_bucket_delete.yaml + test_manual_resharding_without_bucket_delete.yaml + test_dynamic_resharding_with_version_without_bucket_delete.yaml + + multisite_configs/test_bucket_generation.yaml + multisite_configs/test_resharding_disable_in_zonegroup.yaml + multisite_configs/test_dynamic_resharding_quota_exceed.yaml + + configs/test_bucket_index_shards.yaml + configs/test_dbr_with_custom_objs_per_shard_and_max_dynamic_shard.yaml + configs/test_dbr_with_custom_objs_per_shard_max_dynamic_shard_and_reshard_thread_interval.yaml + configs/test_manual_resharding_with_version.yaml + configs/test_dynamic_resharding_with_version.yaml + configs/test_disable_and_enable_dynamic_resharding.yaml + configs/test_disable_and_enable_dynamic_resharding_with_1k_bucket.yaml Operation: Create user diff --git a/rgw/v2/tests/s3cmd/configs/test_disable_and_enable_dynamic_resharding_with_10k_bucket.yaml b/rgw/v2/tests/s3cmd/configs/test_disable_and_enable_dynamic_resharding_with_10k_bucket.yaml new file mode 100644 index 000000000..6c61dfe27 --- /dev/null +++ b/rgw/v2/tests/s3cmd/configs/test_disable_and_enable_dynamic_resharding_with_10k_bucket.yaml @@ -0,0 +1,8 @@ +# Polarian TC : CEPH-11478 +# script: test_s3cmd.py +config: + bucket_count: 10000 + objects_count: 15 + max_objects_per_shard: 1 + test_ops: + disable_and_enable_dynamic_reshard: true diff --git a/rgw/v2/tests/s3cmd/configs/test_disable_and_enable_dynamic_resharding_with_1k_bucket.yaml b/rgw/v2/tests/s3cmd/configs/test_disable_and_enable_dynamic_resharding_with_1k_bucket.yaml new file mode 100644 index 000000000..99af578e7 --- /dev/null +++ b/rgw/v2/tests/s3cmd/configs/test_disable_and_enable_dynamic_resharding_with_1k_bucket.yaml @@ -0,0 +1,8 @@ +# Polarian TC : CEPH-11478 +# script: test_s3cmd.py +config: + bucket_count: 1000 + objects_count: 15 + max_objects_per_shard: 1 + test_ops: + disable_and_enable_dynamic_reshard: true diff --git a/rgw/v2/tests/s3cmd/test_s3cmd.py b/rgw/v2/tests/s3cmd/test_s3cmd.py index a97b94d22..5d0c35d84 100644 --- a/rgw/v2/tests/s3cmd/test_s3cmd.py +++ b/rgw/v2/tests/s3cmd/test_s3cmd.py @@ -7,6 +7,8 @@ Note: Following yaml can be used test_s3cmd.yaml test_multiple_delete_marker_check.yaml + configs/test_disable_and_enable_dynamic_resharding_with_10k_bucket.yaml + configs/test_disable_and_enable_dynamic_resharding_with_1k_bucket.yaml Operation: Create an user @@ -19,10 +21,13 @@ import argparse +import datetime +import json import logging import os import socket import sys +import time import traceback sys.path.append(os.path.abspath(os.path.join(__file__, "../../../.."))) @@ -31,7 +36,7 @@ from v2.lib import resource_op from v2.lib.admin import UserMgmt from v2.lib.exceptions import RGWBaseException, TestExecError -from v2.lib.rgw_config_opts import CephConfOp +from v2.lib.rgw_config_opts import CephConfOp, ConfigOpts from v2.lib.s3.auth import Auth from v2.lib.s3.write_io_info import BasicIOInfoStructure, BucketIoInfo, IOInfoInitialize from v2.lib.s3cmd import auth as s3_auth @@ -199,6 +204,128 @@ def test_exec(config, ssh_con): else: log.info(f"Deleting life cycle rule via s3cmd is successful!") + elif config.test_ops.get("disable_and_enable_dynamic_reshard", False): + log.info("making changes to ceph.conf") + ceph_conf.set_to_ceph_conf( + "global", + ConfigOpts.rgw_max_objs_per_shard, + str(config.max_objects_per_shard), + ssh_con, + ) + + ceph_conf.set_to_ceph_conf( + "global", ConfigOpts.rgw_dynamic_resharding, "False", ssh_con + ) + ceph_conf.set_to_ceph_conf( + "global", + ConfigOpts.rgw_max_dynamic_shards, + str(config.max_rgw_dynamic_shards), + ssh_con, + ) + + ceph_conf.set_to_ceph_conf( + "global", + ConfigOpts.rgw_reshard_thread_interval, + str(config.rgw_reshard_thread_interval), + ssh_con, + ) + + log.info("trying to restart rgw services") + srv_restarted = rgw_service.restart(ssh_con) + time.sleep(30) + if srv_restarted is False: + raise TestExecError("RGW service restart failed") + else: + log.info("RGW service restarted sucessfully") + + log.info(f"Create buckets {config.bucket_count} and upload objects {config.objects_count}") + user_info = resource_op.create_users(no_of_users_to_create=1) + if config.bucket_count > 1000: + cmd = f"radosgw-admin user modify --uid={user_info[0]['user_id']} --max-buckets {config.bucket_count}" + utils.exec_shell_cmd(cmd) + s3_auth.do_auth(user_info[0], ip_and_port) + auth = Auth(user_info[0], ssh_con, ssl=config.ssl, haproxy=config.haproxy) + rgw_conn = auth.do_auth() + s3cmd_path = "/home/cephuser/venv/bin/s3cmd" + buckets = [] + + for bc in range(config.bucket_count): + bucket_name = utils.gen_bucket_name_from_userid( + user_info[0]["user_id"], rand_no=bc + ) + if bc == 0: + bucket_prefix = bucket_name[:-2] + s3cmd_reusable.create_bucket(bucket_name) + log.info(f"Bucket {bucket_name} created") + buckets.append(bucket_name) + log.info(f"uploading {config.objects_count} objects to bucket {bucket_name}") + utils.exec_shell_cmd(f"fallocate -l 1k obj1k") + + for obj in range(config.objects_count): + cmd = f"{s3cmd_path} put obj1k s3://{bucket_name}/object-{obj}" + utils.exec_shell_cmd(cmd) + + for bkt in buckets: + log.info("Expecting num shards of buckets to be a default value") + json_doc = json.loads(utils.exec_shell_cmd(f"radosgw-admin bucket stats --bucket {bkt}")) + num_objects = json_doc["usage"]["rgw.main"]["num_objects"] + if json_doc["num_shards"] != 11: + raise AssertionError( + f"disabling dynamic Re-sharding FAILED!, found {json_doc['num_shards']} shards expected 11" + ) + + log.info("Verify resharding list should not list the buckets as dynamic reshard is disabled") + reshard_list_op = json.loads(utils.exec_shell_cmd("radosgw-admin reshard list")) + if reshard_list_op: + reshard_list = [] + for reshard in reshard_list_op: + if reshard["bucket_name"].startswith(bucket_prefix): + reshard_list.append(reshard["bucket_name"]) + if len(reshard_list) != 0: + raise TestExecError( + f"Expected reshard list to be empty as dynamic reshard is deisabled {reshard_list}" + ) + + log.info("Set rgw_dynamic_resharding to True") + ceph_conf.set_to_ceph_conf( + "global", ConfigOpts.rgw_dynamic_resharding, "True", ssh_con + ) + + log.info("trying to restart rgw services") + srv_restarted = rgw_service.restart(ssh_con) + time.sleep(30) + if srv_restarted is False: + raise TestExecError("RGW service restart failed") + else: + log.info("RGW service restarted sucessfully") + + time_now = datetime.datetime.now() + log.info(f"Upload few more objects to the buckets at {time_now}") + for bkt in buckets: + num_obj = config.max_objects_per_shard * 2 + log.info(f"objects to create: {num_obj}") + for oc in range(num_obj): + cmd = f"{s3cmd_path} put obj1k s3://{bkt}/new-object-{oc}" + utils.exec_shell_cmd(cmd) + + time.sleep(config.rgw_reshard_thread_interval) + for bkt in buckets: + log.info("Expecting num shards to be a greater than 11 as rgw_dynamic_resharding enabled") + json_doc = json.loads(utils.exec_shell_cmd(f"radosgw-admin bucket stats --bucket {bkt}")) + num_objects = json_doc["usage"]["rgw.main"]["num_objects"] + if json_doc["num_shards"] == 11: + raise AssertionError( + f"Enabling dynamic Re-sharding FAILED !, found {json_doc['num_shards']} shards expected > 11" + ) + + time_curr = datetime.datetime.now() + log.info( + f"sucessfully completed dynamic resharding it took {time_now} to {time_curr}" + ) + + log.info("remove user created") + reusable.remove_user(user_info[0]) + else: user_name = resource_op.create_users(no_of_users_to_create=1)[0]["user_id"] tenant = "tenant"