From b53a5e7b09045c4fdd3dc1261f8dfc4a61d82751 Mon Sep 17 00:00:00 2001 From: ckulal Date: Mon, 4 Mar 2024 17:45:02 +0530 Subject: [PATCH] [AWS]:Verification -ETag should not be empty during complete multipart upload Signed-off-by: ckulal --- ...plete_multipart_upload_etag_not_empty.yaml | 13 ++ rgw/v2/tests/aws/reusable.py | 218 +++++++++++++++++- rgw/v2/tests/aws/test_aws.py | 21 ++ 3 files changed, 251 insertions(+), 1 deletion(-) create mode 100644 rgw/v2/tests/aws/configs/test_complete_multipart_upload_etag_not_empty.yaml diff --git a/rgw/v2/tests/aws/configs/test_complete_multipart_upload_etag_not_empty.yaml b/rgw/v2/tests/aws/configs/test_complete_multipart_upload_etag_not_empty.yaml new file mode 100644 index 000000000..4b764d4b8 --- /dev/null +++ b/rgw/v2/tests/aws/configs/test_complete_multipart_upload_etag_not_empty.yaml @@ -0,0 +1,13 @@ +# BZ: BZ2266579 +# Polarian-id: CEPH-9801 +# Script: ceph-qe-scripts/rgw/v2/tests/aws/test_aws.py +config: + user_count: 1 + bucket_count: 1 + objects_count: 1 + local_file_delete: true + objects_size_range: + min: 15M + max: 30M + test_ops: + verify_etag_for_complete_multipart_upload: true diff --git a/rgw/v2/tests/aws/reusable.py b/rgw/v2/tests/aws/reusable.py index 835c8fa27..6b442400a 100644 --- a/rgw/v2/tests/aws/reusable.py +++ b/rgw/v2/tests/aws/reusable.py @@ -3,6 +3,7 @@ """ +import glob import json import logging import os @@ -18,7 +19,7 @@ import v2.utils.utils as utils from v2.lib.aws.resource_op import AWS -from v2.lib.exceptions import AWSCommandExecError +from v2.lib.exceptions import AWSCommandExecError, TestExecError from v2.lib.manage_data import io_generator @@ -75,6 +76,132 @@ def list_object_versions(bucket_name, end_point, ssl=None): raise AWSCommandExecError(message=str(e)) +def create_multipart_upload(bucket_name, key_name, end_point, ssl=None): + """ + Initiate multipart uploads for given object on a given bucket + Ex: /usr/local/bin/aws s3api create-multipart-upload --bucket --key --endpoint + Args: + bucket_name(str): Name of the bucket + key_name(str): Name of the object for which multipart upload has to be initiated + end_point(str): endpoint + ssl: + Return: + Response of create-multipart-upload + """ + create_mp_method = AWS(operation="create-multipart-upload") + if ssl: + ssl_param = "-s" + else: + ssl_param = " " + command = create_mp_method.command( + params=[ + f"--bucket {bucket_name} --key {key_name} --endpoint-url {end_point}", + ssl_param, + ] + ) + try: + response = utils.exec_shell_cmd(command) + if not response: + raise Exception( + f"creating multipart upload failed for bucket {bucket_name} with object name {key_name}" + ) + return response + except Exception as e: + raise AWSCommandExecError(message=str(e)) + + +def upload_part( + bucket_name, key_name, part_number, upload_id, body, end_point, ssl=None +): + """ + Upload part to the key in a bucket + Ex: /usr/local/bin/aws s3api upload-part --bucket --key --part-number + --upload-id --body --endpoint + + Args: + bucket_name(str): Name of the bucket + key_name(str): Name of the object for which part has to be uploaded + part_number(int): part number + upload_id(str): upload id fetched during initiating multipart upload + body(str): part file which needed to be uploaded + end_point(str): endpoint + ssl: + Return: + Response of uplaod_part i.e Etag + """ + upload_part_method = AWS(operation="upload-part") + if ssl: + ssl_param = "-s" + else: + ssl_param = " " + command = upload_part_method.command( + params=[ + f"--bucket {bucket_name} --key {key_name} --part-number {part_number} --upload-id {upload_id}" + f" --body {body} --endpoint-url {end_point}", + ssl_param, + ] + ) + try: + response = utils.exec_shell_cmd(command) + if not response: + raise Exception( + f"Uploading part failed for bucket {bucket_name} with key {key_name} and upload id" + f" {upload_id}" + ) + return response + except Exception as e: + raise AWSCommandExecError(message=str(e)) + + +def complete_multipart_upload( + bucket_name, key_name, upload_file, upload_id, end_point, ssl=None +): + """ + Complete multipart uploads for given object on a given bucket + Ex: /usr/local/bin/aws s3api complete-multipart-upload --multipart-upload file:// + --bucket --key --upload-id --endpoint + Args: + upload_file(str): Name of a file containing mpstructure + ex: { + "Parts": [ + { + "ETag": "e868e0f4719e394144ef36531ee6824c", + "PartNumber": 1 + } + ] + } + bucket_name(str): Name of the bucket + key_name(str): Name of the object for which multipart upload has to be Completed + upload_id(str): upload id fetched during initiating multipart upload + end_point(str): endpoint + ssl: + Return: + Response of create-multipart-upload + """ + complete_mp_method = AWS(operation="complete-multipart-upload") + if ssl: + ssl_param = "-s" + else: + ssl_param = " " + command = complete_mp_method.command( + params=[ + f"--multipart-upload file://{upload_file} --bucket {bucket_name} --key {key_name} --upload-id {upload_id} " + f"--endpoint-url {end_point}", + ssl_param, + ] + ) + try: + response = utils.exec_shell_cmd(command) + if not response: + raise Exception( + f"creating multipart upload failed for bucket {bucket_name} with key {key_name} and" + f" upload id {upload_id}" + ) + return response + except Exception as e: + raise AWSCommandExecError(message=str(e)) + + def put_object(bucket_name, object_name, end_point, ssl=None): """ Put/uploads object to the bucket @@ -265,3 +392,92 @@ def verify_object_with_version_id_null( raise AssertionError( f"Object with version id null is not Deleted at the endpoint {endpoint}!" ) + + +def upload_multipart_aws( + bucket_name, + key_name, + TEST_DATA_PATH, + endpoint, + config, + append_data=False, + append_msg=None, +): + """ + Args: + bucket_name(str): Name of the bucket + key_name(str): Name of the object + TEST_DATA_PATH(str): Test data path + endpoint(str): endpoint url + config: configuration used + append_data(boolean) + append_msg(str) + Return: + Response of aws complete multipart upload operation + """ + log.info("Create multipart upload") + create_mp_upload_resp = create_multipart_upload(bucket_name, key_name, endpoint) + upload_id = json.loads(create_mp_upload_resp)["UploadId"] + + log.info(f"object name: {key_name}") + object_path = os.path.join(TEST_DATA_PATH, key_name) + log.info(f"object path: {object_path}") + object_size = config.obj_size + log.info(f"object_size: {object_size}") + split_size = config.split_size if hasattr(config, "split_size") else 5 + log.info(f"split size: {split_size}") + if append_data is True: + data_info = io_generator( + object_path, + object_size, + op="append", + **{"message": "\n%s" % append_msg}, + ) + else: + data_info = io_generator(object_path, object_size) + if data_info is False: + TestExecError("data creation failed") + + mp_dir = os.path.join(TEST_DATA_PATH, key_name + ".mp.parts") + log.info(f"mp part dir: {mp_dir}") + log.info("making multipart object part dir") + mkdir = utils.exec_shell_cmd(f"sudo mkdir {mp_dir}") + if mkdir is False: + raise TestExecError("mkdir failed creating mp_dir_name") + utils.split_file(object_path, split_size, mp_dir + "/") + parts_list = sorted(glob.glob(mp_dir + "/" + "*")) + log.info("parts_list: %s" % parts_list) + + part_number = 1 + mpstructure = {"Parts": []} + log.info("no of parts: %s" % len(parts_list)) + + for each_part in parts_list: + log.info(f"upload part {part_number} of object: {key_name}") + upload_part_resp = json.loads( + upload_part( + bucket_name, key_name, part_number, upload_id, each_part, endpoint + ) + ) + part_info = {"PartNumber": part_number, "ETag": upload_part_resp["ETag"]} + mpstructure["Parts"].append(part_info) + if each_part != parts_list[-1]: + # increase the part number only if the current part is not the last part + part_number += 1 + log.info("curr part_number: %s" % part_number) + os.system("touch mpstructure.json") + with open("mpstructure.json", "w") as fd: + json.dump(mpstructure, fd) + log.info(f"mpstructure data is: {mpstructure}") + if config.local_file_delete is True: + log.info("deleting local file part") + utils.exec_shell_cmd(f"rm -rf {mp_dir}") + + if len(parts_list) == part_number: + log.info("all parts upload completed") + complete_multipart_upload_resp = json.loads( + complete_multipart_upload( + bucket_name, key_name, "mpstructure.json", upload_id, endpoint + ) + ) + return complete_multipart_upload_resp diff --git a/rgw/v2/tests/aws/test_aws.py b/rgw/v2/tests/aws/test_aws.py index 5e31df418..87286c9ce 100644 --- a/rgw/v2/tests/aws/test_aws.py +++ b/rgw/v2/tests/aws/test_aws.py @@ -4,6 +4,7 @@ Note: Following yaml can be used configs/test_aws_versioned_bucket_creation.yaml + configs/test_complete_multipart_upload_etag_not_empty.yaml Operation: @@ -70,6 +71,24 @@ def test_exec(config, ssh_con): if config.test_ops.get("enable_version", False): log.info(f"bucket versioning test on bucket: {bucket_name}") aws_reusable.put_get_bucket_versioning(bucket_name, endpoint) + if config.test_ops.get("verify_etag_for_complete_multipart_upload", False): + log.info( + f"Verifying ETag element for complete multipart upload is not empty string" + ) + for oc, size in list(config.mapped_sizes.items()): + config.obj_size = size + key_name = utils.gen_s3_object_name(bucket_name, oc) + complete_multipart_upload_resp = aws_reusable.upload_multipart_aws( + bucket_name, + key_name, + TEST_DATA_PATH, + endpoint, + config, + ) + if not complete_multipart_upload_resp["ETag"]: + raise AssertionError( + "Etag not generated during complete multipart upload operation" + ) if config.user_remove is True: s3_reusable.remove_user(user) @@ -115,6 +134,8 @@ def test_exec(config, ssh_con): configure_logging(f_name=log_f_name, set_level=args.log_level.upper()) config = resource_op.Config(yaml_file) config.read() + if config.mapped_sizes is None: + config.mapped_sizes = utils.make_mapped_sizes(config) test_exec(config, ssh_con) test_info.success_status("test passed") sys.exit(0)