Skip to content

Commit

Permalink
[AWS]:Verification -ETag should not be empty during complete multipar…
Browse files Browse the repository at this point in the history 
…t upload

Signed-off-by: ckulal <[email protected]>
  • Loading branch information
@ckulal
ckulal committed Mar 7, 2024
1 parent 7d0f766 commit b53a5e7
Show file tree
Hide file tree
Showing 3 changed files with 251 additions and 1 deletion.
13 changes: 13 additions & 0 deletions rgw/v2/tests/aws/configs/test_complete_multipart_upload_etag_not_empty.yaml
View file
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
# BZ: BZ2266579
# Polarian-id: CEPH-9801
# Script: ceph-qe-scripts/rgw/v2/tests/aws/test_aws.py
config:
user_count: 1
bucket_count: 1
objects_count: 1
local_file_delete: true
objects_size_range:
min: 15M
max: 30M
test_ops:
verify_etag_for_complete_multipart_upload: true
218 changes: 217 additions & 1 deletion rgw/v2/tests/aws/reusable.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
"""


import glob
import json
import logging
import os
Expand All @@ -18,7 +19,7 @@

import v2.utils.utils as utils
from v2.lib.aws.resource_op import AWS
from v2.lib.exceptions import AWSCommandExecError
from v2.lib.exceptions import AWSCommandExecError, TestExecError
from v2.lib.manage_data import io_generator


Expand Down Expand Up @@ -75,6 +76,132 @@ def list_object_versions(bucket_name, end_point, ssl=None):
raise AWSCommandExecError(message=str(e))


def create_multipart_upload(bucket_name, key_name, end_point, ssl=None):
"""
Initiate multipart uploads for given object on a given bucket
Ex: /usr/local/bin/aws s3api create-multipart-upload --bucket <bucket_name> --key <key_name> --endpoint <endpoint_url>
Args:
bucket_name(str): Name of the bucket
key_name(str): Name of the object for which multipart upload has to be initiated
end_point(str): endpoint
ssl:
Return:
Response of create-multipart-upload
"""
create_mp_method = AWS(operation="create-multipart-upload")
if ssl:
ssl_param = "-s"
else:
ssl_param = " "
command = create_mp_method.command(
params=[
f"--bucket {bucket_name} --key {key_name} --endpoint-url {end_point}",
ssl_param,
]
)
try:
response = utils.exec_shell_cmd(command)
if not response:
raise Exception(
f"creating multipart upload failed for bucket {bucket_name} with object name {key_name}"
)
return response
except Exception as e:
raise AWSCommandExecError(message=str(e))


def upload_part(
bucket_name, key_name, part_number, upload_id, body, end_point, ssl=None
):
"""
Upload part to the key in a bucket
Ex: /usr/local/bin/aws s3api upload-part --bucket <bucket_name> --key <key_name> --part-number <part_number>
--upload-id <upload_id> --body <body> --endpoint <endpoint_url>
Args:
bucket_name(str): Name of the bucket
key_name(str): Name of the object for which part has to be uploaded
part_number(int): part number
upload_id(str): upload id fetched during initiating multipart upload
body(str): part file which needed to be uploaded
end_point(str): endpoint
ssl:
Return:
Response of uplaod_part i.e Etag
"""
upload_part_method = AWS(operation="upload-part")
if ssl:
ssl_param = "-s"
else:
ssl_param = " "
command = upload_part_method.command(
params=[
f"--bucket {bucket_name} --key {key_name} --part-number {part_number} --upload-id {upload_id}"
f" --body {body} --endpoint-url {end_point}",
ssl_param,
]
)
try:
response = utils.exec_shell_cmd(command)
if not response:
raise Exception(
f"Uploading part failed for bucket {bucket_name} with key {key_name} and upload id"
f" {upload_id}"
)
return response
except Exception as e:
raise AWSCommandExecError(message=str(e))


def complete_multipart_upload(
bucket_name, key_name, upload_file, upload_id, end_point, ssl=None
):
"""
Complete multipart uploads for given object on a given bucket
Ex: /usr/local/bin/aws s3api complete-multipart-upload --multipart-upload file://<upload_file>
--bucket <bucket_name> --key <key_name> --upload-id <upload_id> --endpoint <endpoint_url>
Args:
upload_file(str): Name of a file containing mpstructure
ex: {
"Parts": [
{
"ETag": "e868e0f4719e394144ef36531ee6824c",
"PartNumber": 1
}
]
}
bucket_name(str): Name of the bucket
key_name(str): Name of the object for which multipart upload has to be Completed
upload_id(str): upload id fetched during initiating multipart upload
end_point(str): endpoint
ssl:
Return:
Response of create-multipart-upload
"""
complete_mp_method = AWS(operation="complete-multipart-upload")
if ssl:
ssl_param = "-s"
else:
ssl_param = " "
command = complete_mp_method.command(
params=[
f"--multipart-upload file://{upload_file} --bucket {bucket_name} --key {key_name} --upload-id {upload_id} "
f"--endpoint-url {end_point}",
ssl_param,
]
)
try:
response = utils.exec_shell_cmd(command)
if not response:
raise Exception(
f"creating multipart upload failed for bucket {bucket_name} with key {key_name} and"
f" upload id {upload_id}"
)
return response
except Exception as e:
raise AWSCommandExecError(message=str(e))


def put_object(bucket_name, object_name, end_point, ssl=None):
"""
Put/uploads object to the bucket
Expand Down Expand Up @@ -265,3 +392,92 @@ def verify_object_with_version_id_null(
raise AssertionError(
f"Object with version id null is not Deleted at the endpoint {endpoint}!"
)


def upload_multipart_aws(
bucket_name,
key_name,
TEST_DATA_PATH,
endpoint,
config,
append_data=False,
append_msg=None,
):
"""
Args:
bucket_name(str): Name of the bucket
key_name(str): Name of the object
TEST_DATA_PATH(str): Test data path
endpoint(str): endpoint url
config: configuration used
append_data(boolean)
append_msg(str)
Return:
Response of aws complete multipart upload operation
"""
log.info("Create multipart upload")
create_mp_upload_resp = create_multipart_upload(bucket_name, key_name, endpoint)
upload_id = json.loads(create_mp_upload_resp)["UploadId"]

log.info(f"object name: {key_name}")
object_path = os.path.join(TEST_DATA_PATH, key_name)
log.info(f"object path: {object_path}")
object_size = config.obj_size
log.info(f"object_size: {object_size}")
split_size = config.split_size if hasattr(config, "split_size") else 5
log.info(f"split size: {split_size}")
if append_data is True:
data_info = io_generator(
object_path,
object_size,
op="append",
**{"message": "\n%s" % append_msg},
)
else:
data_info = io_generator(object_path, object_size)
if data_info is False:
TestExecError("data creation failed")

mp_dir = os.path.join(TEST_DATA_PATH, key_name + ".mp.parts")
log.info(f"mp part dir: {mp_dir}")
log.info("making multipart object part dir")
mkdir = utils.exec_shell_cmd(f"sudo mkdir {mp_dir}")
if mkdir is False:
raise TestExecError("mkdir failed creating mp_dir_name")
utils.split_file(object_path, split_size, mp_dir + "/")
parts_list = sorted(glob.glob(mp_dir + "/" + "*"))
log.info("parts_list: %s" % parts_list)

part_number = 1
mpstructure = {"Parts": []}
log.info("no of parts: %s" % len(parts_list))

for each_part in parts_list:
log.info(f"upload part {part_number} of object: {key_name}")
upload_part_resp = json.loads(
upload_part(
bucket_name, key_name, part_number, upload_id, each_part, endpoint
)
)
part_info = {"PartNumber": part_number, "ETag": upload_part_resp["ETag"]}
mpstructure["Parts"].append(part_info)
if each_part != parts_list[-1]:
# increase the part number only if the current part is not the last part
part_number += 1
log.info("curr part_number: %s" % part_number)
os.system("touch mpstructure.json")
with open("mpstructure.json", "w") as fd:
json.dump(mpstructure, fd)
log.info(f"mpstructure data is: {mpstructure}")
if config.local_file_delete is True:
log.info("deleting local file part")
utils.exec_shell_cmd(f"rm -rf {mp_dir}")

if len(parts_list) == part_number:
log.info("all parts upload completed")
complete_multipart_upload_resp = json.loads(
complete_multipart_upload(
bucket_name, key_name, "mpstructure.json", upload_id, endpoint
)
)
return complete_multipart_upload_resp
21 changes: 21 additions & 0 deletions rgw/v2/tests/aws/test_aws.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
<input_yaml>
Note: Following yaml can be used
configs/test_aws_versioned_bucket_creation.yaml
configs/test_complete_multipart_upload_etag_not_empty.yaml
Operation:
Expand Down Expand Up @@ -70,6 +71,24 @@ def test_exec(config, ssh_con):
if config.test_ops.get("enable_version", False):
log.info(f"bucket versioning test on bucket: {bucket_name}")
aws_reusable.put_get_bucket_versioning(bucket_name, endpoint)
if config.test_ops.get("verify_etag_for_complete_multipart_upload", False):
log.info(
f"Verifying ETag element for complete multipart upload is not empty string"
)
for oc, size in list(config.mapped_sizes.items()):
config.obj_size = size
key_name = utils.gen_s3_object_name(bucket_name, oc)
complete_multipart_upload_resp = aws_reusable.upload_multipart_aws(
bucket_name,
key_name,
TEST_DATA_PATH,
endpoint,
config,
)
if not complete_multipart_upload_resp["ETag"]:
raise AssertionError(
"Etag not generated during complete multipart upload operation"
)

if config.user_remove is True:
s3_reusable.remove_user(user)
Expand Down Expand Up @@ -115,6 +134,8 @@ def test_exec(config, ssh_con):
configure_logging(f_name=log_f_name, set_level=args.log_level.upper())
config = resource_op.Config(yaml_file)
config.read()
if config.mapped_sizes is None:
config.mapped_sizes = utils.make_mapped_sizes(config)
test_exec(config, ssh_con)
test_info.success_status("test passed")
sys.exit(0)
Expand Down

0 comments on commit b53a5e7

Please sign in to comment.