Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[AWS]:Verification -ETag should not be empty during complete multipar… #564

Merged
merged 1 commit into from
Mar 7, 2024
Merged

[AWS]:Verification -ETag should not be empty during complete multipar… #564

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions rgw/v2/tests/aws/configs/test_complete_multipart_upload_etag_not_empty.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
# BZ: BZ2266579
# Polarian-id: CEPH-9801
# Script: ceph-qe-scripts/rgw/v2/tests/aws/test_aws.py
config:
user_count: 1
bucket_count: 1
objects_count: 1
local_file_delete: true
objects_size_range:
min: 15M
max: 30M
test_ops:
verify_etag_for_complete_multipart_upload: true
218 changes: 217 additions & 1 deletion rgw/v2/tests/aws/reusable.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@
"""


import glob
import json
import logging
import os
Expand All @@ -18,7 +19,7 @@

import v2.utils.utils as utils
from v2.lib.aws.resource_op import AWS
from v2.lib.exceptions import AWSCommandExecError
from v2.lib.exceptions import AWSCommandExecError, TestExecError
from v2.lib.manage_data import io_generator


Expand Down Expand Up @@ -75,6 +76,132 @@ def list_object_versions(bucket_name, end_point, ssl=None):
raise AWSCommandExecError(message=str(e))


def create_multipart_upload(bucket_name, key_name, end_point, ssl=None):
"""
Initiate multipart uploads for given object on a given bucket
Ex: /usr/local/bin/aws s3api create-multipart-upload --bucket <bucket_name> --key <key_name> --endpoint <endpoint_url>
Args:
bucket_name(str): Name of the bucket
key_name(str): Name of the object for which multipart upload has to be initiated
end_point(str): endpoint
ssl:
Return:
Response of create-multipart-upload
"""
create_mp_method = AWS(operation="create-multipart-upload")
if ssl:
ssl_param = "-s"
else:
ssl_param = " "
command = create_mp_method.command(
params=[
f"--bucket {bucket_name} --key {key_name} --endpoint-url {end_point}",
ssl_param,
]
)
try:
response = utils.exec_shell_cmd(command)
if not response:
raise Exception(
f"creating multipart upload failed for bucket {bucket_name} with object name {key_name}"
)
return response
except Exception as e:
raise AWSCommandExecError(message=str(e))


def upload_part(
bucket_name, key_name, part_number, upload_id, body, end_point, ssl=None
):
"""
Upload part to the key in a bucket
Ex: /usr/local/bin/aws s3api upload-part --bucket <bucket_name> --key <key_name> --part-number <part_number>
--upload-id <upload_id> --body <body> --endpoint <endpoint_url>

Args:
bucket_name(str): Name of the bucket
key_name(str): Name of the object for which part has to be uploaded
part_number(int): part number
upload_id(str): upload id fetched during initiating multipart upload
body(str): part file which needed to be uploaded
end_point(str): endpoint
ssl:
Return:
Response of uplaod_part i.e Etag
"""
upload_part_method = AWS(operation="upload-part")
if ssl:
ssl_param = "-s"
else:
ssl_param = " "
command = upload_part_method.command(
params=[
f"--bucket {bucket_name} --key {key_name} --part-number {part_number} --upload-id {upload_id}"
f" --body {body} --endpoint-url {end_point}",
ssl_param,
]
)
try:
response = utils.exec_shell_cmd(command)
if not response:
raise Exception(
f"Uploading part failed for bucket {bucket_name} with key {key_name} and upload id"
f" {upload_id}"
)
return response
except Exception as e:
raise AWSCommandExecError(message=str(e))


def complete_multipart_upload(
bucket_name, key_name, upload_file, upload_id, end_point, ssl=None
):
"""
Complete multipart uploads for given object on a given bucket
Ex: /usr/local/bin/aws s3api complete-multipart-upload --multipart-upload file://<upload_file>
--bucket <bucket_name> --key <key_name> --upload-id <upload_id> --endpoint <endpoint_url>
Args:
upload_file(str): Name of a file containing mpstructure
ex: {
"Parts": [
{
"ETag": "e868e0f4719e394144ef36531ee6824c",
"PartNumber": 1
}
]
}
bucket_name(str): Name of the bucket
key_name(str): Name of the object for which multipart upload has to be Completed
upload_id(str): upload id fetched during initiating multipart upload
end_point(str): endpoint
ssl:
Return:
Response of create-multipart-upload
"""
complete_mp_method = AWS(operation="complete-multipart-upload")
if ssl:
ssl_param = "-s"
else:
ssl_param = " "
command = complete_mp_method.command(
params=[
f"--multipart-upload file://{upload_file} --bucket {bucket_name} --key {key_name} --upload-id {upload_id} "
f"--endpoint-url {end_point}",
ssl_param,
]
)
try:
response = utils.exec_shell_cmd(command)
if not response:
raise Exception(
f"creating multipart upload failed for bucket {bucket_name} with key {key_name} and"
f" upload id {upload_id}"
)
return response
except Exception as e:
raise AWSCommandExecError(message=str(e))


def put_object(bucket_name, object_name, end_point, ssl=None):
"""
Put/uploads object to the bucket
Expand Down Expand Up @@ -265,3 +392,92 @@ def verify_object_with_version_id_null(
raise AssertionError(
f"Object with version id null is not Deleted at the endpoint {endpoint}!"
)


def upload_multipart_aws(
bucket_name,
key_name,
TEST_DATA_PATH,
endpoint,
config,
append_data=False,
append_msg=None,
):
"""
Args:
bucket_name(str): Name of the bucket
key_name(str): Name of the object
TEST_DATA_PATH(str): Test data path
endpoint(str): endpoint url
config: configuration used
append_data(boolean)
append_msg(str)
Return:
Response of aws complete multipart upload operation
"""
log.info("Create multipart upload")
create_mp_upload_resp = create_multipart_upload(bucket_name, key_name, endpoint)
upload_id = json.loads(create_mp_upload_resp)["UploadId"]

log.info(f"object name: {key_name}")
object_path = os.path.join(TEST_DATA_PATH, key_name)
log.info(f"object path: {object_path}")
object_size = config.obj_size
log.info(f"object_size: {object_size}")
split_size = config.split_size if hasattr(config, "split_size") else 5
log.info(f"split size: {split_size}")
if append_data is True:
data_info = io_generator(
object_path,
object_size,
op="append",
**{"message": "\n%s" % append_msg},
)
else:
data_info = io_generator(object_path, object_size)
if data_info is False:
TestExecError("data creation failed")

mp_dir = os.path.join(TEST_DATA_PATH, key_name + ".mp.parts")
log.info(f"mp part dir: {mp_dir}")
log.info("making multipart object part dir")
mkdir = utils.exec_shell_cmd(f"sudo mkdir {mp_dir}")
if mkdir is False:
raise TestExecError("mkdir failed creating mp_dir_name")
utils.split_file(object_path, split_size, mp_dir + "/")
parts_list = sorted(glob.glob(mp_dir + "/" + "*"))
log.info("parts_list: %s" % parts_list)

part_number = 1
mpstructure = {"Parts": []}
log.info("no of parts: %s" % len(parts_list))

for each_part in parts_list:
log.info(f"upload part {part_number} of object: {key_name}")
upload_part_resp = json.loads(
upload_part(
bucket_name, key_name, part_number, upload_id, each_part, endpoint
)
)
part_info = {"PartNumber": part_number, "ETag": upload_part_resp["ETag"]}
mpstructure["Parts"].append(part_info)
if each_part != parts_list[-1]:
# increase the part number only if the current part is not the last part
part_number += 1
log.info("curr part_number: %s" % part_number)
os.system("touch mpstructure.json")
with open("mpstructure.json", "w") as fd:
json.dump(mpstructure, fd)
log.info(f"mpstructure data is: {mpstructure}")
if config.local_file_delete is True:
log.info("deleting local file part")
utils.exec_shell_cmd(f"rm -rf {mp_dir}")

if len(parts_list) == part_number:
log.info("all parts upload completed")
complete_multipart_upload_resp = json.loads(
complete_multipart_upload(
bucket_name, key_name, "mpstructure.json", upload_id, endpoint
)
)
return complete_multipart_upload_resp
21 changes: 21 additions & 0 deletions rgw/v2/tests/aws/test_aws.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,7 @@
<input_yaml>
Note: Following yaml can be used
configs/test_aws_versioned_bucket_creation.yaml
configs/test_complete_multipart_upload_etag_not_empty.yaml

Operation:

Expand Down Expand Up @@ -70,6 +71,24 @@ def test_exec(config, ssh_con):
if config.test_ops.get("enable_version", False):
log.info(f"bucket versioning test on bucket: {bucket_name}")
aws_reusable.put_get_bucket_versioning(bucket_name, endpoint)
if config.test_ops.get("verify_etag_for_complete_multipart_upload", False):
log.info(
f"Verifying ETag element for complete multipart upload is not empty string"
)
for oc, size in list(config.mapped_sizes.items()):
config.obj_size = size
key_name = utils.gen_s3_object_name(bucket_name, oc)
complete_multipart_upload_resp = aws_reusable.upload_multipart_aws(
bucket_name,
key_name,
TEST_DATA_PATH,
endpoint,
config,
)
ckulal marked this conversation as resolved.
Show resolved Hide resolved
if not complete_multipart_upload_resp["ETag"]:
raise AssertionError(
"Etag not generated during complete multipart upload operation"
)

if config.user_remove is True:
s3_reusable.remove_user(user)
Expand Down Expand Up @@ -115,6 +134,8 @@ def test_exec(config, ssh_con):
configure_logging(f_name=log_f_name, set_level=args.log_level.upper())
config = resource_op.Config(yaml_file)
config.read()
if config.mapped_sizes is None:
config.mapped_sizes = utils.make_mapped_sizes(config)
test_exec(config, ssh_con)
test_info.success_status("test passed")
sys.exit(0)
Expand Down
Loading