adding support for integration testing of SSE with other features

Signed-off-by: Hemanth Sai Maheswarla <[email protected]>

rgw/v2/tests/s3_swift/configs/test_bucket_notification_kafka_none_multipart.yaml

@@ -15,5 +15,5 @@ config:
   ack_type: none
   put_get_bucket_notification: true
   event_type: Multipart
-  upload_type: mulitpart
+  upload_type: multipart
   delete_bucket_object: false

rgw/v2/tests/s3_swift/configs/test_s3select_query_gen_csv_depth1.yaml

@@ -1,4 +1,5 @@
 # script: test_s3select.py
+# polarion-id: CEPH-83575176
 config:
   user_count: 1
   bucket_count: 1

rgw/v2/tests/s3_swift/configs/test_s3select_query_gen_csv_depth2.yaml

@@ -1,4 +1,5 @@
 # script: test_s3select.py
+# polarion-id: CEPH-83575176
 config:
   user_count: 1
   bucket_count: 1

rgw/v2/tests/s3_swift/configs/test_sse_kms_per_bucket_multipart_object_download_after_transition.yaml

@@ -0,0 +1,38 @@
+# script: test_bucket_lifecycle_object_expiration_transition.py
+# polarion id: CEPH-83586489
+config:
+  user_count: 1
+  encryption_keys: kms
+  bucket_count: 1
+  objects_count: 25
+  test_lc_transition: True
+  parallel_lc: False
+  enable_resharding: True
+  sharding_type: manual
+  shards: 211
+  pool_name: data.cold
+  storage_class: cold
+  ec_pool_transition: False
+  multiple_transitions: False
+  objects_size_range:
+    min: 10M
+    max: 20M
+  test_ops:
+    create_bucket: true
+    create_object: true
+    enable_versioning: false
+    version_count: 1
+    delete_marker: false
+    sse_s3_per_bucket: true
+    upload_type: multipart
+    delete_bucket_object: false
+    download_object_after_transition: true
+    actual_lc_days: 6
+  lifecycle_conf:
+    - ID: LC_Rule_1
+      Filter:
+        Prefix: key
+      Status: Enabled
+      Transitions:
+        - Days: 6
+          StorageClass: cold

rgw/v2/tests/s3_swift/configs/test_sse_kms_per_bucket_with_bucket_policy.yaml

@@ -0,0 +1,33 @@
+# script: test_bucket_policy_ops.py
+# bucket policy with server side encryption
+# polarion id: CEPH-83586489
+config:
+  objects_count: 25
+  encryption_keys: kms
+  objects_size_range:
+    min: 5
+    max: 15
+  test_ops:
+    upload_type: normal
+    sse_s3_per_bucket: true
+    verify_policy: True
+    policy_document:
+      {
+        "Version": "2012-10-17",
+        "Statement": [
+          {
+            "Action": ["s3:GetObject", "s3:DeleteObject", "s3:PutObject", "s3:AbortMultipartUpload"],
+            "Principal": {"AWS": "arn:aws:iam::<tenant_name>:user/<user_name>"},
+            "Resource": "arn:aws:s3:::<bucket_name>/*",
+            "Effect": "Allow",
+            "Sid": "statement1",
+          },
+          {
+            "Action": ["s3:PutBucketVersioning", "s3:GetBucketVersioning", "s3:PutBucketPolicy", "s3:GetBucketPolicy", "s3:DeleteBucketPolicy"],
+            "Principal": {"AWS": "arn:aws:iam::<tenant_name>:user/<user_name>"},
+            "Resource": "arn:aws:s3:::<bucket_name>",
+            "Effect": "Allow",
+            "Sid": "statement3",
+          }
+        ],
+      }

rgw/v2/tests/s3_swift/configs/test_sse_s3_per_bucket_with_notifications_dynamic_reshard.yaml

@@ -0,0 +1,31 @@
+# script: test_bucket_notifications.py
+# polarion id: CEPH-83586489
+config:
+ user_count: 1
+ bucket_count: 1
+ objects_count: 25
+ encryption_keys: s3
+ local_file_delete: true
+ enable_resharding: true
+ sharding_type: dynamic
+ max_objects_per_shard: 5
+ objects_size_range:
+  min: 10M
+  max: 20M
+ test_ops:
+  create_bucket: true
+  create_object: true
+  enable_version: false
+  sse_s3_per_bucket: true
+  create_topic: true
+  get_topic_info: true
+  endpoint: kafka
+  ack_type: broker
+  put_get_bucket_notification: true
+  event_type:
+   - Multipart
+   - Copy
+   - Delete
+  upload_type: multipart
+  copy_object: true
+  delete_bucket_object: true

rgw/v2/tests/s3_swift/configs/test_sse_s3_per_object_with_sts.yaml

@@ -0,0 +1,36 @@
+# polarion test case id: CEPH-83586489
+# test scripts : test_sts_using_boto.py
+config:
+     bucket_count: 2
+     objects_count: 50
+     encryption_keys: s3
+     objects_size_range:
+          min: 5
+          max: 15
+     test_ops:
+          create_bucket: true
+          create_object: true
+          sse_s3_per_object: true
+     sts:
+          policy_document:
+               "Version": "2012-10-17"
+               "Statement":
+                    [
+                         {
+                              "Effect": "Allow",
+                              "Principal":
+                                   {
+                                        "AWS":
+                                             ["arn:aws:iam:::user/<user_name>"],
+                                   },
+                              "Action": ["sts:AssumeRole"],
+                         },
+                    ]
+          role_policy:
+               "Version": "2012-10-17"
+               "Statement":
+                    {
+                         "Effect": "Allow",
+                         "Action": "s3:*",
+                         "Resource": "arn:aws:s3:::*",
+                    }

rgw/v2/tests/s3_swift/reusable.py

@@ -28,6 +28,7 @@
     KeyIoInfo,
 )
 from v2.lib.sync_status import sync_status
+from v2.tests.s3_swift.reusables import server_side_encryption_s3 as sse_s3
 from v2.utils.utils import HttpResponseParser, RGWService
 
 rgw_service = RGWService()
@@ -194,11 +195,22 @@ def upload_object(
             "args": [s3_object_name],
         }
     )
+
+    args = [s3_object_path]
+    if config.test_ops.get("sse_s3_per_object") is True:
+        if config.encryption_keys == "s3":
+            log.info("SSE S3 AES256 encryption method applied")
+            extra_args = {"ServerSideEncryption": "AES256"}
+            args.append(extra_args)
+        elif config.encryption_keys == "kms":
+            log.info("SSE KMS encryption method applied with vault backend")
+            extra_args = {"ServerSideEncryption": "aws:kms", "SSEKMSKeyId": "testKey01"}
+            args.append(extra_args)
     object_uploaded_status = s3lib.resource_op(
         {
             "obj": s3_obj,
             "resource": "upload_file",
-            "args": [s3_object_path],
+            "args": args,
             "extra_info": upload_info,
         }
     )
@@ -2376,3 +2388,14 @@ def test_bucket_stats_colocated_archive_zone(bucket_name_to_create, each_user, c
         log.info(
             "Bucket versioning is enabled in archive zone when  colocated with primary zone"
         )
+
+
+def put_get_bucket_encryption(rgw_s3_client, bucket_name, config):
+    log.info(f"Encryption type is per-bucket, enable it on bucket : {bucket_name}")
+    # Choose the encryption_method sse-s3 or sse-kms
+    encryption_method = config.encryption_keys
+    log.info(f"Encryption method is : {encryption_method}")
+    sse_s3.put_bucket_encryption(rgw_s3_client, bucket_name, encryption_method)
+    # get bucket encryption
+    log.info(f"get bucket encryption for bucket : {bucket_name}")
+    sse_s3.get_bucket_encryption(rgw_s3_client, bucket_name)

rgw/v2/tests/s3_swift/test_bucket_lifecycle_object_expiration_transition.py

@@ -17,6 +17,7 @@
  test_lc_with_custom_worktime.yaml
  test_lc_process_without_applying_rule.yaml
  test_lc_transition_with_lc_process.yaml
+ test_sse_kms_per_bucket_multipart_object_download_after_transition.yaml
 
 Operation:
 
@@ -165,6 +166,9 @@ def test_exec(config, ssh_con):
                 events = ["s3:ObjectLifecycle:Expiration:*"]
                 notification.apply(bucket_name, events)
 
+            if config.test_ops.get("sse_s3_per_bucket") is True:
+                reusable.put_get_bucket_encryption(rgw_conn2, bucket_name, config)
+
             if config.test_ops["enable_versioning"] is True:
                 reusable.enable_versioning(
                     bucket, rgw_conn, each_user, write_bucket_io_info
@@ -357,14 +361,24 @@ def test_exec(config, ssh_con):
                         prefix.insert(0, key)
                         s3_object_name = key + "." + bucket.name + "." + str(oc)
                         obj_list.append(s3_object_name)
-                        reusable.upload_object_with_tagging(
-                            s3_object_name,
-                            bucket,
-                            TEST_DATA_PATH,
-                            config,
-                            each_user,
-                            obj_tag,
-                        )
+                        if config.test_ops.get("upload_type") == "multipart":
+                            log.info("upload type: multipart")
+                            reusable.upload_mutipart_object(
+                                s3_object_name,
+                                bucket,
+                                TEST_DATA_PATH,
+                                config,
+                                each_user,
+                            )
+                        else:
+                            reusable.upload_object_with_tagging(
+                                s3_object_name,
+                                bucket,
+                                TEST_DATA_PATH,
+                                config,
+                                each_user,
+                                obj_tag,
+                            )
                 upload_end_time = time.time()
 
                 if config.enable_resharding and config.sharding_type == "dynamic":
@@ -460,6 +474,20 @@ def test_exec(config, ssh_con):
                             raise TestExecError(
                                 "Put bucket lifecycle Succeeded, expected failure due to invalid date in LC rule"
                             )
+
+                    if config.test_ops.get("download_object_after_transition", False):
+                        for s3_object_name in obj_list:
+                            s3_object_path = os.path.join(
+                                TEST_DATA_PATH, s3_object_name
+                            )
+                            reusable.download_object(
+                                s3_object_name,
+                                bucket,
+                                TEST_DATA_PATH,
+                                s3_object_path,
+                                config,
+                            )
+
                 else:
                     log.info("Inside parallel lc")
                     buckets.append(bucket)

rgw/v2/tests/s3_swift/test_bucket_notifications.py

@@ -23,6 +23,7 @@
     test_bucket_notification_with_tenant_user.yaml
     test_bucket_notification_kafka_broker_persistent_dynamic_reshard.yaml
     test_bucket_notification_kafka_broker_persistent_manual_reshard.yaml
+    test_sse_s3_per_bucket_with_notifications_dynamic_reshard.yaml
 Operation:
     create user (tenant/non-tenant)
     Create topic and get topic
@@ -238,6 +239,11 @@ def test_exec(config, ssh_con):
                 if config.test_ops.get("verify_persistence_with_kafka_stop", False):
                     notification.start_stop_kafka_server("stop")
 
+                if config.test_ops.get("sse_s3_per_bucket") is True:
+                    reusable.put_get_bucket_encryption(
+                        rgw_s3_client, bucket_name_to_create, config
+                    )
+
                 # create objects
                 if config.test_ops.get("create_object", False):
                     # uploading data

rgw/v2/tests/s3_swift/test_bucket_policy_ops.py

@@ -5,7 +5,7 @@
 where input-yaml test_bucket_policy_delete.yaml, test_bucket_policy_modify.yaml and test_bucket_policy_replace.yaml,
   test_bucket_policy_multiple_conflicting_statements.yaml, test_bucket_policy_multiple_statements.yaml,
   test_bucket_policy_condition.yaml, test_bucket_policy_condition_explicit_deny.yaml,
-  test_bucket_policy_invalid_*.yaml
+  test_bucket_policy_invalid_*.yaml, test_sse_kms_per_bucket_with_bucket_policy.yaml
 
 Operation:
 - create bucket in tenant1 for user1
@@ -56,7 +56,6 @@
 
 
 def get_svc_time(ssh_con=None):
-
     cmd = "pidof radosgw"
     if ssh_con:
         _, pid, _ = ssh_con.exec_command(cmd)
@@ -82,7 +81,6 @@ def get_svc_time(ssh_con=None):
 
 
 def test_exec(config, ssh_con):
-
     io_info_initialize = IOInfoInitialize()
     basic_io_structure = BasicIOInfoStructure()
     io_info_initialize.initialize(basic_io_structure.initial())
@@ -115,6 +113,10 @@ def test_exec(config, ssh_con):
         rgw_tenant1_user1,
         tenant1_user1_info,
     )
+
+    if config.test_ops.get("sse_s3_per_bucket") is True:
+        reusable.put_get_bucket_encryption(rgw_tenant1_user1_c, bucket_name1, config)
+
     bucket_name2 = utils.gen_bucket_name_from_userid(
         tenant1_user1_info["user_id"], rand_no=2
     )
@@ -352,7 +354,6 @@ def test_exec(config, ssh_con):
 
 
 if __name__ == "__main__":
-
     test_info = AddTestInfo("test bucket policy")
     test_info.started_info()
 

rgw/v2/tests/s3_swift/test_sts_using_boto.py

@@ -3,7 +3,7 @@
 
 Usage: test_sts_using_boto.py -c <input_yaml>
 <input_yaml>
-    test_sts_using_boto.yaml
+    test_sts_using_boto.yaml, test_sse_s3_per_object_with_sts.yaml
 
 Operation:
     s1: Create 2 Users.
@@ -202,7 +202,6 @@ def test_exec(config, ssh_con):
 
 
 if __name__ == "__main__":
-
     test_info = AddTestInfo("Starting STS test for assume-role operation")
     test_info.started_info()