From a1179901aee5f65f4060b202612ed0a137674f92 Mon Sep 17 00:00:00 2001 From: DevanathanSabapathy1 Date: Tue, 21 Nov 2023 15:20:17 -0800 Subject: [PATCH 1/3] uploading the replay worker logs to s3 --- core/replay/replay.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/core/replay/replay.py b/core/replay/replay.py index 2fca2532..b4930832 100644 --- a/core/replay/replay.py +++ b/core/replay/replay.py @@ -188,10 +188,10 @@ def main(): # uploading replay logs to s3 bucket = bucket_dict(config["workload_location"]) - object_key = "replay_logs.zip" - zip_file_name = f"replay_logs.zip" if bucket.get("bucket_name", ""): logger.info(f"Uploading replay logs to {bucket['bucket_name']}/{bucket['prefix']}") + object_key = "replay_logs.zip" + zip_file_name = f"replay_logs.zip" dir = f"core/logs/replay/replay_log-{replay_id}" with zipfile.ZipFile(zip_file_name, "w", zipfile.ZIP_DEFLATED) as zip_object: for folder_name, sub_folders, file_names in os.walk(dir): @@ -202,6 +202,21 @@ def main(): aws_service_helper.s3_put_object( f, bucket["bucket_name"], f"{bucket['prefix']}{object_key}" ) + + logger.info(f"Uploading replay worker logs to {bucket['bucket_name']}/{bucket['prefix']}") + dir = f"core/logs/replay_log-{replay_id}" + object_key = "replay_worker_logs.zip" + zip_file_name = f"replay_worker_logs.zip" + with zipfile.ZipFile(zip_file_name, "w", zipfile.ZIP_DEFLATED) as zip_object: + for folder_name, sub_folders, file_names in os.walk(dir): + for filename in file_names: + file_path = os.path.join(folder_name, filename) + zip_object.write(file_path) + with open(zip_file_name, "rb") as f: + aws_service_helper.s3_put_object( + f, bucket["bucket_name"], f"{bucket['prefix']}{object_key}" + ) + else: logger.info("Invalid bucket name") From 122e97ac9186f26149ea743b0bf7f9a9cad72140 Mon Sep 17 00:00:00 2001 From: DevanathanSabapathy1 Date: Wed, 22 Nov 2023 08:28:56 -0800 Subject: [PATCH 2/3] simplifying code to upload logs to s3 --- core/replay/replay.py | 41 ++++++++++++++--------------------------- 1 file changed, 14 insertions(+), 27 deletions(-) diff --git a/core/replay/replay.py b/core/replay/replay.py index b4930832..648c2ed6 100644 --- a/core/replay/replay.py +++ b/core/replay/replay.py @@ -189,33 +189,20 @@ def main(): bucket = bucket_dict(config["workload_location"]) if bucket.get("bucket_name", ""): - logger.info(f"Uploading replay logs to {bucket['bucket_name']}/{bucket['prefix']}") - object_key = "replay_logs.zip" - zip_file_name = f"replay_logs.zip" - dir = f"core/logs/replay/replay_log-{replay_id}" - with zipfile.ZipFile(zip_file_name, "w", zipfile.ZIP_DEFLATED) as zip_object: - for folder_name, sub_folders, file_names in os.walk(dir): - for filename in file_names: - file_path = os.path.join(folder_name, filename) - zip_object.write(file_path) - with open(zip_file_name, "rb") as f: - aws_service_helper.s3_put_object( - f, bucket["bucket_name"], f"{bucket['prefix']}{object_key}" - ) - - logger.info(f"Uploading replay worker logs to {bucket['bucket_name']}/{bucket['prefix']}") - dir = f"core/logs/replay_log-{replay_id}" - object_key = "replay_worker_logs.zip" - zip_file_name = f"replay_worker_logs.zip" - with zipfile.ZipFile(zip_file_name, "w", zipfile.ZIP_DEFLATED) as zip_object: - for folder_name, sub_folders, file_names in os.walk(dir): - for filename in file_names: - file_path = os.path.join(folder_name, filename) - zip_object.write(file_path) - with open(zip_file_name, "rb") as f: - aws_service_helper.s3_put_object( - f, bucket["bucket_name"], f"{bucket['prefix']}{object_key}" - ) + for log_type, directory in [("replay_logs","core/logs/replay/replay_log"), ("replay_worker_logs","core/logs/replay_log")]: + logger.info(f"Uploading replay logs to {bucket['bucket_name']}/{bucket['prefix']}") + object_key = f"{log_type}.zip" + zip_file_name = f"{log_type}.zip" + dir = f"{directory}-{replay_id}" + with zipfile.ZipFile(zip_file_name, "w", zipfile.ZIP_DEFLATED) as zip_object: + for folder_name, sub_folders, file_names in os.walk(dir): + for filename in file_names: + file_path = os.path.join(folder_name, filename) + zip_object.write(file_path) + with open(zip_file_name, "rb") as f: + aws_service_helper.s3_put_object( + f, bucket["bucket_name"], f"{bucket['prefix']}{object_key}" + ) else: logger.info("Invalid bucket name") From 272bd067db27bf3e4fb46cba7e5b4a28b4bce198 Mon Sep 17 00:00:00 2001 From: DevanathanSabapathy1 Date: Wed, 22 Nov 2023 08:40:01 -0800 Subject: [PATCH 3/3] removing redundant logging --- core/replay/replay.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/replay/replay.py b/core/replay/replay.py index 648c2ed6..978828a5 100644 --- a/core/replay/replay.py +++ b/core/replay/replay.py @@ -189,8 +189,8 @@ def main(): bucket = bucket_dict(config["workload_location"]) if bucket.get("bucket_name", ""): + logger.info(f"Uploading replay logs to {bucket['bucket_name']}/{bucket['prefix']}") for log_type, directory in [("replay_logs","core/logs/replay/replay_log"), ("replay_worker_logs","core/logs/replay_log")]: - logger.info(f"Uploading replay logs to {bucket['bucket_name']}/{bucket['prefix']}") object_key = f"{log_type}.zip" zip_file_name = f"{log_type}.zip" dir = f"{directory}-{replay_id}"