Skip to content

Commit

Permalink
Merge pull request #133 from aws/workload_replicator_fixes
Browse files Browse the repository at this point in the history
uploading the replay worker logs to s3
  • Loading branch information
jiezhen-chen authored Nov 27, 2023
2 parents 4f17714 + 272bd06 commit 9673e86
Showing 1 changed file with 14 additions and 12 deletions.
26 changes: 14 additions & 12 deletions core/replay/replay.py
Original file line number Diff line number Diff line change
Expand Up @@ -188,20 +188,22 @@ def main():
# uploading replay logs to s3

bucket = bucket_dict(config["workload_location"])
object_key = "replay_logs.zip"
zip_file_name = f"replay_logs.zip"
if bucket.get("bucket_name", ""):
logger.info(f"Uploading replay logs to {bucket['bucket_name']}/{bucket['prefix']}")
dir = f"core/logs/replay/replay_log-{replay_id}"
with zipfile.ZipFile(zip_file_name, "w", zipfile.ZIP_DEFLATED) as zip_object:
for folder_name, sub_folders, file_names in os.walk(dir):
for filename in file_names:
file_path = os.path.join(folder_name, filename)
zip_object.write(file_path)
with open(zip_file_name, "rb") as f:
aws_service_helper.s3_put_object(
f, bucket["bucket_name"], f"{bucket['prefix']}{object_key}"
)
for log_type, directory in [("replay_logs","core/logs/replay/replay_log"), ("replay_worker_logs","core/logs/replay_log")]:
object_key = f"{log_type}.zip"
zip_file_name = f"{log_type}.zip"
dir = f"{directory}-{replay_id}"
with zipfile.ZipFile(zip_file_name, "w", zipfile.ZIP_DEFLATED) as zip_object:
for folder_name, sub_folders, file_names in os.walk(dir):
for filename in file_names:
file_path = os.path.join(folder_name, filename)
zip_object.write(file_path)
with open(zip_file_name, "rb") as f:
aws_service_helper.s3_put_object(
f, bucket["bucket_name"], f"{bucket['prefix']}{object_key}"
)

else:
logger.info("Invalid bucket name")

Expand Down

0 comments on commit 9673e86

Please sign in to comment.