From a6c0819abb84fbf0bfe4424e0ddcbe1a92c999d3 Mon Sep 17 00:00:00 2001 From: ITBM <22393016+itbm@users.noreply.github.com> Date: Mon, 12 Nov 2018 11:50:29 +0000 Subject: [PATCH] Delete files older than --- mysql-backup-s3/Dockerfile | 1 + mysql-backup-s3/README.md | 7 +++++++ mysql-backup-s3/backup.sh | 18 ++++++++++++++++++ postgres-backup-s3/Dockerfile | 1 + postgres-backup-s3/README.md | 6 ++++++ postgres-backup-s3/backup.sh | 18 ++++++++++++++++++ 6 files changed, 51 insertions(+) diff --git a/mysql-backup-s3/Dockerfile b/mysql-backup-s3/Dockerfile index 415abaf..d51e62c 100644 --- a/mysql-backup-s3/Dockerfile +++ b/mysql-backup-s3/Dockerfile @@ -20,6 +20,7 @@ ENV S3_PREFIX 'backup' ENV S3_FILENAME **None** ENV MULTI_FILES no ENV SCHEDULE **None** +ENV DELETE_OLDER_THAN **None** ADD run.sh run.sh ADD backup.sh backup.sh diff --git a/mysql-backup-s3/README.md b/mysql-backup-s3/README.md index bb0d3f7..bb46fa6 100644 --- a/mysql-backup-s3/README.md +++ b/mysql-backup-s3/README.md @@ -26,9 +26,16 @@ $ docker run -e S3_ACCESS_KEY_ID=key -e S3_SECRET_ACCESS_KEY=secret -e S3_BUCKET - `S3_S3V4` set to `yes` to enable AWS Signature Version 4, required for [minio](https://minio.io) servers (default: no) - `MULTI_FILES` Allow to have one file per database if set `yes` default: no) - `SCHEDULE` backup schedule time, see explainatons below +- `DELETE_OLDER_THAN` delete old backups, see explanation and warning below ### Automatic Periodic Backups You can additionally set the `SCHEDULE` environment variable like `-e SCHEDULE="@daily"` to run the backup automatically. More information about the scheduling can be found [here](http://godoc.org/github.com/robfig/cron#hdr-Predefined_schedules). + +### Delete Old Backups + +You can additionally set the `DELETE_OLDER_THAN` environment variable like `-e DELETE_OLDER_THAN="30 days ago"` to delete old backups. + +WARNING: this will delete all files in the S3_PREFIX path, not just those created by this script. diff --git a/mysql-backup-s3/backup.sh b/mysql-backup-s3/backup.sh index eb604e4..6b6baa5 100644 --- a/mysql-backup-s3/backup.sh +++ b/mysql-backup-s3/backup.sh @@ -107,4 +107,22 @@ else fi fi +if [ "${DELETE_OLDER_THAN}" != "**None**" ]; then + aws $AWS_ARGS s3 ls s3://$S3_BUCKET/$S3_PREFIX/ | grep " PRE " -v | while read -r line; + do + created=`echo $line|awk {'print $1" "$2'}` + created=`date -d "$created" +%s` + older_than=`date -d "$DELETE_OLDER_THAN" +%s` + if [ $created -lt $older_than ] + then + fileName=`echo $line|awk {'print $4'}` + if [ $fileName != "" ] + then + printf 'Deleting "%s"\n' $fileName + aws $AWS_ARGS s3 rm s3://$S3_BUCKET/$S3_PREFIX/$fileName + fi + fi + done; +fi + echo "SQL backup finished" diff --git a/postgres-backup-s3/Dockerfile b/postgres-backup-s3/Dockerfile index d5eed1c..a5b2959 100644 --- a/postgres-backup-s3/Dockerfile +++ b/postgres-backup-s3/Dockerfile @@ -18,6 +18,7 @@ ENV S3_PATH 'backup' ENV S3_ENDPOINT **None** ENV S3_S3V4 no ENV SCHEDULE **None** +ENV DELETE_OLDER_THAN **None** ADD run.sh run.sh ADD backup.sh backup.sh diff --git a/postgres-backup-s3/README.md b/postgres-backup-s3/README.md index a5339de..81fb776 100644 --- a/postgres-backup-s3/README.md +++ b/postgres-backup-s3/README.md @@ -40,3 +40,9 @@ You can additionally set the `SCHEDULE` environment variable like `-e SCHEDULE=" More information about the scheduling can be found [here](http://godoc.org/github.com/robfig/cron#hdr-Predefined_schedules). +### Delete Old Backups + +You can additionally set the `DELETE_OLDER_THAN` environment variable like `-e DELETE_OLDER_THAN="30 days ago"` to delete old backups. + +WARNING: this will delete all files in the S3_PREFIX path, not just those created by this script. + diff --git a/postgres-backup-s3/backup.sh b/postgres-backup-s3/backup.sh index 6e5a7f0..9b37253 100644 --- a/postgres-backup-s3/backup.sh +++ b/postgres-backup-s3/backup.sh @@ -65,4 +65,22 @@ echo "Uploading dump to $S3_BUCKET" cat dump.sql.gz | aws $AWS_ARGS s3 cp - s3://$S3_BUCKET/$S3_PREFIX/${POSTGRES_DATABASE}_$(date +"%Y-%m-%dT%H:%M:%SZ").sql.gz || exit 2 +if [ "${DELETE_OLDER_THAN}" != "**None**" ]; then + aws $AWS_ARGS s3 ls s3://$S3_BUCKET/$S3_PREFIX/ | grep " PRE " -v | while read -r line; + do + created=`echo $line|awk {'print $1" "$2'}` + created=`date -d "$created" +%s` + older_than=`date -d "$DELETE_OLDER_THAN" +%s` + if [ $created -lt $older_than ] + then + fileName=`echo $line|awk {'print $4'}` + if [ $fileName != "" ] + then + printf 'Deleting "%s"\n' $fileName + aws $AWS_ARGS s3 rm s3://$S3_BUCKET/$S3_PREFIX/$fileName + fi + fi + done; +fi + echo "SQL backup uploaded successfully"