add db backup cronjob

apps/anyway/.env

@@ -1,4 +1,5 @@
 PIN_DB_IMAGE=ghcr.io/hasadna/anyway/db:sha-6dfd43b
+PIN_DB_BACKUP_IMAGE=ghcr.io/hasadna/anyway/db_backup:sha-6dfd43b
 ANYWAY_IMAGE=ghcr.io/data-for-change/anyway/anyway:sha-020da4e
 AIRFLOW_IMAGE=ghcr.io/data-for-change/anyway-etl/anyway-etl-airflow:v0.0.55
 ETL_NGINX_IMAGE=ghcr.io/data-for-change/anyway-etl/anyway-etl-nginx:v0.0.55

apps/anyway/README.md

@@ -6,8 +6,6 @@ https://docs.google.com/presentation/d/1bXkcCgsXUr1FQA7hCZdb5_m7IXIiP1UixuOHuV88
 
 ![](image.png)
 
-### TODO: db-backup-cronjob
-
 ## Enable DB Redash read-only user
 
 Start a shell on DB container and run the following to start an sql session:

apps/anyway/compose.yaml

@@ -68,6 +68,8 @@ services:
   airflow-scheduler:
     image: ${AIRFLOW_IMAGE:-ghcr.io/data-for-change/anyway-etl/anyway-etl-airflow:latest}
     restart: unless-stopped
+    depends_on:
+      - airflow-db
     environment:
       ANYWAY_ETL_AIRFLOW_ROLE: "scheduler"
       ANYWAY_ETL_AIRFLOW_PIP_INSTALL_DEPS: "yes"
@@ -85,6 +87,8 @@ services:
     networks: [dfc]
 
   airflow-nginx:
+    depends_on:
+      - airflow-webserver
     image: ${ETL_NGINX_IMAGE:-ghcr.io/data-for-change/anyway-etl/anyway-etl-nginx:latest}
     restart: unless-stopped
     volumes:
@@ -98,6 +102,8 @@ services:
       # - "traefik.http.routers.airflow-nginx.tls.certresolver=dfc"
 
   airflow-webserver:
+    depends_on:
+      - airflow-scheduler
     image: ${AIRFLOW_IMAGE:-ghcr.io/data-for-change/anyway-etl/anyway-etl-airflow:latest}
     restart: unless-stopped
     environment:
@@ -130,6 +136,8 @@ services:
       # - "traefik.http.routers.anyway-reports.tls.certresolver=dfc"
 
   nginx:
+    depends_on:
+      - anyway-secondary
     hostname: anyway-nginx
     image: ${NGINX_IMAGE:-ghcr.io/data-for-change/anyway/nginx:latest}
     restart: unless-stopped
@@ -143,6 +151,20 @@ services:
       # - "traefik.http.routers.anyway-nginx.tls=true"
       # - "traefik.http.routers.anyway-nginx.tls.certresolver=dfc"
 
+  db-backup-cronjob:
+    x-cronjob:
+      schedule: "10 0 * * *"
+    networks: [dfc]
+    image: ${PIN_DB_BACKUP_IMAGE:-ghcr.io/hasadna/anyway/db_backup:latest}
+    environment:
+      DBDUMP_S3_FILE_PREFIX: "test-docker-migration"
+      DBDUMP_HOST: "anyway-db"
+      DBDUMP_USER: "postgres"
+      DBDUMP_FULL_BUCKET: "dfc-anyway-full-db-dumps"
+      DBDUMP_PARTIAL_BUCKET: "dfc-anyway-partial-db-dumps"
+    env_file:
+      - ./secrets/db-backup.env
+
 networks:
   dfc:
     external: true
@@ -214,3 +236,7 @@ x-secrets:
       DBRESTORE_AWS_SECRET_ACCESS_KEY: "vault:projects/anyway/prod/aws_db_dumps_reader_user:secret_access_key"
       DBRESTORE_SET_ANYWAY_PASSWORD: "vault:projects/anyway/prod/k8s-secret-db:DBRESTORE_SET_ANYWAY_PASSWORD"
       POSTGRES_PASSWORD: "vault:projects/anyway/prod/k8s-secret-db:POSTGRES_PASSWORD"
+    db-backup:
+      DBDUMP_AWS_ACCESS_KEY_ID: "vault:projects/anyway/prod/aws_db_dumps_writer_user:access_key_id"
+      DBDUMP_AWS_SECRET_ACCESS_KEY: "vault:projects/anyway/prod/aws_db_dumps_writer_user:secret_access_key"
+      DBDUMP_PASSWORD: "vault:projects/anyway/prod/k8s-secret-db-backup:DBDUMP_PASSWORD"

apps/redash/compose.yaml

@@ -10,6 +10,7 @@ services:
     hostname: redash-adhoc-worker
     networks: [dfc]
     image: *x-redash-image
+    restart: unless-stopped
     command: ["worker"]
     environment:
       <<: *x-redash-common-env
@@ -22,6 +23,7 @@ services:
     hostname: redash-scheduled-worker
     networks: [dfc]
     image: *x-redash-image
+    restart: unless-stopped
     command: ["worker"]
     environment:
       <<: *x-redash-common-env
@@ -34,6 +36,7 @@ services:
     hostname: redash-scheduler
     networks: [dfc]
     image: *x-redash-image
+    restart: unless-stopped
     command: ["scheduler"]
     environment:
       <<: *x-redash-common-env
@@ -46,6 +49,7 @@ services:
     hostname: redash
     networks: [dfc]
     image: *x-redash-image
+    restart: unless-stopped
     command: ["server"]
     environment:
       <<: *x-redash-common-env
@@ -64,13 +68,15 @@ services:
       # - "traefik.http.routers.anyway-reports.tls.certresolver=dfc"
     # pulled Nov 1, 2021
     image: redash/nginx:latest@sha256:4eaaa7af6476b0422058b0022661ad6129dfbf9065c506fb0904bbf0a16f2007
+    restart: unless-stopped
     volumes:
       - ./nginx-default.conf:/etc/nginx/conf.d/default.conf:ro
 
   postgres:
     hostname: redash-postgres
     networks: [dfc]
     image: postgres:9.6-alpine@sha256:8c3455a1c953c7c4fbcedca28a0c9c665cb6175934148335b3b86df5d106b9b5
+    restart: unless-stopped
     env_file:
       - ./secrets/redash.env
     volumes:
@@ -80,6 +86,7 @@ services:
     hostname: redash-redis
     networks: [dfc]
     image: redis:5.0-alpine@sha256:cb5840073dc475b45bf856eb666f5e4d19e3b1ae882e0f3a1396d399cfa23702
+    restart: unless-stopped
 
 networks:
   dfc:

apps/selenium/compose.yaml

@@ -3,6 +3,7 @@ services:
   selenium-chrome-node:
     networks: [dfc]
     image: selenium/node-chrome:4.8.3-20230403
+    restart: unless-stopped
     environment:
       SE_EVENT_BUS_HOST: "selenium-hub"
       SE_EVENT_BUS_PUBLISH_PORT: "4442"
@@ -22,6 +23,7 @@ services:
       - "traefik.http.routers.selenium-chrome-node-nginx.tls=true"
       - "traefik.http.routers.selenium-chrome-node-nginx.tls.certresolver=dfc"
     image: nginx@sha256:63b44e8ddb83d5dd8020327c1f40436e37a6fffd3ef2498a6204df23be6e7e94
+    restart: unless-stopped
     volumes:
       - /tmp/selenium-tempdata:/var/selenium/tempdata
       - ./nginx-default.conf:/etc/nginx/conf.d/default.conf:ro
@@ -30,6 +32,7 @@ services:
   selenium-hub:
     networks: [dfc]
     image: selenium/hub:4.8.3-20230403
+    restart: unless-stopped
     labels:
       - "traefik.enable=true"
       - "traefik.http.services.selenium-hub.loadbalancer.server.port=4444"

bin/apps.py

@@ -3,6 +3,8 @@
 import sys
 import json
 import subprocess
+import tempfile
+
 from ruamel import yaml
 from functools import lru_cache
 from tempfile import TemporaryDirectory
@@ -120,6 +122,16 @@ def copy_files(app_conf, app_name, stats):
         scp(f'apps/{app_name}/{src}', f'~/apps/.new.{app_name}/{dst}', stats)
 
 
+def deploy_cronjob(service_name, service, app_name):
+    print(f'Deploying cronjob {service_name}')
+    schedule = service['x-cronjob']['schedule']
+    cronline = f'{schedule} ubuntu cd ~/apps/{app_name} && docker compose run --rm {service_name}'
+    ssh(f'''
+        sudo rm -f /etc/cron.d/{app_name}-{service_name}
+        echo "{cronline}" | sudo tee /etc/cron.d/{app_name}-{service_name}
+    ''')
+
+
 def deploy_app(app_name, *args):
     skip_deploy = '--skip-deploy' in args
     assert os.path.exists(f'./apps/{app_name}/compose.yaml')
@@ -142,9 +154,15 @@ def deploy_app(app_name, *args):
         for line in app_conf.get('x-pre-deploy', []):
             print('Running x-pre-deploy:', line)
             ssh(f'cd ~/apps/{app_name} && {line}')
+        start_services = set()
+        for service_name, service in app_conf['services'].items():
+            if service.get('x-cronjob'):
+                deploy_cronjob(service_name, service, app_name)
+            else:
+                start_services.add(service_name)
         ssh(f'''
             cd ~/apps/{app_name}
-            docker compose up -d
+            docker compose up -d {' '.join(start_services)}
         ''')
         for on_file_change in app_conf.get('x-on-file-change', []):
             if not isinstance(on_file_change, dict):