docker-compose.yml

version: "3.6"

services:

  postgres:
    image: "postgres:13"
    container_name: "postgres"
    environment:
      - POSTGRES_USER=airflow
      - POSTGRES_PASSWORD=airflow
      - POSTGRES_DB=airflow
    expose: 
      - 5432   
    volumes:
        - ../../untracked/pipeline_logs/postgresql:/var/lib/postgresql/data:rw
    user: "${UID}:${GID}"  # This requires the enviroment variables (at host): export UID=$(id -u) and export GID=$(id -g)

  initdb:
    build: .
    entrypoint: airflow db init
    depends_on:
        - postgres
    volumes:
      - ../../untracked/pipeline_logs/airflow_logs:/home/datawhale/airflow/logs:rw
    env_file:
      - airflow.env

  create_user:
    build: .
    entrypoint: airflow users create -r Admin -u airflow -f datawhale -l datawhale -p airflow -e datawhale@rawthil.com.ar
    depends_on:
        - postgres
        - initdb
    env_file:
      - airflow.env


  webserver:
    build: .
    restart: always
    entrypoint: airflow webserver
    healthcheck:
        test: ["CMD-SHELL", "[ -f /home/datawhale/airflow/airflow-webserver.pid ]"]
        interval: 30s
        timeout: 30s
        retries: 3
    ports:
        - "8080:8080"
    depends_on:
        - postgres
    volumes:
        - ../../untracked/pipeline_logs/airflow_logs:/home/datawhale/airflow/logs:rw
    env_file:
      - airflow.env
        

  scheduler:
    build: .
    restart: always
    entrypoint: airflow scheduler
    healthcheck:
        test: ["CMD-SHELL", "[ -f /home/datawhale/airflow/airflow-scheduler.pid ]"]
        interval: 30s
        timeout: 30s
        retries: 3
    depends_on:
        - postgres
    volumes:
        - ../../untracked/pipeline_logs/airflow_logs:/home/datawhale/airflow/logs:rw
    env_file:
      - airflow.env

  redis:
    image: redis:latest
    expose:
      - 6379
    healthcheck:
      test: ["CMD", "redis-cli", "ping"]
      interval: 5s
      timeout: 30s
      retries: 50
    restart: always
    volumes:
      - ../../untracked/pipeline_logs/airflow_logs:/home/datawhale/airflow/logs:rw

  worker:
    build: .
    command: airflow celery worker
    restart: always
    env_file:
      - airflow.env
    depends_on:
      - redis
    volumes:
      - ../../untracked/pipeline_logs/airflow_logs:/home/datawhale/airflow/logs:rw
      - ../../untracked/Datasets:/home/datawhale/datasets:rw
      - ../../untracked/pipeline_data:/home/datawhale/pipeline_data:rw
      - ../../untracked/tf_serving_models:/home/datawhale/output:rw
    deploy:
      resources:
        reservations:
          devices:
            - driver: nvidia
              device_ids: ['0']
              capabilities: [gpu, compute, utility]