docker-compose.yml

version: '3'
services:
  elasticsearch:
    image: elasticsearch:7.2.1
    environment:
      cluster.name: "combine-elasticsearch"
      ES_JAVA_OPTS: "-Xms1024m -Xmx1024m"
      xpack.security.enabled: "false"
      http.max_initial_line_length: 5m
      path.repo: "/tmp/es_repo"
      http.port: 9200
      node.master: "true"
      node.name: "combine_es"
      transport.tcp.port: 9300
      cluster.initial_master_nodes: "combine_es"
    volumes:
      - esdata:/usr/share/elasticsearch/data
    ports:
      - 9200:9200
    healthcheck:
        test: ["CMD-SHELL", "curl --silent --fail localhost:9200/_cluster/health || exit 1"]
        interval: 30s
        timeout: 30s
        retries: 3

  mysql:
    image: mysql:5
    environment:
      MYSQL_ROOT_PASSWORD: ${MYSQL_ROOT_PASSWORD}
    volumes:
      - mysqldata:/var/lib/mysql
    healthcheck:
        test: ["CMD-SHELL", "mysqladmin -h 'localhost' -u root -p${MYSQL_ROOT_PASSWORD} ping --silent"]
        interval: 30s
        timeout: 30s
        retries: 3
    command: '--default-authentication-plugin=mysql_native_password --interactive_timeout=31536000 --wait_timeout=31536000'

  mongo:
    image: mongo:4
    volumes:
      - mongodata:/data/db
    ports:
      - 27018:27017
    command: '--dbpath /data/db --wiredTigerCacheSizeGB 4 --bind_ip 0.0.0.0 --port 27017'

  redis:
    image: redis:5
    healthcheck:
      test: ["CMD", "redis-cli", "ping"]
      interval: 30s
      timeout: 10s
      retries: 3

  hadoop-namenode-format:
    image: tulibraries/combine-hadoop:0.0.6
    volumes:
      - hadoop_binaries:/opt/hadoop
      - hdfs:/hdfs
    command: '/bin/bash -c "mkdir -p /hdfs/namenode && yes Y | /opt/hadoop/bin/hdfs namenode -format"'

  hadoop-namenode:
    image: tulibraries/combine-hadoop:0.0.6
    volumes:
      - hadoop_binaries:/opt/hadoop
      - hdfs:/hdfs
    depends_on:
      - hadoop-namenode-format
    ports:
      - 8020:8020
    command: ['/opt/hadoop/bin/hdfs', '--config', '/opt/hadoop/etc/hadoop', 'namenode']

  hadoop-datanode:
    image: tulibraries/combine-hadoop:0.0.6
    ports:
      - 50071:50071
      - 50075:50075
    command: ['/opt/hadoop/bin/hdfs', '--config', '/opt/hadoop/etc/hadoop', 'datanode']
    depends_on:
      - hadoop-namenode

  combine-django-migration:
    image: tulibraries/combine-django:0.1.1
    env_file:
      - combine_django/.env
    environment:
      MYSQL_ADMIN_PASSWORD: ${MYSQL_ROOT_PASSWORD}
      COMBINE_SUPERADMIN_USERNAME: ${COMBINE_SUPERADMIN_USERNAME}
      COMBINE_SUPERADMIN_EMAIL: ${COMBINE_SUPERADMIN_EMAIL}
      COMBINE_SUPERADMIN_PASSWORD: ${COMBINE_SUPERADMIN_PASSWORD}
    volumes:
      - combine_home:/home/combine
      - combine_static:/opt/combine/static
      - combine_python_env:/opt/conda/envs/combine
      - combine_tmp:/tmp
      - hadoop_binaries:/opt/hadoop
      - hdfs:/hdfs
      - livy_binaries:/opt/livy
      - spark_binaries:/opt/spark
    command: '/tmp/wait-for-it/wait-for-it.sh -h mysql -p 3306 -- /bin/bash -c "/tmp/combine_db.sh"'
    depends_on:
      - mysql
      - elasticsearch

  combine-django:
    image: tulibraries/combine-django:0.1.1
    tty: true
    env_file:
      - combine_django/.env
    environment:
      MYSQL_ADMIN_PASSWORD: ${MYSQL_ROOT_PASSWORD}
      COMBINE_SUPERADMIN_USERNAME: ${COMBINE_SUPERADMIN_USERNAME}
      COMBINE_SUPERADMIN_EMAIL: ${COMBINE_SUPERADMIN_EMAIL}
      COMBINE_SUPERADMIN_PASSWORD: ${COMBINE_SUPERADMIN_PASSWORD}
    volumes:
      - combine_home:/home/combine
      - combine_static:/opt/combine/static
      - combine_python_env:/opt/conda/envs/combine
      - combine_tmp:/tmp
      - hadoop_binaries:/opt/hadoop
      - hdfs:/hdfs
      - livy_binaries:/opt/livy
      - spark_binaries:/opt/spark
    ports:
      - 8000
    command: '/tmp/wait-for-it/wait-for-it.sh -h mysql -p 3306 -- /bin/bash -c "sleep 10 && nohup pyjxslt 6767 & > /tmp/pyjxslt.out && python /opt/combine/manage.py runserver 0.0.0.0:8000"'
    depends_on:
      - combine-django-migration

  combine-celery:
    image: tulibraries/combine-django:0.1.1
    env_file:
      - combine_django/.env
    volumes:
      - combine_home:/home/combine
      - combine_static:/opt/combine/static
      - combine_python_env:/opt/conda/envs/combine
      - combine_tmp:/tmp
      - hadoop_binaries:/opt/hadoop
      - hdfs:/hdfs
      - livy_binaries:/opt/livy
      - spark_binaries:/opt/spark
    working_dir: /opt/combine
    command: '/tmp/wait-for-it/wait-for-it.sh -h redis -p 6379 -- /bin/bash -c "sleep 20 && nohup pyjxslt 6767 & > /tmp/pyjxslt.out && celery -A core worker -l info --concurrency 1"'
    depends_on:
      - redis

  combine-nginx:
    build: ./combine_nginx
    volumes:
      - combine_static:/opt/combine/static
    ports:
      - ${COMBINE_DJANGO_HOST_PORT}:80
    depends_on:
      - combine-django

  combine-livy:
    image: tulibraries/combine-livy:0.1.2
    env_file:
      - combine_django/.env
    volumes:
      - combine_home:/home/combine
      - combine_python_env:/opt/conda/envs/combine
      - combine_tmp:/tmp
      - hadoop_binaries:/opt/hadoop
      - hdfs:/hdfs
      - livy_binaries:/opt/livy
      - spark_binaries:/opt/spark
    ports:
      - 8998:8998
      - 4040:4040
    command: '/bin/bash -c "sleep 20 && nohup pyjxslt 6767 & > /tmp/pyjxslt.out && /opt/livy/bin/livy-server"'

volumes:
  # non-volatile
  combine_home:
    driver: local
  esdata:
    driver: local
  hdfs:
    driver: local
  mongodata:
    driver: local
  mysqldata:
    driver: local
  # volatile
  combine_python_env:
    driver: local
  combine_tmp:
    driver: local
  hadoop_binaries:
    driver: local
  livy_binaries:
    driver: local
  spark_binaries:
    driver: local
  combine_static:
    driver: local