docker-compose.yml

version: '3'

services:

  nginx:
    image: uscisii2/nginx:auth-1.0
    ports:
      - "${PORT}:80"
    environment:
      DIG_AUTH_USER: ${DIG_AUTH_USER}
      DIG_AUTH_PASSWORD: ${DIG_AUTH_PASSWORD}
#      DIG_AUTH_ENABLE: ${DIG_AUTH_ENABLE}
    volumes:
      - ./nginx/sandbox/conf.d/dig.conf:/etc/nginx/conf.d/default.conf
      - ${DIG_PROJECTS_DIR_PATH}/.nginx/logs:/var/log/nginx
      - ${DIG_PROJECTS_DIR_PATH}/.nginx/logs/digui:/var/log/diglogs
      - ./nginx/sandbox/index.html:/usr/share/nginx/html/index.html
    networks:
      - dig_net
    depends_on:
      - elasticsearch
      - sandpaper
      - mydig_ws
      - digui

  dig_etl_engine:
    image: uscisii2/dig-etl-engine:2.0.5
    environment:
      KAFKA_NUM_PARTITIONS: ${KAFKA_NUM_PARTITIONS:-4}
    volumes:
      - ${DIG_PROJECTS_DIR_PATH}:/shared_data/projects
      - dig3-resources-volume:/shared_data/dig3-resources
      - ./logstash/sandbox/pipeline/:/app/logstash/default_pipeline
      - ${DIG_PROJECTS_DIR_PATH}/.ls/pipeline/:/app/logstash/pipeline
#    ports:
#      - "9999"
    networks:
      - dig_net
    depends_on:
      - elasticsearch

  zookeeper:
    image: wurstmeister/zookeeper
    environment:
      KAFKA_HEAP_OPTS: "-Xmx${ZK_HEAP_SIZE:-256m} -Xms${ZK_HEAP_SIZE:-256m}"
#    ports:
#      - "2181"
    networks:
      - dig_net

  kafka:
    image: wurstmeister/kafka
    ports:
      - "9092:9092"
    environment:
      KAFKA_ADVERTISED_HOST_NAME: kafka
      KAFKA_ADVERTISED_PORT: 9092
      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
      KAFKA_CREATE_TOPICS: "ache:1:1,rss:1:1,crawler:1:1"
      KAFKA_HEAP_OPTS: "-Xmx${KAFKA_HEAP_SIZE:-256m} -Xms${KAFKA_HEAP_SIZE:-256m}"
      KAFKA_LOG_CLEANER_ENABLE: "true"
      KAFKA_LOG_CLEANUP_POLICY: delete
      KAFKA_LOG_CLEANER_BACKOFF_MS: 3600 # 3600
      KAFKA_LOG_RETENTION_CHECK_INTERVAL_MS: 21600 # 3600 * 6
      KAFKA_LOG_CLEANER_DELETE_RETENTION_MS: 86400 # 3600 * 24
      KAFKA_NUM_PARTITIONS: ${KAFKA_NUM_PARTITIONS:-4}
      KAFKA_AUTO_CREATE_TOPICS_ENABLE: "true"
      KAFKA_DELETE_TOPIC_ENABLE: "true"
      KAFKA_MESSAGE_MAX_BYTES: 10485760 # 10MB
      KAFKA_REPLICA_FETCH_MAX_BYTES: 10485760 # 10MB
      KAFKA_REPLICA_FETCH_RESPONSE_MAX_BYTES: 10485760 # 10MB
      KAFKA_HEARTBEAT_INTERVAL_MS: 30000
#      KAFKA_SESSION_TIMEOUT_MS: 100000
      KAFKA_GROUP_MAX_SESSION_TIMEOUT_MS: 300000
    volumes:
      - /var/run/docker.sock:/var/run/docker.sock
      - ${DIG_PROJECTS_DIR_PATH}/.kafka/data:/kafka
    depends_on:
      - zookeeper
    networks:
      dig_net:
        # can not set ip range in docker-compose v3 to whitelist fixed ip
        # so set it to 200, DNS still works
        ipv4_address: ${DIG_NET_KAFKA_IP:-172.19.0.200}

  logstash:
    image: docker.elastic.co/logstash/logstash:5.6.4
    environment:
      PATH_CONFIG: /usr/share/logstash/pipeline
      LS_JAVA_OPTS: "-Xmx${LS_HEAP_SIZE:-256m} -Xms${LS_HEAP_SIZE:-256m}"
      LS_LOG_LEVEL: "debug"
    volumes:
      - ${DIG_PROJECTS_DIR_PATH}/.ls/pipeline/:/usr/share/logstash/pipeline
      - ./logstash/sandbox/settings/logstash.yml:/usr/share/logstash/config/logstash.yml
      - ${DIG_PROJECTS_DIR_PATH}/.ls/log/:/var/log/logstash
    networks:
      - dig_net
    depends_on:
      - kafka
      - elasticsearch
      - dig_etl_engine

  elasticsearch:
    image: docker.elastic.co/elasticsearch/elasticsearch:5.6.4
    volumes:
#      - ./elasticsearch/sandbox/config/logging.yml:/usr/share/elasticsearch/config/logging.yml
      - ./elasticsearch/sandbox/config/elasticsearch.yml:/usr/share/elasticsearch/config/elasticsearch.yml
      - ${DIG_PROJECTS_DIR_PATH}/.es/data:/usr/share/elasticsearch/data
#    ports:
#      - "9200:9200"
#      - "9300:9300"
    environment:
      ES_JAVA_OPTS: "-Xmx${ES_HEAP_SIZE:-512m} -Xms${ES_HEAP_SIZE:-512m}"
    networks:
      - dig_net

  kibana:
    image: docker.elastic.co/kibana/kibana:5.6.4
    volumes:
      - ./kibana/sandbox/config:/opt/kibana/config
#    ports:
#      - "5601:5601"
    networks:
      - dig_net
    depends_on:
      - elasticsearch

  sandpaper:
    image: uscisii2/digsandpaper:0.2.7
#    ports:
#      - "9876"
    volumes:
      - ./sandpaper/sandbox/config:/etc/sandpaper/config
#    environment:
#      ES_MAJOR_VERSION: 5
    links:
      - elasticsearch
    command:
      - --host
      - 0.0.0.0
      - --endpoint
      - http://elasticsearch:9200
      - --config
      - config/sandpaper.json
    networks:
      - dig_net
    depends_on:
      - elasticsearch

  mydig_ws:
    image: uscisii2/mydig_ws:2.0.4
    environment:
      DOMAIN: ${DOMAIN}
      PORT: ${PORT}
      NUM_ETK_PROCESSES: ${NUM_ETK_PROCESSES}
      DIG_AUTH_USER: ${DIG_AUTH_USER}
      DIG_AUTH_PASSWORD: ${DIG_AUTH_PASSWORD}
      DEFAULT_EXTERNAL_CRAWLER_PROJECT: ${DEFAULT_EXTERNAL_CRAWLER_PROJECT:-crawler}
#    ports:
#      - "9879:9879"
#      - "9880:9880"
#      - "9881:9881"
    volumes:
      - ./mydig-webservice/sandbox/config_docker.py:/app/mydig-webservice/ws/config.py
      - ${DIG_PROJECTS_DIR_PATH}:/shared_data/projects
      - dig3-resources-volume:/shared_data/dig3-resources
    networks:
      - dig_net
    depends_on:
      - sandpaper
      - elasticsearch

  digui:
    image: uscisii2/digui:3.4.6
#    ports:
#      - "8080"
    volumes:
      - ${DIG_PROJECTS_DIR_PATH}/.digui/log:/var/log
    environment:
      NODE_ENV: 'production'
      ES_HOST: '{"host": "http://${DOMAIN}:${PORT}/es", "apiVersion":"5.0"}'
      ES_HOST_STRING: 'http://elasticsearch:9200/'
      CONFIG_ENDPOINT: 'http://${DOMAIN}:${PORT}/mydig_projects/'
      TAGS_ENTITY_ENDPOINT: 'http://${DOMAIN}:${PORT}/mydig_projects/PROJECT/tags/TAG/annotations/Ad/annotations'
      TAGS_EXTRACTION_ENDPOINT: 'http://${DOMAIN}:${PORT}/mydig_projects/PROJECT/entities/ENTITY_ID/fields/EXTRACTION_FIELD/annotations'
      TAGS_LIST_ENDPOINT: 'http://${DOMAIN}:${PORT}/mydig_projects/PROJECT/tags'
      SEARCH_CONFIG: '{
        "http://sandpaper:9876": "http://${DOMAIN}:${PORT}/search/coarse"
      }'
      PATH_PREFIX: '/dig-ui/'
      SHOW_ES_DATA: 'true'
    depends_on:
      - elasticsearch
    networks:
      - dig_net

  landmark-mysql:
    image: inferlink/landmark-mysql:1.1.0
#    container_name: landmark-mysql
    env_file:
      - ./landmark/sandbox/landmark-mysql.docker.env
    volumes:
      - ${DIG_PROJECTS_DIR_PATH}/.landmark/mysql:/var/lib/mysql
#    ports:
#      - "3306"
    networks:
      - dig_net
  landmark-rest:
    image: inferlink/landmark-rest:1.1.7
#    container_name: landmark-rest
    env_file:
      - ./landmark/sandbox/landmark-rest.docker.env
    volumes:
      - ${DIG_PROJECTS_DIR_PATH}:/shared_data/projects
#    ports:
#      - "5000:5000"
    networks:
      - dig_net
  landmark-portal:
    image: inferlink/landmark-portal:1.1.2
#    container_name: landmark-portal
    env_file:
      - ./landmark/sandbox/landmark-portal.docker.env
#    ports:
#     - "3333:3333"
    networks:
      - dig_net


networks:
  dig_net:
    driver: bridge
    ipam:
      driver: default
      config:
        - subnet: ${DIG_NET_SUBNET:-172.19.0.0/16}

volumes:
  dig3-resources-volume: