docker-compose.yml

# Copyright (c) 2020 Battelle Energy Alliance, LLC.  All rights reserved.

version: '3.7'

################################################################################
# Commonly tweaked configuration options
#-------------------------------------------------------------------------------
x-process-variables: &process-variables
  # docker containers will run processes as unprivileged user with UID:GID
  PUID : 1000
  PGID : 1000

x-auth-variables: &auth-variables
  # authentication method: encrypted HTTP basic authentication ('true') vs LDAP ('false')
  NGINX_BASIC_AUTH : 'true'
  # NGINX LDAP (NGINX_BASIC_AUTH=false) can support LDAP, LDAPS, or LDAP+StartTLS.
  #   For StartTLS, set NGINX_LDAP_TLS_STUNNEL=true to issue the StartTLS command
  #   and use stunnel to tunnel the connection.
  NGINX_LDAP_TLS_STUNNEL : 'false'
  # When NGINX_LDAP_TLS_STUNNEL=true, which protocol to use when initiating the
  # "extendedReq(1) LDAP_START_TLS_OID" command: winldap or openldap
  NGINX_LDAP_TLS_STUNNEL_PROTOCOL : 'winldap'
  # stunnel will require and verify certificates for StartTLS when one or more
  # trusted CA certificate files are placed in the ./nginx/ca-trust directory.
  # For additional security, hostname or IP address checking of the associated
  # CA certificate(s) can be enabled by providing these values.
  NGINX_LDAP_TLS_STUNNEL_CHECK_HOST : ''
  NGINX_LDAP_TLS_STUNNEL_CHECK_IP : ''
  NGINX_LDAP_TLS_STUNNEL_VERIFY_LEVEL : 2

x-nginx-variables: &nginx-variables
  # Whether or not to write nginx's access.log and error.log to Elasticsearch
  NGINX_LOG_ACCESS_AND_ERRORS : 'false'

x-moloch-variables: &moloch-variables
  MANAGE_PCAP_FILES : 'false'
  MOLOCH_ANALYZE_PCAP_THREADS : 1
  MAXMIND_GEOIP_DB_LICENSE_KEY : '0'

x-zeek-variables: &zeek-variables
  ZEEK_AUTO_ANALYZE_PCAP_FILES : 'true'
  ZEEK_AUTO_ANALYZE_PCAP_THREADS : 1
  ZEEK_EXTRACTOR_MODE : 'none'
  EXTRACTED_FILE_IGNORE_EXISTING : 'false'
  EXTRACTED_FILE_PRESERVATION : 'quarantined'
  EXTRACTED_FILE_MIN_BYTES : 64
  EXTRACTED_FILE_MAX_BYTES : 134217728
  VTOT_API2_KEY : '0'
  VTOT_REQUESTS_PER_MINUTE : 4
  EXTRACTED_FILE_ENABLE_CLAMAV : 'false'
  EXTRACTED_FILE_ENABLE_FRESHCLAM : 'false'
  EXTRACTED_FILE_PIPELINE_DEBUG : 'false'
  EXTRACTED_FILE_PIPELINE_DEBUG_EXTRA : 'false'
  # environment variables for tweaking Zeek at runtime (see local.zeek)
  # set to a non-blank value to disable the corresponding feature
  ZEEK_DISABLE_MITRE_BZAR : ''
  ZEEK_DISABLE_HASH_ALL_FILES : ''
  ZEEK_DISABLE_LOG_PASSWORDS : ''
  ZEEK_DISABLE_MODBUS_TRACKING : ''
  ZEEK_DISABLE_MQTT : ''
  ZEEK_DISABLE_PE_XOR : ''
  ZEEK_DISABLE_QUIC : ''
  ZEEK_DISABLE_SSL_VALIDATE_CERTS : ''
  ZEEK_DISABLE_TELNET : ''
  ZEEK_DISABLE_TRACK_ALL_ASSETS : ''
  ZEEK_DISABLE_WIREGUARD : ''
  ZEEK_DISABLE_WIREGUARD_TRANSPORT_PACKETS : 'true'

x-logstash-variables: &logstash-variables
  LOGSTASH_OUI_LOOKUP  : 'true'
  LOGSTASH_REVERSE_DNS : 'false'
  # ES_EXTERNAL_HOSTS : '10.0.0.123:9200'
  # ES_EXTERNAL_SSL : 'true'
  # ES_EXTERNAL_SSL_CERTIFICATE_VERIFICATION : 'false'
  # For security, ES_EXTERNAL_USER and ES_EXTERNAL_PASSWORD should be stored in Logstash keystore using ./scripts/auth_setup
  #   See also volume mount for logstash.keystore below.
  # ES_EXTERNAL_USER : 'janedoe'
  # ES_EXTERNAL_PASSWORD : 'secret'

x-kibana-variables: &kibana-variables
  KIBANA_OFFLINE_REGION_MAPS : 'true'

x-curator-variables: &curator-variables
  # get index dates from actual date index was created (creation_date) vs. extracted from index name (name)
  CURATOR_TIME_SOURCE : 'name'
  # for managing automatic closing of older Elasticsearch indices (to conserve memory)
  CURATOR_CLOSE_UNITS : years
  CURATOR_CLOSE_COUNT : 10
  # for managing automatic deleting of older Elasticsearch indices (to reduce disk consumption)
  CURATOR_DELETE_UNITS : years
  CURATOR_DELETE_COUNT : 99
  # for managing automatic deleting of Elasticsearch indices based on cumulative size (to reduce disk consumption)
  CURATOR_DELETE_GIGS : 1000000
  # for managing automatic snapshots of Elasticsearch indices
  CURATOR_SNAPSHOT_DISABLED : 'True'
  # name of Elasticsearch snapshot repository under path.repo
  CURATOR_SNAPSHOT_REPO : 'logs'
  # controls compression of Elasticsearch snapshots managed by the curator
  CURATOR_SNAPSHOT_COMPRESSED : 'false'

x-common-upload-variables: &common-upload-variables
  AUTO_TAG : 'true'
  PCAP_PIPELINE_DEBUG : 'false'
  PCAP_PIPELINE_DEBUG_EXTRA : 'false'
  PCAP_PIPELINE_IGNORE_PREEXISTING : 'false'
  PCAP_MONITOR_HOST : 'pcap-monitor'

x-common-lookup-variables: &common-lookup-variables
  FREQ_LOOKUP : 'false'

x-common-beats-variables: &common-beats-variables
  BEATS_SSL : 'false'

x-pcap-capture-variables: &pcap-capture-variables
  PCAP_ENABLE_NETSNIFF : 'false'
  PCAP_ENABLE_TCPDUMP : 'false'
  PCAP_IFACE : 'eth0'
  PCAP_ROTATE_MEGABYTES : 1024
  PCAP_ROTATE_MINUTES : 10
  PCAP_FILTER : ''
################################################################################

services:
  elasticsearch:
    build:
      context: .
      dockerfile: Dockerfiles/elasticsearch.Dockerfile
    image: malcolmnetsec/elasticsearch-oss:2.2.0
    restart: "no"
    stdin_open: false
    tty: true
    hostname: elasticsearch
    environment:
      << : *process-variables
      logger.level : 'INFO'
      bootstrap.memory_lock : 'true'
      ES_JAVA_OPTS : '-Xms4g -Xmx4g -Xss256k -Djava.security.egd=file:/dev/./urandom'
      VIRTUAL_HOST : 'es.malcolm.local'
      discovery.type : 'single-node'
      # cluster.initial_master_nodes : 'elasticsearch'
      cluster.routing.allocation.disk.threshold_enabled : 'false'
      cluster.routing.allocation.node_initial_primaries_recoveries : 8
      indices.query.bool.max_clause_count : 2048
      path.repo : '/opt/elasticsearch/backup'
    expose:
      - 9200
    ulimits:
      memlock:
        soft: -1
        hard: -1
    cap_add:
      - IPC_LOCK
    volumes:
      - ./elasticsearch:/usr/share/elasticsearch/data:delegated
      - ./elasticsearch-backup:/opt/elasticsearch/backup:delegated
    healthcheck:
      test: ["CMD", "curl", "--silent", "--fail", "http://localhost:9200"]
      interval: 30s
      timeout: 15s
      retries: 3
      start_period: 180s
  kibana:
    build:
      context: .
      dockerfile: Dockerfiles/kibana.Dockerfile
    image: malcolmnetsec/kibana-oss:2.2.0
    restart: "no"
    stdin_open: false
    tty: true
    hostname: kibana
    environment:
      << : *process-variables
      << : *kibana-variables
      ELASTICSEARCH_URL : 'http://elasticsearch:9200'
      VIRTUAL_HOST : 'kibana.malcolm.local'
      CREATE_ES_MOLOCH_SESSION_INDEX : 'true'
      MOLOCH_INDEX_PATTERN : 'sessions2-*'
      MOLOCH_INDEX_PATTERN_ID : 'sessions2-*'
      MOLOCH_INDEX_TIME_FIELD : 'firstPacket'
    depends_on:
      - elasticsearch
    expose:
      - 5601
      - 28991
    healthcheck:
      test: ["CMD", "curl", "--silent", "--fail", "http://localhost:5601/kibana/api/status"]
      interval: 30s
      timeout: 15s
      retries: 3
      start_period: 210s
  elastalert:
    build:
      context: .
      dockerfile: Dockerfiles/elastalert.Dockerfile
    image: malcolmnetsec/elastalert:2.2.0
    restart: "no"
    stdin_open: false
    tty: true
    hostname: elastalert
    environment:
      << : *process-variables
      ELASTICSEARCH_URL : 'http://elasticsearch:9200'
      ES_HOST : 'elasticsearch'
      ES_PORT : 9200
      VIRTUAL_HOST : 'elastalert.malcolm.local'
    depends_on:
      - elasticsearch
    expose:
      - 3030
      - 3333
    volumes:
      - ./elastalert/config/elastalert.yaml:/opt/elastalert/config.yaml
      - ./elastalert/config/smtp-auth.yaml:/opt/elastalert/smtp-auth.yaml:ro
      - ./elastalert/config/config.json:/opt/elastalert-server/config/config.json
      - ./elastalert/rules/:/opt/elastalert/rules/
    healthcheck:
      test: ["CMD", "curl", "--silent", "--fail", "http://localhost:3030"]
      interval: 30s
      timeout: 15s
      retries: 3
      start_period: 210s
  curator:
    build:
      context: .
      dockerfile: Dockerfiles/curator.Dockerfile
    image: malcolmnetsec/curator:2.2.0
    restart: "no"
    stdin_open: false
    tty: true
    hostname: curator
    environment:
      << : *process-variables
      << : *curator-variables
      ES_HOST : 'elasticsearch'
      ES_PORT : 9200
    depends_on:
      - elasticsearch
    volumes:
      - ./curator/config/action_file.yml:/config/action_file.yml
    healthcheck:
      test: ["CMD", "pidof", "supercronic"]
      interval: 30s
      timeout: 5s
      retries: 3
      start_period: 30s
  logstash:
    build:
      context: .
      dockerfile: Dockerfiles/logstash.Dockerfile
    image: malcolmnetsec/logstash-oss:2.2.0
    restart: "no"
    stdin_open: false
    tty: true
    hostname: logstash
    environment:
      << : *process-variables
      << : *logstash-variables
      << : *common-beats-variables
      << : *common-lookup-variables
      ES_HOSTS : 'elasticsearch:9200'
      LS_JAVA_OPTS : '-Xms2g -Xmx2g -Xss2m -Djava.security.egd=file:/dev/./urandom'
    depends_on:
      - elasticsearch
    expose:
      - 5044
      - 9001
      - 9600
    volumes:
      - ./logstash/config/logstash.yml:/usr/share/logstash/config/logstash.yml:ro
      - ./logstash/pipelines:/usr/share/logstash/malcolm-pipelines.available:ro
      # - ./logstash/certs/logstash.keystore:/usr/share/logstash/config/logstash.keystore:ro
      - ./logstash/certs/ca.crt:/certs/ca.crt:ro
      - ./logstash/certs/server.crt:/certs/server.crt:ro
      - ./logstash/certs/server.key:/certs/server.key:ro
      - ./cidr-map.txt:/usr/share/logstash/config/cidr-map.txt:ro
      - ./host-map.txt:/usr/share/logstash/config/host-map.txt:ro
      - ./net-map.json:/usr/share/logstash/config/net-map.json:ro
    healthcheck:
      test: ["CMD", "curl", "--silent", "--fail", "http://localhost:9600"]
      interval: 30s
      timeout: 15s
      retries: 3
      start_period: 600s
  filebeat:
    build:
      context: .
      dockerfile: Dockerfiles/filebeat.Dockerfile
    image: malcolmnetsec/filebeat-oss:2.2.0
    restart: "no"
    stdin_open: false
    tty: true
    hostname: filebeat
    environment:
      << : *process-variables
      << : *nginx-variables
      << : *common-upload-variables
      << : *common-beats-variables
      FILEBEAT_LOG_PATH : '/data/zeek/current'
      FILEBEAT_NGINX_LOG_PATH : '/data/nginx'
      FILEBEAT_LOG_CLEANUP_MINUTES : 180
      FILEBEAT_ZIP_CLEANUP_MINUTES : 360
      FILEBEAT_SCAN_FREQUENCY : '10s'
      FILEBEAT_CLEAN_INACTIVE : '45m'
      FILEBEAT_IGNORE_OLDER : '30m'
      FILEBEAT_CLOSE_INACTIVE : '30s'
      FILEBEAT_CLOSE_RENAMED : 'true'
      FILEBEAT_CLOSE_REMOVED : 'true'
      FILEBEAT_CLOSE_EOF : 'true'
      FILEBEAT_CLEAN_REMOVED : 'true'
    depends_on:
      - logstash
    volumes:
      - nginx-log-path:/data/nginx:ro
      - ./zeek-logs:/data/zeek
      - ./filebeat/certs/ca.crt:/certs/ca.crt:ro
      - ./filebeat/certs/client.crt:/certs/client.crt:ro
      - ./filebeat/certs/client.key:/certs/client.key:ro
      - ./filebeat/filebeat.yml:/usr/share/filebeat/filebeat.yml:ro
    healthcheck:
      test: ["CMD", "supervisorctl", "status", "filebeat"]
      interval: 30s
      timeout: 15s
      retries: 3
      start_period: 60s
  moloch:
    build:
      context: .
      dockerfile: Dockerfiles/moloch.Dockerfile
    image: malcolmnetsec/moloch:2.2.0
    restart: "no"
    stdin_open: false
    tty: true
    hostname: moloch
    env_file:
      - ./auth.env
    environment:
      << : *process-variables
      << : *common-upload-variables
      << : *moloch-variables
      MOLOCH_VERSION : '2.3.2'
      VIRTUAL_HOST : 'moloch.malcolm.local'
      ES_HOST : 'elasticsearch'
      ES_PORT : 9200
      ES_MAX_SHARDS_PER_NODE : 2500
      VIEWER : 'on'
      WISE : 'on'
    ulimits:
      memlock:
        soft: -1
        hard: -1
    depends_on:
      - elasticsearch
    expose:
      - 8000
      - 8005
      - 8081
    volumes:
      - ./pcap:/data/pcap
      - ./moloch-logs:/data/moloch/logs
      - ./moloch-raw:/data/moloch/raw
      - ./moloch/etc/config.ini:/data/moloch/etc/config.ini:ro
      - ./moloch/etc/user_settings.json:/data/moloch/etc/user_settings.json:ro
      - ./moloch/wise/source.zeeklogs.js:/data/moloch/wiseService/source.zeeklogs.js:ro
    healthcheck:
      test: ["CMD", "curl", "--silent", "--fail", "http://localhost:8005/_ns_/nstest.html"]
      interval: 90s
      timeout: 30s
      retries: 3
      start_period: 210s
  zeek:
    build:
      context: .
      dockerfile: Dockerfiles/zeek.Dockerfile
    image: malcolmnetsec/zeek:2.2.0
    restart: "no"
    stdin_open: false
    tty: true
    hostname: zeek
    environment:
      << : *process-variables
      << : *common-upload-variables
      << : *zeek-variables
    ulimits:
      memlock:
        soft: -1
        hard: -1
    depends_on:
      - elasticsearch
    volumes:
      - ./pcap:/pcap
      - ./zeek-logs/upload:/zeek/upload
      - ./zeek-logs/extract_files:/zeek/extract_files
      - ./zeek/config/local.zeek:/opt/zeek/share/zeek/site/local.zeek:ro
    healthcheck:
      test: ["CMD", "supervisorctl", "status", "pcap-zeek"]
      interval: 30s
      timeout: 15s
      retries: 3
      start_period: 60s
  file-monitor:
    build:
      context: .
      dockerfile: Dockerfiles/file-monitor.Dockerfile
    image: malcolmnetsec/file-monitor:2.2.0
    restart: "no"
    stdin_open: false
    tty: true
    hostname: filemon
    environment:
      << : *process-variables
      << : *zeek-variables
    expose:
      - 3310
    volumes:
      - ./zeek-logs/extract_files:/data/zeek/extract_files
      - ./zeek-logs/current:/data/zeek/logs
    healthcheck:
      test: ["CMD", "supervisorctl", "status", "watcher", "scanner", "logger"]
      interval: 30s
      timeout: 15s
      retries: 3
      start_period: 60s
  pcap-capture:
    build:
      context: .
      dockerfile: Dockerfiles/pcap-capture.Dockerfile
    image: malcolmnetsec/pcap-capture:2.2.0
    restart: "no"
    stdin_open: false
    tty: true
    network_mode: host
    ulimits:
      memlock:
        soft: -1
        hard: -1
    cap_add:
      - IPC_LOCK
      - NET_ADMIN
      - NET_RAW
      - SYS_ADMIN
    environment:
      << : *process-variables
      << : *pcap-capture-variables
    volumes:
      - ./pcap/upload:/pcap
    healthcheck:
      test: ["CMD", "supervisorctl", "status"]
      interval: 30s
      timeout: 15s
      retries: 3
      start_period: 60s
  pcap-monitor:
    build:
      context: .
      dockerfile: Dockerfiles/pcap-monitor.Dockerfile
    image: malcolmnetsec/pcap-monitor:2.2.0
    restart: "no"
    stdin_open: false
    tty: true
    hostname: pcapmon
    environment:
      << : *process-variables
      << : *common-upload-variables
      ELASTICSEARCH_URL : 'http://elasticsearch:9200'
    depends_on:
      - elasticsearch
    expose:
      - 30441
    volumes:
      - ./zeek-logs:/zeek
      - ./pcap:/pcap
    healthcheck:
      test: ["CMD", "supervisorctl", "status", "watch-upload", "pcap-publisher"]
      interval: 30s
      timeout: 15s
      retries: 3
      start_period: 90s
  upload:
    build:
      context: .
      dockerfile: Dockerfiles/file-upload.Dockerfile
    image: malcolmnetsec/file-upload:2.2.0
    restart: "no"
    stdin_open: false
    tty: true
    hostname: upload
    env_file:
      - ./auth.env
    environment:
      << : *process-variables
      SITE_NAME : 'Capture File and Log Archive Upload'
      VIRTUAL_HOST : 'upload.malcolm.local'
    depends_on:
      - moloch
    expose:
      - 80
    ports:
      - 127.0.0.1:8022:22
    volumes:
      - ./pcap/upload:/var/www/upload/server/php/chroot/files
    healthcheck:
      test: ["CMD", "wget", "-qO-", "http://localhost"]
      interval: 30s
      timeout: 15s
      retries: 3
      start_period: 60s
  htadmin:
    image: malcolmnetsec/htadmin:2.2.0
    build:
      context: .
      dockerfile: Dockerfiles/htadmin.Dockerfile
    restart: "no"
    stdin_open: false
    tty: true
    hostname: htadmin
    environment:
      << : *process-variables
      << : *auth-variables
      VIRTUAL_HOST : 'htadmin.malcolm.local'
    expose:
      - 80
    volumes:
      - ./htadmin/config.ini:/var/www/htadmin/config/config.ini:rw
      - ./htadmin/metadata:/var/www/htadmin/config/metadata:rw
      - ./nginx/htpasswd:/var/www/htadmin/config/htpasswd:rw
    healthcheck:
      test: ["CMD", "curl", "--silent", "--fail", "http://localhost"]
      interval: 30s
      timeout: 15s
      retries: 3
      start_period: 60s
  freq:
    image: malcolmnetsec/freq:2.2.0
    build:
      context: .
      dockerfile: Dockerfiles/freq.Dockerfile
    restart: "no"
    stdin_open: false
    tty: true
    hostname: freq
    environment:
      << : *process-variables
      << : *common-lookup-variables
      VIRTUAL_HOST : 'freq.malcolm.local'
    expose:
      - 10004
    healthcheck:
      test: ["CMD", "curl", "--silent", "--fail", "http://localhost:10004"]
      interval: 30s
      timeout: 15s
      retries: 3
      start_period: 60s
  name-map-ui:
    image: malcolmnetsec/name-map-ui:2.2.0
    build:
      context: .
      dockerfile: Dockerfiles/name-map-ui.Dockerfile
    restart: "no"
    stdin_open: false
    tty: true
    hostname: name-map-ui
    environment:
      << : *process-variables
      VIRTUAL_HOST : 'name-map-ui.malcolm.local'
    expose:
      - 8080
    volumes:
      - ./cidr-map.txt:/var/www/html/maps/cidr-map.txt:ro
      - ./host-map.txt:/var/www/html/maps/host-map.txt:ro
      - ./net-map.json:/var/www/html/maps/net-map.json:rw
    healthcheck:
      test: ["CMD", "curl", "--silent", "--fail", "http://localhost:8080/fpm-ping"]
      interval: 30s
      timeout: 15s
      retries: 3
      start_period: 60s
  nginx-proxy:
    build:
      context: .
      dockerfile: Dockerfiles/nginx.Dockerfile
    image: malcolmnetsec/nginx-proxy:2.2.0
    restart: "no"
    stdin_open: false
    tty: true
    hostname: nginx-proxy
    environment:
      << : *process-variables
      << : *auth-variables
      << : *nginx-variables
    depends_on:
      - moloch
      - kibana
      - upload
      - htadmin
      - name-map-ui
    ports:
      - "443:443"
      - "488:488"
      - "3030:3030"
      - "5601:5601"
      - "8443:8443"
      - "9200:9200"
      - "9600:9600"
    volumes:
      - nginx-log-path:/var/log/nginx:rw
      - ./nginx/nginx.conf:/etc/nginx/nginx.conf:ro
      - ./nginx/nginx_ldap.conf:/etc/nginx/nginx_ldap.conf:ro
      - ./nginx/htpasswd:/etc/nginx/.htpasswd:ro
      - ./nginx/ca-trust:/etc/nginx/ca-trust:ro
      - ./nginx/certs:/etc/nginx/certs:ro
      - ./nginx/certs/dhparam.pem:/etc/nginx/dhparam/dhparam.pem:ro
    healthcheck:
      test: ["CMD", "curl", "--insecure", "--silent", "https://localhost:443"]
      interval: 30s
      timeout: 15s
      retries: 3
      start_period: 120s

# shared named volume so filebeat can access nginx access logs
volumes:
  nginx-log-path: