docker-compose.yml

version: "3.7"
services:
  localstack:
    image: localstack/localstack:latest
    container_name: localstack_dap
    ports:
      - '4563-4599:4563-4599'
      - '8055:8080'
    environment:
      - SERVICES=s3,lambda,dynamodb
      - DEBUG=1
      - DATA_DIR=/tmp/localstack/data
    volumes:
      - './localstack/.localstack:/tmp/localstack'
      - '/var/run/docker.sock:/var/run/docker.sock'
  spark-master:
    image: nilan3/spark-master:2.4.4
    container_name: spark-master
    hostname: spark-master
    ports:
      - "7080:8080"
      - "7077:7077"
    volumes:
       - ./spark/data:/opt/local-data
    environment:
      - "SPARK_LOCAL_IP=spark-master"
  spark-worker:
    image: nilan3/spark-worker:2.4.4
    depends_on:
      - spark-master
    environment:
      - SPARK_MASTER=spark://spark-master:7077
      - SPARK_WORKER_CORES=1
      - SPARK_WORKER_MEMORY=1G
      - SPARK_DRIVER_MEMORY=512m
      - SPARK_EXECUTOR_MEMORY=512m
    volumes:
       - ./spark/data:/opt/local-data
  zeppelin:
    image: nilan3/spark-zeppelin:2.4.4-0.8.2
    ports:
      - "7000:7000"
    depends_on:
      - spark-master
    environment:
      - MASTER=spark://spark-master:7077
      - SPARK_MASTER=spark://spark-master:7077
      - SPARK_WORKER_CORES=1
      - SPARK_WORKER_MEMORY=1G
      - SPARK_DRIVER_MEMORY=512m
      - SPARK_EXECUTOR_MEMORY=512m
      # - SPARK_SUBMIT_OPTIONS="--packages com.databricks:spark-csv_2.10:1.2.0,io.delta:delta-core_2.11:0.4.0"
    volumes:
      - ./spark/data:/opt/local-data
      - ./zeppelin/notebook:/zeppelin/notebook