docker-compose.yaml

version: '3'
services:
  jlama-coordinator:
    image: jlama
    restart: always
    container_name: jlama-coordinator
    ports:
      - "8080:8080"
    environment:
      - JLAMA_JVM_ARGS_EXTRA=-Xmx1g -Djava.net.preferIPv4Stack=true
    deploy:
      resources:
        limits:
          memory: 1g
          cpus: "1"
    command:
      - cluster-coordinator
      - --threads=2
      - --worker-count=8
      - tjake/Mistral-7B-Instruct-v0.3-jlama-Q4
    volumes:
      - "./models:/models"
    healthcheck:
      test: ["CMD", "curl", "-f", "http://localhost:8080/ui/index.html"]
      interval: 5s
  jlama-worker:
    image: jlama
    restart: always
    depends_on: 
      jlama-coordinator:
        condition: service_healthy
    environment:
      - JLAMA_JVM_ARGS_EXTRA=-Xmx500M -Djava.net.preferIPv4Stack=true -Djlama.use_hostname_as_workerid=true
    deploy:
      mode: replicated
      replicas: 8
      resources:
        limits:
          cpus: "1"
          memory: 500m
    command:
      - cluster-worker
      - --threads=1
      - --coordinator=jlama-coordinator
      - tjake/Mistral-7B-Instruct-v0.3-jlama-Q4
    volumes:
      - "./models:/models"