docker-compose.yaml

version: "3.8"
services:
  spdk:
    image: $QUAY_SPDK:$SPDK_VERSION
    profiles:
      - build
    build:
      context: spdk/
      dockerfile: ../Dockerfile.spdk
      target: spdk
      args:
        SPDK_VERSION:
        SPDK_CEPH_VERSION:
        SPDK_PKGDEP_ARGS:
        SPDK_CONFIGURE_ARGS:
        SPDK_TARGET_ARCH:
        SPDK_MAKEFLAGS:
        SPDK_NAME:
        SPDK_SUMMARY:
        SPDK_DESCRIPTION:
        SPDK_URL:
        SPDK_MAINTAINER: $MAINTAINER
        BUILD_DATE:
        SPDK_GIT_REPO:
        SPDK_GIT_BRANCH:
        SPDK_GIT_COMMIT:
        CEPH_CLUSTER_CEPH_REPO_BASEURL:
      labels:
        io.ceph.nvmeof:
  # Used to export RPM packages externally (via bind mount)
  spdk-rpm-export:
    extends:
      service: spdk
    image: spdk-rpm-export
    build:
      target: rpm-export
  bdevperf:
    extends:
      service: spdk
    image: bdevperf
    build:
      target: bdevperf
    volumes:
      # sudo bash -c 'echo 2048 > /sys/devices/system/node/node0/hugepages/hugepages-2048kB/nr_hugepages'
      # https://spdk.io/doc/containers.html
      # TODO: Pending of https://github.com/spdk/spdk/issues/2973
      - /dev/hugepages:/dev/hugepages
      - /dev/vfio/vfio:/dev/vfio/vfio
    cap_add:
      - SYS_ADMIN # huge-pages
      - CAP_SYS_NICE # RTE
      - SYS_PTRACE # gdb
    ulimits:
      nofile: $NVMEOF_NOFILE
      memlock: -1
  ceph-base:
    build:
      context: .
      dockerfile: Dockerfile.ceph
      args:
        CEPH_CLUSTER_VERSION:
        CEPH_CLUSTER_CEPH_REPO_BASEURL:
      labels:
        io.ceph.nvmeof:
    environment:
      TOUCHFILE: /tmp/ceph.touch
    entrypoint: >-
      sh -c './vstart.sh --new $$CEPH_VSTART_ARGS &&
      ceph osd pool create rbd &&
      echo ceph dashboard nvmeof-gateway-add -i <(echo nvmeof-devel:5500) nvmeof.1 &&
      sleep infinity'
    healthcheck:
      test: ceph osd pool stats rbd
      start_period: 6s
      interval: 3s
      retries: 10
    volumes:
      - ceph-conf:/etc/ceph
    ulimits:
      nofile: 1024
    networks:
      default:
        ipv4_address: 192.168.13.2
        ipv6_address: 2001:db8::2
  ceph:
    extends:
      service: ceph-base
    image: $QUAY_CEPH:$CEPH_CLUSTER_VERSION
    container_name: ceph
  ceph-devel:
    extends:
      service: ceph-base
    image: $QUAY_CEPH-devel:$CEPH_CLUSTER_VERSION
    container_name: ceph-devel
    volumes:
      - ${CEPH_DEVEL_MGR_PATH?Define the path of your local Ceph repo}/src/pybind/mgr:/usr/share/ceph/mgr
  nvmeof-base:
    build:
      context: .
      args:
        NVMEOF_TARGET: gateway
        NVMEOF_SPDK_VERSION:
        NVMEOF_NAME:
        NVMEOF_SUMMARY:
        NVMEOF_DESCRIPTION:
        NVMEOF_URL:
        NVMEOF_VERSION:
        NVMEOF_MAINTAINER: $MAINTAINER
        NVMEOF_TAGS:
        NVMEOF_WANTS:
        NVMEOF_EXPOSE_SERVICES:
        BUILD_DATE:
        NVMEOF_GIT_REPO:
        NVMEOF_GIT_BRANCH:
        NVMEOF_GIT_COMMIT:
        NVMEOF_GIT_MODIFIED_FILES:
        NVMEOF_CEPH_VERSION:
        SPDK_GIT_REPO:
        SPDK_GIT_BRANCH:
        SPDK_GIT_COMMIT:
        HUGEPAGES:
        HUGEPAGES_DIR:
      labels:
        io.ceph.nvmeof:
    volumes:
      # sudo bash -c 'echo 2048 > /sys/devices/system/node/node0/hugepages/hugepages-2048kB/nr_hugepages'
      # https://spdk.io/doc/containers.html
      # TODO: Pending of https://github.com/spdk/spdk/issues/2973
      - /dev/hugepages:/dev/hugepages
      - /dev/vfio/vfio:/dev/vfio/vfio
      - $NVMEOF_CONFIG:/src/ceph-nvmeof.conf
      - /tmp/coredump:/tmp/coredump # core dump
      - /var/log/ceph:/var/log/ceph
      - ceph-conf:/etc/ceph:ro
    cap_add:
      - SYS_ADMIN # huge-pages
      - CAP_SYS_NICE # RTE
      - SYS_PTRACE # gdb
    ulimits:
      nofile: $NVMEOF_NOFILE
      memlock: -1
      core:
        soft: -1
        hard: -1
    ports:
      - "$NVMEOF_IO_PORT" # I/O controllers
      - "$NVMEOF_GW_PORT" # Gateway
      - "$NVMEOF_DISC_PORT" # Discovery
  nvmeof:
    extends:
      service: nvmeof-base
    image: $QUAY_NVMEOF:$NVMEOF_VERSION
    ports:
      - "$NVMEOF_IO_PORT" # I/O controllers
      - "$NVMEOF_GW_PORT" # Gateway
      - "$NVMEOF_PROMETHEUS_PORT" # Prometheus Exporter
    depends_on:
      ceph:
        condition: service_healthy
  discovery:
    extends:
      service: nvmeof-base
    image: $QUAY_NVMEOF:$NVMEOF_VERSION
    ports:
      - "$NVMEOF_DISC_PORT" # Discovery
    depends_on:
      ceph:
        condition: service_healthy
    entrypoint: >-
      python3 -m control.discovery
  # Used to update lockfile (make update-lockfile)
  nvmeof-builder-base:
    extends:
      service: nvmeof-base
    image: nvmeof-builder-base
    build:
      target: builder-base
      args:
        NVMEOF_TARGET: cli
    volumes:
      - .:/src
  # Used to regenerate gRPC protobuf files
  nvmeof-builder:
    extends:
      service: nvmeof-base
    image: nvmeof-builder
    build:
      target: builder
      args:
        NVMEOF_TARGET: cli
    volumes:
      - ./control:/src/control
  # Used to export Python package
  nvmeof-python-export:
    extends:
      service: nvmeof-base
    image: nvmeof-python-export
    build:
      target: builder
      args:
        NVMEOF_TARGET: cli
  nvmeof-devel:
    image: nvmeof-devel:$NVMEOF_VERSION
    # Runs from source code in current dir
    extends:
      service: nvmeof-base
    build:
      target: builder
      args:
        # https://daobook.github.io/pdm/usage/dependency/#add-development-only-dependencies
        PDM_INSTALL_DEV: "-d"
    depends_on:
      ceph-devel:
        condition: service_healthy
    volumes:
      - ./control:/src/control
  nvmeof-cli:
    image: $QUAY_NVMEOFCLI:$NVMEOF_VERSION
    build:
      context: .
      args:
        NVMEOF_TARGET: cli
        NVMEOF_NAME: $NVMEOF_CLI_NAME
        NVMEOF_SUMMARY: $NVMEOF_CLI_SUMMARY
        NVMEOF_DESCRIPTION: $NVMEOF_CLI_DESCRIPTION
        NVMEOF_URL:
        NVMEOF_VERSION:
        NVMEOF_MAINTAINER: $MAINTAINER
        NVMEOF_TAGS: ""
        NVMEOF_WANTS: ""
        NVMEOF_EXPOSE_SERVICES: ""
        BUILD_DATE:
        NVMEOF_GIT_REPO:
        NVMEOF_GIT_BRANCH:
        NVMEOF_GIT_COMMIT:
        SPDK_GIT_REPO:
        SPDK_GIT_BRANCH:
        SPDK_GIT_COMMIT:
        HUGEPAGES:
        HUGEPAGES_DIR:
      labels:
        io.ceph.nvmeof:
volumes:
  ceph-conf:
networks:
  default:
    enable_ipv6: true
    ipam:
      config:
        - subnet: 192.168.13.0/24
        - subnet: 2001:0DB8::/112