docker-compose.yml

# Bring up all the pieces necessary to run the workbench
# Data persists in Docker volumes and in local dir

# This file passes through all necessary env variables to requisite
# Docker containers and makes them available when running commands via
# `docker exec`.

version: '3.4'

services:
  database:
    image: postgres:10.10
    environment:
      POSTGRES_USER: cjworkbench
      POSTGRES_PASSWORD: cjworkbench
      POSTGRES_DB: cjworkbench
      PGDATA: /var/lib/postgresql/data/10.4
    networks: [ 'dev' ]
    volumes:
      - dbdata:/var/lib/postgresql/data

  rabbitmq:
    image: rabbitmq:3.8.11-management
    ports: [ '15672' ] # open management port, for debugging
    networks: [ 'dev' ]
    environment:
      # Use just one CPU
      RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS: '+S 1:1'

  minio:
    image: minio/minio:RELEASE.2021-07-08T01-15-01Z
    networks: [ 'dev' ]
    ports: [ '8001:9000' ]
    environment:
      MINIO_ACCESS_KEY: minio_access
      MINIO_SECRET_KEY: minio_secret
    volumes:
      - minio_data:/data
    entrypoint: ''
    command:
      - sh
      - '-c'
      - |
        PREFIX=dev
        for bucket in user-files static stored-objects external-modules cached-render-results upload; do
            mkdir -p /data/$$PREFIX-$$bucket
        done
        rm -rf /data/.minio.sys
        mkdir -p /data/.minio.sys/buckets/$$PREFIX-static/
        cat > /data/.minio.sys/buckets/$$PREFIX-static/policy.json <<EOT
        {
            "Version": "2012-10-17",
            "Statement": [
                {
                    "Sid": "",
                    "Effect": "Allow",
                    "Principal": {"AWS": "*"},
                    "Action": ["s3:GetBucketLocation", "s3:ListBucket"],
                    "Resource": ["arn:aws:s3:::$$PREFIX-static"]
                },
                {
                    "Sid": "",
                    "Effect": "Allow",
                    "Principal": {"AWS": "*"},
                    "Action": ["s3:GetObject"],
                    "Resource": ["arn:aws:s3:::$$PREFIX-static/*"]
                }
            ]
        }
        EOT
        minio server /data

  webpack:
    build:
      context: .
      target: jsbase
    volumes:
      # Migrate
      - type: bind
        source: ./
        target: /app/
        consistency: cached
      - node_modules:/app/node_modules:rw
      - jest_cache:/tmp/jest_0:rw
    networks: [ 'dev' ]
    command: [ 'node_modules/.bin/webpack', '--mode', 'development', '--watch' ]

  frontend:
    build:
      context: .
      target: pydev
    volumes:
      - type: bind
        source: ./
        target: /app/
        consistency: cached
    security_opt:
      - seccomp=docker/pyspawner-seccomp-profile.json
    privileged: true
    cap_add: [ SYS_ADMIN ] # for setup-sandboxes.sh to overlay-mount chroots (on k8s we use an init container instead)
    environment: &django-env
      PYTHONUNBUFFERED: '1'
      PYTHONPYCACHEPREFIX: /tmp/pycache
      ASGI_THREADS: '3'
      CJW_HELP_EMAIL: root@localhost
      CJW_HOME_URL: 'http://localhost:8000'
      CJW_DB_HOST: database
      CJW_RABBITMQ_HOST: amqp://guest:guest@rabbitmq
      CJW_DB_PASSWORD: cjworkbench
      CJW_SECRET_KEY: cjw-secret-key
      CJW_INTERCOM_APP_ID: fs80154z
      CJW_INTERCOM_IDENTITY_VERIFICATION_SECRET: sfrVVkJJhMHyLrecJDxLrJAefmV_1COh2NJbjP3G
      TUS_CREATE_UPLOAD_URL: http://tusd/files/
      TUS_EXTERNAL_URL_PREFIX_OVERRIDE: http://localhost:8002/files/
      AWS_ACCESS_KEY_ID: minio_access
      AWS_SECRET_ACCESS_KEY: minio_secret
      AWS_S3_ENDPOINT: http://minio:9000
      S3_BUCKET_NAME_PATTERN: "dev-%s"
    ports: [ '8000:8080' ]
    networks: [ 'dev' ]
    depends_on: [ 'database', 'rabbitmq', 'minio', 'tusd' ]
    command: [
      'sh',
      '-c',
      'cjwkernel/setup-sandboxes.sh only-readonly && /opt/venv/django/bin/python bin/watch-and-restart --exclude "**/*.pyc" "**/tests/**/*" --pattern "{intercom_secret,twitter_secret,client_secret}.json" "stripe.env" "cjwkernel/**/*.py" "cjwstate/**/*.py" "cjwstate/**/*.yaml" "server/**/*.py" "templates/**/*.html" "templates/**/*.txt" "server/**/*.yaml" "server/lessons/**/*.html" "server/courses/**/*.html" "cjworkbench/**/*.py" "assets/locale/*/messages.po" --exec bin/frontend-prod'
    ]

  staticfiles:
    build:
      context: .
      target: pydev
    volumes:
      - type: bind
        source: ./
        target: /app/
        consistency: cached
    ports: [ '8003:8003' ]
    networks: [ 'dev' ]
    environment:
      DJANGO_SETTINGS_MODULE: staticfilesdev.settings
    command: [
      '/opt/venv/django/bin/python',
      '-m',
      'uvicorn',
      '--host',
      '0.0.0.0',
      '--port',
      '8003',
      '--use-colors',
      'staticfilesdev.asgi:application'
    ]

  renderer:
    # It'd be nice to use YAML anchors to copy these parameters ... but
    # PyCharm's YAML parser seems to die when we use YAML anchors.
    build:
      context: .
      target: pydev
    volumes:
      - type: bind
        source: ./
        target: /app/
        consistency: cached
    security_opt:
      - seccomp=docker/pyspawner-seccomp-profile.json
    privileged: true
    cap_add:
      - SYS_ADMIN # for setup-sandboxes.sh to overlay-mount chroots (on k8s we use an init container instead)
      - NET_ADMIN # for pyspawner to create new network namespace without access to private network
    environment:
      <<: *django-env
    depends_on: [ 'database', 'rabbitmq', 'minio' ]
    networks: [ 'dev' ]
    command: [
      'sh',
      '-c',
      # Use watch-and-restart, not Django autoreload. autoreload crashes when there's a
      # syntax error.
      'cjwkernel/setup-sandboxes.sh all && /opt/venv/django/bin/python bin/watch-and-restart --exclude "**/tests/**/*" --pattern "cjworkbench/**/*.py" "cjwkernel/**/*.py" "cjwstate/**/*.py" "renderer/**/*.py" --exec bin/renderer-prod'
    ]

  fetcher:
    # It'd be nice to use YAML anchors to copy these parameters ... but
    # PyCharm's YAML parser seems to die when we use YAML anchors.
    build:
      context: .
      target: pydev
    volumes:
      - type: bind
        source: ./
        target: /app/
        consistency: cached
    security_opt:
      - seccomp=docker/pyspawner-seccomp-profile.json
    privileged: true
    cap_add:
      - SYS_ADMIN # for setup-sandboxes.sh to overlay-mount chroots (on k8s we use an init container instead)
      - NET_ADMIN # for pyspawner to create new network namespace without access to private network
    environment:
      <<: *django-env
    depends_on: [ 'database', 'rabbitmq', 'minio' ]
    networks: [ 'dev' ]
    command: [
      'sh', '-c',
      'cjwkernel/setup-sandboxes.sh all && /opt/venv/django/bin/python bin/watch-and-restart --exclude "**/*.pyc" "**/tests/**/*" --pattern "{intercom_secret,twitter_secret,client_secret}.json" "cjworkbench/**/*.py" "cjwkernel/**/*.py" "cjwstate/**/*.py" "fetcher/**/*.py" --exec bin/fetcher-prod'
    ]

  cron: &cron
    build:
      context: .
      target: pydev
    volumes:
      - type: bind
        source: ./
        target: /app/
        consistency: cached
    environment:
      <<: *django-env
    depends_on: [ 'database', 'rabbitmq', 'minio' ]
    networks: [ 'dev' ]
    command: [
      '/opt/venv/django/bin/python',
      'bin/watch-and-restart',
      '--exclude', '**/*.pyc', '**/tests/**/*',
      '--pattern', 'cjworkbench/**/*.py', 'cron/**/*.py', 'cjwstate/models/**/*.*',
      '--exec', 'bin/cron-prod'
    ]

  cron-expired-session-deleter:
    <<: *cron
    depends_on: [ 'database', 'minio' ]
    command: [
      '/opt/venv/django/bin/python',
      'bin/watch-and-restart',
      '--exclude', '**/*.pyc', '**/tests/**/*',
      '--pattern', 'cjworkbench/**/*.py', 'cron/**/*.py', 'cjwstate/models/**/*.*',
      '--exec', 'bin/cron-expired-session-deleter'
    ]

  cron-lesson-workflow-deleter:
    <<: *cron
    depends_on: [ 'database' ]
    command: [
      '/opt/venv/django/bin/python',
      'bin/watch-and-restart',
      '--exclude', '**/*.pyc', '**/tests/**/*',
      '--pattern', 'cjworkbench/**/*.py', 'cron/**/*.py', 'cjwstate/models/**/*.*',
      '--exec', 'bin/cron-lesson-workflow-deleter'
    ]

  cron-delta-deleter:
    <<: *cron
    depends_on: [ 'database' ]
    command: [
      '/opt/venv/django/bin/python',
      'bin/watch-and-restart',
      '--exclude', '**/*.pyc', '**/tests/**/*',
      '--pattern', 'cjworkbench/**/*.py', 'cron/**/*.py', 'cjwstate/models/**/*.*',
      '--exec', 'bin/cron-delta-deleter'
    ]

  tusd-hooks:
    build:
      context: .
      target: pydev
    volumes:
      - type: bind
        source: ./
        target: /app/
        consistency: cached
    environment: *django-env
    networks: [ 'dev' ]
    depends_on: [ 'database', 'rabbitmq', 'minio' ]
    # we need all this sandboxing stuff just to invoke migrate_params(). Rrgh.
    security_opt:
      - seccomp=docker/pyspawner-seccomp-profile.json
    privileged: true
    cap_add: [ SYS_ADMIN ] # for setup-sandboxes.sh to overlay-mount chroots (on k8s we use an init container instead)
    command: [
      'sh',
      '-c',
      'cjwkernel/setup-sandboxes.sh only-readonly && /opt/venv/django/bin/python bin/watch-and-restart --exclude "**/*.pyc" "**/tests/**/*" --pattern "cjwstate/**/*.py" "cjworkbench/**/*.py" "tusdhooks/**/*.py" --exec bin/tusd-hooks-prod'
    ]

  tusd:
    image: tusproject/tusd:v1.6.0
    networks: [ 'dev' ]
    depends_on: [ 'minio', 'tusd-hooks' ]
    ports: [ '8002:80' ]
    environment:
      AWS_ACCESS_KEY_ID: minio_access
      AWS_SECRET_ACCESS_KEY: minio_secret
      AWS_REGION: us-east-1
    # No volumes: we don't save data between restarts. (Uploads are temporary.)
    command: [
      '-port=80',
      # Use frontend:8080 -- the Django app, not the proxy. The proxy tries to
      # keep connections alive and clients can sometimes notice.
      '-hooks-http=http://tusd-hooks:8080/tusd-hooks',
      '-hooks-enabled-events=pre-finish',
      '-s3-endpoint=http://minio:9000',
      '-s3-bucket=dev-upload',
    ]

  testdatabase:
    image: postgres:12
    environment:
      POSTGRES_USER: cjworkbench
      POSTGRES_PASSWORD: cjworkbench
      POSTGRES_DB: cjworkbench
      PGDATA: /var/lib/postgresql/data/12
    networks: [ 'test' ]
    volumes:
    - type: tmpfs
      target: /var/lib/postgresql/data
    command: [
      '-c', 'wal_level=minimal',
      '-c', 'max_wal_senders=0',
      '-c', 'fsync=off',
      '-c', 'synchronous_commit=off',
      '-c', 'full_page_writes=off'
    ]

  testrabbitmq:
    image: rabbitmq:3.8.11-management
    networks: [ 'test' ]
    environment:
      # Use just one CPU
      RABBITMQ_SERVER_ADDITIONAL_ERL_ARGS: '+S 1:1'

  testminio:
    image: minio/minio:RELEASE.2021-07-08T01-15-01Z
    networks: [ 'test' ]
    volumes:
      - type: tmpfs
        target: /data
    environment:
      MINIO_ACCESS_KEY: minio_access
      MINIO_SECRET_KEY: minio_secret
    entrypoint: ''
    command:
      - sh
      - '-c'
      - |
        for bucket in user-files static stored-objects external-modules cached-render-results upload; do
            mkdir -p /data/$$bucket
        done
        rm -rf /data/.minio.sys
        mkdir -p /data/.minio.sys/buckets/static/
        cat > /data/.minio.sys/buckets/static/policy.json <<EOT
        {
            "Version": "2012-10-17",
            "Statement": [
                {
                    "Sid": "",
                    "Effect": "Allow",
                    "Principal": {"AWS": "*"},
                    "Action": ["s3:GetBucketLocation", "s3:ListBucket"],
                    "Resource": ["arn:aws:s3:::static"]
                },
                {
                    "Sid": "",
                    "Effect": "Allow",
                    "Principal": {"AWS": "*"},
                    "Action": ["s3:GetObject"],
                    "Resource": ["arn:aws:s3:::static/*"]
                }
            ]
        }
        EOT
        minio server /data

  testtusd:
    image: tusproject/tusd:v1.6.0
    networks: [ 'test' ]
    depends_on: [ 'testminio' ]
    environment:
      AWS_ACCESS_KEY_ID: minio_access
      AWS_SECRET_ACCESS_KEY: minio_secret
      AWS_REGION: us-east-1
    command: [
      '-port=8080',  # no permission to bind port 80 on google-cloud-build won't bind port 80
      # no hooks in unit tests
      '-s3-endpoint=http://testminio:9000',
      '-s3-bucket=upload',
    ]

networks:
  dev: {}
  test: {}

volumes:
  dbdata: {}
  minio_data: {}
  # Let's not make node_modules a bind mount: Windows bind mounts seem
  # to behave a bit differently from Linux bind mounts. Use a Docker volume
  # instead.
  node_modules: {}
  jest_cache: {}