feat: support opensearch setup in docker-compose (#768)

Co-authored-by: Igor Petrov <[email protected]>
Co-authored-by: Yevhen Podluzhnyi <[email protected]>

.env

@@ -22,6 +22,7 @@ CAMUNDA_WEB_MODELER_VERSION=8.5.7
 # renovate: datasource=docker depName=elasticsearch
 ELASTIC_VERSION=8.15.1
 KEYCLOAK_SERVER_VERSION=24.0.5
+OPENSEARCH_VERSION=2.16.0
 # renovate: datasource=docker depName=axllent/mailpit
 MAILPIT_VERSION=v1.20.4
 POSTGRES_VERSION=14.5-alpine
@@ -45,3 +46,8 @@ RESOURCE_AUTHORIZATIONS_ENABLED=false
 #  ZEEBE_AUTHENTICATION_MODE=identity
 #
 MULTI_TENANCY_ENABLED=false
+
+OPENSEARCH_INITIAL_ADMIN_PASSWORD=OpenSearch_Admin1234!
+
+# search engine app: opensearch or elasticsearch
+SEARCH_DB=elasticsearch

.env.elasticsearch

@@ -0,0 +1,10 @@
+OPTIMIZE_ELASTICSEARCH_HOST=elasticsearch
+OPTIMIZE_ELASTICSEARCH_HTTP_PORT=9200
+ZEEBE_BROKER_EXPORTERS_ELASTICSEARCH_CLASSNAME=io.camunda.zeebe.exporter.ElasticsearchExporter
+ZEEBE_BROKER_EXPORTERS_ELASTICSEARCH_ARGS_URL=http://elasticsearch:9200
+# default is 1000, see here: https://github.com/camunda/zeebe/blob/main/exporters/elasticsearch-exporter/src/main/java/io/camunda/zeebe/exporter/ElasticsearchExporterConfiguration.java#L259
+ZEEBE_BROKER_EXPORTERS_ELASTICSEARCH_ARGS_BULK_SIZE=1
+CAMUNDA_OPERATE_ELASTICSEARCH_URL=http://elasticsearch:9200
+CAMUNDA_OPERATE_ZEEBEELASTICSEARCH_URL=http://elasticsearch:9200
+CAMUNDA_TASKLIST_ELASTICSEARCH_URL=http://elasticsearch:9200
+CAMUNDA_TASKLIST_ZEEBEELASTICSEARCH_URL=http://elasticsearch:9200

.env.elasticsearch.core

@@ -0,0 +1,7 @@
+ZEEBE_BROKER_EXPORTERS_ELASTICSEARCH_CLASSNAME=io.camunda.zeebe.exporter.ElasticsearchExporter
+ZEEBE_BROKER_EXPORTERS_ELASTICSEARCH_ARGS_URL=http://elasticsearch:9200
+ZEEBE_BROKER_EXPORTERS_ELASTICSEARCH_ARGS_BULK_SIZE=1
+CAMUNDA_OPERATE_ELASTICSEARCH_URL=http://elasticsearch:9200
+CAMUNDA_OPERATE_ZEEBEELASTICSEARCH_URL=http://elasticsearch:9200
+CAMUNDA_TASKLIST_ELASTICSEARCH_URL=http://elasticsearch:9200
+CAMUNDA_TASKLIST_ZEEBEELASTICSEARCH_URL=http://elasticsearch:9200

.env.opensearch

@@ -0,0 +1,21 @@
+# Optimize
+CAMUNDA_OPTIMIZE_OPENSEARCH_HOST=opensearch
+CAMUNDA_OPTIMIZE_OPENSEARCH_HTTP_PORT=9200
+CAMUNDA_OPTIMIZE_DATABASE=opensearch
+
+# Zeebe
+ZEEBE_BROKER_EXPORTERS_OPENSEARCH_CLASSNAME=io.camunda.zeebe.exporter.opensearch.OpensearchExporter
+ZEEBE_BROKER_EXPORTERS_OPENSEARCH_ARGS_URL=http://opensearch:9200
+ZEEBE_BROKER_EXPORTERS_OPENSEARCH_ARGS_BULK_SIZE=1
+
+# Operate
+CAMUNDA_OPERATE_DATABASE=opensearch
+CAMUNDA_OPERATE_OPENSEARCH_URL=http://opensearch:9200
+CAMUNDA_OPERATE_ZEEBEOPENSEARCH_URL=http://opensearch:9200
+
+# Tasklist
+CAMUNDA_TASKLIST_OPENSEARCH_URL=http://opensearch:9200
+CAMUNDA_TASKLIST_ZEEBEEOPENSEARCH_URL=http://opensearch:9200
+camunda.tasklist.database=opensearch
+camunda.tasklist.opensearch.url=http://opensearch:9200
+camunda.tasklist.zeebeOpensearch.url=http://opensearch:9200

.env.opensearch.core

@@ -0,0 +1,11 @@
+CAMUNDA_OPERATE_DATABASE=opensearch
+CAMUNDA_OPERATE_OPENSEARCH_URL=http://opensearch:9200
+CAMUNDA_OPERATE_ZEEBEOPENSEARCH_URL=http://opensearch:9200
+ZEEBE_BROKER_EXPORTERS_OPENSEARCH_CLASSNAME=io.camunda.zeebe.exporter.opensearch.OpensearchExporter
+ZEEBE_BROKER_EXPORTERS_OPENSEARCH_ARGS_INDEX_DEPLOYMENT=false
+ZEEBE_BROKER_EXPORTERS_OPENSEARCH_ARGS_URL=http://opensearch:9200
+ZEEBE_BROKER_EXPORTERS_OPENSEARCH_ARGS_BULK_SIZE=1
+ZEEBE_BROKER_EXPORTERS_OPENSEARCH_ARGS_INDEX_PROCESSMESSAGESUBSCRIPTION=true
+camunda.tasklist.opensearch.url=http://opensearch:9200
+camunda.tasklist.zeebeOpensearch.url=http://opensearch:9200
+camunda.tasklist.database=opensearch

.github/workflows/template-deploy.yaml

@@ -19,6 +19,11 @@ on:
         description: runs a single test
         required: false
         type: string
+      search_db:
+        description: elasticsearch or opensearch
+        required: false
+        default: elasticsearch
+        type: string
 
 jobs:
   test_compose_deploy:
@@ -34,6 +39,10 @@ jobs:
              sudo killall xsp4 || true
 
       - uses: actions/checkout@v4
+      - name: change search db
+        if: ${{ contains(inputs.search_db, 'opensearch') }}
+        run: |
+          sudo sed -i 's/SEARCH_DB=elasticsearch/SEARCH_DB=opensearch/g' .env
 
       - name: Login to private registry
         if: ${{ inputs.directory != 'docker-compose/camunda-8.6' }}

.github/workflows/test-core-opensearch.yaml

@@ -0,0 +1,13 @@
+name: "Test docker-compose-core.opensearch.yaml"
+on:
+  push:
+    branches:
+      - "**"
+jobs:
+  test_core:
+    uses: ./.github/workflows/template-deploy.yaml
+    secrets: inherit
+    with:
+      compose_args: "-f docker-compose-core.yaml --profile opensearch"
+      run_e2e_tests: false
+      search_db: opensearch

.github/workflows/test-default.opensearch.yaml

@@ -0,0 +1,13 @@
+name: "Test docker-compose.opensearch.yaml"
+on:
+  push:
+    branches:
+      - "**"
+jobs:
+  test_docker_compose_yaml:
+    uses: ./.github/workflows/template-deploy.yaml
+    secrets: inherit
+    with:
+      compose_args: "-f docker-compose.yaml --profile opensearch"
+      run_e2e_tests: false
+      search_db: opensearch

.github/workflows/test-modeler.opensearch.yaml

@@ -0,0 +1,13 @@
+name: "Test docker-compose-web-modeler.yaml with OpenSearch"
+on:
+  push:
+    branches:
+      - "**"
+jobs:
+  test_modeler_yaml:
+    uses: ./.github/workflows/template-deploy.yaml
+    secrets: inherit
+    with:
+      compose_args: "-f docker-compose.yaml -f docker-compose-web-modeler.yaml --profile opensearch"
+      run_e2e_tests: true
+      search_db: opensearch

README.md

@@ -39,7 +39,7 @@ The full environment contains these components:
 - Connectors
 - Optimize
 - Identity
-- Elasticsearch
+- Elasticsearch/Opensearch
 - Keycloak
 - PostgreSQL
 
@@ -80,6 +80,12 @@ If Optimize, Identity, and Keycloak are not needed you can use the [docker-compo
 ```
 docker compose -f docker-compose-core.yaml up -d
 ```
+**OR**
+
+Set parameter ```SEARCH_DB=opensearch``` in the file ```.env``` if you want to use OpenSearch instead of ElasticSearch
+```
+docker compose -f docker-compose-core.yaml --profile opensearch up -d
+```
 
 ### Deploying BPMN diagrams
 
@@ -167,10 +173,16 @@ $ docker login registry.camunda.cloud
 Username: your_username
 Password: ******
 Login Succeeded
-$ docker compose -f docker-compose.yaml -f docker-compose-web-modeler.yaml up -d
 ```
 
-To tear down the whole environment run the following command
+#### To run Camunda Platform with Elasticsearch execute this commands
+
+1. Edit ```.env``` file and set parameter ```SEARCH_DB=elasticserach``` (this default value)
+2. Run command
+```
+$ docker compose -f docker-compose.yaml -f docker-compose-web-modeler.yaml up -d
+```
+3. To tear down the whole environment with ```ElasticSearch``` run the following command
 
 ```
 $ docker compose -f docker-compose.yaml -f docker-compose-web-modeler.yaml down -v
@@ -183,6 +195,19 @@ Alternatively, if you want to keep the data run:
 $ docker compose -f docker-compose.yaml -f docker-compose-web-modeler.yaml down
 ```
 
+To run Camunda Platform with ```OpenSearch``` execute this commands
+
+1. Edit ```.env``` file and set parameter ```SEARCH_DB=opensearch```
+2. Run command
+```
+$ docker compose -f docker-compose.yaml -f docker-compose-web-modeler.yaml --profile opensearch up -d
+```
+3. To tear down the whole environment with Elasticsearch run the following command (-v is optional flag. Use it, if you want to delete all the data)
+
+```
+$ docker compose -f docker-compose.yaml -f docker-compose-web-modeler.yaml --profile opensearch down -v
+```
+
 ### Login
 You can access Web Modeler Self-Managed and log in with the user `demo` and password `demo` at [http://localhost:8070](http://localhost:8070).
 

docker-compose-core.yaml

@@ -10,19 +10,24 @@
 # See docker-compose.yml for a configuration that also includes Optimize, Identity, and Keycloak.
 
 services:
+  init: # recommended config https://opensearch.org/docs/1.1/opensearch/install/important-settings/
+    image: bash
+    profiles: ["opensearch"]
+    privileged: true
+    user: root
+    command: [ "sysctl", "-w", "vm.max_map_count=262144" ]
 
   zeebe: # https://docs.camunda.io/docs/self-managed/platform-deployment/docker/#zeebe
     image: camunda/zeebe:${CAMUNDA_PLATFORM_VERSION}
     container_name: zeebe
+    profiles: ["", "opensearch"]
     ports:
       - "26500:26500"
       - "9600:9600"
       - "8088:8080"
+    env_file:
+      - .env.${SEARCH_DB}.core
     environment: # https://docs.camunda.io/docs/self-managed/zeebe-deployment/configuration/environment-variables/
-      - ZEEBE_BROKER_EXPORTERS_ELASTICSEARCH_CLASSNAME=io.camunda.zeebe.exporter.ElasticsearchExporter
-      - ZEEBE_BROKER_EXPORTERS_ELASTICSEARCH_ARGS_URL=http://elasticsearch:9200
-      # default is 1000, see here: https://github.com/camunda/zeebe/blob/main/exporters/elasticsearch-exporter/src/main/java/io/camunda/zeebe/exporter/ElasticsearchExporterConfiguration.java#L259
-      - ZEEBE_BROKER_EXPORTERS_ELASTICSEARCH_ARGS_BULK_SIZE=1
       # allow running with low disk space
       - ZEEBE_BROKER_DATA_DISKUSAGECOMMANDWATERMARK=0.998
       - ZEEBE_BROKER_DATA_DISKUSAGEREPLICATIONWATERMARK=0.999
@@ -39,17 +44,18 @@ services:
     networks:
       - camunda-platform
     depends_on:
-      - elasticsearch
+      - ${SEARCH_DB}
 
   operate: # https://docs.camunda.io/docs/self-managed/platform-deployment/docker/#operate
     image: camunda/operate:${CAMUNDA_OPERATE_VERSION}
     container_name: operate
+    profiles: ["", "opensearch"]
     ports:
       - "8081:8080"
+    env_file:
+      - .env.${SEARCH_DB}.core
     environment: # https://docs.camunda.io/docs/self-managed/operate-deployment/configuration/
       - CAMUNDA_OPERATE_ZEEBE_GATEWAYADDRESS=zeebe:26500
-      - CAMUNDA_OPERATE_ELASTICSEARCH_URL=http://elasticsearch:9200
-      - CAMUNDA_OPERATE_ZEEBEELASTICSEARCH_URL=http://elasticsearch:9200
       - CAMUNDA_OPERATE_CSRFPREVENTIONENABLED=false
       - management.endpoints.web.exposure.include=health
       - management.endpoint.health.probes.enabled=true
@@ -63,18 +69,19 @@ services:
       - camunda-platform
     depends_on:
       - zeebe
-      - elasticsearch
+      - ${SEARCH_DB}
 
   tasklist: # https://docs.camunda.io/docs/self-managed/platform-deployment/docker/#tasklist
     image: camunda/tasklist:${CAMUNDA_TASKLIST_VERSION}
     container_name: tasklist
+    profiles: ["", "opensearch"]
     ports:
       - "8082:8080"
+    env_file:
+      - .env.${SEARCH_DB}.core
     environment: # https://docs.camunda.io/docs/self-managed/tasklist-deployment/configuration/
       - CAMUNDA_TASKLIST_ZEEBE_GATEWAYADDRESS=zeebe:26500
       - CAMUNDA_TASKLIST_ZEEBE_RESTADDRESS=http://zeebe:8080
-      - CAMUNDA_TASKLIST_ELASTICSEARCH_URL=http://elasticsearch:9200
-      - CAMUNDA_TASKLIST_ZEEBEELASTICSEARCH_URL=http://elasticsearch:9200
       - CAMUNDA_TASKLIST_CSRFPREVENTIONENABLED=false
       - management.endpoints.web.exposure.include=health
       - management.endpoint.health.probes.enabled=true
@@ -88,11 +95,12 @@ services:
       - camunda-platform
     depends_on:
       - zeebe
-      - elasticsearch
+      - ${SEARCH_DB}
 
   connectors: # https://docs.camunda.io/docs/components/integration-framework/connectors/out-of-the-box-connectors/available-connectors-overview/
     image: camunda/connectors-bundle:${CAMUNDA_CONNECTORS_VERSION}
     container_name: connectors
+    profiles: ["", "opensearch"]
     ports:
       - "8085:8080"
     environment:
@@ -116,9 +124,48 @@ services:
       - zeebe
       - operate
 
+  opensearch:
+    image: opensearchproject/opensearch:${OPENSEARCH_VERSION}
+    container_name: opensearch
+    profiles: ["opensearch"]
+    depends_on:
+      - init
+    environment:
+      cluster.name: opensearch-cluster
+      network.bind_host: 0.0.0.0
+      node.name: opensearch
+      plugins.security.disabled: true
+      discovery.type: single-node
+      discovery.seed_hosts: opensearch
+      bootstrap.memory_lock: true
+      OPENSEARCH_JAVA_OPTS: "-Xms1G -Xmx1G"
+      OPENSEARCH_INITIAL_ADMIN_PASSWORD: ${OPENSEARCH_INITIAL_ADMIN_PASSWORD}
+    ulimits:
+      memlock:
+        soft: -1
+        hard: -1
+      nofile:
+        soft: 65536
+        hard: 65536
+    ports:
+      - "9200:9200"
+      - "9601:9600"
+    restart: always
+    healthcheck:
+     # Single node OpenSearch clusters are considered 'yellow', see https://github.com/opensearch-project/opensearch-build/issues/4285
+      test: [ "CMD-SHELL", "curl -k -f http://localhost:9200/_cluster/health -u admin:${OPENSEARCH_INITIAL_ADMIN_PASSWORD} | grep -q -E 'yellow|green'" ]
+      interval: 60s
+      timeout: 10s
+      retries: 5
+    volumes:
+      - opensearch-data:/usr/share/opensearch/data
+    networks:
+      - camunda-platform
+
   elasticsearch: # https://hub.docker.com/_/elasticsearch
     image: docker.elastic.co/elasticsearch/elasticsearch:${ELASTIC_VERSION}
     container_name: elasticsearch
+    profiles: [""]
     ports:
       - "9200:9200"
       - "9300:9300"
@@ -154,13 +201,14 @@ services:
     networks:
       - camunda-platform
     depends_on:
-      - elasticsearch
+      - ${SEARCH_DB}
     profiles:
       - kibana
 
 volumes:
   zeebe:
   elastic:
+  opensearch-data:
   kibana:
 
 networks: