Deploy Data Pipelines (#12)

* Added initial dreem pipeline

* Bump image versions

* Update docker-compose.yml

* Update docker-compose.yml

* Define data volume and ignore local folder on server

* Ignore CSV folder from server repo

* Remove consumer from internet

As we need to access inventory/support APIs via the consumer API I kept it on the intranet as an interim solution.

* Store data in local volume

Useful for debugging

* Ignore log files and create dev pipeline container

* Added UCAM example env variables

.gitignore

@@ -5,3 +5,7 @@
 core/acme.json
 core/usersfile
 instances/supportdocs/docs
+instances/middleware/local
+data/
+local/
+*.log

instances/middleware/.consumer.env.example

@@ -5,3 +5,7 @@ VARIABLE_NAME=consumer
 LOG_LEVEL=INFO
 SUPPORT_BASE_URL=
 SUPPORT_TOKEN=
+
+UCAM_URI=""
+UCAM_USERNAME=""
+UCAM_PASSWORD=""

instances/middleware/.dtransfer.env.example

@@ -0,0 +1,44 @@
+IS_DEV="False"
+
+# IDEAFAST (WP3)
+INVENTORY_API=""http://consumer_dev/inventory/""
+DATABASE_URI="mongodb://USER:PASSWORD@mongo:27017/?authSource=DATABASE&authMechanism=SCRAM-SHA-256"
+SUPPORT_BASE_URL=""
+SUPPORT_TOKEN=""
+
+# DREEM DEVICE
+DREEM_LOGIN_URL=""
+DREEM_API_URL=""
+
+DREEM_KIEL_USERNAME=""
+DREEM_KIEL_PASSWORD=""
+
+DREEM_NEWCASTLE_USERNAME=""
+DREEM_NEWCASTLE_PASSWORD=""
+
+DREEM_MUNSTER_USERNAME=""
+DREEM_MUNSTER_PASSWORD=""
+
+DREEM_ROTTERDAM_USERNAME=""
+DREEM_ROTTERDAM_PASSWORD=""
+
+# VTT's Stress Monitor (SMA)
+VTTSMA_AWS_ACCESSKEY=""
+VTTSMA_AWS_SECRET_ACCESSKEY=""
+VTTSMA_AWS_BUCKET_NAME=""
+VTTSMA_GLOBAL_DEVICE_ID=""
+
+# BYTEFLIES DEVICE
+BYTEFLIES_API_URL=""
+BYTEFLIES_USERNAME=""
+BYTEFLIES_PASSWORD=""
+BYTEFLIES_AWS_CLIENT_ID=""
+BYTEFLIES_AWS_AUTH_URL=""
+
+# DATA MANAGEMENT PORTAL
+DMP_STUDY_ID=""
+DMP_URL=""
+DMP_PUBLIC_KEY=""
+DMP_SIGNATURE=""
+
+UCAM_API="http://consumer_dev/ucam/"

instances/middleware/docker-compose.yml

@@ -9,32 +9,48 @@ networks:
 services:
   consumer_live:
     container_name: consumer_live
-    image: ideafast/middleware:0.1.0
+    image: ideafast/middleware-consumer:0.1.0-DEV
     restart: unless-stopped
     networks:
       - web
     env_file:
       - .consumer.live.env
-    labels:
-      - "traefik.enable=true"
-      - "traefik.http.routers.consumer_live.rule=Host(`api.wp3.ideafast.eu`)"
-      - "traefik.http.routers.consumer_live.entrypoints=websecure"
-      - "traefik.http.routers.consumer_live.tls.certresolver=leresolver"
 
   consumer_dev:
     container_name: consumer_dev
-    image: ideafast/middleware:0.1.0
+    image: ideafast/middleware-consumer:0.1.1-DEV
     restart: unless-stopped
     networks:
       - web
     env_file:
       - .consumer.dev.env
-    labels:
-      - "traefik.enable=true"
-      - "traefik.http.routers.consumer_dev.rule=Host(`dev.api.wp3.ideafast.eu`)"
-      - "traefik.http.routers.consumer_dev.entrypoints=websecure"
-      - "traefik.http.routers.consumer_dev.tls.certresolver=leresolver"
 
+  dtransfer_live:
+    container_name: dtransfer_live
+    image: ideafast/middleware-dtransfer:0.1.2-DEV
+    restart: unless-stopped
+    networks:
+      - web
+      - database
+    env_file:
+      - .dtransfer.live.env
+    volumes:
+      - ./local:/app/local/
+      - ./live/data:/app/data/
+
+  dtransfer_dev:
+    container_name: dtransfer_dev
+    image: ideafast/middleware-dtransfer:0.1.3-DEV
+    restart: unless-stopped
+    networks:
+      - web
+      - database
+    env_file:
+      - .dtransfer.dev.env
+    volumes:
+      - ./local:/app/local/
+      - ./dev/data:/app/data/
+
   mongodb:
     container_name: mongo
     image: mongo:4.4.3