feat: improve dev setup (#651)

* fix: use proper dummy app key length

* feat: mount plugin to host path

* feat: parameterize dag path, plugin path, and port

* feat: plugin artifact via variable

* feat: remove artifact path

* feat: add load secret script

* feat: use relative path instead

* feat: adding dummy plugin

* feat: multiple secret load

* feat: load secret from file contains key value

* feat: use load.sh to load plugin and secrets

* fix: readme

* feat: add image busybox on dummy plugin

* refactor: gitignore and add parameterize optimus namespace

* feat: add script for load artifacts

* doc: update docs for mounting plugings

* docs: add port-forward + fix typo

* feat: add load secret message

dev/.gitignore

@@ -0,0 +1,2 @@
+plugins/*
+!plugins/optimus-plugin-echo.yaml 

dev/Makefile

@@ -1,35 +1,50 @@
 EXPECTED_CONTEXT=colima
 NAMESPACE=optimus-dev
 OPTIMUS_ODPF_CHART=odpf/optimus
+OPTIMUS_PLUGINS_PATH=/tmp/colima/plugins
+
+DAGS_PATH?=/tmp/colima/dags
+OPTIMUS_SERVE_PORT?=9100
+SETUP_FILE_PATH?=./setup.yaml
+
+PROJECT?=project-a
+HOST?=localhost:9100
 
 apply.airflow: _context
+	mkdir -p ${DAGS_PATH}
 	helm repo add apache-airflow https://airflow.apache.org
-	helm upgrade --install airflow apache-airflow/airflow\
+	helm upgrade --install airflow apache-airflow/airflow \
 		--namespace ${NAMESPACE} \
+		--set scheduler.extraVolumes[0].hostPath.path=$(abspath ${DAGS_PATH}) \
 		-f airflow.values.yaml
 
 apply.optimus: _context
-	# to create placeholder plugins for internal and odpf plugins
-	echo "test" >  /tmp/colima/test
-	tar -czvf /tmp/colima/internal-plugins.tar.gz /tmp/colima/test
-	tar -czvf /tmp/colima/transformers.tar.gz /tmp/colima/test
-	rm /tmp/colima/test
-
+	mkdir -p ${DAGS_PATH}
+	mkdir -p ${OPTIMUS_PLUGINS_PATH}
 	helm repo add odpf https://odpf.github.io/charts/
-	helm upgrade --install optimus-dev ${OPTIMUS_ODPF_CHART} -n ${NAMESPACE} -f optimus.values.yaml --version '>0.0.2'
+	$(eval OPTIMUS_PLUGIN_ARTIFACTS = $(shell ./get_artifacts.sh ${SETUP_FILE_PATH}))
+	helm upgrade --install optimus-dev ${OPTIMUS_ODPF_CHART} \
+	-n ${NAMESPACE} \
+	--set volumes[0].hostPath.path=$(abspath ${DAGS_PATH}) \
+	--set volumes[1].hostPath.path=$(abspath ${OPTIMUS_PLUGINS_PATH}) \
+	--set config.OPTIMUS_SERVE_PORT=${OPTIMUS_SERVE_PORT} \
+	--set config.OPTIMUS_PLUGIN_ARTIFACTS=${OPTIMUS_PLUGIN_ARTIFACTS} \
+	-f optimus.values.yaml --version '>0.0.2'
+
+_load.secrets:
+	PROJECT=${PROJECT} HOST=${HOST} ./load_secrets.sh ${SETUP_FILE_PATH}
 
 _context:
 	kubectl config use-context $(EXPECTED_CONTEXT)
 
 _pre: _context
-	mkdir -p /tmp/colima/dags
-
 	kubectl create ns ${NAMESPACE} | true
 	kubectl apply -f pvc.yaml -n ${NAMESPACE}
 
 	kubectl apply -f postgres.yaml -n ${NAMESPACE}
 	kubectl wait --namespace=${NAMESPACE} --timeout=120s --for=condition=Available deployment/optimus-db
 
+	mkdir -p $(abspath ${DAGS_PATH})
 
 apply: _pre apply.optimus apply.airflow
 

dev/README.md

@@ -1,11 +1,21 @@
 
 # Optimus Dev setup
 
-## setup
+## Setup
 + `brew install colima`
 + `make start-colima`
 + check kubernetes context
 + `make apply`
++ expose the port to local machine:
+    + `kubectl port-forward svc/optimus-dev 9100:80`
+    + `kubectl port-forward svc/airflow-webserver 8080:8080`
+
+Some optional variable you can set alongside with `make apply`
+```sh
+DAGS_PATH=                # default /tmp/colima/dags
+OPTIMUS_SERVE_PORT=       # default 9100
+SETUP_FILE_PATH=          # default ./setup.yaml
+```
 
 ## Components
 + optimus server
@@ -14,21 +24,28 @@
 + airflow db (postgres)
 
 ### Dag file location on your laptop
-+  `/tmp/colima/dags`
-
-### Mounting plugins
-+ upadate `path`'s for `internal-plugins` and `odpf-plugins` volumes in optimus.values.yaml with the actual path to the plugins and upgrade the helm chart for optimus (`make upgrade.optimus`)
-
-```yaml
-volumes:
-  - name: internal-plugins
-    hostPath: 
-      path: /Users/../Documents/proj/optimus-plugins/dist/optimus-plugins_0.6.1_linux_amd64.tar.gz
-      type: File
-  - name: odpf-plugins
-    hostPath: 
-      path: /Users/../Documents/proj/transformers/dist/transformers_0.1.1_linux_arm64.tar.gz
-      type: File
++ `/tmp/colima/dags` or specified by `DAGS_PATH`
+
+### Spinning up the project
++ `mkdir project-a`
++ `cd project-a`
++ `optimus init`
++ `optimus project register`
++ `optimus plugin sync`
++ then load the secret. [ref](#load-secrets)
+
+### Mounting the plugins
++ define plugin artifacts on `setup.yaml` under section `plugins`
++ `make apply`. You can pass `SETUP_FILE_PATH` if the path is not the default one
+
+### Load secrets
++ define key value pair of secrets on `setup.yaml` under section `secrets`
++ `make _load.secrets`
+Some optional variable you can set alongside with `make _load.secrets`
+```sh
+SETUP_FILE_PATH= # default ./setup.yaml
+PROJECT=         # default project-a
+HOST=            # default localhost:9100
 ```
 
 ### Connect to optimus db

dev/airflow.values.yaml

@@ -10,6 +10,8 @@ scheduler:
           type: Directory
 
 config:
+  core:
+    dags_folder: /opt/airflow/dags/dags
   webserver:
     expose_config: 'True'  # by default this is 'False'
 

dev/get_artifacts.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+# this script is used to copy plugin artifact to plugins mounted location
+# then return the artifact paths which inside the plugins mounted location
+
+OPTIMUS_PLUGINS_PATH=/tmp/colima/plugins
+SETUP_PATH=$1
+
+is_yq_installed(){
+  if [ -z $(command -v yq) ]; then
+    >&2 echo "yq must be installed: \`brew install yq\`"
+    exit 1
+  fi
+}
+
+if [ -z "$SETUP_PATH" ]; then
+  >&2 echo "must provide setup_path: ./get_artifacts.sh <setup_path>"
+  exit 1
+fi
+
+# load plugins
+is_yq_installed
+mkdir -p ${OPTIMUS_PLUGINS_PATH}
+OPTIMUS_PLUGIN_ARTIFACTS=""
+yq '.plugins[]' ${SETUP_PATH} >> p.tmp
+while read artifact; do
+  artifact=$(cp $artifact $OPTIMUS_PLUGINS_PATH 2> /dev/null && echo "/app/plugins/$(basename ${artifact})" || echo $artifact)
+  OPTIMUS_PLUGIN_ARTIFACTS="${OPTIMUS_PLUGIN_ARTIFACTS}\,${artifact}"
+done < p.tmp && rm p.tmp
+OPTIMUS_PLUGIN_ARTIFACTS="\"$(echo $OPTIMUS_PLUGIN_ARTIFACTS | sed 's/^\\,//g')\""
+echo $OPTIMUS_PLUGIN_ARTIFACTS

dev/load_secrets.sh

@@ -0,0 +1,36 @@
+#!/bin/sh
+
+OPTIMUS_NAMESPACE=optimus-dev
+
+PROJECT=${PROJECT:-project-a}
+HOST=${HOST:-"localhost:9100"}
+SETUP_PATH=$1
+
+is_yq_installed(){
+  if [ -z $(command -v yq) ]; then
+    >&2 echo "yq must be installed: \`brew install yq\`"
+    exit 1
+  fi
+}
+
+if [ -z "$SETUP_PATH" ]; then
+  >&2 echo "must provide setup_path: ./load_secrets.sh <setup_path>"
+  exit 1
+fi
+
+# load secrets
+echo ">> load secrets into project ${PROJECT}"
+is_yq_installed
+if ! curl --output /dev/null --silent ${HOST}/ping; then
+  >&2 echo "can't connect to optimus host ${HOST}"
+  exit 1
+fi
+
+yq '.secrets[] | (.name | . + " ") + (.value | @base64)' ${SETUP_PATH} >> s.tmp
+while read key value; do
+    resp=$(curl -XPOST -H "Content-Type: application/json" \
+    "${HOST}/api/v1beta1/project/${PROJECT}/secret/${key}" \
+    -d '{"value": "'${value}'"}' -s -o s.result -w "%{http_code}")
+    if [ $resp -eq 200 ]; then echo "set secret ${key}: success"; else echo "set secret ${key}: fail\n$(cat s.result)"; fi
+    rm s.result
+done < s.tmp && rm s.tmp

dev/optimus.values.yaml

@@ -18,25 +18,18 @@ container:
   volumeMounts:
     - mountPath: /airflow-dags/dags
       name: dags-path
-    - mountPath: /app/optimus-plugins.tar.gz
-      name: internal-plugins
-    - mountPath: /app/transformers.tar.gz
-      name: odpf-plugins
-
+    - mountPath: /app/plugins
+      name: plugins-path
 
 volumes:
   - name: dags-path
     hostPath: 
-      path: /tmp/colima/dags
+      path: /tmp/colima/dags # will be replaced through makefile
       type: Directory
-  - name: internal-plugins
+  - name: plugins-path
     hostPath: 
-      path: /tmp/colima/internal-plugins.tar.gz  # replace this with localpath for plugin dist tar file
-      type: File
-  - name: odpf-plugins
-    hostPath: 
-      path: /tmp/colima/transformers.tar.gz  # replace this with localpath for plugin dist tar file
-      type: File
+      path: /tmp/colima/plugins
+      type: Directory
 
 telegraf:
   enabled: false
@@ -48,17 +41,10 @@ configYaml: |-
   log:
     level: DEBUG
   serve:
-    port: 9100
-    ingress_host: optimus.optimus.svc.cluster.local
+    ingress_host: optimus-dev.optimus-dev.svc.cluster.local:80
   scheduler:
     name: airflow2
-  plugin:
-    artifacts:
-      - https://raw.githubusercontent.com/odpf/transformers/v0.1.7/task/bq2bq/optimus-plugin-bq2bq.yaml
-      - /app/optimus-plugins.tar.gz
-      - /app/transformers.tar.gz
-
 
 config:
   OPTIMUS_SERVE_DB_DSN: postgres://optimus:optimus@database-service/optimus?sslmode=disable
-  OPTIMUS_SERVE_APP_KEY: hgfhjsdfhsdhjsdfgjhfhdf
+  OPTIMUS_SERVE_APP_KEY: TuMxWWp4NkqNqaes6vK9DeqgUkk8I97s

dev/plugins/optimus-plugin-echo.yaml

@@ -0,0 +1,10 @@
+name: echo
+description: Echo sample plugin for optimus
+plugintype: task
+pluginversion: 0.1.0
+image: docker.io/busybox
+questions:
+- name: HELLO
+  prompt: Hello World
+  help: Example
+  minlength: 3

dev/setup.yaml

@@ -0,0 +1,18 @@
+plugins:
+  - https://raw.githubusercontent.com/odpf/transformers/main/task/bq2bq/optimus-plugin-bq2bq.yaml
+  - ./plugins/optimus-plugin-echo.yaml
+secrets:
+  - name: BQ_SERVICE_ACCOUNT
+    value: >
+      {
+        "type": "service_account",
+        "project_id": "default",
+        "private_key": "-----BEGIN PRIVATE KEY-----\nEXAMPLE==\n-----END PRIVATE KEY-----\n",
+      }
+  - name: EXAMPLE
+    value: >
+      {
+        "type": "service_account",
+        "project_id": "default",
+        "private_key": "-----BEGIN PRIVATE KEY-----\nEXAMPLE==\n-----END PRIVATE KEY-----\n",
+      }