From a7fc786d1103c62229b00976a2e95a53f210b21a Mon Sep 17 00:00:00 2001 From: Thomas Cassaert Date: Wed, 2 Oct 2024 15:46:30 +0200 Subject: [PATCH] Add documentation on deployment of openEO Geotrellis and openEO Aggregator Signed-off-by: Thomas Cassaert --- docs/eoepca/openeo-aggregator.md | 33 +++++++++ docs/eoepca/openeo-geotrellis.md | 115 +++++++++++++++++++++++++++++++ mkdocs.yml | 4 +- 3 files changed, 151 insertions(+), 1 deletion(-) create mode 100644 docs/eoepca/openeo-aggregator.md create mode 100644 docs/eoepca/openeo-geotrellis.md diff --git a/docs/eoepca/openeo-aggregator.md b/docs/eoepca/openeo-aggregator.md new file mode 100644 index 00000000..66642c1c --- /dev/null +++ b/docs/eoepca/openeo-aggregator.md @@ -0,0 +1,33 @@ +# openEO Aggregator + +The openEO Aggregator is a software component to group multiple openEO back-ends together into a unified, federated openEO processing platform. + +## Helm chart + +A Chart can be found at the [openeo-geotrellis-kubernetes](https://github.com/Open-EO/openeo-geotrellis-kubernetes/tree/master/kubernetes/charts/openeo-aggregator) repo. + +The releases of the Helm chart are also hosted on the [VITO Artifactory](https://artifactory.vgt.vito.be/helm-charts) instance. + +Install openEO Aggregator as follows in your cluster: + +```bash +helm install openeo-aggregator openeo-aggregator \ + --repo https://artifactory.vgt.vito.be/artifactory/helm-charts \ + --version 0.1.2 \ + --namespace openeo-aggregator \ + --create-namespace \ + --values values.yaml +``` + +An example `values.yaml` file: + +```yaml +--- +envVars: + ENV: "prod" + ZOOKEEPERNODES: zookeeper.zookeeper-prod.svc.cluster.local:2181 + GUNICORN_CMD_ARGS: "--bind=0.0.0.0:8080 --workers=10 --threads=1 --timeout=900" +image: + repository: vito-docker.artifactory.vgt.vito.be/openeo-aggregator + tag: 0.39 +``` diff --git a/docs/eoepca/openeo-geotrellis.md b/docs/eoepca/openeo-geotrellis.md new file mode 100644 index 00000000..3f868479 --- /dev/null +++ b/docs/eoepca/openeo-geotrellis.md @@ -0,0 +1,115 @@ +# openEO Geotrellis + +openEO develops an API that allows users to connect to Earth observation cloud back-ends in a simple and unified way. +The project maintains the API and process specifications, and an open-source ecosystem with clients and server implementations. + +## Prerequisites + +### Spark Operator + +As openEO runs on Apache Spark, we need a way to run this in a Kubernetes cluster. For this requirement, we leverage the [Kubeflow Spark-Operator](https://github.com/kubeflow/spark-operator). Basic instructions on how to get it running inside you cluster are: + +```bash +helm install spark-operator spark-operator \ + --namespace spark-operator \ + --create-namespace \ + --repo https://kubeflow.github.io/spark-operator \ + --set webhook.enable=true \ + --set spark.jobNamespaces[0]="" +``` + +Take a look at the [values.yaml](https://github.com/kubeflow/spark-operator/blob/master/charts/spark-operator-chart/values.yaml) file for all the possible configuration options. + +### ZooKeeper + +openEO uses [Apache ZooKeeper](https://zookeeper.apache.org/) under the hood. To get a basic ZK installed in your cluster, follow these steps: + +```bash +helm install zookeeper oci://registry-1.docker.io/bitnamicharts/zookeeper \ + --create-namespace \ + --namespace zookeeper +``` + +The possible configuration values can be found in the [values.yaml](https://github.com/bitnami/charts/blob/main/bitnami/zookeeper/values.yaml) file. + +## Helm Chart + +openEO can be deployed by Helm. A Chart can be found at the [openeo-geotrellis-kubernetes](https://github.com/Open-EO/openeo-geotrellis-kubernetes/tree/master/kubernetes/charts/sparkapplication) repo. + +The releases of the Helm chart are also hosted on the [VITO Artifactory](https://artifactory.vgt.vito.be/helm-charts) instance. + +Install openEO as follows in your cluster: + +```bash +helm install openeo sparkapplication \ + --repo https://artifactory.vgt.vito.be/artifactory/helm-charts \ + --version 0.14.9 \ + --namespace openeo \ + --create-namespace \ + --values values.yaml +``` + +Example values.yaml file: +```yaml +--- +image: vito-docker.artifactory.vgt.vito.be/openeo-geotrellis-kube +imageVersion: latest +sparkVersion: 3.2.0 +type: Java +driver: + env: + KUBE: "true" + KUBE_OPENEO_API_PORT: "50001" + PYTHONPATH: $PYTHONPATH:/opt/tensorflow/python38/2.3.0/:/opt/openeo/lib/python3.8/site-packages/ + ZOOKEEPERNODES: zookeeper.zookeeper.svc.cluster.local:2181 + podSecurityContext: + fsGroup: 18585 + fsGroupChangePolicy: Always +executor: + env: + PYTHONPATH: $PYTHONPATH:/opt/tensorflow/python38/2.3.0/:/opt/openeo/lib/python3.8/site-packages/ +fileDependencies: + - local:///opt/layercatalog.json + - local:///opt/log4j2.xml +jarDependencies: + - local:///opt/geotrellis-extensions-static.jar +mainApplicationFile: local:///opt/openeo/lib64/python3.8/site-packages/openeogeotrellis/deploy/kube.py +sparkConf: + spark.executorEnv.DRIVER_IMPLEMENTATION_PACKAGE: openeogeotrellis + spark.appMasterEnv.DRIVER_IMPLEMENTATION_PACKAGE: openeogeotrellis +service: + enabled: true + port: 50001 +ha: + enabled: false +rbac: + create: true + role: + rules: + - apiGroups: + - "" + resources: + - pods + verbs: + - create + - delete + - deletecollection + - get + - list + - patch + - watch + - apiGroups: + - "" + resources: + - configmaps + verbs: + - create + - delete + - deletecollection + - list + serviceAccountDriver: openeo +``` + +This gives you an `openeo-driver` pod that you can `port-forward` to on port 50001. + +With the port-forward activated, you can access the openEO API with `curl -L localhost:50001`. diff --git a/mkdocs.yml b/mkdocs.yml index e176da9a..80f09217 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -43,6 +43,8 @@ nav: - Registration API: eoepca/registration-api.md - Workspace: eoepca/workspace.md - Container Registry: eoepca/container-registry.md + - openEO Geotrellis (Processing): eoepca/openeo-geotrellis.md + - Federated orchestrator: eoepca/openeo-aggregator.md theme: # name: mkdocs @@ -80,7 +82,7 @@ extra_css: - css/eoepca.css plugins: - - autorefs +# - autorefs # - pub-debugger: # console_log: # enabled: true