From 63229609af76c5c48aeba24f96fa5ce47e6c35e3 Mon Sep 17 00:00:00 2001 From: Matthias Mohr Date: Fri, 23 Aug 2024 16:03:59 +0200 Subject: [PATCH 1/8] Implementation guidelines for EOAP #507 --- meta/implementation.md | 61 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 61 insertions(+) diff --git a/meta/implementation.md b/meta/implementation.md index c022ca52..d818f7f2 100644 --- a/meta/implementation.md +++ b/meta/implementation.md @@ -230,3 +230,64 @@ We have found some libraries that can be used for an implementation: - Julia: [Statistics.quantile](https://docs.julialang.org/en/v1/stdlib/Statistics/#Statistics.quantile!), type 7 is the default. - Python: [numpy](https://numpy.org/doc/stable/reference/generated/numpy.quantile.html), [pandas](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.quantile.html), [xarray](http://xarray.pydata.org/en/stable/generated/xarray.DataArray.quantile.html) - type 7 (called 'linear' for the interpolation parameter) is the default for all of them. - R: [quantile](https://stat.ethz.ch/R-manual/R-patched/library/stats/html/quantile.html) - type 7 is the default. + +## OGC API - Processes + +OGC API - Processes and OGC EO Application Packages (AP) can generally be unilized in openEO in three different ways: + +1. **openEO process** + + As a pre-defined process that exposes itself as a normal openEO process. + It is not exposed to the user that in the background an AP is invoked. +2. **Pre-deployment through ADES** + + In addition to the openEO API, a provider can offer access to an Application Deployment and Execution Service (ADES). + The ADES is likely external to the openEO API tree due to the conflicting `GET /processes` endpoint. + As such the ADES exposes itself in the `GET /` endpoint of the openEO API instance through a link. + The link must have the relation type `http://www.opengis.net/def/rel/ogc/1.0/processes`, which points to the ADES `/processes` endpoint. + Users can deploy APs through the ADES and use them through the process `run_ogcapi`. + + If the provider doesn't offer an ADES itself, users could also deploy their AP with another provider. + In this case use the process `run_ogcapi_externally` instead. + + Example process node: + + ```json + { + "process_id": "run_ogcapi", + "arguments": { + "data": ..., // Data, e.g. subtypes datacube or stac + "id": "my-ap", // Identifier of the application package in the ADES + "context": { // Parameters as defined in the CWL file + "cwl_param1": true, + "param2": 99 + } + } + } + ``` +3. **CWL provided at runtime (UDF runtime)** + + Providers can also provide a UDF runtime for the language CWL (instead of e.g. Python or R). + The runtime is exposed through the endpoint `GET /udf_runtimes`. + + Example process node: + + ```json + { + "process_id": "run_udf", + "arguments": { + "data": ..., // Data, e.g. subtypes datacube or stac + "udf": "...", // CWL as YAML string/JSON object, URL, or file on the API user workspace + "runtime": "cwl", // Assuming the UDF runtime is named "cwl" + "context": { // Parameters as defined in the CWL file + "cwl_param1": true, + "param2": 99 + } + } + } + ``` + +Generally, we recommend to use the following types and formats for the CWL inputs and outputs: + +- `type`: `File` or `File[]` depending on the capabilities of the CWL workflow +- `format`: For STAC inputs and outputs either `stac:item`, `stac:collection`, `stac:catalog`, `stac:itemcollection`, or `stac:stac` (i.e. and of the types before). From 4c56f7754282814605cdf6a9fdda22f3e7bb49d5 Mon Sep 17 00:00:00 2001 From: Matthias Mohr Date: Fri, 23 Aug 2024 16:19:28 +0200 Subject: [PATCH 2/8] Add processes --- proposals/run_ogcapi.json | 47 +++++++++++++++++++++++ proposals/run_ogcapi_externally.json | 57 ++++++++++++++++++++++++++++ 2 files changed, 104 insertions(+) create mode 100644 proposals/run_ogcapi.json create mode 100644 proposals/run_ogcapi_externally.json diff --git a/proposals/run_ogcapi.json b/proposals/run_ogcapi.json new file mode 100644 index 00000000..72c37e63 --- /dev/null +++ b/proposals/run_ogcapi.json @@ -0,0 +1,47 @@ +{ + "id": "run_ogcapi", + "summary": "Run a OGC API process", + "description": "Runs an OGC API - Processes process that the service provider offers through an Application Deployment and Execution Service (ADES).", + "categories": [ + "cubes", + "import", + "udf" + ], + "experimental": true, + "parameters": [ + { + "name": "data", + "description": "The data to be passed to the OGC API process.\n\nThe data must be given in a way that the external process can understand it.", + "schema": { + "description": "A value of any data type." + } + }, + { + "name": "id", + "description": "The identifier of the OGC API process.", + "schema": { + "type": "string" + } + }, + { + "name": "context", + "description": "Additional parameters as defined by the OGC API process.", + "schema": { + "type": "object" + }, + "default": {}, + "optional": true + } + ], + "exceptions": { + "InvalidOgcApiProcess": { + "message": "The specified OGC API process identifier does not exist." + } + }, + "returns": { + "description": "The data processed by the OGC API process. The returned value can be of any data type and is exactly what the OGC API process returns.", + "schema": { + "description": "Any data type." + } + } +} diff --git a/proposals/run_ogcapi_externally.json b/proposals/run_ogcapi_externally.json new file mode 100644 index 00000000..76b40fcc --- /dev/null +++ b/proposals/run_ogcapi_externally.json @@ -0,0 +1,57 @@ +{ + "id": "run_ogcapi_externally", + "summary": "Run an externally hosted OGC API process", + "description": "Runs an OGC API - Processes process that is either externally hosted by a service provider or running on a local machine of the user.", + "categories": [ + "cubes", + "import", + "udf" + ], + "experimental": true, + "parameters": [ + { + "name": "data", + "description": "The data to be passed to the external process.\n\nThe data must be given in a way that the external process can understand it.", + "schema": { + "description": "A value of any data type." + } + }, + { + "name": "url", + "description": "Absolute URL to the OGC API - Processes landing page.", + "schema": { + "type": "string", + "format": "uri", + "subtype": "uri", + "pattern": "^https?://" + } + }, + { + "name": "id", + "description": "The identifier of the OGC API process.", + "schema": { + "type": "string" + } + }, + { + "name": "context", + "description": "Additional parameters as defined by the OGC API process.", + "schema": { + "type": "object" + }, + "default": {}, + "optional": true + } + ], + "exceptions": { + "InvalidOgcApiProcess": { + "message": "The specified OGC API process does not exist." + } + }, + "returns": { + "description": "The data processed by the OGC API process. The returned value can be of any data type and is exactly what the OGC API process returns.", + "schema": { + "description": "Any data type." + } + } +} From 7f883c19f303d2742729f1e22bb2ee72d9ba9619 Mon Sep 17 00:00:00 2001 From: Matthias Mohr Date: Fri, 23 Aug 2024 16:20:25 +0200 Subject: [PATCH 3/8] Add changelog --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index cf37ada6..3411d637 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,12 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## Unreleased / Draft +### Added + +- `run_ogcapi` +- `run_ogcapi_processes` +- Implementation guide for implementing OGC API - Processes in openEO + ### Changed - `clip`: Throw an exception if min > max [#472](https://github.com/Open-EO/openeo-processes/issues/472) From b5cd5af066254fbefd7e5e9e4e36cc10c5fc0ec4 Mon Sep 17 00:00:00 2001 From: Matthias Mohr Date: Mon, 2 Sep 2024 11:45:55 +0200 Subject: [PATCH 4/8] Apply suggestions from code review --- meta/implementation.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/meta/implementation.md b/meta/implementation.md index d818f7f2..7eb6785e 100644 --- a/meta/implementation.md +++ b/meta/implementation.md @@ -233,7 +233,7 @@ We have found some libraries that can be used for an implementation: ## OGC API - Processes -OGC API - Processes and OGC EO Application Packages (AP) can generally be unilized in openEO in three different ways: +OGC API - Processes and OGC EO Application Packages (AP) can generally be utilized in openEO in three different ways: 1. **openEO process** @@ -241,7 +241,7 @@ OGC API - Processes and OGC EO Application Packages (AP) can generally be uniliz It is not exposed to the user that in the background an AP is invoked. 2. **Pre-deployment through ADES** - In addition to the openEO API, a provider can offer access to an Application Deployment and Execution Service (ADES). + In addition to the openEO API, a provider can offer access to an [Application Deployment and Execution Service (ADES)](https://github.com/eoepca/proc-ades). The ADES is likely external to the openEO API tree due to the conflicting `GET /processes` endpoint. As such the ADES exposes itself in the `GET /` endpoint of the openEO API instance through a link. The link must have the relation type `http://www.opengis.net/def/rel/ogc/1.0/processes`, which points to the ADES `/processes` endpoint. From e7ba6866d4b3691342433ca4bf6c0189c2fb4266 Mon Sep 17 00:00:00 2001 From: Matthias Mohr Date: Mon, 2 Sep 2024 13:48:00 +0200 Subject: [PATCH 5/8] ADES => OGC DRU --- dev/.words | 1 + meta/implementation.md | 18 +++++++++--------- proposals/run_ogcapi.json | 2 +- 3 files changed, 11 insertions(+), 10 deletions(-) diff --git a/dev/.words b/dev/.words index 846e729a..a8f3d8a9 100644 --- a/dev/.words +++ b/dev/.words @@ -48,3 +48,4 @@ Hyndman date1 date2 favor +Undeploy diff --git a/meta/implementation.md b/meta/implementation.md index 7eb6785e..67c4f0e6 100644 --- a/meta/implementation.md +++ b/meta/implementation.md @@ -239,15 +239,15 @@ OGC API - Processes and OGC EO Application Packages (AP) can generally be utiliz As a pre-defined process that exposes itself as a normal openEO process. It is not exposed to the user that in the background an AP is invoked. -2. **Pre-deployment through ADES** - - In addition to the openEO API, a provider can offer access to an [Application Deployment and Execution Service (ADES)](https://github.com/eoepca/proc-ades). - The ADES is likely external to the openEO API tree due to the conflicting `GET /processes` endpoint. - As such the ADES exposes itself in the `GET /` endpoint of the openEO API instance through a link. - The link must have the relation type `http://www.opengis.net/def/rel/ogc/1.0/processes`, which points to the ADES `/processes` endpoint. - Users can deploy APs through the ADES and use them through the process `run_ogcapi`. +2. **Pre-deployment through *OGC API - Processes - Part 2: Deploy, Replace, Undeploy*** + + In addition to the openEO API, a provider can offer access to an instance of *OGC API - Processes - Part 2: Deploy, Replace, Undeploy* (OGC DRU). + The OGC DRU instance is likely external to the openEO API tree due to the conflicting `GET /processes` endpoint. + As such the OGC DRU instance exposes itself in the `GET /` endpoint of the openEO API instance through a link. + The link must have the relation type `http://www.opengis.net/def/rel/ogc/1.0/processes`, which points to the `/processes` of the OGC API - Processes endpoint. + Users can deploy APs through the OGC DRU instance and use them through the process `run_ogcapi`. - If the provider doesn't offer an ADES itself, users could also deploy their AP with another provider. + If the provider doesn't offer an OGC DRU instance itself, users could also deploy their AP with another provider. In this case use the process `run_ogcapi_externally` instead. Example process node: @@ -257,7 +257,7 @@ OGC API - Processes and OGC EO Application Packages (AP) can generally be utiliz "process_id": "run_ogcapi", "arguments": { "data": ..., // Data, e.g. subtypes datacube or stac - "id": "my-ap", // Identifier of the application package in the ADES + "id": "my-ap", // Identifier of the application package in OGC API - Processes "context": { // Parameters as defined in the CWL file "cwl_param1": true, "param2": 99 diff --git a/proposals/run_ogcapi.json b/proposals/run_ogcapi.json index 72c37e63..9b54eb65 100644 --- a/proposals/run_ogcapi.json +++ b/proposals/run_ogcapi.json @@ -1,7 +1,7 @@ { "id": "run_ogcapi", "summary": "Run a OGC API process", - "description": "Runs an OGC API - Processes process that the service provider offers through an Application Deployment and Execution Service (ADES).", + "description": "Runs an OGC API - Processes process that the service provider offers through an instance of OGC API - Processes - Part 2 (Deploy, Replace, Undeploy).", "categories": [ "cubes", "import", From 4bf009455d6a2b85b6af45c5cf58eae3421d8b46 Mon Sep 17 00:00:00 2001 From: Matthias Mohr Date: Tue, 15 Oct 2024 11:11:11 +0200 Subject: [PATCH 6/8] Fix code blocks --- meta/implementation.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/meta/implementation.md b/meta/implementation.md index 67c4f0e6..4579981e 100644 --- a/meta/implementation.md +++ b/meta/implementation.md @@ -164,7 +164,7 @@ For the data types boolean, numbers, strings and null it is recommended to log t It is recommended to summarize arrays as follows: -```js +```json5 { "data": [3,1,6,4,8], // Return a reasonable excerpt of the data, e.g. the first 5 or 10 elements "length": 10, // Return the length of the array, this is important to determine whether the data above is complete or an excerpt @@ -179,7 +179,7 @@ It is recommended to return them summarized in a structure compliant to the [STA If reasonsable, it gives a valuable benefit for users to provide all dimension labels (e.g. individual timestamps for the temporal dimension) instead of values ranges. The top-level object and/or each dimension can be enhanced with additional statstics if possible, ideally use the corresponsing openEO process names as keys. -```js +```json5 { "cube:dimensions": { "x": { @@ -252,7 +252,7 @@ OGC API - Processes and OGC EO Application Packages (AP) can generally be utiliz Example process node: - ```json + ```json5 { "process_id": "run_ogcapi", "arguments": { @@ -272,7 +272,7 @@ OGC API - Processes and OGC EO Application Packages (AP) can generally be utiliz Example process node: - ```json + ```json5 { "process_id": "run_udf", "arguments": { From 751e22b9655ba75639701f74b1240b7fadf4d76f Mon Sep 17 00:00:00 2001 From: Matthias Mohr Date: Mon, 28 Oct 2024 15:31:55 +0100 Subject: [PATCH 7/8] Update meta/implementation.md --- meta/implementation.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/implementation.md b/meta/implementation.md index 4579981e..2a533ae4 100644 --- a/meta/implementation.md +++ b/meta/implementation.md @@ -290,4 +290,4 @@ OGC API - Processes and OGC EO Application Packages (AP) can generally be utiliz Generally, we recommend to use the following types and formats for the CWL inputs and outputs: - `type`: `File` or `File[]` depending on the capabilities of the CWL workflow -- `format`: For STAC inputs and outputs either `stac:item`, `stac:collection`, `stac:catalog`, `stac:itemcollection`, or `stac:stac` (i.e. and of the types before). +- `format`: For STAC inputs and outputs either `stac-item`, `stac-collection`, `stac-catalog`, `stac-itemcollection`, or `stac-stac` (i.e. and of the types before). From b8e17b490fa29b660c3d53d338e67c1608705409 Mon Sep 17 00:00:00 2001 From: Matthias Mohr Date: Tue, 29 Oct 2024 10:35:26 +0100 Subject: [PATCH 8/8] Update meta/implementation.md --- meta/implementation.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta/implementation.md b/meta/implementation.md index ad2900f8..f180968f 100644 --- a/meta/implementation.md +++ b/meta/implementation.md @@ -311,4 +311,4 @@ OGC API - Processes and OGC EO Application Packages (AP) can generally be utiliz Generally, we recommend to use the following types and formats for the CWL inputs and outputs: - `type`: `File` or `File[]` depending on the capabilities of the CWL workflow -- `format`: For STAC inputs and outputs either `stac-item`, `stac-collection`, `stac-catalog`, `stac-itemcollection`, or `stac-stac` (i.e. and of the types before). +- `format`: For STAC inputs and outputs either `stac-item`, `stac-collection`, `stac-catalog`, `stac-itemcollection`, or `stac-stac` (i.e. any of the types before).