diff --git a/.github/workflows/docker-publish.yml b/.github/workflows/docker-publish.yml index 4f768f3..f074aed 100644 --- a/.github/workflows/docker-publish.yml +++ b/.github/workflows/docker-publish.yml @@ -9,11 +9,11 @@ on: schedule: - cron: '43 1 * * *' push: - branches: [ "master" ] + branches: [ "main" ] # Publish semver tags as releases. tags: [ 'v*.*.*' ] pull_request: - branches: [ "master" ] + branches: [ "main" ] env: # Use docker.io for Docker Hub if empty diff --git a/README.md b/README.md index b67317e..638e703 100644 --- a/README.md +++ b/README.md @@ -25,7 +25,7 @@ The CDAB Test Suite is built automatically providing a docker image available pu # Getting Started -You can start now using the Test Suite following the [Getting Started guide](https://github.com/Terradue/cdab-testsuite/wiki) +You can start now using the Test Suite following the [Getting Started guide](https://github.com/esacdab/cdab-testsuite/wiki) # Software licenses diff --git a/Use Cases/Scenario 1 - NDVI Mapping/config.sample.yaml b/Use Cases/Scenario 1 - NDVI Mapping/config.sample.yaml deleted file mode 100644 index 24282fc..0000000 --- a/Use Cases/Scenario 1 - NDVI Mapping/config.sample.yaml +++ /dev/null @@ -1,502 +0,0 @@ -global: - docker_config: "docker-config.json" - country_shapefile_path: /usr/lib/cdab-client/App_Data/TM_WORLD_BORDERS-0.3/TM_WORLD_BORDERS-0.3 - # Test Mode (Download limited to 20MB) - # test_mode: true -# Service Providers definition -service_providers: - # Service Provider Name (used in the CLI target_name argument) - ApiHub: - # Maximum number of thread querying the target catalogue service in parallel - max_catalogue_thread: 5 - # Maximum number of thread downloading the target download service in parallel - max_download_thread: 2 - # Data Access - data: - # Entry Point (usually the URL to the catalogue) - url: https://scihub.copernicus.eu/apihub - # Credentials (usually username:password/apiKey) - credentials: username:password - catalogue: - # ONDA Catalogue collections' set definition - sets: - # ONDA has the Copernicus catalogue (see baselines definition) ... - Copernicus: - # Identifier of the cxatalogue set configured in the data section - reference_set_id: Copernicus - # ... as a baseline (used in TC5O1, TC502, TC601 & TC602)... - type: baseline # The catalogue collections' set of type baseline are referering to common baseline defined globally - # ... with ApiHub as a reference ... - reference_target_site: ApiHub - # ... and with the following parameters to filter out only baseline - ONDA: - max_catalogue_thread: 5 - max_download_thread: 2 - data: - url: https://catalogue.onda-dias.eu/dias-catalogue/ - credentials: username:password - class: ONDA - # Catalogue specific configuration - catalogue: - # ONDA Catalogue collections' set definition - sets: - # ONDA has the Copernicus catalogue (see baselines definition) ... - Copernicus: - # Identifier of the cxatalogue set configured in the data section - reference_set_id: Copernicus - # ... as a baseline (used in TC5O1, TC502, TC601 & TC602)... - type: baseline # The catalogue collections' set of type baseline are referering to common baseline defined globally - # ... with ApiHub as a reference ... - reference_target_site: ApiHub - # ... and with the following parameters to filter out only baseline - parameters: # All Copernicus products are baseline. Just remove the very recent one to avoid newly ingested bias - - key: sensingStop - full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}end" - value: "[NOW-1D]" - label: "before yesterday" - # and the Sentinel-5P (see baselines definition) ... - Copernicus-S5P: - # ... as a baseline (used in TC5O1, TC502, TC601 & TC602)... - type: baseline # The catalogue collections' set of type baseline are referering to common baseline defined globally - # ... with Sentinel-5P pre-Ops Hub as a reference - reference_target_site: S5PHub - # ... and with the following parameters to filter out only baseline - parameters: # All Copernicus products are baseline. Just remove the very recent one to avoid newly ingested bias - - key: sensingStop - full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}end" - value: "[NOW-1D]" - label: "before yesterday" - # Onda has a data offering... - DataOffer: - # ... defined as locally managed (TC503) - type: local - # ... with ApiHub as a reference - reference_target_site: ApiHub - # collections' set definition - collections: - Sentinel2-MSI1C: - label: "Sentinel-2 Level-1C" - parameters: - - key: missionName - full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" - value: "Sentinel-2" - label: "Sentinel-2" - - key: productType - full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" - value: "S2MSI1C" - label: "Level-1C" - parameters: # Local data offer common filters. Just remove the very recent one to avoid newly ingested bias - - key: sensingStop - full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}end" - value: "[NOW-1D]" - label: "before yesterday" - # Compute Services Access. For TS1X - compute: - # Authentication access point (obtain value from auth_url key in clouds.yaml). - auth_url: https://auth.cloud.ovh.net/v3 - # Cloud username (same username as for access to the OpenStack dashboard). - username: "login" - # Cloud password (password for user). - # password: "" - # Project ID (obtain value from project_id key in clouds.yaml). - # This setting is optional. - project_id: d6fae7e318114c248623562c0adce408 - # Project name (obtain value from project_name key in clouds.yaml). - project_name: "6611565740751529" - # User domain name (obtain value from user_domain_name key in clouds.yaml). - user_domain_name: "Default" - # Authentication region name (obtain value from region_name key in clouds.yaml). - # This setting is optional. - region_name: "GRA7" - # Interface (obtain value from interface key in clouds.yaml). - interface: "public" - # Identity API version (obtain value from identity_api_version key in clouds.yaml). - identity_api_version: 3 - # Preferred name of virtual machines to be created (sequential number is appended). - vm_name: "cdab-test-onda" - # Name of predefined public key for SSH connection to new virtual machine - # (key pairs can be created on the OpenStack dashboard and the private key can be downloaded, - # check under Compute > Key Pairs on the OpenStack dashboard). - key_name: "cdab-key" - # Name of image to be used for new virtual machine - # (choose from Compute > Images on the OpenStack dashboard). - image_name: "Centos 7" - # Name of flavour for new virtual machine - # (check under Compute > Instances > Launch Instance > Flavours/Flavors on the OpenStack dashboard). - flavor_name: "b2-7" - # Location of the private key file for SSH connections to virtual machine - # (must correspond to public key in key_name). - private_key_file: "cdab-key-onda.pem" - # User on virtual machine for SSH connections. - remote_user: "centos" - CREO: - max_catalogue_thread: 5 - max_download_thread: 5 - class: CREO - data: - url: https://finder.creodias.eu/resto/api/collections/describe.xml - credentials: username:password - catalogue: - # CREODias Catalogue collections' set definition - sets: - # CREODias has the Copernicus catalogue (see baselines definition) ... - Copernicus: - # Identifier of the cxatalogue set configured in the data section - reference_set_id: Copernicus - # ... as a baseline (used in TC5O1, TC502, TC601 & TC602)... - type: baseline # The catalogue collections' set of type baseline are referering to common baseline defined globally - # ... with ApiHub as a reference ... - reference_target_site: ApiHub - # ... and with the following parameters to filter out only baseline - parameters: # All Copernicus products are baseline. Just remove the very recent one to avoid newly ingested bias - - key: sensingStop - full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}end" - value: "[NOW-1D]" - label: "before yesterday" - # ESA-DATASET filter - - key: dataset - full_name: "{http://creodias.eu/namespace/sentinel}dataset" - value: "ESA-DATASET" - label: "ESA Baseline" - # exact count for creodias - - key: exactCount - full_name: "{http://mapshup.info/-/resto/2.0/}exactCount" - value: "1" - label: "Exact count" - # and the Sentinel-5P (see baselines definition) ... - Copernicus-S5P: - # ... as a baseline (used in TC5O1, TC502, TC601 & TC602)... - type: baseline # The catalogue collections' set of type baseline are referering to common baseline defined globally - # ... with Sentinel-5P pre-Ops Hub as a reference - reference_target_site: S5PHub - # ... and with the following parameters to filter out only baseline - parameters: # All Copernicus products are baseline. Just remove the very recent one to avoid newly ingested bias - - key: sensingStop - full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}end" - value: "[NOW-1D]" - label: "before yesterday" - # ESA-DATASET filter - - key: dataset - full_name: "{http://creodias.eu/namespace/sentinel}dataset" - value: "ESA-DATASET" - label: "ESA Baseline" - # exact count for creodias - - key: exactCount - full_name: "{http://mapshup.info/-/resto/2.0/}exactCount" - value: "1" - label: "Exact count" - # CREODias has a data offering... - DataOffer: - # ... defined as locally managed (TC503) - type: local - # ... with ApiHub as a reference - reference_target_site: ApiHub - # collections' set definition - parameters: - # Parameters to filter only locally managed products (for TC503) - # ESA-DATASET filter - - key: dataset - full_name: "{http://creodias.eu/namespace/sentinel}dataset" - value: "ESA-DATASET" - label: "ESA Baseline" - # exact count for creodias - - key: exactCount - full_name: "{http://mapshup.info/-/resto/2.0/}exactCount" - value: "1" - label: "Exact count" - collections: - Sentinel2-MSI1C: - label: "Sentinel-2 MSI Level-1C World" - parameters: - - key: missionName - full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" - value: "Sentinel-2" - label: "Sentinel-2" - - key: productType - full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" - value: "S2MSI1C" - label: "S2MSI1C" - compute: - auth_url: https://cf2.cloudferro.com:5000/v3 - username: "login" - # password: "" - project_id: 7236455993264b0c8970d22751b23562 - project_name: "cloud_07455 project_without_eo" - user_domain_name: "cloud_07455" - region_name: "RegionOne" - interface: "public" - identity_api_version: 3 - vm_name: "cdab-test-creo" - key_name: "cdab-key" - image_name: "CentOS 7" - flavor_name: "eo2.large" - security_group: "allow_ping_ssh_rdp" - floating_ip: True - private_key_file: "cdab-key-creodias.pem" - remote_user: "eouser" - storage: - auth_url: https://cf2.cloudferro.com:5000/v3 - username: "login" - # password: "" - project_id: 7236455993264b0c8970d22751b23562 - project_name: "cloud_07455 project_without_eo" - user_domain_name: "cloud_07455" - region_name: "RegionOne" - storage_name: "teststorage" - test_file: "path/to/testfile" - MUNDI: - max_catalogue_thread: 5 - max_download_thread: 2 - data: - url: https://mundiwebservices.com/acdc/catalog/proxy/search - credentials: username:password - s3_secret_key: "secret" - s3_key_id: id - class: MUNDI - catalogue: - # Mundi Catalogue collections' set definition - sets: - # Mundi has the Copernicus catalogue (see baselines definition) ... - Copernicus: - # Identifier of the cxatalogue set configured in the data section - reference_set_id: Copernicus - # ... as a baseline (used in TC5O1, TC502, TC601 & TC602)... - type: baseline # The catalogue collections' set of type baseline are referering to common baseline defined globally - # ... with ApiHub as a reference ... - reference_target_site: ApiHub - # ... and with the following parameters to filter out only baseline - parameters: # All Copernicus products are baseline. Just remove the very recent one to avoid newly ingested bias - - key: sensingStop - full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}end" - value: "[NOW-1D]" - label: "before yesterday" - # ESA-DATASET filter - - key: processingCenter - full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}processingCenter" - value: "ESA" - label: "ESA Baseline" - # and the Sentinel-5P (see baselines definition) ... - Copernicus-S5P: - # ... as a baseline (used in TC5O1, TC502, TC601 & TC602)... - type: baseline # The catalogue collections' set of type baseline are referering to common baseline defined globally - # ... with Sentinel-5P pre-Ops Hub as a reference - reference_target_site: S5PHub - # ... and with the following parameters to filter out only baseline - parameters: # All Copernicus products are baseline. Just remove the very recent one to avoid newly ingested bias - - - # ESA-DATASET filter - - key: processingCenter - full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}processingCenter" - value: "ESA" - label: "ESA Baseline" - # Mundi has a data offering... - DataOffer: - # ... defined as locally managed (TC503) - type: local - # ... with ApiHub as a reference - reference_target_site: ApiHub - parameters: - # ESA-DATASET filter - - key: processingCenter - full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}processingCenter" - value: "ESA" - label: "ESA Baseline" - # collections' set definition - collections: - Sentinel2-MSI1C-World: - label: "Sentinel-2 Level-1C World" - parameters: - - key: missionName - full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" - value: "Sentinel-2" - label: "Sentinel-2" - - key: productType - full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" - value: "S2MSI1C" - label: "Level-1C" - - key: sensingStart - full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}start" - value: "[NOW-12M]" - label: "Last 12 months" - Sentinel2-MSI1C-Europe: - label: "Sentinel-2 Level-1C Europe" - parameters: - - key: missionName - full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" - value: "Sentinel-2" - label: "Sentinel-2" - - key: productType - full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" - value: "S2MSI1C" - label: "Level-1C" - - key: geom - full_name: "{http://a9.com/-/opensearch/extensions/geo/1.0/}geometry" - value: "POLYGON((-10.547 36.173,-2.109 36.031,6.855 38.548,11.25 37.996,20.391 34.452,35.156 34.162,42.363 67.942,26.895 72.342,-26.016 67.136,-10.547 36.173))" - label: "Europe" - - key: sensingStart - full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}start" - value: "[NOW-48M]" - label: "Last 48 months" - Sentinel2-MSI2A-Europe: - label: "Sentinel-2 Level-2A Europe" - parameters: - - key: missionName - full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" - value: "Sentinel-2" - label: "Sentinel-2" - - key: productType - full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" - value: "S2MSI2A" - label: "Level-2A" - - key: geom - full_name: "{http://a9.com/-/opensearch/extensions/geo/1.0/}geometry" - value: "POLYGON((-10.547 36.173,-2.109 36.031,6.855 38.548,11.25 37.996,20.391 34.452,35.156 34.162,42.363 67.942,26.895 72.342,-26.016 67.136,-10.547 36.173))" - label: "Europe" - - key: sensingStart - full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}start" - value: "[NOW-48M]" - label: "Last 48 months" - compute: - auth_url: https://iam.eu-de.otc.t-systems.com:443/v3 - username: "login" - # password: "" - project_name: "eu-de" - user_domain_name: "OTC-EU-DE-00000000001000041106" - interface: "public" - identity_api_version: 3 - vm_name: "cdab-test-mundi" - key_name: "cdab-key" - image_name: "Standard_CentOS_7_latest" - flavor_name: "s2.large.4" - security_group: "default" - floating_ip: True - private_key_file: "cdab-key-mundi.pem" - remote_user: "linux" - storage: - auth_url: https://iam.eu-de.otc.t-systems.com:443/v3 - username: "login" - # password: "" - project_name: "eu-de" - user_domain_name: "OTC-EU-DE-00000000001000041106" - SOBLOO: - max_catalogue_thread: 5 - max_download_thread: 2 - data: - url: https://sobloo.eu/api/v1/services/search - credentials: username:apikey - class: SOBLOO - catalogue: - # Sobloo Catalogue collections' set definition - sets: - # Sobloo has the Copernicus catalogue (see baselines definition) ... - Copernicus: - # Identifier of the cxatalogue set configured in the data section - reference_set_id: Copernicus - # ... as a baseline (used in TC5O1, TC502, TC601 & TC602)... - type: baseline # The catalogue collections' set of type baseline are referering to common baseline defined globally - # ... with ApiHub as a reference ... - reference_target_site: ApiHub - # ... and with the following parameters to filter out only baseline - parameters: # All Copernicus products are baseline. Just remove the very recent one to avoid newly ingested bias - - key: sensingStop - full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}end" - value: "[NOW-1D]" - label: "before yesterday" - # and the Sentinel-5P (see baselines definition) ... - Copernicus-S5P: - # ... as a baseline (used in TC5O1, TC502, TC601 & TC602)... - type: baseline # The catalogue collections' set of type baseline are referering to common baseline defined globally - # ... with Sentinel-5P pre-Ops Hub as a reference - reference_target_site: S5PHub - # ... and with the following parameters to filter out only baseline - parameters: # All Copernicus products are baseline. Just remove the very recent one to avoid newly ingested bias - - key: sensingStop - full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}end" - value: "[NOW-1D]" - label: "before yesterday" - # Sobloo has a data offering... - DataOffer: - # ... defined as locally managed (TC503) - type: local - # ... with ApiHub as a reference - reference_target_site: ApiHub - # collections' set definition - collections: - Sentinel2-L1-Europe-Africa: - label: "Sentinel-2 MSI L1C over Europe and Africa" - parameters: - - key: missionName - full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" - value: "Sentinel-2" - label: "Sentinel-2" - - key: productType - full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" - value: "S2MSI1C" - label: "S2MSI1C" - - key: geometry - full_name: "{http://a9.com/-/opensearch/extensions/geo/1.0/}geometry" - value: "POLYGON((-24.961 16.973,-21.094 4.215,4.57 0.352,13.711 -34.016,21.445 -36.031,55.195 -24.847,54.141 16.973,41.484 11.178,34.453 31.653,41.484 68.528,26.367 72.396,-26.719 67.475,-11.25 36.315,-24.961 16.973))" - label: "over Europe & Africa" - Sentinel2-L1-World: - label: "Sentinel-2 MSI L1C World last 12 months" - parameters: - - key: missionName - full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" - value: "Sentinel-2" - label: "Sentinel-2" - - key: productType - full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" - value: "S2MSI1C" - label: "S2MSI1C" - - key: sensingStart - full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}start" - value: "[NOW-12M]" - label: "last 12 months" - Sentinel2-L2-Europe-Africa: - label: "Sentinel-2 MSI L2A over Europe and Africa" - parameters: - - key: missionName - full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" - value: "Sentinel-2" - label: "Sentinel-2" - - key: productType - full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" - value: "S2MSI2A" - label: "S2MSI2A" - - key: geometry - full_name: "{http://a9.com/-/opensearch/extensions/geo/1.0/}geometry" - value: "POLYGON((-24.961 16.973,-21.094 4.215,4.57 0.352,13.711 -34.016,21.445 -36.031,55.195 -24.847,54.141 16.973,41.484 11.178,34.453 31.653,41.484 68.528,26.367 72.396,-26.719 67.475,-11.25 36.315,-24.961 16.973))" - label: "over Europe & Africa" - Sentinel2-L1-World: - label: "Sentinel-2 MSI L2A World last 12 months" - parameters: - - key: missionName - full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" - value: "Sentinel-2" - label: "Sentinel-2" - - key: productType - full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" - value: "S2MSI2A" - label: "S2MSI2A" - - key: sensingStart - full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}start" - value: "[NOW-12M]" - label: "last 12 months" -data: - # This section defines all the logical sets of data - sets: - # Copernicus set (without S5P not yet considered as operational) - Copernicus: - name: "Copernicus Product Types" - collections: - Sentinel2-MSI1C: - label: "Sentinel-2 MSI Level-1C" - parameters: - - key: missionName - full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" - value: "Sentinel-2" - label: "Sentinel-2" - - key: productType - full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" - value: "S2MSI1C" - label: "S2MSI1C" \ No newline at end of file diff --git a/Use Cases/Scenario 1 - NDVI Mapping/integration.md b/Use Cases/Scenario 1 - NDVI Mapping/integration.md index 0f3d773..4d26457 100644 --- a/Use Cases/Scenario 1 - NDVI Mapping/integration.md +++ b/Use Cases/Scenario 1 - NDVI Mapping/integration.md @@ -21,7 +21,7 @@ If Python is not available, install it with the package manager and do point 3 again. ```console - $ sudo yum install python3 python3-pip + $ sudo apt-get install -y python3 python3-pip ``` 4. Check that GDAL is available. diff --git a/Use Cases/Scenario 2 - Rapid Mapping/conda-install.sh b/Use Cases/Scenario 2 - Rapid Mapping/conda-install.sh index 4749d23..a675d00 100644 --- a/Use Cases/Scenario 2 - Rapid Mapping/conda-install.sh +++ b/Use Cases/Scenario 2 - Rapid Mapping/conda-install.sh @@ -8,7 +8,6 @@ URL="https://github.com/conda-forge/miniforge/releases/download/${MINIFORGE_VERS INSTALLER_PATH=/tmp/miniforge-installer.sh # Make sure user's $HOME is not tampered with since this is run as root unset HOME -yum install -y wget wget --quiet $URL -O ${INSTALLER_PATH} chmod +x ${INSTALLER_PATH} # Check sha256 checksum diff --git a/Use Cases/Scenario 2 - Rapid Mapping/integration.md b/Use Cases/Scenario 2 - Rapid Mapping/integration.md index f3ad1c4..b987cad 100644 --- a/Use Cases/Scenario 2 - Rapid Mapping/integration.md +++ b/Use Cases/Scenario 2 - Rapid Mapping/integration.md @@ -4,7 +4,7 @@ 1. Provision a virtual machine on the target site. Preferably with the following specification - 2 CPUs, 8GB RAM, 30GB disk - - CentOS 7 + - Debian 11+ - Jupyter Lab (with Python 3 support), it can also be installed if not present. - With data offer access if required @@ -227,8 +227,8 @@ ```console $ ls -l ~ - -rw-rw-r--. 1 centos centos 16212154 Jun 24 17:37 burned_area_20210305_20210424.rgb.tif - -rw-rw-r--. 1 centos centos 130225905 Jun 24 17:38 burned_area_20210305_20210424.tif + -rw-rw-r--. 1 debian debian 16212154 Jun 24 17:37 burned_area_20210305_20210424.rgb.tif + -rw-rw-r--. 1 debian debian 130225905 Jun 24 17:38 burned_area_20210305_20210424.tif ``` 7. Download them to your computer and open it with any tool that can visualise TIFF files. Verify that the band/layer `NBR` shows an RGB image of the burn severity of the area of interest. [100%] diff --git a/Use Cases/Scenario 3 - Mosaicking/conda-install.sh b/Use Cases/Scenario 3 - Mosaicking/conda-install.sh index 06f4142..17d92bf 100644 --- a/Use Cases/Scenario 3 - Mosaicking/conda-install.sh +++ b/Use Cases/Scenario 3 - Mosaicking/conda-install.sh @@ -12,7 +12,7 @@ URL="https://github.com/conda-forge/miniforge/releases/download/${MINIFORGE_VERS INSTALLER_PATH=/tmp/miniforge-installer.sh # Make sure user's $HOME is not tampered with since this is run as root unset HOME -yum install -y wget +sudo apt-get install -y wget wget --quiet $URL -O ${INSTALLER_PATH} chmod +x ${INSTALLER_PATH} # Check sha256 checksum diff --git a/Use Cases/Scenario 3 - Mosaicking/integration.md b/Use Cases/Scenario 3 - Mosaicking/integration.md index 15295d5..0b730b4 100644 --- a/Use Cases/Scenario 3 - Mosaicking/integration.md +++ b/Use Cases/Scenario 3 - Mosaicking/integration.md @@ -5,7 +5,7 @@ 1. Provision a virtual machine on the target site. Preferably with the following specification - 2 CPUs, 8 GB RAM, 100 GB disk - - CentOS 7 + - Debian 11+ - Python 3 - With data offer access if available diff --git a/Use Cases/Scenario 4 - Trends Mapping/conda-install.sh b/Use Cases/Scenario 4 - Trends Mapping/conda-install.sh index 06f4142..17d92bf 100644 --- a/Use Cases/Scenario 4 - Trends Mapping/conda-install.sh +++ b/Use Cases/Scenario 4 - Trends Mapping/conda-install.sh @@ -12,7 +12,7 @@ URL="https://github.com/conda-forge/miniforge/releases/download/${MINIFORGE_VERS INSTALLER_PATH=/tmp/miniforge-installer.sh # Make sure user's $HOME is not tampered with since this is run as root unset HOME -yum install -y wget +sudo apt-get install -y wget wget --quiet $URL -O ${INSTALLER_PATH} chmod +x ${INSTALLER_PATH} # Check sha256 checksum diff --git a/Use Cases/Scenario 4 - Trends Mapping/integration.md b/Use Cases/Scenario 4 - Trends Mapping/integration.md index 2de77c3..134dae3 100644 --- a/Use Cases/Scenario 4 - Trends Mapping/integration.md +++ b/Use Cases/Scenario 4 - Trends Mapping/integration.md @@ -5,7 +5,7 @@ 1. Provision a virtual machine on the target site. Preferably with the following specification - 2 CPUs, 8 GB RAM, 100 GB disk - - CentOS 7 + - debian 7 - Python 3 - With data offer access if available @@ -191,12 +191,12 @@ ```console $ ls -l output_data/ - -rw-rw-r--. 1 centos centos 1702 May 31 16:40 graph_S3A_SL_2_LST____20210425T110535_20210425T110835_20210426T211759_0179_071_094_2160_LN2_O_NT_004.xml - -rw-rw-r--. 1 centos centos 1702 May 31 16:37 graph_S3B_SL_2_LST____20210425T102904_20210425T103204_20210426T210428_0179_051_336_2340_LN2_O_NT_004.xml - -rw-rw-r--. 1 centos centos 294126 May 31 16:42 lst_S3A_SL_2_LST____20210425T110535_20210425T110835_20210426T211759_0179_071_094_2160_LN2_O_NT_004.tif - -rw-rw-r--. 1 centos centos 925298 May 31 16:40 lst_S3B_SL_2_LST____20210425T102904_20210425T103204_20210426T210428_0179_051_336_2340_LN2_O_NT_004.tif - -rw-rw-r--. 1 centos centos 12339468 May 31 16:42 rgba_S3A_SL_2_LST____20210425T110535_20210425T110835_20210426T211759_0179_071_094_2160_LN2_O_NT_004.tif - -rw-rw-r--. 1 centos centos 10451496 May 31 16:40 rgba_S3B_SL_2_LST____20210425T102904_20210425T103204_20210426T210428_0179_051_336_2340_LN2_O_NT_004.tif + -rw-rw-r--. 1 debian debian 1702 May 31 16:40 graph_S3A_SL_2_LST____20210425T110535_20210425T110835_20210426T211759_0179_071_094_2160_LN2_O_NT_004.xml + -rw-rw-r--. 1 debian debian 1702 May 31 16:37 graph_S3B_SL_2_LST____20210425T102904_20210425T103204_20210426T210428_0179_051_336_2340_LN2_O_NT_004.xml + -rw-rw-r--. 1 debian debian 294126 May 31 16:42 lst_S3A_SL_2_LST____20210425T110535_20210425T110835_20210426T211759_0179_071_094_2160_LN2_O_NT_004.tif + -rw-rw-r--. 1 debian debian 925298 May 31 16:40 lst_S3B_SL_2_LST____20210425T102904_20210425T103204_20210426T210428_0179_051_336_2340_LN2_O_NT_004.tif + -rw-rw-r--. 1 debian debian 12339468 May 31 16:42 rgba_S3A_SL_2_LST____20210425T110535_20210425T110835_20210426T211759_0179_071_094_2160_LN2_O_NT_004.tif + -rw-rw-r--. 1 debian debian 10451496 May 31 16:40 rgba_S3B_SL_2_LST____20210425T102904_20210425T103204_20210426T210428_0179_051_336_2340_LN2_O_NT_004.tif ``` 7. Download the files `rgba_*.tif` onto your computer and open it with a tool such as QGIS. Verify that it shows the desired information (land surface temperature) of the area of interest. [100%] diff --git a/Use Cases/Scenario 5 - Interferogram Computation/conda-install.sh b/Use Cases/Scenario 5 - Interferogram Computation/conda-install.sh index 06f4142..17d92bf 100644 --- a/Use Cases/Scenario 5 - Interferogram Computation/conda-install.sh +++ b/Use Cases/Scenario 5 - Interferogram Computation/conda-install.sh @@ -12,7 +12,7 @@ URL="https://github.com/conda-forge/miniforge/releases/download/${MINIFORGE_VERS INSTALLER_PATH=/tmp/miniforge-installer.sh # Make sure user's $HOME is not tampered with since this is run as root unset HOME -yum install -y wget +sudo apt-get install -y wget wget --quiet $URL -O ${INSTALLER_PATH} chmod +x ${INSTALLER_PATH} # Check sha256 checksum diff --git a/Use Cases/Scenario 5 - Interferogram Computation/integration.md b/Use Cases/Scenario 5 - Interferogram Computation/integration.md index 44ea917..3458c3c 100644 --- a/Use Cases/Scenario 5 - Interferogram Computation/integration.md +++ b/Use Cases/Scenario 5 - Interferogram Computation/integration.md @@ -4,13 +4,13 @@ 1. Provision a virtual machine on the target site. Preferably with the following specification: - 4 CPU, 32GB RAM, 100GB disk - - CentOS 7 + - Debian 11+ or similar (Ubuntu) - With data offer access if required 2. Open a terminal on the provisioned machine and install some prequisites, in case they are not yet present on the machine. ``` - sudo yum install -y vim tree wget unzip libgfortran-4.8.5 + sudo apt-get install -y bc wget unzip libgfortran5-amd64-cross ``` 3. Install, if necessary, **conda** on the virtual machine and create the conda environment. Conda is needed as the vehicle to install the SNAP toolbox. diff --git a/cdab-testsuite.wiki b/cdab-testsuite.wiki index 7d23a07..d8c62aa 160000 --- a/cdab-testsuite.wiki +++ b/cdab-testsuite.wiki @@ -1 +1 @@ -Subproject commit 7d23a07ccab158b8de3a121d6363e3e12f4b4338 +Subproject commit d8c62aa7059616b397e5730e5fc55a527ea57a35 diff --git a/src/cdab-client/Target/TargetSiteWrapper.cs b/src/cdab-client/Target/TargetSiteWrapper.cs index e96b6f7..5d2e2f8 100644 --- a/src/cdab-client/Target/TargetSiteWrapper.cs +++ b/src/cdab-client/Target/TargetSiteWrapper.cs @@ -39,6 +39,8 @@ the licensors of this Program grant you additional permission to convey or distr using Terradue.OpenSearch.DataHub.MicrosoftPlanetaryComputer; // using Terradue.OpenSearch.Usgs; using System.Net.Http; +using Terradue.OpenSearch.Asf; +using Terradue.OpenSearch.Usgs; namespace cdabtesttools.Target { @@ -325,15 +327,15 @@ public static IDataHubSourceWrapper CreateDataAccessWrapper(TargetSiteConfigurat return wekeoDiasWrapper; } - // if (targetUri.Host == "api.daac.asf.alaska.edu") - // { - // return new AsfApiWrapper(targetUri, (NetworkCredential)targetCredentials); - // } + if (targetUri.Host == "api.daac.asf.alaska.edu") + { + return new AsfApiWrapper(targetUri, (NetworkCredential)targetCredentials); + } - // if (targetUri.Host == "m2m.cr.usgs.gov") - // { - // return new UsgsDataWrapper(new Uri(string.Format("https://m2m.cr.usgs.gov/api/api")), (NetworkCredential)targetCredentials); - // } + if (targetUri.Host == "m2m.cr.usgs.gov") + { + return new UsgsDataWrapper(new Uri(string.Format("https://m2m.cr.usgs.gov")), (NetworkCredential)targetCredentials); + } if (targetUri.Host.EndsWith("copernicus.eu") || targetUri.AbsolutePath.EndsWith("/dhus")) { @@ -349,8 +351,6 @@ public static IDataHubSourceWrapper CreateDataAccessWrapper(TargetSiteConfigurat if (targetUri.Host.EndsWith("amazon.com")) { - //var searchWrapper = new DHuSWrapper(new Uri("https://scihub.copernicus.eu/apihub"), (NetworkCredential)targetCredentials); - //var amazonWrapper = new AmazonOldWrapper(targetSiteConfig.Data.S3SecretKey, targetSiteConfig.Data.S3KeyId, searchWrapper); var amazonWrapper = new AmazonStacWrapper(targetSiteConfig.Data.S3SecretKey, targetSiteConfig.Data.S3KeyId, (NetworkCredential)targetCredentials); amazonWrapper.AllowOpenSearch = targetSiteConfig.Data.Catalogue.AllowOpenSearch; return amazonWrapper; @@ -358,8 +358,6 @@ public static IDataHubSourceWrapper CreateDataAccessWrapper(TargetSiteConfigurat if (targetUri.Host.EndsWith("googleapis.com") || targetUri.Host.EndsWith("google.com")) { - //var searchWrapper = new DHuSWrapper(new Uri("https://scihub.copernicus.eu/apihub"), (NetworkCredential)targetCredentials); - //var googleWrapper = new GoogleWrapper(targetSiteConfig.AccountFile, targetSiteConfig.ProjectId, searchWrapper); var googleWrapper = new GoogleWrapper(targetSiteConfig.AccountFile, targetSiteConfig.ProjectId, (NetworkCredential)targetCredentials, "https://cloud.google.com"); return googleWrapper; } diff --git a/src/cdab-client/cdab-client b/src/cdab-client/cdab-client index 8fd46cb..1dc5f0a 100755 --- a/src/cdab-client/cdab-client +++ b/src/cdab-client/cdab-client @@ -27,20 +27,8 @@ export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:/usr/lib/cdab-client if [[ -n "$@" && $@ == *"TS1"* ]]; then - newargs="$@" - newargs=${newargs/-tn/-ts} - newargs=${newargs/--target_name/-ts} - newargs=${newargs/--target_credentials/-tc} - newargs=${newargs/-tu/-te} - newargs=${newargs/-tsn/-n} - newargs=${newargs/--testsite_name/-n} - newargs=${newargs/--load_factor/-lf} - - newargs=`echo ${newargs} | sed -e 's/\( -\w*\)\ /\1=/g'` - - echo "Command: /usr/bin/cdab-remote-client $newargs" >&2 - - exec /usr/bin/cdab-remote-client $newargs + echo "Use cdab-remote-client for this test" >&2 + exit 1 fi exec /usr/lib/cdab-client/cdab-testtools "$@" diff --git a/src/cdab-client/cdab-client.csproj b/src/cdab-client/cdab-client.csproj index 4f3a02e..5acca21 100644 --- a/src/cdab-client/cdab-client.csproj +++ b/src/cdab-client/cdab-client.csproj @@ -21,13 +21,13 @@ - - + + - - + + diff --git a/src/cdab-client/config.sample.yaml b/src/cdab-client/config.sample.yaml index 318b328..af42e68 100644 --- a/src/cdab-client/config.sample.yaml +++ b/src/cdab-client/config.sample.yaml @@ -1,752 +1,1263 @@ global: - docker_config: "docker-config.json" + # Reference target site for TS05 and TS06 + reference_target_site: cdse country_shapefile_path: /usr/lib/cdab-client/App_Data/TM_WORLD_BORDERS-0.3/TM_WORLD_BORDERS-0.3 + connect_retries: 60 + # Interval between initial SSH connections (in seconds, fraction also possible) + connect_interval: 1 + max_retention_hours: 12 + # Docker configuration file location in the container + docker_config: ./docker/config.json + # ca_certificate: "/home/jenkins/docker/ca.crt" # Test Mode (Download limited to 20MB) # test_mode: true -# Service Providers definition + # Scenario specific parameters + scenarios: + TS13: + timeout: 7200 + das_service_provider: cdse + # Maximum number of filters used in TC201 (including mission and product type) + #simple_filter_limit: 3 + # Maximum number of filters used in TC202 (including mission and product type) + #complex_filter_limit: 2 service_providers: # Service Provider Name (used in the CLI target_name argument) - ApiHub: - # Maximum number of thread querying the target catalogue service in parallel + # Copernicus Data Space Ecosystem (reference site) + cdse: max_catalogue_thread: 5 - # Maximum number of thread downloading the target download service in parallel - max_download_thread: 2 - # Data Access - data: - # Entry Point (usually the URL to the catalogue) - url: https://scihub.copernicus.eu/apihub - # Credentials (usually username:password/apiKey) - credentials: username:password - catalogue: - # ONDA Catalogue collections' set definition - sets: - # ONDA has the Copernicus catalogue (see baselines definition) ... - Copernicus: - # Identifier of the cxatalogue set configured in the data section - reference_set_id: Copernicus - # ... as a baseline (used in TC5O1, TC502, TC601 & TC602)... - type: baseline # The catalogue collections' set of type baseline are referering to common baseline defined globally - # ... with ApiHub as a reference ... - reference_target_site: ApiHub - # ... and with the following parameters to filter out only baseline - S5PHub: - # Maximum number of thread querying the target catalogue service in parallel - max_catalogue_thread: 5 - # Maximum number of thread downloading the target download service in parallel - max_download_thread: 2 - # Data Access + max_download_thread: 1 + max_upload_thread: 3 + max_download_size: 1395864371 data: - # Entry Point (usually the URL to the catalogue) - url: https://s5phub.copernicus.eu/dhus - # Credentials (usually username:password/apiKey) - credentials: s5pguest:s5pguest + url: https://catalogue.dataspace.copernicus.eu/odata/v1 + credentials: username:secret + s3_key_id: s3_key_id + s3_secret_key: "secret" + s3_endpoint: "https://eodata.dataspace.copernicus.eu" catalogue: local_parameters: - # Parameters to filter only locally managed products (for TC301, TC502) + # Parameters to filter only locally managed products for TC502 and TC503 dataCoverage. + # If not present, only "Online" is considered. - key: "archiveStatus" full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}statusSubType" - value: "online" - label: "online" - # ONDA Catalogue collections' set definition - sets: - # S5PHub has the Sentinel-5P (see baselines definition) ... - Copernicus-S5P: - # ... as a baseline (used in TC5O1, TC502, TC601 & TC602)... - type: baseline # The catalogue collections' set of type baseline are referering to common baseline defined globally - # ... with Sentinel-5P pre-Ops Hub as a reference - reference_target_site: S5PHub - SciHub: - max_catalogue_thread: 1 - max_download_thread: 2 - data: - url: https://scihub.copernicus.eu/dhus - credentials: username:password - # Catalogue specific configuration - catalogue: - # CopHub Catalogue collections' set definition + value: "" + label: "All" + # Split coverage queries by years or months, to prevent timeouts + split_coverage_queries: true + coverage_months: 3 sets: - # CopHub has the Copernicus catalogue (see baselines definition) ... Copernicus: - # ... as a baseline (used in TC5O1, TC502, TC601 & TC602)... - type: baseline # The catalogue collections' set of type baseline are referering to common baseline defined globally - # ... with ApiHub as a reference ... - reference_target_site: ApiHub - # ... and with the following parameters to filter out only baseline - parameters: # None for ONDA. All Copernicus products are baseline - # and the Sentinel-5P (see baselines definition) ... + reference_set_id: Copernicus + type: baseline + reference_target_site: cdse + parameters: # All Copernicus products are baseline. Just remove the very recent one to avoid newly ingested bias + - key: sensingStop + full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}end" + value: "[NOW-1D]" + label: "before 1D" Copernicus-S5P: - # ... as a baseline (used in TC5O1, TC502, TC601 & TC602)... - type: baseline # The catalogue collections' set of type baseline are referering to common baseline defined globally - # ... with Sentinel-5P pre-Ops Hub as a reference - reference_target_site: S5PHub - # ... and with the following parameters to filter out only baseline - parameters: # None for ONDA. All S5P products are baseline - # Onda has a data offering... + reference_set_id: Copernicus-S5P + type: baseline + reference_target_site: cdse + parameters: # All Copernicus products are baseline. Just remove the very recent one to avoid newly ingested bias + - key: sensingStop + full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}end" + value: "[NOW-1D]" + label: "before 1D" DataOffer: - # ... defined as locally managed (TC503) type: local - # ... with ApiHub as a reference - reference_target_site: ApiHub - # collections' set definition + reference_target_site: cdse + parameters: [] collections: - Sentinel1: - label: "Sentinel-1 last year" + # https://documentation.dataspace.copernicus.eu/Data.html + Sentinel1-GRD: + label: "Sentinel-1 GRD World" parameters: - key: missionName full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" value: "Sentinel-1" label: "Sentinel-1" - - key: sensingStart - full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}start" - value: "[NOW-12M]" - label: "last 12 months" - Sentinel2: - label: "Sentinel-2 MSI last year" + - key: productType + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" + value: "GRD" + label: "GRD" + Sentinel1-OCN: + label: "Sentinel-1 OCN World" parameters: - key: missionName full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" - value: "Sentinel-2" - label: "Sentinel-2" - - key: sensingStart - full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}start" - value: "[NOW-12M]" - label: "last 12 months" - Sentinel3-NonTimeCritical: - label: "Sentinel-3 Non Time Critical last year" + value: "Sentinel-1" + label: "Sentinel-1" + - key: productType + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" + value: "OCN" + label: "OCN" + Sentinel1-SLC-Europe: + label: "Sentinel-1 SLC Europe" parameters: - key: missionName full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" - value: "Sentinel-3" - label: "Sentinel-3" - - key: timeliness - full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" - value: "NTC" - label: "Non Time Critical" - - key: sensingStart - full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}start" - value: "[NOW-12M]" - label: "last 12 months" - Sentinel3-NearRealTime: - label: "Sentinel-3 Near Real Time last month" + value: "Sentinel-1" + label: "Sentinel-1" + - key: productType + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" + value: "SLC" + label: "SLC" + - key: geom + full_name: "{http://a9.com/-/opensearch/extensions/geo/1.0/}geometry" + value: "POLYGON((-10.547 36.173,-2.109 36.031,6.855 38.548,11.25 37.996,20.391 34.452,35.156 34.162,42.363 67.942,26.895 72.342,-26.016 67.136,-10.547 36.173))" + label: "Europe" + Sentinel1-SLC-World: + label: "Sentinel-1 SLC World since 2021" parameters: - key: missionName full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" - value: "Sentinel-3" - label: "Sentinel-3" - - key: timeliness - full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" - value: "NRT" - label: "Near Real Time" + value: "Sentinel-1" + label: "Sentinel-1" + - key: productType + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" + value: "SLC" + label: "SLC" - key: sensingStart full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}start" - value: "[NOW-12M]" - label: "last 12 months" - Sentinel3-ShortTimeCritical: - label: "Sentinel-3 Short Time Critical last month" + value: "2021-02-01T00:00:00Z" + label: "since 2021" + Sentinel1-RAW: + label: "Sentinel-1 RAW since 2021" parameters: - key: missionName full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" - value: "Sentinel-3" - label: "Sentinel-3" - - key: timeliness - full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" - value: "STC" - label: "Short Time Critical" + value: "Sentinel-1" + label: "Sentinel-1" + - key: productType + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" + value: "RAW" + label: "RAW" - key: sensingStart full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}start" - value: "[NOW-12M]" - label: "last 12 months" - HellenicHub: - max_catalogue_thread: 1 - max_download_thread: 2 - data: - url: https://sentinels.space.noa.gr/dhus - credentials: username:password - # Catalogue specific configuration - catalogue: - # CopHub Catalogue collections' set definition - sets: - # CopHub has the Copernicus catalogue (see baselines definition) ... - Copernicus: - # ... as a baseline (used in TC5O1, TC502, TC601 & TC602)... - type: baseline # The catalogue collections' set of type baseline are referering to common baseline defined globally - # ... with ApiHub as a reference ... - reference_target_site: ApiHub - # ... and with the following parameters to filter out only baseline - parameters: # None for ONDA. All Copernicus products are baseline - # and the Sentinel-5P (see baselines definition) ... - Copernicus-S5P: - # ... as a baseline (used in TC5O1, TC502, TC601 & TC602)... - type: baseline # The catalogue collections' set of type baseline are referering to common baseline defined globally - # ... with Sentinel-5P pre-Ops Hub as a reference - reference_target_site: S5PHub - # ... and with the following parameters to filter out only baseline - parameters: # None for ONDA. All S5P products are baseline - # Onda has a data offering... - DataOffer: - # ... defined as locally managed (TC503) - type: local - # ... with ApiHub as a reference - reference_target_site: ApiHub - # collections' set definition - collections: - Sentinel1: - label: "Sentinel-1 last year" + value: "2021-01-01T00:00:00Z" + label: "since 2021" + Sentinel2-L1C: + label: "Sentinel-2 Level-1C" parameters: - key: missionName full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" - value: "Sentinel-1" - label: "Sentinel-1" + value: "Sentinel-2" + label: "Sentinel-2" + - key: productType + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" + value: "S2MSI1C" + label: "L1C" + Sentinel2-L2A-Europe: + label: "Sentinel-2 Level-2A Europe since Mar 2017" + parameters: + - key: missionName + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" + value: "Sentinel-2" + label: "Sentinel-2" + - key: productType + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" + value: "S2MSI2A" + label: "L2A" + - key: geom + full_name: "{http://a9.com/-/opensearch/extensions/geo/1.0/}geometry" + value: "POLYGON((-10.547 36.173,-2.109 36.031,6.855 38.548,11.25 37.996,20.391 34.452,35.156 34.162,42.363 67.942,26.895 72.342,-26.016 67.136,-10.547 36.173))" + label: "Europe" - key: sensingStart full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}start" - value: "[NOW-12M]" - label: "last 12 months" - Sentinel2: - label: "Sentinel-2 MSI last year" + value: "2017-03-01T00:00:00Z" + label: "since Mar 2017" + Sentinel2-L2A-World: + label: "Sentinel-2 Level-2A World since Dec 2021" parameters: - key: missionName full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" value: "Sentinel-2" label: "Sentinel-2" + - key: productType + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" + value: "S2MSI2A" + label: "L2A" - key: sensingStart full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}start" - value: "[NOW-12M]" - label: "last 12 months" - Sentinel3-NonTimeCritical: - label: "Sentinel-3 Non Time Critical last year" + value: "2021-12-01T00:00:00Z" + label: "since Dec 2021" + Sentinel3-OLCI-L1-NonTimeCritical: + label: "Sentinel-3 OLCI Level-1 Non Time Critical" parameters: - key: missionName full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" value: "Sentinel-3" label: "Sentinel-3" + - key: instrument + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}instrument" + value: "OLCI" + label: "OLCI" + - key: processingLevel + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}processingLevel" + value: "L1" + label: "L1" - key: timeliness full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" value: "NTC" - label: "Non Time Critical" - - key: sensingStart - full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}start" - value: "[NOW-12M]" - label: "last 12 months" - Sentinel3-NearRealTime: - label: "Sentinel-3 Near Real Time last month" + label: "NTC" + Sentinel3-OLCI-L1-NearRealTime-1Y: + label: "Sentinel-3 OLCI Level-1 Near Real Time last year" parameters: - key: missionName full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" value: "Sentinel-3" label: "Sentinel-3" + - key: instrument + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}instrument" + value: "OLCI" + label: "OLCI" + - key: processingLevel + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}processingLevel" + value: "L1" + label: "L1" - key: timeliness full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" value: "NRT" - label: "Near Real Time" + label: "NRT" - key: sensingStart full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}start" value: "[NOW-12M]" - label: "last 12 months" - Sentinel3-ShortTimeCritical: - label: "Sentinel-3 Short Time Critical last month" + label: "last 1Y" + # We need to filter out L2 OCEAN/WATER products which are not hosted on the reference + # CDSE, so we only consider LAND products (LFR and LRR) + Sentinel3-OLCI-L2-LFR-NonTimeCritical: + label: "Sentinel-3 OLCI Level-2 LFR Non Time Critical" parameters: - key: missionName full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" value: "Sentinel-3" label: "Sentinel-3" + - key: productType + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" + value: "OL_2_LFR___" + label: "OLCI L2 Land FR" - key: timeliness full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" - value: "STC" - label: "Short Time Critical" - - key: sensingStart - full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}start" - value: "[NOW-12M]" - label: "last 12 months" - IntHub: - max_catalogue_thread: 5 - max_download_thread: 2 - data: - url: https://inthub.copernicus.eu/dhus - credentials: username:password - # Catalogue specific configuration - catalogue: - # IntHub Catalogue collections' set definition - sets: - # IntHub has the Copernicus catalogue (see baselines definition) ... - Copernicus: - # ... as a baseline (used in TC5O1, TC502, TC601 & TC602)... - type: baseline # The catalogue collections' set of type baseline are referering to common baseline defined globally - # ... with ApiHub as a reference ... - reference_target_site: ApiHub - # ... and with the following parameters to filter out only baseline - parameters: # None for ONDA. All Copernicus products are baseline - # and the Sentinel-5P (see baselines definition) ... - Copernicus-S5P: - # ... as a baseline (used in TC5O1, TC502, TC601 & TC602)... - type: baseline # The catalogue collections' set of type baseline are referering to common baseline defined globally - # ... with Sentinel-5P pre-Ops Hub as a reference - reference_target_site: S5PHub - # ... and with the following parameters to filter out only baseline - parameters: # None for ONDA. All S5P products are baseline - # Onda has a data offering... - DataOffer: - # ... defined as locally managed (TC503) - type: local - # ... with ApiHub as a reference - reference_target_site: ApiHub - # collections' set definition - collections: - All: - label: "All 3 weeks" + value: "NTC" + label: "NTC" + Sentinel3-OLCI-L2-LRR-NonTimeCritical: + label: "Sentinel-3 OLCI Level-2 LRR Non Time Critical" parameters: - key: missionName full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" - value: "Sentinel-1" - label: "Sentinel-1" - - key: sensingStart - full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}start" - value: "[NOW-21D]" - label: "last 3 weeks" - ColHub: - max_catalogue_thread: 1 - max_download_thread: 2 - max_download_size: 2000 - data: - url: https://colhub.copernicus.eu/dhus - credentials: username:password - # Catalogue specific configuration - catalogue: - # ColHub Catalogue collections' set definition - sets: - # ColHub has the Copernicus catalogue (see baselines definition) ... - Copernicus: - # ... as a baseline (used in TC5O1, TC502, TC601 & TC602)... - type: baseline # The catalogue collections' set of type baseline are referering to common baseline defined globally - # ... with ApiHub as a reference ... - reference_target_site: ApiHub - # ... and with the following parameters to filter out only baseline - parameters: # None for ONDA. All Copernicus products are baseline - # and the Sentinel-5P (see baselines definition) ... - Copernicus-S5P: - # ... as a baseline (used in TC5O1, TC502, TC601 & TC602)... - type: baseline # The catalogue collections' set of type baseline are referering to common baseline defined globally - # ... with Sentinel-5P pre-Ops Hub as a reference - reference_target_site: S5PHub - # ... and with the following parameters to filter out only baseline - parameters: # None for ONDA. All S5P products are baseline - # Onda has a data offering... - DataOffer: - # ... defined as locally managed (TC503) - type: local - # ... with ApiHub as a reference - reference_target_site: ApiHub - # collections' set definition - collections: - Sentinel1: - label: "Sentinel-1 last year" + value: "Sentinel-3" + label: "Sentinel-3" + - key: productType + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" + value: "OL_2_LRR___" + label: "OLCI L2 Land RR" + - key: timeliness + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" + value: "NTC" + label: "NTC" + Sentinel3-OLCI-L2-LFR-NearRealTime-1Y: + label: "Sentinel-3 OLCI Level-2 LFR Near Real Time last year" parameters: - key: missionName full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" - value: "Sentinel-1" - label: "Sentinel-1" + value: "Sentinel-3" + label: "Sentinel-3" + - key: productType + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" + value: "OL_2_LFR___" + label: "OLCI L2 Land FR" + - key: timeliness + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" + value: "NRT" + label: "NRT" - key: sensingStart full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}start" value: "[NOW-12M]" - label: "last 12 months" - Sentinel2: - label: "Sentinel-2 MSI last year" + label: "last 1Y" + Sentinel3-OLCI-L2-LRR-NearRealTime-1Y: + label: "Sentinel-3 OLCI Level-2 LRR Near Real Time last year" parameters: - key: missionName full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" - value: "Sentinel-2" - label: "Sentinel-2" + value: "Sentinel-3" + label: "Sentinel-3" + - key: productType + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" + value: "OL_2_LRR___" + label: "OLCI L2 Land RR" + - key: timeliness + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" + value: "NRT" + label: "NRT" - key: sensingStart full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}start" value: "[NOW-12M]" - label: "last 12 months" - Sentinel3-NonTimeCritical: - label: "Sentinel-3 Non Time Critical last year" + label: "last 1Y" + Sentinel3-SLSTR-L1-NonTimeCritical: + label: "Sentinel-3 SLSTR Level-1 Non Time Critical" parameters: - key: missionName full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" value: "Sentinel-3" label: "Sentinel-3" + - key: instrument + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}instrument" + value: "SLSTR" + label: "SLSTR" + - key: processingLevel + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}processingLevel" + value: "L1" + label: "L1" - key: timeliness full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" value: "NTC" - label: "Non Time Critical" - - key: sensingStart - full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}start" - value: "[NOW-12M]" - label: "last 12 months" - Sentinel3-NearRealTime: - label: "Sentinel-3 Near Real Time last month" + label: "NTC" + Sentinel3-SLSTR-L1-NearRealTime-1Y: + label: "Sentinel-3 SLSTR Level-1 Near Real Time last year" parameters: - key: missionName full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" value: "Sentinel-3" label: "Sentinel-3" + - key: instrument + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}instrument" + value: "SLSTR" + label: "SLSTR" + - key: processingLevel + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}processingLevel" + value: "L1" + label: "L1" - key: timeliness full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" value: "NRT" - label: "Near Real Time" + label: "NRT" - key: sensingStart full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}start" value: "[NOW-12M]" - label: "last 12 months" - Sentinel3-ShortTimeCritical: - label: "Sentinel-3 Short Time Critical last month" + label: "last 1Y" + Sentinel3-SLSTR-L2-LST-NonTimeCritical: + label: "Sentinel-3 SLSTR Level-2 LST Non Time Critical" parameters: - key: missionName full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" value: "Sentinel-3" label: "Sentinel-3" + - key: productType + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" + value: "SL_2_LST___" + label: "SLSTR L2 Land" - key: timeliness full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" - value: "STC" - label: "Short Time Critical" - - key: sensingStart - full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}start" - value: "[NOW-12M]" - label: "last 12 months" - CopHub: - max_catalogue_thread: 5 - max_download_thread: 2 + value: "NTC" + label: "NTC" + Sentinel3-SLSTR-L2-FRP-NonTimeCritical: + label: "Sentinel-3 SLSTR Level-2 FRP Non Time Critical" + parameters: + - key: missionName + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" + value: "Sentinel-3" + label: "Sentinel-3" + - key: productType + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" + value: "SL_2_FRP___" + label: "SLSTR L2 FRP" + - key: timeliness + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" + value: "NTC" + label: "NTC" + Sentinel3-SLSTR-L2-LST-NearRealTime-1Y: + label: "Sentinel-3 SLSTR Level-2 LST Near Real Time last year" + parameters: + - key: missionName + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" + value: "Sentinel-3" + label: "Sentinel-3" + - key: productType + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" + value: "SL_2_LST___" + label: "SLSTR L2 Land" + - key: timeliness + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" + value: "NRT" + label: "NRT" + - key: sensingStart + full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}start" + value: "[NOW-12M]" + label: "last 1Y" + Sentinel3-SRAL-L1-NonTimeCritical: + label: "Sentinel-3 SRAL Level-1 Non Time Critical" + parameters: + - key: missionName + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" + value: "Sentinel-3" + label: "Sentinel-3" + - key: instrument + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}instrument" + value: "SRAL" + label: "SRAL" + - key: processingLevel + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}processingLevel" + value: "L1" + label: "L1" + - key: timeliness + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" + value: "NTC" + label: "NTC" + Sentinel3-SRAL-L1-ShortTimeCritical-1M: + label: "Sentinel-3 SRAL Level-1 Short Time Critical last month" + parameters: + - key: missionName + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" + value: "Sentinel-3" + label: "Sentinel-3" + - key: instrument + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}instrument" + value: "SRAL" + label: "SRAL" + - key: processingLevel + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}processingLevel" + value: "L1" + label: "L1" + - key: timeliness + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" + value: "STC" + label: "STC" + - key: sensingStart + full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}start" + value: "[NOW-1M]" + label: "last 1M" + Sentinel3-SRAL-L1-NearRealTime-1Y: + label: "Sentinel-3 SRAL Level-1 Near Real Time last year" + parameters: + - key: missionName + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" + value: "Sentinel-3" + label: "Sentinel-3" + - key: instrument + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}instrument" + value: "SRAL" + label: "SRAL" + - key: processingLevel + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}processingLevel" + value: "L1" + label: "L1" + - key: timeliness + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" + value: "NRT" + label: "NRT" + - key: sensingStart + full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}start" + value: "[NOW-12M]" + label: "last 1Y" + Sentinel3-SRAL-L2-LAN-NonTimeCritical: + label: "Sentinel-3 SRAL Level-2 Land Non Time Critical" + parameters: + - key: missionName + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" + value: "Sentinel-3" + label: "Sentinel-3" + - key: productType + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" + value: "SR_2_LAN___" + label: "SRAL L2 Land" + - key: timeliness + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" + value: "NTC" + label: "NTC" + Sentinel3-SRAL-L2-LAN-ShortTimeCritical-1M: + label: "Sentinel-3 SRAL Level-2 Land Short Time Critical last month" + parameters: + - key: missionName + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" + value: "Sentinel-3" + label: "Sentinel-3" + - key: productType + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" + value: "SR_2_LAN___" + label: "SRAL L2 Land" + - key: timeliness + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" + value: "STC" + label: "STC" + - key: sensingStart + full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}start" + value: "[NOW-1M]" + label: "last 1M" + Sentinel3-SRAL-L2-LAN-NearRealTime-1Y: + label: "Sentinel-3 SRAL Level-2 Land Near Real Time last year" + parameters: + - key: missionName + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" + value: "Sentinel-3" + label: "Sentinel-3" + - key: productType + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" + value: "SR_2_LAN___" + label: "SRAL L2 Land" + - key: timeliness + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" + value: "NRT" + label: "NRT" + - key: sensingStart + full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}start" + value: "[NOW-12M]" + label: "last 1Y" + Sentinel3-SYNERGY-L2-SYN: + label: "Sentinel-3 SYNERGY L2 SYN NTC" + parameters: + - key: missionName + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" + value: "Sentinel-3" + label: "Sentinel-3" + - key: productType + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" + value: "SY_2_SYN___" + label: "SY L2 SYN" + - key: timeliness + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" + value: "NTC" + label: "NTC" + Sentinel3-SYNERGY-L2-VGP: + label: "Sentinel-3 SYNERGY L2 VGP NTC" + parameters: + - key: missionName + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" + value: "Sentinel-3" + label: "Sentinel-3" + - key: productType + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" + value: "SY_2_VGP___" + label: "SY L2 VGP" + - key: timeliness + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" + value: "NTC" + label: "NTC" + Sentinel3-SYNERGY-L2-VG1: + label: "Sentinel-3 SYNERGY L2 VG1 NTC" + parameters: + - key: missionName + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" + value: "Sentinel-3" + label: "Sentinel-3" + - key: productType + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" + value: "SY_2_VG1___" + label: "SY L2 VG1" + - key: timeliness + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" + value: "NTC" + label: "NTC" + Sentinel3-SYNERGY-L2-V10: + label: "Sentinel-3 SYNERGY L2 V10 NTC" + parameters: + - key: missionName + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" + value: "Sentinel-3" + label: "Sentinel-3" + - key: productType + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" + value: "SY_2_V10___" + label: "SY L2 V10" + - key: timeliness + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" + value: "NTC" + label: "NTC" + Sentinel3-SYNERGY-L2-AOD: + label: "Sentinel-3 SYNERGY L2 AOD NTC" + parameters: + - key: missionName + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" + value: "Sentinel-3" + label: "Sentinel-3" + - key: productType + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" + value: "SY_2_AOD___" + label: "SY L2 AOD" + - key: timeliness + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" + value: "NTC" + label: "NTC" + # We gather all subtypes in only one STC definition, for 1 month only it's not worth the subdivision + Sentinel3-SYNERGY-L2-STC: + label: "Sentinel-3 SYNERGY L2 Short Time Critical" + parameters: + - key: missionName + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" + value: "Sentinel-3" + label: "Sentinel-3" + - key: instrument + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}instrument" + value: "SYNERGY" + label: "SYNERGY" + - key: processingLevel + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}processingLevel" + value: "L2" + label: "L2" + - key: timeliness + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" + value: "STC" + label: "STC" + DataOffer-S5P: + type: local + reference_target_site: cdse + parameters: [] + collections: + # This includes both Offline and Reprocessing. L1B NRT do not exist. + Sentinel5P-L1B: + label: "Sentinel-5P L1B" + parameters: + - key: missionName + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" + value: "Sentinel-5 Precursor" + label: "Sentinel-5P" + - key: processingLevel + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}processingLevel" + value: "L1B" + label: "L1B" + Sentinel5P-L2-Offline: + label: "Sentinel-5P L2 Offline" + parameters: + - key: missionName + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" + value: "Sentinel-5 Precursor" + label: "Sentinel-5P" + - key: processingLevel + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}processingLevel" + value: "L2" + label: "L2" + - key: processingMode + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}processingMode" + value: "Offline" + label: "Offline" + Sentinel5P-L2-Reprocessing: + label: "Sentinel-5P L2 Reprocessing" + parameters: + - key: missionName + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" + value: "Sentinel-5 Precursor" + label: "Sentinel-5P" + - key: processingLevel + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}processingLevel" + value: "L2" + label: "L2" + - key: processingMode + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}processingMode" + value: "Reprocessing" + label: "RPRO" + # Reference CDSE only keeps 1 month, so we cannot compare a longer time range anyway + Sentinel5P-L2-NRT: + label: "Sentinel-5P L2 NRT last month" + parameters: + - key: missionName + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" + value: "Sentinel-5 Precursor" + label: "Sentinel-5P" + - key: processingLevel + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}processingLevel" + value: "L2" + label: "L2" + - key: processingMode + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}processingMode" + value: "Near real time" + label: "NRT" + - key: sensingStart + full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}start" + value: "[NOW-1M]" + label: "last 1M" + compute: + auth_url: https://cf2.cloudferro.com:5000/v3 + username: "username" + password: "secret" + project_id: project_id + project_name: "project_name" + identity_provider: "creodias_new_provider" + protocol: openid + auth_type: v3oidcaccesstoken + project_domain_id: project_domain_id + project_domain_name: None + region_name: RegionOne + interface: public + identity_api_version: "3" + client_id: "openstack" + client_secret: "secret" + identity_provider_url: "https://identity.cloudferro.com/auth/realms/Creodias-new/protocol/openid-connect/token" + two_factor_authentication_key: "KEY" + vm_name: "cdab-test" + key_name: "cdab-cdse-vmkey" + image_name: "Debian 12" + flavor_name: "hm.medium" + cost_monthly: 70.04 + cost_hourly: 0.114 + currency: "EUR" + network_name: [ "cloud_00710_2" ] + security_group: "allow_ping_ssh_icmp_rdp" + floating_ip: True + floating_ip_network: "external3" + private_key_file: "/home/jenkins/cdab-key-cdse-ts.pem" + remote_user: "eouser" + # COPDAS via ODATA using T-Systems compute (for TS11 to TS15) + cdse-ts: + max_catalogue_thread: 5 + max_download_thread: 1 + max_upload_thread: 3 + # TODO: CHECK IF THE FOLLOWING IS APPLICABLE TO DAS, TOO + # Creodias metadata reports a size smaller than actual download size, see ticket + # https://portal.creodias.eu/viewticket.php?tid=766361&c=CLmnSIzQ and comments in CDAB-57; + # indeed the downloadable ZIP files are not compressed, while the reported size is related + # to the compressed data. + # So, maybe we should use a lower size limit for Creodias, because with 1GB limit we are + # downloading up to 1.7GB and more, but it doesn't seem to work, a lower limit blocks too many downloads. + max_download_size: 1395864371 data: - url: https://cophub.copernicus.eu/dhus - credentials: username:password - # Catalogue specific configuration + url: https://catalogue.dataspace.copernicus.eu/odata/v1 + credentials: user:secret + class: MUNDI catalogue: - # CopHub Catalogue collections' set definition + local_parameters: + # Parameters to filter only locally managed products for TC502 dataCoverage. + # If not present, only "Online" is considered. + - key: "archiveStatus" + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}statusSubType" + value: "" + label: "All" + # Split coverage queries by years or months, to prevent timeouts + split_coverage_queries: true + coverage_months: 3 sets: - # CopHub has the Copernicus catalogue (see baselines definition) ... Copernicus: - # ... as a baseline (used in TC5O1, TC502, TC601 & TC602)... - type: baseline # The catalogue collections' set of type baseline are referering to common baseline defined globally - # ... with ApiHub as a reference ... - reference_target_site: ApiHub - # ... and with the following parameters to filter out only baseline - parameters: # None for ONDA. All Copernicus products are baseline - # and the Sentinel-5P (see baselines definition) ... + reference_set_id: Copernicus + type: baseline + reference_target_site: cdse + parameters: # All Copernicus products are baseline. Just remove the very recent one to avoid newly ingested bias + - key: sensingStop + full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}end" + value: "[NOW-1D]" + label: "before 1D" Copernicus-S5P: - # ... as a baseline (used in TC5O1, TC502, TC601 & TC602)... - type: baseline # The catalogue collections' set of type baseline are referering to common baseline defined globally - # ... with Sentinel-5P pre-Ops Hub as a reference - reference_target_site: S5PHub - # ... and with the following parameters to filter out only baseline - parameters: # None for ONDA. All S5P products are baseline - # Onda has a data offering... + reference_set_id: Copernicus-S5P + type: baseline + reference_target_site: cdse + parameters: # All Copernicus products are baseline. Just remove the very recent one to avoid newly ingested bias + - key: sensingStop + full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}end" + value: "[NOW-1D]" + label: "before 1D" DataOffer: - # ... defined as locally managed (TC503) type: local - # ... with ApiHub as a reference - reference_target_site: ApiHub - # collections' set definition + reference_target_site: cdse + parameters: [] collections: - Sentinel1: - label: "Sentinel-1 last year" + # https://documentation.dataspace.copernicus.eu/Data.html + Sentinel1-GRD: + label: "Sentinel-1 GRD World" parameters: - key: missionName full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" value: "Sentinel-1" label: "Sentinel-1" + - key: productType + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" + value: "GRD" + label: "GRD" + Sentinel1-OCN: + label: "Sentinel-1 OCN World" + parameters: + - key: missionName + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" + value: "Sentinel-1" + label: "Sentinel-1" + - key: productType + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" + value: "OCN" + label: "OCN" + Sentinel1-SLC-Europe: + label: "Sentinel-1 SLC Europe" + parameters: + - key: missionName + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" + value: "Sentinel-1" + label: "Sentinel-1" + - key: productType + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" + value: "SLC" + label: "SLC" + - key: geom + full_name: "{http://a9.com/-/opensearch/extensions/geo/1.0/}geometry" + value: "POLYGON((-10.547 36.173,-2.109 36.031,6.855 38.548,11.25 37.996,20.391 34.452,35.156 34.162,42.363 67.942,26.895 72.342,-26.016 67.136,-10.547 36.173))" + label: "Europe" + Sentinel1-SLC-World: + label: "Sentinel-1 SLC World since 2021" + parameters: + - key: missionName + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" + value: "Sentinel-1" + label: "Sentinel-1" + - key: productType + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" + value: "SLC" + label: "SLC" + - key: sensingStart + full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}start" + value: "2021-02-01T00:00:00Z" + label: "since 2021" + Sentinel1-RAW: + label: "Sentinel-1 RAW since 2021" + parameters: + - key: missionName + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" + value: "Sentinel-1" + label: "Sentinel-1" + - key: productType + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" + value: "RAW" + label: "RAW" + - key: sensingStart + full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}start" + value: "2021-01-01T00:00:00Z" + label: "since 2021" + Sentinel2-L1C: + label: "Sentinel-2 Level-1C" + parameters: + - key: missionName + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" + value: "Sentinel-2" + label: "Sentinel-2" + - key: productType + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" + value: "S2MSI1C" + label: "L1C" + Sentinel2-L2A-Europe: + label: "Sentinel-2 Level-2A Europe since Mar 2017" + parameters: + - key: missionName + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" + value: "Sentinel-2" + label: "Sentinel-2" + - key: productType + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" + value: "S2MSI2A" + label: "L2A" + - key: geom + full_name: "{http://a9.com/-/opensearch/extensions/geo/1.0/}geometry" + value: "POLYGON((-10.547 36.173,-2.109 36.031,6.855 38.548,11.25 37.996,20.391 34.452,35.156 34.162,42.363 67.942,26.895 72.342,-26.016 67.136,-10.547 36.173))" + label: "Europe" + - key: sensingStart + full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}start" + value: "2017-03-01T00:00:00Z" + label: "since Mar 2017" + Sentinel2-L2A-World: + label: "Sentinel-2 Level-2A World since Dec 2021" + parameters: + - key: missionName + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" + value: "Sentinel-2" + label: "Sentinel-2" + - key: productType + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" + value: "S2MSI2A" + label: "L2A" + - key: sensingStart + full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}start" + value: "2021-12-01T00:00:00Z" + label: "since Dec 2021" + Sentinel3-OLCI-L1-NonTimeCritical: + label: "Sentinel-3 OLCI Level-1 Non Time Critical" + parameters: + - key: missionName + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" + value: "Sentinel-3" + label: "Sentinel-3" + - key: instrument + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}instrument" + value: "OLCI" + label: "OLCI" + - key: processingLevel + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}processingLevel" + value: "L1" + label: "L1" + - key: timeliness + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" + value: "NTC" + label: "NTC" + Sentinel3-OLCI-L1-NearRealTime-1Y: + label: "Sentinel-3 OLCI Level-1 Near Real Time last year" + parameters: + - key: missionName + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" + value: "Sentinel-3" + label: "Sentinel-3" + - key: instrument + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}instrument" + value: "OLCI" + label: "OLCI" + - key: processingLevel + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}processingLevel" + value: "L1" + label: "L1" + - key: timeliness + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" + value: "NRT" + label: "NRT" - key: sensingStart full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}start" value: "[NOW-12M]" - label: "last 12 months" - Sentinel2: - label: "Sentinel-2 MSI last year" + label: "last 1Y" + Sentinel3-OLCI-L2-LFR-NonTimeCritical: + label: "Sentinel-3 OLCI Level-2 LFR Non Time Critical" + parameters: + - key: missionName + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" + value: "Sentinel-3" + label: "Sentinel-3" + - key: productType + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" + value: "OL_2_LFR___" + label: "OLCI L2 Land FR" + - key: timeliness + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" + value: "NTC" + label: "NTC" + Sentinel3-OLCI-L2-LRR-NonTimeCritical: + label: "Sentinel-3 OLCI Level-2 LRR Non Time Critical" + parameters: + - key: missionName + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" + value: "Sentinel-3" + label: "Sentinel-3" + - key: productType + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" + value: "OL_2_LRR___" + label: "OLCI L2 Land RR" + - key: timeliness + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" + value: "NTC" + label: "NTC" + Sentinel3-OLCI-L2-LFR-NearRealTime-1Y: + label: "Sentinel-3 OLCI Level-2 LFR Near Real Time last year" + parameters: + - key: missionName + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" + value: "Sentinel-3" + label: "Sentinel-3" + - key: productType + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" + value: "OL_2_LFR___" + label: "OLCI L2 Land FR" + - key: timeliness + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" + value: "NRT" + label: "NRT" + - key: sensingStart + full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}start" + value: "[NOW-12M]" + label: "last 1Y" + Sentinel3-OLCI-L2-LRR-NearRealTime-1Y: + label: "Sentinel-3 OLCI Level-2 LRR Near Real Time last year" + parameters: + - key: missionName + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" + value: "Sentinel-3" + label: "Sentinel-3" + - key: productType + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" + value: "OL_2_LRR___" + label: "OLCI L2 Land RR" + - key: timeliness + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" + value: "NRT" + label: "NRT" + - key: sensingStart + full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}start" + value: "[NOW-12M]" + label: "last 1Y" + Sentinel3-SLSTR-L1-NonTimeCritical: + label: "Sentinel-3 SLSTR Level-1 Non Time Critical" + parameters: + - key: missionName + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" + value: "Sentinel-3" + label: "Sentinel-3" + - key: instrument + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}instrument" + value: "SLSTR" + label: "SLSTR" + - key: processingLevel + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}processingLevel" + value: "L1" + label: "L1" + - key: timeliness + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" + value: "NTC" + label: "NTC" + Sentinel3-SLSTR-L1-NearRealTime-1Y: + label: "Sentinel-3 SLSTR Level-1 Near Real Time last year" + parameters: + - key: missionName + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" + value: "Sentinel-3" + label: "Sentinel-3" + - key: instrument + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}instrument" + value: "SLSTR" + label: "SLSTR" + - key: processingLevel + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}processingLevel" + value: "L1" + label: "L1" + - key: timeliness + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" + value: "NRT" + label: "NRT" + - key: sensingStart + full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}start" + value: "[NOW-12M]" + label: "last 1Y" + Sentinel3-SLSTR-L2-LST-NonTimeCritical: + label: "Sentinel-3 SLSTR Level-2 LST Non Time Critical" + parameters: + - key: missionName + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" + value: "Sentinel-3" + label: "Sentinel-3" + - key: productType + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" + value: "SL_2_LST___" + label: "SLSTR L2 Land" + - key: timeliness + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" + value: "NTC" + label: "NTC" + Sentinel3-SLSTR-L2-FRP-NonTimeCritical: + label: "Sentinel-3 SLSTR Level-2 FRP Non Time Critical" + parameters: + - key: missionName + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" + value: "Sentinel-3" + label: "Sentinel-3" + - key: productType + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" + value: "SL_2_FRP___" + label: "SLSTR L2 FRP" + - key: timeliness + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" + value: "NTC" + label: "NTC" + Sentinel3-SLSTR-L2-LST-NearRealTime-1Y: + label: "Sentinel-3 SLSTR Level-2 LST Near Real Time last year" parameters: - key: missionName full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" - value: "Sentinel-2" - label: "Sentinel-2" + value: "Sentinel-3" + label: "Sentinel-3" + - key: productType + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" + value: "SL_2_LST___" + label: "SLSTR L2 Land" + - key: timeliness + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" + value: "NRT" + label: "NRT" - key: sensingStart full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}start" value: "[NOW-12M]" - label: "last 12 months" - Sentinel3-NonTimeCritical: - label: "Sentinel-3 Non Time Critical last year" + label: "last 1Y" + Sentinel3-SRAL-L1-NonTimeCritical: + label: "Sentinel-3 SRAL Level-1 Non Time Critical" parameters: - key: missionName full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" value: "Sentinel-3" label: "Sentinel-3" + - key: instrument + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}instrument" + value: "SRAL" + label: "SRAL" + - key: processingLevel + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}processingLevel" + value: "L1" + label: "L1" - key: timeliness full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" value: "NTC" - label: "Non Time Critical" - - key: sensingStart - full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}start" - value: "[NOW-12M]" - label: "last 12 months" - Sentinel3-NearRealTime: - label: "Sentinel-3 Near Real Time last month" + label: "NTC" + Sentinel3-SRAL-L1-ShortTimeCritical-1M: + label: "Sentinel-3 SRAL Level-1 Short Time Critical last month" parameters: - key: missionName full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" value: "Sentinel-3" label: "Sentinel-3" + - key: instrument + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}instrument" + value: "SRAL" + label: "SRAL" + - key: processingLevel + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}processingLevel" + value: "L1" + label: "L1" - key: timeliness full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" - value: "NRT" - label: "Near Real Time" + value: "STC" + label: "STC" - key: sensingStart full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}start" - value: "[NOW-12M]" - label: "last 12 months" - Sentinel3-ShortTimeCritical: - label: "Sentinel-3 Short Time Critical last month" + value: "[NOW-1M]" + label: "last 1M" + Sentinel3-SRAL-L1-NearRealTime-1Y: + label: "Sentinel-3 SRAL Level-1 Near Real Time last year" parameters: - key: missionName full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" value: "Sentinel-3" label: "Sentinel-3" + - key: instrument + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}instrument" + value: "SRAL" + label: "SRAL" + - key: processingLevel + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}processingLevel" + value: "L1" + label: "L1" - key: timeliness full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" - value: "STC" - label: "Short Time Critical" + value: "NRT" + label: "NRT" - key: sensingStart full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}start" value: "[NOW-12M]" - label: "last 12 months" - ASF: - max_catalogue_thread: 5 - max_download_thread: 2 - data: - url: https://api.daac.asf.alaska.edu - credentials: username:password - catalogue: - # ONDA Catalogue collections' set definition - sets: - # ONDA has the Copernicus catalogue (see baselines definition) ... - Sentinel1: - # Identifier of the cxatalogue set configured in the data section - reference_set_id: Sentinel1 - # ... as a baseline (used in TC5O1, TC502, TC601 & TC602)... - type: baseline # The catalogue collections' set of type baseline are referering to common baseline defined globally - # ... with ApiHub as a reference ... - reference_target_site: ApiHub - # ... and with the following parameters to filter out only baseline - ONDA: - max_catalogue_thread: 5 - max_download_thread: 2 - data: - url: https://catalogue.onda-dias.eu/dias-catalogue/ - credentials: username:password - class: ONDA - # Catalogue specific configuration - catalogue: - # ONDA Catalogue collections' set definition - sets: - # ONDA has the Copernicus catalogue (see baselines definition) ... - Copernicus: - # Identifier of the cxatalogue set configured in the data section - reference_set_id: Copernicus - # ... as a baseline (used in TC5O1, TC502, TC601 & TC602)... - type: baseline # The catalogue collections' set of type baseline are referering to common baseline defined globally - # ... with ApiHub as a reference ... - reference_target_site: ApiHub - # ... and with the following parameters to filter out only baseline - parameters: # All Copernicus products are baseline. Just remove the very recent one to avoid newly ingested bias - - key: sensingStop - full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}end" - value: "[NOW-1D]" - label: "before yesterday" - # and the Sentinel-5P (see baselines definition) ... - Copernicus-S5P: - # ... as a baseline (used in TC5O1, TC502, TC601 & TC602)... - type: baseline # The catalogue collections' set of type baseline are referering to common baseline defined globally - # ... with Sentinel-5P pre-Ops Hub as a reference - reference_target_site: S5PHub - # ... and with the following parameters to filter out only baseline - parameters: # All Copernicus products are baseline. Just remove the very recent one to avoid newly ingested bias - - key: sensingStop - full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}end" - value: "[NOW-1D]" - label: "before yesterday" - # Onda has a data offering... - DataOffer: - # ... defined as locally managed (TC503) - type: local - # ... with ApiHub as a reference - reference_target_site: ApiHub - # collections' set definition - collections: - Sentinel1-SLC: - label: "Sentinel-1 SLC" - parameters: - - key: missionName - full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" - value: "Sentinel-1" - label: "Sentinel-1" - - key: productType - full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" - value: "SLC" - label: "SLC" - Sentinel1-GRD: - label: "Sentinel-1 GRD" - parameters: - - key: missionName - full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" - value: "Sentinel-1" - label: "Sentinel-1" - - key: productType - full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" - value: "GRD" - label: "GRD" - Sentinel1-OCN: - label: "Sentinel-1 OCN" - parameters: - - key: missionName - full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" - value: "Sentinel-1" - label: "Sentinel-1" - - key: productType - full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" - value: "OCN" - label: "OCN" - Sentinel2-MSI1C: - label: "Sentinel-2 Level-1C" + label: "last 1Y" + Sentinel3-SRAL-L2-LAN-NonTimeCritical: + label: "Sentinel-3 SRAL Level-2 Land Non Time Critical" parameters: - key: missionName full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" - value: "Sentinel-2" - label: "Sentinel-2" + value: "Sentinel-3" + label: "Sentinel-3" - key: productType full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" - value: "S2MSI1C" - label: "Level-1C" - Sentinel2-MSI2A: - label: "Sentinel-2 Level-2A" + value: "SR_2_LAN___" + label: "SRAL L2 Land" + - key: timeliness + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" + value: "NTC" + label: "NTC" + Sentinel3-SRAL-L2-LAN-ShortTimeCritical-1M: + label: "Sentinel-3 SRAL Level-2 Land Short Time Critical last month" parameters: - key: missionName full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" - value: "Sentinel-2" - label: "Sentinel-2" + value: "Sentinel-3" + label: "Sentinel-3" - key: productType full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" - value: "S2MSI2A" - label: "Level-2A" - Sentinel3-OLCI-OL_2_WFR___: - label: "Sentinel-3 OLCI OL_2_WFR___ since May 2018" + value: "SR_2_LAN___" + label: "SRAL L2 Land" + - key: timeliness + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" + value: "STC" + label: "STC" + - key: sensingStart + full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}start" + value: "[NOW-1M]" + label: "last 1M" + Sentinel3-SRAL-L2-LAN-NearRealTime-1Y: + label: "Sentinel-3 SRAL Level-2 Land Near Real Time last year" parameters: - key: missionName full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" value: "Sentinel-3" label: "Sentinel-3" - - key: instrument - full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}instrument" - value: "OLCI" - label: "OLCI" - key: productType full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" - value: "OL_2_WFR___" - label: "OL_2_WFR___" + value: "SR_2_LAN___" + label: "SRAL L2 Land" - key: timeliness full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" - value: "NTC" - label: "Non Time Critical" + value: "NRT" + label: "NRT" - key: sensingStart full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}start" - value: "2018-05-01T00:00:00" - label: "since May 2018" - Sentinel3-OLCI-OL_2_WRR___: - label: "Sentinel-3 OLCI OL_2_WRR___ since May 2018" + value: "[NOW-12M]" + label: "last 1Y" + Sentinel3-SYNERGY-L2-SYN: + label: "Sentinel-3 SYNERGY L2 SYN NTC" parameters: - key: missionName full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" value: "Sentinel-3" label: "Sentinel-3" - - key: instrument - full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}instrument" - value: "OLCI" - label: "OLCI" - key: productType full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" - value: "OL_2_WRR___" - label: "OL_2_WRR___" + value: "SY_2_SYN___" + label: "SY L2 SYN" - key: timeliness full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" value: "NTC" - label: "Non Time Critical" - - key: sensingStart - full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}start" - value: "2018-05-01T00:00:00" - label: "since May 2018" - Sentinel3-OLCI-OL_1_EFR___: - label: "Sentinel-3 OLCI OL_1_EFR___" + label: "NTC" + Sentinel3-SYNERGY-L2-VGP: + label: "Sentinel-3 SYNERGY L2 VGP NTC" parameters: - key: missionName full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" value: "Sentinel-3" label: "Sentinel-3" - - key: instrument - full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}instrument" - value: "OLCI" - label: "OLCI" - key: productType full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" - value: "OL_1_EFR___" - label: "OL_1_EFR___" + value: "SY_2_VGP___" + label: "SY L2 VGP" - key: timeliness full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" value: "NTC" - label: "Non Time Critical" - Sentinel3-OLCI-OL_1_ERR___: - label: "Sentinel-3 OLCI OL_1_ERR___" + label: "NTC" + Sentinel3-SYNERGY-L2-VG1: + label: "Sentinel-3 SYNERGY L2 VG1 NTC" parameters: - key: missionName full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" value: "Sentinel-3" label: "Sentinel-3" - - key: instrument - full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}instrument" - value: "OLCI" - label: "OLCI" - key: productType full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" - value: "OL_1_ERR___" - label: "OL_1_ERR___" + value: "SY_2_VG1___" + label: "SY L2 VG1" - key: timeliness full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" value: "NTC" - label: "Non Time Critical" - Sentinel3-OLCI-OL_2_LFR___: - label: "Sentinel-3 OLCI OL_2_LFR___" + label: "NTC" + Sentinel3-SYNERGY-L2-V10: + label: "Sentinel-3 SYNERGY L2 V10 NTC" parameters: - key: missionName full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" value: "Sentinel-3" label: "Sentinel-3" - - key: instrument - full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}instrument" - value: "OLCI" - label: "OLCI" - key: productType full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" - value: "OL_2_LFR___" - label: "OL_2_LFR___" + value: "SY_2_V10___" + label: "SY L2 V10" - key: timeliness full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" value: "NTC" - label: "Non Time Critical" - Sentinel3-OLCI-OL_2_LRR_: - label: "Sentinel-3 OLCI OL_2_LRR_" + label: "NTC" + Sentinel3-SYNERGY-L2-AOD: + label: "Sentinel-3 SYNERGY L2 AOD NTC" parameters: - key: missionName full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" value: "Sentinel-3" label: "Sentinel-3" - - key: instrument - full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}instrument" - value: "OLCI" - label: "OLCI" - key: productType full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" - value: "OL_2_LRR_" - label: "OL_2_LRR_" + value: "SY_2_AOD___" + label: "SY L2 AOD" - key: timeliness full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" value: "NTC" - label: "Non Time Critical" - Sentinel3-SLSTR-SL_2_WST___: - label: "Sentinel-3 SLSTR SL_2_WST___ since May 2018" + label: "NTC" + # We gather all subtypes in only one STC definition, for 1 month only it's not worth the subdivision + Sentinel3-SYNERGY-L2-STC: + label: "Sentinel-3 SYNERGY L2 Short Time Critical" parameters: - key: missionName full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" @@ -754,60 +1265,228 @@ service_providers: label: "Sentinel-3" - key: instrument full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}instrument" - value: "SLSTR" - label: "SLSTR" - - key: productType - full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" - value: "SL_2_WST___" - label: "SL_2_WST___" + value: "SYNERGY" + label: "SYNERGY" + - key: processingLevel + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}processingLevel" + value: "L2" + label: "L2" - key: timeliness full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" - value: "NTC" - label: "Non Time Critical" + value: "STC" + label: "STC" + DataOffer-S5P: + type: local + reference_target_site: cdse + parameters: [] + collections: + # This includes both Offline and Reprocessing. L1B NRT do not exist. + Sentinel5P-L1B: + label: "Sentinel-5P L1B" + parameters: + - key: missionName + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" + value: "Sentinel-5 Precursor" + label: "Sentinel-5P" + - key: processingLevel + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}processingLevel" + value: "L1B" + label: "L1B" + Sentinel5P-L2-Offline: + label: "Sentinel-5P L2 Offline" + parameters: + - key: missionName + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" + value: "Sentinel-5 Precursor" + label: "Sentinel-5P" + - key: processingLevel + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}processingLevel" + value: "L2" + label: "L2" + - key: processingMode + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}processingMode" + value: "Offline" + label: "Offline" + Sentinel5P-L2-Reprocessing: + label: "Sentinel-5P L2 Reprocessing" + parameters: + - key: missionName + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" + value: "Sentinel-5 Precursor" + label: "Sentinel-5P" + - key: processingLevel + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}processingLevel" + value: "L2" + label: "L2" + - key: processingMode + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}processingMode" + value: "Reprocessing" + label: "RPRO" + # Reference s5phub only keeps 1 month, so we cannot compare a longer time range anyway + Sentinel5P-L2-NRT: + label: "Sentinel-5P L2 NRT last month" + parameters: + - key: missionName + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" + value: "Sentinel-5 Precursor" + label: "Sentinel-5P" + - key: processingLevel + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}processingLevel" + value: "L2" + label: "L2" + - key: processingMode + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}processingMode" + value: "Near real time" + label: "NRT" + - key: sensingStart + full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}start" + value: "[NOW-1M]" + label: "last 1M" + compute: + auth_url: https://iam.eu-de.otc.t-systems.com:443/v3 + username: "username" + password: "secret" + project_name: "eu-de" + user_domain_name: "user_domain_name" + interface: "public" + identity_api_version: 3 + vm_name: "cdab-test" + key_name: "cdab-cdse-vmkey" + image_name: "Standard_Debian_12_latest" + # From T-Systems Elastic Cloud Server creation wizard + # https://open-telekom-cloud.com/en/prices/price-calculator + flavor_name: "s3.large.8" + cost_monthly: 87.79 + cost_hourly: 0.1203 + currency: "EUR" + security_group: "default" + # As of 22/12/2021 we have 6 floating IPs (they get removed sometimes) + floating_ip: True + private_key_file: "/home/jenkins/cdab-key-cdse-ts.pem" + remote_user: "linux" + # 2022/12/20: we add a 100 GB volume because the image we use now only has 6 GB + use_volume: True + storage: + auth_url: https://iam.eu-de.otc.t-systems.com:443/v3 + username: "user" + password: "secret" + project_name: "eu-de" + user_domain_name: "user_domain_name" + min_upload_size: 10 + max_upload_size: 300 + onda: + # LIMITATION on simultaneous queries because of catalogue issues and errors "429 Too Many Requests" + max_catalogue_thread: 1 + max_download_thread: 5 + max_download_size: 1395864371 + max_upload_thread: 3 + data: + url: https://catalogue.onda-dias.eu/dias-catalogue/ + web_url: https://catalogue.onda-dias.eu/catalogue/ + credentials: user:secret + class: ONDA + catalogue: + local_parameters: + # Parameters to filter only locally managed products for TC502 dataCoverage + - key: "archiveStatus" + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}statusSubType" + value: "" + label: "All" + sets: + # ONDA has the Copernicus catalogue (see baselines definition) ... + Copernicus: + # Identifier of the catalogue set configured in the main data section + reference_set_id: Copernicus + # ... as a baseline (used in TC5O1, TC502, TC601 & TC602)... + type: baseline # The catalogue collections' set of type baseline are referering to common baseline defined globally + # ... with scihub as a reference ... + reference_target_site: cdse + Copernicus-S5P: + reference_set_id: Copernicus-S5P + type: baseline + reference_target_site: cdse + DataOffer: + # ... defined as locally managed (TC503 and TC602) + type: local + # ... with scihub as a reference + reference_target_site: cdse + # collections' set definition + # https://www.onda-dias.eu/cms/update-on-onda-data-offer + collections: + Sentinel1-RAW-NonTimeCritical: + label: "Sentinel-1 RAW" + parameters: + - key: missionName + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" + value: "Sentinel-1" + label: "Sentinel-1" + - key: productType + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" + value: "RAW" + label: "RAW" + # This gets overridden by the global "last 30D". If this is an issue, we can try moving the global + # filter at this level, repeated for all other collections. - key: sensingStart full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}start" - value: "2018-05-01T00:00:00" - label: "since May 2018" - Sentinel3-SLSTR-SL_1_RBT___: - label: "Sentinel-3 SLSTR SL_1_RBT___" + value: "[NOW-3D]" + label: "last 3D" + Sentinel1-SLC-NonTimeCritical: + label: "Sentinel-1 SLC" parameters: - key: missionName full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" - value: "Sentinel-3" - label: "Sentinel-3" - - key: instrument - full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}instrument" - value: "SLSTR" - label: "SLSTR" + value: "Sentinel-1" + label: "Sentinel-1" + - key: productType + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" + value: "SLC" + label: "SLC" + Sentinel1-GRD-NonTimeCritical: + label: "Sentinel-1 GRD" + parameters: + - key: missionName + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" + value: "Sentinel-1" + label: "Sentinel-1" + - key: productType + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" + value: "GRD" + label: "GRD" + Sentinel1-OCN-NonTimeCritical: + label: "Sentinel-1 OCN" + parameters: + - key: missionName + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" + value: "Sentinel-1" + label: "Sentinel-1" + - key: productType + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" + value: "OCN" + label: "OCN" + Sentinel2-L1C: + label: "Sentinel-2 Level-1C" + parameters: + - key: missionName + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" + value: "Sentinel-2" + label: "Sentinel-2" - key: productType full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" - value: "SL_1_RBT___" - label: "SL_1_RBT___" - - key: timeliness - full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" - value: "NTC" - label: "Non Time Critical" - Sentinel3-SLSTR-SL_2_LST___: - label: "Sentinel-3 SLSTR SL_2_LST___" + value: "S2MSI1C" + label: "L1C" + Sentinel2-L2A: + label: "Sentinel-2 Level-2A" parameters: - key: missionName full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" - value: "Sentinel-3" - label: "Sentinel-3" - - key: instrument - full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}instrument" - value: "SLSTR" - label: "SLSTR" + value: "Sentinel-2" + label: "Sentinel-2" - key: productType full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" - value: "SL_2_LST___" - label: "SL_2_LST___" - - key: timeliness - full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" - value: "NTC" - label: "Non Time Critical" - Sentinel3-SRAL-SR_2_WAT___: - label: "Sentinel-3 SRAL SR_2_WAT___ since May 2018" + value: "S2MSI2A" + label: "L2A" + Sentinel3-OLCI-L1-NonTimeCritical: + label: "Sentinel-3 OLCI Level-1 Non Time Critical" parameters: - key: missionName full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" @@ -815,60 +1494,48 @@ service_providers: label: "Sentinel-3" - key: instrument full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}instrument" - value: "SRAL" - label: "SRAL" - - key: productType - full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" - value: "SR_2_WAT___" - label: "SR_2_WAT___" + value: "OLCI" + label: "OLCI" + - key: processingLevel + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}processingLevel" + value: "L1" + label: "L1" - key: timeliness full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" value: "NTC" - label: "Non Time Critical" - - key: sensingStart - full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}start" - value: "2018-05-01T00:00:00" - label: "since May 2018" - Sentinel3-SRAL-SR_1_SRA___: - label: "Sentinel-3 SRAL SR_1_SRA___" + label: "NTC" + Sentinel3-OLCI-L2-LFR-NonTimeCritical: + label: "Sentinel-3 OLCI Level-2 LFR Non Time Critical" parameters: - key: missionName full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" value: "Sentinel-3" label: "Sentinel-3" - - key: instrument - full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}instrument" - value: "SRAL" - label: "SRAL" - key: productType full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" - value: "SR_1_SRA___" - label: "SR_1_SRA___" + value: "OL_2_LFR___" + label: "OLCI L2 Land FR" - key: timeliness full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" value: "NTC" - label: "Non Time Critical" - Sentinel3-SRAL-SR_1_SRA_A_: - label: "Sentinel-3 SRAL SR_1_SRA_A_" + label: "NTC" + Sentinel3-OLCI-L2-LRR-NonTimeCritical: + label: "Sentinel-3 OLCI Level-2 LRR Non Time Critical" parameters: - key: missionName full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" value: "Sentinel-3" label: "Sentinel-3" - - key: instrument - full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}instrument" - value: "SRAL" - label: "SRAL" - key: productType full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" - value: "SR_1_SRA_A_" - label: "SR_1_SRA_A_" + value: "OL_2_LRR___" + label: "OLCI L2 Land RR" - key: timeliness full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" value: "NTC" - label: "Non Time Critical" - Sentinel3-SRAL-SR_1_SRA_BS: - label: "Sentinel-3 SRAL SR_1_SRA_BS" + label: "NTC" + Sentinel3-SRAL-L1-NonTimeCritical: + label: "Sentinel-3 SRAL Level-1 Non Time Critical" parameters: - key: missionName full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" @@ -878,35 +1545,31 @@ service_providers: full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}instrument" value: "SRAL" label: "SRAL" - - key: productType - full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" - value: "SR_1_SRA_BS" - label: "SR_1_SRA_BS" + - key: processingLevel + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}processingLevel" + value: "L1" + label: "L1" - key: timeliness full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" value: "NTC" - label: "Non Time Critical" - Sentinel3-SRAL-SR_2_LAN___: - label: "Sentinel-3 SRAL SR_2_LAN___" + label: "NTC" + Sentinel3-SRAL-L2-LAN-NonTimeCritical: + label: "Sentinel-3 SRAL Level-2 Land Non Time Critical" parameters: - key: missionName full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" value: "Sentinel-3" label: "Sentinel-3" - - key: instrument - full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}instrument" - value: "SRAL" - label: "SRAL" - key: productType full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" value: "SR_2_LAN___" - label: "SR_2_LAN___" + label: "SRAL L2 Land" - key: timeliness full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" value: "NTC" - label: "Non Time Critical" - Sentinel3-SYNERGY-SY_2_SYN___: - label: "Sentinel-3 SRAL SY_2_SYN___" + label: "NTC" + Sentinel3-SLSTR-L1-NonTimeCritical: + label: "Sentinel-3 SLSTR Level-1 Non Time Critical" parameters: - key: missionName full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" @@ -914,37 +1577,33 @@ service_providers: label: "Sentinel-3" - key: instrument full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}instrument" - value: "SYNERGY" - label: "SYNERGY" - - key: productType - full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" - value: "SY_2_SYN___" - label: "SY_2_SYN___" + value: "SLSTR" + label: "SLSTR" + - key: processingLevel + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}processingLevel" + value: "L1" + label: "L1" - key: timeliness full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" value: "NTC" - label: "Non Time Critical" - Sentinel3-SYNERGY-SY_2_V10___: - label: "Sentinel-3 SRAL SY_2_V10___" + label: "NTC" + Sentinel3-SLSTR-L2-NonTimeCritical: + label: "Sentinel-3 SLSTR Level-2 Non Time Critical" parameters: - key: missionName full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" value: "Sentinel-3" label: "Sentinel-3" - - key: instrument - full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}instrument" - value: "SYNERGY" - label: "SYNERGY" - key: productType full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" - value: "SY_2_V10___" - label: "SY_2_V10___" + value: "SL_2_LST___" + label: "SLSTR L2 Land" - key: timeliness full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" value: "NTC" - label: "Non Time Critical" - Sentinel3-SYNERGY-SY_2_VG1___: - label: "Sentinel-3 SRAL SY_2_VG1___" + label: "NTC" + Sentinel3-SYNERGY-L2-NonTimeCritical: + label: "Sentinel-3 SYNERGY Level-2 Non Time Critical" parameters: - key: missionName full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" @@ -954,153 +1613,239 @@ service_providers: full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}instrument" value: "SYNERGY" label: "SYNERGY" - - key: productType - full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" - value: "SY_2_VG1___" - label: "SY_2_VG1___" + - key: processingLevel + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}processingLevel" + value: "L2" + label: "L2" - key: timeliness full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" value: "NTC" - label: "Non Time Critical" - Sentinel3-SYNERGY-SY_2_VGP___: - label: "Sentinel-3 SRAL SY_2_VGP___" + label: "NTC" + # TODO: exclude Marine products + Sentinel3-ShortTimeCritical: + label: "Sentinel-3 Short Time Critical" parameters: - key: missionName full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" value: "Sentinel-3" label: "Sentinel-3" - - key: instrument - full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}instrument" - value: "SYNERGY" - label: "SYNERGY" - - key: productType - full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" - value: "SY_2_VGP___" - label: "SY_2_VGP___" - key: timeliness full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" - value: "NTC" - label: "Non Time Critical" - parameters: # Local data offer common filters. Just remove the very recent one to avoid newly ingested bias - - key: sensingStop - full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}end" - value: "[NOW-1D]" - label: "before yesterday" + value: "STC" + label: "STC" + DataOffer-S5P: + type: local + reference_target_site: cdse + collections: + Sentinel5P-L1B: + label: "Sentinel-5P L1B" + parameters: + - key: missionName + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" + value: "Sentinel-5 Precursor" + label: "Sentinel-5P" + - key: processingLevel + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}processingLevel" + value: "L1B" + label: "L1B" + Sentinel5P-L2-Offline: + label: "Sentinel-5P L2 Offline" + parameters: + - key: missionName + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" + value: "Sentinel-5 Precursor" + label: "Sentinel-5P" + - key: processingLevel + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}processingLevel" + value: "L2" + label: "L2" + - key: processingMode + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}processingMode" + value: "Offline" + label: "Offline" + Sentinel5P-L2-NRT: + label: "Sentinel-5P L2 NRT" + parameters: + - key: missionName + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" + value: "Sentinel-5 Precursor" + label: "Sentinel-5P" + - key: processingLevel + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}processingLevel" + value: "L2" + label: "L2" + - key: processingMode + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}processingMode" + value: "Near real time" + label: "NRT" + Sentinel5P-L2-Reprocessing: + label: "Sentinel-5P L2 Reprocessing" + parameters: + - key: missionName + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" + value: "Sentinel-5 Precursor" + label: "Sentinel-5P" + - key: processingLevel + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}processingLevel" + value: "L2" + label: "L2" + - key: processingMode + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}processingMode" + value: "Reprocessing" + label: "RPRO" # Compute Services Access. For TS1X + # ONDA compute: # Authentication access point (obtain value from auth_url key in clouds.yaml). auth_url: https://auth.cloud.ovh.net/v3 # Cloud username (same username as for access to the OpenStack dashboard). - username: "login" + username: "username" # Cloud password (password for user). - # password: "" + password: "secret" # Project ID (obtain value from project_id key in clouds.yaml). # This setting is optional. - project_id: d6fae7e318114c248623562c0adce408 + project_id: project_id # Project name (obtain value from project_name key in clouds.yaml). - project_name: "6611565740751529" + project_name: "project_name" + # Project ID to be used for EO data (obtain value from `project_id` key in clouds.yaml). + #eo_project_id: + # Project name to be used for EO data (obtain value from `project_id` key in clouds.yaml). + #eo_project_name: # User domain name (obtain value from user_domain_name key in clouds.yaml). user_domain_name: "Default" # Authentication region name (obtain value from region_name key in clouds.yaml). # This setting is optional. - region_name: "GRA7" + region_name: "SBG5" # Interface (obtain value from interface key in clouds.yaml). interface: "public" # Identity API version (obtain value from identity_api_version key in clouds.yaml). identity_api_version: 3 # Preferred name of virtual machines to be created (sequential number is appended). - vm_name: "cdab-test-onda" + vm_name: "cdab-test" # Name of predefined public key for SSH connection to new virtual machine # (key pairs can be created on the OpenStack dashboard and the private key can be downloaded, # check under Compute > Key Pairs on the OpenStack dashboard). - key_name: "cdab-key" + key_name: "cdab-key-onda" # Name of image to be used for new virtual machine # (choose from Compute > Images on the OpenStack dashboard). - image_name: "Centos 7" + image_name: "Debian 12" # Name of flavour for new virtual machine # (check under Compute > Instances > Launch Instance > Flavours/Flavors on the OpenStack dashboard). - flavor_name: "b2-7" + # From https://www.onda-dias.eu/cms/services/catalogues/virtual-servers + # Hourly prices from https://www.onda-dias.eu/cms/faqs/#cloud-resources (no more available as of 20231002) + # r2-15 = 2 vCPU, 15GB RAM, 50GB HD + flavor_name: "r2-15" + # Monthly cost for VM of specified flavour. The default is 0. + # If there is more than one flavour, the value has to be an array of the same size. + # NB: Exprivia CDAB contract has higher prices due to Enterprise support. We report here the standard prices. + cost_monthly: 35.20 + # Hourly cost of VM of specified flavour. The default is 0. + # If there is more than one flavour, the value has to be an array of the same size. + cost_hourly: 0.09 + # Payment currency. The default is 'EUR' + currency: "EUR" + # Name of network to which new virtual machine is connected (check under Network > Networks on the OpenStack dashboard). + # This setting is optional. The value can also be an array, e.g. ['networkA', 'networkB'] + network_name: "Ext-Net" + # Name of specific network for EO data to which new VM is connected (check under Network > Networks on the OpenStack dashboard). + # This setting is optional. The value can also be an array, e.g. ['networkA', 'networkB'] + #eo_network_name: + # Name of security group for new virtual machine (optional; this setting might be necessary + # in order to permit remote access to virtual machines, + # check under Network > Security groups on the OpenStack dashboard, if available). This setting is optional. + #security_group: "default" + # Explicitly assign floating IP. Set this to True if public IP addresses are not assigned automatically + # at the creation of a virtual machine and otherwise to False, check under Network > Floating IPs on + # the OpenStack dashboard, if available. + # + # Automatic assignment is working on Onda. + # As of 12/02/2020, as far as I can see there is no fixed limit to the number of floating IPs on Ext-Net network. + #floating_ip: False + # Network from which to assign floating IP (if used). This setting is optional. + #floating_ip_network: "Ext-Net" # Location of the private key file for SSH connections to virtual machine # (must correspond to public key in key_name). - private_key_file: "cdab-key-onda.pem" + private_key_file: "/home/jenkins/cdab-key-onda.pem" # User on virtual machine for SSH connections. - remote_user: "centos" - CREO: - max_catalogue_thread: 5 - max_download_thread: 5 - class: CREO + remote_user: "debian" + #Create an external volume for docker image and test execution; this is useful for flavours that have very limited main disk. + # The size of the additional disk is 20 GB. + #use_volume: True + storage: + auth_url: https://auth.cloud.ovh.net/v3 + username: "username" + password: "secret" + project_id: project_id + project_name: "project_name" + user_domain_name: "Default" + #region_name: "SERCO-DIAS1" + min_upload_size: 10 + max_upload_size: 300 + creodias: + max_catalogue_thread: 1 + max_download_thread: 1 + max_upload_thread: 3 + max_download_size: 1395864371 + # Info on CreoDIAS collections: https://finder.creodias.eu/resto/collections.xml data: - url: https://finder.creodias.eu/resto/api/collections/describe.xml - credentials: username:password + url: https://datahub.creodias.eu/odata/v1 + web_url: https://finder.creodias.eu/ + # We authenticate using DAS/Copernicus credentials to avoid using 2FA (CDAB-537) + credentials: user:secret + class: CREO catalogue: - # CREODias Catalogue collections' set definition + split_coverage_queries: true + coverage_months: 3 sets: - # CREODias has the Copernicus catalogue (see baselines definition) ... Copernicus: - # Identifier of the cxatalogue set configured in the data section reference_set_id: Copernicus - # ... as a baseline (used in TC5O1, TC502, TC601 & TC602)... - type: baseline # The catalogue collections' set of type baseline are referering to common baseline defined globally - # ... with ApiHub as a reference ... - reference_target_site: ApiHub - # ... and with the following parameters to filter out only baseline + type: baseline + reference_target_site: cdse parameters: # All Copernicus products are baseline. Just remove the very recent one to avoid newly ingested bias - key: sensingStop full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}end" value: "[NOW-1D]" - label: "before yesterday" - # ESA-DATASET filter - - key: dataset - full_name: "{http://creodias.eu/namespace/sentinel}dataset" - value: "ESA-DATASET" - label: "ESA Baseline" - # exact count for creodias - - key: exactCount - full_name: "{http://mapshup.info/-/resto/2.0/}exactCount" - value: "1" - label: "Exact count" - # and the Sentinel-5P (see baselines definition) ... + label: "before 1D" + # It seems CREODIAS keeps all S5P NRTI products since mission start, while cdse only keeps them for ~1 month, + # so the CREODIAS collection is much larger. Copernicus-S5P: - # ... as a baseline (used in TC5O1, TC502, TC601 & TC602)... - type: baseline # The catalogue collections' set of type baseline are referering to common baseline defined globally - # ... with Sentinel-5P pre-Ops Hub as a reference - reference_target_site: S5PHub - # ... and with the following parameters to filter out only baseline + reference_set_id: Copernicus-S5P + type: baseline + reference_target_site: cdse parameters: # All Copernicus products are baseline. Just remove the very recent one to avoid newly ingested bias - key: sensingStop full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}end" value: "[NOW-1D]" - label: "before yesterday" - # ESA-DATASET filter - - key: dataset - full_name: "{http://creodias.eu/namespace/sentinel}dataset" - value: "ESA-DATASET" - label: "ESA Baseline" - # exact count for creodias - - key: exactCount - full_name: "{http://mapshup.info/-/resto/2.0/}exactCount" - value: "1" - label: "Exact count" - # CREODias has a data offering... + label: "before 1D" DataOffer: - # ... defined as locally managed (TC503) type: local - # ... with ApiHub as a reference - reference_target_site: ApiHub - # collections' set definition - parameters: - # Parameters to filter only locally managed products (for TC503) - # ESA-DATASET filter - - key: dataset - full_name: "{http://creodias.eu/namespace/sentinel}dataset" - value: "ESA-DATASET" - label: "ESA Baseline" - # exact count for creodias - - key: exactCount - full_name: "{http://mapshup.info/-/resto/2.0/}exactCount" - value: "1" - label: "Exact count" + reference_target_site: cdse + # Seems the dataset and exactCount parameters are not available in the new API + parameters: [] collections: + # https://creodias.eu/data-offer + Sentinel1-GRD: + label: "Sentinel-1 GRD World" + parameters: + - key: missionName + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" + value: "Sentinel-1" + label: "Sentinel-1" + - key: productType + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" + value: "GRD" + label: "GRD" + Sentinel1-OCN: + label: "Sentinel-1 OCN World" + parameters: + - key: missionName + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" + value: "Sentinel-1" + label: "Sentinel-1" + - key: productType + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" + value: "OCN" + label: "OCN" Sentinel1-RAW: label: "Sentinel-1 RAW World last 6 months" parameters: @@ -1115,9 +1860,9 @@ service_providers: - key: sensingStart full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}start" value: "[NOW-6M]" - label: "Last 6 months" - Sentinel1-GRD: - label: "Sentinel-1 GRD World" + label: "Last 6M" + Sentinel1-SLC-Europe: + label: "Sentinel-1 SLC Europe" parameters: - key: missionName full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" @@ -1125,10 +1870,14 @@ service_providers: label: "Sentinel-1" - key: productType full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" - value: "GRD" - label: "GRD" - Sentinel1-OCN: - label: "Sentinel-1 GRD World" + value: "SLC" + label: "SLC" + - key: geom + full_name: "{http://a9.com/-/opensearch/extensions/geo/1.0/}geometry" + value: "POLYGON((-10.547 36.173,-2.109 36.031,6.855 38.548,11.25 37.996,20.391 34.452,35.156 34.162,42.363 67.942,26.895 72.342,-26.016 67.136,-10.547 36.173))" + label: "Europe" + Sentinel1-SLC-World: + label: "Sentinel-1 SLC World last 6 months" parameters: - key: missionName full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" @@ -1136,165 +1885,340 @@ service_providers: label: "Sentinel-1" - key: productType full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" - value: "OCN" - label: "OCN" - Sentinel1-SLC-Europe: - label: "Sentinel-1 SLC Europe" + value: "SLC" + label: "SLC" + - key: sensingStart + full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}start" + value: "[NOW-6M]" + label: "Last 6M" + Sentinel2-L1C: + label: "Sentinel-2 Level-1C World" + parameters: + - key: missionName + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" + value: "Sentinel-2" + label: "Sentinel-2" + - key: productType + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" + value: "S2MSI1C" + label: "L1C" + Sentinel3-OLCI-L1-NonTimeCritical: + label: "Sentinel-3 OLCI Level-1 Non Time Critical" + parameters: + - key: missionName + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" + value: "Sentinel-3" + label: "Sentinel-3" + - key: instrument + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}instrument" + value: "OLCI" + label: "OLCI" + - key: processingLevel + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}processingLevel" + value: "L1" + label: "L1" + - key: timeliness + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" + value: "NTC" + label: "NTC" + Sentinel3-OLCI-L2-LFR-NonTimeCritical: + label: "Sentinel-3 OLCI Level-2 LFR Non Time Critical" + parameters: + - key: missionName + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" + value: "Sentinel-3" + label: "Sentinel-3" + - key: productType + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" + value: "OL_2_LFR___" + label: "OLCI L2 Land FR" + - key: timeliness + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" + value: "NTC" + label: "NTC" + Sentinel3-OLCI-L2-LRR-NonTimeCritical: + label: "Sentinel-3 OLCI Level-2 LRR Non Time Critical" + parameters: + - key: missionName + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" + value: "Sentinel-3" + label: "Sentinel-3" + - key: productType + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" + value: "OL_2_LRR___" + label: "OLCI L2 Land RR" + - key: timeliness + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" + value: "NTC" + label: "NTC" + Sentinel3-SRAL-L1-NonTimeCritical: + label: "Sentinel-3 SRAL Level-1 Non Time Critical" + parameters: + - key: missionName + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" + value: "Sentinel-3" + label: "Sentinel-3" + - key: instrument + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}instrument" + value: "SRAL" + label: "SRAL" + - key: processingLevel + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}processingLevel" + value: "L1" + label: "L1" + - key: timeliness + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" + value: "NTC" + label: "NTC" + Sentinel3-SRAL-L2-LAN-NonTimeCritical: + label: "Sentinel-3 SRAL Level-2 Land Non Time Critical" + parameters: + - key: missionName + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" + value: "Sentinel-3" + label: "Sentinel-3" + - key: productType + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" + value: "SR_2_LAN___" + label: "SRAL L2 Land" + - key: timeliness + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" + value: "NTC" + label: "NTC" + Sentinel3-SLSTR-L1-NonTimeCritical: + label: "Sentinel-3 SLSTR Level-1 Non Time Critical" + parameters: + - key: missionName + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" + value: "Sentinel-3" + label: "Sentinel-3" + - key: instrument + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}instrument" + value: "SLSTR" + label: "SLSTR" + - key: processingLevel + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}processingLevel" + value: "L1" + label: "L1" + - key: timeliness + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" + value: "NTC" + label: "NTC" + Sentinel3-SLSTR-L2-NonTimeCritical: + label: "Sentinel-3 SLSTR Level-2 Non Time Critical" + parameters: + - key: missionName + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" + value: "Sentinel-3" + label: "Sentinel-3" + - key: productType + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" + value: "SL_2_LST___" + label: "SLSTR L2 Land" + - key: timeliness + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" + value: "NTC" + label: "NTC" + # We count only S3 SYN NTC products because the reference scihub does not have RT data + Sentinel3-SYNERGY-L2-SYN: + label: "Sentinel-3 SYNERGY L2 SYN NTC" parameters: - key: missionName full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" - value: "Sentinel-1" - label: "Sentinel-1" + value: "Sentinel-3" + label: "Sentinel-3" - key: productType full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" - value: "SLC" - label: "SLC" - - key: geom - full_name: "{http://a9.com/-/opensearch/extensions/geo/1.0/}geometry" - value: "POLYGON((-10.547 36.173,-2.109 36.031,6.855 38.548,11.25 37.996,20.391 34.452,35.156 34.162,42.363 67.942,26.895 72.342,-26.016 67.136,-10.547 36.173))" - label: "Europe" - Sentinel1-SLC-World: - label: "Sentinel-1 SLC World last 6 months" + value: "SY_2_SYN___" + label: "SY L2 SYN" + - key: timeliness + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" + value: "NTC" + label: "NTC" + Sentinel3-SYNERGY-L2-VGP: + label: "Sentinel-3 SYNERGY L2 VGP NTC" parameters: - key: missionName full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" - value: "Sentinel-1" - label: "Sentinel-1" + value: "Sentinel-3" + label: "Sentinel-3" - key: productType full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" - value: "SLC" - label: "SLC" - - key: sensingStart - full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}start" - value: "[NOW-6M]" - label: "Last 6 months" - Sentinel2-MSI1C: - label: "Sentinel-2 MSI Level-1C World" + value: "SY_2_VGP___" + label: "SY L2 VGP" + - key: timeliness + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" + value: "NTC" + label: "NTC" + Sentinel3-SYNERGY-L2-VG1: + label: "Sentinel-3 SYNERGY L2 VG1 NTC" parameters: - key: missionName full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" - value: "Sentinel-2" - label: "Sentinel-2" + value: "Sentinel-3" + label: "Sentinel-3" - key: productType full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" - value: "S2MSI1C" - label: "S2MSI1C" - Sentinel3: - label: "Sentinel-3 World Non Time Critical" + value: "SY_2_VG1___" + label: "SY L2 VG1" + - key: timeliness + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" + value: "NTC" + label: "NTC" + Sentinel3-SYNERGY-L2-V10: + label: "Sentinel-3 SYNERGY L2 V10 NTC" parameters: - key: missionName full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" value: "Sentinel-3" label: "Sentinel-3" + - key: productType + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" + value: "SY_2_V10___" + label: "SY L2 V10" - key: timeliness full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" value: "NTC" - label: "Non Time Critical" + label: "NTC" + DataOffer-S5P: + type: local + reference_target_site: cdse + parameters: + - key: dataset + full_name: "{http://creodias.eu/namespace/sentinel}dataset" + value: "ESA-DATASET" + label: "ESA Baseline" + - key: exactCount + full_name: "{http://mapshup.info/-/resto/2.0/}exactCount" + value: "1" + label: "" + collections: + Sentinel5P-L1B: + label: "Sentinel-5P L1B" + parameters: + - key: missionName + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" + value: "Sentinel-5 Precursor" + label: "Sentinel-5P" + - key: processingLevel + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}processingLevel" + value: "L1B" + label: "L1B" + Sentinel5P-L2-Offline: + label: "Sentinel-5P L2 Offline" + parameters: + - key: missionName + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" + value: "Sentinel-5 Precursor" + label: "Sentinel-5P" + - key: processingLevel + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}processingLevel" + value: "L2" + label: "L2" + - key: processingMode + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}processingMode" + value: "Offline" + label: "Offline" + # Creodias actually keeps S5P L2 NRT forever but reference s5phub only keeps 1 month, so we cannot compare a longer time range + # 20231002: now reference is DAS and it seems to host old products, too. I remove the time range. + Sentinel5P-L2-NRT: + # label: "Sentinel-5P L2 NRT last month" + label: "Sentinel-5P L2 NRT" + parameters: + - key: missionName + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" + value: "Sentinel-5 Precursor" + label: "Sentinel-5P" + - key: processingLevel + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}processingLevel" + value: "L2" + label: "L2" + - key: processingMode + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}processingMode" + value: "Near real time" + label: "NRT" + # - key: sensingStart + # full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}start" + # value: "[NOW-1M]" + # label: "last 1M" + Sentinel5P-L2-Reprocessing: + label: "Sentinel-5P L2 Reprocessing" + parameters: + - key: missionName + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" + value: "Sentinel-5 Precursor" + label: "Sentinel-5P" + - key: processingLevel + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}processingLevel" + value: "L2" + label: "L2" + - key: processingMode + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}processingMode" + value: "Reprocessing" + label: "RPRO" + # CREODIAS compute: auth_url: https://cf2.cloudferro.com:5000/v3 - username: "login" - # password: "" - project_id: 7236455993264b0c8970d22751b23562 - project_name: "cloud_07455 project_without_eo" - user_domain_name: "cloud_07455" + username: "username" + password: "secret" + project_id: project_id + project_name: "project_name" + user_domain_name: "user_domain_name" region_name: "RegionOne" interface: "public" identity_api_version: 3 - vm_name: "cdab-test-creo" - key_name: "cdab-key" - image_name: "CentOS 7" - flavor_name: "eo2.large" + vm_name: "cdab-test" + key_name: "cdab-key-creodias" + image_name: "Debian 12" + # From https://creodias.eu/price-list + # eo2.large = 2 vCPU, 8GB RAM, 32GB HD, 51.57 EUR per month, 0.084 EUR per hour + # eo2.xlarge = 4 vCPU, 16GB RAM, 64GB HD, 103.15 EUR per month, 0.169 EUR per hour + # hm.medium = 2 vCPU, 16GB RAM, 64GB HD, 70.04 EUR per month, 0.114 EUR per hour + # It appears 8 GB RAM is not enough for TS13 + flavor_name: "hm.medium" + cost_monthly: 70.04 + cost_hourly: 0.114 + currency: "EUR" + network_name: [ "private_network_06973", "eodata" ] security_group: "allow_ping_ssh_rdp" + # https://creodias.eu/-/how-to-add-floating-ip-to-vm-from-external2-pool- floating_ip: True - private_key_file: "cdab-key-creodias.pem" + floating_ip_network: "external" + private_key_file: "/home/jenkins/cdab-key-creodias.pem" remote_user: "eouser" storage: auth_url: https://cf2.cloudferro.com:5000/v3 - username: "login" - # password: "" - project_id: 7236455993264b0c8970d22751b23562 - project_name: "cloud_07455 project_without_eo" - user_domain_name: "cloud_07455" + username: "username" + password: "secret" + project_id: project_id + project_name: "project_name" + user_domain_name: "user_domain_name" region_name: "RegionOne" - storage_name: "teststorage" - test_file: "path/to/testfile" - CODE_DE: + #storage_name: "teststorage" + #test_file: "path/to/testfile" + min_upload_size: 10 + max_upload_size: 300 + asf: max_catalogue_thread: 5 + # Cannot find any download limits on the ASF docs on website max_download_thread: 5 + max_download_size: 1395864371 data: - url: https://finder.code-de.org/resto/api/collections/describe.xml - credentials: username:password + url: https://api.daac.asf.alaska.edu + web_url: https://search.asf.alaska.edu/ + credentials: username:secret catalogue: - # CREODias Catalogue collections' set definition sets: - # CREODias has the Copernicus catalogue (see baselines definition) ... - Copernicus: - # Identifier of the cxatalogue set configured in the data section - reference_set_id: Copernicus - # ... as a baseline (used in TC5O1, TC502, TC601 & TC602)... - type: baseline # The catalogue collections' set of type baseline are referering to common baseline defined globally - # ... with ApiHub as a reference ... - reference_target_site: ApiHub - # ... and with the following parameters to filter out only baseline - parameters: # All Copernicus products are baseline. Just remove the very recent one to avoid newly ingested bias - - key: sensingStop - full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}end" - value: "[NOW-1D]" - label: "before yesterday" - # ESA-DATASET filter - - key: dataset - full_name: "{http://creodias.eu/namespace/sentinel}dataset" - value: "ESA-DATASET" - label: "ESA Baseline" - # exact count for creodias - - key: exactCount - full_name: "{http://mapshup.info/-/resto/2.0/}exactCount" - value: "1" - label: "Exact count" - # and the Sentinel-5P (see baselines definition) ... - Copernicus-S5P: - # ... as a baseline (used in TC5O1, TC502, TC601 & TC602)... - type: baseline # The catalogue collections' set of type baseline are referering to common baseline defined globally - # ... with Sentinel-5P pre-Ops Hub as a reference - reference_target_site: S5PHub - # ... and with the following parameters to filter out only baseline - parameters: # All Copernicus products are baseline. Just remove the very recent one to avoid newly ingested bias - - key: sensingStop - full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}end" - value: "[NOW-1D]" - label: "before yesterday" - # ESA-DATASET filter - - key: dataset - full_name: "{http://creodias.eu/namespace/sentinel}dataset" - value: "ESA-DATASET" - label: "ESA Baseline" - # exact count for creodias - - key: exactCount - full_name: "{http://mapshup.info/-/resto/2.0/}exactCount" - value: "1" - label: "Exact count" - # CREODias has a data offering... + Sentinel1: + reference_set_id: Sentinel1 + type: baseline + reference_target_site: cdse DataOffer: - # ... defined as locally managed (TC503) type: local - # ... with ApiHub as a reference - reference_target_site: ApiHub - # collections' set definition - parameters: - # Parameters to filter only locally managed products (for TC503) - # ESA-DATASET filter - - key: dataset - full_name: "{http://creodias.eu/namespace/sentinel}dataset" - value: "ESA-DATASET" - label: "ESA Baseline" - # exact count for creodias - - key: exactCount - full_name: "{http://mapshup.info/-/resto/2.0/}exactCount" - value: "1" - label: "Exact count" + reference_target_site: cdse collections: Sentinel1-RAW: - label: "Sentinel-1 RAW World last 6 months" + label: "Sentinel-1 RAW" parameters: - key: missionName full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" @@ -1304,23 +2228,8 @@ service_providers: full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" value: "RAW" label: "RAW" - - key: sensingStart - full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}start" - value: "[NOW-6M]" - label: "Last 6 months" - Sentinel1-GRD: - label: "Sentinel-1 GRD World" - parameters: - - key: missionName - full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" - value: "Sentinel-1" - label: "Sentinel-1" - - key: productType - full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" - value: "GRD" - label: "GRD" - Sentinel1-OCN: - label: "Sentinel-1 GRD World" + Sentinel1-SLC: + label: "Sentinel-1 SLC" parameters: - key: missionName full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" @@ -1328,10 +2237,10 @@ service_providers: label: "Sentinel-1" - key: productType full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" - value: "OCN" - label: "OCN" - Sentinel1-SLC-Europe: - label: "Sentinel-1 SLC Europe" + value: "SLC" + label: "SLC" + Sentinel1-GRD: + label: "Sentinel-1 GRD" parameters: - key: missionName full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" @@ -1339,14 +2248,10 @@ service_providers: label: "Sentinel-1" - key: productType full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" - value: "SLC" - label: "SLC" - - key: geom - full_name: "{http://a9.com/-/opensearch/extensions/geo/1.0/}geometry" - value: "POLYGON((-10.547 36.173,-2.109 36.031,6.855 38.548,11.25 37.996,20.391 34.452,35.156 34.162,42.363 67.942,26.895 72.342,-26.016 67.136,-10.547 36.173))" - label: "Europe" - Sentinel1-SLC-World: - label: "Sentinel-1 SLC World last 6 months" + value: "GRD" + label: "GRD" + Sentinel1-OCN: + label: "Sentinel-1 OCN" parameters: - key: missionName full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" @@ -1354,14 +2259,80 @@ service_providers: label: "Sentinel-1" - key: productType full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" - value: "SLC" - label: "SLC" - - key: sensingStart - full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}start" - value: "[NOW-6M]" - label: "Last 6 months" - Sentinel2-MSI1C: - label: "Sentinel-2 MSI Level-1C World" + value: "OCN" + label: "OCN" + google: + max_catalogue_thread: 5 + max_download_thread: 5 + max_download_size: 1395864371 + # Path to the Google Key Service Account File (https://console.developers.google.com/iam-admin/serviceaccounts) + account_file: ./account.json + # Google Project ID + project_id: "project_id" + data: + url: https://storage.googleapis.com + # Here in the case of google, use the cdse credentials to make the search + credentials: username:secret + class: GOOGLE + #s3_key_id: "secret" + # https://cloud.google.com/storage/docs/public-datasets + catalogue: + # TS03 parameters + # Mission (e.g. Sentinel-1) + systematic_search_mission: "Sentinel-2" + # Regular expression to validate correct mission in result items (via identifier) + systematic_search_mission_regex: "^S2.*" + # Product type (e.g. GRD) + systematic_search_product_type: "S2MSI1C" + # Regular expression to validate correct product type in result items (via identifier) + systematic_search_product_type_regex: "^S2._MSIL1C_.*" + # Search area geometry (WKT) + systematic_search_aoi_wkt: "POLYGON((9.75 47.55,13.04 47.51,12.74 48.12,13.86 48.75,12.70 49.38,12.05 50.35,10.11 50.56,9.05 50.01,9.10 49.56,9.73 49.73,10.50 48.96,9.75 47.55))" + # Search area description + systematic_search_aoi_description: "over Bavaria" + # Search period (days before current day) + systematic_search_days: 7 + # End TS03 parameters + # TS06 parameters + # enables polling + # latency_polling: true + # target polling interval in seconds + # latency_check_interval: 900 + # maximum duration of polling attempts in seconds + # latency_check_max_duration: 172800 + # End TS06 parameters + sets: + Google: + type: baseline + reference_set_id: Google + reference_target_site: cdse + parameters: + # Already taken into account in the reference set definition. + # - key: sensingStop + # full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}end" + # value: "[NOW-5D]" + # label: "before 5 days ago" + DataOffer: + # ... defined as locally managed (TC503 and TC602) + type: local + # ... with scihub as a reference + reference_target_site: cdse + # collections' set definition + # It looks like Google does not host very recent products so we need to use + # a "before 5 days" filter or maybe even "before 10 days". + collections: + Sentinel2-L1C: + # TS06 parameters + # enables polling + latency_polling: true + # target polling interval in seconds + latency_check_interval: 900 + # maximum duration of polling attempts in seconds + latency_check_max_duration: 259200 + # time offset (seconds) for recent product search + # latency_check_offset: 286400 + # End TS06 parameters + label: "Sentinel-2 Level-1C" parameters: - key: missionName full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" @@ -1370,113 +2341,131 @@ service_providers: - key: productType full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" value: "S2MSI1C" - label: "S2MSI1C" - Sentinel3: - label: "Sentinel-3 World Non Time Critical" - parameters: - - key: missionName - full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" - value: "Sentinel-3" - label: "Sentinel-3" - - key: timeliness - full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" - value: "NTC" - label: "Non Time Critical" - MUNDI: + label: "L1C" + - key: sensingStop + full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}end" + value: "[NOW-5D]" + label: "before 5D" + # Not applicable when we use BigQuery + # Sentinel2-L2A: + # label: "Sentinel-2 Level-2A" + # parameters: + # - key: missionName + # full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" + # value: "Sentinel-2" + # label: "Sentinel-2" + # - key: productType + # full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" + # value: "S2MSI2A" + # label: "L2A" + # - key: sensingStop + # full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}end" + # value: "[NOW-5D]" + # label: "before 5D" + # GOOGLE + compute: + connector: google + region_name: "europe-west1-b" + image_name: "product/google/debian12" + #flavor_name: "projects/modular-buffer-235216/zones/europe-west1-b/machineTypes/n1-highmem-2" + # n1-highmem-2 = 2 vCPU, 13GB RAM + # n2-highmem-2 = 2 vCPU, 16GB RAM + flavor_name: "n2-highmem-2" + vm_name: "cdab-test" + private_key_file: "/home/jenkins/cdab-key-google.pem" + remote_user: "cdab" + floating_ip: False + cost_monthly: 98.38 + cost_hourly: 0.1366 + currency: "EUR" + storage: + min_upload_size: 10 + max_upload_size: 300 + amazon: max_catalogue_thread: 5 - max_download_thread: 2 + max_download_thread: 5 + # Larger download size for Amazon, especially for TS03 + max_download_size: 2147483648 data: - url: https://mundiwebservices.com/acdc/catalog/proxy/search - credentials: username:password - s3_secret_key: "secret" - s3_key_id: id - class: MUNDI + url: https://aws.amazon.com + # sentinel-hub.com credentials + credentials: "username:secret" + class: AMAZON + # AWS credentials + s3_key_id: "s3_key_id" + s3_secret_key: "s3_secret_key" + # https://registry.opendata.aws/ catalogue: - # Mundi Catalogue collections' set definition + # Use CDSE for Sentinel-2 L1C + #allow_open_search: True + # TS03 parameters + systematic_search_mission: "Sentinel-2" + systematic_search_mission_regex: "^S2.*" + systematic_search_product_type: "S2MSI1C" + systematic_search_product_type_regex: "^S2._MSIL1C_.*" + systematic_search_aoi_wkt: "POLYGON((9.75 47.55,13.04 47.51,12.74 48.12,13.86 48.75,12.70 49.38,12.05 50.35,10.11 50.56,9.05 50.01,9.10 49.56,9.73 49.73,10.50 48.96,9.75 47.55))" + systematic_search_aoi_description: "over Bavaria" + systematic_search_days: 7 + # End TS03 parameters sets: - # Mundi has the Copernicus catalogue (see baselines definition) ... - Copernicus: - # Identifier of the cxatalogue set configured in the data section - reference_set_id: Copernicus - # ... as a baseline (used in TC5O1, TC502, TC601 & TC602)... - type: baseline # The catalogue collections' set of type baseline are referering to common baseline defined globally - # ... with ApiHub as a reference ... - reference_target_site: ApiHub - # ... and with the following parameters to filter out only baseline - parameters: # All Copernicus products are baseline. Just remove the very recent one to avoid newly ingested bias - - key: sensingStop - full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}end" - value: "[NOW-1D]" - label: "before yesterday" - # ESA-DATASET filter - - key: processingCenter - full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}processingCenter" - value: "ESA" - label: "ESA Baseline" - # and the Sentinel-5P (see baselines definition) ... - Copernicus-S5P: - # ... as a baseline (used in TC5O1, TC502, TC601 & TC602)... - type: baseline # The catalogue collections' set of type baseline are referering to common baseline defined globally - # ... with Sentinel-5P pre-Ops Hub as a reference - reference_target_site: S5PHub - # ... and with the following parameters to filter out only baseline - parameters: # All Copernicus products are baseline. Just remove the very recent one to avoid newly ingested bias - - - # ESA-DATASET filter - - key: processingCenter - full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}processingCenter" - value: "ESA" - label: "ESA Baseline" - # Mundi has a data offering... + Amazon: + type: baseline + reference_set_id: Amazon + reference_target_site: cdse DataOffer: - # ... defined as locally managed (TC503) + # ... defined as locally managed (TC503 and TC602) type: local - # ... with ApiHub as a reference - reference_target_site: ApiHub + # ... with scihub as a reference + reference_target_site: cdse parameters: - # ESA-DATASET filter - - key: processingCenter - full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}processingCenter" - value: "ESA" - label: "ESA Baseline" + - key: sensingStop + full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}end" + value: "[NOW-3D]" + label: "before 3D" # collections' set definition collections: - Sentinel1-SLC-World: - label: "Sentinel-1 SLC World" + Sentinel2-L1C: + latency_polling: true + latency_check_interval: 900 + latency_check_max_duration: 259200 + label: "Sentinel-2 Level-1C" parameters: - key: missionName full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" - value: "Sentinel-1" - label: "Sentinel-1" + value: "Sentinel-2" + label: "Sentinel-2" - key: productType full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" - value: "SLC" - label: "SLC" - - key: sensingStart - full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}start" - value: "[NOW-12M]" - label: "Last 12 months" - Sentinel1-SLC-Europe: - label: "Sentinel-1 SLC Europe" + value: "S2MSI1C" + label: "L1C" + # Sentinel-2 L2A is only accessible using STAC + Sentinel2-L2A: + # TS06 parameters, not needed for Opensearch catalogue + # enables polling + latency_polling: true + # target polling interval in seconds + latency_check_interval: 900 + # maximum duration of polling attempts in seconds + latency_check_max_duration: 259200 + # time offset (seconds) for recent product search + # latency_check_offset: 286400 + # End TS06 parameters + label: "Sentinel-2 Level-2A" parameters: - key: missionName full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" - value: "Sentinel-1" - label: "Sentinel-1" + value: "Sentinel-2" + label: "Sentinel-2" - key: productType full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" - value: "SLC" - label: "SLC" - - key: geometry - full_name: "{http://a9.com/-/opensearch/extensions/geo/1.0/}geometry" - value: "POLYGON((-10.547 36.173,-2.109 36.031,6.855 38.548,11.25 37.996,20.391 34.452,35.156 34.162,42.363 67.942,26.895 72.342,-26.016 67.136,-10.547 36.173))" - label: "Europe" - - key: sensingStart - full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}start" - value: "[NOW-24M]" - label: "Last 24 months" - Sentinel1-GRD-World: - label: "Sentinel-1 GRD World" + value: "S2MSI2A" + label: "L2A" + # Sentinel-1 GRD is only accessible using STAC + Sentinel1-GRD: + latency_polling: true + latency_check_interval: 900 + latency_check_max_duration: 259200 + label: "Sentinel-1 GRD" parameters: - key: missionName full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" @@ -1486,12 +2475,89 @@ service_providers: full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" value: "GRD" label: "GRD" - - key: sensingStart - full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}start" - value: "[NOW-12M]" - label: "Last 12 months" - Sentinel1-SLC-Europe: - label: "Sentinel-1 GRD Europe" + # AMAZON + compute: + connector: "amazon" + username: "username" + password: "secret" + # eu-west-2 = London + region_name: "eu-west-2" + vm_name: "cdab-test" + key_name: "cdab-key-aws" + # Immagine trovata nel "Catalogo AMI" fra le "AMI di AWS Marketplace", 07/07/2022 + image_name: "ami-0a7de6eb3a5ad1610" + # t2.large = 2 vCPU, 8 GB RAM + # r6g.large = 2 vCPU, 16 GB RAM (NOT OK, uses ARM CPU) + # r5a.large = 2 vCPU, 16 GB RAM + flavor_name: "r5a.large" + security_group: "sg-0cc75812b649925ce" + private_key_file: "/home/jenkins/cdab-key-aws.pem" + remote_user: "debian" + # https://calculator.aws -> EC2 + cost_monthly: 94.27 + cost_hourly: 0.1291 + currency: "EUR" + storage: + min_upload_size: 10 + max_upload_size: 300 + hellenic: + max_catalogue_thread: 5 + # We lower parallelism to avoid hitting the daily limit. + max_download_thread: 2 + max_download_size: 13958643710 + data: + url: https://sentinels.space.noa.gr/dhus + credentials: username:password + catalogue: + # TS03 parameters + # Mission (e.g. Sentinel-1) + systematic_search_mission: "Sentinel-1" + # Regular expression to validate correct mission in result items (via identifier) + systematic_search_mission_regex: "^S1.*" + # Product type (e.g. GRD) + systematic_search_product_type: "GRD" + # Regular expression to validate correct product type in result items (via identifier) + systematic_search_product_type_regex: "^S1.*_GRD._.*" + # Search area geometry (WKT) + systematic_search_aoi_wkt: "POLYGON((9.75 47.55,13.04 47.51,12.74 48.12,13.86 48.75,12.70 49.38,12.05 50.35,10.11 50.56,9.05 50.01,9.10 49.56,9.73 49.73,10.50 48.96,9.75 47.55))" + # Search area description + systematic_search_aoi_description: "over Bavaria" + # Search period (days before current day) + systematic_search_days: 7 + # End TS03 parameters + sets: + Copernicus: + type: baseline + reference_target_site: cdse + parameters: + - key: sensingStop + full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}end" + value: "[NOW-1D]" + label: "before 1D" + Copernicus-S5P: + reference_set_id: Copernicus-S5P + type: baseline + reference_target_site: cdse + parameters: # All Copernicus products are baseline. Just remove the very recent one to avoid newly ingested bias + - key: sensingStop + full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}end" + value: "[NOW-1D]" + label: "before 1D" + DataOffer: + type: local + reference_target_site: cdse + parameters: + - key: sensingStart + full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}start" + value: "[NOW-23D]" + label: "last 23D" + - key: geom + full_name: "{http://a9.com/-/opensearch/extensions/geo/1.0/}geometry" + value: "POLYGON((-9.84 27.99, 48.87 27.99, 48.87 48.46, -9.84 48.46, -9.84 27.99))" + label: "Mediterranean" + collections: + Sentinel1-RAW-NonTimeCritical: + label: "Sentinel-1 RAW" parameters: - key: missionName full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" @@ -1499,18 +2565,10 @@ service_providers: label: "Sentinel-1" - key: productType full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" - value: "GRD" - label: "GRD" - - key: geometry - full_name: "{http://a9.com/-/opensearch/extensions/geo/1.0/}geometry" - value: "POLYGON((-10.547 36.173,-2.109 36.031,6.855 38.548,11.25 37.996,20.391 34.452,35.156 34.162,42.363 67.942,26.895 72.342,-26.016 67.136,-10.547 36.173))" - label: "Europe" - - key: sensingStart - full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}start" - value: "[NOW-48M]" - label: "Last 48 months" - Sentinel1-OCN-World: - label: "Sentinel-1 OCN World" + value: "RAW" + label: "RAW" + Sentinel1-SLC-NonTimeCritical: + label: "Sentinel-1 SLC" parameters: - key: missionName full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" @@ -1518,14 +2576,10 @@ service_providers: label: "Sentinel-1" - key: productType full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" - value: "OCN" - label: "OCN" - - key: sensingStart - full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}start" - value: "[NOW-12M]" - label: "Last 12 months" - Sentinel1-OCN-Europe: - label: "Sentinel-1 OCN Europe" + value: "SLC" + label: "SLC" + Sentinel1-GRD-NonTimeCritical: + label: "Sentinel-1 GRD" parameters: - key: missionName full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" @@ -1533,33 +2587,21 @@ service_providers: label: "Sentinel-1" - key: productType full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" - value: "OCN" - label: "OCN" - - key: geometry - full_name: "{http://a9.com/-/opensearch/extensions/geo/1.0/}geometry" - value: "POLYGON((-10.547 36.173,-2.109 36.031,6.855 38.548,11.25 37.996,20.391 34.452,35.156 34.162,42.363 67.942,26.895 72.342,-26.016 67.136,-10.547 36.173))" - label: "Europe" - - key: sensingStart - full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}start" - value: "[NOW-48M]" - label: "Last 48 months" - Sentinel2-MSI1C-World: - label: "Sentinel-2 Level-1C World" + value: "GRD" + label: "GRD" + Sentinel1-OCN-NonTimeCritical: + label: "Sentinel-1 OCN" parameters: - key: missionName full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" - value: "Sentinel-2" - label: "Sentinel-2" + value: "Sentinel-1" + label: "Sentinel-1" - key: productType full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" - value: "S2MSI1C" - label: "Level-1C" - - key: sensingStart - full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}start" - value: "[NOW-12M]" - label: "Last 12 months" - Sentinel2-MSI1C-Europe: - label: "Sentinel-2 Level-1C Europe" + value: "OCN" + label: "OCN" + Sentinel2-L1C: + label: "Sentinel-2 Level-1C" parameters: - key: missionName full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" @@ -1568,17 +2610,9 @@ service_providers: - key: productType full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" value: "S2MSI1C" - label: "Level-1C" - - key: geom - full_name: "{http://a9.com/-/opensearch/extensions/geo/1.0/}geometry" - value: "POLYGON((-10.547 36.173,-2.109 36.031,6.855 38.548,11.25 37.996,20.391 34.452,35.156 34.162,42.363 67.942,26.895 72.342,-26.016 67.136,-10.547 36.173))" - label: "Europe" - - key: sensingStart - full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}start" - value: "[NOW-48M]" - label: "Last 48 months" - Sentinel2-MSI2A-Europe: - label: "Sentinel-2 Level-2A Europe" + label: "L1C" + Sentinel2-L2A: + label: "Sentinel-2 Level-2A" parameters: - key: missionName full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" @@ -1587,17 +2621,9 @@ service_providers: - key: productType full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" value: "S2MSI2A" - label: "Level-2A" - - key: geom - full_name: "{http://a9.com/-/opensearch/extensions/geo/1.0/}geometry" - value: "POLYGON((-10.547 36.173,-2.109 36.031,6.855 38.548,11.25 37.996,20.391 34.452,35.156 34.162,42.363 67.942,26.895 72.342,-26.016 67.136,-10.547 36.173))" - label: "Europe" - - key: sensingStart - full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}start" - value: "[NOW-48M]" - label: "Last 48 months" - Sentinel3-OLCI-World: - label: "Sentinel-3 OLCI World" + label: "L2A" + Sentinel3-OLCI-L1: + label: "Sentinel-3 OLCI Level-1" parameters: - key: missionName full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" @@ -1607,16 +2633,12 @@ service_providers: full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}instrument" value: "OLCI" label: "OLCI" - - key: sensingStart - full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}start" - value: "[NOW-12M]" - label: "Last 12 months" - - key: timeliness - full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" - value: "NTC" - label: "Non Time Critical" - Sentinel3-OLCI-Europe: - label: "Sentinel-3 OLCI Europe" + - key: processingLevel + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}processingLevel" + value: "L1" + label: "L1" + Sentinel3-OLCI-L2: + label: "Sentinel-3 OLCI Level-2" parameters: - key: missionName full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" @@ -1626,86 +2648,203 @@ service_providers: full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}instrument" value: "OLCI" label: "OLCI" - - key: geometry - full_name: "{http://a9.com/-/opensearch/extensions/geo/1.0/}geometry" - value: "POLYGON((-10.547 36.173,-2.109 36.031,6.855 38.548,11.25 37.996,20.391 34.452,35.156 34.162,42.363 67.942,26.895 72.342,-26.016 67.136,-10.547 36.173))" - label: "Europe" + - key: processingLevel + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}processingLevel" + value: "L2" + label: "L2" + # It appears Hellenic does not host Short Time Critical (STC) SRAL products, and its retention policy (23 days) + # is too short for NTC data, so we only search for NRT data in the SRAL collection. + Sentinel3-SRAL-L1: + label: "Sentinel-3 SRAL Level-1 NRT" + parameters: + - key: missionName + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" + value: "Sentinel-3" + label: "Sentinel-3" + - key: instrument + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}instrument" + value: "SRAL" + label: "SRAL" + - key: processingLevel + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}processingLevel" + value: "L1" + label: "L1" - key: timeliness full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" - value: "NTC" - label: "Non Time Critical" - - key: sensingStart - full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}start" - value: "[NOW-48M]" - label: "Last 48 months" - compute: - auth_url: https://iam.eu-de.otc.t-systems.com:443/v3 - username: "login" - # password: "" - project_name: "eu-de" - user_domain_name: "OTC-EU-DE-00000000001000041106" - interface: "public" - identity_api_version: 3 - vm_name: "cdab-test-mundi" - key_name: "cdab-key" - image_name: "Standard_CentOS_7_latest" - flavor_name: "s2.large.4" - security_group: "default" - floating_ip: True - private_key_file: "cdab-key-mundi.pem" - remote_user: "linux" - storage: - auth_url: https://iam.eu-de.otc.t-systems.com:443/v3 - username: "login" - # password: "" - project_name: "eu-de" - user_domain_name: "OTC-EU-DE-00000000001000041106" - SOBLOO: - max_catalogue_thread: 5 - max_download_thread: 2 + value: "NRT" + label: "NRT" + Sentinel3-SRAL-L2: + label: "Sentinel-3 SRAL Level-2 NRT" + parameters: + - key: missionName + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" + value: "Sentinel-3" + label: "Sentinel-3" + - key: instrument + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}instrument" + value: "SRAL" + label: "SRAL" + - key: processingLevel + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}processingLevel" + value: "L2" + label: "L2" + - key: timeliness + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" + value: "NRT" + label: "NRT" + Sentinel3-SLSTR-L1: + label: "Sentinel-3 SLSTR Level-1" + parameters: + - key: missionName + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" + value: "Sentinel-3" + label: "Sentinel-3" + - key: instrument + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}instrument" + value: "SLSTR" + label: "SLSTR" + - key: processingLevel + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}processingLevel" + value: "L1" + label: "L1" + # It appears Hellenic does not host SLSTR FRP data, so we only search for LST Land data + Sentinel3-SLSTR-L2: + label: "Sentinel-3 SLSTR Level-2" + parameters: + - key: missionName + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" + value: "Sentinel-3" + label: "Sentinel-3" + - key: productType + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" + value: "SL_2_LST___" + label: "SLSTR L2 Land" + Sentinel3-SYNERGY-L2: + label: "Sentinel-3 SYNERGY Level-2" + parameters: + - key: missionName + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" + value: "Sentinel-3" + label: "Sentinel-3" + - key: instrument + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}instrument" + value: "SYNERGY" + label: "SYNERGY" + - key: processingLevel + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}processingLevel" + value: "L2" + label: "L2" + DataOffer-S5P: + type: local + reference_target_site: cdse + parameters: + - key: sensingStart + full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}start" + value: "[NOW-23D]" + label: "last 23D" + - key: geom + full_name: "{http://a9.com/-/opensearch/extensions/geo/1.0/}geometry" + value: "POLYGON((-9.84 27.99, 48.87 27.99, 48.87 48.46, -9.84 48.46, -9.84 27.99))" + label: "Mediterranean" + collections: + Sentinel5P-L1B: + label: "Sentinel-5P L1B" + parameters: + - key: missionName + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" + value: "Sentinel-5 Precursor" + label: "Sentinel-5P" + - key: processingLevel + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}processingLevel" + value: "L1B" + label: "L1B" + Sentinel5P-L2-Offline: + label: "Sentinel-5P L2 Offline" + parameters: + - key: missionName + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" + value: "Sentinel-5 Precursor" + label: "Sentinel-5P" + - key: processingLevel + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}processingLevel" + value: "L2" + label: "L2" + - key: processingMode + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}processingMode" + value: "Offline" + label: "Offline" + Sentinel5P-L2-NRT: + label: "Sentinel-5P L2 NRT" + parameters: + - key: missionName + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" + value: "Sentinel-5 Precursor" + label: "Sentinel-5P" + - key: processingLevel + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}processingLevel" + value: "L2" + label: "L2" + - key: processingMode + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}processingMode" + value: "Near real time" + label: "NRT" + wekeo: + max_catalogue_thread: 1 + max_download_thread: 1 + # We try using a larger size threshold for Wekeo because download scenarios fail too often for other reasons, too + max_download_size: 2147483648 data: - url: https://sobloo.eu/api/v1/services/search - credentials: username:apikey - class: SOBLOO + url: https://wekeo-broker.apps.mercator.dpi.wekeo.eu/databroker + credentials: username:secret + class: CREO catalogue: - # Sobloo Catalogue collections' set definition + # Interval (seconds) when polling to check for completed catalogue queries. Affects response time metric. Default=4. + # Number of polls is fixed to 30, so timeout is = query_polling_interval * 30. + query_polling_interval: 4 + limit_query: True + default_bounding_box: -1 , 37 , 25 , 55 + default_start_time: 2022-05-01T00:00:00Z + default_end_time: 2022-11-01T00:00:00Z + # Time range (seconds) for queries that only contain a start time or an end time. Default=432000=5 days. + # 7776000 = 90 days + max_period_seconds: 7776000 + # Geometries from the geometry query parameter (such as a POLYGON around a country) are now also taken into account and its enclosing bounding box is treated as if that bounding box was given as the query parameter. If limit_query is True, the central area of that bounding box is used. The size of that area can be set using the new configuration key max_area_degrees (double), which is the length of each side of the bounding box. The default value is 1. + max_area_degrees: 3.0 sets: - # Sobloo has the Copernicus catalogue (see baselines definition) ... Copernicus: - # Identifier of the cxatalogue set configured in the data section reference_set_id: Copernicus - # ... as a baseline (used in TC5O1, TC502, TC601 & TC602)... - type: baseline # The catalogue collections' set of type baseline are referering to common baseline defined globally - # ... with ApiHub as a reference ... - reference_target_site: ApiHub - # ... and with the following parameters to filter out only baseline - parameters: # All Copernicus products are baseline. Just remove the very recent one to avoid newly ingested bias + type: baseline + reference_target_site: cdse + parameters: - key: sensingStop full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}end" value: "[NOW-1D]" - label: "before yesterday" - # and the Sentinel-5P (see baselines definition) ... + label: "before 1D" Copernicus-S5P: - # ... as a baseline (used in TC5O1, TC502, TC601 & TC602)... - type: baseline # The catalogue collections' set of type baseline are referering to common baseline defined globally - # ... with Sentinel-5P pre-Ops Hub as a reference - reference_target_site: S5PHub - # ... and with the following parameters to filter out only baseline - parameters: # All Copernicus products are baseline. Just remove the very recent one to avoid newly ingested bias + reference_set_id: Copernicus-S5P + type: baseline + reference_target_site: cdse + parameters: - key: sensingStop full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}end" value: "[NOW-1D]" - label: "before yesterday" - # Sobloo has a data offering... + label: "before 1D" DataOffer: - # ... defined as locally managed (TC503) + # ... defined as locally managed (TC503 and TC602) type: local - # ... with ApiHub as a reference - reference_target_site: ApiHub + # ... with scihub as a reference + reference_target_site: cdse # collections' set definition collections: - Sentinel1-RAW: - label: "Sentinel-1 RAW last 3 Months" + # All Sentinel-1 products on scihub are NTC. + # The correct filter should be timeliness:"Fast-24h" but only part of the collection has this field, + # many products do not have it, so we should filter exluding NRT i.e. "NOT timeliness:NRT-3h" but NOT + # filters cannot be defined yet. So for the time being we include all products. + Sentinel1-RAW-NonTimeCritical: + latency_polling: true + latency_check_interval: 900 + latency_check_max_duration: 259200 + label: "Sentinel-1 RAW" parameters: - key: missionName full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" @@ -1715,12 +2854,11 @@ service_providers: full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" value: "RAW" label: "RAW" - - key: sensingStart - full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}start" - value: "[NOW-3M]" - label: "last 3 months" - Sentinel1-SLC-Europe-Africa: - label: "Sentinel-1 SLC over Europe and Africa" + Sentinel1-SLC-NonTimeCritical: + latency_polling: true + latency_check_interval: 900 + latency_check_max_duration: 259200 + label: "Sentinel-1 SLC" parameters: - key: missionName full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" @@ -1730,12 +2868,11 @@ service_providers: full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" value: "SLC" label: "SLC" - - key: geometry - full_name: "{http://a9.com/-/opensearch/extensions/geo/1.0/}geometry" - value: "POLYGON((-24.961 16.973,-21.094 4.215,4.57 0.352,13.711 -34.016,21.445 -36.031,55.195 -24.847,54.141 16.973,41.484 11.178,34.453 31.653,41.484 68.528,26.367 72.396,-26.719 67.475,-11.25 36.315,-24.961 16.973))" - label: "over Europe & Africa" - Sentinel1-SLC-World: - label: "Sentinel-1 SLC World last 12 months" + Sentinel1-GRD-NonTimeCritical: + latency_polling: true + latency_check_interval: 900 + latency_check_max_duration: 259200 + label: "Sentinel-1 GRD" parameters: - key: missionName full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" @@ -1743,14 +2880,13 @@ service_providers: label: "Sentinel-1" - key: productType full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" - value: "SLC" - label: "SLC" - - key: sensingStart - full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}start" - value: "[NOW-12M]" - label: "last 12 months" - Sentinel1-GRD-Europe-Africa: - label: "Sentinel-1 GRD over Europe and Africa" + value: "GRD" + label: "GRD" + Sentinel1-OCN-NonTimeCritical: + latency_polling: true + latency_check_interval: 900 + latency_check_max_duration: 259200 + label: "Sentinel-1 OCN" parameters: - key: missionName full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" @@ -1758,104 +2894,377 @@ service_providers: label: "Sentinel-1" - key: productType full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" - value: "GRD" - label: "GRD" - - key: geometry - full_name: "{http://a9.com/-/opensearch/extensions/geo/1.0/}geometry" - value: "POLYGON((-24.961 16.973,-21.094 4.215,4.57 0.352,13.711 -34.016,21.445 -36.031,55.195 -24.847,54.141 16.973,41.484 11.178,34.453 31.653,41.484 68.528,26.367 72.396,-26.719 67.475,-11.25 36.315,-24.961 16.973))" - label: "over Europe & Africa" - Sentinel1-SLC-World: - label: "Sentinel-1 GRD World last 12 months" + value: "OCN" + label: "OCN" + Sentinel2-L1C: + latency_polling: true + latency_check_interval: 900 + latency_check_max_duration: 259200 + label: "Sentinel-2 Level-1C" parameters: - key: missionName full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" - value: "Sentinel-1" - label: "Sentinel-1" + value: "Sentinel-2" + label: "Sentinel-2" - key: productType full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" - value: "GRD" - label: "GRD" - - key: sensingStart - full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}start" - value: "[NOW-12M]" - label: "last 12 months" - Sentinel1-OCN-Europe-Africa: - label: "Sentinel-1 OCN over Europe and Africa" + value: "S2MSI1C" + label: "L1C" + Sentinel2-L2A: + latency_polling: true + latency_check_interval: 900 + latency_check_max_duration: 259200 + label: "Sentinel-2 Level-2A" parameters: - key: missionName full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" - value: "Sentinel-1" - label: "Sentinel-1" + value: "Sentinel-2" + label: "Sentinel-2" - key: productType full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" - value: "OCN" - label: "OCN" - - key: geometry - full_name: "{http://a9.com/-/opensearch/extensions/geo/1.0/}geometry" - value: "POLYGON((-24.961 16.973,-21.094 4.215,4.57 0.352,13.711 -34.016,21.445 -36.031,55.195 -24.847,54.141 16.973,41.484 11.178,34.453 31.653,41.484 68.528,26.367 72.396,-26.719 67.475,-11.25 36.315,-24.961 16.973))" - label: "over Europe & Africa" - Sentinel1-OCN-World: - label: "Sentinel-1 OCN World last 12 months" + value: "S2MSI2A" + label: "L2A" + Sentinel3-OLCI-L1-NonTimeCritical: + latency_polling: true + latency_check_interval: 900 + latency_check_max_duration: 259200 + label: "Sentinel-3 OLCI Level-1 Non Time Critical" parameters: - key: missionName full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" - value: "Sentinel-1" - label: "Sentinel-1" - - key: productType - full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" - value: "OCN" - label: "OCN" - - key: sensingStart - full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}start" - value: "[NOW-12M]" - label: "last 12 months" - Sentinel2-L1-Europe-Africa: - label: "Sentinel-2 MSI L1C over Europe and Africa" + value: "Sentinel-3" + label: "Sentinel-3" + - key: instrument + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}instrument" + value: "OLCI" + label: "OLCI" + - key: processingLevel + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}processingLevel" + value: "L1" + label: "L1" + - key: timeliness + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" + value: "NTC" + label: "NTC" + Sentinel3-OLCI-L2-NonTimeCritical: + latency_polling: true + latency_check_interval: 900 + latency_check_max_duration: 259200 + label: "Sentinel-3 OLCI Level-2 Non Time Critical" + parameters: + - key: missionName + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" + value: "Sentinel-3" + label: "Sentinel-3" + - key: instrument + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}instrument" + value: "OLCI" + label: "OLCI" + - key: processingLevel + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}processingLevel" + value: "L2" + label: "L2" + - key: timeliness + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" + value: "NTC" + label: "NTC" + Sentinel3-SRAL-L1-NonTimeCritical: + latency_polling: true + latency_check_interval: 900 + latency_check_max_duration: 259200 + label: "Sentinel-3 SRAL Level-1 Non Time Critical" + parameters: + - key: missionName + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" + value: "Sentinel-3" + label: "Sentinel-3" + - key: instrument + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}instrument" + value: "SRAL" + label: "SRAL" + - key: processingLevel + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}processingLevel" + value: "L1" + label: "L1" + - key: timeliness + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" + value: "NTC" + label: "NTC" + Sentinel3-SRAL-L2-NonTimeCritical: + latency_polling: true + latency_check_interval: 900 + latency_check_max_duration: 259200 + label: "Sentinel-3 SRAL Level-2 Non Time Critical" + parameters: + - key: missionName + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" + value: "Sentinel-3" + label: "Sentinel-3" + - key: instrument + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}instrument" + value: "SRAL" + label: "SRAL" + - key: processingLevel + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}processingLevel" + value: "L2" + label: "L2" + - key: timeliness + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" + value: "NTC" + label: "NTC" + Sentinel3-SLSTR-L1-NonTimeCritical: + latency_polling: true + latency_check_interval: 900 + latency_check_max_duration: 259200 + label: "Sentinel-3 SLSTR Level-1 Non Time Critical" + parameters: + - key: missionName + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" + value: "Sentinel-3" + label: "Sentinel-3" + - key: instrument + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}instrument" + value: "SLSTR" + label: "SLSTR" + - key: processingLevel + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}processingLevel" + value: "L1" + label: "L1" + - key: timeliness + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" + value: "NTC" + label: "NTC" + Sentinel3-SLSTR-L2-NonTimeCritical: + latency_polling: true + latency_check_interval: 900 + latency_check_max_duration: 259200 + label: "Sentinel-3 SLSTR Level-2 Non Time Critical" + parameters: + - key: missionName + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" + value: "Sentinel-3" + label: "Sentinel-3" + - key: instrument + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}instrument" + value: "SLSTR" + label: "SLSTR" + - key: processingLevel + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}processingLevel" + value: "L2" + label: "L2" + - key: timeliness + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" + value: "NTC" + label: "NTC" + Sentinel3-SYNERGY-L2-NonTimeCritical: + latency_polling: true + latency_check_interval: 900 + latency_check_max_duration: 259200 + label: "Sentinel-3 SYNERGY Level-2 Non Time Critical" + parameters: + - key: missionName + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" + value: "Sentinel-3" + label: "Sentinel-3" + - key: instrument + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}instrument" + value: "SYNERGY" + label: "SYNERGY" + - key: processingLevel + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}processingLevel" + value: "L2" + label: "L2" + - key: timeliness + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" + value: "NTC" + label: "NTC" + Sentinel3-NearRealTime: + latency_polling: true + latency_check_interval: 900 + latency_check_max_duration: 259200 + label: "Sentinel-3 Near Real Time" parameters: - key: missionName full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" - value: "Sentinel-2" - label: "Sentinel-2" - - key: productType - full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" - value: "S2MSI1C" - label: "S2MSI1C" - - key: geometry - full_name: "{http://a9.com/-/opensearch/extensions/geo/1.0/}geometry" - value: "POLYGON((-24.961 16.973,-21.094 4.215,4.57 0.352,13.711 -34.016,21.445 -36.031,55.195 -24.847,54.141 16.973,41.484 11.178,34.453 31.653,41.484 68.528,26.367 72.396,-26.719 67.475,-11.25 36.315,-24.961 16.973))" - label: "over Europe & Africa" - Sentinel2-L1-World: - label: "Sentinel-2 MSI L1C World last 12 months" + value: "Sentinel-3" + label: "Sentinel-3" + - key: timeliness + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" + value: "NRT" + label: "NRT" + Sentinel3-ShortTimeCritical: + latency_polling: true + latency_check_interval: 900 + latency_check_max_duration: 259200 + label: "Sentinel-3 Short Time Critical" parameters: - key: missionName full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" - value: "Sentinel-2" - label: "Sentinel-2" - - key: productType - full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" - value: "S2MSI1C" - label: "S2MSI1C" + value: "Sentinel-3" + label: "Sentinel-3" + - key: timeliness + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" + value: "STC" + label: "STC" + DataOffer-S5P: + type: local + reference_target_site: cdse + parameters: + - key: sensingStart + full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}start" + value: "2019-05-01T00:00:00Z" + label: "Since May 2019" + collections: + Sentinel5P-L1B: + latency_polling: true + latency_check_interval: 900 + latency_check_max_duration: 259200 + label: "Sentinel-5P L1B" + parameters: + - key: missionName + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" + value: "Sentinel-5 Precursor" + label: "Sentinel-5P" + - key: processingLevel + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}processingLevel" + value: "L1B" + label: "L1B" + Sentinel5P-L2-Offline: + latency_polling: true + latency_check_interval: 900 + latency_check_max_duration: 259200 + label: "Sentinel-5P L2 Offline" + parameters: + - key: missionName + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" + value: "Sentinel-5 Precursor" + label: "Sentinel-5P" + - key: processingLevel + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}processingLevel" + value: "L2" + label: "L2" + - key: processingMode + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}processingMode" + value: "Offline" + label: "Offline" + # We limit S5P L2 NRT to 1 month because reference s5phub only keeps 1 month, so we cannot compare a longer time range + Sentinel5P-L2-NRT: + latency_polling: true + latency_check_interval: 900 + latency_check_max_duration: 259200 + label: "Sentinel-5P L2 NRT last month" + parameters: + - key: missionName + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" + value: "Sentinel-5 Precursor" + label: "Sentinel-5P" + - key: processingLevel + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}processingLevel" + value: "L2" + label: "L2" + - key: processingMode + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}processingMode" + value: "Near real time" + label: "NRT" - key: sensingStart full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}start" - value: "[NOW-12M]" - label: "last 12 months" - Sentinel2-L2-Europe-Africa: - label: "Sentinel-2 MSI L2A over Europe and Africa" + value: "[NOW-1M]" + label: "last 1M" + Sentinel5P-L2-Reprocessing: + latency_polling: true + latency_check_interval: 900 + latency_check_max_duration: 259200 + label: "Sentinel-5P L2 Reprocessing" + parameters: + - key: missionName + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" + value: "Sentinel-5 Precursor" + label: "Sentinel-5P" + - key: processingLevel + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}processingLevel" + value: "L2" + label: "L2" + - key: processingMode + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}processingMode" + value: "Reprocessing" + label: "RPRO" + # WEKEO + compute: + auth_url: https://keystone.cloudferro.com:5000/v3 + username: "username" + password: "secret" + project_id: "project_id" + project_name: "project_name" + project_domain_id: "project_domain_id" + region_name: "WAW3-1" + interface: "public" + identity_api_version: 3 + auth_type: v3oidcpassword + protocol: openid + identity_provider: oidc + discovery_endpoint: https://identity.cloudferro.com/auth/realms/wekeo-elasticity/.well-known/openid-configuration + client_id: openstack + client_secret: 16df101f-7a04-4344-9f79-451329ec318a + vm_name: "cdab-test" + key_name: "cdab-key-wekeo" + private_key_file: "/home/jenkins/cdab-key-wekeo.pem" + image_name: "Debian 12" + # hm.medium = 2 vCPU, 16GB RAM, 64GB HD + flavor_name: "hm.medium" + cost_monthly: 70.04 + cost_hourly: 0.114 + currency: "EUR" + network_name: [ "net_1" , "net_2" ] + security_group: "allow_ping_ssh_icmp_rdp" + # 20230927: we have 6 preallocated floating IPs in wekeoelasticity Horizon dashboard. + # They got deleted somehow, and we allocated them again. + floating_ip: True + floating_ip_network: "external" + remote_user: "eouser" + #use_volume: True + storage: + auth_url: https://keystone.cloudferro.com:5000/v3 + username: "username" + password: "secret" + project_id: "project_id" + project_name: "project_name" + user_domain_name: "user_domain_name" + min_upload_size: 10 + max_upload_size: 300 + planetary: + max_catalogue_thread: 5 + max_download_thread: 5 + # Higher size because we have so few collections here + max_download_size: 2147483648 + data: + url: https://planetarycomputer.microsoft.com + # No credentials are needed for Planetary + credentials: none:none + class: CREO + # https://planetarycomputer.microsoft.com/docs/overview/changelog + catalogue: + sets: + Planetary_Baseline: + reference_set_id: Planetary_Baseline + type: baseline + reference_target_site: cdse + DataOffer: + type: local + reference_target_site: cdse + collections: + Sentinel1-GRD-NonTimeCritical: + label: "Sentinel-1 GRD" parameters: - key: missionName full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" - value: "Sentinel-2" - label: "Sentinel-2" + value: "Sentinel-1" + label: "Sentinel-1" - key: productType full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" - value: "S2MSI2A" - label: "S2MSI2A" - - key: geometry - full_name: "{http://a9.com/-/opensearch/extensions/geo/1.0/}geometry" - value: "POLYGON((-24.961 16.973,-21.094 4.215,4.57 0.352,13.711 -34.016,21.445 -36.031,55.195 -24.847,54.141 16.973,41.484 11.178,34.453 31.653,41.484 68.528,26.367 72.396,-26.719 67.475,-11.25 36.315,-24.961 16.973))" - label: "over Europe & Africa" - Sentinel2-L1-World: - label: "Sentinel-2 MSI L2A World last 12 months" + value: "GRD" + label: "GRD" + Sentinel2-L2A: + label: "Sentinel-2 Level-2A" parameters: - key: missionName full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" @@ -1864,32 +3273,46 @@ service_providers: - key: productType full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" value: "S2MSI2A" - label: "S2MSI2A" - - key: sensingStart - full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}start" - value: "[NOW-12M]" - label: "last 12 months" - Sentinel3-World: - label: "Sentinel-3 World" - parameters: - - key: missionName - full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" - value: "Sentinel-3" - label: "Sentinel-3" - - key: timeliness - full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" - value: "NTC" - label: "NTC" - - key: timeliness - full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" - value: "NTC" - label: "Non Time Critical" + label: "L2A" + compute: + connector: azure + subscription_id: "subscription_id" + tenant_id: "tenant_id" + client_id: "client_id" + client_secret: "secret" + resource_group_name: "resource_group_name" + region_name: "northeurope" + vm_name: "cdab-test" + image: + publisher: "OpenLogic" + offer: "Debian" + sku: "12" + version: "latest" + # Memory optimized E2s v3 = 2 vCPU, 16GB RAM, 32GB HD + flavor_name: "Standard_E2s_v3" + cost_monthly: 97.92 + cost_hourly: 0.1341 + currency: "EUR" + private_key_file: "/home/jenkins/cdab-key-planetery.pem" + public_key_file: "/home/jenkins/cdab-key-planetery.pub" + remote_user: "azureuser" +# +# +# =========================================================================== +# DATA SETS DEFINITIONS +# =========================================================================== data: # This section defines all the logical sets of data + # OpenSearch syntax reference: http://docs.opengeospatial.org/is/13-026r8/13-026r8.html#17 sets: # Copernicus set (without S5P not yet considered as operational) Copernicus: name: "Copernicus Product Types" + parameters: # Remove very recent products to avoid newly ingested bias + - key: sensingStop + full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}end" + value: "[NOW-1D]" + label: "before 1D" collections: Sentinel1-RAW: label: "Sentinel-1 RAW" @@ -1902,6 +3325,17 @@ data: full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" value: "RAW" label: "RAW" + Sentinel1-SLC: + label: "Sentinel-1 SLC" + parameters: + - key: missionName + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" + value: "Sentinel-1" + label: "Sentinel-1" + - key: productType + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" + value: "SLC" + label: "SLC" Sentinel1-GRD: label: "Sentinel-1 GRD" parameters: @@ -1924,19 +3358,8 @@ data: full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" value: "OCN" label: "OCN" - Sentinel1-SLC-Europe: - label: "Sentinel-1 SLC" - parameters: - - key: missionName - full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" - value: "Sentinel-1" - label: "Sentinel-1" - - key: productType - full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" - value: "SLC" - label: "SLC" - Sentinel2-MSI1C: - label: "Sentinel-2 MSI Level-1C" + Sentinel2-L1C: + label: "Sentinel-2 Level-1C" parameters: - key: missionName full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" @@ -1945,9 +3368,9 @@ data: - key: productType full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" value: "S2MSI1C" - label: "S2MSI1C" - Sentinel2-MSI2A: - label: "Sentinel-2 MSI Level-2A" + label: "L1C" + Sentinel2-L2A: + label: "Sentinel-2 Level-2A" parameters: - key: missionName full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" @@ -1956,7 +3379,12 @@ data: - key: productType full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" value: "S2MSI2A" - label: "S2MSI2A" + label: "L2A" + # We filter out L2Ap products by ignoring the time range before operational distribution + - key: sensingStart + full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}start" + value: "2018-03-20T00:00:00Z" + label: "" Sentinel3-OL_1_EFR___: label: "Sentinel-3 OLCI L1 FR NTC" parameters: @@ -1971,7 +3399,7 @@ data: - key: timeliness full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" value: "NTC" - label: "Non Time Critical" + label: "NTC" Sentinel3-OL_1_ERR___: label: "Sentinel-3 OLCI L1 RR NTC" parameters: @@ -1986,7 +3414,7 @@ data: - key: timeliness full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" value: "NTC" - label: "Non Time Critical" + label: "NTC" Sentinel3-OL_2_LRR___: label: "Sentinel-3 OLCI L2 Land RR NTC" parameters: @@ -2001,7 +3429,7 @@ data: - key: timeliness full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" value: "NTC" - label: "Non Time Critical" + label: "NTC" Sentinel3-OL_2_LFR___: label: "Sentinel-3 OLCI L2 Land FR NTC" parameters: @@ -2016,7 +3444,7 @@ data: - key: timeliness full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" value: "NTC" - label: "Non Time Critical" + label: "NTC" Sentinel3-SL_1_RBT___: label: "Sentinel-3 SLSTR L1 RBT NTC" parameters: @@ -2031,7 +3459,7 @@ data: - key: timeliness full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" value: "NTC" - label: "Non Time Critical" + label: "NTC" Sentinel3-SL_2_LST___: label: "Sentinel-3 SLSTR L2 Land NTC" parameters: @@ -2046,7 +3474,7 @@ data: - key: timeliness full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" value: "NTC" - label: "Non Time Critical" + label: "NTC" Sentinel3-SR_1_SRA___: label: "Sentinel-3 SRAL L1 NTC" parameters: @@ -2061,7 +3489,7 @@ data: - key: timeliness full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" value: "NTC" - label: "Non Time Critical" + label: "NTC" Sentinel3-SR_1_SRA_A_: label: "Sentinel-3 SRAL L1 A NTC" parameters: @@ -2076,7 +3504,7 @@ data: - key: timeliness full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" value: "NTC" - label: "Non Time Critical" + label: "NTC" Sentinel3-SR_1_SRA_BS: label: "Sentinel-3 SRAL L1 BS NTC" parameters: @@ -2091,7 +3519,7 @@ data: - key: timeliness full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" value: "NTC" - label: "Non Time Critical" + label: "NTC" Sentinel3-SR_2_LAN___: label: "Sentinel-3 SRAL L2 Land NTC" parameters: @@ -2106,7 +3534,7 @@ data: - key: timeliness full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" value: "NTC" - label: "Non Time Critical" + label: "NTC" Sentinel3-SY_2_SYN___: label: "Sentinel-3 SYN L2 NTC" parameters: @@ -2117,11 +3545,11 @@ data: - key: productType full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" value: "SY_2_SYN___" - label: "SYN L2" + label: "SY L2 SYN" - key: timeliness full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" value: "NTC" - label: "Non Time Critical" + label: "NTC" Sentinel3-SY_2_VGP___: label: "Sentinel-3 SYN L2 VGP NTC" parameters: @@ -2132,238 +3560,223 @@ data: - key: productType full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" value: "SY_2_VGP___" - label: "SYN L2 VGP" + label: "SY L2 VGP" - key: timeliness full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" value: "NTC" - label: "Non Time Critical" - # Copernicus Sentinel-5p separate set (since not yet considered as operational) - Copernicus-S5P: - name: "Copernicus Sentinel-5P Product Types Pre-Ops" - collections: - Sentinel5P-L2__AER_AI: - label: "Sentinel-5P L2 AER AI" - parameters: - - key: missionName - full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" - value: "Sentinel-5 Precursor" - label: "Sentinel-5P" - - key: productType - full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" - value: "L2__AER_AI" - label: "L2 AER AI" - Sentinel5P-L2__CH4___: - label: "Sentinel-5P L2 CH4" - parameters: - - key: missionName - full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" - value: "Sentinel-5 Precursor" - label: "Sentinel-5P" - - key: productType - full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" - value: "L2__CH4___" - label: "L2 CH4" - Sentinel5P-L2__CLOUD_: - label: "Sentinel-5P L2 CLOUD" - parameters: - - key: missionName - full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" - value: "Sentinel-5 Precursor" - label: "Sentinel-5P" - - key: productType - full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" - value: "L2__CLOUD_" - label: "L2 CLOUD" - Sentinel5P-L2__CO_____: - label: "Sentinel-5P L2 CO" - parameters: - - key: missionName - full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" - value: "Sentinel-5 Precursor" - label: "Sentinel-5P" - - key: productType - full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" - value: "L2__CO_____" - label: "L2 CO" - Sentinel5P-L2__HCHO___: - label: "Sentinel-5P L2 HCHO" - parameters: - - key: missionName - full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" - value: "Sentinel-5 Precursor" - label: "Sentinel-5P" - - key: productType - full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" - value: "L2__HCHO___" - label: "L2 HCHO" - Sentinel5P-L2__NO2___: - label: "Sentinel-5P L2 NO2" - parameters: - - key: missionName - full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" - value: "Sentinel-5 Precursor" - label: "Sentinel-5P" - - key: productType - full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" - value: "L2__NO2___" - label: "L2 NO2" - Sentinel5P-L2__O3____: - label: "Sentinel-5P L2 O3" + label: "NTC" + Sentinel3-SY_2_VG1___: + label: "Sentinel-3 SYN L2 VG1 NTC" parameters: - key: missionName full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" - value: "Sentinel-5 Precursor" - label: "Sentinel-5P" + value: "Sentinel-3" + label: "Sentinel-3" - key: productType full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" - value: "L2__O3____" - label: "L2 O3" - Sentinel5P-L2__SO2___: - label: "Sentinel-5P L2 SO2" + value: "SY_2_VG1___" + label: "SY L2 VG1" + - key: timeliness + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" + value: "NTC" + label: "NTC" + Sentinel3-SY_2_V10___: + label: "Sentinel-3 SYN L2 V10 NTC" parameters: - key: missionName full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" - value: "Sentinel-5 Precursor" - label: "Sentinel-5P" + value: "Sentinel-3" + label: "Sentinel-3" - key: productType full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" - value: "L2__SO2___" - label: "L2 SO2" - Sentinel5P-L2__NP_BD3: - label: "Sentinel-5P L2 NP BAND 3" + value: "SY_2_V10___" + label: "SY L2 V10" + - key: timeliness + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}timeliness" + value: "NTC" + label: "NTC" + # Copernicus Sentinel-5P separate set (since not yet considered as operational) + Copernicus-S5P: + name: "Copernicus Sentinel-5P Product Types Pre-Ops" + parameters: # Remove very recent products to avoid newly ingested bias + - key: sensingStop + full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}end" + value: "[NOW-1D]" + label: "before 1D" + collections: + Sentinel5P-L1B: + label: "Sentinel-5P L1B" parameters: - key: missionName full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" value: "Sentinel-5 Precursor" label: "Sentinel-5P" - - key: productType - full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" - value: "L2__NP_BD3" - label: "L2 NP BAND 3" - Sentinel5P-L2__NP_BD6: - label: "Sentinel-5P L2 NP BAND 6" + - key: processingLevel + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}processingLevel" + value: "L1B" + label: "L1B" + Sentinel5P-L2-Offline: + label: "Sentinel-5P L2 Offline" parameters: - key: missionName full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" value: "Sentinel-5 Precursor" label: "Sentinel-5P" - - key: productType - full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" - value: "L2__NP_BD6" - label: "L2 NP BAND 6" - Sentinel5P-L2__NP_BD7: - label: "Sentinel-5P L2 NP BAND 7" + - key: processingLevel + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}processingLevel" + value: "L2" + label: "L2" + - key: processingMode + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}processingMode" + value: "Offline" + label: "Offline" + # S5PHub only keeps L2 NRT products since last 30 days, so we count them separately from Offline products + # and we limit to last month for comparison with targets which keep them for longer instead. + Sentinel5P-L2-NRT: + label: "Sentinel-5P L2 NRT last month" parameters: - key: missionName full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" value: "Sentinel-5 Precursor" label: "Sentinel-5P" - - key: productType - full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" - value: "L2__NP_BD7" - label: "L2 NP BAND 7" - Sentinel5P-L2__RA_BD1: - label: "Sentinel-5P L1B RA BAND 1" + - key: processingLevel + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}processingLevel" + value: "L2" + label: "L2" + - key: processingMode + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}processingMode" + value: "Near real time" + label: "NRT" + - key: sensingStart + full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}start" + value: "[NOW-1M]" + label: "last 1M" + Sentinel5P-L2-Reprocessing: + label: "Sentinel-5P L2 Reprocessing" parameters: - key: missionName full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" value: "Sentinel-5 Precursor" label: "Sentinel-5P" - - key: productType - full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" - value: "L2__RA_BD1" - label: "L1B RA BAND 1" - Sentinel5P-L2__RA_BD2: - label: "Sentinel-5P L1B RA BAND 2" + - key: processingLevel + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}processingLevel" + value: "L2" + label: "L2" + - key: processingMode + full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}processingMode" + value: "Reprocessing" + label: "RPRO" + Sentinel1: + name: "Sentinel-1 Product Types" + collections: + Sentinel1-RAW: + label: "Sentinel-1 RAW" parameters: - key: missionName full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" - value: "Sentinel-5 Precursor" - label: "Sentinel-5P" + value: "Sentinel-1" + label: "Sentinel-1" - key: productType full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" - value: "L2__RA_BD2" - label: "L1B RA BAND 2" - Sentinel5P-L2__RA_BD3: - label: "Sentinel-5P L1B RA BAND 3" + value: "RAW" + label: "RAW" + Sentinel1-GRD: + label: "Sentinel-1 GRD" parameters: - key: missionName full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" - value: "Sentinel-5 Precursor" - label: "Sentinel-5P" + value: "Sentinel-1" + label: "Sentinel-1" - key: productType full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" - value: "L2__RA_BD3" - label: "L1B RA BAND 3" - Sentinel5P-L2__RA_BD4: - label: "Sentinel-5P L1B RA BAND 4" + value: "GRD" + label: "GRD" + Sentinel1-OCN: + label: "Sentinel-1 OCN" parameters: - key: missionName full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" - value: "Sentinel-5 Precursor" - label: "Sentinel-5P" + value: "Sentinel-1" + label: "Sentinel-1" - key: productType full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" - value: "L2__RA_BD4" - label: "L1B RA BAND 4" - Sentinel5P-L2__RA_BD5: - label: "Sentinel-5P L1B RA BAND 5" + value: "OCN" + label: "OCN" + Sentinel1-SLC-Europe: + label: "Sentinel-1 SLC" parameters: - key: missionName full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" - value: "Sentinel-5 Precursor" - label: "Sentinel-5P" + value: "Sentinel-1" + label: "Sentinel-1" - key: productType full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" - value: "L2__RA_BD5" - label: "L1B RA BAND 5" - Sentinel5P-L2__RA_BD6: - label: "Sentinel-5P L1B RA BAND 6" + value: "SLC" + label: "SLC" + Amazon: + name: "Amazon Baseline" + parameters: + - key: sensingStop + full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}end" + value: "[NOW-3D]" + label: "before 3D" + collections: + Sentinel2-L1C: + label: "Sentinel-2 Level-1C" parameters: - key: missionName full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" - value: "Sentinel-5 Precursor" - label: "Sentinel-5P" + value: "Sentinel-2" + label: "Sentinel-2" - key: productType full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" - value: "L2__RA_BD6" - label: "L1B RA BAND 6" - Sentinel5P-L2__RA_BD7: - label: "Sentinel-5P L1B RA BAND 7" + value: "S2MSI1C" + label: "L1C" + # STAC-only + Sentinel2-L2A: + label: "Sentinel-2 Level-2A" parameters: - key: missionName full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" - value: "Sentinel-5 Precursor" - label: "Sentinel-5P" + value: "Sentinel-2" + label: "Sentinel-2" - key: productType full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" - value: "L2__RA_BD7" - label: "L1B RA BAND 7" - Sentinel5P-L2__RA_BD8: - label: "Sentinel-5P L1B RA BAND 8" + value: "S2MSI2A" + label: "L2A" + # STAC-only + Sentinel1-GRD: + label: "Sentinel-1 GRD" parameters: - key: missionName full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" - value: "Sentinel-5 Precursor" - label: "Sentinel-5P" + value: "Sentinel-1" + label: "Sentinel-1" - key: productType full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" - value: "L2__RA_BD8" - label: "L1B RA BAND 8" - Sentinel1: - name: "Sentinel-1 Product Types" + value: "GRD" + label: "GRD" + Google: + name: "Google Baseline" collections: - Sentinel1-RAW: - label: "Sentinel-1 RAW" + Sentinel2-L1C: + label: "Sentinel-2 Level-1C" parameters: - key: missionName full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" - value: "Sentinel-1" - label: "Sentinel-1" + value: "Sentinel-2" + label: "Sentinel-2" - key: productType full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" - value: "RAW" - label: "RAW" + value: "S2MSI1C" + label: "L1C" + - key: sensingStop + full_name: "{http://a9.com/-/opensearch/extensions/time/1.0/}end" + value: "[NOW-5D]" + label: "before 5D" + Planetary_Baseline: + name: "Planetary Baseline" + collections: Sentinel1-GRD: label: "Sentinel-1 GRD" parameters: @@ -2375,25 +3788,97 @@ data: full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" value: "GRD" label: "GRD" - Sentinel1-OCN: - label: "Sentinel-1 OCN" - parameters: - - key: missionName - full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" - value: "Sentinel-1" - label: "Sentinel-1" - - key: productType - full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" - value: "OCN" - label: "OCN" - Sentinel1-SLC-Europe: - label: "Sentinel-1 SLC" + Sentinel2-L2A: + label: "Sentinel-2 Level-2A" parameters: - key: missionName full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}platform" - value: "Sentinel-1" - label: "Sentinel-1" + value: "Sentinel-2" + label: "Sentinel-2" - key: productType full_name: "{http://a9.com/-/opensearch/extensions/eo/1.0/}productType" - value: "SLC" - label: "SLC" + value: "S2MSI2A" + label: "L2A" +geometry_filters: + Europe: + - Albania + - Austria + - Belarus + - Belgium + - Bosnia and Herzegovina + - Bulgaria + - Croatia + - Cyprus + - Czech Republic + - Denmark + - Estonia + - Finland + - France + - Germany + - Greece + - Hungary + - Ireland + - Italy + - Latvia + - Lithuania + - Luxembourg + - Malta + - Montenegro + - Netherlands + - Norway + - Poland + - Portugal + - Romania + - Serbia + - Slovakia + - Slovenia + - Spain + - Sweden + - Switzerland + - The former Yugoslav Republic of Macedonia + - Turkey + - United Kingdom + "mainland Europe": + - Austria + - Belgium + - Bulgaria + - Croatia + - Cyprus + - Czech Republic + - Denmark + - Estonia + - Finland + - France + - Germany + - Greece + - Hungary + - Ireland + - Italy + - Latvia + - Lithuania + - Luxembourg + - Malta + - Netherlands + - Poland + - Portugal + - Romania + - Slovakia + - Slovenia + - Spain + - Sweden + - Switzerland + Mediterranean: + - Greece + - Italy + - Spain + - Slovenia + - Croatia + - Bosnia and Herzegovina + - Montenegro + - Albania + - Tunisia + - Turkey + - Syria + - Lebanon + Germany: + - Germany \ No newline at end of file diff --git a/src/cdab-remote-client/README.md b/src/cdab-remote-client/README.md index 490b694..4fc939a 100644 --- a/src/cdab-remote-client/README.md +++ b/src/cdab-remote-client/README.md @@ -16,26 +16,6 @@ The **cdab-remote-client** is a command line utility that performs a local CDAB Note: The Openstack terminology is inconsistent and might be confusing; **servers** (as in the command-line client) and **instances** (as on the dashboard) refer to the same thing, **virtual machines**, as they are called below. -### Prerequisites - -* CentOS 7 -* Python 3 -* OpenStack client (a Python module that provides a command-line tool for OpenStack access) -* The Amazon AWS SDK for Python -* The Google API Client Library for Python -* The Microsoft Azure libraries for Python -* Root or proper _sudo_ permissions - -### Installation - -* Add the terradue-stable-el7 repository configuration on YUM, -* Install the tool with: - -``` -$ sudo yum install cdab-remote-client -y -``` - - ## Usage The main parameters are shown on the help page. @@ -166,7 +146,7 @@ Most of the values for the various keys can be obtained from the OpenStack dashb * **project_id**: Name of the project on the Google Cloud Platform. * **region_name**: Zone, e.g. *europe-west2-c*. * **key_name**: This setting is not applicable, as Google creates all accounts for which there are keys defined and authorises all keys. Key pairs are defined under https://console.cloud.google.com/compute/metadata/sshKeys. -* **image_name**: Fully qualified name of boot disk image to be used for new virtual machine, e.g. *projects/centos-cloud/global/images/centos-7-v20200309*. +* **image_name**: Fully qualified name of boot disk image to be used for new virtual machine, e.g. *projects/debian-cloud/global/images/debian-12*. * **flavor_name**: The machine type name name, e.g. *e2-standard-2*. Can also be specified, as in previous version, as fully qualified name of machine type for new virtual machine, e.g *projects//zones/europe-west2-c/machineTypes/e2-standard-2*. ##### Specific settings for Amazon AWS EC2 diff --git a/src/cdab-remote-client/bin/cdab-remote-client b/src/cdab-remote-client/bin/cdab-remote-client index 446a1ec..fe157c9 100755 --- a/src/cdab-remote-client/bin/cdab-remote-client +++ b/src/cdab-remote-client/bin/cdab-remote-client @@ -3,4 +3,4 @@ export PYTHONIOENCODING='utf8' command="python3 /usr/lib/cdab-remote-client/cdab-remote-client.py3"; for a in "$@"; do command="${command} \"${a/"/\\\\"/}\""; done - +python3 /usr/lib/cdab-remote-client/cdab-remote-client.py3 "$@" \ No newline at end of file diff --git a/src/cdab-remote-client/etc/config.yaml.sample b/src/cdab-remote-client/etc/config.yaml.sample deleted file mode 100644 index 1ce7c84..0000000 --- a/src/cdab-remote-client/etc/config.yaml.sample +++ /dev/null @@ -1,98 +0,0 @@ -global: - # Docker configuration file to be used for docker repository connections on virtual machine - docker_config: "docker-config.json" - # Number of retries for initial SSH connections - connect_retries: 60 - # Interval between initial SSH connections (in seconds, fraction also possible) - connect_interval: 1 -service_providers: - ApiHub: - # Data Access - data: - # Entry Point (usually the URL to the catalogue) - url: https://scihub.copernicus.eu/apihub - # Credentials (usually username:password/apiKey) - # credentials: - SciHub: - data: - url: https://scihub.copernicus.eu/dhus - # credentials: - IntHub: - data: - url: https://inthub.copernicus.eu/dhus - # credentials: - ColHub: - data: - url: https://colhub.copernicus.eu/dhus - # credentials: - ASF: - data: - url: https://api.daac.asf.alaska.edu - # credentials: - CREO: - data: - url: https://finder.creodias.eu/resto/api/collections/describe.xml - # credentials: - compute: - auth_url: https://cf2.cloudferro.com:5000/v3 - username: "udig4wa@gmail.com" - # password: - project_id: 7236455993264b0c8970d22751b23562 - project_name: "cloud_07455 project_without_eo" - user_domain_name: "cloud_07455" - region_name: "RegionOne" - interface: "public" - identity_api_version: 3 - vm_name: "cdab-test-creo" - key_name: "cdab-key" - image_name: "CentOS 7" - flavor_name: "eo2.large" - security_group: "allow_ping_ssh_rdp" - floating_ip: True - private_key_file: "cdab-key-creodias.pem" - remote_user: "eouser" - MUNDI: - data: - url: https://mundiwebservices.com/acdc/catalog/proxy/search - # credentials: - compute: - auth_url: https://iam.eu-de.otc.t-systems.com:443/v3 - username: "cdab_mundi" - # password: - project_name: "eu-de" - user_domain_name: "OTC-EU-DE-00000000001000041106" - interface: "public" - identity_api_version: 3 - vm_name: "cdab-test-mundi" - key_name: "cdab-key" - image_name: "Standard_CentOS_7_latest" - flavor_name: "s2.large.4" - security_group: "default" - floating_ip: True - private_key_file: "cdab-key-mundi.pem" - remote_user: "linux" - use_volume: True - use_tmp_volume: True - ONDA: - data: - url: https://catalogue.onda-dias.eu/dias-catalogue/ - # credentials: - compute: - auth_url: https://auth.cloud.ovh.net/v3 - username: "cjEkBasH97Gt" - # password: - project_id: d6fae7e318114c248623562c0adce408 - project_name: "6611565740751529" - user_domain_name: "Default" - region_name: "GRA7" - interface: "public" - identity_api_version: 3 - vm_name: "cdab-test-onda" - key_name: "cdab-key" - image_name: "Centos 7" - flavor_name: "b2-7" - private_key_file: "cdab-key-ovh.pem" - remote_user: "centos" - SOBLOO: - -reference_target_site: ApiHub diff --git a/src/cdab-remote-client/libexec/cdab-remote-client.py3 b/src/cdab-remote-client/libexec/cdab-remote-client.py3 index 205d16e..c3a3080 100755 --- a/src/cdab-remote-client/libexec/cdab-remote-client.py3 +++ b/src/cdab-remote-client/libexec/cdab-remote-client.py3 @@ -63,7 +63,7 @@ class TestClient: 'TS11': { 'test_scenario_description': 'Remote execution of catalogue access and download test on VM (single)', 'test_case_name': 'TC411', - 'docker_image_id': 'esacdab/testsuite:latest', + 'docker_image_id': 'ghcr.io/esacdab/cdab-testsuite:main', 'docker_run_command': 'CDAB_CLIENT_DEFAULT', 'cdab_client_test_scenario_id': 'TS11', 'timeout': 1 * 60 * 60, @@ -71,7 +71,7 @@ class TestClient: 'TS12': { 'test_scenario_description': 'Remote execution of catalogue access and download test on VM (multiple)', 'test_case_name': 'TC412', - 'docker_image_id': 'esacdab/testsuite:latest', + 'docker_image_id': 'ghcr.io/esacdab/cdab-testsuite:main', 'docker_run_command': 'CDAB_CLIENT_DEFAULT', 'cdab_client_test_scenario_id': 'TS12', 'timeout': 1 * 60 * 60, @@ -79,7 +79,7 @@ class TestClient: 'TS13': { 'test_scenario_description': 'Remote execution of processing test (multiple)', 'test_case_name': 'TC413', - 'docker_image_id': 'esacdab/ewf-s3-olci-composites:0.41', + 'docker_image_id': 'docker-co.terradue.com/geohazards-tep/ewf-s3-olci-composites:0.41', 'docker_run_command': 'PROCESSING', 'test_target_url': 'https://catalog.terradue.com/sentinel3/search?uid=S3A_OL_1_EFR____20191110T230850_20191110T231150_20191112T030831_0179_051_215_3600_LN1_O_NT_002', 'files': [ 's3-olci-composites.py' ], @@ -1049,38 +1049,25 @@ class TestClient: run.install_start_time = datetime.datetime.utcnow() # Software installation - execute_remote_command(self.compute_config, run, "sudo yum install -y yum-utils device-mapper-persistent-data lvm2 wget") - execute_remote_command(self.compute_config, run, "sudo yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo") - # Fails since November 2023 because of broken dependency - # execute_remote_command(self.compute_config, run, "sudo yum install docker-ce docker-ce-cli containerd.io -y") - execute_remote_command(self.compute_config, run, "sudo yum install -y http://mirror.centos.org/centos/7/extras/x86_64/Packages/container-selinux-2.107-1.el7_6.noarch.rpm") - execute_remote_command(self.compute_config, run, "sudo yum-config-manager --add-repo https://download.docker.com/linux/centos/docker-ce.repo") + execute_remote_command(self.compute_config, run, "sudo apt-get update") + execute_remote_command(self.compute_config, run, "sudo apt-get install ca-certificates curl") + execute_remote_command(self.compute_config, run, "sudo install -m 0755 -d /etc/apt/keyrings") + execute_remote_command(self.compute_config, run, "sudo curl -fsSL https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc") + execute_remote_command(self.compute_config, run, "sudo chmod a+r /etc/apt/keyrings/docker.asc" ) execute_remote_command(self.compute_config, run, - "wget -q http://mirror.centos.org/centos/7/extras/x86_64/Packages/slirp4netns-0.4.3-4.el7_8.x86_64.rpm " - "http://mirror.centos.org/centos/7/extras/x86_64/Packages/fuse3-devel-3.6.1-4.el7.x86_64.rpm " - "http://mirror.centos.org/centos/7/extras/x86_64/Packages/fuse3-libs-3.6.1-4.el7.x86_64.rpm " - "http://mirror.centos.org/centos/7/extras/x86_64/Packages/fuse-overlayfs-0.7.2-6.el7_8.x86_64.rpm" + "echo " + "\"deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/debian " + "$(. /etc/os-release && echo \"$VERSION_CODENAME\") stable\" | " + "sudo tee /etc/apt/sources.list.d/docker.list > /dev/null" ) - - execute_remote_command(self.compute_config, run, "sudo yum install -y slirp4netns-0.4.3-4.el7_8.x86_64.rpm fuse3-libs-3.6.1-4.el7.x86_64.rpm fuse-overlayfs-0.7.2-6.el7_8.x86_64.rpm fuse3-devel-3.6.1-4.el7.x86_64.rpm") - - execute_remote_command(self.compute_config, run, - "wget -q https://download.docker.com/linux/centos/7/x86_64/stable/Packages/docker-ce-cli-24.0.6-1.el7.x86_64.rpm " - "https://download.docker.com/linux/centos/7/x86_64/stable/Packages/containerd.io-1.6.18-3.1.el7.x86_64.rpm " - "https://download.docker.com/linux/centos/7/x86_64/stable/Packages/docker-ce-24.0.6-1.el7.x86_64.rpm" - ) - - execute_remote_command(self.compute_config, run, "sudo yum install -y containerd.io-1.6.18-3.1.el7.x86_64.rpm docker-ce-24.0.6-1.el7.x86_64.rpm docker-ce-cli-24.0.6-1.el7.x86_64.rpm") + execute_remote_command(self.compute_config, run, "sudo apt-get update") + execute_remote_command(self.compute_config, run, "sudo apt-get install -y wget docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin") # Preparation for volume use (in case main disk is too small) if self.compute_config['use_volume']: - execute_remote_command(self.compute_config, run, "sudo systemctl start docker") # to create /var/lib/docker directory - execute_remote_command(self.compute_config, run, "sudo systemctl stop docker") + execute_remote_command(self.compute_config, run, "sudo service docker stop") execute_remote_command(self.compute_config, run, "sudo mv /var/lib/docker /mnt/cdab-volume/docker") - # fails since 2020-07 (or earlier): - # execute_remote_command(self.compute_config, run, "sudo mv /var/lib/docker-engine /mnt/cdab-volume/docker-engine") execute_remote_command(self.compute_config, run, "sudo ln -s /mnt/cdab-volume/docker /var/lib/docker") - execute_remote_command(self.compute_config, run, "sudo ln -s /mnt/cdab-volume/docker-engine /var/lib/docker-engine") # Copy CA certificates if needed if self.ca_certificates: @@ -1088,13 +1075,13 @@ class TestClient: for cert in self.ca_certificates: cert_basename = os.path.basename(cert) copy_file(self.compute_config, run, cert, cert_basename) - execute_remote_command(self.compute_config, run, "sudo mv {0} /etc/pki/ca-trust/source/anchors/{0}".format(cert_basename)) - execute_remote_command(self.compute_config, run, "sudo chown root:root /etc/pki/ca-trust/source/anchors/{0}".format(cert_basename)) - execute_remote_command(self.compute_config, run, "sudo update-ca-trust") + execute_remote_command(self.compute_config, run, "sudo mv {0} /usr/local/share/ca-certificates/{0}".format(cert_basename)) + execute_remote_command(self.compute_config, run, "sudo chown root:root /usr/local/share/ca-certificates/{0}".format(cert_basename)) + execute_remote_command(self.compute_config, run, "sudo update-ca-certificates") Logger.log(LogLevel.INFO, "CA certificate(s) copied", run=run) # Start Docker service - execute_remote_command(self.compute_config, run, "sudo systemctl start docker") + execute_remote_command(self.compute_config, run, "sudo service docker start") execute_remote_command(self.compute_config, run, "sudo usermod -a -G docker $USER") Logger.log(LogLevel.INFO, "Docker service started", run=run) @@ -1190,13 +1177,6 @@ class TestClient: if 'opensearch-client' in tools: execute_remote_command(self.compute_config, run, "docker pull terradue/opensearch-client:2.1.2") - # Previously used: - execute_remote_command(self.compute_config, run, "sudo yum install -y unzip yum-utils") - execute_remote_command(self.compute_config, run, "sudo yum-config-manager --add-repo http://download.mono-project.com/repo/centos/") - execute_remote_command(self.compute_config, run, "sudo yum install -y mono-devel --nogpgcheck > /dev/null 2> /dev/null") - copy_file(self.compute_config, run, "{0}/ts-scripts/opensearch-client.zip".format(os.path.dirname(sys.argv[0])), "opensearch-client.zip") - execute_remote_command(self.compute_config, run, "sudo unzip -d /usr/lib/ opensearch-client.zip") - execute_remote_command(self.compute_config, run, "sudo mv /usr/lib/opensearch-client/bin/opensearch-client /usr/bin/") if 'Stars' in tools and 'uri_prefix' in self.target_site_class: execute_remote_command(self.compute_config, run, "docker pull terradue/stars:1.3.5") diff --git a/src/cdab-remote-client/libexec/ts-scripts/TS11-remote.sh b/src/cdab-remote-client/libexec/ts-scripts/TS11-remote.sh index dfb18db..6a82139 100644 --- a/src/cdab-remote-client/libexec/ts-scripts/TS11-remote.sh +++ b/src/cdab-remote-client/libexec/ts-scripts/TS11-remote.sh @@ -17,8 +17,13 @@ fi cd "$working_dir" -echo "DOCKER COMMAND: docker run --workdir /res -v ${PWD}:/res${eodata_volume_option} --env DOWNLOAD_ORIGIN=${download_origin} \"${docker_image}\" mono $MONO_OPTIONS /usr/lib/cdab-client/bin/cdab-testtools.exe --target_name \"${target_name}\" --target_url \"${target_url}\" --target_credentials \"${target_credentials_display}\" --testsite_name \"${test_site}\" -lf \"${load_factor}\" --conf /res/config.yaml -v TS11" > cdab.stderr -docker run --workdir /res -v ${PWD}:/res${eodata_volume_option} --env DOWNLOAD_ORIGIN=${download_origin} "${docker_image}" mono $MONO_OPTIONS /usr/lib/cdab-client/bin/cdab-testtools.exe --target_name "${target_name}" --target_url "${target_url}" --target_credentials "${target_credentials}" --testsite_name "${test_site}" -lf "${load_factor}" --conf /res/config.yaml -v TS11 > cdab.stdout 2>> cdab.stderr +# run the docker detached +docker run --detach --name ${test_site} ${eodata_volume_option} --env DOWNLOAD_ORIGIN=${download_origin} "${docker_image}" +# copy the config file to the docker +docker cp ${PWD}/config.yaml ${test_site}:/home/jenkins/config.yaml +# run the test +echo "DOCKER COMMAND: docker exec --workdir /home/jenkins -it ${test_site} /usr/lib/cdab-client/cdab-testtools --target_name \"${target_name}\" --target_url \"${target_url}\" --target_credentials \"${target_credentials_display}\" --testsite_name \"${test_site}\" -lf \"${load_factor}\" --conf /home/jenkins/config.yaml -v TS11" > cdab.stderr +docker exec --workdir /home/jenkins -t ${test_site} /usr/lib/cdab-client/cdab-testtools --target_name "${target_name}" --target_url "${target_url}" --target_credentials "${target_credentials}" --testsite_name "${test_site}" -lf "${load_factor}" --conf /home/jenkins/config.yaml -v TS11 > cdab.stdout 2>> cdab.stderr res=$? if [ $res -ne 0 ] then @@ -26,3 +31,8 @@ then cat cdab.stderr >&2 exit 1 fi +# Copy the results back to the working directory +docker cp ${test_site}:/home/jenkins/TS11Results.json ${PWD}/TS11Results.json +docker cp ${test_site}:/home/jenkins/junit.xml ${PWD}/junit.xml +# Delete the container +docker rm -f ${test_site} diff --git a/src/cdab-remote-client/libexec/ts-scripts/TS12-remote.sh b/src/cdab-remote-client/libexec/ts-scripts/TS12-remote.sh index f1aa80e..c832855 100644 --- a/src/cdab-remote-client/libexec/ts-scripts/TS12-remote.sh +++ b/src/cdab-remote-client/libexec/ts-scripts/TS12-remote.sh @@ -17,8 +17,13 @@ fi cd "$working_dir" -echo "DOCKER COMMAND: docker run --workdir /res -v ${PWD}:/res${eodata_volume_option} --env DOWNLOAD_ORIGIN=${download_origin} \"${docker_image}\" mono $MONO_OPTIONS /usr/lib/cdab-client/bin/cdab-testtools.exe --target_name \"${target_name}\" --target_url \"${target_url}\" --target_credentials \"${target_credentials_display}\" --testsite_name \"${test_site}\" -lf \"${load_factor}\" --conf /res/config.yaml -v TS12" > cdab.stderr -docker run --workdir /res -v ${PWD}:/res${eodata_volume_option} --env DOWNLOAD_ORIGIN=${download_origin} "${docker_image}" mono $MONO_OPTIONS /usr/lib/cdab-client/bin/cdab-testtools.exe --target_name "${target_name}" --target_url "${target_url}" --target_credentials "${target_credentials}" --testsite_name "${test_site}" -lf "${load_factor}" --conf /res/config.yaml -v TS12 > cdab.stdout 2>> cdab.stderr +# run the docker detached +docker run --detach --name ${test_site} ${eodata_volume_option} --env DOWNLOAD_ORIGIN=${download_origin} "${docker_image}" +# copy the config file to the docker +docker cp ${PWD}/config.yaml ${test_site}:/home/jenkins/config.yaml +# run the test +echo "DOCKER COMMAND: docker exec --workdir /home/jenkins -it ${test_site} /usr/lib/cdab-client/cdab-testtools --target_name \"${target_name}\" --target_url \"${target_url}\" --target_credentials \"${target_credentials_display}\" --testsite_name \"${test_site}\" -lf \"${load_factor}\" --conf /home/jenkins/config.yaml -v TS12" > cdab.stderr +docker exec --workdir /home/jenkins -t ${test_site} /usr/lib/cdab-client/cdab-testtools --target_name "${target_name}" --target_url "${target_url}" --target_credentials "${target_credentials}" --testsite_name "${test_site}" -lf "${load_factor}" --conf /home/jenkins/config.yaml -v TS12 > cdab.stdout 2>> cdab.stderr res=$? if [ $res -ne 0 ] then @@ -26,3 +31,8 @@ then cat cdab.stderr >&2 exit 1 fi +# Copy the results back to the working directory +docker cp ${test_site}:/home/jenkins/TS12Results.json ${PWD}/TS12Results.json +docker cp ${test_site}:/home/jenkins/junit.xml ${PWD}/junit.xml +# Delete the container +docker rm -f ${test_site} \ No newline at end of file diff --git a/src/cdab-remote-client/libexec/ts-scripts/TS13-remote.sh b/src/cdab-remote-client/libexec/ts-scripts/TS13-remote.sh index 54876fe..d069a81 100644 --- a/src/cdab-remote-client/libexec/ts-scripts/TS13-remote.sh +++ b/src/cdab-remote-client/libexec/ts-scripts/TS13-remote.sh @@ -1,6 +1,6 @@ function prepare() { echo "Installing tools" >> cdab.stderr - sudo yum install -y bc unzip + sudo apt-get install -y bc unzip echo "Done" >> cdab.stderr mkdir -p input_data diff --git a/src/cdab-remote-client/libexec/ts-scripts/TS15-remote.sh b/src/cdab-remote-client/libexec/ts-scripts/TS15-remote.sh index 39bdee0..844bef8 100644 --- a/src/cdab-remote-client/libexec/ts-scripts/TS15-remote.sh +++ b/src/cdab-remote-client/libexec/ts-scripts/TS15-remote.sh @@ -1,6 +1,6 @@ function prepare() { echo "Installing basic packages" - sudo yum install -y bc wget unzip libgfortran-4.8.5-39.el7.x86_64 + sudo apt-get install -y bc wget unzip libgfortran5-amd64-cross echo "Done (basic packages)" echo "Installing conda" diff --git a/src/cdab-remote-client/libexec/ts-scripts/TS15.1-remote.sh b/src/cdab-remote-client/libexec/ts-scripts/TS15.1-remote.sh index 42e0f01..2026f75 100644 --- a/src/cdab-remote-client/libexec/ts-scripts/TS15.1-remote.sh +++ b/src/cdab-remote-client/libexec/ts-scripts/TS15.1-remote.sh @@ -1,6 +1,6 @@ function prepare() { echo "Installing tools" >> cdab.stderr - sudo yum install -y bc unzip gcc python3-devel + sudo apt-get install -y bc unzip gcc python3-dev python3-pip sudo pip3 install requests echo "Done" >> cdab.stderr diff --git a/src/cdab-remote-client/libexec/ts-scripts/TS15.3-remote.sh b/src/cdab-remote-client/libexec/ts-scripts/TS15.3-remote.sh index 8880765..9231499 100644 --- a/src/cdab-remote-client/libexec/ts-scripts/TS15.3-remote.sh +++ b/src/cdab-remote-client/libexec/ts-scripts/TS15.3-remote.sh @@ -2,7 +2,7 @@ function prepare() { tar xvzf s3-olci.tgz echo "$(date +%Y-"%m-%dT%H:%M:%SZ") - Installing basic packages" >> cdab.stderr - sudo yum install -y bc tree wget unzip libgfortran-4.8.5 >> cdab.stderr 2>&1 + sudo apt-get install -y bc wget unzip libgfortran5-amd64-cross >> cdab.stderr 2>&1 echo "$(date +%Y-"%m-%dT%H:%M:%SZ") - Done (basic packages)" >> cdab.stderr echo "$(date +%Y-"%m-%dT%H:%M:%SZ") - Creating conda environment with snap and cwltool" >> cdab.stderr diff --git a/src/cdab-remote-client/libexec/ts-scripts/TS15.4-remote.sh b/src/cdab-remote-client/libexec/ts-scripts/TS15.4-remote.sh index 5349616..4cb6772 100644 --- a/src/cdab-remote-client/libexec/ts-scripts/TS15.4-remote.sh +++ b/src/cdab-remote-client/libexec/ts-scripts/TS15.4-remote.sh @@ -2,7 +2,7 @@ function prepare() { tar xvzf s3-slstr.tgz echo "$(date +%Y-"%m-%dT%H:%M:%SZ") - Installing basic packages" >> cdab.stderr - sudo yum install -y bc tree wget unzip libgfortran-4.8.5 >> cdab.stderr 2>&1 + sudo apt-get install -y bc wget unzip libgfortran5-amd64-cross >> cdab.stderr 2>&1 echo "$(date +%Y-"%m-%dT%H:%M:%SZ") - Done (basic packages)" >> cdab.stderr echo "$(date +%Y-"%m-%dT%H:%M:%SZ") - Creating conda environment with snap and cwltool" >> cdab.stderr diff --git a/src/cdab-remote-client/libexec/ts-scripts/TS15.5-remote.sh b/src/cdab-remote-client/libexec/ts-scripts/TS15.5-remote.sh index 34a0eee..83465ac 100644 --- a/src/cdab-remote-client/libexec/ts-scripts/TS15.5-remote.sh +++ b/src/cdab-remote-client/libexec/ts-scripts/TS15.5-remote.sh @@ -1,6 +1,6 @@ function prepare() { echo "$(date +%Y-"%m-%dT%H:%M:%SZ") - Installing basic packages" >> cdab.stderr - sudo yum install -y bc tree wget unzip libgfortran-4.8.5 >> cdab.stderr 2>&1 + sudo apt-get install -y bc wget unzip libgfortran5-amd64-cross >> cdab.stderr 2>&1 echo "$(date +%Y-"%m-%dT%H:%M:%SZ") - Done (basic packages)" >> cdab.stderr echo "$(date +%Y-"%m-%dT%H:%M:%SZ") - Creating conda environment with snap and cwltool" >> cdab.stderr diff --git a/src/cdab-remote-client/libexec/ts-scripts/conda-install.sh b/src/cdab-remote-client/libexec/ts-scripts/conda-install.sh index 06f4142..716238a 100644 --- a/src/cdab-remote-client/libexec/ts-scripts/conda-install.sh +++ b/src/cdab-remote-client/libexec/ts-scripts/conda-install.sh @@ -12,7 +12,6 @@ URL="https://github.com/conda-forge/miniforge/releases/download/${MINIFORGE_VERS INSTALLER_PATH=/tmp/miniforge-installer.sh # Make sure user's $HOME is not tampered with since this is run as root unset HOME -yum install -y wget wget --quiet $URL -O ${INSTALLER_PATH} chmod +x ${INSTALLER_PATH} # Check sha256 checksum diff --git a/src/cdab-remote-client/libexec/ts-scripts/link-onda-eodata.sh b/src/cdab-remote-client/libexec/ts-scripts/link-onda-eodata.sh index e37646a..15c28fe 100644 --- a/src/cdab-remote-client/libexec/ts-scripts/link-onda-eodata.sh +++ b/src/cdab-remote-client/libexec/ts-scripts/link-onda-eodata.sh @@ -1,3 +1,3 @@ -yum install -y nfs-utils +sudo apt-get install -y libnfs-utils mkdir /eodata sudo mount -t nfs4 -o nfsvers=4.1 ens-legacy.onda-dias.eu:/ /eodata diff --git a/src/docker/Dockerfile b/src/docker/Dockerfile index 2bb8133..08bdfe3 100644 --- a/src/docker/Dockerfile +++ b/src/docker/Dockerfile @@ -24,7 +24,7 @@ LABEL org.opencontainers.image.description DESCRIPTION="CDAB Test Suite" \ RUN apt-get update \ && apt-get upgrade -y \ && apt-get install -y python3 python3-pip shunit2 traceroute libssl1.1 libgssapi-krb5-2 sysstat sed ca-certificates jq curl \ - openssh-server ssh sudo cron supervisor \ + openssh-server ssh sudo cron supervisor libxml2-utils \ && rm -rf /var/lib/apt/lists/* /tmp/* # Install OpenStack client, Google Cloud Platform Python API, Amazon AWS EC2 Python API and Microsoft Azure Python API @@ -38,8 +38,6 @@ COPY src/cdab-client/App_Data /usr/lib/cdab-client/App_Data COPY --chmod=755 src/cdab-client/cdab-client /usr/bin/cdab-client COPY src/cdab-remote-client/bin/cdab-remote-client /usr/bin/cdab-remote-client COPY src/cdab-remote-client/libexec /usr/lib/cdab-remote-client -COPY src/cdab-remote-client/etc /usr/lib/cdab-remote-client/etc -COPY src/cdab-remote-client/etc/config.yaml.sample /usr/lib/cdab-remote-client/etc/config.yaml ENV PATH="/usr/bin:${PATH}" @@ -95,16 +93,6 @@ RUN chown -R jenkins:jenkins /home/jenkins/cdab-tools #### -# Add new root certificate ISRG Root X1 -RUN curl -O https://letsencrypt.org/certs/isrgrootx1.pem && \ - sudo mv isrgrootx1.pem /usr/local/share/ca-certificates/isrgrootx1.crt -RUN curl -O https://curl.se/ca/cacert.pem && \ - sudo mv cacert.pem /usr/local/share/ca-certificates/cacert.crt -RUN sudo update-ca-certificates - -ENV SSL_CERT_FILE=/etc/ssl/certs/ca-certificates.crt -ENV SSL_CERT_DIR=/dev/null - # make the terminal prettier RUN echo 'export PS1="[\u@docker] \W # "' >> /root/.bash_profile