From 3c6bbd71751e173d936724500a3095a58fc1b446 Mon Sep 17 00:00:00 2001 From: tinaok Date: Fri, 3 Nov 2023 18:40:43 +0000 Subject: [PATCH] deploy: d76fc9dc4c387267fe111d97be1824dd8e8e2e47 --- ... openEO-Client_Side_Land_Cover_Alps.ipynb} | 0 ...nb => openEO-Corine_Land_Cover_Alps.ipynb} | 0 ...el_2.ipynb => openEO-boa_sentinel_2.ipynb} | 0 about/timeline.html | 6 +- afterword/envds-book.html | 6 +- afterword/pythia.html | 6 +- afterword/resources.html | 6 +- before/EOSC.html | 6 +- before/copernicus.html | 6 +- before/openeo.html | 6 +- before/setup.html | 6 +- genindex.html | 6 +- intro.html | 6 +- objects.inv | Bin 901 -> 879 bytes part1/motivation.html | 6 +- part1/overview.html | 6 +- part1/visualization.html | 66 +-- part1/xarray_pitfalls.html | 176 +++---- part2/advanced_workflows.html | 10 +- ...> openEO-Client_Side_Land_Cover_Alps.html} | 18 +- ...tml => openEO-Corine_Land_Cover_Alps.html} | 24 +- ...inel_2.html => openEO-boa_sentinel_2.html} | 20 +- part2/stac_metadata.html | 6 +- part3/advanced_udf.html | 6 +- part3/chunking_introduction.html | 192 +++---- part3/data_discovery.html | 6 +- part3/data_exploitability_openEO.html | 13 +- part3/data_exploitability_pangeo.html | 12 +- part3/scaling_dask.html | 478 ++++++------------ part3/scaling_openeo.html | 6 +- ...penEO-Client_Side_Land_Cover_Alps.err.log} | 0 ... => openEO-Corine_Land_Cover_Alps.err.log} | 0 ....err.log => openEO-boa_sentinel_2.err.log} | 0 search.html | 6 +- searchindex.js | 2 +- 35 files changed, 468 insertions(+), 645 deletions(-) rename _sources/part2/{openEO - Client Side Land Cover Alps.ipynb => openEO-Client_Side_Land_Cover_Alps.ipynb} (100%) rename _sources/part2/{openEO - Corine Land Cover Alps.ipynb => openEO-Corine_Land_Cover_Alps.ipynb} (100%) rename _sources/part2/{openEO - boa_sentinel_2.ipynb => openEO-boa_sentinel_2.ipynb} (100%) rename part2/{openEO - Client Side Land Cover Alps.html => openEO-Client_Side_Land_Cover_Alps.html} (98%) rename part2/{openEO - Corine Land Cover Alps.html => openEO-Corine_Land_Cover_Alps.html} (98%) rename part2/{openEO - boa_sentinel_2.html => openEO-boa_sentinel_2.html} (98%) rename reports/part2/{openEO - Client Side Land Cover Alps.err.log => openEO-Client_Side_Land_Cover_Alps.err.log} (100%) rename reports/part2/{openEO - Corine Land Cover Alps.err.log => openEO-Corine_Land_Cover_Alps.err.log} (100%) rename reports/part2/{openEO - boa_sentinel_2.err.log => openEO-boa_sentinel_2.err.log} (100%) diff --git a/_sources/part2/openEO - Client Side Land Cover Alps.ipynb b/_sources/part2/openEO-Client_Side_Land_Cover_Alps.ipynb similarity index 100% rename from _sources/part2/openEO - Client Side Land Cover Alps.ipynb rename to _sources/part2/openEO-Client_Side_Land_Cover_Alps.ipynb diff --git a/_sources/part2/openEO - Corine Land Cover Alps.ipynb b/_sources/part2/openEO-Corine_Land_Cover_Alps.ipynb similarity index 100% rename from _sources/part2/openEO - Corine Land Cover Alps.ipynb rename to _sources/part2/openEO-Corine_Land_Cover_Alps.ipynb diff --git a/_sources/part2/openEO - boa_sentinel_2.ipynb b/_sources/part2/openEO-boa_sentinel_2.ipynb similarity index 100% rename from _sources/part2/openEO - boa_sentinel_2.ipynb rename to _sources/part2/openEO-boa_sentinel_2.ipynb diff --git a/about/timeline.html b/about/timeline.html index 633c60f..2179734 100644 --- a/about/timeline.html +++ b/about/timeline.html @@ -190,9 +190,9 @@

Part 2 - OpenEO

diff --git a/afterword/envds-book.html b/afterword/envds-book.html index 3c6b9d6..03a62b6 100644 --- a/afterword/envds-book.html +++ b/afterword/envds-book.html @@ -189,9 +189,9 @@

Part 2 - OpenEO

diff --git a/afterword/pythia.html b/afterword/pythia.html index 35ce64e..95cac45 100644 --- a/afterword/pythia.html +++ b/afterword/pythia.html @@ -190,9 +190,9 @@

Part 2 - OpenEO

diff --git a/afterword/resources.html b/afterword/resources.html index 2cf9fd2..6f949b6 100644 --- a/afterword/resources.html +++ b/afterword/resources.html @@ -190,9 +190,9 @@

Part 2 - OpenEO

diff --git a/before/EOSC.html b/before/EOSC.html index c04cf27..c494ab1 100644 --- a/before/EOSC.html +++ b/before/EOSC.html @@ -190,9 +190,9 @@

Part 2 - OpenEO

diff --git a/before/copernicus.html b/before/copernicus.html index edd68c7..50488f3 100644 --- a/before/copernicus.html +++ b/before/copernicus.html @@ -190,9 +190,9 @@

Part 2 - OpenEO

diff --git a/before/openeo.html b/before/openeo.html index 981e65a..e693c50 100644 --- a/before/openeo.html +++ b/before/openeo.html @@ -190,9 +190,9 @@

Part 2 - OpenEO

diff --git a/before/setup.html b/before/setup.html index 6f13d5a..58b8686 100644 --- a/before/setup.html +++ b/before/setup.html @@ -190,9 +190,9 @@

Part 2 - OpenEO

diff --git a/genindex.html b/genindex.html index 222c315..e862f53 100644 --- a/genindex.html +++ b/genindex.html @@ -187,9 +187,9 @@

Part 2 - OpenEO

diff --git a/intro.html b/intro.html index b0b1cd4..57391ad 100644 --- a/intro.html +++ b/intro.html @@ -189,9 +189,9 @@

Part 2 - OpenEO

diff --git a/objects.inv b/objects.inv index 7a0770a6cb048e7efe0248b31cc93a3d4c281c1f..57f89d016ab40c91b0c8929aa9753a30f3c18aad 100644 GIT binary patch delta 776 zcmV+j1NZ!e2k!=ubbpOiOLE&V4Bh(_?4~={lXTZr)7Y7$(@Z>dKT8cQ!WL_a(2$fI z_X=J01ifO9P)Lw$$zq!9B)o_Bzz;yV#xl%;&cSF4!Uy$EVJfB=3s*~Vt+1!6suH&O+kDhQct-! zgiOo0Dr2m;boc|(AeK$6C*Z(i=~D1~Zfnyg`i4rYso)pcK^ z7j8?Cb_@8v4s@Ld8yZUpYZE%AkngBDtgQ4vM@%PW~Uy~eF zM6x(cJAiZUQI!T-@j|+AG0QRNN(PO#?{kiIMUlHI(}pdrBE62#s$m7L(y$+Ebx6B& z9k0Fae5wMD$G}OI_N6rX=M(h};A4s4Bvn{}C=7<6?SEG6b;!gm8vI6pBC-sN4|LOW5(}$E8Vo? z-G6&J!cuL{^@eDFdWfm`>cNRQY^Y`>Zc5wqmv2?mth772aEb0W-$*`6W1nWsb$F{< zuOLy{r}YsW#&Scvr_;scS{s_%%@pt;U7EF#zJCEcQVZ(PB1wA=akD75?K#|wK*NC< zqS^hX%X)1PS1EX`yvUKcPdn1C^OS1+d@AcfkSOVQV~0N4JzS2)2n}YM+C_PqOLJnJ zNsb?%7~Vc99P2LmYf;j0cg;}E5Y?pj20ZgP)Q>tXa_~m*(GC+i);(wqzLZ;-`~zt} GK>>vWGKkFp delta 798 zcmV+(1L6Gd289QZbbpmq&2rN)5WeRr!~t$0wA?sRl5|>zPBVm`qernev4|{rBspn# z1x`ExugD{?(#nY)yFd?}I=kP`>StGcWoke1vIHeHFz3Y^VRAOUV7!xgS@=@1PYuqj znJ<9tjorG!RBpuA9xVP86VN;1#w%mC9kqcxY0mXdTBA$Qo_{Mg*tSA zX~ckJbr?4Z;@lZgD@fuMxBg;M8ZUR;OQYL9=eedRbQg6V(Oe7EYXC{j?7;3M?E6|x zChc5EGb8RnRPJ<>gpl+iHjI*kBRv$c(tr>2kuwj$_k_EHw36nSIY} z2t$5XDb7w@>>WcFkEry6joDOTb9=Uw0@#OeJkzl0>`GMwhQxY)`D%=R#Ke!l#eabv z!hw(fgMR{*le`v+d%W8vo6Ya~Wf&U<*%K>>q$gOPyRHn+94-P?RCf6fw=Vc^`p)w_ zE5UOShDA@clO~<(tJ{z4$iS|~`*`e-*QbN7#rj;Yn{Ce+T;OMiJ2`_j)-2d{tz#Fu zq?#AC-r}{)WV=Os@`Et)iNmnTC)KcmOh}i9!+$vR<+}NnP7~uRsc^c-S>b@&yhxF@ z0X$%%N}n=WI)`DiL2T+wvu7Sh2q9S9{n(4GW*JtgX!!h>BViwR#9gOZmi%1K!??%< zciX;0JKH>O7KaUvZ=CH_eVSe~M4U

Part 2 - OpenEO

diff --git a/part1/overview.html b/part1/overview.html index 54d29c6..abd568b 100644 --- a/part1/overview.html +++ b/part1/overview.html @@ -190,9 +190,9 @@

Part 2 - OpenEO

diff --git a/part1/visualization.html b/part1/visualization.html index 87688ad..98e084a 100644 --- a/part1/visualization.html +++ b/part1/visualization.html @@ -67,7 +67,7 @@ - + @@ -190,9 +190,9 @@

Part 2 - OpenEO

@@ -970,7 +970,7 @@

Open local dataset @@ -1572,7 +1572,7 @@

Read a shapefile with the Area Of Interest (AOI) @@ -2196,11 +2196,11 @@

Visualization with HoloViews
-
+
- + - + @@ -190,8 +190,8 @@

Part 2 - OpenEO

Part 2 - OpenEO

@@ -269,7 +269,7 @@ -
  • Connect to openEO Platform using python -
    [-------------------------------------] ❌ Timed out
    ---------------------------------------------------------------------------
    +
    [-------------------------------------] ❌ Timed out
    ---------------------------------------------------------------------------
     OidcDeviceCodePollTimeout                 Traceback (most recent call last)
     Cell In[3], line 1
     ----> 1 conn = conn.authenticate_oidc()
    @@ -1005,7 +1005,7 @@ 

    Looking at the results using python matplotlib
    @@ -1014,7 +1014,7 @@

    Looking at the results using python matplotlib

    next

    diff --git a/part2/openEO - boa_sentinel_2.html b/part2/openEO-boa_sentinel_2.html similarity index 98% rename from part2/openEO - boa_sentinel_2.html rename to part2/openEO-boa_sentinel_2.html index f821bb4..6026c85 100644 --- a/part2/openEO - boa_sentinel_2.html +++ b/part2/openEO-boa_sentinel_2.html @@ -64,10 +64,10 @@ const thebe_selector_output = ".output, .cell_output" - + - + @@ -191,8 +191,8 @@

    Part 2 - OpenEO

    @@ -269,7 +269,7 @@ -
  • Connect to openEO Platform using python -
    [-------------------------------------] ❌ Timed out
    ---------------------------------------------------------------------------
    +
    [-------------------------------------] ❌ Timed out
    ---------------------------------------------------------------------------
     OidcDeviceCodePollTimeout                 Traceback (most recent call last)
     Cell In[3], line 1
     ----> 1 conn = conn.authenticate_oidc()
    @@ -1491,7 +1491,7 @@ 

    Look at the results using python matplotlib

    next

    diff --git a/part2/stac_metadata.html b/part2/stac_metadata.html index 2b74b0b..be92519 100644 --- a/part2/stac_metadata.html +++ b/part2/stac_metadata.html @@ -190,9 +190,9 @@

    Part 2 - OpenEO

    diff --git a/part3/advanced_udf.html b/part3/advanced_udf.html index 474e5bd..cedd8c4 100644 --- a/part3/advanced_udf.html +++ b/part3/advanced_udf.html @@ -190,9 +190,9 @@

    Part 2 - OpenEO

    diff --git a/part3/chunking_introduction.html b/part3/chunking_introduction.html index 30bbc08..4a23ede 100644 --- a/part3/chunking_introduction.html +++ b/part3/chunking_introduction.html @@ -64,7 +64,7 @@ const thebe_selector_output = ".output, .cell_output" - + @@ -193,9 +193,9 @@

    Part 2 - OpenEO

    @@ -606,8 +606,8 @@

    Global LTS -
    CPU times: user 328 ms, sys: 69.2 ms, total: 397 ms
    -Wall time: 2.06 s
    +
    CPU times: user 431 ms, sys: 71.1 ms, total: 502 ms
    +Wall time: 2.78 s
     
    @@ -999,9 +999,9 @@

    Global LTS @@ -1080,7 +1080,7 @@

    Global LTS @@ -1159,7 +1159,7 @@

    Global LTS @@ -1238,7 +1238,7 @@

    Global LTS @@ -1317,7 +1317,7 @@

    Global LTS @@ -1396,7 +1396,7 @@

    Global LTS @@ -1475,7 +1475,7 @@

    Global LTSopen_mfdataset automatically switch from Numpy Arrays to Dask Arrays as the data structure used by Xarray.

    test.data is the backend array Python representation of Xarray’s Data Array, Dask Array when using chunking, Numpy by default.

    We will introduce Dask arrays and Dask graphs visualization in the next section Scaling with Dask.

    @@ -3223,7 +3223,7 @@

    Zarr storage format - @@ -3787,9 +3787,9 @@

    Extract chunk information @@ -3868,7 +3868,7 @@

    Extract chunk information @@ -3947,7 +3947,7 @@

    Extract chunk information @@ -4026,7 +4026,7 @@

    Extract chunk information @@ -4105,7 +4105,7 @@

    Extract chunk information @@ -4184,7 +4184,7 @@

    Extract chunk information @@ -4263,7 +4263,7 @@

    Extract chunk information
    working on  foss4g-data/CGLS_LTS_1999_2019/c_gls_NDVI-LTS_1999-2019-0721_GLOBE_VGT-PROBAV_V3.0.1.nc
     

    -
    CPU times: user 872 ms, sys: 235 ms, total: 1.11 s
    -Wall time: 22.3 s
    +
    CPU times: user 1.19 s, sys: 249 ms, total: 1.44 s
    +Wall time: 24 s
     
    @@ -4398,8 +4398,8 @@

    We have 36 files to process, but for this chunking_introduction example, we

    -
    CPU times: user 34.7 ms, sys: 0 ns, total: 34.7 ms
    -Wall time: 33.6 ms
    +
    CPU times: user 30.8 ms, sys: 9.42 ms, total: 40.2 ms
    +Wall time: 40.8 ms
     
    @@ -4423,8 +4423,8 @@

    We have 36 files to process, but for this chunking_introduction example, we

    -
    CPU times: user 16.4 ms, sys: 4.42 ms, total: 20.8 ms
    -Wall time: 237 ms
    +
    CPU times: user 23 ms, sys: 501 µs, total: 23.5 ms
    +Wall time: 245 ms
     
    @@ -4817,9 +4817,9 @@

    We have 36 files to process, but for this chunking_introduction example, we source: Derived from EO satellite imagery time_coverage_end: 2019-12-31T23:59:59Z time_coverage_start: 1999-01-01T00:00:00Z - title: Normalized Difference Vegetation Index: Long Term S...

    + dtype='float64', name='lon', length=40320))

  • Conventions :
    CF-1.6
    archive_facility :
    VITO
    copyright :
    Copernicus Service information 2021
    history :
    2021-03-01 - Processing line NDVI LTS
    identifier :
    urn:cgls:global:ndvi_stats_all:NDVI-LTS_1999-2019-0701_GLOBE_V3.0.1
    institution :
    VITO NV
    long_name :
    Normalized Difference Vegetation Index
    orbit_type :
    LEO
    parent_identifier :
    urn:cgls:global:ndvi_stats_all
    platform :
    SPOT-4, SPOT-5, Proba-V
    processing_level :
    L4
    processing_mode :
    Offline
    product_version :
    V3.0.1
    references :
    https://land.copernicus.eu/global/products/ndvi
    sensor :
    VEGETATION-1, VEGETATION-2, VEGETATION
    source :
    Derived from EO satellite imagery
    time_coverage_end :
    2019-12-31T23:59:59Z
    time_coverage_start :
    1999-01-01T00:00:00Z
    title :
    Normalized Difference Vegetation Index: Long Term Statistics 1KM: GLOBE 1999-2019 0701
  • We can save the consolidated metadata for our dataset in a file, and reuse it later to access the dataset. We used json for next step, but we can also use parquet.

    @@ -5258,9 +5258,9 @@

    We have 36 files to process, but for this chunking_introduction example, we source: Derived from EO satellite imagery time_coverage_end: 2019-12-31T23:59:59Z time_coverage_start: 1999-01-01T00:00:00Z - title: Normalized Difference Vegetation Index: Long Term S... + dtype='float64', name='lon', length=40320))

  • Conventions :
    CF-1.6
    archive_facility :
    VITO
    copyright :
    Copernicus Service information 2021
    history :
    2021-03-01 - Processing line NDVI LTS
    identifier :
    urn:cgls:global:ndvi_stats_all:NDVI-LTS_1999-2019-0701_GLOBE_V3.0.1
    institution :
    VITO NV
    long_name :
    Normalized Difference Vegetation Index
    orbit_type :
    LEO
    parent_identifier :
    urn:cgls:global:ndvi_stats_all
    platform :
    SPOT-4, SPOT-5, Proba-V
    processing_level :
    L4
    processing_mode :
    Offline
    product_version :
    V3.0.1
    references :
    https://land.copernicus.eu/global/products/ndvi
    sensor :
    VEGETATION-1, VEGETATION-2, VEGETATION
    source :
    Derived from EO satellite imagery
    time_coverage_end :
    2019-12-31T23:59:59Z
    time_coverage_start :
    1999-01-01T00:00:00Z
    title :
    Normalized Difference Vegetation Index: Long Term Statistics 1KM: GLOBE 1999-2019 0701
  • The catalog (json file we created) can be shared on the cloud (or GitHub, etc.) and anyone can load it from there too.

    This approach allows anyone to easily access LTS data and select the Area of Interest for their own study.

    @@ -5691,11 +5691,11 @@

    We have 36 files to process, but for this chunking_introduction example, we source: Derived from EO satellite imagery time_coverage_end: 2019-12-31T23:59:59Z time_coverage_start: 1999-01-01T00:00:00Z - title: Normalized Difference Vegetation Index: Long Term S... + dtype='float64', name='time'))
  • Conventions :
    CF-1.6
    archive_facility :
    VITO
    copyright :
    Copernicus Service information 2021
    history :
    2021-03-01 - Processing line NDVI LTS
    identifier :
    urn:cgls:global:ndvi_stats_all:NDVI-LTS_1999-2019-0101_GLOBE_V3.0.1
    institution :
    VITO NV
    long_name :
    Normalized Difference Vegetation Index
    orbit_type :
    LEO
    parent_identifier :
    urn:cgls:global:ndvi_stats_all
    platform :
    SPOT-4, SPOT-5, Proba-V
    processing_level :
    L4
    processing_mode :
    Offline
    product_version :
    V3.0.1
    references :
    https://land.copernicus.eu/global/products/ndvi
    sensor :
    VEGETATION-1, VEGETATION-2, VEGETATION
    source :
    Derived from EO satellite imagery
    time_coverage_end :
    2019-12-31T23:59:59Z
    time_coverage_start :
    1999-01-01T00:00:00Z
    title :
    Normalized Difference Vegetation Index: Long Term Statistics 1KM: GLOBE 1999-2019 0101
  • The kerchunk catalogues can be placed in an intake catalogue, then loading multiple NetCDF file in the cloud can be just done in following 3 lines, chunked and fast.

    @@ -6118,11 +6118,11 @@

    We have 36 files to process, but for this chunking_introduction example, we source: Derived from EO satellite imagery time_coverage_end: 2019-12-31T23:59:59Z time_coverage_start: 1999-01-01T00:00:00Z - title: Normalized Difference Vegetation Index: Long Term S...