From f488748d7105e64db11a3fd8b6626c6dbccf1201 Mon Sep 17 00:00:00 2001 From: annefou Date: Fri, 3 Nov 2023 20:50:46 +0000 Subject: [PATCH] deploy: f00c912c74a964a09c0397508f96f2a0799ff9ae --- part1/visualization.html | 56 ++--- part1/xarray_pitfalls.html | 170 +++++++-------- part3/chunking_introduction.html | 184 ++++++++-------- part3/data_exploitability_openEO.html | 5 +- part3/data_exploitability_pangeo.html | 6 +- part3/scaling_dask.html | 292 +++++++++++++------------- searchindex.js | 2 +- 7 files changed, 358 insertions(+), 357 deletions(-) diff --git a/part1/visualization.html b/part1/visualization.html index 1dd6e11..dcbec3b 100644 --- a/part1/visualization.html +++ b/part1/visualization.html @@ -970,7 +970,7 @@

Open local dataset @@ -1572,7 +1572,7 @@

Read a shapefile with the Area Of Interest (AOI) @@ -2196,11 +2196,11 @@

Visualization with HoloViews
-
+
- + @@ -606,8 +606,8 @@

Global LTS -
CPU times: user 339 ms, sys: 58.4 ms, total: 397 ms
-Wall time: 4.09 s
+
CPU times: user 344 ms, sys: 65.2 ms, total: 409 ms
+Wall time: 2.91 s
 

 

 

@@ -999,9 +999,9 @@ 
Global LTS
@@ -1080,7 +1080,7 @@ 
Global LTS
@@ -1159,7 +1159,7 @@ 
Global LTS
@@ -1238,7 +1238,7 @@ 
Global LTS
@@ -1317,7 +1317,7 @@ 
Global LTS
@@ -1396,7 +1396,7 @@ 
Global LTS
@@ -1475,7 +1475,7 @@ 
Global LTSopen_mfdataset automatically switch from Numpy Arrays to Dask Arrays as the data structure used by Xarray.

 

@@ -1925,9 +1925,9 @@ 
Using chunks within a single file
@@ -1984,7 +1984,7 @@ 
Using chunks within a single file
@@ -2041,7 +2041,7 @@ 
Using chunks within a single file
@@ -2098,7 +2098,7 @@ 
Using chunks within a single file
@@ -2155,7 +2155,7 @@ 
Using chunks within a single file
@@ -2212,7 +2212,7 @@ 
Using chunks within a single file
@@ -2269,7 +2269,7 @@ 
Using chunks within a single file
@@ -2687,7 +2687,7 @@ 
Using chunks within a single file(1121, 2240).  We will chunk it on 600x600 pieces using the following command.

 

@@ -3089,7 +3089,7 @@ 
Using chunks within a single file
@@ -3148,7 +3148,7 @@ 
Using chunks within a single file
-

+

 

 
 test.data is the backend array Python representation of Xarray’s Data Array, Dask Array when using chunking, Numpy by default.

 
We will introduce Dask arrays and Dask graphs visualization in the next section Scaling with Dask.

@@ -3223,7 +3223,7 @@ 
Zarr storage format
-
<xarray.backends.zarr.ZarrStore at 0x7f19ae44e9d0>
+
<xarray.backends.zarr.ZarrStore at 0x7f13b1639cb0>
 

 

 

@@ -3262,18 +3262,18 @@ 
Zarr storage format
 
total 48
-drwxr-xr-x 2 runner docker 4096 Nov  3 20:30 .
-drwxr-xr-x 5 runner docker 4096 Nov  3 20:30 ..
--rw-r--r-- 1 runner docker  341 Nov  3 20:30 .zarray
--rw-r--r-- 1 runner docker  304 Nov  3 20:30 .zattrs
--rw-r--r-- 1 runner docker 1482 Nov  3 20:30 0.0
--rw-r--r-- 1 runner docker 1482 Nov  3 20:30 0.1
--rw-r--r-- 1 runner docker 1482 Nov  3 20:30 0.2
--rw-r--r-- 1 runner docker 1482 Nov  3 20:30 0.3
--rw-r--r-- 1 runner docker 1482 Nov  3 20:30 1.0
--rw-r--r-- 1 runner docker 1482 Nov  3 20:30 1.1
--rw-r--r-- 1 runner docker 1482 Nov  3 20:30 1.2
--rw-r--r-- 1 runner docker 1482 Nov  3 20:30 1.3
+drwxr-xr-x 2 runner docker 4096 Nov  3 20:48 .
+drwxr-xr-x 5 runner docker 4096 Nov  3 20:48 ..
+-rw-r--r-- 1 runner docker  341 Nov  3 20:48 .zarray
+-rw-r--r-- 1 runner docker  304 Nov  3 20:48 .zattrs
+-rw-r--r-- 1 runner docker 1482 Nov  3 20:48 0.0
+-rw-r--r-- 1 runner docker 1482 Nov  3 20:48 0.1
+-rw-r--r-- 1 runner docker 1482 Nov  3 20:48 0.2
+-rw-r--r-- 1 runner docker 1482 Nov  3 20:48 0.3
+-rw-r--r-- 1 runner docker 1482 Nov  3 20:48 1.0
+-rw-r--r-- 1 runner docker 1482 Nov  3 20:48 1.1
+-rw-r--r-- 1 runner docker 1482 Nov  3 20:48 1.2
+-rw-r--r-- 1 runner docker 1482 Nov  3 20:48 1.3
 

 

 

@@ -3787,9 +3787,9 @@ 
Extract chunk information
@@ -3868,7 +3868,7 @@ 
Extract chunk information
@@ -3947,7 +3947,7 @@ 
Extract chunk information
@@ -4026,7 +4026,7 @@ 
Extract chunk information
@@ -4105,7 +4105,7 @@ 
Extract chunk information
@@ -4184,7 +4184,7 @@ 
Extract chunk information
@@ -4263,7 +4263,7 @@ 
Extract chunk information
working on  foss4g-data/CGLS_LTS_1999_2019/c_gls_NDVI-LTS_1999-2019-0721_GLOBE_VGT-PROBAV_V3.0.1.nc
 

 

-
CPU times: user 931 ms, sys: 241 ms, total: 1.17 s
-Wall time: 36 s
+
CPU times: user 937 ms, sys: 301 ms, total: 1.24 s
+Wall time: 36.1 s
 

 

 

@@ -4398,8 +4398,8 @@ 
We have 36 files to process, but for this chunking_introduction example, we
-
CPU times: user 27.8 ms, sys: 3.77 ms, total: 31.5 ms
-Wall time: 31 ms
+
CPU times: user 32.8 ms, sys: 0 ns, total: 32.8 ms
+Wall time: 31.8 ms
@@ -4423,7 +4423,7 @@

We have 36 files to process, but for this chunking_introduction example, we

-
CPU times: user 16.6 ms, sys: 2.7 ms, total: 19.3 ms
+
CPU times: user 18 ms, sys: 435 µs, total: 18.4 ms
 Wall time: 366 ms
 

 

@@ -4817,9 +4817,9 @@ 
We have 36 files to process, but for this chunking_introduction example, we
     source:               Derived from EO satellite imagery
     time_coverage_end:    2019-12-31T23:59:59Z
     time_coverage_start:  1999-01-01T00:00:00Z
-    title:                Normalized Difference Vegetation Index: Long Term S...
+ dtype='float64', name='lon', length=40320))
  • Conventions :
    CF-1.6
    archive_facility :
    VITO
    copyright :
    Copernicus Service information 2021
    history :
    2021-03-01 - Processing line NDVI LTS
    identifier :
    urn:cgls:global:ndvi_stats_all:NDVI-LTS_1999-2019-0701_GLOBE_V3.0.1
    institution :
    VITO NV
    long_name :
    Normalized Difference Vegetation Index
    orbit_type :
    LEO
    parent_identifier :
    urn:cgls:global:ndvi_stats_all
    platform :
    SPOT-4, SPOT-5, Proba-V
    processing_level :
    L4
    processing_mode :
    Offline
    product_version :
    V3.0.1
    references :
    https://land.copernicus.eu/global/products/ndvi
    sensor :
    VEGETATION-1, VEGETATION-2, VEGETATION
    source :
    Derived from EO satellite imagery
    time_coverage_end :
    2019-12-31T23:59:59Z
    time_coverage_start :
    1999-01-01T00:00:00Z
    title :
    Normalized Difference Vegetation Index: Long Term Statistics 1KM: GLOBE 1999-2019 0701

    • We can save the consolidated metadata for our dataset in a file, and reuse it later to access the dataset. We used json for next step, but we can also use parquet.

    @@ -5258,9 +5258,9 @@

    We have 36 files to process, but for this chunking_introduction example, we source: Derived from EO satellite imagery time_coverage_end: 2019-12-31T23:59:59Z time_coverage_start: 1999-01-01T00:00:00Z - title: Normalized Difference Vegetation Index: Long Term S... + dtype='float64', name='lon', length=40320))

  • Conventions :
    CF-1.6
    archive_facility :
    VITO
    copyright :
    Copernicus Service information 2021
    history :
    2021-03-01 - Processing line NDVI LTS
    identifier :
    urn:cgls:global:ndvi_stats_all:NDVI-LTS_1999-2019-0701_GLOBE_V3.0.1
    institution :
    VITO NV
    long_name :
    Normalized Difference Vegetation Index
    orbit_type :
    LEO
    parent_identifier :
    urn:cgls:global:ndvi_stats_all
    platform :
    SPOT-4, SPOT-5, Proba-V
    processing_level :
    L4
    processing_mode :
    Offline
    product_version :
    V3.0.1
    references :
    https://land.copernicus.eu/global/products/ndvi
    sensor :
    VEGETATION-1, VEGETATION-2, VEGETATION
    source :
    Derived from EO satellite imagery
    time_coverage_end :
    2019-12-31T23:59:59Z
    time_coverage_start :
    1999-01-01T00:00:00Z
    title :
    Normalized Difference Vegetation Index: Long Term Statistics 1KM: GLOBE 1999-2019 0701

    • The catalog (json file we created) can be shared on the cloud (or GitHub, etc.) and anyone can load it from there too.

    This approach allows anyone to easily access LTS data and select the Area of Interest for their own study.

    @@ -5691,11 +5691,11 @@

    We have 36 files to process, but for this chunking_introduction example, we source: Derived from EO satellite imagery time_coverage_end: 2019-12-31T23:59:59Z time_coverage_start: 1999-01-01T00:00:00Z - title: Normalized Difference Vegetation Index: Long Term S... + dtype='float64', name='time'))
  • Conventions :
    CF-1.6
    archive_facility :
    VITO
    copyright :
    Copernicus Service information 2021
    history :
    2021-03-01 - Processing line NDVI LTS
    identifier :
    urn:cgls:global:ndvi_stats_all:NDVI-LTS_1999-2019-0101_GLOBE_V3.0.1
    institution :
    VITO NV
    long_name :
    Normalized Difference Vegetation Index
    orbit_type :
    LEO
    parent_identifier :
    urn:cgls:global:ndvi_stats_all
    platform :
    SPOT-4, SPOT-5, Proba-V
    processing_level :
    L4
    processing_mode :
    Offline
    product_version :
    V3.0.1
    references :
    https://land.copernicus.eu/global/products/ndvi
    sensor :
    VEGETATION-1, VEGETATION-2, VEGETATION
    source :
    Derived from EO satellite imagery
    time_coverage_end :
    2019-12-31T23:59:59Z
    time_coverage_start :
    1999-01-01T00:00:00Z
    title :
    Normalized Difference Vegetation Index: Long Term Statistics 1KM: GLOBE 1999-2019 0101

    • The kerchunk catalogues can be placed in an intake catalogue, then loading multiple NetCDF file in the cloud can be just done in following 3 lines, chunked and fast.

    @@ -6118,11 +6118,11 @@

    We have 36 files to process, but for this chunking_introduction example, we source: Derived from EO satellite imagery time_coverage_end: 2019-12-31T23:59:59Z time_coverage_start: 1999-01-01T00:00:00Z - title: Normalized Difference Vegetation Index: Long Term S...