diff --git a/_sources/part1/visualization.ipynb b/_sources/part1/visualization.ipynb index 8c8afaf..3dfdb07 100644 --- a/_sources/part1/visualization.ipynb +++ b/_sources/part1/visualization.ipynb @@ -22,7 +22,7 @@ "### Contributors\n", "\n", "- Alejandro Coca-Castro, The Alan Turing Institute (United Kingdom), [@acocac](https://github.com/acocac)\n", - "- Anne Fouilloux, University of Oslo (Norway), [@annefou](https://github.com/annefou)" + "- Anne Fouilloux, Simula Research Laboratory (Norway), [@annefou](https://github.com/annefou)" ] }, { diff --git a/_sources/part1/xarray_pitfalls.ipynb b/_sources/part1/xarray_pitfalls.ipynb index cecf53b..e7c80da 100644 --- a/_sources/part1/xarray_pitfalls.ipynb +++ b/_sources/part1/xarray_pitfalls.ipynb @@ -22,7 +22,7 @@ "### Contributors\n", "\n", "- Alejandro Coca-Castro, The Alan Turing Institute (United Kingdom), [@acocac](https://github.com/acocac)\n", - "- Anne Fouilloux, University of Oslo (Norway), [@annefou](https://github.com/annefou)\n", + "- Anne Fouilloux, Simula Research Laboratory (Norway), [@annefou](https://github.com/annefou)\n", "- Guillaume Eynard-Bontemps, CNES (France), [@guillaumeeb](https://github.com/guillaumeeb)" ] }, diff --git a/_sources/part3/data_exploitability_pangeo.ipynb b/_sources/part3/data_exploitability_pangeo.ipynb index b80a0ea..70cec08 100644 --- a/_sources/part3/data_exploitability_pangeo.ipynb +++ b/_sources/part3/data_exploitability_pangeo.ipynb @@ -25,7 +25,7 @@ "\n", "### Contributors\n", "- Alejandro Coca-Castro, The Alan Turing Institure, [acocac](https://github.com/acocac) (author)\n", - "- Anne Fouilloux, University of Oslo (Norway), [@annefou](https://github.com/annefou)\n", + "- Anne Fouilloux, Simula Research Laboratory (Norway), [@annefou](https://github.com/annefou)\n", "- Justus Magin, UMR-LOPS CNRS(France), [@justusmagin](https://github.com/justusmagin)\n", "- Tina Odaka, UMR-LOPS Ifremer (France), [@tinaok](https://github.com/tinaok)\n", "\n", diff --git a/_sources/part3/scaling_dask.ipynb b/_sources/part3/scaling_dask.ipynb index b8441b8..c3719be 100644 --- a/_sources/part3/scaling_dask.ipynb +++ b/_sources/part3/scaling_dask.ipynb @@ -20,7 +20,7 @@ "\n", "### Contributors\n", "- Alejandro Coca-Castro, The Alan Turing Institute, [acocac](https://github.com/acocac)\n", - "- Anne Fouilloux, University of Oslo (Norway), [@annefou](https://github.com/annefou)\n", + "- Anne Fouilloux, Simula Research Laboratory (Norway), [@annefou](https://github.com/annefou)\n", "- Guillaume Eynard-Bontemps, CNES (France), [@guillaumeeb](https://github.com/guillaumeeb)" ] }, diff --git a/part1/visualization.html b/part1/visualization.html index 2a0197f..a736509 100644 --- a/part1/visualization.html +++ b/part1/visualization.html @@ -507,7 +507,7 @@

Authors#

Overview @@ -971,7 +971,7 @@

Open local dataset @@ -1573,7 +1573,7 @@

Read a shapefile with the Area Of Interest (AOI) @@ -2197,11 +2197,11 @@

Visualization with HoloViews
-
+
- + @@ -607,8 +607,8 @@

Global LTS -
CPU times: user 300 ms, sys: 88.6 ms, total: 389 ms
-Wall time: 1.95 s
+
CPU times: user 341 ms, sys: 51.7 ms, total: 392 ms
+Wall time: 2.74 s
 

 

 

@@ -1000,9 +1000,9 @@ 
Global LTS
@@ -1081,7 +1081,7 @@ 
Global LTS
@@ -1160,7 +1160,7 @@ 
Global LTS
@@ -1239,7 +1239,7 @@ 
Global LTS
@@ -1318,7 +1318,7 @@ 
Global LTS
@@ -1397,7 +1397,7 @@ 
Global LTS
@@ -1476,7 +1476,7 @@ 
Global LTSopen_mfdataset automatically switch from Numpy Arrays to Dask Arrays as the data structure used by Xarray.

 

@@ -1926,9 +1926,9 @@ 
Using chunks within a single file
@@ -1985,7 +1985,7 @@ 
Using chunks within a single file
@@ -2042,7 +2042,7 @@ 
Using chunks within a single file
@@ -2099,7 +2099,7 @@ 
Using chunks within a single file
@@ -2156,7 +2156,7 @@ 
Using chunks within a single file
@@ -2213,7 +2213,7 @@ 
Using chunks within a single file
@@ -2270,7 +2270,7 @@ 
Using chunks within a single file
@@ -2688,7 +2688,7 @@ 
Using chunks within a single file(1121, 2240).  We will chunk it on 600x600 pieces using the following command.

 

@@ -3090,7 +3090,7 @@ 
Using chunks within a single file
@@ -3149,7 +3149,7 @@ 
Using chunks within a single file
-

+

 

 
 test.data is the backend array Python representation of Xarray’s Data Array, Dask Array when using chunking, Numpy by default.

 
We will introduce Dask arrays and Dask graphs visualization in the next section Scaling with Dask.

@@ -3224,7 +3224,7 @@ 
Zarr storage format
-
<xarray.backends.zarr.ZarrStore at 0x7f1ed4cf5770>
+
<xarray.backends.zarr.ZarrStore at 0x7fcdec141d20>
 

 

 

@@ -3263,18 +3263,18 @@ 
Zarr storage format
 
total 48
-drwxr-xr-x 2 runner docker 4096 Nov  3 21:40 .
-drwxr-xr-x 5 runner docker 4096 Nov  3 21:40 ..
--rw-r--r-- 1 runner docker  341 Nov  3 21:40 .zarray
--rw-r--r-- 1 runner docker  304 Nov  3 21:40 .zattrs
--rw-r--r-- 1 runner docker 1482 Nov  3 21:40 0.0
--rw-r--r-- 1 runner docker 1482 Nov  3 21:40 0.1
--rw-r--r-- 1 runner docker 1482 Nov  3 21:40 0.2
--rw-r--r-- 1 runner docker 1482 Nov  3 21:40 0.3
--rw-r--r-- 1 runner docker 1482 Nov  3 21:40 1.0
--rw-r--r-- 1 runner docker 1482 Nov  3 21:40 1.1
--rw-r--r-- 1 runner docker 1482 Nov  3 21:40 1.2
--rw-r--r-- 1 runner docker 1482 Nov  3 21:40 1.3
+drwxr-xr-x 2 runner docker 4096 Nov  3 21:58 .
+drwxr-xr-x 5 runner docker 4096 Nov  3 21:58 ..
+-rw-r--r-- 1 runner docker  341 Nov  3 21:58 .zarray
+-rw-r--r-- 1 runner docker  304 Nov  3 21:58 .zattrs
+-rw-r--r-- 1 runner docker 1482 Nov  3 21:58 0.0
+-rw-r--r-- 1 runner docker 1482 Nov  3 21:58 0.1
+-rw-r--r-- 1 runner docker 1482 Nov  3 21:58 0.2
+-rw-r--r-- 1 runner docker 1482 Nov  3 21:58 0.3
+-rw-r--r-- 1 runner docker 1482 Nov  3 21:58 1.0
+-rw-r--r-- 1 runner docker 1482 Nov  3 21:58 1.1
+-rw-r--r-- 1 runner docker 1482 Nov  3 21:58 1.2
+-rw-r--r-- 1 runner docker 1482 Nov  3 21:58 1.3
 

 

 

@@ -3788,9 +3788,9 @@ 
Extract chunk information
@@ -3869,7 +3869,7 @@ 
Extract chunk information
@@ -3948,7 +3948,7 @@ 
Extract chunk information
@@ -4027,7 +4027,7 @@ 
Extract chunk information
@@ -4106,7 +4106,7 @@ 
Extract chunk information
@@ -4185,7 +4185,7 @@ 
Extract chunk information
@@ -4264,7 +4264,7 @@ 
Extract chunk information
working on  foss4g-data/CGLS_LTS_1999_2019/c_gls_NDVI-LTS_1999-2019-0721_GLOBE_VGT-PROBAV_V3.0.1.nc
 

 

-
CPU times: user 933 ms, sys: 269 ms, total: 1.2 s
-Wall time: 23.1 s
+
CPU times: user 894 ms, sys: 331 ms, total: 1.23 s
+Wall time: 26 s
 

 

 

@@ -4399,8 +4399,8 @@ 
We have 36 files to process, but for this chunking_introduction example, we
-
CPU times: user 30.8 ms, sys: 3.24 ms, total: 34.1 ms
-Wall time: 33.1 ms
+
CPU times: user 30.8 ms, sys: 0 ns, total: 30.8 ms
+Wall time: 30.1 ms
@@ -4424,8 +4424,8 @@

We have 36 files to process, but for this chunking_introduction example, we

-
CPU times: user 17 ms, sys: 0 ns, total: 17 ms
-Wall time: 243 ms
+
CPU times: user 17.2 ms, sys: 0 ns, total: 17.2 ms
+Wall time: 262 ms
 

 

 

@@ -4818,9 +4818,9 @@ 
We have 36 files to process, but for this chunking_introduction example, we
     source:               Derived from EO satellite imagery
     time_coverage_end:    2019-12-31T23:59:59Z
     time_coverage_start:  1999-01-01T00:00:00Z
-    title:                Normalized Difference Vegetation Index: Long Term S...
+ dtype='float64', name='lon', length=40320))
  • Conventions :
    CF-1.6
    archive_facility :
    VITO
    copyright :
    Copernicus Service information 2021
    history :
    2021-03-01 - Processing line NDVI LTS
    identifier :
    urn:cgls:global:ndvi_stats_all:NDVI-LTS_1999-2019-0701_GLOBE_V3.0.1
    institution :
    VITO NV
    long_name :
    Normalized Difference Vegetation Index
    orbit_type :
    LEO
    parent_identifier :
    urn:cgls:global:ndvi_stats_all
    platform :
    SPOT-4, SPOT-5, Proba-V
    processing_level :
    L4
    processing_mode :
    Offline
    product_version :
    V3.0.1
    references :
    https://land.copernicus.eu/global/products/ndvi
    sensor :
    VEGETATION-1, VEGETATION-2, VEGETATION
    source :
    Derived from EO satellite imagery
    time_coverage_end :
    2019-12-31T23:59:59Z
    time_coverage_start :
    1999-01-01T00:00:00Z
    title :
    Normalized Difference Vegetation Index: Long Term Statistics 1KM: GLOBE 1999-2019 0701

    • We can save the consolidated metadata for our dataset in a file, and reuse it later to access the dataset. We used json for next step, but we can also use parquet.

    @@ -5259,9 +5259,9 @@

    We have 36 files to process, but for this chunking_introduction example, we source: Derived from EO satellite imagery time_coverage_end: 2019-12-31T23:59:59Z time_coverage_start: 1999-01-01T00:00:00Z - title: Normalized Difference Vegetation Index: Long Term S... + dtype='float64', name='lon', length=40320))

  • Conventions :
    CF-1.6
    archive_facility :
    VITO
    copyright :
    Copernicus Service information 2021
    history :
    2021-03-01 - Processing line NDVI LTS
    identifier :
    urn:cgls:global:ndvi_stats_all:NDVI-LTS_1999-2019-0701_GLOBE_V3.0.1
    institution :
    VITO NV
    long_name :
    Normalized Difference Vegetation Index
    orbit_type :
    LEO
    parent_identifier :
    urn:cgls:global:ndvi_stats_all
    platform :
    SPOT-4, SPOT-5, Proba-V
    processing_level :
    L4
    processing_mode :
    Offline
    product_version :
    V3.0.1
    references :
    https://land.copernicus.eu/global/products/ndvi
    sensor :
    VEGETATION-1, VEGETATION-2, VEGETATION
    source :
    Derived from EO satellite imagery
    time_coverage_end :
    2019-12-31T23:59:59Z
    time_coverage_start :
    1999-01-01T00:00:00Z
    title :
    Normalized Difference Vegetation Index: Long Term Statistics 1KM: GLOBE 1999-2019 0701

    • The catalog (json file we created) can be shared on the cloud (or GitHub, etc.) and anyone can load it from there too.

    This approach allows anyone to easily access LTS data and select the Area of Interest for their own study.

    @@ -5692,11 +5692,11 @@

    We have 36 files to process, but for this chunking_introduction example, we source: Derived from EO satellite imagery time_coverage_end: 2019-12-31T23:59:59Z time_coverage_start: 1999-01-01T00:00:00Z - title: Normalized Difference Vegetation Index: Long Term S... + dtype='float64', name='time'))
  • Conventions :
    CF-1.6
    archive_facility :
    VITO
    copyright :
    Copernicus Service information 2021
    history :
    2021-03-01 - Processing line NDVI LTS
    identifier :
    urn:cgls:global:ndvi_stats_all:NDVI-LTS_1999-2019-0101_GLOBE_V3.0.1
    institution :
    VITO NV
    long_name :
    Normalized Difference Vegetation Index
    orbit_type :
    LEO
    parent_identifier :
    urn:cgls:global:ndvi_stats_all
    platform :
    SPOT-4, SPOT-5, Proba-V
    processing_level :
    L4
    processing_mode :
    Offline
    product_version :
    V3.0.1
    references :
    https://land.copernicus.eu/global/products/ndvi
    sensor :
    VEGETATION-1, VEGETATION-2, VEGETATION
    source :
    Derived from EO satellite imagery
    time_coverage_end :
    2019-12-31T23:59:59Z
    time_coverage_start :
    1999-01-01T00:00:00Z
    title :
    Normalized Difference Vegetation Index: Long Term Statistics 1KM: GLOBE 1999-2019 0101

    • The kerchunk catalogues can be placed in an intake catalogue, then loading multiple NetCDF file in the cloud can be just done in following 3 lines, chunked and fast.

    @@ -6119,11 +6119,11 @@

    We have 36 files to process, but for this chunking_introduction example, we source: Derived from EO satellite imagery time_coverage_end: 2019-12-31T23:59:59Z time_coverage_start: 1999-01-01T00:00:00Z - title: Normalized Difference Vegetation Index: Long Term S...