Feature/cryoclim (#5)

* Make branch from Jostein's work

* Implement and test cryo

* Add cryoclim to suite based on option

* remove prints

* Add cryo_varname in config

* Remove list notation. Needed for json data set. What about the others...

* Token

* Not update documentation for PRs

* Not update documentation for PRs

.github/workflows/tests.yml

@@ -93,6 +93,8 @@ jobs:
 
       - name: Coveralls
         if: ${{ matrix.python-version == 3.8 }}
+        env:
+          GITHUB_TOKEN: ${{secrets.GITHUB_TOKEN}}
         run: |
           export COVERALLS_REPO_TOKEN=${{secrets.COVERALLS_REPO_TOKEN}}
           poetry run coveralls
@@ -102,7 +104,7 @@ jobs:
         run: |
           poetry run sphinx-build . docs/build/html
       - name: Commit documentation changes
-        if: ${{ matrix.python-version == 3.8 }}
+        if: ${{ matrix.python-version == 3.8 && github.event_name != 'pull_request' }}
         run: |
           git clone https://github.com/metno/pysurfex-experiment.git --branch gh-pages --single-branch gh-pages
           cp -r docs/build/html/* gh-pages/
@@ -115,7 +117,7 @@ jobs:
           # the return code.
 
       - name: Push changes
-        if: ${{ matrix.python-version == 3.8 }}
+        if: ${{ matrix.python-version == 3.8  && github.event_name != 'pull_request' }}
         uses: ad-m/github-push-action@master
         with:
           branch: gh-pages

data/config/config_exp_observations.toml

@@ -6,11 +6,18 @@
 synop_obs_t2m = true             # SYNOP 2m air temperature from bufr files
 synop_obs_rh2m = true            # SYNOP 2m relative humidity from bufr files
 synop_obs_sd = true              # SYNOP snow depth from bufr files
+cryo_obs_sd = false              # Use cryoclim snow pseudo-observations
 netatmo_obs_t2m = true           # 2m air temperature observations from Netatmo stations
 netatmo_obs_rh2m = true          # 2m relative humidity observations from Netatmo stations
 netatmo_filepattern = "@sfx_exp_data@/archive/observations/@YYYY@/@MM@/@DD@/@HH@/@RRR@"
 # netatmo_filepattern = "/lustre/storeB/project/metproduction/products/netatmo/@YYYY@/@MM@/@DD@/@YYYY@@MM@@DD@T@HH@@[email protected]"
 # netatmo_filepattern = "/tmp/host1/testdata/@YYYY@@MM@@DD@T@HH@@mm@*Z_all.json"
+cryo_filepattern = "@sfx_exp_data@/archive/observations/@YYYY@/@MM@/@DD@/@HH@/@RRR@/daily-avhrr-sce-nhl_ease-50_@YYYY@@MM@@DD@@HH@"
+cryo_laf_threshold = 0.1
+cryo_step = 2
+cryo_fg_threshold = 0.4
+cryo_new_snow = 0.1
+cryo_var_name = "classed_value_c"
 
 [observations.qc]
 # QC_TESTS = ["domain", "blacklist", "nometa", "redundancy", "plausibility", "sct"]

experiment/suites.py

@@ -493,6 +493,21 @@ def __init__(
                     )
                     fg4oi_complete = EcflowSuiteTrigger(fg4oi)
 
+                    cryo_obs_sd = config.get_value("observations.cryo_obs_sd")
+                    cryo2json_complete = fg4oi_complete
+                    if cryo_obs_sd:
+                        cryo_trigger = EcflowSuiteTriggers(fg4oi_complete)
+                        cryo2json = EcflowSuiteTask(
+                            "CryoClim2json",
+                            analysis,
+                            config,
+                            task_settings,
+                            ecf_files,
+                            triggers=cryo_trigger,
+                            input_template=template,
+                        )
+                        cryo2json_complete = EcflowSuiteTrigger(cryo2json)
+
                     fetchobs_complete = None
                     if platform_name == "ECMWF-atos":
                         if need_obs:
@@ -516,7 +531,11 @@ def __init__(
                             qc_triggers = None
                             if var == "sd":
                                 qc_triggers = EcflowSuiteTriggers(
-                                    [fg4oi_complete, fetchobs_complete]
+                                    [
+                                        fg4oi_complete,
+                                        fetchobs_complete,
+                                        cryo2json_complete,
+                                    ]
                                 )
                             else:
                                 qc_triggers = EcflowSuiteTriggers(fetchobs_complete)

experiment/tasks/tasks.py

@@ -20,6 +20,7 @@
     write_analysis_netcdf_file,
 )
 from pysurfex.obsmon import write_obsmon_sqlite_file
+from pysurfex.pseudoobs import CryoclimObservationSet
 from pysurfex.read import ConvertedInput, Converter
 from pysurfex.run import BatchJob
 from pysurfex.titan import TitanDataSet, dataset_from_file, define_quality_control
@@ -388,6 +389,7 @@ def execute(self):
         # Snow Depth
         elif self.var_name == "sd":
             synop_obs = self.config.get_value("observations.synop_obs_sd")
+            cryo_obs = self.config.get_value("observations.cryo_obs_sd")
             data_sets = {}
             if synop_obs:
                 bufr_tests = default_tests
@@ -409,6 +411,25 @@ def execute(self):
                     }
                 )
 
+            if cryo_obs:
+                cryo_tests = default_tests
+                cryo_tests.update(
+                    {
+                        "plausibility": {"do_test": True, "maxval": 1000, "minval": 0},
+                        "firstguess": {"do_test": True, "negdiff": 0.5, "posdiff": 0.5},
+                    }
+                )
+                filepattern = self.obsdir + "/cryo.json"
+                data_sets.update(
+                    {
+                        "cryo": {
+                            "filepattern": filepattern,
+                            "filetype": "json",
+                            "varname": "totalSnowDepth",
+                            "tests": cryo_tests,
+                        }
+                    }
+                )
             settings.update({"sets": data_sets})
         else:
             raise NotImplementedError
@@ -420,15 +441,13 @@ def execute(self):
 
         try:
             tests = self.config.get_value(f"observations.qc.{lname}.tests")
-            self.logger.info("Using observations.qc.{lname}.tests")
+            self.logger.info("Using observations.qc.%s.tests", lname)
         except AttributeError:
             self.logger.info("Using default test observations.qc.tests")
             tests = self.config.get_value("observations.qc.tests")
 
         indent = 2
         blacklist = {}
-        # TODO
-        an_time = an_time.replace(tzinfo=None)
         json.dump(settings, open("settings.json", mode="w", encoding="utf-8"), indent=2)
         tests = define_quality_control(
             tests, settings, an_time, domain_geo=self.geo, blacklist=blacklist
@@ -472,6 +491,7 @@ def execute(self):
         max_locations = 20
         elev_gradient = 0
         epsilon = 0.25
+        only_diff = False
 
         lname = self.var_name.lower()
         hlength = self.config.get_value(
@@ -492,6 +512,9 @@ def execute(self):
         epsilon = self.config.get_value(
             f"observations.oi.{lname}.epsilon", default=epsilon
         )
+        only_diff = self.config.get_value(
+            f"observations.oi.{lname}.only_diff", default=only_diff
+        )
         try:
             minvalue = self.config.get_value(
                 f"observations.oi.{lname}.minvalue", default=None
@@ -532,7 +555,8 @@ def execute(self):
             epsilon=epsilon,
             minvalue=minvalue,
             maxvalue=maxvalue,
-            interpol="nearest",
+            interpol="bilinear",
+            only_diff=only_diff,
         )
         self.logger.info("Write output file %s", output_file)
         if os.path.exists(output_file):
@@ -579,6 +603,75 @@ def execute(self):
         os.symlink(fg_file, self.fg_guess_sfx)
 
 
+class CryoClim2json(AbstractTask):
+    """Find first guess.
+
+    Args:
+        AbstractTask (AbstractTask): Base class
+
+    """
+
+    def __init__(self, config, name=None):
+        """Construct a FistGuess task.
+
+        Args:
+            config (ParsedObject): Parsed configuration
+            name (str, optional): Task name. Defaults to None
+
+        """
+        if name is None:
+            name = "CryoClim2json"
+        AbstractTask.__init__(self, config, name)
+        self.var_name = self.config.get_value("task.var_name", default=None)
+
+    def execute(self):
+        """Execute."""
+        var = "surface_snow_thickness"
+        input_file = self.archive + "/raw_" + var + ".nc"
+
+        # Get input fields
+        geo, validtime, background, glafs, gelevs = read_first_guess_netcdf_file(
+            input_file, var
+        )
+
+        obs_file = self.config.get_value("observations.cryo_filepattern")
+        obs_file = [self.platform.substitute(obs_file)]
+        try:
+            laf_threshold = self.config.get_value("observations.cryo_laf_threshold")
+        except AttributeError:
+            laf_threshold = 0.1
+        try:
+            step = self.config.get_value("observations.cryo_step")
+        except AttributeError:
+            step = 2
+        try:
+            fg_threshold = self.config.get_value("observations.cryo_fg_threshold")
+        except AttributeError:
+            fg_threshold = 0.4
+        try:
+            new_snow_depth = self.config.get_value("observations.cryo_new_snow")
+        except AttributeError:
+            new_snow_depth = 0.1
+        try:
+            cryo_varname = self.config.get_value("observations.cryo_varname")
+        except AttributeError:
+            cryo_varname = None
+        obs_set = CryoclimObservationSet(
+            [obs_file],
+            validtime,
+            geo,
+            background,
+            gelevs,
+            step=step,
+            fg_threshold=fg_threshold,
+            new_snow_depth=new_snow_depth,
+            glaf=glafs,
+            laf_threshold=laf_threshold,
+            cryo_varname=cryo_varname,
+        )
+        obs_set.write_json_file(f"{self.platform.get_system_value('obs_dir')}/cryo.json")
+
+
 class CycleFirstGuess(FirstGuess):
     """Cycle the first guess.
 

tests/unit/test_tasks.py

@@ -201,9 +201,15 @@ def new_read_first_guess_netcdf_file(*args, **kwargs):
         }
         geo = ConfProj(geo_dict)
         validtime = as_datetime("2023-01-01 T03:00:00Z")
-        dummy = np.empty([60, 50])
+        dummy = np.empty([50, 60])
         return geo, validtime, dummy, dummy, dummy
 
+    def new_read_cryoclim_nc(infiles, cryo_varname=""):
+        lons = np.ma.array([[10, 10, 10], [10.1, 10.1, 10.1]])
+        lats = np.ma.array([[60, 60, 60], [60.1, 60.1, 60.1]])
+        vals = np.ma.array([[[0, 1, 2], [3, 4, 5]]])
+        return lons, lats, vals
+
     def new_write_analysis_netcdf_file(*args, **kwargs):
         pass
 
@@ -250,6 +256,7 @@ def new_batchjob_run_method(self, cmd):
         "experiment.tasks.tasks.write_analysis_netcdf_file",
         new=new_write_analysis_netcdf_file,
     )
+    session_mocker.patch("pysurfex.pseudoobs.read_cryoclim_nc", new=new_read_cryoclim_nc)
     session_mocker.patch(
         "experiment.tasks.tasks.write_obsmon_sqlite_file",
         new=new_write_obsmon_sqlite_file,