Fix-parser-logger-tests (#58)

* review each parser tests and flag when logger error is encountered * fix NMEA GPRMC missing datestamp handling * fail tests if logger.error is generated * fix some amundsen vocabulary issues * fix some onset issues
cioos-siooc · Nov 15, 2023 · 9c007bd · 9c007bd
1 parent a106e56
commit 9c007bd
Show file tree

Hide file tree

Showing 8 changed files with 12,199 additions and 113 deletions.
diff --git a/ocean_data_parser/parsers/amundsen.py b/ocean_data_parser/parsers/amundsen.py
@@ -108,6 +108,16 @@ def int_format(
             elif ":" in line:
                 key, value = line.strip()[1:].split(":", 1)
                 metadata[key.strip()] = value.strip()
+            elif line == "% Fluorescence [ug/L]":
+                metadata['Fluo'] = "Fluorescence [ug/L]"
+            elif line == "% Conservative Temperature (TEOS-10) [deg C]":
+                metadata['CONT'] = "Conservative Temperature (TEOS-10) [deg C]"
+            elif line == "% In situ density TEOS10 ((s, t, p) - 1000) [kg/m^3]":
+                metadata['D_CT'] = "In situ density TEOS10 ((s, t, p) - 1000) [kg/m^3]"
+            elif line == "% Potential density TEOS10 ((s, t, 0) - 1000) [kg/m^3]":
+                metadata['D0CT'] = "Potential density TEOS10 ((s, t, 0) - 1000) [kg/m^3]"
+            elif line == "% Potential density TEOS10 (s, t, 0) [kg/m^3]":
+                metadata['D0CT'] = "Potential density TEOS10 (s, t, 0) [kg/m^3]"
             elif re.match(r"% .* \[.+\]", line):
                 logger.warning(
                     "Unknown variable name will be saved to unknown_variables_information: %s",
@@ -191,7 +201,7 @@ def int_format(
             )
             ds["instrument_depth"] = -z_from_p(ds[pressure[0]], latitude)
 
-        # Map varibles to vocabulary
+        # Map variables to vocabulary
         variables_to_rename = {}
         for var in ds:
             if var not in variables:
@@ -215,7 +225,7 @@ def int_format(
                 if (
                     var_units is None  # Consider first if no units
                     or var_units == item.get("units")
-                    or (accepted_units and re.match(accepted_units, var_units))
+                    or (accepted_units and re.fullmatch(accepted_units, var_units))
                 ):
                     if "rename" in item:
                         variables_to_rename[var] = item["rename"]

diff --git a/ocean_data_parser/parsers/nmea.py b/ocean_data_parser/parsers/nmea.py
@@ -179,7 +179,7 @@ def _generate_extra_terms(nmea):
             f"{nmea['year']}-{nmea['month']}-{nmea['day']}T{nmea['timestamp']} UTC",
             f"%Y-%m-%dT%H%M%S{'.%f' if len(nmea['timestamp'])>6 else''} %Z",
         )
-    if nmea["sentence_type"] == "RMC":
+    if nmea["sentence_type"] == "RMC" and nmea.get('timestamp') and nmea.get('datestamp'):
         extra[("GPS Time", "gps_datetime")] = datetime.strptime(
             f"{nmea['datestamp']}T{nmea['timestamp']} UTC",
             f"%d%m%yT%H%M%S{'.%f' if len(nmea['timestamp'])>6 else''} %Z",

diff --git a/ocean_data_parser/parsers/onset.py b/ocean_data_parser/parsers/onset.py
@@ -67,7 +67,8 @@
     (r"^\d\d\-\d\d\-\d\d\s+\d{1,2}\:\d\d$", r"%y-%m-%d %H:%M"),
     (r"^\d\d\-\d\d\-\d\d\s+\d{1,2}\:\d\d\:\d\d$", r"%y-%m-%d %H:%M:%S"),
     (r"^\d\d\d\d\-\d\d\-\d\d\s+\d{1,2}\:\d\d$", r"%Y-%m-%d %H:%M"),
-    (r"^\d\d\/\d\d\/\d\d\d\d\s\d\d\:\d\d", r"%m/%d/%Y %H:%M"),
+    (r"^\d{1,2}\/\d{1,2}\/\d\d\d\d\s\d{1,2}\:\d\d", r"%m/%d/%Y %H:%M"),
+    (r"^\d\d\d\d\-\d\d\-\d\d\s+\d{1,2}:\d\d:\d\d (AM|PM)", r"%Y-%m-%d %I:%M:%S %p"),
 ]
 
 

diff --git a/ocean_data_parser/vocabularies/amundsen_vocabulary.json b/ocean_data_parser/vocabularies/amundsen_vocabulary.json
@@ -39,14 +39,16 @@
     {
       "long_name": "Mean Temperature (ITS-90)",
       "units": "degC",
-      "standard_name": "sea_water_temperature"
+      "standard_name": "sea_water_temperature",
+      "accepted_units": "degC|degrees C"
     }
   ],
   "Tem_std": [
     {
       "long_name": "Standard Deviation Temperature (ITS-90)",
       "units": "degC",
-      "standard_name": "sea_water_temperature"
+      "standard_name": "sea_water_temperature",
+      "accepted_units": "degC|degrees C"
     }
   ],
   "Sal_mean": [
@@ -75,7 +77,7 @@
   "Temp": [
     {
       "long_name": "Temperature (ITS-90)",
-      "accepted_units": "degC|deg C",
+      "accepted_units": "degC|deg C|degrees C",
       "units": "degC",
       "standard_name": "sea_water_temperature"
     }
@@ -140,14 +142,16 @@
     {
       "long_name": "Air Temperature",
       "units": "degC",
-      "standard_name": "air_temperature"
+      "standard_name": "air_temperature",
+      "accepted_units": "degC|deg C"
     }
   ],
   "Dew point": [
     {
       "long_name": "Dew Point",
       "units": "degC",
-      "standard_name": "dew_point_temperature"
+      "standard_name": "dew_point_temperature",
+      "accepted_units": "degC|deg C"
     }
   ],
   "Pressure": [
@@ -223,15 +227,17 @@
     {
       "long_name": "Conservative Temperature (TEOS-10)",
       "units": "degC",
-      "standard_name": "sea_water_conservative_temperature"
+      "standard_name": "sea_water_conservative_temperature",
+      "accepted_units": "degC|deg C"
     }
   ],
   "Cont": [
     {
       "long_name": "Conservative Temperature TEOS-10",
       "units": "degC",
       "standard_name": "sea_water_conservative_temperature",
-      "rename": "CONT"
+      "rename": "CONT",
+      "accepted_units": "degC|deg C"
     }
   ],
   "D_CT": [
@@ -241,13 +247,59 @@
       "standard_name": "sea_water_sigma_t"
     }
   ],
+  "D_ct": [
+    {
+      "long_name": "Density TEOS-10 ((S, T, P) - 1000)",
+      "units": "kg/m^3",
+      "standard_name": "sea_water_sigma_t",
+      "rename": "D_CT"
+    }
+  ],
+  "Sigt": [
+    {
+      "long_name": "Sigma-T (Rho(S, T, 0)-1000)",
+      "units": "kg/m^3",
+      "standard_name": "sea_water_sigma_t",
+      "rename": "sigt"
+    }
+  ],
+  "sigt": [
+    {
+      "long_name": "Density TEOS-10 ((S, T, P) - 1000)",
+      "units": "kg/m^3",
+      "standard_name": "sea_water_sigma_t"
+    }
+  ],
   "D0CT": [
     {
       "long_name": "Potential Density TEOS-10 ((S, T, 0) - 1000)",
       "units": "kg/m^3",
       "standard_name": "sea_water_sigma_theta"
     }
   ],
+  "D0ct": [
+    {
+      "long_name": "Density TEOS-10 (S, T, 0)",
+      "units": "kg/m^3",
+      "standard_name": "sea_water_sigma_theta",
+      "rename": "D0CT"
+    }
+  ],
+  "sigthe": [
+    {
+      "long_name": "Sigma-Theta (Rho(S, Theta, 0)-1000)",
+      "units": "kg/m^3",
+      "standard_name": "sea_water_sigma_theta"
+    }
+  ],
+  "Sigthe": [
+    {
+      "long_name": "Sigma-Theta (Rho(S, Theta, 0)-1000)",
+      "units": "kg/m^3",
+      "standard_name": "sea_water_sigma_theta",
+      "rename": "sigthe"
+    }
+  ],
   "Tem-TSG": [
     {
       "long_name": "Temperature From Thermo-Salinometer",
@@ -304,34 +356,14 @@
       "standard_name": "square_of_brunt_vaisala_frequency_in_sea_water"
     }
   ],
-  "Sigt": [
-    {
-      "long_name": "Sigma-T (Rho(S, T, 0)-1000)",
-      "units": "kg/m^3",
-      "standard_name": "sea_water_sigma_t",
-      "rename": "sigt"
-    }
-  ],
-  "sigt": [
-    {
-      "long_name": "Density TEOS-10 ((S, T, P) - 1000)",
-      "units": "kg/m^3",
-      "standard_name": "sea_water_sigma_t"
-    }
-  ],
-  "D_ct": [
-    {
-      "long_name": "Density TEOS-10 ((S, T, P) - 1000)",
-      "units": "kg/m^3",
-      "standard_name": "sea_water_sigma_t"
-    }
-  ],
+
   "Theta": [
     {
       "long_name": "Potential Temperature",
       "units": "degC",
       "standard_name": "sea_water_potential_temperature",
-      "rename": "theta"
+      "rename": "theta",
+      "accepted_units": "degC|deg C"
     }
   ],
   "theta": [
@@ -341,28 +373,6 @@
       "standard_name": "sea_water_potential_temperature"
     }
   ],
-  "sigthe": [
-    {
-      "long_name": "Sigma-Theta (Rho(S, Theta, 0)-1000)",
-      "units": "kg/m^3",
-      "standard_name": "sea_water_sigma_theta"
-    }
-  ],
-  "Sigthe": [
-    {
-      "long_name": "Sigma-Theta (Rho(S, Theta, 0)-1000)",
-      "units": "kg/m^3",
-      "standard_name": "sea_water_sigma_theta",
-      "rename": "sigthe"
-    }
-  ],
-  "D0ct": [
-    {
-      "long_name": "Density TEOS-10 (S, T, 0)",
-      "units": "kg/m^3",
-      "standard_name": "sea_water_sigma_theta"
-    }
-  ],
   "CDOM": [
     {
       "long_name": "Fluorescence Of CDOM (Colored Dissolved Organic Matter)",
@@ -420,7 +430,8 @@
     {
       "long_name": "Freezing Temperature",
       "units": "degC",
-      "standard_name": "freezing_temperature_of_sea_water"
+      "standard_name": "freezing_temperature_of_sea_water",
+      "accepted_units": "degC|deg C"
     }
   ],
   "FreezT-": [