Merge pull request #1 from OWI-Lab/update/check_bad_fiber

Update/check bad fiber

sep005_io_fbgs/__init__.py

@@ -1,7 +1,3 @@
-"""
-A project template for the sdPy effort..
-"""
-
-__version__ = "0.0.1"
+__version__ = "0.0.2"
 
 from .fbgs import read_fbgs

sep005_io_fbgs/fbgs.py

@@ -21,9 +21,10 @@ def __init__(self, channels:list, error_status=None):
 
 
     @classmethod
-    def from_df(cls, df:pd.DataFrame, mode='engineering_units', dt_format='%Y-%m-%dT%H:%M:%S%z'):
+    def from_df(cls, df:pd.DataFrame, mode='engineering_units', dt_format='%Y-%m-%dT%H:%M:%S%z', qa:bool=True):
         """
         Import FBGS data from a dataframe
+
         """
         if mode == 'engineering_units' or mode == 'eu':
             pass # do Nothing
@@ -40,11 +41,24 @@ def from_df(cls, df:pd.DataFrame, mode='engineering_units', dt_format='%Y-%m-%dT
         timestamp = datetime.datetime.strptime(df['Date'][0], dt_format)
 
         # FBGS data sometimes has a bad timestamping, e.g. only to second precision.
-        for idx, row in df.iterrows():
-            timestamp_i = datetime.datetime.strptime(row['Date'], dt_format)
-            if timestamp_i > timestamp:
-                fs = idx/(timestamp_i-timestamp).total_seconds()
-                break
+        dt_e = datetime.datetime.strptime(df['Date'].iloc[-1], dt_format).replace(microsecond=0) \
+                + datetime.timedelta(seconds=1) # Floor to the second then add 1
+        duration = (dt_e-timestamp).total_seconds()  # Due to the rounding to second precision this
+        fs = round(len(df)/duration, 2) # Assumption: Sampling frequency is defined up to 2 decimal points
+
+
+
+        if duration*fs != len(df):
+            warnings.warn(
+                f'QA: Inconsistent number of samples found ({len(df)}) for estimated sampling frequency of {fs},'
+                f' set qa to False if you want to still consider this file.',
+                UserWarning
+            )
+            if qa:
+                # Failed QA: Returning empty
+                return cls(channels=[], error_status=error_status)
+
+
 
         # Convert timestamp to UTC
         timestamp = timestamp.astimezone(utc)
@@ -65,10 +79,11 @@ def from_df(cls, df:pd.DataFrame, mode='engineering_units', dt_format='%Y-%m-%dT
         return cls(channels=channels, error_status=error_status)
 
 
-def read_fbgs(path: Union[str, Path]) -> list:
+def read_fbgs(path: Union[str, Path], qa:bool = True) -> list:
     """
     Primary function to read fbgs files based on path
 
+    qa : Quality assurance applied, rejecting files with a bad length. When set to False this check is skipped
 
     """
     if not os.path.isfile(path):
@@ -77,7 +92,7 @@ def read_fbgs(path: Union[str, Path]) -> list:
         return signals
 
     df = _open_fbgs_file(path, sep='\t')
-    fbg = FBG.from_df(df)
+    fbg = FBG.from_df(df, qa=qa)
 
     return fbg.channels
 

tests/test_fbgs.py

@@ -8,6 +8,7 @@
 current_dir = os.path.dirname(os.path.abspath(__file__))
 static_dir = os.path.join(current_dir, 'static')
 GOOD_FILES = os.listdir(os.path.join(static_dir, 'good'))
+BAD_FILES = os.listdir(os.path.join(static_dir, 'bad'))
 
 
 @pytest.mark.parametrize("filename", GOOD_FILES)
@@ -21,3 +22,25 @@ def test_compliance_sep005(filename):
     assert len(signals) != 0  # Not an empty response
     assert_sep005(signals)
 
+    for s in signals:
+        assert s['fs'] == 20
+
+@pytest.mark.parametrize("filename", BAD_FILES)
+def test_bad_files(filename):
+    """
+    Test the behavior for files with bad data. Currently, the examples contain one sample too little
+    """
+    #%% Default behaviour with QA = True
+    file_path = os.path.join(static_dir, 'bad', filename)
+    signals = read_fbgs(file_path)  # By default does a quality assurance that should kick the file
+
+    assert signals == []
+
+    #%% QA turned off
+    signals = read_fbgs(file_path, qa=False)
+    assert len(signals) != 0  # Not an empty response
+    assert_sep005(signals)
+
+    for s in signals:
+        assert s['fs'] == 20
+        assert len(s['data']) == 11999