Merge pull request #76 from biomarkersParkinson/tremor_classification

Tremor classification

docs/notebooks/tremor/tremor_analysis.ipynb

@@ -16,8 +16,8 @@
     "import os\n",
     "from paradigma.preprocessing_config import IMUPreprocessingConfig\n",
     "from paradigma.imu_preprocessing import preprocess_imu_data_io\n",
-    "from paradigma.tremor.tremor_analysis_config import TremorFeatureExtractionConfig\n",
-    "from paradigma.tremor.tremor_analysis import extract_tremor_features_io"
+    "from paradigma.tremor.tremor_analysis_config import TremorFeatureExtractionConfig, TremorDetectionConfig\n",
+    "from paradigma.tremor.tremor_analysis import extract_tremor_features_io, detect_tremor_io"
    ]
   },
   {
@@ -67,6 +67,23 @@
     "config = TremorFeatureExtractionConfig()\n",
     "extract_tremor_features_io(path_to_preprocessed_data, path_to_extracted_features, config)"
    ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "## Detect tremor"
+   ]
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "config = TremorDetectionConfig()\n",
+    "detect_tremor_io(path_to_extracted_features, path_to_predictions, path_to_classifier, config)"
+   ]
   }
  ],
  "metadata": {

src/paradigma/constants.py

@@ -28,6 +28,11 @@ class DataColumns():
     VELOCITY : str = "velocity"
     SEGMENT_NR : str = "segment_nr"
 
+    # The following are used in tremor analysis
+    PRED_TREMOR_PROBA: str = "pred_tremor_proba"
+    PRED_TREMOR_LOGREG : str = "pred_tremor_logreg"
+    PRED_TREMOR_CHECKED : str = "pred_tremor_checked"
+
     # Constants for PPG features
     VARIANCE: str = "variance"
     MEAN: str = "mean"

src/paradigma/tremor/tremor_analysis.py

@@ -1,11 +1,13 @@
+import os
 import tsdf
 import pandas as pd
+import numpy as np
 from pathlib import Path
 from typing import Union
 from sklearn.linear_model import LogisticRegression
 
 from paradigma.constants import DataColumns
-from paradigma.tremor.tremor_analysis_config import TremorFeatureExtractionConfig
+from paradigma.tremor.tremor_analysis_config import TremorFeatureExtractionConfig, TremorDetectionConfig
 from paradigma.tremor.feature_extraction import extract_spectral_domain_features
 from paradigma.windowing import tabulate_windows
 from paradigma.util import get_end_iso8601, write_df_data, read_metadata
@@ -47,3 +49,56 @@ def extract_tremor_features_io(input_path: Union[str, Path], output_path: Union[
     write_df_data(metadata_time, metadata_samples, output_path, 'tremor_meta.json', df_windowed)
 
 
+def detect_tremor(df: pd.DataFrame, config: TremorDetectionConfig, path_to_classifier_input: Union[str, Path]) -> pd.DataFrame:
+
+    # Initialize the classifier
+    coefficients = np.loadtxt(os.path.join(path_to_classifier_input, config.coefficients_file_name))
+    threshold = np.loadtxt(os.path.join(path_to_classifier_input, config.thresholds_file_name))
+
+    # Scale the mfcc's
+    mean_scaling = np.loadtxt(os.path.join(path_to_classifier_input, config.mean_scaling_file_name))
+    std_scaling = np.loadtxt(os.path.join(path_to_classifier_input, config.std_scaling_file_name))
+    mfcc = df.loc[:, df.columns.str.startswith('mfcc')]
+    mfcc_scaled = (mfcc-mean_scaling)/std_scaling
+
+    # Create a logistic regression model with pre-defined coefficients
+    log_reg = LogisticRegression(penalty = None)
+    log_reg.classes_ = np.array([0, 1]) 
+    log_reg.intercept_ = coefficients[0]
+    log_reg.coef_ = coefficients[1:].reshape(1, -1)
+    log_reg.n_features_in_ = int(mfcc.shape[1])
+    log_reg.feature_names_in_ = mfcc.columns
+
+    # Get the tremor probability 
+    df[DataColumns.PRED_TREMOR_PROBA] = log_reg.predict_proba(mfcc_scaled)[:, 1] 
+
+    # Make prediction based on pre-defined threshold
+    df[DataColumns.PRED_TREMOR_LOGREG] = (df[DataColumns.PRED_TREMOR_PROBA] >= threshold).astype(int)
+
+    # Perform extra checks for rest tremor 
+    peak_check = (df['freq_peak'] >= config.fmin_peak) & (df['freq_peak']<=config.fmax_peak) # peak within 3-7 Hz
+    movement_check = df['low_freq_power'] <= config.movement_treshold # little non-tremor arm movement
+    df[DataColumns.PRED_TREMOR_CHECKED] = ((df[DataColumns.PRED_TREMOR_LOGREG]==1) & (peak_check==True) & (movement_check == True)).astype(int)
+
+    return df
+
+
+def detect_tremor_io(input_path: Union[str, Path], output_path: Union[str, Path], path_to_classifier_input: Union[str, Path], config: TremorDetectionConfig) -> None:
+
+    # Load the data
+    metadata_time, metadata_samples = read_metadata(input_path, config.meta_filename, config.time_filename, config.values_filename)
+    df = tsdf.load_dataframe_from_binaries([metadata_time, metadata_samples], tsdf.constants.ConcatenationType.columns)
+
+    df = detect_tremor(df, config, path_to_classifier_input)
+
+    # Prepare the metadata
+    metadata_samples.file_name = 'tremor_values.bin'
+    metadata_time.file_name = 'tremor_time.bin'
+
+    metadata_samples.channels = list(config.d_channels_values.keys())
+    metadata_samples.units = list(config.d_channels_values.values())
+
+    metadata_time.channels = [config.time_colname]
+    metadata_time.units = ['relative_time_ms']
+
+    write_df_data(metadata_time, metadata_samples, output_path, 'tremor_meta.json', df)

src/paradigma/tremor/tremor_analysis_config.py

@@ -1,6 +1,6 @@
 from typing import Dict, List
 
-from paradigma.constants import DataColumns, DataUnits
+from paradigma.constants import DataColumns
 
 class IMUConfig:
     """
@@ -94,4 +94,26 @@ def __init__(self) -> None:
         self.d_channels_values["freq_peak"] = "Hz"
         self.d_channels_values["low_freq_power"] = "(deg/s)^2"
         self.d_channels_values["tremor_power"] = "(deg/s)^2"
-
+
+
+class TremorDetectionConfig(IMUConfig):
+
+    def __init__(self) -> None:
+        super().__init__()
+        self.coefficients_file_name = "tremor_detection_coefficients.txt"
+        self.thresholds_file_name = "tremor_detection_threshold.txt"
+        self.mean_scaling_file_name = "tremor_detection_mean_scaling.txt"
+        self.std_scaling_file_name = "tremor_detection_std_scaling.txt"
+
+        self.fmin_peak: float = 3
+        self.fmax_peak: float = 7
+        self.movement_treshold: float = 50
+
+        self.d_channels_values = {
+        "pred_tremor_proba": "probability",
+        "pred_tremor_logreg": "boolean",
+        "pred_tremor_checked": "boolean"
+        }
+
+        self.set_filenames_values("tremor")
+

tests/data/0.classification/tremor/tremor_detection_coefficients.txt

@@ -0,0 +1,13 @@
+  -5.8764059e+00
+   8.4610178e-01
+   5.9007168e-01
+  -8.5049211e-01
+  -6.2201523e-01
+  -7.5012010e-01
+  -1.9672478e-01
+  -9.0230967e-03
+   1.2943513e-01
+   3.9977200e-01
+   1.5316181e-01
+   1.1048012e-01
+   6.6611830e-02

tests/data/0.classification/tremor/tremor_detection_mean_scaling.txt

@@ -0,0 +1 @@
+   8.7446235e+00   1.8507160e+00   8.0358797e-02   1.9517555e-01   6.4314154e-02   1.0056287e-01   3.2248067e-02   4.6983209e-02   2.6788370e-02   3.9139464e-02   1.4924832e-02   1.5485990e-02

tests/data/0.classification/tremor/tremor_detection_std_scaling.txt

@@ -0,0 +1 @@
+   2.5413890e+00   7.1223807e-01   2.4485624e-01   1.9031539e-01   1.6081742e-01   1.3204677e-01   9.9856516e-02   8.4755261e-02   7.9769296e-02   7.3536817e-02   6.5684145e-02   5.1479819e-02

tests/data/0.classification/tremor/tremor_detection_threshold.txt

@@ -0,0 +1 @@
+0.029570239921802

tests/data/4.predictions/tremor/tremor_meta.json

@@ -0,0 +1,41 @@
+{
+    "study_id": "PPP",
+    "device_id": "Verily Study Watch",
+    "subject_id": "X",
+    "ppp_source_protobuf": "WatchData.IMU.Week104.raw",
+    "metadata_version": "0.1",
+    "start_iso8601": "2021-06-27T16:52:20Z",
+    "end_iso8601": "2021-06-27T17:04:28Z",
+    "rows": 182,
+    "endianness": "little",
+    "data_type": "float",
+    "bits": 64,
+    "sensors": [
+        {
+            "scale_factors": [
+                1
+            ],
+            "file_name": "tremor_time.bin",
+            "channels": [
+                "time"
+            ],
+            "units": [
+                "relative_time_ms"
+            ]
+        },
+        {
+            "file_name": "tremor_values.bin",
+            "channels": [
+                "pred_tremor_proba",
+                "pred_tremor_logreg",
+                "pred_tremor_checked"
+            ],
+            "units": [
+                "probability",
+                "boolean",
+                "boolean"
+            ],
+            "scale_factors": []
+        }
+    ]
+}

tests/test_tremor_analysis.py

@@ -1,7 +1,7 @@
 from pathlib import Path
 
-from paradigma.tremor.tremor_analysis import extract_tremor_features_io
-from paradigma.tremor.tremor_analysis_config import TremorFeatureExtractionConfig
+from paradigma.tremor.tremor_analysis import extract_tremor_features_io, detect_tremor_io
+from paradigma.tremor.tremor_analysis_config import TremorFeatureExtractionConfig, TremorDetectionConfig
 from test_notebooks import compare_data
 
 
@@ -27,3 +27,21 @@ def test_2_extract_features_tremor_output(shared_datadir: Path):
     extract_tremor_features_io(input_path, tested_output_path, config)
     compare_data(reference_output_path, tested_output_path, tremor_binaries_pairs)
 
+def test_3_tremor_detection_output(shared_datadir: Path):
+    """
+    This function is used to evaluate the output of the gait detection. It evaluates it by comparing the output to a reference output.
+    """
+
+    input_dir_name: str = "3.extracted_features"
+    output_dir_name: str = "4.predictions"
+    data_type: str = "tremor"
+
+    # Temporary path to store the output of the notebook
+    path_to_classifier_input = shared_datadir / '0.classification' / 'tremor'
+    input_path = shared_datadir / input_dir_name / data_type
+    reference_output_path = shared_datadir / output_dir_name / data_type
+    tested_output_path = reference_output_path / "test-output"
+
+    config = TremorDetectionConfig()
+    detect_tremor_io(input_path, tested_output_path, path_to_classifier_input, config)
+    compare_data(reference_output_path, tested_output_path, tremor_binaries_pairs)