diff --git a/docs/notebooks/gait/gait_analysis.ipynb b/docs/notebooks/gait/gait_analysis.ipynb index 7cd4880..6ec0b35 100644 --- a/docs/notebooks/gait/gait_analysis.ipynb +++ b/docs/notebooks/gait/gait_analysis.ipynb @@ -9,7 +9,7 @@ }, { "cell_type": "code", - "execution_count": 1, + "execution_count": null, "metadata": {}, "outputs": [], "source": [ @@ -19,14 +19,14 @@ "\n", "import os\n", "from paradigma.preprocessing_config import IMUPreprocessingConfig\n", - "from paradigma.gait_analysis import extract_gait_features, detect_gait, extract_arm_swing_features, detect_arm_swing, quantify_arm_swing\n", + "from paradigma.gait_analysis import extract_gait_features_io, detect_gait_io, extract_arm_swing_features_io, detect_arm_swing_io, quantify_arm_swing_io\n", "from paradigma.gait_analysis_config import GaitFeatureExtractionConfig, GaitDetectionConfig, ArmSwingFeatureExtractionConfig, ArmSwingDetectionConfig, ArmSwingQuantificationConfig\n", - "from paradigma.imu_preprocessing import preprocess_imu_data" + "from paradigma.imu_preprocessing import preprocess_imu_data_io" ] }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 5, "metadata": { "tags": [ "parameters" @@ -54,12 +54,12 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 6, "metadata": {}, "outputs": [], "source": [ "config = IMUPreprocessingConfig()\n", - "preprocess_imu_data(path_to_sensor_data, path_to_preprocessed_data, config)" + "preprocess_imu_data_io(path_to_sensor_data, path_to_preprocessed_data, config)" ] }, { @@ -71,13 +71,13 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 8, "metadata": {}, "outputs": [], "source": [ "config = GaitFeatureExtractionConfig()\n", "#config.set_sampling_frequency(50)\n", - "extract_gait_features(path_to_preprocessed_data, path_to_extracted_features, config)" + "extract_gait_features_io(path_to_preprocessed_data, path_to_extracted_features, config)" ] }, { @@ -89,12 +89,12 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 11, "metadata": {}, "outputs": [], "source": [ "config = GaitDetectionConfig()\n", - "detect_gait(path_to_extracted_features, path_to_predictions, path_to_classifier, config)" + "detect_gait_io(path_to_extracted_features, path_to_predictions, path_to_classifier, config)" ] }, { @@ -106,12 +106,12 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 13, "metadata": {}, "outputs": [], "source": [ "config = ArmSwingFeatureExtractionConfig()\n", - "extract_arm_swing_features(path_to_preprocessed_data, path_to_extracted_features, config)" + "extract_arm_swing_features_io(path_to_preprocessed_data, path_to_extracted_features, config)" ] }, { @@ -123,12 +123,12 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 14, "metadata": {}, "outputs": [], "source": [ "config = ArmSwingDetectionConfig()\n", - "detect_arm_swing(path_to_extracted_features, path_to_predictions, path_to_classifier, config)" + "detect_arm_swing_io(path_to_extracted_features, path_to_predictions, path_to_classifier, config)" ] }, { @@ -140,12 +140,12 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 16, "metadata": {}, "outputs": [], "source": [ "config = ArmSwingQuantificationConfig()\n", - "quantify_arm_swing(path_to_extracted_features, path_to_predictions, path_to_quantification, config)" + "quantify_arm_swing_io(path_to_extracted_features, path_to_predictions, path_to_quantification, config)" ] } ], @@ -165,7 +165,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.9.13" + "version": "3.11.5" } }, "nbformat": 4, diff --git a/src/paradigma/constants.py b/src/paradigma/constants.py index 8640214..986591b 100644 --- a/src/paradigma/constants.py +++ b/src/paradigma/constants.py @@ -19,7 +19,9 @@ class DataColumns(): GRAV_ACCELEROMETER_X : str = "grav_accelerometer_x" GRAV_ACCELEROMETER_Y : str = "grav_accelerometer_y" GRAV_ACCELEROMETER_Z : str = "grav_accelerometer_z" + PRED_GAIT_PROBA: str = "pred_gait_proba" PRED_GAIT : str = "pred_gait" + PRED_ARM_SWING_PROBA: str = "pred_arm_swing_proba" PRED_ARM_SWING : str = "pred_arm_swing" ANGLE : str = "angle" ANGLE_SMOOTH : str = "angle_smooth" diff --git a/src/paradigma/gait_analysis.py b/src/paradigma/gait_analysis.py index 2652c71..062de11 100644 --- a/src/paradigma/gait_analysis.py +++ b/src/paradigma/gait_analysis.py @@ -3,6 +3,8 @@ import pandas as pd from pathlib import Path from typing import Union +from sklearn.linear_model import LogisticRegression +from sklearn.ensemble import RandomForestClassifier import tsdf @@ -18,11 +20,7 @@ from paradigma.util import get_end_iso8601, write_data, read_metadata -def extract_gait_features(input_path: Union[str, Path], output_path: Union[str, Path], config: GaitFeatureExtractionConfig) -> None: - # load data - metadata_time, metadata_samples = read_metadata(input_path, config.meta_filename, config.time_filename, config.values_filename) - df = tsdf.load_dataframe_from_binaries([metadata_time, metadata_samples], tsdf.constants.ConcatenationType.columns) - +def extract_gait_features(df: pd.DataFrame, config: GaitFeatureExtractionConfig) -> pd.DataFrame: # group sequences of timestamps into windows df_windowed = tabulate_windows( df=df, @@ -41,6 +39,18 @@ def extract_gait_features(input_path: Union[str, Path], output_path: Union[str, # and extract spectral features df_windowed = extract_spectral_domain_features(config, df_windowed, config.sensor, config.l_accelerometer_cols) + return df_windowed + + +def extract_gait_features_io(input_path: Union[str, Path], output_path: Union[str, Path], config: GaitFeatureExtractionConfig) -> None: + # Load data + metadata_time, metadata_samples = read_metadata(input_path, config.meta_filename, config.time_filename, config.values_filename) + df = tsdf.load_dataframe_from_binaries([metadata_time, metadata_samples], tsdf.constants.ConcatenationType.columns) + + # Extract gait features + df_windowed = extract_gait_features(df, config) + + # Store data end_iso8601 = get_end_iso8601(start_iso8601=metadata_time.start_iso8601, window_length_seconds=int(df_windowed[config.time_colname][-1:].values[0] + config.window_length_s)) @@ -52,18 +62,13 @@ def extract_gait_features(input_path: Union[str, Path], output_path: Union[str, metadata_samples.channels = list(config.d_channels_values.keys()) metadata_samples.units = list(config.d_channels_values.values()) - metadata_time.channels = ['time'] + metadata_time.channels = [DataColumns.TIME] metadata_time.units = ['relative_time_ms'] write_data(metadata_time, metadata_samples, output_path, 'gait_meta.json', df_windowed) -def detect_gait(input_path: Union[str, Path], output_path: Union[str, Path], path_to_classifier_input: Union[str, Path], config: GaitDetectionConfig) -> None: - - # Load the data - metadata_time, metadata_samples = read_metadata(input_path, config.meta_filename, config.time_filename, config.values_filename) - df = tsdf.load_dataframe_from_binaries([metadata_time, metadata_samples], tsdf.constants.ConcatenationType.columns) - +def detect_gait(df: pd.DataFrame, config: GaitDetectionConfig, path_to_classifier_input: Union[str, Path]) -> pd.DataFrame: # Initialize the classifier clf = pd.read_pickle(os.path.join(path_to_classifier_input, config.classifier_file_name)) with open(os.path.join(path_to_classifier_input, config.thresholds_file_name), 'r') as f: @@ -80,7 +85,18 @@ def detect_gait(input_path: Union[str, Path], output_path: Union[str, Path], pat # Make prediction df['pred_gait_proba'] = clf.predict_proba(X)[:, 1] - df['pred_gait'] = df['pred_gait_proba'] > threshold + df['pred_gait'] = df['pred_gait_proba'] >= threshold + + return df + + +def detect_gait_io(input_path: Union[str, Path], output_path: Union[str, Path], path_to_classifier_input: Union[str, Path], config: GaitDetectionConfig) -> None: + + # Load the data + metadata_time, metadata_samples = read_metadata(input_path, config.meta_filename, config.time_filename, config.values_filename) + df = tsdf.load_dataframe_from_binaries([metadata_time, metadata_samples], tsdf.constants.ConcatenationType.columns) + + df = detect_gait(df, config, path_to_classifier_input) # Prepare the metadata metadata_samples.file_name = 'gait_values.bin' @@ -95,22 +111,7 @@ def detect_gait(input_path: Union[str, Path], output_path: Union[str, Path], pat write_data(metadata_time, metadata_samples, output_path, 'gait_meta.json', df) -def extract_arm_swing_features(input_path: Union[str, Path], output_path: Union[str, Path], config: ArmSwingFeatureExtractionConfig) -> None: - # load accelerometer and gyroscope data - l_dfs = [] - for sensor in ['accelerometer', 'gyroscope']: - config.set_sensor(sensor) - meta_filename = f'{sensor}_meta.json' - values_filename = f'{sensor}_samples.bin' - time_filename = f'{sensor}_time.bin' - - metadata_dict = tsdf.load_metadata_from_path(os.path.join(input_path, meta_filename)) - metadata_time = metadata_dict[time_filename] - metadata_samples = metadata_dict[values_filename] - l_dfs.append(tsdf.load_dataframe_from_binaries([metadata_time, metadata_samples], tsdf.constants.ConcatenationType.columns)) - - df = pd.merge(l_dfs[0], l_dfs[1], on=config.time_colname) - +def extract_arm_swing_features(df: pd.DataFrame, config: ArmSwingFeatureExtractionConfig) -> pd.DataFrame: # temporary add "random" predictions df[config.pred_gait_colname] = np.concatenate([np.repeat([1], df.shape[0]//3), np.repeat([0], df.shape[0]//3), np.repeat([1], df.shape[0] + 1 - 2*df.shape[0]//3)], axis=0) @@ -143,7 +144,7 @@ def extract_arm_swing_features(input_path: Union[str, Path], output_path: Union[ df_segments = create_segments( df=df, time_colname=config.time_colname, - segment_nr_colname='segment_nr', + segment_nr_colname=DataColumns.SEGMENT_NR, minimum_gap_s=3 ) @@ -151,7 +152,7 @@ def extract_arm_swing_features(input_path: Union[str, Path], output_path: Union[ df_segments = discard_segments( df=df_segments, time_colname=config.time_colname, - segment_nr_colname='segment_nr', + segment_nr_colname=DataColumns.SEGMENT_NR, minimum_segment_length_s=3 ) @@ -257,6 +258,27 @@ def extract_arm_swing_features(input_path: Union[str, Path], output_path: Union[ for sensor, l_sensor_colnames in zip(['accelerometer', 'gyroscope'], [config.l_accelerometer_cols, config.l_gyroscope_cols]): df_windowed = extract_spectral_domain_features(config, df_windowed, sensor, l_sensor_colnames) + return df_windowed + + +def extract_arm_swing_features_io(input_path: Union[str, Path], output_path: Union[str, Path], config: ArmSwingFeatureExtractionConfig) -> None: + # load accelerometer and gyroscope data + l_dfs = [] + for sensor in ['accelerometer', 'gyroscope']: + config.set_sensor(sensor) + meta_filename = f'{sensor}_meta.json' + values_filename = f'{sensor}_samples.bin' + time_filename = f'{sensor}_time.bin' + + metadata_dict = tsdf.load_metadata_from_path(os.path.join(input_path, meta_filename)) + metadata_time = metadata_dict[time_filename] + metadata_samples = metadata_dict[values_filename] + l_dfs.append(tsdf.load_dataframe_from_binaries([metadata_time, metadata_samples], tsdf.constants.ConcatenationType.columns)) + + df = pd.merge(l_dfs[0], l_dfs[1], on=config.time_colname) + + df_windowed = extract_arm_swing_features(df, config) + end_iso8601 = get_end_iso8601(metadata_samples.start_iso8601, df_windowed[config.time_colname][-1:].values[0] + config.window_length_s) @@ -274,13 +296,7 @@ def extract_arm_swing_features(input_path: Union[str, Path], output_path: Union[ write_data(metadata_time, metadata_samples, output_path, 'arm_swing_meta.json', df_windowed) -def detect_arm_swing(input_path: Union[str, Path], output_path: Union[str, Path], path_to_classifier_input: Union[str, Path], config: ArmSwingDetectionConfig) -> None: - # Load the data - metadata_time, metadata_samples = read_metadata(input_path, config.meta_filename, config.time_filename, config.values_filename) - df = tsdf.load_dataframe_from_binaries([metadata_time, metadata_samples], tsdf.constants.ConcatenationType.columns) - - # Initialize the classifier - clf = pd.read_pickle(os.path.join(path_to_classifier_input, config.classifier_file_name)) +def detect_arm_swing(df: pd.DataFrame, config: ArmSwingDetectionConfig, clf: Union[LogisticRegression, RandomForestClassifier]) -> pd.DataFrame: # Prepare the data clf.feature_names_in_ = ['std_norm_acc'] + [f'{x}_power_below_gait' for x in config.l_accelerometer_cols] + \ @@ -292,57 +308,44 @@ def detect_arm_swing(input_path: Union[str, Path], output_path: Union[str, Path] ['range_of_motion', 'forward_peak_ang_vel_mean', 'backward_peak_ang_vel_mean', 'forward_peak_ang_vel_std', 'backward_peak_ang_vel_std', 'angle_perc_power', 'angle_dominant_frequency'] + \ [f'{x}_dominant_frequency' for x in config.l_accelerometer_cols] - X = df.loc[:, clf.feature_names_in_] # Make prediction - # df['pred_arm_swing_proba'] = clf.predict_proba(X)[:, 1] - df['pred_arm_swing'] = clf.predict(X) + df[DataColumns.PRED_ARM_SWING_PROBA] = clf.predict_proba(X)[:, 1] + + return df + +def detect_arm_swing_io(input_path: Union[str, Path], output_path: Union[str, Path], path_to_classifier_input: Union[str, Path], config: ArmSwingDetectionConfig) -> None: + # Load the data + metadata_time, metadata_samples = read_metadata(input_path, config.meta_filename, config.time_filename, config.values_filename) + df = tsdf.load_dataframe_from_binaries([metadata_time, metadata_samples], tsdf.constants.ConcatenationType.columns) + + # Load the classifier + clf = pd.read_pickle(os.path.join(path_to_classifier_input, config.classifier_file_name)) + + df = detect_arm_swing(df, config, clf) # Prepare the metadata metadata_samples.file_name = 'arm_swing_values.bin' metadata_time.file_name = 'arm_swing_time.bin' - metadata_samples.channels = ['pred_arm_swing'] - metadata_samples.units = ['boolean'] + metadata_samples.channels = [DataColumns.PRED_ARM_SWING_PROBA] + metadata_samples.units = ['probability'] - metadata_time.channels = ['time'] + metadata_time.channels = [DataColumns.TIME] metadata_time.units = ['relative_time_ms'] write_data(metadata_time, metadata_samples, output_path, 'arm_swing_meta.json', df) -def quantify_arm_swing(path_to_feature_input: Union[str, Path], path_to_prediction_input: Union[str, Path], output_path: Union[str, Path], config: ArmSwingQuantificationConfig) -> None: - # Load the features & predictions - metadata_time, metadata_samples = read_metadata(path_to_feature_input, config.meta_filename, config.time_filename, config.values_filename) - df_features = tsdf.load_dataframe_from_binaries([metadata_time, metadata_samples], tsdf.constants.ConcatenationType.columns) - - metadata_dict = tsdf.load_metadata_from_path(os.path.join(path_to_prediction_input, config.meta_filename)) - metadata_time = metadata_dict[config.time_filename] - metadata_samples = metadata_dict[config.values_filename] - df_predictions = tsdf.load_dataframe_from_binaries([metadata_time, metadata_samples], tsdf.constants.ConcatenationType.columns) - - # Validate - # dataframes have same length - assert df_features.shape[0] == df_predictions.shape[0] - - # dataframes have same time column - assert df_features['time'].equals(df_predictions['time']) - - # Prepare the data - - # subset features - l_feature_cols = ['time', 'range_of_motion', 'forward_peak_ang_vel_mean', 'backward_peak_ang_vel_mean'] - df_features = df_features[l_feature_cols] - - # concatenate features and predictions - df = pd.concat([df_features, df_predictions[config.pred_arm_swing_colname]], axis=1) +def quantify_arm_swing(df: pd.DataFrame, config: ArmSwingQuantificationConfig) -> pd.DataFrame: # temporarily for testing: manually determine predictions - df[config.pred_arm_swing_colname] = np.concatenate([np.repeat([1], df.shape[0]//3), np.repeat([0], df.shape[0]//3), np.repeat([1], df.shape[0] - 2*df.shape[0]//3)], axis=0) + df[DataColumns.PRED_ARM_SWING_PROBA] = np.concatenate([np.repeat([1], df.shape[0]//3), np.repeat([0], df.shape[0]//3), np.repeat([1], df.shape[0] - 2*df.shape[0]//3)], axis=0) # keep only predicted arm swing - df_arm_swing = df.loc[df[config.pred_arm_swing_colname]==1].copy().reset_index(drop=True) + # TODO: Aggregate overlapping windows for probabilities + df_arm_swing = df.loc[df[DataColumns.PRED_ARM_SWING_PROBA]>=0.5].copy().reset_index(drop=True) del df @@ -354,30 +357,59 @@ def quantify_arm_swing(path_to_feature_input: Union[str, Path], path_to_predicti df_arm_swing = create_segments( df=df_arm_swing, - time_colname='time', - segment_nr_colname='segment_nr', + time_colname=DataColumns.TIME, + segment_nr_colname=DataColumns.SEGMENT_NR, minimum_gap_s=config.segment_gap_s ) df_arm_swing = discard_segments( df=df_arm_swing, - time_colname='time', - segment_nr_colname='segment_nr', + time_colname=DataColumns.TIME, + segment_nr_colname=DataColumns.SEGMENT_NR, minimum_segment_length_s=config.min_segment_length_s ) # Quantify arm swing df_aggregates = aggregate_segments( df=df_arm_swing, - time_colname='time', - segment_nr_colname='segment_nr', + time_colname=DataColumns.TIME, + segment_nr_colname=DataColumns.SEGMENT_NR, window_step_size_s=config.window_step_size, l_metrics=['range_of_motion', 'peak_ang_vel'], l_aggregates=['median'], l_quantiles=[0.95] ) - df_aggregates['segment_duration_ms'] = df_aggregates['segment_duration_s'] * 1000 - df_aggregates = df_aggregates.drop(columns=['segment_nr']) + df_aggregates['segment_duration_ms'] = (df_aggregates['segment_duration_s'] * 1000).round().astype(int) + df_aggregates = df_aggregates.drop(columns=[DataColumns.SEGMENT_NR]) + + return df_aggregates + + +def quantify_arm_swing_io(path_to_feature_input: Union[str, Path], path_to_prediction_input: Union[str, Path], output_path: Union[str, Path], config: ArmSwingQuantificationConfig) -> None: + # Load the features & predictions + metadata_time, metadata_samples = read_metadata(path_to_feature_input, config.meta_filename, config.time_filename, config.values_filename) + df_features = tsdf.load_dataframe_from_binaries([metadata_time, metadata_samples], tsdf.constants.ConcatenationType.columns) + + metadata_dict = tsdf.load_metadata_from_path(os.path.join(path_to_prediction_input, config.meta_filename)) + metadata_time = metadata_dict[config.time_filename] + metadata_samples = metadata_dict[config.values_filename] + df_predictions = tsdf.load_dataframe_from_binaries([metadata_time, metadata_samples], tsdf.constants.ConcatenationType.columns) + + # Validate + # Dataframes have same length + assert df_features.shape[0] == df_predictions.shape[0] + + # Dataframes have same time column + assert df_features[DataColumns.TIME].equals(df_predictions[DataColumns.TIME]) + + # Subset features + l_feature_cols = [DataColumns.TIME, 'range_of_motion', 'forward_peak_ang_vel_mean', 'backward_peak_ang_vel_mean'] + df_features = df_features[l_feature_cols] + + # Concatenate features and predictions + df = pd.concat([df_features, df_predictions[DataColumns.PRED_ARM_SWING_PROBA]], axis=1) + + df_aggregates = quantify_arm_swing(df, config) # Store data metadata_samples.file_name = 'arm_swing_values.bin' @@ -387,7 +419,7 @@ def quantify_arm_swing(path_to_feature_input: Union[str, Path], path_to_predicti 'peak_ang_vel_median', 'peak_ang_vel_quantile_95'] metadata_samples.units = ['deg', 'deg', 'deg/s', 'deg/s'] - metadata_time.channels = ['time', 'segment_duration_ms'] + metadata_time.channels = [DataColumns.TIME, 'segment_duration_ms'] metadata_time.units = ['relative_time_ms', 'ms'] write_data(metadata_time, metadata_samples, output_path, 'arm_swing_meta.json', df_aggregates) diff --git a/src/paradigma/gait_analysis_config.py b/src/paradigma/gait_analysis_config.py index b073969..37d9d2e 100644 --- a/src/paradigma/gait_analysis_config.py +++ b/src/paradigma/gait_analysis_config.py @@ -177,6 +177,7 @@ def initialize_column_names( self ) -> None: + self.pred_gait_proba_colname=DataColumns.PRED_GAIT_PROBA self.pred_gait_colname=DataColumns.PRED_GAIT self.angle_smooth_colname: str = DataColumns.ANGLE_SMOOTH self.angle_colname=DataColumns.ANGLE @@ -258,6 +259,7 @@ def __init__(self) -> None: super().__init__() self.set_filenames_values("arm_swing") + self.pred_arm_swing_proba_colname = DataColumns.PRED_ARM_SWING_PROBA self.pred_arm_swing_colname = DataColumns.PRED_ARM_SWING self.window_length_s = 3 diff --git a/src/paradigma/imu_preprocessing.py b/src/paradigma/imu_preprocessing.py index 812da15..7006399 100644 --- a/src/paradigma/imu_preprocessing.py +++ b/src/paradigma/imu_preprocessing.py @@ -11,12 +11,7 @@ from paradigma.preprocessing_config import IMUPreprocessingConfig -def preprocess_imu_data(input_path: Union[str, Path], output_path: Union[str, Path], config: IMUPreprocessingConfig) -> None: - - # Load data - metadata_time, metadata_samples = read_metadata(str(input_path), str(config.meta_filename), - str(config.time_filename), str(config.values_filename)) - df = tsdf.load_dataframe_from_binaries([metadata_time, metadata_samples], tsdf.constants.ConcatenationType.columns) +def preprocess_imu_data(df: pd.DataFrame, config: IMUPreprocessingConfig, scale_factors: list) -> pd.DataFrame: # Rename columns df = df.rename(columns={f'rotation_{a}': f'gyroscope_{a}' for a in ['x', 'y', 'z']}) @@ -35,7 +30,7 @@ def preprocess_imu_data(input_path: Union[str, Path], output_path: Union[str, Pa time_column=config.time_colname, time_unit_type=TimeUnit.RELATIVE_MS, unscaled_column_names = list(config.d_channels_imu.keys()), - scale_factors=metadata_samples.scale_factors, + scale_factors=scale_factors, resampling_frequency=config.sampling_frequency) if config.side_watch == 'left': @@ -59,6 +54,19 @@ def preprocess_imu_data(input_path: Union[str, Path], output_path: Union[str, Pa df = df.drop(columns=[col]) df = df.rename(columns={f'filt_{col}': col}) + return df + + +def preprocess_imu_data_io(input_path: Union[str, Path], output_path: Union[str, Path], config: IMUPreprocessingConfig) -> None: + + # Load data + metadata_time, metadata_samples = read_metadata(str(input_path), str(config.meta_filename), + str(config.time_filename), str(config.values_filename)) + df = tsdf.load_dataframe_from_binaries([metadata_time, metadata_samples], tsdf.constants.ConcatenationType.columns) + + # Preprocess data + df = preprocess_imu_data(df=df, config=config, scale_factors=metadata_samples.scale_factors) + # Store data for sensor, units in zip(['accelerometer', 'gyroscope'], ['g', config.rotation_units]): df_sensor = df[[config.time_colname] + [x for x in df.columns if sensor in x]] @@ -72,6 +80,7 @@ def preprocess_imu_data(input_path: Union[str, Path], output_path: Union[str, Pa write_data(metadata_time, metadata_samples, output_path, f'{sensor}_meta.json', df_sensor) + def transform_time_array( time_array: pd.Series, scale_factor: float, diff --git a/src/paradigma/windowing.py b/src/paradigma/windowing.py index 150df90..a90bbf0 100644 --- a/src/paradigma/windowing.py +++ b/src/paradigma/windowing.py @@ -203,7 +203,7 @@ def discard_segments( pd.DataFrame The dataframe with segments that are longer than the specified length """ - segment_length_bool = df.groupby(segment_nr_colname)[time_colname].apply(lambda x: x.max() - x.min()) > minimum_segment_length_s + segment_length_bool = df.groupby(segment_nr_colname)[time_colname].apply(lambda x: x.max() - x.min()) >= minimum_segment_length_s df = df.loc[df[segment_nr_colname].isin(segment_length_bool.loc[segment_length_bool.values].index)] diff --git a/tests/data/3.extracted_features/gait/arm_swing_values.bin b/tests/data/3.extracted_features/gait/arm_swing_values.bin index a0b0806..f72f4a2 100644 Binary files a/tests/data/3.extracted_features/gait/arm_swing_values.bin and b/tests/data/3.extracted_features/gait/arm_swing_values.bin differ diff --git a/tests/data/4.predictions/gait/arm_swing_meta.json b/tests/data/4.predictions/gait/arm_swing_meta.json index ee42b0e..db62bf5 100644 --- a/tests/data/4.predictions/gait/arm_swing_meta.json +++ b/tests/data/4.predictions/gait/arm_swing_meta.json @@ -8,10 +8,10 @@ "end_iso8601": "2021-06-27T17:04:30Z", "rows": 642, "endianness": "little", + "data_type": "float", "bits": 64, "sensors": [ { - "data_type": "float", "scale_factors": [ 1 ], @@ -24,13 +24,12 @@ ] }, { - "data_type": "int", "file_name": "arm_swing_values.bin", "channels": [ - "pred_arm_swing" + "pred_arm_swing_proba" ], "units": [ - "boolean" + "probability" ], "scale_factors": [ 0.00469378, diff --git a/tests/data/4.predictions/gait/arm_swing_values.bin b/tests/data/4.predictions/gait/arm_swing_values.bin index 43ed663..c8fa711 100644 Binary files a/tests/data/4.predictions/gait/arm_swing_values.bin and b/tests/data/4.predictions/gait/arm_swing_values.bin differ diff --git a/tests/data/5.quantification/gait/arm_swing_meta.json b/tests/data/5.quantification/gait/arm_swing_meta.json index 80150f1..f2ebc40 100644 --- a/tests/data/5.quantification/gait/arm_swing_meta.json +++ b/tests/data/5.quantification/gait/arm_swing_meta.json @@ -8,8 +8,8 @@ "end_iso8601": "2021-06-27T17:04:30Z", "rows": 2, "endianness": "little", - "bits": 64, "data_type": "float", + "bits": 64, "sensors": [ { "scale_factors": [ diff --git a/tests/data/5.quantification/gait/arm_swing_values.bin b/tests/data/5.quantification/gait/arm_swing_values.bin index 8e6ecf9..997859b 100644 Binary files a/tests/data/5.quantification/gait/arm_swing_values.bin and b/tests/data/5.quantification/gait/arm_swing_values.bin differ diff --git a/tests/test_gait_analysis.py b/tests/test_gait_analysis.py index c1ae115..506fe13 100644 --- a/tests/test_gait_analysis.py +++ b/tests/test_gait_analysis.py @@ -1,8 +1,8 @@ from pathlib import Path -from paradigma.gait_analysis import detect_arm_swing, detect_gait, extract_arm_swing_features, extract_gait_features, quantify_arm_swing +from paradigma.gait_analysis import detect_arm_swing_io, detect_gait_io, extract_arm_swing_features_io, extract_gait_features_io, quantify_arm_swing_io from paradigma.gait_analysis_config import ArmSwingDetectionConfig, ArmSwingFeatureExtractionConfig, ArmSwingQuantificationConfig, GaitDetectionConfig, GaitFeatureExtractionConfig -from paradigma.imu_preprocessing import preprocess_imu_data +from paradigma.imu_preprocessing import preprocess_imu_data_io from paradigma.preprocessing_config import IMUPreprocessingConfig from test_notebooks import compare_data @@ -42,7 +42,7 @@ def test_1_imu_preprocessing_outputs(shared_datadir: Path): tested_output_path = reference_output_path / "test-output" config = IMUPreprocessingConfig() - preprocess_imu_data(input_path, tested_output_path, config) + preprocess_imu_data_io(input_path, tested_output_path, config) compare_data(reference_output_path, tested_output_path, imu_binaries_pairs) @@ -60,7 +60,7 @@ def test_2_extract_features_gait_output(shared_datadir: Path): tested_output_path = reference_output_path / "test-output" config = GaitFeatureExtractionConfig() - extract_gait_features(input_path, tested_output_path, config) + extract_gait_features_io(input_path, tested_output_path, config) compare_data(reference_output_path, tested_output_path, gait_binaries_pairs) @@ -80,7 +80,7 @@ def test_3_gait_detection_output(shared_datadir: Path): tested_output_path = reference_output_path / "test-output" config = GaitDetectionConfig() - detect_gait(input_path, tested_output_path, path_to_classifier_input, config) + detect_gait_io(input_path, tested_output_path, path_to_classifier_input, config) compare_data(reference_output_path, tested_output_path, gait_binaries_pairs) @@ -99,7 +99,7 @@ def test_4_extract_features_arm_swing_output(shared_datadir: Path): tested_output_path = reference_output_path / "test-output" config = ArmSwingFeatureExtractionConfig() - extract_arm_swing_features(input_path, tested_output_path, config) + extract_arm_swing_features_io(input_path, tested_output_path, config) compare_data(reference_output_path, tested_output_path, arm_swing_binaries_pairs) @@ -120,7 +120,7 @@ def test_5_arm_swing_detection_output(shared_datadir: Path): tested_output_path = reference_output_path / "test-output" config = ArmSwingDetectionConfig() - detect_arm_swing(input_path, tested_output_path, path_to_classifier_input, config) + detect_arm_swing_io(input_path, tested_output_path, path_to_classifier_input, config) compare_data(reference_output_path, tested_output_path, arm_swing_binaries_pairs) @@ -141,5 +141,5 @@ def test_6_arm_swing_quantification_output(shared_datadir: Path): tested_output_path = reference_output_path / "test-output" config = ArmSwingQuantificationConfig() - quantify_arm_swing(path_to_feature_input, path_to_prediction_input, tested_output_path, config) + quantify_arm_swing_io(path_to_feature_input, path_to_prediction_input, tested_output_path, config) compare_data(reference_output_path, tested_output_path, arm_swing_binaries_pairs)