rest condition should have task name starting with rest not == to rest (

#585) * rest condition should have task name starting with rest not == to rest * more fixes + add ds003775 to the documentation * [circle full] * fix config ds003775 [circle ds003775] * one more try and add support for reader_extra_params for ds003775 * update doc * code spell * oups * adding better name to new dataset * Apply suggestions from code review * Update config.py [circle full] Co-authored-by: Richard Höchenberger <[email protected]>
mne-tools · Sep 2, 2022 · 2b16d44 · 2b16d44
1 parent 143654a
commit 2b16d44
Show file tree

Hide file tree

Showing 17 changed files with 175 additions and 22 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -77,6 +77,25 @@ jobs:
           paths:
             - ~/mne_data/ds000117
 
+  cache_ds003775:
+    <<: *imageconfig
+    steps:
+      - attach_workspace:
+          at: ~/
+      # - restore_cache:
+      #     keys:
+      #       - data-cache-ds003775
+      - run:
+          <<: *bashenv
+      - run:
+          name: Get ds003775
+          command: |
+            python ~/project/tests/download_test_data.py -d ds003775
+      - save_cache:
+          key: data-cache-ds003775-{{ .Revision }}
+          paths:
+            - ~/mne_data/ds003775
+
   cache_ds000246:
     <<: *imageconfig
     steps:
@@ -276,6 +295,32 @@ jobs:
           paths:
             - mne_data/derivatives/mne-bids-pipeline/ds000117/*/*/*/*.html
 
+  test_ds003775:
+    <<: *imageconfig
+    steps:
+      - attach_workspace:
+          at: ~/
+      - run:
+          <<: *bashenv
+      - restore_cache:
+          keys:
+            - data-cache-ds003775-{{ .Revision }}
+      - run:
+          name: test ds003775
+          command: |
+            export DS=ds003775
+            python tests/run_tests.py --download=0 ${DS}
+            mkdir -p ~/reports/${DS}
+            cp ~/mne_data/derivatives/mne-bids-pipeline/${DS}/*/**/*.html ~/reports/${DS}/
+            cp ~/mne_data/derivatives/mne-bids-pipeline/${DS}/*.xlsx ~/reports/${DS}/
+      - store_artifacts:
+          path: /home/circleci/reports/ds003775
+          destination: reports/ds003775
+      - persist_to_workspace:
+          root: ~/
+          paths:
+            - mne_data/derivatives/mne-bids-pipeline/ds003775/*/*/*/*.html
+
   test_ds000246:
     <<: *imageconfig
     steps:
@@ -812,6 +857,9 @@ jobs:
       - restore_cache:
           keys:
             - data-cache-ds000117-{{ .Revision }}
+      - restore_cache:
+          keys:
+            - data-cache-ds003775-{{ .Revision }}
       - restore_cache:
           keys:
             - data-cache-ds000246-{{ .Revision }}
@@ -891,6 +939,13 @@ workflows:
           requires:
             - cache_ds000117
 
+      - cache_ds003775:
+          requires:
+            - setup_env
+      - test_ds003775:
+          requires:
+            - cache_ds003775
+
       - cache_ds000246:
           requires:
             - setup_env
@@ -990,6 +1045,7 @@ workflows:
       - build_docs:
           requires:
             - test_ds000117
+            - test_ds003775
             - test_ds000246
             - test_ds000247
             - test_ds000248

diff --git a/config.py b/config.py
@@ -428,6 +428,17 @@ class ArbitraryContrast(TypedDict):
     ```
 """
 
+reader_extra_params: dict = {}
+"""
+Parameters to be passed to `read_raw_bids()` calls when importing raw data.
+
+???+ example "Example"
+    Enforce units for EDF files:
+    ```python
+    reader_extra_params = {"units": "uV"}
+    ```
+"""
+
 ###############################################################################
 # BREAK DETECTION
 # ---------------
@@ -860,8 +871,8 @@ class ArbitraryContrast(TypedDict):
 (value) to a more legible one (value).
 
 This is a **required** parameter in the configuration file, unless you are
-processing resting-state data. If left as `None` and [`task`][config.task]
-is not `'rest'`, we will raise an error.
+processing resting-state data. If left as `None` and
+[`task_is_rest`][config.task_is_rest] is not `True`, we will raise an error.
 
 ???+ example "Example"
     Specifying conditions as lists of strings:
@@ -899,6 +910,11 @@ class ArbitraryContrast(TypedDict):
     ```
 """
 
+task_is_rest: bool = False
+"""
+Whether the task should be treated as resting-state data.
+"""
+
 rest_epochs_duration: Optional[float] = None
 """
 Duration of epochs in seconds.
@@ -3116,7 +3132,8 @@ def make_epochs(
     metadata_keep_last: Optional[Iterable[str]],
     metadata_query: Optional[str],
     event_repeated: Literal['error', 'drop', 'merge'],
-    decim: int
+    decim: int,
+    task_is_rest: bool
 ) -> mne.Epochs:
     """Generate Epochs from raw data.
 
@@ -3127,7 +3144,7 @@ def make_epochs(
     - No rejection thresholds will be applied.
     - No baseline-correction will be performed.
     """
-    if task.lower() == 'rest':
+    if task_is_rest:
         stop = raw.times[-1] - rest_epochs_duration
         assert epochs_tmin == 0., "epochs_tmin must be 0 for rest"
         assert rest_epochs_overlap is not None, \
@@ -3414,7 +3431,8 @@ def _load_data(cfg, bids_path):
     # - sets raw.annotations using the BIDS events.tsv
 
     subject = bids_path.subject
-    raw = read_raw_bids(bids_path=bids_path)
+    raw = read_raw_bids(bids_path=bids_path,
+                        extra_params=cfg.reader_extra_params)
 
     # Save only the channel types we wish to analyze (including the
     # channels marked as "bad").
@@ -3607,7 +3625,8 @@ def import_er_data(
     # But at least using the union here should reduce them.
     # TODO: We should also uso automatic bad finding on the empty room data
     if cfg.use_maxwell_filter:
-        raw_ref = mne_bids.read_raw_bids(bids_path_ref_in)
+        raw_ref = mne_bids.read_raw_bids(bids_path_ref_in,
+                                         extra_params=cfg.reader_extra_params)
         # We need to include any automatically found bad channels, if relevant.
         # TODO this is a bit of a hack because we don't use "in_files" access
         # here, but this is *in the same step where this file is generated*
@@ -3785,10 +3804,11 @@ def save_logs(logs):
 
 
 # Another check that depends on some of the functions defined above
-if (get_task() is not None and
-        get_task().lower() != 'rest' and
-        conditions is None and
-        'MKDOCS' not in os.environ):
+if (
+    not task_is_rest and
+    conditions is None and
+    'MKDOCS' not in os.environ
+):
     msg = ('Please indicate the name of your conditions in your '
            'configuration. Currently the `conditions` parameter is empty. '
            'This is only allowed for resting-state analysis.')

diff --git a/docs/mkdocs.yml b/docs/mkdocs.yml
@@ -76,6 +76,7 @@ nav:
         - examples/eeg_matchingpennies.md
         - examples/ds001810.md
         - examples/ds000117.md
+        - examples/ds003775.md
         - examples/ERP_CORE.md
     - What's new: changes.md
 

diff --git a/docs/source/changes.md b/docs/source/changes.md
@@ -241,6 +241,12 @@ authors:
   [`report_evoked_n_time_points`][config.report_evoked_n_time_points] and
   [`report_stc_n_time_points`][config.report_stc_n_time_points], respectively.
   ({{ gh(542) }} by {{ authors.agramfort }})
+- Add [`reader_extra_params`][config.reader_extra_params] parameter to pass
+  parameters to read_raw_bids.
+  ({{ gh(585) }} by {{ authors.agramfort }})
+- Add [`task_is_rest`][config.task_is_rest] parameter to be explicit that
+  the data must be analyzed as resting state.
+  ({{ gh(585) }} by {{ authors.agramfort }})
 - Patch information is now incorporated when computing surface source spaces,
   which should slightly improve the surface normals
   ({{ gh(588) }} by {{ authors.larsoner }})

diff --git a/docs/source/settings/general.md b/docs/source/settings/general.md
@@ -5,6 +5,7 @@
 ::: config.interactive
 ::: config.sessions
 ::: config.task
+::: config.task_is_rest
 ::: config.runs
 ::: config.exclude_runs
 ::: config.crop_runs
@@ -22,6 +23,7 @@
 ::: config.eeg_reference
 ::: config.eeg_template_montage
 ::: config.drop_channels
+::: config.reader_extra_params
 ::: config.analyze_channels
 ::: config.plot_psd_for_runs
 ::: config.N_JOBS

diff --git a/scripts/preprocessing/_01_maxfilter.py b/scripts/preprocessing/_01_maxfilter.py
@@ -103,7 +103,8 @@ def run_maxwell_filter(*, cfg, subject, session=None, run=None, in_files=None):
         logger.info(**gen_log_kwargs(message=msg, subject=subject,
                                      session=session, run=run))
 
-    raw = read_raw_bids(bids_path=in_files["raw_ref_run"])
+    raw = read_raw_bids(bids_path=in_files["raw_ref_run"],
+                        extra_params=cfg.reader_extra_params)
     dev_head_t = raw.info['dev_head_t']
     del raw
 
@@ -258,6 +259,7 @@ def get_config(
     session: Optional[str] = None
 ) -> SimpleNamespace:
     cfg = SimpleNamespace(
+        reader_extra_params=config.reader_extra_params,
         mf_cal_fname=config.get_mf_cal_fname(subject, session),
         mf_ctc_fname=config.get_mf_ctc_fname(subject, session),
         mf_st_duration=config.mf_st_duration,

diff --git a/scripts/preprocessing/_02_frequency_filter.py b/scripts/preprocessing/_02_frequency_filter.py
@@ -271,6 +271,7 @@ def get_config(
     session: Optional[str] = None
 ) -> SimpleNamespace:
     cfg = SimpleNamespace(
+        reader_extra_params=config.reader_extra_params,
         process_er=config.process_er,
         runs=config.get_runs(subject=subject),
         use_maxwell_filter=config.use_maxwell_filter,

diff --git a/scripts/preprocessing/_03_make_epochs.py b/scripts/preprocessing/_03_make_epochs.py
@@ -52,7 +52,7 @@ def run_epochs(*, cfg, subject, session=None):
 
     # Generate a unique event name -> event code mapping that can be used
     # across all runs.
-    if cfg.task.lower() != 'rest':
+    if not cfg.task_is_rest:
         event_name_to_code_map = config.annotations_to_events(
             raw_paths=raw_fnames)
 
@@ -75,7 +75,7 @@ def run_epochs(*, cfg, subject, session=None):
         raw = mne.io.read_raw_fif(raw_fname, preload=True)
 
         # Only keep the subset of the mapping that applies to the current run
-        if cfg.task.lower() == 'rest':
+        if cfg.task_is_rest:
             event_id = None  # make_epochs takes care of it.
         else:
             event_id = event_name_to_code_map.copy()
@@ -101,7 +101,8 @@ def run_epochs(*, cfg, subject, session=None):
             metadata_keep_last=cfg.epochs_metadata_keep_last,
             metadata_query=cfg.epochs_metadata_query,
             event_repeated=cfg.event_repeated,
-            decim=cfg.decim
+            decim=cfg.decim,
+            task_is_rest=cfg.task_is_rest
         )
 
         epochs.load_data()  # Remove reference to raw
@@ -215,6 +216,7 @@ def get_config(
         bids_root=config.get_bids_root(),
         deriv_root=config.get_deriv_root(),
         interactive=config.interactive,
+        task_is_rest=config.task_is_rest,
         conditions=config.conditions,
         epochs_tmin=config.epochs_tmin,
         epochs_tmax=config.epochs_tmax,

diff --git a/scripts/preprocessing/_04a_run_ica.py b/scripts/preprocessing/_04a_run_ica.py
@@ -345,7 +345,8 @@ def run_ica(*, cfg, subject, session=None):
             metadata_keep_last=cfg.epochs_metadata_keep_last,
             metadata_query=cfg.epochs_metadata_query,
             event_repeated=cfg.event_repeated,
-            decim=cfg.decim
+            decim=cfg.decim,
+            task_is_rest=cfg.task_is_rest,
         )
 
         epochs.load_data()  # Remove reference to raw
@@ -497,6 +498,7 @@ def get_config(
     cfg = SimpleNamespace(
         conditions=config.conditions,
         task=config.get_task(),
+        task_is_rest=config.task_is_rest,
         datatype=config.get_datatype(),
         runs=config.get_runs(subject=subject),
         acq=config.acq,

diff --git a/scripts/report/_01_make_reports.py b/scripts/report/_01_make_reports.py
@@ -561,7 +561,7 @@ def run_report_preprocessing(
             plt.close(fig)
 
     # Visualize events.
-    if cfg.task.lower() != 'rest':
+    if not cfg.task_is_rest:
         msg = 'Adding events plot to report.'
         logger.info(
             **gen_log_kwargs(
@@ -1539,6 +1539,7 @@ def get_config(
 
     cfg = SimpleNamespace(
         task=config.get_task(),
+        task_is_rest=config.task_is_rest,
         runs=config.get_runs(subject=subject),
         datatype=config.get_datatype(),
         acq=config.acq,
@@ -1596,8 +1597,7 @@ def main():
         if not sessions:
             sessions = [None]
 
-        if (config.get_task() is not None and
-                config.get_task().lower() == 'rest'):
+        if config.task_is_rest:
             msg = '    … skipping "average" report for "rest" task.'
             logger.info(**gen_log_kwargs(message=msg))
             return

diff --git a/scripts/sensor/_01_make_evoked.py b/scripts/sensor/_01_make_evoked.py
@@ -110,7 +110,7 @@ def get_config(
 
 def main():
     """Run evoked."""
-    if config.get_task().lower() == 'rest':
+    if config.task_is_rest:
         msg = '    … skipping: for resting-state task.'
         logger.info(**gen_log_kwargs(message=msg))
         return

diff --git a/scripts/sensor/_99_group_average.py b/scripts/sensor/_99_group_average.py
@@ -377,6 +377,7 @@ def get_config(
     cfg = SimpleNamespace(
         subjects=config.get_subjects(),
         task=config.get_task(),
+        task_is_rest=config.task_is_rest,
         datatype=config.get_datatype(),
         acq=config.acq,
         rec=config.rec,
@@ -405,7 +406,7 @@ def get_config(
 # pass 'average' subject for logging
 @failsafe_run(on_error=on_error, script_path=__file__)
 def run_group_average_sensor(*, cfg, subject='average'):
-    if config.get_task().lower() == 'rest':
+    if cfg.task_is_rest:
         msg = '    … skipping: for "rest" task.'
         logger.info(**gen_log_kwargs(message=msg))
         return

diff --git a/scripts/source/_99_group_average.py b/scripts/source/_99_group_average.py
@@ -36,8 +36,8 @@ def morph_stc(cfg, subject, fs_subject, session=None):
 
     morphed_stcs = []
 
-    if cfg.task == 'rest':
-        conditions = ['rest']
+    if cfg.task_is_rest:
+        conditions = [cfg.task.lower()]
     else:
         if isinstance(cfg.conditions, dict):
             conditions = list(cfg.conditions.keys())
@@ -104,6 +104,7 @@ def run_average(cfg, session, mean_morphed_stcs):
 def get_config() -> SimpleNamespace:
     cfg = SimpleNamespace(
         task=config.get_task(),
+        task_is_rest=config.task_is_rest,
         datatype=config.get_datatype(),
         acq=config.acq,
         rec=config.rec,

diff --git a/tests/configs/config_ds000247.py b/tests/configs/config_ds000247.py
@@ -11,6 +11,7 @@
 subjects = ['0002']
 sessions = ['01']
 task = 'rest'
+task_is_rest = True
 
 crop_runs = (0, 100)  # to speed up computations