diff --git a/.github/workflows/linting.yml b/.github/workflows/lint_and_test.yml similarity index 52% rename from .github/workflows/linting.yml rename to .github/workflows/lint_and_test.yml index ef50157..3782d6e 100644 --- a/.github/workflows/linting.yml +++ b/.github/workflows/lint_and_test.yml @@ -1,9 +1,11 @@ -name: lint and install +name: tests on: push: branches: - main + tags: + - '*' pull_request: jobs: @@ -12,24 +14,25 @@ jobs: steps: - uses: neuroinformatics-unit/actions/lint@main - - install: + test: needs: [linting] name: ${{ matrix.os }} py${{ matrix.python-version }} runs-on: ${{ matrix.os }} - defaults: - run: - shell: bash -l {0} # https://stackoverflow.com/questions/69070754/shell-bash-l-0-in-github-actions - strategy: matrix: os: [windows-latest, ubuntu-latest, macos-latest] python-version: ["3.8", "3.9", "3.10", "3.11"] steps: - - uses: actions/checkout@v4 - - name: Install dependencies - run: - python3 -m pip install --upgrade pip - pip install . + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install .[dev] + - name: Test + run: pytest diff --git a/.gitignore b/.gitignore index 152c86f..86c6ab4 100644 --- a/.gitignore +++ b/.gitignore @@ -2,7 +2,7 @@ .obsidian slurm_logs/ derivatives/ -tests/data/ +tests/data/steve_multi_run # Byte-compiled / optimized / DLL files __pycache__/ *.py[cod] diff --git a/pyproject.toml b/pyproject.toml index 0604bad..27a0630 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -20,8 +20,8 @@ classifiers = [ ] dependencies = [ - "spikeinterface==0.98.2", - "probeinterface==0.2.18", + "spikeinterface[full] @git+https://github.com/JoeZiminski/spikeinterface.git@random_chunks_chunk_size_to_warning", + "probeinterface", "neo==0.12.0", "submitit", "PyYAML", @@ -30,7 +30,7 @@ dependencies = [ # sorter-specific "tridesclous", # "spyking-circus", TODO: this is not straightforward, requires mpi4py. TBD if we want to manage this. - "mountainsort5; platform_system != 'Darwin'", + "mountainsort5", "docker; platform_system=='Windows'", "docker; platform_system=='Darwin'", "spython; platform_system=='Linux'", # I think missing from SI? diff --git a/spikewrap/configs/fast_test_pipeline.yaml b/spikewrap/configs/fast_test_pipeline.yaml new file mode 100644 index 0000000..ec6f56b --- /dev/null +++ b/spikewrap/configs/fast_test_pipeline.yaml @@ -0,0 +1,38 @@ +'preprocessing': + '1': + - phase_shift + - {} + '2': + - bandpass_filter + - freq_min: 300 + freq_max: 6000 + '3': + - common_reference + - operator: median + reference: global + +'sorting': + 'kilosort2': + 'car': False # common average referencing + 'freq_min': 150 # highpass filter cutoff, False nor 0 does not work to turn off. (results in KS error) + 'kilosort2_5': + 'car': False + 'freq_min': 150 + 'kilosort3': + 'car': False + 'freq_min': 300 + 'mountainsort5': + 'scheme': '2' + 'filter': False + 'whiten': False + +'waveforms': + 'ms_before': 2 + 'ms_after': 2 + 'max_spikes_per_unit': 500 + 'return_scaled': True + # Sparsity Options + 'sparse': True + 'peak_sign': "neg" + 'method': "radius" + 'radius_um': 75 diff --git a/spikewrap/configs/test_default.yaml b/spikewrap/configs/test_default.yaml index 432526f..1dc610a 100644 --- a/spikewrap/configs/test_default.yaml +++ b/spikewrap/configs/test_default.yaml @@ -27,6 +27,7 @@ 'mountainsort5': 'scheme': '2' 'filter': False + 'whiten': False # TODO: add this back. 'waveforms': 'ms_before': 2 @@ -38,3 +39,4 @@ 'peak_sign': "neg" 'method': "radius" 'radius_um': 75 + 'chunk_size': 1 diff --git a/spikewrap/examples/example_full_pipeline.py b/spikewrap/examples/example_full_pipeline.py index b1cc773..87cc0a3 100644 --- a/spikewrap/examples/example_full_pipeline.py +++ b/spikewrap/examples/example_full_pipeline.py @@ -4,8 +4,8 @@ from spikewrap.pipeline.full_pipeline import run_full_pipeline base_path = Path( - "/ceph/neuroinformatics/neuroinformatics/scratch/jziminski/ephys/code/spikewrap/tests/data/steve_multi_run/time-short-multises" - # "/ceph/neuroinformatics/neuroinformatics/scratch/jziminski/ephys/test_data/steve_multi_run/1119617/time-short-multises" + # "/ceph/neuroinformatics/neuroinformatics/scratch/jziminski/ephys/code/spikewrap/tests/data/steve_multi_run/time-short-multises" + "/ceph/neuroinformatics/neuroinformatics/scratch/jziminski/ephys/test_data/steve_multi_run/1119617/time-short-multises" # r"C:\data\ephys\test_data\steve_multi_run\1119617\time-miniscule-mutlises" # r"C:\data\ephys\test_data\steve_multi_run\1119617\time-short-multises" # "/ceph/neuroinformatics/neuroinformatics/scratch/jziminski/ephys/test_data/steve_multi_run/1119617/time-short" @@ -36,7 +36,7 @@ sessions_and_runs, config_name, sorter, - concat_sessions_for_sorting=True, # TODO: validate this at the start, in `run_full_pipeline` + concat_sessions_for_sorting=False, # TODO: validate this at the start, in `run_full_pipeline` concat_runs_for_sorting=True, existing_preprocessed_data="skip_if_exists", # this is kind of confusing... existing_sorting_output="overwrite", diff --git a/spikewrap/pipeline/sort.py b/spikewrap/pipeline/sort.py index 80cdf81..5680002 100644 --- a/spikewrap/pipeline/sort.py +++ b/spikewrap/pipeline/sort.py @@ -1,7 +1,6 @@ from __future__ import annotations import os -import platform from pathlib import Path from typing import TYPE_CHECKING, Dict, List, Literal, Optional, Union @@ -152,9 +151,6 @@ def _run_sorting( passed_arguments = locals() validate.check_function_arguments(passed_arguments) - if sorter == "mountainsort5" and platform.system() == "Darwin": - raise EnvironmentError("Mountainsort is not currently supported on macOS.") - logs = logging_sw.get_started_logger( utils.get_logging_path(base_path, sub_name), "sorting", diff --git a/tests/data/small_toy_data/generate_test_data.py b/tests/data/small_toy_data/generate_test_data.py new file mode 100644 index 0000000..806f287 --- /dev/null +++ b/tests/data/small_toy_data/generate_test_data.py @@ -0,0 +1,64 @@ +from pathlib import Path + +import numpy as np +import spikeinterface.extractors as se +import spikeinterface.preprocessing as spre + +print("Starting...") + +sessions_and_runs = { + "ses-001": ["ses-001_run-001", "ses-001_run-002"], + "ses-002": ["ses-002_run-001", "ses-002_run-002"], + "ses-003": ["ses-003_run-001", "ses-003_run-002"], +} + +if False: + from spikewrap.pipeline.preprocess import run_preprocessing + + sub_name = "sub-001_type-test" + base_path = Path(__file__).parent.resolve() + + from spikewrap.pipeline.load_data import load_data + + preprocess_data = load_data( + base_path, sub_name, sessions_and_runs, "spikeinterface" + ) + + run_preprocessing( + preprocess_data, + "default", + handle_existing_data="overwrite", + slurm_batch=False, + log=False, + ) # TODO: use config_name for all funcs. + + +# if False: + +sub = "sub-001_type-test" +base_path = Path(__file__).parent.resolve() / "rawdata" + +for ses in sessions_and_runs.keys(): + for run in sessions_and_runs[ses]: + num_channels = 16 + recording, _ = se.toy_example( + duration=[0.05], num_segments=1, num_channels=num_channels, num_units=2 + ) + four_shank_groupings = np.repeat([0, 1, 2, 3], 4) + recording.set_property("group", four_shank_groupings) + recording.set_property("inter_sample_shift", np.arange(16) * 0.0001) + + recording._main_ids = np.array( + [f"imec0.ap#AP{i}" for i in range(num_channels)] + ) # for consistency with spikeglx dataset TODO this is a hack + + recording = spre.scale(recording, gain=50, offset=20) + + output_path = base_path / sub / ses / "ephys" / run + + recording.save(folder=output_path, chunk_size=1000000) + +# shifted_recording = spre.phase_shift(recording) +# filtered_recording = spre.bandpass_filter(recording) +# referenced_recording = spre.common_reference(filtered_recording) +# run_sorter("mountainsort5", referenced_recording, remove_existing_folder=True, whiten=False) diff --git a/tests/data/small_toy_data/rawdata/sub-001_type-test/ses-001/ephys/ses-001_run-001/binary.json b/tests/data/small_toy_data/rawdata/sub-001_type-test/ses-001/ephys/ses-001_run-001/binary.json new file mode 100644 index 0000000..0fd0e1a --- /dev/null +++ b/tests/data/small_toy_data/rawdata/sub-001_type-test/ses-001/ephys/ses-001_run-001/binary.json @@ -0,0 +1,115 @@ +{ + "class": "spikeinterface.core.binaryrecordingextractor.BinaryRecordingExtractor", + "module": "spikeinterface", + "kwargs": { + "file_paths": [ + "traces_cached_seg0.raw" + ], + "sampling_frequency": 30000.0, + "t_starts": null, + "num_channels": 16, + "dtype": "