Merge pull request #262 from khanlab/pythondeps

Remove script python dependencies from hippunfold package, moved this to hippunfold_deps instead. This container is now used for *all* rules

docs/contributing/contributing.md

@@ -4,7 +4,7 @@ Hippunfold python package dependencies are managed with Poetry, which you\'ll ne
 installed on your machine. You can find instructions on the [poetry
 website](https://python-poetry.org/docs/master/#installation).
 
-HippUnfold also has a number of dependencies outside of python, including popular neuroimaging tools like `wb_command`, `ANTs`, `c3d`, and others listed in https://github.com/khanlab/autotop_deps. Thus we strongly recommend running HippUnfold with the `--use-singularity` flag, which will pull this container automatically and use it when required, unless you are comfortable using all of these tools yourself. 
+HippUnfold also has a number of dependencies outside of python, including popular neuroimaging tools like `wb_command`, `ANTs`, `c3d`, and others listed in https://github.com/khanlab/hippunfold_deps. Thus we strongly recommend running HippUnfold with the `--use-singularity` flag, which will pull this container automatically and use it when required, unless you are comfortable using all of these tools yourself. 
 
 Note: These instructions are only recommended if you are making changes to HippUnfold code to commit back to this repository, or if you are using Snakemake's cluster execution profiles. If not, it is easier to run HippUnfold when it is packaged into a single singularity container (e.g. `docker://khanlab/hippunfold:latest`). 
 
@@ -33,6 +33,8 @@ or you can activate a virtualenv shell and then run hippunfold directly:
 
 You can exit the poetry shell with `exit`.
 
+Note: you can alternatively use `pip install` to install dependencies.
+
 ## Running code format quality checking and fixing:
 
 Hippunfold uses [poethepoet](https://github.com/nat-n/poethepoet) as a

docs/getting_started/installation.md

@@ -74,7 +74,7 @@ Instructions for this can be found in the **Contributing** documentation page.
 
 #### Pros:
 - Complete flexibility to modify code
-- External (Non-Python) dependencies as Singularity containers
+- External (python and non-python) dependencies as Singularity containers
 
 #### Cons:
 - Must use Python virtual environment

hippunfold/config/snakebids.yml

@@ -335,10 +335,7 @@ cifti_metric_types:
 #--- workflow specific configuration -- 
 
 singularity:
-  prepdwi: 'docker://khanlab/prepdwi:latest'  
-  autotop: 'docker://khanlab/autotop_deps:v0.4.3'
-  fsl: '/project/6050199/akhanf/singularity/bids-apps/fsl_6.0.3_cuda9.1.sif'  #fsl with cuda container not on docker hub yet.. only used for dwi workflow anyhow..
-  ants: 'docker://kaczmarj/ants:2.3.4'
+  autotop: 'docker://khanlab/hippunfold_deps:v0.5.0'
 
 xfm_identity: resources/identity_xfm.txt
 template: CITI168

hippunfold/workflow/Snakefile

@@ -118,12 +118,10 @@ if config["modality"] == "segT1w" or config["modality"] == "segT2w":
 
     include: "rules/preproc_seg.smk"
 
-
 elif config["modality"] == "cropseg":
 
     include: "rules/preproc_cropseg.smk"
 
-
 else:
 
     include: "rules/nnunet.smk"

hippunfold/workflow/rules/autotop.smk

@@ -1,5 +1,4 @@
 import os
-import numpy as np
 
 
 def get_cmd_laplace_coords():
@@ -79,6 +78,8 @@ rule laplace_coords_hipp:
             hemi="{hemi}",
             suffix="laplace-hipp.txt"
         ),
+    container:
+        config["singularity"]["autotop"]
     script:
         get_cmd_laplace_coords()
 
@@ -120,6 +121,8 @@ rule laplace_coords_dentate:
             hemi="{hemi}",
             suffix="laplace-dentate.txt"
         ),
+    container:
+        config["singularity"]["autotop"]
     shell:
         "cp {input} {output}"
 
@@ -160,6 +163,8 @@ rule prep_equivolume_coords:
         ),
     group:
         "subj"
+    container:
+        config["singularity"]["autotop"]
     script:
         "../scripts/prep_equivolume_coords.py"
 

hippunfold/workflow/rules/gifti.smk

@@ -136,6 +136,8 @@ rule constrain_surf_to_bbox:
         ),
     group:
         "subj"
+    container:
+        config["singularity"]["autotop"]
     script:
         "../scripts/constrain_surf_to_bbox.py"
 
@@ -225,6 +227,8 @@ rule correct_nan_vertices1:
         ),
     group:
         "subj"
+    container:
+        config["singularity"]["autotop"]
     script:
         "../scripts/fillnanvertices.py"
 
@@ -379,6 +383,8 @@ rule normalize_curvature1:
         ),
     group:
         "subj"
+    container:
+        config["singularity"]["autotop"]
     script:
         "../scripts/normalize_tanh.py"
 
@@ -637,6 +643,8 @@ rule warp_gii_unfoldreg:
         ),
     group:
         "subj"
+    container:
+        config["singularity"]["autotop"]
     script:
         "../scripts/warp_flatsurf.py"
 
@@ -669,6 +677,8 @@ rule dentate_skip_unfoldreg:
         ),
     group:
         "subj"
+    container:
+        config["singularity"]["autotop"]
     shell:
         "cp {input} {output}"
 
@@ -773,6 +783,8 @@ rule correct_nan_vertices2:
         ),
     group:
         "subj"
+    container:
+        config["singularity"]["autotop"]
     script:
         "../scripts/fillnanvertices.py"
 
@@ -803,6 +815,8 @@ rule cp_corobl_root:
         ),
     group:
         "subj"
+    container:
+        config["singularity"]["autotop"]
     shell:
         "cp {input} {output}"
 
@@ -990,6 +1004,8 @@ rule normalize_curvature2:
         ),
     group:
         "subj"
+    container:
+        config["singularity"]["autotop"]
     script:
         "../scripts/normalize_tanh.py"
 
@@ -1074,7 +1090,7 @@ rule resample_atlas_to_refvol:
             **config["subj_wildcards"]
         ),
     container:
-        config["singularity"]["ants"]
+        config["singularity"]["autotop"]
     group:
         "subj"
     shell:
@@ -1304,6 +1320,8 @@ rule cp_unfolded_noconstrain:
         ),
     group:
         "subj"
+    container:
+        config["singularity"]["autotop"]
     shell:
         "cp {input} {output}"
 

hippunfold/workflow/rules/nnunet.smk

@@ -96,6 +96,8 @@ rule download_model:
         else config["nnunet_model"][config["modality"]],
     output:
         model_tar=get_model_tar(),
+    container:
+        config["singularity"]["autotop"]
     shell:
         "wget https://{params.url} -O {output}"
 
@@ -143,6 +145,8 @@ rule run_inference:
         time=30 if config["use_gpu"] else 60,
     group:
         "subj"
+    container:
+        config["singularity"]["autotop"]
     shell:
         #create temp folders
         #cp input image to temp folder
@@ -326,5 +330,7 @@ rule qc_nnunet_dice:
         ),
     group:
         "subj"
+    container:
+        config["singularity"]["autotop"]
     script:
         "../scripts/dice.py"

hippunfold/workflow/rules/preproc_cropseg.smk

@@ -12,6 +12,8 @@ rule import_cropseg:
         ),
     group:
         "subj"
+    container:
+        config["singularity"]["autotop"]
     shell:
         "cp {input} {output}"
 

hippunfold/workflow/rules/preproc_hippb500.smk

@@ -84,5 +84,7 @@ rule cp_b500_to_anat_dir:
         ),
     group:
         "subj"
+    container:
+        config["singularity"]["autotop"]
     shell:
         "cp {input} {output}"

hippunfold/workflow/rules/preproc_seg.smk

@@ -14,6 +14,8 @@ rule import_seg:
         ),
     group:
         "subj"
+    container:
+        config["singularity"]["autotop"]
     shell:
         "cp {input} {output}"
 

hippunfold/workflow/rules/preproc_t1.smk

@@ -42,7 +42,6 @@ if config["skip_preproc"]:
         shell:
             "cp {input} {output}"
 
-
 else:
 
     rule n4_t1:

hippunfold/workflow/rules/preproc_t2.smk

@@ -158,7 +158,6 @@ if config["skip_preproc"]:
         shell:
             "cp {input} {output}"
 
-
 else:
 
     # average aligned n4 images to get preproc T2w (or if single scan, just copy it)

hippunfold/workflow/rules/qc.smk

@@ -28,6 +28,8 @@ rule qc_reg_to_template:
         ),
     group:
         "subj"
+    container:
+        config["singularity"]["autotop"]
     script:
         "../scripts/vis_regqc.py"
 
@@ -67,6 +69,8 @@ rule get_subfield_vols_subj:
             suffix="volumes.tsv",
             **config["subj_wildcards"]
         ),
+    container:
+        config["singularity"]["autotop"]
     script:
         "../scripts/gen_volume_tsv.py"
 
@@ -98,6 +102,8 @@ rule plot_subj_subfields:
         ),
     group:
         "subj"
+    container:
+        config["singularity"]["autotop"]
     script:
         "../scripts/plot_subj_subfields.py"
 
@@ -180,6 +186,8 @@ rule qc_subfield:
         ),
     group:
         "subj"
+    container:
+        config["singularity"]["autotop"]
     script:
         "../scripts/vis_qc_dseg.py"
 
@@ -214,6 +222,8 @@ rule qc_subfield_surf:
         ),
     group:
         "subj"
+    container:
+        config["singularity"]["autotop"]
     script:
         "../scripts/vis_qc_surf.py"
 
@@ -250,9 +260,7 @@ rule concat_subj_vols_tsv:
             from_="{modality}",
             suffix="volumes.tsv",
         ),
-    run:
-        import pandas as pd
-
-        pd.concat([pd.read_table(in_tsv) for in_tsv in input]).to_csv(
-            output.tsv, sep="\t", index=False
-        )
+    container:
+        config["singularity"]["autotop"]
+    script:
+        "../scripts/concat_tsv.py"

hippunfold/workflow/rules/shape_inject.smk

@@ -106,6 +106,8 @@ rule import_template_shape:
         ),
     group:
         "subj"
+    container:
+        config["singularity"]["autotop"]
     shell:
         "cp {input} {output}"
 

hippunfold/workflow/rules/subfields.smk

@@ -116,6 +116,8 @@ rule label_subfields_from_vol_coords_corobl:
         ),
     group:
         "subj"
+    container:
+        config["singularity"]["autotop"]
     script:
         "../scripts/label_subfields_from_vol_coords.py"
 

hippunfold/workflow/rules/warps.smk

@@ -226,6 +226,8 @@ rule create_warps_hipp:
             hemi="{hemi}",
             suffix="create_warps-hipp.txt"
         ),
+    container:
+        config["singularity"]["autotop"]
     script:
         "../scripts/create_warps.py"
 
@@ -351,6 +353,8 @@ rule create_warps_dentate:
             hemi="{hemi}",
             suffix="create_warps-dentate.txt"
         ),
+    container:
+        config["singularity"]["autotop"]
     script:
         "../scripts/create_warps.py"
 
@@ -393,6 +397,8 @@ rule expand_unfolded_warps:
         ),
     group:
         "subj"
+    container:
+        config["singularity"]["autotop"]
     script:
         "../scripts/expand_2Dwarp.py"
 
@@ -475,7 +481,7 @@ rule compose_warps_native_to_unfold:
             mode="image"
         ),
     container:
-        config["singularity"]["ants"]
+        config["singularity"]["autotop"]
     group:
         "subj"
     shell:
@@ -578,7 +584,7 @@ rule compose_warps_unfold_to_crop_native:
     params:
         cmd=get_cmd_compose_warps_unfold_to_crop_native,
     container:
-        config["singularity"]["ants"]
+        config["singularity"]["autotop"]
     group:
         "subj"
     shell:

hippunfold/workflow/scripts/concat_tsv.py

@@ -0,0 +1,5 @@
+import pandas as pd
+
+pd.concat([pd.read_table(in_tsv) for in_tsv in snakemake.input]).to_csv(
+    snakemake.output.tsv, sep="\t", index=False
+)