Add toil-lib dependency (resolves BD2KGenomics/toil#922)

setup.py

@@ -73,7 +73,7 @@ def check_provided(distribution, min_version, max_version=None, optional=False):
     author_email='[email protected]',
     url="https://github.com/BD2KGenomics/toil-scripts",
     install_requires=[
-        'tqdm==3.8.0',  # FIXME: Remove once ADAM stops using it (superfluous import)
+        'toil-lib==1.0.3',
         'pyyaml==3.11'],
     tests_require=[
         'pytest==2.8.3'],

src/toil_scripts/__init__.py

@@ -2,7 +2,7 @@
 
 _log = logging.getLogger(__name__)
 
-# TODO: replace this with toil_scripts.lib.urls._download_s3_url
+# TODO: replace this with toil_lib.urls._download_s3_url
 
 def download_from_s3_url(file_path, url):
     from urlparse import urlparse

src/toil_scripts/adam_gatk_pipeline/align_and_call.py

@@ -123,14 +123,15 @@
 # these don't seem necessary! but, must be imported here due to a serialization issue
 from toil.lib.spark import spawn_spark_cluster
 
+from toil_lib.programs import mock_mode
+
 # import job steps from other toil pipelines
 from toil_scripts.adam_pipeline.adam_preprocessing import * #static_adam_preprocessing_dag
 from toil_scripts.bwa_alignment.bwa_alignment import * #download_shared_files
 from toil_scripts.gatk_germline.germline import * #batch_start
 from toil_scripts.gatk_processing.gatk_preprocessing import * #download_gatk_files
 from toil_scripts.rnaseq_cgl.rnaseq_cgl_pipeline import generate_file
 
-from toil_scripts.lib.programs import mock_mode
 
 def sample_loop(job, uuid_list, inputs):
   """

src/toil_scripts/adam_kmers/count_kmers.py

@@ -11,8 +11,8 @@
 from toil.lib.spark import spawn_spark_cluster
 
 # imports from toil_scripts
-from toil_scripts.lib import require
-from toil_scripts.tools.spark_tools import call_adam, call_conductor, \
+from toil_lib import require
+from toil_lib.tools.spark_tools import call_adam, call_conductor, \
     MasterAddress, HDFS_MASTER_PORT, SPARK_MASTER_PORT
 
 

src/toil_scripts/adam_pipeline/adam_preprocessing.py

@@ -39,11 +39,12 @@
 from toil.job import Job
 from toil.lib.spark import spawn_spark_cluster
 
-from toil_scripts.lib import require
-from toil_scripts.lib.files import copy_files, move_files
-from toil_scripts.lib.programs import docker_call, mock_mode
+from toil_lib import require
+from toil_lib.files import copy_files, move_files
+from toil_lib.programs import docker_call, mock_mode
+from toil_lib.tools.spark_tools import call_adam, call_conductor, MasterAddress, HDFS_MASTER_PORT, SPARK_MASTER_PORT
+
 from toil_scripts.rnaseq_cgl.rnaseq_cgl_pipeline import generate_file
-from toil_scripts.tools.spark_tools import call_adam, call_conductor, MasterAddress, HDFS_MASTER_PORT, SPARK_MASTER_PORT
 
 log = logging.getLogger(__name__)
 

src/toil_scripts/bwa_alignment/bwa_alignment.py

@@ -8,14 +8,14 @@
 
 import yaml
 from toil.job import Job
+from toil_lib import require, required_length
+from toil_lib.files import copy_file_job
+from toil_lib.jobs import map_job
+from toil_lib.tools.aligners import run_bwakit
+from toil_lib.tools.indexing import run_samtools_faidx, run_bwa_index
+from toil_lib.urls import download_url_job, s3am_upload_job
 
-from toil_scripts.lib import require, required_length
-from toil_scripts.lib.files import copy_file_job
-from toil_scripts.lib.jobs import map_job
-from toil_scripts.lib.urls import download_url_job, s3am_upload_job
 from toil_scripts.rnaseq_cgl.rnaseq_cgl_pipeline import generate_file
-from toil_scripts.tools.aligners import run_bwakit
-from toil_scripts.tools.indexing import run_samtools_faidx, run_bwa_index
 
 
 def download_reference_files(job, inputs, samples):

src/toil_scripts/exome_variant_pipeline/exome_variant_pipeline.py

@@ -12,18 +12,17 @@
 from bd2k.util.files import mkdir_p
 from bd2k.util.processes import which
 from toil.job import Job
-
-from toil_scripts.lib import require
-from toil_scripts.lib.files import copy_files
-from toil_scripts.lib.jobs import map_job
-from toil_scripts.lib.urls import download_url_job, s3am_upload
-from toil_scripts.tools.mutation_callers import run_muse
-from toil_scripts.tools.mutation_callers import run_mutect
-from toil_scripts.tools.mutation_callers import run_pindel
-from toil_scripts.tools.preprocessing import run_gatk_preprocessing
-from toil_scripts.tools.preprocessing import run_picard_create_sequence_dictionary
-from toil_scripts.tools.preprocessing import run_samtools_faidx
-from toil_scripts.tools.preprocessing import run_samtools_index
+from toil_lib import require
+from toil_lib.files import copy_files
+from toil_lib.jobs import map_job
+from toil_lib.tools.mutation_callers import run_muse
+from toil_lib.tools.mutation_callers import run_mutect
+from toil_lib.tools.mutation_callers import run_pindel
+from toil_lib.tools.preprocessing import run_gatk_preprocessing
+from toil_lib.tools.preprocessing import run_picard_create_sequence_dictionary
+from toil_lib.tools.preprocessing import run_samtools_faidx
+from toil_lib.tools.preprocessing import run_samtools_index
+from toil_lib.urls import download_url_job, s3am_upload
 
 
 # Start of Job Functions

src/toil_scripts/gatk_germline/germline.py

@@ -42,10 +42,11 @@
 
 import yaml
 from toil.job import Job
+from toil_lib import require
+from toil_lib.programs import docker_call
+from toil_lib.urls import s3am_upload
+
 from toil_scripts import download_from_s3_url
-from toil_scripts.lib import require
-from toil_scripts.lib.programs import docker_call
-from toil_scripts.lib.urls import s3am_upload
 from toil_scripts.rnaseq_cgl.rnaseq_cgl_pipeline import generate_file
 
 

src/toil_scripts/gatk_processing/gatk_preprocessing.py

@@ -33,10 +33,11 @@
 
 import yaml
 from toil.job import Job
+from toil_lib import require
+from toil_lib.programs import docker_call
+from toil_lib.urls import s3am_upload
+
 from toil_scripts import download_from_s3_url
-from toil_scripts.lib import require
-from toil_scripts.lib.programs import docker_call
-from toil_scripts.lib.urls import s3am_upload
 from toil_scripts.rnaseq_cgl.rnaseq_cgl_pipeline import generate_file
 
 _log = logging.getLogger(__name__)