diff --git a/FactorNet/CEBPB/meta_Unique35_DGF/dataloader.py b/FactorNet/CEBPB/meta_Unique35_DGF/dataloader.py index fe485d008..08a180cc9 100644 --- a/FactorNet/CEBPB/meta_Unique35_DGF/dataloader.py +++ b/FactorNet/CEBPB/meta_Unique35_DGF/dataloader.py @@ -29,9 +29,9 @@ def download_gencode_dir(output_dir): """Download all the required gencode files """ makedir_exist_ok(output_dir) - url_template = ("https://github.com/kipoi/models/blob/" - "7648d3fd57def50934835b52acadd26bcaaa275c/FactorNet/" - "template/dataloader_files/gencode_features/{}?raw=true") + + url_template = ("https://s3.eu-central-1.amazonaws.com/kipoi-models/" + "dataloader_files/FactorNet/dataloader_files/gencode_features/{}") # url_template = "https://github.com/uci-cbcl/FactorNet/blob/master/resources/{}?raw=true" fnames = [('cpgisland.bed.gz', 'ac7dc007d7019c05adb7a331d1d6721d'), @@ -87,8 +87,6 @@ def __init__(self, self.bt = BT(intervals_file) - - # Fasta self.fasta_file = fasta_file self.fasta_extractor = None # initialize later @@ -97,7 +95,7 @@ def __init__(self, self.dnase_extractor = None # mappability if mappability_file is None: - # download the mappability file if not existing + # download the mappability file if not existing common_dl_dir = os.path.join(this_dir, "../../template/downloaded/dataloader_files") makedir_exist_ok(common_dl_dir) rf = RemoteFile(url="http://hgdownload.cse.ucsc.edu/goldenPath/hg19/encodeDCC/wgEncodeMapability/wgEncodeDukeMapabilityUniqueness35bp.bigWig", @@ -118,10 +116,10 @@ def __init__(self, output_dir = os.path.join(this_dir, "../../template/downloaded/dataloader_files/RNAseq_features/") makedir_exist_ok(output_dir) RNAseq_PC_file = os.path.join(output_dir, cell_line, "meta.txt") - url_template = ('https://github.com/kipoi/models/blob/7648d3fd57def50934835b52acadd26bcaaa275c' - '/FactorNet/template/dataloader_files/RNAseq_features/{}/meta.txt?raw=true') + url_template = ('https://s3.eu-central-1.amazonaws.com/kipoi-models/dataloader_files/' + 'FactorNet/dataloader_files/RNAseq_features/{}/meta.txt') # rf = RemoteFile(url=url_template.format(cell_line)) - if not os.path.exists(RNAseq_PC_file): # or not rf.validate(mappability_file): + if not os.path.exists(RNAseq_PC_file): # or not rf.validate(mappability_file): # download the path download_url(url_template.format(cell_line), os.path.join(output_dir, cell_line), "meta.txt") # rf.get_file(RNAseq_PC_file) @@ -181,4 +179,4 @@ def __getitem__(self, idx): "ranges": ranges, "ranges_rc": ranges_rc } - } \ No newline at end of file + } diff --git a/FactorNet/CEBPB/meta_Unique35_DGF/dataloader.yaml b/FactorNet/CEBPB/meta_Unique35_DGF/dataloader.yaml index 5f16f5a9b..b843c7857 100644 --- a/FactorNet/CEBPB/meta_Unique35_DGF/dataloader.yaml +++ b/FactorNet/CEBPB/meta_Unique35_DGF/dataloader.yaml @@ -26,7 +26,7 @@ args: mappability_file: doc: USCS mappability track - http://hgdownload.cse.ucsc.edu/goldenpath/hg19/encodeDCC/wgEncodeMapability/wgEncodeDukeMapabilityUniqueness35bp.bigWig. by deafult, provide this file with the dataloader, download in background example: - url: https://github.com/kipoi/models/blob/92b23ed62956d56eb457488b431fc27550ddbf32/FactorNet/template/dataloader_files/wgEncodeDukeMapabilityUniqueness35bp.chr22.bigWig?raw=true + url: https://s3.eu-central-1.amazonaws.com/kipoi-models/dataloader_files/FactorNet/dataloader_files/wgEncodeDukeMapabilityUniqueness35bp.chr22.bigWig md5: 912e1bb77fb67c1a792ac4a05d9416f9 optional: True use_linecache: @@ -68,4 +68,4 @@ output_schema: doc: Ranges describing inputs.seq ranges_rc: type: GenomicRanges - doc: Ranges describing inputs.seq_rc \ No newline at end of file + doc: Ranges describing inputs.seq_rc diff --git a/FactorNet/CEBPB/onePeak_1_DGF/dataloader.py b/FactorNet/CEBPB/onePeak_1_DGF/dataloader.py index 5e737679a..3c201bf4f 100644 --- a/FactorNet/CEBPB/onePeak_1_DGF/dataloader.py +++ b/FactorNet/CEBPB/onePeak_1_DGF/dataloader.py @@ -29,9 +29,8 @@ def download_gencode_dir(output_dir): """Download all the required gencode files """ makedir_exist_ok(output_dir) - url_template = ("https://github.com/kipoi/models/blob/" - "7648d3fd57def50934835b52acadd26bcaaa275c/FactorNet/" - "template/dataloader_files/gencode_features/{}?raw=true") + url_template = ("https://s3.eu-central-1.amazonaws.com/kipoi-models/" + "dataloader_files/FactorNet/dataloader_files/gencode_features/{}") # url_template = "https://github.com/uci-cbcl/FactorNet/blob/master/resources/{}?raw=true" fnames = [('cpgisland.bed.gz', 'ac7dc007d7019c05adb7a331d1d6721d'), @@ -84,8 +83,6 @@ def __init__(self, self.bt = BT(intervals_file) - - # Fasta self.fasta_file = fasta_file self.fasta_extractor = None # initialize later @@ -138,4 +135,4 @@ def __getitem__(self, idx): "ranges": ranges, "ranges_rc": ranges_rc } - } \ No newline at end of file + } diff --git a/FactorNet/CEBPB/onePeak_2_Unique35_DGF/dataloader.py b/FactorNet/CEBPB/onePeak_2_Unique35_DGF/dataloader.py index d36ad58ef..775a01792 100644 --- a/FactorNet/CEBPB/onePeak_2_Unique35_DGF/dataloader.py +++ b/FactorNet/CEBPB/onePeak_2_Unique35_DGF/dataloader.py @@ -29,9 +29,8 @@ def download_gencode_dir(output_dir): """Download all the required gencode files """ makedir_exist_ok(output_dir) - url_template = ("https://github.com/kipoi/models/blob/" - "7648d3fd57def50934835b52acadd26bcaaa275c/FactorNet/" - "template/dataloader_files/gencode_features/{}?raw=true") + url_template = ("https://s3.eu-central-1.amazonaws.com/kipoi-models/" + "dataloader_files/FactorNet/dataloader_files/gencode_features/{}") # url_template = "https://github.com/uci-cbcl/FactorNet/blob/master/resources/{}?raw=true" fnames = [('cpgisland.bed.gz', 'ac7dc007d7019c05adb7a331d1d6721d'), @@ -85,8 +84,6 @@ def __init__(self, self.bt = BT(intervals_file) - - # Fasta self.fasta_file = fasta_file self.fasta_extractor = None # initialize later @@ -95,7 +92,7 @@ def __init__(self, self.dnase_extractor = None # mappability if mappability_file is None: - # download the mappability file if not existing + # download the mappability file if not existing common_dl_dir = os.path.join(this_dir, "../../template/downloaded/dataloader_files") makedir_exist_ok(common_dl_dir) rf = RemoteFile(url="http://hgdownload.cse.ucsc.edu/goldenPath/hg19/encodeDCC/wgEncodeMapability/wgEncodeDukeMapabilityUniqueness35bp.bigWig", @@ -158,4 +155,4 @@ def __getitem__(self, idx): "ranges": ranges, "ranges_rc": ranges_rc } - } \ No newline at end of file + } diff --git a/FactorNet/CEBPB/onePeak_2_Unique35_DGF/dataloader.yaml b/FactorNet/CEBPB/onePeak_2_Unique35_DGF/dataloader.yaml index 97a5f127b..96a8750c6 100644 --- a/FactorNet/CEBPB/onePeak_2_Unique35_DGF/dataloader.yaml +++ b/FactorNet/CEBPB/onePeak_2_Unique35_DGF/dataloader.yaml @@ -19,7 +19,7 @@ args: mappability_file: doc: USCS mappability track - http://hgdownload.cse.ucsc.edu/goldenpath/hg19/encodeDCC/wgEncodeMapability/wgEncodeDukeMapabilityUniqueness35bp.bigWig. by deafult, provide this file with the dataloader, download in background example: - url: https://github.com/kipoi/models/blob/92b23ed62956d56eb457488b431fc27550ddbf32/FactorNet/template/dataloader_files/wgEncodeDukeMapabilityUniqueness35bp.chr22.bigWig?raw=true + url: https://s3.eu-central-1.amazonaws.com/kipoi-models/dataloader_files/FactorNet/dataloader_files/wgEncodeDukeMapabilityUniqueness35bp.chr22.bigWig md5: 912e1bb77fb67c1a792ac4a05d9416f9 optional: True use_linecache: @@ -58,4 +58,4 @@ output_schema: doc: Ranges describing inputs.seq ranges_rc: type: GenomicRanges - doc: Ranges describing inputs.seq_rc \ No newline at end of file + doc: Ranges describing inputs.seq_rc diff --git a/FactorNet/CTCF/metaGENCODE_RNAseq_Unique35_DGF/dataloader.py b/FactorNet/CTCF/metaGENCODE_RNAseq_Unique35_DGF/dataloader.py index a82d2add4..463d66275 100644 --- a/FactorNet/CTCF/metaGENCODE_RNAseq_Unique35_DGF/dataloader.py +++ b/FactorNet/CTCF/metaGENCODE_RNAseq_Unique35_DGF/dataloader.py @@ -29,9 +29,9 @@ def download_gencode_dir(output_dir): """Download all the required gencode files """ makedir_exist_ok(output_dir) - url_template = ("https://github.com/kipoi/models/blob/" - "7648d3fd57def50934835b52acadd26bcaaa275c/FactorNet/" - "template/dataloader_files/gencode_features/{}?raw=true") + + url_template = ("https://s3.eu-central-1.amazonaws.com/kipoi-models/" + "dataloader_files/FactorNet/dataloader_files/gencode_features/{}") # url_template = "https://github.com/uci-cbcl/FactorNet/blob/master/resources/{}?raw=true" fnames = [('cpgisland.bed.gz', 'ac7dc007d7019c05adb7a331d1d6721d'), @@ -88,8 +88,6 @@ def __init__(self, self.bt = BT(intervals_file) - - # Fasta self.fasta_file = fasta_file self.fasta_extractor = None # initialize later @@ -98,7 +96,7 @@ def __init__(self, self.dnase_extractor = None # mappability if mappability_file is None: - # download the mappability file if not existing + # download the mappability file if not existing common_dl_dir = os.path.join(this_dir, "../../template/downloaded/dataloader_files") makedir_exist_ok(common_dl_dir) rf = RemoteFile(url="http://hgdownload.cse.ucsc.edu/goldenPath/hg19/encodeDCC/wgEncodeMapability/wgEncodeDukeMapabilityUniqueness35bp.bigWig", @@ -141,10 +139,10 @@ def __init__(self, output_dir = os.path.join(this_dir, "../../template/downloaded/dataloader_files/RNAseq_features/") makedir_exist_ok(output_dir) RNAseq_PC_file = os.path.join(output_dir, cell_line, "meta.txt") - url_template = ('https://github.com/kipoi/models/blob/7648d3fd57def50934835b52acadd26bcaaa275c' - '/FactorNet/template/dataloader_files/RNAseq_features/{}/meta.txt?raw=true') + url_template = ('https://s3.eu-central-1.amazonaws.com/kipoi-models/dataloader_files/' + 'FactorNet/dataloader_files/RNAseq_features/{}/meta.txt') # rf = RemoteFile(url=url_template.format(cell_line)) - if not os.path.exists(RNAseq_PC_file): # or not rf.validate(mappability_file): + if not os.path.exists(RNAseq_PC_file): # or not rf.validate(mappability_file): # download the path download_url(url_template.format(cell_line), os.path.join(output_dir, cell_line), "meta.txt") # rf.get_file(RNAseq_PC_file) diff --git a/FactorNet/CTCF/metaGENCODE_RNAseq_Unique35_DGF/dataloader.yaml b/FactorNet/CTCF/metaGENCODE_RNAseq_Unique35_DGF/dataloader.yaml index 56715a39e..bb2be34cc 100644 --- a/FactorNet/CTCF/metaGENCODE_RNAseq_Unique35_DGF/dataloader.yaml +++ b/FactorNet/CTCF/metaGENCODE_RNAseq_Unique35_DGF/dataloader.yaml @@ -26,7 +26,7 @@ args: mappability_file: doc: USCS mappability track - http://hgdownload.cse.ucsc.edu/goldenpath/hg19/encodeDCC/wgEncodeMapability/wgEncodeDukeMapabilityUniqueness35bp.bigWig. by deafult, provide this file with the dataloader, download in background example: - url: https://github.com/kipoi/models/blob/92b23ed62956d56eb457488b431fc27550ddbf32/FactorNet/template/dataloader_files/wgEncodeDukeMapabilityUniqueness35bp.chr22.bigWig?raw=true + url: https://s3.eu-central-1.amazonaws.com/kipoi-models/dataloader_files/FactorNet/dataloader_files/wgEncodeDukeMapabilityUniqueness35bp.chr22.bigWig md5: 912e1bb77fb67c1a792ac4a05d9416f9 optional: True GENCODE_dir: diff --git a/FactorNet/CTCF/meta_RNAseq_Unique35_DGF/dataloader.py b/FactorNet/CTCF/meta_RNAseq_Unique35_DGF/dataloader.py index fe485d008..1620ead6a 100644 --- a/FactorNet/CTCF/meta_RNAseq_Unique35_DGF/dataloader.py +++ b/FactorNet/CTCF/meta_RNAseq_Unique35_DGF/dataloader.py @@ -29,9 +29,9 @@ def download_gencode_dir(output_dir): """Download all the required gencode files """ makedir_exist_ok(output_dir) - url_template = ("https://github.com/kipoi/models/blob/" - "7648d3fd57def50934835b52acadd26bcaaa275c/FactorNet/" - "template/dataloader_files/gencode_features/{}?raw=true") + + url_template = ("https://s3.eu-central-1.amazonaws.com/kipoi-models/" + "dataloader_files/FactorNet/dataloader_files/gencode_features/{}") # url_template = "https://github.com/uci-cbcl/FactorNet/blob/master/resources/{}?raw=true" fnames = [('cpgisland.bed.gz', 'ac7dc007d7019c05adb7a331d1d6721d'), @@ -87,8 +87,6 @@ def __init__(self, self.bt = BT(intervals_file) - - # Fasta self.fasta_file = fasta_file self.fasta_extractor = None # initialize later @@ -97,7 +95,7 @@ def __init__(self, self.dnase_extractor = None # mappability if mappability_file is None: - # download the mappability file if not existing + # download the mappability file if not existing common_dl_dir = os.path.join(this_dir, "../../template/downloaded/dataloader_files") makedir_exist_ok(common_dl_dir) rf = RemoteFile(url="http://hgdownload.cse.ucsc.edu/goldenPath/hg19/encodeDCC/wgEncodeMapability/wgEncodeDukeMapabilityUniqueness35bp.bigWig", @@ -118,10 +116,10 @@ def __init__(self, output_dir = os.path.join(this_dir, "../../template/downloaded/dataloader_files/RNAseq_features/") makedir_exist_ok(output_dir) RNAseq_PC_file = os.path.join(output_dir, cell_line, "meta.txt") - url_template = ('https://github.com/kipoi/models/blob/7648d3fd57def50934835b52acadd26bcaaa275c' - '/FactorNet/template/dataloader_files/RNAseq_features/{}/meta.txt?raw=true') + url_template = ('https://s3.eu-central-1.amazonaws.com/kipoi-models/dataloader_files/' + 'FactorNet/dataloader_files/RNAseq_features/{}/meta.txt') # rf = RemoteFile(url=url_template.format(cell_line)) - if not os.path.exists(RNAseq_PC_file): # or not rf.validate(mappability_file): + if not os.path.exists(RNAseq_PC_file): # or not rf.validate(mappability_file): # download the path download_url(url_template.format(cell_line), os.path.join(output_dir, cell_line), "meta.txt") # rf.get_file(RNAseq_PC_file) diff --git a/FactorNet/CTCF/meta_RNAseq_Unique35_DGF/dataloader.yaml b/FactorNet/CTCF/meta_RNAseq_Unique35_DGF/dataloader.yaml index 5f16f5a9b..bd2b01c55 100644 --- a/FactorNet/CTCF/meta_RNAseq_Unique35_DGF/dataloader.yaml +++ b/FactorNet/CTCF/meta_RNAseq_Unique35_DGF/dataloader.yaml @@ -26,7 +26,7 @@ args: mappability_file: doc: USCS mappability track - http://hgdownload.cse.ucsc.edu/goldenpath/hg19/encodeDCC/wgEncodeMapability/wgEncodeDukeMapabilityUniqueness35bp.bigWig. by deafult, provide this file with the dataloader, download in background example: - url: https://github.com/kipoi/models/blob/92b23ed62956d56eb457488b431fc27550ddbf32/FactorNet/template/dataloader_files/wgEncodeDukeMapabilityUniqueness35bp.chr22.bigWig?raw=true + url: https://s3.eu-central-1.amazonaws.com/kipoi-models/dataloader_files/FactorNet/dataloader_files/wgEncodeDukeMapabilityUniqueness35bp.chr22.bigWig md5: 912e1bb77fb67c1a792ac4a05d9416f9 optional: True use_linecache: diff --git a/FactorNet/E2F1/GENCODE_Unique35_DGF/dataloader.py b/FactorNet/E2F1/GENCODE_Unique35_DGF/dataloader.py index 7e71a5de1..9809179bd 100644 --- a/FactorNet/E2F1/GENCODE_Unique35_DGF/dataloader.py +++ b/FactorNet/E2F1/GENCODE_Unique35_DGF/dataloader.py @@ -29,9 +29,9 @@ def download_gencode_dir(output_dir): """Download all the required gencode files """ makedir_exist_ok(output_dir) - url_template = ("https://github.com/kipoi/models/blob/" - "7648d3fd57def50934835b52acadd26bcaaa275c/FactorNet/" - "template/dataloader_files/gencode_features/{}?raw=true") + + url_template = ("https://s3.eu-central-1.amazonaws.com/kipoi-models/" + "dataloader_files/FactorNet/dataloader_files/gencode_features/{}") # url_template = "https://github.com/uci-cbcl/FactorNet/blob/master/resources/{}?raw=true" fnames = [('cpgisland.bed.gz', 'ac7dc007d7019c05adb7a331d1d6721d'), @@ -87,8 +87,6 @@ def __init__(self, self.bt = BT(intervals_file) - - # Fasta self.fasta_file = fasta_file self.fasta_extractor = None # initialize later @@ -97,7 +95,7 @@ def __init__(self, self.dnase_extractor = None # mappability if mappability_file is None: - # download the mappability file if not existing + # download the mappability file if not existing common_dl_dir = os.path.join(this_dir, "../../template/downloaded/dataloader_files") makedir_exist_ok(common_dl_dir) rf = RemoteFile(url="http://hgdownload.cse.ucsc.edu/goldenPath/hg19/encodeDCC/wgEncodeMapability/wgEncodeDukeMapabilityUniqueness35bp.bigWig", diff --git a/FactorNet/E2F1/GENCODE_Unique35_DGF/dataloader.yaml b/FactorNet/E2F1/GENCODE_Unique35_DGF/dataloader.yaml index 23fb41cd3..ca9c01c47 100644 --- a/FactorNet/E2F1/GENCODE_Unique35_DGF/dataloader.yaml +++ b/FactorNet/E2F1/GENCODE_Unique35_DGF/dataloader.yaml @@ -23,7 +23,7 @@ args: mappability_file: doc: USCS mappability track - http://hgdownload.cse.ucsc.edu/goldenpath/hg19/encodeDCC/wgEncodeMapability/wgEncodeDukeMapabilityUniqueness35bp.bigWig. by deafult, provide this file with the dataloader, download in background example: - url: https://github.com/kipoi/models/blob/92b23ed62956d56eb457488b431fc27550ddbf32/FactorNet/template/dataloader_files/wgEncodeDukeMapabilityUniqueness35bp.chr22.bigWig?raw=true + url: https://s3.eu-central-1.amazonaws.com/kipoi-models/dataloader_files/FactorNet/dataloader_files/wgEncodeDukeMapabilityUniqueness35bp.chr22.bigWig md5: 912e1bb77fb67c1a792ac4a05d9416f9 optional: True GENCODE_dir: diff --git a/FactorNet/E2F1/onePeak_Unique35_DGF/dataloader.py b/FactorNet/E2F1/onePeak_Unique35_DGF/dataloader.py index d36ad58ef..33a036afd 100644 --- a/FactorNet/E2F1/onePeak_Unique35_DGF/dataloader.py +++ b/FactorNet/E2F1/onePeak_Unique35_DGF/dataloader.py @@ -29,9 +29,9 @@ def download_gencode_dir(output_dir): """Download all the required gencode files """ makedir_exist_ok(output_dir) - url_template = ("https://github.com/kipoi/models/blob/" - "7648d3fd57def50934835b52acadd26bcaaa275c/FactorNet/" - "template/dataloader_files/gencode_features/{}?raw=true") + + url_template = ("https://s3.eu-central-1.amazonaws.com/kipoi-models/" + "dataloader_files/FactorNet/dataloader_files/gencode_features/{}") # url_template = "https://github.com/uci-cbcl/FactorNet/blob/master/resources/{}?raw=true" fnames = [('cpgisland.bed.gz', 'ac7dc007d7019c05adb7a331d1d6721d'), @@ -85,8 +85,6 @@ def __init__(self, self.bt = BT(intervals_file) - - # Fasta self.fasta_file = fasta_file self.fasta_extractor = None # initialize later @@ -95,7 +93,7 @@ def __init__(self, self.dnase_extractor = None # mappability if mappability_file is None: - # download the mappability file if not existing + # download the mappability file if not existing common_dl_dir = os.path.join(this_dir, "../../template/downloaded/dataloader_files") makedir_exist_ok(common_dl_dir) rf = RemoteFile(url="http://hgdownload.cse.ucsc.edu/goldenPath/hg19/encodeDCC/wgEncodeMapability/wgEncodeDukeMapabilityUniqueness35bp.bigWig", diff --git a/FactorNet/E2F1/onePeak_Unique35_DGF/dataloader.yaml b/FactorNet/E2F1/onePeak_Unique35_DGF/dataloader.yaml index 97a5f127b..f5fd23fff 100644 --- a/FactorNet/E2F1/onePeak_Unique35_DGF/dataloader.yaml +++ b/FactorNet/E2F1/onePeak_Unique35_DGF/dataloader.yaml @@ -19,7 +19,7 @@ args: mappability_file: doc: USCS mappability track - http://hgdownload.cse.ucsc.edu/goldenpath/hg19/encodeDCC/wgEncodeMapability/wgEncodeDukeMapabilityUniqueness35bp.bigWig. by deafult, provide this file with the dataloader, download in background example: - url: https://github.com/kipoi/models/blob/92b23ed62956d56eb457488b431fc27550ddbf32/FactorNet/template/dataloader_files/wgEncodeDukeMapabilityUniqueness35bp.chr22.bigWig?raw=true + url: https://s3.eu-central-1.amazonaws.com/kipoi-models/dataloader_files/FactorNet/dataloader_files/wgEncodeDukeMapabilityUniqueness35bp.chr22.bigWig md5: 912e1bb77fb67c1a792ac4a05d9416f9 optional: True use_linecache: diff --git a/FactorNet/EGR1/meta_RNAseq_Unique35_DGF/dataloader.py b/FactorNet/EGR1/meta_RNAseq_Unique35_DGF/dataloader.py index fe485d008..1620ead6a 100644 --- a/FactorNet/EGR1/meta_RNAseq_Unique35_DGF/dataloader.py +++ b/FactorNet/EGR1/meta_RNAseq_Unique35_DGF/dataloader.py @@ -29,9 +29,9 @@ def download_gencode_dir(output_dir): """Download all the required gencode files """ makedir_exist_ok(output_dir) - url_template = ("https://github.com/kipoi/models/blob/" - "7648d3fd57def50934835b52acadd26bcaaa275c/FactorNet/" - "template/dataloader_files/gencode_features/{}?raw=true") + + url_template = ("https://s3.eu-central-1.amazonaws.com/kipoi-models/" + "dataloader_files/FactorNet/dataloader_files/gencode_features/{}") # url_template = "https://github.com/uci-cbcl/FactorNet/blob/master/resources/{}?raw=true" fnames = [('cpgisland.bed.gz', 'ac7dc007d7019c05adb7a331d1d6721d'), @@ -87,8 +87,6 @@ def __init__(self, self.bt = BT(intervals_file) - - # Fasta self.fasta_file = fasta_file self.fasta_extractor = None # initialize later @@ -97,7 +95,7 @@ def __init__(self, self.dnase_extractor = None # mappability if mappability_file is None: - # download the mappability file if not existing + # download the mappability file if not existing common_dl_dir = os.path.join(this_dir, "../../template/downloaded/dataloader_files") makedir_exist_ok(common_dl_dir) rf = RemoteFile(url="http://hgdownload.cse.ucsc.edu/goldenPath/hg19/encodeDCC/wgEncodeMapability/wgEncodeDukeMapabilityUniqueness35bp.bigWig", @@ -118,10 +116,10 @@ def __init__(self, output_dir = os.path.join(this_dir, "../../template/downloaded/dataloader_files/RNAseq_features/") makedir_exist_ok(output_dir) RNAseq_PC_file = os.path.join(output_dir, cell_line, "meta.txt") - url_template = ('https://github.com/kipoi/models/blob/7648d3fd57def50934835b52acadd26bcaaa275c' - '/FactorNet/template/dataloader_files/RNAseq_features/{}/meta.txt?raw=true') + url_template = ('https://s3.eu-central-1.amazonaws.com/kipoi-models/dataloader_files/' + 'FactorNet/dataloader_files/RNAseq_features/{}/meta.txt') # rf = RemoteFile(url=url_template.format(cell_line)) - if not os.path.exists(RNAseq_PC_file): # or not rf.validate(mappability_file): + if not os.path.exists(RNAseq_PC_file): # or not rf.validate(mappability_file): # download the path download_url(url_template.format(cell_line), os.path.join(output_dir, cell_line), "meta.txt") # rf.get_file(RNAseq_PC_file) diff --git a/FactorNet/EGR1/meta_RNAseq_Unique35_DGF/dataloader.yaml b/FactorNet/EGR1/meta_RNAseq_Unique35_DGF/dataloader.yaml index 5f16f5a9b..bd2b01c55 100644 --- a/FactorNet/EGR1/meta_RNAseq_Unique35_DGF/dataloader.yaml +++ b/FactorNet/EGR1/meta_RNAseq_Unique35_DGF/dataloader.yaml @@ -26,7 +26,7 @@ args: mappability_file: doc: USCS mappability track - http://hgdownload.cse.ucsc.edu/goldenpath/hg19/encodeDCC/wgEncodeMapability/wgEncodeDukeMapabilityUniqueness35bp.bigWig. by deafult, provide this file with the dataloader, download in background example: - url: https://github.com/kipoi/models/blob/92b23ed62956d56eb457488b431fc27550ddbf32/FactorNet/template/dataloader_files/wgEncodeDukeMapabilityUniqueness35bp.chr22.bigWig?raw=true + url: https://s3.eu-central-1.amazonaws.com/kipoi-models/dataloader_files/FactorNet/dataloader_files/wgEncodeDukeMapabilityUniqueness35bp.chr22.bigWig md5: 912e1bb77fb67c1a792ac4a05d9416f9 optional: True use_linecache: diff --git a/FactorNet/EGR1/onePeak_DGF/dataloader.py b/FactorNet/EGR1/onePeak_DGF/dataloader.py index 5e737679a..c34d3df22 100644 --- a/FactorNet/EGR1/onePeak_DGF/dataloader.py +++ b/FactorNet/EGR1/onePeak_DGF/dataloader.py @@ -29,9 +29,9 @@ def download_gencode_dir(output_dir): """Download all the required gencode files """ makedir_exist_ok(output_dir) - url_template = ("https://github.com/kipoi/models/blob/" - "7648d3fd57def50934835b52acadd26bcaaa275c/FactorNet/" - "template/dataloader_files/gencode_features/{}?raw=true") + + url_template = ("https://s3.eu-central-1.amazonaws.com/kipoi-models/" + "dataloader_files/FactorNet/dataloader_files/gencode_features/{}") # url_template = "https://github.com/uci-cbcl/FactorNet/blob/master/resources/{}?raw=true" fnames = [('cpgisland.bed.gz', 'ac7dc007d7019c05adb7a331d1d6721d'), @@ -84,8 +84,6 @@ def __init__(self, self.bt = BT(intervals_file) - - # Fasta self.fasta_file = fasta_file self.fasta_extractor = None # initialize later diff --git a/FactorNet/FOXA1/multiTask_DGF/dataloader.py b/FactorNet/FOXA1/multiTask_DGF/dataloader.py index 5e737679a..c34d3df22 100644 --- a/FactorNet/FOXA1/multiTask_DGF/dataloader.py +++ b/FactorNet/FOXA1/multiTask_DGF/dataloader.py @@ -29,9 +29,9 @@ def download_gencode_dir(output_dir): """Download all the required gencode files """ makedir_exist_ok(output_dir) - url_template = ("https://github.com/kipoi/models/blob/" - "7648d3fd57def50934835b52acadd26bcaaa275c/FactorNet/" - "template/dataloader_files/gencode_features/{}?raw=true") + + url_template = ("https://s3.eu-central-1.amazonaws.com/kipoi-models/" + "dataloader_files/FactorNet/dataloader_files/gencode_features/{}") # url_template = "https://github.com/uci-cbcl/FactorNet/blob/master/resources/{}?raw=true" fnames = [('cpgisland.bed.gz', 'ac7dc007d7019c05adb7a331d1d6721d'), @@ -84,8 +84,6 @@ def __init__(self, self.bt = BT(intervals_file) - - # Fasta self.fasta_file = fasta_file self.fasta_extractor = None # initialize later diff --git a/FactorNet/FOXA1/onePeak_DGF/dataloader.py b/FactorNet/FOXA1/onePeak_DGF/dataloader.py index 5e737679a..c34d3df22 100644 --- a/FactorNet/FOXA1/onePeak_DGF/dataloader.py +++ b/FactorNet/FOXA1/onePeak_DGF/dataloader.py @@ -29,9 +29,9 @@ def download_gencode_dir(output_dir): """Download all the required gencode files """ makedir_exist_ok(output_dir) - url_template = ("https://github.com/kipoi/models/blob/" - "7648d3fd57def50934835b52acadd26bcaaa275c/FactorNet/" - "template/dataloader_files/gencode_features/{}?raw=true") + + url_template = ("https://s3.eu-central-1.amazonaws.com/kipoi-models/" + "dataloader_files/FactorNet/dataloader_files/gencode_features/{}") # url_template = "https://github.com/uci-cbcl/FactorNet/blob/master/resources/{}?raw=true" fnames = [('cpgisland.bed.gz', 'ac7dc007d7019c05adb7a331d1d6721d'), @@ -84,8 +84,6 @@ def __init__(self, self.bt = BT(intervals_file) - - # Fasta self.fasta_file = fasta_file self.fasta_extractor = None # initialize later diff --git a/FactorNet/FOXA2/multiTask_DGF/dataloader.py b/FactorNet/FOXA2/multiTask_DGF/dataloader.py index 5e737679a..c34d3df22 100644 --- a/FactorNet/FOXA2/multiTask_DGF/dataloader.py +++ b/FactorNet/FOXA2/multiTask_DGF/dataloader.py @@ -29,9 +29,9 @@ def download_gencode_dir(output_dir): """Download all the required gencode files """ makedir_exist_ok(output_dir) - url_template = ("https://github.com/kipoi/models/blob/" - "7648d3fd57def50934835b52acadd26bcaaa275c/FactorNet/" - "template/dataloader_files/gencode_features/{}?raw=true") + + url_template = ("https://s3.eu-central-1.amazonaws.com/kipoi-models/" + "dataloader_files/FactorNet/dataloader_files/gencode_features/{}") # url_template = "https://github.com/uci-cbcl/FactorNet/blob/master/resources/{}?raw=true" fnames = [('cpgisland.bed.gz', 'ac7dc007d7019c05adb7a331d1d6721d'), @@ -84,8 +84,6 @@ def __init__(self, self.bt = BT(intervals_file) - - # Fasta self.fasta_file = fasta_file self.fasta_extractor = None # initialize later diff --git a/FactorNet/FOXA2/onePeak_Unique35_DGF/dataloader.py b/FactorNet/FOXA2/onePeak_Unique35_DGF/dataloader.py index d36ad58ef..33a036afd 100644 --- a/FactorNet/FOXA2/onePeak_Unique35_DGF/dataloader.py +++ b/FactorNet/FOXA2/onePeak_Unique35_DGF/dataloader.py @@ -29,9 +29,9 @@ def download_gencode_dir(output_dir): """Download all the required gencode files """ makedir_exist_ok(output_dir) - url_template = ("https://github.com/kipoi/models/blob/" - "7648d3fd57def50934835b52acadd26bcaaa275c/FactorNet/" - "template/dataloader_files/gencode_features/{}?raw=true") + + url_template = ("https://s3.eu-central-1.amazonaws.com/kipoi-models/" + "dataloader_files/FactorNet/dataloader_files/gencode_features/{}") # url_template = "https://github.com/uci-cbcl/FactorNet/blob/master/resources/{}?raw=true" fnames = [('cpgisland.bed.gz', 'ac7dc007d7019c05adb7a331d1d6721d'), @@ -85,8 +85,6 @@ def __init__(self, self.bt = BT(intervals_file) - - # Fasta self.fasta_file = fasta_file self.fasta_extractor = None # initialize later @@ -95,7 +93,7 @@ def __init__(self, self.dnase_extractor = None # mappability if mappability_file is None: - # download the mappability file if not existing + # download the mappability file if not existing common_dl_dir = os.path.join(this_dir, "../../template/downloaded/dataloader_files") makedir_exist_ok(common_dl_dir) rf = RemoteFile(url="http://hgdownload.cse.ucsc.edu/goldenPath/hg19/encodeDCC/wgEncodeMapability/wgEncodeDukeMapabilityUniqueness35bp.bigWig", diff --git a/FactorNet/FOXA2/onePeak_Unique35_DGF/dataloader.yaml b/FactorNet/FOXA2/onePeak_Unique35_DGF/dataloader.yaml index 97a5f127b..f5fd23fff 100644 --- a/FactorNet/FOXA2/onePeak_Unique35_DGF/dataloader.yaml +++ b/FactorNet/FOXA2/onePeak_Unique35_DGF/dataloader.yaml @@ -19,7 +19,7 @@ args: mappability_file: doc: USCS mappability track - http://hgdownload.cse.ucsc.edu/goldenpath/hg19/encodeDCC/wgEncodeMapability/wgEncodeDukeMapabilityUniqueness35bp.bigWig. by deafult, provide this file with the dataloader, download in background example: - url: https://github.com/kipoi/models/blob/92b23ed62956d56eb457488b431fc27550ddbf32/FactorNet/template/dataloader_files/wgEncodeDukeMapabilityUniqueness35bp.chr22.bigWig?raw=true + url: https://s3.eu-central-1.amazonaws.com/kipoi-models/dataloader_files/FactorNet/dataloader_files/wgEncodeDukeMapabilityUniqueness35bp.chr22.bigWig md5: 912e1bb77fb67c1a792ac4a05d9416f9 optional: True use_linecache: diff --git a/FactorNet/GABPA/metaGENCODE_RNAseq_Unique35_DGF/dataloader.py b/FactorNet/GABPA/metaGENCODE_RNAseq_Unique35_DGF/dataloader.py index a82d2add4..463d66275 100644 --- a/FactorNet/GABPA/metaGENCODE_RNAseq_Unique35_DGF/dataloader.py +++ b/FactorNet/GABPA/metaGENCODE_RNAseq_Unique35_DGF/dataloader.py @@ -29,9 +29,9 @@ def download_gencode_dir(output_dir): """Download all the required gencode files """ makedir_exist_ok(output_dir) - url_template = ("https://github.com/kipoi/models/blob/" - "7648d3fd57def50934835b52acadd26bcaaa275c/FactorNet/" - "template/dataloader_files/gencode_features/{}?raw=true") + + url_template = ("https://s3.eu-central-1.amazonaws.com/kipoi-models/" + "dataloader_files/FactorNet/dataloader_files/gencode_features/{}") # url_template = "https://github.com/uci-cbcl/FactorNet/blob/master/resources/{}?raw=true" fnames = [('cpgisland.bed.gz', 'ac7dc007d7019c05adb7a331d1d6721d'), @@ -88,8 +88,6 @@ def __init__(self, self.bt = BT(intervals_file) - - # Fasta self.fasta_file = fasta_file self.fasta_extractor = None # initialize later @@ -98,7 +96,7 @@ def __init__(self, self.dnase_extractor = None # mappability if mappability_file is None: - # download the mappability file if not existing + # download the mappability file if not existing common_dl_dir = os.path.join(this_dir, "../../template/downloaded/dataloader_files") makedir_exist_ok(common_dl_dir) rf = RemoteFile(url="http://hgdownload.cse.ucsc.edu/goldenPath/hg19/encodeDCC/wgEncodeMapability/wgEncodeDukeMapabilityUniqueness35bp.bigWig", @@ -141,10 +139,10 @@ def __init__(self, output_dir = os.path.join(this_dir, "../../template/downloaded/dataloader_files/RNAseq_features/") makedir_exist_ok(output_dir) RNAseq_PC_file = os.path.join(output_dir, cell_line, "meta.txt") - url_template = ('https://github.com/kipoi/models/blob/7648d3fd57def50934835b52acadd26bcaaa275c' - '/FactorNet/template/dataloader_files/RNAseq_features/{}/meta.txt?raw=true') + url_template = ('https://s3.eu-central-1.amazonaws.com/kipoi-models/dataloader_files/' + 'FactorNet/dataloader_files/RNAseq_features/{}/meta.txt') # rf = RemoteFile(url=url_template.format(cell_line)) - if not os.path.exists(RNAseq_PC_file): # or not rf.validate(mappability_file): + if not os.path.exists(RNAseq_PC_file): # or not rf.validate(mappability_file): # download the path download_url(url_template.format(cell_line), os.path.join(output_dir, cell_line), "meta.txt") # rf.get_file(RNAseq_PC_file) diff --git a/FactorNet/GABPA/metaGENCODE_RNAseq_Unique35_DGF/dataloader.yaml b/FactorNet/GABPA/metaGENCODE_RNAseq_Unique35_DGF/dataloader.yaml index 56715a39e..bb2be34cc 100644 --- a/FactorNet/GABPA/metaGENCODE_RNAseq_Unique35_DGF/dataloader.yaml +++ b/FactorNet/GABPA/metaGENCODE_RNAseq_Unique35_DGF/dataloader.yaml @@ -26,7 +26,7 @@ args: mappability_file: doc: USCS mappability track - http://hgdownload.cse.ucsc.edu/goldenpath/hg19/encodeDCC/wgEncodeMapability/wgEncodeDukeMapabilityUniqueness35bp.bigWig. by deafult, provide this file with the dataloader, download in background example: - url: https://github.com/kipoi/models/blob/92b23ed62956d56eb457488b431fc27550ddbf32/FactorNet/template/dataloader_files/wgEncodeDukeMapabilityUniqueness35bp.chr22.bigWig?raw=true + url: https://s3.eu-central-1.amazonaws.com/kipoi-models/dataloader_files/FactorNet/dataloader_files/wgEncodeDukeMapabilityUniqueness35bp.chr22.bigWig md5: 912e1bb77fb67c1a792ac4a05d9416f9 optional: True GENCODE_dir: diff --git a/FactorNet/GABPA/meta_RNAseq_Unique35_DGF/dataloader.py b/FactorNet/GABPA/meta_RNAseq_Unique35_DGF/dataloader.py index fe485d008..1620ead6a 100644 --- a/FactorNet/GABPA/meta_RNAseq_Unique35_DGF/dataloader.py +++ b/FactorNet/GABPA/meta_RNAseq_Unique35_DGF/dataloader.py @@ -29,9 +29,9 @@ def download_gencode_dir(output_dir): """Download all the required gencode files """ makedir_exist_ok(output_dir) - url_template = ("https://github.com/kipoi/models/blob/" - "7648d3fd57def50934835b52acadd26bcaaa275c/FactorNet/" - "template/dataloader_files/gencode_features/{}?raw=true") + + url_template = ("https://s3.eu-central-1.amazonaws.com/kipoi-models/" + "dataloader_files/FactorNet/dataloader_files/gencode_features/{}") # url_template = "https://github.com/uci-cbcl/FactorNet/blob/master/resources/{}?raw=true" fnames = [('cpgisland.bed.gz', 'ac7dc007d7019c05adb7a331d1d6721d'), @@ -87,8 +87,6 @@ def __init__(self, self.bt = BT(intervals_file) - - # Fasta self.fasta_file = fasta_file self.fasta_extractor = None # initialize later @@ -97,7 +95,7 @@ def __init__(self, self.dnase_extractor = None # mappability if mappability_file is None: - # download the mappability file if not existing + # download the mappability file if not existing common_dl_dir = os.path.join(this_dir, "../../template/downloaded/dataloader_files") makedir_exist_ok(common_dl_dir) rf = RemoteFile(url="http://hgdownload.cse.ucsc.edu/goldenPath/hg19/encodeDCC/wgEncodeMapability/wgEncodeDukeMapabilityUniqueness35bp.bigWig", @@ -118,10 +116,10 @@ def __init__(self, output_dir = os.path.join(this_dir, "../../template/downloaded/dataloader_files/RNAseq_features/") makedir_exist_ok(output_dir) RNAseq_PC_file = os.path.join(output_dir, cell_line, "meta.txt") - url_template = ('https://github.com/kipoi/models/blob/7648d3fd57def50934835b52acadd26bcaaa275c' - '/FactorNet/template/dataloader_files/RNAseq_features/{}/meta.txt?raw=true') + url_template = ('https://s3.eu-central-1.amazonaws.com/kipoi-models/dataloader_files/' + 'FactorNet/dataloader_files/RNAseq_features/{}/meta.txt') # rf = RemoteFile(url=url_template.format(cell_line)) - if not os.path.exists(RNAseq_PC_file): # or not rf.validate(mappability_file): + if not os.path.exists(RNAseq_PC_file): # or not rf.validate(mappability_file): # download the path download_url(url_template.format(cell_line), os.path.join(output_dir, cell_line), "meta.txt") # rf.get_file(RNAseq_PC_file) diff --git a/FactorNet/GABPA/meta_RNAseq_Unique35_DGF/dataloader.yaml b/FactorNet/GABPA/meta_RNAseq_Unique35_DGF/dataloader.yaml index 5f16f5a9b..bd2b01c55 100644 --- a/FactorNet/GABPA/meta_RNAseq_Unique35_DGF/dataloader.yaml +++ b/FactorNet/GABPA/meta_RNAseq_Unique35_DGF/dataloader.yaml @@ -26,7 +26,7 @@ args: mappability_file: doc: USCS mappability track - http://hgdownload.cse.ucsc.edu/goldenpath/hg19/encodeDCC/wgEncodeMapability/wgEncodeDukeMapabilityUniqueness35bp.bigWig. by deafult, provide this file with the dataloader, download in background example: - url: https://github.com/kipoi/models/blob/92b23ed62956d56eb457488b431fc27550ddbf32/FactorNet/template/dataloader_files/wgEncodeDukeMapabilityUniqueness35bp.chr22.bigWig?raw=true + url: https://s3.eu-central-1.amazonaws.com/kipoi-models/dataloader_files/FactorNet/dataloader_files/wgEncodeDukeMapabilityUniqueness35bp.chr22.bigWig md5: 912e1bb77fb67c1a792ac4a05d9416f9 optional: True use_linecache: diff --git a/FactorNet/HNF4A/multiTask_DGF/dataloader.py b/FactorNet/HNF4A/multiTask_DGF/dataloader.py index 5e737679a..c34d3df22 100644 --- a/FactorNet/HNF4A/multiTask_DGF/dataloader.py +++ b/FactorNet/HNF4A/multiTask_DGF/dataloader.py @@ -29,9 +29,9 @@ def download_gencode_dir(output_dir): """Download all the required gencode files """ makedir_exist_ok(output_dir) - url_template = ("https://github.com/kipoi/models/blob/" - "7648d3fd57def50934835b52acadd26bcaaa275c/FactorNet/" - "template/dataloader_files/gencode_features/{}?raw=true") + + url_template = ("https://s3.eu-central-1.amazonaws.com/kipoi-models/" + "dataloader_files/FactorNet/dataloader_files/gencode_features/{}") # url_template = "https://github.com/uci-cbcl/FactorNet/blob/master/resources/{}?raw=true" fnames = [('cpgisland.bed.gz', 'ac7dc007d7019c05adb7a331d1d6721d'), @@ -84,8 +84,6 @@ def __init__(self, self.bt = BT(intervals_file) - - # Fasta self.fasta_file = fasta_file self.fasta_extractor = None # initialize later diff --git a/FactorNet/HNF4A/onePeak_Unique35_DGF/dataloader.py b/FactorNet/HNF4A/onePeak_Unique35_DGF/dataloader.py index d36ad58ef..33a036afd 100644 --- a/FactorNet/HNF4A/onePeak_Unique35_DGF/dataloader.py +++ b/FactorNet/HNF4A/onePeak_Unique35_DGF/dataloader.py @@ -29,9 +29,9 @@ def download_gencode_dir(output_dir): """Download all the required gencode files """ makedir_exist_ok(output_dir) - url_template = ("https://github.com/kipoi/models/blob/" - "7648d3fd57def50934835b52acadd26bcaaa275c/FactorNet/" - "template/dataloader_files/gencode_features/{}?raw=true") + + url_template = ("https://s3.eu-central-1.amazonaws.com/kipoi-models/" + "dataloader_files/FactorNet/dataloader_files/gencode_features/{}") # url_template = "https://github.com/uci-cbcl/FactorNet/blob/master/resources/{}?raw=true" fnames = [('cpgisland.bed.gz', 'ac7dc007d7019c05adb7a331d1d6721d'), @@ -85,8 +85,6 @@ def __init__(self, self.bt = BT(intervals_file) - - # Fasta self.fasta_file = fasta_file self.fasta_extractor = None # initialize later @@ -95,7 +93,7 @@ def __init__(self, self.dnase_extractor = None # mappability if mappability_file is None: - # download the mappability file if not existing + # download the mappability file if not existing common_dl_dir = os.path.join(this_dir, "../../template/downloaded/dataloader_files") makedir_exist_ok(common_dl_dir) rf = RemoteFile(url="http://hgdownload.cse.ucsc.edu/goldenPath/hg19/encodeDCC/wgEncodeMapability/wgEncodeDukeMapabilityUniqueness35bp.bigWig", diff --git a/FactorNet/HNF4A/onePeak_Unique35_DGF/dataloader.yaml b/FactorNet/HNF4A/onePeak_Unique35_DGF/dataloader.yaml index 97a5f127b..f5fd23fff 100644 --- a/FactorNet/HNF4A/onePeak_Unique35_DGF/dataloader.yaml +++ b/FactorNet/HNF4A/onePeak_Unique35_DGF/dataloader.yaml @@ -19,7 +19,7 @@ args: mappability_file: doc: USCS mappability track - http://hgdownload.cse.ucsc.edu/goldenpath/hg19/encodeDCC/wgEncodeMapability/wgEncodeDukeMapabilityUniqueness35bp.bigWig. by deafult, provide this file with the dataloader, download in background example: - url: https://github.com/kipoi/models/blob/92b23ed62956d56eb457488b431fc27550ddbf32/FactorNet/template/dataloader_files/wgEncodeDukeMapabilityUniqueness35bp.chr22.bigWig?raw=true + url: https://s3.eu-central-1.amazonaws.com/kipoi-models/dataloader_files/FactorNet/dataloader_files/wgEncodeDukeMapabilityUniqueness35bp.chr22.bigWig md5: 912e1bb77fb67c1a792ac4a05d9416f9 optional: True use_linecache: diff --git a/FactorNet/JUND/meta_Unique35_DGF/dataloader.py b/FactorNet/JUND/meta_Unique35_DGF/dataloader.py index fe485d008..1620ead6a 100644 --- a/FactorNet/JUND/meta_Unique35_DGF/dataloader.py +++ b/FactorNet/JUND/meta_Unique35_DGF/dataloader.py @@ -29,9 +29,9 @@ def download_gencode_dir(output_dir): """Download all the required gencode files """ makedir_exist_ok(output_dir) - url_template = ("https://github.com/kipoi/models/blob/" - "7648d3fd57def50934835b52acadd26bcaaa275c/FactorNet/" - "template/dataloader_files/gencode_features/{}?raw=true") + + url_template = ("https://s3.eu-central-1.amazonaws.com/kipoi-models/" + "dataloader_files/FactorNet/dataloader_files/gencode_features/{}") # url_template = "https://github.com/uci-cbcl/FactorNet/blob/master/resources/{}?raw=true" fnames = [('cpgisland.bed.gz', 'ac7dc007d7019c05adb7a331d1d6721d'), @@ -87,8 +87,6 @@ def __init__(self, self.bt = BT(intervals_file) - - # Fasta self.fasta_file = fasta_file self.fasta_extractor = None # initialize later @@ -97,7 +95,7 @@ def __init__(self, self.dnase_extractor = None # mappability if mappability_file is None: - # download the mappability file if not existing + # download the mappability file if not existing common_dl_dir = os.path.join(this_dir, "../../template/downloaded/dataloader_files") makedir_exist_ok(common_dl_dir) rf = RemoteFile(url="http://hgdownload.cse.ucsc.edu/goldenPath/hg19/encodeDCC/wgEncodeMapability/wgEncodeDukeMapabilityUniqueness35bp.bigWig", @@ -118,10 +116,10 @@ def __init__(self, output_dir = os.path.join(this_dir, "../../template/downloaded/dataloader_files/RNAseq_features/") makedir_exist_ok(output_dir) RNAseq_PC_file = os.path.join(output_dir, cell_line, "meta.txt") - url_template = ('https://github.com/kipoi/models/blob/7648d3fd57def50934835b52acadd26bcaaa275c' - '/FactorNet/template/dataloader_files/RNAseq_features/{}/meta.txt?raw=true') + url_template = ('https://s3.eu-central-1.amazonaws.com/kipoi-models/dataloader_files/' + 'FactorNet/dataloader_files/RNAseq_features/{}/meta.txt') # rf = RemoteFile(url=url_template.format(cell_line)) - if not os.path.exists(RNAseq_PC_file): # or not rf.validate(mappability_file): + if not os.path.exists(RNAseq_PC_file): # or not rf.validate(mappability_file): # download the path download_url(url_template.format(cell_line), os.path.join(output_dir, cell_line), "meta.txt") # rf.get_file(RNAseq_PC_file) diff --git a/FactorNet/JUND/meta_Unique35_DGF/dataloader.yaml b/FactorNet/JUND/meta_Unique35_DGF/dataloader.yaml index 5f16f5a9b..bd2b01c55 100644 --- a/FactorNet/JUND/meta_Unique35_DGF/dataloader.yaml +++ b/FactorNet/JUND/meta_Unique35_DGF/dataloader.yaml @@ -26,7 +26,7 @@ args: mappability_file: doc: USCS mappability track - http://hgdownload.cse.ucsc.edu/goldenpath/hg19/encodeDCC/wgEncodeMapability/wgEncodeDukeMapabilityUniqueness35bp.bigWig. by deafult, provide this file with the dataloader, download in background example: - url: https://github.com/kipoi/models/blob/92b23ed62956d56eb457488b431fc27550ddbf32/FactorNet/template/dataloader_files/wgEncodeDukeMapabilityUniqueness35bp.chr22.bigWig?raw=true + url: https://s3.eu-central-1.amazonaws.com/kipoi-models/dataloader_files/FactorNet/dataloader_files/wgEncodeDukeMapabilityUniqueness35bp.chr22.bigWig md5: 912e1bb77fb67c1a792ac4a05d9416f9 optional: True use_linecache: diff --git a/FactorNet/JUND/meta_Unique35_DGF_2/dataloader.py b/FactorNet/JUND/meta_Unique35_DGF_2/dataloader.py index fe485d008..1620ead6a 100644 --- a/FactorNet/JUND/meta_Unique35_DGF_2/dataloader.py +++ b/FactorNet/JUND/meta_Unique35_DGF_2/dataloader.py @@ -29,9 +29,9 @@ def download_gencode_dir(output_dir): """Download all the required gencode files """ makedir_exist_ok(output_dir) - url_template = ("https://github.com/kipoi/models/blob/" - "7648d3fd57def50934835b52acadd26bcaaa275c/FactorNet/" - "template/dataloader_files/gencode_features/{}?raw=true") + + url_template = ("https://s3.eu-central-1.amazonaws.com/kipoi-models/" + "dataloader_files/FactorNet/dataloader_files/gencode_features/{}") # url_template = "https://github.com/uci-cbcl/FactorNet/blob/master/resources/{}?raw=true" fnames = [('cpgisland.bed.gz', 'ac7dc007d7019c05adb7a331d1d6721d'), @@ -87,8 +87,6 @@ def __init__(self, self.bt = BT(intervals_file) - - # Fasta self.fasta_file = fasta_file self.fasta_extractor = None # initialize later @@ -97,7 +95,7 @@ def __init__(self, self.dnase_extractor = None # mappability if mappability_file is None: - # download the mappability file if not existing + # download the mappability file if not existing common_dl_dir = os.path.join(this_dir, "../../template/downloaded/dataloader_files") makedir_exist_ok(common_dl_dir) rf = RemoteFile(url="http://hgdownload.cse.ucsc.edu/goldenPath/hg19/encodeDCC/wgEncodeMapability/wgEncodeDukeMapabilityUniqueness35bp.bigWig", @@ -118,10 +116,10 @@ def __init__(self, output_dir = os.path.join(this_dir, "../../template/downloaded/dataloader_files/RNAseq_features/") makedir_exist_ok(output_dir) RNAseq_PC_file = os.path.join(output_dir, cell_line, "meta.txt") - url_template = ('https://github.com/kipoi/models/blob/7648d3fd57def50934835b52acadd26bcaaa275c' - '/FactorNet/template/dataloader_files/RNAseq_features/{}/meta.txt?raw=true') + url_template = ('https://s3.eu-central-1.amazonaws.com/kipoi-models/dataloader_files/' + 'FactorNet/dataloader_files/RNAseq_features/{}/meta.txt') # rf = RemoteFile(url=url_template.format(cell_line)) - if not os.path.exists(RNAseq_PC_file): # or not rf.validate(mappability_file): + if not os.path.exists(RNAseq_PC_file): # or not rf.validate(mappability_file): # download the path download_url(url_template.format(cell_line), os.path.join(output_dir, cell_line), "meta.txt") # rf.get_file(RNAseq_PC_file) diff --git a/FactorNet/JUND/meta_Unique35_DGF_2/dataloader.yaml b/FactorNet/JUND/meta_Unique35_DGF_2/dataloader.yaml index 5f16f5a9b..bd2b01c55 100644 --- a/FactorNet/JUND/meta_Unique35_DGF_2/dataloader.yaml +++ b/FactorNet/JUND/meta_Unique35_DGF_2/dataloader.yaml @@ -26,7 +26,7 @@ args: mappability_file: doc: USCS mappability track - http://hgdownload.cse.ucsc.edu/goldenpath/hg19/encodeDCC/wgEncodeMapability/wgEncodeDukeMapabilityUniqueness35bp.bigWig. by deafult, provide this file with the dataloader, download in background example: - url: https://github.com/kipoi/models/blob/92b23ed62956d56eb457488b431fc27550ddbf32/FactorNet/template/dataloader_files/wgEncodeDukeMapabilityUniqueness35bp.chr22.bigWig?raw=true + url: https://s3.eu-central-1.amazonaws.com/kipoi-models/dataloader_files/FactorNet/dataloader_files/wgEncodeDukeMapabilityUniqueness35bp.chr22.bigWig md5: 912e1bb77fb67c1a792ac4a05d9416f9 optional: True use_linecache: diff --git a/FactorNet/MAFK/meta_1_Unique35_DGF/dataloader.py b/FactorNet/MAFK/meta_1_Unique35_DGF/dataloader.py index fe485d008..74bf0ddb8 100644 --- a/FactorNet/MAFK/meta_1_Unique35_DGF/dataloader.py +++ b/FactorNet/MAFK/meta_1_Unique35_DGF/dataloader.py @@ -29,9 +29,8 @@ def download_gencode_dir(output_dir): """Download all the required gencode files """ makedir_exist_ok(output_dir) - url_template = ("https://github.com/kipoi/models/blob/" - "7648d3fd57def50934835b52acadd26bcaaa275c/FactorNet/" - "template/dataloader_files/gencode_features/{}?raw=true") + url_template = ("https://s3.eu-central-1.amazonaws.com/kipoi-models/" + "dataloader_files/FactorNet/dataloader_files/gencode_features/{}") # url_template = "https://github.com/uci-cbcl/FactorNet/blob/master/resources/{}?raw=true" fnames = [('cpgisland.bed.gz', 'ac7dc007d7019c05adb7a331d1d6721d'), @@ -87,8 +86,6 @@ def __init__(self, self.bt = BT(intervals_file) - - # Fasta self.fasta_file = fasta_file self.fasta_extractor = None # initialize later @@ -97,7 +94,7 @@ def __init__(self, self.dnase_extractor = None # mappability if mappability_file is None: - # download the mappability file if not existing + # download the mappability file if not existing common_dl_dir = os.path.join(this_dir, "../../template/downloaded/dataloader_files") makedir_exist_ok(common_dl_dir) rf = RemoteFile(url="http://hgdownload.cse.ucsc.edu/goldenPath/hg19/encodeDCC/wgEncodeMapability/wgEncodeDukeMapabilityUniqueness35bp.bigWig", @@ -118,10 +115,10 @@ def __init__(self, output_dir = os.path.join(this_dir, "../../template/downloaded/dataloader_files/RNAseq_features/") makedir_exist_ok(output_dir) RNAseq_PC_file = os.path.join(output_dir, cell_line, "meta.txt") - url_template = ('https://github.com/kipoi/models/blob/7648d3fd57def50934835b52acadd26bcaaa275c' - '/FactorNet/template/dataloader_files/RNAseq_features/{}/meta.txt?raw=true') + url_template = ('https://s3.eu-central-1.amazonaws.com/kipoi-models/dataloader_files/' + 'FactorNet/dataloader_files/RNAseq_features/{}/meta.txt') # rf = RemoteFile(url=url_template.format(cell_line)) - if not os.path.exists(RNAseq_PC_file): # or not rf.validate(mappability_file): + if not os.path.exists(RNAseq_PC_file): # or not rf.validate(mappability_file): # download the path download_url(url_template.format(cell_line), os.path.join(output_dir, cell_line), "meta.txt") # rf.get_file(RNAseq_PC_file) @@ -181,4 +178,4 @@ def __getitem__(self, idx): "ranges": ranges, "ranges_rc": ranges_rc } - } \ No newline at end of file + } diff --git a/FactorNet/MAFK/meta_1_Unique35_DGF/dataloader.yaml b/FactorNet/MAFK/meta_1_Unique35_DGF/dataloader.yaml index 5f16f5a9b..b843c7857 100644 --- a/FactorNet/MAFK/meta_1_Unique35_DGF/dataloader.yaml +++ b/FactorNet/MAFK/meta_1_Unique35_DGF/dataloader.yaml @@ -26,7 +26,7 @@ args: mappability_file: doc: USCS mappability track - http://hgdownload.cse.ucsc.edu/goldenpath/hg19/encodeDCC/wgEncodeMapability/wgEncodeDukeMapabilityUniqueness35bp.bigWig. by deafult, provide this file with the dataloader, download in background example: - url: https://github.com/kipoi/models/blob/92b23ed62956d56eb457488b431fc27550ddbf32/FactorNet/template/dataloader_files/wgEncodeDukeMapabilityUniqueness35bp.chr22.bigWig?raw=true + url: https://s3.eu-central-1.amazonaws.com/kipoi-models/dataloader_files/FactorNet/dataloader_files/wgEncodeDukeMapabilityUniqueness35bp.chr22.bigWig md5: 912e1bb77fb67c1a792ac4a05d9416f9 optional: True use_linecache: @@ -68,4 +68,4 @@ output_schema: doc: Ranges describing inputs.seq ranges_rc: type: GenomicRanges - doc: Ranges describing inputs.seq_rc \ No newline at end of file + doc: Ranges describing inputs.seq_rc diff --git a/FactorNet/MAFK/onePeak_1_DGF/dataloader.py b/FactorNet/MAFK/onePeak_1_DGF/dataloader.py index 5e737679a..3c201bf4f 100644 --- a/FactorNet/MAFK/onePeak_1_DGF/dataloader.py +++ b/FactorNet/MAFK/onePeak_1_DGF/dataloader.py @@ -29,9 +29,8 @@ def download_gencode_dir(output_dir): """Download all the required gencode files """ makedir_exist_ok(output_dir) - url_template = ("https://github.com/kipoi/models/blob/" - "7648d3fd57def50934835b52acadd26bcaaa275c/FactorNet/" - "template/dataloader_files/gencode_features/{}?raw=true") + url_template = ("https://s3.eu-central-1.amazonaws.com/kipoi-models/" + "dataloader_files/FactorNet/dataloader_files/gencode_features/{}") # url_template = "https://github.com/uci-cbcl/FactorNet/blob/master/resources/{}?raw=true" fnames = [('cpgisland.bed.gz', 'ac7dc007d7019c05adb7a331d1d6721d'), @@ -84,8 +83,6 @@ def __init__(self, self.bt = BT(intervals_file) - - # Fasta self.fasta_file = fasta_file self.fasta_extractor = None # initialize later @@ -138,4 +135,4 @@ def __getitem__(self, idx): "ranges": ranges, "ranges_rc": ranges_rc } - } \ No newline at end of file + } diff --git a/FactorNet/MAFK/onePeak_2_Unique35_DGF/dataloader.py b/FactorNet/MAFK/onePeak_2_Unique35_DGF/dataloader.py index d36ad58ef..775a01792 100644 --- a/FactorNet/MAFK/onePeak_2_Unique35_DGF/dataloader.py +++ b/FactorNet/MAFK/onePeak_2_Unique35_DGF/dataloader.py @@ -29,9 +29,8 @@ def download_gencode_dir(output_dir): """Download all the required gencode files """ makedir_exist_ok(output_dir) - url_template = ("https://github.com/kipoi/models/blob/" - "7648d3fd57def50934835b52acadd26bcaaa275c/FactorNet/" - "template/dataloader_files/gencode_features/{}?raw=true") + url_template = ("https://s3.eu-central-1.amazonaws.com/kipoi-models/" + "dataloader_files/FactorNet/dataloader_files/gencode_features/{}") # url_template = "https://github.com/uci-cbcl/FactorNet/blob/master/resources/{}?raw=true" fnames = [('cpgisland.bed.gz', 'ac7dc007d7019c05adb7a331d1d6721d'), @@ -85,8 +84,6 @@ def __init__(self, self.bt = BT(intervals_file) - - # Fasta self.fasta_file = fasta_file self.fasta_extractor = None # initialize later @@ -95,7 +92,7 @@ def __init__(self, self.dnase_extractor = None # mappability if mappability_file is None: - # download the mappability file if not existing + # download the mappability file if not existing common_dl_dir = os.path.join(this_dir, "../../template/downloaded/dataloader_files") makedir_exist_ok(common_dl_dir) rf = RemoteFile(url="http://hgdownload.cse.ucsc.edu/goldenPath/hg19/encodeDCC/wgEncodeMapability/wgEncodeDukeMapabilityUniqueness35bp.bigWig", @@ -158,4 +155,4 @@ def __getitem__(self, idx): "ranges": ranges, "ranges_rc": ranges_rc } - } \ No newline at end of file + } diff --git a/FactorNet/MAFK/onePeak_2_Unique35_DGF/dataloader.yaml b/FactorNet/MAFK/onePeak_2_Unique35_DGF/dataloader.yaml index 97a5f127b..96a8750c6 100644 --- a/FactorNet/MAFK/onePeak_2_Unique35_DGF/dataloader.yaml +++ b/FactorNet/MAFK/onePeak_2_Unique35_DGF/dataloader.yaml @@ -19,7 +19,7 @@ args: mappability_file: doc: USCS mappability track - http://hgdownload.cse.ucsc.edu/goldenpath/hg19/encodeDCC/wgEncodeMapability/wgEncodeDukeMapabilityUniqueness35bp.bigWig. by deafult, provide this file with the dataloader, download in background example: - url: https://github.com/kipoi/models/blob/92b23ed62956d56eb457488b431fc27550ddbf32/FactorNet/template/dataloader_files/wgEncodeDukeMapabilityUniqueness35bp.chr22.bigWig?raw=true + url: https://s3.eu-central-1.amazonaws.com/kipoi-models/dataloader_files/FactorNet/dataloader_files/wgEncodeDukeMapabilityUniqueness35bp.chr22.bigWig md5: 912e1bb77fb67c1a792ac4a05d9416f9 optional: True use_linecache: @@ -58,4 +58,4 @@ output_schema: doc: Ranges describing inputs.seq ranges_rc: type: GenomicRanges - doc: Ranges describing inputs.seq_rc \ No newline at end of file + doc: Ranges describing inputs.seq_rc diff --git a/FactorNet/MAX/onePeak_Unique35_DGF/dataloader.py b/FactorNet/MAX/onePeak_Unique35_DGF/dataloader.py index d36ad58ef..33a036afd 100644 --- a/FactorNet/MAX/onePeak_Unique35_DGF/dataloader.py +++ b/FactorNet/MAX/onePeak_Unique35_DGF/dataloader.py @@ -29,9 +29,9 @@ def download_gencode_dir(output_dir): """Download all the required gencode files """ makedir_exist_ok(output_dir) - url_template = ("https://github.com/kipoi/models/blob/" - "7648d3fd57def50934835b52acadd26bcaaa275c/FactorNet/" - "template/dataloader_files/gencode_features/{}?raw=true") + + url_template = ("https://s3.eu-central-1.amazonaws.com/kipoi-models/" + "dataloader_files/FactorNet/dataloader_files/gencode_features/{}") # url_template = "https://github.com/uci-cbcl/FactorNet/blob/master/resources/{}?raw=true" fnames = [('cpgisland.bed.gz', 'ac7dc007d7019c05adb7a331d1d6721d'), @@ -85,8 +85,6 @@ def __init__(self, self.bt = BT(intervals_file) - - # Fasta self.fasta_file = fasta_file self.fasta_extractor = None # initialize later @@ -95,7 +93,7 @@ def __init__(self, self.dnase_extractor = None # mappability if mappability_file is None: - # download the mappability file if not existing + # download the mappability file if not existing common_dl_dir = os.path.join(this_dir, "../../template/downloaded/dataloader_files") makedir_exist_ok(common_dl_dir) rf = RemoteFile(url="http://hgdownload.cse.ucsc.edu/goldenPath/hg19/encodeDCC/wgEncodeMapability/wgEncodeDukeMapabilityUniqueness35bp.bigWig", diff --git a/FactorNet/MAX/onePeak_Unique35_DGF/dataloader.yaml b/FactorNet/MAX/onePeak_Unique35_DGF/dataloader.yaml index 97a5f127b..f5fd23fff 100644 --- a/FactorNet/MAX/onePeak_Unique35_DGF/dataloader.yaml +++ b/FactorNet/MAX/onePeak_Unique35_DGF/dataloader.yaml @@ -19,7 +19,7 @@ args: mappability_file: doc: USCS mappability track - http://hgdownload.cse.ucsc.edu/goldenpath/hg19/encodeDCC/wgEncodeMapability/wgEncodeDukeMapabilityUniqueness35bp.bigWig. by deafult, provide this file with the dataloader, download in background example: - url: https://github.com/kipoi/models/blob/92b23ed62956d56eb457488b431fc27550ddbf32/FactorNet/template/dataloader_files/wgEncodeDukeMapabilityUniqueness35bp.chr22.bigWig?raw=true + url: https://s3.eu-central-1.amazonaws.com/kipoi-models/dataloader_files/FactorNet/dataloader_files/wgEncodeDukeMapabilityUniqueness35bp.chr22.bigWig md5: 912e1bb77fb67c1a792ac4a05d9416f9 optional: True use_linecache: diff --git a/FactorNet/MAX/onePeak_Unique35_DGF_2/dataloader.py b/FactorNet/MAX/onePeak_Unique35_DGF_2/dataloader.py index d36ad58ef..33a036afd 100644 --- a/FactorNet/MAX/onePeak_Unique35_DGF_2/dataloader.py +++ b/FactorNet/MAX/onePeak_Unique35_DGF_2/dataloader.py @@ -29,9 +29,9 @@ def download_gencode_dir(output_dir): """Download all the required gencode files """ makedir_exist_ok(output_dir) - url_template = ("https://github.com/kipoi/models/blob/" - "7648d3fd57def50934835b52acadd26bcaaa275c/FactorNet/" - "template/dataloader_files/gencode_features/{}?raw=true") + + url_template = ("https://s3.eu-central-1.amazonaws.com/kipoi-models/" + "dataloader_files/FactorNet/dataloader_files/gencode_features/{}") # url_template = "https://github.com/uci-cbcl/FactorNet/blob/master/resources/{}?raw=true" fnames = [('cpgisland.bed.gz', 'ac7dc007d7019c05adb7a331d1d6721d'), @@ -85,8 +85,6 @@ def __init__(self, self.bt = BT(intervals_file) - - # Fasta self.fasta_file = fasta_file self.fasta_extractor = None # initialize later @@ -95,7 +93,7 @@ def __init__(self, self.dnase_extractor = None # mappability if mappability_file is None: - # download the mappability file if not existing + # download the mappability file if not existing common_dl_dir = os.path.join(this_dir, "../../template/downloaded/dataloader_files") makedir_exist_ok(common_dl_dir) rf = RemoteFile(url="http://hgdownload.cse.ucsc.edu/goldenPath/hg19/encodeDCC/wgEncodeMapability/wgEncodeDukeMapabilityUniqueness35bp.bigWig", diff --git a/FactorNet/MAX/onePeak_Unique35_DGF_2/dataloader.yaml b/FactorNet/MAX/onePeak_Unique35_DGF_2/dataloader.yaml index 97a5f127b..f5fd23fff 100644 --- a/FactorNet/MAX/onePeak_Unique35_DGF_2/dataloader.yaml +++ b/FactorNet/MAX/onePeak_Unique35_DGF_2/dataloader.yaml @@ -19,7 +19,7 @@ args: mappability_file: doc: USCS mappability track - http://hgdownload.cse.ucsc.edu/goldenpath/hg19/encodeDCC/wgEncodeMapability/wgEncodeDukeMapabilityUniqueness35bp.bigWig. by deafult, provide this file with the dataloader, download in background example: - url: https://github.com/kipoi/models/blob/92b23ed62956d56eb457488b431fc27550ddbf32/FactorNet/template/dataloader_files/wgEncodeDukeMapabilityUniqueness35bp.chr22.bigWig?raw=true + url: https://s3.eu-central-1.amazonaws.com/kipoi-models/dataloader_files/FactorNet/dataloader_files/wgEncodeDukeMapabilityUniqueness35bp.chr22.bigWig md5: 912e1bb77fb67c1a792ac4a05d9416f9 optional: True use_linecache: diff --git a/FactorNet/NANOG/GENCODE_Unique35_DGF/dataloader.py b/FactorNet/NANOG/GENCODE_Unique35_DGF/dataloader.py index 7e71a5de1..9809179bd 100644 --- a/FactorNet/NANOG/GENCODE_Unique35_DGF/dataloader.py +++ b/FactorNet/NANOG/GENCODE_Unique35_DGF/dataloader.py @@ -29,9 +29,9 @@ def download_gencode_dir(output_dir): """Download all the required gencode files """ makedir_exist_ok(output_dir) - url_template = ("https://github.com/kipoi/models/blob/" - "7648d3fd57def50934835b52acadd26bcaaa275c/FactorNet/" - "template/dataloader_files/gencode_features/{}?raw=true") + + url_template = ("https://s3.eu-central-1.amazonaws.com/kipoi-models/" + "dataloader_files/FactorNet/dataloader_files/gencode_features/{}") # url_template = "https://github.com/uci-cbcl/FactorNet/blob/master/resources/{}?raw=true" fnames = [('cpgisland.bed.gz', 'ac7dc007d7019c05adb7a331d1d6721d'), @@ -87,8 +87,6 @@ def __init__(self, self.bt = BT(intervals_file) - - # Fasta self.fasta_file = fasta_file self.fasta_extractor = None # initialize later @@ -97,7 +95,7 @@ def __init__(self, self.dnase_extractor = None # mappability if mappability_file is None: - # download the mappability file if not existing + # download the mappability file if not existing common_dl_dir = os.path.join(this_dir, "../../template/downloaded/dataloader_files") makedir_exist_ok(common_dl_dir) rf = RemoteFile(url="http://hgdownload.cse.ucsc.edu/goldenPath/hg19/encodeDCC/wgEncodeMapability/wgEncodeDukeMapabilityUniqueness35bp.bigWig", diff --git a/FactorNet/NANOG/GENCODE_Unique35_DGF/dataloader.yaml b/FactorNet/NANOG/GENCODE_Unique35_DGF/dataloader.yaml index 23fb41cd3..ca9c01c47 100644 --- a/FactorNet/NANOG/GENCODE_Unique35_DGF/dataloader.yaml +++ b/FactorNet/NANOG/GENCODE_Unique35_DGF/dataloader.yaml @@ -23,7 +23,7 @@ args: mappability_file: doc: USCS mappability track - http://hgdownload.cse.ucsc.edu/goldenpath/hg19/encodeDCC/wgEncodeMapability/wgEncodeDukeMapabilityUniqueness35bp.bigWig. by deafult, provide this file with the dataloader, download in background example: - url: https://github.com/kipoi/models/blob/92b23ed62956d56eb457488b431fc27550ddbf32/FactorNet/template/dataloader_files/wgEncodeDukeMapabilityUniqueness35bp.chr22.bigWig?raw=true + url: https://s3.eu-central-1.amazonaws.com/kipoi-models/dataloader_files/FactorNet/dataloader_files/wgEncodeDukeMapabilityUniqueness35bp.chr22.bigWig md5: 912e1bb77fb67c1a792ac4a05d9416f9 optional: True GENCODE_dir: diff --git a/FactorNet/NANOG/onePeak_Unique35_DGF/dataloader.py b/FactorNet/NANOG/onePeak_Unique35_DGF/dataloader.py index d36ad58ef..33a036afd 100644 --- a/FactorNet/NANOG/onePeak_Unique35_DGF/dataloader.py +++ b/FactorNet/NANOG/onePeak_Unique35_DGF/dataloader.py @@ -29,9 +29,9 @@ def download_gencode_dir(output_dir): """Download all the required gencode files """ makedir_exist_ok(output_dir) - url_template = ("https://github.com/kipoi/models/blob/" - "7648d3fd57def50934835b52acadd26bcaaa275c/FactorNet/" - "template/dataloader_files/gencode_features/{}?raw=true") + + url_template = ("https://s3.eu-central-1.amazonaws.com/kipoi-models/" + "dataloader_files/FactorNet/dataloader_files/gencode_features/{}") # url_template = "https://github.com/uci-cbcl/FactorNet/blob/master/resources/{}?raw=true" fnames = [('cpgisland.bed.gz', 'ac7dc007d7019c05adb7a331d1d6721d'), @@ -85,8 +85,6 @@ def __init__(self, self.bt = BT(intervals_file) - - # Fasta self.fasta_file = fasta_file self.fasta_extractor = None # initialize later @@ -95,7 +93,7 @@ def __init__(self, self.dnase_extractor = None # mappability if mappability_file is None: - # download the mappability file if not existing + # download the mappability file if not existing common_dl_dir = os.path.join(this_dir, "../../template/downloaded/dataloader_files") makedir_exist_ok(common_dl_dir) rf = RemoteFile(url="http://hgdownload.cse.ucsc.edu/goldenPath/hg19/encodeDCC/wgEncodeMapability/wgEncodeDukeMapabilityUniqueness35bp.bigWig", diff --git a/FactorNet/NANOG/onePeak_Unique35_DGF/dataloader.yaml b/FactorNet/NANOG/onePeak_Unique35_DGF/dataloader.yaml index 97a5f127b..f5fd23fff 100644 --- a/FactorNet/NANOG/onePeak_Unique35_DGF/dataloader.yaml +++ b/FactorNet/NANOG/onePeak_Unique35_DGF/dataloader.yaml @@ -19,7 +19,7 @@ args: mappability_file: doc: USCS mappability track - http://hgdownload.cse.ucsc.edu/goldenpath/hg19/encodeDCC/wgEncodeMapability/wgEncodeDukeMapabilityUniqueness35bp.bigWig. by deafult, provide this file with the dataloader, download in background example: - url: https://github.com/kipoi/models/blob/92b23ed62956d56eb457488b431fc27550ddbf32/FactorNet/template/dataloader_files/wgEncodeDukeMapabilityUniqueness35bp.chr22.bigWig?raw=true + url: https://s3.eu-central-1.amazonaws.com/kipoi-models/dataloader_files/FactorNet/dataloader_files/wgEncodeDukeMapabilityUniqueness35bp.chr22.bigWig md5: 912e1bb77fb67c1a792ac4a05d9416f9 optional: True use_linecache: diff --git a/FactorNet/REST/GENCODE_Unique35_DGF/dataloader.py b/FactorNet/REST/GENCODE_Unique35_DGF/dataloader.py index 7e71a5de1..9809179bd 100644 --- a/FactorNet/REST/GENCODE_Unique35_DGF/dataloader.py +++ b/FactorNet/REST/GENCODE_Unique35_DGF/dataloader.py @@ -29,9 +29,9 @@ def download_gencode_dir(output_dir): """Download all the required gencode files """ makedir_exist_ok(output_dir) - url_template = ("https://github.com/kipoi/models/blob/" - "7648d3fd57def50934835b52acadd26bcaaa275c/FactorNet/" - "template/dataloader_files/gencode_features/{}?raw=true") + + url_template = ("https://s3.eu-central-1.amazonaws.com/kipoi-models/" + "dataloader_files/FactorNet/dataloader_files/gencode_features/{}") # url_template = "https://github.com/uci-cbcl/FactorNet/blob/master/resources/{}?raw=true" fnames = [('cpgisland.bed.gz', 'ac7dc007d7019c05adb7a331d1d6721d'), @@ -87,8 +87,6 @@ def __init__(self, self.bt = BT(intervals_file) - - # Fasta self.fasta_file = fasta_file self.fasta_extractor = None # initialize later @@ -97,7 +95,7 @@ def __init__(self, self.dnase_extractor = None # mappability if mappability_file is None: - # download the mappability file if not existing + # download the mappability file if not existing common_dl_dir = os.path.join(this_dir, "../../template/downloaded/dataloader_files") makedir_exist_ok(common_dl_dir) rf = RemoteFile(url="http://hgdownload.cse.ucsc.edu/goldenPath/hg19/encodeDCC/wgEncodeMapability/wgEncodeDukeMapabilityUniqueness35bp.bigWig", diff --git a/FactorNet/REST/GENCODE_Unique35_DGF/dataloader.yaml b/FactorNet/REST/GENCODE_Unique35_DGF/dataloader.yaml index 23fb41cd3..ca9c01c47 100644 --- a/FactorNet/REST/GENCODE_Unique35_DGF/dataloader.yaml +++ b/FactorNet/REST/GENCODE_Unique35_DGF/dataloader.yaml @@ -23,7 +23,7 @@ args: mappability_file: doc: USCS mappability track - http://hgdownload.cse.ucsc.edu/goldenpath/hg19/encodeDCC/wgEncodeMapability/wgEncodeDukeMapabilityUniqueness35bp.bigWig. by deafult, provide this file with the dataloader, download in background example: - url: https://github.com/kipoi/models/blob/92b23ed62956d56eb457488b431fc27550ddbf32/FactorNet/template/dataloader_files/wgEncodeDukeMapabilityUniqueness35bp.chr22.bigWig?raw=true + url: https://s3.eu-central-1.amazonaws.com/kipoi-models/dataloader_files/FactorNet/dataloader_files/wgEncodeDukeMapabilityUniqueness35bp.chr22.bigWig md5: 912e1bb77fb67c1a792ac4a05d9416f9 optional: True GENCODE_dir: diff --git a/FactorNet/REST/GENCODE_Unique35_DGF_2/dataloader.py b/FactorNet/REST/GENCODE_Unique35_DGF_2/dataloader.py index 7e71a5de1..9809179bd 100644 --- a/FactorNet/REST/GENCODE_Unique35_DGF_2/dataloader.py +++ b/FactorNet/REST/GENCODE_Unique35_DGF_2/dataloader.py @@ -29,9 +29,9 @@ def download_gencode_dir(output_dir): """Download all the required gencode files """ makedir_exist_ok(output_dir) - url_template = ("https://github.com/kipoi/models/blob/" - "7648d3fd57def50934835b52acadd26bcaaa275c/FactorNet/" - "template/dataloader_files/gencode_features/{}?raw=true") + + url_template = ("https://s3.eu-central-1.amazonaws.com/kipoi-models/" + "dataloader_files/FactorNet/dataloader_files/gencode_features/{}") # url_template = "https://github.com/uci-cbcl/FactorNet/blob/master/resources/{}?raw=true" fnames = [('cpgisland.bed.gz', 'ac7dc007d7019c05adb7a331d1d6721d'), @@ -87,8 +87,6 @@ def __init__(self, self.bt = BT(intervals_file) - - # Fasta self.fasta_file = fasta_file self.fasta_extractor = None # initialize later @@ -97,7 +95,7 @@ def __init__(self, self.dnase_extractor = None # mappability if mappability_file is None: - # download the mappability file if not existing + # download the mappability file if not existing common_dl_dir = os.path.join(this_dir, "../../template/downloaded/dataloader_files") makedir_exist_ok(common_dl_dir) rf = RemoteFile(url="http://hgdownload.cse.ucsc.edu/goldenPath/hg19/encodeDCC/wgEncodeMapability/wgEncodeDukeMapabilityUniqueness35bp.bigWig", diff --git a/FactorNet/REST/GENCODE_Unique35_DGF_2/dataloader.yaml b/FactorNet/REST/GENCODE_Unique35_DGF_2/dataloader.yaml index 23fb41cd3..ca9c01c47 100644 --- a/FactorNet/REST/GENCODE_Unique35_DGF_2/dataloader.yaml +++ b/FactorNet/REST/GENCODE_Unique35_DGF_2/dataloader.yaml @@ -23,7 +23,7 @@ args: mappability_file: doc: USCS mappability track - http://hgdownload.cse.ucsc.edu/goldenpath/hg19/encodeDCC/wgEncodeMapability/wgEncodeDukeMapabilityUniqueness35bp.bigWig. by deafult, provide this file with the dataloader, download in background example: - url: https://github.com/kipoi/models/blob/92b23ed62956d56eb457488b431fc27550ddbf32/FactorNet/template/dataloader_files/wgEncodeDukeMapabilityUniqueness35bp.chr22.bigWig?raw=true + url: https://s3.eu-central-1.amazonaws.com/kipoi-models/dataloader_files/FactorNet/dataloader_files/wgEncodeDukeMapabilityUniqueness35bp.chr22.bigWig md5: 912e1bb77fb67c1a792ac4a05d9416f9 optional: True GENCODE_dir: diff --git a/FactorNet/TAF1/GENCODE_Unique35_DGF/dataloader.py b/FactorNet/TAF1/GENCODE_Unique35_DGF/dataloader.py index 7e71a5de1..9809179bd 100644 --- a/FactorNet/TAF1/GENCODE_Unique35_DGF/dataloader.py +++ b/FactorNet/TAF1/GENCODE_Unique35_DGF/dataloader.py @@ -29,9 +29,9 @@ def download_gencode_dir(output_dir): """Download all the required gencode files """ makedir_exist_ok(output_dir) - url_template = ("https://github.com/kipoi/models/blob/" - "7648d3fd57def50934835b52acadd26bcaaa275c/FactorNet/" - "template/dataloader_files/gencode_features/{}?raw=true") + + url_template = ("https://s3.eu-central-1.amazonaws.com/kipoi-models/" + "dataloader_files/FactorNet/dataloader_files/gencode_features/{}") # url_template = "https://github.com/uci-cbcl/FactorNet/blob/master/resources/{}?raw=true" fnames = [('cpgisland.bed.gz', 'ac7dc007d7019c05adb7a331d1d6721d'), @@ -87,8 +87,6 @@ def __init__(self, self.bt = BT(intervals_file) - - # Fasta self.fasta_file = fasta_file self.fasta_extractor = None # initialize later @@ -97,7 +95,7 @@ def __init__(self, self.dnase_extractor = None # mappability if mappability_file is None: - # download the mappability file if not existing + # download the mappability file if not existing common_dl_dir = os.path.join(this_dir, "../../template/downloaded/dataloader_files") makedir_exist_ok(common_dl_dir) rf = RemoteFile(url="http://hgdownload.cse.ucsc.edu/goldenPath/hg19/encodeDCC/wgEncodeMapability/wgEncodeDukeMapabilityUniqueness35bp.bigWig", diff --git a/FactorNet/TAF1/GENCODE_Unique35_DGF/dataloader.yaml b/FactorNet/TAF1/GENCODE_Unique35_DGF/dataloader.yaml index 23fb41cd3..ca9c01c47 100644 --- a/FactorNet/TAF1/GENCODE_Unique35_DGF/dataloader.yaml +++ b/FactorNet/TAF1/GENCODE_Unique35_DGF/dataloader.yaml @@ -23,7 +23,7 @@ args: mappability_file: doc: USCS mappability track - http://hgdownload.cse.ucsc.edu/goldenpath/hg19/encodeDCC/wgEncodeMapability/wgEncodeDukeMapabilityUniqueness35bp.bigWig. by deafult, provide this file with the dataloader, download in background example: - url: https://github.com/kipoi/models/blob/92b23ed62956d56eb457488b431fc27550ddbf32/FactorNet/template/dataloader_files/wgEncodeDukeMapabilityUniqueness35bp.chr22.bigWig?raw=true + url: https://s3.eu-central-1.amazonaws.com/kipoi-models/dataloader_files/FactorNet/dataloader_files/wgEncodeDukeMapabilityUniqueness35bp.chr22.bigWig md5: 912e1bb77fb67c1a792ac4a05d9416f9 optional: True GENCODE_dir: diff --git a/FactorNet/TAF1/onePeak_Unique35_DGF/dataloader.py b/FactorNet/TAF1/onePeak_Unique35_DGF/dataloader.py index d36ad58ef..33a036afd 100644 --- a/FactorNet/TAF1/onePeak_Unique35_DGF/dataloader.py +++ b/FactorNet/TAF1/onePeak_Unique35_DGF/dataloader.py @@ -29,9 +29,9 @@ def download_gencode_dir(output_dir): """Download all the required gencode files """ makedir_exist_ok(output_dir) - url_template = ("https://github.com/kipoi/models/blob/" - "7648d3fd57def50934835b52acadd26bcaaa275c/FactorNet/" - "template/dataloader_files/gencode_features/{}?raw=true") + + url_template = ("https://s3.eu-central-1.amazonaws.com/kipoi-models/" + "dataloader_files/FactorNet/dataloader_files/gencode_features/{}") # url_template = "https://github.com/uci-cbcl/FactorNet/blob/master/resources/{}?raw=true" fnames = [('cpgisland.bed.gz', 'ac7dc007d7019c05adb7a331d1d6721d'), @@ -85,8 +85,6 @@ def __init__(self, self.bt = BT(intervals_file) - - # Fasta self.fasta_file = fasta_file self.fasta_extractor = None # initialize later @@ -95,7 +93,7 @@ def __init__(self, self.dnase_extractor = None # mappability if mappability_file is None: - # download the mappability file if not existing + # download the mappability file if not existing common_dl_dir = os.path.join(this_dir, "../../template/downloaded/dataloader_files") makedir_exist_ok(common_dl_dir) rf = RemoteFile(url="http://hgdownload.cse.ucsc.edu/goldenPath/hg19/encodeDCC/wgEncodeMapability/wgEncodeDukeMapabilityUniqueness35bp.bigWig", diff --git a/FactorNet/TAF1/onePeak_Unique35_DGF/dataloader.yaml b/FactorNet/TAF1/onePeak_Unique35_DGF/dataloader.yaml index 97a5f127b..f5fd23fff 100644 --- a/FactorNet/TAF1/onePeak_Unique35_DGF/dataloader.yaml +++ b/FactorNet/TAF1/onePeak_Unique35_DGF/dataloader.yaml @@ -19,7 +19,7 @@ args: mappability_file: doc: USCS mappability track - http://hgdownload.cse.ucsc.edu/goldenpath/hg19/encodeDCC/wgEncodeMapability/wgEncodeDukeMapabilityUniqueness35bp.bigWig. by deafult, provide this file with the dataloader, download in background example: - url: https://github.com/kipoi/models/blob/92b23ed62956d56eb457488b431fc27550ddbf32/FactorNet/template/dataloader_files/wgEncodeDukeMapabilityUniqueness35bp.chr22.bigWig?raw=true + url: https://s3.eu-central-1.amazonaws.com/kipoi-models/dataloader_files/FactorNet/dataloader_files/wgEncodeDukeMapabilityUniqueness35bp.chr22.bigWig md5: 912e1bb77fb67c1a792ac4a05d9416f9 optional: True use_linecache: diff --git a/FactorNet/template/template_dataloader.py b/FactorNet/template/template_dataloader.py index 3c582d04e..a01362ee3 100644 --- a/FactorNet/template/template_dataloader.py +++ b/FactorNet/template/template_dataloader.py @@ -29,9 +29,9 @@ def download_gencode_dir(output_dir): """Download all the required gencode files """ makedir_exist_ok(output_dir) - url_template = ("https://github.com/kipoi/models/blob/" - "7648d3fd57def50934835b52acadd26bcaaa275c/FactorNet/" - "template/dataloader_files/gencode_features/{}?raw=true") + + url_template = ("https://s3.eu-central-1.amazonaws.com/kipoi-models/" + "dataloader_files/FactorNet/dataloader_files/gencode_features/{}") # url_template = "https://github.com/uci-cbcl/FactorNet/blob/master/resources/{}?raw=true" fnames = [('cpgisland.bed.gz', 'ac7dc007d7019c05adb7a331d1d6721d'), @@ -96,8 +96,6 @@ def __init__(self, self.bt = BT(intervals_file) - - # Fasta self.fasta_file = fasta_file self.fasta_extractor = None # initialize later @@ -105,11 +103,10 @@ def __init__(self, self.dnase_file = dnase_file self.dnase_extractor = None - {%- if needs_mappability %} # mappability if mappability_file is None: - # download the mappability file if not existing + # download the mappability file if not existing common_dl_dir = os.path.join(this_dir, "../../template/downloaded/dataloader_files") makedir_exist_ok(common_dl_dir) rf = RemoteFile(url="http://hgdownload.cse.ucsc.edu/goldenPath/hg19/encodeDCC/wgEncodeMapability/wgEncodeDukeMapabilityUniqueness35bp.bigWig", @@ -158,10 +155,10 @@ def __init__(self, output_dir = os.path.join(this_dir, "../../template/downloaded/dataloader_files/RNAseq_features/") makedir_exist_ok(output_dir) RNAseq_PC_file = os.path.join(output_dir, cell_line, "meta.txt") - url_template = ('https://github.com/kipoi/models/blob/7648d3fd57def50934835b52acadd26bcaaa275c' - '/FactorNet/template/dataloader_files/RNAseq_features/{}/meta.txt?raw=true') + url_template = ('https://s3.eu-central-1.amazonaws.com/kipoi-models/dataloader_files/' + 'FactorNet/dataloader_files/RNAseq_features/{}/meta.txt') # rf = RemoteFile(url=url_template.format(cell_line)) - if not os.path.exists(RNAseq_PC_file): # or not rf.validate(mappability_file): + if not os.path.exists(RNAseq_PC_file): # or not rf.validate(mappability_file): # download the path download_url(url_template.format(cell_line), os.path.join(output_dir, cell_line), "meta.txt") # rf.get_file(RNAseq_PC_file) diff --git a/FactorNet/template/template_dataloader.yaml b/FactorNet/template/template_dataloader.yaml index 1db345c55..8357cd8d5 100644 --- a/FactorNet/template/template_dataloader.yaml +++ b/FactorNet/template/template_dataloader.yaml @@ -31,7 +31,7 @@ args: mappability_file: doc: USCS mappability track - http://hgdownload.cse.ucsc.edu/goldenpath/hg19/encodeDCC/wgEncodeMapability/wgEncodeDukeMapabilityUniqueness35bp.bigWig. by deafult, provide this file with the dataloader, download in background example: - url: https://github.com/kipoi/models/blob/92b23ed62956d56eb457488b431fc27550ddbf32/FactorNet/template/dataloader_files/wgEncodeDukeMapabilityUniqueness35bp.chr22.bigWig?raw=true + url: https://s3.eu-central-1.amazonaws.com/kipoi-models/dataloader_files/FactorNet/dataloader_files/wgEncodeDukeMapabilityUniqueness35bp.chr22.bigWig md5: 912e1bb77fb67c1a792ac4a05d9416f9 optional: True {%- endif %}