Integrate JBrowse into Varify

Replace Alamut with JBrowse
Fixes #338

Signed-off-by: Ryan O'Hara <[email protected]>

README.md

@@ -164,6 +164,65 @@ or run a `uwsgi` process:
 uwsgi --ini server/uwsgi/local.ini --protocol http --socket 127.0.0.1:8000 --check-static _site
 ```
 
+## Optional JBrowse Setup
+
+Install JBrowse in $project_root/jbrowse
+
+```bash
+curl -O http://jbrowse.org/releases/JBrowse-x.x.x.zip (ie 1.11.3)
+unzip JBrowse-x.x.x.zip -d jbrowse
+cd jbrowse
+./setup.sh
+```
+
+Download data files (ie BAM, GFF, VCF) to /your/directory/of/files/ and add the following to nginx.conf
+
+```bash
+location /files/ {
+    alias /your/directory/of/files/;
+    internal;
+}
+```
+
+Load reference sequences and load features tracks (in that order) using JBrowse loading scripts
+
+```bash
+sudo ./bin/prepare-refseqs.pl --fasta ../files/chr1.fa
+...
+sudo ./bin/prepare-refseqs.pl --fasta ../files/chrY.fa
+sudo ./bin/flatfile-to-json.pl --gff ../files/ALL_COSMIC_POINT_MUTS_v65.gff3 --trackLabel Cosmic --trackType CanvasFeatures
+...
+
+Note: Segment large Cosmic GFF files with synchronization marks, a line containing '###', to prevent (really) slow loading. Once a loading script executes, a data directory will exist in $project_root/jbrowse.
+```
+
+Run bgzip and tabix (via samtools/htslib) on VCF files
+
+```bash
+git clone git://github.com/samtools/samtools.git
+bgzip my.vcf
+tabix -p vcf my.vcf.gz
+```
+
+Make one edit to JBrowse source (BAM.js), regex change related to custom URL
+
+```bash
+/\/([^/\#\?]+)($|[\#\?])/
+
+to
+
+/.*filename=([^&]+)/
+```
+
+Lastly, save and load filenames correctly! Currently, the sample section key values of the manifest are concatenated (and '_' delimited) to build filename roots in the JBrowse URL. BAM, BAI, VCF, and TBI files are hard-codedly suffixed '.sorted.mdup.bam','.sorted.mdup.bai','.var_raw.vcf.gz', and '.var_raw.vcf.gz.tbi', respectively.
+
+```bash
+[sample]
+batch = Pseq_batch9
+sample = P-Pseq_0019-P-A
+version = 1
+```
+
 ## Makefile Commands
 
 - `build` - builds and initializes all submodules, compiles SCSS and optimizes JavaScript

varify/conf/global_settings.py

@@ -167,7 +167,6 @@
 TEMPLATE_CONTEXT_PROCESSORS += (
     'django.core.context_processors.request',
     'varify.context_processors.static',
-    'varify.context_processors.alamut',
 )
 
 
@@ -187,7 +186,7 @@
 LOGOUT_URL = '/logout/'
 LOGIN_REDIRECT_URL = '/workspace/'
 
-ALAMUT_URL = 'http://localhost:10000'
+JBROWSE_HOST = 'localhost'
 
 # For non-publicly accessible applications, the siteauth app can be used to
 # restrict access site-wide.

varify/context_processors.py

@@ -14,9 +14,3 @@ def static(request):
         'IMAGES_URL': os.path.join(static_url, 'images'),
         'JAVASCRIPT_URL': os.path.join(static_url, 'js', prefix),
     }
-
-
-def alamut(request):
-    return {
-        'ALAMUT_URL': settings.ALAMUT_URL,
-    }

varify/samples/formatters.py

@@ -5,27 +5,6 @@
 from django.db.models import Q
 from avocado.formatters import registry as formatters
 from serrano.formatters import HTMLFormatter
-from django.conf import settings
-
-
-class AlamutFormatter(HTMLFormatter):
-    href = settings.ALAMUT_URL + '/show?request={0}'
-    request = 'chr{chr}:g.{pos}{ref}>{alt}'
-
-    def to_html(self, values, **context):
-        request = self.request.format(**values)
-        href = self.href.format(request)
-        return '<a target=_blank href="{href}">{label}</a>'.format(
-            href=href, label=request)
-    to_html.process_multiple = True
-
-    def to_excel(self, values, **context):
-        request = self.request.format(**values)
-        href = self.href.format(request)
-
-        return '=HYPERLINK("{href}", "{label}")'.format(
-            href=href, label=request)
-    to_excel.process_multiple = True
 
 
 class SampleFormatter(HTMLFormatter):
@@ -105,7 +84,6 @@ def to_excel(self, value, **context):
     to_csv = to_excel
 
 
-formatters.register(AlamutFormatter, 'Alamut Query Link')
 formatters.register(SampleFormatter, 'Sample')
 formatters.register(ReadDepthFormatter, 'Read Depth')
 formatters.register(CohortsFormatter, 'Cohorts')

varify/samples/resources.py

@@ -22,10 +22,12 @@
 from varify import api
 from vdw.assessments.models import Assessment
 from vdw.genome.models import Chromosome
-from vdw.samples.models import Sample, Result, ResultScore, ResultSet
+from vdw.samples.models import (Sample, SampleManifest, Result, ResultScore,
+                                ResultSet)
 from vdw.variants.models import Variant
 from .forms import ResultSetForm
 
+
 log = logging.getLogger(__name__)
 OPENPYXL_MAJOR_VERSION = int(openpyxl.__version__[0])
 
@@ -274,6 +276,34 @@ def _cache_data(self, request, pk, key):
         data['variant'] = VariantResource.get(request, data['variant_id'])
         data.pop('variant_id')
 
+        # Integrate the SampleManifest data
+        path = SampleManifest.objects.get(sample__id=data['sample']['id']).path
+        manifest = open(path, 'r')
+        found = False
+        filestem = ''
+
+        # Add JBrowse data requirements, including sample_manifest which is
+        # constructed by appending batch, sample, and version
+        for line in manifest:
+            if '[sample]' in line:
+                found = True
+
+            if found:
+                if 'batch' in line or 'sample' in line or 'version' in line:
+                    chunks = line.split('=')
+
+                    if len(chunks) > 1:
+                        if 'batch' in line:
+                            filestem += chunks[1].strip()
+                        else:
+                            filestem += '_' + chunks[1].strip()
+
+                            if 'version' in line:
+                                break
+
+        data['sample_manifest'] = filestem
+        data['jbrowse_host'] = settings.JBROWSE_HOST
+
         try:
             score = ResultScore.objects.get(result=result)
             data['score'] = {
@@ -310,10 +340,15 @@ class ResultsResource(ThrottledResource):
     template = api.templates.SampleResultVariant
 
     def post(self, request):
-        if (not request.data.get('ids') or
-                not isinstance(request.data['ids'], list)):
-            return HttpResponse(status=codes.unprocessable_entity,
-                                content='Array of "ids" is required')
+        ids_not_found = 'ids' not in request.data
+        not_a_list = not isinstance(request.data['ids'], list)
+
+        if ids_not_found or not_a_list:
+            return self.render(
+                request,
+                {'message': 'An array of "ids" is required'},
+                status=codes.unprocessable_entity
+            )
 
         data = []
         resource = SampleResultResource()
@@ -686,6 +721,43 @@ def get(self, request, pk):
         return data
 
 
+class JbrowseResource(ThrottledResource):
+    def get(self, request):
+        data = request.GET
+
+        if 'id' not in data or 'filename' not in data:
+            return self.render(
+                request,
+                {'message': 'An "id" and "filename" are required'},
+                status=codes.unprocessable_entity
+            )
+
+        try:
+            if Sample.objects.get(id=data['id']):
+                response = HttpResponse()
+                f = data['filename']
+
+                if '.bai' in f or '.tbi' in f:
+                    response['Content-Type'] = 'application/octet-stream'
+                else:
+                    response['Content-Type'] = 'text/plain'
+
+                # Control access to files hosted by nginx
+                response['X-Accel-Redirect'] = '/files/' + f
+                # Control access to files hosted by Apache
+                response['X-Sendfile'] = '/files/' + f
+
+                response['Content-Disposition'] = 'attachment;filename=' + f
+        except Exception:
+            return self.render(
+                request,
+                {'message': 'No sample found for "id"'},
+                status=codes.unprocessable_entity
+            )
+
+        return response
+
+
 sample_resource = never_cache(SampleResource())
 samples_resource = never_cache(SamplesResource())
 named_sample_resource = never_cache(NamedSampleResource())
@@ -699,6 +771,7 @@ def get(self, request, pk):
 
 phenotype_resource = never_cache(PhenotypeResource())
 pedigree_resource = never_cache(PedigreeResource())
+jbrowse_resource = never_cache(JbrowseResource())
 
 urlpatterns = patterns(
     '',
@@ -731,6 +804,10 @@ def get(self, request, pk):
         sample_result_set_resource,
         name='variant-set'),
 
+    url(r'^jbrowse/$',
+        jbrowse_resource,
+        name='jbrowse_resource'),
+
     url(r'^(?P<sample_id>.+)/phenotypes/$',
         phenotype_resource,
         name='phenotype'),

varify/static/templates/variant/summary.html

@@ -48,4 +48,4 @@ <h4><%= data.sample.label %> <small>in <%= data.sample.project %></small></h4>
     </li>
 </ul>
 
-<a href='http://localhost:10000/show?request=chr<%= data.variant.chr %>:g.<%= data.variant.pos %><%= data.variant.ref %>><%= data.variant.alt %>' target=_blank class='btn btn-primary btn-small alamut-button'>Query Alamut</a>
+<a href='http://<%= data.jbrowse_host%>/jbrowse/?loc=chr<%= data.variant.chr %>:<%= data.variant.pos-25|0 %>..<%= data.variant.pos+25 %>&tracks=DNA,Cosmic,VCF,BAM&highlight=&addTracks=%5B%7B%22label%22%3A%22BAM%22%2C%22store%22%3A%22testStore%22%2C%22type%22%3A%22JBrowse/View/Track/Alignments2%22%7D%2C%7B%22label%22%3A%22VCF%22%2C%22store%22%3A%22testStoreTwo%22%2C%22type%22%3A%22JBrowse/View/Track/CanvasVariants%22%7D%5D&addStores=%7B%22testStore%22%3A%7B%22baiUrlTemplate%22%3A%22http%3A//<%= data.jbrowse_host%>/api/samples/jbrowse/?id=<%= data.sample.id%>%26filename%3D<%= data.sample_manifest%>.sorted.mdup.bai%22%2C%22type%22%3A%22JBrowse/Store/SeqFeature/BAM%22%2C%22urlTemplate%22%3A%22http%3A//<%= data.jbrowse_host%>/api/samples/jbrowse/?id=<%= data.sample.id%>%26filename%3D<%= data.sample_manifest%>.sorted.mdup.bam%22%7D%2C%22testStoreTwo%22%3A%7B%22tbiUrlTemplate%22%3A%22http%3A//<%= data.jbrowse_host%>/api/samples/jbrowse/?id=<%= data.sample.id%>%26filename%3D<%= data.sample_manifest%>.var_raw.vcf.gz.tbi%22%2C%22type%22%3A%22JBrowse/Store/SeqFeature/VCFTabix%22%2C%22urlTemplate%22%3A%22http%3A//<%= data.jbrowse_host%>/api/samples/jbrowse/?id=<%= data.sample.id%>%26filename%3D<%= data.sample_manifest%>.var_raw.vcf.gz%22%7D%7D' target=_blank class='btn btn-primary btn-small'>Query JBrowse</a>