Integrate JBrowse into Varify

Supplement Alamut with JBrowse
Fixes #338

Signed-off-by: Ryan O'Hara <[email protected]>

README.md

@@ -164,6 +164,56 @@ or run a `uwsgi` process:
 uwsgi --ini server/uwsgi/local.ini --protocol http --socket 127.0.0.1:8000 --check-static _site
 ```
 
+## Optional JBrowse Setup
+
+*Note: By default, both Alamut and JBrowse are enabled. To disable, comment out one or both buttons in varify/static/templates/variant/summary.html.*
+
+Install JBrowse in $project_root/jbrowse
+
+```bash
+curl -O http://jbrowse.org/releases/JBrowse-x.x.x.zip (ie 1.11.3)
+unzip JBrowse-x.x.x.zip -d jbrowse
+cd jbrowse
+./setup.sh
+```
+
+Download data files (ie BAM, GFF, VCF) to /your/directory/of/files/ and add the following to nginx.conf. This corresponds to the JbrowseResource endpoint defined in varify/samples/resources.py.
+
+```bash
+location /files/ {
+    alias /your/directory/of/files/;
+    internal;
+}
+```
+
+Load reference sequences and load features tracks (in that order) using JBrowse loading scripts. This only needs to be done once.
+
+```bash
+sudo ./bin/prepare-refseqs.pl --fasta ../files/chr1.fa
+...
+sudo ./bin/prepare-refseqs.pl --fasta ../files/chrY.fa
+sudo ./bin/flatfile-to-json.pl --gff ../files/ALL_COSMIC_POINT_MUTS_v65.gff3 --trackLabel Cosmic --trackType CanvasFeatures
+...
+
+Note: Segment large Cosmic GFF files with synchronization marks, a line containing '###', to prevent (really) slow loading. Once a loading script executes, a data directory will exist in $project_root/jbrowse.
+```
+
+Run bgzip and tabix (via samtools/htslib) on VCF files
+
+```bash
+git clone git://github.com/samtools/samtools.git
+bgzip my.vcf
+tabix -p vcf my.vcf.gz
+```
+
+**Lastly, make sure data files are named correctly!** Currently, the md5 in the [vcf] section of the sample manifest is used as the filename root for the BAM, BAI, VCF, and TBI files in the JBrowse URL. Suffixes are hard-coded '.bam','.bai','.vcf.gz', and '.vcf.gz.tbi', respectively.
+
+```bash
+[vcf]
+file = Pseq_batch9.example
+md5 = 1z2k5jhf8ab3rexample
+```
+
 ## Makefile Commands
 
 - `build` - builds and initializes all submodules, compiles SCSS and optimizes JavaScript

varify/conf/global_settings.py

@@ -167,7 +167,7 @@
 TEMPLATE_CONTEXT_PROCESSORS += (
     'django.core.context_processors.request',
     'varify.context_processors.static',
-    'varify.context_processors.alamut',
+    'varify.context_processors.alamut'
 )
 
 
@@ -188,6 +188,7 @@
 LOGIN_REDIRECT_URL = '/workspace/'
 
 ALAMUT_URL = 'http://localhost:10000'
+JBROWSE_HOST = 'localhost'
 
 # For non-publicly accessible applications, the siteauth app can be used to
 # restrict access site-wide.

varify/samples/formatters.py

@@ -4,8 +4,8 @@
     from ordereddict import OrderedDict
 from django.db.models import Q
 from avocado.formatters import registry as formatters
-from serrano.formatters import HTMLFormatter
 from django.conf import settings
+from serrano.formatters import HTMLFormatter
 
 
 class AlamutFormatter(HTMLFormatter):

varify/samples/resources.py

@@ -22,10 +22,12 @@
 from varify import api
 from vdw.assessments.models import Assessment
 from vdw.genome.models import Chromosome
-from vdw.samples.models import Sample, Result, ResultScore, ResultSet
+from vdw.samples.models import (Sample, SampleManifest, Result, ResultScore,
+                                ResultSet)
 from vdw.variants.models import Variant
 from .forms import ResultSetForm
 
+
 log = logging.getLogger(__name__)
 OPENPYXL_MAJOR_VERSION = int(openpyxl.__version__[0])
 
@@ -274,6 +276,28 @@ def _cache_data(self, request, pk, key):
         data['variant'] = VariantResource.get(request, data['variant_id'])
         data.pop('variant_id')
 
+        # Integrate the SampleManifest data
+        sm = SampleManifest.objects.get(sample__id=data['sample']['id'])
+        found = False
+        filestem = ''
+
+        # Add JBrowse data requirements, including sample_manifest which is
+        # constructed using the md5 from the [vcf] section
+        for line in sm.content.split('\n'):
+            if '[vcf]' in line:
+                found = True
+
+            if found:
+                if line.startswith('md5 = '):
+                    chunks = line.split('=')
+
+                    if len(chunks) > 1:
+                        filestem = chunks[1].strip()
+                        break
+
+        data['sample_manifest'] = filestem
+        data['jbrowse_host'] = settings.JBROWSE_HOST
+
         try:
             score = ResultScore.objects.get(result=result)
             data['score'] = {
@@ -310,10 +334,15 @@ class ResultsResource(ThrottledResource):
     template = api.templates.SampleResultVariant
 
     def post(self, request):
-        if (not request.data.get('ids') or
-                not isinstance(request.data['ids'], list)):
-            return HttpResponse(status=codes.unprocessable_entity,
-                                content='Array of "ids" is required')
+        ids_not_found = 'ids' not in request.data
+        not_a_list = not isinstance(request.data['ids'], list)
+
+        if ids_not_found or not_a_list:
+            return self.render(
+                request,
+                {'message': 'An array of "ids" is required'},
+                status=codes.unprocessable_entity
+            )
 
         data = []
         resource = SampleResultResource()
@@ -686,6 +715,31 @@ def get(self, request, pk):
         return data
 
 
+class JbrowseResource(ThrottledResource):
+    def get(self, request, filename):
+        response = HttpResponse()
+        try:
+            if '.bai' in filename or '.tbi' in filename:
+                response['Content-Type'] = 'application/octet-stream'
+            else:
+                response['Content-Type'] = 'text/plain'
+
+            # Control access to files hosted by nginx
+            response['X-Accel-Redirect'] = '/files/' + filename
+            # Control access to files hosted by Apache
+            response['X-Sendfile'] = '/files/' + filename
+
+            response['Content-Disposition'] = 'attachment;filename=' + filename
+        except Exception:
+            return self.render(
+                request,
+                {'message': 'No sample found for "id"'},
+                status=codes.unprocessable_entity
+            )
+
+        return response
+
+
 sample_resource = never_cache(SampleResource())
 samples_resource = never_cache(SamplesResource())
 named_sample_resource = never_cache(NamedSampleResource())
@@ -699,6 +753,7 @@ def get(self, request, pk):
 
 phenotype_resource = never_cache(PhenotypeResource())
 pedigree_resource = never_cache(PedigreeResource())
+jbrowse_resource = never_cache(JbrowseResource())
 
 urlpatterns = patterns(
     '',
@@ -731,6 +786,10 @@ def get(self, request, pk):
         sample_result_set_resource,
         name='variant-set'),
 
+    url(r'^jbrowse/(?P<filename>.+)$',
+        jbrowse_resource,
+        name='jbrowse_resource'),
+
     url(r'^(?P<sample_id>.+)/phenotypes/$',
         phenotype_resource,
         name='phenotype'),

varify/static/templates/variant/summary.html

@@ -49,3 +49,5 @@ <h4><%= data.sample.label %> <small>in <%= data.sample.project %></small></h4>
 </ul>
 
 <a href='http://localhost:10000/show?request=chr<%= data.variant.chr %>:g.<%= data.variant.pos %><%= data.variant.ref %>><%= data.variant.alt %>' target=_blank class='btn btn-primary btn-small alamut-button'>Query Alamut</a>
+
+<a href='http://<%= data.jbrowse_host%>/jbrowse/?loc=chr<%= data.variant.chr %>:<%= data.variant.pos-25|0 %>..<%= data.variant.pos+25 %>&tracks=DNA,Cosmic,VCF,BAM&highlight=&addTracks=%5B%7B%22label%22%3A%22BAM%22%2C%22store%22%3A%22testStore%22%2C%22type%22%3A%22JBrowse/View/Track/Alignments2%22%7D%2C%7B%22label%22%3A%22VCF%22%2C%22store%22%3A%22testStoreTwo%22%2C%22type%22%3A%22JBrowse/View/Track/CanvasVariants%22%7D%5D&addStores=%7B%22testStore%22%3A%7B%22baiUrlTemplate%22%3A%22http%3A//<%= data.jbrowse_host%>/api/samples/jbrowse/<%= data.sample_manifest%>.bai%22%2C%22type%22%3A%22JBrowse/Store/SeqFeature/BAM%22%2C%22urlTemplate%22%3A%22http%3A//<%= data.jbrowse_host%>/api/samples/jbrowse/<%= data.sample_manifest%>.bam%22%7D%2C%22testStoreTwo%22%3A%7B%22tbiUrlTemplate%22%3A%22http%3A//<%= data.jbrowse_host%>/api/samples/jbrowse/<%= data.sample_manifest%>.vcf.gz.tbi%22%2C%22type%22%3A%22JBrowse/Store/SeqFeature/VCFTabix%22%2C%22urlTemplate%22%3A%22http%3A//<%= data.jbrowse_host%>/api/samples/jbrowse/<%= data.sample_manifest%>.vcf.gz%22%7D%7D' target=_blank class='btn btn-primary btn-small'>Query JBrowse</a>