Merge branch 'bug_fixes'

MANUAL

@@ -1516,7 +1516,7 @@ by tabs; from left to right, the fields are:
     will truncate the name at the first whitespace character. This is
     similar to the behavior of other tools. The standard behavior of
     truncating at the first whitespace can be suppressed with
-    --sam-noqname-trunc at the expense of generating non-standard SAM.
+    --sam-no-qname-trunc at the expense of generating non-standard SAM.
 
 2.  Sum of all applicable flags. Flags relevant to Bowtie are:
 

MANUAL.markdown

@@ -1927,8 +1927,8 @@ left to right, the fields are:
 	If the read name contains any whitespace characters, Bowtie 2 will truncate
 	the name at the first whitespace character.  This is similar to the
 	behavior of other tools. The standard behavior of truncating at the first
-    whitespace can be suppressed with `--sam-noqname-trunc` at the expense of
-    generating non-standard SAM.
+	whitespace can be suppressed with `--sam-no-qname-trunc` at the expense of
+	generating non-standard SAM.
 
 2.  Sum of all applicable flags.  Flags relevant to Bowtie are:
 

Makefile

@@ -24,20 +24,20 @@
 prefix = /usr/local
 bindir = $(prefix)/bin
 
-INC =
-LIBS = -lz
+INC = $(if $(RELEASE_BUILD),-I$(CURDIR)/.include)
+LIBS = $(LDFLAGS) $(if $(RELEASE_BUILD),-L$(CURDIR)/.lib) -lz
 GCC_PREFIX = $(shell dirname `which gcc`)
 GCC_SUFFIX =
 CC ?= $(GCC_PREFIX)/gcc$(GCC_SUFFIX)
 CPP ?= $(GCC_PREFIX)/g++$(GCC_SUFFIX)
 CXX ?= $(CPP)
 HEADERS = $(wildcard *.h)
 BOWTIE_MM = 1
-BOWTIE_SHARED_MEM = 0
+BOWTIE_SHARED_MEM =
 
 # Detect Cygwin or MinGW
-WINDOWS = 0
-MINGW = 0
+WINDOWS =
+MINGW =
 ifneq (,$(findstring MINGW,$(shell uname)))
 	WINDOWS = 1
 	MINGW = 1
@@ -47,14 +47,17 @@ ifneq (,$(findstring MINGW,$(shell uname)))
 	override EXTRA_FLAGS += -ansi
 endif
 
-MACOS = 0
+MACOS =
 ifneq (,$(findstring Darwin,$(shell uname)))
 	MACOS = 1
 	ifneq (,$(findstring 13,$(shell uname -r)))
 		CPP = clang++
 		CC = clang
 		override EXTRA_FLAGS += -stdlib=libstdc++
 	endif
+	ifeq (1, $(RELEASE_BUILD))
+		EXTRA_FLAGS += -mmacosx-version-min=10.9
+	endif
 endif
 
 POPCNT_CAPABILITY ?= 1
@@ -95,7 +98,7 @@ endif
 
 #default is to use Intel TBB
 ifneq (1,$(NO_TBB))
-	LIBS += $(PTHREAD_LIB) -ltbb -ltbbmalloc_proxy
+	LIBS += $(PTHREAD_LIB) -ltbb -ltbbmalloc$(if $(RELEASE_BUILD),,_proxy)
 	override EXTRA_FLAGS += -DWITH_TBB
 else
 	LIBS += $(PTHREAD_LIB)
@@ -433,9 +436,8 @@ bowtie2-src: $(SRC_PKG_LIST)
 	rm -rf .src.tmp
 
 .PHONY: bowtie2-pkg
-bowtie2-pkg: $(BIN_PKG_LIST) $(BOWTIE2_BIN_LIST) $(BOWTIE2_BIN_LIST_AUX)
-	$(eval HAS_TBB=$(shell strings bowtie2-align-l* | grep tbb))
-	$(eval PKG_DIR=bowtie2-$(VERSION)$(if $(HAS_TBB),,-legacy))
+bowtie2-pkg: static-libs $(BIN_PKG_LIST) $(BOWTIE2_BIN_LIST) $(BOWTIE2_BIN_LIST_AUX)
+	$(eval PKG_DIR=bowtie2-$(VERSION)-$(if $(MACOS),macos,$(if $(MINGW),mingw,linux))-x86_64)
 	chmod a+x scripts/*.sh scripts/*.pl
 	rm -rf .bin.tmp
 	mkdir -p .bin.tmp/$(PKG_DIR)
@@ -494,13 +496,30 @@ random-test: all perl-deps
 .PHONY: perl-deps
 perl-deps:
 	if [ ! -e .perllib.tmp ]; then \
-		DL=$$([ `which wget` ] && echo wget -O- || echo curl -L) ; \
+		DL=$$([ `which wget` ] && echo "wget --no-check-certificate -O-" || echo "curl -L") ; \
 		mkdir .perllib.tmp ; \
 		$$DL http://cpanmin.us | perl - -l $(CURDIR)/.perllib.tmp App::cpanminus local::lib ; \
 		eval `perl -I $(CURDIR)/.perllib.tmp/lib/perl5 -Mlocal::lib=$(CURDIR)/.perllib.tmp` ; \
 		cpanm --force Math::Random Clone Test::Deep Sys::Info ; \
 	fi
 
+static-libs:
+	if [[ ! -d $(CURDIR)/.lib || ! -d $(CURDIR)/.inc ]]; then \
+		mkdir $(CURDIR)/.lib $(CURDIR)/.include ; \
+	fi ; \
+	if [[ `uname` = "Darwin" ]]; then \
+		export CFLAGS=-mmacosx-version-min=10.9 ; \
+		export CXXFLAGS=-mmacosx-version-min=10.9 ; \
+	fi ; \
+	DL=$$([ `which wget` ] && echo "wget --no-check-certificate" || echo "curl -LO") ; \
+	cd /tmp ; \
+	$$DL https://zlib.net/zlib-1.2.11.tar.gz && tar xzf zlib-1.2.11.tar.gz && cd zlib-1.2.11 ; \
+	$(if $(MINGW), mingw32-make -f win32/Makefile.gcc, ./configure --static && make) && cp libz.a $(CURDIR)/.lib && cp zconf.h zlib.h $(CURDIR)/.include ; \
+	cd .. ; \
+	$$DL https://github.com/01org/tbb/archive/2017_U8.tar.gz && tar xzf 2017_U8.tar.gz && cd tbb-2017_U8; \
+	$(if $(MINGW), mingw32-make compiler=gcc arch=ia64 runtime=mingw, make) extra_inc=big_iron.inc -j4 \
+	&& cp -r include/tbb $(CURDIR)/.include && cp build/*_release/*.a $(CURDIR)/.lib
+
 .PHONY: test
 test: simple-test random-test
 
@@ -512,3 +531,4 @@ clean:
 	rm -f core.* .tmp.head
 	rm -rf *.dSYM
 	rm -rf .perllib.tmp
+	rm -rf .include .lib

NEWS

@@ -19,6 +19,12 @@ Please report any issues to the Bowtie 2 Github page or using the Sourceforge bu
 Version Release History
 =======================
 
+Version 2.3.3.1 - Oct 05, 2017
+    * Fixed an issue causing input files to be skipped when running
+    multi-threaded alignment
+    * Fixed an issue causing the first character of a read name to be
+    dropped while parsing reads split across multiple input files
+
 Version 2.3.3 - Sep 09, 2017
 From this release forward prepackaged bowtie2 binaries are now
 statically linked to the zlib compression library and, the recommended

VERSION

@@ -1 +1 @@
-2.3.3
+2.3.3.1

bt2_search.cpp

@@ -29,7 +29,11 @@
 #include <limits>
 #include <time.h>
 #include <dirent.h>
+
+#ifndef _WIN32
 #include <signal.h>
+#endif
+
 #include "alphabet.h"
 #include "assert_helpers.h"
 #include "endian_swap.h"
@@ -819,7 +823,7 @@ static void printUsage(ostream& out) {
 	    << "  --rg <text>        add <text> (\"lab:value\") to @RG line of SAM header." << endl
 	    << "                     Note: @RG line only printed when --rg-id is set." << endl
 	    << "  --omit-sec-seq     put '*' in SEQ and QUAL fields for secondary alignments." << endl
-	    << "  --sam-noqname-trunc Suppress standard behavior of truncating readname at first whitespace " << endl
+	    << "  --sam-no-qname-trunc Suppress standard behavior of truncating readname at first whitespace " << endl
 	    << "                      at the expense of generating non-standard SAM." << endl
 	    << "  --xeq              Use '='/'X', instead of 'M,' to specify matches/mismatches in SAM record." << endl
 	    << "  --soft-clipped-unmapped-tlen Exclude soft-clipped bases when reporting TLEN" << endl
@@ -4353,7 +4357,7 @@ static void multiseedSearchWorker_2p5(void *vp) {
 	return;
 }
 
-
+#ifndef _WIN32
 /**
  * Print friendly-ish message pertaining to failed system call.
  */
@@ -4524,7 +4528,7 @@ static void thread_monitor(int pid, int orig_threads, EList<int>& tids, EList<T*
 		}
 	}
 }
-
+#endif
 /**
  * Called once per alignment job.  Sets up global pointers to the
  * shared global data structures, creates per-thread structures, then
@@ -4564,10 +4568,12 @@ static void multiseedSearch(
 	delete _t;
 	if(!refs->loaded()) throw 1;
 	multiseed_refs = refs.get();
-    sigset_t set;
-    sigemptyset(&set);
-    sigaddset(&set, SIGPIPE);
-    pthread_sigmask(SIG_BLOCK, &set, NULL);
+#ifndef _WIN32
+	sigset_t set;
+	sigemptyset(&set);
+	sigaddset(&set, SIGPIPE);
+	pthread_sigmask(SIG_BLOCK, &set, NULL);
+#endif
 	EList<int> tids;
 #ifdef WITH_TBB
 	//tbb::task_group tbb_grp;
@@ -4614,12 +4620,14 @@ static void multiseedSearch(
 	{
 		Timer _t(cerr, "Multiseed full-index search: ", timing);
 
+#ifndef _WIN32
 		int pid = 0;
 		if(thread_stealing) {
 			pid = getpid();
 			write_pid(thread_stealing_dir.c_str(), pid);
 			thread_counter = 0;
 		}
+#endif
 
 		for(int i = 0; i < nthreads; i++) {
 #ifdef WITH_TBB
@@ -4644,10 +4652,12 @@ static void multiseedSearch(
 #endif
 		}
 
+#ifndef _WIN32
 		if(thread_stealing) {
 			int orig_threads = nthreads;
 			thread_monitor(pid, orig_threads, tids, threads);
 		}
+#endif
 
 #ifdef WITH_TBB
 		while(all_threads_done < nthreads) {
@@ -4659,9 +4669,11 @@ static void multiseedSearch(
 		}
 #endif
 
+#ifndef _WIN32
 		if(thread_stealing) {
 			del_pid(thread_stealing_dir.c_str(), pid);
 		}
+#endif
 	}
 	if(!metricsPerRead && (metricsOfb != NULL || metricsStderr)) {
 		metrics.reportInterval(metricsOfb, metricsStderr, true, NULL);
@@ -5012,7 +5024,9 @@ int bowtie(int argc, const char **argv) {
 				return 1;
 			}
 
+#ifndef _WIN32
 			thread_stealing = thread_ceiling > nthreads;
+#endif
 			if(thread_stealing && thread_stealing_dir.empty()) {
 				cerr << "When --thread-ceiling is specified, must also specify --thread-piddir" << endl;
 				printUsage(cerr);

doc/manual.html

@@ -1058,7 +1058,7 @@ <h2 id="sam-output">SAM output</h2>
 <p>Each subsequent line describes an alignment or, if the read failed to align, a read. Each line is a collection of at least 12 fields separated by tabs; from left to right, the fields are:</p>
 <ol style="list-style-type: decimal">
 <li><p>Name of read that aligned.</p>
-<p>Note that the <a href="http://samtools.sourceforge.net/SAM1.pdf">SAM specification</a> disallows whitespace in the read name. If the read name contains any whitespace characters, Bowtie 2 will truncate the name at the first whitespace character. This is similar to the behavior of other tools. The standard behavior of truncating at the first whitespace can be suppressed with <code>--sam-noqname-trunc</code> at the expense of generating non-standard SAM.</p></li>
+<p>Note that the <a href="http://samtools.sourceforge.net/SAM1.pdf">SAM specification</a> disallows whitespace in the read name. If the read name contains any whitespace characters, Bowtie 2 will truncate the name at the first whitespace character. This is similar to the behavior of other tools. The standard behavior of truncating at the first whitespace can be suppressed with <code>--sam-no-qname-trunc</code> at the expense of generating non-standard SAM.</p></li>
 <li><p>Sum of all applicable flags. Flags relevant to Bowtie are:</p>
 <table>
 <tr>

doc/website/manual.ssi

@@ -1058,7 +1058,7 @@ Seed 4 rc:                   TTATGCATGA</code></pre>
 <p>Each subsequent line describes an alignment or, if the read failed to align, a read. Each line is a collection of at least 12 fields separated by tabs; from left to right, the fields are:</p>
 <ol style="list-style-type: decimal">
 <li><p>Name of read that aligned.</p>
-<p>Note that the <a href="http://samtools.sourceforge.net/SAM1.pdf">SAM specification</a> disallows whitespace in the read name. If the read name contains any whitespace characters, Bowtie 2 will truncate the name at the first whitespace character. This is similar to the behavior of other tools. The standard behavior of truncating at the first whitespace can be suppressed with <code>--sam-noqname-trunc</code> at the expense of generating non-standard SAM.</p></li>
+<p>Note that the <a href="http://samtools.sourceforge.net/SAM1.pdf">SAM specification</a> disallows whitespace in the read name. If the read name contains any whitespace characters, Bowtie 2 will truncate the name at the first whitespace character. This is similar to the behavior of other tools. The standard behavior of truncating at the first whitespace can be suppressed with <code>--sam-no-qname-trunc</code> at the expense of generating non-standard SAM.</p></li>
 <li><p>Sum of all applicable flags. Flags relevant to Bowtie are:</p>
 <table>
 <tr>

doc/website/old_news.ssi

@@ -1,3 +1,52 @@
+<h2>Bowtie2 developers note</h2>
+<p>As of Nov 2015 we had to fix the bowtie2 github repo and relabel the entire history. Developers and contributors should re-clone the bowtie2 github repo from this current state. </p>
+<h2>Version 2.2.9 - Apr 22, 2016</h2>
+<ul>
+   <li>Fixed the multiple threads issue for the bowtie2-build.</li>
+   <li>Fixed a TBB related build issue impacting TBB v4.4.</li>
+</ul>
+<h2>Version 2.2.8 - Mar 10, 2016</h2>
+<ul>
+   <li>Various website updates.</li>
+   <li>Fixed the bowtie2-build issue that made TBB compilation fail.</li>
+   <li>Fixed the static build for Win32 platform.</li>
+</ul>
+<h2>Version 2.2.7 - Feb 10, 2016</h2>
+<ul>
+   <li>Added a parallel index build option: bowtie2-build --threads &lt;# threads&gt;.</li>
+   <li>Fixed an issue whereby IUPAC codes (other than A/C/G/T/N) in reads were converted to As. Now all non-A/C/G/T characters in reads become Ns.</li>
+   <li>Fixed some compilation issues, including for the Intel C++ Compiler.</li>
+   <li>Removed debugging code that could impede performance for many alignment threads.</li>
+   <li>Fixed a few typos in documentation.</li>
+</ul>
+<h2>Version 2.2.6 - Jul 22, 2015</h2>
+<ul>
+   <li>Switched to a stable sort to avoid some potential reproducibility confusions.</li>
+   <li>Added <tt>'install'</tt> target for *nix platforms.</li>
+   <li>Added the Intel TBB option which provides in most situations a better performance output. TBB is not present by default in the current build but can be added by compiling the source code with <tt>WITH_TBB=1</tt> option.</li>
+   <li>Fixed a bug that caused seed lenght to be dependent of the <tt><a href="manual.shtml#bowtie2-options-L">-L</a></tt> and <tt><a href="manual.shtml#bowtie2-options-N">-N</a></tt> parameters order.</li>
+   <li>Fixed a bug that caused <tt><a href="manual.shtml#bowtie2-options-local">--local</a></tt> followed by <tt><a href="manual.shtml#bowtie2-options-N">-N</a></tt> to reset seed lenght to 22 which is actually the default value for global.</li>
+   <li>Enable compilation on FreeBDS and clang, although gmake port is still required.</li>
+   <li>Fixed an issue that made bowtie2 compilation process to fail on Snow Leopard.</li>
+</ul>
+
+<h2>Version 2.2.5 - Mar 9, 2015</h2>
+<ul>
+   <li>Fixed some situations where incorrectly we could detect a Mavericks platform.</li>
+   <li>Fixed some manual issues including some HTML bad formating.</li>
+   <li>Make sure the wrapper correctly identifies the platform under OSX.</li>
+   <li>Fixed <tt><a href="manual.shtml#bowtie2-options-rg">--rg</a></tt>/<tt><a href="manual.shtml#bowtie2-options-rg-id">--rg-id</a></tt> options where included spaces were incorrectly treated.</li>
+   <li>Various documentation fixes added by contributors.</li>
+   <li>Fixed the incorrect behavior where parameter file names may contain spaces.</li>
+   <li>Fixed bugs related with the presence of spaces in the path where bowtie binaries are stored.</li>
+   <li>Improved exception handling for missformated quality values.</li>
+   <li>Improved redundancy checks by correctly account for soft clipping.</li>
+</ul>
+
+<h2>Lighter released</h2>
+<ul>
+   <li>Lighter is an extremely fast and memory-efficient program for correcting sequencing errors in DNA sequencing data.  For details on how error correction can help improve the speed and accuracy of downstream analysis tools, see the <a href="http://genomebiology.com/2014/15/11/509">paper in Genome Biology</a>.  Source and software <a href="https://github.com/mourisl/Lighter">available at GitHub</a></li>.
+</ul>
 <h2>Version 2.2.4 - Oct 22, 2014</h2>
 <ul>
    <li>Fixed a Mavericks OSX specific bug caused by some linkage ambiguities.</li>