[med-svn] [bowtie2] 02/10: New upstream version 2.3.0
Andreas Tille
tille at debian.org
Mon Jan 9 12:30:41 UTC 2017
This is an automated email from the git hooks/post-receive script.
tille pushed a commit to branch master
in repository bowtie2.
commit 7b7c16b4fb27697c91f6a582084e35d35b71799f
Author: Andreas Tille <tille at debian.org>
Date: Mon Jan 9 12:53:15 2017 +0100
New upstream version 2.3.0
---
.gitignore | 26 +
.travis.yml | 20 +
MANUAL | 25 +-
MANUAL.markdown | 26 +-
Makefile | 87 +-
NEWS | 23 +-
README | 8 +
VERSION | 2 +-
aligner_result.cpp | 2 +-
aligner_result.h | 2 +-
aligner_seed.cpp | 16 +-
aligner_seed2.cpp | 19 +-
aligner_seed2.h | 2 -
aligner_sw_common.h | 13 +-
aligner_swsse_ee_i16.cpp | 1 -
aligner_swsse_ee_u8.cpp | 1 -
aligner_swsse_loc_u8.cpp | 1 -
aln_sink.h | 8 -
alphabet.cpp | 199 -
alphabet.h | 72 -
bt2_build.cpp | 14 +-
bt2_inspect.cpp | 6 +-
bt2_search.cpp | 207 +-
cpu_numa_info.cpp | 9 +
cpu_numa_info.h | 6 +
diff_sample.h | 2 +-
doc/images/bowtie_logo.png | Bin 0 -> 16419 bytes
doc/images/osi-certified.gif | Bin 0 -> 2712 bytes
doc/images/sflogo.png | Bin 0 -> 1177 bytes
doc/images/university.small.horizontal.white.png | Bin 0 -> 35224 bytes
doc/manual.html | 1417 +++---
doc/release.txt | 153 +
doc/website/faq.shtml | 34 +
doc/website/faq.ssi | 68 +
doc/website/foot.ssi | 12 +
doc/website/index.html | 9 +
doc/website/index.shtml | 60 +
doc/website/manual.shtml | 33 +
doc/{manual.html => website/manual.ssi} | 36 +-
doc/website/news.shtml | 35 +
doc/website/old_news.ssi | 359 ++
doc/website/other_tools.shtml | 33 +
doc/website/push.sh | 17 +
doc/website/push_images.sh | 16 +
doc/website/recent_news.ssi | 104 +
doc/website/rhsidebar.ssi | 196 +
doc/website/top.ssi | 16 +
dp_framer.cpp | 3 +-
dp_framer.h | 3 +-
filebuf.h | 1 +
opts.h | 2 +-
pat.cpp | 2030 ++++-----
pat.h | 1852 +++-----
pthreadGC2.dll | Bin 0 -> 60273 bytes
read.h | 102 +-
read_qseq.cpp | 383 +-
ref_read.cpp | 34 +-
sam.cpp | 120 +-
sam.h | 8 -
scoring.cpp | 1 -
scoring.h | 2 +-
scripts/sa.py | 79 +
scripts/sim/AlignmentCheck.pm | 859 ++++
scripts/sim/DNA.pm | 287 ++
scripts/sim/Mutate.pm | 301 ++
scripts/sim/RandDNA.pm | 191 +
scripts/sim/SampleRead.pm | 244 ++
scripts/sim/Sim.pm | 1040 +++++
scripts/sim/Test.pm | 47 +
scripts/sim/contrib/ForkManager.pm | 412 ++
scripts/sim/run.pl | 133 +
scripts/sim/run.sh | 37 +
scripts/sim/unit.sh | 23 +
scripts/test/.gitignore | 1 +
scripts/test/DNA.pm | 127 +
scripts/test/README.md | 44 +
scripts/test/benchmark/benchmarks.py | 275 ++
scripts/test/benchmark/data/conf/acc.json | 44 +
scripts/test/benchmark/data/conf/seta.bench | 59 +
scripts/test/benchmark/data/conf/speed.json | 44 +
scripts/test/benchmark/run.py | 128 +
scripts/test/benchmark/samreader.py | 128 +
scripts/test/big_data/reads/human_reads.fa | 8 +
scripts/test/big_data/reads/mouse_reads.fa | 8 +
scripts/test/bt2face.py | 39 +
scripts/test/btdata.py | 199 +
scripts/test/dataface.py | 81 +
scripts/test/large_idx.py | 110 +
scripts/test/regressions.py | 207 +
scripts/test/simple_tests.pl | 5070 ++++++++++++++++++++++
scripts/test/simple_tests.sh | 34 +
search_globals.h | 5 -
simple_func.h | 4 +
sstring.cpp | 4 +-
sstring.h | 142 +-
threading.h | 46 +-
96 files changed, 14429 insertions(+), 3967 deletions(-)
diff --git a/.gitignore b/.gitignore
new file mode 100644
index 0000000..f5e3327
--- /dev/null
+++ b/.gitignore
@@ -0,0 +1,26 @@
+.project
+bowtie2-align
+bowtie2-build
+bowtie2-inspect
+*.zip
+*.bt2
+*.bt2l
+*.dSYM
+*.pyc
+*.swp
+.run.pl.child.*
+.simple_tests*
+.idea
+bowtie2-build-s
+bowtie2-build-l
+bowtie2-align-s
+bowtie2-align-l
+bowtie2-inspect-s
+bowtie2-inspect-l
+bowtie2-build-s-debug
+bowtie2-build-l-debug
+bowtie2-align-s-debug
+bowtie2-align-l-debug
+bowtie2-inspect-s-debug
+bowtie2-inspect-l-debug
+Xcode
diff --git a/.travis.yml b/.travis.yml
new file mode 100644
index 0000000..e855a60
--- /dev/null
+++ b/.travis.yml
@@ -0,0 +1,20 @@
+language: c++
+compiler:
+- clang
+- gcc
+cache: apt
+addons:
+ apt:
+ packages:
+ - libtbb-dev
+branches:
+ only:
+ - master
+ - cleaner_parsing
+ - batch_parsing
+ - no-io
+ - rel2.3
+script: make allall && make simple-test
+notifications:
+ slack:
+ secure: tfzT8N1fNV+oSV7tide9WrAj3ifs+LONJ3fCH1tUzexqrx23te4lE0oAj9C1cEMJ4evyRYwHNG8HZoLCOy8EfapqbWm6vgLIlkIBpeZ9E6f2jG6v0YuVDWWpqQC3qdGXCqWtHPjgs3i5OLsLwwQ/LItLoTqpBk2aYv+vGNs2F9g=
diff --git a/MANUAL b/MANUAL
index 28ae790..359bb68 100644
--- a/MANUAL
+++ b/MANUAL
@@ -155,15 +155,18 @@ Bowtie 2 tools by running GNU `make` (usually with the command `make`, but
sometimes with `gmake`) with no arguments. If building with MinGW, run `make`
from the MSYS environment.
-Bowtie 2 is using the multithreading software model in order to speed up
-execution times on SMP architectures where this is possible. On POSIX
-platforms (like linux, Mac OS, etc) it needs the pthread library. Although
-it is possible to use pthread library on non-POSIX platform like Windows, due
-to performance reasons bowtie 2 will try to use Windows native multithreading
-if possible. We recommend that you first install the [Threading Building Blocks library],
-also known as TBB, and then build using `make WITH_TBB=1`. TBB comes installed
-by default on many popular linux distros. If TBB is not available, then simply omit
-the `WITH_TBB=1` option.
++Bowtie 2 is using the multithreading software model in order to
++speed up execution times on SMP architectures where this is possible.
++The Threading Building Blocks library, TBB, is now the default
++threading library in bowtie2. On POSIX platforms (like linux, Mac
++OS, etc) if TBB is not available the pthread library will be used.
++Although it is possible to use pthread library on Windows, a non-POSIX
++platform, due to performance reasons bowtie 2 will try to use Windows
++native multithreading if possible. We recommend that you first
++install the [Threading Building Blocks library], but if unable to
++do so please specify `make NO_TBB=1`. TBB comes installed by default
++on many popular linux distros. Please note, packages built without
++TBB will have _-legacy_ appended to the name.
[MinGW]: http://www.mingw.org/
[MSYS]: http://www.mingw.org/wiki/msys
@@ -1713,14 +1716,14 @@ automatically by default; use `-a`/`--noauto` to configure manually.
The maximum number of suffixes allowed in a block. Allowing more suffixes per
block makes indexing faster, but increases peak memory usage. Setting this
option overrides any previous setting for `--bmax`, or `--bmaxdivn`.
-Default (in terms of the `--bmaxdivn` parameter) is `--bmaxdivn` 4. This is
+Default (in terms of the `--bmaxdivn` parameter) is `--bmaxdivn` 4 * number of threads. This is
configured automatically by default; use `-a`/`--noauto` to configure manually.
--bmaxdivn <int>
The maximum number of suffixes allowed in a block, expressed as a fraction of
the length of the reference. Setting this option overrides any previous setting
-for `--bmax`, or `--bmaxdivn`. Default: `--bmaxdivn` 4. This is
+for `--bmax`, or `--bmaxdivn`. Default: `--bmaxdivn` 4 * number of threads. This is
configured automatically by default; use `-a`/`--noauto` to configure manually.
--dcv <int>
diff --git a/MANUAL.markdown b/MANUAL.markdown
index 1f0d29f..2ec2828 100644
--- a/MANUAL.markdown
+++ b/MANUAL.markdown
@@ -165,16 +165,18 @@ Bowtie 2 tools by running GNU `make` (usually with the command `make`, but
sometimes with `gmake`) with no arguments. If building with MinGW, run `make`
from the MSYS environment.
-Bowtie 2 is using the multithreading software model in order to speed up
-execution times on SMP architectures where this is possible. On POSIX
-platforms (like linux, Mac OS, etc) it needs the pthread library. Although
-it is possible to use pthread library on non-POSIX platform like Windows, due
-to performance reasons bowtie 2 will try to use Windows native multithreading
-if possible. We recommend that you first install the [Threading Building Blocks library],
-also known as TBB, and then build using `make WITH_TBB=1`. TBB comes installed
-by default on many popular linux distros. If TBB is not available, then simply omit
-the `WITH_TBB=1` option.
-
++Bowtie 2 is using the multithreading software model in order to
++speed up execution times on SMP architectures where this is possible.
++The Threading Building Blocks library, TBB, is now the default
++threading library in bowtie2. On POSIX platforms (like linux, Mac
++OS, etc) if TBB is not available the pthread library will be used.
++Although it is possible to use pthread library on Windows, a non-POSIX
++platform, due to performance reasons bowtie 2 will try to use Windows
++native multithreading if possible. We recommend that you first
++install the [Threading Building Blocks library], but if unable to
++do so please specify `make NO_TBB=1`. TBB comes installed by default
++on many popular linux distros. Please note, packages built without
++TBB will have _-legacy_ appended to the name.
[MinGW]: http://www.mingw.org/
[MSYS]: http://www.mingw.org/wiki/msys
@@ -2502,7 +2504,7 @@ automatically by default; use [`-a`/`--noauto`] to configure manually.
The maximum number of suffixes allowed in a block. Allowing more suffixes per
block makes indexing faster, but increases peak memory usage. Setting this
option overrides any previous setting for [`--bmax`], or [`--bmaxdivn`].
-Default (in terms of the [`--bmaxdivn`] parameter) is [`--bmaxdivn`] 4. This is
+Default (in terms of the [`--bmaxdivn`] parameter) is [`--bmaxdivn`] 4 * number of threads. This is
configured automatically by default; use [`-a`/`--noauto`] to configure manually.
</td></tr><tr><td id="bowtie2-build-options-bmaxdivn">
@@ -2515,7 +2517,7 @@ configured automatically by default; use [`-a`/`--noauto`] to configure manually
The maximum number of suffixes allowed in a block, expressed as a fraction of
the length of the reference. Setting this option overrides any previous setting
-for [`--bmax`], or [`--bmaxdivn`]. Default: [`--bmaxdivn`] 4. This is
+for [`--bmax`], or [`--bmaxdivn`]. Default: [`--bmaxdivn`] 4 * number of threads. This is
configured automatically by default; use [`-a`/`--noauto`] to configure manually.
</td></tr><tr><td id="bowtie2-build-options-dcv">
diff --git a/Makefile b/Makefile
index 667eda1..f2aa557 100644
--- a/Makefile
+++ b/Makefile
@@ -43,6 +43,7 @@ ifneq (,$(findstring MINGW,$(shell uname)))
# POSIX memory-mapped files not currently supported on Windows
BOWTIE_MM = 0
BOWTIE_SHARED_MEM = 0
+ override EXTRA_FLAGS += -ansi
endif
MACOS = 0
@@ -61,23 +62,28 @@ ifeq (1, $(POPCNT_CAPABILITY))
INC += -I third_party
endif
-MM_DEF =
+MM_DEF =
ifeq (1,$(BOWTIE_MM))
MM_DEF = -DBOWTIE_MM
endif
-SHMEM_DEF =
+SHMEM_DEF =
ifeq (1,$(BOWTIE_SHARED_MEM))
SHMEM_DEF = -DBOWTIE_SHARED_MEM
endif
PTHREAD_PKG =
-PTHREAD_LIB =
+PTHREAD_LIB =
+
+#if we're not using TBB, then we can't use queuing locks
+ifeq (1,$(NO_TBB))
+ NO_QUEUELOCK=1
+endif
ifeq (1,$(MINGW))
- PTHREAD_LIB =
+ PTHREAD_LIB =
else
PTHREAD_LIB = -lpthread
endif
@@ -86,19 +92,20 @@ ifeq (1,$(NO_SPINLOCK))
override EXTRA_FLAGS += -DNO_SPINLOCK
endif
-ifeq (1,$(WITH_TBB))
+#default is to use Intel TBB
+ifneq (1,$(NO_TBB))
LIBS = $(PTHREAD_LIB) -ltbb -ltbbmalloc_proxy
override EXTRA_FLAGS += -DWITH_TBB
else
LIBS = $(PTHREAD_LIB)
endif
-SEARCH_LIBS =
-BUILD_LIBS =
+SEARCH_LIBS =
+BUILD_LIBS =
INSPECT_LIBS =
ifeq (1,$(MINGW))
- BUILD_LIBS =
- INSPECT_LIBS =
+ BUILD_LIBS =
+ INSPECT_LIBS =
endif
ifeq (1,$(WITH_THREAD_PROFILING))
@@ -109,11 +116,19 @@ ifeq (1,$(WITH_AFFINITY))
override EXTRA_FLAGS += -DWITH_AFFINITY=1
endif
+#default is to use Intel TBB's queuing lock for better thread scaling performance
+ifneq (1,$(NO_QUEUELOCK))
+ override EXTRA_FLAGS += -DNO_SPINLOCK
+ override EXTRA_FLAGS += -DWITH_QUEUELOCK=1
+endif
+
+
SHARED_CPPS = ccnt_lut.cpp ref_read.cpp alphabet.cpp shmem.cpp \
edit.cpp bt2_idx.cpp bt2_io.cpp bt2_util.cpp \
reference.cpp ds.cpp multikey_qsort.cpp limit.cpp \
random_source.cpp
-ifneq (1,$(WITH_TBB))
+
+ifeq (1,$(NO_TBB))
SHARED_CPPS += tinythread.cpp
endif
@@ -135,6 +150,7 @@ SEARCH_CPPS = qual.cpp pat.cpp sam.cpp \
aligner_swsse_loc_u8.cpp \
aligner_swsse_ee_u8.cpp \
aligner_driver.cpp
+
SEARCH_CPPS_MAIN = $(SEARCH_CPPS) bowtie_main.cpp
DP_CPPS = qual.cpp aligner_sw.cpp aligner_result.cpp ref_coord.cpp mask.cpp \
@@ -165,7 +181,7 @@ ifeq (32,$(BITS))
$(error bowtie2 compilation requires a 64-bit platform )
endif
-SSE_FLAG=-msse2
+SSE_FLAG=-msse2
DEBUG_FLAGS = -O0 -g3 -m64 $(SSE_FLAG)
DEBUG_DEFS = -DCOMPILER_OPTIONS="\"$(DEBUG_FLAGS) $(EXTRA_FLAGS)\""
@@ -209,8 +225,8 @@ GENERAL_LIST = $(wildcard scripts/*.sh) \
VERSION
ifeq (1,$(WINDOWS))
- BOWTIE2_BIN_LIST := $(BOWTIE2_BIN_LIST) bowtie2.bat bowtie2-build.bat bowtie2-inspect.bat
- ifeq (1,$(WITH_TBB))
+ BOWTIE2_BIN_LIST := $(BOWTIE2_BIN_LIST) bowtie2.bat bowtie2-build.bat bowtie2-inspect.bat
+ ifneq (1,$(NO_TBB))
override EXTRA_FLAGS += -static-libgcc -static-libstdc++
else
override EXTRA_FLAGS += -static -static-libgcc -static-libstdc++
@@ -231,7 +247,7 @@ SRC_PKG_LIST = $(wildcard *.h) \
$(GENERAL_LIST)
ifeq (1,$(WINDOWS))
- BIN_PKG_LIST = $(GENERAL_LIST) bowtie2.bat bowtie2-build.bat bowtie2-inspect.bat
+ BIN_PKG_LIST = $(GENERAL_LIST) bowtie2.bat bowtie2-build.bat bowtie2-inspect.bat
else
BIN_PKG_LIST = $(GENERAL_LIST)
endif
@@ -352,7 +368,7 @@ bowtie2-inspect-l: bt2_inspect.cpp $(HEADERS) $(SHARED_CPPS)
$(SHARED_CPPS) \
$(LIBS) $(INSPECT_LIBS)
-bowtie2-inspect-s-debug: bt2_inspect.cpp $(HEADERS) $(SHARED_CPPS)
+bowtie2-inspect-s-debug: bt2_inspect.cpp $(HEADERS) $(SHARED_CPPS)
$(CXX) $(DEBUG_FLAGS) \
$(DEBUG_DEFS) $(EXTRA_FLAGS) \
$(DEFS) -DBOWTIE2 -DBOWTIE_INSPECT_MAIN -Wall \
@@ -416,21 +432,22 @@ bowtie2-src: $(SRC_PKG_LIST)
cp .src.tmp/bowtie2-$(VERSION)-source.zip .
rm -rf .src.tmp
-.PHONY: bowtie2-bin
-bowtie2-bin: $(BIN_PKG_LIST) $(BOWTIE2_BIN_LIST) $(BOWTIE2_BIN_LIST_AUX)
+.PHONY: bowtie2-pkg
+bowtie2-pkg: $(BIN_PKG_LIST) $(BOWTIE2_BIN_LIST) $(BOWTIE2_BIN_LIST_AUX)
+ $(eval HAS_TBB=$(shell strings bowtie2-align-l* | grep tbb))
+ $(eval PKG_DIR=bowtie2-$(VERSION)$(if $(HAS_TBB),,-legacy))
chmod a+x scripts/*.sh scripts/*.pl
rm -rf .bin.tmp
- mkdir .bin.tmp
- mkdir .bin.tmp/bowtie2-$(VERSION)
+ mkdir -p .bin.tmp/$(PKG_DIR)
if [ -f bowtie2-align-s.exe ] ; then \
zip tmp.zip $(BIN_PKG_LIST) $(addsuffix .exe,$(BOWTIE2_BIN_LIST) $(BOWTIE2_BIN_LIST_AUX)) ; \
else \
zip tmp.zip $(BIN_PKG_LIST) $(BOWTIE2_BIN_LIST) $(BOWTIE2_BIN_LIST_AUX) ; \
fi
- mv tmp.zip .bin.tmp/bowtie2-$(VERSION)
- cd .bin.tmp/bowtie2-$(VERSION) ; unzip tmp.zip ; rm -f tmp.zip
- cd .bin.tmp ; zip -r bowtie2-$(VERSION).zip bowtie2-$(VERSION)
- cp .bin.tmp/bowtie2-$(VERSION).zip .
+ mv tmp.zip .bin.tmp/$(PKG_DIR)
+ cd .bin.tmp/$(PKG_DIR) ; unzip tmp.zip ; rm -f tmp.zip
+ cd .bin.tmp ; zip -r $(PKG_DIR).zip $(PKG_DIR)
+ cp .bin.tmp/$(PKG_DIR).zip .
rm -rf .bin.tmp
bowtie2-seeds-debug: aligner_seed.cpp ccnt_lut.cpp alphabet.cpp aligner_seed.h bt2_idx.cpp bt2_io.cpp
@@ -464,6 +481,29 @@ install: all
cp -f $$file $(DESTDIR)$(bindir) ; \
done
+.PHONY: simple-test
+simple-test: all perl-deps
+ eval `perl -I $(CURDIR)/.perllib.tmp/lib/perl5 -Mlocal::lib=$(CURDIR)/.perllib.tmp` ; \
+ sh ./scripts/test/simple_tests.sh
+
+.PHONY: random-test
+random-test: all perl-deps
+ eval `perl -I $(CURDIR)/.perllib.tmp/lib/perl5 -Mlocal::lib=$(CURDIR)/.perllib.tmp` ; \
+ sh ./scripts/sim/run.sh $(if $(NUM_CORES), $(NUM_CORES), 2)
+
+.PHONY: perl-deps
+perl-deps:
+ if [ ! -e .perllib.tmp ]; then \
+ DL=$$([ `which wget` ] && echo wget -O- || echo curl -L) ; \
+ mkdir .perllib.tmp ; \
+ $$DL http://cpanmin.us | perl - -l $(CURDIR)/.perllib.tmp App::cpanminus local::lib ; \
+ eval `perl -I $(CURDIR)/.perllib.tmp/lib/perl5 -Mlocal::lib=$(CURDIR)/.perllib.tmp` ; \
+ cpanm Math::Random Clone Test::Deep Sys::Info ; \
+ fi
+
+.PHONY: test
+test: simple-test random-test
+
.PHONY: clean
clean:
rm -f $(BOWTIE2_BIN_LIST) $(BOWTIE2_BIN_LIST_AUX) \
@@ -471,3 +511,4 @@ clean:
bowtie2-src.zip bowtie2-bin.zip
rm -f core.* .tmp.head
rm -rf *.dSYM
+ rm -rf .perllib.tmp
diff --git a/NEWS b/NEWS
index a25813b..c52d521 100644
--- a/NEWS
+++ b/NEWS
@@ -3,7 +3,7 @@ Bowtie 2 NEWS
Bowtie 2 is now available for download from the project website,
http://bowtie-bio.sf.net/bowtie2. 2.0.0-beta1 is the first version released to
-the public and 2.2.5 is the latest version. Bowtie 2 is licensed under
+the public and 2.3.0 is the latest version. Bowtie 2 is licensed under
the GPLv3 license. See `LICENSE' file for details.
Reporting Issues
@@ -16,6 +16,27 @@ Please report any issues using the Sourceforge bug tracker:
Version Release History
=======================
+Version 2.3.0 - Dec 13, 2016
+ * Code related to read parsing was completely rewritten to improve
+ scalability to many threads. In short, the critical section is
+ simpler and parses input reads in batches rather than one at a time.
+ The improvement applies to all read formats.
+ * TBB is now the default threading library. We consistently found TBB to give
+ superior thread scaling. It is widely available and widely installed.
+ That said, we are also preserving a "legacy" version of Bowtie that,
+ like previous releases, does not use TBB. To compile Bowtie source
+ in legacy mode use `NO_TBB=1`. To
+ use legacy binaries, download the appropriate binary archive with
+ "legacy" in the name.
+ * Bowtie now uses a queue-based lock rather
+ than a spin or heavyweight lock. We find this gives superior thread
+ scaling; we saw an order-of-magnitude throughput improvements at
+ 120 threads in one experiment, for example.
+ * Unnecessary thread synchronization removed
+ * Fixed issue with parsing FASTA records with greater-than symbol in the name
+ * Now detects and reports inconsistencies between `--score-min` and `--ma`
+ * Changed default for `--bmaxdivn` to yield better memory footprint and running time when building an index with many threads
+
Version 2.2.9 - Apr 22, 2016
* Fixed the multiple threads issue for the bowtie2-build.
* Fixed a TBB related build issue impacting TBB v4.4.
diff --git a/README b/README
new file mode 100644
index 0000000..37c370a
--- /dev/null
+++ b/README
@@ -0,0 +1,8 @@
+Bowtie 2: http://bowtie-bio.sf.net/bowtie2
+
+ - See AUTHORS for information about who wrote Bowtie 2 and its various
+ components.
+ - See LICENSE for license information.
+ - See MANUAL for detailed information on how to use Bowtie 2.
+ - See NEWS for information about changes in this and previous versions
+ of Bowtie 2.
diff --git a/VERSION b/VERSION
index a6333e4..276cbf9 100644
--- a/VERSION
+++ b/VERSION
@@ -1 +1 @@
-2.2.9
+2.3.0
diff --git a/aligner_result.cpp b/aligner_result.cpp
index 5a2b87e..2e8ebba 100644
--- a/aligner_result.cpp
+++ b/aligner_result.cpp
@@ -474,7 +474,7 @@ bool AlnRes::matchesRef(
rdseq.clear();
rdseq = rd.patFw;
if(!fw) {
- rdseq.reverseComp(false);
+ rdseq.reverseComp();
}
assert_eq(rdrows_, rdseq.length());
// rdseq is the nucleotide sequence from upstream to downstream on the
diff --git a/aligner_result.h b/aligner_result.h
index 38fd4e4..203999e 100644
--- a/aligner_result.h
+++ b/aligner_result.h
@@ -904,7 +904,7 @@ public:
trim_st += (fw() ? pretrim5p_ : pretrim3p_);
trim_en += (fw() ? pretrim3p_ : pretrim5p_);
st.adjustOff(-trim_st);
- en.adjustOff( trim_st);
+ en.adjustOff( trim_en);
}
/**
diff --git a/aligner_seed.cpp b/aligner_seed.cpp
index cbd167b..7050a5a 100644
--- a/aligner_seed.cpp
+++ b/aligner_seed.cpp
@@ -359,7 +359,7 @@ SeedAligner::instantiateSeq(
// If fw is false, we take characters starting at the 3' end of the
// reverse complement of the read.
for(int i = 0; i < len; i++) {
- seq.set(read.patFw.windowGetDna(i, fw, read.color, depth, len), i);
+ seq.set(read.patFw.windowGetDna(i, fw, depth, len), i);
qual.set(read.qual.windowGet(i, fw, depth, len), i);
}
}
@@ -1686,7 +1686,6 @@ enum {
static const char *short_opts = "vCt";
static struct option long_opts[] = {
{(char*)"verbose", no_argument, 0, 'v'},
- {(char*)"color", no_argument, 0, 'C'},
{(char*)"timing", no_argument, 0, 't'},
{(char*)"nofw", no_argument, 0, ARG_NOFW},
{(char*)"norc", no_argument, 0, ARG_NORC},
@@ -1705,16 +1704,13 @@ static void printUsage(ostream& os) {
os << " --nofw don't align forward-oriented read" << endl;
os << " --norc don't align reverse-complemented read" << endl;
os << " -t/--timing show timing information" << endl;
- os << " -C/--color colorspace mode" << endl;
os << " -v/--verbose talkative mode" << endl;
}
bool gNorc = false;
bool gNofw = false;
-bool gColor = false;
int gVerbose = 0;
int gGapBarrier = 1;
-bool gColorExEnds = true;
int gSnpPhred = 30;
bool gReportOverhangs = true;
@@ -1723,7 +1719,6 @@ extern void aligner_random_seed_tests(
int num_tests,
TIndexOffU qslo,
TIndexOffU qshi,
- bool color,
uint32_t seed);
/**
@@ -1744,7 +1739,6 @@ int main(int argc, char **argv) {
argc, argv, short_opts, long_opts, &option_index);
switch (next_option) {
case 'v': gVerbose = true; break;
- case 'C': gColor = true; break;
case 't': timing = true; break;
case ARG_NOFW: gNofw = true; break;
case ARG_NORC: gNorc = true; break;
@@ -1757,7 +1751,6 @@ int main(int argc, char **argv) {
100, // num references
100, // queries per reference lo
400, // queries per reference hi
- false, // true -> generate colorspace reference/reads
18); // pseudo-random seed
return 0;
}
@@ -1767,7 +1760,6 @@ int main(int argc, char **argv) {
100, // num references
100, // queries per reference lo
400, // queries per reference hi
- false, // true -> generate colorspace reference/reads
seed); // pseudo-random seed
return 0;
}
@@ -1792,7 +1784,7 @@ int main(int argc, char **argv) {
string ebwtBase(reffn);
BitPairReference ref(
ebwtBase, // base path
- gColor, // whether we expect it to be colorspace
+ false, // whether we expect it to be colorspace
sanity, // whether to sanity-check reference as it's loaded
NULL, // fasta files to sanity check reference against
NULL, // another way of specifying original sequences
@@ -1805,7 +1797,7 @@ int main(int argc, char **argv) {
Timer *t = new Timer(cerr, "Time loading fw index: ", timing);
Ebwt ebwtFw(
ebwtBase,
- gColor, // index is colorspace
+ false, // index is colorspace
0, // don't need entireReverse for fw index
true, // index is for the forward direction
-1, // offrate (irrelevant)
@@ -1825,7 +1817,7 @@ int main(int argc, char **argv) {
t = new Timer(cerr, "Time loading bw index: ", timing);
Ebwt ebwtBw(
ebwtBase + ".rev",
- gColor, // index is colorspace
+ false, // index is colorspace
1, // need entireReverse
false, // index is for the backward direction
-1, // offrate (irrelevant)
diff --git a/aligner_seed2.cpp b/aligner_seed2.cpp
index 2bf75a8..0a9d90a 100644
--- a/aligner_seed2.cpp
+++ b/aligner_seed2.cpp
@@ -1825,11 +1825,10 @@ int main(int argc, char **argv) {
// GCTATATAGCGCGCTTGCATCATTTTGTGT
// ^
bool packed = false;
- int color = 0;
pair<Ebwt*, Ebwt*> ebwts = Ebwt::fromStrings<SString<char> >(
strs,
packed,
- color,
+ 0,
REF_READ_REVERSE,
Ebwt::default_bigEndian,
Ebwt::default_lineRate,
@@ -1846,8 +1845,8 @@ int main(int argc, char **argv) {
false, // autoMem
false); // sanity
- ebwts.first->loadIntoMemory (color, -1, true, true, true, true, false);
- ebwts.second->loadIntoMemory(color, 1, true, true, true, true, false);
+ ebwts.first->loadIntoMemory (0, -1, true, true, true, true, false);
+ ebwts.second->loadIntoMemory(0, 1, true, true, true, true, false);
int testnum = 0;
@@ -2036,7 +2035,7 @@ int main(int argc, char **argv) {
ebwts = Ebwt::fromStrings<SString<char> >(
strs,
packed,
- color,
+ 0,
REF_READ_REVERSE,
Ebwt::default_bigEndian,
Ebwt::default_lineRate,
@@ -2053,8 +2052,8 @@ int main(int argc, char **argv) {
false, // autoMem
false); // sanity
- ebwts.first->loadIntoMemory (color, -1, true, true, true, true, false);
- ebwts.second->loadIntoMemory(color, 1, true, true, true, true, false);
+ ebwts.first->loadIntoMemory (0, -1, true, true, true, true, false);
+ ebwts.second->loadIntoMemory(0, 1, true, true, true, true, false);
// Test to see if the root selector works as we expect
{
@@ -2985,7 +2984,7 @@ int main(int argc, char **argv) {
ebwts = Ebwt::fromStrings<SString<char> >(
strs,
packed,
- color,
+ 0,
REF_READ_REVERSE,
Ebwt::default_bigEndian,
Ebwt::default_lineRate,
@@ -3002,8 +3001,8 @@ int main(int argc, char **argv) {
false, // autoMem
false); // sanity
- ebwts.first->loadIntoMemory (color, -1, true, true, true, true, false);
- ebwts.second->loadIntoMemory(color, 1, true, true, true, true, false);
+ ebwts.first->loadIntoMemory (0, -1, true, true, true, true, false);
+ ebwts.second->loadIntoMemory(0, 1, true, true, true, true, false);
// Query is longer than ftab and matches exactly once with one read gap,
// one ref gap, and one mismatch
diff --git a/aligner_seed2.h b/aligner_seed2.h
index e7c4e8c..b184dbb 100644
--- a/aligner_seed2.h
+++ b/aligner_seed2.h
@@ -2252,8 +2252,6 @@ public:
-1, // seed length
-1, // seed interval
dr.minScore(), // minimum score for valid alignment
- -1, // nuc5p (for colorspace)
- -1, // nuc3p (for colorspace)
false, // soft pre-trimming?
0, // 5p pre-trimming
0, // 3p pre-trimming
diff --git a/aligner_sw_common.h b/aligner_sw_common.h
index 639a3c6..be3f388 100644
--- a/aligner_sw_common.h
+++ b/aligner_sw_common.h
@@ -23,16 +23,14 @@
#include "aligner_result.h"
/**
- * Encapsulates the result of a dynamic programming alignment, including
- * colorspace alignments. In our case, the result is a combination of:
+ * Encapsulates the result of a dynamic programming alignment. In our
+ * case, the result is a combination of:
*
* 1. All the nucleotide edits
* 2. All the "edits" where an ambiguous reference char is resolved to
* an unambiguous char.
- * 3. All the color edits (if applicable)
- * 4. All the color miscalls (if applicable). This is a subset of 3.
- * 5. The score of the best alginment
- * 6. The score of the second-best alignment
+ * 3. The score of the best alginment
+ * 4. The score of the second-best alignment
*
* Having scores for the best and second-best alignments gives us an
* idea of where gaps may make reassembly beneficial.
@@ -101,9 +99,6 @@ struct SwResult {
uint64_t swsucc; // # DP problems resulting in alignment
uint64_t swfail; // # DP problems not resulting in alignment
uint64_t swbts; // # DP backtrace steps
-
- int nup; // upstream decoded nucleotide; for colorspace reads
- int ndn; // downstream decoded nucleotide; for colorspace reads
};
/**
diff --git a/aligner_swsse_ee_i16.cpp b/aligner_swsse_ee_i16.cpp
index f5796a4..f329364 100644
--- a/aligner_swsse_ee_i16.cpp
+++ b/aligner_swsse_ee_i16.cpp
@@ -1654,7 +1654,6 @@ bool SwAligner::backtraceNucleotidesEnd2EndSseI16(
// it and add a nucleotide mismatch.
case SW_BT_OALL_DIAG: {
assert_gt(row, 0); assert_gt(col, 0);
- // Check for color mismatch
int readC = (*rd_)[row];
int refNmask = (int)rf_[rfi_+col];
assert_gt(refNmask, 0);
diff --git a/aligner_swsse_ee_u8.cpp b/aligner_swsse_ee_u8.cpp
index d53800a..6f68a47 100644
--- a/aligner_swsse_ee_u8.cpp
+++ b/aligner_swsse_ee_u8.cpp
@@ -1642,7 +1642,6 @@ bool SwAligner::backtraceNucleotidesEnd2EndSseU8(
// it and add a nucleotide mismatch.
case SW_BT_OALL_DIAG: {
assert_gt(row, 0); assert_gt(col, 0);
- // Check for color mismatch
int readC = (*rd_)[row];
int refNmask = (int)rf_[rfi_+col];
assert_gt(refNmask, 0);
diff --git a/aligner_swsse_loc_u8.cpp b/aligner_swsse_loc_u8.cpp
index 71819f2..9c31e27 100644
--- a/aligner_swsse_loc_u8.cpp
+++ b/aligner_swsse_loc_u8.cpp
@@ -2011,7 +2011,6 @@ bool SwAligner::backtraceNucleotidesLocalSseU8(
// it and add a nucleotide mismatch.
case SW_BT_OALL_DIAG: {
assert_gt(row, 0); assert_gt(col, 0);
- // Check for color mismatch
int readC = (*rd_)[row];
int refNmask = (int)rf_[rfi_+col];
assert_gt(refNmask, 0);
diff --git a/aln_sink.h b/aln_sink.h
index 4bdfd78..0bb0e84 100644
--- a/aln_sink.h
+++ b/aln_sink.h
@@ -610,13 +610,6 @@ public:
virtual ~AlnSink() { }
/**
- * Called when the AlnSink is wrapped by a new AlnSinkWrap. This helps us
- * keep track of whether the main lock or any of the per-stream locks will
- * be contended by multiple threads.
- */
- void addWrapper() { numWrappers_++; }
-
- /**
* Append a single hit to the given output stream. If
* synchronization is required, append() assumes the caller has
* already grabbed the appropriate lock.
@@ -868,7 +861,6 @@ public:
protected:
OutputQueue& oq_; // output queue
- int numWrappers_; // # threads owning a wrapper for this HitSink
const StrList& refnames_; // reference names
bool quiet_; // true -> don't print alignment stats at the end
ReportingMetrics met_; // global repository of reporting metrics
diff --git a/alphabet.cpp b/alphabet.cpp
index 630a3bb..c31aeed 100644
--- a/alphabet.cpp
+++ b/alphabet.cpp
@@ -115,164 +115,6 @@ uint8_t asc2dnamask[] = {
};
/**
- * Convert a pair of DNA masks to a color mask
- *
- *
- */
-uint8_t dnamasks2colormask[16][16] = {
- /* 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 */
- /* 0 */ { 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0 },
- /* 1 */ { 0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15 },
- /* 2 */ { 0, 2, 1, 3, 8, 10, 9, 11, 4, 6, 5, 7, 12, 14, 13, 15 },
- /* 3 */ { 0, 3, 3, 3, 12, 15, 15, 15, 12, 15, 15, 15, 12, 15, 15, 15 },
- /* 4 */ { 0, 4, 8, 12, 1, 5, 9, 13, 2, 6, 10, 14, 3, 7, 11, 15 },
- /* 5 */ { 0, 5, 10, 15, 5, 5, 15, 15, 10, 15, 10, 15, 15, 15, 15, 15 },
- /* 6 */ { 0, 6, 9, 15, 9, 15, 9, 15, 6, 6, 15, 15, 15, 15, 15, 15 },
- /* 7 */ { 0, 7, 11, 15, 13, 15, 15, 15, 14, 15, 15, 15, 15, 15, 15, 15 },
- /* 8 */ { 0, 8, 4, 12, 2, 10, 6, 14, 1, 9, 5, 13, 3, 11, 7, 15 },
- /* 9 */ { 0, 9, 6, 15, 6, 15, 6, 15, 9, 9, 15, 15, 15, 15, 15, 15 },
- /* 10 */ { 0, 10, 5, 15, 10, 10, 15, 15, 5, 15, 5, 15, 15, 15, 15, 15 },
- /* 11 */ { 0, 11, 7, 15, 14, 15, 15, 15, 13, 15, 15, 15, 15, 15, 15, 15 },
- /* 12 */ { 0, 12, 12, 12, 3, 15, 15, 15, 3, 15, 15, 15, 3, 15, 15, 15 },
- /* 13 */ { 0, 13, 14, 15, 7, 15, 15, 15, 11, 15, 15, 15, 15, 15, 15, 15 },
- /* 14 */ { 0, 14, 13, 15, 11, 15, 15, 15, 7, 15, 15, 15, 15, 15, 15, 15 },
- /* 15 */ { 0, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15, 15 }
-};
-
-/**
- * Mapping from ASCII characters for ambiguous nucleotides into masks:
- */
-char asc2dnacomp[] = {
- /* 0 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- /* 16 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- /* 32 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,'-', 0, 0,
- /* 48 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- /* 64 */ 0,'T','V','G','H', 0, 0,'C','D', 0, 0,'M', 0,'K','N', 0,
- /* A B C D G H K M N */
- /* 80 */ 0, 0,'Y','S','A', 0,'B','W', 0,'R', 0, 0, 0, 0, 0, 0,
- /* R S T V W Y */
- /* 96 */ 0,'T','V','G','H', 0, 0,'C','D', 0, 0,'M', 0,'K','N', 0,
- /* a b c d g h k m n */
- /* 112 */ 0, 0,'Y','S','A', 0,'B','W', 0,'R', 0, 0, 0, 0, 0, 0,
- /* r s t v w y */
- /* 128 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- /* 144 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- /* 160 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- /* 176 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- /* 192 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- /* 208 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- /* 224 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- /* 240 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-};
-
-/**
- * Mapping from ASCII characters for ambiguous nucleotides into masks:
- */
-char col2dna[] = {
- /* 0 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- /* 16 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- /* 32 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,'-','N', 0,
- /* - . */
- /* 48 */'A','C','G','T','N', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- /* 0 1 2 3 4 */
- /* 64 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- /* 80 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- /* 96 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- /* 112 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- /* 128 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- /* 144 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- /* 160 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- /* 176 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- /* 192 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- /* 208 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- /* 224 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- /* 240 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-};
-
-/**
- * Mapping from ASCII characters for ambiguous nucleotides into masks:
- */
-char dna2col[] = {
- /* 0 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- /* 16 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- /* 32 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,'-', 0, 0,
- /* 48 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- /* 64 */ 0,'0', 0,'1', 0, 0, 0,'2', 0, 0, 0, 0, 0, 0,'.', 0,
- /* A C G N */
- /* 80 */ 0, 0, 0, 0,'3', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- /* T */
- /* 92 */ 0,'0', 0,'1', 0, 0, 0,'2', 0, 0, 0, 0, 0, 0,'.', 0,
- /* a c g n */
- /* 112 */ 0, 0, 0, 0,'3', 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- /* t */
- /* 128 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- /* 144 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- /* 160 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- /* 176 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- /* 192 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- /* 208 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- /* 224 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- /* 240 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0
-};
-
-/**
- * Mapping from ASCII characters for ambiguous nucleotides into masks:
- */
-const char* dna2colstr[] = {
- /* 0 */ "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?",
- /* 16 */ "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?",
- /* 32 */ "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "-", "?", "?",
- /* 48 */ "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?",
- /* 64 */ "?", "0","1|2|3","1","0|2|3","?", "?", "2","0|1|3","?", "?", "2|3", "?", "0|1", ".", "?",
- /* A B C D G H K M N */
- /* 80 */ "?", "?", "0|2","1|2", "3", "?","0|1|2","0|3","?", "1|3", "?", "?", "?", "?", "?", "?",
- /* R S T V W Y */
- /* 92 */ "?", "?","1|2|3","1","0|2|3","?", "?", "2","0|1|3","?", "?", "2|3", "?", "0|1", ".", "?",
- /* a b c d g h k m n */
- /* 112 */ "?", "0", "0|2","1|2", "3", "?","0|1|2","0|3","?", "1|3", "?", "?", "?", "?", "?", "?",
- /* r s t v w y */
- /* 128 */ "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?",
- /* 144 */ "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?",
- /* 160 */ "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?",
- /* 176 */ "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?",
- /* 192 */ "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?",
- /* 208 */ "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?",
- /* 224 */ "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?",
- /* 240 */ "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?", "?"
-};
-
-/**
- * Mapping from ASCII characters to color categories:
- *
- * 0 = invalid - error
- * 1 = valid color
- * 2 = IUPAC (ambiguous DNA) - there is no such thing for colors to my
- * knowledge
- * 3 = not an error, but unmatchable; alignments containing this
- * character are invalid
- */
-uint8_t asc2colcat[] = {
- /* 0 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- /* 16 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- /* 32 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 3, 3, 0,
- /* - . */
- /* 48 */ 1, 1, 1, 1, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- /* 0 1 2 3 4 */
- /* 64 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- /* 80 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- /* 96 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- /* 112 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- /* 128 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- /* 144 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- /* 160 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- /* 176 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- /* 192 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- /* 208 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- /* 224 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- /* 240 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-};
-
-/**
* Set the category for all IUPAC codes. By default they're in
* category 2 (IUPAC), but sometimes we'd like to put them in category
* 3 (unmatchable), for example.
@@ -321,33 +163,6 @@ uint8_t asc2dna[] = {
/* 240 */ 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4, 4,
};
-/// Convert an ascii char representing a base or a color to a 2-bit
-/// code: 0=A,0; 1=C,1; 2=G,2; 3=T,3; 4=N,.
-uint8_t asc2dnaOrCol[] = {
- /* 0 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- /* 16 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- /* 32 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 4, 4, 0,
- /* - . */
- /* 48 */ 0, 1, 2, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- /* 0 1 2 3 */
- /* 64 */ 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4, 0,
- /* A C G N */
- /* 80 */ 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- /* T */
- /* 96 */ 0, 0, 0, 1, 0, 0, 0, 2, 0, 0, 0, 0, 0, 0, 4, 0,
- /* a c g n */
- /* 112 */ 0, 0, 0, 0, 3, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- /* t */
- /* 128 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- /* 144 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- /* 160 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- /* 176 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- /* 192 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- /* 208 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- /* 224 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
- /* 240 */ 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,
-};
-
/// For converting from ASCII to the Dna5 code where A=0, C=1, G=2,
/// T=3, N=4
uint8_t asc2col[] = {
@@ -372,20 +187,6 @@ uint8_t asc2col[] = {
};
/**
- * Convert a nucleotide and a color to the paired nucleotide. Indexed
- * first by nucleotide then by color. Note that this is exactly the
- * same as the dinuc2color array.
- */
-uint8_t nuccol2nuc[5][5] = {
- /* B G O R . */
- /* A */ {0, 1, 2, 3, 4},
- /* C */ {1, 0, 3, 2, 4},
- /* G */ {2, 3, 0, 1, 4},
- /* T */ {3, 2, 1, 0, 4},
- /* N */ {4, 4, 4, 4, 4}
-};
-
-/**
* Convert a pair of nucleotides to a color.
*/
uint8_t dinuc2color[5][5] = {
diff --git a/alphabet.h b/alphabet.h
index 340942e..adf9a8f 100644
--- a/alphabet.h
+++ b/alphabet.h
@@ -42,38 +42,15 @@ extern uint8_t asc2dnamask[];
extern int mask2popcnt[];
/// Convert an ascii char to a 2-bit base: 0=A, 1=C, 2=G, 3=T, 4=N
extern uint8_t asc2dna[];
-/// Convert an ascii char representing a base or a color to a 2-bit
-/// code: 0=A,0; 1=C,1; 2=G,2; 3=T,3; 4=N,.
-extern uint8_t asc2dnaOrCol[];
-/// Convert a pair of DNA masks to a color mask
-extern uint8_t dnamasks2colormask[16][16];
-/// Convert an ascii char to a color category. Categories are:
-/// 0 -> invalid
-/// 1 -> unambiguous 0, 1, 2 or 3
-/// 2 -> ambiguous (not applicable for colors)
-/// 3 -> unmatchable
-extern uint8_t asc2colcat[];
/// Convert an ascii char to a 2-bit base: 0=A, 1=C, 2=G, 3=T, 4=N
extern uint8_t asc2col[];
-/// Convert an ascii char to its DNA complement, including IUPACs
-extern char asc2dnacomp[];
/// Convert a pair of 2-bit (and 4=N) encoded DNA bases to a color
extern uint8_t dinuc2color[5][5];
-/// Convert a 2-bit nucleotide (and 4=N) and a color to the
-/// corresponding 2-bit nucleotide
-extern uint8_t nuccol2nuc[5][5];
/// Convert a 4-bit mask into an IUPAC code
extern char mask2iupac[16];
-/// Convert an ascii color to an ascii dna char
-extern char col2dna[];
-/// Convert an ascii dna to a color char
-extern char dna2col[];
-/// Convert an ascii dna to a color char
-extern const char* dna2colstr[];
-
/// Convert bit encoded DNA char to its complement
extern int dnacomp[5];
@@ -84,41 +61,6 @@ extern const char *iupacs;
extern int maskcomp[16];
/**
- * Return true iff c is a Dna character.
- */
-static inline bool isDna(char c) {
- return asc2dnacat[(int)c] > 0;
-}
-
-/**
- * Return true iff c is a color character.
- */
-static inline bool isColor(char c) {
- return asc2colcat[(int)c] > 0;
-}
-
-/**
- * Return true iff c is an ambiguous Dna character.
- */
-static inline bool isAmbigNuc(char c) {
- return asc2dnacat[(int)c] == 2;
-}
-
-/**
- * Return true iff c is an ambiguous color character.
- */
-static inline bool isAmbigColor(char c) {
- return asc2colcat[(int)c] == 2;
-}
-
-/**
- * Return true iff c is an ambiguous character.
- */
-static inline bool isAmbig(char c, bool color) {
- return (color ? asc2colcat[(int)c] : asc2dnacat[(int)c]) == 2;
-}
-
-/**
* Return true iff c is an unambiguous DNA character.
*/
static inline bool isUnambigNuc(char c) {
@@ -150,20 +92,6 @@ static inline int compDna(int c) {
return dnacomp[c];
}
-/**
- * Return true iff c is an unambiguous Dna character.
- */
-static inline bool isUnambigDna(char c) {
- return asc2dnacat[(int)c] == 1;
-}
-
-/**
- * Return true iff c is an unambiguous color character (0,1,2,3).
- */
-static inline bool isUnambigColor(char c) {
- return asc2colcat[(int)c] == 1;
-}
-
/// Convert a pair of 2-bit (and 4=N) encoded DNA bases to a color
extern uint8_t dinuc2color[5][5];
diff --git a/bt2_build.cpp b/bt2_build.cpp
index e0d87ad..0497f98 100644
--- a/bt2_build.cpp
+++ b/bt2_build.cpp
@@ -198,7 +198,6 @@ static struct option long_options[] = {
{(char*)"ntoa", no_argument, 0, ARG_NTOA},
{(char*)"justref", no_argument, 0, '3'},
{(char*)"noref", no_argument, 0, 'r'},
- {(char*)"color", no_argument, 0, 'C'},
{(char*)"sa", no_argument, 0, ARG_SA},
{(char*)"reverse-each", no_argument, 0, ARG_REVERSE_EACH},
{(char*)"threads", required_argument, 0, ARG_THREADS},
@@ -236,6 +235,7 @@ static T parseNumber(T lower, const char *errmsg) {
static bool parseOptions(int argc, const char **argv) {
int option_index = 0;
int next_option;
+ bool bmaxDivNSet = false;
bool abort = false;
do {
next_option = getopt_long(
@@ -248,10 +248,6 @@ static bool parseOptions(int argc, const char **argv) {
case 'f': format = FASTA; break;
case 'c': format = CMDLINE; break;
case 'p': packed = true; break;
- case 'C':
- cerr << "Error: -C specified but Bowtie 2 does not support colorspace input." << endl;
- throw 1;
- break;
case 'l':
lineRate = parseNumber<int>(3, "-l/--lineRate arg must be at least 3");
break;
@@ -287,6 +283,7 @@ static bool parseOptions(int argc, const char **argv) {
bmaxDivN = 0xffffffff; // don't use multSqrt
break;
case ARG_BMAX_DIV:
+ bmaxDivNSet = true;
bmaxDivN = parseNumber<uint32_t>(1, "--bmaxdivn arg must be at least 1");
bmax = OFF_MASK; // don't use bmax
bmaxMultSqrt = OFF_MASK; // don't use multSqrt
@@ -327,6 +324,9 @@ static bool parseOptions(int argc, const char **argv) {
<< "extremely slow performance and memory exhaustion. Perhaps you meant to specify" << endl
<< "a small --bmaxdivn?" << endl;
}
+ if (!bmaxDivNSet) {
+ bmaxDivN *= nthreads;
+ }
return abort;
}
@@ -405,9 +405,9 @@ static void driver(
}
if(!reverse) {
#ifdef BOWTIE_64BIT_INDEX
- cerr << "Building a LARGE index" << endl;
+ if (verbose) cerr << "Building a LARGE index" << endl;
#else
- cerr << "Building a SMALL index" << endl;
+ if (verbose) cerr << "Building a SMALL index" << endl;
#endif
}
// Vector for the ordered list of "records" comprising the input
diff --git a/bt2_inspect.cpp b/bt2_inspect.cpp
index 9094aa0..215ee8e 100644
--- a/bt2_inspect.cpp
+++ b/bt2_inspect.cpp
@@ -71,7 +71,7 @@ static void printUsage(ostream& out) {
<< endl
<< " By default, prints FASTA records of the indexed nucleotide sequences to" << endl
<< " standard out. With -n, just prints names. With -s, just prints a summary of" << endl
- << " the index parameters and sequences. With -e, preserves colors if applicable." << endl
+ << " the index parameters and sequences." << endl
<< endl
<< "Options:" << endl;
if(wrapper == "basic-0") {
@@ -81,7 +81,6 @@ static void printUsage(ostream& out) {
out << " -a/--across <int> Number of characters across in FASTA output (default: 60)" << endl
<< " -n/--names Print reference sequence names only" << endl
<< " -s/--summary Print summary incl. ref names, lengths, index properties" << endl
- << " -e/--bt2-ref Reconstruct reference from ." + gEbwt_ext + " (slow, preserves colors)" << endl
<< " -v/--verbose Verbose output (for debugging)" << endl
<< " -h/--help print detailed description of tool and its options" << endl
<< " --help print this usage message" << endl
@@ -204,7 +203,7 @@ static void print_ref_sequence(
}
fout << "\n";
}
- delete buf;
+ delete [] buf;
}
/**
@@ -352,7 +351,6 @@ static void print_index_summary(
readEbwtRefnames(fname, p_refnames);
cout << "Flags" << '\t' << (-flags) << endl;
cout << "Reverse flags" << '\t' << (-flagsr) << endl;
- cout << "Colorspace" << '\t' << (color ? "1" : "0") << endl;
cout << "2.0-compatible" << '\t' << (entireReverse ? "1" : "0") << endl;
cout << "SA-Sample" << "\t1 in " << (1 << ebwt.eh().offRate()) << endl;
cout << "FTab-Chars" << '\t' << ebwt.eh().ftabChars() << endl;
diff --git a/bt2_search.cpp b/bt2_search.cpp
index b481f44..783f544 100644
--- a/bt2_search.cpp
+++ b/bt2_search.cpp
@@ -61,7 +61,6 @@ static EList<string> mates1; // mated reads (first mate)
static EList<string> mates2; // mated reads (second mate)
static EList<string> mates12; // mated reads (1st/2nd interleaved in 1 file)
static string adjIdxBase;
-bool gColor; // colorspace (not supported)
int gVerbose; // be talkative
static bool startVerbose; // be talkative at startup
int gQuiet; // print nothing but the alignments
@@ -90,6 +89,7 @@ static bool noRefNames; // true -> print reference indexes; not names
static uint32_t khits; // number of hits per read; >1 is much slower
static uint32_t mhits; // don't report any hits if there are > mhits
static int partitionSz; // output a partitioning key in first field
+static int readsPerBatch; // # reads to read from input file at once
static bool fileParallel; // separate threads read separate input files in parallel
static bool useShmem; // use shared memory to hold the index
static bool useMm; // use memory-mapped files to hold the index
@@ -113,7 +113,6 @@ bool gNorc; // don't align rc orientation of read
static uint32_t fastaContLen;
static uint32_t fastaContFreq;
static bool hadoopOut; // print Hadoop status and summary messages
-static bool fuzzy;
static bool fullRef;
static bool samTruncQname; // whether to truncate QNAME to 255 chars
static bool samOmitSecSeqQual; // omit SEQ/QUAL for 2ndary alignments?
@@ -125,8 +124,6 @@ static bool sam_print_xs; // XS:i
static bool sam_print_xss; // Xs:i and Ys:i
static bool sam_print_yn; // YN:i and Yn:i
static bool sam_print_xn;
-static bool sam_print_cs;
-static bool sam_print_cq;
static bool sam_print_x0;
static bool sam_print_x1;
static bool sam_print_xm;
@@ -250,7 +247,6 @@ static void resetOptions() {
mates2.clear();
mates12.clear();
adjIdxBase = "";
- gColor = false;
gVerbose = 0;
startVerbose = 0;
gQuiet = false;
@@ -279,6 +275,7 @@ static void resetOptions() {
khits = 1; // number of hits per read; >1 is much slower
mhits = 50; // stop after finding this many alignments+1
partitionSz = 0; // output a partitioning key in first field
+ readsPerBatch = 16; // # reads to read from input file at once
fileParallel = false; // separate threads read separate input files in parallel
useShmem = false; // use shared memory to hold the index
useMm = false; // use memory-mapped files to hold the index
@@ -303,7 +300,6 @@ static void resetOptions() {
fastaContLen = 0;
fastaContFreq = 0;
hadoopOut = false; // print Hadoop status and summary messages
- fuzzy = false; // reads will have alternate basecalls w/ qualities
fullRef = false; // print entire reference name instead of just up to 1st space
samTruncQname = true; // whether to truncate QNAME to 255 chars
samOmitSecSeqQual = false; // omit SEQ/QUAL for 2ndary alignments?
@@ -315,8 +311,6 @@ static void resetOptions() {
sam_print_xss = false; // Xs:i and Ys:i
sam_print_yn = false; // YN:i and Yn:i
sam_print_xn = true;
- sam_print_cs = false;
- sam_print_cq = false;
sam_print_x0 = true;
sam_print_x1 = true;
sam_print_xm = true;
@@ -460,6 +454,7 @@ static struct option long_options[] = {
{(char*)"qupto", required_argument, 0, 'u'},
{(char*)"upto", required_argument, 0, 'u'},
{(char*)"version", no_argument, 0, ARG_VERSION},
+ {(char*)"reads-per-batch", required_argument, 0, ARG_READS_PER_BATCH},
{(char*)"filepar", no_argument, 0, ARG_FILEPAR},
{(char*)"help", no_argument, 0, 'h'},
{(char*)"threads", required_argument, 0, 'p'},
@@ -491,7 +486,6 @@ static struct option long_options[] = {
{(char*)"shmem", no_argument, 0, ARG_SHMEM},
{(char*)"mmsweep", no_argument, 0, ARG_MMSWEEP},
{(char*)"hadoopout", no_argument, 0, ARG_HADOOPOUT},
- {(char*)"fuzzy", no_argument, 0, ARG_FUZZY},
{(char*)"fullref", no_argument, 0, ARG_FULLREF},
{(char*)"usage", no_argument, 0, ARG_USAGE},
{(char*)"sam-no-qname-trunc", no_argument, 0, ARG_SAM_NO_QNAME_TRUNC},
@@ -510,7 +504,6 @@ static struct option long_options[] = {
{(char*)"no-HD", no_argument, 0, ARG_SAM_NOHEAD},
{(char*)"no-SQ", no_argument, 0, ARG_SAM_NOSQ},
{(char*)"no-unal", no_argument, 0, ARG_SAM_NO_UNAL},
- {(char*)"color", no_argument, 0, 'C'},
{(char*)"sam-RG", required_argument, 0, ARG_SAM_RG},
{(char*)"sam-rg", required_argument, 0, ARG_SAM_RG},
{(char*)"sam-rg-id", required_argument, 0, ARG_SAM_RGID},
@@ -967,11 +960,6 @@ static void parseOption(int next_option, const char *arg) {
case 'r': format = RAW; break;
case 'c': format = CMDLINE; break;
case ARG_QSEQ: format = QSEQ; break;
- case 'C': {
- cerr << "Error: -C specified but Bowtie 2 does not support colorspace input." << endl;
- throw 1;
- break;
- }
case 'I':
gMinInsert = parseInt(0, "-I arg must be positive", arg);
break;
@@ -1016,7 +1004,6 @@ static void parseOption(int next_option, const char *arg) {
seedCacheCurrentMB = (uint32_t)parseInt(1, "--seed-cache-sz arg must be at least 1", arg);
break;
case ARG_REFIDX: noRefNames = true; break;
- case ARG_FUZZY: fuzzy = true; break;
case ARG_FULLREF: fullRef = true; break;
case ARG_GAP_BAR:
gGapBarrier = parseInt(1, "--gbar must be no less than 1", arg);
@@ -1160,9 +1147,6 @@ static void parseOption(int next_option, const char *arg) {
case ARG_SAM_PRINT_YI: sam_print_yi = true; break;
case ARG_REORDER: reorder = true; break;
case ARG_MAPQ_EX: {
- //sam_print_zp = true;
- //sam_print_xss = true;
- //sam_print_yn = true;
sam_print_zt = true;
break;
}
@@ -1221,6 +1205,9 @@ static void parseOption(int next_option, const char *arg) {
break;
}
case ARG_PARTITION: partitionSz = parse<int>(arg); break;
+ case ARG_READS_PER_BATCH:
+ readsPerBatch = parseInt(1, "--reads-per-batch arg must be at least 1", arg);
+ break;
case ARG_DPAD:
maxhalf = parseInt(0, "--dpad must be no less than 0", arg);
break;
@@ -1397,7 +1384,7 @@ static void parseOption(int next_option, const char *arg) {
polstr += ";";
EList<string> args;
tokenize(arg, ",", args);
- if(args.size() > 3 && args.size() == 0) {
+ if(args.size() > 3 || args.size() == 0) {
cerr << "Error: expected 3 or fewer comma-separated "
<< "arguments to --n-ceil option, got "
<< args.size() << endl;
@@ -1579,6 +1566,13 @@ static void parseOptions(int argc, const char **argv) {
<< "); setting to 1 instead" << endl;
gGapBarrier = 1;
}
+ if(bonusMatch > 0 && !scoreMin.alwaysPositive()) {
+ cerr << "Error: the match penalty is greater than 0 (" << bonusMatch
+ << ") but the --score-min function can be less than or equal to "
+ << "zero. Either let the match penalty be 0 or make --score-min "
+ << "always positive." << endl;
+ throw 1;
+ }
if(multiseedMms >= multiseedLen) {
assert_gt(multiseedLen, 0);
cerr << "Warning: seed mismatches (" << multiseedMms
@@ -1602,16 +1596,21 @@ static const char *argv0 = NULL;
/// Create a PatternSourcePerThread for the current thread according
/// to the global params and return a pointer to it
static PatternSourcePerThreadFactory*
-createPatsrcFactory(PairedPatternSource& _patsrc, int tid) {
+createPatsrcFactory(
+ PatternComposer& patcomp,
+ const PatternParams& pp,
+ int tid)
+{
PatternSourcePerThreadFactory *patsrcFact;
- patsrcFact = new WrappedPatternSourcePerThreadFactory(_patsrc);
+ patsrcFact = new PatternSourcePerThreadFactory(patcomp, pp);
assert(patsrcFact != NULL);
return patsrcFact;
}
#define PTHREAD_ATTRS (PTHREAD_CREATE_JOINABLE | PTHREAD_CREATE_DETACHED)
-static PairedPatternSource* multiseed_patsrc;
+static PatternComposer* multiseed_patsrc;
+static PatternParams multiseed_pp;
static Ebwt* multiseed_ebwtFw;
static Ebwt* multiseed_ebwtBw;
static Scoring* multiseed_sc;
@@ -2588,12 +2587,12 @@ static inline void printMmsSkipMsg(
ostringstream os;
if(paired) {
os << "Warning: skipping mate #" << (mate1 ? '1' : '2')
- << " of read '" << (mate1 ? ps.bufa().name : ps.bufb().name)
- << "' because length (" << (mate1 ? ps.bufa().patFw.length() : ps.bufb().patFw.length())
+ << " of read '" << (mate1 ? ps.read_a().name : ps.read_b().name)
+ << "' because length (" << (mate1 ? ps.read_a().patFw.length() : ps.read_b().patFw.length())
<< ") <= # seed mismatches (" << seedmms << ")" << endl;
} else {
- os << "Warning: skipping read '" << (mate1 ? ps.bufa().name : ps.bufb().name)
- << "' because length (" << (mate1 ? ps.bufa().patFw.length() : ps.bufb().patFw.length())
+ os << "Warning: skipping read '" << (mate1 ? ps.read_a().name : ps.read_b().name)
+ << "' because length (" << (mate1 ? ps.read_a().patFw.length() : ps.read_b().patFw.length())
<< ") <= # seed mismatches (" << seedmms << ")" << endl;
}
cerr << os.str().c_str();
@@ -2607,10 +2606,10 @@ static inline void printLenSkipMsg(
ostringstream os;
if(paired) {
os << "Warning: skipping mate #" << (mate1 ? '1' : '2')
- << " of read '" << (mate1 ? ps.bufa().name : ps.bufb().name)
+ << " of read '" << (mate1 ? ps.read_a().name : ps.read_b().name)
<< "' because it was < 2 characters long" << endl;
} else {
- os << "Warning: skipping read '" << (mate1 ? ps.bufa().name : ps.bufb().name)
+ os << "Warning: skipping read '" << (mate1 ? ps.read_a().name : ps.read_b().name)
<< "' because it was < 2 characters long" << endl;
}
cerr << os.str().c_str();
@@ -2625,11 +2624,11 @@ static inline void printLocalScoreMsg(
if(paired) {
os << "Warning: minimum score function gave negative number in "
<< "--local mode for mate #" << (mate1 ? '1' : '2')
- << " of read '" << (mate1 ? ps.bufa().name : ps.bufb().name)
+ << " of read '" << (mate1 ? ps.read_a().name : ps.read_b().name)
<< "; setting to 0 instead" << endl;
} else {
os << "Warning: minimum score function gave negative number in "
- << "--local mode for read '" << (mate1 ? ps.bufa().name : ps.bufb().name)
+ << "--local mode for read '" << (mate1 ? ps.read_a().name : ps.read_b().name)
<< "; setting to 0 instead" << endl;
}
cerr << os.str().c_str();
@@ -2644,11 +2643,11 @@ static inline void printEEScoreMsg(
if(paired) {
os << "Warning: minimum score function gave positive number in "
<< "--end-to-end mode for mate #" << (mate1 ? '1' : '2')
- << " of read '" << (mate1 ? ps.bufa().name : ps.bufb().name)
+ << " of read '" << (mate1 ? ps.read_a().name : ps.read_b().name)
<< "; setting to 0 instead" << endl;
} else {
os << "Warning: minimum score function gave positive number in "
- << "--end-to-end mode for read '" << (mate1 ? ps.bufa().name : ps.bufb().name)
+ << "--end-to-end mode for read '" << (mate1 ? ps.read_a().name : ps.read_b().name)
<< "; setting to 0 instead" << endl;
}
cerr << os.str().c_str();
@@ -2798,7 +2797,8 @@ static void multiseedSearchWorker(void *vp) {
#endif
assert(multiseed_ebwtFw != NULL);
assert(multiseedMms == 0 || multiseed_ebwtBw != NULL);
- PairedPatternSource& patsrc = *multiseed_patsrc;
+ PatternComposer& patsrc = *multiseed_patsrc;
+ PatternParams pp = multiseed_pp;
const Ebwt& ebwtFw = *multiseed_ebwtFw;
const Ebwt& ebwtBw = *multiseed_ebwtBw;
const Scoring& sc = *multiseed_sc;
@@ -2827,7 +2827,7 @@ static void multiseedSearchWorker(void *vp) {
// problems, or generally characterize performance.
//const BitPairReference& refs = *multiseed_refs;
- auto_ptr<PatternSourcePerThreadFactory> patsrcFact(createPatsrcFactory(patsrc, tid));
+ auto_ptr<PatternSourcePerThreadFactory> patsrcFact(createPatsrcFactory(patsrc, pp, tid));
auto_ptr<PatternSourcePerThread> ps(patsrcFact->create());
// Thread-local cache for seed alignments
@@ -2860,7 +2860,7 @@ static void multiseedSearchWorker(void *vp) {
AlnSinkWrap msinkwrap(
msink, // global sink
rp, // reporting parameters
- *bmapq.get(), // MAPQ calculator
+ *bmapq, // MAPQ calculator
(size_t)tid); // thread id
// Write dynamic-programming problem descriptions here
@@ -2945,22 +2945,24 @@ static void multiseedSearchWorker(void *vp) {
rndArb.init((uint32_t)time(0));
int mergei = 0;
int mergeival = 16;
- while(true) {
- bool success = false, done = false, paired = false;
- ps->nextReadPair(success, done, paired, outType != OUTPUT_SAM);
+ bool done = false;
+ while(!done) {
+ pair<bool, bool> ret = ps->nextReadPair();
+ bool success = ret.first;
+ done = ret.second;
if(!success && done) {
break;
} else if(!success) {
continue;
}
- TReadId rdid = ps->rdid();
+ TReadId rdid = ps->read_a().rdid;
bool sample = true;
if(arbitraryRandom) {
- ps->bufa().seed = rndArb.nextU32();
- ps->bufb().seed = rndArb.nextU32();
+ ps->read_a().seed = rndArb.nextU32();
+ ps->read_b().seed = rndArb.nextU32();
}
if(sampleFrac < 1.0f) {
- rnd.init(ROTL(ps->bufa().seed, 2));
+ rnd.init(ROTL(ps->read_a().seed, 2));
sample = rnd.nextFloat() < sampleFrac;
}
if(rdid >= skipReads && rdid < qUpto && sample) {
@@ -3008,17 +3010,16 @@ static void multiseedSearchWorker(void *vp) {
// Try to align this read
while(retry) {
retry = false;
- assert_eq(ps->bufa().color, false);
ca.nextRead(); // clear the cache
olm.reads++;
assert(!ca.aligning());
- bool pair = paired;
- const size_t rdlen1 = ps->bufa().length();
- const size_t rdlen2 = pair ? ps->bufb().length() : 0;
+ bool paired = !ps->read_b().empty();
+ const size_t rdlen1 = ps->read_a().length();
+ const size_t rdlen2 = paired ? ps->read_b().length() : 0;
olm.bases += (rdlen1 + rdlen2);
msinkwrap.nextRead(
- &ps->bufa(),
- pair ? &ps->bufb() : NULL,
+ &ps->read_a(),
+ paired ? &ps->read_b() : NULL,
rdid,
sc.qualitiesMatter());
assert(msinkwrap.inited());
@@ -3063,8 +3064,8 @@ static void multiseedSearchWorker(void *vp) {
// N filter; does the read have too many Ns?
size_t readns[2] = {0, 0};
sc.nFilterPair(
- &ps->bufa().patFw,
- pair ? &ps->bufb().patFw : NULL,
+ &ps->read_a().patFw,
+ paired ? &ps->read_b().patFw : NULL,
readns[0],
readns[1],
nfilt[0],
@@ -3092,13 +3093,13 @@ static void multiseedSearchWorker(void *vp) {
}
qcfilt[0] = qcfilt[1] = true;
if(qcFilter) {
- qcfilt[0] = (ps->bufa().filter != '0');
- qcfilt[1] = (ps->bufb().filter != '0');
+ qcfilt[0] = (ps->read_a().filter != '0');
+ qcfilt[1] = (ps->read_b().filter != '0');
}
filt[0] = (nfilt[0] && scfilt[0] && lenfilt[0] && qcfilt[0]);
filt[1] = (nfilt[1] && scfilt[1] && lenfilt[1] && qcfilt[1]);
prm.nFilt += (filt[0] ? 0 : 1) + (filt[1] ? 0 : 1);
- Read* rds[2] = { &ps->bufa(), &ps->bufb() };
+ Read* rds[2] = { &ps->read_a(), &ps->read_b() };
// For each mate...
assert(msinkwrap.empty());
sd.nextRead(paired, rdrows[0], rdrows[1]); // SwDriver
@@ -3123,13 +3124,13 @@ static void multiseedSearchWorker(void *vp) {
size_t matemap[2] = { 0, 1 };
bool pairPostFilt = filt[0] && filt[1];
if(pairPostFilt) {
- rnd.init(ps->bufa().seed ^ ps->bufb().seed);
+ rnd.init(ps->read_a().seed ^ ps->read_b().seed);
} else {
- rnd.init(ps->bufa().seed);
+ rnd.init(ps->read_a().seed);
}
// Calculate interval length for both mates
int interval[2] = { 0, 0 };
- for(size_t mate = 0; mate < (pair ? 2:1); mate++) {
+ for(size_t mate = 0; mate < (paired ? 2:1); mate++) {
interval[mate] = msIval.f<int>((double)rdlens[mate]);
if(filt[0] && filt[1]) {
// Boost interval length by 20% for paired-end reads
@@ -3173,14 +3174,14 @@ static void multiseedSearchWorker(void *vp) {
}
assert_gt(nrounds[0], 0);
// Increment counters according to what got filtered
- for(size_t mate = 0; mate < (pair ? 2:1); mate++) {
+ for(size_t mate = 0; mate < (paired ? 2:1); mate++) {
if(!filt[mate]) {
// Mate was rejected by N filter
olm.freads++; // reads filtered out
olm.fbases += rdlens[mate]; // bases filtered out
} else {
shs[mate].clear();
- shs[mate].nextRead(mate == 0 ? ps->bufa() : ps->bufb());
+ shs[mate].nextRead(mate == 0 ? ps->read_a() : ps->read_b());
assert(shs[mate].empty());
olm.ureads++; // reads passing filter
olm.ubases += rdlens[mate]; // bases passing filter
@@ -3193,7 +3194,7 @@ static void multiseedSearchWorker(void *vp) {
// Find end-to-end exact alignments for each read
if(doExactUpFront) {
- for(size_t matei = 0; matei < (pair ? 2:1); matei++) {
+ for(size_t matei = 0; matei < (paired ? 2:1); matei++) {
size_t mate = matemap[matei];
if(!filt[mate] || done[mate] || msinkwrap.state().doneWithMate(mate == 0)) {
continue;
@@ -3239,11 +3240,11 @@ static void multiseedSearchWorker(void *vp) {
continue;
}
assert(filt[mate]);
- assert(matei == 0 || pair);
+ assert(matei == 0 || paired);
assert(!msinkwrap.maxed());
assert(msinkwrap.repOk());
int ret = 0;
- if(pair) {
+ if(paired) {
// Paired-end dynamic programming driver
ret = sd.extendSeedsPaired(
*rds[mate], // mate to align as anchor
@@ -3372,7 +3373,7 @@ static void multiseedSearchWorker(void *vp) {
}
// 1-mismatch
if(do1mmUpFront && !seedSumm) {
- for(size_t matei = 0; matei < (pair ? 2:1); matei++) {
+ for(size_t matei = 0; matei < (paired ? 2:1); matei++) {
size_t mate = matemap[matei];
if(!filt[mate] || done[mate] || nelt[mate] > eePeEeltLimit) {
// Done with this mate
@@ -3424,7 +3425,7 @@ static void multiseedSearchWorker(void *vp) {
continue;
}
int ret = 0;
- if(pair) {
+ if(paired) {
// Paired-end dynamic programming driver
ret = sd.extendSeedsPaired(
*rds[mate], // mate to align as anchor
@@ -3569,7 +3570,7 @@ static void multiseedSearchWorker(void *vp) {
// if(seedlens[0] > 8) seedlens[0]--;
// if(seedlens[1] > 8) seedlens[1]--;
//}
- for(size_t matei = 0; matei < (pair ? 2:1); matei++) {
+ for(size_t matei = 0; matei < (paired ? 2:1); matei++) {
size_t mate = matemap[matei];
if(done[mate] || msinkwrap.state().doneWithMate(mate == 0)) {
// Done with this mate
@@ -3666,7 +3667,7 @@ static void multiseedSearchWorker(void *vp) {
// TODO: Consider mates & orientations separately?
matemap[0] = 1; matemap[1] = 0;
}
- for(size_t matei = 0; matei < (pair ? 2:1); matei++) {
+ for(size_t matei = 0; matei < (paired ? 2:1); matei++) {
size_t mate = matemap[matei];
if(done[mate] || msinkwrap.state().doneWithMate(mate == 0)) {
// Done with this mate
@@ -3685,7 +3686,7 @@ static void multiseedSearchWorker(void *vp) {
// Sort seed hits into ranks
shs[mate].rankSeedHits(rnd, msinkwrap.allHits());
int ret = 0;
- if(pair) {
+ if(paired) {
// Paired-end dynamic programming driver
ret = sd.extendSeedsPaired(
*rds[mate], // mate to align as anchor
@@ -3819,7 +3820,7 @@ static void multiseedSearchWorker(void *vp) {
prm.seedHitAvg = (float)seedHitTot / seedsTried;
}
size_t totnucs = 0;
- for(size_t mate = 0; mate < (pair ? 2:1); mate++) {
+ for(size_t mate = 0; mate < (paired ? 2:1); mate++) {
if(filt[mate]) {
size_t len = rdlens[mate];
if(!nofw[mate] && !norc[mate]) {
@@ -3870,7 +3871,7 @@ static void multiseedSearchWorker(void *vp) {
}
if(metricsPerRead) {
MERGE_METRICS(metricsPt, nthreads > 1);
- nametmp = ps->bufa().name;
+ nametmp = ps->read_a().name;
metricsPt.reportInterval(
metricsOfb, metricsStderr, true, true, &nametmp);
metricsPt.reset();
@@ -3902,7 +3903,8 @@ static void multiseedSearchWorker_2p5(void *vp) {
#endif
assert(multiseed_ebwtFw != NULL);
assert(multiseedMms == 0 || multiseed_ebwtBw != NULL);
- PairedPatternSource& patsrc = *multiseed_patsrc;
+ PatternComposer& patsrc = *multiseed_patsrc;
+ PatternParams pp = multiseed_pp;
const Ebwt& ebwtFw = *multiseed_ebwtFw;
const Ebwt& ebwtBw = *multiseed_ebwtBw;
const Scoring& sc = *multiseed_sc;
@@ -3915,7 +3917,7 @@ static void multiseedSearchWorker_2p5(void *vp) {
// level. These in turn can be used to diagnose performance
// problems, or generally characterize performance.
- auto_ptr<PatternSourcePerThreadFactory> patsrcFact(createPatsrcFactory(patsrc, tid));
+ auto_ptr<PatternSourcePerThreadFactory> patsrcFact(createPatsrcFactory(patsrc, pp, tid));
auto_ptr<PatternSourcePerThread> ps(patsrcFact->create());
// Instantiate an object for holding reporting-related parameters.
@@ -3934,7 +3936,7 @@ static void multiseedSearchWorker_2p5(void *vp) {
AlnSinkWrap msinkwrap(
msink, // global sink
rp, // reporting parameters
- *bmapq.get(), // MAPQ calculator
+ *bmapq, // MAPQ calculator
(size_t)tid); // thread id
OuterLoopMetrics olm;
@@ -4013,21 +4015,22 @@ static void multiseedSearchWorker_2p5(void *vp) {
int mergei = 0;
int mergeival = 16;
while(true) {
- bool success = false, done = false, paired = false;
- ps->nextReadPair(success, done, paired, outType != OUTPUT_SAM);
+ pair<bool, bool> ret = ps->nextReadPair();
+ bool success = ret.first;
+ bool done = ret.second;
if(!success && done) {
break;
} else if(!success) {
continue;
}
- TReadId rdid = ps->rdid();
+ TReadId rdid = ps->read_a().rdid;
bool sample = true;
if(arbitraryRandom) {
- ps->bufa().seed = rndArb.nextU32();
- ps->bufb().seed = rndArb.nextU32();
+ ps->read_a().seed = rndArb.nextU32();
+ ps->read_b().seed = rndArb.nextU32();
}
if(sampleFrac < 1.0f) {
- rnd.init(ROTL(ps->bufa().seed, 2));
+ rnd.init(ROTL(ps->read_a().seed, 2));
sample = rnd.nextFloat() < sampleFrac;
}
if(rdid >= skipReads && rdid < qUpto && sample) {
@@ -4061,17 +4064,16 @@ static void multiseedSearchWorker_2p5(void *vp) {
gettimeofday(&prm.tv_beg, &prm.tz_beg);
}
// Try to align this read
- assert_eq(ps->bufa().color, false);
olm.reads++;
- bool pair = paired;
- const size_t rdlen1 = ps->bufa().length();
- const size_t rdlen2 = pair ? ps->bufb().length() : 0;
+ bool paired = !ps->read_b().empty();
+ const size_t rdlen1 = ps->read_a().length();
+ const size_t rdlen2 = paired ? ps->read_b().length() : 0;
olm.bases += (rdlen1 + rdlen2);
// Check if read is identical to previous read
- rnd.init(ROTL(ps->bufa().seed, 5));
+ rnd.init(ROTL(ps->read_a().seed, 5));
msinkwrap.nextRead(
- &ps->bufa(),
- pair ? &ps->bufb() : NULL,
+ &ps->read_a(),
+ paired ? &ps->read_b() : NULL,
rdid,
sc.qualitiesMatter());
assert(msinkwrap.inited());
@@ -4083,8 +4085,8 @@ static void multiseedSearchWorker_2p5(void *vp) {
// N filter; does the read have too many Ns?
size_t readns[2] = {0, 0};
sc.nFilterPair(
- &ps->bufa().patFw,
- pair ? &ps->bufb().patFw : NULL,
+ &ps->read_a().patFw,
+ paired ? &ps->read_b().patFw : NULL,
readns[0],
readns[1],
nfilt[0],
@@ -4112,13 +4114,13 @@ static void multiseedSearchWorker_2p5(void *vp) {
}
qcfilt[0] = qcfilt[1] = true;
if(qcFilter) {
- qcfilt[0] = (ps->bufa().filter != '0');
- qcfilt[1] = (ps->bufb().filter != '0');
+ qcfilt[0] = (ps->read_a().filter != '0');
+ qcfilt[1] = (ps->read_b().filter != '0');
}
filt[0] = (nfilt[0] && scfilt[0] && lenfilt[0] && qcfilt[0]);
filt[1] = (nfilt[1] && scfilt[1] && lenfilt[1] && qcfilt[1]);
prm.nFilt += (filt[0] ? 0 : 1) + (filt[1] ? 0 : 1);
- Read* rds[2] = { &ps->bufa(), &ps->bufb() };
+ Read* rds[2] = { &ps->read_a(), &ps->read_b() };
assert(msinkwrap.empty());
// Calcualte nofw / no rc
bool nofw[2] = { false, false };
@@ -4170,7 +4172,7 @@ static void multiseedSearchWorker_2p5(void *vp) {
}
assert_gt(streak[0], 0);
// Increment counters according to what got filtered
- for(size_t mate = 0; mate < (pair ? 2:1); mate++) {
+ for(size_t mate = 0; mate < (paired ? 2:1); mate++) {
if(!filt[mate]) {
// Mate was rejected by N filter
olm.freads++; // reads filtered out
@@ -4181,9 +4183,9 @@ static void multiseedSearchWorker_2p5(void *vp) {
}
}
if(filt[0]) {
- ald.initRead(ps->bufa(), nofw[0], norc[0], minsc[0], maxpen[0], filt[1] ? &ps->bufb() : NULL);
+ ald.initRead(ps->read_a(), nofw[0], norc[0], minsc[0], maxpen[0], filt[1] ? &ps->read_b() : NULL);
} else if(filt[1]) {
- ald.initRead(ps->bufb(), nofw[1], norc[1], minsc[1], maxpen[1], NULL);
+ ald.initRead(ps->read_b(), nofw[1], norc[1], minsc[1], maxpen[1], NULL);
}
if(filt[0] || filt[1]) {
ald.go(sc, ebwtFw, ebwtBw, ref, descm, wlm, prm, rnd, msinkwrap);
@@ -4216,7 +4218,7 @@ static void multiseedSearchWorker_2p5(void *vp) {
}
if(metricsPerRead) {
MERGE_METRICS(metricsPt, nthreads > 1);
- nametmp = ps->bufa().name;
+ nametmp = ps->read_a().name;
metricsPt.reportInterval(
metricsOfb, metricsStderr, true, true, &nametmp);
metricsPt.reset();
@@ -4236,13 +4238,15 @@ static void multiseedSearchWorker_2p5(void *vp) {
*/
static void multiseedSearch(
Scoring& sc,
- PairedPatternSource& patsrc, // pattern source
- AlnSink& msink, // hit sink
+ const PatternParams& pp,
+ PatternComposer& patsrc, // pattern source
+ AlnSink& msink, // hit sink
Ebwt& ebwtFw, // index of original text
Ebwt& ebwtBw, // index of mirror text
OutFileBuf *metricsOfb)
{
multiseed_patsrc = &patsrc;
+ multiseed_pp = pp;
multiseed_msink = &msink;
multiseed_ebwtFw = &ebwtFw;
multiseed_ebwtBw = &ebwtBw;
@@ -4357,18 +4361,20 @@ static void driver(
format, // file format
fileParallel, // true -> wrap files with separate PairedPatternSources
seed, // pseudo-random seed
+ readsPerBatch, // # reads in a light parsing batch
solexaQuals, // true -> qualities are on solexa64 scale
phred64Quals, // true -> qualities are on phred64 scale
integerQuals, // true -> qualities are space-separated numbers
- fuzzy, // true -> try to parse fuzzy fastq
fastaContLen, // length of sampled reads for FastaContinuous...
fastaContFreq, // frequency of sampled reads for FastaContinuous...
- skipReads // skip the first 'skip' patterns
+ skipReads, // skip the first 'skip' patterns
+ nthreads, //number of threads for locking
+ outType != OUTPUT_SAM // whether to fix mate names
);
if(gVerbose || startVerbose) {
cerr << "Creating PatternSource: "; logTime(cerr, true);
}
- PairedPatternSource *patsrc = PairedPatternSource::setupPatternSources(
+ PatternComposer *patsrc = PatternComposer::setupPatternComposer(
queries, // singles, from argv
mates1, // mate1's, from -1 arg
mates2, // mate2's, from -2 arg
@@ -4507,8 +4513,6 @@ static void driver(
sam_print_xss,
sam_print_yn,
sam_print_xn,
- sam_print_cs,
- sam_print_cq,
sam_print_x0,
sam_print_x1,
sam_print_xm,
@@ -4576,6 +4580,7 @@ static void driver(
assert(mssink != NULL);
multiseedSearch(
sc, // scoring scheme
+ pp, // pattern params
*patsrc, // pattern source
*mssink, // hit sink
ebwt, // BWT
diff --git a/cpu_numa_info.cpp b/cpu_numa_info.cpp
new file mode 100644
index 0000000..fb5ea0b
--- /dev/null
+++ b/cpu_numa_info.cpp
@@ -0,0 +1,9 @@
+#include "cpu_numa_info.h"
+
+/// Based on http://stackoverflow.com/questions/16862620/numa-get-current-node-core
+void get_cpu_and_node_(int& cpu, int& node) {
+ unsigned long a,d,c;
+ __asm__ volatile("rdtscp" : "=a" (a), "=d" (d), "=c" (c));
+ node = (c & 0xFFF000)>>12;
+ cpu = c & 0xFFF;
+}
diff --git a/cpu_numa_info.h b/cpu_numa_info.h
new file mode 100644
index 0000000..1125df6
--- /dev/null
+++ b/cpu_numa_info.h
@@ -0,0 +1,6 @@
+#ifndef CPU_AND_NODE_H_
+#define CPU_AND_NODE_H_
+
+extern void get_cpu_and_node_(int& cpu, int& node);
+
+#endif
diff --git a/diff_sample.h b/diff_sample.h
index 8c1064f..91acc2f 100644
--- a/diff_sample.h
+++ b/diff_sample.h
@@ -638,7 +638,7 @@ void DifferenceCoverSample<TStr>::buildSPrime(
// mapping; where the mapping would leave a blank, insert a 0
TIndexOffU added = 0;
TIndexOffU i = 0;
- for(uint64_t ti = 0; ti <= tlen; ti += v) {
+ for(TIndexOffU ti = 0; ti <= tlen; ti += v) {
for(uint32_t di = 0; di < d; di++) {
TIndexOffU tti = ti + ds[di];
if(tti > tlen) break;
diff --git a/doc/images/bowtie_logo.png b/doc/images/bowtie_logo.png
new file mode 100644
index 0000000..381bcd9
Binary files /dev/null and b/doc/images/bowtie_logo.png differ
diff --git a/doc/images/osi-certified.gif b/doc/images/osi-certified.gif
new file mode 100644
index 0000000..fd1a07d
Binary files /dev/null and b/doc/images/osi-certified.gif differ
diff --git a/doc/images/sflogo.png b/doc/images/sflogo.png
new file mode 100644
index 0000000..dbacfdc
Binary files /dev/null and b/doc/images/sflogo.png differ
diff --git a/doc/images/university.small.horizontal.white.png b/doc/images/university.small.horizontal.white.png
new file mode 100644
index 0000000..e872adc
Binary files /dev/null and b/doc/images/university.small.horizontal.white.png differ
diff --git a/doc/manual.html b/doc/manual.html
index bb4b79c..0b2ba00 100644
--- a/doc/manual.html
+++ b/doc/manual.html
@@ -4,7 +4,7 @@
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
<meta http-equiv="Content-Style-Type" content="text/css" />
<meta name="generator" content="pandoc" />
- <title>Bowtie 2 Manual - </title>
+ <title>Bowtie 2 Manual – </title>
<style type="text/css">code{white-space: pre;}</style>
<link rel="stylesheet" href="style.css" type="text/css" />
</head>
@@ -93,7 +93,6 @@
! distracting formatting clutter. See 'MANUAL' for an easier-to-read version
! of this text document, or see the HTML manual online.
! -->
-
<h1 id="introduction">Introduction</h1>
<h2 id="what-is-bowtie-2">What is Bowtie 2?</h2>
<p><a href="http://bowtie-bio.sf.net/bowtie2">Bowtie 2</a> is an ultrafast and memory-efficient tool for aligning sequencing reads to long reference sequences. It is particularly good at aligning reads of about 50 up to 100s or 1,000s of characters to relatively long (e.g. mammalian) genomes. Bowtie 2 indexes the genome with an <a href="http://portal.acm.org/citation.cfm?id=796543">FM Index</a> (based on the <a href="http://en.wikipedia.org/wiki/Burrows-Wheeler_transform">Burrows-Wheeler [...]
@@ -125,7 +124,7 @@
<h2 id="building-from-source">Building from source</h2>
<p>Building Bowtie 2 from source requires a GNU-like environment with GCC, GNU Make and other basics. It should be possible to build Bowtie 2 on most vanilla Linux installations or on a Mac installation with <a href="http://developer.apple.com/xcode/">Xcode</a> installed. Bowtie 2 can also be built on Windows using a 64-bit MinGW distribution and MSYS. In order to simplify the MinGW setup it might be worth investigating popular MinGW personal builds since these are coming already prepare [...]
<p>First, download the source package from the <a href="https://sourceforge.net/projects/bowtie-bio/files/bowtie2/">sourceforge site</a>. Make sure you're getting the source package; the file downloaded should end in <code>-source.zip</code>. Unzip the file, change to the unzipped directory, and build the Bowtie 2 tools by running GNU <code>make</code> (usually with the command <code>make</code>, but sometimes with <code>gmake</code>) with no arguments. If building with MinGW, run <code> [...]
-<p>Bowtie 2 is using the multithreading software model in order to speed up execution times on SMP architectures where this is possible. On POSIX platforms (like linux, Mac OS, etc) it needs the pthread library. Although it is possible to use pthread library on non-POSIX platform like Windows, due to performance reasons bowtie 2 will try to use Windows native multithreading if possible. We recommend that you first install the <a href="https://www.threadingbuildingblocks.org">Threading Bu [...]
+<p>+Bowtie 2 is using the multithreading software model in order to +speed up execution times on SMP architectures where this is possible. +The Threading Building Blocks library, TBB, is now the default +threading library in bowtie2. On POSIX platforms (like linux, Mac +OS, etc) if TBB is not available the pthread library will be used. +Although it is possible to use pthread library on Windows, a non-POSIX +platform, due to performance reasons bowtie 2 will try to use Windows +native mul [...]
<h2 id="adding-to-path">Adding to PATH</h2>
<p>By adding your new Bowtie 2 directory to your <a href="http://en.wikipedia.org/wiki/PATH_(variable)">PATH environment variable</a>, you ensure that whenever you run <code>bowtie2</code>, <code>bowtie2-build</code> or <code>bowtie2-inspect</code> from the command line, you will get the version you just installed without having to specify the entire path. This is recommended for most users. To do this, follow your operating system's instructions for adding the directory to your <a href= [...]
<p>If you would like to install Bowtie 2 by copying the Bowtie 2 executable files to an existing directory in your <a href="http://en.wikipedia.org/wiki/PATH_(variable)">PATH</a>, make sure that you copy all the executables, including <code>bowtie2</code>, <code>bowtie2-align-s</code>, <code>bowtie2-align-l</code>, <code>bowtie2-build</code>, <code>bowtie2-build-s</code>, <code>bowtie2-build-l</code>, <code>bowtie2-inspect</code>, <code>bowtie2-inspect-s</code> and <code>bowtie2-inspect- [...]
@@ -305,218 +304,249 @@ Reference: GCAGATTATATGAGTCAGCTACGATATTGTTTGGGGTGACACATTACGCGTCTTTGAC</code></pr
<h3 id="usage">Usage</h3>
<pre><code>bowtie2 [options]* -x <bt2-idx> {-1 <m1> -2 <m2> | -U <r>} -S [<hit>]</code></pre>
<h3 id="main-arguments">Main arguments</h3>
-<table><tr><td>
-
+<table>
+<tr>
+<td>
<pre><code>-x <bt2-idx></code></pre>
-</td><td>
-
+</td>
+<td>
<p>The basename of the index for the reference genome. The basename is the name of any of the index files up to but not including the final <code>.1.bt2</code> / <code>.rev.1.bt2</code> / etc. <code>bowtie2</code> looks for the specified index first in the current directory, then in the directory specified in the <code>BOWTIE2_INDEXES</code> environment variable.</p>
-</td></tr><tr><td>
-
+</td>
+</tr>
+<tr>
+<td>
<pre><code>-1 <m1></code></pre>
-</td><td>
-
+</td>
+<td>
<p>Comma-separated list of files containing mate 1s (filename usually includes <code>_1</code>), e.g. <code>-1 flyA_1.fq,flyB_1.fq</code>. Sequences specified with this option must correspond file-for-file and read-for-read with those specified in <code><m2></code>. Reads may be a mix of different lengths. If <code>-</code> is specified, <code>bowtie2</code> will read the mate 1s from the "standard in" or "stdin" filehandle.</p>
-</td></tr><tr><td>
-
+</td>
+</tr>
+<tr>
+<td>
<pre><code>-2 <m2></code></pre>
-</td><td>
-
+</td>
+<td>
<p>Comma-separated list of files containing mate 2s (filename usually includes <code>_2</code>), e.g. <code>-2 flyA_2.fq,flyB_2.fq</code>. Sequences specified with this option must correspond file-for-file and read-for-read with those specified in <code><m1></code>. Reads may be a mix of different lengths. If <code>-</code> is specified, <code>bowtie2</code> will read the mate 2s from the "standard in" or "stdin" filehandle.</p>
-</td></tr><tr><td>
-
+</td>
+</tr>
+<tr>
+<td>
<pre><code>-U <r></code></pre>
-</td><td>
-
+</td>
+<td>
<p>Comma-separated list of files containing unpaired reads to be aligned, e.g. <code>lane1.fq,lane2.fq,lane3.fq,lane4.fq</code>. Reads may be a mix of different lengths. If <code>-</code> is specified, <code>bowtie2</code> gets the reads from the "standard in" or "stdin" filehandle.</p>
-</td></tr><tr><td>
-
+</td>
+</tr>
+<tr>
+<td>
<pre><code>-S <hit></code></pre>
-</td><td>
-
+</td>
+<td>
<p>File to write SAM alignments to. By default, alignments are written to the "standard out" or "stdout" filehandle (i.e. the console).</p>
-</td></tr></table>
-
+</td>
+</tr>
+</table>
<h3 id="options">Options</h3>
<h4 id="input-options">Input options</h4>
<table>
-<tr><td id="bowtie2-options-q">
-
+<tr>
+<td id="bowtie2-options-q">
<pre><code>-q</code></pre>
-</td><td>
-
+</td>
+<td>
<p>Reads (specified with <code><m1></code>, <code><m2></code>, <code><s></code>) are FASTQ files. FASTQ files usually have extension <code>.fq</code> or <code>.fastq</code>. FASTQ is the default format. See also: <a href="#bowtie2-options-solexa-quals"><code>--solexa-quals</code></a> and <a href="#bowtie2-options-int-quals"><code>--int-quals</code></a>.</p>
-</td></tr>
-<tr><td id="bowtie2-options-qseq">
-
+</td>
+</tr>
+<tr>
+<td id="bowtie2-options-qseq">
<pre><code>--qseq</code></pre>
-</td><td>
-
+</td>
+<td>
<p>Reads (specified with <code><m1></code>, <code><m2></code>, <code><s></code>) are QSEQ files. QSEQ files usually end in <code>_qseq.txt</code>. See also: <a href="#bowtie2-options-solexa-quals"><code>--solexa-quals</code></a> and <a href="#bowtie2-options-int-quals"><code>--int-quals</code></a>.</p>
-</td></tr>
-<tr><td id="bowtie2-options-f">
-
+</td>
+</tr>
+<tr>
+<td id="bowtie2-options-f">
<pre><code>-f</code></pre>
-</td><td>
-
+</td>
+<td>
<p>Reads (specified with <code><m1></code>, <code><m2></code>, <code><s></code>) are FASTA files. FASTA files usually have extension <code>.fa</code>, <code>.fasta</code>, <code>.mfa</code>, <code>.fna</code> or similar. FASTA files do not have a way of specifying quality values, so when <code>-f</code> is set, the result is as if <code>--ignore-quals</code> is also set.</p>
-</td></tr>
-<tr><td id="bowtie2-options-r">
-
+</td>
+</tr>
+<tr>
+<td id="bowtie2-options-r">
<pre><code>-r</code></pre>
-</td><td>
-
+</td>
+<td>
<p>Reads (specified with <code><m1></code>, <code><m2></code>, <code><s></code>) are files with one input sequence per line, without any other information (no read names, no qualities). When <code>-r</code> is set, the result is as if <code>--ignore-quals</code> is also set.</p>
-</td></tr>
-<tr><td id="bowtie2-options-c">
-
+</td>
+</tr>
+<tr>
+<td id="bowtie2-options-c">
<pre><code>-c</code></pre>
-</td><td>
-
+</td>
+<td>
<p>The read sequences are given on command line. I.e. <code><m1></code>, <code><m2></code> and <code><singles></code> are comma-separated lists of reads rather than lists of read files. There is no way to specify read names or qualities, so <code>-c</code> also implies <code>--ignore-quals</code>.</p>
-</td></tr>
-<tr><td id="bowtie2-options-s">
-
+</td>
+</tr>
+<tr>
+<td id="bowtie2-options-s">
<pre><code>-s/--skip <int></code></pre>
-</td><td>
-
+</td>
+<td>
<p>Skip (i.e. do not align) the first <code><int></code> reads or pairs in the input.</p>
-</td></tr>
-<tr><td id="bowtie2-options-u">
-
+</td>
+</tr>
+<tr>
+<td id="bowtie2-options-u">
<pre><code>-u/--qupto <int></code></pre>
-</td><td>
-
+</td>
+<td>
<p>Align the first <code><int></code> reads or read pairs from the input (after the <a href="#bowtie2-options-s"><code>-s</code>/<code>--skip</code></a> reads or pairs have been skipped), then stop. Default: no limit.</p>
-</td></tr>
-<tr><td id="bowtie2-options-5">
-
+</td>
+</tr>
+<tr>
+<td id="bowtie2-options-5">
<pre><code>-5/--trim5 <int></code></pre>
-</td><td>
-
+</td>
+<td>
<p>Trim <code><int></code> bases from 5' (left) end of each read before alignment (default: 0).</p>
-</td></tr>
-<tr><td id="bowtie2-options-3">
-
+</td>
+</tr>
+<tr>
+<td id="bowtie2-options-3">
<pre><code>-3/--trim3 <int></code></pre>
-</td><td>
-
+</td>
+<td>
<p>Trim <code><int></code> bases from 3' (right) end of each read before alignment (default: 0).</p>
-</td></tr><tr><td id="bowtie2-options-phred33-quals">
-
+</td>
+</tr>
+<tr>
+<td id="bowtie2-options-phred33-quals">
<pre><code>--phred33</code></pre>
-</td><td>
-
+</td>
+<td>
<p>Input qualities are ASCII chars equal to the <a href="http://en.wikipedia.org/wiki/Phred_quality_score">Phred quality</a> plus 33. This is also called the "Phred+33" encoding, which is used by the very latest Illumina pipelines.</p>
-</td></tr>
-<tr><td id="bowtie2-options-phred64-quals">
-
+</td>
+</tr>
+<tr>
+<td id="bowtie2-options-phred64-quals">
<pre><code>--phred64</code></pre>
-</td><td>
-
+</td>
+<td>
<p>Input qualities are ASCII chars equal to the <a href="http://en.wikipedia.org/wiki/Phred_quality_score">Phred quality</a> plus 64. This is also called the "Phred+64" encoding.</p>
-</td></tr>
-<tr><td id="bowtie2-options-solexa-quals">
-
+</td>
+</tr>
+<tr>
+<td id="bowtie2-options-solexa-quals">
<pre><code>--solexa-quals</code></pre>
-</td><td>
-
+</td>
+<td>
<p>Convert input qualities from <a href="http://en.wikipedia.org/wiki/Phred_quality_score">Solexa</a> (which can be negative) to <a href="http://en.wikipedia.org/wiki/Phred_quality_score">Phred</a> (which can't). This scheme was used in older Illumina GA Pipeline versions (prior to 1.3). Default: off.</p>
-</td></tr>
-<tr><td id="bowtie2-options-int-quals">
-
+</td>
+</tr>
+<tr>
+<td id="bowtie2-options-int-quals">
<pre><code>--int-quals</code></pre>
-</td><td>
-
+</td>
+<td>
<p>Quality values are represented in the read input file as space-separated ASCII integers, e.g., <code>40 40 30 40</code>..., rather than ASCII characters, e.g., <code>II?I</code>.... Integers are treated as being on the <a href="http://en.wikipedia.org/wiki/Phred_quality_score">Phred quality</a> scale unless <a href="#bowtie2-options-solexa-quals"><code>--solexa-quals</code></a> is also specified. Default: off.</p>
-</td></tr></table>
-
+</td>
+</tr>
+</table>
<h4 id="preset-options-in---end-to-end-mode">Preset options in <code>--end-to-end</code> mode</h4>
<table>
-<tr><td id="bowtie2-options-very-fast">
-
+<tr>
+<td id="bowtie2-options-very-fast">
<pre><code>--very-fast</code></pre>
-</td><td>
-
+</td>
+<td>
<p>Same as: <code>-D 5 -R 1 -N 0 -L 22 -i S,0,2.50</code></p>
-</td></tr>
-<tr><td id="bowtie2-options-fast">
-
+</td>
+</tr>
+<tr>
+<td id="bowtie2-options-fast">
<pre><code>--fast</code></pre>
-</td><td>
-
+</td>
+<td>
<p>Same as: <code>-D 10 -R 2 -N 0 -L 22 -i S,0,2.50</code></p>
-</td></tr>
-<tr><td id="bowtie2-options-sensitive">
-
+</td>
+</tr>
+<tr>
+<td id="bowtie2-options-sensitive">
<pre><code>--sensitive</code></pre>
-</td><td>
-
+</td>
+<td>
<p>Same as: <code>-D 15 -R 2 -L 22 -i S,1,1.15</code> (default in <a href="#bowtie2-options-end-to-end"><code>--end-to-end</code></a> mode)</p>
-</td></tr>
-<tr><td id="bowtie2-options-very-sensitive">
-
+</td>
+</tr>
+<tr>
+<td id="bowtie2-options-very-sensitive">
<pre><code>--very-sensitive</code></pre>
-</td><td>
-
+</td>
+<td>
<p>Same as: <code>-D 20 -R 3 -N 0 -L 20 -i S,1,0.50</code></p>
-</td></tr>
+</td>
+</tr>
</table>
-
<h4 id="preset-options-in---local-mode">Preset options in <code>--local</code> mode</h4>
<table>
-<tr><td id="bowtie2-options-very-fast-local">
-
+<tr>
+<td id="bowtie2-options-very-fast-local">
<pre><code>--very-fast-local</code></pre>
-</td><td>
-
+</td>
+<td>
<p>Same as: <code>-D 5 -R 1 -N 0 -L 25 -i S,1,2.00</code></p>
-</td></tr>
-<tr><td id="bowtie2-options-fast-local">
-
+</td>
+</tr>
+<tr>
+<td id="bowtie2-options-fast-local">
<pre><code>--fast-local</code></pre>
-</td><td>
-
+</td>
+<td>
<p>Same as: <code>-D 10 -R 2 -N 0 -L 22 -i S,1,1.75</code></p>
-</td></tr>
-<tr><td id="bowtie2-options-sensitive-local">
-
+</td>
+</tr>
+<tr>
+<td id="bowtie2-options-sensitive-local">
<pre><code>--sensitive-local</code></pre>
-</td><td>
-
+</td>
+<td>
<p>Same as: <code>-D 15 -R 2 -N 0 -L 20 -i S,1,0.75</code> (default in <a href="#bowtie2-options-local"><code>--local</code></a> mode)</p>
-</td></tr>
-<tr><td id="bowtie2-options-very-sensitive-local">
-
+</td>
+</tr>
+<tr>
+<td id="bowtie2-options-very-sensitive-local">
<pre><code>--very-sensitive-local</code></pre>
-</td><td>
-
+</td>
+<td>
<p>Same as: <code>-D 20 -R 3 -N 0 -L 20 -i S,1,0.50</code></p>
-</td></tr>
+</td>
+</tr>
</table>
-
<h4 id="alignment-options">Alignment options</h4>
<table>
-
-<tr><td id="bowtie2-options-N">
-
+<tr>
+<td id="bowtie2-options-N">
<pre><code>-N <int></code></pre>
-</td><td>
-
+</td>
+<td>
<p>Sets the number of mismatches to allowed in a seed alignment during <a href="#multiseed-heuristic">multiseed alignment</a>. Can be set to 0 or 1. Setting this higher makes alignment slower (often much slower) but increases sensitivity. Default: 0.</p>
-</td></tr>
-<tr><td id="bowtie2-options-L">
-
+</td>
+</tr>
+<tr>
+<td id="bowtie2-options-L">
<pre><code>-L <int></code></pre>
-</td><td>
-
+</td>
+<td>
<p>Sets the length of the seed substrings to align during <a href="#multiseed-heuristic">multiseed alignment</a>. Smaller values make alignment slower but more sensitive. Default: the <a href="#bowtie2-options-sensitive"><code>--sensitive</code></a> preset is used by default, which sets <code>-L</code> to 20 both in <a href="#bowtie2-options-end-to-end"><code>--end-to-end</code></a> mode and in <a href="#bowtie2-options-local"><code>--local</code></a> mode.</p>
-</td></tr>
-<tr><td id="bowtie2-options-i">
-
+</td>
+</tr>
+<tr>
+<td id="bowtie2-options-i">
<pre><code>-i <func></code></pre>
-</td><td>
-
+</td>
+<td>
<p>Sets a function governing the interval between seed substrings to use during <a href="#multiseed-heuristic">multiseed alignment</a>. For instance, if the read has 30 characters, and seed length is 10, and the seed interval is 6, the seeds extracted will be:</p>
<pre><code>Read: TAGCTACGCTCTACGCTATCATGCATAAAC
Seed 1 fw: TAGCTACGCT
@@ -528,415 +558,452 @@ Seed 3 rc: ATGATAGCGT
Seed 4 fw: TCATGCATAA
Seed 4 rc: TTATGCATGA</code></pre>
<p>Since it's best to use longer intervals for longer reads, this parameter sets the interval as a function of the read length, rather than a single one-size-fits-all number. For instance, specifying <code>-i S,1,2.5</code> sets the interval function <code>f</code> to <code>f(x) = 1 + 2.5 * sqrt(x)</code>, where x is the read length. See also: <a href="#setting-function-options">setting function options</a>. If the function returns a result less than 1, it is rounded up to 1. Default: th [...]
-</td></tr>
-<tr><td id="bowtie2-options-n-ceil">
-
+</td>
+</tr>
+<tr>
+<td id="bowtie2-options-n-ceil">
<pre><code>--n-ceil <func></code></pre>
-</td><td>
-
+</td>
+<td>
<p>Sets a function governing the maximum number of ambiguous characters (usually <code>N</code>s and/or <code>.</code>s) allowed in a read as a function of read length. For instance, specifying <code>-L,0,0.15</code> sets the N-ceiling function <code>f</code> to <code>f(x) = 0 + 0.15 * x</code>, where x is the read length. See also: <a href="#setting-function-options">setting function options</a>. Reads exceeding this ceiling are <a href="#filtering">filtered out</a>. Default: <code>L,0, [...]
-</td></tr>
-<tr><td id="bowtie2-options-dpad">
-
+</td>
+</tr>
+<tr>
+<td id="bowtie2-options-dpad">
<pre><code>--dpad <int></code></pre>
-</td><td>
-
+</td>
+<td>
<p>"Pads" dynamic programming problems by <code><int></code> columns on either side to allow gaps. Default: 15.</p>
-</td></tr>
-<tr><td id="bowtie2-options-gbar">
-
+</td>
+</tr>
+<tr>
+<td id="bowtie2-options-gbar">
<pre><code>--gbar <int></code></pre>
-</td><td>
-
+</td>
+<td>
<p>Disallow gaps within <code><int></code> positions of the beginning or end of the read. Default: 4.</p>
-</td></tr>
-<tr><td id="bowtie2-options-ignore-quals">
-
+</td>
+</tr>
+<tr>
+<td id="bowtie2-options-ignore-quals">
<pre><code>--ignore-quals</code></pre>
-</td><td>
-
+</td>
+<td>
<p>When calculating a mismatch penalty, always consider the quality value at the mismatched position to be the highest possible, regardless of the actual value. I.e. input is treated as though all quality values are high. This is also the default behavior when the input doesn't specify quality values (e.g. in <a href="#bowtie2-options-f"><code>-f</code></a>, <a href="#bowtie2-options-R"><code>-r</code></a>, or <a href="#bowtie2-options-c"><code>-c</code></a> modes).</p>
-</td></tr>
-<tr><td id="bowtie2-options-nofw">
-
+</td>
+</tr>
+<tr>
+<td id="bowtie2-options-nofw">
<pre><code>--nofw/--norc</code></pre>
-</td><td>
-
+</td>
+<td>
<p>If <code>--nofw</code> is specified, <code>bowtie2</code> will not attempt to align unpaired reads to the forward (Watson) reference strand. If <code>--norc</code> is specified, <code>bowtie2</code> will not attempt to align unpaired reads against the reverse-complement (Crick) reference strand. In paired-end mode, <code>--nofw</code> and <code>--norc</code> pertain to the fragments; i.e. specifying <code>--nofw</code> causes <code>bowtie2</code> to explore only those paired-end confi [...]
-</td></tr>
-<tr><td id="bowtie2-options-no-1mm-upfront">
-
+</td>
+</tr>
+<tr>
+<td id="bowtie2-options-no-1mm-upfront">
<pre><code>--no-1mm-upfront</code></pre>
-</td><td>
-
+</td>
+<td>
<p>By default, Bowtie 2 will attempt to find either an exact or a 1-mismatch end-to-end alignment for the read <em>before</em> trying the <a href="#multiseed-heuristic">multiseed heuristic</a>. Such alignments can be found very quickly, and many short read alignments have exact or near-exact end-to-end alignments. However, this can lead to unexpected alignments when the user also sets options governing the <a href="#multiseed-heuristic">multiseed heuristic</a>, like <a href="#bowtie2-opt [...]
-</td></tr><tr><td id="bowtie2-options-end-to-end">
-
+</td>
+</tr>
+<tr>
+<td id="bowtie2-options-end-to-end">
<pre><code>--end-to-end</code></pre>
-</td><td>
-
+</td>
+<td>
<p>In this mode, Bowtie 2 requires that the entire read align from one end to the other, without any trimming (or "soft clipping") of characters from either end. The match bonus <a href="#bowtie2-options-ma"><code>--ma</code></a> always equals 0 in this mode, so all alignment scores are less than or equal to 0, and the greatest possible alignment score is 0. This is mutually exclusive with <a href="#bowtie2-options-local"><code>--local</code></a>. <code>--end-to-end</code> is t [...]
-</td></tr>
-<tr><td id="bowtie2-options-local">
-
+</td>
+</tr>
+<tr>
+<td id="bowtie2-options-local">
<pre><code>--local</code></pre>
-</td><td>
-
+</td>
+<td>
<p>In this mode, Bowtie 2 does not require that the entire read align from one end to the other. Rather, some characters may be omitted ("soft clipped") from the ends in order to achieve the greatest possible alignment score. The match bonus <a href="#bowtie2-options-ma"><code>--ma</code></a> is used in this mode, and the best possible alignment score is equal to the match bonus (<a href="#bowtie2-options-ma"><code>--ma</code></a>) times the length of the read. Specifying <code [...]
-</td></tr>
+</td>
+</tr>
</table>
-
<h4 id="scoring-options">Scoring options</h4>
<table>
-
-<tr><td id="bowtie2-options-ma">
-
+<tr>
+<td id="bowtie2-options-ma">
<pre><code>--ma <int></code></pre>
-</td><td>
-
+</td>
+<td>
<p>Sets the match bonus. In <a href="#bowtie2-options-local"><code>--local</code></a> mode <code><int></code> is added to the alignment score for each position where a read character aligns to a reference character and the characters match. Not used in <a href="#bowtie2-options-end-to-end"><code>--end-to-end</code></a> mode. Default: 2.</p>
-</td></tr>
-<tr><td id="bowtie2-options-mp">
-
+</td>
+</tr>
+<tr>
+<td id="bowtie2-options-mp">
<pre><code>--mp MX,MN</code></pre>
-</td><td>
-
+</td>
+<td>
<p>Sets the maximum (<code>MX</code>) and minimum (<code>MN</code>) mismatch penalties, both integers. A number less than or equal to <code>MX</code> and greater than or equal to <code>MN</code> is subtracted from the alignment score for each position where a read character aligns to a reference character, the characters do not match, and neither is an <code>N</code>. If <a href="#bowtie2-options-ignore-quals"><code>--ignore-quals</code></a> is specified, the number subtracted quals <cod [...]
-</td></tr>
-<tr><td id="bowtie2-options-np">
-
+</td>
+</tr>
+<tr>
+<td id="bowtie2-options-np">
<pre><code>--np <int></code></pre>
-</td><td>
-
+</td>
+<td>
<p>Sets penalty for positions where the read, reference, or both, contain an ambiguous character such as <code>N</code>. Default: 1.</p>
-</td></tr>
-<tr><td id="bowtie2-options-rdg">
-
+</td>
+</tr>
+<tr>
+<td id="bowtie2-options-rdg">
<pre><code>--rdg <int1>,<int2></code></pre>
-</td><td>
-
+</td>
+<td>
<p>Sets the read gap open (<code><int1></code>) and extend (<code><int2></code>) penalties. A read gap of length N gets a penalty of <code><int1></code> + N * <code><int2></code>. Default: 5, 3.</p>
-</td></tr>
-<tr><td id="bowtie2-options-rfg">
-
+</td>
+</tr>
+<tr>
+<td id="bowtie2-options-rfg">
<pre><code>--rfg <int1>,<int2></code></pre>
-</td><td>
-
+</td>
+<td>
<p>Sets the reference gap open (<code><int1></code>) and extend (<code><int2></code>) penalties. A reference gap of length N gets a penalty of <code><int1></code> + N * <code><int2></code>. Default: 5, 3.</p>
-</td></tr>
-<tr><td id="bowtie2-options-score-min">
-
+</td>
+</tr>
+<tr>
+<td id="bowtie2-options-score-min">
<pre><code>--score-min <func></code></pre>
-</td><td>
-
+</td>
+<td>
<p>Sets a function governing the minimum alignment score needed for an alignment to be considered "valid" (i.e. good enough to report). This is a function of read length. For instance, specifying <code>L,0,-0.6</code> sets the minimum-score function <code>f</code> to <code>f(x) = 0 + -0.6 * x</code>, where <code>x</code> is the read length. See also: <a href="#setting-function-options">setting function options</a>. The default in <a href="#bowtie2-options-end-to-end"><code>--en [...]
-</td></tr>
+</td>
+</tr>
</table>
-
<h4 id="reporting-options">Reporting options</h4>
<table>
-
-<tr><td id="bowtie2-options-k">
-
+<tr>
+<td id="bowtie2-options-k">
<pre><code>-k <int></code></pre>
-</td><td>
-
+</td>
+<td>
<p>By default, <code>bowtie2</code> searches for distinct, valid alignments for each read. When it finds a valid alignment, it continues looking for alignments that are nearly as good or better. The best alignment found is reported (randomly selected from among best if tied). Information about the best alignments is used to estimate mapping quality and to set SAM optional fields, such as <a href="#bowtie2-build-opt-fields-as"><code>AS:i</code></a> and <a href="#bowtie2-build-opt-fields-x [...]
<p>When <code>-k</code> is specified, however, <code>bowtie2</code> behaves differently. Instead, it searches for at most <code><int></code> distinct, valid alignments for each read. The search terminates when it can't find more distinct valid alignments, or when it finds <code><int></code>, whichever happens first. All alignments found are reported in descending order by alignment score. The alignment score for a paired-end alignment equals the sum of the alignment scores of [...]
<p>Note: Bowtie 2 is not designed with large values for <code>-k</code> in mind, and when aligning reads to long, repetitive genomes large <code>-k</code> can be very, very slow.</p>
-</td></tr>
-<tr><td id="bowtie2-options-a">
-
+</td>
+</tr>
+<tr>
+<td id="bowtie2-options-a">
<pre><code>-a</code></pre>
-</td><td>
-
+</td>
+<td>
<p>Like <a href="#bowtie2-options-k"><code>-k</code></a> but with no upper limit on number of alignments to search for. <code>-a</code> is mutually exclusive with <a href="#bowtie2-options-k"><code>-k</code></a>.</p>
<p>Note: Bowtie 2 is not designed with <code>-a</code> mode in mind, and when aligning reads to long, repetitive genomes this mode can be very, very slow.</p>
-</td></tr>
+</td>
+</tr>
</table>
-
<h4 id="effort-options">Effort options</h4>
<table>
-
-<tr><td id="bowtie2-options-D">
-
+<tr>
+<td id="bowtie2-options-D">
<pre><code>-D <int></code></pre>
-</td><td>
-
+</td>
+<td>
<p>Up to <code><int></code> consecutive seed extension attempts can "fail" before Bowtie 2 moves on, using the alignments found so far. A seed extension "fails" if it does not yield a new best or a new second-best alignment. This limit is automatically adjusted up when -k or -a are specified. Default: 15.</p>
-</td></tr>
-<tr><td id="bowtie2-options-R">
-
+</td>
+</tr>
+<tr>
+<td id="bowtie2-options-R">
<pre><code>-R <int></code></pre>
-</td><td>
-
+</td>
+<td>
<p><code><int></code> is the maximum number of times Bowtie 2 will "re-seed" reads with repetitive seeds. When "re-seeding," Bowtie 2 simply chooses a new set of reads (same length, same number of mismatches allowed) at different offsets and searches for more alignments. A read is considered to have repetitive seeds if the total number of seed hits divided by the number of seeds that aligned at least once is greater than 300. Default: 2.</p>
-</td></tr>
+</td>
+</tr>
</table>
-
<h4 id="paired-end-options">Paired-end options</h4>
<table>
-
-<tr><td id="bowtie2-options-I">
-
+<tr>
+<td id="bowtie2-options-I">
<pre><code>-I/--minins <int></code></pre>
-</td><td>
-
+</td>
+<td>
<p>The minimum fragment length for valid paired-end alignments. E.g. if <code>-I 60</code> is specified and a paired-end alignment consists of two 20-bp alignments in the appropriate orientation with a 20-bp gap between them, that alignment is considered valid (as long as <a href="#bowtie2-options-X"><code>-X</code></a> is also satisfied). A 19-bp gap would not be valid in that case. If trimming options <a href="#bowtie2-options-3"><code>-3</code></a> or <a href="#bowtie2-options-5"><cod [...]
<p>The larger the difference between <a href="#bowtie2-options-I"><code>-I</code></a> and <a href="#bowtie2-options-X"><code>-X</code></a>, the slower Bowtie 2 will run. This is because larger differences bewteen <a href="#bowtie2-options-I"><code>-I</code></a> and <a href="#bowtie2-options-X"><code>-X</code></a> require that Bowtie 2 scan a larger window to determine if a concordant alignment exists. For typical fragment length ranges (200 to 400 nucleotides), Bowtie 2 is very efficient.</p>
<p>Default: 0 (essentially imposing no minimum)</p>
-</td></tr>
-<tr><td id="bowtie2-options-X">
-
+</td>
+</tr>
+<tr>
+<td id="bowtie2-options-X">
<pre><code>-X/--maxins <int></code></pre>
-</td><td>
-
+</td>
+<td>
<p>The maximum fragment length for valid paired-end alignments. E.g. if <code>-X 100</code> is specified and a paired-end alignment consists of two 20-bp alignments in the proper orientation with a 60-bp gap between them, that alignment is considered valid (as long as <a href="#bowtie2-options-I"><code>-I</code></a> is also satisfied). A 61-bp gap would not be valid in that case. If trimming options <a href="#bowtie2-options-3"><code>-3</code></a> or <a href="#bowtie2-options-5"><code>-5 [...]
<p>The larger the difference between <a href="#bowtie2-options-I"><code>-I</code></a> and <a href="#bowtie2-options-X"><code>-X</code></a>, the slower Bowtie 2 will run. This is because larger differences bewteen <a href="#bowtie2-options-I"><code>-I</code></a> and <a href="#bowtie2-options-X"><code>-X</code></a> require that Bowtie 2 scan a larger window to determine if a concordant alignment exists. For typical fragment length ranges (200 to 400 nucleotides), Bowtie 2 is very efficient.</p>
<p>Default: 500.</p>
-</td></tr>
-<tr><td id="bowtie2-options-fr">
-
+</td>
+</tr>
+<tr>
+<td id="bowtie2-options-fr">
<pre><code>--fr/--rf/--ff</code></pre>
-</td><td>
-
+</td>
+<td>
<p>The upstream/downstream mate orientations for a valid paired-end alignment against the forward reference strand. E.g., if <code>--fr</code> is specified and there is a candidate paired-end alignment where mate 1 appears upstream of the reverse complement of mate 2 and the fragment length constraints (<a href="#bowtie2-options-I"><code>-I</code></a> and <a href="#bowtie2-options-X"><code>-X</code></a>) are met, that alignment is valid. Also, if mate 2 appears upstream of the reverse co [...]
-</td></tr>
-<tr><td id="bowtie2-options-no-mixed">
-
+</td>
+</tr>
+<tr>
+<td id="bowtie2-options-no-mixed">
<pre><code>--no-mixed</code></pre>
-</td><td>
-
+</td>
+<td>
<p>By default, when <code>bowtie2</code> cannot find a concordant or discordant alignment for a pair, it then tries to find alignments for the individual mates. This option disables that behavior.</p>
-</td></tr>
-<tr><td id="bowtie2-options-no-discordant">
-
+</td>
+</tr>
+<tr>
+<td id="bowtie2-options-no-discordant">
<pre><code>--no-discordant</code></pre>
-</td><td>
-
+</td>
+<td>
<p>By default, <code>bowtie2</code> looks for discordant alignments if it cannot find any concordant alignments. A discordant alignment is an alignment where both mates align uniquely, but that does not satisfy the paired-end constraints (<a href="#bowtie2-options-fr"><code>--fr</code>/<code>--rf</code>/<code>--ff</code></a>, <a href="#bowtie2-options-I"><code>-I</code></a>, <a href="#bowtie2-options-X"><code>-X</code></a>). This option disables that behavior.</p>
-</td></tr>
-<tr><td id="bowtie2-options-dovetail">
-
+</td>
+</tr>
+<tr>
+<td id="bowtie2-options-dovetail">
<pre><code>--dovetail</code></pre>
-</td><td>
-
+</td>
+<td>
<p>If the mates "dovetail", that is if one mate alignment extends past the beginning of the other such that the wrong mate begins upstream, consider that to be concordant. See also: <a href="#mates-can-overlap-contain-or-dovetail-each-other">Mates can overlap, contain or dovetail each other</a>. Default: mates cannot dovetail in a concordant alignment.</p>
-</td></tr>
-<tr><td id="bowtie2-options-no-contain">
-
+</td>
+</tr>
+<tr>
+<td id="bowtie2-options-no-contain">
<pre><code>--no-contain</code></pre>
-</td><td>
-
+</td>
+<td>
<p>If one mate alignment contains the other, consider that to be non-concordant. See also: <a href="#mates-can-overlap-contain-or-dovetail-each-other">Mates can overlap, contain or dovetail each other</a>. Default: a mate can contain the other in a concordant alignment.</p>
-</td></tr>
-<tr><td id="bowtie2-options-no-overlap">
-
+</td>
+</tr>
+<tr>
+<td id="bowtie2-options-no-overlap">
<pre><code>--no-overlap</code></pre>
-</td><td>
-
+</td>
+<td>
<p>If one mate alignment overlaps the other at all, consider that to be non-concordant. See also: <a href="#mates-can-overlap-contain-or-dovetail-each-other">Mates can overlap, contain or dovetail each other</a>. Default: mates can overlap in a concordant alignment.</p>
-</td></tr></table>
-
+</td>
+</tr>
+</table>
<h4 id="output-options">Output options</h4>
<table>
-
-<tr><td id="bowtie2-options-t">
-
+<tr>
+<td id="bowtie2-options-t">
<pre><code>-t/--time</code></pre>
-</td><td>
-
+</td>
+<td>
<p>Print the wall-clock time required to load the index files and align the reads. This is printed to the "standard error" ("stderr") filehandle. Default: off.</p>
-</td></tr>
-<tr><td id="bowtie2-options-un">
-
+</td>
+</tr>
+<tr>
+<td id="bowtie2-options-un">
<pre><code>--un <path>
--un-gz <path>
--un-bz2 <path>
--un-lz4 <path></code></pre>
-</td><td>
-
+</td>
+<td>
<p>Write unpaired reads that fail to align to file at <code><path></code>. These reads correspond to the SAM records with the FLAGS <code>0x4</code> bit set and neither the <code>0x40</code> nor <code>0x80</code> bits set. If <code>--un-gz</code> is specified, output will be gzip compressed. If <code>--un-bz2</code> or <code>--un-lz4</code> is specified, output will be bzip2 or lz4 compressed. Reads written in this way will appear exactly as they did in the input file, without any [...]
-</td></tr>
-<tr><td id="bowtie2-options-al">
-
+</td>
+</tr>
+<tr>
+<td id="bowtie2-options-al">
<pre><code>--al <path>
--al-gz <path>
--al-bz2 <path>
--al-lz4 <path></code></pre>
-</td><td>
-
+</td>
+<td>
<p>Write unpaired reads that align at least once to file at <code><path></code>. These reads correspond to the SAM records with the FLAGS <code>0x4</code>, <code>0x40</code>, and <code>0x80</code> bits unset. If <code>--al-gz</code> is specified, output will be gzip compressed. If <code>--al-bz2</code> is specified, output will be bzip2 compressed. Similarly if <code>--al-lz4</code> is specified, output will be lz4 compressed. Reads written in this way will appear exactly as they d [...]
-</td></tr>
-<tr><td id="bowtie2-options-un-conc">
-
+</td>
+</tr>
+<tr>
+<td id="bowtie2-options-un-conc">
<pre><code>--un-conc <path>
--un-conc-gz <path>
--un-conc-bz2 <path>
--un-conc-lz4 <path></code></pre>
-</td><td>
-
+</td>
+<td>
<p>Write paired-end reads that fail to align concordantly to file(s) at <code><path></code>. These reads correspond to the SAM records with the FLAGS <code>0x4</code> bit set and either the <code>0x40</code> or <code>0x80</code> bit set (depending on whether it's mate #1 or #2). <code>.1</code> and <code>.2</code> strings are added to the filename to distinguish which file contains mate #1 and mate #2. If a percent symbol, <code>%</code>, is used in <code><path></code>, the p [...]
-</td></tr>
-<tr><td id="bowtie2-options-al-conc">
-
+</td>
+</tr>
+<tr>
+<td id="bowtie2-options-al-conc">
<pre><code>--al-conc <path>
--al-conc-gz <path>
--al-conc-bz2 <path>
--al-conc-lz4 <path></code></pre>
-</td><td>
-
+</td>
+<td>
<p>Write paired-end reads that align concordantly at least once to file(s) at <code><path></code>. These reads correspond to the SAM records with the FLAGS <code>0x4</code> bit unset and either the <code>0x40</code> or <code>0x80</code> bit set (depending on whether it's mate #1 or #2). <code>.1</code> and <code>.2</code> strings are added to the filename to distinguish which file contains mate #1 and mate #2. If a percent symbol, <code>%</code>, is used in <code><path></code [...]
-</td></tr>
-<tr><td id="bowtie2-options-quiet">
-
+</td>
+</tr>
+<tr>
+<td id="bowtie2-options-quiet">
<pre><code>--quiet</code></pre>
-</td><td>
-
+</td>
+<td>
<p>Print nothing besides alignments and serious errors.</p>
-</td></tr>
-<tr><td id="bowtie2-options-met-file">
-
+</td>
+</tr>
+<tr>
+<td id="bowtie2-options-met-file">
<pre><code>--met-file <path></code></pre>
-</td><td>
-
+</td>
+<td>
<p>Write <code>bowtie2</code> metrics to file <code><path></code>. Having alignment metric can be useful for debugging certain problems, especially performance issues. See also: <a href="#bowtie2-options-met"><code>--met</code></a>. Default: metrics disabled.</p>
-</td></tr>
-<tr><td id="bowtie2-options-met-stderr">
-
+</td>
+</tr>
+<tr>
+<td id="bowtie2-options-met-stderr">
<pre><code>--met-stderr <path></code></pre>
-</td><td>
-
+</td>
+<td>
<p>Write <code>bowtie2</code> metrics to the "standard error" ("stderr") filehandle. This is not mutually exclusive with <a href="#bowtie2-options-met-file"><code>--met-file</code></a>. Having alignment metric can be useful for debugging certain problems, especially performance issues. See also: <a href="#bowtie2-options-met"><code>--met</code></a>. Default: metrics disabled.</p>
-</td></tr>
-<tr><td id="bowtie2-options-met">
-
+</td>
+</tr>
+<tr>
+<td id="bowtie2-options-met">
<pre><code>--met <int></code></pre>
-</td><td>
-
+</td>
+<td>
<p>Write a new <code>bowtie2</code> metrics record every <code><int></code> seconds. Only matters if either <a href="#bowtie2-options-met-stderr"><code>--met-stderr</code></a> or <a href="#bowtie2-options-met-file"><code>--met-file</code></a> are specified. Default: 1.</p>
-</td></tr>
+</td>
+</tr>
</table>
-
<h4 id="sam-options">SAM options</h4>
<table>
-
-<tr><td id="bowtie2-options-no-unal">
-
+<tr>
+<td id="bowtie2-options-no-unal">
<pre><code>--no-unal</code></pre>
-</td><td>
-
+</td>
+<td>
<p>Suppress SAM records for reads that failed to align.</p>
-</td></tr>
-<tr><td id="bowtie2-options-no-hd">
-
+</td>
+</tr>
+<tr>
+<td id="bowtie2-options-no-hd">
<pre><code>--no-hd</code></pre>
-</td><td>
-
+</td>
+<td>
<p>Suppress SAM header lines (starting with <code>@</code>).</p>
-</td></tr>
-<tr><td id="bowtie2-options-no-sq">
-
+</td>
+</tr>
+<tr>
+<td id="bowtie2-options-no-sq">
<pre><code>--no-sq</code></pre>
-</td><td>
-
+</td>
+<td>
<p>Suppress <code>@SQ</code> SAM header lines.</p>
-</td></tr>
-<tr><td id="bowtie2-options-rg-id">
-
+</td>
+</tr>
+<tr>
+<td id="bowtie2-options-rg-id">
<pre><code>--rg-id <text></code></pre>
-</td><td>
-
+</td>
+<td>
<p>Set the read group ID to <code><text></code>. This causes the SAM <code>@RG</code> header line to be printed, with <code><text></code> as the value associated with the <code>ID:</code> tag. It also causes the <code>RG:Z:</code> extra field to be attached to each SAM output record, with value set to <code><text></code>.</p>
-</td></tr>
-<tr><td id="bowtie2-options-rg">
-
+</td>
+</tr>
+<tr>
+<td id="bowtie2-options-rg">
<pre><code>--rg <text></code></pre>
-</td><td>
-
+</td>
+<td>
<p>Add <code><text></code> (usually of the form <code>TAG:VAL</code>, e.g. <code>SM:Pool1</code>) as a field on the <code>@RG</code> header line. Note: in order for the <code>@RG</code> line to appear, <a href="#bowtie2-options-rg-id"><code>--rg-id</code></a> must also be specified. This is because the <code>ID</code> tag is required by the <a href="http://samtools.sourceforge.net/SAM1.pdf">SAM Spec</a>. Specify <code>--rg</code> multiple times to set multiple fields. See the <a hr [...]
-</td></tr>
-<tr><td id="bowtie2-options-omit-sec-seq">
-
+</td>
+</tr>
+<tr>
+<td id="bowtie2-options-omit-sec-seq">
<pre><code>--omit-sec-seq</code></pre>
-</td><td>
-
+</td>
+<td>
<p>When printing secondary alignments, Bowtie 2 by default will write out the <code>SEQ</code> and <code>QUAL</code> strings. Specifying this option causes Bowtie 2 to print an asterix in those fields instead.</p>
-</td></tr>
-
-
+</td>
+</tr>
</table>
-
<h4 id="performance-options">Performance options</h4>
-<table><tr>
-
+<table>
+<tr>
<td id="bowtie2-options-o">
-
<pre><code>-o/--offrate <int></code></pre>
-</td><td>
-
+</td>
+<td>
<p>Override the offrate of the index with <code><int></code>. If <code><int></code> is greater than the offrate used to build the index, then some row markings are discarded when the index is read into memory. This reduces the memory footprint of the aligner but requires more time to calculate text offsets. <code><int></code> must be greater than the value used to build the index.</p>
-</td></tr>
-<tr><td id="bowtie2-options-p">
-
+</td>
+</tr>
+<tr>
+<td id="bowtie2-options-p">
<pre><code>-p/--threads NTHREADS</code></pre>
-</td><td>
-
+</td>
+<td>
<p>Launch <code>NTHREADS</code> parallel search threads (default: 1). Threads will run on separate processors/cores and synchronize when parsing reads and outputting alignments. Searching for alignments is highly parallel, and speedup is close to linear. Increasing <code>-p</code> increases Bowtie 2's memory footprint. E.g. when aligning to a human genome index, increasing <code>-p</code> from 1 to 8 increases the memory footprint by a few hundred megabytes. This option is only available [...]
-</td></tr>
-<tr><td id="bowtie2-options-reorder">
-
+</td>
+</tr>
+<tr>
+<td id="bowtie2-options-reorder">
<pre><code>--reorder</code></pre>
-</td><td>
-
+</td>
+<td>
<p>Guarantees that output SAM records are printed in an order corresponding to the order of the reads in the original input file, even when <a href="#bowtie2-options-p"><code>-p</code></a> is set greater than 1. Specifying <code>--reorder</code> and setting <a href="#bowtie2-options-p"><code>-p</code></a> greater than 1 causes Bowtie 2 to run somewhat slower and use somewhat more memory then if <code>--reorder</code> were not specified. Has no effect if <a href="#bowtie2-options-p"><code [...]
-</td></tr>
-<tr><td id="bowtie2-options-mm">
-
+</td>
+</tr>
+<tr>
+<td id="bowtie2-options-mm">
<pre><code>--mm</code></pre>
-</td><td>
-
+</td>
+<td>
<p>Use memory-mapped I/O to load the index, rather than typical file I/O. Memory-mapping allows many concurrent <code>bowtie</code> processes on the same computer to share the same memory image of the index (i.e. you pay the memory overhead just once). This facilitates memory-efficient parallelization of <code>bowtie</code> in situations where using <a href="#bowtie2-options-p"><code>-p</code></a> is not possible or not preferable.</p>
-</td></tr></table>
-
+</td>
+</tr>
+</table>
<h4 id="other-options">Other options</h4>
<table>
-<tr><td id="bowtie2-options-qc-filter">
-
+<tr>
+<td id="bowtie2-options-qc-filter">
<pre><code>--qc-filter</code></pre>
-</td><td>
-
+</td>
+<td>
<p>Filter out reads for which the QSEQ filter field is non-zero. Only has an effect when read format is <a href="#bowtie2-options-qseq"><code>--qseq</code></a>. Default: off.</p>
-</td></tr>
-<tr><td id="bowtie2-options-seed">
-
+</td>
+</tr>
+<tr>
+<td id="bowtie2-options-seed">
<pre><code>--seed <int></code></pre>
-</td><td>
-
+</td>
+<td>
<p>Use <code><int></code> as the seed for pseudo-random number generator. Default: 0.</p>
-</td></tr>
-<tr><td id="bowtie2-options-non-deterministic">
-
+</td>
+</tr>
+<tr>
+<td id="bowtie2-options-non-deterministic">
<pre><code>--non-deterministic</code></pre>
-</td><td>
-
+</td>
+<td>
<p>Normally, Bowtie 2 re-initializes its pseudo-random generator for each read. It seeds the generator with a number derived from (a) the read name, (b) the nucleotide sequence, (c) the quality sequence, (d) the value of the <a href="#bowtie2-options-seed"><code>--seed</code></a> option. This means that if two reads are identical (same name, same nucleotides, same qualities) Bowtie 2 will find and report the same alignment(s) for both, even if there was ambiguity. When <code>--non-determ [...]
-</td></tr>
-<tr><td id="bowtie2-options-version">
-
+</td>
+</tr>
+<tr>
+<td id="bowtie2-options-version">
<pre><code>--version</code></pre>
-</td><td>
-
+</td>
+<td>
<p>Print version information and quit.</p>
-</td></tr>
-<tr><td id="bowtie2-options-h">
-
+</td>
+</tr>
+<tr>
+<td id="bowtie2-options-h">
<pre><code>-h/--help</code></pre>
-</td><td>
-
+</td>
+<td>
<p>Print usage information and quit.</p>
-</td></tr></table>
-
+</td>
+</tr>
+</table>
<h2 id="sam-output">SAM output</h2>
<p>Following is a brief description of the <a href="http://samtools.sourceforge.net/SAM1.pdf">SAM</a> format as output by <code>bowtie2</code>. For more details, see the <a href="http://samtools.sourceforge.net/SAM1.pdf">SAM format specification</a>.</p>
<p>By default, <code>bowtie2</code> prints a SAM header with <code>@HD</code>, <code>@SQ</code> and <code>@PG</code> lines. When one or more <a href="#bowtie2-options-rg"><code>--rg</code></a> arguments are specified, <code>bowtie2</code> will also print an <code>@RG</code> line that includes all user-specified <a href="#bowtie2-options-rg"><code>--rg</code></a> tokens separated by tabs.</p>
@@ -945,56 +1012,72 @@ Seed 4 rc: TTATGCATGA</code></pre>
<li><p>Name of read that aligned.</p>
<p>Note that the <a href="http://samtools.sourceforge.net/SAM1.pdf">SAM specification</a> disallows whitespace in the read name. If the read name contains any whitespace characters, Bowtie 2 will truncate the name at the first whitespace character. This is similar to the behavior of other tools.</p></li>
<li><p>Sum of all applicable flags. Flags relevant to Bowtie are:</p>
-<table><tr><td>
-
+<table>
+<tr>
+<td>
<pre><code>1</code></pre>
-</td><td>
-
+</td>
+<td>
<p>The read is one of a pair</p>
-</td></tr><tr><td>
-
+</td>
+</tr>
+<tr>
+<td>
<pre><code>2</code></pre>
-</td><td>
-
+</td>
+<td>
<p>The alignment is one end of a proper paired-end alignment</p>
-</td></tr><tr><td>
-
+</td>
+</tr>
+<tr>
+<td>
<pre><code>4</code></pre>
-</td><td>
-
+</td>
+<td>
<p>The read has no reported alignments</p>
-</td></tr><tr><td>
-
+</td>
+</tr>
+<tr>
+<td>
<pre><code>8</code></pre>
-</td><td>
-
+</td>
+<td>
<p>The read is one of a pair and has no reported alignments</p>
-</td></tr><tr><td>
-
+</td>
+</tr>
+<tr>
+<td>
<pre><code>16</code></pre>
-</td><td>
-
+</td>
+<td>
<p>The alignment is to the reverse reference strand</p>
-</td></tr><tr><td>
-
+</td>
+</tr>
+<tr>
+<td>
<pre><code>32</code></pre>
-</td><td>
-
+</td>
+<td>
<p>The other mate in the paired-end alignment is aligned to the reverse reference strand</p>
-</td></tr><tr><td>
-
+</td>
+</tr>
+<tr>
+<td>
<pre><code>64</code></pre>
-</td><td>
-
+</td>
+<td>
<p>The read is mate 1 in a pair</p>
-</td></tr><tr><td>
-
+</td>
+</tr>
+<tr>
+<td>
<pre><code>128</code></pre>
-</td><td>
-
+</td>
+<td>
<p>The read is mate 2 in a pair</p>
-</td></tr></table>
-
+</td>
+</tr>
+</table>
<p>Thus, an unpaired read that aligns to the reverse reference strand will have flag 16. A paired-end read that aligns and is the first mate in the pair will have flag 83 (= 64 + 16 + 2 + 1).</p></li>
<li><p>Name of reference sequence where alignment occurs</p></li>
<li><p>1-based offset into the forward reference strand where leftmost character of the alignment occurs</p></li>
@@ -1008,82 +1091,118 @@ Seed 4 rc: TTATGCATGA</code></pre>
<li><p>Optional fields. Fields are tab-separated. <code>bowtie2</code> outputs zero or more of these optional fields for each alignment, depending on the type of the alignment:</p></li>
</ol>
<table>
-<tr><td id="bowtie2-build-opt-fields-as">
+<tr>
+<td id="bowtie2-build-opt-fields-as">
<pre><code> AS:i:<N></code></pre>
</td>
<td>
-
-Alignment score. Can be negative. Can be greater than 0 in <a href="#bowtie2-options-local"><code>--local</code></a> mode (but not in <a href="#bowtie2-options-end-to-end"><code>--end-to-end</code></a> mode). Only present if SAM record is for an aligned read.
-</td></tr>
-<tr><td id="bowtie2-build-opt-fields-xs">
+<pre><code>Alignment score. Can be negative. Can be greater than 0 in [`--local`]
+mode (but not in [`--end-to-end`] mode). Only present if SAM record is for
+an aligned read.</code></pre>
+</td>
+</tr>
+<tr>
+<td id="bowtie2-build-opt-fields-xs">
<pre><code> XS:i:<N></code></pre>
</td>
<td>
-
-Alignment score for the best-scoring alignment found other than the alignment reported. Can be negative. Can be greater than 0 in <a href="#bowtie2-options-local"><code>--local</code></a> mode (but not in <a href="#bowtie2-options-end-to-end"><code>--end-to-end</code></a> mode). Only present if the SAM record is for an aligned read and more than one alignment was found for the read. Note that, when the read is part of a concordantly-aligned pair, this score could be greater than <a href= [...]
-</td></tr>
-<tr><td id="bowtie2-build-opt-fields-ys">
+<pre><code>Alignment score for the best-scoring alignment found other than the
+alignment reported. Can be negative. Can be greater than 0 in [`--local`]
+mode (but not in [`--end-to-end`] mode). Only present if the SAM record is
+for an aligned read and more than one alignment was found for the read.
+Note that, when the read is part of a concordantly-aligned pair, this score
+could be greater than [`AS:i`].</code></pre>
+</td>
+</tr>
+<tr>
+<td id="bowtie2-build-opt-fields-ys">
<pre><code> YS:i:<N></code></pre>
</td>
<td>
-
-Alignment score for opposite mate in the paired-end alignment. Only present if the SAM record is for a read that aligned as part of a paired-end alignment.
-</td></tr>
-<tr><td id="bowtie2-build-opt-fields-xn">
+<pre><code>Alignment score for opposite mate in the paired-end alignment. Only present
+if the SAM record is for a read that aligned as part of a paired-end
+alignment.</code></pre>
+</td>
+</tr>
+<tr>
+<td id="bowtie2-build-opt-fields-xn">
<pre><code> XN:i:<N></code></pre>
</td>
<td>
-
-The number of ambiguous bases in the reference covering this alignment. Only present if SAM record is for an aligned read.
-</td></tr>
-<tr><td id="bowtie2-build-opt-fields-xm">
+<pre><code>The number of ambiguous bases in the reference covering this alignment.
+Only present if SAM record is for an aligned read.</code></pre>
+</td>
+</tr>
+<tr>
+<td id="bowtie2-build-opt-fields-xm">
<pre><code> XM:i:<N></code></pre>
</td>
<td>
-
-The number of mismatches in the alignment. Only present if SAM record is for an aligned read.
-</td></tr>
-<tr><td id="bowtie2-build-opt-fields-xo">
+<pre><code>The number of mismatches in the alignment. Only present if SAM record is
+for an aligned read.</code></pre>
+</td>
+</tr>
+<tr>
+<td id="bowtie2-build-opt-fields-xo">
<pre><code> XO:i:<N></code></pre>
</td>
<td>
-
-The number of gap opens, for both read and reference gaps, in the alignment. Only present if SAM record is for an aligned read.
-</td></tr>
-<tr><td id="bowtie2-build-opt-fields-xg">
+<pre><code>The number of gap opens, for both read and reference gaps, in the alignment.
+Only present if SAM record is for an aligned read.</code></pre>
+</td>
+</tr>
+<tr>
+<td id="bowtie2-build-opt-fields-xg">
<pre><code> XG:i:<N></code></pre>
</td>
<td>
-
-The number of gap extensions, for both read and reference gaps, in the alignment. Only present if SAM record is for an aligned read.
-</td></tr>
-<tr><td id="bowtie2-build-opt-fields-nm">
+<pre><code>The number of gap extensions, for both read and reference gaps, in the
+alignment. Only present if SAM record is for an aligned read.</code></pre>
+</td>
+</tr>
+<tr>
+<td id="bowtie2-build-opt-fields-nm">
<pre><code> NM:i:<N></code></pre>
</td>
<td>
-
-The edit distance; that is, the minimal number of one-nucleotide edits (substitutions, insertions and deletions) needed to transform the read string into the reference string. Only present if SAM record is for an aligned read.
-</td></tr>
-<tr><td id="bowtie2-build-opt-fields-yf">
+<pre><code>The edit distance; that is, the minimal number of one-nucleotide edits
+(substitutions, insertions and deletions) needed to transform the read
+string into the reference string. Only present if SAM record is for an
+aligned read.</code></pre>
+</td>
+</tr>
+<tr>
+<td id="bowtie2-build-opt-fields-yf">
<pre><code> YF:Z:<S></code></pre>
-</td><td>
-
-String indicating reason why the read was filtered out. See also: <a href="#filtering">Filtering</a>. Only appears for reads that were filtered out.
-</td></tr>
-<tr><td id="bowtie2-build-opt-fields-yt">
+</td>
+<td>
+<pre><code>String indicating reason why the read was filtered out. See also:
+[Filtering]. Only appears for reads that were filtered out.</code></pre>
+</td>
+</tr>
+<tr>
+<td id="bowtie2-build-opt-fields-yt">
<pre><code> YT:Z:<S></code></pre>
-</td><td>
-
-Value of <code>UU</code> indicates the read was not part of a pair. Value of <code>CP</code> indicates the read was part of a pair and the pair aligned concordantly. Value of <code>DP</code> indicates the read was part of a pair and the pair aligned discordantly. Value of <code>UP</code> indicates the read was part of a pair but the pair failed to aligned either concordantly or discordantly. <a href="#filtering">Filtering</a>: #filtering
-</td></tr>
-<tr><td id="bowtie2-build-opt-fields-md">
+</td>
+<td>
+<pre><code>Value of `UU` indicates the read was not part of a pair. Value of `CP`
+indicates the read was part of a pair and the pair aligned concordantly.
+Value of `DP` indicates the read was part of a pair and the pair aligned
+discordantly. Value of `UP` indicates the read was part of a pair but the
+pair failed to aligned either concordantly or discordantly.</code></pre>
+</td>
+</tr>
+<tr>
+<td id="bowtie2-build-opt-fields-md">
<pre><code> MD:Z:<S></code></pre>
-</td><td>
-
-A string representation of the mismatched reference bases in the alignment. See <a href="http://samtools.sourceforge.net/SAM1.pdf">SAM</a> format specification for details. Only present if SAM record is for an aligned read.
-</td></tr>
+</td>
+<td>
+<pre><code>A string representation of the mismatched reference bases in the alignment.
+See [SAM] format specification for details. Only present if SAM record is
+for an aligned read.</code></pre>
+</td>
+</tr>
</table>
-
<h1 id="the-bowtie2-build-indexer">The <code>bowtie2-build</code> indexer</h1>
<p><code>bowtie2-build</code> builds a Bowtie index from a set of DNA sequences. <code>bowtie2-build</code> outputs a set of 6 files with suffixes <code>.1.bt2</code>, <code>.2.bt2</code>, <code>.3.bt2</code>, <code>.4.bt2</code>, <code>.rev.1.bt2</code>, and <code>.rev.2.bt2</code>. In the case of a large index these suffixes will have a <code>bt2l</code> termination. These files together constitute the index: they are all that is needed to align reads to that reference. The original se [...]
<p>Bowtie 2's <code>.bt2</code> index format is different from Bowtie 1's <code>.ebwt</code> format, and they are not compatible with each other.</p>
@@ -1095,171 +1214,220 @@ A string representation of the mismatched reference bases in the alignment. See
<p>Usage:</p>
<pre><code>bowtie2-build [options]* <reference_in> <bt2_base></code></pre>
<h3 id="main-arguments-1">Main arguments</h3>
-<table><tr><td>
-
+<table>
+<tr>
+<td>
<pre><code><reference_in></code></pre>
-</td><td>
-
+</td>
+<td>
<p>A comma-separated list of FASTA files containing the reference sequences to be aligned to, or, if <a href="#bowtie2-build-options-c"><code>-c</code></a> is specified, the sequences themselves. E.g., <code><reference_in></code> might be <code>chr1.fa,chr2.fa,chrX.fa,chrY.fa</code>, or, if <a href="#bowtie2-build-options-c"><code>-c</code></a> is specified, this might be <code>GGTCATCCT,ACGGGTCGT,CCGTTCTATGCGGCTTA</code>.</p>
-</td></tr><tr><td>
-
+</td>
+</tr>
+<tr>
+<td>
<pre><code><bt2_base></code></pre>
-</td><td>
-
+</td>
+<td>
<p>The basename of the index files to write. By default, <code>bowtie2-build</code> writes files named <code>NAME.1.bt2</code>, <code>NAME.2.bt2</code>, <code>NAME.3.bt2</code>, <code>NAME.4.bt2</code>, <code>NAME.rev.1.bt2</code>, and <code>NAME.rev.2.bt2</code>, where <code>NAME</code> is <code><bt2_base></code>.</p>
-</td></tr></table>
-
+</td>
+</tr>
+</table>
<h3 id="options-1">Options</h3>
-<table><tr><td>
-
+<table>
+<tr>
+<td>
<pre><code>-f</code></pre>
-</td><td>
-
+</td>
+<td>
<p>The reference input files (specified as <code><reference_in></code>) are FASTA files (usually having extension <code>.fa</code>, <code>.mfa</code>, <code>.fna</code> or similar).</p>
-</td></tr><tr><td id="bowtie2-build-options-c">
-
+</td>
+</tr>
+<tr>
+<td id="bowtie2-build-options-c">
<pre><code>-c</code></pre>
-</td><td>
-
+</td>
+<td>
<p>The reference sequences are given on the command line. I.e. <code><reference_in></code> is a comma-separated list of sequences rather than a list of FASTA files.</p>
-</td></tr>
-</td></tr><tr><td id="bowtie2-build-options-large-index">
-
+</td>
+</tr>
+</td>
+</tr>
+<tr>
+<td id="bowtie2-build-options-large-index">
<pre><code>--large-index</code></pre>
-</td><td>
-
+</td>
+<td>
<p>Force <code>bowtie2-build</code> to build a <a href="#small-and-large-indexes">large index</a>, even if the reference is less than ~ 4 billion nucleotides inlong.</p>
-</td></tr>
-<tr><td id="bowtie2-build-options-a">
-
+</td>
+</tr>
+<tr>
+<td id="bowtie2-build-options-a">
<pre><code>-a/--noauto</code></pre>
-</td><td>
-
+</td>
+<td>
<p>Disable the default behavior whereby <code>bowtie2-build</code> automatically selects values for the <a href="#bowtie2-build-options-bmax"><code>--bmax</code></a>, <a href="#bowtie2-build-options-dcv"><code>--dcv</code></a> and <a href="#bowtie2-build-options-p"><code>--packed</code></a> parameters according to available memory. Instead, user may specify values for those parameters. If memory is exhausted during indexing, an error message will be printed; it is up to the user to try n [...]
-</td></tr><tr><td id="bowtie2-build-options-p">
-
+</td>
+</tr>
+<tr>
+<td id="bowtie2-build-options-p">
<pre><code>-p/--packed</code></pre>
-</td><td>
-
+</td>
+<td>
<p>Use a packed (2-bits-per-nucleotide) representation for DNA strings. This saves memory but makes indexing 2-3 times slower. Default: off. This is configured automatically by default; use <a href="#bowtie2-build-options-a"><code>-a</code>/<code>--noauto</code></a> to configure manually.</p>
-</td></tr><tr><td id="bowtie2-build-options-bmax">
-
+</td>
+</tr>
+<tr>
+<td id="bowtie2-build-options-bmax">
<pre><code>--bmax <int></code></pre>
-</td><td>
-
-<p>The maximum number of suffixes allowed in a block. Allowing more suffixes per block makes indexing faster, but increases peak memory usage. Setting this option overrides any previous setting for <a href="#bowtie2-build-options-bmax"><code>--bmax</code></a>, or <a href="#bowtie2-build-options-bmaxdivn"><code>--bmaxdivn</code></a>. Default (in terms of the <a href="#bowtie2-build-options-bmaxdivn"><code>--bmaxdivn</code></a> parameter) is <a href="#bowtie2-build-options-bmaxdivn"><code> [...]
-</td></tr><tr><td id="bowtie2-build-options-bmaxdivn">
-
+</td>
+<td>
+<p>The maximum number of suffixes allowed in a block. Allowing more suffixes per block makes indexing faster, but increases peak memory usage. Setting this option overrides any previous setting for <a href="#bowtie2-build-options-bmax"><code>--bmax</code></a>, or <a href="#bowtie2-build-options-bmaxdivn"><code>--bmaxdivn</code></a>. Default (in terms of the <a href="#bowtie2-build-options-bmaxdivn"><code>--bmaxdivn</code></a> parameter) is <a href="#bowtie2-build-options-bmaxdivn"><code> [...]
+</td>
+</tr>
+<tr>
+<td id="bowtie2-build-options-bmaxdivn">
<pre><code>--bmaxdivn <int></code></pre>
-</td><td>
-
-<p>The maximum number of suffixes allowed in a block, expressed as a fraction of the length of the reference. Setting this option overrides any previous setting for <a href="#bowtie2-build-options-bmax"><code>--bmax</code></a>, or <a href="#bowtie2-build-options-bmaxdivn"><code>--bmaxdivn</code></a>. Default: <a href="#bowtie2-build-options-bmaxdivn"><code>--bmaxdivn</code></a> 4. This is configured automatically by default; use <a href="#bowtie2-build-options-a"><code>-a</code>/<code>-- [...]
-</td></tr><tr><td id="bowtie2-build-options-dcv">
-
+</td>
+<td>
+<p>The maximum number of suffixes allowed in a block, expressed as a fraction of the length of the reference. Setting this option overrides any previous setting for <a href="#bowtie2-build-options-bmax"><code>--bmax</code></a>, or <a href="#bowtie2-build-options-bmaxdivn"><code>--bmaxdivn</code></a>. Default: <a href="#bowtie2-build-options-bmaxdivn"><code>--bmaxdivn</code></a> 4 * number of threads. This is configured automatically by default; use <a href="#bowtie2-build-options-a"><cod [...]
+</td>
+</tr>
+<tr>
+<td id="bowtie2-build-options-dcv">
<pre><code>--dcv <int></code></pre>
-</td><td>
-
+</td>
+<td>
<p>Use <code><int></code> as the period for the difference-cover sample. A larger period yields less memory overhead, but may make suffix sorting slower, especially if repeats are present. Must be a power of 2 no greater than 4096. Default: 1024. This is configured automatically by default; use <a href="#bowtie2-build-options-a"><code>-a</code>/<code>--noauto</code></a> to configure manually.</p>
-</td></tr><tr><td id="bowtie2-build-options-nodc">
-
+</td>
+</tr>
+<tr>
+<td id="bowtie2-build-options-nodc">
<pre><code>--nodc</code></pre>
-</td><td>
-
+</td>
+<td>
<p>Disable use of the difference-cover sample. Suffix sorting becomes quadratic-time in the worst case (where the worst case is an extremely repetitive reference). Default: off.</p>
-</td></tr><tr><td>
-
+</td>
+</tr>
+<tr>
+<td>
<pre><code>-r/--noref</code></pre>
-</td><td>
-
+</td>
+<td>
<p>Do not build the <code>NAME.3.bt2</code> and <code>NAME.4.bt2</code> portions of the index, which contain a bitpacked version of the reference sequences and are used for paired-end alignment.</p>
-</td></tr><tr><td>
-
+</td>
+</tr>
+<tr>
+<td>
<pre><code>-3/--justref</code></pre>
-</td><td>
-
+</td>
+<td>
<p>Build only the <code>NAME.3.bt2</code> and <code>NAME.4.bt2</code> portions of the index, which contain a bitpacked version of the reference sequences and are used for paired-end alignment.</p>
-</td></tr><tr><td id="bowtie2-build-options-o">
-
+</td>
+</tr>
+<tr>
+<td id="bowtie2-build-options-o">
<pre><code>-o/--offrate <int></code></pre>
-</td><td>
-
+</td>
+<td>
<p>To map alignments back to positions on the reference sequences, it's necessary to annotate ("mark") some or all of the <a href="http://en.wikipedia.org/wiki/Burrows-Wheeler_transform">Burrows-Wheeler</a> rows with their corresponding location on the genome. <a href="#bowtie2-build-options-o"><code>-o</code>/<code>--offrate</code></a> governs how many rows get marked: the indexer will mark every 2^<code><int></code> rows. Marking more rows makes reference-position looku [...]
-</td></tr><tr><td>
-
+</td>
+</tr>
+<tr>
+<td>
<pre><code>-t/--ftabchars <int></code></pre>
-</td><td>
-
+</td>
+<td>
<p>The ftab is the lookup table used to calculate an initial <a href="http://en.wikipedia.org/wiki/Burrows-Wheeler_transform">Burrows-Wheeler</a> range with respect to the first <code><int></code> characters of the query. A larger <code><int></code> yields a larger lookup table but faster query times. The ftab has size 4^(<code><int></code>+1) bytes. The default setting is 10 (ftab is 4MB).</p>
-</td></tr><tr><td>
-
+</td>
+</tr>
+<tr>
+<td>
<pre><code>--seed <int></code></pre>
-</td><td>
-
+</td>
+<td>
<p>Use <code><int></code> as the seed for pseudo-random number generator.</p>
-</td></tr><tr><td>
-
+</td>
+</tr>
+<tr>
+<td>
<pre><code>--cutoff <int></code></pre>
-</td><td>
-
+</td>
+<td>
<p>Index only the first <code><int></code> bases of the reference sequences (cumulative across sequences) and ignore the rest.</p>
-</td></tr><tr><td>
-
+</td>
+</tr>
+<tr>
+<td>
<pre><code>-q/--quiet</code></pre>
-</td><td>
-
+</td>
+<td>
<p><code>bowtie2-build</code> is verbose by default. With this option <code>bowtie2-build</code> will print only error messages.</p>
-</td></tr><tr><td>
-
+</td>
+</tr>
+<tr>
+<td>
<pre><code>--threads <int></code></pre>
-</td><td>
-
+</td>
+<td>
<p>By default <code>bowtie2-build</code> is using only one thread. Increasing the number of threads will speed up the index building considerably in most cases.</p>
-</td></tr><tr><td>
-
+</td>
+</tr>
+<tr>
+<td>
<pre><code>-h/--help</code></pre>
-</td><td>
-
+</td>
+<td>
<p>Print usage information and quit.</p>
-</td></tr><tr><td>
-
+</td>
+</tr>
+<tr>
+<td>
<pre><code>--version</code></pre>
-</td><td>
-
+</td>
+<td>
<p>Print version information and quit.</p>
-</td></tr></table>
-
+</td>
+</tr>
+</table>
<h1 id="the-bowtie2-inspect-index-inspector">The <code>bowtie2-inspect</code> index inspector</h1>
<p><code>bowtie2-inspect</code> extracts information from a Bowtie index about what kind of index it is and what reference sequences were used to build it. When run without any options, the tool will output a FASTA file containing the sequences of the original references (with all non-<code>A</code>/<code>C</code>/<code>G</code>/<code>T</code> characters converted to <code>N</code>s). It can also be used to extract just the reference sequence names using the <a href="#bowtie2-inspect-opt [...]
<h2 id="command-line-2">Command Line</h2>
<p>Usage:</p>
<pre><code>bowtie2-inspect [options]* <bt2_base></code></pre>
<h3 id="main-arguments-2">Main arguments</h3>
-<table><tr><td>
-
+<table>
+<tr>
+<td>
<pre><code><bt2_base></code></pre>
-</td><td>
-
+</td>
+<td>
<p>The basename of the index to be inspected. The basename is name of any of the index files but with the <code>.X.bt2</code> or <code>.rev.X.bt2</code> suffix omitted. <code>bowtie2-inspect</code> first looks in the current directory for the index files, then in the directory specified in the <code>BOWTIE2_INDEXES</code> environment variable.</p>
-</td></tr></table>
-
+</td>
+</tr>
+</table>
<h3 id="options-2">Options</h3>
-<table><tr><td>
-
+<table>
+<tr>
+<td>
<pre><code>-a/--across <int></code></pre>
-</td><td>
-
+</td>
+<td>
<p>When printing FASTA output, output a newline character every <code><int></code> bases (default: 60).</p>
-</td></tr><tr><td id="bowtie2-inspect-options-n">
-
+</td>
+</tr>
+<tr>
+<td id="bowtie2-inspect-options-n">
<pre><code>-n/--names</code></pre>
-</td><td>
-
+</td>
+<td>
<p>Print reference sequence names, one per line, and quit.</p>
-</td></tr><tr><td id="bowtie2-inspect-options-s">
-
+</td>
+</tr>
+<tr>
+<td id="bowtie2-inspect-options-s">
<pre><code>-s/--summary</code></pre>
-</td><td>
-
+</td>
+<td>
<p>Print a summary that includes information about index settings, as well as the names and lengths of the input sequences. The summary has this format:</p>
<pre><code>Colorspace <0 or 1>
SA-Sample 1 in <sample>
@@ -1269,26 +1437,33 @@ Sequence-2 <name> <len>
...
Sequence-N <name> <len></code></pre>
<p>Fields are separated by tabs. Colorspace is always set to 0 for Bowtie 2.</p>
-</td></tr><tr><td>
-
+</td>
+</tr>
+<tr>
+<td>
<pre><code>-v/--verbose</code></pre>
-</td><td>
-
+</td>
+<td>
<p>Print verbose output (for debugging).</p>
-</td></tr><tr><td>
-
+</td>
+</tr>
+<tr>
+<td>
<pre><code>--version</code></pre>
-</td><td>
-
+</td>
+<td>
<p>Print version information and quit.</p>
-</td></tr><tr><td>
-
+</td>
+</tr>
+<tr>
+<td>
<pre><code>-h/--help</code></pre>
-</td><td>
-
+</td>
+<td>
<p>Print usage information and quit.</p>
-</td></tr></table>
-
+</td>
+</tr>
+</table>
<h1 id="getting-started-with-bowtie-2-lambda-phage-example">Getting started with Bowtie 2: Lambda phage example</h1>
<p>Bowtie 2 comes with some example files to get you started. The example files are not scientifically significant; we use the <a href="http://en.wikipedia.org/wiki/Lambda_phage">Lambda phage</a> reference genome simply because it's short, and the reads were generated by a computer program, not a sequencer. However, these files will let you start running Bowtie 2 and downstream tools right away.</p>
<p>First follow the manual instructions to <a href="#obtaining-bowtie-2">obtain Bowtie 2</a>. Set the <code>BT2_HOME</code> environment variable to point to the new Bowtie 2 directory containing the <code>bowtie2</code>, <code>bowtie2-build</code> and <code>bowtie2-inspect</code> binaries. This is important, as the <code>BT2_HOME</code> variable is used in the commands below to refer to that directory.</p>
diff --git a/doc/release.txt b/doc/release.txt
new file mode 100644
index 0000000..b1dd592
--- /dev/null
+++ b/doc/release.txt
@@ -0,0 +1,153 @@
+Instructions for releasing a new version of Bowtie 2
+====================================================
+
+Author: Ben Langmead (langmea at cs.jhu.edu)
+Date: 12/21/2012
+
+Prerequisites:
+- Need a way of building Linux binaries. It's recommended that you not build
+ on a Linux 3.X distro (for portability reasons)
+- Need a way of building binaries for Mac OS X.
+- Need a way of building Windows binaries, both 32-bit and 64-bit.
+- Need "pandoc" installed on at least one of those machines.
+- Need access to the GitHub repository.
+- Need a sourceforge account with the appropriate permissions.
+- TBB is now the default threading library and should be installed
+ either from source or using a package manager
+
+ 1. Check that the code compiles with no issues, including with very recent
+ versions of g++, and with clang on a Mac.
+ - To use clang on the Mac: "make CPP=clang++ allall"
+
+ 2. Check that all the simple tests pass: "make simple-test"
+
+ 3. Check that random tests pass: "make random-test"
+
+ 4. Check that performance, sensitivity and accuracy are not substantially
+ changed.
+ - TODO! Need infrastructure for this.
+
+ 5. Update VERSION with new version number if this hasn't already been done.
+
+ 6. Update NEWS and doc/website/recent_news.ssi if this hasn't already been
+ done. (And even if it has, make sure the release date mentioned in those
+ documents is correct.)
+ - In both cases, add a section at the top describing the new release, along
+ with bullet points describing improvements and changes in that release.
+ - For recent_news.ssi, hyperlink concepts or command-line options so that
+ they point into the manual.
+ - Make sure to update the version numbers mentioned in the paragraph at the
+ beginning of the NEWS file.
+ - Potentially move an entry or two from the bottom of
+ doc/website/recent_news.ssi to the top of doc/website/old_news.ssi if it
+ gets too long.
+
+ 7. Update the text and html version of the manual, which are derived from the
+ source MANUAL.markdown file.
+ - In Bowtie 2 root directory, do "make doc" (you'll have to have 'pandoc'
+ installed). The output will be stored both in MANUAL and
+ doc/manual.html.
+ - In doc/manual.html, highlight and copy everything from <div id="TOC">
+ down to but not including </body>. Then paste it in the corresponding
+ part of doc/website/manual.ssi.
+ - Update the version number at the top of doc/website/manual.ssi.
+
+ 8. Go to the sourceforge site for the Bowtie 2 files:
+ https://sourceforge.net/projects/bowtie-bio/files/bowtie2/
+ And make a new directory (with "Add Folder" button) for the new version.
+
+ 9. Update doc/website/rhsidebar.ssi to refer to the new version in the section
+ immediately following "<h2>Latest Release</h2>"
+
+10. Commit all changes due to the steps taken so far.
+
+11. Upload updated webpages
+ - In doc/website directory, do "sh push.sh"
+
+12. Make Bowtie 2 source archive:
+ - From the Bowtie 2 git root, run "make bowtie2-src"
+ - Upload to sourceforge:
+ scp bowtie2-X.Y.Z-source.zip USERNAME at frs.sourceforge.net:/home/frs/project/bowtie-bio/bowtie2/X.Y.Z/
+ (But replace X.Y.Z and USERNAME with appropriate values)
+
+13. Make Bowtie 2 Linux 64-bit binaries.
+ - On a Linux machine (e.g. igmX), from the Bowtie 2 git root run "make clean"
+ - Then "make bowtie2-pkg". (Use "make -j12" or similar to use many
+ CPUs at once).
+ - Rename the resulting archive to match the typical naming scheme. E.g.:
+ "mv bowtie2-X.Y.Z.zip bowtie2-X.Y.Z-linux-x86_64.zip"
+ - Upload to sourceforge:
+ scp bowtie2-X.Y.Z-linux-x86_64.zip USERNAME at frs.sourceforge.net:/home/frs/project/bowtie-bio/bowtie2/X.Y.Z/
+ (But replace X.Y.Z and USERNAME with appropriate values)
+ - Repeat from the beginning adding the NO_TBB=1 flag to the second step to create the legacy packages
+
+14. Make Bowtie 2 Mac OS X 64-bit binaries.
+ - From the Bowtie 2 git root run "make clean"
+ - Then "make EXTRA_FLAGS=-mmacosx-version-min=10.5 bowtie2-pkg".
+ (Use "make -j12" or similar to use many CPUs at once). The -mmacosx...
+ flag forces the binaries to be backward compatible with Mac OS version
+ 10.5 and up.
+ - Rename the resulting archive to match the typical naming scheme. E.g.:
+ "mv bowtie2-X.Y.Z.zip bowtie2-X.Y.Z-macos-x86_64.zip"
+ - Upload to sourceforge:
+ scp bowtie2-X.Y.Z-macos-x86_64.zip USERNAME at frs.sourceforge.net:/home/frs/project/bowtie-bio/bowtie2/X.Y.Z/
+ (But replace X.Y.Z and USERNAME with appropriate values)
+ - Repeat from the beginning adding the NO_TBB=1 flag to the second step to create the legacy packages
+
+15. Make Windows 64-bit binaries
+ Prerequisites
+ -------------
+ By default MinGW can only compile 32 bit binaries, so MinGW-w64 will be needed for 64 bit binaries
+ MinGW installer - https://sourceforge.net/projects/mingw/files/Installer/mingw-get-setup.exe/download
+ MinGW64 - https://sourceforge.net/projects/tdm-gcc/files/latest/download
+ TBB - https://www.threadingbuildingblocks.org/sites/default/files/software_releases/source/tbb2017_20161128oss_src.tgz
+
+ - Download and run the mingw-get-setup installer and set the install directory to C:\MinGW\32
+ - When presented with the "MinGW Installation Manager" window click on "All Packages" and mark the msys-zip, msys-unzip, and msys-wget packages for installation.
+ - Download and run the mingw-w64-install installer and set the install directory to C:\MinGW\64
+ - Once all MinGW packages are installed open Windows Explorer and navigate to C:\MinGW\msys\1.0 and run the msys.bat file to start the MinGW shell.
+ - To build TBB from source
+ * wget --no-check-certificate <tbb url>
+ * tar -xzf <tbb_archive_name>.tgz
+ * cd into the TBB source directory
+ * run mingw32-make.exe compiler=gcc arch=intel64 runtime=mingw
+ - Upon successful compilation there should now exist a build directory which contains the dlls needed by the linker. Before building bowtie make sure that the following environment variables contain the path to the TBB headers and dlls respectfully. From the root of the TBB directory run:
+ * export CPATH=$CPATH:`pwd`/include
+ * export LIBRARY_PATH=$LIBRARY_PATH:`pwd`/build/windows_intel64_gcc_mingw_release
+ * the changes to the above environment variables will only available for the duration of the current shell session. To make them permanent make sure they are added to the ~/.profile file or your preferred shell’s rc file e.g. ~/.bashrc.
+
+ - From the MinGW shell cd to the Bowtie 2 git root run "make clean"
+ - Then "make bowtie2-pkg". (Use "make -j12" or similar to use many
+ CPUs at once).
+ - Rename the resulting archive to match the typical naming scheme. E.g.:
+ "mv bowtie2-X.Y.Z.zip bowtie2-X.Y.Z-mingw-x86_64.zip"
+ - Upload to sourceforge:
+ scp bowtie2-X.Y.Z-linux-x86_64.zip USERNAME at frs.sourceforge.net:/home/frs/project/bowtie-bio/bowtie2/X.Y.Z/
+ (But replace X.Y.Z and USERNAME with appropriate values)
+ - Repeat the above 4 steps adding the NO_TBB=1 flag to the second step to create the legacy packages
+
+16. Make a git annotated tag for the new version using "git tag -a"
+ - See: http://git-scm.com/book/en/Git-Basics-Tagging
+ - Basically, do something like:
+ + git tag -a v2.0.5 -m 'Just prior to 2.0.5 release on 1/4/2013'
+ + git push origin --tags
+
+17. Update the sourceforge "Looking for the latest version?" pointers:
+ - Go to: https://sourceforge.net/projects/bowtie-bio/files/bowtie2/X.Y.Z
+ - Click the "i" ("info") button next to the Mac OS X 64-bit archive. Check
+ the Mac os "Default Download For:" checkbox and save.
+ - Do same for Linux 64-bit archive and the Linux checkbox.
+ - Do same for Windows.
+ - Click the "i" ("info") button next to the source archive. Check all the
+ "Default Download For:" checkboxes not checked so far and save.
+
+18. Upload the release binaries to github release area as well.
+ - Go to the releases area and click 'Edit' button for the current release.
+ - Change the release description so it will include the NEWS.
+ - Upload the tgz archives for current release.
+
+19. Send an email to bowtie-bio-announce at lists.sourceforge.net announcing the
+ release.
+ - Subject: "Bowtie 2 X.Y.Z released"
+ - Body: Basically a copy and paste of the release notes for the most recent
+ release from the NEWS file.
diff --git a/doc/website/faq.shtml b/doc/website/faq.shtml
new file mode 100644
index 0000000..af6d918
--- /dev/null
+++ b/doc/website/faq.shtml
@@ -0,0 +1,34 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head>
+<title>Bowtie 2: fast and sensitive read alignment</title>
+<meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
+<link rel="stylesheet" type="text/css" href="style.css" media="screen" />
+<meta name="verify-v1" content="YJT1CfXN3kzE9cr+jvNB+Q73lTfHrv8eivoY+xjblc0=" />
+</head>
+<body>
+<div id="wrap">
+ <!--#include virtual="top.ssi" -->
+ <div id="main">
+ <!--#include virtual="rhsidebar.ssi" -->
+ <div id="leftside">
+ <h2>Frequently Asked Questions</h2>
+ <!--#include virtual="faq.ssi" -->
+ </div>
+ </div>
+ <!--#include virtual="foot.ssi" -->
+</div>
+
+<!-- Google analytics code -->
+<script type="text/javascript">
+var gaJsHost = (("https:" == document.location.protocol) ? "https://ssl." : "http://www.");
+document.write(unescape("%3Cscript src='" + gaJsHost + "google-analytics.com/ga.js' type='text/javascript'%3E%3C/script%3E"));
+</script>
+<script type="text/javascript">
+var pageTracker = _gat._getTracker("UA-5334290-1");
+pageTracker._trackPageview();
+</script>
+<!-- End google analytics code -->
+
+</body>
+</html>
diff --git a/doc/website/faq.ssi b/doc/website/faq.ssi
new file mode 100644
index 0000000..913ff6e
--- /dev/null
+++ b/doc/website/faq.ssi
@@ -0,0 +1,68 @@
+<h2>How is Bowtie 2 different from Bowtie 1?</h2>
+
+Bowtie 1 was released in 2009 and was geared toward aligning the relatively
+short type of sequencing reads (up to 50 bp) prevalent at the time.
+Since then, sequencing technology has improved both in terms of throughput (more
+bp produced per sequencer per day) and in terms of read length (more
+bp per read).
+
+Chief differences between Bowtie 1 and Bowtie 2 are:
+
+<ul>
+<li>Bowtie 2 fully supports gapped alignment with affine gap penalties. Number
+ of gaps and gap lengths are not restricted, except via the user-supplied
+ scoring scheme. Bowtie 1 only finds ungapped alignments.</li>
+<li> For reads longer than about 50 bp Bowtie 2 is generally faster, more
+ sensitive, and uses less memory than Bowtie 1. For relatively short reads
+ (e.g. less than 50 bp) Bowtie 1 is sometimes faster and/or more sensitive.</li>
+<li>Bowtie 2 supports a "local" alignment mode, which doesn't require that reads
+ align end-to-end. This produces alignments that might be "trimmed" (or "soft
+ clipped") at one or both extremes in a way that optimizes alignment score.
+ Bowtie 2 also supports an "end-to-end" alignment mode which, like Bowtie 1,
+ requires that the read align entirely.</li>
+<li>There is no upper limit on read length in Bowtie 2. Bowtie 1 had an upper
+ limit of around 1000 bp.</li>
+<li>Bowtie 2 does away with Bowtie 1's notion of alignment "stratum". In Bowtie
+ 2 all alignments lie along a continuous spectrum of alignment scores.</li>
+<li>There is no longer a distinction between "end-to-end" and "Maq-like" modes as
+ in Bowtie 1. There is just one scoring scheme, similar to Needleman-Wunsch
+ and Smith-Waterman.</li>
+<li>Bowtie 2's paired-end alignment mode is more flexible than Bowtie 1's. For
+ example, for pairs that do not align in a paired fashion, it will attempt to
+ find unpaired alignments for each mate.</li>
+<li>Bowtie 2 does not align colorspace reads.</li>
+</ul>
+
+<h2>Is Bowtie 2 a straightforward upgrade from Bowtie 1?</h2>
+Bowtie 2 is not a "drop-in" replacement for Bowtie 1. Bowtie 2's command-line
+arguments and genome index format are similar to, but different from Bowtie 1's,
+so that simply running the Bowtie 2 executable instead of the Bowtie 1
+executable will not work.
+
+<h2>Are Bowtie 2 and Bowtie 1 genome indexes compatible?</h2>
+No. Bowtie 2 indexes are formatted differently. Bowtie 1 indexes do not work
+with Bowtie 2 and Bowtie 2 indexes do not work with Bowtie 1.
+
+<h2>Does Bowtie 2 supersede Bowtie 1?</h2>
+Mostly, but not entirely. If your reads are shorter than 50 bp, you
+might want to try both Bowtie 1 and Bowtie 2 and see which gives better
+results in terms of speed and sensitivity. In our experiments, Bowtie 2 is
+generally superior to Bowtie 1 for reads longer than 50 bp. For reads shorter
+than 50 bp, Bowtie 1 may or may not be preferable.
+
+<h2>I used Bowtie 2 to align my paired-end data and very few of my paired-end
+reads align concordantly. What could have happened?</h2>
+
+This is usually because Bowtie 2's paired-end options have been been set to
+properly reflect how the sequencing library was prepared. See the
+<a href="manual.shtml#paired-end-options">Paired-end Options</a> section of the
+manual.
+
+<h2>How many mismatches and gaps does Bowtie 2 allow?</h2>
+Bowtie 2 using a scoring scheme that assigns various scores to matches,
+mismatches, gap opens, and gap extensions. These are all user-configurable.
+An alignment with a total score that rises above a user-configurable threshold
+are considered "valid" and eligible for reporting. For information on how to
+adjust these settings, see the
+<a href="manual.shtml#scoring-options">Scoring Options</a> section of the
+manual.
diff --git a/doc/website/foot.ssi b/doc/website/foot.ssi
new file mode 100644
index 0000000..57a73b3
--- /dev/null
+++ b/doc/website/foot.ssi
@@ -0,0 +1,12 @@
+ <div id="footer">
+ <table width="100%">
+ <tr>
+ <td>
+ This research was supported in part by NIH grants R01-HG006677 and R01-HG006102 and AWS in Education research grants.
+ </td>
+ </tr>
+ </table>
+ <div style="text-align:center">
+ <a href="http://www.sourceforge.net"><img src="../images/sflogo.png" alt="Sourceforge.net" style="border-style: none"></img></a>
+ </div>
+ </div>
diff --git a/doc/website/index.html b/doc/website/index.html
new file mode 100644
index 0000000..beb1350
--- /dev/null
+++ b/doc/website/index.html
@@ -0,0 +1,9 @@
+<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.0 Transitional//EN">
+<html>
+<head>
+<title>Bowtie 2</title>
+<meta http-equiv="REFRESH" content="0;url=index.shtml"></HEAD>
+<BODY>
+Redirecting to Bowtie 2.
+</BODY>
+</HTML>
diff --git a/doc/website/index.shtml b/doc/website/index.shtml
new file mode 100644
index 0000000..a05e793
--- /dev/null
+++ b/doc/website/index.shtml
@@ -0,0 +1,60 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head>
+<title>Bowtie 2: fast and sensitive read alignment</title>
+<meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
+<link rel="stylesheet" type="text/css" href="style.css" media="screen" />
+<meta name="verify-v1" content="YJT1CfXN3kzE9cr+jvNB+Q73lTfHrv8eivoY+xjblc0=" />
+</head>
+<body>
+<div id="wrap">
+ <!--#include virtual="top.ssi" -->
+ <div id="subheader">
+ <table width="100%"><tr>
+ <td>
+
+ <strong>Bowtie 2</strong> is an ultrafast and memory-efficient tool for aligning sequencing
+reads to long reference sequences. It is particularly good at aligning reads of
+about 50 up to 100s or 1,000s of characters, and particularly good at aligning to relatively long (e.g. mammalian)
+genomes. Bowtie 2 indexes the genome with an FM Index to keep its memory footprint small: for the human
+genome, its memory footprint is typically around 3.2 GB. Bowtie 2 supports
+gapped, local, and paired-end alignment modes.
+
+</td>
+<td align="right" valign="middle">
+ <a href="http://opensource.org"><img alt="Open Source Software" src="/images/osi-certified.gif"></img></a>
+ </td></tr>
+ </table>
+ </div>
+ <div id="main">
+ <!--#include virtual="rhsidebar.ssi" -->
+ <div id="leftside">
+ <!--#include virtual="recent_news.ssi" -->
+ </div>
+ </div>
+ <div id="footer">
+ <table width="100%"><tr><td>
+ This research was supported in part by NIH grants R01-HG006102 and R01-GM083873 and AWS in Education research grants.
+ </td><td align="right">
+ Administrator: <a href="mailto:langmea at cs.jhu.edu">Ben Langmead</a>. Design by <a href="http://www.free-css-templates.com" title="Design by David Herreman">David Herreman</a>
+ </td></tr>
+ </table>
+ <div style="text-align:center">
+ <a href="http://www.sourceforge.net"><img src="/images/sflogo.png" alt="Sourceforge.net" style="border-style: none"></img></a>
+ </div>
+ </div>
+</div>
+
+<!-- Google analytics code -->
+<script type="text/javascript">
+var gaJsHost = (("https:" == document.location.protocol) ? "https://ssl." : "http://www.");
+document.write(unescape("%3Cscript src='" + gaJsHost + "google-analytics.com/ga.js' type='text/javascript'%3E%3C/script%3E"));
+</script>
+<script type="text/javascript">
+var pageTracker = _gat._getTracker("UA-5334290-1");
+pageTracker._trackPageview();
+</script>
+<!-- End google analytics code -->
+
+</body>
+</html>
diff --git a/doc/website/manual.shtml b/doc/website/manual.shtml
new file mode 100644
index 0000000..42b3e44
--- /dev/null
+++ b/doc/website/manual.shtml
@@ -0,0 +1,33 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head>
+<title>Bowtie 2: Manual</title>
+<meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
+<link rel="stylesheet" type="text/css" href="style.css" media="screen" />
+<meta name="verify-v1" content="YJT1CfXN3kzE9cr+jvNB+Q73lTfHrv8eivoY+xjblc0=" />
+</head>
+<body>
+<div id="wrap">
+ <!--#include virtual="top.ssi" -->
+ <div id="main">
+ <!--#include virtual="rhsidebar.ssi" -->
+ <div id="manual" style="margin-right: 310px; margin-left: 0px; width: auto;">
+ <!--#include virtual="manual.ssi" -->
+ </div>
+ </div>
+ <!--#include virtual="foot.ssi" -->
+</div>
+
+<!-- Google analytics code -->
+<script type="text/javascript">
+var gaJsHost = (("https:" == document.location.protocol) ? "https://ssl." : "http://www.");
+document.write(unescape("%3Cscript src='" + gaJsHost + "google-analytics.com/ga.js' type='text/javascript'%3E%3C/script%3E"));
+</script>
+<script type="text/javascript">
+var pageTracker = _gat._getTracker("UA-5334290-1");
+pageTracker._trackPageview();
+</script>
+<!-- End google analytics code -->
+
+</body>
+</html>
diff --git a/doc/manual.html b/doc/website/manual.ssi
similarity index 97%
copy from doc/manual.html
copy to doc/website/manual.ssi
index bb4b79c..038aa6a 100644
--- a/doc/manual.html
+++ b/doc/website/manual.ssi
@@ -1,15 +1,15 @@
+<h1>Table of Contents</h1>
+<p> Version <b>2.3.0</b></p>
<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Transitional//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-transitional.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head>
+ <title>Bowtie 2 Manual - </title>
<meta http-equiv="Content-Type" content="text/html; charset=utf-8" />
- <meta http-equiv="Content-Style-Type" content="text/css" />
<meta name="generator" content="pandoc" />
- <title>Bowtie 2 Manual - </title>
- <style type="text/css">code{white-space: pre;}</style>
<link rel="stylesheet" href="style.css" type="text/css" />
</head>
<body>
-<h1>Table of Contents</h1>
+
<div id="TOC">
<ul>
<li><a href="#introduction">Introduction</a><ul>
@@ -55,7 +55,7 @@
<li><a href="#ambiguous-characters">Ambiguous characters</a></li>
<li><a href="#presets-setting-many-settings-at-once">Presets: setting many settings at once</a></li>
<li><a href="#filtering">Filtering</a></li>
-<li><a href="#alignment-summmary">Alignment summmary</a></li>
+<li><a href="#alignment-summary">Alignment summary</a></li>
<li><a href="#wrapper-scripts">Wrapper scripts</a></li>
<li><a href="#small-and-large-indexes">Small and large indexes</a></li>
<li><a href="#performance-tuning">Performance tuning</a></li>
@@ -97,7 +97,7 @@
<h1 id="introduction">Introduction</h1>
<h2 id="what-is-bowtie-2">What is Bowtie 2?</h2>
<p><a href="http://bowtie-bio.sf.net/bowtie2">Bowtie 2</a> is an ultrafast and memory-efficient tool for aligning sequencing reads to long reference sequences. It is particularly good at aligning reads of about 50 up to 100s or 1,000s of characters to relatively long (e.g. mammalian) genomes. Bowtie 2 indexes the genome with an <a href="http://portal.acm.org/citation.cfm?id=796543">FM Index</a> (based on the <a href="http://en.wikipedia.org/wiki/Burrows-Wheeler_transform">Burrows-Wheeler [...]
-<p><a href="http://bowtie-bio.sf.net/bowtie2">Bowtie 2</a> is often the first step in pipelines for comparative genomics, including for variation calling, ChIP-seq, RNA-seq, BS-seq. <a href="http://bowtie-bio.sf.net/bowtie2">Bowtie 2</a> and <a href="http://bowtie-bio.sf.net">Bowtie</a> (also called "<a href="http://bowtie-bio.sf.net">Bowtie 1</a>" here) are also tightly integrated into some tools, including <a href="http://tophat.cbcb.umd.edu/">TopHat</a>: a fast splice juncti [...]
+<p><a href="http://bowtie-bio.sf.net/bowtie2">Bowtie 2</a> is often the first step in pipelines for comparative genomics, including for variation calling, ChIP-seq, RNA-seq, BS-seq. <a href="http://bowtie-bio.sf.net/bowtie2">Bowtie 2</a> and <a href="http://bowtie-bio.sf.net">Bowtie</a> (also called "<a href="http://bowtie-bio.sf.net">Bowtie 1</a>" here) are also tightly integrated into some tools, including <a href="http://tophat.cbcb.umd.edu/">TopHat</a>: a fast splice juncti [...]
<p>If you use <a href="http://bowtie-bio.sf.net/bowtie2">Bowtie 2</a> for your published research, please cite the <a href="http://genomebiology.com/2009/10/3/R25">Bowtie paper</a>. Thank you!</p>
<h2 id="how-is-bowtie-2-different-from-bowtie-1">How is Bowtie 2 different from Bowtie 1?</h2>
<p>Bowtie 1 was released in 2009 and was geared toward aligning the relatively short sequencing reads (up to 25-50 nucleotides) prevalent at the time. Since then, technology has improved both sequencing throughput (more nucleotides produced per sequencer per day) and read length (more nucleotides per read).</p>
@@ -125,7 +125,7 @@
<h2 id="building-from-source">Building from source</h2>
<p>Building Bowtie 2 from source requires a GNU-like environment with GCC, GNU Make and other basics. It should be possible to build Bowtie 2 on most vanilla Linux installations or on a Mac installation with <a href="http://developer.apple.com/xcode/">Xcode</a> installed. Bowtie 2 can also be built on Windows using a 64-bit MinGW distribution and MSYS. In order to simplify the MinGW setup it might be worth investigating popular MinGW personal builds since these are coming already prepare [...]
<p>First, download the source package from the <a href="https://sourceforge.net/projects/bowtie-bio/files/bowtie2/">sourceforge site</a>. Make sure you're getting the source package; the file downloaded should end in <code>-source.zip</code>. Unzip the file, change to the unzipped directory, and build the Bowtie 2 tools by running GNU <code>make</code> (usually with the command <code>make</code>, but sometimes with <code>gmake</code>) with no arguments. If building with MinGW, run <code> [...]
-<p>Bowtie 2 is using the multithreading software model in order to speed up execution times on SMP architectures where this is possible. On POSIX platforms (like linux, Mac OS, etc) it needs the pthread library. Although it is possible to use pthread library on non-POSIX platform like Windows, due to performance reasons bowtie 2 will try to use Windows native multithreading if possible. We recommend that you first install the <a href="https://www.threadingbuildingblocks.org">Threading Bu [...]
+<p>Bowtie 2 is using the multithreading software model in order to +speed up execution times on SMP architectures where this is possible. +The Threading Building Blocks library, TBB, is now the default +threading library in bowtie2. On POSIX platforms (like linux, Mac +OS, etc) if TBB is not available the pthread library will be used. +Although it is possible to use pthread library on Windows, a non-POSIX +platform, due to performance reasons bowtie 2 will try to use Windows +native mult [...]
<h2 id="adding-to-path">Adding to PATH</h2>
<p>By adding your new Bowtie 2 directory to your <a href="http://en.wikipedia.org/wiki/PATH_(variable)">PATH environment variable</a>, you ensure that whenever you run <code>bowtie2</code>, <code>bowtie2-build</code> or <code>bowtie2-inspect</code> from the command line, you will get the version you just installed without having to specify the entire path. This is recommended for most users. To do this, follow your operating system's instructions for adding the directory to your <a href= [...]
<p>If you would like to install Bowtie 2 by copying the Bowtie 2 executable files to an existing directory in your <a href="http://en.wikipedia.org/wiki/PATH_(variable)">PATH</a>, make sure that you copy all the executables, including <code>bowtie2</code>, <code>bowtie2-align-s</code>, <code>bowtie2-align-l</code>, <code>bowtie2-build</code>, <code>bowtie2-build-s</code>, <code>bowtie2-build-l</code>, <code>bowtie2-inspect</code>, <code>bowtie2-inspect-s</code> and <code>bowtie2-inspect- [...]
@@ -192,7 +192,7 @@ Alignment:
<h3 id="some-sam-flags-describe-paired-end-properties">Some SAM FLAGS describe paired-end properties</h3>
<p>The SAM <code>FLAGS</code> field, the second field in a SAM record, has multiple bits that describe the paired-end nature of the read and alignment. The first (least significant) bit (1 in decimal, 0x1 in hexadecimal) is set if the read is part of a pair. The second bit (2 in decimal, 0x2 in hexadecimal) is set if the read is part of a pair that aligned in a paired-end fashion. The fourth bit (8 in decimal, 0x8 in hexadecimal) is set if the read is part of a pair and the other mate in [...]
<h3 id="some-sam-optional-fields-describe-more-paired-end-properties">Some SAM optional fields describe more paired-end properties</h3>
-<p>The last severeal fields of each SAM record usually contain SAM optional fields, which are simply tab-separated strings conveying additional information about the reads and alignments. A SAM optional field is formatted like this: "XP:i:1" where "XP" is the <code>TAG</code>, "i" is the <code>TYPE</code> ("integer" in this case), and "1" is the <code>VALUE</code>. See the <a href="http://samtools.sourceforge.net/SAM1.pdf">SAM specificati [...]
+<p>The last several fields of each SAM record usually contain SAM optional fields, which are simply tab-separated strings conveying additional information about the reads and alignments. A SAM optional field is formatted like this: "XP:i:1" where "XP" is the <code>TAG</code>, "i" is the <code>TYPE</code> ("integer" in this case), and "1" is the <code>VALUE</code>. See the <a href="http://samtools.sourceforge.net/SAM1.pdf">SAM specificatio [...]
<h3 id="mates-can-overlap-contain-or-dovetail-each-other">Mates can overlap, contain, or dovetail each other</h3>
<p>The fragment and read lengths might be such that alignments for the two mates from a pair overlap each other. Consider this example:</p>
<p>(For these examples, assume we expect mate 1 to align to the left of mate 2.)</p>
@@ -225,17 +225,17 @@ Reference: GCAGATTATATGAGTCAGCTACGATATTGTTTGGGGTGACACATTACGCGTCTTTGAC</code></pr
<p>See also: <a href="#bowtie2-options-R"><code>-R</code></a>, which sets the maximum number of times Bowtie 2 will "re-seed" when attempting to align a read with repetitive seeds. Increasing <a href="#bowtie2-options-R"><code>-R</code></a> makes Bowtie 2 slower, but increases the likelihood that it will report the correct alignment for a read that aligns many places.</p>
<h3 id="k-mode-search-for-one-or-more-alignments-report-each">-k mode: search for one or more alignments, report each</h3>
<p>In <a href="#bowtie2-options-k"><code>-k</code></a> mode, Bowtie 2 searches for up to N distinct, valid alignments for each read, where N equals the integer specified with the <code>-k</code> parameter. That is, if <code>-k 2</code> is specified, Bowtie 2 will search for at most 2 distinct alignments. It reports all alignments found, in descending order by alignment score. The alignment score for a paired-end alignment equals the sum of the alignment scores of the individual mates. Ea [...]
-<p>Bowtie 2 does not "find" alignments in any specific order, so for reads that have more than N distinct, valid alignments, Bowtie 2 does not garantee that the N alignments reported are the best possible in terms of alignment score. Still, this mode can be effective and fast in situations where the user cares more about whether a read aligns (or aligns a certain number of times) than where exactly it originated.</p>
+<p>Bowtie 2 does not "find" alignments in any specific order, so for reads that have more than N distinct, valid alignments, Bowtie 2 does not guarantee that the N alignments reported are the best possible in terms of alignment score. Still, this mode can be effective and fast in situations where the user cares more about whether a read aligns (or aligns a certain number of times) than where exactly it originated.</p>
<h3 id="a-mode-search-for-and-report-all-alignments">-a mode: search for and report all alignments</h3>
<p><a href="#bowtie2-options-a"><code>-a</code></a> mode is similar to <a href="#bowtie2-options-k"><code>-k</code></a> mode except that there is no upper limit on the number of alignments Bowtie 2 should report. Alignments are reported in descending order by alignment score. The alignment score for a paired-end alignment equals the sum of the alignment scores of the individual mates. Each reported read or pair alignment beyond the first has the SAM 'secondary' bit (which equals 256) set [...]
<p>Some tools are designed with this reporting mode in mind. Bowtie 2 is not! For very large genomes, this mode is very slow.</p>
<h3 id="randomness-in-bowtie-2">Randomness in Bowtie 2</h3>
-<p>Bowtie 2's search for alignments for a given read is "randomized." That is, when Bowtie 2 encounters a set of equally-good choices, it uses a pseudo-random number to choose. For example, if Bowtie 2 discovers a set of 3 equally-good alignments and wants to decide which to report, it picks a pseudo-random integer 0, 1 or 2 and reports the corresponding alignment. Abitrary choices can crop up at various points during alignment.</p>
+<p>Bowtie 2's search for alignments for a given read is "randomized." That is, when Bowtie 2 encounters a set of equally-good choices, it uses a pseudo-random number to choose. For example, if Bowtie 2 discovers a set of 3 equally-good alignments and wants to decide which to report, it picks a pseudo-random integer 0, 1 or 2 and reports the corresponding alignment. Arbitrary choices can crop up at various points during alignment.</p>
<p>The pseudo-random number generator is re-initialized for every read, and the seed used to initialize it is a function of the read name, nucleotide string, quality string, and the value specified with <a href="#bowtie2-options-seed"><code>--seed</code></a>. If you run the same version of Bowtie 2 on two reads with identical names, nucleotide strings, and quality strings, and if <a href="#bowtie2-options-seed"><code>--seed</code></a> is set the same for both runs, Bowtie 2 will produce [...]
<p>However, when the user specifies the <a href="#bowtie2-options-non-deterministic"><code>--non-deterministic</code></a> option, Bowtie 2 will use the current time to re-initialize the pseudo-random number generator. When this is specified, Bowtie 2 might report different alignments for identical reads. This is counter-intuitive for some users, but might be more appropriate in situations where the input consists of many identical reads.</p>
<h2 id="multiseed-heuristic">Multiseed heuristic</h2>
<p>To rapidly narrow the number of possible alignments that must be considered, Bowtie 2 begins by extracting substrings ("seeds") from the read and its reverse complement and aligning them in an ungapped fashion with the help of the <a href="http://portal.acm.org/citation.cfm?id=796543">FM Index</a>. This is "multiseed alignment" and it is similar to what <a href="http://genomebiology.com/2009/10/3/R25">Bowtie 1 does</a>, except Bowtie 1 attempts to align the entire [...]
-<p>This initial step makes Bowtie 2 much faster than it would be without such a filter, but at the expense of missing some valid alignments. For instance, it is possible for a read to have a valid overall alignment but to have no valid seed alignments because each potential seed alignment is interruped by too many mismatches or gaps.</p>
+<p>This initial step makes Bowtie 2 much faster than it would be without such a filter, but at the expense of missing some valid alignments. For instance, it is possible for a read to have a valid overall alignment but to have no valid seed alignments because each potential seed alignment is interrupted by too many mismatches or gaps.</p>
<p>The tradeoff between speed and sensitivity/accuracy can be adjusted by setting the seed length (<a href="#bowtie2-options-L"><code>-L</code></a>), the interval between extracted seeds (<a href="#bowtie2-options-I"><code>-i</code></a>), and the number of mismatches permitted per seed (<a href="#bowtie2-options-N"><code>-N</code></a>). For more sensitive alignment, set these parameters to (a) make the seeds closer together, (b) make the seeds shorter, and/or (c) allow more mismatches. Y [...]
<p><a href="#bowtie2-options-D"><code>-D</code></a> and <a href="#bowtie2-options-R"><code>-R</code></a> are also options that adjust the tradeoff between speed and sensitivity/accuracy.</p>
<h3 id="fm-index-memory-footprint">FM Index memory footprint</h3>
@@ -255,7 +255,7 @@ Reference: GCAGATTATATGAGTCAGCTACGATATTGTTTGGGGTGACACATTACGCGTCTTTGAC</code></pr
<li><code>YF:Z:QC</code>: the read was filtered because it was marked as failing quality control and the user specified the <a href="#bowtie2-options-qc-filter"><code>--qc-filter</code></a> option. This only happens when the input is in Illumina's QSEQ format (i.e. when <a href="#bowtie2-options-qseq"><code>--qseq</code></a> is specified) and the last (11th) field of the read's QSEQ record contains <code>1</code>.</li>
</ul>
<p>If a read could be filtered for more than one reason, the value <code>YF:Z</code> flag will reflect only one of those reasons.</p>
-<h2 id="alignment-summmary">Alignment summmary</h2>
+<h2 id="alignment-summary">Alignment summary</h2>
<p>When Bowtie 2 finishes running, it prints messages summarizing what happened. These messages are printed to the "standard error" ("stderr") filehandle. For datasets consisting of unpaired reads, the summary might look like this:</p>
<pre><code>20000 reads; of these:
20000 (100.00%) were unpaired; of these:
@@ -281,7 +281,7 @@ Reference: GCAGATTATATGAGTCAGCTACGATATTGTTTGGGGTGACACATTACGCGTCTTTGAC</code></pr
96.70% overall alignment rate</code></pre>
<p>The indentation indicates how subtotals relate to totals.</p>
<h2 id="wrapper-scripts">Wrapper scripts</h2>
-<p>The <code>bowtie2</code>, <code>bowtie2-build</code> and <code>bowtie2-inspect</code> executables are actually wrapper scripts that call binary programs as appropriate. The wrappers shield users from having to distinguish between "small" and "large" index formats, discussed briefly in the following section. Also, the <code>bowtie2</code> wrapper provides some key functionality, like the ability to handle compressed inputs, and the fucntionality for <a href="#bowtie [...]
+<p>The <code>bowtie2</code>, <code>bowtie2-build</code> and <code>bowtie2-inspect</code> executables are actually wrapper scripts that call binary programs as appropriate. The wrappers shield users from having to distinguish between "small" and "large" index formats, discussed briefly in the following section. Also, the <code>bowtie2</code> wrapper provides some key functionality, like the ability to handle compressed inputs, and the functionality for <a href="#bowtie [...]
<p>It is recommended that you always run the bowtie2 wrappers and not run the binaries directly.</p>
<h2 id="small-and-large-indexes">Small and large indexes</h2>
<p><code>bowtie2-build</code> can index reference genomes of any size. For genomes less than about 4 billion nucleotides in length, <code>bowtie2-build</code> builds a "small" index using 32-bit numbers in various parts of the index. When the genome is longer, <code>bowtie2-build</code> builds a "large" index using 64-bit numbers. Small indexes are stored in files with the <code>.bt2</code> extension, and large indexes are stored in files with the <code>.bt2l</code> e [...]
@@ -683,7 +683,7 @@ Seed 4 rc: TTATGCATGA</code></pre>
</td><td>
<p>The minimum fragment length for valid paired-end alignments. E.g. if <code>-I 60</code> is specified and a paired-end alignment consists of two 20-bp alignments in the appropriate orientation with a 20-bp gap between them, that alignment is considered valid (as long as <a href="#bowtie2-options-X"><code>-X</code></a> is also satisfied). A 19-bp gap would not be valid in that case. If trimming options <a href="#bowtie2-options-3"><code>-3</code></a> or <a href="#bowtie2-options-5"><cod [...]
-<p>The larger the difference between <a href="#bowtie2-options-I"><code>-I</code></a> and <a href="#bowtie2-options-X"><code>-X</code></a>, the slower Bowtie 2 will run. This is because larger differences bewteen <a href="#bowtie2-options-I"><code>-I</code></a> and <a href="#bowtie2-options-X"><code>-X</code></a> require that Bowtie 2 scan a larger window to determine if a concordant alignment exists. For typical fragment length ranges (200 to 400 nucleotides), Bowtie 2 is very efficient.</p>
+<p>The larger the difference between <a href="#bowtie2-options-I"><code>-I</code></a> and <a href="#bowtie2-options-X"><code>-X</code></a>, the slower Bowtie 2 will run. This is because larger differences between <a href="#bowtie2-options-I"><code>-I</code></a> and <a href="#bowtie2-options-X"><code>-X</code></a> require that Bowtie 2 scan a larger window to determine if a concordant alignment exists. For typical fragment length ranges (200 to 400 nucleotides), Bowtie 2 is very efficient.</p>
<p>Default: 0 (essentially imposing no minimum)</p>
</td></tr>
<tr><td id="bowtie2-options-X">
@@ -692,7 +692,7 @@ Seed 4 rc: TTATGCATGA</code></pre>
</td><td>
<p>The maximum fragment length for valid paired-end alignments. E.g. if <code>-X 100</code> is specified and a paired-end alignment consists of two 20-bp alignments in the proper orientation with a 60-bp gap between them, that alignment is considered valid (as long as <a href="#bowtie2-options-I"><code>-I</code></a> is also satisfied). A 61-bp gap would not be valid in that case. If trimming options <a href="#bowtie2-options-3"><code>-3</code></a> or <a href="#bowtie2-options-5"><code>-5 [...]
-<p>The larger the difference between <a href="#bowtie2-options-I"><code>-I</code></a> and <a href="#bowtie2-options-X"><code>-X</code></a>, the slower Bowtie 2 will run. This is because larger differences bewteen <a href="#bowtie2-options-I"><code>-I</code></a> and <a href="#bowtie2-options-X"><code>-X</code></a> require that Bowtie 2 scan a larger window to determine if a concordant alignment exists. For typical fragment length ranges (200 to 400 nucleotides), Bowtie 2 is very efficient.</p>
+<p>The larger the difference between <a href="#bowtie2-options-I"><code>-I</code></a> and <a href="#bowtie2-options-X"><code>-X</code></a>, the slower Bowtie 2 will run. This is because larger differences between <a href="#bowtie2-options-I"><code>-I</code></a> and <a href="#bowtie2-options-X"><code>-X</code></a> require that Bowtie 2 scan a larger window to determine if a concordant alignment exists. For typical fragment length ranges (200 to 400 nucleotides), Bowtie 2 is very efficient.</p>
<p>Default: 500.</p>
</td></tr>
<tr><td id="bowtie2-options-fr">
@@ -1147,13 +1147,13 @@ A string representation of the mismatched reference bases in the alignment. See
<pre><code>--bmax <int></code></pre>
</td><td>
-<p>The maximum number of suffixes allowed in a block. Allowing more suffixes per block makes indexing faster, but increases peak memory usage. Setting this option overrides any previous setting for <a href="#bowtie2-build-options-bmax"><code>--bmax</code></a>, or <a href="#bowtie2-build-options-bmaxdivn"><code>--bmaxdivn</code></a>. Default (in terms of the <a href="#bowtie2-build-options-bmaxdivn"><code>--bmaxdivn</code></a> parameter) is <a href="#bowtie2-build-options-bmaxdivn"><code> [...]
+<p>The maximum number of suffixes allowed in a block. Allowing more suffixes per block makes indexing faster, but increases peak memory usage. Setting this option overrides any previous setting for <a href="#bowtie2-build-options-bmax"><code>--bmax</code></a>, or <a href="#bowtie2-build-options-bmaxdivn"><code>--bmaxdivn</code></a>. Default (in terms of the <a href="#bowtie2-build-options-bmaxdivn"><code>--bmaxdivn</code></a> parameter) is <a href="#bowtie2-build-options-bmaxdivn"><code> [...]
</td></tr><tr><td id="bowtie2-build-options-bmaxdivn">
<pre><code>--bmaxdivn <int></code></pre>
</td><td>
-<p>The maximum number of suffixes allowed in a block, expressed as a fraction of the length of the reference. Setting this option overrides any previous setting for <a href="#bowtie2-build-options-bmax"><code>--bmax</code></a>, or <a href="#bowtie2-build-options-bmaxdivn"><code>--bmaxdivn</code></a>. Default: <a href="#bowtie2-build-options-bmaxdivn"><code>--bmaxdivn</code></a> 4. This is configured automatically by default; use <a href="#bowtie2-build-options-a"><code>-a</code>/<code>-- [...]
+<p>The maximum number of suffixes allowed in a block, expressed as a fraction of the length of the reference. Setting this option overrides any previous setting for <a href="#bowtie2-build-options-bmax"><code>--bmax</code></a>, or <a href="#bowtie2-build-options-bmaxdivn"><code>--bmaxdivn</code></a>. Default: <a href="#bowtie2-build-options-bmaxdivn"><code>--bmaxdivn</code></a> 4 * number of threads. This is configured automatically by default; use <a href="#bowtie2-build-options-a"><cod [...]
</td></tr><tr><td id="bowtie2-build-options-dcv">
<pre><code>--dcv <int></code></pre>
@@ -1336,5 +1336,3 @@ r7 16 gi|9626243|ref|NC_001416.1| 4692 42 143M * 0 0 TCAGCCGGACGC
<p>Then to view the variants, run:</p>
<pre><code>bcftools view eg2.raw.bcf</code></pre>
<p>See the official SAMtools guide to <a href="http://samtools.sourceforge.net/mpileup.shtml">Calling SNPs/INDELs with SAMtools/BCFtools</a> for more details and variations on this process.</p>
-</body>
-</html>
diff --git a/doc/website/news.shtml b/doc/website/news.shtml
new file mode 100644
index 0000000..596b4d6
--- /dev/null
+++ b/doc/website/news.shtml
@@ -0,0 +1,35 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head>
+<title>Bowtie 2: fast and sensitive read alignment</title>
+<meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
+<link rel="stylesheet" type="text/css" href="style.css" media="screen" />
+<meta name="verify-v1" content="YJT1CfXN3kzE9cr+jvNB+Q73lTfHrv8eivoY+xjblc0=" />
+</head>
+<body>
+<div id="wrap">
+ <!--#include virtual="top.ssi" -->
+ <div id="main">
+ <!--#include virtual="rhsidebar.ssi" -->
+ <div id="leftside">
+ <h2>News archive</h2>
+ <!--#include virtual="recent_news.ssi" -->
+ <!--#include virtual="old_news.ssi" -->
+ </div>
+ </div>
+ <!--#include virtual="foot.ssi" -->
+</div>
+
+<!-- Google analytics code -->
+<script type="text/javascript">
+var gaJsHost = (("https:" == document.location.protocol) ? "https://ssl." : "http://www.");
+document.write(unescape("%3Cscript src='" + gaJsHost + "google-analytics.com/ga.js' type='text/javascript'%3E%3C/script%3E"));
+</script>
+<script type="text/javascript">
+var pageTracker = _gat._getTracker("UA-5334290-1");
+pageTracker._trackPageview();
+</script>
+<!-- End google analytics code -->
+
+</body>
+</html>
diff --git a/doc/website/old_news.ssi b/doc/website/old_news.ssi
new file mode 100644
index 0000000..dc0983b
--- /dev/null
+++ b/doc/website/old_news.ssi
@@ -0,0 +1,359 @@
+<h2>Bowtie 2 on GitHub - February 4, 2014</h2>
+<ul>
+ <li>Bowtie 2 source now lives in a <a href="https://github.com/BenLangmead/bowtie2">public GitHub repository</a>.</li>
+</ul>
+
+<h2>Version 2.2.0 - February 17, 2014</h2>
+<ul>
+ <li>Improved index querying efficiency using "population count"
+ instructions available since
+ <a href="http://en.wikipedia.org/wiki/SSE4.2#SSE4.2">SSE4.2</a></li>
+ <li>Added support for large and small indexes, removing 4-billion-nucleotide
+ barrier. Bowtie 2 can now be used with reference genomes of any size.</li>
+ <li>Fixed bug that could cause bowtie2-build to crash when reference length
+ is close to 4 billion.</li>
+ <li>Added a <tt>CL:</tt> string to the <tt>@PG</tt> SAM header to preserve information about
+ the aligner binary and paramteres.</li>
+ <li>Fixed bug that could cause <tt>bowtie2-build</tt> to crash when reference length
+ is close to 4 billion.</li>
+ <li>No longer releasing 32-bit binaries. Simplified manual and Makefile
+ accordingly.</li>
+ <li>Credits to the Intel® enabling team for performance optimizations
+ included in this release. Thank you!</li>
+ <li>Phased out CygWin support. MinGW can still be used for Windows building.</li>
+ <li>Added the .bat generation for Windows.</li>
+ <li>Fixed some issues with some uncommon chars in fasta files.</li>
+ <li>Fixed wrappers so bowtie can now be used with symlinks.</li>
+</ul>
+
+<h2>Version 2.1.0 - February 21, 2013</h2>
+<ul>
+ <li>Improved multithreading support so that Bowtie 2 now uses native Windows
+ threads when compiled on Windows and uses a faster mutex. Threading
+ performance should improve on all platforms.</li>
+ <li>Improved support for building 64-bit binaries for Windows x64
+ platforms.</li>
+ <li>Bowtie 2 uses a lightweight mutex by default.</li>
+ <li>Test option <tt>--nospin</tt> is no longer available. However bowtie2
+ can always be recompiled with <tt>EXTRA_FLAGS="-DNO_SPINLOCK"</tt> in
+ order to drop the default spinlock usage.
+</ul>
+
+<h2>Version 2.0.6 - January 27, 2013</h2>
+<ul>
+ <li>Fixed issue whereby spurious output would be written in
+ <tt><a href="manual.shtml#bowtie2-options-no-unal">--no-unal</a></tt> mode.
+ <li>Fixed issue whereby multiple input files combined with
+ <tt><a href="manual.shtml#bowtie2-options-reorder">--reorder</a></tt>
+ would
+ cause truncated output and a memory spike.
+ <li>Fixed spinlock datatype for Win64 API (LLP64 data model) which made it
+ crash when compiled under Windows 7 x64.
+ <li>Fixed bowtie2 wrapper to handle filename/paths operations in a more
+ platform independent manner.
+ <li>Added pthread as a default library option under cygwin, and pthreadGC
+ for MinGW.
+ <li>Fixed some minor issues that made MinGW compilation fail.
+</ul>
+
+<h2>Version 2.0.5 - January 4, 2013</h2>
+<ul>
+ <li>Fixed an issue that would cause excessive memory allocation when aligning
+ to very repetitive genomes.</li>
+ <li>Fixed an issue that would cause a pseudo-randomness-related assert to be
+ thrown in debug mode under rare circumstances.</li>
+ <li>When <tt>bowtie2-build</tt> fails, it will now delete index files created so far so
+ that invalid index files don't linger.</li>
+ <li>Tokenizer no longer has limit of 10,000 tokens, which was a problem for
+ users trying to index a very large number of FASTA files.</li>
+ <li>Updated manual's discussion of the
+ <tt><a href="manual.shtml#bowtie2-options-I">-I</a></tt> and
+ <tt><a href="manual.shtml#bowtie2-options-X">-X</a></tt> options to mention that
+ setting them farther apart makes Bowtie 2 slower.</li>
+ <li>Renamed <tt>COPYING</tt> to <tt>LICENSE</tt> and created a <tt>README</tt> to be GitHub-friendly.</li>
+</ul>
+
+<h2>Version 2.0.4 - December 17, 2012</h2>
+<ul>
+ <li>Fixed issue whereby
+<tt><a href="manual.shtml#bowtie2-options-un">--un</a></tt>,
+<tt><a href="manual.shtml#bowtie2-options-al">--al</a></tt>,
+<tt><a href="manual.shtml#bowtie2-options-un-conc">--un-conc</a></tt>, and
+<tt><a href="manual.shtml#bowtie2-options-al-conc">--al-conc</a></tt>
+ options would
+ incorrectly suppress SAM output.</li>
+ <li>Fixed minor command-line parsing issue in wrapper script.</li>
+ <li>Fixed issue on Windows where wrapper script would fail to find
+ <tt>bowtie2-align.exe</tt> binary.</li>
+ <li>Updated some of the index-building scripts and documentation.</li>
+ <li>Updated author's contact info in usage message</li>
+</ul>
+
+<h2>Version 2.0.3 - December 14, 2012</h2>
+<ul>
+<li>Fixed thread safely issues that could cause crashes with a large number of
+ threads. Thanks to John O’Neill for identifying these issues.</li>
+<li>Fixed some problems with pseudo-random number generation that could cause
+ unequal distribution of alignments across equally good candidate loci.</li>
+<li>The
+<tt><a href="manual.shtml#bowtie2-options-un">--un</a></tt>,
+<tt><a href="manual.shtml#bowtie2-options-al">--al</a></tt>,
+<tt><a href="manual.shtml#bowtie2-options-un-conc">--un-conc</a></tt>, and
+<tt><a href="manual.shtml#bowtie2-options-al-conc">--al-conc</a></tt>
+ options (and their compressed
+ analogs) are all much faster now, making it less likely that they become
+ the bottleneck when Bowtie 2 is run with large
+<tt><a href="manual.shtml#bowtie2-options-p">-p</a></tt>.</li>
+<li>Fixed issue with innaccurate mapping qualities, <tt>XS:i</tt>, and <tt>YS:i</tt> flags when
+<tt><a href="manual.shtml#bowtie2-options-no-mixed">--no-mixed</a></tt> and
+<tt><a href="manual.shtml#bowtie2-options-no-discordant">--no-discordant</a></tt>
+are specified at the same time.</li>
+<li>Fixed some compiler warnings and errors when using <tt>clang++</tt> to compile.</li>
+<li>Fixed race condition in <tt>bowtie2</tt> script when named pipes are used.</li>
+<li>Added more discussion of whitespace in read names to manual.</li>
+</ul>
+
+<h2>Sourceforge spam</h2>
+<ul>
+<li>Spam on the sourceforge tracker (i.e. where feature requests and bug
+ reports go) was getting out of control, so I disabled posting by anonymous
+ users. This means you'll have to use some set of credentials when posting
+ on the tracker. Sourceforge allows you to use various, e.g., your Google
+ credentials. Sorry for the invonvenience, but I think this will make the
+ experience better overall.
+</ul>
+
+<h2>Version 2.0.2 - October 31, 2012</h2>
+<ul>
+ <li>Fixes a couple small issues pointed out to me immediately after 2.0.1
+ release</li>
+ <li>Mac binaries now built on 10.6 in order to be forward-compatible with more
+ Mac OS versions</li>
+ <li>Small changes to source to make it compile with gcc versions up to 4.7
+ without warnings</li>
+</ul>
+
+<h2>Version 2.0.1 - October 31, 2012</h2>
+<ul>
+ <li>First non-beta release.</li>
+ <li>Fixed an issue that would cause Bowtie 2 to use excessive amounts of
+ memory for closely-matching and highly repetitive reads under some
+ circumstances.</li>
+ <li>Fixed a bug in
+ <tt><a href="manual.shtml#bowtie2-options-mm">--mm</a></tt>
+ mode that would fail to report when an index file
+ could not be memory-mapped.</li>
+ <li>Added
+ <tt><a href="manual.shtml#bowtie2-options-non-deterministic">--non-deterministic</a></tt>
+ option, which better matches how some users
+ expect the pseudo-random generator inside Bowtie 2 to work. Normally, if
+ you give the same read (same name, sequence and qualities) and
+ <tt><a href="manual.shtml#bowtie2-options-seed">--seed</a></tt>, you
+ get the same answer.
+ <tt><a href="manual.shtml#bowtie2-options-non-deterministic">--non-deterministic</a></tt>
+ breaks that guarantee. This can
+ be more appropriate for datasets where the input contains many identical
+ reads (same name, same sequence, same qualities).</li>
+ <li>Fixed a bug in <tt>bowtie2-build</tt> would yield corrupt index files when memory
+ settings were adjusted in the middle of indexing.</li>
+ <li>Clarified in manual that <tt>--un-*</tt> options print reads exactly as
+ they appeared in the input, and that they are not necessarily written in
+ the same order as they appeared in the input.</li>
+ <li>Fixed issue whereby wrapper would incorrectly interpret arguments with
+ <tt>--al</tt> as a prefix (e.g.
+ <tt><a href="manual.shtml#bowtie2-options-a">--all</a></tt>) as
+ <tt><a href="manual.shtml#bowtie2-options-al">--al</a></tt>.</li>
+</ul>
+
+<h2>Version 2.0.0-beta7 - July 13, 2012</h2>
+<ul>
+ <li>Fixed an issue in how Bowtie 2 aligns longer reads in
+ <tt><a href="manual.shtml#bowtie2-options-local">--local</a></tt>.
+ mode. Some
+ alignments were incorrectly curtailed on the left-hand side.</li>
+ <li>Fixed issue whereby
+ <tt><a href="manual.shtml#bowtie2-options-un">--un</a></tt> or
+ <tt><a href="manual.shtml#bowtie2-options-un-conc">--un-conc</a></tt>
+ would fail to output unaligned reads when
+ <tt><a href="manual.shtml#bowtie2-options-no-unal">--no-unal</a></tt>
+ was also specified.</li>
+ <li>Fixed issue whereby
+ <tt><a href="manual.shtml#bowtie2-options-un">--un</a></tt> or
+ <tt><a href="manual.shtml#bowtie2-options-un-conc">--un-conc</a></tt>
+ were significantly slowing down Bowtie 2 when
+ <tt><a href="manual.shtml#bowtie2-options-p">-p</a></tt>
+ was set greater than 1.</li>
+ <li>Fixed issue that would could cause hangs in
+ <tt><a href="manual.shtml#bowtie2-options-a">-a</a></tt>
+ mode or when
+ <tt><a href="manual.shtml#bowtie2-options-k">-k</a></tt>
+ was set high.</li>
+ <li>Fixed issue whereby the SAM FLAGS field could be set incorrectly for
+ secondary paired-end alignments with
+ <tt><a href="manual.shtml#bowtie2-options-a">-a</a></tt>
+ or
+ <tt><a href="manual.shtml#bowtie2-options-k">-k</a></tt>
+ > 1.</li>
+ <li>When input reads are unpaired, Bowtie 2 no longer removes the trailing
+ <tt>/1</tt> or <tt>/2</tt> from the read name.</li>
+ <li><tt>-M</tt> option is now deprecated. It will be removed in subsequent versions.
+ What used to be called <tt>-M</tt> mode is still the default mode, and
+ <tt><a href="manual.shtml#bowtie2-options-k">-k</a></tt>
+ and
+ <tt><a href="manual.shtml#bowtie2-options-a">-a</a></tt>
+ are still there alternatives to the default mode, but adjusting the <tt>-M</tt>
+ setting is deprecated. Use the
+ <tt><a href="manual.shtml#bowtie2-options-D">-D</a></tt>
+ and
+ <tt><a href="manual.shtml#bowtie2-options-R">-R</a></tt>
+ options to adjust the effort
+ expended to find valid alignments.</li>
+ <li>Gaps are now left-aligned in a manner similar to BWA and other tools.</li>
+ <li>Fixed issue whereby wrapper script would not pass on exitlevel correctly,
+ sometimes spuriously hiding non-0 exitlevel.</li>
+ <li>Added documentation for <tt>YT:Z</tt> to manual.</li>
+ <li>Fixed documentation describing how Bowtie 2 searches for an index given an
+ index basename.</li>
+ <li>Fixed inconsistent documentation for the default value of the
+ <tt><a href="manual.shtml#bowtie2-options-i">-i</a></tt>
+ parameter </li>
+</ul>
+
+<h2>Added rat pre-built index - May 17 2012</h2>
+
+<h2>Version 2.0.0-beta6 - May 7, 2012</h2>
+<ul>
+ <li>Bowtie 2 now handles longer reads in a more memory-economical fashion,
+ which should prevent many out-of-memory issues for longer reads.</li>
+ <li>Error message now produced when -L is set greater than 32.</li>
+ <li>Added a warning message to warn when <tt>bowtie2-align</tt> binary is
+ being run directly, rather than via the wrapper. Some functionality is
+ provided by the wrapper, so Bowtie 2 should always be run via the
+ <tt>bowtie2</tt> executable rather than <tt>bowtie2-align</tt>.</li>
+ <li>Fixed an occasional crashing bug that was usually caused by setting the
+ seed length relatively short.</li>
+ <li>Fixed an issue whereby the FLAG, RNEXT and PNEXT fields were incorrect
+ for some paired-end alignments. Specifically, this affected paired-end
+ alignments where both mates aligned and one or both mates aligned
+ non-uniquely.</li>
+ <li>Fixed issue whereby compressed input would sometimes be mishandled.</li>
+ <li>Renamed <tt>--sam-*</tt> options to omit the <tt>sam-</tt> part for
+ brevity. The old option names will also work.</li>
+ <li>Added
+ <tt><a href="manual.shtml#bowtie2-options-no-unal">--no-unal</a></tt>
+ option to suppress SAM records corresponding to unaligned
+ reads, i.e., records where <tt>FLAG</tt> field has <tt>0x4</tt> set.</li>
+ <li>Added
+ <tt><a href="manual.shtml#bowtie2-options-rg-id">--rg-id</a></tt>
+ option and enhanced the documentation for both
+ <tt><a href="manual.shtml#bowtie2-options-rg-id">--rg-id</a></tt>
+ and
+ <tt><a href="manual.shtml#bowtie2-options-rg">--rg</a></tt>.
+ Users were confused by the need to specify
+ <tt>--rg "ID:(something)"</tt>
+ in order for the <tt>@RG</tt> line to be printed; hopefully this is
+ clearer now.</li>
+ <li>Index updates: indexes linked to in the right-hand sidebar have been
+ updated to include the unplaced contigs appearing in the UCSC "random"
+ FASTA files. This makes the indexes more complete. Also, an index for
+ the latest mouse assembly, mm10 (AKA "GRCm38") has been added.</li>
+</ul>
+
+<h2>Version 2.0.0-beta5 - December 15, 2011</h2>
+<ul>
+ <li>Added
+ <tt><a href="manual.shtml#bowtie2-options-un">--un</a></tt>,
+ <tt><a href="manual.shtml#bowtie2-options-al">--al</a></tt>,
+ <tt><a href="manual.shtml#bowtie2-options-un-conc">--un-conc</a></tt>, and
+ <tt><a href="manual.shtml#bowtie2-options-al-conc">--al-conc</a></tt> options that write unpaired
+ and/or paired-end reads to files depending on whether they align at least
+ once or fail to align.</li>
+ <li>Added
+ <tt><a href="manual.shtml#bowtie2-options-reorder">--reorder</a></tt>
+ option. When enabled, the order of the SAM records output
+ by Bowtie 2 will match the order of the input reads even when
+ <tt><a href="manual.shtml#bowtie2-options-p">-p</a></tt>
+ is set
+ greater than 1. This is disabled by default; enabling it makes Bowtie 2
+ somewhat slower and use somewhat more memory when
+ <tt><a href="manual.shtml#bowtie2-options-p">-p</a></tt>
+ is set greater than 1.</li>
+ <li>Changed the default
+ <tt><a href="manual.shtml#bowtie2-options-score-min">--score-min</a></tt> in
+ <tt><a href="manual.shtml#bowtie2-options-local">--local</a></tt>
+ mode to <tt>G,20,8</tt>. This ought to improve sensitivity and accuracy in
+ many cases.</li>
+ <li>Fixed some minor documentation issues.</li>
+ <li>Note: I am aware of an issue whereby longer reads (>10,000 bp) drive the
+ memory footprint way up and often cause an out-of-memory exception. This
+ will be fixed in a future version.</li>
+</ul>
+
+<h2>Version 2.0.0-beta4 - December 6, 2011</h2>
+<ul>
+ <li>Accuracy improvements.</li>
+ <li>Speed improvements in some situations.</li>
+ <li>Fixed a handful of crashing bugs.</li>
+ <li>Fixed some documentation bugs.</li>
+ <li>Fixed bug whereby <tt><a href="manual.shtml#bowtie2-options-version">--version</tt></a> worked incorrectly.</li>
+ <li>Fixed formatting bug with MD:Z optional field that would sometimes fail to follow a mismatch with a number.</li>
+ <li>Added
+ <tt><a href="manual.shtml#bowtie2-options-D">-D</a></tt>
+ option for controlling the maximum number of seed extensions that can fail in a row before we move on. This option or something like it will eventually replace the argument to
+ <tt><a href="manual.shtml#bowtie2-options-M">-M</a></tt>.</li>
+ <li>Added
+ <tt><a href="manual.shtml#bowtie2-options-R">-R</a></tt>
+ option to control maximum number of times re-seeding is attempted for a read with repetitive seeds.</li>
+ <li>Changed default to
+ <tt>--no-dovetail</tt>. Specifying
+ <tt><a href="manual.shtml#bowtie2-options-dovetail">--dovetail</a></tt> turns it back on.</li>
+ <li>Added second argument for
+ <tt><a href="manual.shtml#bowtie2-options-mp">--mp</a></tt> option so that user can set maximum and minimum mismatch penalties at once. Also tweaked the formula for calculating the quality-aware mismatch penalty.</li>
+</ul>
+
+<h2>Version 2.0.0-beta3 - November 1, 2011</h2>
+<ul>
+ <li>Accuracy improvements.</li>
+ <li>Speed improvements in some situations.</li>
+ <li>Fixed a handful of crashing bugs.</li>
+ <li>Fixed a bug whereby number of repetitively aligned reads could be misreported in the summary output.</li>
+ <li>As always, thanks to everyone for their reports and feedback! Please keep it coming.</li>
+</ul>
+
+<h2>Version 2.0.0-beta2 - October 16, 2011</h2>
+<ul>
+ <li>Added manual, both included in the download package and
+ <a href="manual.shtml">on the website</a>.
+ The website will always have the manual for the latest version.</li>
+ <li>Added Linux 32-bit and 64-bit binary packages. Mac OS X packages to come.
+ Still working on a Windows package.</li>
+ <li>Fixed a bug that led to crashes when seed-alignment result memory was
+ exhausted.</li>
+ <li>Changed the
+ <tt><a href="manual.shtml#bowtie2-options-end-to-end">--end-to-end</a></tt>
+ mode
+ <tt><a href="manual.shtml#bowtie2-options-score-min">--score-min</a></tt>
+ default to be less permissive.
+ The previous threshold seemed to be having an adverse effect on accuracy,
+ though the fix implemented in this version comes at the expense of some
+ sensitivity.</li>
+ <li>Changed the
+ <tt><a href="manual.shtml#bowtie2-options-end-to-end">--end-to-end</a></tt>
+ mode
+ <tt><a href="manual.shtml#bowtie2-options-M">-M</a></tt>
+ default to be lower by 2 notches. This
+ offsets any adverse effect that the previous change would have had on
+ speed, without a large adverse impact on accuracy. As always, setting
+ <tt><a href="manual.shtml#bowtie2-options-M">-M</a></tt>
+ higher will yield still greater accuracy at the expense of speed.</li>
+</ul>
+
+<h2>Version 2.0.0-beta1 - September 22, 2011</h2>
+<ul>
+ <li>First public release</li>
+ <li>Caveats: as of now, the manual is incomplete, there's no tutorial, and no
+ example genome or example reads. All these will be fixed in upcoming
+ releases.</li>
+ <li>Only a source package is currently available. Platform-specific binaries
+ will be included in future releases.</li>
+</ul>
diff --git a/doc/website/other_tools.shtml b/doc/website/other_tools.shtml
new file mode 100644
index 0000000..1d9a754
--- /dev/null
+++ b/doc/website/other_tools.shtml
@@ -0,0 +1,33 @@
+<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.0 Strict//EN" "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd">
+<html xmlns="http://www.w3.org/1999/xhtml">
+<head>
+<title>Bowtie 2: fast and sensitive read alignment</title>
+<meta http-equiv="Content-Type" content="text/html; charset=UTF-8" />
+<link rel="stylesheet" type="text/css" href="style.css" media="screen" />
+<meta name="verify-v1" content="YJT1CfXN3kzE9cr+jvNB+Q73lTfHrv8eivoY+xjblc0=" />
+</head>
+<body>
+<div id="wrap">
+ <!--#include virtual="top.ssi" -->
+ <div id="main">
+ <!--#include virtual="rhsidebar.ssi" -->
+ <div id="leftside">
+ <!--#include virtual="other_tools.ssi" -->
+ </div>
+ </div>
+ <!--#include virtual="foot.ssi" -->
+</div>
+
+<!-- Google analytics code -->
+<script type="text/javascript">
+var gaJsHost = (("https:" == document.location.protocol) ? "https://ssl." : "http://www.");
+document.write(unescape("%3Cscript src='" + gaJsHost + "google-analytics.com/ga.js' type='text/javascript'%3E%3C/script%3E"));
+</script>
+<script type="text/javascript">
+var pageTracker = _gat._getTracker("UA-5334290-1");
+pageTracker._trackPageview();
+</script>
+<!-- End google analytics code -->
+
+</body>
+</html>
diff --git a/doc/website/push.sh b/doc/website/push.sh
new file mode 100644
index 0000000..23f3f81
--- /dev/null
+++ b/doc/website/push.sh
@@ -0,0 +1,17 @@
+#!/bin/sh
+
+##
+# push.sh
+#
+# Run this from the $BOWTIE2_HOME/doc/website subdirectory.
+#
+# Copies the files that comprise the website at
+# http://bowtie-bio.sourceforge.net/bowtie2 to sourceforge. You must
+# have the right sourceforge privileges to do this. The SF_USER
+# environment variable must be set appropriately.
+#
+
+[ -z "$SF_USER" ] && echo "Must set SF_USER" && exit 1
+[ -z "$BOWTIE_DOCS" ] && echo "Must set BOWTIE_DOCS" && exit 1
+
+scp -r $BOWTIE_DOCS/other_tools.ssi ../style.css *.ssi *.shtml *.html $SF_USER,bowtie-bio at web.sourceforge.net:/home/groups/b/bo/bowtie-bio/htdocs/bowtie2/
diff --git a/doc/website/push_images.sh b/doc/website/push_images.sh
new file mode 100644
index 0000000..ef7371c
--- /dev/null
+++ b/doc/website/push_images.sh
@@ -0,0 +1,16 @@
+#!/bin/sh
+
+##
+# push.sh
+#
+# Run this from the $BOWTIE2_HOME/doc/website subdirectory.
+#
+# Copies the files that comprise the website at
+# http://bowtie-bio.sourceforge.net/bowtie2 to sourceforge. You must
+# have the right sourceforge privileges to do this. The SF_USER
+# environment variable must be set appropriately.
+#
+
+[ -z "$SF_USER" ] && echo "Must set SF_USER" && exit 1
+
+scp -r ../images $SF_USER,bowtie-bio at web.sourceforge.net:/home/groups/b/bo/bowtie-bio/htdocs/bowtie2/
diff --git a/doc/website/recent_news.ssi b/doc/website/recent_news.ssi
new file mode 100644
index 0000000..958a2fa
--- /dev/null
+++ b/doc/website/recent_news.ssi
@@ -0,0 +1,104 @@
+<h2>Version 2.3.0 - Dec 13, 2016</h2>
+<p>This is a major release with some larger and many smaller changes. These notes emphasize the large changes. See commit history for details.</p>
+<ul>
+ <li>Code related to read parsing was completely rewritten to improve scalability to many threads. In short, the critical section is simpler and parses input reads in batches rather than one at a time. The improvement applies to all read formats.</li>
+ <li>TBB is now the default threading library. We consistently found TBB to give superior thread scaling. It is widely available and widely installed. That said, we are also preserving a "legacy" version of Bowtie that, like previous releases, does not use TBB. To compile Bowtie source in legacy mode use <tt>NO_TBB=1</tt>. To use legacy binaries, download the appropriate binary archive with "legacy" in the name.</li>
+ <li>Bowtie now uses a queue-based lock rather than a spin or heavyweight lock. We find this gives superior thread scaling; we saw an order-of-magnitude throughput improvements at 120 threads in one experiment, for example.</li>
+ <li>Unnecessary thread synchronization removed</li>
+ <li>Fixed issue with parsing FASTA records with greater-than symbol in the name</li>
+ <li>Now detects and reports inconsistencies between <tt><a href="manual.shtml#bowtie2-options-score-min">--score-min</a></tt> and <tt><a href="manual.shtml#bowtie2-options-ma">--ma</a></tt></li>
+ <li>Changed default for <tt><a href="manual.shtml#bowtie2-build-options-bmaxdivn">--bmaxdivn</a></tt> to yield better memory footprint and running time when building an index with many threads</li>
+</ul>
+<h2>Bowtie2 developers note</h2>
+<p>As of Nov 2015 we had to fix the bowtie2 github repo and relabel the entire history. Developers and contributors should re-clone the bowtie2 github repo from this current state. </p>
+<h2>Version 2.2.9 - Apr 22, 2016</h2>
+<ul>
+ <li>Fixed the multiple threads issue for the bowtie2-build.</li>
+ <li>Fixed a TBB related build issue impacting TBB v4.4.</li>
+</ul>
+<h2>Version 2.2.8 - Mar 10, 2016</h2>
+<ul>
+ <li>Various website updates.</li>
+ <li>Fixed the bowtie2-build issue that made TBB compilation fail.</li>
+ <li>Fixed the static build for Win32 platform.</li>
+</ul>
+<h2>Version 2.2.7 - Feb 10, 2016</h2>
+<ul>
+ <li>Added a parallel index build option: bowtie2-build --threads <# threads>.</li>
+ <li>Fixed an issue whereby IUPAC codes (other than A/C/G/T/N) in reads were converted to As. Now all non-A/C/G/T characters in reads become Ns.</li>
+ <li>Fixed some compilation issues, including for the Intel C++ Compiler.</li>
+ <li>Removed debugging code that could impede performance for many alignment threads.</li>
+ <li>Fixed a few typos in documentation.</li>
+</ul>
+<h2>Version 2.2.6 - Jul 22, 2015</h2>
+<ul>
+ <li>Switched to a stable sort to avoid some potential reproducibility confusions.</li>
+ <li>Added <tt>'install'</tt> target for *nix platforms.</li>
+ <li>Added the Intel TBB option which provides in most situations a better performance output. TBB is not present by default in the current build but can be added by compiling the source code with <tt>WITH_TBB=1</tt> option.</li>
+ <li>Fixed a bug that caused seed lenght to be dependent of the <tt><a href="manual.shtml#bowtie2-options-L">-L</a></tt> and <tt><a href="manual.shtml#bowtie2-options-N">-N</a></tt> parameters order.</li>
+ <li>Fixed a bug that caused <tt><a href="manual.shtml#bowtie2-options-local">--local</a></tt> followed by <tt><a href="manual.shtml#bowtie2-options-N">-N</a></tt> to reset seed lenght to 22 which is actually the default value for global.</li>
+ <li>Enable compilation on FreeBDS and clang, although gmake port is still required.</li>
+ <li>Fixed an issue that made bowtie2 compilation process to fail on Snow Leopard.</li>
+</ul>
+
+<h2>Version 2.2.5 - Mar 9, 2015</h2>
+<ul>
+ <li>Fixed some situations where incorrectly we could detect a Mavericks platform.</li>
+ <li>Fixed some manual issues including some HTML bad formating.</li>
+ <li>Make sure the wrapper correctly identifies the platform under OSX.</li>
+ <li>Fixed <tt><a href="manual.shtml#bowtie2-options-rg">--rg</a></tt>/<tt><a href="manual.shtml#bowtie2-options-rg-id">--rg-id</a></tt> options where included spaces were incorrectly treated.</li>
+ <li>Various documentation fixes added by contributors.</li>
+ <li>Fixed the incorrect behavior where parameter file names may contain spaces.</li>
+ <li>Fixed bugs related with the presence of spaces in the path where bowtie binaries are stored.</li>
+ <li>Improved exception handling for missformated quality values.</li>
+ <li>Improved redundancy checks by correctly account for soft clipping.</li>
+</ul>
+
+<h2>Lighter released</h2>
+<ul>
+ <li>Lighter is an extremely fast and memory-efficient program for correcting sequencing errors in DNA sequencing data. For details on how error correction can help improve the speed and accuracy of downstream analysis tools, see the <a href="http://genomebiology.com/2014/15/11/509">paper in Genome Biology</a>. Source and software <a href="https://github.com/mourisl/Lighter">available at GitHub</a>.
+</ul>
+
+
+<h2>Version 2.2.4 - Oct 22, 2014</h2>
+<ul>
+ <li>Fixed a Mavericks OSX specific bug caused by some linkage ambiguities.</li>
+ <li>Added lz4 compression option for the wrapper.</li>
+ <li>Fixed the vanishing <tt><a href="manual.shtml#bowtie2-options-no-unal">--no-unal</a></tt> help line.</li>
+ <li>Added the static linkage for MinGW builds.</li>
+ <li>Added extra seed-hit output.</li>
+ <li>Fixed missing 0-length read in fastq <tt>--passthrough</tt> output.</li>
+ <li>Fixed an issue that would cause different output in <tt>-a</tt> mode depending on random seed.</li>
+</ul>
+
+<h2>Version 2.2.3 - May 30, 2014</h2>
+<ul>
+ <li>Fixed a bug that made loading an index into memory crash sometimes.</li>
+ <li>Fixed a silent failure to warn the user in case the <tt><a href="manual.shtml#bowtie2-options-x">-x</a></tt> option is missing.</li>
+ <li>Updated <tt><a href="manual.shtml#bowtie2-options-al">--al</a></tt>, <tt><a href="manual.shtml#bowtie2-options-un">--un</a></tt>, <tt><a href="manual.shtml#bowtie2-options-al-conc">al-conc</a></tt> and <tt><a href="manual.shtml#bowtie2-options-un-conc">un-conc</a></tt> options to avoid confusion in cases where the user does not provide a base file name.</li>
+ <li>Fixed a spurious assert that made bowtie2-inspect debug fail.</li>
+</ul>
+
+<h2>Version 2.2.2 - April 10, 2014</h2>
+<ul>
+ <li>Improved performance for cases where the reference contains ambiguous
+ or masked nucleobases represented by Ns. </li>
+</ul>
+
+<h2>Version 2.2.1 - February 28, 2014</h2>
+<ul>
+ <li>Improved way in which index files are loaded for alignment. Should fix
+ efficiency problems on some filesystems.</li>
+ <li>Fixed a bug that made older systems unable to correctly deal with bowtie
+ relative symbolic links.</li>
+ <li>Fixed a bug that, for very big indexes, could determine to determine file
+ offsets correctly.</li>
+ <li>Fixed a bug where using <tt><a href="manual.shtml#bowtie2-options-no-unal">--no-unal</a></tt> option incorrectly suppressed
+ <tt><a href="manual.shtml#bowtie2-options-un">--un</a></tt>/<tt><a href="manual.shtml#bowtie2-options-un-conc">--un-conc</a></tt> output.</li>
+ <li>Dropped a perl dependency that could cause problems on old systems.</li>
+ <li>Added <tt><a href="manual.shtml#bowtie2-options-no-1mm-upfront">--no-1mm-upfront</a></tt> option and clarified documentation for parameters
+ governing the multiseed heuristic.</li>
+</ul>
+
+
+
diff --git a/doc/website/rhsidebar.ssi b/doc/website/rhsidebar.ssi
new file mode 100644
index 0000000..ba11d60
--- /dev/null
+++ b/doc/website/rhsidebar.ssi
@@ -0,0 +1,196 @@
+<div id="rightside" style="padding-left: 20px">
+ <h2>Site Map</h2>
+ <div class="box">
+ <ul>
+ <li><a href="index.shtml">Home</a></li>
+ <li><a href="news.shtml">News archive</a></li>
+ <li><a href="manual.shtml">Manual</a></li>
+ <li><a href="manual.shtml#getting-started-with-bowtie-2-lambda-phage-example">Getting started</a></li>
+ <li><a href="faq.shtml">Frequently Asked Questions</a></li>
+ <li><a href="other_tools.shtml">Tools that use Bowtie</a></li>
+ </ul>
+ </div>
+ <h2>Latest Release</h2>
+ <div class="box">
+ <table width="100%">
+ <tr>
+ <td colspan="2"><a href="http://bioconda.github.io/recipes/bowtie2/README.html"><img src="https://img.shields.io/badge/install%20with-bioconda-brightgreen.svg?style=flat-square"/></a></td>
+ </tr>
+ <tr>
+ <td>
+ <a href="https://sourceforge.net/projects/bowtie-bio/files/bowtie2/2.3.0">Bowtie2 2.3.0</a>
+ </td>
+ <td align="right">
+ 12/13/16
+ </td>
+ </tr>
+ <tr>
+ <td colspan="2">
+ <p style="font-size: x-small; line-height: 130%">Please cite: Langmead B, Salzberg S. <a href="http://www.nature.com/nmeth/journal/v9/n4/full/nmeth.1923.html">Fast gapped-read alignment with Bowtie 2</a>. <i><a href="http://www.nature.com/nmeth">Nature Methods</a></i>. 2012, 9:357-359.</p>
+ </td>
+ </tr>
+ </table>
+ </div>
+ <h2>Links</h2>
+ <div class="box">
+ <ul>
+ <li><a href="https://github.com/BenLangmead/bowtie2">Bowtie GitHub repository</a></li>
+ <li><a href="https://github.com/BenLangmead/bowtie2/issues">Report an issue</a></li>
+ <li><a href="https://lists.sourceforge.net/lists/listinfo/bowtie-bio-announce">Bowtie Mailing list</a></li>
+ </ul>
+ </div>
+ <h2>Related Tools</h2>
+ <div class="box">
+ <table width="100%">
+ <tr><td><a href="http://bowtie-bio.sf.net">Bowtie</a>: Ultrafast short read alignment</td></tr>
+ <tr><td><a href="http://bowtie-bio.sf.net/crossbow">Crossbow</a>: Genotyping, cloud computing</td></tr>
+ <tr><td><a href="http://bowtie-bio.sf.net/myrna">Myrna</a>: Cloud, differential gene expression</td></tr>
+ <tr><td><a href="http://tophat.cbcb.umd.edu">Tophat</a>: RNA-Seq splice junction mapper</td></tr>
+ <tr><td><a href="http://cufflinks.cbcb.umd.edu">Cufflinks</a>: Isoform assembly, quantitation</td></tr>
+ <tr><td><a href="https://github.com/mourisl/Lighter">Lighter</a>: Fast error correction</td></tr>
+ </table>
+ </div>
+ <h2>Indexes</h2>
+ <div class="box">
+ <table width="100%">
+ <p style="font-size: small; line-height: 130%">
+ Consider using Illumina's
+ <a href="http://support.illumina.com/sequencing/sequencing_software/igenome.ilmn">iGenomes</a>
+ collection. Each iGenomes archive contains pre-built
+ Bowtie 2 and <a href="http://bowtie-bio.sourceforge.net/index.shtml">Bowtie</a> indexes.
+ <br/><br/>
+ </p>
+ <!-- hg18 -->
+ <tr>
+ <td>
+ <a href="ftp://ftp.ccb.jhu.edu/pub/data/bowtie2_indexes/hg18.zip"><i>H. sapiens</i>, UCSC hg18</a>
+ </td>
+ <td align="right">
+ <b>3.5 GB</b>
+ </td>
+ </tr>
+ <tr>
+ <td colspan="2" style="font-size: x-small"> or:
+ <a href="ftp://ftp.ccb.jhu.edu/pub/data/bowtie2_indexes/hg18.1.zip">part 1</a> (1.5 GB),
+ <a href="ftp://ftp.ccb.jhu.edu/pub/data/bowtie2_indexes/hg18.2.zip">part 2</a> (651 MB),
+ <a href="ftp://ftp.ccb.jhu.edu/pub/data/bowtie2_indexes/hg18.3.zip">part 3</a> (1.5 GB)
+ </td>
+ </tr>
+
+ <!-- hg19 -->
+ <tr>
+ <td>
+ <a href="ftp://ftp.ccb.jhu.edu/pub/data/bowtie2_indexes/hg19.zip"><i>H. sapiens</i>, UCSC hg19</a>
+ </td>
+ <td align="right">
+ <b>3.5 GB</b>
+ </td>
+ </tr>
+ <tr>
+ <td colspan="2" style="font-size: x-small"> or:
+ <a href="ftp://ftp.ccb.jhu.edu/pub/data/bowtie2_indexes/hg19.1.zip">part 1</a> (1.5 GB),
+ <a href="ftp://ftp.ccb.jhu.edu/pub/data/bowtie2_indexes/hg19.2.zip">part 2</a> (650 MB),
+ <a href="ftp://ftp.ccb.jhu.edu/pub/data/bowtie2_indexes/hg19.3.zip">part 3</a> (1.5 GB)
+ </td>
+ </tr>
+
+ <!-- GRCh38 -->
+ <tr>
+ <td>
+ <a href="ftp://ftp.ncbi.nlm.nih.gov/genomes/archive/old_genbank/Eukaryotes/vertebrates_mammals/Homo_sapiens/GRCh38/seqs_for_alignment_pipelines/GCA_000001405.15_GRCh38_no_alt_analysis_set.fna.bowtie_index.tar.gz"><i>H. sapiens</i>, NCBI GRCh38</a>
+ </td>
+ <td align="right">
+ <b>3.5 GB</b>
+ </td>
+ </tr>
+
+ <!-- mm10 -->
+ <tr>
+ <td>
+ <a href="ftp://ftp.ccb.jhu.edu/pub/data/bowtie2_indexes/mm10.zip"><i>M. musculus</i>, UCSC mm10</a>
+ </td>
+ <td align="right">
+ <b>3.2 GB</b>
+ </td>
+ </tr>
+ <tr>
+ <td colspan="2" style="font-size: x-small"> or:
+ <a href="ftp://ftp.ccb.jhu.edu/pub/data/bowtie2_indexes/mm10.1.zip">part 1</a> (1.3 GB),
+ <a href="ftp://ftp.ccb.jhu.edu/pub/data/bowtie2_indexes/mm10.2.zip">part 2</a> (600 MB),
+ <a href="ftp://ftp.ccb.jhu.edu/pub/data/bowtie2_indexes/mm10.3.zip">part 3</a> (1.3 GB)
+ </td>
+ </tr>
+
+ <!-- mm9 -->
+ <tr>
+ <td>
+ <a href="ftp://ftp.ccb.jhu.edu/pub/data/bowtie2_indexes/mm9.zip"><i>M. musculus</i>, UCSC mm9</a>
+ </td>
+ <td align="right">
+ <b>3.2 GB</b>
+ </td>
+ </tr>
+ <tr>
+ <td colspan="2" style="font-size: x-small"> or:
+ <a href="ftp://ftp.ccb.jhu.edu/pub/data/bowtie2_indexes/mm9.1.zip">part 1</a> (1.3 GB),
+ <a href="ftp://ftp.ccb.jhu.edu/pub/data/bowtie2_indexes/mm9.2.zip">part 2</a> (593 MB),
+ <a href="ftp://ftp.ccb.jhu.edu/pub/data/bowtie2_indexes/mm9.3.zip">part 3</a> (1.3 GB)
+ </td>
+ </tr>
+
+ <!-- rn4 -->
+ <tr>
+ <td>
+ <a href="ftp://ftp.ccb.jhu.edu/pub/data/bowtie2_indexes/rn4.zip"><i>R. norvegicus</i>, UCSC rn4</a>
+ </td>
+ <td align="right">
+ <b>3.1 GB</b>
+ </td>
+ </tr>
+ <tr>
+ <td colspan="2" style="font-size: x-small"> or:
+ <a href="ftp://ftp.ccb.jhu.edu/pub/data/bowtie2_indexes/rn4.1.zip">part 1</a> (1.3 GB),
+ <a href="ftp://ftp.ccb.jhu.edu/pub/data/bowtie2_indexes/rn4.2.zip">part 2</a> (580 MB),
+ <a href="ftp://ftp.ccb.jhu.edu/pub/data/bowtie2_indexes/rn4.3.zip">part 3</a> (1.3 GB)
+ </td>
+ </tr>
+
+ </table>
+ <p style="font-size: x-small; line-height: 130%">
+ Some unzip programs cannot handle archives >2 GB. If you have
+ problems downloading or unzipping a >2 GB index, try downloading
+ in parts.</p>
+ </div>
+ <h2>Related publications</h2>
+ <div class="box">
+ <ul>
+ <li style="font-size: x-small; line-height: 130%">Langmead B, Salzberg S. <a href="http://www.nature.com/nmeth/journal/v9/n4/full/nmeth.1923.html"><b>Fast gapped-read alignment with Bowtie 2</b></a>. <i><a href="http://www.nature.com/nmeth">Nature Methods</a></i>. 2012, 9:357-359.<br/><br/></li>
+ <li style="font-size: x-small; line-height: 130%">Langmead B, Trapnell C, Pop M, Salzberg SL. <a href="http://genomebiology.com/2009/10/3/R25"><b>Ultrafast and memory-efficient alignment of short DNA sequences to the human genome</b></a>. <i><a href="http://genomebiology.com">Genome Biology</a></i> <b>10</b>:R25.<br/><br/></li>
+ </ul>
+ </div>
+ <h2>Contributors</h2>
+ <div class="box">
+ <ul>
+ <li><a href="http://www.cs.jhu.edu/~langmea/index.shtml">Ben Langmead</a></li>
+ <li><a href="http://www.ccb.jhu.edu/people/infphilo">Daehwan Kim</a></li>
+ <li>Valentin Antonescu</li>
+ <li>Chris Wilks</li>
+ </ul>
+ </div>
+ <h2>Other Documentation</h2>
+ <div class="box">
+ <ul>
+ <li>Bio. of Genomes poster, 5/11 (<a href="http://www.cbcb.umd.edu/~langmead/bog2011_poster.ppt">.ppt</a>, <a href="http://www.cbcb.umd.edu/~langmead/bog2011_poster.pdf">.pdf</a>)</li>
+ </ul>
+ </div>
+ <h2>Related links</h2>
+ <div class="box">
+ <ul>
+ <li><a href="http://scholar.google.com/scholar?oi=bibs&hl=en&cites=4636016874467197548">Papers citing Bowtie 2</a></li>
+ <li><a href="http://www.jhu.edu/">Johns Hopkins University</a></li>
+ <li><a href="http://www.cs.jhu.edu/">JHU Computer Science</a></li>
+ <li><a href="http://www.biostat.jhsph.edu/">JHSPH Biostatistics</a></li>
+ <li><a href="http://seqanswers.com/">SEQanswers</a></li>
+ </ul>
+ </div>
+</div> <!-- End of "rightside" -->
diff --git a/doc/website/top.ssi b/doc/website/top.ssi
new file mode 100644
index 0000000..8345b8d
--- /dev/null
+++ b/doc/website/top.ssi
@@ -0,0 +1,16 @@
+ <div id="top">
+ <div class="lefts">
+ <table width="100%" cellpadding="2">
+ <tr>
+ <td align="left" valign="middle" width="145">
+ <a href="./index.shtml"><img style="padding-left: 7pt; padding-top: 6pt" height="65" src="images/bowtie_logo.png"/></a>
+ </td>
+ <td align="left" valign="top">
+ <h1><a href="./index.shtml" style="color: white; text-decoration: none">Bowtie 2</a></h1>
+ <h2>Fast and sensitive read alignment</h2>
+ </td><td align="right" valign="middle">
+ <a href="http://www.jhu.edu/"><img style="padding-top: 7pt; padding-right: 7pt" border=0 height="55" src="images/university.small.horizontal.white.png"></a>
+ </td></tr>
+ </table>
+ </div>
+ </div>
diff --git a/dp_framer.cpp b/dp_framer.cpp
index d7e359a..c8d6a73 100644
--- a/dp_framer.cpp
+++ b/dp_framer.cpp
@@ -80,8 +80,7 @@ using namespace std;
*/
bool DynProgFramer::frameSeedExtensionRect(
int64_t off, // ref offset implied by seed hit assuming no gaps
- size_t rdlen, // length of read sequence used in DP table (so len
- // of +1 nucleotide sequence for colorspace reads)
+ size_t rdlen, // length of read sequence used in DP table
int64_t reflen, // length of reference sequence aligned to
size_t maxrdgap, // max # of read gaps permitted in opp mate alignment
size_t maxrfgap, // max # of ref gaps permitted in opp mate alignment
diff --git a/dp_framer.h b/dp_framer.h
index e27dca5..abdda89 100644
--- a/dp_framer.h
+++ b/dp_framer.h
@@ -138,8 +138,7 @@ public:
*/
bool frameSeedExtensionRect(
int64_t off, // ref offset implied by seed hit assuming no gaps
- size_t rdlen, // length of read sequence used in DP table (so len
- // of +1 nucleotide sequence for colorspace reads)
+ size_t rdlen, // length of read sequence used in DP table
int64_t reflen, // length of reference sequence aligned to
size_t maxrdgap, // max # of read gaps permitted in opp mate alignment
size_t maxrfgap, // max # of ref gaps permitted in opp mate alignment
diff --git a/filebuf.h b/filebuf.h
index 66dffb4..f3e7b9c 100644
--- a/filebuf.h
+++ b/filebuf.h
@@ -197,6 +197,7 @@ public:
_buf_sz = _ins->gcount();
} else {
assert(_in != NULL);
+ // TODO: consider an _unlocked function
_buf_sz = fread(_buf, 1, BUF_SZ, _in);
}
_cur = 0;
diff --git a/opts.h b/opts.h
index bba213d..a90669b 100644
--- a/opts.h
+++ b/opts.h
@@ -36,6 +36,7 @@ enum {
ARG_PARTITION, // --partition
ARG_INTEGER_QUALS, // --int-quals
ARG_FILEPAR, // --filepar
+ ARG_READS_PER_BATCH, // --reads-per-batch
ARG_SHMEM, // --shmem
ARG_MM, // --mm
ARG_MMSWEEP, // --mmsweep
@@ -53,7 +54,6 @@ enum {
ARG_PHRED64, // --phred64
ARG_PHRED33, // --phred33
ARG_HADOOPOUT, // --hadoopout
- ARG_FUZZY, // --fuzzy
ARG_FULLREF, // --fullref
ARG_USAGE, // --usage
ARG_SNPPHRED, // --snpphred
diff --git a/pat.cpp b/pat.cpp
index 7d287c9..782130f 100644
--- a/pat.cpp
+++ b/pat.cpp
@@ -26,10 +26,56 @@
#include "pat.h"
#include "filebuf.h"
#include "formats.h"
+#include "util.h"
+#include "tokenize.h"
using namespace std;
/**
+ * Calculate a per-read random seed based on a combination of
+ * the read data (incl. sequence, name, quals) and the global
+ * seed in '_randSeed'.
+ */
+static uint32_t genRandSeed(
+ const BTDnaString& qry,
+ const BTString& qual,
+ const BTString& name,
+ uint32_t seed)
+{
+ // Calculate a per-read random seed based on a combination of
+ // the read data (incl. sequence, name, quals) and the global
+ // seed
+ uint32_t rseed = (seed + 101) * 59 * 61 * 67 * 71 * 73 * 79 * 83;
+ size_t qlen = qry.length();
+ // Throw all the characters of the read into the random seed
+ for(size_t i = 0; i < qlen; i++) {
+ int p = (int)qry[i];
+ assert_leq(p, 4);
+ size_t off = ((i & 15) << 1);
+ rseed ^= (p << off);
+ }
+ // Throw all the quality values for the read into the random
+ // seed
+ for(size_t i = 0; i < qlen; i++) {
+ int p = (int)qual[i];
+ assert_leq(p, 255);
+ size_t off = ((i & 3) << 3);
+ rseed ^= (p << off);
+ }
+ // Throw all the characters in the read name into the random
+ // seed
+ size_t namelen = name.length();
+ for(size_t i = 0; i < namelen; i++) {
+ int p = (int)name[i];
+ if(p == '/') break;
+ assert_leq(p, 255);
+ size_t off = ((i & 3) << 3);
+ rseed ^= (p << off);
+ }
+ return rseed;
+}
+
+/**
* Return a new dynamically allocated PatternSource for the given
* format, using the given list of strings as the filenames to read
* from or as the sequences themselves (i.e. if -c was used).
@@ -55,83 +101,73 @@ PatternSource* PatternSource::patsrcFromStrings(
}
/**
- * The main member function for dispensing patterns.
- *
- * Returns true iff a pair was parsed succesfully.
+ * Once name/sequence/qualities have been parsed for an
+ * unpaired read, set all the other key fields of the Read
+ * struct.
*/
-bool PatternSource::nextReadPair(
- Read& ra,
- Read& rb,
- TReadId& rdid,
- TReadId& endid,
- bool& success,
- bool& done,
- bool& paired,
- bool fixName)
-{
- // nextPatternImpl does the reading from the ultimate source;
- // it is implemented in concrete subclasses
- success = done = paired = false;
- nextReadPairImpl(ra, rb, rdid, endid, success, done, paired);
- if(success) {
- // Construct reversed versions of fw and rc seqs/quals
- ra.finalize();
- if(!rb.empty()) {
- rb.finalize();
- }
- // Fill in the random-seed field using a combination of
- // information from the user-specified seed and the read
- // sequence, qualities, and name
- ra.seed = genRandSeed(ra.patFw, ra.qual, ra.name, seed_);
- if(!rb.empty()) {
- rb.seed = genRandSeed(rb.patFw, rb.qual, rb.name, seed_);
- }
+void PatternSourcePerThread::finalize(Read& ra) {
+ ra.mate = 1;
+ ra.rdid = buf_.rdid();
+ ra.seed = genRandSeed(ra.patFw, ra.qual, ra.name, pp_.seed);
+ ra.finalize();
+ if(pp_.fixName) {
+ ra.fixMateName(1);
}
- return success;
}
/**
- * The main member function for dispensing patterns.
+ * Once name/sequence/qualities have been parsed for a
+ * paired-end read, set all the other key fields of the Read
+ * structs.
*/
-bool PatternSource::nextRead(
- Read& r,
- TReadId& rdid,
- TReadId& endid,
- bool& success,
- bool& done)
-{
- // nextPatternImpl does the reading from the ultimate source;
- // it is implemented in concrete subclasses
- nextReadImpl(r, rdid, endid, success, done);
- if(success) {
- // Construct the reversed versions of the fw and rc seqs
- // and quals
- r.finalize();
- // Fill in the random-seed field using a combination of
- // information from the user-specified seed and the read
- // sequence, qualities, and name
- r.seed = genRandSeed(r.patFw, r.qual, r.name, seed_);
+void PatternSourcePerThread::finalizePair(Read& ra, Read& rb) {
+ ra.mate = 1;
+ rb.mate = 2;
+ ra.rdid = rb.rdid = buf_.rdid();
+ ra.seed = genRandSeed(ra.patFw, ra.qual, ra.name, pp_.seed);
+ rb.seed = genRandSeed(rb.patFw, rb.qual, rb.name, pp_.seed);
+ ra.finalize();
+ rb.finalize();
+ if(pp_.fixName) {
+ ra.fixMateName(1);
+ rb.fixMateName(2);
}
- return success;
}
/**
* Get the next paired or unpaired read from the wrapped
- * PairedPatternSource.
+ * PatternComposer. Returns a pair of bools; first indicates
+ * whether we were successful, second indicates whether we're
+ * done.
*/
-bool WrappedPatternSourcePerThread::nextReadPair(
- bool& success,
- bool& done,
- bool& paired,
- bool fixName)
-{
- PatternSourcePerThread::nextReadPair(success, done, paired, fixName);
- ASSERT_ONLY(TReadId lastRdId = rdid_);
- buf1_.reset();
- buf2_.reset();
- patsrc_.nextReadPair(buf1_, buf2_, rdid_, endid_, success, done, paired, fixName);
- assert(!success || rdid_ != lastRdId);
- return success;
+pair<bool, bool> PatternSourcePerThread::nextReadPair() {
+ // Prepare batch
+ if(buf_.exhausted()) {
+ pair<bool, int> res = nextBatch(); // more parsing needed!
+ if(res.first && res.second == 0) {
+ return make_pair(false, true);
+ }
+ last_batch_ = res.first;
+ last_batch_size_ = res.second;
+ assert_eq(0, buf_.cur_buf_);
+ } else {
+ buf_.next(); // advance cursor; no parsing or locking needed
+ assert_gt(buf_.cur_buf_, 0);
+ }
+ // Now fully parse read/pair *outside* the critical section
+ assert(!buf_.read_a().readOrigBuf.empty());
+ assert(buf_.read_a().empty());
+ if(!parse(buf_.read_a(), buf_.read_b())) {
+ return make_pair(false, false);
+ }
+ // Finalize read/pair
+ if(!buf_.read_b().patFw.empty()) {
+ finalizePair(buf_.read_a(), buf_.read_b());
+ } else {
+ finalize(buf_.read_a());
+ }
+ bool this_is_last = buf_.cur_buf_ == last_batch_size_-1;
+ return make_pair(true, this_is_last ? last_batch_ : false);
}
/**
@@ -139,56 +175,29 @@ bool WrappedPatternSourcePerThread::nextReadPair(
* singleton reads. Returns true iff ra and rb contain a new
* pair; returns false if ra contains a new unpaired read.
*/
-bool PairedSoloPatternSource::nextReadPair(
- Read& ra,
- Read& rb,
- TReadId& rdid,
- TReadId& endid,
- bool& success,
- bool& done,
- bool& paired,
- bool fixName)
-{
- uint32_t cur = cur_;
- success = false;
+pair<bool, int> SoloPatternComposer::nextBatch(PerThreadReadBuf& pt) {
+ size_t cur = cur_;
while(cur < src_->size()) {
// Patterns from srca_[cur_] are unpaired
+ pair<bool, int> res;
do {
- (*src_)[cur]->nextReadPair(
- ra, rb, rdid, endid, success, done, paired, fixName);
- } while(!success && !done);
- if(!success) {
- assert(done);
- // If patFw is empty, that's our signal that the
- // input dried up
- lock();
- if(cur + 1 > cur_) cur_++;
+ res = (*src_)[cur]->nextBatch(
+ pt,
+ true, // batch A (or pairs)
+ true); // grab lock below
+ } while(!res.first && res.second == 0);
+ if(res.second == 0) {
+ ThreadSafe ts(&mutex_m);
+ if(cur + 1 > cur_) {
+ cur_++;
+ }
cur = cur_;
- unlock();
continue; // on to next pair of PatternSources
}
- assert(success);
- ra.seed = genRandSeed(ra.patFw, ra.qual, ra.name, seed_);
- if(!rb.empty()) {
- rb.seed = genRandSeed(rb.patFw, rb.qual, rb.name, seed_);
- if(fixName) {
- ra.fixMateName(1);
- rb.fixMateName(2);
- }
- }
- ra.rdid = rdid;
- ra.endid = endid;
- if(!rb.empty()) {
- rb.rdid = rdid;
- rb.endid = endid+1;
- }
- ra.mate = 1;
- rb.mate = 2;
- return true; // paired
+ return res;
}
assert_leq(cur, src_->size());
- done = (cur == src_->size());
- return false;
+ return make_pair(true, 0);
}
/**
@@ -196,113 +205,64 @@ bool PairedSoloPatternSource::nextReadPair(
* singleton reads. Returns true iff ra and rb contain a new
* pair; returns false if ra contains a new unpaired read.
*/
-bool PairedDualPatternSource::nextReadPair(
- Read& ra,
- Read& rb,
- TReadId& rdid,
- TReadId& endid,
- bool& success,
- bool& done,
- bool& paired,
- bool fixName)
-{
+pair<bool, int> DualPatternComposer::nextBatch(PerThreadReadBuf& pt) {
// 'cur' indexes the current pair of PatternSources
- uint32_t cur;
- {
- lock();
- cur = cur_;
- unlock();
- }
- success = false;
- done = true;
+ size_t cur = cur_;
while(cur < srca_->size()) {
if((*srcb_)[cur] == NULL) {
- paired = false;
// Patterns from srca_ are unpaired
- do {
- (*srca_)[cur]->nextRead(ra, rdid, endid, success, done);
- } while(!success && !done);
- if(!success) {
- assert(done);
- lock();
+ pair<bool, int> res = (*srca_)[cur]->nextBatch(
+ pt,
+ true, // batch A (or pairs)
+ true); // grab lock below
+ bool done = res.first;
+ if(!done && res.second == 0) {
+ ThreadSafe ts(&mutex_m);
if(cur + 1 > cur_) cur_++;
cur = cur_; // Move on to next PatternSource
- unlock();
continue; // on to next pair of PatternSources
}
- ra.rdid = rdid;
- ra.endid = endid;
- ra.mate = 0;
- return success;
+ return make_pair(done, res.second);
} else {
- paired = true;
- // Patterns from srca_[cur_] and srcb_[cur_] are paired
- TReadId rdid_a = 0, endid_a = 0;
- TReadId rdid_b = 0, endid_b = 0;
- bool success_a = false, done_a = false;
- bool success_b = false, done_b = false;
+ pair<bool, int> resa, resb;
// Lock to ensure that this thread gets parallel reads
// in the two mate files
- lock();
- do {
- (*srca_)[cur]->nextRead(ra, rdid_a, endid_a, success_a, done_a);
- } while(!success_a && !done_a);
- do {
- (*srcb_)[cur]->nextRead(rb, rdid_b, endid_b, success_b, done_b);
- } while(!success_b && !done_b);
- if(!success_a && success_b) {
- cerr << "Error, fewer reads in file specified with -1 than in file specified with -2" << endl;
+ {
+ ThreadSafe ts(&mutex_m);
+ resa = (*srca_)[cur]->nextBatch(
+ pt,
+ true, // batch A
+ false); // don't grab lock below
+ resb = (*srcb_)[cur]->nextBatch(
+ pt,
+ false, // batch B
+ false); // don't grab lock below
+ assert_eq((*srca_)[cur]->readCount(),
+ (*srcb_)[cur]->readCount());
+ }
+ if(resa.second < resb.second) {
+ cerr << "Error, fewer reads in file specified with -1 "
+ << "than in file specified with -2" << endl;
throw 1;
- } else if(!success_a) {
- assert(done_a && done_b);
- if(cur + 1 > cur_) cur_++;
+ } else if(resa.second == 0 && resb.second == 0) {
+ ThreadSafe ts(&mutex_m);
+ if(cur + 1 > cur_) {
+ cur_++;
+ }
cur = cur_; // Move on to next PatternSource
- unlock();
continue; // on to next pair of PatternSources
- } else if(!success_b) {
- cerr << "Error, fewer reads in file specified with -2 than in file specified with -1" << endl;
+ } else if(resb.second < resa.second) {
+ cerr << "Error, fewer reads in file specified with -2 "
+ << "than in file specified with -1" << endl;
throw 1;
}
- assert_eq(rdid_a, rdid_b);
- //assert_eq(endid_a+1, endid_b);
- assert_eq(success_a, success_b);
- unlock();
- if(fixName) {
- ra.fixMateName(1);
- rb.fixMateName(2);
- }
- rdid = rdid_a;
- endid = endid_a;
- success = success_a;
- done = done_a;
- ra.rdid = rdid;
- ra.endid = endid;
- if(!rb.empty()) {
- rb.rdid = rdid;
- rb.endid = endid+1;
- }
- ra.mate = 1;
- rb.mate = 2;
- return success;
- }
- }
- return success;
-}
-
-/**
- * Return the number of reads attempted.
- */
-pair<TReadId, TReadId> PairedDualPatternSource::readCnt() const {
- uint64_t rets = 0llu, retp = 0llu;
- for(size_t i = 0; i < srca_->size(); i++) {
- if((*srcb_)[i] == NULL) {
- rets += (*srca_)[i]->readCnt();
- } else {
- assert_eq((*srca_)[i]->readCnt(), (*srcb_)[i]->readCnt());
- retp += (*srca_)[i]->readCnt();
+ assert_eq(resa.first, resb.first);
+ assert_eq(resa.second, resb.second);
+ return make_pair(resa.first, resa.second);
}
}
- return make_pair(rets, retp);
+ assert_leq(cur, srca_->size());
+ return make_pair(true, 0);
}
/**
@@ -310,7 +270,7 @@ pair<TReadId, TReadId> PairedDualPatternSource::readCnt() const {
* the read and quality input, create a list of pattern sources to
* dispense them.
*/
-PairedPatternSource* PairedPatternSource::setupPatternSources(
+PatternComposer* PatternComposer::setupPatternComposer(
const EList<string>& si, // singles, from argv
const EList<string>& m1, // mate1's, from -1 arg
const EList<string>& m2, // mate2's, from -2 arg
@@ -398,1082 +358,898 @@ PairedPatternSource* PairedPatternSource::setupPatternSources(
}
}
- PairedPatternSource *patsrc = NULL;
+ PatternComposer *patsrc = NULL;
if(m12.size() > 0) {
- patsrc = new PairedSoloPatternSource(ab, p);
+ patsrc = new SoloPatternComposer(ab, p);
for(size_t i = 0; i < a->size(); i++) delete (*a)[i];
for(size_t i = 0; i < b->size(); i++) delete (*b)[i];
delete a; delete b;
} else {
- patsrc = new PairedDualPatternSource(a, b, p);
+ patsrc = new DualPatternComposer(a, b, p);
for(size_t i = 0; i < ab->size(); i++) delete (*ab)[i];
delete ab;
}
return patsrc;
}
-void PairedPatternSource::free_EList_pmembers( const EList<PatternSource*> &elist) {
+void PatternComposer::free_EList_pmembers( const EList<PatternSource*> &elist) {
for (size_t i = 0; i < elist.size(); i++)
if (elist[i] != NULL)
delete elist[i];
}
+/**
+ * Fill Read with the sequence, quality and name for the next
+ * read in the list of read files. This function gets called by
+ * all the search threads, so we must handle synchronization.
+ *
+ * Returns pair<bool, int> where bool indicates whether we're
+ * completely done, and int indicates how many reads were read.
+ */
+pair<bool, int> CFilePatternSource::nextBatch(
+ PerThreadReadBuf& pt,
+ bool batch_a,
+ bool lock)
+{
+ bool done = false;
+ int nread = 0;
+
+ // synchronization at this level because both reading and manipulation of
+ // current file pointer have to be protected
+ ThreadSafe ts(&mutex, lock);
+ pt.setReadId(readCnt_);
+ while(true) { // loop that moves on to next file when needed
+ do {
+ pair<bool, int> ret = nextBatchFromFile(pt, batch_a);
+ done = ret.first;
+ nread = ret.second;
+ } while(!done && nread == 0); // not sure why this would happen
+ if(done && filecur_ < infiles_.size()) { // finished with this file
+ open();
+ resetForNextFile(); // reset state to handle a fresh file
+ filecur_++;
+ if(nread == 0) {
+ continue;
+ }
+ }
+ break;
+ }
+ assert_geq(nread, 0);
+ readCnt_ += nread;
+ return make_pair(done, nread);
+}
+
+/**
+ * Open the next file in the list of input files.
+ */
+void CFilePatternSource::open() {
+ if(is_open_) {
+ is_open_ = false;
+ fclose(fp_);
+ fp_ = NULL;
+ }
+ while(filecur_ < infiles_.size()) {
+ if(infiles_[filecur_] == "-") {
+ fp_ = stdin;
+ } else if((fp_ = fopen(infiles_[filecur_].c_str(), "rb")) == NULL) {
+ if(!errs_[filecur_]) {
+ cerr << "Warning: Could not open read file \""
+ << infiles_[filecur_].c_str()
+ << "\" for reading; skipping..." << endl;
+ errs_[filecur_] = true;
+ }
+ filecur_++;
+ continue;
+ }
+ is_open_ = true;
+ setvbuf(fp_, buf_, _IOFBF, 64*1024);
+ return;
+ }
+ cerr << "Error: No input read files were valid" << endl;
+ exit(1);
+ return;
+}
+
+/**
+ * Constructor for vector pattern source, used when the user has
+ * specified the input strings on the command line using the -c
+ * option.
+ */
VectorPatternSource::VectorPatternSource(
- const EList<string>& v,
+ const EList<string>& seqs,
const PatternParams& p) :
PatternSource(p),
cur_(p.skip),
skip_(p.skip),
paired_(false),
- v_(),
- quals_()
+ tokbuf_(),
+ bufs_()
{
- for(size_t i = 0; i < v.size(); i++) {
- EList<string> ss;
- tokenize(v[i], ":", ss, 2);
- assert_gt(ss.size(), 0);
- assert_leq(ss.size(), 2);
- // Initialize s
- string s = ss[0];
- int mytrim5 = gTrim5;
- if(gColor && s.length() > 1) {
- // This may be a primer character. If so, keep it in the
- // 'primer' field of the read buf and parse the rest of the
- // read without it.
- int c = toupper(s[0]);
- if(asc2dnacat[c] > 0) {
- // First char is a DNA char
- int c2 = toupper(s[1]);
- // Second char is a color char
- if(asc2colcat[c2] > 0) {
- mytrim5 += 2; // trim primer and first color
- }
+ // Install sequences in buffers, ready for immediate copying in
+ // nextBatch(). Formatting of the buffer is just like
+ // TabbedPatternSource.
+ const size_t seqslen = seqs.size();
+ for(size_t i = 0; i < seqslen; i++) {
+ tokbuf_.clear();
+ tokenize(seqs[i], ":", tokbuf_, 2);
+ assert_gt(tokbuf_.size(), 0);
+ assert_leq(tokbuf_.size(), 2);
+ // Get another buffer ready
+ bufs_.expand();
+ bufs_.back().clear();
+ // Install name
+ itoa10<TReadId>(static_cast<TReadId>(i), nametmp_);
+ bufs_.back().install(nametmp_);
+ bufs_.back().append('\t');
+ // Install sequence
+ bufs_.back().append(tokbuf_[0].c_str());
+ bufs_.back().append('\t');
+ // Install qualities
+ if(tokbuf_.size() > 1) {
+ bufs_.back().append(tokbuf_[1].c_str());
+ } else {
+ const size_t len = tokbuf_[0].length();
+ for(size_t i = 0; i < len; i++) {
+ bufs_.back().append('I');
}
}
- if(gColor) {
- // Convert '0'-'3' to 'A'-'T'
- for(size_t i = 0; i < s.length(); i++) {
- if(s[i] >= '0' && s[i] <= '4') {
- s[i] = "ACGTN"[(int)s[i] - '0'];
- }
- if(s[i] == '.') s[i] = 'N';
+ }
+}
+
+/**
+ * Read next batch. However, batch concept is not very applicable for this
+ * PatternSource where all the info has already been parsed into the fields
+ * in the contsructor. This essentially modifies the pt as though we read
+ * in some number of patterns.
+ */
+pair<bool, int> VectorPatternSource::nextBatch(
+ PerThreadReadBuf& pt,
+ bool batch_a,
+ bool lock)
+{
+ ThreadSafe ts(&mutex, lock);
+ pt.setReadId(cur_);
+ EList<Read>& readbuf = batch_a ? pt.bufa_ : pt.bufb_;
+ size_t readi = 0;
+ for(; readi < pt.max_buf_ && cur_ < bufs_.size(); readi++, cur_++) {
+ readbuf[readi].readOrigBuf = bufs_[cur_];
+ }
+ readCnt_ += readi;
+ return make_pair(cur_ == bufs_.size(), readi);
+}
+
+/**
+ * Finishes parsing outside the critical section.
+ */
+bool VectorPatternSource::parse(Read& ra, Read& rb, TReadId rdid) const {
+ // Very similar to TabbedPatternSource
+
+ // Light parser (nextBatchFromFile) puts unparsed data
+ // into Read& r, even when the read is paired.
+ assert(ra.empty());
+ assert(!ra.readOrigBuf.empty()); // raw data for read/pair is here
+ int c = '\t';
+ size_t cur = 0;
+ const size_t buflen = ra.readOrigBuf.length();
+
+ // Loop over the two ends
+ for(int endi = 0; endi < 2 && c == '\t'; endi++) {
+ Read& r = ((endi == 0) ? ra : rb);
+ assert(r.name.empty());
+ // Parse name if (a) this is the first end, or
+ // (b) this is tab6
+ if(endi < 1 || paired_) {
+ // Parse read name
+ c = ra.readOrigBuf[cur++];
+ while(c != '\t' && cur < buflen) {
+ r.name.append(c);
+ c = ra.readOrigBuf[cur++];
}
- }
- if(s.length() <= (size_t)(gTrim3 + mytrim5)) {
- // Entire read is trimmed away
- s.clear();
- } else {
- // Trim on 5' (high-quality) end
- if(mytrim5 > 0) {
- s.erase(0, mytrim5);
+ assert_eq('\t', c);
+ if(cur >= buflen) {
+ return false; // record ended prematurely
}
- // Trim on 3' (low-quality) end
- if(gTrim3 > 0) {
- s.erase(s.length()-gTrim3);
+ } else if(endi > 0) {
+ // if this is the second end and we're parsing
+ // tab5, copy name from first end
+ rb.name = ra.name;
+ }
+
+ // Parse sequence
+ assert(r.patFw.empty());
+ c = ra.readOrigBuf[cur++];
+ int nchar = 0;
+ while(c != '\t' && cur < buflen) {
+ if(isalpha(c)) {
+ assert_in(toupper(c), "ACGTN");
+ if(nchar++ >= gTrim5) {
+ assert_neq(0, asc2dnacat[c]);
+ r.patFw.append(asc2dna[c]); // ascii to int
+ }
}
+ c = ra.readOrigBuf[cur++];
}
- // Initialize vq
- string vq;
- if(ss.size() == 2) {
- vq = ss[1];
+ assert_eq('\t', c);
+ if(cur >= buflen) {
+ return false; // record ended prematurely
}
- // Trim qualities
- if(vq.length() > (size_t)(gTrim3 + mytrim5)) {
- // Trim on 5' (high-quality) end
- if(mytrim5 > 0) {
- vq.erase(0, mytrim5);
+ // record amt trimmed from 5' end due to --trim5
+ r.trimmed5 = (int)(nchar - r.patFw.length());
+ // record amt trimmed from 3' end due to --trim3
+ r.trimmed3 = (int)(r.patFw.trimEnd(gTrim3));
+
+ // Parse qualities
+ assert(r.qual.empty());
+ c = ra.readOrigBuf[cur++];
+ int nqual = 0;
+ while(c != '\t' && c != '\n' && c != '\r') {
+ if(c == ' ') {
+ wrongQualityFormat(r.name);
+ return false;
}
- // Trim on 3' (low-quality) end
- if(gTrim3 > 0) {
- vq.erase(vq.length()-gTrim3);
+ char cadd = charToPhred33(c, false, false);
+ if(++nqual > gTrim5) {
+ r.qual.append(cadd);
}
+ if(cur >= buflen) break;
+ c = ra.readOrigBuf[cur++];
}
- // Pad quals with Is if necessary; this shouldn't happen
- while(vq.length() < s.length()) {
- vq.push_back('I');
- }
- // Truncate quals to match length of read if necessary;
- // this shouldn't happen
- if(vq.length() > s.length()) {
- vq.erase(s.length());
+ if(nchar > nqual) {
+ tooFewQualities(r.name);
+ return false;
+ } else if(nqual > nchar) {
+ tooManyQualities(r.name);
+ return false;
}
- assert_eq(vq.length(), s.length());
- v_.expand();
- v_.back().installChars(s);
- quals_.push_back(BTString(vq));
- trimmed3_.push_back(gTrim3);
- trimmed5_.push_back(mytrim5);
- ostringstream os;
- os << (names_.size());
- names_.push_back(BTString(os.str()));
+ r.qual.trimEnd(gTrim3);
+ assert(c == '\t' || c == '\n' || c == '\r' || cur >= buflen);
+ assert_eq(r.patFw.length(), r.qual.length());
}
- assert_eq(v_.size(), quals_.size());
-}
-
-bool VectorPatternSource::nextReadImpl(
- Read& r,
- TReadId& rdid,
- TReadId& endid,
- bool& success,
- bool& done)
-{
- // Let Strings begin at the beginning of the respective bufs
- r.reset();
- lock();
- if(cur_ >= v_.size()) {
- unlock();
- // Clear all the Strings, as a signal to the caller that
- // we're out of reads
- r.reset();
- success = false;
- done = true;
- assert(r.empty());
- return false;
+ ra.parsed = true;
+ if(!rb.parsed && !rb.readOrigBuf.empty()) {
+ return parse(rb, ra, rdid);
}
- // Copy v_*, quals_* strings into the respective Strings
- r.color = gColor;
- r.patFw = v_[cur_];
- r.qual = quals_[cur_];
- r.trimmed3 = trimmed3_[cur_];
- r.trimmed5 = trimmed5_[cur_];
- ostringstream os;
- os << cur_;
- r.name = os.str();
- cur_++;
- done = cur_ == v_.size();
- rdid = endid = readCnt_;
- readCnt_++;
- unlock();
- success = true;
return true;
}
-
+
/**
- * This is unused, but implementation is given for completeness.
+ * Light-parse a FASTA batch into the given buffer.
*/
-bool VectorPatternSource::nextReadPairImpl(
- Read& ra,
- Read& rb,
- TReadId& rdid,
- TReadId& endid,
- bool& success,
- bool& done,
- bool& paired)
+pair<bool, int> FastaPatternSource::nextBatchFromFile(
+ PerThreadReadBuf& pt,
+ bool batch_a)
{
- // Let Strings begin at the beginning of the respective bufs
- ra.reset();
- rb.reset();
- paired = true;
- if(!paired_) {
- paired_ = true;
- cur_ <<= 1;
+ int c;
+ EList<Read>& readbuf = batch_a ? pt.bufa_ : pt.bufb_;
+ if(first_) {
+ c = getc_unlocked(fp_);
+ while(c == '\r' || c == '\n') {
+ c = getc_unlocked(fp_);
+ }
+ if(c != '>') {
+ cerr << "Error: reads file does not look like a FASTA file" << endl;
+ throw 1;
+ }
+ first_ = false;
}
- lock();
- if(cur_ >= v_.size()-1) {
- unlock();
- // Clear all the Strings, as a signal to the caller that
- // we're out of reads
- ra.reset();
- rb.reset();
- assert(ra.empty());
- assert(rb.empty());
- success = false;
+ bool done = false;
+ size_t readi = 0;
+ if (feof(fp_)) {
done = true;
- return false;
}
- // Copy v_*, quals_* strings into the respective Strings
- ra.patFw = v_[cur_];
- ra.qual = quals_[cur_];
- ra.trimmed3 = trimmed3_[cur_];
- ra.trimmed5 = trimmed5_[cur_];
- cur_++;
- rb.patFw = v_[cur_];
- rb.qual = quals_[cur_];
- rb.trimmed3 = trimmed3_[cur_];
- rb.trimmed5 = trimmed5_[cur_];
- ostringstream os;
- os << readCnt_;
- ra.name = os.str();
- rb.name = os.str();
- ra.color = rb.color = gColor;
- cur_++;
- done = cur_ >= v_.size()-1;
- rdid = endid = readCnt_;
- readCnt_++;
- unlock();
- success = true;
- return true;
+ // Read until we run out of input or until we've filled the buffer
+ for(; readi < pt.max_buf_ && !done; readi++) {
+ Read::TBuf& buf = readbuf[readi].readOrigBuf;
+ buf.clear();
+ buf.append('>');
+ while(true) {
+ c = getc_unlocked(fp_);
+ if(c < 0 || c == '>') {
+ done = c < 0;
+ break;
+ }
+ buf.append(c);
+ }
+ }
+ return make_pair(done, readi);
}
/**
- * Parse a single quality string from fb and store qualities in r.
- * Assume the next character obtained via fb.get() is the first
- * character of the quality string. When returning, the next
- * character returned by fb.peek() or fb.get() should be the first
- * character of the following line.
+ * Finalize FASTA parsing outside critical section.
*/
-int parseQuals(
- Read& r,
- FileBuf& fb,
- int firstc,
- int readLen,
- int trim3,
- int trim5,
- bool intQuals,
- bool phred64,
- bool solexa64)
-{
- int c = firstc;
+bool FastaPatternSource::parse(Read& r, Read& rb, TReadId rdid) const {
+ // We assume the light parser has put the raw data for the separate ends
+ // into separate Read objects. That doesn't have to be the case, but
+ // that's how we've chosen to do it for FastqPatternSource
+ assert(!r.readOrigBuf.empty());
+ assert(r.empty());
+ int c = -1;
+ size_t cur = 1;
+ const size_t buflen = r.readOrigBuf.length();
+
+ // Parse read name
+ assert(r.name.empty());
+ while(cur < buflen) {
+ c = r.readOrigBuf[cur++];
+ if(c == '\n' || c == '\r') {
+ do {
+ c = r.readOrigBuf[cur++];
+ } while((c == '\n' || c == '\r') && cur < buflen);
+ break;
+ }
+ r.name.append(c);
+ }
+ if(cur >= buflen) {
+ return false; // FASTA ended prematurely
+ }
+
+ // Parse sequence
+ int nchar = 0;
+ assert(r.patFw.empty());
assert(c != '\n' && c != '\r');
- r.qual.clear();
- if (intQuals) {
- while (c != '\r' && c != '\n' && c != -1) {
- bool neg = false;
- int num = 0;
- while(!isspace(c) && !fb.eof()) {
- if(c == '-') {
- neg = true;
- assert_eq(num, 0);
- } else {
- if(!isdigit(c)) {
- char buf[2048];
- cerr << "Warning: could not parse quality line:" << endl;
- fb.getPastNewline();
- cerr << fb.copyLastN(buf);
- buf[2047] = '\0';
- cerr << buf;
- throw 1;
- }
- assert(isdigit(c));
- num *= 10;
- num += (c - '0');
- }
- c = fb.get();
- }
- if(neg) num = 0;
- // Phred-33 ASCII encode it and add it to the back of the
- // quality string
- r.qual.append('!' + num);
- // Skip over next stretch of whitespace
- while(c != '\r' && c != '\n' && isspace(c) && !fb.eof()) {
- c = fb.get();
- }
+ assert_lt(cur, buflen);
+ while(c != '\n' && cur < buflen) {
+ if(c == '.') {
+ c = 'N';
}
- } else {
- while (c != '\r' && c != '\n' && c != -1) {
- r.qual.append(charToPhred33(c, solexa64, phred64));
- c = fb.get();
- while(c != '\r' && c != '\n' && isspace(c) && !fb.eof()) {
- c = fb.get();
+ if(isalpha(c)) {
+ // If it's past the 5'-end trim point
+ if(nchar++ >= gTrim5) {
+ r.patFw.append(asc2dna[c]);
}
}
+ assert_lt(cur, buflen);
+ c = r.readOrigBuf[cur++];
}
- if ((int)r.qual.length() < readLen-1 ||
- ((int)r.qual.length() < readLen && !r.color))
- {
- tooFewQualities(r.name);
- }
- r.qual.trimEnd(trim3);
- if(r.qual.length()-trim5 < r.patFw.length()) {
- assert(gColor && r.primer != -1);
- assert_gt(trim5, 0);
- trim5--;
+ r.trimmed5 = (int)(nchar - r.patFw.length());
+ r.trimmed3 = (int)(r.patFw.trimEnd(gTrim3));
+
+ for(size_t i = 0; i < r.patFw.length(); i++) {
+ r.qual.append('I');
}
- r.qual.trimBegin(trim5);
- if(r.qual.length() <= 0) return 0;
- assert_eq(r.qual.length(), r.patFw.length());
- while(fb.peek() == '\n' || fb.peek() == '\r') fb.get();
- return (int)r.qual.length();
-}
-/// Read another pattern from a FASTA input file
-bool FastaPatternSource::read(
- Read& r,
- TReadId& rdid,
- TReadId& endid,
- bool& success,
- bool& done)
-{
- int c, qc = 0;
- success = true;
- done = false;
- assert(fb_.isOpen());
- r.reset();
- r.color = gColor;
- // Pick off the first carat
- c = fb_.get();
- if(c < 0) {
- bail(r); success = false; done = true; return success;
- }
- while(c == '#' || c == ';' || c == '\r' || c == '\n') {
- c = fb_.peekUptoNewline();
- fb_.resetLastN();
- c = fb_.get();
+ // Set up a default name if one hasn't been set
+ if(r.name.empty()) {
+ char cbuf[20];
+ itoa10<TReadId>(static_cast<TReadId>(rdid), cbuf);
+ r.name.install(cbuf);
}
- assert_eq(1, fb_.lastNLen());
-
- // Pick off the first carat
- if(first_) {
- if(c != '>') {
- cerr << "Error: reads file does not look like a FASTA file" << endl;
- throw 1;
- }
- first_ = false;
+ r.parsed = true;
+ if(!rb.parsed && !rb.readOrigBuf.empty()) {
+ return parse(rb, r, rdid);
}
- assert_eq('>', c);
- c = fb_.get(); // get next char after '>'
+ return true;
+}
- // Read to the end of the id line, sticking everything after the '>'
- // into *name
- //bool warning = false;
- while(true) {
- if(c < 0 || qc < 0) {
- bail(r); success = false; done = true; return success;
+/**
+ * Light-parse a FASTA-continuous batch into the given buffer.
+ * This is trickier for FASTA-continuous than for other formats,
+ * for several reasons:
+ *
+ * 1. Reads are substrings of a longer FASTA input string
+ * 2. Reads may overlap w/r/t the longer FASTA string
+ * 3. Read names depend on the most recently observed FASTA
+ * record name
+ */
+pair<bool, int> FastaContinuousPatternSource::nextBatchFromFile(
+ PerThreadReadBuf& pt,
+ bool batch_a)
+{
+ int c = -1;
+ EList<Read>& readbuf = batch_a ? pt.bufa_ : pt.bufb_;
+ size_t readi = 0;
+ while(readi < pt.max_buf_) {
+ c = getc_unlocked(fp_);
+ if(c < 0) {
+ break;
}
- if(c == '\n' || c == '\r') {
- // Break at end of line, after consuming all \r's, \n's
- while(c == '\n' || c == '\r') {
- if(fb_.peek() == '>') {
- // Empty sequence
- break;
+ if(c == '>') {
+ resetForNextFile();
+ c = getc_unlocked(fp_);
+ bool sawSpace = false;
+ while(c != '\n' && c != '\r') {
+ if(!sawSpace) {
+ sawSpace = isspace(c);
}
- c = fb_.get();
- if(c < 0 || qc < 0) {
- bail(r); success = false; done = true; return success;
+ if(!sawSpace) {
+ name_prefix_buf_.append(c);
}
+ c = getc_unlocked(fp_);
}
- break;
+ while(c == '\n' || c == '\r') {
+ c = getc_unlocked(fp_);
+ }
+ if(c < 0) {
+ break;
+ }
+ name_prefix_buf_.append('_');
}
- r.name.append(c);
- if(fb_.peek() == '>') {
- // Empty sequence
- break;
+ int cat = asc2dnacat[c];
+ if(cat >= 2) c = 'N';
+ if(cat == 0) {
+ // Non-DNA, non-IUPAC char; skip
+ continue;
+ } else {
+ // DNA char
+ buf_[bufCur_++] = c;
+ if(bufCur_ == 1024) {
+ bufCur_ = 0; // wrap around circular buf
+ }
+ if(eat_ > 0) {
+ eat_--;
+ // Try to keep readCnt_ aligned with the offset
+ // into the reference; that lets us see where
+ // the sampling gaps are by looking at the read
+ // name
+ if(!beginning_) {
+ readCnt_++;
+ }
+ continue;
+ }
+ // install name
+ readbuf[readi].readOrigBuf = name_prefix_buf_;
+ itoa10<TReadId>(readCnt_ - subReadCnt_, name_int_buf_);
+ readbuf[readi].readOrigBuf.append(name_int_buf_);
+ readbuf[readi].readOrigBuf.append('\t');
+ // install sequence
+ for(size_t i = 0; i < length_; i++) {
+ if(length_ - i <= bufCur_) {
+ c = buf_[bufCur_ - (length_ - i)];
+ } else {
+ // Rotate
+ c = buf_[bufCur_ - (length_ - i) + 1024];
+ }
+ readbuf[readi].readOrigBuf.append(c);
+ }
+ eat_ = freq_-1;
+ readCnt_++;
+ beginning_ = false;
+ readi++;
}
- c = fb_.get();
}
- if(c == '>') {
- // Empty sequences!
- cerr << "Warning: skipping empty FASTA read with name '" << r.name << "'" << endl;
- fb_.resetLastN();
- rdid = endid = readCnt_;
- readCnt_++;
- success = true; done = false; return success;
+ return make_pair(c < 0, readi);
+}
+
+/**
+ * Finalize FASTA-continuous parsing outside critical section.
+ */
+bool FastaContinuousPatternSource::parse(
+ Read& ra,
+ Read& rb,
+ TReadId rdid) const
+{
+ // Light parser (nextBatchFromFile) puts unparsed data
+ // into Read& r, even when the read is paired.
+ assert(ra.empty());
+ assert(rb.empty());
+ assert(!ra.readOrigBuf.empty()); // raw data for read/pair is here
+ assert(rb.readOrigBuf.empty());
+ int c = '\t';
+ size_t cur = 0;
+ const size_t buflen = ra.readOrigBuf.length();
+
+ // Parse read name
+ c = ra.readOrigBuf[cur++];
+ while(c != '\t' && cur < buflen) {
+ ra.name.append(c);
+ c = ra.readOrigBuf[cur++];
+ }
+ assert_eq('\t', c);
+ if(cur >= buflen) {
+ return false; // record ended prematurely
}
- assert_neq('>', c);
- // _in now points just past the first character of a sequence
- // line, and c holds the first character
- int begin = 0;
- int mytrim5 = gTrim5;
- if(gColor) {
- // This is the primer character, keep it in the
- // 'primer' field of the read buf and keep parsing
- c = toupper(c);
- if(asc2dnacat[c] > 0) {
- // First char is a DNA char
- int c2 = toupper(fb_.peek());
- if(asc2colcat[c2] > 0) {
- // Second char is a color char
- r.primer = c;
- r.trimc = c2;
- mytrim5 += 2;
+ // Parse sequence
+ assert(ra.patFw.empty());
+ c = ra.readOrigBuf[cur++];
+ int nchar = 0;
+ while(cur < buflen) {
+ if(isalpha(c)) {
+ assert_in(toupper(c), "ACGTN");
+ if(nchar++ >= gTrim5) {
+ assert_neq(0, asc2dnacat[c]);
+ ra.patFw.append(asc2dna[c]); // ascii to int
}
}
- if(c < 0) {
- bail(r); success = false; done = true; return success;
- }
+ c = ra.readOrigBuf[cur++];
}
- while(c != '>' && c >= 0) {
- if(gColor) {
- if(c >= '0' && c <= '4') c = "ACGTN"[(int)c - '0'];
- if(c == '.') c = 'N';
- }
- if(asc2dnacat[c] > 0 && begin++ >= mytrim5) {
- r.patFw.append(asc2dna[c]);
- r.qual.append('I');
- }
- if(fb_.peek() == '>') break;
- c = fb_.get();
- }
- r.patFw.trimEnd(gTrim3);
- r.qual.trimEnd(gTrim3);
- r.trimmed3 = gTrim3;
- r.trimmed5 = mytrim5;
- // Set up a default name if one hasn't been set
- if(r.name.empty()) {
- char cbuf[20];
- itoa10<TReadId>(readCnt_, cbuf);
- r.name.install(cbuf);
+ // record amt trimmed from 5' end due to --trim5
+ ra.trimmed5 = (int)(nchar - ra.patFw.length());
+ // record amt trimmed from 3' end due to --trim3
+ ra.trimmed3 = (int)(ra.patFw.trimEnd(gTrim3));
+
+ // Make fake qualities
+ assert(ra.qual.empty());
+ const size_t len = ra.patFw.length();
+ for(size_t i = 0; i < len; i++) {
+ ra.qual.append('I');
}
- assert_gt(r.name.length(), 0);
- r.readOrigBuf.install(fb_.lastN(), fb_.lastNLen());
- fb_.resetLastN();
- rdid = endid = readCnt_;
- readCnt_++;
- return success;
+ return true;
}
-/// Read another pattern from a FASTQ input file
-bool FastqPatternSource::read(
- Read& r,
- TReadId& rdid,
- TReadId& endid,
- bool& success,
- bool& done)
+
+/**
+ * "Light" parser. This is inside the critical section, so the key is to do
+ * just enough parsing so that another function downstream (finalize()) can do
+ * the rest of the parsing. Really this function's only job is to stick every
+ * for lines worth of the input file into a buffer (r.readOrigBuf). finalize()
+ * then parses the contents of r.readOrigBuf later.
+ */
+pair<bool, int> FastqPatternSource::nextBatchFromFile(
+ PerThreadReadBuf& pt,
+ bool batch_a)
{
int c;
- int dstLen = 0;
- success = true;
- done = false;
- r.reset();
- r.color = gColor;
- r.fuzzy = fuzzy_;
- // Pick off the first at
+ EList<Read>& readbuf = batch_a ? pt.bufa_ : pt.bufb_;
if(first_) {
- c = fb_.get();
- if(c != '@') {
- c = getOverNewline(fb_);
- if(c < 0) {
- bail(r); success = false; done = true; return success;
- }
+ c = getc_unlocked(fp_);
+ while(c == '\r' || c == '\n') {
+ c = getc_unlocked(fp_);
}
if(c != '@') {
cerr << "Error: reads file does not look like a FASTQ file" << endl;
throw 1;
}
- assert_eq('@', c);
first_ = false;
+ readbuf[0].readOrigBuf.append('@');
+ }
+ bool done = false, aborted = false;
+ size_t readi = 0;
+ // Read until we run out of input or until we've filled the buffer
+ for(; readi < pt.max_buf_ && !done; readi++) {
+ Read::TBuf& buf = readbuf[readi].readOrigBuf;
+ assert(readi == 0 || buf.empty());
+ int newlines = 4;
+ while(newlines) {
+ c = getc_unlocked(fp_);
+ done = c < 0;
+ if(c == '\n' || (done && newlines == 1)) {
+ // Saw newline, or EOF that we're
+ // interpreting as final newline
+ newlines--;
+ c = '\n';
+ } else if(done) {
+ aborted = true; // Unexpected EOF
+ break;
+ }
+ buf.append(c);
+ }
}
+ if(aborted) {
+ readi--;
+ }
+ return make_pair(done, readi);
+}
- // Read to the end of the id line, sticking everything after the '@'
- // into *name
+/**
+ * Finalize FASTQ parsing outside critical section.
+ */
+bool FastqPatternSource::parse(Read &r, Read& rb, TReadId rdid) const {
+ // We assume the light parser has put the raw data for the separate ends
+ // into separate Read objects. That doesn't have to be the case, but
+ // that's how we've chosen to do it for FastqPatternSource
+ assert(!r.readOrigBuf.empty());
+ assert(r.empty());
+ int c;
+ size_t cur = 1;
+ const size_t buflen = r.readOrigBuf.length();
+
+ // Parse read name
+ assert(r.name.empty());
while(true) {
- c = fb_.get();
- if(c < 0) {
- bail(r); success = false; done = true; return success;
- }
+ assert_lt(cur, buflen);
+ c = r.readOrigBuf[cur++];
if(c == '\n' || c == '\r') {
- // Break at end of line, after consuming all \r's, \n's
- while(c == '\n' || c == '\r') {
- c = fb_.get();
- if(c < 0) {
- bail(r); success = false; done = true; return success;
- }
- }
+ do {
+ c = r.readOrigBuf[cur++];
+ } while(c == '\n' || c == '\r');
break;
}
r.name.append(c);
}
- // fb_ now points just past the first character of a
- // sequence line, and c holds the first character
- int charsRead = 0;
- BTDnaString *sbuf = &r.patFw;
- int dstLens[] = {0, 0, 0, 0};
- int *dstLenCur = &dstLens[0];
- int mytrim5 = gTrim5;
- int altBufIdx = 0;
- if(gColor && c != '+') {
- // This may be a primer character. If so, keep it in the
- // 'primer' field of the read buf and parse the rest of the
- // read without it.
- c = toupper(c);
- if(asc2dnacat[c] > 0) {
- // First char is a DNA char
- int c2 = toupper(fb_.peek());
- // Second char is a color char
- if(asc2colcat[c2] > 0) {
- r.primer = c;
- r.trimc = c2;
- mytrim5 += 2; // trim primer and first color
- }
- }
- if(c < 0) {
- bail(r); success = false; done = true; return success;
+
+ // Parse sequence
+ int nchar = 0;
+ assert(r.patFw.empty());
+ while(c != '+') {
+ if(c == '.') {
+ c = 'N';
}
- }
- int trim5 = 0;
- if(c != '+') {
- trim5 = mytrim5;
- while(c != '+') {
- // Convert color numbers to letters if necessary
- if(c == '.') c = 'N';
- if(gColor) {
- if(c >= '0' && c <= '4') c = "ACGTN"[(int)c - '0'];
- }
- if(fuzzy_ && c == '-') c = 'A';
- if(isalpha(c)) {
- // If it's past the 5'-end trim point
- if(charsRead >= trim5) {
- sbuf->append(asc2dna[c]);
- (*dstLenCur)++;
- }
- charsRead++;
- } else if(fuzzy_ && c == ' ') {
- trim5 = 0; // disable 5' trimming for now
- if(charsRead == 0) {
- c = fb_.get();
- continue;
- }
- charsRead = 0;
- if(altBufIdx >= 3) {
- cerr << "At most 3 alternate sequence strings permitted; offending read: " << r.name << endl;
- throw 1;
- }
- // Move on to the next alternate-sequence buffer
- sbuf = &r.altPatFw[altBufIdx++];
- dstLenCur = &dstLens[altBufIdx];
- }
- c = fb_.get();
- if(c < 0) {
- bail(r); success = false; done = true; return success;
+ if(isalpha(c)) {
+ // If it's past the 5'-end trim point
+ if(nchar++ >= gTrim5) {
+ r.patFw.append(asc2dna[c]);
}
}
- dstLen = dstLens[0];
- charsRead = dstLen + mytrim5;
- }
- // Trim from 3' end
- if(gTrim3 > 0) {
- if((int)r.patFw.length() > gTrim3) {
- r.patFw.resize(r.patFw.length() - gTrim3);
- dstLen -= gTrim3;
- assert_eq((int)r.patFw.length(), dstLen);
- } else {
- // Trimmed the whole read; we won't be using this read,
- // but we proceed anyway so that fb_ is advanced
- // properly
- r.patFw.clear();
- dstLen = 0;
- }
+ assert(cur < r.readOrigBuf.length());
+ c = r.readOrigBuf[cur++];
}
+ r.trimmed5 = (int)(nchar - r.patFw.length());
+ r.trimmed3 = (int)(r.patFw.trimEnd(gTrim3));
+
assert_eq('+', c);
-
- // Chew up the optional name on the '+' line
- ASSERT_ONLY(int pk =) peekToEndOfLine(fb_);
- if(charsRead == 0) {
- assert(pk == '@' || pk == -1);
- fb_.get();
- r.readOrigBuf.install(fb_.lastN(), fb_.lastNLen());
- fb_.resetLastN();
- rdid = endid = readCnt_;
- readCnt_++;
- return success;
+ do {
+ assert(cur < r.readOrigBuf.length());
+ c = r.readOrigBuf[cur++];
+ } while(c != '\n' && c != '\r');
+ while(cur < buflen && (c == '\n' || c == '\r')) {
+ c = r.readOrigBuf[cur++];
}
-
- // Now read the qualities
- if (intQuals_) {
- assert(!fuzzy_);
- int qualsRead = 0;
- char buf[4096];
- if(gColor && r.primer != -1) {
- // In case the original quality string is one shorter
- mytrim5--;
- }
- qualToks_.clear();
- tokenizeQualLine(fb_, buf, 4096, qualToks_);
- for(unsigned int j = 0; j < qualToks_.size(); ++j) {
- char c = intToPhred33(atoi(qualToks_[j].c_str()), solQuals_);
- assert_geq(c, 33);
- if (qualsRead >= mytrim5) {
- r.qual.append(c);
- }
- ++qualsRead;
- } // done reading integer quality lines
- if(gColor && r.primer != -1) mytrim5++;
- r.qual.trimEnd(gTrim3);
- if(r.qual.length() < r.patFw.length()) {
- tooFewQualities(r.name);
- } else if(r.qual.length() > r.patFw.length() + 1) {
- tooManyQualities(r.name);
- }
- if(r.qual.length() == r.patFw.length()+1 && gColor && r.primer != -1) {
- r.qual.remove(0);
- }
- // Trim qualities on 3' end
- if(r.qual.length() > r.patFw.length()) {
- r.qual.resize(r.patFw.length());
- assert_eq((int)r.qual.length(), dstLen);
- }
- peekOverNewline(fb_);
- } else {
- // Non-integer qualities
- altBufIdx = 0;
- trim5 = mytrim5;
- int qualsRead[4] = {0, 0, 0, 0};
- int *qualsReadCur = &qualsRead[0];
- BTString *qbuf = &r.qual;
- if(gColor && r.primer != -1) {
- // In case the original quality string is one shorter
- trim5--;
- }
- while(true) {
- c = fb_.get();
- if (!fuzzy_ && c == ' ') {
- wrongQualityFormat(r.name);
- } else if(c == ' ') {
- trim5 = 0; // disable 5' trimming for now
- if((*qualsReadCur) == 0) continue;
- if(altBufIdx >= 3) {
- cerr << "At most 3 alternate quality strings permitted; offending read: " << r.name << endl;
- throw 1;
+
+ assert(r.qual.empty());
+ if(nchar > 0) {
+ int nqual = 0;
+ if (intQuals_) {
+ int cur_int = 0;
+ while(c != '\t' && c != '\n' && c != '\r') {
+ cur_int *= 10;
+ cur_int += (int)(c - '0');
+ c = r.readOrigBuf[cur++];
+ if(c == ' ' || c == '\t' || c == '\n' || c == '\r') {
+ char cadd = intToPhred33(cur_int, solQuals_);
+ cur_int = 0;
+ assert_geq(cadd, 33);
+ if(++nqual > gTrim5) {
+ r.qual.append(cadd);
+ }
}
- qbuf = &r.altQual[altBufIdx++];
- qualsReadCur = &qualsRead[altBufIdx];
- continue;
}
- if(c < 0) {
- break; // let the file end just at the end of a quality line
- //bail(r); success = false; done = true; return success;
+ } else {
+ c = charToPhred33(c, solQuals_, phred64Quals_);
+ if(nqual++ >= r.trimmed5) {
+ r.qual.append(c);
}
- if (c != '\r' && c != '\n') {
- if (*qualsReadCur >= trim5) {
- try {
- c = charToPhred33(c, solQuals_, phred64Quals_);
- }
- catch (...) {
- cout << "Error encountered at sequence id: " << r.name << endl;
- throw;
- }
- assert_geq(c, 33);
- qbuf->append(c);
+ while(cur < r.readOrigBuf.length()) {
+ c = r.readOrigBuf[cur++];
+ if (c == ' ') {
+ wrongQualityFormat(r.name);
+ return false;
}
- (*qualsReadCur)++;
- } else {
- break;
- }
- }
- qualsRead[0] -= gTrim3;
- r.qual.trimEnd(gTrim3);
- if(r.qual.length() < r.patFw.length()) {
- tooFewQualities(r.name);
- } else if(r.qual.length() > r.patFw.length()+1) {
- tooManyQualities(r.name);
- }
- if(r.qual.length() == r.patFw.length()+1 && gColor && r.primer != -1) {
- r.qual.remove(0);
- }
-
- if(fuzzy_) {
- // Trim from 3' end of alternate basecall and quality strings
- if(gTrim3 > 0) {
- for(int i = 0; i < 3; i++) {
- assert_eq(r.altQual[i].length(), r.altPatFw[i].length());
- if((int)r.altQual[i].length() > gTrim3) {
- r.altPatFw[i].resize(gTrim3);
- r.altQual[i].resize(gTrim3);
- } else {
- r.altPatFw[i].clear();
- r.altQual[i].clear();
- }
- qualsRead[i+1] = dstLens[i+1] =
- max<int>(0, dstLens[i+1] - gTrim3);
+ if(c == '\r' || c == '\n') {
+ break;
}
- }
- // Shift to RHS, and install in Strings
- assert_eq(0, r.alts);
- for(int i = 1; i < 4; i++) {
- if(qualsRead[i] == 0) continue;
- if(qualsRead[i] > dstLen) {
- // Shift everybody up
- int shiftAmt = qualsRead[i] - dstLen;
- for(int j = 0; j < dstLen; j++) {
- r.altQual[i-1].set(r.altQual[i-1][j+shiftAmt], j);
- r.altPatFw[i-1].set(r.altPatFw[i-1][j+shiftAmt], j);
- }
- r.altQual[i-1].resize(dstLen);
- r.altPatFw[i-1].resize(dstLen);
- } else if (qualsRead[i] < dstLen) {
- r.altQual[i-1].resize(dstLen);
- r.altPatFw[i-1].resize(dstLen);
- // Shift everybody down
- int shiftAmt = dstLen - qualsRead[i];
- for(int j = dstLen-1; j >= shiftAmt; j--) {
- r.altQual[i-1].set(r.altQual[i-1][j-shiftAmt], j);
- r.altPatFw[i-1].set(r.altPatFw[i-1][j-shiftAmt], j);
- }
- // Fill in unset positions
- for(int j = 0; j < shiftAmt; j++) {
- // '!' - indicates no alternate basecall at
- // this position
- r.altQual[i-1].set(33, j);
- }
+ c = charToPhred33(c, solQuals_, phred64Quals_);
+ if(nqual++ >= r.trimmed5) {
+ r.qual.append(c);
}
- r.alts++;
}
- }
-
- if(c == '\r' || c == '\n') {
- c = peekOverNewline(fb_);
- } else {
- c = peekToEndOfLine(fb_);
+ r.qual.trimEnd(r.trimmed3);
+ if(r.qual.length() < r.patFw.length()) {
+ tooFewQualities(r.name);
+ return false;
+ } else if(r.qual.length() > r.patFw.length()) {
+ tooManyQualities(r.name);
+ return false;
+ }
}
}
- r.readOrigBuf.install(fb_.lastN(), fb_.lastNLen());
- fb_.resetLastN();
-
- c = fb_.get();
- // Should either be at end of file or at beginning of next record
- assert(c == -1 || c == '@');
-
// Set up a default name if one hasn't been set
if(r.name.empty()) {
char cbuf[20];
- itoa10<TReadId>(readCnt_, cbuf);
+ itoa10<TReadId>(static_cast<TReadId>(readCnt_), cbuf);
r.name.install(cbuf);
}
- r.trimmed3 = gTrim3;
- r.trimmed5 = mytrim5;
- rdid = endid = readCnt_;
- readCnt_++;
- return success;
-}
-
-/// Read another pattern from a FASTA input file
-bool TabbedPatternSource::read(
- Read& r,
- TReadId& rdid,
- TReadId& endid,
- bool& success,
- bool& done)
-{
- r.reset();
- r.color = gColor;
- success = true;
- done = false;
- // fb_ is about to dish out the first character of the
- // name field
- if(parseName(r, NULL, '\t') == -1) {
- peekOverNewline(fb_); // skip rest of line
- r.reset();
- success = false;
- done = true;
- return false;
- }
- assert_neq('\t', fb_.peek());
-
- // fb_ is about to dish out the first character of the
- // sequence field
- int charsRead = 0;
- int mytrim5 = gTrim5;
- int dstLen = parseSeq(r, charsRead, mytrim5, '\t');
- assert_neq('\t', fb_.peek());
- if(dstLen < 0) {
- peekOverNewline(fb_); // skip rest of line
- r.reset();
- success = false;
- done = true;
- return false;
- }
-
- // fb_ is about to dish out the first character of the
- // quality-string field
- char ct = 0;
- if(parseQuals(r, charsRead, dstLen, mytrim5, ct, '\n') < 0) {
- peekOverNewline(fb_); // skip rest of line
- r.reset();
- success = false;
- done = true;
- return false;
+ r.parsed = true;
+ if(!rb.parsed && !rb.readOrigBuf.empty()) {
+ return parse(rb, r, rdid);
}
- r.trimmed3 = gTrim3;
- r.trimmed5 = mytrim5;
- assert_eq(ct, '\n');
- assert_neq('\n', fb_.peek());
- r.readOrigBuf.install(fb_.lastN(), fb_.lastNLen());
- fb_.resetLastN();
- rdid = endid = readCnt_;
- readCnt_++;
return true;
}
-/// Read another pair of patterns from a FASTA input file
-bool TabbedPatternSource::readPair(
- Read& ra,
- Read& rb,
- TReadId& rdid,
- TReadId& endid,
- bool& success,
- bool& done,
- bool& paired)
+/**
+ * Light-parse a batch of tabbed-format reads into given buffer.
+ */
+pair<bool, int> TabbedPatternSource::nextBatchFromFile(
+ PerThreadReadBuf& pt,
+ bool batch_a)
{
- success = true;
- done = false;
-
- // Skip over initial vertical whitespace
- if(fb_.peek() == '\r' || fb_.peek() == '\n') {
- fb_.peekUptoNewline();
- fb_.resetLastN();
- }
-
- // fb_ is about to dish out the first character of the
- // name field
- int mytrim5_1 = gTrim5;
- if(parseName(ra, &rb, '\t') == -1) {
- peekOverNewline(fb_); // skip rest of line
- ra.reset();
- rb.reset();
- fb_.resetLastN();
- success = false;
- done = true;
- return false;
+ int c = getc_unlocked(fp_);
+ while(c >= 0 && (c == '\n' || c == '\r')) {
+ c = getc_unlocked(fp_);
}
- assert_neq('\t', fb_.peek());
-
- // fb_ is about to dish out the first character of the
- // sequence field for the first mate
- int charsRead1 = 0;
- int dstLen1 = parseSeq(ra, charsRead1, mytrim5_1, '\t');
- if(dstLen1 < 0) {
- peekOverNewline(fb_); // skip rest of line
- ra.reset();
- rb.reset();
- fb_.resetLastN();
- success = false;
- done = true;
- return false;
+ EList<Read>& readbuf = batch_a ? pt.bufa_ : pt.bufb_;
+ size_t readi = 0;
+ // Read until we run out of input or until we've filled the buffer
+ for(; readi < pt.max_buf_ && c >= 0; readi++) {
+ readbuf[readi].readOrigBuf.clear();
+ while(c >= 0 && c != '\n' && c != '\r') {
+ readbuf[readi].readOrigBuf.append(c);
+ c = getc_unlocked(fp_);
+ }
+ while(c >= 0 && (c == '\n' || c == '\r')) {
+ c = getc_unlocked(fp_);
+ }
}
- assert_neq('\t', fb_.peek());
+ return make_pair(c < 0, readi);
+}
- // fb_ is about to dish out the first character of the
- // quality-string field
- char ct = 0;
- if(parseQuals(ra, charsRead1, dstLen1, mytrim5_1, ct, '\t', '\n') < 0) {
- peekOverNewline(fb_); // skip rest of line
- ra.reset();
- rb.reset();
- fb_.resetLastN();
- success = false;
- done = true;
- return false;
- }
- ra.trimmed3 = gTrim3;
- ra.trimmed5 = mytrim5_1;
- assert(ct == '\t' || ct == '\n' || ct == '\r' || ct == -1);
- if(ct == '\r' || ct == '\n' || ct == -1) {
- // Only had 3 fields prior to newline, so this must be an unpaired read
- rb.reset();
- ra.readOrigBuf.install(fb_.lastN(), fb_.lastNLen());
- fb_.resetLastN();
- success = true;
- done = false;
- paired = false;
- rdid = endid = readCnt_;
- readCnt_++;
- return success;
- }
- paired = true;
- assert_neq('\t', fb_.peek());
+/**
+ * Finalize tabbed parsing outside critical section.
+ */
+bool TabbedPatternSource::parse(Read& ra, Read& rb, TReadId rdid) const {
+ // Light parser (nextBatchFromFile) puts unparsed data
+ // into Read& r, even when the read is paired.
+ assert(ra.empty());
+ assert(rb.empty());
+ assert(!ra.readOrigBuf.empty()); // raw data for read/pair is here
+ assert(rb.readOrigBuf.empty());
+ int c = '\t';
+ size_t cur = 0;
+ const size_t buflen = ra.readOrigBuf.length();
- // Saw another tab after the third field, so this must be a pair
- if(secondName_) {
- // The second mate has its own name
- if(parseName(rb, NULL, '\t') == -1) {
- peekOverNewline(fb_); // skip rest of line
- ra.reset();
- rb.reset();
- fb_.resetLastN();
- success = false;
- done = true;
- return false;
+ // Loop over the two ends
+ for(int endi = 0; endi < 2 && c == '\t'; endi++) {
+ Read& r = ((endi == 0) ? ra : rb);
+ assert(r.name.empty());
+ // Parse name if (a) this is the first end, or
+ // (b) this is tab6
+ if(endi < 1 || secondName_) {
+ // Parse read name
+ c = ra.readOrigBuf[cur++];
+ while(c != '\t' && cur < buflen) {
+ r.name.append(c);
+ c = ra.readOrigBuf[cur++];
+ }
+ assert_eq('\t', c);
+ if(cur >= buflen) {
+ return false; // record ended prematurely
+ }
+ } else if(endi > 0) {
+ // if this is the second end and we're parsing
+ // tab5, copy name from first end
+ rb.name = ra.name;
}
- assert_neq('\t', fb_.peek());
- }
-
- // fb_ about to give the first character of the second mate's sequence
- int charsRead2 = 0;
- int mytrim5_2 = gTrim5;
- int dstLen2 = parseSeq(rb, charsRead2, mytrim5_2, '\t');
- if(dstLen2 < 0) {
- peekOverNewline(fb_); // skip rest of line
- ra.reset();
- rb.reset();
- fb_.resetLastN();
- success = false;
- done = true;
- return false;
- }
- assert_neq('\t', fb_.peek());
- // fb_ is about to dish out the first character of the
- // quality-string field
- if(parseQuals(rb, charsRead2, dstLen2, mytrim5_2, ct, '\n') < 0) {
- peekOverNewline(fb_); // skip rest of line
- ra.reset();
- rb.reset();
- fb_.resetLastN();
- success = false;
- done = true;
- return false;
+ // Parse sequence
+ assert(r.patFw.empty());
+ c = ra.readOrigBuf[cur++];
+ int nchar = 0;
+ while(c != '\t' && cur < buflen) {
+ if(isalpha(c)) {
+ assert_in(toupper(c), "ACGTN");
+ if(nchar++ >= gTrim5) {
+ assert_neq(0, asc2dnacat[c]);
+ r.patFw.append(asc2dna[c]); // ascii to int
+ }
+ }
+ c = ra.readOrigBuf[cur++];
+ }
+ assert_eq('\t', c);
+ if(cur >= buflen) {
+ return false; // record ended prematurely
+ }
+ // record amt trimmed from 5' end due to --trim5
+ r.trimmed5 = (int)(nchar - r.patFw.length());
+ // record amt trimmed from 3' end due to --trim3
+ r.trimmed3 = (int)(r.patFw.trimEnd(gTrim3));
+
+ // Parse qualities
+ assert(r.qual.empty());
+ c = ra.readOrigBuf[cur++];
+ int nqual = 0;
+ if (intQuals_) {
+ int cur_int = 0;
+ while(c != '\t' && c != '\n' && c != '\r' && cur < buflen) {
+ cur_int *= 10;
+ cur_int += (int)(c - '0');
+ c = ra.readOrigBuf[cur++];
+ if(c == ' ' || c == '\t' || c == '\n' || c == '\r') {
+ char cadd = intToPhred33(cur_int, solQuals_);
+ cur_int = 0;
+ assert_geq(cadd, 33);
+ if(++nqual > gTrim5) {
+ r.qual.append(cadd);
+ }
+ }
+ }
+ } else {
+ while(c != '\t' && c != '\n' && c != '\r') {
+ if(c == ' ') {
+ wrongQualityFormat(r.name);
+ return false;
+ }
+ char cadd = charToPhred33(c, solQuals_, phred64Quals_);
+ if(++nqual > gTrim5) {
+ r.qual.append(cadd);
+ }
+ if(cur >= buflen) break;
+ c = ra.readOrigBuf[cur++];
+ }
+ }
+ if(nchar > nqual) {
+ tooFewQualities(r.name);
+ return false;
+ } else if(nqual > nchar) {
+ tooManyQualities(r.name);
+ return false;
+ }
+ r.qual.trimEnd(gTrim3);
+ assert(c == '\t' || c == '\n' || c == '\r' || cur >= buflen);
+ assert_eq(r.patFw.length(), r.qual.length());
}
- ra.readOrigBuf.install(fb_.lastN(), fb_.lastNLen());
- fb_.resetLastN();
- rb.trimmed3 = gTrim3;
- rb.trimmed5 = mytrim5_2;
- rdid = endid = readCnt_;
- readCnt_++;
return true;
}
/**
- * Parse a name from fb_ and store in r. Assume that the next
- * character obtained via fb_.get() is the first character of
- * the sequence and the string stops at the next char upto (could
- * be tab, newline, etc.).
+ * Light-parse a batch of raw-format reads into given buffer.
*/
-int TabbedPatternSource::parseName(
- Read& r,
- Read* r2,
- char upto /* = '\t' */)
+pair<bool, int> RawPatternSource::nextBatchFromFile(
+ PerThreadReadBuf& pt,
+ bool batch_a)
{
- // Read the name out of the first field
- int c = 0;
- if(r2 != NULL) r2->name.clear();
- r.name.clear();
- while(true) {
- if((c = fb_.get()) < 0) {
- return -1;
- }
- if(c == upto) {
- // Finished with first field
- break;
+ int c = getc_unlocked(fp_);
+ while(c >= 0 && (c == '\n' || c == '\r')) {
+ c = getc_unlocked(fp_);
+ }
+ EList<Read>& readbuf = batch_a ? pt.bufa_ : pt.bufb_;
+ size_t readi = 0;
+ // Read until we run out of input or until we've filled the buffer
+ for(; readi < pt.max_buf_ && c >= 0; readi++) {
+ readbuf[readi].readOrigBuf.clear();
+ while(c >= 0 && c != '\n' && c != '\r') {
+ readbuf[readi].readOrigBuf.append(c);
+ c = getc_unlocked(fp_);
}
- if(c == '\n' || c == '\r') {
- return -1;
+ while(c >= 0 && (c == '\n' || c == '\r')) {
+ c = getc_unlocked(fp_);
}
- if(r2 != NULL) r2->name.append(c);
- r.name.append(c);
}
- // Set up a default name if one hasn't been set
- if(r.name.empty()) {
- char cbuf[20];
- itoa10<TReadId>(readCnt_, cbuf);
- r.name.install(cbuf);
- if(r2 != NULL) r2->name.install(cbuf);
+ // incase a valid character is consumed between batches
+ if (c >= 0 && c != '\n' && c != '\r') {
+ ungetc(c, fp_);
}
- return (int)r.name.length();
+ return make_pair(c < 0, readi);
}
/**
- * Parse a single sequence from fb_ and store in r. Assume
- * that the next character obtained via fb_.get() is the first
- * character of the sequence and the sequence stops at the next
- * char upto (could be tab, newline, etc.).
+ * Finalize raw parsing outside critical section.
*/
-int TabbedPatternSource::parseSeq(
- Read& r,
- int& charsRead,
- int& trim5,
- char upto /*= '\t'*/)
-{
- int begin = 0;
- int c = fb_.get();
- assert(c != upto);
- r.patFw.clear();
- r.color = gColor;
- if(gColor) {
- // This may be a primer character. If so, keep it in the
- // 'primer' field of the read buf and parse the rest of the
- // read without it.
- c = toupper(c);
- if(asc2dnacat[c] > 0) {
- // First char is a DNA char
- int c2 = toupper(fb_.peek());
- // Second char is a color char
- if(asc2colcat[c2] > 0) {
- r.primer = c;
- r.trimc = c2;
- trim5 += 2; // trim primer and first color
- }
- }
- if(c < 0) { return -1; }
- }
- while(c != upto) {
- if(gColor) {
- if(c >= '0' && c <= '4') c = "ACGTN"[(int)c - '0'];
- if(c == '.') c = 'N';
- }
+bool RawPatternSource::parse(Read& r, Read& rb, TReadId rdid) const {
+ assert(r.empty());
+ assert(!r.readOrigBuf.empty()); // raw data for read/pair is here
+ int c = '\n';
+ size_t cur = 0;
+ const size_t buflen = r.readOrigBuf.length();
+
+ // Parse sequence
+ assert(r.patFw.empty());
+ int nchar = 0;
+ while(cur < buflen) {
+ c = r.readOrigBuf[cur++];
+ assert(c != '\r' && c != '\n');
if(isalpha(c)) {
assert_in(toupper(c), "ACGTN");
- if(begin++ >= trim5) {
+ if(nchar++ >= gTrim5) {
assert_neq(0, asc2dnacat[c]);
- r.patFw.append(asc2dna[c]);
+ r.patFw.append(asc2dna[c]); // ascii to int
}
- charsRead++;
- }
- if((c = fb_.get()) < 0) {
- return -1;
}
}
- r.patFw.trimEnd(gTrim3);
- return (int)r.patFw.length();
-}
-
-/**
- * Parse a single quality string from fb_ and store in r.
- * Assume that the next character obtained via fb_.get() is
- * the first character of the quality string and the string stops
- * at the next char upto (could be tab, newline, etc.).
- */
-int TabbedPatternSource::parseQuals(
- Read& r,
- int charsRead,
- int dstLen,
- int trim5,
- char& c2,
- char upto /*= '\t'*/,
- char upto2 /*= -1*/)
-{
- int qualsRead = 0;
- int c = 0;
- if (intQuals_) {
- char buf[4096];
- while (qualsRead < charsRead) {
- qualToks_.clear();
- if(!tokenizeQualLine(fb_, buf, 4096, qualToks_)) break;
- for (unsigned int j = 0; j < qualToks_.size(); ++j) {
- char c = intToPhred33(atoi(qualToks_[j].c_str()), solQuals_);
- assert_geq(c, 33);
- if (qualsRead >= trim5) {
- r.qual.append(c);
- }
- ++qualsRead;
- }
- } // done reading integer quality lines
- if (charsRead > qualsRead) tooFewQualities(r.name);
- } else {
- // Non-integer qualities
- while((qualsRead < dstLen + trim5) && c >= 0) {
- c = fb_.get();
- c2 = c;
- if (c == ' ') wrongQualityFormat(r.name);
- if(c < 0) {
- // EOF occurred in the middle of a read - abort
- return -1;
- }
- if(!isspace(c) && c != upto && (upto2 == -1 || c != upto2)) {
- if (qualsRead >= trim5) {
- c = charToPhred33(c, solQuals_, phred64Quals_);
- assert_geq(c, 33);
- r.qual.append(c);
- }
- qualsRead++;
- } else {
- break;
- }
- }
- if(qualsRead < dstLen + trim5) {
- tooFewQualities(r.name);
- } else if(qualsRead > dstLen + trim5) {
- tooManyQualities(r.name);
- }
+ assert_eq(cur, buflen);
+ // record amt trimmed from 5' end due to --trim5
+ r.trimmed5 = (int)(nchar - r.patFw.length());
+ // record amt trimmed from 3' end due to --trim3
+ r.trimmed3 = (int)(r.patFw.trimEnd(gTrim3));
+
+ // Give the name field a dummy value
+ char cbuf[20];
+ itoa10<TReadId>(rdid, cbuf);
+ r.name.install(cbuf);
+
+ // Give the base qualities dummy values
+ assert(r.qual.empty());
+ const size_t len = r.patFw.length();
+ for(size_t i = 0; i < len; i++) {
+ r.qual.append('I');
}
- r.qual.resize(dstLen);
- while(c != upto && (upto2 == -1 || c != upto2) && c != -1) {
- c = fb_.get();
- c2 = c;
+ r.parsed = true;
+ if(!rb.parsed && !rb.readOrigBuf.empty()) {
+ return parse(rb, r, rdid);
}
- return qualsRead;
+ return true;
}
+
void wrongQualityFormat(const BTString& read_name) {
cerr << "Error: Encountered one or more spaces while parsing the quality "
<< "string for read " << read_name << ". If this is a FASTQ file "
diff --git a/pat.h b/pat.h
index cf3e92b..e5071db 100644
--- a/pat.h
+++ b/pat.h
@@ -21,19 +21,12 @@
#define PAT_H_
#include <cassert>
-#include <cmath>
-#include <stdexcept>
-#include <vector>
#include <string>
-#include <cstring>
#include <ctype.h>
-#include <fstream>
#include "alphabet.h"
#include "assert_helpers.h"
-#include "tokenize.h"
#include "random_source.h"
#include "threading.h"
-#include "filebuf.h"
#include "qual.h"
#include "search_globals.h"
#include "sstring.h"
@@ -41,658 +34,208 @@
#include "read.h"
#include "util.h"
-/**
- * Classes and routines for reading reads from various input sources.
- */
-
-using namespace std;
+#ifdef _WIN32
+#define getc_unlocked _fgetc_nolock
+#endif
/**
- * Calculate a per-read random seed based on a combination of
- * the read data (incl. sequence, name, quals) and the global
- * seed in '_randSeed'.
+ * Classes and routines for reading reads from various input sources.
*/
-static inline uint32_t genRandSeed(const BTDnaString& qry,
- const BTString& qual,
- const BTString& name,
- uint32_t seed)
-{
- // Calculate a per-read random seed based on a combination of
- // the read data (incl. sequence, name, quals) and the global
- // seed
- uint32_t rseed = (seed + 101) * 59 * 61 * 67 * 71 * 73 * 79 * 83;
- size_t qlen = qry.length();
- // Throw all the characters of the read into the random seed
- for(size_t i = 0; i < qlen; i++) {
- int p = (int)qry[i];
- assert_leq(p, 4);
- size_t off = ((i & 15) << 1);
- rseed ^= (p << off);
- }
- // Throw all the quality values for the read into the random
- // seed
- for(size_t i = 0; i < qlen; i++) {
- int p = (int)qual[i];
- assert_leq(p, 255);
- size_t off = ((i & 3) << 3);
- rseed ^= (p << off);
- }
- // Throw all the characters in the read name into the random
- // seed
- size_t namelen = name.length();
- for(size_t i = 0; i < namelen; i++) {
- int p = (int)name[i];
- if(p == '/') break;
- assert_leq(p, 255);
- size_t off = ((i & 3) << 3);
- rseed ^= (p << off);
- }
- return rseed;
-}
/**
* Parameters affecting how reads and read in.
*/
struct PatternParams {
+
+ PatternParams() { }
+
PatternParams(
int format_,
bool fileParallel_,
uint32_t seed_,
+ size_t max_buf_,
bool solexa64_,
bool phred64_,
bool intQuals_,
- bool fuzzy_,
int sampleLen_,
int sampleFreq_,
- uint32_t skip_) :
+ size_t skip_,
+ int nthreads_,
+ bool fixName_) :
format(format_),
fileParallel(fileParallel_),
seed(seed_),
+ max_buf(max_buf_),
solexa64(solexa64_),
phred64(phred64_),
intQuals(intQuals_),
- fuzzy(fuzzy_),
sampleLen(sampleLen_),
sampleFreq(sampleFreq_),
- skip(skip_) { }
+ skip(skip_),
+ nthreads(nthreads_),
+ fixName(fixName_) { }
int format; // file format
- bool fileParallel; // true -> wrap files with separate PairedPatternSources
+ bool fileParallel; // true -> wrap files with separate PatternComposers
uint32_t seed; // pseudo-random seed
+ size_t max_buf; // number of reads to buffer in one read
bool solexa64; // true -> qualities are on solexa64 scale
bool phred64; // true -> qualities are on phred64 scale
bool intQuals; // true -> qualities are space-separated numbers
- bool fuzzy; // true -> try to parse fuzzy fastq
int sampleLen; // length of sampled reads for FastaContinuous...
int sampleFreq; // frequency of sampled reads for FastaContinuous...
- uint32_t skip; // skip the first 'skip' patterns
+ size_t skip; // skip the first 'skip' patterns
+ int nthreads; // number of threads for locking
+ bool fixName; //
};
/**
- * Encapsulates a synchronized source of patterns; usually a file.
- * Optionally reverses reads and quality strings before returning them,
- * though that is usually more efficiently done by the concrete
- * subclass. Concrete subclasses should delimit critical sections with
- * calls to lock() and unlock().
+ * All per-thread storage for input read data.
*/
-class PatternSource {
-
-public:
-
- PatternSource(const PatternParams& p) :
- seed_(p.seed),
- readCnt_(0),
- numWrappers_(0),
- doLocking_(true),
- mutex()
+struct PerThreadReadBuf {
+
+ PerThreadReadBuf(size_t max_buf) :
+ max_buf_(max_buf),
+ bufa_(max_buf),
+ bufb_(max_buf),
+ rdid_()
{
- }
-
- virtual ~PatternSource() { }
-
- /**
- * Call this whenever this PatternSource is wrapped by a new
- * WrappedPatternSourcePerThread. This helps us keep track of
- * whether locks will be contended.
- */
- void addWrapper() {
- lock();
- numWrappers_++;
- unlock();
+ bufa_.resize(max_buf);
+ bufb_.resize(max_buf);
+ reset();
}
- /**
- * The main member function for dispensing patterns.
- *
- * Returns true iff a pair was parsed succesfully.
- */
- virtual bool nextReadPair(
- Read& ra,
- Read& rb,
- TReadId& rdid,
- TReadId& endid,
- bool& success,
- bool& done,
- bool& paired,
- bool fixName);
-
- /**
- * The main member function for dispensing patterns.
- */
- virtual bool nextRead(
- Read& r,
- TReadId& rdid,
- TReadId& endid,
- bool& success,
- bool& done);
-
- /**
- * Implementation to be provided by concrete subclasses. An
- * implementation for this member is only relevant for formats that
- * can read in a pair of reads in a single transaction with a
- * single input source. If paired-end input is given as a pair of
- * parallel files, this member should throw an error and exit.
- */
- virtual bool nextReadPairImpl(
- Read& ra,
- Read& rb,
- TReadId& rdid,
- TReadId& endid,
- bool& success,
- bool& done,
- bool& paired) = 0;
-
- /**
- * Implementation to be provided by concrete subclasses. An
- * implementation for this member is only relevant for formats
- * where individual input sources look like single-end-read
- * sources, e.g., formats where paired-end reads are specified in
- * parallel read files.
- */
- virtual bool nextReadImpl(
- Read& r,
- TReadId& rdid,
- TReadId& endid,
- bool& success,
- bool& done) = 0;
-
- /// Reset state to start over again with the first read
- virtual void reset() { readCnt_ = 0; }
-
- /**
- * Concrete subclasses call lock() to enter a critical region.
- * What constitutes a critical region depends on the subclass.
- */
- void lock() {
- if(!doLocking_) return; // no contention
- mutex.lock();
- }
-
- /**
- * Concrete subclasses call unlock() to exit a critical region
- * What constitutes a critical region depends on the subclass.
- */
- void unlock() {
- if(!doLocking_) return; // no contention
- mutex.unlock();
- }
-
- /**
- * Return a new dynamically allocated PatternSource for the given
- * format, using the given list of strings as the filenames to read
- * from or as the sequences themselves (i.e. if -c was used).
- */
- static PatternSource* patsrcFromStrings(
- const PatternParams& p,
- const EList<string>& qs);
-
- /**
- * Return the number of reads attempted.
- */
- TReadId readCnt() const { return readCnt_ - 1; }
-
-protected:
-
- uint32_t seed_;
-
- /// The number of reads read by this PatternSource
- TReadId readCnt_;
-
- int numWrappers_; /// # threads that own a wrapper for this PatternSource
- bool doLocking_; /// override whether to lock (true = don't override)
- MUTEX_T mutex;
-};
-
-/**
- * Abstract parent class for synhconized sources of paired-end reads
- * (and possibly also single-end reads).
- */
-class PairedPatternSource {
-public:
- PairedPatternSource(const PatternParams& p) : mutex_m(), seed_(p.seed) {}
- virtual ~PairedPatternSource() { }
-
- virtual void addWrapper() = 0;
- virtual void reset() = 0;
+ Read& read_a() { return bufa_[cur_buf_]; }
+ Read& read_b() { return bufb_[cur_buf_]; }
- virtual bool nextReadPair(
- Read& ra,
- Read& rb,
- TReadId& rdid,
- TReadId& endid,
- bool& success,
- bool& done,
- bool& paired,
- bool fixName) = 0;
+ const Read& read_a() const { return bufa_[cur_buf_]; }
+ const Read& read_b() const { return bufb_[cur_buf_]; }
- virtual pair<TReadId, TReadId> readCnt() const = 0;
-
- /**
- * Lock this PairedPatternSource, usually because one of its shared
- * fields is being updated.
- */
- void lock() {
- mutex_m.lock();
- }
-
/**
- * Unlock this PairedPatternSource.
+ * Return read id for read/pair currently in the buffer.
*/
- void unlock() {
- mutex_m.unlock();
+ TReadId rdid() const {
+ assert_neq(rdid_, std::numeric_limits<TReadId>::max());
+ return rdid_ + cur_buf_;
}
-
- /**
- * Given the values for all of the various arguments used to specify
- * the read and quality input, create a list of pattern sources to
- * dispense them.
- */
- static PairedPatternSource* setupPatternSources(
- const EList<string>& si, // singles, from argv
- const EList<string>& m1, // mate1's, from -1 arg
- const EList<string>& m2, // mate2's, from -2 arg
- const EList<string>& m12, // both mates on each line, from --12 arg
- const EList<string>& q, // qualities associated with singles
- const EList<string>& q1, // qualities associated with m1
- const EList<string>& q2, // qualities associated with m2
- const PatternParams& p, // read-in params
- bool verbose); // be talkative?
-
-protected:
- static void free_EList_pmembers(const EList<PatternSource*>&);
-
- MUTEX_T mutex_m; /// mutex for syncing over critical regions
- uint32_t seed_;
-};
-
-/**
- * Encapsulates a synchronized source of both paired-end reads and
- * unpaired reads, where the paired-end must come from parallel files.
- */
-class PairedSoloPatternSource : public PairedPatternSource {
-
-public:
-
- PairedSoloPatternSource(
- const EList<PatternSource*>* src,
- const PatternParams& p) :
- PairedPatternSource(p),
- cur_(0),
- src_(src)
- {
- assert(src_ != NULL);
- for(size_t i = 0; i < src_->size(); i++) {
- assert((*src_)[i] != NULL);
- }
- }
-
- virtual ~PairedSoloPatternSource() {
- free_EList_pmembers(*src_);
- delete src_;
- }
-
+
/**
- * Call this whenever this PairedPatternSource is wrapped by a new
- * WrappedPatternSourcePerThread. This helps us keep track of
- * whether locks within PatternSources will be contended.
+ * Reset state as though no reads have been read.
*/
- virtual void addWrapper() {
- for(size_t i = 0; i < src_->size(); i++) {
- (*src_)[i]->addWrapper();
+ void reset() {
+ cur_buf_ = bufa_.size();
+ for(size_t i = 0; i < max_buf_; i++) {
+ bufa_[i].reset();
+ bufb_[i].reset();
}
+ rdid_ = std::numeric_limits<TReadId>::max();
}
-
+
/**
- * Reset this object and all the PatternSources under it so that
- * the next call to nextReadPair gets the very first read pair.
+ * Advance cursor to next element
*/
- virtual void reset() {
- for(size_t i = 0; i < src_->size(); i++) {
- (*src_)[i]->reset();
- }
- cur_ = 0;
+ void next() {
+ assert_lt(cur_buf_, bufa_.size());
+ cur_buf_++;
}
-
- /**
- * The main member function for dispensing pairs of reads or
- * singleton reads. Returns true iff ra and rb contain a new
- * pair; returns false if ra contains a new unpaired read.
- */
- virtual bool nextReadPair(
- Read& ra,
- Read& rb,
- TReadId& rdid,
- TReadId& endid,
- bool& success,
- bool& done,
- bool& paired,
- bool fixName);
-
+
/**
- * Return the number of reads attempted.
+ * Return true when there's nothing left for next().
*/
- virtual pair<TReadId, TReadId> readCnt() const {
- uint64_t ret = 0llu;
- for(size_t i = 0; i < src_->size(); i++) ret += (*src_)[i]->readCnt();
- return make_pair(ret, 0llu);
- }
-
-protected:
- volatile uint32_t cur_; // current element in parallel srca_, srcb_ vectors
- const EList<PatternSource*>* src_; /// PatternSources for paired-end reads
-};
-
-/**
- * Encapsulates a synchronized source of both paired-end reads and
- * unpaired reads, where the paired-end must come from parallel files.
- */
-class PairedDualPatternSource : public PairedPatternSource {
-
-public:
-
- PairedDualPatternSource(
- const EList<PatternSource*>* srca,
- const EList<PatternSource*>* srcb,
- const PatternParams& p) :
- PairedPatternSource(p), cur_(0), srca_(srca), srcb_(srcb)
- {
- assert(srca_ != NULL);
- assert(srcb_ != NULL);
- // srca_ and srcb_ must be parallel
- assert_eq(srca_->size(), srcb_->size());
- for(size_t i = 0; i < srca_->size(); i++) {
- // Can't have NULL first-mate sources. Second-mate sources
- // can be NULL, in the case when the corresponding first-
- // mate source is unpaired.
- assert((*srca_)[i] != NULL);
- for(size_t j = 0; j < srcb_->size(); j++) {
- assert_neq((*srca_)[i], (*srcb_)[j]);
- }
- }
- }
-
- virtual ~PairedDualPatternSource() {
- free_EList_pmembers(*srca_);
- delete srca_;
- free_EList_pmembers(*srcb_);
- delete srcb_;
+ bool exhausted() {
+ assert_leq(cur_buf_, bufa_.size());
+ return cur_buf_ >= bufa_.size()-1;
}
-
+
/**
- * Call this whenever this PairedPatternSource is wrapped by a new
- * WrappedPatternSourcePerThread. This helps us keep track of
- * whether locks within PatternSources will be contended.
+ * Just after a new batch has been loaded, use init to
+ * set cur_buf_ appropriately.
*/
- virtual void addWrapper() {
- for(size_t i = 0; i < srca_->size(); i++) {
- (*srca_)[i]->addWrapper();
- if((*srcb_)[i] != NULL) {
- (*srcb_)[i]->addWrapper();
- }
- }
+ void init() {
+ cur_buf_ = 0;
}
-
+
/**
- * Reset this object and all the PatternSources under it so that
- * the next call to nextReadPair gets the very first read pair.
+ * Set read id of first read in buffer.
*/
- virtual void reset() {
- for(size_t i = 0; i < srca_->size(); i++) {
- (*srca_)[i]->reset();
- if((*srcb_)[i] != NULL) {
- (*srcb_)[i]->reset();
- }
- }
- cur_ = 0;
+ void setReadId(TReadId rdid) {
+ rdid_ = rdid;
}
-
- /**
- * The main member function for dispensing pairs of reads or
- * singleton reads. Returns true iff ra and rb contain a new
- * pair; returns false if ra contains a new unpaired read.
- */
- virtual bool nextReadPair(
- Read& ra,
- Read& rb,
- TReadId& rdid,
- TReadId& endid,
- bool& success,
- bool& done,
- bool& paired,
- bool fixName);
- /**
- * Return the number of reads attempted.
- */
- virtual pair<TReadId, TReadId> readCnt() const;
-
-protected:
-
- volatile uint32_t cur_; // current element in parallel srca_, srcb_ vectors
- const EList<PatternSource*>* srca_; /// PatternSources for 1st mates and/or unpaired reads
- const EList<PatternSource*>* srcb_; /// PatternSources for 2nd mates
+ const size_t max_buf_; // max # reads to read into buffer at once
+ EList<Read> bufa_; // Read buffer for mate as
+ EList<Read> bufb_; // Read buffer for mate bs
+ size_t cur_buf_; // Read buffer currently active
+ TReadId rdid_; // index of read at offset 0 of bufa_/bufb_
};
+extern void wrongQualityFormat(const BTString& read_name);
+extern void tooFewQualities(const BTString& read_name);
+extern void tooManyQualities(const BTString& read_name);
+
/**
- * Encapsulates a single thread's interaction with the PatternSource.
- * Most notably, this class holds the buffers into which the
- * PatterSource will write sequences. This class is *not* threadsafe
- * - it doesn't need to be since there's one per thread. PatternSource
- * is thread-safe.
+ * Encapsulates a synchronized source of patterns; usually a file.
+ * Optionally reverses reads and quality strings before returning them,
+ * though that is usually more efficiently done by the concrete
+ * subclass. Concrete subclasses should delimit critical sections with
+ * calls to lock() and unlock().
*/
-class PatternSourcePerThread {
-
+class PatternSource {
+
public:
-
- PatternSourcePerThread() :
- buf1_(), buf2_(), rdid_(0xffffffff), endid_(0xffffffff) { }
-
- virtual ~PatternSourcePerThread() { }
-
+
+ PatternSource(const PatternParams& p) :
+ readCnt_(0),
+ mutex() { }
+
+ virtual ~PatternSource() { }
+
/**
- * Read the next read pair.
+ * Implementation to be provided by concrete subclasses. An
+ * implementation for this member is only relevant for formats
+ * where individual input sources look like single-end-read
+ * sources, e.g., formats where paired-end reads are specified in
+ * parallel read files.
*/
- virtual bool nextReadPair(
- bool& success,
- bool& done,
- bool& paired,
- bool fixName)
- {
- return success;
- }
-
- Read& bufa() { return buf1_; }
- Read& bufb() { return buf2_; }
- const Read& bufa() const { return buf1_; }
- const Read& bufb() const { return buf2_; }
-
- TReadId rdid() const { return rdid_; }
- TReadId endid() const { return endid_; }
- virtual void reset() { rdid_ = endid_ = 0xffffffff; }
+ virtual std::pair<bool, int> nextBatch(
+ PerThreadReadBuf& pt,
+ bool batch_a,
+ bool lock = true) = 0;
/**
- * Return the length of mate 1 or mate 2.
+ * Finishes parsing a given read. Happens outside the critical section.
*/
- size_t length(int mate) const {
- return (mate == 1) ? buf1_.length() : buf2_.length();
- }
-
-protected:
-
- Read buf1_; // read buffer for mate a
- Read buf2_; // read buffer for mate b
- TReadId rdid_; // index of read just read
- TReadId endid_; // index of read just read
-};
-
-/**
- * Abstract parent factory for PatternSourcePerThreads.
- */
-class PatternSourcePerThreadFactory {
-public:
- virtual ~PatternSourcePerThreadFactory() { }
- virtual PatternSourcePerThread* create() const = 0;
- virtual EList<PatternSourcePerThread*>* create(uint32_t n) const = 0;
-
- /// Free memory associated with a pattern source
- virtual void destroy(PatternSourcePerThread* patsrc) const {
- assert(patsrc != NULL);
- // Free the PatternSourcePerThread
- delete patsrc;
- }
-
- /// Free memory associated with a pattern source list
- virtual void destroy(EList<PatternSourcePerThread*>* patsrcs) const {
- assert(patsrcs != NULL);
- // Free all of the PatternSourcePerThreads
- for(size_t i = 0; i < patsrcs->size(); i++) {
- if((*patsrcs)[i] != NULL) {
- delete (*patsrcs)[i];
- (*patsrcs)[i] = NULL;
- }
- }
- // Free the vector
- delete patsrcs;
- }
-};
-
-/**
- * A per-thread wrapper for a PairedPatternSource.
- */
-class WrappedPatternSourcePerThread : public PatternSourcePerThread {
-public:
- WrappedPatternSourcePerThread(PairedPatternSource& __patsrc) :
- patsrc_(__patsrc)
- {
- patsrc_.addWrapper();
- }
-
+ virtual bool parse(Read& ra, Read& rb, TReadId rdid) const = 0;
+
/**
- * Get the next paired or unpaired read from the wrapped
- * PairedPatternSource.
+ * Reset so that next call to nextBatch* gets the first batch.
*/
- virtual bool nextReadPair(
- bool& success,
- bool& done,
- bool& paired,
- bool fixName);
-
-private:
-
- /// Container for obtaining paired reads from PatternSources
- PairedPatternSource& patsrc_;
-};
-
-/**
- * Abstract parent factory for PatternSourcePerThreads.
- */
-class WrappedPatternSourcePerThreadFactory : public PatternSourcePerThreadFactory {
-public:
- WrappedPatternSourcePerThreadFactory(PairedPatternSource& patsrc) :
- patsrc_(patsrc) { }
-
+ virtual void reset() { readCnt_ = 0; }
+
/**
- * Create a new heap-allocated WrappedPatternSourcePerThreads.
+ * Return a new dynamically allocated PatternSource for the given
+ * format, using the given list of strings as the filenames to read
+ * from or as the sequences themselves (i.e. if -c was used).
*/
- virtual PatternSourcePerThread* create() const {
- return new WrappedPatternSourcePerThread(patsrc_);
- }
-
+ static PatternSource* patsrcFromStrings(
+ const PatternParams& p,
+ const EList<std::string>& qs);
+
/**
- * Create a new heap-allocated vector of heap-allocated
- * WrappedPatternSourcePerThreads.
+ * Return number of reads light-parsed by this stream so far.
*/
- virtual EList<PatternSourcePerThread*>* create(uint32_t n) const {
- EList<PatternSourcePerThread*>* v = new EList<PatternSourcePerThread*>;
- for(size_t i = 0; i < n; i++) {
- v->push_back(new WrappedPatternSourcePerThread(patsrc_));
- assert(v->back() != NULL);
- }
- return v;
- }
-
-private:
- /// Container for obtaining paired reads from PatternSources
- PairedPatternSource& patsrc_;
+ TReadId readCount() const { return readCnt_; }
+
+protected:
+
+ /// The number of reads read by this PatternSource
+ volatile TReadId readCnt_;
+
+ /// Lock enforcing mutual exclusion for (a) file I/O, (b) writing fields
+ /// of this or another other shared object.
+ MUTEX_T mutex;
};
-/// Skip to the end of the current string of newline chars and return
-/// the first character after the newline chars, or -1 for EOF
-static inline int getOverNewline(FileBuf& in) {
- int c;
- while(isspace(c = in.get()));
- return c;
-}
-
-/// Skip to the end of the current string of newline chars such that
-/// the next call to get() returns the first character after the
-/// whitespace
-static inline int peekOverNewline(FileBuf& in) {
- while(true) {
- int c = in.peek();
- if(c != '\r' && c != '\n') {
- return c;
- }
- in.get();
- }
-}
-
-/// Skip to the end of the current line; return the first character
-/// of the next line or -1 for EOF
-static inline int getToEndOfLine(FileBuf& in) {
- while(true) {
- int c = in.get(); if(c < 0) return -1;
- if(c == '\n' || c == '\r') {
- while(c == '\n' || c == '\r') {
- c = in.get(); if(c < 0) return -1;
- }
- // c now holds first character of next line
- return c;
- }
- }
-}
-
-/// Skip to the end of the current line such that the next call to
-/// get() returns the first character on the next line
-static inline int peekToEndOfLine(FileBuf& in) {
- while(true) {
- int c = in.get(); if(c < 0) return c;
- if(c == '\n' || c == '\r') {
- c = in.peek();
- while(c == '\n' || c == '\r') {
- in.get(); if(c < 0) return c; // consume \r or \n
- c = in.peek();
- }
- // next get() gets first character of next line
- return c;
- }
- }
-}
-
-extern void wrongQualityFormat(const BTString& read_name);
-extern void tooFewQualities(const BTString& read_name);
-extern void tooManyQualities(const BTString& read_name);
-
/**
* Encapsulates a source of patterns which is an in-memory vector.
*/
@@ -700,145 +243,103 @@ class VectorPatternSource : public PatternSource {
public:
+ /**
+ * Populate member lists, v_, quals_, names_, etc, with information parsed
+ * from the given list of strings.
+ */
VectorPatternSource(
- const EList<string>& v,
+ const EList<std::string>& v,
const PatternParams& p);
virtual ~VectorPatternSource() { }
- virtual bool nextReadImpl(
- Read& r,
- TReadId& rdid,
- TReadId& endid,
- bool& success,
- bool& done);
-
/**
- * This is unused, but implementation is given for completeness.
+ * Read next batch. However, batch concept is not very applicable for this
+ * PatternSource where all the info has already been parsed into the fields
+ * in the contsructor. This essentially modifies the pt as though we read
+ * in some number of patterns.
*/
- virtual bool nextReadPairImpl(
- Read& ra,
- Read& rb,
- TReadId& rdid,
- TReadId& endid,
- bool& success,
- bool& done,
- bool& paired);
+ virtual std::pair<bool, int> nextBatch(
+ PerThreadReadBuf& pt,
+ bool batch_a,
+ bool lock = true);
+ /**
+ * Reset so that next call to nextBatch* gets the first batch.
+ */
virtual void reset() {
PatternSource::reset();
cur_ = skip_;
paired_ = false;
}
+
+ /**
+ * Finishes parsing outside the critical section
+ */
+ virtual bool parse(Read& ra, Read& rb, TReadId rdid) const;
private:
- size_t cur_;
- uint32_t skip_;
- bool paired_;
- EList<BTDnaString> v_; // forward sequences
- EList<BTString> quals_; // forward qualities
- EList<BTString> names_; // names
- EList<int> trimmed3_; // names
- EList<int> trimmed5_; // names
+ size_t cur_; // index for first read of next batch
+ size_t skip_; // # reads to skip
+ bool paired_; // whether reads are paired
+ EList<string> tokbuf_; // buffer for storing parsed tokens
+ EList<Read::TBuf> bufs_; // per-read buffers
+ char nametmp_[20]; // temp buffer for constructing name
};
/**
- *
+ * Parent class for PatternSources that read from a file.
+ * Uses unlocked C I/O, on the assumption that all reading
+ * from the file will take place in an otherwise-protected
+ * critical section.
*/
-class BufferedFilePatternSource : public PatternSource {
+class CFilePatternSource : public PatternSource {
public:
- BufferedFilePatternSource(
- const EList<string>& infiles,
+ CFilePatternSource(
+ const EList<std::string>& infiles,
const PatternParams& p) :
PatternSource(p),
infiles_(infiles),
filecur_(0),
- fb_(),
+ fp_(NULL),
+ is_open_(false),
skip_(p.skip),
first_(true)
{
assert_gt(infiles.size(), 0);
errs_.resize(infiles_.size());
errs_.fill(0, infiles_.size(), false);
- assert(!fb_.isOpen());
open(); // open first file in the list
filecur_++;
}
- virtual ~BufferedFilePatternSource() {
- if(fb_.isOpen()) fb_.close();
+ /**
+ * Close open file.
+ */
+ virtual ~CFilePatternSource() {
+ if(is_open_) {
+ assert(fp_ != NULL);
+ fclose(fp_);
+ }
}
/**
* Fill Read with the sequence, quality and name for the next
* read in the list of read files. This function gets called by
* all the search threads, so we must handle synchronization.
- */
- virtual bool nextReadImpl(
- Read& r,
- TReadId& rdid,
- TReadId& endid,
- bool& success,
- bool& done)
- {
- // We'll be manipulating our file handle/filecur_ state
- lock();
- while(true) {
- do { read(r, rdid, endid, success, done); }
- while(!success && !done);
- if(!success && filecur_ < infiles_.size()) {
- assert(done);
- open();
- resetForNextFile(); // reset state to handle a fresh file
- filecur_++;
- continue;
- }
- break;
- }
- assert(r.repOk());
- // Leaving critical region
- unlock();
- return success;
- }
-
- /**
*
+ * Returns pair<bool, int> where bool indicates whether we're
+ * completely done, and int indicates how many reads were read.
*/
- virtual bool nextReadPairImpl(
- Read& ra,
- Read& rb,
- TReadId& rdid,
- TReadId& endid,
- bool& success,
- bool& done,
- bool& paired)
- {
- // We'll be manipulating our file handle/filecur_ state
- lock();
- while(true) {
- do { readPair(ra, rb, rdid, endid, success, done, paired); }
- while(!success && !done);
- if(!success && filecur_ < infiles_.size()) {
- assert(done);
- open();
- resetForNextFile(); // reset state to handle a fresh file
- filecur_++;
- continue;
- }
- break;
- }
- assert(ra.repOk());
- assert(rb.repOk());
- // Leaving critical region
- unlock();
- return success;
- }
+ virtual std::pair<bool, int> nextBatch(
+ PerThreadReadBuf& pt,
+ bool batch_a,
+ bool lock = true);
/**
- * Reset state so that we read start reading again from the
- * beginning of the first file. Should only be called by the
- * master thread.
+ * Reset so that next call to nextBatch* gets the first batch.
+ * Should only be called by the master thread.
*/
virtual void reset() {
PatternSource::reset();
@@ -849,94 +350,70 @@ public:
protected:
- /// Read another pattern from the input file; this is overridden
- /// to deal with specific file formats
- virtual bool read(
- Read& r,
- TReadId& rdid,
- TReadId& endid,
- bool& success,
- bool& done) = 0;
-
- /// Read another pattern pair from the input file; this is
- /// overridden to deal with specific file formats
- virtual bool readPair(
- Read& ra,
- Read& rb,
- TReadId& rdid,
- TReadId& endid,
- bool& success,
- bool& done,
- bool& paired) = 0;
-
- /// Reset state to handle a fresh file
+ /**
+ * Light-parse a batch of unpaired reads from current file into the given
+ * buffer. Called from CFilePatternSource.nextBatch().
+ */
+ virtual std::pair<bool, int> nextBatchFromFile(
+ PerThreadReadBuf& pt,
+ bool batch_a) = 0;
+
+ /**
+ * Reset state to handle a fresh file
+ */
virtual void resetForNextFile() { }
- void open() {
- if(fb_.isOpen()) fb_.close();
- while(filecur_ < infiles_.size()) {
- // Open read
- FILE *in;
- if(infiles_[filecur_] == "-") {
- in = stdin;
- } else if((in = fopen(infiles_[filecur_].c_str(), "rb")) == NULL) {
- if(!errs_[filecur_]) {
- cerr << "Warning: Could not open read file \"" << infiles_[filecur_].c_str() << "\" for reading; skipping..." << endl;
- errs_[filecur_] = true;
- }
- filecur_++;
- continue;
- }
- fb_.newFile(in);
- return;
- }
- cerr << "Error: No input read files were valid" << endl;
- exit(1);
- return;
- }
+ /**
+ * Open the next file in the list of input files.
+ */
+ void open();
- EList<string> infiles_; // filenames for read files
+ EList<std::string> infiles_; // filenames for read files
EList<bool> errs_; // whether we've already printed an error for each file
size_t filecur_; // index into infiles_ of next file to read
- FileBuf fb_; // read file currently being read from
+ FILE *fp_; // read file currently being read from
+ bool is_open_; // whether fp_ is currently open
TReadId skip_; // number of reads to skip
- bool first_;
+ bool first_; // parsing first record in first file?
+ char buf_[64*1024]; // file buffer
};
/**
- * Parse a single quality string from fb and store qualities in r.
- * Assume the next character obtained via fb.get() is the first
- * character of the quality string. When returning, the next
- * character returned by fb.peek() or fb.get() should be the first
- * character of the following line.
+ * Synchronized concrete pattern source for a list of FASTA files.
*/
-int parseQuals(
- Read& r,
- FileBuf& fb,
- int firstc,
- int readLen,
- int trim3,
- int trim5,
- bool intQuals,
- bool phred64,
- bool solexa64);
+class FastaPatternSource : public CFilePatternSource {
-/**
- * Synchronized concrete pattern source for a list of FASTA or CSFASTA
- * (if color = true) files.
- */
-class FastaPatternSource : public BufferedFilePatternSource {
public:
- FastaPatternSource(const EList<string>& infiles,
- const PatternParams& p) :
- BufferedFilePatternSource(infiles, p),
- first_(true)
- { }
+
+ FastaPatternSource(
+ const EList<std::string>& infiles,
+ const PatternParams& p) :
+ CFilePatternSource(infiles, p),
+ first_(true) { }
+
+ /**
+ * Reset so that next call to nextBatch* gets the first batch.
+ * Should only be called by the master thread.
+ */
virtual void reset() {
first_ = true;
- BufferedFilePatternSource::reset();
+ CFilePatternSource::reset();
}
+
+ /**
+ * Finalize FASTA parsing outside critical section.
+ */
+ virtual bool parse(Read& ra, Read& rb, TReadId rdid) const;
+
protected:
+
+ /**
+ * Light-parse a FASTA batch into the given buffer.
+ */
+ virtual std::pair<bool, int> nextBatchFromFile(
+ PerThreadReadBuf& pt,
+ bool batch_a);
+
/**
* Scan to the next FASTA record (starting with >) and return the first
* character of the record (which will always be >).
@@ -948,133 +425,54 @@ protected:
}
return c;
}
-
- /// Called when we have to bail without having parsed a read.
- void bail(Read& r) {
- r.reset();
- fb_.resetLastN();
- }
-
- /// Read another pattern from a FASTA input file
- virtual bool read(
- Read& r,
- TReadId& rdid,
- TReadId& endid,
- bool& success,
- bool& done);
-
- /// Read another pair of patterns from a FASTA input file
- virtual bool readPair(
- Read& ra,
- Read& rb,
- TReadId& rdid,
- TReadId& endid,
- bool& success,
- bool& done,
- bool& paired)
- {
- // (For now, we shouldn't ever be here)
- cerr << "In FastaPatternSource.readPair()" << endl;
- throw 1;
- return false;
- }
+ /**
+ * Reset state to handle a fresh file
+ */
virtual void resetForNextFile() {
first_ = true;
}
-private:
bool first_;
-};
-
-
-/**
- * Tokenize a line of space-separated integer quality values.
- */
-static inline bool tokenizeQualLine(
- FileBuf& filebuf,
- char *buf,
- size_t buflen,
- EList<string>& toks)
-{
- size_t rd = filebuf.gets(buf, buflen);
- if(rd == 0) return false;
- assert(NULL == strrchr(buf, '\n'));
- tokenize(string(buf), " ", toks);
- return true;
-}
+};
/**
* Synchronized concrete pattern source for a list of files with tab-
* delimited name, seq, qual fields (or, for paired-end reads,
* basename, seq1, qual1, seq2, qual2).
*/
-class TabbedPatternSource : public BufferedFilePatternSource {
+class TabbedPatternSource : public CFilePatternSource {
public:
TabbedPatternSource(
- const EList<string>& infiles,
+ const EList<std::string>& infiles,
const PatternParams& p,
- bool secondName) :
- BufferedFilePatternSource(infiles, p),
+ bool secondName) : // whether it's --12/--tab5 or --tab6
+ CFilePatternSource(infiles, p),
solQuals_(p.solexa64),
phred64Quals_(p.phred64),
intQuals_(p.intQuals),
secondName_(secondName) { }
-protected:
-
- /// Read another pattern from a FASTA input file
- virtual bool read(
- Read& r,
- TReadId& rdid,
- TReadId& endid,
- bool& success,
- bool& done);
-
- /// Read another pair of patterns from a FASTA input file
- virtual bool readPair(
- Read& ra,
- Read& rb,
- TReadId& rdid,
- TReadId& endid,
- bool& success,
- bool& done,
- bool& paired);
-
-private:
-
/**
- * Parse a name from fb_ and store in r. Assume that the next
- * character obtained via fb_.get() is the first character of
- * the sequence and the string stops at the next char upto (could
- * be tab, newline, etc.).
+ * Finalize tabbed parsing outside critical section.
*/
- int parseName(Read& r, Read* r2, char upto = '\t');
+ virtual bool parse(Read& ra, Read& rb, TReadId rdid) const;
- /**
- * Parse a single sequence from fb_ and store in r. Assume
- * that the next character obtained via fb_.get() is the first
- * character of the sequence and the sequence stops at the next
- * char upto (could be tab, newline, etc.).
- */
- int parseSeq(Read& r, int& charsRead, int& trim5, char upto = '\t');
+protected:
/**
- * Parse a single quality string from fb_ and store in r.
- * Assume that the next character obtained via fb_.get() is
- * the first character of the quality string and the string stops
- * at the next char upto (could be tab, newline, etc.).
+ * Light-parse a batch of tabbed-format reads into given buffer.
*/
- int parseQuals(Read& r, int charsRead, int dstLen, int trim5,
- char& c2, char upto = '\t', char upto2 = -1);
-
- bool solQuals_;
- bool phred64Quals_;
- bool intQuals_;
- EList<string> qualToks_;
- bool secondName_;
+ virtual std::pair<bool, int> nextBatchFromFile(
+ PerThreadReadBuf& pt,
+ bool batch_a);
+
+ bool solQuals_; // base qualities are log odds
+ bool phred64Quals_; // base qualities are on -64 scale
+ bool intQuals_; // base qualities are space-separated strings
+ bool secondName_; // true if --tab6, false if --tab5
};
/**
@@ -1096,243 +494,101 @@ private:
* 10. Quality
* 11. Filter: 1 = passed, 0 = didn't
*/
-class QseqPatternSource : public BufferedFilePatternSource {
+class QseqPatternSource : public CFilePatternSource {
public:
QseqPatternSource(
- const EList<string>& infiles,
- const PatternParams& p) :
- BufferedFilePatternSource(infiles, p),
+ const EList<std::string>& infiles,
+ const PatternParams& p) :
+ CFilePatternSource(infiles, p),
solQuals_(p.solexa64),
phred64Quals_(p.phred64),
intQuals_(p.intQuals) { }
-protected:
-
-#define BAIL_UNPAIRED() { \
- peekOverNewline(fb_); \
- r.reset(); \
- success = false; \
- done = true; \
- return success; \
-}
-
- /**
- * Parse a name from fb_ and store in r. Assume that the next
- * character obtained via fb_.get() is the first character of
- * the sequence and the string stops at the next char upto (could
- * be tab, newline, etc.).
- */
- int parseName(
- Read& r, // buffer for mate 1
- Read* r2, // buffer for mate 2 (NULL if mate2 is read separately)
- bool append, // true -> append characters, false -> skip them
- bool clearFirst, // clear the name buffer first
- bool warnEmpty, // emit a warning if nothing was added to the name
- bool useDefault, // if nothing is read, put readCnt_ as a default value
- int upto); // stop parsing when we first reach character 'upto'
-
- /**
- * Parse a single sequence from fb_ and store in r. Assume
- * that the next character obtained via fb_.get() is the first
- * character of the sequence and the sequence stops at the next
- * char upto (could be tab, newline, etc.).
- */
- int parseSeq(
- Read& r, // buffer for read
- int& charsRead,
- int& trim5,
- char upto);
-
/**
- * Parse a single quality string from fb_ and store in r.
- * Assume that the next character obtained via fb_.get() is
- * the first character of the quality string and the string stops
- * at the next char upto (could be tab, newline, etc.).
+ * Finalize qseq parsing outside critical section.
*/
- int parseQuals(
- Read& r, // buffer for read
- int charsRead,
- int dstLen,
- int trim5,
- char& c2,
- char upto,
- char upto2);
-
- /**
- * Read another pattern from a Qseq input file.
- */
- virtual bool read(
- Read& r,
- TReadId& rdid,
- TReadId& endid,
- bool& success,
- bool& done);
+ virtual bool parse(Read& ra, Read& rb, TReadId rdid) const;
+protected:
+
/**
- * Read a pair of patterns from 1 Qseq file. Note: this is never used.
+ * Light-parse a batch into the given buffer.
*/
- virtual bool readPair(
- Read& ra,
- Read& rb,
- TReadId& rdid,
- TReadId& endid,
- bool& success,
- bool& done,
- bool& paired)
- {
- // (For now, we shouldn't ever be here)
- cerr << "In QseqPatternSource.readPair()" << endl;
- throw 1;
- return false;
- }
+ virtual std::pair<bool, int> nextBatchFromFile(
+ PerThreadReadBuf& pt,
+ bool batch_a);
- bool solQuals_;
- bool phred64Quals_;
- bool intQuals_;
- EList<string> qualToks_;
+ bool solQuals_; // base qualities are log odds
+ bool phred64Quals_; // base qualities are on -64 scale
+ bool intQuals_; // base qualities are space-separated strings
+ EList<std::string> qualToks_;
};
/**
* Synchronized concrete pattern source for a list of FASTA files where
* reads need to be extracted from long continuous sequences.
*/
-class FastaContinuousPatternSource : public BufferedFilePatternSource {
+class FastaContinuousPatternSource : public CFilePatternSource {
public:
- FastaContinuousPatternSource(const EList<string>& infiles, const PatternParams& p) :
- BufferedFilePatternSource(infiles, p),
- length_(p.sampleLen), freq_(p.sampleFreq),
- eat_(length_-1), beginning_(true),
- bufCur_(0), subReadCnt_(0llu)
+ FastaContinuousPatternSource(
+ const EList<std::string>& infiles,
+ const PatternParams& p) :
+ CFilePatternSource(infiles, p),
+ length_(p.sampleLen),
+ freq_(p.sampleFreq),
+ eat_(length_-1),
+ beginning_(true),
+ bufCur_(0),
+ subReadCnt_(0llu)
{
+ assert_gt(freq_, 0);
resetForNextFile();
+ assert_leq(length_, 1024);
}
virtual void reset() {
- BufferedFilePatternSource::reset();
+ CFilePatternSource::reset();
resetForNextFile();
}
+ /**
+ * Finalize FASTA parsing outside critical section.
+ */
+ virtual bool parse(Read& ra, Read& rb, TReadId rdid) const;
+
protected:
- /// Read another pattern from a FASTA input file
- virtual bool read(
- Read& r,
- TReadId& rdid,
- TReadId& endid,
- bool& success,
- bool& done)
- {
- success = true;
- done = false;
- r.reset();
- while(true) {
- r.color = gColor;
- int c = fb_.get();
- if(c < 0) { success = false; done = true; return success; }
- if(c == '>') {
- resetForNextFile();
- c = fb_.peek();
- bool sawSpace = false;
- while(c != '\n' && c != '\r') {
- if(!sawSpace) {
- sawSpace = isspace(c);
- }
- if(!sawSpace) {
- nameBuf_.append(c);
- }
- fb_.get();
- c = fb_.peek();
- }
- while(c == '\n' || c == '\r') {
- fb_.get();
- c = fb_.peek();
- }
- nameBuf_.append('_');
- } else {
- int cat = asc2dnacat[c];
- if(cat >= 2) c = 'N';
- if(cat == 0) {
- // Encountered non-DNA, non-IUPAC char; skip it
- continue;
- } else {
- // DNA char
- buf_[bufCur_++] = c;
- if(bufCur_ == 1024) bufCur_ = 0;
- if(eat_ > 0) {
- eat_--;
- // Try to keep readCnt_ aligned with the offset
- // into the reference; that lets us see where
- // the sampling gaps are by looking at the read
- // name
- if(!beginning_) readCnt_++;
- continue;
- }
- for(size_t i = 0; i < length_; i++) {
- if(length_ - i <= bufCur_) {
- c = buf_[bufCur_ - (length_ - i)];
- } else {
- // Rotate
- c = buf_[bufCur_ - (length_ - i) + 1024];
- }
- r.patFw.append(asc2dna[c]);
- r.qual.append('I');
- }
- // Set up a default name if one hasn't been set
- r.name = nameBuf_;
- char cbuf[20];
- itoa10<TReadId>(readCnt_ - subReadCnt_, cbuf);
- r.name.append(cbuf);
- eat_ = freq_-1;
- readCnt_++;
- beginning_ = false;
- rdid = endid = readCnt_-1;
- break;
- }
- }
- }
- return true;
- }
+ /**
+ * Light-parse a batch into the given buffer.
+ */
+ virtual std::pair<bool, int> nextBatchFromFile(
+ PerThreadReadBuf& pt,
+ bool batch_a);
- /// Shouldn't ever be here; it's not sensible to obtain read pairs
- // from a continuous input.
- virtual bool readPair(
- Read& ra,
- Read& rb,
- TReadId& rdid,
- TReadId& endid,
- bool& success,
- bool& done,
- bool& paired)
- {
- cerr << "In FastaContinuousPatternSource.readPair()" << endl;
- throw 1;
- return false;
- }
-
/**
* Reset state to be read for the next file.
*/
virtual void resetForNextFile() {
eat_ = length_-1;
+ name_prefix_buf_.clear();
beginning_ = true;
bufCur_ = 0;
- nameBuf_.clear();
subReadCnt_ = readCnt_;
}
private:
- size_t length_; /// length of reads to generate
- size_t freq_; /// frequency to sample reads
+ const size_t length_; /// length of reads to generate
+ const size_t freq_; /// frequency to sample reads
size_t eat_; /// number of characters we need to skip before
/// we have flushed all of the ambiguous or
/// non-existent characters out of our read
/// window
bool beginning_; /// skipping over the first read length?
- char buf_[1024]; /// read buffer
- BTString nameBuf_; /// read buffer for name of fasta record being
- /// split into mers
+ char buf_[1024]; /// FASTA sequence buffer
+ Read::TBuf name_prefix_buf_; /// FASTA sequence name buffer
+ char name_int_buf_[20]; /// for composing offsets for names
size_t bufCur_; /// buffer cursor; points to where we should
/// insert the next character
uint64_t subReadCnt_;/// number to subtract from readCnt_ to get
@@ -1344,117 +600,49 @@ private:
* Read a FASTQ-format file.
* See: http://maq.sourceforge.net/fastq.shtml
*/
-class FastqPatternSource : public BufferedFilePatternSource {
+class FastqPatternSource : public CFilePatternSource {
public:
- FastqPatternSource(const EList<string>& infiles, const PatternParams& p) :
- BufferedFilePatternSource(infiles, p),
+ FastqPatternSource(
+ const EList<std::string>& infiles,
+ const PatternParams& p) :
+ CFilePatternSource(infiles, p),
first_(true),
solQuals_(p.solexa64),
phred64Quals_(p.phred64),
- intQuals_(p.intQuals),
- fuzzy_(p.fuzzy)
- { }
+ intQuals_(p.intQuals) { }
virtual void reset() {
first_ = true;
- fb_.resetLastN();
- BufferedFilePatternSource::reset();
+ CFilePatternSource::reset();
}
-
-protected:
/**
- * Scan to the next FASTQ record (starting with @) and return the first
- * character of the record (which will always be @). Since the quality
- * line may start with @, we keep scanning until we've seen a line
- * beginning with @ where the line two lines back began with +.
+ * Finalize FASTQ parsing outside critical section.
*/
- static int skipToNextFastqRecord(FileBuf& in, bool sawPlus) {
- int line = 0;
- int plusLine = -1;
- int c = in.get();
- int firstc = c;
- while(true) {
- if(line > 20) {
- // If we couldn't find our desired '@' in the first 20
- // lines, it's time to give up
- if(firstc == '>') {
- // That firstc is '>' may be a hint that this is
- // actually a FASTA file, so return it intact
- return '>';
- }
- // Return an error
- return -1;
- }
- if(c == -1) return -1;
- if(c == '\n') {
- c = in.get();
- if(c == '@' && sawPlus && plusLine == (line-2)) {
- return '@';
- }
- else if(c == '+') {
- // Saw a '+' at the beginning of a line; remember where
- // we saw it
- sawPlus = true;
- plusLine = line;
- }
- else if(c == -1) {
- return -1;
- }
- line++;
- }
- c = in.get();
- }
- }
+ virtual bool parse(Read& ra, Read& rb, TReadId rdid) const;
- /// Read another pattern from a FASTQ input file
- virtual bool read(
- Read& r,
- TReadId& rdid,
- TReadId& endid,
- bool& success,
- bool& done);
-
- /// Read another read pair from a FASTQ input file
- virtual bool readPair(
- Read& ra,
- Read& rb,
- TReadId& rdid,
- TReadId& endid,
- bool& success,
- bool& done,
- bool& paired)
- {
- // (For now, we shouldn't ever be here)
- cerr << "In FastqPatternSource.readPair()" << endl;
- throw 1;
- return false;
- }
-
- virtual void resetForNextFile() {
- first_ = true;
- }
-
-private:
+protected:
/**
- * Do things we need to do if we have to bail in the middle of a
- * read, usually because we reached the end of the input without
- * finishing.
+ * Light-parse a batch into the given buffer.
+ */
+ virtual std::pair<bool, int> nextBatchFromFile(
+ PerThreadReadBuf& pt,
+ bool batch_a);
+
+ /**
+ * Reset state to be ready for the next file.
*/
- void bail(Read& r) {
- r.patFw.clear();
- fb_.resetLastN();
+ virtual void resetForNextFile() {
+ first_ = true;
}
- bool first_;
- bool solQuals_;
- bool phred64Quals_;
- bool intQuals_;
- bool fuzzy_;
- EList<string> qualToks_;
+ bool first_; // parsing first read in file
+ bool solQuals_; // base qualities are log odds
+ bool phred64Quals_; // base qualities are on -64 scale
+ bool intQuals_; // base qualities are space-separated strings
};
/**
@@ -1462,148 +650,334 @@ private:
* allowed. All qualities are assumed to be 'I' (40 on the Phred-33
* scale).
*/
-class RawPatternSource : public BufferedFilePatternSource {
+class RawPatternSource : public CFilePatternSource {
public:
- RawPatternSource(const EList<string>& infiles, const PatternParams& p) :
- BufferedFilePatternSource(infiles, p), first_(true) { }
+ RawPatternSource(
+ const EList<std::string>& infiles,
+ const PatternParams& p) :
+ CFilePatternSource(infiles, p), first_(true) { }
virtual void reset() {
first_ = true;
- BufferedFilePatternSource::reset();
+ CFilePatternSource::reset();
+ }
+
+ /**
+ * Finalize raw parsing outside critical section.
+ */
+ virtual bool parse(Read& ra, Read& rb, TReadId rdid) const;
+
+protected:
+
+ /**
+ * Light-parse a batch into the given buffer.
+ */
+ virtual std::pair<bool, int> nextBatchFromFile(
+ PerThreadReadBuf& pt,
+ bool batch_a);
+
+ /**
+ * Reset state to be ready for the next file.
+ */
+ virtual void resetForNextFile() {
+ first_ = true;
}
+
+private:
+
+ bool first_;
+};
+/**
+ * Abstract parent class for synhconized sources of paired-end reads
+ * (and possibly also single-end reads).
+ */
+class PatternComposer {
+public:
+ PatternComposer(const PatternParams& p) : mutex_m() { }
+
+ virtual ~PatternComposer() { }
+
+ virtual void reset() = 0;
+
+ /**
+ * Member function override by concrete, format-specific classes.
+ */
+ virtual std::pair<bool, int> nextBatch(PerThreadReadBuf& pt) = 0;
+
+ /**
+ * Make appropriate call into the format layer to parse individual read.
+ */
+ virtual bool parse(Read& ra, Read& rb, TReadId rdid) = 0;
+
+ /**
+ * Given the values for all of the various arguments used to specify
+ * the read and quality input, create a list of pattern sources to
+ * dispense them.
+ */
+ static PatternComposer* setupPatternComposer(
+ const EList<std::string>& si, // singles, from argv
+ const EList<std::string>& m1, // mate1's, from -1 arg
+ const EList<std::string>& m2, // mate2's, from -2 arg
+ const EList<std::string>& m12, // both mates on each line, from --12
+ const EList<std::string>& q, // qualities associated with singles
+ const EList<std::string>& q1, // qualities associated with m1
+ const EList<std::string>& q2, // qualities associated with m2
+ const PatternParams& p, // read-in params
+ bool verbose); // be talkative?
+
protected:
+
+ static void free_EList_pmembers(const EList<PatternSource*>&);
+
+ /// Lock enforcing mutual exclusion for (a) file I/O, (b) writing fields
+ /// of this or another other shared object.
+ MUTEX_T mutex_m;
+};
- /// Read another pattern from a Raw input file
- virtual bool read(
- Read& r,
- TReadId& rdid,
- TReadId& endid,
- bool& success,
- bool& done)
+/**
+ * Encapsulates a synchronized source of both paired-end reads and
+ * unpaired reads, where the paired-end must come from parallel files.
+ */
+class SoloPatternComposer : public PatternComposer {
+
+public:
+
+ SoloPatternComposer(
+ const EList<PatternSource*>* src,
+ const PatternParams& p) :
+ PatternComposer(p),
+ cur_(0),
+ src_(src)
{
- int c;
- success = true;
- done = false;
- r.reset();
- c = getOverNewline(this->fb_);
- if(c < 0) {
- bail(r); success = false; done = true; return success;
+ assert(src_ != NULL);
+ for(size_t i = 0; i < src_->size(); i++) {
+ assert((*src_)[i] != NULL);
}
- assert(!isspace(c));
- r.color = gColor;
- int mytrim5 = gTrim5;
- if(first_) {
- // Check that the first character is sane for a raw file
- int cc = c;
- if(gColor) {
- if(cc >= '0' && cc <= '4') cc = "ACGTN"[(int)cc - '0'];
- if(cc == '.') cc = 'N';
- }
- if(asc2dnacat[cc] == 0) {
- cerr << "Error: reads file does not look like a Raw file" << endl;
- if(c == '>') {
- cerr << "Reads file looks like a FASTA file; please use -f" << endl;
- }
- if(c == '@') {
- cerr << "Reads file looks like a FASTQ file; please use -q" << endl;
- }
- throw 1;
- }
- first_ = false;
+ }
+
+ virtual ~SoloPatternComposer() {
+ free_EList_pmembers(*src_);
+ delete src_;
+ }
+
+ /**
+ * Reset this object and all the PatternSources under it so that
+ * the next call to nextBatch gets the very first read.
+ */
+ virtual void reset() {
+ for(size_t i = 0; i < src_->size(); i++) {
+ (*src_)[i]->reset();
}
- if(gColor) {
- // This may be a primer character. If so, keep it in the
- // 'primer' field of the read buf and parse the rest of the
- // read without it.
- c = toupper(c);
- if(asc2dnacat[c] > 0) {
- // First char is a DNA char
- int c2 = toupper(fb_.peek());
- // Second char is a color char
- if(asc2colcat[c2] > 0) {
- r.primer = c;
- r.trimc = c2;
- mytrim5 += 2; // trim primer and first color
- }
- }
- if(c < 0) {
- bail(r); success = false; done = true; return success;
+ cur_ = 0;
+ }
+
+ /**
+ * Calls member functions of the individual PatternSource objects
+ * to get more reads. Since there's no need to keep two separate
+ * files in sync (as there is for DualPatternComposer),
+ * synchronization can be handed by the PatternSource contained
+ * in the src_ field.
+ */
+ virtual std::pair<bool, int> nextBatch(PerThreadReadBuf& pt);
+
+ /**
+ * Make appropriate call into the format layer to parse individual read.
+ */
+ virtual bool parse(Read& ra, Read& rb, TReadId rdid) {
+ return (*src_)[0]->parse(ra, rb, rdid);
+ }
+
+protected:
+ volatile size_t cur_; // current element in parallel srca_, srcb_ vectors
+ const EList<PatternSource*>* src_; /// PatternSources for paired-end reads
+};
+
+/**
+ * Encapsulates a synchronized source of both paired-end reads and
+ * unpaired reads, where the paired-end must come from parallel files.
+ */
+class DualPatternComposer : public PatternComposer {
+
+public:
+
+ DualPatternComposer(
+ const EList<PatternSource*>* srca,
+ const EList<PatternSource*>* srcb,
+ const PatternParams& p) :
+ PatternComposer(p), cur_(0), srca_(srca), srcb_(srcb)
+ {
+ assert(srca_ != NULL);
+ assert(srcb_ != NULL);
+ // srca_ and srcb_ must be parallel
+ assert_eq(srca_->size(), srcb_->size());
+ for(size_t i = 0; i < srca_->size(); i++) {
+ // Can't have NULL first-mate sources. Second-mate sources
+ // can be NULL, in the case when the corresponding first-
+ // mate source is unpaired.
+ assert((*srca_)[i] != NULL);
+ for(size_t j = 0; j < srcb_->size(); j++) {
+ assert_neq((*srca_)[i], (*srcb_)[j]);
}
}
- // _in now points just past the first character of a sequence
- // line, and c holds the first character
- int chs = 0;
- while(!isspace(c) && c >= 0) {
- if(gColor) {
- if(c >= '0' && c <= '4') c = "ACGTN"[(int)c - '0'];
- if(c == '.') c = 'N';
- }
- // 5' trimming
- if(isalpha(c) && chs >= mytrim5) {
- //size_t len = chs - mytrim5;
- //if(len >= 1024) tooManyQualities(BTString("(no name)"));
- r.patFw.append(asc2dna[c]);
- r.qual.append('I');
+ }
+
+ virtual ~DualPatternComposer() {
+ free_EList_pmembers(*srca_);
+ delete srca_;
+ free_EList_pmembers(*srcb_);
+ delete srcb_;
+ }
+
+ /**
+ * Reset this object and all the PatternSources under it so that
+ * the next call to nextBatch gets the very first read.
+ */
+ virtual void reset() {
+ for(size_t i = 0; i < srca_->size(); i++) {
+ (*srca_)[i]->reset();
+ if((*srcb_)[i] != NULL) {
+ (*srcb_)[i]->reset();
}
- chs++;
- if(isspace(fb_.peek())) break;
- c = fb_.get();
}
- // 3' trimming
- r.patFw.trimEnd(gTrim3);
- r.qual.trimEnd(gTrim3);
- c = peekToEndOfLine(fb_);
- r.trimmed3 = gTrim3;
- r.trimmed5 = mytrim5;
- r.readOrigBuf.install(fb_.lastN(), fb_.lastNLen());
- fb_.resetLastN();
-
- // Set up name
- char cbuf[20];
- itoa10<TReadId>(readCnt_, cbuf);
- r.name.install(cbuf);
- readCnt_++;
-
- rdid = endid = readCnt_-1;
- return success;
+ cur_ = 0;
}
- /// Read another read pair from a FASTQ input file
- virtual bool readPair(
- Read& ra,
- Read& rb,
- TReadId& rdid,
- TReadId& endid,
- bool& success,
- bool& done,
- bool& paired)
- {
- // (For now, we shouldn't ever be here)
- cerr << "In RawPatternSource.readPair()" << endl;
- throw 1;
- return false;
+ /**
+ * Calls member functions of the individual PatternSource objects
+ * to get more reads. Since we need to keep the two separate
+ * files in sync, synchronization can be handed at this level, with
+ * one critical section surrounding both calls into the srca_ and
+ * srcb_ member functions.
+ */
+ virtual std::pair<bool, int> nextBatch(PerThreadReadBuf& pt);
+
+ /**
+ * Make appropriate call into the format layer to parse individual read.
+ */
+ virtual bool parse(Read& ra, Read& rb, TReadId rdid) {
+ return (*srca_)[0]->parse(ra, rb, rdid);
}
+
+protected:
- virtual void resetForNextFile() {
- first_ = true;
- }
+ volatile size_t cur_; // current element in parallel srca_, srcb_ vectors
+ const EList<PatternSource*>* srca_; // for 1st matesunpaired
+ const EList<PatternSource*>* srcb_; // for 2nd mates
+};
+
+/**
+ * Encapsulates a single thread's interaction with the PatternSource.
+ * Most notably, this class holds the buffers into which the
+ * PatterSource will write sequences. This class is *not* threadsafe
+ * - it doesn't need to be since there's one per thread. PatternSource
+ * is thread-safe.
+ */
+class PatternSourcePerThread {
+
+public:
+
+ PatternSourcePerThread(
+ PatternComposer& composer,
+ const PatternParams& pp) :
+ composer_(composer),
+ buf_(pp.max_buf),
+ pp_(pp),
+ last_batch_(false),
+ last_batch_size_(0) { }
+
+ /**
+ * Use objects in the PatternSource and/or PatternComposer
+ * hierarchies to populate the per-thread buffers.
+ */
+ std::pair<bool, bool> nextReadPair();
+
+ Read& read_a() { return buf_.read_a(); }
+ Read& read_b() { return buf_.read_b(); }
+
+ const Read& read_a() const { return buf_.read_a(); }
+ const Read& read_b() const { return buf_.read_b(); }
private:
+
+ /**
+ * When we've finished fully parsing and dishing out reads in
+ * the current batch, we go get the next one by calling into
+ * the composition layer.
+ */
+ std::pair<bool, int> nextBatch() {
+ buf_.reset();
+ std::pair<bool, int> res = composer_.nextBatch(buf_);
+ buf_.init();
+ return res;
+ }
+
+ /**
+ * Once name/sequence/qualities have been parsed for an
+ * unpaired read, set all the other key fields of the Read
+ * struct.
+ */
+ void finalize(Read& ra);
/**
- * Do things we need to do if we have to bail in the middle of a
- * read, usually because we reached the end of the input without
- * finishing.
+ * Once name/sequence/qualities have been parsed for a
+ * paired-end read, set all the other key fields of the Read
+ * structs.
+ */
+ void finalizePair(Read& ra, Read& rb);
+
+ /**
+ * Call into composition layer (which in turn calls into
+ * format layer) to parse the read.
*/
- void bail(Read& r) {
- r.patFw.clear();
- fb_.resetLastN();
+ bool parse(Read& ra, Read& rb) {
+ return composer_.parse(ra, rb, buf_.rdid());
}
+
+ PatternComposer& composer_; // pattern composer
+ PerThreadReadBuf buf_; // read data buffer
+ const PatternParams& pp_; // pattern-related parameters
+ bool last_batch_; // true if this is final batch
+ int last_batch_size_; // # reads read in previous batch
+};
+
+/**
+ * Abstract parent factory for PatternSourcePerThreads.
+ */
+class PatternSourcePerThreadFactory {
+public:
+ PatternSourcePerThreadFactory(
+ PatternComposer& composer,
+ const PatternParams& pp) :
+ composer_(composer),
+ pp_(pp) { }
- bool first_;
+ /**
+ * Create a new heap-allocated PatternSourcePerThreads.
+ */
+ virtual PatternSourcePerThread* create() const {
+ return new PatternSourcePerThread(composer_, pp_);
+ }
+
+ /**
+ * Create a new heap-allocated vector of heap-allocated
+ * PatternSourcePerThreads.
+ */
+ virtual EList<PatternSourcePerThread*>* create(uint32_t n) const {
+ EList<PatternSourcePerThread*>* v = new EList<PatternSourcePerThread*>;
+ for(size_t i = 0; i < n; i++) {
+ v->push_back(new PatternSourcePerThread(composer_, pp_));
+ assert(v->back() != NULL);
+ }
+ return v;
+ }
+
+private:
+ /// Container for obtaining paired reads from PatternSources
+ PatternComposer& composer_;
+ const PatternParams& pp_;
};
#endif /*PAT_H_*/
diff --git a/pthreadGC2.dll b/pthreadGC2.dll
new file mode 100644
index 0000000..8b9116c
Binary files /dev/null and b/pthreadGC2.dll differ
diff --git a/read.h b/read.h
index 7fe26ef..15c721f 100644
--- a/read.h
+++ b/read.h
@@ -38,14 +38,14 @@ class HitSet;
*/
struct Read {
+ typedef SStringExpandable<char, 1024, 2, 1024> TBuf;
+
Read() { reset(); }
Read(const char *nm, const char *seq, const char *ql) { init(nm, seq, ql); }
void reset() {
rdid = 0;
- endid = 0;
- alts = 0;
trimmed5 = trimmed3 = 0;
readOrigBuf.clear();
patFw.clear();
@@ -55,19 +55,9 @@ struct Read {
patRcRev.clear();
qualRev.clear();
name.clear();
- for(int j = 0; j < 3; j++) {
- altPatFw[j].clear();
- altPatFwRev[j].clear();
- altPatRc[j].clear();
- altPatRcRev[j].clear();
- altQual[j].clear();
- altQualRev[j].clear();
- }
- color = fuzzy = false;
- primer = '?';
- trimc = '?';
filter = '?';
seed = 0;
+ parsed = false;
ns_ = 0;
}
@@ -124,21 +114,10 @@ struct Read {
/**
* Construct reverse complement of the pattern and the fuzzy
- * alternative patters. If read is in colorspace, just reverse
- * them.
+ * alternative patters.
*/
void constructRevComps() {
- if(color) {
- patRc.installReverse(patFw);
- for(int j = 0; j < alts; j++) {
- altPatRc[j].installReverse(altPatFw[j]);
- }
- } else {
- patRc.installReverseComp(patFw);
- for(int j = 0; j < alts; j++) {
- altPatRc[j].installReverseComp(altPatFw[j]);
- }
- }
+ patRc.installReverseComp(patFw);
}
/**
@@ -149,11 +128,6 @@ struct Read {
patFwRev.installReverse(patFw);
patRcRev.installReverse(patRc);
qualRev.installReverse(qual);
- for(int j = 0; j < alts; j++) {
- altPatFwRev[j].installReverse(altPatFw[j]);
- altPatRcRev[j].installReverse(altPatRc[j]);
- altQualRev[j].installReverse(altQual[j]);
- }
}
/**
@@ -192,55 +166,9 @@ struct Read {
void dump(std::ostream& os) const {
using namespace std;
os << name << ' ';
- if(color) {
- os << patFw.toZBufXForm("0123.");
- } else {
- os << patFw;
- }
+ os << patFw;
os << ' ';
- // Print out the fuzzy alternative sequences
- for(int j = 0; j < 3; j++) {
- bool started = false;
- if(!altQual[j].empty()) {
- for(size_t i = 0; i < length(); i++) {
- if(altQual[j][i] != '!') {
- started = true;
- }
- if(started) {
- if(altQual[j][i] == '!') {
- os << '-';
- } else {
- if(color) {
- os << "0123."[(int)altPatFw[j][i]];
- } else {
- os << altPatFw[j][i];
- }
- }
- }
- }
- }
- cout << " ";
- }
os << qual.toZBuf() << " ";
- // Print out the fuzzy alternative quality strings
- for(int j = 0; j < 3; j++) {
- bool started = false;
- if(!altQual[j].empty()) {
- for(size_t i = 0; i < length(); i++) {
- if(altQual[j][i] != '!') {
- started = true;
- }
- if(started) {
- os << altQual[j][i];
- }
- }
- }
- if(j == 2) {
- os << endl;
- } else {
- os << " ";
- }
- }
}
/**
@@ -318,33 +246,19 @@ struct Read {
BTDnaString patRc; // reverse-complement sequence
BTString qual; // quality values
- BTDnaString altPatFw[3];
- BTDnaString altPatRc[3];
- BTString altQual[3];
-
BTDnaString patFwRev;
BTDnaString patRcRev;
BTString qualRev;
- BTDnaString altPatFwRev[3];
- BTDnaString altPatRcRev[3];
- BTString altQualRev[3];
-
// For remembering the exact input text used to define a read
- SStringExpandable<char> readOrigBuf;
+ TBuf readOrigBuf;
BTString name; // read name
TReadId rdid; // 0-based id based on pair's offset in read file(s)
- TReadId endid; // 0-based id based on pair's offset in read file(s)
- // and which mate ("end") this is
int mate; // 0 = single-end, 1 = mate1, 2 = mate2
uint32_t seed; // random seed
+ bool parsed; // true iff read has been fully parsed
size_t ns_; // # Ns
- int alts; // number of alternatives
- bool fuzzy; // whether to employ fuzziness
- bool color; // whether read is in color space
- char primer; // primer base, for csfasta files
- char trimc; // trimmed color, for csfasta files
char filter; // if read format permits filter char, set it here
int trimmed5; // amount actually trimmed off 5' end
int trimmed3; // amount actually trimmed off 3' end
diff --git a/read_qseq.cpp b/read_qseq.cpp
index ced6c68..be1e868 100644
--- a/read_qseq.cpp
+++ b/read_qseq.cpp
@@ -25,280 +25,209 @@
* the sequence and the string stops at the next char upto (could
* be tab, newline, etc.).
*/
-int QseqPatternSource::parseName(
- Read& r, // buffer for mate 1
- Read* r2, // buffer for mate 2 (NULL if mate2 is read separately)
- bool append, // true -> append characters, false -> skip them
- bool clearFirst, // clear the name buffer first
- bool warnEmpty, // emit a warning if nothing was added to the name
- bool useDefault, // if nothing is read, put readCnt_ as a default value
+static int parseName(
+ Read::TBuf& buf, // buffer w/ raw qseq data
+ size_t& cur, // buffer cursor
+ Read& r, // buffer for mate 1
int upto) // stop parsing when we first reach character 'upto'
{
- if(clearFirst) {
- if(r2 != NULL) r2->name.clear();
- r.name.clear();
- }
- while(true) {
- int c;
- if((c = fb_.get()) < 0) {
- // EOF reached in the middle of the name
- return -1;
- }
- if(c == '\n' || c == '\r') {
- // EOL reached in the middle of the name
- return -1;
- }
+ const size_t buflen = buf.length();
+ int c;
+ while(cur < buflen) {
+ c = buf[cur++];
+ assert(c != '\r' && c != '\n');
if(c == upto) {
- // Finished with field
- break;
- }
- if(append) {
- if(r2 != NULL) r2->name.append(c);
- r.name.append(c);
+ break; // Finished with field
}
+ r.name.append(c);
}
- // Set up a default name if one hasn't been set
- if(r.name.empty() && useDefault && append) {
- char cbuf[20];
- itoa10(readCnt_, cbuf);
- r.name.append(cbuf);
- if(r2 != NULL) r2->name.append(cbuf);
- }
- if(r.name.empty() && warnEmpty) {
- cerr << "Warning: read had an empty name field" << endl;
+ if(cur >= buflen) {
+ return -1; // Error: buffer ended prematurely
}
return (int)r.name.length();
}
+
/**
- * Parse a single sequence from fb_ and store in r. Assume
- * that the next character obtained via fb_.get() is the first
- * character of the sequence and the sequence stops at the next
- * char upto (could be tab, newline, etc.).
+ * Read another pattern from a Qseq input file.
*/
-int QseqPatternSource::parseSeq(
- Read& r,
- int& charsRead,
- int& trim5,
- char upto)
+pair<bool, int> QseqPatternSource::nextBatchFromFile(
+ PerThreadReadBuf& pt,
+ bool batch_a)
{
- int begin = 0;
- int c = fb_.get();
- assert(c != upto);
- r.patFw.clear();
- r.color = gColor;
- if(gColor) {
- // NOTE: clearly this is not relevant for Illumina output, but
- // I'm keeping it here in case there's some reason to put SOLiD
- // data in this format in the future.
-
- // This may be a primer character. If so, keep it in the
- // 'primer' field of the read buf and parse the rest of the
- // read without it.
- c = toupper(c);
- if(asc2dnacat[c] > 0) {
- // First char is a DNA char
- int c2 = toupper(fb_.peek());
- // Second char is a color char
- if(asc2colcat[c2] > 0) {
- r.primer = c;
- r.trimc = c2;
- trim5 += 2; // trim primer and first color
- }
- }
- if(c < 0) { return -1; }
+ int c = getc_unlocked(fp_);
+ while(c >= 0 && (c == '\n' || c == '\r')) {
+ c = getc_unlocked(fp_);
}
- while(c != upto) {
- if(c == '.') c = 'N';
- if(gColor) {
- if(c >= '0' && c <= '4') c = "ACGTN"[(int)c - '0'];
- }
- if(isalpha(c)) {
- assert_in(toupper(c), "ACGTN");
- if(begin++ >= trim5) {
- assert_neq(0, asc2dnacat[c]);
- r.patFw.append(asc2dna[c]);
- }
- charsRead++;
+ EList<Read>& readbuf = batch_a ? pt.bufa_ : pt.bufb_;
+ size_t readi = 0;
+ // Read until we run out of input or until we've filled the buffer
+ for(; readi < pt.max_buf_ && c >= 0; readi++) {
+ readbuf[readi].readOrigBuf.clear();
+ while(c >= 0 && c != '\n' && c != '\r') {
+ readbuf[readi].readOrigBuf.append(c);
+ c = getc_unlocked(fp_);
}
- if((c = fb_.get()) < 0) {
- return -1;
+ while(c >= 0 && (c == '\n' || c == '\r')) {
+ c = getc_unlocked(fp_);
}
}
- r.patFw.trimEnd(gTrim3);
- return (int)r.patFw.length();
+ return make_pair(c < 0, readi);
}
/**
- * Parse a single quality string from fb_ and store in r.
- * Assume that the next character obtained via fb_.get() is
- * the first character of the quality string and the string stops
- * at the next char upto (could be tab, newline, etc.).
- */
-int QseqPatternSource::parseQuals(
- Read& r,
- int charsRead,
- int dstLen,
- int trim5,
- char& c2,
- char upto = '\t',
- char upto2 = -1)
-{
- int qualsRead = 0;
- int c = 0;
- if (intQuals_) {
- // Probably not relevant
- char buf[4096];
- while (qualsRead < charsRead) {
- qualToks_.clear();
- if(!tokenizeQualLine(fb_, buf, 4096, qualToks_)) break;
- for (unsigned int j = 0; j < qualToks_.size(); ++j) {
- char c = intToPhred33(atoi(qualToks_[j].c_str()), solQuals_);
- assert_geq(c, 33);
- if (qualsRead >= trim5) {
- r.qual.append(c);
- }
- ++qualsRead;
- }
- } // done reading integer quality lines
- if (charsRead > qualsRead) tooFewQualities(r.name);
- } else {
- // Non-integer qualities
- while((qualsRead < dstLen + trim5) && c >= 0) {
- c = fb_.get();
- c2 = c;
- if (c == ' ') wrongQualityFormat(r.name);
- if(c < 0) {
- // EOF occurred in the middle of a read - abort
- return -1;
- }
- if(!isspace(c) && c != upto && (upto2 == -1 || c != upto2)) {
- if (qualsRead >= trim5) {
- c = charToPhred33(c, solQuals_, phred64Quals_);
- assert_geq(c, 33);
- r.qual.append(c);
- }
- qualsRead++;
- } else {
- break;
- }
- }
- }
- if(r.qual.length() < (size_t)dstLen) {
- tooFewQualities(r.name);
- }
- // TODO: How to detect too many qualities??
- r.qual.resize(dstLen);
- while(c != -1 && c != upto && (upto2 == -1 || c != upto2)) {
- c = fb_.get();
- c2 = c;
- }
- return qualsRead;
-}
-
-/**
- * Read another pattern from a Qseq input file.
+ *
*/
-bool QseqPatternSource::read(
- Read& r,
- TReadId& rdid,
- TReadId& endid,
- bool& success,
- bool& done)
-{
- r.reset();
- r.color = gColor;
- success = true;
- done = false;
- readCnt_++;
- rdid = endid = readCnt_-1;
- peekOverNewline(fb_);
- fb_.resetLastN();
+bool QseqPatternSource::parse(Read& r, Read& rb, TReadId rdid) const {
+ // Light parser (nextBatchFromFile) puts unparsed data
+ // into Read& r, even when the read is paired.
+ assert(r.empty());
+ assert(!r.readOrigBuf.empty()); // raw data for read/pair is here
+ int c = '\t';
+ size_t cur = 0;
+ const size_t buflen = r.readOrigBuf.length();
+ assert(r.name.empty());
+
// 1. Machine name
- if(parseName(r, NULL, true, true, true, false, '\t') == -1) BAIL_UNPAIRED();
- assert_neq('\t', fb_.peek());
+ if(parseName(r.readOrigBuf, cur, r, '\t') == -1) {
+ return false;
+ }
r.name.append('_');
// 2. Run number
- if(parseName(r, NULL, true, false, true, false, '\t') == -1) BAIL_UNPAIRED();
- assert_neq('\t', fb_.peek());
+ if(parseName(r.readOrigBuf, cur, r, '\t') == -1) {
+ return false;
+ }
r.name.append('_');
// 3. Lane number
- if(parseName(r, NULL, true, false, true, false, '\t') == -1) BAIL_UNPAIRED();
- assert_neq('\t', fb_.peek());
+ if(parseName(r.readOrigBuf, cur, r, '\t') == -1) {
+ return false;
+ }
r.name.append('_');
// 4. Tile number
- if(parseName(r, NULL, true, false, true, false, '\t') == -1) BAIL_UNPAIRED();
- assert_neq('\t', fb_.peek());
+ if(parseName(r.readOrigBuf, cur, r, '\t') == -1) {
+ return false;
+ }
r.name.append('_');
// 5. X coordinate of spot
- if(parseName(r, NULL, true, false, true, false, '\t') == -1) BAIL_UNPAIRED();
- assert_neq('\t', fb_.peek());
+ if(parseName(r.readOrigBuf, cur, r, '\t') == -1) {
+ return false;
+ }
r.name.append('_');
// 6. Y coordinate of spot
- if(parseName(r, NULL, true, false, true, false, '\t') == -1) BAIL_UNPAIRED();
- assert_neq('\t', fb_.peek());
+ if(parseName(r.readOrigBuf, cur, r, '\t') == -1) {
+ return false;
+ }
r.name.append('_');
// 7. Index
- if(parseName(r, NULL, true, false, true, false, '\t') == -1) BAIL_UNPAIRED();
- assert_neq('\t', fb_.peek());
+ if(parseName(r.readOrigBuf, cur, r, '\t') == -1) {
+ return false;
+ }
r.name.append('/');
// 8. Mate number
- if(parseName(r, NULL, true, false, true, false, '\t') == -1) BAIL_UNPAIRED();
- // Empty sequence??
- if(fb_.peek() == '\t') {
- // Get tab that separates seq from qual
- ASSERT_ONLY(int c =) fb_.get();
- assert_eq('\t', c);
- assert_eq('\t', fb_.peek());
- // Get tab that separates qual from filter
- ASSERT_ONLY(c =) fb_.get();
+ if(parseName(r.readOrigBuf, cur, r, '\t') == -1) {
+ return false;
+ }
+ if(cur >= buflen) {
+ return false; // ended prematurely
+ }
+ c = r.readOrigBuf[cur++];
+ assert(c != '\r' && c != '\n');
+ // 9. Sequence & 10. Qualities
+ if(c == '\t') {
+ // empty sequence & qualities
+ c = r.readOrigBuf[cur++];
+ assert(c != '\r' && c != '\n');
assert_eq('\t', c);
- // Next char is first char of filter flag
- assert_neq('\t', fb_.peek());
- fb_.resetLastN();
cerr << "Warning: skipping empty QSEQ read with name '" << r.name << "'" << endl;
} else {
- assert_neq('\t', fb_.peek());
- int charsRead = 0;
- int mytrim5 = gTrim5;
// 9. Sequence
- int dstLen = parseSeq(r, charsRead, mytrim5, '\t');
- assert_neq('\t', fb_.peek());
- if(dstLen < 0) BAIL_UNPAIRED();
- char ct = 0;
+ int nchar = 0;
+ while(c != '\t') {
+ if(c == '.') {
+ c = 'N';
+ }
+ if(isalpha(c)) {
+ assert_in(toupper(c), "ACGTN");
+ if(++nchar > gTrim5) {
+ assert_neq(0, asc2dnacat[c]);
+ r.patFw.append(asc2dna[c]);
+ }
+ }
+ if(cur >= buflen) {
+ break;
+ }
+ c = r.readOrigBuf[cur++];
+ }
+ if(cur >= buflen) {
+ return false; // ended prematurely
+ }
+ // record amt trimmed from 5' end due to --trim5
+ r.trimmed5 = (int)(nchar - r.patFw.length());
+ // record amt trimmed from 3' end due to --trim3
+ r.trimmed3 = (int)(r.patFw.trimEnd(gTrim3));
+
// 10. Qualities
- if(parseQuals(r, charsRead, dstLen, mytrim5, ct, '\t', -1) < 0) BAIL_UNPAIRED();
- r.trimmed3 = gTrim3;
- r.trimmed5 = mytrim5;
- if(ct != '\t') {
- cerr << "Error: QSEQ with name " << r.name << " did not have tab after qualities" << endl;
- throw 1;
+ assert(r.qual.empty());
+ int nqual = 0;
+ if (intQuals_) {
+ int cur_int = 0;
+ while(c != '\t') {
+ cur_int *= 10;
+ cur_int += (int)(c - '0');
+ c = r.readOrigBuf[cur++];
+ assert(c != '\r' && c != '\n');
+ if(c == ' ' || c == '\t') {
+ char cadd = intToPhred33(cur_int, solQuals_);
+ cur_int = 0;
+ assert_geq(cadd, 33);
+ if(++nqual > gTrim5) {
+ r.qual.append(cadd);
+ }
+ }
+ }
+ } else {
+ while(cur < buflen) {
+ c = r.readOrigBuf[cur++];
+ assert(c != '\r' && c != '\n');
+ if (c == ' ') {
+ wrongQualityFormat(r.name);
+ return false;
+ } else if(c == '\t') {
+ break;
+ }
+ c = charToPhred33(c, solQuals_, phred64Quals_);
+ if(++nqual > r.trimmed5) {
+ r.qual.append(c);
+ }
+ }
+ r.qual.trimEnd(r.trimmed3);
+ if(r.qual.length() < r.patFw.length()) {
+ tooFewQualities(r.name);
+ return false;
+ } else if(r.qual.length() > r.patFw.length()) {
+ tooManyQualities(r.name);
+ return false;
+ }
}
- assert_eq(ct, '\t');
}
+ assert_eq('\t', c);
+
// 11. Filter flag
- int filt = fb_.get();
- if(filt == -1) BAIL_UNPAIRED();
+ if(cur >= buflen) {
+ return false;
+ }
+ int filt = r.readOrigBuf[cur++];
r.filter = filt;
if(filt != '0' && filt != '1') {
// Bad value for filt
+ cerr << "Error: Bad value '" << filt
+ << "' for qseq filter flag" << endl;
+ throw 1;
}
- if(fb_.peek() != -1 && fb_.peek() != '\n') {
- // Bad value right after the filt field
- }
- fb_.get();
- r.readOrigBuf.install(fb_.lastN(), fb_.lastNLen());
- fb_.resetLastN();
- if(r.qual.length() < r.patFw.length()) {
- tooFewQualities(r.name);
- } else if(r.qual.length() > r.patFw.length()) {
- tooManyQualities(r.name);
- }
-#ifndef NDEBUG
- assert_eq(r.patFw.length(), r.qual.length());
- for(size_t i = 0; i < r.qual.length(); i++) {
- assert_geq((int)r.qual[i], 33);
+ assert_eq(cur, buflen);
+ r.parsed = true;
+ if(!rb.parsed && !rb.readOrigBuf.empty()) {
+ return parse(rb, r, rdid);
}
-#endif
return true;
}
diff --git a/ref_read.cpp b/ref_read.cpp
index 40dd454..abf4914 100644
--- a/ref_read.cpp
+++ b/ref_read.cpp
@@ -84,26 +84,12 @@ RefRecord fastaRefReadSize(
// Now skip to the first DNA character, counting gap characters
// as we go
- int lc = -1; // last-DNA char variable for color conversion
+ int lc = -1;
while(true) {
int cat = asc2dnacat[c];
if(rparms.nsToAs && cat >= 2) c = 'A';
if(cat == 1) {
- // This is a DNA character
- if(rparms.color) {
- if(lc != -1) {
- // Got two consecutive unambiguous DNAs
- break; // to read-in loop
- }
- // Keep going; we need two consecutive unambiguous DNAs
- lc = asc2dna[(int)c];
- // The 'if(off > 0)' takes care of the case where
- // the reference is entirely unambiguous and we don't
- // want to incorrectly increment off.
- if(off > 0) off++;
- } else {
- break; // to read-in loop
- }
+ break; // to read-in loop
} else if(cat >= 2) {
if(lc != -1 && off == 0) off++;
lc = -1;
@@ -131,14 +117,7 @@ RefRecord fastaRefReadSize(
return RefRecord((TIndexOffU)off, 0, first);
}
}
- assert(!rparms.color || (lc != -1));
assert_eq(1, asc2dnacat[c]); // C must be unambiguous base
- if(off > 0 && rparms.color && first) {
- // Handle the case where the first record has ambiguous
- // characters but we're in color space; one of those counts is
- // spurious
- off--;
- }
// in now points just past the first character of a sequence
// line, and c holds the first character
@@ -158,13 +137,8 @@ RefRecord fastaRefReadSize(
len++;
// Output it
if(bpout != NULL) {
- if(rparms.color) {
- // output color
- bpout->write(dinuc2color[asc2dna[(int)c]][lc]);
- } else if(!rparms.color) {
- // output nucleotide
- bpout->write(asc2dna[c]);
- }
+ // output nucleotide
+ bpout->write(asc2dna[c]);
}
lc = asc2dna[(int)c];
} else if(cat >= 2) {
diff --git a/sam.cpp b/sam.cpp
index 278b8be..ca9ca3e 100644
--- a/sam.cpp
+++ b/sam.cpp
@@ -135,22 +135,6 @@ void SamConfig::printAlignedOptFlags(
{
char buf[1024];
assert(summ.best(rd.mate < 2).valid());
- if(print_zt_) {
- // ZT:Z: Extra features for MAPQ estimation
- WRITE_SEP();
- o.append("ZT:Z:");
- itoa10<TAlScore>((int)(prm.seedsPerNuc * 1000), buf);
- o.append(buf);
- o.append(",");
- itoa10<TAlScore>((int)(prm.seedPctUnique * 1000), buf);
- o.append(buf);
- o.append(",");
- itoa10<TAlScore>((int)(prm.seedPctRep * 1000), buf);
- o.append(buf);
- o.append(",");
- itoa10<TAlScore>((int)(prm.seedHitAvg + 0.5), buf);
- o.append(buf);
- }
if(print_as_) {
// AS:i: Alignment score generated by aligner
itoa10<TAlScore>(res.score().score(), buf);
@@ -535,6 +519,110 @@ void SamConfig::printAlignedOptFlags(
o.append("\n");
printOptFieldNewlineEscapedZ(o, rd.readOrigBuf);
}
+ if(print_zt_) {
+ // ZT:Z: Extra features for MAPQ estimation
+
+ // 1. AS:i for current mate
+ // 2. diff or NA for current mate
+ // 3. Like 1 for opposite mate
+ // 4. Like 2 for opposite mate
+
+ WRITE_SEP();
+ const bool paired = flags.partOfPair();
+ const TAlScore MN = std::numeric_limits<TAlScore>::min();
+ TAlScore secondBest[2] = {MN, MN};
+ TAlScore thirdBest[2] = {MN, MN};
+ {
+ for (int self = 0; self < (paired ? 2 : 1); self++) {
+ // Second-best
+ AlnScore sco;
+ bool mate1 = rd.mate < 2;
+ if(self > 0) mate1 = !mate1;
+ if(flags.alignedConcordant()) {
+ sco = summ.bestUnchosen(mate1);
+ } else {
+ sco = summ.secbest(mate1);
+ }
+ if(sco.valid()) {
+ secondBest[self] = sco.score();
+ }
+
+ // Third-best
+ thirdBest[self] = mate1 ? prm.bestLtMinscMate1 : prm.bestLtMinscMate2;
+ }
+ }
+ TAlScore best[2] = {res.score().score(), res.oscore().score()};
+ TAlScore diff[2] = {MN, MN};
+ for(int self = 0; self < 2; self++) {
+ const TAlScore mx = max(secondBest[self], thirdBest[self]);
+ if(best[self] > MN && mx > MN) {
+ diff[self] = best[self] - mx;
+ }
+ }
+ TAlScore best_conc = MN, diff_conc = MN;
+ if(paired && summ.bestPaired().valid()) {
+ best_conc = summ.bestPaired().score();
+ if(summ.secbestPaired().valid()) {
+ diff_conc = best_conc - summ.secbestPaired().score();
+ }
+ }
+ o.append("ZT:Z:");
+ // AS:i for current mate
+ itoa10<TAlScore>((int)best[0], buf);
+ o.append(buf);
+ o.append(",");
+ // diff for current mate
+ if(diff[0] > MN) {
+ itoa10<TAlScore>((int)diff[0], buf);
+ o.append(buf);
+ } else {
+ o.append("NA");
+ }
+ o.append(",");
+ // AS:i for other mate
+ if(best[1] > MN) {
+ itoa10<TAlScore>((int)best[1], buf);
+ o.append(buf);
+ } else {
+ o.append("NA");
+ }
+ o.append(",");
+ // diff for other mate
+ if(diff[1] > MN) {
+ itoa10<TAlScore>((int)diff[1], buf);
+ o.append(buf);
+ } else {
+ o.append("NA");
+ }
+ o.append(",");
+ // Sum of AS:i for aligned pairs
+ if(best_conc > MN) {
+ itoa10<TAlScore>((int)best_conc, buf);
+ o.append(buf);
+ } else {
+ o.append("NA");
+ }
+ o.append(",");
+ // Diff for aligned pairs
+ if(diff_conc > MN) {
+ itoa10<TAlScore>((int)diff_conc, buf);
+ o.append(buf);
+ } else {
+ o.append("NA");
+ }
+ o.append(",");
+ itoa10<TAlScore>((int)(prm.seedsPerNuc * 1000), buf);
+ o.append(buf);
+ o.append(",");
+ itoa10<TAlScore>((int)(prm.seedPctUnique * 1000), buf);
+ o.append(buf);
+ o.append(",");
+ itoa10<TAlScore>((int)(prm.seedPctRep * 1000), buf);
+ o.append(buf);
+ o.append(",");
+ itoa10<TAlScore>((int)(prm.seedHitAvg + 0.5), buf);
+ o.append(buf);
+ }
}
/**
diff --git a/sam.h b/sam.h
index 9e2d77e..4170477 100644
--- a/sam.h
+++ b/sam.h
@@ -73,8 +73,6 @@ public:
bool print_xss,
bool print_yn,
bool print_xn,
- bool print_cs,
- bool print_cq,
bool print_x0,
bool print_x1,
bool print_xm,
@@ -121,8 +119,6 @@ public:
print_xss_(print_xss),
print_yn_(print_yn), // minimum valid score and perfect score
print_xn_(print_xn),
- print_cs_(print_cs),
- print_cq_(print_cq),
print_x0_(print_x0),
print_x1_(print_x1),
print_xm_(print_xm),
@@ -348,10 +344,6 @@ protected:
bool print_yn_; // YN:i:, Yn:i: minimum valid score and perfect score
bool print_xn_; // XN:i: Number of ambiguous bases in the referenece
- // Other optional flags
- bool print_cs_; // CS:Z: Color read sequence on the original strand
- bool print_cq_; // CQ:Z: Color read quality on the original strand
-
// Following are printed by BWA
bool print_x0_; // X0:i: Number of best hits
bool print_x1_; // X1:i: Number of sub-optimal best hits
diff --git a/scoring.cpp b/scoring.cpp
index 1348821..0672fa4 100644
--- a/scoring.cpp
+++ b/scoring.cpp
@@ -221,7 +221,6 @@ int main() {
4, // reward for a match
COST_MODEL_QUAL, // how to penalize mismatches
0, // constant if mm pelanty is a constant
- 30, // penalty for nuc mm in decoded colorspace als
-3.0f, // constant coeff for minimum score
-3.0f, // linear coeff for minimum score
DEFAULT_FLOOR_CONST, // constant coeff for score floor
diff --git a/scoring.h b/scoring.h
index 55f4893..2ac50a4 100644
--- a/scoring.h
+++ b/scoring.h
@@ -51,7 +51,7 @@
// Linear coefficient a
#define DEFAULT_MIN_LINEAR (-0.6f)
// Different defaults for --local mode
-#define DEFAULT_MIN_CONST_LOCAL (0.0f)
+#define DEFAULT_MIN_CONST_LOCAL (1.0f)
#define DEFAULT_MIN_LINEAR_LOCAL (10.0f)
// Constant coefficient b in linear function f(x) = ax + b determining
diff --git a/scripts/sa.py b/scripts/sa.py
new file mode 100644
index 0000000..d27857d
--- /dev/null
+++ b/scripts/sa.py
@@ -0,0 +1,79 @@
+#!/usr/bin/env python
+
+"""
+sa.py
+
+Parse and possibly sanity-check a .sa file output by bowtie2-build in --sa
+mode. These files have a very simple format: first is a uint32_t containing
+the length of the suffix array, the rest is an array of that many uint32_ts
+containing the suffix array.
+"""
+
+import sys
+import struct
+
+def loadBowtieSa(fh):
+ """ Load a .sa file from handle into an array of ints """
+ nsa = struct.unpack('I', fh.read(4))[0]
+ return [ struct.unpack('I', fh.read(4))[0] for i in xrange(0, nsa) ]
+
+def loadBowtieSaFilename(fn):
+ """ Load a .sa file from filename into an array of ints """
+ with open(fn, 'rb') as fh:
+ return loadBowtieSa(fh)
+
+def loadFasta(fns):
+ """ Load the concatenation of all the A/C/G/T characters """
+ falist = []
+ dna = set(['A', 'C', 'G', 'T', 'a', 'c', 'g', 't'])
+ for fn in fns:
+ with open(fn, 'r') as fh:
+ for line in fh:
+ if line[0] == '>':
+ continue
+ for c in line:
+ if c in dna:
+ falist.append(c)
+ return ''.join(falist)
+
+if __name__ == "__main__":
+ import argparse
+
+ parser = argparse.ArgumentParser(\
+ description='Parse suffix array built from bowtie2-build')
+
+ parser.add_argument(\
+ '--sa', metavar='string', required=True, type=str,
+ help='Suffix array file')
+ parser.add_argument(\
+ '--fa', metavar='string', type=str, nargs='+', help='FASTA file')
+
+ args = parser.parse_args()
+
+ def go():
+ ref = None
+ if args.fa is not None:
+ ref = loadFasta(args.fa)
+ sas = loadBowtieSaFilename(args.sa)
+ # Suffix array is in sas; note that $ is considered greater than all
+ # other characters
+ if ref is not None:
+ for i in xrange(1, len(sas)):
+ sa1, sa2 = sas[i-1], sas[i]
+ assert sa1 != sa2
+ # Sanity check that suffixes are really in order
+ while sa1 < len(ref) and sa2 < len(ref):
+ if ref[sa1] < ref[sa2]:
+ break
+ assert ref[sa1] == ref[sa2]
+ sa1 += 1
+ sa2 += 1
+ else:
+ # Note: Bowtie treats $ as greater than all other
+ # characters; so if these strings are tied up to the end of
+ # one or the other, the longer string is prior
+ assert sa1 < sa2, "%d, %d" % (sas[i-1], sas[i])
+ assert sas[-1] == len(ref)
+
+ go()
+
\ No newline at end of file
diff --git a/scripts/sim/AlignmentCheck.pm b/scripts/sim/AlignmentCheck.pm
new file mode 100644
index 0000000..fb259c4
--- /dev/null
+++ b/scripts/sim/AlignmentCheck.pm
@@ -0,0 +1,859 @@
+#!/usr/bin/perl -w
+
+#
+# Copyright 2011, Ben Langmead <langmea at cs.jhu.edu>
+#
+# This file is part of Bowtie 2.
+#
+# Bowtie 2 is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Bowtie 2 is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Bowtie 2. If not, see <http://www.gnu.org/licenses/>.
+#
+
+##
+# AlignmentCheck.pm
+#
+# Read in fasta files containing reference sequences that might be
+# aligned to, then read in alignment files, checking each alignment to
+# be sure it's sane and consistent with the reference sequence it
+# aligns to.
+#
+
+package AlignmentCheck;
+use strict;
+use warnings;
+use FindBin qw($Bin);
+use lib $Bin;
+use DNA;
+use Data::Dumper;
+
+##
+# Parse a fasta file into the %ref hash
+#
+sub parseFasta($$) {
+ my ($fa, $ref) = @_;
+ print STDERR "Parsing FASTA file $fa...\n";
+ my $fapipe = "$fa";
+ $fapipe = "gzip -dc $fa |" if $fa =~ /\.gz$/;
+ open(FA, $fapipe) || die "Could not open '$fapipe' for reading";
+ my $name = "";
+ my $bases = 0;
+ while(<FA>) {
+ chomp;
+ if(/^>/) {
+ $name = substr($_, 1);
+ $name =~ s/\s.*//;
+ print STDERR " parsing sequence $name...\n";
+ $ref->{std}{$name} = "";
+ } else {
+ $name ne "" || die "sequence before name";
+ $bases += length($_);
+ $ref->{std}{$name} .= $_;
+ }
+ }
+ print STDERR "Parsed $bases reference bases\n";
+ close(FA);
+}
+
+##
+# Create a new alignment checker
+#
+sub new {
+ my (
+ $class,
+ $name, # name of checker
+ $fas, # list of fasta files containing reference sequences
+ # or hash of the references themselves (former if it's
+ # an array ref latter if it's a hash ref)
+ $format, # alignment format
+ $bisC, # whether alignment was w/r/t ref w/ all Cs converted to Ys
+ $bisCpG, # whether alignment was w/r/t ref w/ all CpGs converted to YpGs
+ ) = @_;
+ (defined($fas)) || die "Must specify non-empty list of fasta files";
+ # Set defaults for format, bisC, bisCpG, name
+ $format = "bowtie" unless defined($format);
+ $bisC = 0 unless defined($bisC);
+ $bisCpG = 0 unless defined($bisCpG);
+ $name = "noname" unless defined($name);
+ # Parse all the fasta files into the ref hash
+ my %ref = ();
+ if(ref($fas) eq "HASH") {
+ for (keys %$fas) {
+ $ref{std}{$_} = $fas->{$_}
+ }
+ } else {
+ ref($fas) eq "ARRAY" || die;
+ foreach (@$fas) { parseFasta($_, \%ref); }
+ }
+ return bless {
+ _name => $name,
+ _fas => $fas,
+ _format => $format,
+ _bisC => $bisC,
+ _bisCpG => $bisCpG,
+ _refs => \%ref,
+ _nals => 0,
+ _nedits => 0
+ }, $class;
+}
+sub name { return $_[0]->{_name} }
+sub fas { return $_[0]->{_fas} }
+sub format { return $_[0]->{_format} }
+sub bisC { return $_[0]->{_bisC} }
+sub bisCpG { return $_[0]->{_bisCpG} }
+sub refs { return $_[0]->{_refs} }
+sub nals { return $_[0]->{_nals} }
+sub nedits { return $_[0]->{_nedits} }
+sub nrefs { return scalar(keys %{$_[0]->{_refs}}); }
+
+##
+# Given a sequence that represents the read oriented s.t. the Watson-
+# upstream end is on the left, and given a set of edits to apply to the
+# read, also oriented assuming that Watson-upstream is on the left,
+# return the string corresponding to the read mutated to match the
+# reference.
+#
+my $nedits = 0;
+sub applyEdits($$$) {
+ my ($seq, $edits, $line) = @_;
+ my $rfseq = $seq;
+ my $lpos = length($seq)+1;
+ $nedits += scalar(@$edits);
+ foreach (reverse @$edits) {
+ next unless defined($_);
+ #print STDERR "Applying edit at $_->{pos}\n";
+ # Apply the edit
+ $_->{pos} <= $lpos || die "Edit position $_->{pos} was greater than previous $lpos";
+ if($_->{qchr} eq "-") {
+ # Insert
+ $_->{pos} <= length($rfseq) || die "Read gap pos $_->{pos} was not <= string len ".length($rfseq)."\n$line";
+ substr($rfseq, $_->{pos}, 0) = $_->{chr};
+ } elsif($_->{chr} eq "-") {
+ # Deletion
+ $_->{pos} < length($rfseq) || die "Ref gap pos $_->{pos} was not < string len ".length($rfseq)."\n$line";
+ my $dc = substr($rfseq, $_->{pos}, 1);
+ $dc eq $_->{qchr} ||
+ die "Edit: $_->{pos}:$_->{chr}>$_->{qchr} but ref char was $dc".
+ "\n$rfseq\n$line";
+ substr($rfseq, $_->{pos}, 1) = "";
+ } else {
+ # Mismatch
+ $_->{pos} < length($rfseq) || die "Mismatch pos $_->{pos} was not < string len ".length($rfseq)."\n$line";
+ substr($rfseq, $_->{pos}, 1) eq $_->{qchr} ||
+ die "Edit: $_->{pos}:$_->{chr}>$_->{qchr}\n$rfseq\n$line";
+ substr($rfseq, $_->{pos}, 1) = $_->{chr};
+ }
+ }
+ return $rfseq;
+}
+
+##
+# Given a list of Bowtie edits, invert them by reversing the list and
+# changing the poss to be with respect to the other end of the read.
+#
+sub invertEdits($$) {
+ my ($edits, $len) = @_;
+ @$edits = reverse @$edits;
+ for (@$edits) {
+ next unless defined($_);
+ defined($_->{qchr}) || die;
+ if($_->{qchr} eq "-") {
+ $_->{pos} = $len - $_->{pos};
+ length($_->{chr}) >= 1 || die;
+ $_->{chr} = reverse $_->{chr};
+ } else {
+ $_->{pos} = $len - $_->{pos} - 1;
+ length($_->{chr}) == 1 || die;
+ }
+ }
+}
+
+##
+# Given an edit string, parses it into a list of hashes and returns
+# the list.
+#
+sub parseEdits($) {
+ my $editstr = shift;
+ return undef if (!defined($editstr) || $editstr eq "-" || $editstr eq "");
+ my @edits = ();
+ # For each edit
+ for (split(/,/, $editstr)) {
+ # Parse pos
+ my ($pos, $ed) = split(/:/);
+ defined($ed) || die;
+ # Parse chr, qchr
+ my ($chr, $qchr) = split(/>/, $ed);
+ push @edits, { pos => $pos, chr => $chr, qchr => $qchr };
+ }
+ return \@edits;
+}
+
+##
+# Given an edit string, possibly with 4 semicolon-delimited fields,
+# parse it into a set of 4 lists of hashes and return the set as an
+# array ref.
+#
+sub parseAllEdits($) {
+ my $editstr = shift;
+ return [ undef, undef, undef, undef ] if ($editstr eq "-" || $editstr eq "");
+ my @editls = split(/;/, $editstr, -1);
+ if(scalar(@editls) > 1) {
+ scalar(@editls) == 4 || die;
+ return [
+ parseEdits($editls[0]),
+ parseEdits($editls[1]),
+ parseEdits($editls[2]),
+ parseEdits($editls[3]) ];
+ } else {
+ scalar(@editls) == 1 || die;
+ return [
+ parseEdits($editls[0]),
+ undef,
+ undef,
+ undef ];
+ }
+}
+
+##
+# Given array refs for two lists of edits, one corresponding to the
+# nucleotide edit list and the other corresponding to the resolved
+# ambiguous base list, eliminate any edits that appear in both lists.
+# Really this shouldn't happen, but I observe that merman does report
+# an edit in both categories if the reference base is being resolved to
+# an incompatible nucleotide, e.g. R>C.
+#
+sub removeDups($$) {
+ my ($ned, $aed) = @_;
+ return unless (defined($ned) && defined($aed));
+ for my $i (0..scalar(@$ned)-1) {
+ next unless defined($ned->[$i]);
+ for my $j (0..scalar(@$aed)-1) {
+ next unless defined($aed->[$j]);
+ if($ned->[$i]->{qchr} eq $aed->[$j]->{qchr} &&
+ $ned->[$i]->{chr} eq $aed->[$j]->{chr} &&
+ $ned->[$i]->{pos} eq $aed->[$j]->{pos})
+ {
+ #print STDERR " Eliminated a duplicate edit\n";
+ $aed->[$j] = undef;
+ }
+ }
+ }
+}
+
+##
+# Take all the references in %ref and make both Watson and Crick
+# versions where the sequence is in-silico bisulfite treated.
+#
+sub bisulfiteC($) {
+ my $ref = shift;
+ for(keys %{$ref->{std}}) {
+ $ref->{bisc_fw}{$_} = $ref->{std}{$_};
+ $ref->{bisc_fw}{$_} = s/C/Y/g;
+ $ref->{bisc_rc}{$_} = $ref->{std}{$_};
+ $ref->{bisc_rc}{$_} = s/G/R/g;
+ }
+}
+
+##
+# Take all the references in %ref and make both Watson and Crick
+# versions where the sequence is in-silico bisulfite treated.
+#
+sub bisulfiteCpG($) {
+ my $ref = shift;
+ for(keys %{$ref->{std}}) {
+ $ref->{biscpg_fw}{$_} = $ref->{std}{$_};
+ $ref->{biscpg_fw}{$_} =~ s/CG/YG/g;
+ $ref->{biscpg_fw}{$_} =~ s/C/T/g;
+ $ref->{biscpg_rc}{$_} = $ref->{std}{$_};
+ $ref->{biscpg_rc}{$_} =~ s/CG/CR/g;
+ $ref->{biscpg_rc}{$_} =~ s/G/A/g;
+ }
+}
+
+##
+# Given a Bowtie orientation string, return true iff the 5' end of the
+# read is at the left of the printed sequence.
+#
+sub fivePrimeLeft($) {
+ return ($_[0] eq "+" || $_[0] eq "W" || $_[0] eq "CR");
+}
+
+##
+# Given the orientation of the read and the state of the global
+# bisulfite variables, determine which version of the reference to
+# compare against.
+#
+sub calcRefType {
+ my ($self, $orient) = @_;
+ if($self->bisC || $self->bisCpG) {
+ if($orient eq "W" || $orient eq "WR" || $orient eq "+") {
+ return $self->bisC ? "bisc_fw" : "biscpg_fw";
+ } else {
+ $orient eq "C" || $orient eq "CR" || $orient eq "-" || die;
+ return $self->bisC ? "bisc_rc" : "biscpg_rc";
+ }
+ } else {
+ return "std";
+ }
+}
+
+##
+# Parse a CIGAR string into parallel arrays of CIGAR operations (M, I, D)
+#
+sub cigarParse($$$) {
+ my ($cigar, $ops, $runs) = @_;
+ my $i = 0;
+ while($i < length($cigar)) {
+ substr($cigar, $i) =~ /^([0-9]+)/;
+ defined($1) || die "Could not parse number at pos $i: '$cigar'";
+ $i += length($1);
+ $i < length($cigar) || die;
+ push @$runs, $1;
+ my $op = substr($cigar, $i, 1);
+ defined($op) || die "Could not parse operation at pos $i: '$cigar'";
+ push @$ops, $op;
+ $i++;
+ }
+}
+
+##
+# Trim a read sequence according to the soft clipping in the CIGAR string.
+#
+sub cigarTrim($$) {
+ my ($seq, $cigar) = @_;
+ my @ops = ();
+ my @runs = ();
+ cigarParse($cigar, \@ops, \@runs);
+ my ($trimup, $trimdn) = (0, 0);
+ if($ops[0] eq 'S') {
+ $runs[0] < length($seq) || die "Soft clipped entire alignment!";
+ $seq = substr($seq, $runs[0]);
+ $trimup = $runs[0];
+ }
+ if(scalar(@ops) > 1 && $ops[-1] eq 'S') {
+ $runs[-1] < length($seq) || die "Soft clipped entire alignment!";
+ $seq = substr($seq, 0, -$runs[-1]);
+ $trimdn = $runs[-1];
+ }
+ return ($seq, $trimup, $trimdn);
+}
+
+##
+# Parse a CIGAR string into a string of operators. Operators are expanded into
+# runs where appropriate. = and X are collapsed into M.
+#
+sub parse_cigar($) {
+ my ($cigar) = @_;
+ my $ret = "";
+ my $i = 0;
+ my ($rdlen, $rflen) = (0, 0);
+ while($i < length($cigar)) {
+ substr($cigar, $i) =~ /^([0-9]+)/;
+ defined($1) || die "Could not parse number at pos $i: '$cigar'";
+ my $runlen = $1;
+ $i += length($1);
+ $i < length($cigar) || die;
+ my $op = substr($cigar, $i, 1);
+ defined($op) || die "Could not parse operation at pos $i: '$cigar'";
+ if($op eq "X" || $op eq "=") {
+ $op = "M";
+ }
+ $rdlen += $runlen if $op ne "D";
+ $rflen += $runlen if $op ne "I";
+ $ret .= ($op x $runlen);
+ $i++;
+ }
+ return ($ret, $rdlen, $rflen);
+}
+
+##
+# Parse an MD:Z string into a string with length equal to query length. Each
+# position contains either a space, if the read matches the reference at that
+# position, or a character, if the reference contains a character that doesn't
+# match its opposite in the alignment. In the latter case, the character in
+# the string is the reference character.
+#
+sub parse_md($) {
+ my ($md) = @_;
+ my $i = 0;
+ my $ret = "";
+ while($i < length($md)) {
+ # Starts with a number?
+ my $ch = substr($md, $i, 1);
+ if($ch =~ /[0-9]/) {
+ # Parse the number off the beginning
+ substr($md, $i) =~ /^([0-9]+)/;
+ defined($1) || die "Could not parse number at pos $i: '$md'";
+ my $runlen = $1;
+ $ret .= (" " x $runlen) if $runlen > 0;
+ $i += length($runlen);
+ } elsif($ch eq "^") {
+ # Read gap
+ $i++;
+ substr($md, $i) =~ /^([A-Za-z]+)/;
+ defined($1) || die "Could not parse read gap at pos $i: '$md'";
+ my $chrs = $1;
+ $i += length($chrs);
+ $ret .= $chrs;
+ } else {
+ # DNA character
+ $ch =~ /[ACGTN.]/i || die "Bad char '$ch' at pos $i: '$md'";
+ $ret .= $ch;
+ $i++;
+ }
+ }
+ return $ret;
+}
+
+##
+# Given a read sequence (with characters in upstream-to-downstream order with
+# respect to the reference - NOT necessarily 5'-to-3') and a CIGAR string and
+# an MD:Z string, build the alignment strings. The alignment strings will only
+# contain the portion of the read that aligned. Any portions that were either
+# hard-trimmed or soft-trimmed are trimmed from this function's result.
+#
+# For now, I'm assuming that the MD:Z string only describes aligned characters,
+# i.e. *after* trimming.
+#
+sub _read_md_cigar_to_al($$$) {
+ my ($seq, $md, $cigar) = @_;
+ my $alread = "";
+ my $alref = "";
+ $cigar ne "*" || die "CIGAR string was star!";
+ $seq ne "" || die "Empty sequence given to _read_md_cigar_to_al";
+ my $parsed_md = parse_md($md);
+ my ($parsed_cig, $cigar_rdlen, $cigar_rflen) = parse_cigar($cigar);
+ my ($rdoff, $mdoff) = (0, 0);
+ my ($htriml, $striml, $htrimr, $strimr) = (0, 0, 0, 0);
+ my $nonsh = 0; # have I seen a non-S, non-H CIGAR op?
+ my $nonh = 0; # have I seen a non-H CIGAR op?
+ for(my $i = 0; $i < length($parsed_cig); $i++) {
+ my $cigop = substr($parsed_cig, $i, 1);
+ $nonh++ if $cigop ne "H";
+ $nonsh++ if ($cigop ne "H" && $cigop ne "S");
+ if($cigop eq "S") {
+ if($nonsh) {
+ $strimr++;
+ } else {
+ $striml++;
+ }
+ $rdoff++;
+ next;
+ }
+ if($cigop eq "H") {
+ if($nonh) {
+ $htrimr++;
+ } else {
+ $htriml++;
+ }
+ next;
+ }
+ $cigop = "M" if $cigop eq "=" || $cigop eq "X";
+ if($cigop eq "P") {
+ # Padding
+ $alread .= "-";
+ $alref .= "-";
+ } elsif($cigop eq "M") {
+ my $rdc = substr($seq, $rdoff, 1);
+ $mdoff < length($parsed_md) ||
+ die "Bad mdoff ($mdoff)\nlength(parsed_md)=".length($parsed_md)."\nseq:\n$seq\ncigar:\n$cigar\nmd:\n$md\nparsed md:\n$parsed_md";
+ my $rfc = substr($parsed_md, $mdoff, 1);
+ $rfc = $rdc if $rfc eq " ";
+ $alread .= $rdc;
+ $alref .= $rfc;
+ $rdoff++;
+ $mdoff++;
+ } elsif($cigop eq "D") {
+ # Read gap
+ # Read: AAA-AAA
+ # Ref: AAAAAAA
+ my $rfc = substr($parsed_md, $mdoff, 1);
+ $rfc ne " " ||
+ die "Must have a ref character opposite a gap in the read:\n".
+ "cig: $parsed_cig ($i)\nmd: $parsed_md ($mdoff)\n";
+ $alread .= "-";
+ $alref .= $rfc;
+ $mdoff++;
+ } else {
+ # Reference gap
+ # Read: AAAAAAA
+ # Ref: AAA-AAA
+ $cigop eq "I" || die "Unsupported cigop: $cigop in cigar $cigar";
+ my $rdc = substr($seq, $rdoff, 1);
+ $alread .= $rdc;
+ $alref .= "-";
+ $rdoff++;
+ # $mdoff SHOULD NOT be incremented here
+ }
+ $rdoff <= length($seq) ||
+ die "Bad rdoff:$rdoff for seq '$seq' cigop=$cigop\nseq: $seq\ncigar=$cigar\nmd=$md";
+ }
+ return ($alread, $alref, $htriml, $striml, $htrimr, $strimr);
+}
+
+##
+# Parse a line from a Bowtie alignment file and check that the
+# alignment is sane and consistent with the reference.
+#
+sub parseBowtieLines {
+ my ($self, $lines) = @_;
+ for my $line (@$lines) {
+ chomp($line);
+ my ($rdname, $orient, $refname, $off, $seq, $qual, $oms, $editstr,
+ $flags) = split(/\t/, $line, -1);
+ next if $refname eq "*";
+ $flags =~ /XC:([^,\s]+)/;
+ my $cigar = $1;
+ defined($cigar) ||
+ die "Could not parse CIGAR string from flags: '$flags'";
+ defined($editstr) || die "Failed to get 8 tokens from line:\n$_";
+ $off == int($off) || die "Offset field (col 4) must be an integer:\n$_";
+ $oms == int($oms) || die "OMS field (col 7) must be an integer:\n$_";
+ my $reftype = $self->calcRefType($orient);
+ defined($self->refs->{$reftype}{$refname}) ||
+ die "No such refname as $refname for reftype $reftype:\n".
+ Dumper($self->refs->{$reftype});
+ my $edits4 = parseAllEdits($editstr);
+ my ($ned, $aed) = ($edits4->[0], $edits4->[1]);
+ removeDups($ned, $aed);
+ my $fpl = fivePrimeLeft($orient);
+ # Trim seq according to CIGAR string
+ my $rfseq = $seq;
+ my ($trimup, $trimdn);
+ ($rfseq, $trimup, $trimdn) = cigarTrim($rfseq, $cigar);
+ invertEdits($ned, length($rfseq)) unless ($fpl || !defined($ned));
+ invertEdits($aed, length($rfseq)) unless ($fpl || !defined($aed));
+ $rfseq = applyEdits($rfseq, $ned, $line) if defined($ned);
+ $rfseq = applyEdits($rfseq, $aed, $line) if defined($aed);
+ # Check if our alignment falls off the end of the reference, in
+ # which case we need to pad the reference string with Ns
+ my $exoff = $off;
+ my $padleft = "";
+ my $exlen = length($rfseq);
+ my $tlen = length($self->refs->{$reftype}{$refname});
+ if($exoff < 0) {
+ # Alignment hangs off LHS; pad it
+ my $npad = -$exoff;
+ $padleft = "N" x $npad;
+ $exlen += $exoff;
+ $exlen >= 0 ||
+ die "Read was entirely off the LHS of the reference\n".
+ "Referemce len=$tlen\n".
+ "Alignment referemce len=$tlen\n".
+ "$line\n";
+ $exoff = 0;
+ }
+ my $padright = "";
+ my $roverhang = $off + length($rfseq) - $tlen;
+ if($roverhang > 0) {
+ $padright = "N" x $roverhang;
+ $exlen -= $roverhang;
+ $exlen >= 0 ||
+ die "Read was entirely off the RHS of the reference\n".
+ "Referemce len=$tlen\n".
+ "Alignment referemce len=$tlen\n".
+ "$line\n";
+ }
+ my $refsub = substr($self->refs->{$reftype}{$refname}, $exoff, $exlen);
+ length($refsub) == $exlen ||
+ die "Tried to extract ref substring of length $exlen, got ".
+ "\"$refsub\" from \"".$self->refs->{$reftype}{$refname}."\"".
+ "\n$line\n".
+ "\noff=$off, rfseq=$rfseq\n";
+ $refsub = DNA::iupacSubN($refsub);
+ my $trueRfseq = $padleft . $refsub . $padright;
+ length($trueRfseq) == length($rfseq) ||
+ die "Different lengths for edited read and ref:\n".
+ " Read: $seq\n".
+ "Edited read: $rfseq\n".
+ " Ref: $trueRfseq\n";
+ $rfseq eq $trueRfseq ||
+ die "Did not match:\n".
+ " Read: $seq\n".
+ "Edited read: $rfseq\n".
+ " Ref: $trueRfseq\n";
+ $self->{_nals}++;
+ }
+}
+
+##
+# Parse a line from a SAM alignment file and check that the
+# alignment is sane and consistent with the reference.
+#
+sub parseSamLines {
+ my ($self, $lines) = @_;
+ my ($lastseq, $lastqual) = ("", "");
+ my $idx = 0;
+ for my $line (@$lines) {
+ $idx++;
+ print STDERR "Processing line...\n";
+ chomp($line);
+ next if $line =~ /^\@/;
+ my @toks = split(/\t/, $line, -1);
+ my (
+ $qname, #1
+ $flag, #2
+ $rname, #3
+ $pos, #4
+ $mapq, #5
+ $cigar, #6
+ $rnext, #7
+ $pnext, #8
+ $tlen, #9
+ $seq, #10
+ $qual) = @toks;
+ defined($qual) || die "Not enough SAM tokens:\n$line\n";
+ my @opt_flags_list = @toks[11..$#toks];
+ my %opt_flags = ();
+ next if $cigar eq "*"; # Failed to align
+ # Get the read sequence & qualities from previous record if necessary
+ if($seq eq "*") {
+ $lastseq ne "" || die "Line #$idx:\n$line";
+ $seq = $lastseq;
+ $qual = $lastqual;
+ } else {
+ $lastseq = $seq;
+ $lastqual = $qual;
+ }
+ $seq ne "*" || die;
+ my ($parsed_cigar, $rdlen_cigar, $rflen_cigar) = parse_cigar($cigar);
+ length($seq) == $rdlen_cigar ||
+ die "Sequence length and parsed cigar string length ($rdlen_cigar) mismatch:\n".
+ "$seq\n$parsed_cigar\nLine:\n$line";
+ # Stick optional flags into a hash
+ for my $fl (@opt_flags_list) {
+ my @fs = split(/:/, $fl, -1);
+ scalar(@fs) > 2 || die "Bad optional flag: $fl\n$line\n";
+ $opt_flags{$fs[0]}{type} = $fs[1];
+ $opt_flags{$fs[0]}{value} = join(":", @fs[2..$#fs]);
+ }
+ defined($opt_flags{"MD"}) || die "No MD:Z flag:\n$line\n";
+ $opt_flags{"MD"}{type} eq "Z" || die "Bad type for MD:Z flag\n$line\n";
+ my $md = $opt_flags{"MD"}{value};
+ $pos == int($pos) || die "POS field (col 4) must be an int:\n$line\n";
+ $pnext == int($pnext) || die "PNEXT field (col 8) must be an int:\n$line\n";
+ $tlen == int($tlen) || die "TLEN field (col 9) must be an int:\n$line\n";
+ $mapq == int($mapq) || die "MAPQ field (col 5) must be an int:\n$line\n";
+ # TODO: deal with bisulfite strands??
+ my $fw = (($flag & 0x10) == 0);
+ my $orient = $fw ? "+" : "-";
+ my $reftype = $self->calcRefType($orient);
+ defined($self->refs->{$reftype}{$rname}) ||
+ die "No such refname as $rname for reftype $reftype:\n$line\n".
+ Dumper($self->refs->{$reftype});
+ my $exoff = $pos-1; # expected 0-based reference offset
+ my ($alread, $alref, $htriml, $striml, $htrimr, $strimr) =
+ _read_md_cigar_to_al($seq, $md, $cigar);
+ print STDERR "$alread\n$alref\n";
+ my $rfseq = $alref;
+ $rfseq =~ s/[ -]//g; # remove spaces & gaps
+ my $exlen = length($rfseq);
+ my $refsub = substr($self->refs->{$reftype}{$rname}, $exoff, $exlen);
+ length($refsub) == $exlen ||
+ die "Tried to extract ref substring of length $exlen from:\n".
+ $self->refs->{$reftype}{$rname}.
+ "\ngot string of length ".length($refsub).":\n".
+ $refsub.
+ "\nfrom:\n".
+ $line.
+ "\nexlen is the length of:\n$rfseq\npos=$pos, rfseq=$rfseq\n";
+ $refsub = DNA::iupacSubN($refsub);
+ my $trueRfseq = $refsub;
+ length($trueRfseq) == length($rfseq) ||
+ die "Different lengths for edited read and ref:\n".
+ " Read: $seq\n".
+ "Edited read: $rfseq\n".
+ " Ref: $trueRfseq\n";
+ $rfseq eq $trueRfseq ||
+ die "Did not match:\n".
+ " Read: $seq\n".
+ "Edited read: $rfseq\n".
+ " Ref: $trueRfseq\n".
+ "$line";
+ $self->{_nals}++;
+ }
+}
+
+##
+# Parse lines from a Bowtie alignment file and check that the
+# alignments are sane and consistent with the reference.
+#
+sub parseBowtie {
+ my ($self, $fh) = @_;
+ while(<$fh>) {
+ $self->parseBowtieLines([$_]);
+ }
+}
+
+##
+# Parse lines from a SAM alignment file and check that alignments are
+# sane and consistent with the reference.
+#
+sub parseSam {
+ my ($self, $fh) = @_;
+ my @lines = ();
+ while(<$fh>) { push @lines, $_; }
+ $self->parseSamLines(\@lines);
+}
+
+##
+# Parse lines from an alignment file of the type given by self->format
+#
+sub parseLines {
+ my ($self, $lines) = @_;
+ if($self->format eq "bowtie") {
+ $self->parseBowtieLines($lines);
+ } else {
+ $self->format eq "sam" || die;
+ $self->parseSamLines($lines);
+ }
+}
+
+##
+# Parse lines from an alignment file of the type given by self->format
+#
+sub parse {
+ my ($self, $fh) = @_;
+ if($self->format eq "bowtie") {
+ $self->parseBowtie($fh);
+ } else {
+ $self->format eq "sam" || die;
+ $self->parseSam($fh);
+ }
+}
+
+##
+# Print summary of how many alignments and edits were checked.
+#
+sub printSummary {
+ my $self = shift;
+ print STDERR "--- Summary ---\n";
+ print STDERR "Read ".scalar(keys %{$self->refs})." reference strings\n";
+ print STDERR "Checked $self->{_nals} alignments, $self->{_nedits} edits\n";
+ print STDERR "---------------\n";
+ print STDERR "PASSED\n";
+}
+
+##
+# Check the given batch of alignments. We check that they're
+# internally consistent in some basic ways, and we check that the
+# sequence and edits are consistent with the reference.
+#
+# The $als argument is either a list of (possibly compressed) filenames
+# of files containing alignments, or a list of alignment strings. If
+# the former, $filenames is non-zero.
+#
+sub checkAlignments {
+ my ($self, $als, $filenames) = @_;
+ if($self->bisC) {
+ print STDERR "Generating all-C bisulfite-treated references\n";
+ bisulfiteC($self->refs);
+ }
+ if($self->bisCpG) {
+ print STDERR "Generating all-CpG bisulfite-treated references\n";
+ bisulfiteCpG($self->refs);
+ }
+ if($filenames) {
+ foreach (@$als) {
+ my $alnpipe = $_;
+ print STDERR "Processing alignment file '$_'\n";
+ $alnpipe = "gzip -dc $_ |" if ($_ =~ /\.gz$/);
+ my $alnfh = undef;
+ open($alnfh, $alnpipe) || die "Could not open '$alnpipe' for reading";
+ $self->parse($alnfh);
+ close($alnfh);
+ }
+ } else {
+ $self->parseLines($als);
+ }
+}
+
+##
+# Check simple alignments
+#
+sub test1 {
+ my $ac = AlignmentCheck->new(
+ "AlignmentCheck.pm test1 checker",
+ { "r1" => "TTGTTCGT" },
+ "bowtie",
+ 0,
+ 0
+ );
+ $ac->checkAlignments([
+ "0\t+\tr1\t1\tTGTTCGT\tIIIIIII\t40\t-",
+ "1\t+\tr1\t0\tTTGTTCG\tIIIIIII\t40\t-",
+ "2\t+\tr1\t2\tGTTCGTA\tIIIIIII\t40\t6:N>A",
+ "3\t+\tr1\t-1\tATTGTTC\tIIIIIII\t40\t0:N>A"], 0);
+ return 1;
+}
+
+##
+# Check simple alignments from files
+#
+sub test2 {
+ open(TMP, ">/tmp/.AlignmentCheck.pm.fa") || die;
+ print TMP ">r1\n";
+ print TMP "TTGTTCGT\n";
+ close(TMP);
+ my $ac = AlignmentCheck->new(
+ "AlignmentCheck.pm test1 checker",
+ [ "/tmp/.AlignmentCheck.pm.fa" ],
+ "bowtie",
+ 0,
+ 0
+ );
+ $ac->checkAlignments([
+ "0\t+\tr1\t1\tTGTTCGT\tIIIIIII\t40\t-",
+ "1\t+\tr1\t0\tTTGTTCG\tIIIIIII\t40\t-",
+ "2\t+\tr1\t2\tGTTCGTA\tIIIIIII\t40\t6:N>A",
+ "3\t+\tr1\t-1\tATTGTTC\tIIIIIII\t40\t0:N>A"], 0);
+ return 1;
+}
+
+if($0 =~ /[^0-9a-zA-Z_]?AlignmentCheck\.pm$/) {
+ my @fas = ();
+ my @als = ();
+ my $format = "sam";
+ my $bisC = 0;
+ my $bisCpG = 0;
+ my $test = 0;
+
+ use Getopt::Long;
+ GetOptions (
+ "test" => \$test,
+ "fasta|ref=s" => \@fas,
+ "als|aln=s" => \@als,
+ "format=s" => \$format,
+ "bis-C|bisulfite-C" => \$bisC,
+ "bis-CpG|bisulfite-CpG" => \$bisCpG,
+ "bowtie" => sub {$format = "bowtie"},
+ "sam" => sub {$format = "sam"}) || die;
+
+ if($test) {
+ use Test;
+ print "Running unit tests\n";
+ # Run unit tests
+ Test::shouldSucceed("test1", \&test1);
+ Test::shouldSucceed("test2", \&test2);
+ exit 0;
+ }
+
+ my $ac = AlignmentCheck->new(
+ "AlignmentCheck.pm checker",
+ \@fas,
+ $format,
+ $bisC,
+ $bisCpG);
+ $ac->checkAlignments(\@als, 1);
+}
+
+1;
diff --git a/scripts/sim/DNA.pm b/scripts/sim/DNA.pm
new file mode 100644
index 0000000..bf6b4e2
--- /dev/null
+++ b/scripts/sim/DNA.pm
@@ -0,0 +1,287 @@
+#!/usr/bin/perl -w
+
+#
+# Copyright 2011, Ben Langmead <langmea at cs.jhu.edu>
+#
+# This file is part of Bowtie 2.
+#
+# Bowtie 2 is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Bowtie 2 is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Bowtie 2. If not, see <http://www.gnu.org/licenses/>.
+#
+
+package DNA;
+use strict;
+use warnings;
+use Carp;
+use FindBin qw($Bin);
+use lib $Bin;
+use Test;
+
+##
+# Set up uppercase IUPAC characters minus N
+#
+my %iupac_u_nn = (
+ "R" => 1,
+ "Y" => 1,
+ "M" => 1,
+ "K" => 1,
+ "S" => 1,
+ "W" => 1,
+ "B" => 1,
+ "V" => 1,
+ "H" => 1,
+ "D" => 1
+);
+
+##
+# Return true iff arg is an IUPAC code and not ACGT or N.
+#
+sub isIUPAC($) {
+ return defined($iupac_u_nn{$_[0]});
+}
+
+##
+# Replace IUPAC characters with Ns.
+#
+sub iupacSubN($) {
+ $_[0] =~ tr/RYMKSWBVHDrymkswbvhd/NNNNNNNNNNnnnnnnnnnn/;
+ return $_[0];
+}
+
+my %compat = (
+ "A" => "A",
+ "T" => "T",
+ "C" => "C",
+ "G" => "G",
+ "R" => "AG",
+ "Y" => "CT",
+ "M" => "AC",
+ "K" => "GT",
+ "S" => "CG",
+ "W" => "AT",
+ "B" => "CGT",
+ "V" => "ACG",
+ "H" => "ACT",
+ "D" => "AGT",
+ "N" => "ACGT"
+);
+
+##
+# Pick a random character that's compatible with input character.
+#
+sub pickCompat($) {
+ my $c = uc $_[0];
+ defined($compat{$c}) || die "Bad input $c";
+ my $cc = $compat{$c};
+ if(length($cc) == 1) {
+ return $cc;
+ } else {
+ length($cc) > 1 || die;
+ return substr($cc, int(rand(length($cc))), 1);
+ }
+}
+
+my %incompat = (
+ "A" => "CGT",
+ "T" => "ACG",
+ "C" => "AGT",
+ "G" => "ACT",
+ "R" => "CT",
+ "Y" => "AG",
+ "M" => "GT",
+ "K" => "AC",
+ "S" => "AT",
+ "W" => "CG",
+ "B" => "A",
+ "V" => "T",
+ "H" => "G",
+ "D" => "C",
+ "N" => "ACGT"
+);
+
+##
+# Pick a random character that's compatible with input character.
+#
+sub pickIncompat($) {
+ my $c = uc $_[0];
+ defined($incompat{$c}) || die "Bad input $c";
+ my $cc = $incompat{$c};
+ if(length($cc) == 1) {
+ return $cc;
+ } else {
+ length($cc) > 1 || die;
+ return substr($cc, int(rand(length($cc))), 1);
+ }
+}
+
+##
+# Set up lowercase IUPAC characters minus N
+#
+my %iupac_l_nn = (
+ "r" => 1,
+ "y" => 1,
+ "m" => 1,
+ "k" => 1,
+ "s" => 1,
+ "w" => 1,
+ "b" => 1,
+ "v" => 1,
+ "h" => 1,
+ "d" => 1
+);
+
+my %revcompMap = (
+ "A" => "T",
+ "T" => "A",
+ "C" => "G",
+ "G" => "C",
+ "R" => "Y",
+ "Y" => "R",
+ "M" => "K",
+ "K" => "M",
+ "S" => "S",
+ "W" => "W",
+ "B" => "V",
+ "V" => "B",
+ "H" => "D",
+ "D" => "H",
+ "N" => "N",
+ "a" => "t",
+ "t" => "a",
+ "c" => "g",
+ "g" => "c",
+ "r" => "y",
+ "y" => "r",
+ "m" => "k",
+ "k" => "m",
+ "s" => "s",
+ "w" => "w",
+ "b" => "v",
+ "v" => "b",
+ "h" => "d",
+ "d" => "h",
+ "n" => "n"
+);
+
+my %unambigSet = (
+ "A" => 1, "a" => 1,
+ "C" => 1, "c" => 1,
+ "G" => 1, "g" => 1,
+ "T" => 1, "t" => 1
+);
+
+##
+# Return the complement, incl. if it's IUPAC.
+#
+sub comp($) {
+ my $s = uc shift;
+ return $s =~ tr/ACGTRYMKSWBVHDN/TGCAYRKMSWVBDHN/;
+}
+
+##
+# Return the reverse complement of a string.
+#
+sub revcomp($) {
+ my $s = uc shift;
+ $s = reverse $s;
+ $s =~ tr/ACGTRYMKSWBVHDN/TGCAYRKMSWVBDHN/;
+ return $s;
+}
+
+##
+# Return true iff it's unambiguous.
+#
+sub unambig($) {
+ return $unambigSet{$_[0]};
+}
+
+##
+# Manipulate DNA in an integer-indexed fashion.
+#
+sub plus($$) {
+ my ($c, $amt) = @_;
+ my %ctoi = ("A" => 0, "C" => 1, "G" => 2, "T" => 3);
+ my %itoc = (0 => "A", 1 => "C", 2 => "G", 3 => "T");
+ $c = uc $c;
+ defined($ctoi{$c}) || die "Not an unambiguous nucleotide: $c";
+ return $itoc{($ctoi{$c}+$amt) % 4};
+}
+
+my %dinucToColorMap = (
+ "AA" => "0",
+ "AC" => "1",
+ "AG" => "2",
+ "AT" => "3",
+ "CC" => "0",
+ "CG" => "3",
+ "CT" => "2",
+ "GG" => "0",
+ "GT" => "1",
+ "TT" => "0",
+
+ "AN" => ".",
+ "CN" => ".",
+ "GN" => ".",
+ "NT" => ".",
+ "NN" => ".",
+);
+
+sub dinucToColor($$) {
+ my ($n1, $n2) = @_;
+ if(ord($n2) < ord($n1)) {
+ my $tmp = $n1;
+ $n1 = $n2;
+ $n2 = $tmp;
+ }
+ ord($n1) <= ord($n2) || die;
+ defined($dinucToColorMap{"$n1$n2"}) ||
+ die "Bad nucleotide dinuc: '$n1$n2'";
+ return $dinucToColorMap{"$n1$n2"};
+}
+
+sub test1 {
+ plus("A", 1) eq "C" || die;
+ plus("C", 1) eq "G" || die;
+ plus("G", 1) eq "T" || die;
+ plus("T", 1) eq "A" || die;
+ return 1;
+}
+
+sub test2 {
+ plus("A", 2) eq "G" || die;
+ plus("C", 2) eq "T" || die;
+ plus("G", 2) eq "A" || die;
+ plus("T", 2) eq "C" || die;
+ return 1;
+}
+
+sub test3 {
+ revcomp("ACGT") eq "ACGT" || die;
+ return 1;
+}
+
+sub test4 {
+ revcomp("ACGTYR") eq "YRACGT" || die;
+ return 1;
+}
+
+if($0 =~ /[^0-9a-zA-Z_]?DNA\.pm$/) {
+ print "Running unit tests\n";
+ # Run unit tests
+ Test::shouldSucceed("test1", \&test1);
+ Test::shouldSucceed("test2", \&test2);
+ Test::shouldSucceed("test3", \&test3);
+ Test::shouldSucceed("test4", \&test4);
+}
+
+1;
diff --git a/scripts/sim/Mutate.pm b/scripts/sim/Mutate.pm
new file mode 100644
index 0000000..2157f73
--- /dev/null
+++ b/scripts/sim/Mutate.pm
@@ -0,0 +1,301 @@
+#!/usr/bin/perl -w
+
+#
+# Copyright 2011, Ben Langmead <langmea at cs.jhu.edu>
+#
+# This file is part of Bowtie 2.
+#
+# Bowtie 2 is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Bowtie 2 is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Bowtie 2. If not, see <http://www.gnu.org/licenses/>.
+#
+
+package Mutate;
+use strict;
+use Carp;
+use FindBin qw($Bin);
+use lib $Bin;
+use DNA;
+use Test;
+use List::Util qw(max min);
+use Math::Random;
+
+##
+# Default SNP rate generator. Doesn't generate the SNP per se, just
+# the rate.
+#
+sub defaultSNPGen() {
+ return Math::Random::random_uniform(1, 0, 0.05);
+}
+
+##
+# Default read gap rate generator. Doesn't generate the gaps or
+# lengths, just the rate.
+#
+sub defaultRdGapGen() {
+ return Math::Random::random_uniform(1, 0, 0.005);
+}
+
+##
+# Default reference gap rate generator. Doesn't generate the gaps or
+# lengths, just the rate.
+#
+sub defaultRfGapGen() {
+ return Math::Random::random_uniform(1, 0, 0.005);
+}
+
+##
+# Default rearrangement rate generator.
+#
+sub defaultRearrGen() {
+ return Math::Random::random_uniform(1, 0, 0.005);
+}
+
+##
+# Default function for generating gap lengths when introducing a gap.
+#
+sub defaultGapLenGen() {
+ return int(Math::Random::random_exponential(1, 3))+1;
+}
+
+##
+# Default function for generating random sequence to insert into a gap.
+#
+sub defaultSeqGen($) {
+ my $len = shift;
+ ($len == int($len) && $len > 0) ||
+ die "Bad length for sequence generator: $len";
+ my $ret = "";
+ for (1..$len) {
+ $ret .= substr("ACGT", int(rand(4)), 1);
+ }
+ return $ret;
+}
+
+##
+# Create a new DNA mutator
+#
+sub new {
+ my (
+ $class,
+ $name, # name
+ $snp, # SNP rate
+ $rdgap, # read gap rate
+ $rfgap, # ref gap rate
+ $rearr, # rearrangement rate
+ $gaplen, # gap length
+ $seqgen, # DNA generator
+ ) = @_;
+ $name = "noname" unless defined($name);
+ $snp = \&defaultSNPGen unless defined($snp);
+ $rdgap = \&defaultRdGapGen unless defined($rdgap);
+ $rfgap = \&defaultRfGapGen unless defined($rfgap);
+ $rearr = \&defaultRearrGen unless defined($rearr);
+ $gaplen = \&defaultGapLenGen unless defined($gaplen);
+ $seqgen = \&defaultSeqGen unless defined($seqgen);
+ return bless {
+ _name => $name,
+ _snp => $snp,
+ _rdgap => $rdgap,
+ _rfgap => $rfgap,
+ _rearr => $rearr,
+ _gaplen => $gaplen,
+ _seqgen => $seqgen,
+ }, $class;
+}
+sub snp { return $_[0]->{_snp} }
+sub rdgap { return $_[0]->{_rdgap} }
+sub rfgap { return $_[0]->{_rfgap} }
+sub rearr { return $_[0]->{_rearr} }
+sub gaplen { return $_[0]->{_gaplen} }
+sub seqgen { return $_[0]->{_seqgen} }
+
+##
+# Given a sequence (i.e. a key $srcchr into the reference hash),
+# mutate that string. Note that rearrangement mutations can affect
+# more than one sequence at a time.
+#
+# Returns a list containing counts for:
+#
+# 1: number of SNPs added
+# 2: number of read gaps added
+# 3: number of ref gaps added
+# 4: number of rearrangements added
+#
+sub mutateSeq {
+ my ($self, $srcchr, $ref) = @_;
+ my ($nsnp, $nrfgap, $nrdgap, $nrearr) = (0, 0, 0, 0);
+ my $mutseq = $ref->{$srcchr};
+ # Calculate # SNPs to add
+ my $len = length($mutseq);
+ my $snpRate = $self->snp->();
+ my $rfgapRate = $self->rfgap->();
+ my $rdgapRate = $self->rdgap->();
+ my $rearrRate = $self->rearr->();
+ $nsnp = Math::Random::random_binomial(1, $len, $snpRate);
+ $nrfgap = Math::Random::random_binomial(1, $len, $rfgapRate);
+ $nrdgap = Math::Random::random_binomial(1, $len, $rdgapRate);
+ $nrearr = Math::Random::random_binomial(1, $len, $rearrRate);
+ print STDERR " Introducing $nsnp SNPs, $nrfgap/$nrdgap ref/read gaps, and $nrearr rearrangements\n";
+ $nsnp = min($nsnp, $len);
+ # Add the SNPs
+ for (1..$nsnp) {
+ my $off = int(rand($len)); # where to mutate
+ my $add = int(rand(3))+1; # how to mutate
+ my $c = substr($mutseq, $off, 1);
+ $c eq "A" || $c eq "C" || $c eq "G" || $c eq "T" || $c eq "N" || die "Bad char '$c' in:\n$ref->{$srcchr}";
+ substr($mutseq, $off, 1) = DNA::plus(substr($mutseq, $off, 1), $add);
+ }
+ print STDERR " Finished SNPs\n";
+ # Calculate # ref gaps to add
+ for (1..$nrfgap) {
+ my $off = int(rand($len)); # where to mutate
+ my $gaplen = $self->gaplen->(); # how many gap positions in ref
+ # Insert characters into the subject genome
+ my $insseq = $self->seqgen->($gaplen);
+ substr($mutseq, $off, 0) = $insseq;
+ $len = length($mutseq);
+ }
+ print STDERR " Finished ref gaps\n";
+ # Calculate # read gaps to add
+ for (1..$nrdgap) {
+ my $off = int(rand($len)); # where to mutate
+ my $gaplen = $self->gaplen->(); # how many gap positions in ref
+ # Delete characters from subject genome
+ substr($mutseq, $off, $gaplen) = "";
+ $len = length($mutseq);
+ }
+ print STDERR " Finished read gaps\n";
+ $ref->{$srcchr} = $mutseq;
+ return ($nsnp, $nrfgap, $nrdgap, $nrearr);
+
+ my $totlen = 0;
+ for (keys %$ref) { $totlen += length($ref->{$_}); }
+ # Calculate # rearrangements to add
+ for (1..$nrearr) {
+ # Pick two loci, at least one on this reference sequence and
+ # then cross them over somehow
+ my $off = int(rand($len));
+ my @refkeys = keys %$ref;
+ my $ochr = $refkeys[int(rand(scalar(@refkeys)))];
+ my $oseq = $ref->{$ochr};
+ my $ooff = int(rand(length($oseq)));
+ my $srcleft = int(rand(2));
+ my $dstleft = int(rand(2));
+ my $srcrc = int(rand(2));
+ my $dstrc = int(rand(2));
+ # Check that the source and dest don't overlap
+ next if $srcchr eq $ochr;
+ # Get the sequence to move
+ my $presrclen = length($mutseq);
+ my $predstlen = length($oseq);
+ my $srcseq;
+ if($srcleft) {
+ $srcseq = substr($mutseq, 0, $off);
+ } else {
+ $srcseq = substr($mutseq, $off);
+ }
+ my $dstseq;
+ if($dstleft) {
+ $dstseq = substr($oseq, 0, $ooff);
+ } else {
+ $dstseq = substr($oseq, $ooff);
+ }
+ # Delete the sequence from the source
+ length($srcseq) <= length($mutseq) || die;
+ length($dstseq) <= length($oseq) || die;
+ if($srcleft) {
+ substr($mutseq, 0, length($srcseq)) = "";
+ } else {
+ substr($mutseq, -length($srcseq)) = "";
+ }
+ if($dstleft) {
+ substr($oseq, 0, length($dstseq)) = "";
+ } else {
+ substr($oseq, -length($dstseq)) = "";
+ }
+ # Possibly reverse the pieces we broke off
+ my $len1 = length($srcseq);
+ my $len2 = length($dstseq);
+ $srcseq = DNA::revcomp($srcseq) if $srcrc;
+ $dstseq = DNA::revcomp($dstseq) if $dstrc;
+ length($srcseq) == $len1 || die "$srcseq";
+ length($dstseq) == $len2 || die;
+ # Mutate the current chromosome
+ if($srcleft) {
+ $mutseq = $dstseq . $mutseq;
+ } else {
+ $mutseq = $mutseq . $dstseq;
+ }
+ # Mutate the other chromosome
+ if($dstleft) {
+ $oseq = $srcseq . $oseq;
+ } else {
+ $oseq = $oseq . $srcseq;
+ }
+ my $postsrclen = length($mutseq);
+ my $postdstlen = length($oseq);
+ ($presrclen + $presrclen) == ($postsrclen + $postsrclen) ||
+ die "from $srcchr to $ochr: $presrclen + $presrclen != $postsrclen + $postsrclen";
+ $ref->{$srcchr} = $mutseq;
+ $ref->{$ochr} = $oseq;
+ my $ntotlen = 0;
+ for (keys %$ref) { $ntotlen += length($ref->{$_}); }
+ $totlen == $ntotlen || die "Total length changed after rearrangements from $srcchr to $ochr ($totlen -> $ntotlen)";
+ }
+ print STDERR " Finished rearrangements\n";
+ $ref->{$srcchr} = $mutseq;
+ return ($nsnp, $nrfgap, $nrdgap, $nrearr);
+}
+
+sub test1 {
+ my $mut = Mutate->new("UnitTest mutator");
+ my %refs = (
+ "r1" => "TATGACGGTCGAAACCAGGCGA",
+ "r2" => "TATATTTAGTCTCGTCTGGCTGTCTCGGCTGCGCGCGAGTAAAGACCGGCCTGATC");
+ $mut->mutateSeq("r1", \%refs);
+ $mut->mutateSeq("r2", \%refs);
+ return 1;
+}
+
+sub test2 {
+ my $mut = Mutate->new(
+ "UnitTest mutator",
+ \&defaultSNPGen,
+ \&defaultRdGapGen,
+ \&defaultRfGapGen,
+ sub { return 0.1 },
+ \&defaultGapLenGen,
+ \&defaultSeqGen);
+ my %refs = (
+ "r1" => "TATGACGGTCGAAACCAGGCGA",
+ "r2" => "TATATTTAGTCTCGTCTGGCTGTCTCGGCTGCGCGCGAGTAAAGACCGGCCTGATC",
+ "r3" => "TATATTTAGTCTCGTCTGGCTGTCTCGGCTGCGCGCGAGTAAAGACCGGCCTGATC".
+ "ATTGGTGTCGCGGCGCGCGTATATATATATATATATAGCCTGCTACGTCAGCTAGC",
+ "r4" => "TATATTTAGTCTCGTCTGGCTGTCTCGGCTGCGCGCGAGTAAAGACCGGCCTGATC".
+ "ATTGGTGTCGCGGCGCGCGTATATATATATATATATAGCCTGCTACGTCAGCTAGC".
+ "ATATAACAAAAAAACCCCACACGACGCGGACTCTAGCACTATCGGACTATCATCGG");
+ $mut->mutateSeq("r1", \%refs);
+ $mut->mutateSeq("r2", \%refs);
+ $mut->mutateSeq("r3", \%refs);
+ $mut->mutateSeq("r4", \%refs);
+ return 1;
+}
+
+if($0 =~ /[^0-9a-zA-Z_]?Mutate\.pm$/) {
+ print "Running unit tests\n";
+ # Run unit tests
+ Test::shouldSucceed("test1", \&test1);
+ Test::shouldSucceed("test2", \&test2);
+}
+
+1;
diff --git a/scripts/sim/RandDNA.pm b/scripts/sim/RandDNA.pm
new file mode 100644
index 0000000..a924992
--- /dev/null
+++ b/scripts/sim/RandDNA.pm
@@ -0,0 +1,191 @@
+#!/usr/bin/perl -w
+
+#
+# Copyright 2011, Ben Langmead <langmea at cs.jhu.edu>
+#
+# This file is part of Bowtie 2.
+#
+# Bowtie 2 is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Bowtie 2 is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Bowtie 2. If not, see <http://www.gnu.org/licenses/>.
+#
+
+package RandDNA;
+use strict;
+use Carp;
+use FindBin qw($Bin);
+use lib $Bin;
+use DNA;
+use Test;
+use Math::Random;
+
+##
+# Create a new random DNA generator
+#
+sub new {
+ my (
+ $class,
+ $name, # name of generator
+ $n, # N fraction
+ $iupac, # Non-A/C/G/T/N IUPAC fraction (after N fraction removed)
+ $at, # AT fraction (after N/IUPAC fractions removed)
+ $a, # A fraction (of AT)
+ $c, # C fraction (of CG)
+ ) = @_;
+ $name = "noname" unless defined($name);
+ return bless {
+ _name => $name,
+ _n => $n || croak("No N frac"),
+ _iupac => $iupac || croak("No IUPAC frac"),
+ _at => $at || rand(),
+ _a => $a || rand(),
+ _c => $c || rand()
+ }, $class;
+}
+sub name { return $_[0]->{_name} }
+sub nFrac { return $_[0]->{_n} }
+sub iupacFrac { return $_[0]->{_iupac} }
+sub atFrac { return $_[0]->{_at} }
+sub aFrac { return $_[0]->{_a} }
+sub cFrac { return $_[0]->{_c} }
+
+##
+# Return a random IUPAC character.
+#
+sub randIUPAC() {
+ my @iu = (
+ "R",
+ "Y",
+ "M",
+ "K",
+ "S",
+ "W",
+ "B",
+ "V",
+ "H",
+ "D"
+ );
+ my $iui = int(rand(scalar(@iu)));
+ defined($iu[$iui]) || die "Bad index $iui";
+ return $iu[$iui];
+}
+
+##
+# Given parameters controlling character frequencies, build a palette
+# of short sequences that can be composed to make longer sequences.
+#
+sub genBuildingBlocks {
+ my ($self, $arr, $num, $bbrnd) = @_;
+ defined($arr) || die;
+ $num = 30 unless defined($num);
+ # Random generator for length
+ $bbrnd = sub { return int(rand(100))+1 } unless defined($bbrnd);
+ for my $i (1..$num) {
+ my $seq = "";
+ # Generate length
+ my $len = $bbrnd->();
+ $len > 0 || die "Bad length: $len";
+ # Generate characters
+ for my $j (1..$len) {
+ my $c = "";
+ if(rand() < $self->nFrac()) {
+ $c = "N";
+ } elsif(rand() < $self->iupacFrac()) {
+ $c = randIUPAC();
+ defined($c) || die;
+ } else {
+ $c = (rand() < $self->atFrac()) ? "AT" : "CG";
+ if($c eq "AT") {
+ $c = (rand() < $self->aFrac()) ? "A" : "T";
+ } else {
+ $c = (rand() < $self->cFrac()) ? "C" : "G";
+ }
+ }
+ $seq .= $c;
+ }
+ # Add to return list
+ push @$arr, $seq;
+ }
+}
+
+##
+# Use this generator to generate another random sequence.
+#
+sub nextSeq {
+ my ($self, $len, $bbsr, $runrnd) = @_;
+ my $seq = "";
+ # Generate building blocks
+ my @bbs = ();
+ if(defined($bbsr)) {
+ @bbs = @$bbsr;
+ } else {
+ $self->genBuildingBlocks(\@bbs);
+ }
+ scalar(@bbs) > 0 || die;
+ # Random generator for run length
+ my $defaultRnd = sub {
+ # Mean of exp is 1/lambda
+ return int(Math::Random::random_exponential(1, 2))+1;
+ };
+ $runrnd = $defaultRnd unless defined($runrnd);
+ # Build the sequence by repeatedly inserting runs of building blocks
+ while(length($seq) < $len * 5) {
+ # Choose building block
+ my $bbi = int(rand(scalar(@bbs)));
+ # Choose how many times to add it
+ my $runlen = $runrnd->();
+ # Choose insert point
+ my $insat = int(rand(length($seq)));
+ # Repeatedly insert building lock
+ for my $i (1..$runlen) {
+ substr($seq, $insat, 0) = $bbs[$bbi];
+ }
+ }
+ # Return chopped out piece
+ return substr($seq, $len * 2, $len);
+}
+
+sub test1 {
+ my $rd = RandDNA->new(
+ "randtest1", # name
+ 0.02, # n frac
+ 0.01, # IUPAC frac
+ 0.4, # AT frac
+ 0.45, # A/AT frac
+ 0.35); # C/CG frac
+ my $seq = $rd->nextSeq(200, undef);
+ length($seq) == 200 || die;
+ return 1;
+}
+
+sub test2 {
+ my @bb = ("AAA", "CCCC");
+ my $rd = RandDNA->new(
+ "randtest2", # name
+ 0.02, # n frac
+ 0.01, # IUPAC frac
+ 0.4, # AT frac
+ 0.45, # A/AT frac
+ 0.35); # C/CG frac
+ my $seq = $rd->nextSeq(300, \@bb);
+ length($seq) == 300 || die;
+ return 1;
+}
+
+if($0 =~ /[^0-9a-zA-Z_]?RandDNA\.pm$/) {
+ print "Running unit tests\n";
+ # Run unit tests
+ Test::shouldSucceed("test1", \&test1);
+ Test::shouldSucceed("test2", \&test2);
+}
+
+1;
diff --git a/scripts/sim/SampleRead.pm b/scripts/sim/SampleRead.pm
new file mode 100644
index 0000000..1defeed
--- /dev/null
+++ b/scripts/sim/SampleRead.pm
@@ -0,0 +1,244 @@
+#!/usr/bin/perl -w
+
+#
+# Copyright 2011, Ben Langmead <langmea at cs.jhu.edu>
+#
+# This file is part of Bowtie 2.
+#
+# Bowtie 2 is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Bowtie 2 is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Bowtie 2. If not, see <http://www.gnu.org/licenses/>.
+#
+
+package SampleRead;
+use strict;
+use Carp;
+use FindBin qw($Bin);
+use lib $Bin;
+use DNA;
+use Test;
+use List::Util qw(max min);
+use Math::Random;
+
+##
+# Default sequencing miscall rate generator.
+#
+sub defaultSeqMmGen() {
+ return Math::Random::random_uniform(1, 0, 0.1);
+}
+
+##
+# Default random generator for read length.
+#
+sub defaultFragLenGen() {
+ return int(Math::Random::random_normal(1, 200, 40))+1;
+}
+
+##
+# Default random generator for read length.
+#
+sub defaultReadLenGen() {
+ my $r = int(rand(3));
+ if($r == 0) {
+ return int(Math::Random::random_exponential(1, 60))+1;
+ } elsif($r == 1) {
+ return int(Math::Random::random_exponential(1, 20))+1;
+ } else {
+ return int(Math::Random::random_exponential(1, 150))+1;
+ }
+}
+
+##
+# Create a new read sampler
+#
+sub new {
+ my (
+ $class,
+ $name, # name of simulator
+ $fraglengen, # paired-end fragment length generator
+ $m1lengen, # random mate1 length generator
+ $m2lengen, # random mate2 length generator
+ ) = @_;
+ $fraglengen = \&defaultFragLenGen unless defined($fraglengen);
+ $m1lengen = \&defaultReadLenGen unless defined($m1lengen);
+ $m2lengen = \&defaultReadLenGen unless defined($m2lengen);
+ $name = "noname" unless defined($name);
+ return bless {
+ _name => $name,
+ _fraglengen => $fraglengen,
+ _m1lengen => $m1lengen,
+ _m2lengen => $m2lengen,
+ }, $class;
+}
+sub name { return $_[0]->{_name} }
+sub fraglengen { return $_[0]->{_fraglengen} }
+sub m1lengen { return $_[0]->{_m1lengen} }
+sub m2lengen { return $_[0]->{_m2lengen} }
+
+##
+# Generate a set of reads from a subject genome encoded in a hash ref.
+#
+sub genReads {
+ my (
+ $self,
+ $num, # number of reads/fragments to generate
+ $color, # colorize?
+ $refs, # hash ref holding reference sequences
+ $seqs, # put generated read sequences here
+ $quals, # put generated quality sequences here
+ $lengen) = @_; # length generator
+
+ ref($refs) eq "HASH" || die "Reference input must be hash ref, is ".ref($refs);
+ ref($seqs) eq "ARRAY" || die "seqs input must be array ref, is ".ref($seqs);
+ ref($quals) eq "ARRAY" || die "quals input must be array ref, is".ref($quals);
+ $lengen = $self->m1lengen() unless defined($lengen);
+ my $totreflen = 0;
+ my @keys = keys %$refs;
+ for (@keys) { $totreflen += length($refs->{$_}); }
+ for(1..$num) {
+ if(rand() < 0.05 && scalar(@$seqs) > 0) {
+ # Clone a previous read
+ my $ci = int(rand(scalar(@$seqs)));
+ push @$seqs, $seqs->[$ci];
+ push @$quals, $quals->[$ci];
+ } else {
+ while(1) {
+ my $ro = int(rand($totreflen));
+ my $len = $lengen->();
+ $len = 1 if $len < 1;
+ my $key = undef;
+ my $rflen = 0;
+ for (@keys) {
+ $rflen = length($refs->{$_});
+ if($ro < $rflen) {
+ $key = $_;
+ last;
+ }
+ $ro -= $rflen;
+ }
+ defined($key) || die;
+ $rflen > 0 || die;
+ # If we are overhanging the end, discard and try again
+ next if $ro + $len > $rflen;
+ my $rfseq = substr($refs->{$key}, $ro, $len);
+ length($rfseq) == $len || die;
+ my $rc = int(rand(2));
+ # Possibly reverse-complement it
+ $rfseq = DNA::revcomp($rfseq) if $rc == 1;
+ # Possible colorize
+ if($color) {
+ my $cseq = "";
+ for(0..$len-2) {
+ my ($c1, $c2) = (substr($rfseq, $_, 1), substr($rfseq, $_+1, 1));
+ my $col = DNA::dinucToColor($c1, $c2);
+ $cseq .= $col;
+ }
+ $rfseq = $cseq;
+ $len = length($rfseq);
+ }
+ push @$seqs, $rfseq;
+ # TODO: generate interesting qualities
+ push @$quals, "I" x $len;
+ last;
+ }
+ }
+ # Simulate next read
+ }
+}
+
+##
+# Generate a set of read pairs from a subject genome encoded in a hash
+# ref. First we extract unpaired fragments, then take sequences from
+# either end to make the mates.
+#
+sub genReadPairs {
+ my (
+ $self,
+ $num, # number of reads/fragments to generate
+ $color, # colorize?
+ $refs, # hash ref holding reference sequences
+ $m1fw, # orientation of mate 1 when fragment comes from Watson strand
+ $m2fw, # orientation of mate 2 when fragment comes from Watson strand
+ $seq1s, # put generated mate1 sequences here
+ $seq2s, # put generated mate2 sequences here
+ $qual1s, # put generated mate1 quality sequences here
+ $qual2s) = @_; # put generated mate2 quality sequences here
+
+ # First simulate fragments
+ ref($refs) eq "HASH" || die "Reference input must be hash ref";
+ ref($seq1s) eq "ARRAY" || die "seq1s input must be array ref";
+ ref($seq2s) eq "ARRAY" || die "seq2s input must be array ref";
+ ref($qual1s) eq "ARRAY" || die "qual1s input must be array ref";
+ ref($qual2s) eq "ARRAY" || die "qual2s input must be array ref";
+ my @fragseqs = ();
+ my @fragquals = ();
+ $self->genReads(
+ $num,
+ $color,
+ $refs,
+ \@fragseqs,
+ \@fragquals,
+ $self->fraglengen);
+ scalar(@fragseqs) == scalar(@fragquals) || die;
+ # For each fragment
+ for (1..scalar(@fragseqs)) {
+ # Take mates from either end
+ my $m1len = $self->m1lengen->();
+ my $m2len = $self->m2lengen->();
+ $m1len = min($m1len, length($fragseqs[$_-1]));
+ $m2len = min($m2len, length($fragseqs[$_-1]));
+ my $m1seq = substr($fragseqs [$_-1], 0, $m1len);
+ my $m2seq = substr($fragseqs [$_-1], -$m2len);
+ my $m1qual = substr($fragquals[$_-1], 0, $m1len);
+ my $m2qual = substr($fragquals[$_-1], -$m2len);
+ if(!$m1fw) {
+ $m1seq = DNA::revcomp($m1seq);
+ $m1qual = reverse $m1qual;
+ }
+ if(!$m2fw) {
+ $m2seq = DNA::revcomp($m2seq);
+ $m2qual = reverse $m2qual;
+ }
+ # Commit new pair to the list
+ push @$seq1s, $m1seq;
+ push @$seq2s, $m2seq;
+ push @$qual1s, $m1qual;
+ push @$qual2s, $m2qual;
+ # Simulate next pair
+ }
+}
+
+sub test1 {
+ my $samp = SampleRead->new("UnitTest read sampler");
+ my %refs = (
+ "r1" => "TATGACGGTCGAAACCAGGCGA",
+ "r2" => "TATATTTAGTCTCGTCTGGCTGTCTCGGCTGCGCGCGAGTAAAGACCGGCCTGATC");
+ my @seqs = ();
+ my @quals = ();
+ $samp->genReads(10, 0, \%refs, \@seqs, \@quals, \&defaultReadLenGen);
+ scalar(@seqs) == 10 || die;
+ scalar(@quals) == 10 || die;
+ return 1;
+}
+
+sub test2 {
+ return 1;
+}
+
+if($0 =~ /[^0-9a-zA-Z_]?SampleRead\.pm$/) {
+ print "Running unit tests\n";
+ # Run unit tests
+ Test::shouldSucceed("test1", \&test1);
+ Test::shouldSucceed("test2", \&test2);
+}
+
+1;
diff --git a/scripts/sim/Sim.pm b/scripts/sim/Sim.pm
new file mode 100644
index 0000000..8c27b05
--- /dev/null
+++ b/scripts/sim/Sim.pm
@@ -0,0 +1,1040 @@
+#!/usr/bin/perl -w
+
+#
+# Copyright 2011, Ben Langmead <langmea at cs.jhu.edu>
+#
+# This file is part of Bowtie 2.
+#
+# Bowtie 2 is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Bowtie 2 is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Bowtie 2. If not, see <http://www.gnu.org/licenses/>.
+#
+
+package Sim;
+use strict;
+use Carp;
+use FindBin qw($Bin);
+use lib $Bin;
+use DNA;
+use Test;
+use RandDNA;
+use SampleRead;
+use Mutate;
+use AlignmentCheck;
+use Math::Random;
+use List::Util qw(max min);
+use POSIX;
+use Sys::Info;
+use Sys::Info::Constants qw( :device_cpu );
+
+##
+# Replacement for "die" that additionally writes error message to file so that
+# run.pl can read it later.
+#
+sub mydie($) {
+ my $fn = ".run.pl.child.$$";
+ open(EO, ">$fn") || die "Could not open $fn for writing";
+ print EO "$_[0]\n";
+ close(EO);
+ confess $_[0];
+}
+
+# Generates random printable strings of a given length
+sub randStr($) {
+ my $len = shift;
+ my @chars = ('a'..'z', 'A'..'Z', '0'..'9', '_');
+ my $str = "";
+ foreach (1..$len) {
+ $str .= $chars[int(rand(scalar(@chars)))];
+ }
+ return $str;
+}
+
+##
+# Default random generator for number of reference per test case.
+#
+sub defaultRefNumGen() { return int(Math::Random::random_exponential(1, 8))+1; }
+
+##
+# Default random generator for reference length.
+#
+sub defaultRefLenGen() {
+ return int(Math::Random::random_exponential(1, 50000))+1;
+}
+
+##
+# Default random generator for number of reference per test case.
+#
+sub defaultReadNumGen() {
+ return int(Math::Random::random_exponential(1, 10000))+1;
+}
+
+##
+# Default random generator for read length.
+#
+sub defaultFragLenGen() {
+ return int(Math::Random::random_normal(1, 200, 40))+1;
+}
+
+##
+# Default random generator for reference length.
+#
+sub defaultReadLenGen() {
+ my $r = int(rand(3));
+ if($r == 0) {
+ return int(Math::Random::random_exponential(1, 60))+1;
+ } elsif($r == 1) {
+ return int(Math::Random::random_exponential(1, 20))+1;
+ } else {
+ return int(Math::Random::random_exponential(1, 150))+1;
+ }
+}
+
+##
+# Default random generator for fraction of reference characters = N.
+#
+sub defaultNGen() {
+ return Math::Random::random_uniform(1, 0, 0.05);
+}
+
+##
+# Default random generator for fraction of reference characters = an
+# ambiguous IUPAC code.
+#
+sub defaultIupacGen() {
+ return Math::Random::random_uniform(1, 0, 0.01);
+}
+
+##
+# Default random generator for AT/ACGT fraction.
+#
+sub defaultAtGen() {
+ return min(max(Math::Random::random_normal(1, 0.5, 0.18), 0), 1);
+}
+
+##
+# Default random generator for A/AT fraction.
+#
+sub defaultAGen() {
+ return min(max(Math::Random::random_normal(1, 0.5, 0.18), 0), 1);
+}
+
+##
+# Default random generator for C/CG fraction.
+#
+sub defaultCGen() {
+ return min(max(Math::Random::random_normal(1, 0.5, 0.18), 0), 1);
+}
+
+##
+# Default SNP rate generator. Doesn't generate the SNP per se, just
+# the rate.
+#
+sub defaultSNPGen() {
+ return Math::Random::random_uniform(1, 0, 0.05);
+}
+
+##
+# Default read gap rate generator. Doesn't generate the gaps or
+# lengths, just the rate.
+#
+sub defaultRdGapGen() {
+ return Math::Random::random_uniform(1, 0, 0.005);
+}
+
+##
+# Default reference gap rate generator. Doesn't generate the gaps or
+# lengths, just the rate.
+#
+sub defaultRfGapGen() {
+ return Math::Random::random_uniform(1, 0, 0.005);
+}
+
+##
+# Default rearrangement rate generator.
+#
+sub defaultRearrGen() {
+ return Math::Random::random_uniform(1, 0, 0.005);
+}
+
+##
+# Default function for generating gap lengths when introducing a gap.
+#
+sub defaultGapLenGen($) {
+ return int(Math::Random::random_exponential(1, 3))+1;
+}
+
+##
+# Default function for generating random sequence to insert into a gap.
+#
+sub defaultSeqGen($) {
+ my $len = shift;
+ ($len == int($len) && $len > 0) ||
+ mydie("Bad length for sequence generator: $len");
+ my $ret = "";
+ for (1..$len) {
+ $ret .= substr("ACGT", int(rand(4)), 1);
+ }
+ return $ret;
+}
+
+##
+# Default sequencing miscall rate generator.
+#
+sub defaultSeqMmGen() {
+ return Math::Random::random_uniform(1, 0, 0.1);
+}
+
+##
+# Create a new test case simulator
+#
+sub new {
+ my (
+ $class,
+ $name, # name of simulator
+ $rfnumgen, # number of reference sequences
+ $rflengen, # reference length
+ $rdnumgen, # number of read sequences per run
+ $rdlengen, # read length generator
+ $fraglengen, # fragment length generator
+ $ngen, # N fraction
+ $iupacgen, # Non-A/C/G/T/N IUPAC fraction (after N fraction removed)
+ $atgen, # AT fraction (after N/IUPAC fractions removed)
+ $agen, # A fraction (of AT)
+ $cgen, # C fraction (of CG)
+ $snpgen, # SNP rate gen
+ $rdgapgen, # read gap generator
+ $rfgapgen, # ref gap generator
+ $rearrgen, # rearrangement generator
+ $gaplengen, # gap length generator
+ $seqgen, # gap filler sequence generator
+ $seqmm, # sequencing error generator
+ ) = @_;
+ $rfnumgen = \&defaultRefNumGen unless defined($rfnumgen);
+ $rflengen = \&defaultRefLenGen unless defined($rflengen);
+ $rdnumgen = \&defaultReadNumGen unless defined($rdnumgen);
+ $rdlengen = \&defaultReadLenGen unless defined($rdlengen);
+ $fraglengen = \&defaultFragLenGen unless defined($fraglengen);
+ $ngen = \&defaultNGen unless defined($ngen);
+ $iupacgen = \&defaultIupacGen unless defined($iupacgen);
+ $atgen = \&defaultAtGen unless defined($atgen);
+ $agen = \&defaultAGen unless defined($agen);
+ $cgen = \&defaultCGen unless defined($cgen);
+ $snpgen = \&defaultSNPGen unless defined($snpgen);
+ $rdgapgen = \&defaultRdGapGen unless defined($rdgapgen);
+ $rfgapgen = \&defaultRfGapGen unless defined($rfgapgen);
+ $rearrgen = \&defaultRearrGen unless defined($rearrgen);
+ $gaplengen = \&defaultGapLenGen unless defined($gaplengen);
+ $seqgen = \&defaultSeqGen unless defined($seqgen);
+ $seqmm = \&defaultSeqMmGen unless defined($seqmm);
+ $name = "noname" unless defined($name);
+ return bless {
+ _name => $name,
+ _rfnumgen => $rfnumgen,
+ _rflengen => $rflengen,
+ _rdnumgen => $rdnumgen,
+ _rdlengen => $rdlengen,
+ _fraglengen => $fraglengen,
+ _ngen => $ngen,
+ _iupacgen => $iupacgen,
+ _atgen => $atgen,
+ _agen => $agen,
+ _cgen => $cgen,
+ _snpgen => $snpgen,
+ _rdgapgen => $rdgapgen,
+ _rfgapgen => $rfgapgen,
+ _rearrgen => $rearrgen,
+ _gaplengen => $gaplengen,
+ _seqgen => $seqgen,
+ _seqmm => $seqmm,
+ }, $class;
+}
+sub rfnumgen { return $_[0]->{_rfnumgen} }
+sub rflengen { return $_[0]->{_rflengen} }
+sub rdnumgen { return $_[0]->{_rdnumgen} }
+sub rdlengen { return $_[0]->{_rdlengen} }
+sub fraglengen { return $_[0]->{_fraglengen} }
+sub ngen { return $_[0]->{_ngen} }
+sub iupacgen { return $_[0]->{_iupacgen} }
+sub atgen { return $_[0]->{_atgen} }
+sub agen { return $_[0]->{_agen} }
+sub cgen { return $_[0]->{_cgen} }
+sub snpgen { return $_[0]->{_snpgen} }
+sub rdgapgen { return $_[0]->{_rdgapgen} }
+sub rfgapgen { return $_[0]->{_rfgapgen} }
+sub rearrgen { return $_[0]->{_rearrgen} }
+sub gaplengen { return $_[0]->{_gaplengen} }
+sub seqgen { return $_[0]->{_seqgen} }
+sub seqmm { return $_[0]->{_seqmm} }
+
+##
+# Generate DNA generator.
+#
+sub genDNAgen {
+ my $self = shift;
+ my $nfrac = $self->ngen->();
+ my $iupacfrac = $self->iupacgen->();
+ my $atfrac = $self->atgen->();
+ my $afrac = $self->agen->();
+ my $cfrac = $self->cgen->();
+ my $refdnagen = RandDNA->new(
+ "Sim.pm gen",
+ $nfrac,
+ $iupacfrac,
+ $atfrac,
+ $afrac,
+ $cfrac);
+ printf STDERR "Created DNA generator\n";
+ printf STDERR " N frac: %0.3f\n", $nfrac;
+ printf STDERR " IUPAC frac: %0.3f\n", $iupacfrac;
+ printf STDERR " AT/ACGT frac: %0.3f\n", $atfrac;
+ printf STDERR " A/AT frac: %0.3f\n", $afrac;
+ printf STDERR " C/CG frac: %0.3f\n", $cfrac;
+ return $refdnagen;
+}
+
+##
+# Generate and print reference sequences to file of given name. Also,
+# install reference sequences into hash ref $ref. To allow for
+# "overhang" (alignment that hang off the end of the reference), we
+# actually write out a little bit less than the full reference sequence
+# for each sequence.
+#
+sub genRef {
+ my ($self, $ref, $refdnagen, $conf, $tmpfn) = @_;
+ # Get a generator for reference length
+ my $reflen = $self->rflengen;
+ # Generate the number of references
+ my $refnum = $self->rfnumgen->();
+ $refnum = sqrt($refnum) if $conf->{small};
+ $refnum = 1 if $refnum <= 0;
+ $refnum = sqrt($refnum) if $conf->{small};
+ $refnum = 1 if $refnum <= 0;
+ $refnum = ceil($refnum);
+ $refnum = $conf->{numrefs} if defined($conf->{numrefs});
+ # Open output file
+ open (FA, ">$tmpfn") ||
+ mydie("Could not open temporary fasta file '$tmpfn' for writing");
+ my %ccnt = ();
+ print STDERR "Generating $refnum references\n";
+ for (1..$refnum) {
+ # Randomly generate length
+ my $len = $reflen->();
+ $len = sqrt($len) if $conf->{small};
+ $len = 1 if $len <= 0;
+ $len = ceil($len);
+ my $seq = $refdnagen->nextSeq($len);
+ length($seq) >= $len || die;
+ my $name = "Sim.pm.$_";
+ $ref->{$name} = $seq;
+ # Select amount to trim from LHS
+ my $trimleft = int(Math::Random::random_exponential(1, 200));
+ # Select amount to trim from RHS
+ my $trimright = int(Math::Random::random_exponential(1, 200));
+ # Make sure we're leaving some sequence after trimming
+ while($trimleft + $trimright > $len) {
+ if(int(rand(2))) {
+ $trimleft = int($trimleft*0.5);
+ } else {
+ $trimright = int($trimright*0.5);
+ }
+ }
+ # Trim the sequence
+ substr($seq, 0, $trimleft) = "";
+ $seq = substr($seq, 0, length($seq)-$trimright);
+ my $trimlen = length($seq);
+ $trimlen == $len - $trimleft - $trimright ||
+ mydie("Unexpected trim combo: $len, $trimleft, $trimright, $trimlen");
+ print STDERR " Generated reference '$name' of untrimmed length $len, trimmed length $trimlen\n";
+ print FA ">$name\n";
+ my $buf = "";
+ length($seq) >= $trimlen || die;
+ for my $i (1..$trimlen) {
+ my $c = substr($seq, $i-1, 1);
+ defined($c) || die;
+ $ccnt{$c}++;
+ $buf .= $c;
+ $ref->{$name} .= $c;
+ if($i % 60 == 0) {
+ print FA "$buf\n";
+ $buf = "";
+ }
+ }
+ print FA "$buf\n" if $buf ne "";
+ }
+ close(FA);
+ print STDERR "Wrote references to $tmpfn\n";
+ for my $k (sort keys %ccnt) {
+ print STDERR " $k: $ccnt{$k}\n";
+ }
+}
+
+##
+# Generate a hash of key/value arguments to pass to bowtie2.
+#
+sub genBuildArgs {
+ my ($self, $large_index) = @_;
+ my %args = ();
+ my $r1 = int(rand(3));
+ if($r1 == 0) {
+ $args{"--bmaxdivn"} = int(Math::Random::random_exponential(1, 4))+1;
+ } elsif($r1 == 1) {
+ $args{"--bmax"} = int(Math::Random::random_exponential(1, 10000))+100;
+ }
+ my $r2 = int(rand(2));
+ if($r2 == 0) {
+ $args{"--dcv"} = 2 ** (int(rand(10))+4);
+ }
+ my $r3 = int(rand(5));
+ if($r3 == 0) {
+ $args{"--packed"} = "";
+ }
+ my $r4 = int(rand(3));
+ if($r4 == 0) {
+ $args{"--offrate"} = int(rand(8))+1;
+ }
+ my $info = Sys::Info->new;
+ my $cpu = $info->device('CPU');
+ $args{"--threads"} = int(rand($cpu->count || 1)) + 1;
+ $args{"--large-index"} = "" if $large_index;
+ return \%args;
+}
+
+##
+# Given a fasta filename, an index basename, and a path to the
+# bowtie2-build executable, build nucleotide-space and colorpace
+# indexes for the sequences in the fasta file.
+#
+sub build {
+ my ($self, $fa, $idx, $conf, $args) = @_;
+ my $argstr = "";
+ for (keys %$args) {
+ $argstr .= " $_";
+ if($args->{$_} ne "") {
+ $argstr .= " ".$args->{$_};
+ }
+ }
+ $argstr .= " --sanity";
+ # Build nucleotide index
+ my $cmd = "$conf->{bowtie2_build_debug} $argstr $fa $idx";
+ print STDERR "Executing: $cmd\n";
+ system($cmd);
+ $? == 0 || mydie("Error running '$cmd'; exitlevel=$?");
+ print STDERR "Built nucleotide index '$idx'\n";
+ # Build colorspace index
+ unless($conf->{no_color}) {
+ $cmd = "$conf->{bowtie2_build_debug} $argstr -C $fa ${idx}.c";
+ print STDERR "$cmd\n";
+ system($cmd);
+ $? == 0 || mydie("Error running '$cmd'; exitlevel=$?");
+ print STDERR "Built colorspace index '$idx'\n";
+ }
+}
+
+##
+# Given a hash of sequences, flatten all IUPAC codes into unambiguous
+# nucleotides.
+#
+sub flattenIUPAC() {
+ my ($self, $h) = @_;
+ for my $c (keys %$h) {
+ my $len = length($h->{$c});
+ for my $i (0..$len-1) {
+ my $ch = uc substr($h->{$c}, $i, 1);
+ my $nc = $ch;
+ if(DNA::isIUPAC($ch) || $ch eq "N") {
+ if(rand() < $self->snpgen->()) {
+ $nc = DNA::pickIncompat($ch);
+ defined($nc) || mydie("Couldn't find incompatible base for $ch");
+ } else {
+ $nc = DNA::pickCompat($ch);
+ defined($nc) || mydie("Couldn't find compatible base for $ch");
+ }
+ }
+ if($ch ne $nc) {
+ substr($h->{$c}, $i, 1) = $nc;
+ }
+ }
+ }
+}
+
+##
+# Mutate reference genome into a subject genome.
+#
+sub mutate() {
+ my ($self, $refs) = @_;
+ my %subj = %$refs;
+ $self->flattenIUPAC(\%subj);
+ print STDERR "Flattened IUPAC characters\n";
+ my $mutator = Mutate->new(
+ "Sim.pm mutator",
+ $self->snpgen,
+ $self->rdgapgen,
+ $self->rfgapgen,
+ $self->rearrgen,
+ $self->gaplengen,
+ $self->seqgen);
+ my ($nsnp, $nrfgap, $nrdgap, $nrearr) = (0, 0, 0, 0);
+ for(keys %subj) {
+ print STDERR " Mutating sequence $_\n";
+ my ($nsnp_, $nrfgap_, $nrdgap_, $nrearr_) = $mutator->mutateSeq($_, \%subj);
+ $nsnp += $nsnp_;
+ $nrfgap += $nrfgap_;
+ $nrdgap += $nrdgap_;
+ $nrearr += $nrearr_;
+ }
+ print STDERR "Mutated reference genome to subject genome\n";
+ print STDERR " SNPs introduced: $nsnp\n";
+ print STDERR " Reference gaps introduced: $nrfgap\n";
+ print STDERR " Read gaps introduced: $nrdgap\n";
+ print STDERR " Rearrangements introduced: $nrearr\n";
+ return \%subj;
+}
+
+sub dumpFastq {
+ my ($self, $input, $fh1, $fh2) = @_;
+ for (1..scalar(@{$input->{seq1s}})) {
+ my $seq1 = $input->{seq1s}->[$_-1];
+ my $qual1 = $input->{qual1s}->[$_-1];
+ print {$fh1} "\@$_\n";
+ print {$fh1} "$seq1\n";
+ print {$fh1} "+$_\n";
+ print {$fh1} "$qual1\n";
+ if($input->{paired}) {
+ my $seq2 = $input->{seq2s}->[$_-1];
+ my $qual2 = $input->{qual2s}->[$_-1];
+ print {$fh2} "\@$_\n";
+ print {$fh2} "$seq2\n";
+ print {$fh2} "+$_\n";
+ print {$fh2} "$qual2\n";
+ }
+ }
+}
+
+sub dumpQseq {
+ my ($self, $input, $fh1, $fh2) = @_;
+ for (1..scalar(@{$input->{seq1s}})) {
+ my $seq1 = $input->{seq1s}->[$_-1];
+ my $qual1 = $input->{qual1s}->[$_-1];
+ print {$fh1} "R\t1\t1\t1\t$_\t$_\t1\t1\t$seq1\t$qual1\t1\n";
+ if($input->{paired}) {
+ my $seq2 = $input->{seq2s}->[$_-1];
+ my $qual2 = $input->{qual2s}->[$_-1];
+ print {$fh2} "R\t1\t1\t1\t$_\t$_\t1\t1\t$seq2\t$qual2\t1\n";
+ }
+ }
+}
+
+sub dumpFasta {
+ my ($self, $input, $fh1, $fh2) = @_;
+ for (1..scalar(@{$input->{seq1s}})) {
+ my $seq1 = $input->{seq1s}->[$_-1];
+ print {$fh1} ">$_\n";
+ print {$fh1} "$seq1\n";
+ if($input->{paired}) {
+ my $seq2 = $input->{seq2s}->[$_-1];
+ print {$fh2} ">$_\n";
+ print {$fh2} "$seq2\n";
+ }
+ }
+}
+
+sub dumpRaw {
+ my ($self, $input, $fh1, $fh2) = @_;
+ for (1..scalar(@{$input->{seq1s}})) {
+ my $seq1 = $input->{seq1s}->[$_-1];
+ print {$fh1} "$seq1\n";
+ if($input->{paired}) {
+ my $seq2 = $input->{seq2s}->[$_-1];
+ print {$fh2} "$seq2\n";
+ }
+ }
+}
+
+##
+# Generate the input (reads plus paired/fragment information)
+#
+sub genInput {
+ my ($self, $refs, $conf) = @_;
+ # Select whether we're doing colorspace
+ my $color = int(rand(2));
+ $color = 0 if $conf->{no_color};
+ # Select whether we're doing unpaired or paired-end.
+ my $paired = int(rand(2));
+ $paired = 0 if $conf->{no_paired};
+ # Select format for read file
+ my @formats = ("fastq", "qseq", "fasta", "raw");
+ my @format_arg = ( "-q", "--qseq", "-f", "-r");
+ my $formati = int(rand(scalar(@formats)));
+ my $format = $formats[$formati];
+ my $format_arg = $format_arg[$formati];
+ my $tmprdfn1 = "$conf->{tempdir}/Sim.pm.$conf->{randstr}_1.$format";
+ my $tmprdfn2 = "$conf->{tempdir}/Sim.pm.$conf->{randstr}_2.$format";
+ # Generate reads from the subject genome; no sequencing error yet
+ my %input = (
+ seq1s => [],
+ seq2s => [],
+ qual1s => [],
+ qual2s => [],
+ mate1fw => 1,
+ mate2fw => 0,
+ paired => $paired,
+ color => $color,
+ format => $format,
+ format_arg => $format_arg,
+ file1 => $tmprdfn1,
+ file2 => $tmprdfn2 );
+ my $read_sampler = SampleRead->new(
+ "Sim.pm read sampler",
+ $self->fraglengen,
+ $self->rdlengen,
+ $self->rdlengen);
+ print STDERR "Created read sampler\n";
+ my $numreads = $self->rdnumgen->();
+ $numreads = ceil(sqrt($numreads)) if $conf->{small};
+ $numreads == int($numreads) || mydie("numreads $numreads not a number");
+ my $tmp = int(rand(3));
+ if($tmp == 0) {
+ $input{mate2fw} = 1;
+ } elsif($tmp == 1) {
+ $input{mate1fw} = 0;
+ $input{mate2fw} = 1;
+ }
+ print STDERR "Sampling $numreads reads\n";
+ ref($refs) eq "HASH" || mydie("Reference input must be hash ref");
+ if($paired) {
+ $read_sampler->genReadPairs(
+ $numreads, # number of reads/fragments to generate
+ $input{color}, # colorize?
+ $refs, # hash ref holding reference sequences
+ $input{mate1fw}, # orientation of mate 1 when fragment comes from Watson strand
+ $input{mate2fw}, # orientation of mate 2 when fragment comes from Watson strand
+ $input{seq1s}, # put generated mate1 sequences here
+ $input{seq2s}, # put generated mate2 sequences here
+ $input{qual1s}, # put generated mate1 quality sequences here
+ $input{qual2s}); # put generated mate2 quality sequences here
+ } else {
+ $read_sampler->genReads(
+ $numreads, # number of reads/fragments to generate
+ $input{color}, # colorize?
+ $refs, # hash ref holding reference sequences
+ $input{seq1s}, # put generated sequences here
+ $input{qual1s}); # put generated quality sequences here
+ }
+ # TODO: with some probability, sort the reads
+ print STDERR "Dumping reads to temporary files $tmprdfn1 & $tmprdfn2\n";
+ # Dump reads to output file
+ my ($fh1, $fh2);
+ open($fh1, ">$tmprdfn1") || mydie("Could not open '$tmprdfn1' for writing");
+ open($fh2, ">$tmprdfn2") || mydie("Could not open '$tmprdfn2' for writing");
+ if($format eq "fastq") {
+ $self->dumpFastq(\%input, $fh1, $fh2);
+ } elsif($format eq "qseq") {
+ $self->dumpQseq(\%input, $fh1, $fh2);
+ } elsif($format eq "fasta") {
+ $self->dumpFasta(\%input, $fh1, $fh2);
+ } elsif($format eq "raw") {
+ $self->dumpRaw(\%input, $fh1, $fh2);
+ }
+ close($fh1);
+ close($fh2);
+ return \%input;
+}
+
+##
+# Mutate reads according to sequencing error model.
+#
+sub mutateSeq {
+ my ($self, $input) = @_;
+ return $input;
+}
+
+##
+# Generate a setting for MA (match bonus).
+#
+sub genPolicyMA($) {
+ my $local = shift;
+ return undef if $local;
+ return Math::Random::random_uniform(1, 1, 40)
+}
+
+##
+# Generate a setting for MMP (mismatch penalty).
+#
+sub genPolicyMMP() {
+ #my $op = substr("CQR", int(rand(3)), 1);
+ #if($op eq "C") {
+ my $op1 = Math::Random::random_uniform(1, 1, 10);
+ my $op2 = Math::Random::random_uniform(1, 1, 10);
+ #}
+ return max($op1, $op2).",".min($op1, $op2);
+}
+
+##
+# Generate a setting for NP (penalty for a mismatch involving an N).
+#
+sub genPolicyNP() {
+ #my $op = substr("CQR", int(rand(3)), 1);
+ #if($op eq "C") {
+ my $op = int(Math::Random::random_exponential(1, 3))+1;
+ #}
+ return $op;
+}
+
+##
+# Generate a setting for RDG (read gap open and extend penalties).
+#
+sub genPolicyRDG() {
+ my $op = Math::Random::random_uniform(1, 1, 50);
+ if(int(rand(2)) == 0) {
+ $op .= ",";
+ $op .= Math::Random::random_uniform(1, 1, 20);
+ }
+ return "$op";
+}
+
+##
+# Generate a setting for RFG (ref gap open and extend penalties).
+#
+sub genPolicyRFG() {
+ my $op = Math::Random::random_uniform(1, 1, 50);
+ if(int(rand(2)) == 0) {
+ $op .= ",";
+ $op .= Math::Random::random_uniform(1, 1, 20);
+ }
+ return "$op";
+}
+
+##
+# Generate a setting for MIN (function determining minimum acceptable score).
+#
+sub genPolicyMIN($) {
+ my $local = shift;
+ my $xx = Math::Random::random_uniform(1, 1, 10);
+ my $yy = Math::Random::random_uniform(1, 1, 10);
+ return "L,$xx,$yy";
+}
+
+##
+# Generate a setting for NCEIL (function determining maximum number of Ns
+# allowed).
+#
+sub genPolicyNCEIL() {
+ return undef if int(rand(2)) == 0;
+ my $xx = Math::Random::random_uniform(1, 0, 1.5);
+ my $yy = Math::Random::random_uniform(1, 0, 1.5);
+ return "$xx,$yy";
+}
+
+##
+# Generate a setting for SEED (# mismatches, length, interval).
+#
+sub genPolicySEED() {
+ return undef if int(rand(2)) == 0;
+ # Pick a number of mismatches
+ my $sd = substr("012", int(rand(2)), 1);
+ if(rand() < 0.9) {
+ # Length
+ $sd .= ",".int(Math::Random::random_uniform(1, 12, 32));
+ }
+ return $sd;
+}
+
+##
+# Generate a setting for -D (# DP fails in a row).
+#
+sub genPolicyFailStreak() {
+ return undef if int(rand(2)) == 0;
+ return int(Math::Random::random_uniform(1, 2, 50));
+}
+
+##
+# Generate a setting for -R (# seeding rounds).
+#
+sub genPolicySeedRounds() {
+ return undef if int(rand(2)) == 0;
+ return int(Math::Random::random_uniform(1, 1, 5));
+}
+
+##
+# Generate a setting for IVAL. Interval between seeds is a function of the
+# read length OR sqaure root of read length OR cube root of read length.
+#
+sub genPolicyIVAL() {
+ return undef if int(rand(2)) == 0;
+ # Pick a number of mismatches
+ my $iv = substr("LSC", int(rand(3)), 1);
+ if($iv eq "L") {
+ if(rand() < 0.9) {
+ # Multiplier
+ $iv .= ",".Math::Random::random_uniform(1, 0.0, 0.5);
+ }
+ if(rand() < 0.3) {
+ # Offset
+ $iv .= ",".Math::Random::random_uniform(1, 0.0, 4.0);
+ }
+ } elsif($iv eq "S") {
+ if(rand() < 0.9) {
+ # Multiplier
+ $iv .= ",".Math::Random::random_uniform(1, 0.0, 3.0);
+ }
+ if(rand() < 0.3) {
+ # Offset
+ $iv .= ",".Math::Random::random_uniform(1, 0.0, 7.0);
+ }
+ } elsif($iv eq "C") {
+ if(rand() < 0.9) {
+ # Multiplier
+ $iv .= ",".Math::Random::random_uniform(1, 0.0, 5.0);
+ }
+ if(rand() < 0.3) {
+ # Offset
+ $iv .= ",".Math::Random::random_uniform(1, 0.0, 14.0);
+ }
+ }
+ return $iv;
+}
+
+##
+# Generate a hash of key/value arguments to pass to bowtie2.
+#
+sub genAlignArgs {
+ my ($self, $input, $color, $large_index, $conf) = @_;
+ my %args = ();
+ my $local = int(rand(2)) == 0;
+ $args{"-u"} = $conf->{maxreads} if defined($conf->{maxreads});
+ $args{"--mm"} = "" if int(rand(2)) == 0;
+ $args{"--trim3"} = int(rand(10)) if int(rand(2)) == 0;
+ $args{"--trim5"} = int(rand(10)) if int(rand(2)) == 0;
+ $args{"--nofw"} = "" if int(rand(4)) == 0;
+ $args{"--norc"} = "" if int(rand(4)) == 0;
+ $args{"--col-keepends"} = "" if ($color && int(rand(3)) == 0);
+ $args{"--gbar"} = int(Math::Random::random_exponential(1, 3))+1 if int(rand(4)) == 0;
+ $args{"--local"} = "" if $local;
+ my $rep = int(rand(5));
+ if($rep == 0) {
+ $args{"-a"} = "";
+ } elsif($rep == 1) {
+ $args{"-k"} = int(Math::Random::random_exponential(1, 3))+2;
+ } elsif($rep == 2) {
+ $args{"-M"} = int(Math::Random::random_exponential(1, 3))+2;
+ }
+ $args{"--rdg"} = genPolicyRDG() if rand() < 0.5;
+ $args{"--rfg"} = genPolicyRFG() if rand() < 0.5;
+ $args{"--score-min"} = genPolicyMIN($local);
+ $args{"--n-ceil"} = genPolicyNCEIL() if rand() < 0.5;
+ $args{"-N"} = genPolicySEED() if rand() < 0.5;
+ $args{"-D"} = genPolicyFailStreak() if rand() < 0.5;
+ $args{"-R"} = genPolicySeedRounds() if rand() < 0.5;
+ $args{"--ma"} = genPolicyMA($local) if rand() < 0.5;
+ $args{"--mp"} = genPolicyMMP() if rand() < 0.5;
+ $args{"--np"} = genPolicyNP() if rand() < 0.5;
+ $args{"-i"} = genPolicyIVAL() if rand() < 0.5;
+ $args{"--large-index"} = "" if $large_index;
+ $args{"--cp-min"} = int(Math::Random::random_exponential(1, 3)) + 2;
+ $args{"--cp-ival"} = int(Math::Random::random_exponential(1, 1)) + 1;
+ return \%args;
+}
+
+##
+# Align the given input set against the given index using the given
+# bowtie2 binary and arguments. Sanity-check the SAM output.
+#
+sub align {
+ my ($self, $fa, $idx, $input, $conf, $args) = @_;
+ my $argstr = "";
+ for (keys %$args) {
+ if(defined($args->{$_})) {
+ $argstr .= " $_";
+ if($args->{$_} ne "") {
+ $argstr .= " ".$args->{$_};
+ }
+ }
+ }
+ $argstr .= " -C" if $input->{color};
+ $argstr .= " ".$input->{format_arg};
+ $idx .= ".c" if $input->{color};
+ my $inputfn;
+ if($input->{paired}) {
+ $inputfn = "-1 $input->{file1} -2 $input->{file2}";
+ } else {
+ $inputfn = $input->{file1};
+ }
+ # Create object that will help us sanity-check alignments
+ my $ac = AlignmentCheck->new(
+ "Sim.pm alignment checker", # name
+ [ $fa ], # fasta
+ "sam", # SAM-formatted alignments
+ 0, # no bis-C
+ 0 # no bis-CpG
+ );
+ $ac->nrefs() > 0 || mydie("No references");
+ # Run normal (non-debug) Bowtie
+ defined($conf->{tempdir}) || mydie("No tmp dir");
+ my $als = "$conf->{tempdir}/Sim.pm.$conf->{randstr}.als";
+ my $als_debug = "$conf->{tempdir}/Sim.pm.$conf->{randstr}.debug.als";
+ my $als_px = "$conf->{tempdir}/Sim.pm.$conf->{randstr}.px.als";
+ my $als_px_reord = "$conf->{tempdir}/Sim.pm.$conf->{randstr}.px.reord.als";
+ my $cmd = "$conf->{bowtie2_debug} $argstr -x $idx $inputfn";
+ print "$cmd\n";
+ open(ALSDEB, ">$als_debug") || mydie("Could not open '$als_debug' for writing");
+ open(ALSDEBCMD, "$cmd |") || mydie("Could not open pipe '$cmd |'");
+ my $ival = 50;
+ my $nals = 0;
+ my @lines = ();
+ while(<ALSDEBCMD>) {
+ # Remove @PG line because CL: tag can legitimately differ
+ print ALSDEB $_ unless /^\@PG/;
+ push @lines, $_;
+ $nals++;
+ print STDERR " Read $nals alignments...\n" if ($nals % $ival) == 0;
+ }
+ close(ALSDEBCMD);
+ $ac->checkAlignments(\@lines, 0);
+ $? == 0 || mydie("bowtie2-align-debug exited with exitlevel $?:\n$cmd\n");
+ close(ALSDEB);
+ $ac->printSummary();
+ # With some probability, also run debug Bowtie and check that
+ # results are identical
+ if(int(rand(3)) == 0) {
+ print STDERR "ALSO checking that bowtie2 and bowtie2-align-debug match up\n";
+ # Remove @PG line because CL: tag can legitimately differ
+ $cmd = "$conf->{bowtie2} $argstr -x $idx $inputfn | grep -v '^\@PG' > $als";
+ print "$cmd\n";
+ system($cmd);
+ $? == 0 ||
+ mydie("Command '$cmd' failed with exitlevel $?");
+ $cmd = "diff -uw $als $als_debug";
+ print "$cmd\n";
+ system($cmd);
+ $? == 0 ||
+ mydie("diff found a difference between bowtie2 and bowtie2-align-debug ".
+ "output for same input (above)\n");
+ }
+ # With some probability, also run debug Bowtie in -p X mode with X > 1 and
+ # without the --reorder argument and check that results are identical
+ if(int(rand(3)) == 0) {
+ print STDERR "ALSO checking that bowtie2 and bowtie2 -p X w/ X > 1 match up\n";
+ my $p = int(rand(3))+2;
+ $cmd = "$conf->{bowtie2} $argstr -p $p -x $idx $inputfn | grep -v '^\@PG' > $als_px";
+ print "$cmd\n";
+ system($cmd);
+ $? == 0 ||
+ mydie("Command '$cmd' failed with exitlevel $?");
+ # Sort the $als_px and $als_debug files to guarantee that reads and
+ # alignments for a given read appear in the same order in both
+ $cmd = "sort -k 1,1 -n -k 2,2 -k 3,3 -k 4,4 < $als_px | grep -v '^\@PG' > $als_px.sorted";
+ print "$cmd\n";
+ system($cmd);
+ $? == 0 ||
+ mydie("Failed to sort alignment file $als_px\n");
+ # Sort the $als_px and $als_debug files to guarantee that reads and
+ # alignments for a given read appear in the same order in both
+ $cmd = "sort -k 1,1 -n -k 2,2 -k 3,3 -k 4,4 < $als_debug | grep -v '^\@PG' > $als_debug.sorted";
+ print "$cmd\n";
+ system($cmd);
+ $? == 0 ||
+ mydie("Failed to sort alignment file $als_debug\n");
+ $cmd = "diff -uw $als_debug.sorted $als_px.sorted";
+ print "$cmd\n";
+ system($cmd);
+ $? == 0 ||
+ mydie("diff found a difference between bowtie2-align-debug and bowtie2 ".
+ "-p output for same input (above)\n");
+ }
+
+ # With some probability, also run debug Bowtie in -p X mode with X > 1 and
+ # with the --reorder argument and check that results are identical
+ if(int(rand(3)) == 0) {
+ print STDERR "ALSO checking that bowtie2 and bowtie2 -p X --reorder w/ X > 1 match up\n";
+ my $p = int(rand(3))+2;
+ $cmd = "$conf->{bowtie2} $argstr -p $p -x $idx --reorder $inputfn | grep -v '^\@PG' > $als_px_reord";
+ print "$cmd\n";
+ system($cmd);
+ $? == 0 || mydie("Command '$cmd' failed with exitlevel $?");
+ $cmd = "diff -uw $als_debug $als_px_reord";
+ print "$cmd\n";
+ system($cmd);
+ $? == 0 ||
+ mydie("diff found a difference between bowtie2-align-debug and bowtie2 ".
+ "-p --reorder output for same input (above)\n");
+ }
+}
+
+##
+# Generate a new test case
+#
+# Possible key/value pairs in $conf hash:
+#
+# 1. bowtie2_build: path to bowtie2-build binary
+# 2. bowtie2: path to bowtie2 binary
+# 3. bowtie2_build_debug: path to bowtie2-build-debug binary
+# 4. bowtie2_debug: path to bowtie2-debug binary
+# 5. tempdir: temporary directory for reference/reads/index
+# 6. no_paired: defined & non-0 -> don't generate paired-end datasets
+# 7. no_color: defined & non-0 -> don't generate colorspace datasets
+# 8. single_thread: defined & non-0 -> don't use -p X where X > 1
+#
+sub nextCase {
+ my ($self, $conf) = @_;
+
+ $conf->{bowtie2_build} = "bowtie2-build" unless defined($conf->{bowtie2_build});
+ $conf->{bowtie2} = "bowtie2-align" unless defined($conf->{bowtie2});
+ $conf->{bowtie2_build_debug} = $conf->{bowtie2_build}." --debug";
+ $conf->{bowtie2_debug} = $conf->{bowtie2}." --debug";
+ $conf->{tempdir} = "/tmp" unless defined($conf->{tempdir});
+ srand(time ^ $$);
+ $conf->{randstr} = randStr(8);
+
+ print "*** TEST CASE ***\n";
+
+ # Build a large index?
+ my $large_index = int(rand(2)) == 0;
+
+ # Generate the references
+ my $refdnagen = $self->genDNAgen();
+ # Generate references and write them to a temporary fasta file
+ my $tmpfn = "$conf->{tempdir}/Sim.pm.$conf->{randstr}.fa";
+ my %refs = ();
+ $self->genRef(\%refs, $refdnagen, $conf, $tmpfn);
+ # Run bowtie2-build
+ my $tmpidxfn = "$conf->{tempdir}/Sim.pm.$conf->{randstr}";
+ my $buildArgs = $self->genBuildArgs($large_index);
+ $self->build($tmpfn, $tmpidxfn, $conf, $buildArgs);
+ my $numruns = 10;
+ $numruns *= 10 if $conf->{small}; # Lots of short runs
+ # For each batch of reads / bowtie options
+ for(1..$numruns) {
+ print "*** Run $_ of $numruns\n";
+ # Generate mutated version of the reference as our subject genome
+ my $subj = $self->mutate(\%refs);
+ # Generate all the input, including reads, pairedness,
+ # fragment information, whether it's colorspace, etc
+ my $input = $self->genInput($subj, $conf);
+ # Mutate the input
+ my $mutinput = $self->mutateSeq($input);
+ # Select Bowtie arguments
+ my $args = $self->genAlignArgs($mutinput, $input->{color}, $large_index, $conf);
+ $self->align($tmpfn, $tmpidxfn, $mutinput, $conf, $args);
+ # Sanity check output. Possible sanity checks are:
+ # 1. Check alignments & edits against reference
+ # 2. Compare bowtie2 and bowtie2-debug
+ # 3. Compare -p X>1 and -p 1
+ }
+}
+
+if($0 =~ /Sim\.pm$/) {
+ print "Running unit tests\n";
+ # Run unit tests
+}
+
+1;
diff --git a/scripts/sim/Test.pm b/scripts/sim/Test.pm
new file mode 100644
index 0000000..82b3793
--- /dev/null
+++ b/scripts/sim/Test.pm
@@ -0,0 +1,47 @@
+#!/usr/bin/perl -w
+
+#
+# Copyright 2011, Ben Langmead <langmea at cs.jhu.edu>
+#
+# This file is part of Bowtie 2.
+#
+# Bowtie 2 is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Bowtie 2 is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Bowtie 2. If not, see <http://www.gnu.org/licenses/>.
+#
+
+package Test;
+use strict;
+use Carp;
+
+sub test($$) {
+ my ($name, $f) = @_;
+ print "Test \"$name\"...";
+ $f->();
+ print "PASSED\n";
+}
+
+sub shouldSucceed($$) {
+ my ($name, $f) = @_;
+ print "Test \"$name\"...";
+ $f->() || die "";
+ print "PASSED\n";
+}
+
+sub shouldFail($$) {
+ my ($name, $f) = @_;
+ print "Test \"$name\"...";
+ $f->() && die "";
+ print "PASSED\n";
+}
+
+1;
diff --git a/scripts/sim/contrib/ForkManager.pm b/scripts/sim/contrib/ForkManager.pm
new file mode 100644
index 0000000..331c59c
--- /dev/null
+++ b/scripts/sim/contrib/ForkManager.pm
@@ -0,0 +1,412 @@
+=head1 NAME
+
+Parallel::ForkManager - A simple parallel processing fork manager
+
+=head1 SYNOPSIS
+
+ use Parallel::ForkManager;
+
+ $pm = new Parallel::ForkManager($MAX_PROCESSES);
+
+ foreach $data (@all_data) {
+ # Forks and returns the pid for the child:
+ my $pid = $pm->start and next;
+
+ ... do some work with $data in the child process ...
+
+ $pm->finish; # Terminates the child process
+ }
+
+=head1 DESCRIPTION
+
+This module is intended for use in operations that can be done in parallel
+where the number of processes to be forked off should be limited. Typical
+use is a downloader which will be retrieving hundreds/thousands of files.
+
+The code for a downloader would look something like this:
+
+ use LWP::Simple;
+ use Parallel::ForkManager;
+
+ ...
+
+ @links=(
+ ["http://www.foo.bar/rulez.data","rulez_data.txt"],
+ ["http://new.host/more_data.doc","more_data.doc"],
+ ...
+ );
+
+ ...
+
+ # Max 30 processes for parallel download
+ my $pm = new Parallel::ForkManager(30);
+
+ foreach my $linkarray (@links) {
+ $pm->start and next; # do the fork
+
+ my ($link,$fn) = @$linkarray;
+ warn "Cannot get $fn from $link"
+ if getstore($link,$fn) != RC_OK;
+
+ $pm->finish; # do the exit in the child process
+ }
+ $pm->wait_all_children;
+
+First you need to instantiate the ForkManager with the "new" constructor.
+You must specify the maximum number of processes to be created. If you
+specify 0, then NO fork will be done; this is good for debugging purposes.
+
+Next, use $pm->start to do the fork. $pm returns 0 for the child process,
+and child pid for the parent process (see also L<perlfunc(1p)/fork()>).
+The "and next" skips the internal loop in the parent process. NOTE:
+$pm->start dies if the fork fails.
+
+$pm->finish terminates the child process (assuming a fork was done in the
+"start").
+
+NOTE: You cannot use $pm->start if you are already in the child process.
+If you want to manage another set of subprocesses in the child process,
+you must instantiate another Parallel::ForkManager object!
+
+=head1 METHODS
+
+=over 5
+
+=item new $processes
+
+Instantiate a new Parallel::ForkManager object. You must specify the maximum
+number of children to fork off. If you specify 0 (zero), then no children
+will be forked. This is intended for debugging purposes.
+
+=item start [ $process_identifier ]
+
+This method does the fork. It returns the pid of the child process for
+the parent, and 0 for the child process. If the $processes parameter
+for the constructor is 0 then, assuming you're in the child process,
+$pm->start simply returns 0.
+
+An optional $process_identifier can be provided to this method... It is used by
+the "run_on_finish" callback (see CALLBACKS) for identifying the finished
+process.
+
+=item finish [ $exit_code ]
+
+Closes the child process by exiting and accepts an optional exit code
+(default exit code is 0) which can be retrieved in the parent via callback.
+If you use the program in debug mode ($processes == 0), this method doesn't
+do anything.
+
+=item set_max_procs $processes
+
+Allows you to set a new maximum number of children to maintain. Returns
+the previous setting.
+
+=item wait_all_children
+
+You can call this method to wait for all the processes which have been
+forked. This is a blocking wait.
+
+=back
+
+=head1 CALLBACKS
+
+You can define callbacks in the code, which are called on events like starting
+a process or upon finish.
+
+The callbacks can be defined with the following methods:
+
+=over 4
+
+=item run_on_finish $code [, $pid ]
+
+You can define a subroutine which is called when a child is terminated. It is
+called in the parent process.
+
+The paremeters of the $code are the following:
+
+ - pid of the process, which is terminated
+ - exit code of the program
+ - identification of the process (if provided in the "start" method)
+ - exit signal (0-127: signal name)
+ - core dump (1 if there was core dump at exit)
+
+=item run_on_start $code
+
+You can define a subroutine which is called when a child is started. It called
+after the successful startup of a child in the parent process.
+
+The parameters of the $code are the following:
+
+ - pid of the process which has been started
+ - identification of the process (if provided in the "start" method)
+
+=item run_on_wait $code, [$period]
+
+You can define a subroutine which is called when the child process needs to wait
+for the startup. If $period is not defined, then one call is done per
+child. If $period is defined, then $code is called periodically and the
+module waits for $period seconds betwen the two calls. Note, $period can be
+fractional number also. The exact "$period seconds" is not guarranteed,
+signals can shorten and the process scheduler can make it longer (on busy
+systems).
+
+The $code called in the "start" and the "wait_all_children" method also.
+
+No parameters are passed to the $code on the call.
+
+=back
+
+=head1 EXAMPLE
+
+=head2 Parallel get
+
+This small example can be used to get URLs in parallel.
+
+ use Parallel::ForkManager;
+ use LWP::Simple;
+ my $pm=new Parallel::ForkManager(10);
+ for my $link (@ARGV) {
+ $pm->start and next;
+ my ($fn)= $link =~ /^.*\/(.*?)$/;
+ if (!$fn) {
+ warn "Cannot determine filename from $fn\n";
+ } else {
+ $0.=" ".$fn;
+ print "Getting $fn from $link\n";
+ my $rc=getstore($link,$fn);
+ print "$link downloaded. response code: $rc\n";
+ };
+ $pm->finish;
+ };
+
+=head2 Callbacks
+
+Example of a program using callbacks to get child exit codes:
+
+ use strict;
+ use Parallel::ForkManager;
+
+ my $max_procs = 5;
+ my @names = qw( Fred Jim Lily Steve Jessica Bob Dave Christine Rico Sara );
+ # hash to resolve PID's back to child specific information
+
+ my $pm = new Parallel::ForkManager($max_procs);
+
+ # Setup a callback for when a child finishes up so we can
+ # get it's exit code
+ $pm->run_on_finish(
+ sub { my ($pid, $exit_code, $ident) = @_;
+ print "** $ident just got out of the pool ".
+ "with PID $pid and exit code: $exit_code\n";
+ }
+ );
+
+ $pm->run_on_start(
+ sub { my ($pid,$ident)=@_;
+ print "** $ident started, pid: $pid\n";
+ }
+ );
+
+ $pm->run_on_wait(
+ sub {
+ print "** Have to wait for one children ...\n"
+ },
+ 0.5
+ );
+
+ foreach my $child ( 0 .. $#names ) {
+ my $pid = $pm->start($names[$child]) and next;
+
+ # This code is the child process
+ print "This is $names[$child], Child number $child\n";
+ sleep ( 2 * $child );
+ print "$names[$child], Child $child is about to get out...\n";
+ sleep 1;
+ $pm->finish($child); # pass an exit code to finish
+ }
+
+ print "Waiting for Children...\n";
+ $pm->wait_all_children;
+ print "Everybody is out of the pool!\n";
+
+=head1 BUGS AND LIMITATIONS
+
+Do not use Parallel::ForkManager in an environment, where other child
+processes can affect the run of the main program, so using this module
+is not recommended in an environment where fork() / wait() is already used.
+
+If you want to use more than one copies of the Parallel::ForkManager, then
+you have to make sure that all children processes are terminated, before you
+use the second object in the main program.
+
+You are free to use a new copy of Parallel::ForkManager in the child
+processes, although I don't think it makes sense.
+
+=head1 COPYRIGHT
+
+Copyright (c) 2000 Szab�, Bal�zs (dLux)
+
+All right reserved. This program is free software; you can redistribute it
+and/or modify it under the same terms as Perl itself.
+
+=head1 AUTHOR
+
+ dLux (Szab�, Bal�zs) <dlux at kapu.hu>
+
+=head1 CREDITS
+
+ Noah Robin <sitz at onastick.net> (documentation tweaks)
+ Chuck Hirstius <chirstius at megapathdsl.net> (callback exit status, example)
+ Grant Hopwood <hopwoodg at valero.com> (win32 port)
+ Mark Southern <mark_southern at merck.com> (bugfix)
+
+=cut
+
+package Parallel::ForkManager;
+use POSIX ":sys_wait_h";
+use strict;
+use vars qw($VERSION);
+$VERSION='0.7.5';
+
+sub new { my ($c,$processes)=@_;
+ my $h={
+ max_proc => $processes,
+ processes => {},
+ in_child => 0,
+ };
+ return bless($h,ref($c)||$c);
+};
+
+sub start { my ($s,$identification)=@_;
+ die "Cannot start another process while you are in the child process"
+ if $s->{in_child};
+ while ($s->{max_proc} && ( keys %{ $s->{processes} } ) >= $s->{max_proc}) {
+ $s->on_wait;
+ $s->wait_one_child(defined $s->{on_wait_period} ? &WNOHANG : undef);
+ };
+ $s->wait_children;
+ if ($s->{max_proc}) {
+ my $pid=fork();
+ die "Cannot fork: $!" if !defined $pid;
+ if ($pid) {
+ $s->{processes}->{$pid}=$identification;
+ $s->on_start($pid,$identification);
+ } else {
+ $s->{in_child}=1 if !$pid;
+ }
+ return $pid;
+ } else {
+ $s->{processes}->{$$}=$identification;
+ $s->on_start($$,$identification);
+ return 0; # Simulating the child which returns 0
+ }
+}
+
+sub finish { my ($s, $x)=@_;
+ if ( $s->{in_child} ) {
+ exit ($x || 0);
+ }
+ if ($s->{max_proc} == 0) { # max_proc == 0
+ $s->on_finish($$, $x ,$s->{processes}->{$$}, 0, 0);
+ delete $s->{processes}->{$$};
+ }
+ return 0;
+}
+
+sub wait_children { my ($s)=@_;
+ return if !keys %{$s->{processes}};
+ my $kid;
+ do {
+ $kid = $s->wait_one_child(&WNOHANG);
+ } while $kid > 0 || $kid < -1; # AS 5.6/Win32 returns negative PIDs
+};
+
+sub wait_one_child { my ($s,$par)=@_;
+ my $kid;
+ while (1) {
+ $kid = $s->_waitpid(-1,$par||=0);
+ last if $kid == 0 || $kid == -1; # AS 5.6/Win32 returns negative PIDs
+ redo if !exists $s->{processes}->{$kid};
+ my $id = delete $s->{processes}->{$kid};
+ $s->on_finish( $kid, $? >> 8 , $id, $? & 0x7f, $? & 0x80 ? 1 : 0);
+ last;
+ }
+ $kid;
+};
+
+sub wait_all_children { my ($s)=@_;
+ while (keys %{ $s->{processes} }) {
+ $s->on_wait;
+ $s->wait_one_child(defined $s->{on_wait_period} ? &WNOHANG : undef);
+ };
+}
+
+sub run_on_finish { my ($s,$code,$pid)=@_;
+ $s->{on_finish}->{$pid || 0}=$code;
+}
+
+sub on_finish { my ($s,$pid, at par)=@_;
+ my $code=$s->{on_finish}->{$pid} || $s->{on_finish}->{0} or return 0;
+ $code->($pid, at par);
+};
+
+sub run_on_wait { my ($s,$code, $period)=@_;
+ $s->{on_wait}=$code;
+ $s->{on_wait_period} = $period;
+}
+
+sub on_wait { my ($s)=@_;
+ if(ref($s->{on_wait}) eq 'CODE') {
+ $s->{on_wait}->();
+ if (defined $s->{on_wait_period}) {
+ local $SIG{CHLD} = sub { } if ! defined $SIG{CHLD};
+ select undef, undef, undef, $s->{on_wait_period}
+ };
+ };
+};
+
+sub run_on_start { my ($s,$code)=@_;
+ $s->{on_start}=$code;
+}
+
+sub on_start { my ($s, at par)=@_;
+ $s->{on_start}->(@par) if ref($s->{on_start}) eq 'CODE';
+};
+
+sub set_max_procs { my ($s, $mp)=@_;
+ $s->{max_proc} = $mp;
+}
+
+# OS dependant code follows...
+
+sub _waitpid { # Call waitpid() in the standard Unix fashion.
+ return waitpid($_[1],$_[2]);
+}
+
+# On ActiveState Perl 5.6/Win32 build 625, waitpid(-1, &WNOHANG) always
+# blocks unless an actual PID other than -1 is given.
+sub _NT_waitpid { my ($s, $pid, $par) = @_;
+ if ($par == &WNOHANG) { # Need to nonblock on each of our PIDs in the pool.
+ my @pids = keys %{ $s->{processes} };
+ # Simulate -1 (no processes awaiting cleanup.)
+ return -1 unless scalar(@pids);
+ # Check each PID in the pool.
+ my $kid;
+ foreach $pid (@pids) {
+ $kid = waitpid($pid, $par);
+ return $kid if $kid != 0; # AS 5.6/Win32 returns negative PIDs.
+ }
+ return $kid;
+ } else { # Normal waitpid() call.
+ return waitpid($pid, $par);
+ }
+}
+
+{
+ local $^W = 0;
+ if ($^O eq 'NT' or $^O eq 'MSWin32') {
+ *_waitpid = \&_NT_waitpid;
+ }
+}
+
+1;
diff --git a/scripts/sim/run.pl b/scripts/sim/run.pl
new file mode 100644
index 0000000..a67894a
--- /dev/null
+++ b/scripts/sim/run.pl
@@ -0,0 +1,133 @@
+#!/usr/bin/perl -w
+
+#
+# Copyright 2011, Ben Langmead <langmea at cs.jhu.edu>
+#
+# This file is part of Bowtie 2.
+#
+# Bowtie 2 is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Bowtie 2 is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Bowtie 2. If not, see <http://www.gnu.org/licenses/>.
+#
+
+use strict;
+use warnings;
+use Getopt::Long;
+use FindBin qw($Bin);
+use lib "$Bin";
+use lib "$Bin/contrib";
+use Sim;
+use ForkManager;
+
+# Simulator configuration
+our %conf = (
+ bowtie2_build => "bowtie2-build",
+ bowtie2 => "bowtie2",
+ bowtie2_build_debug => "bowtie2-build --debug",
+ bowtie2_debug => "bowtie2 --debug",
+ tempdir => "/tmp",
+ no_color => 1,
+ small => 1
+);
+
+# Number of parallel processes to use
+my $cpus = 1;
+
+my $usage = qq!
+run.pl [options*]
+
+Options:
+
+ --bowtie2 <path> Path to bowtie2 release binary
+ --bowtie2-debug <path> Path to bowtie2 debug binary
+ --bowtie2-build <path> Path to bowtie2-build release binary
+ --bowtie2-build-debug <path> Path to bowtie2-build debug binary
+ --tempdir <path> Put temporary files here
+ --cases <int> Each thread runs around <int> cases (def: 5)
+ --cpus <int> / -p <int> Run test cases in <int> threads at once
+ --maxreads <int> Handle at most <int> reads per case
+ --numrefs <int> Generate <int> refs per case
+ --die-with-child Kill parent as soon as 1 child dies
+ --no-die-with-child Don\'t kill parent as soon as 1 child dies
+ --small Make small test cases
+ --help Print this usage message
+
+!;
+
+my $help = 0;
+my $ncases = 5;
+my $dieWithChild = 1;
+
+GetOptions(
+ "bowtie2=s" => \$conf{bowtie2},
+ "bowtie2-build=s" => \$conf{bowtie2_build},
+ "tempdir|tmpdir=s" => \$conf{tempdir},
+ "cases-per-thread=i" => \$ncases,
+ "small" => \$conf{small},
+ "large" => sub { $conf{small} = 0 },
+ "no-paired" => \$conf{no_paired},
+ "color" => sub { $conf{no_color} = 0 },
+ "no-color" => \$conf{no_color},
+ "help" => \$help,
+ "die-with-child" => \$dieWithChild,
+ "no-die-with-child" => sub { $dieWithChild = 0 },
+ "p|cpus=i" => \$cpus,
+ "u|qupto|maxreads=i" => \$conf{maxreads},
+ "numrefs|num-refs=i" => \$conf{numrefs},
+) || die "Bad options;";
+
+if($help) {
+ print $usage;
+ exit 0;
+}
+
+my $sim = Sim->new();
+my $pm = new Parallel::ForkManager($cpus);
+
+# Callback for when a child finishes so we can get its exit code
+my @childFailed = ();
+my @childFailedPid = ();
+
+$pm->run_on_finish(sub {
+ my ($pid, $exit_code, $ident) = @_;
+ if($exit_code != 0) {
+ push @childFailed, $exit_code;
+ push @childFailedPid, $pid;
+ !$dieWithChild || die "Dying with child with PID $pid";
+ }
+});
+
+my $totcases = $ncases * $cpus;
+for(1..$totcases) {
+ my $childPid = $pm->start;
+ if($childPid != 0) {
+ next; # spawn the next child
+ }
+ $sim->nextCase(\%conf);
+ $pm->finish;
+}
+$pm->wait_all_children;
+for(@childFailedPid) {
+ print STDERR "Error message from child with pid $_:\n";
+ my $fn = ".run.pl.child.$_";
+ if(open(ER, $fn)) {
+ print STDERR "---------\n";
+ while(<ER>) {
+ print STDERR $_;
+ }
+ print STDERR "---------\n";
+ close(ER);
+ } else {
+ print STDERR "(could not open $fn)\n";
+ }
+}
+print STDERR "PASSED\n";
diff --git a/scripts/sim/run.sh b/scripts/sim/run.sh
new file mode 100644
index 0000000..232b7b8
--- /dev/null
+++ b/scripts/sim/run.sh
@@ -0,0 +1,37 @@
+#!/bin/sh
+
+#
+# Copyright 2011, Ben Langmead <langmea at cs.jhu.edu>
+#
+# This file is part of Bowtie 2.
+#
+# Bowtie 2 is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Bowtie 2 is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Bowtie 2. If not, see <http://www.gnu.org/licenses/>.
+#
+
+CPUS=$1
+shift
+make -j$CPUS \
+ bowtie2-align-s \
+ bowtie2-align-l \
+ bowtie2-align-s-debug \
+ bowtie2-align-l-debug \
+ bowtie2-build-s \
+ bowtie2-build-l \
+ bowtie2-build-s-debug \
+ bowtie2-build-l-debug && \
+perl scripts/sim/run.pl \
+ --bowtie2=./bowtie2 \
+ --bowtie2-build=./bowtie2-build \
+ --cpus=$CPUS \
+ $*
diff --git a/scripts/sim/unit.sh b/scripts/sim/unit.sh
new file mode 100644
index 0000000..f4fdcf8
--- /dev/null
+++ b/scripts/sim/unit.sh
@@ -0,0 +1,23 @@
+#!/bin/sh
+
+#
+# Copyright 2011, Ben Langmead <langmea at cs.jhu.edu>
+#
+# This file is part of Bowtie 2.
+#
+# Bowtie 2 is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Bowtie 2 is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Bowtie 2. If not, see <http://www.gnu.org/licenses/>.
+#
+
+d=`dirname $0`
+for i in `ls $d/*.pm` ; do echo $i ; perl $i --test ; done
diff --git a/scripts/test/.gitignore b/scripts/test/.gitignore
new file mode 100644
index 0000000..0d20b64
--- /dev/null
+++ b/scripts/test/.gitignore
@@ -0,0 +1 @@
+*.pyc
diff --git a/scripts/test/DNA.pm b/scripts/test/DNA.pm
new file mode 100644
index 0000000..e9ba818
--- /dev/null
+++ b/scripts/test/DNA.pm
@@ -0,0 +1,127 @@
+#!/usr/bin/perl -w
+
+#
+# Copyright 2011, Ben Langmead <langmea at cs.jhu.edu>
+#
+# This file is part of Bowtie 2.
+#
+# Bowtie 2 is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Bowtie 2 is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Bowtie 2. If not, see <http://www.gnu.org/licenses/>.
+#
+
+package DNA;
+use strict;
+use warnings;
+use Carp;
+
+my %revcompMap = (
+ "A" => "T", "a" => "t",
+ "T" => "A", "t" => "a",
+ "C" => "G", "c" => "g",
+ "G" => "C", "g" => "c",
+ "R" => "Y", "r" => "y",
+ "Y" => "R", "y" => "r",
+ "M" => "K", "m" => "k",
+ "K" => "M", "k" => "m",
+ "S" => "S", "s" => "s",
+ "W" => "W", "w" => "w",
+ "B" => "V", "b" => "v",
+ "V" => "B", "v" => "b",
+ "H" => "D", "h" => "d",
+ "D" => "H", "d" => "h",
+ "N" => "N", "n" => "n"
+);
+
+my %compat = (
+ "A" => "A",
+ "T" => "T",
+ "C" => "C",
+ "G" => "G",
+ "R" => "AG",
+ "Y" => "CT",
+ "M" => "AC",
+ "K" => "GT",
+ "S" => "CG",
+ "W" => "AT",
+ "B" => "CGT",
+ "V" => "ACG",
+ "H" => "ACT",
+ "D" => "AGT",
+ "N" => "N"
+);
+
+my %incompat = (
+ "A" => "CGT",
+ "T" => "ACG",
+ "C" => "AGT",
+ "G" => "ACT",
+ "R" => "CT",
+ "Y" => "AG",
+ "M" => "GT",
+ "K" => "AC",
+ "S" => "AT",
+ "W" => "CG",
+ "B" => "A",
+ "V" => "T",
+ "H" => "G",
+ "D" => "C",
+ "N" => "N"
+);
+
+my %unambigSet = (
+ "A" => 1, "a" => 1,
+ "C" => 1, "c" => 1,
+ "G" => 1, "g" => 1,
+ "T" => 1, "t" => 1
+);
+
+##
+# Return the complement, incl. if it's IUPAC.
+#
+sub comp($) {
+ my $ret = $revcompMap{$_[0]} || die "Can't reverse-complement '$_[0]'";
+ return $ret;
+}
+
+##
+# Return the complement, incl. if it's IUPAC.
+#
+sub revcomp {
+ my ($ret) = @_;
+ $ret = reverse $ret;
+ for(my $i = 0; $i < length($ret); $i++) {
+ substr($ret, $i, 1) = comp(substr($ret, $i, 1));
+ }
+ return $ret;
+}
+
+##
+# Return true iff it's unambiguous.
+#
+sub unambig($) {
+ return $unambigSet{$_[0]};
+}
+
+##
+# Manipulate DNA in an integer-indexed fashion.
+#
+sub plus($$) {
+ my ($c, $amt) = @_;
+ my %ctoi = ("A" => 0, "C" => 1, "G" => 2, "T" => 3);
+ my %itoc = (0 => "A", 1 => "C", 2 => "G", 3 => "T");
+ $c = uc $c;
+ defined($ctoi{$c}) || die;
+ return $itoc{($ctoi{$c}+$amt) % 4};
+}
+
+1;
diff --git a/scripts/test/README.md b/scripts/test/README.md
new file mode 100644
index 0000000..2e57a08
--- /dev/null
+++ b/scripts/test/README.md
@@ -0,0 +1,44 @@
+Bowtie 2 tests
+==============
+
+Many of the tests use the debug versions of the binaries, so first do `make allall` in the root.
+
+#### Simple test cases
+
+A collection of small tests with hand-crafted input and output checks are collected in `scripts/test/simple_tests.pl`, with `scripts/test/simple_tests.sh` as a driver script. Uses Perl module `DNA.pm`.
+
+From root:
+
+ sh scripts/test/simple_tests.sh
+
+#### Regression tests
+
+They use the Python infrastructure in the `scripts/test` subdirectory, including `bt2face.py`, `dataface.py` and `btdata.py`.
+
+From root:
+
+ python scripts/test/regressions.py --verbose
+
+Val Antonescu originally set these up.
+
+#### Big index test
+
+Builds an index consisting of both human and mouse genomes, pushing the genome size above the 2^32 limit, and necessitating a "big" 64-bit index. This takes a lot of time and RAM.
+
+From root:
+
+ python scripts/test/large_idx.py --verbose
+
+#### Randomized tests
+
+`scripts/sim` contains Perl infrastructure for running randomized tests. These scripts generate random reference genomes and reads, and Bowtie 2 programs are run in various combinations with cross-checks to ensure, for example, that release-binary output matches debug-binary output and that single-threaded output matches multithreaded output. The non-core `Math::Random` Perl module is needed. You can use CPAN to install it.
+
+As of 10/1/2016, the unit tests in `AlignmentCheck.pm` don't all succeed. Also, random tests fail for reasons that seem related to the test infrastructure, e.g., not specifying the command-line arguments properly.
+
+To run the unit tests for the infrastructure, from `scripts/sim`:
+
+ sh unit.sh
+
+To run the actual suite of random tests in parallel on 8 cores, from the root:
+
+ sh scripts/sim/run.sh 8
diff --git a/scripts/test/benchmark/benchmarks.py b/scripts/test/benchmark/benchmarks.py
new file mode 100755
index 0000000..6cb48fd
--- /dev/null
+++ b/scripts/test/benchmark/benchmarks.py
@@ -0,0 +1,275 @@
+#!/usr/bin/env python
+"""
+A few items to deal with sets of benchmarks.
+"""
+
+import os
+import re
+import csv
+import glob
+import json
+import logging
+import subprocess
+import samreader as Sr
+
+
+class Benchmarks(object):
+ """
+ Iterable for all benchmarks found in our test directory.
+ """
+
+ def __init__(self,
+ benchmarks_dir=None,
+ benchmark_test=None,
+ data_dir=None,
+ output_dir=None,
+ bin_dir=None,
+ benchmark_id=None):
+ self.set_idx = 0
+ self.values = list()
+ self.benchmark_id = benchmark_id
+ self.data_dir = data_dir
+ self.bin_dir = bin_dir
+ self.benchmarks_dir = benchmarks_dir
+ self.output_dir = os.path.join(output_dir, self.benchmark_id)
+ #
+ if os.path.exists(self.output_dir):
+ logging.error("A benchmark with the same name already exists (%s)" % self.output_dir)
+ raise OSError("Directory already exists: %s" % self.output_dir)
+ #
+ logging.debug("Creating test output directory: %s" % self.output_dir)
+ os.mkdir(self.output_dir)
+
+ if benchmark_test is not None:
+ self.values.append(self._load_benchmark(benchmark_test))
+
+ if self.benchmarks_dir is not None and os.path.isdir(self.benchmarks_dir):
+ logging.debug("Parse all json files from: %s" % self.benchmarks_dir)
+ all_tests = glob.glob(os.path.join(self.benchmarks_dir, "*.json"))
+ if all_tests:
+ for b_set in all_tests:
+ logging.debug("Adding test: %s" % b_set)
+ self.values.append(self._load_benchmark(b_set))
+ else:
+ logging.warn("No json files found!")
+
+ def __iter__(self):
+ self.set_idx = 0
+ return self
+
+ def next(self):
+ if self.set_idx == len(self.values):
+ raise StopIteration
+
+ value = self.values[self.set_idx]
+ self.set_idx += 1
+ return value
+
+ def _load_benchmark(self, bench_fname):
+ logging.debug("Loading test: %s" % bench_fname)
+ with open(bench_fname) as fp:
+ set_data = json.load(fp)
+ test_list = set_data["tests"]
+
+ for test_desc in test_list:
+ data_load_list = test_desc["input_data"]["loading"]
+ for i, load_cmd in enumerate(data_load_list):
+ load_cmd = re.sub(r'##BT2DIR##', self.bin_dir, load_cmd)
+ load_cmd = re.sub(r'##DATADIR##', self.data_dir, load_cmd)
+ data_load_list[i] = load_cmd
+
+ runable_opt = test_desc["runable"]
+ for key in ["options", "parameters", "outfiles"]:
+ try:
+ runable_prop = runable_opt[key]
+ except KeyError:
+ continue
+
+ for i, item in enumerate(runable_prop):
+ item = re.sub(r'##BT2DIR##', self.bin_dir, item)
+ item = re.sub(r'##DATADIR##', self.data_dir, item)
+ item = re.sub(r'##OUTDIR##', self.output_dir, item)
+ runable_prop[i] = item
+
+ return BenchmarkSet(set_data, self.data_dir, self.output_dir, self.bin_dir)
+
+
+class BenchmarkSet(object):
+ """ A Benchmark item
+ """
+
+ def __init__(self, data, data_dir, out_dir, bin_dir):
+ self.data = data
+ self.data_dir = data_dir
+ self.out_dir = out_dir
+ self.bin_dir = bin_dir
+ self.input_data_loaded = False
+
+ def run(self):
+ logging.info("Running benchmark set: %s" % self.data["description"])
+ all_tests = self.data["tests"]
+ for test in all_tests:
+ bench_cmd = globals()[test["metric"]](self, test)
+ bench_cmd.launch()
+
+ def load(self):
+ if self.input_data_loaded:
+ return
+ logging.info("Loading data for %s" % self.data["name"])
+ # load data
+ test_list = self.data["tests"]
+
+ for test in test_list:
+ data_is_here = True
+ data_set = test["input_data"]["files"]
+ for data_file in data_set:
+ if not os.path.isfile(os.path.join(self.data_dir, data_file)):
+ data_is_here = False
+
+ if not data_is_here:
+ logging.info("Generate data for %s" % test["name"])
+ for cmd in test["input_data"]["loading"]:
+ logging.info("running: %s" % cmd)
+ subprocess.check_call(cmd, shell=True)
+
+ self.input_data_loaded = True
+
+
+class Runable(object):
+ """ cmd line and run helper """
+
+ def __init__(self, main_set, test):
+ """ start """
+ self.benchmark_set = main_set
+ self.test = test
+ self.prologue = ''
+ self.err_log = os.path.join(self.benchmark_set.out_dir, test["name"]) + ".metric"
+
+ def launch(self):
+ """ builds cmd and launch"""
+ logging.debug("Building command.")
+ cmd = self._build_cmd()
+ logging.debug("Running command: %s" % cmd)
+ self._run_cmd(cmd)
+ logging.debug("Calling report formating.")
+ self._format_report()
+
+ def _build_cmd(self):
+ """ build command line"""
+ space = " "
+ test = self.test
+ prg = os.path.join(self.benchmark_set.bin_dir, test["runable"]["program"])
+ cmd = self.prologue + space + prg
+
+ for opt in test["runable"]["options"]:
+ cmd = cmd + space + opt
+
+ for parm in test["runable"]["parameters"]:
+ cmd = cmd + space + parm
+
+ return cmd
+
+ def _run_cmd(self, cmd):
+ """ running """
+ logging.info("Start Benchmark %s" % self.test["name"])
+ logging.info("Running: %s" % cmd)
+ with open(self.err_log, 'w') as errlog:
+ subprocess.check_call(cmd, shell=True, stderr=errlog)
+
+ def _format_report(self, cmd):
+ """ not always required """
+ pass
+
+
+class TestTime(Runable):
+ """ Time benchmarks """
+
+ def __init__(self, main_set, test):
+ super(TestTime, self).__init__(main_set, test)
+ self.prologue = "/usr/bin/time -f %U,%S,%E "
+
+ def _format_report(self):
+ """ formats data in csv format"""
+ csv_file = os.path.join(self.benchmark_set.out_dir, self.test["name"]) + ".csv"
+ p1 = subprocess.Popen(["tail -1 %s" % self.err_log], shell=True, stdout=subprocess.PIPE)
+ line = p1.communicate()[0].rstrip()
+
+ with open(csv_file, 'w') as csvf:
+ writer = csv.writer(csvf)
+ writer.writerow(["Test", "User Time", "System Time", "Wall Time"])
+ row = [self.test["name"]]
+ row.extend(line.split(","))
+ writer.writerow(row)
+
+
+class TestAccuracy(Runable):
+ """ accuracy """
+
+ def __init__(self, main_set, test):
+ super(TestAccuracy, self).__init__(main_set, test)
+
+ def _format_report(self):
+ """ collect data in csv format"""
+ in_sam_file = self._get_first_sam_input_file()
+ in_sam_file = os.path.join(self.benchmark_set.data_dir, in_sam_file)
+ initial_data = dict()
+ mapq_summary = dict()
+ with open(in_sam_file, "r") as fh_sin:
+ in_reader = Sr.SamReader(fh_sin)
+ for rec in in_reader:
+ fl = rec.flag & 255
+ if fl % 2 == 0:
+ logging.error("Initial data does not look like a paired search.")
+ if fl > 128:
+ q_name = rec.qname + "_2"
+ elif fl > 64:
+ q_name = rec.qname + "_1"
+ else:
+ logging.error("Again, initial data does not look like a paired search.")
+ q_name = rec.qname
+ initial_data[q_name] = (rec.rname, rec.pos)
+
+ out_sam_file = self.test["runable"]["outfiles"][0]
+ with open(out_sam_file, "r") as fh_out:
+ in_reader = Sr.SamReader(fh_out)
+ for rec in in_reader:
+ fl = rec.flag & 255
+ if fl % 2 == 0:
+ logging.error("This does not look like a paired search.")
+ if fl > 128:
+ q_name = rec.qname + "_2"
+ elif fl > 64:
+ q_name = rec.qname + "_1"
+ else:
+ logging.error("Again, initial data does not look like a paired search.")
+ q_name = rec.qname
+ orig = initial_data[q_name]
+ if orig[0] == rec.rname and orig[1] - rec.pos < 3:
+ delta = [1, 0]
+ else:
+ delta = [0, 1]
+ logging.debug("%s: missed (pos:%d vs %d)" % (q_name, orig[1], rec.pos))
+ try:
+ mapq_summary[rec.mapq] = map(sum, zip(delta, mapq_summary[rec.mapq]))
+ except KeyError:
+ mapq_summary[rec.mapq] = delta
+
+ csv_file = os.path.join(self.benchmark_set.out_dir, self.test["name"]) + ".csv"
+ with open(csv_file, 'w') as csvf:
+ writer = csv.writer(csvf)
+ writer.writerow(["Test name", "MAPQ", "No. Correct", "No. Misses"])
+ for k in mapq_summary:
+ row = [self.test["name"], k]
+ row.extend(mapq_summary[k])
+ writer.writerow(row)
+
+ def _get_first_sam_input_file(self):
+ all_input_files = self.test["input_data"]["files"]
+ for fname in all_input_files:
+ if fname[-4:] == ".sam":
+ logging.debug("Compare with origin SAM file: %s" % fname)
+ return fname
+
+ raise LookupError("No SAM data input file defined for this test!")
+
+
diff --git a/scripts/test/benchmark/data/conf/acc.json b/scripts/test/benchmark/data/conf/acc.json
new file mode 100644
index 0000000..84ce280
--- /dev/null
+++ b/scripts/test/benchmark/data/conf/acc.json
@@ -0,0 +1,44 @@
+{"description":"A simple accuracy test. Run for each bowtie version separately.",
+ "name" : "Acc_1",
+ "tests": [
+ {"description":"An accuracy test.",
+ "name":"pair wise accuracy",
+ "input_data":{
+ "files": [
+ "hg19.1.bt2",
+ "hg19.2.bt2",
+ "hg19.3.bt2",
+ "hg19.4.bt2",
+ "hg19.rev.1.bt2",
+ "hg19.rev.2.bt2",
+ "hg19.fa",
+ "art_150_mil_1.fq",
+ "art_150_mil_2.fq"
+ ],
+ "loading":[ " ln -s ../work/genomes/human_hg19/hg19.fa ##DATADIR##/",
+ " ln -s ../work/genomes/human_hg19/hg19.1.bt2 ##DATADIR##/",
+ " ln -s ../work/genomes/human_hg19/hg19.2.bt2 ##DATADIR##/",
+ " ln -s ../work/genomes/human_hg19/hg19.3.bt2 ##DATADIR##/",
+ " ln -s ../work/genomes/human_hg19/hg19.4.bt2 ##DATADIR##/",
+ " ln -s ../work/genomes/human_hg19/hg19.rev.1.bt2 ##DATADIR##/",
+ " ln -s ../work/genomes/human_hg19/hg19.rev.2.bt2 ##DATADIR##/",
+ "art_illumina -sam -mp -l 150 -m 200 -s 10 -o ##DATADIR##/art_150 -f 5 -i ##DATADIR##/hg19.fa -rs 1415911971",
+ "cat ##DATADIR##/art_1501.fq | head -4000000 > ##DATADIR##/art_150_mil_1.fq",
+ "cat ##DATADIR##/art_1502.fq | head -4000000 > ##DATADIR##/art_150_mil_2.fq"
+ ]
+ },
+ "runable":{
+ "program":"bowtie2",
+ "options":[
+ "-x ##DATADIR##/hg19",
+ "-1 ##DATADIR##/art_150_mil_1.fq",
+ "-2 ##DATADIR##/art_150_mil_2.fq",
+ "-S ##OUTDIR##/accuracy.sam"
+ ],
+ "parameters":[],
+ "outfiles":["##OUTDIR##/accuracy.sam"]
+ },
+ "metric":"TestAccuracy"
+ }
+ ]
+}
diff --git a/scripts/test/benchmark/data/conf/seta.bench b/scripts/test/benchmark/data/conf/seta.bench
new file mode 100644
index 0000000..c98ed58
--- /dev/null
+++ b/scripts/test/benchmark/data/conf/seta.bench
@@ -0,0 +1,59 @@
+{"description":"A suite of simple tests like the one bowtie2 already comes with",
+ "name" : "set_a",
+ "tests": [
+ {"description":"The lambda virus genome test.",
+ "name":"lambda_long",
+ "input_data":{
+ "files": [
+ "lambda_virus.1.bt2",
+ "lambda_virus.2.bt2",
+ "lambda_virus.3.bt2",
+ "lambda_virus.4.bt2",
+ "lambda_virus.rev.1.bt2",
+ "lambda_virus.rev.2.bt2",
+ "longreads.fq"
+ ],
+ "loading":[ " ##BT2DIR##/bowtie2-build ##BT2DIR##/example/reference/lambda_virus.fa ##DATADIR##/lambda_virus",
+ "cp ##BT2DIR##/example/reads/longreads.fq ##DATADIR##/"
+ ]
+ },
+ "runable":{
+ "program":"bowtie2",
+ "options":[
+ "-x ##DATADIR##/lambda_virus",
+ "-U ##DATADIR##/longreads.fq",
+ "-S /dev/null"
+ ],
+ "parameters":[]
+ },
+ "metric":"TestTime"
+ },
+ {"description":"lambda virus long reads test.",
+ "name":"lambda_s",
+ "input_data":{
+ "files": [
+ "lambda_virus.1.bt2",
+ "lambda_virus.2.bt2",
+ "lambda_virus.3.bt2",
+ "lambda_virus.4.bt2",
+ "lambda_virus.rev.1.bt2",
+ "lambda_virus.rev.2.bt2",
+ "longreads.fq"
+ ],
+ "loading":[ " ##BT2DIR##/bowtie2-build ##BT2DIR##/example/reference/lambda_virus.fa ##DATADIR##/lambda_virus",
+ "cp ##BT2DIR##/example/reads/longreads.fq ##DATADIR##/"
+ ]
+ },
+ "runable":{
+ "program":"bowtie2",
+ "options":[
+ "-x ##DATADIR##/lambda_virus",
+ "-U ##DATADIR##/longreads.fq",
+ "-S /dev/null"
+ ],
+ "parameters":[]
+ },
+ "metric":"TestTime"
+ }
+ ]
+}
diff --git a/scripts/test/benchmark/data/conf/speed.json b/scripts/test/benchmark/data/conf/speed.json
new file mode 100644
index 0000000..bae453e
--- /dev/null
+++ b/scripts/test/benchmark/data/conf/speed.json
@@ -0,0 +1,44 @@
+{"description":"A simple speed test",
+ "name" : "Speed_1",
+ "tests": [
+ {"description":"A speed test.",
+ "name":"pair wise speed",
+ "input_data":{
+ "files": [
+ "hg19.1.bt2",
+ "hg19.2.bt2",
+ "hg19.3.bt2",
+ "hg19.4.bt2",
+ "hg19.rev.1.bt2",
+ "hg19.rev.2.bt2",
+ "hg19.fa",
+ "art_150_mil_1.fq",
+ "art_150_mil_2.fq"
+ ],
+ "loading":[ " ln -s ../work/genomes/human_hg19/hg19.fa ##DATADIR##/",
+ " ln -s ../work/genomes/human_hg19/hg19.1.bt2 ##DATADIR##/",
+ " ln -s ../work/genomes/human_hg19/hg19.2.bt2 ##DATADIR##/",
+ " ln -s ../work/genomes/human_hg19/hg19.3.bt2 ##DATADIR##/",
+ " ln -s ../work/genomes/human_hg19/hg19.4.bt2 ##DATADIR##/",
+ " ln -s ../work/genomes/human_hg19/hg19.rev.1.bt2 ##DATADIR##/",
+ " ln -s ../work/genomes/human_hg19/hg19.rev.2.bt2 ##DATADIR##/",
+ "art_illumina -sam -mp -l 150 -m 200 -s 10 -o ##DATADIR##/art_150 -f 5 -i ##DATADIR##/hg19.fa -rs 1415911971",
+ "cat ##DATADIR##/art_1501.fq | head -4000000 > ##DATADIR##/art_150_mil_1.fq",
+ "cat ##DATADIR##/art_1502.fq | head -4000000 > ##DATADIR##/art_150_mil_2.fq"
+ ]
+ },
+ "runable":{
+ "program":"bowtie2",
+ "options":[
+ "-x ##DATADIR##/hg19",
+ "-1 ##DATADIR##/art_150_mil_1.fq",
+ "-2 ##DATADIR##/art_150_mil_2.fq",
+ "-S /dev/null"
+ ],
+ "parameters":[],
+ "outfiles":[]
+ },
+ "metric":"TestTime"
+ }
+ ]
+}
diff --git a/scripts/test/benchmark/run.py b/scripts/test/benchmark/run.py
new file mode 100755
index 0000000..747c197
--- /dev/null
+++ b/scripts/test/benchmark/run.py
@@ -0,0 +1,128 @@
+#!/usr/bin/env python
+"""
+ Runs benchmark sets specified in JSON files.
+
+Example:
+ # Runs a test set specified in simple.json and
+ # record this run under id "test".
+
+ run.py -t simple.json -i test
+
+ # runs same test using another bowtie binaries from
+ # directory bowtie-devel.
+
+ run.py -t simple.json -i test -b ~/bowtie-devel
+
+ # Runs all *.json benchmark test sets from ~/bt2_benchmarks
+ # recording them under the output directory "records/all_tests".
+
+ run.py -s ~/bt2_benchmarks -i all_tests -o records
+
+"""
+
+import os
+import logging
+import benchmarks as bm
+from optparse import OptionParser
+
+
+def parse_args():
+ usage = " %prog [options] \n\n"
+ usage += "Runs specified bowtie2 benchmarking tests.\n"
+ usage += "Benchmarks are defined using JSON test sets files. For each test\n"
+ usage += "it is defined what input data prerequisites are required, commands\n"
+ usage += "used to generate them in case they are missing, what command to test\n"
+ usage += "and what metrics to record."
+ parser = OptionParser(usage=usage)
+ parser.add_option("-t", "--benchmark-test",
+ action="store", type="string", dest="benchmark_test", default=None,
+ help="One benchmark tests to run. Tests are files in JSON "
+ "format describing a set of test, input data requirements and what "
+ "metrics to record.")
+ parser.add_option("-s", "--benchmarks-dir",
+ action="store", type="string", dest="benchmarks_dir", default=None,
+ help="A path to a directory with benchmark tests to run. All files "
+ "with .json extension will be loaded and run."
+ "metrics to record.")
+ parser.add_option("-i", "--benchmark-id",
+ action="store", type="string", dest="benchmark_id", default=None,
+ help="(mandatory).What name/id to use for this recording.")
+ parser.add_option("-d", "--download-dir",
+ action="store", type="string", dest="download_dir", default='download',
+ help=" (Default: ./download).The directory path where all input "
+ "data used by the benchmarks should be stored.")
+ parser.add_option("-b", "--bowtie-dir",
+ action="store", type="string", dest="bowtie_dir", default='bowtie2',
+ help="(Default: ./bowtie2).Bowtie directory.")
+ parser.add_option("-o", "--output-dir",
+ action="store", type="string", dest="out_dir", default='benchmark_rezults',
+ help="(Default: ./out). Directory where tests results are going to "
+ "be written.")
+ parser.add_option("-l", "--log-file",
+ action="store", type="string", dest="log_fname", default=None,
+ help="(Default: stderr). Log file name if desired.")
+ parser.add_option("-v", "--verbose",
+ action="store_true", dest="verbose", default=False,
+ help="Print more info about each step.")
+ parser.add_option("-u", "--debug",
+ action="store_true", dest="debug", default=False,
+ help="Print extra debug info. Used only for diagnosing purpose.")
+
+ (options, args) = parser.parse_args()
+
+ if options.benchmark_id is None:
+ print("Mandatory option is missing (--benchmark-id)")
+ parser.print_help()
+ exit(-1)
+
+ return options
+
+
+if __name__ == "__main__":
+ options = parse_args()
+
+ if options.log_fname is not None:
+ logging.basicConfig(format='%(levelname)s:%(message)s',
+ filename=options.log_fname,
+ level=logging.ERROR)
+ else:
+ logging.basicConfig(format='%(levelname)s:%(message)s',
+ level=logging.ERROR)
+
+ if options.verbose:
+ logging.getLogger().setLevel(level=logging.INFO)
+
+ if options.debug:
+ logging.getLogger().setLevel(level=logging.DEBUG)
+
+ curr_path = os.getcwd()
+
+ if not os.path.exists(options.out_dir):
+ logging.debug("Creating output dir: %s" % options.out_dir)
+ os.mkdir(options.out_dir)
+
+ if not os.path.exists(options.download_dir):
+ logging.debug("Creating download dir %s" % options.download_dir)
+ os.mkdir(options.download_dir)
+
+ if options.benchmarks_dir is not None:
+ if not os.path.exists(options.benchmarks_dir):
+ logging.error("Cannot find benchmark directory %s" % options.benchmarks_dir)
+
+ batch_benchmarks = bm.Benchmarks(benchmarks_dir=options.benchmarks_dir,
+ benchmark_test=options.benchmark_test,
+ data_dir=options.download_dir,
+ output_dir=options.out_dir,
+ bin_dir=options.bowtie_dir,
+ benchmark_id=options.benchmark_id)
+
+ for test_set in batch_benchmarks:
+ if not test_set.input_data_loaded:
+ test_set.load()
+ test_set.run()
+
+
+
+
+
+
diff --git a/scripts/test/benchmark/samreader.py b/scripts/test/benchmark/samreader.py
new file mode 100755
index 0000000..28a81e2
--- /dev/null
+++ b/scripts/test/benchmark/samreader.py
@@ -0,0 +1,128 @@
+#!/usr/bin/env python
+"""
+A reader of SAM format.
+
+ import samreader
+
+ # ...
+ # Attach it to stdin and print the header.
+ sam_reader = SamReader(sys.stdin)
+ print sam_reader.header
+
+ # ...
+ # Open a SAM file and store each read starting position.
+ with fopen(myfile) as fp:
+ sr = SamReader(fp)
+ for rec in sr:
+ start_coord[rec.qname] = rec.pos
+
+
+"""
+
+
+class CigarString(object):
+ """
+ A basic Cigar.
+ """
+
+ def __init__(self, cigar):
+ self.cigar = cigar
+ self.set_idx = 0
+
+
+ def __str__(self):
+ return self.cigar
+
+
+class SamHeader(object):
+ """
+ Sam Header.
+ """
+
+ def __init__(self, file_handler):
+ self.header_lines = list()
+ self.curr_idx = 0
+ self._source_fh = file_handler
+ self.end_header_pointer = self._load_header()
+
+ def __iter__(self):
+ self.curr_idx = 0
+ return self
+
+ def next(self):
+ if self.curr_idx == len(self.header_lines):
+ raise StopIteration
+
+ value = self.header_lines[self.curr_idx]
+ self.curr_idx += 1
+ return value
+
+ def __str__(self):
+ return "\n".join(self.header_lines)
+
+ def _load_header(self):
+ fh = self._source_fh
+ fh.seek(0)
+
+ last_pos = fh.tell()
+ line = fh.readline().rstrip()
+ while line[0] == '@':
+ self.header_lines.append(line)
+ last_pos = fh.tell()
+ line = fh.readline().rstrip()
+
+ fh.seek(last_pos)
+ return last_pos
+
+
+class SamRecord(object):
+ """
+ Record Item for SAM.
+ """
+
+ def __init__(self, sam_line):
+ all_tokens = sam_line.rstrip().split('\t')
+ # NOTE: not care about optional fields for now.
+ self.rec_keys = ['QNAME', 'FLAG', 'RNAME', 'POS', 'MAPQ', 'CIGAR', 'RNEXT', 'PNEXT', 'TLEN', 'SEQ', 'QUAL']
+ rec = {k: all_tokens[i] for i, k in enumerate(self.rec_keys)}
+ rec['POS'] = int(rec['POS'])
+ rec['MAPQ'] = int(rec['MAPQ'])
+ rec['CIGAR'] = CigarString(rec['CIGAR'])
+ rec['PNEXT'] = int(rec['PNEXT'])
+ rec['TLEN'] = int(rec['TLEN'])
+ self.record = rec
+ self.pos = rec['POS']
+ self.qname = rec['QNAME']
+ self.mapq = rec['MAPQ']
+ self.rname = rec['RNAME']
+ self.flag = int(rec['FLAG'])
+
+ def __str__(self):
+ return "\t".join([str(self.record[self.rec_keys[i]]) for i in range(len(self.rec_keys))])
+
+
+class SamReader(object):
+ """
+ Iterable for all SAM records.
+ """
+
+ def __init__(self, file_handle):
+ self._source_fh = file_handle
+ self.header = SamHeader(file_handle)
+
+ def __iter__(self):
+ self._source_fh.seek(self.header.end_header_pointer)
+ return self
+
+ def next(self):
+ line = self._source_fh.readline()
+ if not line:
+ raise StopIteration
+
+ return SamRecord(line)
+
+
+
+
+
+
diff --git a/scripts/test/big_data/reads/human_reads.fa b/scripts/test/big_data/reads/human_reads.fa
new file mode 100644
index 0000000..1c901bd
--- /dev/null
+++ b/scripts/test/big_data/reads/human_reads.fa
@@ -0,0 +1,8 @@
+>Fragment_1
+ACAAATGCACTGCTAGGCACCACCCCCAGTTCTAGAATCACACCAGCCAG
+TTCACCCTCCAGATGGTTCACCCTCAACTTCATAAAAGTTCCCTACCTAA
+TCTACTGACAGGCTCATCCCCGACCTAATTTTAAAGATTTCCTAGGAGCT
+GCAGTGGGAATCCTGGACCTCAGCCTGGACAAAGAACAGCTGCAGGTCAT
+TCTCATGTGTGGACACAGAAGCTCTGCCTGCCTTTGCTGGCCAGCTGGGC
+TGAGCGGGCCTGGGAATTAAGGCTGCAGGGTTGGTCCCAGGCAGTCTTGC
+
diff --git a/scripts/test/big_data/reads/mouse_reads.fa b/scripts/test/big_data/reads/mouse_reads.fa
new file mode 100644
index 0000000..1f104d5
--- /dev/null
+++ b/scripts/test/big_data/reads/mouse_reads.fa
@@ -0,0 +1,8 @@
+>Fragment_2
+GAATAGACAGGATGAGTATTGTCCTGTAATTTCATAAATTTGTTATTACA
+CTCTTGTAGTTCATGTTTAAAACACTTTCTGTCACCTTTATCTTCTGAAA
+ATTTACTATGTTATGAATTTCTTGAAGTTTTGATAGTTCGGGAAAAAGTT
+TTAGTGTCCTCTTCTAAAGGCTTGCCTCTCCTCATTTCTTGAGAAATCTG
+TGTTTTCCAGTTTTTGTAATCATTAATTTTGATtgtgtgtgtgtgtgtgt
+gtgtgGTTTGTTTGTTGTTTTTTTTGCTACTTAAACGATATTGTTAACAC
+
diff --git a/scripts/test/bt2face.py b/scripts/test/bt2face.py
new file mode 100755
index 0000000..c13c0eb
--- /dev/null
+++ b/scripts/test/bt2face.py
@@ -0,0 +1,39 @@
+#!/usr/bin/env python
+
+import os
+import logging
+import subprocess
+
+
+class BowtieSuite(object):
+
+ def __init__(self,bt2_path=''):
+ curr_path = os.path.realpath(bt2_path)
+ self.bowtie_bin = os.path.join(curr_path,'bowtie2')
+ self.bowtie_build = os.path.join(curr_path,'bowtie2-build')
+ self.bowtie_inspect = os.path.join(curr_path,'bowtie2-inspect')
+ logging.info('bowtie2 path: ' + self.bowtie_bin)
+ logging.info('bowtie2-build path: ' + self.bowtie_build)
+ logging.info('bowtie2-inspect path: ' + self.bowtie_inspect)
+
+
+ def run(self, *args):
+ cmd = self.bowtie_bin + " " + " ".join([i for i in args])
+ logging.debug('align cmd: ' + cmd)
+ return(subprocess.call(cmd,shell=True))
+
+
+ def silent_run(self, *args):
+ cmd = self.bowtie_bin + " " + " ".join([i for i in args])
+ logging.debug('align cmd (silent): ' + cmd)
+ return(subprocess.call(cmd,shell=True,stderr=open(os.devnull, 'w')))
+
+
+ def build(self, *args):
+ cmd = self.bowtie_build + " " + " ".join([i for i in args])
+ curr_dir = os.getcwd()
+ os.chdir(os.path.dirname(cmd.split()[-2]))
+ logging.debug('build cmd: ' + cmd)
+ ret = subprocess.check_call(cmd,shell=True)
+ os.chdir(curr_dir)
+ return(ret)
diff --git a/scripts/test/btdata.py b/scripts/test/btdata.py
new file mode 100755
index 0000000..f4bdd54
--- /dev/null
+++ b/scripts/test/btdata.py
@@ -0,0 +1,199 @@
+#!/usr/bin/env python
+"""
+Note: This would look so much better replaced by XML or at least JSON. But
+ is not worth to do it for now.
+"""
+
+import os
+import gzip
+import urllib2
+import logging
+
+
+class LargeTestsData(object):
+ """
+ Large index tests use quite big datasets. Make sure these
+ are present before starting the time consuming tests.
+ """
+
+ def __init__(self,bt2_path=''):
+ self.data_dir = 'big_data'
+ curr_path = os.path.realpath(bt2_path)
+
+ curr_path = os.path.join(curr_path,'scripts')
+ curr_path = os.path.join(curr_path,'test')
+ self.data_dir_path = os.path.join(curr_path,self.data_dir)
+ self.reads_dir_path = os.path.join(curr_path,'reads')
+
+ try:
+ os.stat(self.data_dir_path)
+ except:
+ logging.error("Cannot find the working datadir %s!" % self.data_dir_path)
+ raise
+
+ self.genomes = dict()
+ self.genomes['human'] = dict()
+ hm = self.genomes['human']
+ hm['link'] = "ftp://hgdownload.cse.ucsc.edu/goldenPath/hg19/chromosomes/"
+ hm['ref_name'] = 'human.fa'
+ hm['chromosomes'] = []
+ chromosomes = hm['chromosomes']
+ for i in range(1,22):
+ chromosomes.append('chr%d' % i)
+ chromosomes.extend(['chrX', 'chrY', 'chrM'])
+
+ self.genomes['mouse'] = dict()
+ ms = self.genomes['mouse']
+ ms['link'] = "ftp://hgdownload.cse.ucsc.edu/goldenPath/rn4/chromosomes"
+ ms['ref_name'] = 'mouse.fa'
+ ms['chromosomes'] = []
+ chromosomes = ms['chromosomes']
+ for i in range(1,21):
+ chromosomes.append('chr%d' % i)
+ chromosomes.extend(['chrX', 'chrM'])
+
+ self.joint_genomes = dict()
+ self.joint_genomes['ms_hum'] = dict()
+ mh = self.joint_genomes['ms_hum']
+ mh['link'] = None
+ mh['ref_name'] = 'ms_hum.fa'
+ mh['genomes'] = ['human','mouse']
+
+ self.init_data()
+
+
+
+ def init_data(self):
+ """ Try and init the data we need.
+ """
+ for genome,gdata in self.genomes.iteritems():
+ gn_path = os.path.join(self.data_dir_path,genome)
+ gn_fasta = os.path.join(gn_path,gdata['ref_name'])
+ if not os.path.exists(gn_fasta):
+ self._get_genome(genome)
+ self._build_genome(genome)
+
+ for genome,gdata in self.joint_genomes.iteritems():
+ gn_path = os.path.join(self.data_dir_path,genome)
+ gn_fasta = os.path.join(gn_path,gdata['ref_name'])
+ if not os.path.exists(gn_fasta):
+ self._build_joint_genome(genome)
+
+
+
+ def _get_genome(self,genome):
+ g = self.genomes[genome]
+ gn_path = os.path.join(self.data_dir_path,genome)
+
+ if not os.path.exists(gn_path):
+ os.mkdir(gn_path)
+
+ logging.info("Downloading genome: %s " % genome)
+
+ for chrs in g['chromosomes']:
+ chr_file = chrs + ".fa.gz"
+ fname = os.path.join(gn_path,chr_file)
+
+ if os.path.exists(fname):
+ logging.info("Skip %s (already present)" % chr_file)
+ continue
+
+ uri = g['link'] + r"/" + chr_file
+ logging.info("file: %s" % chr_file)
+
+ try:
+ f = open(fname,'wb')
+ u = urllib2.urlopen(uri)
+ f.write(u.read())
+ except:
+ f.close()
+ os.remove(fname)
+ os.close(u.fileno())
+ raise
+ else:
+ os.close(u.fileno())
+ u.close()
+ f.close()
+
+
+
+ def _build_genome(self,genome):
+ g = self.genomes[genome]
+ gn_path = os.path.join(self.data_dir_path,genome)
+ gn_fasta = os.path.join(gn_path,g['ref_name'])
+
+ logging.info("Building fasta file for genome: %s" % genome)
+
+ f_gn = open(gn_fasta,'wb')
+
+ for chrs in g['chromosomes']:
+ chr_file = chrs + ".fa.gz"
+ fname = os.path.join(gn_path,chr_file)
+
+ try:
+ f_chr = gzip.open(fname,'rb')
+ f_gn.write(f_chr.read())
+ except:
+ f_chr_close()
+ f_gn.close()
+ os.remove(gn_fasta)
+ raise
+ else:
+ f_chr.close()
+
+ f_gn.close()
+
+
+
+ def _build_joint_genome(self,genome):
+ jg = self.joint_genomes[genome]
+ jgn_path = os.path.join(self.data_dir_path,genome)
+ jgn_fasta = os.path.join(jgn_path,jg['ref_name'])
+
+ if not os.path.exists(jgn_path):
+ os.mkdir(jgn_path)
+
+ logging.info("Building fasta file for genome: %s" % genome)
+
+ f_jg = open(jgn_fasta,'wb')
+ for g in jg['genomes']:
+ gn_path = os.path.join(self.data_dir_path,g)
+ fasta_file = os.path.join(gn_path,self.genomes[g]['ref_name'])
+ try:
+ fin = open(fasta_file,'rb')
+ f_jg.write(fin.read())
+ except:
+ fin.close()
+ f_jg.close()
+ os.remove(jgn_fasta)
+ raise
+ else:
+ fin.close()
+
+ f_jg.close()
+
+
+
+class ExampleData(object):
+ """ The example data.
+ """
+
+ def __init__(self,bt2_path=''):
+ curr_path = os.path.realpath(bt2_path)
+ curr_path = os.path.join(curr_path,'example')
+ self.index_dir_path = os.path.join(curr_path,'index')
+ self.reads_dir_path = os.path.join(curr_path,'reads')
+ self.ref_dir_path = os.path.join(curr_path,'reference')
+
+ try:
+ os.stat(curr_path)
+ except:
+ logging.error("Cannot find the example datadir %s!" % curr_path)
+ raise
+
+
+
+
+
+
+
diff --git a/scripts/test/dataface.py b/scripts/test/dataface.py
new file mode 100755
index 0000000..e48cdca
--- /dev/null
+++ b/scripts/test/dataface.py
@@ -0,0 +1,81 @@
+#!/usr/bin/env python
+
+import os
+import logging
+
+
+class DataFace(object):
+ """ Some data IFace bowtie can work with.
+ """
+
+ def size(self):
+ raise NotImplementedError("size() needs to be implemented!")
+
+
+
+class SamFile(DataFace):
+
+ def __init__(self,sam_desc):
+ self.file_desc = sam_desc
+
+
+ def size(self):
+ if hasattr(self,'no_frags'):
+ return self.no_frags
+
+ count = 0
+ try:
+ fh = open(self.file_desc,"r")
+ for line in fh:
+ if line[0] != "@":
+ count += 1
+ except:
+ logging.error("Exception reading sam file!")
+ fh.close()
+ raise
+ else:
+ fh.close()
+
+ self.no_frags = count
+ return count
+
+
+
+class FastaQFile(DataFace):
+
+ def __init__(self,fastq_desc):
+ self.file_desc = fastq_desc
+
+
+ def size(self):
+ if hasattr(self,'no_seqs'):
+ return self.no_seqs
+
+ count = 0
+ try:
+ fh = open(self.file_desc,"r")
+ lno = 0
+ for line in fh:
+ if lno == 0:
+ if line[0] != '@':
+ raise TypeError("This does not look like a fastq file!")
+ count += 1
+ if lno == 2:
+ if line[0] != '+':
+ raise TypeError("This does not look like a fastq file!")
+ lno = (lno + 1) % 4
+ except:
+ logging.error("Exception reading fastq file!")
+ fh.close()
+ raise
+ else:
+ fh.close()
+
+ self.no_seqs = count
+ return self.no_seqs
+
+
+
+
+
+
\ No newline at end of file
diff --git a/scripts/test/large_idx.py b/scripts/test/large_idx.py
new file mode 100755
index 0000000..d7a93cd
--- /dev/null
+++ b/scripts/test/large_idx.py
@@ -0,0 +1,110 @@
+#!/usr/bin/env python
+
+import os
+import gzip
+import urllib2
+import inspect
+import unittest
+import logging
+import bt2face
+import btdata
+import subprocess
+from optparse import OptionParser
+
+
+
+class TestLargeIndex(unittest.TestCase):
+ """
+ Main fixture for large index testing.
+ """
+
+ def test_human(self):
+ wdir = g_bdata.data_dir_path
+ wdir = os.path.join(wdir,'human')
+ rdir = g_bdata.reads_dir_path
+ genome = g_bdata.genomes['human']
+ genome_fasta = os.path.join(wdir,genome['ref_name'])
+ genome_index = os.path.join(wdir,'human')
+ reads = os.path.join(rdir,'human_reads.fa')
+ ret = g_bt.build("%s human" % genome_fasta)
+ self.assertEqual(ret,0)
+ args = "-x %s -f -U %s" % (genome_index,reads)
+ ret = g_bt.run(args)
+ self.assertEqual(ret,0)
+
+
+ def test_mouse(self):
+ wdir = g_bdata.data_dir_path
+ wdir = os.path.join(wdir,'mouse')
+ rdir = g_bdata.reads_dir_path
+ genome = g_bdata.genomes['mouse']
+ genome_fasta = os.path.join(wdir,genome['ref_name'])
+ genome_index = os.path.join(wdir,'mouse')
+ reads = os.path.join(rdir,'mouse_reads.fa')
+ ret = g_bt.build("%s mouse" % genome_fasta)
+ self.assertEqual(ret,0)
+ args = "-x %s -f -U %s" % (genome_index,reads)
+ ret = g_bt.run(args)
+ self.assertEqual(ret,0)
+
+
+ def test_large_index(self):
+ wdir = g_bdata.data_dir_path
+ wdir = os.path.join(wdir,'ms_hum')
+ rdir = g_bdata.reads_dir_path
+ genome = g_bdata.joint_genomes['ms_hum']
+ genome_fasta = os.path.join(wdir,genome['ref_name'])
+ genome_index = os.path.join(wdir,'ms_hum')
+ reads_human = os.path.join(rdir,'human_reads.fa')
+ reads_mouse = os.path.join(rdir,'mouse_reads.fa')
+ ret = g_bt.build("%s ms_hum" % genome_fasta)
+ self.assertEqual(ret,0)
+ args = "-x %s -f -U %s" % (genome_index,reads_human)
+ ret = g_bt.run(args)
+ args = "-x %s -f -U %s" % (genome_index,reads_mouse)
+ ret = g_bt.run(args)
+ self.assertEqual(ret,0)
+
+
+
+def get_suite():
+ tests = ['test_human','test_mouse','test_large_index']
+ return unittest.TestSuite(map(TestLargeIndex,tests))
+
+
+
+def parse_args():
+ usage = " %prog [options] \n\n"
+ usage += "Warning, this test runs some VERY resource consuming tests.\n"
+ parser = OptionParser(usage=usage)
+ parser.add_option("-v", "--verbose",
+ action="store_true",dest="verbose", default=False,
+ help="Print more info about each test.")
+
+ (options, args) = parser.parse_args()
+ return options
+
+
+g_bdata = None
+g_bt = None
+
+if __name__ == "__main__":
+ logging.basicConfig(format='%(levelname)s:%(message)s',level=logging.ERROR)
+ options = parse_args()
+
+ runner = unittest.TextTestRunner()
+ if options.verbose:
+ logging.getLogger().setLevel(level=logging.INFO)
+ runner = unittest.TextTestRunner(verbosity=2)
+
+ src_file_path = os.path.realpath(inspect.getsourcefile(parse_args))
+ curr_path = os.path.dirname(src_file_path)
+ bw2_subdir = 'bowtie2'
+
+ i = curr_path.find(bw2_subdir)
+ bt2_path = curr_path[:i+len(bw2_subdir)]
+
+ g_bdata = btdata.LargeTestsData(bt2_path)
+ g_bt = bt2face.BowtieSuite(bt2_path)
+ runner.run(get_suite())
+
diff --git a/scripts/test/regressions.py b/scripts/test/regressions.py
new file mode 100755
index 0000000..99e28f9
--- /dev/null
+++ b/scripts/test/regressions.py
@@ -0,0 +1,207 @@
+#!/usr/bin/env python
+
+import os
+import inspect
+import unittest
+import logging
+import shutil
+import bt2face
+import dataface
+import btdata
+from optparse import OptionParser
+
+
+class TestRegressions(unittest.TestCase):
+ """
+ Main regression fixture.
+ """
+
+ def test_288(self):
+ """ Check if --un option works correctly when used with --no-unal
+ """
+ # run --un and record unaligned file size and orig file size
+ # run --no-unal and record file size
+ # check if --no-unali + --un files size equal the orig file size
+ out_sam = 'test288_out.sam'
+ not_algn = 'test288_nal.fastq'
+
+ ref_index = os.path.join(g_bdata.index_dir_path,'lambda_virus')
+ reads = os.path.join(g_bdata.reads_dir_path,'longreads.fq')
+
+ args = "--quiet -x %s -U %s -a --un %s -S %s" % (ref_index,reads,not_algn,out_sam)
+ ret = g_bt.run(args)
+ self.assertEqual(ret,0)
+ sam_size = dataface.SamFile(out_sam).size()
+ no_algn_size = dataface.FastaQFile(not_algn).size()
+ args = "--quiet -x %s -U %s -a --un %s --no-unal -S %s" % (ref_index,reads,not_algn,out_sam)
+ ret = g_bt.run(args)
+ self.assertEqual(ret,0)
+ no_al_sam_size = dataface.SamFile(out_sam).size()
+ self.assertEqual(no_al_sam_size + no_algn_size, sam_size)
+ os.remove(out_sam)
+ os.remove(not_algn)
+
+
+ def test_279(self):
+ """ Check if --un-conc option works correctly when used with --no-unal
+ """
+ out_no_conc = 'test279_nconc.fq'
+ out_conc = 'test279_conc.fq'
+ out_sam = 'test279.sam'
+ out_p1_nc = 'test279_nconc.1.fq'
+ out_p2_nc = 'test279_nconc.2.fq'
+ out_p1_conc = 'test279_conc.1.fq'
+ out_p2_conc = 'test279_conc.2.fq'
+ ref_index = os.path.join(g_bdata.index_dir_path,'lambda_virus')
+ pairs_1 = os.path.join(g_bdata.reads_dir_path,'reads_1.fq')
+ pairs_2 = os.path.join(g_bdata.reads_dir_path,'reads_2.fq')
+
+ args = "--quiet -x %s -1 %s -2 %s --un-conc %s --dovetail --al-conc %s -S %s" % (ref_index,pairs_1,pairs_2,out_no_conc,out_conc,out_sam)
+ ret = g_bt.run(args)
+ self.assertEqual(ret,0)
+ conc_p1_size_step1 = dataface.FastaQFile(out_p1_conc).size()
+ conc_p2_size_step1 = dataface.FastaQFile(out_p2_conc).size()
+ nc_p1_size_step1 = dataface.FastaQFile(out_p1_nc).size()
+ nc_p2_size_step1 = dataface.FastaQFile(out_p2_nc).size()
+ self.assertEqual(conc_p1_size_step1, conc_p2_size_step1, "Number of concordant sequences in pass 1 should match.")
+ self.assertEqual(nc_p1_size_step1, nc_p2_size_step1, "Number of non-concordant sequences in pass 1 should match.")
+
+ args = "--quiet -x %s -1 %s -2 %s --no-unal --un-conc %s --dovetail --al-conc %s -S %s" % (ref_index,pairs_1,pairs_2,out_no_conc,out_conc,out_sam)
+ ret = g_bt.run(args)
+ self.assertEqual(ret,0)
+ conc_p1_size_step2 = dataface.FastaQFile(out_p1_conc).size()
+ conc_p2_size_step2 = dataface.FastaQFile(out_p2_conc).size()
+ nc_p1_size_step2 = dataface.FastaQFile(out_p1_nc).size()
+ nc_p2_size_step2 = dataface.FastaQFile(out_p2_nc).size()
+ self.assertEqual(conc_p1_size_step2, conc_p2_size_step2, "Number of concordant sequences in pass 2 should match.")
+ self.assertEqual(nc_p1_size_step2, nc_p2_size_step2, "Number of non-concordant sequences in pass 2 should match.")
+ self.assertEqual(conc_p1_size_step1, conc_p1_size_step2, "Number of concordant sequences in both steps should match.")
+ self.assertEqual(conc_p2_size_step1, conc_p2_size_step2, "Number of concordant sequences in both steps should match.")
+ self.assertEqual(nc_p1_size_step1, nc_p1_size_step2, "Number of non-concordant sequences in both steps should match.")
+ self.assertEqual(nc_p2_size_step1, nc_p2_size_step2, "Number of non-concordant sequences in both steps should match.")
+
+ self.assertTrue(1,"temp")
+ os.remove(out_p1_conc)
+ os.remove(out_p2_conc)
+ os.remove(out_p1_nc)
+ os.remove(out_p2_nc)
+ os.remove(out_sam)
+
+
+ def test_version(self):
+ """ Check if --version option works correctly.
+ """
+ args = "--version"
+ ret = g_bt.silent_run(args)
+ self.assertEqual(ret,0)
+
+
+ def test_x_option(self):
+ """ Check if -x is indeed mandatory.
+ """
+ ref_index = os.path.join(g_bdata.index_dir_path,'lambda_virus')
+ reads = os.path.join(g_bdata.reads_dir_path,'longreads.fq')
+ out_sam = 'test_x_option.sam'
+
+ args = "%s -U %s -S %s " % (ref_index,reads,out_sam)
+ ret = g_bt.silent_run(args)
+ self.assertNotEqual(ret,0)
+ args = "-x %s -U %s -S %s " % (ref_index,reads,out_sam)
+ ret = g_bt.silent_run(args)
+ self.assertEqual(ret, 0)
+ os.remove(out_sam)
+
+
+ def test_313(self):
+ """ Check if --un-conc, --al-conc, --un, --al do not fail in case of a dot
+ character present in their path name.
+ """
+ ref_index = os.path.join(g_bdata.index_dir_path,'lambda_virus')
+ pairs_1 = os.path.join(g_bdata.reads_dir_path,'reads_1.fq')
+ pairs_2 = os.path.join(g_bdata.reads_dir_path,'reads_2.fq')
+ no_dot_dir = 'nodotdir'
+ dot_dir = 'v.12_.test'
+ un_basename = 'bname'
+ out_spec = os.path.join(no_dot_dir,un_basename)
+ out_spec_dot= os.path.join(dot_dir,un_basename)
+ args = "--quiet -x %s -1 %s -2 %s --un-conc %s -S %s" % (ref_index,pairs_1,pairs_2,out_spec,os.devnull)
+
+ try:
+ os.makedirs(no_dot_dir)
+ os.makedirs(dot_dir)
+ except:
+ pass
+
+ ret = g_bt.silent_run(args)
+ self.assertEqual(ret, 0)
+ mate1_count = dataface.FastaQFile(out_spec + '.1').size()
+ args = "--quiet -x %s -1 %s -2 %s --un-conc %s -S %s" % (ref_index,pairs_1,pairs_2,out_spec_dot,os.devnull)
+ ret = g_bt.silent_run(args)
+ self.assertEqual(ret, 0, "--un-conc should work with dots in its path.")
+ mate1_count_dot = dataface.FastaQFile(out_spec_dot + '.1').size()
+ self.assertEqual(mate1_count, mate1_count_dot, "Number of seqs should be the same.")
+
+ args = "--quiet -x %s -1 %s -2 %s --un-conc %s -S %s" % (ref_index,pairs_1,pairs_2,dot_dir,os.devnull)
+ ret = g_bt.silent_run(args)
+ self.assertEqual(ret, 0, "--un-conc should work with a path only.")
+ mate1_fs = os.path.join(dot_dir,"un-conc-mate.1")
+ self.assertTrue(os.path.isfile(mate1_fs), "--un-conc output is missing for path only case.")
+ mate1_count_dot_only = dataface.FastaQFile(mate1_fs).size()
+ self.assertEqual(mate1_count, mate1_count_dot_only, "Number of seqs should be the same.")
+
+ args= "--quiet -x %s -1 %s -2 %s --un-conc %s --al-conc %s --un %s -S %s" % (ref_index,pairs_1,pairs_2,dot_dir,dot_dir,dot_dir,os.devnull)
+ ret = g_bt.run(args)
+ self.assertEqual(ret, 0)
+ mate1_al = os.path.join(dot_dir,"al-conc-mate.1")
+ mate1_un = os.path.join(dot_dir,"un-seqs")
+ self.assertTrue(
+ os.path.isfile(mate1_fs) and
+ os.path.isfile(mate1_al) and
+ os.path.isfile(mate1_un),
+ "--un-conc, --al-conc and --un should not clobber"
+ )
+ shutil.rmtree(no_dot_dir)
+ shutil.rmtree(dot_dir)
+
+
+
+def get_suite():
+ loader = unittest.TestLoader()
+ return loader.loadTestsFromTestCase(TestRegressions)
+
+
+def parse_args():
+ usage = " %prog [options] \n\n"
+ usage += "Some regression tests.\n"
+ parser = OptionParser(usage=usage)
+ parser.add_option("-v", "--verbose",
+ action="store_true",dest="verbose", default=False,
+ help="Print more info about each test.")
+
+ (options, args) = parser.parse_args()
+ return options
+
+
+g_bdata = None
+g_bt = None
+
+if __name__ == "__main__":
+ logging.basicConfig(format='%(levelname)s:%(message)s',level=logging.ERROR)
+ options = parse_args()
+
+ runner = unittest.TextTestRunner()
+ if options.verbose:
+ logging.getLogger().setLevel(level=logging.INFO)
+ runner = unittest.TextTestRunner(verbosity=2)
+
+ src_file_path = os.path.realpath(inspect.getsourcefile(parse_args))
+ curr_path = os.path.dirname(src_file_path)
+ bw2_subdir = 'bowtie2'
+
+ i = curr_path.find(bw2_subdir)
+ bt2_path = curr_path[:i+len(bw2_subdir)]
+
+ g_bdata = btdata.ExampleData(bt2_path)
+ g_bt = bt2face.BowtieSuite(bt2_path)
+ runner.run(get_suite())
+
diff --git a/scripts/test/simple_tests.pl b/scripts/test/simple_tests.pl
new file mode 100644
index 0000000..c27744e
--- /dev/null
+++ b/scripts/test/simple_tests.pl
@@ -0,0 +1,5070 @@
+#!/usr/bin/perl -w
+
+#
+# Copyright 2011, Ben Langmead <langmea at cs.jhu.edu>
+#
+# This file is part of Bowtie 2.
+#
+# Bowtie 2 is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Bowtie 2 is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Bowtie 2. If not, see <http://www.gnu.org/licenses/>.
+#
+
+##
+# Give simple tests with known results to bowtie2.
+#
+
+use strict;
+use warnings;
+use Getopt::Long;
+use FindBin qw($Bin);
+use lib $Bin;
+use List::Util qw(max min);
+use Data::Dumper;
+use DNA;
+use Clone qw(clone);
+use Test::Deep;
+
+my $bowtie2 = "";
+my $bowtie2_build = "";
+
+GetOptions(
+ "bowtie2=s" => \$bowtie2,
+ "bowtie2-build=s" => \$bowtie2_build) || die "Bad options";
+
+if(! -x $bowtie2 || ! -x $bowtie2_build) {
+ my $bowtie2_dir = `dirname $bowtie2`;
+ my $bowtie2_exe = `basename $bowtie2`;
+ my $bowtie2_build_exe = `basename $bowtie2_build`;
+ chomp($bowtie2_dir);
+ chomp($bowtie2_exe);
+ chomp($bowtie2_build_exe);
+ system("make -C $bowtie2_dir $bowtie2_exe $bowtie2_build_exe") && die;
+}
+
+(-x $bowtie2) || die "Cannot run '$bowtie2'";
+(-x $bowtie2_build) || die "Cannot run '$bowtie2_build'";
+
+my @cases = (
+
+ # File format cases
+
+ # -F: FASTA continuous
+
+ { name => "FASTA-continuous 1",
+ ref => [ "AGCATCGATCAGTATCTGA" ],
+ # 0123456789012345678
+ # AGCATCGATC
+ # CAGTATCTGA
+ cont_fasta_reads => ">seq1\nAGCATCGATCAGTATCTGA\n",
+ idx_map => { "seq1_0" => 0, "seq1_9" => 1 },
+ args => "-F 10,9",
+ hits => [{ 0 => 1 }, { 9 => 1 }] },
+
+ { name => "FASTA-continuous 2",
+ ref => [ "AGCATCGATCAGTATCTGA" ],
+ # 0123456789012345678
+ # seq1: AGCATCGATCAGTATCTG
+ # AGCATCGATC
+ # seq2: AGCATCGATCAGTATCTGA
+ # AGCATCGATC
+ # CAGTATCTGA
+ cont_fasta_reads => ">seq1\nAGCATCGATCAGTATCTG\n".
+ ">seq2\nAGCATCGATCAGTATCTGA\n",
+ idx_map => { "seq1_0" => 0, "seq2_0" => 1, "seq2_9" => 2 },
+ args => "-F 10,9",
+ hits => [{ 0 => 1 }, { 0 => 1 }, { 9 => 1 }] },
+
+ { name => "FASTA-continuous 3",
+ ref => [ "AGCATCGATCAGTATCTGA" ],
+ # 0123456789012345678
+ # AGCATCGATC
+ # CAGTATCTGA
+ cont_fasta_reads => ">seq1\nAGCATCGATCAGTATCTGA\n",
+ idx_map => { "seq1_0" => 0 },
+ args => "-F 10,9 -u 1",
+ hits => [{ 0 => 1 }] },
+
+ { name => "FASTA-continuous 4",
+ ref => [ "AGCATCGATCAGTATCTGA" ],
+ # 0123456789012345678
+ # AGCATCGATC
+ # CAGTATCTGA
+ cont_fasta_reads => ">seq1\nAGCATCGATCAGTATCTGA\n",
+ idx_map => { "seq1_9" => 0 },
+ args => "-F 10,9 -s 1",
+ hits => [{ 9 => 1 }] },
+
+ { name => "FASTA-continuous 5",
+ ref => [ "AGCATCGATCAGTATCTGA" ],
+ # 0123456789012345678
+ # seq1: AGCATCGATCAGTATCTG
+ # AGCATCGATC
+ # seq2: AGCATCGATCAGTATCTGA
+ # AGCATCGATC
+ # CAGTATCTGA
+ cont_fasta_reads => ">seq1\nAGCATCGATCAGTATCTG\n".
+ ">seq2\nAGCATCGATCAGTATCTGA\n",
+ idx_map => { "seq2_0" => 0 },
+ args => "-F 10,9 -u 1 -s 1",
+ hits => [{ 0 => 1 }] },
+
+ { name => "FASTA-continuous 6",
+ ref => [ "AGCATCGATCAG" ],
+ # 012345678901
+ # seq1: AGCATCGATC
+ # GCATCGATCA
+ # CATCGATCAG
+ cont_fasta_reads => ">seq1\nAGCATCGATCAG\n",
+ idx_map => { "seq1_0" => 0, "seq1_1" => 1, "seq1_2" => 2 },
+ args => "-F 10,1",
+ hits => [{ 0 => 1 }, { 1 => 1 }, { 2 => 1 }] },
+
+ # -c
+
+ { name => "Cline 1",
+ ref => [ "AGCATCGATCAGTATCTGA" ],
+ cline_reads => "CATCGATCAGTATCTG",
+ hits => [{ 2 => 1 }] },
+
+ { name => "Cline 2",
+ ref => [ "AGCATCGATCAGTATCTGA" ],
+ cline_reads => "CATCGATCAGTATCTG:IIIIIIIIIIIIIIII",
+ hits => [{ 2 => 1 }] },
+
+ { name => "Cline 2",
+ ref => [ "AGCATCGATCAGTATCTGA" ],
+ cline_reads => "CATCGATCAGTATCTG:ABCDEDGHIJKLMNOP",
+ hits => [{ 2 => 1 }] },
+
+ { name => "Cline 4",
+ ref => [ "AGCATCGATCAGTATCTGA" ],
+ cline_reads => "CATCGATCAGTATCTG:ABCDEDGHIJKLMNO", # qual too short
+ should_abort => 1},
+
+ { name => "Cline 5",
+ ref => [ "AGCATCGATCAGTATCTGA" ],
+ cline_reads => "CATCGATCAGTATCTG:ABCDEDGHIJKLMNOPQ", # qual too long
+ should_abort => 1},
+
+ # Part of sequence is trimmed
+ { name => "Cline 7",
+ ref => [ "AGCATCGATCAGTATCTGA" ],
+ cline_reads => "CATCGATCAGTATCTG:IIIIIIIIIIIIIIII\n",
+ args => "--trim3 4",
+ norc => 1,
+ hits => [{ 2 => 1 }] },
+
+ # Whole sequence is trimmed
+ { name => "Cline 8",
+ ref => [ "AGCATCGATCAGTATCTGA" ],
+ cline_reads => "CATCGATCAGTATCTG:IIIIIIIIIIIIIIII",
+ args => "--trim5 16",
+ hits => [{ "*" => 1 }] },
+
+ # Sequence is skipped
+ { name => "Cline 9",
+ ref => [ "AGCATCGATCAGTATCTGA" ],
+ cline_reads => "CATCGATCAGTATCTG:IIIIIIIIIIIIIIII",
+ args => "-s 1",
+ hits => [{ }] },
+
+ { name => "Cline multiread 1",
+ ref => [ "AGCATCGATCAGTATCTGA" ],
+ cline_reads => "CATCGATCAGTATCTG:IIIIIIIIIIIIIIII,".
+ "ATCGATCAGTATCTG:IIIIIIIIIIIIIII\n\n",
+ hits => [{ 2 => 1 }, { 3 => 1 }] },
+
+ { name => "Cline multiread 2",
+ ref => [ "AGCATCGATCAGTATCTGA" ],
+ args => "-u 1",
+ cline_reads => "CATCGATCAGTATCTG:IIIIIIIIIIIIIIII,".
+ "ATCGATCAGTATCTG:IIIIIIIIIIIIIII\n\n",
+ hits => [{ 2 => 1 }] },
+
+ { name => "Cline multiread 3",
+ ref => [ "AGCATCGATCAGTATCTGA" ],
+ args => "-u 2",
+ cline_reads => "CATCGATCAGTATCTG,".
+ "ATCGATCAGTATCTG\r\n",
+ hits => [{ 2 => 1 }, { 3 => 1 }] },
+
+ # Paired-end reads that should align
+ { name => "Cline paired 1",
+ ref => [ "AGCATCGATCAAAAACTGA" ],
+ # AGCATCGATC
+ # TCAAAAACTGA
+ # 0123456789012345678
+ cline_reads1 => "AGCATCGATC:IIIIIIIIII,".
+ "TCAGTTTTTGA",
+ cline_reads2 => "TCAGTTTTTGA,".
+ "AGCATCGATC:IIIIIIIIII",
+ pairhits => [ { "0,8" => 1 }, { "0,8" => 1 } ] },
+
+ # Paired-end reads that should align
+ { name => "Cline paired 2",
+ ref => [ "AGCATCGATCAAAAACTGA" ],
+ args => "-s 1",
+ # AGCATCGATC
+ # TCAAAAACTGA
+ # 0123456789012345678
+ cline_reads1 => "AGCATCGATC:IIIIIIIIII,".
+ "TCAGTTTTTGA:IIIIIIIIIII",
+ cline_reads2 => "TCAGTTTTTGA:IIIIIIIIIII,".
+ "AGCATCGATC:IIIIIIIIII",
+ pairhits => [ { }, { "0,8" => 1 } ] },
+
+ # Paired-end reads that should align
+ { name => "Cline paired 3",
+ ref => [ "AGCATCGATCAAAAACTGA" ],
+ args => "-u 1",
+ # AGCATCGATC
+ # TCAAAAACTGA
+ # 0123456789012345678
+ cline_reads1 => "AGCATCGATC:IIIIIIIIII,".
+ "TCAGTTTTTGA:IIIIIIIIIII",
+ cline_reads2 => "TCAGTTTTTGA:IIIIIIIIIII,".
+ "AGCATCGATC:IIIIIIIIII",
+ pairhits => [ { "0,8" => 1 }, { } ] },
+
+ # Paired-end reads with left end entirely trimmed away
+ { name => "Cline paired 4",
+ ref => [ "AGCATCGATCAAAAACTGA" ],
+ args => "-3 7",
+ # AGCATCG
+ # GATCAAAAACTGA
+ # 0123456789012345678
+ cline_reads1 => "AGCATCG:IIIIIII",
+ cline_reads2 => "GATCAAAAACTGA:IIIIIIIIIIIII",
+ # GATCAGTTTTTGA
+ pairhits => [ { "*,6" => 1 } ] },
+
+ # -q
+
+ { name => "Fastq 1",
+ ref => [ "AGCATCGATCAGTATCTGA" ],
+ fastq => "\@r0\nCATCGATCAGTATCTG\n+\nIIIIIIIIIIIIIIII",
+ hits => [{ 2 => 1 }] },
+
+ { name => "Fastq 2",
+ ref => [ "AGCATCGATCAGTATCTGA" ],
+ fastq => "\@r0\nCATCGATCAGTATCTG\n+\nIIIIIIIIIIIIIIII\n", # extra newline
+ hits => [{ 2 => 1 }] },
+
+ { name => "Fastq 3",
+ ref => [ "AGCATCGATCAGTATCTGA" ],
+ fastq => "\@r0\nCATCGATCAGTATCTG\r\n+\nIIIIIIIIIIIIIIII\n",
+ hits => [{ 2 => 1 }] },
+
+ { name => "Fastq 4",
+ ref => [ "AGCATCGATCAGTATCTGA" ],
+ fastq => "\@r0\nCATCGATCAGTATCTG\r\n+\nIIIIIIIIIIIIIII\n", # qual too short
+ # CATCGATCAGTATCTG
+ should_abort => 1},
+
+ # Name line doesn't start with @
+ { name => "Fastq 5",
+ ref => [ "AGCATCGATCAGTATCTGA" ],
+ fastq => "\@r0\nCATCGATCAGTATCTG\r\n+\nIIIIIIIIIIIIIII\n",
+ should_abort => 1,
+ hits => [{ }] },
+
+ # Name line doesn't start with @ (2)
+ { name => "Fastq 6",
+ ref => [ "AGCATCGATCAGTATCTGA" ],
+ fastq => "r0\nCATCGATCAGTATCTG\r\n+\nIIIIIIIIIIIIIII\n",
+ should_abort => 1,
+ hits => [{ }] },
+
+ # Part of sequence is trimmed
+ { name => "Fastq 7",
+ ref => [ "AGCATCGATCAGTATCTGA" ],
+ fastq => "\@r0\nCATCGATCAGTATCTG\r\n+\nIIIIIIIIIIIIIIII\n",
+ args => "--trim3 4",
+ norc => 1,
+ hits => [{ 2 => 1 }] },
+
+ # Whole sequence is trimmed
+ { name => "Fastq 8",
+ ref => [ "AGCATCGATCAGTATCTGA" ],
+ fastq => "\@r0\nCATCGATCAGTATCTG\r\n+\nIIIIIIIIIIIIIIII\n",
+ args => "--trim5 16",
+ hits => [{ "*" => 1 }] },
+
+ # Sequence is skipped
+ { name => "Fastq 9",
+ ref => [ "AGCATCGATCAGTATCTGA" ],
+ fastq => "\@r0\nCATCGATCAGTATCTG\r\n+\nIIIIIIIIIIIIIIII\n",
+ args => "-s 1",
+ hits => [{ }] },
+
+ # Like Fastq 1 but with many extra newlines
+ { name => "Fastq multiread 1",
+ ref => [ "AGCATCGATCAGTATCTGA" ],
+ fastq => "\@r0\nCATCGATCAGTATCTG\r\n+\nIIIIIIIIIIIIIIII\n".
+ "\@r1\nATCGATCAGTATCTG\r\n+\nIIIIIIIIIIIIIII\n",
+ hits => [{ 2 => 1 }, { 3 => 1 }] },
+
+ # Like Fastq multiread 1 but with -u 1
+ { name => "Fastq multiread 2",
+ ref => [ "AGCATCGATCAGTATCTGA" ],
+ args => "-u 1",
+ fastq => "\@r0\nCATCGATCAGTATCTG\r\n+\nIIIIIIIIIIIIIIII\n".
+ "\@r1\nATCGATCAGTATCTG\r\n+\nIIIIIIIIIIIIIII\n",
+ hits => [{ 2 => 1 }] },
+
+ # Like Fastq multiread 1 but with -u 2
+ { name => "Fastq multiread 3",
+ ref => [ "AGCATCGATCAGTATCTGA" ],
+ args => "-u 2",
+ fastq => "\@r0\nCATCGATCAGTATCTG\r\n+\nIIIIIIIIIIIIIIII\n".
+ "\@r1\nATCGATCAGTATCTG\r\n+\nIIIIIIIIIIIIIII\n",
+ hits => [{ 2 => 1 }, { 3 => 1 }] },
+
+ # Paired-end reads that should align
+ { name => "Fastq paired 1",
+ ref => [ "AGCATCGATCAAAAACTGA" ],
+ # AGCATCGATC
+ # TCAAAAACTGA
+ # 0123456789012345678
+ fastq1 => "\@r0\nAGCATCGATC\r\n+\nIIIIIIIIII\n".
+ "\@r1\nTCAGTTTTTGA\r\n+\nIIIIIIIIIII\n",
+ fastq2 => "\@r0\nTCAGTTTTTGA\n+\nIIIIIIIIIII\n".
+ "\@r1\nAGCATCGATC\r\n+\nIIIIIIIIII",
+ pairhits => [ { "0,8" => 1 }, { "0,8" => 1 } ] },
+
+ # Paired-end reads that should align
+ { name => "Fastq paired 2",
+ ref => [ "AGCATCGATCAAAAACTGA" ],
+ args => "-s 1",
+ # AGCATCGATC
+ # TCAAAAACTGA
+ # 0123456789012345678
+ fastq1 => "\@r0\nAGCATCGATC\r\n+\nIIIIIIIIII\n".
+ "\@r1\nTCAGTTTTTGA\n+\nIIIIIIIIIII\n",
+ fastq2 => "\@r0\nTCAGTTTTTGA\n+\nIIIIIIIIIII\n".
+ "\@r1\nAGCATCGATC\r\n+\nIIIIIIIIII",
+ pairhits => [ { }, { "0,8" => 1 } ] },
+
+ # Paired-end reads that should align
+ { name => "Fastq paired 3",
+ ref => [ "AGCATCGATCAAAAACTGA" ],
+ args => "-u 1",
+ # AGCATCGATC
+ # TCAAAAACTGA
+ # 0123456789012345678
+ fastq1 => "\@r0\nAGCATCGATC\r\n+\nIIIIIIIIII\n".
+ "\@r1\nTCAGTTTTTGA\r\n+\nIIIIIIIIIII\n",
+ fastq2 => "\@r0\nTCAGTTTTTGA\n+\nIIIIIIIIIII\n".
+ "\@r1\nAGCATCGATC\r\n+\nIIIIIIIIII",
+ pairhits => [ { "0,8" => 1 }, { } ] },
+
+ # Paired-end reads with left end entirely trimmed away
+ { name => "Fastq paired 4",
+ ref => [ "AGCATCGATCAAAAACTGA" ],
+ args => "-3 7",
+ # AGCATCG
+ # GATCAAAAACTGA
+ # 0123456789012345678
+ fastq1 => "\@r0\nAGCATCG\n+\nIIIIIII\n",
+ fastq2 => "\@r0\nGATCAAAAACTGA\n+\nIIIIIIIIIIIII\n",
+ # GATCAGTTTTTGA
+ pairhits => [ { "*,6" => 1 } ] },
+
+ # -f
+
+ { name => "Fasta 1",
+ ref => [ "AGCATCGATCAGTATCTGA" ],
+ fasta => ">r0\nCATCGATCAGTATCTG",
+ hits => [{ 2 => 1 }] },
+
+ { name => "Fasta 2",
+ ref => [ "AGCATCGATCAGTATCTGA" ],
+ fasta => ">r0\nCATCGATCAGTATCTG\n",
+ hits => [{ 2 => 1 }] },
+
+ { name => "Fasta 3",
+ ref => [ "AGCATCGATCAGTATCTGA" ],
+ fasta => "\n\n\r\n>r0\nCATCGATCAGTATCTG\r\n\n",
+ hits => [{ 2 => 1 }] },
+
+ # Name line doesn't start with >
+ { name => "Fasta 5",
+ ref => [ "AGCATCGATCAGTATCTGA" ],
+ fasta => "\n\n\r\nr0\nCATCGATCAGTATCTG\r",
+ should_abort => 1,
+ hits => [{ }] },
+
+ # Name line doesn't start with > (2)
+ { name => "Fasta 6",
+ ref => [ "AGCATCGATCAGTATCTGA" ],
+ fasta => "r0\nCATCGATCAGTATCTG\r",
+ should_abort => 1,
+ hits => [{ }] },
+
+ { name => "Fasta 7",
+ ref => [ "AGCATCGATCAGTATCTGA" ],
+ fasta => "\n\n\r\n\>r0\nCATCGATCAGTATCTG\r\n",
+ args => "--trim3 4",
+ norc => 1,
+ hits => [{ 2 => 1 }] },
+
+ { name => "Fasta 8",
+ ref => [ "AGCATCGATCAGTATCTGA" ],
+ fasta => "\n\n\r\n\>r0\nCATCGATCAGTATCTG\r\n",
+ args => "--trim3 16",
+ hits => [{ "*" => 1 }] },
+
+ { name => "Fasta 9",
+ ref => [ "AGCATCGATCAGTATCTGA" ],
+ fasta => "\n\n\r\n>r0\nCATCGATCAGTATCTG\r\n",
+ args => "-s 1",
+ hits => [{ }] },
+
+ { name => "Fasta multiread 1",
+ ref => [ "AGCATCGATCAGTATCTGA" ],
+ fasta => "\n\n\r\n>r0\nCATCGATCAGTATCTG\n\n".
+ "\n\n\r\n>r1\nATCGATCAGTATCTG\n\n",
+ hits => [{ 2 => 1 }, { 3 => 1 }] },
+
+ { name => "Fasta multiread 2",
+ ref => [ "AGCATCGATCAGTATCTGA" ],
+ args => "-u 1",
+ fasta => "\n\n\r\n>r0\nCATCGATCAGTATCTG\r\n".
+ "\n\n\r\n>r1\nATCGATCAGTATCTG\r\n",
+ hits => [{ 2 => 1 }] },
+
+ { name => "Fasta multiread 3",
+ ref => [ "AGCATCGATCAGTATCTGA" ],
+ args => "-u 2",
+ fasta => "\n\n\r\n>r0\nCATCGATCAGTATCTG\r\n".
+ "\n\n\r\n>r1\nATCGATCAGTATCTG\r\n",
+ hits => [{ 2 => 1 }, { 3 => 1 }] },
+
+ { name => "Fasta paired 1",
+ ref => [ "AGCATCGATCAAAAACTGA" ],
+ # AGCATCGATC
+ # TCAAAAACTGA
+ # 0123456789012345678
+ fasta1 => "\n\n\r\n>r0\nAGCATCGATC\r\n".
+ "\n\n>r1\nTCAGTTTTTGA\r\n",
+ fasta2 => "\n\n\r\n>r0\nTCAGTTTTTGA\n".
+ "\n\n\r\n>r1\nAGCATCGATC",
+ pairhits => [ { "0,8" => 1 }, { "0,8" => 1 } ] },
+
+ { name => "Fasta paired 2",
+ ref => [ "AGCATCGATCAAAAACTGA" ],
+ args => "-s 1",
+ # AGCATCGATC
+ # TCAAAAACTGA
+ # 0123456789012345678
+ fasta1 => ">r0\nAGCATCGATC\r\n".
+ "\n\n>r1\nTCAGTTTTTGA\n",
+ fasta2 => "\n\n\r\n>r0\nTCAGTTTTTGA\n".
+ "\n\n\r\n>r1\nAGCATCGATC",
+ pairhits => [ { }, { "0,8" => 1 } ] },
+
+ { name => "Fasta paired 3",
+ ref => [ "AGCATCGATCAAAAACTGA" ],
+ args => "-u 1",
+ # AGCATCGATC
+ # TCAAAAACTGA
+ # 0123456789012345678
+ fasta1 => "\n\n\r\n>r0\nAGCATCGATC\r\n".
+ "\n\n>r1\nTCAGTTTTTGA\r\n",
+ fasta2 => "\n\n\r\n>r0\nTCAGTTTTTGA\n".
+ "\n\n\r\n>r1\nAGCATCGATC",
+ pairhits => [ { "0,8" => 1 }, { } ] },
+
+ # Paired-end reads with left end entirely trimmed away
+ { name => "Fasta paired 4",
+ ref => [ "AGCATCGATCAAAAACTGA" ],
+ args => "-3 7",
+ # AGCATCG
+ # GATCAAAAACTGA
+ # 0123456789012345678
+ fasta1 => ">\nAGCATCG\n",
+ fasta2 => ">\nGATCAAAAACTGA\n",
+ # GATCAGTTTTTGA
+ pairhits => [ { "*,6" => 1 } ] },
+
+ # -r
+
+ { name => "Raw 1",
+ ref => [ "AGCATCGATCAGTATCTGA" ],
+ raw => "CATCGATCAGTATCTG",
+ hits => [{ 2 => 1 }] },
+
+ { name => "Raw 2",
+ ref => [ "AGCATCGATCAGTATCTGA" ],
+ raw => "CATCGATCAGTATCTG\n",
+ hits => [{ 2 => 1 }] },
+
+ { name => "Raw 3",
+ ref => [ "AGCATCGATCAGTATCTGA" ],
+ raw => "\n\n\nCATCGATCAGTATCTG\n\n",
+ hits => [{ 2 => 1 }] },
+
+ { name => "Raw 7",
+ ref => [ "AGCATCGATCAGTATCTGA" ],
+ raw => "\n\n\r\nCATCGATCAGTATCTG\r\n",
+ args => "--trim3 4",
+ norc => 1,
+ hits => [{ 2 => 1 }] },
+
+ { name => "Raw 8",
+ ref => [ "AGCATCGATCAGTATCTGA" ],
+ raw => "\n\n\r\nCATCGATCAGTATCTG\r\n",
+ args => "--trim3 16",
+ hits => [{ "*" => 1 }] },
+
+ { name => "Raw 9",
+ ref => [ "AGCATCGATCAGTATCTGA" ],
+ raw => "CATCGATCAGTATCTG\n",
+ args => "-s 1",
+ hits => [{ }] },
+
+ { name => "Raw multiread 1",
+ ref => [ "AGCATCGATCAGTATCTGA" ],
+ raw => "\n\n\r\nCATCGATCAGTATCTG\n\n".
+ "\n\n\r\nATCGATCAGTATCTG\n\n",
+ hits => [{ 2 => 1 }, { 3 => 1 }] },
+
+ { name => "Raw multiread 2",
+ ref => [ "AGCATCGATCAGTATCTGA" ],
+ args => "-u 1",
+ raw => "\n\n\r\nCATCGATCAGTATCTG\r\n".
+ "\n\n\r\nATCGATCAGTATCTG\r\n",
+ hits => [{ 2 => 1 }] },
+
+ { name => "Raw multiread 3",
+ ref => [ "AGCATCGATCAGTATCTGA" ],
+ args => "-u 2",
+ raw => "\n\n\r\nCATCGATCAGTATCTG\r\n".
+ "\n\n\r\nATCGATCAGTATCTG\r\n",
+ hits => [{ 2 => 1 }, { 3 => 1 }] },
+
+ { name => "Raw paired 1",
+ ref => [ "AGCATCGATCAAAAACTGA" ],
+ # AGCATCGATC
+ # TCAAAAACTGA
+ # 0123456789012345678
+ raw1 => "\n\n\r\nAGCATCGATC\r\n".
+ "\n\nTCAGTTTTTGA\r\n",
+ raw2 => "\n\n\r\nTCAGTTTTTGA\n".
+ "\n\n\r\nAGCATCGATC",
+ pairhits => [ { "0,8" => 1 }, { "0,8" => 1 } ] },
+
+ { name => "Raw paired 2",
+ ref => [ "AGCATCGATCAAAAACTGA" ],
+ args => "-s 1",
+ # AGCATCGATC
+ # TCAAAAACTGA
+ # 0123456789012345678
+ raw1 => "AGCATCGATC\r\n".
+ "\n\nTCAGTTTTTGA\n",
+ raw2 => "\n\n\r\nTCAGTTTTTGA\n".
+ "\n\n\r\nAGCATCGATC",
+ pairhits => [ { }, { "0,8" => 1 } ] },
+
+ { name => "Raw paired 3",
+ ref => [ "AGCATCGATCAAAAACTGA" ],
+ args => "-u 1",
+ # AGCATCGATC
+ # TCAAAAACTGA
+ # 0123456789012345678
+ raw1 => "\n\n\r\nAGCATCGATC\r\n".
+ "\n\nTCAGTTTTTGA\r\n",
+ raw2 => "\n\n\r\nTCAGTTTTTGA\n".
+ "\n\n\r\nAGCATCGATC",
+ pairhits => [ { "0,8" => 1 }, { } ] },
+
+ # Paired-end reads with left end entirely trimmed away
+ { name => "Raw paired 4",
+ ref => [ "AGCATCGATCAAAAACTGA" ],
+ args => "-3 7",
+ # AGCATCG
+ # GATCAAAAACTGA
+ # 0123456789012345678
+ raw1 => "\nAGCATCG\n",
+ raw2 => "\nGATCAAAAACTGA\n",
+ # GATCAGTTTTTGA
+ pairhits => [ { "*,6" => 1 } ] },
+
+ # --12 / --tab5 / --tab6
+
+ { name => "Tabbed 1",
+ ref => [ "AGCATCGATCAGTATCTGA" ],
+ tabbed => "r0\tCATCGATCAGTATCTG\tIIIIIIIIIIIIIIII",
+ hits => [{ 2 => 1 }] },
+
+ { name => "Tabbed 2",
+ ref => [ "AGCATCGATCAGTATCTGA" ],
+ tabbed => "r0\tCATCGATCAGTATCTG\tIIIIIIIIIIIIIIII\n", # extra newline
+ hits => [{ 2 => 1 }] },
+
+ { name => "Tabbed 3",
+ ref => [ "AGCATCGATCAGTATCTGA" ],
+ tabbed => "\n\n\r\nr0\tCATCGATCAGTATCTG\tIIIIIIIIIIIIIIII\n\n",
+ hits => [{ 2 => 1 }] },
+
+ { name => "Tabbed 4",
+ ref => [ "AGCATCGATCAGTATCTGA" ],
+ tabbed => "\n\n\r\nr0\tCATCGATCAGTATCTG\tIIIIIIIIIIIIIII\n\n", # qual too short
+ should_abort => 1},
+
+ { name => "Tabbed 5",
+ ref => [ "AGCATCGATCAGTATCTGA" ],
+ tabbed => "\n\n\r\nr0\tCATCGATCAGTATCTG\tIIIIIIIIIIIIIIIII\n\n", # qual too long
+ should_abort => 1},
+
+ { name => "Tabbed 7",
+ ref => [ "AGCATCGATCAGTATCTGA" ],
+ tabbed => "\n\n\r\nr0\tCATCGATCAGTATCTG\tIIIIIIIIIIIIIIII\n\n",
+ args => "--trim3 4",
+ norc => 1,
+ hits => [{ 2 => 1 }] },
+
+ { name => "Tabbed 8",
+ ref => [ "AGCATCGATCAGTATCTGA" ],
+ tabbed => "\n\n\r\nr0\tCATCGATCAGTATCTG\tIIIIIIIIIIIIIIII\n\n",
+ args => "--trim5 16",
+ hits => [{ "*" => 1 }] },
+
+ { name => "Tabbed 9",
+ ref => [ "AGCATCGATCAGTATCTGA" ],
+ tabbed => "\n\n\r\nr0\tCATCGATCAGTATCTG\tIIIIIIIIIIIIIIII\n\n",
+ args => "-s 1",
+ hits => [{ }] },
+
+ { name => "Tabbed multiread 1",
+ ref => [ "AGCATCGATCAGTATCTGA" ],
+ tabbed => "\n\n\r\nr0\tCATCGATCAGTATCTG\tIIIIIIIIIIIIIIII\n\n".
+ "\n\n\r\nr1\tATCGATCAGTATCTG\tIIIIIIIIIIIIIII\n\n",
+ hits => [{ 2 => 1 }, { 3 => 1 }] },
+
+ { name => "Tabbed multiread 2",
+ ref => [ "AGCATCGATCAGTATCTGA" ],
+ args => "-u 1",
+ tabbed => "\n\n\r\nr0\tCATCGATCAGTATCTG\tIIIIIIIIIIIIIIII\n\n".
+ "\n\n\r\nr1\tATCGATCAGTATCTG\tIIIIIIIIIIIIIII\n\n",
+ hits => [{ 2 => 1 }] },
+
+ { name => "Tabbed multiread 3",
+ ref => [ "AGCATCGATCAGTATCTGA" ],
+ args => "-u 2",
+ tabbed => "\n\n\r\nr0\tCATCGATCAGTATCTG\tIIIIIIIIIIIIIIII\n\n".
+ "\n\n\r\nr1\tATCGATCAGTATCTG\tIIIIIIIIIIIIIII\n\n",
+ hits => [{ 2 => 1 }, { 3 => 1 }] },
+
+ { name => "Tabbed paired 1",
+ ref => [ "AGCATCGATCAAAAACTGA" ],
+ # AGCATCGATC
+ # TCAAAAACTGA
+ # 0123456789012345678
+ tabbed => "\n\n\r\nr0\tAGCATCGATC\tIIIIIIIIII\tTCAGTTTTTGA\tIIIIIIIIIII\n\n".
+ "\n\nr1\tTCAGTTTTTGA\tIIIIIIIIIII\tAGCATCGATC\tIIIIIIIIII\n\n",
+ paired => 1,
+ pairhits => [ { "0,8" => 1 }, { "0,8" => 1 } ] },
+
+ { name => "Tabbed paired 2",
+ ref => [ "AGCATCGATCAAAAACTGA" ],
+ args => "-s 1",
+ # AGCATCGATC
+ # TCAAAAACTGA
+ # 0123456789012345678
+ tabbed => "r0\tAGCATCGATC\tIIIIIIIIII\tTCAGTTTTTGA\tIIIIIIIIIII\n\n".
+ "\nr1\tTCAGTTTTTGA\tIIIIIIIIIII\tAGCATCGATC\tIIIIIIIIII",
+ paired => 1,
+ pairhits => [ { }, { "0,8" => 1 } ] },
+
+ { name => "Tabbed paired 3",
+ ref => [ "AGCATCGATCAAAAACTGA" ],
+ args => "-u 1",
+ # AGCATCGATC
+ # TCAAAAACTGA
+ # 0123456789012345678
+ tabbed => "\n\n\r\nr0\tAGCATCGATC\tIIIIIIIIII\tTCAGTTTTTGA\tIIIIIIIIIII\n\n".
+ "\n\nr1\tTCAGTTTTTGA\tIIIIIIIIIII\tAGCATCGATC\tIIIIIIIIII",
+ paired => 1,
+ pairhits => [ { "0,8" => 1 }, { } ] },
+
+ # Paired-end reads with left end entirely trimmed away
+ { name => "Tabbed paired 4",
+ ref => [ "AGCATCGATCAAAAACTGA" ],
+ args => "-3 7",
+ # AGCATCG
+ # GATCAAAAACTGA
+ # 0123456789012345678
+ tabbed => "\nr0\tAGCATCG\tIIIIIII\tGATCAAAAACTGA\tIIIIIIIIIIIII\n",
+ paired => 1,
+ pairhits => [ { "*,6" => 1 } ] },
+
+ # --qseq
+
+ { name => "Qseq 1",
+ ref => [ "AGCATCGATCAGTATCTGA" ],
+ qseq => join("\t", "MachName",
+ "RunNum",
+ "Lane",
+ "Tile",
+ "X", "Y",
+ "Index",
+ "0", # Mate
+ "CATCGATCAGTATCTG",
+ "IIIIIIIIIIIIIIII",
+ "1"),
+ hits => [{ 2 => 1 }] },
+
+ { name => "Qseq 2",
+ ref => [ "AGCATCGATCAGTATCTGA" ],
+ qseq => join("\t", "MachName",
+ "RunNum",
+ "Lane",
+ "Tile",
+ "X", "Y",
+ "Index",
+ "0", # Mate
+ "CATCGATCAGTATCTG",
+ "IIIIIIIIIIIIIIII",
+ "1")."\n",
+ hits => [{ 2 => 1 }] },
+
+ { name => "Qseq 3",
+ ref => [ "AGCATCGATCAGTATCTGA" ],
+ qseq => "\n\n\n".join("\t", "MachName",
+ "RunNum",
+ "Lane",
+ "Tile",
+ "X", "Y",
+ "Index",
+ "0", # Mate
+ "CATCGATCAGTATCTG",
+ "IIIIIIIIIIIIIIII",
+ "1")."\n\n",
+ hits => [{ 2 => 1 }] },
+
+ { name => "Qseq 4",
+ ref => [ "AGCATCGATCAGTATCTGA" ],
+ qseq => "\n\n\n".join("\t", "MachName",
+ "RunNum",
+ "Lane",
+ "Tile",
+ "X", "Y",
+ "Index",
+ "0", # Mate
+ "CATCGATCAGTATCTG",
+ "IIIIIIIIIIIIIII",
+ "1")."\n\n",
+ should_abort => 1},
+
+ { name => "Qseq 7",
+ ref => [ "AGCATCGATCAGTATCTGA" ],
+ qseq => "\n\n\n".join("\t", "MachName",
+ "RunNum",
+ "Lane",
+ "Tile",
+ "X", "Y",
+ "Index",
+ "0", # Mate
+ "CATCGATCAGTATCTG",
+ "IIIIIIIIIIIIIIII",
+ "1")."\n\n",
+ args => "--trim3 4",
+ norc => 1,
+ hits => [{ 2 => 1 }] },
+
+ { name => "Qseq 8",
+ ref => [ "AGCATCGATCAGTATCTGA" ],
+ qseq => "\n\n\n".join("\t", "MachName",
+ "RunNum",
+ "Lane",
+ "Tile",
+ "X", "Y",
+ "Index",
+ "0", # Mate
+ "CATCGATCAGTATCTG",
+ "IIIIIIIIIIIIIIII",
+ "1")."\n\n",
+ args => "--trim3 16",
+ hits => [{ "*" => 1 }] },
+
+ { name => "Qseq 9",
+ ref => [ "AGCATCGATCAGTATCTGA" ],
+ qseq => "\n\n\n".join("\t", "MachName",
+ "RunNum",
+ "Lane",
+ "Tile",
+ "X", "Y",
+ "Index",
+ "0", # Mate
+ "CATCGATCAGTATCTG",
+ "IIIIIIIIIIIIIIII",
+ "1")."\n\n",
+ args => "-s 1",
+ hits => [{ }] },
+
+ { name => "Qseq multiread 1",
+ ref => [ "AGCATCGATCAGTATCTGA" ],
+ qseq => "\n\n\n".join("\t", "MachName",
+ "RunNum",
+ "Lane",
+ "Tile",
+ "10", "10",
+ "Index",
+ "0", # Mate
+ "CATCGATCAGTATCTG",
+ "IIIIIIIIIIIIIIII",
+ "1")."\n\n".
+ join("\t", "MachName",
+ "RunNum",
+ "Lane",
+ "Tile",
+ "12", "15",
+ "Index",
+ "0", # Mate
+ "ATCGATCAGTATCTG",
+ "IIIIIIIIIIIIIII",
+ "1")."\n\n",
+ idx_map => { "MachName_RunNum_Lane_Tile_10_10_Index" => 0,
+ "MachName_RunNum_Lane_Tile_12_15_Index" => 1 },
+ hits => [{ 2 => 1 }, { 3 => 1 }] },
+
+ { name => "Qseq multiread 2",
+ ref => [ "AGCATCGATCAGTATCTGA" ],
+ args => "-u 1",
+ qseq => "\n\n\n".join("\t", "MachName",
+ "RunNum",
+ "Lane",
+ "Tile",
+ "10", "10",
+ "Index",
+ "0", # Mate
+ "CATCGATCAGTATCTG",
+ "IIIIIIIIIIIIIIII",
+ "1")."\n\n".
+ join("\t", "MachName",
+ "RunNum",
+ "Lane",
+ "Tile",
+ "12", "15",
+ "Index",
+ "0", # Mate
+ "ATCGATCAGTATCTG",
+ "IIIIIIIIIIIIIII",
+ "1")."\n\n",
+ idx_map => { "MachName_RunNum_Lane_Tile_10_10_Index" => 0,
+ "MachName_RunNum_Lane_Tile_12_15_Index" => 1 },
+ hits => [{ 2 => 1 }] },
+
+ { name => "Qseq multiread 3",
+ ref => [ "AGCATCGATCAGTATCTGA" ],
+ args => "-u 2",
+ qseq => "\n\n\n".join("\t", "MachName",
+ "RunNum",
+ "Lane",
+ "Tile",
+ "10", "10",
+ "Index",
+ "0", # Mate
+ "CATCGATCAGTATCTG",
+ "IIIIIIIIIIIIIIII",
+ "1")."\n\n".
+ join("\t", "MachName",
+ "RunNum",
+ "Lane",
+ "Tile",
+ "12", "15",
+ "Index",
+ "0", # Mate
+ "ATCGATCAGTATCTG",
+ "IIIIIIIIIIIIIII",
+ "1")."\n\n",
+ idx_map => { "MachName_RunNum_Lane_Tile_10_10_Index" => 0,
+ "MachName_RunNum_Lane_Tile_12_15_Index" => 1 },
+ hits => [{ 2 => 1 }, { 3 => 1 }] },
+
+ { name => "Qseq paired 1",
+ ref => [ "AGCATCGATCAAAAACTGA" ],
+ # AGCATCGATC
+ # TCAAAAACTGA
+ # 0123456789012345678
+ qseq1 => "\n\n\n".join("\t", "MachName",
+ "RunNum",
+ "Lane",
+ "Tile",
+ "10", "10",
+ "Index",
+ "1", # Mate
+ "AGCATCGATC",
+ "ABCBGACBCB",
+ "1")."\n\n".
+ join("\t", "MachName",
+ "RunNum",
+ "Lane",
+ "Tile",
+ "12", "15",
+ "Index",
+ "1", # Mate
+ "TCAGTTTTTGA",
+ "95849456875",
+ "1")."\n\n",
+ qseq2 => "\n\n\n".join("\t", "MachName",
+ "RunNum",
+ "Lane",
+ "Tile",
+ "10", "10",
+ "Index",
+ "2", # Mate
+ "TCAGTTTTTGA",
+ "ABCBGACBCBA",
+ "1")."\n\n".
+ join("\t", "MachName",
+ "RunNum",
+ "Lane",
+ "Tile",
+ "12", "15",
+ "Index",
+ "2", # Mate
+ "AGCATCGATC",
+ "AGGCBBGCBG",
+ "1")."\n\n",
+ idx_map => { "MachName_RunNum_Lane_Tile_10_10_Index" => 0,
+ "MachName_RunNum_Lane_Tile_12_15_Index" => 1 },
+ pairhits => [ { "0,8" => 1 }, { "0,8" => 1 } ] },
+
+ { name => "Qseq paired 2",
+ ref => [ "AGCATCGATCAAAAACTGA" ],
+ # AGCATCGATC
+ # TCAAAAACTGA
+ # 0123456789012345678
+ args => "-s 1",
+ qseq1 => "\n\n\n".join("\t", "MachName",
+ "RunNum",
+ "Lane",
+ "Tile",
+ "10", "10",
+ "Index",
+ "1", # Mate
+ "AGCATCGATC",
+ "ABCBGACBCB",
+ "1")."\n\n".
+ join("\t", "MachName",
+ "RunNum",
+ "Lane",
+ "Tile",
+ "12", "15",
+ "Index",
+ "1", # Mate
+ "TCAGTTTTTGA",
+ "95849456875",
+ "1")."\n\n",
+ qseq2 => "\n\n\n".join("\t", "MachName",
+ "RunNum",
+ "Lane",
+ "Tile",
+ "10", "10",
+ "Index",
+ "2", # Mate
+ "TCAGTTTTTGA",
+ "ABCBGACBCBA",
+ "1")."\n\n".
+ join("\t", "MachName",
+ "RunNum",
+ "Lane",
+ "Tile",
+ "12", "15",
+ "Index",
+ "2", # Mate
+ "AGCATCGATC",
+ "AGGCBBGCBG",
+ "1")."\n\n",
+ idx_map => { "MachName_RunNum_Lane_Tile_10_10_Index" => 0,
+ "MachName_RunNum_Lane_Tile_12_15_Index" => 1 },
+ pairhits => [ { }, { "0,8" => 1 } ] },
+
+ { name => "Left-align insertion",
+ ref => [ "GCGATATCTACGACTGCTACGTACAAAAAAAAAAAAAAGTGTTTACGTTGCTAGACTCGATCGATCTGACAGC" ],
+ norc => 1,
+ reads => [ "ACAAAAAAAAAAAAAAAGTGTTTACGTTGCTAGACTCGATCGA" ],
+ # ref: AC-AAAAAAAAAAAAAAGTGTTTACGTTGCTAGACTCGATCGA
+ # read: ACAAAAAAAAAAAAAAAGTGTTTACGTTGCTAGACTCGATCGA
+ # 0123456789012345678901234567890123456789
+ cigar => [ "2M1I40M" ],
+ samoptflags => [ {
+ "MD:Z:42" => 1,
+ "YT:Z:UU" => 1,
+ "NM:i:1" => 1,
+ "XG:i:1" => 1,
+ "XO:i:1" => 1,
+ "AS:i:-8" => 1 } ],
+ report => "",
+ args => ""
+ },
+
+ { name => "Left-align deletion",
+ ref => [ "GCGATATCTACGACTGCTACGTACAAAAAAAAAAAAAAGTGTTTACGTTGCTAGACTCGATCGATCTGACAGC" ],
+ norc => 1,
+ reads => [ "ACGTACAAAAAAAAAAAAAGTGTTTACGTTGCTAGACTCGATCGA" ],
+ # ref: ACGTACAAAAAAAAAAAAAAGTGTTTACGTTGCTAGACTCGATCGA
+ # read: ACGTAC-AAAAAAAAAAAAAGTGTTTACGTTGCTAGACTCGATCGA
+ # 012345678901234567890123456789012345678
+ cigar => [ "6M1D39M" ],
+ samoptflags => [ {
+ "MD:Z:6^A39" => 1,
+ "YT:Z:UU" => 1,
+ "NM:i:1" => 1,
+ "XG:i:1" => 1,
+ "XO:i:1" => 1,
+ "AS:i:-8" => 1 } ],
+ report => "",
+ args => ""
+ },
+
+ { name => "Left-align insertion with mismatch at LHS",
+ ref => [ "GCGATATCTACGACTGCTACGCCCAAAAAAAAAAAAAAGTGTTTACGTTGCTAGACTCGATCGATCTGACAGC" ],
+ norc => 1,
+ reads => [ "TATCTACGACTGCTACGCCCTAAAAAAAAAAAAGTGTTTACGTTGCTAGACTCGATCGATCTGAC" ],
+ # ref: GCGATATCTACGACTGCTACGCCCAAAAAAAAAAAAAAGTGTTTACGTTGCTAGACTCGATCGATCTGACAGC
+ # read: TATCTACGACTGCTACGCCC-TAAAAAAAAAAAAGTGTTTACGTTGCTAGACTCGATCGATCTGAC
+ # 01234567890123456789-012345678901234567890123456789012345678901234
+ # 0 1 2 3 4 5 6
+ cigar => [ "20M1D45M" ],
+ samoptflags => [ {
+ "MD:Z:20^A0A44" => 1,
+ "YT:Z:UU" => 1,
+ "NM:i:2" => 1,
+ "XG:i:1" => 1,
+ "XO:i:1" => 1,
+ "XM:i:1" => 1,
+ "AS:i:-14" => 1 } ],
+ report => "",
+ args => ""
+ },
+
+ # This won't necessarily pass because the original location of the deletion
+ # might
+ #{ name => "Left-align deletion with mismatch at LHS",
+ # ref => [ "GCGATATCTACGACTGCTACGCCCAAAAAAAAAAAAAAGTGTTTACGTTGCTAGACTCGATCGATCTGACAGC" ],
+ # norc => 1,
+ # reads => [ "TATCTACGACTGCTACGCCAAAAAAAAAAAAAAAAGTGTTTACGTTGCTAGACTCGATCGATCTGAC" ],
+ # # ref: GCGATATCTACGACTGCTACGCCC-AAAAAAAAAAAAAAGTGTTTACGTTGCTAGACTCGATCGATCTGACAGC
+ # # read: TATCTACGACTGCTACGCCAAAAAAAAAAAAAAAAGTGTTTACGTTGCTAGACTCGATCGATCTGAC
+ # # 01234567890123456789-012345678901234567890123456789012345678901234
+ # # 0 1 0 1 2 3 4
+ # cigar => [ "20M1I45M" ],
+ # samoptflags => [ {
+ # "MD:Z:19C45" => 1,
+ # "YT:Z:UU" => 1,
+ # "NM:i:2" => 1,
+ # "XG:i:1" => 1,
+ # "XO:i:1" => 1,
+ # "XM:i:1" => 1,
+ # "AS:i:-14" => 1 } ],
+ # report => "",
+ # args => ""
+ #},
+
+ { name => "Flags for when mates align non-concordantly, with many alignments for one",
+ # 012345678
+ ref => [ "CAGCGGCTAGCTATCGATCGTCCGGCAGCTATCATTATGATNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN [...]
+ # 01234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345 [...]
+ # 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 [...]
+ # 0 1 2 3 4 [...]
+ # 0 [...]
+ norc => 1,
+ mate1s => [ "GCGGCTAGCTATCGATCGTCCGGCAGCTATCATTATGA" ],
+ mate2s => [ [...]
+ # [...]
+ # [...]
+ samflags_map => [{ 981 => (1 | 128), 1064 => (1 | 128), 1147 => (1 | 128), 2 => (1 | 64) }],
+ report => "",
+ args => ""
+ },
+
+ { name => "Flags for when mates align non-concordantly, with many alignments for one",
+ # 012345678
+ ref => [ "CAGCGGCTAGCTATCGATCGTCCGGCAGCTATCATTATGATNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNNN [...]
+ # 01234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345 [...]
+ # 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 [...]
+ # 0 1 2 3 4 [...]
+ # 0 [...]
+ norc => 1,
+ mate1s => [ "GCGGCTAGCTATCGATCGTCCGGCAGCTATCATTATGA" ],
+ mate2s => [ [...]
+ tlen_map => [{ 2 => 1021, 981 => -1021 }],
+ samflags_map => [{ 981 => (1 | 128), 2 => (1 | 64) }],
+ report => "",
+ args => ""
+ },
+
+ { name => "Flags for when mates align non-concordantly, with many alignments for one",
+ # 012345678
+ ref => [ "CAGCGGCTAGCTATCGATCGTCCGGCAGCTATCATTATGATAGGATAGATCGCTCGCCTGACCTATATCGCTCGCGATTACGAGCTACGTACTGGCTATCCGAGCTGACGCATCACGACGATCGAGGATAGATCGCTCGCCTGACCTATATCGCTCGCGATTACGAGCTACGTACTGGCTATCCGAGCTGACGCATCACGACGATCGAGGATAGATCGCTCGCCTGACCTATATCGCTCGCGATTACGAGCTACGTACTGGCTATCCGAGCTGACGCATCACGACGATCG" ],
+ # 01234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789
+ # 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8
+ # 0 * 1 * 2 *
+ # 0 [...]
+ norc => 1,
+ mate1s => [ "GCGGCTAGCTATCGATCGTCCGGCAGCTATCATTATGA" ],
+ mate2s => [ "TCGTCGTGATGCGTCAGCTCGGATAGCCAGTACGTAGCTCGT" ],
+ # [...]
+ # [...]
+ samflags_map => [{ 79 => (1 | 2 | 16 | 128), 162 => (1 | 2 | 16 | 128), 245 => (1 | 2 | 16 | 128), 2 => (1 | 2 | 32 | 64) }],
+ report => "",
+ args => ""
+ },
+
+ # Checking MD:Z strings for alignment
+ { name => "MD:Z 1",
+ ref => [ "CACGATCGACTTGA"."C"."TCATCGACGCTATCATTAATATATATAAGCCCGCATCTA" ],
+ reads => [ "CACGATCGACTTGG". "TCATCGACGCTATCATTAATATATATAAGCCCGCATCTA" ],
+ hits => [ { 0 => 1 } ],
+ samoptflags => [ {
+ "AS:i:-14" => 1, # alignment score
+ "XM:i:1" => 1, # num mismatches
+ "XO:i:1" => 1, # num gap opens
+ "XG:i:1" => 1, # num gap extensions
+ "NM:i:2" => 1, # num edits
+ "MD:Z:13^A0C39" => 1, # mismatching positions/bases
+ "YT:Z:UU" => 1, # type of alignment (concordant/discordant/etc)
+ } ] },
+ { name => "MD:Z 2",
+ ref => [ "CACGATCGACTTGA"."A"."TCATCGACGCTATCATTAATATATATAAGCCCGCATCTA" ],
+ reads => [ "CACGATCGACTTGG". "TCATCGACGCTATCATTAATATATATAAGCCCGCATCTA" ],
+ # 0123456789012 012345678901234567890123456789012345678
+ hits => [ { 0 => 1 } ],
+ samoptflags => [ {
+ "AS:i:-14" => 1, # alignment score
+ "XM:i:1" => 1, # num mismatches
+ "XO:i:1" => 1, # num gap opens
+ "XG:i:1" => 1, # num gap extensions
+ "NM:i:2" => 1, # num edits
+ "MD:Z:13^A0A39" => 1, # mismatching positions/bases
+ "YT:Z:UU" => 1, # type of alignment (concordant/discordant/etc)
+ } ] },
+ { name => "MD:Z 3",
+ ref => [ "CACGATCGACTTGT"."AA"."TCATCGACGCTATCATTAATATATATAAGCCCGCATCTA" ],
+ reads => [ "CACGATCGACTTGC". "TCATCGACGCTATCATTAATATATATAAGCCCGCATCTA" ],
+ # 0123456789012 012345678901234567890123456789012345678
+ hits => [ { 0 => 1 } ],
+ samoptflags => [ {
+ "AS:i:-17" => 1, # alignment score
+ "XM:i:1" => 1, # num mismatches
+ "XO:i:1" => 1, # num gap opens
+ "XG:i:2" => 1, # num gap extensions
+ "NM:i:3" => 1, # num edits
+ "MD:Z:13^TA0A39" => 1, # mismatching positions/bases
+ "YT:Z:UU" => 1, # type of alignment (concordant/discordant/etc)
+ } ] },
+ { name => "MD:Z 4",
+ ref => [ "CACGATCGACTTGN"."NN"."TCATCGACGCTATCATTAATATATATAAGCCCGCATCTA" ],
+ reads => [ "CACGATCGACTTGC". "TCATCGACGCTATCATTAATATATATAAGCCCGCATCTA" ],
+ # 0123456789012 012345678901234567890123456789012345678
+ hits => [ { 0 => 1 } ],
+ samoptflags => [ {
+ "AS:i:-12" => 1, # alignment score
+ "XN:i:3" => 1, # num ambiguous ref bases
+ "XM:i:1" => 1, # num mismatches
+ "XO:i:1" => 1, # num gap opens
+ "XG:i:2" => 1, # num gap extensions
+ "NM:i:3" => 1, # num edits
+ "MD:Z:13^NN0N39" => 1, # mismatching positions/bases
+ "YT:Z:UU" => 1, # type of alignment (concordant/discordant/etc)
+ } ] },
+
+ #
+ # Local alignment
+ #
+
+ # Local alignment for a short perfect hit where hit spans the read
+ { name => "Local alignment 1",
+ ref => [ "TTGT" ],
+ reads => [ "TTGT" ],
+ args => "--local --policy \"MIN=L,1.0,0.75\"",
+ hits => [ { 0 => 1 } ],
+ flags => [ "XM:0,XP:0,XT:UU,XC:4=" ],
+ cigar => [ "4M" ],
+ samoptflags => [ {
+ "AS:i:8" => 1, # alignment score
+ "XS:i:0" => 1, # suboptimal alignment score
+ "XN:i:0" => 1, # num ambiguous ref bases
+ "XM:i:0" => 1, # num mismatches
+ "XO:i:0" => 1, # num gap opens
+ "XG:i:0" => 1, # num gap extensions
+ "NM:i:0" => 1, # num edits
+ "MD:Z:4" => 1, # mismatching positions/bases
+ "YM:i:0" => 1, # read aligned repetitively in unpaired fashion
+ "YP:i:0" => 1, # read aligned repetitively in paired fashion
+ "YT:Z:UU" => 1, # type of alignment (concordant/discordant/etc)
+ } ] },
+
+ # T T G A T T G A
+ # T x T x
+ # T x T
+ # G x G
+ # T T
+
+ # Local alignment for a short hit where hit is trimmed at one end
+ { name => "Local alignment 2",
+ ref => [ "TTGA" ],
+ reads => [ "TTGT" ],
+ args => "--local --policy \"MIN=L,1.0,0.75\\;SEED=0,3\\;IVAL=C,1,0\"",
+ report => "-a",
+ hits => [ { 0 => 1 } ],
+ flags => [ "XM:0,XP:0,XT:UU,XC:3=1S" ],
+ cigar => [ "3M1S" ],
+ samoptflags => [ {
+ "AS:i:6" => 1, # alignment score
+ "XS:i:0" => 1, # suboptimal alignment score
+ "XN:i:0" => 1, # num ambiguous ref bases
+ "XM:i:0" => 1, # num mismatches
+ "XO:i:0" => 1, # num gap opens
+ "XG:i:0" => 1, # num gap extensions
+ "NM:i:0" => 1, # num edits
+ "MD:Z:3" => 1, # mismatching positions/bases
+ "YM:i:0" => 1, # read aligned repetitively in unpaired fashion
+ "YP:i:0" => 1, # read aligned repetitively in paired fashion
+ "YT:Z:UU" => 1, # type of alignment (concordant/discordant/etc)
+ } ] },
+
+ # 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5
+ # T T G T T C G T T T G T T C G T
+ # 0 T x
+ # 1 T x
+ # 2 G x
+ # 3 T x
+ # 4 T x
+ # 5 C x
+ # 6 G x
+ # 7 T x
+ # 8 T x
+ # 9 T x
+ # 0 G x
+ # 1 T x
+ # 2 T x
+ #
+ # Score=130
+
+ # 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5
+ # T T G T T C G T T T G T T C G T
+ # 0 T x
+ # 1 T x
+ # 2 G x
+ # 3 T x
+ # 4 T x
+ # 5 C x
+ # 6 G x
+ # 7 T x
+ # 8 T
+ # 9 T
+ # 0 G
+ # 1 T
+ # 2 T
+ #
+ # Score=80
+
+ # Local alignment for a perfect hit
+ { name => "Local alignment 3",
+ # TTGTTCGT
+ # TTGTTCGT
+ ref => [ "TTGTTCGTTTGTTCGT" ],
+ # 0123456789012345
+ # TTGTTCGTTTGTT
+ # TTGTTCGT-----
+ reads => [ "TTGTTCGTTTGTT" ],
+ args => "--local -L 8 -i C,1,0 --score-min=C,12",
+ report => "-a",
+ hits => [ { 0 => 1, 8 => 1 } ],
+ flags_map => [{
+ 0 => "XM:0,XP:0,XT:UU,XC:13=",
+ 8 => "XM:0,XP:0,XT:UU,XC:8="
+ }],
+ cigar_map => [{
+ 0 => "13M",
+ 8 => "8M5S"
+ }],
+ samoptflags_map => [{
+ 0 => { "AS:i:26" => 1, "XS:i:16" => 1, "YT:Z:UU" => 1, "MD:Z:13" => 1 },
+ 8 => { "AS:i:16" => 1, "XS:i:16" => 1, "YT:Z:UU" => 1, "MD:Z:8" => 1 }
+ }]
+ },
+
+ # 1
+ # 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5
+ # T T G T T C G T T T G T T C G T
+ # 0 T x
+ # 1 T x
+ # 2 G x
+ # 3 T x
+ # 4 T x
+ # 5 C x
+ # 6 G x
+ # 7 T x
+ # 8 T
+ # 9 T
+ # 10 G
+ # 1 T
+
+ # Local alignment for a hit that should be trimmed from the right end
+ { name => "Local alignment 4",
+ ref => [ "TTGTTCGTTTGTTCGT" ],
+ reads => [ "TTGTTCGTTTGT" ],
+ args => "--local --policy \"SEED=0,3\\;IVAL=C,1,0\" --score-min=C,12",
+ report => "-a",
+ hits => [ { 0 => 1, 8 => 1 } ],
+ flags_map => [{
+ 0 => "XM:0,XP:0,XT:UU,XC:12=",
+ 8 => "XM:0,XP:0,XT:UU,XC:8=4S"
+ }],
+ cigar_map => [{
+ 0 => "12M",
+ 8 => "8M4S"
+ }],
+ samoptflags_map => [{
+ 0 => { "AS:i:24" => 1, "XS:i:16" => 1, "YT:Z:UU" => 1, "MD:Z:12" => 1 },
+ 8 => { "AS:i:16" => 1, "XS:i:16" => 1, "YT:Z:UU" => 1, "MD:Z:8" => 1 }
+ }]
+ },
+
+ #
+ # Test some common featuers for the manual. E.g. when more than one
+ # alignment is reported in -k mode, what order are they reported in? They
+ # should be in order by alignment score.
+ #
+
+ { name => "Alignment order -k",
+ # 012345678
+ ref => [ "GCGCATGCACATATCANNNNNGCGCATGCACATATCTNNNNNNNNGCGCATGCACATATTTNNNNNNNNNGCGCATGGTGTTATCA" ],
+ reads => [ "GCGCATGCACATATCA" ],
+ quals => [ "GOAIYEFGFIWDSFIU" ],
+ args => "--min-score C,-24,0 -L 4",
+ report => "-k 4"
+ },
+
+ { name => "Alignment order -a",
+ # 012345678
+ ref => [ "GCGCATGCACATATCANNNNNGCGCATGCACATATCTNNNNNNNNGCGCATGCACATATTTNNNNNNNNNGCGCATGGTGTTATCA" ],
+ reads => [ "GCGCATGCACATATCA" ],
+ quals => [ "GOAIYEFGFIWDSFIU" ],
+ args => "--min-score C,-24,0 -L 4",
+ report => "-a"
+ },
+
+ #
+ # What order are mates reported in? Should be reporting in mate1/mate2
+ # order.
+ #
+
+ { name => "Mate reporting order, -a",
+ # 012345678
+ ref => [ "AGCTATCATCACGCGGATATTAGCGCATCGACATTAATATCCCCAAAATAGACGACTCGATCGCGGATTAGGGGTAGACCCCCCCCCGACTNNNNNNNNNNAGCTATCATCACGCGGATATTAGCGCATCGACATTAATATCCCCAAAATAGACGACTCGATCGCGGATTAGGGGTAGACCCCCCCCCGACTNNNNNNNNNNAGCTATCATCACGCGGATATTAGCGCATCGACATTAATATCCCCAAAATAGACGACTCGATCGCGGATTAGGGGTAGACCCCCCCCCGACTNNNNNNNNCGGTAATACGGCCATCGCGGCGGCATTACTCGGCGACTGCACGAGCAGATATTGGGGGTCTAATATAACGTCTCATTAAAACGCTCTAGTCAGCTCATTGGCTCTA" ],
+ mate1s => [ "CTATCATCACGCGGATATT", "GGGGGGGGTCTACCCCTAA", "ATACGGCCATCGCGGCGGCATTACTCGGCG" ],
+ mate2s => [ "GGGGGGGGTCTACCCCTAA", "CTATCATCACGCGGATATT", "AGCCAATGAGCTGACTAGAGCGTTTT" ],
+ quals => [ "GOAIYEFGFIWDSFIU" ],
+ args => "",
+ report => "-a"
+ },
+
+ { name => "Mate reporting order, -M 1",
+ # 012345678
+ ref => [ "AGCTATCATCACGCGGATATTAGCGCATCGACATTAATATCCCCAAAATAGACGACTCGATCGCGGATTAGGGGTAGACCCCCCCCCGACTNNNNNNNNNNAGCTATCATCACGCGGATATTAGCGCATCGACATTAATATCCCCAAAATAGACGACTCGATCGCGGATTAGGGGTAGACCCCCCCCCGACTNNNNNNNNNNAGCTATCATCACGCGGATATTAGCGCATCGACATTAATATCCCCAAAATAGACGACTCGATCGCGGATTAGGGGTAGACCCCCCCCCGACTNNNNNNNNCGGTAATACGGCCATCGCGGCGGCATTACTCGGCGACTGCACGAGCAGATATTGGGGGTCTAATATAACGTCTCATTAAAACGCTCTAGTCAGCTCATTGGCTCTA" ],
+ mate1s => [ "CTATCATCACGCGGATATT", "GGGGGGGGTCTACCCCTAA", "ATACGGCCATCGCGGCGGCATTACTCGGCG" ],
+ mate2s => [ "GGGGGGGGTCTACCCCTAA", "CTATCATCACGCGGATATT", "AGCCAATGAGCTGACTAGAGCGTTTT" ],
+ quals => [ "GOAIYEFGFIWDSFIU" ],
+ args => "",
+ report => "-M 1"
+ },
+
+ #
+ # Test dovetailing, containment, and overlapping
+ #
+ { name => "Non-overlapping; no args",
+ ref => [ "AGCTATCATCACGCGGATATTAGCGCATCGACATTAATATCCCCAAA" ],
+ # 01234567890123456789012345678901234567890123456
+ mate1s => [ "GCTATCATCACGCGGATA" ],
+ mate2s => [ "CGCATCGACATTAATATCC" ],
+ pairhits => [{ "1,23" => 1 }],
+ mate1fw => 1, mate2fw => 1,
+ report => "-M 1"
+ },
+ { name => "Non-overlapping; --no-discordant",
+ ref => [ "AGCTATCATCACGCGGATATTAGCGCATCGACATTAATATCCCCAAA" ],
+ # 01234567890123456789012345678901234567890123456
+ mate1s => [ "GCTATCATCACGCGGATA" ],
+ mate2s => [ "CGCATCGACATTAATATCC" ],
+ pairhits => [{ "1,23" => 1 }],
+ mate1fw => 1, mate2fw => 1,
+ report => "-M 1 --no-discordant"
+ },
+ { name => "Non-overlapping; --no-discordant --no-mixed",
+ ref => [ "AGCTATCATCACGCGGATATTAGCGCATCGACATTAATATCCCCAAA" ],
+ # 01234567890123456789012345678901234567890123456
+ mate1s => [ "GCTATCATCACGCGGATA" ],
+ mate2s => [ "CGCATCGACATTAATATCC" ],
+ pairhits => [{ "1,23" => 1 }],
+ mate1fw => 1, mate2fw => 1,
+ report => "-M 1 --no-discordant --no-mixed"
+ },
+ { name => "Non-overlapping; --no-discordant --no-mixed",
+ ref => [ "AGCTATCATCACGCGGATATTAGCGCATCGACATTAATATCCCCAAA" ],
+ # 01234567890123456789012345678901234567890123456
+ mate1s => [ "GCTATCATCACGCGGATA" ],
+ mate2s => [ "CGCATCGACATTAATATCC" ],
+ pairhits => [{ "1,23" => 1 }],
+ mate1fw => 1, mate2fw => 1,
+ report => "-M 1 --no-discordant --no-mixed"
+ },
+ { name => "Non-overlapping; --no-dovetail",
+ ref => [ "AGCTATCATCACGCGGATATTAGCGCATCGACATTAATATCCCCAAA" ],
+ # 01234567890123456789012345678901234567890123456
+ mate1s => [ "GCTATCATCACGCGGATA" ],
+ mate2s => [ "CGCATCGACATTAATATCC" ],
+ pairhits => [{ "1,23" => 1 }],
+ mate1fw => 1, mate2fw => 1,
+ args => "--no-dovetail",
+ report => "-M 1"
+ },
+ { name => "Non-overlapping; --un-conc=.tmp.simple_tests.pl",
+ ref => [ "AGCTATCATCACGCGGATATTAGCGCATCGACATTAATATCCCCAAA" ],
+ # 01234567890123456789012345678901234567890123456
+ mate1s => [ "GCTATCATCACGCGGATA" ],
+ mate2s => [ "CGCATCGACATTAATATCC" ],
+ pairhits => [{ "1,23" => 1 }],
+ mate1fw => 1, mate2fw => 1,
+ args => "--un-conc=.tmp.simple_tests.pl",
+ report => "-M 1"
+ },
+ { name => "Non-overlapping; --no-overlap",
+ ref => [ "AGCTATCATCACGCGGATATTAGCGCATCGACATTAATATCCCCAAA" ],
+ # 01234567890123456789012345678901234567890123456
+ mate1s => [ "GCTATCATCACGCGGATA" ],
+ mate2s => [ "CGCATCGACATTAATATCC" ],
+ pairhits => [{ "1,23" => 1 }],
+ mate1fw => 1, mate2fw => 1,
+ args => "--no-overlap",
+ report => "-M 1"
+ },
+
+ { name => "Overlapping; no args",
+ ref => [ "AGCTATCATCACGCGGATATTAGCGCATCGACATTAATATCCCCAAA" ],
+ # 01234567890123456789012345678901234567890123456
+ mate1s => [ "GCTATCATCACGCGGATATTA" ],
+ mate2s => [ "TTAGCGCATCGACATTAATATCC" ],
+ pairhits => [{ "1,19" => 1 }],
+ mate1fw => 1, mate2fw => 1,
+ args => "",
+ report => "-M 1"
+ },
+ { name => "Overlapping; --no-dovetail",
+ ref => [ "AGCTATCATCACGCGGATATTAGCGCATCGACATTAATATCCCCAAA" ],
+ # 01234567890123456789012345678901234567890123456
+ mate1s => [ "GCTATCATCACGCGGATATTA" ],
+ mate2s => [ "TTAGCGCATCGACATTAATATCC" ],
+ pairhits => [{ "1,19" => 1 }],
+ mate1fw => 1, mate2fw => 1,
+ args => "--no-dovetail",
+ report => "-M 1"
+ },
+ { name => "Overlapping; --no-contain",
+ ref => [ "AGCTATCATCACGCGGATATTAGCGCATCGACATTAATATCCCCAAA" ],
+ # 01234567890123456789012345678901234567890123456
+ mate1s => [ "GCTATCATCACGCGGATATTA" ],
+ mate2s => [ "TTAGCGCATCGACATTAATATCC" ],
+ pairhits => [{ "1,19" => 1 }],
+ mate1fw => 1, mate2fw => 1,
+ args => "--no-contain",
+ report => "-M 1"
+ },
+ { name => "Overlapping; --no-overlap",
+ ref => [ "AGCTATCATCACGCGGATATTAGCGCATCGACATTAATATCCCCAAA" ],
+ # 01234567890123456789012345678901234567890123456
+ mate1s => [ "GCTATCATCACGCGGATATTA" ],
+ mate2s => [ "TTAGCGCATCGACATTAATATCC" ],
+ pairhits => [],
+ mate1fw => 1, mate2fw => 1,
+ args => "--no-overlap",
+ report => "-M 1"
+ },
+
+ #
+ # Test XS:i with quality scaling
+ #
+
+ { name => "Scoring params 1",
+ # 012345678
+ ref => [ "ACTATTGCGCGCATGCACATATCAATTAAGCCGTCTCTCTAAAGAGACCCCAATCTCGCGCGCTAGACGTCAGTAGTTTAATTTTATAAACACCTCGCTGCGGGG" ],
+ reads => [ "GCGCATGCACATATCAATTAAGCCGTCTCTCTAAAGAGACCCCAATCTCGCGCGCTAGACGTCAGTAGTTTAATTTTATAAACACCTC" ],
+ quals => [ "GOAIYEFGFIWDSFIUYWEHRIWQWLFNSLDKkjdfglduhiuevhsiuqkAUHFIUEHGIUDJFHSKseuweyriwfskdgbiuuhh" ],
+ # 0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567
+ # 0 1 2 3 4 5 6 7 8
+ args => "",
+ report => "-M 1",
+ hits => [ { 8 => 1 } ],
+ cigar => [ "88M" ],
+ samoptflags => [ {
+ "AS:i:0" => 1,
+ "YT:Z:UU" => 1,
+ "MD:Z:88" => 1 } ],
+ },
+
+ { name => "Scoring params 2",
+ # 012345678
+ ref => [ "ACTATTGCGCGCATGCACATATCAATTAAGCCGTCTCTCTAAAGAGACCCCAATCTCGCGCGCTAGACGTCAGTAGTTT"."TTTATAAACACCTCGCTGCGGGG" ],
+ reads => [ "NCGCATGCACATtTCAATTAAGCCGTCTCTCTAAAGA". "CCAATCTCGCGCGCTAGACGTCAGTAGTTTAAATTTATAAACACCTC" ],
+ # * -1 * -6 **** -5 -3 -3 -3 -3 *** -5 -3 -3 -3
+ quals => [ "GOAIYEFGFIWDSFIUYWEHRIWQWLFNSLDKkjdfg". "iuevhsiuqkAUHFIUEHGIUDJFHSKseuweyriwfskdgbiuuhh" ],
+ # 0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567
+ # 0 1 2 3 4 5 6 7 8
+ args => "--ignore-quals --score-min C,-40,0 -N 1 -L 20",
+ report => "-M 1",
+ hits => [ { 8 => 1 } ],
+ cigar => [ "37M4D30M3I14M" ],
+ # 37M4D30M13I4M
+ samoptflags => [ {
+ "AS:i:-38" => 1,
+ "YT:Z:UU" => 1,
+ "MD:Z:0G11A24^GACC44" => 1,
+ "NM:i:9" => 1,
+ "XM:i:2" => 1,
+ "XG:i:7" => 1,
+ "XO:i:2" => 1 } ],
+ },
+
+ { name => "Scoring params 3",
+ # 012345678
+ ref => [ "ACTATTGCGCGCATGCACATATCAATTAAGCCGTCTCTCTAAAGAGACCCCAATCTCGCGCGCTAGACGTCAGTAGTTT"."TTTATAAACACCTCGCTGCGGGG" ],
+ reads => [ "NCGCATGCACATtTCAATTAAGCCGTCTCTCTAAAGA". "CCAATCTCGCGCGCTAGACGTCAGTAGTTTAAATTTATAAACACCTC" ],
+ # * -1 * -6 **** -5 -3 -3 -3 -3 *** -1 -2 -2 -2
+ quals => [ "GOAIYEFGFIWDSFIUYWEHRIWQWLFNSLDKkjdfg". "iuevhsiuqkAUHFIUEHGIUDJFHSKseuweyriwfskdgbiuuhh" ],
+ # 0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567
+ # 0 1 2 3 4 5 6 7 8
+ args => "--ignore-quals --rfg 1,2 --score-min C,-40,0 -N 1 -L 20",
+ report => "-M 1",
+ hits => [ { 8 => 1 } ],
+ cigar => [ "37M4D30M3I14M" ],
+ samoptflags => [ {
+ "AS:i:-31" => 1,
+ "YT:Z:UU" => 1,
+ "MD:Z:0G11A24^GACC44" => 1,
+ "NM:i:9" => 1,
+ "XM:i:2" => 1,
+ "XG:i:7" => 1,
+ "XO:i:2" => 1 } ],
+ },
+
+ { name => "Scoring params 4",
+ # 012345678
+ ref => [ "ACTATTGCGCGCATGCACATATCAATTAAGCCGTCTCTCTAAAGAGACCCCAATCTCGCGCGCTAGACGTCAGTAGTTT"."TTTATAAACACCTCGCTGCGGGG" ],
+ reads => [ "NCGCATGCACATtTCAATTAAGCCGTCTCTCTAAAGA". "CCAATCTCGCGCGCTAGACGTCAGTAGTTTAAATTTATAAACACCTC" ],
+ # * -1 * -6 **** -1 -2 -2 -2 -2 *** -5 -3 -3 -3
+ quals => [ "GOAIYEFGFIWDSFIUYWEHRIWQWLFNSLDKkjdfg". "iuevhsiuqkAUHFIUEHGIUDJFHSKseuweyriwfskdgbiuuhh" ],
+ # 0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567
+ # 0 1 2 3 4 5 6 7 8
+ args => "--ignore-quals --rdg 1,2 --score-min C,-40,0 -N 1 -L 20",
+ report => "-M 1",
+ hits => [ { 8 => 1 } ],
+ cigar => [ "37M4D30M3I14M" ],
+ samoptflags => [ {
+ "AS:i:-30" => 1,
+ "YT:Z:UU" => 1,
+ "MD:Z:0G11A24^GACC44" => 1,
+ "NM:i:9" => 1,
+ "XM:i:2" => 1,
+ "XG:i:7" => 1,
+ "XO:i:2" => 1 } ],
+ },
+
+ { name => "Scoring params 5",
+ # 012345678
+ ref => [ "ACTATTGCGCGCATGCACATATCAATTAAGCCGTCTCTCTAAAGAGACCCCAATCTCGCGCGCTAGACGTCAGTAGTTT"."TTTATAAACACCTCGCTGCGGGG" ],
+ reads => [ "NCGCATGCACATtTCAATTAAGCCGTCTCTCTAAAGA". "CCAATCTCGCGCGCTAGACGTCAGTAGTTTAAATTTATAAACACCTC" ],
+ # * -1 * -8 **** -5 -3 -3 -3 -3 *** -5 -3 -3 -3
+ quals => [ "GOAIYEFGFIWDSFIUYWEHRIWQWLFNSLDKkjdfg". "iuevhsiuqkAUHFIUEHGIUDJFHSKseuweyriwfskdgbiuuhh" ],
+ # 0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567
+ # 0 1 2 3 4 5 6 7 8
+ args => "--ignore-quals --mp 8 --score-min C,-40,0 -N 1 -L 20",
+ report => "-M 1",
+ hits => [ { 8 => 1 } ],
+ cigar => [ "37M4D30M3I14M" ],
+ samoptflags => [ {
+ "AS:i:-40" => 1,
+ "YT:Z:UU" => 1,
+ "MD:Z:0G11A24^GACC44" => 1,
+ "NM:i:9" => 1,
+ "XM:i:2" => 1,
+ "XG:i:7" => 1,
+ "XO:i:2" => 1 } ],
+ },
+
+ { name => "Scoring params 6",
+ # 012345678
+ ref => [ "ACTATTGCGCGCATGCACATATCAATTAAGCCGTCTCTCTAAAGAGACCCCAATCTCGCGCGCTAGACGTCAGTAGTTT"."TTTATAAACACCTCGCTGCGGGG" ],
+ reads => [ "NCGCATGCACATtTCAATTAAGCCGTCTCTCTAAAGA". "CCAATCTCGCGCGCTAGACGTCAGTAGTTTAAATTTATAAACACCTC" ],
+ # * -4 * -6 **** -5 -3 -3 -3 -3 *** -5 -3 -3 -3
+ quals => [ "GOAIYEFGFIWDSFIUYWEHRIWQWLFNSLDKkjdfg". "iuevhsiuqkAUHFIUEHGIUDJFHSKseuweyriwfskdgbiuuhh" ],
+ # 0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567
+ # 0 1 2 3 4 5 6 7 8
+ args => "--ignore-quals --np 4 --score-min C,-41,0 -N 1 -L 20",
+ report => "-M 1",
+ hits => [ { 8 => 1 } ],
+ cigar => [ "37M4D30M3I14M" ],
+ samoptflags => [ {
+ "AS:i:-41" => 1,
+ "YT:Z:UU" => 1,
+ "MD:Z:0G11A24^GACC44" => 1,
+ "NM:i:9" => 1,
+ "XM:i:2" => 1,
+ "XG:i:7" => 1,
+ "XO:i:2" => 1 } ],
+ },
+
+ #
+ # Test XS:i with quality scaling
+ #
+
+ { name => "Q XS:i 1a",
+ ref => [ "TTGTTCGATTGTTCGA" ],
+ reads => [ "TTGTTCGT" ],
+ quals => [ "IIIIIIIA" ],
+ args => "--multiseed=0,7,C,1 --score-min=C,-6",
+ report => "-M 1",
+ hits => [ { 0 => 1, 8 => 1 } ],
+ hits_are_superset => [ 1 ],
+ cigar => [ "8M" ],
+ samoptflags => [ {
+ "AS:i:-5" => 1, "XS:i:-5" => 1,
+ "YM:i:1" => 1, "YT:Z:UU" => 1,
+ "MD:Z:7A0" => 1,
+ "NM:i:1" => 1, "XM:i:1" => 1 } ],
+ },
+
+ { name => "Q XS:i 1a ! --mp 3,3",
+ ref => [ "TTGTTCGATTGTTCGA" ],
+ reads => [ "TTGTTCGT" ],
+ quals => [ "IIIIIII!" ],
+ args => "-L 6 --mp 3,3 --score-min=C,-6",
+ report => "-M 1",
+ hits => [ { 0 => 1, 8 => 1 } ],
+ hits_are_superset => [ 1 ],
+ cigar => [ "8M" ],
+ samoptflags => [ {
+ "AS:i:-3" => 1, "XS:i:-3" => 1,
+ "YM:i:1" => 1, "YT:Z:UU" => 1,
+ "MD:Z:7A0" => 1,
+ "NM:i:1" => 1, "XM:i:1" => 1 } ],
+ },
+
+ { name => "Q XS:i 1a ! --mp 3,6",
+ ref => [ "TTGTTCGATTGTTCGA" ],
+ reads => [ "TTGTTCGT" ],
+ quals => [ "IIIIIII!" ],
+ args => "-L 6 --mp 6,3 --score-min=C,-6",
+ report => "-M 1",
+ hits => [ { 0 => 1, 8 => 1 } ],
+ hits_are_superset => [ 1 ],
+ cigar => [ "8M" ],
+ samoptflags => [ {
+ "AS:i:-3" => 1, "XS:i:-3" => 1,
+ "YM:i:1" => 1, "YT:Z:UU" => 1,
+ "MD:Z:7A0" => 1,
+ "NM:i:1" => 1, "XM:i:1" => 1 } ],
+ },
+
+ { name => "Q XS:i 1a I --mp 3,3",
+ ref => [ "TTGTTCGATTGTTCGA" ],
+ reads => [ "TTGTTCGT" ],
+ quals => [ "IIIIIIII" ],
+ args => "-L 6 --mp 3,3 --score-min=C,-6",
+ report => "-M 1",
+ hits => [ { 0 => 1, 8 => 1 } ],
+ hits_are_superset => [ 1 ],
+ cigar => [ "8M" ],
+ samoptflags => [ {
+ "AS:i:-3" => 1, "XS:i:-3" => 1,
+ "YM:i:1" => 1, "YT:Z:UU" => 1,
+ "MD:Z:7A0" => 1,
+ "NM:i:1" => 1, "XM:i:1" => 1 } ],
+ },
+
+ { name => "Q XS:i 1a I --mp 3,6",
+ ref => [ "TTGTTCGATTGTTCGA" ],
+ reads => [ "TTGTTCGT" ],
+ quals => [ "IIIIIIII" ],
+ args => "-L 6 --mp 6,3 --score-min=C,-6",
+ report => "-M 1",
+ hits => [ { 0 => 1, 8 => 1 } ],
+ hits_are_superset => [ 1 ],
+ cigar => [ "8M" ],
+ samoptflags => [ {
+ "AS:i:-6" => 1, "XS:i:-6" => 1,
+ "YM:i:1" => 1, "YT:Z:UU" => 1,
+ "MD:Z:7A0" => 1,
+ "NM:i:1" => 1, "XM:i:1" => 1 } ],
+ },
+
+ { name => "Q XS:i 1a --ignore-quals",
+ ref => [ "TTGTTCGATTGTTCGA" ],
+ reads => [ "TTGTTCGT" ],
+ quals => [ "IIIIIIIA" ],
+ args => "--multiseed=0,7,C,1 --score-min=C,-6 --ignore-quals",
+ report => "-M 1",
+ hits => [ { 0 => 1, 8 => 1 } ],
+ hits_are_superset => [ 1 ],
+ cigar => [ "8M" ],
+ samoptflags => [ {
+ "AS:i:-6" => 1, "XS:i:-6" => 1,
+ "YM:i:1" => 1, "YT:Z:UU" => 1,
+ "MD:Z:7A0" => 1,
+ "NM:i:1" => 1, "XM:i:1" => 1 } ],
+ },
+
+ { name => "Q XS:i 1b",
+ ref => [ "TTGTTCGATTGTTCGA" ],
+ reads => [ "TTGTTCGT" ],
+ quals => [ "IIIIIII5" ],
+ args => "--multiseed=0,7,C,1 --score-min=C,-6",
+ report => "-M 1",
+ hits => [ { 0 => 1, 8 => 1 } ],
+ hits_are_superset => [ 1 ],
+ cigar => [ "8M" ],
+ samoptflags => [ {
+ "AS:i:-4" => 1, "XS:i:-4" => 1,
+ "YM:i:1" => 1, "YT:Z:UU" => 1,
+ "MD:Z:7A0" => 1,
+ "NM:i:1" => 1, "XM:i:1" => 1 } ],
+ },
+
+ { name => "Q XS:i 1b --ignore-quals",
+ ref => [ "TTGTTCGATTGTTCGA" ],
+ reads => [ "TTGTTCGT" ],
+ quals => [ "IIIIIII5" ],
+ args => "--multiseed=0,7,C,1 --score-min=C,-6 --ignore-quals",
+ report => "-M 1",
+ hits => [ { 0 => 1, 8 => 1 } ],
+ hits_are_superset => [ 1 ],
+ cigar => [ "8M" ],
+ samoptflags => [ {
+ "AS:i:-6" => 1, "XS:i:-6" => 1,
+ "YM:i:1" => 1, "YT:Z:UU" => 1,
+ "MD:Z:7A0" => 1,
+ "NM:i:1" => 1, "XM:i:1" => 1 } ],
+ },
+
+ { name => "Q XS:i 1c",
+ ref => [ "TTGTTCGATTGTTCGA" ],
+ reads => [ "TTGTTCGT" ],
+ quals => [ "IIIIIII4" ],
+ args => "--multiseed=0,7,C,1 --score-min=C,-6",
+ report => "-M 1",
+ hits => [ { 0 => 1, 8 => 1 } ],
+ hits_are_superset => [ 1 ],
+ cigar => [ "8M" ],
+ samoptflags => [ {
+ "AS:i:-3" => 1, "XS:i:-3" => 1,
+ "YM:i:1" => 1, "YT:Z:UU" => 1,
+ "MD:Z:7A0" => 1,
+ "NM:i:1" => 1, "XM:i:1" => 1 } ],
+ },
+
+ { name => "Q XS:i 1c --ignore-quals",
+ ref => [ "TTGTTCGATTGTTCGA" ],
+ reads => [ "TTGTTCGT" ],
+ quals => [ "IIIIIII4" ],
+ args => "--multiseed=0,7,C,1 --score-min=C,-6 --ignore-quals",
+ report => "-M 1",
+ hits => [ { 0 => 1, 8 => 1 } ],
+ hits_are_superset => [ 1 ],
+ cigar => [ "8M" ],
+ samoptflags => [ {
+ "AS:i:-6" => 1, "XS:i:-6" => 1,
+ "YM:i:1" => 1, "YT:Z:UU" => 1,
+ "MD:Z:7A0" => 1,
+ "NM:i:1" => 1, "XM:i:1" => 1 } ],
+ },
+
+ # One mate aligns. Ensuring that the unmapped mate gets reference
+ # information filled in from the other mate.
+ { ref => [ "CATCGACTGAGACTCGTACGACAATTACGCGCATTATTCGCATCACCAGCGCGGCGCGCGCCCCCTAT" ],
+ # 01234567890123456789012345678901234567890123456789012345678901234567
+ # 0 1 2 3 4 5 6
+ # ATCACCAGCGTTTCGCGCGAAACCTA
+ mate1s => [ "ATCGACTGAGACTCGTACGACAATTAC" ],
+ mate2s => [ "TAGGTTTCGCGCGAAACGCTGGTGAT" ],
+ pairhits_orig => [{ "1,1" => 1}]
+ },
+
+ { ref => [ "TTGTTCGTTTGTTCGTTTGTTCGTTTGTTCGTTTGTTCGTTTGTTCGTTTGTTCGTTTGTTCGTTTGTTCGTTTGTTCGTTTGTTCGTTTGTTCGTTTGTTCGTTTGTTCGTTTGTTCGTTTGTTCGTTTGTTCGTTTGTTCGTTTGTTCGTTTGTTCGTTTGTTCGTTTGTTCGTTTGTTCGTTTGTTCGTTTGTTCGTTTGTTCGTTTGTTCGTTTGTTCGTTTGTTCGTTTGTTCGTTTGTTCGTTTGTTCGTTTGTTCGTTTGTTCGTTTGTTCGTTTGTTCGTTTGTTCGTTTGTTCGTTTGTTCGTTTGTTCGTTTGTTCGTTTGTTCGTTTGTTCGTTTGTTCGTTTGTTCGTTTGTTCGTTTGTTCGTTTGTTCGTTTGTTCGTTTGTTCGTTTGTTCGTTTGTTCGTTTGTTCGTTTGTTCGTTTGTTCGTTTGTTCGTTTGTTCGTTTGTTCGTTTGTTCGTTTGT [...]
+ reads => [ "TTGTTCGT" ],
+ args => "--multiseed=0,4,C,1,0",
+ report => "-M 1"
+ },
+
+ # Testing that DEFAULT is -M 1
+ { ref => [ "TTGTTCGTTTGTTCGT" ],
+ reads => [ "TTGTTCGT" ],
+ report => "-M 1",
+ hits => [ { 0 => 1, 8 => 1 } ],
+ hits_are_superset => [ 1 ],
+ cigar => [ "8M" ],
+ samoptflags => [
+ { "YM:i:1" => 1, "YT:Z:UU" => 1, "MD:Z:8" => 1, "YM:i:1" => 1 }
+ ],
+ },
+ { ref => [ "TTGTTCGTTTGTTCGT" ],
+ reads => [ "TTGTTCGT" ],
+ report => "-M 1",
+ hits => [ { 0 => 1, 8 => 1 } ],
+ hits_are_superset => [ 1 ],
+ cigar => [ "8M" ],
+ samoptflags => [
+ { "YM:i:1" => 1, "YT:Z:UU" => 1, "MD:Z:8" => 1, "YM:i:1" => 1 }
+ ],
+ },
+
+ #
+ # Test XS:i
+ #
+
+ { name => "XS:i 1",
+ ref => [ "TTGTTCGATTGTTCGA" ],
+ reads => [ "TTGTTCGT" ],
+ args => "--multiseed=0,7,C,1 --score-min=C,-6",
+ report => "-M 1",
+ hits => [ { 0 => 1, 8 => 1 } ],
+ hits_are_superset => [ 1 ],
+ cigar => [ "8M" ],
+ samoptflags => [ {
+ "AS:i:-6" => 1, "XS:i:-6" => 1,
+ "YM:i:1" => 1, "YT:Z:UU" => 1,
+ "MD:Z:7A0" => 1,
+ "NM:i:1" => 1, "XM:i:1" => 1 } ],
+ },
+
+ { name => "XS:i 2",
+ ref => [ "TTGTTCGATTGTTCGA" ],
+ reads => [ "TTGTTCGT" ],
+ args => "--multiseed=0,7,C,1 --score-min=C,-5",
+ report => "",
+ cigar => [ "*" ],
+ samoptflags => [{ "YT:Z:UU" => 1, "YM:i:0" => 1 }],
+ },
+
+ { name => "XS:i 3a",
+ ref => [ "TTGTTCGATTGTTCGT" ],
+ # TTGTTCGT
+ reads => [ "TTGTTCGT" ],
+ args => "--multiseed=0,7,C,1 --score-min=C,-6",
+ report => "-M 1",
+ hits => [ { 8 => 1 } ],
+ cigar => [ "8M" ],
+ samoptflags => [ {
+ "AS:i:0" => 1, "XS:i:-6" => 1,
+ "YM:i:1" => 1, "YT:Z:UU" => 1,
+ "MD:Z:8" => 1,
+ "NM:i:0" => 1, "XM:i:0" => 1 } ],
+ },
+
+ { name => "XS:i 3b",
+ ref => [ "TTGTTCGATTGTTCGT" ],
+ # TTGTTCGT
+ reads => [ "TTGTTCGT" ],
+ args => "--multiseed=0,7,C,1 --score-min=C,-6 --seed=52",
+ report => "-M 1",
+ hits => [ { 8 => 1 } ],
+ cigar => [ "8M" ],
+ samoptflags => [ {
+ "AS:i:0" => 1, "XS:i:-6" => 1,
+ "YM:i:1" => 1, "YT:Z:UU" => 1,
+ "MD:Z:8" => 1,
+ "NM:i:0" => 1, "XM:i:0" => 1 } ],
+ },
+
+ { name => "XS:i 3c",
+ ref => [ "TTGTTCGATTGTTCGT" ],
+ # TTGTTCGT
+ reads => [ "TTGTTCGT" ],
+ args => "--multiseed=0,7,C,1 --score-min=C,-6 --seed=53",
+ report => "-M 2",
+ hits => [ { 8 => 1 } ],
+ cigar => [ "8M" ],
+ samoptflags => [ {
+ "AS:i:0" => 1, "XS:i:-6" => 1,
+ "YM:i:0" => 1, "YT:Z:UU" => 1,
+ "MD:Z:8" => 1,
+ "NM:i:0" => 1, "XM:i:0" => 1 } ],
+ },
+
+ { name => "XS:i 4a",
+ ref => [ "TTGTTCAATTGTTCGATTGTTCGT" ],
+ # |||||| ||||||| ||||||||
+ # TTGTTCGT||||||| ||||||||
+ # TTGTTCGT||||||||
+ # TTGTTCGT
+ reads => [ "TTGTTCGT" ],
+ args => "--multiseed=0,6,C,1 --score-min=C,-12 --seed=53",
+ report => "-M 2",
+ hits => [ { 16 => 1 } ],
+ cigar => [ "8M" ],
+ samoptflags => [ {
+ "AS:i:0" => 1, "XS:i:-6" => 1,
+ "YM:i:1" => 1, "YT:Z:UU" => 1,
+ "MD:Z:8" => 1,
+ "NM:i:0" => 1, "XM:i:0" => 1 } ],
+ },
+
+ { name => "XS:i 4b",
+ ref => [ "TTGTTCAATTGTTCGATTGTTCGT" ],
+ # |||||| ||||||| ||||||||
+ # TTGTTCGT||||||| ||||||||
+ # TTGTTCGT||||||||
+ # TTGTTCGT
+ reads => [ "TTGTTCGT" ],
+ args => "--multiseed=0,6,C,1 --score-min=C,-12 --seed=54",
+ report => "-M 3",
+ hits => [ { 16 => 1 } ],
+ cigar => [ "8M" ],
+ samoptflags => [ {
+ "AS:i:0" => 1, "XS:i:-6" => 1,
+ "YM:i:0" => 1, "YT:Z:UU" => 1,
+ "MD:Z:8" => 1,
+ "NM:i:0" => 1, "XM:i:0" => 1 } ],
+ },
+
+ { name => "XS:i 5a",
+ ref => [ "TTGTTCAATTGTTCGATTGTTCGTTTGTTCAATTGTTCAATTGTTCAATTGTTCAATTGTTCAATTGTTCAATTGTTCAATTGTTCAATTGTTCAATTGTTCAATTGTTCAATTGTTCAA" ],
+ # |||||| ||||||| |||||||||||||| |||||| |||||| |||||| |||||| |||||| |||||| |||||| |||||| |||||| |||||| ||||||
+ # TTGTTCGT||||||| ||||||||TTGTTCGT|||||| TTGTTCGT|||||| TTGTTCGT|||||| TTGTTCGT|||||| TTGTTCGT|||||| TTGTTCGT||||||
+ # TTGTTCGT|||||||| TTGTTCGT TTGTTCGT TTGTTCGT TTGTTCGT TTGTTCGT TTGTTCGT
+ # TTGTTCGT
+ reads => [ "TTGTTCGT" ],
+ args => "--multiseed=0,6,C,1,1 --score-min=C,-12 --seed=54",
+ report => "-M 1",
+ hits => [ { 16 => 1 } ],
+ cigar => [ "8M" ],
+ samoptflags => [ {
+ "AS:i:0" => 1, "XS:i:-6" => 1,
+ "YM:i:1" => 1, "YT:Z:UU" => 1,
+ "MD:Z:8" => 1,
+ "NM:i:0" => 1, "XM:i:0" => 1 } ],
+ },
+
+ { name => "XS:i 5b",
+ ref => [ "TTGTTCAATTGTTCGATTGTTCGTTTGTTCAATTGTTCAATTGTTCAATTGTTCAATTGTTCAATTGTTCAATTGTTCAATTGTTCAATTGTTCAATTGTTCAATTGTTCAATTGTTCAA" ],
+ # |||||| ||||||| |||||||||||||| |||||| |||||| |||||| |||||| |||||| |||||| |||||| |||||| |||||| |||||| ||||||
+ # TTGTTCGT||||||| ||||||||TTGTTCGT|||||| TTGTTCGT|||||| TTGTTCGT|||||| TTGTTCGT|||||| TTGTTCGT|||||| TTGTTCGT||||||
+ # TTGTTCGT|||||||| TTGTTCGT TTGTTCGT TTGTTCGT TTGTTCGT TTGTTCGT TTGTTCGT
+ # TTGTTCGT
+ reads => [ "TTGTTCGT" ],
+ args => "--multiseed=0,5,C,1,1 --score-min=C,-12 --seed=55",
+ report => "-M 1",
+ hits => [ { 16 => 1 } ],
+ cigar => [ "8M" ],
+ samoptflags => [ {
+ "AS:i:0" => 1, "XS:i:-6" => 1,
+ "YM:i:1" => 1, "YT:Z:UU" => 1,
+ "MD:Z:8" => 1,
+ "NM:i:0" => 1, "XM:i:0" => 1 } ],
+ },
+
+ # Testing BWA-SW-like scoring
+ #
+ # a*max{T,c*log(l)} = 1 * max(30, 5.5 * log(56)) = 1 * max(30, 22.139) = 30
+ #
+ { name => "BWA-SW-like 1",
+ ref => [ "GTTTAGATTCCACTACGCTAACCATCGAGAACTCGTCTCAGAGTTTCGATAGGAAAATCTGCGA" ],
+ # ||||||||||||||||||||||||||||||||||||||||||||||||||||||||
+ reads => [ "TAGATTCCACTACGCTAACCATCGAGAACTCGTCTCAGAGTTTCGATAGGAAAATC" ],
+ # 01234567890123456789012345678901234567890123456789012345
+ # 1 2 3 4 5
+ args => "--bwa-sw-like",
+ hits => [{ 3 => 1 }],
+ samoptflags => [{ "AS:i:56" => 1, "NM:i:0" => 1,
+ "MD:Z:56" => 1, "YT:Z:UU" => 1 }]
+ },
+ { name => "BWA-SW-like 2",
+ # 0123
+ ref => [ "GTTTAGATTCCACTACGCTAACCATCGAGAACTCGTCTCAGAGTTTCGATAGGAAAATCTGCGA" ],
+ # |||||||||||||||||||||||||| ||||||||||||||||||||||||||||
+ reads => [ "TAGATTCCACTACGCTAACCATCGAGTTCTCGTCTCAGAGTTTCGATAGGAAAATC" ],
+ # 01234567890123456789012345678901234567890123456789012345
+ # 1 2 3 4 5
+ args => "--bwa-sw-like -L 18",
+ hits => [{ 3 => 1 }],
+ # Tot matches = 54
+ # Tot penalties = 6
+ samoptflags => [{ "AS:i:48" => 1, "NM:i:2" => 1, "XM:i:2" => 1,
+ "MD:Z:26A0A28" => 1, "YT:Z:UU" => 1 }]
+ },
+ { name => "BWA-SW-like 3",
+ # 0123
+ ref => [ "GTTTAGATTCCACTACGCTAACCATCGAGAACTCGTCTCAGAGTTTCGATAGGAAAATCTGCGA" ],
+ # |||||||||||||||||||||||||| |||||||||||||||||||||||||||
+ reads => [ "TAGATTCCACTACGCTAACCATCGAG"."TCGTCTCAGAGTTTCGATAGGAAAATC" ],
+ # 01234567890123456789012345678901234567890123456789012345
+ # 1 2 3 4 5
+ args => "--bwa-sw-like -i C,1,0",
+ hits => [{ 3 => 1 }],
+ # Tot matches = 53
+ # Tot penalties = 11
+ samoptflags => [{ "AS:i:42" => 1, "NM:i:3" => 1, "XM:i:0" => 1,
+ "XO:i:1" => 1, "XG:i:3" => 1,
+ "MD:Z:26^AAC27" => 1, "YT:Z:UU" => 1 }]
+ },
+
+ # Some tricky SAM FLAGS field tests
+
+ { name => "SAM paired-end where both mates align 1",
+ ref => [ "GCACTATCTACGCTTCGGCGTCGGCGAAAAAACGCACGACCGGGTGTGTGACAATCATATATAGCGCGC" ],
+ # 012345678901234567890123456789012345678901234567890123456789012345678
+ # 0 1 2 3 4 5 6
+ mate1s => [ "CTATCTACGCTTCGGCGTCGGTGA" ],
+ mate2s => [ "GATTGTCACACACCCGGTCGT" ],
+ # -----------------------------------------------------
+ # 01234567890123456789012345678901234567890123456789012
+ # 0 1 2 3 4 5
+ # 0x1 template having multiple fragments in sequencing
+ # 0x2 each fragment properly aligned according to the aligner
+ # 0x4 fragment unmapped
+ # 0x8 next fragment in the template unmapped
+ # 0x10 SEQ being reverse complemented
+ # 0x20 SEQ of the next fragment in the template being reversed
+ # 0x40 the first fragment in the template
+ # 0x80 the last fragment in the template
+ pairhits => [{ "3,35" => 1 }],
+ norc => 1,
+ samflags_map => [{ 3 => (1 | 2 | 32 | 64), 35 => (1 | 2 | 16 | 128) }],
+ tlen_map => [{ 3 => 53, 35 => -53 }] },
+
+ { name => "SAM paired-end where both mates align 2",
+ ref => [ "GCACTATCTACGCTTCGGCGTCGGCGAAAAAACGCACGACCGGGTGTGTGACAATCATATATAGCGCGC" ],
+ # 012345678901234567890123456789012345678901234567890123456789012345678
+ # 0 1 2 3 4 5 6
+ mate1s => [ "TCACCGACGCCGAAGCGTAGATAG" ],
+ mate2s => [ "ACGACCGGGTGTGTGACAATC" ],
+ # -----------------------------------------------------
+ # 01234567890123456789012345678901234567890123456789012
+ # 0 1 2 3 4 5
+ # 0x1 template having multiple fragments in sequencing
+ # 0x2 each fragment properly aligned according to the aligner
+ # 0x4 fragment unmapped
+ # 0x8 next fragment in the template unmapped
+ # 0x10 SEQ being reverse complemented
+ # 0x20 SEQ of the next fragment in the template being reversed
+ # 0x40 the first fragment in the template
+ # 0x80 the last fragment in the template
+ mate1fw => 0,
+ mate2fw => 1,
+ pairhits => [{ "3,35" => 1 }],
+ norc => 1,
+ samflags_map => [{ 3 => (1 | 2 | 16 | 64), 35 => (1 | 2 | 32 | 128) }],
+ tlen_map => [{ 3 => 53, 35 => -53 }] },
+
+ { name => "SAM paired-end where both mates align 3",
+ ref => [ "GCACTATCTACGCTTCGGCGTCGGCGAAAAAACGCACGACCGGGTGTGTGACAATCATATATAGCGCGC" ],
+ # 012345678901234567890123456789012345678901234567890123456789012345678
+ # 0 1 2 3 4 5 6
+ mate1s => [ "CTATCTACGCTTCGGCGTCGGTGA" ],
+ mate2s => [ "ACGACCGGGTGTGTGACAATC" ],
+ # -----------------------------------------------------
+ # 01234567890123456789012345678901234567890123456789012
+ # 0 1 2 3 4 5
+ # 0x1 template having multiple fragments in sequencing
+ # 0x2 each fragment properly aligned according to the aligner
+ # 0x4 fragment unmapped
+ # 0x8 next fragment in the template unmapped
+ # 0x10 SEQ being reverse complemented
+ # 0x20 SEQ of the next fragment in the template being reversed
+ # 0x40 the first fragment in the template
+ # 0x80 the last fragment in the template
+ mate1fw => 1,
+ mate2fw => 1,
+ pairhits => [{ "3,35" => 1 }],
+ norc => 1,
+ samflags_map => [{ 3 => (1 | 2 | 64), 35 => (1 | 2 | 128) }],
+ tlen_map => [{ 3 => 53, 35 => -53 }] },
+
+ { name => "SAM paired-end where mate #1 aligns but mate #2 doesn't",
+ ref => [ "GCACTATCTACGCTTCGGCGTCGGCGAAAAAACGCACGACCGGGTGTGTGACAATCATATATAGCGCGC" ],
+ # 012345678901234567890123456789012345678901234567890123456789012345678
+ # 0 1 2 3 4 5 6
+ mate1s => [ "CTATCTACGCTTCGGCGTCGGCGA" ],
+ mate2s => [ "GATTGTCTTTTCCCGGAAAAATCGT" ],
+ # 0x1 template having multiple fragments in sequencing
+ # 0x2 each fragment properly aligned according to the aligner
+ # 0x4 fragment unmapped
+ # 0x8 next fragment in the template unmapped
+ # 0x10 SEQ being reverse complemented
+ # 0x20 SEQ of the next fragment in the template being reversed
+ # 0x40 the first fragment in the template
+ # 0x80 the last fragment in the template
+ pairhits => [{ "*,3" => 1 }],
+ norc => 1,
+ samflags_map => [{ 3 => (1 | 8 | 64), "*" => (1 | 4 | 128) }] },
+
+ { name => "SAM paired-end where neither mate aligns",
+ ref => [ "GCACTATCTACGCTTCGGCGTCGGCGAAAAAACGCACGACCGGGTGTGTGACAATCATATATAGCGCGC" ],
+ # 012345678901234567890123456789012345678901234567890123456789012345678
+ # 0 1 2 3 4 5 6
+ mate1s => [ "CTATATACGAAAAAGCGTCGGCGA" ],
+ mate2s => [ "GATTGTCTTTTCCCGGAAAAATCGT" ],
+ # 0x1 template having multiple fragments in sequencing
+ # 0x2 each fragment properly aligned according to the aligner
+ # 0x4 fragment unmapped
+ # 0x8 next fragment in the template unmapped
+ # 0x10 SEQ being reverse complemented
+ # 0x20 SEQ of the next fragment in the template being reversed
+ # 0x40 the first fragment in the template
+ # 0x80 the last fragment in the template
+ pairhits => [{ "*,*" => 1 }],
+ norc => 1,
+ samoptflags_flagmap => [{
+ (1 | 4 | 8 | 64) => { "YT:Z:UP" => 1 },
+ (1 | 4 | 8 | 128) => { "YT:Z:UP" => 1 }
+ }] },
+
+ { name => "SAM paired-end where both mates align, but discordantly",
+ ref => [ "GCACTATCTACGCTTCGGCGTCGGCGAAAAAACGCACGACCGGGTGTGTGACAATCATATATAGCGCGC" ],
+ # 012345678901234567890123456789012345678901234567890123456789012345678
+ # 0 1 2 3 4 5 6
+ mate1s => [ "CTATCTACGCTTCGGCGTCGGCGA" ],
+ mate2s => [ "ACGACCGGGTGTGTGACAATC" ],
+ # -----------------------------------------------------
+ # 01234567890123456789012345678901234567890123456789012
+ # 0 1 2 3 4 5
+ # 0x1 template having multiple fragments in sequencing
+ # 0x2 each fragment properly aligned according to the aligner
+ # 0x4 fragment unmapped
+ # 0x8 next fragment in the template unmapped
+ # 0x10 SEQ being reverse complemented
+ # 0x20 SEQ of the next fragment in the template being reversed
+ # 0x40 the first fragment in the template
+ # 0x80 the last fragment in the template
+ pairhits => [{ "3,35" => 1 }],
+ norc => 1,
+ samflags_map => [{ 3 => (1 | 64), 35 => (1 | 128) }],
+ # Which TLEN is right? Depends on criteria for when to infer TLEN. If
+ # criterion is mates are concordant, then it should be 0 here. If the
+ # criterion is that both mates align to the same chromosome, should be
+ # +-53
+ #tlen_map => [{ 3 => 0, 35 => 0 }] },
+ tlen_map => [{ 3 => 53, 35 => -53 }] },
+
+ { name => "matchesRef regression 4",
+ ref => [ "CCGGGTCGTCACGCCCCGCTTGCGTCANGCCCCTCACCCTCCCTTTGTCGGCTCCCACCCCTCCCCATCCGTTGTCCCCGCCCCCGCCCGCCGGGTCGTCACGCCCCGCTTGCGTCANGC",
+ "GCTCGGAATTCGTGCTCCGNCCCGTACGGTT" ],
+ #
+ # NNNNNGA------A-------------------G-NTTT
+ # ||||||||||||||||||||||||||||||||||
+ # CCAAT-ATTTTTAATTTCCTCTATTTTTCTCTCGTCTTG
+ args => "--policy \"NP=Q\\;RDG=46.3220993654702\\;RFG=41.3796024365659\\;MIN=L,5.57015383125426,-3.28597145122829\\;NCEIL=L,0.263054599454459,0.130843661549367\\;SEED=1,29\\;IVAL=L,0.0169183264663712,3.75762168662522\" --overhang --trim5 6",
+ reads => [ "CTTTGCACCCCTCCCTTGTCGGCTCCCACCCATCCCCATCCGTTGTCCCCGCCCCCGCCCGCCGGTCGTCACTCCCCGTTTGCGTCATGCCCCTCACCCTCCCTTTGTCGGCTCGCACCCCTCCCCATCCGTTGTCCCCGCCCCCGCTCTCGGGGTCTTCACGCCCCGCTTGCTTCATGCCCCTCACTCGCACCCCG" ],
+ },
+
+ { name => "matchesRef regression 3",
+ ref => [ "GAAGNTTTTCCAATATTTTTAATTTCCTCTATTTTTCTCTCGTCTTGNTCTAC" ],
+ #
+ # NNNNNGA------A-------------------G-NTTT
+ # ||||||||||||||||||||||||||||||||||
+ # CCAAT-ATTTTTAATTTCCTCTATTTTTCTCTCGTCTTG
+ args => "--policy \"MMP=R\\;MIN=L,8.8,-8.1\" --overhang",
+ reads => [ "CAAGACGAGAGAAAAATAGAGGAAATTAAAAATATTGG" ],
+ },
+
+ { name => "matchesRef regression 2",
+ ref => ["GTTGTCGGCAGCTCTGGATATGTGNTCTCGGGTTTATNTCGTTGTCG",
+ "CCTTGTTNTTAATGCTGCCTGGTTTNG"],
+ args => "--policy \"RDG=2.02030755427021,2.81949533273331\\;MIN=L,-6.52134769703939,-3.39889659588514\\;IVAL=L,0.127835912101927\" --overhang --trim5 5",
+ mate1s => ["TCTGGCGGTTGCGAAGGCCCCTGGCGGTTGCTATGTCCTCTGGCGGTTGCGTTGTCGGCAGCTCG"],
+ mate2s => ["AGAACACATATCCAGAGCTGCCGACAACGAAATGAACCCGAGAGCACAAATCCAGAG"] },
+
+ # Regression test for an issue observed once
+ { name => "matchesRef regression 1",
+ # 0 1 2 3 4 5 6 7
+ # 01234567890123456789012345678901234567890123456789012345678901234567890
+ ref => [ "AGGTCGACCGAAAGGCCTAGAGGTCGACCGACAATCTGACCATGGGGCGAGGAGCGAGTAC" ],
+ # ||||||||||||||||||||||||||||||||||||||||||||||||||
+ reads => [ "AAGGCCTAGAGGTCGACCGACAATCTGACCATGGGGCGAGGAGCGAGTACTGGTCTGGGG" ],
+ # 012345678901234567890123456789012345678901234567890123456789
+ # 0 1 2 3 4 5
+ args => "--overhang" },
+
+ # 1 discordant alignment and one concordant alignment. Discordant because
+ # the fragment is too long.
+
+ { name => "Discordant with different chromosomes",
+ ref => [ "TTTATAAAAATATTTCCCCCCCC",
+ "CCCCCCTGTCGCTACCGCCCCCCCCCCC" ],
+ # ATAAAAATAT GTCGCTACCG
+ # ATAAAAATAT TGTCGCTACC
+ # 01234567890123456789012
+ # 0 1 2
+ # 0123456789012345678901234567
+ # 0 1 2
+ mate1s => [ "ATAAAAATAT", "ATAAAAATAT" ],
+ mate2s => [ "GTCGCTACCG", "TGTCGCTACC" ],
+ mate1fw => 1,
+ mate2fw => 1,
+ args => "-I 0 -X 35",
+ # Not really any way to flag an alignment as discordant
+ pairhits => [ { "3,7" => 1 }, { "3,6" => 1 } ],
+ rnext_map => [ { 3 => 1, 7 => 0 }, { 3 => 1, 6 => 0 } ],
+ pnext_map => [ { 3 => 7, 7 => 3 }, { 3 => 6, 6 => 3 } ] },
+
+ # Paired-end reads that should align
+ #{ name => "Fastq paired 4",
+ # ref => [ "AGCATCGATCAAAAACTGA" ],
+ # args => "-s 1 -L 4 -i C,1,0",
+ # # AGCATCGATC
+ # # TCAAAAACTGA
+ # # 0123456789012345678
+ # fastq1 => "\n\n\r\n\@r0\nAGCATCGATC\r\n+\n\nIIIIIIIIII\n\n".
+ # #"\n\n\@r1\nTC\r\n+\n\nII\n\n".
+ # "\n\n\@r2\nTCAGTTTTTGA\r\n+\n\nIIIIIIIIIII\n\n",
+ # fastq2 => "\n\n\r\n\@r0\nTCAGTTTTTGA\n+\n\nIIIIIIIIIII\n\n".
+ # #"\n\n\r\n\@r1\nAG\r\n+\nII".
+ # "\n\@r2\nAGCATCGATC\r\n+\nIIIIIIIIII",
+ # paired => 1,
+ # pairhits => [ { }, { "*,*" => 1 }, { "0,8" => 1 } ],
+ # pairhits => [ { "0,8" => 1 } ],
+ # samoptflags_map => [
+ # { },
+ # { "*" => { "YT:Z:UP" => 1, "YF:Z:LN" => 1 } },
+ ## { 0 => { "MD:Z:10" => 1, "YT:Z:CP" => 1 },
+ # 8 => { "MD:Z:11" => 1, "YT:Z:CP" => 1 } }]
+ #},
+
+ #{ name => "Tabbed paired 4",
+ # ref => [ "AGCATCGATCAAAAACTGA" ],
+ # args => "-s 1 -L 4 -i C,1,0",
+ # # AGCATCGATC
+ # # TCAAAAACTGA
+ # # 0123456789012345678
+ # tabbed => "\n\n\r\nr0\tAGCATCGATC\tIIIIIIIIII\tTCAGTTTTTGA\tIIIIIIIIIII\n\n".
+ # "\n\nr1\tTC\tII\tAG\tII".
+ # "\n\nr2\tTCAGTTTTTGA\tIIIIIIIIIII\tAGCATCGATC\tIIIIIIIIII\n\n",
+ # paired => 1,
+ # #pairhits => [ { }, { "*,*" => 1 }, { "0,8" => 1 } ],
+ # pairhits => [ { }, { "0,8" => 1 } ],
+ # samoptflags_map => [
+ # { },
+ # #{ "*" => { "YT:Z:UP" => 1, "YF:Z:LN" => 1 } },
+ # { 0 => { "MD:Z:10" => 1, "YT:Z:CP" => 1 },
+# 8 => { "MD:Z:11" => 1, "YT:Z:CP" => 1 } }]
+ #},
+
+ #{ name => "Fasta paired 4",
+ # ref => [ "AGCATCGATCAAAAACTGA" ],
+ # args => "-s 1 -L 4 -i C,1,0",
+ # # AGCATCGATC
+ # # TCAAAAACTGA
+ # # 0123456789012345678
+ # fasta1 => "\n\n\r\n>r0\nAGCATCGATC\r\n".
+ # # "\n\n>r1\nTC\r\n".
+ # "\n\n>r2\nTCAGTTTTTGA\r\n",
+ # fasta2 => "\n\n\r\n>r0\nTCAGTTTTTGA\n\n".
+ # # "\n\n\r\n>r1\nAG".
+ # "\n>r2\nAGCATCGATC",
+ # # pairhits => [ { }, { "*,*" => 1 }, { "0,8" => 1 } ],
+ # pairhits => [ { }, { "0,8" => 1 } ],
+ # samoptflags_map => [
+ # { },
+ # #{ "*" => { "YT:Z:UP" => 1, "YF:Z:LN" => 1 } },
+ # { 0 => { "MD:Z:10" => 1, "YT:Z:CP" => 1 },
+ # 8 => { "MD:Z:11" => 1, "YT:Z:CP" => 1 } }]
+ #},
+
+ #{ name => "Raw paired 4",
+ # ref => [ "AGCATCGATCAAAAACTGA" ],
+ # args => "-s 1 -L 4 -i C,1,0",
+ # # AGCATCGATC
+ # # TCAAAAACTGA
+ # # 0123456789012345678
+ # raw1 => "\n\n\r\nAGCATCGATC\r\n".
+ ## "\n\nTC\r\n".
+ # "\n\nTCAGTTTTTGA\r\n",
+ # raw2 => "\n\n\r\nTCAGTTTTTGA\n\n".
+ # "\n\n\r\nAG".
+ # "\nAGCATCGATC",
+ # pairhits => [ { }, { "*,*" => 1 }, { "0,8" => 1 } ],
+ # pairhits => [ { }, { "0,8" => 1 } ],
+ # samoptflags_map => [
+ # { },
+ # { "*" => { "YT:Z:UP" => 1, "YF:Z:LN" => 1 } },
+ # { 0 => { "MD:Z:10" => 1, "YT:Z:CP" => 1 },
+# 8 => { "MD:Z:11" => 1, "YT:Z:CP" => 1 } }]
+ #},
+
+ #
+ # Check that skipping of empty reads is handled correctly. A read that is
+ # empty or becomes empty after --trim3/--trim5 are applied should still
+ # count as a first-class read that gets propagated up into the alignment
+ # loop. And it should be counted in the -s/-u totals.
+ #
+
+ { ref => [ "AGCATCGATCAGTATCTGA" ],
+ reads => [ "", "ATCGATCAGTA" ],
+ args => "-s 1",
+ hits => [ {}, { 3 => 1 }] },
+
+ { ref => [ "AGCATCGATCAGTATCTGA" ],
+ mate1s => [ "", "AGCATCGATC" ],
+ mate2s => [ "", "TCAGATACTG" ],
+ args => "-s 1",
+ pairhits => [ {}, { "0,9" => 1 }] },
+
+ { ref => [ "AGCATCGATCAGTATCTGA" ],
+ reads => [ "", "ATCGATCAGTA" ],
+ args => "-s 2",
+ hits => [ {}, {} ] },
+
+ { ref => [ "AGCATCGATCAGTATCTGA" ],
+ mate1s => [ "", "AGCATCGATC" ],
+ mate2s => [ "", "TCAGATACTG" ],
+ args => "-s 2",
+ pairhits => [ {}, {} ] },
+
+ { ref => [ "AGCATCGATCAGTATCTGA" ],
+ reads => [ "", "ATCGATCAGTA", "AGTATCTGA" ],
+ args => "-s 1 -u 1",
+ hits => [ {}, { 3 => 1 }] },
+
+ { ref => [ "AGCATCGATCAGTATCTGA" ],
+ reads => [ "AC", "ATCGATCAGTA" ],
+ args => "-s 1 --trim3 2",
+ norc => 1,
+ hits => [ {}, { 3 => 1 }] },
+
+ { ref => [ "AGCATCGATCAGTATCTGA" ],
+ reads => [ "AC", "ATCGATCAGTA" ],
+ args => "-s 1 --trim3 2",
+ nofw => 1,
+ hits => [ {}, { 5 => 1 }] },
+
+ { ref => [ "AGCATCGATCAGTATCTGA" ],
+ reads => [ "AC", "ATCGATCAGTA" ],
+ args => "-s 1 --trim5 2",
+ nofw => 1,
+ hits => [ {}, { 3 => 1 }] },
+
+ { ref => [ "AGCATCGATCAGTATCTGA" ],
+ reads => [ "AC", "ATCGATCAGTA" ],
+ args => "-s 1 --trim5 2",
+ norc => 1,
+ hits => [ {}, { 5 => 1 }] },
+
+ #
+ # Alignment with overhang
+ #
+
+ { ref => [ "TGC" ],
+ reads => [ "ATGC" ],
+ args => "--overhang --policy \"SEED=0,3\\;IVAL=C,1,0\\;NCEIL=L,1,0\"",
+ hits => [ { 0 => 1 } ],
+ cigar => [ "1S3M" ],
+ samoptflags => [
+ { "AS:i:-1" => 1, "YT:Z:UU" => 1, "MD:Z:3" => 1, "XN:i:1" => 1 } ]
+ },
+
+ { ref => [ "TTGTTCGT" ],
+ reads => [ "TTGTTCG" ],
+ args => "--policy \"SEED=0,2\\;IVAL=C,1,0\\;NCEIL=L,2,0\"",
+ hits => [ { 0 => 1 } ],
+ cigar => [ "7M" ],
+ samoptflags => [ { "AS:i:0" => 1, "YT:Z:UU" => 1, "MD:Z:7" => 1 } ]
+ },
+
+ { ref => [ "TTGTTCGT" ],
+ reads => [ "TTGTTCG" ],
+ args => "",
+ hits => [ { 0 => 1 } ],
+ flags => [ "XM:0,XP:0,XT:UU,XC:7=" ],
+ cigar => [ "7M" ],
+ samoptflags => [ { "AS:i:0" => 1, "YT:Z:UU" => 1, "MD:Z:7" => 1 } ]
+ },
+
+ { ref => [ "TTGTTCGT" ],
+ reads => [ "TGTTCGT", "TTGTTCG" ],
+ args => "--overhang",
+ hits => [ { 1 => 1 }, { 0 => 1 } ],
+ flags => [ "XM:0,XP:0,XT:UU,XC:7=", "XM:0,XP:0,XT:UU,XC:7=" ],
+ cigar => [ "7M", "7M" ],
+ samoptflags => [
+ { "YT:Z:UU" => 1, "MD:Z:7" => 1 },
+ { "YT:Z:UU" => 1, "MD:Z:7" => 1 }
+ ]
+ },
+
+ { ref => [ "TTGTTCGT" ],
+ reads => [ "TGTTCGT", "TTGTTCG" ],
+ args => "",
+ hits => [ { 1 => 1 }, { 0 => 1 } ],
+ flags => [ "XM:0,XP:0,XT:UU,XC:7=", "XM:0,XP:0,XT:UU,XC:7=" ],
+ cigar => [ "7M", "7M" ],
+ samoptflags => [
+ { "YT:Z:UU" => 1, "MD:Z:7" => 1 },
+ { "YT:Z:UU" => 1, "MD:Z:7" => 1 }
+ ]
+ },
+
+ # Reads 1 and 2 don't have overhang, reads 3 and 4 overhang opposite ends
+ { ref => [ "TTGTTCGT" ],
+ # TGTTCGT
+ # GTTCGTA
+ # ATTGTTC
+ reads => [ "TGTTCGT", "GTTCGTA", "ATTGTTC" ],
+ args => "--overhang --policy \"SEED=0,2\\;IVAL=C,1,0\\;NCEIL=L,2,0\"",
+ hits => [ { 1 => 1 }, { 2 => 1 }, { 0 => 1 } ],
+ cigar => [ "7M", "6M1S", "1S6M" ],
+ samoptflags => [
+ { "YT:Z:UU" => 1, "MD:Z:7" => 1 },
+ { "AS:i:-1" => 1, "XN:i:1" => 1, "YT:Z:UU" => 1, "MD:Z:6" => 1 },
+ { "AS:i:-1" => 1, "XN:i:1" => 1, "YT:Z:UU" => 1, "MD:Z:6" => 1 }
+ ]},
+
+ # Same as previous case but --overhang not specified
+ { ref => [ "TTGTTCGT" ],
+ reads => [ "TGTTCGT", "TTGTTCG", "GTTCGTA", "ATTGTTC" ],
+ args => "--policy \"SEED=0,2\\;IVAL=C,1,0\\;NCEIL=L,2,0\"",
+ hits => [ { 1 => 1 }, { 0 => 1 } ], # only the internal hits
+ cigar => [ "7M", "7M", "*", "*" ],
+ samoptflags => [
+ { "YT:Z:UU" => 1, "MD:Z:7" => 1 },
+ { "YT:Z:UU" => 1, "MD:Z:7" => 1 },
+ { "YT:Z:UU" => 1 },
+ { "YT:Z:UU" => 1 }
+ ]
+ },
+
+ # A simple case that should align with or without overhang, with or without
+ # a special NCEIL setting.
+ { ref => [ "TTGTTCGT" ],
+ reads => [ "TTGTTCG" ],
+ args => "--overhang --policy \"SEED=0,2\\;IVAL=C,1,0\\;NCEIL=L,2,0\"",
+ hits => [ { 0 => 1 } ]},
+
+ { ref => [ "TTGTTCGT" ],
+ reads => [ "TTGTTCG" ],
+ args => "--overhang",
+ hits => [ { 0 => 1 } ]},
+
+ #
+ # Testing the various -M/-m/-k/-a alignment modes in both unpaired and
+ # paired-end modes. Ensuring that SAM optional flags such as YM:i, YP:i
+ # are set properly in all cases.
+ #
+
+ #
+ # Paired-end
+ #
+
+ { name => "P.M.58.G.b Unpaired -M 5 w/ 8 hits global, but mate #1 has just 1",
+ # 0 1 2 3 0 1 2 0 1 2 0 1 2
+ # 012345678901234567890123456789012 0123456789012345678901234567 0123456789012345678901234567 0123456789012345678901234567 [...]
+ # CAGCGTACGGTATCTAGCTATGGGCATCGATCG ACACACCCCTATAGCTCGGAGCTGACTG ACACACCCCTATAGCTCGGAGCTGACTG ACACACCCCTATAGCTCGGAGCTGACTG [...]
+ ref => [ "AGACGCAGTCACCAGCGTACGGTATCTAGCTATGGGCATCGATCGACGACGTACGAGCGGTATCTACAGCCACTCATCACACACCCCTATAGCTCGGAGCTGACTGGGTTACTGGGGGGGATGCGTATCGACTATCGACAATATGACGCGTCGGTCACCCCATAATATGCAAAAATTATAGCTCACGACGCGTACTAATAGAAAACGCGCTATCAGCCTCCGACGCGGCGGTATCGAAGACGCAGTCACACACACCCCTATAGCTCGGAGCTGACTGGATCGACGACGTACGAGCGGTATCTACAGCCACTCATCACACACCCCTATAGCTCGGAGCTGACTGGGTTACTGGGGGGGATGCGTATCGACTATCGACAATATGACGCGTCGGTCACCCCATAATATGCAAAAATTATAGCTCACGACGCGTACTAATAGAAAACGCGCTATCAGCCTCCGACGCGGCGGTATCGAAG [...]
+ # 01234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345 [...]
+ # 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 [...]
+ # 0 1 2 3 4 [...]
+ # 0 0 0 0 0 [...]
+ mate1s => [ "CAGCGTACGGTATCTAGCTATGGGCATCGATCG" ],
+ mate2s => [ "CAGTCAGCTCCGAGCTATAGGGGTGTGT" ], # rev comped
+ args => "-X 1000",
+ report => "-M 5",
+ pairhits => [{ "12,78" => 1, "12,249" => 1, "12,315" => 1,
+ "12,486" => 1, "12,552" => 1, "12,723" => 1,
+ "12,789" => 1, "12,960" => 1 }],
+ hits_are_superset => [ 1 ],
+ cigar_map => [{
+ 12 => "33M", 78 => "28M",
+ 249 => "28M", 315 => "28M",
+ 486 => "28M", 552 => "28M",
+ 723 => "28M", 789 => "28M",
+ 960 => "28M"
+ }],
+ samoptflags_map => [ {
+ 12 => { "AS:i:0" => 1, "XS:i:0" => 1, "XN:i:0" => 1, "XM:i:0" => 1,
+ "XO:i:0" => 1, "XG:i:0" => 1, "NM:i:0" => 1, "MD:Z:33" => 1,
+ "YM:i:0" => 1, "YP:i:1" => 1, "YT:Z:CP" => 1, "YS:i:0" => 1 },
+ 78 => { "AS:i:0" => 1, "XS:i:0" => 1, "XN:i:0" => 1, "XM:i:0" => 1,
+ "XO:i:0" => 1, "XG:i:0" => 1, "NM:i:0" => 1, "MD:Z:28" => 1,
+ "YM:i:1" => 1, "YP:i:1" => 1, "YT:Z:CP" => 1, "YS:i:0" => 1 },
+ 249 => { "AS:i:0" => 1, "XS:i:0" => 1, "XN:i:0" => 1, "XM:i:0" => 1,
+ "XO:i:0" => 1, "XG:i:0" => 1, "NM:i:0" => 1, "MD:Z:28" => 1,
+ "YM:i:1" => 1, "YP:i:1" => 1, "YT:Z:CP" => 1, "YS:i:0" => 1 },
+ 315 => { "AS:i:0" => 1, "XS:i:0" => 1, "XN:i:0" => 1, "XM:i:0" => 1,
+ "XO:i:0" => 1, "XG:i:0" => 1, "NM:i:0" => 1, "MD:Z:28" => 1,
+ "YM:i:1" => 1, "YP:i:1" => 1, "YT:Z:CP" => 1, "YS:i:0" => 1 },
+ 486 => { "AS:i:0" => 1, "XS:i:0" => 1, "XN:i:0" => 1, "XM:i:0" => 1,
+ "XO:i:0" => 1, "XG:i:0" => 1, "NM:i:0" => 1, "MD:Z:28" => 1,
+ "YM:i:1" => 1, "YP:i:1" => 1, "YT:Z:CP" => 1, "YS:i:0" => 1 },
+ 552 => { "AS:i:0" => 1, "XS:i:0" => 1, "XN:i:0" => 1, "XM:i:0" => 1,
+ "XO:i:0" => 1, "XG:i:0" => 1, "NM:i:0" => 1, "MD:Z:28" => 1,
+ "YM:i:1" => 1, "YP:i:1" => 1, "YT:Z:CP" => 1, "YS:i:0" => 1 },
+ 723 => { "AS:i:0" => 1, "XS:i:0" => 1, "XN:i:0" => 1, "XM:i:0" => 1,
+ "XO:i:0" => 1, "XG:i:0" => 1, "NM:i:0" => 1, "MD:Z:28" => 1,
+ "YM:i:1" => 1, "YP:i:1" => 1, "YT:Z:CP" => 1, "YS:i:0" => 1 },
+ 789 => { "AS:i:0" => 1, "XS:i:0" => 1, "XN:i:0" => 1, "XM:i:0" => 1,
+ "XO:i:0" => 1, "XG:i:0" => 1, "NM:i:0" => 1, "MD:Z:28" => 1,
+ "YM:i:1" => 1, "YP:i:1" => 1, "YT:Z:CP" => 1, "YS:i:0" => 1 },
+ 960 => { "AS:i:0" => 1, "XS:i:0" => 1, "XN:i:0" => 1, "XM:i:0" => 1,
+ "XO:i:0" => 1, "XG:i:0" => 1, "NM:i:0" => 1, "MD:Z:28" => 1,
+ "YM:i:1" => 1, "YP:i:1" => 1, "YT:Z:CP" => 1, "YS:i:0" => 1 },
+ } ] },
+
+ { name => "P.M.58.L.b Unpaired -M 5 w/ 8 hits local, but mate #1 has just 1",
+ # 0 1 2 3 0 1 2 0 1 2 0 1 2
+ # 012345678901234567890123456789012 0123456789012345678901234567 0123456789012345678901234567 0123456789012345678901234567 [...]
+ # CAGCGTACGGTATCTAGCTATGGGCATCGATCG ACACACCCCTATAGCTCGGAGCTGACTG ACACACCCCTATAGCTCGGAGCTGACTG ACACACCCCTATAGCTCGGAGCTGACTG [...]
+ ref => [ "AGACGCAGTCACCAGCGTACGGTATCTAGCTATGGGCATCGATCGACGACGTACGAGCGGTATCTACAGCCACTCATCACACACCCCTATAGCTCGGAGCTGACTGGGTTACTGGGGGGGATGCGTATCGACTATCGACAATATGACGCGTCGGTCACCCCATAATATGCAAAAATTATAGCTCACGACGCGTACTAATAGAAAACGCGCTATCAGCCTCCGACGCGGCGGTATCGAAGACGCAGTCACACACACCCCTATAGCTCGGAGCTGACTGGATCGACGACGTACGAGCGGTATCTACAGCCACTCATCACACACCCCTATAGCTCGGAGCTGACTGGGTTACTGGGGGGGATGCGTATCGACTATCGACAATATGACGCGTCGGTCACCCCATAATATGCAAAAATTATAGCTCACGACGCGTACTAATAGAAAACGCGCTATCAGCCTCCGACGCGGCGGTATCGAAG [...]
+ # 01234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345 [...]
+ # 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 [...]
+ # 0 1 2 3 4 [...]
+ # 0 0 0 0 0 [...]
+ mate1s => [ "CAGCGTACGGTATCTAGCTATGGGCATCGATCG" ],
+ mate2s => [ "CAGTCAGCTCCGAGCTATAGGGGTGTGT" ], # rev comped
+ args => "-X 1000 --local",
+ report => "-M 5",
+ pairhits => [{ "12,78" => 1, "12,249" => 1, "12,315" => 1,
+ "12,486" => 1, "12,552" => 1, "12,723" => 1,
+ "12,789" => 1, "12,960" => 1 }],
+ hits_are_superset => [ 1 ],
+ cigar_map => [{
+ 12 => "33M", 78 => "28M",
+ 249 => "28M", 315 => "28M",
+ 486 => "28M", 552 => "28M",
+ 723 => "28M", 789 => "28M",
+ 960 => "28M"
+ }],
+ samoptflags_map => [ {
+ 12 => { "AS:i:66" => 1, "XS:i:0" => 1, "XN:i:0" => 1, "XM:i:0" => 1,
+ "XO:i:0" => 1, "XG:i:0" => 1, "NM:i:0" => 1, "MD:Z:33" => 1,
+ "YM:i:0" => 1, "YP:i:1" => 1, "YT:Z:CP" => 1, "YS:i:56" => 1 },
+ 78 => { "AS:i:56" => 1, "XS:i:56" => 1, "XN:i:0" => 1, "XM:i:0" => 1,
+ "XO:i:0" => 1, "XG:i:0" => 1, "NM:i:0" => 1, "MD:Z:28" => 1,
+ "YM:i:1" => 1, "YP:i:1" => 1, "YT:Z:CP" => 1, "YS:i:66" => 1 },
+ 249 => { "AS:i:56" => 1, "XS:i:56" => 1, "XN:i:0" => 1, "XM:i:0" => 1,
+ "XO:i:0" => 1, "XG:i:0" => 1, "NM:i:0" => 1, "MD:Z:28" => 1,
+ "YM:i:1" => 1, "YP:i:1" => 1, "YT:Z:CP" => 1, "YS:i:66" => 1 },
+ 315 => { "AS:i:56" => 1, "XS:i:56" => 1, "XN:i:0" => 1, "XM:i:0" => 1,
+ "XO:i:0" => 1, "XG:i:0" => 1, "NM:i:0" => 1, "MD:Z:28" => 1,
+ "YM:i:1" => 1, "YP:i:1" => 1, "YT:Z:CP" => 1, "YS:i:66" => 1 },
+ 486 => { "AS:i:56" => 1, "XS:i:56" => 1, "XN:i:0" => 1, "XM:i:0" => 1,
+ "XO:i:0" => 1, "XG:i:0" => 1, "NM:i:0" => 1, "MD:Z:28" => 1,
+ "YM:i:1" => 1, "YP:i:1" => 1, "YT:Z:CP" => 1, "YS:i:66" => 1 },
+ 552 => { "AS:i:56" => 1, "XS:i:56" => 1, "XN:i:0" => 1, "XM:i:0" => 1,
+ "XO:i:0" => 1, "XG:i:0" => 1, "NM:i:0" => 1, "MD:Z:28" => 1,
+ "YM:i:1" => 1, "YP:i:1" => 1, "YT:Z:CP" => 1, "YS:i:66" => 1 },
+ 723 => { "AS:i:56" => 1, "XS:i:56" => 1, "XN:i:0" => 1, "XM:i:0" => 1,
+ "XO:i:0" => 1, "XG:i:0" => 1, "NM:i:0" => 1, "MD:Z:28" => 1,
+ "YM:i:1" => 1, "YP:i:1" => 1, "YT:Z:CP" => 1, "YS:i:66" => 1 },
+ 789 => { "AS:i:56" => 1, "XS:i:56" => 1, "XN:i:0" => 1, "XM:i:0" => 1,
+ "XO:i:0" => 1, "XG:i:0" => 1, "NM:i:0" => 1, "MD:Z:28" => 1,
+ "YM:i:1" => 1, "YP:i:1" => 1, "YT:Z:CP" => 1, "YS:i:66" => 1 },
+ 960 => { "AS:i:56" => 1, "XS:i:56" => 1, "XN:i:0" => 1, "XM:i:0" => 1,
+ "XO:i:0" => 1, "XG:i:0" => 1, "NM:i:0" => 1, "MD:Z:28" => 1,
+ "YM:i:1" => 1, "YP:i:1" => 1, "YT:Z:CP" => 1, "YS:i:66" => 1 },
+ } ] },
+
+ { name => "P.k.58.G.b Unpaired -k 5 w/ 8 hits global, but mate #1 has just 1",
+ # 0 1 2 3 0 1 2 0 1 2 0 1 2
+ # 012345678901234567890123456789012 0123456789012345678901234567 0123456789012345678901234567 0123456789012345678901234567 [...]
+ # CAGCGTACGGTATCTAGCTATGGGCATCGATCG ACACACCCCTATAGCTCGGAGCTGACTG ACACACCCCTATAGCTCGGAGCTGACTG ACACACCCCTATAGCTCGGAGCTGACTG [...]
+ ref => [ "AGACGCAGTCACCAGCGTACGGTATCTAGCTATGGGCATCGATCGACGACGTACGAGCGGTATCTACAGCCACTCATCACACACCCCTATAGCTCGGAGCTGACTGGGTTACTGGGGGGGATGCGTATCGACTATCGACAATATGACGCGTCGGTCACCCCATAATATGCAAAAATTATAGCTCACGACGCGTACTAATAGAAAACGCGCTATCAGCCTCCGACGCGGCGGTATCGAAGACGCAGTCACACACACCCCTATAGCTCGGAGCTGACTGGATCGACGACGTACGAGCGGTATCTACAGCCACTCATCACACACCCCTATAGCTCGGAGCTGACTGGGTTACTGGGGGGGATGCGTATCGACTATCGACAATATGACGCGTCGGTCACCCCATAATATGCAAAAATTATAGCTCACGACGCGTACTAATAGAAAACGCGCTATCAGCCTCCGACGCGGCGGTATCGAAG [...]
+ # 01234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345 [...]
+ # 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 [...]
+ # 0 1 2 3 4 [...]
+ # 0 0 0 0 0 [...]
+ mate1s => [ "CAGCGTACGGTATCTAGCTATGGGCATCGATCG" ],
+ mate2s => [ "CAGTCAGCTCCGAGCTATAGGGGTGTGT" ], # rev comped
+ args => "-X 1000",
+ report => "-k 5",
+ pairhits => [{ "12,78" => 1, "12,249" => 1, "12,315" => 1,
+ "12,486" => 1, "12,552" => 1, "12,723" => 1,
+ "12,789" => 1, "12,960" => 1 }],
+ hits_are_superset => [ 1 ],
+ cigar_map => [{
+ 12 => "33M", 78 => "28M",
+ 249 => "28M", 315 => "28M",
+ 486 => "28M", 552 => "28M",
+ 723 => "28M", 789 => "28M",
+ 960 => "28M"
+ }],
+ samoptflags_map => [ {
+ 12 => { "AS:i:0" => 1, "XS:i:0" => 1, "XN:i:0" => 1, "XM:i:0" => 1,
+ "XO:i:0" => 1, "XG:i:0" => 1, "NM:i:0" => 1, "MD:Z:33" => 1,
+ "YM:i:0" => 1, "YP:i:1" => 1, "YT:Z:CP" => 1, "YS:i:0" => 1 },
+ 78 => { "AS:i:0" => 1, "XS:i:0" => 1, "XN:i:0" => 1, "XM:i:0" => 1,
+ "XO:i:0" => 1, "XG:i:0" => 1, "NM:i:0" => 1, "MD:Z:28" => 1,
+ "YM:i:1" => 1, "YP:i:1" => 1, "YT:Z:CP" => 1, "YS:i:0" => 1 },
+ 249 => { "AS:i:0" => 1, "XS:i:0" => 1, "XN:i:0" => 1, "XM:i:0" => 1,
+ "XO:i:0" => 1, "XG:i:0" => 1, "NM:i:0" => 1, "MD:Z:28" => 1,
+ "YM:i:1" => 1, "YP:i:1" => 1, "YT:Z:CP" => 1, "YS:i:0" => 1 },
+ 315 => { "AS:i:0" => 1, "XS:i:0" => 1, "XN:i:0" => 1, "XM:i:0" => 1,
+ "XO:i:0" => 1, "XG:i:0" => 1, "NM:i:0" => 1, "MD:Z:28" => 1,
+ "YM:i:1" => 1, "YP:i:1" => 1, "YT:Z:CP" => 1, "YS:i:0" => 1 },
+ 486 => { "AS:i:0" => 1, "XS:i:0" => 1, "XN:i:0" => 1, "XM:i:0" => 1,
+ "XO:i:0" => 1, "XG:i:0" => 1, "NM:i:0" => 1, "MD:Z:28" => 1,
+ "YM:i:1" => 1, "YP:i:1" => 1, "YT:Z:CP" => 1, "YS:i:0" => 1 },
+ 552 => { "AS:i:0" => 1, "XS:i:0" => 1, "XN:i:0" => 1, "XM:i:0" => 1,
+ "XO:i:0" => 1, "XG:i:0" => 1, "NM:i:0" => 1, "MD:Z:28" => 1,
+ "YM:i:1" => 1, "YP:i:1" => 1, "YT:Z:CP" => 1, "YS:i:0" => 1 },
+ 723 => { "AS:i:0" => 1, "XS:i:0" => 1, "XN:i:0" => 1, "XM:i:0" => 1,
+ "XO:i:0" => 1, "XG:i:0" => 1, "NM:i:0" => 1, "MD:Z:28" => 1,
+ "YM:i:1" => 1, "YP:i:1" => 1, "YT:Z:CP" => 1, "YS:i:0" => 1 },
+ 789 => { "AS:i:0" => 1, "XS:i:0" => 1, "XN:i:0" => 1, "XM:i:0" => 1,
+ "XO:i:0" => 1, "XG:i:0" => 1, "NM:i:0" => 1, "MD:Z:28" => 1,
+ "YM:i:1" => 1, "YP:i:1" => 1, "YT:Z:CP" => 1, "YS:i:0" => 1 },
+ 960 => { "AS:i:0" => 1, "XS:i:0" => 1, "XN:i:0" => 1, "XM:i:0" => 1,
+ "XO:i:0" => 1, "XG:i:0" => 1, "NM:i:0" => 1, "MD:Z:28" => 1,
+ "YM:i:1" => 1, "YP:i:1" => 1, "YT:Z:CP" => 1, "YS:i:0" => 1 },
+ } ] },
+
+ { name => "P.k.58.L.b Unpaired -k 5 w/ 8 hits local, but mate #1 has just 1",
+ # 0 1 2 3 0 1 2 0 1 2 0 1 2
+ # 012345678901234567890123456789012 0123456789012345678901234567 0123456789012345678901234567 0123456789012345678901234567 [...]
+ # CAGCGTACGGTATCTAGCTATGGGCATCGATCG ACACACCCCTATAGCTCGGAGCTGACTG ACACACCCCTATAGCTCGGAGCTGACTG ACACACCCCTATAGCTCGGAGCTGACTG [...]
+ ref => [ "AGACGCAGTCACCAGCGTACGGTATCTAGCTATGGGCATCGATCGACGACGTACGAGCGGTATCTACAGCCACTCATCACACACCCCTATAGCTCGGAGCTGACTGGGTTACTGGGGGGGATGCGTATCGACTATCGACAATATGACGCGTCGGTCACCCCATAATATGCAAAAATTATAGCTCACGACGCGTACTAATAGAAAACGCGCTATCAGCCTCCGACGCGGCGGTATCGAAGACGCAGTCACACACACCCCTATAGCTCGGAGCTGACTGGATCGACGACGTACGAGCGGTATCTACAGCCACTCATCACACACCCCTATAGCTCGGAGCTGACTGGGTTACTGGGGGGGATGCGTATCGACTATCGACAATATGACGCGTCGGTCACCCCATAATATGCAAAAATTATAGCTCACGACGCGTACTAATAGAAAACGCGCTATCAGCCTCCGACGCGGCGGTATCGAAG [...]
+ # 01234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345 [...]
+ # 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 [...]
+ # 0 1 2 3 4 [...]
+ # 0 0 0 0 0 [...]
+ mate1s => [ "CAGCGTACGGTATCTAGCTATGGGCATCGATCG" ],
+ mate2s => [ "CAGTCAGCTCCGAGCTATAGGGGTGTGT" ], # rev comped
+ args => "-X 1000 --local",
+ report => "-k 5",
+ pairhits => [{ "12,78" => 1, "12,249" => 1, "12,315" => 1,
+ "12,486" => 1, "12,552" => 1, "12,723" => 1,
+ "12,789" => 1, "12,960" => 1 }],
+ hits_are_superset => [ 1 ],
+ cigar_map => [{
+ 12 => "33M", 78 => "28M",
+ 249 => "28M", 315 => "28M",
+ 486 => "28M", 552 => "28M",
+ 723 => "28M", 789 => "28M",
+ 960 => "28M"
+ }],
+ samoptflags_map => [ {
+ 12 => { "AS:i:66" => 1, "XS:i:0" => 1, "XN:i:0" => 1, "XM:i:0" => 1,
+ "XO:i:0" => 1, "XG:i:0" => 1, "NM:i:0" => 1, "MD:Z:33" => 1,
+ "YM:i:0" => 1, "YP:i:1" => 1, "YT:Z:CP" => 1, "YS:i:56" => 1 },
+ 78 => { "AS:i:56" => 1, "XS:i:56" => 1, "XN:i:0" => 1, "XM:i:0" => 1,
+ "XO:i:0" => 1, "XG:i:0" => 1, "NM:i:0" => 1, "MD:Z:28" => 1,
+ "YM:i:1" => 1, "YP:i:1" => 1, "YT:Z:CP" => 1, "YS:i:66" => 1 },
+ 249 => { "AS:i:56" => 1, "XS:i:56" => 1, "XN:i:0" => 1, "XM:i:0" => 1,
+ "XO:i:0" => 1, "XG:i:0" => 1, "NM:i:0" => 1, "MD:Z:28" => 1,
+ "YM:i:1" => 1, "YP:i:1" => 1, "YT:Z:CP" => 1, "YS:i:66" => 1 },
+ 315 => { "AS:i:56" => 1, "XS:i:56" => 1, "XN:i:0" => 1, "XM:i:0" => 1,
+ "XO:i:0" => 1, "XG:i:0" => 1, "NM:i:0" => 1, "MD:Z:28" => 1,
+ "YM:i:1" => 1, "YP:i:1" => 1, "YT:Z:CP" => 1, "YS:i:66" => 1 },
+ 486 => { "AS:i:56" => 1, "XS:i:56" => 1, "XN:i:0" => 1, "XM:i:0" => 1,
+ "XO:i:0" => 1, "XG:i:0" => 1, "NM:i:0" => 1, "MD:Z:28" => 1,
+ "YM:i:1" => 1, "YP:i:1" => 1, "YT:Z:CP" => 1, "YS:i:66" => 1 },
+ 552 => { "AS:i:56" => 1, "XS:i:56" => 1, "XN:i:0" => 1, "XM:i:0" => 1,
+ "XO:i:0" => 1, "XG:i:0" => 1, "NM:i:0" => 1, "MD:Z:28" => 1,
+ "YM:i:1" => 1, "YP:i:1" => 1, "YT:Z:CP" => 1, "YS:i:66" => 1 },
+ 723 => { "AS:i:56" => 1, "XS:i:56" => 1, "XN:i:0" => 1, "XM:i:0" => 1,
+ "XO:i:0" => 1, "XG:i:0" => 1, "NM:i:0" => 1, "MD:Z:28" => 1,
+ "YM:i:1" => 1, "YP:i:1" => 1, "YT:Z:CP" => 1, "YS:i:66" => 1 },
+ 789 => { "AS:i:56" => 1, "XS:i:56" => 1, "XN:i:0" => 1, "XM:i:0" => 1,
+ "XO:i:0" => 1, "XG:i:0" => 1, "NM:i:0" => 1, "MD:Z:28" => 1,
+ "YM:i:1" => 1, "YP:i:1" => 1, "YT:Z:CP" => 1, "YS:i:66" => 1 },
+ 960 => { "AS:i:56" => 1, "XS:i:56" => 1, "XN:i:0" => 1, "XM:i:0" => 1,
+ "XO:i:0" => 1, "XG:i:0" => 1, "NM:i:0" => 1, "MD:Z:28" => 1,
+ "YM:i:1" => 1, "YP:i:1" => 1, "YT:Z:CP" => 1, "YS:i:66" => 1 },
+ } ] },
+
+ { name => "P.M.22.G. Paired -M 2 w/ 2 paired hit, 2 unpaired hits each, global",
+ # 0 1 2 3 0 1 2 0 1 2 3 0 1 2
+ # 012345678901234567890123456789012 0123456789012345678901234567 012345678901234567890123456789012 0123456789012345678901234567
+ # CAGCGTACGGTATCTAGCTATGGGCATCGATCG ACACACCCCTATAGCTCGGAGCTGACTG CAGCGTACGGTATCTAGCTATGGGCATCGATCG ACACACCCCTATAGCTCGGAGCTGACTG
+ ref => [ "AGACGCAGTCACCAGCGTACGGTATCTAGCTATGGGCATCGATCGACGACGTACGAGCGGTATCTACAGCCACTCATCACACACCCCTATAGCTCGGAGCTGACTGGGTTACTGGGGGGGATGCGTATCGACTATCGACAATATGACGCGTCGGTCACCCCATAATATGCAAAAATTATAGCTCACGACGCGTACTAATAGAAAACGCGCTATCAGCCTCCGACGCGGCGGTATCGAAGACGCAGTCACCAGCGTACGGTATCTAGCTATGGGCATCGATCGACGACGTACGAGCGGTATCTACAGCCACTCATCACACACCCCTATAGCTCGGAGCTGACTGGGTTACTGGGGGGGTATCGA" ],
+ # 012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012
+ # 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6
+ # 0 1 2 3
+ mate1s => [ "CAGCGTACGGTATCTAGCTATGGGCATCGATCG" ],
+ mate2s => [ "CAGTCAGCTCCGAGCTATAGGGGTGTGT" ], # rev comped
+ mate1fw => 1, mate2fw => 0,
+ args => "-X 150",
+ report => "-M 2",
+ pairhits => [ { "12,78" => 1, "249,315" => 1 } ],
+ cigar_map => [{
+ 12 => "33M", 249 => "33M",
+ 78 => "28M", 315 => "28M"
+ }],
+ hits_are_superset => [ 1 ],
+ samoptflags_map => [{
+ 12 => { "AS:i:0" => 1, "XS:i:0" => 1, "MD:Z:33" => 1,
+ "YM:i:0" => 1, "YP:i:0" => 1, "YT:Z:CP" => 1, "YS:i:0" => 1 },
+ 78 => { "AS:i:0" => 1, "XS:i:0" => 1, "MD:Z:28" => 1,
+ "YM:i:0" => 1, "YP:i:0" => 1, "YT:Z:CP" => 1, "YS:i:0" => 1 },
+ 249 => { "AS:i:0" => 1, "XS:i:0" => 1, "MD:Z:33" => 1,
+ "YM:i:0" => 1, "YP:i:0" => 1, "YT:Z:CP" => 1, "YS:i:0" => 1 },
+ 315 => { "AS:i:0" => 1, "XS:i:0" => 1, "MD:Z:28" => 1,
+ "YM:i:0" => 1, "YP:i:0" => 1, "YT:Z:CP" => 1, "YS:i:0" => 1 },
+ }]
+ },
+
+ { name => "P.M.22.L. Paired -M 2 w/ 2 paired hit, 2 unpaired hits each, local",
+ # 0 1 2 3 0 1 2 0 1 2 3 0 1 2
+ # 012345678901234567890123456789012 0123456789012345678901234567 012345678901234567890123456789012 0123456789012345678901234567
+ # CAGCGTACGGTATCTAGCTATGGGCATCGATCG ACACACCCCTATAGCTCGGAGCTGACTG CAGCGTACGGTATCTAGCTATGGGCATCGATCG ACACACCCCTATAGCTCGGAGCTGACTG
+ ref => [ "AGACGCAGTCACCAGCGTACGGTATCTAGCTATGGGCATCGATCGACGACGTACGAGCGGTATCTACAGCCACTCATCACACACCCCTATAGCTCGGAGCTGACTGGGTTACTGGGGGGGATGCGTATCGACTATCGACAATATGACGCGTCGGTCACCCCATAATATGCAAAAATTATAGCTCACGACGCGTACTAATAGAAAACGCGCTATCAGCCTCCGACGCGGCGGTATCGAAGACGCAGTCACCAGCGTACGGTATCTAGCTATGGGCATCGATCGACGACGTACGAGCGGTATCTACAGCCACTCATCACACACCCCTATAGCTCGGAGCTGACTGGGTTACTGGGGGGGTATCGA" ],
+ # 012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012
+ # 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6
+ # 0 1 2 3
+ mate1s => [ "CAGCGTACGGTATCTAGCTATGGGCATCGATCG" ],
+ mate2s => [ "CAGTCAGCTCCGAGCTATAGGGGTGTGT" ], # rev comped
+ mate1fw => 1, mate2fw => 0,
+ args => "--local -X 150",
+ report => "-M 2",
+ pairhits => [ { "12,78" => 1, "249,315" => 1 } ],
+ cigar_map => [{
+ 12 => "33M", 249 => "33M",
+ 78 => "28M", 315 => "28M"
+ }],
+ hits_are_superset => [ 1 ],
+ samoptflags_map => [{
+ 12 => { "AS:i:66" => 1, "XS:i:66" => 1, "MD:Z:33" => 1,
+ "YM:i:0" => 1, "YP:i:0" => 1, "YT:Z:CP" => 1, "YS:i:56" => 1 },
+ 78 => { "AS:i:56" => 1, "XS:i:56" => 1, "MD:Z:28" => 1,
+ "YM:i:0" => 1, "YP:i:0" => 1, "YT:Z:CP" => 1, "YS:i:66" => 1 },
+ 249 => { "AS:i:66" => 1, "XS:i:66" => 1, "MD:Z:33" => 1,
+ "YM:i:0" => 1, "YP:i:0" => 1, "YT:Z:CP" => 1, "YS:i:56" => 1 },
+ 315 => { "AS:i:56" => 1, "XS:i:56" => 1, "MD:Z:28" => 1,
+ "YM:i:0" => 1, "YP:i:0" => 1, "YT:Z:CP" => 1, "YS:i:66" => 1 },
+ }]
+ },
+
+ { name => "P.k.2.G. Paired -k 1 w/ 2 paired hit, 2 unpaired hits each, global",
+ # 0 1 2 3 0 1 2 0 1 2 3 0 1 2
+ # 012345678901234567890123456789012 0123456789012345678901234567 012345678901234567890123456789012 0123456789012345678901234567
+ # CAGCGTACGGTATCTAGCTATGGGCATCGATCG ACACACCCCTATAGCTCGGAGCTGACTG CAGCGTACGGTATCTAGCTATGGGCATCGATCG ACACACCCCTATAGCTCGGAGCTGACTG
+ ref => [ "AGACGCAGTCACCAGCGTACGGTATCTAGCTATGGGCATCGATCGACGACGTACGAGCGGTATCTACAGCCACTCATCACACACCCCTATAGCTCGGAGCTGACTGGGTTACTGGGGGGGATGCGTATCGACTATCGACAATATGACGCGTCGGTCACCCCATAATATGCAAAAATTATAGCTCACGACGCGTACTAATAGAAAACGCGCTATCAGCCTCCGACGCGGCGGTATCGAAGACGCAGTCACCAGCGTACGGTATCTAGCTATGGGCATCGATCGACGACGTACGAGCGGTATCTACAGCCACTCATCACACACCCCTATAGCTCGGAGCTGACTGGGTTACTGGGGGGGTATCGA" ],
+ # 012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012
+ # 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6
+ # 0 1 2 3
+ mate1s => [ "CAGCGTACGGTATCTAGCTATGGGCATCGATCG" ],
+ mate2s => [ "CAGTCAGCTCCGAGCTATAGGGGTGTGT" ], # rev comped
+ mate1fw => 1, mate2fw => 0,
+ args => "-X 150",
+ report => "-k 1",
+ pairhits => [ { "12,78" => 1, "249,315" => 1 } ],
+ hits_are_superset => [ 1 ],
+ cigar_map => [{
+ 12 => "33M", 249 => "33M",
+ 78 => "28M", 315 => "28M"
+ }],
+ samoptflags_map => [{
+ 12 => { "AS:i:0" => 1, "XS:i:0" => 1, "MD:Z:33" => 1,
+ "YT:Z:CP" => 1, "YS:i:0" => 1 },
+ 78 => { "AS:i:0" => 1, "XS:i:0" => 1, "MD:Z:28" => 1,
+ "YT:Z:CP" => 1, "YS:i:0" => 1 },
+ 249 => { "AS:i:0" => 1, "XS:i:0" => 1, "MD:Z:33" => 1,
+ "YT:Z:CP" => 1, "YS:i:0" => 1 },
+ 315 => { "AS:i:0" => 1, "XS:i:0" => 1, "MD:Z:28" => 1,
+ "YT:Z:CP" => 1, "YS:i:0" => 1 },
+ }]
+ },
+
+ { name => "P.k.2.L. Paired -k 1 w/ 2 paired hit, 2 unpaired hits each, local",
+ # 0 1 2 3 0 1 2 0 1 2 3 0 1 2
+ # 012345678901234567890123456789012 0123456789012345678901234567 012345678901234567890123456789012 0123456789012345678901234567
+ # CAGCGTACGGTATCTAGCTATGGGCATCGATCG ACACACCCCTATAGCTCGGAGCTGACTG CAGCGTACGGTATCTAGCTATGGGCATCGATCG ACACACCCCTATAGCTCGGAGCTGACTG
+ ref => [ "AGACGCAGTCACCAGCGTACGGTATCTAGCTATGGGCATCGATCGACGACGTACGAGCGGTATCTACAGCCACTCATCACACACCCCTATAGCTCGGAGCTGACTGGGTTACTGGGGGGGATGCGTATCGACTATCGACAATATGACGCGTCGGTCACCCCATAATATGCAAAAATTATAGCTCACGACGCGTACTAATAGAAAACGCGCTATCAGCCTCCGACGCGGCGGTATCGAAGACGCAGTCACCAGCGTACGGTATCTAGCTATGGGCATCGATCGACGACGTACGAGCGGTATCTACAGCCACTCATCACACACCCCTATAGCTCGGAGCTGACTGGGTTACTGGGGGGGTATCGA" ],
+ # 012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012
+ # 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6
+ # 0 1 2 3
+ mate1s => [ "CAGCGTACGGTATCTAGCTATGGGCATCGATCG" ],
+ mate2s => [ "CAGTCAGCTCCGAGCTATAGGGGTGTGT" ], # rev comped
+ mate1fw => 1, mate2fw => 0,
+ args => "--local -X 150",
+ report => "-k 1",
+ pairhits => [ { "12,78" => 1, "249,315" => 1 } ],
+ hits_are_superset => [ 1 ],
+ cigar_map => [{
+ 12 => "33M", 249 => "33M",
+ 78 => "28M", 315 => "28M"
+ }],
+ samoptflags_map => [{
+ 12 => { "AS:i:66" => 1, "XS:i:0" => 1, "MD:Z:33" => 1,
+ "YT:Z:CP" => 1, "YS:i:56" => 1 },
+ 78 => { "AS:i:56" => 1, "XS:i:0" => 1, "MD:Z:28" => 1,
+ "YT:Z:CP" => 1, "YS:i:66" => 1 },
+ 249 => { "AS:i:66" => 1, "XS:i:0" => 1, "MD:Z:33" => 1,
+ "YT:Z:CP" => 1, "YS:i:56" => 1 },
+ 315 => { "AS:i:56" => 1, "XS:i:0" => 1, "MD:Z:28" => 1,
+ "YT:Z:CP" => 1, "YS:i:66" => 1 },
+ }]
+ },
+
+ { name => "P.M.2.G. Paired -M 1 w/ 2 paired hit, 2 unpaired hits each, global",
+ # 0 1 2 3 0 1 2 0 1 2 3 0 1 2
+ # 012345678901234567890123456789012 0123456789012345678901234567 012345678901234567890123456789012 0123456789012345678901234567
+ # CAGCGTACGGTATCTAGCTATGGGCATCGATCG ACACACCCCTATAGCTCGGAGCTGACTG CAGCGTACGGTATCTAGCTATGGGCATCGATCG ACACACCCCTATAGCTCGGAGCTGACTG
+ ref => [ "AGACGCAGTCACCAGCGTACGGTATCTAGCTATGGGCATCGATCGACGACGTACGAGCGGTATCTACAGCCACTCATCACACACCCCTATAGCTCGGAGCTGACTGGGTTACTGGGGGGGATGCGTATCGACTATCGACAATATGACGCGTCGGTCACCCCATAATATGCAAAAATTATAGCTCACGACGCGTACTAATAGAAAACGCGCTATCAGCCTCCGACGCGGCGGTATCGAAGACGCAGTCACCAGCGTACGGTATCTAGCTATGGGCATCGATCGACGACGTACGAGCGGTATCTACAGCCACTCATCACACACCCCTATAGCTCGGAGCTGACTGGGTTACTGGGGGGGTATCGA" ],
+ # 012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012
+ # 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6
+ # 0 1 2 3
+ mate1s => [ "CAGCGTACGGTATCTAGCTATGGGCATCGATCG" ],
+ mate2s => [ "CAGTCAGCTCCGAGCTATAGGGGTGTGT" ], # rev comped
+ mate1fw => 1, mate2fw => 0,
+ report => "-M 1 -X 150",
+ pairhits => [ { "12,78" => 1, "249,315" => 1 } ],
+ hits_are_superset => [ 1 ],
+ cigar_map => [{
+ 12 => "33M", 249 => "33M",
+ 78 => "28M", 315 => "28M"
+ }],
+ samoptflags_map => [{
+ 12 => { "AS:i:0" => 1, "XS:i:0" => 1, "MD:Z:33" => 1,
+ "YM:i:1" => 1, "YP:i:1" => 1, "YT:Z:CP" => 1, "YS:i:0" => 1 },
+ 78 => { "AS:i:0" => 1, "XS:i:0" => 1, "MD:Z:28" => 1,
+ "YM:i:1" => 1, "YP:i:1" => 1, "YT:Z:CP" => 1, "YS:i:0" => 1 },
+ 249 => { "AS:i:0" => 1, "XS:i:0" => 1, "MD:Z:33" => 1,
+ "YM:i:1" => 1, "YP:i:1" => 1, "YT:Z:CP" => 1, "YS:i:0" => 1 },
+ 315 => { "AS:i:0" => 1, "XS:i:0" => 1, "MD:Z:28" => 1,
+ "YM:i:1" => 1, "YP:i:1" => 1, "YT:Z:CP" => 1, "YS:i:0" => 1 },
+ }]
+ },
+
+ { name => "P.M.2.L. Paired -M 1 w/ 2 paired hit, 2 unpaired hits each, local",
+ # 0 1 2 3 0 1 2 0 1 2 3 0 1 2
+ # 012345678901234567890123456789012 0123456789012345678901234567 012345678901234567890123456789012 0123456789012345678901234567
+ # CAGCGTACGGTATCTAGCTATGGGCATCGATCG ACACACCCCTATAGCTCGGAGCTGACTG CAGCGTACGGTATCTAGCTATGGGCATCGATCG ACACACCCCTATAGCTCGGAGCTGACTG
+ ref => [ "AGACGCAGTCACCAGCGTACGGTATCTAGCTATGGGCATCGATCGACGACGTACGAGCGGTATCTACAGCCACTCATCACACACCCCTATAGCTCGGAGCTGACTGGGTTACTGGGGGGGATGCGTATCGACTATCGACAATATGACGCGTCGGTCACCCCATAATATGCAAAAATTATAGCTCACGACGCGTACTAATAGAAAACGCGCTATCAGCCTCCGACGCGGCGGTATCGAAGACGCAGTCACCAGCGTACGGTATCTAGCTATGGGCATCGATCGACGACGTACGAGCGGTATCTACAGCCACTCATCACACACCCCTATAGCTCGGAGCTGACTGGGTTACTGGGGGGGTATCGA" ],
+ # 012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012
+ # 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6
+ # 0 1 2 3
+ mate1s => [ "CAGCGTACGGTATCTAGCTATGGGCATCGATCG" ],
+ mate2s => [ "CAGTCAGCTCCGAGCTATAGGGGTGTGT" ], # rev comped
+ mate1fw => 1, mate2fw => 0,
+ report => "-M 1 --local -X 150",
+ pairhits => [ { "12,78" => 1, "249,315" => 1 } ],
+ hits_are_superset => [ 1 ],
+ cigar_map => [{
+ 12 => "33M", 249 => "33M",
+ 78 => "28M", 315 => "28M"
+ }],
+ samoptflags_map => [{
+ 12 => { "AS:i:66" => 1, "XS:i:66" => 1, "MD:Z:33" => 1,
+ "YM:i:1" => 1, "YP:i:1" => 1, "YT:Z:CP" => 1, "YS:i:56" => 1 },
+ 78 => { "AS:i:56" => 1, "XS:i:56" => 1, "MD:Z:28" => 1,
+ "YM:i:1" => 1, "YP:i:1" => 1, "YT:Z:CP" => 1, "YS:i:66" => 1 },
+ 249 => { "AS:i:66" => 1, "XS:i:66" => 1, "MD:Z:33" => 1,
+ "YM:i:1" => 1, "YP:i:1" => 1, "YT:Z:CP" => 1, "YS:i:56" => 1 },
+ 315 => { "AS:i:56" => 1, "XS:i:56" => 1, "MD:Z:28" => 1,
+ "YM:i:1" => 1, "YP:i:1" => 1, "YT:Z:CP" => 1, "YS:i:66" => 1 },
+ }]
+ },
+
+ { name => "P.k.1.G. Paired -k w/ 1 paired hit, 1 unpaired hit each, global",
+ # 0 1 2 3 0 1 2
+ # 012345678901234567890123456789012 0123456789012345678901234567
+ # CAGCGTACGGTATCTAGCTATGGGCATCGATCG ACACACCCCTATAGCTCGGAGCTGACTG
+ ref => [ "AGACGCAGTCACCAGCGTACGGTATCTAGCTATGGGCATCGATCGACGACGTACGAGCGGTATCTACAGCCACTCATCACACACCCCTATAGCTCGGAGCTGACTGGGTTACTGGGGGGGTATCGA" ],
+ # 012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345
+ # 0 1 2 3 4 5 6 7 8 9 0 1 2
+ mate1s => [ "CAGCGTACGGTATCTAGCTATGGGCATCGATCG" ],
+ mate2s => [ "CAGTCAGCTCCGAGCTATAGGGGTGTGT" ], # rev comped
+ mate1fw => 1, mate2fw => 0,
+ report => "-k 1 -X 150",
+ pairhits => [ { "12,78" => 1 } ],
+ cigar_map => [{
+ 12 => "33M",
+ 78 => "28M"
+ }],
+ samoptflags_map => [{
+ 12 => { "AS:i:0" => 1, "XS:i:0" => 1, "MD:Z:33" => 1,
+ "YM:i:0" => 1, "YP:i:0" => 1, "YT:Z:CP" => 1, "YS:i:0" => 1 },
+ 78 => { "AS:i:0" => 1, "XS:i:0" => 1, "MD:Z:28" => 1,
+ "YM:i:0" => 1, "YP:i:0" => 1, "YT:Z:CP" => 1, "YS:i:0" => 1 },
+ }]
+ },
+
+ { name => "P.k.1.L. Paired -k 1 w/ 1 paired hit, 1 unpaired hit each, local",
+ # 0 1 2 3 0 1 2
+ # 012345678901234567890123456789012 0123456789012345678901234567
+ # CAGCGTACGGTATCTAGCTATGGGCATCGATCG ACACACCCCTATAGCTCGGAGCTGACTG
+ ref => [ "AGACGCAGTCACCAGCGTACGGTATCTAGCTATGGGCATCGATCGACGACGTACGAGCGGTATCTACAGCCACTCATCACACACCCCTATAGCTCGGAGCTGACTGGGTTACTGGGGGGGTATCGA" ],
+ # 012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345
+ # 0 1 2 3 4 5 6 7 8 9 0 1 2
+ mate1s => [ "CAGCGTACGGTATCTAGCTATGGGCATCGATCG" ],
+ mate2s => [ "CAGTCAGCTCCGAGCTATAGGGGTGTGT" ], # rev comped
+ mate1fw => 1, mate2fw => 0,
+ args => "--local -X 150",
+ report => "-k 1",
+ pairhits => [ { "12,78" => 1 } ],
+ cigar_map => [{
+ 12 => "33M",
+ 78 => "28M"
+ }],
+ samoptflags_map => [{
+ 12 => { "AS:i:66" => 1, "XS:i:0" => 1, "MD:Z:33" => 1,
+ "YM:i:0" => 1, "YP:i:0" => 1, "YT:Z:CP" => 1, "YS:i:56" => 1 },
+ 78 => { "AS:i:56" => 1, "XS:i:0" => 1, "MD:Z:28" => 1,
+ "YM:i:0" => 1, "YP:i:0" => 1, "YT:Z:CP" => 1, "YS:i:66" => 1 },
+ }]
+ },
+
+ { name => "P.M.1.G. Paired -M w/ 1 paired hit, 1 unpaired hit each, global",
+ # 0 1 2 3 0 1 2
+ # 012345678901234567890123456789012 0123456789012345678901234567
+ # CAGCGTACGGTATCTAGCTATGGGCATCGATCG ACACACCCCTATAGCTCGGAGCTGACTG
+ ref => [ "AGACGCAGTCACCAGCGTACGGTATCTAGCTATGGGCATCGATCGACGACGTACGAGCGGTATCTACAGCCACTCATCACACACCCCTATAGCTCGGAGCTGACTGGGTTACTGGGGGGGTATCGA" ],
+ # 012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345
+ # 0 1 2 3 4 5 6 7 8 9 0 1 2
+ mate1s => [ "CAGCGTACGGTATCTAGCTATGGGCATCGATCG" ],
+ mate2s => [ "CAGTCAGCTCCGAGCTATAGGGGTGTGT" ], # rev comped
+ mate1fw => 1, mate2fw => 0,
+ args => "-X 150",
+ report => "-M 1",
+ pairhits => [ { "12,78" => 1 } ],
+ cigar_map => [{
+ 12 => "33M",
+ 78 => "28M"
+ }],
+ samoptflags_map => [{
+ 12 => { "AS:i:0" => 1, "XS:i:0" => 1, "MD:Z:33" => 1,
+ "YM:i:0" => 1, "YP:i:0" => 1, "YT:Z:CP" => 1, "YS:i:0" => 1 },
+ 78 => { "AS:i:0" => 1, "XS:i:0" => 1, "MD:Z:28" => 1,
+ "YM:i:0" => 1, "YP:i:0" => 1, "YT:Z:CP" => 1, "YS:i:0" => 1 },
+ }]
+ },
+
+ { name => "P.M.1.L. Paired -M w/ 1 paired hit, 1 unpaired hit each, local",
+ # 0 1 2 3 0 1 2
+ # 012345678901234567890123456789012 0123456789012345678901234567
+ # CAGCGTACGGTATCTAGCTATGGGCATCGATCG ACACACCCCTATAGCTCGGAGCTGACTG
+ ref => [ "AGACGCAGTCACCAGCGTACGGTATCTAGCTATGGGCATCGATCGACGACGTACGAGCGGTATCTACAGCCACTCATCACACACCCCTATAGCTCGGAGCTGACTGGGTTACTGGGGGGGTATCGA" ],
+ # 012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345
+ # 0 1 2 3 4 5 6 7 8 9 0 1 2
+ mate1s => [ "CAGCGTACGGTATCTAGCTATGGGCATCGATCG" ],
+ mate2s => [ "CAGTCAGCTCCGAGCTATAGGGGTGTGT" ], # rev comped
+ mate1fw => 1, mate2fw => 0,
+ args => "-X 150 --local",
+ report => "-M 1",
+ pairhits => [ { "12,78" => 1 } ],
+ cigar_map => [{
+ 12 => "33M",
+ 78 => "28M"
+ }],
+ samoptflags_map => [{
+ 12 => { "AS:i:66" => 1, "XS:i:0" => 1, "MD:Z:33" => 1,
+ "YM:i:0" => 1, "YP:i:0" => 1, "YT:Z:CP" => 1, "YS:i:56" => 1 },
+ 78 => { "AS:i:56" => 1, "XS:i:0" => 1, "MD:Z:28" => 1,
+ "YM:i:0" => 1, "YP:i:0" => 1, "YT:Z:CP" => 1, "YS:i:66" => 1 },
+ }]
+ },
+
+ { name => "P.M.58.G. Unpaired -M 5 w/ 8 hits global",
+ # 0 1 2 3 0 1 2 0 1 2 3 0 1 2
+ # 012345678901234567890123456789012 0123456789012345678901234567 012345678901234567890123456789012 0123456789012345678901234567
+ # CAGCGTACGGTATCTAGCTATGGGCATCGATCG ACACACCCCTATAGCTCGGAGCTGACTG CAGCGTACGGTATCTAGCTATGGGCATCGATCG ACACACCCCTATAGCTCGGAGCTGACTG [...]
+ ref => [ "AGACGCAGTCACCAGCGTACGGTATCTAGCTATGGGCATCGATCGACGACGTACGAGCGGTATCTACAGCCACTCATCACACACCCCTATAGCTCGGAGCTGACTGGGTTACTGGGGGGGATGCGTATCGACTATCGACAATATGACGCGTCGGTCACCCCATAATATGCAAAAATTATAGCTCACGACGCGTACTAATAGAAAACGCGCTATCAGCCTCCGACGCGGCGGTATCGAAGACGCAGTCACCAGCGTACGGTATCTAGCTATGGGCATCGATCGACGACGTACGAGCGGTATCTACAGCCACTCATCACACACCCCTATAGCTCGGAGCTGACTGGGTTACTGGGGGGGATGCGTATCGACTATCGACAATATGACGCGTCGGTCACCCCATAATATGCAAAAATTATAGCTCACGACGCGTACTAATAGAAAACGCGCTATCAGCCTCCGACGCGGCGGTATCGAAG [...]
+ # 01234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345 [...]
+ # 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 [...]
+ # 0 1 2 3 4 [...]
+ # 0 0 0 0 0 [...]
+ mate1s => [ "CAGCGTACGGTATCTAGCTATGGGCATCGATCG" ],
+ mate2s => [ "CAGTCAGCTCCGAGCTATAGGGGTGTGT" ], # rev comped
+ args => "-X 150",
+ report => "-M 5",
+ pairhits => [ { "12,78" => 1, "249,315" => 1, "486,552" => 1,
+ "723,789" => 1, "960,1026" => 1, "1197,1263" => 1,
+ "1434,1500" => 1, "1671,1737" => 1, "1908,1974" => 1,
+ "2145,2211" => 1, "2382,2448" => 1 } ],
+ hits_are_superset => [ 1 ],
+ cigar_map => [{
+ 12 => "33M", 78 => "28M",
+ 249 => "33M", 315 => "28M",
+ 486 => "33M", 552 => "28M",
+ 723 => "33M", 789 => "28M",
+ 960 => "33M", 1026 => "28M",
+ 1197 => "33M", 1263 => "28M",
+ 1434 => "33M", 1500 => "28M",
+ 1671 => "33M", 1737 => "28M",
+ 1908 => "33M", 1974 => "28M",
+ 2145 => "33M", 2211 => "28M",
+ 2382 => "33M", 2448 => "28M",
+ }],
+ samoptflags_map => [ {
+ 12 => { "AS:i:0" => 1, "XS:i:0" => 1, "XN:i:0" => 1, "XM:i:0" => 1,
+ "XO:i:0" => 1, "XG:i:0" => 1, "NM:i:0" => 1, "MD:Z:33" => 1,
+ "YM:i:1" => 1, "YP:i:1" => 1, "YT:Z:CP" => 1, "YS:i:0" => 1 },
+ 78 => { "AS:i:0" => 1, "XS:i:0" => 1, "XN:i:0" => 1, "XM:i:0" => 1,
+ "XO:i:0" => 1, "XG:i:0" => 1, "NM:i:0" => 1, "MD:Z:28" => 1,
+ "YM:i:1" => 1, "YP:i:1" => 1, "YT:Z:CP" => 1, "YS:i:0" => 1 },
+ 249 => { "AS:i:0" => 1, "XS:i:0" => 1, "XN:i:0" => 1, "XM:i:0" => 1,
+ "XO:i:0" => 1, "XG:i:0" => 1, "NM:i:0" => 1, "MD:Z:33" => 1,
+ "YM:i:1" => 1, "YP:i:1" => 1, "YT:Z:CP" => 1, "YS:i:0" => 1 },
+ 315 => { "AS:i:0" => 1, "XS:i:0" => 1, "XN:i:0" => 1, "XM:i:0" => 1,
+ "XO:i:0" => 1, "XG:i:0" => 1, "NM:i:0" => 1, "MD:Z:28" => 1,
+ "YM:i:1" => 1, "YP:i:1" => 1, "YT:Z:CP" => 1, "YS:i:0" => 1 },
+ 486 => { "AS:i:0" => 1, "XS:i:0" => 1, "XN:i:0" => 1, "XM:i:0" => 1,
+ "XO:i:0" => 1, "XG:i:0" => 1, "NM:i:0" => 1, "MD:Z:33" => 1,
+ "YM:i:1" => 1, "YP:i:1" => 1, "YT:Z:CP" => 1, "YS:i:0" => 1 },
+ 552 => { "AS:i:0" => 1, "XS:i:0" => 1, "XN:i:0" => 1, "XM:i:0" => 1,
+ "XO:i:0" => 1, "XG:i:0" => 1, "NM:i:0" => 1, "MD:Z:28" => 1,
+ "YM:i:1" => 1, "YP:i:1" => 1, "YT:Z:CP" => 1, "YS:i:0" => 1 },
+ 723 => { "AS:i:0" => 1, "XS:i:0" => 1, "XN:i:0" => 1, "XM:i:0" => 1,
+ "XO:i:0" => 1, "XG:i:0" => 1, "NM:i:0" => 1, "MD:Z:33" => 1,
+ "YM:i:1" => 1, "YP:i:1" => 1, "YT:Z:CP" => 1, "YS:i:0" => 1 },
+ 789 => { "AS:i:0" => 1, "XS:i:0" => 1, "XN:i:0" => 1, "XM:i:0" => 1,
+ "XO:i:0" => 1, "XG:i:0" => 1, "NM:i:0" => 1, "MD:Z:28" => 1,
+ "YM:i:1" => 1, "YP:i:1" => 1, "YT:Z:CP" => 1, "YS:i:0" => 1 },
+ 960 => { "AS:i:0" => 1, "XS:i:0" => 1, "XN:i:0" => 1, "XM:i:0" => 1,
+ "XO:i:0" => 1, "XG:i:0" => 1, "NM:i:0" => 1, "MD:Z:33" => 1,
+ "YM:i:1" => 1, "YP:i:1" => 1, "YT:Z:CP" => 1, "YS:i:0" => 1 },
+ 1026 => { "AS:i:0" => 1, "XS:i:0" => 1, "XN:i:0" => 1, "XM:i:0" => 1,
+ "XO:i:0" => 1, "XG:i:0" => 1, "NM:i:0" => 1, "MD:Z:28" => 1,
+ "YM:i:1" => 1, "YP:i:1" => 1, "YT:Z:CP" => 1, "YS:i:0" => 1 },
+ 1197 => { "AS:i:0" => 1, "XS:i:0" => 1, "XN:i:0" => 1, "XM:i:0" => 1,
+ "XO:i:0" => 1, "XG:i:0" => 1, "NM:i:0" => 1, "MD:Z:33" => 1,
+ "YM:i:1" => 1, "YP:i:1" => 1, "YT:Z:CP" => 1, "YS:i:0" => 1 },
+ 1263 => { "AS:i:0" => 1, "XS:i:0" => 1, "XN:i:0" => 1, "XM:i:0" => 1,
+ "XO:i:0" => 1, "XG:i:0" => 1, "NM:i:0" => 1, "MD:Z:28" => 1,
+ "YM:i:1" => 1, "YP:i:1" => 1, "YT:Z:CP" => 1, "YS:i:0" => 1 },
+ 1434 => { "AS:i:0" => 1, "XS:i:0" => 1, "XN:i:0" => 1, "XM:i:0" => 1,
+ "XO:i:0" => 1, "XG:i:0" => 1, "NM:i:0" => 1, "MD:Z:33" => 1,
+ "YM:i:1" => 1, "YP:i:1" => 1, "YT:Z:CP" => 1, "YS:i:0" => 1 },
+ 1500 => { "AS:i:0" => 1, "XS:i:0" => 1, "XN:i:0" => 1, "XM:i:0" => 1,
+ "XO:i:0" => 1, "XG:i:0" => 1, "NM:i:0" => 1, "MD:Z:28" => 1,
+ "YM:i:1" => 1, "YP:i:1" => 1, "YT:Z:CP" => 1, "YS:i:0" => 1 },
+ 1671 => { "AS:i:0" => 1, "XS:i:0" => 1, "XN:i:0" => 1, "XM:i:0" => 1,
+ "XO:i:0" => 1, "XG:i:0" => 1, "NM:i:0" => 1, "MD:Z:33" => 1,
+ "YM:i:1" => 1, "YP:i:1" => 1, "YT:Z:CP" => 1, "YS:i:0" => 1 },
+ 1737 => { "AS:i:0" => 1, "XS:i:0" => 1, "XN:i:0" => 1, "XM:i:0" => 1,
+ "XO:i:0" => 1, "XG:i:0" => 1, "NM:i:0" => 1, "MD:Z:28" => 1,
+ "YM:i:1" => 1, "YP:i:1" => 1, "YT:Z:CP" => 1, "YS:i:0" => 1 },
+ 1908 => { "AS:i:0" => 1, "XS:i:0" => 1, "XN:i:0" => 1, "XM:i:0" => 1,
+ "XO:i:0" => 1, "XG:i:0" => 1, "NM:i:0" => 1, "MD:Z:33" => 1,
+ "YM:i:1" => 1, "YP:i:1" => 1, "YT:Z:CP" => 1, "YS:i:0" => 1 },
+ 1974 => { "AS:i:0" => 1, "XS:i:0" => 1, "XN:i:0" => 1, "XM:i:0" => 1,
+ "XO:i:0" => 1, "XG:i:0" => 1, "NM:i:0" => 1, "MD:Z:28" => 1,
+ "YM:i:1" => 1, "YP:i:1" => 1, "YT:Z:CP" => 1, "YS:i:0" => 1 },
+ 2145 => { "AS:i:0" => 1, "XS:i:0" => 1, "XN:i:0" => 1, "XM:i:0" => 1,
+ "XO:i:0" => 1, "XG:i:0" => 1, "NM:i:0" => 1, "MD:Z:33" => 1,
+ "YM:i:1" => 1, "YP:i:1" => 1, "YT:Z:CP" => 1, "YS:i:0" => 1 },
+ 2211 => { "AS:i:0" => 1, "XS:i:0" => 1, "XN:i:0" => 1, "XM:i:0" => 1,
+ "XO:i:0" => 1, "XG:i:0" => 1, "NM:i:0" => 1, "MD:Z:28" => 1,
+ "YM:i:1" => 1, "YP:i:1" => 1, "YT:Z:CP" => 1, "YS:i:0" => 1 },
+ 2382 => { "AS:i:0" => 1, "XS:i:0" => 1, "XN:i:0" => 1, "XM:i:0" => 1,
+ "XO:i:0" => 1, "XG:i:0" => 1, "NM:i:0" => 1, "MD:Z:33" => 1,
+ "YM:i:1" => 1, "YP:i:1" => 1, "YT:Z:CP" => 1, "YS:i:0" => 1 },
+ 2448 => { "AS:i:0" => 1, "XS:i:0" => 1, "XN:i:0" => 1, "XM:i:0" => 1,
+ "XO:i:0" => 1, "XG:i:0" => 1, "NM:i:0" => 1, "MD:Z:28" => 1,
+ "YM:i:1" => 1, "YP:i:1" => 1, "YT:Z:CP" => 1, "YS:i:0" => 1 },
+ } ] },
+
+ { name => "P.M.58.L. Unpaired -M 5 w/ 8 hits local",
+ # 0 1 2 3 0 1 2 0 1 2 3 0 1 2
+ # 012345678901234567890123456789012 0123456789012345678901234567 012345678901234567890123456789012 0123456789012345678901234567
+ # CAGCGTACGGTATCTAGCTATGGGCATCGATCG ACACACCCCTATAGCTCGGAGCTGACTG CAGCGTACGGTATCTAGCTATGGGCATCGATCG ACACACCCCTATAGCTCGGAGCTGACTG [...]
+ ref => [ "AGACGCAGTCACCAGCGTACGGTATCTAGCTATGGGCATCGATCGACGACGTACGAGCGGTATCTACAGCCACTCATCACACACCCCTATAGCTCGGAGCTGACTGGGTTACTGGGGGGGATGCGTATCGACTATCGACAATATGACGCGTCGGTCACCCCATAATATGCAAAAATTATAGCTCACGACGCGTACTAATAGAAAACGCGCTATCAGCCTCCGACGCGGCGGTATCGAAGACGCAGTCACCAGCGTACGGTATCTAGCTATGGGCATCGATCGACGACGTACGAGCGGTATCTACAGCCACTCATCACACACCCCTATAGCTCGGAGCTGACTGGGTTACTGGGGGGGATGCGTATCGACTATCGACAATATGACGCGTCGGTCACCCCATAATATGCAAAAATTATAGCTCACGACGCGTACTAATAGAAAACGCGCTATCAGCCTCCGACGCGGCGGTATCGAAG [...]
+ # 01234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345 [...]
+ # 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 [...]
+ # 0 1 2 3 4 [...]
+ # 0 0 0 0 0 [...]
+ mate1s => [ "CAGCGTACGGTATCTAGCTATGGGCATCGATCG" ],
+ mate2s => [ "CAGTCAGCTCCGAGCTATAGGGGTGTGT" ], # rev comped
+ args => "--local -X 150",
+ report => "-M 5",
+ pairhits => [ { "12,78" => 1, "249,315" => 1, "486,552" => 1,
+ "723,789" => 1, "960,1026" => 1, "1197,1263" => 1,
+ "1434,1500" => 1, "1671,1737" => 1 } ],
+ hits_are_superset => [ 1 ],
+ cigar_map => [{
+ 12 => "33M", 78 => "28M",
+ 249 => "33M", 315 => "28M",
+ 486 => "33M", 552 => "28M",
+ 723 => "33M", 789 => "28M",
+ 960 => "33M", 1026 => "28M",
+ 1197 => "33M", 1263 => "28M",
+ 1434 => "33M", 1500 => "28M",
+ 1671 => "33M", 1737 => "28M"
+ }],
+ samoptflags_map => [ {
+ 12 => { "AS:i:66" => 1, "XS:i:66" => 1, "XN:i:0" => 1, "XM:i:0" => 1,
+ "XO:i:0" => 1, "XG:i:0" => 1, "NM:i:0" => 1, "MD:Z:33" => 1,
+ "YM:i:1" => 1, "YP:i:1" => 1, "YT:Z:CP" => 1, "YS:i:56" => 1 },
+ 78 => { "AS:i:56" => 1, "XS:i:56" => 1, "XN:i:0" => 1, "XM:i:0" => 1,
+ "XO:i:0" => 1, "XG:i:0" => 1, "NM:i:0" => 1, "MD:Z:28" => 1,
+ "YM:i:1" => 1, "YP:i:1" => 1, "YT:Z:CP" => 1, "YS:i:66" => 1 },
+ 249 => { "AS:i:66" => 1, "XS:i:66" => 1, "XN:i:0" => 1, "XM:i:0" => 1,
+ "XO:i:0" => 1, "XG:i:0" => 1, "NM:i:0" => 1, "MD:Z:33" => 1,
+ "YM:i:1" => 1, "YP:i:1" => 1, "YT:Z:CP" => 1, "YS:i:56" => 1 },
+ 315 => { "AS:i:56" => 1, "XS:i:56" => 1, "XN:i:0" => 1, "XM:i:0" => 1,
+ "XO:i:0" => 1, "XG:i:0" => 1, "NM:i:0" => 1, "MD:Z:28" => 1,
+ "YM:i:1" => 1, "YP:i:1" => 1, "YT:Z:CP" => 1, "YS:i:66" => 1 },
+ 486 => { "AS:i:66" => 1, "XS:i:66" => 1, "XN:i:0" => 1, "XM:i:0" => 1,
+ "XO:i:0" => 1, "XG:i:0" => 1, "NM:i:0" => 1, "MD:Z:33" => 1,
+ "YM:i:1" => 1, "YP:i:1" => 1, "YT:Z:CP" => 1, "YS:i:56" => 1 },
+ 552 => { "AS:i:56" => 1, "XS:i:56" => 1, "XN:i:0" => 1, "XM:i:0" => 1,
+ "XO:i:0" => 1, "XG:i:0" => 1, "NM:i:0" => 1, "MD:Z:28" => 1,
+ "YM:i:1" => 1, "YP:i:1" => 1, "YT:Z:CP" => 1, "YS:i:66" => 1 },
+ 723 => { "AS:i:66" => 1, "XS:i:66" => 1, "XN:i:0" => 1, "XM:i:0" => 1,
+ "XO:i:0" => 1, "XG:i:0" => 1, "NM:i:0" => 1, "MD:Z:33" => 1,
+ "YM:i:1" => 1, "YP:i:1" => 1, "YT:Z:CP" => 1, "YS:i:56" => 1 },
+ 789 => { "AS:i:56" => 1, "XS:i:56" => 1, "XN:i:0" => 1, "XM:i:0" => 1,
+ "XO:i:0" => 1, "XG:i:0" => 1, "NM:i:0" => 1, "MD:Z:28" => 1,
+ "YM:i:1" => 1, "YP:i:1" => 1, "YT:Z:CP" => 1, "YS:i:66" => 1 },
+ 960 => { "AS:i:66" => 1, "XS:i:66" => 1, "XN:i:0" => 1, "XM:i:0" => 1,
+ "XO:i:0" => 1, "XG:i:0" => 1, "NM:i:0" => 1, "MD:Z:33" => 1,
+ "YM:i:1" => 1, "YP:i:1" => 1, "YT:Z:CP" => 1, "YS:i:56" => 1 },
+ 1026 => { "AS:i:56" => 1, "XS:i:56" => 1, "XN:i:0" => 1, "XM:i:0" => 1,
+ "XO:i:0" => 1, "XG:i:0" => 1, "NM:i:0" => 1, "MD:Z:28" => 1,
+ "YM:i:1" => 1, "YP:i:1" => 1, "YT:Z:CP" => 1, "YS:i:66" => 1 },
+ 1197 => { "AS:i:66" => 1, "XS:i:66" => 1, "XN:i:0" => 1, "XM:i:0" => 1,
+ "XO:i:0" => 1, "XG:i:0" => 1, "NM:i:0" => 1, "MD:Z:33" => 1,
+ "YM:i:1" => 1, "YP:i:1" => 1, "YT:Z:CP" => 1, "YS:i:56" => 1 },
+ 1263 => { "AS:i:56" => 1, "XS:i:56" => 1, "XN:i:0" => 1, "XM:i:0" => 1,
+ "XO:i:0" => 1, "XG:i:0" => 1, "NM:i:0" => 1, "MD:Z:28" => 1,
+ "YM:i:1" => 1, "YP:i:1" => 1, "YT:Z:CP" => 1, "YS:i:66" => 1 },
+ 1434 => { "AS:i:66" => 1, "XS:i:66" => 1, "XN:i:0" => 1, "XM:i:0" => 1,
+ "XO:i:0" => 1, "XG:i:0" => 1, "NM:i:0" => 1, "MD:Z:33" => 1,
+ "YM:i:1" => 1, "YP:i:1" => 1, "YT:Z:CP" => 1, "YS:i:56" => 1 },
+ 1500 => { "AS:i:56" => 1, "XS:i:56" => 1, "XN:i:0" => 1, "XM:i:0" => 1,
+ "XO:i:0" => 1, "XG:i:0" => 1, "NM:i:0" => 1, "MD:Z:28" => 1,
+ "YM:i:1" => 1, "YP:i:1" => 1, "YT:Z:CP" => 1, "YS:i:66" => 1 },
+ 1671 => { "AS:i:66" => 1, "XS:i:66" => 1, "XN:i:0" => 1, "XM:i:0" => 1,
+ "XO:i:0" => 1, "XG:i:0" => 1, "NM:i:0" => 1, "MD:Z:33" => 1,
+ "YM:i:1" => 1, "YP:i:1" => 1, "YT:Z:CP" => 1, "YS:i:56" => 1 },
+ 1737 => { "AS:i:56" => 1, "XS:i:56" => 1, "XN:i:0" => 1, "XM:i:0" => 1,
+ "XO:i:0" => 1, "XG:i:0" => 1, "NM:i:0" => 1, "MD:Z:28" => 1,
+ "YM:i:1" => 1, "YP:i:1" => 1, "YT:Z:CP" => 1, "YS:i:66" => 1 },
+ } ] },
+
+ #
+ # Unpaired
+ #
+
+ { name => "U.M.1.G. Unpaired -M w/ 1 hit global",
+ # 0 1 2 3
+ # 012345678901234567890123456789012
+ # CAGCGTACGGTATCTAGCTATGGGCATCGATCG
+ # CAGCGTACGGTATCTAGCTATG
+ # GGTATCTAGCTATGGGCATCGA
+ # AGACGCAGTCACCAGCGTACGGTATCTAGCTATGGGCATCGATCGACGACGTACGA
+ ref => [ "AGACGCAGTCACCAGCGTACGGTATCTAGCTATGGGCATCGATCGACGACGTACGA" ],
+ # 01234567890123456789012345678901234567890123456789012345
+ # 0 1 2 3 4 5
+ reads => [ "CAGCGTACGGTATCTAGCTATGGGCATCGATCG" ],
+ report => "-M 1",
+ hits => [ { 12 => 1 } ],
+ cigar => [ "33M" ],
+ samoptflags => [{
+ "AS:i:0" => 1, # alignment score
+ "XS:i:0" => 1, # suboptimal alignment score
+ "XN:i:0" => 1, # num ambiguous ref bases
+ "XM:i:0" => 1, # num mismatches
+ "XO:i:0" => 1, # num gap opens
+ "XG:i:0" => 1, # num gap extensions
+ "NM:i:0" => 1, # num edits
+ "MD:Z:33" => 1, # mismatching positions/bases
+ "YM:i:0" => 1, # read aligned repetitively in unpaired fashion
+ "YT:Z:UU" => 1, # unpaired read aligned in unpaired fashion
+ }]
+ },
+
+ { name => "U.M.1.L. Unpaired -M w/ 1 hit local",
+ # 0 1 2 3
+ # 012345678901234567890123456789012
+ # CAGCGTACGGTATCTAGCTATGGGCATCGATCG
+ ref => [ "AGACGCAGTCACCAGCGTACGGTATCTAGCTATGGGCATCGATCGACGACGTACGA" ],
+ # 01234567890123456789012345678901234567890123456789012345
+ # 0 1 2 3 4 5
+ reads => [ "CAGCGTACGGTATCTAGCTATGGGCATCGATCG" ],
+ args => "--local",
+ report => "-M 1",
+ hits => [ { 12 => 1 } ],
+ cigar => [ "33M" ],
+ samoptflags => [{
+ "AS:i:66" => 1, # alignment score
+ "XS:i:0" => 1, # suboptimal alignment score
+ "XN:i:0" => 1, # num ambiguous ref bases
+ "XM:i:0" => 1, # num mismatches
+ "XO:i:0" => 1, # num gap opens
+ "XG:i:0" => 1, # num gap extensions
+ "NM:i:0" => 1, # num edits
+ "MD:Z:33" => 1, # mismatching positions/bases
+ "YM:i:0" => 1, # read aligned repetitively in unpaired fashion
+ "YT:Z:UU" => 1, # unpaired read aligned in unpaired fashion
+ }]
+ },
+
+ { name => "U.k.1.G. Unpaired -k 1 w/ 1 hit global",
+ # 0 1 2 3
+ # 012345678901234567890123456789012
+ # CAGCGTACGGTATCTAGCTATGGGCATCGATCG
+ ref => [ "AGACGCAGTCACCAGCGTACGGTATCTAGCTATGGGCATCGATCGACGACGTACGA" ],
+ # 01234567890123456789012345678901234567890123456789012345
+ # 0 1 2 3 4 5
+ reads => [ "CAGCGTACGGTATCTAGCTATGGGCATCGATCG" ],
+ report => "-k 1",
+ hits => [ { 12 => 1 } ],
+ cigar => [ "33M" ],
+ samoptflags => [ {
+ "AS:i:0" => 1, # alignment score
+ "XS:i:0" => 1, # suboptimal alignment score
+ "XN:i:0" => 1, # num ambiguous ref bases
+ "XM:i:0" => 1, # num mismatches
+ "XO:i:0" => 1, # num gap opens
+ "XG:i:0" => 1, # num gap extensions
+ "NM:i:0" => 1, # num edits
+ "MD:Z:33" => 1, # mismatching positions/bases
+ "YT:Z:UU" => 1, # unpaired read aligned in unpaired fashion
+ } ] },
+
+ { name => "U.M.1.L. Unpaired -m w/ 1 hit local",
+ # 0 1 2 3
+ # 012345678901234567890123456789012
+ # CAGCGTACGGTATCTAGCTATGGGCATCGATCG
+ ref => [ "AGACGCAGTCACCAGCGTACGGTATCTAGCTATGGGCATCGATCGACGACGTACGA" ],
+ # 01234567890123456789012345678901234567890123456789012345
+ # 0 1 2 3 4 5
+ reads => [ "CAGCGTACGGTATCTAGCTATGGGCATCGATCG" ],
+ args => "--local",
+ report => "-k 1",
+ hits => [ { 12 => 1 } ],
+ cigar => [ "33M" ],
+ samoptflags => [ {
+ "AS:i:66" => 1, # alignment score
+ "XS:i:0" => 1, # suboptimal alignment score
+ "XN:i:0" => 1, # num ambiguous ref bases
+ "XM:i:0" => 1, # num mismatches
+ "XO:i:0" => 1, # num gap opens
+ "XG:i:0" => 1, # num gap extensions
+ "NM:i:0" => 1, # num edits
+ "MD:Z:33" => 1, # mismatching positions/bases
+ "YT:Z:UU" => 1, # unpaired read aligned in unpaired fashion
+ } ] },
+
+ { name => "U.M.2.G. Unpaired -M 1 w/ 2 hit global",
+ # 0 1 2 0 1 2
+ # 012345678901234567890123456789 012345678901234567890123456789
+ # AGATTACGGATCTACGATTCGAGTCGGTCA AGATTACGGATCTACGATTCGAGTCGGTCA
+ ref => [ "AGACGCAGATTACGGATCTACGATTCGAGTCGGTCAGTCACCAGCGTAAGATTACGGATCTACGATTCGAGTCGGTCAAGTGCGA" ],
+ # 0123456789012345678901234567890123456789012345678901234567890123456789012345678901234
+ # 0 1 2 3 4 5 6 7 8
+ reads => [ "AGATTACGGATCTACGATTCGAGTCGGTCA" ],
+ args => "",
+ report => "-M 1",
+ hits => [ { 6 => 1, 48 => 1 } ],
+ hits_are_superset => [ 1 ],
+ cigar => [ "30M" ],
+ samoptflags => [ {
+ "AS:i:0" => 1, # alignment score
+ "XS:i:0" => 1, # suboptimal alignment score
+ "XN:i:0" => 1, # num ambiguous ref bases
+ "XM:i:0" => 1, # num mismatches
+ "XO:i:0" => 1, # num gap opens
+ "XG:i:0" => 1, # num gap extensions
+ "NM:i:0" => 1, # num edits
+ "MD:Z:30" => 1, # mismatching positions/bases
+ "YM:i:1" => 1, # read aligned repetitively in unpaired fashion
+ "YT:Z:UU" => 1, # unpaired read aligned in unpaired fashion
+ } ] },
+
+ { name => "U.M.2.L. Unpaired -M 1 w/ 2 hit local",
+ # 0 1 2 0 1 2
+ # 012345678901234567890123456789 012345678901234567890123456789
+ # AGATTACGGATCTACGATTCGAGTCGGTCA AGATTACGGATCTACGATTCGAGTCGGTCA
+ ref => [ "AGACGCAGATTACGGATCTACGATTCGAGTCGGTCAGTCACCAGCGTAAGATTACGGATCTACGATTCGAGTCGGTCAAGTGCGA" ],
+ # 0123456789012345678901234567890123456789012345678901234567890123456789012345678901234
+ # 0 1 2 3 4 5 6 7 8
+ reads => [ "AGATTACGGATCTACGATTCGAGTCGGTCA" ],
+ args => "--local",
+ report => "-M 1",
+ hits => [ { 6 => 1, 48 => 1 } ],
+ hits_are_superset => [ 1 ],
+ cigar => [ "30M" ],
+ samoptflags => [ {
+ "AS:i:60" => 1, # alignment score
+ "XS:i:60" => 1, # suboptimal alignment score
+ "XN:i:0" => 1, # num ambiguous ref bases
+ "XM:i:0" => 1, # num mismatches
+ "XO:i:0" => 1, # num gap opens
+ "XG:i:0" => 1, # num gap extensions
+ "NM:i:0" => 1, # num edits
+ "MD:Z:30" => 1, # mismatching positions/bases
+ "YM:i:1" => 1, # read aligned repetitively in unpaired fashion
+ "YT:Z:UU" => 1, # unpaired read aligned in unpaired fashion
+ } ] },
+
+ { name => "U.k.2.G. Unpaired -k 1 w/ 2 hit global",
+ # 0 1 2 0 1 2
+ # 012345678901234567890123456789 012345678901234567890123456789
+ # AGATTACGGATCTACGATTCGAGTCGGTCA AGATTACGGATCTACGATTCGAGTCGGTCA
+ ref => [ "AGACGCAGATTACGGATCTACGATTCGAGTCGGTCAGTCACCAGCGTAAGATTACGGATCTACGATTCGAGTCGGTCAAGTGCGA" ],
+ # 0123456789012345678901234567890123456789012345678901234567890123456789012345678901234
+ # 0 1 2 3 4 5 6 7 8
+ reads => [ "AGATTACGGATCTACGATTCGAGTCGGTCA" ],
+ report => "-k 1",
+ hits => [ { 6 => 1, 48 => 1 } ],
+ hits_are_superset => [ 1 ],
+ cigar => [ "30M" ],
+ samoptflags => [ {
+ "AS:i:0" => 1, # alignment score
+ "XS:i:0" => 1, # suboptimal alignment score
+ "XN:i:0" => 1, # num ambiguous ref bases
+ "XM:i:0" => 1, # num mismatches
+ "XO:i:0" => 1, # num gap opens
+ "XG:i:0" => 1, # num gap extensions
+ "NM:i:0" => 1, # num edits
+ "MD:Z:30" => 1, # mismatching positions/bases
+ "YM:i:0" => 1, # read aligned repetitively in unpaired fashion
+ "YT:Z:UU" => 1, # unpaired read aligned in unpaired fashion
+ } ] },
+
+ { name => "U.k.2.L. Unpaired -k 1 w/ 2 hit local",
+ # 0 1 2 0 1 2
+ # 012345678901234567890123456789 012345678901234567890123456789
+ # AGATTACGGATCTACGATTCGAGTCGGTCA AGATTACGGATCTACGATTCGAGTCGGTCA
+ ref => [ "AGACGCAGATTACGGATCTACGATTCGAGTCGGTCAGTCACCAGCGTAAGATTACGGATCTACGATTCGAGTCGGTCAAGTGCGA" ],
+ # 0123456789012345678901234567890123456789012345678901234567890123456789012345678901234
+ # 0 1 2 3 4 5 6 7 8
+ reads => [ "AGATTACGGATCTACGATTCGAGTCGGTCA" ],
+ report => "-k 1",
+ hits => [ { 6 => 1, 48 => 1 } ],
+ hits_are_superset => [ 1 ],
+ cigar => [ "30M" ],
+ samoptflags => [ {
+ "AS:i:0" => 1, # alignment score
+ "XS:i:0" => 1, # suboptimal alignment score
+ "XN:i:0" => 1, # num ambiguous ref bases
+ "XM:i:0" => 1, # num mismatches
+ "XO:i:0" => 1, # num gap opens
+ "XG:i:0" => 1, # num gap extensions
+ "NM:i:0" => 1, # num edits
+ "MD:Z:30" => 1, # mismatching positions/bases
+ "YM:i:0" => 1, # read aligned repetitively in unpaired fashion
+ "YT:Z:UU" => 1, # unpaired read aligned in unpaired fashion
+ } ] },
+
+ { name => "U.M.22.G. Unpaired -M 2 w/ 2 hit global",
+ # 0 1 2 0 1 2
+ # 012345678901234567890123456789 012345678901234567890123456789
+ # AGATTACGGATCTACGATTCGAGTCGGTCA AGATTACGGATCTACGATTCGAGTCGGTCA
+ ref => [ "AGACGCAGATTACGGATCTACGATTCGAGTCGGTCAGTCACCAGCGTAAGATTACGGATCTACGATTCGAGTCGGTCAAGTGCGA" ],
+ # 0123456789012345678901234567890123456789012345678901234567890123456789012345678901234
+ # 0 1 2 3 4 5 6 7 8
+ reads => [ "AGATTACGGATCTACGATTCGAGTCGGTCA" ],
+ report => "-M 2",
+ hits => [ { 6 => 1, 48 => 1 } ],
+ cigar => [ "30M" ],
+ hits_are_superset => [ 1 ],
+ samoptflags => [ {
+ "AS:i:0" => 1, # alignment score
+ "XS:i:0" => 1, # suboptimal alignment score
+ "XN:i:0" => 1, # num ambiguous ref bases
+ "XM:i:0" => 1, # num mismatches
+ "XO:i:0" => 1, # num gap opens
+ "XG:i:0" => 1, # num gap extensions
+ "NM:i:0" => 1, # num edits
+ "MD:Z:30" => 1, # mismatching positions/bases
+ "YM:i:0" => 1, # read aligned repetitively in unpaired fashion
+ "YT:Z:UU" => 1, # unpaired read aligned in unpaired fashion
+ } ] },
+
+ { name => "U.M.22.L. Unpaired -M 2 w/ 2 hit local",
+ # 0 1 2 0 1 2
+ # 012345678901234567890123456789 012345678901234567890123456789
+ # AGATTACGGATCTACGATTCGAGTCGGTCA AGATTACGGATCTACGATTCGAGTCGGTCA
+ ref => [ "AGACGCAGATTACGGATCTACGATTCGAGTCGGTCAGTCACCAGCGTAAGATTACGGATCTACGATTCGAGTCGGTCAAGTGCGA" ],
+ # 0123456789012345678901234567890123456789012345678901234567890123456789012345678901234
+ # 0 1 2 3 4 5 6 7 8
+ reads => [ "AGATTACGGATCTACGATTCGAGTCGGTCA" ],
+ report => "-M 2 --local",
+ hits => [ { 6 => 1, 48 => 1 } ],
+ cigar => [ "30M" ],
+ hits_are_superset => [ 1 ],
+ samoptflags => [ {
+ "AS:i:60" => 1, # alignment score
+ "XS:i:60" => 1, # suboptimal alignment score
+ "XN:i:0" => 1, # num ambiguous ref bases
+ "XM:i:0" => 1, # num mismatches
+ "XO:i:0" => 1, # num gap opens
+ "XG:i:0" => 1, # num gap extensions
+ "NM:i:0" => 1, # num edits
+ "MD:Z:30" => 1, # mismatching positions/bases
+ "YM:i:0" => 1, # read aligned repetitively in unpaired fashion
+ "YT:Z:UU" => 1, # unpaired read aligned in unpaired fashion
+ } ] },
+
+ { name => "U.k.22.G. Unpaired -k 2 w/ 2 hit global",
+ # 0 1 2 0 1 2
+ # 012345678901234567890123456789 012345678901234567890123456789
+ # AGATTACGGATCTACGATTCGAGTCGGTCA AGATTACGGATCTACGATTCGAGTCGGTCA
+ ref => [ "AGACGCAGATTACGGATCTACGATTCGAGTCGGTCAGTCACCAGCGTAAGATTACGGATCTACGATTCGAGTCGGTCAAGTGCGA" ],
+ # 0123456789012345678901234567890123456789012345678901234567890123456789012345678901234
+ # 0 1 2 3 4 5 6 7 8
+ reads => [ "AGATTACGGATCTACGATTCGAGTCGGTCA" ],
+ report => "-k 2",
+ hits => [ { 6 => 1, 48 => 1 } ],
+ cigar => [ "30M" ],
+ samoptflags => [ {
+ "AS:i:0" => 1, # alignment score
+ "XS:i:0" => 1, # suboptimal alignment score
+ "XN:i:0" => 1, # num ambiguous ref bases
+ "XM:i:0" => 1, # num mismatches
+ "XO:i:0" => 1, # num gap opens
+ "XG:i:0" => 1, # num gap extensions
+ "NM:i:0" => 1, # num edits
+ "MD:Z:30" => 1, # mismatching positions/bases
+ "YM:i:0" => 1, # read aligned repetitively in unpaired fashion
+ "YT:Z:UU" => 1, # unpaired read aligned in unpaired fashion
+ } ] },
+
+ { name => "U.k.22.L. Unpaired -k 2 w/ 2 hit local",
+ # 0 1 2 0 1 2
+ # 012345678901234567890123456789 012345678901234567890123456789
+ # AGATTACGGATCTACGATTCGAGTCGGTCA AGATTACGGATCTACGATTCGAGTCGGTCA
+ ref => [ "AGACGCAGATTACGGATCTACGATTCGAGTCGGTCAGTCACCAGCGTAAGATTACGGATCTACGATTCGAGTCGGTCAAGTGCGA" ],
+ # 0123456789012345678901234567890123456789012345678901234567890123456789012345678901234
+ # 0 1 2 3 4 5 6 7 8
+ reads => [ "AGATTACGGATCTACGATTCGAGTCGGTCA" ],
+ args => "--local",
+ report => "-k 2",
+ hits => [ { 6 => 1, 48 => 1 } ],
+ cigar => [ "30M" ],
+ samoptflags => [ {
+ "AS:i:60" => 1, # alignment score
+ "XS:i:60" => 1, # suboptimal alignment score
+ "XN:i:0" => 1, # num ambiguous ref bases
+ "XM:i:0" => 1, # num mismatches
+ "XO:i:0" => 1, # num gap opens
+ "XG:i:0" => 1, # num gap extensions
+ "NM:i:0" => 1, # num edits
+ "MD:Z:30" => 1, # mismatching positions/bases
+ "YM:i:0" => 1, # read aligned repetitively in unpaired fashion
+ "YT:Z:UU" => 1, # unpaired read aligned in unpaired fashion
+ } ] },
+
+ { name => "U.M.58.G. Unpaired -M 5 w/ 8 hits global",
+ # 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2
+ # 012345678901234567890123456789 012345678901234567890123456789 012345678901234567890123456789 012345678901234567890123456789 012345678901234567890123456789 012345678901234567890123456789 012345678901234567890123456789 012345678901234567890123456789
+ # AGATTACGGATCTACGATTCGAGTCGGTCA AGATTACGGATCTACGATTCGAGTCGGTCA AGATTACGGATCTACGATTCGAGTCGGTCA AGATTACGGATCTACGATTCGAGTCGGTCA AGATTACGGATCTACGATTCGAGTCGGTCA AGATTACGGATCTACGATTCGAGTCGGTCA AGATTACGGATCTACGATTCGAGTCGGTCA AGATTACGGATCTACGATTCGAGTCGGTCA
+ ref => [ "AGACGCAGATTACGGATCTACGATTCGAGTCGGTCAGTCACCAGCGTAAGATTACGGATCTACGATTCGAGTCGGTCAAGTGCGAAGACGCAGATTACGGATCTACGATTCGAGTCGGTCAGTCACCAGCGTAAGATTACGGATCTACGATTCGAGTCGGTCAAGTGCGAAGACGCAGATTACGGATCTACGATTCGAGTCGGTCAGTCACCAGCGTAAGATTACGGATCTACGATTCGAGTCGGTCAAGTGCGAAGACGCAGATTACGGATCTACGATTCGAGTCGGTCAGTCACCAGCGTAAGATTACGGATCTACGATTCGAGTCGGTCAAGTGCGA" ],
+ # 0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789
+ # 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3
+ # 0 1 2 3
+ reads => [ "AGATTACGGATCTACGATTCGAGTCGGTCA" ],
+ args => "-X 150",
+ report => "-M 5",
+ hits => [ { 6 => 1, 48 => 1, 91 => 1, 133 => 1, 176 => 1, 218 => 1, 261 => 1, 303 => 1 } ],
+ hits_are_superset => [ 1 ],
+ cigar => [ "30M" ],
+ samoptflags => [ {
+ "AS:i:0" => 1, # alignment score
+ "XS:i:0" => 1, # suboptimal alignment score
+ "XN:i:0" => 1, # num ambiguous ref bases
+ "XM:i:0" => 1, # num mismatches
+ "XO:i:0" => 1, # num gap opens
+ "XG:i:0" => 1, # num gap extensions
+ "NM:i:0" => 1, # num edits
+ "MD:Z:30" => 1, # mismatching positions/bases
+ "YM:i:1" => 1, # read aligned repetitively in unpaired fashion
+ "YT:Z:UU" => 1, # unpaired read aligned in unpaired fashion
+ } ] },
+
+ { name => "U.M.58.L. Unpaired -M 5 w/ 8 hits global",
+ # 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2
+ # 012345678901234567890123456789 012345678901234567890123456789 012345678901234567890123456789 012345678901234567890123456789 012345678901234567890123456789 012345678901234567890123456789 012345678901234567890123456789 012345678901234567890123456789
+ # AGATTACGGATCTACGATTCGAGTCGGTCA AGATTACGGATCTACGATTCGAGTCGGTCA AGATTACGGATCTACGATTCGAGTCGGTCA AGATTACGGATCTACGATTCGAGTCGGTCA AGATTACGGATCTACGATTCGAGTCGGTCA AGATTACGGATCTACGATTCGAGTCGGTCA AGATTACGGATCTACGATTCGAGTCGGTCA AGATTACGGATCTACGATTCGAGTCGGTCA
+ ref => [ "AGACGCAGATTACGGATCTACGATTCGAGTCGGTCAGTCACCAGCGTAAGATTACGGATCTACGATTCGAGTCGGTCAAGTGCGAAGACGCAGATTACGGATCTACGATTCGAGTCGGTCAGTCACCAGCGTAAGATTACGGATCTACGATTCGAGTCGGTCAAGTGCGAAGACGCAGATTACGGATCTACGATTCGAGTCGGTCAGTCACCAGCGTAAGATTACGGATCTACGATTCGAGTCGGTCAAGTGCGAAGACGCAGATTACGGATCTACGATTCGAGTCGGTCAGTCACCAGCGTAAGATTACGGATCTACGATTCGAGTCGGTCAAGTGCGA" ],
+ # 0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789
+ # 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3
+ # 0 1 2 3
+ reads => [ "AGATTACGGATCTACGATTCGAGTCGGTCA" ],
+ args => "--local",
+ report => "-M 5",
+ hits => [ { 6 => 1, 48 => 1, 91 => 1, 133 => 1, 176 => 1, 218 => 1, 261 => 1, 303 => 1 } ],
+ hits_are_superset => [ 1 ],
+ cigar => [ "30M" ],
+ samoptflags => [ {
+ "AS:i:60" => 1, # alignment score
+ "XS:i:60" => 1, # suboptimal alignment score
+ "XN:i:0" => 1, # num ambiguous ref bases
+ "XM:i:0" => 1, # num mismatches
+ "XO:i:0" => 1, # num gap opens
+ "XG:i:0" => 1, # num gap extensions
+ "NM:i:0" => 1, # num edits
+ "MD:Z:30" => 1, # mismatching positions/bases
+ "YM:i:1" => 1, # read aligned repetitively in unpaired fashion
+ "YT:Z:UU" => 1, # unpaired read aligned in unpaired fashion
+ } ] },
+
+ { name => "U.k.58.G. Unpaired -k 5 w/ 8 hits global",
+ # 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2
+ # 012345678901234567890123456789 012345678901234567890123456789 012345678901234567890123456789 012345678901234567890123456789 012345678901234567890123456789 012345678901234567890123456789 012345678901234567890123456789 012345678901234567890123456789
+ # AGATTACGGATCTACGATTCGAGTCGGTCA AGATTACGGATCTACGATTCGAGTCGGTCA AGATTACGGATCTACGATTCGAGTCGGTCA AGATTACGGATCTACGATTCGAGTCGGTCA AGATTACGGATCTACGATTCGAGTCGGTCA AGATTACGGATCTACGATTCGAGTCGGTCA AGATTACGGATCTACGATTCGAGTCGGTCA AGATTACGGATCTACGATTCGAGTCGGTCA
+ ref => [ "AGACGCAGATTACGGATCTACGATTCGAGTCGGTCAGTCACCAGCGTAAGATTACGGATCTACGATTCGAGTCGGTCAAGTGCGAAGACGCAGATTACGGATCTACGATTCGAGTCGGTCAGTCACCAGCGTAAGATTACGGATCTACGATTCGAGTCGGTCAAGTGCGAAGACGCAGATTACGGATCTACGATTCGAGTCGGTCAGTCACCAGCGTAAGATTACGGATCTACGATTCGAGTCGGTCAAGTGCGAAGACGCAGATTACGGATCTACGATTCGAGTCGGTCAGTCACCAGCGTAAGATTACGGATCTACGATTCGAGTCGGTCAAGTGCGA" ],
+ # 0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789
+ # 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3
+ # 0 1 2 3
+ reads => [ "AGATTACGGATCTACGATTCGAGTCGGTCA" ],
+ report => "-k 5",
+ hits => [ { 6 => 1, 48 => 1, 91 => 1, 133 => 1, 176 => 1, 218 => 1, 261 => 1, 303 => 1 } ],
+ hits_are_superset => [ 1 ],
+ cigar => [ "30M" ],
+ samoptflags => [ {
+ "AS:i:0" => 1, # alignment score
+ "XS:i:0" => 1, # suboptimal alignment score
+ "XN:i:0" => 1, # num ambiguous ref bases
+ "XM:i:0" => 1, # num mismatches
+ "XO:i:0" => 1, # num gap opens
+ "XG:i:0" => 1, # num gap extensions
+ "NM:i:0" => 1, # num edits
+ "MD:Z:30" => 1, # mismatching positions/bases
+ "YM:i:0" => 1, # read aligned repetitively in unpaired fashion
+ "YT:Z:UU" => 1, # unpaired read aligned in unpaired fashion
+ } ] },
+
+ { name => "U.k.58.L. Unpaired -k 5 w/ 8 hits local",
+ # 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2 0 1 2
+ # 012345678901234567890123456789 012345678901234567890123456789 012345678901234567890123456789 012345678901234567890123456789 012345678901234567890123456789 012345678901234567890123456789 012345678901234567890123456789 012345678901234567890123456789
+ # AGATTACGGATCTACGATTCGAGTCGGTCA AGATTACGGATCTACGATTCGAGTCGGTCA AGATTACGGATCTACGATTCGAGTCGGTCA AGATTACGGATCTACGATTCGAGTCGGTCA AGATTACGGATCTACGATTCGAGTCGGTCA AGATTACGGATCTACGATTCGAGTCGGTCA AGATTACGGATCTACGATTCGAGTCGGTCA AGATTACGGATCTACGATTCGAGTCGGTCA
+ ref => [ "AGACGCAGATTACGGATCTACGATTCGAGTCGGTCAGTCACCAGCGTAAGATTACGGATCTACGATTCGAGTCGGTCAAGTGCGAAGACGCAGATTACGGATCTACGATTCGAGTCGGTCAGTCACCAGCGTAAGATTACGGATCTACGATTCGAGTCGGTCAAGTGCGAAGACGCAGATTACGGATCTACGATTCGAGTCGGTCAGTCACCAGCGTAAGATTACGGATCTACGATTCGAGTCGGTCAAGTGCGAAGACGCAGATTACGGATCTACGATTCGAGTCGGTCAGTCACCAGCGTAAGATTACGGATCTACGATTCGAGTCGGTCAAGTGCGA" ],
+ # 0123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789012345678901234567890123456789
+ # 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3
+ # 0 1 2 3
+ reads => [ "AGATTACGGATCTACGATTCGAGTCGGTCA" ],
+ args => "--local",
+ report => "-k 5",
+ hits => [ { 6 => 1, 48 => 1, 91 => 1, 133 => 1, 176 => 1, 218 => 1, 261 => 1, 303 => 1 } ],
+ hits_are_superset => [ 1 ],
+ cigar => [ "30M" ],
+ samoptflags => [ {
+ "AS:i:60" => 1, # alignment score
+ "XS:i:60" => 1, # suboptimal alignment score
+ "XN:i:0" => 1, # num ambiguous ref bases
+ "XM:i:0" => 1, # num mismatches
+ "XO:i:0" => 1, # num gap opens
+ "XG:i:0" => 1, # num gap extensions
+ "NM:i:0" => 1, # num edits
+ "MD:Z:30" => 1, # mismatching positions/bases
+ "YM:i:0" => 1, # read aligned repetitively in unpaired fashion
+ "YT:Z:UU" => 1, # unpaired read aligned in unpaired fashion
+ } ] },
+
+ # Following cases depend on this being the case:
+ #
+ # static const float DEFAULT_CEIL_CONST = 3.0f;
+ # static const float DEFAULT_CEIL_LINEAR = 3.0f;
+
+ # Just enough budget for hits, so it should align
+ { ref => [ "TTGTTCGTTTGTTCGT" ],
+ reads => [ "TTGTTCAT" ], # budget = 3 + 8 * 3 = 27
+ args => "-L 6 -i C,1,0 --policy \"MMP=C27\\;MIN=L,-3,-3\\;RDG=25,15\\;RFG=25,15\"", # penalty = 27
+ report => "-a",
+ hits => [ { 0 => 1, 8 => 1 } ],
+ flags => [ "XM:0,XP:0,XT:UU,XC:6=1X1=" ],
+ cigar => [ "8M" ],
+ samoptflags => [ { "AS:i:-27" => 1, "XS:i:-27" => 1, "NM:i:1" => 1,
+ "XM:i:1" => 1, "YT:Z:UU" => 1, "MD:Z:6G1" => 1 } ] },
+
+ # Not quite enough budget for hits, so it should NOT align
+ { ref => [ "TTGTTCGTTTGTTCGT" ],
+ reads => [ "TTGTTCAT" ], # budget = 3 + 8 * 3 = 27
+ args => "-L 6 -i C,1,0 --policy \"MMP=C28\\;MIN=L,-3,-3\\;RDG=25,15\\;RFG=25,15\"", # penalty = 28
+ report => "-a",
+ hits => [ { "*" => 1 } ],
+ flags => [ "XM:0,XP:0,XT:UU" ],
+ cigar => [ "*" ],
+ samoptflags => [ { "YT:Z:UU" => 1 } ] },
+
+ # Check that using a seed of length 1 with 1-mismatch doesn't crash.
+ # Perhaps we should disallow it though?
+
+ { ref => [ "AAAAAAAAAAAAAAAAAAAAAAAAACCCCCCCCCCCCCCCCCCCCCCCC" ],
+ reads => [ "AA", "AA", "AA", "AA", "CC", "CC", "CC", "CC", "AA", "AA", "AA", "AA", "CC", "CC", "CC", "CC" ],
+ names => [ "r1", "r1", "r1", "r1", "r2", "r2", "r2", "r2", "r3", "r3", "r3", "r3", "r4", "r4", "r4", "r4" ],
+ args => "--policy \"SEED=1,1\"",
+ check_random => 1,
+ report => "-k 1" },
+
+ #
+ # Gap penalties
+ #
+
+ # Alignment with 1 read gap
+ { name => "Gap penalties 1",
+ ref => [ "TTGTTCGTTTGTTCGT" ],
+ reads => [ "TTGTTCTTTGTT" ], # budget = 3 + 12 * 3 = 39
+ args => "--policy \"MMP=C30\\;SEED=0,3\\;IVAL=C,1,0\\;RDG=29,10\\;RFG=25,15\\;MIN=L,-3,-3\"",
+ report => "-a",
+ hits => [ { 0 => 1 } ],
+ flags => [ "XM:0,XP:0,XT:UU,XC:6=1D6=" ],
+ cigar => [ "6M1D6M" ],
+ samoptflags => [{
+ "AS:i:-39" => 1, "NM:i:1" => 1, "XO:i:1" => 1, "XG:i:1" => 1,
+ "YT:Z:UU" => 1, "MD:Z:6^G6" => 1 }]
+ },
+
+ # Alignment with 1 read gap, but not enough budget
+ { name => "Gap penalties 2",
+ ref => [ "TTGTTCGTTTGTTCGT" ],
+ reads => [ "TTGTTCTTTGTT" ], # budget = 3 + 12 * 3 = 39
+ args => "--policy \"MMP=C30\\;SEED=0,3\\;IVAL=C,1,0\\;RDG=30,10\\;RFG=25,15\\;MIN=L,-3,-3\"",
+ report => "-a",
+ hits => [ { "*" => 1 } ],
+ flags => [ "XM:0,XP:0,XT:UU" ],
+ cigar => [ "*" ],
+ samoptflags => [{ "YT:Z:UU" => 1 }]
+ },
+
+ # Alignment with 1 reference gap
+ { name => "Gap penalties 3",
+ ref => [ "TTGTTCGTTTGTTCGT" ],
+ reads => [ "TTGTTCGATTTGTT" ], # budget = 3 + 14 * 3 = 45
+ args => "--policy \"MMP=C30\\;SEED=0,3\\;IVAL=C,1,0\\;RDG=25,15\\;RFG=30,15\\;MIN=L,-3,-3\"",
+ report => "-a",
+ hits => [ { 0 => 1 } ],
+ flags => [ "XM:0,XP:0,XT:UU,XC:7=1I6=" ],
+ cigar => [ "7M1I6M" ],
+ samoptflags => [{ "AS:i:-45" => 1, "NM:i:1" => 1, "XO:i:1" => 1,
+ "XG:i:1" => 1, "YT:Z:UU" => 1, "MD:Z:13" => 1 }]
+ },
+
+ # 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5
+ # T T G T T C G T T T G T T C G T
+ # 0 1 1 0 2 3 1 0 0 1 1 0 2 3 1
+ # 0 T x
+ # 0 0 x
+ # 1 T x
+ # 1 1 x
+ # 2 G x
+ # 2 1 x
+ # 3 T x
+ # 3 0 x
+ # 4 T x
+ # 4 2 x
+ # 5 C x
+ # 5 3 x
+ # 6 G
+ # 6 2 x
+ # 7 A
+ # 7 3 x
+ # 8 T x
+ # 8 0 x
+ # 9 T x
+ # 9 0 x
+ # 0 T x
+ # 0 1 x
+ # 1 G x
+ # 1 1 x
+ # 2 T x
+ # 2 0 x
+ # 3 T
+ #
+
+ # Alignment with 1 reference gap, but not enough budget
+ { name => "Gap penalties 4",
+ ref => [ "TTGTTCGTTTGTTCGT" ],
+ reads => [ "TTGTTCGATTTGTT" ], # budget = 3 + 14 * 3 = 45
+ args => "--policy \"MMP=C30\\;SEED=0,3\\;IVAL=C,1,0\\;RDG=25,15\\;RFG=30,16\\;MIN=L,-3,-3\"",
+ report => "-a",
+ hits => [ { "*" => 1 } ],
+ flags => [ "XM:0,XP:0,XT:UU" ],
+ cigar => [ "*" ],
+ samoptflags => [ { "YT:Z:UU" => 1 } ] },
+
+ # Alignment with 1 reference gap, but not enough budget
+ { name => "Gap penalties 5",
+ ref => [ "TTGTTCGTTTGTTCGT" ],
+ reads => [ "TTGTTCGATTTGTT" ], # budget = 3 + 14 * 3 = 45
+ args => "--policy \"MMP=C30\\;SEED=0,3\\;IVAL=C,1,0\\;RDG=25,15\\;RFG=31,15\\;MIN=L,-3,-3\"",
+ report => "-a",
+ hits => [ { "*" => 1 } ],
+ flags => [ "XM:0,XP:0,XT:UU" ],
+ cigar => [ "*" ],
+ samoptflags => [ { "YT:Z:UU" => 1 } ] },
+
+ # Alignment with 1 reference gap and 1 read gap
+ { name => "Gap penalties 6",
+ ref => [ "ATTGTTCGTTTGTTCGTA" ],
+ reads => [ "ATTGTTGTTTGATTCGTA" ], # budget = 3 + 18 * 3 = 57
+ args => "--policy \"MMP=C30\\;SEED=0,3\\;IVAL=C,1,0\\;RDG=19,10\\;RFG=18,10\\;MIN=L,-3,-3\"",
+ report => "-a",
+ hits => [ { 0 => 1 } ],
+ flags => [ "XM:0,XP:0,XT:UU,XC:6=1D5=1I6=" ],
+ cigar => [ "6M1D5M1I6M" ] },
+
+ # Alignment with 1 reference gap and 1 read gap, but not enough budget
+ { name => "Gap penalties 7",
+ ref => [ "TTGTTCGTTTGTTCGT" ],
+ reads => [ "TTGTTGTTTGATTCGT" ], # budget = 3 + 16 * 3 = 51
+ args => "--policy \"MMP=C30\\;SEED=0,3\\;IVAL=C,1,0\\;RDG=16,10\\;RFG=16,10\\;MIN=L,-3,-3\"",
+ report => "-a",
+ hits => [ { "*" => 1 } ],
+ flags => [ "XM:0,XP:0,XT:UU" ],
+ cigar => [ "*" ] },
+
+ # Experiment with N filtering
+
+ { name => "N filtering 1",
+ ref => [ "GAGACTTTATACGCATCGAACTATCGCTCTA" ],
+ reads => [ "ATACGCATCGAAC" ],
+ # 0123456789012345678901234567890
+ # 1 2 3
+ args => "--policy \"NCEIL=L,0,0\"",
+ report => "-a",
+ hits => [ { 8 => 1 } ],
+ flags => [ "XM:0,XP:0,XT:UU,XC:13=" ] },
+
+ { name => "N filtering 2",
+ ref => [ "GAGACTTTATNCGCATCGAACTATCGCTCTA" ],
+ reads => [ "ATACGCATCGAAC" ],
+ # 0123456789012345678901234567890
+ # 1 2 3
+ args => "--policy \"NCEIL=L,0,0\"",
+ report => "-a",
+ hits => [ { "*" => 1 } ] },
+
+ { name => "N filtering 3",
+ ref => [ "GAGACTTTATACGCATCGAANTATCGCTCTA" ],
+ reads => [ "ATACGCATCGAAC" ],
+ # 0123456789012345678901234567890
+ # 1 2 3
+ args => "--policy \"NCEIL=L,0,0\"",
+ report => "-a",
+ hits => [ { "*" => 1 } ] },
+
+ { name => "N filtering 4",
+ ref => [ "GAGACTTTNTACGCATCGAACTATCGCTCTA" ],
+ reads => [ "ATACGCATCGAAC" ],
+ # 0123456789012345678901234567890
+ # 1 2 3
+ args => "--policy \"NCEIL=L,0,0\"",
+ report => "-a",
+ hits => [ { "*" => 1 } ] },
+
+ { name => "N filtering 5",
+ ref => [ "GAGACTTTATNCGCATCGAACTATCGCTCTA" ],
+ reads => [ "ATACGCATCGAAC" ],
+ # 0123456789012345678901234567890
+ # 1 2 3
+ args => "--policy \"NCEIL=L,0,0.1\\;SEED=0,10\\;IVAL=C,1,0\"",
+ report => "-a",
+ hits => [ { 8 => 1 } ],
+ flags => [ "XM:0,XP:0,XT:UU,XC:2=1X10=" ] },
+
+ { name => "N filtering 6",
+ ref => [ "GAGACTTTNTACGCATCGAANTATCGCTCTA" ],
+ reads => [ "ATACGCATCGAAC" ],
+ # 0123456789012345678901234567890
+ # 1 2 3
+ args => "--policy \"NCEIL=L,0,0.1\\;SEED=0,10\\;IVAL=C,1,0\"",
+ report => "-a",
+ hits => [ { "*" => 1 } ] },
+
+ # No discordant alignment because one mate is repetitive.
+
+ # Alignment with 1 reference gap
+ { ref => [ "TTTTGTTCGTTTG" ],
+ reads => [ "TTTTGTTCGATTTG" ], # budget = 3 + 14 * 3 = 45
+ args => "--policy \"SEED=0,8\\;IVAL=C,1,0\\;MMP=C30\\;RDG=25,15\\;RFG=25,20\\;MIN=L,-3,-3\"",
+ report => "-a",
+ hits => [ { 0 => 1 } ],
+ flags => [ "XM:0,XP:0,XT:UU,XC:9=1I4=" ],
+ cigar => [ "9M1I4M" ],
+ samoptflags => [
+ { "AS:i:-45" => 1, "NM:i:1" => 1, "XO:i:1" => 1, "XG:i:1" => 1,
+ "YT:Z:UU" => 1, "MD:Z:13" => 1 },
+ ]
+ },
+
+ # TTGTTCGTTTGTT
+ # Tx
+ # T x
+ # G x
+ # T x
+ # T x
+ # C x
+ # G x
+ # A x
+ # T x
+ # T x
+ # T x
+ # G x
+ # T x
+ # T x
+
+ # Alignment with 1 reference gap
+ { ref => [ "TTGTTCGTTTGTT" ],
+ reads => [ "TTGTTCGATTTGTT" ], # budget = 3 + 14 * 3 = 45
+ args => "--policy \"SEED=0,3\\;IVAL=C,1,0\\;MMP=C30\\;RDG=25,15\\;RFG=25,20\\;MIN=L,-3,-3\"",
+ report => "-a",
+ hits => [ { 0 => 1 } ],
+ flags => [ "XM:0,XP:0,XT:UU,XC:7=1I6=" ],
+ cigar => [ "7M1I6M" ],
+ samoptflags => [
+ { "AS:i:-45" => 1, "NM:i:1" => 1, "XO:i:1" => 1, "XG:i:1" => 1,
+ "YT:Z:UU" => 1, "MD:Z:13" => 1 },
+ ]
+ },
+
+ { ref => [ "ACNCA" ],
+ reads => [ "CA" ],
+ args => "",
+ report => "-a --policy \"SEED=0,2\\;IVAL=C,1,0\\;NCEIL=L,0,0\"",
+ hits => [ { 3 => 1 } ],
+ edits => [ ],
+ flags => [ "XM:0,XP:0,XT:UU,XC:2=" ],
+ cigar => [ "2M" ],
+ samoptflags => [
+ { "YT:Z:UU" => 1, "MD:Z:2" => 1 },
+ ]
+ },
+
+ { name => "N ceil = 0, 2 legit hits (1)",
+ ref => [ "ACNCA" ],
+ reads => [ "AC" ],
+ args => "",
+ report => "-a --policy \"SEED=0,2\\;IVAL=C,1,0\\;NCEIL=L,0,0\"",
+ hits => [ { 0 => 1 } ],
+ edits => [ ],
+ flags => [ ] },
+
+ { name => "N ceil = 0, 2 legit hits (2)",
+ ref => [ "ACNCANNNNNNNNCGNNNNNNNNCG" ],
+ # 0123456789012345678901234
+ # 0 1 2
+ reads => [ "CG" ],
+ args => "",
+ report => "-a --policy \"SEED=0,2\\;IVAL=C,1,0\\;NCEIL=L,0,0\"",
+ hits => [ { 13 => 2, 23 => 2 } ],
+ edits => [ ],
+ cigar => [ "2M", "2M" ],
+ samoptflags => [
+ { "YT:Z:UU" => 1, "MD:Z:2" => 1 },
+ { "YT:Z:UU" => 1, "MD:Z:2" => 1 },
+ ]
+ },
+
+ { ref => [ "ACNCANNNNNNAACGNNNNNNNACGAANNNNCGAAAN" ],
+ # 0123456789012345678901234567890123456
+ # 0 1 2 3
+ reads => [ "CG" ],
+ args => "",
+ report => "-a --policy \"SEED=0,2\\;IVAL=C,1,0\\;NCEIL=L,0,0\"",
+ hits => [ { 13 => 2, 23 => 2, 31 => 2 } ],
+ edits => [ ],
+ flags => [ "XM:0,XP:0,XT:UU,XC:2=",
+ "XM:0,XP:0,XT:UU,XC:2=",
+ "XM:0,XP:0,XT:UU,XC:2=" ],
+ cigar => [ "2M", "2M", "2M" ],
+ samoptflags => [
+ { "YT:Z:UU" => 1, "MD:Z:2" => 1 },
+ { "YT:Z:UU" => 1, "MD:Z:2" => 1 },
+ { "YT:Z:UU" => 1, "MD:Z:2" => 1 },
+ ]
+ },
+
+ { ref => [ "ACNCANNNNNNAACGNNNNNNNACGAANNNNCGAAAN" ],
+ # 0123456789012345678901234567890123456
+ # 0 1 2 3
+ reads => [ "CG" ],
+ args => "",
+ report => "-a --policy \"SEED=0,1\\;IVAL=C,1,0\\;NCEIL=L,0,0\"",
+ hits => [ { 13 => 2, 23 => 2, 31 => 2 } ],
+ edits => [ ],
+ flags => [ "XM:0,XP:0,XT:UU,XC:2=",
+ "XM:0,XP:0,XT:UU,XC:2=",
+ "XM:0,XP:0,XT:UU,XC:2=" ],
+ cigar => [ "2M", "2M", "2M" ],
+ samoptflags => [
+ { "YT:Z:UU" => 1, "MD:Z:2" => 1 },
+ { "YT:Z:UU" => 1, "MD:Z:2" => 1 },
+ { "YT:Z:UU" => 1, "MD:Z:2" => 1 },
+ ]
+ },
+
+ #
+ # Alignment involving ambiguous reference character
+ #
+
+ # First read has non-compatible unambiguous charcacter (G for Y),
+ # second read has compatible one
+ { ref => [ "TTGTTYGT" ],
+ reads => [ "TTGTTGGT", "TTGTTCGT" ],
+ args => "",
+ report => "-a --policy \"SEED=0,5\\;IVAL=C,1,0\\;NCEIL=L,2,0\"",
+ hits => [ { 0 => 1 }, { 0 => 1 } ],
+ norc => 1,
+ edits => [ "5:N>G", "5:N>C" ],
+ flags => [ "XM:0,XP:0,XT:UU,XC:5=1X2=", "XM:0,XP:0,XT:UU,XC:5=1X2=" ],
+ cigar => [ "8M", "8M" ],
+ samoptflags => [
+ { "AS:i:-1" => 1, "NM:i:1" => 1, "XM:i:1" => 1, "XN:i:1" => 1,
+ "YT:Z:UU" => 1, "MD:Z:5N2" => 1 },
+ { "AS:i:-1" => 1, "NM:i:1" => 1, "XM:i:1" => 1, "XN:i:1" => 1,
+ "YT:Z:UU" => 1, "MD:Z:5N2" => 1 },
+ ]
+ },
+
+ #
+ # Alignment with multi-character read gap
+ #
+
+ # Relatively small example with a read gap extend
+ { ref => [ "ATAACCTTCG" ],
+ reads => [ "ATAATTCG" ], # 3 * 19 + 3 = 60
+ # ^
+ # 4:CC>-
+ args => "",
+ report => "-a --overhang --gbar 3 --policy \"MMP=C30\\;RDG=5,5\\;SEED=0,4\\;IVAL=C,1,0\\;RFG=25,20\\;MIN=L,-3,-3\"",
+ hits => [ { 0 => 1 } ],
+ edits => [ "4:CC>-" ],
+ flags => [ "XM:0,XP:0,XT:UU,XC:4=2D4=" ],
+ cigar => [ "4M2D4M" ],
+ samoptflags => [
+ { "AS:i:-15" => 1, "NM:i:2" => 1,
+ "XO:i:1" => 1, "XG:i:2" => 3, "YT:Z:UU" => 1, "MD:Z:4^CC4" => 1 }
+ ]
+ },
+
+ # Reads 1 and 2 don't have overhang, reads 3 and 4 overhang opposite ends
+ { ref => [ "ATATGCCCCATGCCCCCCTCCG" ],
+ reads => [ "ATATGCCCCCCCCCCTCCG" ], # 3 * 19 + 3 = 60
+ # ^
+ # 9:ATG>-
+ args => "--policy \"SEED=0,8\\;IVAL=C,1,0\\;MMP=C30\\;RDG=5,5\\;RFG=25,15\\;MIN=L,-3,-3\"",
+ hits => [ { 0 => 1 } ],
+ edits => [ "9:ATG>-" ],
+ norc => 1,
+ flags => [ "XM:0,XP:0,XT:UU,XC:9=3D10=" ],
+ cigar => [ "9M3D10M" ],
+ samoptflags => [
+ { "AS:i:-20" => 1, "NM:i:3" => 1,
+ "XO:i:1" => 1, "XG:i:3" => 3, "YT:Z:UU" => 1, "MD:Z:9^ATG10" => 1 }
+ ]
+ },
+
+ # Reads 1 and 2 don't have overhang, reads 3 and 4 overhang opposite ends
+ { ref => [ "ATATGCCCCATGCCCCCCTCCG" ],
+ reads => [ "CGGAGGGGGGGGGGCATAT" ],
+ # ATATGCCCCCCCCCCTCCG
+ # ^
+ # 10:GTA>-
+ args => "",
+ report => "-a --overhang --policy \"SEED=0,8\\;IVAL=C,1,0\\;MMP=C30\\;RDG=5,5\\;RFG=25,20\\;MIN=L,-3,-3\"",
+ hits => [ { 0 => 1 } ],
+ edits => [ "10:GTA>-" ],
+ norc => 1,
+ flags => [ "XM:0,XP:0,XT:UU,XC:9=3D10=" ],
+ cigar => [ "9M3D10M" ],
+ samoptflags => [
+ { "AS:i:-20" => 1, "NM:i:3" => 1,
+ "XO:i:1" => 1, "XG:i:3" => 3, "YT:Z:UU" => 1, "MD:Z:9^ATG10" => 1 }
+ ]
+ },
+
+ # 1 discordant alignment and one concordant alignment. Discordant because
+ # the fragment is too long.
+
+ { name => "Simple paired-end 13",
+ ref => [ "TTTATAAAAATATTTCCCCCCCCCCCCCCTGTCGCTACCGCCCCCCCCCCC" ],
+ # 012345678901234567890123456789012345678901234567890
+ # 0 1 2 3 4 5
+ # ATAAAAATAT GTCGCTACCG
+ # ATAAAAATAT TGTCGCTACC
+ # ATAAAAATAT CTGTCGCTAC
+ # ATAAAAATAT CCTGTCGCTA
+ # TAAAAATATT GTCGCTACCG
+ # TAAAAATATT TGTCGCTACC
+ # TAAAAATATT CTGTCGCTAC
+ # TAAAAATATT CCTGTCGCTA
+ # 012345678901234567890123456789012345678901234567890
+ # 0 1 2 3 4 5
+ # -----------------------------------
+ # 012345678901234567890123456789012345678901234567
+ # 0 1 2 3 4
+ mate1s => [ "ATAAAAATAT", "ATAAAAATAT", "ATAAAAATAT", "ATAAAAATAT",
+ "TAAAAATATT", "TAAAAATATT", "TAAAAATATT", "TAAAAATATT", ],
+ mate2s => [ "GTCGCTACCG", "TGTCGCTACC", "CTGTCGCTAC", "CCTGTCGCTA",
+ "GTCGCTACCG", "TGTCGCTACC", "CTGTCGCTAC", "CCTGTCGCTA" ],
+ mate1fw => 1, mate2fw => 1,
+ args => "-I 0 -X 35",
+ # Not really any way to flag an alignment as discordant
+ pairhits => [ { "3,30" => 1 }, { "3,29" => 1 }, { "3,28" => 1 }, { "3,27" => 1 },
+ { "4,30" => 1 }, { "4,29" => 1 }, { "4,28" => 1 }, { "4,27" => 1 } ],
+ flags => [ "XM:0,XP:0,XT:DP,XC:10=", "XM:0,XP:0,XT:DP,XC:10=",
+ "XM:0,XP:0,XT:CP,XC:10=", "XM:0,XP:0,XT:CP,XC:10=",
+ "XM:0,XP:0,XT:DP,XC:10=", "XM:0,XP:0,XT:CP,XC:10=",
+ "XM:0,XP:0,XT:CP,XC:10=", "XM:0,XP:0,XT:CP,XC:10=" ] },
+
+ # 1 discordant alignment and one concordant alignment. Discordant because
+ # the fragment is too long.
+
+ { name => "Simple paired-end 12",
+ ref => [ "TTTATAAAAATATTTCCCCCCCCCCCCCCGGGCCCGCCCGCCCCCCCCCCC" ],
+ # ATAAAAATAT GGCCCGCCCG
+ # ATAAAAATAT CCGGGCCCGC
+ # 012345678901234567890123456789012345678901234567890
+ # 0 1 2 3 4 5
+ # -------------------------------------
+ # 012345678901234567890123456789012345678901234567
+ mate1s => [ "ATAAAAATAT", "ATAAAAATAT" ],
+ mate2s => [ "GGCCCGCCCG", "CCGGGCCCGC" ],
+ mate1fw => 1, mate2fw => 1,
+ args => "-I 0 -X 36",
+ # Not really any way to flag an alignment as discordant
+ pairhits => [ { "3,30" => 1 }, { "3,27" => 1 } ],
+ flags => [ "XM:0,XP:0,XT:DP,XC:10=", "XM:0,XP:0,XT:CP,XC:10=" ] },
+
+ # 1 discordant alignment. Discordant because the fragment is too long.
+
+ { name => "Simple paired-end 11",
+ ref => [ "TTTATAAAAATATTTCCCCCCCCCCCCCCCCGATCGCCCGCCCCCCCCCCC" ],
+ # ATAAAAATAT CGATCGCCCG
+ # 012345678901234567890123456789012345678901234567890
+ # 0 1 2 3 4 5
+ # -------------------------------------
+ # 012345678901234567890123456789012345678901234567
+ mate1s => [ "ATAAAAATAT" ],
+ mate2s => [ "CGATCGCCCG" ],
+ mate1fw => 1, mate2fw => 1,
+ args => "-I 0 -X 36",
+ # Not really any way to flag an alignment as discordant
+ pairhits => [ { "3,30" => 1 } ],
+ flags => [ "XM:0,XP:0,XT:DP,XC:10=" ] },
+
+ # 1 discordant alignment. Discordant because the fragment is too short.
+
+ { name => "Simple paired-end 10",
+ ref => [ "TTTATAAAAATATTTCCCCCCGATCGCCCGCCCCCCCCCCC" ],
+ # ATAAAAATAT CGATCGCCCG
+ # 01234567890123456789012345678901234567890
+ # 0 1 2 3 4
+ # ---------------------------
+ # 012345678901234567890123456
+ mate1s => [ "ATAAAAATAT" ],
+ mate2s => [ "CGATCGCCCG" ],
+ mate1fw => 1, mate2fw => 1,
+ args => "-I 28 -X 80",
+ # Not really any way to flag an alignment as discordant
+ pairhits => [ { "3,20" => 1 } ],
+ flags => [ "XM:0,XP:0,XT:DP,XC:10=" ] },
+
+ # Like 6, but with -M limit
+
+ { name => "Simple paired-end 9",
+ ref => [ "CCCATATATATATCCTCCCATATATATATCCCTCCCCATATATATATCCCTTTTCCTTTCGCGCGCGCGTTTCCCCCCCCC" ],
+ # ATATATATAT ATATATATAT ATATATATAT CGCGCGCGCG
+ # 012345678901234567890123456789012345678901234567890123456789012345678901234567890
+ # 0 1 2 3 4 5 6 7 8
+ mate1s => [ "ATATATATAT" ],
+ mate2s => [ "CGCGCGCGCG" ],
+ mate1fw => 1, mate2fw => 0,
+ args => "-I 0 -X 80",
+ report => "-M 2",
+ lines => 2,
+ pairhits => [ { "3,59" => 1, "19,59" => 1, "37,59" => 1 } ],
+ hits_are_superset => [ 1 ],
+ flags => [ "XM:1,XP:1,XT:CP,XC:10=", "XM:1,XP:1,XT:CP,XC:10=" ] },
+
+ # Like 6, but without -m limit
+
+ { name => "Simple paired-end 8",
+ ref => [ "CCCATATATATATCCTCCCATATATATATCCCTTCCCATATATATATCCCTTTTTTTTTCGCGCGCGCGTTTCCCCCCCCC" ],
+ # ATATATATAT ATATATATAT ATATATATAT CGCGCGCGCG
+ # 012345678901234567890123456789012345678901234567890123456789012345678901234567890
+ # 0 1 2 3 4 5 6 7 8
+ mate1s => [ "ATATATATAT" ],
+ mate2s => [ "CGCGCGCGCG" ],
+ mate1fw => 1, mate2fw => 0,
+ args => "-I 0 -X 80",
+ pairhits => [ { "3,59" => 1, "19,59" => 1, "37,59" => 1 } ],
+ flags => [ "XM:0,XP:0,XT:CP,XC:10=" ] },
+
+ # Paired-end read, but only one mate aligns
+
+ { name => "Simple paired-end 2; no --no-mixed",
+ ref => [ "CCCATATATATATCCCTTTTTTTCCCCCCCCCCTTCGCGCGCGCGTTTCCCCC" ],
+ # ATATATATAT CGCGCGCGCG
+ # 01234567890123456789012345678901234567890123456789012
+ # 0 1 2 3 4 5
+ mate1s => [ "ATATATATAT" ],
+ mate2s => [ "CCCCCGGGGG" ],
+ mate1fw => 1, mate2fw => 1,
+ args => "-I 0 -X 50 --nofw",
+ nofw => 1,
+ pairhits => [ { "*,3" => 1 } ],
+ flags_map => [ { 3 => "XM:0,XP:0,XT:UP,XC:10=",
+ "*" => "XM:0,XP:0,XT:UP" } ],
+ cigar_map => [{
+ 3 => "10M",
+ "*" => "*"
+ }],
+ samoptflags_map => [{
+ 3 => {
+ "MD:Z:10" => 1, # mismatching positions/bases
+ "YT:Z:UP" => 1, # type of alignment (concordant/discordant/etc)
+ },
+ "*" => {
+ "YT:Z:UP" => 1, # type of alignment (concordant/discordant/etc)
+ }
+ }]
+ },
+
+ { name => "Simple paired-end 2; --no-mixed",
+ ref => [ "CCCATATATATATCCCTTTTTTTCCCCCCCCTTTTCGCGCGCGCGTTTCCCCC" ],
+ # ATATATATAT CGCGCGCGCG
+ # 01234567890123456789012345678901234567890123456789012
+ # 0 1 2 3 4 5
+ mate1s => [ "ATATATATAT" ],
+ mate2s => [ "CCCCCGGGGG" ],
+ mate1fw => 1, mate2fw => 1,
+ args => "-I 0 -X 50 --no-mixed",
+ pairhits => [ { "*,*" => 1 } ] },
+
+ # Simple paired-end alignment
+
+ { name => "Simple paired-end 1",
+ ref => [ "CCCATATATATATCCCTTTTTTTCCCCCCCCTTTTCGCGCGCGCGTTTTCCCC" ],
+ # ATATATATAT CGCGCGCGCG
+ # 01234567890123456789012345678901234567890123456789012
+ # 0 1 2 3 4 5
+ mate1s => [ "ATATATATAT" ],
+ mate2s => [ "CGCGCGCGCG" ],
+ mate1fw => 1, mate2fw => 1,
+ args => "-I 0 -X 50",
+ pairhits => [ { "3,35" => 1 } ],
+ flags => [ "XM:0,XP:0,XT:CP,XC:10=" ],
+ cigar_map => [{
+ 3 => "10M",
+ 35 => "10M"
+ }],
+ samoptflags_map => [{
+ 3 => {
+ "MD:Z:10" => 1, # mismatching positions/bases
+ "YT:Z:CP" => 1, # type of alignment (concordant/discordant/etc)
+ },
+ 35 => {
+ "MD:Z:10" => 1, # mismatching positions/bases
+ "YT:Z:CP" => 1, # type of alignment (concordant/discordant/etc)
+ }
+ }]
+ },
+
+ # Check that pseudo-random generation is always the same for
+ # same-sequence, same-name reads
+
+ { ref => [ "AAAAAAAAAAAAAAAAAAAAAAAAACCCCCCCCCCCCCCCCCCCCCCCC" ],
+ reads => [ "AA", "AA", "AA", "AA", "CC", "CC", "CC", "CC", "AA", "AA", "AA", "AA", "CC", "CC", "CC", "CC" ],
+ names => [ "r1", "r1", "r1", "r1", "r2", "r2", "r2", "r2", "r3", "r3", "r3", "r3", "r4", "r4", "r4", "r4" ],
+ args => "",
+ check_random => 1,
+ report => "-k 1" },
+
+ { ref => [ "TTGTTCGTTTGTTCGT" ],
+ reads => [ "TTGTTCGT" ],
+ report => "-M 1",
+ hits => [ { 0 => 1, 8 => 1 } ],
+ flags => [ "XM:1,XP:0,XT:UU,XC:8=" ],
+ hits_are_superset => [ 1 ],
+ cigar => [ "8M" ],
+ samoptflags => [
+ { "YM:i:1" => 1, "YT:Z:UU" => 1, "MD:Z:8" => 1, "YM:i:1" => 1 }
+ ],
+ },
+
+ # Read 3 overhangs right end
+ { ref => [ "TTGTTCGT" ],
+ reads => [ "GTTCGTA" ],
+ args => "--overhang --policy \"SEED=0,3\\;IVAL=C,1,0\\;NCEIL=L,2,0\"",
+ hits => [ { 2 => 1 } ],
+ flags => [ "XM:0,XP:0,XT:UU,XC:6=1X" ] },
+
+ # Mess with arguments
+
+ # Default should be 1-mismatch, so this shouldn't align
+ { ref => [ "TTGTTCGTTTGTTCGT" ],
+ reads => [ "TTATTAGT" ],
+ args => "",
+ hits => [ { "*" => 1 } ],
+ flags => [ "XM:0,XP:0,XT:UU" ],
+ cigar => [ "*" ],
+ samoptflags => [{ "YT:Z:UU" => 1 }],
+ },
+
+ # Shouldn't align with 0 mismatches either
+ { ref => [ "TTGTTCGTTTGTTCGT" ],
+ reads => [ "TTATTAGT" ],
+ args => "--policy SEED=0",
+ hits => [ { "*" => 1 } ],
+ flags => [ "XM:0,XP:0,XT:UU" ],
+ cigar => [ "*" ],
+ samoptflags => [{ "YT:Z:UU" => 1 }],
+ },
+
+ # Should align with 0 mismatches if we can wedge a seed into the 2
+ # matching characters between the two mismatches. Here we fail to
+ # wedge a length-3 seed in (there's no room)
+ { ref => [ "TTGTTCGTTTGTTCGT" ],
+ reads => [ "TTATTAGT" ],
+ args => "--policy \"SEED=0,3\\;IVAL=C,1,0\\;MMP=C1\"",
+ hits => [ { "*" => 1 } ],
+ flags => [ "XM:0,XP:0,XT:UU" ],
+ cigar => [ "*" ],
+ samoptflags => [{ "YT:Z:UU" => 1 }],
+ },
+
+ # Should align with 0 mismatches if we can wedge a seed into the 2
+ # matching characters between the two mismatches. Here we wedge a
+ # length-2 seed in
+ { ref => [ "TTGTTCGTTTGTTCGT" ],
+ reads => [ "TTATTAGT" ],
+ args => "--policy \"SEED=0,2\\;IVAL=C,1,0\\;MMP=C1\"",
+ #
+ # TTGTTCGTTTGTTCGT TTGTTCGTTTGTTCGT TTGTTCGTTTGTTCGT
+ # || || || || | | || ||
+ # TTATTAGT TTATTAGT TTATTAGT
+ #
+ # TTGTTCGTTTGTTCGT TTGTTCGTTTGTTCGT TTGTTCGTTTGTTCGT
+ # || | || | || || ||
+ # TTATTAGT TTATTAGT TTATTAGT
+ #
+ hits => [ { 0 => 1, 3 => 1, 4 => 1,
+ 5 => 1, 7 => 1, 8 => 1} ],
+ flag_map => [ { 0 => "XM:0,XP:0,XT:UU,XC:2=1X2=1X2=",
+ 3 => "XM:0,XP:0,XT:UU,XC:2=2X1=3X",
+ 4 => "XM:0,XP:0,XT:UU,XC:1=2X2=1X2=",
+ 5 => "XM:0,XP:0,XT:UU,XC:3X2=2X1=",
+ 7 => "XM:0,XP:0,XT:UU,XC:2=2X1=3X",
+ 8 => "XM:0,XP:0,XT:UU,XC:2=1X2=1X2="} ],
+ cigar_map => [ { 0 => "8M", 3 => "8M", 4 => "8M",
+ 5 => "8M", 7 => "8M", 8 => "8M" } ],
+ samoptflags_map => [{
+ 0 => { "AS:i:-2" => 1, "XS:i:-2" => 1, "NM:i:2" => 1, "XM:i:2" => 1,
+ "YT:Z:UU" => 1, "MD:Z:2G2C2" => 1 },
+ 3 => { "AS:i:-5" => 1, "XS:i:-2" => 1, "NM:i:5" => 1, "XM:i:5" => 1,
+ "YT:Z:UU" => 1, "MD:Z:2C0G1T0T0G0" => 1 },
+ 4 => { "AS:i:-3" => 1, "XS:i:-2" => 1, "NM:i:3" => 1, "XM:i:3" => 1,
+ "YT:Z:UU" => 1, "MD:Z:1C0G2T2" => 1 },
+ 5 => { "AS:i:-5" => 1, "XS:i:-2" => 1, "NM:i:5" => 1, "XM:i:5" => 1,
+ "YT:Z:UU" => 1, "MD:Z:0C0G0T2G0T1" => 1 },
+ 7 => { "AS:i:-5" => 1, "XS:i:-2" => 1, "NM:i:5" => 1, "XM:i:5" => 1,
+ "YT:Z:UU" => 1, "MD:Z:2T0G1T0C0G0" => 1 },
+ 8 => { "AS:i:-2" => 1, "XS:i:-2" => 1, "NM:i:2" => 1, "XM:i:2" => 1,
+ "YT:Z:UU" => 1, "MD:Z:2G2C2" => 1 },
+ }],
+ },
+);
+
+##
+# Take a list of reference sequences and write them to a temporary
+# FASTA file of the given name.
+#
+sub writeFasta($$) {
+ my ($l, $fa) = @_;
+ open(FA, ">$fa") || die "Could not open $fa for writing";
+ my $idx = 0;
+ for(my $i = 0; $i < scalar(@$l); $i++) {
+ print FA ">$idx\n".$l->[$i]."\n";
+ $idx++;
+ }
+ close(FA);
+}
+
+##
+# Take a lists of named reads/mates and write them to appropriate
+# files.
+#
+sub writeReads($$$$$$$$$) {
+ my (
+ $reads,
+ $quals,
+ $mate1s,
+ $qual1s,
+ $mate2s,
+ $qual2s,
+ $names,
+ $fq1,
+ $fq2) = @_;
+
+ open(FQ1, ">$fq1") || die "Could not open '$fq1' for writing";
+ open(FQ2, ">$fq2") || die "Could not open '$fq2' for writing";
+ my $pe = (defined($mate1s) && $mate1s ne "");
+ if($pe) {
+ for (0..scalar(@$mate1s)-1) {
+ my $m1 = $mate1s->[$_];
+ my $m2 = $mate2s->[$_];
+ my $q1 = $qual1s->[$_];
+ my $q2 = $qual2s->[$_];
+ my $nm = $names->[$_];
+ defined($m1) || die;
+ defined($m2) || die;
+ $q1 = $q1 || ("I" x length($m1));
+ $q2 = $q2 || ("I" x length($m2));
+ $nm = $nm || "r$_";
+ print FQ1 "\@$nm/1\n$m1\n+\n$q1\n";
+ print FQ2 "\@$nm/2\n$m2\n+\n$q2\n";
+ }
+ } else {
+ for (0..scalar(@$reads)-1) {
+ my $read = $reads->[$_];
+ defined($read) || die;
+ my $qual = $quals->[$_];
+ my $nm = $names->[$_];
+ $qual = $qual || ("I" x length($read));
+ $nm = $nm || "r$_";
+ print FQ1 "\@$nm\n$read\n+\n$qual\n";
+ }
+ }
+ close(FQ1);
+ close(FQ2);
+}
+
+##
+# Run bowtie2 with given arguments
+#
+sub runbowtie2($$$$$$$$$$$$$$$$$$$$$$) {
+
+ my (
+ $do_build,
+ $large_idx,
+ $debug_mode,
+ $args,
+ $fa,
+ $reportargs, #5
+ $read_file_format,
+ $read_file,
+ $mate1_file,
+ $mate2_file,
+ $reads,
+ $quals,
+ $mate1s,
+ $qual1s,
+ $mate2s,
+ $qual2s,
+ $names,
+ $ls,
+ $rawls,
+ $header_ls,
+ $raw_header_ls,
+ $should_abort) = @_;
+
+my $idx_type = "";
+ $args .= " --quiet";
+ $reportargs = "-a" unless defined($reportargs);
+ $args .= " $reportargs";
+ if ($large_idx){
+ $idx_type = "--large-index";
+ }
+ # Write the reference to a fasta file
+ print "References:\n";
+ open(FA, $fa) || die;
+ while(<FA>) { print $_; }
+ close(FA);
+ if($do_build) {
+ my $build_args = "";
+ my $cmd = "$bowtie2_build $idx_type --quiet --sanity $build_args $fa .simple_tests.tmp";
+ print "$cmd\n";
+ system($cmd);
+ ($? == 0) || die "Bad exitlevel from bowtie2-build: $?";
+ }
+ my $pe = (defined($mate1s) && $mate1s ne "");
+ $pe = $pe || (defined($mate1_file));
+ my $mate1arg;
+ my $mate2arg;
+ my $readarg;
+ my $formatarg = "-c";
+ my ($readstr, $m1str, $m2str) = (undef, undef, undef);
+ $readstr = join(",", @$reads) if defined($reads);
+ $m1str = join(",", @$mate1s) if defined($mate1s);
+ $m2str = join(",", @$mate2s) if defined($mate2s);
+ if(defined($read_file) || defined($mate1_file)) {
+ defined($read_file_format) || die;
+ my $ext = "";
+ if($read_file_format eq "fastq") {
+ $formatarg = "-q";
+ $ext = ".fq";
+ } elsif($read_file_format eq "tabbed") {
+ $formatarg = "--12";
+ $ext = ".tab";
+ } elsif($read_file_format eq "cline_reads") {
+ $formatarg = "-c";
+ $readarg = $read_file;
+ $mate1arg = $mate1_file;
+ $mate2arg = $mate2_file;
+ } elsif($read_file_format eq "cont_fasta_reads") {
+ $formatarg = "";
+ $readarg = $read_file;
+ $mate1arg = $mate1_file;
+ $mate2arg = $mate2_file;
+ } elsif($read_file_format eq "fasta") {
+ $formatarg = "-f";
+ $ext = ".fa";
+ } elsif($read_file_format eq "qseq") {
+ $formatarg = "--qseq";
+ $ext = "_qseq.txt";
+ } elsif($read_file_format eq "raw") {
+ $formatarg = "-r";
+ $ext = ".raw";
+ } else {
+ die "Bad format: $read_file_format";
+ }
+ if($formatarg ne "-c") {
+ if(defined($read_file)) {
+ # Unpaired
+ open(RD, ">.simple_tests$ext") || die;
+ print RD $read_file;
+ close(RD);
+ $readarg = ".simple_tests$ext";
+ } else {
+ defined($mate1_file) || die;
+ defined($mate2_file) || die;
+ # Paired
+ open(M1, ">.simple_tests.1$ext") || die;
+ print M1 $mate1_file;
+ close(M1);
+ open(M2, ">.simple_tests.2$ext") || die;
+ print M2 $mate2_file;
+ close(M2);
+ $mate1arg = ".simple_tests.1$ext";
+ $mate2arg = ".simple_tests.2$ext";
+ }
+ }
+ } else {
+ writeReads(
+ $reads,
+ $quals,
+ $mate1s,
+ $qual1s,
+ $mate2s,
+ $qual2s,
+ $names,
+ ".simple_tests.1.fq",
+ ".simple_tests.2.fq");
+ $mate1arg = ".simple_tests.1.fq";
+ $mate2arg = ".simple_tests.2.fq";
+ $formatarg = "-q";
+ $readarg = $mate1arg;
+ }
+ # Possibly add debug mode string
+ my $debug_arg = "";
+ $debug_arg = "--debug" if $debug_mode;
+ my $cmd;
+ if($pe) {
+ # Paired-end case
+ $cmd = "$bowtie2 $debug_arg $idx_type $args -x .simple_tests.tmp $formatarg -1 $mate1arg -2 $mate2arg";
+ } else {
+ # Unpaired case
+ $cmd = "$bowtie2 $debug_arg $idx_type $args -x .simple_tests.tmp $formatarg $readarg";
+ }
+ print "$cmd\n";
+ open(BT, "$cmd |") || die "Could not open pipe '$cmd |'";
+ while(<BT>) {
+ print $_;
+ chomp;
+ if(substr($_, 0, 1) eq "@") {
+ push @$header_ls, [ split(/\t/, $_, -1) ];
+ push @$raw_header_ls, $_;
+ } else {
+ push @$ls, [ split(/\t/, $_, -1) ];
+ push @$rawls, $_;
+ }
+ }
+ close(BT);
+ ($? == 0 || $should_abort) || die "bowtie2 aborted with exitlevel $?\n";
+ ($? != 0 || !$should_abort) || die "bowtie2 failed to abort!\n";
+}
+
+##
+# Compare a hash ref of expected SAM flags with a hash ref of observed SAM
+# flags.
+#
+sub matchSamOptionalFlags($$) {
+ my ($flags, $ex_flags) = @_;
+ my %ex = ();
+ for(keys %$ex_flags) {
+ my ($nm, $ty, $vl) = split(/:/, $_);
+ defined($vl) || die "Could not parse optional flag field \"$_\"";
+ ($ex{$nm}{ty}, $ex{$nm}{vl}) = ($ty, $vl);
+ }
+ for(keys %$flags) {
+ my ($ex_ty, $ex_vl);
+ if(defined($ex{$_})) {
+ ($ex_ty, $ex_vl) = ($ex{$_}{ty}, $ex{$_}{vl});
+ } else {
+ ($ex_ty, $ex_vl) = ("i", "0");
+ }
+ defined($ex_ty) || die;
+ defined($ex_vl) || die;
+ my ($ty, $vl) = ($flags->{$_}{ty}, $flags->{$_}{vl});
+ defined($ty) || die;
+ defined($vl) || die;
+ $ex_ty eq $ty ||
+ die "Expected SAM optional flag $_ to have type $ex_ty, had $ty";
+ $ex_vl eq $vl ||
+ die "Expected SAM optional flag $_ to have value $ex_vl, had $vl";
+ }
+ return 1;
+}
+
+my $tmpfafn = ".simple_tests.pl.fa";
+my $last_ref = undef;
+foreach my $large_idx (undef,1) {
+ foreach my $debug_mode (undef,1) {
+ for (my $ci = 0; $ci < scalar(@cases); $ci++) {
+ my $c = $cases[$ci];
+ last unless defined($c);
+ # If there's any skipping of cases to be done, do it here prior to the
+ # eq_deeply check
+ my $do_build = 0;
+ unless(defined($last_ref) && eq_deeply($c->{ref}, $last_ref)) {
+ writeFasta($c->{ref}, $tmpfafn);
+ $do_build = 1;
+ }
+ $last_ref = $c->{ref};
+ # For each set of arguments...
+ my $case_args = $c->{args};
+ $case_args = "" unless defined($case_args);
+ my $first = 1; # did we build the index yet?
+ # Forward, then reverse-complemented
+ my $fwlo = ($c->{nofw} ? 1 : 0);
+ my $fwhi = ($c->{norc} ? 0 : 1);
+ for(my $fwi = $fwlo; $fwi <= $fwhi; $fwi++) {
+ my $fw = ($fwi == 0);
+ my $sam = 1;
+
+ my $reads = $c->{reads};
+ my $quals = $c->{quals};
+ my $m1s = $c->{mate1s};
+ my $q1s = $c->{qual1s};
+ my $m2s = $c->{mate2s};
+ my $q2s = $c->{qual2s};
+
+ my $read_file = undef;
+ my $mate1_file = undef;
+ my $mate2_file = undef;
+
+ $read_file = $c->{fastq} if defined($c->{fastq});
+ $read_file = $c->{tabbed} if defined($c->{tabbed});
+ $read_file = $c->{fasta} if defined($c->{fasta});
+ $read_file = $c->{qseq} if defined($c->{qseq});
+ $read_file = $c->{raw} if defined($c->{raw});
+ $read_file = $c->{cline_reads} if defined($c->{cline_reads});
+ $read_file = $c->{cont_fasta_reads} if defined($c->{cont_fasta_reads});
+
+ $mate1_file = $c->{fastq1} if defined($c->{fastq1});
+ $mate1_file = $c->{tabbed1} if defined($c->{tabbed1});
+ $mate1_file = $c->{fasta1} if defined($c->{fasta1});
+ $mate1_file = $c->{qseq1} if defined($c->{qseq1});
+ $mate1_file = $c->{raw1} if defined($c->{raw1});
+ $mate1_file = $c->{cline_reads1} if defined($c->{cline_reads1});
+ $mate1_file = $c->{cont_fasta_reads1} if defined($c->{cont_fasta_reads1});
+
+ $mate2_file = $c->{fastq2} if defined($c->{fastq2});
+ $mate2_file = $c->{tabbed2} if defined($c->{tabbed2});
+ $mate2_file = $c->{fasta2} if defined($c->{fasta2});
+ $mate2_file = $c->{qseq2} if defined($c->{qseq2});
+ $mate2_file = $c->{raw2} if defined($c->{raw2});
+ $mate2_file = $c->{cline_reads2} if defined($c->{cline_reads2});
+ $mate2_file = $c->{cont_fasta_reads2} if defined($c->{cont_fasta_reads2});
+
+ my $read_file_format = undef;
+ if(!defined($reads) && !defined($m1s) && !defined($m2s)) {
+ defined($read_file) || defined($mate1_file) || die;
+ $read_file_format = "fastq" if defined($c->{fastq}) || defined($c->{fastq1});
+ $read_file_format = "tabbed" if defined($c->{tabbed}) || defined($c->{tabbed});
+ $read_file_format = "fasta" if defined($c->{fasta}) || defined($c->{fasta1});
+ $read_file_format = "qseq" if defined($c->{qseq}) || defined($c->{qseq1});
+ $read_file_format = "raw" if defined($c->{raw}) || defined($c->{raw1});
+ $read_file_format = "cline_reads" if defined($c->{cline_reads}) || defined($c->{cline_reads1});
+ $read_file_format = "cont_fasta_reads" if defined($c->{cont_fasta_reads}) || defined($c->{cont_fasta_reads1});
+ next unless $fw;
+ }
+ # Run bowtie2
+ my @lines = ();
+ my @rawlines = ();
+ my @header_lines = ();
+ my @header_rawlines = ();
+ print $c->{name}." " if defined($c->{name});
+ print "(fw:".($fw ? 1 : 0).", sam:$sam)\n";
+ my $mate1fw = 1;
+ my $mate2fw = 0;
+ $mate1fw = $c->{mate1fw} if defined($c->{mate1fw});
+ $mate2fw = $c->{mate2fw} if defined($c->{mate2fw});
+ if(!$fw) {
+ # Reverse-complement the reads
+ my @s = (); @s = @$reads if defined($reads);
+ my @q = (); @q = @$quals if defined($quals);
+ # Reverse-complement mates and switch mate1 with mate2
+ my @m1 = (); @m1 = @$m1s if defined($m1s);
+ my @m2 = (); @m2 = @$m2s if defined($m2s);
+ my @q1 = (); @q1 = @$q1s if defined($q1s);
+ my @q2 = (); @q2 = @$q2s if defined($q2s);
+ for(0..scalar(@s)-1) {
+ $s[$_] = DNA::revcomp($s[$_]);
+ $q[$_] = reverse $q[$_] if $_ < scalar(@q);
+ }
+ if($mate1fw == $mate2fw) {
+ for(0..$#m1) { $m1[$_] = DNA::revcomp($m1[$_]); }
+ for(0..$#m2) { $m2[$_] = DNA::revcomp($m2[$_]); }
+ for(0..$#q1) { $q1[$_] = reverse $q1[$_]; }
+ for(0..$#q2) { $q2[$_] = reverse $q2[$_]; }
+ }
+ $reads = \@s if defined($reads);
+ $quals = \@q if defined($quals);
+ $m1s = \@m2 if defined($m1s);
+ $q1s = \@q2 if defined($q1s);
+ $m2s = \@m1 if defined($m2s);
+ $q2s = \@q1 if defined($q2s);
+ }
+ my $a = $case_args;
+ if(defined($m2s)) {
+ $a .= " --";
+ $a .= ($mate1fw ? "f" : "r");
+ $a .= ($mate2fw ? "f" : "r");
+ }
+ runbowtie2(
+ $do_build && $first,
+ $large_idx,
+ $debug_mode,
+ "$a",
+ $tmpfafn,
+ $c->{report},
+ $read_file_format, # formate of read/mate files
+ $read_file, # read file
+ $mate1_file, # mate #1 file
+ $mate2_file, # mate #2 file
+ $reads, # read list
+ $quals, # quality list
+ $m1s, # mate #1 sequence list
+ $q1s, # mate #1 quality list
+ $m2s, # mate #2 sequence list
+ $q2s, # mate #2 quality list
+ $c->{names},
+ \@lines,
+ \@rawlines,
+ \@header_lines,
+ \@header_rawlines,
+ $c->{should_abort});
+ $first = 0;
+ my $pe = defined($c->{mate1s}) && $c->{mate1s} ne "";
+ $pe = $pe || defined($mate1_file);
+ $pe = $pe || $c->{paired};
+ my ($lastchr, $lastoff, $lastoff_orig) = ("", -1, -1);
+ # Keep temporary copies of hits and pairhits so that we can
+ # restore for the next orientation
+ my $hitstmp = [];
+ $hitstmp = clone($c->{hits}) if defined($c->{hits});
+ my $pairhitstmp = [];
+ $pairhitstmp = clone($c->{pairhits}) if defined($c->{pairhits});
+ my $pairhits_orig_tmp = [];
+ $pairhits_orig_tmp = clone($c->{pairhits_orig}) if defined($c->{pairhits_orig});
+ # Record map from already-seen read name, read sequence and
+ # quality to the place on the reference where it's reported.
+ # This allows us to check that the pseudo-random generator
+ # isn't mistakenly yielding different alignments for identical
+ # reads.
+ my %seenNameSeqQual = ();
+ if(defined($c->{lines})) {
+ my $l = scalar(@lines);
+ $l == $c->{lines} || die "Expected $c->{lines} lines, got $l";
+ }
+ for my $li (0 .. scalar(@lines)-1) {
+ my $l = $lines[$li];
+ my ($readname, $orient, $chr, $off_orig, $off, $seq, $qual, $mapq,
+ $oms, $editstr, $flagstr, $samflags, $cigar, $rnext, $pnext,
+ $tlen);
+ my %samoptflags = ();
+ if($sam) {
+ scalar(@$l) >= 11 ||
+ die "Bad number of fields; expected at least 11 got ".
+ scalar(@$l).":\n$rawlines[$li]\n";
+ ($readname, $samflags, $chr, $off) = @$l[0..3];
+ ($seq, $qual) = @$l[9..10];
+ $orient = ((($samflags >> 4) & 1) == 0) ? "+" : "-";
+ $mapq = $l->[4]; # mapping quality
+ $cigar = $l->[5]; # CIGAR string
+ $rnext = $l->[6]; # ref seq of next frag in template
+ $pnext = $l->[7]; # position of next frag in template
+ $tlen = $l->[8]; # template length
+ if($pnext == 0) { $pnext = "*"; } else { $pnext--; }
+ for(my $m = 11; $m < scalar(@$l); $m++) {
+ next if $l->[$m] eq "";
+ my ($nm, $ty, $vl) = split(/:/, $l->[$m]);
+ defined($vl) ||
+ die "Could not parse optional flag field $m: ".
+ "\"$l->[$m]\"";
+ $samoptflags{$nm}{ty} = $ty;
+ $samoptflags{$nm}{vl} = $vl;
+ }
+ if($off > 0) { $off--; }
+ else { $off = "*"; }
+ $off_orig = $off;
+ $off = "*" if $cigar eq "*";
+ } else {
+ scalar(@$l) == 9 ||
+ die "Bad number of fields; expected 9 got ".
+ scalar(@$l).":\n$rawlines[$li]\n";
+ ($readname, $orient, $chr, $off, $seq,
+ $qual, $oms, $editstr, $flagstr) = @$l;
+ $off_orig = $off;
+ }
+ if($c->{check_random}) {
+ my $rsqKey = "$readname\t$orient\t$seq\t$qual";
+ my $rsqVal = "$chr\t$off";
+ if(defined($seenNameSeqQual{$rsqKey})) {
+ $seenNameSeqQual{$rsqKey} eq $rsqVal ||
+ die "Two hits for read/seq/qual:\n$rsqKey\n".
+ "had different alignments:\n".
+ "$seenNameSeqQual{$rsqKey}\n$rsqVal\n";
+ }
+ $seenNameSeqQual{$rsqKey} = $rsqVal;
+ }
+ $readname ne "" || die "readname was blank:\n".Dumper($c);
+ my $rdi = $readname;
+ $rdi = substr($rdi, 1) if substr($rdi, 0, 1) eq "r";
+ my $mate = 0;
+ if($readname =~ /\//) {
+ ($rdi, $mate) = split(/\//, $readname);
+ defined($rdi) || die;
+ }
+ $rdi = $c->{idx_map}{$rdi} if defined($c->{idx_map}{$rdi});
+ $rdi ne "" || die "rdi was blank:\nreadname=$readname\n".Dumper($c);
+ if($rdi != int($rdi)) {
+ # Read name has non-numeric characters. Figure out
+ # what number it is by scanning the names list.
+ my $found = 0;
+ for(my $i = 0; $i < scalar(@{$c->{names}}); $i++) {
+ if($c->{names}->[$i] eq $readname) {
+ $rdi = $i;
+ $found = 1;
+ last;
+ }
+ }
+ $found || die "No specified name matched reported name $readname";
+ }
+ # Make simply-named copies of some portions of the test case
+ # 'hits'
+ my %hits = ();
+ %hits = %{$c->{hits}->[$rdi]} if
+ defined($c->{hits}->[$rdi]);
+ # 'flags'
+ my $flags = undef;
+ $flags = $c->{flags}->[$rdi] if
+ defined($c->{flags}->[$rdi]);
+ # 'samflags'
+ my $ex_samflags = undef;
+ $ex_samflags = $c->{ex_samflags}->[$rdi] if
+ defined($c->{ex_samflags}->[$rdi]);
+ # 'samflags_map'
+ my $ex_samflags_map = undef;
+ $ex_samflags_map = $c->{samflags_map}->[$rdi] if
+ defined($c->{samflags_map}->[$rdi]);
+ # 'samoptflags'
+ my $ex_samoptflags = undef;
+ $ex_samoptflags = $c->{samoptflags}->[$rdi] if
+ defined($c->{samoptflags}->[$rdi]);
+ # 'cigar'
+ my $ex_cigar = undef;
+ $ex_cigar = $c->{cigar}->[$rdi] if
+ defined($c->{cigar}->[$rdi]);
+ # 'cigar_map'
+ my $ex_cigar_map = undef;
+ $ex_cigar_map = $c->{cigar_map}->[$rdi] if
+ defined($c->{cigar_map}->[$rdi]);
+ # 'mapq_hi' - boolean indicating whether mapq is hi/lo
+ my $ex_mapq_hi = undef;
+ $ex_mapq_hi = $c->{mapq_hi}->[$rdi] if
+ defined($c->{mapq_hi}->[$rdi]);
+ # 'mapq'
+ my $ex_mapq = undef;
+ $ex_mapq = $c->{mapq}->[$rdi] if
+ defined($c->{mapq}->[$rdi]);
+ # 'mapq_map'
+ my $ex_mapq_map = undef;
+ $ex_mapq_map = $c->{mapq_map}->[$rdi] if
+ defined($c->{mapq_map}->[$rdi]);
+ # 'rnext_map'
+ my $ex_rnext_map = undef;
+ $ex_rnext_map = $c->{rnext_map}->[$rdi] if
+ defined($c->{rnext_map}) && defined($c->{rnext_map}->[$rdi]);
+ # 'pnext_map'
+ my $ex_pnext_map = undef;
+ $ex_pnext_map = $c->{pnext_map}->[$rdi] if
+ defined($c->{pnext_map}) && defined($c->{pnext_map}->[$rdi]);
+ # 'tlen_map'
+ my $ex_tlen_map = undef;
+ $ex_tlen_map = $c->{tlen_map}->[$rdi] if
+ defined($c->{tlen_map}) && defined($c->{tlen_map}->[$rdi]);
+ # 'flags_fw'
+ my $flags_fw = undef;
+ $flags_fw = $c->{flags_fw}->[$rdi] if
+ defined($c->{flags_fw}->[$rdi]);
+ # 'flags_rc'
+ my $flags_rc = undef;
+ $flags_rc = $c->{flags_rc}->[$rdi] if
+ defined($c->{flags_rc}->[$rdi]);
+ # 'pairhits'
+ my %pairhits = ();
+ %pairhits = %{$c->{pairhits}->[$rdi]} if
+ defined($c->{pairhits}->[$rdi]);
+ # 'pairhits_orig'
+ my %pairhits_orig = ();
+ %pairhits_orig = %{$c->{pairhits_orig}->[$rdi]} if
+ defined($c->{pairhits_orig}->[$rdi]);
+ # 'pairflags'
+ my %pairflags = ();
+ %pairflags = %{$c->{pairflags}->[$rdi]} if
+ defined($c->{pairflags}->[$rdi]);
+ # 'hits_are_superset'
+ my $hits_are_superset = 0;
+ $hits_are_superset = $c->{hits_are_superset}->[$rdi] if
+ defined($ci);
+ # edits
+ my $ex_edits = undef;
+ $ex_edits = $c->{edits}->[$rdi] if
+ defined($c->{edits}->[$rdi]);
+ if(!$sam) {
+ # Bowtie flags
+ if(defined($flags)) {
+ $flagstr eq $flags ||
+ die "Expected flags=\"$flags\", got \"$flagstr\"";
+ }
+ if(defined($flags_fw) && $fw) {
+ $flagstr eq $flags_fw ||
+ die "Expected flags=\"$flags_fw\", got \"$flagstr\"";
+ }
+ if(defined($flags_rc) && !$fw) {
+ $flagstr eq $flags_rc ||
+ die "Expected flags=\"$flags_rc\", got \"$flagstr\"";
+ }
+ if(defined($c->{flag_map})) {
+ if(defined($c->{flag_map}->[$rdi]->{$off})) {
+ $flagstr eq $c->{flag_map}->[$rdi]->{$off} ||
+ die "Expected flags=\"$c->{flag_map}->[$rdi]->{$off}\"".
+ " at offset $off, got \"$flagstr\"";
+ }
+ }
+ }
+ if($sam) {
+ # SAM flags
+ if(defined($ex_samflags)) {
+ $samflags eq $ex_samflags ||
+ die "Expected flags $ex_samflags, got $samflags";
+ }
+ if(defined($ex_samflags_map)) {
+ if(defined($c->{samflags_map}->[$rdi]->{$off})) {
+ my $ex = $c->{samflags_map}->[$rdi]->{$off};
+ $samflags eq $ex || die
+ "Expected FLAGS value $ex at offset $off, got $samflags"
+ } else {
+ die "Expected to see alignment with offset $off parsing samflags_map";
+ }
+ }
+ # CIGAR string
+ if(defined($ex_cigar)) {
+ $cigar eq $ex_cigar ||
+ die "Expected CIGAR string $ex_cigar, got $cigar";
+ }
+ if(defined($ex_cigar_map)) {
+ if(defined($c->{cigar_map}->[$rdi]->{$off})) {
+ my $ex = $c->{cigar_map}->[$rdi]->{$off};
+ $cigar eq $ex || die
+ "Expected CIGAR string $ex at offset $off, got $cigar"
+ } else {
+ die "Expected to see alignment with offset $off parsing cigar_map";
+ }
+ }
+ # MAPQ
+ if(defined($ex_mapq)) {
+ $mapq eq $ex_mapq ||
+ die "Expected MAPQ $ex_mapq, got $mapq";
+ }
+ if(defined($ex_mapq_map)) {
+ if(defined($c->{mapq_map}->[$rdi]->{$off})) {
+ my $ex = $c->{mapq_map}->[$rdi]->{$off};
+ $mapq eq $ex || die
+ "Expected MAPQ string $ex at offset $off, got $mapq"
+ } else {
+ die "Expected to see alignment with offset $off parsing mapq_map";
+ }
+ }
+ # MAPQ
+ if(defined($ex_mapq_hi)) {
+ if($ex_mapq_hi == 0) {
+ $mapq < 20 || die "Expected MAPQ < 20, got $mapq";
+ } else {
+ $mapq >= 20 || die "Expected MAPQ >= 20, got $mapq";
+ }
+ }
+ if(defined($ex_mapq_map)) {
+ if(defined($c->{mapq_map}->[$rdi]->{$off})) {
+ my $ex = $c->{mapq_map}->[$rdi]->{$off};
+ $mapq eq $ex || die
+ "Expected MAPQ string $ex at offset $off, got $mapq"
+ } else {
+ die "Expected to see alignment with offset $off parsing mapq_map";
+ }
+ }
+ # SAM optional flags
+ if(defined($ex_samoptflags)) {
+ matchSamOptionalFlags(\%samoptflags, $ex_samoptflags);
+ }
+ if(defined($c->{samoptflags_map})) {
+ if(defined($c->{samoptflags_map}->[$rdi]->{$off})) {
+ matchSamOptionalFlags(
+ \%samoptflags,
+ $c->{samoptflags_map}->[$rdi]->{$off});
+ } else {
+ die "Expected to see alignment with offset $off parsing samoptflags_map";
+ }
+ }
+ if(defined($c->{samoptflags_flagmap})) {
+ if(defined($c->{samoptflags_flagmap}->[$rdi]->{$samflags})) {
+ matchSamOptionalFlags(
+ \%samoptflags,
+ $c->{samoptflags_flagmap}->[$rdi]->{$samflags});
+ } else {
+ die "Expected to see alignment with flag $samflags parsing samoptflags_flagmap";
+ }
+ }
+ # RNEXT map
+ if(defined($c->{rnext_map})) {
+ if(defined($c->{rnext_map}->[$rdi]->{$off})) {
+ my $ex = $c->{rnext_map}->[$rdi]->{$off};
+ $rnext eq $ex || die
+ "Expected RNEXT '$ex' at offset $off, got '$rnext'"
+ } else {
+ die "Expected to see alignment with offset $off parsing rnext_map".Dumper($c);
+ }
+ }
+ # PNEXT map
+ if(defined($c->{pnext_map})) {
+ if(defined($c->{pnext_map}->[$rdi]->{$off})) {
+ my $ex = $c->{pnext_map}->[$rdi]->{$off};
+ $pnext eq $ex || die
+ "Expected PNEXT '$ex' at offset $off, got '$pnext'"
+ } else {
+ die "Expected to see alignment with offset $off parsing pnext_map";
+ }
+ }
+ # TLEN map
+ if(defined($c->{tlen_map})) {
+ if(defined($c->{tlen_map}->[$rdi]->{$off})) {
+ my $ex = $c->{tlen_map}->[$rdi]->{$off};
+ $tlen eq $ex || die
+ "Expected TLEN '$ex' at offset $off, got '$tlen'"
+ } else {
+ die "Expected to see alignment with offset $off parsing tlen_map";
+ }
+ }
+ }
+ if($pe && $lastchr ne "") {
+ my $offkey_orig = $lastoff.",".$off_orig;
+ $offkey_orig = $off_orig.",".$lastoff_orig if $off_orig eq "*";
+
+ my $offkey = $lastoff.",".$off;
+ $offkey = $off.",".$lastoff if $off eq "*";
+
+ if($lastoff ne "*" && $off ne "*") {
+ $offkey = min($lastoff, $off).",".max($lastoff, $off);
+ }
+ if(defined($c->{pairhits}->[$rdi])) {
+ defined($pairhits{$offkey}) ||
+ die "No such paired off as $offkey in pairhits list: ".Dumper(\%pairhits)."\n";
+ $c->{pairhits}->[$rdi]->{$offkey}--;
+ delete $c->{pairhits}->[$rdi]->{$offkey} if $c->{pairhits}->[$rdi]->{$offkey} == 0;
+ %pairhits = %{$c->{pairhits}->[$rdi]};
+ }
+ if(defined($c->{pairhits_orig}->[$rdi])) {
+ defined($pairhits_orig{$offkey_orig}) ||
+ die "No such paired off as $offkey in pairhits_orig list: ".Dumper(\%pairhits_orig)."\n";
+ $c->{pairhits_orig}->[$rdi]->{$offkey_orig}--;
+ delete $c->{pairhits_orig}->[$rdi]->{$offkey_orig} if $c->{pairhits_orig}->[$rdi]->{$offkey_orig} == 0;
+ %pairhits_orig = %{$c->{pairhits_orig}->[$rdi]};
+ }
+ ($lastchr, $lastoff, $lastoff_orig) = ("", -1, -1);
+ } elsif($pe) {
+ # Found an unpaired alignment from aligning a pair
+ my $foundSe =
+ defined($c->{pairhits}->[$rdi]) &&
+ $c->{pairhits}->[$rdi]->{$off};
+ if($foundSe) {
+ $c->{pairhits}->[$rdi]->{$off}--;
+ delete $c->{pairhits}->[$rdi]->{$off}
+ if $c->{pairhits}->[$rdi]->{$off} == 0;
+ %pairhits = %{$c->{pairhits}->[$rdi]};
+ } else {
+ ($lastchr, $lastoff) = ($chr, $off);
+ }
+ # Found an unpaired alignment from aligning a pair
+ $foundSe =
+ defined($c->{pairhits_orig}->[$rdi]) &&
+ $c->{pairhits_orig}->[$rdi]->{$off_orig};
+ if($foundSe) {
+ $c->{pairhits_orig}->[$rdi]->{$off_orig}--;
+ delete $c->{pairhits_orig}->[$rdi]->{$off_orig}
+ if $c->{pairhits_orig}->[$rdi]->{$off_orig} == 0;
+ %pairhits_orig = %{$c->{pairhits_orig}->[$rdi]};
+ } else {
+ ($lastchr, $lastoff, $lastoff_orig) = ($chr, $off, $off_orig);
+ }
+ } else {
+ if(defined($c->{hits}->[$rdi])) {
+ defined($hits{$off}) ||
+ die "No such off as $off in hits list: ".Dumper(\%hits)."\n";
+ $c->{hits}->[$rdi]->{$off}--;
+ delete $c->{hits}->[$rdi]->{$off} if $c->{hits}->[$rdi]->{$off} == 0;
+ %hits = %{$c->{hits}->[$rdi]};
+ }
+ }
+ if(!$sam && defined($ex_edits)) {
+ my $eds = $l->[7];
+ $eds eq $ex_edits ||
+ die "For edit string, expected \"$ex_edits\" got \"$eds\"\n";
+ }
+ }
+ # Go through all the per-read
+ my $klim = 0;
+ $klim = scalar(@{$c->{hits}}) if defined($c->{hits});
+ $klim = max($klim, scalar(@{$c->{pairhits}})) if defined($c->{pairhits});
+ for (my $k = 0; $k < $klim; $k++) {
+ # For each read
+ my %hits = %{$c->{hits}->[$k]} if defined($c->{hits}->[$k]);
+ my %pairhits = %{$c->{pairhits}->[$k]} if defined($c->{pairhits}->[$k]);
+ my %pairhits_orig = %{$c->{pairhits_orig}->[$k]} if defined($c->{pairhits_orig}->[$k]);
+ my $hits_are_superset = $c->{hits_are_superset}->[$k];
+ # Check if there are any hits left over
+ my $hitsLeft = scalar(keys %hits);
+ if($hitsLeft != 0 && !$hits_are_superset) {
+ print Dumper(\%hits);
+ die "Had $hitsLeft hit(s) left over at position $k";
+ }
+ my $pairhitsLeft = scalar(keys %pairhits);
+ if($pairhitsLeft != 0 && !$hits_are_superset) {
+ print Dumper(\%pairhits);
+ die "Had $pairhitsLeft hit(s) left over at position $k";
+ }
+ my $pairhits_orig_Left = scalar(keys %pairhits_orig);
+ if($pairhits_orig_Left != 0 && !$hits_are_superset) {
+ print Dumper(\%pairhits_orig);
+ die "Had $pairhits_orig_Left hit(s) left over at position $k";
+ }
+ }
+
+ $c->{hits} = $hitstmp;
+ $c->{pairhits} = $pairhitstmp;
+ $c->{pairhits_orig} = $pairhits_orig_tmp;
+ }
+ $last_ref = undef if $first;
+ }
+ }
+}
+print "PASSED\n";
diff --git a/scripts/test/simple_tests.sh b/scripts/test/simple_tests.sh
new file mode 100644
index 0000000..90b13f2
--- /dev/null
+++ b/scripts/test/simple_tests.sh
@@ -0,0 +1,34 @@
+#!/bin/sh
+
+#
+# Copyright 2011, Ben Langmead <langmea at cs.jhu.edu>
+#
+# This file is part of Bowtie 2.
+#
+# Bowtie 2 is free software: you can redistribute it and/or modify
+# it under the terms of the GNU General Public License as published by
+# the Free Software Foundation, either version 3 of the License, or
+# (at your option) any later version.
+#
+# Bowtie 2 is distributed in the hope that it will be useful,
+# but WITHOUT ANY WARRANTY; without even the implied warranty of
+# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
+# GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License
+# along with Bowtie 2. If not, see <http://www.gnu.org/licenses/>.
+#
+
+# simple_tests.sh
+
+make $* bowtie2-align-s \
+ bowtie2-align-l \
+ bowtie2-align-s-debug \
+ bowtie2-align-l-debug \
+ bowtie2-build-s \
+ bowtie2-build-l \
+ bowtie2-build-s-debug \
+ bowtie2-build-l-debug && \
+perl scripts/test/simple_tests.pl \
+ --bowtie2=./bowtie2 \
+ --bowtie2-build=./bowtie2-build
diff --git a/search_globals.h b/search_globals.h
index bd2704f..448c9c7 100644
--- a/search_globals.h
+++ b/search_globals.h
@@ -23,12 +23,7 @@
#include <stdint.h>
// declared in ebwt_search.cpp
-extern bool gColor;
-extern bool gColorExEnds;
extern bool gReportOverhangs;
-extern bool gColorSeq;
-extern bool gColorEdit;
-extern bool gColorQual;
extern bool gNoMaqRound;
extern bool gStrandFix;
extern bool gRangeMode;
diff --git a/simple_func.h b/simple_func.h
index ca76869..4d7ff78 100644
--- a/simple_func.h
+++ b/simple_func.h
@@ -82,6 +82,10 @@ public:
bool initialized() const { return type_ != 0; }
void reset() { type_ = 0; }
+ bool alwaysPositive() const {
+ return f<int>(1.0) > 0 && (SIMPLE_FUNC_CONST || L_ >= 0.0);
+ }
+
template<typename T>
T f(double x) const {
assert(type_ >= SIMPLE_FUNC_CONST && type_ <= SIMPLE_FUNC_LOG);
diff --git a/sstring.cpp b/sstring.cpp
index 3b26587..8569768 100644
--- a/sstring.cpp
+++ b/sstring.cpp
@@ -115,9 +115,9 @@ int main(void) {
assert(sstr_eq(s, s2));
if(i >= 10) {
BTDnaString s3;
- s.windowGetDna(s3, true, false, 3, 4);
+ s.windowGetDna(s3, true, 3, 4);
assert(sstr_eq(s3.toZBuf(), (const char*)"TACG"));
- s.windowGetDna(s3, false, false, 3, 4);
+ s.windowGetDna(s3, false, 3, 4);
assert(sstr_eq(s3.toZBuf(), (const char*)"CGTA"));
assert_eq('A', s.toChar(0));
assert_eq('G', s.toChar(2));
diff --git a/sstring.h b/sstring.h
index c06bbc6..403b843 100644
--- a/sstring.h
+++ b/sstring.h
@@ -1520,7 +1520,6 @@ public:
char windowGetDna(
size_t i,
bool fw,
- bool color,
size_t depth = 0,
size_t len = 0) const
{
@@ -1530,10 +1529,7 @@ public:
if(fw) {
return get(depth+i);
} else {
- return
- color ?
- get(depth+len-i-1) :
- compDna(get(depth+len-i-1));
+ return compDna(get(depth+len-i-1));
}
}
@@ -1545,7 +1541,6 @@ public:
void windowGetDna(
T& buf,
bool fw,
- bool color,
size_t depth = 0,
size_t len = 0) const
{
@@ -1556,9 +1551,7 @@ public:
buf.set(
(fw ?
get(depth+i) :
- (color ?
- get(depth+len-i-1) :
- compDna(get(depth+len-i-1)))), i);
+ compDna(get(depth+len-i-1))), i);
}
}
@@ -1619,7 +1612,7 @@ protected:
/**
* Simple string class with backing memory that automatically expands as needed.
*/
-template<typename T, int S = 1024, int M = 2>
+template<typename T, int S = 1024, int M = 2, int I = 0>
class SStringExpandable {
public:
@@ -1629,7 +1622,11 @@ public:
printcs_(NULL),
len_(0),
sz_(0)
- { }
+ {
+ if(I > 0) {
+ expandNoCopy(I);
+ }
+ }
explicit SStringExpandable(size_t sz) :
cs_(NULL),
@@ -1738,9 +1735,9 @@ public:
}
/**
- * Assignment to other SStringFixed.
+ * Assignment to other SStringExpandable.
*/
- SStringExpandable<T,S>& operator=(const SStringExpandable<T,S>& o) {
+ SStringExpandable<T,S,M,I>& operator=(const SStringExpandable<T,S,M,I>& o) {
install(o.cs_, o.len_);
return *this;
}
@@ -1984,9 +1981,14 @@ public:
/**
* Trim len characters from the end of the string.
*/
- void trimEnd(size_t len) {
- if(len >= len_) len_ = 0;
- else len_ -= len;
+ size_t trimEnd(size_t len) {
+ if(len >= len_) {
+ size_t ret = len_;
+ len_ = 0;
+ return ret;
+ }
+ len_ -= len;
+ return len;
}
/**
@@ -2631,22 +2633,18 @@ public:
* Either reverse or reverse-complement (depending on "color") this
* DNA buffer in-place.
*/
- void reverseComp(bool color = false) {
- if(color) {
- this->reverse();
- } else {
- for(size_t i = 0; i < (this->len_ >> 1); i++) {
- char tmp1 = (this->cs_[i] == 4 ? 4 : this->cs_[i] ^ 3);
- char tmp2 = (this->cs_[this->len_-i-1] == 4 ? 4 : this->cs_[this->len_-i-1] ^ 3);
- this->cs_[i] = tmp2;
- this->cs_[this->len_-i-1] = tmp1;
- }
- // Do middle element iff there are an odd number
- if((this->len_ & 1) != 0) {
- char tmp = this->cs_[this->len_ >> 1];
- tmp = (tmp == 4 ? 4 : tmp ^ 3);
- this->cs_[this->len_ >> 1] = tmp;
- }
+ void reverseComp() {
+ for(size_t i = 0; i < (this->len_ >> 1); i++) {
+ char tmp1 = (this->cs_[i] == 4 ? 4 : this->cs_[i] ^ 3);
+ char tmp2 = (this->cs_[this->len_-i-1] == 4 ? 4 : this->cs_[this->len_-i-1] ^ 3);
+ this->cs_[i] = tmp2;
+ this->cs_[this->len_-i-1] = tmp1;
+ }
+ // Do middle element iff there are an odd number
+ if((this->len_ & 1) != 0) {
+ char tmp = this->cs_[this->len_ >> 1];
+ tmp = (tmp == 4 ? 4 : tmp ^ 3);
+ this->cs_[this->len_ >> 1] = tmp;
}
}
@@ -2782,7 +2780,6 @@ public:
char windowGetDna(
size_t i,
bool fw,
- bool color,
size_t depth = 0,
size_t len = 0) const
{
@@ -2790,8 +2787,7 @@ public:
assert_lt(i, len);
assert_leq(len, this->len_ - depth);
if(fw) return this->cs_[depth+i];
- else return color ? this->cs_[depth+len-i-1] :
- compDna(this->cs_[depth+len-i-1]);
+ else return compDna(this->cs_[depth+len-i-1]);
}
/**
@@ -2801,7 +2797,6 @@ public:
void windowGetDna(
SDnaStringFixed<S>& buf,
bool fw,
- bool color,
size_t depth = 0,
size_t len = 0) const
{
@@ -2809,8 +2804,7 @@ public:
assert_leq(len, this->len_ - depth);
for(size_t i = 0; i < len; i++) {
buf.append(fw ? this->cs_[depth+i] :
- (color ? this->cs_[depth+len-i-1] :
- compDna(this->cs_[depth+len-i-1])));
+ compDna(this->cs_[depth+len-i-1]));
}
}
@@ -2924,22 +2918,18 @@ public:
* Either reverse or reverse-complement (depending on "color") this
* DNA buffer in-place.
*/
- void reverseComp(bool color = false) {
- if(color) {
- this->reverse();
- } else {
- for(size_t i = 0; i < (this->len_ >> 1); i++) {
- char tmp1 = (this->cs_[i] == 4 ? 4 : this->cs_[i] ^ 3);
- char tmp2 = (this->cs_[this->len_-i-1] == 4 ? 4 : this->cs_[this->len_-i-1] ^ 3);
- this->cs_[i] = tmp2;
- this->cs_[this->len_-i-1] = tmp1;
- }
- // Do middle element iff there are an odd number
- if((this->len_ & 1) != 0) {
- char tmp = this->cs_[this->len_ >> 1];
- tmp = (tmp == 4 ? 4 : tmp ^ 3);
- this->cs_[this->len_ >> 1] = tmp;
- }
+ void reverseComp() {
+ for(size_t i = 0; i < (this->len_ >> 1); i++) {
+ char tmp1 = (this->cs_[i] == 4 ? 4 : this->cs_[i] ^ 3);
+ char tmp2 = (this->cs_[this->len_-i-1] == 4 ? 4 : this->cs_[this->len_-i-1] ^ 3);
+ this->cs_[i] = tmp2;
+ this->cs_[this->len_-i-1] = tmp1;
+ }
+ // Do middle element iff there are an odd number
+ if((this->len_ & 1) != 0) {
+ char tmp = this->cs_[this->len_ >> 1];
+ tmp = (tmp == 4 ? 4 : tmp ^ 3);
+ this->cs_[this->len_ >> 1] = tmp;
}
}
@@ -3091,7 +3081,6 @@ public:
char windowGetDna(
size_t i,
bool fw,
- bool color,
size_t depth = 0,
size_t len = 0) const
{
@@ -3099,8 +3088,7 @@ public:
assert_lt(i, len);
assert_leq(len, this->len_ - depth);
if(fw) return this->cs_[depth+i];
- else return color ? this->cs_[depth+len-i-1] :
- compDna(this->cs_[depth+len-i-1]);
+ else return compDna(this->cs_[depth+len-i-1]);
}
/**
@@ -3110,7 +3098,6 @@ public:
void windowGetDna(
SDnaStringExpandable<S, M>& buf,
bool fw,
- bool color,
size_t depth = 0,
size_t len = 0) const
{
@@ -3118,8 +3105,7 @@ public:
assert_leq(len, this->len_ - depth);
for(size_t i = 0; i < len; i++) {
buf.append(fw ? this->cs_[depth+i] :
- (color ? this->cs_[depth+len-i-1] :
- compDna(this->cs_[depth+len-i-1])));
+ compDna(this->cs_[depth+len-i-1]));
}
}
@@ -3208,22 +3194,18 @@ public:
* Either reverse or reverse-complement (depending on "color") this
* DNA buffer in-place.
*/
- void reverseComp(bool color = false) {
- if(color) {
- this->reverse();
- } else {
- for(size_t i = 0; i < (this->len_ >> 1); i++) {
- char tmp1 = maskcomp[(int)this->cs_[i]];
- char tmp2 = maskcomp[(int)this->cs_[this->len_-i-1]];
- this->cs_[i] = tmp2;
- this->cs_[this->len_-i-1] = tmp1;
- }
- // Do middle element iff there are an odd number
- if((this->len_ & 1) != 0) {
- char tmp = this->cs_[this->len_ >> 1];
- tmp = maskcomp[(int)tmp];
- this->cs_[this->len_ >> 1] = tmp;
- }
+ void reverseComp() {
+ for(size_t i = 0; i < (this->len_ >> 1); i++) {
+ char tmp1 = maskcomp[(int)this->cs_[i]];
+ char tmp2 = maskcomp[(int)this->cs_[this->len_-i-1]];
+ this->cs_[i] = tmp2;
+ this->cs_[this->len_-i-1] = tmp1;
+ }
+ // Do middle element iff there are an odd number
+ if((this->len_ & 1) != 0) {
+ char tmp = this->cs_[this->len_ >> 1];
+ tmp = maskcomp[(int)tmp];
+ this->cs_[this->len_ >> 1] = tmp;
}
}
@@ -3353,7 +3335,6 @@ public:
char windowGetDna(
size_t i,
bool fw,
- bool color,
size_t depth = 0,
size_t len = 0) const
{
@@ -3361,8 +3342,7 @@ public:
assert_lt(i, len);
assert_leq(len, this->len_ - depth);
if(fw) return this->cs_[depth+i];
- else return color ? this->cs_[depth+len-i-1] :
- maskcomp[this->cs_[depth+len-i-1]];
+ else return maskcomp[this->cs_[depth+len-i-1]];
}
/**
@@ -3372,7 +3352,6 @@ public:
void windowGetDna(
SDnaStringFixed<S>& buf,
bool fw,
- bool color,
size_t depth = 0,
size_t len = 0) const
{
@@ -3380,8 +3359,7 @@ public:
assert_leq(len, this->len_ - depth);
for(size_t i = 0; i < len; i++) {
buf.append(fw ? this->cs_[depth+i] :
- (color ? this->cs_[depth+len-i-1] :
- maskcomp[this->cs_[depth+len-i-1]]));
+ maskcomp[this->cs_[depth+len-i-1]]);
}
}
diff --git a/threading.h b/threading.h
index ec9b504..98cdec3 100644
--- a/threading.h
+++ b/threading.h
@@ -21,12 +21,15 @@
#define THREADING_H_
#include <iostream>
+#include <cstdio>
+#include <cstdlib>
+#include <cstring>
#ifdef WITH_TBB
# include <tbb/mutex.h>
# include <tbb/spin_mutex.h>
+# include <tbb/queuing_mutex.h>
# ifdef WITH_AFFINITY
-# include <cstdlib>
# include <sched.h>
# include <tbb/task_group.h>
# include <tbb/task_scheduler_observer.h>
@@ -40,15 +43,19 @@
#ifdef NO_SPINLOCK
# ifdef WITH_TBB
-# define MUTEX_T tbb::mutex
+# ifdef WITH_QUEUELOCK
+# define MUTEX_T tbb::queuing_mutex
+# else
+# define MUTEX_T tbb::mutex
+# endif
# else
# define MUTEX_T tthread::mutex
# endif
#else
# ifdef WITH_TBB
-# define MUTEX_T tbb::spin_mutex
+# define MUTEX_T tbb::spin_mutex
# else
-# define MUTEX_T tthread::fast_mutex
+# define MUTEX_T tthread::fast_mutex
# endif
#endif /* NO_SPINLOCK */
@@ -58,22 +65,37 @@
*/
class ThreadSafe {
public:
- ThreadSafe(MUTEX_T* ptr_mutex, bool locked = true) {
+
+ ThreadSafe() : ptr_mutex(NULL) { }
+
+ ThreadSafe(MUTEX_T* ptr_mutex, bool locked = true) : ptr_mutex(NULL) {
if(locked) {
- this->ptr_mutex = ptr_mutex;
- ptr_mutex->lock();
+#if WITH_TBB && NO_SPINLOCK && WITH_QUEUELOCK
+ //have to use the heap as we can't copy
+ //the scoped lock
+ this->ptr_mutex = new MUTEX_T::scoped_lock(*ptr_mutex);
+#else
+ this->ptr_mutex = ptr_mutex;
+ ptr_mutex->lock();
+#endif
}
- else
- this->ptr_mutex = NULL;
}
~ThreadSafe() {
- if (ptr_mutex != NULL)
- ptr_mutex->unlock();
+ if (ptr_mutex != NULL)
+#if WITH_TBB && NO_SPINLOCK && WITH_QUEUELOCK
+ delete ptr_mutex;
+#else
+ ptr_mutex->unlock();
+#endif
}
-
+
private:
+#if WITH_TBB && NO_SPINLOCK && WITH_QUEUELOCK
+ MUTEX_T::scoped_lock* ptr_mutex;
+#else
MUTEX_T *ptr_mutex;
+#endif
};
#ifdef WITH_TBB
--
Alioth's /usr/local/bin/git-commit-notice on /srv/git.debian.org/git/debian-med/bowtie2.git
More information about the debian-med-commit
mailing list